diff --git a/core/libs/prx/libSceAgcDriver/CMakeLists.txt b/core/libs/prx/libSceAgcDriver/CMakeLists.txt index 8cb5e76be..8cd0d4edf 100644 --- a/core/libs/prx/libSceAgcDriver/CMakeLists.txt +++ b/core/libs/prx/libSceAgcDriver/CMakeLists.txt @@ -47,14 +47,15 @@ add_custom_command( ) add_library(libSceAgcDriver SHARED EXCLUDE_FROM_ALL + ${CMAKE_CURRENT_SOURCE_DIR}/Unimplemented.cpp ${CMAKE_CURRENT_SOURCE_DIR}/Execution/src/DisplayBuffer.cpp ${CMAKE_CURRENT_SOURCE_DIR}/Execution/src/Driver.cpp + ${CMAKE_CURRENT_SOURCE_DIR}/Execution/src/WorkerSampler.cpp ${CMAKE_CURRENT_SOURCE_DIR}/Execution/src/GuestMemory.cpp ${CMAKE_CURRENT_SOURCE_DIR}/Execution/src/ShaderMemory.cpp ${CMAKE_CURRENT_SOURCE_DIR}/Execution/src/BdaFeatures.cpp ${CMAKE_CURRENT_SOURCE_DIR}/Execution/src/Pm4.cpp ${CMAKE_CURRENT_SOURCE_DIR}/Execution/src/VulkanDevice.cpp - ${CMAKE_CURRENT_SOURCE_DIR}/Execution/src/SwapchainState.cpp ${CMAKE_CURRENT_SOURCE_DIR}/Execution/src/AspectFit.cpp ${CMAKE_CURRENT_SOURCE_DIR}/Execution/src/PresentationScaler.cpp ${CMAKE_CURRENT_SOURCE_DIR}/Graphics/src/Draw.cpp @@ -62,6 +63,7 @@ add_library(libSceAgcDriver SHARED EXCLUDE_FROM_ALL ${CMAKE_CURRENT_SOURCE_DIR}/Graphics/src/ColorTargetTransfer.cpp ${CMAKE_CURRENT_SOURCE_DIR}/Graphics/src/Pipeline.cpp ${CMAKE_CURRENT_SOURCE_DIR}/Graphics/src/Resources.cpp + ${CMAKE_CURRENT_SOURCE_DIR}/Graphics/src/Recorder.cpp ${CMAKE_CURRENT_SOURCE_DIR}/Graphics/src/ShaderInputState.cpp ${CMAKE_CURRENT_SOURCE_DIR}/Graphics/src/ShaderResources.cpp ${CMAKE_CURRENT_SOURCE_DIR}/Graphics/src/ShaderAddressResources.cpp @@ -73,6 +75,7 @@ add_library(libSceAgcDriver SHARED EXCLUDE_FROM_ALL ${CMAKE_CURRENT_SOURCE_DIR}/Graphics/src/GuestTextureResource.cpp ${CMAKE_CURRENT_SOURCE_DIR}/Graphics/src/GuestSamplerResource.cpp ${CMAKE_CURRENT_SOURCE_DIR}/Graphics/src/TextureFormat.cpp + ${CMAKE_CURRENT_SOURCE_DIR}/Graphics/src/DccMetadata.cpp ${CMAKE_CURRENT_SOURCE_DIR}/Graphics/src/TextureTiling.cpp ${CMAKE_CURRENT_SOURCE_DIR}/Graphics/src/TextureDetiler.cpp ${CMAKE_CURRENT_SOURCE_DIR}/Graphics/src/Texture.cpp @@ -208,6 +211,7 @@ add_executable(agc_driver_graphics_tests EXCLUDE_FROM_ALL Graphics/src/GuestTextureResource.cpp Graphics/src/GuestSamplerResource.cpp Graphics/src/TextureFormat.cpp + Graphics/src/DccMetadata.cpp Graphics/src/TextureTiling.cpp Graphics/src/TextureDetiler.cpp Graphics/src/Texture.cpp @@ -282,7 +286,6 @@ if(AGC_BUILD_VISUAL_TEST) add_executable(agc_driver_visual_test EXCLUDE_FROM_ALL tests/Visual.cpp Execution/src/VulkanDevice.cpp - Execution/src/SwapchainState.cpp Execution/src/AspectFit.cpp Execution/src/PresentationScaler.cpp Execution/src/GuestMemory.cpp @@ -301,6 +304,7 @@ if(AGC_BUILD_VISUAL_TEST) Graphics/src/GuestTextureResource.cpp Graphics/src/GuestSamplerResource.cpp Graphics/src/TextureFormat.cpp + Graphics/src/DccMetadata.cpp Graphics/src/TextureTiling.cpp Graphics/src/TextureDetiler.cpp Graphics/src/Texture.cpp @@ -363,3 +367,14 @@ if(AGC_BUILD_VISUAL_TEST) endif() include(PostCompile.cmake) +# Replays shader_*.req files the driver writes when a compute dispatch cannot be recompiled. +add_executable(agc_shader_replay EXCLUDE_FROM_ALL + tools/ShaderReplay.cpp +) +target_include_directories(agc_shader_replay PRIVATE ${LIBS_INCLUDE_DIR}) +target_link_libraries(agc_shader_replay PRIVATE shader_recompiler_agc_driver libc) +if(ANYPS5_ENABLE_SPIRV_TOOLS) + target_link_libraries(agc_shader_replay PRIVATE SPIRV-Tools-static) +endif() +set_target_properties(agc_shader_replay PROPERTIES CXX_EXTENSIONS OFF) +configure_windows_unwind(agc_shader_replay) diff --git a/core/libs/prx/libSceAgcDriver/Execution/include/GuestMemory.hpp b/core/libs/prx/libSceAgcDriver/Execution/include/GuestMemory.hpp index ab0b5fe4f..7528b331d 100644 --- a/core/libs/prx/libSceAgcDriver/Execution/include/GuestMemory.hpp +++ b/core/libs/prx/libSceAgcDriver/Execution/include/GuestMemory.hpp @@ -1,16 +1,169 @@ #ifndef CORE_LIBS_PRX_LIBSCEAGCDRIVER_EXECUTION_INCLUDE_GUESTMEMORY_HPP #define CORE_LIBS_PRX_LIBSCEAGCDRIVER_EXECUTION_INCLUDE_GUESTMEMORY_HPP +#include #include #include +#include #include +#include +#include namespace AgcDriver::GuestMemory { void CheckRange(const void* pointer, std::size_t bytes, std::size_t alignment, bool writable = false); -void CheckGpuRange(const void* pointer, std::size_t bytes, std::size_t alignment, bool writable = false); +// CheckRange without throwing: whether the whole range is mapped with the requested access. +bool Accessible(const void* pointer, std::size_t bytes, bool writable = false); +// The accessible parts [begin, end) of a range, in address order. GPU heaps are often bound whole while +// the guest commits their pages on demand, so a binding can cover reserved but uncommitted pages. +std::vector> CommittedRanges(std::uint64_t address, std::size_t bytes, bool writable = false); +// Both facts from one walk of the page tables: the accessible parts of a range and whether they cover +// all of it (`whole`, what Accessible answers). Reserved pages are queried each time, so asking twice +// costs two VirtualQuery per run. +struct Commitment { + std::vector> ranges; + bool whole = false; +}; +Commitment DescribeCommitted(std::uint64_t address, std::size_t bytes, bool writable = false); void Read(std::uint64_t address, std::span destination, std::size_t alignment = 1); +// Variants for resources in heaps the guest commits on demand (partially resident textures, GPU +// scratch heaps): uncommitted pages read as zeros, are not compared, and are never stored. +void ReadCommitted(std::uint64_t address, std::span destination); +bool EqualsCommitted(std::uint64_t address, std::span bytes); +void WriteChangedCommitted(std::uint64_t address, std::span current, std::span original); void Write(std::uint64_t address, std::span source, std::size_t alignment = 1); +// Stores the parts of `current` that differ from `original` (the guest bytes the GPU started from), so +// guest writes made meanwhile to untouched bytes survive. Compares in 256-byte blocks. +void WriteChanged(std::uint64_t address, std::span current, std::span original); + +// Write tracking. The guest arena is reserved with page write watching, so CollectWrites learns which +// pages of a range the CPU wrote since they were last collected, stamps their 64 KiB blocks with a new +// generation and returns it; a snapshot of the range taken after that call is current while +// UnchangedSince(range, that generation) holds. Every validation collects its own range first, so +// resources sharing pages see each other's writes. GPU writes into host-imported memory bypass the +// page tables and are reported with MarkWritten. Without write watching CollectWrites returns 0 and +// UnchangedSince is always false, so callers fall back to comparing bytes. +std::uint64_t CollectWrites(std::uint64_t address, std::size_t bytes); +bool UnchangedSince(std::uint64_t address, std::size_t bytes, std::uint64_t generation); +void MarkWritten(std::uint64_t address, std::size_t bytes); +// Collect epoch: within one epoch a range already collected is not walked again, CollectWrites +// returns the current generation instead. A guest write landing between two collects of the same +// epoch is seen by the next epoch, which is the ordering real hardware gives a CPU write made while a +// submission runs. The driver's own writes (Write, WriteChanged, MarkWritten) stamp their blocks, so +// they are never hidden. APS5_NO_COLLECT_MEMO=1 disables the memo. CollectWritesUncached always walks: +// for decisions about which blocks the CPU wrote while a GPU result was pending (a stale answer there +// stores old texels over reused memory). +// The epoch is per queue worker: BumpCollectEpoch advances the calling thread's epoch to a value no +// other thread ever holds (one worker's walk never satisfies another's check) at the hardware +// ordering points of its queue: the start of a submission, a WAIT_REG_MEM the CPU may have satisfied +// (at entry, polled, timed out, or by another queue's recorded label; a label this queue recorded +// itself orders no CPU write), a device drain, and before reaping completions outside a submission. +// A thread that never bumps (the presenter, game threads inside the flush hook) gets a fresh epoch +// for every collect: nothing is reused without an ordering point. APS5_PACKET_EPOCH=1 makes the +// workers bump before every packet as before. CollectEpochBumps counts the bumps ([guestmem] line). +void BumpCollectEpoch(); +std::uint64_t CollectEpochBumps(); +std::uint64_t CollectWritesUncached(std::uint64_t address, std::size_t bytes); + +// Serializes device work: draws, dispatches, presentation and the deferred write-backs below. The +// mutex is recursive; it is wrapped so every acquisition (std::lock_guard at any site) measures how +// long it waited: the graphics worker's frame is mostly spent waiting for this lock behind the +// compute workers, and that wait has to stay visible. Under APS5_PROFILE_DRAW each thread reports its +// waits in a [lock] line every 10 s (APS5_NO_LOCK_PROFILE=1 turns the timing off); a queue worker tags +// its thread with TagGpuLockThread so the line names the queue. +class GpuMutexType { +public: + void lock(); + bool try_lock(); + void unlock(); + // Whether the calling thread holds the mutex (at any depth): the APS5_ASSERT_GPU_LOCK checks + // (AssertGpuLockHeld) and code that must know whether it runs inside the device lock. + bool HeldByThisThread() const; + // The calling thread's recursion depth (0 when it does not hold the mutex): 1 means its + // current acquisition is the outermost, so code that releases the mutex around a wait + // (Recorder::SyncThrough for the flush hook) gives up no caller's hold. + std::uint32_t DepthOnThisThread() const; + +private: + void acquired(); + std::recursive_mutex mutex; + // The holder's thread token and recursion depth, written by the holder only; another thread + // reads the owner just to see that it is not itself. + std::atomic owner{nullptr}; + std::uint32_t depth = 0; +}; +GpuMutexType& GpuMutex(); +// Called on the thread that just gave up its outermost hold, right after the mutex was released: +// work the hold deferred to run without it (the recorder's release of the objects completed batches +// kept). One hook, set once; it must not take GpuMutex itself. +void SetGpuUnlockHook(void (*hook)()); +// Debug aid (APS5_ASSERT_GPU_LOCK=1): aborts with a [lock] line naming `where` when the calling +// thread does not hold GpuMutex. Recording, submits, command batches and the detiler's pools must +// only be reached under it (a resource build's stage A runs without it, see ShaderResources). +void AssertGpuLockHeld(const char* where); +void TagGpuLockThread(std::uint32_t queue); +// The queue tag of the calling thread (0xffffffff when untagged), for reports that name the thread. +std::uint32_t GpuLockThreadTag(); +// Lock sites (APS5_PROFILE_DRAW): a caller names the site of its next lock() so the [lock] line +// splits acquisitions and waits per site (whose holds queue 0 waits behind is then measurable). The +// tag is consumed by that acquisition, so a nested lock inside the flush hook is attributed to the +// hook, and an acquisition nobody tagged counts as "other". Try-only acquisitions are not counted +// as acquisitions; the hold they start is timed under "try". Holds are timed per site as well (the +// outermost acquisition of a thread until its unlock; APS5_NO_HOLD_PROFILE=1 turns that off), so +// the line shows who holds the mutex, not only who waits for it. +enum class GpuLockSite : std::uint8_t { Other = 0, Dispatch, Indirect, Draw, Hook, Wait, Label, Flush, Present, Fill, Try, Count }; +void TagGpuLockSite(GpuLockSite site); +// A code address (a return address) as an offset into its module, in the form of the [guestmem] +// callers (symbolized with nm against the driver's image). Takes the loader lock: for reports only. +unsigned long long CodeOffset(const void* address); + +// Deferred GPU writes. The graphics layer keeps the results of storage images on the GPU until guest +// memory is read; it registers a hook that stores every pending result overlapping a range, which +// every read and store through this interface calls first. Code that reads guest memory directly +// calls FlushGpuWrites itself. +void SetFlushHook(void (*hook)(std::uint64_t address, std::size_t bytes)); +void FlushGpuWrites(std::uint64_t address, std::size_t bytes); + +// Attribution of guest memory accesses (the [hooksync] line in Recorder.cpp and the read-site +// counts of the [guestmem] line, APS5_PROFILE_DRAW): a sync the flush hook makes for an access has +// to be charged to the PM4 packet and the driver code that made the access. A queue worker names +// the packet it executes at the top of its packet loop (SetCurrentPacket; opcode 0xffff is the +// flip); a thread that runs no packets (the presenter, game threads) reads back NoPacket. The main +// read paths name themselves with a ReadSiteScope; an access made by none of them is attributed by +// its return addresses instead (CaptureCallerOffsets). +struct PacketTag { + std::uint32_t opcode; + std::uint32_t queue; +}; +constexpr std::uint32_t NoPacket = 0xfffffffeu; +void SetCurrentPacket(std::uint32_t opcode, std::uint32_t queue); +PacketTag CurrentPacket(); + +// Store: a CPU write (Write/WriteChanged) rather than a read; a hook sync made for one orders a +// write after a pending GPU write, so whether the bytes change says nothing about its necessity. +// MirrorRefresh: an address-based build bringing an image mirror up to date with guest memory +// (GuestBufferMemory.cpp: the flush before a writable mirror's compare, a new mirror's fill). +// Every enumerator needs its name in ReadSiteName's table (GuestMemory.cpp). +enum class ReadSite : std::uint8_t { Unknown = 0, Capture, DispatchCache, TextureCompare, TextureRead, BufferUpload, IndexBuffer, VertexBuffer, Registers, IndirectArguments, Wait, Label, Scanout, Store, MirrorRefresh, Count }; +const char* ReadSiteName(ReadSite site); +// Sets the calling thread's read site and returns the previous one (nested scopes restore it). +ReadSite SetReadSite(ReadSite site); +ReadSite CurrentReadSite(); +class ReadSiteScope { +public: + explicit ReadSiteScope(ReadSite site) : previous(SetReadSite(site)) {} + ~ReadSiteScope() { SetReadSite(previous); } + ReadSiteScope(const ReadSiteScope&) = delete; + ReadSiteScope& operator=(const ReadSiteScope&) = delete; + +private: + ReadSite previous; +}; +// Return addresses above the caller as offsets into the driver's module (the same form as the +// [guestmem] callers, symbolized with nm against the driver's image; an address outside the module +// is written as 0), newest first, skipping `skip` frames above the caller; returns how many were +// written. Stack unwinding, so only for rare, profiled events. +std::size_t CaptureCallerOffsets(std::span frames, unsigned skip); } diff --git a/core/libs/prx/libSceAgcDriver/Execution/include/Pm4.hpp b/core/libs/prx/libSceAgcDriver/Execution/include/Pm4.hpp index 88c105566..475995ccc 100644 --- a/core/libs/prx/libSceAgcDriver/Execution/include/Pm4.hpp +++ b/core/libs/prx/libSceAgcDriver/Execution/include/Pm4.hpp @@ -3,6 +3,9 @@ #include "prx/libSceAgcDriver/Execution/include/QueueState.hpp" #include "prx/libSceAgcDriver/Execution/include/Pm4Opcodes.hpp" +#include +#include +#include #include #include #include @@ -21,11 +24,59 @@ struct DrawParameters { }; std::string Name(std::uint32_t header); +// A PM4 type-2 packet is a one-dword filler (command-buffer padding); type 3 and type 0 carry a +// dword count in bits 29:16. Type 1 is undefined. +inline bool FillerPacket(std::uint32_t header) { return (header >> 30u) == 2u; } +inline std::size_t PacketWords(std::uint32_t header) { return FillerPacket(header) ? 1u : static_cast((header >> 16u) & 0x3fffu) + 2u; } std::string_view UnsupportedReason(std::uint32_t header); void Validate(std::span packet, std::uint32_t queue); void Execute(std::span packet, QueueState& queue); bool AccessesMemory(std::uint32_t header); +// Whether an ACQUIRE_MEM packet asks only for GPU cache actions (no CPU-visible memory +// synchronization): such a packet needs a pipeline barrier, not a device drain. bool UsesGpuCacheBarrier(std::span packet); +bool IsTagMarker(std::span packet); +bool WaitSatisfied(std::span packet); +// WaitSatisfied for a polling loop: the caller has validated the address once, so the value is read +// directly instead of through the checked guest memory path. +bool WaitSatisfiedUnchecked(std::span packet); +// Whether a WAIT_REG_MEM's compare holds for `value` (the 4 or 8 bytes a label the recorder still +// holds will store), without reading memory. +bool WaitComparesValue(std::span packet, std::uint64_t value); +// A label write (RELEASE_MEM with a data select, WRITE_DATA to memory): the destination and the +// bytes it stores, so the write can be recorded on the GPU behind the work it signals. Packets +// without a memory destination decode to nothing. No allocation per label (tens of thousands per +// second): WRITE_DATA's bytes are a view of the packet (valid while the packet is), a RELEASE_MEM +// value is held inline. +struct LabelWrite { + std::uint64_t address; + std::span packetBytes; + std::array inlineBytes{}; + std::size_t inlineSize = 0; + std::span Bytes() const { return inlineSize != 0 ? std::span(inlineBytes).first(inlineSize) : packetBytes; } +}; +std::optional DecodeLabelWrite(std::span packet); +// A memory store the CPU can resolve before the GPU runs it (COPY_DATA and DMA_DATA to memory, +// DUMP_CONST_RAM): the destination and the bytes it stores, so the driver can record the store on +// the GPU like a label instead of draining the device and storing on the CPU. Immediate and constant +// RAM bytes are known at once; a memory source is read here through the checked guest memory path +// (which waits only for recorded GPU work that writes the source). Nothing when the packet is not +// such a store, when it stores more than `limit` bytes, or when its destination or size is not a +// multiple of 4 (the GPU store needs both; the source is then not read: the caller keeps its CPU +// path). A store of zero bytes resolves with empty bytes. +struct StoreWrite { + std::uint64_t address; + std::span viewBytes; + std::vector ownedBytes; + std::span Bytes() const { return ownedBytes.empty() ? viewBytes : std::span(ownedBytes); } +}; +std::optional ResolveStore(std::span packet, const QueueState& queue, std::size_t limit); +// A DISPATCH_INDIRECT's arguments: the guest address of its three group-count dwords (no memory +// access, so the GPU can read them in place: VulkanDevice::DispatchIndirect), the DISPATCH_DIRECT +// packet made by reading them there (through the checked guest memory path, which waits for +// recorded GPU work that writes them), and both steps in one for the packet. +std::uint64_t DispatchArgumentAddress(std::span packet, const QueueState& queue); +std::array ReadDispatchArguments(std::uint64_t arguments, std::uint32_t initiator); std::array ResolveDispatch(std::span packet, const QueueState& queue); DrawParameters ResolveDraw(std::span packet, const QueueState& queue); diff --git a/core/libs/prx/libSceAgcDriver/Execution/include/PresentationScaler.hpp b/core/libs/prx/libSceAgcDriver/Execution/include/PresentationScaler.hpp index 307ceeb48..c3fb1e15c 100644 --- a/core/libs/prx/libSceAgcDriver/Execution/include/PresentationScaler.hpp +++ b/core/libs/prx/libSceAgcDriver/Execution/include/PresentationScaler.hpp @@ -18,7 +18,18 @@ public: void EnsureSourceImage(std::uint32_t width, std::uint32_t height); void RecordUpload(VkCommandBuffer commands, VkBuffer uploadBuffer); void RecordImage(VkCommandBuffer commands, VkImage image); + // Fills the source image from another image of the same extent (any blittable format, layout + // unchanged), leaving the source image ready to blit or read back. + void RecordBlitInto(VkCommandBuffer commands, VkImage image, VkImageLayout layout, VkFilter filter); + // Copies the source image's texels (tightly packed rows of its format) into a host-readable buffer + // and makes them visible to the host once the submission completes. + void RecordReadback(VkCommandBuffer commands, VkBuffer destination); void RecordBlit(VkCommandBuffer commands, VkImage destinationImage, std::uint32_t destinationWidth, std::uint32_t destinationHeight); + // The aspect-fitting blit of RecordBlit from any image: presenting a resident render target needs + // no copy into the source image first. + static void RecordBlitFrom(const Graphics::Context& context, VkCommandBuffer commands, VkImage image, VkImageLayout layout, std::uint32_t width, std::uint32_t height, VkFilter filter, VkImage destinationImage, std::uint32_t destinationWidth, std::uint32_t destinationHeight); + std::uint32_t SourceWidth() const { return sourceWidth; } + std::uint32_t SourceHeight() const { return sourceHeight; } private: void release() noexcept; diff --git a/core/libs/prx/libSceAgcDriver/Execution/include/ShaderMemory.hpp b/core/libs/prx/libSceAgcDriver/Execution/include/ShaderMemory.hpp index 25693df61..ce2ab7bfa 100644 --- a/core/libs/prx/libSceAgcDriver/Execution/include/ShaderMemory.hpp +++ b/core/libs/prx/libSceAgcDriver/Execution/include/ShaderMemory.hpp @@ -2,19 +2,45 @@ #define CORE_LIBS_PRX_LIBSCEAGCDRIVER_EXECUTION_INCLUDE_SHADERMEMORY_HPP #include "Recompiler.hpp" +#include +#include +#include #include +#include +#include +#include namespace AgcDriver { +// Records the guest words a shader's resource analysis reads, as the memory regions its recompile +// request carries. Guest memory is fetched a 4 KiB page at a time; the regions report exactly the +// dwords read, so cache keys built from them do not change with unrelated bytes nearby. class ShaderMemory { public: explicit ShaderMemory(std::span initial); - void Capture(const ShaderRecompiler::RecompileRequest& request); + // Returns what the capture resolved (plan, snapshot, specialization) for + // ShaderRecompiler::Recompile(request, capture), which then skips its own materialization. One + // result per call: the draw path captures several stages on one ShaderMemory. + std::shared_ptr Capture(const ShaderRecompiler::RecompileRequest& request); [[nodiscard]] std::vector Regions() const; private: + static constexpr std::size_t PageBytes = 4096; + static constexpr std::size_t PageWords = PageBytes / sizeof(std::uint32_t); + + struct Page { + std::array words{}; + std::bitset valid; + std::bitset read; + }; + static bool read(void* context, std::uint64_t address, std::uint32_t* value); - std::map> regions; + Page& page(std::uint64_t base); + + // Regions given at construction (the registered shader's code and header), referenced as given: + // the caller keeps them alive for as long as the capture is used. + std::map> initial; + std::map pages; }; } diff --git a/core/libs/prx/libSceAgcDriver/Execution/include/VulkanDevice.hpp b/core/libs/prx/libSceAgcDriver/Execution/include/VulkanDevice.hpp index 50f172636..2c1fef9ac 100644 --- a/core/libs/prx/libSceAgcDriver/Execution/include/VulkanDevice.hpp +++ b/core/libs/prx/libSceAgcDriver/Execution/include/VulkanDevice.hpp @@ -8,9 +8,17 @@ #include "prx/libSceAgcDriver/Execution/include/DisplayBuffer.hpp" #include "prx/libSceAgcDriver/Graphics/include/Draw.hpp" #include +#include namespace AgcDriver { +namespace Graphics { +class StorageTexture; +} + +// Stage A of a dispatch's resource build (see VulkanDevice::PrepareDispatch); opaque to callers. +struct PreparedDispatch; + class VulkanDevice { public: explicit VulkanDevice(const PresentationWindow* window = nullptr); @@ -18,23 +26,99 @@ public: VulkanDevice(const VulkanDevice&) = delete; VulkanDevice& operator=(const VulkanDevice&) = delete; ShaderRecompiler::SpirvTarget Target() const; + // Drains everything under the caller's GpuMutex: recorder Sync plus vkDeviceWaitIdle. For suspend, + // resize, device replacement, CPU fill fallback and APS5_DRAIN_ALL; the packet-loop drains use the + // three-step form below so the GPU wait happens without the mutex. void WaitIdle(); - void WaitDraws(); - void AcquireGpuMemory(); + // Sends recorded work to the GPU without waiting for it. With `reapFirst` it first retires batches + // that already finished, so the in-flight list stays short (APS5_NO_OPPORTUNISTIC_REAP=1 skips + // that). A reap runs completion actions, and a write-back can wait for a later batch under the + // mutex through the flush hook, so a thread submitting on another queue's behalf (a WAIT_REG_MEM + // poller, a compute worker's between-packet flush) passes false: it must only submit, never wait. + void SubmitRecorded(bool reapFirst = true); + // Unlocked drain: SubmitAndEpoch (under the mutex) returns the serial covering every recorded + // batch (0: nothing in flight); WaitRecorded(serial) waits for it WITHOUT the mutex (the caller + // holds a shared_ptr to this device); ReapRecorded(serial) (under the mutex) runs the completions + // of the batches up to it. CanWaitUnlocked is false without timeline semaphores: use WaitIdle. + bool CanWaitUnlocked() const; + std::uint64_t SubmitAndEpoch(); + void WaitRecorded(std::uint64_t serial); + void ReapRecorded(std::uint64_t serial); + // Completes the batches that already finished (non-blocking); under the mutex. + void ReapRecorded(); + // Records a store of `bytes` at a guest address behind the recorded work, so the value appears + // once that work completed; the batch is submitted by the queue worker's flush rules (or at once + // with APS5_LABEL_SUBMIT_NOW=1). `stamp` is the record-order stamp and `queue` the recording + // queue for the recorder's pending-label table. Returns 0 when recorded on the GPU, 5 when kept + // as a completion action behind pending write-backs and 6 when kept as one because the memory is + // not host-imported so the GPU cannot store it (both land when their batch is reaped), else why + // the CPU must write it: + // 1 nothing recorded (the write is already ordered), 2 write-backs pending and 3 memory not + // imported (both only with APS5_DRAIN_COMPLETION_LABELS=1), 4 unsuitable size or alignment. + // `reapFirst` retires finished batches before the checks; a caller recording a group of labels + // under one lock passes it for the first label only. + int WriteLabelOnGpu(std::uint64_t address, std::span bytes, std::uint64_t stamp, std::uint32_t queue, bool reapFirst = true); + // Pending-label table lookup and open-batch overlap test for WAIT_REG_MEM (see Recorder). + std::optional PendingLabel(std::uint64_t address, std::size_t bytes, std::uint64_t afterStamp, std::uint32_t& queue) const; + bool OpenWriteOverlaps(std::uint64_t address, std::size_t bytes) const; + // Fills [address, address + bytes) of host-imported guest memory with a repeating 16-byte pattern, + // recorded behind the open batch; false when the range is not imported (the caller stores it). + // Recorded GPU stores over the range are ordered before the fill by its barrier; finished + // batches are retired first, and it waits only for stores a batch's completion makes on the + // CPU (a copied buffer's write-back, a deferred label). Debug aid: APS5_FILL_SYNC=1 waits for + // every recorded store over the range. + bool FillBuffer(std::uint64_t address, std::size_t bytes, std::span pattern); void ResolveMemory(std::uint64_t address, std::size_t bytes, bool writable); void* Window() const; void Resize(std::uint32_t width, std::uint32_t height); bool Presentable() const; - void PresentClear(std::uint32_t width, std::uint32_t height, bool opaque); + // A presentation is four steps so the presenter holds GuestMemory::GpuMutex only while it touches + // the queue: AcquireImage takes the next swapchain image (no mutex needed: the swapchain and its + // fences are the presenter's own, and in FIFO mode this is where a frame waits for the vblank); + // PresentClear/PresentDisplayBuffer record and submit the frame (under the mutex) and return + // whether one is in flight; FinishPresent waits for its render fence (no mutex); QueuePresent + // hands the image to the swapchain (under the mutex again). AcquireImage returns false when the + // swapchain is out of date (the frame is dropped; the next Resize recreates it); a present without + // a prior AcquireImage acquires itself. PresentPixels does all steps itself. + bool AcquireImage(); + bool PresentClear(std::uint32_t width, std::uint32_t height, bool opaque); void PresentPixels(std::uint32_t width, std::uint32_t height, std::span pixels); - void PresentDisplayBuffer(const DisplayBuffer& buffer); - void Dispatch(const ShaderRecompiler::RecompileResult& shader, std::uint32_t x, std::uint32_t y, std::uint32_t z, std::span snapshots = {}); + bool PresentDisplayBuffer(const DisplayBuffer& buffer); + void FinishPresent(); + void QueuePresent(); + // Stage A of a dispatch's resource build, run WITHOUT GuestMemory::GpuMutex before Dispatch or + // DispatchIndirect: the binding plan, data buffers, imports already serving the guest buffers, + // read-only copies and the descriptor set (see ShaderResources). The dispatch completes it + // under the mutex (stage B: texture lookups, the rest of the upload, descriptor writes) and + // records. Null when nothing is prepared: the resource cache may serve the dispatch (its + // Revalidate stays under the mutex), or APS5_LOCKED_BUILD=1 keeps the whole build under it as + // before. `shader` and `snapshots` must outlive the dispatch. + std::shared_ptr PrepareDispatch(const ShaderRecompiler::RecompileResult& shader, std::span snapshots); + void Dispatch(const ShaderRecompiler::RecompileResult& shader, std::uint32_t x, std::uint32_t y, std::uint32_t z, std::span snapshots = {}, std::uint64_t programAddress = 0, std::shared_ptr prepared = nullptr); + // A dispatch whose group counts are the three dwords at `arguments` in guest memory + // (DISPATCH_INDIRECT): the GPU reads them in place from the host import, ordered after everything + // recorded before, so the CPU never waits for the shader that wrote them. When the GPU could not + // see the current bytes the counts are read on the CPU instead (through the flush hook, as the + // driver resolved every indirect dispatch before) and the dispatch is recorded as a direct one: + // cpuReason 1 storage-image results were pending over them, 2 a recorded dispatch writes them + // through a copied buffer (its CPU write-back lands only when the batch is reaped) or a label + // over those dwords is pending, 3 the memory is not host-imported; 0 when recorded GPU-side. argumentReadMs is + // what that CPU read (its sync) took. The device limit on group counts is not checked GPU-side. + // Debug aid: APS5_NO_GPU_INDIRECT=1 (in the driver) keeps every indirect dispatch on the CPU path. + struct IndirectOutcome { + int cpuReason; + double argumentReadMs; + }; + IndirectOutcome DispatchIndirect(const ShaderRecompiler::RecompileResult& shader, std::uint64_t arguments, std::span snapshots = {}, std::uint64_t programAddress = 0, std::shared_ptr prepared = nullptr); void Draw(const Graphics::State& graphics, const Pm4::DrawParameters& draw, std::span shaders, std::span snapshots = {}); void EnqueueDraw(const Graphics::State& graphics, const Pm4::DrawParameters& draw, std::span shaders, std::span snapshots = {}); private: Graphics::Context graphicsContext() const; - void present(std::uint32_t width, std::uint32_t height, bool opaque, std::span pixels, const DisplayBuffer* display = nullptr); + // Body of Dispatch and DispatchIndirect: `arguments` 0 dispatches x, y, z groups. + IndirectOutcome dispatch(const ShaderRecompiler::RecompileResult& shader, std::uint32_t x, std::uint32_t y, std::uint32_t z, std::uint64_t arguments, std::span snapshots, std::uint64_t programAddress, std::shared_ptr prepared); + bool present(std::uint32_t width, std::uint32_t height, bool opaque, std::span pixels, const DisplayBuffer* display = nullptr, const std::shared_ptr& resident = nullptr, VkFilter residentFilter = VK_FILTER_LINEAR, bool dumpFrame = false); + void writeFrameDump(); struct State; std::unique_ptr state; }; diff --git a/core/libs/prx/libSceAgcDriver/Execution/include/WorkerSampler.hpp b/core/libs/prx/libSceAgcDriver/Execution/include/WorkerSampler.hpp new file mode 100644 index 000000000..65b272671 --- /dev/null +++ b/core/libs/prx/libSceAgcDriver/Execution/include/WorkerSampler.hpp @@ -0,0 +1,11 @@ +#ifndef CORE_LIBS_PRX_LIBSCEAGCDRIVER_EXECUTION_INCLUDE_WORKERSAMPLER_HPP +#define CORE_LIBS_PRX_LIBSCEAGCDRIVER_EXECUTION_INCLUDE_WORKERSAMPLER_HPP + +namespace AgcDriver { + +// Starts sampling the calling thread when APS5_SAMPLE_WORKER names an output file. +void StartWorkerSampler(); + +} + +#endif diff --git a/core/libs/prx/libSceAgcDriver/Execution/src/DisplayBuffer.cpp b/core/libs/prx/libSceAgcDriver/Execution/src/DisplayBuffer.cpp index 7de87a438..e9294fea7 100644 --- a/core/libs/prx/libSceAgcDriver/Execution/src/DisplayBuffer.cpp +++ b/core/libs/prx/libSceAgcDriver/Execution/src/DisplayBuffer.cpp @@ -1,6 +1,9 @@ #include "prx/libSceAgcDriver/Execution/include/DisplayBuffer.hpp" #include "prx/libSceAgcDriver/Execution/include/GuestMemory.hpp" #include "prx/libSceAgcDriver/Execution/include/PerformanceTimer.hpp" +#include +#include +#include #include #include #include @@ -12,6 +15,15 @@ void require(bool condition, const char* reason) { if (!condition) throw std::runtime_error(reason); } +// Bit 56 selects the 10-bit (A2B10G10R10) variant of the format. +constexpr std::uint64_t PixelFormatUnormBit = 0x0100000000000000ull; +constexpr std::uint64_t PixelFormatB8G8R8A8 = 0x8000000000000000ull; +constexpr std::uint64_t PixelFormatR8G8B8A8 = 0x8000000022000000ull; + +std::uint64_t baseFormat(std::uint64_t pixelFormat) { + return pixelFormat & ~PixelFormatUnormBit; +} + std::uint32_t tileOffset(std::uint32_t x, std::uint32_t y) { return ((y << 4u) & 0x0070u) ^ ((y << 5u) & 0x0f00u) ^ ((y << 9u) & 0x1000u) ^ ((y << 8u) & 0x4000u) ^ ((x << 2u) & 0x000cu) ^ ((x << 5u) & 0x0380u) ^ ((x << 4u) & 0x0400u) ^ ((x << 6u) & 0x0800u) ^ ((x << 9u) & 0xa000u); } @@ -20,7 +32,11 @@ std::uint32_t tileOffset(std::uint32_t x, std::uint32_t y) { std::size_t DisplayBufferSize(const DisplayBuffer& buffer) { require(buffer.width != 0 && buffer.height != 0 && buffer.width <= 16384 && buffer.height <= 16384, "VideoOut: invalid display buffer dimensions"); - require(buffer.pixelFormat == 0x8000000000000000ull || buffer.pixelFormat == 0x8000000022000000ull, "VideoOut: unsupported display pixel format"); + if (baseFormat(buffer.pixelFormat) != PixelFormatB8G8R8A8 && baseFormat(buffer.pixelFormat) != PixelFormatR8G8B8A8) { + char message[96]; + std::snprintf(message, sizeof(message), "VideoOut: unsupported display pixel format 0x%016llx", static_cast(buffer.pixelFormat)); + throw std::runtime_error(message); + } require(buffer.address != 0 && (buffer.address & 65535u) == 0, "VideoOut: display buffer requires 64 KiB alignment"); const auto size = static_cast((buffer.width + 127u) / 128u) * ((buffer.height + 127u) / 128u) * 65536u; require(size <= std::numeric_limits::max() && size <= std::numeric_limits::max() - buffer.address, "VideoOut: display buffer range overflow"); @@ -32,12 +48,27 @@ std::vector DecodeDisplayBuffer(const DisplayBuffer& buffer, std::spa require(source.size() == DisplayBufferSize(buffer), "VideoOut: invalid tiled display buffer size"); std::vector pixels(static_cast(buffer.width) * buffer.height * 4); const auto blocksPerRow = (buffer.width + 127u) / 128u; - const bool rgba = buffer.pixelFormat == 0x8000000022000000ull; + const bool rgba = baseFormat(buffer.pixelFormat) == PixelFormatR8G8B8A8; + // Bit 56 marks the 10-bit variant: A2B10G10R10 with red in the low bits (the scanout draws render + // COLOR_2_10_10_10 into it). Presented as 8-bit BGRA by dropping the low two bits of each channel. + const bool tenBit = (buffer.pixelFormat & PixelFormatUnormBit) != 0; timing.Mark("validate_allocate"); for (std::uint32_t y = 0; y < buffer.height; ++y) { for (std::uint32_t x = 0; x < buffer.width; ++x) { const auto tiled = (static_cast(y / 128u) * blocksPerRow + x / 128u) * 65536u + tileOffset(x, y); const auto linear = (static_cast(y) * buffer.width + x) * 4; + if (tenBit) { + std::uint32_t word = 0; + std::memcpy(&word, source.data() + tiled, 4); + const auto red = static_cast((word >> 2u) & 0xffu); + const auto green = static_cast((word >> 12u) & 0xffu); + const auto blue = static_cast((word >> 22u) & 0xffu); + pixels[linear] = rgba ? blue : red; + pixels[linear + 1] = green; + pixels[linear + 2] = rgba ? red : blue; + pixels[linear + 3] = std::byte{0xff}; + continue; + } pixels[linear] = source[tiled + (rgba ? 2 : 0)]; pixels[linear + 1] = source[tiled + 1]; pixels[linear + 2] = source[tiled + (rgba ? 0 : 2)]; @@ -50,6 +81,12 @@ std::vector DecodeDisplayBuffer(const DisplayBuffer& buffer, std::spa std::vector ReadDisplayBuffer(const DisplayBuffer& buffer) { const auto size = DisplayBufferSize(buffer); + static const bool traceGpu = std::getenv("APS5_TRACE_GPU") != nullptr; + if (traceGpu) { + static const auto start = std::chrono::steady_clock::now(); + std::fprintf(stderr, "[gpu] scanout display buffer 0x%llx %ux%u format 0x%016llx at %.1f s\n", static_cast(buffer.address), buffer.width, buffer.height, static_cast(buffer.pixelFormat), std::chrono::duration(std::chrono::steady_clock::now() - start).count()); + } + GuestMemory::FlushGpuWrites(buffer.address, size); GuestMemory::CheckRange(reinterpret_cast(buffer.address), size, 65536); const auto* source = reinterpret_cast(buffer.address); return DecodeDisplayBuffer(buffer, {source, size}); diff --git a/core/libs/prx/libSceAgcDriver/Execution/src/Driver.cpp b/core/libs/prx/libSceAgcDriver/Execution/src/Driver.cpp index efd60cf2e..2a82386bb 100644 --- a/core/libs/prx/libSceAgcDriver/Execution/src/Driver.cpp +++ b/core/libs/prx/libSceAgcDriver/Execution/src/Driver.cpp @@ -6,30 +6,71 @@ #include "prx/libSceAgcDriver/Execution/include/QueueState.hpp" #include "prx/libSceAgcDriver/Execution/include/VulkanDevice.hpp" #include "prx/libSceAgcDriver/Execution/include/VideoOutput.hpp" +#include "prx/libSceAgcDriver/Execution/include/WorkerSampler.hpp" +#include "prx/libSceAgcDriver/Graphics/include/Recorder.hpp" #include "prx/libSceAgcDriver/Graphics/include/ShaderInputState.hpp" +#include "prx/libSceAgcDriver/Graphics/include/Texture.hpp" #include "prx/libc/include/Shutdown.hpp" -#include "prx/libSceAgcDriver/Execution/include/MemoryAccessScope.hpp" -#include "prx/libc/include/GuestMemoryTracking.hpp" +#include "ControlFlow/RequestSerializer.hpp" +#ifdef _WIN32 +#include +#endif +#include #include +#include +#include +#include +#include +#include #include #include +#include #include #include #include #include #include +#include #include #include #include #include +#include #include #include +#include #include +#include #include namespace AgcDriver { namespace { +// Milliseconds since the first trace line, for the APS5_TRACE_GPU timeline. +double TraceMs() { + static const auto origin = std::chrono::steady_clock::now(); + return std::chrono::duration(std::chrono::steady_clock::now() - origin).count(); +} + +// A short sleep between polls of a label another queue or the CPU writes. std::this_thread::sleep_for +// goes through winpthreads' nanosleep and rounds up to the scheduler tick (15.6 ms unless raised), +// which multiplied by the dozens of queue-to-queue hand-offs in a frame; a high-resolution waitable +// timer sleeps for the 200 us asked. +void PollSleep() { +#ifdef _WIN32 + thread_local HANDLE timer = CreateWaitableTimerExW(nullptr, nullptr, CREATE_WAITABLE_TIMER_HIGH_RESOLUTION, TIMER_ALL_ACCESS); + if (timer != nullptr) { + LARGE_INTEGER due{}; + due.QuadPart = -2000; // 200 us in 100 ns units + if (SetWaitableTimer(timer, &due, 0, nullptr, nullptr, FALSE)) { + WaitForSingleObject(timer, INFINITE); + return; + } + } +#endif + std::this_thread::sleep_for(std::chrono::microseconds(200)); +} + void require(bool condition, const char* reason) { if (!condition) { throw std::runtime_error(std::string("AGC driver: ") + reason); @@ -51,11 +92,9 @@ struct Submission { std::map> shaders; std::map> flips; bool suspend = false; - FrameTiming::Clock::time_point received; - FrameTiming::Clock::time_point copied; - FrameTiming::Clock::time_point validated; - FrameTiming::Clock::time_point enqueued; - FrameTiming::Clock::time_point dequeued; + // Record-order stamp (Driver::eventSerial) taken when the game submitted: a WAIT_REG_MEM of this + // submission trusts only labels the recorder noted with a newer stamp (see Recorder::NoteLabel). + std::uint64_t received = 0; }; std::uint32_t readRegister(const Registers& registers, std::uint32_t offset) { @@ -64,6 +103,161 @@ std::uint32_t readRegister(const Registers& registers, std::uint32_t offset) { return it->second; } +// The last packets of a submission, for the WAIT_REG_MEM timeout report and the draw-failure dump. +// Only offsets are kept per packet: formatting every packet's line cost more than most packets +// themselves, and the reports are rare. The commands outlive the history (both live in execute). +struct PacketHistory { + std::span commands; + std::array offsets{}; + std::size_t count = 0; + + void Record(std::size_t offset) { + offsets[count % offsets.size()] = offset; + ++count; + } + + std::string Format(std::size_t offset) const { + const auto header = commands[offset]; + const auto words = std::min(Pm4::PacketWords(header), commands.size() - offset); + char line[160]; + int length = std::snprintf(line, sizeof(line), "%s", Pm4::Name(header).c_str()); + for (std::size_t i = 1; i < words && i < 9 && length < 140; ++i) length += std::snprintf(line + length, sizeof(line) - length, " %08x", commands[offset + i]); + return line; + } + + // Visits the recorded packets' lines oldest to newest. + template + void Each(Visit&& visit) const { + const auto shown = std::min(count, offsets.size()); + const auto first = count > shown ? count % offsets.size() : 0; + for (std::size_t i = 0; i < shown; ++i) visit(Format(offsets[(first + i) % offsets.size()])); + } +}; + +// The driver's device. It is created, replaced and reset under GuestMemory::GpuMutex only; the +// atomic lets a dispatch's prologue (capture, cache validation, recompile), which runs without that +// lock, fetch it without waiting behind another queue's device phase. Arrow access is for holders of +// the GpuMutex, under which the pointee cannot be released. +class DevicePointer { +public: + std::shared_ptr load() const { return pointer.load(std::memory_order_acquire); } + operator std::shared_ptr() const { return load(); } + DevicePointer& operator=(std::shared_ptr value) { + pointer.store(std::move(value), std::memory_order_release); + return *this; + } + void reset() { *this = nullptr; } + VulkanDevice* operator->() const { return load().get(); } + explicit operator bool() const { return load() != nullptr; } + bool operator==(std::nullptr_t) const { return load() == nullptr; } + +private: + std::atomic> pointer; +}; + +// [sync] statistics (APS5_PROFILE_DRAW): device drains by packet, and the label outcomes of +// VulkanDevice::WriteLabelOnGpu (reason 0 GPU, 5 GPU store plus a completion action behind +// write-backs, 6 completion action because the memory is not host-imported, 1-4 CPU fallbacks by +// reason). Not-imported labels +// are counted apart: a title whose CPU-side label polling stalls looks there first. +// Counted from the label path and the drain sites of Driver::execute; the report itself is printed +// under GuestMemory::GpuMutex only (lastSyncReport is plain). +std::atomic gpuLabels{0}, completionLabels{0}, notImportedLabels{0}, noOpLabels{0}, unlockedDrains{0}; +std::atomic labelFallbacks[5] = {}; +// Labels queued in a worker's deferred list, the groups they were recorded in, and labels that +// took the locked path at once (too large for the list, or APS5_LABEL_LOCK_EACH=1). +std::atomic queuedLabels{0}, labelGroups{0}, immediateLabels{0}; +// Label groups recorded and label batches submitted at the following packet's own GPU mutex +// acquisition (recordLabelsForPacket), groups and submits a failed try in flushBetweenPackets left +// to that acquisition (counted apart: a compute worker's try is often submit-only), groups a +// self-locking packet recorded right after its capture (a try that succeeded there, or a queued +// label overlapping the capture, which the packet then redoes), and suspend points (all in the +// [labels] line). +std::atomic packetLockRecords{0}, packetLockSubmits{0}, packetLockDeferred{0}, packetSubmitDeferred{0}, captureTryRecords{0}, captureRetries{0}, suspendPoints{0}; +// APS5_PROFILE_DRAW: the time (microseconds) the record and the submit take at the packet's lock. +std::atomic packetLockRecordUs{0}, packetLockSubmitUs{0}; +// Non-label stores the driver makes for the title (COPY_DATA, DMA_DATA, DUMP_CONST_RAM; see the +// store path of Driver::execute): recorded on the GPU, recorded on the GPU and stored again by a +// completion action (behind a pending write-back, or memory the GPU has no view of: reasons 5 and +// 6, kept apart so a stale GPU read of a store is looked for there first), stored by the CPU under +// the mutex because the recorder was idle (nothing to order after), or stored by the CPU behind a +// device drain as before (too large, misaligned, not decodable, no device, or APS5_CPU_STORES=1). +std::atomic storesOnGpu{0}, storesBehindCompletions{0}, storesOnCpu{0}, storesDrained{0}; +std::map drainCounts; +std::uint64_t drainTotal = 0; +std::chrono::steady_clock::time_point lastSyncReport = std::chrono::steady_clock::now(); +void reportSync() { + std::string report; + for (const auto& [code, count] : drainCounts) report += " " + (code == 0xffffu ? std::string("flip") : Pm4::Name(code << 8u)) + "=" + std::to_string(count); + std::fprintf(stderr, "[sync] %llu device drains by packet:%s (%llu waited without the GPU mutex); %llu labels written on the GPU, %llu deferred behind completions, %llu deferred not imported, %llu no-op, fallbacks: idle %llu, completions %llu, not imported %llu, undecodable %llu; %llu labels queued per worker, recorded in %llu groups, %llu recorded at once; stores: %llu recorded on the GPU, %llu behind completions, %llu on the CPU (idle), %llu synced (drained)\n", static_cast(drainTotal), report.c_str(), static_cast(unlockedDrains.load()), static_cast(gpuLabels.load()), static_cast(completionLabels.load()), static_cast(notImportedLabels.load()), static_cast(noOpLabels.load()), static_cast(labelFallbacks[1].load()), static_cast(labelFallbacks[2].load()), static_cast(labelFallbacks[3].load()), static_cast(labelFallbacks[4].load()), static_cast(queuedLabels.load()), static_cast(labelGroups.load()), static_cast(immediateLabels.load()), static_cast(storesOnGpu.load()), static_cast(storesBehindCompletions.load()), static_cast(storesOnCpu.load()), static_cast(storesDrained.load())); +} +void countLabelOutcome(int reason) { + if (reason == 0) ++gpuLabels; + else if (reason == 5) ++completionLabels; + else if (reason == 6) ++notImportedLabels; + else ++labelFallbacks[std::min(reason, 4)]; +} + +// A worker's deferred labels. A label packet (RELEASE_MEM, WRITE_DATA) no longer takes the GPU +// mutex to record its store: the label is queued here, on the worker's own thread, and the whole +// group is recorded under the mutex the next time this worker needs it anyway (before any packet +// of its queue that touches memory or the device: the label still precedes that work in queue +// order), or at the label flush deadline (APS5_LABEL_FLUSH_US) checked between packets. Other +// queues and game threads poll memory for the value, so their latency stays bounded by the +// deadline. Debug aid: APS5_LABEL_LOCK_EACH=1 records every label under the mutex as before. +struct DeferredLabel { + static constexpr std::size_t Capacity = 32; + std::uint64_t address; + std::size_t size; + std::array bytes; +}; +struct DeferredLabels { + std::vector labels; + // When the first label of the group was queued (the deadline counts from here). + std::chrono::steady_clock::time_point since; +}; +DeferredLabels& deferredLabels() { + static thread_local DeferredLabels deferred; + return deferred; +} +bool DeferLabels() { + // APS5_DRAIN_COMPLETION_LABELS=1 implies the locked path too: a label that must drain then + // keeps the packet path's unlocked timeline wait instead of a WaitIdle under the mutex, so + // that switch bisects the same way it did before labels were deferred. + static const bool defer = std::getenv("APS5_LABEL_LOCK_EACH") == nullptr && std::getenv("APS5_DRAIN_COMPLETION_LABELS") == nullptr; + return defer; +} +// Whether a packet needs this worker's deferred labels recorded before it runs: everything except +// pure register and state writes, NOP, and the events the driver does not execute. A wait, dispatch, +// draw, copy, indirect register load, constant RAM dump or flip either reads memory a label may +// write (through the flush hook, which only knows recorded stores) or records device work that must +// follow the label in queue order. +bool NeedsRecordedLabels(std::uint32_t header) { + if (header == FlipPacketHeader) return true; + switch ((header >> 8u) & 0xffu) { + case 0x10: case 0x11: case 0x12: case 0x13: case 0x26: case 0x28: case 0x2a: case 0x2f: + case 0x42: case 0x46: case 0x58: case 0x68: case 0x69: case 0x76: case 0x78: case 0x79: case 0x7a: case 0x81: + return false; + default: return true; + } +} +// Packets whose own execution takes GuestMemory::GpuMutex and records this worker's deferred labels +// first thing inside it (Driver::recordLabelsForPacket): dispatches (direct and indirect, including +// the fill HLE), draws and the flip. Their unlocked prologue (capture, resource stage A) reads guest +// memory through a flush hook that does not know a still-queued label, so a capture is checked +// against the queued labels once it is known what it read (recordQueuedLabelsAfterCapture: an +// overlap records them and the packet starts over), and a CPU read of dispatch arguments records +// the queue's labels first (recordQueuedLabelsBeforeRead). Stores (COPY_DATA, DMA_DATA, +// DUMP_CONST_RAM) lock too but resolve their source before that lock, so they keep the record +// ahead of them: a copy of a label value this queue just wrote must see it. +bool PacketLocksItself(std::uint32_t header) { + if (header == FlipPacketHeader) return true; + switch ((header >> 8u) & 0xffu) { + case 0x15: case 0x16: case 0x2d: case 0x35: return true; + default: return false; + } +} + class Driver { public: static Driver& Get() { @@ -82,20 +276,24 @@ public: private: void stop() { - require(std::this_thread::get_id() != worker.get_id(), "worker cannot stop itself"); + require(!OnWorkerThread(), "worker cannot stop itself"); std::lock_guard shutdownLock(shutdownMutex); { std::lock_guard lock(mutex); stopping = true; } changed.notify_all(); - if (worker.joinable()) worker.join(); + // No worker is added once `stopping` is set, so the map is stable while the threads finish. + for (auto& [queue, worker] : workers) { + if (worker.thread.joinable()) worker.thread.join(); + } + std::lock_guard gpuLock(GuestMemory::GpuMutex()); + device.reset(); } public: void Submit(const Packet* packet, std::uint32_t queue) { - const auto received = FrameTiming::Clock::now(); CheckFailure(); require(queue == 0 || (queue >= 0x20 && queue < 0x58), "unsupported compute queue"); GuestMemory::CheckRange(packet, sizeof(Packet), alignof(Packet)); @@ -103,15 +301,14 @@ public: require(descriptor.flags == 0, "nonzero submission flags are not implemented"); Submission submission{}; submission.queue = queue; - submission.received = received; if (descriptor.dw_num != 0) { require(descriptor.dw_num <= std::numeric_limits::max() / sizeof(std::uint32_t), "command size overflow"); GuestMemory::CheckRange(descriptor.addr, static_cast(descriptor.dw_num) * sizeof(std::uint32_t), alignof(std::uint32_t)); submission.commands.assign(descriptor.addr, descriptor.addr + descriptor.dw_num); } - submission.copied = FrameTiming::Clock::now(); - validate(submission.commands, queue); - submission.validated = FrameTiming::Clock::now(); + validate(submission.commands, queue, descriptor.addr); + static const bool trace = std::getenv("APS5_TRACE_GPU") != nullptr; + if (trace) std::fprintf(stderr, "[gpu] %.1f submit queue=0x%x dwords=%zu at %p\n", TraceMs(), queue, submission.commands.size(), static_cast(descriptor.addr)); { std::lock_guard lock(mutex); rethrowFailure(); @@ -127,19 +324,21 @@ public: require(request != nullptr, "video output returned a null flip reservation"); submission.flips.emplace(cursor, std::move(request)); } - cursor += static_cast((words[0] >> 16u) & 0x3fffu) + 2; + cursor += Pm4::PacketWords(words[0]); } submission.shaders = shaders; submission.serial = accepted + 1; - submission.enqueued = FrameTiming::Clock::now(); - pending.push_back(std::move(submission)); + // Every CPU store the game made before this call precedes the stamp; labels recorded + // later carry a larger one. + submission.received = ++eventSerial; + enqueue(std::move(submission)); ++accepted; } changed.notify_all(); } void WaitIdle() { - require(std::this_thread::get_id() != worker.get_id(), "worker cannot wait for itself"); + require(!OnWorkerThread(), "worker cannot wait for itself"); std::unique_lock lock(mutex); const auto target = accepted; changed.wait(lock, [&] { return failure != nullptr || completed >= target; }); @@ -147,8 +346,7 @@ public: } void SuspendPoint() { - const auto received = FrameTiming::Clock::now(); - require(std::this_thread::get_id() != worker.get_id(), "worker cannot suspend itself"); + require(!OnWorkerThread(), "worker cannot suspend itself"); std::unique_lock lock(mutex); rethrowFailure(); require(!stopping, "suspend during shutdown"); @@ -156,15 +354,13 @@ public: Submission boundary{}; boundary.serial = accepted + 1; boundary.suspend = true; - boundary.received = received; - boundary.copied = received; - boundary.validated = received; - boundary.enqueued = FrameTiming::Clock::now(); - pending.push_back(std::move(boundary)); - const auto target = ++accepted; + // The boundary resets graphics state, so it runs in order with the graphics queue. + boundary.queue = 0; + enqueue(std::move(boundary)); + ++accepted; + // The suspend point only marks where the system may suspend the title; it does not wait for + // the GPU. Blocking here deadlocks frames whose GPU work waits on labels the CPU writes later. changed.notify_all(); - changed.wait(lock, [&] { return failure != nullptr || completed >= target; }); - rethrowFailure(); } void RegisterVideoOutput(std::uint32_t handle, const std::shared_ptr& output) { @@ -182,7 +378,10 @@ public: outputs.erase(it); } + // Called per packet and per poll of a wait: the flag keeps the driver mutex out of those loops + // until a failure exists, and `failure` itself is still read under the mutex. void CheckFailure() { + if (!failed.load(std::memory_order_acquire)) return; std::lock_guard lock(mutex); rethrowFailure(); } @@ -192,26 +391,41 @@ public: { std::lock_guard lock(mutex); if (!failure) failure = error; + failed.store(true, std::memory_order_release); for (const auto& [handle, output] : outputs) output->Fail(failure); - for (const auto& item : pending) { - for (const auto& [offset, flip] : item.flips) flip->Fail(failure); + for (auto& [queue, worker] : workers) { + for (const auto& item : worker.pending) { + for (const auto& [offset, flip] : item.flips) flip->Fail(failure); + } + worker.pending.clear(); } - pending.clear(); } changed.notify_all(); } void Present(const PresentationWindow& window, const DisplayBuffer* buffer, bool opaque, void (*gpuReady)(void*), void* context) { + // The presenter's frame record travels in the window descriptor: PerformanceContext is a + // thread_local per DLL, so the one the video-out library set is invisible here. PerformanceContext timingContext(window.timing.get()); PerformanceTimer timing("Driver.Present"); CheckFailure(); require(gpuReady != nullptr && context != nullptr, "missing GPU completion callback"); require(window.getDrawableSize != nullptr, "missing window drawable size query"); + // The device is held only while the presentation is recorded and submitted; the swapchain + // acquire (where FIFO mode waits for the vblank) and the render fence wait happen outside + // GpuMutex so the queue workers are not blocked for the present's GPU or display time (the + // presenter submits on the same VkQueue as the recorder, so queue order makes the frame's + // batches complete first). Debug aid: APS5_SYNC_FLIP=1 drains the device and presents + // synchronously under one mutex hold, as before. + static const bool syncFlip = std::getenv("APS5_SYNC_FLIP") != nullptr; std::shared_ptr presenting; timing.Mark("validate"); try { + bool submitted = false; + bool presentable = false; { - std::lock_guard lock(gpuMutex); + GuestMemory::TagGpuLockSite(GuestMemory::GpuLockSite::Present); + std::lock_guard lock(GuestMemory::GpuMutex()); timing.Mark("gpu_mutex_wait"); if (device == nullptr || device->Window() == nullptr) { if (device) device->WaitIdle(); @@ -225,18 +439,46 @@ public: window.getDrawableSize(window.context, &drawableWidth, &drawableHeight); presenting->Resize(drawableWidth, drawableHeight); timing.Mark("resize"); - if (presenting->Presentable()) { - if (buffer != nullptr) { - require(buffer->width == window.width && buffer->height == window.height, "display buffer extent differs from output"); - presenting->WaitDraws(); - timing.Mark("draw_wait"); - presenting->PresentDisplayBuffer(*buffer); - timing.Mark("present_display_buffer"); - } else { - presenting->PresentClear(window.width, window.height, opaque); - timing.Mark("present_clear"); + presentable = presenting->Presentable(); + if (buffer != nullptr) require(buffer->width == window.width && buffer->height == window.height, "display buffer extent differs from output"); + } + // The swapchain, its fences and semaphores are the presenter's own: only the VkQueue needs + // the mutex, so the acquire (and its vblank wait) is taken first without it. + if (presentable && !syncFlip) { + presentable = presenting->AcquireImage(); + timing.Mark("acquire_image"); + } + if (presentable) { + GuestMemory::TagGpuLockSite(GuestMemory::GpuLockSite::Present); + std::lock_guard lock(GuestMemory::GpuMutex()); + timing.Mark("gpu_mutex_wait"); + if (buffer != nullptr) { + if (syncFlip) { + presenting->WaitIdle(); + timing.Mark("device_idle_wait"); } + submitted = presenting->PresentDisplayBuffer(*buffer); + timing.Mark("present_display_buffer"); + } else { + submitted = presenting->PresentClear(window.width, window.height, opaque); + timing.Mark("present_clear"); } + if (submitted && syncFlip) { + presenting->FinishPresent(); + timing.Mark("render_fence_wait"); + presenting->QueuePresent(); + timing.Mark("queue_present"); + submitted = false; + } + } + if (submitted) { + presenting->FinishPresent(); + timing.Mark("render_fence_wait"); + // The single VkQueue is shared with the recorder: presenting on it needs the mutex. + GuestMemory::TagGpuLockSite(GuestMemory::GpuLockSite::Present); + std::lock_guard lock(GuestMemory::GpuMutex()); + presenting->QueuePresent(); + timing.Mark("queue_present"); } gpuReady(context); timing.Mark("release_and_callback"); @@ -248,7 +490,7 @@ public: } void ReleaseWindow(void* window) { - std::lock_guard lock(gpuMutex); + std::lock_guard lock(GuestMemory::GpuMutex()); if (device && device->Window() == window) device.reset(); } @@ -266,6 +508,15 @@ public: std::memcpy(snapshot.code.data(), code, shader->shader_size); snapshot.header.resize(shader->header_size); std::memcpy(snapshot.header.data(), shader, shader->header_size); + // Debug aid: APS5_TRACE_SHADER_REGS= (or "all") prints a shader's register lists. + static const char* traceRegs = std::getenv("APS5_TRACE_SHADER_REGS"); + if (traceRegs != nullptr && (std::string(traceRegs) == "all" || std::strtoull(traceRegs, nullptr, 16) == snapshot.codeAddress)) { + std::fprintf(stderr, "[shader] 0x%llx type %u cx", static_cast(snapshot.codeAddress), shader->type); + for (std::uint32_t i = 0; i < shader->num_cx_registers && shader->cx_registers != nullptr; ++i) std::fprintf(stderr, " %x=%08x", shader->cx_registers[i].offset, shader->cx_registers[i].value); + std::fprintf(stderr, " sh"); + for (std::uint32_t i = 0; i < shader->num_sh_registers && shader->sh_registers != nullptr; ++i) std::fprintf(stderr, " %x=%08x", shader->sh_registers[i].offset, shader->sh_registers[i].value); + std::fprintf(stderr, "\n"); + } std::lock_guard lock(mutex); rethrowFailure(); const auto address = snapshot.codeAddress; @@ -276,22 +527,68 @@ private: std::mutex mutex; std::mutex shutdownMutex; std::condition_variable changed; - std::deque pending; - std::map> shaders; + // Each queue runs on its own thread, so a WAIT_REG_MEM blocks only its own queue as on the GPU. + // Draws and dispatches take GuestMemory::GpuMutex, so device work stays serialized. + struct QueueWorker { + std::deque pending; + std::thread thread; + }; + std::map workers; + std::uint64_t frameSerial = 0; + // Packets executing now and packets finished, across all queues; waits time out only when neither + // moves. On a cache line of their own: the waiting queues poll them while the others update them. + alignas(64) std::atomic packetsInFlight{0}; + std::atomic packetsDone{0}; + // Orders the game's submit calls against the labels the workers record (Submission::received). + std::atomic eventSerial{0}; + alignas(64) std::map> shaders; + // Dispatch cache: a program with the same user data and shader registers whose captured memory + // (SRT chains, descriptors) is unchanged reuses its capture and recompile result. Guarded by + // dispatchCacheMutex; entries are evicted least recently used. + struct DispatchEntry { + std::shared_ptr memory; + std::vector captured; + std::shared_ptr compiled; + std::vector> spans; + std::uint64_t generation = 0; + // The registered shader whose code and header the capture references. + std::shared_ptr shader; + // This entry's position in dispatchOrder. + std::list::iterator order; + }; + static constexpr std::size_t DispatchCacheEntries = 4096; + std::unordered_map dispatchCache; + // Keys most recently used first. + std::list dispatchOrder; + std::mutex dispatchCacheMutex; + std::uint64_t dispatchCacheHits = 0; + std::uint64_t dispatchCacheEvictions = 0; std::map queues; std::map> outputs; - std::recursive_mutex& gpuMutex = GuestMemoryTracking::GuestMemoryTrackingMutex_nid_postfix(); - std::shared_ptr device; + DevicePointer device; std::uint64_t accepted = 0; std::uint64_t completed = 0; + std::set completedOutOfOrder; std::exception_ptr failure; + // Set once `failure` is, so hot loops can check for one without the mutex. + std::atomic failed{false}; bool stopping = false; bool resetGraphics = false; - std::shared_ptr frameTiming; - std::uint64_t frameSerial = 0; - std::thread worker; - Driver() : worker([this] { run(); }) { + static bool& OnWorkerThread() { + static thread_local bool worker = false; + return worker; + } + + // Caller holds `mutex`. + void enqueue(Submission submission) { + const auto queue = submission.queue; + auto& worker = workers[queue]; + worker.pending.push_back(std::move(submission)); + if (!worker.thread.joinable()) worker.thread = std::thread([this, queue] { run(queue); }); + } + + Driver() { try { LibcRegisterShutdown_nid_postfix([] { Driver::Get().Shutdown(); }); } catch (...) { @@ -306,25 +603,227 @@ private: } } - static void validate(std::span commands, std::uint32_t queue) { + // Summarizes a rejected submission (packet names with counts and the first rejection reason per name). + static void dumpPackets(std::span commands, const std::uint32_t* guest = nullptr) { + std::map> summary; + std::vector> walk; for (std::size_t cursor = 0; cursor < commands.size();) { const auto header = commands[cursor]; - require((header & 0xc0000000u) == 0xc0000000u, "unsupported PM4 packet type"); - const auto count = static_cast((header >> 16u) & 0x3fffu) + 2; + if (Pm4::FillerPacket(header)) { ++cursor; continue; } + if ((header & 0xc0000000u) != 0xc0000000u) break; + const auto count = Pm4::PacketWords(header); + if (count > commands.size() - cursor) break; + walk.emplace_back(cursor, header); + auto& entry = summary[Pm4::Name(header)]; + if (entry.first++ == 0) { + try { + Pm4::Validate(commands.subspan(cursor, count), 0); + } catch (const std::exception& error) { + entry.second = error.what(); + } + } + cursor += count; + } + std::fprintf(stderr, "[gpu] rejected submission of %zu dwords at guest %p:\n", commands.size(), static_cast(guest)); + for (const auto& [name, entry] : summary) std::fprintf(stderr, "[gpu] %-28s x%-5zu %s\n", name.c_str(), entry.first, entry.second.c_str()); + // The last packets walked before the parse broke and, with APS5_DUMP_REJECTED=1, the whole + // stream for offline analysis. + for (std::size_t i = walk.size() > 24 ? walk.size() - 24 : 0; i < walk.size(); ++i) { + const auto [offset, header] = walk[i]; + std::fprintf(stderr, "[gpu] at DWORD %-6zu header 0x%08x %-24s %zu dwords:", offset, header, Pm4::Name(header).c_str(), Pm4::PacketWords(header)); + for (std::size_t j = offset + 1; j < std::min(commands.size(), offset + std::min(Pm4::PacketWords(header), 12)); ++j) std::fprintf(stderr, " %08x", commands[j]); + std::fprintf(stderr, "\n"); + } + static const bool dumpRejected = std::getenv("APS5_DUMP_REJECTED") != nullptr; + if (!dumpRejected) return; + static int dumps = 0; + char fileName[64]; + std::snprintf(fileName, sizeof(fileName), "rejected_submission_%d.bin", dumps++); + if (FILE* file = std::fopen(fileName, "wb")) { + std::fwrite(commands.data(), sizeof(std::uint32_t), commands.size(), file); + std::fclose(file); + std::fprintf(stderr, "[gpu] rejected submission written to %s\n", fileName); + } + } + + static void validate(std::span commands, std::uint32_t queue, const std::uint32_t* guest = nullptr) { + for (std::size_t cursor = 0; cursor < commands.size();) { + const auto header = commands[cursor]; + if (Pm4::FillerPacket(header)) { ++cursor; continue; } + if ((header & 0xc0000000u) != 0xc0000000u) { + char what[96]; + std::snprintf(what, sizeof(what), "unsupported PM4 packet type: header 0x%08x at DWORD %zu of %zu", header, cursor, commands.size()); + // The packets before it (and the dwords around it) show which packet was mis-sized. + dumpPackets(commands, guest); + std::fprintf(stderr, "[gpu] dwords %zu..%zu:", cursor >= 8 ? cursor - 8 : 0, std::min(commands.size(), cursor + 8)); + for (std::size_t i = cursor >= 8 ? cursor - 8 : 0; i < std::min(commands.size(), cursor + 8); ++i) std::fprintf(stderr, " %08x", commands[i]); + std::fprintf(stderr, "\n"); + throw std::runtime_error(std::string("AGC driver: ") + what); + } + const auto count = Pm4::PacketWords(header); require(count <= commands.size() - cursor, "truncated PM4 packet"); try { Pm4::Validate(commands.subspan(cursor, count), queue); } catch (const std::exception& error) { + dumpPackets(commands, guest); throw std::runtime_error("AGC driver: " + Pm4::Name(header) + " at DWORD " + std::to_string(cursor) + ": " + error.what()); } cursor += count; } } - void dispatch(QueueState& queue, std::span packet, const Submission& submission) { - PerformanceTimer timing("Driver.Dispatch"); - std::lock_guard gpuLock(gpuMutex); - timing.Mark("gpu_mutex_wait"); + // Worker time per packet class, reported every 10 s (APS5_PROFILE_DRAW) so throughput problems + // show where time goes. + struct WorkerProfile { + double dispatchMs = 0; + double drawMs = 0; + double waitMs = 0; + std::chrono::steady_clock::time_point start = std::chrono::steady_clock::now(); + std::chrono::steady_clock::time_point reported = start; + }; + template + static void timed(double WorkerProfile::*bucket, Work&& work) { + static thread_local WorkerProfile profile; + const auto begin = std::chrono::steady_clock::now(); + work(); + const auto end = std::chrono::steady_clock::now(); + profile.*bucket += std::chrono::duration(end - begin).count(); + static const bool report = std::getenv("APS5_PROFILE_DRAW") != nullptr; + if (report && end - profile.reported > std::chrono::seconds(10)) { + profile.reported = end; + std::fprintf(stderr, "[gpu] worker at %.0f s: dispatch %.1f s, draw %.1f s, wait %.1f s\n", std::chrono::duration(end - profile.start).count(), profile.dispatchMs / 1000, profile.drawMs / 1000, profile.waitMs / 1000); + } + } + + // A draw or dispatch the translator cannot handle is skipped so the rest of the frame still runs; + // each distinct failure is reported once. + template + void tolerate(const char* kind, Work&& work) { + try { + work(); + } catch (const std::exception& error) { + static std::mutex reportedMutex; + static std::set reported; + std::lock_guard lock(reportedMutex); + // APS5_TRACE_SKIPS reports every skip (shortened), not only the first per reason. + static const bool traceSkips = std::getenv("APS5_TRACE_SKIPS") != nullptr; + if (reported.insert(error.what()).second) std::fprintf(stderr, "[gpu] skipped %s: %s\n", kind, error.what()); + else if (traceSkips) std::fprintf(stderr, "[gpu] skipped %s again: %.100s\n", kind, error.what()); + } + } + + // Saves a request once per shader address so it can be replayed with agc_shader_replay. + static std::string dumpRequest(std::uint64_t address, const ShaderRecompiler::RecompileRequest& request) { + static std::mutex dumpMutex; + static std::set dumped; + char name[64]; + std::snprintf(name, sizeof(name), "shader_%llx.req", static_cast(address)); + std::lock_guard lock(dumpMutex); + if (!dumped.insert(address).second) return name; + try { + const auto text = ShaderRecompiler::RequestSerializer{}.Serialize(request); + if (std::FILE* file = std::fopen(name, "wb")) { + std::fwrite(text.data(), 1, text.size(), file); + std::fclose(file); + } + } catch (const std::exception& error) { + std::fprintf(stderr, "[gpu] could not serialize request for 0x%llx: %s\n", static_cast(address), error.what()); + } + return name; + } + + // Capture and recompile run without the GPU lock, so the compute queues prepare their dispatches + // in parallel; only resource building and recording are serialized. + // The engine clears and initializes buffers with a nine-dword compute kernel (every thread stores one + // 16-byte record of a 32_32_32_32_UINT typed buffer: v_lshl_add_u32 v4, s8, 6, v0; four v_mov from + // the pattern in user data; buffer_store_format_xyzw v[0:3], v4, s[0:3] idxen). Demon's Souls runs + // it ~13 times per frame over ~80 MB, and typed 16-byte stores into host-imported system memory took + // ~22 ms each on the GPU while every drain waited for them. The fill is done as a transfer instead + // (or a CPU store when the memory is not imported). Debug aid: APS5_NO_FILL_HLE=1 runs the kernel. + // Whether `code` is that kernel with a V# the transfer can reproduce: everything fillBuffer + // decides on except the group count, so an indirect dispatch can tell before reading its count. + static bool matchesFillKernel(std::span code, const std::vector& userData, const ShaderRecompiler::ShaderComputeStageInfo& compute) { + static const bool enabled = std::getenv("APS5_NO_FILL_HLE") == nullptr; + if (!enabled || userData.size() < 8 || compute.numThreads[0] != 64 || compute.numThreads[1] != 1 || compute.numThreads[2] != 1) return false; + static constexpr std::array fillKernel{0xd7460004u, 0x04010c08u, 0x7e000204u, 0x7e020205u, 0x7e040206u, 0x7e060207u, 0xe01c2000u, 0x80000004u, 0xbf810000u}; + if (code.size() < fillKernel.size() || !std::equal(fillKernel.begin(), fillKernel.end(), code.begin())) return false; + // V# in user[0..3]: base, stride 16, record count, 32_32_32_32_UINT with an identity swizzle. + const auto stride = (userData[1] >> 16u) & 0x3fffu; + const bool swizzled = ((userData[1] >> 31u) & 1u) != 0; + const auto dstSel = userData[3] & 0xfffu; + const bool addTid = ((userData[3] >> 23u) & 1u) != 0; + const auto type = userData[3] >> 30u; + const auto format = (userData[3] >> 12u) & 0x7fu; + return type == 0 && stride == 16 && !swizzled && !addTid && dstSel == 0xfacu && format == 0x4bu; + } + + bool fillBuffer(QueueState& queue, std::uint32_t queueId, std::span packet, std::span code, const std::vector& userData, const ShaderRecompiler::ShaderComputeStageInfo& compute, const std::shared_ptr& localDevice) { + if (!matchesFillKernel(code, userData, compute)) return false; + const auto numRecords = userData[2]; + std::array groups{packet[1], packet[2], packet[3]}; + if ((packet[4] & 0x20u) != 0) { + for (std::uint32_t axis = 0; axis < 3; ++axis) { + const auto threads = std::max(readRegister(queue.shader, 0x207 + axis) & 0xffffu, 1u); + groups[axis] = (groups[axis] + threads - 1) / threads; + } + } + if (groups[1] != 1 || groups[2] != 1) return false; + const auto records = std::min(static_cast(groups[0]) * 64u, numRecords); + const auto base = userData[0] | (static_cast(userData[1] & 0xffffu) << 32u); + const auto bytes = static_cast(records * 16u); + const std::array pattern{userData[4], userData[5], userData[6], userData[7]}; + static const bool profile = std::getenv("APS5_PROFILE_DRAW") != nullptr; + static std::atomic fills{0}, filledBytes{0}, cpuFills{0}; + ++fills; + filledBytes += bytes; + if (bytes != 0) { + GuestMemory::TagGpuLockSite(GuestMemory::GpuLockSite::Fill); + std::lock_guard gpuLock(GuestMemory::GpuMutex()); + // This queue's labels first (queue order), as at every dispatch's lock. + recordLabelsForPacket(localDevice.get(), queueId); + GuestMemory::CheckRange(reinterpret_cast(base), bytes, 16, true); + // Results still on the GPU for the range would be stored over the fill later. + Graphics::StorageTexture::FlushPending(base, bytes, nullptr, "buffer fill"); + if (!localDevice->FillBuffer(base, bytes, pattern)) { + ++cpuFills; + // Ordered after recorded work like any CPU store from the queue. + localDevice->WaitIdle(); + static thread_local std::vector block; + const auto chunk = std::min(bytes, 1u << 20u); + block.resize(chunk); + for (std::size_t at = 0; at < chunk; at += 16) std::memcpy(block.data() + at, pattern.data(), 16); + for (std::size_t done = 0; done < bytes; done += chunk) GuestMemory::Write(base + done, std::span(block).first(std::min(chunk, bytes - done)), 16); + } + } + if (profile && fills % 500 == 0) std::fprintf(stderr, "[fill] %llu buffer fills (%.0f MiB), %llu stored by the CPU\n", static_cast(fills.load()), filledBytes.load() / 1048576.0, static_cast(cpuFills.load())); + return true; + } + + // How DISPATCH_INDIRECT group counts were resolved, for the [indirect] line (APS5_PROFILE_DRAW, + // every 10 s). 0..3 are VulkanDevice::DispatchIndirect's outcomes (0 recorded GPU-side), the rest + // are decided here before the dispatch is prepared. `readMs` is the CPU read's time (its sync). + enum IndirectPath { IndirectGpu = 0, IndirectPendingImage = 1, IndirectCopiedWrite = 2, IndirectNotImported = 3, IndirectThreadDimensions = 4, IndirectFillKernel = 5, IndirectMisaligned = 6, IndirectDisabled = 7, IndirectPaths = 8 }; + static void countIndirect(int path, double readMs) { + static const bool profile = std::getenv("APS5_PROFILE_DRAW") != nullptr; + if (!profile || path < 0 || path >= IndirectPaths) return; + static std::mutex countsMutex; + static std::uint64_t counts[IndirectPaths] = {}; + static double readWaitedMs = 0; + static auto lastReport = std::chrono::steady_clock::now(); + std::lock_guard lock(countsMutex); + ++counts[path]; + readWaitedMs += readMs; + const auto now = std::chrono::steady_clock::now(); + if (now - lastReport < std::chrono::seconds(10)) return; + lastReport = now; + std::uint64_t cpu = 0; + for (int i = 1; i < IndirectPaths; ++i) cpu += counts[i]; + std::fprintf(stderr, "[indirect] gpu-side %llu, cpu-side %llu (thread dimensions %llu, fill kernel %llu, pending image results %llu, pending label or copied write %llu, not imported %llu, misaligned %llu, disabled %llu); CPU argument reads took %.1f s\n", static_cast(counts[IndirectGpu]), static_cast(cpu), static_cast(counts[IndirectThreadDimensions]), static_cast(counts[IndirectFillKernel]), static_cast(counts[IndirectPendingImage]), static_cast(counts[IndirectCopiedWrite]), static_cast(counts[IndirectNotImported]), static_cast(counts[IndirectMisaligned]), static_cast(counts[IndirectDisabled]), readWaitedMs / 1000); + } + + // `indirectArguments` (non-zero): a DISPATCH_INDIRECT whose group counts, at that guest address, + // are not read here; `packet` then carries only the initiator (see dispatchIndirect). + void dispatch(QueueState& queue, std::span packet, const Submission& submission, std::uint64_t indirectArguments = 0) { const auto address = (static_cast(readRegister(queue.shader, 0x20c)) << 8u) | (static_cast(readRegister(queue.shader, 0x20d) & 0xffu) << 40u); auto it = submission.shaders.upper_bound(address); require(it != submission.shaders.begin(), "compute program does not belong to a registered shader"); @@ -339,39 +838,390 @@ private: } const auto compute = Graphics::DecodeComputeStageInfo(queue.shader); const std::array memory{{{snapshot.codeAddress, std::as_bytes(std::span(snapshot.code))}, {snapshot.headerAddress, snapshot.header}}}; - if (device == nullptr) { - device = std::make_shared(); + // The GpuMutex is only taken to create the device: taking it just to copy the pointer made + // every dispatch wait behind another queue's whole device phase before its lock-free prologue. + // Debug aid: APS5_NO_UNLOCKED_DEVICE=1 takes the lock to read it as before. + static const bool unlockedDevice = std::getenv("APS5_NO_UNLOCKED_DEVICE") == nullptr; + std::shared_ptr localDevice = unlockedDevice ? device.load() : nullptr; + if (localDevice == nullptr) { + GuestMemory::TagGpuLockSite(GuestMemory::GpuLockSite::Dispatch); + std::lock_guard gpuLock(GuestMemory::GpuMutex()); + if (device == nullptr) device = std::make_shared(); + localDevice = device; } - const GuestMemory::MemoryAccessScope memoryScope(device.get(), [](void* context, std::uint64_t address, std::size_t bytes, bool writable) { - static_cast(context)->ResolveMemory(address, bytes, writable); - }); const auto codeOffset = static_cast((address - snapshot.codeAddress) / sizeof(std::uint32_t)); + std::array resolved{}; + if (indirectArguments != 0 && matchesFillKernel(std::span(snapshot.code).subspan(codeOffset), userData, compute)) { + // The fill HLE consumes the group count on the CPU: read it as before (waiting for the + // producer through the flush hook) and continue as a direct dispatch. + recordQueuedLabelsBeforeRead(submission.queue); + const auto readStart = std::chrono::steady_clock::now(); + resolved = Pm4::ReadDispatchArguments(indirectArguments, packet[4]); + countIndirect(IndirectFillKernel, std::chrono::duration(std::chrono::steady_clock::now() - readStart).count()); + packet = resolved; + indirectArguments = 0; + } + if (fillBuffer(queue, submission.queue, packet, std::span(snapshot.code).subspan(codeOffset), userData, compute, localDevice)) return; ShaderRecompiler::RecompileRequest request{ {ShaderRecompiler::ShaderStage::Compute, address, std::span(snapshot.code).subspan(codeOffset), snapshot.headerAddress, snapshot.header}, {(packet[4] & 0x8000u) != 0 ? 32u : 64u, 0, userData, compute, std::nullopt, std::nullopt, memory}, - device->Target(), + localDevice->Target(), {0, 0, 0, 128} }; - ShaderMemory shaderMemory(memory); - timing.Mark("prepare"); - shaderMemory.Capture(request); - timing.Mark("shader_memory_capture"); - const auto captured = shaderMemory.Regions(); - request.context.memory = captured; - timing.Mark("request_memory"); - const auto compiled = ShaderRecompiler::Recompile(request); - timing.Mark(compiled.cacheHit ? "shader_cache_hit" : "shader_compile"); + static const bool profile = std::getenv("APS5_PROFILE_DRAW") != nullptr; + static double captureMs = 0, keyMs = 0, recompileMs = 0, deviceMs = 0; + static std::uint64_t cacheHits = 0; + static std::uint64_t dispatches = 0; + auto lap = std::chrono::steady_clock::now(); + const auto elapsed = [&] { + const auto now = std::chrono::steady_clock::now(); + const auto ms = std::chrono::duration(now - lap).count(); + lap = now; + return ms; + }; + // Debug aid: APS5_NO_DISPATCH_CACHE=1 captures and recompiles every dispatch. + static const bool noDispatchCacheEnv = std::getenv("APS5_NO_DISPATCH_CACHE") != nullptr; + // Debug aid: APS5_PROBE_DISPATCH=: applies the recompiler's APS5_PROBE + // register probe to the n-th (0-based) dispatch of that program only; that dispatch is + // recompiled outside the dispatch cache. + static const std::pair probeDispatch = [] { + const char* text = std::getenv("APS5_PROBE_DISPATCH"); + if (text == nullptr) return std::pair{0, 0}; + char* end = nullptr; + const auto probeAddress = std::strtoull(text, &end, 16); + const auto index = end != nullptr && *end == ':' ? std::strtoull(end + 1, nullptr, 10) : 0ull; + return std::pair{probeAddress, index}; + }(); + bool probeThis = false; + // The heap base differs between runs (0x10…, 0x20…, 0x30…), so only the low 36 bits are compared. + if (probeDispatch.first != 0 && (address & 0xfffffffffull) == (probeDispatch.first & 0xfffffffffull)) { + static std::atomic dispatchesSeen{0}; + probeThis = dispatchesSeen.fetch_add(1) == probeDispatch.second; + if (probeThis) std::fprintf(stderr, "[gpu] probing dispatch %llu of 0x%llx\n", static_cast(probeDispatch.second), static_cast(address)); + } + const bool noDispatchCache = noDispatchCacheEnv || probeThis; + std::uint64_t key = 0xcbf29ce484222325ull; + const auto mix = [&](std::uint64_t value) { + key ^= value; + key *= 0x100000001b3ull; + }; + mix(address); + mix(packet[4] & 0x8000u); + for (const auto word : userData) mix(word); + // Only the shader registers the request reads (thread counts and RSRC1/2; the program + // address is `address`): on queue 0 the bank also holds the graphics stages' registers, + // which every draw rewrites. Debug aid: APS5_NO_DISPATCH_KEY_HYGIENE=1 mixes the whole bank. + static const bool keyHygiene = std::getenv("APS5_NO_DISPATCH_KEY_HYGIENE") == nullptr; + if (keyHygiene) { + for (const auto offset : {0x207u, 0x208u, 0x209u, 0x212u, 0x213u}) { + const auto found = queue.shader.find(offset); + // An absent register mixes a value no 32-bit register can hold. + mix(found == queue.shader.end() ? (1ull << 32u) : found->second); + } + } else { + for (const auto& [offset, value] : queue.shader) { + mix(offset); + mix(value); + } + } + std::shared_ptr compiledResult; + std::shared_ptr keepMemory; + // The captured code and header spans point into the entry's snapshot, which may differ + // from this submission's (re-registered, identical bytes); a hit keeps it past eviction. + std::shared_ptr keepShader; + std::vector captured; + bool cached = false; + // Debug aid: APS5_TRACE_DISPATCH_CACHE reports, per program, what changed between dispatches. + static const bool traceCache = std::getenv("APS5_TRACE_DISPATCH_CACHE") != nullptr; + if (traceCache) { + std::lock_guard traceLock(dispatchCacheMutex); + struct Last { std::vector userData; std::map shader; std::uint64_t key; }; + static std::map last; + static int reports = 0; + auto& previous = last[address]; + if (previous.key != 0 && previous.key != key && reports < 200) { + std::string what; + for (std::size_t i = 0; i < userData.size(); ++i) { + if (i >= previous.userData.size() || previous.userData[i] != userData[i]) { + char text[48]; + std::snprintf(text, sizeof(text), " user[%zu] %08x->%08x", i, i < previous.userData.size() ? previous.userData[i] : 0u, userData[i]); + what += text; + } + } + for (const auto& [offset, value] : queue.shader) { + const auto old = previous.shader.find(offset); + if (old == previous.shader.end() || old->second != value) { + char text[48]; + std::snprintf(text, sizeof(text), " sh[%x] %08x->%08x", offset, old == previous.shader.end() ? 0u : old->second, value); + what += text; + } + } + ++reports; + std::fprintf(stderr, "[dispatch-cache] 0x%llx key changed:%s\n", static_cast(address), what.c_str()); + } + previous.userData = userData; + previous.shader = std::map(queue.shader.begin(), queue.shader.end()); + previous.key = key; + } + if (!noDispatchCache) { + // The entry is copied out and validated without the cache lock: CollectWrites and + // EqualsCommitted (which can wait for the GpuMutex through the flush hook) would + // otherwise stall every other queue's lookup behind this one. The write watch keeps + // concurrent validations of one entry correct, so the re-lock only has to skip an entry + // another worker changed meanwhile. Debug aid: APS5_NO_UNLOCKED_VALIDATE=1 holds the lock. + static const bool validateUnlocked = std::getenv("APS5_NO_UNLOCKED_VALIDATE") == nullptr; + std::unique_lock cacheLock(dispatchCacheMutex); + if (const auto found = dispatchCache.find(key); found != dispatchCache.end()) { + auto entry = found->second; + if (validateUnlocked) cacheLock.unlock(); + bool current = true; + for (const auto& [begin, bytes] : entry.spans) { + GuestMemory::CollectWrites(begin, bytes); + if (!GuestMemory::UnchangedSince(begin, bytes, entry.generation)) { + current = false; + break; + } + } + std::uint64_t restamped = 0; + if (!current) { + // A write landed in a watched span (spans merge regions up to 64 KiB apart, so it + // is often unrelated): the captured bytes themselves decide, and an unchanged + // capture is re-stamped at a fresh generation. The generation is collected + // BEFORE the compare, as on insert: a write landing after the collect dirties + // the pages again and fails the next validation, whereas one landing between a + // compare and a later collect would be stamped into the entry and never seen. + std::uint64_t generation = 0; + for (const auto& [begin, bytes] : entry.spans) generation = std::max(generation, GuestMemory::CollectWrites(begin, bytes)); + bool same = generation != 0; + const GuestMemory::ReadSiteScope site(GuestMemory::ReadSite::DispatchCache); + for (const auto& region : entry.captured) { + if (!same) break; + same = GuestMemory::EqualsCommitted(region.guestAddress, region.bytes); + } + if (same) { + restamped = generation; + current = true; + } + } + if (!cacheLock.owns_lock()) cacheLock.lock(); + // Only an entry still at the copied generation is re-stamped or erased; the validated + // copy is used either way. + const auto again = dispatchCache.find(key); + const bool untouched = again != dispatchCache.end() && again->second.generation == entry.generation; + if (current) { + compiledResult = std::move(entry.compiled); + keepMemory = std::move(entry.memory); + keepShader = std::move(entry.shader); + captured = std::move(entry.captured); + cached = true; + ++dispatchCacheHits; + if (untouched) { + if (restamped != 0) again->second.generation = restamped; + dispatchOrder.splice(dispatchOrder.begin(), dispatchOrder, again->second.order); + } + } else { + if (traceCache) std::fprintf(stderr, "[dispatch-cache] 0x%llx captured memory changed\n", static_cast(address)); + if (untouched) { + dispatchOrder.erase(again->second.order); + dispatchCache.erase(again); + } + } + } + } + if (cached) { + captureMs += elapsed(); + } else { + auto shaderMemory = std::make_shared(memory); + // Debug aid: APS5_DUMP_SHADERS=1 saves every request, failing ones included, for agc_shader_replay. + static const bool dumpShaders = std::getenv("APS5_DUMP_SHADERS") != nullptr; + try { + // The probe is part of the recompiler's cache key, so it covers the capture's plan + // lookup as well as the recompile that reuses it. + struct ProbeScope { + bool active; + explicit ProbeScope(bool active) : active(active) { if (active) ShaderRecompiler::SetDebugProbeActive(true); } + ~ProbeScope() { if (active) ShaderRecompiler::SetDebugProbeActive(false); } + } probeScope{probeThis}; + const auto capture = shaderMemory->Capture(request); + captured = shaderMemory->Regions(); + request.context.memory = captured; + captureMs += elapsed(); + if (dumpShaders) static_cast(dumpRequest(address, request)); + const auto started = std::chrono::steady_clock::now(); + // The recompile reuses the capture's plan and materialization rather than walking + // the captured regions again. Debug aid: APS5_NO_CAPTURE_REUSE=1 walks them again. + static const bool reuseCapture = std::getenv("APS5_NO_CAPTURE_REUSE") == nullptr; + compiledResult = std::make_shared(reuseCapture ? ShaderRecompiler::Recompile(request, *capture) : ShaderRecompiler::Recompile(request)); + if (compiledResult->cacheHit) ++cacheHits; + const auto elapsed = std::chrono::duration(std::chrono::steady_clock::now() - started).count(); + static double totalMs = 0; + totalMs += elapsed; + if (profile && elapsed > 200) std::fprintf(stderr, "[gpu] compute shader 0x%llx recompile took %.0f ms (%zu SPIR-V words, %zu captured regions, total %.1f s)\n", static_cast(address), elapsed, compiledResult->spirv.size(), captured.size(), totalMs / 1000); + } catch (const std::exception& error) { + const auto dump = dumpShaders ? dumpRequest(address, request) : std::string{}; + // Recompile errors append the serialized request; keep only the first line in the report. + std::string reason = error.what(); + if (const auto newline = reason.find('\n'); newline != std::string::npos) reason.resize(newline); + char where[96]; + if (dump.empty()) std::snprintf(where, sizeof(where), "compute shader 0x%llx: ", static_cast(address)); + else std::snprintf(where, sizeof(where), "compute shader 0x%llx (%s): ", static_cast(address), dump.c_str()); + throw std::runtime_error(where + reason); + } + recompileMs += elapsed(); + if (!noDispatchCache) { + // The captured regions, merged into spans when close, are watched for writes from now + // on; bytes that already differ show a write racing the capture, and nothing is kept. + DispatchEntry fresh{shaderMemory, captured, compiledResult, {}, 0, it->second}; + for (const auto& region : captured) { + const auto begin = region.guestAddress; + const auto end = begin + region.bytes.size(); + if (!fresh.spans.empty() && begin <= fresh.spans.back().first + fresh.spans.back().second + 65536) { + auto& last = fresh.spans.back(); + last.second = static_cast(std::max(last.first + last.second, end) - last.first); + } else { + fresh.spans.emplace_back(begin, static_cast(end - begin)); + } + } + std::uint64_t generation = 0; + for (const auto& [begin, bytes] : fresh.spans) generation = std::max(generation, GuestMemory::CollectWrites(begin, bytes)); + bool stable = generation != 0; + const GuestMemory::ReadSiteScope site(GuestMemory::ReadSite::DispatchCache); + for (const auto& region : captured) { + if (!stable) break; + stable = GuestMemory::EqualsCommitted(region.guestAddress, region.bytes); + } + if (stable) { + fresh.generation = generation; + std::lock_guard cacheLock(dispatchCacheMutex); + // Another worker may have inserted the key meanwhile; its entry is as good. + const auto [inserted, isNew] = dispatchCache.try_emplace(key, std::move(fresh)); + if (isNew) { + dispatchOrder.push_front(key); + inserted->second.order = dispatchOrder.begin(); + } + // Debug aid: APS5_NO_DISPATCH_LRU=1 clears the cache when it fills, as before. + static const bool lru = std::getenv("APS5_NO_DISPATCH_LRU") == nullptr; + if (dispatchCache.size() > DispatchCacheEntries) { + if (lru) { + dispatchCache.erase(dispatchOrder.back()); + dispatchOrder.pop_back(); + ++dispatchCacheEvictions; + } else { + dispatchCacheEvictions += dispatchCache.size(); + dispatchCache.clear(); + dispatchOrder.clear(); + } + } + } + } + } + // A label this queue still has queued over a captured region: recorded now, and the dispatch + // starts over (the cache entry just validated or inserted fails its next validation, the + // label store marks the tracker). The queue is empty after, so this happens once. + if (recordQueuedLabelsAfterCapture(submission.queue, captured)) { + dispatch(queue, packet, submission, indirectArguments); + return; + } + const auto& compiled = *compiledResult; std::vector snapshots; for (const auto& region : captured) snapshots.push_back({region.guestAddress, region.bytes}); - timing.Mark("snapshots"); - device->Dispatch(compiled, packet[1], packet[2], packet[3], snapshots); - timing.Mark("dispatch_and_resource_release"); + std::array groups{packet[1], packet[2], packet[3]}; + if (indirectArguments == 0 && (packet[4] & 0x20u) != 0) { + // USE_THREAD_DIMENSIONS: the packet counts threads; launch enough whole groups to cover them. + for (std::uint32_t axis = 0; axis < 3; ++axis) { + const auto threads = std::max(readRegister(queue.shader, 0x207 + axis) & 0xffffu, 1u); + groups[axis] = (groups[axis] + threads - 1) / threads; + } + } + static const bool traceIo = std::getenv("APS5_TRACE_DISPATCH_IO") != nullptr; + if (traceIo) { + // APS5_TRACE_DISPATCH_IO=2 also lists the user data. + std::string words; + if (std::getenv("APS5_TRACE_DISPATCH_IO")[0] == '2') { + for (const auto word : userData) { + char text[12]; + std::snprintf(text, sizeof(text), " %08x", word); + words += text; + } + } + std::fprintf(stderr, "[dispatch-io] shader 0x%llx%s\n", static_cast(address), words.c_str()); + } + // A failure of the build or the device work is reported against the shader. + const auto rethrow = [&](const std::exception& error) { + char where[64]; + std::snprintf(where, sizeof(where), "compute shader 0x%llx: ", static_cast(address)); + throw std::runtime_error(where + std::string(error.what())); + }; + // Stage A of the resource build (copies, buffers, the descriptor set) runs before the + // lock, which then covers only stage B (texture lookups, imports, descriptor writes) and + // the record: a compute queue's build no longer stalls the graphics queue for its whole + // resources phase. Null when the resource cache may serve this dispatch, or with + // APS5_LOCKED_BUILD=1 (the whole build under the lock as before). + std::shared_ptr prepared; + try { + prepared = localDevice->PrepareDispatch(compiled, snapshots); + } catch (const std::exception& error) { + rethrow(error); + } + GuestMemory::TagGpuLockSite(indirectArguments != 0 ? GuestMemory::GpuLockSite::Indirect : GuestMemory::GpuLockSite::Dispatch); + std::lock_guard gpuLock(GuestMemory::GpuMutex()); + // This queue's labels first (queue order), and their batch out before the dispatch. Not + // under the shader's name: a label that fails to record is not a shader failure. + recordLabelsForPacket(localDevice.get(), submission.queue); + try { + if (indirectArguments != 0) { + const auto outcome = localDevice->DispatchIndirect(compiled, indirectArguments, snapshots, address, std::move(prepared)); + countIndirect(outcome.cpuReason, outcome.argumentReadMs); + } else { + localDevice->Dispatch(compiled, groups[0], groups[1], groups[2], snapshots, address, std::move(prepared)); + } + } catch (const std::exception& error) { + rethrow(error); + } + deviceMs += elapsed(); + if (profile && ++dispatches % 100 == 0) std::fprintf(stderr, "[gpu] %llu dispatches (%llu dispatch cache hits, %llu evictions, %llu recompile cache hits): capture %.1f s, cache key %.1f s, recompile %.1f s, device %.1f s\n", static_cast(dispatches), static_cast(dispatchCacheHits), static_cast(dispatchCacheEvictions), static_cast(cacheHits), captureMs / 1000, keyMs / 1000, recompileMs / 1000, deviceMs / 1000); + } + + // DISPATCH_INDIRECT: the group counts are three dwords a shader usually wrote, and reading them + // on the CPU waited for that shader (the flush hook syncs on its batch) before this dispatch could + // even be prepared. The GPU reads them in place instead (VulkanDevice::DispatchIndirect, which + // still falls back to a CPU read when it cannot see the current bytes); the CPU path stays for + // thread-dimension initiators (the counts are rounded to groups here) and the fill HLE (needs the + // count, decided in dispatch once the code is known). Debug aid: APS5_NO_GPU_INDIRECT=1 reads + // every count on the CPU as before. + void dispatchIndirect(QueueState& queue, std::span packet, const Submission& submission) { + static const bool gpuIndirect = std::getenv("APS5_NO_GPU_INDIRECT") == nullptr; + const auto arguments = Pm4::DispatchArgumentAddress(packet, queue); + const auto initiator = packet.back(); + int path = IndirectGpu; + if (!gpuIndirect) path = IndirectDisabled; + else if ((initiator & 0x20u) != 0) path = IndirectThreadDimensions; + else if (arguments % 4 != 0) path = IndirectMisaligned; + if (path != IndirectGpu) { + recordQueuedLabelsBeforeRead(submission.queue); + const auto readStart = std::chrono::steady_clock::now(); + const auto direct = Pm4::ReadDispatchArguments(arguments, initiator); + countIndirect(path, std::chrono::duration(std::chrono::steady_clock::now() - readStart).count()); + dispatch(queue, direct, submission); + return; + } + // A DISPATCH_DIRECT shape with the counts unknown: dispatch reads only the initiator from it. + const std::array unresolved{0xc0031500u, 0, 0, 0, initiator}; + dispatch(queue, unresolved, submission, arguments); } void draw(QueueState& queue, std::span packet, const Submission& submission) { PerformanceTimer timing("Driver.Draw"); auto drawParameters = Pm4::ResolveDraw(packet, queue); if (!drawParameters.indexed && (drawParameters.indexCount == 0 || drawParameters.instanceCount == 0)) return; + // Depth/stencil-only passes (no color writes, no pixel shader) have no effect without depth + // targets, which are not emulated. + { + const auto targetMask = queue.context.find(0x8e); + const auto shaderMask = queue.context.find(0x8f); + const bool colorWrites = targetMask != queue.context.end() && shaderMask != queue.context.end() && (targetMask->second & shaderMask->second) != 0; + if (!colorWrites && !queue.shader.contains(0x8)) return; + } const auto graphics = Graphics::DecodeState(queue); struct Program { ShaderRecompiler::ShaderBinary binary; @@ -458,13 +1308,32 @@ private: linked.push_back({roles[i], program.binary, program.userDataBase, program.firstUserSgpr, program.userData}); } timing.Mark("prepare"); - std::lock_guard gpuLock(gpuMutex); - timing.Mark("gpu_mutex_wait"); - if (device == nullptr) device = std::make_shared(); + // As in dispatch, the capture, the recompiles and the rect-list shaders run without the + // GpuMutex (guest memory is read through the self-locking flush hook, the recompiler has its + // own locks); only the device creation and the device Draw take it, so the compute workers + // do not pile up behind a draw's whole preparation. The device is held by this shared_ptr + // meanwhile (DevicePointer's arrow form is for holders of the mutex only). Debug aid: + // APS5_LOCKED_DRAW_PREPARE=1 holds the mutex across the preparation as before. + static const bool lockedPrepare = std::getenv("APS5_LOCKED_DRAW_PREPARE") != nullptr; + std::unique_lock gpuLock(GuestMemory::GpuMutex(), std::defer_lock); + std::shared_ptr localDevice; + if (lockedPrepare) { + // The device is created under the lock already held: one "draw" acquisition per draw, + // as in the baseline, so the [lock] site counts stay comparable. + GuestMemory::TagGpuLockSite(GuestMemory::GpuLockSite::Draw); + gpuLock.lock(); + timing.Mark("gpu_mutex_wait"); + if (device == nullptr) device = std::make_shared(); + localDevice = device; + // This queue's labels first (queue order), as at the unlocked path's acquisition below. + recordLabelsForPacket(localDevice.get(), submission.queue); + } else if ((localDevice = device.load()) == nullptr) { + GuestMemory::TagGpuLockSite(GuestMemory::GpuLockSite::Draw); + std::lock_guard createLock(GuestMemory::GpuMutex()); + if (device == nullptr) device = std::make_shared(); + localDevice = device; + } timing.Mark("device_setup"); - const GuestMemory::MemoryAccessScope memoryScope(device.get(), [](void* context, std::uint64_t address, std::size_t bytes, bool writable) { - static_cast(context)->ResolveMemory(address, bytes, writable); - }); ShaderMemory shaderMemory(memory); std::vector results; std::vector stages; @@ -478,18 +1347,40 @@ private: ShaderRecompiler::RecompileRequest request{ program.binary, {waveSize, program.firstUserSgpr, program.userData, std::nullopt, program.binary.stage == Stage::Fragment ? std::optional(pixel) : std::nullopt, program.binary.stage == Stage::Fragment ? std::nullopt : std::optional(Graphics::DecodeVertexStageInfo(program.binary.header, program.binary.headerAddress, program.userData)), memory}, - device->Target(), + localDevice->Target(), {0, 0, pushCursorBytes, Graphics::PipelinePushConstantBytes - pushCursorBytes}, ShaderRecompiler::GraphicsCompileContext{program.firstUserSgpr, linked, graphics.stages.mesh, graphics.stages.tessellation, {drawParameters.indexAddress, drawParameters.indexCount, drawParameters.indexSize, drawParameters.instanceCount}} }; - PerformanceTimer shaderTiming("Driver.GraphicsShader"); - shaderMemory.Capture(request); - shaderTiming.Mark("memory_capture"); + const auto capture = shaderMemory.Capture(request); memory = shaderMemory.Regions(); request.context.memory = memory; - shaderTiming.Mark("request_memory"); - results.push_back(ShaderRecompiler::Recompile(request)); - shaderTiming.Mark(results.back().cacheHit ? "cache_hit" : "compile"); + // Debug aid: APS5_DUMP_DRAW_SHADERS= saves the requests of draws into that + // target as shader_
.req for agc_shader_replay. + static const std::uint64_t dumpTarget = [] { const char* text = std::getenv("APS5_DUMP_DRAW_SHADERS"); return text ? std::strtoull(text, nullptr, 16) : 0ull; }(); + if (dumpTarget != 0) { + // Match the first enabled target or the raw slot-0 base, which skip reports name. + const auto slot0 = (static_cast(readRegister(queue.context, 0x390)) << 40u) | (static_cast(readRegister(queue.context, 0x318)) << 8u); + if ((graphics.hasColorTarget && graphics.color.address == dumpTarget) || slot0 == dumpTarget) static_cast(dumpRequest(program.binary.codeAddress, request)); + } + // Debug aid: APS5_DUMP_DRAW_SLOT1= saves the requests and registers of draws whose + // second color target (slot 1) is at that address, as shader_
.req and draw_slot1.regs. + static const std::uint64_t dumpSlot1 = [] { const char* text = std::getenv("APS5_DUMP_DRAW_SLOT1"); return text ? std::strtoull(text, nullptr, 16) : 0ull; }(); + if (dumpSlot1 != 0) { + const auto value = [&](std::uint32_t offset) -> std::uint64_t { const auto it = queue.context.find(offset); return it == queue.context.end() ? 0u : it->second; }; + const auto slot1 = (value(0x391) << 40u) | (value(0x327) << 8u); + if (slot1 == dumpSlot1) { + static_cast(dumpRequest(program.binary.codeAddress, request)); + if (std::FILE* file = std::fopen("draw_slot1.regs", "w")) { + for (const auto& [offset, value] : queue.context) std::fprintf(file, "context %x %08x\n", offset, value); + for (const auto& [offset, value] : queue.userConfig) std::fprintf(file, "uconfig %x %08x\n", offset, value); + for (const auto& [offset, value] : queue.shader) std::fprintf(file, "shader %x %08x\n", offset, value); + std::fclose(file); + } + } + } + // As in dispatch: the recompile reuses the capture (APS5_NO_CAPTURE_REUSE=1 restores). + static const bool reuseCapture = std::getenv("APS5_NO_CAPTURE_REUSE") == nullptr; + results.push_back(reuseCapture ? ShaderRecompiler::Recompile(request, *capture) : ShaderRecompiler::Recompile(request)); const auto& result = results.back(); if (!drawParameters.indexed && i == 0) { const auto offsetValue = [&](std::int32_t sgpr) { @@ -505,10 +1396,16 @@ private: stages.push_back({program.binary.stage, &result, result.pushConstants.empty() ? 0u : pushCursorBytes}); pushCursorBytes += static_cast(result.pushConstants.size()); } - timing.Mark("shaders"); + timing.Mark("shader_compile_and_link"); + // As in dispatch: a label this queue still has queued over a captured region is recorded + // now and the draw starts over (nothing queued on the locked-prepare path, recorded above). + if (recordQueuedLabelsAfterCapture(submission.queue, memory)) { + draw(queue, packet, submission); + return; + } if (graphics.rectList) { require(stages.size() == 2, "rect-list requires vertex and fragment programs"); - auto rectangle = ShaderRecompiler::BuildRectListShaders(results[0], results[1], device->Target()); + auto rectangle = ShaderRecompiler::BuildRectListShaders(results[0], results[1], localDevice->Target()); results.push_back(std::move(rectangle.control)); results.push_back(std::move(rectangle.evaluation)); stages.insert(stages.begin() + 1, {{Stage::TessellationControl, &results[2], 0}, {Stage::TessellationEvaluation, &results[3], 0}}); @@ -516,151 +1413,1070 @@ private: std::vector snapshots; for (const auto& region : memory) snapshots.push_back({region.guestAddress, region.bytes}); timing.Mark("post_compile_prepare"); - device->EnqueueDraw(graphics, drawParameters, stages, snapshots); + if (!gpuLock.owns_lock()) { + GuestMemory::TagGpuLockSite(GuestMemory::GpuLockSite::Draw); + gpuLock.lock(); + timing.Mark("gpu_mutex_wait"); + // Present replaces a windowless device by a windowed one (after WaitIdle) while draws + // may be preparing unlocked: the draw then records on the current device, whose + // recorder Recorder::Active() already is, rather than mixing the old device's images + // and pipelines into the new recorder's batch. The preparation is device-independent + // (SPIR-V for the same physical device; pipelines are created inside Draw). + if (auto current = device.load(); current != nullptr && current != localDevice) { + static std::atomic replaced{0}; + std::fprintf(stderr, "[draw] device replaced during unlocked preparation (%llu)\n", static_cast(++replaced)); + localDevice = std::move(current); + } + // This queue's labels first (queue order), and their batch out before the draw. + recordLabelsForPacket(localDevice.get(), submission.queue); + } + localDevice->Draw(graphics, drawParameters, stages, snapshots); timing.Mark("draw_and_resource_release"); } - void includeSubmission(const Submission& submission, bool firstSegment) { - if (frameTiming == nullptr) { - require(frameSerial != std::numeric_limits::max(), "frame serial overflow"); - frameTiming = std::make_shared(++frameSerial); + // Serials complete out of order when a wait runs other queues' work; `completed` stays the + // highest serial below which everything has finished. Caller holds `mutex`. + void markCompleted(std::uint64_t serial) { + completedOutOfOrder.insert(serial); + while (!completedOutOfOrder.empty() && *completedOutOfOrder.begin() == completed + 1) { + completed = *completedOutOfOrder.begin(); + completedOutOfOrder.erase(completedOutOfOrder.begin()); } - const auto dequeued = firstSegment ? submission.dequeued : FrameTiming::Clock::now(); - frameTiming->IncludeSubmission(submission.serial, submission.received, submission.enqueued, dequeued, firstSegment); - if (firstSegment) { - frameTiming->Add(frameTiming->Get("Submission", "copy"), submission.copied - submission.received, submission.commands.size() * sizeof(std::uint32_t)); - frameTiming->Add(frameTiming->Get("Submission", "validate"), submission.validated - submission.copied); - frameTiming->Add(frameTiming->Get("Submission", "reserve_enqueue"), submission.enqueued - submission.validated); + } + + // Debug aid: APS5_TRACE_LABEL= logs every GPU write to or wait on memory within 256 + // bytes of that address. + static void traceLabel(std::span packet, std::uint32_t queue) { + static const std::uint64_t watched = [] { + const char* value = std::getenv("APS5_TRACE_LABEL"); + return value ? std::strtoull(value, nullptr, 16) : 0ull; + }(); + const auto opcode = (packet[0] >> 8u) & 0xffu; + std::uint64_t target = 0; + const char* kind = nullptr; + if (opcode == 0x49 && packet.size() >= 7) { target = packet[3] | (static_cast(packet[4]) << 32u); kind = "RELEASE_MEM"; } + else if (opcode == 0x37 && packet.size() >= 5) { target = packet[2] | (static_cast(packet[3]) << 32u); kind = "WRITE_DATA"; } + else if ((opcode == 0x3c || opcode == 0x93) && packet.size() >= 7) { target = packet[2] | (static_cast(packet[3]) << 32u); kind = "WAIT_REG_MEM"; } + else if (opcode == 0x40 && packet.size() >= 6) { target = packet[4] | (static_cast(packet[5]) << 32u); kind = "COPY_DATA"; } + std::uint64_t length = 4; + if (opcode == 0x50 && packet.size() >= 7) { + target = packet[4] | (static_cast(packet[5]) << 32u); + length = packet[6] & 0x3ffffffu; + kind = "DMA_DATA"; } + if (kind == nullptr) return; + if (std::string_view(kind) != "WAIT_REG_MEM") { + static std::mutex historyMutex; + std::lock_guard lock(historyMutex); + auto& entry = writeHistory()[writeCursor()++ % writeHistory().size()]; + entry = {target, length, queue, opcode}; + } + if (watched == 0 || target + length + 0x100 < watched || target > watched + 0x100) return; + std::fprintf(stderr, "[label] %lld ms queue 0x%x %s 0x%llx:", static_cast(std::chrono::duration_cast(std::chrono::steady_clock::now().time_since_epoch()).count()), queue, kind, static_cast(target)); + for (std::size_t i = 1; i < packet.size() && i < 9; ++i) std::fprintf(stderr, " %08x", packet[i]); + std::fprintf(stderr, "\n"); + } + + struct WriteRecord { + std::uint64_t target; + std::uint64_t length; + std::uint32_t queue; + std::uint32_t opcode; + }; + static std::array& writeHistory() { + static std::array history{}; + return history; + } + static std::size_t& writeCursor() { + static std::size_t cursor = 0; + return cursor; + } + + static int WaitTimeoutMs() { + static const int value = [] { const char* text = std::getenv("APS5_GPU_WAIT_TIMEOUT_MS"); return text ? std::atoi(text) : 1000; }(); + return value; + } + + // How WAIT_REG_MEM packets were satisfied, per queue worker (reported in its [packets] line): + // already true when reached; from the recorder's pending-label table (and how many of those by + // a label of this same queue); by polling memory (the GPU, another queue or the CPU wrote it); + // timed out. Plus the submits and reaps the poll loop made on behalf of the producer. + // fromRecorderUnlocked: table hits (at entry or polling) that took no GPU mutex; entriesUnlocked: + // waits that went to polling without the GPU mutex (nothing recorded writes the range); + // entryTriesFailed: overlapping entries whose try of the mutex failed and went to polling + // instead of waiting for it (see waitMemory). + struct WaitOutcomes { + std::uint64_t atEntry = 0, fromRecorder = 0, fromRecorderSameQueue = 0, fromRecorderPolling = 0, polled = 0, timedOut = 0, pollSubmits = 0, pollReaps = 0; + std::uint64_t fromRecorderUnlocked = 0, entriesUnlocked = 0, entryTriesFailed = 0; + }; + static WaitOutcomes& waitOutcomes() { + static thread_local WaitOutcomes outcomes; + return outcomes; + } + + // Collect-epoch bumps by ordering point, per queue worker (reported in its [packets] line, see + // GuestMemory::BumpCollectEpoch): the start of a submission, a wait the CPU may have satisfied, + // a drain, a reap outside a submission, and (APS5_PACKET_EPOCH=1 only) every packet. + struct EpochBumps { + std::uint64_t submissions = 0, waits = 0, drains = 0, reaps = 0, packets = 0; + }; + static EpochBumps& epochBumps() { + static thread_local EpochBumps bumps; + return bumps; + } + static void bumpEpoch(std::uint64_t EpochBumps::*counter) { + GuestMemory::BumpCollectEpoch(); + ++(epochBumps().*counter); + } + static bool PacketEpoch() { + static const bool packet = std::getenv("APS5_PACKET_EPOCH") != nullptr; + return packet; + } + + // Labels recorded on the GPU are not submitted one by one (see VulkanDevice::WriteLabelOnGpu): + // the open batch goes out at the next non-label packet of the recording worker, and at the + // latest APS5_LABEL_FLUSH_US (default 250) after its first label, checked by every worker + // between packets and by waiting workers inside their poll loop, so a game thread polling the + // label is never left waiting on an unsubmitted batch for long. The deadline is read lock-free. + static std::chrono::microseconds LabelFlushDeadline() { + static const std::chrono::microseconds value = [] { + const char* text = std::getenv("APS5_LABEL_FLUSH_US"); + return std::chrono::microseconds(text ? std::atoi(text) : 250); + }(); + return value; + } + static bool LabelFlushDue() { + const auto since = Graphics::Recorder::PendingLabelSince(); + return since.has_value() && std::chrono::steady_clock::now() - *since >= LabelFlushDeadline(); + } + // Submits the open batch after this many dispatches/draws even without a label, so the GPU + // starts on them while the CPU records the rest (APS5_BATCH_CAP, default 16; 0 disables). + static std::uint64_t BatchCap() { + static const std::uint64_t value = [] { + const char* text = std::getenv("APS5_BATCH_CAP"); + return text ? std::strtoull(text, nullptr, 10) : 16ull; + }(); + return value; + } + + // Records this worker's deferred labels (see DeferredLabels) under GuestMemory::GpuMutex, in + // queue order, each as VulkanDevice::WriteLabelOnGpu would have at its packet: on the GPU, as a + // completion action, or by a CPU store when nothing is recorded (the store is ordered already). + // The stamp is taken here, right before the label is noted, so it orders after every submission + // the game made before this point (a wait received later never trusts the label). Only an + // unusable recorder (reason 4) can still ask for a drain here (APS5_DRAIN_COMPLETION_LABELS=1 + // disables deferral, see DeferLabels); it waits idle under the mutex, which is cheap then. + // A throw (lost device) leaves nothing queued: the worker fails with the packet, or, when the + // record ran inside a tolerated dispatch or draw, that packet is skipped; the dropped group is + // reported either way, so a WAIT_REG_MEM timeout later is not the first sign of it. + void recordDeferredLabels(VulkanDevice* localDevice, std::uint32_t queue) { + auto& deferred = deferredLabels(); + if (deferred.labels.empty()) return; + struct Clear { + std::vector& labels; + std::uint32_t queue; + ~Clear() { + if (std::uncaught_exceptions() != 0) std::fprintf(stderr, "[gpu] queue 0x%x dropped %zu queued labels: their record failed\n", queue, labels.size()); + labels.clear(); + } + } clear{deferred.labels, queue}; + bool first = true; + for (const auto& label : deferred.labels) { + const auto bytes = std::span(label.bytes).first(label.size); + const int reason = localDevice != nullptr ? localDevice->WriteLabelOnGpu(label.address, bytes, ++eventSerial, queue, first) : 4; + first = false; + countLabelOutcome(reason); + if (reason == 0 || reason == 5 || reason == 6) continue; + if (reason != 1 && localDevice != nullptr) localDevice->WaitIdle(); + GuestMemory::Write(label.address, bytes, 4); + } + ++labelGroups; + static const bool profile = std::getenv("APS5_PROFILE_DRAW") != nullptr; + // The [sync] line every 10 s (checked every 64 groups; drains alone are too rare now). + if (profile && (labelGroups.load(std::memory_order_relaxed) & 63u) == 0 && std::chrono::steady_clock::now() - lastSyncReport > std::chrono::seconds(10)) { + lastSyncReport = std::chrono::steady_clock::now(); + reportSync(); + } + } + + // The first thing a packet does under its own GpuMutex hold (dispatch, draw, flip: see + // PacketLocksItself), before its work is recorded: this worker's deferred labels go in, in + // queue order ahead of that work, and the open batch is submitted when it carries labels, + // exactly what flushBetweenPackets did under an acquisition of its own before the packet + // (APS5_LABEL_OWN_LOCK=1 restores that). One acquisition instead of two: the label one was + // queue 0's first after a run of unlocked packets and absorbed the compute queues' whole + // critical sections ([lock] site 'label'), after which the packet's own came uncontended. + // Returns whether the batch was submitted (the flip then needs no submit of its own). + bool recordLabelsForPacket(VulkanDevice* localDevice, std::uint32_t queue) { + static const bool packetFlush = std::getenv("APS5_NO_LABEL_PACKET_FLUSH") == nullptr; + // APS5_PROFILE_DRAW: what this costs inside the packet's own hold ([labels] line): the + // record (with queue 0's reap of finished batches before the first label) and the submit + // (with another reap on queue 0), both of which run completions, so a dispatch's or + // indirect dispatch's hold ([lock] line) is partly this. + static const bool profile = std::getenv("APS5_PROFILE_DRAW") != nullptr; + const auto start = profile ? std::chrono::steady_clock::now() : std::chrono::steady_clock::time_point{}; + if (!deferredLabels().labels.empty()) { + recordDeferredLabels(localDevice, queue); + ++packetLockRecords; + if (profile) packetLockRecordUs += static_cast(std::chrono::duration_cast(std::chrono::steady_clock::now() - start).count()); + } + if (localDevice == nullptr || !packetFlush || !Graphics::Recorder::PendingLabelSince().has_value()) return false; + // As in flushBetweenPackets: a compute worker submits without reaping. + const auto submitStart = profile ? std::chrono::steady_clock::now() : std::chrono::steady_clock::time_point{}; + localDevice->SubmitRecorded(queue == 0); + ++packetLockSubmits; + if (profile) packetLockSubmitUs += static_cast(std::chrono::duration_cast(std::chrono::steady_clock::now() - submitStart).count()); + return true; + } + + // Before a packet's CPU read of guest memory outside any lock (the group counts of a + // DISPATCH_INDIRECT resolved on the CPU): a label this queue still has queued may write those + // bytes and the flush hook only knows recorded stores, so the queued labels are recorded first. + // Rare (a label is normally followed by a wait or a dispatch that records it) and cheap. + void recordQueuedLabelsBeforeRead(std::uint32_t queue) { + if (deferredLabels().labels.empty()) return; + GuestMemory::TagGpuLockSite(GuestMemory::GpuLockSite::Label); + std::lock_guard gpuLock(GuestMemory::GpuMutex()); + const auto localDevice = device.load(); + recordDeferredLabels(localDevice.get(), queue); + } + + // After a self-locking packet's unlocked capture (dispatch, draw), when this worker still has + // labels queued (the try in flushBetweenPackets failed): a queued label whose bytes lie in a + // captured region (a fence value in SRT-chased data, a patched constant) was read before its + // write, and the hook could not sync on it, so the labels are recorded outright now and the + // caller redoes the packet from the top (returns true; once, the queue is empty after). With no + // overlap the labels are recorded if the mutex happens to be free, so a poller of the label + // does not wait for the rest of this prologue (stage A takes milliseconds for a BDA build); + // otherwise the packet's own acquisition records them as planned. The check itself is a few + // dozen compares: a group holds a handful of labels, a capture a few dozen regions. + bool recordQueuedLabelsAfterCapture(std::uint32_t queue, std::span regions) { + const auto& labels = deferredLabels().labels; + if (labels.empty()) return false; + for (const auto& label : labels) { + for (const auto& region : regions) { + if (label.address < region.guestAddress + region.bytes.size() && region.guestAddress < label.address + label.size) { + recordQueuedLabelsBeforeRead(queue); + ++captureRetries; + return true; + } + } + } + GuestMemory::TagGpuLockSite(GuestMemory::GpuLockSite::Label); + std::unique_lock gpuLock(GuestMemory::GpuMutex(), std::defer_lock); + if (!gpuLock.try_lock()) return false; + const auto localDevice = device.load(); + recordDeferredLabels(localDevice.get(), queue); + ++captureTryRecords; + return false; + } + + // Between packets: records this worker's deferred labels before a packet that needs them (or + // once their deadline passed), submits the open batch when a label is pending (at once on a + // non-label packet, else once the deadline passed) or the batch holds enough work, and reaps + // finished batches whose completion actions still hold labels. The mutex is taken outright when + // the packet needs the labels or a deadline passed, and by the graphics worker for a pending + // submit; otherwise it is only tried (a register-only packet must not queue behind a 2-4 ms + // dispatch for this) and the next packet retries. A packet that takes the mutex itself + // (PacketLocksItself) gets both the record and the submit at its own acquisition + // (recordLabelsForPacket), so for it the mutex is only tried here as well: a try that succeeds + // records at once (the packet's unlocked prologue then reads through a hook that knows the + // labels, as before), one that fails leaves the group to the packet. Only a deadline still + // locks outright before such a packet: its prologue may take milliseconds, and pollers of the + // label are promised the deadline. APS5_NO_LABEL_PACKET_FLUSH=1 leaves recorded labels pending + // until the deadline (the batch then absorbs the following work); APS5_LABEL_OWN_LOCK=1 takes + // the mutex outright before every packet that needs the labels, as before. + void flushBetweenPackets(std::uint32_t queue, std::uint32_t header, bool labelPacket) { + static const bool packetFlush = std::getenv("APS5_NO_LABEL_PACKET_FLUSH") == nullptr; + static const bool ownLock = std::getenv("APS5_LABEL_OWN_LOCK") != nullptr; + static std::atomic packetFlushes{0}, deadlineFlushes{0}, capFlushes{0}, boundaryReaps{0}; + auto& deferred = deferredLabels(); + const bool queued = !deferred.labels.empty(); + const bool selfLocking = !ownLock && PacketLocksItself(header); + bool submit = false, record = false, mustLock = false, deferredDue = false, pendingDue = false; + const auto pending = Graphics::Recorder::PendingLabelSince(); + // The clock is read only while some label waits (every packet passes here). + const auto now = queued || pending.has_value() ? std::chrono::steady_clock::now() : std::chrono::steady_clock::time_point{}; + if (queued) { + if (!labelPacket) { + record = true; + mustLock = NeedsRecordedLabels(header) && !selfLocking; + } + if (now - deferred.since >= LabelFlushDeadline()) record = mustLock = deferredDue = true; + } + if (pending.has_value()) { + if (!labelPacket && packetFlush) submit = true; + if (now - *pending >= LabelFlushDeadline()) submit = pendingDue = true; + } + const bool capped = !submit && !record && BatchCap() != 0 && Graphics::Recorder::RecordedWorkSinceSubmit() >= BatchCap(); + const bool reap = !submit && !record && !capped && Graphics::Recorder::PendingCompletionLabels() != 0; + if (!submit && !record && !capped && !reap) return; + std::unique_lock gpuLock(GuestMemory::GpuMutex(), std::defer_lock); + // The site names the hold too (a try's hold is charged to it; a reap alone shows as 'try'). + if (record) GuestMemory::TagGpuLockSite(GuestMemory::GpuLockSite::Label); + else if (submit || capped) GuestMemory::TagGpuLockSite(GuestMemory::GpuLockSite::Flush); + // A reap alone is never worth waiting for the mutex (the next packet retries); neither is a + // submit the self-locking packet makes itself, unless the label's deadline passed. + if (mustLock || (queue == 0 && (capped || (submit && (!selfLocking || pendingDue))))) { + gpuLock.lock(); + } else if (!gpuLock.try_lock()) { + if (selfLocking && record) ++packetLockDeferred; + else if (selfLocking && submit) ++packetSubmitDeferred; + return; + } + const auto localDevice = device.load(); + if (record) { + recordDeferredLabels(localDevice.get(), queue); + // The group goes out with the packet flush (or at once past the deadline), as a label + // recorded at its own packet would have. + if ((!labelPacket && packetFlush) || deferredDue) submit = true; + } + if (localDevice == nullptr) return; + if (submit || capped) { + // Rechecked under the lock: another worker may have submitted meanwhile. A compute + // worker only submits (no reap: a write-back's sync would hold the mutex against queue 0). + if (!Graphics::Recorder::PendingLabelSince().has_value() && (BatchCap() == 0 || Graphics::Recorder::RecordedWorkSinceSubmit() < BatchCap())) return; + localDevice->SubmitRecorded(queue == 0); + if (capped) ++capFlushes; + else if (!labelPacket && packetFlush) ++packetFlushes; + else ++deadlineFlushes; + } else if (reap) { + localDevice->ReapRecorded(); + ++boundaryReaps; + } + static const bool profile = std::getenv("APS5_PROFILE_DRAW") != nullptr; + static std::atomic total{0}; + if (profile && (++total & 4095u) == 0) std::fprintf(stderr, "[labels] batch flushes between packets: %llu at the next packet, %llu by deadline, %llu by size cap; %llu boundary reaps; at the packet's own lock: %llu label groups (%.0f ms), %llu submits (%.0f ms incl. queue 0's reaps), left to it by a failed try: %llu groups, %llu submits; after a capture: %llu groups recorded by a try, %llu packets redone for a queued label over the capture; %llu suspend points\n", static_cast(packetFlushes.load()), static_cast(deadlineFlushes.load()), static_cast(capFlushes.load()), static_cast(boundaryReaps.load()), static_cast(packetLockRecords.load()), packetLockRecordUs.load() / 1000.0, static_cast(packetLockSubmits.load()), packetLockSubmitUs.load() / 1000.0, static_cast(packetLockDeferred.load()), static_cast(packetSubmitDeferred.load()), static_cast(captureTryRecords.load()), static_cast(captureRetries.load()), static_cast(suspendPoints.load())); + } + + // WAIT_REG_MEM: another queue or the CPU produces the value; the other queues run on their own threads. + void waitMemory(std::span packet, std::uint32_t queue, const PacketHistory& context, std::uint64_t received) { + // The timeout counts from the last sign of GPU progress: while any queue is executing packets, + // the producer may still be on its way (shader compiles alone take hundreds of milliseconds). + auto start = std::chrono::steady_clock::now(); + auto lastDone = packetsDone.load(); + // Names this worker's thread in the [lock] GpuMutex wait report. + GuestMemory::TagGpuLockThread(queue); + auto& outcomes = waitOutcomes(); + const bool wide = ((packet[0] >> 8u) & 0xffu) == 0x93u; + const std::uint64_t awaited = packet[2] | (static_cast(packet[3]) << 32u); + const std::size_t awaitedBytes = wide ? 8 : 4; + static const bool traceGpu = std::getenv("APS5_TRACE_GPU") != nullptr; + if (traceGpu) std::fprintf(stderr, "[gpu] %.1f queue 0x%x waits 0x%llx == 0x%x (now 0x%x)\n", TraceMs(), queue, static_cast(packet[2] | (static_cast(packet[3]) << 32u)), packet[4], + *reinterpret_cast(packet[2] | (static_cast(packet[3]) << 32u))); + struct WaitTrace { + bool enabled; std::uint32_t queue; std::uint64_t address; std::chrono::steady_clock::time_point begin; + ~WaitTrace() { + if (!enabled) return; + const auto ms = std::chrono::duration(std::chrono::steady_clock::now() - begin).count(); + if (ms >= 0.5) std::fprintf(stderr, "[gpu] %.1f queue 0x%x wait on 0x%llx done after %.1f ms\n", TraceMs(), queue, static_cast(address), ms); + } + } waitTrace{traceGpu, queue, packet[2] | (static_cast(packet[3]) << 32u), std::chrono::steady_clock::now()}; + bool warned = false; + // The collect epoch advances once the wait is over (at entry, polled, timed out, or from a + // label another queue recorded: the CPU's writes before that queue's submission are ordered + // through it), unless a label this queue itself recorded satisfied it, which orders no CPU + // write. After the wait, never before: completions run inside pollService make memo entries + // during the wait that may predate the CPU's store of the flag. + struct EpochPoint { + bool bump = true; + ~EpochPoint() { + if (bump) bumpEpoch(&EpochBumps::waits); + } + } epochPoint; + GuestMemory::CheckRange(reinterpret_cast(awaited), awaitedBytes, awaitedBytes); + if (Pm4::WaitSatisfiedUnchecked(packet)) { + ++outcomes.atEntry; + return; + } + // Debug aids: APS5_NO_LABEL_SHORTCUT=1 never satisfies a wait from the pending-label table; + // APS5_NO_WAIT_OVERLAP_SUBMIT=1 submits the open batch at every wait, as before, instead of + // only when it writes the awaited range. + static const bool labelShortcut = std::getenv("APS5_NO_LABEL_SHORTCUT") == nullptr; + static const bool overlapSubmit = std::getenv("APS5_NO_WAIT_OVERLAP_SUBMIT") == nullptr; + // APS5_WAIT_LOCK=1 takes the GPU mutex at every wait's entry and poll check, as before. + static const bool waitLock = std::getenv("APS5_WAIT_LOCK") != nullptr; + // Read before the checks below: a write noted before the (locked or snapshot) check is seen + // by that check, one noted after it changes the generation the poll loop compares against. + std::uint64_t seenGeneration = Graphics::Recorder::WriteGeneration(); + // Without the GPU mutex first: the label table has its own small mutex, and the lock-free + // pending-write snapshot says whether any recorded work writes the range at all. Only an + // overlap needs the mutex (to submit the open batch); a miss goes straight to polling (its + // producer is the CPU, an in-flight batch, or a queue that has not recorded the label yet: + // the poll loop watches the write generation for that). The snapshot may lag the generation + // (Recorder::noteWrite bumps it before publishing, and NoteLabel follows the note), so a + // decision made from the snapshot never consumes a generation: an unlocked entry makes the + // poll loop's first check a locked (try_lock) one, which sees the producer's consistent view. + const bool unlocked = !waitLock && overlapSubmit; + if (unlocked) { + if (labelShortcut) { + std::uint32_t producer = 0; + if (const auto value = Graphics::Recorder::LookupLabel(awaited, awaitedBytes, received, producer); value.has_value() && Pm4::WaitComparesValue(packet, *value)) { + ++outcomes.fromRecorder; + ++outcomes.fromRecorderUnlocked; + if (producer == queue) { + ++outcomes.fromRecorderSameQueue; + epochPoint.bump = false; + } + return; + } + } + } + const bool lockedEntry = !unlocked || Graphics::Recorder::SnapshotWriteOverlaps(awaited, awaitedBytes); + if (!lockedEntry) ++outcomes.entriesUnlocked; + // An overlapping entry has one thing to do under the mutex: submit the open batch when it + // writes the range (the table lookup already ran without it). It only TRIES the mutex: the + // wait's hold is microseconds, but waiting for the mutex behind a compute queue's build cost + // queue 0 ~4 ms per entry ([lock] 'wait' site). A failed try goes to the poll loop, whose + // first service call repeats the entry's checks under a try of its own (`recheck`) and every + // later one re-tries as well, so the submit happens as soon as the mutex is free, while the + // value may arrive meanwhile (an in-flight batch, another queue, the CPU) and end the wait + // without any hold. Debug aid: APS5_WAIT_LOCK_ENTRY=1 waits for the mutex at entry as before. + static const bool blockingEntry = std::getenv("APS5_WAIT_LOCK_ENTRY") != nullptr; + bool recheck = !lockedEntry; + bool entryTryFailed = false; + if (lockedEntry) { + GuestMemory::TagGpuLockSite(GuestMemory::GpuLockSite::Wait); + std::unique_lock gpuLock(GuestMemory::GpuMutex(), std::defer_lock); + if (blockingEntry || !unlocked) { + gpuLock.lock(); + } else if (!gpuLock.try_lock()) { + ++outcomes.entryTriesFailed; + recheck = true; + entryTryFailed = true; + } + if (const auto localDevice = gpuLock.owns_lock() ? device.load() : std::shared_ptr{}) { + // A label the recorder holds for this range, recorded after the game submitted this + // packet, will store the value in order (see Recorder::NoteLabel): the wait is over + // without the GPU round trip. The label's batch is submitted by the flush rules. + std::uint32_t producer = 0; + if (labelShortcut) { + if (const auto value = localDevice->PendingLabel(awaited, awaitedBytes, received, producer); value.has_value() && Pm4::WaitComparesValue(packet, *value)) { + ++outcomes.fromRecorder; + if (producer == queue) { + ++outcomes.fromRecorderSameQueue; + epochPoint.bump = false; + } + return; + } + } + // Recorded work may write the awaited value: send it to the GPU; the poll below sees + // the store once it lands in the imported memory. Work not writing the range stays + // in the open batch (its producer is the CPU, an in-flight batch or a queue that has + // not recorded the label yet; the poll loop submits it once it does). Submit only, + // no reap: this thread must not wait for the GPU under the mutex on another queue's + // behalf (a reaped write-back can sync a later batch through the flush hook). + if (!overlapSubmit || localDevice->OpenWriteOverlaps(awaited, awaitedBytes)) localDevice->SubmitRecorded(false); + } + } + // Consumer-driven submission: the producer's label may be recorded into the open batch after + // this wait began, and the producing worker submits it only at its next packet, so the + // poller (idle anyway) watches the recorder's write generation and submits the batch itself + // when it writes the awaited range, or when any label passed its deadline. In the sleeping + // phase it also reaps finished batches when labels wait in completion actions (a wait on + // such a label converges without a reaper thread). The mutex is only tried: a failed try + // keeps the generation unconsumed and the next check retries. Returns true when the label + // table satisfied the wait meanwhile: a label recorded after this wait began (the common + // queue 0 -> compute -> queue 0 chain when queue 0 reaches its wait first) is accepted on + // the same stamp argument as at entry, sparing the GPU round trip. Every submit here is + // reap-free: a poller never waits for the GPU under the mutex on the producer's behalf. + // An unlocked entry decided from the snapshot alone (or one whose try failed): the first + // check re-does the entry's overlap test under the mutex (a note whose snapshot was not yet + // published when the entry looked must still get its batch submitted; see the entry comment). + const auto pollService = [&](bool sleeping) { + const auto generation = Graphics::Recorder::WriteGeneration(); + const bool changed = generation != seenGeneration || recheck; + const bool labelDue = LabelFlushDue(); + const bool reap = sleeping && Graphics::Recorder::PendingCompletionLabels() != 0; + if (!changed && !labelDue && !reap) return false; + if (changed && unlocked && labelShortcut) { + // The table first, without the mutex: a hit ends the wait. A miss is not final (the + // producer may be between its generation bump and its table entry or snapshot), so + // the generation is consumed only below, under a successful try_lock, where the + // table, the open batch and the snapshot are consistent (a try_lock never waits, + // so this costs no mutex wait; a failed one leaves the generation for the next check). + std::uint32_t producer = 0; + if (const auto value = Graphics::Recorder::LookupLabel(awaited, awaitedBytes, received, producer); value.has_value() && Pm4::WaitComparesValue(packet, *value)) { + ++outcomes.fromRecorderPolling; + ++outcomes.fromRecorderUnlocked; + if (producer == queue) { + ++outcomes.fromRecorderSameQueue; + epochPoint.bump = false; + } + return true; + } + } + std::unique_lock gpuLock(GuestMemory::GpuMutex(), std::try_to_lock); + if (!gpuLock.owns_lock()) return false; + const auto localDevice = device.load(); + if (localDevice == nullptr) { + seenGeneration = generation; + recheck = false; + return false; + } + bool submitted = false; + if (changed) { + seenGeneration = generation; + recheck = false; + if (labelShortcut) { + std::uint32_t producer = 0; + if (const auto value = localDevice->PendingLabel(awaited, awaitedBytes, received, producer); value.has_value() && Pm4::WaitComparesValue(packet, *value)) { + ++outcomes.fromRecorderPolling; + if (producer == queue) { + ++outcomes.fromRecorderSameQueue; + epochPoint.bump = false; + } + return true; + } + } + if (localDevice->OpenWriteOverlaps(awaited, awaitedBytes)) { + localDevice->SubmitRecorded(false); + submitted = true; + } + } + if (labelDue && !submitted && Graphics::Recorder::PendingLabelSince().has_value()) { + localDevice->SubmitRecorded(false); + submitted = true; + } + if (submitted) ++outcomes.pollSubmits; + if (reap) { + localDevice->ReapRecorded(); + ++outcomes.pollReaps; + } + return false; + }; + // A failed entry try is retried once at once, not only after the spin's first 64 pauses: + // the mutex may have freed during the entry's own lookup, and the submit a blocking entry + // made the instant it got the mutex should not wait for the loop's first service. + if (entryTryFailed && pollService(false)) return; + // The producer is usually another queue or the CPU: spin briefly with pause (the value is + // often written within microseconds; the graphics queue's waits average under a millisecond + // and it is the frame-critical thread, so it keeps its core longer than the compute queues, + // which mostly wait tens of milliseconds for it), then poll every 200 us so waiting queues + // do not burn cores. While spinning, the clock, the failure flag and the progress counters + // are read every 64 pauses only; the timeout accounting below stays on the sleeping path. + // Debug aid: APS5_NO_PAUSE_SPIN=1 yields 4000 times before sleeping as before. + static const bool pauseSpin = std::getenv("APS5_NO_PAUSE_SPIN") == nullptr; + const auto spinLimit = queue == 0 ? std::chrono::microseconds(1500) : std::chrono::microseconds(100); + const auto spinStart = std::chrono::steady_clock::now(); + bool spinning = pauseSpin; + std::uint32_t polls = 0; + while (!Pm4::WaitSatisfiedUnchecked(packet)) { + ++polls; + if (spinning) { + _mm_pause(); + if ((polls & 63u) != 0) continue; + if (std::chrono::steady_clock::now() - spinStart > spinLimit) spinning = false; + } else if (pauseSpin || polls >= 4000) { + PollSleep(); + } else { + std::this_thread::yield(); + } + CheckFailure(); + if (pollService(!spinning)) return; + if (const auto done = packetsDone.load(); done != lastDone || packetsInFlight.load() != 0) { + lastDone = done; + start = std::chrono::steady_clock::now(); + } + // Some producers (skipped shaders, interrupt-driven CPU work) are not modeled; rather than deadlock + // the queue, give up on a wait after a while and report it once. + if (!warned && std::chrono::steady_clock::now() - start > std::chrono::milliseconds(WaitTimeoutMs())) { + warned = true; + ++outcomes.timedOut; + static std::set reported; + static std::uint64_t timeouts = 0; + if (++timeouts % 20 == 0) std::fprintf(stderr, "[gpu] %llu GPU waits have timed out\n", static_cast(timeouts)); + if (!reported.insert(packet[2] | (static_cast(packet[3]) << 32u)).second) return; + std::fprintf(stderr, "[gpu] queue 0x%x WAIT_REG_MEM at 0x%llx timed out after %dms (function %u ref 0x%x mask 0x%x value 0x%x)\n", queue, + static_cast(packet[2] | (static_cast(packet[3]) << 32u)), WaitTimeoutMs(), packet[1] & 7u, packet[4], packet[5], + *reinterpret_cast(packet[2] | (static_cast(packet[3]) << 32u))); + const auto address = packet[2] | (static_cast(packet[3]) << 32u); + for (const auto& record : writeHistory()) { + if (record.length != 0 && record.target <= address && address < record.target + std::max(record.length, 4)) + std::fprintf(stderr, "[gpu] earlier write by queue 0x%x opcode 0x%x at 0x%llx+0x%llx\n", record.queue, record.opcode, static_cast(record.target), static_cast(record.length)); + } + context.Each([](const std::string& line) { std::fprintf(stderr, "[gpu] preceding packet %s\n", line.c_str()); }); + } + if (warned) return; + } + ++outcomes.polled; } void execute(const Submission& submission) { - includeSubmission(submission, true); if (submission.suspend) { - PerformanceContext timingContext(frameTiming.get()); - PerformanceTimer timing("Driver.Suspend"); - std::lock_guard gpuLock(gpuMutex); - timing.Mark("gpu_mutex_wait"); - if (device != nullptr) device->WaitIdle(); - timing.Mark("device_idle_wait"); + // A suspend point only marks where the system may suspend the title (the system, not the + // title, waits for the GPU there); this drained the device under the mutex once per + // frame, which was every 'idle' fence wait of queue 0 in the [recorder] line (~10 ms + // each, the frame's whole batch list) although nothing reads results here. Now the + // frame's batches are only submitted, this worker's queued labels first; the register + // reset that follows needs no idle GPU (recorded work keeps what it uses). + // Debug aid: APS5_SUSPEND_DRAIN=1 drains as before. + static const bool suspendDrain = std::getenv("APS5_SUSPEND_DRAIN") != nullptr; + GuestMemory::TagGpuLockSite(GuestMemory::GpuLockSite::Flush); + std::lock_guard gpuLock(GuestMemory::GpuMutex()); + ++suspendPoints; + if (const auto localDevice = device.load()) { + recordDeferredLabels(localDevice.get(), submission.queue); + if (suspendDrain) localDevice->WaitIdle(); + else localDevice->SubmitRecorded(submission.queue == 0); + } resetGraphics = true; return; } - if (submission.queue == 0 && resetGraphics) { - queues.erase(0); - resetGraphics = false; + QueueState* state = nullptr; + { + std::lock_guard lock(mutex); + if (submission.queue == 0 && resetGraphics) { + queues.erase(0); + resetGraphics = false; + } + state = &queues[submission.queue]; } - auto& queue = queues[submission.queue]; + auto& queue = *state; + static const bool traceGpu = std::getenv("APS5_TRACE_GPU") != nullptr; + if (traceGpu) std::fprintf(stderr, "[gpu] %.1f execute serial=%llu queue=0x%x dwords=%zu\n", TraceMs(), static_cast(submission.serial), submission.queue, submission.commands.size()); + // Debug aid: APS5_DUMP_QUEUE= prints the packets of that queue's first 40 submissions. + static const long dumpQueue = [] { const char* text = std::getenv("APS5_DUMP_QUEUE"); return text ? std::strtol(text, nullptr, 16) : -1L; }(); + if (static_cast(submission.queue) == dumpQueue) { + // Only this queue's thread gets here. + static int dumped = 0; + if (dumped++ < 40) { + std::string text = "[queue] submission " + std::to_string(submission.serial) + ":\n"; + for (std::size_t cursor = 0; cursor < submission.commands.size();) { + const auto header = submission.commands[cursor]; + const auto count = Pm4::PacketWords(header); + char line[200]; + int length = std::snprintf(line, sizeof(line), "[queue] %s", Pm4::Name(header).c_str()); + for (std::size_t i = 1; i < count && i < 10 && length < 180; ++i) length += std::snprintf(line + length, sizeof(line) - length, " %08x", submission.commands[cursor + i]); + text += line; + text += "\n"; + cursor += count; + } + std::fputs(text.c_str(), stderr); + } + } + PacketHistory recent{submission.commands}; + // APS5_PROFILE_DRAW: per-queue time spent handling each packet type, reported every 10 s, so + // the packets a frame's submission spends its time on are visible. + static const bool profilePackets = std::getenv("APS5_PROFILE_DRAW") != nullptr; + struct PacketProfile { + std::map> byOpcode; + std::chrono::steady_clock::time_point lastReport = std::chrono::steady_clock::now(); + std::uint64_t submissions = 0; + // Time in flushBetweenPackets (deferred label submits, reaps and, on queue 0, the mutex + // wait for them), kept apart from the packets so their buckets show their own cost. + double flushMs = 0; + }; + thread_local PacketProfile packetProfile; + ++packetProfile.submissions; + // A submission orders every CPU write the game made before submitting it (the collect memo + // of this worker starts over here); inside it only the ordering points below do, unless + // APS5_PACKET_EPOCH=1 restores an epoch per packet. + bumpEpoch(&EpochBumps::submissions); for (std::size_t cursor = 0; cursor < submission.commands.size();) { - if (frameTiming == nullptr) includeSubmission(submission, false); + if (PacketEpoch()) bumpEpoch(&EpochBumps::packets); + CheckFailure(); const auto header = submission.commands[cursor]; - const auto count = static_cast((header >> 16u) & 0x3fffu) + 2; + if (Pm4::FillerPacket(header)) { ++cursor; continue; } + const auto count = Pm4::PacketWords(header); const auto packet = std::span(submission.commands).subspan(cursor, count); const auto opcode = (header >> 8u) & 0xffu; - { - PerformanceContext timingContext(frameTiming.get()); - PerformanceTimer timing("Driver.Packet"); - CheckFailure(); - timing.Mark("failure_check"); - if (opcode == 0x37 || opcode == 0x40 || opcode == 0x50 || opcode == 0x42 || opcode == 0x46 || opcode == 0x58 || header == FlipPacketHeader) { - std::lock_guard gpuLock(gpuMutex); - timing.Mark("gpu_mutex_wait"); - const auto eventType = opcode == 0x46 ? packet[1] & 0x3fu : 0u; - const auto memoryTransfer = opcode == 0x37 || opcode == 0x40 || opcode == 0x50; - const auto waitDraws = memoryTransfer || opcode == 0x42 || (opcode == 0x46 && (eventType == 0x07 || eventType == 0x0f || eventType == 0x10)); - const auto gpuCacheBarrier = opcode == 0x58 && Pm4::UsesGpuCacheBarrier(packet); - if (device != nullptr) { - if (gpuCacheBarrier) device->AcquireGpuMemory(); - else if (waitDraws) device->WaitDraws(); - else { - const auto scope = header == FlipPacketHeader ? "Driver.FlipWait" : opcode == 0x58 ? "Driver.AcquireMemoryWait" : "Driver.CacheEventWait"; - PerformanceTimer waitTiming(scope); - device->WaitIdle(); + // Names this packet for the flush hook's sync attribution ([hooksync]); the flip is 0xffff. + GuestMemory::SetCurrentPacket(header == FlipPacketHeader ? 0xffffu : opcode, submission.queue); + // Pending labels and full batches go to the GPU before this packet's own work starts + // (see flushBetweenPackets); labels themselves only check the deadline, so a label + // group shares one submission. Timed on its own, before the packet's timer starts. + const auto flushStart = profilePackets ? std::chrono::steady_clock::now() : std::chrono::steady_clock::time_point{}; + flushBetweenPackets(submission.queue, header, opcode == 0x49 || opcode == 0x37); + struct PacketTimer { + bool enabled; std::uint32_t key; std::uint32_t queue; PacketProfile& profile; std::chrono::steady_clock::time_point start; + ~PacketTimer() { + if (!enabled) return; + const auto now = std::chrono::steady_clock::now(); + auto& entry = profile.byOpcode[key]; + ++entry.first; + entry.second += std::chrono::duration(now - start).count(); + if (now - profile.lastReport < std::chrono::seconds(10)) return; + profile.lastReport = now; + std::vector>> hot(profile.byOpcode.begin(), profile.byOpcode.end()); + std::sort(hot.begin(), hot.end(), [](const auto& a, const auto& b) { return a.second.second > b.second.second; }); + std::string report; + for (std::size_t i = 0; i < hot.size() && i < 10; ++i) { + char text[96]; + std::snprintf(text, sizeof(text), " %s x%llu %.0fms", hot[i].first == 0xffffu ? "flip" : Pm4::Name(hot[i].first << 8u).c_str(), static_cast(hot[i].second.first), hot[i].second.second); + report += text; + } + auto& waits = waitOutcomes(); + auto& epochs = epochBumps(); + std::fprintf(stderr, "[packets] queue 0x%x %llu submissions, time by packet (10 s):%s, flush %.0fms; waits satisfied: at entry %llu, from recorder %llu (%llu while polling, same queue %llu, %llu without the GPU mutex), polled %llu (%llu entered without the GPU mutex, %llu entry tries failed), timed out %llu; poll submits %llu, poll reaps %llu; epoch bumps: submissions %llu, waits %llu, drains %llu, reaps %llu, packets %llu\n", queue, static_cast(profile.submissions), report.c_str(), profile.flushMs, static_cast(waits.atEntry), static_cast(waits.fromRecorder + waits.fromRecorderPolling), static_cast(waits.fromRecorderPolling), static_cast(waits.fromRecorderSameQueue), static_cast(waits.fromRecorderUnlocked), static_cast(waits.polled), static_cast(waits.entriesUnlocked), static_cast(waits.entryTriesFailed), static_cast(waits.timedOut), static_cast(waits.pollSubmits), static_cast(waits.pollReaps), static_cast(epochs.submissions), static_cast(epochs.waits), static_cast(epochs.drains), static_cast(epochs.reaps), static_cast(epochs.packets)); + waits = WaitOutcomes{}; + epochs = EpochBumps{}; + profile.byOpcode.clear(); + profile.submissions = 0; + profile.flushMs = 0; + } + } packetTimer{profilePackets, header == FlipPacketHeader ? 0xffffu : opcode, submission.queue, packetProfile, std::chrono::steady_clock::now()}; + if (profilePackets) packetProfile.flushMs += std::chrono::duration(packetTimer.start - flushStart).count(); + // Label writes are recorded on the GPU behind the work they signal whenever possible, so + // the CPU never waits for them. Otherwise packets that write guest memory in order with + // GPU work (labels, copies, constant RAM dumps), draws and flips drain the recorded batches + // first; reads (indirect arguments, register lists, waits) go through the flush hook. + // Debug aid: APS5_DRAIN_ALL=1 drains before every memory packet as before. + static const bool drainAll = std::getenv("APS5_DRAIN_ALL") != nullptr; + static const bool profile = std::getenv("APS5_PROFILE_DRAW") != nullptr; + bool wroteOnGpu = false; + // A label the CPU writes while the recorder is idle needs no drain: nothing the GPU was + // given is still running (draws complete synchronously), so the write is already ordered + // after every earlier packet. That was 70% of all drains at the movie stage. + bool orderedAlready = false; + // Labels that store nothing (RELEASE_MEM without data select or destination: + // interrupt-only) need no drain because Pm4::Execute is a no-op for them. The label + // counters and the [sync] report live at namespace scope (see reportSync). + if (!drainAll && (opcode == 0x49 || opcode == 0x37)) { + if (const auto label = Pm4::DecodeLabelWrite(packet)) { + const auto bytes = label->Bytes(); + if (DeferLabels() && bytes.size() <= DeferredLabel::Capacity && bytes.size() % 4 == 0 && label->address % 4 == 0) { + // Queued on this worker, no mutex: recorded with the group by + // flushBetweenPackets before the next packet that needs it (or by the + // deadline), still ahead of every later packet of this queue. + auto& deferred = deferredLabels(); + if (deferred.labels.empty()) deferred.since = std::chrono::steady_clock::now(); + auto& entry = deferred.labels.emplace_back(); + entry.address = label->address; + entry.size = bytes.size(); + std::memcpy(entry.bytes.data(), bytes.data(), bytes.size()); + ++queuedLabels; + wroteOnGpu = true; + } else { + GuestMemory::TagGpuLockSite(GuestMemory::GpuLockSite::Label); + std::lock_guard gpuLock(GuestMemory::GpuMutex()); + const auto localDevice = device.load(); + // Earlier labels of this queue go first (queue order). The stamp is taken + // under the mutex, right before the label is noted, so it orders after every + // submission the game made before this point. Without a device (reason 4) + // the packet drains nothing and Pm4::Execute stores it, as before. + recordDeferredLabels(localDevice.get(), submission.queue); + const auto reason = localDevice != nullptr ? localDevice->WriteLabelOnGpu(label->address, bytes, ++eventSerial, submission.queue) : 4; + wroteOnGpu = reason == 0 || reason == 5 || reason == 6; + if (reason == 1) { + // Idle recorder: stored here, still under the mutex, as + // recordDeferredLabels does. Stored later by Pm4::Execute outside it, + // another worker could record work over the range first and the flush + // hook would then sync for the store. + GuestMemory::Write(label->address, bytes, 4); + wroteOnGpu = true; + } + countLabelOutcome(reason); + ++immediateLabels; + } + } else if (opcode == 0x49 ? ((packet[2] >> 29u) == 0 || (packet[3] | (static_cast(packet[4]) << 32u)) == 0) : (packet[2] | (static_cast(packet[3]) << 32u)) == 0) { + orderedAlready = true; + ++noOpLabels; + } else { + ++labelFallbacks[4]; + } + } + // The other stores the driver makes for the title (COPY_DATA and DMA_DATA to memory, + // DUMP_CONST_RAM) used to drain the device and store on the CPU. Their bytes are known + // before the GPU runs them (immediate, constant RAM, or a source read through the flush + // hook, which waits only for recorded work writing the source), so the store is recorded + // on the GPU exactly like a label (VulkanDevice::WriteLabelOnGpu: host import, + // barriers, label table, pending-write note): a destination that recorded work also + // writes is then ordered by the queue and no CPU wait happens. An idle recorder keeps + // the CPU store (nothing to order after; done under the mutex so no work can be recorded + // over the range first). A store WriteLabelOnGpu cannot take (more than 64 KiB, not + // 4-byte aligned, no device) or that does not decode drains and stores as before. + // Kill switch: APS5_CPU_STORES=1 keeps the drain and CPU store for every such packet. + static const bool cpuStores = std::getenv("APS5_CPU_STORES") != nullptr; + if (!drainAll && !cpuStores && (opcode == 0x40 || opcode == 0x50 || opcode == 0x83)) { + constexpr std::size_t gpuStoreLimit = 65536; + bool drained = true; + if (const auto store = Pm4::ResolveStore(packet, queue, gpuStoreLimit)) { + const auto bytes = store->Bytes(); + if (bytes.empty()) { + // Nothing to store: Pm4::Execute is a no-op for it too. + orderedAlready = true; + drained = false; + } else { + // No lock site of its own: these stores count under [lock] 'label'. + GuestMemory::TagGpuLockSite(GuestMemory::GpuLockSite::Label); + std::lock_guard gpuLock(GuestMemory::GpuMutex()); + const auto localDevice = device.load(); + // Results still on the GPU for the range would be stored over (or skipped + // around, as a block MarkWritten stamps) the store later: written back + // first, like a buffer fill (a GPU-direct write-back is recorded before the + // store and ordered by its ALL_COMMANDS -> TRANSFER barrier). The CPU and + // drained paths get this through the flush hook of GuestMemory::Write. + Graphics::StorageTexture::FlushPending(store->address, bytes.size(), nullptr, "packet store"); + // Earlier labels of this queue go first (queue order); the stamp is taken + // under the mutex like a label's. + recordDeferredLabels(localDevice.get(), submission.queue); + const auto reason = localDevice != nullptr ? localDevice->WriteLabelOnGpu(store->address, bytes, ++eventSerial, submission.queue) : 4; + if (reason == 0 || reason == 5 || reason == 6) { + if (reason == 0) ++storesOnGpu; + else ++storesBehindCompletions; + wroteOnGpu = true; + drained = false; + } else if (reason == 1) { + GuestMemory::Write(store->address, bytes, 1); + ++storesOnCpu; + wroteOnGpu = true; + drained = false; } } - timing.Mark(gpuCacheBarrier ? "gpu_cache_barrier" : waitDraws ? "draw_wait" : "device_idle_wait"); } - if (header == FlipPacketHeader) { - CheckFailure(); - timing.Mark("flip_prepare"); - } else if (opcode == 0x15) { - dispatch(queue, packet, submission); - } else if (opcode == 0x16) { - std::array direct; - { - std::lock_guard gpuLock(gpuMutex); - const GuestMemory::MemoryAccessScope memoryScope(device.get(), [](void* context, std::uint64_t address, std::size_t bytes, bool writable) { - if (context) static_cast(context)->ResolveMemory(address, bytes, writable); - }); - direct = Pm4::ResolveDispatch(packet, queue); + if (drained) ++storesDrained; + } + // Draws no longer drain: a recorded draw follows earlier recorded work in queue order, and + // a synchronous draw's batch submits the recorder first (CommandBatch); the guest memory a + // draw's preparation reads goes through the flush hook. Debug aid: APS5_DRAW_DRAIN=1 restores. + static const bool drawDrain = std::getenv("APS5_DRAW_DRAIN") != nullptr; + const bool drawPacket = opcode == 0x2d || opcode == 0x35; + // Flips no longer drain either: the frame's batches are submitted and the presenter's blit + // follows them on the same queue (see Driver::Present). Debug aid: APS5_SYNC_FLIP=1 restores. + static const bool syncFlip = std::getenv("APS5_SYNC_FLIP") != nullptr; + const bool drains = drainAll ? ((Pm4::AccessesMemory(header) && opcode != 0x16) || opcode == 0x42 || opcode == 0x46 || opcode == 0x58 || header == FlipPacketHeader) + : (!wroteOnGpu && !orderedAlready && (opcode == 0x49 || opcode == 0x37 || opcode == 0x40 || opcode == 0x50 || opcode == 0x83 || (drawPacket && drawDrain) || (header == FlipPacketHeader && syncFlip))); + if (drains) { + // The GPU wait happens without the mutex: the batches are submitted under it, the + // timeline value is waited for outside (the other workers keep recording), then the + // completions of the batches up to that serial run under it again, in order, before + // the packet's own CPU store. Work another thread submits meanwhile is not waited + // for: a RELEASE_MEM only orders after earlier work, and a CPU store into a range + // such work writes is caught by the flush hook. vkDeviceWaitIdle is not needed: the + // presenter waits its own render fence and every other submission is synchronous. + // The device is held by this shared_ptr across the wait and released under the + // mutex (a replacement on another thread must not destroy it outside). + // Debug aids: APS5_NO_UNLOCKED_DRAIN=1 and APS5_DRAIN_ALL=1 drain under the mutex. + static const bool unlockedDrain = std::getenv("APS5_NO_UNLOCKED_DRAIN") == nullptr && !drainAll; + // A drain is an ordering point for the collect memo (see GuestMemory::BumpCollectEpoch). + bumpEpoch(&EpochBumps::drains); + std::shared_ptr draining; + std::uint64_t epoch = 0; + { + GuestMemory::TagGpuLockSite(GuestMemory::GpuLockSite::Flush); + std::lock_guard gpuLock(GuestMemory::GpuMutex()); + if (profile) { + ++drainCounts[header == FlipPacketHeader ? 0xffffu : opcode]; + ++drainTotal; + if (std::chrono::steady_clock::now() - lastSyncReport > std::chrono::seconds(10)) { + lastSyncReport = std::chrono::steady_clock::now(); + reportSync(); + } } - dispatch(queue, direct, submission); - } else if (opcode == 0x35 || opcode == 0x2d) { - draw(queue, packet, submission); - } else if (opcode != 0x42 && opcode != 0x46 && opcode != 0x58) { - std::lock_guard gpuLock(gpuMutex); - const GuestMemory::MemoryAccessScope memoryScope(device.get(), [](void* context, std::uint64_t address, std::size_t bytes, bool writable) { - if (context) static_cast(context)->ResolveMemory(address, bytes, writable); - }); - Pm4::Execute(packet, queue); - timing.Mark("pm4_execute"); + draining = device.load(); + // A drain is the ordering point: this worker's queued labels (an undecodable + // label packet skips the record in flushBetweenPackets) go in first, so the + // packet's own CPU store lands after them in queue order. + recordDeferredLabels(draining.get(), submission.queue); + if (draining != nullptr) { + if (unlockedDrain && draining->CanWaitUnlocked()) epoch = draining->SubmitAndEpoch(); + else draining->WaitIdle(); + } + if (epoch == 0) draining.reset(); + } + if (epoch != 0) { + ++unlockedDrains; + try { + draining->WaitRecorded(epoch); + } catch (...) { + // A lost device: if Present replaced it meanwhile this copy is the last + // owner, and its recorder must still be torn down under the mutex. + GuestMemory::TagGpuLockSite(GuestMemory::GpuLockSite::Flush); + std::lock_guard gpuLock(GuestMemory::GpuMutex()); + draining.reset(); + throw; + } + GuestMemory::TagGpuLockSite(GuestMemory::GpuLockSite::Flush); + std::lock_guard gpuLock(GuestMemory::GpuMutex()); + draining->ReapRecorded(epoch); + draining.reset(); } } + traceLabel(packet, submission.queue); + // Waits do not count as progress; every other packet does, including while it runs. + const bool waitPacket = opcode == 0x3c || opcode == 0x93; + struct Progress { + Driver& driver; + bool counted; + ~Progress() { + if (!counted) return; + --driver.packetsInFlight; + ++driver.packetsDone; + } + } progress{*this, !waitPacket}; + if (!waitPacket) ++packetsInFlight; + recent.Record(cursor); if (header == FlipPacketHeader) { - frameTiming->SetFlip(submission.serial, cursor, submission.received, FrameTiming::Clock::now()); - const auto completedFrame = std::exchange(frameTiming, nullptr); - submission.flips.at(cursor)->GpuReady(completedFrame); + CheckFailure(); + if (!drains) { + // Frame N's recorded batches go to the GPU now; the presenter's own submission + // follows them in queue order, so no wait is needed here and GpuReady returns at + // once (the title paces on flipPendingNum, which the presenter drops after showing + // the frame). + GuestMemory::TagGpuLockSite(GuestMemory::GpuLockSite::Flush); + std::lock_guard gpuLock(GuestMemory::GpuMutex()); + const auto localDevice = device.load(); + // This worker's queued labels belong to the frame: recorded first (queue order), + // then everything goes out (an empty open batch submits nothing) unless the + // record submitted already (a second call would only scan the fences again). + if (!recordLabelsForPacket(localDevice.get(), submission.queue) && localDevice != nullptr) localDevice->SubmitRecorded(submission.queue == 0); + } + // Each flip carries a fresh frame timing record; the per-stage frame statistics upstream + // collects are not gathered on this path. + auto frame = std::make_shared(++frameSerial); + const auto now = FrameTiming::Clock::now(); + frame->IncludeSubmission(submission.serial, now, now, now, true); + frame->SetFlip(submission.serial, cursor, now, now); + submission.flips.at(cursor)->GpuReady(frame); + } else if (opcode == 0x15) { + timed(&WorkerProfile::dispatchMs, [&] { tolerate("dispatch", [&] { dispatch(queue, packet, submission); }); }); + Graphics::Recorder::CountRecordedWork(); + } else if (opcode == 0x16) { + timed(&WorkerProfile::dispatchMs, [&] { tolerate("indirect dispatch", [&] { dispatchIndirect(queue, packet, submission); }); }); + Graphics::Recorder::CountRecordedWork(); + } else if (opcode == 0x3c || opcode == 0x93) { + static const bool traceGpu = std::getenv("APS5_TRACE_GPU") != nullptr; + const auto waitStart = std::chrono::steady_clock::now(); + timed(&WorkerProfile::waitMs, [&] { waitMemory(packet, submission.queue, recent, submission.received); }); + if (traceGpu && std::chrono::steady_clock::now() - waitStart > std::chrono::milliseconds(200)) { + // List the rest of the submission to show what the stalled queue would have done next. + for (std::size_t next = cursor + count, shown = 0; next < submission.commands.size() && shown < 48; ++shown) { + const auto nextHeader = submission.commands[next]; + const auto nextCount = Pm4::PacketWords(nextHeader); + const auto nextPacket = std::span(submission.commands).subspan(next, nextCount); + std::fprintf(stderr, "[gpu] then %s", Pm4::Name(nextHeader).c_str()); + for (std::size_t i = 1; i < nextPacket.size() && i < 7; ++i) std::fprintf(stderr, " %08x", nextPacket[i]); + std::fprintf(stderr, "\n"); + next += nextCount; + } + } + } else if (opcode == 0x35 || opcode == 0x2d) { + timed(&WorkerProfile::drawMs, [&] { tolerate("draw", [&] { + static const bool traceDraws = std::getenv("APS5_TRACE_DRAWS") != nullptr; + const auto color = (static_cast(readRegister(queue.context, 0x390)) << 40u) | (static_cast(readRegister(queue.context, 0x318)) << 8u); + try { + draw(queue, packet, submission); + if (traceDraws) std::fprintf(stderr, "[draw] target 0x%llx mask 0x%x ok\n",static_cast(color), readRegister(queue.context, 0x8e)); + } catch (const std::exception& error) { + if (traceDraws) std::fprintf(stderr, "[draw] target 0x%llx mask 0x%x failed: %.160s\n",static_cast(color), readRegister(queue.context, 0x8e), error.what()); + // Name the color target so skipped draws can be matched against the scanout buffer. + char suffix[48]; + std::snprintf(suffix, sizeof(suffix), " [color target 0x%llx]", static_cast(color)); + // Debug aid: the first failing draw per color target and reason saves its register banks to + // draw__.regs. + static std::set> dumpedTargets; + const std::string reason = std::string(error.what()).substr(0, 48); + if (dumpedTargets.insert({color, reason}).second) { + char name[64]; + std::snprintf(name, sizeof(name), "draw_%llx_%08x.regs", static_cast(color), static_cast(std::hash{}(reason))); + if (std::FILE* file = std::fopen(name, "w")) { + std::fprintf(file, "# %s\n", error.what()); + recent.Each([&](const std::string& line) { std::fprintf(file, "# packet %s\n", line.c_str()); }); + for (const auto& [offset, value] : queue.context) std::fprintf(file, "context %x %08x\n", offset, value); + for (const auto& [offset, value] : queue.userConfig) std::fprintf(file, "uconfig %x %08x\n", offset, value); + for (const auto& [offset, value] : queue.shader) std::fprintf(file, "shader %x %08x\n", offset, value); + std::fclose(file); + } + } + throw std::runtime_error(error.what() + std::string(suffix)); + } + }); }); + } else if (opcode != 0x42 && opcode != 0x46 && opcode != 0x58) { + if (!wroteOnGpu) Pm4::Execute(packet, queue); } + if (drawPacket) Graphics::Recorder::CountRecordedWork(); cursor += count; } + // The submission's last labels and dispatches go to the GPU before it counts as completed + // (Driver::WaitIdle and the game threads polling the labels must not depend on a later + // packet of some queue). Checked lock-free first: an empty open batch needs no submit. A + // compute worker submits without reaping (it must not wait under the mutex for queue 0). + if (!deferredLabels().labels.empty() || Graphics::Recorder::PendingLabelSince().has_value() || Graphics::Recorder::RecordedWorkSinceSubmit() != 0) { + GuestMemory::TagGpuLockSite(GuestMemory::GpuLockSite::Flush); + std::lock_guard gpuLock(GuestMemory::GpuMutex()); + const auto localDevice = device.load(); + // This worker's deferred labels are recorded first (they belong to this submission). + recordDeferredLabels(localDevice.get(), submission.queue); + if (localDevice != nullptr) localDevice->SubmitRecorded(submission.queue == 0); + } } - void run() noexcept { + // An idle worker retires labels waiting in completion actions (see Recorder::AfterCompletions): + // without it a label whose only consumer is a game thread, submitted last before the game waits + // for it, would never land once every worker sleeps. Non-blocking (a fence status check under + // the mutex, tried only); the caller polls it every millisecond while such labels exist. + void reapCompletionLabels() { + std::unique_lock gpuLock(GuestMemory::GpuMutex(), std::try_to_lock); + if (!gpuLock.owns_lock()) return; + // Every idle worker polls this; once another worker reaped the labels there is nothing to + // do, and the epoch bump below would only inflate the "reaps" and [guestmem] epoch counts. + if (Graphics::Recorder::PendingCompletionLabels() == 0) return; + // The write-backs the reap runs collect outside any submission: a fresh epoch, so no walk + // of this worker's last packet is reused for them. + bumpEpoch(&EpochBumps::reaps); + if (const auto localDevice = device.load()) localDevice->ReapRecorded(); + } + + void run(std::uint32_t id) noexcept { + OnWorkerThread() = true; + // Named from the start, so a worker's fence waits before its first WAIT_REG_MEM land in its + // own row of the [recorder] "fence waits by thread" report and "untagged" is exactly the + // presenter and the game threads. + GuestMemory::TagGpuLockThread(id); + if (id == 0) StartWorkerSampler(); Submission submission; try { for (;;) { + submission = Submission{}; + static const bool traceGpu = std::getenv("APS5_TRACE_GPU") != nullptr; { - PerformanceContext timingContext(frameTiming.get()); - PerformanceTimer timing("Driver.Worker"); - submission = Submission{}; - timing.Mark("submission_release"); std::unique_lock lock(mutex); - timing.Mark("queue_mutex_wait"); - changed.wait(lock, [&] { return failure || stopping || !pending.empty(); }); - timing.Mark("wait_for_submission"); + auto& pending = workers.at(id).pending; + if (traceGpu && pending.empty()) std::fprintf(stderr, "[gpu] %.1f idle queue=0x%x\n", TraceMs(), id); + const auto ready = [&] { return failure || stopping || !pending.empty(); }; + // While completion labels are pending, the idle wait wakes every millisecond to + // reap them (the driver mutex is dropped for that: it is never held with the GPU + // mutex); otherwise it sleeps until work arrives. + while (!ready()) { + if (Graphics::Recorder::PendingCompletionLabels() == 0) { + changed.wait(lock, ready); + break; + } + if (changed.wait_for(lock, std::chrono::milliseconds(1), ready)) break; + lock.unlock(); + reapCompletionLabels(); + lock.lock(); + } rethrowFailure(); if (pending.empty()) { break; } submission = std::move(pending.front()); pending.pop_front(); - submission.dequeued = FrameTiming::Clock::now(); } execute(submission); + if (traceGpu) std::fprintf(stderr, "[gpu] %.1f done serial=%llu queue=0x%x\n", TraceMs(), static_cast(submission.serial), id); { - PerformanceContext timingContext(frameTiming.get()); - PerformanceTimer timing("Driver.SubmissionCompletion"); - std::lock_guard gpuLock(gpuMutex); - if (device) device->WaitIdle(); - } - { - PerformanceContext timingContext(frameTiming.get()); - PerformanceTimer timing("Driver.Completion"); std::lock_guard lock(mutex); - timing.Mark("mutex_wait"); rethrowFailure(); - completed = submission.serial; + markCompleted(submission.serial); } changed.notify_all(); } - std::lock_guard gpuLock(gpuMutex); - device.reset(); } catch (...) { const auto error = std::current_exception(); + try { + std::rethrow_exception(error); + } catch (const std::exception& reason) { + std::fprintf(stderr, "[gpu] worker failed: %s\n", reason.what()); + } catch (...) { + std::fprintf(stderr, "[gpu] worker failed with a non-standard exception\n"); + } for (const auto& [offset, flip] : submission.flips) flip->Fail(error); ReportFailure(error); { - std::lock_guard gpuLock(gpuMutex); + std::lock_guard gpuLock(GuestMemory::GpuMutex()); device.reset(); } } diff --git a/core/libs/prx/libSceAgcDriver/Execution/src/GuestMemory.cpp b/core/libs/prx/libSceAgcDriver/Execution/src/GuestMemory.cpp index 52e4ccc09..0789c853b 100644 --- a/core/libs/prx/libSceAgcDriver/Execution/src/GuestMemory.cpp +++ b/core/libs/prx/libSceAgcDriver/Execution/src/GuestMemory.cpp @@ -1,12 +1,18 @@ #include "prx/libSceAgcDriver/Execution/include/GuestMemory.hpp" -#include "prx/libc/include/GuestMemoryTracking.hpp" -#include "prx/libSceAgcDriver/Execution/include/MemoryAccessScope.hpp" -#include "prx/libSceAgcDriver/Execution/include/PerformanceTimer.hpp" +#include "prx/libc/include/GuestAllocations.hpp" +#include "prx/libc/include/GuestArena.hpp" +#include +#include +#include +#include #include +#include +#include #include #include #include #include +#include #ifdef _WIN32 #ifndef NOMINMAX #define NOMINMAX @@ -22,6 +28,471 @@ namespace { void require(bool condition, const char* reason) { if (!condition) throw std::runtime_error(std::string("AGC driver: ") + reason); } + +// A code address relative to its module, for symbolizing debug traces with nm. +unsigned long long ModuleOffset(const void* address) { +#ifdef _WIN32 + HMODULE module = nullptr; + GetModuleHandleExA(GET_MODULE_HANDLE_EX_FLAG_FROM_ADDRESS | GET_MODULE_HANDLE_EX_FLAG_UNCHANGED_REFCOUNT, static_cast(address), &module); + return reinterpret_cast(address) - reinterpret_cast(module); +#else + return reinterpret_cast(address); +#endif +} + +#ifdef _WIN32 +bool readableProtection(DWORD protection) { + return protection == PAGE_READONLY || protection == PAGE_READWRITE || protection == PAGE_WRITECOPY || protection == PAGE_EXECUTE_READ || protection == PAGE_EXECUTE_READWRITE || protection == PAGE_EXECUTE_WRITECOPY; +} + +bool writableProtection(DWORD protection) { + return protection == PAGE_READWRITE || protection == PAGE_WRITECOPY || protection == PAGE_EXECUTE_READWRITE || protection == PAGE_EXECUTE_WRITECOPY; +} +#endif + +} + +namespace { +std::atomic forgetCalls{0}; +std::atomic forgetBytes{0}; +std::atomic collectMemoHits{0}; +std::atomic collectEpochBumps{0}; +// Walks that reported at least one written page (the ones the probe pass used to double), and the +// tracker mutex acquisitions that had to wait (APS5_PROFILE_DRAW; see lockTracker). +std::atomic collectDirty{0}; +std::atomic trackerWaits{0}; +std::atomic trackerAcquisitions{0}; +std::uintptr_t PagesBase(); +std::size_t PagesSize(); +std::uintptr_t ImagePagesBase(); +std::size_t ImagePagesSize(); +} + +// Profile (APS5_PROFILE_DRAW): calls, bytes and time of the guest memory copies and comparisons the +// driver makes, printed every 10 s as [guestmem]. +struct MemoryCounter { + std::atomic calls{0}; + std::atomic bytes{0}; + std::atomic nanoseconds{0}; +}; + +struct MemoryProfile { + MemoryCounter read; + MemoryCounter compare; + MemoryCounter write; + MemoryCounter changedWrite; + MemoryCounter collect; + MemoryCounter verify; + MemoryCounter query; + std::atomic lastReport{0}; + // Sampled callers of Read and Write (module offsets), to name what touches guest memory most. + std::mutex callersMutex; + std::array, 16> callers{}; + std::array, 16> writeCallers{}; +}; + + +MemoryProfile& Profile() { + static MemoryProfile profile; + return profile; +} + +void CountCaller(std::array, 16>& callers, std::uint32_t& sampled, std::uint32_t every, const void* returnAddress) { + if (++sampled % every != 0) return; + auto& state = Profile(); + const auto offset = ModuleOffset(returnAddress); + std::lock_guard lock(state.callersMutex); + for (auto& [caller, count] : callers) { + if (caller == offset || count == 0) { + caller = offset; + count += every; + return; + } + } +} + +// Attribution tags (see the header): the packet a queue worker executes and the read site the +// driver code named. Plain thread-locals; a store per packet costs nothing measurable. +thread_local PacketTag currentPacket{NoPacket, 0xffffffffu}; +thread_local ReadSite currentReadSite = ReadSite::Unknown; +// Sampled reads by site (every 256th Read/ReadCommitted, like the callers), for the [guestmem] line. +std::atomic readSiteSamples[static_cast(ReadSite::Count)] = {}; + +void SetCurrentPacket(std::uint32_t opcode, std::uint32_t queue) { + currentPacket = {opcode, queue}; +} + +PacketTag CurrentPacket() { + return currentPacket; +} + +const char* ReadSiteName(ReadSite site) { + static const char* const names[static_cast(ReadSite::Count)] = {"unknown", "capture", "dispatch-cache", "texture-compare", "texture-read", "buffer-upload", "index-buffer", "vertex-buffer", "registers", "indirect-args", "wait", "label", "scanout", "store"}; + const auto index = static_cast(site); + return index < static_cast(ReadSite::Count) ? names[index] : "?"; +} + +ReadSite SetReadSite(ReadSite site) { + return std::exchange(currentReadSite, site); +} + +ReadSite CurrentReadSite() { + return currentReadSite; +} + +std::size_t CaptureCallerOffsets(std::span frames, unsigned skip) { + if (frames.empty()) return 0; +#ifdef _WIN32 + // Walks the x64 unwind tables, so it needs no frame pointers (unlike __builtin_return_address(n) + // for n > 0); frame 0 of the capture is this function's caller. + void* raw[16]; + const auto wanted = static_cast(std::min(frames.size(), std::size(raw))); + const auto captured = RtlCaptureStackBackTrace(1 + skip, wanted, raw, nullptr); + // The driver's module is resolved once (this function lives in it): a per-frame + // GetModuleHandleEx would take the loader lock several times per sync under the GpuMutex. + static const auto module = [] { + HMODULE handle = nullptr; + GetModuleHandleExA(GET_MODULE_HANDLE_EX_FLAG_FROM_ADDRESS | GET_MODULE_HANDLE_EX_FLAG_UNCHANGED_REFCOUNT, reinterpret_cast(&CaptureCallerOffsets), &handle); + std::uintptr_t size = 0; + if (handle != nullptr) { + // The image size comes from the mapped PE headers (no psapi dependency). + const auto* base = reinterpret_cast(handle); + const auto* dos = reinterpret_cast(base); + const auto* nt = reinterpret_cast(base + dos->e_lfanew); + size = nt->OptionalHeader.SizeOfImage; + } + return std::pair(reinterpret_cast(handle), size); + }(); + for (std::size_t i = 0; i < captured; ++i) { + const auto address = reinterpret_cast(raw[i]); + frames[i] = module.first != 0 && address >= module.first && address - module.first < module.second ? address - module.first : 0; + } + return captured; +#else + static_cast(skip); + frames[0] = ModuleOffset(__builtin_return_address(0)); + return 1; +#endif +} + +void CountReadCaller(const void* returnAddress) { + thread_local std::uint32_t sampled = 0; + // Sampled at the same rate as the callers, in step with them (CountCaller advances the counter). + if ((sampled + 1) % 256 == 0) readSiteSamples[static_cast(currentReadSite)].fetch_add(256, std::memory_order_relaxed); + CountCaller(Profile().callers, sampled, 256, returnAddress); +} + +void CountWriteCaller(const void* returnAddress) { + thread_local std::uint32_t sampled = 0; + CountCaller(Profile().writeCallers, sampled, 16, returnAddress); +} + +class TimedAccess { +public: + TimedAccess(MemoryCounter& counter, std::uint64_t bytes) : counter(counter), bytes(bytes), start(std::chrono::steady_clock::now()) {} + ~TimedAccess() { + const auto now = std::chrono::steady_clock::now(); + counter.calls += 1; + counter.bytes += bytes; + counter.nanoseconds += static_cast(std::chrono::duration_cast(now - start).count()); + static const bool profile = std::getenv("APS5_PROFILE_DRAW") != nullptr; + if (!profile) return; + auto& state = Profile(); + const auto nowMs = std::chrono::duration_cast(now.time_since_epoch()).count(); + auto last = state.lastReport.load(); + if (nowMs - last < 10000 || !state.lastReport.compare_exchange_strong(last, nowMs)) return; + const auto line = [](const char* name, const MemoryCounter& c) { + const double seconds = c.nanoseconds / 1e9; + std::fprintf(stderr, " %s %llu calls %.0f MiB %.1f s", name, static_cast(c.calls.load()), c.bytes / 1048576.0, seconds); + }; + std::fprintf(stderr, "[guestmem]"); + line("read", state.read); + line("compare", state.compare); + line("write", state.write); + line("write-changed", state.changedWrite); + line("collect", state.collect); + line("verify", state.verify); + line("query", state.query); + std::fprintf(stderr, " collect-memo hits %llu, collect epochs %llu", static_cast(collectMemoHits.load()), static_cast(collectEpochBumps.load())); + std::fprintf(stderr, " collect-dirty %llu tracker waits %llu / %llu", static_cast(collectDirty.load()), static_cast(trackerWaits.load()), static_cast(trackerAcquisitions.load())); + std::fprintf(stderr, " | arena 0x%llx+0x%llx image 0x%llx+0x%llx forgets %llu (%.0f MiB)", static_cast(PagesBase()), static_cast(PagesSize()), static_cast(ImagePagesBase()), static_cast(ImagePagesSize()), static_cast(forgetCalls.load()), forgetBytes.load() / 1048576.0); + { + std::lock_guard lock(state.callersMutex); + std::fprintf(stderr, " | read callers:"); + for (const auto& [caller, count] : state.callers) { + if (count != 0) std::fprintf(stderr, " +0x%llx=%llu", caller, static_cast(count)); + } + std::fprintf(stderr, " | write callers:"); + for (const auto& [caller, count] : state.writeCallers) { + if (count != 0) std::fprintf(stderr, " +0x%llx=%llu", caller, static_cast(count)); + } + } + std::fprintf(stderr, " | read sites:"); + for (std::size_t site = 0; site < static_cast(ReadSite::Count); ++site) { + const auto count = readSiteSamples[site].load(std::memory_order_relaxed); + if (count != 0) std::fprintf(stderr, " %s=%llu", ReadSiteName(static_cast(site)), static_cast(count)); + } + std::fprintf(stderr, "\n"); + } +private: + MemoryCounter& counter; + std::uint64_t bytes; + std::chrono::steady_clock::time_point start; +}; + +namespace { + +// Page-state cache. VirtualQuery finds the extent of a run of equally mapped pages by walking page +// tables, which costs tens of milliseconds inside the game's committed gigabyte heap, and GPU workers +// verify ranges thousands of times per second. Committed pages of the guest arena are therefore +// remembered, one byte of access flags per 4 KiB page (the table is committed lazily, so only the +// pages ever verified cost memory). libc reports every range whose mapping or protection changed +// (registry mutations and guest heap decommits) through the invalidator, which forgets those pages. +// Only committed pages are cached; reserved pages are queried each time, so a commit made outside the +// registry (the guest heap grows that way) is seen at once. +// A second table covers the main guest image (the PE at 0x7ff6b...): its protections change only +// through registry mutations (mprotect replaces the registered range and invalidates through the same +// callback), and address-based builds verify every image section on each build, so without the table +// each of those verifications was a VirtualQuery. APS5_NO_IMAGE_PAGE_CACHE=1 leaves the image uncached. +constexpr std::size_t PageBytes = 4096; +constexpr std::uint8_t PageReadable = 1; +constexpr std::uint8_t PageWritable = 2; + +struct PageSpan { + std::uintptr_t base = 0; + std::size_t size = 0; + // Per page: access flags, zero while unknown. + std::uint8_t* pages = nullptr; + + // Reserves and commits the table; untouched table pages cost nothing until a guest page is recorded. + bool allocate() { + if (size == 0) return false; + const auto count = (size / PageBytes + 8) & ~static_cast(7); +#ifdef _WIN32 + pages = static_cast(VirtualAlloc(nullptr, count, MEM_RESERVE | MEM_COMMIT, PAGE_READWRITE)); +#else + pages = static_cast(std::calloc(count, 1)); +#endif + if (pages == nullptr) size = 0; + return pages != nullptr; + } + + bool covers(std::uintptr_t address) const { + return size != 0 && address >= base && address - base < size; + } + + std::uint8_t load(std::uintptr_t address) const { + return std::atomic_ref(pages[(address - base) / PageBytes]).load(std::memory_order_relaxed); + } + + void store(std::uintptr_t address, std::uint8_t value) { + std::atomic_ref(pages[(address - base) / PageBytes]).store(value, std::memory_order_relaxed); + } + + // End of the run of pages equal to `value` that starts at `address`, at most `limit`. + std::uintptr_t runEnd(std::uintptr_t address, std::uint8_t value, std::uintptr_t limit) const { + auto index = (address - base) / PageBytes; + const auto stop = std::min((std::min(limit, base + size) - base + PageBytes - 1) / PageBytes, size / PageBytes); + const std::uint64_t wide = 0x0101010101010101ull * value; + while (index < stop) { + if (index % 8 == 0 && index + 8 <= stop && std::atomic_ref(*reinterpret_cast(pages + index)).load(std::memory_order_relaxed) == wide) { + index += 8; + continue; + } + if (std::atomic_ref(pages[index]).load(std::memory_order_relaxed) != value) break; + ++index; + } + return std::min(limit, base + index * PageBytes); + } + + void forget(std::uintptr_t address, std::size_t bytes) { + if (size == 0 || bytes == 0 || address >= base + size || address + bytes <= base) return; + const auto first = (std::max(address, base) - base) / PageBytes; + const auto last = (std::min(address + bytes, base + size) - 1 - base) / PageBytes; + for (auto page = first; page <= last; ++page) std::atomic_ref(pages[page]).store(0, std::memory_order_relaxed); + } +}; + +struct PageStates { + std::once_flag once; + PageSpan arena; + PageSpan image; + + void initialize(); + + PageSpan* spanOf(std::uintptr_t address) { + if (arena.covers(address)) return &arena; + if (image.covers(address)) return ℑ + return nullptr; + } + + void forget(std::uintptr_t address, std::size_t bytes) { + arena.forget(address, bytes); + image.forget(address, bytes); + } +}; + +PageStates& Pages() { + static PageStates pages; + return pages; +} + +std::uintptr_t PagesBase() { + return Pages().arena.base; +} + +std::size_t PagesSize() { + return Pages().arena.size; +} + +std::uintptr_t ImagePagesBase() { + return Pages().image.base; +} + +std::size_t ImagePagesSize() { + return Pages().image.size; +} + +void ForgetPages(std::uintptr_t address, std::size_t bytes) { + forgetCalls.fetch_add(1, std::memory_order_relaxed); + forgetBytes.fetch_add(bytes, std::memory_order_relaxed); + Pages().forget(address, bytes); +} + +void PageStates::initialize() { + std::call_once(once, [&] { + GuestArena::GuestArenaRange_nid_postfix(&arena.base, &arena.size); + const bool arenaCached = arena.allocate(); + bool imageCached = false; +#ifdef _WIN32 + // The image extent is the run of regions sharing the module's allocation base, as the registry + // walks it when it registers the main image. + static const bool noImageCache = std::getenv("APS5_NO_IMAGE_PAGE_CACHE") != nullptr; + if (const auto module = noImageCache ? nullptr : GetModuleHandleW(nullptr)) { + const auto start = reinterpret_cast(module); + auto cursor = start; + for (;;) { + MEMORY_BASIC_INFORMATION memory{}; + if (VirtualQuery(reinterpret_cast(cursor), &memory, sizeof(memory)) != sizeof(memory) || memory.AllocationBase != module || memory.RegionSize == 0 || memory.RegionSize > std::numeric_limits::max() - cursor) break; + cursor += memory.RegionSize; + } + image.base = start; + image.size = cursor - start; + imageCached = image.allocate(); + } +#endif + if (arenaCached || imageCached) GuestAllocations::GuestAllocationsSetInvalidator_nid_postfix(&ForgetPages); + }); +} + +struct PageRun { + std::uintptr_t begin; + std::uintptr_t end; + bool readable; + bool writable; +}; + +// Calls `emit` with consecutive runs of uniform accessibility covering [address, address + bytes) in +// order, stopping early when it returns false. Returns false when the address space cannot be queried. +template +bool describePages(std::uintptr_t address, std::size_t bytes, Emit&& emit) { + auto& pages = Pages(); + pages.initialize(); + const auto end = address + bytes; + auto cursor = address; + while (cursor < end) { + if (const auto* span = pages.spanOf(cursor)) { + const auto value = span->load(cursor); + if ((value & 3u) != 0) { + const auto next = span->runEnd(cursor, value, end); + if (!emit(PageRun{cursor, next, true, (value & PageWritable) != 0})) return true; + cursor = next; + continue; + } + } +#ifdef _WIN32 + const TimedAccess timed(Profile().query, 0); + MEMORY_BASIC_INFORMATION memory{}; + const auto queryStart = std::chrono::steady_clock::now(); + // A mutation that changes these pages while they are being queried bumps the generation + // before it invalidates, so a generation change across the query and the store below means + // the stored pages may describe the old state and are forgotten again. + const auto generation = GuestAllocations::GuestAllocationsGeneration_nid_postfix(); + if (VirtualQuery(reinterpret_cast(cursor), &memory, sizeof(memory)) != sizeof(memory)) return false; + // Debug aid: APS5_TRACE_QUERY prints every 16th page query with its cost. + static const bool traceQuery = std::getenv("APS5_TRACE_QUERY") != nullptr; + if (traceQuery) { + static std::atomic queries{0}; + if (queries.fetch_add(1) % 16 == 0) { + const auto us = std::chrono::duration(std::chrono::steady_clock::now() - queryStart).count(); + std::fprintf(stderr, "[query] 0x%llx range 0x%zx: base 0x%llx size 0x%llx state 0x%lx protect 0x%lx type 0x%lx %.0f us gen %llu from +0x%llx\n", static_cast(cursor), bytes, reinterpret_cast(memory.BaseAddress), static_cast(memory.RegionSize), memory.State, memory.Protect, memory.Type, us, static_cast(GuestAllocations::GuestAllocationsGeneration_nid_postfix()), ModuleOffset(__builtin_return_address(0))); + } + } + const auto base = reinterpret_cast(memory.BaseAddress); + if (memory.RegionSize > std::numeric_limits::max() - base || base + memory.RegionSize <= cursor) return false; + const auto regionEnd = base + memory.RegionSize; + const auto protection = memory.Protect & 0xffu; + const bool committed = memory.State == MEM_COMMIT && (memory.Protect & (PAGE_GUARD | PAGE_NOACCESS)) == 0; + const bool readable = committed && readableProtection(protection); + const bool writable = readable && writableProtection(protection); + for (PageSpan* span : {&pages.arena, &pages.image}) { + if (!readable || span->size == 0 || regionEnd <= span->base || base >= span->base + span->size) continue; + const std::uint8_t value = PageReadable | (writable ? PageWritable : 0u); + const auto first = std::max(base, span->base); + const auto last = std::min(regionEnd, span->base + span->size); + for (auto at = first; at < last; at += PageBytes) span->store(at, value); + if (GuestAllocations::GuestAllocationsGeneration_nid_postfix() != generation) span->forget(first, static_cast(last - first)); + } + const auto next = std::min(end, regionEnd); + if (!emit(PageRun{cursor, next, readable, writable})) return true; + cursor = next; +#else + std::ifstream maps("/proc/self/maps"); + if (!maps.is_open()) return false; + std::string line; + bool found = false; + while (std::getline(maps, line)) { + std::istringstream fields(line); + std::uintptr_t first = 0; + std::uintptr_t last = 0; + char separator = 0; + std::string permissions; + if (!(fields >> std::hex >> first >> separator >> last >> permissions) || separator != '-' || first >= last || permissions.size() < 2) return false; + if (last <= cursor || first > cursor) continue; + const auto next = std::min(end, last); + if (!emit(PageRun{cursor, next, permissions[0] == 'r', permissions[0] == 'r' && permissions[1] == 'w'})) return true; + cursor = next; + found = true; + break; + } + if (!found) { + static_cast(emit(PageRun{cursor, end, false, false})); + return true; + } +#endif + } + return true; +} + +// Checks that [address, address + bytes) is mapped with read (and, if asked, write) access. Returns +// an empty string when it is, otherwise why it is not. +std::string verify(std::uintptr_t address, std::size_t bytes, bool writable) { + const TimedAccess timed(Profile().verify, bytes); + std::string reason; + const bool queried = describePages(address, bytes, [&](const PageRun& run) { + if (!run.readable) { + char text[128]; + std::snprintf(text, sizeof(text), "guest memory is not readable at 0x%llx (range 0x%llx+0x%zx)", static_cast(run.begin), static_cast(address), bytes); + reason = text; + } else if (writable && !run.writable) { + reason = "guest memory has no write permission"; + } + return reason.empty(); + }); + if (!queried && reason.empty()) reason = "cannot query guest memory"; + return reason; +} + } void CheckRange(const void* pointer, std::size_t bytes, std::size_t alignment, bool writable) { @@ -29,60 +500,594 @@ void CheckRange(const void* pointer, std::size_t bytes, std::size_t alignment, b const auto address = reinterpret_cast(pointer); require(address != 0 && address % alignment == 0, "null or misaligned address"); require(bytes <= std::numeric_limits::max() - address, "address range overflow"); - MemoryAccessScope::Resolve(address, bytes, writable); - GuestMemoryTracking::GuestMemoryTrackingResolve_nid_postfix(address, bytes, writable); - auto cursor = address; - const auto end = address + bytes; + const auto reason = verify(address, bytes, writable); + if (!reason.empty()) { + // Debug aid: APS5_TRACE_UNREADABLE names the code that checked an inaccessible range. + static const bool trace = std::getenv("APS5_TRACE_UNREADABLE") != nullptr; + if (trace) std::fprintf(stderr, "[unreadable] %s from +0x%llx\n", reason.c_str(), ModuleOffset(__builtin_return_address(0))); + require(false, reason.c_str()); + } +} + +bool Accessible(const void* pointer, std::size_t bytes, bool writable) { + const auto address = reinterpret_cast(pointer); + return address != 0 && bytes <= std::numeric_limits::max() - address && verify(address, bytes, writable).empty(); +} + +Commitment DescribeCommitted(std::uint64_t address, std::size_t bytes, bool writable) { + require(bytes <= std::numeric_limits::max() - address, "address range overflow"); + const TimedAccess timed(Profile().verify, bytes); + Commitment result; + const bool queried = describePages(static_cast(address), bytes, [&](const PageRun& run) { + if (run.readable && (!writable || run.writable)) { + if (!result.ranges.empty() && result.ranges.back().second == run.begin) result.ranges.back().second = run.end; + else result.ranges.emplace_back(run.begin, run.end); + } + return true; + }); + require(queried, "cannot query guest memory"); + result.whole = bytes == 0 || (result.ranges.size() == 1 && result.ranges.front().first == address && result.ranges.front().second == address + bytes); + return result; +} + +std::vector> CommittedRanges(std::uint64_t address, std::size_t bytes, bool writable) { + return DescribeCommitted(address, bytes, writable).ranges; +} + +namespace { + +constexpr std::size_t WriteBlockBytes = 65536; + +struct WriteTracker { + std::mutex mutex; + bool initialized = false; + bool watched = false; + std::uintptr_t base = 0; + std::size_t size = 0; + // Generation of the last collected write per 64 KiB block of the arena. + std::vector blocks; + // Atomic only so a per-thread memo hit can read it without the mutex (every mutation and every + // stamp still happen under it): a hit must return the current value, including MarkWritten + // bumps, or an image refreshed after a stamp would keep failing UnchangedSince until the next + // walk of its range and re-upload at every use of the epoch. + std::atomic generation{1}; + std::vector pages; + // Collect memo: page-rounded ranges walked during the current epoch (see BumpCollectEpoch). A + // collect fully inside one skips GetWriteWatch, whose cost is proportional to the range's pages, + // and the same surfaces are collected several times per epoch (each storage mip binding, a sampled + // view plus its storage refresh, a write-back followed by a refresh, and every dispatch of the + // epoch binding the same targets). Entries carry the walking thread's epoch, so no entry matches + // another thread: the ring consulted first is per thread (ThreadCollectMemo, no mutex, no + // eviction by other threads' walks); this shared ring (8 workers with epochs of several packets + // hold a few hundred distinct ranges) serves APS5_SHARED_COLLECT_MEMO=1. + struct Memo { + std::uintptr_t begin = 0; + std::uintptr_t end = 0; + std::uint64_t epoch = 0; + }; + std::array memo{}; + std::size_t nextMemo = 0; + + void initialize() { + if (initialized) return; + initialized = true; + GuestArena::GuestArenaRange_nid_postfix(&base, &size); + watched = size != 0 && GuestArena::GuestArenaWriteWatched_nid_postfix(); + if (!watched) return; + blocks.assign(size / WriteBlockBytes + 1, 0); + pages.resize(1u << 16); + } +}; + +WriteTracker& Tracker() { + static WriteTracker tracker; + return tracker; +} + +// Takes the tracker mutex; under APS5_PROFILE_DRAW an acquisition that found it held is counted +// ('tracker waits N / M' in [guestmem]), which says whether walking outside the lock would pay. +std::unique_lock lockTracker(WriteTracker& tracker) { + static const bool profile = std::getenv("APS5_PROFILE_DRAW") != nullptr; + if (!profile) return std::unique_lock(tracker.mutex); + std::unique_lock lock(tracker.mutex, std::try_to_lock); + if (!lock.owns_lock()) { + trackerWaits.fetch_add(1, std::memory_order_relaxed); + lock.lock(); + } + trackerAcquisitions.fetch_add(1, std::memory_order_relaxed); + return lock; +} + +// Collect epochs are per thread and globally unique: every bump takes a fresh value from one counter, +// so a memo entry (stamped with the epoch of its walk) can only match the thread that made it, and +// only until that thread's next ordering point (the queue workers bump at theirs, see the header). A +// thread that never bumped is given a new epoch for every collect: its walks are never reused (the +// presenter, game threads inside the flush hook), as nothing orders the CPU's writes for it. +std::atomic nextCollectEpoch{1}; +thread_local std::uint64_t threadCollectEpoch = 0; + +std::uint64_t currentCollectEpoch() { + if (threadCollectEpoch != 0) return threadCollectEpoch; + return nextCollectEpoch.fetch_add(1, std::memory_order_relaxed) + 1; +} + +bool collectMemoEnabled() { + static const bool disabled = std::getenv("APS5_NO_COLLECT_MEMO") != nullptr; + return !disabled; +} + +// The per-thread memo ring (see WriteTracker::Memo): sized to a packet's working set (a dispatch +// collects each of its few dozen image surfaces in stage A and again in stage B). Entries of a +// thread that never bumps its epoch never match (each collect gets a fresh epoch), as intended. +// APS5_SHARED_COLLECT_MEMO=1 uses the tracker's shared ring under its mutex instead. +struct ThreadCollectMemo { + std::array entries{}; + std::size_t next = 0; +}; +thread_local ThreadCollectMemo threadCollectMemo; + +bool sharedCollectMemo() { + static const bool shared = std::getenv("APS5_SHARED_COLLECT_MEMO") != nullptr; + return shared; +} + +std::uint64_t collectWrites(std::uint64_t address, std::size_t bytes, bool memoized) { + auto& tracker = Tracker(); #ifdef _WIN32 - while (cursor < end) { - MEMORY_BASIC_INFORMATION memory{}; - require(VirtualQuery(reinterpret_cast(cursor), &memory, sizeof(memory)) == sizeof(memory), "cannot query guest memory"); - require(memory.State == MEM_COMMIT && (memory.Protect & (PAGE_GUARD | PAGE_NOACCESS)) == 0, "guest memory is not readable"); - const auto protection = memory.Protect & 0xffu; - require(protection == PAGE_READONLY || protection == PAGE_READWRITE || protection == PAGE_WRITECOPY || protection == PAGE_EXECUTE_READ || protection == PAGE_EXECUTE_READWRITE || protection == PAGE_EXECUTE_WRITECOPY, "guest memory has no read permission"); - const auto base = reinterpret_cast(memory.BaseAddress); - require(memory.RegionSize <= std::numeric_limits::max() - base && base + memory.RegionSize > cursor, "invalid guest memory mapping"); - require(!writable || protection == PAGE_READWRITE || protection == PAGE_WRITECOPY || protection == PAGE_EXECUTE_READWRITE || protection == PAGE_EXECUTE_WRITECOPY, "guest memory has no write permission"); - cursor = std::min(end, base + memory.RegionSize); + // Whole pages, so a page shared with the next range is collected with either. + constexpr std::uint64_t page = 4096; + const auto first = address & ~(page - 1); + const auto stop = (address + bytes + page - 1) & ~(page - 1); + // One epoch for the lookup and the entry made after the walk: an unbumped thread's fresh epoch + // must not differ between them. + const auto epoch = currentCollectEpoch(); + const bool useMemo = memoized && collectMemoEnabled() && bytes != 0; + if (useMemo && !sharedCollectMemo()) { + // An entry exists only for a completed walk of an in-arena range, so the tracker is + // initialized and watched; nothing below the lock needs asking. + for (const auto& entry : threadCollectMemo.entries) { + if (entry.epoch == epoch && entry.begin <= first && stop <= entry.end) { + // The current generation, not the memoized one: blocks stamped since (MarkWritten, an + // overlapping collect) were written before this caller reads, and an older value would + // make every later UnchangedSince fail until the epoch ends. + collectMemoHits.fetch_add(1, std::memory_order_relaxed); + return tracker.generation.load(std::memory_order_relaxed); + } + } } +#endif + const auto lock = lockTracker(tracker); + tracker.initialize(); + if (!tracker.watched || bytes == 0 || address < tracker.base || address - tracker.base > tracker.size - bytes) return 0; +#ifdef _WIN32 + auto cursor = first; + if (useMemo && sharedCollectMemo()) { + for (const auto& entry : tracker.memo) { + if (entry.epoch == epoch && entry.begin <= cursor && stop <= entry.end) { + collectMemoHits.fetch_add(1, std::memory_order_relaxed); + return tracker.generation; + } + } + } + const TimedAccess timed(Profile().collect, bytes); + ++tracker.generation; + // One resetting walk: the kernel reports and clears a page's dirty bit together, a write landing + // after the walk passed a page is reported by the next walk, and a clean page is not touched by the + // reset, so a probe pass first (which doubled the walk of every dirty range) buys nothing. + // APS5_NO_SINGLE_PASS_COLLECT=1 restores the probe pass followed by a resetting pass when dirty. + static const bool singlePass = std::getenv("APS5_NO_SINGLE_PASS_COLLECT") == nullptr; + bool dirty = false; + while (cursor < stop) { + ULONG_PTR count = tracker.pages.size(); + DWORD granularity = 0; + if (GetWriteWatch(singlePass ? WRITE_WATCH_FLAG_RESET : 0, reinterpret_cast(cursor), static_cast(stop - cursor), tracker.pages.data(), &count, &granularity) != 0) { + // Uncommitted pages inside the range make the call fail; such ranges are compared instead. + return 0; + } + if (count != 0) dirty = true; + for (ULONG_PTR i = 0; i < count; ++i) tracker.blocks[(reinterpret_cast(tracker.pages[i]) - tracker.base) / WriteBlockBytes] = tracker.generation; + if (count < tracker.pages.size()) break; + cursor = reinterpret_cast(tracker.pages[count - 1]) + (granularity != 0 ? granularity : page); + } + if (dirty) collectDirty.fetch_add(1, std::memory_order_relaxed); + if (dirty && !singlePass) { + // The resetting pass reports pages again, so a write racing the first pass is stamped too. + cursor = address & ~(page - 1); + while (cursor < stop) { + ULONG_PTR count = tracker.pages.size(); + DWORD granularity = 0; + if (GetWriteWatch(WRITE_WATCH_FLAG_RESET, reinterpret_cast(cursor), static_cast(stop - cursor), tracker.pages.data(), &count, &granularity) != 0) return 0; + for (ULONG_PTR i = 0; i < count; ++i) tracker.blocks[(reinterpret_cast(tracker.pages[i]) - tracker.base) / WriteBlockBytes] = tracker.generation; + if (count < tracker.pages.size()) break; + cursor = reinterpret_cast(tracker.pages[count - 1]) + (granularity != 0 ? granularity : page); + } + } + // Only a completed walk is remembered; a failed one (uncommitted pages) returned 0 above. + if (collectMemoEnabled()) { + if (sharedCollectMemo()) tracker.memo[tracker.nextMemo++ % tracker.memo.size()] = {first, stop, epoch}; + else threadCollectMemo.entries[threadCollectMemo.next++ % threadCollectMemo.entries.size()] = {first, stop, epoch}; + } + return tracker.generation; #else - std::ifstream maps("/proc/self/maps"); - require(maps.is_open(), "cannot query guest memory maps"); - std::string line; - while (cursor < end && std::getline(maps, line)) { - std::istringstream fields(line); - std::uintptr_t first = 0; - std::uintptr_t last = 0; - char separator = 0; - std::string permissions; - require(static_cast(fields >> std::hex >> first >> separator >> last >> permissions) && separator == '-' && first < last && !permissions.empty(), "invalid guest memory map entry"); - if (last <= cursor) continue; - require(first <= cursor && permissions[0] == 'r', "guest memory is not readable"); - require(!writable || (permissions.size() > 1 && permissions[1] == 'w'), "guest memory has no write permission"); - cursor = std::min(end, last); - } - require(cursor == end, "guest address range is not mapped"); + static_cast(memoized); + static_cast(threadCollectMemo); + return 0; #endif } +} + +std::uint64_t CollectWrites(std::uint64_t address, std::size_t bytes) { + return collectWrites(address, bytes, true); +} + +std::uint64_t CollectWritesUncached(std::uint64_t address, std::size_t bytes) { + return collectWrites(address, bytes, false); +} + +void BumpCollectEpoch() { + threadCollectEpoch = nextCollectEpoch.fetch_add(1, std::memory_order_relaxed) + 1; + collectEpochBumps.fetch_add(1, std::memory_order_relaxed); +} + +std::uint64_t CollectEpochBumps() { + return collectEpochBumps.load(std::memory_order_relaxed); +} + +bool UnchangedSince(std::uint64_t address, std::size_t bytes, std::uint64_t generation) { + auto& tracker = Tracker(); + const auto lock = lockTracker(tracker); + tracker.initialize(); + if (!tracker.watched || generation == 0 || bytes == 0 || address < tracker.base || address - tracker.base > tracker.size - bytes) return false; + const auto first = (address - tracker.base) / WriteBlockBytes; + const auto last = (address - tracker.base + bytes - 1) / WriteBlockBytes; + for (auto block = first; block <= last; ++block) { + if (tracker.blocks[block] > generation) return false; + } + return true; +} + +void MarkWritten(std::uint64_t address, std::size_t bytes) { + auto& tracker = Tracker(); + const auto lock = lockTracker(tracker); + tracker.initialize(); + if (!tracker.watched || bytes == 0 || address < tracker.base || address - tracker.base > tracker.size - bytes) return; + const auto first = (address - tracker.base) / WriteBlockBytes; + const auto last = (address - tracker.base + bytes - 1) / WriteBlockBytes; + ++tracker.generation; + for (auto block = first; block <= last; ++block) tracker.blocks[block] = tracker.generation; +} + +namespace { + +// Per-thread GpuMutex wait accounting (APS5_PROFILE_DRAW): a queue worker tags its thread, other +// threads (the presenter) report untagged. +constexpr std::size_t GpuLockSiteCount = static_cast(GpuLockSite::Count); +constexpr const char* GpuLockSiteNames[GpuLockSiteCount] = {"other", "dispatch", "indirect", "draw", "hook", "wait", "label", "flush", "present", "fill", "try"}; + +struct GpuLockStats { + std::uint32_t tag = 0xffffffffu; + std::uint64_t acquisitions = 0; + std::uint64_t waits = 0; + double waitedMs = 0; + // The same split by the site tagged before the acquisition (TagGpuLockSite); the tag of the + // next lock() is kept here and consumed by it. + std::array siteAcquisitions{}; + std::array siteWaits{}; + std::array siteWaitedMs{}; + GpuLockSite nextSite = GpuLockSite::Other; + // Holds: the outermost (depth 0 -> 1) acquisition of this thread until the matching unlock, + // charged to that acquisition's site (a nested hook lock inside a dispatch's hold extends the + // dispatch's hold). The waits above say who waits; these say whom they wait for. + std::array siteHolds{}; + std::array siteHeldMs{}; + std::array siteMaxHeldMs{}; + std::size_t holdSite = 0; + std::chrono::steady_clock::time_point holdStart; + std::chrono::steady_clock::time_point lastReport = std::chrono::steady_clock::now(); +}; + +GpuLockStats& LockStats() { + thread_local GpuLockStats stats; + return stats; +} + +bool GpuLockProfiled() { + static const bool profiled = std::getenv("APS5_PROFILE_DRAW") != nullptr && std::getenv("APS5_NO_LOCK_PROFILE") == nullptr; + return profiled; +} + +// Hold timing costs two clock reads per outermost acquisition (~20000 per 10 s on queue 0, well +// under a millisecond); APS5_NO_HOLD_PROFILE=1 keeps only the wait timing. +bool GpuHoldProfiled() { + static const bool profiled = GpuLockProfiled() && std::getenv("APS5_NO_HOLD_PROFILE") == nullptr; + return profiled; +} + +void BeginHold(std::size_t site) { + auto& stats = LockStats(); + stats.holdSite = site; + stats.holdStart = std::chrono::steady_clock::now(); +} + +void EndHold() { + auto& stats = LockStats(); + const auto heldMs = std::chrono::duration(std::chrono::steady_clock::now() - stats.holdStart).count(); + ++stats.siteHolds[stats.holdSite]; + stats.siteHeldMs[stats.holdSite] += heldMs; + stats.siteMaxHeldMs[stats.holdSite] = std::max(stats.siteMaxHeldMs[stats.holdSite], heldMs); +} + +} + +// The owner token is a thread-local's address: unique per live thread and free to read. +namespace { +const void* ThreadToken() { + thread_local char token; + return &token; +} +} + +void GpuMutexType::acquired() { + owner.store(ThreadToken(), std::memory_order_relaxed); + ++depth; +} + +bool GpuMutexType::try_lock() { + if (!GpuLockProfiled()) { + if (!mutex.try_lock()) return false; + acquired(); + return true; + } + // The tag is consumed whether or not the try succeeds (a failed try must not name the next + // acquisition); an untagged try's hold is charged to 'try' (a boundary reap, an idle worker). + auto& stats = LockStats(); + const auto tagged = std::exchange(stats.nextSite, GpuLockSite::Other); + if (!mutex.try_lock()) return false; + acquired(); + if (depth == 1 && GpuHoldProfiled()) BeginHold(static_cast(tagged == GpuLockSite::Other ? GpuLockSite::Try : tagged)); + return true; +} + +namespace { +std::atomic gpuUnlockHook{nullptr}; +} + +void SetGpuUnlockHook(void (*hook)()) { + gpuUnlockHook.store(hook, std::memory_order_release); +} + +void GpuMutexType::unlock() { + const bool outermost = --depth == 0; + if (outermost) { + owner.store(nullptr, std::memory_order_relaxed); + // Still the holder here: the hold ends when the mutex is given up, not before. + if (GpuHoldProfiled()) EndHold(); + } + mutex.unlock(); + // After the release, so what the hook does (destroying a batch's kept objects) is neither part + // of the hold nor waited for by the next holder. + if (outermost) { + if (auto* hook = gpuUnlockHook.load(std::memory_order_acquire); hook != nullptr) hook(); + } +} + +bool GpuMutexType::HeldByThisThread() const { + return owner.load(std::memory_order_relaxed) == ThreadToken(); +} + +std::uint32_t GpuMutexType::DepthOnThisThread() const { + // `depth` is written by the holder only, so it is read only when that is this thread. + return HeldByThisThread() ? depth : 0; +} + +void AssertGpuLockHeld(const char* where) { + static const bool enabled = std::getenv("APS5_ASSERT_GPU_LOCK") != nullptr; + if (!enabled || GpuMutex().HeldByThisThread()) return; + std::fprintf(stderr, "[lock] ASSERT: %s reached without GuestMemory::GpuMutex on thread tagged 0x%x\n", where, GpuLockThreadTag()); + std::abort(); +} + +void GpuMutexType::lock() { + if (!GpuLockProfiled()) { + mutex.lock(); + acquired(); + return; + } + auto& stats = LockStats(); + ++stats.acquisitions; + const auto site = static_cast(std::exchange(stats.nextSite, GpuLockSite::Other)); + ++stats.siteAcquisitions[site]; + // An uncontended (or recursive) acquisition costs no clock reads for the wait; only a wait is + // timed. The hold timer (outermost acquisitions only) starts once the mutex is held. + if (!mutex.try_lock()) { + const auto start = std::chrono::steady_clock::now(); + mutex.lock(); + acquired(); + const auto now = std::chrono::steady_clock::now(); + if (depth == 1 && GpuHoldProfiled()) { + stats.holdSite = site; + stats.holdStart = now; + } + const auto waitedMs = std::chrono::duration(now - start).count(); + stats.waitedMs += waitedMs; + ++stats.waits; + stats.siteWaitedMs[site] += waitedMs; + ++stats.siteWaits[site]; + if (now - stats.lastReport < std::chrono::seconds(10)) return; + // The line is printed from inside a contended acquisition, so a thread that rarely waits + // reports a longer span than 10 s: the real interval is printed. + const auto interval = std::chrono::duration(now - stats.lastReport).count(); + stats.lastReport = now; + std::string sites; + for (std::size_t i = 0; i < GpuLockSiteCount; ++i) { + if (stats.siteAcquisitions[i] == 0) continue; + char text[96]; + std::snprintf(text, sizeof(text), " %s %llu/%llu %.0f ms", GpuLockSiteNames[i], static_cast(stats.siteAcquisitions[i]), static_cast(stats.siteWaits[i]), stats.siteWaitedMs[i]); + sites += text; + } + // Holds by site, longest total first: the sites whose holds the other threads wait behind. + std::string holds; + if (GpuHoldProfiled()) { + std::array order{}; + for (std::size_t i = 0; i < GpuLockSiteCount; ++i) order[i] = i; + std::sort(order.begin(), order.end(), [&](std::size_t a, std::size_t b) { return stats.siteHeldMs[a] > stats.siteHeldMs[b]; }); + double heldMs = 0; + for (std::size_t i = 0; i < GpuLockSiteCount; ++i) heldMs += stats.siteHeldMs[i]; + char text[112]; + std::snprintf(text, sizeof(text), "; held %.0f ms in total, by site (holds, held, max):", heldMs); + holds += text; + for (const auto i : order) { + if (stats.siteHolds[i] == 0) continue; + std::snprintf(text, sizeof(text), " %s %llu %.0f ms (max %.1f)", GpuLockSiteNames[i], static_cast(stats.siteHolds[i]), stats.siteHeldMs[i], stats.siteMaxHeldMs[i]); + holds += text; + } + } + if (stats.tag == 0xffffffffu) std::fprintf(stderr, "[lock] untagged thread waited %.0f ms for the GPU mutex in %llu of %llu acquisitions (%.0f s); by site (acquisitions/waits, waited):%s%s\n", stats.waitedMs, static_cast(stats.waits), static_cast(stats.acquisitions), interval, sites.c_str(), holds.c_str()); + else std::fprintf(stderr, "[lock] queue 0x%x waited %.0f ms for the GPU mutex in %llu of %llu acquisitions (%.0f s); by site (acquisitions/waits, waited):%s%s\n", stats.tag, stats.waitedMs, static_cast(stats.waits), static_cast(stats.acquisitions), interval, sites.c_str(), holds.c_str()); + stats.waitedMs = 0; + stats.waits = 0; + stats.acquisitions = 0; + stats.siteAcquisitions.fill(0); + stats.siteWaits.fill(0); + stats.siteWaitedMs.fill(0); + stats.siteHolds.fill(0); + stats.siteHeldMs.fill(0); + stats.siteMaxHeldMs.fill(0); + return; + } + acquired(); + if (depth == 1 && GpuHoldProfiled()) BeginHold(site); +} + +void TagGpuLockSite(GpuLockSite site) { + // Off the profiled path the tag is never read, so the thread-local is not touched either. + if (!GpuLockProfiled()) return; + LockStats().nextSite = site; +} + +std::uint32_t GpuLockThreadTag() { + return LockStats().tag; +} + +GpuMutexType& GpuMutex() { + static GpuMutexType mutex; + return mutex; +} + +void TagGpuLockThread(std::uint32_t queue) { + LockStats().tag = queue; +} + +unsigned long long CodeOffset(const void* address) { + return ModuleOffset(address); +} + +namespace { +std::atomic flushHook{nullptr}; +} + +void SetFlushHook(void (*hook)(std::uint64_t, std::size_t)) { + flushHook.store(hook, std::memory_order_release); +} + +void FlushGpuWrites(std::uint64_t address, std::size_t bytes) { + if (const auto hook = flushHook.load(std::memory_order_acquire)) hook(address, bytes); +} + void Read(std::uint64_t address, std::span destination, std::size_t alignment) { - PerformanceTimer timing("GuestMemory.Read"); if (destination.empty()) return; + FlushGpuWrites(address, destination.size()); + const TimedAccess timed(Profile().read, destination.size()); + CountReadCaller(__builtin_return_address(0)); const auto* source = reinterpret_cast(address); - CheckRange(source, destination.size(), alignment); - timing.Mark("range_check"); + try { + CheckRange(source, destination.size(), alignment); + } catch (const std::runtime_error&) { + static const bool trace = std::getenv("APS5_TRACE_UNREADABLE") != nullptr; + if (trace) std::fprintf(stderr, "[unreadable] read from +0x%llx\n", ModuleOffset(__builtin_return_address(0))); + throw; + } std::memcpy(destination.data(), source, destination.size()); - timing.Mark("copy", destination.size()); +} + +void ReadCommitted(std::uint64_t address, std::span destination) { + if (destination.empty()) return; + FlushGpuWrites(address, destination.size()); + const TimedAccess timed(Profile().read, destination.size()); + CountReadCaller(__builtin_return_address(0)); + if (Accessible(reinterpret_cast(address), destination.size())) { + std::memcpy(destination.data(), reinterpret_cast(address), destination.size()); + return; + } + std::memset(destination.data(), 0, destination.size()); + for (const auto& [begin, end] : CommittedRanges(address, destination.size())) std::memcpy(destination.data() + (begin - address), reinterpret_cast(begin), static_cast(end - begin)); +} + +bool EqualsCommitted(std::uint64_t address, std::span bytes) { + if (bytes.empty()) return true; + FlushGpuWrites(address, bytes.size()); + const TimedAccess timed(Profile().compare, bytes.size()); + if (Accessible(reinterpret_cast(address), bytes.size())) return std::memcmp(reinterpret_cast(address), bytes.data(), bytes.size()) == 0; + for (const auto& [begin, end] : CommittedRanges(address, bytes.size())) { + if (std::memcmp(reinterpret_cast(begin), bytes.data() + (begin - address), static_cast(end - begin)) != 0) return false; + } + return true; +} + +void WriteChangedCommitted(std::uint64_t address, std::span current, std::span original) { + require(current.size() == original.size(), "write-back snapshot sizes differ"); + if (Accessible(reinterpret_cast(address), current.size(), true)) { + WriteChanged(address, current, original); + return; + } + for (const auto& [begin, end] : CommittedRanges(address, current.size(), true)) { + const auto offset = static_cast(begin - address); + const auto length = static_cast(end - begin); + WriteChanged(begin, current.subspan(offset, length), original.subspan(offset, length)); + } +} + +void WriteChanged(std::uint64_t address, std::span current, std::span original) { + require(current.size() == original.size(), "write-back snapshot sizes differ"); + if (current.empty()) return; + // A store: the hook sync it makes orders this write after a pending GPU write, whatever a read + // site above it was doing (the [hooksync] line counts stores apart). + const ReadSiteScope site(ReadSite::Store); + FlushGpuWrites(address, current.size()); + const TimedAccess timed(Profile().changedWrite, current.size()); + auto* destination = reinterpret_cast(address); + CheckRange(destination, current.size(), 1, true); + constexpr std::size_t block = 256; + const auto size = current.size(); + const auto differs = [&](std::size_t at) { + const auto length = std::min(block, size - at); + return std::memcmp(current.data() + at, original.data() + at, length) != 0; + }; + std::size_t firstChanged = size; + std::size_t lastChanged = 0; + for (std::size_t at = 0; at < size; at += block) { + if (!differs(at)) continue; + std::memcpy(destination + at, current.data() + at, std::min(block, size - at)); + firstChanged = std::min(firstChanged, at); + lastChanged = std::min(at + block, size); + } + // Stamped like a GPU write: a collect memoized for this packet would not see the page fault. + if (firstChanged < lastChanged) MarkWritten(address + firstChanged, lastChanged - firstChanged); } void Write(std::uint64_t address, std::span source, std::size_t alignment) { - PerformanceTimer timing("GuestMemory.Write"); if (source.empty()) return; + // A store, as in WriteChanged. + const ReadSiteScope site(ReadSite::Store); + FlushGpuWrites(address, source.size()); + const TimedAccess timed(Profile().write, source.size()); + CountWriteCaller(__builtin_return_address(0)); auto* destination = reinterpret_cast(address); CheckRange(destination, source.size(), alignment, true); - timing.Mark("range_check"); std::memcpy(destination, source.data(), source.size()); - timing.Mark("copy", source.size()); + // Stamped like a GPU write: a collect memoized for this packet would not see the page fault. + MarkWritten(address, source.size()); } } diff --git a/core/libs/prx/libSceAgcDriver/Execution/src/Pm4.cpp b/core/libs/prx/libSceAgcDriver/Execution/src/Pm4.cpp index af771512d..6e23d0166 100644 --- a/core/libs/prx/libSceAgcDriver/Execution/src/Pm4.cpp +++ b/core/libs/prx/libSceAgcDriver/Execution/src/Pm4.cpp @@ -2,14 +2,23 @@ #include "prx/libSceAgcDriver/Execution/include/GuestMemory.hpp" #include "prx/libc/include/General.hpp" #include +#include +#include #include #include #include #include +#include namespace AgcDriver::Pm4 { namespace { +std::string ToHex(std::uint32_t value) { + char buffer[16]; + std::snprintf(buffer, sizeof(buffer), "%x", value); + return buffer; +} + void require(bool condition, const char* reason) { if (!condition) throw std::runtime_error(reason); } @@ -25,6 +34,13 @@ std::uint32_t registerOffset(std::uint32_t value) { return offset; } +// Debug aid: APS5_TRACE_CONTEXT_STATE logs context-state operations and the context registers +// written while a context is pushed. Read once: getenv scans the environment under a lock. +bool TraceContextState() { + static const bool value = std::getenv("APS5_TRACE_CONTEXT_STATE") != nullptr; + return value; +} + Registers& registersFor(QueueState& queue, std::uint32_t opcode) { if (opcode == 0x69 || opcode == 0x9f) return queue.context; if (opcode == 0x76 || opcode == 0x63) return queue.shader; @@ -32,7 +48,10 @@ Registers& registersFor(QueueState& queue, std::uint32_t opcode) { } void writeRegister(QueueState& queue, std::uint32_t opcode, std::uint32_t offset, std::uint32_t value) { + if ((opcode == 0x69 || opcode == 0x9f) && (offset == 0x8e || offset == 0x8f || offset == 0x318 || offset == 0x31b || offset == 0x31c || offset == 0x31d || offset == 0x390 || offset == 0x3b0 || offset == 0x3b8)) + APS5_LOG_OUT_DEBUG("CONTEXT WRITE opcode=0x%x offset=0x%x value=0x%x", opcode, offset, value); registersFor(queue, opcode).insert_or_assign(offset, value); + if (TraceContextState() && (opcode == 0x69 || opcode == 0x9f) && ((offset >= 0x318 && offset < 0x318 + 8 * 0xf && (offset - 0x318) % 0xf == 0) || offset == 0x8e)) std::fprintf(stderr, "[context] write %x = %08x (0x%x)\n", offset, value, opcode); if ((opcode == 0x64 || opcode == 0x79 || opcode == 0x7a) && offset == 0x243) queue.indexType = value & 3u; } @@ -48,9 +67,9 @@ std::uint32_t dmaDestination(std::span packet) { return ((packet[1] >> 20u) & 3u) | ((packet[6] >> 25u) & 4u) | ((packet[6] >> 26u) & 8u); } -void copyMemory(std::uint64_t source, std::uint64_t destination, std::size_t bytes, bool immediate) { - if (bytes == 0) return; - GuestMemory::CheckRange(reinterpret_cast(destination), bytes, 1, true); +// The bytes a COPY_DATA or DMA_DATA stores: an immediate is a repeating 32-bit pattern, a memory +// source is read through the checked path (it waits for recorded GPU work that writes it). +std::vector copySource(std::uint64_t source, std::size_t bytes, bool immediate) { std::vector data(bytes); if (immediate) { const auto value = static_cast(source); @@ -58,12 +77,19 @@ void copyMemory(std::uint64_t source, std::uint64_t destination, std::size_t byt } else { GuestMemory::Read(source, data); } - GuestMemory::Write(destination, data); + return data; +} + +void copyMemory(std::uint64_t source, std::uint64_t destination, std::size_t bytes, bool immediate) { + if (bytes == 0) return; + GuestMemory::CheckRange(reinterpret_cast(destination), bytes, 1, true); + GuestMemory::Write(destination, copySource(source, bytes, immediate)); } } std::string Name(std::uint32_t header) { + if (FillerPacket(header)) return "PM4_FILLER"; const auto opcode = (header >> 8u) & 0xffu; if (opcode == 0x10 && (header & 0xfcu) != 0) { switch ((header >> 2u) & 0x3fu) { @@ -107,11 +133,13 @@ std::string_view UnsupportedReason(std::uint32_t header) { case 0x20: return "GPU query predication is not implemented"; case 0x22: return "conditional command execution and conditional flip reservation are not implemented"; case 0x33: case 0x3f: return "nested command buffers, branching and nested flip reservation are not implemented"; - case 0x39: case 0x3c: case 0x59: case 0x93: + case 0x3c: case 0x93: return {}; + case 0x39: case 0x59: return "cooperative command-queue waits are not implemented"; case 0x84: case 0x85: case 0x86: case 0x88: return "separate CE/DE execution and counter synchronization are not implemented"; - case 0x43: case 0x47: case 0x48: case 0x49: + case 0x49: return {}; + case 0x43: case 0x47: case 0x48: return "guest cache actions, GPU events and interrupt delivery are not implemented"; case 0x8e: return "GPU LOD statistics are not implemented; synthetic results are forbidden"; case 0x28: case 0x41: case 0x68: case 0x78: @@ -121,8 +149,13 @@ std::string_view UnsupportedReason(std::uint32_t header) { } void Validate(std::span packet, std::uint32_t queue) { - require(packet.size() >= 2, "truncated PM4 header or payload"); + require(!packet.empty(), "truncated PM4 header"); const auto header = packet[0]; + if (FillerPacket(header)) { + require(packet.size() == 1, "invalid PM4 filler size"); + return; + } + require(packet.size() >= 2, "truncated PM4 header or payload"); require((header & 0xc0000000u) == 0xc0000000u, "unsupported PM4 packet type"); require(packet.size() == ((header >> 16u) & 0x3fffu) + 2u, "invalid PM4 packet size"); const auto opcode = (header >> 8u) & 0xffu; @@ -153,7 +186,13 @@ void Validate(std::span packet, std::uint32_t queue) { } return; } - require((header & 0xffu) == 0 || (opcode == 0x11 && (header & 0xffu) == 2), "PM4 header flags are not implemented"); + // Header bit 1 selects the compute shader type and bit 2 (register writes) resets the filter CAM; + // neither changes what the packet writes. Predication (bit 0) is not implemented. + const auto flags = header & 0xffu; + const bool registerWrite = opcode == 0x69 || opcode == 0x76 || opcode == 0x79 || opcode == 0x7a; + auto allowedFlags = opcode == 0x11 ? 2u : registerWrite ? 6u : opcode == 0x3c || opcode == 0x93 ? 2u : 0u; + if (IsTagMarker(packet)) allowedFlags |= 1u; + if ((flags & ~allowedFlags) != 0) throw std::runtime_error("PM4 header flags 0x" + ToHex(flags) + " are not implemented"); switch (opcode) { case 0x11: size(4); @@ -175,10 +214,10 @@ void Validate(std::span packet, std::uint32_t queue) { require(packet[3] <= packet[1], "index count exceeds maximum index size"); require((packet[4] & ~0x20u) == 0, "unsupported indexed draw flags"); break; - case 0x15: size(5); require((packet[4] & ~0x8000u) == 0x41u, "dispatch modifiers are not implemented"); break; + case 0x15: size(5); if ((packet[4] & ~0x8020u) != 0x41u) throw std::runtime_error("dispatch modifiers 0x" + ToHex(packet[4]) + " are not implemented"); break; case 0x16: require(packet.size() == 3 || packet.size() == 4, "invalid indirect dispatch size"); - require((packet.back() & ~0x8000u) == 0x41u, "indirect dispatch modifiers are not implemented"); + require((packet.back() & ~0x8020u) == 0x41u, "indirect dispatch modifiers are not implemented"); break; case 0x42: size(2); require(packet[1] == 0, "unsupported PFP_SYNC_ME payload"); break; case 0x46: { @@ -222,6 +261,7 @@ void Validate(std::span packet, std::uint32_t queue) { require((packet[1] & 3u) == 0 && packet[3] == 0x80000000u && packet[4] <= 0x3fffu, "unsupported indirect-register address or control fields"); break; case 0x69: case 0x76: case 0x79: case 0x7a: { + if (IsTagMarker(packet)) break; if (opcode != 0x76) graphics(); require(packet.size() >= 3, "register packet has no values"); if (opcode == 0x7a) require((packet[1] & 0xf0000000u) == 0 || (packet.size() == 3 && packet[1] == 0x20000243u), "indexed register bank selection is not implemented"); @@ -229,6 +269,20 @@ void Validate(std::span packet, std::uint32_t queue) { require(packet.size() - 2 <= 0x10000u - offset, "register range overflow"); break; } + case 0x3c: case 0x93: { + size(opcode == 0x3c ? 7 : 9); + require((packet[1] & 0x10u) != 0, "register-space WAIT_REG_MEM is not implemented"); + require((packet[1] & 7u) <= 6u, "invalid WAIT_REG_MEM compare function"); + require((packet[2] & (opcode == 0x3c ? 3u : 7u)) == 0, "misaligned WAIT_REG_MEM address"); + break; + } + case 0x49: { + size(8); + const auto dataSelect = packet[2] >> 29u; + require(dataSelect <= 3, "GDS RELEASE_MEM data is not implemented"); + require(dataSelect == 0 || address(packet[3], packet[4]) == 0 || (packet[3] & (dataSelect == 1 ? 3u : 7u)) == 0, "misaligned RELEASE_MEM destination"); + break; + } case 0x81: graphics(); require(packet[1] <= 0xbffcu && (packet[1] & 3u) == 0 && packet.size() - 2 <= 0x3000u - packet[1] / 4u, "constant RAM write range overflow or misalignment"); @@ -266,6 +320,111 @@ void Validate(std::span packet, std::uint32_t queue) { } } +bool IsTagMarker(std::span packet) { + // libSceAgc brackets waits and draws with predicated SET_UCONFIG_REG writes of a tag value to + // 0x342; they carry no register state. + return packet.size() >= 3 && ((packet[0] >> 8u) & 0xffu) == 0x79u && (packet[0] & 1u) != 0 && packet[1] == 0x342u; +} + +namespace { + +bool waitCompares(std::span packet, bool wide, std::uint64_t value) { + const std::uint64_t reference = wide ? address(packet[4], packet[5]) : packet[4]; + const std::uint64_t mask = wide ? address(packet[6], packet[7]) : packet[5]; + const auto masked = value & mask; + switch (packet[1] & 7u) { + case 0: return true; + case 1: return masked < reference; + case 2: return masked <= reference; + case 3: return masked == reference; + case 4: return masked != reference; + case 5: return masked >= reference; + case 6: return masked > reference; + default: return false; + } +} + +} + +bool WaitSatisfied(std::span packet) { + const bool wide = ((packet[0] >> 8u) & 0xffu) == 0x93u; + const auto source = address(packet[2], packet[3]); + std::uint64_t value = 0; + // Named for the [hooksync] attribution (the read goes through the flush hook). + const GuestMemory::ReadSiteScope site(GuestMemory::ReadSite::Wait); + GuestMemory::Read(source, std::as_writable_bytes(std::span(&value, 1)).first(wide ? 8 : 4), wide ? 8 : 4); + return waitCompares(packet, wide, value); +} + +bool WaitSatisfiedUnchecked(std::span packet) { + const bool wide = ((packet[0] >> 8u) & 0xffu) == 0x93u; + const auto source = address(packet[2], packet[3]); + const std::uint64_t value = wide ? *reinterpret_cast(source) : *reinterpret_cast(source); + return waitCompares(packet, wide, value); +} + +bool WaitComparesValue(std::span packet, std::uint64_t value) { + const bool wide = ((packet[0] >> 8u) & 0xffu) == 0x93u; + return waitCompares(packet, wide, value); +} + +std::optional DecodeLabelWrite(std::span packet) { + const auto opcode = (packet[0] >> 8u) & 0xffu; + if (opcode == 0x49 && packet.size() >= 7) { + const auto dataSelect = packet[2] >> 29u; + const auto destination = address(packet[3], packet[4]); + if (dataSelect == 0 || dataSelect > 3) return std::nullopt; + if (destination == 0) return std::nullopt; + std::uint64_t value = address(packet[5], packet[6]); + if (dataSelect == 3) value = static_cast(std::chrono::duration_cast(std::chrono::steady_clock::now().time_since_epoch()).count() / 10); + LabelWrite label{destination, {}}; + label.inlineSize = dataSelect == 1 ? 4 : 8; + std::memcpy(label.inlineBytes.data(), &value, label.inlineSize); + return label; + } + if (opcode == 0x37 && packet.size() > 4) { + const auto destination = address(packet[2], packet[3]); + if (destination == 0) return std::nullopt; + // WR_ONE_ADDR stores every value at the same address: the last one remains. + const auto words = (packet[1] & 0x10000u) != 0 ? packet.last(1) : packet.subspan(4); + return LabelWrite{destination, std::as_bytes(words)}; + } + return std::nullopt; +} + +std::optional ResolveStore(std::span packet, const QueueState& queue, std::size_t limit) { + // The same decode as Execute's cases below (a packet Validate accepted): a store that is not + // resolved here still runs there. The size and alignment tests precede any memory read: a store + // the GPU path would reject anyway must not read (and sync for) its source here and again there. + const auto fits = [limit](std::uint64_t destination, std::size_t bytes) { return bytes <= limit && destination % 4 == 0 && bytes % 4 == 0; }; + switch ((packet[0] >> 8u) & 0xffu) { + case 0x40: { + if (packet.size() < 6) return std::nullopt; + const auto source = ((packet[1] & 0xfu) << 1u) | ((packet[1] >> 30u) & 1u); + const std::size_t bytes = (packet[1] & 0x10000u) != 0 ? 8 : 4; + const auto destination = address(packet[4], packet[5]); + if (!fits(destination, bytes)) return std::nullopt; + return StoreWrite{destination, {}, copySource(address(packet[2], packet[3]), bytes, source >= 10)}; + } + case 0x50: { + if (packet.size() < 7) return std::nullopt; + const std::size_t bytes = packet[6] & 0x3ffffffu; + const auto destination = address(packet[4], packet[5]); + if (!fits(destination, bytes)) return std::nullopt; + return StoreWrite{destination, {}, copySource(address(packet[2], packet[3]), bytes, dmaSource(packet) == 2)}; + } + case 0x83: { + if (packet.size() < 5 || packet[1] / 4 > queue.constantRam.size() || packet[2] > queue.constantRam.size() - packet[1] / 4) return std::nullopt; + const auto words = std::span(queue.constantRam).subspan(packet[1] / 4, packet[2]); + if (words.size_bytes() > limit) return std::nullopt; + // A view of the queue's constant RAM: valid until the next CONST_RAM write on this queue, + // which is a later packet of the same worker. + return StoreWrite{address(packet[3], packet[4]), std::as_bytes(words), {}}; + } + default: return std::nullopt; + } +} + bool UsesGpuCacheBarrier(std::span packet) { require(!packet.empty() && ((packet[0] >> 8u) & 0xffu) == 0x58, "cache barrier requires ACQUIRE_MEM"); Validate(packet, 0); @@ -274,24 +433,30 @@ bool UsesGpuCacheBarrier(std::span packet) { bool AccessesMemory(std::uint32_t header) { switch ((header >> 8u) & 0xffu) { - case 0x16: case 0x2d: case 0x35: case 0x37: case 0x40: case 0x50: case 0x63: case 0x64: case 0x83: case 0x9f: return true; + case 0x3c: case 0x93: case 0x49: case 0x16: case 0x2d: case 0x35: case 0x37: case 0x40: case 0x50: case 0x63: case 0x64: case 0x83: case 0x9f: return true; default: return false; } } -std::array ResolveDispatch(std::span packet, const QueueState& queue) { - std::uint64_t source = 0; - if (packet.size() == 4) source = address(packet[1], packet[2]); - else { - require(queue.dispatchIndirectBase != 0, "indirect dispatch base has not been set"); - require(packet[1] <= std::numeric_limits::max() - queue.dispatchIndirectBase, "indirect dispatch address overflow"); - source = queue.dispatchIndirectBase + packet[1]; - } - std::array result{0xc0031500u, 0, 0, 0, packet.back()}; - GuestMemory::Read(source, std::as_writable_bytes(std::span(result).subspan(1, 3)), 4); +std::uint64_t DispatchArgumentAddress(std::span packet, const QueueState& queue) { + if (packet.size() == 4) return address(packet[1], packet[2]); + require(queue.dispatchIndirectBase != 0, "indirect dispatch base has not been set"); + require(packet[1] <= std::numeric_limits::max() - queue.dispatchIndirectBase, "indirect dispatch address overflow"); + return queue.dispatchIndirectBase + packet[1]; +} + +std::array ReadDispatchArguments(std::uint64_t arguments, std::uint32_t initiator) { + std::array result{0xc0031500u, 0, 0, 0, initiator}; + // Named for the [hooksync] attribution (the read goes through the flush hook). + const GuestMemory::ReadSiteScope site(GuestMemory::ReadSite::IndirectArguments); + GuestMemory::Read(arguments, std::as_writable_bytes(std::span(result).subspan(1, 3)), 4); return result; } +std::array ResolveDispatch(std::span packet, const QueueState& queue) { + return ReadDispatchArguments(DispatchArgumentAddress(packet, queue), packet.back()); +} + DrawParameters ResolveDraw(std::span packet, const QueueState& queue) { Validate(packet, 0); if (((packet[0] >> 8u) & 0xffu) == 0x2d) { @@ -311,6 +476,7 @@ DrawParameters ResolveDraw(std::span packet, const QueueSta const auto bytes = static_cast(packet[3]) * indexSize; require(bytes <= std::numeric_limits::max(), "index range size overflow"); GuestMemory::CheckRange(reinterpret_cast(address), static_cast(bytes), indexSize); + APS5_LOG_OUT_DEBUG("ResolveDraw context targetMask=0x%x shaderMask=0x%x indexCount=%u indexType=%u instances=%u", queue.context.contains(0x8e) ? queue.context.at(0x8e) : 0u, queue.context.contains(0x8f) ? queue.context.at(0x8f) : 0u, packet[3], queue.indexType, queue.instanceCount); return {address, packet[3], indexSize, queue.instanceCount, packet[4]}; } @@ -327,6 +493,8 @@ void Execute(std::span packet, QueueState& queue) { queue.markers.pop_back(); return; case 0x1a: + if (TraceContextState()) std::fprintf(stderr, "[context] op %u: %zu registers, saved %d, cb0 %x info %x mask %x\n", packet[1], queue.context.size(), queue.savedContext.has_value() ? 1 : 0, queue.context.contains(0x318) ? queue.context.at(0x318) : 0u, queue.context.contains(0x31c) ? queue.context.at(0x31c) : 0u, queue.context.contains(0x8e) ? queue.context.at(0x8e) : 0u); + APS5_LOG_OUT_DEBUG("CONTEXT_STATE operation=%u targetMaskBefore=0x%x shaderMaskBefore=0x%x", packet[1], queue.context.contains(0x8e) ? queue.context.at(0x8e) : 0u, queue.context.contains(0x8f) ? queue.context.at(0x8f) : 0u); switch (packet[1]) { case 0: queue.ClearContext(); break; case 1: case 3: @@ -340,6 +508,7 @@ void Execute(std::span packet, QueueState& queue) { queue.savedContext.reset(); break; } + APS5_LOG_OUT_DEBUG("CONTEXT_STATE done operation=%u targetMaskAfter=0x%x shaderMaskAfter=0x%x", packet[1], queue.context.contains(0x8e) ? queue.context.at(0x8e) : 0u, queue.context.contains(0x8f) ? queue.context.at(0x8f) : 0u); return; default: throw std::runtime_error("custom packet requires driver execution"); } @@ -347,6 +516,7 @@ void Execute(std::span packet, QueueState& queue) { ((packet[0] & 2u) == 0 ? queue.drawIndirectBase : queue.dispatchIndirectBase) = address(packet[2], packet[3]); return; case 0x12: + APS5_LOG_OUT_DEBUG("CLEAR_STATE targetMaskBefore=0x%x shaderMaskBefore=0x%x", queue.context.contains(0x8e) ? queue.context.at(0x8e) : 0u, queue.context.contains(0x8f) ? queue.context.at(0x8f) : 0u); queue.ClearContext(); return; case 0x13: queue.indexBufferSize = packet[1]; return; case 0x26: queue.indexBase = address(packet[1], packet[2]); return; @@ -354,12 +524,47 @@ void Execute(std::span packet, QueueState& queue) { case 0x2f: queue.instanceCount = packet[1]; return; case 0x63: case 0x64: case 0x9f: { std::vector pairs(static_cast(packet[4]) * 2); + // Named for the [hooksync] attribution (the read goes through the flush hook). + const GuestMemory::ReadSiteScope site(GuestMemory::ReadSite::Registers); GuestMemory::Read(address(packet[1], packet[2]), std::as_writable_bytes(std::span(pairs)), 4); for (std::size_t i = 0; i < pairs.size(); i += 2) registerOffset(pairs[i]); for (std::size_t i = 0; i < pairs.size(); i += 2) writeRegister(queue, opcode, registerOffset(pairs[i]), pairs[i + 1]); + if (queue.savedContext.has_value() && TraceContextState()) { + std::fprintf(stderr, "[context] indirect 0x%x:", opcode); + for (std::size_t i = 0; i < pairs.size(); i += 2) std::fprintf(stderr, " %x", registerOffset(pairs[i])); + std::fprintf(stderr, "\n"); + } + return; + } + case 0x3c: case 0x93: { + // The waited-on value is written by the CPU or another queue; poll it like the CP would. + const auto start = std::chrono::steady_clock::now(); + bool warned = false; + while (!WaitSatisfied(packet)) { + std::this_thread::sleep_for(std::chrono::microseconds(50)); + if (!warned && std::chrono::steady_clock::now() - start > std::chrono::seconds(5)) { + warned = true; + std::fprintf(stderr, "[gpu] WAIT_REG_MEM at 0x%llx still waiting after 5s (function %u ref 0x%x mask 0x%x)\n", + static_cast(address(packet[2], packet[3])), packet[1] & 7u, packet[4], packet[5]); + } + } + return; + } + case 0x49: { + // Earlier work has drained by the time the driver reaches this packet, so the end-of-pipe + // write can happen immediately. Interrupt delivery is not modeled. + const auto dataSelect = packet[2] >> 29u; + const auto destination = address(packet[3], packet[4]); + if (dataSelect == 0 || destination == 0) return; + std::uint64_t value = address(packet[5], packet[6]); + if (dataSelect == 3) { + value = static_cast(std::chrono::duration_cast(std::chrono::steady_clock::now().time_since_epoch()).count() / 10); + } + GuestMemory::Write(destination, std::as_bytes(std::span(&value, 1)).first(dataSelect == 1 ? 4 : 8), dataSelect == 1 ? 4 : 8); return; } case 0x69: case 0x76: case 0x79: case 0x7a: { + if (IsTagMarker(packet)) return; const auto offset = registerOffset(packet[1]); for (std::size_t i = 2; i < packet.size(); ++i) writeRegister(queue, opcode, offset + static_cast(i - 2), packet[i]); return; diff --git a/core/libs/prx/libSceAgcDriver/Execution/src/PresentationScaler.cpp b/core/libs/prx/libSceAgcDriver/Execution/src/PresentationScaler.cpp index fedb9a12b..aa7cd9bc9 100644 --- a/core/libs/prx/libSceAgcDriver/Execution/src/PresentationScaler.cpp +++ b/core/libs/prx/libSceAgcDriver/Execution/src/PresentationScaler.cpp @@ -1,5 +1,6 @@ #include "prx/libSceAgcDriver/Execution/include/PresentationScaler.hpp" #include "prx/libSceAgcDriver/Execution/include/AspectFit.hpp" +#include "prx/libSceAgcDriver/Graphics/include/Resources.hpp" namespace AgcDriver { @@ -77,17 +78,56 @@ void PresentationScaler::RecordUpload(VkCommandBuffer commands, VkBuffer uploadB pipelineBarrier(commands, VK_PIPELINE_STAGE_TRANSFER_BIT, VK_PIPELINE_STAGE_TRANSFER_BIT, 0, 0, nullptr, 0, nullptr, 1, &barrier); } +void PresentationScaler::RecordBlitInto(VkCommandBuffer commands, VkImage image, VkImageLayout layout, VkFilter filter) { + Graphics::Require(sourceImage != VK_NULL_HANDLE && image != VK_NULL_HANDLE, "presentation image is unavailable"); + const auto pipelineBarrier = context.Function("vkCmdPipelineBarrier"); + VkImageMemoryBarrier barrier{VK_STRUCTURE_TYPE_IMAGE_MEMORY_BARRIER}; + barrier.dstAccessMask = VK_ACCESS_TRANSFER_WRITE_BIT; + barrier.oldLayout = VK_IMAGE_LAYOUT_UNDEFINED; + barrier.newLayout = VK_IMAGE_LAYOUT_TRANSFER_DST_OPTIMAL; + barrier.srcQueueFamilyIndex = VK_QUEUE_FAMILY_IGNORED; + barrier.dstQueueFamilyIndex = VK_QUEUE_FAMILY_IGNORED; + barrier.image = sourceImage; + barrier.subresourceRange = {VK_IMAGE_ASPECT_COLOR_BIT, 0, 1, 0, 1}; + pipelineBarrier(commands, VK_PIPELINE_STAGE_TOP_OF_PIPE_BIT, VK_PIPELINE_STAGE_TRANSFER_BIT, 0, 0, nullptr, 0, nullptr, 1, &barrier); + VkImageBlit blit{}; + blit.srcSubresource = {VK_IMAGE_ASPECT_COLOR_BIT, 0, 0, 1}; + blit.srcOffsets[1] = {static_cast(sourceWidth), static_cast(sourceHeight), 1}; + blit.dstSubresource = blit.srcSubresource; + blit.dstOffsets[1] = blit.srcOffsets[1]; + context.Function("vkCmdBlitImage")(commands, image, layout, sourceImage, VK_IMAGE_LAYOUT_TRANSFER_DST_OPTIMAL, 1, &blit, filter); + barrier.srcAccessMask = VK_ACCESS_TRANSFER_WRITE_BIT; + barrier.dstAccessMask = VK_ACCESS_TRANSFER_READ_BIT; + barrier.oldLayout = VK_IMAGE_LAYOUT_TRANSFER_DST_OPTIMAL; + barrier.newLayout = VK_IMAGE_LAYOUT_TRANSFER_SRC_OPTIMAL; + pipelineBarrier(commands, VK_PIPELINE_STAGE_TRANSFER_BIT, VK_PIPELINE_STAGE_TRANSFER_BIT, 0, 0, nullptr, 0, nullptr, 1, &barrier); +} + +void PresentationScaler::RecordReadback(VkCommandBuffer commands, VkBuffer destination) { + Graphics::Require(sourceImage != VK_NULL_HANDLE && destination != VK_NULL_HANDLE, "presentation readback is unavailable"); + VkBufferImageCopy copy{}; + copy.imageSubresource = {VK_IMAGE_ASPECT_COLOR_BIT, 0, 0, 1}; + copy.imageExtent = {sourceWidth, sourceHeight, 1u}; + context.Function("vkCmdCopyImageToBuffer")(commands, sourceImage, VK_IMAGE_LAYOUT_TRANSFER_SRC_OPTIMAL, destination, 1, ©); + Graphics::RecordMemoryBarrier(context, commands, VK_PIPELINE_STAGE_TRANSFER_BIT, VK_PIPELINE_STAGE_HOST_BIT, VK_ACCESS_TRANSFER_WRITE_BIT, VK_ACCESS_HOST_READ_BIT); +} + void PresentationScaler::RecordBlit(VkCommandBuffer commands, VkImage destinationImage, std::uint32_t destinationWidth, std::uint32_t destinationHeight) { Graphics::Require(sourceImage != VK_NULL_HANDLE, "presentation source image has not been created"); - const auto rect = ComputeContainRect_nid_postfix(sourceWidth, sourceHeight, destinationWidth, destinationHeight); + RecordBlitFrom(context, commands, sourceImage, VK_IMAGE_LAYOUT_TRANSFER_SRC_OPTIMAL, sourceWidth, sourceHeight, VK_FILTER_LINEAR, destinationImage, destinationWidth, destinationHeight); +} + +void PresentationScaler::RecordBlitFrom(const Graphics::Context& context, VkCommandBuffer commands, VkImage image, VkImageLayout layout, std::uint32_t width, std::uint32_t height, VkFilter filter, VkImage destinationImage, std::uint32_t destinationWidth, std::uint32_t destinationHeight) { + Graphics::Require(image != VK_NULL_HANDLE && width != 0 && height != 0, "presentation blit source is unavailable"); + const auto rect = ComputeContainRect_nid_postfix(width, height, destinationWidth, destinationHeight); VkImageBlit blit{}; blit.srcSubresource = {VK_IMAGE_ASPECT_COLOR_BIT, 0, 0, 1}; blit.srcOffsets[0] = {0, 0, 0}; - blit.srcOffsets[1] = {static_cast(sourceWidth), static_cast(sourceHeight), 1}; + blit.srcOffsets[1] = {static_cast(width), static_cast(height), 1}; blit.dstSubresource = {VK_IMAGE_ASPECT_COLOR_BIT, 0, 0, 1}; blit.dstOffsets[0] = {rect.x, rect.y, 0}; blit.dstOffsets[1] = {rect.x + static_cast(rect.width), rect.y + static_cast(rect.height), 1}; - context.Function("vkCmdBlitImage")(commands, sourceImage, VK_IMAGE_LAYOUT_TRANSFER_SRC_OPTIMAL, destinationImage, VK_IMAGE_LAYOUT_TRANSFER_DST_OPTIMAL, 1, &blit, VK_FILTER_LINEAR); + context.Function("vkCmdBlitImage")(commands, image, layout, destinationImage, VK_IMAGE_LAYOUT_TRANSFER_DST_OPTIMAL, 1, &blit, filter); } } diff --git a/core/libs/prx/libSceAgcDriver/Execution/src/ShaderMemory.cpp b/core/libs/prx/libSceAgcDriver/Execution/src/ShaderMemory.cpp index 50d913bc1..03853d079 100644 --- a/core/libs/prx/libSceAgcDriver/Execution/src/ShaderMemory.cpp +++ b/core/libs/prx/libSceAgcDriver/Execution/src/ShaderMemory.cpp @@ -3,68 +3,128 @@ #include "Optimization/RequestMemoryView.hpp" #include "Optimization/ResourceMaterializer.hpp" #include "Optimization/ResourceProgram.hpp" +#include +#include +#include +#include #include -#include #include #include -#include namespace AgcDriver { +namespace { -ShaderMemory::ShaderMemory(std::span initial) { - const ShaderRecompiler::RequestMemoryView validated(initial); - for (const auto& region : initial) { - regions.emplace(region.guestAddress, std::vector(region.bytes.begin(), region.bytes.end())); +struct CaptureProfile { + std::atomic captures{0}; + std::atomic reads{0}; + std::atomic pages{0}; + // The whole CaptureResources call: the plan lookup and the materialization are one call. + std::atomic captureNanoseconds{0}; +}; + +CaptureProfile& CaptureTotals() { + static CaptureProfile profile; + return profile; +} + +} + +ShaderMemory::ShaderMemory(std::span regions) { + const ShaderRecompiler::RequestMemoryView validated(regions); + for (const auto& region : regions) { + initial.emplace(region.guestAddress, region.bytes); } } +ShaderMemory::Page& ShaderMemory::page(std::uint64_t base) { + const auto found = pages.find(base); + if (found != pages.end()) return found->second; + auto& page = pages[base]; + ++CaptureTotals().pages; + // A page is either mapped whole or read word by word where the guest mapped less than a page. + if (GuestMemory::Accessible(reinterpret_cast(base), PageBytes)) { + GuestMemory::Read(base, std::as_writable_bytes(std::span(page.words)), sizeof(std::uint32_t)); + page.valid.set(); + } + return page; +} + bool ShaderMemory::read(void* context, std::uint64_t address, std::uint32_t* value) { auto& self = *static_cast(context); if (address % sizeof(*value) != 0 || address > std::numeric_limits::max() - sizeof(*value)) { throw std::runtime_error("AGC driver: invalid shader memory read address"); } - const auto next = self.regions.upper_bound(address); - if (next != self.regions.begin()) { - const auto previous = std::prev(next); - const auto offset = address - previous->first; - if (offset < previous->second.size()) { - if (previous->second.size() - offset < sizeof(*value)) { - throw std::runtime_error("AGC driver: shader memory read crosses a snapshot boundary"); + ++CaptureTotals().reads; + if (!self.initial.empty()) { + const auto next = self.initial.upper_bound(address); + if (next != self.initial.begin()) { + const auto previous = std::prev(next); + const auto offset = address - previous->first; + if (offset < previous->second.size()) { + if (previous->second.size() - offset < sizeof(*value)) throw std::runtime_error("AGC driver: shader memory read crosses a snapshot boundary"); + std::memcpy(value, previous->second.data() + offset, sizeof(*value)); + return true; } - std::memcpy(value, previous->second.data() + offset, sizeof(*value)); - return true; } + if (next != self.initial.end() && next->first - address < sizeof(*value)) throw std::runtime_error("AGC driver: shader memory read overlaps a snapshot boundary"); } - if (next != self.regions.end() && next->first - address < sizeof(*value)) { - throw std::runtime_error("AGC driver: shader memory read overlaps a snapshot boundary"); + auto& page = self.page(address & ~static_cast(PageBytes - 1)); + const auto index = static_cast((address % PageBytes) / sizeof(*value)); + if (!page.valid.test(index)) { + std::uint32_t word = 0; + GuestMemory::Read(address, std::as_writable_bytes(std::span(&word, 1)), alignof(std::uint32_t)); + page.words[index] = word; + page.valid.set(index); } - std::vector bytes(sizeof(*value)); - GuestMemory::Read(address, bytes, alignof(std::uint32_t)); - std::memcpy(value, bytes.data(), sizeof(*value)); - self.regions.emplace(address, std::move(bytes)); + page.read.set(index); + *value = page.words[index]; return true; } -void ShaderMemory::Capture(const ShaderRecompiler::RecompileRequest& request) { - const auto plan = ShaderRecompiler::GetResourcePlan(request); - constexpr ShaderRecompiler::ResourceMaterializer materializer; +std::shared_ptr ShaderMemory::Capture(const ShaderRecompiler::RecompileRequest& request) { + // The capture's word and page reads (through `read`) are attributed to it ([hooksync], [guestmem]). + const GuestMemory::ReadSiteScope site(GuestMemory::ReadSite::Capture); + const auto started = std::chrono::steady_clock::now(); ShaderRecompiler::SrtRuntime runtime; runtime.userData = request.context.userData; runtime.shaderBase = request.shader.codeAddress; runtime.userContext = this; runtime.readMemory = &read; runtime.readSpecializationMemory = &read; - ShaderRecompiler::ResourceSnapshot snapshot; - ShaderRecompiler::ResourceSpecialization specialization; - materializer.Materialize(*plan, runtime, snapshot, specialization); + auto capture = ShaderRecompiler::CaptureResources(request, runtime); + static const bool profile = std::getenv("APS5_PROFILE_DRAW") != nullptr; + auto& totals = CaptureTotals(); + if (profile) { + const auto done = std::chrono::steady_clock::now(); + totals.captureNanoseconds += static_cast(std::chrono::duration_cast(done - started).count()); + std::uint64_t initialBytes = 0; + for (const auto& [address, bytes] : initial) initialBytes += bytes.size(); + if (++totals.captures % 500 == 0) std::fprintf(stderr, "[capture] %llu captures: %llu word reads, %llu pages fetched, capture (plan lookup + materialize) %.1f s, %llu KiB registered code this capture\n", static_cast(totals.captures.load()), static_cast(totals.reads.load()), static_cast(totals.pages.load()), totals.captureNanoseconds.load() / 1e9, static_cast(initialBytes / 1024)); + } + return capture; } std::vector ShaderMemory::Regions() const { std::vector result; - result.reserve(regions.size()); - for (const auto& [address, bytes] : regions) { - result.push_back({address, bytes}); + result.reserve(initial.size() + pages.size()); + auto next = initial.begin(); + // Both maps are ordered by address and never overlap, so a merge keeps the result sorted. + for (const auto& [base, page] : pages) { + while (next != initial.end() && next->first < base) { + result.push_back({next->first, next->second}); + ++next; + } + for (std::size_t index = 0; index < PageWords;) { + if (!page.read.test(index)) { + ++index; + continue; + } + const auto first = index; + while (index < PageWords && page.read.test(index)) ++index; + result.push_back({base + first * sizeof(std::uint32_t), std::as_bytes(std::span(page.words).subspan(first, index - first))}); + } } + for (; next != initial.end(); ++next) result.push_back({next->first, next->second}); return result; } diff --git a/core/libs/prx/libSceAgcDriver/Execution/src/VulkanDevice.cpp b/core/libs/prx/libSceAgcDriver/Execution/src/VulkanDevice.cpp index 6f28b5525..651bd0bac 100644 --- a/core/libs/prx/libSceAgcDriver/Execution/src/VulkanDevice.cpp +++ b/core/libs/prx/libSceAgcDriver/Execution/src/VulkanDevice.cpp @@ -1,30 +1,38 @@ #include "BdaAbi.hpp" -#include "prx/libSceAgcDriver/Graphics/include/RenderCache.hpp" -#include "prx/libSceAgcDriver/Graphics/include/DrawQueue.hpp" -#include "prx/libSceAgcDriver/Graphics/include/GraphicsPipelineCache.hpp" -#include "prx/libSceAgcDriver/Execution/include/MemoryAccessScope.hpp" #include "prx/libSceAgcDriver/Execution/include/VulkanDevice.hpp" #include "prx/libSceAgcDriver/Execution/include/PerformanceTimer.hpp" #include "prx/libSceAgcDriver/Execution/include/BdaFeatures.hpp" #include "prx/libSceAgcDriver/Execution/include/PresentationScaler.hpp" -#include "prx/libSceAgcDriver/Execution/include/SwapchainState.hpp" #include "prx/libSceAgcDriver/Graphics/include/TextureDetiler.hpp" #include "prx/libSceAgcDriver/Graphics/include/GpuColorTransfer.hpp" #include "prx/libSceAgcDriver/Graphics/include/BufferPool.hpp" -#include "prx/libSceAgcDriver/Graphics/include/DescriptorCache.hpp" -#include "prx/libSceAgcDriver/Graphics/include/Sampler.hpp" #include "prx/libSceAgcDriver/Graphics/include/TextureCache.hpp" #include "prx/libSceAgcDriver/Graphics/include/PipelineCache.hpp" +#include "prx/libSceAgcDriver/Graphics/include/Recorder.hpp" +#include "prx/libSceAgcDriver/Graphics/include/Resources.hpp" +#include "prx/libSceAgcDriver/Graphics/include/GuestBufferMemory.hpp" +#include "prx/libSceAgcDriver/Graphics/include/ShaderResources.hpp" +#include "prx/libSceAgcDriver/Graphics/include/Texture.hpp" +#include "prx/libSceAgcDriver/Execution/include/GuestMemory.hpp" #include "prx/libc/include/General.hpp" +#ifdef _WIN32 +#include +#endif +#include +#include #include #include #include #include +#include #include #include #include +#include #include #include +#include +#include #include #include @@ -41,6 +49,29 @@ void require(bool condition, const char* reason) { if (!condition) throw std::runtime_error(std::string("Vulkan presentation: ") + reason); } +// A compute dispatch's Vulkan objects, shared by every dispatch of the same compiled variant and +// kept by the recorder until the batches using them completed. +struct ComputePipelineObjects { + VkDevice device = VK_NULL_HANDLE; + PFN_vkDestroyShaderModule destroyModule = nullptr; + PFN_vkDestroyPipelineLayout destroyLayout = nullptr; + PFN_vkDestroyPipeline destroyPipeline = nullptr; + VkShaderModule module = VK_NULL_HANDLE; + VkPipelineLayout layout = VK_NULL_HANDLE; + VkPipeline pipeline = VK_NULL_HANDLE; + ~ComputePipelineObjects() { + if (pipeline != VK_NULL_HANDLE) destroyPipeline(device, pipeline, nullptr); + if (layout != VK_NULL_HANDLE) destroyLayout(device, layout, nullptr); + if (module != VK_NULL_HANDLE) destroyModule(device, module, nullptr); + } +}; + +// The ShaderResources content cache dispatches share with recorded draws: Graphics::ResourceCache, +// one process-wide instance (see SharedResourceCache) that the State references so this file keeps +// its Find/Insert/Remove/Clear calls; the device clears it at teardown before its descriptor caches +// go. APS5_NO_RESOURCE_CACHE=1 builds every dispatch's resources as before. +using ResourceCache = Graphics::ResourceCache; + } struct VulkanDevice::State { @@ -55,7 +86,6 @@ struct VulkanDevice::State { void* window = nullptr; VkSurfaceKHR surface = VK_NULL_HANDLE; VkSwapchainKHR swapchain = VK_NULL_HANDLE; - SwapchainState swapchainState; VkExtent2D extent{}; std::vector images; VkFence acquireFence = VK_NULL_HANDLE; @@ -80,22 +110,45 @@ struct VulkanDevice::State { bool meshShader = false; bool fragmentShaderBarycentric = false; bool depthClipControl = false; + bool depthClamp = false; + VkDeviceSize hostImportAlignment = 0; bool depthRangeUnrestricted = false; bool samplerAnisotropy = false; bool textureCompressionBC = false; + // VK_KHR_timeline_semaphore enabled: the recorder's unlocked waits are available. + bool timelineSemaphores = false; std::unique_ptr detiler; + std::unique_ptr recorder; + std::map> computePipelines; std::unique_ptr colorTransfer; std::shared_ptr bufferPool; - std::shared_ptr descriptorCache; - std::shared_ptr samplerCache; std::unique_ptr textureCache; std::unique_ptr pipelineCache; - std::unique_ptr drawQueue; - std::unique_ptr renderCache; - std::unique_ptr graphicsPipelines; + std::unique_ptr descriptorCache; + std::unique_ptr samplerCache; + Graphics::ResourceCache& resourceCache = Graphics::SharedResourceCache(); + // Recorded dispatches that write a copied buffer (their results reach guest memory by a CPU + // write-back when the batch is reaped), listed until that write-back ran. An indirect dispatch + // whose arguments one of them writes must read them on the CPU (DispatchIndirect); GPU-direct + // writes into host imports need no entry. Under GuestMemory::GpuMutex; shared with the completion + // actions so they never outlive it. + std::shared_ptr>> copiedWriters = std::make_shared>>(); VkPhysicalDeviceMeshShaderPropertiesEXT meshLimits{VK_STRUCTURE_TYPE_PHYSICAL_DEVICE_MESH_SHADER_PROPERTIES_EXT}; std::unique_ptr scaler; - std::unique_ptr rgbaScaler; + // The swapchain image AcquireImage took for the next present(), consumed by that present. + bool imageAcquired = false; + std::uint32_t acquiredIndex = 0; + // The presentation in flight between present() and QueuePresent(): its swapchain image, and the + // resident image it blits from, alive until the render fence has been waited for. + bool presentPending = false; + std::uint32_t presentIndex = 0; + std::shared_ptr presentKept; + // APS5_DUMP_FRAMES: the presented frame is read back into this buffer by the same submission and + // written as a BMP by the writer thread after the render fence. + std::unique_ptr dumpBuffer; + bool dumpRecorded = false; + int dumpIndex = 0; + std::thread dumpWriter; template TFunction InstanceFunction(const char* name) const { @@ -116,7 +169,9 @@ struct VulkanDevice::State { } void Upload(std::span pixels) { + APS5_LOG_OUT("Upload pixels=%zu uploadSize=%llu buffer=%p memory=%p mapping=%p", pixels.size(), static_cast(uploadSize), reinterpret_cast(uploadBuffer), reinterpret_cast(uploadMemory), uploadMapping); if (uploadSize < pixels.size()) { + APS5_LOG_OUT("Upload reallocating oldSize=%llu newSize=%zu", static_cast(uploadSize), pixels.size()); if (uploadMapping) DeviceFunction("vkUnmapMemory")(device, uploadMemory); uploadMapping = nullptr; if (uploadBuffer) DeviceFunction("vkDestroyBuffer")(device, uploadBuffer, nullptr); @@ -140,6 +195,7 @@ struct VulkanDevice::State { } } require(memoryType < memoryProperties.memoryTypeCount, "coherent host upload memory is unavailable"); + APS5_LOG_OUT("Upload requirements size=%llu alignment=%llu typeBits=0x%x memoryType=%u", static_cast(requirements.size), static_cast(requirements.alignment), requirements.memoryTypeBits, memoryType); VkMemoryAllocateInfo allocation{VK_STRUCTURE_TYPE_MEMORY_ALLOCATE_INFO}; allocation.allocationSize = requirements.size; allocation.memoryTypeIndex = memoryType; @@ -147,8 +203,10 @@ struct VulkanDevice::State { check(DeviceFunction("vkBindBufferMemory")(device, uploadBuffer, uploadMemory, 0), "vkBindBufferMemory display upload"); check(DeviceFunction("vkMapMemory")(device, uploadMemory, 0, pixels.size(), 0, &uploadMapping), "vkMapMemory display upload"); uploadSize = pixels.size(); + APS5_LOG_OUT("Upload allocation complete buffer=%p memory=%p mapping=%p size=%llu", reinterpret_cast(uploadBuffer), reinterpret_cast(uploadMemory), uploadMapping, static_cast(uploadSize)); } std::memcpy(uploadMapping, pixels.data(), pixels.size()); + APS5_LOG_OUT("Upload memcpy complete bytes=%zu", pixels.size()); } void DestroyRetiredSwapchains() { @@ -165,24 +223,29 @@ struct VulkanDevice::State { } ~State() { - std::lock_guard memoryLock(GuestMemoryTracking::GuestMemoryTrackingMutex_nid_postfix()); + if (dumpWriter.joinable()) dumpWriter.join(); if (device != VK_NULL_HANDLE) { - if (drawQueue) drawQueue->Wait(); - if (renderCache) renderCache->Flush(); const auto idle = reinterpret_cast(deviceProc(device, "vkDeviceWaitIdle"))(device); if (idle != VK_SUCCESS && idle != VK_ERROR_DEVICE_LOST) std::terminate(); - drawQueue.reset(); - graphicsPipelines.reset(); - renderCache.reset(); + // A resident image kept for an unfinished presentation goes before the caches it came from. + presentKept.reset(); + dumpBuffer.reset(); + recorder.reset(); + // Cached graphics pipelines (with their framebuffers, modules, render passes and layouts) + // belong to this device and must be destroyed while it lives. + Graphics::ClearCachedPipelines(device); + computePipelines.clear(); + // Every ShaderResources (kept by the recorder or the resource cache) is gone now, so the + // sets and samplers they borrowed can go. + resourceCache.Clear(); + descriptorCache.reset(); + samplerCache.reset(); textureCache.reset(); detiler.reset(); colorTransfer.reset(); scaler.reset(); - rgbaScaler.reset(); pipelineCache.reset(); bufferPool.reset(); - descriptorCache.reset(); - samplerCache.reset(); const auto destroyFence = reinterpret_cast(deviceProc(device, "vkDestroyFence")); if (acquireFence) destroyFence(device, acquireFence, nullptr); if (renderFence) destroyFence(device, renderFence, nullptr); @@ -213,6 +276,7 @@ struct VulkanDevice::State { }; VulkanDevice::VulkanDevice(const PresentationWindow* window) : state(std::make_unique()) { + APS5_LOG_OUT("VulkanDevice constructor window=%p", static_cast(window)); #ifdef _WIN32 state->library = SDL_LoadObject("vulkan-1.dll"); #else @@ -221,6 +285,7 @@ VulkanDevice::VulkanDevice(const PresentationWindow* window) : state(std::make_u if (state->library == nullptr) { throw std::runtime_error(std::string("Vulkan loader: ") + SDL_GetError()); } + APS5_LOG_OUT("Vulkan loader loaded library=%p", state->library); state->instanceProc = reinterpret_cast(SDL_LoadFunction(state->library, "vkGetInstanceProcAddr")); if (state->instanceProc == nullptr) { throw std::runtime_error("Vulkan loader: vkGetInstanceProcAddr missing"); @@ -232,6 +297,7 @@ VulkanDevice::VulkanDevice(const PresentationWindow* window) : state(std::make_u create.pApplicationInfo = &application; std::vector instanceExtensions; if (window != nullptr) { + APS5_LOG_OUT("Presentation window context=%p extent=%ux%u extensions=%zu", window->context, window->width, window->height, window->extensions.size()); require(window->context && window->createSurface && window->getDrawableSize && window->width && window->height, "invalid window descriptor"); instanceExtensions.assign(window->extensions.begin(), window->extensions.end()); std::uint32_t availableCount = 0; @@ -249,15 +315,18 @@ VulkanDevice::VulkanDevice(const PresentationWindow* window) : state(std::make_u create.ppEnabledExtensionNames = instanceExtensions.data(); } check(state->InstanceFunction("vkCreateInstance")(&create, nullptr, &state->instance), "vkCreateInstance"); + APS5_LOG_OUT("Vulkan instance created instance=%p", reinterpret_cast(state->instance)); if (window != nullptr) { state->surface = window->createSurface(window->context, state->instance); require(state->surface != VK_NULL_HANDLE, "window returned a null surface"); state->window = window->context; + APS5_LOG_OUT("Vulkan surface created surface=%p window=%p", reinterpret_cast(state->surface), state->window); } state->deviceProc = state->InstanceFunction("vkGetDeviceProcAddr"); const auto enumerate = state->InstanceFunction("vkEnumeratePhysicalDevices"); std::uint32_t count = 0; check(enumerate(state->instance, &count, nullptr), "vkEnumeratePhysicalDevices"); + APS5_LOG_OUT("Physical device count=%u", count); std::vector devices(count); check(enumerate(state->instance, &count, devices.data()), "vkEnumeratePhysicalDevices"); devices.resize(count); @@ -267,6 +336,7 @@ VulkanDevice::VulkanDevice(const PresentationWindow* window) : state(std::make_u for (auto physical : devices) { VkPhysicalDeviceProperties properties{}; state->InstanceFunction("vkGetPhysicalDeviceProperties")(physical, &properties); + APS5_LOG_OUT("Physical device candidate=%p name=%s api=0x%x", reinterpret_cast(physical), properties.deviceName, properties.apiVersion); if (properties.apiVersion < VK_API_VERSION_1_1) { continue; } @@ -305,6 +375,7 @@ VulkanDevice::VulkanDevice(const PresentationWindow* window) : state(std::make_u if (selected == VK_NULL_HANDLE) { throw std::runtime_error(window ? "Vulkan: no Vulkan 1.1 device with graphics, compute and swapchain presentation" : "Vulkan: no Vulkan 1.1 graphics and compute queue"); } + APS5_LOG_OUT("Physical device selected physical=%p queueFamily=%u", reinterpret_cast(selected), family); VkPhysicalDeviceProperties2 properties{VK_STRUCTURE_TYPE_PHYSICAL_DEVICE_PROPERTIES_2}; properties.pNext = &state->subgroup; state->InstanceFunction("vkGetPhysicalDeviceProperties2")(selected, &properties); @@ -315,7 +386,10 @@ VulkanDevice::VulkanDevice(const PresentationWindow* window) : state(std::make_u if ((state->subgroup.supportedOperations & VK_SUBGROUP_FEATURE_BALLOT_BIT) != 0) state->capabilities.push_back(spv::CapabilityGroupNonUniformBallot); if ((state->subgroup.supportedOperations & VK_SUBGROUP_FEATURE_SHUFFLE_BIT) != 0) state->capabilities.push_back(spv::CapabilityGroupNonUniformShuffle); } + APS5_LOG_OUT("Selected GPU name=%s vendor=0x%x device=0x%x subgroup=%u", state->properties.deviceName, state->properties.vendorID, state->properties.deviceID, state->subgroup.subgroupSize); state->InstanceFunction("vkGetPhysicalDeviceMemoryProperties")(selected, &state->memoryProperties); + for (std::uint32_t i = 0; i < state->memoryProperties.memoryHeapCount; ++i) APS5_LOG_OUT("Memory heap %u size=%.1f MiB flags=0x%x", i, state->memoryProperties.memoryHeaps[i].size / 1048576.0, state->memoryProperties.memoryHeaps[i].flags); + for (std::uint32_t i = 0; i < state->memoryProperties.memoryTypeCount; ++i) APS5_LOG_OUT("Memory type %u heap=%u flags=0x%x", i, state->memoryProperties.memoryTypes[i].heapIndex, state->memoryProperties.memoryTypes[i].propertyFlags); std::uint32_t extensionCount = 0; const auto enumerateDeviceExtensions = state->InstanceFunction("vkEnumerateDeviceExtensionProperties"); check(enumerateDeviceExtensions(selected, nullptr, &extensionCount, nullptr), "vkEnumerateDeviceExtensionProperties"); @@ -366,6 +440,26 @@ VulkanDevice::VulkanDevice(const PresentationWindow* window) : state(std::make_u state->spirvExtensions.push_back("SPV_KHR_physical_storage_buffer"); state->depthRangeUnrestricted = hasExtension(VK_EXT_DEPTH_RANGE_UNRESTRICTED_EXTENSION_NAME); if (state->depthRangeUnrestricted) deviceExtensions.push_back(VK_EXT_DEPTH_RANGE_UNRESTRICTED_EXTENSION_NAME); + // Guest dispatches cover whole thread groups past the edge of small images; robust image access + // drops those writes instead of faulting the device. + const bool imageRobustness = hasExtension(VK_EXT_IMAGE_ROBUSTNESS_EXTENSION_NAME); + if (imageRobustness) deviceExtensions.push_back(VK_EXT_IMAGE_ROBUSTNESS_EXTENSION_NAME); + // Guest memory is host memory: importing it lets address-based shaders use it in place instead of + // copying every registered allocation per draw. + if (hasExtension(VK_EXT_EXTERNAL_MEMORY_HOST_EXTENSION_NAME) && std::getenv("APS5_NO_HOST_IMPORT") == nullptr) { + VkPhysicalDeviceExternalMemoryHostPropertiesEXT hostProperties{VK_STRUCTURE_TYPE_PHYSICAL_DEVICE_EXTERNAL_MEMORY_HOST_PROPERTIES_EXT}; + VkPhysicalDeviceProperties2 properties{VK_STRUCTURE_TYPE_PHYSICAL_DEVICE_PROPERTIES_2, &hostProperties}; + state->InstanceFunction("vkGetPhysicalDeviceProperties2")(selected, &properties); + state->hostImportAlignment = hostProperties.minImportedHostPointerAlignment; +#ifdef _WIN32 + // Imported pages are locked; locking is bounded by the process working-set minimum. + if (const char* value = std::getenv("APS5_WORKING_SET_MIB")) { + const SIZE_T bytes = static_cast(std::strtoull(value, nullptr, 10)) << 20u; + if (!SetProcessWorkingSetSizeEx(GetCurrentProcess(), bytes, bytes * 2, QUOTA_LIMITS_HARDWS_MIN_DISABLE | QUOTA_LIMITS_HARDWS_MAX_DISABLE)) std::fprintf(stderr, "[gpu] SetProcessWorkingSetSizeEx failed: %lu\n", GetLastError()); + } +#endif + deviceExtensions.push_back(VK_EXT_EXTERNAL_MEMORY_HOST_EXTENSION_NAME); + } VkPhysicalDeviceDepthClipControlFeaturesEXT depthClipFeatures{VK_STRUCTURE_TYPE_PHYSICAL_DEVICE_DEPTH_CLIP_CONTROL_FEATURES_EXT}; if (hasExtension(VK_EXT_DEPTH_CLIP_CONTROL_EXTENSION_NAME)) { VkPhysicalDeviceFeatures2 features{VK_STRUCTURE_TYPE_PHYSICAL_DEVICE_FEATURES_2, &depthClipFeatures}; @@ -378,6 +472,7 @@ VulkanDevice::VulkanDevice(const PresentationWindow* window) : state(std::make_u state->capabilities.push_back(5283); state->spirvExtensions.push_back("SPV_EXT_mesh_shader"); } + APS5_LOG_OUT("Vulkan features tessellationAvailable=%u mesh=%u depthClip=%u depthRangeUnrestricted=%u", static_cast(state->tessellationShader), static_cast(state->meshShader), static_cast(state->depthClipControl), static_cast(state->depthRangeUnrestricted)); const float priority = 1.0f; VkDeviceQueueCreateInfo queueInfo{VK_STRUCTURE_TYPE_DEVICE_QUEUE_CREATE_INFO}; queueInfo.queueFamilyIndex = family; @@ -399,6 +494,20 @@ VulkanDevice::VulkanDevice(const PresentationWindow* window) : state(std::make_u require(available.samplerAnisotropy && available.textureCompressionBC, "device lacks sampler anisotropy or BC texture compression support required for texture sampling"); enabled.samplerAnisotropy = VK_TRUE; enabled.textureCompressionBC = VK_TRUE; + // Guest shaders routinely read past descriptor ranges; robust access turns that into zeros + // instead of a GPU fault that loses the device. + enabled.robustBufferAccess = available.robustBufferAccess; + // PA_CL_CLIP_CNTL near/far clip disable maps to depth clamping. + enabled.depthClamp = available.depthClamp; + state->depthClamp = enabled.depthClamp == VK_TRUE; + // Recompiled storage-image access declares no format (the guest descriptor decides it). + enabled.shaderStorageImageWriteWithoutFormat = available.shaderStorageImageWriteWithoutFormat; + enabled.shaderStorageImageReadWithoutFormat = available.shaderStorageImageReadWithoutFormat; + // Gathers with non-constant offsets (ImageGatherExtended). + enabled.shaderImageGatherExtended = available.shaderImageGatherExtended; + if (enabled.shaderImageGatherExtended) state->capabilities.push_back(spv::CapabilityImageGatherExtended); + if (enabled.shaderStorageImageWriteWithoutFormat) state->capabilities.push_back(spv::CapabilityStorageImageWriteWithoutFormat); + if (enabled.shaderStorageImageReadWithoutFormat) state->capabilities.push_back(spv::CapabilityStorageImageReadWithoutFormat); state->samplerAnisotropy = true; state->textureCompressionBC = true; deviceInfo.pEnabledFeatures = &enabled; @@ -417,24 +526,49 @@ VulkanDevice::VulkanDevice(const PresentationWindow* window) : state(std::make_u barycentricFeatures.pNext = byteFeatures.pNext; byteFeatures.pNext = &barycentricFeatures; } + VkPhysicalDeviceImageRobustnessFeaturesEXT imageRobustnessFeatures{VK_STRUCTURE_TYPE_PHYSICAL_DEVICE_IMAGE_ROBUSTNESS_FEATURES_EXT, nullptr, VK_TRUE}; + if (imageRobustness) { + imageRobustnessFeatures.pNext = byteFeatures.pNext; + byteFeatures.pNext = &imageRobustnessFeatures; + } + // Timeline semaphores let a queue worker wait for recorded batches without holding the GPU mutex + // (see Recorder::WaitSerial). The instance is 1.1, so the KHR extension is used even on 1.2+ + // devices. Debug aid: APS5_NO_TIMELINE=1 leaves it off (drains wait under the mutex as before). + VkPhysicalDeviceTimelineSemaphoreFeaturesKHR timelineFeatures{VK_STRUCTURE_TYPE_PHYSICAL_DEVICE_TIMELINE_SEMAPHORE_FEATURES_KHR}; + if (hasExtension(VK_KHR_TIMELINE_SEMAPHORE_EXTENSION_NAME) && std::getenv("APS5_NO_TIMELINE") == nullptr) { + VkPhysicalDeviceFeatures2 features{VK_STRUCTURE_TYPE_PHYSICAL_DEVICE_FEATURES_2, &timelineFeatures}; + state->InstanceFunction("vkGetPhysicalDeviceFeatures2")(selected, &features); + state->timelineSemaphores = timelineFeatures.timelineSemaphore == VK_TRUE; + } + timelineFeatures = {VK_STRUCTURE_TYPE_PHYSICAL_DEVICE_TIMELINE_SEMAPHORE_FEATURES_KHR}; + timelineFeatures.timelineSemaphore = VK_TRUE; + if (state->timelineSemaphores) { + deviceExtensions.push_back(VK_KHR_TIMELINE_SEMAPHORE_EXTENSION_NAME); + deviceInfo.enabledExtensionCount = static_cast(deviceExtensions.size()); + deviceInfo.ppEnabledExtensionNames = deviceExtensions.data(); + timelineFeatures.pNext = byteFeatures.pNext; + byteFeatures.pNext = &timelineFeatures; + } else { + std::fprintf(stderr, "[gpu] timeline semaphores unavailable or disabled; drains wait under the GPU mutex\n"); + } bdaFeatures.pNext = &byteFeatures; deviceInfo.pNext = &bdaFeatures; check(state->InstanceFunction("vkCreateDevice")(selected, &deviceInfo, nullptr, &state->device), "vkCreateDevice"); state->DeviceFunction("vkGetDeviceQueue")(state->device, family, 0, &state->queue); + APS5_LOG_OUT("Vulkan device ready device=%p queue=%p family=%u", reinterpret_cast(state->device), reinterpret_cast(state->queue), family); VkCommandPoolCreateInfo poolInfo{VK_STRUCTURE_TYPE_COMMAND_POOL_CREATE_INFO}; poolInfo.flags = VK_COMMAND_POOL_CREATE_RESET_COMMAND_BUFFER_BIT; poolInfo.queueFamilyIndex = family; check(state->DeviceFunction("vkCreateCommandPool")(state->device, &poolInfo, nullptr, &state->pool), "vkCreateCommandPool"); state->bufferPool = std::make_shared(graphicsContext()); - state->descriptorCache = std::make_shared(); - state->samplerCache = std::make_shared(); state->pipelineCache = std::make_unique(graphicsContext()); state->detiler = std::make_unique(graphicsContext()); - state->drawQueue = std::make_unique(); - state->renderCache = std::make_unique(graphicsContext()); - state->graphicsPipelines = std::make_unique(graphicsContext()); state->textureCache = std::make_unique(graphicsContext()); state->colorTransfer = std::make_unique(graphicsContext()); + state->descriptorCache = std::make_unique(graphicsContext()); + state->samplerCache = std::make_unique(); + state->recorder = std::make_unique(graphicsContext(), state->timelineSemaphores); + state->recorder->Activate(); if (window != nullptr) { require(window->getDrawableSize != nullptr, "missing window drawable size query"); std::uint32_t drawableWidth = 0; @@ -444,7 +578,14 @@ VulkanDevice::VulkanDevice(const PresentationWindow* window) : state(std::make_u VkSurfaceCapabilitiesKHR surface{}; check(state->InstanceFunction("vkGetPhysicalDeviceSurfaceCapabilitiesKHR")(selected, state->surface, &surface), "vkGetPhysicalDeviceSurfaceCapabilitiesKHR"); state->extent = {drawableWidth, drawableHeight}; - require(surface.currentExtent.width == std::numeric_limits::max() || (surface.currentExtent.width == drawableWidth && surface.currentExtent.height == drawableHeight), "window extent differs from the real drawable size"); + APS5_LOG_OUT("Surface capabilities drawable=%ux%u min=%ux%u max=%ux%u minImages=%u maxImages=%u usage=0x%x", drawableWidth, drawableHeight, surface.minImageExtent.width, surface.minImageExtent.height, surface.maxImageExtent.width, surface.maxImageExtent.height, surface.minImageCount, surface.maxImageCount, surface.supportedUsageFlags); + // A surface that reports its extent (a fullscreen transition can make it differ from the + // drawable size for a moment) wins: the swapchain must match the surface. + if (surface.currentExtent.width != std::numeric_limits::max() && surface.currentExtent.width != 0 && surface.currentExtent.height != 0) { + drawableWidth = surface.currentExtent.width; + drawableHeight = surface.currentExtent.height; + state->extent = {drawableWidth, drawableHeight}; + } require(drawableWidth >= surface.minImageExtent.width && drawableWidth <= surface.maxImageExtent.width && drawableHeight >= surface.minImageExtent.height && drawableHeight <= surface.maxImageExtent.height, "unsupported output extent"); require((surface.supportedUsageFlags & VK_IMAGE_USAGE_TRANSFER_DST_BIT) != 0, "surface does not support transfer destination images"); require((surface.supportedCompositeAlpha & VK_COMPOSITE_ALPHA_OPAQUE_BIT_KHR) != 0, "opaque composition is unavailable"); @@ -474,6 +615,7 @@ VulkanDevice::VulkanDevice(const PresentationWindow* window) : state(std::make_u check(getImages(state->device, state->swapchain, &imageCount, nullptr), "vkGetSwapchainImagesKHR"); state->images.resize(imageCount); check(getImages(state->device, state->swapchain, &imageCount, state->images.data()), "vkGetSwapchainImagesKHR"); + APS5_LOG_OUT("Swapchain created swapchain=%p extent=%ux%u images=%u", reinterpret_cast(state->swapchain), state->extent.width, state->extent.height, imageCount); VkFenceCreateInfo fence{VK_STRUCTURE_TYPE_FENCE_CREATE_INFO}; for (auto* destination : {&state->acquireFence, &state->renderFence}) { check(state->DeviceFunction("vkCreateFence")(state->device, &fence, nullptr, destination), "vkCreateFence"); @@ -486,25 +628,237 @@ VulkanDevice::VulkanDevice(const PresentationWindow* window) : state(std::make_u allocation.commandBufferCount = 1; check(state->DeviceFunction("vkAllocateCommandBuffers")(state->device, &allocation, &state->clearCommands), "vkAllocateCommandBuffers"); state->scaler = std::make_unique(graphicsContext(), VK_FORMAT_B8G8R8A8_UNORM, VK_FORMAT_B8G8R8A8_UNORM); - state->rgbaScaler = std::make_unique(graphicsContext(), VK_FORMAT_R8G8B8A8_UNORM, VK_FORMAT_B8G8R8A8_UNORM); } } VulkanDevice::~VulkanDevice() = default; void VulkanDevice::WaitIdle() { - std::lock_guard memoryLock(GuestMemoryTracking::GuestMemoryTrackingMutex_nid_postfix()); - PerformanceTimer timing("Vulkan.WaitIdle"); - state->drawQueue->Wait(); - timing.Mark("draw_wait"); + APS5_LOG_CHARS_OUT_DEBUG("VulkanDevice::WaitIdle begin"); + if (state->recorder) { + // Announced with this call's return address: the [recorder] site table then names the + // driver site that drains (suspend point, label fallback, fill fallback, device replacement). + if (!state->recorder->Idle()) Graphics::Recorder::CountSync(0, __builtin_return_address(0)); + state->recorder->Sync(); + } check(state->DeviceFunction("vkDeviceWaitIdle")(state->device), "vkDeviceWaitIdle"); - timing.Mark("device_wait"); + APS5_LOG_CHARS_OUT_DEBUG("VulkanDevice::WaitIdle complete"); } -void VulkanDevice::WaitDraws() { - std::lock_guard memoryLock(GuestMemoryTracking::GuestMemoryTrackingMutex_nid_postfix()); - PerformanceTimer timing("Vulkan.WaitDraws"); - state->drawQueue->Wait(); +namespace { + +bool OpportunisticReap() { + static const bool enabled = std::getenv("APS5_NO_OPPORTUNISTIC_REAP") == nullptr; + return enabled; +} + +} + +void VulkanDevice::SubmitRecorded(bool reapFirst) { + if (!state->recorder) return; + // Finished batches are retired first (a non-blocking fence status check on the front of the + // in-flight list), so PendingWriteOverlaps/HasCompletions scans stay short and a label recorded + // after a completing batch already finished can go to the GPU instead of behind a completion. + // Not for cross-queue submitters: a retired batch's write-back may sync a later batch under the + // mutex (see the header), which a poller or compute worker must never do for queue 0. + if (reapFirst && OpportunisticReap()) state->recorder->Reap(); + state->recorder->Submit(); +} + +bool VulkanDevice::CanWaitUnlocked() const { + return state->recorder && state->recorder->HasTimeline(); +} + +std::uint64_t VulkanDevice::SubmitAndEpoch() { + if (!state->recorder) return 0; + if (OpportunisticReap()) state->recorder->Reap(); + return state->recorder->SubmitAndEpoch(); +} + +void VulkanDevice::WaitRecorded(std::uint64_t serial) { + if (state->recorder) state->recorder->WaitSerial(serial); +} + +void VulkanDevice::ReapRecorded(std::uint64_t serial) { + if (!state->recorder) return; + // The drain's site in the [recorder] table is this call's caller (Driver::execute's drain, + // PrepareDispatch's presync), not this wrapper; FinishUpTo consumes the announcement. + Graphics::Recorder::AnnounceSyncSite(__builtin_return_address(0)); + state->recorder->FinishUpTo(serial); +} + +void VulkanDevice::ReapRecorded() { + if (state->recorder) state->recorder->Reap(); +} + +std::optional VulkanDevice::PendingLabel(std::uint64_t address, std::size_t bytes, std::uint64_t afterStamp, std::uint32_t& queue) const { + if (!state->recorder) return std::nullopt; + return state->recorder->PendingLabel(address, bytes, afterStamp, queue); +} + +bool VulkanDevice::OpenWriteOverlaps(std::uint64_t address, std::size_t bytes) const { + return state->recorder && state->recorder->OpenWriteOverlaps(address, bytes); +} + +int VulkanDevice::WriteLabelOnGpu(std::uint64_t address, std::span bytes, std::uint64_t stamp, std::uint32_t queue, bool reapFirst) { + if (!state->recorder || bytes.empty() || bytes.size() > 65536 || address % 4 != 0 || bytes.size() % 4 != 0) return 4; + auto& recorder = *state->recorder; + // Batches that already finished are retired before the checks: their completions ran, so they + // neither keep the recorder busy nor force this label behind a completion. Only for the + // graphics worker: a reap runs completions (copied write-backs, whose guest stores can re-enter + // the flush hook and sync a later batch) under the mutex, which a compute worker must never do + // against queue 0; its label goes behind a completion instead (reason 5, still no wait). + // Debug aid: APS5_LABEL_REAP_ALL_QUEUES=1 reaps on every queue as before. + static const bool reapAllQueues = std::getenv("APS5_LABEL_REAP_ALL_QUEUES") != nullptr; + if (reapFirst && (queue == 0 || reapAllQueues) && OpportunisticReap()) recorder.Reap(); + if (recorder.Idle()) return 1; + static const bool drain = std::getenv("APS5_DRAIN_COMPLETION_LABELS") != nullptr; + const auto context = graphicsContext(); + const auto* import = Graphics::HostImportFor(context, address, bytes.size()); + if (import == nullptr) { + // Memory the GPU has no view of: the store is a completion action of the batch (it runs after + // the recorded work, in order, like a label behind write-backs) instead of a device drain. + if (drain) return 3; + recorder.AfterCompletions(address, bytes, stamp, queue, false); + return 6; + } + // The store on the GPU, into the open batch: everything recorded so far completes before it, + // and it is visible to the host after. Both barriers stay per store: the first orders + // consecutive stores to one address (WAW), the second makes each store host-visible on its own, + // before the rest of the batch completes. + const auto recordStore = [&] { + const auto commands = recorder.Commands(); + Graphics::RecordMemoryBarrier(context, commands, VK_PIPELINE_STAGE_ALL_COMMANDS_BIT, VK_PIPELINE_STAGE_TRANSFER_BIT, VK_ACCESS_SHADER_WRITE_BIT | VK_ACCESS_TRANSFER_WRITE_BIT | VK_ACCESS_COLOR_ATTACHMENT_WRITE_BIT, VK_ACCESS_TRANSFER_WRITE_BIT); + state->DeviceFunction("vkCmdUpdateBuffer")(commands, import->buffer, address - import->base, bytes.size(), bytes.data()); + Graphics::RecordMemoryBarrier(context, commands, VK_PIPELINE_STAGE_TRANSFER_BIT, VK_PIPELINE_STAGE_HOST_BIT | VK_PIPELINE_STAGE_ALL_COMMANDS_BIT, VK_ACCESS_TRANSFER_WRITE_BIT, VK_ACCESS_HOST_READ_BIT | VK_ACCESS_SHADER_READ_BIT | VK_ACCESS_TRANSFER_READ_BIT); + }; + if (recorder.HasCompletions()) { + // A copied buffer's write-back is still to run: the store must land after it. It is stored + // by a completion action of the batch (a CPU memcpy when the batch is reaped) instead of + // draining the device here. The same bytes are also recorded on the GPU into the open batch + // first, so GPU work recorded after this point that reads the range through the host import + // (a shader binding constants a DUMP_CONST_RAM dumped, a dispatch consuming a COPY_DATA or + // DMA_DATA fill) sees them in order; the completion memcpy stores identical bytes again + // after any write-back that overwrote them, so either order of the two stores is correct. + // CPU readers are covered by the pending-write note (the flush hook syncs) and waits by the + // label table. Debug aid: APS5_DRAIN_COMPLETION_LABELS=1 drains as before; try it first when + // a title misbehaves with this path. + if (drain) return 2; + recordStore(); + GuestMemory::MarkWritten(address, bytes.size()); + // Commands() opened the batch the store went into, so the completion is appended to that + // batch and runs at its reap, after the write-backs of every batch before it. + recorder.AfterCompletions(address, bytes, stamp, queue, true); + return 5; + } + recordStore(); + // The table entry goes in before the pending-write note: the note bumps the write generation a + // poller watches, and a poller that sees the bump then finds the label without the GPU mutex. + recorder.NoteLabel(address, bytes, stamp, queue); + recorder.NotePendingWrite(address, bytes.size()); + GuestMemory::MarkWritten(address, bytes.size()); + // No submit per label: the batch goes out at the queue worker's next non-label packet, at a + // wait on its range, after APS5_LABEL_FLUSH_US, or at the submission's end (Driver.cpp). + // Debug aid: APS5_LABEL_SUBMIT_NOW=1 submits every label at once, as before. + static const bool submitNow = std::getenv("APS5_LABEL_SUBMIT_NOW") != nullptr; + if (submitNow) recorder.Submit(); + return 0; +} + +bool VulkanDevice::FillBuffer(std::uint64_t address, std::size_t bytes, std::span pattern) { + if (!state->recorder || bytes == 0 || bytes % 16 != 0 || address % 16 != 0) return false; + auto& recorder = *state->recorder; + // Finished batches are retired first (one fence status check, as WriteLabelOnGpu does): their + // completion labels landed and their copied writers are delisted, so a batch the idle GPU already + // finished does not force a wait below. Before the import lookup: a completion may refresh the + // import table. Debug aid: APS5_NO_OPPORTUNISTIC_REAP=1 skips it. + if (OpportunisticReap()) recorder.Reap(); + const auto context = graphicsContext(); + const Graphics::HostImport* import = Graphics::HostImportFor(context, address, bytes); + if (import == nullptr) return false; + // Earlier recorded work that stores into the range must land before the fill, or it would + // overwrite the fill later. Stores the GPU makes (direct dispatch writes, GPU labels, earlier + // fills, GPU-direct write-backs) are ordered by the fill's own barrier below (ALL_COMMANDS -> + // TRANSFER), so only the stores the CPU makes when a batch is reaped need a wait: a recorded + // dispatch whose copied buffer overlaps the range (state->copiedWriters, listed until its + // write-back ran; per object over all its written V# ranges, as DispatchIndirect tests it) and a + // completion-deferred label (Recorder::AfterCompletions) over the range. The label table has no + // range query and a fill spans MiBs, so a small range is looked up per dword (a GPU label there + // counts too: a harmless extra wait) and a larger one waits whenever any completion label is + // pending at all. Batches recorded after the last such store keep running (SyncThrough). + // Debug aid: APS5_FILL_SYNC=1 waits for every recorded store over the range, as before. + static const bool alwaysSync = std::getenv("APS5_FILL_SYNC") != nullptr; + static const bool profile = std::getenv("APS5_PROFILE_DRAW") != nullptr; + // APS5_PROFILE_DRAW: the fills' outcomes, cumulative, every 10 s (under the GpuMutex like the + // rest) on a [fill-sync] line; Driver.cpp's [fill] line counts the fills and their bytes. + // `labelSynced` counts syncs taken because a completion label was pending: over the range for + // a small fill, anywhere at all for a larger one. + static std::uint64_t fills = 0, synced = 0, labelSynced = 0, ordered = 0; + static double syncedMs = 0; + static auto lastReport = std::chrono::steady_clock::now(); + ++fills; + if (recorder.PendingWriteOverlaps(address, bytes)) { + // Recorded draws with copied writes keep their own registry (Graphics::DrawCopiedWriters): + // their CPU write-back would land after a fill recorded on the GPU, so they count too. + bool sync = alwaysSync || std::any_of(state->copiedWriters->begin(), state->copiedWriters->end(), [&](const auto& writer) { return writer->WritesOverlap(address, bytes); }) + || std::any_of(Graphics::DrawCopiedWriters()->begin(), Graphics::DrawCopiedWriters()->end(), [&](const auto& writer) { return writer->WritesOverlap(address, bytes); }); + if (!sync && Graphics::Recorder::PendingCompletionLabels() != 0) { + // A label the CPU will store after a completion must not be overwritten by the fill: + // only a label inside the filled range matters, whatever the range's size. + sync = recorder.PendingLabelIn(address, bytes); + if (sync) ++labelSynced; + } + if (sync) { + ++synced; + const auto syncStart = std::chrono::steady_clock::now(); + Graphics::Recorder::CountSync(4); + recorder.SyncThrough(address, bytes); + syncedMs += std::chrono::duration(std::chrono::steady_clock::now() - syncStart).count(); + // The sync ran completions, whose write-backs can refresh the import table and retire + // the import looked up above; the caller stores the fill itself when it is gone. + import = Graphics::HostImportFor(context, address, bytes); + if (import == nullptr) return false; + } else { + ++ordered; + } + } + if (profile) { + const auto now = std::chrono::steady_clock::now(); + if (now - lastReport > std::chrono::seconds(10)) { + lastReport = now; + std::fprintf(stderr, "[fill-sync] fills: %llu synced (%llu with a completion label pending) waited %.0f ms, %llu ordered by barrier, %llu with no pending write\n", static_cast(synced), static_cast(labelSynced), syncedMs, static_cast(ordered), static_cast(fills - synced - ordered)); + } + } + const auto commands = recorder.Commands(); + // Every earlier recorded read or write of the range precedes the fill (WAR by the execution + // dependency, WAW by the writes made available), whichever stage made it. + Graphics::RecordMemoryBarrier(context, commands, VK_PIPELINE_STAGE_ALL_COMMANDS_BIT, VK_PIPELINE_STAGE_TRANSFER_BIT, VK_ACCESS_SHADER_WRITE_BIT | VK_ACCESS_TRANSFER_WRITE_BIT | VK_ACCESS_HOST_WRITE_BIT, VK_ACCESS_TRANSFER_WRITE_BIT); + const auto offset = address - import->base; + if (pattern[0] == pattern[1] && pattern[1] == pattern[2] && pattern[2] == pattern[3]) { + state->DeviceFunction("vkCmdFillBuffer")(commands, import->buffer, offset, bytes, pattern[0]); + } else { + // A 16-byte pattern is seeded once and doubled in place until the range is covered. + auto seed = std::make_shared(context, 16, VK_BUFFER_USAGE_TRANSFER_SRC_BIT); + std::memcpy(seed->Bytes().data(), pattern.data(), 16); + recorder.Keep(seed); + const VkBufferCopy first{0, offset, 16}; + state->DeviceFunction("vkCmdCopyBuffer")(commands, seed->Handle(), import->buffer, 1, &first); + for (std::size_t done = 16; done < bytes;) { + const auto chunk = std::min(done, bytes - done); + Graphics::RecordMemoryBarrier(context, commands, VK_PIPELINE_STAGE_TRANSFER_BIT, VK_PIPELINE_STAGE_TRANSFER_BIT, VK_ACCESS_TRANSFER_WRITE_BIT, VK_ACCESS_TRANSFER_READ_BIT); + const VkBufferCopy copy{offset, offset + done, chunk}; + state->DeviceFunction("vkCmdCopyBuffer")(commands, import->buffer, import->buffer, 1, ©); + done += chunk; + } + } + // The filled bytes are visible to everything recorded after: shaders, transfers, the host, and + // an indirect dispatch reading its group counts in place (DispatchIndirect adds its own + // ALL_COMMANDS -> DRAW_INDIRECT barrier as well). A later GPU label store over the range is + // ordered behind the fill by its ALL_COMMANDS -> TRANSFER barrier (WriteLabelOnGpu). + Graphics::RecordMemoryBarrier(context, commands, VK_PIPELINE_STAGE_TRANSFER_BIT, VK_PIPELINE_STAGE_ALL_COMMANDS_BIT | VK_PIPELINE_STAGE_HOST_BIT, VK_ACCESS_TRANSFER_WRITE_BIT, VK_ACCESS_SHADER_READ_BIT | VK_ACCESS_SHADER_WRITE_BIT | VK_ACCESS_TRANSFER_READ_BIT | VK_ACCESS_INDIRECT_COMMAND_READ_BIT | VK_ACCESS_HOST_READ_BIT); + recorder.NotePendingWrite(address, bytes); + GuestMemory::MarkWritten(address, bytes); + return true; } void* VulkanDevice::Window() const { @@ -512,24 +866,27 @@ void* VulkanDevice::Window() const { } void VulkanDevice::Resize(std::uint32_t width, std::uint32_t height) { - std::lock_guard memoryLock(GuestMemoryTracking::GuestMemoryTrackingMutex_nid_postfix()); + APS5_LOG_OUT_DEBUG("Resize requested=%ux%u current=%ux%u", width, height, state->extent.width, state->extent.height); require(state->swapchain != VK_NULL_HANDLE, "cannot resize an unavailable swapchain"); if (width == 0 || height == 0) { state->extent = {0, 0}; return; } + if (state->extent.width == width && state->extent.height == height) return; + WaitIdle(); VkSurfaceCapabilitiesKHR surface{}; check(state->InstanceFunction("vkGetPhysicalDeviceSurfaceCapabilitiesKHR")(state->physical, state->surface, &surface), "vkGetPhysicalDeviceSurfaceCapabilitiesKHR resize"); + // The window's drawable size and the surface's extent can disagree for a moment while the window + // is being resized; the swapchain must match the surface, and the next present catches up. if (surface.currentExtent.width != std::numeric_limits::max()) { width = surface.currentExtent.width; height = surface.currentExtent.height; + if (width == 0 || height == 0) { + state->extent = {0, 0}; + return; + } + if (state->extent.width == width && state->extent.height == height) return; } - if (width == 0 || height == 0) { - state->extent = {0, 0}; - return; - } - if (!state->swapchainState.NeedsRecreation() && state->extent.width == width && state->extent.height == height) return; - WaitIdle(); require(width >= surface.minImageExtent.width && width <= surface.maxImageExtent.width && height >= surface.minImageExtent.height && height <= surface.maxImageExtent.height, "unsupported resized output extent"); require((surface.supportedUsageFlags & VK_IMAGE_USAGE_TRANSFER_DST_BIT) != 0 && (surface.supportedCompositeAlpha & VK_COMPOSITE_ALPHA_OPAQUE_BIT_KHR) != 0 && (surface.supportedTransforms & VK_SURFACE_TRANSFORM_IDENTITY_BIT_KHR) != 0, "resized surface capabilities are unsupported"); VkSwapchainCreateInfoKHR create{VK_STRUCTURE_TYPE_SWAPCHAIN_CREATE_INFO_KHR}; @@ -558,60 +915,200 @@ void VulkanDevice::Resize(std::uint32_t width, std::uint32_t height) { check(getImages(state->device, replacement, &count, state->images.data()), "vkGetSwapchainImagesKHR resize"); state->images.resize(count); state->rendered.assign(count, VK_NULL_HANDLE); - state->swapchainState.Recreated(); + APS5_LOG_OUT_DEBUG("Resize complete swapchain=%p extent=%ux%u images=%u", reinterpret_cast(state->swapchain), state->extent.width, state->extent.height, count); } bool VulkanDevice::Presentable() const { return state->extent.width != 0 && state->extent.height != 0; } -void VulkanDevice::PresentClear(std::uint32_t width, std::uint32_t height, bool opaque) { - present(width, height, opaque, {}); +bool VulkanDevice::PresentClear(std::uint32_t width, std::uint32_t height, bool opaque) { + APS5_LOG_OUT_DEBUG("PresentClear width=%u height=%u opaque=%u", width, height, static_cast(opaque)); + return present(width, height, opaque, {}); } void VulkanDevice::PresentPixels(std::uint32_t width, std::uint32_t height, std::span pixels) { + APS5_LOG_OUT("PresentPixels width=%u height=%u pixels=%zu expected=%llu", width, height, pixels.size(), static_cast(width) * height * 4u); require(width != 0 && height != 0 && width <= 16384 && height <= 16384, "invalid display image extent"); require(pixels.size() == static_cast(width) * height * 4, "invalid display pixel buffer size"); - present(width, height, true, pixels); -} - -void VulkanDevice::PresentDisplayBuffer(const DisplayBuffer& buffer) { - present(buffer.width, buffer.height, true, {}, &buffer); -} - -void VulkanDevice::present(std::uint32_t width, std::uint32_t height, bool opaque, std::span pixels, const DisplayBuffer* display) { - std::lock_guard memoryLock(GuestMemoryTracking::GuestMemoryTrackingMutex_nid_postfix()); - PerformanceTimer timing("Vulkan.Present"); - state->drawQueue->Wait(); - timing.Mark("draw_wait"); - require(state->swapchain != VK_NULL_HANDLE, "device has no swapchain"); - require(state->extent.width != 0 && state->extent.height != 0, "output window is minimized"); - std::shared_ptr resident; - if (display != nullptr) { - const auto bytes = DisplayBufferSize(*display); - resident = state->renderCache->Find(display->address); - if (resident) { - const auto& color = resident->Description(); - if (color.extent.width != width || color.extent.height != height || color.bytes != bytes || color.tileMode != Graphics::ColorTileMode::RenderTarget) resident.reset(); - } - if (!resident) { - ResolveMemory(display->address, bytes, false); - state->colorTransfer->Upload(display->address, width, height, Graphics::ColorTileMode::RenderTarget); + // Debug aid: APS5_DUMP_FRAMES= saves the first n presented frames as frame_.bmp. + static const int dumpLimit = [] { + const char* value = std::getenv("APS5_DUMP_FRAMES"); + return value ? std::atoi(value) : 0; + }(); + static int dumped = 0; + if (dumped < dumpLimit) { + char name[32]; + std::snprintf(name, sizeof(name), "frame_%03d.bmp", dumped++); + if (std::FILE* file = std::fopen(name, "wb")) { + const std::uint32_t imageBytes = width * height * 4; + const std::uint32_t header[13] = {0, 0, 54, 40, width, static_cast(-static_cast(height)), 1u | (32u << 16u), 0, imageBytes, 2835, 2835, 0, 0}; + const std::uint16_t magic = 0x4d42; + const std::uint32_t fileBytes = 54 + imageBytes; + std::fwrite(&magic, 2, 1, file); + std::fwrite(&fileBytes, 4, 1, file); + std::fwrite(header + 1, 4, 12, file); + std::fwrite(pixels.data(), 1, pixels.size(), file); + std::fclose(file); } } - auto* scaler = resident && display->pixelFormat == 0x8000000022000000ull ? state->rgbaScaler.get() : state->scaler.get(); - if (!pixels.empty()) state->Upload(pixels); - timing.Mark("pixel_upload"); + // Tests and tools present pixels synchronously; the game path splits the steps (see Driver::Present). + if (!present(width, height, true, pixels)) return; + FinishPresent(); + QueuePresent(); +} + +namespace { + +// APS5_DUMP_SCALE= (default 4) keeps every n-th pixel in each direction, so long runs of 4K +// frames stay small. +std::uint32_t DumpScale() { + static const std::uint32_t scale = [] { + const char* value = std::getenv("APS5_DUMP_SCALE"); + const int parsed = value ? std::atoi(value) : 4; + return static_cast(parsed > 0 ? parsed : 1); + }(); + return scale; +} + +// frame_.bmp: every DumpScale()-th pixel of a BGRA8 image, 32-bit top-down. +void WriteFrameBmp(int index, std::uint32_t fullWidth, std::uint32_t fullHeight, std::span full) { + const auto scale = DumpScale(); + const std::uint32_t width = (fullWidth + scale - 1) / scale; + const std::uint32_t height = (fullHeight + scale - 1) / scale; + std::vector pixels(static_cast(width) * height * 4); + for (std::uint32_t y = 0; y < height; ++y) { + for (std::uint32_t x = 0; x < width; ++x) std::memcpy(pixels.data() + (static_cast(y) * width + x) * 4, full.data() + (static_cast(y) * scale * fullWidth + x * scale) * 4, 4); + } + char name[32]; + std::snprintf(name, sizeof(name), "frame_%03d.bmp", index); + if (std::FILE* file = std::fopen(name, "wb")) { + const std::uint32_t imageBytes = width * height * 4; + const std::uint32_t header[13] = {0, 0, 54, 40, width, static_cast(-static_cast(height)), 1u | (32u << 16u), 0, imageBytes, 2835, 2835, 0, 0}; + const std::uint16_t magic = 0x4d42; + const std::uint32_t fileBytes = 54 + imageBytes; + std::fwrite(&magic, 2, 1, file); + std::fwrite(&fileBytes, 4, 1, file); + std::fwrite(header + 1, 4, 12, file); + std::fwrite(pixels.data(), 1, pixels.size(), file); + std::fclose(file); + } +} + +// Whether a resident image of the display buffer can be blitted to the swapchain as it is: one 2D +// level of the display's extent in a normalized 4-byte color format (a blit converts components, +// so an integer or float reinterpretation would present wrong values). `filter` gets the best +// filter the format allows. +bool ResidentPresentable(const Graphics::Context& context, const Graphics::StorageTexture& image, const DisplayBuffer& buffer, VkFilter& filter) { + const auto& descriptor = image.Descriptor(); + if (descriptor.width != buffer.width || descriptor.height != buffer.height || descriptor.mipCount != 1 || image.ImageLayers() != 1 || image.ImageDepth() != 1) return false; + if (image.GuestBytes() != DisplayBufferSize(buffer)) return false; + const auto format = Graphics::StorageFormatForGuest(context, descriptor.format); + switch (format) { + case VK_FORMAT_R8G8B8A8_UNORM: case VK_FORMAT_B8G8R8A8_UNORM: case VK_FORMAT_A8B8G8R8_UNORM_PACK32: + case VK_FORMAT_A2B10G10R10_UNORM_PACK32: case VK_FORMAT_A2R10G10B10_UNORM_PACK32: + break; + default: return false; + } + VkFormatProperties properties{}; + context.formatProperties(context.physical, format, &properties); + if ((properties.optimalTilingFeatures & VK_FORMAT_FEATURE_BLIT_SRC_BIT) == 0) return false; + filter = (properties.optimalTilingFeatures & VK_FORMAT_FEATURE_SAMPLED_IMAGE_FILTER_LINEAR_BIT) != 0 ? VK_FILTER_LINEAR : VK_FILTER_NEAREST; + return true; +} + +} + +bool VulkanDevice::PresentDisplayBuffer(const DisplayBuffer& buffer) { + static const bool profile = std::getenv("APS5_PROFILE_DRAW") != nullptr; + // The display buffer is usually a resident render target whose results are still on the GPU: it + // is blitted from that image, which skips the write-back retile, the 33 MB read of guest memory, + // the upload and the detile dispatch. The deferred write-back stays pending as for any storage + // image. Debug aid: APS5_NO_RESIDENT_PRESENT=1 always goes through guest memory. + static const bool noResidentPresent = std::getenv("APS5_NO_RESIDENT_PRESENT") != nullptr; + static std::uint64_t residentPresents = 0, refreshedPresents = 0, notPending = 0, unsuitable = 0, gpuDumps = 0; + static auto lastReport = std::chrono::steady_clock::now(); + const auto bytes = DisplayBufferSize(buffer); + std::shared_ptr resident; + VkFilter filter = VK_FILTER_LINEAR; + if (!noResidentPresent) { + resident = Graphics::StorageTexture::FindPending(buffer.address, bytes); + if (resident == nullptr) { + ++notPending; + } else if (!ResidentPresentable(graphicsContext(), *resident, buffer, filter)) { + resident.reset(); + ++unsuitable; + } else { + // 64 KiB blocks the CPU wrote since the image last matched guest memory would be shown + // stale (the write-back keeps the CPU's bytes for them): Refresh merges them into the + // image first, as a sampled texture of the image would (see cachedTexture). The uncached + // walk: the collect memo is per worker packet, so on this thread a memoized answer could + // miss a CPU write that landed after a worker's walk of the same range. + GuestMemory::CollectWritesUncached(buffer.address, bytes); + if (!GuestMemory::UnchangedSince(buffer.address, bytes, resident->Generation())) { + resident->Refresh(); + ++refreshedPresents; + } + ++residentPresents; + } + } + // Debug aid: APS5_DUMP_FRAMES= also covers the GPU display path: the presented frame is read + // back by the same submission and written after its fence (see writeFrameDump). + // APS5_NO_GPU_DUMP=1 decodes the tiled guest buffer on the CPU instead, as before. + static const int dumpLimit = [] { + const char* value = std::getenv("APS5_DUMP_FRAMES"); + return value ? std::atoi(value) : 0; + }(); + static const bool cpuDump = std::getenv("APS5_NO_GPU_DUMP") != nullptr; + static int dumped = 0; + bool dumpFrame = false; + if (dumped < dumpLimit) { + if (cpuDump) { + const auto full = ReadDisplayBuffer(buffer); + WriteFrameBmp(dumped++, buffer.width, buffer.height, full); + } else { + state->dumpIndex = dumped++; + dumpFrame = true; + ++gpuDumps; + } + } + if (profile && std::chrono::steady_clock::now() - lastReport > std::chrono::seconds(10)) { + lastReport = std::chrono::steady_clock::now(); + std::fprintf(stderr, "[flip] %llu presents from the resident image (%llu refreshed first), through guest memory: %llu not pending, %llu unsuitable; %llu GPU frame dumps\n", static_cast(residentPresents), static_cast(refreshedPresents), static_cast(notPending), static_cast(unsuitable), static_cast(gpuDumps)); + } + if (!present(buffer.width, buffer.height, true, {}, &buffer, resident, filter, dumpFrame)) { + // A dropped frame (swapchain out of date) keeps the dump numbering contiguous. + if (dumpFrame) --dumped; + return false; + } + return true; +} + +bool VulkanDevice::AcquireImage() { + PerformanceTimer timing("Vulkan.Acquire"); + require(state->swapchain != VK_NULL_HANDLE, "device has no swapchain"); + require(state->extent.width != 0 && state->extent.height != 0, "output window is minimized"); + require(!state->presentPending, "the previous presentation was not finished"); + require(!state->imageAcquired, "the previously acquired image was not presented"); auto wait = state->DeviceFunction("vkWaitForFences"); auto reset = state->DeviceFunction("vkResetFences"); const std::array fences{state->acquireFence, state->renderFence}; check(reset(state->device, static_cast(fences.size()), fences.data()), "vkResetFences"); + APS5_LOG_CHARS_OUT_DEBUG("present fences reset"); std::uint32_t index = 0; timing.Mark("fence_reset"); - if (!state->swapchainState.ProcessResult(state->DeviceFunction("vkAcquireNextImageKHR")(state->device, state->swapchain, 5'000'000'000ULL, VK_NULL_HANDLE, state->acquireFence, &index), "vkAcquireNextImageKHR")) return; + // An out-of-date swapchain (the window changed) drops this frame; the next Resize recreates it. + const auto acquired = state->DeviceFunction("vkAcquireNextImageKHR")(state->device, state->swapchain, 5'000'000'000ULL, VK_NULL_HANDLE, state->acquireFence, &index); + if (acquired == VK_ERROR_OUT_OF_DATE_KHR) { + state->extent = {0, 0}; + return false; + } + if (acquired != VK_SUBOPTIMAL_KHR) check(acquired, "vkAcquireNextImageKHR"); timing.Mark("acquire_image"); + APS5_LOG_OUT_DEBUG("vkAcquireNextImageKHR index=%u imageCount=%zu", index, state->images.size()); check(wait(state->device, 1, &state->acquireFence, VK_TRUE, std::numeric_limits::max()), "vkWaitForFences acquire"); timing.Mark("acquire_fence_wait"); + APS5_LOG_OUT_DEBUG("Acquire fence complete index=%u", index); require(index < state->images.size() && index < state->rendered.size(), "acquired image index is out of range"); auto& rendered = state->rendered[index]; if (rendered != VK_NULL_HANDLE) { @@ -620,12 +1117,41 @@ void VulkanDevice::present(std::uint32_t width, std::uint32_t height, bool opaqu VkSemaphoreCreateInfo semaphore{VK_STRUCTURE_TYPE_SEMAPHORE_CREATE_INFO}; check(state->DeviceFunction("vkCreateSemaphore")(state->device, &semaphore, nullptr, &rendered), "vkCreateSemaphore presentation"); } - auto commands = state->clearCommands; timing.Mark("retired_swapchains"); + state->acquiredIndex = index; + state->imageAcquired = true; + return true; +} + +bool VulkanDevice::present(std::uint32_t width, std::uint32_t height, bool opaque, std::span pixels, const DisplayBuffer* display, const std::shared_ptr& resident, VkFilter residentFilter, bool dumpFrame) { + PerformanceTimer timing("Vulkan.Present"); + APS5_LOG_OUT_DEBUG("present begin width=%u height=%u opaque=%u pixels=%zu", width, height, static_cast(opaque), pixels.size()); + require(state->swapchain != VK_NULL_HANDLE, "device has no swapchain"); + require(state->extent.width != 0 && state->extent.height != 0, "output window is minimized"); + require(!state->presentPending, "the previous presentation was not finished"); + // The game path acquired the image before taking GpuMutex (Driver::Present); tests, tools and + // APS5_SYNC_FLIP=1 acquire here. + if (!state->imageAcquired && !AcquireImage()) return false; + state->imageAcquired = false; + const auto index = state->acquiredIndex; + require(index < state->images.size() && index < state->rendered.size(), "acquired image index is out of range"); + auto& rendered = state->rendered[index]; + if (display != nullptr && resident == nullptr) { + static_cast(DisplayBufferSize(*display)); + state->colorTransfer->Upload(display->address, width, height, Graphics::ColorTileMode::RenderTarget); + } + if (!pixels.empty()) state->Upload(pixels); + // The frame's recorded work (and a refresh of the resident image) must reach the queue before the + // presentation's own submission, which reads the image in queue order. + SubmitRecorded(); + timing.Mark("pixel_upload"); + APS5_LOG_OUT_DEBUG("present source=%s bytes=%zu", pixels.empty() ? "clear" : "pixels", pixels.size()); + auto commands = state->clearCommands; check(state->DeviceFunction("vkResetCommandBuffer")(commands, 0), "vkResetCommandBuffer"); VkCommandBufferBeginInfo begin{VK_STRUCTURE_TYPE_COMMAND_BUFFER_BEGIN_INFO}; begin.flags = VK_COMMAND_BUFFER_USAGE_ONE_TIME_SUBMIT_BIT; check(state->DeviceFunction("vkBeginCommandBuffer")(commands, &begin), "vkBeginCommandBuffer"); + APS5_LOG_OUT_DEBUG("Presentation command buffer begin commands=%p image=%p", reinterpret_cast(commands), reinterpret_cast(state->images[index])); VkImageMemoryBarrier barrier{VK_STRUCTURE_TYPE_IMAGE_MEMORY_BARRIER}; barrier.dstAccessMask = VK_ACCESS_TRANSFER_WRITE_BIT; barrier.oldLayout = VK_IMAGE_LAYOUT_UNDEFINED; @@ -637,18 +1163,41 @@ void VulkanDevice::present(std::uint32_t width, std::uint32_t height, bool opaqu auto pipelineBarrier = state->DeviceFunction("vkCmdPipelineBarrier"); pipelineBarrier(commands, VK_PIPELINE_STAGE_TOP_OF_PIPE_BIT, VK_PIPELINE_STAGE_TRANSFER_BIT, 0, 0, nullptr, 0, nullptr, 1, &barrier); if (pixels.empty() && display == nullptr) { + APS5_LOG_OUT_DEBUG("Recording swapchain clear opaque=%u image=%p", static_cast(opaque), reinterpret_cast(barrier.image)); VkClearColorValue clear{}; clear.float32[3] = opaque ? 1.0f : 0.0f; state->DeviceFunction("vkCmdClearColorImage")(commands, barrier.image, VK_IMAGE_LAYOUT_TRANSFER_DST_OPTIMAL, &clear, 1, &barrier.subresourceRange); } else { - require(scaler != nullptr, "presentation scaler is unavailable"); - scaler->EnsureSourceImage(width, height); - if (resident) { - resident->Transition(commands, VK_IMAGE_LAYOUT_TRANSFER_SRC_OPTIMAL); - scaler->RecordImage(commands, resident->Target().Image()); + APS5_LOG_OUT_DEBUG("Recording swapchain scaled blit width=%u height=%u bytes=%zu buffer=%p image=%p", width, height, pixels.size(), reinterpret_cast(state->uploadBuffer), reinterpret_cast(barrier.image)); + require(state->scaler != nullptr, "presentation scaler is unavailable"); + state->scaler->EnsureSourceImage(width, height); + // A resident image is blitted straight to the swapchain; a frame dump goes through the + // scaler's BGRA8 image so the readback has one format. + const bool direct = resident != nullptr && !dumpFrame; + VkImageMemoryBarrier residentBarrier{VK_STRUCTURE_TYPE_IMAGE_MEMORY_BARRIER}; + if (resident != nullptr) { + // The draws and dispatches that produced the image were submitted before this command + // buffer; their writes become visible to the blit here. The layout stays GENERAL, the + // one the storage image is tracked in. + residentBarrier.srcAccessMask = VK_ACCESS_COLOR_ATTACHMENT_WRITE_BIT | VK_ACCESS_SHADER_WRITE_BIT | VK_ACCESS_TRANSFER_WRITE_BIT; + residentBarrier.dstAccessMask = VK_ACCESS_TRANSFER_READ_BIT; + residentBarrier.oldLayout = VK_IMAGE_LAYOUT_GENERAL; + residentBarrier.newLayout = VK_IMAGE_LAYOUT_GENERAL; + residentBarrier.srcQueueFamilyIndex = VK_QUEUE_FAMILY_IGNORED; + residentBarrier.dstQueueFamilyIndex = VK_QUEUE_FAMILY_IGNORED; + residentBarrier.image = resident->Image(); + residentBarrier.subresourceRange = {VK_IMAGE_ASPECT_COLOR_BIT, 0, VK_REMAINING_MIP_LEVELS, 0, VK_REMAINING_ARRAY_LAYERS}; + pipelineBarrier(commands, VK_PIPELINE_STAGE_ALL_COMMANDS_BIT, VK_PIPELINE_STAGE_TRANSFER_BIT, 0, 0, nullptr, 0, nullptr, 1, &residentBarrier); + if (!direct) state->scaler->RecordBlitInto(commands, resident->Image(), VK_IMAGE_LAYOUT_GENERAL, residentFilter); } else { - if (display != nullptr) state->colorTransfer->Detile(commands, display->pixelFormat == 0x8000000022000000ull); - scaler->RecordUpload(commands, display != nullptr ? state->colorTransfer->LinearBuffer() : state->uploadBuffer); + if (display != nullptr) { + // Bit 56 selects the 10-bit A2B10G10R10 variant; R8G8B8A8-ordered formats need red and blue + // swapped for the B8G8R8A8 swapchain. + constexpr std::uint64_t tenBitFormat = 0x0100000000000000ull; + const bool tenBit = (display->pixelFormat & tenBitFormat) != 0; + state->colorTransfer->Detile(commands, (display->pixelFormat & ~tenBitFormat) == 0x8000000022000000ull, tenBit); + } + state->scaler->RecordUpload(commands, display != nullptr ? state->colorTransfer->LinearBuffer() : state->uploadBuffer); } VkClearColorValue letterbox{}; letterbox.float32[3] = 1.0f; @@ -663,7 +1212,20 @@ void VulkanDevice::present(std::uint32_t width, std::uint32_t height, bool opaqu letterboxBarrier.image = barrier.image; letterboxBarrier.subresourceRange = barrier.subresourceRange; pipelineBarrier(commands, VK_PIPELINE_STAGE_TRANSFER_BIT, VK_PIPELINE_STAGE_TRANSFER_BIT, 0, 0, nullptr, 0, nullptr, 1, &letterboxBarrier); - scaler->RecordBlit(commands, barrier.image, state->extent.width, state->extent.height); + if (direct) PresentationScaler::RecordBlitFrom(graphicsContext(), commands, resident->Image(), VK_IMAGE_LAYOUT_GENERAL, width, height, residentFilter, barrier.image, state->extent.width, state->extent.height); + else state->scaler->RecordBlit(commands, barrier.image, state->extent.width, state->extent.height); + if (resident != nullptr) { + // Later batches write the image again (the next frame into this buffer, a refresh): they + // start after the blit has read it. + residentBarrier.srcAccessMask = VK_ACCESS_TRANSFER_READ_BIT; + residentBarrier.dstAccessMask = VK_ACCESS_COLOR_ATTACHMENT_WRITE_BIT | VK_ACCESS_SHADER_WRITE_BIT | VK_ACCESS_TRANSFER_WRITE_BIT; + pipelineBarrier(commands, VK_PIPELINE_STAGE_TRANSFER_BIT, VK_PIPELINE_STAGE_ALL_COMMANDS_BIT, 0, 0, nullptr, 0, nullptr, 1, &residentBarrier); + } + if (dumpFrame) { + const auto dumpBytes = static_cast(state->scaler->SourceWidth()) * state->scaler->SourceHeight() * 4; + if (!state->dumpBuffer || state->dumpBuffer->Bytes().size() != dumpBytes) state->dumpBuffer = std::make_unique(graphicsContext(), dumpBytes, VK_BUFFER_USAGE_TRANSFER_DST_BIT); + state->scaler->RecordReadback(commands, state->dumpBuffer->Handle()); + } } barrier.srcAccessMask = VK_ACCESS_TRANSFER_WRITE_BIT; barrier.dstAccessMask = 0; @@ -671,6 +1233,7 @@ void VulkanDevice::present(std::uint32_t width, std::uint32_t height, bool opaqu barrier.newLayout = VK_IMAGE_LAYOUT_PRESENT_SRC_KHR; pipelineBarrier(commands, VK_PIPELINE_STAGE_TRANSFER_BIT, VK_PIPELINE_STAGE_BOTTOM_OF_PIPE_BIT, 0, 0, nullptr, 0, nullptr, 1, &barrier); check(state->DeviceFunction("vkEndCommandBuffer")(commands), "vkEndCommandBuffer"); + APS5_LOG_CHARS_OUT_DEBUG("Presentation command buffer recorded"); VkSubmitInfo submit{VK_STRUCTURE_TYPE_SUBMIT_INFO}; submit.commandBufferCount = 1; submit.pCommandBuffers = &commands; @@ -679,16 +1242,64 @@ void VulkanDevice::present(std::uint32_t width, std::uint32_t height, bool opaqu timing.Mark("command_record_scale"); check(state->DeviceFunction("vkQueueSubmit")(state->queue, 1, &submit, state->renderFence), "vkQueueSubmit clear"); timing.Mark("queue_submit"); + APS5_LOG_CHARS_OUT_DEBUG("Presentation vkQueueSubmit OK"); + // The image stays alive until FinishPresent has waited for the blit (storage-cache eviction only + // drops the cache's reference). + state->presentKept = resident; + state->presentIndex = index; + state->presentPending = true; + // Only a submitted readback is written after the fence (a failed submit tears the device down). + state->dumpRecorded = dumpFrame; + return true; +} + +void VulkanDevice::FinishPresent() { + if (!state->presentPending) return; + PerformanceTimer timing("Vulkan.PresentWait"); + const auto wait = state->DeviceFunction("vkWaitForFences"); check(wait(state->device, 1, &state->renderFence, VK_TRUE, std::numeric_limits::max()), "vkWaitForFences clear"); timing.Mark("render_fence_wait"); + APS5_LOG_CHARS_OUT_DEBUG("Presentation render fence complete"); + if (state->dumpRecorded) { + state->dumpRecorded = false; + writeFrameDump(); + timing.Mark("frame_dump"); + } +} + +void VulkanDevice::writeFrameDump() { + require(state->dumpBuffer != nullptr && state->scaler != nullptr, "frame dump was not recorded"); + // The readback buffer is reused by the next dump, so its bytes are copied out before the writer + // thread (one at a time) downscales and writes them. + std::vector full(state->dumpBuffer->Bytes().begin(), state->dumpBuffer->Bytes().end()); + if (state->dumpWriter.joinable()) state->dumpWriter.join(); + const auto index = state->dumpIndex; + const auto width = state->scaler->SourceWidth(); + const auto height = state->scaler->SourceHeight(); + static const auto start = std::chrono::steady_clock::now(); + std::fprintf(stderr, "[gpu] frame_%03d.bmp: %ux%u display read back on the GPU at %.1f s\n", index, width, height, std::chrono::duration(std::chrono::steady_clock::now() - start).count()); + state->dumpWriter = std::thread([index, width, height, pixels = std::move(full)] { WriteFrameBmp(index, width, height, pixels); }); +} + +void VulkanDevice::QueuePresent() { + if (!state->presentPending) return; + PerformanceTimer timing("Vulkan.QueuePresent"); + state->presentPending = false; + // The finished submission was the last reader of the resident image. + state->presentKept.reset(); + const auto index = state->presentIndex; + require(index < state->rendered.size(), "presented image index is out of range"); VkPresentInfoKHR present{VK_STRUCTURE_TYPE_PRESENT_INFO_KHR}; present.waitSemaphoreCount = 1; - present.pWaitSemaphores = &rendered; + present.pWaitSemaphores = &state->rendered[index]; present.swapchainCount = 1; present.pSwapchains = &state->swapchain; present.pImageIndices = &index; - state->swapchainState.ProcessResult(state->DeviceFunction("vkQueuePresentKHR")(state->queue, &present), "vkQueuePresentKHR"); + const auto presented = state->DeviceFunction("vkQueuePresentKHR")(state->queue, &present); + if (presented == VK_ERROR_OUT_OF_DATE_KHR || presented == VK_SUBOPTIMAL_KHR) state->extent = {0, 0}; + else check(presented, "vkQueuePresentKHR"); timing.Mark("queue_present"); + APS5_LOG_OUT_DEBUG("vkQueuePresentKHR queued imageIndex=%u", index); } ShaderRecompiler::SpirvTarget VulkanDevice::Target() const { @@ -703,7 +1314,7 @@ ShaderRecompiler::SpirvTarget VulkanDevice::Target() const { } Graphics::Context VulkanDevice::graphicsContext() const { - return Graphics::Context{ + auto context = Graphics::Context{ state->device, state->physical, state->queue, @@ -728,40 +1339,160 @@ Graphics::Context VulkanDevice::graphicsContext() const { state->bufferPool, state->textureCache.get(), state->pipelineCache ? state->pipelineCache->Handle() : VK_NULL_HANDLE, - state->renderCache.get(), - state->drawQueue.get(), - state->graphicsPipelines.get(), - state->descriptorCache, - state->samplerCache + state->depthClamp, + state->hostImportAlignment }; -} - -void VulkanDevice::ResolveMemory(std::uint64_t address, std::size_t bytes, bool writable) { - std::lock_guard memoryLock(GuestMemoryTracking::GuestMemoryTrackingMutex_nid_postfix()); - state->drawQueue->Resolve(address, bytes); - state->renderCache->Resolve(address, bytes, writable); + context.recorder = state->recorder.get(); + context.descriptorCache = state->descriptorCache.get(); + context.samplerCache = state->samplerCache.get(); + return context; } void VulkanDevice::Draw(const Graphics::State& graphics, const Pm4::DrawParameters& draw, std::span shaders, std::span snapshots) { - EnqueueDraw(graphics, draw, shaders, snapshots); - WaitIdle(); -} - -void VulkanDevice::EnqueueDraw(const Graphics::State& graphics, const Pm4::DrawParameters& draw, std::span shaders, std::span snapshots) { - std::lock_guard memoryLock(GuestMemoryTracking::GuestMemoryTrackingMutex_nid_postfix()); - const GuestMemory::MemoryAccessScope memoryScope(this, [](void* context, std::uint64_t address, std::size_t bytes, bool writable) { - static_cast(context)->ResolveMemory(address, bytes, writable); - }); + APS5_LOG_OUT("VulkanDevice::Draw indices=%u instances=%u indexSize=%u address=0x%llx shaders=%zu colorTarget=%u", draw.indexCount, draw.instanceCount, draw.indexSize, static_cast(draw.indexAddress), shaders.size(), static_cast(graphics.hasColorTarget)); const auto context = graphicsContext(); Graphics::Draw(context, graphics, draw, shaders, snapshots); + APS5_LOG_CHARS_OUT("VulkanDevice::Draw complete"); } -void VulkanDevice::Dispatch(const ShaderRecompiler::RecompileResult& shader, std::uint32_t x, std::uint32_t y, std::uint32_t z, std::span snapshots) { - std::lock_guard memoryLock(GuestMemoryTracking::GuestMemoryTrackingMutex_nid_postfix()); +namespace { + +// The resource cache serves dispatches unless disabled; Revalidate proves a hit by texture identity, +// which needs the texture caches: without them every lookup makes a fresh object, so a hit could +// never validate and would only upload everything twice. +bool ResourceCacheEnabled() { + static const bool noResourceCache = std::getenv("APS5_NO_RESOURCE_CACHE") != nullptr; + static const bool noTextureCache = std::getenv("APS5_NO_TEXTURE_CACHE") != nullptr; + return !noResourceCache && !noTextureCache; +} + +// The content key of a compute stage, naming the device: the cache is process-wide and the driver +// replaces the headless device with the windowed one while workers may still use the old one, so +// the key names the device that built the entry (its descriptor set and pooled buffers belong to it). +ResourceCache::Key DispatchContentKey(const Graphics::CompiledShader& shader, VkDevice device) { + auto key = Graphics::ShaderResources::ContentKey(shader); + const auto deviceHandle = reinterpret_cast(device); + key.push_back(static_cast(deviceHandle)); + key.push_back(static_cast(deviceHandle >> 32u)); + return key; +} + +} + +struct PreparedDispatch { + // Stage A done; stage B (Complete) runs in the dispatch under the mutex. Null when the resource + // cache held the key's object at prepare time: the dispatch looks it up and revalidates it. + std::shared_ptr resources; + ResourceCache::Key key; + // APS5_PROFILE_DRAW: stage A's time, added to the [dispatch] phase totals by the dispatch. + double prepareMs = 0; + // The stage-A pre-sync (see PrepareDispatch): the newest recorder serial waited for without the + // mutex, whose batches the dispatch reaps under it before stage B (0: nothing waited for). + std::uint64_t presyncSerial = 0; +}; + +namespace { + +// APS5_PROFILE_DRAW: the pre-syncs made and the time they waited (unlocked), as [presync] every 10 s. +struct PresyncCounters { + std::atomic checked{0}; + std::atomic presyncs{0}; + std::atomic waitedUs{0}; + std::atomic lastReport{0}; +}; + +PresyncCounters& Presyncs() { + static PresyncCounters counters; + return counters; +} + +} + +std::shared_ptr VulkanDevice::PrepareDispatch(const ShaderRecompiler::RecompileResult& shader, std::span snapshots) { + // APS5_LOCKED_BUILD=1: the whole build under the mutex, as before the split. + static const bool lockedBuild = std::getenv("APS5_LOCKED_BUILD") != nullptr; + if (lockedBuild || shader.spirv.size() < 5 || shader.spirv[0] != 0x07230203u) return nullptr; + static const bool profile = std::getenv("APS5_PROFILE_DRAW") != nullptr; + const auto start = std::chrono::steady_clock::now(); + const Graphics::CompiledShader compute{ShaderRecompiler::ShaderStage::Compute, &shader, 0}; + const auto context = graphicsContext(); + auto prepared = std::make_shared(); + // A cached build (the map locks itself) is revalidated under the mutex by the dispatch, which + // looks it up again by the key made here; the object stays local, only its surfaces matter + // below. Anything inserted between here and the dispatch is simply replaced by this build. + std::shared_ptr cached; + if (ResourceCacheEnabled() && shader.variantId != 0) { + prepared->key = DispatchContentKey(compute, context.device); + cached = state->resourceCache.Find(prepared->key); + } + if (cached == nullptr) prepared->resources = std::make_shared(context, compute, snapshots, true); + // The pre-sync. Stage B's image lookups that read guest memory on the CPU (PresyncSurfaces) + // wait, through the flush hook and under the mutex, for the recorded work writing those + // surfaces: every other worker queues behind that wait. So the wait is made here instead, + // without the mutex: the lock-free pending-write snapshot says whether any recorded batch + // writes a surface; if so the mutex is taken only long enough to learn the newest such batch + // (submitting the open one when it is that batch), the timeline wait runs unlocked, and the + // dispatch reaps those batches at its own lock so the hook then finds nothing pending. Work + // noted in between falls back to the locked wait as before. A cached object's Revalidate + // repeats the same lookups when its stamps fail, so it gets the same pre-sync. Needs timeline + // semaphores. APS5_NO_PRESYNC=1 disables it. + static const bool noPresync = std::getenv("APS5_NO_PRESYNC") != nullptr; + if (!noPresync && CanWaitUnlocked()) { + auto& counters = Presyncs(); + counters.checked.fetch_add(1, std::memory_order_relaxed); + const auto surfaces = cached != nullptr ? cached->PresyncSurfaces() : prepared->resources->PresyncSurfaces(); + bool overlaps = false; + for (const auto& [address, bytes] : surfaces) overlaps = overlaps || Graphics::Recorder::SnapshotWriteOverlaps(address, static_cast(bytes)); + if (overlaps) { + std::uint64_t serial = 0; + { + // No reap here (SubmitAndEpoch's would run completions, which a compute worker must + // not do on queue 0's behalf, see SubmitRecorded): the dispatch reaps after the wait. + std::lock_guard lock(GuestMemory::GpuMutex()); + if (state->recorder) { + bool open = false; + for (const auto& [address, bytes] : surfaces) { + const auto info = state->recorder->DescribePendingWrite(address, static_cast(bytes)); + if (!info.has_value()) continue; + open = open || info->open; + serial = std::max(serial, info->serial); + } + if (open) serial = state->recorder->SubmitAndEpoch(); + } + } + if (serial != 0) { + const auto waitStart = std::chrono::steady_clock::now(); + WaitRecorded(serial); + prepared->presyncSerial = serial; + counters.presyncs.fetch_add(1, std::memory_order_relaxed); + counters.waitedUs.fetch_add(static_cast(std::chrono::duration_cast(std::chrono::steady_clock::now() - waitStart).count()), std::memory_order_relaxed); + } + } + if (profile) { + const auto nowMs = std::chrono::duration_cast(std::chrono::steady_clock::now().time_since_epoch()).count(); + auto last = counters.lastReport.load(); + if (nowMs - last >= 10000 && counters.lastReport.compare_exchange_strong(last, nowMs)) std::fprintf(stderr, "[presync] %llu dispatches checked, %llu pre-syncs waited %.0f ms (cumulative)\n", static_cast(counters.checked.load()), static_cast(counters.presyncs.load()), counters.waitedUs.load() / 1000.0); + } + } + if (profile) prepared->prepareMs = std::chrono::duration(std::chrono::steady_clock::now() - start).count(); + return prepared; +} + +void VulkanDevice::Dispatch(const ShaderRecompiler::RecompileResult& shader, std::uint32_t x, std::uint32_t y, std::uint32_t z, std::span snapshots, std::uint64_t programAddress, std::shared_ptr prepared) { + static_cast(dispatch(shader, x, y, z, 0, snapshots, programAddress, std::move(prepared))); +} + +VulkanDevice::IndirectOutcome VulkanDevice::DispatchIndirect(const ShaderRecompiler::RecompileResult& shader, std::uint64_t arguments, std::span snapshots, std::uint64_t programAddress, std::shared_ptr prepared) { + return dispatch(shader, 0, 0, 0, arguments, snapshots, programAddress, std::move(prepared)); +} + +VulkanDevice::IndirectOutcome VulkanDevice::dispatch(const ShaderRecompiler::RecompileResult& shader, std::uint32_t x, std::uint32_t y, std::uint32_t z, std::uint64_t arguments, std::span snapshots, std::uint64_t programAddress, std::shared_ptr prepared) { PerformanceTimer timing("Vulkan.Dispatch"); - const GuestMemory::MemoryAccessScope memoryScope(this, [](void* context, std::uint64_t address, std::size_t bytes, bool writable) { - static_cast(context)->ResolveMemory(address, bytes, writable); - }); + // Group counts for the trace lines; an indirect dispatch does not know them. + char groupsText[40]; + if (arguments != 0) std::snprintf(groupsText, sizeof(groupsText), "indirect"); + else std::snprintf(groupsText, sizeof(groupsText), "%ux%ux%u", x, y, z); + APS5_LOG_OUT_DEBUG("Dispatch groups=%s spirvWords=%zu bindings=%zu pushConstants=%zu", groupsText, shader.spirv.size(), shader.bindings.size(), shader.pushConstants.size()); if (shader.spirv.size() < 5 || shader.spirv[0] != 0x07230203u) { throw std::runtime_error("Vulkan dispatch: invalid SPIR-V"); } @@ -773,99 +1504,343 @@ void VulkanDevice::Dispatch(const ShaderRecompiler::RecompileResult& shader, std const auto pushBytes = Graphics::AssemblePushConstants(shaders); const auto context = graphicsContext(); const auto* limit = state->properties.limits.maxComputeWorkGroupCount; - if (x > limit[0] || y > limit[1] || z > limit[2]) { + if (arguments == 0 && (x > limit[0] || y > limit[1] || z > limit[2])) { throw std::runtime_error("Vulkan dispatch: workgroup count exceeds device limits"); } - state->drawQueue->Wait(); - timing.Mark("draw_wait"); - const auto destroyModule = state->DeviceFunction("vkDestroyShaderModule"); - const auto destroyLayout = state->DeviceFunction("vkDestroyPipelineLayout"); - const auto destroyPipeline = state->DeviceFunction("vkDestroyPipeline"); - const auto destroyFence = state->DeviceFunction("vkDestroyFence"); - const auto freeCommands = state->DeviceFunction("vkFreeCommandBuffers"); - const auto wait = state->DeviceFunction("vkWaitForFences"); - const auto submit = state->DeviceFunction("vkQueueSubmit"); - VkShaderModule module = VK_NULL_HANDLE; - VkPipelineLayout layout = VK_NULL_HANDLE; - VkPipeline pipeline = VK_NULL_HANDLE; - VkFence fence = VK_NULL_HANDLE; - VkCommandBuffer commands = VK_NULL_HANDLE; - const auto cleanup = [&] { - if (commands != VK_NULL_HANDLE) freeCommands(state->device, state->pool, 1, &commands); - if (fence != VK_NULL_HANDLE) destroyFence(state->device, fence, nullptr); - if (pipeline != VK_NULL_HANDLE) destroyPipeline(state->device, pipeline, nullptr); - if (layout != VK_NULL_HANDLE) destroyLayout(state->device, layout, nullptr); - if (module != VK_NULL_HANDLE) destroyModule(state->device, module, nullptr); + // Pipelines are shared by dispatches of one compiled variant; descriptor set layouts built from the + // same bindings are compatible, so the pipeline layout of the first dispatch serves them all. + const std::uint64_t pipelineKey = shader.variantId != 0 ? (shader.variantId << 1u) | (pushStages != 0 ? 1u : 0u) : 0u; + std::shared_ptr objects; + if (pipelineKey != 0) { + if (const auto found = state->computePipelines.find(pipelineKey); found != state->computePipelines.end()) objects = found->second; + } + static const bool profile = std::getenv("APS5_PROFILE_DRAW") != nullptr; + // Debug aid: APS5_SYNC_DISPATCH=1 waits for every dispatch, as before batching. + static const bool syncEachDispatch = std::getenv("APS5_SYNC_DISPATCH") != nullptr; + auto phaseStart = std::chrono::steady_clock::now(); + const auto dispatchStart = phaseStart; + std::string phases; + // Totals per phase across dispatches, reported every 1000 dispatches under APS5_PROFILE_DRAW. + static std::map phaseTotals; + static std::uint64_t profiledDispatches = 0; + // The same phases for indirect dispatches alone, every 10 s on an [indirect] line: what the + // 'indirect' GpuMutex hold ([lock] line) spends its time on inside this function (the label + // record before it is timed in the driver, [labels] line), with the longest hold seen and the + // copied-writer lists the indirect decision scans. All under the mutex, like the rest. + struct IndirectHold { + std::map phaseMs; + std::uint64_t count = 0; + double totalMs = 0; + double maxMs = 0; + std::uint64_t writersScanned = 0; + std::uint64_t drawWritersScanned = 0; + std::size_t maxWriters = 0; + std::chrono::steady_clock::time_point lastReport = std::chrono::steady_clock::now(); }; - try { + static IndirectHold indirectHold; + const auto phase = [&](const char* name) { + if (!profile) return; + const auto now = std::chrono::steady_clock::now(); + const auto ms = std::chrono::duration(now - phaseStart).count(); + char text[64]; + std::snprintf(text, sizeof(text), " %s=%.0fms", name, ms); + phases += text; + phaseTotals[name] += ms; + if (arguments != 0) indirectHold.phaseMs[name] += ms; + phaseStart = now; + // Slow phases are reported as they finish, so a dispatch that never completes shows where it is. + if (ms > 1000) std::fprintf(stderr, "[dispatch] %s took %.0f ms (%zu words)\n", name, ms, shader.spirv.size()); + }; + if (profile && ++profiledDispatches % 1000 == 0) { + std::string report; + for (const auto& [name, ms] : phaseTotals) report += " " + name + "=" + std::to_string(static_cast(ms / 1000)) + "s"; + std::fprintf(stderr, "[dispatch] %llu dispatches, phase totals:%s\n", static_cast(profiledDispatches), report.c_str()); + } + // A dispatch whose compiled content repeats an earlier one reuses that build's descriptor set + // when it is still valid (see ResourceCache); push constants still come from this dispatch. + static std::uint64_t cacheHits = 0, cacheMisses = 0, cacheInvalidated = 0; + static auto cacheReport = std::chrono::steady_clock::now(); + std::shared_ptr resources; + ResourceCache::Key contentKey; + const bool cacheable = ResourceCacheEnabled() && shader.variantId != 0; + // The batches the pre-sync (PrepareDispatch) waited for are reaped first: their completions + // (CPU write-backs) land before stage B or a cached object's Revalidate reads, and the flush + // hook then finds nothing pending over their surfaces. + if (prepared != nullptr && prepared->presyncSerial != 0) { + // Timed alone: the phase entry exists only when a reap ran, and measures just the reap. + phaseStart = std::chrono::steady_clock::now(); + ReapRecorded(prepared->presyncSerial); + phase("reap"); + } + // The 'resources' phase below, split (APS5_PROFILE_DRAW) into what runs under the mutex: the + // sub-phases are extra rows named "resources: ..." in the [dispatch] totals and the [indirect] + // line, so the hold's biggest part (stage B's image lookups, the whole build of an address-based + // shader) is named, and "resources: hook waits" says how much of the build was the nested flush + // hook waiting for recorded work (a locked wait: it overlaps the other rows). + const auto subPhase = [&](const char* name, double ms) { + if (!profile) return; + phaseTotals[name] += ms; + if (arguments != 0) indirectHold.phaseMs[name] += ms; + }; + const auto subPhaseStart = profile ? std::chrono::steady_clock::now() : std::chrono::steady_clock::time_point{}; + const auto waitedBefore = profile ? Graphics::Recorder::ThreadWaitedMs() : 0.0; + if (prepared != nullptr && prepared->resources != nullptr) { + // Stage A ran without the mutex (PrepareDispatch); stage B completes the build here. + resources = std::move(prepared->resources); + contentKey = std::move(prepared->key); + // The build's own sub-phase totals before and after: the differences are stage B's parts + // (an address-based build also runs its stage A here, under the mutex: 'A locked'). + const auto before = profile ? resources->Timing() : Graphics::ShaderResources::BuildTiming{}; + resources->Complete(); + if (profile) { + const auto& after = resources->Timing(); + const auto images = after.bindingsMs - before.bindingsMs; + const auto upload = after.uploadMs - before.uploadMs; + const auto descriptors = after.descriptorsMs - before.descriptorsMs; + const auto stageA = after.prepareMs - before.prepareMs; + subPhase("resources: B images", images); + subPhase("resources: B upload", upload); + subPhase("resources: B descriptors", descriptors); + subPhase("resources: B bda+other", std::max(0.0, after.completeMs - before.completeMs - images - upload - descriptors)); + subPhase("resources: A locked (bda)", stageA); + char text[64]; + std::snprintf(text, sizeof(text), " resourcesA=%.0fms", prepared->prepareMs); + phases += text; + phaseTotals["resources A (unlocked)"] += prepared->prepareMs; + } + if (cacheable) { + ++cacheMisses; + if (resources->Reusable()) state->resourceCache.Insert(contentKey, resources); + } + } else if (cacheable) { + // PrepareDispatch made the key already when it found the cached object. + contentKey = prepared != nullptr && !prepared->key.empty() ? std::move(prepared->key) : DispatchContentKey(shaders[0], context.device); + if (auto cached = state->resourceCache.Find(contentKey)) { + if (cached->Revalidate(shaders[0])) { + resources = std::move(cached); + ++cacheHits; + } else { + state->resourceCache.Remove(contentKey); + ++cacheInvalidated; + } + if (profile) subPhase("resources: revalidate", std::chrono::duration(std::chrono::steady_clock::now() - subPhaseStart).count()); + } + } + if (resources == nullptr) { + const auto buildStart = profile ? std::chrono::steady_clock::now() : std::chrono::steady_clock::time_point{}; + resources = std::make_shared(context, shaders[0], snapshots); + if (profile) subPhase("resources: full build (locked)", std::chrono::duration(std::chrono::steady_clock::now() - buildStart).count()); + if (cacheable) { + ++cacheMisses; + if (resources->Reusable()) state->resourceCache.Insert(contentKey, resources); + } + } + if (profile) subPhase("resources: hook waits", Graphics::Recorder::ThreadWaitedMs() - waitedBefore); + if (profile && std::chrono::steady_clock::now() - cacheReport > std::chrono::seconds(10)) { + cacheReport = std::chrono::steady_clock::now(); + std::fprintf(stderr, "[rescache] %llu hits, %llu misses, %llu invalidated, %zu entries (dispatch + draw)\n", static_cast(cacheHits), static_cast(cacheMisses), static_cast(cacheInvalidated), state->resourceCache.Size()); + } + timing.Mark("shader_resources"); + phase("resources"); + if (objects == nullptr) { + objects = std::make_shared(); + objects->device = state->device; + objects->destroyModule = state->DeviceFunction("vkDestroyShaderModule"); + objects->destroyLayout = state->DeviceFunction("vkDestroyPipelineLayout"); + objects->destroyPipeline = state->DeviceFunction("vkDestroyPipeline"); VkShaderModuleCreateInfo moduleInfo{VK_STRUCTURE_TYPE_SHADER_MODULE_CREATE_INFO}; moduleInfo.codeSize = shader.spirv.size() * sizeof(std::uint32_t); moduleInfo.pCode = shader.spirv.data(); - check(state->DeviceFunction("vkCreateShaderModule")(state->device, &moduleInfo, nullptr, &module), "vkCreateShaderModule"); + check(state->DeviceFunction("vkCreateShaderModule")(state->device, &moduleInfo, nullptr, &objects->module), "vkCreateShaderModule"); timing.Mark("validate_shader_module"); - Graphics::ShaderResources resources(context, shaders[0], snapshots); - timing.Mark("shader_resources"); - const auto setLayout = resources.Layout(); + const auto setLayout = resources->Layout(); const VkPushConstantRange push{VK_SHADER_STAGE_COMPUTE_BIT, 0, Graphics::PipelinePushConstantBytes}; VkPipelineLayoutCreateInfo layoutInfo{VK_STRUCTURE_TYPE_PIPELINE_LAYOUT_CREATE_INFO}; layoutInfo.setLayoutCount = 1; layoutInfo.pSetLayouts = &setLayout; layoutInfo.pushConstantRangeCount = pushStages != 0 ? 1 : 0; layoutInfo.pPushConstantRanges = pushStages != 0 ? &push : nullptr; - check(state->DeviceFunction("vkCreatePipelineLayout")(state->device, &layoutInfo, nullptr, &layout), "vkCreatePipelineLayout"); + check(state->DeviceFunction("vkCreatePipelineLayout")(state->device, &layoutInfo, nullptr, &objects->layout), "vkCreatePipelineLayout"); VkComputePipelineCreateInfo pipelineInfo{VK_STRUCTURE_TYPE_COMPUTE_PIPELINE_CREATE_INFO}; pipelineInfo.stage.sType = VK_STRUCTURE_TYPE_PIPELINE_SHADER_STAGE_CREATE_INFO; pipelineInfo.stage.stage = VK_SHADER_STAGE_COMPUTE_BIT; - pipelineInfo.stage.module = module; + pipelineInfo.stage.module = objects->module; pipelineInfo.stage.pName = "main"; - pipelineInfo.layout = layout; - check(state->DeviceFunction("vkCreateComputePipelines")(state->device, context.pipelineCache, 1, &pipelineInfo, nullptr, &pipeline), "vkCreateComputePipelines"); + pipelineInfo.layout = objects->layout; + if (profile && shader.spirv.size() > 100000) std::fprintf(stderr, "[dispatch] creating a pipeline for %zu SPIR-V words\n", shader.spirv.size()); + check(state->DeviceFunction("vkCreateComputePipelines")(state->device, context.pipelineCache, 1, &pipelineInfo, nullptr, &objects->pipeline), "vkCreateComputePipelines"); timing.Mark("pipeline_create"); - VkCommandBufferAllocateInfo allocation{VK_STRUCTURE_TYPE_COMMAND_BUFFER_ALLOCATE_INFO}; - allocation.commandPool = state->pool; - allocation.level = VK_COMMAND_BUFFER_LEVEL_PRIMARY; - allocation.commandBufferCount = 1; - check(state->DeviceFunction("vkAllocateCommandBuffers")(state->device, &allocation, &commands), "vkAllocateCommandBuffers"); - VkCommandBufferBeginInfo begin{VK_STRUCTURE_TYPE_COMMAND_BUFFER_BEGIN_INFO}; - begin.flags = VK_COMMAND_BUFFER_USAGE_ONE_TIME_SUBMIT_BIT; - check(state->DeviceFunction("vkBeginCommandBuffer")(commands, &begin), "vkBeginCommandBuffer"); - VkMemoryBarrier upload{VK_STRUCTURE_TYPE_MEMORY_BARRIER}; - upload.srcAccessMask = VK_ACCESS_HOST_WRITE_BIT; - upload.dstAccessMask = VK_ACCESS_SHADER_READ_BIT | VK_ACCESS_SHADER_WRITE_BIT; - state->DeviceFunction("vkCmdPipelineBarrier")(commands, VK_PIPELINE_STAGE_HOST_BIT, VK_PIPELINE_STAGE_COMPUTE_SHADER_BIT, 0, 1, &upload, 0, nullptr, 0, nullptr); - state->DeviceFunction("vkCmdBindPipeline")(commands, VK_PIPELINE_BIND_POINT_COMPUTE, pipeline); - resources.Bind(commands, VK_PIPELINE_BIND_POINT_COMPUTE, layout); - if (pushStages != 0) { - state->DeviceFunction("vkCmdPushConstants")(commands, layout, VK_SHADER_STAGE_COMPUTE_BIT, 0, Graphics::PipelinePushConstantBytes, pushBytes.data()); - } - state->DeviceFunction("vkCmdDispatch")(commands, x, y, z); - VkMemoryBarrier download{VK_STRUCTURE_TYPE_MEMORY_BARRIER}; - download.srcAccessMask = VK_ACCESS_SHADER_WRITE_BIT; - download.dstAccessMask = VK_ACCESS_HOST_READ_BIT; - state->DeviceFunction("vkCmdPipelineBarrier")(commands, VK_PIPELINE_STAGE_COMPUTE_SHADER_BIT, VK_PIPELINE_STAGE_HOST_BIT, 0, 1, &download, 0, nullptr, 0, nullptr); - check(state->DeviceFunction("vkEndCommandBuffer")(commands), "vkEndCommandBuffer"); - VkFenceCreateInfo fenceInfo{VK_STRUCTURE_TYPE_FENCE_CREATE_INFO}; - check(state->DeviceFunction("vkCreateFence")(state->device, &fenceInfo, nullptr, &fence), "vkCreateFence"); - VkSubmitInfo submission{VK_STRUCTURE_TYPE_SUBMIT_INFO}; - submission.commandBufferCount = 1; - submission.pCommandBuffers = &commands; - timing.Mark("command_record"); - check(submit(state->queue, 1, &submission, fence), "vkQueueSubmit"); - timing.Mark("queue_submit"); - const auto result = wait(state->device, 1, &fence, VK_TRUE, std::numeric_limits::max()); - timing.Mark("fence_wait"); - if (result != VK_SUCCESS && result != VK_ERROR_DEVICE_LOST) { - const auto idle = state->DeviceFunction("vkDeviceWaitIdle")(state->device); - check(idle, "vkDeviceWaitIdle after fence failure"); - } - check(result, "vkWaitForFences"); - resources.WriteBack(); - timing.Mark("resources_writeback"); - } catch (...) { - cleanup(); - throw; + if (pipelineKey != 0) state->computePipelines[pipelineKey] = objects; + phase("pipeline"); } - cleanup(); + // Recorded into the device's open batch: the CPU moves on to the next command while the GPU + // works; the batch is waited for where the guest expects results (labels, flips, CPU reads). + auto& recorder = *state->recorder; + IndirectOutcome outcome{0, 0}; + const Graphics::HostImport* argumentImport = nullptr; + if (arguments != 0) { + // Decided here, after the resource build (which may retire imports and wait for recorded + // work) and before anything of this dispatch is recorded, so the CPU fallback's sync cannot + // split the dispatch across two batches and the import cannot be dropped before the record. + // A label over the argument dwords (a completion-deferred one lands by a CPU store the GPU + // read would miss; a GPU one is ordered but rare enough to share the fallback). Only those + // dwords: a global "any completion label pending" test would send most indirect dispatches + // to the CPU, as ~5 such labels are pending per frame. Stamp 0 matches every live entry. + const auto labelPending = [&] { + std::uint32_t labelQueue = 0; + for (std::uint64_t dword = arguments; dword < arguments + 12; dword += 4) { + if (recorder.PendingLabel(dword, 4, 0, labelQueue).has_value()) return true; + } + return false; + }; + if (Graphics::StorageTexture::FlushPending(arguments, 12, nullptr, "indirect dispatch arguments")) { + // As the flush hook does: the stores were only recorded and the CPU is about to read them. + outcome.cpuReason = 1; + Graphics::Recorder::CountSync(2); + recorder.Sync(); + } else if (labelPending() || std::any_of(state->copiedWriters->begin(), state->copiedWriters->end(), [&](const auto& writer) { return writer->WritesOverlap(arguments, 12); }) + || std::any_of(Graphics::DrawCopiedWriters()->begin(), Graphics::DrawCopiedWriters()->end(), [&](const auto& writer) { return writer->WritesOverlap(arguments, 12); })) { + outcome.cpuReason = 2; + } else if ((argumentImport = Graphics::HostImportFor(context, arguments, 12)) == nullptr) { + // Valid until the next refreshImports, which only runs under GuestMemory::GpuMutex (held + // here, by every Dispatch caller) and Keeps a retired VkBuffer while the recorder is busy. + outcome.cpuReason = 3; + } + if (profile) { + indirectHold.writersScanned += state->copiedWriters->size(); + indirectHold.drawWritersScanned += Graphics::DrawCopiedWriters()->size(); + indirectHold.maxWriters = std::max(indirectHold.maxWriters, state->copiedWriters->size()); + } + phase("decide"); + if (outcome.cpuReason != 0) { + // The read waits for the producing batch through the flush hook, as the driver's resolve + // did for every indirect dispatch before; the counts then go through the direct checks. + const auto readStart = std::chrono::steady_clock::now(); + std::array groups{}; + GuestMemory::Read(arguments, std::as_writable_bytes(std::span(groups)), 4); + outcome.argumentReadMs = std::chrono::duration(std::chrono::steady_clock::now() - readStart).count(); + x = groups[0]; + y = groups[1]; + z = groups[2]; + // Timed before `arguments` is cleared, so the read counts as an indirect phase. + phase("argument read"); + arguments = 0; + std::snprintf(groupsText, sizeof(groupsText), "%ux%ux%u", x, y, z); + if (x > limit[0] || y > limit[1] || z > limit[2]) throw std::runtime_error("Vulkan dispatch: workgroup count exceeds device limits"); + } + } + // From here on the phases of a CPU-resolved indirect dispatch are charged as direct ones + // (`arguments` is 0); the indirect hold total below still covers the whole call. + const bool indirect = arguments != 0 || outcome.cpuReason != 0; + const auto commands = recorder.Commands(); + recorder.Keep(objects); + recorder.Keep(resources); + if (argumentImport != nullptr) { + // The group counts were stored by earlier recorded work (a dispatch in place, a fill) or the + // host; the indirect read follows all of it. + Graphics::RecordMemoryBarrier(context, commands, VK_PIPELINE_STAGE_ALL_COMMANDS_BIT, VK_PIPELINE_STAGE_DRAW_INDIRECT_BIT, VK_ACCESS_SHADER_WRITE_BIT | VK_ACCESS_TRANSFER_WRITE_BIT | VK_ACCESS_HOST_WRITE_BIT, VK_ACCESS_INDIRECT_COMMAND_READ_BIT); + } + // Results of earlier recorded work are visible to this dispatch, its own to everything after. + Graphics::RecordMemoryBarrier(context, commands, VK_PIPELINE_STAGE_ALL_COMMANDS_BIT, VK_PIPELINE_STAGE_COMPUTE_SHADER_BIT, VK_ACCESS_SHADER_WRITE_BIT | VK_ACCESS_TRANSFER_WRITE_BIT | VK_ACCESS_HOST_WRITE_BIT, VK_ACCESS_SHADER_READ_BIT | VK_ACCESS_SHADER_WRITE_BIT); + state->DeviceFunction("vkCmdBindPipeline")(commands, VK_PIPELINE_BIND_POINT_COMPUTE, objects->pipeline); + resources->Bind(commands, VK_PIPELINE_BIND_POINT_COMPUTE, objects->layout); + if (pushStages != 0) { + state->DeviceFunction("vkCmdPushConstants")(commands, objects->layout, VK_SHADER_STAGE_COMPUTE_BIT, 0, Graphics::PipelinePushConstantBytes, pushBytes.data()); + } + const auto gpuTiming = recorder.BeginGpuTiming(programAddress != 0 ? programAddress : shader.variantId); + if (argumentImport != nullptr) state->DeviceFunction("vkCmdDispatchIndirect")(commands, argumentImport->buffer, arguments - argumentImport->base); + else state->DeviceFunction("vkCmdDispatch")(commands, x, y, z); + Graphics::RecordMemoryBarrier(context, commands, VK_PIPELINE_STAGE_COMPUTE_SHADER_BIT, VK_PIPELINE_STAGE_ALL_COMMANDS_BIT, VK_ACCESS_SHADER_WRITE_BIT, VK_ACCESS_SHADER_READ_BIT | VK_ACCESS_SHADER_WRITE_BIT | VK_ACCESS_TRANSFER_READ_BIT | VK_ACCESS_HOST_READ_BIT); + recorder.EndGpuTiming(gpuTiming); + resources->MarkGpuWrites(recorder); + // Only copied written buffers (and BDA fault checks) need work once the GPU is done; without them + // the batch can signal its labels from the GPU. + if (resources->NeedsCompletion()) { + // Listed for DispatchIndirect until the write-back ran (a non-reusable object, so once). + auto writers = state->copiedWriters; + recorder.OnComplete([resources, writers] { + // Delisted before the write-back: one that fails must not keep indirect dispatches on the CPU. + writers->erase(std::remove(writers->begin(), writers->end(), resources), writers->end()); + resources->WriteBackBuffers(); + }); + // Listed after the registration: a throw there leaves nothing that would pin the CPU path forever. + writers->push_back(resources); + } + timing.Mark("command_record"); + phase("record"); + APS5_LOG_OUT_DEBUG("Dispatch recorded groups=%s", groupsText); + // Address-based shaders pin guest allocations until their write-back, which the completion runs + // when the batch finished (deferred lease release, see Graphics::SyncLeaseWork): a guest thread + // that needs a leased allocation syncs the recorder itself through the registry's pin waiter. + // APS5_SYNC_LEASE_DISPATCH=1 completes such dispatches at once, as before. + if (syncEachDispatch || (resources->HoldsLease() && Graphics::SyncLeaseWork())) { + Graphics::Recorder::CountSync(3); + const auto syncStart = std::chrono::steady_clock::now(); + recorder.Sync(); + phase("sync"); + if (resources->HoldsLease()) Graphics::CountLeaseOutcome(true, 0); + if (profile) { + // APS5_PROFILE_DRAW: the [recorder] line's "address-based" syncs by program, every 10 s, + // so the programs still waiting here are known (under the GpuMutex, like the rest). + struct Waits { std::uint64_t count = 0; double ms = 0; }; + static std::map byProgram; + static std::uint64_t syncs = 0; + static double waitedMs = 0; + static auto lastReport = std::chrono::steady_clock::now(); + const auto now = std::chrono::steady_clock::now(); + const auto ms = std::chrono::duration(now - syncStart).count(); + auto& waits = byProgram[programAddress != 0 ? programAddress : shader.variantId]; + ++waits.count; + waits.ms += ms; + ++syncs; + waitedMs += ms; + if (now - lastReport > std::chrono::seconds(10)) { + lastReport = now; + std::vector> hot(byProgram.begin(), byProgram.end()); + std::sort(hot.begin(), hot.end(), [](const auto& a, const auto& b) { return a.second.ms > b.second.ms; }); + std::fprintf(stderr, "[address-sync] %llu address-based dispatch syncs waited %.1f s in total; by program (10 s):", static_cast(syncs), waitedMs / 1000); + for (std::size_t i = 0; i < hot.size() && i < 8; ++i) std::fprintf(stderr, " 0x%llx x%llu %.0fms", static_cast(hot[i].first), static_cast(hot[i].second.count), hot[i].second.ms); + std::fprintf(stderr, "\n"); + byProgram.clear(); + } + } + } else if (resources->HoldsLease()) { + // The lease is in the open batch (kept above, nothing submitted since): the pin waiter + // finishes the recorder up to that batch's serial. Prints the [address-sync] leases line. + Graphics::CountLeaseOutcome(false, recorder.Submissions() + 1); + } + // Debug aid: APS5_TRACE_DISPATCH_IO prints every dispatch's resources (after write-back when synced). + static const bool traceIo = std::getenv("APS5_TRACE_DISPATCH_IO") != nullptr; + if (traceIo) std::fprintf(stderr, "[dispatch-io] %s:%s\n", groupsText, resources->Describe().c_str()); + if (profile) { + const auto now = std::chrono::steady_clock::now(); + const auto totalMs = std::chrono::duration(now - dispatchStart).count(); + if (totalMs > 100) std::fprintf(stderr, "[dispatch] %s %zu words:%s\n", groupsText, shader.spirv.size(), phases.c_str()); + if (indirect) { + auto& hold = indirectHold; + ++hold.count; + hold.totalMs += totalMs; + hold.maxMs = std::max(hold.maxMs, totalMs); + if (now - hold.lastReport > std::chrono::seconds(10)) { + hold.lastReport = now; + std::string report; + for (const auto& [name, ms] : hold.phaseMs) { + char text[80]; + std::snprintf(text, sizeof(text), " %s %.0f ms", name.c_str(), ms); + report += text; + } + std::fprintf(stderr, "[indirect] %llu indirect dispatches spent %.0f ms inside the device call (10 s; max %.1f ms), by phase (the 'resources: ...' rows split 'resources'; 'hook waits' overlaps them):%s; copied-writer lists scanned per decision: dispatch avg %.1f (max %zu), draw avg %.1f\n", static_cast(hold.count), hold.totalMs, hold.maxMs, report.c_str(), hold.count != 0 ? static_cast(hold.writersScanned) / hold.count : 0.0, hold.maxWriters, hold.count != 0 ? static_cast(hold.drawWritersScanned) / hold.count : 0.0); + hold.phaseMs.clear(); + hold.count = 0; + hold.totalMs = 0; + hold.maxMs = 0; + hold.writersScanned = 0; + hold.drawWritersScanned = 0; + hold.maxWriters = 0; + } + } + } + return outcome; } } diff --git a/core/libs/prx/libSceAgcDriver/Execution/src/WorkerSampler.cpp b/core/libs/prx/libSceAgcDriver/Execution/src/WorkerSampler.cpp new file mode 100644 index 000000000..7e4d7a381 --- /dev/null +++ b/core/libs/prx/libSceAgcDriver/Execution/src/WorkerSampler.cpp @@ -0,0 +1,181 @@ +#include "prx/libSceAgcDriver/Execution/include/WorkerSampler.hpp" +#include +#include +#include +#ifdef _WIN32 +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#endif + +namespace AgcDriver { + +#ifdef _WIN32 +namespace { + +// Debug aid: APS5_SAMPLE_WORKER= samples the calling thread every millisecond and, every 20 s, +// writes "module+offset self inclusive" lines to for offline symbolization with nm. +struct Sampler { + HANDLE target = nullptr; + std::string path; + std::mutex mutex; + std::map> counts; + std::uint64_t samples = 0; + + void sample() { + CONTEXT context{}; + context.ContextFlags = CONTEXT_FULL; + if (SuspendThread(target) == static_cast(-1)) return; + const bool captured = GetThreadContext(target, &context) != 0; + std::vector frames; + if (captured) { + // Unwinding reads the target's stack while it is suspended; frames without unwind data + // (JIT/guest code) end the walk. + for (int depth = 0; depth < 12 && context.Rip != 0; ++depth) { + frames.push_back(context.Rip); + DWORD64 imageBase = 0; + auto* entry = RtlLookupFunctionEntry(context.Rip, &imageBase, nullptr); + if (entry == nullptr) break; + PVOID handler = nullptr; + DWORD64 establisher = 0; + RtlVirtualUnwind(UNW_FLAG_NHANDLER, imageBase, context.Rip, entry, &context, &handler, &establisher, nullptr); + } + } + ResumeThread(target); + std::lock_guard lock(mutex); + ++samples; + for (std::size_t i = 0; i < frames.size(); ++i) { + auto& count = counts[frames[i]]; + if (i == 0) ++count.first; + ++count.second; + } + } + + void write() { + std::lock_guard lock(mutex); + std::FILE* file = std::fopen(path.c_str(), "w"); + if (file == nullptr) return; + std::fprintf(file, "# %llu samples\n", static_cast(samples)); + for (const auto& [address, count] : counts) { + HMODULE module = nullptr; + char name[MAX_PATH] = "?"; + if (GetModuleHandleExA(GET_MODULE_HANDLE_EX_FLAG_FROM_ADDRESS | GET_MODULE_HANDLE_EX_FLAG_UNCHANGED_REFCOUNT, reinterpret_cast(address), &module)) GetModuleFileNameA(module, name, sizeof(name)); + const char* base = std::strrchr(name, '\\'); + std::fprintf(file, "%s 0x%llx %llu %llu\n", base ? base + 1 : name, static_cast(address - reinterpret_cast(module)), static_cast(count.first), static_cast(count.second)); + } + std::fclose(file); + } +}; + +// Debug aid: APS5_SAMPLE_THREADS= samples every thread in the process every 2 ms and, every +// 20 s, writes each busy thread's hottest frames ("module+offset self inclusive") to . +struct ProcessSampler { + std::string path; + DWORD self = 0; + std::map threads; + std::uint64_t rounds = 0; + + std::chrono::steady_clock::time_point enumerated{}; + + // The system-wide thread snapshot is slow (tens of ms); the thread list is refreshed every 5 s and + // the rounds in between only sample the known threads. + void enumerate() { + const HANDLE snapshot = CreateToolhelp32Snapshot(TH32CS_SNAPTHREAD, 0); + if (snapshot == INVALID_HANDLE_VALUE) return; + THREADENTRY32 entry{}; + entry.dwSize = sizeof(entry); + const DWORD process = GetCurrentProcessId(); + for (BOOL more = Thread32First(snapshot, &entry); more; more = Thread32Next(snapshot, &entry)) { + if (entry.th32OwnerProcessID != process || entry.th32ThreadID == self) continue; + auto& thread = threads[entry.th32ThreadID]; + if (thread.target == nullptr) thread.target = OpenThread(THREAD_SUSPEND_RESUME | THREAD_GET_CONTEXT | THREAD_QUERY_INFORMATION, FALSE, entry.th32ThreadID); + } + CloseHandle(snapshot); + enumerated = std::chrono::steady_clock::now(); + } + + void sample() { + if (threads.empty() || std::chrono::steady_clock::now() - enumerated > std::chrono::seconds(5)) enumerate(); + for (auto& [id, thread] : threads) { + if (thread.target != nullptr) thread.sample(); + } + ++rounds; + } + + void write() { + std::FILE* file = std::fopen(path.c_str(), "w"); + if (file == nullptr) return; + std::fprintf(file, "# %llu rounds\n", static_cast(rounds)); + for (auto& [id, thread] : threads) { + std::lock_guard lock(thread.mutex); + std::vector>> hot(thread.counts.begin(), thread.counts.end()); + std::sort(hot.begin(), hot.end(), [](const auto& a, const auto& b) { return a.second.second > b.second.second; }); + std::uint64_t leaves = 0; + for (const auto& [address, count] : hot) leaves += count.first; + if (leaves == 0) continue; + std::fprintf(file, "thread %lu samples %llu\n", static_cast(id), static_cast(thread.samples)); + for (std::size_t i = 0; i < hot.size() && i < 40; ++i) { + const auto address = hot[i].first; + HMODULE module = nullptr; + char name[MAX_PATH] = "?"; + if (GetModuleHandleExA(GET_MODULE_HANDLE_EX_FLAG_FROM_ADDRESS | GET_MODULE_HANDLE_EX_FLAG_UNCHANGED_REFCOUNT, reinterpret_cast(address), &module)) GetModuleFileNameA(module, name, sizeof(name)); + const char* base = std::strrchr(name, '\\'); + std::fprintf(file, " %s 0x%llx %llu %llu\n", base ? base + 1 : name, static_cast(address - reinterpret_cast(module)), static_cast(hot[i].second.first), static_cast(hot[i].second.second)); + } + } + std::fclose(file); + } +}; + +void StartProcessSampler() { + const char* path = std::getenv("APS5_SAMPLE_THREADS"); + if (path == nullptr) return; + std::thread([path = std::string(path)] { + auto* sampler = new ProcessSampler(); + sampler->path = path; + sampler->self = GetCurrentThreadId(); + auto flushed = std::chrono::steady_clock::now(); + for (;;) { + std::this_thread::sleep_for(std::chrono::milliseconds(2)); + sampler->sample(); + if (std::chrono::steady_clock::now() - flushed > std::chrono::seconds(20)) { + sampler->write(); + flushed = std::chrono::steady_clock::now(); + } + } + }).detach(); +} + +} + +void StartWorkerSampler() { + StartProcessSampler(); + const char* path = std::getenv("APS5_SAMPLE_WORKER"); + if (path == nullptr) return; + auto* sampler = new Sampler(); + sampler->path = path; + DuplicateHandle(GetCurrentProcess(), GetCurrentThread(), GetCurrentProcess(), &sampler->target, THREAD_ALL_ACCESS, FALSE, 0); + std::thread([sampler] { + auto flushed = std::chrono::steady_clock::now(); + for (;;) { + std::this_thread::sleep_for(std::chrono::milliseconds(1)); + sampler->sample(); + if (std::chrono::steady_clock::now() - flushed > std::chrono::seconds(20)) { + sampler->write(); + flushed = std::chrono::steady_clock::now(); + } + } + }).detach(); +} +#else +void StartWorkerSampler() {} +#endif + +} diff --git a/core/libs/prx/libSceAgcDriver/Graphics/include/BdaResources.hpp b/core/libs/prx/libSceAgcDriver/Graphics/include/BdaResources.hpp index 0163b2103..6e525bf90 100644 --- a/core/libs/prx/libSceAgcDriver/Graphics/include/BdaResources.hpp +++ b/core/libs/prx/libSceAgcDriver/Graphics/include/BdaResources.hpp @@ -12,9 +12,20 @@ public: VkDescriptorBufferInfo Table() const; VkDescriptorBufferInfo Fault() const; void CheckFault() const; + // APS5_PROFILE_DRAW: page tables served from the per-device cache and built anew (cumulative), + // and how many recent tables the cache holds right now. + struct TableCacheStats { + std::uint64_t hits = 0; + std::uint64_t misses = 0; + std::size_t held = 0; + }; + static TableCacheStats TableCacheCounters(); private: - std::unique_ptr table; + // The page table is read-only to the shader, so consecutive builds mapping the same ranges to the + // same device addresses share one buffer while one of them is alive (see the table cache in + // BdaResources.cpp, which refers to it weakly). + std::shared_ptr table; std::unique_ptr fault; std::size_t tableBytes = 0; }; diff --git a/core/libs/prx/libSceAgcDriver/Graphics/include/BufferPool.hpp b/core/libs/prx/libSceAgcDriver/Graphics/include/BufferPool.hpp index 5e78a981c..efc1bb176 100644 --- a/core/libs/prx/libSceAgcDriver/Graphics/include/BufferPool.hpp +++ b/core/libs/prx/libSceAgcDriver/Graphics/include/BufferPool.hpp @@ -2,10 +2,11 @@ #define CORE_LIBS_PRX_LIBSCEAGCDRIVER_GRAPHICS_INCLUDE_BUFFERPOOL_HPP #include "prx/libSceAgcDriver/Graphics/include/Context.hpp" -#include +#include #include #include #include +#include namespace AgcDriver::Graphics { @@ -15,46 +16,80 @@ struct BufferAllocation { void* mapping; VkDeviceAddress address; VkDeviceSize allocationBytes; + // Size the VkBuffer was created with (see BufferPool::Capacity), not the size a user asked for. std::size_t bytes; VkBufferUsageFlags usage; VkMemoryPropertyFlags properties = VK_MEMORY_PROPERTY_HOST_VISIBLE_BIT | VK_MEMORY_PROPERTY_HOST_COHERENT_BIT; }; +// Released buffer allocations kept for reuse, since creating, binding and mapping one costs tens of +// microseconds and a build makes several. Small requests are served by size class (the next power of +// two), so the many differently sized data and copied-region buffers of consecutive builds hit; +// requests of a MiB and more keep their exact size (the multi-MiB registered-range snapshots repeat +// exactly, and rounding them would waste pinned host memory). Retained allocations are evicted least +// recently used under a byte budget and a slot count, so a burst of small buffers between two large +// builds no longer sweeps the large ones out. Reuse is safe because a Buffer is only released once +// the GPU work using it completed (kept until the batch fence, or after a CommandBatch wait). +// APS5_NO_BUFFER_CLASSES=1 matches exact sizes only, as before; APS5_BUFFER_POOL_SLOTS=64 restores +// the old slot count (the bound applies to each tier below, so twice that many slots in all). +// +// The size classes and the exact-size allocations are retained in two tiers with a budget each: +// the multi-MiB texture scratch buffers that come back from a batch (GPU-direct uploads and +// write-backs of storage images) filled the one budget, so every one returned evicted hundreds of +// the 256-byte copied-region buffers to fit, and those then missed on every build (81869 misses +// and 80850 evictions per run, each a Vulkan create or destroy under the pool mutex, 3.5 s per +// run). APS5_BUFFER_POOL_SHARED=1 keeps one tier as before. class BufferPool { public: explicit BufferPool(const Context& context); ~BufferPool(); BufferPool(const BufferPool&) = delete; BufferPool& operator=(const BufferPool&) = delete; + // The size a buffer for `bytes` is created with: its size class, or `bytes` itself when large. + static std::size_t Capacity(std::size_t bytes); std::optional Take(std::size_t bytes, VkBufferUsageFlags usage, VkMemoryPropertyFlags properties = VK_MEMORY_PROPERTY_HOST_VISIBLE_BIT | VK_MEMORY_PROPERTY_HOST_COHERENT_BIT); void Put(const BufferAllocation& allocation) noexcept; private: - static constexpr std::size_t capacity = 1024; - static constexpr std::size_t bucketCount = 256; - static constexpr std::size_t none = capacity; struct Slot { - std::optional allocation; - std::size_t next = none; - std::size_t older = none; - std::size_t newer = none; + BufferAllocation allocation; + std::uint64_t lastUse; }; - static std::size_t bucket(std::size_t bytes, VkBufferUsageFlags usage, VkMemoryPropertyFlags properties) noexcept; - BufferAllocation remove(std::size_t index) noexcept; + // One retention tier: its slots, their bytes, the byte budget they are evicted under and its + // counters (APS5_PROFILE_DRAW, reported every 10 s from Take). + struct Tier { + std::vector free; + VkDeviceSize retainedBytes = 0; + VkDeviceSize budget = 0; + std::uint64_t hits = 0; + std::uint64_t misses = 0; + std::uint64_t evictions = 0; + }; + // The tier a buffer of `capacity` is retained in (the large one for everything when shared). + Tier& tierFor(std::size_t capacity); void destroy(const BufferAllocation& allocation) noexcept; + // Moves the tier's least recently used slot to `evicted`; the caller destroys those after + // releasing the mutex, so builds taking buffers on other threads do not wait behind the + // Vulkan destroy calls. Nothing changes when the vector cannot grow. + void evictOldest(Tier& tier, std::vector& evicted); + // The retained-slot bound of each tier (APS5_BUFFER_POOL_SLOTS, default `defaultSlots`), read once. + static std::size_t MaxSlots(); VkDevice device; PFN_vkUnmapMemory unmap; PFN_vkDestroyBuffer destroyBuffer; PFN_vkFreeMemory freeMemory; std::mutex mutex; - std::array slots; - std::array buckets; - std::array freeSlots; - std::size_t freeCount = capacity; - std::size_t oldest = none; - std::size_t newest = none; - VkDeviceSize retainedBytes = 0; + // Not `small`/`large`: (via ) defines `small` as a macro. + Tier smallTier; + Tier largeTier; + std::uint64_t clock = 0; static constexpr VkDeviceSize budget = 512ull * 1024 * 1024; + // The small tier's own budget (512 slots of at most half a MiB each): pinned host memory the + // large tier's budget does not count. + static constexpr VkDeviceSize smallBudget = 64ull * 1024 * 1024; + // Requests of this size and more keep their exact size and go to the large tier. + static constexpr std::size_t classLimit = std::size_t{1} << 20u; + static constexpr std::size_t defaultSlots = 512; }; std::shared_ptr GetBufferPool(const Context& context); diff --git a/core/libs/prx/libSceAgcDriver/Graphics/include/ColorTargetLayout.hpp b/core/libs/prx/libSceAgcDriver/Graphics/include/ColorTargetLayout.hpp index 511e41751..860584f7b 100644 --- a/core/libs/prx/libSceAgcDriver/Graphics/include/ColorTargetLayout.hpp +++ b/core/libs/prx/libSceAgcDriver/Graphics/include/ColorTargetLayout.hpp @@ -4,6 +4,7 @@ #include #include #include +#include namespace AgcDriver::Graphics { @@ -16,10 +17,11 @@ ColorTileMode DecodeColorTileMode(std::uint32_t attrib3); class ColorTargetLayout { public: - ColorTargetLayout(std::uint32_t width, std::uint32_t height, ColorTileMode mode); + ColorTargetLayout(std::uint32_t width, std::uint32_t height, ColorTileMode mode, std::uint32_t bytesPerElement = 4); std::size_t Bytes() const { return bytes; } - std::size_t LinearBytes() const { return static_cast(width) * height * 4u; } + std::size_t LinearBytes() const { return static_cast(width) * height * elementBytes; } std::size_t Alignment() const { return mode == ColorTileMode::Linear ? 256u : 65536u; } + std::uint32_t BlocksPerRow() const { return pitch / blockWidth; } std::size_t Offset(std::uint32_t x, std::uint32_t y) const; void Detile(std::span source, std::span destination) const; void Tile(std::span source, std::span destination) const; @@ -31,6 +33,12 @@ private: std::uint32_t pitch; ColorTileMode mode; std::size_t bytes; + std::uint32_t elementBytes; + std::uint32_t blockWidth = 1; + std::uint32_t blockHeight = 1; + // The XOR swizzle is linear over GF(2), so a block offset is xOffsets[x] ^ yOffsets[y]. + std::vector xOffsets; + std::vector yOffsets; }; } diff --git a/core/libs/prx/libSceAgcDriver/Graphics/include/Context.hpp b/core/libs/prx/libSceAgcDriver/Graphics/include/Context.hpp index d5c2ec543..a1c7ab64d 100644 --- a/core/libs/prx/libSceAgcDriver/Graphics/include/Context.hpp +++ b/core/libs/prx/libSceAgcDriver/Graphics/include/Context.hpp @@ -19,6 +19,7 @@ class TextureCache; class RenderCache; class DrawQueue; class GraphicsPipelineCache; +class Recorder; class DescriptorCache; class SamplerCache; @@ -59,11 +60,18 @@ struct Context { mutable std::shared_ptr bufferPool; TextureCache* textureCache = nullptr; VkPipelineCache pipelineCache = VK_NULL_HANDLE; + bool depthClamp = false; + // Nonzero when VK_EXT_external_memory_host is enabled: the required host pointer alignment. + VkDeviceSize hostImportAlignment = 0; RenderCache* renderCache = nullptr; DrawQueue* drawQueue = nullptr; GraphicsPipelineCache* graphicsPipelines = nullptr; - mutable std::shared_ptr descriptorCache; - mutable std::shared_ptr samplerCache; + // Batches GPU work across guest commands (see Recorder); null before the device finished setup. + Recorder* recorder = nullptr; + // Per-device caches of the immutable descriptor objects a ShaderResources build needs (set + // layouts, descriptor pools, samplers); null (tests) means every build makes and destroys its own. + DescriptorCache* descriptorCache = nullptr; + SamplerCache* samplerCache = nullptr; template TFunction Function(const char* name) const { diff --git a/core/libs/prx/libSceAgcDriver/Graphics/include/DccMetadata.hpp b/core/libs/prx/libSceAgcDriver/Graphics/include/DccMetadata.hpp new file mode 100644 index 000000000..80961435d --- /dev/null +++ b/core/libs/prx/libSceAgcDriver/Graphics/include/DccMetadata.hpp @@ -0,0 +1,53 @@ +#ifndef CORE_LIBS_PRX_LIBSCEAGCDRIVER_GRAPHICS_INCLUDE_DCCMETADATA_HPP +#define CORE_LIBS_PRX_LIBSCEAGCDRIVER_GRAPHICS_INCLUDE_DCCMETADATA_HPP + +#ifndef VK_NO_PROTOTYPES +#define VK_NO_PROTOTYPES +#endif +#include +#include "prx/libSceAgcDriver/Graphics/include/GuestTextureResource.hpp" +#include +#include +#include + +namespace AgcDriver::Graphics { + +struct Context; + +// Delta color compression keeps one key byte per 256 bytes of a color surface. Surfaces are written +// uncompressed here, so the keys that matter are the fast-clear codes a title writes into the metadata +// (the surface then reads as a constant whatever its texels hold) and "uncompressed", which the driver +// stores after it writes a surface so later reads see the texels. +enum class DccKeys { Uncompressed, Clear0000, Clear0001, Clear1110, Clear1111, ClearRegister, Mixed, Unreadable }; + +const char* DccKeysName(DccKeys keys); +// The keys covering a surface of `surfaceBytes`, when they all agree. +DccKeys ReadDccKeys(std::uint64_t metaAddress, std::uint64_t surfaceBytes); +bool IsDccClear(DccKeys keys); +// Stores "uncompressed" keys over the surface's metadata on the CPU (a guest memory write: it waits +// for recorded GPU work that writes the keys first). +void MarkDccUncompressed(std::uint64_t metaAddress, std::uint64_t surfaceBytes); +// The same after a write-back recorded under GuestMemory::GpuMutex: when the keys are in host-imported +// memory and a recorder is active, the store is a fill recorded into the open batch (ordered behind +// the title's key-writing kernels like the write-back itself, nothing waits on the CPU) and the range +// reads as uncompressed from ReadDccKeys while that batch is pending; otherwise the CPU store above. +// APS5_CPU_DCC_KEYS=1 always stores on the CPU. +void MarkDccUncompressed(const Context& context, std::uint64_t metaAddress, std::uint64_t surfaceBytes); +// Fills `bytes` with the texel a 0000/0001/1110/1111 clear code stands for ("1" is 1.0 or the integer +// maximum; the alpha channel is the last one in memory when alphaOnMsb, else the first, and 3-channel +// formats have none). False when the format has no encoding here. +bool FillDccClear(VkFormat format, DccKeys keys, bool alphaOnMsb, std::span bytes); + +// Where a color target's metadata puts alpha: the last channel in memory unless the component swap is +// reversed (single-channel formats: only with the alternate reversed swap). +bool DccAlphaOnMsb(VkFormat format, std::uint32_t componentSwap); + +// The keys of a texture's DCC surface when they fast-clear it to a value encodable here, else +// Uncompressed (the texels are read as stored; other keys are reported once). +DccKeys TextureClearKeys(const GuestTextureResource& resource, std::uint64_t guestBytes); +// A surface's texels as a read sees them: the guest bytes, or the clear value of fast-cleared keys. +void ReadTextureSurface(const GuestTextureResource& resource, DccKeys keys, std::span bytes); + +} + +#endif diff --git a/core/libs/prx/libSceAgcDriver/Graphics/include/Draw.hpp b/core/libs/prx/libSceAgcDriver/Graphics/include/Draw.hpp index f535b67fc..e3016a2ae 100644 --- a/core/libs/prx/libSceAgcDriver/Graphics/include/Draw.hpp +++ b/core/libs/prx/libSceAgcDriver/Graphics/include/Draw.hpp @@ -3,11 +3,28 @@ #include "prx/libSceAgcDriver/Graphics/include/Pipeline.hpp" #include "prx/libSceAgcDriver/Execution/include/Pm4.hpp" +#include +#include namespace AgcDriver::Graphics { void Draw(const Context& context, const State& state, const Pm4::DrawParameters& draw, std::span shaders, std::span snapshots = {}); +// Recorded draws whose written guest buffers were copied (their results reach guest memory by a CPU +// write-back when their batch completes), listed until that write-back ran. It is the draw +// counterpart of VulkanDevice's State::copiedWriters for dispatches, which VulkanDevice.cpp consults +// in two places: DispatchIndirect (an indirect dispatch whose arguments such a writer produced reads +// them on the CPU, behind the write-back, instead of from the host import on the GPU) and the fill +// HLE (a vkCmdFillBuffer over a range a listed write-back will land on must wait for it, or the +// write-back overwrites the fill). Draws are listed here only under APS5_RECORD_COPIED_DRAWS=1, +// which is safe only once BOTH of those checks also test this list, i.e. each extends its +// `std::any_of(state->copiedWriters ...)` with +// `|| std::any_of(Graphics::DrawCopiedWriters()->begin(), Graphics::DrawCopiedWriters()->end(), [&](const auto& writer) { return writer->WritesOverlap(); })`; +// cleaner still, State::copiedWriters can become this list (one registry for both producers), after +// which the switch can turn into APS5_NO_RECORD_COPIED_DRAWS. Until then such draws are recorded and +// waited for at once. Read and written under GuestMemory::GpuMutex, like the device's list. +std::shared_ptr>> DrawCopiedWriters(); + } #endif diff --git a/core/libs/prx/libSceAgcDriver/Graphics/include/GpuColorTransfer.hpp b/core/libs/prx/libSceAgcDriver/Graphics/include/GpuColorTransfer.hpp index 72cbe172e..63c5724fb 100644 --- a/core/libs/prx/libSceAgcDriver/Graphics/include/GpuColorTransfer.hpp +++ b/core/libs/prx/libSceAgcDriver/Graphics/include/GpuColorTransfer.hpp @@ -13,18 +13,15 @@ public: GpuColorTransfer(const GpuColorTransfer&) = delete; GpuColorTransfer& operator=(const GpuColorTransfer&) = delete; void Upload(std::uint64_t address, std::uint32_t width, std::uint32_t height, ColorTileMode mode); - void Detile(VkCommandBuffer commands, bool swapRedBlue = false); + void Detile(VkCommandBuffer commands, bool swapRedBlue = false, bool tenBit = false); void Tile(VkCommandBuffer commands); void WriteBack(std::uint64_t address); - void WriteBackTracked(std::uint64_t address); - bool MatchesGuest(std::uint64_t address); VkBuffer LinearBuffer() const; RenderTarget& Target(const ColorTarget& color, bool blending); private: - void writeBack(std::uint64_t address, bool tracked); void prepare(std::uint32_t width, std::uint32_t height, ColorTileMode mode); - void convert(VkCommandBuffer commands, bool toTiled, bool swapRedBlue); + void convert(VkCommandBuffer commands, bool toTiled, bool swapRedBlue, bool tenBit = false); void release() noexcept; Context context; VkDescriptorSetLayout descriptorLayout = VK_NULL_HANDLE; @@ -32,10 +29,10 @@ private: VkDescriptorSet descriptorSet = VK_NULL_HANDLE; VkPipelineLayout pipelineLayout = VK_NULL_HANDLE; VkPipeline pipeline = VK_NULL_HANDLE; + // Guest bytes in host memory, and the device-local copies the conversion shader works on. std::unique_ptr tiled; - std::unique_ptr linear; - std::unique_ptr readback; - std::unique_ptr upload; + std::unique_ptr tiledDevice; + std::unique_ptr linear; std::uint32_t width = 0; std::uint32_t height = 0; ColorTileMode mode = ColorTileMode::Linear; diff --git a/core/libs/prx/libSceAgcDriver/Graphics/include/GraphicsPipelineCache.hpp b/core/libs/prx/libSceAgcDriver/Graphics/include/GraphicsPipelineCache.hpp deleted file mode 100644 index 17299f660..000000000 --- a/core/libs/prx/libSceAgcDriver/Graphics/include/GraphicsPipelineCache.hpp +++ /dev/null @@ -1,30 +0,0 @@ -#ifndef CORE_LIBS_PRX_LIBSCEAGCDRIVER_GRAPHICS_INCLUDE_GRAPHICSPIPELINECACHE_HPP -#define CORE_LIBS_PRX_LIBSCEAGCDRIVER_GRAPHICS_INCLUDE_GRAPHICSPIPELINECACHE_HPP - -#include "prx/libSceAgcDriver/Graphics/include/Pipeline.hpp" -#include "prx/libSceAgcDriver/Graphics/include/RenderCache.hpp" -#include -#include -#include - -namespace AgcDriver::Graphics { - -class GraphicsPipelineCache { -public: - explicit GraphicsPipelineCache(const Context& context) : context(context) {} - std::shared_ptr Get(const State& state, const std::shared_ptr& target, const ShaderResources& resources, std::span shaders); - -private: - struct Entry { - std::string key; - std::shared_ptr target; - std::shared_ptr pipeline; - }; - Context context; - std::list entries; - std::map::iterator> lookup; -}; - -} - -#endif diff --git a/core/libs/prx/libSceAgcDriver/Graphics/include/GuestBufferMemory.hpp b/core/libs/prx/libSceAgcDriver/Graphics/include/GuestBufferMemory.hpp index cb5f0011f..06484200d 100644 --- a/core/libs/prx/libSceAgcDriver/Graphics/include/GuestBufferMemory.hpp +++ b/core/libs/prx/libSceAgcDriver/Graphics/include/GuestBufferMemory.hpp @@ -5,6 +5,8 @@ #include "BdaAbi.hpp" #include "prx/libc/include/GuestAllocations.hpp" #include +#include +#include #include namespace AgcDriver::Graphics { @@ -14,17 +16,118 @@ struct GuestMemorySnapshot { std::span bytes; }; +// A guest allocation imported with VK_EXT_external_memory_host, usable by the GPU in place. +struct HostImport { + std::uint64_t base; + std::uint64_t bytes; + VkBuffer buffer; + VkDeviceMemory memory; + VkDeviceAddress address; + // Identity for the life of this import (see HostImportSerial); 0 until first asked for. + std::uint64_t serial = 0; +}; + +// The host import of the registered allocation containing [address, address + bytes), made on demand +// (alignment and budget permitting), or null. Bytes at `address` are at `address - import->base` in +// the import's buffer. +const HostImport* HostImportFor(const Context& context, std::uint64_t address, std::size_t bytes); +// Whether an existing import covers [address, address + bytes), without reconciling the imports +// with the registry or making one (HostImportFor may take a registry lease): a hint for choices +// made outside the device lock (a sampled texture's path, a dispatch's pre-sync); the path taken +// re-checks with HostImportFor when it binds the import. +bool HostImportCovers(const Context& context, std::uint64_t address, std::size_t bytes); + +// A persistent device copy of one registered range of the main guest image (which cannot be host +// imported); see GuestBufferMemory.cpp. +struct ImageMirror; +class Recorder; + +// Identity of the import serving [address, address + bytes): a serial unique for the life of one +// import, 0 when no import serves the range. An equal serial later means the same VkBuffer still +// backs the range (a re-import gets a new serial), which is what a descriptor set written against it +// needs to stay valid (see the ShaderResources cache). With `reconcile`, imports whose registered +// range changed are dropped first, as an upload does; without it (right after an upload, before the +// work using the import is recorded) the imports are left as they are. +std::uint64_t HostImportSerial(const Context& context, std::uint64_t address, std::size_t bytes, bool reconcile); + +// Identity of the read-only image mirror serving [address, address + bytes) (the counterpart of +// HostImportSerial for main-image ranges, which no import serves): a serial unique for the life of +// one mirror with its top bit set, 0 when no read-only mirror whose registered range still exists +// covers the range. A read-only mirror's bytes and VkBuffer never change, so a descriptor set written +// against it stays valid while the serial is unchanged. +std::uint64_t ImageMirrorSerial(const Context& context, std::uint64_t address, std::size_t bytes); + +// Deferred lease release (see GuestBufferMemory.cpp). An address-based build leases every readable +// registered allocation until its write-back, which runs when its batch completed; the guest's +// unmap/mprotect/free of a leased allocation waits for that through the registry's pin waiter, which +// submits the recorder and finishes its batches up to the newest one recorded with a lease. SyncLeaseWork +// says whether the build's work must instead be synced as soon as it is recorded +// (APS5_SYNC_LEASE_DISPATCH=1, the behaviour before deferral). The driver and the draw path count +// their choice with CountLeaseOutcome right after the lease-holding resources were kept by the open +// batch: `batchSerial` is that batch's serial (Recorder::Submissions() + 1 under the device lock), 0 +// when synced, and the waiter targets the newest of them. Under APS5_PROFILE_DRAW CountLeaseOutcome +// prints the [address-sync] leases line every 10 s from LeaseCounters (cumulative). +bool SyncLeaseWork(); +void CountLeaseOutcome(bool synced, std::uint64_t batchSerial); +struct LeaseStats { + std::uint64_t deferred = 0; + std::uint64_t synced = 0; + // Pin-contention waits made by guest threads (the waiter calls), how many of them finished the + // newest lease batch, how many had to drain the whole recorder instead (no lease batch noted), + // and their total time. + std::uint64_t contentionWaits = 0; + std::uint64_t contentionSyncs = 0; + std::uint64_t contentionDrains = 0; + double contentionMs = 0; +}; +LeaseStats LeaseCounters(); + class GuestBufferMemory { public: explicit GuestBufferMemory(const Context& context); void AcquireRegistered(); + // A guest range bound through a descriptor. Both read live guest memory at upload and bind the + // same way (a storage buffer); AddWritable also notes the range in Writes(), so it gets the + // write-back's reference copy, a write-back, the recorder's pending-write note and the + // direct-write marks. AddReadable is for an element the shader is proved never to store to + // (DescriptorBinding::bufferWritten): the CPU never has to wait for it. void AddWritable(std::uint64_t address, std::size_t bytes); + void AddReadable(std::uint64_t address, std::size_t bytes); void AddSnapshot(const GuestMemorySnapshot& snapshot); + // Upload is the two stages below back to back. UploadPrepare needs no device lock: it merges the + // regions, binds the image mirrors and host imports that already serve them (an import pointer is + // re-checked against the registry epoch later) and copies read-only regions. UploadFinish runs + // under GuestMemory::GpuMutex: it reconciles and makes imports (retiring one hands its buffer to + // the recorder), refreshes writable mirrors and flushes pending results (both may wait for + // recorded work) and copies the regions a descriptor may write, so the window between that copy + // and the recorder's pending-write note stays closed. void Upload(bool addressable); + void UploadPrepare(bool addressable); + void UploadFinish(bool addressable); VkDescriptorBufferInfo Descriptor(std::uint64_t address, std::size_t bytes) const; std::vector AddressRanges() const; void WriteBack(); bool WritesOverlap(std::uint64_t address, std::size_t bytes) const; + // Guest ranges the shader may write through descriptors, as [begin, end). + const std::vector>& Writes() const { return writes; } + // Whether any written range lives in a copied buffer, so a write-back must run once the GPU is done. + bool HasCopiedWrites() const; + // Reports writes into host-imported memory (made by the GPU in place, or copied back into it by + // RecordCopyBacks) to the write tracking now. + void MarkDirectWrites() const; + // Records, into the recorder's open batch, the copy of every written sub-range of a region the + // GPU copied out of a host import (see Region::gpuCopy) back into the import: what the shader + // wrote lands in guest memory by the GPU, ordered after the recorded work, so the region needs + // no CPU write-back (HasCopiedWrites no longer counts it) and the caller notes the ranges as + // pending writes and marks them as MarkDirectWrites does. Called right after the work using the + // regions was recorded (ShaderResources::MarkGpuWrites), under GuestMemory::GpuMutex. A use that + // never calls it (a synchronous draw) stores the staging bytes from the CPU in WriteBack. + void RecordCopyBacks(Recorder& recorder); + // Whether registered allocations are pinned until write-back (address-based shaders). + bool HoldsLease() const { return !lease.empty(); } + // Every uploaded region as [begin, end) when all of them are served by host imports (nothing was + // copied, so the upload can serve a later identical build), else nothing. + std::optional>> DirectRegions() const; private: struct Region { @@ -32,14 +135,66 @@ private: std::uint64_t end; bool writable; std::vector snapshot; - std::unique_ptr buffer; + // Shared so a recorded GPU copy into it (see gpuCopy) can keep it in the batch itself, before + // the caller keeps the whole resources: a throw between the two would otherwise return it to + // the pool under an unsubmitted copy command. + std::shared_ptr buffer; + // Guest bytes as uploaded; write-back only stores bytes the GPU changed. + std::vector uploaded {}; + // Registered allocation that is imported: its bytes are read from live guest memory, not a snapshot. + bool hostBacked = false; + // Set when the region is served by an imported allocation; nothing is copied or written back. + const HostImport* direct = nullptr; + // Set when the region covers uncommitted pages: only the `backed` parts (possibly none) are guest + // memory; the rest reads as zeros and is never stored. + bool sparse = false; + std::vector> backed {}; + // Set when the region is served by an image mirror: nothing is copied; writable mirrors are + // written back by comparing with the mirror's shadow. Kept alive here for recorded work. + std::shared_ptr mirror {}; + // The mirror is a descriptor sub-range one found by UploadPrepare (not a lease mirror), which + // UploadFinish confirms still has its Range before the region binds it. + bool subrangeMirror = false; + // Set by UploadPrepare when UploadFinish still has device-lock work for the region: a copy a + // descriptor writes, a writable mirror's refresh, an import to reconcile or make, or the + // pending-results flush of an import it took. + bool pending = false; + // The region lies inside a host import but is misaligned for binding in place: `buffer` is + // filled by a vkCmdCopyBuffer from the import recorded into the batch (no CPU read, so no + // flush-hook wait), from `copySource` (the import's VkBuffer, alive until the batch + // completed, with the guest address of its first byte). `copiedBack` once RecordCopyBacks + // recorded the written sub-ranges' copies back into the import. + bool gpuCopy = false; + VkBuffer copySource = VK_NULL_HANDLE; + std::uint64_t copySourceBase = 0; + bool copiedBack = false; }; void validate(std::uint64_t address, std::size_t bytes) const; + // The region of a descriptor-bound range (AddWritable/AddReadable), committed pages only. + void addDescriptorRegion(std::uint64_t address, std::size_t bytes); + // Gives a region a buffer of its own with its bytes (guest memory for host-backed and writable + // ranges, plus the write-back's reference copy for a range a descriptor writes; else its snapshot). + void copyRegion(Region& region, bool addressable); + // Whether a region inside a host import that cannot be bound in place is copied by the GPU + // instead of the CPU (see Region::gpuCopy): live guest bytes, not sparse, at most the size + // APS5_GPU_COPY_MAX_KIB allows, and APS5_CPU_COPIES unset. + bool gpuCopyEligible(const Region& region) const; + // Records the import-to-buffer copies of the given gpuCopy regions into the open batch, with + // the barriers that order them after earlier recorded writes and before the shaders reading them. + void recordGpuCopies(std::span copies, bool addressable); Context context; GuestAllocations::Lease lease; + // Import registry epoch when `direct` pointers were taken at acquire time; they are reused while + // no import was destroyed since. + std::uint64_t importsEpoch = 0; std::vector regions; + // Whether `regions` is in ascending address order (true right after AcquireRegistered, whose + // regions follow the registry's order), so AddSnapshot can search instead of scanning. + bool regionsSorted = false; std::vector> writes; + // UploadPrepare ran (regions are frozen); `uploaded` once UploadFinish ran. + bool prepared = false; bool uploaded = false; bool committed = false; }; diff --git a/core/libs/prx/libSceAgcDriver/Graphics/include/GuestTextureResource.hpp b/core/libs/prx/libSceAgcDriver/Graphics/include/GuestTextureResource.hpp index 7dfb9855d..7d5393043 100644 --- a/core/libs/prx/libSceAgcDriver/Graphics/include/GuestTextureResource.hpp +++ b/core/libs/prx/libSceAgcDriver/Graphics/include/GuestTextureResource.hpp @@ -12,14 +12,32 @@ enum class TextureTileMode { kStandard256B, kStandard4KB, kStandard64KB, - RenderTarget64KB + // 64 KiB XOR swizzles; their address equations live in TextureSwizzleEquations.hpp. + kZ64KBX, + kS64KBX, + kD64KBX, + kR64KBX, + // Upstream's name for SW_64KB_R_X (tile mode 0x1b). + RenderTarget64KB = kR64KBX }; +// The hardware SW_MODE of an XOR swizzle tile mode, or 0 for the modes addressed without an equation table. +constexpr std::uint32_t XorSwizzleMode(TextureTileMode mode) { + switch (mode) { + case TextureTileMode::kZ64KBX: return 24u; + case TextureTileMode::kS64KBX: return 25u; + case TextureTileMode::kD64KBX: return 26u; + case TextureTileMode::kR64KBX: return 27u; + default: return 0u; + } +} + enum class TextureDimension { k1D, k2D, k2DArray, - kCube + kCube, + k3D }; struct GuestTextureResource { @@ -37,6 +55,11 @@ struct GuestTextureResource { std::uint8_t dstSelY; std::uint8_t dstSelZ; std::uint8_t dstSelW; + // Last mip level the view exposes; the surface itself holds mipCount levels. + std::uint32_t lastLevel = 0; + // DCC metadata of a compressed surface, or 0 (see DccMetadata.hpp). + std::uint64_t dccAddress = 0; + bool dccAlphaOnMsb = false; }; GuestTextureResource DecodeTextureResource(std::span words); diff --git a/core/libs/prx/libSceAgcDriver/Graphics/include/Pipeline.hpp b/core/libs/prx/libSceAgcDriver/Graphics/include/Pipeline.hpp index 87171cff3..a0722e148 100644 --- a/core/libs/prx/libSceAgcDriver/Graphics/include/Pipeline.hpp +++ b/core/libs/prx/libSceAgcDriver/Graphics/include/Pipeline.hpp @@ -2,31 +2,86 @@ #define CORE_LIBS_PRX_LIBSCEAGCDRIVER_GRAPHICS_INCLUDE_PIPELINE_HPP #include "prx/libSceAgcDriver/Graphics/include/ShaderResources.hpp" +#include namespace AgcDriver::Graphics { +struct VertexInputLayout; + +// The attachments of one render pass instance. Work recorded with it references the handle until +// the batch completes, so a recorded draw keeps the object (Recorder::Keep) like its pipeline. +class Framebuffer { +public: + Framebuffer(const Context& context, VkRenderPass renderPass, std::span targets, VkExtent2D extent); + ~Framebuffer(); + Framebuffer(const Framebuffer&) = delete; + Framebuffer& operator=(const Framebuffer&) = delete; + VkFramebuffer Handle() const { return framebuffer; } + // Forgets the handle without destroying it (the device it belongs to is already gone). + void Abandon() noexcept; + +private: + Context context; + VkFramebuffer framebuffer = VK_NULL_HANDLE; +}; + +// The shader modules, layout, render pass and VkPipeline of one draw configuration. Viewport and +// scissor are dynamic state set at Begin, so pipelines are shared by draws that differ only there +// (see CachedPipeline). class Pipeline { public: - Pipeline(const Context& context, const State& state, const RenderTarget* target, const ShaderResources& resources, std::span shaders); + // `state` carries the blend states with the outputs the pixel shader lacks already masked and + // `vertexInput` the layout of the draw's vertex descriptors; the shaders were validated by the + // caller (ValidateShaders). + Pipeline(const Context& context, const State& state, const VertexInputLayout& vertexInput, const ShaderResources& resources, std::span shaders); ~Pipeline(); Pipeline(const Pipeline&) = delete; Pipeline& operator=(const Pipeline&) = delete; VkPipelineLayout Layout() const; - void Begin(VkCommandBuffer commands, VkExtent2D extent) const; + // The framebuffer of these attachment views. A resident target's view (`owners[i]` set) is stable + // while its StorageTexture lives, so framebuffers made of resident views are kept in the pipeline + // and reused when every owner is still the same object; any other set is built per call. + std::shared_ptr AcquireFramebuffer(std::span targets, std::span> owners, VkExtent2D extent); + // Begins the render pass on the framebuffer, binds the pipeline and sets viewport and scissor. + void Begin(VkCommandBuffer commands, const Framebuffer& framebuffer, VkExtent2D extent, const VkViewport& viewport, const VkRect2D& scissor) const; void PushConstants(VkCommandBuffer commands, std::span shaders) const; + // Forgets the Vulkan objects without destroying them: for entries of a device that is already gone. + void Abandon() noexcept; private: + struct CachedFramebuffer { + std::vector views; + std::vector> owners; + VkExtent2D extent; + std::shared_ptr framebuffer; + }; void release() noexcept; Context context; std::vector _modules; VkPipelineLayout layout = VK_NULL_HANDLE; VkRenderPass renderPass = VK_NULL_HANDLE; - VkFramebuffer framebuffer = VK_NULL_HANDLE; VkPipeline pipeline = VK_NULL_HANDLE; + std::size_t attachments = 0; + std::vector framebuffers; }; +// The pipeline for this configuration, shared across draws and frames: keyed by the stages' variant +// ids (the rect-list tessellation pair by the vertex and fragment ids that generate it), the vertex +// input layout, the descriptor set layout key, push constant stages and every pipeline state field. +// A configuration whose stage has no variant id gets a private pipeline. Entries are LRU-bounded and +// only evicted once no recorded draw holds them. Debug aid: APS5_NO_PIPELINE_CACHE=1 builds one per +// draw as before. +std::shared_ptr CachedPipeline(const Context& context, const State& state, const VertexInputLayout& vertexInput, const ShaderResources& resources, std::span shaders); +// Destroys the cached pipelines of a device; to be called before the device goes away. Without it, +// entries of a gone device are recognised by their buffer pool (made and reset with the device, so +// it tells device instances apart when the loader reuses a VkDevice handle) and forgotten unused. +void ClearCachedPipelines(VkDevice device); +// Device limit checks of the viewport, which is dynamic state and so no longer checked by Pipeline. +void ValidateViewport(const Context& context, const VkViewport& viewport); + void ValidateShaderPair(const ShaderRecompiler::RecompileResult& vertex, const ShaderRecompiler::RecompileResult& fragment); -void ValidateShaders(std::span shaders, const State& state, const VkPhysicalDeviceSubgroupProperties& subgroup, bool fragmentShaderBarycentric); +// Returns the color attachment locations the pixel shader writes. +std::set ValidateShaders(std::span shaders, const State& state, const VkPhysicalDeviceSubgroupProperties& subgroup, bool fragmentShaderBarycentric); } diff --git a/core/libs/prx/libSceAgcDriver/Graphics/include/Recorder.hpp b/core/libs/prx/libSceAgcDriver/Graphics/include/Recorder.hpp new file mode 100644 index 000000000..0f37b4497 --- /dev/null +++ b/core/libs/prx/libSceAgcDriver/Graphics/include/Recorder.hpp @@ -0,0 +1,262 @@ +#ifndef CORE_LIBS_PRX_LIBSCEAGCDRIVER_GRAPHICS_INCLUDE_RECORDER_HPP +#define CORE_LIBS_PRX_LIBSCEAGCDRIVER_GRAPHICS_INCLUDE_RECORDER_HPP + +#include "prx/libSceAgcDriver/Graphics/include/Context.hpp" +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include + +namespace AgcDriver::Graphics { + +// Accumulates GPU work across guest commands so the CPU does not wait for each one. Dispatches and the +// copies that feed them record into one open batch; Submit sends it to the queue without waiting and +// Sync waits for every batch, then runs its completion actions (write-backs) in order. Objects handed +// to Keep live until the batch that recorded them completed. Guest memory ranges the recorded work +// will write are noted, so a CPU access to such a range (the GuestMemory flush hook) syncs first. +// Every call happens under GuestMemory::GpuMutex, except WaitSerial and the static lock-free readers +// below. The noted ranges are also published as an immutable snapshot (rebuilt under the mutex at +// every note and after a batch's completions ran) that the flush hook reads without the mutex, so +// accesses overlapping nothing never wait behind device work. +// +// Submissions are numbered (serials) and, when the device has timeline semaphores, every Submit +// signals a timeline semaphore with its serial: a thread can then wait for "everything submitted +// up to serial S" WITHOUT the mutex (WaitSerial), because the monotonic value cannot be reset or +// recycled the way the pooled batch fences are, and afterwards run the completions under the mutex +// (FinishUpTo). Without timeline semaphores callers use the locked Sync as before. +class Recorder { +public: + // `timelineSemaphores`: the device enabled VK_KHR_timeline_semaphore (WaitSerial is usable). + explicit Recorder(const Context& context, bool timelineSemaphores = false); + ~Recorder(); + Recorder(const Recorder&) = delete; + Recorder& operator=(const Recorder&) = delete; + + // The open batch's command buffer, starting a batch when none is open. + VkCommandBuffer Commands(); + bool Recording() const { return open != nullptr; } + bool Idle() const { return open == nullptr && inFlight.empty(); } + // Whether recorded work still has completion actions (write-backs the CPU must see) to run. + bool HasCompletions() const; + // The object lives at least until the batch open now completed; it is then destroyed AFTER the + // finishing thread released GuestMemory::GpuMutex, on a release thread of its own (never under + // a hold, see Recorder.cpp ReleaseDeferredKeeps; the finishing thread destroys it itself when + // the thread's queue is full, APS5_RELEASE_QUEUE_MAX batches, or with APS5_RELEASE_ON_UNLOCK=1; + // APS5_RELEASE_UNDER_LOCK=1 destroys it in the reap as before), so its destructor must need + // neither the mutex nor the recorder nor a particular thread. ~Recorder joins the release + // thread and waits for every release in progress before the device goes. + void Keep(std::shared_ptr object); + void OnComplete(std::function action); + void NotePendingWrite(std::uint64_t address, std::size_t bytes); + // Notes several [begin, end) ranges and publishes the snapshot once (a dispatch writes many buffers). + void NotePendingWrites(std::span> ranges); + bool PendingWriteOverlaps(std::uint64_t address, std::size_t bytes) const; + // Whether the OPEN (unsubmitted) batch writes the range: a wait on such a range must submit it. + bool OpenWriteOverlaps(std::uint64_t address, std::size_t bytes) const; + // What SyncThrough(address, bytes) would wait for, for the [hooksync] attribution: the newest + // batch whose noted range overlaps the access (the open batch counts first, with the serial it + // will get), whether that batch's fence has already signaled (its GPU work is done, so the + // access's bytes are already final; never for the open batch), that noted range, and how many + // batches the wait finishes. Nullopt: no overlap. + struct PendingWriteInfo { + std::uint64_t serial; + bool open; + bool signaled; + std::uint64_t rangeBegin; + std::uint64_t rangeEnd; + std::size_t batchesToFinish; + }; + std::optional DescribePendingWrite(std::uint64_t address, std::size_t bytes) const; + // Ends and submits the open batch without waiting. + void Submit(); + // Submits the open batch and returns the serial of the newest submitted batch (0 when nothing was + // ever submitted); WaitSerial(serial) then covers all recorded work. + std::uint64_t SubmitAndEpoch(); + bool HasTimeline() const { return timeline != VK_NULL_HANDLE; } + // Waits until every batch up to `serial` completed on the GPU. Called WITHOUT GuestMemory::GpuMutex: + // it touches only the immutable timeline semaphore (the caller keeps the device alive). Requires + // HasTimeline(); the debug switch APS5_NO_TIMELINE makes the device create the recorder without one. + void WaitSerial(std::uint64_t serial); + // Under the mutex: finishes (completions, release) the in-flight batches up to `serial`, from the + // front only; tolerates batches another thread finished meanwhile. Later batches stay in flight. + void FinishUpTo(std::uint64_t serial); + // Submits and waits for every batch, running completions in order. + void Sync(); + // Waits only for the batches up to the newest one that writes the range (submitting the open + // batch when it is that one); later batches stay in flight. Fences of one queue signal in + // submission order, so completions still run in order. Debug aid: APS5_NO_SYNC_THROUGH=1 syncs all. + // `waitUnlocked` (the flush hook): when the caller's GpuMutex acquisition is the outermost one + // on this thread and the device has a timeline, the GPU wait runs WITHOUT the mutex (released + // and retaken here; the caller must hold nothing else that orders after it) and only the + // completions run under it, so other queues do not queue behind a CPU read's wait. Inside a + // batch's completion action no wait is made at all: every batch still in flight was recorded + // after the completing one, and its store is that batch's in-order write (see Recorder.cpp). + void SyncThrough(std::uint64_t address, std::size_t bytes, bool waitUnlocked = false); + // Completes batches that already finished; returns whether nothing is in flight. + bool Reap(); + std::uint64_t Submissions() const { return submissions; } + + // Pending-label table. A GPU label (RELEASE_MEM/WRITE_DATA recorded as a store into the batch) + // is noted per 4-byte dword with the value it will store and a record-order stamp (the driver's + // event serial, taken when the label is recorded). A WAIT_REG_MEM whose submission was received + // (stamped) BEFORE the label was recorded may take the value from the table instead of waiting + // for the GPU: any CPU store the game ordered before that submit call precedes the label in + // program order exactly as on hardware, and later recorded work follows the label in queue + // order. Entries recorded before the submission (the previous frame's label at the same address) + // are never trusted. Entries leave the table with their batch (a memory bound only). + void NoteLabel(std::uint64_t address, std::span bytes, std::uint64_t stamp, std::uint32_t queue); + // The 4- or 8-byte value the table holds for `address` when every dword is present with a stamp + // newer than `afterStamp`; `queue` receives the recording queue of the first dword. + std::optional PendingLabel(std::uint64_t address, std::size_t bytes, std::uint64_t afterStamp, std::uint32_t& queue) const; + std::size_t PendingLabels() const { return labels.size(); } + // Whether any tracked label dword lies inside [address, address + bytes): a range query for + // large ranges (a fill of megabytes), where a per-dword lookup would not do. + bool PendingLabelIn(std::uint64_t address, std::size_t bytes) const; + // PendingLabel of the active recorder WITHOUT GuestMemory::GpuMutex: the table has a small mutex + // of its own (every mutation holds both), so a WAIT_REG_MEM consults it without queueing behind + // device work. Nothing is done under the table mutex but the lookup (it never takes the GPU mutex). + static std::optional LookupLabel(std::uint64_t address, std::size_t bytes, std::uint64_t afterStamp, std::uint32_t& queue); + // Whether the lock-free pending-write snapshot (open, in-flight and finishing batches) overlaps + // the range: false means no recorded work writes it, so a wait on it has nothing to submit. + static bool SnapshotWriteOverlaps(std::uint64_t address, std::size_t bytes); + // A label behind pending completions (CPU write-backs the label must not precede): stored to + // guest memory by a completion action appended to the open batch, or to the newest in-flight + // batch when none is open (no empty submission), so it runs after every earlier write-back in + // submission order. The range is noted as a pending write of that batch so CPU reads through the + // flush hook still sync, and the label enters the table like a GPU one. + // `storedOnGpu`: the same bytes were also recorded into the open batch (a host import), so the + // completion re-stores them only when a CPU write-back noted by NoteWrittenBack overlapped the + // range since; the plain store of a label the GPU has no view of always runs. An unconditional + // completion store would land on memory the game may have reused by then (a stale label value + // over a fresh command buffer). Debug aid: APS5_LABEL_STORE_ALWAYS=1 stores unconditionally. + void AfterCompletions(std::uint64_t address, std::span bytes, std::uint64_t stamp, std::uint32_t queue, bool storedOnGpu); + // A CPU store of GPU results into guest memory (GuestBufferMemory::WriteBack): recorded so a + // completion label store can tell whether its bytes were overwritten. + static void NoteWrittenBack(std::uint64_t address, std::size_t bytes); + + // Lock-free readers for the queue workers' packet loops and WAIT_REG_MEM polls (values of the + // active recorder; a torn-down recorder never publishes): + // when the open batch received its first label (nullopt: none pending), so the label's + // submission can be bounded (APS5_LABEL_FLUSH_US) without taking the mutex to look; + static std::optional PendingLabelSince(); + // bumped by every note of a pending write into the open batch, so a poller learns that the + // producer may have recorded the label it waits for; + static std::uint64_t WriteGeneration(); + // labels waiting in completion actions (the CPU must reap their batch for them to land); + static std::uint64_t PendingCompletionLabels(); + // dispatches and draws recorded since the last submit (the driver counts them: CountRecordedWork), + // so a batch is submitted after a bounded amount of work (APS5_BATCH_CAP). + static std::uint64_t RecordedWorkSinceSubmit(); + static void CountRecordedWork(); + + // The recorder of the current device, for code that only has guest addresses (the flush hook) + // and for helpers that must not recycle resources the recorded work still uses. + static Recorder* Active(); + void Activate(); + // APS5_PROFILE_DRAW: syncs by source (0 device idle, 1 CPU access to pending writes, 2 CPU read + // after a recorded store, 3 address-based dispatch, 4 other). Contract: the call announces the + // source for the Sync/SyncThrough this thread makes NEXT, which consumes it (a sync without a + // preceding CountSync counts as "other"); call it right before the sync and never without one, + // or the announcement attributes an unrelated later wait. `site` names the call site for the + // [recorder] "top sync sites" table (a return address, printed as a module offset like the + // [guestmem] callers): a caller that syncs on behalf of others (VulkanDevice::WaitIdle) passes + // its own return address, so the table names the driver site, not the wrapper; without it the + // return address of the Sync/SyncThrough call itself is taken, so an unannounced sync still + // names its real caller. + static void CountSync(int source, const void* site = nullptr); + // Names only the site for the sync or drain this thread makes next (its source stays as + // announced, or as the drain counts it): for a wrapper (VulkanDevice::ReapRecorded) whose + // FinishUpTo counts the drain itself, so a CountSync there would count it twice. + static void AnnounceSyncSite(const void* site); + // APS5_PROFILE_DRAW: the calling thread's fence and timeline waits so far, in milliseconds (0 + // when not profiling): a caller reads it around a span of its own work to learn how much of + // that span waited for the GPU (a resource build's nested flush-hook waits, a draw's). + static double ThreadWaitedMs(); + // APS5_PROFILE_GPU=1: GPU time of recorded work by key (a guest program address), from timestamp + // queries around each timed range; the totals per key are reported every 10 s. Begin returns the + // range index to pass to End, or NoTiming when timing is off or the batch's queries are used up. + static constexpr std::uint32_t NoTiming = 0xffffffffu; + std::uint32_t BeginGpuTiming(std::uint64_t key); + void EndGpuTiming(std::uint32_t index); + +private: + struct Batch { + VkCommandBuffer commands = VK_NULL_HANDLE; + VkFence fence = VK_NULL_HANDLE; + std::vector> kept; + std::vector> completions; + std::vector> writes; + // Submission number (1-based): identifies a batch after its allocation may have been reused. + std::uint64_t serial = 0; + bool submitted = false; + VkQueryPool queries = VK_NULL_HANDLE; + std::vector timedKeys; + // Dword addresses this batch noted in the label table (removed when it finishes). + std::vector labelDwords; + // Labels stored by this batch's completion actions (see AfterCompletions); subtracted from + // the lock-free pending count when the batch finishes, whether or not its completions ran. + std::uint32_t completionLabelCount = 0; + }; + struct LabelEntry { + std::uint32_t value; + std::uint32_t queue; + std::uint64_t stamp; + const Batch* batch; + }; + void readGpuTiming(Batch& batch); + + // The unlocked wait of SyncThrough(waitUnlocked): the target batch (submitting the open one when + // it is the target), the timeline wait with the mutex released, then the completions up to it. + // Returns false when the locked path must run instead (no timeline, nested acquisition, off). + bool syncThroughUnlocked(std::uint64_t address, std::uint64_t end, int source, const void* site); + // `source` is the CountSync source the wait is attributed to. + void finish(std::unique_ptr batch, bool wait, int source); + void release(Batch& batch) noexcept; + static bool overlaps(const Batch& batch, std::uint64_t address, std::uint64_t end); + // Rebuilds the lock-free snapshot of pending writes from open, inFlight and finishing. + void publishPendingWrites() const; + // Appends one range to the open batch; returns whether the snapshot must be rebuilt for it. + bool noteWrite(std::uint64_t address, std::size_t bytes); + // Appends one range to `batch` (open or in flight) and publishes the snapshot if needed. + void noteWriteOn(Batch& batch, std::uint64_t address, std::size_t bytes); + void noteLabelOn(Batch& batch, std::uint64_t address, std::span bytes, std::uint64_t stamp, std::uint32_t queue); + // Whether a write-back noted after `sequence` overlapped [begin, end); true when the ring no + // longer reaches back to `sequence` (conservative: the store runs as before). + bool writtenBackSince(std::uint64_t sequence, std::uint64_t begin, std::uint64_t end); + std::mutex writtenBackMutex; + std::deque> writtenBack; + std::uint64_t writtenBackSequence = 0; + // PendingLabel without the table mutex (the caller holds it, or the GPU mutex). + std::optional lookupLabel(std::uint64_t address, std::size_t bytes, std::uint64_t afterStamp, std::uint32_t& queue) const; + + Context context; + VkSemaphore timeline = VK_NULL_HANDLE; + // Identity for a thread that released the mutex around a wait (syncThroughUnlocked): the + // recorder may have been torn down meanwhile, so it is looked up by this id, not by pointer. + std::uint64_t id = 0; + // Mutated only under both GuestMemory::GpuMutex and the table mutex (Recorder.cpp), so a holder + // of either reads it consistently. + std::unordered_map labels; + std::unique_ptr open; + std::deque> inFlight; + // Batches popped from inFlight whose completions have not run yet: their writes stay in the + // snapshot (a rebuild from inside a completion must not drop them) until finish returns. + std::vector finishing; + std::uint64_t submissions = 0; + // Command buffers and fences of completed batches, reused by later ones (hundreds of batches per + // frame would otherwise allocate and free their objects each time). + std::vector> spare; +}; + +} + +#endif diff --git a/core/libs/prx/libSceAgcDriver/Graphics/include/Resources.hpp b/core/libs/prx/libSceAgcDriver/Graphics/include/Resources.hpp index c205f5bb4..4d4f226f2 100644 --- a/core/libs/prx/libSceAgcDriver/Graphics/include/Resources.hpp +++ b/core/libs/prx/libSceAgcDriver/Graphics/include/Resources.hpp @@ -26,13 +26,44 @@ private: VkDeviceMemory memory = VK_NULL_HANDLE; void* mapping = nullptr; std::size_t size; + // The VkBuffer's own size, `size` rounded up to its pool class (see BufferPool::Capacity). + std::size_t capacity; VkDeviceSize allocationBytes = 0; VkBufferUsageFlags usage; VkMemoryPropertyFlags properties; - bool reusable = false; std::shared_ptr cache; }; +// Device-local scratch memory for GPU-side layout conversion. The detiler reads and writes scattered +// elements, which crawls across PCIe, so guest bytes move between host and device buffers with DMA +// copies and are only swizzled in video memory. +class DeviceBuffer { +public: + DeviceBuffer(const Context& context, std::size_t size, VkBufferUsageFlags usage); + ~DeviceBuffer(); + DeviceBuffer(const DeviceBuffer&) = delete; + DeviceBuffer& operator=(const DeviceBuffer&) = delete; + VkBuffer Handle() const; + std::size_t Size() const; + +private: + void release() noexcept; + Context context; + VkBuffer buffer = VK_NULL_HANDLE; + VkDeviceMemory memory = VK_NULL_HANDLE; + std::size_t size; + std::size_t capacity; + VkDeviceSize allocationBytes = 0; + VkBufferUsageFlags usage; + std::shared_ptr cache; +}; + +// Records a whole-range buffer copy. +void CopyBuffer(const Context& context, VkCommandBuffer commands, VkBuffer source, VkDeviceSize sourceOffset, VkBuffer destination, VkDeviceSize destinationOffset, VkDeviceSize bytes); + +// Records a global memory barrier. +void RecordMemoryBarrier(const Context& context, VkCommandBuffer commands, VkPipelineStageFlags sourceStage, VkPipelineStageFlags destinationStage, VkAccessFlags sourceAccess, VkAccessFlags destinationAccess); + class RenderTarget { public: RenderTarget(const Context& context, const ColorTarget& target, bool blending); @@ -60,8 +91,6 @@ public: void SubmitAndWait(); void Submit(); void Wait(); - bool IsComplete(); - void Reset(); private: void release() noexcept; diff --git a/core/libs/prx/libSceAgcDriver/Graphics/include/Sampler.hpp b/core/libs/prx/libSceAgcDriver/Graphics/include/Sampler.hpp index 2e190d67c..e560d391a 100644 --- a/core/libs/prx/libSceAgcDriver/Graphics/include/Sampler.hpp +++ b/core/libs/prx/libSceAgcDriver/Graphics/include/Sampler.hpp @@ -4,8 +4,11 @@ #include "prx/libSceAgcDriver/Graphics/include/Context.hpp" #include "prx/libSceAgcDriver/Graphics/include/GuestSamplerResource.hpp" #include +#include #include +#include #include +#include namespace AgcDriver::Graphics { @@ -21,18 +24,36 @@ public: private: void release() noexcept; - VkDevice device; - PFN_vkDestroySampler destroySampler; + Context context; VkSampler sampler = VK_NULL_HANDLE; }; +// One VkSampler per distinct S# (its 4 words plus the shader's depth-compare use): samplers are +// immutable and DecodeSamplerResource is a pure function of the words, so equal keys mean equal +// samplers and nothing ever invalidates an entry. Holders keep a shared_ptr, so an entry evicted from +// the LRU-capped cache lives on while a descriptor set in flight references it. One cache per device. +// APS5_NO_SAMPLER_CACHE=1 creates a sampler per binding as before. class SamplerCache { public: - std::shared_ptr Get(const Context& context, std::span words, const GuestSamplerResource& descriptor); + explicit SamplerCache(std::size_t capacity = 1024); + SamplerCache(const SamplerCache&) = delete; + SamplerCache& operator=(const SamplerCache&) = delete; + std::shared_ptr Get(const Context& context, std::span words, bool compareEnable); + // APS5_PROFILE_DRAW counters: lookups served by an existing sampler, and samplers created. + std::uint64_t Hits() const { return hits; } + std::uint64_t Misses() const { return misses; } private: + struct Entry { + std::shared_ptr sampler; + std::uint64_t lastUse; + }; std::mutex mutex; - std::map, std::shared_ptr> entries; + std::map, Entry> entries; + std::uint64_t clock = 0; + std::size_t capacity; + std::uint64_t hits = 0; + std::uint64_t misses = 0; }; } diff --git a/core/libs/prx/libSceAgcDriver/Graphics/include/ShaderResources.hpp b/core/libs/prx/libSceAgcDriver/Graphics/include/ShaderResources.hpp index 773633095..25b3629aa 100644 --- a/core/libs/prx/libSceAgcDriver/Graphics/include/ShaderResources.hpp +++ b/core/libs/prx/libSceAgcDriver/Graphics/include/ShaderResources.hpp @@ -5,36 +5,166 @@ #include "prx/libSceAgcDriver/Graphics/include/BdaResources.hpp" #include "prx/libSceAgcDriver/Graphics/include/Texture.hpp" #include "prx/libSceAgcDriver/Graphics/include/Sampler.hpp" -#include "prx/libSceAgcDriver/Graphics/include/DescriptorCache.hpp" #include "Recompiler.hpp" #include "prx/libSceAgcDriver/Graphics/include/Shaders.hpp" #include +#include +#include +#include #include +#include +#include +#include #include namespace AgcDriver::Graphics { +class Recorder; + +// The cached storage image of a surface (render targets use it as their resident image); brought up +// to date with guest memory before it is returned. +std::shared_ptr CachedStorageSurface(const Context& context, const GuestTextureResource& resource); + +// Defined in Texture.cpp beside the pending-results registry, for the fast Revalidate below: whether +// a storage image other than `except` has results pending in [address, address + bytes). +bool PendingStorageOverlaps(std::uint64_t address, std::size_t bytes, const StorageTexture* except); + +// Per-device descriptor objects shared by ShaderResources builds: set layouts by their binding list +// (immutable; kept until the device is torn down, which Vulkan allows even for pipeline layouts made +// from them) and a chain of descriptor pools that sets are freed back to, so a build creates and +// destroys no layout or pool of its own. A set lives exactly as long as its ShaderResources, which +// its batch keeps until the GPU completed. The cache locks itself: stage A of a dispatch build +// (ShaderResources::buildPrepare, from VulkanDevice::PrepareDispatch) takes layouts and sets from it +// without GuestMemory::GpuMutex, while other threads free and update sets of the same pools under +// that lock (Vulkan synchronizes the pool for allocate/free and the set for update, so that is +// legal). APS5_NO_LAYOUT_CACHE=1 / APS5_NO_POOL_CACHE=1 restore the per-build objects. +class DescriptorCache { +public: + explicit DescriptorCache(const Context& context); + ~DescriptorCache(); + DescriptorCache(const DescriptorCache&) = delete; + DescriptorCache& operator=(const DescriptorCache&) = delete; + + // The layout for `key` (binding, type, count, stage flags per binding, as ShaderResources builds + // it from `bindings`), created on first use. + VkDescriptorSetLayout Layout(std::span key, std::span bindings); + struct SetAllocation { + VkDescriptorSet set = VK_NULL_HANDLE; + VkDescriptorPool pool = VK_NULL_HANDLE; + }; + // A set of `layout` needing `sizes` descriptors from the pool chain, opening a pool when no pool + // has room; a null set when the needs exceed what one chain pool holds (the caller then makes a + // dedicated pool, as before). + SetAllocation Allocate(VkDescriptorSetLayout layout, std::span sizes); + void Free(const SetAllocation& allocation) noexcept; + // APS5_PROFILE_DRAW counters: layouts served from the map / created, sets allocated, pools opened. + struct Stats { + std::uint64_t layoutHits = 0; + std::uint64_t layoutMisses = 0; + std::uint64_t sets = 0; + std::uint64_t pools = 0; + }; + Stats Counters() const; + +private: + Context context; + PFN_vkDestroyDescriptorSetLayout destroyLayout; + PFN_vkDestroyDescriptorPool destroyPool; + PFN_vkFreeDescriptorSets freeSets; + mutable std::mutex mutex; + std::map, VkDescriptorSetLayout> layouts; + std::vector pools; + Stats stats; +}; + class ShaderResources { public: ShaderResources(const Context& context, const ShaderRecompiler::RecompileResult& vertex, const ShaderRecompiler::RecompileResult& fragment, const ColorTarget& target, std::uint64_t indexAddress, std::size_t indexBytes); ShaderResources(const Context& context, std::span shaders, const ColorTarget& target, std::uint64_t indexAddress, std::size_t indexBytes, std::span snapshots = {}); ShaderResources(const Context& context, const CompiledShader& compute, std::span snapshots = {}); + // Two-stage build for dispatches (see build): with `deferred` the constructor runs stage A only, + // which needs no device lock, and Complete() runs stage B under GuestMemory::GpuMutex; until + // then no other member may be used. `compute` and `snapshots` must outlive Complete(). An + // address-based shader (BDA tables) builds entirely in Complete(): its lease acquisition + // reconciles imports and refreshes mirrors, which needs the lock. + ShaderResources(const Context& context, const CompiledShader& compute, std::span snapshots, bool deferred); + void Complete(); + bool Completed() const { return completed; } ~ShaderResources(); ShaderResources(const ShaderResources&) = delete; ShaderResources& operator=(const ShaderResources&) = delete; VkDescriptorSetLayout Layout() const; void Bind(VkCommandBuffer commands, VkPipelineBindPoint bindPoint, VkPipelineLayout layout) const; void WriteBack(); + // Deferred completion: MarkGpuWrites registers the results the recorded work leaves on the GPU + // (storage images stay there; buffer ranges are noted so CPU reads wait); WriteBackBuffers runs + // once the work completed. + void MarkGpuWrites(Recorder& recorder); + void WriteBackBuffers(); + // Whether WriteBackBuffers has anything the CPU must see (copied written buffers, BDA faults). + bool NeedsCompletion() const { return bda != nullptr || guestMemory.HasCopiedWrites(); } + // Whether a written buffer was copied (its results reach guest memory by the CPU write-back). + bool HasCopiedWrites() const { return guestMemory.HasCopiedWrites(); } + bool HoldsLease() const { return guestMemory.HoldsLease(); } bool WritesOverlap(std::uint64_t address, std::size_t bytes) const { return guestMemory.WritesOverlap(address, bytes); } const std::vector& LayoutKey() const { return layoutKey; } + // Debug aid: each bound guest resource with the fraction of sampled bytes that are nonzero. + std::string Describe() const; + + // Reuse across dispatches of identical content (see the resource cache in VulkanDevice): the + // key is everything a compute build consumes from its compiled shader (variant, and per binding + // kind, role, slot, count, guest descriptor words, written images, depth-compare samplers); push + // constants are not part of it because every dispatch pushes its own. Reusable says whether this + // object can serve a later dispatch with the same key: nothing needs completion work and every + // guest buffer is read in place through a host import. Revalidate brings a reusable object up to + // date for another use, or says it cannot be reused any more (a buffer's import went away, or a + // texture or storage image the set references is no longer the one its memory maps to). + static std::vector ContentKey(const CompiledShader& shader); + bool Reusable() const { return reusable; } + // `shaders` are the stages the object was built from, in build order (a recorded draw's vertex + // and fragment stages, or one compute stage): their bindings are walked like the build did. + bool Revalidate(std::span shaders); + bool Revalidate(const CompiledShader& shader) { return Revalidate(std::span(&shader, 1)); } + // APS5_PROFILE_DRAW: the build's sub-phase times in milliseconds (zero when not profiling), so + // the draw path can total them separately from dispatches. bindingsMs covers the binding plan + // (stage A) and the image lookups (stage B); prepareMs/completeMs are the two stages' totals. + struct BuildTiming { + double bindingsMs = 0; + double uploadMs = 0; + double descriptorsMs = 0; + double prepareMs = 0; + double completeMs = 0; + }; + const BuildTiming& Timing() const { return timing; } + // The image surfaces (address, bytes) whose stage-B lookup reads guest memory on the CPU and so + // would wait, under the device lock, for recorded work writing them: a surface without a host + // import, or a sampled one that stays a CPU snapshot (block compressed, or the storage-view path + // is off). VulkanDevice::PrepareDispatch waits for that work before the lock (the pre-sync). + // Usable right after the deferred constructor, for address-based builds too (it decodes the + // compiled shader's bindings itself when stage A has not run), and on a completed object from + // the resource cache (its Revalidate repeats the same lookups), without the device lock. + std::vector> PresyncSurfaces() const; private: + struct DescribedRange { + const char* kind; + std::uint64_t address; + std::uint64_t bytes; + std::uint32_t width = 0; + std::uint32_t height = 0; + std::uint32_t format = 0; + int tileMode = 0; + std::uint64_t dccAddress = 0; + }; + std::vector describedRanges; struct Allocation { std::uint64_t address; std::size_t size; bool guest; std::unique_ptr buffer; ShaderRecompiler::DescriptorRole role = ShaderRecompiler::DescriptorRole::ShaderData; + // Guest buffers: whether the shader may store to the range (see addGuestBuffer). + bool written = true; }; struct Binding { @@ -43,13 +173,92 @@ private: std::vector imageAllocations; }; + // A host-imported buffer region the set reads in place, with its import's identity at build time. + struct DirectRegion { + std::uint64_t begin; + std::uint64_t end; + std::uint64_t serial; + }; + + // The build is two stages. A (buildPrepare, no device lock): the binding plan with its layout + // entries, samplers, data buffers, the upload's prepare stage (imports already serving the + // regions, read-only copies), the set layout and the descriptor set. B (buildComplete, under + // GuestMemory::GpuMutex): the texture and storage image lookups (they refresh, upload and flush + // through the recorder), the rest of the upload, the BDA objects, the descriptor writes and the + // reusability record. build runs both. void build(std::span shaders, const ColorTarget* target, std::uint64_t indexAddress, std::size_t indexBytes); - std::size_t addGuestBuffer(std::span words, const ColorTarget* target, std::uint64_t indexAddress, std::size_t indexBytes); + void buildPrepare(std::span shaders, const ColorTarget* target, std::uint64_t indexAddress, std::size_t indexBytes); + void buildComplete(); + // APS5_PROFILE_DRAW: closes the current sub-phase of the build into the [resources] totals. + double phase(const char* name); + // `written` is the element's DescriptorBinding::bufferWritten: a read-only element binds the + // same way but is left out of the write set (no write-back, no pending-write note). + std::size_t addGuestBuffer(std::span words, const ColorTarget* target, std::uint64_t indexAddress, std::size_t indexBytes, bool written); std::size_t addDataBuffer(std::span words); - void addImageBinding(const ShaderRecompiler::DescriptorBinding& binding, VkShaderStageFlags flags, std::vector& bindings); + // Stage A: the layout entry of an image binding (samplers are taken at once, the sampler cache + // locks itself); stage B looks the sampled textures and storage images up (resolveImageBinding). + void addImageBinding(const ShaderRecompiler::DescriptorBinding& binding, VkShaderStageFlags flags); + // Stage A: one record per planned image element (ImageRecord), in plan order: the decoded + // descriptor and its surface size (computed once for the build), the write watch walked so stage + // B's collects are memo hits (APS5_NO_PRECOLLECT=1 skips the pass entirely), and for a sampled + // element the cache entry it maps to, taken now so stage B needs only the fastRevalidate + // predicate under the lock (APS5_NO_STAGE_A_IMAGES=1 leaves every element to the full lookup). + // Whether it ran. + bool precollectImages(); + struct ImageRecord { + bool sampled = false; + // The descriptor decoded and its surface described; false leaves the element to stage B's + // own decode (which reports the error). + bool decoded = false; + std::array words{}; + GuestTextureResource resource{}; + std::uint64_t guestBytes = 0; + VkComponentMapping components{}; + // Sampled elements: the keys read in stage A and the collect made BEFORE stage B's checks + // (the generation the entry moves to when they pass). + DccKeys keys = DccKeys::Uncompressed; + std::uint64_t generation = 0; + // The cache entry's object, storage source, keys and generation as of stage A (texture null: + // no entry, the full lookup makes one). + std::shared_ptr texture; + std::shared_ptr source; + DccKeys entryKeys = DccKeys::Uncompressed; + std::uint64_t entryGeneration = 0; + }; + std::vector imageRecords; + // The next record resolveImageBinding consumes (records follow the deferredImages order). + std::size_t nextImageRecord = 0; + // Stage B: the record's texture when the fastRevalidate predicate proves it current under the + // lock and the cache still holds it; null sends the element to cachedTexture. + std::shared_ptr fastTexture(const ImageRecord& record); + void resolveImageBinding(const ShaderRecompiler::DescriptorBinding& binding, Binding& item); + void forgetDeferredInputs(); void release() noexcept; void prepareAddressBindings(std::span shaders, std::span snapshots); VkDescriptorBufferInfo descriptor(const Allocation& allocation) const; + void noteReusable(); + void reportDescriptorCaches() const; + // What a sampled texture was proved current against when the build (or the last full Revalidate) + // looked it up, so the next Revalidate can repeat the proof from write stamps and the DCC keys + // alone instead of repeating the lookup (see fastRevalidate). One record per textures[] entry; an + // element without a record always takes the full lookup. Storage images need no record: their + // proof comes from the image itself (its own surface and generation). + struct ValidatedSurface { + GuestTextureResource resource{}; + std::uint64_t bytes = 0; + // Keys read at the lookup (a snapshot texture holds its clear texels while they are unchanged). + DccKeys keys = DccKeys::Uncompressed; + // Snapshot textures: write generation the content was known current at, always from a + // collect issued before the checks that proved it. + std::uint64_t generation = 0; + // The collect of the check in progress, moved into `generation` once every element passed. + std::uint64_t collected = 0; + // Storage-sourced textures: the image the view follows (kept alive by the texture). + const StorageTexture* source = nullptr; + bool valid = false; + }; + void captureValidation(); + bool fastRevalidate(); Context context; std::vector layoutKey; GuestBufferMemory guestMemory; @@ -57,13 +266,93 @@ private: bool usesBda = false; bool usesFaultBuffer = false; VkDescriptorSetLayout _layout = VK_NULL_HANDLE; + // Whether the layout is this object's own (no cache) and destroyed with it. + bool ownsLayout = false; VkDescriptorSet _set = VK_NULL_HANDLE; - std::unique_ptr descriptors; + // Dedicated pool of this object's set (no cache, or a set too large for a cache pool). + VkDescriptorPool pool = VK_NULL_HANDLE; + // The cache pool the set was allocated from, freed back to it on release. + VkDescriptorPool cachePool = VK_NULL_HANDLE; std::vector allocations; + // Guest buffer elements bound read-only: each use of this object skips that many pending-write + // notes (counted in MarkGpuWrites for the [buffers] line). + std::size_t readOnlyBuffers = 0; std::vector> textures; + std::vector> storageTextures; + std::vector storageMips; + std::vector storageWritten; std::vector> samplers; + bool reusable = false; + std::vector directRegions; + std::vector validatedTextures; + BuildTiming timing; + // Build state carried from stage A to stage B: the bindings in plan order, the image bindings + // still to look up (index into `bindings`; the DescriptorBinding lives in the compiled shader), + // the descriptor counts the set was sized for, and the compute stage of a deferred build. + std::vector bindings; + struct DeferredImages { + const ShaderRecompiler::DescriptorBinding* binding; + std::size_t index; + }; + std::vector deferredImages; + std::uint64_t storageBuffers = 0; + std::uint32_t plannedSampledImages = 0; + std::uint32_t plannedStorageImages = 0; + // The compute constructor's shader and captured regions: the caller's objects, valid only until + // the build (Complete() for a deferred one) is done, and reset then (forgetDeferredInputs). + CompiledShader deferredCompute{}; + std::span deferredSnapshots; + // The deferred build is address-based: everything runs in Complete(). + bool lockedBuild = false; + // Stage A runs without the device lock (a deferred, non-address-based build). + bool unlockedPrepare = false; + bool completed = false; + std::chrono::steady_clock::time_point phaseStart; }; +// Built ShaderResources by the content the build consumed (see ShaderResources::ContentKey), so a +// dispatch or recorded draw repeating an earlier one binds the earlier descriptor set instead of +// building one (import lookups, sampler and descriptor objects, guest buffer uploads). Only reusable +// objects (nothing to complete, every buffer read in place) are kept, and an entry serves a later +// use only after ShaderResources::Revalidate brought it up to date. Entries hold their object alive; +// the recorder keeps it as well while its batch runs. One LRU bound covers dispatch and draw entries +// together (APS5_RESOURCE_CACHE_ENTRIES, default 1024); the map has its own mutex and locks itself: +// VulkanDevice::PrepareDispatch probes it without GuestMemory::GpuMutex (a hit there is revalidated +// under the lock by the dispatch; inserts and Revalidate stay under it). Compute keys start with the +// stage, draw keys with a marker word, so the two never meet, and both name the device handle: the driver replaces +// the headless device with the windowed one while workers may still hold the old one, so a replaced +// device's entries (whose descriptor sets and pooled buffers belong to it) are unreachable by key +// and dropped by that device's Clear(). APS5_NO_RESOURCE_CACHE=1 (dispatches) and +// APS5_NO_DRAW_RESOURCE_CACHE=1 (draws) build every object as before; the device clears the cache at +// teardown before its descriptor caches go. +class ResourceCache { +public: + using Key = std::vector; + std::shared_ptr Find(const Key& key); + void Insert(const Key& key, std::shared_ptr resources); + void Remove(const Key& key); + void Clear(); + std::size_t Size() const; + +private: + void erase(const Key& key); + void noteMiss(const Key& key); + struct KeyHash { + std::size_t operator()(const Key& key) const noexcept { + std::uint64_t hash = 14695981039346656037ull; + for (const auto word : key) hash = (hash ^ word) * 1099511628211ull; + return static_cast(hash); + } + }; + mutable std::mutex mutex; + std::list>> entries; + std::unordered_map index; +}; + +// The process-wide cache the device and the draw path share (keys name the device; a device clears +// it when it goes). Never destroyed: entries belong to the device, not to static teardown. +ResourceCache& SharedResourceCache(); + } #endif diff --git a/core/libs/prx/libSceAgcDriver/Graphics/include/State.hpp b/core/libs/prx/libSceAgcDriver/Graphics/include/State.hpp index 590dd23f6..b20568e1f 100644 --- a/core/libs/prx/libSceAgcDriver/Graphics/include/State.hpp +++ b/core/libs/prx/libSceAgcDriver/Graphics/include/State.hpp @@ -4,6 +4,7 @@ #include "prx/libSceAgcDriver/Graphics/include/Context.hpp" #include "prx/libSceAgcDriver/Graphics/include/ColorTargetLayout.hpp" #include "prx/libSceAgcDriver/Execution/include/QueueState.hpp" +#include #include #include #include @@ -34,17 +35,25 @@ struct ColorTarget { std::size_t bytes; std::uint8_t componentMapping; ColorTileMode tileMode = ColorTileMode::Linear; + std::uint32_t elementBytes = 4; + // DCC metadata of a compressed target (CB_COLOR_INFO DCC_ENABLE), or 0 (see DccMetadata.hpp). + std::uint64_t dccAddress = 0; + bool dccAlphaOnMsb = false; }; struct State { ShaderStages stages; + // MRT slot 0; `colors`/`blends` hold every written slot, attachment i being slot i. ColorTarget color; + std::vector colors; + std::vector blends; bool hasColorTarget; bool rectList = false; VkExtent2D renderExtent; VkPrimitiveTopology topology; VkViewport viewport; bool negativeOneToOne; + bool depthClamp = false; VkRect2D scissor; VkCullModeFlags cullMode; VkFrontFace frontFace; diff --git a/core/libs/prx/libSceAgcDriver/Graphics/include/Texture.hpp b/core/libs/prx/libSceAgcDriver/Graphics/include/Texture.hpp index aca05fbf3..b1d9f9353 100644 --- a/core/libs/prx/libSceAgcDriver/Graphics/include/Texture.hpp +++ b/core/libs/prx/libSceAgcDriver/Graphics/include/Texture.hpp @@ -2,35 +2,174 @@ #define CORE_LIBS_PRX_LIBSCEAGCDRIVER_GRAPHICS_INCLUDE_TEXTURE_HPP #include "prx/libSceAgcDriver/Graphics/include/Context.hpp" +#include "prx/libSceAgcDriver/Graphics/include/DccMetadata.hpp" #include "prx/libSceAgcDriver/Graphics/include/GuestTextureResource.hpp" #include "prx/libSceAgcDriver/Graphics/include/TextureDetiler.hpp" +#include "prx/libSceAgcDriver/Graphics/include/TextureTiling.hpp" +#include +#include +#include +#include namespace AgcDriver::Graphics { class ResidentColor; class CommandBatch; +class StorageTexture; + +// The Vulkan format storage images of a guest format use (sRGB formats store as their UNORM form). +// Throws when the format has no storage form; the query is cached per format (a lookup cost a +// vkGetPhysicalDeviceFormatProperties call before). +VkFormat StorageFormatForGuest(const Context& context, std::uint32_t guestFormat); +// Whether StorageFormatForGuest would succeed, without throwing (unknown guest formats included). +bool StorageFormatAvailable(const Context& context, std::uint32_t guestFormat); + +// A sampled texture's own VkImage with its memory, shared with the recorder while a recorded upload +// still writes it (see the snapshot constructor), so the texture may go before the batch completes. +struct OwnedImage { + OwnedImage(const Context& context, VkImage image, VkDeviceMemory memory) : context(context), image(image), memory(memory) {} + OwnedImage(const OwnedImage&) = delete; + OwnedImage& operator=(const OwnedImage&) = delete; + ~OwnedImage() { + if (image) context.Function("vkDestroyImage")(context.device, image, nullptr); + if (memory) context.Function("vkFreeMemory")(context.device, memory, nullptr); + } + Context context; + VkImage image; + VkDeviceMemory memory; +}; class Texture { public: Texture(const Context& context, TextureDetiler& detiler, const GuestTextureResource& descriptor, VkComponentMapping components, std::span snapshot); Texture(const Context& context, const std::shared_ptr& source, const GuestTextureResource& descriptor, VkComponentMapping components); + // A view of a storage image's own VkImage: the sampled texture follows the image's content, so a + // compute pass writing it and the next pass sampling it share one image and copy nothing. + // CanCopyFrom says whether the two descriptors address the same surface compatibly. + Texture(const Context& context, const std::shared_ptr& source, const GuestTextureResource& descriptor, VkComponentMapping components); + static bool CanCopyFrom(const StorageTexture& source, const GuestTextureResource& descriptor); ~Texture(); Texture(const Texture&) = delete; Texture& operator=(const Texture&) = delete; VkImageView View() const; + // The layout the image is kept in while sampled. + VkImageLayout Layout() const { return layout; } VkDeviceSize AllocationBytes() const { return allocationBytes; } + // Whether this texture is a view of a storage image (no snapshot of its own). + bool ViewsStorageImage() const { return storageSource != nullptr; } private: void release() noexcept; + // Held by value: cached textures outlive the Context of the draw that created them. Context context; + // The snapshot constructor's image (null for a view of a storage image); `owned` frees it. + VkImage image = VK_NULL_HANDLE; + std::shared_ptr owned; + VkImageView view = VK_NULL_HANDLE; + VkImageLayout layout = VK_IMAGE_LAYOUT_SHADER_READ_ONLY_OPTIMAL; + VkDeviceSize allocationBytes = 0; + std::shared_ptr source; + std::shared_ptr storageSource; + std::unique_ptr upload; +}; + +// A guest texture a shader writes through a storage image. It is uploaded like a sampled texture; +// after the GPU work completes its results are stored to guest memory (retiled, changed bytes only), +// either at once (WriteBack) or deferred: MarkDirty keeps them on the GPU until something reads that +// memory (FlushPending, through the GuestMemory flush hook), the image is refreshed after a CPU write, +// or it leaves the cache. Dispatches reusing the image meanwhile skip the round trip entirely. +class StorageTexture : public std::enable_shared_from_this { +public: + StorageTexture(const Context& context, TextureDetiler& detiler, const GuestTextureResource& descriptor, std::uint32_t mipLevel); + ~StorageTexture(); + StorageTexture(const StorageTexture&) = delete; + StorageTexture& operator=(const StorageTexture&) = delete; + + VkImageView View() const; + // The image holds every mip of the surface; one storage view per written mip is made on demand, + // so successive mip writes of a chain share one image and one write-back. + VkImageView View(std::uint32_t mip); + // Render targets live in the same images: draws attach mip 0 through a view of the color + // buffer's format and mark the image dirty like a storage write. + bool Attachable() const { return attachable; } + VkImageView AttachmentView(VkFormat format); + void WriteBack(); + // Deferred write-back (APS5_EAGER_WRITEBACK=1 stores at once instead). + void MarkDirty(); + void Flush(); + // Stores every pending image overlapping the range, except `except`; returns whether any was. + // Stores into host-imported memory are recorded (not waited for): a CPU reader syncs afterwards. + static bool FlushPending(std::uint64_t address, std::size_t bytes, const StorageTexture* except = nullptr, const char* reason = "memory access"); + static void FlushAllPending(const char* reason); + // The pending image whose surface starts at `address` and covers at least `bytes` (a mip chain + // contains a descriptor of its first mips), if any; the caller decides whether the geometry fits + // (Texture::CanCopyFrom, ResidentPresentable). + static std::shared_ptr FindPending(std::uint64_t address, std::uint64_t bytes); + VkImage Image() const { return image; } + const GuestTextureResource& Descriptor() const { return descriptor; } + std::uint32_t ImageLayers() const { return geometry.imageLayers; } + std::uint32_t ImageDepth() const { return geometry.imageDepth; } + // Content version: advances when the image is re-uploaded or a shader wrote it. Together with + // Generation (the write generation guest memory was last known to match the content at) it + // validates textures copied from this image. + std::uint64_t Version() const { return version; } + std::uint64_t Generation() const { return generation; } + // Brings the image up to date with guest memory before another use; returns whether its content + // was still current (nothing uploaded). + // Keeps the image current with guest memory (see GuestMemory::CollectWrites). + bool Refresh(); + std::uint64_t GuestBytes() const; + +private: + std::vector CopyRegions() const; + void upload(); + // Stores the image to guest memory; with `skipWrittenSince` set, 64 KiB blocks the CPU wrote since + // that write generation keep the CPU's bytes. + void writeBack(std::uint64_t skipWrittenSince); + // Pending images whose surface shares a 64 KiB write-stamp block with this one without + // overlapping it, and whose memory is unchanged since their generation (see writeBack), with + // the generation seen at the check. + struct Adjacent { + std::shared_ptr texture; + std::uint64_t generation; + }; + std::vector adjacentPendingUnchanged() const; + // Advances the entries still at their seen generation to `now`, the generation this write-back's + // stamps of blocks firstBlock..lastBlock (64 KiB) are below, unless the neighbour's other blocks + // were stamped since the check. + static void advanceAdjacent(const std::vector& adjacent, std::uint64_t now, std::uint64_t firstBlock, std::uint64_t lastBlock); + bool overlaps(std::uint64_t address, std::size_t bytes) const; + VkImageView createView(std::uint32_t mip) const; + void release() noexcept; + + Context context; + TextureDetiler& detiler; + GuestTextureResource descriptor; + std::vector mips; + std::uint32_t arrayLayers = 1; + std::uint64_t guestBytes = 0; + std::uint64_t sliceLinearBytes = 0; + SurfaceGeometry geometry; + std::vector original; + // DCC keys the image content was uploaded under: a fast-cleared surface starts as its clear value. + DccKeys uploadedKeys = DccKeys::Uncompressed; + // Write generation `original` is known current at. + std::uint64_t generation = 0; VkImage image = VK_NULL_HANDLE; VkDeviceMemory memory = VK_NULL_HANDLE; VkImageView view = VK_NULL_HANDLE; - VkDeviceSize allocationBytes = 0; - std::shared_ptr source; - std::unique_ptr upload; + std::uint32_t defaultMip = 0; + std::map extraViews; + bool attachable = false; + std::map attachmentViews; + VkFormat storageFormat = VK_FORMAT_UNDEFINED; + // Results are on the GPU only (guarded by the pending-write registry lock). + bool dirty = false; + std::uint64_t version = 0; + // `original` holds the guest bytes; false after a GPU-side clear, which never read them. + bool originalValid = true; }; } diff --git a/core/libs/prx/libSceAgcDriver/Graphics/include/TextureDetiler.hpp b/core/libs/prx/libSceAgcDriver/Graphics/include/TextureDetiler.hpp index f6782186e..9436f2597 100644 --- a/core/libs/prx/libSceAgcDriver/Graphics/include/TextureDetiler.hpp +++ b/core/libs/prx/libSceAgcDriver/Graphics/include/TextureDetiler.hpp @@ -1,6 +1,7 @@ #ifndef CORE_LIBS_PRX_LIBSCEAGCDRIVER_GRAPHICS_INCLUDE_TEXTUREDETILER_HPP #define CORE_LIBS_PRX_LIBSCEAGCDRIVER_GRAPHICS_INCLUDE_TEXTUREDETILER_HPP +#include #include "prx/libSceAgcDriver/Graphics/include/Context.hpp" #include "prx/libSceAgcDriver/Graphics/include/GuestTextureResource.hpp" #include "prx/libSceAgcDriver/Graphics/include/TextureTiling.hpp" @@ -17,11 +18,14 @@ namespace AgcDriver::Graphics { TextureDetiler(const TextureDetiler&) = delete; TextureDetiler& operator=(const TextureDetiler&) = delete; - void Dispatch(VkCommandBuffer commands, TextureTileMode tileMode, std::uint32_t elementBytes, VkBuffer source, std::uint64_t sourceOffset, VkBuffer destination, std::uint64_t destinationOffset, const TileMipLayout& layout, std::uint32_t arrayLayer); + // Detiles `source` (tiled) into `destination` (linear), or with `retile` writes linear `source` + // into tiled `destination`; offsets always refer to the respective buffers. + void Dispatch(VkCommandBuffer commands, TextureTileMode tileMode, std::uint32_t elementBytes, VkBuffer source, std::uint64_t sourceOffset, VkBuffer destination, std::uint64_t destinationOffset, const TileMipLayout& layout, bool retile = false, std::uint32_t slice = 0, bool thick = false); + // Recycles the descriptor sets of the previous batch; call before recording a new command batch. void BeginBatch(); private: - VkPipeline pipeline(TextureTileMode tileMode, std::uint32_t elementBytes); + VkPipeline pipeline(TextureTileMode tileMode, std::uint32_t elementBytes, bool retile, bool thick); void release() noexcept; VkDescriptorSet allocateSet(); diff --git a/core/libs/prx/libSceAgcDriver/Graphics/include/TextureSwizzleEquations.hpp b/core/libs/prx/libSceAgcDriver/Graphics/include/TextureSwizzleEquations.hpp new file mode 100644 index 000000000..99dc592e8 --- /dev/null +++ b/core/libs/prx/libSceAgcDriver/Graphics/include/TextureSwizzleEquations.hpp @@ -0,0 +1,63 @@ +#ifndef CORE_LIBS_PRX_LIBSCEAGCDRIVER_GRAPHICS_INCLUDE_TEXTURESWIZZLEEQUATIONS_HPP +#define CORE_LIBS_PRX_LIBSCEAGCDRIVER_GRAPHICS_INCLUDE_TEXTURESWIZZLEEQUATIONS_HPP + +#include +#include + +namespace AgcDriver::Graphics { + +// Address equations of the 64 KiB XOR swizzle modes (SW_64KB_Z_X, S_X, D_X, R_X) for the PS5 GPU, a +// Navi1x-class part with 16 pipes. Generated from AMD addrlib's GFX10 non-RB+ swizzle pattern tables +// (the 32 bpp R_X entry reproduces the scanout layout in ColorTargetLayout). +// Each entry gives, for byte-address bit 0..15 inside a block, the element-coordinate bits XORed into it: +// bits 0-11 select x bits, bits 12-23 select y bits and bits 24-31 select slice (or depth) bits. +struct TextureSwizzleEquation { + std::uint32_t swizzleMode; + std::uint32_t elementBytes; + std::array bits; +}; + +inline constexpr TextureSwizzleEquation kTextureSwizzleEquations[] = { + {24u, 1u, {0x00000001u, 0x00001000u, 0x00000002u, 0x00002000u, 0x00000004u, 0x00004000u, 0x00000008u, 0x00010000u, 0x08008008u, 0x04010010u, 0x02020040u, 0x01040020u, 0x00040000u, 0x00000040u, 0x00080000u, 0x00000080u}}, + {24u, 2u, {0x00000000u, 0x00000001u, 0x00001000u, 0x00000002u, 0x00002000u, 0x00000004u, 0x00004000u, 0x00000008u, 0x08008008u, 0x04010010u, 0x02020040u, 0x01040020u, 0x00010000u, 0x00000040u, 0x00040000u, 0x00000080u}}, + {24u, 4u, {0x00000000u, 0x00000000u, 0x00000001u, 0x00001000u, 0x00000002u, 0x00002000u, 0x00000004u, 0x00004000u, 0x08008008u, 0x04010010u, 0x02020040u, 0x01040020u, 0x00008000u, 0x00000010u, 0x00040000u, 0x00000040u}}, + {24u, 8u, {0x00000000u, 0x00000000u, 0x00000000u, 0x00000001u, 0x00001000u, 0x00000002u, 0x00002000u, 0x00000004u, 0x08008008u, 0x04010010u, 0x02020040u, 0x01040020u, 0x00004000u, 0x00000008u, 0x00010000u, 0x00000040u}}, + {24u, 16u, {0x00000000u, 0x00000000u, 0x00000000u, 0x00000000u, 0x00000001u, 0x00001000u, 0x00000002u, 0x00002000u, 0x08008008u, 0x04010010u, 0x02020040u, 0x01040020u, 0x00004000u, 0x00000004u, 0x00008000u, 0x00000010u}}, + {25u, 1u, {0x00000001u, 0x00000002u, 0x00000004u, 0x00000008u, 0x00001000u, 0x00002000u, 0x00004000u, 0x00008000u, 0x08010080u, 0x04080010u, 0x02020040u, 0x01040020u, 0x00040000u, 0x00000040u, 0x00080000u, 0x00000080u}}, + {25u, 2u, {0x00000000u, 0x00000001u, 0x00000002u, 0x00000004u, 0x00001000u, 0x00002000u, 0x00004000u, 0x00000008u, 0x08008080u, 0x04040010u, 0x02010040u, 0x01020020u, 0x00020000u, 0x00000040u, 0x00040000u, 0x00000080u}}, + {25u, 4u, {0x00000000u, 0x00000000u, 0x00000001u, 0x00000002u, 0x00001000u, 0x00002000u, 0x00004000u, 0x00000004u, 0x08008040u, 0x04040008u, 0x02010020u, 0x01020010u, 0x00020000u, 0x00000020u, 0x00040000u, 0x00000040u}}, + {25u, 8u, {0x00000000u, 0x00000000u, 0x00000000u, 0x00000001u, 0x00001000u, 0x00002000u, 0x00000002u, 0x00000004u, 0x08004040u, 0x04020008u, 0x02008020u, 0x01010010u, 0x00010000u, 0x00000020u, 0x00020000u, 0x00000040u}}, + {25u, 16u, {0x00000000u, 0x00000000u, 0x00000000u, 0x00000000u, 0x00001000u, 0x00002000u, 0x00000001u, 0x00000002u, 0x08004020u, 0x04020004u, 0x02008010u, 0x01010008u, 0x00010000u, 0x00000010u, 0x00020000u, 0x00000020u}}, + {26u, 1u, {0x00000001u, 0x00000002u, 0x00000004u, 0x00002000u, 0x00001000u, 0x00004000u, 0x00000008u, 0x00008000u, 0x08010080u, 0x04080010u, 0x02020040u, 0x01040020u, 0x00040000u, 0x00000040u, 0x00080000u, 0x00000080u}}, + {26u, 2u, {0x00000000u, 0x00000001u, 0x00000002u, 0x00000004u, 0x00001000u, 0x00002000u, 0x00004000u, 0x00000008u, 0x08008080u, 0x04040010u, 0x02010040u, 0x01020020u, 0x00020000u, 0x00000040u, 0x00040000u, 0x00000080u}}, + {26u, 4u, {0x00000000u, 0x00000000u, 0x00000001u, 0x00000002u, 0x00001000u, 0x00002000u, 0x00004000u, 0x00000004u, 0x08008040u, 0x04040008u, 0x02010020u, 0x01020010u, 0x00020000u, 0x00000020u, 0x00040000u, 0x00000040u}}, + {26u, 8u, {0x00000000u, 0x00000000u, 0x00000000u, 0x00000001u, 0x00001000u, 0x00000002u, 0x00000004u, 0x00002000u, 0x08004040u, 0x04020008u, 0x02008020u, 0x01010010u, 0x00010000u, 0x00000020u, 0x00020000u, 0x00000040u}}, + {26u, 16u, {0x00000000u, 0x00000000u, 0x00000000u, 0x00000000u, 0x00000001u, 0x00001000u, 0x00000002u, 0x00002000u, 0x08004020u, 0x04020004u, 0x02008010u, 0x01010008u, 0x00010000u, 0x00000010u, 0x00020000u, 0x00000020u}}, + {27u, 1u, {0x00000001u, 0x00000002u, 0x00000004u, 0x00002000u, 0x00001000u, 0x00004000u, 0x00000008u, 0x00010000u, 0x08008008u, 0x04010010u, 0x02020040u, 0x01040020u, 0x00040000u, 0x00000040u, 0x00080000u, 0x00000080u}}, + {27u, 2u, {0x00000000u, 0x00000001u, 0x00000002u, 0x00000004u, 0x00001000u, 0x00002000u, 0x00004000u, 0x00000008u, 0x08008008u, 0x04010010u, 0x02020040u, 0x01040020u, 0x00010000u, 0x00000040u, 0x00040000u, 0x00000080u}}, + {27u, 4u, {0x00000000u, 0x00000000u, 0x00000001u, 0x00000002u, 0x00001000u, 0x00002000u, 0x00004000u, 0x00000004u, 0x08008008u, 0x04010010u, 0x02020040u, 0x01040020u, 0x00008000u, 0x00000010u, 0x00040000u, 0x00000040u}}, + {27u, 8u, {0x00000000u, 0x00000000u, 0x00000000u, 0x00000001u, 0x00001000u, 0x00000002u, 0x00000004u, 0x00002000u, 0x08008008u, 0x04010010u, 0x02020040u, 0x01040020u, 0x00004000u, 0x00000008u, 0x00010000u, 0x00000040u}}, + {27u, 16u, {0x00000000u, 0x00000000u, 0x00000000u, 0x00000000u, 0x00000001u, 0x00001000u, 0x00000002u, 0x00002000u, 0x08008008u, 0x04010010u, 0x02020040u, 0x01040020u, 0x00004000u, 0x00000004u, 0x00008000u, 0x00000010u}}, + // Thick (3D) standard swizzles SW_4KB_S and SW_64KB_S use the S3 patterns, keyed 0x100 | SW_MODE. + {0x105u, 1u, {0x00000001u, 0x00000002u, 0x01000000u, 0x00001000u, 0x02000000u, 0x00002000u, 0x00000004u, 0x04000000u, 0x00004000u, 0x00000008u, 0x08000000u, 0x00008000u, 0x00000000u, 0x00000000u, 0x00000000u, 0x00000000u}}, + {0x105u, 2u, {0x00000000u, 0x00000001u, 0x01000000u, 0x00001000u, 0x02000000u, 0x00002000u, 0x00000002u, 0x04000000u, 0x00004000u, 0x00000004u, 0x08000000u, 0x00008000u, 0x00000000u, 0x00000000u, 0x00000000u, 0x00000000u}}, + {0x105u, 4u, {0x00000000u, 0x00000000u, 0x00000001u, 0x00001000u, 0x01000000u, 0x00002000u, 0x00000002u, 0x02000000u, 0x00004000u, 0x00000004u, 0x04000000u, 0x00008000u, 0x00000000u, 0x00000000u, 0x00000000u, 0x00000000u}}, + {0x105u, 8u, {0x00000000u, 0x00000000u, 0x00000000u, 0x00000001u, 0x01000000u, 0x00001000u, 0x00000002u, 0x02000000u, 0x00002000u, 0x00000004u, 0x04000000u, 0x00004000u, 0x00000000u, 0x00000000u, 0x00000000u, 0x00000000u}}, + {0x105u, 16u, {0x00000000u, 0x00000000u, 0x00000000u, 0x00000000u, 0x01000000u, 0x00001000u, 0x00000001u, 0x02000000u, 0x00002000u, 0x00000002u, 0x04000000u, 0x00004000u, 0x00000000u, 0x00000000u, 0x00000000u, 0x00000000u}}, + {0x109u, 1u, {0x00000001u, 0x00000002u, 0x01000000u, 0x00001000u, 0x02000000u, 0x00002000u, 0x00000004u, 0x04000000u, 0x00004000u, 0x00000008u, 0x08000000u, 0x00008000u, 0x00000010u, 0x10000000u, 0x00010000u, 0x00000020u}}, + {0x109u, 2u, {0x00000000u, 0x00000001u, 0x01000000u, 0x00001000u, 0x02000000u, 0x00002000u, 0x00000002u, 0x04000000u, 0x00004000u, 0x00000004u, 0x08000000u, 0x00008000u, 0x00000008u, 0x10000000u, 0x00010000u, 0x00000010u}}, + {0x109u, 4u, {0x00000000u, 0x00000000u, 0x00000001u, 0x00001000u, 0x01000000u, 0x00002000u, 0x00000002u, 0x02000000u, 0x00004000u, 0x00000004u, 0x04000000u, 0x00008000u, 0x00000008u, 0x08000000u, 0x00010000u, 0x00000010u}}, + {0x109u, 8u, {0x00000000u, 0x00000000u, 0x00000000u, 0x00000001u, 0x01000000u, 0x00001000u, 0x00000002u, 0x02000000u, 0x00002000u, 0x00000004u, 0x04000000u, 0x00004000u, 0x00000008u, 0x08000000u, 0x00008000u, 0x00000010u}}, + {0x109u, 16u, {0x00000000u, 0x00000000u, 0x00000000u, 0x00000000u, 0x01000000u, 0x00001000u, 0x00000001u, 0x02000000u, 0x00002000u, 0x00000002u, 0x04000000u, 0x00004000u, 0x00000004u, 0x08000000u, 0x00008000u, 0x00000008u}}, +}; + +inline const TextureSwizzleEquation* FindTextureSwizzleEquation(std::uint32_t swizzleMode, std::uint32_t elementBytes) { + for (const auto& equation : kTextureSwizzleEquations) { + if (equation.swizzleMode == swizzleMode && equation.elementBytes == elementBytes) return &equation; + } + return nullptr; +} + +} + +#endif diff --git a/core/libs/prx/libSceAgcDriver/Graphics/include/TextureTiling.hpp b/core/libs/prx/libSceAgcDriver/Graphics/include/TextureTiling.hpp index fbfe21b38..337214291 100644 --- a/core/libs/prx/libSceAgcDriver/Graphics/include/TextureTiling.hpp +++ b/core/libs/prx/libSceAgcDriver/Graphics/include/TextureTiling.hpp @@ -2,6 +2,7 @@ #define CORE_LIBS_PRX_LIBSCEAGCDRIVER_GRAPHICS_INCLUDE_TEXTURETILING_HPP #include "prx/libSceAgcDriver/Graphics/include/GuestTextureResource.hpp" +#include #include #include @@ -23,6 +24,43 @@ struct TileMipLayout { std::vector ComputeMipLayout(TextureTileMode tileMode, std::uint32_t format, std::uint32_t width, std::uint32_t height, std::uint32_t mipCount); std::uint64_t ComputeSurfaceSize(const std::vector& mips, std::uint32_t arrayLayers); +// Mip chain of an uncompressed surface given its element size (color targets have no texture format). +std::vector ComputeElementMipLayout(TextureTileMode tileMode, std::uint32_t bytesPerElement, std::uint32_t width, std::uint32_t height, std::uint32_t mipCount); + +// Thick (3D) SW_4KB_S / SW_64KB_S blocks span several depth slices: {width, height, depth} in elements. +std::array ThickBlockExtent(TextureTileMode tileMode, std::uint32_t bytesPerElement); + +// Single-mip 3D surface. Each depth slice z is detiled from slab z / blockDepth (slabBytes apart) with the +// swizzle's slice input set to z, into linear slices sliceLinearBytes apart. +struct ThickLayout { + TileMipLayout mip; + std::uint32_t depth; + std::uint32_t blockDepth; + std::uint64_t slabBytes; + std::uint64_t sliceLinearBytes; + std::uint64_t guestBytes; +}; +ThickLayout ComputeThickLayout(TextureTileMode tileMode, std::uint32_t format, std::uint32_t width, std::uint32_t height, std::uint32_t depth); + +// Guest surface as the texture upload/write-back loops walk it: array layers of thin surfaces, or the +// depth slices of a volume (which Vulkan holds as one layer with extent depth). +struct SurfaceGeometry { + std::vector mips; + std::uint32_t layers = 1; + std::uint32_t imageLayers = 1; + std::uint32_t imageDepth = 1; + std::uint64_t guestBytes = 0; + std::uint64_t sliceLinearBytes = 0; + bool thick = false; + std::uint32_t blockDepth = 1; + std::uint64_t layerBytes = 0; + + std::uint64_t GuestLayerOffset(std::uint32_t layer) const { return thick ? static_cast(layer / blockDepth) * layerBytes : static_cast(layer) * layerBytes; } + std::uint64_t LinearLayerOffset(std::uint32_t layer) const { return static_cast(layer) * sliceLinearBytes; } + std::uint32_t CopyLayer(std::uint32_t layer) const { return imageDepth > 1 ? 0u : layer; } + std::int32_t CopyDepth(std::uint32_t layer) const { return imageDepth > 1 ? static_cast(layer) : 0; } +}; +SurfaceGeometry DescribeSurface(const GuestTextureResource& descriptor); } diff --git a/core/libs/prx/libSceAgcDriver/Graphics/shaders/ColorTransfer.comp b/core/libs/prx/libSceAgcDriver/Graphics/shaders/ColorTransfer.comp index 30d6f3238..d9ae43c66 100644 --- a/core/libs/prx/libSceAgcDriver/Graphics/shaders/ColorTransfer.comp +++ b/core/libs/prx/libSceAgcDriver/Graphics/shaders/ColorTransfer.comp @@ -22,6 +22,8 @@ void main() { tiledIndex = ((y / 128u) * parameters.blocksPerRow + x / 128u) * 16384u + offset / 4u; } uint pixel = (parameters.mode & 1u) != 0u ? linear[linearIndex] : tiled[tiledIndex]; + // Mode 8: A2B10G10R10 source (red in the low bits) presented as 8-bit RGBA. + if ((parameters.mode & 8u) != 0u) pixel = ((pixel >> 2u) & 0xffu) | (((pixel >> 12u) & 0xffu) << 8u) | (((pixel >> 22u) & 0xffu) << 16u) | 0xff000000u; if ((parameters.mode & 2u) != 0u) pixel = (pixel & 0xff00ff00u) | ((pixel & 0xffu) << 16u) | ((pixel >> 16u) & 0xffu); if ((parameters.mode & 1u) != 0u) tiled[tiledIndex] = pixel; else linear[linearIndex] = pixel; diff --git a/core/libs/prx/libSceAgcDriver/Graphics/shaders/TextureDetile.comp b/core/libs/prx/libSceAgcDriver/Graphics/shaders/TextureDetile.comp index 2c8d49d46..912df386d 100644 --- a/core/libs/prx/libSceAgcDriver/Graphics/shaders/TextureDetile.comp +++ b/core/libs/prx/libSceAgcDriver/Graphics/shaders/TextureDetile.comp @@ -3,6 +3,29 @@ layout(constant_id = 0) const uint ELEMENT_BYTES = 4u; layout(constant_id = 1) const uint BLOCK_BYTES = 256u; layout(constant_id = 2) const uint TILE_FAMILY = 0u; +// 0: tiled input -> linear output (detile); 1: linear input -> tiled output (retile). +layout(constant_id = 3) const uint RETILE = 0u; +// TILE_FAMILY 2: XOR swizzle given per byte-address bit as masks of x (bits 0-11), y (12-23) and +// slice (24-31) element-coordinate bits. +layout(constant_id = 4) const uint EQ0 = 0u; +layout(constant_id = 5) const uint EQ1 = 0u; +layout(constant_id = 6) const uint EQ2 = 0u; +layout(constant_id = 7) const uint EQ3 = 0u; +layout(constant_id = 8) const uint EQ4 = 0u; +layout(constant_id = 9) const uint EQ5 = 0u; +layout(constant_id = 10) const uint EQ6 = 0u; +layout(constant_id = 11) const uint EQ7 = 0u; +layout(constant_id = 12) const uint EQ8 = 0u; +layout(constant_id = 13) const uint EQ9 = 0u; +layout(constant_id = 14) const uint EQ10 = 0u; +layout(constant_id = 15) const uint EQ11 = 0u; +layout(constant_id = 16) const uint EQ12 = 0u; +layout(constant_id = 17) const uint EQ13 = 0u; +layout(constant_id = 18) const uint EQ14 = 0u; +layout(constant_id = 19) const uint EQ15 = 0u; +// Nonzero: block extent in elements (thick 3D blocks are not the thin 2D shape derived below). +layout(constant_id = 20) const uint BLOCK_WIDTH = 0u; +layout(constant_id = 21) const uint BLOCK_HEIGHT = 0u; layout(local_size_x = 8, local_size_y = 8) in; @@ -20,9 +43,23 @@ layout(push_constant) uniform Push { uint tailX; uint tailY; uint elementBytes; - uint arrayLayer; + uint slice; } params; +uint equationBit(uint mask, uvec2 p) { + uint selected = (p.x & (mask & 0xfffu)) ^ ((p.y << 12) & (mask & 0xfff000u)) ^ ((params.slice << 24) & (mask & 0xff000000u)); + return bitCount(selected) & 1u; +} + +uint equationOffset(uvec2 p) { + const uint eq[16] = uint[16](EQ0, EQ1, EQ2, EQ3, EQ4, EQ5, EQ6, EQ7, EQ8, EQ9, EQ10, EQ11, EQ12, EQ13, EQ14, EQ15); + uint offset = 0u; + for (uint bit = 0u; bit < 16u; ++bit) { + offset |= equationBit(eq[bit], p) << bit; + } + return offset; +} + uint standardOffset(uint x, uint y) { switch (ELEMENT_BYTES) { case 1u: @@ -67,76 +104,9 @@ uint standard64Extra(uint x, uint y) { } -uint renderTargetOffset(uint x, uint y) { - uint offset = 0u; - switch (ELEMENT_BYTES) { - case 1u: - offset ^= (y << 2u) & 0x0008u; - offset ^= (y << 4u) & 0x0010u; - offset ^= (y << 3u) & 0x00a0u; - offset ^= (y << 5u) & 0x0f00u; - offset ^= (y << 6u) & 0x1000u; - offset ^= (y << 7u) & 0x4000u; - offset ^= x & 0x0007u; - offset ^= (x << 3u) & 0x0040u; - offset ^= (x << 5u) & 0x0300u; - offset ^= (x << 4u) & 0x0400u; - offset ^= (x << 6u) & 0x0800u; - offset ^= (x << 7u) & 0x2000u; - offset ^= (x << 8u) & 0x8000u; - break; - case 2u: - offset ^= (y << 4u) & 0x0070u; - offset ^= (y << 5u) & 0x0f00u; - offset ^= (y << 8u) & 0x5000u; - offset ^= (x << 1u) & 0x000eu; - offset ^= (x << 4u) & 0x0480u; - offset ^= (x << 5u) & 0x0300u; - offset ^= (x << 6u) & 0x0800u; - offset ^= (x << 7u) & 0x2000u; - offset ^= (x << 8u) & 0x8000u; - break; - case 4u: - offset ^= (y << 4u) & 0x0070u; - offset ^= (y << 5u) & 0x0f00u; - offset ^= (y << 9u) & 0x1000u; - offset ^= (y << 8u) & 0x4000u; - offset ^= (x << 2u) & 0x000cu; - offset ^= (x << 5u) & 0x0380u; - offset ^= (x << 4u) & 0x0400u; - offset ^= (x << 6u) & 0x0800u; - offset ^= (x << 9u) & 0xa000u; - break; - case 8u: - offset ^= (y << 4u) & 0x0010u; - offset ^= (y << 6u) & 0x0080u; - offset ^= (y << 5u) & 0x0f00u; - offset ^= (y << 10u) & 0x5000u; - offset ^= (x << 3u) & 0x0008u; - offset ^= (x << 4u) & 0x0460u; - offset ^= (x << 5u) & 0x0300u; - offset ^= (x << 6u) & 0x0800u; - offset ^= (x << 10u) & 0x2000u; - offset ^= (x << 9u) & 0x8000u; - break; - case 16u: - offset ^= (x << 4u) & 0x0410u; - offset ^= (x << 5u) & 0x0340u; - offset ^= (x << 6u) & 0x0800u; - offset ^= (x << 11u) & 0xa000u; - offset ^= (y << 5u) & 0x0f20u; - offset ^= (y << 6u) & 0x0080u; - offset ^= (y << 10u) & 0x1000u; - offset ^= (y << 11u) & 0x4000u; - break; - } - uint z = params.arrayLayer; - return offset ^ ((z & 8u) << 5u) ^ ((z & 4u) << 7u) ^ ((z & 2u) << 9u) ^ ((z & 1u) << 11u); -} - uint blockOffset(uvec2 p) { if (TILE_FAMILY == 2u) { - return renderTargetOffset(p.x, p.y); + return equationOffset(p); } uint offset = standardOffset(p.x, p.y); if (BLOCK_BYTES > 4096u) { @@ -146,6 +116,9 @@ uint blockOffset(uvec2 p) { } uvec2 blockExtent() { + if (BLOCK_WIDTH != 0u) { + return uvec2(BLOCK_WIDTH, BLOCK_HEIGHT); + } uint width; if (BLOCK_BYTES <= 256u) { width = ELEMENT_BYTES <= 2u ? 16u : (ELEMENT_BYTES <= 8u ? 8u : 4u); @@ -197,6 +170,10 @@ void main() { uint blockIndex = block.y * params.blocksPerRow + block.x; uint tiledOffset = blockIndex * BLOCK_BYTES + blockOffset(swizzle); - copyElement(params.srcBase + tiledOffset, params.dstBase + linearOffset); + if (RETILE != 0u) { + copyElement(params.srcBase + linearOffset, params.dstBase + tiledOffset); + } else { + copyElement(params.srcBase + tiledOffset, params.dstBase + linearOffset); + } } diff --git a/core/libs/prx/libSceAgcDriver/Graphics/src/BdaResources.cpp b/core/libs/prx/libSceAgcDriver/Graphics/src/BdaResources.cpp index 94793d333..f9978ce2c 100644 --- a/core/libs/prx/libSceAgcDriver/Graphics/src/BdaResources.cpp +++ b/core/libs/prx/libSceAgcDriver/Graphics/src/BdaResources.cpp @@ -1,11 +1,90 @@ #include "prx/libSceAgcDriver/Graphics/include/BdaResources.hpp" +#include #include +#include +#include #include #include +#include +#include #include namespace AgcDriver::Graphics { +namespace { + +// The page tables built per device, reused by a build whose ranges (guest ranges, device +// addresses, permissions) equal one of them: consecutive address-based builds map the same registered +// set through the same imports and mirrors, so the ~1200-entry table repeats. The shader only reads +// the table (its lookup keeps its state in function variables), and a batch in flight keeps its +// BdaResources, so sharing the buffer is safe. The cache holds the tables weakly: they never outlive +// the builds that own them (a static owner would destroy the buffer after its device, at exit or on +// device replacement, and would fail the tests' leak check), and with deferred lease release the +// previous builds are still in flight when the next one asks, which is what makes the hit rate. +// One table was enough while every build repeated the last one exactly; now the misaligned +// descriptor sub-ranges the GPU copies out of imports sit in pool staging buffers whose device +// addresses cycle through the few buffers the batches in flight leave free, and the three +// address-based programs of one queue interleave with the lease draws, so a table differs from the +// previous one and comes back a few builds later (69% misses in the wave-8 run, 4% before the GPU +// copies). Hence several recent tables, most recently used first, matched by a word hash before +// the byte compare. APS5_BDA_TABLE_CACHE_ENTRIES sets how many (default 8; 1 is the previous +// behaviour); APS5_NO_BDA_TABLE_CACHE=1 builds every table. +struct TableEntry { + std::uint64_t hash; + std::vector ranges; + std::weak_ptr buffer; +}; + +struct TableCache { + std::mutex mutex; + VkDevice device = VK_NULL_HANDLE; + std::list entries; + std::uint64_t hits = 0; + std::uint64_t misses = 0; +}; + +TableCache& Tables() { + static TableCache cache; + return cache; +} + +bool tableCacheEnabled() { + static const bool disabled = std::getenv("APS5_NO_BDA_TABLE_CACHE") != nullptr; + return !disabled; +} + +std::size_t tableCacheEntries() { + static const std::size_t entries = [] { + const char* value = std::getenv("APS5_BDA_TABLE_CACHE_ENTRIES"); + const auto parsed = value != nullptr ? std::strtoull(value, nullptr, 10) : 8ull; + return static_cast(std::clamp(parsed, 1, 64)); + }(); + return entries; +} + +bool sameRanges(const std::vector& left, const std::vector& right) { + return left.size() == right.size() && (left.empty() || std::memcmp(left.data(), right.data(), left.size() * sizeof(left.front())) == 0); +} + +// A 64-bit mix of the table's words: a table is ~1200 ranges of four words, and comparing the bytes +// against every held table would cost as much as building it. +std::uint64_t hashRanges(const std::vector& ranges) { + static_assert(sizeof(ShaderRecompiler::BdaAbi::Range) % sizeof(std::uint64_t) == 0); + std::uint64_t hash = 0x9e3779b97f4a7c15ull ^ ranges.size(); + for (const auto& range : ranges) { + std::uint64_t words[sizeof(range) / sizeof(std::uint64_t)]; + std::memcpy(words, &range, sizeof(range)); + for (const auto word : words) { + hash ^= word; + hash *= 0xff51afd7ed558ccdull; + hash ^= hash >> 33u; + } + } + return hash; +} + +} + BdaResources::BdaResources(const Context& context) { static_assert(std::endian::native == std::endian::little); Require(sizeof(ShaderRecompiler::BdaAbi::Fault) <= context.limits.maxStorageBufferRange, "BDA fault buffer exceeds storage buffer range limit"); @@ -14,15 +93,67 @@ BdaResources::BdaResources(const Context& context) { } BdaResources::BdaResources(const Context& context, const GuestBufferMemory& memory) : BdaResources(context) { - const auto ranges = memory.AddressRanges(); + auto ranges = memory.AddressRanges(); Require(ranges.size() <= std::numeric_limits::max(), "BDA table range count overflow"); Require(ranges.size() <= (std::numeric_limits::max() - sizeof(ShaderRecompiler::BdaAbi::Header)) / sizeof(ShaderRecompiler::BdaAbi::Range), "BDA table size overflow"); tableBytes = sizeof(ShaderRecompiler::BdaAbi::Header) + ranges.size() * sizeof(ShaderRecompiler::BdaAbi::Range); Require(tableBytes <= context.limits.maxStorageBufferRange && sizeof(ShaderRecompiler::BdaAbi::Fault) <= context.limits.maxStorageBufferRange, "BDA descriptors exceed storage buffer range limit"); - table = std::make_unique(context, tableBytes, VK_BUFFER_USAGE_STORAGE_BUFFER_BIT); + auto& cache = Tables(); + const auto hash = tableCacheEnabled() ? hashRanges(ranges) : 0; + if (tableCacheEnabled()) { + std::lock_guard lock(cache.mutex); + // Another device's tables are not this one's (their buffers are dead or foreign). + if (cache.device != context.device) { + cache.entries.clear(); + cache.device = context.device; + } + for (auto it = cache.entries.begin(); it != cache.entries.end();) { + // Compare the hash before taking a strong reference: locking a non-matching entry's + // buffer could make this thread its last owner (a concurrent reap releasing it) and + // run the pool release under the cache mutex; expired() takes no reference. + if (it->hash != hash) { + if (it->buffer.expired()) { + it = cache.entries.erase(it); + } else { + ++it; + } + continue; + } + auto shared = it->buffer.lock(); + if (shared == nullptr) { + // Every build that owned the table is gone, and its buffer with it. + it = cache.entries.erase(it); + continue; + } + if (sameRanges(it->ranges, ranges)) { + ++cache.hits; + table = std::move(shared); + cache.entries.splice(cache.entries.begin(), cache.entries, it); + return; + } + ++it; + } + } + table = std::make_shared(context, tableBytes, VK_BUFFER_USAGE_STORAGE_BUFFER_BIT); const ShaderRecompiler::BdaAbi::Header header{ShaderRecompiler::BdaAbi::Version, static_cast(ranges.size()), sizeof(ShaderRecompiler::BdaAbi::Range), 0}; std::memcpy(table->Bytes().data(), &header, sizeof(header)); if (!ranges.empty()) std::memcpy(table->Bytes().data() + sizeof(header), ranges.data(), ranges.size() * sizeof(ranges.front())); + if (tableCacheEnabled()) { + std::lock_guard lock(cache.mutex); + ++cache.misses; + if (cache.device != context.device) { + cache.entries.clear(); + cache.device = context.device; + } + cache.entries.push_front({hash, std::move(ranges), table}); + while (cache.entries.size() > tableCacheEntries()) cache.entries.pop_back(); + } +} + +BdaResources::TableCacheStats BdaResources::TableCacheCounters() { + auto& cache = Tables(); + std::lock_guard lock(cache.mutex); + return {cache.hits, cache.misses, cache.entries.size()}; } VkDescriptorBufferInfo BdaResources::Table() const { @@ -43,6 +174,12 @@ void BdaResources::CheckFault() const { } Require(report.state == ShaderRecompiler::BdaAbi::FaultState::Ready && report.reserved == 0, "incomplete or invalid BDA fault record"); Require(report.reason != ShaderRecompiler::BdaAbi::FaultReason::InvalidRectangle, "rect-list requires finite nondegenerate axis-aligned positions with equal positive W"); + if (report.reason == ShaderRecompiler::BdaAbi::FaultReason::LoopLimit) { + // APS5_LOOP_GUARD: the shader left a loop that ran past the guard; the dispatch result is kept. + std::fprintf(stderr, "[gpu] loop guard: the loop exit at pc 0x%x ran past %u evaluations\n", report.instruction, report.bytes); + std::memset(fault->Bytes().data(), 0, fault->Bytes().size()); + return; + } std::ostringstream message; message << "BDA access failed: address=0x" << std::hex << report.address << " instruction=0x" << report.instruction << std::dec << " bytes=" << report.bytes << " stage=" << report.stage << " reason=" << static_cast(report.reason); throw std::runtime_error(message.str()); diff --git a/core/libs/prx/libSceAgcDriver/Graphics/src/BufferPool.cpp b/core/libs/prx/libSceAgcDriver/Graphics/src/BufferPool.cpp index 6c4477baf..b3896cc46 100644 --- a/core/libs/prx/libSceAgcDriver/Graphics/src/BufferPool.cpp +++ b/core/libs/prx/libSceAgcDriver/Graphics/src/BufferPool.cpp @@ -1,74 +1,118 @@ #include "prx/libSceAgcDriver/Graphics/include/BufferPool.hpp" +#include +#include +#include +#include +#include namespace AgcDriver::Graphics { +namespace { + +// APS5_BUFFER_POOL_SHARED=1: one tier for every size, as before the split. +bool sharedTiers() { + static const bool shared = std::getenv("APS5_BUFFER_POOL_SHARED") != nullptr; + return shared; +} + +} + BufferPool::BufferPool(const Context& context) : device(context.device), unmap(context.Function("vkUnmapMemory")), destroyBuffer(context.Function("vkDestroyBuffer")), freeMemory(context.Function("vkFreeMemory")) { - buckets.fill(none); - for (std::size_t index = 0; index < capacity; ++index) freeSlots[index] = index; + smallTier.budget = smallBudget; + largeTier.budget = budget; } BufferPool::~BufferPool() { - for (const auto& slot : slots) { - if (slot.allocation) destroy(*slot.allocation); - } -} - -std::size_t BufferPool::bucket(std::size_t bytes, VkBufferUsageFlags usage, VkMemoryPropertyFlags properties) noexcept { - auto hash = bytes ^ (bytes >> 16u); - hash = hash * 16777619u ^ usage; - hash = hash * 16777619u ^ properties; - return (hash ^ (hash >> 8u) ^ (hash >> 16u)) % bucketCount; -} - -BufferAllocation BufferPool::remove(std::size_t index) noexcept { - auto& slot = slots[index]; - const auto allocation = *slot.allocation; - auto* link = &buckets[bucket(allocation.bytes, allocation.usage, allocation.properties)]; - while (*link != index) link = &slots[*link].next; - *link = slot.next; - if (slot.older != none) slots[slot.older].newer = slot.newer; - else oldest = slot.newer; - if (slot.newer != none) slots[slot.newer].older = slot.older; - else newest = slot.older; - retainedBytes -= allocation.allocationBytes; - slot = Slot{}; - freeSlots[freeCount++] = index; - return allocation; + for (const auto& slot : smallTier.free) destroy(slot.allocation); + for (const auto& slot : largeTier.free) destroy(slot.allocation); } void BufferPool::destroy(const BufferAllocation& allocation) noexcept { + // Device-local allocations (see DeviceBuffer) are never mapped. if (allocation.mapping != nullptr) unmap(device, allocation.memory); destroyBuffer(device, allocation.buffer, nullptr); freeMemory(device, allocation.memory, nullptr); } +std::size_t BufferPool::Capacity(std::size_t bytes) { + static const bool exact = std::getenv("APS5_NO_BUFFER_CLASSES") != nullptr; + constexpr std::size_t smallest = 256; + if (exact || bytes >= classLimit) return bytes; + return std::max(smallest, std::bit_ceil(bytes)); +} + +BufferPool::Tier& BufferPool::tierFor(std::size_t capacity) { + return !sharedTiers() && capacity < classLimit ? smallTier : largeTier; +} + std::optional BufferPool::Take(std::size_t bytes, VkBufferUsageFlags usage, VkMemoryPropertyFlags properties) { + static const bool profile = std::getenv("APS5_PROFILE_DRAW") != nullptr; + const auto capacity = Capacity(bytes); std::lock_guard lock(mutex); - for (auto index = buckets[bucket(bytes, usage, properties)]; index != none; index = slots[index].next) { - const auto& allocation = *slots[index].allocation; - if (allocation.bytes == bytes && allocation.usage == usage && allocation.properties == properties) return remove(index); + ++clock; + if (profile) { + static auto lastReport = std::chrono::steady_clock::now(); + const auto now = std::chrono::steady_clock::now(); + if (now - lastReport > std::chrono::seconds(10)) { + lastReport = now; + std::fprintf(stderr, "[bufferpool] small: %llu hits, %llu misses, %llu evictions, %zu retained (%.0f MiB); large: %llu hits, %llu misses, %llu evictions, %zu retained (%.0f MiB)%s\n", static_cast(smallTier.hits), static_cast(smallTier.misses), static_cast(smallTier.evictions), smallTier.free.size(), smallTier.retainedBytes / 1048576.0, static_cast(largeTier.hits), static_cast(largeTier.misses), static_cast(largeTier.evictions), largeTier.free.size(), largeTier.retainedBytes / 1048576.0, sharedTiers() ? " (shared)" : ""); + } } + auto& tier = tierFor(capacity); + for (auto it = tier.free.begin(); it != tier.free.end(); ++it) { + const auto& allocation = it->allocation; + if (allocation.bytes != capacity || allocation.usage != usage || allocation.properties != properties) continue; + auto result = allocation; + tier.retainedBytes -= allocation.allocationBytes; + *it = std::move(tier.free.back()); + tier.free.pop_back(); + ++tier.hits; + return result; + } + ++tier.misses; return std::nullopt; } +void BufferPool::evictOldest(Tier& tier, std::vector& evicted) { + const auto oldest = std::min_element(tier.free.begin(), tier.free.end(), [](const Slot& left, const Slot& right) { return left.lastUse < right.lastUse; }); + // First: a throw here leaves the slot retained and counted. + evicted.push_back(oldest->allocation); + tier.retainedBytes -= oldest->allocation.allocationBytes; + *oldest = std::move(tier.free.back()); + tier.free.pop_back(); + ++tier.evictions; +} + +std::size_t BufferPool::MaxSlots() { + // APS5_BUFFER_POOL_SLOTS= bounds the retained allocations; 64 is the capacity the pool had + // before least-recently-used retention, for A/B runs. + static const std::size_t slots = [] { + const char* value = std::getenv("APS5_BUFFER_POOL_SLOTS"); + const auto parsed = value != nullptr ? std::strtoull(value, nullptr, 10) : 0; + return parsed != 0 ? static_cast(parsed) : defaultSlots; + }(); + return slots; +} + void BufferPool::Put(const BufferAllocation& allocation) noexcept { - std::lock_guard lock(mutex); - if (allocation.allocationBytes > budget) { + // Evicted allocations are destroyed after the mutex is released (see evictOldest). The vector + // may throw on growth; a Put that cannot retain simply destroys, as noexcept requires. + std::vector evicted; + try { + std::lock_guard lock(mutex); + auto& tier = tierFor(allocation.bytes); + if (allocation.allocationBytes > tier.budget) { + evicted.push_back(allocation); + } else { + const auto maxSlots = MaxSlots(); + while (!tier.free.empty() && (tier.retainedBytes + allocation.allocationBytes > tier.budget || tier.free.size() >= maxSlots)) evictOldest(tier, evicted); + tier.free.push_back({allocation, ++clock}); + tier.retainedBytes += allocation.allocationBytes; + } + } catch (...) { destroy(allocation); - return; } - while (retainedBytes > budget - allocation.allocationBytes || freeCount == 0) destroy(remove(oldest)); - const auto index = freeSlots[--freeCount]; - auto& slot = slots[index]; - auto& head = buckets[bucket(allocation.bytes, allocation.usage, allocation.properties)]; - slot.allocation = allocation; - slot.next = head; - slot.older = newest; - if (newest != none) slots[newest].newer = index; - else oldest = index; - newest = index; - head = index; - retainedBytes += allocation.allocationBytes; + for (const auto& gone : evicted) destroy(gone); } std::shared_ptr GetBufferPool(const Context& context) { diff --git a/core/libs/prx/libSceAgcDriver/Graphics/src/ColorTargetLayout.cpp b/core/libs/prx/libSceAgcDriver/Graphics/src/ColorTargetLayout.cpp index 563a39d2e..f9d4c41e9 100644 --- a/core/libs/prx/libSceAgcDriver/Graphics/src/ColorTargetLayout.cpp +++ b/core/libs/prx/libSceAgcDriver/Graphics/src/ColorTargetLayout.cpp @@ -1,4 +1,6 @@ #include "prx/libSceAgcDriver/Graphics/include/ColorTargetLayout.hpp" +#include "prx/libSceAgcDriver/Graphics/include/TextureSwizzleEquations.hpp" +#include #include #include #include @@ -10,9 +12,8 @@ void require(bool condition, const char* reason) { if (!condition) throw std::runtime_error(reason); } -std::uint32_t blockOffset(std::uint32_t x, std::uint32_t y) { - return ((y << 4u) & 0x0070u) ^ ((y << 5u) & 0x0f00u) ^ ((y << 9u) & 0x1000u) ^ ((y << 8u) & 0x4000u) - ^ ((x << 2u) & 0x000cu) ^ ((x << 5u) & 0x0380u) ^ ((x << 4u) & 0x0400u) ^ ((x << 6u) & 0x0800u) ^ ((x << 9u) & 0xa000u); +std::uint32_t parity(std::uint32_t value) { + return static_cast(std::popcount(value)) & 1u; } } @@ -26,28 +27,48 @@ ColorTileMode DecodeColorTileMode(std::uint32_t attrib3) { return static_cast(mode); } -ColorTargetLayout::ColorTargetLayout(std::uint32_t width, std::uint32_t height, ColorTileMode mode) : width(width), height(height), pitch(width), mode(mode), bytes(0) { +ColorTargetLayout::ColorTargetLayout(std::uint32_t width, std::uint32_t height, ColorTileMode mode, std::uint32_t bytesPerElement) : width(width), height(height), pitch(width), mode(mode), bytes(0), elementBytes(bytesPerElement) { require(width != 0 && height != 0 && width <= 16384 && height <= 16384, "AGC graphics: invalid color surface extent"); + require(std::has_single_bit(bytesPerElement) && bytesPerElement <= 16u, "AGC graphics: unsupported color element size"); std::uint32_t paddedHeight = height; switch (mode) { case ColorTileMode::Linear: - require(width % 64u == 0, "AGC graphics: linear surface pitch requires a width aligned to 256 bytes"); + require(width * bytesPerElement % 256u == 0, "AGC graphics: linear surface pitch requires a width aligned to 256 bytes"); break; - case ColorTileMode::RenderTarget: - pitch = (width + 127u) & ~127u; - paddedHeight = (height + 127u) & ~127u; + case ColorTileMode::RenderTarget: { + // SW_64KB_R_X: 64 KiB blocks of 2^(16 - log2(bpe)) elements, wider than tall for odd powers. + const auto log2Elements = 16u - static_cast(std::countr_zero(bytesPerElement)); + blockWidth = 1u << ((log2Elements + 1u) / 2u); + blockHeight = 1u << (log2Elements / 2u); + pitch = (width + blockWidth - 1u) / blockWidth * blockWidth; + paddedHeight = (height + blockHeight - 1u) / blockHeight * blockHeight; + const auto* equation = FindTextureSwizzleEquation(27u, bytesPerElement); + require(equation != nullptr, "AGC graphics: no SW_64KB_R_X equation for the color element size"); + xOffsets.resize(blockWidth); + yOffsets.resize(blockHeight); + for (std::uint32_t x = 0; x < blockWidth; ++x) { + std::uint32_t offset = 0; + for (std::uint32_t bit = 0; bit < 16u; ++bit) offset |= parity(x & equation->bits[bit] & 0xfffu) << bit; + xOffsets[x] = offset; + } + for (std::uint32_t y = 0; y < blockHeight; ++y) { + std::uint32_t offset = 0; + for (std::uint32_t bit = 0; bit < 16u; ++bit) offset |= parity((y << 12u) & equation->bits[bit] & 0xfff000u) << bit; + yOffsets[y] = offset; + } break; + } default: throw std::runtime_error("AGC graphics: unsupported color tile mode"); } - const auto size = static_cast(pitch) * paddedHeight * 4u; + const auto size = static_cast(pitch) * paddedHeight * bytesPerElement; require(size <= std::numeric_limits::max(), "AGC graphics: color surface size overflow"); bytes = static_cast(size); } std::size_t ColorTargetLayout::offset(std::uint32_t x, std::uint32_t y) const { - if (mode == ColorTileMode::Linear) return (static_cast(y) * pitch + x) * 4u; - const auto block = static_cast(y / 128u) * (pitch / 128u) + x / 128u; - return block * 65536u + blockOffset(x, y); + if (mode == ColorTileMode::Linear) return (static_cast(y) * pitch + x) * elementBytes; + const auto block = static_cast(y / blockHeight) * (pitch / blockWidth) + x / blockWidth; + return block * 65536u + (xOffsets[x % blockWidth] ^ yOffsets[y % blockHeight]); } std::size_t ColorTargetLayout::Offset(std::uint32_t x, std::uint32_t y) const { @@ -58,18 +79,16 @@ std::size_t ColorTargetLayout::Offset(std::uint32_t x, std::uint32_t y) const { void ColorTargetLayout::Detile(std::span source, std::span destination) const { require(source.size() == Bytes() && destination.size() == LinearBytes(), "AGC graphics: color detile buffer size mismatch"); for (std::uint32_t y = 0; y < height; ++y) { - for (std::uint32_t x = 0; x < width; ++x) { - std::memcpy(destination.data() + (static_cast(y) * width + x) * 4u, source.data() + offset(x, y), 4); - } + auto* row = destination.data() + static_cast(y) * width * elementBytes; + for (std::uint32_t x = 0; x < width; ++x) std::memcpy(row + static_cast(x) * elementBytes, source.data() + offset(x, y), elementBytes); } } void ColorTargetLayout::Tile(std::span source, std::span destination) const { require(source.size() == LinearBytes() && destination.size() == Bytes(), "AGC graphics: color tile buffer size mismatch"); for (std::uint32_t y = 0; y < height; ++y) { - for (std::uint32_t x = 0; x < width; ++x) { - std::memcpy(destination.data() + offset(x, y), source.data() + (static_cast(y) * width + x) * 4u, 4); - } + const auto* row = source.data() + static_cast(y) * width * elementBytes; + for (std::uint32_t x = 0; x < width; ++x) std::memcpy(destination.data() + offset(x, y), row + static_cast(x) * elementBytes, elementBytes); } } diff --git a/core/libs/prx/libSceAgcDriver/Graphics/src/ColorTargetTransfer.cpp b/core/libs/prx/libSceAgcDriver/Graphics/src/ColorTargetTransfer.cpp index 0b3003e57..9d9d7d346 100644 --- a/core/libs/prx/libSceAgcDriver/Graphics/src/ColorTargetTransfer.cpp +++ b/core/libs/prx/libSceAgcDriver/Graphics/src/ColorTargetTransfer.cpp @@ -7,7 +7,7 @@ namespace AgcDriver::Graphics { void ReadColorTarget(const ColorTarget& target, std::span destination) { PerformanceTimer timing("ColorTarget.Read"); - const ColorTargetLayout layout(target.extent.width, target.extent.height, target.tileMode); + const ColorTargetLayout layout(target.extent.width, target.extent.height, target.tileMode, target.elementBytes); Require(target.bytes == layout.Bytes() && destination.size() == layout.LinearBytes(), "color target transfer size mismatch"); if (target.tileMode == ColorTileMode::Linear) { GuestMemory::Read(target.address, destination, layout.Alignment()); @@ -23,7 +23,7 @@ void ReadColorTarget(const ColorTarget& target, std::span destination void WriteColorTarget(const ColorTarget& target, std::span source) { PerformanceTimer timing("ColorTarget.Write"); - const ColorTargetLayout layout(target.extent.width, target.extent.height, target.tileMode); + const ColorTargetLayout layout(target.extent.width, target.extent.height, target.tileMode, target.elementBytes); Require(target.bytes == layout.Bytes() && source.size() == layout.LinearBytes(), "color target transfer size mismatch"); if (target.tileMode == ColorTileMode::Linear) { GuestMemory::Write(target.address, source, layout.Alignment()); diff --git a/core/libs/prx/libSceAgcDriver/Graphics/src/DccMetadata.cpp b/core/libs/prx/libSceAgcDriver/Graphics/src/DccMetadata.cpp new file mode 100644 index 000000000..4693139ac --- /dev/null +++ b/core/libs/prx/libSceAgcDriver/Graphics/src/DccMetadata.cpp @@ -0,0 +1,433 @@ +#include "prx/libSceAgcDriver/Graphics/include/DccMetadata.hpp" +#include "prx/libSceAgcDriver/Execution/include/GuestMemory.hpp" +#include "prx/libSceAgcDriver/Graphics/include/GuestBufferMemory.hpp" +#include "prx/libSceAgcDriver/Graphics/include/Recorder.hpp" +#include "prx/libSceAgcDriver/Graphics/include/Resources.hpp" +#include "prx/libSceAgcDriver/Graphics/include/TextureFormat.hpp" +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include + +namespace AgcDriver::Graphics { +namespace { + +constexpr std::uint64_t KeyBytes = 256; + +// Whether every key in [keys, keys + count) equals `first`. Eight keys per compare: the scan runs +// on every texture and storage validation, and an all-0xff (uncompressed) surface's keys are read +// to the end each time (132 KiB for a 4K RGBA8 target). APS5_NO_DCC_WORD_SCAN=1 restores the byte loop. +bool AllKeysEqual(const std::uint8_t* keys, std::size_t count, std::uint8_t first) { + static const bool byteScan = std::getenv("APS5_NO_DCC_WORD_SCAN") != nullptr; + if (byteScan) return std::all_of(keys, keys + count, [&](std::uint8_t key) { return key == first; }); + const std::uint64_t pattern = 0x0101010101010101ull * first; + std::size_t i = 0; + for (; i + 8 <= count; i += 8) { + std::uint64_t word; + std::memcpy(&word, keys + i, sizeof(word)); + if (word != pattern) return false; + } + for (; i < count; ++i) { + if (keys[i] != first) return false; + } + return true; +} + +// APS5_PROFILE_DRAW: key scans, their bytes and time, printed as [dcc] every 10 s, with the hits of +// the GPU-store memo below and the "uncompressed" key stores by where they were made. Flush syncs +// stay 0 (no key read syncs the recorder); the field is kept so the line's shape does not change. +struct ScanProfile { + std::atomic scans{0}; + std::atomic bytes{0}; + std::atomic nanoseconds{0}; + std::atomic memoHits{0}; + std::atomic flushSyncs{0}; + std::atomic gpuStores{0}; + // GPU stores whose range had a 4-byte unaligned head or tail (copied instead of filled). + std::atomic gpuSplitStores{0}; + std::atomic cpuStores{0}; + std::atomic lastReport{0}; +}; + +ScanProfile& Scans() { + static ScanProfile profile; + return profile; +} + +bool ScanProfileEnabled() { + static const bool profile = std::getenv("APS5_PROFILE_DRAW") != nullptr; + return profile; +} + +void ReportScans(std::chrono::steady_clock::time_point now) { + auto& profile = Scans(); + const auto nowMs = std::chrono::duration_cast(now.time_since_epoch()).count(); + auto last = profile.lastReport.load(); + // The first event starts the period rather than reporting an empty one. + if (last == 0) { + profile.lastReport.compare_exchange_strong(last, nowMs); + return; + } + if (nowMs - last < 10000 || !profile.lastReport.compare_exchange_strong(last, nowMs)) return; + std::fprintf(stderr, "[dcc] %llu scans, %.1f MiB scanned, %llu memo hits, %llu flush syncs, %.1f ms; uncompressed keys stored: %llu on the GPU (%llu with an unaligned head or tail), %llu on the CPU\n", static_cast(profile.scans.load()), profile.bytes.load() / 1048576.0, static_cast(profile.memoHits.load()), static_cast(profile.flushSyncs.load()), profile.nanoseconds.load() / 1e6, static_cast(profile.gpuStores.load()), static_cast(profile.gpuSplitStores.load()), static_cast(profile.cpuStores.load())); +} + +void CountScan(std::size_t bytes, std::chrono::steady_clock::time_point start) { + auto& profile = Scans(); + const auto now = std::chrono::steady_clock::now(); + profile.scans.fetch_add(1, std::memory_order_relaxed); + profile.bytes.fetch_add(bytes, std::memory_order_relaxed); + profile.nanoseconds.fetch_add(static_cast(std::chrono::duration_cast(now - start).count()), std::memory_order_relaxed); + ReportScans(now); +} + +void CountStore(std::atomic& counter) { + if (!ScanProfileEnabled()) return; + counter.fetch_add(1, std::memory_order_relaxed); + ReportScans(std::chrono::steady_clock::now()); +} + +// "Uncompressed" keys the driver stored on the GPU (MarkDccUncompressed's recorded fill) whose batch +// may not have run yet: the bytes ReadDccKeys scans would still show the keys from before the store +// (a fast-clear code), and a storage image whose write-back made the store would be re-uploaded as +// cleared. While recorded work that writes the range is still running, the range reads as +// uncompressed from here; the bytes are trusted again only once the newest recorded writer of the +// range has signaled its fence (batches of the recorder's queue finish in order, so the fill and +// every writer up to that one have landed and the bytes are final, whichever batch wrote last). +// Dropping an entry on a later batch's serial alone would not do: until the fill's batch runs the +// bytes PREDATE the fill (the previous clear code), and a later writer that is not a same-code clear +// (a decompress kernel, a partial key write, a false overlap from an unrelated V#) would make the +// next Refresh compare that stale code against the uploaded keys. The lookup runs with or without +// GuestMemory::GpuMutex (a resource build's stage A reads keys unlocked, its stage B re-reads them +// under the lock): unlocked it can only ask the lock-free snapshot whether the range is written at +// all. Entries belong to one recorder: when the active recorder changes (device replacement; a new +// recorder can reuse the old one's address) they are all dropped. Its mutex is a leaf (nothing is +// taken under it). +struct GpuKeyStore { + std::uint64_t begin; + std::uint64_t end; +}; + +struct KeyStoreMemo { + std::mutex mutex; + std::vector entries; + const Recorder* recorder = nullptr; +}; + +KeyStoreMemo& Memo() { + static KeyStoreMemo memo; + return memo; +} + +// Under memo.mutex: the active recorder, with the entries of any other recorder dropped. +Recorder* MemoRecorder(KeyStoreMemo& memo) { + auto* active = Recorder::Active(); + if (active != memo.recorder) { + memo.entries.clear(); + memo.recorder = active; + } + return active; +} + +void ForgetStores(std::uint64_t begin, std::uint64_t end) { + auto& memo = Memo(); + std::lock_guard lock(memo.mutex); + std::erase_if(memo.entries, [&](const GpuKeyStore& store) { return begin < store.end && store.begin < end; }); +} + +// Uncompressed when a GPU store still pending covers the whole range, else nullopt (the bytes decide). +std::optional MemoizedKeys(std::uint64_t begin, std::uint64_t end) { + auto& memo = Memo(); + std::lock_guard lock(memo.mutex); + auto* recorder = MemoRecorder(memo); + for (auto it = memo.entries.begin(); it != memo.entries.end(); ++it) { + if (it->begin > begin || it->end < end) continue; + const auto bytes = static_cast(it->end - it->begin); + bool pending = false; + if (recorder != nullptr) { + if (GuestMemory::GpuMutex().HeldByThisThread()) { + // The open batch is never signaled (also with APS5_NO_SYNC_THROUGH, where the info + // carries the open flag with the newest in-flight batch's fence state). + const auto info = recorder->DescribePendingWrite(it->begin, bytes); + pending = info.has_value() && !info->signaled; + } else { + pending = Recorder::SnapshotWriteOverlaps(it->begin, bytes); + } + } + if (!pending) { + memo.entries.erase(it); + return std::nullopt; + } + if (ScanProfileEnabled()) Scans().memoHits.fetch_add(1, std::memory_order_relaxed); + return DccKeys::Uncompressed; + } + return std::nullopt; +} + +// The 0xff keys as a fill recorded into the active recorder's open batch, into the range's host +// import (the WriteLabelOnGpu / FillBuffer pattern): ordered behind every earlier recorded read or +// write of the range (the title's DCC clear or decompress kernel storing keys through a V#, which the +// CPU store had to wait for) and visible to the work recorded after it and to the host. vkCmdFillBuffer +// needs a 4-byte aligned offset and size; an unaligned head or tail is copied from a kept 4-byte +// buffer of 0xff instead. Under GuestMemory::GpuMutex (it records). False when the keys cannot be +// stored this way (memory the GPU has no view of): the caller stores them on the CPU. +bool StoreUncompressedOnGpu(const Context& context, Recorder& recorder, std::uint64_t begin, std::size_t count) { + const auto* import = HostImportFor(context, begin, count); + if (import == nullptr) return false; + const VkDeviceSize first = begin - import->base; + const VkDeviceSize last = first + count; + const VkDeviceSize fillBegin = std::min((first + 3) & ~VkDeviceSize{3}, last); + const VkDeviceSize fillEnd = std::max(last & ~VkDeviceSize{3}, fillBegin); + // Everything that can throw (the seed's allocation, the batch's command buffer) comes before the + // first recorded command: a fill left in the batch without its note and memo entry would be an + // unordered write of the keys. + std::shared_ptr seed; + if (fillBegin > first || last > fillEnd) { + seed = std::make_shared(context, 4, VK_BUFFER_USAGE_TRANSFER_SRC_BIT); + std::memset(seed->Bytes().data(), 0xff, seed->Bytes().size()); + recorder.Keep(seed); + } + const auto commands = recorder.Commands(); + RecordMemoryBarrier(context, commands, VK_PIPELINE_STAGE_ALL_COMMANDS_BIT, VK_PIPELINE_STAGE_TRANSFER_BIT, VK_ACCESS_SHADER_WRITE_BIT | VK_ACCESS_TRANSFER_WRITE_BIT | VK_ACCESS_HOST_WRITE_BIT, VK_ACCESS_TRANSFER_WRITE_BIT | VK_ACCESS_TRANSFER_READ_BIT); + if (fillEnd > fillBegin) context.Function("vkCmdFillBuffer")(commands, import->buffer, fillBegin, fillEnd - fillBegin, 0xffffffffu); + if (seed != nullptr) { + VkBufferCopy copies[2]; + std::uint32_t copyCount = 0; + if (fillBegin > first) copies[copyCount++] = {0, first, fillBegin - first}; + if (last > fillEnd) copies[copyCount++] = {0, fillEnd, last - fillEnd}; + context.Function("vkCmdCopyBuffer")(commands, seed->Handle(), import->buffer, copyCount, copies); + CountStore(Scans().gpuSplitStores); + } + RecordMemoryBarrier(context, commands, VK_PIPELINE_STAGE_TRANSFER_BIT, VK_PIPELINE_STAGE_ALL_COMMANDS_BIT | VK_PIPELINE_STAGE_HOST_BIT, VK_ACCESS_TRANSFER_WRITE_BIT, VK_ACCESS_SHADER_READ_BIT | VK_ACCESS_SHADER_WRITE_BIT | VK_ACCESS_TRANSFER_READ_BIT | VK_ACCESS_HOST_READ_BIT); + recorder.NotePendingWrite(begin, count); + GuestMemory::MarkWritten(begin, count); + // After the note: an unlocked memo lookup drops an entry the snapshot does not cover yet. The + // entry is kept only for the active recorder (the one the flush hook and the lookups consult). + { + auto& memo = Memo(); + std::lock_guard lock(memo.mutex); + const auto end = begin + count; + std::erase_if(memo.entries, [&](const GpuKeyStore& store) { return begin < store.end && store.begin < end; }); + if (MemoRecorder(memo) == &recorder) memo.entries.push_back({begin, end}); + } + CountStore(Scans().gpuStores); + return true; +} + +void StoreUncompressedOnCpu(std::uint64_t begin, std::size_t count) { + // The write waits for recorded work over the range (the flush hook), so the bytes are current after. + ForgetStores(begin, begin + count); + const std::vector uncompressed(count, std::byte{0xff}); + GuestMemory::Write(begin, uncompressed); + CountStore(Scans().cpuStores); +} + +// Whether the keys already read as uncompressed: scans the same bytes as ReadDccKeys, so it counts +// as a scan too. +bool AlreadyUncompressed(std::uint64_t metaAddress, std::size_t count) { + const auto* keys = reinterpret_cast(metaAddress); + const auto start = ScanProfileEnabled() ? std::chrono::steady_clock::now() : std::chrono::steady_clock::time_point{}; + const bool uncompressed = AllKeysEqual(keys, count, 0xff); + if (ScanProfileEnabled()) CountScan(count, start); + return uncompressed; +} + +bool CpuKeysOnly() { + static const bool cpuKeys = std::getenv("APS5_CPU_DCC_KEYS") != nullptr; + return cpuKeys; +} + +enum class Kind { Unorm, Snorm, Uint, Sint, Float }; + +struct Layout { + std::uint32_t channels; + std::uint32_t bits[4]; + Kind kind; +}; + +bool LayoutFor(VkFormat format, Layout& layout) { + switch (format) { + case VK_FORMAT_R8_UNORM: case VK_FORMAT_R8_SRGB: layout = {1, {8}, Kind::Unorm}; return true; + case VK_FORMAT_R8_UINT: layout = {1, {8}, Kind::Uint}; return true; + case VK_FORMAT_R16_UNORM: layout = {1, {16}, Kind::Unorm}; return true; + case VK_FORMAT_R16_SNORM: layout = {1, {16}, Kind::Snorm}; return true; + case VK_FORMAT_R16_UINT: layout = {1, {16}, Kind::Uint}; return true; + case VK_FORMAT_R16_SINT: layout = {1, {16}, Kind::Sint}; return true; + case VK_FORMAT_R16_SFLOAT: layout = {1, {16}, Kind::Float}; return true; + case VK_FORMAT_R8G8_UNORM: case VK_FORMAT_R8G8_SRGB: layout = {2, {8, 8}, Kind::Unorm}; return true; + case VK_FORMAT_R8G8_SNORM: layout = {2, {8, 8}, Kind::Snorm}; return true; + case VK_FORMAT_R8G8_UINT: layout = {2, {8, 8}, Kind::Uint}; return true; + case VK_FORMAT_R8G8_SINT: layout = {2, {8, 8}, Kind::Sint}; return true; + case VK_FORMAT_R32_UINT: layout = {1, {32}, Kind::Uint}; return true; + case VK_FORMAT_R32_SINT: layout = {1, {32}, Kind::Sint}; return true; + case VK_FORMAT_R32_SFLOAT: layout = {1, {32}, Kind::Float}; return true; + case VK_FORMAT_R16G16_UNORM: layout = {2, {16, 16}, Kind::Unorm}; return true; + case VK_FORMAT_R16G16_SNORM: layout = {2, {16, 16}, Kind::Snorm}; return true; + case VK_FORMAT_R16G16_UINT: layout = {2, {16, 16}, Kind::Uint}; return true; + case VK_FORMAT_R16G16_SINT: layout = {2, {16, 16}, Kind::Sint}; return true; + case VK_FORMAT_R16G16_SFLOAT: layout = {2, {16, 16}, Kind::Float}; return true; + case VK_FORMAT_A2B10G10R10_UNORM_PACK32: layout = {4, {10, 10, 10, 2}, Kind::Unorm}; return true; + case VK_FORMAT_A2B10G10R10_UINT_PACK32: layout = {4, {10, 10, 10, 2}, Kind::Uint}; return true; + case VK_FORMAT_R8G8B8A8_UNORM: case VK_FORMAT_R8G8B8A8_SRGB: case VK_FORMAT_B8G8R8A8_UNORM: case VK_FORMAT_B8G8R8A8_SRGB: layout = {4, {8, 8, 8, 8}, Kind::Unorm}; return true; + case VK_FORMAT_R8G8B8A8_SNORM: layout = {4, {8, 8, 8, 8}, Kind::Snorm}; return true; + case VK_FORMAT_R8G8B8A8_UINT: layout = {4, {8, 8, 8, 8}, Kind::Uint}; return true; + case VK_FORMAT_R8G8B8A8_SINT: layout = {4, {8, 8, 8, 8}, Kind::Sint}; return true; + case VK_FORMAT_R32G32_UINT: layout = {2, {32, 32}, Kind::Uint}; return true; + case VK_FORMAT_R32G32_SINT: layout = {2, {32, 32}, Kind::Sint}; return true; + case VK_FORMAT_R32G32_SFLOAT: layout = {2, {32, 32}, Kind::Float}; return true; + case VK_FORMAT_R16G16B16A16_UNORM: layout = {4, {16, 16, 16, 16}, Kind::Unorm}; return true; + case VK_FORMAT_R16G16B16A16_SNORM: layout = {4, {16, 16, 16, 16}, Kind::Snorm}; return true; + case VK_FORMAT_R16G16B16A16_UINT: layout = {4, {16, 16, 16, 16}, Kind::Uint}; return true; + case VK_FORMAT_R16G16B16A16_SINT: layout = {4, {16, 16, 16, 16}, Kind::Sint}; return true; + case VK_FORMAT_R16G16B16A16_SFLOAT: layout = {4, {16, 16, 16, 16}, Kind::Float}; return true; + case VK_FORMAT_R32G32B32A32_UINT: layout = {4, {32, 32, 32, 32}, Kind::Uint}; return true; + case VK_FORMAT_R32G32B32A32_SINT: layout = {4, {32, 32, 32, 32}, Kind::Sint}; return true; + case VK_FORMAT_R32G32B32A32_SFLOAT: layout = {4, {32, 32, 32, 32}, Kind::Float}; return true; + default: return false; + } +} + +// "1" in a channel of this kind and width. +std::uint64_t One(Kind kind, std::uint32_t bits) { + switch (kind) { + case Kind::Unorm: case Kind::Uint: return bits >= 64 ? ~0ull : (1ull << bits) - 1u; + case Kind::Snorm: case Kind::Sint: return (1ull << (bits - 1u)) - 1u; + case Kind::Float: return bits == 16 ? 0x3c00u : 0x3f800000u; + } + return 0; +} + +} + +const char* DccKeysName(DccKeys keys) { + switch (keys) { + case DccKeys::Uncompressed: return "uncompressed"; + case DccKeys::Clear0000: return "0000"; + case DccKeys::Clear0001: return "0001"; + case DccKeys::Clear1110: return "1110"; + case DccKeys::Clear1111: return "1111"; + case DccKeys::ClearRegister: return "register"; + case DccKeys::Mixed: return "mixed"; + case DccKeys::Unreadable: return "unreadable"; + } + return "?"; +} + +DccKeys ReadDccKeys(std::uint64_t metaAddress, std::uint64_t surfaceBytes) { + const auto count = static_cast(surfaceBytes / KeyBytes); + if (metaAddress == 0 || count == 0 || !GuestMemory::Accessible(reinterpret_cast(metaAddress), count)) return DccKeys::Unreadable; + // A driver store still pending on the GPU is not in the bytes yet. + if (const auto memoized = MemoizedKeys(metaAddress, metaAddress + count)) return *memoized; + const auto* keys = reinterpret_cast(metaAddress); + const auto first = keys[0]; + const auto start = ScanProfileEnabled() ? std::chrono::steady_clock::now() : std::chrono::steady_clock::time_point{}; + const bool uniform = AllKeysEqual(keys + 1, count - 1, first); + if (ScanProfileEnabled()) CountScan(count, start); + if (!uniform) return DccKeys::Mixed; + switch (first) { + case 0x00: return DccKeys::Clear0000; + case 0x40: return DccKeys::Clear0001; + case 0x80: return DccKeys::Clear1110; + case 0xc0: return DccKeys::Clear1111; + case 0x20: return DccKeys::ClearRegister; + case 0xff: return DccKeys::Uncompressed; + default: return DccKeys::Mixed; + } +} + +bool IsDccClear(DccKeys keys) { + return keys == DccKeys::Clear0000 || keys == DccKeys::Clear0001 || keys == DccKeys::Clear1110 || keys == DccKeys::Clear1111 || keys == DccKeys::ClearRegister; +} + +void MarkDccUncompressed(std::uint64_t metaAddress, std::uint64_t surfaceBytes) { + const auto count = static_cast(surfaceBytes / KeyBytes); + if (metaAddress == 0 || count == 0 || !GuestMemory::Accessible(reinterpret_cast(metaAddress), count, true)) return; + // The bytes are trusted only when no recorded work writes them (lock-free, with or without + // GuestMemory::GpuMutex); otherwise the store runs and waits through the flush hook. A pending + // GPU store of the driver's own is no reason to skip: a title kernel recorded into the same + // batch after it may write clear keys, and the memo cannot order inside a batch. + if (!Recorder::SnapshotWriteOverlaps(metaAddress, count) && AlreadyUncompressed(metaAddress, count)) return; + StoreUncompressedOnCpu(metaAddress, count); +} + +void MarkDccUncompressed(const Context& context, std::uint64_t metaAddress, std::uint64_t surfaceBytes) { + const auto count = static_cast(surfaceBytes / KeyBytes); + if (metaAddress == 0 || count == 0 || !GuestMemory::Accessible(reinterpret_cast(metaAddress), count, true)) return; + auto* recorder = CpuKeysOnly() || !GuestMemory::GpuMutex().HeldByThisThread() ? nullptr : Recorder::Active(); + // The bytes are scanned only when no recorded work writes them: a pending clear kernel's keys are + // not in memory yet, so the bytes could read as uncompressed while the kernel will store a clear + // code, and the range is stored without looking (the fill lands after the kernel). That includes + // a pending GPU store of the driver's own (the memo): a title kernel recorded into the same batch + // after it may write clear keys, and nothing orders inside a batch, so a fresh fill is recorded + // (idempotent, no wait) rather than trusting the earlier one. + const bool pending = recorder != nullptr && recorder->PendingWriteOverlaps(metaAddress, count); + if (!pending && AlreadyUncompressed(metaAddress, count)) return; + if (recorder != nullptr && StoreUncompressedOnGpu(context, *recorder, metaAddress, count)) return; + StoreUncompressedOnCpu(metaAddress, count); +} + +bool FillDccClear(VkFormat format, DccKeys keys, bool alphaOnMsb, std::span bytes) { + if (keys == DccKeys::Clear0000) { + std::fill(bytes.begin(), bytes.end(), std::byte{0}); + return true; + } + Layout layout{}; + if (!IsDccClear(keys) || keys == DccKeys::ClearRegister || !LayoutFor(format, layout)) return false; + const bool color = keys == DccKeys::Clear1110 || keys == DccKeys::Clear1111; + const bool alpha = keys == DccKeys::Clear0001 || keys == DccKeys::Clear1111; + const int alphaChannel = layout.channels == 3 ? -1 : alphaOnMsb ? static_cast(layout.channels) - 1 : 0; + std::uint32_t elementBits = 0; + for (std::uint32_t channel = 0; channel < layout.channels; ++channel) elementBits += layout.bits[channel]; + std::vector element(elementBits / 8u); + std::uint32_t bit = 0; + for (std::uint32_t channel = 0; channel < layout.channels; ++channel) { + const bool set = static_cast(channel) == alphaChannel ? alpha : color; + const auto value = set ? One(layout.kind, layout.bits[channel]) : 0u; + for (std::uint32_t i = 0; i < layout.bits[channel]; ++i, ++bit) { + if (((value >> i) & 1u) != 0) element[bit / 8u] |= static_cast(1u << (bit % 8u)); + } + } + for (std::size_t offset = 0; offset + element.size() <= bytes.size(); offset += element.size()) std::memcpy(bytes.data() + offset, element.data(), element.size()); + return true; +} + +bool DccAlphaOnMsb(VkFormat format, std::uint32_t componentSwap) { + Layout layout{}; + const auto channels = LayoutFor(format, layout) ? layout.channels : 4u; + constexpr std::uint32_t standardReversed = 2; + constexpr std::uint32_t alternateReversed = 3; + if (channels == 1) return componentSwap == alternateReversed; + return componentSwap != standardReversed && componentSwap != alternateReversed; +} + +DccKeys TextureClearKeys(const GuestTextureResource& resource, std::uint64_t guestBytes) { + if (resource.dccAddress == 0) return DccKeys::Uncompressed; + const auto keys = ReadDccKeys(resource.dccAddress, guestBytes); + if (keys == DccKeys::Uncompressed) return keys; + std::byte probe[16]{}; + if (!IsDccClear(keys) || !FillDccClear(ResolveTextureFormat(resource.format), keys, resource.dccAlphaOnMsb, std::span(probe, std::min(sizeof(probe), BytesPerElement(resource.format))))) { + static std::mutex reportedMutex; + static std::set> reported; + std::lock_guard lock(reportedMutex); + if (reported.size() < 32 && reported.insert({resource.baseAddress, static_cast(keys)}).second) std::fprintf(stderr, "[gpu] texture 0x%llx (format %u) has %s DCC keys at 0x%llx; its texels are read as stored\n", static_cast(resource.baseAddress), resource.format, DccKeysName(keys), static_cast(resource.dccAddress)); + return DccKeys::Uncompressed; + } + return keys; +} + +void ReadTextureSurface(const GuestTextureResource& resource, DccKeys keys, std::span bytes) { + // Named for the [hooksync] attribution: the read goes through the flush hook. + const GuestMemory::ReadSiteScope site(GuestMemory::ReadSite::TextureRead); + if (keys == DccKeys::Uncompressed) GuestMemory::ReadCommitted(resource.baseAddress, bytes); + else FillDccClear(ResolveTextureFormat(resource.format), keys, resource.dccAlphaOnMsb, bytes); +} + +} diff --git a/core/libs/prx/libSceAgcDriver/Graphics/src/Draw.cpp b/core/libs/prx/libSceAgcDriver/Graphics/src/Draw.cpp index b0a7ba2d6..f73afaee7 100644 --- a/core/libs/prx/libSceAgcDriver/Graphics/src/Draw.cpp +++ b/core/libs/prx/libSceAgcDriver/Graphics/src/Draw.cpp @@ -2,30 +2,491 @@ #include "prx/libSceAgcDriver/Graphics/include/ColorTargetTransfer.hpp" #include "prx/libSceAgcDriver/Graphics/include/GpuColorTransfer.hpp" #include "prx/libSceAgcDriver/Graphics/include/VertexInput.hpp" +#include "prx/libSceAgcDriver/Graphics/include/TextureDetiler.hpp" +#include "prx/libSceAgcDriver/Graphics/include/DccMetadata.hpp" +#include "prx/libSceAgcDriver/Graphics/include/ShaderResources.hpp" +#include "prx/libSceAgcDriver/Graphics/include/Recorder.hpp" +#include "prx/libSceAgcDriver/Graphics/include/TextureFormat.hpp" #include "prx/libSceAgcDriver/Execution/include/GuestMemory.hpp" #include "prx/libSceAgcDriver/Execution/include/PerformanceTimer.hpp" #include "prx/libc/include/General.hpp" -#include "prx/libSceAgcDriver/Graphics/include/RenderCache.hpp" -#include "prx/libSceAgcDriver/Graphics/include/DrawQueue.hpp" -#include "prx/libSceAgcDriver/Graphics/include/GraphicsPipelineCache.hpp" +#include "Optimization/include/Optimization/ShaderStageInputInfo.hpp" +#include +#include +#include +#include +#include +#include +#include #include #include +#include #include +#include +#include +#include namespace AgcDriver::Graphics { + namespace { -struct DrawStorage { - std::unique_ptr indices; - std::vector> vertices; - std::shared_ptr color; - std::shared_ptr pipeline; +// A guest texture format code with the same Vulkan format and texel size as a color buffer. +std::uint32_t GuestFormatFor(VkFormat format, std::uint32_t elementBytes) { + static std::mutex mutex; + static std::map, std::uint32_t> known; + std::lock_guard lock(mutex); + if (const auto found = known.find({format, elementBytes}); found != known.end()) return found->second; + for (std::uint32_t code = 0; code < 256; ++code) { + try { + if (!IsBlockCompressed(code) && ResolveTextureFormat(code) == format && BytesPerElement(code) == elementBytes) { + known.emplace(std::make_pair(format, elementBytes), code); + return code; + } + } catch (const std::exception&) { + } + } + throw std::runtime_error("AGC graphics: no guest texture format matches the color buffer format " + std::to_string(static_cast(format))); +} + +// The color buffer as a single-mip 2D surface descriptor (tile mode SW_64KB_R_X). +GuestTextureResource SurfaceForTarget(const ColorTarget& color) { + Require(color.tileMode == ColorTileMode::RenderTarget, "only 64 KiB tiled color targets are resident"); + GuestTextureResource surface{}; + surface.baseAddress = color.address; + surface.width = color.extent.width; + surface.height = color.extent.height; + surface.depthOrLastArray = 0; + surface.baseArray = 0; + surface.mipCount = 1; + surface.baseLevel = 0; + surface.lastLevel = 0; + surface.tileMode = TextureTileMode::kR64KBX; + surface.dimension = TextureDimension::k2D; + surface.format = GuestFormatFor(color.format, color.elementBytes); + surface.dstSelX = 4; + surface.dstSelY = 5; + surface.dstSelZ = 6; + surface.dstSelW = 7; + surface.dccAddress = color.dccAddress; + surface.dccAlphaOnMsb = color.dccAlphaOnMsb; + return surface; +} + +} + +namespace { + +void imageBarrier(const Context& context, VkCommandBuffer commands, VkImage image, VkImageLayout oldLayout, VkImageLayout newLayout, VkPipelineStageFlags sourceStage, VkPipelineStageFlags destinationStage, VkAccessFlags sourceAccess, VkAccessFlags destinationAccess) { + VkImageMemoryBarrier barrier{VK_STRUCTURE_TYPE_IMAGE_MEMORY_BARRIER}; + barrier.srcAccessMask = sourceAccess; + barrier.dstAccessMask = destinationAccess; + barrier.oldLayout = oldLayout; + barrier.newLayout = newLayout; + barrier.srcQueueFamilyIndex = VK_QUEUE_FAMILY_IGNORED; + barrier.dstQueueFamilyIndex = VK_QUEUE_FAMILY_IGNORED; + barrier.image = image; + barrier.subresourceRange = {VK_IMAGE_ASPECT_COLOR_BIT, 0, 1, 0, 1}; + context.Function("vkCmdPipelineBarrier")(commands, sourceStage, destinationStage, 0, 0, nullptr, 0, nullptr, 1, &barrier); +} + +void memoryBarrier(const Context& context, VkCommandBuffer commands, VkPipelineStageFlags sourceStage, VkPipelineStageFlags destinationStage, VkAccessFlags sourceAccess, VkAccessFlags destinationAccess) { + const VkMemoryBarrier barrier{VK_STRUCTURE_TYPE_MEMORY_BARRIER, nullptr, sourceAccess, destinationAccess}; + context.Function("vkCmdPipelineBarrier")(commands, sourceStage, destinationStage, 0, 1, &barrier, 0, nullptr, 0, nullptr); +} + +// The color surface as one untailed SW_64KB_R_X mip for the GPU detiler, with tightly packed linear rows. +TileMipLayout ColorTargetMip(const ColorTarget& color, const ColorTargetLayout& layout) { + TileMipLayout mip{}; + mip.width = color.extent.width; + mip.height = color.extent.height; + mip.blocksPerRow = layout.BlocksPerRow(); + mip.pitchBytes = color.extent.width * color.elementBytes; + mip.tiledSize = layout.Bytes(); + mip.linearSize = layout.LinearBytes(); + return mip; +} + +// APS5_PROFILE_DRAW: per-draw phase timers in microseconds (a recorded draw's phases are far below +// the millisecond the old print rounded to), totalled over 10 s in the [draws] line. +enum DrawPhase : std::size_t { PhaseValidate, PhaseVertex, PhaseSetup, PhaseReadTarget, PhasePrepare, PhaseLookup, PhaseResources, PhasePipeline, PhaseRecord, PhaseKeep, PhaseSync, PhaseWriteBack, PhaseDescribe, PhaseCount }; +constexpr std::array DrawPhaseNames{"validate", "vertex", "setup", "readTarget", "prepare", "lookup", "resources", "pipeline", "record", "keep", "sync", "writeBack", "describe"}; + +// Why a draw did not go into the recorder without a wait (counted in the [draws] line): its targets +// are not all resident, no recorder is active, a switch (APS5_SYNC_DRAWS, APS5_DUMP_TARGETS, +// APS5_SYNC_COMPLETION_DRAWS) forced it, or its completion work writes copied guest buffers or +// releases an address-based build's lease, which the CPU must not outrun (see `recorded` in Draw). +enum SyncReason : std::size_t { SyncNone, SyncNotResident, SyncNoRecorder, SyncDisabled, SyncCopiedWrites, SyncLease, SyncCount }; +constexpr std::array SyncReasonNames{"none", "non-resident target", "no recorder", "disabled", "copied writes", "lease"}; + +// What became of one draw, for the totals. +struct DrawOutcome { + // In the recorder; `waited` when the recorder was synced right after (copied writes or a lease). + bool recorded = false; + bool waited = false; + bool completion = false; + SyncReason reason = SyncNone; + // Shader validation memo (see CachedFragmentOutputs). + bool validateMemoized = false; + bool validateHit = false; + // A build that acquired the allocation registry lease (BDA), whose cost sits outside the + // build's own sub-phases. + bool addressBased = false; + // Resident target lookups, and those that took a millisecond or more. Whether a slow one + // re-uploaded the image or just walked its pages under contention is not visible from here + // (StorageTexture::Version advances on every MarkDirty as well as on an upload, and Generation + // is stamped afresh by every write-watch walk): the [texture] line's "reused" and "direct + // uploads" counters and APS5_TRACE_UPLOAD name the actual uploads. + std::uint64_t targetLookups = 0; + std::uint64_t slowLookups = 0; + double slowLookupUs = 0; + // The flush hook's fence waits made inside this draw (index and vertex reads, the target + // lookups' flushes, the build's texture lookups): nested in the draw's GpuMutex hold, so they + // are the part of the hold that is waiting rather than working. The draw's own syncs (a + // recorded-then-waited draw's recorder Sync, a synchronous draw's SubmitAndWait) are kept apart + // in ownSyncUs: those wait by design and would otherwise count as nested hook waits. + double hookWaitUs = 0; + double ownSyncUs = 0; }; +struct DrawProfile { + std::mutex mutex; + std::array totalsUs{}; + // The longest single draw's time per phase, and the longest draw: the [lock] line's 'draw' + // hold max (tens of ms against an average well under a millisecond) needs a phase name. + std::array maxUs{}; + double maxDrawUs = 0; + double hookWaitUs = 0; + double maxHookWaitUs = 0; + double ownSyncUs = 0; + // The resources build's sub-phases (ShaderResources::Timing) of draws only; dispatches report + // theirs in the [resources] line. `other` is the rest of the build (prepareAddressBindings: + // the registry lease and snapshots of an address-based build, the BDA table, the layout + // lookup), and how much of it address-based builds account for. + double bindingsUs = 0; + double uploadUs = 0; + double descriptorsUs = 0; + double otherUs = 0; + double addressOtherUs = 0; + std::uint64_t addressBuilds = 0; + std::uint64_t draws = 0; + std::uint64_t recorded = 0; + std::uint64_t waited = 0; + // Recorded draws whose write-back (BDA fault check) runs as a completion action instead of + // making the draw synchronous (see APS5_SYNC_COMPLETION_DRAWS). + std::uint64_t completion = 0; + std::array reasons{}; + // The CPU inside Graphics::Draw, split by outcome: a synchronous draw costs ten times a recorded + // one, so one average would only show the synchronous population. + double recordedUs = 0; + double waitedUs = 0; + double synchronousUs = 0; + std::uint64_t targetLookups = 0; + std::uint64_t slowLookups = 0; + double slowLookupUs = 0; + // Resource cache outcomes: hits, misses (built and inserted when reusable), entries that failed + // Revalidate, and draws that could not use the cache (synchronous, no variant id, or disabled). + // Which key words the misses differ in is the cache's own "[rescache] miss churn" line + // (ResourceCache::noteMiss compares each miss with the last key of the same variants). + std::uint64_t cacheHits = 0; + std::uint64_t cacheMisses = 0; + std::uint64_t cacheInvalidated = 0; + std::uint64_t uncacheable = 0; + // Shader validation memo (see CachedFragmentOutputs). + std::uint64_t validateHits = 0; + std::uint64_t validateMisses = 0; + std::chrono::steady_clock::time_point lastReport = std::chrono::steady_clock::now(); +}; + +DrawProfile& Profile() { + static DrawProfile profile; + return profile; +} + +// Adds one draw's phases to the totals and prints the [draws] and [rescache] lines every 10 s. +void reportDraw(const std::array& us, const ShaderResources::BuildTiming* built, const DrawOutcome& outcome) { + auto& profile = Profile(); + std::lock_guard lock(profile.mutex); + double drawUs = 0; + for (std::size_t i = 0; i < PhaseCount; ++i) { + profile.totalsUs[i] += us[i]; + profile.maxUs[i] = std::max(profile.maxUs[i], us[i]); + drawUs += us[i]; + } + profile.maxDrawUs = std::max(profile.maxDrawUs, drawUs); + profile.hookWaitUs += outcome.hookWaitUs; + profile.maxHookWaitUs = std::max(profile.maxHookWaitUs, outcome.hookWaitUs); + profile.ownSyncUs += outcome.ownSyncUs; + if (built != nullptr) { + profile.bindingsUs += built->bindingsMs * 1000.0; + profile.uploadUs += built->uploadMs * 1000.0; + profile.descriptorsUs += built->descriptorsMs * 1000.0; + const auto other = std::max(0.0, us[PhaseResources] - (built->bindingsMs + built->uploadMs + built->descriptorsMs) * 1000.0); + profile.otherUs += other; + if (outcome.addressBased) { + profile.addressOtherUs += other; + ++profile.addressBuilds; + } + } + ++profile.draws; + if (outcome.recorded) ++(outcome.waited ? profile.waited : profile.recorded); + if (outcome.completion) ++profile.completion; + ++profile.reasons[outcome.reason]; + (outcome.recorded ? (outcome.waited ? profile.waitedUs : profile.recordedUs) : profile.synchronousUs) += drawUs; + profile.targetLookups += outcome.targetLookups; + profile.slowLookups += outcome.slowLookups; + profile.slowLookupUs += outcome.slowLookupUs; + if (outcome.validateMemoized) ++(outcome.validateHit ? profile.validateHits : profile.validateMisses); + const auto now = std::chrono::steady_clock::now(); + if (now - profile.lastReport < std::chrono::seconds(10)) return; + profile.lastReport = now; + const auto synchronous = profile.draws - profile.recorded - profile.waited; + const auto average = [](double total, std::uint64_t count) { return count != 0 ? total / static_cast(count) : 0.0; }; + char line[1536]; + int n = std::snprintf(line, sizeof(line), "[draws] %llu draws over 10 s (%llu recorded avg %.0f us, of them %llu with completion; %llu recorded then waited avg %.0f us; %llu synchronous avg %.0f us; waited or synchronous because:", static_cast(profile.draws), static_cast(profile.recorded), average(profile.recordedUs, profile.recorded), static_cast(profile.completion), static_cast(profile.waited), average(profile.waitedUs, profile.waited), static_cast(synchronous), average(profile.synchronousUs, synchronous)); + const auto room = [&] { return n > 0 && static_cast(n) < sizeof(line); }; + for (std::size_t i = SyncNone + 1; i < SyncCount && room(); ++i) { + if (profile.reasons[i] != 0) n += std::snprintf(line + n, sizeof(line) - static_cast(n), " %s %llu", SyncReasonNames[i], static_cast(profile.reasons[i])); + } + if (room()) n += std::snprintf(line + n, sizeof(line) - static_cast(n), "):"); + for (std::size_t i = 0; i < PhaseCount && room(); ++i) { + if (profile.totalsUs[i] <= 0) continue; + n += std::snprintf(line + n, sizeof(line) - static_cast(n), " %s=%.1fms", DrawPhaseNames[i], profile.totalsUs[i] / 1000.0); + if (i == PhaseResources && room()) n += std::snprintf(line + n, sizeof(line) - static_cast(n), " (bindings %.1f, upload %.1f, descriptors %.1f, other %.1f of which %.1f in %llu address-based builds)", profile.bindingsUs / 1000.0, profile.uploadUs / 1000.0, profile.descriptorsUs / 1000.0, profile.otherUs / 1000.0, profile.addressOtherUs / 1000.0, static_cast(profile.addressBuilds)); + if (i == PhaseReadTarget && room()) n += std::snprintf(line + n, sizeof(line) - static_cast(n), " (%llu resident lookups, %llu of them >= 1 ms = %.1f)", static_cast(profile.targetLookups), static_cast(profile.slowLookups), profile.slowLookupUs / 1000.0); + } + // The longest draw and the longest single phase of any draw (which phase a long 'draw' hold + // was), plus the hook waits nested inside the draws. + if (room()) n += std::snprintf(line + n, sizeof(line) - static_cast(n), "; longest draw %.1f ms, longest phase of any draw:", profile.maxDrawUs / 1000.0); + for (std::size_t i = 0; i < PhaseCount && room(); ++i) { + if (profile.maxUs[i] < 500.0) continue; + n += std::snprintf(line + n, sizeof(line) - static_cast(n), " %s %.1f", DrawPhaseNames[i], profile.maxUs[i] / 1000.0); + } + if (room()) n += std::snprintf(line + n, sizeof(line) - static_cast(n), "; hook waits inside draws %.1f ms (max %.1f; the draws' own syncs, %.1f ms, not counted)", profile.hookWaitUs / 1000.0, profile.maxHookWaitUs / 1000.0, profile.ownSyncUs / 1000.0); + std::fprintf(stderr, "%s\n", line); + std::fprintf(stderr, "[rescache] draws: %llu hits, %llu misses, %llu invalidated, %llu uncacheable; validation memo %llu hits / %llu misses (which key words the misses differ in: the miss churn line)\n", static_cast(profile.cacheHits), static_cast(profile.cacheMisses), static_cast(profile.cacheInvalidated), static_cast(profile.uncacheable), static_cast(profile.validateHits), static_cast(profile.validateMisses)); + profile.totalsUs.fill(0); + profile.maxUs.fill(0); + profile.maxDrawUs = profile.hookWaitUs = profile.maxHookWaitUs = profile.ownSyncUs = 0; + profile.bindingsUs = profile.uploadUs = profile.descriptorsUs = profile.otherUs = profile.addressOtherUs = 0; + profile.addressBuilds = 0; + profile.draws = profile.recorded = profile.waited = profile.completion = 0; + profile.reasons.fill(0); + profile.recordedUs = profile.waitedUs = profile.synchronousUs = 0; + profile.targetLookups = profile.slowLookups = 0; + profile.slowLookupUs = 0; + profile.cacheHits = profile.cacheMisses = profile.cacheInvalidated = profile.uncacheable = 0; + profile.validateHits = profile.validateMisses = 0; +} + +void countCache(std::uint64_t DrawProfile::*counter) { + static const bool profile = std::getenv("APS5_PROFILE_DRAW") != nullptr; + if (!profile) return; + auto& stats = Profile(); + std::lock_guard lock(stats.mutex); + ++(stats.*counter); +} + +// ValidateShaders decodes every SPIR-V instruction of every stage on every draw. Its outcome depends +// only on the stages' compiled variants (a variant id names identical SPIR-V and binding layout), +// their push constant placement and vertex attribute shapes, and the state fields it checks, so the +// fragment output set is remembered by exactly those. Nothing here is keyed by pointer: the results +// a draw hands in live in a per-draw vector. A stage without a variant id is validated as before, +// except the rect-list control and evaluation stages, which are generated from the vertex and +// fragment results the key already names (the pipeline key treats them the same way). `memoized` +// says whether the memo applied, `hit` whether it answered. APS5_NO_VALIDATE_CACHE=1 validates +// every draw. +std::set CachedFragmentOutputs(const Context& context, std::span shaders, const State& state, bool& memoized, bool& hit) { + using Stage = ShaderRecompiler::ShaderStage; + static const bool disabled = std::getenv("APS5_NO_VALIDATE_CACHE") != nullptr; + memoized = false; + hit = false; + std::vector key; + const auto add = [&](auto value) { key.push_back(static_cast(value)); }; + bool keyed = !disabled; + if (keyed) { + key.reserve(40 + shaders.size() * 12); + add(shaders.size()); + for (const auto& shader : shaders) { + Require(shader.program != nullptr, "missing compiled shader"); + const auto& program = *shader.program; + const bool generated = state.rectList && (shader.stage == Stage::TessellationControl || shader.stage == Stage::TessellationEvaluation); + if (!generated && program.variantId == 0) { + keyed = false; + break; + } + add(shader.stage); + add(generated ? std::uint64_t{0} : program.variantId); + add(shader.pushConstantOffset); + add(program.pushConstants.size()); + add(program.bdaAbiVersion); + add(program.vertexAttributes.size()); + for (const auto& attribute : program.vertexAttributes) { + add(attribute.location); + add(attribute.components); + add(attribute.fetchIndex); + // The data format bits of the V# decide the attribute's signature. + add((attribute.resource.fields[3] >> 12u) & 0x7fu); + } + } + } + if (keyed) { + add(state.stages.path); + add(state.rectList); + add(state.topology); + add(state.cullMode); + add(state.colors.size()); + add(context.subgroup.subgroupSize); + add(context.subgroup.supportedStages); + add(context.subgroup.supportedOperations); + add(context.fragmentShaderBarycentric); + add(state.stages.mesh.has_value()); + if (state.stages.mesh) { + const auto& mesh = *state.stages.mesh; + add(mesh.inputPrimitive); + add(mesh.primitivesPerGroup); + add(mesh.verticesPerGroup); + add(mesh.maxVertices); + add(mesh.maxPrimitives); + add(mesh.threadsPerGroup); + add(mesh.ldsSizeDwords); + add(mesh.provokingVertex); + } + add(state.stages.tessellation.has_value()); + if (state.stages.tessellation) { + const auto& tessellation = *state.stages.tessellation; + add(tessellation.inputControlPoints); + add(tessellation.outputControlPoints); + add(tessellation.domain); + add(tessellation.partitioning); + add(tessellation.outputTopology); + } + } + static std::mutex mutex; + static std::map, std::set> memo; + if (keyed) { + memoized = true; + std::lock_guard lock(mutex); + if (const auto found = memo.find(key); found != memo.end()) { + hit = true; + return found->second; + } + } + auto outputs = ValidateShaders(shaders, state, context.subgroup, context.fragmentShaderBarycentric); + if (keyed) { + std::lock_guard lock(mutex); + // A handful of configurations recur; a runaway key space is dropped wholesale. + if (memo.size() >= 1024) memo.clear(); + memo.emplace(std::move(key), outputs); + } + return outputs; +} + +// The resource cache key of a recorded draw: a marker no compute key starts with (those begin with +// the stage), the device handle (the cache is process-wide and the driver replaces the headless +// device with the windowed one while workers may still use the old one: an entry's descriptor set +// and pooled buffers belong to the device that built it) and every stage's content key +// (ResourceCache::noteMiss walks this layout to attribute misses, so it changes together with it). +// With `ranges`, also the render target and the index buffer the build's alias checks compared the +// guest buffers against; without it the checks are repeated for a hit (CheckBufferAliases), since +// neither range is part of the descriptor set (the target is attached, the index buffer copied per +// draw), so a target or index ring that moves per frame does not miss on every draw. That guards a +// workload with such rings; in the profiled menu stage every draw was DRAW_INDEX_AUTO (index range +// 0) onto one fixed target, so its misses come from the stages' descriptor words themselves. +ResourceCache::Key DrawResourceKey(const Context& context, std::span shaders, const ColorTarget& target, std::uint64_t indexAddress, std::uint64_t indexBytes, bool ranges) { + ResourceCache::Key key{0xffffffffu}; + const auto append64 = [&](std::uint64_t value) { + key.push_back(static_cast(value)); + key.push_back(static_cast(value >> 32u)); + }; + append64(reinterpret_cast(context.device)); + key.push_back(static_cast(shaders.size())); + for (const auto& shader : shaders) { + const auto part = ShaderResources::ContentKey(shader); + key.push_back(static_cast(part.size())); + key.insert(key.end(), part.begin(), part.end()); + } + if (ranges) { + append64(target.address); + append64(target.bytes); + append64(indexAddress); + append64(indexBytes); + } + return key; +} + +// The alias checks the build makes for every guest buffer descriptor (ShaderResources:: +// addGuestBuffer), repeated for a cached build whose own checks compared the buffers against +// another draw's render target and index buffer. The same conditions and messages, so a draw that +// would have failed its build fails its hit. +void CheckBufferAliases(std::span shaders, const ColorTarget& target, std::uint64_t indexAddress, std::uint64_t indexBytes) { + const auto overlap = [](std::uint64_t first, std::uint64_t firstSize, std::uint64_t second, std::uint64_t secondSize) { return first < second + secondSize && second < first + firstSize; }; + for (const auto& shader : shaders) { + for (const auto& binding : shader.program->bindings) { + if (binding.role != ShaderRecompiler::DescriptorRole::GuestBuffers) continue; + const auto& words = binding.guestDescriptor; + for (std::size_t offset = 0; offset + 4 <= words.size(); offset += 4) { + const ShaderRecompiler::ShaderBufferResource descriptor{{words[offset], words[offset + 1], words[offset + 2], words[offset + 3]}}; + const auto address = descriptor.Base48(); + const auto size = descriptor.GetSize(); + Require(!overlap(address, size, target.address, target.bytes), "shader buffer aliases the render target"); + Require(!overlap(address, size, indexAddress, indexBytes), "writable shader buffer aliases the index buffer"); + } + } + } +} + +} + +std::shared_ptr>> DrawCopiedWriters() { + // Never destroyed: a completion action of a batch still in flight at static teardown may erase from it. + static auto* const writers = new std::shared_ptr>>(std::make_shared>>()); + return *writers; } void Draw(const Context& context, const State& state, const Pm4::DrawParameters& draw, std::span shaders, std::span snapshots) { PerformanceTimer timing("Graphics.Draw"); + // APS5_PROFILE_DRAW prints the time of each phase of the draw (microseconds) and the [draws] totals. + static const bool profile = std::getenv("APS5_PROFILE_DRAW") != nullptr; + auto phaseStart = std::chrono::steady_clock::now(); + std::array us{}; + const auto phase = [&](DrawPhase which) { + if (!profile) return; + const auto now = std::chrono::steady_clock::now(); + us[which] += std::chrono::duration(now - phaseStart).count(); + phaseStart = now; + }; + // APS5_TRACE_DRAWS (together with APS5_PROFILE_DRAW; alone it only enables Driver.cpp's own + // [draw] lines) additionally prints one line per draw with its phases and, for synchronous + // draws, their inputs: per-draw string building at ~1400 draws per 10 s is itself a cost the + // totals must not carry. + static const bool traceDraws = std::getenv("APS5_TRACE_DRAWS") != nullptr; + DrawOutcome outcome; + const ShaderResources::BuildTiming* built = nullptr; + // The thread's GPU waits so far: the difference at the report, less the draw's own syncs + // (`ownWaitedMs`, sampled around them below), is what this draw waited for through the flush + // hook (nested in the draw's hold, see DrawOutcome::hookWaitUs). + const auto waitedBefore = profile ? Recorder::ThreadWaitedMs() : 0.0; + double ownWaitedMs = 0; + // One line per draw with the phases that took time (tracing), and the 10 s totals. + const auto report = [&](const char* suffix) { + if (!profile) return; + outcome.hookWaitUs = std::max(0.0, Recorder::ThreadWaitedMs() - waitedBefore - ownWaitedMs) * 1000.0; + outcome.ownSyncUs = ownWaitedMs * 1000.0; + if (traceDraws) { + char line[512]; + int n = std::snprintf(line, sizeof(line), "[draw] %ux%u %zu targets%s:", state.renderExtent.width, state.renderExtent.height, state.colors.size(), suffix); + for (std::size_t i = 0; i < PhaseCount && n > 0 && static_cast(n) < sizeof(line); ++i) { + if (us[i] <= 0) continue; + n += std::snprintf(line + n, sizeof(line) - static_cast(n), " %s=%.0fus", DrawPhaseNames[i], us[i]); + } + std::fprintf(stderr, "%s\n", line); + } + reportDraw(us, built, outcome); + }; + APS5_LOG_OUT_DEBUG("Draw indices=%u instances=%u indexSize=%u flags=%u indexAddress=0x%llx", draw.indexCount, draw.instanceCount, draw.indexSize, draw.flags, static_cast(draw.indexAddress)); + APS5_LOG_OUT_DEBUG("State colorTarget=%u render=%ux%u colorAddress=0x%llx colorBytes=%llu colorExtent=%ux%u", state.hasColorTarget ? 1u : 0u, state.renderExtent.width, state.renderExtent.height, static_cast(state.color.address), static_cast(state.color.bytes), state.color.extent.width, state.color.extent.height); + APS5_LOG_OUT_DEBUG("Viewport x=%f y=%f w=%f h=%f minDepth=%f maxDepth=%f", state.viewport.x, state.viewport.y, state.viewport.width, state.viewport.height, state.viewport.minDepth, state.viewport.maxDepth); + APS5_LOG_OUT_DEBUG("Scissor x=%d y=%d w=%u h=%u topology=%u cullMode=0x%x frontFace=%u", state.scissor.offset.x, state.scissor.offset.y, state.scissor.extent.width, state.scissor.extent.height, static_cast(state.topology), static_cast(state.cullMode), static_cast(state.frontFace)); Require(draw.indexed ? draw.flags == 0 : (draw.flags & ~0x20u) == 0, "draw modifiers are unsupported"); if (draw.indexed) { Require(draw.indexSize == 2 || draw.indexSize == 4, "only uint16 and uint32 index buffers are supported"); @@ -38,15 +499,21 @@ void Draw(const Context& context, const State& state, const Pm4::DrawParameters& } Require(draw.indexCount != 0 && draw.instanceCount != 0, "zero-count indexed draws are unsupported"); const auto indexBytes = static_cast(draw.indexCount) * draw.indexSize; + APS5_LOG_OUT_DEBUG("Index buffer bytes=%llu", static_cast(indexBytes)); Require(indexBytes <= std::numeric_limits::max(), "index buffer size overflow"); if (draw.indexed) GuestMemory::CheckRange(reinterpret_cast(draw.indexAddress), static_cast(indexBytes), draw.indexSize); + APS5_LOG_CHARS_OUT_DEBUG("Index buffer range OK"); Require(!draw.indexed || !state.hasColorTarget || draw.indexAddress + indexBytes <= state.color.address || state.color.address + state.color.bytes <= draw.indexAddress, "index buffer aliases the render target"); if (state.rectList) Require(draw.indexCount % 3 == 0, "incomplete rect-list primitive"); - ValidateShaders(shaders, state, context.subgroup, context.fragmentShaderBarycentric); + APS5_LOG_CHARS_OUT_DEBUG("ValidateShaders"); + const auto fragmentOutputs = CachedFragmentOutputs(context, shaders, state, outcome.validateMemoized, outcome.validateHit); + APS5_LOG_CHARS_OUT_DEBUG("ValidateShaders OK"); const auto shaderStages = PipelineStages(shaders); + APS5_LOG_OUT_DEBUG("PipelineStages=0x%x", static_cast(shaderStages)); std::uint32_t meshGroups = 0; if (state.stages.mesh) { Require(draw.firstVertex == 0 && draw.firstInstance == 0, "mesh draw offsets are unsupported"); + APS5_LOG_CHARS_OUT_DEBUG("Mesh path"); Require(context.meshShader, "device does not support mesh shaders"); const auto& mesh = *state.stages.mesh; const auto inputSize = mesh.inputPrimitive == 1 ? 1u : mesh.inputPrimitive == 2 ? 2u : 3u; @@ -54,13 +521,15 @@ void Draw(const Context& context, const State& state, const Pm4::DrawParameters& const auto step = mesh.inputPrimitive == 6 ? 1u : inputSize; const auto primitives = (draw.indexCount - inputSize) / step + 1u; meshGroups = (primitives - 1u) / mesh.primitivesPerGroup + 1u; + APS5_LOG_OUT_DEBUG("Mesh primitives=%u groups=%u", primitives, meshGroups); Require(meshGroups <= context.meshLimits.maxMeshWorkGroupCount[0] && draw.instanceCount <= context.meshLimits.maxMeshWorkGroupCount[1] && static_cast(meshGroups) * draw.instanceCount <= context.meshLimits.maxMeshWorkGroupTotalCount, "mesh draw exceeds workgroup count limits"); } if (state.stages.tessellation) Require(draw.indexCount % state.stages.tessellation->inputControlPoints == 0, "incomplete tessellation patch"); + // Viewport and scissor are dynamic pipeline state, so their limits are checked here per draw. + ValidateViewport(context, state.viewport); timing.Mark("validate"); - Require(context.renderCache != nullptr && context.drawQueue != nullptr && context.graphicsPipelines != nullptr, "device graphics execution caches are unavailable"); - auto storage = std::make_shared(); - auto& indices = storage->indices; + phase(PhaseValidate); + std::unique_ptr indices; std::uint32_t maxIndex = draw.indexed ? 0u : draw.firstVertex + draw.indexCount - 1u; if (draw.indexed) { indices = std::make_unique(context, static_cast(indexBytes), VK_BUFFER_USAGE_INDEX_BUFFER_BIT); @@ -78,10 +547,12 @@ void Draw(const Context& context, const State& state, const Pm4::DrawParameters& maxIndex = std::max(maxIndex, index); } } + APS5_LOG_CHARS_OUT_DEBUG("Index validation OK"); timing.Mark("index_upload"); const auto& attributes = shaders.front().program->vertexAttributes; - static_cast(BuildVertexInputLayout(context, attributes)); - auto& vertexBuffers = storage->vertices; + // Validates the vertex descriptors; the layout also keys and builds the pipeline below. + const auto vertexInput = BuildVertexInputLayout(context, attributes); + std::vector> vertexBuffers; std::vector vertexHandles; std::vector vertexOffsets(attributes.size(), 0); for (const auto& attribute : attributes) { @@ -95,28 +566,261 @@ void Draw(const Context& context, const State& state, const Pm4::DrawParameters& vertexHandles.push_back(buffer->Handle()); vertexBuffers.push_back(std::move(buffer)); } - timing.Mark("vertex_upload"); - auto resources = std::make_shared(context, shaders, state.color, draw.indexAddress, static_cast(indexBytes), snapshots); - timing.Mark("shader_resources"); - if (state.hasColorTarget) { - const ColorTargetLayout colorLayout(state.color.extent.width, state.color.extent.height, state.color.tileMode); - Require(state.color.bytes == colorLayout.Bytes(), "color target transfer size mismatch"); - storage->color = context.renderCache->Get(state.color, state.blend.blendEnable != 0); + phase(PhaseVertex); + // One binding per color attachment. Tiled targets are detiled and retiled on the GPU in video memory; + // only changed guest blocks are stored back. + struct TargetBinding { + ColorTarget color; + bool gpuTiling = false; + // Resident target: the surface's cached storage image is attached directly and marked dirty + // afterwards, so nothing is copied in or out per draw. + std::shared_ptr resident; + // Linear pixels converted on the CPU, for targets the GPU detiler does not handle. + std::unique_ptr transfer; + // Guest bytes in host memory, and their device-local copies for the detiler. + std::unique_ptr tiled; + std::unique_ptr tiledDevice; + std::unique_ptr linearDevice; + std::vector original; + TileMipLayout mip{}; + std::unique_ptr target; + // APS5_DUMP_TARGETS: the rendered linear pixels, read back for target_
_.raw. + std::unique_ptr dump; + }; + // Debug aid: APS5_DUMP_TARGETS= saves the first n renders of every color target as a raw file + // (u32 width, u32 height, u32 VkFormat, then tightly packed rows). + static const int dumpLimit = [] { const char* text = std::getenv("APS5_DUMP_TARGETS"); return text ? std::atoi(text) : 0; }(); + static std::mutex dumpMutex; + static std::map dumped; + std::vector targets(state.colors.size()); + std::vector targetViews; + for (std::size_t index = 0; index < state.colors.size(); ++index) { + auto& binding = targets[index]; + binding.color = state.colors[index]; + const auto& color = binding.color; + binding.gpuTiling = color.tileMode == ColorTileMode::RenderTarget && context.detiler != nullptr; + APS5_LOG_OUT_DEBUG("Creating color target %zu address=0x%llx bytes=%llu extent=%ux%u", index, static_cast(color.address), static_cast(color.bytes), color.extent.width, color.extent.height); + const ColorTargetLayout colorLayout(color.extent.width, color.extent.height, color.tileMode, color.elementBytes); + constexpr VkBufferUsageFlags copies = VK_BUFFER_USAGE_TRANSFER_SRC_BIT | VK_BUFFER_USAGE_TRANSFER_DST_BIT; + phase(PhaseSetup); + // Debug aid: APS5_NO_RESIDENT_TARGETS=1 copies every target in and out again. + static const bool residentTargets = std::getenv("APS5_NO_RESIDENT_TARGETS") == nullptr; + if (binding.gpuTiling && residentTargets) { + // The lookup refreshes the image on every draw (StorageTexture::Refresh: FlushPending, + // CollectWrites over the target's pages, the DCC key scan of TextureClearKeys, then + // UnchangedSince). The page walk is skipped while the worker's collect epoch lasts + // (GuestMemory::BumpCollectEpoch: ordering points of the queue, or every packet under + // APS5_PACKET_EPOCH=1); the key scan runs on every lookup regardless. Lookups of a + // millisecond or more are counted apart; the [texture] line says how many uploaded. + const auto lookupStart = std::chrono::steady_clock::now(); + try { + auto resident = CachedStorageSurface(context, SurfaceForTarget(color)); + Require(resident->Attachable(), "storage format cannot be a color attachment"); + Require(resident->GuestBytes() == colorLayout.Bytes(), "resident image layout differs from the color layout"); + binding.resident = std::move(resident); + } catch (const std::exception& error) { + static std::mutex reportMutex; + static std::set reported; + std::lock_guard lock(reportMutex); + if (reported.insert(color.address).second) std::fprintf(stderr, "[gpu] color target 0x%llx stays non-resident: %s\n", static_cast(color.address), error.what()); + } + if (profile) { + const auto lookupUs = std::chrono::duration(std::chrono::steady_clock::now() - lookupStart).count(); + ++outcome.targetLookups; + if (lookupUs >= 1000.0) { + ++outcome.slowLookups; + outcome.slowLookupUs += lookupUs; + } + } + } + if (binding.resident != nullptr) { + phase(PhaseReadTarget); + targetViews.push_back(binding.resident->AttachmentView(color.format)); + continue; + } + if (binding.gpuTiling) { + binding.mip = ColorTargetMip(color, colorLayout); + binding.tiled = std::make_unique(context, colorLayout.Bytes(), copies); + binding.tiledDevice = std::make_unique(context, colorLayout.Bytes(), copies | VK_BUFFER_USAGE_STORAGE_BUFFER_BIT); + binding.linearDevice = std::make_unique(context, colorLayout.LinearBytes(), copies | VK_BUFFER_USAGE_STORAGE_BUFFER_BIT); + binding.original.resize(colorLayout.Bytes()); + GuestMemory::Read(color.address, binding.original, colorLayout.Alignment()); + std::memcpy(binding.tiled->Bytes().data(), binding.original.data(), binding.original.size()); + } else { + binding.transfer = std::make_unique(context, colorLayout.LinearBytes(), copies); + ReadColorTarget(color, binding.transfer->Bytes()); + } + // A fast-cleared DCC target holds its clear value wherever the draw does not write. + if (color.dccAddress != 0) { + const auto keys = ReadDccKeys(color.dccAddress, colorLayout.Bytes()); + if (IsDccClear(keys)) { + const auto pixels = binding.gpuTiling ? binding.tiled->Bytes() : binding.transfer->Bytes(); + if (!FillDccClear(color.format, keys, color.dccAlphaOnMsb, pixels)) { + static std::set> reported; + if (reported.size() < 32 && reported.insert({color.address, static_cast(keys)}).second) std::fprintf(stderr, "[gpu] color target 0x%llx (VkFormat %d) has %s DCC keys; its stored texels are used\n", static_cast(color.address), static_cast(color.format), DccKeysName(keys)); + if (binding.gpuTiling) std::memcpy(pixels.data(), binding.original.data(), binding.original.size()); + else ReadColorTarget(color, pixels); + } + } + } + phase(PhaseReadTarget); + binding.target = std::make_unique(context, color, state.blends[index].blendEnable != 0); + targetViews.push_back(binding.target->View()); } - timing.Mark("render_target_cache"); - storage->pipeline = context.graphicsPipelines->Get(state, storage->color, *resources, shaders); - auto& pipeline = *storage->pipeline; - timing.Mark("pipeline_cache"); - const auto commands = context.drawQueue->Begin(context); + phase(PhasePrepare); + // A draw whose targets are all resident is recorded into the device's open batch like a dispatch: + // the CPU does not wait for it, and the drain before it goes away (queue order and the barriers + // below keep it behind earlier recorded work); see `recorded` below for the buffers that still + // need a synchronous draw. Other draws keep their own synchronous batch. Debug aid: + // APS5_SYNC_DRAWS=1 makes every draw synchronous. + static const bool recordDraws = std::getenv("APS5_SYNC_DRAWS") == nullptr; + auto* recorder = Recorder::Active(); + const bool recordable = recordDraws && recorder != nullptr && dumpLimit == 0 && std::all_of(targets.begin(), targets.end(), [](const TargetBinding& binding) { return binding.resident != nullptr; }); + APS5_LOG_CHARS_OUT_DEBUG("Creating ShaderResources"); + // A recordable draw whose stages' compiled content repeats an earlier one binds that build's + // descriptor set when it is still valid (see ResourceCache; the dispatch path does the same). + // Only recordable draws take part: a synchronous draw writes its resources back, which a shared + // object must never do. Revalidate proves a hit by texture identity, which needs the texture + // caches. Push constants still come from this draw's stages. APS5_NO_DRAW_RESOURCE_CACHE=1 + // builds every draw's resources as before. + static const bool noDrawResourceCache = std::getenv("APS5_NO_DRAW_RESOURCE_CACHE") != nullptr; + static const bool noTextureCache = std::getenv("APS5_NO_TEXTURE_CACHE") != nullptr; + // The render target and index ranges stay out of the key (see DrawResourceKey); a hit repeats + // the alias checks instead. Debug aid: APS5_NO_DRAW_KEY_TRIM=1 keys them as before. + static const bool trimKey = std::getenv("APS5_NO_DRAW_KEY_TRIM") == nullptr; + const bool cacheable = recordable && !noDrawResourceCache && !noTextureCache && std::all_of(shaders.begin(), shaders.end(), [](const CompiledShader& shader) { return shader.program != nullptr && shader.program->variantId != 0; }); + std::shared_ptr resources; + ResourceCache::Key contentKey; + if (cacheable) { + contentKey = DrawResourceKey(context, shaders, state.color, draw.indexAddress, indexBytes, !trimKey); + if (auto cached = SharedResourceCache().Find(contentKey)) { + if (cached->Revalidate(shaders)) { + if (trimKey) CheckBufferAliases(shaders, state.color, draw.indexAddress, indexBytes); + resources = std::move(cached); + countCache(&DrawProfile::cacheHits); + } else { + SharedResourceCache().Remove(contentKey); + countCache(&DrawProfile::cacheInvalidated); + } + } + } else { + countCache(&DrawProfile::uncacheable); + } + phase(PhaseLookup); + if (resources == nullptr) { + resources = std::make_shared(context, shaders, state.color, draw.indexAddress, static_cast(indexBytes), snapshots); + built = &resources->Timing(); + outcome.addressBased = resources->HoldsLease(); + if (cacheable) countCache(&DrawProfile::cacheMisses); + } + phase(PhaseResources); + APS5_LOG_CHARS_OUT_DEBUG("ShaderResources created"); + APS5_LOG_CHARS_OUT_DEBUG("Creating Pipeline"); + // Vulkan leaves attachments a pixel shader has no output for undefined, so their writes are masked + // (shaders that only store to images or buffers keep their targets as they were). The state is + // copied only when a mask must change. + std::optional masked; + for (std::size_t index = 0; index < state.blends.size(); ++index) { + if (fragmentOutputs.contains(static_cast(index)) || state.blends[index].colorWriteMask == 0) continue; + if (!masked.has_value()) masked = state; + masked->blends[index].colorWriteMask = 0; + } + const State& pipelineState = masked.has_value() ? *masked : state; + auto pipeline = CachedPipeline(context, pipelineState, vertexInput, *resources, shaders); + // Resident targets' views are stable while their storage image lives, so the framebuffer is + // reused with the pipeline; a per-draw RenderTarget gets a framebuffer of its own. + std::vector> owners; + owners.reserve(targets.size()); + for (const auto& binding : targets) owners.push_back(binding.resident); + auto framebuffer = pipeline->AcquireFramebuffer(targetViews, owners, state.renderExtent); + phase(PhasePipeline); + APS5_LOG_CHARS_OUT_DEBUG("Pipeline created"); + APS5_LOG_CHARS_OUT_DEBUG("Creating CommandBatch"); + // A draw whose only completion work is a BDA fault check (the rect-list stages carry a fault + // buffer, so every rect-list draw needed one) is recorded too, with the check as a completion + // action like a dispatch's write-back: its own batch would first wait for everything recorded + // before it. A draw that writes a copied buffer (at the menu stage: 0x48-byte constant buffers + // inside an import but not at its storage offset alignment, so GuestBufferMemory copies them + // and every guest buffer counts as writable) is recorded and then waited for at once: the + // write-back runs before the next packet, without the separate command batch and fence a + // synchronous draw takes. The copied write-back is listed in DrawCopiedWriters, which lets such + // draws run without the wait; both consumers of the device's list in VulkanDevice.cpp + // (DispatchIndirect's argument check and the fill HLE's range check, see Draw.hpp) consult this + // list too. A draw holding an address-based build's lease (which pins guest allocations until + // the write-back releases it) runs free like the dispatch path's: the completion releases the + // lease, and a guest thread that needs a leased allocation syncs the recorder through the + // registry's pin waiter (see SyncLeaseWork); APS5_SYNC_LEASE_DISPATCH=1 waits for such draws + // at once as before. Debug aids: + // APS5_SYNC_COMPLETION_DRAWS=1 keeps every draw with completion work synchronous; + // APS5_NO_RECORDER_SYNC_DRAWS=1 gives the waited-for draws their own batch as before; + // APS5_NO_RECORD_COPIED_DRAWS=1 waits for copied-write draws at once instead of listing them. + static const bool syncCompletionDraws = std::getenv("APS5_SYNC_COMPLETION_DRAWS") != nullptr; + static const bool recorderSyncDraws = std::getenv("APS5_NO_RECORDER_SYNC_DRAWS") == nullptr; + static const bool recordCopiedDraws = std::getenv("APS5_NO_RECORD_COPIED_DRAWS") == nullptr; + const bool completion = resources->NeedsCompletion(); + const bool copiedWrites = completion && resources->HasCopiedWrites(); + // A lease only forces the wait when deferred release is off. + const bool lease = resources->HoldsLease() && SyncLeaseWork(); + outcome.completion = completion; + outcome.recorded = recordable; + // Whether the copied write-back is listed in DrawCopiedWriters instead of waited for. + bool listed = false; + if (!recordable) { + outcome.reason = !recordDraws || dumpLimit != 0 ? SyncDisabled : recorder == nullptr ? SyncNoRecorder : SyncNotResident; + } else if (completion && (syncCompletionDraws || copiedWrites || lease)) { + outcome.reason = syncCompletionDraws ? SyncDisabled : lease ? SyncLease : SyncCopiedWrites; + if (syncCompletionDraws) outcome.recorded = false; + else if (copiedWrites && !lease && recordCopiedDraws) listed = true; + else if (recorderSyncDraws) outcome.waited = true; + else outcome.recorded = false; + } + const bool recorded = outcome.recorded; + // A build this recorded draw can share with later identical ones goes into the cache (a cache + // hit is reusable by construction, so `recorded` holds for it; one with completion work is + // never reusable). + if (cacheable && built != nullptr && recorded && resources->Reusable()) SharedResourceCache().Insert(contentKey, resources); + std::optional batch; + if (!recorded) { + // The color-target detiles below take their descriptor sets from a fresh batch. + if (context.detiler != nullptr) context.detiler->BeginBatch(); + batch.emplace(context); + } + const auto commands = recorded ? recorder->Commands() : batch->Handle(); + APS5_LOG_CHARS_OUT_DEBUG("CommandBatch created"); VkMemoryBarrier upload{VK_STRUCTURE_TYPE_MEMORY_BARRIER}; upload.srcAccessMask = VK_ACCESS_HOST_WRITE_BIT; upload.dstAccessMask = VK_ACCESS_TRANSFER_READ_BIT | VK_ACCESS_INDEX_READ_BIT | VK_ACCESS_VERTEX_ATTRIBUTE_READ_BIT | VK_ACCESS_UNIFORM_READ_BIT | VK_ACCESS_SHADER_READ_BIT | VK_ACCESS_SHADER_WRITE_BIT; context.Function("vkCmdPipelineBarrier")(commands, VK_PIPELINE_STAGE_HOST_BIT, VK_PIPELINE_STAGE_TRANSFER_BIT | VK_PIPELINE_STAGE_VERTEX_INPUT_BIT | shaderStages, 0, 1, &upload, 0, nullptr, 0, nullptr); - if (storage->color) storage->color->Begin(commands); - pipeline.Begin(commands, state.renderExtent); - resources->Bind(commands, VK_PIPELINE_BIND_POINT_GRAPHICS, pipeline.Layout()); - pipeline.PushConstants(commands, shaders); + APS5_LOG_CHARS_OUT_DEBUG("Upload barrier recorded"); + for (auto& binding : targets) { + if (binding.resident != nullptr) { + // Earlier recorded work (dispatches, the previous draw) wrote the image in the general layout. + imageBarrier(context, commands, binding.resident->Image(), VK_IMAGE_LAYOUT_GENERAL, VK_IMAGE_LAYOUT_COLOR_ATTACHMENT_OPTIMAL, VK_PIPELINE_STAGE_ALL_COMMANDS_BIT, VK_PIPELINE_STAGE_COLOR_ATTACHMENT_OUTPUT_BIT, VK_ACCESS_SHADER_WRITE_BIT | VK_ACCESS_TRANSFER_WRITE_BIT, VK_ACCESS_COLOR_ATTACHMENT_READ_BIT | VK_ACCESS_COLOR_ATTACHMENT_WRITE_BIT); + continue; + } + VkBufferImageCopy copy{}; + copy.imageSubresource = {VK_IMAGE_ASPECT_COLOR_BIT, 0, 0, 1}; + copy.imageExtent = {binding.color.extent.width, binding.color.extent.height, 1}; + const VkBuffer linear = binding.gpuTiling ? binding.linearDevice->Handle() : binding.transfer->Handle(); + if (binding.gpuTiling) { + CopyBuffer(context, commands, binding.tiled->Handle(), 0, binding.tiledDevice->Handle(), 0, binding.original.size()); + memoryBarrier(context, commands, VK_PIPELINE_STAGE_TRANSFER_BIT, VK_PIPELINE_STAGE_COMPUTE_SHADER_BIT, VK_ACCESS_TRANSFER_WRITE_BIT, VK_ACCESS_SHADER_READ_BIT | VK_ACCESS_SHADER_WRITE_BIT); + context.detiler->Dispatch(commands, TextureTileMode::kR64KBX, binding.color.elementBytes, binding.tiledDevice->Handle(), 0, binding.linearDevice->Handle(), 0, binding.mip); + memoryBarrier(context, commands, VK_PIPELINE_STAGE_COMPUTE_SHADER_BIT, VK_PIPELINE_STAGE_TRANSFER_BIT, VK_ACCESS_SHADER_WRITE_BIT, VK_ACCESS_TRANSFER_READ_BIT); + } + imageBarrier(context, commands, binding.target->Image(), VK_IMAGE_LAYOUT_UNDEFINED, VK_IMAGE_LAYOUT_TRANSFER_DST_OPTIMAL, VK_PIPELINE_STAGE_TOP_OF_PIPE_BIT, VK_PIPELINE_STAGE_TRANSFER_BIT, 0, VK_ACCESS_TRANSFER_WRITE_BIT); + context.Function("vkCmdCopyBufferToImage")(commands, linear, binding.target->Image(), VK_IMAGE_LAYOUT_TRANSFER_DST_OPTIMAL, 1, ©); + imageBarrier(context, commands, binding.target->Image(), VK_IMAGE_LAYOUT_TRANSFER_DST_OPTIMAL, VK_IMAGE_LAYOUT_COLOR_ATTACHMENT_OPTIMAL, VK_PIPELINE_STAGE_TRANSFER_BIT, VK_PIPELINE_STAGE_COLOR_ATTACHMENT_OUTPUT_BIT, VK_ACCESS_TRANSFER_WRITE_BIT, VK_ACCESS_COLOR_ATTACHMENT_READ_BIT | VK_ACCESS_COLOR_ATTACHMENT_WRITE_BIT); + } + APS5_LOG_OUT_DEBUG("Beginning pipeline renderExtent=%ux%u", state.renderExtent.width, state.renderExtent.height); + pipeline->Begin(commands, *framebuffer, state.renderExtent, state.viewport, state.scissor); + APS5_LOG_CHARS_OUT_DEBUG("Pipeline Begin OK"); + resources->Bind(commands, VK_PIPELINE_BIND_POINT_GRAPHICS, pipeline->Layout()); + APS5_LOG_CHARS_OUT_DEBUG("Resources bound"); + pipeline->PushConstants(commands, shaders); + APS5_LOG_CHARS_OUT_DEBUG("Push constants recorded"); if (state.stages.mesh) { + APS5_LOG_OUT_DEBUG("vkCmdDrawMeshTasksEXT groups=%u instances=%u", meshGroups, draw.instanceCount); context.Function("vkCmdDrawMeshTasksEXT")(commands, meshGroups, draw.instanceCount, 1); } else { if (!vertexHandles.empty()) context.Function("vkCmdBindVertexBuffers")(commands, 0, static_cast(vertexHandles.size()), vertexHandles.data(), vertexOffsets.data()); @@ -127,14 +831,170 @@ void Draw(const Context& context, const State& state, const Pm4::DrawParameters& context.Function("vkCmdDraw")(commands, draw.indexCount, draw.instanceCount, draw.firstVertex, draw.firstInstance); } } + APS5_LOG_CHARS_OUT_DEBUG("Draw recorded"); context.Function("vkCmdEndRenderPass")(commands); + APS5_LOG_CHARS_OUT_DEBUG("Render pass ended"); + for (auto& binding : targets) { + if (binding.resident != nullptr) { + imageBarrier(context, commands, binding.resident->Image(), VK_IMAGE_LAYOUT_COLOR_ATTACHMENT_OPTIMAL, VK_IMAGE_LAYOUT_GENERAL, VK_PIPELINE_STAGE_COLOR_ATTACHMENT_OUTPUT_BIT, VK_PIPELINE_STAGE_ALL_COMMANDS_BIT, VK_ACCESS_COLOR_ATTACHMENT_WRITE_BIT, VK_ACCESS_SHADER_READ_BIT | VK_ACCESS_SHADER_WRITE_BIT | VK_ACCESS_TRANSFER_READ_BIT); + continue; + } + VkBufferImageCopy copy{}; + copy.imageSubresource = {VK_IMAGE_ASPECT_COLOR_BIT, 0, 0, 1}; + copy.imageExtent = {binding.color.extent.width, binding.color.extent.height, 1}; + imageBarrier(context, commands, binding.target->Image(), VK_IMAGE_LAYOUT_COLOR_ATTACHMENT_OPTIMAL, VK_IMAGE_LAYOUT_TRANSFER_SRC_OPTIMAL, VK_PIPELINE_STAGE_COLOR_ATTACHMENT_OUTPUT_BIT, VK_PIPELINE_STAGE_TRANSFER_BIT, VK_ACCESS_COLOR_ATTACHMENT_WRITE_BIT, VK_ACCESS_TRANSFER_READ_BIT); + const VkBuffer linear = binding.gpuTiling ? binding.linearDevice->Handle() : binding.transfer->Handle(); + VkBufferMemoryBarrier reuse{VK_STRUCTURE_TYPE_BUFFER_MEMORY_BARRIER}; + reuse.srcAccessMask = VK_ACCESS_TRANSFER_READ_BIT; + reuse.dstAccessMask = VK_ACCESS_TRANSFER_WRITE_BIT; + reuse.srcQueueFamilyIndex = VK_QUEUE_FAMILY_IGNORED; + reuse.dstQueueFamilyIndex = VK_QUEUE_FAMILY_IGNORED; + reuse.buffer = linear; + reuse.size = VK_WHOLE_SIZE; + context.Function("vkCmdPipelineBarrier")(commands, VK_PIPELINE_STAGE_TRANSFER_BIT, VK_PIPELINE_STAGE_TRANSFER_BIT, 0, 0, nullptr, 1, &reuse, 0, nullptr); + context.Function("vkCmdCopyImageToBuffer")(commands, binding.target->Image(), VK_IMAGE_LAYOUT_TRANSFER_SRC_OPTIMAL, linear, 1, ©); + if (binding.gpuTiling) { + memoryBarrier(context, commands, VK_PIPELINE_STAGE_TRANSFER_BIT, VK_PIPELINE_STAGE_COMPUTE_SHADER_BIT | VK_PIPELINE_STAGE_TRANSFER_BIT, VK_ACCESS_TRANSFER_WRITE_BIT, VK_ACCESS_SHADER_READ_BIT | VK_ACCESS_SHADER_WRITE_BIT | VK_ACCESS_TRANSFER_READ_BIT); + if (dumpLimit > 0) { + std::lock_guard lock(dumpMutex); + if (dumped[binding.color.address] < dumpLimit) { + binding.dump = std::make_unique(context, binding.linearDevice->Size(), VK_BUFFER_USAGE_TRANSFER_DST_BIT); + CopyBuffer(context, commands, binding.linearDevice->Handle(), 0, binding.dump->Handle(), 0, binding.linearDevice->Size()); + } + } + context.detiler->Dispatch(commands, TextureTileMode::kR64KBX, binding.color.elementBytes, binding.linearDevice->Handle(), 0, binding.tiledDevice->Handle(), 0, binding.mip, true); + memoryBarrier(context, commands, VK_PIPELINE_STAGE_COMPUTE_SHADER_BIT, VK_PIPELINE_STAGE_TRANSFER_BIT, VK_ACCESS_SHADER_WRITE_BIT, VK_ACCESS_TRANSFER_READ_BIT); + CopyBuffer(context, commands, binding.tiledDevice->Handle(), 0, binding.tiled->Handle(), 0, binding.original.size()); + } + } VkMemoryBarrier download{VK_STRUCTURE_TYPE_MEMORY_BARRIER}; download.srcAccessMask = VK_ACCESS_TRANSFER_WRITE_BIT | VK_ACCESS_SHADER_WRITE_BIT; download.dstAccessMask = VK_ACCESS_HOST_READ_BIT; context.Function("vkCmdPipelineBarrier")(commands, VK_PIPELINE_STAGE_TRANSFER_BIT | shaderStages, VK_PIPELINE_STAGE_HOST_BIT, 0, 1, &download, 0, nullptr, 0, nullptr); - timing.Mark("command_record"); - context.drawQueue->Enqueue(std::move(resources), std::move(storage)); - timing.Mark("enqueue"); + APS5_LOG_CHARS_OUT_DEBUG("Download barrier recorded"); + APS5_LOG_CHARS_OUT_DEBUG("SubmitAndWait"); + phase(PhaseRecord); + if (recorded) { + // Everything the recorded draw uses lives until its batch completes. + struct Kept { + std::shared_ptr resources; + std::shared_ptr pipeline; + std::shared_ptr framebuffer; + std::unique_ptr indices; + std::vector> vertexBuffers; + std::vector> targets; + }; + auto kept = std::make_shared(); + kept->resources = resources; + kept->pipeline = pipeline; + kept->framebuffer = framebuffer; + kept->indices = std::move(indices); + kept->vertexBuffers = std::move(vertexBuffers); + for (auto& binding : targets) kept->targets.push_back(binding.resident); + recorder->Keep(std::move(kept)); + // Counted for the [address-sync] line: a lease released by the completion (the pin waiter + // finishes the recorder up to the open batch, which holds the kept resources and gets the + // next serial), or waited for below (any wait releases it at once). + if (resources->HoldsLease()) CountLeaseOutcome(outcome.waited, outcome.waited ? 0 : recorder->Submissions() + 1); + // Every use of a (possibly shared) resources object registers its GPU writes anew. + resources->MarkGpuWrites(*recorder); + // The write-back (fault check, copied buffers) runs when the batch completed; a fault is + // reported by the recorder ("deferred write-back failed") instead of thrown out of the draw. + if (listed) { + // As VulkanDevice::dispatch lists its copied writers: delisted before the write-back + // (one that fails must not keep indirect dispatches on the CPU), listed after the + // registration (a throw there leaves nothing behind). + auto writers = DrawCopiedWriters(); + recorder->OnComplete([resources, writers] { + writers->erase(std::remove(writers->begin(), writers->end(), resources), writers->end()); + resources->WriteBackBuffers(); + }); + writers->push_back(resources); + } else if (completion) { + recorder->OnComplete([resources] { resources->WriteBackBuffers(); }); + } + for (auto& binding : targets) binding.resident->MarkDirty(); + phase(PhaseKeep); + if (outcome.waited) { + // The wait the dispatch path makes for such work (source 3, "address-based"): the + // write-back runs before the next packet, so the CPU never reads copied results or + // touches a leased allocation before they landed. + Recorder::CountSync(3); + const auto ownBefore = profile ? Recorder::ThreadWaitedMs() : 0.0; + recorder->Sync(); + if (profile) ownWaitedMs += Recorder::ThreadWaitedMs() - ownBefore; + phase(PhaseSync); + } + report(outcome.waited ? (outcome.reason == SyncLease ? " recorded then waited (lease)" : " recorded then waited (copied writes)") : completion ? " recorded with completion" : " recorded"); + return; + } + { + // The recorder sync SubmitAndWait makes is the draw's own wait, not a nested hook wait. + const auto ownBefore = profile ? Recorder::ThreadWaitedMs() : 0.0; + batch->SubmitAndWait(); + if (profile) ownWaitedMs += Recorder::ThreadWaitedMs() - ownBefore; + } + phase(PhaseSync); + APS5_LOG_CHARS_OUT_DEBUG("SubmitAndWait OK"); + for (const auto& binding : targets) GuestMemory::CheckRange(reinterpret_cast(binding.color.address), binding.color.bytes, 256, true); + for (auto& binding : targets) { + if (!binding.dump) continue; + std::lock_guard lock(dumpMutex); + char name[64]; + std::snprintf(name, sizeof(name), "target_%llx_%d.raw", static_cast(binding.color.address), dumped[binding.color.address]++); + if (std::FILE* file = std::fopen(name, "wb")) { + const std::uint32_t header[3] = {binding.color.extent.width, binding.color.extent.height, static_cast(binding.color.format)}; + std::fwrite(header, sizeof(header), 1, file); + std::fwrite(binding.dump->Bytes().data(), 1, binding.dump->Bytes().size(), file); + std::fclose(file); + } + } + APS5_LOG_CHARS_OUT_DEBUG("Shader resources WriteBack"); + resources->WriteBack(); + APS5_LOG_CHARS_OUT_DEBUG("Shader resources WriteBack OK"); + static const bool skipTargetWrite = std::getenv("APS5_NO_TARGET_WRITEBACK") != nullptr; + for (const auto& binding : targets) { + if (skipTargetWrite) break; + if (binding.resident != nullptr) { + // The results stay on the GPU until something reads the target's memory. + binding.resident->MarkDirty(); + continue; + } + if (binding.gpuTiling) GuestMemory::WriteChanged(binding.color.address, binding.tiled->Bytes(), binding.original); + else WriteColorTarget(binding.color, binding.transfer->Bytes()); + // The stored texels are the whole target now, so later reads must see them rather than a fast clear. + MarkDccUncompressed(binding.color.dccAddress, ColorTargetLayout(binding.color.extent.width, binding.color.extent.height, binding.color.tileMode, binding.color.elementBytes).Bytes()); + } + phase(PhaseWriteBack); + if (profile && traceDraws) { + // A synchronous draw's inputs and target sample are described as a rendering debug aid; + // the ~0.5 ms that takes (Describe samples every bound range and reads its DCC keys) is + // shown as its own phase and kept out of the plain profile. + std::size_t nonzero = 0; + std::size_t sampled = 0; + if (!targets.empty() && targets.front().resident == nullptr) { + const auto bytes = targets.front().gpuTiling ? targets.front().tiled->Bytes() : targets.front().transfer->Bytes(); + sampled = bytes.size() / 64; + for (std::size_t i = 0; i < bytes.size(); i += 64) nonzero += bytes[i] != std::byte{0}; + } + std::fprintf(stderr, "[draw] inputs:%s\n", resources->Describe().c_str()); + if (targets.size() > 1) { + std::string list; + for (const auto& binding : targets) { + char entry[64]; + std::snprintf(entry, sizeof(entry), " 0x%llx(%d,%ux%u)", static_cast(binding.color.address), static_cast(binding.color.format), binding.color.extent.width, binding.color.extent.height); + list += entry; + } + std::fprintf(stderr, "[draw] targets:%s\n", list.c_str()); + } + phase(PhaseDescribe); + char suffix[160]; + std::snprintf(suffix, sizeof(suffix), " synchronous (%s), first 0x%llx format %d (%zu of %zu sampled bytes nonzero)", SyncReasonNames[outcome.reason], static_cast(state.color.address), static_cast(state.color.format), nonzero, sampled); + report(suffix); + } else { + report(" synchronous"); + } + APS5_LOG_CHARS_OUT_DEBUG("Draw finished"); } } diff --git a/core/libs/prx/libSceAgcDriver/Graphics/src/GpuColorTransfer.cpp b/core/libs/prx/libSceAgcDriver/Graphics/src/GpuColorTransfer.cpp index 1f01f2768..55c236b72 100644 --- a/core/libs/prx/libSceAgcDriver/Graphics/src/GpuColorTransfer.cpp +++ b/core/libs/prx/libSceAgcDriver/Graphics/src/GpuColorTransfer.cpp @@ -2,8 +2,6 @@ #include "prx/libSceAgcDriver/Graphics/shaders/ColorTransfer_spv.h" #include "prx/libSceAgcDriver/Execution/include/GuestMemory.hpp" #include -#include -#include "prx/libSceAgcDriver/Execution/include/PerformanceTimer.hpp" namespace AgcDriver::Graphics { @@ -72,11 +70,11 @@ void GpuColorTransfer::prepare(std::uint32_t newWidth, std::uint32_t newHeight, Require(layout.Bytes() <= context.limits.maxStorageBufferRange && layout.LinearBytes() <= context.limits.maxStorageBufferRange, "color transfer exceeds storage buffer limits"); Require((newWidth + 7u) / 8u <= context.limits.maxComputeWorkGroupCount[0] && (newHeight + 7u) / 8u <= context.limits.maxComputeWorkGroupCount[1], "color transfer exceeds workgroup limits"); if (tiled && width == newWidth && height == newHeight && mode == newMode) return; - auto newTiled = std::make_unique(context, layout.Bytes(), VK_BUFFER_USAGE_STORAGE_BUFFER_BIT | VK_BUFFER_USAGE_TRANSFER_SRC_BIT | VK_BUFFER_USAGE_TRANSFER_DST_BIT, VK_MEMORY_PROPERTY_DEVICE_LOCAL_BIT); - auto newLinear = std::make_unique(context, layout.LinearBytes(), VK_BUFFER_USAGE_STORAGE_BUFFER_BIT | VK_BUFFER_USAGE_TRANSFER_SRC_BIT | VK_BUFFER_USAGE_TRANSFER_DST_BIT, VK_MEMORY_PROPERTY_DEVICE_LOCAL_BIT); - auto newReadback = std::make_unique(context, layout.Bytes(), VK_BUFFER_USAGE_TRANSFER_DST_BIT, VK_MEMORY_PROPERTY_HOST_VISIBLE_BIT | VK_MEMORY_PROPERTY_HOST_CACHED_BIT); - auto newUpload = std::make_unique(context, layout.Bytes(), VK_BUFFER_USAGE_TRANSFER_SRC_BIT); - const std::array buffers{{{newTiled->Handle(), 0, layout.Bytes()}, {newLinear->Handle(), 0, layout.LinearBytes()}}}; + constexpr VkBufferUsageFlags copies = VK_BUFFER_USAGE_TRANSFER_SRC_BIT | VK_BUFFER_USAGE_TRANSFER_DST_BIT; + auto newTiled = std::make_unique(context, layout.Bytes(), copies); + auto newTiledDevice = std::make_unique(context, layout.Bytes(), copies | VK_BUFFER_USAGE_STORAGE_BUFFER_BIT); + auto newLinear = std::make_unique(context, layout.LinearBytes(), copies | VK_BUFFER_USAGE_STORAGE_BUFFER_BIT); + const std::array buffers{{{newTiledDevice->Handle(), 0, layout.Bytes()}, {newLinear->Handle(), 0, layout.LinearBytes()}}}; std::array writes{}; for (std::uint32_t i = 0; i < writes.size(); ++i) { writes[i].sType = VK_STRUCTURE_TYPE_WRITE_DESCRIPTOR_SET; @@ -88,31 +86,30 @@ void GpuColorTransfer::prepare(std::uint32_t newWidth, std::uint32_t newHeight, } context.Function("vkUpdateDescriptorSets")(context.device, static_cast(writes.size()), writes.data(), 0, nullptr); tiled = std::move(newTiled); + tiledDevice = std::move(newTiledDevice); linear = std::move(newLinear); - readback = std::move(newReadback); - upload = std::move(newUpload); width = newWidth; height = newHeight; mode = newMode; } void GpuColorTransfer::Upload(std::uint64_t address, std::uint32_t newWidth, std::uint32_t newHeight, ColorTileMode newMode) { - PerformanceTimer timing("ColorTransfer.Upload"); prepare(newWidth, newHeight, newMode); - timing.Mark("prepare"); const ColorTargetLayout layout(width, height, mode); - GuestMemory::Read(address, upload->Bytes(), layout.Alignment()); - timing.Mark("guest_read", layout.Bytes()); + GuestMemory::Read(address, tiled->Bytes(), layout.Alignment()); } -void GpuColorTransfer::convert(VkCommandBuffer commands, bool toTiled, bool swapRedBlue) { - Require(commands != VK_NULL_HANDLE && tiled && linear, "color transfer is not prepared"); +void GpuColorTransfer::convert(VkCommandBuffer commands, bool toTiled, bool swapRedBlue, bool tenBit) { + Require(commands != VK_NULL_HANDLE && tiled && tiledDevice && linear, "color transfer is not prepared"); + // The guest bytes move to video memory with one DMA copy; the shader's scattered accesses stay local. + RecordMemoryBarrier(context, commands, VK_PIPELINE_STAGE_HOST_BIT | VK_PIPELINE_STAGE_TRANSFER_BIT | VK_PIPELINE_STAGE_COMPUTE_SHADER_BIT, VK_PIPELINE_STAGE_TRANSFER_BIT, VK_ACCESS_HOST_WRITE_BIT | VK_ACCESS_TRANSFER_WRITE_BIT | VK_ACCESS_TRANSFER_READ_BIT | VK_ACCESS_SHADER_READ_BIT | VK_ACCESS_SHADER_WRITE_BIT, VK_ACCESS_TRANSFER_READ_BIT | VK_ACCESS_TRANSFER_WRITE_BIT); + CopyBuffer(context, commands, tiled->Handle(), 0, tiledDevice->Handle(), 0, tiledDevice->Size()); VkMemoryBarrier before{VK_STRUCTURE_TYPE_MEMORY_BARRIER}; - before.srcAccessMask = VK_ACCESS_HOST_WRITE_BIT | VK_ACCESS_TRANSFER_WRITE_BIT | VK_ACCESS_TRANSFER_READ_BIT | VK_ACCESS_SHADER_READ_BIT | VK_ACCESS_SHADER_WRITE_BIT; + before.srcAccessMask = VK_ACCESS_TRANSFER_WRITE_BIT | VK_ACCESS_TRANSFER_READ_BIT | VK_ACCESS_SHADER_READ_BIT | VK_ACCESS_SHADER_WRITE_BIT; before.dstAccessMask = VK_ACCESS_SHADER_READ_BIT | VK_ACCESS_SHADER_WRITE_BIT; const auto barrier = context.Function("vkCmdPipelineBarrier"); - barrier(commands, VK_PIPELINE_STAGE_HOST_BIT | VK_PIPELINE_STAGE_TRANSFER_BIT | VK_PIPELINE_STAGE_COMPUTE_SHADER_BIT, VK_PIPELINE_STAGE_COMPUTE_SHADER_BIT, 0, 1, &before, 0, nullptr, 0, nullptr); - const std::array push{width, height, (width + 127u) / 128u, (toTiled ? 1u : 0u) | (swapRedBlue ? 2u : 0u) | (mode == ColorTileMode::RenderTarget ? 4u : 0u)}; + barrier(commands, VK_PIPELINE_STAGE_TRANSFER_BIT | VK_PIPELINE_STAGE_COMPUTE_SHADER_BIT, VK_PIPELINE_STAGE_COMPUTE_SHADER_BIT, 0, 1, &before, 0, nullptr, 0, nullptr); + const std::array push{width, height, (width + 127u) / 128u, (toTiled ? 1u : 0u) | (swapRedBlue ? 2u : 0u) | (mode == ColorTileMode::RenderTarget ? 4u : 0u) | (tenBit ? 8u : 0u)}; context.Function("vkCmdBindPipeline")(commands, VK_PIPELINE_BIND_POINT_COMPUTE, pipeline); context.Function("vkCmdBindDescriptorSets")(commands, VK_PIPELINE_BIND_POINT_COMPUTE, pipelineLayout, 0, 1, &descriptorSet, 0, nullptr); context.Function("vkCmdPushConstants")(commands, pipelineLayout, VK_SHADER_STAGE_COMPUTE_BIT, 0, sizeof(push), push.data()); @@ -121,32 +118,24 @@ void GpuColorTransfer::convert(VkCommandBuffer commands, bool toTiled, bool swap after.srcAccessMask = VK_ACCESS_SHADER_WRITE_BIT; after.dstAccessMask = VK_ACCESS_TRANSFER_READ_BIT; barrier(commands, VK_PIPELINE_STAGE_COMPUTE_SHADER_BIT, VK_PIPELINE_STAGE_TRANSFER_BIT, 0, 1, &after, 0, nullptr, 0, nullptr); + if (toTiled) { + CopyBuffer(context, commands, tiledDevice->Handle(), 0, tiled->Handle(), 0, tiledDevice->Size()); + RecordMemoryBarrier(context, commands, VK_PIPELINE_STAGE_TRANSFER_BIT, VK_PIPELINE_STAGE_HOST_BIT, VK_ACCESS_TRANSFER_WRITE_BIT, VK_ACCESS_HOST_READ_BIT); + } } -void GpuColorTransfer::Detile(VkCommandBuffer commands, bool swapRedBlue) { - Require(commands != VK_NULL_HANDLE && upload && tiled, "color upload is not prepared"); - VkMemoryBarrier before{VK_STRUCTURE_TYPE_MEMORY_BARRIER}; - before.srcAccessMask = VK_ACCESS_HOST_WRITE_BIT | VK_ACCESS_MEMORY_READ_BIT | VK_ACCESS_MEMORY_WRITE_BIT; - before.dstAccessMask = VK_ACCESS_TRANSFER_READ_BIT | VK_ACCESS_TRANSFER_WRITE_BIT; - context.Function("vkCmdPipelineBarrier")(commands, VK_PIPELINE_STAGE_HOST_BIT | VK_PIPELINE_STAGE_ALL_COMMANDS_BIT, VK_PIPELINE_STAGE_TRANSFER_BIT, 0, 1, &before, 0, nullptr, 0, nullptr); - const VkBufferCopy copy{0, 0, upload->Bytes().size()}; - context.Function("vkCmdCopyBuffer")(commands, upload->Handle(), tiled->Handle(), 1, ©); - convert(commands, false, swapRedBlue); +void GpuColorTransfer::Detile(VkCommandBuffer commands, bool swapRedBlue, bool tenBit) { + convert(commands, false, swapRedBlue, tenBit); } void GpuColorTransfer::Tile(VkCommandBuffer commands) { convert(commands, true, false); - VkMemoryBarrier before{VK_STRUCTURE_TYPE_MEMORY_BARRIER}; - before.srcAccessMask = VK_ACCESS_SHADER_WRITE_BIT; - before.dstAccessMask = VK_ACCESS_TRANSFER_READ_BIT; - const auto barrier = context.Function("vkCmdPipelineBarrier"); - barrier(commands, VK_PIPELINE_STAGE_COMPUTE_SHADER_BIT, VK_PIPELINE_STAGE_TRANSFER_BIT, 0, 1, &before, 0, nullptr, 0, nullptr); - const VkBufferCopy copy{0, 0, readback->Bytes().size()}; - context.Function("vkCmdCopyBuffer")(commands, tiled->Handle(), readback->Handle(), 1, ©); - VkMemoryBarrier after{VK_STRUCTURE_TYPE_MEMORY_BARRIER}; - after.srcAccessMask = VK_ACCESS_TRANSFER_WRITE_BIT; - after.dstAccessMask = VK_ACCESS_HOST_READ_BIT; - barrier(commands, VK_PIPELINE_STAGE_TRANSFER_BIT, VK_PIPELINE_STAGE_HOST_BIT, 0, 1, &after, 0, nullptr, 0, nullptr); +} + +void GpuColorTransfer::WriteBack(std::uint64_t address) { + Require(tiled != nullptr, "color transfer is not prepared for writeback"); + const ColorTargetLayout layout(width, height, mode); + GuestMemory::Write(address, tiled->Bytes(), layout.Alignment()); } VkBuffer GpuColorTransfer::LinearBuffer() const { @@ -154,13 +143,6 @@ VkBuffer GpuColorTransfer::LinearBuffer() const { return linear->Handle(); } -bool GpuColorTransfer::MatchesGuest(std::uint64_t address) { - Require(readback != nullptr, "color readback is unavailable"); - const auto bytes = readback->Bytes(); - GuestMemory::CheckRange(reinterpret_cast(address), bytes.size(), 1); - return std::memcmp(reinterpret_cast(address), bytes.data(), bytes.size()) == 0; -} - RenderTarget& GpuColorTransfer::Target(const ColorTarget& color, bool blending) { if (!target || targetExtent.width != color.extent.width || targetExtent.height != color.extent.height || targetFormat != color.format || targetBlending != blending) { auto replacement = std::make_unique(context, color, blending); diff --git a/core/libs/prx/libSceAgcDriver/Graphics/src/GraphicsPipelineCache.cpp b/core/libs/prx/libSceAgcDriver/Graphics/src/GraphicsPipelineCache.cpp deleted file mode 100644 index 7414c171c..000000000 --- a/core/libs/prx/libSceAgcDriver/Graphics/src/GraphicsPipelineCache.cpp +++ /dev/null @@ -1,138 +0,0 @@ -#include "prx/libSceAgcDriver/Graphics/include/GraphicsPipelineCache.hpp" -#include "prx/libSceAgcDriver/Graphics/include/VertexInput.hpp" -#include "prx/libSceAgcDriver/Execution/include/PerformanceTimer.hpp" -#include -#include -#include - -namespace AgcDriver::Graphics { -namespace { - -template -void append(std::string& key, const TValue& value) { - static_assert(std::is_trivially_copyable_v); - key.append(reinterpret_cast(&value), sizeof(value)); -} - -std::string makeKey(const Context& context, const State& state, const std::shared_ptr& target, const ShaderResources& resources, std::span shaders) { - PerformanceTimer timing("Graphics.PipelineKey"); - std::string key; - key.reserve(512); - append(key, target ? target->Target().View() : VK_NULL_HANDLE); - append(key, state.hasColorTarget); - append(key, state.rectList); - append(key, state.renderExtent.width); - append(key, state.renderExtent.height); - append(key, state.topology); - append(key, state.viewport.x); - append(key, state.viewport.y); - append(key, state.viewport.width); - append(key, state.viewport.height); - append(key, state.viewport.minDepth); - append(key, state.viewport.maxDepth); - append(key, state.negativeOneToOne); - append(key, state.scissor.offset.x); - append(key, state.scissor.offset.y); - append(key, state.scissor.extent.width); - append(key, state.scissor.extent.height); - append(key, state.cullMode); - append(key, state.frontFace); - append(key, state.blend.blendEnable); - append(key, state.blend.srcColorBlendFactor); - append(key, state.blend.dstColorBlendFactor); - append(key, state.blend.colorBlendOp); - append(key, state.blend.srcAlphaBlendFactor); - append(key, state.blend.dstAlphaBlendFactor); - append(key, state.blend.alphaBlendOp); - append(key, state.blend.colorWriteMask); - for (const auto value : state.blendConstants) append(key, value); - append(key, state.stages.mesh.has_value()); - append(key, state.stages.tessellation.has_value()); - if (state.stages.mesh) { - const auto& mesh = *state.stages.mesh; - append(key, mesh.inputPrimitive); - append(key, mesh.primitivesPerGroup); - append(key, mesh.verticesPerGroup); - append(key, mesh.maxVertices); - append(key, mesh.maxPrimitives); - append(key, mesh.threadsPerGroup); - append(key, mesh.ldsSizeDwords); - append(key, mesh.provokingVertex); - } - if (state.stages.tessellation) { - const auto& tessellation = *state.stages.tessellation; - append(key, tessellation.inputControlPoints); - append(key, tessellation.outputControlPoints); - append(key, tessellation.domain); - append(key, tessellation.partitioning); - append(key, tessellation.outputTopology); - } - timing.Mark("state"); - const auto input = BuildVertexInputLayout(context, shaders.front().program->vertexAttributes); - timing.Mark("vertex_layout"); - append(key, input.bindings.size()); - for (const auto& binding : input.bindings) { - append(key, binding.binding); - append(key, binding.stride); - append(key, binding.inputRate); - } - append(key, input.attributes.size()); - for (const auto& attribute : input.attributes) { - append(key, attribute.location); - append(key, attribute.binding); - append(key, attribute.format); - append(key, attribute.offset); - } - append(key, resources.LayoutKey().size()); - const auto layout = std::span(resources.LayoutKey()); - if (!layout.empty()) key.append(reinterpret_cast(layout.data()), layout.size_bytes()); - append(key, PushConstantStages(shaders)); - append(key, shaders.size()); - timing.Mark("resources"); - std::uint64_t shaderBytes = 0; - for (const auto& shader : shaders) { - append(key, shader.stage); - append(key, shader.program->spirv.size()); - const auto bytes = std::as_bytes(std::span(shader.program->spirv)); - if (!bytes.empty()) key.append(reinterpret_cast(bytes.data()), bytes.size()); - shaderBytes += bytes.size(); - } - timing.Mark("shader_code", shaderBytes); - return key; -} - -} - -std::shared_ptr GraphicsPipelineCache::Get(const State& state, const std::shared_ptr& target, const ShaderResources& resources, std::span shaders) { - PerformanceTimer timing("Graphics.PipelineCache"); - auto key = makeKey(context, state, target, resources, shaders); - timing.Mark("key"); - const auto found = lookup.find(key); - if (found != lookup.end()) { - const auto it = found->second; - auto pipeline = it->pipeline; - entries.splice(entries.end(), entries, it); - timing.Mark("hit"); - return pipeline; - } - timing.Mark("miss"); - auto pipeline = std::make_shared(context, state, target ? &target->Target() : nullptr, resources, shaders); - timing.Mark("create"); - entries.push_back({std::move(key), target, pipeline}); - try { - const auto it = std::prev(entries.end()); - Require(lookup.emplace(it->key, it).second, "duplicate graphics pipeline cache key"); - } catch (...) { - entries.pop_back(); - throw; - } - while (entries.size() > 128) { - const auto it = std::find_if(entries.begin(), entries.end(), [](const auto& entry) { return entry.pipeline.use_count() == 1; }); - if (it == entries.end()) break; - lookup.erase(it->key); - entries.erase(it); - } - return pipeline; -} - -} diff --git a/core/libs/prx/libSceAgcDriver/Graphics/src/GuestBufferMemory.cpp b/core/libs/prx/libSceAgcDriver/Graphics/src/GuestBufferMemory.cpp index b2a6eeb03..3d0c25004 100644 --- a/core/libs/prx/libSceAgcDriver/Graphics/src/GuestBufferMemory.cpp +++ b/core/libs/prx/libSceAgcDriver/Graphics/src/GuestBufferMemory.cpp @@ -1,51 +1,969 @@ #include "prx/libSceAgcDriver/Graphics/include/GuestBufferMemory.hpp" +#include "prx/libSceAgcDriver/Graphics/include/BdaResources.hpp" +#include "prx/libSceAgcDriver/Graphics/include/Texture.hpp" +#include "prx/libSceAgcDriver/Graphics/include/Recorder.hpp" #include "prx/libSceAgcDriver/Execution/include/GuestMemory.hpp" +#ifdef _WIN32 +#include +#endif +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include #include #include #include +#include namespace AgcDriver::Graphics { +// Image mirrors. The main guest image (the PE at 0x7ff6b...) is MEM_IMAGE memory, which +// VK_EXT_external_memory_host refuses, and address-based shaders map every registered range, so each +// of their builds copied the whole image (~55 MiB: ~40 MiB of read-only sections snapshotted and copied +// again, ~15 MiB of writable ones copied twice). Instead one host-cached device buffer per registered +// image range persists here. Read-only ranges are filled once: their pages change only through an +// mprotect, which replaces the registered Range object, and a mirror whose Range is gone is rebuilt. +// Writable ranges keep a shadow of the guest bytes and copy the 64 KiB blocks that differ on each use. +// A mirror never holds the Range shared_ptr: a pinned range makes the guest's mprotect spin and fail. +struct ImageMirror { + std::uint64_t base = 0; + std::uint64_t bytes = 0; + bool writable = false; + std::weak_ptr range; + std::shared_ptr buffer; + // Writable ranges only: the guest bytes the mirror holds, compared on refresh and at write-back. + std::vector shadow; + // Identity for the life of this mirror (see ImageMirrorSerial); the top bit keeps mirror serials + // apart from host import serials. + std::uint64_t serial = 0; +}; + +namespace { + +// Imports persist across draws, keyed by allocation base, and are dropped when their allocation leaves +// the registered set; a dropped import is destroyed once the recorded work that may read it completed. +struct HostImports { + std::mutex mutex; + VkDevice device = VK_NULL_HANDLE; + std::map imports; + std::set failed; + // Registry generation the imports were last reconciled with. + std::uint64_t refreshedGeneration = 0; + // Bumped whenever an import is dropped, so HostImport pointers taken under the lock earlier can be + // reused while it is unchanged. + std::uint64_t epoch = 1; +}; + +HostImports& Imports() { + static HostImports imports; + return imports; +} + +void destroyImport(const Context& context, const HostImport& entry) { + context.Function("vkDestroyBuffer")(context.device, entry.buffer, nullptr); + context.Function("vkFreeMemory")(context.device, entry.memory, nullptr); +} + +// Frees a dropped import's Vulkan objects when it is released. Never copied: a copy would destroy the +// same handles twice (and a temporary would destroy them at once). +struct RetiredImport { + RetiredImport(const Context& context, const HostImport& entry) : context(context), entry(entry) {} + RetiredImport(const RetiredImport&) = delete; + RetiredImport& operator=(const RetiredImport&) = delete; + ~RetiredImport() { destroyImport(context, entry); } + Context context; + HostImport entry; +}; + +// Drops an import. Recorded batches (dispatches, recorded draws, GPU label writes) may still read it, +// so the objects live until the batch open now completed; with no batches in flight all GPU work that +// used it was synchronous and it is destroyed at once. +void retireImport(const Context& context, HostImports& state, std::map::iterator it) { + auto holder = std::make_shared(context, it->second); + if (auto* recorder = Recorder::Active(); recorder != nullptr && !recorder->Idle()) recorder->Keep(std::move(holder)); + ++state.epoch; + state.imports.erase(it); +} + +const HostImport* importAllocation(const Context& context, HostImports& state, std::uint64_t base, std::uint64_t bytes) { + if (const auto found = state.imports.find(base); found != state.imports.end()) { + if (found->second.bytes == bytes) return &found->second; + retireImport(context, state, found); + } + const auto alignment = context.hostImportAlignment; + if (alignment == 0 || base % alignment != 0 || bytes % alignment != 0 || state.failed.contains(base)) return nullptr; + // Pinned imports count against the driver's system memory budget; past it ordinary host + // allocations fail, so imports stop at APS5_HOST_IMPORT_MIB (default 6 GiB, which covers the + // registered memory of address-based shaders; past it they copy gigabytes per dispatch). + static const std::uint64_t budget = [] { + const char* value = std::getenv("APS5_HOST_IMPORT_MIB"); + return (value ? std::strtoull(value, nullptr, 10) : 6144ull) << 20u; + }(); + std::uint64_t live = 0; + for (const auto& [address, existing] : state.imports) live += existing.bytes; + if (live + bytes > budget) { + // A refused import turns every later use of the range into CPU copies, so say so. + static std::uint64_t refused = 0, refusedBytes = 0; + static auto lastReport = std::chrono::steady_clock::now() - std::chrono::seconds(60); + ++refused; + refusedBytes += bytes; + if (std::chrono::steady_clock::now() - lastReport > std::chrono::seconds(10)) { + lastReport = std::chrono::steady_clock::now(); + std::fprintf(stderr, "[gpu] host import budget: %llu MiB live of %llu MiB (APS5_HOST_IMPORT_MIB); %llu imports (%llu MiB) refused so far, last 0x%llx+0x%llx\n", static_cast(live >> 20u), static_cast(budget >> 20u), static_cast(refused), static_cast(refusedBytes >> 20u), static_cast(base), static_cast(bytes)); + } + return nullptr; + } +#ifdef _WIN32 + // Drivers pin imported pages, so every page must be committed and accessible. + for (std::uint64_t cursor = base; cursor < base + bytes;) { + MEMORY_BASIC_INFORMATION info{}; + if (VirtualQuery(reinterpret_cast(cursor), &info, sizeof(info)) == 0 || info.State != MEM_COMMIT || (info.Protect & (PAGE_NOACCESS | PAGE_GUARD)) != 0) { + state.failed.insert(base); + return nullptr; + } + cursor = reinterpret_cast(info.BaseAddress) + info.RegionSize; + } +#endif + HostImport entry{base, bytes, VK_NULL_HANDLE, VK_NULL_HANDLE, 0}; + const auto fail = [&](const char* step, VkResult result) -> const HostImport* { + if (entry.buffer) context.Function("vkDestroyBuffer")(context.device, entry.buffer, nullptr); + if (entry.memory) context.Function("vkFreeMemory")(context.device, entry.memory, nullptr); + state.failed.insert(base); + std::fprintf(stderr, "[gpu] host import of 0x%llx+0x%llx failed at %s (%d); falling back to copies\n", static_cast(base), static_cast(bytes), step, static_cast(result)); +#ifdef _WIN32 + static const bool trace = std::getenv("APS5_TRACE_HOST_IMPORT") != nullptr; + for (std::uint64_t cursor = base; trace && cursor < base + bytes;) { + MEMORY_BASIC_INFORMATION info{}; + if (VirtualQuery(reinterpret_cast(cursor), &info, sizeof(info)) == 0) break; + const auto regionEnd = reinterpret_cast(info.BaseAddress) + info.RegionSize; + std::fprintf(stderr, "[gpu] 0x%llx+0x%llx state 0x%lx protect 0x%lx type 0x%lx allocation 0x%llx\n", static_cast(cursor), static_cast(std::min(regionEnd, base + bytes) - cursor), info.State, info.Protect, info.Type, reinterpret_cast(info.AllocationBase)); + cursor = regionEnd; + } +#endif + return nullptr; + }; + const VkExternalMemoryBufferCreateInfo external{VK_STRUCTURE_TYPE_EXTERNAL_MEMORY_BUFFER_CREATE_INFO, nullptr, VK_EXTERNAL_MEMORY_HANDLE_TYPE_HOST_ALLOCATION_BIT_EXT}; + VkBufferCreateInfo info{VK_STRUCTURE_TYPE_BUFFER_CREATE_INFO, &external}; + info.size = bytes; + // INDIRECT_BUFFER: DISPATCH_INDIRECT group counts are read in place (VulkanDevice::DispatchIndirect). + info.usage = VK_BUFFER_USAGE_STORAGE_BUFFER_BIT | VK_BUFFER_USAGE_SHADER_DEVICE_ADDRESS_BIT | VK_BUFFER_USAGE_TRANSFER_SRC_BIT | VK_BUFFER_USAGE_TRANSFER_DST_BIT | VK_BUFFER_USAGE_INDIRECT_BUFFER_BIT; + info.sharingMode = VK_SHARING_MODE_EXCLUSIVE; + if (const auto result = context.Function("vkCreateBuffer")(context.device, &info, nullptr, &entry.buffer); result != VK_SUCCESS) return fail("vkCreateBuffer", result); + VkMemoryHostPointerPropertiesEXT pointer{VK_STRUCTURE_TYPE_MEMORY_HOST_POINTER_PROPERTIES_EXT}; + if (const auto result = context.Function("vkGetMemoryHostPointerPropertiesEXT")(context.device, VK_EXTERNAL_MEMORY_HANDLE_TYPE_HOST_ALLOCATION_BIT_EXT, reinterpret_cast(base), &pointer); result != VK_SUCCESS) return fail("vkGetMemoryHostPointerPropertiesEXT", result); + VkMemoryRequirements requirements{}; + context.Function("vkGetBufferMemoryRequirements")(context.device, entry.buffer, &requirements); + const auto types = requirements.memoryTypeBits & pointer.memoryTypeBits; + if (types == 0) return fail("memory type selection", VK_ERROR_FORMAT_NOT_SUPPORTED); + const VkImportMemoryHostPointerInfoEXT import{VK_STRUCTURE_TYPE_IMPORT_MEMORY_HOST_POINTER_INFO_EXT, nullptr, VK_EXTERNAL_MEMORY_HANDLE_TYPE_HOST_ALLOCATION_BIT_EXT, reinterpret_cast(base)}; + const VkMemoryAllocateFlagsInfo flags{VK_STRUCTURE_TYPE_MEMORY_ALLOCATE_FLAGS_INFO, &import, VK_MEMORY_ALLOCATE_DEVICE_ADDRESS_BIT, 0}; + VkMemoryAllocateInfo allocation{VK_STRUCTURE_TYPE_MEMORY_ALLOCATE_INFO, &flags}; + allocation.allocationSize = bytes; + allocation.memoryTypeIndex = static_cast(std::countr_zero(types)); + if (const auto result = context.Function("vkAllocateMemory")(context.device, &allocation, nullptr, &entry.memory); result != VK_SUCCESS) return fail("vkAllocateMemory", result); + if (const auto result = context.Function("vkBindBufferMemory")(context.device, entry.buffer, entry.memory, 0); result != VK_SUCCESS) return fail("vkBindBufferMemory", result); + const VkBufferDeviceAddressInfo addressInfo{VK_STRUCTURE_TYPE_BUFFER_DEVICE_ADDRESS_INFO, nullptr, entry.buffer}; + entry.address = context.Function("vkGetBufferDeviceAddressKHR")(context.device, &addressInfo); + if (entry.address == 0) return fail("vkGetBufferDeviceAddressKHR", VK_ERROR_UNKNOWN); + static std::uint64_t importedBytes = 0; + importedBytes += bytes; + static const bool trace = std::getenv("APS5_TRACE_HOST_IMPORT") != nullptr; + std::uint64_t liveBytes = bytes; + for (const auto& [address, existing] : state.imports) liveBytes += existing.bytes; + if (trace) std::fprintf(stderr, "[gpu] host import of 0x%llx+0x%llx ok (%zu live, %.1f MiB live, %.1f MiB ever)\n", static_cast(base), static_cast(bytes), state.imports.size() + 1, liveBytes / 1048576.0, importedBytes / 1048576.0); + return &state.imports.emplace(base, entry).first->second; +} + +// The lease is in address order (it is built from the registry map), so a range is found by binary search. +const GuestAllocations::Range* leasedRangeAt(const GuestAllocations::Lease& lease, std::uint64_t base) { + const auto found = std::lower_bound(lease.begin(), lease.end(), base, [](const auto& range, std::uint64_t value) { return range->address < value; }); + return found != lease.end() && (*found)->address == base ? found->get() : nullptr; +} + +// The readable registered range containing [begin, end), or null. +const GuestAllocations::Range* containingRange(const GuestAllocations::Lease& lease, std::uint64_t begin, std::uint64_t end) { + const auto found = std::upper_bound(lease.begin(), lease.end(), begin, [](std::uint64_t value, const auto& range) { return value < range->address; }); + if (found == lease.begin()) return nullptr; + const auto& range = *std::prev(found); + return range->readable && begin >= range->address && end <= range->address + range->bytes ? range.get() : nullptr; +} + +// Drops imports whose registered range changed or disappeared: their pages may no longer back the +// guest addresses. Walks the imports only when the registry changed since the last walk. +void refreshImports(const Context& context, HostImports& state, const GuestAllocations::Lease& lease) { + if (state.device != context.device) { + state.imports.clear(); + state.failed.clear(); + state.device = context.device; + state.refreshedGeneration = 0; + ++state.epoch; + } + const auto generation = GuestAllocations::GuestAllocationsGeneration_nid_postfix(); + if (generation == state.refreshedGeneration) return; + state.refreshedGeneration = generation; + for (auto it = state.imports.begin(); it != state.imports.end();) { + const auto* range = leasedRangeAt(lease, it->first); + if (range != nullptr && range->bytes == it->second.bytes) { + ++it; + continue; + } + const auto next = std::next(it); + retireImport(context, state, it); + it = next; + } + for (auto it = state.failed.begin(); it != state.failed.end();) it = leasedRangeAt(lease, *it) != nullptr ? std::next(it) : state.failed.erase(it); +} + +const HostImport* findImport(HostImports& state, std::uint64_t begin, std::uint64_t end) { + auto found = state.imports.upper_bound(begin); + if (found == state.imports.begin()) return nullptr; + --found; + const auto& entry = found->second; + return begin >= entry.base && end <= entry.base + entry.bytes ? &entry : nullptr; +} + +// Whether the imports need reconciling before a lookup: the registry changed since the last walk. +bool importsStale(const Context& context, const HostImports& state) { + return state.device != context.device || GuestAllocations::GuestAllocationsGeneration_nid_postfix() != state.refreshedGeneration; +} + +// The mirror registry: by range base. The map is protected by `mutex` (findMirror runs in a build's +// unlocked stage A, UploadPrepare); the mirrors' contents (fills, refreshes, sweeps) change under +// GuestMemory::GpuMutex only (every completion and every stage B holds it). APS5_NO_LEASE_MIRROR=1 +// restores copies. +struct ImageMirrors { + std::mutex mutex; + VkDevice device = VK_NULL_HANDLE; + std::map> entries; + std::set failed; + // APS5_PROFILE_DRAW counters, reported every 10 s as [buffers] image mirrors. + std::uint64_t builds = 0; + std::uint64_t subranges = 0; + std::uint64_t rebuilds = 0; + std::uint64_t refreshes = 0; + std::uint64_t blocksCompared = 0; + std::uint64_t blocksCopied = 0; + // Refreshes that had to wait for recorded work writing the range. + std::uint64_t syncs = 0; + std::uint64_t serials = 0; + std::chrono::steady_clock::time_point lastReport = std::chrono::steady_clock::now(); +}; + +ImageMirrors& Mirrors() { + static ImageMirrors mirrors; + return mirrors; +} + +bool mirrorsEnabled() { + static const bool disabled = std::getenv("APS5_NO_LEASE_MIRROR") != nullptr; + return !disabled; +} + +bool sameRange(const std::weak_ptr& mirrored, const std::shared_ptr& range) { + return !mirrored.owner_before(range) && !range.owner_before(mirrored); +} + +// One 64 KiB (or shorter, at a range's ends) block of a writable mirror to compare with guest memory. +struct RefreshBlock { + ImageMirror* mirror; + std::uint64_t address; + std::size_t length; +}; + +// Compares a block's guest bytes with the mirror's shadow and copies a changed block into the device +// buffer and the shadow. Blocks are distinct, so blocks of one refresh can be compared in parallel. +bool compareBlock(const RefreshBlock& block) { + const auto offset = static_cast(block.address - block.mirror->base); + const auto* guest = reinterpret_cast(block.address); + if (std::memcmp(block.mirror->shadow.data() + offset, guest, block.length) == 0) return false; + std::memcpy(block.mirror->buffer->Bytes().data() + offset, guest, block.length); + std::memcpy(block.mirror->shadow.data() + offset, guest, block.length); + return true; +} + +// Helper threads for the block compares. An address-based build refreshes every writable image range +// (~15 MiB in ~300 blocks) under the device lock, and that memcmp was most of its cost; the blocks +// are handed out by an atomic cursor to the helpers and the calling thread alike, and Run returns +// once every block was taken and finished. The threads are started on first use and never joined +// (they only wait on the condition variable). APS5_MIRROR_REFRESH_THREADS sets the helper count +// (default 3); APS5_NO_PARALLEL_MIRROR_REFRESH=1 compares on the calling thread only. +class RefreshPool { +public: + static RefreshPool& Get() { + // Never destroyed (as WorkerSampler's): the helpers block on its condition variable, which a + // static destructor at std::exit would tear down under them. + static RefreshPool* pool = new RefreshPool; + return *pool; + } + + // Compares `blocks` and returns how many were copied. + std::uint64_t Run(std::span blocks) { + // Small jobs are not worth waking the helpers (a sub-range refresh at Upload is a few blocks). + if (helpers == 0 || blocks.size() < 8) { + std::uint64_t copied = 0; + for (const auto& block : blocks) copied += compareBlock(block) ? 1 : 0; + return copied; + } + // Every caller holds the device lock today; the pool serializes its jobs regardless. + std::lock_guard running(runMutex); + { + std::lock_guard lock(mutex); + job = blocks; + next.store(0, std::memory_order_relaxed); + copiedBlocks.store(0, std::memory_order_relaxed); + finished = 0; + ++serial; + } + wake.notify_all(); + work(blocks); + std::unique_lock lock(mutex); + done.wait(lock, [&] { return finished == helpers; }); + job = {}; + return copiedBlocks.load(std::memory_order_relaxed); + } + +private: + RefreshPool() { + static const bool disabled = std::getenv("APS5_NO_PARALLEL_MIRROR_REFRESH") != nullptr; + const char* text = std::getenv("APS5_MIRROR_REFRESH_THREADS"); + const auto requested = text != nullptr ? std::atoi(text) : 3; + const auto wanted = disabled ? 0u : static_cast(std::clamp(requested, 0, 16)); + // Only started threads are counted: Run waits for `helpers` finishes, and a thread that could + // not be started would leave it waiting forever. + for (unsigned i = 0; i < wanted; ++i) { + try { + std::thread([this] { helper(); }).detach(); + ++helpers; + } catch (const std::system_error& error) { + std::fprintf(stderr, "[gpu] mirror refresh helper %u not started: %s\n", i, error.what()); + break; + } + } + } + + // Takes blocks until none is left. + void work(std::span blocks) { + std::uint64_t copied = 0; + for (auto index = next.fetch_add(1, std::memory_order_relaxed); index < blocks.size(); index = next.fetch_add(1, std::memory_order_relaxed)) copied += compareBlock(blocks[index]) ? 1 : 0; + copiedBlocks.fetch_add(copied, std::memory_order_relaxed); + } + + void helper() { + std::uint64_t seen = 0; + for (;;) { + std::span blocks; + { + std::unique_lock lock(mutex); + wake.wait(lock, [&] { return serial != seen; }); + seen = serial; + blocks = job; + } + work(blocks); + { + std::lock_guard lock(mutex); + ++finished; + } + done.notify_all(); + } + } + + unsigned helpers = 0; + std::mutex runMutex; + std::mutex mutex; + std::condition_variable wake; + std::condition_variable done; + std::span job; + std::uint64_t serial = 0; + unsigned finished = 0; + std::atomic next{0}; + std::atomic copiedBlocks{0}; +}; + +// The device-lock work before [address, address + bytes) of a writable mirror can be compared with +// guest memory: recorded GPU work that writes the range (a descriptor dispatch bound to a sub-range) +// completes first, so its results reach the guest and the shadow before the compare and no batch +// writes a block being replaced; GPU reads of the mirror by earlier recorded work (an address-based +// dispatch whose lease is released at completion) see the newer bytes, as they would on hardware. +// Returns whether there is anything to compare. +bool prepareRefresh(ImageMirror& mirror, std::uint64_t address, std::uint64_t bytes) { + if (!mirror.writable || bytes == 0) return false; + // The compare reads the pages directly; a mirror is only reused while its Range exists, but a + // lease of a build in flight can keep a replaced Range alive past a protection change. + Require(GuestMemory::Accessible(reinterpret_cast(address), static_cast(bytes)), "image mirror range became inaccessible"); + // Named for the [hooksync] attribution: the flush goes through the hook, which waits when + // recorded work notes a write inside the range (a descriptor written inside a writable exe + // range; the lease itself maps the range read-only). The access is the whole range. + const GuestMemory::ReadSiteScope site(GuestMemory::ReadSite::MirrorRefresh); + GuestMemory::FlushGpuWrites(address, static_cast(bytes)); + auto& state = Mirrors(); + if (auto* recorder = Recorder::Active(); recorder != nullptr && recorder->PendingWriteOverlaps(address, static_cast(bytes))) { + Recorder::CountSync(4); + ++state.syncs; + recorder->Sync(); + } + ++state.refreshes; + return true; +} + +void appendBlocks(std::vector& blocks, ImageMirror& mirror, std::uint64_t address, std::uint64_t bytes) { + constexpr std::uint64_t block = 65536; + const auto end = address + bytes; + for (auto at = address; at < end;) { + const auto next = std::min(end, (at / block + 1) * block); + blocks.push_back({&mirror, at, static_cast(next - at)}); + at = next; + } +} + +// Compares the gathered blocks (of one or many mirrors) and counts them. +void compareBlocks(std::span blocks) { + if (blocks.empty()) return; + const auto copied = RefreshPool::Get().Run(blocks); + auto& state = Mirrors(); + state.blocksCompared += blocks.size(); + state.blocksCopied += copied; +} + +// Brings [address, address + bytes) of a writable mirror up to date with guest memory. +void refreshMirror(ImageMirror& mirror, std::uint64_t address, std::uint64_t bytes) { + if (!prepareRefresh(mirror, address, bytes)) return; + std::vector blocks; + appendBlocks(blocks, mirror, address, bytes); + compareBlocks(blocks); +} + +// The mirror for a leased image range: the existing one, or a new one when none exists or the +// registered Range object changed (a protection change). Null when the range cannot be mirrored. An +// existing writable mirror's blocks are appended to `blocks` for the caller's one compare of every +// mirror of the build (prepareRefresh ran); a new mirror is filled here. +std::shared_ptr acquireMirror(const Context& context, const std::shared_ptr& range, std::vector& blocks) { + auto& state = Mirrors(); + std::shared_ptr mirror; + { + std::lock_guard lock(state.mutex); + if (state.device != context.device) { + state.entries.clear(); + state.failed.clear(); + state.device = context.device; + } + if (state.failed.contains(range->address)) return nullptr; + const auto found = state.entries.find(range->address); + if (found != state.entries.end() && sameRange(found->second->range, range) && found->second->bytes == range->bytes && found->second->writable == range->writable) mirror = found->second; + } + if (mirror != nullptr) { + if (prepareRefresh(*mirror, range->address, range->bytes)) appendBlocks(blocks, *mirror, range->address, range->bytes); + return mirror; + } + // The registered flags were taken from the pages; a range that disagrees is copied by this build. + if (!GuestMemory::Accessible(reinterpret_cast(range->address), range->bytes, range->writable)) return nullptr; + mirror = std::make_shared(); + mirror->base = range->address; + mirror->bytes = range->bytes; + mirror->writable = range->writable; + mirror->range = range; + try { + mirror->buffer = std::make_shared(context, range->bytes, VK_BUFFER_USAGE_STORAGE_BUFFER_BIT | VK_BUFFER_USAGE_SHADER_DEVICE_ADDRESS_BIT); + } catch (const std::runtime_error& error) { + std::fprintf(stderr, "[gpu] image mirror of 0x%llx+0x%llx failed: %s; falling back to copies\n", static_cast(range->address), static_cast(range->bytes), error.what()); + std::lock_guard lock(state.mutex); + state.failed.insert(range->address); + return nullptr; + } + // Read stores GPU results pending in the range first (write-backs of earlier copies of it); + // named for the [hooksync] attribution like the refresh's flush. + { + const GuestMemory::ReadSiteScope site(GuestMemory::ReadSite::MirrorRefresh); + GuestMemory::Read(range->address, mirror->buffer->Bytes()); + } + if (range->writable) mirror->shadow.assign(mirror->buffer->Bytes().begin(), mirror->buffer->Bytes().end()); + std::lock_guard lock(state.mutex); + ++state.rebuilds; + mirror->serial = (1ull << 63u) | ++state.serials; + // A replaced mirror lives on in the regions of recorded work that bound it. + state.entries[range->address] = mirror; + return mirror; +} + +// The mirror containing [begin, end) whose registered range still exists, or null. A mirror whose +// Range was replaced is only rebuilt by the next address-based build; until then the range is copied. +std::shared_ptr findMirror(const Context& context, std::uint64_t begin, std::uint64_t end) { + auto& state = Mirrors(); + std::lock_guard lock(state.mutex); + if (state.device != context.device || state.entries.empty()) return nullptr; + auto found = state.entries.upper_bound(begin); + if (found == state.entries.begin()) return nullptr; + --found; + const auto& mirror = found->second; + if (begin < mirror->base || end > mirror->base + mirror->bytes || mirror->range.expired()) return nullptr; + return mirror; +} + +// Drops mirrors whose registered range is gone (a later build made a new one for the replacement). +void sweepMirrors() { + auto& state = Mirrors(); + std::lock_guard lock(state.mutex); + for (auto it = state.entries.begin(); it != state.entries.end();) it = it->second->range.expired() ? state.entries.erase(it) : std::next(it); +} + +void reportMirrors() { + static const bool profile = std::getenv("APS5_PROFILE_DRAW") != nullptr; + if (!profile) return; + auto& state = Mirrors(); + const auto now = std::chrono::steady_clock::now(); + if (now - state.lastReport < std::chrono::seconds(10)) return; + state.lastReport = now; + std::size_t count = 0; + std::size_t writable = 0; + std::uint64_t bytes = 0; + { + std::lock_guard lock(state.mutex); + for (const auto& [base, mirror] : state.entries) { + ++count; + bytes += mirror->bytes; + if (mirror->writable) ++writable; + } + } + std::fprintf(stderr, "[buffers] image mirrors: %zu ranges (%.1f MiB, %zu writable), %llu address-based builds served, %llu descriptor sub-ranges bound, %llu rebuilds, %llu refreshes: %llu blocks compared, %llu copied, %llu refresh syncs\n", count, bytes / 1048576.0, writable, static_cast(state.builds), static_cast(state.subranges), static_cast(state.rebuilds), static_cast(state.refreshes), static_cast(state.blocksCompared), static_cast(state.blocksCopied), static_cast(state.syncs)); +} + +// Deferred lease release. The lease an address-based build takes (AcquireRegistered) is dropped by its +// write-back, which runs as a completion action when the batch that recorded the work finished; the +// work is no longer synced as soon as it is recorded (that sync was ~30000 waits per 200 s). A guest +// thread that unmaps, mprotects or frees a leased allocation finds it pinned in the registry and calls +// this waiter (with the registry lock released, see GuestAllocationsRequireUnpinned): under the device +// lock it submits the recorder and finishes its batches up to the newest one that was recorded with a +// lease (CountLeaseOutcome notes that serial); the completions release the leases, later batches stay +// in flight, and the registry rescans. A lease held by a build in progress belongs to a worker holding +// the device lock, so the waiter waits for that worker first; once it holds the lock every lease of +// recorded work is in a batch. A lease whose batch was not noted (the record threw between the keep +// and the note) is covered by draining the recorder when the noted serial is already finished. The +// waiter yields instead when its own thread holds the device lock (a driver thread mutating the +// registry mid-packet, as the plain spin did) and reports whether it finished any work. +struct LeaseState { + std::mutex mutex; + LeaseStats stats; + std::chrono::steady_clock::time_point lastReport = std::chrono::steady_clock::now(); + // Under GuestMemory::GpuMutex: the recorder whose batches hold leases (serials restart with a + // replaced device's recorder), the serial of its newest batch recorded with a lease, and the + // newest one a waiter finished. + const Recorder* leaseRecorder = nullptr; + std::uint64_t newestLeaseSerial = 0; + std::uint64_t finishedLeaseSerial = 0; +}; + +LeaseState& Leases() { + static LeaseState state; + return state; +} + +// APS5_PROFILE_DRAW: AddSnapshot's consistency compares of a captured region against the region +// serving it, made and skipped (see AddSnapshot); printed in the [address-sync] line. Atomic: the +// unlocked stage A of descriptor builds adds snapshots too. +struct SnapshotStats { + std::atomic checked{0}; + std::atomic skipped{0}; +}; + +SnapshotStats& Snapshots() { + static SnapshotStats stats; + return stats; +} + +bool WaitForLeases() noexcept { + const auto start = std::chrono::steady_clock::now(); + bool synced = false; + bool drained = false; + if (GuestMemory::GpuMutex().HeldByThisThread()) { + std::this_thread::yield(); + } else { + try { + std::lock_guard lock(GuestMemory::GpuMutex()); + auto& state = Leases(); + auto* recorder = Recorder::Active(); + const auto target = state.newestLeaseSerial; + if (recorder != nullptr && recorder == state.leaseRecorder && target > state.finishedLeaseSerial) { + // The open batch may be the target; FinishUpTo waits front to back up to it only. + Recorder::CountSync(4); + recorder->Submit(); + recorder->FinishUpTo(target); + state.finishedLeaseSerial = target; + synced = true; + } else if (recorder != nullptr && !recorder->Idle()) { + Recorder::CountSync(4); + recorder->Sync(); + synced = true; + drained = true; + } else { + // Nothing recorded holds the lease: the holder is a lease-holding object that has + // yet to record (a test's, or a build whose worker gave up the lock); give it time. + std::this_thread::yield(); + } + } catch (const std::exception& error) { + std::fprintf(stderr, "[gpu] lease wait failed: %s\n", error.what()); + } + } + auto& state = Leases(); + std::lock_guard lock(state.mutex); + ++state.stats.contentionWaits; + if (synced) ++state.stats.contentionSyncs; + if (drained) ++state.stats.contentionDrains; + state.stats.contentionMs += std::chrono::duration(std::chrono::steady_clock::now() - start).count(); + return synced; +} + +void ensurePinWaiter() { + static const bool registered = [] { + GuestAllocations::GuestAllocationsSetPinWaiter_nid_postfix(&WaitForLeases); + return true; + }(); + static_cast(registered); +} + +} + +bool SyncLeaseWork() { + static const bool sync = std::getenv("APS5_SYNC_LEASE_DISPATCH") != nullptr; + return sync; +} + +void CountLeaseOutcome(bool synced, std::uint64_t batchSerial) { + static const bool profile = std::getenv("APS5_PROFILE_DRAW") != nullptr; + auto& state = Leases(); + if (!synced) { + // Serials grow with the submissions, so the newest noted one covers every earlier lease batch + // of the same recorder. + GuestMemory::AssertGpuLockHeld("CountLeaseOutcome"); + if (const auto* recorder = Recorder::Active(); recorder != state.leaseRecorder) { + state.leaseRecorder = recorder; + state.newestLeaseSerial = 0; + state.finishedLeaseSerial = 0; + } + state.newestLeaseSerial = std::max(state.newestLeaseSerial, batchSerial); + } + std::lock_guard lock(state.mutex); + ++(synced ? state.stats.synced : state.stats.deferred); + if (!profile) return; + // APS5_PROFILE_DRAW: the leases (dispatches and draws) deferred and synced, the guest's + // pin-contention waits and the BDA table cache, every 10 s (cumulative counts, like the + // [recorder] line's), printed by whichever path counts a lease. + const auto now = std::chrono::steady_clock::now(); + if (now - state.lastReport < std::chrono::seconds(10)) return; + state.lastReport = now; + const auto& stats = state.stats; + const auto table = BdaResources::TableCacheCounters(); + const auto& snapshots = Snapshots(); + std::fprintf(stderr, "[address-sync] leases: %llu released at completion, %llu synced at once; %llu pin-contention waits by guest threads (%llu finished the lease batch, %llu drained the recorder) %.1f s; BDA table cache %llu hits / %llu misses (%zu tables held); snapshot compares: %llu skipped (live-backed region), %llu made\n", static_cast(stats.deferred), static_cast(stats.synced), static_cast(stats.contentionWaits), static_cast(stats.contentionSyncs), static_cast(stats.contentionDrains), stats.contentionMs / 1000, static_cast(table.hits), static_cast(table.misses), table.held, static_cast(snapshots.skipped.load(std::memory_order_relaxed)), static_cast(snapshots.checked.load(std::memory_order_relaxed))); +} + +LeaseStats LeaseCounters() { + auto& state = Leases(); + std::lock_guard lock(state.mutex); + return state.stats; +} + +const HostImport* HostImportFor(const Context& context, std::uint64_t address, std::size_t bytes) { + if (context.hostImportAlignment == 0 || bytes == 0 || bytes > std::numeric_limits::max() - address) return nullptr; + auto& state = Imports(); + std::lock_guard lock(state.mutex); + // A hit is only valid while the registry has not changed since the imports were reconciled. + if (!importsStale(context, state)) { + if (const auto* entry = findImport(state, address, address + bytes)) return entry; + } + const auto lease = GuestAllocations::GuestAllocationsAcquire_nid_postfix(); + refreshImports(context, state, lease); + if (const auto* range = containingRange(lease, address, address + bytes)) return importAllocation(context, state, range->address, range->bytes); + return nullptr; +} + +bool HostImportCovers(const Context& context, std::uint64_t address, std::size_t bytes) { + if (context.hostImportAlignment == 0 || bytes == 0 || bytes > std::numeric_limits::max() - address) return false; + auto& state = Imports(); + std::lock_guard lock(state.mutex); + // Imports of a replaced device are not this device's; a stale registry only means an import + // may be dropped at the next reconcile, which the binding path handles. + return state.device == context.device && findImport(state, address, address + bytes) != nullptr; +} + GuestBufferMemory::GuestBufferMemory(const Context& context) : context(context) {} bool GuestBufferMemory::WritesOverlap(std::uint64_t address, std::size_t bytes) const { return std::any_of(writes.begin(), writes.end(), [&](const auto& range) { return address < range.second && range.first < address + bytes; }); } +void GuestBufferMemory::AcquireRegistered() { + Require(!uploaded && regions.empty() && lease.empty(), "guest allocation lease must precede resource registration"); + ensurePinWaiter(); + // Pending GPU results in these ranges are stored when Upload prepares each region. + lease = GuestAllocations::GuestAllocationsAcquire_nid_postfix(); + const bool importable = context.hostImportAlignment != 0; + if (importable) { + auto& state = Imports(); + std::lock_guard lock(state.mutex); + refreshImports(context, state, lease); + // Taken before the pass: the imports it takes are the registry's ones (just reconciled, so + // none is retired by a make below), and a mirror's preparation may sync the recorder, whose + // completions can retire an import already taken; that moves the epoch, and Upload then + // re-checks every pointer instead of trusting it. + importsEpoch = state.epoch; + } + // One pass in the registry's (ascending) order, so the regions come out sorted for AddSnapshot. + // The import lock is taken per range: a mirror's preparation may wait for recorded work, which + // must not happen under it. + regions.reserve(lease.size()); + std::vector blocks; + bool mirrored = false; + for (const auto& range : lease) { + if (!range->readable) continue; + validate(range->address, range->bytes); + Region region{range->address, range->address + range->bytes, range->writable, {}, nullptr}; + const HostImport* entry = nullptr; + if (importable) { + auto& state = Imports(); + std::lock_guard lock(state.mutex); + entry = importAllocation(context, state, range->address, range->bytes); + } + if (entry != nullptr) { + region.hostBacked = true; + // Reused by Upload while no import was dropped since (see importsEpoch). + region.direct = entry; + regions.push_back(std::move(region)); + continue; + } + if (!range->releasable && mirrorsEnabled()) { + // Main image: served by a persistent mirror (see ImageMirror); the writable ones are + // compared with guest memory below, all at once. + region.mirror = acquireMirror(context, range, blocks); + if (region.mirror != nullptr) { + mirrored = true; + regions.push_back(std::move(region)); + continue; + } + } + // Read-only ranges are read from live guest memory at Upload: the lease pins them and the guest + // cannot write them, so that equals a snapshot taken now without the extra copy. Reserved heaps + // are registered whole while the guest commits pages on demand: committed pages only. + region.hostBacked = !range->writable; + auto committed = GuestMemory::DescribeCommitted(range->address, range->bytes, range->writable); + if (!committed.whole) { + region.sparse = true; + region.backed = std::move(committed.ranges); + } + regions.push_back(std::move(region)); + } + // APS5_NO_SORTED_SNAPSHOT_LOOKUP=1: AddSnapshot scans the regions as before instead of searching. + static const bool sortedLookup = std::getenv("APS5_NO_SORTED_SNAPSHOT_LOOKUP") == nullptr; + regionsSorted = sortedLookup; + compareBlocks(blocks); + if (mirrorsEnabled()) { + if (mirrored) ++Mirrors().builds; + sweepMirrors(); + reportMirrors(); + } +} + void GuestBufferMemory::validate(std::uint64_t address, std::size_t bytes) const { Require(!uploaded, "guest memory ownership is frozen for GPU execution"); Require(address != 0 && bytes != 0, "empty guest memory range"); Require(bytes <= std::numeric_limits::max() - address, "guest memory range overflow"); } -void GuestBufferMemory::AddWritable(std::uint64_t address, std::size_t bytes) { +void GuestBufferMemory::addDescriptorRegion(std::uint64_t address, std::size_t bytes) { validate(address, bytes); - GuestMemory::CheckRange(reinterpret_cast(address), bytes, 1, true); - regions.push_back({address, address + bytes, true, {}, nullptr}); + // `writable` here means the bytes come from live guest memory (not a snapshot), whether or not + // the shader stores to them; what is written back is decided by Writes() alone. + Region region{address, address + bytes, true, {}, nullptr}; + auto committed = GuestMemory::DescribeCommitted(address, bytes, true); + if (!committed.whole) { + // A GPU heap bound whole while the guest commits its pages on demand, or a descriptor left + // pointing at unmapped memory: only committed pages are copied in and stored back. Shaders + // must not touch the rest, which reads as zeros. + region.sparse = true; + region.backed = std::move(committed.ranges); + } + regions.push_back(std::move(region)); + regionsSorted = false; +} + +void GuestBufferMemory::AddWritable(std::uint64_t address, std::size_t bytes) { + addDescriptorRegion(address, bytes); writes.emplace_back(address, address + bytes); } +void GuestBufferMemory::AddReadable(std::uint64_t address, std::size_t bytes) { + // Not in `writes`: no reference copy (copyRegion), no write-back, no pending-write note, no + // direct-write mark, and UploadPrepare copies it without the device lock. A written descriptor + // overlapping the range still covers it through its own Writes() entry. + addDescriptorRegion(address, bytes); +} + void GuestBufferMemory::AddSnapshot(const GuestMemorySnapshot& snapshot) { validate(snapshot.address, snapshot.bytes.size()); - for (const auto& region : regions) { - if (region.begin <= snapshot.address && snapshot.address + snapshot.bytes.size() <= region.end) { - const auto offset = static_cast(snapshot.address - region.begin); - const auto* source = region.writable ? reinterpret_cast(snapshot.address) : region.snapshot.data() + offset; - Require(std::memcmp(source, snapshot.bytes.data(), snapshot.bytes.size()) == 0, "guest snapshot differs from registered memory"); - return; + const auto end = snapshot.address + snapshot.bytes.size(); + // A snapshot inside a region needs no copy: the region's bytes serve it (they are checked to + // agree). Right after AcquireRegistered the regions are sorted and non-overlapping (registered + // ranges), so the one that can contain the snapshot is found by search rather than by scanning + // the ~1200 of an address-based build for each captured region. + const Region* owner = nullptr; + if (regionsSorted) { + const auto found = std::upper_bound(regions.begin(), regions.end(), snapshot.address, [](std::uint64_t value, const Region& region) { return value < region.begin; }); + if (found != regions.begin() && end <= std::prev(found)->end) owner = &*std::prev(found); + } else { + for (const auto& region : regions) { + if (region.begin <= snapshot.address && end <= region.end) { + owner = ®ion; + break; + } } } - regions.push_back({snapshot.address, snapshot.address + snapshot.bytes.size(), false, {snapshot.bytes.begin(), snapshot.bytes.end()}, nullptr}); + if (owner != nullptr) { + // A region read live (an import or mirror bound in place, a copy of live bytes) serves the + // GPU whatever the snapshot holds: the compare only checked that the capture (made before + // the device lock) still agreed with memory the guest, or a dispatch writing a descriptor + // range around the captured words, may since have changed, and a difference failed the + // whole dispatch where hardware would have run with the bytes in memory. A snapshot-backed + // region uploads its own snapshot, so two snapshots of one range must agree. The live + // compare reads the pages directly (no flush hook), so it costs no sync, only the memcmp; + // it stays on because it is the one check that the words the recompiler baked into + // descriptors and specialization still equal what the GPU reads. APS5_NO_SNAPSHOT_CHECK=1 + // skips the compare of live-backed regions (the region serves the GPU whatever the snapshot + // held; only the diagnostic is lost). + static const bool checkLive = std::getenv("APS5_NO_SNAPSHOT_CHECK") == nullptr; + static const bool profile = std::getenv("APS5_PROFILE_DRAW") != nullptr; + const bool live = owner->writable || owner->hostBacked || owner->mirror != nullptr; + if (live && !checkLive) { + if (profile) Snapshots().skipped.fetch_add(1, std::memory_order_relaxed); + return; + } + const auto offset = static_cast(snapshot.address - owner->begin); + const auto* source = live ? reinterpret_cast(snapshot.address) : owner->snapshot.data() + offset; + Require(std::memcmp(source, snapshot.bytes.data(), snapshot.bytes.size()) == 0, "guest snapshot differs from registered memory"); + if (profile) Snapshots().checked.fetch_add(1, std::memory_order_relaxed); + return; + } + regions.push_back({snapshot.address, end, false, {snapshot.bytes.begin(), snapshot.bytes.end()}, nullptr}); + regionsSorted = false; } void GuestBufferMemory::Upload(bool addressable) { - Require(!uploaded, "guest memory was already uploaded"); - uploaded = true; + UploadPrepare(addressable); + UploadFinish(addressable); +} + +namespace { + +// APS5_PROFILE_DRAW: sub-phase totals of the uploads in microseconds, reported every 2000 uploads +// ([buffers] line); atomic because the prepare stage of several builds runs at once. +std::atomic importUs{0}; +std::atomic allocateUs{0}; +std::atomic readUs{0}; +std::atomic uploadsProfiled{0}; + +std::uint64_t microsecondsSince(std::chrono::steady_clock::time_point start) { + return static_cast(std::chrono::duration_cast(std::chrono::steady_clock::now() - start).count()); +} + +// GPU-side copies. A descriptor sub-range inside a host import whose offset is not a multiple of +// minStorageBufferOffsetAlignment cannot be bound in place; ~31000 of them per run (about 34 bytes +// each) were copied through the CPU, and every such read went through the flush hook and waited +// for the producer batch ([hooksync] 'buffer-upload'). Instead the copy is a vkCmdCopyBuffer from +// the import into the region's buffer, recorded into the open batch after the producer, and the +// written sub-ranges are copied back into the import by the GPU (RecordCopyBacks), so neither +// direction touches guest memory from the CPU. APS5_CPU_COPIES=1 copies on the CPU as before; +// APS5_GPU_COPY_MAX_KIB (default 1024) bounds the region size the GPU copies (larger ones and +// regions outside imports keep the CPU path). +bool gpuCopiesEnabled() { + static const bool disabled = std::getenv("APS5_CPU_COPIES") != nullptr; + return !disabled; +} + +std::uint64_t gpuCopyLimit() { + static const std::uint64_t limit = [] { + const char* value = std::getenv("APS5_GPU_COPY_MAX_KIB"); + return (value != nullptr ? std::strtoull(value, nullptr, 10) : 1024ull) << 10u; + }(); + return limit; +} + +VkBufferUsageFlags gpuCopyUsage(bool addressable) { + return VK_BUFFER_USAGE_STORAGE_BUFFER_BIT | VK_BUFFER_USAGE_TRANSFER_SRC_BIT | VK_BUFFER_USAGE_TRANSFER_DST_BIT | (addressable ? VK_BUFFER_USAGE_SHADER_DEVICE_ADDRESS_BIT : 0u); +} + +// APS5_PROFILE_DRAW: the GPU copies, reported in the [buffers] uploads line (cumulative). +struct CopyStats { + std::atomic gpuCopies{0}; + std::atomic gpuCopyBytes{0}; + std::atomic gpuCopyBacks{0}; + std::atomic gpuCopyBackBytes{0}; + // Written gpuCopy regions of a use that recorded no copy-back (a synchronous draw): stored + // from the staging buffer by the CPU in WriteBack. + std::atomic stagingStores{0}; +}; + +CopyStats& Copies() { + static CopyStats stats; + return stats; +} + +} + +bool GuestBufferMemory::gpuCopyEligible(const Region& region) const { + if (!gpuCopiesEnabled() || region.sparse || !(region.hostBacked || region.writable)) return false; + return region.end - region.begin <= gpuCopyLimit(); +} + +void GuestBufferMemory::UploadPrepare(bool addressable) { + Require(!prepared && !uploaded, "guest memory was already uploaded"); + prepared = true; std::sort(regions.begin(), regions.end(), [](const Region& left, const Region& right) { return left.begin < right.begin; }); + // Merged regions stay sparse when either part covered uncommitted pages. + const auto mergeBacked = [](Region& into, const Region& from) { + if (!into.sparse && !from.sparse) return; + auto ranges = into.sparse ? into.backed : decltype(into.backed){{into.begin, into.end}}; + if (from.sparse) ranges.insert(ranges.end(), from.backed.begin(), from.backed.end()); + else ranges.emplace_back(from.begin, from.end); + std::sort(ranges.begin(), ranges.end()); + into.backed.clear(); + for (const auto& range : ranges) { + if (!into.backed.empty() && range.first <= into.backed.back().second) into.backed.back().second = std::max(into.backed.back().second, range.second); + else into.backed.push_back(range); + } + const auto end = std::max(into.end, from.end); + into.sparse = !(into.backed.size() == 1 && into.backed.front().first <= into.begin && into.backed.front().second >= end); + if (!into.sparse) into.backed.clear(); + }; std::vector merged; for (auto& region : regions) { if (!merged.empty() && region.begin < merged.back().end) { auto& previous = merged.back(); + if (previous.mirror != nullptr || region.mirror != nullptr) { + // A descriptor range inside a mirrored image range binds the mirror's bytes (registered + // ranges never overlap, so the other part is such a descriptor; it sorts first when it + // starts at the range's base). Writes into a read-only mirror are never stored. One + // crossing the mirror's end cannot be served by it: this build copies the range. + if (previous.mirror == nullptr) std::swap(previous, region); + Require(region.mirror == nullptr, "image mirrors overlap"); + if (region.begin >= previous.begin && region.end <= previous.end) continue; + previous.mirror = nullptr; + previous.hostBacked = true; + } + mergeBacked(previous, region); + // A range starting before the mirror (only possible after the swap) keeps its prefix; the + // earlier merged region ends at or before it, so the merged list stays sorted. + previous.begin = std::min(previous.begin, region.begin); + if (previous.hostBacked || region.hostBacked) { + // Live guest bytes back the merged range, so snapshots inside it add nothing. + previous.hostBacked = true; + previous.writable = previous.writable || region.writable; + previous.snapshot.clear(); + previous.end = std::max(previous.end, region.end); + continue; + } Require(previous.writable == region.writable, "writable guest memory overlaps an immutable snapshot"); if (!region.writable) { const auto offset = static_cast(region.begin - previous.begin); @@ -59,15 +977,358 @@ void GuestBufferMemory::Upload(bool addressable) { } } regions = std::move(merged); + static const bool profile = std::getenv("APS5_PROFILE_DRAW") != nullptr; + const auto started = std::chrono::steady_clock::now(); for (auto& region : regions) { - const auto bytes = region.end - region.begin; - Require(bytes <= std::numeric_limits::max(), "guest GPU allocation size overflow"); - const auto usage = VK_BUFFER_USAGE_STORAGE_BUFFER_BIT | (addressable ? VK_BUFFER_USAGE_SHADER_DEVICE_ADDRESS_BIT : 0u); - region.buffer = std::make_unique(context, static_cast(bytes), usage); - if (region.writable) GuestMemory::Read(region.begin, region.buffer->Bytes()); - else std::memcpy(region.buffer->Bytes().data(), region.snapshot.data(), region.snapshot.size()); - region.snapshot.clear(); + Require(region.end - region.begin <= std::numeric_limits::max(), "guest GPU allocation size overflow"); + // Served by a lease mirror, brought up to date when it was acquired. + if (region.mirror != nullptr || region.direct != nullptr || !mirrorsEnabled()) continue; + // A descriptor range inside a mirrored image range binds the mirror's sub-range. Looked up + // before the imports: the image can never be imported, and the import fallback would acquire + // a registry lease for it on nearly every descriptor upload. + auto mirror = findMirror(context, region.begin, region.end); + if (mirror == nullptr || (region.begin - mirror->base) % context.limits.minStorageBufferOffsetAlignment != 0) continue; + // A read-only mirror's bytes never change while its Range exists; a writable one is + // refreshed by UploadFinish (the refresh may wait for recorded work). Either is confirmed + // there, under the lock, before it serves the region: the Range can go between the stages, + // and until then the region keeps its snapshot for the copy it would need instead. + region.pending = mirror->writable; + region.mirror = std::move(mirror); + region.subrangeMirror = true; } + // Imports already serving regions, taken under one hold of the import lock so every pointer + // belongs to the same registry epoch (UploadFinish trusts them while the epoch is unchanged). No + // reconcile and no new import here: retiring one hands its buffer to the recorder, and a region + // without an import waits for UploadFinish, which may make one. + if (context.hostImportAlignment != 0) { + auto& state = Imports(); + std::lock_guard lock(state.mutex); + const bool stale = importsStale(context, state); + for (auto& region : regions) { + if (region.mirror != nullptr) continue; + // An import found when the lease was acquired is reused while none was dropped since. + const HostImport* entry = region.direct != nullptr && state.epoch == importsEpoch ? region.direct : nullptr; + region.direct = nullptr; + if (stale) { + region.pending = true; + continue; + } + if (entry == nullptr) entry = findImport(state, region.begin, region.end); + if (entry == nullptr) { + region.pending = true; + continue; + } + // Misaligned inside an import: copied, whatever the registry does meanwhile. By the GPU + // when eligible (see Region::gpuCopy): UploadFinish records the copy from the import it + // confirms then (the pointer is provisional, as for an aligned region); the buffer the + // copy fills is made here, without the device lock, and serves the CPU fallback too. + if ((region.begin - entry->base) % context.limits.minStorageBufferOffsetAlignment != 0) { + if (!gpuCopyEligible(region)) continue; + const auto allocateStart = std::chrono::steady_clock::now(); + region.buffer = std::make_shared(context, static_cast(region.end - region.begin), gpuCopyUsage(addressable)); + if (profile) allocateUs.fetch_add(microsecondsSince(allocateStart), std::memory_order_relaxed); + region.direct = entry; + region.pending = true; + continue; + } + // Provisional until UploadFinish confirms it against the epoch; the snapshot stays for + // the copy the region falls back to when the import is gone and none can be made. + region.direct = entry; + // Storage results pending in the range are flushed by UploadFinish. + region.pending = true; + } + importsEpoch = state.epoch; + } + if (profile) importUs.fetch_add(microsecondsSince(started), std::memory_order_relaxed); + // Copies that need no device work: a range no descriptor writes needs no reference copy, and + // GuestMemory::Read stores pending GPU results first (the flush hook locks for itself). + for (auto& region : regions) { + if (region.pending || region.mirror != nullptr || region.direct != nullptr) continue; + if (region.writable && WritesOverlap(region.begin, static_cast(region.end - region.begin))) { + region.pending = true; + continue; + } + copyRegion(region, addressable); + } +} + +void GuestBufferMemory::UploadFinish(bool addressable) { + Require(prepared && !uploaded, "guest memory upload was not prepared"); + uploaded = true; + static const bool profile = std::getenv("APS5_PROFILE_DRAW") != nullptr; + const auto started = std::chrono::steady_clock::now(); + // Registered ranges to import from. Address-based shaders hold their lease until write-back; + // descriptor-only uploads acquire one only when the registry changed or an import must be made + // (a lease copies every registered range's shared_ptr under the registry lock), so the guest can + // keep managing memory. + GuestAllocations::Lease acquired; + const auto importRanges = [&]() -> const GuestAllocations::Lease& { + if (lease.empty() && acquired.empty()) acquired = GuestAllocations::GuestAllocationsAcquire_nid_postfix(); + return lease.empty() ? acquired : lease; + }; + // Sub-range mirrors were found by UploadPrepare, possibly long before this lock was taken: one + // whose Range went meanwhile (the guest reprotected its image) is dropped as findMirror would + // have dropped it, and the region takes the import or copy path below instead; a confirmed one + // serves the region, so its snapshot goes. (Lease mirrors need no check: the lease pins their + // Range.) + for (auto& region : regions) { + if (region.mirror == nullptr || !region.subrangeMirror) continue; + if (region.mirror->range.expired()) { + region.mirror = nullptr; + region.subrangeMirror = false; + region.pending = true; + continue; + } + region.snapshot.clear(); + std::lock_guard lock(Mirrors().mutex); + ++Mirrors().subranges; + } + bool importsRefreshed = false; + // Regions the GPU copies out of their import (see Region::gpuCopy), recorded together below. + // No recorder (tests) keeps them on the CPU path. + auto* recorder = Recorder::Active(); + std::vector gpuCopies; + for (auto& region : regions) { + if (!region.pending) continue; + region.pending = false; + const auto bytes = region.end - region.begin; + if (region.mirror != nullptr) { + // A writable sub-range mirror: refreshed here, where waiting for recorded work is allowed. + // Read-only mirrors are never written back; a writable one is diffed for this sub-range + // at write-back. + const auto refreshStart = std::chrono::steady_clock::now(); + refreshMirror(*region.mirror, region.begin, bytes); + if (profile) readUs.fetch_add(microsecondsSince(refreshStart), std::memory_order_relaxed); + continue; + } + bool copyOnGpu = false; + if (context.hostImportAlignment != 0) { + const auto lookupStart = std::chrono::steady_clock::now(); + auto& state = Imports(); + std::lock_guard lock(state.mutex); + // Imports are made on demand for the registered range around each region. + if (!importsRefreshed) { + if (importsStale(context, state)) refreshImports(context, state, importRanges()); + importsRefreshed = true; + } + // An import taken by UploadPrepare (or at the lease) is still the registry's unless one + // was dropped since. + const HostImport* entry = region.direct != nullptr && state.epoch == importsEpoch ? region.direct : nullptr; + region.direct = nullptr; + if (entry == nullptr) entry = findImport(state, region.begin, region.end); + if (entry == nullptr) { + if (const auto* range = containingRange(importRanges(), region.begin, region.end)) entry = importAllocation(context, state, range->address, range->bytes); + } + if (entry != nullptr && (region.begin - entry->base) % context.limits.minStorageBufferOffsetAlignment == 0) { + region.direct = entry; + region.snapshot.clear(); + // A buffer UploadPrepare made for a GPU copy is not needed: the import serves the + // region in place (Descriptor prefers `direct`), and a kept buffer would count as copied. + region.buffer.reset(); + } else if (entry != nullptr && recorder != nullptr && gpuCopyEligible(region)) { + // Misaligned in the import: the GPU copies the sub-range out of it. The handle and + // base are taken now, under the import lock: a reconcile by the flush below could + // retire the entry (the batch opened below keeps a retired import's buffer alive). + copyOnGpu = true; + region.copySource = entry->buffer; + region.copySourceBase = entry->base; + } + if (profile) importUs.fetch_add(microsecondsSince(lookupStart), std::memory_order_relaxed); + } + if (region.direct != nullptr) { + // The GPU reads this range in place, so storage results pending in it must be stored first + // (outside the import lock: the store may import memory itself). The whole registered heaps + // an address-based shader maps are exempt (they hold every pending image, and such shaders + // do not read texture memory through pointers); APS5_BDA_FLUSH=1 flushes for them too. + static const bool bdaFlush = std::getenv("APS5_BDA_FLUSH") != nullptr; + if (!(addressable && region.hostBacked) || bdaFlush) StorageTexture::FlushPending(region.begin, static_cast(bytes), nullptr, "imported buffer region"); + continue; + } + if (copyOnGpu) { + // As for a direct region, storage results pending in the range reach the import before + // the copy reads it: the flush is recorded (GPU-direct into the import) ahead of the copy + // in the same batch. The batch is opened first so that an import the flush's own + // reconcile retires is handed to it (retireImport keeps nothing while the recorder is + // idle) and outlives the copy. + static_cast(recorder->Commands()); + StorageTexture::FlushPending(region.begin, static_cast(bytes), nullptr, "copied buffer region"); + region.gpuCopy = true; + gpuCopies.push_back(®ion); + continue; + } + copyRegion(region, addressable); + } + if (!gpuCopies.empty()) recordGpuCopies(gpuCopies, addressable); + if (!profile) return; + const auto uploads = uploadsProfiled.fetch_add(1, std::memory_order_relaxed) + 1; + if (uploads % 2000 == 0) { + const auto& copies = Copies(); + std::fprintf(stderr, "[buffers] %llu uploads: import lookup %.0f ms, buffer allocation %.0f ms, guest read %.0f ms; copies on the GPU: %llu regions (%.0f KiB) copied out of imports, %llu written sub-ranges (%.0f KiB) copied back, %llu staging stores by the CPU at write-back\n", static_cast(uploads), importUs.load(std::memory_order_relaxed) / 1000.0, allocateUs.load(std::memory_order_relaxed) / 1000.0, readUs.load(std::memory_order_relaxed) / 1000.0, static_cast(copies.gpuCopies.load(std::memory_order_relaxed)), copies.gpuCopyBytes.load(std::memory_order_relaxed) / 1024.0, static_cast(copies.gpuCopyBacks.load(std::memory_order_relaxed)), copies.gpuCopyBackBytes.load(std::memory_order_relaxed) / 1024.0, static_cast(copies.stagingStores.load(std::memory_order_relaxed))); + } + const auto ms = std::chrono::duration(std::chrono::steady_clock::now() - started).count(); + if (ms > 50) { + std::uint64_t total = 0; + std::uint64_t largest = 0; + std::uint64_t copied = 0; + for (const auto& region : regions) { + total += region.end - region.begin; + largest = std::max(largest, region.end - region.begin); + if (region.buffer != nullptr) copied += region.end - region.begin; + } + std::fprintf(stderr, "[resources] guest upload of %zu regions, %.1f MiB (largest %.1f MiB, copied %.1f MiB, addressable %d) took %.0f ms to finish\n", regions.size(), total / 1048576.0, largest / 1048576.0, copied / 1048576.0, addressable ? 1 : 0, ms); + } + // Debug aid: APS5_TRACE_BIGBUF lists every copied region of at least 1 MiB. + static const bool traceBig = std::getenv("APS5_TRACE_BIGBUF") != nullptr; + if (traceBig) { + for (const auto& region : regions) { + if (region.buffer != nullptr && region.end - region.begin >= (1u << 20u)) std::fprintf(stderr, "[bigbuf] upload 0x%llx+0x%llx writable=%d hostBacked=%d sparse=%d addressable=%d\n", static_cast(region.begin), static_cast(region.end - region.begin), region.writable ? 1 : 0, region.hostBacked ? 1 : 0, region.sparse ? 1 : 0, addressable ? 1 : 0); + } + } +} + +void GuestBufferMemory::copyRegion(Region& region, bool addressable) { + static const bool profile = std::getenv("APS5_PROFILE_DRAW") != nullptr; + const auto bytes = region.end - region.begin; + const auto usage = VK_BUFFER_USAGE_STORAGE_BUFFER_BIT | (addressable ? VK_BUFFER_USAGE_SHADER_DEVICE_ADDRESS_BIT : 0u); + if (profile) { + // Why the region is copied rather than bound in place, totalled every 1000 uploads (under a + // mutex: prepare stages of several builds copy at once). + static std::mutex reasonsMutex; + static std::uint64_t uploads = 0, noImport = 0, noImportBytes = 0, misaligned = 0, misalignedBytes = 0, snapshotOnly = 0, snapshotBytes = 0; + static std::map> outside; + bool inImport = false; + if (context.hostImportAlignment != 0) { + auto& state = Imports(); + std::lock_guard lock(state.mutex); + inImport = findImport(state, region.begin, region.end) != nullptr; + } + std::lock_guard lock(reasonsMutex); + if (!region.hostBacked && !region.writable) { ++snapshotOnly; snapshotBytes += bytes; } + else if (inImport) { ++misaligned; misalignedBytes += bytes; } + else { + ++noImport; + noImportBytes += bytes; + auto& bucket = outside[region.begin >> 28u]; + ++bucket.first; + bucket.second += bytes; + } + if (++uploads % 1000 == 0) { + // 'misaligned in imports' are the ones the GPU did not copy: over APS5_GPU_COPY_MAX_KIB, + // sparse, APS5_CPU_COPIES, no recorder, or an import gone by UploadFinish. + std::fprintf(stderr, "[buffers] %llu copied regions on the CPU: %llu snapshots (%.0f MiB), %llu misaligned in imports (%.0f MiB), %llu outside imports (%.0f MiB):", static_cast(uploads), static_cast(snapshotOnly), snapshotBytes / 1048576.0, static_cast(misaligned), misalignedBytes / 1048576.0, static_cast(noImport), noImportBytes / 1048576.0); + for (const auto& [granule, counts] : outside) std::fprintf(stderr, " 0x%llx0000000:%llu/%.0fMiB", static_cast(granule), static_cast(counts.first), counts.second / 1048576.0); + std::fprintf(stderr, "\n"); + } + } + const auto allocateStart = std::chrono::steady_clock::now(); + // A buffer made by UploadPrepare for a GPU copy that fell back here (its import went) is kept. + if (region.buffer == nullptr) region.buffer = std::make_shared(context, static_cast(bytes), usage); + const auto readStart = std::chrono::steady_clock::now(); + if (profile) allocateUs.fetch_add(microsecondsSince(allocateStart), std::memory_order_relaxed); + // Named for the [hooksync] attribution: the reads below go through the flush hook. + const GuestMemory::ReadSiteScope site(GuestMemory::ReadSite::BufferUpload); + if (region.hostBacked || region.writable) { + // Only ranges a descriptor may write need the copy the write-back diffs against; the + // registered ranges of an address-based shader are read-only to it (AddressRanges). + const bool written = region.writable && WritesOverlap(region.begin, static_cast(bytes)); + if (!region.sparse) { + GuestMemory::Read(region.begin, region.buffer->Bytes()); + if (written) region.uploaded.assign(region.buffer->Bytes().begin(), region.buffer->Bytes().end()); + } else { + // `uploaded` holds the backed pieces back to back. + for (const auto& [first, last] : region.backed) { + const auto piece = region.buffer->Bytes().subspan(static_cast(first - region.begin), static_cast(last - first)); + GuestMemory::Read(first, piece); + if (written) region.uploaded.insert(region.uploaded.end(), piece.begin(), piece.end()); + } + } + } + else std::memcpy(region.buffer->Bytes().data(), region.snapshot.data(), region.snapshot.size()); + region.snapshot.clear(); + if (profile) readUs.fetch_add(microsecondsSince(readStart), std::memory_order_relaxed); +} + +void GuestBufferMemory::recordGpuCopies(std::span copies, bool addressable) { + static const bool profile = std::getenv("APS5_PROFILE_DRAW") != nullptr; + auto* recorder = Recorder::Active(); + Require(recorder != nullptr, "GPU buffer copies need an active recorder"); + const auto commands = recorder->Commands(); + // Every earlier recorded store into the imports (dispatches in place, fills, GPU labels, + // GPU-direct write-backs and the flushes just recorded) and the host's own writes precede the + // copies. A CPU write made after the batch is submitted is the game's race, as on hardware. + RecordMemoryBarrier(context, commands, VK_PIPELINE_STAGE_ALL_COMMANDS_BIT, VK_PIPELINE_STAGE_TRANSFER_BIT, VK_ACCESS_SHADER_WRITE_BIT | VK_ACCESS_TRANSFER_WRITE_BIT | VK_ACCESS_HOST_WRITE_BIT, VK_ACCESS_TRANSFER_READ_BIT); + for (auto* region : copies) { + const auto bytes = region->end - region->begin; + if (region->buffer == nullptr) { + // Not made by UploadPrepare (the registry was stale then): made here, under the lock. + const auto allocateStart = std::chrono::steady_clock::now(); + region->buffer = std::make_shared(context, static_cast(bytes), gpuCopyUsage(addressable)); + if (profile) allocateUs.fetch_add(microsecondsSince(allocateStart), std::memory_order_relaxed); + } + CopyBuffer(context, commands, region->copySource, region->begin - region->copySourceBase, region->buffer->Handle(), 0, bytes); + // Kept by the batch at record time, as every other recorded target is: the caller keeps + // the resources only after descriptor and pipeline work that may throw, and a released + // buffer would go back to the pool (reused or destroyed) under this copy command. The + // source is a live import or one retireImport already handed to this batch. + recorder->Keep(region->buffer); + region->snapshot.clear(); + if (profile) { + Copies().gpuCopies.fetch_add(1, std::memory_order_relaxed); + Copies().gpuCopyBytes.fetch_add(bytes, std::memory_order_relaxed); + } + } + // The copied bytes are visible to the shaders bound to the buffers, which may also store into + // them. Every stage: dispatches and draws share this path, and draws run mesh, task and + // tessellation shaders too (the batch's own barriers around dispatches are as wide). + RecordMemoryBarrier(context, commands, VK_PIPELINE_STAGE_TRANSFER_BIT, VK_PIPELINE_STAGE_ALL_COMMANDS_BIT, VK_ACCESS_TRANSFER_WRITE_BIT, VK_ACCESS_SHADER_READ_BIT | VK_ACCESS_UNIFORM_READ_BIT | VK_ACCESS_SHADER_WRITE_BIT); +} + +void GuestBufferMemory::RecordCopyBacks(Recorder& recorder) { + if (!uploaded || committed) return; + static const bool profile = std::getenv("APS5_PROFILE_DRAW") != nullptr; + // The written ranges, merged: a V# bound twice would otherwise copy the same bytes twice. + std::vector> merged; + bool recording = false; + VkCommandBuffer commands = VK_NULL_HANDLE; + for (auto& region : regions) { + if (!region.gpuCopy || region.copiedBack) continue; + if (merged.empty() && !writes.empty()) { + auto sorted = writes; + std::sort(sorted.begin(), sorted.end()); + for (const auto& range : sorted) { + if (!merged.empty() && range.first < merged.back().second) merged.back().second = std::max(merged.back().second, range.second); + else merged.push_back(range); + } + } + for (const auto& [begin, end] : merged) { + const auto from = std::max(begin, region.begin); + const auto to = std::min(end, region.end); + if (from >= to) continue; + if (!recording) { + commands = recorder.Commands(); + // The shader's stores into the buffer (whichever stage made them) precede the copy. + RecordMemoryBarrier(context, commands, VK_PIPELINE_STAGE_ALL_COMMANDS_BIT, VK_PIPELINE_STAGE_TRANSFER_BIT, VK_ACCESS_SHADER_WRITE_BIT, VK_ACCESS_TRANSFER_READ_BIT); + recording = true; + } + // Exactly the sub-range the shader may write, in place in the import: what the shader + // would have stored there itself had the range been bindable. + CopyBuffer(context, commands, region.buffer->Handle(), from - region.begin, region.copySource, from - region.copySourceBase, to - from); + recorder.Keep(region.buffer); + if (profile) { + Copies().gpuCopyBacks.fetch_add(1, std::memory_order_relaxed); + Copies().gpuCopyBackBytes.fetch_add(to - from, std::memory_order_relaxed); + } + } + // Only once its copies are recorded: a throw above leaves the region counted by + // HasCopiedWrites, so the caller still registers the CPU write-back that stores the + // staging bytes, instead of losing the shader's results. + region.copiedBack = true; + } + if (!recording) return; + // As a GPU label store or fill: visible to everything recorded after (shaders, transfers, an + // indirect dispatch's arguments) and to the host once the batch completed; the caller's + // pending-write note makes CPU readers wait for that. + RecordMemoryBarrier(context, commands, VK_PIPELINE_STAGE_TRANSFER_BIT, VK_PIPELINE_STAGE_ALL_COMMANDS_BIT | VK_PIPELINE_STAGE_HOST_BIT, VK_ACCESS_TRANSFER_WRITE_BIT, VK_ACCESS_SHADER_READ_BIT | VK_ACCESS_SHADER_WRITE_BIT | VK_ACCESS_TRANSFER_READ_BIT | VK_ACCESS_INDIRECT_COMMAND_READ_BIT | VK_ACCESS_HOST_READ_BIT); } VkDescriptorBufferInfo GuestBufferMemory::Descriptor(std::uint64_t address, std::size_t bytes) const { @@ -76,25 +1337,52 @@ VkDescriptorBufferInfo GuestBufferMemory::Descriptor(std::uint64_t address, std: const auto found = std::upper_bound(regions.begin(), regions.end(), address, [](std::uint64_t value, const Region& region) { return value < region.begin; }); Require(found != regions.begin(), "guest buffer has no GPU owner"); const auto& region = *std::prev(found); - Require(address >= region.begin && address + bytes <= region.end && region.buffer != nullptr, "guest buffer view exceeds its GPU owner"); - const auto offset = address - region.begin; + Require(address >= region.begin && address + bytes <= region.end && (region.buffer != nullptr || region.direct != nullptr || region.mirror != nullptr), "guest buffer view exceeds its GPU owner"); + const auto base = region.direct != nullptr ? region.direct->base : region.mirror != nullptr ? region.mirror->base : region.begin; + const auto offset = address - base; Require(context.limits.minStorageBufferOffsetAlignment != 0 && offset % context.limits.minStorageBufferOffsetAlignment == 0, "guest buffer view violates storage buffer offset alignment"); Require(bytes <= context.limits.maxStorageBufferRange, "guest buffer view exceeds descriptor range limit"); - return {region.buffer->Handle(), offset, bytes}; + const auto handle = region.direct != nullptr ? region.direct->buffer : region.mirror != nullptr ? region.mirror->buffer->Handle() : region.buffer->Handle(); + return {handle, offset, bytes}; } std::vector GuestBufferMemory::AddressRanges() const { Require(uploaded && !committed, "guest GPU address ranges are not available"); std::vector result; for (const auto& region : regions) { - Require(region.buffer != nullptr, "incomplete guest GPU upload"); - const auto address = region.buffer->DeviceAddress(); + Require(region.buffer != nullptr || region.direct != nullptr || region.mirror != nullptr, "incomplete guest GPU upload"); + const auto address = region.direct != nullptr ? region.direct->address + (region.begin - region.direct->base) : region.mirror != nullptr ? region.mirror->buffer->DeviceAddress() + (region.begin - region.mirror->base) : region.buffer->DeviceAddress(); Require(region.end - region.begin <= std::numeric_limits::max() - address, "GPU address range overflow"); result.push_back({region.begin, region.end, address, ShaderRecompiler::BdaAbi::Read, 0}); } return result; } +bool GuestBufferMemory::HasCopiedWrites() const { + for (const auto& [begin, end] : writes) { + const auto found = std::upper_bound(regions.begin(), regions.end(), begin, [](auto address, const auto& region) { return address < region.begin; }); + if (found == regions.begin()) continue; + const auto& region = *std::prev(found); + if (region.direct != nullptr) continue; + // A GPU copy whose written sub-ranges were copied back by the GPU lands like a direct + // write; until RecordCopyBacks ran it still counts (a use without it stores in WriteBack). + if (region.gpuCopy && region.copiedBack) continue; + // Writes into a read-only image mirror are never stored (its pages could not take them). + if (region.mirror != nullptr && !region.mirror->writable) continue; + return true; + } + return false; +} + +void GuestBufferMemory::MarkDirectWrites() const { + for (const auto& [begin, end] : writes) { + const auto found = std::upper_bound(regions.begin(), regions.end(), begin, [](auto address, const auto& region) { return address < region.begin; }); + if (found == regions.begin()) continue; + const auto& region = *std::prev(found); + if (region.direct != nullptr || (region.gpuCopy && region.copiedBack)) GuestMemory::MarkWritten(begin, static_cast(end - begin)); + } +} + void GuestBufferMemory::WriteBack() { Require(uploaded && !committed, "guest memory cannot be committed twice or before upload"); std::sort(writes.begin(), writes.end()); @@ -103,18 +1391,109 @@ void GuestBufferMemory::WriteBack() { if (!merged.empty() && range.first < merged.back().second) merged.back().second = std::max(merged.back().second, range.second); else merged.push_back(range); } - std::vector> sources; + // The CPU keeps running while the GPU works, so storing whole ranges would roll back its writes to + // bytes the shader never touched. Only runs that differ from the uploaded copy are stored. for (const auto& [begin, end] : merged) { - GuestMemory::CheckRange(reinterpret_cast(begin), static_cast(end - begin), 1, true); const auto found = std::upper_bound(regions.begin(), regions.end(), begin, [](auto address, const auto& region) { return address < region.begin; }); Require(found != regions.begin(), "write-back range has no GPU owner"); const auto& region = *std::prev(found); + // Every branch below but the direct and copied-back ones stores from the CPU. + if (region.direct == nullptr && !(region.gpuCopy && region.copiedBack) && !(region.mirror != nullptr && !region.mirror->writable)) Recorder::NoteWrittenBack(begin, static_cast(end - begin)); + if (region.direct != nullptr) { + // The GPU wrote imported guest memory directly; page write watching did not see it. + GuestMemory::MarkWritten(begin, static_cast(end - begin)); + continue; + } + if (region.mirror != nullptr) { + if (!region.mirror->writable) continue; + Require(end <= region.end, "write-back range exceeds its GPU owner"); + // The shadow is what the GPU started from; it then takes the mirror's bytes so the next + // refresh does not mistake the GPU's results for a guest change. + auto& mirror = *region.mirror; + const auto offset = static_cast(begin - mirror.base); + const auto length = static_cast(end - begin); + const auto current = mirror.buffer->Bytes().subspan(offset, length); + GuestMemory::WriteChanged(begin, current, std::span(mirror.shadow).subspan(offset, length)); + std::memcpy(mirror.shadow.data() + offset, current.data(), length); + continue; + } + if (region.gpuCopy) { + Require(region.buffer != nullptr && end <= region.end, "write-back range exceeds its GPU owner"); + if (region.copiedBack) { + // The GPU copied the written sub-range back into the import; page write watching + // did not see it. + GuestMemory::MarkWritten(begin, static_cast(end - begin)); + continue; + } + // No copy-back was recorded (a synchronous draw): the staging bytes of the written + // sub-range are stored whole, which is what the copy-back would have done. There is no + // reference copy to diff against (the copy in never passed through the CPU). + static const bool profile = std::getenv("APS5_PROFILE_DRAW") != nullptr; + GuestMemory::Write(begin, region.buffer->Bytes().subspan(static_cast(begin - region.begin), static_cast(end - begin))); + if (profile) Copies().stagingStores.fetch_add(1, std::memory_order_relaxed); + continue; + } Require(region.buffer != nullptr && region.writable && end <= region.end, "write-back range exceeds its GPU owner"); - sources.push_back(region.buffer->Bytes().subspan(static_cast(begin - region.begin), static_cast(end - begin))); + static const bool traceBig = std::getenv("APS5_TRACE_BIGBUF") != nullptr; + if (traceBig && end - begin >= (1u << 20u)) std::fprintf(stderr, "[bigbuf] writeback 0x%llx+0x%llx sparse=%d\n", static_cast(begin), static_cast(end - begin), region.sparse ? 1 : 0); + if (!region.sparse) { + const auto first = static_cast(begin - region.begin); + const auto length = static_cast(end - begin); + GuestMemory::WriteChanged(begin, region.buffer->Bytes().subspan(first, length), std::span(region.uploaded).subspan(first, length)); + continue; + } + std::size_t packed = 0; + for (const auto& [first, last] : region.backed) { + const auto from = std::max(first, begin); + const auto to = std::min(last, end); + if (from < to) { + const auto length = static_cast(to - from); + GuestMemory::WriteChanged(from, region.buffer->Bytes().subspan(static_cast(from - region.begin), length), std::span(region.uploaded).subspan(packed + static_cast(from - first), length)); + } + packed += static_cast(last - first); + } } - for (std::size_t i = 0; i < merged.size(); ++i) GuestMemory::Write(merged[i].first, sources[i]); committed = true; lease.clear(); } +std::optional>> GuestBufferMemory::DirectRegions() const { + if (!uploaded || committed) return std::nullopt; + std::vector> result; + for (const auto& region : regions) { + // A read-only mirror is as fixed as an import: its bytes and VkBuffer never change while its + // Range exists (see ImageMirrorSerial). A writable one is refreshed and written back per build. + const bool fixedMirror = region.mirror != nullptr && !region.mirror->writable; + if (region.direct == nullptr && !fixedMirror) return std::nullopt; + result.emplace_back(region.begin, region.end); + } + return result; +} + +std::uint64_t ImageMirrorSerial(const Context& context, std::uint64_t address, std::size_t bytes) { + if (bytes == 0 || bytes > std::numeric_limits::max() - address || !mirrorsEnabled()) return 0; + // A replaced Range expires the mirror (findMirror), and a replacement mirror has a new serial. + const auto mirror = findMirror(context, address, address + bytes); + return mirror != nullptr && !mirror->writable ? mirror->serial : 0; +} + +std::uint64_t HostImportSerial(const Context& context, std::uint64_t address, std::size_t bytes, bool reconcile) { + if (context.hostImportAlignment == 0 || bytes == 0 || bytes > std::numeric_limits::max() - address) return 0; + auto& state = Imports(); + std::lock_guard lock(state.mutex); + // An import whose registered range changed must not be reused: reconcile first, as an upload + // does (the walk only runs when the registry changed since the last one). + if (reconcile && GuestAllocations::GuestAllocationsGeneration_nid_postfix() != state.refreshedGeneration) static_cast(refreshImports(context, state, GuestAllocations::GuestAllocationsAcquire_nid_postfix())); + auto found = state.imports.upper_bound(address); + if (found == state.imports.begin()) return 0; + --found; + auto& entry = found->second; + if (address < entry.base || address + bytes > entry.base + entry.bytes) return 0; + // Serials are handed out on first use: a fresh import starts at 0, so one made after an earlier + // import of the same range was dropped can never repeat that import's serial. + static std::uint64_t serials = 0; + if (entry.serial == 0) entry.serial = ++serials; + return entry.serial; +} + } diff --git a/core/libs/prx/libSceAgcDriver/Graphics/src/GuestSamplerResource.cpp b/core/libs/prx/libSceAgcDriver/Graphics/src/GuestSamplerResource.cpp index 9528d9f99..a3f2890c8 100644 --- a/core/libs/prx/libSceAgcDriver/Graphics/src/GuestSamplerResource.cpp +++ b/core/libs/prx/libSceAgcDriver/Graphics/src/GuestSamplerResource.cpp @@ -75,14 +75,17 @@ GuestSamplerResource DecodeSamplerResource(std::span words) const auto borderColorType = (words[3] >> 30u) & 0x3u; Require(!forceUnormCoords, "guest sampler descriptor uses unnormalized coordinates which are not implemented"); - Require(anisoThreshold == 0, "guest sampler descriptor uses an anisotropy threshold override which is not implemented"); Require(!forceSrgb, "guest sampler descriptor forces sRGB decoding which is not implemented"); - Require(anisoBias == 0, "guest sampler descriptor uses an anisotropy bias which is not implemented"); - Require(!truncCoord, "guest sampler descriptor uses coordinate truncation which is not implemented"); + // TRUNC_COORD picks point-sampled texels by truncation instead of rounding, and the perf fields + // trade mip/depth precision for speed; Vulkan's nearest filtering already floors, so these only + // move texel selection by half a texel at most and are accepted as is. ANISO_THRESHOLD and + // ANISO_BIAS only tune how many anisotropic taps the hardware takes; the host filter picks its own. + static_cast(truncCoord); + static_cast(anisoThreshold); + static_cast(anisoBias); Require(!disableCubeWrap, "guest sampler descriptor disables seamless cube filtering which is not implemented"); Require(filterMode == 0, "guest sampler descriptor uses a reduction filter mode which is not implemented"); Require(!disableDegamma, "guest sampler descriptor disables degamma which is not implemented"); - Require(perfMip == 0 && perfZ == 0, "guest sampler descriptor uses performance counters which are not implemented"); Require(lodBiasSec == 0, "guest sampler descriptor uses a secondary LOD bias which is not implemented"); Require(!pointPreclamp, "guest sampler descriptor uses point preclamping which is not implemented"); Require(!anisoOverride, "guest sampler descriptor uses an anisotropy override which is not implemented"); diff --git a/core/libs/prx/libSceAgcDriver/Graphics/src/GuestTextureResource.cpp b/core/libs/prx/libSceAgcDriver/Graphics/src/GuestTextureResource.cpp index b7e13b992..3cd364eae 100644 --- a/core/libs/prx/libSceAgcDriver/Graphics/src/GuestTextureResource.cpp +++ b/core/libs/prx/libSceAgcDriver/Graphics/src/GuestTextureResource.cpp @@ -1,5 +1,7 @@ +#include #include "prx/libSceAgcDriver/Graphics/include/GuestTextureResource.hpp" #include "prx/libSceAgcDriver/Graphics/include/Context.hpp" +#include #include #include @@ -17,7 +19,10 @@ TextureTileMode resolveTileMode(std::uint32_t raw) { case 0x01: return TextureTileMode::kStandard256B; case 0x05: return TextureTileMode::kStandard4KB; case 0x09: return TextureTileMode::kStandard64KB; - case 0x1b: return TextureTileMode::RenderTarget64KB; + case 0x18: return TextureTileMode::kZ64KBX; + case 0x19: return TextureTileMode::kS64KBX; + case 0x1a: return TextureTileMode::kD64KBX; + case 0x1b: return TextureTileMode::kR64KBX; default: throw std::runtime_error("AGC graphics: guest texture descriptor uses an unsupported tile mode " + std::to_string(raw)); } } @@ -26,6 +31,7 @@ TextureDimension resolveDimension(std::uint32_t raw) { switch (raw) { case 8: return TextureDimension::k1D; case 9: return TextureDimension::k2D; + case 10: return TextureDimension::k3D; case 11: return TextureDimension::kCube; case 13: return TextureDimension::k2DArray; default: throw std::runtime_error("AGC graphics: guest texture descriptor uses an unsupported image type " + std::to_string(raw)); @@ -51,7 +57,7 @@ GuestTextureResource DecodeTextureResource(std::span words) const auto dstSelZ = (words[3] >> 6u) & 0x7u; const auto dstSelW = (words[3] >> 9u) & 0x7u; const auto baseLevel = (words[3] >> 12u) & 0xfu; - const auto lastLevel = (words[3] >> 16u) & 0xfu; + auto lastLevel = (words[3] >> 16u) & 0xfu; const auto tileModeRaw = (words[3] >> 20u) & 0x1fu; const auto bcSwizzle = (words[3] >> 25u) & 0x7u; const auto typeRaw = (words[3] >> 28u) & 0xfu; @@ -83,21 +89,54 @@ GuestTextureResource DecodeTextureResource(std::span words) requireValidDstSel(dstSelW); Require(minLod == 0, "guest texture descriptor uses a nonzero minimum LOD clamp which is not implemented"); - Require(minLodWarn == 0, "guest texture descriptor uses a minimum LOD warning threshold which is not implemented"); - Require(mipStatsCntId == 0 && !mipStatsCntEn, "guest texture descriptor uses mip statistics counters which are not implemented"); + // The LOD warning threshold and mip statistics counters are texture-streaming feedback: the GPU + // reports which mips were wanted. Nothing is reported back here, which only affects what the title + // chooses to stream, not what this access returns. + if (minLodWarn != 0 || mipStatsCntId != 0 || mipStatsCntEn) { + static bool reported = false; + if (!reported) { + reported = true; + std::fprintf(stderr, "[gpu] texture streaming feedback (LOD warning / mip statistics) is not reported\n"); + } + } Require(!cornerSample, "guest texture descriptor uses corner sampling which is not implemented"); Require(!prtDefColor, "guest texture descriptor uses a partially resident default color which is not implemented"); Require(arrayPitch == 0, "guest texture descriptor uses a nonzero array pitch which is not implemented"); Require(!msaaDepth, "guest texture descriptor uses MSAA which is not implemented"); - Require(maxUncompBlkSize == 0 && maxCompBlkSize == 0, "guest texture descriptor uses DCC block size overrides which are not implemented"); - Require(!metaPipeAligned && !writeCompress && !metaCompress && !dccAlphaPos && !dccColorTransf && metaAddr == 0, "guest texture descriptor uses metadata compression which is not implemented"); + // DCC block sizes only apply with metadata compression, which is rejected above. + static_cast(maxUncompBlkSize); + static_cast(maxCompBlkSize); + // Surfaces are always written uncompressed here (render targets and storage images bypass DCC), so + // only the fast-clear keys in DCC metadata change what a read returns (see DccMetadata.hpp). + static_cast(metaPipeAligned); + static_cast(writeCompress); + if (dccColorTransf) { + static bool reported = false; + if (!reported) { + reported = true; + std::fprintf(stderr, "[gpu] texture DCC color transform is ignored (word6=0x%08x word7=0x%08x)\n", words[6], words[7]); + } + } Require(bcSwizzle == 0, "guest texture descriptor uses a BC swizzle which is not implemented"); Require(baseLevel <= lastLevel, "guest texture descriptor has a base mip level past its last mip level"); - Require(lastLevel == maxMip, "guest texture descriptor must expose every mip level down to the last one"); - const auto tileMode = resolveTileMode(tileModeRaw); - const auto dimension = resolveDimension(typeRaw); + + const auto describe = [&] { return " (format " + std::to_string(format) + ", " + std::to_string(width) + "x" + std::to_string(height) + "x" + std::to_string(depth + 1) + ", type " + std::to_string(typeRaw) + ", tile " + std::to_string(tileModeRaw) + ", mips " + std::to_string(baseLevel) + ".." + std::to_string(lastLevel) + " of " + std::to_string(maxMip + 1) + ")"; }; + TextureTileMode tileMode; + TextureDimension dimension; + try { + tileMode = resolveTileMode(tileModeRaw); + dimension = resolveDimension(typeRaw); + } catch (const std::exception& error) { + throw std::runtime_error(std::string(error.what()) + describe()); + } + Require(baseLevel <= maxMip, "guest texture descriptor starts past the surface's last mip level" + describe()); + // Views may name levels past MAX_MIP (a 512x512 view through 1x1 over a 9-level surface); the + // hardware never addresses them, so the view ends at the surface's last level. + lastLevel = std::min(lastLevel, maxMip); + // XOR swizzles fold a pipe/bank XOR into the low address bits; only unmodified 64 KiB bases are modeled. + Require(XorSwizzleMode(tileMode) == 0 || (baseAddress & 0xffffu) == 0, "guest texture descriptor combines an XOR swizzle with a pipe/bank XOR base which is not implemented"); switch (dimension) { case TextureDimension::k1D: @@ -106,6 +145,10 @@ GuestTextureResource DecodeTextureResource(std::span words) case TextureDimension::k2D: Require(depth == 0 && baseArray == 0, "guest 2D texture descriptor has a nonzero depth or base array"); break; + case TextureDimension::k3D: + // word4 DEPTH holds the depth minus one; BASE_ARRAY has no meaning for volumes. + Require(baseArray == 0, "guest 3D texture descriptor has a nonzero base array"); + break; case TextureDimension::k2DArray: Require(baseArray <= depth, "guest 2D array texture descriptor has a base array past its last array slice"); break; @@ -124,6 +167,7 @@ GuestTextureResource DecodeTextureResource(std::span words) result.baseArray = baseArray; result.mipCount = maxMip + 1u; result.baseLevel = baseLevel; + result.lastLevel = lastLevel; result.tileMode = tileMode; result.dimension = dimension; result.format = format; @@ -131,6 +175,8 @@ GuestTextureResource DecodeTextureResource(std::span words) result.dstSelY = static_cast(dstSelY); result.dstSelZ = static_cast(dstSelZ); result.dstSelW = static_cast(dstSelW); + result.dccAddress = metaCompress ? metaAddr << 8u : 0u; + result.dccAlphaOnMsb = dccAlphaPos; return result; } @@ -138,9 +184,9 @@ bool MatchesGuestDimension(ShaderRecompiler::DescriptorImageShape shape, Texture switch (shape) { case ShaderRecompiler::DescriptorImageShape::Image1D: return dimension == TextureDimension::k1D; case ShaderRecompiler::DescriptorImageShape::Image2D: return dimension == TextureDimension::k2D; - case ShaderRecompiler::DescriptorImageShape::Image2DArray: return dimension == TextureDimension::k2DArray; + case ShaderRecompiler::DescriptorImageShape::Image2DArray: return dimension == TextureDimension::k2DArray || dimension == TextureDimension::kCube; case ShaderRecompiler::DescriptorImageShape::ImageCube: return dimension == TextureDimension::kCube; - case ShaderRecompiler::DescriptorImageShape::Image3D: return false; + case ShaderRecompiler::DescriptorImageShape::Image3D: return dimension == TextureDimension::k3D; } throw std::runtime_error("AGC graphics: MatchesGuestDimension encountered an unknown descriptor image shape"); } diff --git a/core/libs/prx/libSceAgcDriver/Graphics/src/Pipeline.cpp b/core/libs/prx/libSceAgcDriver/Graphics/src/Pipeline.cpp index 3d55bc878..517072b9a 100644 --- a/core/libs/prx/libSceAgcDriver/Graphics/src/Pipeline.cpp +++ b/core/libs/prx/libSceAgcDriver/Graphics/src/Pipeline.cpp @@ -1,16 +1,53 @@ #include "prx/libSceAgcDriver/Graphics/include/Pipeline.hpp" #include "prx/libSceAgcDriver/Graphics/include/VertexInput.hpp" #include +#include +#include #include +#include +#include +#include +#include +#include +#include #include namespace AgcDriver::Graphics { -Pipeline::Pipeline(const Context& context, const State& state, const RenderTarget* target, const ShaderResources& resources, std::span shaders) : context(context), _modules(shaders.size()) { - Require((target != nullptr) == state.hasColorTarget, "render target does not match decoded color state"); - Require(state.renderExtent.width != 0 && state.renderExtent.height != 0 && state.renderExtent.width <= context.limits.maxFramebufferWidth && state.renderExtent.height <= context.limits.maxFramebufferHeight, "framebuffer extent exceeds device limits"); +Framebuffer::Framebuffer(const Context& context, VkRenderPass renderPass, std::span targets, VkExtent2D extent) : context(context) { + // Cached objects outlive their device's teardown; they must not keep its buffer pool alive past it. + this->context.bufferPool.reset(); + Require(extent.width != 0 && extent.height != 0 && extent.width <= context.limits.maxFramebufferWidth && extent.height <= context.limits.maxFramebufferHeight, "framebuffer extent exceeds device limits"); + VkFramebufferCreateInfo framebufferInfo{VK_STRUCTURE_TYPE_FRAMEBUFFER_CREATE_INFO}; + framebufferInfo.renderPass = renderPass; + framebufferInfo.attachmentCount = static_cast(targets.size()); + framebufferInfo.pAttachments = targets.empty() ? nullptr : targets.data(); + framebufferInfo.width = extent.width; + framebufferInfo.height = extent.height; + framebufferInfo.layers = 1; + Check(context.Function("vkCreateFramebuffer")(context.device, &framebufferInfo, nullptr, &framebuffer), "vkCreateFramebuffer"); +} + +Framebuffer::~Framebuffer() { + if (framebuffer) context.Function("vkDestroyFramebuffer")(context.device, framebuffer, nullptr); +} + +void ValidateViewport(const Context& context, const VkViewport& viewport) { + Require(std::isfinite(viewport.minDepth) && std::isfinite(viewport.maxDepth), "non-finite viewport depth range"); + Require(context.depthRangeUnrestricted || (viewport.minDepth >= 0 && viewport.minDepth <= 1 && viewport.maxDepth >= 0 && viewport.maxDepth <= 1), "viewport depth outside [0, 1] requires VK_EXT_depth_range_unrestricted"); + Require(std::isfinite(viewport.x) && std::isfinite(viewport.y) && std::isfinite(viewport.width) && std::isfinite(viewport.height), "viewport arithmetic overflow"); + Require(viewport.width <= context.limits.maxViewportDimensions[0] && std::abs(viewport.height) <= context.limits.maxViewportDimensions[1], "viewport dimensions exceed device limits"); + Require(viewport.x >= context.limits.viewportBoundsRange[0] && viewport.x + viewport.width <= context.limits.viewportBoundsRange[1], "viewport X exceeds device bounds"); + Require(std::min(viewport.y, viewport.y + viewport.height) >= context.limits.viewportBoundsRange[0] && std::max(viewport.y, viewport.y + viewport.height) <= context.limits.viewportBoundsRange[1], "viewport Y exceeds device bounds"); +} + +Pipeline::Pipeline(const Context& context, const State& state, const VertexInputLayout& vertexInput, const ShaderResources& resources, std::span shaders) : context(context), _modules(shaders.size()), attachments(state.colors.size()) { + // A cached pipeline may outlive its device's teardown (see ClearCachedPipelines); it must not keep + // the buffer pool, which is reset with the device, alive past it. + this->context.bufferPool.reset(); + Require(state.blends.size() == state.colors.size(), "blend states do not match decoded color state"); + Require(state.colors.size() <= context.limits.maxColorAttachments, "color targets exceed device attachment limits"); Require(state.hasColorTarget || (context.limits.framebufferNoAttachmentsSampleCounts & VK_SAMPLE_COUNT_1_BIT) != 0, "device does not support single-sample rendering without attachments"); - ValidateShaders(shaders, state, context.subgroup, context.fragmentShaderBarycentric); Require(!state.negativeOneToOne || context.depthClipControl, "negative-one-to-one depth clipping requires VK_EXT_depth_clip_control with depthClipControl enabled"); if (state.rectList) Require(context.tessellationShader && context.limits.maxTessellationPatchSize >= 4, "rect-list requires tessellation with four output control points"); if (state.stages.tessellation) { @@ -23,13 +60,6 @@ Pipeline::Pipeline(const Context& context, const State& state, const RenderTarge Require(mesh.threadsPerGroup <= context.meshLimits.maxMeshWorkGroupInvocations && mesh.threadsPerGroup <= context.meshLimits.maxMeshWorkGroupSize[0], "mesh workgroup exceeds device limits"); Require(mesh.maxVertices <= context.meshLimits.maxMeshOutputVertices && mesh.maxPrimitives <= context.meshLimits.maxMeshOutputPrimitives && static_cast(mesh.ldsSizeDwords) * 4 <= context.meshLimits.maxMeshSharedMemorySize, "mesh output or LDS exceeds device limits"); } - const auto& viewport = state.viewport; - Require(std::isfinite(viewport.minDepth) && std::isfinite(viewport.maxDepth), "non-finite viewport depth range"); - Require(context.depthRangeUnrestricted || (viewport.minDepth >= 0 && viewport.minDepth <= 1 && viewport.maxDepth >= 0 && viewport.maxDepth <= 1), "viewport depth outside [0, 1] requires VK_EXT_depth_range_unrestricted"); - Require(std::isfinite(viewport.x) && std::isfinite(viewport.y) && std::isfinite(viewport.width) && std::isfinite(viewport.height), "viewport arithmetic overflow"); - Require(viewport.width <= context.limits.maxViewportDimensions[0] && std::abs(viewport.height) <= context.limits.maxViewportDimensions[1], "viewport dimensions exceed device limits"); - Require(viewport.x >= context.limits.viewportBoundsRange[0] && viewport.x + viewport.width <= context.limits.viewportBoundsRange[1], "viewport X exceeds device bounds"); - Require(std::min(viewport.y, viewport.y + viewport.height) >= context.limits.viewportBoundsRange[0] && std::max(viewport.y, viewport.y + viewport.height) <= context.limits.viewportBoundsRange[1], "viewport Y exceeds device bounds"); const auto pushStages = PushConstantStages(shaders); Require(pushStages == 0 || context.limits.maxPushConstantsSize >= PipelinePushConstantBytes, "graphics push constant range exceeds device limit"); try { @@ -46,6 +76,8 @@ Pipeline::Pipeline(const Context& context, const State& state, const RenderTarge stages[i].module = _modules[i]; stages[i].pName = "main"; } + // A descriptor set layout with the same bindings as this one is compatible with the pipeline + // layout, so later draws bind their own ShaderResources' set under it. const auto setLayout = resources.Layout(); const VkPushConstantRange push{pushStages, 0, PipelinePushConstantBytes}; VkPipelineLayoutCreateInfo layoutInfo{VK_STRUCTURE_TYPE_PIPELINE_LAYOUT_CREATE_INFO}; @@ -54,61 +86,61 @@ Pipeline::Pipeline(const Context& context, const State& state, const RenderTarge layoutInfo.pushConstantRangeCount = pushStages != 0 ? 1 : 0; layoutInfo.pPushConstantRanges = pushStages != 0 ? &push : nullptr; Check(context.Function("vkCreatePipelineLayout")(context.device, &layoutInfo, nullptr, &layout), "vkCreatePipelineLayout graphics"); - VkAttachmentDescription color{}; - color.format = state.color.format; - color.samples = VK_SAMPLE_COUNT_1_BIT; - color.loadOp = VK_ATTACHMENT_LOAD_OP_LOAD; - color.storeOp = VK_ATTACHMENT_STORE_OP_STORE; - color.stencilLoadOp = VK_ATTACHMENT_LOAD_OP_DONT_CARE; - color.stencilStoreOp = VK_ATTACHMENT_STORE_OP_DONT_CARE; - color.initialLayout = VK_IMAGE_LAYOUT_COLOR_ATTACHMENT_OPTIMAL; - color.finalLayout = VK_IMAGE_LAYOUT_COLOR_ATTACHMENT_OPTIMAL; - VkAttachmentReference reference{0, VK_IMAGE_LAYOUT_COLOR_ATTACHMENT_OPTIMAL}; + std::vector colors; + std::vector references; + for (std::uint32_t index = 0; index < state.colors.size(); ++index) { + VkAttachmentDescription color{}; + color.format = state.colors[index].format; + color.samples = VK_SAMPLE_COUNT_1_BIT; + color.loadOp = VK_ATTACHMENT_LOAD_OP_LOAD; + color.storeOp = VK_ATTACHMENT_STORE_OP_STORE; + color.stencilLoadOp = VK_ATTACHMENT_LOAD_OP_DONT_CARE; + color.stencilStoreOp = VK_ATTACHMENT_STORE_OP_DONT_CARE; + color.initialLayout = VK_IMAGE_LAYOUT_COLOR_ATTACHMENT_OPTIMAL; + color.finalLayout = VK_IMAGE_LAYOUT_COLOR_ATTACHMENT_OPTIMAL; + colors.push_back(color); + references.push_back({index, VK_IMAGE_LAYOUT_COLOR_ATTACHMENT_OPTIMAL}); + } VkSubpassDescription subpass{}; subpass.pipelineBindPoint = VK_PIPELINE_BIND_POINT_GRAPHICS; - subpass.colorAttachmentCount = state.hasColorTarget ? 1 : 0; - subpass.pColorAttachments = state.hasColorTarget ? &reference : nullptr; + subpass.colorAttachmentCount = static_cast(references.size()); + subpass.pColorAttachments = references.empty() ? nullptr : references.data(); VkRenderPassCreateInfo passInfo{VK_STRUCTURE_TYPE_RENDER_PASS_CREATE_INFO}; - passInfo.attachmentCount = state.hasColorTarget ? 1 : 0; - passInfo.pAttachments = state.hasColorTarget ? &color : nullptr; + passInfo.attachmentCount = static_cast(colors.size()); + passInfo.pAttachments = colors.empty() ? nullptr : colors.data(); passInfo.subpassCount = 1; passInfo.pSubpasses = &subpass; Check(context.Function("vkCreateRenderPass")(context.device, &passInfo, nullptr, &renderPass), "vkCreateRenderPass"); - const auto view = target ? target->View() : VK_NULL_HANDLE; - VkFramebufferCreateInfo framebufferInfo{VK_STRUCTURE_TYPE_FRAMEBUFFER_CREATE_INFO}; - framebufferInfo.renderPass = renderPass; - framebufferInfo.attachmentCount = state.hasColorTarget ? 1 : 0; - framebufferInfo.pAttachments = state.hasColorTarget ? &view : nullptr; - framebufferInfo.width = state.renderExtent.width; - framebufferInfo.height = state.renderExtent.height; - framebufferInfo.layers = 1; - Check(context.Function("vkCreateFramebuffer")(context.device, &framebufferInfo, nullptr, &framebuffer), "vkCreateFramebuffer"); VkPipelineVertexInputStateCreateInfo input{VK_STRUCTURE_TYPE_PIPELINE_VERTEX_INPUT_STATE_CREATE_INFO}; - const auto vertexInput = BuildVertexInputLayout(context, shaders.front().program->vertexAttributes); input.vertexBindingDescriptionCount = static_cast(vertexInput.bindings.size()); input.pVertexBindingDescriptions = vertexInput.bindings.data(); input.vertexAttributeDescriptionCount = static_cast(vertexInput.attributes.size()); input.pVertexAttributeDescriptions = vertexInput.attributes.data(); VkPipelineInputAssemblyStateCreateInfo assembly{VK_STRUCTURE_TYPE_PIPELINE_INPUT_ASSEMBLY_STATE_CREATE_INFO}; assembly.topology = state.topology; + // Viewport and scissor are set per draw (Begin), so they do not multiply pipelines; the depth + // clip control stays baked in. VkPipelineViewportStateCreateInfo viewports{VK_STRUCTURE_TYPE_PIPELINE_VIEWPORT_STATE_CREATE_INFO}; VkPipelineViewportDepthClipControlCreateInfoEXT depthClip{VK_STRUCTURE_TYPE_PIPELINE_VIEWPORT_DEPTH_CLIP_CONTROL_CREATE_INFO_EXT}; depthClip.negativeOneToOne = state.negativeOneToOne; if (state.negativeOneToOne) viewports.pNext = &depthClip; viewports.viewportCount = 1; - viewports.pViewports = &state.viewport; viewports.scissorCount = 1; - viewports.pScissors = &state.scissor; + const std::array dynamicStates{VK_DYNAMIC_STATE_VIEWPORT, VK_DYNAMIC_STATE_SCISSOR}; + VkPipelineDynamicStateCreateInfo dynamic{VK_STRUCTURE_TYPE_PIPELINE_DYNAMIC_STATE_CREATE_INFO}; + dynamic.dynamicStateCount = static_cast(dynamicStates.size()); + dynamic.pDynamicStates = dynamicStates.data(); VkPipelineRasterizationStateCreateInfo raster{VK_STRUCTURE_TYPE_PIPELINE_RASTERIZATION_STATE_CREATE_INFO}; raster.polygonMode = VK_POLYGON_MODE_FILL; + raster.depthClampEnable = state.depthClamp && context.depthClamp ? VK_TRUE : VK_FALSE; raster.cullMode = state.cullMode; raster.frontFace = state.frontFace; raster.lineWidth = 1; VkPipelineMultisampleStateCreateInfo samples{VK_STRUCTURE_TYPE_PIPELINE_MULTISAMPLE_STATE_CREATE_INFO}; samples.rasterizationSamples = VK_SAMPLE_COUNT_1_BIT; VkPipelineColorBlendStateCreateInfo blend{VK_STRUCTURE_TYPE_PIPELINE_COLOR_BLEND_STATE_CREATE_INFO}; - blend.attachmentCount = state.hasColorTarget ? 1 : 0; - blend.pAttachments = state.hasColorTarget ? &state.blend : nullptr; + blend.attachmentCount = static_cast(state.blends.size()); + blend.pAttachments = state.blends.empty() ? nullptr : state.blends.data(); std::copy(state.blendConstants.begin(), state.blendConstants.end(), blend.blendConstants); VkGraphicsPipelineCreateInfo pipelineInfo{VK_STRUCTURE_TYPE_GRAPHICS_PIPELINE_CREATE_INFO}; pipelineInfo.stageCount = static_cast(stages.size()); @@ -124,6 +156,7 @@ Pipeline::Pipeline(const Context& context, const State& state, const RenderTarge pipelineInfo.pRasterizationState = &raster; pipelineInfo.pMultisampleState = &samples; pipelineInfo.pColorBlendState = &blend; + pipelineInfo.pDynamicState = &dynamic; pipelineInfo.layout = layout; pipelineInfo.renderPass = renderPass; Check(context.Function("vkCreateGraphicsPipelines")(context.device, context.pipelineCache, 1, &pipelineInfo, nullptr, &pipeline), "vkCreateGraphicsPipelines"); @@ -138,26 +171,83 @@ Pipeline::~Pipeline() { } void Pipeline::release() noexcept { + framebuffers.clear(); if (pipeline) context.Function("vkDestroyPipeline")(context.device, pipeline, nullptr); - if (framebuffer) context.Function("vkDestroyFramebuffer")(context.device, framebuffer, nullptr); if (renderPass) context.Function("vkDestroyRenderPass")(context.device, renderPass, nullptr); if (layout) context.Function("vkDestroyPipelineLayout")(context.device, layout, nullptr); for (auto module : _modules) { if (module) context.Function("vkDestroyShaderModule")(context.device, module, nullptr); } + pipeline = VK_NULL_HANDLE; + renderPass = VK_NULL_HANDLE; + layout = VK_NULL_HANDLE; + _modules.clear(); +} + +void Pipeline::Abandon() noexcept { + for (auto& entry : framebuffers) entry.framebuffer->Abandon(); + framebuffers.clear(); + pipeline = VK_NULL_HANDLE; + renderPass = VK_NULL_HANDLE; + layout = VK_NULL_HANDLE; + _modules.clear(); +} + +void Framebuffer::Abandon() noexcept { + framebuffer = VK_NULL_HANDLE; } VkPipelineLayout Pipeline::Layout() const { return layout; } -void Pipeline::Begin(VkCommandBuffer commands, VkExtent2D extent) const { +std::shared_ptr Pipeline::AcquireFramebuffer(std::span targets, std::span> owners, VkExtent2D extent) { + Require(targets.size() == attachments && targets.size() == owners.size(), "render targets do not match the pipeline's color attachments"); + const bool resident = std::all_of(owners.begin(), owners.end(), [](const auto& owner) { return owner != nullptr; }); + if (resident) { + // Entries whose views are gone can never match again and go as soon as no recorded draw holds + // them (Kept keeps its framebuffer until the batch completes). + std::erase_if(framebuffers, [](const CachedFramebuffer& entry) { + return entry.framebuffer.use_count() == 1 && std::any_of(entry.owners.begin(), entry.owners.end(), [](const auto& owner) { return owner.expired(); }); + }); + for (auto it = framebuffers.begin(); it != framebuffers.end(); ++it) { + if (it->extent.width != extent.width || it->extent.height != extent.height || !std::equal(it->views.begin(), it->views.end(), targets.begin(), targets.end())) continue; + // View handles are recycled once a StorageTexture is destroyed, so the owners must be the + // very objects the views were made for. + bool same = true; + for (std::size_t i = 0; i < owners.size() && same; ++i) same = it->owners[i].lock().get() == owners[i].get(); + if (!same) continue; + std::rotate(it, std::next(it), framebuffers.end()); + return framebuffers.back().framebuffer; + } + } + auto framebuffer = std::make_shared(context, renderPass, targets, extent); + if (!resident) return framebuffer; + // Beyond the bound the least recently used unreferenced entry goes. + constexpr std::size_t bound = 8; + while (framebuffers.size() >= bound) { + const auto victim = std::find_if(framebuffers.begin(), framebuffers.end(), [](const CachedFramebuffer& entry) { return entry.framebuffer.use_count() == 1; }); + if (victim == framebuffers.end()) break; + framebuffers.erase(victim); + } + CachedFramebuffer entry; + entry.views.assign(targets.begin(), targets.end()); + entry.owners.assign(owners.begin(), owners.end()); + entry.extent = extent; + entry.framebuffer = framebuffer; + framebuffers.push_back(std::move(entry)); + return framebuffer; +} + +void Pipeline::Begin(VkCommandBuffer commands, const Framebuffer& framebuffer, VkExtent2D extent, const VkViewport& viewport, const VkRect2D& scissor) const { VkRenderPassBeginInfo begin{VK_STRUCTURE_TYPE_RENDER_PASS_BEGIN_INFO}; begin.renderPass = renderPass; - begin.framebuffer = framebuffer; + begin.framebuffer = framebuffer.Handle(); begin.renderArea = {{0, 0}, extent}; context.Function("vkCmdBeginRenderPass")(commands, &begin, VK_SUBPASS_CONTENTS_INLINE); context.Function("vkCmdBindPipeline")(commands, VK_PIPELINE_BIND_POINT_GRAPHICS, pipeline); + context.Function("vkCmdSetViewport")(commands, 0, 1, &viewport); + context.Function("vkCmdSetScissor")(commands, 0, 1, &scissor); } void Pipeline::PushConstants(VkCommandBuffer commands, std::span shaders) const { @@ -167,4 +257,211 @@ void Pipeline::PushConstants(VkCommandBuffer commands, std::span("vkCmdPushConstants")(commands, layout, stages, 0, PipelinePushConstantBytes, bytes.data()); } +namespace { + +template +void append(std::vector& key, const TValue& value) { + static_assert(std::is_trivially_copyable_v); + const auto bytes = std::as_bytes(std::span(&value, 1)); + key.insert(key.end(), bytes.begin(), bytes.end()); +} + +// Everything the Pipeline objects are built from, or empty when a stage's result has no variant id +// (the recompiler could not identify it, so nothing else may share its pipeline). The rect-list +// control and evaluation stages are generated from the vertex and fragment results, which the key +// already names, so they carry no id of their own. +std::vector pipelineKey(const Context& context, const State& state, const VertexInputLayout& input, const ShaderResources& resources, std::span shaders) { + using Stage = ShaderRecompiler::ShaderStage; + std::vector key; + append(key, context.device); + append(key, shaders.size()); + for (const auto& shader : shaders) { + Require(shader.program != nullptr, "missing compiled shader"); + const bool generated = state.rectList && (shader.stage == Stage::TessellationControl || shader.stage == Stage::TessellationEvaluation); + if (!generated && shader.program->variantId == 0) return {}; + append(key, shader.stage); + append(key, generated ? std::uint64_t{0} : shader.program->variantId); + // Where the stage's push constants sit in the block (AssemblePushConstants). + append(key, shader.pushConstantOffset); + } + append(key, PushConstantStages(shaders)); + append(key, input.bindings.size()); + for (const auto& binding : input.bindings) { + append(key, binding.binding); + append(key, binding.stride); + append(key, binding.inputRate); + } + append(key, input.attributes.size()); + for (const auto& attribute : input.attributes) { + append(key, attribute.location); + append(key, attribute.binding); + append(key, attribute.format); + append(key, attribute.offset); + } + append(key, resources.LayoutKey().size()); + for (const auto word : resources.LayoutKey()) append(key, word); + append(key, state.hasColorTarget); + append(key, state.rectList); + append(key, state.topology); + append(key, state.cullMode); + append(key, state.frontFace); + append(key, state.negativeOneToOne); + append(key, state.depthClamp && context.depthClamp); + append(key, state.blends.size()); + for (const auto& blend : state.blends) append(key, blend); + for (const auto value : state.blendConstants) append(key, value); + append(key, state.colors.size()); + for (const auto& color : state.colors) append(key, color.format); + append(key, state.stages.mesh.has_value()); + if (state.stages.mesh) { + const auto& mesh = *state.stages.mesh; + append(key, mesh.inputPrimitive); + append(key, mesh.primitivesPerGroup); + append(key, mesh.verticesPerGroup); + append(key, mesh.maxVertices); + append(key, mesh.maxPrimitives); + append(key, mesh.threadsPerGroup); + append(key, mesh.ldsSizeDwords); + append(key, mesh.provokingVertex); + } + append(key, state.stages.tessellation.has_value()); + if (state.stages.tessellation) { + const auto& tessellation = *state.stages.tessellation; + append(key, tessellation.inputControlPoints); + append(key, tessellation.outputControlPoints); + append(key, tessellation.domain); + append(key, tessellation.partitioning); + append(key, tessellation.outputTopology); + } + return key; +} + +std::uint64_t hashKey(const std::vector& key) { + std::uint64_t hash = 14695981039346656037ull; + for (const auto byte : key) { + hash ^= static_cast(byte); + hash *= 1099511628211ull; + } + return hash; +} + +struct PipelineStore { + struct Entry { + VkDevice device; + // The device's buffer pool at insertion: it is made and reset with the device, so it tells + // the device instance apart from a later one the loader gave the same handle value. + std::weak_ptr pool; + std::uint64_t hash; + std::vector key; + std::shared_ptr pipeline; + }; + std::mutex mutex; + // Least recently used first. + std::list entries; + std::unordered_map::iterator> index; + std::uint64_t hits = 0; + std::uint64_t misses = 0; + std::uint64_t uncached = 0; + std::uint64_t evicted = 0; + std::chrono::steady_clock::time_point lastReport = std::chrono::steady_clock::now(); +}; + +// Never destroyed: the pipelines belong to a device that may already be gone when statics die, and +// the device's teardown (ClearCachedPipelines) is the place to destroy them. +PipelineStore& Pipelines() { + static auto* store = new PipelineStore(); + return *store; +} + +// Whether the entry's objects belong to the device the context names. A context without a buffer +// pool (tests) is identified by the handle alone. +bool alive(const PipelineStore::Entry& entry, const Context& context) { + if (entry.device != context.device) return false; + return context.bufferPool == nullptr || entry.pool.lock() == context.bufferPool; +} + +// Drops an entry of a device that is gone: its objects went with the device, so they are forgotten, +// not destroyed. +std::list::iterator abandon(PipelineStore& store, std::list::iterator it) { + it->pipeline->Abandon(); + store.index.erase(it->hash); + return store.entries.erase(it); +} + +void reportPipelines(PipelineStore& store) { + static const bool profile = std::getenv("APS5_PROFILE_DRAW") != nullptr; + if (!profile) return; + const auto now = std::chrono::steady_clock::now(); + if (now - store.lastReport < std::chrono::seconds(10)) return; + store.lastReport = now; + const auto lookups = store.hits + store.misses + store.uncached; + std::fprintf(stderr, "[pipecache] %llu lookups over 10 s: %llu hits (%.0f%%), %llu misses, %llu private (no variant id), %llu evicted, %zu cached\n", static_cast(lookups), static_cast(store.hits), lookups != 0 ? 100.0 * static_cast(store.hits) / static_cast(lookups) : 0.0, static_cast(store.misses), static_cast(store.uncached), static_cast(store.evicted), store.entries.size()); + store.hits = store.misses = store.uncached = store.evicted = 0; +} + +} + +std::shared_ptr CachedPipeline(const Context& context, const State& state, const VertexInputLayout& vertexInput, const ShaderResources& resources, std::span shaders) { + static const bool disabled = std::getenv("APS5_NO_PIPELINE_CACHE") != nullptr; + if (disabled) return std::make_shared(context, state, vertexInput, resources, shaders); + auto& store = Pipelines(); + std::lock_guard lock(store.mutex); + reportPipelines(store); + const auto key = pipelineKey(context, state, vertexInput, resources, shaders); + if (key.empty()) { + ++store.uncached; + return std::make_shared(context, state, vertexInput, resources, shaders); + } + const auto hash = hashKey(key); + if (const auto found = store.index.find(hash); found != store.index.end()) { + const auto it = found->second; + if (it->key == key) { + // The key names the device handle, which the loader may reuse for a device created after + // this one was destroyed without ClearCachedPipelines: such an entry is a miss. + if (alive(*it, context)) { + ++store.hits; + store.entries.splice(store.entries.end(), store.entries, it); + return it->pipeline; + } + abandon(store, it); + } else { + // A different configuration with the same hash keeps the resident entry; this one stays private. + ++store.uncached; + return std::make_shared(context, state, vertexInput, resources, shaders); + } + } + ++store.misses; + // Entries of another device belong to one the driver replaced (it does so under the GpuMutex + // before any draw reaches the new device), whose objects went with it: forget them. + for (auto it = store.entries.begin(); it != store.entries.end();) { + it = alive(*it, context) ? std::next(it) : abandon(store, it); + } + auto pipeline = std::make_shared(context, state, vertexInput, resources, shaders); + store.entries.push_back({context.device, context.bufferPool, hash, key, pipeline}); + store.index[hash] = std::prev(store.entries.end()); + constexpr std::size_t bound = 256; + while (store.entries.size() > bound) { + // Only an entry no recorded draw still holds may go (Kept keeps its shared_ptr until the fence). + const auto victim = std::find_if(store.entries.begin(), store.entries.end(), [](const PipelineStore::Entry& entry) { return entry.pipeline.use_count() == 1; }); + if (victim == store.entries.end()) break; + store.index.erase(victim->hash); + store.entries.erase(victim); + ++store.evicted; + } + return pipeline; +} + +void ClearCachedPipelines(VkDevice device) { + auto& store = Pipelines(); + std::lock_guard lock(store.mutex); + for (auto it = store.entries.begin(); it != store.entries.end();) { + if (it->device != device) { + ++it; + continue; + } + store.index.erase(it->hash); + it = store.entries.erase(it); + } +} + } diff --git a/core/libs/prx/libSceAgcDriver/Graphics/src/Recorder.cpp b/core/libs/prx/libSceAgcDriver/Graphics/src/Recorder.cpp new file mode 100644 index 000000000..938c438b0 --- /dev/null +++ b/core/libs/prx/libSceAgcDriver/Graphics/src/Recorder.cpp @@ -0,0 +1,1592 @@ +#include "prx/libSceAgcDriver/Graphics/include/Recorder.hpp" +#include "prx/libSceAgcDriver/Graphics/include/Texture.hpp" +#include "prx/libSceAgcDriver/Execution/include/GuestMemory.hpp" +#include "prx/libSceAgcDriver/Execution/include/Pm4Opcodes.hpp" +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include + +namespace AgcDriver::Graphics { + +namespace { + +Recorder* activeRecorder = nullptr; + +bool DrawProfiled() { + static const bool profiled = std::getenv("APS5_PROFILE_DRAW") != nullptr; + return profiled; +} + +std::uint64_t syncCounts[5] = {}; +// Fence wait time by sync source (APS5_PROFILE_DRAW), and the source CountSync announced for the +// Sync/SyncThrough that follows it on this thread. +double syncWaitedMs[5] = {}; +thread_local int announcedSource = 4; +// The call site CountSync named for that sync (nullptr: none; the sync's own return address is +// taken then), and the fence-wait table per (source, site) it feeds (APS5_PROFILE_DRAW, under the +// GpuMutex like finish()): which caller's syncs wait, when the source alone does not say (source +// 0 is every WaitIdle and drain). `activeSyncSite` is the entry of the sync in progress on this +// thread (an index: a nested sync from a completion replaces and restores it), so finish() charges +// each batch's wait to it. +thread_local const void* announcedSite = nullptr; +struct SyncSiteWaits { + int source; + const void* site; + std::uint64_t syncs; + std::uint64_t batches; + double waitedMs; +}; +std::vector syncSites; +constexpr std::size_t NoSyncSite = static_cast(-1); +constexpr std::size_t SyncSiteLimit = 48; +thread_local std::size_t activeSyncSite = NoSyncSite; + +bool SyncSitesProfiled() { + // APS5_NO_SYNC_SITES=1 leaves only the per-source and per-thread counts. + static const bool profiled = std::getenv("APS5_PROFILE_DRAW") != nullptr && std::getenv("APS5_NO_SYNC_SITES") == nullptr; + return profiled; +} + +// Begins a sync's attribution: returns the previous active entry for the caller to restore, after +// counting the sync under (source, site). Beyond the table's limit every further site shares the +// last entry (a nullptr site), so the table stays small. +std::size_t BeginSyncSite(int source, const void* site) { + const auto previous = activeSyncSite; + if (!SyncSitesProfiled()) return previous; + auto it = std::find_if(syncSites.begin(), syncSites.end(), [&](const SyncSiteWaits& entry) { return entry.source == source && entry.site == site; }); + if (it == syncSites.end()) { + if (syncSites.size() >= SyncSiteLimit) { + it = std::find_if(syncSites.begin(), syncSites.end(), [&](const SyncSiteWaits& entry) { return entry.source == source && entry.site == nullptr; }); + if (it == syncSites.end()) it = syncSites.insert(syncSites.end(), SyncSiteWaits{source, nullptr, 0, 0, 0}); + } else { + it = syncSites.insert(syncSites.end(), SyncSiteWaits{source, site, 0, 0, 0}); + } + } + ++it->syncs; + activeSyncSite = static_cast(it - syncSites.begin()); + return previous; +} + +void CountSiteWait(double ms) { + if (activeSyncSite == NoSyncSite || activeSyncSite >= syncSites.size()) return; + auto& entry = syncSites[activeSyncSite]; + ++entry.batches; + entry.waitedMs += ms; +} + +// The sites by wait, longest first, as "source@+offset syncs/batches/wait" (offsets symbolize with +// nm against the driver's image like the [guestmem] callers; +0x0 is the overflow entry). +std::string SyncSiteReport() { + static const char* const names[5] = {"idle", "pending-write", "recorded-store", "address-based", "other"}; + std::vector order; + for (const auto& entry : syncSites) order.push_back(&entry); + std::sort(order.begin(), order.end(), [](const SyncSiteWaits* a, const SyncSiteWaits* b) { return a->waitedMs > b->waitedMs; }); + std::string report; + for (std::size_t i = 0; i < order.size() && i < 8; ++i) { + char text[96]; + std::snprintf(text, sizeof(text), " %s@+0x%llx %llu/%llu/%.1fs", names[order[i]->source], order[i]->site != nullptr ? GuestMemory::CodeOffset(order[i]->site) : 0ull, static_cast(order[i]->syncs), static_cast(order[i]->batches), order[i]->waitedMs / 1000); + report += text; + } + return report; +} +// Fence waits by thread and source (APS5_PROFILE_DRAW), keyed by the thread's GpuMutex queue tag: +// a sync waits for the GPU under the mutex, so which worker's syncs (and which kind) hold it against +// queue 0 has to be visible. Its own mutex: the table is read while the report line is printed. +struct ThreadSyncs { + std::uint32_t tag; + std::array counts; + std::array waitedMs; +}; +std::vector threadSyncs; +std::mutex threadSyncsMutex; + +// This thread's fence and timeline waits in total (Recorder::ThreadWaitedMs): a caller times a span +// of its own work and reads the difference to learn how much of it was waiting for the GPU. +thread_local double threadWaitedMs = 0; + +void CountThreadSync(int source, double ms) { + threadWaitedMs += ms; + const auto tag = GuestMemory::GpuLockThreadTag(); + std::lock_guard lock(threadSyncsMutex); + auto it = std::find_if(threadSyncs.begin(), threadSyncs.end(), [&](const ThreadSyncs& thread) { return thread.tag == tag; }); + if (it == threadSyncs.end()) it = threadSyncs.insert(threadSyncs.end(), ThreadSyncs{tag, {}, {}}); + ++it->counts[source]; + it->waitedMs[source] += ms; +} + +std::string ThreadSyncReport() { + static const char* const names[5] = {"idle", "pending-write", "recorded-store", "address-based", "other"}; + std::string report; + std::lock_guard lock(threadSyncsMutex); + for (const auto& thread : threadSyncs) { + char text[64]; + if (thread.tag == 0xffffffffu) std::snprintf(text, sizeof(text), " untagged:"); + else std::snprintf(text, sizeof(text), " queue 0x%x:", thread.tag); + report += text; + for (int source = 0; source < 5; ++source) { + if (thread.counts[source] == 0) continue; + std::snprintf(text, sizeof(text), " %s %llu/%.1fs", names[source], static_cast(thread.counts[source]), thread.waitedMs[source] / 1000); + report += text; + } + } + return report; +} +// Unlocked timeline waits (WaitSerial): count and time, for the [recorder] line. +std::atomic unlockedWaits{0}, unlockedWaitedUs{0}; +// Threads inside a timeline wait with the GpuMutex released (WaitSerial, syncThroughUnlocked): a +// recorder's teardown, after its Sync() made every wait satisfiable, waits until they left the +// semaphore before destroying it. Counted per recorder id (a small fixed array; the waiter cannot +// touch a recorder that may be dying, so it is keyed by the id it copied out), so a replaced +// device's old recorder does not spin on the new device's in-flight waits. +constexpr std::size_t WaiterSlots = 16; +std::atomic unlockedWaiters[WaiterSlots]{}; +std::atomic& WaitersOf(std::uint64_t id) { return unlockedWaiters[id % WaiterSlots]; } +// Recorders alive, by id (own mutex, taken under the GpuMutex or with nothing held): a thread that +// released the GpuMutex around a wait learns whether its recorder still exists before touching it. +std::mutex liveRecordersMutex; +std::vector liveRecorders; +std::atomic nextRecorderId{1}; + +bool RecorderAlive(std::uint64_t id) { + std::lock_guard lock(liveRecordersMutex); + return std::find(liveRecorders.begin(), liveRecorders.end(), id) != liveRecorders.end(); +} + +// Completion actions in progress on this thread (finish()): a pending-write sync the flush hook +// makes from inside one (a copied buffer's write-back store) is skipped, see SyncThrough. +thread_local int completionDepth = 0; + +// APS5_COMPLETION_STORE_SYNC=1: a store made by a completion action waits for later batches that +// note its range, as before (the wait runs under whichever hold reaped the batch). +bool CompletionStoreSyncs() { + static const bool enabled = std::getenv("APS5_COMPLETION_STORE_SYNC") != nullptr; + return enabled; +} + +// APS5_HOOK_LOCKED_WAIT=1: the flush hook's pending-write wait stays under the GpuMutex as before. +bool HookLockedWait() { + static const bool locked = std::getenv("APS5_HOOK_LOCKED_WAIT") != nullptr; + return locked; +} + +// What the holds spend on the recorder (APS5_PROFILE_DRAW, all under the GpuMutex, printed at the +// end of the [recorder] line): completions and their time (of which releasing the kept objects), +// pending-write syncs from inside completions (skipped, or waited with the kill switch), reaps and +// the batches they retired, and the flush hook's unlocked waits (with those that found the +// recorder torn down when they retook the mutex, and those that had to wait locked). +struct HoldCounters { + std::uint64_t completions = 0; + double completionMs = 0; + double keptReleaseMs = 0; + std::uint64_t completionSyncsSkipped = 0; + std::uint64_t completionSyncsWaited = 0; + double completionSyncWaitMs = 0; + std::uint64_t reaps = 0; + std::uint64_t reapsWithWork = 0; + std::uint64_t reapBatches = 0; + double reapMs = 0; + std::uint64_t hookUnlockedWaits = 0; + // The GPU wait alone (timeline wait with the mutex released) and, separately, the relock's + // own wait for the GpuMutex (also counted by the [lock] line under 'hook'), so the two + // remaining costs, GPU latency and hold contention, stay apart. + double hookUnlockedWaitMs = 0; + double hookRelockMs = 0; + std::uint64_t hookUnlockedTornDown = 0; + // Genuine nested hooks (depth >= 2 outside a completion action) that had to wait locked. + std::uint64_t hookLockedWaits = 0; +}; +HoldCounters holdCounters; + +// Deferred release of a finished batch's kept objects (and its completion actions, whose captures +// hold some of the same objects): finish() moves them to this list of the finishing thread instead +// of destroying them under the mutex, and the GpuMutex unlock hook (ReleaseDeferredKeeps, set by +// the constructor) hands the list to the release thread once the thread gave up its outermost +// hold. Destroying a ShaderResources with its buffers, descriptor set, textures and pipelines cost +// ~1.5 ms per reap with work, inside whichever hold reaped ('kept objects released', 15.9 s per +// 200 s run), and most reaps are queue 0's own (ReapRecorded before its dispatches), so a release +// on the unlocking thread would still be the frame's time. The destructors touch only their own +// caches' mutexes and the device, never the recorder or GpuMutex, so they need no hold and no +// particular thread (every Vulkan object is externally synchronized per object, and its cache's +// mutex covers its pool); they need the device alive, which ~Recorder guarantees by draining its +// own thread's list, stopping and joining the release thread (it drains the queue before it +// leaves) and waiting for every release still in progress on another thread (`deferredPending`: +// counted up under the mutex, down after each batch's objects are gone) before the device goes. +// The queue is bounded (APS5_RELEASE_QUEUE_MAX batches, default 256): a thread whose hand-off +// would exceed it destroys its batches itself, as it did before the release thread, so memory +// cannot grow behind a release thread that falls behind. +// APS5_RELEASE_UNDER_LOCK=1 destroys them in finish() as before; APS5_RELEASE_ON_UNLOCK=1 destroys +// them on the unlocking thread (no release thread). +struct DeferredBatch { + std::vector> kept; + std::vector> completions; +}; +thread_local std::vector deferredBatches; +std::atomic deferredPending{0}; +// APS5_PROFILE_DRAW, for the [recorder] line: batches and objects released after an unlock and the +// time that took, on the release thread and inline (on the unlocking thread: the kill switch, a +// full queue, a stopping thread or a failed hand-off), the batches that went inline because the +// queue was full, and the longest queue seen (in batches). +std::atomic threadReleases{0}, threadObjects{0}, threadReleaseUs{0}; +std::atomic inlineReleases{0}, inlineObjects{0}, inlineReleaseUs{0}, inlineOverBound{0}; +std::atomic releaseQueueMax{0}; + +bool ReleaseUnderLock() { + static const bool locked = std::getenv("APS5_RELEASE_UNDER_LOCK") != nullptr; + return locked; +} + +bool ReleaseOnUnlock() { + static const bool onUnlock = std::getenv("APS5_RELEASE_ON_UNLOCK") != nullptr; + return onUnlock; +} + +std::size_t ReleaseQueueBound() { + static const std::size_t bound = [] { + const char* value = std::getenv("APS5_RELEASE_QUEUE_MAX"); + const auto parsed = value != nullptr ? std::strtoull(value, nullptr, 10) : 0ull; + return parsed != 0 ? static_cast(parsed) : std::size_t{256}; + }(); + return bound; +} + +// Destroys deferred batches, on the release thread (`onThread`) or on the thread that deferred +// them, and counts each batch off `deferredPending` only once its objects are gone: ~Recorder +// waits for that count. +void DestroyDeferred(std::vector releasing, bool onThread) { + if (releasing.empty()) return; + const bool profile = DrawProfiled(); + const auto start = profile ? std::chrono::steady_clock::now() : std::chrono::steady_clock::time_point{}; + const auto count = releasing.size(); + std::uint64_t objects = 0; + for (auto& batch : releasing) { + objects += batch.kept.size(); + // The completions first (their captures hold some of the objects), then the objects. + batch.completions.clear(); + batch.kept.clear(); + deferredPending.fetch_sub(1, std::memory_order_acq_rel); + } + releasing.clear(); + if (profile) { + const auto us = static_cast(std::chrono::duration_cast(std::chrono::steady_clock::now() - start).count()); + (onThread ? threadReleases : inlineReleases).fetch_add(count, std::memory_order_relaxed); + (onThread ? threadObjects : inlineObjects).fetch_add(objects, std::memory_order_relaxed); + (onThread ? threadReleaseUs : inlineReleaseUs).fetch_add(us, std::memory_order_relaxed); + } +} + +// The release thread's queue: one for the process. The thread is started at the first hand-off and +// stopped and joined by ~Recorder (JoinReleaseThread), after which the next hand-off starts it +// again (a device replacement makes a new recorder). The queue object is leaked so that a static +// destructor never races the thread: at process exit the thread is idle on its condition variable +// (every ~Recorder joined it) or, when no ~Recorder ran, mid-release like any worker thread. Nothing +// is done under `mutex` but the hand-off and the take; `joinMutex` serializes joiners and orders +// before `mutex`; neither is ever taken by a kept object's destructor. +struct ReleaseQueue { + std::mutex mutex; + std::condition_variable wake; + std::vector items; + std::thread thread; + bool started = false; + // Set by a joiner under `mutex` until the thread was joined: the thread leaves once the queue + // is empty, and hand-offs meanwhile destroy inline (nothing may be queued without a taker). + bool stop = false; + std::mutex joinMutex; +}; + +ReleaseQueue& ReleaseThreadQueue() { + static ReleaseQueue* const queue = new ReleaseQueue; + return *queue; +} + +void ReleaseThreadMain() { + auto& queue = ReleaseThreadQueue(); + std::unique_lock lock(queue.mutex); + for (;;) { + queue.wake.wait(lock, [&] { return !queue.items.empty() || queue.stop; }); + if (queue.items.empty()) return; + auto items = std::move(queue.items); + queue.items.clear(); + lock.unlock(); + DestroyDeferred(std::move(items), true); + lock.lock(); + } +} + +// Stops the release thread once it emptied the queue and joins it (~Recorder, before its device +// goes). Hand-offs made while the thread stops destroy inline, so nothing can sit in the queue +// without a taker; a later hand-off starts the thread again. +void JoinReleaseThread() { + auto& queue = ReleaseThreadQueue(); + std::lock_guard joining(queue.joinMutex); + std::thread worker; + { + std::lock_guard lock(queue.mutex); + if (!queue.started) return; + queue.stop = true; + worker = std::move(queue.thread); + } + queue.wake.notify_all(); + worker.join(); + std::lock_guard lock(queue.mutex); + queue.started = false; + queue.stop = false; +} + +// The unlock hook: hands what this thread deferred to the release thread, or destroys it here +// (APS5_RELEASE_ON_UNLOCK=1, the queue is full or stopping, or the hand-off cannot be made). +void ReleaseDeferredKeeps() { + if (deferredBatches.empty()) return; + // Taken off the thread's list first: a destructor that took and released the mutex (none is + // known to) would re-enter here and must find nothing. + auto releasing = std::move(deferredBatches); + deferredBatches.clear(); + if (ReleaseOnUnlock()) { + DestroyDeferred(std::move(releasing), false); + return; + } + auto& queue = ReleaseThreadQueue(); + const bool profile = DrawProfiled(); + bool handedOff = false; + bool overBound = false; + try { + std::lock_guard lock(queue.mutex); + if (queue.items.size() + releasing.size() > ReleaseQueueBound()) { + overBound = true; + } else if (!queue.stop) { + // The thread before the items: items queued with no thread to take them would keep + // `deferredPending` up and ~Recorder waiting forever. + if (!queue.started) { + queue.thread = std::thread(&ReleaseThreadMain); + queue.started = true; + } + // The allocation before any move: a failure here leaves `releasing` intact for the + // fallback below, and the moves cannot throw. + queue.items.reserve(queue.items.size() + releasing.size()); + for (auto& batch : releasing) queue.items.push_back(std::move(batch)); + handedOff = true; + if (profile) { + const auto queued = static_cast(queue.items.size()); + auto seen = releaseQueueMax.load(std::memory_order_relaxed); + while (queued > seen && !releaseQueueMax.compare_exchange_weak(seen, queued, std::memory_order_relaxed)) { + } + } + } + } catch (...) { + } + if (!handedOff) { + if (profile && overBound) inlineOverBound.fetch_add(releasing.size(), std::memory_order_relaxed); + DestroyDeferred(std::move(releasing), false); + return; + } + queue.wake.notify_one(); +} +// Lock-free state of the active recorder's open batch, read by the queue workers between packets +// and inside WAIT_REG_MEM polls (see the static readers in Recorder.hpp). Written under the mutex. +constexpr std::int64_t NoPendingLabel = std::numeric_limits::min(); +std::atomic pendingLabelSince{NoPendingLabel}; +std::atomic writeGeneration{0}; +std::atomic completionLabels{0}; +std::atomic completionStoresSkipped{0}; +std::atomic completionStoresRun{0}; +std::atomic workSinceSubmit{0}; +// The pending-label table's own mutex: every mutation of a recorder's `labels` holds it (under the +// GPU mutex), so a WAIT_REG_MEM looks a label up with this small lock alone (Recorder::LookupLabel) +// instead of queueing behind a dispatch. `labelTableOwner` is the active recorder while it lives, +// set and cleared under this mutex, so a lookup never touches a table being destroyed. Nothing +// under this mutex takes the GPU mutex (no lock-order cycle). +std::mutex labelTableMutex; +Recorder* labelTableOwner = nullptr; +// Flush hook statistics: calls, calls whose snapshot check overlapped (the GpuMutex was taken), +// targeted syncs and the batches they left in flight. +std::atomic hookCalls{0}, hookLocks{0}, targetedSyncs{0}, batchesLeftInFlight{0}; +// Snapshot maintenance (under the GpuMutex): notes whose range the snapshot already covered (no +// rebuild), rebuilds, and the time the rebuilds took (APS5_PROFILE_DRAW), so their cost is visible. +std::uint64_t snapshotCovered = 0, snapshotRebuilds = 0; +double snapshotRebuildMs = 0; + +using WriteRanges = std::vector>; +// The sorted, merged union of the guest ranges every unfinished batch will write. Rebuilt under +// GuestMemory::GpuMutex (the only writer) and read by the flush hook without it: the lock is the +// graphics worker's main stall, so a no-overlap access must not wait for another queue's device +// work. A range leaves the snapshot only after its batch's completions (CPU write-backs) ran, so a +// reader that sees no overlap either precedes the note (the queues are unordered then, as on the +// GPU) or follows the write-back. +std::atomic> pendingWrites; + +bool HookSnapshotEnabled() { + // Debug aid: APS5_NO_HOOK_SNAPSHOT=1 takes the GpuMutex on every access as before. + static const bool enabled = std::getenv("APS5_NO_HOOK_SNAPSHOT") == nullptr; + return enabled; +} + +bool SnapshotOverlaps(std::uint64_t address, std::size_t bytes) { + if (bytes == 0) return false; + const auto snapshot = pendingWrites.load(std::memory_order_acquire); + if (snapshot == nullptr || snapshot->empty()) return false; + // Merged ranges are ordered by both bounds: the first one ending past the access decides. + const auto it = std::partition_point(snapshot->begin(), snapshot->end(), [&](const auto& range) { return range.second <= address; }); + return it != snapshot->end() && it->first < address + bytes; +} + +// Whether one merged range of the snapshot contains [address, end) entirely. +bool SnapshotCovers(std::uint64_t address, std::uint64_t end) { + const auto snapshot = pendingWrites.load(std::memory_order_acquire); + if (snapshot == nullptr || snapshot->empty()) return false; + const auto it = std::partition_point(snapshot->begin(), snapshot->end(), [&](const auto& range) { return range.second <= address; }); + return it != snapshot->end() && it->first <= address && end <= it->second; +} + +// Attribution of the pending-write syncs the hook makes (the [hooksync] line every 10 s, under +// APS5_PROFILE_DRAW; APS5_NO_HOOKSYNC_PROFILE=1 leaves only the plain counts). Every sync is charged +// to the packet the accessing thread executes and the read site it named (or, when it named none, +// its return addresses), together with the noted range it hit, the batch that noted it and the +// time waited. A small read (up to 64 KiB) has its bytes compared before and after the wait: a +// sync whose bytes come out unchanged although its target batch had not run yet was not needed +// for that read (the noted range was larger than what the GPU wrote, or the GPU wrote the same +// values), which separates false overlaps from real producer/consumer dependencies; see +// HookSyncOutcomes for the cases that prove nothing. All state lives under the GpuMutex, which +// the hook holds. +// Debug aid: APS5_NO_SYNC_THROUGH=1 makes every pending-write sync a full Sync (SyncThrough and +// the attribution's DescribePendingWrite read the same switch). +bool SyncThroughEnabled() { + static const bool enabled = std::getenv("APS5_NO_SYNC_THROUGH") == nullptr; + return enabled; +} + +// Six frames: whether the hook's constructor and FlushGpuWrites are inlined into the GuestMemory +// entry decides at which frame the driver caller sits, so enough are kept to see its own caller. +constexpr std::size_t HookSyncFrames = 6; + +struct HookSyncKey { + std::uint32_t queue; + std::uint32_t opcode; + GuestMemory::ReadSite site; + std::array frames; + bool operator<(const HookSyncKey& other) const { + return std::tie(queue, opcode, site, frames) < std::tie(other.queue, other.opcode, other.site, other.frames); + } +}; + +// Whether the bytes of a small access changed over the wait, split by the state of the target +// batch: only an unchanged access whose target had not run yet (open, or in flight with its fence +// unsignaled) shows the sync was not needed; against a signaled fence the copy made before the +// wait already held the GPU's values, so unchanged proves nothing. A store (ReadSite::Store) is +// counted apart: its sync orders a CPU write after the GPU's, whatever the bytes do. +struct HookSyncOutcomes { + std::uint64_t unchangedOpen = 0; + std::uint64_t unchangedPending = 0; + std::uint64_t unchangedSignaled = 0; + std::uint64_t changed = 0; + std::uint64_t stores = 0; + std::uint64_t unchecked = 0; +}; + +struct HookSyncTotals { + std::uint64_t count = 0; + HookSyncOutcomes outcomes; + std::uint64_t rangeBytes = 0; + std::uint64_t accessBytes = 0; + double waitedMs = 0; +}; + +// Size buckets for the noted ranges and the accesses: <=4K, <=64K, <=1M, <=16M, <=256M, larger. +constexpr std::size_t SizeBuckets = 6; +constexpr const char* SizeBucketNames[SizeBuckets] = {"<=4K", "<=64K", "<=1M", "<=16M", "<=256M", ">256M"}; + +std::size_t SizeBucket(std::uint64_t bytes) { + if (bytes <= 4096) return 0; + if (bytes <= 65536) return 1; + if (bytes <= (1u << 20u)) return 2; + if (bytes <= (16u << 20u)) return 3; + if (bytes <= (256u << 20u)) return 4; + return 5; +} + +struct HookSyncStats { + std::map byKey; + std::uint64_t count = 0, openTargets = 0, signaledTargets = 0, batchesFinished = 0; + HookSyncOutcomes outcomes; + double waitedMs = 0; + std::array rangeBuckets{}; + std::array accessBuckets{}; + std::chrono::steady_clock::time_point lastReport = std::chrono::steady_clock::now(); +}; + +HookSyncStats& HookSyncs() { + static HookSyncStats stats; + return stats; +} + +bool HookSyncProfiled() { + static const bool profiled = std::getenv("APS5_PROFILE_DRAW") != nullptr && std::getenv("APS5_NO_HOOKSYNC_PROFILE") == nullptr; + return profiled; +} + +std::string PacketName(std::uint32_t opcode) { + if (opcode == GuestMemory::NoPacket) return "no-packet"; + if (opcode == 0xffffu) return "flip"; + // 0xfffd is reserved for the deferred-label stores (recordDeferredLabels), once its owner + // brackets them with SetCurrentPacket; until then they are charged to the following packet. + if (opcode == 0xfffdu) return "deferred-labels"; + for (const auto& entry : Pm4::Opcodes) { + if (entry.value == opcode) return std::string(entry.name); + } + char text[24]; + std::snprintf(text, sizeof(text), "op 0x%x", opcode); + return text; +} + +void ReportHookSyncs(HookSyncStats& stats) { + std::vector> hot; + hot.reserve(stats.byKey.size()); + for (const auto& [key, totals] : stats.byKey) hot.emplace_back(&key, &totals); + std::sort(hot.begin(), hot.end(), [](const auto& a, const auto& b) { return a.second->waitedMs > b.second->waitedMs; }); + std::string report; + char text[512]; + const auto count = [](std::uint64_t value) { return static_cast(value); }; + const auto& o = stats.outcomes; + std::snprintf(text, sizeof(text), "[hooksync] %llu pending-write syncs waited %.0f ms (10 s); read bytes after the wait: unchanged %llu (target open %llu, in flight unsignaled %llu, signaled %llu), changed %llu; stores %llu, unchecked %llu; targets: %llu open, %llu already signaled, %llu batches finished; noted range:", count(stats.count), stats.waitedMs, count(o.unchangedOpen + o.unchangedPending + o.unchangedSignaled), count(o.unchangedOpen), count(o.unchangedPending), count(o.unchangedSignaled), count(o.changed), count(o.stores), count(o.unchecked), count(stats.openTargets), count(stats.signaledTargets), count(stats.batchesFinished)); + report += text; + for (std::size_t i = 0; i < SizeBuckets; ++i) { + if (stats.rangeBuckets[i] == 0) continue; + std::snprintf(text, sizeof(text), " %s %llu", SizeBucketNames[i], static_cast(stats.rangeBuckets[i])); + report += text; + } + report += "; access:"; + for (std::size_t i = 0; i < SizeBuckets; ++i) { + if (stats.accessBuckets[i] == 0) continue; + std::snprintf(text, sizeof(text), " %s %llu", SizeBucketNames[i], static_cast(stats.accessBuckets[i])); + report += text; + } + report += "; top by wait (queue packet site frames: count/ms, unchanged open/unsignaled/signaled, changed, stores, avg range/access):"; + for (std::size_t i = 0; i < hot.size() && i < 10; ++i) { + const auto& key = *hot[i].first; + const auto& totals = *hot[i].second; + const auto& k = totals.outcomes; + std::snprintf(text, sizeof(text), " [0x%x %s %s +0x%llx/+0x%llx/+0x%llx/+0x%llx/+0x%llx/+0x%llx: %llu/%.0fms u%llu/%llu/%llu c%llu s%llu %.0fK/%.0fK]", key.queue, PacketName(key.opcode).c_str(), GuestMemory::ReadSiteName(key.site), key.frames[0], key.frames[1], key.frames[2], key.frames[3], key.frames[4], key.frames[5], count(totals.count), totals.waitedMs, count(k.unchangedOpen), count(k.unchangedPending), count(k.unchangedSignaled), count(k.changed), count(k.stores), totals.rangeBytes / 1024.0 / totals.count, totals.accessBytes / 1024.0 / totals.count); + report += text; + } + std::fprintf(stderr, "%s\n", report.c_str()); + stats = HookSyncStats{}; +} + +// Wraps one pending-write sync (constructed before it, under the GpuMutex): gathers what the sync +// waits for and, in the destructor, the time it took and whether the bytes changed. +class HookSyncScope { +public: + HookSyncScope(const Recorder& recorder, std::uint64_t address, std::size_t bytes) : enabled(HookSyncProfiled()), address(address), bytes(bytes) { + if (!enabled) return; + info = recorder.DescribePendingWrite(address, bytes); + const auto packet = GuestMemory::CurrentPacket(); + key = HookSyncKey{packet.queue, packet.opcode, GuestMemory::CurrentReadSite(), {}}; + // Frame 0 is the hook's caller (the GuestMemory entry, or the driver code when the entry + // was inlined); the driver code that made the access is one of the next frames. + GuestMemory::CaptureCallerOffsets(key.frames, 1); + // The access's bytes are copied raw: a Read here would re-enter the hook. The same access + // check the accessing code makes guards the copy; a range that fails it is left unchecked. + // A store's bytes are not compared (counted as a store instead). + if (key.site != GuestMemory::ReadSite::Store && bytes <= CompareLimit && GuestMemory::Accessible(reinterpret_cast(address), bytes)) { + before.resize(bytes); + std::memcpy(before.data(), reinterpret_cast(address), bytes); + } + start = std::chrono::steady_clock::now(); + } + ~HookSyncScope() { + if (!enabled) return; + const auto now = std::chrono::steady_clock::now(); + const auto ms = std::chrono::duration(now - start).count(); + auto& stats = HookSyncs(); + auto& totals = stats.byKey[key]; + ++totals.count; + totals.waitedMs += ms; + totals.accessBytes += bytes; + ++stats.count; + stats.waitedMs += ms; + stats.accessBuckets[SizeBucket(bytes)] += 1; + if (info.has_value()) { + const auto rangeBytes = info->rangeEnd - info->rangeBegin; + totals.rangeBytes += rangeBytes; + stats.rangeBuckets[SizeBucket(rangeBytes)] += 1; + if (info->open) ++stats.openTargets; + if (info->signaled) ++stats.signaledTargets; + stats.batchesFinished += info->batchesToFinish; + } + // One outcome per sync, on both the key's and the interval's counters. + const auto outcome = [&]() -> std::uint64_t HookSyncOutcomes::* { + if (key.site == GuestMemory::ReadSite::Store) return &HookSyncOutcomes::stores; + if (before.empty() || !GuestMemory::Accessible(reinterpret_cast(address), bytes)) return &HookSyncOutcomes::unchecked; + if (std::memcmp(before.data(), reinterpret_cast(address), bytes) != 0) return &HookSyncOutcomes::changed; + // A target already signaled (or no target found: the batch is finishing) had run + // before the copy, so its unchanged bytes are not evidence of a false overlap. + if (!info.has_value() || info->signaled) return &HookSyncOutcomes::unchangedSignaled; + return info->open ? &HookSyncOutcomes::unchangedOpen : &HookSyncOutcomes::unchangedPending; + }(); + ++(totals.outcomes.*outcome); + ++(stats.outcomes.*outcome); + if (now - stats.lastReport > std::chrono::seconds(10)) ReportHookSyncs(stats); + } + HookSyncScope(const HookSyncScope&) = delete; + HookSyncScope& operator=(const HookSyncScope&) = delete; + +private: + static constexpr std::size_t CompareLimit = 65536; + bool enabled; + std::uint64_t address; + std::size_t bytes; + std::optional info; + HookSyncKey key{}; + std::vector before; + std::chrono::steady_clock::time_point start; +}; + +// GuestMemory flush hook: a CPU access to memory that recorded GPU work will write waits for that +// work first; then storage image results pending for the range are stored. The recorder is only +// dereferenced under the GpuMutex (it is destroyed with its device under that lock). +void FlushForAccess(std::uint64_t address, std::size_t bytes) { + hookCalls.fetch_add(1, std::memory_order_relaxed); + if (!HookSnapshotEnabled() || SnapshotOverlaps(address, bytes)) { + hookLocks.fetch_add(1, std::memory_order_relaxed); + GuestMemory::TagGpuLockSite(GuestMemory::GpuLockSite::Hook); + std::lock_guard gpu(GuestMemory::GpuMutex()); + if (auto* recorder = Recorder::Active(); recorder != nullptr && recorder->PendingWriteOverlaps(address, bytes)) { + const HookSyncScope attribution(*recorder, address, bytes); + Recorder::CountSync(1); + // The wait runs without the mutex when this acquisition is the outermost one (a stage-A + // read, a game thread); the recorder is not dereferenced after the call. + recorder->SyncThrough(address, bytes, true); + } + } + if (StorageTexture::FlushPending(address, bytes)) { + // Stores into imported memory were only recorded; the CPU is about to read them. + GuestMemory::TagGpuLockSite(GuestMemory::GpuLockSite::Hook); + std::lock_guard gpu(GuestMemory::GpuMutex()); + if (auto* recorder = Recorder::Active(); recorder != nullptr) { + Recorder::CountSync(2); + recorder->Sync(); + } + } +} + +} + +Recorder::Recorder(const Context& context, bool timelineSemaphores) : context(context), id(nextRecorderId.fetch_add(1)) { + { + std::lock_guard lock(liveRecordersMutex); + liveRecorders.push_back(id); + } + // Idempotent: the same hook for every recorder (the deferred lists are per thread, not per recorder). + GuestMemory::SetGpuUnlockHook(&ReleaseDeferredKeeps); + if (!timelineSemaphores) return; + // The timeline starts at 0 and every Submit signals its serial (1, 2, ...): a value that only + // grows, so a thread waiting for it outside the mutex can never observe a reset or a reused + // handle (the batch fences are pooled and reset in release()). A failure here only disables the + // unlocked waits; the locked Sync path still works. + VkSemaphoreTypeCreateInfoKHR type{VK_STRUCTURE_TYPE_SEMAPHORE_TYPE_CREATE_INFO_KHR}; + type.semaphoreType = VK_SEMAPHORE_TYPE_TIMELINE_KHR; + type.initialValue = 0; + VkSemaphoreCreateInfo info{VK_STRUCTURE_TYPE_SEMAPHORE_CREATE_INFO, &type}; + const auto result = context.Function("vkCreateSemaphore")(context.device, &info, nullptr, &timeline); + if (result != VK_SUCCESS) { + std::fprintf(stderr, "[gpu] timeline semaphore creation failed (Vulkan result %d); drains wait under the GPU mutex\n", static_cast(result)); + timeline = VK_NULL_HANDLE; + } +} + +Recorder::~Recorder() { + try { + Sync(); + } catch (const std::exception& error) { + std::fprintf(stderr, "[gpu] recorder teardown: %s\n", error.what()); + } + if (activeRecorder == this) { + activeRecorder = nullptr; + pendingWrites.store(nullptr, std::memory_order_release); + pendingLabelSince.store(NoPendingLabel, std::memory_order_release); + completionLabels.store(0, std::memory_order_release); + workSinceSubmit.store(0, std::memory_order_release); + } + { + // A thread that released the mutex around a wait on this recorder re-checks the id when it + // retakes the mutex and touches nothing then. That check is only sound while the device + // (and so this destructor) is torn down under the GpuMutex, as the driver's drain and + // failure paths do; a worker dropping the last device reference after releasing its lock + // would run this without the mutex, concurrently with a relocked waiter. + std::lock_guard lock(liveRecordersMutex); + liveRecorders.erase(std::remove(liveRecorders.begin(), liveRecorders.end(), id), liveRecorders.end()); + } + // Sync() above completed every batch, so every timeline wait in progress on this recorder + // returns now; the semaphore is destroyed only once they all left it. + while (WaitersOf(id).load(std::memory_order_acquire) != 0) std::this_thread::yield(); + // The kept objects of every batch this thread finished (Sync() above included) belong to the + // device that is going away: destroyed now, here (no hand-off: nothing should stay in flight + // behind this thread); the release thread is then stopped and joined (it drains its queue + // first, holding no mutex this thread holds: the destructors take only their caches' own + // mutexes, which order after the GpuMutex), and the releases still in progress inline on + // other threads (an unlock hook that found the queue stopping or full) are waited for before + // the caller destroys the device. The count is global: a newer recorder's batches (device + // replacement) are waited for too, which only prolongs the spin. + if (!deferredBatches.empty()) { + auto own = std::move(deferredBatches); + deferredBatches.clear(); + DestroyDeferred(std::move(own), false); + } + JoinReleaseThread(); + while (deferredPending.load(std::memory_order_acquire) != 0) std::this_thread::yield(); + { + // Unlocked lookups stop before `labels` is destroyed (after this body). + std::lock_guard tableLock(labelTableMutex); + if (labelTableOwner == this) labelTableOwner = nullptr; + } + for (auto& [commands, fence] : spare) { + context.Function("vkFreeCommandBuffers")(context.device, context.pool, 1, &commands); + context.Function("vkDestroyFence")(context.device, fence, nullptr); + } + spare.clear(); + // Sync() above waited for every batch, so no submission still signals the timeline. + if (timeline != VK_NULL_HANDLE) context.Function("vkDestroySemaphore")(context.device, timeline, nullptr); + timeline = VK_NULL_HANDLE; +} + +std::optional Recorder::PendingLabelSince() { + const auto since = pendingLabelSince.load(std::memory_order_acquire); + if (since == NoPendingLabel) return std::nullopt; + return std::chrono::steady_clock::time_point(std::chrono::steady_clock::duration(since)); +} + +std::uint64_t Recorder::WriteGeneration() { + return writeGeneration.load(std::memory_order_acquire); +} + +std::uint64_t Recorder::PendingCompletionLabels() { + return completionLabels.load(std::memory_order_acquire); +} + +std::uint64_t Recorder::RecordedWorkSinceSubmit() { + return workSinceSubmit.load(std::memory_order_relaxed); +} + +void Recorder::CountRecordedWork() { + workSinceSubmit.fetch_add(1, std::memory_order_relaxed); +} + +Recorder* Recorder::Active() { + return activeRecorder; +} + +void Recorder::Activate() { + activeRecorder = this; + { + std::lock_guard tableLock(labelTableMutex); + labelTableOwner = this; + } + GuestMemory::SetFlushHook(&FlushForAccess); +} + +std::optional Recorder::LookupLabel(std::uint64_t address, std::size_t bytes, std::uint64_t afterStamp, std::uint32_t& queue) { + std::lock_guard tableLock(labelTableMutex); + if (labelTableOwner == nullptr) return std::nullopt; + return labelTableOwner->lookupLabel(address, bytes, afterStamp, queue); +} + +bool Recorder::SnapshotWriteOverlaps(std::uint64_t address, std::size_t bytes) { + return SnapshotOverlaps(address, bytes); +} + +void Recorder::CountSync(int source, const void* site) { + if (source < 0 || source >= 5) return; + ++syncCounts[source]; + announcedSource = source; + announcedSite = site; +} + +void Recorder::AnnounceSyncSite(const void* site) { + announcedSite = site; +} + +double Recorder::ThreadWaitedMs() { + return threadWaitedMs; +} + +VkCommandBuffer Recorder::Commands() { + GuestMemory::AssertGpuLockHeld("Recorder::Commands"); + if (open == nullptr) { + auto batch = std::make_unique(); + try { + if (!spare.empty()) { + // The pool resets command buffers on begin; the fence was reset when the batch completed. + std::tie(batch->commands, batch->fence) = spare.back(); + spare.pop_back(); + } else { + VkCommandBufferAllocateInfo allocation{VK_STRUCTURE_TYPE_COMMAND_BUFFER_ALLOCATE_INFO}; + allocation.commandPool = context.pool; + allocation.level = VK_COMMAND_BUFFER_LEVEL_PRIMARY; + allocation.commandBufferCount = 1; + Check(context.Function("vkAllocateCommandBuffers")(context.device, &allocation, &batch->commands), "vkAllocateCommandBuffers recorder"); + VkFenceCreateInfo info{VK_STRUCTURE_TYPE_FENCE_CREATE_INFO}; + Check(context.Function("vkCreateFence")(context.device, &info, nullptr, &batch->fence), "vkCreateFence recorder"); + } + VkCommandBufferBeginInfo begin{VK_STRUCTURE_TYPE_COMMAND_BUFFER_BEGIN_INFO}; + begin.flags = VK_COMMAND_BUFFER_USAGE_ONE_TIME_SUBMIT_BIT; + Check(context.Function("vkBeginCommandBuffer")(batch->commands, &begin), "vkBeginCommandBuffer recorder"); + } catch (...) { + release(*batch); + throw; + } + open = std::move(batch); + } + return open->commands; +} + +namespace { + +constexpr std::uint32_t MaxTimedRanges = 512; + +bool GpuTimingEnabled() { + static const bool enabled = std::getenv("APS5_PROFILE_GPU") != nullptr; + return enabled; +} + +} + +std::uint32_t Recorder::BeginGpuTiming(std::uint64_t key) { + if (!GpuTimingEnabled()) return NoTiming; + const auto commands = Commands(); + if (open->queries == VK_NULL_HANDLE) { + VkQueryPoolCreateInfo info{VK_STRUCTURE_TYPE_QUERY_POOL_CREATE_INFO}; + info.queryType = VK_QUERY_TYPE_TIMESTAMP; + info.queryCount = MaxTimedRanges * 2; + if (context.Function("vkCreateQueryPool")(context.device, &info, nullptr, &open->queries) != VK_SUCCESS) { + open->queries = VK_NULL_HANDLE; + return NoTiming; + } + context.Function("vkCmdResetQueryPool")(commands, open->queries, 0, info.queryCount); + } + if (open->timedKeys.size() >= MaxTimedRanges) return NoTiming; + const auto index = static_cast(open->timedKeys.size()); + open->timedKeys.push_back(key); + // Both stamps wait for everything before them to complete, so the range is the timed work alone + // (a top-of-pipe stamp is not held back by the barrier that precedes the work). + context.Function("vkCmdWriteTimestamp")(commands, VK_PIPELINE_STAGE_BOTTOM_OF_PIPE_BIT, open->queries, index * 2); + return index; +} + +void Recorder::EndGpuTiming(std::uint32_t index) { + if (index == NoTiming || open == nullptr || open->queries == VK_NULL_HANDLE) return; + context.Function("vkCmdWriteTimestamp")(open->commands, VK_PIPELINE_STAGE_BOTTOM_OF_PIPE_BIT, open->queries, index * 2 + 1); +} + +void Recorder::readGpuTiming(Batch& batch) { + if (batch.queries == VK_NULL_HANDLE || batch.timedKeys.empty()) return; + std::vector stamps(batch.timedKeys.size() * 2); + const auto result = context.Function("vkGetQueryPoolResults")(context.device, batch.queries, 0, static_cast(stamps.size()), stamps.size() * sizeof(std::uint64_t), stamps.data(), sizeof(std::uint64_t), VK_QUERY_RESULT_64_BIT | VK_QUERY_RESULT_WAIT_BIT); + if (result != VK_SUCCESS) return; + struct Totals { std::uint64_t count = 0; double ms = 0; }; + static std::map byKey; + static double totalMs = 0; + static std::uint64_t batches = 0; + static auto lastReport = std::chrono::steady_clock::now(); + for (std::size_t i = 0; i < batch.timedKeys.size(); ++i) { + const auto ns = static_cast(stamps[i * 2 + 1] - stamps[i * 2]) * context.limits.timestampPeriod; + auto& totals = byKey[batch.timedKeys[i]]; + ++totals.count; + totals.ms += ns / 1e6; + totalMs += ns / 1e6; + } + ++batches; + const auto now = std::chrono::steady_clock::now(); + if (now - lastReport < std::chrono::seconds(10)) return; + lastReport = now; + std::vector> hot(byKey.begin(), byKey.end()); + std::sort(hot.begin(), hot.end(), [](const auto& a, const auto& b) { return a.second.ms > b.second.ms; }); + std::fprintf(stderr, "[gputime] %.0f ms of GPU time in %llu batches over 10 s; by program:", totalMs, static_cast(batches)); + for (std::size_t i = 0; i < hot.size() && i < 12; ++i) std::fprintf(stderr, " 0x%llx x%llu %.0fms", static_cast(hot[i].first), static_cast(hot[i].second.count), hot[i].second.ms); + std::fprintf(stderr, "\n"); + byKey.clear(); + totalMs = 0; + batches = 0; +} + +void Recorder::Keep(std::shared_ptr object) { + Commands(); + open->kept.push_back(std::move(object)); +} + +void Recorder::OnComplete(std::function action) { + Commands(); + open->completions.push_back(std::move(action)); +} + +bool Recorder::noteWrite(std::uint64_t address, std::size_t bytes) { + if (bytes == 0) return false; + Commands(); + const auto end = address + bytes; + open->writes.emplace_back(address, end); + // A poller waiting on this range learns that the open batch may now hold its producer. + if (activeRecorder == this) writeGeneration.fetch_add(1, std::memory_order_release); + // Consecutive dispatches write the same output buffers: a range the snapshot already contains + // leaves the published union unchanged, so the O(N log N) rebuild is skipped. + if (SnapshotCovers(address, end)) { + ++snapshotCovered; + return false; + } + return true; +} + +void Recorder::noteWriteOn(Batch& batch, std::uint64_t address, std::size_t bytes) { + if (bytes == 0) return; + if (&batch == open.get()) { + NotePendingWrite(address, bytes); + return; + } + // An in-flight batch: its range joins the snapshot at once (the completion that stores it runs + // when the batch finishes, and the hook must sync for a CPU read until then). The generation + // moves as well, so a poller re-consults the label table for a completion label. + batch.writes.emplace_back(address, address + bytes); + if (activeRecorder == this) writeGeneration.fetch_add(1, std::memory_order_release); + if (!SnapshotCovers(address, address + bytes)) publishPendingWrites(); + std::atomic_thread_fence(std::memory_order_seq_cst); +} + +void Recorder::NotePendingWrite(std::uint64_t address, std::size_t bytes) { + if (!noteWrite(address, bytes)) return; + publishPendingWrites(); + // The note precedes this thread's vkQueueSubmit and the label another queue polls for; the + // fence makes that order hold without relying on x86 store ordering. + std::atomic_thread_fence(std::memory_order_seq_cst); +} + +void Recorder::NotePendingWrites(std::span> ranges) { + bool publish = false; + for (const auto& [begin, end] : ranges) { + if (end > begin && noteWrite(begin, static_cast(end - begin))) publish = true; + } + if (!publish) return; + publishPendingWrites(); + std::atomic_thread_fence(std::memory_order_seq_cst); +} + +void Recorder::publishPendingWrites() const { + // Only the active recorder owns the snapshot: one torn down after its successor was activated, + // or one a worker still dispatches into after device.reset(), must not publish its ranges. + if (activeRecorder != this) return; + const auto started = DrawProfiled() ? std::chrono::steady_clock::now() : std::chrono::steady_clock::time_point{}; + auto merged = std::make_shared(); + if (open != nullptr) merged->insert(merged->end(), open->writes.begin(), open->writes.end()); + for (const auto& batch : inFlight) merged->insert(merged->end(), batch->writes.begin(), batch->writes.end()); + for (const auto* batch : finishing) merged->insert(merged->end(), batch->writes.begin(), batch->writes.end()); + std::sort(merged->begin(), merged->end()); + std::size_t out = 0; + for (const auto& [begin, end] : *merged) { + if (out != 0 && begin <= (*merged)[out - 1].second) (*merged)[out - 1].second = std::max((*merged)[out - 1].second, end); + else (*merged)[out++] = {begin, end}; + } + merged->resize(out); + pendingWrites.store(std::move(merged), std::memory_order_release); + ++snapshotRebuilds; + if (DrawProfiled()) snapshotRebuildMs += std::chrono::duration(std::chrono::steady_clock::now() - started).count(); +} + +bool Recorder::overlaps(const Batch& batch, std::uint64_t address, std::uint64_t end) { + for (const auto& [begin, finish] : batch.writes) { + if (address < finish && begin < end) return true; + } + return false; +} + +bool Recorder::PendingWriteOverlaps(std::uint64_t address, std::size_t bytes) const { + if (bytes == 0) return false; + const auto end = address + bytes; + if (open != nullptr && overlaps(*open, address, end)) return true; + for (const auto& batch : inFlight) { + if (overlaps(*batch, address, end)) return true; + } + return false; +} + +bool Recorder::OpenWriteOverlaps(std::uint64_t address, std::size_t bytes) const { + return bytes != 0 && open != nullptr && overlaps(*open, address, address + bytes); +} + +std::optional Recorder::DescribePendingWrite(std::uint64_t address, std::size_t bytes) const { + if (bytes == 0) return std::nullopt; + const auto end = address + bytes; + // The same choice SyncThrough makes: the open batch forces a full Sync, otherwise the newest + // overlapping in-flight batch is the target. Of that batch's ranges the first overlapping one + // is reported (a dispatch notes each written V# once, so it is the range the access hit). + const auto firstOverlap = [&](const Batch& batch) -> const std::pair* { + for (const auto& range : batch.writes) { + if (address < range.second && range.first < end) return ⦥ + } + return nullptr; + }; + // With SyncThrough disabled every hit is a full Sync: the open batch (if any) is submitted and + // everything in flight finishes, whichever batch noted the range. + const bool syncAll = !SyncThroughEnabled(); + const auto allBatches = inFlight.size() + (open != nullptr ? 1 : 0); + if (open != nullptr) { + if (const auto* range = firstOverlap(*open)) return PendingWriteInfo{submissions + 1, true, false, range->first, range->second, inFlight.size() + 1}; + } + std::size_t finished = inFlight.size(); + for (auto it = inFlight.rbegin(); it != inFlight.rend(); ++it, --finished) { + const auto* range = firstOverlap(**it); + if (range == nullptr) continue; + // In flight means submitted, so the fence is live; signaled = the GPU already ran it. + const bool signaled = context.Function("vkGetFenceStatus")(context.device, (*it)->fence) == VK_SUCCESS; + if (syncAll) return PendingWriteInfo{submissions + (open != nullptr ? 1 : 0), open != nullptr, signaled, range->first, range->second, allBatches}; + return PendingWriteInfo{(*it)->serial, false, signaled, range->first, range->second, finished}; + } + return std::nullopt; +} + +bool Recorder::HasCompletions() const { + if (open != nullptr && !open->completions.empty()) return true; + for (const auto& batch : inFlight) { + if (!batch->completions.empty()) return true; + } + return false; +} + +void Recorder::noteLabelOn(Batch& batch, std::uint64_t address, std::span bytes, std::uint64_t stamp, std::uint32_t queue) { + // One entry per dword; a label overlapping older entries (a 4-byte store inside an 8-byte one or + // the reverse) replaces exactly the dwords it stores, so a lookup composes what memory will hold. + std::lock_guard tableLock(labelTableMutex); + for (std::size_t offset = 0; offset + 4 <= bytes.size(); offset += 4) { + std::uint32_t value = 0; + std::memcpy(&value, bytes.data() + offset, 4); + const auto dword = address + offset; + labels.insert_or_assign(dword, LabelEntry{value, queue, stamp, &batch}); + batch.labelDwords.push_back(dword); + } +} + +void Recorder::NoteLabel(std::uint64_t address, std::span bytes, std::uint64_t stamp, std::uint32_t queue) { + Commands(); + noteLabelOn(*open, address, bytes, stamp, queue); + if (activeRecorder == this && pendingLabelSince.load(std::memory_order_relaxed) == NoPendingLabel) { + pendingLabelSince.store(std::chrono::steady_clock::now().time_since_epoch().count(), std::memory_order_release); + } +} + +std::optional Recorder::PendingLabel(std::uint64_t address, std::size_t bytes, std::uint64_t afterStamp, std::uint32_t& queue) const { + // Under the GPU mutex; the table mutex is taken too so this one path serves both lock regimes. + std::lock_guard tableLock(labelTableMutex); + return lookupLabel(address, bytes, afterStamp, queue); +} + +bool Recorder::PendingLabelIn(std::uint64_t address, std::size_t bytes) const { + std::lock_guard tableLock(labelTableMutex); + if (labels.empty() || bytes == 0) return false; + const auto end = address + bytes; + for (const auto& [dword, entry] : labels) { + if (dword >= address && dword < end) return true; + } + return false; +} + +std::optional Recorder::lookupLabel(std::uint64_t address, std::size_t bytes, std::uint64_t afterStamp, std::uint32_t& queue) const { + if (labels.empty() || (bytes != 4 && bytes != 8) || address % 4 != 0) return std::nullopt; + std::uint64_t value = 0; + for (std::size_t offset = 0; offset < bytes; offset += 4) { + const auto found = labels.find(address + offset); + if (found == labels.end() || found->second.stamp <= afterStamp) return std::nullopt; + value |= static_cast(found->second.value) << (offset * 8u); + if (offset == 0) queue = found->second.queue; + } + return value; +} + +void Recorder::AfterCompletions(std::uint64_t address, std::span bytes, std::uint64_t stamp, std::uint32_t queue, bool storedOnGpu) { + Require(open != nullptr || !inFlight.empty(), "no batch to append a completion label to"); + Batch& batch = open != nullptr ? *open : *inFlight.back(); + std::vector copy(bytes.begin(), bytes.end()); + // The store bypasses the flush hook: it runs inside finish() under the mutex, and the hook would + // find this very range noted (or a later label to it) and sync re-entrantly. Stamped like every + // driver store so a collect memoized for the packet sees it. + static const bool always = std::getenv("APS5_LABEL_STORE_ALWAYS") != nullptr; + const auto sequence = [&] { + std::lock_guard ringLock(writtenBackMutex); + return writtenBackSequence; + }(); + batch.completions.push_back([this, address, copy = std::move(copy), sequence, storedOnGpu] { + if (storedOnGpu && !always && !writtenBackSince(sequence, address, address + copy.size())) { + completionStoresSkipped.fetch_add(1, std::memory_order_relaxed); + return; + } + completionStoresRun.fetch_add(1, std::memory_order_relaxed); + GuestMemory::CheckRange(reinterpret_cast(address), copy.size(), 4, true); + std::memcpy(reinterpret_cast(address), copy.data(), copy.size()); + GuestMemory::MarkWritten(address, copy.size()); + }); + // Counted per batch and counted down when the batch finishes, on every exit path of finish() + // (a label whose range became unmapped is reported there, and a lost device throws before the + // completions run): a label that will never land must not leave the workers reaping forever. + ++batch.completionLabelCount; + completionLabels.fetch_add(1, std::memory_order_acq_rel); + // The table entry before the write note: the note bumps the generation a poller watches, and a + // poller that sees the bump then finds the label without the GPU mutex. + noteLabelOn(batch, address, bytes, stamp, queue); + noteWriteOn(batch, address, bytes.size()); + if (&batch == open.get() && activeRecorder == this && pendingLabelSince.load(std::memory_order_relaxed) == NoPendingLabel) { + pendingLabelSince.store(std::chrono::steady_clock::now().time_since_epoch().count(), std::memory_order_release); + } +} + +void Recorder::NoteWrittenBack(std::uint64_t address, std::size_t bytes) { + Recorder* recorder = activeRecorder; + if (recorder == nullptr || bytes == 0) return; + std::lock_guard ringLock(recorder->writtenBackMutex); + recorder->writtenBack.push_back({++recorder->writtenBackSequence, address, address + bytes}); + while (recorder->writtenBack.size() > 16384) recorder->writtenBack.pop_front(); +} + +bool Recorder::writtenBackSince(std::uint64_t sequence, std::uint64_t begin, std::uint64_t end) { + std::lock_guard ringLock(writtenBackMutex); + if (writtenBack.empty() || writtenBack.back()[0] <= sequence) return false; + if (writtenBack.front()[0] > sequence + 1) return true; + for (auto it = writtenBack.rbegin(); it != writtenBack.rend() && (*it)[0] > sequence; ++it) { + if ((*it)[1] < end && begin < (*it)[2]) return true; + } + return false; +} + +void Recorder::Submit() { + // The work count is cleared even when nothing is open: the driver counts a dispatch after its + // call returns (outside the mutex), so a submit by another thread in between leaves a stale + // count behind, and the callers that act on it would otherwise take the mutex for nothing at + // every packet or submission end until a real batch is next submitted. + if (activeRecorder == this) workSinceSubmit.store(0, std::memory_order_relaxed); + if (open == nullptr) return; + GuestMemory::AssertGpuLockHeld("Recorder::Submit"); + auto batch = std::move(open); + Check(context.Function("vkEndCommandBuffer")(batch->commands), "vkEndCommandBuffer recorder"); + VkSubmitInfo submission{VK_STRUCTURE_TYPE_SUBMIT_INFO}; + submission.commandBufferCount = 1; + submission.pCommandBuffers = &batch->commands; + // The timeline reaches this batch's serial when it completes (see WaitSerial). + const std::uint64_t serial = submissions + 1; + VkTimelineSemaphoreSubmitInfoKHR timelineInfo{VK_STRUCTURE_TYPE_TIMELINE_SEMAPHORE_SUBMIT_INFO_KHR}; + timelineInfo.signalSemaphoreValueCount = 1; + timelineInfo.pSignalSemaphoreValues = &serial; + if (timeline != VK_NULL_HANDLE) { + submission.pNext = &timelineInfo; + submission.signalSemaphoreCount = 1; + submission.pSignalSemaphores = &timeline; + } + Check(context.Function("vkQueueSubmit")(context.queue, 1, &submission, batch->fence), "vkQueueSubmit recorder"); + batch->submitted = true; + batch->serial = ++submissions; + inFlight.push_back(std::move(batch)); + if (activeRecorder == this) pendingLabelSince.store(NoPendingLabel, std::memory_order_release); +} + +std::uint64_t Recorder::SubmitAndEpoch() { + Submit(); + return inFlight.empty() ? 0 : submissions; +} + +namespace { + +// One thread's registration as a timeline waiter (see `unlockedWaiters`): constructed while the +// caller still holds the mutex (or the device), so a teardown that starts after the mutex is +// released sees it and keeps the semaphore until the wait returned. +struct UnlockedWaiter { + std::atomic& count; + explicit UnlockedWaiter(std::uint64_t recorderId) : count(WaitersOf(recorderId)) { count.fetch_add(1, std::memory_order_acq_rel); } + ~UnlockedWaiter() { count.fetch_sub(1, std::memory_order_acq_rel); } + UnlockedWaiter(const UnlockedWaiter&) = delete; + UnlockedWaiter& operator=(const UnlockedWaiter&) = delete; +}; + +// Restores the thread's active sync-site entry on every exit path (a Check throw on a lost device +// included), so the next sync on the thread is not attributed to this one's site. +struct RestoreSyncSite { + std::size_t site; + ~RestoreSyncSite() { activeSyncSite = site; } +}; + +// The timeline wait itself, on handles copied out of the recorder: the caller holds no mutex, and +// the recorder may be torn down meanwhile (its destructor waits for the registered waiters to +// leave after completing every batch, so the handles stay valid until this returns). +VkResult WaitTimeline(VkDevice device, VkSemaphore timeline, PFN_vkWaitSemaphoresKHR waitSemaphores, std::uint64_t serial) { + VkSemaphoreWaitInfoKHR wait{VK_STRUCTURE_TYPE_SEMAPHORE_WAIT_INFO_KHR}; + wait.semaphoreCount = 1; + wait.pSemaphores = &timeline; + wait.pValues = &serial; + auto result = waitSemaphores(device, &wait, 5'000'000'000ull); + for (int waited = 5; result == VK_TIMEOUT; waited += 5) { + // As for the fence wait in finish(): a hung batch is reported every 5 s. + std::fprintf(stderr, "[gpu] recorded batch %llu still running on the GPU after %d s (timeline wait)\n", static_cast(serial), waited); + result = waitSemaphores(device, &wait, 5'000'000'000ull); + } + return result; +} + +} + +void Recorder::WaitSerial(std::uint64_t serial) { + if (timeline == VK_NULL_HANDLE || serial == 0) return; + static const bool profile = std::getenv("APS5_PROFILE_DRAW") != nullptr; + const auto start = profile ? std::chrono::steady_clock::now() : std::chrono::steady_clock::time_point{}; + const UnlockedWaiter waiter{id}; + const auto result = WaitTimeline(context.device, timeline, context.Function("vkWaitSemaphoresKHR"), serial); + if (profile) { + unlockedWaits.fetch_add(1, std::memory_order_relaxed); + unlockedWaitedUs.fetch_add(static_cast(std::chrono::duration_cast(std::chrono::steady_clock::now() - start).count()), std::memory_order_relaxed); + } + Check(result, "vkWaitSemaphoresKHR recorder"); +} + +void Recorder::FinishUpTo(std::uint64_t serial) { + // A drain: the wait already happened (unlocked), the fences are signaled, so the source-0 wait + // recorded here is the bookkeeping only (a caller that did not wait first, WaitForLeases, shows + // up in the site table by its own address). + ++syncCounts[0]; + announcedSource = 4; + const void* site = std::exchange(announcedSite, nullptr); + const auto previousSite = inFlight.empty() || inFlight.front()->serial > serial ? activeSyncSite : BeginSyncSite(0, site != nullptr ? site : __builtin_return_address(0)); + while (!inFlight.empty() && inFlight.front()->serial <= serial) { + auto batch = std::move(inFlight.front()); + inFlight.pop_front(); + finish(std::move(batch), true, 0); + } + activeSyncSite = previousSite; +} + +void Recorder::Sync() { + const auto source = std::exchange(announcedSource, 4); + const void* site = std::exchange(announcedSite, nullptr); + Submit(); + // Counted only when something is waited for: an idle recorder's Sync is free and would only + // swamp the site table with the callers that check nothing first. + const auto previousSite = inFlight.empty() ? activeSyncSite : BeginSyncSite(source, site != nullptr ? site : __builtin_return_address(0)); + while (!inFlight.empty()) { + auto batch = std::move(inFlight.front()); + inFlight.pop_front(); + finish(std::move(batch), true, source); + } + activeSyncSite = previousSite; +} + +void Recorder::SyncThrough(std::uint64_t address, std::size_t bytes, bool waitUnlocked) { + if (bytes == 0) return; + const auto end = address + bytes; + if (completionDepth != 0 && !CompletionStoreSyncs()) { + // A store made by a completion action (a copied buffer's write-back, GuestBufferMemory:: + // WriteBack) reached the flush hook, and a batch still in flight notes the range. Every + // batch in flight now was recorded after the completing one (finish runs front to back), + // so in program order this store precedes their writes: landing it before them is the + // order the hardware gives; waiting for them first (what the hook would do) lands the + // completing batch's bytes over the later batch's, which is wrong, and does so under the + // hold of whoever reaped (the [hooksync] 'store' entries, up to tens of ms per reap). The + // later batch's own ordering against this write-back is its recorder's business at record + // time (FillBuffer and DispatchIndirect consult copiedWriters; a later copied writer's + // write-back runs after this one in finish order; a label behind completions lands after). + // Known gap, not introduced here: a later batch writing the same range GPU-direct in place + // (a writable region bound in place, or a GPU copy-back) is not ordered against this + // write-back by the resource build; it went from deterministically stale to racy. + announcedSource = 4; + announcedSite = nullptr; + ++holdCounters.completionSyncsSkipped; + return; + } + if (completionDepth != 0) ++holdCounters.completionSyncsWaited; + const auto completionWaitStart = completionDepth != 0 && DrawProfiled() ? std::chrono::steady_clock::now() : std::chrono::steady_clock::time_point{}; + struct CompletionWait { + std::chrono::steady_clock::time_point start; + ~CompletionWait() { + if (start != std::chrono::steady_clock::time_point{}) holdCounters.completionSyncWaitMs += std::chrono::duration(std::chrono::steady_clock::now() - start).count(); + } + } completionWait{completionWaitStart}; + if (waitUnlocked) { + // The hook's wait without the mutex: its own acquisition is the outermost on this thread. + const auto source = announcedSource; + const void* site = announcedSite != nullptr ? announcedSite : __builtin_return_address(0); + if (syncThroughUnlocked(address, end, source, site)) return; + // A completion-store wait (kill switch above) is already counted as 'waited'. + if (completionDepth == 0) ++holdCounters.hookLockedWaits; + } + if (!SyncThroughEnabled() || (open != nullptr && overlaps(*open, address, end))) { + // The announced site (or this call's caller) names the sync, not this function. + if (announcedSite == nullptr) announcedSite = __builtin_return_address(0); + Sync(); + return; + } + const auto source = std::exchange(announcedSource, 4); + const void* site = std::exchange(announcedSite, nullptr); + std::uint64_t targetSerial = 0; + for (auto it = inFlight.rbegin(); it != inFlight.rend(); ++it) { + if (overlaps(**it, address, end)) { + targetSerial = (*it)->serial; + break; + } + } + if (targetSerial == 0) return; + targetedSyncs.fetch_add(1, std::memory_order_relaxed); + const RestoreSyncSite restoreSite{BeginSyncSite(source, site != nullptr ? site : __builtin_return_address(0))}; + // Batches are in flight in serial order. A completion's own guest access can sync re-entrantly + // and finish the target first (and a new batch may reuse its allocation, so the serial, not the + // pointer, identifies it); the loop stops then, and later batches stay in flight either way. + while (!inFlight.empty() && inFlight.front()->serial <= targetSerial) { + auto batch = std::move(inFlight.front()); + inFlight.pop_front(); + finish(std::move(batch), true, source); + } + batchesLeftInFlight.fetch_add(inFlight.size(), std::memory_order_relaxed); +} + +bool Recorder::syncThroughUnlocked(std::uint64_t address, std::uint64_t end, int source, const void* site) { + // Only for the outermost acquisition: a nested hook (inside a dispatch's hold) must not give + // up the caller's mutex, and a thread that waits here holds nothing the relock could invert + // (the hook took the mutex from a depth of 0 with the same locks held). + if (HookLockedWait() || timeline == VK_NULL_HANDLE || GuestMemory::GpuMutex().DepthOnThisThread() != 1) return false; + // The target as SyncThrough chooses it: the open batch (submitted here, so the GPU can reach + // it) makes everything the target, else the newest overlapping in-flight batch; with + // SyncThrough disabled everything is the target as well. + std::uint64_t targetSerial = 0; + if (!SyncThroughEnabled() || (open != nullptr && overlaps(*open, address, end))) { + Submit(); + targetSerial = submissions; + } else { + for (auto it = inFlight.rbegin(); it != inFlight.rend(); ++it) { + if (overlaps(**it, address, end)) { + targetSerial = (*it)->serial; + break; + } + } + } + announcedSource = 4; + announcedSite = nullptr; + if (targetSerial == 0) return true; + // Everything the wait needs is copied out: the recorder may be torn down while the mutex is + // released (its destructor completes every batch and waits for the semaphore to be left, see + // WaitTimeline), after which `this` must not be touched. The id re-check below relies on the + // teardown running under the GpuMutex (the driver's drain and failure paths reset the device + // under it; a worker dropping its last device reference outside its lock would not). + const auto myId = id; + const auto device = context.device; + const auto semaphore = timeline; + const auto waitSemaphores = context.Function("vkWaitSemaphoresKHR"); + // The site entry is taken under the mutex (the table is only touched under it; entries are + // appended, never removed, so the index stays valid across the release). + const RestoreSyncSite restoreSite{BeginSyncSite(source, site)}; + const bool profile = DrawProfiled(); + const auto start = profile ? std::chrono::steady_clock::now() : std::chrono::steady_clock::time_point{}; + auto waited = start; + auto& mutex = GuestMemory::GpuMutex(); + VkResult result = VK_SUCCESS; + { + // Registered before the release: a teardown can only start once the mutex is free. + const UnlockedWaiter waiter{myId}; + mutex.unlock(); + result = WaitTimeline(device, semaphore, waitSemaphores, targetSerial); + // Read before the relock: the GPU wait, not the wait for the mutex behind it. + if (profile) waited = std::chrono::steady_clock::now(); + } + GuestMemory::TagGpuLockSite(GuestMemory::GpuLockSite::Hook); + mutex.lock(); + if (profile) { + const auto ms = std::chrono::duration(waited - start).count(); + // The GPU wait is charged like a locked wait, so the per-source, per-thread and per-site + // totals of the [recorder] line keep their meaning; the unlocked count says how many were + // not held. The relock's wait for the mutex is kept apart (the [lock] line also counts it + // under 'hook' waits, for a cross-check). + syncWaitedMs[source] += ms; + CountThreadSync(source, ms); + CountSiteWait(ms); + ++holdCounters.hookUnlockedWaits; + holdCounters.hookUnlockedWaitMs += ms; + holdCounters.hookRelockMs += std::chrono::duration(std::chrono::steady_clock::now() - waited).count(); + } + if (!RecorderAlive(myId)) { + // Torn down meanwhile: its destructor's Sync() ran every completion, nothing is pending. + ++holdCounters.hookUnlockedTornDown; + Check(result, "vkWaitSemaphoresKHR recorder"); + return true; + } + Check(result, "vkWaitSemaphoresKHR recorder"); + // The completions up to the target, under the mutex again; the fences are signaled, so the + // waits inside finish() return at once (charged to the same site entry). Another thread may + // have finished some meanwhile. + targetedSyncs.fetch_add(1, std::memory_order_relaxed); + while (!inFlight.empty() && inFlight.front()->serial <= targetSerial) { + auto batch = std::move(inFlight.front()); + inFlight.pop_front(); + finish(std::move(batch), true, source); + } + batchesLeftInFlight.fetch_add(inFlight.size(), std::memory_order_relaxed); + return true; +} + +bool Recorder::Reap() { + const auto status = context.Function("vkGetFenceStatus"); + const bool profile = DrawProfiled(); + const auto start = profile ? std::chrono::steady_clock::now() : std::chrono::steady_clock::time_point{}; + std::uint64_t retired = 0; + while (!inFlight.empty()) { + if (status(context.device, inFlight.front()->fence) != VK_SUCCESS) break; + auto batch = std::move(inFlight.front()); + inFlight.pop_front(); + finish(std::move(batch), false, 4); + ++retired; + } + if (profile) { + ++holdCounters.reaps; + if (retired != 0) ++holdCounters.reapsWithWork; + holdCounters.reapBatches += retired; + holdCounters.reapMs += std::chrono::duration(std::chrono::steady_clock::now() - start).count(); + } + return inFlight.empty(); +} + +void Recorder::finish(std::unique_ptr batch, bool wait, int source) { + // The batch's writes stay published until its completions ran, on every exit path. + struct Finishing { + Recorder& recorder; + const Batch* batch; + ~Finishing() { + auto& list = recorder.finishing; + list.erase(std::remove(list.begin(), list.end(), batch), list.end()); + // The batch's completion labels have landed, or never will (fence failure below). + if (batch->completionLabelCount != 0) completionLabels.fetch_sub(batch->completionLabelCount, std::memory_order_acq_rel); + // This also runs while the fence-failure throw unwinds: an allocation failure in the + // rebuild keeps the previous snapshot, which is a superset (a stale entry only causes a + // false hit, never a miss) rather than terminating. + try { + recorder.publishPendingWrites(); + } catch (...) { + } + } + } finishingScope{*this, batch.get()}; + finishing.push_back(batch.get()); + if (wait) { + // APS5_PROFILE_DRAW: how long the CPU waits for recorded GPU work, reported every 10 s. + static const bool profile = std::getenv("APS5_PROFILE_DRAW") != nullptr; + static double waitedMs = 0; + static std::uint64_t waits = 0; + static auto lastReport = std::chrono::steady_clock::now(); + const auto waitStart = std::chrono::steady_clock::now(); + struct Report { + bool enabled; + int source; + std::chrono::steady_clock::time_point start; + const Recorder& recorder; + ~Report() { + if (!enabled) return; + const auto now = std::chrono::steady_clock::now(); + const auto ms = std::chrono::duration(now - start).count(); + waitedMs += ms; + syncWaitedMs[source] += ms; + ++waits; + CountThreadSync(source, ms); + CountSiteWait(ms); + if (now - lastReport > std::chrono::seconds(10)) { + lastReport = now; + const auto& h = holdCounters; + std::fprintf(stderr, "[recorder] %llu syncs waited %.1f s for the GPU in total (sources, count/wait: idle %llu/%.1fs, pending write %llu/%.1fs, recorded store %llu/%.1fs, address-based %llu/%.1fs, other %llu/%.1fs); hook %llu calls, %llu locked; %llu targeted syncs left %llu batches in flight; snapshot %llu rebuilds %.0f ms, %llu notes covered; %llu unlocked timeline waits %.1f s; %llu submissions, %zu label entries, %llu completion labels pending; fence waits by thread (count/wait):%s; top sync sites (source@caller syncs/batches/wait):%s; under holds (cumulative): %llu completions ran %.0f ms (kept objects released %.0f ms), pending-write syncs from completions: %llu skipped, %llu waited %.0f ms; %llu reaps (%llu with work) retired %llu batches in %.0f ms; hook waits unlocked %llu / %.0f ms GPU + %.0f ms relock (%llu found the recorder torn down), locked %llu; deferred releases: on the release thread %llu batches (%llu objects) in %.0f ms, inline %llu batches (%llu objects) in %.0f ms (%llu batches over the queue bound of %zu), queue max %llu batches, %llu pending\n",static_cast(waits), waitedMs / 1000, static_cast(syncCounts[0]), syncWaitedMs[0] / 1000, static_cast(syncCounts[1]), syncWaitedMs[1] / 1000, static_cast(syncCounts[2]), syncWaitedMs[2] / 1000, static_cast(syncCounts[3]), syncWaitedMs[3] / 1000, static_cast(syncCounts[4]), syncWaitedMs[4] / 1000, static_cast(hookCalls.load()), static_cast(hookLocks.load()), static_cast(targetedSyncs.load()), static_cast(batchesLeftInFlight.load()), static_cast(snapshotRebuilds), snapshotRebuildMs, static_cast(snapshotCovered), static_cast(unlockedWaits.load()), unlockedWaitedUs.load() / 1e6, static_cast(recorder.submissions), recorder.labels.size(), static_cast(completionLabels.load()), ThreadSyncReport().c_str(), SyncSiteReport().c_str(), static_cast(h.completions), h.completionMs, h.keptReleaseMs, static_cast(h.completionSyncsSkipped), static_cast(h.completionSyncsWaited), h.completionSyncWaitMs, static_cast(h.reaps), static_cast(h.reapsWithWork), static_cast(h.reapBatches), h.reapMs, static_cast(h.hookUnlockedWaits), h.hookUnlockedWaitMs, h.hookRelockMs, static_cast(h.hookUnlockedTornDown), static_cast(h.hookLockedWaits), static_cast(threadReleases.load()), static_cast(threadObjects.load()), threadReleaseUs.load() / 1000.0, static_cast(inlineReleases.load()), static_cast(inlineObjects.load()), inlineReleaseUs.load() / 1000.0, static_cast(inlineOverBound.load()), ReleaseQueueBound(), static_cast(releaseQueueMax.load()), static_cast(deferredPending.load())); + std::fprintf(stderr, "[recorder] completion label stores: %llu run, %llu skipped (no CPU write-back overlapped them)\n", static_cast(completionStoresRun.load()), static_cast(completionStoresSkipped.load())); + } + } + } report{profile, source, waitStart, *this}; + const auto waitFences = context.Function("vkWaitForFences"); + auto result = waitFences(context.device, 1, &batch->fence, VK_TRUE, 5'000'000'000ull); + for (int waited = 5; result == VK_TIMEOUT; waited += 5) { + // A batch still running after 5 s is reported (every 5 s) so a GPU-side hang is visible. + std::fprintf(stderr, "[gpu] recorded batch still running on the GPU after %d s\n", waited); + result = waitFences(context.device, 1, &batch->fence, VK_TRUE, 5'000'000'000ull); + } + if (result != VK_SUCCESS && result != VK_ERROR_DEVICE_LOST) { + const auto idle = context.Function("vkDeviceWaitIdle")(context.device); + Check(idle, "vkDeviceWaitIdle after recorder fence failure"); + } + if (result != VK_SUCCESS) { + release(*batch); + Check(result, "vkWaitForFences recorder"); + } + } + readGpuTiming(*batch); + // Completions store GPU results to guest memory; a failing one is reported, the rest still run. + // Timed (APS5_PROFILE_DRAW) with the release of the kept objects: both run under whichever hold + // reaped or synced the batch. Inside them the flush hook makes no pending-write wait (SyncThrough). + const bool profile = DrawProfiled(); + const auto completionStart = profile ? std::chrono::steady_clock::now() : std::chrono::steady_clock::time_point{}; + { + struct InCompletion { + InCompletion() { ++completionDepth; } + ~InCompletion() { --completionDepth; } + } inCompletion; + for (auto& action : batch->completions) { + try { + action(); + } catch (const std::exception& error) { + std::fprintf(stderr, "[gpu] deferred write-back failed: %s\n", error.what()); + } + } + } + if (profile) holdCounters.completions += batch->completions.size(); + const auto keptStart = profile ? std::chrono::steady_clock::now() : std::chrono::steady_clock::time_point{}; + // Deferred only from inside a hold: the unlock that releases the list is this thread's own, and + // a caller finishing batches without the mutex (a test) might never make one. + if (ReleaseUnderLock() || !GuestMemory::GpuMutex().HeldByThisThread()) { + batch->completions.clear(); + batch->kept.clear(); + } else { + // To this thread's unlock (see ReleaseDeferredKeeps); the completions go with the objects, + // their captures hold the same ones. The push comes first: a failed one (the list's growth) + // destroys the batch's objects here, under the mutex, counts nothing the destructor would + // wait for and must not escape (the rest of finish() releases the fence and command buffer). + try { + deferredBatches.push_back({std::move(batch->kept), std::move(batch->completions)}); + deferredPending.fetch_add(1, std::memory_order_acq_rel); + } catch (...) { + } + batch->completions.clear(); + batch->kept.clear(); + } + if (profile) { + const auto now = std::chrono::steady_clock::now(); + holdCounters.keptReleaseMs += std::chrono::duration(now - keptStart).count(); + holdCounters.completionMs += std::chrono::duration(now - completionStart).count(); + } + // The batch's label entries leave the table (a later label to the same dword already replaced + // its entry and belongs to another batch). Correctness never depended on this removal. + if (!batch->labelDwords.empty()) { + std::lock_guard tableLock(labelTableMutex); + for (const auto dword : batch->labelDwords) { + const auto found = labels.find(dword); + if (found != labels.end() && found->second.batch == batch.get()) labels.erase(found); + } + } + batch->labelDwords.clear(); + release(*batch); +} + +void Recorder::release(Batch& batch) noexcept { + if (batch.queries != VK_NULL_HANDLE) context.Function("vkDestroyQueryPool")(context.device, batch.queries, nullptr); + batch.queries = VK_NULL_HANDLE; + // A completed (or never submitted) batch's objects are kept for reuse: the fence is signaled or + // untouched, so resetting it cannot block, and the command buffer is no longer pending. + if (batch.commands != VK_NULL_HANDLE && batch.fence != VK_NULL_HANDLE && spare.size() < 64 && context.Function("vkResetFences")(context.device, 1, &batch.fence) == VK_SUCCESS) { + spare.emplace_back(batch.commands, batch.fence); + batch.commands = VK_NULL_HANDLE; + batch.fence = VK_NULL_HANDLE; + return; + } + if (batch.commands != VK_NULL_HANDLE) context.Function("vkFreeCommandBuffers")(context.device, context.pool, 1, &batch.commands); + if (batch.fence != VK_NULL_HANDLE) context.Function("vkDestroyFence")(context.device, batch.fence, nullptr); + batch.commands = VK_NULL_HANDLE; + batch.fence = VK_NULL_HANDLE; +} + +} diff --git a/core/libs/prx/libSceAgcDriver/Graphics/src/RegisteredGpuMemory.cpp b/core/libs/prx/libSceAgcDriver/Graphics/src/RegisteredGpuMemory.cpp deleted file mode 100644 index 058dc9229..000000000 --- a/core/libs/prx/libSceAgcDriver/Graphics/src/RegisteredGpuMemory.cpp +++ /dev/null @@ -1,21 +0,0 @@ -#include "prx/libSceAgcDriver/Graphics/include/GuestBufferMemory.hpp" -#include "prx/libSceAgcDriver/Execution/include/GuestMemory.hpp" - -namespace AgcDriver::Graphics { - -void GuestBufferMemory::AcquireRegistered() { - Require(!uploaded && regions.empty() && lease.empty(), "guest allocation lease must precede resource registration"); - lease = GuestAllocations::GuestAllocationsAcquire_nid_postfix(); - regions.reserve(lease.size()); - for (const auto& range : lease) { - validate(range->address, range->bytes); - std::vector snapshot; - if (!range->writable) { - snapshot.resize(range->bytes); - GuestMemory::Read(range->address, snapshot); - } - regions.push_back({range->address, range->address + range->bytes, range->writable, std::move(snapshot), nullptr}); - } -} - -} diff --git a/core/libs/prx/libSceAgcDriver/Graphics/src/Resources.cpp b/core/libs/prx/libSceAgcDriver/Graphics/src/Resources.cpp index b219865d9..a8ad8e72d 100644 --- a/core/libs/prx/libSceAgcDriver/Graphics/src/Resources.cpp +++ b/core/libs/prx/libSceAgcDriver/Graphics/src/Resources.cpp @@ -1,11 +1,13 @@ #include "prx/libSceAgcDriver/Graphics/include/Resources.hpp" #include "prx/libSceAgcDriver/Graphics/include/BufferPool.hpp" +#include "prx/libSceAgcDriver/Graphics/include/Recorder.hpp" #include "prx/libSceAgcDriver/Execution/include/PerformanceTimer.hpp" +#include "prx/libSceAgcDriver/Execution/include/GuestMemory.hpp" #include namespace AgcDriver::Graphics { -Buffer::Buffer(const Context& context, std::size_t size, VkBufferUsageFlags usage, VkMemoryPropertyFlags properties) : context(context), size(size), usage(usage), properties(properties) { +Buffer::Buffer(const Context& context, std::size_t size, VkBufferUsageFlags usage, VkMemoryPropertyFlags properties) : context(context), size(size), capacity(BufferPool::Capacity(size)), usage(usage), properties(properties) { Require(size != 0, "zero-sized GPU buffer"); const bool addressable = (usage & VK_BUFFER_USAGE_SHADER_DEVICE_ADDRESS_BIT) != 0; Require(!addressable || context.bufferDeviceAddress, "buffer device address is not enabled"); @@ -16,12 +18,11 @@ Buffer::Buffer(const Context& context, std::size_t size, VkBufferUsageFlags usag mapping = allocation->mapping; deviceAddress = allocation->address; allocationBytes = allocation->allocationBytes; - reusable = true; return; } try { VkBufferCreateInfo info{VK_STRUCTURE_TYPE_BUFFER_CREATE_INFO}; - info.size = size; + info.size = capacity; info.usage = usage; info.sharingMode = VK_SHARING_MODE_EXCLUSIVE; Check(context.Function("vkCreateBuffer")(context.device, &info, nullptr, &buffer), "vkCreateBuffer"); @@ -32,12 +33,22 @@ Buffer::Buffer(const Context& context, std::size_t size, VkBufferUsageFlags usag if (addressable) allocation.pNext = &flags; allocation.allocationSize = requirements.size; allocationBytes = requirements.size; - allocation.memoryTypeIndex = context.MemoryType(requirements.memoryTypeBits, properties); + // The CPU reads most of these buffers back (write-back, diffs), which is very slow from + // write-combined memory, so the default host properties prefer cached host memory. + constexpr VkMemoryPropertyFlags hostDefault = VK_MEMORY_PROPERTY_HOST_VISIBLE_BIT | VK_MEMORY_PROPERTY_HOST_COHERENT_BIT; + if (properties == hostDefault) { + try { + allocation.memoryTypeIndex = context.MemoryType(requirements.memoryTypeBits, hostDefault | VK_MEMORY_PROPERTY_HOST_CACHED_BIT); + } catch (const std::runtime_error&) { + allocation.memoryTypeIndex = context.MemoryType(requirements.memoryTypeBits, hostDefault); + } + } else { + allocation.memoryTypeIndex = context.MemoryType(requirements.memoryTypeBits, properties); + } Check(context.Function("vkAllocateMemory")(context.device, &allocation, nullptr, &memory), "vkAllocateMemory buffer"); Check(context.Function("vkBindBufferMemory")(context.device, buffer, memory, 0), "vkBindBufferMemory"); initializeAddress(usage); - if ((properties & VK_MEMORY_PROPERTY_HOST_VISIBLE_BIT) != 0) Check(context.Function("vkMapMemory")(context.device, memory, 0, VK_WHOLE_SIZE, 0, &mapping), "vkMapMemory"); - reusable = true; + Check(context.Function("vkMapMemory")(context.device, memory, 0, VK_WHOLE_SIZE, 0, &mapping), "vkMapMemory"); } catch (...) { release(); throw; @@ -49,8 +60,8 @@ Buffer::~Buffer() { } void Buffer::release() noexcept { - if (reusable && buffer && memory && cache) { - cache->Put({buffer, memory, mapping, deviceAddress, allocationBytes, size, usage, properties}); + if (mapping && buffer && memory && cache) { + cache->Put({buffer, memory, mapping, deviceAddress, allocationBytes, capacity, usage, properties}); return; } if (mapping) context.Function("vkUnmapMemory")(context.device, memory); @@ -63,18 +74,76 @@ VkBuffer Buffer::Handle() const { } std::span Buffer::Bytes() { - Require(mapping != nullptr, "GPU-only buffer has no CPU mapping"); return {static_cast(mapping), size}; } void Buffer::Invalidate() { - Require(mapping != nullptr, "cannot invalidate an unmapped GPU buffer"); VkMappedMemoryRange range{VK_STRUCTURE_TYPE_MAPPED_MEMORY_RANGE}; range.memory = memory; range.size = VK_WHOLE_SIZE; Check(context.Function("vkInvalidateMappedMemoryRanges")(context.device, 1, &range), "vkInvalidateMappedMemoryRanges"); } +DeviceBuffer::DeviceBuffer(const Context& context, std::size_t size, VkBufferUsageFlags usage) : context(context), size(size), capacity(BufferPool::Capacity(size)), usage(usage) { + Require(size != 0, "zero-sized device buffer"); + cache = GetBufferPool(context); + if (const auto allocation = cache->Take(size, usage, VK_MEMORY_PROPERTY_DEVICE_LOCAL_BIT)) { + buffer = allocation->buffer; + memory = allocation->memory; + allocationBytes = allocation->allocationBytes; + return; + } + try { + VkBufferCreateInfo info{VK_STRUCTURE_TYPE_BUFFER_CREATE_INFO}; + info.size = capacity; + info.usage = usage; + info.sharingMode = VK_SHARING_MODE_EXCLUSIVE; + Check(context.Function("vkCreateBuffer")(context.device, &info, nullptr, &buffer), "vkCreateBuffer device"); + VkMemoryRequirements requirements{}; + context.Function("vkGetBufferMemoryRequirements")(context.device, buffer, &requirements); + VkMemoryAllocateInfo allocation{VK_STRUCTURE_TYPE_MEMORY_ALLOCATE_INFO}; + allocation.allocationSize = requirements.size; + allocationBytes = requirements.size; + allocation.memoryTypeIndex = context.MemoryType(requirements.memoryTypeBits, VK_MEMORY_PROPERTY_DEVICE_LOCAL_BIT); + Check(context.Function("vkAllocateMemory")(context.device, &allocation, nullptr, &memory), "vkAllocateMemory device buffer"); + Check(context.Function("vkBindBufferMemory")(context.device, buffer, memory, 0), "vkBindBufferMemory device"); + } catch (...) { + release(); + throw; + } +} + +DeviceBuffer::~DeviceBuffer() { + release(); +} + +void DeviceBuffer::release() noexcept { + if (buffer && memory && cache) { + cache->Put({buffer, memory, nullptr, 0, allocationBytes, capacity, usage, VK_MEMORY_PROPERTY_DEVICE_LOCAL_BIT}); + return; + } + if (buffer) context.Function("vkDestroyBuffer")(context.device, buffer, nullptr); + if (memory) context.Function("vkFreeMemory")(context.device, memory, nullptr); +} + +VkBuffer DeviceBuffer::Handle() const { + return buffer; +} + +std::size_t DeviceBuffer::Size() const { + return size; +} + +void CopyBuffer(const Context& context, VkCommandBuffer commands, VkBuffer source, VkDeviceSize sourceOffset, VkBuffer destination, VkDeviceSize destinationOffset, VkDeviceSize bytes) { + const VkBufferCopy region{sourceOffset, destinationOffset, bytes}; + context.Function("vkCmdCopyBuffer")(commands, source, destination, 1, ®ion); +} + +void RecordMemoryBarrier(const Context& context, VkCommandBuffer commands, VkPipelineStageFlags sourceStage, VkPipelineStageFlags destinationStage, VkAccessFlags sourceAccess, VkAccessFlags destinationAccess) { + const VkMemoryBarrier barrier{VK_STRUCTURE_TYPE_MEMORY_BARRIER, nullptr, sourceAccess, destinationAccess}; + context.Function("vkCmdPipelineBarrier")(commands, sourceStage, destinationStage, 0, 1, &barrier, 0, nullptr, 0, nullptr); +} + RenderTarget::RenderTarget(const Context& context, const ColorTarget& target, bool blending) : context(context) { VkFormatProperties properties{}; context.formatProperties(context.physical, target.format, &properties); @@ -136,6 +205,8 @@ VkImageView RenderTarget::View() const { } CommandBatch::CommandBatch(const Context& context) : context(context) { + // Records into the device's one command pool and submits to its queue: device-lock work only. + GuestMemory::AssertGpuLockHeld("CommandBatch"); try { VkCommandBufferAllocateInfo allocation{VK_STRUCTURE_TYPE_COMMAND_BUFFER_ALLOCATE_INFO}; allocation.commandPool = context.pool; @@ -147,6 +218,8 @@ CommandBatch::CommandBatch(const Context& context) : context(context) { VkCommandBufferBeginInfo begin{VK_STRUCTURE_TYPE_COMMAND_BUFFER_BEGIN_INFO}; begin.flags = VK_COMMAND_BUFFER_USAGE_ONE_TIME_SUBMIT_BIT; Check(context.Function("vkBeginCommandBuffer")(commands, &begin), "vkBeginCommandBuffer"); + // Work recorded so far goes first in queue order, so this batch sees its results. + if (auto* recorder = Recorder::Active(); recorder != nullptr && recorder->Recording()) recorder->Submit(); } catch (...) { release(); throw; @@ -176,16 +249,6 @@ void CommandBatch::SubmitAndWait() { Wait(); } -void CommandBatch::Reset() { - Require(submitted && !pending, "command batch must complete before reuse"); - Check(context.Function("vkResetFences")(context.device, 1, &fence), "vkResetFences graphics"); - Check(context.Function("vkResetCommandBuffer")(commands, 0), "vkResetCommandBuffer graphics"); - VkCommandBufferBeginInfo begin{VK_STRUCTURE_TYPE_COMMAND_BUFFER_BEGIN_INFO}; - begin.flags = VK_COMMAND_BUFFER_USAGE_ONE_TIME_SUBMIT_BIT; - Check(context.Function("vkBeginCommandBuffer")(commands, &begin), "vkBeginCommandBuffer graphics"); - submitted = false; -} - void CommandBatch::Submit() { PerformanceTimer timing("Graphics.Submit"); Require(!submitted, "command batch has already been submitted"); @@ -208,6 +271,8 @@ void CommandBatch::Wait() { timing.Mark("fence_wait"); if (result == VK_SUCCESS || result == VK_ERROR_DEVICE_LOST) pending = false; Check(result, "vkWaitForFences graphics"); + // Recorded batches preceded this one, so their completions (write-backs) can run now. + if (auto* recorder = Recorder::Active()) recorder->Reap(); } } diff --git a/core/libs/prx/libSceAgcDriver/Graphics/src/Sampler.cpp b/core/libs/prx/libSceAgcDriver/Graphics/src/Sampler.cpp index 5a0e4cf8f..660b16bb1 100644 --- a/core/libs/prx/libSceAgcDriver/Graphics/src/Sampler.cpp +++ b/core/libs/prx/libSceAgcDriver/Graphics/src/Sampler.cpp @@ -1,8 +1,10 @@ #include "prx/libSceAgcDriver/Graphics/include/Sampler.hpp" +#include +#include namespace AgcDriver::Graphics { - Sampler::Sampler(const Context& context, const GuestSamplerResource& descriptor) : device(context.device), destroySampler(context.Function("vkDestroySampler")) { + Sampler::Sampler(const Context& context, const GuestSamplerResource& descriptor) : context(context) { Require(!descriptor.anisotropyEnable || context.samplerAnisotropy, "guest sampler descriptor requests anisotropic filtering which the device does not support"); Require(descriptor.maxAnisotropy <= context.limits.maxSamplerAnisotropy, "guest sampler descriptor requests an anisotropy ratio beyond the device limit"); Require(descriptor.lodBias >= -context.limits.maxSamplerLodBias && descriptor.lodBias <= context.limits.maxSamplerLodBias, "guest sampler descriptor requests a LOD bias beyond the device limit"); @@ -31,22 +33,36 @@ namespace AgcDriver::Graphics { } void Sampler::release() noexcept { - if (sampler) destroySampler(device, sampler, nullptr); + if (sampler) context.Function("vkDestroySampler")(context.device, sampler, nullptr); } VkSampler Sampler::Handle() const { return sampler; } - std::shared_ptr SamplerCache::Get(const Context& context, std::span words, const GuestSamplerResource& descriptor) { - Require(words.size() == 4, "sampler cache descriptor must contain four DWORDs"); - const std::array key{words[0], words[1], words[2], words[3], descriptor.compareEnable ? 1u : 0u}; + SamplerCache::SamplerCache(std::size_t capacity) : capacity(std::max(capacity, 1)) {} + + std::shared_ptr SamplerCache::Get(const Context& context, std::span words, bool compareEnable) { + Require(words.size() == 4, "guest sampler descriptor must contain 4 dwords"); + const std::array key{words[0], words[1], words[2], words[3], compareEnable ? 1u : 0u}; std::lock_guard lock(mutex); - const auto found = entries.find(key); - if (found != entries.end()) return found->second; - auto sampler = std::make_shared(context, descriptor); - if (entries.size() >= 256) entries.erase(entries.begin()); - entries.emplace(key, sampler); + ++clock; + if (const auto found = entries.find(key); found != entries.end()) { + ++hits; + found->second.lastUse = clock; + return found->second.sampler; + } + ++misses; + auto resource = DecodeSamplerResource(words); + resource.compareEnable = compareEnable; + auto sampler = std::make_shared(context, resource); + // The cap keeps live samplers well below the device's limit (NVIDIA: ~4000); a set in flight + // still holds the evicted sampler through its own shared_ptr. + while (entries.size() >= capacity) { + const auto oldest = std::min_element(entries.begin(), entries.end(), [](const auto& left, const auto& right) { return left.second.lastUse < right.second.lastUse; }); + entries.erase(oldest); + } + entries.emplace(key, Entry{sampler, clock}); return sampler; } diff --git a/core/libs/prx/libSceAgcDriver/Graphics/src/ShaderInputState.cpp b/core/libs/prx/libSceAgcDriver/Graphics/src/ShaderInputState.cpp index 04951b202..b5663629f 100644 --- a/core/libs/prx/libSceAgcDriver/Graphics/src/ShaderInputState.cpp +++ b/core/libs/prx/libSceAgcDriver/Graphics/src/ShaderInputState.cpp @@ -2,7 +2,10 @@ #include "SceShaders.hpp" #include "prx/libSceAgc/Shader/include/ShaderConstants.hpp" #include "prx/libSceAgcDriver/Execution/include/GuestMemory.hpp" +#include #include +#include +#include #include #include #include @@ -25,7 +28,9 @@ constexpr std::uint32_t spiShaderColFormat = 0x1C5; std::uint32_t read(const Registers& registers, std::uint32_t offset) { const auto it = registers.find(offset); if (it == registers.end()) { - throw std::runtime_error("AGC graphics: missing register at DWORD 0x" + std::to_string(offset)); + char text[64]; + std::snprintf(text, sizeof(text), "AGC graphics: missing register at DWORD 0x%x", offset); + throw std::runtime_error(text); } return it->second; } @@ -65,9 +70,12 @@ ShaderRecompiler::ShaderComputeStageInfo DecodeComputeStageInfo(const Registers& if ((rsrc2 & 0x1u) != 0) { throw std::runtime_error("AGC graphics: COMPUTE_PGM_RSRC2.SCRATCH_EN is unsupported"); } + // Debug aid: APS5_LDS_SLACK= grows every dispatch's LDS allocation by that much, to tell + // whether a program depends on addresses past its declared allocation. + static const std::uint32_t ldsSlack = [] { const char* text = std::getenv("APS5_LDS_SLACK"); return text ? static_cast(std::strtoul(text, nullptr, 0)) : 0u; }(); return ShaderRecompiler::ShaderComputeStageInfo{ {numThreadX, numThreadY, numThreadZ}, - ((rsrc2 >> 15u) & 0x1FFu) * 128u, + std::min(((rsrc2 >> 15u) & 0x1FFu) * 128u + ldsSlack, 16384u), {((rsrc2 >> 7u) & 0x1u) != 0, ((rsrc2 >> 8u) & 0x1u) != 0, ((rsrc2 >> 9u) & 0x1u) != 0}, ((rsrc2 >> 10u) & 0x1u) != 0, ((rsrc2 >> 11u) & 0x3u) + 1u @@ -92,12 +100,8 @@ ShaderRecompiler::ShaderPixelStageInfo DecodePixelStageInfo(const Registers& con interpolatorSettings[i] = read(context, spiPsInputCntl0 + i); } const auto shaderControl = read(context, dbShaderControl); - if (((shaderControl >> 9u) & 0x1u) != 0) { - throw std::runtime_error("AGC graphics: DB_SHADER_CONTROL.DUAL_EXPORT_ENABLE is unsupported"); - } - if (((shaderControl >> 11u) & 0x1u) != 0) { - throw std::runtime_error("AGC graphics: DB_SHADER_CONTROL.ALPHA_TO_MASK_DISABLE is unsupported"); - } + // Bits 9 and 11 are EXEC_ON_HIER_FAIL and ALPHA_TO_MASK_DISABLE on GFX10; neither changes what the + // recompiled pixel shader computes. if (((shaderControl >> 13u) & 0x3u) != 0) { throw std::runtime_error("AGC graphics: DB_SHADER_CONTROL.CONSERVATIVE_Z_EXPORT is unsupported"); } diff --git a/core/libs/prx/libSceAgcDriver/Graphics/src/ShaderResources.cpp b/core/libs/prx/libSceAgcDriver/Graphics/src/ShaderResources.cpp index 45db4dda8..637d3ce4f 100644 --- a/core/libs/prx/libSceAgcDriver/Graphics/src/ShaderResources.cpp +++ b/core/libs/prx/libSceAgcDriver/Graphics/src/ShaderResources.cpp @@ -1,13 +1,28 @@ #include "prx/libSceAgcDriver/Graphics/include/ShaderResources.hpp" -#include "prx/libSceAgcDriver/Graphics/include/TextureCache.hpp" +#include "prx/libSceAgcDriver/Graphics/include/Recorder.hpp" +#include +#include +#include +#include +#include +#include +#include +#include #include "prx/libSceAgcDriver/Execution/include/GuestMemory.hpp" -#include "prx/libSceAgcDriver/Execution/include/PerformanceTimer.hpp" +#include "prx/libSceAgcDriver/Graphics/include/TextureTiling.hpp" +#include "prx/libSceAgcDriver/Graphics/include/DccMetadata.hpp" +#include "prx/libSceAgcDriver/Graphics/include/TextureFormat.hpp" #include "prx/libc/include/General.hpp" #include "Optimization/include/Optimization/ShaderStageInputInfo.hpp" #include #include +#include +#include +#include #include #include +#include +#include "prx/libc/include/GuestAllocations.hpp" namespace AgcDriver::Graphics { namespace { @@ -28,6 +43,446 @@ VkComponentSwizzle ComponentSwizzleFor(std::uint8_t dstSel) { } } +// Sampled textures are reused across draws and dispatches while their guest bytes are unchanged; a +// byte copy of the guest surface validates each reuse, so CPU or GPU writes to it force a re-upload. +// The key is everything a lookup matches: the device, the eight descriptor words and the swizzle. +struct TextureKey { + VkDevice device; + std::array words; + std::array components; + bool operator==(const TextureKey&) const = default; +}; + +std::uint64_t hashWords(std::uint64_t hash, const std::uint32_t* words, std::size_t count) { + for (std::size_t i = 0; i < count; ++i) hash = (hash ^ words[i]) * 1099511628211ull; + return hash; +} + +struct TextureKeyHash { + std::size_t operator()(const TextureKey& key) const noexcept { + auto hash = 14695981039346656037ull ^ reinterpret_cast(key.device); + hash = hashWords(hash, key.words.data(), key.words.size()); + return static_cast(hashWords(hash, key.components.data(), key.components.size())); + } +}; + +TextureKey MakeTextureKey(VkDevice device, std::span words, VkComponentMapping components) { + TextureKey key{device, {}, {static_cast(components.r), static_cast(components.g), static_cast(components.b), static_cast(components.a)}}; + std::copy(words.begin(), words.end(), key.words.begin()); + return key; +} + +struct CachedTexture { + TextureKey key; + std::uint64_t address; + std::vector bytes; + std::shared_ptr texture; + // A fast-cleared surface is cached as its clear texels; it stays valid while the keys are unchanged. + DccKeys keys = DccKeys::Uncompressed; + // Write generation the snapshot is known current at (see GuestMemory::CollectWrites). + std::uint64_t generation = 0; + // A texture viewing a storage image on the GPU has no snapshot; it stays valid while that image + // is still its surface's cached image and nothing wrote the guest memory since it matched. + std::shared_ptr source; + std::uint64_t sourceVersion = 0; + std::uint64_t accounted = 0; +}; + +// Entries in use order (front = most recent) with a hash index by key: a lookup is O(1) and the +// eviction takes the back. APS5_NO_TEXTURE_HASH=1 finds entries by scanning the list (the index is +// still kept), the linear lookup of before. +struct TextureCache { + std::mutex mutex; + std::list entries; + std::unordered_map::iterator, TextureKeyHash> index; + std::uint64_t bytes = 0; +}; + +TextureCache& Textures() { + static TextureCache cache; + return cache; +} + +bool TextureHashEnabled() { + static const bool disabled = std::getenv("APS5_NO_TEXTURE_HASH") != nullptr; + return !disabled; +} + +std::list::iterator findTexture(TextureCache& cache, const TextureKey& key) { + if (TextureHashEnabled()) { + const auto found = cache.index.find(key); + return found == cache.index.end() ? cache.entries.end() : found->second; + } + for (auto it = cache.entries.begin(); it != cache.entries.end(); ++it) { + if (it->key == key) return it; + } + return cache.entries.end(); +} + +void eraseTexture(TextureCache& cache, std::list::iterator it) { + cache.bytes -= it->accounted; + cache.index.erase(it->key); + cache.entries.erase(it); +} + +// Moves an entry to the front (most recently used). +void touchTexture(TextureCache& cache, std::list::iterator it) { + cache.entries.splice(cache.entries.begin(), cache.entries, it); +} + +// APS5_PROFILE_DRAW: what the sampled-texture and storage-image lookups did, printed as [textures] +// every 10 s (cumulative). Storage-sourced sampled textures need no CPU read; snapshots do, and a +// pending read is a snapshot compare or read made while recorded work still wrote the surface (the +// flush hook may find that work already signaled, so this bounds the hook syncs from above; the +// [hooksync] line attributes the actual waits). +struct TextureCounters { + std::atomic fromStorage{0}; + std::atomic snapshots{0}; + std::atomic pendingReads{0}; + std::atomic storageFallbacks{0}; + std::atomic records{0}; + std::atomic fastHits{0}; + std::atomic fastMisses{0}; + std::atomic storageHits{0}; + std::atomic storageCreated{0}; + std::atomic lastReport{0}; +}; + +TextureCounters& TextureCounts() { + static TextureCounters counters; + return counters; +} + +void reportTextureCounters() { + static const bool profile = std::getenv("APS5_PROFILE_DRAW") != nullptr; + if (!profile) return; + auto& counters = TextureCounts(); + const auto nowMs = std::chrono::duration_cast(std::chrono::steady_clock::now().time_since_epoch()).count(); + auto last = counters.lastReport.load(); + if (nowMs - last < 10000 || !counters.lastReport.compare_exchange_strong(last, nowMs)) return; + const auto count = [](const std::atomic& value) { return static_cast(value.load(std::memory_order_relaxed)); }; + std::fprintf(stderr, "[textures] sampled created: %llu from storage images, %llu snapshots (%llu snapshot reads over recorded writes inside cachedTexture, %llu storage-path fallbacks); stage-A records %llu: %llu fast hits, %llu full lookups; storage images %llu hits, %llu created\n", count(counters.fromStorage), count(counters.snapshots), count(counters.pendingReads), count(counters.storageFallbacks), count(counters.records), count(counters.fastHits), count(counters.fastMisses), count(counters.storageHits), count(counters.storageCreated)); +} + +// What the sampled-texture lookups on this thread proved their returned objects current against, +// taken by object into the ShaderResources being built or revalidated (captureValidation), which +// then repeats the proof from write stamps on its next Revalidate. The lookups run inside one build +// on one thread; records nobody takes (a build that threw, a resident target looked up outside a +// build, a thread that never captures) are dropped by the next capture or by the bound below. +struct LookupRecord { + const void* object; + GuestTextureResource resource; + std::uint64_t bytes; + DccKeys keys; + std::uint64_t generation; + const StorageTexture* source; +}; + +thread_local std::vector lookupLog; + +void logLookup(const LookupRecord& record) { + // The bound drops the oldest half, never everything: one build's lookups (a few dozen at most) + // are the newest records, and a capture must find them even when the bound trips mid-build. + constexpr std::size_t bound = 1024; + if (lookupLog.size() >= bound) lookupLog.erase(lookupLog.begin(), lookupLog.begin() + bound / 2); + lookupLog.push_back(record); +} + +std::shared_ptr cachedStorageTexture(const Context& context, std::span words, const GuestTextureResource& resource, std::uint32_t mip, std::uint64_t guestBytes = 0); + +// Whether a sampled texture over `resource` can be a view of the surface's cached storage image +// instead of a CPU snapshot (see cachedTexture): the format has a storage form and is not block +// compressed, and the surface lives in host-imported memory, where the image uploads and refreshes +// GPU-direct (a snapshot of such a surface reads and compares its bytes on the CPU, waiting for +// the recorded work that wrote them). APS5_NO_SAMPLED_FROM_STORAGE=1 keeps every sampled texture a +// snapshot as before. +bool SampledFromStorageEligible(const Context& context, std::uint32_t format, std::uint64_t address, std::uint64_t guestBytes) { + static const bool disabled = std::getenv("APS5_NO_SAMPLED_FROM_STORAGE") != nullptr; + if (disabled || IsBlockCompressed(format) || !StorageFormatAvailable(context, format)) return false; + return HostImportCovers(context, address, static_cast(guestBytes)); +} + +bool SampledFromStorageEligible(const Context& context, const GuestTextureResource& resource, std::uint64_t guestBytes) { + return SampledFromStorageEligible(context, resource.format, resource.baseAddress, guestBytes); +} + +// Surfaces whose storage image could not be made (no writable committed pages, say): remembered +// so the sampled lookup does not throw and fall back on every use. A failure is keyed by the surface +// (address and size: the heap reuses addresses) and holds only while the guest mappings are what +// they were when it failed (the allocation generation): pages committed later, or another surface +// at the address, get a fresh attempt. +struct StorageFailures { + std::mutex mutex; + std::map, std::uint64_t> generations; +}; + +StorageFailures& StorageFailed() { + static StorageFailures failures; + return failures; +} + +// The cached storage image for a sampled descriptor, or null when it cannot be made (then the +// snapshot path serves the descriptor, as before). Every null return counts as a fallback. +std::shared_ptr sampledStorageSource(const Context& context, const GuestTextureResource& resource, std::uint64_t guestBytes) { + auto& counters = TextureCounts(); + auto& failures = StorageFailed(); + const auto surface = std::make_pair(resource.baseAddress, guestBytes); + const auto generation = GuestAllocations::GuestAllocationsGeneration_nid_postfix(); + { + std::lock_guard lock(failures.mutex); + if (const auto it = failures.generations.find(surface); it != failures.generations.end()) { + if (it->second == generation) { + counters.storageFallbacks.fetch_add(1, std::memory_order_relaxed); + return nullptr; + } + failures.generations.erase(it); + } + } + try { + return cachedStorageTexture(context, {}, resource, 0, guestBytes); + } catch (const std::exception& error) { + counters.storageFallbacks.fetch_add(1, std::memory_order_relaxed); + std::lock_guard lock(failures.mutex); + if (failures.generations.size() < 4096 && failures.generations.emplace(surface, generation).second) std::fprintf(stderr, "[textures] sampled texture 0x%llx (%ux%u format %u) keeps the snapshot path: %s\n", static_cast(resource.baseAddress), resource.width, resource.height, resource.format, error.what()); + return nullptr; + } +} + +// `guestBytes` is the surface size when the caller described the surface already (0: described here). +std::shared_ptr cachedTexture(const Context& context, std::span words, const GuestTextureResource& resource, VkComponentMapping components, std::uint64_t guestBytes = 0) { + static const bool disabled = std::getenv("APS5_NO_TEXTURE_CACHE") != nullptr; + if (guestBytes == 0) guestBytes = DescribeSurface(resource).guestBytes; + auto& counters = TextureCounts(); + const auto address = resource.baseAddress; + const auto bytes = static_cast(guestBytes); + // A storage image whose results for this surface (or for a mip chain containing it) are still on + // the GPU supplies the texture by a view of it; anything else needs those results in guest + // memory first. + auto source = StorageTexture::FindPending(address, guestBytes); + if (source != nullptr && !Texture::CanCopyFrom(*source, resource)) source.reset(); + // Otherwise a surface in host-imported memory is viewed through its cached storage image (made + // here when there is none): its refresh after a CPU or GPU write is a GPU-direct detile from the + // import, recorded behind the producer, so no bytes are read or compared on the CPU and nothing + // waits for the producer. A fast-cleared surface (keys) stays a snapshot: its texels are not + // read either, and the image's own descriptor may not carry the DCC address. + std::optional keys; + if (source == nullptr && SampledFromStorageEligible(context, resource, guestBytes)) { + keys = TextureClearKeys(resource, guestBytes); + if (*keys == DccKeys::Uncompressed) source = sampledStorageSource(context, resource, guestBytes); + } + // Pages of the surface written since they were last collected are stamped now, before any + // UnchangedSince (here and on a cache hit) looks at them: the checks only compare stamped blocks, + // so a write the game made since the last walk is invisible until a collect stamps its page. The + // walk is memoized per packet, so a texture viewed from a storage image repeats it for free. + auto generation = GuestMemory::CollectWrites(address, bytes); + if (source != nullptr && !GuestMemory::UnchangedSince(address, bytes, source->Generation())) { + // The CPU wrote the memory while results were pending: Refresh merges them the usual way. + source->Refresh(); + } + // Results stored to guest memory just now are stamped newer than `generation`; a snapshot read + // after them is current at the generation of a second (memoized) collect. The store marks the + // surface's DCC keys uncompressed, so the keys are read after it. + if (source == nullptr && StorageTexture::FlushPending(address, bytes, nullptr, "sampled texture")) { + generation = GuestMemory::CollectWrites(address, bytes); + keys.reset(); + } + if (!keys.has_value()) keys = TextureClearKeys(resource, guestBytes); + if (disabled) { + if (source != nullptr) return std::make_shared(context, source, resource, components); + std::vector snapshot(bytes); + ReadTextureSurface(resource, *keys, snapshot); + return std::make_shared(context, *context.detiler, resource, components, snapshot); + } + auto& cache = Textures(); + const auto key = MakeTextureKey(context.device, words, components); + std::lock_guard lock(cache.mutex); + if (auto it = findTexture(cache, key); it != cache.entries.end()) { + if (it->source != nullptr) { + // The view follows the storage image, whatever the GPU wrote to it since; guest memory + // written meanwhile is taken in by refreshing the image. A fast clear the image cannot + // see (keys, and no pending results to prefer) ends the view: a snapshot holds the clear. + if ((source == nullptr || source == it->source) && (source != nullptr || *keys == DccKeys::Uncompressed)) { + if (!GuestMemory::UnchangedSince(address, bytes, it->source->Generation())) it->source->Refresh(); + touchTexture(cache, it); + logLookup({it->texture.get(), resource, guestBytes, *keys, 0, it->source.get()}); + reportTextureCounters(); + return it->texture; + } + } else if (source == nullptr && it->bytes.size() == guestBytes && it->keys == *keys) { + // Unwritten pages need no comparison; partially resident textures compare only committed + // pages. The compare goes through the flush hook: it waits for recorded work over the + // surface (counted, and named for the [hooksync] line). + const auto equalsCommitted = [&] { + if (Recorder::SnapshotWriteOverlaps(address, bytes)) counters.pendingReads.fetch_add(1, std::memory_order_relaxed); + const GuestMemory::ReadSiteScope site(GuestMemory::ReadSite::TextureCompare); + return GuestMemory::EqualsCommitted(address, it->bytes); + }; + if (*keys != DccKeys::Uncompressed || GuestMemory::UnchangedSince(address, it->bytes.size(), it->generation) || equalsCommitted()) { + it->generation = generation; + touchTexture(cache, it); + logLookup({it->texture.get(), resource, guestBytes, *keys, generation, nullptr}); + reportTextureCounters(); + return it->texture; + } + } + eraseTexture(cache, it); + } + CachedTexture entry{key, address, std::vector(source != nullptr ? 0u : bytes), nullptr, *keys, generation}; + entry.accounted = guestBytes; + if (source != nullptr) { + entry.source = source; + entry.sourceVersion = source->Version(); + entry.texture = std::make_shared(context, source, resource, components); + counters.fromStorage.fetch_add(1, std::memory_order_relaxed); + } else { + // Snapshot before the upload so a write racing with it is caught by the next comparison. + if (*keys == DccKeys::Uncompressed && Recorder::SnapshotWriteOverlaps(address, bytes)) counters.pendingReads.fetch_add(1, std::memory_order_relaxed); + ReadTextureSurface(resource, *keys, entry.bytes); + static const bool traceTextures = std::getenv("APS5_TRACE_TEXTURES") != nullptr; + if (traceTextures) { + std::size_t nonzero = 0; + for (std::size_t i = 0; i < entry.bytes.size(); i += 64) nonzero += entry.bytes[i] != std::byte{0}; + std::fprintf(stderr, "[texture] 0x%llx %ux%u format %u tile %d: %zu of %zu sampled bytes nonzero\n", static_cast(resource.baseAddress), resource.width, resource.height, resource.format, static_cast(resource.tileMode), nonzero, entry.bytes.size() / 64); + } + entry.texture = std::make_shared(context, *context.detiler, resource, components, entry.bytes); + counters.snapshots.fetch_add(1, std::memory_order_relaxed); + } + constexpr std::uint64_t budget = 2048ull << 20u; + while (!cache.entries.empty() && cache.bytes + entry.accounted > budget) eraseTexture(cache, std::prev(cache.entries.end())); + cache.bytes += entry.accounted; + auto texture = entry.texture; + logLookup({texture.get(), resource, guestBytes, *keys, generation, source.get()}); + cache.entries.push_front(std::move(entry)); + cache.index[key] = cache.entries.begin(); + reportTextureCounters(); + return texture; +} + +// Storage images stay on the GPU between dispatches: while guest memory still holds what an image was +// last uploaded from or written back as, its next use skips the upload and detile. +struct StorageKey { + VkDevice device; + std::array words; + bool operator==(const StorageKey&) const = default; +}; + +struct StorageKeyHash { + std::size_t operator()(const StorageKey& key) const noexcept { + return static_cast(hashWords(14695981039346656037ull ^ reinterpret_cast(key.device), key.words.data(), key.words.size())); + } +}; + +struct CachedStorageTexture { + StorageKey key; + std::uint32_t mip; + std::shared_ptr texture; +}; + +// As TextureCache: use order with a hash index by key, plus one by image for StorageImageCached. +struct StorageTextureCache { + std::mutex mutex; + std::list entries; + std::unordered_map::iterator, StorageKeyHash> index; + std::unordered_map::iterator> byImage; + std::uint64_t bytes = 0; +}; + +StorageTextureCache& StorageTextures() { + static StorageTextureCache cache; + return cache; +} + +std::list::iterator findStorage(StorageTextureCache& cache, const StorageKey& key) { + if (TextureHashEnabled()) { + const auto found = cache.index.find(key); + return found == cache.index.end() ? cache.entries.end() : found->second; + } + for (auto it = cache.entries.begin(); it != cache.entries.end(); ++it) { + if (it->key == key) return it; + } + return cache.entries.end(); +} + +std::list::iterator findStorageByImage(StorageTextureCache& cache, VkDevice device, const StorageTexture* image) { + if (TextureHashEnabled()) { + const auto found = cache.byImage.find(image); + return found == cache.byImage.end() || found->second->key.device != device ? cache.entries.end() : found->second; + } + for (auto it = cache.entries.begin(); it != cache.entries.end(); ++it) { + if (it->key.device == device && it->texture.get() == image) return it; + } + return cache.entries.end(); +} + +// Evicts an entry: its pending results go to guest memory first (the image may die with the entry). +void evictStorage(StorageTextureCache& cache, std::list::iterator it) { + it->texture->Flush(); + cache.bytes -= it->texture->GuestBytes(); + cache.index.erase(it->key); + cache.byImage.erase(it->texture.get()); + cache.entries.erase(it); +} + +// Storage images are shared by every descriptor of one surface (address, extent, layers, format, tile +// mode): the image holds the whole mip chain, and render targets in the same memory attach to it. +std::array SurfaceKey(const Context& context, const GuestTextureResource& resource) { + // Guest formats that store in the same Vulkan format share the image (views carry the difference). + return {static_cast(resource.baseAddress), static_cast(resource.baseAddress >> 32u), resource.width, resource.height, (resource.depthOrLastArray << 16u) | (resource.mipCount & 0xffffu), (static_cast(resource.tileMode) << 12u) | (static_cast(resource.dimension) << 20u), resource.baseArray, static_cast(StorageFormatForGuest(context, resource.format))}; +} + +// `guestBytes` is the surface size when the caller described the surface already (0: described here). +std::shared_ptr cachedStorageTexture(const Context& context, std::span words, const GuestTextureResource& resource, std::uint32_t mip, std::uint64_t guestBytes) { + static const bool disabled = std::getenv("APS5_NO_TEXTURE_CACHE") != nullptr; + if (disabled) return std::make_shared(context, *context.detiler, resource, mip); + static_cast(words); + auto& counters = TextureCounts(); + const StorageKey key{context.device, SurfaceKey(context, resource)}; + auto& cache = StorageTextures(); + std::lock_guard lock(cache.mutex); + if (auto it = findStorage(cache, key); it != cache.entries.end()) { + it->texture->Refresh(); + cache.entries.splice(cache.entries.begin(), cache.entries, it); + counters.storageHits.fetch_add(1, std::memory_order_relaxed); + return it->texture; + } + // Results other images hold over this memory reach it before the new image reads it: a hit's + // Refresh flushes them, the constructor's upload does not, and a GPU-direct upload reads the + // import buffer without the flush hook. The flush is recorded ahead of the upload in the batch. + if (guestBytes == 0) guestBytes = DescribeSurface(resource).guestBytes; + StorageTexture::FlushPending(resource.baseAddress, static_cast(guestBytes), nullptr, "storage image creation"); + CachedStorageTexture entry{key, mip, std::make_shared(context, *context.detiler, resource, mip)}; + constexpr std::uint64_t budget = 2048ull << 20u; + while (!cache.entries.empty() && cache.bytes + entry.texture->GuestBytes() > budget) evictStorage(cache, std::prev(cache.entries.end())); + cache.bytes += entry.texture->GuestBytes(); + auto texture = entry.texture; + cache.entries.push_front(std::move(entry)); + cache.index[key] = cache.entries.begin(); + cache.byImage[texture.get()] = cache.entries.begin(); + counters.storageCreated.fetch_add(1, std::memory_order_relaxed); + return texture; +} + +// Whether `image` is still the storage cache's image of its surface, i.e. what a lookup would return +// (an evicted image next to a newer one of the same memory must not be reused: two images of one +// surface would hold results). A use through the fast Revalidate counts as a use for the cache's +// eviction order, as a lookup would. +bool StorageImageCached(const Context& context, const StorageTexture* image) { + auto& cache = StorageTextures(); + std::lock_guard lock(cache.mutex); + const auto it = findStorageByImage(cache, context.device, image); + if (it == cache.entries.end()) return false; + cache.entries.splice(cache.entries.begin(), cache.entries, it); + return true; +} + +} + +std::shared_ptr CachedStorageSurface(const Context& context, const GuestTextureResource& resource) { + return cachedStorageTexture(context, {}, resource, 0); +} + +namespace { + const char* roleName(ShaderRecompiler::DescriptorRole role) { switch (role) { case ShaderRecompiler::DescriptorRole::GuestBuffers: return "GuestBuffers"; @@ -55,6 +510,21 @@ const char* kindName(ShaderRecompiler::DescriptorKind kind) { throw std::runtime_error("AGC graphics: unknown descriptor kind"); } +// Consecutive identical storage descriptors address successive mips of one texture (dynamic-mip +// storage writes), so the second and later ones share the first one's image lookup. +// APS5_NO_STORAGE_DEDUPE=1 looks each one up (and refreshes the image) separately as before. +bool StorageDedupeEnabled() { + static const bool disabled = std::getenv("APS5_NO_STORAGE_DEDUPE") != nullptr; + return !disabled; +} + +// Whether storage element `element` of `binding` repeats the previous element's eight words. +bool SameAsPreviousStorageElement(const ShaderRecompiler::DescriptorBinding& binding, std::uint32_t element) { + if (element == 0) return false; + const auto words = binding.guestDescriptor.begin() + static_cast(element) * 8u; + return std::equal(words, words + 8, words - 8); +} + } ShaderResources::ShaderResources(const Context& context, const ShaderRecompiler::RecompileResult& vertex, const ShaderRecompiler::RecompileResult& fragment, const ColorTarget& target, std::uint64_t indexAddress, std::size_t indexBytes) : ShaderResources(context, std::array{{{ShaderRecompiler::ShaderStage::Vertex, &vertex, 0}, {ShaderRecompiler::ShaderStage::Fragment, &fragment, static_cast(vertex.pushConstants.size())}}}, target, indexAddress, indexBytes) {} @@ -64,19 +534,130 @@ ShaderResources::ShaderResources(const Context& context, std::span snapshots) : context(context), guestMemory(context) { +ShaderResources::ShaderResources(const Context& context, const CompiledShader& compute, std::span snapshots) : ShaderResources(context, compute, snapshots, false) {} + +namespace { + +// Whether a compute stage maps registered guest memory through BDA tables (see prepareAddressBindings). +bool UsesAddressTables(const CompiledShader& compute) { + Require(compute.program != nullptr, "missing compiled shader"); + return std::any_of(compute.program->bindings.begin(), compute.program->bindings.end(), [](const auto& binding) { return binding.role == ShaderRecompiler::DescriptorRole::BdaPagetable; }); +} + +} + +ShaderResources::ShaderResources(const Context& context, const CompiledShader& compute, std::span snapshots, bool deferred) : context(context), guestMemory(context), deferredCompute(compute), deferredSnapshots(snapshots) { Require(compute.stage == ShaderRecompiler::ShaderStage::Compute, "compute resources require a compute shader"); - prepareAddressBindings(std::span(&compute, 1), snapshots); - build(std::span(&compute, 1), nullptr, 0, 0); + const std::span shaders(&deferredCompute, 1); + if (!deferred) { + prepareAddressBindings(shaders, snapshots); + build(shaders, nullptr, 0, 0); + forgetDeferredInputs(); + return; + } + // Address-based shaders pin and mirror registered memory in prepareAddressBindings (reconciling + // imports retires buffers to the recorder, refreshing mirrors waits for recorded work): that is + // device-lock work, so their whole build waits for Complete(). Without tables the call only + // checks the bindings. + lockedBuild = UsesAddressTables(compute); + if (lockedBuild) return; + unlockedPrepare = true; + prepareAddressBindings(shaders, snapshots); + buildPrepare(shaders, nullptr, 0, 0); +} + +void ShaderResources::Complete() { + Require(!completed, "shader resources were already completed"); + const std::span shaders(&deferredCompute, 1); + if (lockedBuild) { + prepareAddressBindings(shaders, deferredSnapshots); + buildPrepare(shaders, nullptr, 0, 0); + } + buildComplete(); + forgetDeferredInputs(); +} + +void ShaderResources::forgetDeferredInputs() { + // The compiled shader and the captured regions belong to the dispatch, which returns while this + // object lives on in the resource cache and the recorder: nothing may reach for them after the + // build, so they are dropped rather than left dangling. + deferredCompute = {}; + deferredSnapshots = {}; } void ShaderResources::build(std::span shaders, const ColorTarget* target, std::uint64_t indexAddress, std::size_t indexBytes) { - PerformanceTimer timing("Graphics.ShaderResources"); + buildPrepare(shaders, target, indexAddress, indexBytes); + buildComplete(); +} + +namespace { + +// APS5_PROFILE_DRAW: the [resources] phase totals, under a mutex because stage A of several builds +// runs at once. +struct BuildProfile { + std::mutex mutex; + std::map phaseTotals; + std::uint64_t builds = 0; +}; + +BuildProfile& Builds() { + static BuildProfile profile; + return profile; +} + +bool BuildProfiled() { + static const bool profile = std::getenv("APS5_PROFILE_DRAW") != nullptr; + return profile; +} + +void addBuildPhase(const char* name, double ms) { + auto& profile = Builds(); + std::lock_guard lock(profile.mutex); + profile.phaseTotals[name] += ms; +} + +// APS5_PROFILE_DRAW: guest buffer elements bound by descriptor (addGuestBuffer) and how many the +// recompiler proved read-only, plus the pending-write notes their uses skipped (MarkGpuWrites), +// printed as [buffers] every 10 s from MarkGpuWrites. Cumulative. +struct BufferWriteCounters { + std::atomic elements{0}; + std::atomic readOnly{0}; + std::atomic notesSkipped{0}; + std::atomic lastReport{0}; +}; + +BufferWriteCounters& BufferWrites() { + static BufferWriteCounters counters; + return counters; +} + +} + +double ShaderResources::phase(const char* name) { + if (!BuildProfiled()) return 0.0; + const auto now = std::chrono::steady_clock::now(); + const auto ms = std::chrono::duration(now - phaseStart).count(); + addBuildPhase(name, ms); + if (ms > 50) std::fprintf(stderr, "[resources] %s took %.0f ms (%zu textures, %zu storage images, %zu buffers, bda %d)\n", name, ms, textures.size(), storageTextures.size(), allocations.size(), usesBda ? 1 : 0); + phaseStart = now; + return ms; +} + +void ShaderResources::buildPrepare(std::span shaders, const ColorTarget* target, std::uint64_t indexAddress, std::size_t indexBytes) { + const auto stageStart = std::chrono::steady_clock::now(); + phaseStart = stageStart; + if (BuildProfiled()) { + auto& profile = Builds(); + std::lock_guard lock(profile.mutex); + if (++profile.builds % 1000 == 0) { + std::string report; + for (const auto& [name, ms] : profile.phaseTotals) report += " " + name + "=" + std::to_string(static_cast(ms)) + "ms"; + std::fprintf(stderr, "[resources] %llu builds, phase totals:%s\n", static_cast(profile.builds), report.c_str()); + } + } try { Require(!shaders.empty() && context.limits.maxBoundDescriptorSets >= 1, "shader descriptor set exceeds device limits"); - std::vector bindings; std::set occupied; - std::uint64_t storageBuffers = 0; for (const auto& shader : shaders) { Require(shader.program != nullptr, "missing compiled shader"); const VkShaderStageFlags flags = VulkanStage(shader.stage); @@ -88,7 +669,7 @@ void ShaderResources::build(std::span shaders, const Color const bool bufferRole = addressRole || binding.role == ShaderRecompiler::DescriptorRole::GuestBuffers || binding.role == ShaderRecompiler::DescriptorRole::ShaderData || binding.role == ShaderRecompiler::DescriptorRole::FlattenedSrt; const bool imageRole = binding.role == ShaderRecompiler::DescriptorRole::GuestImages || binding.role == ShaderRecompiler::DescriptorRole::GuestSamplers; if (imageRole) { - addImageBinding(binding, flags, bindings); + addImageBinding(binding, flags); continue; } Require(bufferRole, std::string("unsupported descriptor role ") + roleName(binding.role)); @@ -101,7 +682,12 @@ void ShaderResources::build(std::span shaders, const Color Binding item{{binding.binding, VK_DESCRIPTOR_TYPE_STORAGE_BUFFER, binding.count, flags, nullptr}, {}}; if (binding.role == ShaderRecompiler::DescriptorRole::GuestBuffers) { Require(binding.guestDescriptor.size() == static_cast(binding.count) * 4, "guest buffer descriptor must contain four DWORDs per array element"); - for (std::uint32_t element = 0; element < binding.count; ++element) item.allocations.push_back(addGuestBuffer(std::span(binding.guestDescriptor).subspan(static_cast(element) * 4, 4), target, indexAddress, indexBytes)); + for (std::uint32_t element = 0; element < binding.count; ++element) { + // An element the recompiler did not classify (a producer without the + // vector) counts as written, like imageWritten below. + const bool written = element >= binding.bufferWritten.size() || binding.bufferWritten[element]; + item.allocations.push_back(addGuestBuffer(std::span(binding.guestDescriptor).subspan(static_cast(element) * 4, 4), target, indexAddress, indexBytes, written)); + } } else if (addressRole) { item.allocations.push_back(allocations.size()); allocations.push_back({0, 0, false, nullptr, binding.role}); @@ -114,66 +700,662 @@ void ShaderResources::build(std::span shaders, const Color } } Require(storageBuffers <= context.limits.maxDescriptorSetStorageBuffers, "pipeline descriptors exceed device limits"); - timing.Mark("bindings"); - guestMemory.Upload(usesBda); - if (usesBda) bda = std::make_unique(context, guestMemory); - else if (usesFaultBuffer) bda = std::make_unique(context); - timing.Mark("memory_upload"); + timing.bindingsMs = phase("bindings"); + // For every build, locked ones included: their stage B then takes the fast path too, and the + // collects cost the same wherever they run. + if (precollectImages()) phase("precollect"); + guestMemory.UploadPrepare(usesBda); + timing.uploadMs = phase("guest memory upload"); std::vector description; for (const auto& binding : bindings) { description.push_back(binding.layout); layoutKey.insert(layoutKey.end(), {binding.layout.binding, static_cast(binding.layout.descriptorType), binding.layout.descriptorCount, binding.layout.stageFlags}); } - std::vector sizes; - if (storageBuffers != 0) sizes.push_back({VK_DESCRIPTOR_TYPE_STORAGE_BUFFER, static_cast(storageBuffers)}); - if (!textures.empty()) sizes.push_back({VK_DESCRIPTOR_TYPE_SAMPLED_IMAGE, static_cast(textures.size())}); - if (!samplers.empty()) sizes.push_back({VK_DESCRIPTOR_TYPE_SAMPLER, static_cast(samplers.size())}); - if (!context.descriptorCache) context.descriptorCache = std::make_shared(); - descriptors = context.descriptorCache->Take(layoutKey); - if (!descriptors) descriptors = std::make_unique(context, layoutKey, description, sizes); - _layout = descriptors->Layout(); - _set = descriptors->Set(); - timing.Mark("descriptor_acquire"); - std::vector buffers; - std::vector images; - std::vector writes; - buffers.reserve(allocations.size()); - images.reserve(textures.size() + samplers.size()); - writes.reserve(bindings.size()); - for (const auto& binding : bindings) { - const auto bufferOffset = buffers.size(); - const auto imageOffset = images.size(); - VkWriteDescriptorSet write{VK_STRUCTURE_TYPE_WRITE_DESCRIPTOR_SET}; - write.dstSet = _set; - write.dstBinding = binding.layout.binding; - write.descriptorCount = binding.layout.descriptorCount; - write.descriptorType = binding.layout.descriptorType; - switch (binding.layout.descriptorType) { - case VK_DESCRIPTOR_TYPE_STORAGE_BUFFER: - for (const auto index : binding.allocations) buffers.push_back(descriptor(allocations[index])); - write.pBufferInfo = buffers.data() + bufferOffset; - break; - case VK_DESCRIPTOR_TYPE_SAMPLED_IMAGE: - for (const auto index : binding.imageAllocations) images.push_back({VK_NULL_HANDLE, textures[index]->View(), VK_IMAGE_LAYOUT_SHADER_READ_ONLY_OPTIMAL}); - write.pImageInfo = images.data() + imageOffset; - break; - case VK_DESCRIPTOR_TYPE_SAMPLER: - for (const auto index : binding.imageAllocations) images.push_back({samplers[index]->Handle(), VK_NULL_HANDLE, VK_IMAGE_LAYOUT_UNDEFINED}); - write.pImageInfo = images.data() + imageOffset; - break; - default: throw std::runtime_error("AGC graphics: ShaderResources encountered an unknown descriptor type while writing the descriptor set"); - } - writes.push_back(write); + static const bool noLayoutCache = std::getenv("APS5_NO_LAYOUT_CACHE") != nullptr; + static const bool noPoolCache = std::getenv("APS5_NO_POOL_CACHE") != nullptr; + if (context.descriptorCache != nullptr && !noLayoutCache) { + _layout = context.descriptorCache->Layout(layoutKey, description); + } else { + VkDescriptorSetLayoutCreateInfo info{VK_STRUCTURE_TYPE_DESCRIPTOR_SET_LAYOUT_CREATE_INFO}; + info.bindingCount = static_cast(description.size()); + info.pBindings = description.data(); + Check(context.Function("vkCreateDescriptorSetLayout")(context.device, &info, nullptr, &_layout), "vkCreateDescriptorSetLayout"); + ownsLayout = true; + } + if (!bindings.empty()) { + // The set is sized from the plan: every image element becomes exactly one descriptor + // when stage B looks it up. + std::vector sizes; + if (storageBuffers != 0) sizes.push_back({VK_DESCRIPTOR_TYPE_STORAGE_BUFFER, static_cast(storageBuffers)}); + if (plannedSampledImages != 0) sizes.push_back({VK_DESCRIPTOR_TYPE_SAMPLED_IMAGE, plannedSampledImages}); + if (plannedStorageImages != 0) sizes.push_back({VK_DESCRIPTOR_TYPE_STORAGE_IMAGE, plannedStorageImages}); + if (!samplers.empty()) sizes.push_back({VK_DESCRIPTOR_TYPE_SAMPLER, static_cast(samplers.size())}); + if (context.descriptorCache != nullptr && !noPoolCache) { + const auto allocated = context.descriptorCache->Allocate(_layout, sizes); + _set = allocated.set; + cachePool = allocated.pool; + } + if (_set == VK_NULL_HANDLE) { + VkDescriptorPoolCreateInfo poolInfo{VK_STRUCTURE_TYPE_DESCRIPTOR_POOL_CREATE_INFO}; + poolInfo.maxSets = 1; + poolInfo.poolSizeCount = static_cast(sizes.size()); + poolInfo.pPoolSizes = sizes.data(); + Check(context.Function("vkCreateDescriptorPool")(context.device, &poolInfo, nullptr, &pool), "vkCreateDescriptorPool"); + VkDescriptorSetAllocateInfo allocation{VK_STRUCTURE_TYPE_DESCRIPTOR_SET_ALLOCATE_INFO}; + allocation.descriptorPool = pool; + allocation.descriptorSetCount = 1; + allocation.pSetLayouts = &_layout; + Check(context.Function("vkAllocateDescriptorSets")(context.device, &allocation, &_set), "vkAllocateDescriptorSets"); + } + } + timing.descriptorsMs = phase("descriptors"); + if (BuildProfiled()) { + timing.prepareMs = std::chrono::duration(std::chrono::steady_clock::now() - stageStart).count(); + addBuildPhase("stage A", timing.prepareMs); } - if (!writes.empty()) context.Function("vkUpdateDescriptorSets")(context.device, static_cast(writes.size()), writes.data(), 0, nullptr); - timing.Mark("descriptor_update"); } catch (...) { release(); throw; } } -std::size_t ShaderResources::addGuestBuffer(std::span words, const ColorTarget* target, std::uint64_t indexAddress, std::size_t indexBytes) { +void ShaderResources::buildComplete() { + const auto stageStart = std::chrono::steady_clock::now(); + phaseStart = stageStart; + try { + // The lookups run in plan order: Revalidate walks the bindings the same way, and consecutive + // storage elements of one mip chain share the previous element's image. + for (const auto& deferred : deferredImages) resolveImageBinding(*deferred.binding, bindings[deferred.index]); + deferredImages.clear(); + // The records served their purpose: each holds the cache entry's objects as of stage A, + // which would otherwise keep a replaced texture or an evicted storage image (and its device + // memory) alive, outside the caches' budgets, for as long as this object is cached. + imageRecords.clear(); + imageRecords.shrink_to_fit(); + nextImageRecord = 0; + Require(textures.size() == plannedSampledImages && storageTextures.size() == plannedStorageImages, "image lookups disagree with the descriptor plan"); + timing.bindingsMs += phase("images"); + guestMemory.UploadFinish(usesBda); + timing.uploadMs += phase("guest memory upload"); + if (usesBda) bda = std::make_unique(context, guestMemory); + else if (usesFaultBuffer) bda = std::make_unique(context); + phase("bda"); + if (_set != VK_NULL_HANDLE) { + // One update call for the whole set: the info arrays are sized up front so every write's + // pointer into them stays valid until the call. + std::size_t bufferCount = 0; + std::size_t imageCount = 0; + for (const auto& binding : bindings) { + bufferCount += binding.allocations.size(); + imageCount += binding.imageAllocations.size(); + } + std::vector buffers; + std::vector images; + buffers.reserve(bufferCount); + images.reserve(imageCount); + std::vector writes; + writes.reserve(bindings.size()); + for (const auto& binding : bindings) { + VkWriteDescriptorSet write{VK_STRUCTURE_TYPE_WRITE_DESCRIPTOR_SET}; + write.dstSet = _set; + write.dstBinding = binding.layout.binding; + write.descriptorCount = binding.layout.descriptorCount; + write.descriptorType = binding.layout.descriptorType; + switch (binding.layout.descriptorType) { + case VK_DESCRIPTOR_TYPE_STORAGE_BUFFER: + write.pBufferInfo = buffers.data() + buffers.size(); + for (const auto index : binding.allocations) buffers.push_back(descriptor(allocations[index])); + break; + case VK_DESCRIPTOR_TYPE_SAMPLED_IMAGE: + write.pImageInfo = images.data() + images.size(); + for (const auto index : binding.imageAllocations) images.push_back({VK_NULL_HANDLE, textures[index]->View(), textures[index]->Layout()}); + break; + case VK_DESCRIPTOR_TYPE_STORAGE_IMAGE: + write.pImageInfo = images.data() + images.size(); + for (const auto index : binding.imageAllocations) images.push_back({VK_NULL_HANDLE, storageTextures[index]->View(storageMips[index]), VK_IMAGE_LAYOUT_GENERAL}); + break; + case VK_DESCRIPTOR_TYPE_SAMPLER: + write.pImageInfo = images.data() + images.size(); + for (const auto index : binding.imageAllocations) images.push_back({samplers[index]->Handle(), VK_NULL_HANDLE, VK_IMAGE_LAYOUT_UNDEFINED}); + break; + default: throw std::runtime_error("AGC graphics: ShaderResources encountered an unknown descriptor type while writing the descriptor set"); + } + writes.push_back(write); + } + context.Function("vkUpdateDescriptorSets")(context.device, static_cast(writes.size()), writes.data(), 0, nullptr); + } + timing.descriptorsMs += phase("descriptors"); + noteReusable(); + completed = true; + if (BuildProfiled()) { + timing.completeMs = std::chrono::duration(std::chrono::steady_clock::now() - stageStart).count(); + addBuildPhase("stage B", timing.completeMs); + reportDescriptorCaches(); + } + } catch (...) { + release(); + throw; + } +} + +// Whether this build can serve later dispatches of the same content (see ContentKey): nothing to do +// once the GPU completed (no BDA, no copied written buffers, no lease) and every guest buffer bound +// in place through a host import whose identity is recorded for Revalidate. +void ShaderResources::noteReusable() { + // Taken whether or not the object turns out reusable: the lookups' records are this build's. + captureValidation(); + reusable = false; + directRegions.clear(); + if (NeedsCompletion() || HoldsLease()) return; + const auto regions = guestMemory.DirectRegions(); + if (!regions.has_value()) return; + for (const auto& [begin, end] : *regions) { + // No reconcile here: the upload just took these imports, and the set is about to be recorded + // against them. + auto serial = HostImportSerial(context, begin, static_cast(end - begin), false); + // Read-only image mirrors (exe ranges) are as stable as imports; their serials have the top + // bit set, so the two spaces never collide. + if (serial == 0) serial = ImageMirrorSerial(context, begin, static_cast(end - begin)); + if (serial == 0) return; + directRegions.push_back({begin, end, serial}); + } + reusable = true; +} + +// APS5_PROFILE_DRAW: the per-device descriptor caches' counters, every 10 s. +void ShaderResources::reportDescriptorCaches() const { + static std::mutex reportMutex; + static auto lastReport = std::chrono::steady_clock::now(); + const auto now = std::chrono::steady_clock::now(); + std::lock_guard lock(reportMutex); + if (now - lastReport < std::chrono::seconds(10)) return; + lastReport = now; + const auto descriptors = context.descriptorCache != nullptr ? context.descriptorCache->Counters() : DescriptorCache::Stats{}; + const auto samplerHits = context.samplerCache != nullptr ? context.samplerCache->Hits() : 0; + const auto samplerMisses = context.samplerCache != nullptr ? context.samplerCache->Misses() : 0; + std::fprintf(stderr, "[descriptors] layouts %llu hits / %llu created, sets %llu from %llu pools, samplers %llu hits / %llu created\n", static_cast(descriptors.layoutHits), static_cast(descriptors.layoutMisses), static_cast(descriptors.sets), static_cast(descriptors.pools), static_cast(samplerHits), static_cast(samplerMisses)); +} + +std::vector ShaderResources::ContentKey(const CompiledShader& shader) { + Require(shader.program != nullptr, "missing compiled shader"); + const auto& program = *shader.program; + std::vector key; + key.reserve(8 + program.bindings.size() * 12); + key.push_back(static_cast(shader.stage)); + key.push_back(static_cast(program.variantId)); + key.push_back(static_cast(program.variantId >> 32u)); + key.push_back(static_cast(program.bindings.size())); + const auto packBits = [&](const std::vector& bits) { + key.push_back(static_cast(bits.size())); + std::uint32_t word = 0; + for (std::size_t i = 0; i < bits.size(); ++i) { + if (bits[i]) word |= 1u << (i % 32u); + if (i % 32u == 31u || i + 1 == bits.size()) { + key.push_back(word); + word = 0; + } + } + }; + for (const auto& binding : program.bindings) { + key.insert(key.end(), {static_cast(binding.kind), static_cast(binding.role), binding.descriptorSet, binding.binding, binding.count, binding.readOnly ? 1u : 0u, binding.imageShape.has_value() ? static_cast(*binding.imageShape) + 1u : 0u, static_cast(binding.guestDescriptor.size())}); + key.insert(key.end(), binding.guestDescriptor.begin(), binding.guestDescriptor.end()); + packBits(binding.imageWritten); + packBits(binding.samplerDepthCompare); + // Read-only elements are bound without a write set: an object built for one written set + // must not serve a build with another (the variant implies it, this makes it explicit). + packBits(binding.bufferWritten); + } + return key; +} + +namespace { + +// APS5_PROFILE_DRAW: Revalidate outcomes and time, printed as [rescache] every 10 s next to the +// device's hit/miss line: failed = the object could not be reused (whichever image path it took); +// of the reused ones, fast = every image proved current from stamps, full = the lookups were repeated. +struct RevalidateProfile { + std::atomic calls{0}; + std::atomic nanoseconds{0}; + std::atomic fast{0}; + std::atomic full{0}; + std::atomic failed{0}; + std::atomic lastReport{0}; +}; + +RevalidateProfile& Revalidations() { + static RevalidateProfile profile; + return profile; +} + +void countRevalidate(bool fast, bool ok, std::chrono::steady_clock::time_point start) { + auto& profile = Revalidations(); + const auto now = std::chrono::steady_clock::now(); + profile.calls.fetch_add(1, std::memory_order_relaxed); + profile.nanoseconds.fetch_add(static_cast(std::chrono::duration_cast(now - start).count()), std::memory_order_relaxed); + (!ok ? profile.failed : fast ? profile.fast : profile.full).fetch_add(1, std::memory_order_relaxed); + const auto nowMs = std::chrono::duration_cast(now.time_since_epoch()).count(); + auto last = profile.lastReport.load(); + if (nowMs - last < 10000 || !profile.lastReport.compare_exchange_strong(last, nowMs)) return; + std::fprintf(stderr, "[rescache] revalidate %llu calls %.1f ms: fast %llu, full %llu, failed %llu\n", static_cast(profile.calls.load()), profile.nanoseconds.load() / 1e6, static_cast(profile.fast.load()), static_cast(profile.full.load()), static_cast(profile.failed.load())); +} + +} + +// Turns the lookups' records into this object's per-texture validation records (see +// ValidatedSurface); the last record of an object is the freshest. +void ShaderResources::captureValidation() { + const auto record = [](const void* object) -> const LookupRecord* { + for (auto it = lookupLog.rbegin(); it != lookupLog.rend(); ++it) { + if (it->object == object) return &*it; + } + return nullptr; + }; + validatedTextures.assign(textures.size(), {}); + for (std::size_t i = 0; i < textures.size(); ++i) { + if (const auto* found = record(textures[i].get())) validatedTextures[i] = {found->resource, found->bytes, found->keys, found->generation, 0, found->source, true}; + } + lookupLog.clear(); +} + +// Proves every texture and storage image still current without repeating its lookup: exactly the +// "unchanged" branches of cachedTexture and StorageTexture::Refresh, evaluated from write stamps, +// the pending-results registry and the DCC keys. Per element: collect first (stamps come only from +// collects, and the generation a record moves to must predate the checks, so a write landing between +// them is stamped newer: the rule of Driver's dispatch cache), then no other image may have results +// pending over the memory (a lookup would flush them into it), a storage image must still be the +// cache's image of its surface, the memory must be unchanged since the object's content matched it, +// and the DCC keys must be what the content was made under whenever the surface has any (a fast +// clear touches only the keys). Anything else, including a failed collect (memory outside the arena, +// uncommitted pages), is left to the full walk. +bool ShaderResources::fastRevalidate() { + if (validatedTextures.size() != textures.size()) return false; + for (std::size_t i = 0; i < textures.size(); ++i) { + auto& surface = validatedTextures[i]; + if (!surface.valid) return false; + const auto address = surface.resource.baseAddress; + const auto bytes = static_cast(surface.bytes); + surface.collected = GuestMemory::CollectWrites(address, bytes); + if (surface.collected == 0) return false; + if (PendingStorageOverlaps(address, bytes, surface.source)) return false; + if (surface.source != nullptr) { + // The view follows the image: it needs the image current with guest memory, as the + // lookup's Refresh would make it. + if (!StorageImageCached(context, surface.source) || !GuestMemory::UnchangedSince(address, bytes, surface.source->Generation())) return false; + } else if (!GuestMemory::UnchangedSince(address, bytes, surface.generation)) { + return false; + } + if (surface.resource.dccAddress != 0 && TextureClearKeys(surface.resource, surface.bytes) != surface.keys) return false; + // A view made under fast-clear keys stays one only while its image still has results pending + // over the surface (the lookup prefers them to the clear); once they are flushed the clear + // the image cannot see wins and the lookup makes a snapshot (cachedTexture's hit rule). + if (surface.source != nullptr && surface.keys != DccKeys::Uncompressed && StorageTexture::FindPending(address, surface.bytes).get() != surface.source) return false; + } + for (std::size_t i = 0; i < storageTextures.size(); ++i) { + // Consecutive elements of one mip chain share the image (see addStorageImageBinding). + if (i != 0 && storageTextures[i] == storageTextures[i - 1]) continue; + const auto* image = storageTextures[i].get(); + if (image == nullptr) return false; + // StorageTexture::Refresh checks the image's own surface, not the element's descriptor (one + // image serves every descriptor of its memory, with or without a DCC address), and compares + // its keys with the keys it was uploaded under, which only the image knows (they move to + // Uncompressed on every write-back; a fast clear after it puts the key bytes back to what a + // record saw while the image holds last frame's results), so a surface with keys is left to + // the full walk. Without a DCC address both sides of that compare are Uncompressed whatever + // happened, and the memory checks below are Refresh's. + const auto& own = image->Descriptor(); + if (own.dccAddress != 0) return false; + const auto address = own.baseAddress; + const auto bytes = static_cast(image->GuestBytes()); + if (GuestMemory::CollectWrites(address, bytes) == 0) return false; + if (PendingStorageOverlaps(address, bytes, image)) return false; + if (!StorageImageCached(context, image)) return false; + if (!GuestMemory::UnchangedSince(address, bytes, image->Generation())) return false; + } + // Every check passed: snapshot textures are current at the collects issued before them. Storage + // images keep their own generation: nothing was stamped above it, so the blocks a later write + // stamps are the same set either way. + for (auto& surface : validatedTextures) { + if (surface.source == nullptr) surface.generation = surface.collected; + } + return true; +} + +bool ShaderResources::Revalidate(std::span shaders) { + if (!reusable || shaders.empty()) return false; + static const bool profile = std::getenv("APS5_PROFILE_DRAW") != nullptr; + // APS5_NO_FAST_REVALIDATE=1 always repeats the lookups. + static const bool noFast = std::getenv("APS5_NO_FAST_REVALIDATE") != nullptr; + const auto start = profile ? std::chrono::steady_clock::now() : std::chrono::steady_clock::time_point{}; + const auto finish = [&](bool fast, bool ok) { + if (profile) countRevalidate(fast, ok, start); + return ok; + }; + // (1) Textures and storage images, from stamps when every element allows it (fastRevalidate), + // else by the same lookups as the build (which refresh or replace them as guest memory changed): + // they must hand back the very objects the set's views belong to. The build appended them stage + // by stage, binding by binding, so the walk repeats that order. + const bool fast = !noFast && fastRevalidate(); + if (!fast) { + lookupLog.clear(); + std::size_t textureIndex = 0; + std::size_t storageIndex = 0; + for (const auto& shader : shaders) { + if (shader.program == nullptr) return finish(false, false); + for (const auto& binding : shader.program->bindings) { + if (binding.role != ShaderRecompiler::DescriptorRole::GuestImages) continue; + if (binding.kind == ShaderRecompiler::DescriptorKind::SampledImage) { + const auto elementWords = binding.count != 0 ? binding.guestDescriptor.size() / binding.count : 0; + for (std::uint32_t element = 0; element < binding.count; ++element) { + const auto words = std::span(binding.guestDescriptor).subspan(static_cast(element) * elementWords, elementWords); + const auto resource = DecodeTextureResource(words); + const VkComponentMapping components{ComponentSwizzleFor(resource.dstSelX), ComponentSwizzleFor(resource.dstSelY), ComponentSwizzleFor(resource.dstSelZ), ComponentSwizzleFor(resource.dstSelW)}; + if (textureIndex >= textures.size() || cachedTexture(context, words, resource, components) != textures[textureIndex]) return finish(false, false); + ++textureIndex; + } + } else if (binding.kind == ShaderRecompiler::DescriptorKind::StorageImage) { + for (std::uint32_t element = 0; element < binding.count; ++element) { + const auto words = std::span(binding.guestDescriptor).subspan(static_cast(element) * 8u, 8u); + if (storageIndex >= storageTextures.size()) return finish(false, false); + std::shared_ptr expected; + if (SameAsPreviousStorageElement(binding, element) && StorageDedupeEnabled()) expected = storageTextures[storageIndex - 1]; + else expected = cachedStorageTexture(context, words, DecodeTextureResource(words), storageMips[storageIndex]); + if (expected != storageTextures[storageIndex]) return finish(false, false); + ++storageIndex; + } + } + } + } + if (textureIndex != textures.size() || storageIndex != storageTextures.size()) return finish(false, false); + // The walk proved every object current again: the records move to what it proved, or one + // spurious stamp (a label sharing a 64 KiB block with a surface's edge) would keep this + // object on the full walk for good. + captureValidation(); + } + // (2) The imported buffers the set reads in place: results of storage images pending in them go + // to guest memory first (as an upload does), then each import must still be the one the set was + // written against. This comes last because the lookups and flushes above can reconcile imports + // themselves; nothing else touches them between here and the dispatch being recorded. + for (const auto& region : directRegions) StorageTexture::FlushPending(region.begin, static_cast(region.end - region.begin), nullptr, "imported buffer region"); + for (const auto& region : directRegions) { + auto serial = HostImportSerial(context, region.begin, static_cast(region.end - region.begin), true); + if (serial == 0) serial = ImageMirrorSerial(context, region.begin, static_cast(region.end - region.begin)); + if (serial != region.serial) return finish(fast, false); + } + return finish(fast, true); +} + +namespace { + +// APS5_PROFILE_DRAW: what makes content keys miss. On a miss the key is compared with the last key +// seen for the same shader variant(s) and the first differing word is charged to its binding's role +// (and to its index within the V# or T# element), printed as [rescache] every 10 s: it says whether +// V# bases of ring allocations, SRT words or image descriptors churn. +struct ChurnProfile { + std::mutex mutex; + std::unordered_map lastByVariant; + std::array roles{}; + std::array bufferWords{}; + std::array imageWords{}; + std::uint64_t firstSeen = 0; + std::uint64_t sameKey = 0; + std::uint64_t layout = 0; + std::uint64_t drawWords = 0; + std::chrono::steady_clock::time_point lastReport = std::chrono::steady_clock::now(); +}; + +ChurnProfile& Churn() { + static ChurnProfile profile; + return profile; +} + +// Charges word `diff` of `key` to the binding of the ContentKey that starts at `at` (the layout of +// ShaderResources::ContentKey), moving `at` past that key; false when `diff` lies beyond it. +bool chargeShaderKey(ChurnProfile& profile, const ResourceCache::Key& key, std::size_t& at, std::size_t diff) { + if (at + 4 > key.size() || diff < at + 4) { + ++profile.layout; + return true; + } + const auto bindings = key[at + 3]; + at += 4; + for (std::uint32_t binding = 0; binding < bindings; ++binding) { + if (at + 8 > key.size() || diff < at + 8) { + ++profile.layout; + return true; + } + const auto kind = key[at]; + const auto role = key[at + 1]; + const auto descriptorWords = key[at + 7]; + at += 8; + if (diff < at + descriptorWords) { + if (role < profile.roles.size()) ++profile.roles[role]; + const auto index = diff - at; + if (role == static_cast(ShaderRecompiler::DescriptorRole::GuestBuffers)) ++profile.bufferWords[index % 4]; + else if (role == static_cast(ShaderRecompiler::DescriptorRole::GuestImages) && (kind == static_cast(ShaderRecompiler::DescriptorKind::SampledImage) || kind == static_cast(ShaderRecompiler::DescriptorKind::StorageImage))) ++profile.imageWords[index % 8]; + return true; + } + at += descriptorWords; + // imageWritten, samplerDepthCompare and bufferWritten: a count word, then one word per 32 bits. + for (int flags = 0; flags < 3; ++flags) { + if (at >= key.size()) { + ++profile.layout; + return true; + } + const auto words = 1 + (static_cast(key[at]) + 31) / 32; + if (diff < at + words) { + ++profile.layout; + return true; + } + at += words; + } + } + return false; +} + +} + +std::shared_ptr ResourceCache::Find(const Key& key) { + std::lock_guard lock(mutex); + const auto found = index.find(key); + if (found == index.end()) { + noteMiss(key); + return nullptr; + } + entries.splice(entries.begin(), entries, found->second); + return found->second->second; +} + +void ResourceCache::noteMiss(const Key& key) { + static const bool profile = std::getenv("APS5_PROFILE_DRAW") != nullptr; + if (!profile || key.size() < 4) return; + // The variant(s) the key belongs to: a compute key starts with the stage and its variant, a draw + // key with a marker, the device and its stages' keys (Draw.cpp DrawResourceKey). + const bool draw = key[0] == 0xffffffffu; + std::uint64_t variants = 0; + if (draw) { + std::size_t at = 4; + for (std::uint32_t stage = 0; stage < key[3] && at + 3 < key.size(); ++stage) { + variants = variants * 1000003ull ^ (key[at + 2] | (static_cast(key[at + 3]) << 32u)); + at += 1 + key[at]; + } + } else { + variants = key[1] | (static_cast(key[2]) << 32u); + } + auto& churn = Churn(); + std::lock_guard lock(churn.mutex); + const auto found = churn.lastByVariant.find(variants); + if (found == churn.lastByVariant.end()) { + ++churn.firstSeen; + churn.lastByVariant.emplace(variants, key); + } else { + const auto& previous = found->second; + const auto common = std::min(key.size(), previous.size()); + std::size_t diff = 0; + while (diff < common && key[diff] == previous[diff]) ++diff; + if (diff == common && key.size() == previous.size()) { + ++churn.sameKey; + } else if (draw) { + std::size_t at = 4; + bool charged = false; + for (std::uint32_t stage = 0; stage < key[3] && at < key.size() && !charged; ++stage) { + ++at; + charged = chargeShaderKey(churn, key, at, diff); + } + if (!charged) ++churn.drawWords; + } else { + std::size_t at = 0; + if (!chargeShaderKey(churn, key, at, diff)) ++churn.layout; + } + found->second = key; + } + const auto now = std::chrono::steady_clock::now(); + if (now - churn.lastReport < std::chrono::seconds(10)) return; + churn.lastReport = now; + std::string line = "[rescache] miss churn, first differing word by role:"; + char item[256]; + for (std::size_t role = 0; role < churn.roles.size(); ++role) { + if (churn.roles[role] == 0) continue; + std::snprintf(item, sizeof(item), " %s %llu", roleName(static_cast(role)), static_cast(churn.roles[role])); + line += item; + if (role == static_cast(ShaderRecompiler::DescriptorRole::GuestBuffers)) { + std::snprintf(item, sizeof(item), " (V# word %llu/%llu/%llu/%llu)", static_cast(churn.bufferWords[0]), static_cast(churn.bufferWords[1]), static_cast(churn.bufferWords[2]), static_cast(churn.bufferWords[3])); + line += item; + } else if (role == static_cast(ShaderRecompiler::DescriptorRole::GuestImages)) { + std::snprintf(item, sizeof(item), " (T# word %llu/%llu/%llu/%llu/%llu/%llu/%llu/%llu)", static_cast(churn.imageWords[0]), static_cast(churn.imageWords[1]), static_cast(churn.imageWords[2]), static_cast(churn.imageWords[3]), static_cast(churn.imageWords[4]), static_cast(churn.imageWords[5]), static_cast(churn.imageWords[6]), static_cast(churn.imageWords[7])); + line += item; + } + } + std::snprintf(item, sizeof(item), "; layout %llu, draw target/index %llu, same key %llu (not inserted or evicted), first seen %llu", static_cast(churn.layout), static_cast(churn.drawWords), static_cast(churn.sameKey), static_cast(churn.firstSeen)); + line += item; + std::fprintf(stderr, "%s\n", line.c_str()); +} + +void ResourceCache::Insert(const Key& key, std::shared_ptr resources) { + std::lock_guard lock(mutex); + erase(key); + entries.emplace_front(key, std::move(resources)); + index.emplace(key, entries.begin()); + // Entries pin their textures and storage images past the texture caches' budgets, so the bound + // stays modest: APS5_RESOURCE_CACHE_ENTRIES (default 1024) covers several frames of distinct + // dispatch and draw content. + static const std::size_t capacity = [] { + const char* value = std::getenv("APS5_RESOURCE_CACHE_ENTRIES"); + const auto parsed = value ? std::strtoull(value, nullptr, 10) : 1024ull; + return static_cast(parsed != 0 ? parsed : 1024ull); + }(); + while (entries.size() > capacity) { + index.erase(entries.back().first); + entries.pop_back(); + } +} + +void ResourceCache::Remove(const Key& key) { + std::lock_guard lock(mutex); + erase(key); +} + +void ResourceCache::Clear() { + std::lock_guard lock(mutex); + index.clear(); + entries.clear(); +} + +std::size_t ResourceCache::Size() const { + std::lock_guard lock(mutex); + return entries.size(); +} + +void ResourceCache::erase(const Key& key) { + const auto found = index.find(key); + if (found == index.end()) return; + entries.erase(found->second); + index.erase(found); +} + +ResourceCache& SharedResourceCache() { + static auto* cache = new ResourceCache(); + return *cache; +} + +DescriptorCache::DescriptorCache(const Context& context) : context(context), destroyLayout(context.Function("vkDestroyDescriptorSetLayout")), destroyPool(context.Function("vkDestroyDescriptorPool")), freeSets(context.Function("vkFreeDescriptorSets")) {} + +DescriptorCache::~DescriptorCache() { + for (const auto pool : pools) destroyPool(context.device, pool, nullptr); + for (const auto& [key, layout] : layouts) destroyLayout(context.device, layout, nullptr); +} + +VkDescriptorSetLayout DescriptorCache::Layout(std::span key, std::span bindings) { + std::lock_guard lock(mutex); + std::vector keyCopy(key.begin(), key.end()); + if (const auto found = layouts.find(keyCopy); found != layouts.end()) { + ++stats.layoutHits; + return found->second; + } + ++stats.layoutMisses; + VkDescriptorSetLayoutCreateInfo info{VK_STRUCTURE_TYPE_DESCRIPTOR_SET_LAYOUT_CREATE_INFO}; + info.bindingCount = static_cast(bindings.size()); + info.pBindings = bindings.data(); + VkDescriptorSetLayout layout = VK_NULL_HANDLE; + Check(context.Function("vkCreateDescriptorSetLayout")(context.device, &info, nullptr, &layout), "vkCreateDescriptorSetLayout"); + layouts.emplace(std::move(keyCopy), layout); + return layout; +} + +namespace { +// What one chain pool holds; a set needing more of any type gets a dedicated pool. +constexpr std::uint32_t ChainPoolSets = 1024; +constexpr std::array ChainPoolSizes{{{VK_DESCRIPTOR_TYPE_STORAGE_BUFFER, 4096}, {VK_DESCRIPTOR_TYPE_SAMPLED_IMAGE, 1024}, {VK_DESCRIPTOR_TYPE_STORAGE_IMAGE, 1024}, {VK_DESCRIPTOR_TYPE_SAMPLER, 512}}}; +} + +DescriptorCache::SetAllocation DescriptorCache::Allocate(VkDescriptorSetLayout layout, std::span sizes) { + for (const auto& size : sizes) { + const auto capacity = std::find_if(ChainPoolSizes.begin(), ChainPoolSizes.end(), [&](const auto& item) { return item.type == size.type; }); + if (capacity == ChainPoolSizes.end() || size.descriptorCount > capacity->descriptorCount) return {}; + } + std::lock_guard lock(mutex); + const auto allocate = context.Function("vkAllocateDescriptorSets"); + VkDescriptorSetAllocateInfo allocation{VK_STRUCTURE_TYPE_DESCRIPTOR_SET_ALLOCATE_INFO}; + allocation.descriptorSetCount = 1; + allocation.pSetLayouts = &layout; + // Newest pool first: it has the most room; a full or fragmented pool is left for its sets to + // drain and tried again later. + for (auto it = pools.rbegin(); it != pools.rend(); ++it) { + allocation.descriptorPool = *it; + VkDescriptorSet set = VK_NULL_HANDLE; + const auto result = allocate(context.device, &allocation, &set); + if (result == VK_SUCCESS) { + ++stats.sets; + return {set, *it}; + } + if (result != VK_ERROR_OUT_OF_POOL_MEMORY && result != VK_ERROR_FRAGMENTED_POOL) Check(result, "vkAllocateDescriptorSets"); + } + VkDescriptorPoolCreateInfo poolInfo{VK_STRUCTURE_TYPE_DESCRIPTOR_POOL_CREATE_INFO}; + poolInfo.flags = VK_DESCRIPTOR_POOL_CREATE_FREE_DESCRIPTOR_SET_BIT; + poolInfo.maxSets = ChainPoolSets; + poolInfo.poolSizeCount = static_cast(ChainPoolSizes.size()); + poolInfo.pPoolSizes = ChainPoolSizes.data(); + VkDescriptorPool pool = VK_NULL_HANDLE; + Check(context.Function("vkCreateDescriptorPool")(context.device, &poolInfo, nullptr, &pool), "vkCreateDescriptorPool"); + pools.push_back(pool); + ++stats.pools; + allocation.descriptorPool = pool; + VkDescriptorSet set = VK_NULL_HANDLE; + Check(allocate(context.device, &allocation, &set), "vkAllocateDescriptorSets"); + ++stats.sets; + return {set, pool}; +} + +void DescriptorCache::Free(const SetAllocation& allocation) noexcept { + if (allocation.set == VK_NULL_HANDLE || allocation.pool == VK_NULL_HANDLE) return; + std::lock_guard lock(mutex); + static_cast(freeSets(context.device, allocation.pool, 1, &allocation.set)); +} + +DescriptorCache::Stats DescriptorCache::Counters() const { + std::lock_guard lock(mutex); + return stats; +} + +std::size_t ShaderResources::addGuestBuffer(std::span words, const ColorTarget* target, std::uint64_t indexAddress, std::size_t indexBytes, bool written) { Require(words.size() == 4, "buffer descriptor must contain four DWORDs"); Require((words[1] & 0x40000000u) == 0, "buffer descriptor has reserved bits set"); const ShaderRecompiler::ShaderBufferResource descriptor{{words[0], words[1], words[2], words[3]}}; @@ -185,14 +1367,86 @@ std::size_t ShaderResources::addGuestBuffer(std::span words Require(byteSize <= context.limits.maxStorageBufferRange, "shader buffer exceeds descriptor range limit"); Require(byteSize <= std::numeric_limits::max(), "shader buffer size exceeds host address space"); const auto size = static_cast(byteSize); - GuestMemory::CheckRange(reinterpret_cast(address), size, 1, true); Require(target == nullptr || !overlap(address, size, target->address, target->bytes), "shader buffer aliases the render target"); + // APS5_ALL_BUFFERS_WRITTEN=1: every element is noted as written, as before bufferWritten existed. + static const bool allWritten = std::getenv("APS5_ALL_BUFFERS_WRITTEN") != nullptr; + written = written || allWritten; + // Unconditional (also for read-only elements): Draw.cpp's CheckBufferAliases repeats this check + // on every resource-cache hit, and a draw must fail its hit exactly when it fails its build. Require(!overlap(address, size, indexAddress, indexBytes), "writable shader buffer aliases the index buffer"); - guestMemory.AddWritable(address, size); - allocations.push_back({address, size, true, nullptr}); + if (written) guestMemory.AddWritable(address, size); + else { + guestMemory.AddReadable(address, size); + ++readOnlyBuffers; + } + if (BuildProfiled()) { + auto& counters = BufferWrites(); + counters.elements.fetch_add(1, std::memory_order_relaxed); + if (!written) counters.readOnly.fetch_add(1, std::memory_order_relaxed); + } + allocations.push_back({address, size, true, nullptr, ShaderRecompiler::DescriptorRole::ShaderData, written}); return allocations.size() - 1; } +std::string ShaderResources::Describe() const { + const auto sample = [](std::uint64_t address, std::uint64_t bytes) { + // Heaps bound whole can include uncommitted pages; those ranges are not sampled. + if (!GuestMemory::Accessible(reinterpret_cast(address), static_cast(bytes))) return -1.0; + std::size_t nonzero = 0; + std::size_t samples = 0; + const auto* data = reinterpret_cast(address); + const auto words = bytes / 4; + const auto step = std::max(1, words / 4096); + for (std::uint64_t i = 0; i < words; i += step, ++samples) nonzero += data[i] != 0; + return samples == 0 ? 0.0 : static_cast(nonzero) / samples; + }; + std::string text; + char line[160]; + for (const auto& range : describedRanges) { + std::snprintf(line, sizeof(line), " %s 0x%llx+0x%llx(%ux%u f%u t%d) nz=%.2f", range.kind, static_cast(range.address), static_cast(range.bytes), range.width, range.height, range.format, range.tileMode, sample(range.address, range.bytes)); + text += line; + if (range.dccAddress != 0) { + const auto keys = ReadDccKeys(range.dccAddress, range.bytes); + std::snprintf(line, sizeof(line), " dcc=%s@0x%llx", DccKeysName(keys), static_cast(range.dccAddress)); + text += line; + if (keys == DccKeys::Mixed) { + // Where the keys change: first key, how many leading keys match it, and the next key. + const auto* bytes = reinterpret_cast(range.dccAddress); + const auto count = static_cast(range.bytes / 256u); + std::size_t run = 1; + while (run < count && bytes[run] == bytes[0]) ++run; + std::snprintf(line, sizeof(line), "(%02x x%zu then %02x of %zu)", bytes[0], run, run < count ? bytes[run] : 0u, count); + text += line; + } + } + } + // APS5_TRACE_DISPATCH_IO=2 also lists the words of small buffers (constants). + static const bool words = [] { const char* value = std::getenv("APS5_TRACE_DISPATCH_IO"); return value != nullptr && value[0] == '2'; }(); + const auto appendWords = [&](const std::uint32_t* data, std::size_t bytes) { + if (!words || bytes > 0x200) return; + text += " ["; + for (std::size_t i = 0; i < bytes / 4; ++i) { + char word[12]; + std::snprintf(word, sizeof(word), "%s%08x", i == 0 ? "" : " ", data[i]); + text += word; + } + text += "]"; + }; + for (const auto& allocation : allocations) { + if (allocation.guest) { + std::snprintf(line, sizeof(line), " buffer%s 0x%llx+0x%zx nz=%.2f", allocation.written ? "" : "(ro)", static_cast(allocation.address), allocation.size, sample(allocation.address, allocation.size)); + text += line; + if (GuestMemory::Accessible(reinterpret_cast(allocation.address), allocation.size)) appendWords(reinterpret_cast(allocation.address), allocation.size); + } else if (allocation.buffer) { + const auto bytes = allocation.buffer->Bytes(); + std::snprintf(line, sizeof(line), " data+0x%zx nz=%.2f", allocation.size, sample(reinterpret_cast(bytes.data()), allocation.size)); + text += line; + appendWords(reinterpret_cast(bytes.data()), allocation.size); + } + } + return text; +} + std::size_t ShaderResources::addDataBuffer(std::span words) { const auto size = words.size() * sizeof(std::uint32_t); Require(size <= context.limits.maxStorageBufferRange, "shader data buffer exceeds descriptor range limit"); @@ -202,8 +1456,19 @@ std::size_t ShaderResources::addDataBuffer(std::span words) return allocations.size() - 1; } -void ShaderResources::addImageBinding(const ShaderRecompiler::DescriptorBinding& binding, VkShaderStageFlags flags, std::vector& bindings) { +void ShaderResources::addImageBinding(const ShaderRecompiler::DescriptorBinding& binding, VkShaderStageFlags flags) { Require(binding.count != 0, "empty descriptor binding"); + if (binding.kind == ShaderRecompiler::DescriptorKind::StorageImage) { + // Storage images are guest textures the shader writes (looked up in stage B). + Require(binding.role == ShaderRecompiler::DescriptorRole::GuestImages, "storage image binding has a non-image role"); + Require(binding.guestDescriptor.size() == static_cast(binding.count) * 8u, "guest storage image descriptors must contain 8 dwords each"); + Require(context.detiler != nullptr, "device texture detiler is unavailable"); + Require(binding.count <= context.limits.maxPerStageDescriptorStorageImages, "shader storage-image descriptors exceed per-stage limits"); + bindings.push_back({{binding.binding, VK_DESCRIPTOR_TYPE_STORAGE_IMAGE, binding.count, flags, nullptr}, {}, {}}); + plannedStorageImages += binding.count; + deferredImages.push_back({&binding, bindings.size() - 1}); + return; + } const bool sampledImage = binding.kind == ShaderRecompiler::DescriptorKind::SampledImage; const bool samplerKind = binding.kind == ShaderRecompiler::DescriptorKind::Sampler; Require(sampledImage || samplerKind, std::string("unsupported descriptor kind ") + kindName(binding.kind) + " for role " + roleName(binding.role)); @@ -219,31 +1484,243 @@ void ShaderResources::addImageBinding(const ShaderRecompiler::DescriptorBinding& Require(context.detiler != nullptr, "device texture detiler is unavailable"); Require(context.textureCache != nullptr, "device texture cache is unavailable"); Require(binding.count <= context.limits.maxPerStageDescriptorSampledImages, "shader sampled-image descriptors exceed per-stage limits"); - for (std::uint32_t element = 0; element < binding.count; ++element) { - const auto words = std::span(binding.guestDescriptor).subspan(static_cast(element) * elementWords, elementWords); - const auto resource = DecodeTextureResource(words); - Require(MatchesGuestDimension(*binding.imageShape, resource.dimension), "guest texture dimension disagrees with the shader's declared image shape"); - const VkComponentMapping components{ComponentSwizzleFor(resource.dstSelX), ComponentSwizzleFor(resource.dstSelY), ComponentSwizzleFor(resource.dstSelZ), ComponentSwizzleFor(resource.dstSelW)}; - textures.push_back(context.textureCache->Get(words, resource, components)); - item.imageAllocations.push_back(textures.size() - 1); - } - Require(textures.size() <= context.limits.maxDescriptorSetSampledImages, "pipeline sampled-image descriptors exceed device limits"); + plannedSampledImages += binding.count; + Require(plannedSampledImages <= context.limits.maxDescriptorSetSampledImages, "pipeline sampled-image descriptors exceed device limits"); } else { Require(elementWords == 4, "guest sampler descriptor must contain 4 dwords"); Require(binding.count <= context.limits.maxPerStageDescriptorSamplers, "shader sampler descriptors exceed per-stage limits"); Require(binding.samplerDepthCompare.size() == binding.count, "guest sampler binding is missing depth comparison metadata"); for (std::uint32_t element = 0; element < binding.count; ++element) { const auto words = std::span(binding.guestDescriptor).subspan(static_cast(element) * elementWords, elementWords); - auto resource = DecodeSamplerResource(words); - resource.compareEnable = binding.samplerDepthCompare.at(element); - if (!context.samplerCache) context.samplerCache = std::make_shared(); - samplers.push_back(context.samplerCache->Get(context, words, resource)); + const bool compareEnable = binding.samplerDepthCompare.at(element); + static const bool noSamplerCache = std::getenv("APS5_NO_SAMPLER_CACHE") != nullptr; + if (context.samplerCache != nullptr && !noSamplerCache) { + samplers.push_back(context.samplerCache->Get(context, words, compareEnable)); + } else { + auto resource = DecodeSamplerResource(words); + resource.compareEnable = compareEnable; + samplers.push_back(std::make_shared(context, resource)); + } item.imageAllocations.push_back(samplers.size() - 1); } Require(samplers.size() <= context.limits.maxDescriptorSetSamplers, "pipeline sampler descriptors exceed device limits"); } bindings.push_back(std::move(item)); + if (sampledImage) deferredImages.push_back({&binding, bindings.size() - 1}); +} + +namespace { + +// Calls `visit(binding, element, words)` for every sampled and storage image element of a compiled +// shader's bindings, in plan order (the order addImageBinding and resolveImageBinding walk). +template +void forEachImageElement(const ShaderRecompiler::RecompileResult& program, Visit&& visit) { + for (const auto& binding : program.bindings) { + if (binding.role != ShaderRecompiler::DescriptorRole::GuestImages || binding.count == 0) continue; + if (binding.kind != ShaderRecompiler::DescriptorKind::SampledImage && binding.kind != ShaderRecompiler::DescriptorKind::StorageImage) continue; + const auto elementWords = binding.guestDescriptor.size() / binding.count; + for (std::uint32_t element = 0; element < binding.count; ++element) visit(binding, element, std::span(binding.guestDescriptor).subspan(static_cast(element) * elementWords, elementWords)); + } +} + +} + +bool ShaderResources::precollectImages() { + // The image lookups of stage B (cachedTexture, StorageTexture::Refresh) each start with a + // GuestMemory::CollectWrites of their surface, the GetWriteWatch walk that is most of the + // lookups' time under the lock. The walk is memoized per thread epoch, and the worker thread + // passes no ordering point between the stages (see GuestMemory::BumpCollectEpoch), so walking + // every surface here makes stage B's collects memo hits: the walk leaves the lock, the lookups' + // checks stay where they were. A memo miss (the ring overflowed, an uncommitted page) just walks + // under the lock as before; a descriptor stage B rejects is left to it. APS5_NO_PRECOLLECT=1 + // disables the pass. + // The pass also leaves one ImageRecord per element: the decode and the surface description are + // made once for the build, and a sampled element's cache entry is taken here (a hash lookup + // under the cache's own mutex) so that stage B, under the device lock, runs only the + // fastRevalidate predicate on it (see fastTexture). The collect comes before the entry is read, + // as the predicate's rule demands (the generation the entry moves to must predate the checks). + static const bool disabled = std::getenv("APS5_NO_PRECOLLECT") != nullptr; + static const bool noRecords = std::getenv("APS5_NO_STAGE_A_IMAGES") != nullptr; + if (disabled || deferredImages.empty()) return false; + imageRecords.clear(); + nextImageRecord = 0; + auto& counters = TextureCounts(); + for (const auto& deferred : deferredImages) { + const auto& binding = *deferred.binding; + const auto elementWords = binding.guestDescriptor.size() / binding.count; + for (std::uint32_t element = 0; element < binding.count; ++element) { + const auto words = std::span(binding.guestDescriptor).subspan(static_cast(element) * elementWords, elementWords); + ImageRecord record; + record.sampled = binding.kind == ShaderRecompiler::DescriptorKind::SampledImage; + try { + record.resource = DecodeTextureResource(words); + record.guestBytes = DescribeSurface(record.resource).guestBytes; + record.generation = GuestMemory::CollectWrites(record.resource.baseAddress, static_cast(record.guestBytes)); + record.decoded = true; + if (record.sampled && !noRecords && words.size() == 8) { + std::copy(words.begin(), words.end(), record.words.begin()); + record.components = {ComponentSwizzleFor(record.resource.dstSelX), ComponentSwizzleFor(record.resource.dstSelY), ComponentSwizzleFor(record.resource.dstSelZ), ComponentSwizzleFor(record.resource.dstSelW)}; + record.keys = TextureClearKeys(record.resource, record.guestBytes); + auto& cache = Textures(); + std::lock_guard lock(cache.mutex); + if (const auto it = findTexture(cache, MakeTextureKey(context.device, words, record.components)); it != cache.entries.end()) { + record.texture = it->texture; + record.source = it->source; + record.entryKeys = it->keys; + record.entryGeneration = it->generation; + counters.records.fetch_add(1, std::memory_order_relaxed); + } + } + } catch (const std::exception&) { + record.decoded = false; + } + imageRecords.push_back(std::move(record)); + } + } + return true; +} + +std::shared_ptr ShaderResources::fastTexture(const ImageRecord& record) { + if (record.texture == nullptr) return nullptr; + // Exactly the "unchanged" branches of cachedTexture, from write stamps, the pending-results + // registry and the DCC keys, the way fastRevalidate proves a built object current: the collect + // first (a memo hit: stage A walked the range), then no other image may have results pending + // over the memory (a lookup would flush them into it, or view them instead), a storage-sourced + // view needs its image to be the cache's image of the surface and current with guest memory, + // a snapshot needs the memory unchanged since its content matched, and the keys must be what + // the content was made under whenever the surface has any (re-read under the lock: a flush + // between the stages marks them uncompressed). + const auto address = record.resource.baseAddress; + const auto bytes = static_cast(record.guestBytes); + if (GuestMemory::CollectWrites(address, bytes) == 0) return nullptr; + if (PendingStorageOverlaps(address, bytes, record.source.get())) return nullptr; + auto keys = record.keys; + if (record.resource.dccAddress != 0) keys = TextureClearKeys(record.resource, record.guestBytes); + // The keys the entry was made under (a view of a pending render target was made under its clear + // keys and stays valid while they are unchanged); a change is the full lookup's to judge. + if (keys != record.entryKeys) return nullptr; + if (record.source != nullptr) { + if (!StorageImageCached(context, record.source.get()) || !GuestMemory::UnchangedSince(address, bytes, record.source->Generation())) return nullptr; + // Under fast-clear keys the view holds only while its image's results are still pending over + // the surface; flushed, the clear the image cannot see makes the lookup take a snapshot. + if (keys != DccKeys::Uncompressed && StorageTexture::FindPending(address, record.guestBytes) != record.source) return nullptr; + } else if (keys == DccKeys::Uncompressed && !GuestMemory::UnchangedSince(address, bytes, record.entryGeneration)) { + return nullptr; + } + // The entry must still be the cache's, holding this object (an evicted object is not reused: the + // next lookup would make another), and it takes the stage-A generation like a hit would. + auto& cache = Textures(); + std::lock_guard lock(cache.mutex); + const auto it = findTexture(cache, MakeTextureKey(context.device, record.words, record.components)); + if (it == cache.entries.end() || it->texture != record.texture) return nullptr; + if (it->source == nullptr) it->generation = record.generation; + touchTexture(cache, it); + logLookup({record.texture.get(), record.resource, record.guestBytes, keys, record.source != nullptr ? 0 : record.generation, record.source.get()}); + return record.texture; +} + +void ShaderResources::resolveImageBinding(const ShaderRecompiler::DescriptorBinding& binding, Binding& item) { + auto& counters = TextureCounts(); + // The element's stage-A record, when the pass ran (records follow the plan order exactly). + const auto nextRecord = [&]() -> const ImageRecord* { return nextImageRecord < imageRecords.size() ? &imageRecords[nextImageRecord++] : nullptr; }; + if (binding.kind == ShaderRecompiler::DescriptorKind::SampledImage) { + const auto elementWords = binding.guestDescriptor.size() / binding.count; + for (std::uint32_t element = 0; element < binding.count; ++element) { + const auto words = std::span(binding.guestDescriptor).subspan(static_cast(element) * elementWords, elementWords); + const auto* record = nextRecord(); + const auto resource = record != nullptr && record->decoded ? record->resource : DecodeTextureResource(words); + Require(MatchesGuestDimension(*binding.imageShape, resource.dimension), "guest texture dimension disagrees with the shader's declared image shape"); + const VkComponentMapping components{ComponentSwizzleFor(resource.dstSelX), ComponentSwizzleFor(resource.dstSelY), ComponentSwizzleFor(resource.dstSelZ), ComponentSwizzleFor(resource.dstSelW)}; + const auto guestBytes = record != nullptr && record->decoded ? record->guestBytes : DescribeSurface(resource).guestBytes; + std::shared_ptr texture; + if (record != nullptr && record->texture != nullptr) { + texture = fastTexture(*record); + (texture != nullptr ? counters.fastHits : counters.fastMisses).fetch_add(1, std::memory_order_relaxed); + } + if (texture == nullptr) texture = cachedTexture(context, words, resource, components, guestBytes); + textures.push_back(std::move(texture)); + describedRanges.push_back({"texture", resource.baseAddress, guestBytes, resource.width, resource.height, resource.format, static_cast(resource.tileMode), resource.dccAddress}); + item.imageAllocations.push_back(textures.size() - 1); + } + // The line is due even when every element took the fast path (cachedTexture reports too). + reportTextureCounters(); + return; + } + // Consecutive identical storage descriptors address successive mips of one texture (dynamic-mip + // storage writes). + std::uint32_t mipOffset = 0; + for (std::uint32_t element = 0; element < binding.count; ++element) { + const auto words = std::span(binding.guestDescriptor).subspan(static_cast(element) * 8u, 8u); + const bool sameAsPrevious = SameAsPreviousStorageElement(binding, element); + if (sameAsPrevious) ++mipOffset; + else mipOffset = 0; + const auto* record = nextRecord(); + const auto resource = record != nullptr && record->decoded ? record->resource : DecodeTextureResource(words); + if (binding.imageShape.has_value()) Require(MatchesGuestDimension(*binding.imageShape, resource.dimension), "guest storage texture dimension disagrees with the shader's declared image shape"); + const auto mip = std::min(resource.baseLevel + mipOffset, resource.mipCount - 1u); + const auto guestBytes = record != nullptr && record->decoded ? record->guestBytes : DescribeSurface(resource).guestBytes; + // The same surface as the previous element: its image was just looked up and refreshed. + if (sameAsPrevious && StorageDedupeEnabled()) storageTextures.push_back(storageTextures.back()); + else storageTextures.push_back(cachedStorageTexture(context, words, resource, mip, guestBytes)); + storageMips.push_back(mip); + // Images the shader only reads have nothing to store back. + storageWritten.push_back(element >= binding.imageWritten.size() || binding.imageWritten[element]); + describedRanges.push_back({"storage", resource.baseAddress, guestBytes, resource.width, resource.height, resource.format, static_cast(resource.tileMode), resource.dccAddress}); + item.imageAllocations.push_back(storageTextures.size() - 1); + } +} + +std::vector> ShaderResources::PresyncSurfaces() const { + std::vector> surfaces; + // A surface's lookup reads guest memory on the CPU unless it is served GPU-direct from a host + // import: a storage image of imported memory uploads and refreshes from the import, and a + // sampled texture of one views that image (SampledFromStorageEligible), so neither waits for + // the producer. + const auto consider = [&](std::uint32_t format, std::uint64_t address, std::uint64_t guestBytes, bool sampled) { + const bool gpuDirect = sampled ? SampledFromStorageEligible(context, format, address, guestBytes) : HostImportCovers(context, address, static_cast(guestBytes)); + if (!gpuDirect) surfaces.emplace_back(address, guestBytes); + }; + if (completed) { + // A cached object (the resource cache): Revalidate repeats the lookups of the objects the + // build made, which never change afterwards (a lookup returning another object fails the + // revalidation instead), so the surfaces come from the build's own record of them. A + // sampled texture viewing a storage image refreshes like that image; a snapshot texture + // over an eligible surface is replaced by a view without reading. Storage images are + // checked on their own surface (a mip chain can exceed the element's descriptor). Read + // without the device lock: these members are fixed once the build completed. + std::size_t textureIndex = 0; + for (const auto& range : describedRanges) { + if (std::strcmp(range.kind, "texture") == 0 && textureIndex < textures.size()) { + const auto& texture = textures[textureIndex++]; + if (texture->ViewsStorageImage()) consider(range.format, range.address, range.bytes, false); + else consider(range.format, range.address, range.bytes, true); + } + } + for (std::size_t index = 0; index < storageTextures.size(); ++index) { + if (index != 0 && storageTextures[index] == storageTextures[index - 1]) continue; + const auto& image = *storageTextures[index]; + consider(image.Descriptor().format, image.Descriptor().baseAddress, image.GuestBytes(), false); + } + return surfaces; + } + if (!imageRecords.empty()) { + for (const auto& record : imageRecords) { + if (record.decoded) consider(record.resource.format, record.resource.baseAddress, record.guestBytes, record.sampled); + } + return surfaces; + } + // Stage A has not run (an address-based build, or the pass is off): decoded from the shader. + if (deferredCompute.program == nullptr) return surfaces; + forEachImageElement(*deferredCompute.program, [&](const ShaderRecompiler::DescriptorBinding& binding, std::uint32_t, std::span words) { + try { + const auto resource = DecodeTextureResource(words); + consider(resource.format, resource.baseAddress, DescribeSurface(resource).guestBytes, binding.kind == ShaderRecompiler::DescriptorKind::SampledImage); + } catch (const std::exception&) { + // Stage B reports the bad descriptor. + } + }); + return surfaces; } ShaderResources::~ShaderResources() { @@ -251,9 +1728,14 @@ ShaderResources::~ShaderResources() { } void ShaderResources::release() noexcept { - if (descriptors) context.descriptorCache->Put(std::move(descriptors)); - _layout = VK_NULL_HANDLE; + // A set from the cache's pool chain goes back to it; a dedicated pool dies with its set. + if (cachePool && context.descriptorCache != nullptr) context.descriptorCache->Free({_set, cachePool}); + if (pool) context.Function("vkDestroyDescriptorPool")(context.device, pool, nullptr); + if (_layout && ownsLayout) context.Function("vkDestroyDescriptorSetLayout")(context.device, _layout, nullptr); + cachePool = VK_NULL_HANDLE; + pool = VK_NULL_HANDLE; _set = VK_NULL_HANDLE; + _layout = VK_NULL_HANDLE; } VkDescriptorSetLayout ShaderResources::Layout() const { @@ -265,8 +1747,51 @@ void ShaderResources::Bind(VkCommandBuffer commands, VkPipelineBindPoint bindPoi context.Function("vkCmdBindDescriptorSets")(commands, bindPoint, layout, 0, 1, &_set, 0, nullptr); } +namespace { +// Debug aid: APS5_GPU_NO_WRITEBACK=1 keeps GPU results out of guest memory. +bool SkipWriteBack() { + static const bool skip = std::getenv("APS5_GPU_NO_WRITEBACK") != nullptr; + return skip; +} +} + void ShaderResources::WriteBack() { + WriteBackBuffers(); + if (SkipWriteBack()) return; + for (std::size_t index = 0; index < storageTextures.size(); ++index) { + if (storageWritten[index]) storageTextures[index]->MarkDirty(); + } +} + +void ShaderResources::MarkGpuWrites(Recorder& recorder) { + if (SkipWriteBack()) return; + for (std::size_t index = 0; index < storageTextures.size(); ++index) { + if (storageWritten[index]) storageTextures[index]->MarkDirty(); + } + // Written sub-ranges of buffers the GPU copied out of a host import go back into it by the GPU, + // recorded here after the work: those regions then need no CPU write-back (HasCopiedWrites), + // and the note and mark below cover them like direct writes. + guestMemory.RecordCopyBacks(recorder); + // Only the written elements' ranges (AddWritable): a read-only element is neither noted here + // nor marked as a direct write, so CPU reads of its memory never wait for this work. + recorder.NotePendingWrites(guestMemory.Writes()); + guestMemory.MarkDirectWrites(); + if (!BuildProfiled()) return; + auto& counters = BufferWrites(); + // Recorded uses only (dispatches and recorded draws): a synchronous draw writes back in + // WriteBack() and publishes no pending writes, so it has no notes to skip. + counters.notesSkipped.fetch_add(readOnlyBuffers, std::memory_order_relaxed); + const auto nowMs = std::chrono::duration_cast(std::chrono::steady_clock::now().time_since_epoch()).count(); + auto last = counters.lastReport.load(); + if (nowMs - last < 10000 || !counters.lastReport.compare_exchange_strong(last, nowMs)) return; + const auto elements = counters.elements.load(); + const auto readOnly = counters.readOnly.load(); + std::fprintf(stderr, "[buffers] descriptor elements bound: %llu total, %llu written, %llu read-only; %llu pending-write notes skipped\n", static_cast(elements), static_cast(elements - readOnly), static_cast(readOnly), static_cast(counters.notesSkipped.load())); +} + +void ShaderResources::WriteBackBuffers() { if (bda) bda->CheckFault(); + if (SkipWriteBack()) return; guestMemory.WriteBack(); } diff --git a/core/libs/prx/libSceAgcDriver/Graphics/src/ShaderValidation.cpp b/core/libs/prx/libSceAgcDriver/Graphics/src/ShaderValidation.cpp index e12e886ab..1eb231c1a 100644 --- a/core/libs/prx/libSceAgcDriver/Graphics/src/ShaderValidation.cpp +++ b/core/libs/prx/libSceAgcDriver/Graphics/src/ShaderValidation.cpp @@ -1,7 +1,6 @@ #include "BdaAbi.hpp" #include "prx/libSceAgcDriver/Graphics/include/Pipeline.hpp" #include "prx/libSceAgcDriver/Graphics/include/VertexInput.hpp" -#include "prx/libSceAgcDriver/Execution/include/PerformanceTimer.hpp" #include #include #include @@ -9,8 +8,6 @@ #include #include #include -#include -#include namespace AgcDriver::Graphics { namespace { @@ -205,13 +202,26 @@ Module Inspect(const CompiledShader& compiled, const State& state, const VkPhysi mesh && capability == spv::CapabilityMeshShadingEXT; + // Enabled unconditionally or by the device setup in VulkanDevice. + const bool isFeatureCapability = + capability == spv::CapabilityImageGatherExtended || + capability == spv::CapabilityImageQuery || + capability == spv::CapabilityStorageImageWriteWithoutFormat || + capability == spv::CapabilityStorageImageReadWithoutFormat || + capability == spv::CapabilityInt64 || + capability == spv::CapabilityInt16 || + capability == spv::CapabilityFloat16 || + capability == spv::CapabilityStorageBuffer8BitAccess || + capability == spv::CapabilityPhysicalStorageBufferAddresses; + Require( isBaseCapability || isBarycentricCapability || isSubgroupCapability || isBdaCapability || isTessellationCapability || - isMeshCapability, + isMeshCapability || + isFeatureCapability, std::string("SPIR-V requires unsupported device capability ") + std::to_string(static_cast(capability))); @@ -242,11 +252,10 @@ Module Inspect(const CompiledShader& compiled, const State& state, const VkPhysi case spv::OpSpecConstantOp: throw std::runtime_error("AGC graphics: unsupported SPIR-V extension, grouped decoration or specialization constant"); case spv::OpMemoryModel: { - const auto addressingModel = shader.bdaAbiVersion == ShaderRecompiler::BdaAbi::Version && !rectListControl - ? spv::AddressingModelPhysicalStorageBuffer64 - : spv::AddressingModelLogical; + // Address-based shaders use PhysicalStorageBuffer64; everything else (including the + // generated rect-list stages) is Logical. Require(count == 3, "invalid SPIR-V OpMemoryModel word count: " + std::to_string(count)); - Require(instruction[1] == addressingModel, "unsupported SPIR-V addressing model: " + std::to_string(instruction[1])); + Require(instruction[1] == spv::AddressingModelPhysicalStorageBuffer64 || instruction[1] == spv::AddressingModelLogical, "unsupported SPIR-V addressing model: " + std::to_string(instruction[1])); Require(instruction[2] == spv::MemoryModelGLSL450, "unsupported SPIR-V memory model: " + std::to_string(instruction[2])); ++memoryModels; break; @@ -417,7 +426,6 @@ Module Inspect(const CompiledShader& compiled, const State& state, const VkPhysi const auto binding = std::find_if(shader.bindings.begin(), shader.bindings.end(), [&](const auto& item) { return item.descriptorSet == key.first && item.binding == key.second; }); Require(binding != shader.bindings.end(), "SPIR-V resource is absent from recompiler binding metadata"); Require(binding->kind == ShaderRecompiler::DescriptorKind::Sampler || binding->kind == ShaderRecompiler::DescriptorKind::SampledImage || binding->kind == ShaderRecompiler::DescriptorKind::StorageImage, "SPIR-V descriptor type disagrees with recompiler binding metadata"); - Require(binding->kind != ShaderRecompiler::DescriptorKind::StorageImage, "storage image resources are not implemented"); } else { Require(variable.storage == spv::StorageClassStorageBuffer && decoration.set && decoration.binding, "unsupported or unbound shader resource"); const auto key = std::make_pair(*decoration.set, *decoration.binding); @@ -466,77 +474,7 @@ Module Inspect(const CompiledShader& compiled, const State& state, const VkPhysi } -namespace { - -struct ValidatedInterface { - std::map inputs; - std::map outputs; -}; - -std::shared_ptr inspectCached(const CompiledShader& compiled, const State& state, const VkPhysicalDeviceSubgroupProperties& subgroup, bool fragmentShaderBarycentric) { - PerformanceTimer timing("Graphics.ShaderValidation"); - Require(compiled.program != nullptr, "missing compiled shader"); - const auto& shader = *compiled.program; - std::string key; - const auto append = [&](const TValue& value) { - static_assert(std::is_trivially_copyable_v); - key.append(reinterpret_cast(&value), sizeof(value)); - }; - append(compiled.stage); - append(state.rectList); - append(state.stages.mesh.has_value()); - if (state.stages.mesh) { - append(state.stages.mesh->threadsPerGroup); - append(state.stages.mesh->maxVertices); - append(state.stages.mesh->maxPrimitives); - } - append(state.stages.tessellation.has_value()); - if (state.stages.tessellation) { - append(state.stages.tessellation->inputControlPoints); - append(state.stages.tessellation->outputControlPoints); - } - append(subgroup.supportedStages); - append(subgroup.supportedOperations); - append(fragmentShaderBarycentric); - append(shader.bdaAbiVersion); - append(shader.pushConstants.empty()); - append(shader.bindings.size()); - for (const auto& binding : shader.bindings) { - append(binding.kind); - append(binding.role); - append(binding.descriptorSet); - append(binding.binding); - append(binding.count); - append(binding.readOnly); - append(binding.guestDescriptor.empty()); - } - append(shader.vertexAttributes.size()); - for (const auto& attribute : shader.vertexAttributes) { - append(attribute.location); - append(attribute.components); - append(attribute.resource.fields[3]); - } - append(shader.spirv.size()); - const auto code = std::as_bytes(std::span(shader.spirv)); - if (!code.empty()) key.append(reinterpret_cast(code.data()), code.size()); - timing.Mark("key", key.size()); - static thread_local std::map> cache; - const auto found = cache.find(key); - if (found != cache.end()) { - timing.Mark("hit"); - return found->second; - } - auto module = Inspect(compiled, state, subgroup, fragmentShaderBarycentric); - auto result = std::make_shared(ValidatedInterface{std::move(module.inputs), std::move(module.outputs)}); - if (cache.size() >= 128) cache.clear(); - cache.emplace(std::move(key), result); - timing.Mark("inspect"); - return result; -} - -} - -void ValidateShaders(std::span shaders, const State& state, const VkPhysicalDeviceSubgroupProperties& subgroup, bool fragmentShaderBarycentric) { +std::set ValidateShaders(std::span shaders, const State& state, const VkPhysicalDeviceSubgroupProperties& subgroup, bool fragmentShaderBarycentric) { using Stage = ShaderRecompiler::ShaderStage; const bool tessellation = state.stages.path == ShaderPath::Tessellation; const bool mesh = state.stages.path == ShaderPath::Geometry; @@ -546,22 +484,30 @@ void ValidateShaders(std::span shaders, const State& state Require(shaders.size() == (tessellation || state.rectList ? 4u : 2u), "incorrect graphics stage count"); const std::array tessStages{Stage::Local, Stage::TessellationControl, Stage::TessellationEvaluation, Stage::Fragment}; static_cast(AssemblePushConstants(shaders)); - std::shared_ptr previous; + Module previous; for (std::size_t i = 0; i < shaders.size(); ++i) { const auto expected = state.rectList ? (i == 0 ? Stage::Vertex : tessStages[i]) : tessellation ? tessStages[i] : i == 1 ? Stage::Fragment : state.stages.path == ShaderPath::Geometry ? Stage::Mesh : Stage::Vertex; Require(shaders[i].program != nullptr, "missing compiled shader"); Require(shaders[i].stage == expected, "graphics stage order disagrees"); for (const auto& binding : shaders[i].program->bindings) Require(binding.descriptorSet == 0, "graphics resource uses a descriptor set other than zero"); - const auto current = inspectCached(shaders[i], state, subgroup, fragmentShaderBarycentric); + const auto current = Inspect(shaders[i], state, subgroup, fragmentShaderBarycentric); if (i != 0) { - for (const auto& [location, signature] : current->inputs) { - const auto output = previous->outputs.find(location); - Require(output != previous->outputs.end() && output->second == signature, "graphics interfaces disagree at location " + std::to_string(location)); + for (const auto& [location, signature] : current.inputs) { + const auto output = previous.outputs.find(location); + Require(output != previous.outputs.end() && output->second == signature, "graphics interfaces disagree at location " + std::to_string(location)); } } previous = current; } - Require(previous->outputs.size() == 1 && previous->outputs.contains(0) && previous->outputs.at(0) == "vertex:f32x4", "fragment shader must export one float4 color at location zero"); + // One float4 color per MRT attachment. A pixel shader may also export no color at all when it + // writes storage images or buffers instead; its attachments are then left untouched. + const auto attachments = std::max(state.colors.size(), 1u); + std::set locations; + for (const auto& [location, signature] : previous.outputs) { + Require(location < attachments && signature == "vertex:f32x4", "fragment shader must export float4 colors at locations below the attachment count"); + locations.insert(location); + } + return locations; } void ValidateShaderPair(const ShaderRecompiler::RecompileResult& vertex, const ShaderRecompiler::RecompileResult& fragment) { diff --git a/core/libs/prx/libSceAgcDriver/Graphics/src/State.cpp b/core/libs/prx/libSceAgcDriver/Graphics/src/State.cpp index c759c5dfb..5b71e9fae 100644 --- a/core/libs/prx/libSceAgcDriver/Graphics/src/State.cpp +++ b/core/libs/prx/libSceAgcDriver/Graphics/src/State.cpp @@ -1,4 +1,7 @@ +#include #include "prx/libSceAgcDriver/Graphics/include/State.hpp" +#include "prx/libSceAgcDriver/Graphics/include/TextureTiling.hpp" +#include "prx/libSceAgcDriver/Graphics/include/DccMetadata.hpp" #include "prx/libSceAgcDriver/Execution/include/GuestMemory.hpp" #include "prx/libc/include/General.hpp" #include @@ -27,7 +30,12 @@ float readFloat(const Registers& registers, std::uint32_t offset) { } void zero(const Registers& registers, std::uint32_t offset, std::uint32_t mask, const char* name, const char* bank = "context") { - Require((read(registers, offset, bank) & mask) == 0, std::string(name) + " is unsupported"); + const auto value = read(registers, offset, bank); + if ((value & mask) != 0) { + char detail[64]; + std::snprintf(detail, sizeof(detail), " (register 0x%x = 0x%08x)", offset, value); + throw std::runtime_error(std::string(name) + " is unsupported" + detail); + } } VkBlendFactor blendFactor(std::uint32_t value) { @@ -62,6 +70,82 @@ VkBlendOp blendOp(std::uint32_t value) { } } +struct DecodedColorFormat { + VkFormat format; + std::uint32_t elementBytes; +}; + +// CB_COLOR_INFO FORMAT / NUMBER_TYPE / COMP_SWAP to a Vulkan attachment format. AMD formats list +// components from the least significant bits, as Vulkan's non-packed formats do. +DecodedColorFormat DecodeColorFormat(std::uint32_t format, std::uint32_t number, std::uint32_t swap) { + constexpr std::uint32_t unorm = 0, sint = 5, srgb = 6, floating = 7; + constexpr std::uint32_t uint = 4; + const auto fail = [&]() -> DecodedColorFormat { + throw std::runtime_error("AGC graphics: unsupported color format " + std::to_string(format) + " number type " + std::to_string(number) + " component swap " + std::to_string(swap)); + }; + const bool alternate = swap == 1; + if (swap > 1) return fail(); + switch (format) { + case 1: + if (swap != 0) return fail(); + if (number == unorm) return {VK_FORMAT_R8_UNORM, 1}; + if (number == uint) return {VK_FORMAT_R8_UINT, 1}; + return fail(); + case 2: + if (swap != 0) return fail(); + if (number == unorm) return {VK_FORMAT_R16_UNORM, 2}; + if (number == floating) return {VK_FORMAT_R16_SFLOAT, 2}; + if (number == uint) return {VK_FORMAT_R16_UINT, 2}; + return fail(); + case 3: + if (swap != 0) return fail(); + if (number == unorm) return {VK_FORMAT_R8G8_UNORM, 2}; + return fail(); + case 4: + if (swap != 0) return fail(); + if (number == floating) return {VK_FORMAT_R32_SFLOAT, 4}; + if (number == uint) return {VK_FORMAT_R32_UINT, 4}; + if (number == sint) return {VK_FORMAT_R32_SINT, 4}; + return fail(); + case 5: + if (swap != 0) return fail(); + if (number == floating) return {VK_FORMAT_R16G16_SFLOAT, 4}; + if (number == unorm) return {VK_FORMAT_R16G16_UNORM, 4}; + if (number == uint) return {VK_FORMAT_R16G16_UINT, 4}; + return fail(); + case 6: + // COLOR_10_11_11: red in the low 11 bits, the Vulkan B10G11R11 packing. + if (swap != 0 || number != floating) return fail(); + return {VK_FORMAT_B10G11R11_UFLOAT_PACK32, 4}; + case 9: + // COLOR_2_10_10_10 keeps red in the low bits, the Vulkan A2B10G10R10 packing. + if (number != unorm) return fail(); + return {alternate ? VK_FORMAT_A2R10G10B10_UNORM_PACK32 : VK_FORMAT_A2B10G10R10_UNORM_PACK32, 4}; + case 10: + if (number == unorm) return {alternate ? VK_FORMAT_B8G8R8A8_UNORM : VK_FORMAT_R8G8B8A8_UNORM, 4}; + if (number == srgb) return {alternate ? VK_FORMAT_B8G8R8A8_SRGB : VK_FORMAT_R8G8B8A8_SRGB, 4}; + return fail(); + case 11: + if (swap != 0) return fail(); + if (number == floating) return {VK_FORMAT_R32G32_SFLOAT, 8}; + if (number == uint) return {VK_FORMAT_R32G32_UINT, 8}; + return fail(); + case 12: + if (swap != 0) return fail(); + if (number == floating) return {VK_FORMAT_R16G16B16A16_SFLOAT, 8}; + if (number == unorm) return {VK_FORMAT_R16G16B16A16_UNORM, 8}; + if (number == uint) return {VK_FORMAT_R16G16B16A16_UINT, 8}; + return fail(); + case 14: + if (swap != 0) return fail(); + if (number == floating) return {VK_FORMAT_R32G32B32A32_SFLOAT, 16}; + if (number == uint) return {VK_FORMAT_R32G32B32A32_UINT, 16}; + return fail(); + default: + return fail(); + } +} + void intersect(VkRect2D& result, const Registers& registers, std::uint32_t offset, bool screen) { const auto tl = read(registers, offset); const auto br = read(registers, offset + 1); @@ -95,6 +179,7 @@ ShaderStages DecodeShaderStages(const QueueState& queue) { validate(!tessellation || !geometry, "combined tessellation and geometry is unsupported by the reference path"); const auto path = tessellation ? ShaderPath::Tessellation : geometry ? ShaderPath::Geometry : ShaderPath::Vertex; ShaderStages result{path, value, (value & 0x00400000u) != 0 ? 32u : 64u, (read(queue.context, 0x1b6) & 0x8000u) != 0 ? 32u : 64u, {}, {}}; + APS5_LOG_OUT_DEBUG("DecodeShaderStages value=0x%x primitive=%u path=%u vertexWave=%u", value, primitive, static_cast(result.path), result.vertexWaveSize); if (path == ShaderPath::Vertex) { validate((value & 0x2000u) != 0, "legacy vertex routing without PRIMGEN_EN is unsupported"); validate((value & ~0x02402010u) == 0, "unsupported vertex routing, scheduling or wave-ID state"); @@ -132,6 +217,7 @@ State DecodeState(const QueueState& queue) { State result{}; result.stages = DecodeShaderStages(queue); const auto primitive = read(queue.userConfig, 0x242, "user-config"); + APS5_LOG_OUT_DEBUG("DecodeState primitive=%u path=%u vertexWave=%u", primitive, static_cast(result.stages.path), result.stages.vertexWaveSize); switch (primitive) { case 1: Require(result.stages.mesh.has_value(), "point-list vertex rendering requires point-size output support"); result.topology = VK_PRIMITIVE_TOPOLOGY_POINT_LIST; break; case 2: result.topology = VK_PRIMITIVE_TOPOLOGY_LINE_LIST; break; @@ -147,10 +233,42 @@ State DecodeState(const QueueState& queue) { case 6: result.topology = VK_PRIMITIVE_TOPOLOGY_TRIANGLE_STRIP; break; default: throw std::runtime_error("AGC graphics: unsupported primitive type " + std::to_string(primitive)); } + APS5_LOG_OUT_DEBUG("Topology=%u", static_cast(result.topology)); zero(queue.userConfig, 0x24b, ~0u, "primitive restart (GE_MULTI_PRIM_IB_RESET_EN)", "user-config"); - zero(cx, 0x207, ~0u, "clip distances, layer, viewport or auxiliary vertex exports"); - zero(cx, 0x200, ~0x007007f0u, "depth, stencil or conditional color writes"); - zero(cx, 0x203, ~0x00009870u, "depth export, shader coverage or ordered fragment execution"); + // Render target index, viewport index and the misc export vector that carries them are accepted but + // not routed: color targets are single-layer, so layered draws land in layer 0. + constexpr std::uint32_t layerExports = (1u << 18u) | (1u << 19u) | (1u << 21u) | (1u << 24u); + if ((read(cx, 0x207, "context") & layerExports) != 0) { + static bool reported = false; + if (!reported) { + reported = true; + std::fprintf(stderr, "[gpu] layer/viewport index vertex exports are ignored (PA_CL_VS_OUT_CNTL=0x%08x)\n", read(cx, 0x207, "context")); + } + } + zero(cx, 0x207, ~layerExports, "clip distances, layer, viewport or auxiliary vertex exports"); + // DB_DEPTH_CONTROL: depth and stencil tests whose function is ALWAYS and that write no depth cannot + // change color output, so they are accepted without a depth target (stencil writes are dropped). + { + const auto depthControl = read(cx, 0x200, "context"); + const bool stencil = (depthControl & 1u) != 0; + const bool depth = (depthControl & 2u) != 0; + const bool depthWrite = (depthControl & 4u) != 0; + const bool passThroughDepth = !depth || (!depthWrite && ((depthControl >> 4u) & 7u) == 7u); + const bool passThroughStencil = !stencil || (((depthControl >> 8u) & 7u) == 7u && ((depthControl & 0x80u) == 0 || ((depthControl >> 20u) & 7u) == 7u)); + if ((stencil || depth) && passThroughDepth && passThroughStencil) { + static bool reported = false; + if (!reported) { + reported = true; + std::fprintf(stderr, "[gpu] always-pass depth/stencil state is rendered without a depth target (DB_DEPTH_CONTROL=0x%08x)\n", depthControl); + } + } else { + zero(cx, 0x200, ~0x007007f0u, "depth, stencil or conditional color writes"); + } + Require((depthControl & 0xc0000008u) == 0, "depth bounds or depth-conditional color writes are unsupported"); + } + // EXEC_ON_HIER_FAIL / EXEC_ON_NOOP / EXEC_IF_OVERLAPPED (bits 9, 10, 17) only force the pixel shader + // to run, which it always does here. + zero(cx, 0x203, ~(0x00009870u | 0x00020600u), "depth export, shader coverage or ordered fragment execution"); zero(cx, 0x2dc, ~0x0001ff00u, "alpha-to-coverage"); zero(cx, 0x2f8, ~0u, "multisampling or coverage conversion"); zero(cx, 0x292, ~2u, "scan conversion mode"); @@ -168,53 +286,120 @@ State DecodeState(const QueueState& queue) { message << "AGC graphics: PA_CL_VTE_CNTL=0x" << std::hex << viewportControl << ": expected 0x43f for homogeneous positions and all viewport transforms; pre-divided coordinates, reciprocal W or disabled transforms are unsupported"; throw std::runtime_error(message.str()); } - zero(cx, 0x204, ~0x80000u, "unsupported PA_CL_CLIP_CNTL flags"); + // Bits 26/27 (ZCLIP_NEAR/FAR_DISABLE) become depth clamping; bit 19 selects the [0, 1] clip space. + zero(cx, 0x204, ~(0x80000u | 0x0c000000u), "unsupported PA_CL_CLIP_CNTL flags"); + result.depthClamp = (read(cx, 0x204) & 0x0c000000u) != 0; result.negativeOneToOne = (read(cx, 0x204) & 0x80000u) == 0; const auto raster = read(cx, 0x205); - Require((raster & ~0x7u) == 0 || (raster & ~0x7u) == 0x240u, "polygon mode, depth bias, provoking vertex or nonstandard rasterization is unsupported"); + // Bits 5-10 give the front/back polygon type (2 = filled triangles), which POLY_MODE (bit 3) turns on + // explicitly; KEEP_TOGETHER_ENABLE (bit 24) only affects primitive distribution across the chip. + const auto rasterMode = raster & ~0x7u & ~(1u << 24u); + Require(rasterMode == 0 || rasterMode == 0x240u || rasterMode == 0x248u, "polygon mode, depth bias, provoking vertex or nonstandard rasterization is unsupported"); result.cullMode = ((raster & 1u) != 0 ? VK_CULL_MODE_FRONT_BIT : 0u) | ((raster & 2u) != 0 ? VK_CULL_MODE_BACK_BIT : 0u); if (result.rectList) result.cullMode = VK_CULL_MODE_NONE; result.frontFace = (raster & 4u) != 0 ? VK_FRONT_FACE_CLOCKWISE : VK_FRONT_FACE_COUNTER_CLOCKWISE; - const auto targetMask = read(cx, 0x8e); + APS5_LOG_OUT_DEBUG("Raster=0x%x cullMode=0x%x frontFace=%u negativeOneToOne=%u", raster, static_cast(result.cullMode), static_cast(result.frontFace), result.negativeOneToOne ? 1u : 0u); const auto shaderMask = read(cx, 0x8f); - if ((targetMask & ~0xfu) != 0 || (shaderMask & ~0xfu) != 0) { - std::ostringstream message; - message << "AGC graphics: only color target zero is supported: CB_TARGET_MASK=0x" << std::hex << targetMask << ", CB_SHADER_MASK=0x" << shaderMask; - throw std::runtime_error(message.str()); + // Channels of targets the pixel shader does not export are never written, so the target mask only + // matters where the shader exports. + const auto targetMask = read(cx, 0x8e) & shaderMask; + APS5_LOG_OUT_DEBUG("CB_TARGET_MASK=0x%x CB_SHADER_MASK=0x%x", targetMask, shaderMask); + // MRT slots 0..n-1 become attachments 0..n-1; a slot between written slots must be written too. + std::uint32_t slotCount = 0; + for (std::uint32_t slot = 0; slot < 8; ++slot) { + if (((targetMask >> (4u * slot)) & 0xfu) != 0) slotCount = slot + 1; } - result.hasColorTarget = targetMask != 0; - Require(!result.hasColorTarget || shaderMask == 0xfu, "partial shader color exports are unsupported"); - Require(read(cx, 0x202) == 0xcc0010u, "only normal color rendering with copy ROP is supported"); + for (std::uint32_t slot = 0; slot < slotCount; ++slot) { + if (((targetMask >> (4u * slot)) & 0xfu) == 0) { + std::ostringstream message; + message << "AGC graphics: color targets with gaps are unsupported: CB_TARGET_MASK=0x" << std::hex << targetMask << ", CB_SHADER_MASK=0x" << shaderMask; + throw std::runtime_error(message.str()); + } + } + result.hasColorTarget = slotCount != 0; + APS5_LOG_OUT_DEBUG("hasColorTarget=%u slots=%u", result.hasColorTarget ? 1u : 0u, slotCount); + + // CB_COLOR_CONTROL mode 0 disables color writes, which only matters when a target is written. + Require(read(cx, 0x202) == 0xcc0010u || (!result.hasColorTarget && (read(cx, 0x202) & ~0x70u) == 0xcc0000u), "only normal color rendering with copy ROP is supported"); zero(cx, 0x1c4, ~0u, "depth or sample-mask export"); const auto exportFormat = read(cx, 0x1c5); - Require(exportFormat == 4 || exportFormat == 9, "only FP16_ABGR or 32_ABGR color export is supported"); - Require(read(cx, 0x1c3) == 4, "additional position exports are unsupported"); - if (result.hasColorTarget) { - const auto info = read(cx, 0x31c); + APS5_LOG_OUT_DEBUG("Export format=%u", exportFormat); + // SPI_SHADER_POS_FORMAT: POS0 must be a 4-component position; later vectors carry the misc/clip + // exports that PA_CL_VS_OUT_CNTL validation above already limits to ignored layer/viewport data. + Require((read(cx, 0x1c3) & 0xfu) == 4, "additional position exports are unsupported"); + for (std::uint32_t slot = 0; slot < slotCount; ++slot) { + // Export formats only matter for the targets the draw writes. + const auto slotExport = (exportFormat >> (4u * slot)) & 0xfu; + Require(slotExport == 4 || slotExport == 9, "only FP16_ABGR or 32_ABGR color export is supported"); + const auto stride = slot * 0xfu; + ColorTarget color{}; + const auto info = read(cx, 0x31c + stride); const auto number = (info >> 8u) & 7u; const auto swap = (info >> 11u) & 3u; - Require(((info >> 2u) & 0x1fu) == 10 && (number == 0 || number == 6) && swap <= 1, "unsupported color format or component order"); - Require((info & ~0x00029f7cu) == 0, "color compression, DCC, endian conversion, nonstandard rounding or color optimization is unsupported"); - Require((info & 0x8000u) != 0, "unclamped normalized color is unsupported"); - zero(cx, 0x31b, ~0u, "color mip or array view"); - zero(cx, 0x31d, ~0u, "color samples, fragments or destination alpha override"); - const auto attrib2 = read(cx, 0x3b0); - Require((attrib2 >> 28u) == 0, "mipmapped render targets are unsupported"); - const auto attrib3 = read(cx, 0x3b8); - result.color.tileMode = DecodeColorTileMode(attrib3); - result.color.extent = {((attrib2 >> 14u) & 0x3fffu) + 1u, (attrib2 & 0x3fffu) + 1u}; - const ColorTargetLayout colorLayout(result.color.extent.width, result.color.extent.height, result.color.tileMode); - const auto high = read(cx, 0x390); + APS5_LOG_OUT_DEBUG("Color %u info=0x%x number=%u swap=%u", slot, info, number, swap); + const auto format = (info >> 2u) & 0x1fu; + const auto decoded = DecodeColorFormat(format, number, swap); + // ROUND_MODE (bit 18) only affects unorm rounding. With DCC_ENABLE (bit 28) the target is written + // uncompressed and only its fast-clear keys matter (see DccMetadata.hpp). + Require((info & ~(0x00029f7cu | 0x00040000u | 0x10000000u)) == 0, "color compression, DCC, endian conversion, nonstandard rounding or color optimization is unsupported"); + Require((info & 0x8000u) != 0 || number == 7, "unclamped normalized color is unsupported"); + // CB_COLOR_VIEW: MIP_LEVEL (bits 24-27) selects the rendered mip; array slices are not modeled. + const auto view = read(cx, 0x31b + stride); + Require((view & ~0x0f000000u) == 0, "color array views are unsupported"); + const auto viewMip = (view >> 24u) & 0xfu; + zero(cx, 0x31d + stride, ~0u, "color samples, fragments or destination alpha override"); + const auto attrib2 = read(cx, 0x3b0 + slot); + const auto maxMip = attrib2 >> 28u; + Require(viewMip <= maxMip, "color view mip exceeds the surface"); + const auto attrib3 = read(cx, 0x3b8 + slot); + color.tileMode = DecodeColorTileMode(attrib3); + color.extent = {((attrib2 >> 14u) & 0x3fffu) + 1u, (attrib2 & 0x3fffu) + 1u}; + color.elementBytes = decoded.elementBytes; + std::uint64_t mipOffset = 0; + if (maxMip != 0) { + // A mipmapped surface is addressed like a texture; the view renders into one mip of it. + const auto mips = ComputeElementMipLayout(color.tileMode == ColorTileMode::Linear ? TextureTileMode::kLinear : TextureTileMode::kR64KBX, color.elementBytes, color.extent.width, color.extent.height, maxMip + 1u); + const auto& mip = mips.at(viewMip); + Require(!mip.tail, "rendering into a packed mip tail is unsupported"); + mipOffset = mip.tiledOffset; + color.extent = {mip.width, mip.height}; + } + const ColorTargetLayout colorLayout(color.extent.width, color.extent.height, color.tileMode, color.elementBytes); + const auto high = read(cx, 0x390 + slot); Require((high & ~0xffu) == 0, "invalid color address extension"); - result.color.address = (static_cast(high) << 40u) | (static_cast(read(cx, 0x318)) << 8u); - result.color.bytes = colorLayout.Bytes(); - GuestMemory::CheckGpuRange(reinterpret_cast(result.color.address), result.color.bytes, colorLayout.Alignment(), true); - result.color.format = swap == 0 ? (number == 0 ? VK_FORMAT_R8G8B8A8_UNORM : VK_FORMAT_R8G8B8A8_SRGB) : (number == 0 ? VK_FORMAT_B8G8R8A8_UNORM : VK_FORMAT_B8G8R8A8_SRGB); - result.color.componentMapping = 0xe4u; - result.renderExtent = result.color.extent; + color.address = ((static_cast(high) << 40u) | (static_cast(read(cx, 0x318 + stride)) << 8u)) + mipOffset; + color.bytes = colorLayout.Bytes(); + GuestMemory::CheckRange(reinterpret_cast(color.address), color.bytes, colorLayout.Alignment(), true); + color.format = decoded.format; + color.componentMapping = 0xe4u; + if ((info & 0x10000000u) != 0) { + if (maxMip == 0) { + const auto dccHigh = cx.find(0x3a8 + slot); + color.dccAddress = ((dccHigh == cx.end() ? 0ull : static_cast(dccHigh->second & 0xffu)) << 40u) | (static_cast(read(cx, 0x325 + stride)) << 8u); + color.dccAlphaOnMsb = DccAlphaOnMsb(color.format, swap); + } else { + static bool reported = false; + if (!reported) { + reported = true; + std::fprintf(stderr, "[gpu] DCC keys of mipmapped color targets are ignored\n"); + } + } + } + APS5_LOG_OUT_DEBUG("Color %u address=0x%llx extent=%ux%u bytes=%llu VkFormat=%u", slot, static_cast(color.address), color.extent.width, color.extent.height, static_cast(color.bytes), static_cast(color.format)); + if (slot == 0) { + result.renderExtent = color.extent; + } else { + result.renderExtent = {std::min(result.renderExtent.width, color.extent.width), std::min(result.renderExtent.height, color.extent.height)}; + } + result.colors.push_back(color); + } + if (result.hasColorTarget) { + result.color = result.colors.front(); } else { const auto screenBottomRight = read(cx, 0xd); + APS5_LOG_OUT_DEBUG("No color target, screen BR register=0x%x", screenBottomRight); result.renderExtent = {screenBottomRight & 0xffffu, screenBottomRight >> 16u}; + APS5_LOG_OUT_DEBUG("Render extent from screen=%ux%u", result.renderExtent.width, result.renderExtent.height); Require(result.renderExtent.width != 0 && result.renderExtent.height != 0, "empty framebuffer extent for a draw without color writes"); } const auto xs = readFloat(cx, 0x10f); @@ -225,6 +410,7 @@ State DecodeState(const QueueState& queue) { const auto zo = readFloat(cx, 0x114); const auto minDepth = result.negativeOneToOne ? zo - zs : zo; const auto maxDepth = zo + zs; + APS5_LOG_OUT_DEBUG("Viewport transform scale=(%f,%f,%f) offset=(%f,%f,%f) depth=(%f,%f)", xs, ys, zs, xo, yo, zo, minDepth, maxDepth); if (!(xs > 0 && ys != 0 && std::isfinite(minDepth) && std::isfinite(maxDepth))) { std::ostringstream message; message << "AGC graphics: unsupported viewport transform: scale=(" << xs << ", " << ys << ", " << zs << "), offset=(" << xo << ", " << yo << ", " << zo << "), depth=(" << minDepth << ", " << maxDepth << "), negativeOneToOne=" << result.negativeOneToOne; @@ -232,28 +418,35 @@ State DecodeState(const QueueState& queue) { } Require(readFloat(cx, 0xb4) <= readFloat(cx, 0xb5), "inverted viewport depth clamp bounds"); result.viewport = {xo - xs, yo - ys, 2 * xs, 2 * ys, minDepth, maxDepth}; + APS5_LOG_OUT_DEBUG("Viewport x=%f y=%f w=%f h=%f minDepth=%f maxDepth=%f", result.viewport.x, result.viewport.y, result.viewport.width, result.viewport.height, result.viewport.minDepth, result.viewport.maxDepth); result.scissor = {{0, 0}, result.renderExtent}; intersect(result.scissor, cx, 0xc, true); intersect(result.scissor, cx, 0x81, false); intersect(result.scissor, cx, 0x90, false); if ((read(cx, 0x292) & 2u) != 0) intersect(result.scissor, cx, 0x94, false); - if (result.hasColorTarget) { - const auto blend = read(cx, 0x1e0); + APS5_LOG_OUT_DEBUG("Scissor offset=(%d,%d) extent=%ux%u", result.scissor.offset.x, result.scissor.offset.y, result.scissor.extent.width, result.scissor.extent.height); + for (std::uint32_t slot = 0; slot < result.colors.size(); ++slot) { + const auto blend = read(cx, 0x1e0 + slot); + APS5_LOG_OUT_DEBUG("Blend %u control=0x%x", slot, blend); Require((blend & 0x0000e000u) == 0, "reserved blend control bits"); - result.blend.colorWriteMask = targetMask; - result.blend.blendEnable = (blend >> 30u) & 1u; - if (result.blend.blendEnable) { - Require((read(cx, 0x31c) & 0x10000u) == 0, "blend bypass conflicts with enabled blending"); - result.blend.srcColorBlendFactor = blendFactor(blend & 0x1fu); - result.blend.dstColorBlendFactor = blendFactor((blend >> 8u) & 0x1fu); - result.blend.colorBlendOp = blendOp((blend >> 5u) & 7u); + VkPipelineColorBlendAttachmentState state{}; + state.colorWriteMask = (targetMask >> (4u * slot)) & 0xfu; + state.blendEnable = (blend >> 30u) & 1u; + if (state.blendEnable) { + Require((read(cx, 0x31c + slot * 0xfu) & 0x10000u) == 0, "blend bypass conflicts with enabled blending"); + state.srcColorBlendFactor = blendFactor(blend & 0x1fu); + state.dstColorBlendFactor = blendFactor((blend >> 8u) & 0x1fu); + state.colorBlendOp = blendOp((blend >> 5u) & 7u); const auto alpha = (blend & 0x20000000u) != 0 ? blend >> 16u : blend; - result.blend.srcAlphaBlendFactor = blendFactor(alpha & 0x1fu); - result.blend.dstAlphaBlendFactor = blendFactor((alpha >> 8u) & 0x1fu); - result.blend.alphaBlendOp = blendOp((alpha >> 5u) & 7u); + state.srcAlphaBlendFactor = blendFactor(alpha & 0x1fu); + state.dstAlphaBlendFactor = blendFactor((alpha >> 8u) & 0x1fu); + state.alphaBlendOp = blendOp((alpha >> 5u) & 7u); for (std::uint32_t i = 0; i < 4; ++i) result.blendConstants[i] = readFloat(cx, 0x105 + i); } + result.blends.push_back(state); } + if (!result.blends.empty()) result.blend = result.blends.front(); + APS5_LOG_OUT_DEBUG("DecodeState done colorTarget=%u render=%ux%u topology=%u", result.hasColorTarget ? 1u : 0u, result.renderExtent.width, result.renderExtent.height, static_cast(result.topology)); return result; } diff --git a/core/libs/prx/libSceAgcDriver/Graphics/src/Texture.cpp b/core/libs/prx/libSceAgcDriver/Graphics/src/Texture.cpp index 35bf09e0d..890e9dbcf 100644 --- a/core/libs/prx/libSceAgcDriver/Graphics/src/Texture.cpp +++ b/core/libs/prx/libSceAgcDriver/Graphics/src/Texture.cpp @@ -1,68 +1,103 @@ #include "prx/libSceAgcDriver/Graphics/include/Texture.hpp" +#include "prx/libc/include/General.hpp" #include "prx/libSceAgcDriver/Graphics/include/Resources.hpp" -#include "prx/libSceAgcDriver/Graphics/include/DrawQueue.hpp" +#include "prx/libSceAgcDriver/Graphics/include/Recorder.hpp" +#include "prx/libSceAgcDriver/Graphics/include/GuestBufferMemory.hpp" +// For the declaration of PendingStorageOverlaps, defined below. +#include "prx/libSceAgcDriver/Graphics/include/ShaderResources.hpp" #include "prx/libSceAgcDriver/Graphics/include/TextureFormat.hpp" #include "prx/libSceAgcDriver/Graphics/include/TextureTiling.hpp" #include "prx/libSceAgcDriver/Execution/include/GuestMemory.hpp" #include +#include +#include #include #include +#include +#include #include #include +#include +#include +#include +#include +#include +#include namespace AgcDriver::Graphics { namespace { +// APS5_PROFILE_DRAW: accumulate texture setup phases and report every 200 textures. +struct TextureProfile { + double allocate = 0, read = 0, gpu = 0, view = 0; + std::uint64_t count = 0; + std::uint64_t fromStorage = 0; + double storageCreate = 0, storageWriteBack = 0, storageAlloc = 0, storageHostCopy = 0, storageGpu = 0, storageStore = 0; + std::uint64_t storageCount = 0; + std::uint64_t storageBytes = 0; + std::uint64_t storageReused = 0; + std::uint64_t storageDirectUploads = 0; + std::uint64_t storageDirectWriteBacks = 0; + // Sampled-texture uploads recorded into the open batch instead of waited for. + std::uint64_t recordedUploads = 0; +}; -std::uint32_t FullArrayLayers(const GuestTextureResource& descriptor) { - switch (descriptor.dimension) { - case TextureDimension::k1D: - case TextureDimension::k2D: return 1u; - case TextureDimension::k2DArray: - case TextureDimension::kCube: return descriptor.depthOrLastArray + 1u; - } - throw std::runtime_error("AGC graphics: Texture encountered an unknown guest texture dimension"); +TextureProfile& Profile() { + static TextureProfile profile; + return profile; } +struct PhaseTimer { + std::chrono::steady_clock::time_point last = std::chrono::steady_clock::now(); + double lap() { + const auto now = std::chrono::steady_clock::now(); + const auto ms = std::chrono::duration(now - last).count(); + last = now; + return ms; + } +}; + + VkImageType ImageTypeFor(TextureDimension dimension) { - return dimension == TextureDimension::k1D ? VK_IMAGE_TYPE_1D : VK_IMAGE_TYPE_2D; + return dimension == TextureDimension::k1D ? VK_IMAGE_TYPE_1D : dimension == TextureDimension::k3D ? VK_IMAGE_TYPE_3D : VK_IMAGE_TYPE_2D; } -VkImageViewType ViewTypeFor(TextureDimension dimension, std::uint32_t viewLayerCount) { +VkImageViewType ViewTypeFor(TextureDimension dimension, [[maybe_unused]] std::uint32_t viewLayerCount) { switch (dimension) { case TextureDimension::k1D: return VK_IMAGE_VIEW_TYPE_1D; case TextureDimension::k2D: return VK_IMAGE_VIEW_TYPE_2D; case TextureDimension::k2DArray: return VK_IMAGE_VIEW_TYPE_2D_ARRAY; - case TextureDimension::kCube: return viewLayerCount == 6u ? VK_IMAGE_VIEW_TYPE_CUBE : VK_IMAGE_VIEW_TYPE_CUBE_ARRAY; + // Shaders address cube maps as 2D arrays of faces. + case TextureDimension::kCube: return VK_IMAGE_VIEW_TYPE_2D_ARRAY; + case TextureDimension::k3D: return VK_IMAGE_VIEW_TYPE_3D; } throw std::runtime_error("AGC graphics: Texture encountered an unknown guest texture dimension"); } -std::uint64_t SliceLinearBytes(const std::vector& mips) { - std::uint64_t bytes = 0; - for (const auto& mip : mips) bytes = std::max(bytes, mip.linearOffset + mip.linearSize); - return bytes; -} - } Texture::Texture(const Context& context, TextureDetiler& detiler, const GuestTextureResource& descriptor, VkComponentMapping components, std::span snapshot) : context(context) { + static const bool profile = std::getenv("APS5_PROFILE_DRAW") != nullptr; + PhaseTimer timer; try { const auto vkFormat = ResolveTextureFormat(descriptor.format); if (IsBlockCompressed(descriptor.format)) { Require(context.textureCompressionBC, "device does not support BC compressed textures"); } - const auto mips = ComputeMipLayout(descriptor.tileMode, descriptor.format, descriptor.width, descriptor.height, descriptor.mipCount); - const auto arrayLayers = FullArrayLayers(descriptor); + const auto geometry = DescribeSurface(descriptor); + const auto& mips = geometry.mips; + const auto arrayLayers = geometry.layers; const auto elementBytes = BytesPerElement(descriptor.format); + APS5_LOG_OUT("Texture address=0x%llx %ux%u mips=%u layers=%u dim=%d tile=%d format=%u vk=%d element=%u", static_cast(descriptor.baseAddress), descriptor.width, descriptor.height, descriptor.mipCount, arrayLayers, + static_cast(descriptor.dimension), static_cast(descriptor.tileMode), descriptor.format, static_cast(vkFormat), elementBytes); + for (const auto& mip : mips) APS5_LOG_OUT(" mip %ux%u tiled=0x%llx+0x%llx linear=0x%llx+0x%llx blocksPerRow=%u pitch=%u tail=%d", mip.width, mip.height, static_cast(mip.tiledOffset), static_cast(mip.tiledSize), static_cast(mip.linearOffset), static_cast(mip.linearSize), mip.blocksPerRow, mip.pitchBytes, mip.tail ? 1 : 0); - const auto guestBytes = ComputeSurfaceSize(mips, arrayLayers); - const auto guestSliceBytes = guestBytes / arrayLayers; + const auto guestBytes = geometry.guestBytes; Require(snapshot.size() == guestBytes, "texture snapshot size mismatch"); - const auto sliceLinearBytes = SliceLinearBytes(mips); + const auto sliceLinearBytes = geometry.sliceLinearBytes; Require(arrayLayers == 0 || sliceLinearBytes <= UINT64_MAX / arrayLayers, "detiled texture buffer size overflows"); const auto linearBytes = sliceLinearBytes * arrayLayers; @@ -70,15 +105,16 @@ Texture::Texture(const Context& context, TextureDetiler& detiler, const GuestTex imageInfo.flags = descriptor.dimension == TextureDimension::kCube ? VK_IMAGE_CREATE_CUBE_COMPATIBLE_BIT : 0u; imageInfo.imageType = ImageTypeFor(descriptor.dimension); imageInfo.format = vkFormat; - imageInfo.extent = {descriptor.width, descriptor.height, 1u}; + imageInfo.extent = {descriptor.width, descriptor.height, geometry.imageDepth}; imageInfo.mipLevels = descriptor.mipCount; - imageInfo.arrayLayers = arrayLayers; + imageInfo.arrayLayers = geometry.imageLayers; imageInfo.samples = VK_SAMPLE_COUNT_1_BIT; imageInfo.tiling = VK_IMAGE_TILING_OPTIMAL; imageInfo.usage = VK_IMAGE_USAGE_TRANSFER_DST_BIT | VK_IMAGE_USAGE_SAMPLED_BIT; imageInfo.sharingMode = VK_SHARING_MODE_EXCLUSIVE; imageInfo.initialLayout = VK_IMAGE_LAYOUT_UNDEFINED; Check(context.Function("vkCreateImage")(context.device, &imageInfo, nullptr, &image), "vkCreateImage"); + owned = std::make_shared(context, image, VK_NULL_HANDLE); VkMemoryRequirements requirements{}; context.Function("vkGetImageMemoryRequirements")(context.device, image, &requirements); @@ -86,45 +122,61 @@ Texture::Texture(const Context& context, TextureDetiler& detiler, const GuestTex allocation.allocationSize = requirements.size; allocationBytes = requirements.size; allocation.memoryTypeIndex = context.MemoryType(requirements.memoryTypeBits, VK_MEMORY_PROPERTY_DEVICE_LOCAL_BIT); - Check(context.Function("vkAllocateMemory")(context.device, &allocation, nullptr, &memory), "vkAllocateMemory texture"); - Check(context.Function("vkBindImageMemory")(context.device, image, memory, 0), "vkBindImageMemory"); + Check(context.Function("vkAllocateMemory")(context.device, &allocation, nullptr, &owned->memory), "vkAllocateMemory texture"); + Check(context.Function("vkBindImageMemory")(context.device, image, owned->memory, 0), "vkBindImageMemory"); { - Buffer staging(context, static_cast(guestBytes), VK_BUFFER_USAGE_STORAGE_BUFFER_BIT); - std::memcpy(staging.Bytes().data(), snapshot.data(), snapshot.size()); - Buffer linear(context, static_cast(linearBytes), VK_BUFFER_USAGE_STORAGE_BUFFER_BIT | VK_BUFFER_USAGE_TRANSFER_SRC_BIT); + // Debug aid: APS5_DUMP_TEXTURE= saves the detiled first mip + // of those sampled textures on their first 8 uploads as texture_
_.raw (u32 + // width, height, VkFormat, then tightly packed rows). + static const std::string dumpList = [] { const char* text = std::getenv("APS5_DUMP_TEXTURE"); return text ? std::string(text) : std::string(); }(); + bool dumpWanted = false; + if (!dumpList.empty()) { + char address[32]; + std::snprintf(address, sizeof(address), "%llx", static_cast(descriptor.baseAddress)); + char extent[32]; + std::snprintf(extent, sizeof(extent), "%ux%u", descriptor.width, descriptor.height); + // Entries may also be extents ("3840x2160"), since heap addresses change between runs. + dumpWanted = dumpList.find(address) != std::string::npos || dumpList.find(extent) != std::string::npos; + } + auto staging = std::make_shared(context, static_cast(guestBytes), VK_BUFFER_USAGE_TRANSFER_SRC_BIT); + std::memcpy(staging->Bytes().data(), snapshot.data(), snapshot.size()); + auto tiled = std::make_shared(context, static_cast(guestBytes), VK_BUFFER_USAGE_STORAGE_BUFFER_BIT | VK_BUFFER_USAGE_TRANSFER_DST_BIT); + auto linear = std::make_shared(context, static_cast(linearBytes), VK_BUFFER_USAGE_STORAGE_BUFFER_BIT | VK_BUFFER_USAGE_TRANSFER_SRC_BIT); + if (profile) Profile().allocate += timer.lap(); + if (profile) Profile().read += timer.lap(); detiler.BeginBatch(); - if (context.drawQueue) context.drawQueue->Flush(); - CommandBatch batch(context); - const auto commands = batch.Handle(); + // The upload is recorded into the open batch, like a storage image's: the work that samples + // the texture is recorded after it, and a batch of its own (SubmitAndWait) was a full GPU + // drain under the device lock for every new texture. The buffers and the image live with + // the batch (the cache may drop the texture before it completes). The dump aid needs the + // linear bytes on the CPU, so it keeps the waiting batch, as does a build without a + // recorder (tests). APS5_SYNC_TEXTURE_UPLOAD=1 restores the waiting batch for every upload. + static const bool syncUploads = std::getenv("APS5_SYNC_TEXTURE_UPLOAD") != nullptr; + auto* recorder = dumpWanted || syncUploads ? nullptr : Recorder::Active(); + std::unique_ptr batch; + VkCommandBuffer commands = VK_NULL_HANDLE; + if (recorder != nullptr) { + commands = recorder->Commands(); + recorder->Keep(staging); + recorder->Keep(tiled); + recorder->Keep(linear); + recorder->Keep(owned); + } else { + batch = std::make_unique(context); + commands = batch->Handle(); + } - VkBufferMemoryBarrier stagingReadBarrier{VK_STRUCTURE_TYPE_BUFFER_MEMORY_BARRIER}; - stagingReadBarrier.srcAccessMask = VK_ACCESS_HOST_WRITE_BIT; - stagingReadBarrier.dstAccessMask = VK_ACCESS_SHADER_READ_BIT; - stagingReadBarrier.srcQueueFamilyIndex = VK_QUEUE_FAMILY_IGNORED; - stagingReadBarrier.dstQueueFamilyIndex = VK_QUEUE_FAMILY_IGNORED; - stagingReadBarrier.buffer = staging.Handle(); - stagingReadBarrier.offset = 0; - stagingReadBarrier.size = VK_WHOLE_SIZE; - - VkBufferMemoryBarrier linearWriteBarrier{VK_STRUCTURE_TYPE_BUFFER_MEMORY_BARRIER}; - linearWriteBarrier.srcAccessMask = 0; - linearWriteBarrier.dstAccessMask = VK_ACCESS_SHADER_WRITE_BIT; - linearWriteBarrier.srcQueueFamilyIndex = VK_QUEUE_FAMILY_IGNORED; - linearWriteBarrier.dstQueueFamilyIndex = VK_QUEUE_FAMILY_IGNORED; - linearWriteBarrier.buffer = linear.Handle(); - linearWriteBarrier.offset = 0; - linearWriteBarrier.size = VK_WHOLE_SIZE; - - const VkBufferMemoryBarrier preBarriers[] = {stagingReadBarrier, linearWriteBarrier}; - context.Function("vkCmdPipelineBarrier")(commands, VK_PIPELINE_STAGE_HOST_BIT, VK_PIPELINE_STAGE_COMPUTE_SHADER_BIT, 0, 0, nullptr, 2, preBarriers, 0, nullptr); + RecordMemoryBarrier(context, commands, VK_PIPELINE_STAGE_HOST_BIT, VK_PIPELINE_STAGE_TRANSFER_BIT, VK_ACCESS_HOST_WRITE_BIT, VK_ACCESS_TRANSFER_READ_BIT); + CopyBuffer(context, commands, staging->Handle(), 0, tiled->Handle(), 0, guestBytes); + RecordMemoryBarrier(context, commands, VK_PIPELINE_STAGE_TRANSFER_BIT, VK_PIPELINE_STAGE_COMPUTE_SHADER_BIT, VK_ACCESS_TRANSFER_WRITE_BIT, VK_ACCESS_SHADER_READ_BIT | VK_ACCESS_SHADER_WRITE_BIT); for (std::uint32_t layer = 0; layer < arrayLayers; ++layer) { - const auto guestLayerOffset = static_cast(layer) * guestSliceBytes; - const auto linearLayerOffset = static_cast(layer) * sliceLinearBytes; + const auto guestLayerOffset = geometry.GuestLayerOffset(layer); + const auto linearLayerOffset = geometry.LinearLayerOffset(layer); for (const auto& mip : mips) { - detiler.Dispatch(commands, descriptor.tileMode, elementBytes, staging.Handle(), guestLayerOffset + mip.tiledOffset, linear.Handle(), linearLayerOffset + mip.linearOffset, mip, layer); + detiler.Dispatch(commands, descriptor.tileMode, elementBytes, tiled->Handle(), guestLayerOffset + mip.tiledOffset, linear->Handle(), linearLayerOffset + mip.linearOffset, mip, false, layer, geometry.thick); } } @@ -133,7 +185,7 @@ Texture::Texture(const Context& context, TextureDetiler& detiler, const GuestTex linearReadBarrier.dstAccessMask = VK_ACCESS_TRANSFER_READ_BIT; linearReadBarrier.srcQueueFamilyIndex = VK_QUEUE_FAMILY_IGNORED; linearReadBarrier.dstQueueFamilyIndex = VK_QUEUE_FAMILY_IGNORED; - linearReadBarrier.buffer = linear.Handle(); + linearReadBarrier.buffer = linear->Handle(); linearReadBarrier.offset = 0; linearReadBarrier.size = VK_WHOLE_SIZE; @@ -145,7 +197,7 @@ Texture::Texture(const Context& context, TextureDetiler& detiler, const GuestTex toTransferDst.srcQueueFamilyIndex = VK_QUEUE_FAMILY_IGNORED; toTransferDst.dstQueueFamilyIndex = VK_QUEUE_FAMILY_IGNORED; toTransferDst.image = image; - toTransferDst.subresourceRange = {VK_IMAGE_ASPECT_COLOR_BIT, 0, descriptor.mipCount, 0, arrayLayers}; + toTransferDst.subresourceRange = {VK_IMAGE_ASPECT_COLOR_BIT, 0, descriptor.mipCount, 0, geometry.imageLayers}; context.Function("vkCmdPipelineBarrier")(commands, VK_PIPELINE_STAGE_COMPUTE_SHADER_BIT, VK_PIPELINE_STAGE_TRANSFER_BIT, 0, 0, nullptr, 1, &linearReadBarrier, 1, &toTransferDst); std::vector regions; @@ -156,15 +208,22 @@ Texture::Texture(const Context& context, TextureDetiler& detiler, const GuestTex const auto& mip = mips[level]; VkBufferImageCopy region{}; region.bufferOffset = linearLayerOffset + mip.linearOffset; - region.bufferRowLength = mip.pitchBytes / elementBytes * BlockWidth(descriptor.format); + region.bufferRowLength = mip.pitchBytes / BytesPerElement(descriptor.format) * BlockWidth(descriptor.format); region.bufferImageHeight = 0; - region.imageSubresource = {VK_IMAGE_ASPECT_COLOR_BIT, level, layer, 1}; - region.imageOffset = {0, 0, 0}; + region.imageSubresource = {VK_IMAGE_ASPECT_COLOR_BIT, level, geometry.CopyLayer(layer), 1}; + region.imageOffset = {0, 0, geometry.CopyDepth(layer)}; region.imageExtent = {std::max(descriptor.width >> level, 1u), std::max(descriptor.height >> level, 1u), 1u}; regions.push_back(region); } } - context.Function("vkCmdCopyBufferToImage")(commands, linear.Handle(), image, VK_IMAGE_LAYOUT_TRANSFER_DST_OPTIMAL, static_cast(regions.size()), regions.data()); + context.Function("vkCmdCopyBufferToImage")(commands, linear->Handle(), image, VK_IMAGE_LAYOUT_TRANSFER_DST_OPTIMAL, static_cast(regions.size()), regions.data()); + std::unique_ptr dump; + if (dumpWanted) { + dump = std::make_unique(context, static_cast(mips[0].linearSize), VK_BUFFER_USAGE_TRANSFER_DST_BIT); + RecordMemoryBarrier(context, commands, VK_PIPELINE_STAGE_TRANSFER_BIT, VK_PIPELINE_STAGE_TRANSFER_BIT, VK_ACCESS_TRANSFER_WRITE_BIT, VK_ACCESS_TRANSFER_READ_BIT); + CopyBuffer(context, commands, linear->Handle(), mips[0].linearOffset, dump->Handle(), 0, mips[0].linearSize); + RecordMemoryBarrier(context, commands, VK_PIPELINE_STAGE_TRANSFER_BIT, VK_PIPELINE_STAGE_HOST_BIT, VK_ACCESS_TRANSFER_WRITE_BIT, VK_ACCESS_HOST_READ_BIT); + } VkImageMemoryBarrier toShaderRead{VK_STRUCTURE_TYPE_IMAGE_MEMORY_BARRIER}; toShaderRead.srcAccessMask = VK_ACCESS_TRANSFER_WRITE_BIT; @@ -177,11 +236,30 @@ Texture::Texture(const Context& context, TextureDetiler& detiler, const GuestTex toShaderRead.subresourceRange = toTransferDst.subresourceRange; context.Function("vkCmdPipelineBarrier")(commands, VK_PIPELINE_STAGE_TRANSFER_BIT, VK_PIPELINE_STAGE_ALL_COMMANDS_BIT, 0, 0, nullptr, 0, nullptr, 1, &toShaderRead); - batch.SubmitAndWait(); + if (batch) batch->SubmitAndWait(); + else ++Profile().recordedUploads; + if (profile) Profile().gpu += timer.lap(); + if (dump) { + static std::mutex dumpMutex; + static std::map dumped; + std::lock_guard lock(dumpMutex); + auto& count = dumped[descriptor.baseAddress]; + if (count < 8) { + char name[64]; + std::snprintf(name, sizeof(name), "texture_%llx_%d.raw", static_cast(descriptor.baseAddress), count++); + if (std::FILE* file = std::fopen(name, "wb")) { + const std::uint32_t header[3] = {mips[0].pitchBytes / static_cast(elementBytes), mips[0].height, static_cast(vkFormat)}; + std::fwrite(header, sizeof(header), 1, file); + std::fwrite(dump->Bytes().data(), 1, dump->Bytes().size(), file); + std::fclose(file); + std::fprintf(stderr, "[texture] dumped %s (%ux%u VkFormat %d, tile %d)\n", name, header[0], header[1], static_cast(vkFormat), static_cast(descriptor.tileMode)); + } + } + } } - const auto viewLevelCount = descriptor.mipCount - descriptor.baseLevel; - const auto viewLayerCount = arrayLayers - descriptor.baseArray; + const auto viewLevelCount = std::min(descriptor.lastLevel, descriptor.mipCount - 1u) - descriptor.baseLevel + 1u; + const auto viewLayerCount = geometry.imageLayers - descriptor.baseArray; if (descriptor.dimension == TextureDimension::kCube) { Require(viewLayerCount % 6u == 0, "guest cube texture view does not contain a multiple of 6 array slices"); } @@ -193,6 +271,53 @@ Texture::Texture(const Context& context, TextureDetiler& detiler, const GuestTex viewInfo.components = components; viewInfo.subresourceRange = {VK_IMAGE_ASPECT_COLOR_BIT, descriptor.baseLevel, viewLevelCount, descriptor.baseArray, viewLayerCount}; Check(context.Function("vkCreateImageView")(context.device, &viewInfo, nullptr, &view), "vkCreateImageView"); + if (profile) { + auto& totals = Profile(); + totals.view += timer.lap(); + if (++totals.count % 200 == 0) std::fprintf(stderr, "[texture] %llu textures (%llu copied from storage images, %llu uploads recorded): allocate+image %.0f ms, guest read %.0f ms, detile+copy %.0f ms, view+buffer release %.0f ms\n", static_cast(totals.count), static_cast(totals.fromStorage), static_cast(totals.recordedUploads), totals.allocate, totals.read, totals.gpu, totals.view); + } + } catch (...) { + release(); + throw; + } +} + +bool Texture::CanCopyFrom(const StorageTexture& source, const GuestTextureResource& descriptor) { + const auto& from = source.Descriptor(); + if (IsBlockCompressed(descriptor.format) || IsBlockCompressed(from.format)) return false; + // Same memory, same layout, same texel size: the GPU copy reinterprets the texels exactly as a + // guest read through the sampled descriptor would. + return descriptor.baseAddress == from.baseAddress && descriptor.width == from.width && descriptor.height == from.height && descriptor.dimension == from.dimension && descriptor.tileMode == from.tileMode && descriptor.mipCount <= from.mipCount && descriptor.depthOrLastArray == from.depthOrLastArray && BytesPerElement(descriptor.format) == BytesPerElement(from.format) && BlockWidth(descriptor.format) == BlockWidth(from.format); +} + +Texture::Texture(const Context& context, const std::shared_ptr& source, const GuestTextureResource& descriptor, VkComponentMapping components) : context(context), storageSource(source) { + static const bool profile = std::getenv("APS5_PROFILE_DRAW") != nullptr; + PhaseTimer timer; + try { + Require(source != nullptr && CanCopyFrom(*source, descriptor), "storage image does not match the sampled texture"); + const auto vkFormat = ResolveTextureFormat(descriptor.format); + const auto geometry = DescribeSurface(descriptor); + APS5_LOG_OUT("Texture address=0x%llx %ux%u mips=%u viewed from storage image (vk=%d)", static_cast(descriptor.baseAddress), descriptor.width, descriptor.height, descriptor.mipCount, static_cast(vkFormat)); + // Storage images stay in the general layout; the view samples them there. + layout = VK_IMAGE_LAYOUT_GENERAL; + const auto viewLevelCount = std::min(descriptor.lastLevel, descriptor.mipCount - 1u) - descriptor.baseLevel + 1u; + const auto viewLayerCount = std::min(geometry.imageLayers, source->ImageLayers()) - descriptor.baseArray; + if (descriptor.dimension == TextureDimension::kCube) { + Require(viewLayerCount % 6u == 0, "guest cube texture view does not contain a multiple of 6 array slices"); + } + VkImageViewCreateInfo viewInfo{VK_STRUCTURE_TYPE_IMAGE_VIEW_CREATE_INFO}; + viewInfo.image = source->Image(); + viewInfo.viewType = ViewTypeFor(descriptor.dimension, viewLayerCount); + viewInfo.format = vkFormat; + viewInfo.components = components; + viewInfo.subresourceRange = {VK_IMAGE_ASPECT_COLOR_BIT, descriptor.baseLevel, viewLevelCount, descriptor.baseArray, viewLayerCount}; + Check(context.Function("vkCreateImageView")(context.device, &viewInfo, nullptr, &view), "vkCreateImageView storage view"); + if (profile) { + auto& totals = Profile(); + totals.view += timer.lap(); + ++totals.fromStorage; + if (++totals.count % 200 == 0) std::fprintf(stderr, "[texture] %llu textures (%llu viewed from storage images): allocate+image %.0f ms, guest read %.0f ms, detile+copy %.0f ms, view+buffer release %.0f ms\n", static_cast(totals.count), static_cast(totals.fromStorage), totals.allocate, totals.read, totals.gpu, totals.view); + } } catch (...) { release(); throw; @@ -206,12 +331,894 @@ Texture::~Texture() { void Texture::release() noexcept { upload.reset(); if (view) context.Function("vkDestroyImageView")(context.device, view, nullptr); - if (image) context.Function("vkDestroyImage")(context.device, image, nullptr); - if (memory) context.Function("vkFreeMemory")(context.device, memory, nullptr); + view = VK_NULL_HANDLE; + // The image and its memory go with the last holder: this texture, or the batch still uploading it. + owned.reset(); + image = VK_NULL_HANDLE; } VkImageView Texture::View() const { return view; } + +namespace { + +VkBufferMemoryBarrier WholeBufferBarrier(VkBuffer buffer, VkAccessFlags from, VkAccessFlags to) { + VkBufferMemoryBarrier barrier{VK_STRUCTURE_TYPE_BUFFER_MEMORY_BARRIER}; + barrier.srcAccessMask = from; + barrier.dstAccessMask = to; + barrier.srcQueueFamilyIndex = VK_QUEUE_FAMILY_IGNORED; + barrier.dstQueueFamilyIndex = VK_QUEUE_FAMILY_IGNORED; + barrier.buffer = buffer; + barrier.offset = 0; + barrier.size = VK_WHOLE_SIZE; + return barrier; +} + +// Storage images cannot use sRGB formats; the shader works on the raw encoded values either way. +// The answer is a property of the physical device, so it is computed once per format (every storage +// image lookup asks: a live vkGetPhysicalDeviceFormatProperties call each time was measurable) and +// remembered as VK_FORMAT_UNDEFINED when the format has no storage form. +VkFormat StorageFormatOrUndefined(const Context& context, VkFormat format) { + struct Table { + std::mutex mutex; + std::unordered_map formats; + }; + static Table table; + // The key names the physical device: the headless and the windowed device share one GPU, but a + // second GPU would answer differently. + const auto key = (static_cast(reinterpret_cast(context.physical)) << 20u) ^ static_cast(format); + { + std::lock_guard lock(table.mutex); + if (const auto found = table.formats.find(key); found != table.formats.end()) return found->second; + } + const auto supports = [&](VkFormat candidate) { + VkFormatProperties properties{}; + context.formatProperties(context.physical, candidate, &properties); + return (properties.optimalTilingFeatures & VK_FORMAT_FEATURE_STORAGE_IMAGE_BIT) != 0; + }; + VkFormat storage = VK_FORMAT_UNDEFINED; + if (supports(format)) { + storage = format; + } else { + VkFormat linear = VK_FORMAT_UNDEFINED; + switch (format) { + case VK_FORMAT_R8G8B8A8_SRGB: linear = VK_FORMAT_R8G8B8A8_UNORM; break; + case VK_FORMAT_B8G8R8A8_SRGB: linear = VK_FORMAT_B8G8R8A8_UNORM; break; + case VK_FORMAT_A8B8G8R8_SRGB_PACK32: linear = VK_FORMAT_A8B8G8R8_UNORM_PACK32; break; + case VK_FORMAT_R8_SRGB: linear = VK_FORMAT_R8_UNORM; break; + case VK_FORMAT_R8G8_SRGB: linear = VK_FORMAT_R8G8_UNORM; break; + default: break; + } + if (linear != VK_FORMAT_UNDEFINED && supports(linear)) storage = linear; + } + std::lock_guard lock(table.mutex); + table.formats.emplace(key, storage); + return storage; +} + +VkFormat StorageFormatFor(const Context& context, VkFormat format) { + const auto storage = StorageFormatOrUndefined(context, format); + Require(storage != VK_FORMAT_UNDEFINED, "guest storage texture format " + std::to_string(format) + " cannot be used as a storage image"); + return storage; +} + +} + +bool StorageFormatAvailable(const Context& context, std::uint32_t guestFormat) { + try { + return StorageFormatOrUndefined(context, ResolveTextureFormat(guestFormat)) != VK_FORMAT_UNDEFINED; + } catch (const std::exception&) { + // An unknown guest format: the sampled texture path reports it when it gets there. + return false; + } +} + +// Integer formats take integer clear values; the DCC clear codes are only mapped for the others. +bool IntegerFormat(VkFormat format) { + switch (format) { + case VK_FORMAT_R8_UINT: case VK_FORMAT_R8_SINT: case VK_FORMAT_R8G8_UINT: case VK_FORMAT_R8G8_SINT: + case VK_FORMAT_R8G8B8A8_UINT: case VK_FORMAT_R8G8B8A8_SINT: case VK_FORMAT_B8G8R8A8_UINT: case VK_FORMAT_B8G8R8A8_SINT: + case VK_FORMAT_A8B8G8R8_UINT_PACK32: case VK_FORMAT_A8B8G8R8_SINT_PACK32: case VK_FORMAT_A2R10G10B10_UINT_PACK32: case VK_FORMAT_A2B10G10R10_UINT_PACK32: + case VK_FORMAT_R16_UINT: case VK_FORMAT_R16_SINT: case VK_FORMAT_R16G16_UINT: case VK_FORMAT_R16G16_SINT: + case VK_FORMAT_R16G16B16A16_UINT: case VK_FORMAT_R16G16B16A16_SINT: case VK_FORMAT_R32_UINT: case VK_FORMAT_R32_SINT: + case VK_FORMAT_R32G32_UINT: case VK_FORMAT_R32G32_SINT: case VK_FORMAT_R32G32B32_UINT: case VK_FORMAT_R32G32B32_SINT: + case VK_FORMAT_R32G32B32A32_UINT: case VK_FORMAT_R32G32B32A32_SINT: case VK_FORMAT_R64_UINT: case VK_FORMAT_R64_SINT: + return true; + default: return false; + } +} + +// The clear value a DCC clear code stands for, for non-integer formats. +bool ClearColorFor(VkFormat format, DccKeys keys, VkClearColorValue& clear) { + if (IntegerFormat(format)) return false; + switch (keys) { + case DccKeys::Clear0000: clear.float32[0] = clear.float32[1] = clear.float32[2] = clear.float32[3] = 0.0f; return true; + case DccKeys::Clear0001: clear.float32[0] = clear.float32[1] = clear.float32[2] = 0.0f; clear.float32[3] = 1.0f; return true; + case DccKeys::Clear1110: clear.float32[0] = clear.float32[1] = clear.float32[2] = 1.0f; clear.float32[3] = 0.0f; return true; + case DccKeys::Clear1111: clear.float32[0] = clear.float32[1] = clear.float32[2] = clear.float32[3] = 1.0f; return true; + default: return false; + } +} + +VkFormat StorageFormatForGuest(const Context& context, std::uint32_t guestFormat) { + return StorageFormatFor(context, ResolveTextureFormat(guestFormat)); +} + +StorageTexture::StorageTexture(const Context& context, TextureDetiler& detiler, const GuestTextureResource& descriptor, std::uint32_t mipLevel) : context(context), detiler(detiler), descriptor(descriptor) { + static const bool profile = std::getenv("APS5_PROFILE_DRAW") != nullptr; + PhaseTimer timer; + try { + Require(!IsBlockCompressed(descriptor.format), "block-compressed textures cannot be storage images"); + Require(mipLevel < descriptor.mipCount, "storage texture mip level is outside the texture"); + const auto vkFormat = StorageFormatFor(context, ResolveTextureFormat(descriptor.format)); + storageFormat = vkFormat; + APS5_LOG_OUT("StorageTexture address=0x%llx %ux%u mips=%u mip=%u layers=%u base=%u dim=%d tile=%d format=%u vk=%d", static_cast(descriptor.baseAddress), descriptor.width, descriptor.height, descriptor.mipCount, mipLevel, + descriptor.depthOrLastArray, descriptor.baseArray, static_cast(descriptor.dimension), static_cast(descriptor.tileMode), descriptor.format, static_cast(vkFormat)); + geometry = DescribeSurface(descriptor); + mips = geometry.mips; + arrayLayers = geometry.layers; + const auto elementBytes = BytesPerElement(descriptor.format); + guestBytes = geometry.guestBytes; + // Storage images in heaps the guest commits on demand only read and store committed pages. + Require(!GuestMemory::CommittedRanges(descriptor.baseAddress, static_cast(guestBytes), true).empty(), "storage texture has no committed guest pages"); + sliceLinearBytes = geometry.sliceLinearBytes; + const auto linearBytes = sliceLinearBytes * arrayLayers; + + VkImageCreateInfo imageInfo{VK_STRUCTURE_TYPE_IMAGE_CREATE_INFO}; + // Sampled views of other same-size formats (sRGB, reinterpretations) read the image directly. + imageInfo.flags = (descriptor.dimension == TextureDimension::kCube ? VK_IMAGE_CREATE_CUBE_COMPATIBLE_BIT : 0u) | VK_IMAGE_CREATE_MUTABLE_FORMAT_BIT | VK_IMAGE_CREATE_EXTENDED_USAGE_BIT; + imageInfo.imageType = ImageTypeFor(descriptor.dimension); + imageInfo.format = vkFormat; + imageInfo.extent = {descriptor.width, descriptor.height, geometry.imageDepth}; + imageInfo.mipLevels = descriptor.mipCount; + imageInfo.arrayLayers = geometry.imageLayers; + imageInfo.samples = VK_SAMPLE_COUNT_1_BIT; + imageInfo.tiling = VK_IMAGE_TILING_OPTIMAL; + imageInfo.usage = VK_IMAGE_USAGE_TRANSFER_DST_BIT | VK_IMAGE_USAGE_TRANSFER_SRC_BIT | VK_IMAGE_USAGE_STORAGE_BIT | VK_IMAGE_USAGE_SAMPLED_BIT; + { + VkFormatProperties properties{}; + context.formatProperties(context.physical, vkFormat, &properties); + attachable = descriptor.dimension == TextureDimension::k2D && (properties.optimalTilingFeatures & VK_FORMAT_FEATURE_COLOR_ATTACHMENT_BIT) != 0; + if (attachable) imageInfo.usage |= VK_IMAGE_USAGE_COLOR_ATTACHMENT_BIT; + } + imageInfo.sharingMode = VK_SHARING_MODE_EXCLUSIVE; + imageInfo.initialLayout = VK_IMAGE_LAYOUT_UNDEFINED; + Check(context.Function("vkCreateImage")(context.device, &imageInfo, nullptr, &image), "vkCreateImage storage"); + VkMemoryRequirements requirements{}; + context.Function("vkGetImageMemoryRequirements")(context.device, image, &requirements); + VkMemoryAllocateInfo allocation{VK_STRUCTURE_TYPE_MEMORY_ALLOCATE_INFO}; + allocation.allocationSize = requirements.size; + allocation.memoryTypeIndex = context.MemoryType(requirements.memoryTypeBits, VK_MEMORY_PROPERTY_DEVICE_LOCAL_BIT); + Check(context.Function("vkAllocateMemory")(context.device, &allocation, nullptr, &memory), "vkAllocateMemory storage texture"); + Check(context.Function("vkBindImageMemory")(context.device, image, memory, 0), "vkBindImageMemory storage"); + upload(); + defaultMip = mipLevel; + view = createView(mipLevel); + if (profile) { + auto& totals = Profile(); + totals.storageCreate += timer.lap(); + totals.storageBytes += guestBytes; + if (++totals.storageCount % 50 == 0) std::fprintf(stderr, "[texture] %llu storage images (%.0f MiB): create %.0f ms, write-back %.0f ms (alloc %.0f, host copy %.0f, gpu %.0f, store %.0f), %llu reused, %llu direct uploads, %llu direct write-backs\n", static_cast(totals.storageCount), totals.storageBytes / 1048576.0, totals.storageCreate, totals.storageWriteBack, totals.storageAlloc, totals.storageHostCopy, totals.storageGpu, totals.storageStore, static_cast(totals.storageReused), static_cast(totals.storageDirectUploads), static_cast(totals.storageDirectWriteBacks)); + } + } catch (...) { + release(); + throw; + } +} + +namespace { + +// Storage images whose results have not reached guest memory yet. +struct PendingWrites { + std::mutex mutex; + std::vector textures; + // Images taken out of `textures` by a FlushPending still storing them (see adjacentPendingUnchanged). + std::vector flushing; +}; + +PendingWrites& Pending() { + static PendingWrites pending; + return pending; +} + +// The image being validated by Refresh: its own pending results are what the next dispatch wants, +// so the comparison with guest memory must not flush them. +thread_local const StorageTexture* refreshing = nullptr; + +void FlushHook(std::uint64_t address, std::size_t bytes) { + StorageTexture::FlushPending(address, bytes); +} + +// Write stamps are per 64 KiB block (GuestMemory::UnchangedSince), so a write-back's own MarkWritten +// stamps the blocks its surface shares with an adjacent surface (video planes packed back to back: +// the luma plane's last block is the chroma plane's first). A pending image of that adjacent surface +// would take the stamp for a CPU write since its generation and, at its own write-back, keep the +// guest bytes of the shared block in place of its results (a stale band at the plane's start, zero +// chroma on a fresh buffer). So a write-back first finds the adjacent pending images unchanged since +// their generation and, once its stamps are made, advances them past the stamps: nothing of theirs +// changed. APS5_NO_ADJACENT_GENERATION=1 leaves them at their generation, as before. +bool AdjacentGenerationEnabled() { + static const bool disabled = std::getenv("APS5_NO_ADJACENT_GENERATION") != nullptr; + return !disabled; +} + +} + +VkImageView StorageTexture::createView(std::uint32_t mip) const { + Require(mip < descriptor.mipCount, "storage texture mip level is outside the texture"); + const auto viewLayerCount = geometry.imageLayers - descriptor.baseArray; + VkImageViewCreateInfo viewInfo{VK_STRUCTURE_TYPE_IMAGE_VIEW_CREATE_INFO}; + viewInfo.image = image; + // Storage views address one mip; cube faces are written as array layers. + viewInfo.viewType = descriptor.dimension == TextureDimension::k1D ? VK_IMAGE_VIEW_TYPE_1D : descriptor.dimension == TextureDimension::k2D ? VK_IMAGE_VIEW_TYPE_2D : descriptor.dimension == TextureDimension::k3D ? VK_IMAGE_VIEW_TYPE_3D : VK_IMAGE_VIEW_TYPE_2D_ARRAY; + viewInfo.format = storageFormat; + viewInfo.components = {VK_COMPONENT_SWIZZLE_IDENTITY, VK_COMPONENT_SWIZZLE_IDENTITY, VK_COMPONENT_SWIZZLE_IDENTITY, VK_COMPONENT_SWIZZLE_IDENTITY}; + viewInfo.subresourceRange = {VK_IMAGE_ASPECT_COLOR_BIT, mip, 1u, descriptor.baseArray, viewLayerCount}; + VkImageView created = VK_NULL_HANDLE; + Check(context.Function("vkCreateImageView")(context.device, &viewInfo, nullptr, &created), "vkCreateImageView storage"); + return created; +} + +VkImageView StorageTexture::AttachmentView(VkFormat format) { + Require(attachable, "storage image cannot be a color attachment"); + const auto found = attachmentViews.find(format); + if (found != attachmentViews.end()) return found->second; + VkImageViewCreateInfo viewInfo{VK_STRUCTURE_TYPE_IMAGE_VIEW_CREATE_INFO}; + viewInfo.image = image; + viewInfo.viewType = VK_IMAGE_VIEW_TYPE_2D; + viewInfo.format = format; + viewInfo.components = {VK_COMPONENT_SWIZZLE_IDENTITY, VK_COMPONENT_SWIZZLE_IDENTITY, VK_COMPONENT_SWIZZLE_IDENTITY, VK_COMPONENT_SWIZZLE_IDENTITY}; + viewInfo.subresourceRange = {VK_IMAGE_ASPECT_COLOR_BIT, 0, 1u, descriptor.baseArray, 1u}; + VkImageView created = VK_NULL_HANDLE; + Check(context.Function("vkCreateImageView")(context.device, &viewInfo, nullptr, &created), "vkCreateImageView attachment"); + attachmentViews.emplace(format, created); + return created; +} + +VkImageView StorageTexture::View(std::uint32_t mip) { + if (mip == defaultMip) return view; + const auto found = extraViews.find(mip); + if (found != extraViews.end()) return found->second; + const auto created = createView(mip); + extraViews.emplace(mip, created); + return created; +} + +bool StorageTexture::Refresh() { + // Results of other images pending in this memory must reach it first; this image's own pending + // results stay on the GPU, where the next dispatch wants them. + FlushPending(descriptor.baseAddress, static_cast(guestBytes), this, "storage refresh"); + // `original` holds the guest bytes the image was last uploaded from or written back as; while the + // guest memory and the DCC keys still match, the image content is current. Pages nobody wrote + // since `generation` need no comparison. + const auto current = GuestMemory::CollectWrites(descriptor.baseAddress, static_cast(guestBytes)); + bool unchanged = false; + { + struct Exempt { + const StorageTexture* previous; + ~Exempt() { refreshing = previous; } + } exempt{refreshing}; + refreshing = this; + const auto equalsOriginal = [&] { + // Named for the [hooksync] attribution: the compare goes through the flush hook. + const GuestMemory::ReadSiteScope site(GuestMemory::ReadSite::TextureCompare); + return GuestMemory::EqualsCommitted(descriptor.baseAddress, original); + }; + unchanged = TextureClearKeys(descriptor, guestBytes) == uploadedKeys && (GuestMemory::UnchangedSince(descriptor.baseAddress, static_cast(guestBytes), generation) || (originalValid && equalsOriginal())); + } + if (unchanged) { + ++Profile().storageReused; + generation = current; + return true; + } + // Debug aid: APS5_TRACE_UPLOAD names why a storage image is uploaded again. + static const bool traceUpload = std::getenv("APS5_TRACE_UPLOAD") != nullptr; + if (traceUpload) { + const auto keys = TextureClearKeys(descriptor, guestBytes); + std::fprintf(stderr, "[upload] 0x%llx+0x%llx %ux%u mips %u: %s (keys %s -> %s, originalValid %d, dirty %d)\n", static_cast(descriptor.baseAddress), static_cast(guestBytes), descriptor.width, descriptor.height, descriptor.mipCount, keys != uploadedKeys ? "DCC keys changed" : "guest memory changed", DccKeysName(uploadedKeys), DccKeysName(keys), originalValid ? 1 : 0, dirty ? 1 : 0); + } + bool pendingResults = false; + { + auto& pending = Pending(); + std::lock_guard lock(pending.mutex); + if (dirty) { + dirty = false; + std::erase(pending.textures, this); + pendingResults = true; + } + } + if (pendingResults) { + // The CPU wrote this memory while GPU results were pending: as with an immediate write-back + // followed by the CPU write, its blocks win and the results land everywhere else. + static std::atomic reports{0}; + if (reports.fetch_add(1) < 8) std::fprintf(stderr, "[gpu] storage image 0x%llx: guest memory changed while GPU results were pending; keeping the CPU's blocks\n", static_cast(descriptor.baseAddress)); + writeBack(generation); + } + upload(); + return false; +} + +void StorageTexture::upload() { + const auto elementBytes = BytesPerElement(descriptor.format); + const auto linearBytes = sliceLinearBytes * arrayLayers; + original.resize(static_cast(guestBytes)); + generation = GuestMemory::CollectWrites(descriptor.baseAddress, static_cast(guestBytes)); + uploadedKeys = TextureClearKeys(descriptor, guestBytes); + VkClearColorValue clearValue{}; + if (IsDccClear(uploadedKeys) && ClearColorFor(storageFormat, uploadedKeys, clearValue)) { + // A fast-cleared surface is cleared on the GPU; its texel memory is neither read nor filled. + originalValid = false; + CommandBatch batch(context); + const auto commands = batch.Handle(); + VkImageMemoryBarrier toClear{VK_STRUCTURE_TYPE_IMAGE_MEMORY_BARRIER}; + toClear.srcAccessMask = VK_ACCESS_SHADER_WRITE_BIT | VK_ACCESS_SHADER_READ_BIT | VK_ACCESS_TRANSFER_WRITE_BIT; + toClear.dstAccessMask = VK_ACCESS_TRANSFER_WRITE_BIT; + toClear.oldLayout = VK_IMAGE_LAYOUT_UNDEFINED; + toClear.newLayout = VK_IMAGE_LAYOUT_TRANSFER_DST_OPTIMAL; + toClear.srcQueueFamilyIndex = VK_QUEUE_FAMILY_IGNORED; + toClear.dstQueueFamilyIndex = VK_QUEUE_FAMILY_IGNORED; + toClear.image = image; + toClear.subresourceRange = {VK_IMAGE_ASPECT_COLOR_BIT, 0, descriptor.mipCount, 0, geometry.imageLayers}; + context.Function("vkCmdPipelineBarrier")(commands, VK_PIPELINE_STAGE_ALL_COMMANDS_BIT, VK_PIPELINE_STAGE_TRANSFER_BIT, 0, 0, nullptr, 0, nullptr, 1, &toClear); + context.Function("vkCmdClearColorImage")(commands, image, VK_IMAGE_LAYOUT_TRANSFER_DST_OPTIMAL, &clearValue, 1, &toClear.subresourceRange); + VkImageMemoryBarrier toGeneral = toClear; + toGeneral.srcAccessMask = VK_ACCESS_TRANSFER_WRITE_BIT; + toGeneral.dstAccessMask = VK_ACCESS_SHADER_READ_BIT | VK_ACCESS_SHADER_WRITE_BIT; + toGeneral.oldLayout = VK_IMAGE_LAYOUT_TRANSFER_DST_OPTIMAL; + toGeneral.newLayout = VK_IMAGE_LAYOUT_GENERAL; + context.Function("vkCmdPipelineBarrier")(commands, VK_PIPELINE_STAGE_TRANSFER_BIT, VK_PIPELINE_STAGE_ALL_COMMANDS_BIT, 0, 0, nullptr, 0, nullptr, 1, &toGeneral); + batch.SubmitAndWait(); + ++version; + return; + } + if (const auto* import = uploadedKeys == DccKeys::Uncompressed ? HostImportFor(context, descriptor.baseAddress, static_cast(guestBytes)) : nullptr) { + // The surface lives in host-imported memory: the detiler reads it in place, no guest bytes + // are copied, and write tracking alone validates the image (a change re-runs this). + originalValid = false; + auto linear = std::make_shared(context, static_cast(linearBytes), VK_BUFFER_USAGE_STORAGE_BUFFER_BIT | VK_BUFFER_USAGE_TRANSFER_SRC_BIT); + detiler.BeginBatch(); + auto* recorder = Recorder::Active(); + std::unique_ptr batch; + VkCommandBuffer commands = VK_NULL_HANDLE; + if (recorder != nullptr) { + commands = recorder->Commands(); + recorder->Keep(linear); + // The image itself must outlive the recorded copy: the cache may evict it right after. + if (auto self = weak_from_this().lock()) recorder->Keep(std::move(self)); + } else { + batch = std::make_unique(context); + commands = batch->Handle(); + } + const auto importOffset = descriptor.baseAddress - import->base; + RecordMemoryBarrier(context, commands, VK_PIPELINE_STAGE_ALL_COMMANDS_BIT, VK_PIPELINE_STAGE_COMPUTE_SHADER_BIT, VK_ACCESS_SHADER_WRITE_BIT | VK_ACCESS_TRANSFER_WRITE_BIT | VK_ACCESS_HOST_WRITE_BIT, VK_ACCESS_SHADER_READ_BIT | VK_ACCESS_SHADER_WRITE_BIT); + for (std::uint32_t layer = 0; layer < arrayLayers; ++layer) { + for (const auto& mip : mips) { + detiler.Dispatch(commands, descriptor.tileMode, elementBytes, import->buffer, importOffset + geometry.GuestLayerOffset(layer) + mip.tiledOffset, linear->Handle(), geometry.LinearLayerOffset(layer) + mip.linearOffset, mip, false, layer, geometry.thick); + } + } + const auto linearRead = WholeBufferBarrier(linear->Handle(), VK_ACCESS_SHADER_WRITE_BIT, VK_ACCESS_TRANSFER_READ_BIT); + VkImageMemoryBarrier toTransfer{VK_STRUCTURE_TYPE_IMAGE_MEMORY_BARRIER}; + toTransfer.dstAccessMask = VK_ACCESS_TRANSFER_WRITE_BIT; + toTransfer.oldLayout = VK_IMAGE_LAYOUT_UNDEFINED; + toTransfer.newLayout = VK_IMAGE_LAYOUT_TRANSFER_DST_OPTIMAL; + toTransfer.srcQueueFamilyIndex = VK_QUEUE_FAMILY_IGNORED; + toTransfer.dstQueueFamilyIndex = VK_QUEUE_FAMILY_IGNORED; + toTransfer.image = image; + toTransfer.subresourceRange = {VK_IMAGE_ASPECT_COLOR_BIT, 0, descriptor.mipCount, 0, geometry.imageLayers}; + context.Function("vkCmdPipelineBarrier")(commands, VK_PIPELINE_STAGE_COMPUTE_SHADER_BIT, VK_PIPELINE_STAGE_TRANSFER_BIT, 0, 0, nullptr, 1, &linearRead, 1, &toTransfer); + const auto regions = CopyRegions(); + context.Function("vkCmdCopyBufferToImage")(commands, linear->Handle(), image, VK_IMAGE_LAYOUT_TRANSFER_DST_OPTIMAL, static_cast(regions.size()), regions.data()); + VkImageMemoryBarrier toGeneral = toTransfer; + toGeneral.srcAccessMask = VK_ACCESS_TRANSFER_WRITE_BIT; + toGeneral.dstAccessMask = VK_ACCESS_SHADER_READ_BIT | VK_ACCESS_SHADER_WRITE_BIT; + toGeneral.oldLayout = VK_IMAGE_LAYOUT_TRANSFER_DST_OPTIMAL; + toGeneral.newLayout = VK_IMAGE_LAYOUT_GENERAL; + context.Function("vkCmdPipelineBarrier")(commands, VK_PIPELINE_STAGE_TRANSFER_BIT, VK_PIPELINE_STAGE_ALL_COMMANDS_BIT, 0, 0, nullptr, 0, nullptr, 1, &toGeneral); + if (batch) batch->SubmitAndWait(); + ++Profile().storageDirectUploads; + ++version; + return; + } + originalValid = true; + GuestMemory::ReadCommitted(descriptor.baseAddress, original); + { + Buffer staging(context, original.size(), VK_BUFFER_USAGE_TRANSFER_SRC_BIT); + if (uploadedKeys == DccKeys::Uncompressed) std::memcpy(staging.Bytes().data(), original.data(), original.size()); + else ReadTextureSurface(descriptor, uploadedKeys, staging.Bytes().first(original.size())); + DeviceBuffer tiled(context, original.size(), VK_BUFFER_USAGE_STORAGE_BUFFER_BIT | VK_BUFFER_USAGE_TRANSFER_DST_BIT); + DeviceBuffer linear(context, static_cast(linearBytes), VK_BUFFER_USAGE_STORAGE_BUFFER_BIT | VK_BUFFER_USAGE_TRANSFER_SRC_BIT); + detiler.BeginBatch(); + CommandBatch batch(context); + const auto commands = batch.Handle(); + RecordMemoryBarrier(context, commands, VK_PIPELINE_STAGE_HOST_BIT, VK_PIPELINE_STAGE_TRANSFER_BIT, VK_ACCESS_HOST_WRITE_BIT, VK_ACCESS_TRANSFER_READ_BIT); + CopyBuffer(context, commands, staging.Handle(), 0, tiled.Handle(), 0, original.size()); + RecordMemoryBarrier(context, commands, VK_PIPELINE_STAGE_TRANSFER_BIT, VK_PIPELINE_STAGE_COMPUTE_SHADER_BIT, VK_ACCESS_TRANSFER_WRITE_BIT, VK_ACCESS_SHADER_READ_BIT | VK_ACCESS_SHADER_WRITE_BIT); + for (std::uint32_t layer = 0; layer < arrayLayers; ++layer) { + for (const auto& mip : mips) { + detiler.Dispatch(commands, descriptor.tileMode, elementBytes, tiled.Handle(), geometry.GuestLayerOffset(layer) + mip.tiledOffset, linear.Handle(), geometry.LinearLayerOffset(layer) + mip.linearOffset, mip, false, layer, geometry.thick); + } + } + const auto linearRead = WholeBufferBarrier(linear.Handle(), VK_ACCESS_SHADER_WRITE_BIT, VK_ACCESS_TRANSFER_READ_BIT); + VkImageMemoryBarrier toTransfer{VK_STRUCTURE_TYPE_IMAGE_MEMORY_BARRIER}; + toTransfer.dstAccessMask = VK_ACCESS_TRANSFER_WRITE_BIT; + toTransfer.oldLayout = VK_IMAGE_LAYOUT_UNDEFINED; + toTransfer.newLayout = VK_IMAGE_LAYOUT_TRANSFER_DST_OPTIMAL; + toTransfer.srcQueueFamilyIndex = VK_QUEUE_FAMILY_IGNORED; + toTransfer.dstQueueFamilyIndex = VK_QUEUE_FAMILY_IGNORED; + toTransfer.image = image; + toTransfer.subresourceRange = {VK_IMAGE_ASPECT_COLOR_BIT, 0, descriptor.mipCount, 0, geometry.imageLayers}; + context.Function("vkCmdPipelineBarrier")(commands, VK_PIPELINE_STAGE_COMPUTE_SHADER_BIT, VK_PIPELINE_STAGE_TRANSFER_BIT, 0, 0, nullptr, 1, &linearRead, 1, &toTransfer); + const auto regions = CopyRegions(); + context.Function("vkCmdCopyBufferToImage")(commands, linear.Handle(), image, VK_IMAGE_LAYOUT_TRANSFER_DST_OPTIMAL, static_cast(regions.size()), regions.data()); + VkImageMemoryBarrier toGeneral = toTransfer; + toGeneral.srcAccessMask = VK_ACCESS_TRANSFER_WRITE_BIT; + toGeneral.dstAccessMask = VK_ACCESS_SHADER_READ_BIT | VK_ACCESS_SHADER_WRITE_BIT; + toGeneral.oldLayout = VK_IMAGE_LAYOUT_TRANSFER_DST_OPTIMAL; + toGeneral.newLayout = VK_IMAGE_LAYOUT_GENERAL; + context.Function("vkCmdPipelineBarrier")(commands, VK_PIPELINE_STAGE_TRANSFER_BIT, VK_PIPELINE_STAGE_ALL_COMMANDS_BIT, 0, 0, nullptr, 0, nullptr, 1, &toGeneral); + APS5_LOG_CHARS_OUT("StorageTexture upload submit"); + batch.SubmitAndWait(); + APS5_LOG_CHARS_OUT("StorageTexture upload done"); + } + ++version; +} + +std::vector StorageTexture::CopyRegions() const { + std::vector regions; + for (std::uint32_t layer = 0; layer < arrayLayers; ++layer) { + for (std::uint32_t level = 0; level < descriptor.mipCount; ++level) { + const auto& mip = mips[level]; + VkBufferImageCopy region{}; + region.bufferOffset = layer * sliceLinearBytes + mip.linearOffset; + region.bufferRowLength = mip.pitchBytes / BytesPerElement(descriptor.format) * BlockWidth(descriptor.format); + region.imageSubresource = {VK_IMAGE_ASPECT_COLOR_BIT, level, geometry.CopyLayer(layer), 1}; + region.imageOffset = {0, 0, geometry.CopyDepth(layer)}; + region.imageExtent = {std::max(descriptor.width >> level, 1u), std::max(descriptor.height >> level, 1u), 1u}; + regions.push_back(region); + } + } + return regions; +} + +bool StorageTexture::overlaps(std::uint64_t address, std::size_t bytes) const { + return address < descriptor.baseAddress + guestBytes && descriptor.baseAddress < address + bytes; +} + +void StorageTexture::MarkDirty() { + static const bool eager = std::getenv("APS5_EAGER_WRITEBACK") != nullptr || std::getenv("APS5_NO_TEXTURE_CACHE") != nullptr; + if (eager) { + WriteBack(); + return; + } + auto& pending = Pending(); + std::lock_guard lock(pending.mutex); + // The active recorder installs a hook that also waits for recorded work; without one (tests), + // pending storage results alone are flushed. + if (Recorder::Active() == nullptr) { + static const bool hooked = [] { + GuestMemory::SetFlushHook(&FlushHook); + return true; + }(); + static_cast(hooked); + } + ++version; + if (dirty) return; + dirty = true; + pending.textures.push_back(this); +} + +void StorageTexture::Flush() { + { + auto& pending = Pending(); + std::lock_guard lock(pending.mutex); + if (!dirty) return; + dirty = false; + std::erase(pending.textures, this); + } + std::lock_guard gpu(GuestMemory::GpuMutex()); + writeBack(generation); +} + +bool StorageTexture::FlushPending(std::uint64_t address, std::size_t bytes, const StorageTexture* except, const char* reason) { + // The images stay alive across the scan: their last owner may be a batch's kept list, which + // another thread releases outside the GPU mutex once the batch completed. + std::vector> flush; + { + auto& pending = Pending(); + std::lock_guard lock(pending.mutex); + for (auto it = pending.textures.begin(); it != pending.textures.end();) { + auto* texture = *it; + if (texture != except && texture != refreshing && texture->overlaps(address, bytes)) { + texture->dirty = false; + if (auto alive = texture->weak_from_this().lock()) flush.push_back(std::move(alive)); + it = pending.textures.erase(it); + } else { + ++it; + } + } + // Still pending for the adjacency rule of writeBack until each one is stored. + for (const auto& texture : flush) pending.flushing.push_back(texture.get()); + } + if (flush.empty()) return false; + struct Unregister { + const std::vector>& flush; + ~Unregister() { + auto& pending = Pending(); + std::lock_guard lock(pending.mutex); + for (const auto& texture : flush) std::erase(pending.flushing, texture.get()); + } + } unregister{flush}; + // Debug aid: APS5_TRACE_FLUSH names what forces pending results to guest memory. + static const bool trace = std::getenv("APS5_TRACE_FLUSH") != nullptr; + if (trace) { + for (const auto& texture : flush) std::fprintf(stderr, "[flush] image 0x%llx+0x%llx for %s 0x%llx+0x%zx\n", static_cast(texture->descriptor.baseAddress), static_cast(texture->guestBytes), reason, static_cast(address), bytes); + } + std::lock_guard gpu(GuestMemory::GpuMutex()); + std::exception_ptr failure; + for (const auto& texture : flush) { + try { + texture->writeBack(texture->generation); + } catch (...) { + if (!failure) failure = std::current_exception(); + } + } + if (failure) std::rethrow_exception(failure); + return true; +} + +void StorageTexture::FlushAllPending(const char* reason) { + static_cast(FlushPending(0, std::numeric_limits::max(), nullptr, reason)); +} + +std::shared_ptr StorageTexture::FindPending(std::uint64_t address, std::uint64_t bytes) { + auto& pending = Pending(); + std::lock_guard lock(pending.mutex); + for (auto* texture : pending.textures) { + // Containment, not equality: a descriptor of a chain's first mips (its own guestBytes are + // shorter) is served by the chain's image; CanCopyFrom then checks the geometry. + if (texture->descriptor.baseAddress == address && texture->guestBytes >= bytes) return texture->weak_from_this().lock(); + } + return nullptr; +} + +bool PendingStorageOverlaps(std::uint64_t address, std::size_t bytes, const StorageTexture* except) { + auto& pending = Pending(); + std::lock_guard lock(pending.mutex); + for (const auto* texture : pending.textures) { + // A free function (declared in ShaderResources.hpp): the overlap is computed from the public + // surface description rather than the private helper. + const auto begin = texture->Descriptor().baseAddress; + if (texture != except && address < begin + texture->GuestBytes() && begin < address + bytes) return true; + } + return false; +} + +void StorageTexture::WriteBack() { + writeBack(generation); +} + +// See AdjacentGenerationEnabled above. +std::vector StorageTexture::adjacentPendingUnchanged() const { + std::vector adjacent; + if (!AdjacentGenerationEnabled() || guestBytes == 0) return adjacent; + constexpr std::uint64_t block = 65536; + const auto firstBlock = descriptor.baseAddress / block; + const auto lastBlock = (descriptor.baseAddress + guestBytes - 1) / block; + { + auto& pending = Pending(); + std::lock_guard lock(pending.mutex); + const auto consider = [&](StorageTexture* texture) { + if (texture == this || texture->guestBytes == 0 || texture->overlaps(descriptor.baseAddress, static_cast(guestBytes))) return; + const auto begin = texture->descriptor.baseAddress; + if (begin / block > lastBlock || (begin + texture->guestBytes - 1) / block < firstBlock) return; + if (auto alive = texture->weak_from_this().lock()) adjacent.push_back({std::move(alive), texture->generation}); + }; + for (auto* texture : pending.textures) consider(texture); + for (auto* texture : pending.flushing) consider(texture); + } + // Only an image whose memory has not changed since its generation can be advanced: the collect + // stamps the CPU's writes so far, and a later one lands at a newer generation either way. + std::erase_if(adjacent, [](const Adjacent& entry) { + const auto& texture = entry.texture; + GuestMemory::CollectWritesUncached(texture->descriptor.baseAddress, static_cast(texture->guestBytes)); + return !GuestMemory::UnchangedSince(texture->descriptor.baseAddress, static_cast(texture->guestBytes), texture->generation); + }); + return adjacent; +} + +void StorageTexture::advanceAdjacent(const std::vector& adjacent, std::uint64_t now, std::uint64_t firstBlock, std::uint64_t lastBlock) { + static const bool trace = std::getenv("APS5_TRACE_FLUSH") != nullptr; + constexpr std::uint64_t block = 65536; + for (const auto& [texture, seen] : adjacent) { + // An image stored or refreshed meanwhile set its own generation (an old one on purpose when + // it kept blocks for the CPU): only the value seen at the check is advanced. + if (texture->generation != seen || now <= seen) continue; + // A CPU write to the neighbour landing since the check may have been stamped by another + // thread's walk of its range at a value in (seen, now]; the advance would hide it. Its blocks + // outside this write-back's span carry no stamp of ours, so they are re-checked against the + // seen generation (a stamp scan, no walk). A write inside the shared block itself in that + // window is indistinguishable from this write-back's stamp, as it is for the block rule. + const auto begin = texture->descriptor.baseAddress; + const auto end = begin + texture->guestBytes; + const auto beforeEnd = std::min(end, firstBlock * block); + const auto afterBegin = std::max(begin, (lastBlock + 1) * block); + if (begin < beforeEnd && !GuestMemory::UnchangedSince(begin, static_cast(beforeEnd - begin), seen)) continue; + if (afterBegin < end && !GuestMemory::UnchangedSince(afterBegin, static_cast(end - afterBegin), seen)) continue; + if (trace) std::fprintf(stderr, "[flush] adjacent pending image 0x%llx+0x%llx advanced past a write-back's stamps (generation %llu -> %llu)\n", static_cast(begin), static_cast(texture->guestBytes), static_cast(seen), static_cast(now)); + texture->generation = now; + } +} + +void StorageTexture::writeBack(std::uint64_t skipWrittenSince) { + static const bool profile = std::getenv("APS5_PROFILE_DRAW") != nullptr; + PhaseTimer timer; + struct Account { + bool enabled; + PhaseTimer& timer; + ~Account() { if (enabled) Profile().storageWriteBack += timer.lap(); } + } account{profile, timer}; + const auto elementBytes = BytesPerElement(descriptor.format); + // 64 KiB blocks the CPU wrote since the image was last in sync keep the CPU's bytes: the game may + // have reused the memory for something else entirely (see skipWrittenSince). Zero means no + // tracking, and everything is stored. + std::vector> keep; + bool skippedAny = false; + std::size_t skipped = 0; + { + const auto begin = descriptor.baseAddress; + const auto end = begin + guestBytes; + if (skipWrittenSince != 0) { + constexpr std::uint64_t block = 65536; + // The keep-blocks decision must see every CPU write up to now, so it bypasses the + // per-packet collect memo (a Refresh collect earlier in the same packet would satisfy it). + GuestMemory::CollectWritesUncached(begin, static_cast(guestBytes)); + for (auto at = begin & ~(block - 1); at < end; at += block) { + const auto from = std::max(at, begin); + const auto to = std::min(at + block, end); + if (from >= to) continue; + if (!GuestMemory::UnchangedSince(from, static_cast(to - from), skipWrittenSince)) { + skippedAny = true; + ++skipped; + continue; + } + if (!keep.empty() && keep.back().second == from) keep.back().second = to; + else keep.emplace_back(from, to); + } + } else { + keep.emplace_back(begin, end); + } + } + // Debug aid: APS5_TRACE_FLUSH also names the blocks a write-back leaves to the CPU. + static const bool traceKept = std::getenv("APS5_TRACE_FLUSH") != nullptr; + if (traceKept && skippedAny) { + const auto first = keep.empty() ? descriptor.baseAddress + guestBytes : keep.front().first; + std::fprintf(stderr, "[flush] image 0x%llx+0x%llx keeps %zu CPU-written 64 KiB blocks (first stored byte at +0x%llx, %zu ranges stored, generation %llu)\n", static_cast(descriptor.baseAddress), static_cast(guestBytes), skipped, static_cast(first - descriptor.baseAddress), keep.size(), static_cast(skipWrittenSince)); + } + // Taken before this write-back stamps anything (see advanceAdjacent), with the 64 KiB block span + // its stamps cover. + const auto adjacent = adjacentPendingUnchanged(); + const auto firstBlock = descriptor.baseAddress / 65536; + const auto lastBlock = guestBytes == 0 ? firstBlock : (descriptor.baseAddress + guestBytes - 1) / 65536; + if (const auto* import = HostImportFor(context, descriptor.baseAddress, static_cast(guestBytes))) { + // The surface lives in host-imported memory: the retiler writes into device scratch and the + // untouched blocks are copied into the imported bytes in place, recorded behind the work that + // produced the image; nothing crosses to the CPU. + auto linear = std::make_shared(context, static_cast(sliceLinearBytes * arrayLayers), VK_BUFFER_USAGE_STORAGE_BUFFER_BIT | VK_BUFFER_USAGE_TRANSFER_DST_BIT); + auto tiledScratch = std::make_shared(context, static_cast(guestBytes), VK_BUFFER_USAGE_STORAGE_BUFFER_BIT | VK_BUFFER_USAGE_TRANSFER_SRC_BIT); + detiler.BeginBatch(); + auto* recorder = Recorder::Active(); + std::unique_ptr batch; + VkCommandBuffer commands = VK_NULL_HANDLE; + if (recorder != nullptr) { + commands = recorder->Commands(); + recorder->Keep(linear); + recorder->Keep(tiledScratch); + // The image itself must outlive the recorded retile: the cache may evict it right after. + if (auto self = weak_from_this().lock()) recorder->Keep(std::move(self)); + } else { + batch = std::make_unique(context); + commands = batch->Handle(); + } + const auto importOffset = descriptor.baseAddress - import->base; + VkImageMemoryBarrier toSource{VK_STRUCTURE_TYPE_IMAGE_MEMORY_BARRIER}; + toSource.srcAccessMask = VK_ACCESS_SHADER_WRITE_BIT | VK_ACCESS_COLOR_ATTACHMENT_WRITE_BIT; + toSource.dstAccessMask = VK_ACCESS_TRANSFER_READ_BIT; + toSource.oldLayout = VK_IMAGE_LAYOUT_GENERAL; + toSource.newLayout = VK_IMAGE_LAYOUT_TRANSFER_SRC_OPTIMAL; + toSource.srcQueueFamilyIndex = VK_QUEUE_FAMILY_IGNORED; + toSource.dstQueueFamilyIndex = VK_QUEUE_FAMILY_IGNORED; + toSource.image = image; + toSource.subresourceRange = {VK_IMAGE_ASPECT_COLOR_BIT, 0, descriptor.mipCount, 0, geometry.imageLayers}; + context.Function("vkCmdPipelineBarrier")(commands, VK_PIPELINE_STAGE_ALL_COMMANDS_BIT, VK_PIPELINE_STAGE_TRANSFER_BIT, 0, 0, nullptr, 0, nullptr, 1, &toSource); + const auto regions = CopyRegions(); + context.Function("vkCmdCopyImageToBuffer")(commands, image, VK_IMAGE_LAYOUT_TRANSFER_SRC_OPTIMAL, linear->Handle(), static_cast(regions.size()), regions.data()); + const auto linearRead = WholeBufferBarrier(linear->Handle(), VK_ACCESS_TRANSFER_WRITE_BIT, VK_ACCESS_SHADER_READ_BIT); + const VkMemoryBarrier importReady{VK_STRUCTURE_TYPE_MEMORY_BARRIER, nullptr, VK_ACCESS_SHADER_READ_BIT | VK_ACCESS_SHADER_WRITE_BIT | VK_ACCESS_TRANSFER_READ_BIT | VK_ACCESS_TRANSFER_WRITE_BIT | VK_ACCESS_HOST_WRITE_BIT, VK_ACCESS_SHADER_WRITE_BIT | VK_ACCESS_SHADER_READ_BIT}; + context.Function("vkCmdPipelineBarrier")(commands, VK_PIPELINE_STAGE_ALL_COMMANDS_BIT, VK_PIPELINE_STAGE_COMPUTE_SHADER_BIT, 0, 1, &importReady, 1, &linearRead, 0, nullptr); + for (std::uint32_t layer = 0; layer < arrayLayers; ++layer) { + for (const auto& mip : mips) { + detiler.Dispatch(commands, descriptor.tileMode, elementBytes, linear->Handle(), geometry.LinearLayerOffset(layer) + mip.linearOffset, tiledScratch->Handle(), geometry.GuestLayerOffset(layer) + mip.tiledOffset, mip, true, layer, geometry.thick); + } + } + { + const auto scratchDone = WholeBufferBarrier(tiledScratch->Handle(), VK_ACCESS_SHADER_WRITE_BIT, VK_ACCESS_TRANSFER_READ_BIT); + context.Function("vkCmdPipelineBarrier")(commands, VK_PIPELINE_STAGE_COMPUTE_SHADER_BIT, VK_PIPELINE_STAGE_TRANSFER_BIT, 0, 0, nullptr, 1, &scratchDone, 0, nullptr); + std::vector copies; + for (const auto& [from, to] : keep) copies.push_back({from - descriptor.baseAddress, importOffset + (from - descriptor.baseAddress), to - from}); + if (!copies.empty()) context.Function("vkCmdCopyBuffer")(commands, tiledScratch->Handle(), import->buffer, static_cast(copies.size()), copies.data()); + } + VkImageMemoryBarrier backToGeneral = toSource; + backToGeneral.srcAccessMask = VK_ACCESS_TRANSFER_READ_BIT; + backToGeneral.dstAccessMask = VK_ACCESS_SHADER_READ_BIT | VK_ACCESS_SHADER_WRITE_BIT; + backToGeneral.oldLayout = VK_IMAGE_LAYOUT_TRANSFER_SRC_OPTIMAL; + backToGeneral.newLayout = VK_IMAGE_LAYOUT_GENERAL; + const VkMemoryBarrier stored{VK_STRUCTURE_TYPE_MEMORY_BARRIER, nullptr, VK_ACCESS_TRANSFER_WRITE_BIT | VK_ACCESS_SHADER_WRITE_BIT, VK_ACCESS_SHADER_READ_BIT | VK_ACCESS_SHADER_WRITE_BIT | VK_ACCESS_TRANSFER_READ_BIT | VK_ACCESS_HOST_READ_BIT}; + context.Function("vkCmdPipelineBarrier")(commands, VK_PIPELINE_STAGE_COMPUTE_SHADER_BIT | VK_PIPELINE_STAGE_TRANSFER_BIT, VK_PIPELINE_STAGE_ALL_COMMANDS_BIT | VK_PIPELINE_STAGE_HOST_BIT, 0, 1, &stored, 0, nullptr, 1, &backToGeneral); + if (batch) batch->SubmitAndWait(); + else recorder->NotePendingWrite(descriptor.baseAddress, static_cast(guestBytes)); + if (profile) Profile().storageGpu += timer.lap(); + // The guest bytes now differ from `original`; other caches of the range see the write. With + // every block stored the image is current at a fresh generation; with blocks kept for the CPU + // the image is stale there, so the old generation stays and the next use re-uploads. + originalValid = false; + // The keys go the same way as the texels: a fill recorded behind the retile when the + // metadata is host-imported (no CPU wait for the title's key-writing kernels), else a CPU + // store (APS5_CPU_DCC_KEYS=1 keeps the CPU store; see DccMetadata.hpp). + MarkDccUncompressed(context, descriptor.dccAddress, guestBytes); + uploadedKeys = DccKeys::Uncompressed; + for (const auto& [from, to] : keep) GuestMemory::MarkWritten(from, static_cast(to - from)); + if (!skippedAny || !adjacent.empty()) { + // The walk covers this surface's pages only: an adjacent image's later CPU write is + // stamped newer than this value when its own range is collected. No CPU wrote the pages + // here (MarkWritten made the only stamps), so the memoized collect is exact. + const auto now = GuestMemory::CollectWrites(descriptor.baseAddress, static_cast(guestBytes)); + if (!skippedAny) generation = now; + advanceAdjacent(adjacent, now, firstBlock, lastBlock); + } + ++Profile().storageDirectWriteBacks; + return; + } + if (!originalValid) { + // The image was cleared on the GPU without reading the guest bytes; the store below compares + // against them, so read them now. + GuestMemory::ReadCommitted(descriptor.baseAddress, original); + originalValid = true; + } + DeviceBuffer linear(context, static_cast(sliceLinearBytes * arrayLayers), VK_BUFFER_USAGE_STORAGE_BUFFER_BIT | VK_BUFFER_USAGE_TRANSFER_DST_BIT); + DeviceBuffer tiled(context, original.size(), VK_BUFFER_USAGE_STORAGE_BUFFER_BIT | VK_BUFFER_USAGE_TRANSFER_SRC_BIT | VK_BUFFER_USAGE_TRANSFER_DST_BIT); + Buffer host(context, original.size(), VK_BUFFER_USAGE_TRANSFER_SRC_BIT | VK_BUFFER_USAGE_TRANSFER_DST_BIT); + if (profile) Profile().storageAlloc += timer.lap(); + // Start from the uploaded bytes so padding and untouched texels keep their guest values. + std::memcpy(host.Bytes().data(), original.data(), original.size()); + if (profile) Profile().storageHostCopy += timer.lap(); + detiler.BeginBatch(); + CommandBatch batch(context); + const auto commands = batch.Handle(); + RecordMemoryBarrier(context, commands, VK_PIPELINE_STAGE_HOST_BIT, VK_PIPELINE_STAGE_TRANSFER_BIT, VK_ACCESS_HOST_WRITE_BIT, VK_ACCESS_TRANSFER_READ_BIT); + CopyBuffer(context, commands, host.Handle(), 0, tiled.Handle(), 0, original.size()); + VkImageMemoryBarrier toSource{VK_STRUCTURE_TYPE_IMAGE_MEMORY_BARRIER}; + toSource.srcAccessMask = VK_ACCESS_SHADER_WRITE_BIT; + toSource.dstAccessMask = VK_ACCESS_TRANSFER_READ_BIT; + toSource.oldLayout = VK_IMAGE_LAYOUT_GENERAL; + toSource.newLayout = VK_IMAGE_LAYOUT_TRANSFER_SRC_OPTIMAL; + toSource.srcQueueFamilyIndex = VK_QUEUE_FAMILY_IGNORED; + toSource.dstQueueFamilyIndex = VK_QUEUE_FAMILY_IGNORED; + toSource.image = image; + toSource.subresourceRange = {VK_IMAGE_ASPECT_COLOR_BIT, 0, descriptor.mipCount, 0, geometry.imageLayers}; + context.Function("vkCmdPipelineBarrier")(commands, VK_PIPELINE_STAGE_ALL_COMMANDS_BIT, VK_PIPELINE_STAGE_TRANSFER_BIT, 0, 0, nullptr, 0, nullptr, 1, &toSource); + const auto regions = CopyRegions(); + context.Function("vkCmdCopyImageToBuffer")(commands, image, VK_IMAGE_LAYOUT_TRANSFER_SRC_OPTIMAL, linear.Handle(), static_cast(regions.size()), regions.data()); + const VkBufferMemoryBarrier toShader[] = {WholeBufferBarrier(linear.Handle(), VK_ACCESS_TRANSFER_WRITE_BIT, VK_ACCESS_SHADER_READ_BIT), WholeBufferBarrier(tiled.Handle(), VK_ACCESS_TRANSFER_WRITE_BIT, VK_ACCESS_SHADER_READ_BIT | VK_ACCESS_SHADER_WRITE_BIT)}; + context.Function("vkCmdPipelineBarrier")(commands, VK_PIPELINE_STAGE_TRANSFER_BIT, VK_PIPELINE_STAGE_COMPUTE_SHADER_BIT, 0, 0, nullptr, 2, toShader, 0, nullptr); + for (std::uint32_t layer = 0; layer < arrayLayers; ++layer) { + for (const auto& mip : mips) { + detiler.Dispatch(commands, descriptor.tileMode, elementBytes, linear.Handle(), geometry.LinearLayerOffset(layer) + mip.linearOffset, tiled.Handle(), geometry.GuestLayerOffset(layer) + mip.tiledOffset, mip, true, layer, geometry.thick); + } + } + // Debug aid: APS5_DUMP_STORAGE= saves that storage image's first mip after each of + // its first 8 write-backs as storage_
_.raw (u32 width, height, VkFormat, then rows). + static const std::uint64_t dumpAddress = [] { const char* text = std::getenv("APS5_DUMP_STORAGE"); return text ? std::strtoull(text, nullptr, 16) : 0ull; }(); + static int dumps = 0; + std::unique_ptr dump; + if (dumpAddress != 0 && descriptor.baseAddress == dumpAddress && dumps < 8) { + dump = std::make_unique(context, static_cast(mips[0].linearSize), VK_BUFFER_USAGE_TRANSFER_DST_BIT); + RecordMemoryBarrier(context, commands, VK_PIPELINE_STAGE_TRANSFER_BIT, VK_PIPELINE_STAGE_TRANSFER_BIT, VK_ACCESS_TRANSFER_WRITE_BIT, VK_ACCESS_TRANSFER_READ_BIT); + CopyBuffer(context, commands, linear.Handle(), mips[0].linearOffset, dump->Handle(), 0, mips[0].linearSize); + } + const auto toCopy = WholeBufferBarrier(tiled.Handle(), VK_ACCESS_SHADER_WRITE_BIT, VK_ACCESS_TRANSFER_READ_BIT); + VkImageMemoryBarrier backToGeneral = toSource; + backToGeneral.srcAccessMask = VK_ACCESS_TRANSFER_READ_BIT; + backToGeneral.dstAccessMask = VK_ACCESS_SHADER_READ_BIT | VK_ACCESS_SHADER_WRITE_BIT; + backToGeneral.oldLayout = VK_IMAGE_LAYOUT_TRANSFER_SRC_OPTIMAL; + backToGeneral.newLayout = VK_IMAGE_LAYOUT_GENERAL; + context.Function("vkCmdPipelineBarrier")(commands, VK_PIPELINE_STAGE_COMPUTE_SHADER_BIT | VK_PIPELINE_STAGE_TRANSFER_BIT, VK_PIPELINE_STAGE_TRANSFER_BIT | VK_PIPELINE_STAGE_ALL_COMMANDS_BIT, 0, 0, nullptr, 1, &toCopy, 1, &backToGeneral); + CopyBuffer(context, commands, tiled.Handle(), 0, host.Handle(), 0, original.size()); + RecordMemoryBarrier(context, commands, VK_PIPELINE_STAGE_TRANSFER_BIT, VK_PIPELINE_STAGE_HOST_BIT, VK_ACCESS_TRANSFER_WRITE_BIT, VK_ACCESS_HOST_READ_BIT); + APS5_LOG_CHARS_OUT("StorageTexture writeback submit"); + batch.SubmitAndWait(); + APS5_LOG_CHARS_OUT("StorageTexture writeback done"); + if (profile) Profile().storageGpu += timer.lap(); + if (dump) { + char name[64]; + std::snprintf(name, sizeof(name), "storage_%llx_%d.raw", static_cast(descriptor.baseAddress), dumps++); + if (std::FILE* file = std::fopen(name, "wb")) { + const std::uint32_t header[3] = {mips[0].pitchBytes / static_cast(elementBytes), mips[0].height, static_cast(storageFormat)}; + std::fwrite(header, sizeof(header), 1, file); + std::fwrite(dump->Bytes().data(), 1, dump->Bytes().size(), file); + std::fclose(file); + } + } + // Store only blocks that changed so concurrent CPU writes to untouched texels survive. + const auto current = host.Bytes(); + if (skipWrittenSince == 0) { + GuestMemory::WriteChangedCommitted(descriptor.baseAddress, current, original); + } else { + constexpr std::uint64_t block = 65536; + const auto begin = descriptor.baseAddress; + const auto end = begin + guestBytes; + for (auto at = begin & ~(block - 1); at < end; at += block) { + const auto from = std::max(at, begin); + const auto to = std::min(at + block, end); + if (from >= to || !GuestMemory::UnchangedSince(from, static_cast(to - from), skipWrittenSince)) continue; + const auto offset = static_cast(from - begin); + const auto length = static_cast(to - from); + GuestMemory::WriteChangedCommitted(from, current.subspan(offset, length), std::span(original).subspan(offset, length)); + } + } + std::memcpy(original.data(), current.data(), original.size()); + // The texels now hold the whole image, so later reads must see them rather than a fast clear + // (the keys may be host-imported although the texels were not: then a recorded fill, else a CPU store). + MarkDccUncompressed(context, descriptor.dccAddress, guestBytes); + uploadedKeys = DccKeys::Uncompressed; + // The store above is the only write to these pages, so `original` is current at a fresh + // generation, unless blocks were kept for the CPU: then the image is stale there and the old + // generation stays so the next use re-uploads. + if (skippedAny) originalValid = false; + if (!skippedAny || !adjacent.empty()) { + // As in the GPU-direct path, but the store's memcpy dirtied this surface's pages in the write + // watch: a memoized collect would leave them for the next walk to stamp newer than this value + // (the shared edge block included, undoing the advance), so the walk is made here and + // consumes them. + const auto now = GuestMemory::CollectWritesUncached(descriptor.baseAddress, static_cast(guestBytes)); + if (!skippedAny) generation = now; + advanceAdjacent(adjacent, now, firstBlock, lastBlock); + } + if (profile) Profile().storageStore += timer.lap(); +} + +StorageTexture::~StorageTexture() { + // Cache eviction flushes first; anything still pending here is being torn down with the device. + { + auto& pending = Pending(); + std::lock_guard lock(pending.mutex); + if (dirty) { + dirty = false; + std::erase(pending.textures, this); + static std::atomic reports{0}; + if (reports.fetch_add(1) < 4) std::fprintf(stderr, "[gpu] storage image 0x%llx destroyed with GPU results pending\n", static_cast(descriptor.baseAddress)); + } + } + release(); +} + +void StorageTexture::release() noexcept { + for (const auto& [mip, extra] : extraViews) context.Function("vkDestroyImageView")(context.device, extra, nullptr); + extraViews.clear(); + for (const auto& [format, attachment] : attachmentViews) context.Function("vkDestroyImageView")(context.device, attachment, nullptr); + attachmentViews.clear(); + if (view) context.Function("vkDestroyImageView")(context.device, view, nullptr); + if (image) context.Function("vkDestroyImage")(context.device, image, nullptr); + if (memory) context.Function("vkFreeMemory")(context.device, memory, nullptr); +} + +std::uint64_t StorageTexture::GuestBytes() const { + return guestBytes; +} + +VkImageView StorageTexture::View() const { + return view; +} + } diff --git a/core/libs/prx/libSceAgcDriver/Graphics/src/TextureCache.cpp b/core/libs/prx/libSceAgcDriver/Graphics/src/TextureCache.cpp index 4091d2860..28e970083 100644 --- a/core/libs/prx/libSceAgcDriver/Graphics/src/TextureCache.cpp +++ b/core/libs/prx/libSceAgcDriver/Graphics/src/TextureCache.cpp @@ -1,8 +1,6 @@ #include "prx/libSceAgcDriver/Graphics/include/TextureCache.hpp" #include "prx/libSceAgcDriver/Graphics/include/TextureTiling.hpp" #include "prx/libSceAgcDriver/Execution/include/GuestMemory.hpp" -#include "prx/libSceAgcDriver/Graphics/include/RenderCache.hpp" -#include "prx/libSceAgcDriver/Graphics/include/TextureFormat.hpp" #include #include #include @@ -29,26 +27,9 @@ std::shared_ptr TextureCache::Get(std::span words, trim(); std::array key; std::copy(words.begin(), words.end(), key.begin()); - auto source = context.renderCache ? context.renderCache->Find(resource.baseAddress) : nullptr; - if (source) { - const auto& color = source->Description(); - const auto compatibleTiling = (color.tileMode == ColorTileMode::RenderTarget && resource.tileMode == TextureTileMode::RenderTarget64KB) || (color.tileMode == ColorTileMode::Linear && resource.tileMode == TextureTileMode::kLinear); - if (!compatibleTiling || resource.width != color.extent.width || resource.height != color.extent.height || resource.dimension != TextureDimension::k2D || resource.mipCount != 1 || resource.baseLevel != 0 || resource.baseArray != 0 || IsBlockCompressed(resource.format) || BytesPerElement(resource.format) != 4) source.reset(); - } for (auto it = entries.begin(); it != entries.end(); ++it) { if (it->descriptor != key) continue; - if (source || it->generation != 0) { - if (source && it->source.lock() == source && it->generation == source->Generation()) { - auto result = it->texture; - entries.splice(entries.end(), entries, it); - return result; - } - retainedBytes -= it->snapshot.size() + it->texture->AllocationBytes(); - entries.erase(it); - break; - } - GuestMemory::CheckRange(reinterpret_cast(resource.baseAddress), it->snapshot.size(), 1); - if (std::memcmp(reinterpret_cast(resource.baseAddress), it->snapshot.data(), it->snapshot.size()) == 0) { + if (GuestMemory::EqualsCommitted(resource.baseAddress, it->snapshot)) { auto result = it->texture; entries.splice(entries.end(), entries, it); return result; @@ -57,19 +38,12 @@ std::shared_ptr TextureCache::Get(std::span words, entries.erase(it); break; } - if (source) { - auto texture = std::make_shared(context, source, resource, components); - entries.push_back({key, {}, texture, source, source->Generation()}); - retainedBytes += texture->AllocationBytes(); - trim(); - return texture; - } const auto mips = ComputeMipLayout(resource.tileMode, resource.format, resource.width, resource.height, resource.mipCount); const auto layers = resource.dimension == TextureDimension::k2DArray || resource.dimension == TextureDimension::kCube ? resource.depthOrLastArray + 1u : 1u; const auto bytes = ComputeSurfaceSize(mips, layers); Require(bytes != 0 && bytes <= std::numeric_limits::max(), "texture cache surface size overflow"); std::vector snapshot(static_cast(bytes)); - GuestMemory::Read(resource.baseAddress, snapshot, 1); + GuestMemory::ReadCommitted(resource.baseAddress, snapshot); auto texture = std::make_shared(context, *context.detiler, resource, components, snapshot); const auto retained = snapshot.size() + texture->AllocationBytes(); entries.push_back({key, std::move(snapshot), texture}); diff --git a/core/libs/prx/libSceAgcDriver/Graphics/src/TextureDetiler.cpp b/core/libs/prx/libSceAgcDriver/Graphics/src/TextureDetiler.cpp index 0b65f2b3c..b6bcab80d 100644 --- a/core/libs/prx/libSceAgcDriver/Graphics/src/TextureDetiler.cpp +++ b/core/libs/prx/libSceAgcDriver/Graphics/src/TextureDetiler.cpp @@ -1,4 +1,6 @@ #include "prx/libSceAgcDriver/Graphics/include/TextureDetiler.hpp" +#include "prx/libSceAgcDriver/Graphics/include/TextureSwizzleEquations.hpp" +#include "prx/libSceAgcDriver/Graphics/include/TextureTiling.hpp" #include "prx/libSceAgcDriver/Graphics/shaders/TextureDetile_spv.h" #include #include @@ -22,7 +24,7 @@ struct Push { std::uint32_t tailX; std::uint32_t tailY; std::uint32_t elementBytes; - std::uint32_t arrayLayer; + std::uint32_t slice; }; std::uint32_t BlockBytesFor(TextureTileMode tileMode) { @@ -30,14 +32,17 @@ std::uint32_t BlockBytesFor(TextureTileMode tileMode) { case TextureTileMode::kLinear: return 0u; case TextureTileMode::kStandard256B: return 256u; case TextureTileMode::kStandard4KB: return 4096u; - case TextureTileMode::RenderTarget64KB: - case TextureTileMode::kStandard64KB: return 65536u; + case TextureTileMode::kStandard64KB: + case TextureTileMode::kZ64KBX: + case TextureTileMode::kS64KBX: + case TextureTileMode::kD64KBX: + case TextureTileMode::kR64KBX: return 65536u; } throw std::runtime_error("AGC graphics: TextureDetiler encountered an unknown tile mode"); } -std::uint32_t PipelineKey(TextureTileMode tileMode, std::uint32_t elementBytes) { - return (static_cast(tileMode) << 8) | elementBytes; +std::uint32_t PipelineKey(TextureTileMode tileMode, std::uint32_t elementBytes, bool retile, bool thick) { + return (thick ? 1u << 17 : 0u) | (retile ? 1u << 16 : 0u) | (static_cast(tileMode) << 8) | elementBytes; } } @@ -87,19 +92,38 @@ void TextureDetiler::release() noexcept { if (descriptorLayout) context.Function("vkDestroyDescriptorSetLayout")(context.device, descriptorLayout, nullptr); } -VkPipeline TextureDetiler::pipeline(TextureTileMode tileMode, std::uint32_t elementBytes) { +VkPipeline TextureDetiler::pipeline(TextureTileMode tileMode, std::uint32_t elementBytes, bool retile, bool thick) { Require(std::has_single_bit(elementBytes) && elementBytes <= 16u, "unsupported element size for texture detiling"); - const auto key = PipelineKey(tileMode, elementBytes); + const auto key = PipelineKey(tileMode, elementBytes, retile, thick); for (const auto& entry : pipelines) { if (entry.first == key) return entry.second; } - const std::uint32_t values[3] = {elementBytes, BlockBytesFor(tileMode), tileMode == TextureTileMode::kLinear ? 0u : (tileMode == TextureTileMode::RenderTarget64KB ? 2u : 1u)}; - const VkSpecializationMapEntry entries[3] = {{0, 0, 4}, {1, 4, 4}, {2, 8, 4}}; + // Constants 0-3 select element size, block size, addressing family and direction; 4-19 carry the + // per-bit XOR equation for the equation family (2); 20-21 override the block extent in elements. + std::array values{elementBytes, BlockBytesFor(tileMode), tileMode == TextureTileMode::kLinear ? 0u : 1u, retile ? 1u : 0u}; + if (thick) { + const auto thickMode = tileMode == TextureTileMode::kStandard4KB ? 0x105u : tileMode == TextureTileMode::kStandard64KB ? 0x109u : 0u; + Require(thickMode != 0, "3D textures are only detiled from SW_4KB_S or SW_64KB_S"); + const auto* equation = FindTextureSwizzleEquation(thickMode, elementBytes); + Require(equation != nullptr, "no thick swizzle equation for the element size"); + values[2] = 2u; + std::copy(equation->bits.begin(), equation->bits.end(), values.begin() + 4); + const auto extent = ThickBlockExtent(tileMode, elementBytes); + values[20] = extent[0]; + values[21] = extent[1]; + } else if (const auto mode = XorSwizzleMode(tileMode); mode != 0) { + const auto* equation = FindTextureSwizzleEquation(mode, elementBytes); + Require(equation != nullptr, "no swizzle equation for tile mode " + std::to_string(mode) + " at " + std::to_string(elementBytes) + " bytes per element"); + values[2] = 2u; + std::copy(equation->bits.begin(), equation->bits.end(), values.begin() + 4); + } + std::array entries{}; + for (std::uint32_t index = 0; index < entries.size(); ++index) entries[index] = {index, index * 4u, 4}; VkSpecializationInfo specialization{}; - specialization.mapEntryCount = 3; - specialization.pMapEntries = entries; + specialization.mapEntryCount = static_cast(entries.size()); + specialization.pMapEntries = entries.data(); specialization.dataSize = sizeof(values); - specialization.pData = values; + specialization.pData = values.data(); VkPipelineShaderStageCreateInfo stage{VK_STRUCTURE_TYPE_PIPELINE_SHADER_STAGE_CREATE_INFO}; stage.stage = VK_SHADER_STAGE_COMPUTE_BIT; stage.module = module; @@ -114,20 +138,20 @@ VkPipeline TextureDetiler::pipeline(TextureTileMode tileMode, std::uint32_t elem return result; } -void TextureDetiler::Dispatch(VkCommandBuffer commands, TextureTileMode tileMode, std::uint32_t elementBytes, VkBuffer source, std::uint64_t sourceOffset, VkBuffer destination, std::uint64_t destinationOffset, const TileMipLayout& layout, std::uint32_t arrayLayer) { +void TextureDetiler::Dispatch(VkCommandBuffer commands, TextureTileMode tileMode, std::uint32_t elementBytes, VkBuffer source, std::uint64_t sourceOffset, VkBuffer destination, std::uint64_t destinationOffset, const TileMipLayout& layout, bool retile, std::uint32_t slice, bool thick) { Require(commands != VK_NULL_HANDLE, "texture detiling requires an active command buffer"); Require(source != VK_NULL_HANDLE && destination != VK_NULL_HANDLE, "texture detiling requires source and destination buffers"); Require(layout.width != 0 && layout.height != 0, "texture detiling requires a non-empty mip layout"); Require(layout.tiledSize != 0 && layout.linearSize != 0, "texture detiling requires a non-empty mip layout"); - const auto target = pipeline(tileMode, elementBytes); + const auto target = pipeline(tileMode, elementBytes, retile, thick); const auto alignment = std::max(context.limits.minStorageBufferOffsetAlignment, 4); const auto sourceDescriptorOffset = sourceOffset - sourceOffset % alignment; const auto destinationDescriptorOffset = destinationOffset - destinationOffset % alignment; const auto sourceBase = sourceOffset - sourceDescriptorOffset; const auto destinationBase = destinationOffset - destinationDescriptorOffset; Require(sourceBase <= UINT32_MAX && destinationBase <= UINT32_MAX, "texture detiling buffer offset exceeds addressable range"); - const auto sourceRange = (sourceBase + layout.tiledSize + 3) / 4 * 4; - const auto destinationRange = (destinationBase + layout.linearSize + 3) / 4 * 4; + const auto sourceRange = (sourceBase + (retile ? layout.linearSize : layout.tiledSize) + 3) / 4 * 4; + const auto destinationRange = (destinationBase + (retile ? layout.tiledSize : layout.linearSize) + 3) / 4 * 4; Require(sourceRange <= context.limits.maxStorageBufferRange && destinationRange <= context.limits.maxStorageBufferRange, "texture detiling buffer range exceeds device limits"); const auto set = allocateSet(); const VkDescriptorBufferInfo sourceInfo{source, sourceDescriptorOffset, sourceRange}; @@ -159,7 +183,7 @@ void TextureDetiler::Dispatch(VkCommandBuffer commands, TextureTileMode tileMode push.tailX = layout.tailX; push.tailY = layout.tailY; push.elementBytes = elementBytes; - push.arrayLayer = arrayLayer; + push.slice = slice; context.Function("vkCmdPushConstants")(commands, pipelineLayout, VK_SHADER_STAGE_COMPUTE_BIT, 0, sizeof(Push), &push); const auto groupsX = (layout.width + 7u) / 8u; const auto groupsY = (layout.height + 7u) / 8u; diff --git a/core/libs/prx/libSceAgcDriver/Graphics/src/TextureDetilerDescriptors.cpp b/core/libs/prx/libSceAgcDriver/Graphics/src/TextureDetilerDescriptors.cpp index e9da522f7..ed3b905e4 100644 --- a/core/libs/prx/libSceAgcDriver/Graphics/src/TextureDetilerDescriptors.cpp +++ b/core/libs/prx/libSceAgcDriver/Graphics/src/TextureDetilerDescriptors.cpp @@ -1,8 +1,14 @@ +#include "prx/libSceAgcDriver/Graphics/include/Recorder.hpp" #include "prx/libSceAgcDriver/Graphics/include/TextureDetiler.hpp" +#include "prx/libSceAgcDriver/Execution/include/GuestMemory.hpp" namespace AgcDriver::Graphics { void TextureDetiler::BeginBatch() { + GuestMemory::AssertGpuLockHeld("TextureDetiler::BeginBatch"); + // Sets of recorded work that has not completed stay allocated; the pools are recycled once the + // recorder is idle again. + if (const auto* recorder = Recorder::Active(); recorder != nullptr && !recorder->Idle()) return; for (const auto pool : descriptorPools) { Check(context.Function("vkResetDescriptorPool")(context.device, pool, 0), "vkResetDescriptorPool texture detiler"); } diff --git a/core/libs/prx/libSceAgcDriver/Graphics/src/TextureTiling.cpp b/core/libs/prx/libSceAgcDriver/Graphics/src/TextureTiling.cpp index e56ecffee..9afbe8167 100644 --- a/core/libs/prx/libSceAgcDriver/Graphics/src/TextureTiling.cpp +++ b/core/libs/prx/libSceAgcDriver/Graphics/src/TextureTiling.cpp @@ -45,8 +45,11 @@ BlockLayout GetBlockLayout(TextureTileMode tileMode, std::uint32_t bytesPerEleme case TextureTileMode::kLinear: throw std::runtime_error("AGC graphics: GetBlockLayout does not apply to linear tiling"); case TextureTileMode::kStandard256B: return {256u, 1u << kLog2BlockThin256B[index].width, 1u << kLog2BlockThin256B[index].height}; case TextureTileMode::kStandard4KB: return {4096u, 1u << kLog2BlockThin4KB[index].width, 1u << kLog2BlockThin4KB[index].height}; - case TextureTileMode::RenderTarget64KB: - case TextureTileMode::kStandard64KB: return {65536u, 1u << kLog2BlockThin64KB[index].width, 1u << kLog2BlockThin64KB[index].height}; + case TextureTileMode::kStandard64KB: + case TextureTileMode::kZ64KBX: + case TextureTileMode::kS64KBX: + case TextureTileMode::kD64KBX: + case TextureTileMode::kR64KBX: return {65536u, 1u << kLog2BlockThin64KB[index].width, 1u << kLog2BlockThin64KB[index].height}; } throw std::runtime_error("AGC graphics: GetBlockLayout encountered an unknown tile mode"); } @@ -92,8 +95,11 @@ bool GetMipTailLayout(TextureTileMode tileMode, const BlockLayout& block, std::u case TextureTileMode::kStandard4KB: out = MakeMipTailLayout(kMipTailThin4KB[index], block.blockWidth >> 1u, block.blockHeight); return true; - case TextureTileMode::RenderTarget64KB: case TextureTileMode::kStandard64KB: + case TextureTileMode::kZ64KBX: + case TextureTileMode::kS64KBX: + case TextureTileMode::kD64KBX: + case TextureTileMode::kR64KBX: out = MakeMipTailLayout(kMipTailThin64KB[index], block.blockWidth >> 1u, block.blockHeight); return true; } @@ -167,8 +173,10 @@ std::vector ComputeTiledMipLayout(TextureTileMode tileMode, std:: const auto paddedWidth = AlignUp(std::max(ShiftCeil(elementsWidth0, level), 1u), block.blockWidth); const auto paddedHeight = AlignUp(std::max(ShiftCeil(elementsHeight0, level), 1u), block.blockHeight); mip.blocksPerRow = paddedWidth / block.blockWidth; + mip.pitchBytes = paddedWidth * bytesPerElement; mip.tiledSize = static_cast(paddedWidth) * paddedHeight * bytesPerElement; - mip.linearSize = static_cast(mip.width) * mip.height * bytesPerElement; + // The detiler writes linear rows at pitchBytes, so the region spans whole padded rows. + mip.linearSize = static_cast(mip.pitchBytes) * mip.height; mip.tail = false; mip.tailX = 0; mip.tailY = 0; @@ -182,8 +190,9 @@ std::vector ComputeTiledMipLayout(TextureTileMode tileMode, std:: mip.width = TexelLevelDimension(width, level, texelWidth); mip.height = TexelLevelDimension(height, level, texelHeight); mip.blocksPerRow = 1u; + mip.pitchBytes = block.blockWidth * bytesPerElement; mip.tiledSize = block.blockSize; - mip.linearSize = static_cast(mip.width) * mip.height * bytesPerElement; + mip.linearSize = static_cast(mip.pitchBytes) * mip.height; mip.tail = true; mip.tailX = tail.locations[level - firstTailLevel].x; mip.tailY = tail.locations[level - firstTailLevel].y; @@ -192,10 +201,17 @@ std::vector ComputeTiledMipLayout(TextureTileMode tileMode, std:: std::uint64_t offset = firstTailLevel < mipCount ? block.blockSize : 0; for (auto level = firstTailLevel; level-- > 0;) { mips[level].tiledOffset = offset; + mips[level].linearOffset = offset; offset += mips[level].tiledSize; } + // Tail mips share one tiled block but each needs its own linear region, placed after the chain. + std::uint64_t linearCursor = 0; + for (auto level = std::uint32_t{0}; level < firstTailLevel; ++level) linearCursor = std::max(linearCursor, mips[level].linearOffset + mips[level].linearSize); + linearCursor = std::max(linearCursor, offset); for (auto level = firstTailLevel; level < mipCount; ++level) { mips[level].tiledOffset = 0; + mips[level].linearOffset = linearCursor; + linearCursor += mips[level].linearSize; } Require(offset == blockSliceSize, "tiled texture mip chain geometry is inconsistent"); @@ -203,15 +219,6 @@ std::vector ComputeTiledMipLayout(TextureTileMode tileMode, std:: Require(mip.width != 0 && mip.height != 0, "computed a zero-sized tiled texture mip level"); Require(mip.tiledSize != 0 && mip.linearSize != 0, "computed a zero-sized tiled texture mip level"); } - std::uint64_t linearOffset = 0; - for (auto& mip : mips) { - const auto alignment = std::max(bytesPerElement, 4u); - linearOffset = (linearOffset + alignment - 1u) / alignment * alignment; - mip.linearOffset = linearOffset; - mip.pitchBytes = mip.width * bytesPerElement; - mip.linearSize = (static_cast(mip.pitchBytes) * mip.height + alignment - 1u) / alignment * alignment; - linearOffset += mip.linearSize; - } return mips; } @@ -225,14 +232,88 @@ std::vector ComputeMipLayout(TextureTileMode tileMode, std::uint3 const auto texelWidth = BlockWidth(format); const auto texelHeight = BlockHeight(format); - if (tileMode == TextureTileMode::RenderTarget64KB) { - Require(!IsBlockCompressed(format), "render target tiling does not support block compressed formats"); - Require(format != 128 && format != 129 && format != 132, "texture format does not support render target tiling"); - } if (tileMode == TextureTileMode::kLinear) return ComputeLinearMipLayout(bytesPerElement, texelWidth, texelHeight, width, height, mipCount); return ComputeTiledMipLayout(tileMode, bytesPerElement, texelWidth, texelHeight, width, height, mipCount); } +std::array ThickBlockExtent(TextureTileMode tileMode, std::uint32_t bytesPerElement) { + // addrlib Block4K_Log2_3d / Block64K_Log2_3d. + constexpr std::uint8_t thick4KB[5][3] = {{4, 4, 4}, {3, 4, 4}, {3, 4, 3}, {3, 3, 3}, {2, 3, 3}}; + constexpr std::uint8_t thick64KB[5][3] = {{6, 5, 5}, {5, 5, 5}, {5, 5, 4}, {5, 4, 4}, {4, 4, 4}}; + Require(std::has_single_bit(bytesPerElement) && bytesPerElement <= 16u, "unsupported bytes per element for thick texture geometry"); + const auto index = static_cast(std::countr_zero(bytesPerElement)); + switch (tileMode) { + case TextureTileMode::kStandard4KB: return {1u << thick4KB[index][0], 1u << thick4KB[index][1], 1u << thick4KB[index][2]}; + case TextureTileMode::kStandard64KB: return {1u << thick64KB[index][0], 1u << thick64KB[index][1], 1u << thick64KB[index][2]}; + default: throw std::runtime_error("AGC graphics: 3D textures are only supported linear or in SW_4KB_S / SW_64KB_S"); + } +} + +ThickLayout ComputeThickLayout(TextureTileMode tileMode, std::uint32_t format, std::uint32_t width, std::uint32_t height, std::uint32_t depth) { + Require(width != 0 && height != 0 && depth != 0, "cannot compute layout for a zero-sized 3D texture"); + const auto bytesPerElement = BytesPerElement(format); + Require(BlockWidth(format) == 1u && BlockHeight(format) == 1u, "block-compressed 3D textures are not implemented"); + ThickLayout result{}; + result.depth = depth; + auto& mip = result.mip; + mip.width = width; + mip.height = height; + mip.tail = false; + if (tileMode == TextureTileMode::kLinear) { + const auto paddedWidth = AlignUp(width, CalcLinearBlockWidth(bytesPerElement)); + mip.blocksPerRow = paddedWidth; + mip.pitchBytes = paddedWidth * bytesPerElement; + mip.tiledSize = static_cast(mip.pitchBytes) * height; + result.blockDepth = 1; + result.slabBytes = mip.tiledSize; + } else { + const auto block = ThickBlockExtent(tileMode, bytesPerElement); + const auto paddedWidth = AlignUp(width, block[0]); + const auto paddedHeight = AlignUp(height, block[1]); + mip.blocksPerRow = paddedWidth / block[0]; + mip.pitchBytes = paddedWidth * bytesPerElement; + result.blockDepth = block[2]; + result.slabBytes = static_cast(mip.blocksPerRow) * (paddedHeight / block[1]) * (tileMode == TextureTileMode::kStandard4KB ? 4096u : 65536u); + mip.tiledSize = result.slabBytes; + } + mip.linearSize = static_cast(mip.pitchBytes) * height; + result.sliceLinearBytes = mip.linearSize; + result.guestBytes = result.slabBytes * ((depth + result.blockDepth - 1u) / result.blockDepth); + return result; +} + +SurfaceGeometry DescribeSurface(const GuestTextureResource& descriptor) { + SurfaceGeometry geometry; + if (descriptor.dimension == TextureDimension::k3D) { + Require(descriptor.mipCount == 1, "mipmapped 3D textures are not implemented"); + const auto depth = descriptor.depthOrLastArray + 1u; + const auto thick = ComputeThickLayout(descriptor.tileMode, descriptor.format, descriptor.width, descriptor.height, depth); + geometry.mips = {thick.mip}; + geometry.layers = depth; + geometry.imageDepth = depth; + geometry.guestBytes = thick.guestBytes; + geometry.sliceLinearBytes = thick.sliceLinearBytes; + geometry.thick = descriptor.tileMode != TextureTileMode::kLinear; + geometry.blockDepth = thick.blockDepth; + geometry.layerBytes = thick.slabBytes; + return geometry; + } + geometry.mips = ComputeMipLayout(descriptor.tileMode, descriptor.format, descriptor.width, descriptor.height, descriptor.mipCount); + const bool layered = descriptor.dimension == TextureDimension::k2DArray || descriptor.dimension == TextureDimension::kCube; + geometry.layers = layered ? descriptor.depthOrLastArray + 1u : 1u; + geometry.imageLayers = geometry.layers; + geometry.guestBytes = ComputeSurfaceSize(geometry.mips, geometry.layers); + geometry.layerBytes = geometry.guestBytes / geometry.layers; + for (const auto& mip : geometry.mips) geometry.sliceLinearBytes = std::max(geometry.sliceLinearBytes, mip.linearOffset + mip.linearSize); + return geometry; +} + +std::vector ComputeElementMipLayout(TextureTileMode tileMode, std::uint32_t bytesPerElement, std::uint32_t width, std::uint32_t height, std::uint32_t mipCount) { + Require(width != 0 && height != 0 && mipCount != 0 && mipCount <= 16u, "invalid surface mip chain"); + if (tileMode == TextureTileMode::kLinear) return ComputeLinearMipLayout(bytesPerElement, 1u, 1u, width, height, mipCount); + return ComputeTiledMipLayout(tileMode, bytesPerElement, 1u, 1u, width, height, mipCount); +} + std::uint64_t ComputeSurfaceSize(const std::vector& mips, std::uint32_t arrayLayers) { Require(!mips.empty(), "cannot compute surface size for an empty mip chain"); Require(arrayLayers != 0, "cannot compute surface size for zero array layers"); diff --git a/core/libs/prx/libSceAgcDriver/PostCompile.cmake b/core/libs/prx/libSceAgcDriver/PostCompile.cmake index 072eeed37..b48b71293 100644 --- a/core/libs/prx/libSceAgcDriver/PostCompile.cmake +++ b/core/libs/prx/libSceAgcDriver/PostCompile.cmake @@ -12,25 +12,19 @@ add_custom_command( foreach(agcTarget IN ITEMS libSceAgcDriver agc_driver_visual_test agc_driver_graphics_tests agc_driver_bda_device_tests) if(TARGET ${agcTarget}) - target_sources(${agcTarget} PRIVATE Graphics/src/BufferPool.cpp Graphics/src/CommandCompletion.cpp) - endif() -endforeach() - -foreach(agcTarget IN ITEMS libSceAgcDriver agc_driver_visual_test agc_driver_graphics_tests agc_driver_bda_device_tests) - if(TARGET ${agcTarget}) - target_sources(${agcTarget} PRIVATE Execution/src/GuestGpuRange.cpp) + target_sources(${agcTarget} PRIVATE Graphics/src/BufferPool.cpp) endif() endforeach() foreach(agcTarget IN ITEMS libSceAgcDriver agc_driver_visual_test agc_driver_graphics_tests) if(TARGET ${agcTarget}) - target_sources(${agcTarget} PRIVATE Graphics/src/TextureDetilerDescriptors.cpp Graphics/src/TextureCache.cpp Graphics/src/RegisteredGpuMemory.cpp) + target_sources(${agcTarget} PRIVATE Graphics/src/TextureDetilerDescriptors.cpp Graphics/src/TextureCache.cpp) endif() endforeach() foreach(agcTarget IN ITEMS libSceAgcDriver agc_driver_visual_test agc_driver_graphics_tests agc_driver_bda_device_tests) if(TARGET ${agcTarget}) - target_sources(${agcTarget} PRIVATE Graphics/src/GpuColorTransfer.cpp Graphics/src/ColorMemoryWriteback.cpp ${agcColorTransferHeader}) + target_sources(${agcTarget} PRIVATE Graphics/src/GpuColorTransfer.cpp ${agcColorTransferHeader}) endif() endforeach() @@ -41,14 +35,6 @@ if(TARGET agc_driver_visual_test) target_sources(agc_driver_visual_test PRIVATE Execution/src/DisplayBuffer.cpp) endif() -foreach(agcTarget IN ITEMS libSceAgcDriver agc_driver_visual_test agc_driver_graphics_tests) - if(TARGET ${agcTarget}) - target_sources(${agcTarget} PRIVATE Graphics/src/RenderCache.cpp Graphics/src/RenderMemoryOwnership.cpp Graphics/src/DrawQueue.cpp Graphics/src/DrawCompletion.cpp Graphics/src/RenderTexture.cpp Graphics/src/DescriptorCache.cpp) - endif() -endforeach() - -foreach(agcTarget IN ITEMS libSceAgcDriver agc_driver_visual_test) - if(TARGET ${agcTarget}) - target_sources(${agcTarget} PRIVATE Execution/src/PresentationImage.cpp Execution/src/DeviceSynchronization.cpp Graphics/src/GraphicsPipelineCache.cpp) - endif() -endforeach() +# Upstream's GPU-resident draw path (RenderCache, DrawQueue, RenderTexture, PresentationImage, +# GraphicsPipelineCache) targets the single-target Pipeline/GpuColorTransfer API; the multi-target +# draw path in Draw.cpp does not use it yet, so those sources stay out of the build. diff --git a/core/libs/prx/libSceAgcDriver/Resource/src/Registration.cpp b/core/libs/prx/libSceAgcDriver/Resource/src/Registration.cpp index 35e29c61c..4de64c2dd 100644 --- a/core/libs/prx/libSceAgcDriver/Resource/src/Registration.cpp +++ b/core/libs/prx/libSceAgcDriver/Resource/src/Registration.cpp @@ -5,16 +5,26 @@ #include "SceTypes.hpp" #include "prx/libc/include/General.hpp" +// Resource registration feeds GPU debugging tools, none of which are attached; the game treats this code as benign. +static constexpr int SCE_AGC_ERROR_RESOURCE_REGISTRATION_UNAVAILABLE = static_cast(0x8A6C9018); + extern "C" { -int APS5_VABI sceAgcDriverRegisterOwner(void) { - NotImplemented_nid_no_patch(__func__); - return 0; +int APS5_VABI sceAgcDriverRegisterOwner(uint32_t* owner_handle, const char* name) { + (void)owner_handle; + (void)name; + return SCE_AGC_ERROR_RESOURCE_REGISTRATION_UNAVAILABLE; } -int APS5_VABI sceAgcDriverRegisterResource(void) { - NotImplemented_nid_no_patch(__func__); - return 0; +int APS5_VABI sceAgcDriverRegisterResource(uint32_t* resource_handle, uint32_t owner_handle, const void* memory, size_t size, const char* name, uint32_t type, uint64_t user_data) { + (void)resource_handle; + (void)owner_handle; + (void)memory; + (void)size; + (void)name; + (void)type; + (void)user_data; + return SCE_AGC_ERROR_RESOURCE_REGISTRATION_UNAVAILABLE; } int APS5_VABI sceAgcDriverRegisterWorkloadStream(uint32_t stream_id, const void* stream) { @@ -25,14 +35,13 @@ int APS5_VABI sceAgcDriverRegisterWorkloadStream(uint32_t stream_id, const void* } int APS5_VABI sceAgcDriverUnregisterOwnerAndResources(uint32_t owner_handle) { - (void)owner_handle; - NotImplemented_nid_no_patch(__func__); - return 0; + (void)owner_handle; + return SCE_AGC_ERROR_RESOURCE_REGISTRATION_UNAVAILABLE; } -int APS5_VABI sceAgcDriverUnregisterResource(void) { - NotImplemented_nid_no_patch(__func__); - return 0; +int APS5_VABI sceAgcDriverUnregisterResource(uint32_t resource_handle) { + (void)resource_handle; + return SCE_AGC_ERROR_RESOURCE_REGISTRATION_UNAVAILABLE; } } diff --git a/core/libs/prx/libSceAgcDriver/Unimplemented.cpp b/core/libs/prx/libSceAgcDriver/Unimplemented.cpp new file mode 100644 index 000000000..8166a546f --- /dev/null +++ b/core/libs/prx/libSceAgcDriver/Unimplemented.cpp @@ -0,0 +1,112 @@ +#include +#include +#include "SceTypes.hpp" +#include "prx/libc/include/General.hpp" + +extern "C" { + +int APS5_VABI sceAgcDriverFindResourcesPublic() { + NotImplemented_nid_no_patch(__func__); + return 0; +} + +int APS5_VABI sceAgcDriverGetDefaultOwner() { + NotImplemented_nid_no_patch(__func__); + return 0; +} + +int APS5_VABI sceAgcDriverGetOwnerName() { + NotImplemented_nid_no_patch(__func__); + return 0; +} + +int APS5_VABI sceAgcDriverGetResourceBaseAddressAndSizeInBytes() { + NotImplemented_nid_no_patch(__func__); + return 0; +} + +int APS5_VABI sceAgcDriverGetResourceName() { + NotImplemented_nid_no_patch(__func__); + return 0; +} + +int APS5_VABI sceAgcDriverGetResourceRegistrationMaxNameLength() { + NotImplemented_nid_no_patch(__func__); + return 0; +} + +int APS5_VABI sceAgcDriverGetResourceShaderGuid() { + NotImplemented_nid_no_patch(__func__); + return 0; +} + +int APS5_VABI sceAgcDriverGetResourceType() { + NotImplemented_nid_no_patch(__func__); + return 0; +} + +int APS5_VABI sceAgcDriverGetResourceUserData() { + NotImplemented_nid_no_patch(__func__); + return 0; +} + +int APS5_VABI sceAgcDriverGetShaderDebuggingStatus() { + NotImplemented_nid_no_patch(__func__); + return 0; +} + +int APS5_VABI sceAgcDriverRegisterDefaultOwner(uint32_t* owner_handle) { + (void)owner_handle; + return static_cast(0x8A6C9018); +} + +int APS5_VABI sceAgcDriverRegisterGdsResource() { + NotImplemented_nid_no_patch(__func__); + return 0; +} + +int APS5_VABI sceAgcDriverSetResourceUserData() { + NotImplemented_nid_no_patch(__func__); + return 0; +} + +int APS5_VABI sceAgcDriverSetValidationErrorOutputFrequency() { + NotImplemented_nid_no_patch(__func__); + return 0; +} + +int APS5_VABI sceAgcDriverUnregisterAllResourcesForOwner() { + NotImplemented_nid_no_patch(__func__); + return 0; +} + +int APS5_VABI sceAgcDriverUnregisterWorkloadStream() { + NotImplemented_nid_no_patch(__func__); + return 0; +} + +APS5_EXPORT("+iAOE3jCnkk", sceAgcDriverUnknown__PiAOE3jCnkk); +int APS5_VABI sceAgcDriverUnknown__PiAOE3jCnkk() { + NotImplemented_nid_no_patch(__func__); + return 0; +} + +APS5_EXPORT("ICaGtkEIXTk", sceAgcDriverUnknown_ICaGtkEIXTk); +int APS5_VABI sceAgcDriverUnknown_ICaGtkEIXTk() { + NotImplemented_nid_no_patch(__func__); + return 0; +} + +APS5_EXPORT("WHIOMbb+iIU", sceAgcDriverUnknown_WHIOMbb_PiIU); +int APS5_VABI sceAgcDriverUnknown_WHIOMbb_PiIU() { + NotImplemented_nid_no_patch(__func__); + return 0; +} + +APS5_EXPORT("chJWZcNSzjk", sceAgcDriverUnknown_chJWZcNSzjk); +int APS5_VABI sceAgcDriverUnknown_chJWZcNSzjk() { + NotImplemented_nid_no_patch(__func__); + return 0; +} + +} diff --git a/core/libs/prx/libSceAgcDriver/tests/GuestSamplerResource.cpp b/core/libs/prx/libSceAgcDriver/tests/GuestSamplerResource.cpp index 616cb47b4..fe60064b0 100644 --- a/core/libs/prx/libSceAgcDriver/tests/GuestSamplerResource.cpp +++ b/core/libs/prx/libSceAgcDriver/tests/GuestSamplerResource.cpp @@ -149,21 +149,17 @@ void RunGuestSamplerResourceTests() { badUnorm.forceUnormCoords = true; rejectFields(badUnorm, "unnormalized coordinates"); - Fields badThreshold = base; - badThreshold.anisoThreshold = 1; - rejectFields(badThreshold, "anisotropy threshold override"); - Fields badSrgb = base; badSrgb.forceSrgb = true; rejectFields(badSrgb, "forces sRGB decoding"); - Fields badAnisoBias = base; - badAnisoBias.anisoBias = 1; - rejectFields(badAnisoBias, "anisotropy bias"); - - Fields badTrunc = base; - badTrunc.truncCoord = true; - rejectFields(badTrunc, "coordinate truncation"); + Fields truncated = base; + truncated.truncCoord = true; + truncated.perfMip = 1; + truncated.perfZ = 1; + truncated.anisoThreshold = 1; + truncated.anisoBias = 1; + static_cast(DecodeSamplerResource(pack(truncated))); Fields badCubeWrap = base; badCubeWrap.disableCubeWrap = true; @@ -177,13 +173,6 @@ void RunGuestSamplerResourceTests() { badDegamma.disableDegamma = true; rejectFields(badDegamma, "disables degamma"); - Fields badPerf = base; - badPerf.perfMip = 1; - rejectFields(badPerf, "performance counters"); - badPerf = base; - badPerf.perfZ = 1; - rejectFields(badPerf, "performance counters"); - Fields badLodBiasSec = base; badLodBiasSec.lodBiasSec = 1; rejectFields(badLodBiasSec, "secondary LOD bias"); diff --git a/core/libs/prx/libSceAgcDriver/tests/GuestTextureResource.cpp b/core/libs/prx/libSceAgcDriver/tests/GuestTextureResource.cpp index e22ad9c31..eedc6e271 100644 --- a/core/libs/prx/libSceAgcDriver/tests/GuestTextureResource.cpp +++ b/core/libs/prx/libSceAgcDriver/tests/GuestTextureResource.cpp @@ -179,9 +179,12 @@ void RunGuestTextureResourceTests() { badMinLod.minLod = 1; rejectFields(badMinLod, "nonzero minimum LOD clamp"); - Fields badMinLodWarn = base; - badMinLodWarn.minLodWarn = 1; - rejectFields(badMinLodWarn, "minimum LOD warning threshold"); + // Streaming feedback fields decode; nothing is reported back. + Fields feedback = base; + feedback.minLodWarn = 1; + feedback.mipStatsCntId = 1; + feedback.mipStatsCntEn = true; + static_cast(DecodeTextureResource(pack(feedback))); const auto unmodulated = DecodeTextureResource(pack(base)); for (std::uint32_t perfMod = 0; perfMod < 8; ++perfMod) { @@ -196,13 +199,6 @@ void RunGuestTextureResourceTests() { rejectFields(modulated, "corner sampling"); } - Fields badMipStats = base; - badMipStats.mipStatsCntId = 1; - rejectFields(badMipStats, "mip statistics counters"); - badMipStats = base; - badMipStats.mipStatsCntEn = true; - rejectFields(badMipStats, "mip statistics counters"); - Fields badCorner = base; badCorner.cornerSample = true; rejectFields(badCorner, "corner sampling"); diff --git a/core/libs/prx/libSceAgcDriver/tools/ShaderReplay.cpp b/core/libs/prx/libSceAgcDriver/tools/ShaderReplay.cpp new file mode 100644 index 000000000..4536f474b --- /dev/null +++ b/core/libs/prx/libSceAgcDriver/tools/ShaderReplay.cpp @@ -0,0 +1,125 @@ +// agc_shader_replay ...: replays compute recompile requests the driver saved with +// APS5_DUMP_SHADERS=1, running the same resource analysis and recompile steps without the game. +#include "ControlFlow/RequestSerializer.hpp" +#include "Optimization/ResourceMaterializer.hpp" +#include "Optimization/ResourceProgram.hpp" +#include "Recompiler.hpp" +#include "RdnaDecoder/RdnaInstructionDecoder.hpp" + +#if ANYPS5_ENABLE_SPIRV_TOOLS +#include "spirv-tools/libspirv.hpp" +#endif + +#include +#include +#include +#include +#include + +namespace { + +std::string ReadText(const char* path) { + std::ifstream file(path, std::ios::binary); + std::ostringstream stream; + stream << file.rdbuf(); + auto text = stream.str(); + while (!text.empty() && (text.back() == '\n' || text.back() == '\r' || text.back() == ' ')) text.pop_back(); + return text; +} + +// The error text without the serialized request that Recompile appends. +std::string FirstLine(const std::string& text) { + const auto request = text.find("\nRecompileRequest:"); + auto shown = request == std::string::npos ? text : text.substr(0, request); + if (shown.size() > 2000) shown.resize(2000); + return shown; +} + +bool g_disassemble = false; +bool g_assembly = false; +bool g_memory = false; + +bool Replay(const char* path) { + const auto request = ShaderRecompiler::RequestSerializer{}.Deserialize(ReadText(path)); + std::printf("%s: %zu code words, %zu user data, %zu memory regions, wave%u\n", path, request.request.shader.code.size(), request.request.context.userData.size(), request.request.context.memory.size(), request.request.context.waveSize); + if (request.request.context.compute.has_value()) { + const auto& compute = *request.request.context.compute; + std::printf(" compute: threads %ux%ux%u, lds %u dwords, group ids %d%d%d, tg size %d, thread id components %u\n", compute.numThreads[0], compute.numThreads[1], compute.numThreads[2], compute.ldsSizeDwords, compute.groupIdEnable[0], compute.groupIdEnable[1], compute.groupIdEnable[2], compute.tgSizeEnable, compute.threadIdComponentCount); + } + if (g_memory) { + // --mem: the captured inputs. User data words are printed; each memory region is written to + // mem__.bin next to the request for inspection with other tools. + const auto& context = request.request.context; + for (std::size_t i = 0; i < context.userData.size(); ++i) std::printf(" user[%zu] = 0x%08x\n", i, context.userData[i]); + for (const auto& region : context.memory) { + char name[96]; + std::snprintf(name, sizeof(name), "mem_%llx_%llx.bin", static_cast(request.request.shader.codeAddress), static_cast(region.guestAddress)); + if (std::FILE* file = std::fopen(name, "wb")) { + std::fwrite(region.bytes.data(), 1, region.bytes.size(), file); + std::fclose(file); + } + std::printf(" region 0x%llx + 0x%zx -> %s\n", static_cast(region.guestAddress), region.bytes.size(), name); + } + } + if (g_assembly) { + const auto program = ShaderRecompiler::RdnaInstructionDecoder{}.Decode(request.request.shader.code); + // Raw first words carry what the text omits (branch offsets, waitcnt fields). + for (const auto& instruction : program.instructions) std::printf("raw=%08x %s\n", instruction.rawWords[0], ShaderRecompiler::RdnaInstructionToString(instruction).c_str()); + } + try { + auto program = ShaderRecompiler::PrepareResourceProgram(request.request); + constexpr ShaderRecompiler::ResourceMaterializer materializer; + static_cast(materializer.ExtractPlan(program)); + } catch (const std::exception& error) { + std::printf(" resource analysis failed: %s\n", FirstLine(error.what()).c_str()); + return false; + } + try { + const auto result = ShaderRecompiler::Recompile(request.request); + std::printf(" recompiled: %zu SPIR-V words\n", result.spirv.size()); +#if ANYPS5_ENABLE_SPIRV_TOOLS + if (g_disassemble) { + spvtools::SpirvTools tools(SPV_ENV_VULKAN_1_2); + std::string text; + if (tools.Disassemble(result.spirv, &text, SPV_BINARY_TO_TEXT_OPTION_FRIENDLY_NAMES)) std::printf("%s\n", text.c_str()); + } +#else + if (g_disassemble) std::printf(" (--dis needs ANYPS5_ENABLE_SPIRV_TOOLS=ON)\n"); +#endif + return true; + } catch (const std::exception& error) { + std::printf(" recompile failed: %s\n", FirstLine(error.what()).c_str()); + return false; + } +} + +} + +int main(int argc, char** argv) { + if (argc < 2) { + std::fprintf(stderr, "usage: agc_shader_replay [--dis] ...\n the driver writes shader_
.req files when APS5_DUMP_SHADERS is set\n"); + return 2; + } + int failures = 0; + for (int i = 1; i < argc; ++i) { + if (std::string(argv[i]) == "--dis") { + g_disassemble = true; + continue; + } + if (std::string(argv[i]) == "--asm") { + g_assembly = true; + continue; + } + if (std::string(argv[i]) == "--mem") { + g_memory = true; + continue; + } + try { + if (!Replay(argv[i])) ++failures; + } catch (const std::exception& error) { + std::printf("%s: could not load request: %s\n", argv[i], FirstLine(error.what()).c_str()); + ++failures; + } + } + return failures == 0 ? 0 : 1; +}