diff --git a/core/libs/prx/libSceAgcDriver/Graphics/include/Draw.hpp b/core/libs/prx/libSceAgcDriver/Graphics/include/Draw.hpp index 078e05bb2..f0e0fbf30 100644 --- a/core/libs/prx/libSceAgcDriver/Graphics/include/Draw.hpp +++ b/core/libs/prx/libSceAgcDriver/Graphics/include/Draw.hpp @@ -12,6 +12,8 @@ namespace AgcDriver::Graphics { +struct HostImport; + // `recipe`, when given, receives the DrawRecipe a recorded, cacheable, reusable, non-indirect draw // built for its draw-cache entry (design_cpu_final M8; null otherwise, and always under // APS5_NO_DRAW_RECIPE=1). @@ -27,6 +29,7 @@ struct DrawInputCopy { }; DrawInputCopy CopyDrawInput(const Context& context, Recorder* recorder, std::uint64_t address, std::size_t bytes, std::size_t alignment, Recorder::SnapshotUse use); void KeepDrawInput(Recorder* recorder, std::uint64_t address, const DrawInputCopy& copy, Recorder::SnapshotUse use, std::uint32_t derived); +const HostImport* InPlaceDrawInput(const Context& context, std::uint64_t address, std::size_t bytes, std::size_t alignment); std::array MeshIndexBufferDescriptor(const Pm4::DrawParameters& draw); diff --git a/core/libs/prx/libSceAgcDriver/Graphics/include/Recorder.hpp b/core/libs/prx/libSceAgcDriver/Graphics/include/Recorder.hpp index c683c4079..5cebabacc 100644 --- a/core/libs/prx/libSceAgcDriver/Graphics/include/Recorder.hpp +++ b/core/libs/prx/libSceAgcDriver/Graphics/include/Recorder.hpp @@ -124,7 +124,7 @@ public: // on the open batch, no snapshot: the only reader holds the mutex. `kind` names the reader for // the [recorder] hit counters. APS5_COPY_READ_TRACKING=0 notes nothing (ReadTracking() is then // false and the CPU copy falls back to Idle()). - enum class ReadKind : std::uint8_t { DispatchElement = 0, GpuCopy, AddressBased, Indirect, StorageUpload, CopySource, Count }; + enum class ReadKind : std::uint8_t { DispatchElement = 0, GpuCopy, AddressBased, Indirect, StorageUpload, CopySource, DrawInput, Count }; static bool ReadTracking(); void NotePendingRead(std::uint64_t address, std::size_t bytes, ReadKind kind); void NotePendingReads(std::span> ranges, ReadKind kind); diff --git a/core/libs/prx/libSceAgcDriver/Graphics/include/VertexInput.hpp b/core/libs/prx/libSceAgcDriver/Graphics/include/VertexInput.hpp index 5ae9b14a8..4210d5290 100644 --- a/core/libs/prx/libSceAgcDriver/Graphics/include/VertexInput.hpp +++ b/core/libs/prx/libSceAgcDriver/Graphics/include/VertexInput.hpp @@ -181,6 +181,7 @@ struct VertexCopyPlan { std::vector> copies; std::vector copyOf; std::vector offsets; + std::vector alignments; }; inline VertexCopyPlan PlanVertexCopies(std::span fetches) { @@ -204,7 +205,9 @@ inline VertexCopyPlan PlanVertexCopies(std::span fetches) { if (!joins) { lead = i; plan.copies.emplace_back(fetch.begin, fetch.end); + plan.alignments.push_back(fetch.alignment); } + plan.alignments.back() = std::max(plan.alignments.back(), fetch.alignment); auto& copy = plan.copies.back(); copy.second = std::max(copy.second, fetch.end); plan.copyOf[i] = plan.copies.size() - 1; diff --git a/core/libs/prx/libSceAgcDriver/Graphics/src/Draw.cpp b/core/libs/prx/libSceAgcDriver/Graphics/src/Draw.cpp index b18caafad..583921367 100644 --- a/core/libs/prx/libSceAgcDriver/Graphics/src/Draw.cpp +++ b/core/libs/prx/libSceAgcDriver/Graphics/src/Draw.cpp @@ -895,6 +895,21 @@ void KeepDrawInput(Recorder* recorder, std::uint64_t address, const DrawInputCop recorder->KeepDrawSnapshot(address, copy.buffer->Bytes().size(), copy.generation, copy.registryGeneration, copy.buffer, use, derived); } +const HostImport* InPlaceDrawInput(const Context& context, std::uint64_t address, std::size_t bytes, std::size_t alignment) { + auto* recorder = context.recorder; + if (recorder == nullptr || bytes == 0) return nullptr; + GuestMemory::FlushGpuWrites(address, bytes); + if (recorder->PendingWriteOverlaps(address, bytes) || recorder->PendingLabelIn(address, bytes) || recorder->QueuedStoreOverlaps(address, bytes)) return nullptr; + const auto overlaps = [&](const auto& writer) { return writer->WritesOverlap(address, bytes); }; + if (context.copiedWriters != nullptr && std::any_of(context.copiedWriters->begin(), context.copiedWriters->end(), overlaps)) return nullptr; + const auto drawWriters = DrawCopiedWriters(); + if (std::any_of(drawWriters->begin(), drawWriters->end(), overlaps)) return nullptr; + const auto* import = HostImportFor(context, address, bytes); + if (import == nullptr || (address - import->base) % alignment != 0) return nullptr; + recorder->NotePendingRead(address, bytes, Recorder::ReadKind::DrawInput); + return import; +} + namespace { // The draw's inputs before its resources (prepareDrawInputs): the validated parameters, the index @@ -905,11 +920,14 @@ struct DrawInputs { bool nothing = false; std::uint64_t indexBytes = 0; std::shared_ptr indices; + VkBuffer indexHandle = VK_NULL_HANDLE; + VkDeviceSize indexOffset = 0; std::uint32_t maxIndex = 0; VertexInputLayout vertexInput; std::vector> vertexBuffers; std::vector vertexHandles; std::vector vertexOffsets; + std::vector> inPlaceReads; std::set fragmentOutputs; VkPipelineStageFlags shaderStages = 0; std::uint32_t meshGroups = 0; @@ -993,11 +1011,15 @@ DrawInputs prepareDrawInputs(const Context& context, const State& state, const P const bool fanGeometry = state.stages.mesh && state.stages.mesh->inputPrimitive == 5; const bool skipRestart = state.primitiveRestart && (!listTopology || context.primitiveListRestart) && !fanGeometry; const auto use = draw.indexSize == 2 ? (skipRestart ? Recorder::SnapshotUse::Index16Restart : Recorder::SnapshotUse::Index16) : (skipRestart ? Recorder::SnapshotUse::Index32Restart : Recorder::SnapshotUse::Index32); - auto copy = CopyDrawInput(context, context.recorder, draw.indexAddress, static_cast(indexBytes), draw.indexSize, use); + const auto* import = InPlaceDrawInput(context, draw.indexAddress, static_cast(indexBytes), draw.indexSize); + DrawInputCopy copy; + if (import == nullptr) copy = CopyDrawInput(context, context.recorder, draw.indexAddress, static_cast(indexBytes), draw.indexSize, use); + else GuestMemory::CheckRange(reinterpret_cast(draw.indexAddress), static_cast(indexBytes), draw.indexSize); std::optional highest; if (!copy.reused) { - highest = HighestDrawIndex(copy.buffer->Bytes().first(static_cast(indexBytes)), draw.indexSize, skipRestart); - KeepDrawInput(context.recorder, draw.indexAddress, copy, use, skipRestart ? (highest ? *highest + 1u : 0u) : highest.value_or(0u)); + const auto bytes = import != nullptr ? std::span(reinterpret_cast(draw.indexAddress), static_cast(indexBytes)) : copy.buffer->Bytes().first(static_cast(indexBytes)); + highest = HighestDrawIndex(bytes, draw.indexSize, skipRestart); + if (import == nullptr) KeepDrawInput(context.recorder, draw.indexAddress, copy, use, skipRestart ? (highest ? *highest + 1u : 0u) : highest.value_or(0u)); } else if (!skipRestart) { highest = copy.derived; } else if (copy.derived != 0) { @@ -1010,7 +1032,14 @@ DrawInputs prepareDrawInputs(const Context& context, const State& state, const P Require(*highest <= context.limits.maxDrawIndexedIndexValue, "index exceeds the device's indexed draw limit"); Require(!fanGeometry || !state.primitiveRestart || *highest != (draw.indexSize == 2 ? 0xffffu : 0xffffffffu), "primitive restart in a triangle fan geometry draw is unsupported"); inputs.maxIndex = *highest; - inputs.indices = std::move(copy.buffer); + if (import != nullptr) { + inputs.indexHandle = import->buffer; + inputs.indexOffset = draw.indexAddress - import->base; + inputs.inPlaceReads.emplace_back(draw.indexAddress, draw.indexAddress + indexBytes); + } else { + inputs.indices = std::move(copy.buffer); + inputs.indexHandle = inputs.indices->Handle(); + } } APS5_LOG_CHARS_OUT_DEBUG("Index validation OK"); const auto& attributes = shaders.front().program->vertexAttributes; @@ -1042,11 +1071,24 @@ DrawInputs prepareDrawInputs(const Context& context, const State& state, const P fetches.push_back({address, address + bytes, (fields[1] >> 16u) & 0x3fffu, attribute.fetchIndex, DecodeVertexFormat(attribute).alignment}); } const auto plan = PlanVertexCopies(fetches); - for (const auto& [begin, end] : plan.copies) { + std::vector rangeHandles; + std::vector rangeOffsets; + rangeHandles.reserve(plan.copies.size()); + rangeOffsets.reserve(plan.copies.size()); + for (std::size_t range = 0; range < plan.copies.size(); ++range) { + const auto [begin, end] = plan.copies[range]; const auto bytes = static_cast(end - begin); GuestMemory::CheckRange(reinterpret_cast(begin), bytes, 1); + if (const auto* import = InPlaceDrawInput(context, begin, bytes, plan.alignments[range])) { + rangeHandles.push_back(import->buffer); + rangeOffsets.push_back(begin - import->base); + inputs.inPlaceReads.emplace_back(begin, end); + continue; + } auto copy = CopyDrawInput(context, context.recorder, begin, bytes, 1, Recorder::SnapshotUse::Vertex); KeepDrawInput(context.recorder, begin, copy, Recorder::SnapshotUse::Vertex, 0); + rangeHandles.push_back(copy.buffer->Handle()); + rangeOffsets.push_back(0); inputs.vertexBuffers.push_back(std::move(copy.buffer)); } for (std::size_t i = 0; i < attributes.size(); ++i) { @@ -1056,8 +1098,8 @@ DrawInputs prepareDrawInputs(const Context& context, const State& state, const P inputs.vertexBuffers.push_back(std::move(zeroed[i])); continue; } - inputs.vertexHandles.push_back(inputs.vertexBuffers[plan.copyOf[fetchOf[i]]]->Handle()); - inputs.vertexOffsets[i] = plan.offsets[fetchOf[i]]; + inputs.vertexHandles.push_back(rangeHandles[plan.copyOf[fetchOf[i]]]); + inputs.vertexOffsets[i] = rangeOffsets[plan.copyOf[fetchOf[i]]] + plan.offsets[fetchOf[i]]; } timer.phase(PhaseVertex); return inputs; @@ -1184,7 +1226,7 @@ void recordDrawCommands(const Context& context, VkCommandBuffer commands, const return; } if (!inputs.vertexHandles.empty()) context.Resolved(&DeviceFunctions::cmdBindVertexBuffers, "vkCmdBindVertexBuffers")(commands, 0, static_cast(inputs.vertexHandles.size()), inputs.vertexHandles.data(), inputs.vertexOffsets.data()); - if (draw.indexed) context.Resolved(&DeviceFunctions::cmdBindIndexBuffer, "vkCmdBindIndexBuffer")(commands, inputs.indices->Handle(), 0, draw.indexSize == 2 ? VK_INDEX_TYPE_UINT16 : VK_INDEX_TYPE_UINT32); + if (draw.indexed) context.Resolved(&DeviceFunctions::cmdBindIndexBuffer, "vkCmdBindIndexBuffer")(commands, inputs.indexHandle, inputs.indexOffset, draw.indexSize == 2 ? VK_INDEX_TYPE_UINT16 : VK_INDEX_TYPE_UINT32); if (args == nullptr) { if (draw.indexed) context.Resolved(&DeviceFunctions::cmdDrawIndexed, "vkCmdDrawIndexed")(commands, draw.indexCount, draw.instanceCount, 0, static_cast(draw.firstVertex), draw.firstInstance); else context.Resolved(&DeviceFunctions::cmdDraw, "vkCmdDraw")(commands, draw.indexCount, draw.instanceCount, draw.firstVertex, draw.firstInstance); @@ -1530,6 +1572,7 @@ void recordDraw(const Context& context, const State& state, const Pm4::DrawParam const auto drawTiming = !continued ? recorder->BeginGpuTiming(CommandClass::Draw) : Recorder::NoTiming; if (!continued && Recorder::BarrierValidate()) { auto reads = resources.InPlaceReads(); + reads.insert(reads.end(), inputs.inPlaceReads.begin(), inputs.inPlaceReads.end()); if (gpuIndirect) { reads.emplace_back(args->arguments, args->arguments + args->RangeBytes()); if (args->countIndirect) reads.emplace_back(args->countAddress, args->countAddress + 4); @@ -1983,6 +2026,7 @@ void Draw(const Context& context, const State& state, const Pm4::DrawParameters& const auto drawTiming = recorded ? recorder->BeginGpuTiming(CommandClass::Draw) : Recorder::NoTiming; if (recorded && Recorder::BarrierValidate()) { auto reads = resources->InPlaceReads(); + reads.insert(reads.end(), inputs.inPlaceReads.begin(), inputs.inPlaceReads.end()); if (gpuIndirect) { reads.emplace_back(args->arguments, args->arguments + args->RangeBytes()); if (args->countIndirect) reads.emplace_back(args->countAddress, args->countAddress + 4); diff --git a/core/libs/prx/libSceAgcDriver/Graphics/src/GuestBufferMemory.cpp b/core/libs/prx/libSceAgcDriver/Graphics/src/GuestBufferMemory.cpp index 7e50ddf21..f61c2978c 100644 --- a/core/libs/prx/libSceAgcDriver/Graphics/src/GuestBufferMemory.cpp +++ b/core/libs/prx/libSceAgcDriver/Graphics/src/GuestBufferMemory.cpp @@ -208,7 +208,7 @@ const char* bindImport(const Context& context, HostImport& entry, VkExternalMemo VkBufferCreateInfo info{VK_STRUCTURE_TYPE_BUFFER_CREATE_INFO, &external}; info.size = entry.bytes; // INDIRECT_BUFFER: DISPATCH_INDIRECT group counts are read in place (VulkanDevice::DispatchIndirect). - info.usage = VK_BUFFER_USAGE_STORAGE_BUFFER_BIT | VK_BUFFER_USAGE_SHADER_DEVICE_ADDRESS_BIT | VK_BUFFER_USAGE_TRANSFER_SRC_BIT | VK_BUFFER_USAGE_TRANSFER_DST_BIT | VK_BUFFER_USAGE_INDIRECT_BUFFER_BIT; + info.usage = VK_BUFFER_USAGE_STORAGE_BUFFER_BIT | VK_BUFFER_USAGE_SHADER_DEVICE_ADDRESS_BIT | VK_BUFFER_USAGE_TRANSFER_SRC_BIT | VK_BUFFER_USAGE_TRANSFER_DST_BIT | VK_BUFFER_USAGE_INDIRECT_BUFFER_BIT | VK_BUFFER_USAGE_VERTEX_BUFFER_BIT | VK_BUFFER_USAGE_INDEX_BUFFER_BIT; info.sharingMode = VK_SHARING_MODE_EXCLUSIVE; if (const auto result = context.Function("vkCreateBuffer")(context.device, &info, nullptr, &entry.buffer); result != VK_SUCCESS) return failed("vkCreateBuffer", result); VkMemoryRequirements requirements{}; diff --git a/core/libs/prx/libSceAgcDriver/Graphics/src/Recorder.cpp b/core/libs/prx/libSceAgcDriver/Graphics/src/Recorder.cpp index 5810d65a9..687a5137d 100644 --- a/core/libs/prx/libSceAgcDriver/Graphics/src/Recorder.cpp +++ b/core/libs/prx/libSceAgcDriver/Graphics/src/Recorder.cpp @@ -3496,7 +3496,7 @@ void Recorder::finish(std::unique_ptr batch, bool wait, int source, bool AgcDriver::ProfilePrint_nid_no_patch("[recorder] completion label stores: %llu run, %llu skipped (no CPU write-back overlapped them); %llu counted pending at a write-back; %llu write-backs over a tracked label; %llu write-back completions pending\n", static_cast(completionStoresRun.load()), static_cast(completionStoresSkipped.load()), static_cast(completionLabelsCountedLate.load()), static_cast(writeBacksOverLabels.load()), static_cast(writeBackCompletions.load())); AgcDriver::ProfilePrint_nid_no_patch("[recorder] submits %llu, vkQueueSubmit mean %.1f us, max %.1f us\n", static_cast(submitCount), submitCount != 0 ? submitUs / static_cast(submitCount) : 0.0, submitMaxUs); const auto reads = Recorder::ReadCounts(); - AgcDriver::ProfilePrint_nid_no_patch("[recorder] in-place reads: %llu noted, %llu queries, hits by reader: dispatch element %llu, gpu copy %llu, address-based %llu, indirect %llu, storage upload %llu, copy source %llu; %llu hits on signaled batches ignored\n", static_cast(reads.noted), static_cast(reads.queries), static_cast(reads.hits[0]), static_cast(reads.hits[1]), static_cast(reads.hits[2]), static_cast(reads.hits[3]), static_cast(reads.hits[4]), static_cast(reads.hits[5]), static_cast(reads.staleIgnored)); + AgcDriver::ProfilePrint_nid_no_patch("[recorder] in-place reads: %llu noted, %llu queries, hits by reader: dispatch element %llu, gpu copy %llu, address-based %llu, indirect %llu, storage upload %llu, copy source %llu, draw input %llu; %llu hits on signaled batches ignored\n", static_cast(reads.noted), static_cast(reads.queries), static_cast(reads.hits[0]), static_cast(reads.hits[1]), static_cast(reads.hits[2]), static_cast(reads.hits[3]), static_cast(reads.hits[4]), static_cast(reads.hits[5]), static_cast(reads.hits[6]), static_cast(reads.staleIgnored)); } } } report{profile, source, waitStart, *this, signaledAtStart}; diff --git a/core/libs/prx/libSceAgcDriver/tests/Graphics.cpp b/core/libs/prx/libSceAgcDriver/tests/Graphics.cpp index 5a2842982..32474afe3 100644 --- a/core/libs/prx/libSceAgcDriver/tests/Graphics.cpp +++ b/core/libs/prx/libSceAgcDriver/tests/Graphics.cpp @@ -3317,6 +3317,12 @@ void vertexCopyTests() { const auto plan = PlanVertexCopies(fetches); Require(plan.copies.size() == 1 && plan.copies[0].first == 0x1000 && plan.copies[0].second == 0x1018 + 32 * 9 + 8, "interleaved attributes were not copied as one union"); Require(plan.copyOf == std::vector{0, 0, 0} && plan.offsets == std::vector{0x18, 0, 0xc}, "interleaved attribute offsets are wrong"); + Require(plan.alignments == std::vector{4}, "the union's alignment is not its attributes'"); + } + { + const std::array fetches{{{0x4000, 0x4100, 16, 0, 2}, {0x4004, 0x4104, 16, 0, 4}}}; + const auto plan = PlanVertexCopies(fetches); + Require(plan.copies.size() == 1 && plan.alignments == std::vector{4}, "a union did not take its strictest attribute alignment"); } { const std::array fetches{{ diff --git a/core/libs/prx/libSceAgcDriver/tests/Recorder.cpp b/core/libs/prx/libSceAgcDriver/tests/Recorder.cpp index abf73b6af..145e901bc 100644 --- a/core/libs/prx/libSceAgcDriver/tests/Recorder.cpp +++ b/core/libs/prx/libSceAgcDriver/tests/Recorder.cpp @@ -1490,6 +1490,63 @@ void drawInputReuseTests(const Device& device, Recorder& recorder) { recorder.Sync(); } +void drawInputInPlaceTests(const Device& device, Recorder& recorder) { + auto context = device.GetContext(); + constexpr std::size_t bytes = 65536; + void* block = context.hostImportAlignment != 0 ? AllocateWatched(bytes, 65536) : nullptr; + if (block == nullptr) { + std::cout << "host imports or write watching unavailable: in-place draw inputs not tested\n"; + return; + } + struct Release { + void* block; + ~Release() { ReleaseWatched(block, bytes); } + } release{block}; + const auto address = reinterpret_cast(block); + { + GuestAllocations::Mutation mutation; + mutation.Add(block, bytes, true, true); + } + struct Unregister { + const Context& context; + void* block; + ~Unregister() { + { + GuestAllocations::Mutation mutation; + mutation.Remove(block); + } + HostImportFor(context, reinterpret_cast(block), bytes); + } + } unregister{context, block}; + if (HostImportFor(context, address, bytes) == nullptr) { + std::cout << "host import of the watched block refused: in-place draw inputs not tested\n"; + return; + } + context.recorder = &recorder; + recorder.Sync(); + const auto vertices = address + 4096; + const auto* import = InPlaceDrawInput(context, vertices, 256, 4); + Require(import != nullptr && import->base <= vertices && vertices + 256 <= import->base + import->bytes, "an imported draw input was not bound in place"); + const auto read = recorder.DescribePendingRead(vertices, 256); + Require(read.has_value() && read->kind == Recorder::ReadKind::DrawInput && read->open, "an in-place draw input's read was not noted on the open batch"); + Require(InPlaceDrawInput(context, vertices + 2, 256, 4) == nullptr, "a draw input misaligned for its format was bound in place"); + Require(InPlaceDrawInput(context, vertices + 2, 256, 2) != nullptr, "an aligned 16-bit draw input was not bound in place"); + recorder.NotePendingWrite(address + 8192, 16); + Require(InPlaceDrawInput(context, address + 8192, 256, 4) != nullptr && !recorder.PendingWriteOverlaps(address + 8192, 256), "a draw input under a pending GPU write was bound before the write completed"); + const std::array label{std::byte{1}, std::byte{2}, std::byte{3}, std::byte{4}}; + recorder.RecordStore(import->buffer, address + 12288 - import->base, label, address + 12288); + Require(recorder.QueuedStoreOverlaps(address + 12288, 4), "the label store was not queued"); + Require(InPlaceDrawInput(context, address + 12288, 256, 4) == nullptr, "a draw input under a queued label store was bound in place"); + Require(InPlaceDrawInput(context, address + 16384, 256, 4) != nullptr, "a queued label store refused a disjoint draw input"); + alignas(64) static std::array unregistered{}; + Require(InPlaceDrawInput(context, reinterpret_cast(unregistered.data()), unregistered.size(), 4) == nullptr, "memory without a host import was bound in place"); + auto unrecorded = context; + unrecorded.recorder = nullptr; + Require(InPlaceDrawInput(unrecorded, vertices, 256, 4) == nullptr, "a draw without a recorder bound its input in place"); + recorder.Sync(); + Require(!recorder.PendingReadOverlaps(vertices, 256), "an in-place draw input's read outlived its batch"); +} + // Unit shadows (UnitShadow.hpp) over a host import of write-watched arena memory: a retile piece's // slab destination and its seeds, freshness from the tracker (a publish never stamps, a CPU write // makes the unit stale), the scopes, the slab boundary, and the retire publish. With @@ -3447,6 +3504,7 @@ int main(int argc, char** argv) { drawSnapshotReuseTests(device, recorder); drawSnapshotEvictionTests(device); drawInputReuseTests(device, recorder); + drawInputInPlaceTests(device, recorder); RunResidentPresentTests(device.GetContext()); storeRunTests(device, recorder); queuedWriteGroupTests();