From 25738ff67f94b78cebebb62fc5f6eb0c46ea5de4 Mon Sep 17 00:00:00 2001 From: Dean Galvin Date: Tue, 6 Oct 2026 07:07:05 +1300 Subject: [PATCH] perf(agc): move storage texels between the image and its tiled bytes in one pass A storage write-back copied the image into a linear buffer and retiled that buffer; an upload detiled into a linear buffer and copied it into the image. With storage image access without a format, the retile now reads each texel from the image through an unsigned integer view of the element size, and the detile writes it there, so the linear buffer, its copy and one barrier per move go away. The texels move bit for bit as before, through the same tiling equations and windows. This covers the block-unit windows of write-backs and uploads and the whole-surface direct upload, for 2D, array and cube images. agc_driver_recorder_tests singlePassTests: a first upload, a write-back of new texels in every mip and a second upload over the stored memory leave the images and guest memory byte for byte equal to the copies through a linear buffer, in plain and write-watched memory. --- core/libs/prx/libSceAgcDriver/CMakeLists.txt | 13 ++ .../Execution/src/VulkanDevice.cpp | 3 + .../Graphics/include/Context.hpp | 1 + .../Graphics/include/Texture.hpp | 7 + .../Graphics/include/TextureDetiler.hpp | 12 +- .../Graphics/shaders/TextureDetile.comp | 62 ++++++ .../libSceAgcDriver/Graphics/src/Texture.cpp | 180 +++++++++++++--- .../Graphics/src/TextureDetiler.cpp | 112 +++++++++- .../src/TextureDetilerDescriptors.cpp | 31 ++- .../prx/libSceAgcDriver/tests/Recorder.cpp | 200 ++++++++++++++++++ 10 files changed, 570 insertions(+), 51 deletions(-) diff --git a/core/libs/prx/libSceAgcDriver/CMakeLists.txt b/core/libs/prx/libSceAgcDriver/CMakeLists.txt index 583e96a21..a62c4cae1 100644 --- a/core/libs/prx/libSceAgcDriver/CMakeLists.txt +++ b/core/libs/prx/libSceAgcDriver/CMakeLists.txt @@ -45,6 +45,17 @@ add_custom_command( DEPENDS "${agcTextureDetileShaderSource}" glslang-standalone VERBATIM ) +set(agcTextureDetileImageShaderSpv ${agcTextureDetileGeneratedDir}/TextureDetileImage.spv) +set(agcTextureDetileImageShaderHeader ${agcTextureDetileGeneratedDir}/TextureDetileImage_spv.h) +add_custom_command( + OUTPUT "${agcTextureDetileImageShaderHeader}" + COMMAND ${CMAKE_COMMAND} -E make_directory "${agcTextureDetileGeneratedDir}" + COMMAND "$" -V --target-env vulkan1.1 -DLINEAR_IMAGE=1 -o "${agcTextureDetileImageShaderSpv}" "${agcTextureDetileShaderSource}" + COMMAND ${CMAKE_COMMAND} -DINPUT=${agcTextureDetileImageShaderSpv} -DOUTPUT=${agcTextureDetileImageShaderHeader} + -DSYMBOL=TEXTURE_DETILE_IMAGE_SPV -P "${CMAKE_CURRENT_SOURCE_DIR}/embed_spirv.cmake" + DEPENDS "${agcTextureDetileShaderSource}" glslang-standalone + VERBATIM +) set(agcMeshArgumentsShaderSource ${CMAKE_CURRENT_SOURCE_DIR}/Graphics/shaders/MeshArguments.comp) set(agcMeshArgumentsShaderSpv ${agcTextureDetileGeneratedDir}/MeshArguments.spv) @@ -188,6 +199,7 @@ set(agcDriverSources ${agcTextureDetileShaderHeader} ${agcCmaskClearShaderHeader} ${agcMeshArgumentsShaderHeader} + ${agcTextureDetileImageShaderHeader} ${agcSampleCounterShaderHeader} ${agcSampleDumpsShaderHeader} ${CMAKE_CURRENT_SOURCE_DIR}/Eq/src/Event.cpp @@ -756,6 +768,7 @@ add_test_executable(agc_driver_graphics_tests EXCLUDE_FROM_ALL ${agcTextureDetileShaderHeader} ${agcCmaskClearShaderHeader} ${agcMeshArgumentsShaderHeader} + ${agcTextureDetileImageShaderHeader} ${agcSampleCounterShaderHeader} ${agcSampleDumpsShaderHeader} Execution/src/GuestMemory.cpp diff --git a/core/libs/prx/libSceAgcDriver/Execution/src/VulkanDevice.cpp b/core/libs/prx/libSceAgcDriver/Execution/src/VulkanDevice.cpp index 7cfd5f2b5..11f83cfa5 100644 --- a/core/libs/prx/libSceAgcDriver/Execution/src/VulkanDevice.cpp +++ b/core/libs/prx/libSceAgcDriver/Execution/src/VulkanDevice.cpp @@ -227,6 +227,7 @@ struct VulkanDevice::State { bool pipelineExecutableInfo = false; bool maintenance8 = false; std::uint32_t srgbDecodeFormats = 0; + bool storageImageWithoutFormat = false; bool depthClamp = false; bool depthBounds = false; bool depthBiasClamp = false; @@ -1117,6 +1118,7 @@ VulkanDevice::VulkanDevice(const PresentationWindow* window) : state(std::make_u enabled.shaderClipDistance = available.shaderClipDistance; if (enabled.shaderStorageImageWriteWithoutFormat) state->capabilities.push_back(spv::CapabilityStorageImageWriteWithoutFormat); if (enabled.shaderStorageImageReadWithoutFormat) state->capabilities.push_back(spv::CapabilityStorageImageReadWithoutFormat); + state->storageImageWithoutFormat = enabled.shaderStorageImageWriteWithoutFormat == VK_TRUE && enabled.shaderStorageImageReadWithoutFormat == VK_TRUE; // Bindless image tables index an image array with a wave-uniform runtime slot. enabled.shaderSampledImageArrayDynamicIndexing = available.shaderSampledImageArrayDynamicIndexing; enabled.shaderStorageImageArrayDynamicIndexing = available.shaderStorageImageArrayDynamicIndexing; @@ -2729,6 +2731,7 @@ Graphics::Context VulkanDevice::buildContext() const { context.pipelineExecutableInfo = state->pipelineExecutableInfo; context.srgbDecodeFormats = state->srgbDecodeFormats; context.memoryProperties2 = state->memoryBudget ? state->InstanceFunction("vkGetPhysicalDeviceMemoryProperties2") : nullptr; + context.singlePassStorage = state->storageImageWithoutFormat; return context; } diff --git a/core/libs/prx/libSceAgcDriver/Graphics/include/Context.hpp b/core/libs/prx/libSceAgcDriver/Graphics/include/Context.hpp index b74efb979..a948e08fb 100644 --- a/core/libs/prx/libSceAgcDriver/Graphics/include/Context.hpp +++ b/core/libs/prx/libSceAgcDriver/Graphics/include/Context.hpp @@ -165,6 +165,7 @@ struct Context { std::uint32_t srgbDecodeFormats = 0; bool provokingVertexLast = false; bool provokingVertexModePerPipeline = false; + bool singlePassStorage = false; template TFunction Function(const char* name) const { diff --git a/core/libs/prx/libSceAgcDriver/Graphics/include/Texture.hpp b/core/libs/prx/libSceAgcDriver/Graphics/include/Texture.hpp index 2fa7b67e5..a1f98258b 100644 --- a/core/libs/prx/libSceAgcDriver/Graphics/include/Texture.hpp +++ b/core/libs/prx/libSceAgcDriver/Graphics/include/Texture.hpp @@ -295,6 +295,7 @@ public: // was still current (nothing uploaded). // Keeps the image current with guest memory (see GuestMemory::CollectWrites). bool Refresh(); + static std::uint64_t SinglePassMoves(); std::uint64_t GuestBytes() const; VkDeviceSize AllocationBytes() const { return memoryBytes; } @@ -430,6 +431,10 @@ private: bool keysFillMatches(std::uint64_t address, std::size_t bytes, bool overlapped) const; bool pendingUnitInside(std::uint64_t address, std::size_t bytes) const; VkImageView createView(std::uint32_t mip, bool firstLayer, VkFormat format) const; + bool singlePass(); + VkImageView elementLayerView(std::uint32_t level, std::uint32_t layer); + void recordDirectUploadBarrier(VkCommandBuffer commands, bool discard); + void recordDirectUploadDone(VkCommandBuffer commands); void release() noexcept; Context context; @@ -483,6 +488,8 @@ private: std::map, VkImageView> atomicViews; std::map, VkImageView> uintViews; VkImageView elementView = VK_NULL_HANDLE; + std::map elementLayerViews; + std::int8_t singlePassState = -1; bool attachable = false; std::map, VkImageView> attachmentViews; VkImage proxyImage = VK_NULL_HANDLE; diff --git a/core/libs/prx/libSceAgcDriver/Graphics/include/TextureDetiler.hpp b/core/libs/prx/libSceAgcDriver/Graphics/include/TextureDetiler.hpp index d5af352ab..2efc3bc46 100644 --- a/core/libs/prx/libSceAgcDriver/Graphics/include/TextureDetiler.hpp +++ b/core/libs/prx/libSceAgcDriver/Graphics/include/TextureDetiler.hpp @@ -48,15 +48,18 @@ namespace AgcDriver::Graphics { // restricts the move to part of the mip (a partial upload or write-back, StorageTexture), // leaving the other elements of both buffers alone. void Dispatch(VkCommandBuffer commands, TextureTileMode tileMode, std::uint32_t elementBytes, VkBuffer source, std::uint64_t sourceOffset, VkBuffer destination, std::uint64_t destinationOffset, const TileMipLayout& layout, bool retile = false, std::uint32_t slice = 0, bool thick = false, const DetileWindow& window = {}); + void DispatchImage(VkCommandBuffer commands, TextureTileMode tileMode, std::uint32_t elementBytes, VkBuffer tiled, std::uint64_t tiledOffset, VkImageView view, const TileMipLayout& layout, bool retile = false, std::uint32_t slice = 0, const DetileWindow& window = {}); + static VkFormat ImageElementFormat(std::uint32_t elementBytes); // Recycles the descriptor sets of the previous batch; call before recording a new command batch. void BeginBatch(); void DispatchCmaskClear(VkCommandBuffer commands, VkBuffer cmask, std::uint64_t cmaskOffset, std::size_t cmaskBytes, VkImageView view, std::uint32_t width, std::uint32_t height, std::uint32_t elementBytes, const std::array& clearWords, bool write); std::uint32_t CmaskErrors(); private: - VkPipeline pipeline(TextureTileMode tileMode, std::uint32_t elementBytes, bool retile, bool thick); + VkPipeline pipeline(TextureTileMode tileMode, std::uint32_t elementBytes, bool retile, bool thick, bool image = false); void release() noexcept; - VkDescriptorSet allocateSet(); + VkDescriptorSet allocateSet(bool image = false); + void checkWindow(const TileMipLayout& layout, const DetileWindow& window, std::uint32_t& columnEnd, std::uint32_t& rowEnd) const; const Context context; VkDescriptorSetLayout descriptorLayout = VK_NULL_HANDLE; @@ -71,6 +74,11 @@ namespace AgcDriver::Graphics { std::vector cmaskDescriptorPools; std::size_t allocatedCmaskSets = 0; std::unique_ptr cmaskErrors; + VkDescriptorSetLayout imageDescriptorLayout = VK_NULL_HANDLE; + VkPipelineLayout imagePipelineLayout = VK_NULL_HANDLE; + VkShaderModule imageModule = VK_NULL_HANDLE; + std::vector imageDescriptorPools; + std::size_t allocatedImageSets = 0; }; } diff --git a/core/libs/prx/libSceAgcDriver/Graphics/shaders/TextureDetile.comp b/core/libs/prx/libSceAgcDriver/Graphics/shaders/TextureDetile.comp index 575edeb25..fcff04497 100644 --- a/core/libs/prx/libSceAgcDriver/Graphics/shaders/TextureDetile.comp +++ b/core/libs/prx/libSceAgcDriver/Graphics/shaders/TextureDetile.comp @@ -1,4 +1,7 @@ #version 450 +#ifdef LINEAR_IMAGE +#extension GL_EXT_shader_image_load_formatted : require +#endif layout(constant_id = 0) const uint ELEMENT_BYTES = 4u; layout(constant_id = 1) const uint BLOCK_BYTES = 256u; @@ -29,8 +32,13 @@ layout(constant_id = 21) const uint BLOCK_HEIGHT = 0u; layout(local_size_x = 8, local_size_y = 8) in; +#ifdef LINEAR_IMAGE +layout(set = 0, binding = 0, std430) buffer Tiled { uint data[]; } tiledBuffer; +layout(set = 0, binding = 1) uniform uimage2D linearImage; +#else layout(set = 0, binding = 0, std430) readonly buffer Input { uint data[]; } inputBuffer; layout(set = 0, binding = 1, std430) buffer Output { uint data[]; } outputBuffer; +#endif layout(push_constant) uniform Push { uint srcBase; @@ -142,6 +150,43 @@ uvec2 blockExtent() { return uvec2(width, BLOCK_BYTES / (width * ELEMENT_BYTES)); } +#ifdef LINEAR_IMAGE +void storeElement(uint src, uvec2 p) { + uvec4 value = uvec4(0u); + uint word = src >> 2; + if (ELEMENT_BYTES == 1u) { + value.x = (tiledBuffer.data[word] >> ((src & 3u) * 8u)) & 0xffu; + } else if (ELEMENT_BYTES == 2u) { + value.x = (tiledBuffer.data[word] >> ((src & 3u) * 8u)) & 0xffffu; + } else { + value.x = tiledBuffer.data[word]; + if (ELEMENT_BYTES >= 8u) value.y = tiledBuffer.data[word + 1u]; + if (ELEMENT_BYTES >= 16u) { + value.z = tiledBuffer.data[word + 2u]; + value.w = tiledBuffer.data[word + 3u]; + } + } + imageStore(linearImage, ivec2(p), value); +} + +void loadElement(uvec2 p, uint dst) { + uvec4 value = imageLoad(linearImage, ivec2(p)); + uint word = dst >> 2; + if (ELEMENT_BYTES >= 4u) { + tiledBuffer.data[word] = value.x; + if (ELEMENT_BYTES >= 8u) tiledBuffer.data[word + 1u] = value.y; + if (ELEMENT_BYTES >= 16u) { + tiledBuffer.data[word + 2u] = value.z; + tiledBuffer.data[word + 3u] = value.w; + } + } else { + uint mask = ELEMENT_BYTES == 1u ? 0xffu : 0xffffu; + uint shift = (dst & 3u) * 8u; + atomicAnd(tiledBuffer.data[word], ~(mask << shift)); + atomicOr(tiledBuffer.data[word], (value.x & mask) << shift); + } +} +#else void copyElement(uint src, uint dst) { if (ELEMENT_BYTES >= 4u) { for (uint i = 0u; i < ELEMENT_BYTES; i += 4u) { @@ -155,6 +200,7 @@ void copyElement(uint src, uint dst) { atomicOr(outputBuffer.data[dst >> 2], value << shift); } } +#endif void main() { uvec2 p = gl_GlobalInvocationID.xy + uvec2(params.columnBegin, params.rowBegin); @@ -170,11 +216,19 @@ void main() { if (linearOffset < params.rangeBegin || linearOffset >= params.rangeEnd) { return; } +#ifdef LINEAR_IMAGE + if (RETILE != 0u) { + loadElement(p, params.dstBase + linearOffset - params.tiledBase); + } else { + storeElement(params.srcBase + linearOffset - params.tiledBase, p); + } +#else if (RETILE != 0u) { copyElement(params.srcBase + linearOffset - params.linearBase, params.dstBase + linearOffset - params.tiledBase); } else { copyElement(params.srcBase + linearOffset - params.tiledBase, params.dstBase + linearOffset - params.linearBase); } +#endif return; } @@ -194,10 +248,18 @@ void main() { tiledOffset -= params.tiledBase; linearOffset -= params.linearBase; +#ifdef LINEAR_IMAGE + if (RETILE != 0u) { + loadElement(p, params.dstBase + tiledOffset); + } else { + storeElement(params.srcBase + tiledOffset, p); + } +#else if (RETILE != 0u) { copyElement(params.srcBase + linearOffset, params.dstBase + tiledOffset); } else { copyElement(params.srcBase + tiledOffset, params.dstBase + linearOffset); } +#endif } diff --git a/core/libs/prx/libSceAgcDriver/Graphics/src/Texture.cpp b/core/libs/prx/libSceAgcDriver/Graphics/src/Texture.cpp index 71720b8b5..e538a7bd2 100644 --- a/core/libs/prx/libSceAgcDriver/Graphics/src/Texture.cpp +++ b/core/libs/prx/libSceAgcDriver/Graphics/src/Texture.cpp @@ -77,6 +77,7 @@ thread_local const char* uploadReason = "first"; // selections widened to every pending unit (the hysteresis in writeBack), units dropped as dead // (DiscardPendingInside) and units an alias's own results superseded (markLayersPending); relaxed, // reported only. +std::atomic singlePassMoves{0}; std::atomic partialUploads{0}, partialUploadBytes{0}, partialWriteBacks{0}, partialWriteBackBytes{0}, emptyWriteBacks{0}, coalescedWriteBacks{0}, unitsDropped{0}, unitsSuperseded{0}; // FlushPending listings skipped because no pending unit of the image lay inside the range, images // whose selected pending units were dropped because their memory is no longer registered (see @@ -1107,6 +1108,39 @@ VkImageView StorageTexture::createView(std::uint32_t mip, bool firstLayer, VkFor return created; } +bool StorageTexture::singlePass() { + if (singlePassState < 0) { + const auto elementBytes = BytesPerElement(descriptor.format); + const auto format = TextureDetiler::ImageElementFormat(elementBytes); + bool eligible = context.singlePassStorage && format != VK_FORMAT_UNDEFINED && BlockWidth(descriptor.format) == 1 && !geometry.thick && geometry.imageDepth == 1 && (descriptor.dimension == TextureDimension::k2D || descriptor.dimension == TextureDimension::k2DArray || descriptor.dimension == TextureDimension::kCube); + if (eligible) { + VkFormatProperties properties{}; + context.formatProperties(context.physical, format, &properties); + eligible = (properties.optimalTilingFeatures & VK_FORMAT_FEATURE_STORAGE_IMAGE_BIT) != 0; + } + singlePassState = eligible ? 1 : 0; + } + return singlePassState == 1; +} + +VkImageView StorageTexture::elementLayerView(std::uint32_t level, std::uint32_t layer) { + const auto key = (level << 16) | layer; + if (const auto found = elementLayerViews.find(key); found != elementLayerViews.end()) return found->second; + VkImageViewUsageCreateInfo usage{VK_STRUCTURE_TYPE_IMAGE_VIEW_USAGE_CREATE_INFO}; + usage.usage = VK_IMAGE_USAGE_STORAGE_BIT; + VkImageViewCreateInfo viewInfo{VK_STRUCTURE_TYPE_IMAGE_VIEW_CREATE_INFO}; + viewInfo.pNext = &usage; + viewInfo.image = image; + viewInfo.viewType = VK_IMAGE_VIEW_TYPE_2D; + viewInfo.format = TextureDetiler::ImageElementFormat(BytesPerElement(descriptor.format)); + viewInfo.components = {VK_COMPONENT_SWIZZLE_IDENTITY, VK_COMPONENT_SWIZZLE_IDENTITY, VK_COMPONENT_SWIZZLE_IDENTITY, VK_COMPONENT_SWIZZLE_IDENTITY}; + viewInfo.subresourceRange = {VK_IMAGE_ASPECT_COLOR_BIT, level, 1u, geometry.CopyLayer(layer), 1u}; + VkImageView created = VK_NULL_HANDLE; + Check(context.Function("vkCreateImageView")(context.device, &viewInfo, nullptr, &created), "vkCreateImageView storage element"); + elementLayerViews.emplace(key, created); + return created; +} + VkImageView StorageTexture::AttachmentView(VkFormat format, std::uint32_t mip, std::uint32_t depthSlice) { Require(attachable, "storage image cannot be a color attachment"); Require(mip < descriptor.mipCount, "attachment mip exceeds the storage image"); @@ -1566,6 +1600,10 @@ DccKeys StorageTexture::ProvedKeys() const { return *keys; } +std::uint64_t StorageTexture::SinglePassMoves() { + return singlePassMoves.load(std::memory_order_relaxed); +} + bool StorageTexture::ServesKeysAt(std::uint64_t dccAddress) const { const bool locked = GuestMemory::GpuMutex().HeldByThisThread(); const auto readFollowed = [&] { @@ -1771,7 +1809,8 @@ void StorageTexture::upload(const std::vector* layers) { if ((*layers)[layer]) uploadedBytes += trackedLayerBytes; } } - auto linear = std::make_shared(context, static_cast(linearBytes), VK_BUFFER_USAGE_STORAGE_BUFFER_BIT | VK_BUFFER_USAGE_TRANSFER_SRC_BIT); + const bool direct = singlePass(); + auto linear = direct ? nullptr : std::make_shared(context, static_cast(linearBytes), VK_BUFFER_USAGE_STORAGE_BUFFER_BIT | VK_BUFFER_USAGE_TRANSFER_SRC_BIT); detiler.BeginBatch(); auto* recorder = Recorder::Active(); std::unique_ptr batch; @@ -1782,12 +1821,12 @@ void StorageTexture::upload(const std::vector* layers) { recorder->FlushStoresOverlapping(descriptor.baseAddress, static_cast(guestBytes)); commands = recorder->Commands(); timing = recorder->BeginGpuTiming(Recorder::CommandClass::StorageUpload); - recorder->Keep(linear, linear->Size()); + if (linear) recorder->Keep(linear, linear->Size()); // The image itself must outlive the recorded copy: the cache may evict it right after. if (auto self = weak_from_this().lock()) recorder->Keep(std::move(self)); // The detile reads the tiled bytes from the import when the batch runs. recorder->NotePendingRead(descriptor.baseAddress, static_cast(guestBytes), Recorder::ReadKind::StorageUpload); - Recorder::CountBarriers(Recorder::CommandClass::StorageUpload, 3); + Recorder::CountBarriers(Recorder::CommandClass::StorageUpload, direct ? 2 : 3); if (Recorder::BarrierValidate()) { const std::pair read{descriptor.baseAddress, descriptor.baseAddress + guestBytes}; const std::pair written{image, true}; @@ -1798,6 +1837,26 @@ void StorageTexture::upload(const std::vector* layers) { commands = batch->Handle(); } const auto importOffset = descriptor.baseAddress - import->base; + if (direct) { + recordDirectUploadBarrier(commands, layers == nullptr); + for (std::uint32_t layer = 0; layer < arrayLayers; ++layer) { + if (layers != nullptr && !(*layers)[layer]) continue; + for (std::uint32_t level = 0; level < mips.size(); ++level) { + if (!geometry.HasLayer(level, layer)) continue; + const auto& mip = mips[level]; + detiler.DispatchImage(commands, descriptor.tileMode, elementBytes, import->buffer, importOffset + geometry.GuestLayerOffset(layer) + mip.tiledOffset, elementLayerView(level, layer), mip, false, layer, {.pipeBankXor = descriptor.pipeBankXor}); + } + } + recordDirectUploadDone(commands); + singlePassMoves.fetch_add(1, std::memory_order_relaxed); + if (batch) batch->SubmitAndWait(); + else recorder->EndGpuTiming(timing, uploadedBytes); + countStorageUpload(1, uploadedBytes); + ++Profile().storageDirectUploads; + ++version; + if (profile) LookupOutcomes::Add(LookupOutcomes::UploadDirect, start); + return; + } RecordMemoryBarrier(context, commands, VK_PIPELINE_STAGE_ALL_COMMANDS_BIT | VK_PIPELINE_STAGE_HOST_BIT, VK_PIPELINE_STAGE_COMPUTE_SHADER_BIT, VK_ACCESS_SHADER_WRITE_BIT | VK_ACCESS_TRANSFER_WRITE_BIT | VK_ACCESS_HOST_WRITE_BIT, VK_ACCESS_SHADER_READ_BIT | VK_ACCESS_SHADER_WRITE_BIT); for (std::uint32_t layer = 0; layer < arrayLayers; ++layer) { if (layers != nullptr && !(*layers)[layer]) continue; @@ -2055,6 +2114,33 @@ StorageTexture::PaddingSeeds StorageTexture::paddingSeeds(const HostImport& impo return seeds; } +void StorageTexture::recordDirectUploadBarrier(VkCommandBuffer commands, bool discard) { + const VkMemoryBarrier sourceReady{VK_STRUCTURE_TYPE_MEMORY_BARRIER, nullptr, VK_ACCESS_SHADER_WRITE_BIT | VK_ACCESS_TRANSFER_WRITE_BIT | VK_ACCESS_HOST_WRITE_BIT, VK_ACCESS_SHADER_READ_BIT | VK_ACCESS_SHADER_WRITE_BIT}; + VkImageMemoryBarrier toWrite{VK_STRUCTURE_TYPE_IMAGE_MEMORY_BARRIER}; + toWrite.srcAccessMask = discard ? 0u : VK_ACCESS_SHADER_WRITE_BIT | VK_ACCESS_SHADER_READ_BIT | VK_ACCESS_TRANSFER_WRITE_BIT | VK_ACCESS_COLOR_ATTACHMENT_WRITE_BIT; + toWrite.dstAccessMask = VK_ACCESS_SHADER_WRITE_BIT; + toWrite.oldLayout = discard ? VK_IMAGE_LAYOUT_UNDEFINED : VK_IMAGE_LAYOUT_GENERAL; + toWrite.newLayout = VK_IMAGE_LAYOUT_GENERAL; + toWrite.srcQueueFamilyIndex = VK_QUEUE_FAMILY_IGNORED; + toWrite.dstQueueFamilyIndex = VK_QUEUE_FAMILY_IGNORED; + toWrite.image = image; + toWrite.subresourceRange = {VK_IMAGE_ASPECT_COLOR_BIT, 0, descriptor.mipCount, 0, geometry.imageLayers}; + context.Function("vkCmdPipelineBarrier")(commands, VK_PIPELINE_STAGE_ALL_COMMANDS_BIT | VK_PIPELINE_STAGE_HOST_BIT, VK_PIPELINE_STAGE_COMPUTE_SHADER_BIT, 0, 1, &sourceReady, 0, nullptr, 1, &toWrite); +} + +void StorageTexture::recordDirectUploadDone(VkCommandBuffer commands) { + VkImageMemoryBarrier written{VK_STRUCTURE_TYPE_IMAGE_MEMORY_BARRIER}; + written.srcAccessMask = VK_ACCESS_SHADER_WRITE_BIT; + written.dstAccessMask = VK_ACCESS_SHADER_READ_BIT | VK_ACCESS_SHADER_WRITE_BIT | VK_ACCESS_TRANSFER_READ_BIT | VK_ACCESS_TRANSFER_WRITE_BIT | VK_ACCESS_COLOR_ATTACHMENT_READ_BIT | VK_ACCESS_COLOR_ATTACHMENT_WRITE_BIT; + written.oldLayout = VK_IMAGE_LAYOUT_GENERAL; + written.newLayout = VK_IMAGE_LAYOUT_GENERAL; + written.srcQueueFamilyIndex = VK_QUEUE_FAMILY_IGNORED; + written.dstQueueFamilyIndex = VK_QUEUE_FAMILY_IGNORED; + written.image = image; + written.subresourceRange = {VK_IMAGE_ASPECT_COLOR_BIT, 0, descriptor.mipCount, 0, geometry.imageLayers}; + context.Function("vkCmdPipelineBarrier")(commands, VK_PIPELINE_STAGE_COMPUTE_SHADER_BIT, VK_PIPELINE_STAGE_ALL_COMMANDS_BIT, 0, 0, nullptr, 0, nullptr, 1, &written); +} + std::uint64_t StorageTexture::uploadWindows(const HostImport& import, std::span> runs, bool discard) { const auto elementBytes = BytesPerElement(descriptor.format); // Each run's pieces by source (a unit shadow's slab while fresh, else the import), the tail @@ -2094,7 +2180,8 @@ std::uint64_t StorageTexture::uploadWindows(const HostImport& import, std::span< linearTotal += (window.linearBytes + 255) & ~std::uint64_t{255}; } for (const auto& [begin, end] : runs) uploadedBytes += end - begin; - auto linear = std::make_shared(context, static_cast(linearTotal), VK_BUFFER_USAGE_STORAGE_BUFFER_BIT | VK_BUFFER_USAGE_TRANSFER_SRC_BIT); + const bool direct = singlePass(); + auto linear = direct ? nullptr : std::make_shared(context, static_cast(linearTotal), VK_BUFFER_USAGE_STORAGE_BUFFER_BIT | VK_BUFFER_USAGE_TRANSFER_SRC_BIT); detiler.BeginBatch(); auto* recorder = Recorder::Active(); std::unique_ptr batch; @@ -2111,13 +2198,13 @@ std::uint64_t StorageTexture::uploadWindows(const HostImport& import, std::span< for (const auto& [begin, end] : reads) recorder->FlushStoresOverlapping(begin, static_cast(end - begin)); commands = recorder->Commands(); timing = recorder->BeginGpuTiming(Recorder::CommandClass::StorageUpload); - recorder->Keep(linear, linear->Size()); + if (linear) recorder->Keep(linear, linear->Size()); for (const auto& source : sources) { if (source.slab != nullptr) recorder->Keep(source.slab); } if (auto self = weak_from_this().lock()) recorder->Keep(std::move(self)); if (!reads.empty()) recorder->NotePendingReads(reads, Recorder::ReadKind::StorageUpload); - Recorder::CountBarriers(Recorder::CommandClass::StorageUpload, 3); + Recorder::CountBarriers(Recorder::CommandClass::StorageUpload, direct ? 2 : 3); if (Recorder::BarrierValidate()) { const std::pair written{image, true}; recorder->NoteAccess(Recorder::CommandClass::StorageUpload, Recorder::Access{reads, {}, std::span(&written, 1), VK_PIPELINE_STAGE_COMPUTE_SHADER_BIT | VK_PIPELINE_STAGE_TRANSFER_BIT}); @@ -2127,6 +2214,27 @@ std::uint64_t StorageTexture::uploadWindows(const HostImport& import, std::span< commands = batch->Handle(); } const auto importOffset = descriptor.baseAddress - import.base; + if (direct) { + recordDirectUploadBarrier(commands, discard); + for (const auto& window : windows) { + const auto& mip = mips[window.level]; + const auto& source = sourceOf(window); + auto detile = window.window; + detile.pipeBankXor = descriptor.pipeBankXor; + const auto view = elementLayerView(window.level, window.layer); + if (source.shadow) { + detiler.DispatchImage(commands, descriptor.tileMode, elementBytes, source.buffer, source.offset + (window.tiledBegin - source.begin), view, mip, false, window.layer, detile); + } else { + detile.tiledBase = 0; + detiler.DispatchImage(commands, descriptor.tileMode, elementBytes, import.buffer, importOffset + geometry.GuestLayerOffset(window.layer) + mip.tiledOffset, view, mip, false, window.layer, detile); + } + } + recordDirectUploadDone(commands); + singlePassMoves.fetch_add(1, std::memory_order_relaxed); + if (batch) batch->SubmitAndWait(); + else recorder->EndGpuTiming(timing, uploadedBytes); + return uploadedBytes; + } RecordMemoryBarrier(context, commands, VK_PIPELINE_STAGE_ALL_COMMANDS_BIT | VK_PIPELINE_STAGE_HOST_BIT, VK_PIPELINE_STAGE_COMPUTE_SHADER_BIT, VK_ACCESS_SHADER_WRITE_BIT | VK_ACCESS_TRANSFER_WRITE_BIT | VK_ACCESS_HOST_WRITE_BIT, VK_ACCESS_SHADER_READ_BIT | VK_ACCESS_SHADER_WRITE_BIT); std::vector regions; for (std::size_t i = 0; i < windows.size(); ++i) { @@ -2265,7 +2373,8 @@ std::uint64_t StorageTexture::writeBackWindows(const HostImport& import, std::sp flushBegin = std::min(flushBegin, seed.begin); flushEnd = std::max(flushEnd, seed.end); } - auto linear = std::make_shared(context, static_cast(linearTotal), VK_BUFFER_USAGE_STORAGE_BUFFER_BIT | VK_BUFFER_USAGE_TRANSFER_DST_BIT); + const bool direct = singlePass(); + auto linear = direct ? nullptr : std::make_shared(context, static_cast(linearTotal), VK_BUFFER_USAGE_STORAGE_BUFFER_BIT | VK_BUFFER_USAGE_TRANSFER_DST_BIT); auto tiledScratch = std::make_shared(context, static_cast(scratchTotal), VK_BUFFER_USAGE_STORAGE_BUFFER_BIT | VK_BUFFER_USAGE_TRANSFER_SRC_BIT | VK_BUFFER_USAGE_TRANSFER_DST_BIT); detiler.BeginBatch(); auto* recorder = Recorder::Active(); @@ -2278,12 +2387,12 @@ std::uint64_t StorageTexture::writeBackWindows(const HostImport& import, std::sp recorder->FlushStoresOverlapping(flushBegin, static_cast(flushEnd - flushBegin)); commands = recorder->Commands(&covered); timing = recorder->BeginGpuTiming(Recorder::CommandClass::StorageWriteBack); - recorder->Keep(linear, linear->Size()); + if (linear) recorder->Keep(linear, linear->Size()); recorder->Keep(tiledScratch, tiledScratch->Size()); for (const auto& pieces : slabPieces) recorder->Keep(pieces.slab); for (const auto& slab : padding.slabs) recorder->Keep(slab); if (auto self = weak_from_this().lock()) recorder->Keep(std::move(self)); - Recorder::CountBarriers(Recorder::CommandClass::StorageWriteBack, 4); + Recorder::CountBarriers(Recorder::CommandClass::StorageWriteBack, direct ? 3 : 4); if (Recorder::BarrierValidate()) { const std::pair read{image, false}; std::vector> seedReads; @@ -2304,7 +2413,7 @@ std::uint64_t StorageTexture::writeBackWindows(const HostImport& import, std::sp toSource.dstQueueFamilyIndex = VK_QUEUE_FAMILY_IGNORED; toSource.image = image; toSource.subresourceRange = {VK_IMAGE_ASPECT_COLOR_BIT, 0, descriptor.mipCount, 0, geometry.imageLayers}; - context.Function("vkCmdPipelineBarrier")(commands, VK_PIPELINE_STAGE_ALL_COMMANDS_BIT, VK_PIPELINE_STAGE_TRANSFER_BIT, 0, 0, nullptr, 0, nullptr, 1, &toSource); + if (!direct) context.Function("vkCmdPipelineBarrier")(commands, VK_PIPELINE_STAGE_ALL_COMMANDS_BIT, VK_PIPELINE_STAGE_TRANSFER_BIT, 0, 0, nullptr, 0, nullptr, 1, &toSource); if (!seeds.empty() || !padding.copies.empty()) { // The seeded units' import bytes (every earlier writer of them, host stores included) // precede the seed copies; the previous command's trailing barrier may have covered that. @@ -2325,22 +2434,37 @@ std::uint64_t StorageTexture::writeBackWindows(const HostImport& import, std::sp // orders the scratch writes). for (const auto& [source, copies] : padding.copies) context.Resolved(&DeviceFunctions::cmdCopyBuffer, "vkCmdCopyBuffer")(commands, source, tiledScratch->Handle(), static_cast(copies.size()), copies.data()); } - std::vector regions; - for (std::size_t i = 0; i < windows.size(); ++i) { - for (auto region : windows[i].regions) { - region.bufferOffset += linearPositions[i]; - regions.push_back(region); - } - } - context.Function("vkCmdCopyImageToBuffer")(commands, image, VK_IMAGE_LAYOUT_TRANSFER_SRC_OPTIMAL, linear->Handle(), static_cast(regions.size()), regions.data()); - const auto linearRead = WholeBufferBarrier(linear->Handle(), VK_ACCESS_TRANSFER_WRITE_BIT, VK_ACCESS_SHADER_READ_BIT); const VkMemoryBarrier importReady{VK_STRUCTURE_TYPE_MEMORY_BARRIER, nullptr, VK_ACCESS_SHADER_READ_BIT | VK_ACCESS_SHADER_WRITE_BIT | VK_ACCESS_TRANSFER_READ_BIT | VK_ACCESS_TRANSFER_WRITE_BIT | VK_ACCESS_HOST_WRITE_BIT, VK_ACCESS_SHADER_WRITE_BIT | VK_ACCESS_SHADER_READ_BIT}; - context.Function("vkCmdPipelineBarrier")(commands, VK_PIPELINE_STAGE_ALL_COMMANDS_BIT | VK_PIPELINE_STAGE_HOST_BIT, VK_PIPELINE_STAGE_COMPUTE_SHADER_BIT, 0, 1, &importReady, 1, &linearRead, 0, nullptr); - for (std::size_t i = 0; i < windows.size(); ++i) { - const auto& window = windows[i]; - auto retile = window.window; - retile.pipeBankXor = descriptor.pipeBankXor; - detiler.Dispatch(commands, descriptor.tileMode, elementBytes, linear->Handle(), linearPositions[i], tiledScratch->Handle(), scratchPositions[i], mips[window.level], true, window.layer, false, retile); + if (direct) { + VkImageMemoryBarrier toRead = toSource; + toRead.srcAccessMask = VK_ACCESS_SHADER_WRITE_BIT | VK_ACCESS_COLOR_ATTACHMENT_WRITE_BIT | VK_ACCESS_TRANSFER_WRITE_BIT; + toRead.dstAccessMask = VK_ACCESS_SHADER_READ_BIT; + toRead.newLayout = VK_IMAGE_LAYOUT_GENERAL; + context.Function("vkCmdPipelineBarrier")(commands, VK_PIPELINE_STAGE_ALL_COMMANDS_BIT | VK_PIPELINE_STAGE_HOST_BIT, VK_PIPELINE_STAGE_COMPUTE_SHADER_BIT, 0, 1, &importReady, 0, nullptr, 1, &toRead); + for (std::size_t i = 0; i < windows.size(); ++i) { + const auto& window = windows[i]; + auto retile = window.window; + retile.pipeBankXor = descriptor.pipeBankXor; + detiler.DispatchImage(commands, descriptor.tileMode, elementBytes, tiledScratch->Handle(), scratchPositions[i], elementLayerView(window.level, window.layer), mips[window.level], true, window.layer, retile); + } + singlePassMoves.fetch_add(1, std::memory_order_relaxed); + } else { + std::vector regions; + for (std::size_t i = 0; i < windows.size(); ++i) { + for (auto region : windows[i].regions) { + region.bufferOffset += linearPositions[i]; + regions.push_back(region); + } + } + context.Function("vkCmdCopyImageToBuffer")(commands, image, VK_IMAGE_LAYOUT_TRANSFER_SRC_OPTIMAL, linear->Handle(), static_cast(regions.size()), regions.data()); + const auto linearRead = WholeBufferBarrier(linear->Handle(), VK_ACCESS_TRANSFER_WRITE_BIT, VK_ACCESS_SHADER_READ_BIT); + context.Function("vkCmdPipelineBarrier")(commands, VK_PIPELINE_STAGE_ALL_COMMANDS_BIT | VK_PIPELINE_STAGE_HOST_BIT, VK_PIPELINE_STAGE_COMPUTE_SHADER_BIT, 0, 1, &importReady, 1, &linearRead, 0, nullptr); + for (std::size_t i = 0; i < windows.size(); ++i) { + const auto& window = windows[i]; + auto retile = window.window; + retile.pipeBankXor = descriptor.pipeBankXor; + detiler.Dispatch(commands, descriptor.tileMode, elementBytes, linear->Handle(), linearPositions[i], tiledScratch->Handle(), scratchPositions[i], mips[window.level], true, window.layer, false, retile); + } } { // The retiled scratch (and a seed's slab bytes, which the scratch copies overwrite in @@ -2360,7 +2484,7 @@ std::uint64_t StorageTexture::writeBackWindows(const HostImport& import, std::sp // merges its leading barrier. constexpr VkAccessFlags storedAccess = VK_ACCESS_SHADER_READ_BIT | VK_ACCESS_SHADER_WRITE_BIT | VK_ACCESS_TRANSFER_READ_BIT | VK_ACCESS_TRANSFER_WRITE_BIT | VK_ACCESS_HOST_READ_BIT; const VkMemoryBarrier stored{VK_STRUCTURE_TYPE_MEMORY_BARRIER, nullptr, VK_ACCESS_TRANSFER_WRITE_BIT | VK_ACCESS_SHADER_WRITE_BIT, storedAccess}; - context.Function("vkCmdPipelineBarrier")(commands, VK_PIPELINE_STAGE_COMPUTE_SHADER_BIT | VK_PIPELINE_STAGE_TRANSFER_BIT, VK_PIPELINE_STAGE_ALL_COMMANDS_BIT | VK_PIPELINE_STAGE_HOST_BIT, 0, 1, &stored, 0, nullptr, 1, &backToGeneral); + context.Function("vkCmdPipelineBarrier")(commands, VK_PIPELINE_STAGE_COMPUTE_SHADER_BIT | VK_PIPELINE_STAGE_TRANSFER_BIT, VK_PIPELINE_STAGE_ALL_COMMANDS_BIT | VK_PIPELINE_STAGE_HOST_BIT, 0, 1, &stored, 0, nullptr, direct ? 0u : 1u, &backToGeneral); if (batch) { batch->SubmitAndWait(); } else { @@ -3733,7 +3857,7 @@ void StorageTexture::writeBackLayers(const std::vector& layers) { recorder->FlushStoresOverlapping(firstStored, static_cast(lastStored - firstStored)); commands = recorder->Commands(); timing = recorder->BeginGpuTiming(Recorder::CommandClass::StorageWriteBack); - recorder->Keep(linear, linear->Size()); + if (linear) recorder->Keep(linear, linear->Size()); recorder->Keep(tiledScratch, tiledScratch->Size()); for (const auto& slab : padding.slabs) recorder->Keep(slab); // The image itself must outlive the recorded retile: the cache may evict it right after. @@ -3965,6 +4089,8 @@ void StorageTexture::release() noexcept { uintViews.clear(); if (elementView) context.Function("vkDestroyImageView")(context.device, elementView, nullptr); elementView = VK_NULL_HANDLE; + for (const auto& [key, element] : elementLayerViews) context.Function("vkDestroyImageView")(context.device, element, nullptr); + elementLayerViews.clear(); for (const auto& [format, attachment] : attachmentViews) context.Function("vkDestroyImageView")(context.device, attachment, nullptr); attachmentViews.clear(); if (proxyView) context.Function("vkDestroyImageView")(context.device, proxyView, nullptr); diff --git a/core/libs/prx/libSceAgcDriver/Graphics/src/TextureDetiler.cpp b/core/libs/prx/libSceAgcDriver/Graphics/src/TextureDetiler.cpp index eea951d96..bed858e7a 100644 --- a/core/libs/prx/libSceAgcDriver/Graphics/src/TextureDetiler.cpp +++ b/core/libs/prx/libSceAgcDriver/Graphics/src/TextureDetiler.cpp @@ -4,6 +4,7 @@ #include "prx/libSceAgcDriver/Graphics/include/TextureTiling.hpp" #include "prx/libSceAgcDriver/Graphics/shaders/TextureDetile_spv.h" #include "prx/libSceAgcDriver/Graphics/shaders/CmaskClear_spv.h" +#include "prx/libSceAgcDriver/Graphics/shaders/TextureDetileImage_spv.h" #include #include #include @@ -58,8 +59,8 @@ std::uint32_t BlockBytesFor(TextureTileMode tileMode) { throw std::runtime_error("AGC graphics: TextureDetiler encountered an unknown tile mode"); } -std::uint32_t PipelineKey(TextureTileMode tileMode, std::uint32_t elementBytes, bool retile, bool thick) { - return (thick ? 1u << 17 : 0u) | (retile ? 1u << 16 : 0u) | (static_cast(tileMode) << 8) | elementBytes; +std::uint32_t PipelineKey(TextureTileMode tileMode, std::uint32_t elementBytes, bool retile, bool thick, bool image) { + return (image ? 1u << 18 : 0u) | (thick ? 1u << 17 : 0u) | (retile ? 1u << 16 : 0u) | (static_cast(tileMode) << 8) | elementBytes; } } @@ -91,6 +92,10 @@ TextureDetiler::TextureDetiler(const Context& context) : context(context) { moduleInfo.codeSize = sizeof(TEXTURE_DETILE_SPV); moduleInfo.pCode = TEXTURE_DETILE_SPV; Check(context.Function("vkCreateShaderModule")(context.device, &moduleInfo, nullptr, &module), "vkCreateShaderModule"); + bindings[1].descriptorType = VK_DESCRIPTOR_TYPE_STORAGE_IMAGE; + Check(context.Function("vkCreateDescriptorSetLayout")(context.device, &layoutInfo, nullptr, &imageDescriptorLayout), "vkCreateDescriptorSetLayout"); + pipelineLayoutInfo.pSetLayouts = &imageDescriptorLayout; + Check(context.Function("vkCreatePipelineLayout")(context.device, &pipelineLayoutInfo, nullptr, &imagePipelineLayout), "vkCreatePipelineLayout"); } catch (...) { release(); throw; @@ -108,15 +113,30 @@ void TextureDetiler::release() noexcept { if (cmaskDescriptorLayout) context.Function("vkDestroyDescriptorSetLayout")(context.device, cmaskDescriptorLayout, nullptr); cmaskErrors.reset(); for (const auto pool : descriptorPools) context.Function("vkDestroyDescriptorPool")(context.device, pool, nullptr); + for (const auto pool : imageDescriptorPools) context.Function("vkDestroyDescriptorPool")(context.device, pool, nullptr); + if (imageModule) context.Function("vkDestroyShaderModule")(context.device, imageModule, nullptr); + if (imagePipelineLayout) context.Function("vkDestroyPipelineLayout")(context.device, imagePipelineLayout, nullptr); + if (imageDescriptorLayout) context.Function("vkDestroyDescriptorSetLayout")(context.device, imageDescriptorLayout, nullptr); for (const auto& entry : pipelines) context.Function("vkDestroyPipeline")(context.device, entry.second, nullptr); if (module) context.Function("vkDestroyShaderModule")(context.device, module, nullptr); if (pipelineLayout) context.Function("vkDestroyPipelineLayout")(context.device, pipelineLayout, nullptr); if (descriptorLayout) context.Function("vkDestroyDescriptorSetLayout")(context.device, descriptorLayout, nullptr); } -VkPipeline TextureDetiler::pipeline(TextureTileMode tileMode, std::uint32_t elementBytes, bool retile, bool thick) { +VkFormat TextureDetiler::ImageElementFormat(std::uint32_t elementBytes) { + switch (elementBytes) { + case 1: return VK_FORMAT_R8_UINT; + case 2: return VK_FORMAT_R16_UINT; + case 4: return VK_FORMAT_R32_UINT; + case 8: return VK_FORMAT_R32G32_UINT; + case 16: return VK_FORMAT_R32G32B32A32_UINT; + default: return VK_FORMAT_UNDEFINED; + } +} + +VkPipeline TextureDetiler::pipeline(TextureTileMode tileMode, std::uint32_t elementBytes, bool retile, bool thick, bool image) { Require(std::has_single_bit(elementBytes) && elementBytes <= 16u, "unsupported element size for texture detiling"); - const auto key = PipelineKey(tileMode, elementBytes, retile, thick); + const auto key = PipelineKey(tileMode, elementBytes, retile, thick, image); for (const auto& entry : pipelines) { if (entry.first == key) return entry.second; } @@ -139,6 +159,12 @@ VkPipeline TextureDetiler::pipeline(TextureTileMode tileMode, std::uint32_t elem values[2] = 2u; std::copy(equation->bits.begin(), equation->bits.end(), values.begin() + 4); } + if (image && imageModule == VK_NULL_HANDLE) { + VkShaderModuleCreateInfo moduleInfo{VK_STRUCTURE_TYPE_SHADER_MODULE_CREATE_INFO}; + moduleInfo.codeSize = sizeof(TEXTURE_DETILE_IMAGE_SPV); + moduleInfo.pCode = TEXTURE_DETILE_IMAGE_SPV; + Check(context.Function("vkCreateShaderModule")(context.device, &moduleInfo, nullptr, &imageModule), "vkCreateShaderModule"); + } std::array entries{}; for (std::uint32_t index = 0; index < entries.size(); ++index) entries[index] = {index, index * 4u, 4}; VkSpecializationInfo specialization{}; @@ -148,27 +174,32 @@ VkPipeline TextureDetiler::pipeline(TextureTileMode tileMode, std::uint32_t elem specialization.pData = values.data(); VkPipelineShaderStageCreateInfo stage{VK_STRUCTURE_TYPE_PIPELINE_SHADER_STAGE_CREATE_INFO}; stage.stage = VK_SHADER_STAGE_COMPUTE_BIT; - stage.module = module; + stage.module = image ? imageModule : module; stage.pName = "main"; stage.pSpecializationInfo = &specialization; VkComputePipelineCreateInfo createInfo{VK_STRUCTURE_TYPE_COMPUTE_PIPELINE_CREATE_INFO}; createInfo.stage = stage; - createInfo.layout = pipelineLayout; + createInfo.layout = image ? imagePipelineLayout : pipelineLayout; VkPipeline result = VK_NULL_HANDLE; Check(context.Function("vkCreateComputePipelines")(context.device, context.pipelineCache, 1, &createInfo, nullptr, &result), "vkCreateComputePipelines"); pipelines.emplace_back(key, result); return result; } +void TextureDetiler::checkWindow(const TileMipLayout& layout, const DetileWindow& window, std::uint32_t& columnEnd, std::uint32_t& rowEnd) const { + Require(layout.width != 0 && layout.height != 0, "texture detiling requires a non-empty mip layout"); + Require(layout.tiledSize != 0 && layout.linearSize != 0, "texture detiling requires a non-empty mip layout"); + columnEnd = window.columnEnd != 0 ? std::min(window.columnEnd, layout.width) : layout.width; + rowEnd = window.rowEnd != 0 ? std::min(window.rowEnd, layout.height) : layout.height; + if (!(window.columnBegin < columnEnd && window.rowBegin < rowEnd)) Require(false, "texture detiling window lies outside the mip: columns " + std::to_string(window.columnBegin) + ".." + std::to_string(columnEnd) + ", rows " + std::to_string(window.rowBegin) + ".." + std::to_string(rowEnd) + " of " + std::to_string(layout.width) + "x" + std::to_string(layout.height)); +} + void TextureDetiler::Dispatch(VkCommandBuffer commands, TextureTileMode tileMode, std::uint32_t elementBytes, VkBuffer source, std::uint64_t sourceOffset, VkBuffer destination, std::uint64_t destinationOffset, const TileMipLayout& layout, bool retile, std::uint32_t slice, bool thick, const DetileWindow& window) { Require(commands != VK_NULL_HANDLE, "texture detiling requires an active command buffer"); Require(window.rangeBegin < window.rangeEnd && window.tiledBase <= window.rangeBegin, "texture detiling window is empty"); Require(source != VK_NULL_HANDLE && destination != VK_NULL_HANDLE, "texture detiling requires source and destination buffers"); - Require(layout.width != 0 && layout.height != 0, "texture detiling requires a non-empty mip layout"); - Require(layout.tiledSize != 0 && layout.linearSize != 0, "texture detiling requires a non-empty mip layout"); - const auto columnEnd = window.columnEnd != 0 ? std::min(window.columnEnd, layout.width) : layout.width; - const auto rowEnd = window.rowEnd != 0 ? std::min(window.rowEnd, layout.height) : layout.height; - if (!(window.columnBegin < columnEnd && window.rowBegin < rowEnd)) Require(false, "texture detiling window lies outside the mip: columns " + std::to_string(window.columnBegin) + ".." + std::to_string(columnEnd) + ", rows " + std::to_string(window.rowBegin) + ".." + std::to_string(rowEnd) + " of " + std::to_string(layout.width) + "x" + std::to_string(layout.height)); + std::uint32_t columnEnd = 0, rowEnd = 0; + checkWindow(layout, window, columnEnd, rowEnd); const auto target = pipeline(tileMode, elementBytes, retile, thick); const auto alignment = std::max(context.limits.minStorageBufferOffsetAlignment, 4); const auto sourceDescriptorOffset = sourceOffset - sourceOffset % alignment; @@ -332,4 +363,63 @@ void TextureDetiler::DispatchCmaskClear(VkCommandBuffer commands, VkBuffer cmask barrier(VK_PIPELINE_STAGE_TRANSFER_BIT, VK_PIPELINE_STAGE_ALL_COMMANDS_BIT | VK_PIPELINE_STAGE_HOST_BIT, VK_ACCESS_TRANSFER_WRITE_BIT, anyAccess); } +void TextureDetiler::DispatchImage(VkCommandBuffer commands, TextureTileMode tileMode, std::uint32_t elementBytes, VkBuffer tiled, std::uint64_t tiledOffset, VkImageView view, const TileMipLayout& layout, bool retile, std::uint32_t slice, const DetileWindow& window) { + Require(commands != VK_NULL_HANDLE, "texture detiling requires an active command buffer"); + Require(window.rangeBegin < window.rangeEnd && window.tiledBase <= window.rangeBegin, "texture detiling window is empty"); + Require(tiled != VK_NULL_HANDLE && view != VK_NULL_HANDLE, "texture detiling requires a tiled buffer and an image view"); + Require(ImageElementFormat(elementBytes) != VK_FORMAT_UNDEFINED, "unsupported element size for texture detiling through an image"); + std::uint32_t columnEnd = 0, rowEnd = 0; + checkWindow(layout, window, columnEnd, rowEnd); + const auto target = pipeline(tileMode, elementBytes, retile, false, true); + const auto alignment = std::max(context.limits.minStorageBufferOffsetAlignment, 4); + const auto descriptorOffset = tiledOffset - tiledOffset % alignment; + const auto base = tiledOffset - descriptorOffset; + Require(base <= UINT32_MAX, "texture detiling buffer offset exceeds addressable range"); + const std::uint64_t tiledBytes = window.rangeEnd == 0xffffffffu ? layout.tiledSize : std::min(layout.tiledSize, window.rangeEnd) - window.tiledBase; + const auto range = (base + tiledBytes + 3) / 4 * 4; + Require(range <= context.limits.maxStorageBufferRange, "texture detiling buffer range exceeds device limits"); + const auto set = allocateSet(true); + const VkDescriptorBufferInfo bufferInfo{tiled, descriptorOffset, range}; + const VkDescriptorImageInfo imageInfo{VK_NULL_HANDLE, view, VK_IMAGE_LAYOUT_GENERAL}; + std::array writes{}; + writes[0].sType = VK_STRUCTURE_TYPE_WRITE_DESCRIPTOR_SET; + writes[0].dstSet = set; + writes[0].dstBinding = 0; + writes[0].descriptorCount = 1; + writes[0].descriptorType = VK_DESCRIPTOR_TYPE_STORAGE_BUFFER; + writes[0].pBufferInfo = &bufferInfo; + writes[1].sType = VK_STRUCTURE_TYPE_WRITE_DESCRIPTOR_SET; + writes[1].dstSet = set; + writes[1].dstBinding = 1; + writes[1].descriptorCount = 1; + writes[1].descriptorType = VK_DESCRIPTOR_TYPE_STORAGE_IMAGE; + writes[1].pImageInfo = &imageInfo; + context.Function("vkUpdateDescriptorSets")(context.device, static_cast(writes.size()), writes.data(), 0, nullptr); + context.Function("vkCmdBindPipeline")(commands, VK_PIPELINE_BIND_POINT_COMPUTE, target); + context.Function("vkCmdBindDescriptorSets")(commands, VK_PIPELINE_BIND_POINT_COMPUTE, imagePipelineLayout, 0, 1, &set, 0, nullptr); + Push push{}; + push.srcBase = static_cast(retile ? 0 : base); + push.dstBase = static_cast(retile ? base : 0); + push.width = layout.width; + push.height = layout.height; + push.pitchBytes = layout.pitchBytes; + push.blocksPerRow = layout.blocksPerRow; + push.tail = layout.tail ? 1u : 0u; + push.tailX = layout.tailX; + push.tailY = layout.tailY; + push.elementBytes = elementBytes; + push.slice = slice; + push.rangeBegin = window.rangeBegin; + push.rangeEnd = window.rangeEnd; + push.tiledBase = window.tiledBase; + push.linearBase = 0; + push.columnBegin = window.columnBegin; + push.rowBegin = window.rowBegin; + push.pipeBankXor = window.pipeBankXor; + context.Function("vkCmdPushConstants")(commands, imagePipelineLayout, VK_SHADER_STAGE_COMPUTE_BIT, 0, sizeof(Push), &push); + const auto groupsX = (columnEnd - window.columnBegin + 7u) / 8u; + const auto groupsY = (rowEnd - window.rowBegin + 7u) / 8u; + context.Function("vkCmdDispatch")(commands, groupsX, groupsY, 1); +} + } diff --git a/core/libs/prx/libSceAgcDriver/Graphics/src/TextureDetilerDescriptors.cpp b/core/libs/prx/libSceAgcDriver/Graphics/src/TextureDetilerDescriptors.cpp index cea243e2e..4d482528c 100644 --- a/core/libs/prx/libSceAgcDriver/Graphics/src/TextureDetilerDescriptors.cpp +++ b/core/libs/prx/libSceAgcDriver/Graphics/src/TextureDetilerDescriptors.cpp @@ -1,6 +1,7 @@ #include "prx/libSceAgcDriver/Graphics/include/Recorder.hpp" #include "prx/libSceAgcDriver/Graphics/include/TextureDetiler.hpp" #include "prx/libSceAgcDriver/Execution/include/GuestMemory.hpp" +#include namespace AgcDriver::Graphics { @@ -17,29 +18,37 @@ void TextureDetiler::BeginBatch() { Check(context.Function("vkResetDescriptorPool")(context.device, pool, 0), "vkResetDescriptorPool CMASK clear"); } allocatedCmaskSets = 0; + for (const auto pool : imageDescriptorPools) { + Check(context.Function("vkResetDescriptorPool")(context.device, pool, 0), "vkResetDescriptorPool texture detiler"); + } + allocatedImageSets = 0; } -VkDescriptorSet TextureDetiler::allocateSet() { +VkDescriptorSet TextureDetiler::allocateSet(bool image) { constexpr std::uint32_t setsPerPool = 64; - const auto poolIndex = allocatedSets / setsPerPool; - if (poolIndex == descriptorPools.size()) { - const VkDescriptorPoolSize size{VK_DESCRIPTOR_TYPE_STORAGE_BUFFER, setsPerPool * 2}; + auto& pools = image ? imageDescriptorPools : descriptorPools; + auto& allocated = image ? allocatedImageSets : allocatedSets; + const auto poolIndex = allocated / setsPerPool; + if (poolIndex == pools.size()) { + std::array sizes{}; + sizes[0] = {VK_DESCRIPTOR_TYPE_STORAGE_BUFFER, image ? setsPerPool : setsPerPool * 2}; + sizes[1] = {VK_DESCRIPTOR_TYPE_STORAGE_IMAGE, setsPerPool}; VkDescriptorPoolCreateInfo info{VK_STRUCTURE_TYPE_DESCRIPTOR_POOL_CREATE_INFO}; info.maxSets = setsPerPool; - info.poolSizeCount = 1; - info.pPoolSizes = &size; - descriptorPools.reserve(descriptorPools.size() + 1); + info.poolSizeCount = image ? 2u : 1u; + info.pPoolSizes = sizes.data(); + pools.reserve(pools.size() + 1); VkDescriptorPool pool = VK_NULL_HANDLE; Check(context.Function("vkCreateDescriptorPool")(context.device, &info, nullptr, &pool), "vkCreateDescriptorPool texture detiler"); - descriptorPools.push_back(pool); + pools.push_back(pool); } VkDescriptorSetAllocateInfo allocation{VK_STRUCTURE_TYPE_DESCRIPTOR_SET_ALLOCATE_INFO}; - allocation.descriptorPool = descriptorPools.at(poolIndex); + allocation.descriptorPool = pools.at(poolIndex); allocation.descriptorSetCount = 1; - allocation.pSetLayouts = &descriptorLayout; + allocation.pSetLayouts = image ? &imageDescriptorLayout : &descriptorLayout; VkDescriptorSet set = VK_NULL_HANDLE; Check(context.Function("vkAllocateDescriptorSets")(context.device, &allocation, &set), "vkAllocateDescriptorSets texture detiler"); - ++allocatedSets; + ++allocated; return set; } diff --git a/core/libs/prx/libSceAgcDriver/tests/Recorder.cpp b/core/libs/prx/libSceAgcDriver/tests/Recorder.cpp index 34dc7009f..b0d503dac 100644 --- a/core/libs/prx/libSceAgcDriver/tests/Recorder.cpp +++ b/core/libs/prx/libSceAgcDriver/tests/Recorder.cpp @@ -167,6 +167,9 @@ public: function("vkGetPhysicalDeviceFeatures")(context.physical, &supported); enabled.occlusionQueryPrecise = supported.occlusionQueryPrecise; context.occlusionQueryPrecise = supported.occlusionQueryPrecise == VK_TRUE; + enabled.shaderStorageImageReadWithoutFormat = supported.shaderStorageImageReadWithoutFormat; + enabled.shaderStorageImageWriteWithoutFormat = supported.shaderStorageImageWriteWithoutFormat; + context.singlePassStorage = supported.shaderStorageImageReadWithoutFormat == VK_TRUE && supported.shaderStorageImageWriteWithoutFormat == VK_TRUE; address.pNext = &bytes; std::vector extensionsEnabled{VK_KHR_BUFFER_DEVICE_ADDRESS_EXTENSION_NAME, VK_KHR_8BIT_STORAGE_EXTENSION_NAME}; VkPhysicalDeviceDriverProperties driverProperties{VK_STRUCTURE_TYPE_PHYSICAL_DEVICE_DRIVER_PROPERTIES}; @@ -2366,6 +2369,201 @@ void writeBackPaddingTests(const Device& device, Recorder& recorder, bool watche std::cout << "write-back padding" << (watched ? " (watched)" : "") << ": ok\n"; } +void singlePassTests(const Device& device, Recorder& recorder, bool watched) { + const auto& base = device.GetContext(); + if (base.hostImportAlignment == 0 || !base.singlePassStorage) { + std::cout << "host imports or storage access without a format unavailable: single-pass storage moves not tested\n"; + return; + } + struct Case { + const char* name; + std::uint32_t format; + TextureTileMode tileMode; + std::uint32_t width; + std::uint32_t height; + std::uint32_t mipCount; + std::uint32_t pipeBankXor = 0; + }; + constexpr Case cases[] = { + {"R8 64 KiB R_X 200x150", 1, TextureTileMode::kR64KBX, 200, 150, 1}, + {"R16 64 KiB R_X 200x150, 3 mips", 7, TextureTileMode::kR64KBX, 200, 150, 3}, + {"RGBA8 64 KiB R_X 520x260, 6 mips", 56, TextureTileMode::kR64KBX, 520, 260, 6}, + {"RGBA16F 64 KiB R_X 384x256", 71, TextureTileMode::kR64KBX, 384, 256, 1}, + {"RGBA16F 64 KiB R_X 256x200, 4 mips", 71, TextureTileMode::kR64KBX, 256, 200, 4}, + {"RGBA32F 64 KiB R_X 136x72, 2 mips", 77, TextureTileMode::kR64KBX, 136, 72, 2}, + {"RGBA8 64 KiB S 300x170", 56, TextureTileMode::kStandard64KB, 300, 170, 1}, + {"RGBA16F 4 KiB S 100x70, 2 mips", 71, TextureTileMode::kStandard4KB, 100, 70, 2}, + {"R8 256 B S 90x40", 1, TextureTileMode::kStandard256B, 90, 40, 1}, + {"RGBA8 64 KiB R_X 520x260, 2 mips, pipe/bank XOR 5", 56, TextureTileMode::kR64KBX, 520, 260, 2, 5}, + }; + std::string failures; + std::size_t tested = 0; + for (const auto& test : cases) { + if (!StorageFormatAvailable(base, test.format)) continue; + const std::string what = std::string(watched ? "(watched) " : "") + test.name; + GuestTextureResource resource{}; + resource.width = test.width; + resource.height = test.height; + resource.mipCount = test.mipCount; + resource.tileMode = test.tileMode; + resource.dimension = TextureDimension::k2D; + resource.format = test.format; + resource.pipeBankXor = test.pipeBankXor; + resource.dstSelX = 4; + resource.dstSelY = 5; + resource.dstSelZ = 6; + resource.dstSelW = 7; + const auto geometry = DescribeSurface(resource); + const auto elementBytes = BytesPerElement(test.format); + const auto bytes = (static_cast(geometry.guestBytes) + 65535) / 65536 * 65536 + 65536; + struct Result { + std::vector uploaded, memory, reuploaded; + std::uint64_t moves = 0; + }; + std::array results; + bool skipped = false; + for (int mode = 0; mode < 2 && !skipped; ++mode) { + auto context = base; + context.singlePassStorage = mode == 1; + void* block = nullptr; + if (watched) { + block = AllocateWatched(bytes, 65536); + } else { +#ifdef _WIN32 + block = VirtualAlloc(nullptr, bytes, MEM_RESERVE | MEM_COMMIT, PAGE_READWRITE); +#else + block = std::aligned_alloc(65536, bytes); +#endif + } + if (block == nullptr) { + std::cout << "no write watching: single-pass storage moves in watched memory not tested\n"; + return; + } + auto* memory = static_cast(block); + std::uint32_t seed = 0x13579bdfu; + const auto random = [&] { + seed ^= seed << 13; + seed ^= seed >> 17; + seed ^= seed << 5; + return static_cast(seed >> 9); + }; + for (std::size_t i = 0; i < bytes; ++i) memory[i] = random(); + const auto address = reinterpret_cast(block); + { + GuestAllocations::Mutation mutation; + mutation.Add(block, bytes, true, true, true); + } + struct Unregister { + const Context& context; + void* block; + std::uint64_t address; + std::size_t bytes; + bool watched; + ~Unregister() { + { + GuestAllocations::Mutation mutation; + mutation.Remove(block); + } + HostImportFor(context, address, bytes); + if (watched) ReleaseWatched(block, bytes); +#ifdef _WIN32 + else VirtualFree(block, 0, MEM_RELEASE); +#else + else std::free(block); +#endif + } + } unregister{base, block, address, bytes, watched}; + if (HostImportFor(base, address, bytes) == nullptr) { + std::cout << "host import refused: single-pass storage moves not tested\n"; + return; + } + TextureDetiler detiler(base); + context.detiler = &detiler; + resource.baseAddress = address; + std::size_t linearBytes = 0; + for (const auto& mip : geometry.mips) linearBytes += static_cast(mip.width) * mip.height * elementBytes; + const auto readImage = [&](const StorageTexture& image) { + Buffer readback(context, linearBytes, VK_BUFFER_USAGE_TRANSFER_DST_BIT); + const auto commands = recorder.Commands(); + RecordMemoryBarrier(context, commands, VK_PIPELINE_STAGE_ALL_COMMANDS_BIT, VK_PIPELINE_STAGE_TRANSFER_BIT, VK_ACCESS_SHADER_WRITE_BIT | VK_ACCESS_TRANSFER_WRITE_BIT | VK_ACCESS_COLOR_ATTACHMENT_WRITE_BIT, VK_ACCESS_TRANSFER_READ_BIT); + std::vector copies; + std::size_t at = 0; + for (std::uint32_t level = 0; level < geometry.mips.size(); ++level) { + VkBufferImageCopy copy{}; + copy.bufferOffset = at; + copy.imageSubresource = {VK_IMAGE_ASPECT_COLOR_BIT, level, 0, 1}; + copy.imageExtent = {geometry.mips[level].width, geometry.mips[level].height, 1}; + copies.push_back(copy); + at += static_cast(geometry.mips[level].width) * geometry.mips[level].height * elementBytes; + } + context.Function("vkCmdCopyImageToBuffer")(commands, image.Image(), VK_IMAGE_LAYOUT_GENERAL, readback.Handle(), static_cast(copies.size()), copies.data()); + RecordMemoryBarrier(context, commands, VK_PIPELINE_STAGE_TRANSFER_BIT, VK_PIPELINE_STAGE_ALL_COMMANDS_BIT | VK_PIPELINE_STAGE_HOST_BIT, VK_ACCESS_TRANSFER_READ_BIT | VK_ACCESS_TRANSFER_WRITE_BIT, VK_ACCESS_HOST_READ_BIT | VK_ACCESS_SHADER_READ_BIT | VK_ACCESS_SHADER_WRITE_BIT | VK_ACCESS_TRANSFER_WRITE_BIT); + recorder.Submit(); + device.WaitQueue(); + recorder.Sync(); + const auto read = readback.Bytes(); + return std::vector(reinterpret_cast(read.data()), reinterpret_cast(read.data()) + linearBytes); + }; + std::vector texels(linearBytes); + for (auto& value : texels) value = random(); + const auto movesBefore = StorageTexture::SinglePassMoves(); + { + auto image = std::make_shared(context, detiler, resource, 0); + recorder.Keep(image); + results[mode].uploaded = readImage(*image); + Buffer staging(context, linearBytes, VK_BUFFER_USAGE_TRANSFER_SRC_BIT); + std::memcpy(staging.Bytes().data(), texels.data(), linearBytes); + const auto commands = recorder.Commands(); + RecordMemoryBarrier(context, commands, VK_PIPELINE_STAGE_ALL_COMMANDS_BIT, VK_PIPELINE_STAGE_TRANSFER_BIT, VK_ACCESS_SHADER_WRITE_BIT | VK_ACCESS_TRANSFER_WRITE_BIT | VK_ACCESS_COLOR_ATTACHMENT_WRITE_BIT, VK_ACCESS_TRANSFER_WRITE_BIT); + std::vector copies; + std::size_t at = 0; + for (std::uint32_t level = 0; level < geometry.mips.size(); ++level) { + VkBufferImageCopy copy{}; + copy.bufferOffset = at; + copy.imageSubresource = {VK_IMAGE_ASPECT_COLOR_BIT, level, 0, 1}; + copy.imageExtent = {geometry.mips[level].width, geometry.mips[level].height, 1}; + copies.push_back(copy); + at += static_cast(geometry.mips[level].width) * geometry.mips[level].height * elementBytes; + } + context.Function("vkCmdCopyBufferToImage")(commands, staging.Handle(), image->Image(), VK_IMAGE_LAYOUT_GENERAL, static_cast(copies.size()), copies.data()); + RecordMemoryBarrier(context, commands, VK_PIPELINE_STAGE_TRANSFER_BIT, VK_PIPELINE_STAGE_ALL_COMMANDS_BIT, VK_ACCESS_TRANSFER_WRITE_BIT, VK_ACCESS_SHADER_READ_BIT | VK_ACCESS_SHADER_WRITE_BIT | VK_ACCESS_TRANSFER_READ_BIT | VK_ACCESS_TRANSFER_WRITE_BIT); + image->MarkDirty(); + image->WriteBack(); + recorder.Submit(); + device.WaitQueue(); + recorder.Sync(); + auto again = std::make_shared(context, detiler, resource, 0); + recorder.Keep(again); + results[mode].reuploaded = readImage(*again); + std::vector read(bytes); + if (watched) AgcDriver::GuestMemory::Read(address, read); + else { + StorageTexture::FlushPending(address, bytes, nullptr, "test", PublishScope::Whole); + recorder.Submit(); + device.WaitQueue(); + recorder.Sync(); + std::memcpy(read.data(), memory, bytes); + } + results[mode].memory.assign(reinterpret_cast(read.data()), reinterpret_cast(read.data()) + bytes); + } + recorder.Sync(); + results[mode].moves = StorageTexture::SinglePassMoves() - movesBefore; + if (results[mode].reuploaded != texels) failures += what + (mode == 1 ? " (single pass)" : " (copies)") + ": the second image does not hold the texels written back\n"; + } + if (results[0].moves != 0 || results[1].moves < 3) failures += what + ": single-pass moves " + std::to_string(results[0].moves) + " with the copies and " + std::to_string(results[1].moves) + " in single pass (want 0 and at least 3)\n"; + if (results[0].uploaded != results[1].uploaded) failures += what + ": the first upload differs between the copies and single pass\n"; + if (results[0].memory != results[1].memory) { + std::size_t differ = 0; + for (std::size_t i = 0; i < results[0].memory.size(); ++i) differ += results[0].memory[i] != results[1].memory[i]; + failures += what + ": guest memory after the write-back differs in " + std::to_string(differ) + " bytes between the copies and single pass\n"; + } + ++tested; + } + if (!failures.empty()) throw std::runtime_error("single-pass storage moves:\n" + failures); + std::cout << "single-pass storage moves" << (watched ? " (watched)" : "") << ": ok (" << tested << " surfaces)\n"; +} + + void unchangedCpuStampTests(const Device& device, Recorder& recorder) { const auto& base = device.GetContext(); constexpr std::uint32_t side = 256; @@ -3906,6 +4104,8 @@ int main(int argc, char** argv) { storageRefreshTests(device, recorder, true); writeBackPaddingTests(device, recorder, false); writeBackPaddingTests(device, recorder, true); + singlePassTests(device, recorder, false); + singlePassTests(device, recorder, true); unchangedCpuStampTests(device, recorder); importWatchTests(device); staleGenerationTests(device, recorder);