diff --git a/core/libs/prx/libSceAgcDriver/CMakeLists.txt b/core/libs/prx/libSceAgcDriver/CMakeLists.txt index 583e96a21..a62c4cae1 100644 --- a/core/libs/prx/libSceAgcDriver/CMakeLists.txt +++ b/core/libs/prx/libSceAgcDriver/CMakeLists.txt @@ -45,6 +45,17 @@ add_custom_command( DEPENDS "${agcTextureDetileShaderSource}" glslang-standalone VERBATIM ) +set(agcTextureDetileImageShaderSpv ${agcTextureDetileGeneratedDir}/TextureDetileImage.spv) +set(agcTextureDetileImageShaderHeader ${agcTextureDetileGeneratedDir}/TextureDetileImage_spv.h) +add_custom_command( + OUTPUT "${agcTextureDetileImageShaderHeader}" + COMMAND ${CMAKE_COMMAND} -E make_directory "${agcTextureDetileGeneratedDir}" + COMMAND "$" -V --target-env vulkan1.1 -DLINEAR_IMAGE=1 -o "${agcTextureDetileImageShaderSpv}" "${agcTextureDetileShaderSource}" + COMMAND ${CMAKE_COMMAND} -DINPUT=${agcTextureDetileImageShaderSpv} -DOUTPUT=${agcTextureDetileImageShaderHeader} + -DSYMBOL=TEXTURE_DETILE_IMAGE_SPV -P "${CMAKE_CURRENT_SOURCE_DIR}/embed_spirv.cmake" + DEPENDS "${agcTextureDetileShaderSource}" glslang-standalone + VERBATIM +) set(agcMeshArgumentsShaderSource ${CMAKE_CURRENT_SOURCE_DIR}/Graphics/shaders/MeshArguments.comp) set(agcMeshArgumentsShaderSpv ${agcTextureDetileGeneratedDir}/MeshArguments.spv) @@ -188,6 +199,7 @@ set(agcDriverSources ${agcTextureDetileShaderHeader} ${agcCmaskClearShaderHeader} ${agcMeshArgumentsShaderHeader} + ${agcTextureDetileImageShaderHeader} ${agcSampleCounterShaderHeader} ${agcSampleDumpsShaderHeader} ${CMAKE_CURRENT_SOURCE_DIR}/Eq/src/Event.cpp @@ -756,6 +768,7 @@ add_test_executable(agc_driver_graphics_tests EXCLUDE_FROM_ALL ${agcTextureDetileShaderHeader} ${agcCmaskClearShaderHeader} ${agcMeshArgumentsShaderHeader} + ${agcTextureDetileImageShaderHeader} ${agcSampleCounterShaderHeader} ${agcSampleDumpsShaderHeader} Execution/src/GuestMemory.cpp diff --git a/core/libs/prx/libSceAgcDriver/Execution/src/VulkanDevice.cpp b/core/libs/prx/libSceAgcDriver/Execution/src/VulkanDevice.cpp index 7cfd5f2b5..11f83cfa5 100644 --- a/core/libs/prx/libSceAgcDriver/Execution/src/VulkanDevice.cpp +++ b/core/libs/prx/libSceAgcDriver/Execution/src/VulkanDevice.cpp @@ -227,6 +227,7 @@ struct VulkanDevice::State { bool pipelineExecutableInfo = false; bool maintenance8 = false; std::uint32_t srgbDecodeFormats = 0; + bool storageImageWithoutFormat = false; bool depthClamp = false; bool depthBounds = false; bool depthBiasClamp = false; @@ -1117,6 +1118,7 @@ VulkanDevice::VulkanDevice(const PresentationWindow* window) : state(std::make_u enabled.shaderClipDistance = available.shaderClipDistance; if (enabled.shaderStorageImageWriteWithoutFormat) state->capabilities.push_back(spv::CapabilityStorageImageWriteWithoutFormat); if (enabled.shaderStorageImageReadWithoutFormat) state->capabilities.push_back(spv::CapabilityStorageImageReadWithoutFormat); + state->storageImageWithoutFormat = enabled.shaderStorageImageWriteWithoutFormat == VK_TRUE && enabled.shaderStorageImageReadWithoutFormat == VK_TRUE; // Bindless image tables index an image array with a wave-uniform runtime slot. enabled.shaderSampledImageArrayDynamicIndexing = available.shaderSampledImageArrayDynamicIndexing; enabled.shaderStorageImageArrayDynamicIndexing = available.shaderStorageImageArrayDynamicIndexing; @@ -2729,6 +2731,7 @@ Graphics::Context VulkanDevice::buildContext() const { context.pipelineExecutableInfo = state->pipelineExecutableInfo; context.srgbDecodeFormats = state->srgbDecodeFormats; context.memoryProperties2 = state->memoryBudget ? state->InstanceFunction("vkGetPhysicalDeviceMemoryProperties2") : nullptr; + context.singlePassStorage = state->storageImageWithoutFormat; return context; } diff --git a/core/libs/prx/libSceAgcDriver/Graphics/include/Context.hpp b/core/libs/prx/libSceAgcDriver/Graphics/include/Context.hpp index b74efb979..a948e08fb 100644 --- a/core/libs/prx/libSceAgcDriver/Graphics/include/Context.hpp +++ b/core/libs/prx/libSceAgcDriver/Graphics/include/Context.hpp @@ -165,6 +165,7 @@ struct Context { std::uint32_t srgbDecodeFormats = 0; bool provokingVertexLast = false; bool provokingVertexModePerPipeline = false; + bool singlePassStorage = false; template TFunction Function(const char* name) const { diff --git a/core/libs/prx/libSceAgcDriver/Graphics/include/Texture.hpp b/core/libs/prx/libSceAgcDriver/Graphics/include/Texture.hpp index 2fa7b67e5..a1f98258b 100644 --- a/core/libs/prx/libSceAgcDriver/Graphics/include/Texture.hpp +++ b/core/libs/prx/libSceAgcDriver/Graphics/include/Texture.hpp @@ -295,6 +295,7 @@ public: // was still current (nothing uploaded). // Keeps the image current with guest memory (see GuestMemory::CollectWrites). bool Refresh(); + static std::uint64_t SinglePassMoves(); std::uint64_t GuestBytes() const; VkDeviceSize AllocationBytes() const { return memoryBytes; } @@ -430,6 +431,10 @@ private: bool keysFillMatches(std::uint64_t address, std::size_t bytes, bool overlapped) const; bool pendingUnitInside(std::uint64_t address, std::size_t bytes) const; VkImageView createView(std::uint32_t mip, bool firstLayer, VkFormat format) const; + bool singlePass(); + VkImageView elementLayerView(std::uint32_t level, std::uint32_t layer); + void recordDirectUploadBarrier(VkCommandBuffer commands, bool discard); + void recordDirectUploadDone(VkCommandBuffer commands); void release() noexcept; Context context; @@ -483,6 +488,8 @@ private: std::map, VkImageView> atomicViews; std::map, VkImageView> uintViews; VkImageView elementView = VK_NULL_HANDLE; + std::map elementLayerViews; + std::int8_t singlePassState = -1; bool attachable = false; std::map, VkImageView> attachmentViews; VkImage proxyImage = VK_NULL_HANDLE; diff --git a/core/libs/prx/libSceAgcDriver/Graphics/include/TextureDetiler.hpp b/core/libs/prx/libSceAgcDriver/Graphics/include/TextureDetiler.hpp index d5af352ab..2efc3bc46 100644 --- a/core/libs/prx/libSceAgcDriver/Graphics/include/TextureDetiler.hpp +++ b/core/libs/prx/libSceAgcDriver/Graphics/include/TextureDetiler.hpp @@ -48,15 +48,18 @@ namespace AgcDriver::Graphics { // restricts the move to part of the mip (a partial upload or write-back, StorageTexture), // leaving the other elements of both buffers alone. void Dispatch(VkCommandBuffer commands, TextureTileMode tileMode, std::uint32_t elementBytes, VkBuffer source, std::uint64_t sourceOffset, VkBuffer destination, std::uint64_t destinationOffset, const TileMipLayout& layout, bool retile = false, std::uint32_t slice = 0, bool thick = false, const DetileWindow& window = {}); + void DispatchImage(VkCommandBuffer commands, TextureTileMode tileMode, std::uint32_t elementBytes, VkBuffer tiled, std::uint64_t tiledOffset, VkImageView view, const TileMipLayout& layout, bool retile = false, std::uint32_t slice = 0, const DetileWindow& window = {}); + static VkFormat ImageElementFormat(std::uint32_t elementBytes); // Recycles the descriptor sets of the previous batch; call before recording a new command batch. void BeginBatch(); void DispatchCmaskClear(VkCommandBuffer commands, VkBuffer cmask, std::uint64_t cmaskOffset, std::size_t cmaskBytes, VkImageView view, std::uint32_t width, std::uint32_t height, std::uint32_t elementBytes, const std::array& clearWords, bool write); std::uint32_t CmaskErrors(); private: - VkPipeline pipeline(TextureTileMode tileMode, std::uint32_t elementBytes, bool retile, bool thick); + VkPipeline pipeline(TextureTileMode tileMode, std::uint32_t elementBytes, bool retile, bool thick, bool image = false); void release() noexcept; - VkDescriptorSet allocateSet(); + VkDescriptorSet allocateSet(bool image = false); + void checkWindow(const TileMipLayout& layout, const DetileWindow& window, std::uint32_t& columnEnd, std::uint32_t& rowEnd) const; const Context context; VkDescriptorSetLayout descriptorLayout = VK_NULL_HANDLE; @@ -71,6 +74,11 @@ namespace AgcDriver::Graphics { std::vector cmaskDescriptorPools; std::size_t allocatedCmaskSets = 0; std::unique_ptr cmaskErrors; + VkDescriptorSetLayout imageDescriptorLayout = VK_NULL_HANDLE; + VkPipelineLayout imagePipelineLayout = VK_NULL_HANDLE; + VkShaderModule imageModule = VK_NULL_HANDLE; + std::vector imageDescriptorPools; + std::size_t allocatedImageSets = 0; }; } diff --git a/core/libs/prx/libSceAgcDriver/Graphics/shaders/TextureDetile.comp b/core/libs/prx/libSceAgcDriver/Graphics/shaders/TextureDetile.comp index 575edeb25..fcff04497 100644 --- a/core/libs/prx/libSceAgcDriver/Graphics/shaders/TextureDetile.comp +++ b/core/libs/prx/libSceAgcDriver/Graphics/shaders/TextureDetile.comp @@ -1,4 +1,7 @@ #version 450 +#ifdef LINEAR_IMAGE +#extension GL_EXT_shader_image_load_formatted : require +#endif layout(constant_id = 0) const uint ELEMENT_BYTES = 4u; layout(constant_id = 1) const uint BLOCK_BYTES = 256u; @@ -29,8 +32,13 @@ layout(constant_id = 21) const uint BLOCK_HEIGHT = 0u; layout(local_size_x = 8, local_size_y = 8) in; +#ifdef LINEAR_IMAGE +layout(set = 0, binding = 0, std430) buffer Tiled { uint data[]; } tiledBuffer; +layout(set = 0, binding = 1) uniform uimage2D linearImage; +#else layout(set = 0, binding = 0, std430) readonly buffer Input { uint data[]; } inputBuffer; layout(set = 0, binding = 1, std430) buffer Output { uint data[]; } outputBuffer; +#endif layout(push_constant) uniform Push { uint srcBase; @@ -142,6 +150,43 @@ uvec2 blockExtent() { return uvec2(width, BLOCK_BYTES / (width * ELEMENT_BYTES)); } +#ifdef LINEAR_IMAGE +void storeElement(uint src, uvec2 p) { + uvec4 value = uvec4(0u); + uint word = src >> 2; + if (ELEMENT_BYTES == 1u) { + value.x = (tiledBuffer.data[word] >> ((src & 3u) * 8u)) & 0xffu; + } else if (ELEMENT_BYTES == 2u) { + value.x = (tiledBuffer.data[word] >> ((src & 3u) * 8u)) & 0xffffu; + } else { + value.x = tiledBuffer.data[word]; + if (ELEMENT_BYTES >= 8u) value.y = tiledBuffer.data[word + 1u]; + if (ELEMENT_BYTES >= 16u) { + value.z = tiledBuffer.data[word + 2u]; + value.w = tiledBuffer.data[word + 3u]; + } + } + imageStore(linearImage, ivec2(p), value); +} + +void loadElement(uvec2 p, uint dst) { + uvec4 value = imageLoad(linearImage, ivec2(p)); + uint word = dst >> 2; + if (ELEMENT_BYTES >= 4u) { + tiledBuffer.data[word] = value.x; + if (ELEMENT_BYTES >= 8u) tiledBuffer.data[word + 1u] = value.y; + if (ELEMENT_BYTES >= 16u) { + tiledBuffer.data[word + 2u] = value.z; + tiledBuffer.data[word + 3u] = value.w; + } + } else { + uint mask = ELEMENT_BYTES == 1u ? 0xffu : 0xffffu; + uint shift = (dst & 3u) * 8u; + atomicAnd(tiledBuffer.data[word], ~(mask << shift)); + atomicOr(tiledBuffer.data[word], (value.x & mask) << shift); + } +} +#else void copyElement(uint src, uint dst) { if (ELEMENT_BYTES >= 4u) { for (uint i = 0u; i < ELEMENT_BYTES; i += 4u) { @@ -155,6 +200,7 @@ void copyElement(uint src, uint dst) { atomicOr(outputBuffer.data[dst >> 2], value << shift); } } +#endif void main() { uvec2 p = gl_GlobalInvocationID.xy + uvec2(params.columnBegin, params.rowBegin); @@ -170,11 +216,19 @@ void main() { if (linearOffset < params.rangeBegin || linearOffset >= params.rangeEnd) { return; } +#ifdef LINEAR_IMAGE + if (RETILE != 0u) { + loadElement(p, params.dstBase + linearOffset - params.tiledBase); + } else { + storeElement(params.srcBase + linearOffset - params.tiledBase, p); + } +#else if (RETILE != 0u) { copyElement(params.srcBase + linearOffset - params.linearBase, params.dstBase + linearOffset - params.tiledBase); } else { copyElement(params.srcBase + linearOffset - params.tiledBase, params.dstBase + linearOffset - params.linearBase); } +#endif return; } @@ -194,10 +248,18 @@ void main() { tiledOffset -= params.tiledBase; linearOffset -= params.linearBase; +#ifdef LINEAR_IMAGE + if (RETILE != 0u) { + loadElement(p, params.dstBase + tiledOffset); + } else { + storeElement(params.srcBase + tiledOffset, p); + } +#else if (RETILE != 0u) { copyElement(params.srcBase + linearOffset, params.dstBase + tiledOffset); } else { copyElement(params.srcBase + tiledOffset, params.dstBase + linearOffset); } +#endif } diff --git a/core/libs/prx/libSceAgcDriver/Graphics/src/Texture.cpp b/core/libs/prx/libSceAgcDriver/Graphics/src/Texture.cpp index 71720b8b5..e538a7bd2 100644 --- a/core/libs/prx/libSceAgcDriver/Graphics/src/Texture.cpp +++ b/core/libs/prx/libSceAgcDriver/Graphics/src/Texture.cpp @@ -77,6 +77,7 @@ thread_local const char* uploadReason = "first"; // selections widened to every pending unit (the hysteresis in writeBack), units dropped as dead // (DiscardPendingInside) and units an alias's own results superseded (markLayersPending); relaxed, // reported only. +std::atomic singlePassMoves{0}; std::atomic partialUploads{0}, partialUploadBytes{0}, partialWriteBacks{0}, partialWriteBackBytes{0}, emptyWriteBacks{0}, coalescedWriteBacks{0}, unitsDropped{0}, unitsSuperseded{0}; // FlushPending listings skipped because no pending unit of the image lay inside the range, images // whose selected pending units were dropped because their memory is no longer registered (see @@ -1107,6 +1108,39 @@ VkImageView StorageTexture::createView(std::uint32_t mip, bool firstLayer, VkFor return created; } +bool StorageTexture::singlePass() { + if (singlePassState < 0) { + const auto elementBytes = BytesPerElement(descriptor.format); + const auto format = TextureDetiler::ImageElementFormat(elementBytes); + bool eligible = context.singlePassStorage && format != VK_FORMAT_UNDEFINED && BlockWidth(descriptor.format) == 1 && !geometry.thick && geometry.imageDepth == 1 && (descriptor.dimension == TextureDimension::k2D || descriptor.dimension == TextureDimension::k2DArray || descriptor.dimension == TextureDimension::kCube); + if (eligible) { + VkFormatProperties properties{}; + context.formatProperties(context.physical, format, &properties); + eligible = (properties.optimalTilingFeatures & VK_FORMAT_FEATURE_STORAGE_IMAGE_BIT) != 0; + } + singlePassState = eligible ? 1 : 0; + } + return singlePassState == 1; +} + +VkImageView StorageTexture::elementLayerView(std::uint32_t level, std::uint32_t layer) { + const auto key = (level << 16) | layer; + if (const auto found = elementLayerViews.find(key); found != elementLayerViews.end()) return found->second; + VkImageViewUsageCreateInfo usage{VK_STRUCTURE_TYPE_IMAGE_VIEW_USAGE_CREATE_INFO}; + usage.usage = VK_IMAGE_USAGE_STORAGE_BIT; + VkImageViewCreateInfo viewInfo{VK_STRUCTURE_TYPE_IMAGE_VIEW_CREATE_INFO}; + viewInfo.pNext = &usage; + viewInfo.image = image; + viewInfo.viewType = VK_IMAGE_VIEW_TYPE_2D; + viewInfo.format = TextureDetiler::ImageElementFormat(BytesPerElement(descriptor.format)); + viewInfo.components = {VK_COMPONENT_SWIZZLE_IDENTITY, VK_COMPONENT_SWIZZLE_IDENTITY, VK_COMPONENT_SWIZZLE_IDENTITY, VK_COMPONENT_SWIZZLE_IDENTITY}; + viewInfo.subresourceRange = {VK_IMAGE_ASPECT_COLOR_BIT, level, 1u, geometry.CopyLayer(layer), 1u}; + VkImageView created = VK_NULL_HANDLE; + Check(context.Function("vkCreateImageView")(context.device, &viewInfo, nullptr, &created), "vkCreateImageView storage element"); + elementLayerViews.emplace(key, created); + return created; +} + VkImageView StorageTexture::AttachmentView(VkFormat format, std::uint32_t mip, std::uint32_t depthSlice) { Require(attachable, "storage image cannot be a color attachment"); Require(mip < descriptor.mipCount, "attachment mip exceeds the storage image"); @@ -1566,6 +1600,10 @@ DccKeys StorageTexture::ProvedKeys() const { return *keys; } +std::uint64_t StorageTexture::SinglePassMoves() { + return singlePassMoves.load(std::memory_order_relaxed); +} + bool StorageTexture::ServesKeysAt(std::uint64_t dccAddress) const { const bool locked = GuestMemory::GpuMutex().HeldByThisThread(); const auto readFollowed = [&] { @@ -1771,7 +1809,8 @@ void StorageTexture::upload(const std::vector* layers) { if ((*layers)[layer]) uploadedBytes += trackedLayerBytes; } } - auto linear = std::make_shared(context, static_cast(linearBytes), VK_BUFFER_USAGE_STORAGE_BUFFER_BIT | VK_BUFFER_USAGE_TRANSFER_SRC_BIT); + const bool direct = singlePass(); + auto linear = direct ? nullptr : std::make_shared(context, static_cast(linearBytes), VK_BUFFER_USAGE_STORAGE_BUFFER_BIT | VK_BUFFER_USAGE_TRANSFER_SRC_BIT); detiler.BeginBatch(); auto* recorder = Recorder::Active(); std::unique_ptr batch; @@ -1782,12 +1821,12 @@ void StorageTexture::upload(const std::vector* layers) { recorder->FlushStoresOverlapping(descriptor.baseAddress, static_cast(guestBytes)); commands = recorder->Commands(); timing = recorder->BeginGpuTiming(Recorder::CommandClass::StorageUpload); - recorder->Keep(linear, linear->Size()); + if (linear) recorder->Keep(linear, linear->Size()); // The image itself must outlive the recorded copy: the cache may evict it right after. if (auto self = weak_from_this().lock()) recorder->Keep(std::move(self)); // The detile reads the tiled bytes from the import when the batch runs. recorder->NotePendingRead(descriptor.baseAddress, static_cast(guestBytes), Recorder::ReadKind::StorageUpload); - Recorder::CountBarriers(Recorder::CommandClass::StorageUpload, 3); + Recorder::CountBarriers(Recorder::CommandClass::StorageUpload, direct ? 2 : 3); if (Recorder::BarrierValidate()) { const std::pair read{descriptor.baseAddress, descriptor.baseAddress + guestBytes}; const std::pair written{image, true}; @@ -1798,6 +1837,26 @@ void StorageTexture::upload(const std::vector* layers) { commands = batch->Handle(); } const auto importOffset = descriptor.baseAddress - import->base; + if (direct) { + recordDirectUploadBarrier(commands, layers == nullptr); + for (std::uint32_t layer = 0; layer < arrayLayers; ++layer) { + if (layers != nullptr && !(*layers)[layer]) continue; + for (std::uint32_t level = 0; level < mips.size(); ++level) { + if (!geometry.HasLayer(level, layer)) continue; + const auto& mip = mips[level]; + detiler.DispatchImage(commands, descriptor.tileMode, elementBytes, import->buffer, importOffset + geometry.GuestLayerOffset(layer) + mip.tiledOffset, elementLayerView(level, layer), mip, false, layer, {.pipeBankXor = descriptor.pipeBankXor}); + } + } + recordDirectUploadDone(commands); + singlePassMoves.fetch_add(1, std::memory_order_relaxed); + if (batch) batch->SubmitAndWait(); + else recorder->EndGpuTiming(timing, uploadedBytes); + countStorageUpload(1, uploadedBytes); + ++Profile().storageDirectUploads; + ++version; + if (profile) LookupOutcomes::Add(LookupOutcomes::UploadDirect, start); + return; + } RecordMemoryBarrier(context, commands, VK_PIPELINE_STAGE_ALL_COMMANDS_BIT | VK_PIPELINE_STAGE_HOST_BIT, VK_PIPELINE_STAGE_COMPUTE_SHADER_BIT, VK_ACCESS_SHADER_WRITE_BIT | VK_ACCESS_TRANSFER_WRITE_BIT | VK_ACCESS_HOST_WRITE_BIT, VK_ACCESS_SHADER_READ_BIT | VK_ACCESS_SHADER_WRITE_BIT); for (std::uint32_t layer = 0; layer < arrayLayers; ++layer) { if (layers != nullptr && !(*layers)[layer]) continue; @@ -2055,6 +2114,33 @@ StorageTexture::PaddingSeeds StorageTexture::paddingSeeds(const HostImport& impo return seeds; } +void StorageTexture::recordDirectUploadBarrier(VkCommandBuffer commands, bool discard) { + const VkMemoryBarrier sourceReady{VK_STRUCTURE_TYPE_MEMORY_BARRIER, nullptr, VK_ACCESS_SHADER_WRITE_BIT | VK_ACCESS_TRANSFER_WRITE_BIT | VK_ACCESS_HOST_WRITE_BIT, VK_ACCESS_SHADER_READ_BIT | VK_ACCESS_SHADER_WRITE_BIT}; + VkImageMemoryBarrier toWrite{VK_STRUCTURE_TYPE_IMAGE_MEMORY_BARRIER}; + toWrite.srcAccessMask = discard ? 0u : VK_ACCESS_SHADER_WRITE_BIT | VK_ACCESS_SHADER_READ_BIT | VK_ACCESS_TRANSFER_WRITE_BIT | VK_ACCESS_COLOR_ATTACHMENT_WRITE_BIT; + toWrite.dstAccessMask = VK_ACCESS_SHADER_WRITE_BIT; + toWrite.oldLayout = discard ? VK_IMAGE_LAYOUT_UNDEFINED : VK_IMAGE_LAYOUT_GENERAL; + toWrite.newLayout = VK_IMAGE_LAYOUT_GENERAL; + toWrite.srcQueueFamilyIndex = VK_QUEUE_FAMILY_IGNORED; + toWrite.dstQueueFamilyIndex = VK_QUEUE_FAMILY_IGNORED; + toWrite.image = image; + toWrite.subresourceRange = {VK_IMAGE_ASPECT_COLOR_BIT, 0, descriptor.mipCount, 0, geometry.imageLayers}; + context.Function("vkCmdPipelineBarrier")(commands, VK_PIPELINE_STAGE_ALL_COMMANDS_BIT | VK_PIPELINE_STAGE_HOST_BIT, VK_PIPELINE_STAGE_COMPUTE_SHADER_BIT, 0, 1, &sourceReady, 0, nullptr, 1, &toWrite); +} + +void StorageTexture::recordDirectUploadDone(VkCommandBuffer commands) { + VkImageMemoryBarrier written{VK_STRUCTURE_TYPE_IMAGE_MEMORY_BARRIER}; + written.srcAccessMask = VK_ACCESS_SHADER_WRITE_BIT; + written.dstAccessMask = VK_ACCESS_SHADER_READ_BIT | VK_ACCESS_SHADER_WRITE_BIT | VK_ACCESS_TRANSFER_READ_BIT | VK_ACCESS_TRANSFER_WRITE_BIT | VK_ACCESS_COLOR_ATTACHMENT_READ_BIT | VK_ACCESS_COLOR_ATTACHMENT_WRITE_BIT; + written.oldLayout = VK_IMAGE_LAYOUT_GENERAL; + written.newLayout = VK_IMAGE_LAYOUT_GENERAL; + written.srcQueueFamilyIndex = VK_QUEUE_FAMILY_IGNORED; + written.dstQueueFamilyIndex = VK_QUEUE_FAMILY_IGNORED; + written.image = image; + written.subresourceRange = {VK_IMAGE_ASPECT_COLOR_BIT, 0, descriptor.mipCount, 0, geometry.imageLayers}; + context.Function("vkCmdPipelineBarrier")(commands, VK_PIPELINE_STAGE_COMPUTE_SHADER_BIT, VK_PIPELINE_STAGE_ALL_COMMANDS_BIT, 0, 0, nullptr, 0, nullptr, 1, &written); +} + std::uint64_t StorageTexture::uploadWindows(const HostImport& import, std::span> runs, bool discard) { const auto elementBytes = BytesPerElement(descriptor.format); // Each run's pieces by source (a unit shadow's slab while fresh, else the import), the tail @@ -2094,7 +2180,8 @@ std::uint64_t StorageTexture::uploadWindows(const HostImport& import, std::span< linearTotal += (window.linearBytes + 255) & ~std::uint64_t{255}; } for (const auto& [begin, end] : runs) uploadedBytes += end - begin; - auto linear = std::make_shared(context, static_cast(linearTotal), VK_BUFFER_USAGE_STORAGE_BUFFER_BIT | VK_BUFFER_USAGE_TRANSFER_SRC_BIT); + const bool direct = singlePass(); + auto linear = direct ? nullptr : std::make_shared(context, static_cast(linearTotal), VK_BUFFER_USAGE_STORAGE_BUFFER_BIT | VK_BUFFER_USAGE_TRANSFER_SRC_BIT); detiler.BeginBatch(); auto* recorder = Recorder::Active(); std::unique_ptr batch; @@ -2111,13 +2198,13 @@ std::uint64_t StorageTexture::uploadWindows(const HostImport& import, std::span< for (const auto& [begin, end] : reads) recorder->FlushStoresOverlapping(begin, static_cast(end - begin)); commands = recorder->Commands(); timing = recorder->BeginGpuTiming(Recorder::CommandClass::StorageUpload); - recorder->Keep(linear, linear->Size()); + if (linear) recorder->Keep(linear, linear->Size()); for (const auto& source : sources) { if (source.slab != nullptr) recorder->Keep(source.slab); } if (auto self = weak_from_this().lock()) recorder->Keep(std::move(self)); if (!reads.empty()) recorder->NotePendingReads(reads, Recorder::ReadKind::StorageUpload); - Recorder::CountBarriers(Recorder::CommandClass::StorageUpload, 3); + Recorder::CountBarriers(Recorder::CommandClass::StorageUpload, direct ? 2 : 3); if (Recorder::BarrierValidate()) { const std::pair written{image, true}; recorder->NoteAccess(Recorder::CommandClass::StorageUpload, Recorder::Access{reads, {}, std::span(&written, 1), VK_PIPELINE_STAGE_COMPUTE_SHADER_BIT | VK_PIPELINE_STAGE_TRANSFER_BIT}); @@ -2127,6 +2214,27 @@ std::uint64_t StorageTexture::uploadWindows(const HostImport& import, std::span< commands = batch->Handle(); } const auto importOffset = descriptor.baseAddress - import.base; + if (direct) { + recordDirectUploadBarrier(commands, discard); + for (const auto& window : windows) { + const auto& mip = mips[window.level]; + const auto& source = sourceOf(window); + auto detile = window.window; + detile.pipeBankXor = descriptor.pipeBankXor; + const auto view = elementLayerView(window.level, window.layer); + if (source.shadow) { + detiler.DispatchImage(commands, descriptor.tileMode, elementBytes, source.buffer, source.offset + (window.tiledBegin - source.begin), view, mip, false, window.layer, detile); + } else { + detile.tiledBase = 0; + detiler.DispatchImage(commands, descriptor.tileMode, elementBytes, import.buffer, importOffset + geometry.GuestLayerOffset(window.layer) + mip.tiledOffset, view, mip, false, window.layer, detile); + } + } + recordDirectUploadDone(commands); + singlePassMoves.fetch_add(1, std::memory_order_relaxed); + if (batch) batch->SubmitAndWait(); + else recorder->EndGpuTiming(timing, uploadedBytes); + return uploadedBytes; + } RecordMemoryBarrier(context, commands, VK_PIPELINE_STAGE_ALL_COMMANDS_BIT | VK_PIPELINE_STAGE_HOST_BIT, VK_PIPELINE_STAGE_COMPUTE_SHADER_BIT, VK_ACCESS_SHADER_WRITE_BIT | VK_ACCESS_TRANSFER_WRITE_BIT | VK_ACCESS_HOST_WRITE_BIT, VK_ACCESS_SHADER_READ_BIT | VK_ACCESS_SHADER_WRITE_BIT); std::vector regions; for (std::size_t i = 0; i < windows.size(); ++i) { @@ -2265,7 +2373,8 @@ std::uint64_t StorageTexture::writeBackWindows(const HostImport& import, std::sp flushBegin = std::min(flushBegin, seed.begin); flushEnd = std::max(flushEnd, seed.end); } - auto linear = std::make_shared(context, static_cast(linearTotal), VK_BUFFER_USAGE_STORAGE_BUFFER_BIT | VK_BUFFER_USAGE_TRANSFER_DST_BIT); + const bool direct = singlePass(); + auto linear = direct ? nullptr : std::make_shared(context, static_cast(linearTotal), VK_BUFFER_USAGE_STORAGE_BUFFER_BIT | VK_BUFFER_USAGE_TRANSFER_DST_BIT); auto tiledScratch = std::make_shared(context, static_cast(scratchTotal), VK_BUFFER_USAGE_STORAGE_BUFFER_BIT | VK_BUFFER_USAGE_TRANSFER_SRC_BIT | VK_BUFFER_USAGE_TRANSFER_DST_BIT); detiler.BeginBatch(); auto* recorder = Recorder::Active(); @@ -2278,12 +2387,12 @@ std::uint64_t StorageTexture::writeBackWindows(const HostImport& import, std::sp recorder->FlushStoresOverlapping(flushBegin, static_cast(flushEnd - flushBegin)); commands = recorder->Commands(&covered); timing = recorder->BeginGpuTiming(Recorder::CommandClass::StorageWriteBack); - recorder->Keep(linear, linear->Size()); + if (linear) recorder->Keep(linear, linear->Size()); recorder->Keep(tiledScratch, tiledScratch->Size()); for (const auto& pieces : slabPieces) recorder->Keep(pieces.slab); for (const auto& slab : padding.slabs) recorder->Keep(slab); if (auto self = weak_from_this().lock()) recorder->Keep(std::move(self)); - Recorder::CountBarriers(Recorder::CommandClass::StorageWriteBack, 4); + Recorder::CountBarriers(Recorder::CommandClass::StorageWriteBack, direct ? 3 : 4); if (Recorder::BarrierValidate()) { const std::pair read{image, false}; std::vector> seedReads; @@ -2304,7 +2413,7 @@ std::uint64_t StorageTexture::writeBackWindows(const HostImport& import, std::sp toSource.dstQueueFamilyIndex = VK_QUEUE_FAMILY_IGNORED; toSource.image = image; toSource.subresourceRange = {VK_IMAGE_ASPECT_COLOR_BIT, 0, descriptor.mipCount, 0, geometry.imageLayers}; - context.Function("vkCmdPipelineBarrier")(commands, VK_PIPELINE_STAGE_ALL_COMMANDS_BIT, VK_PIPELINE_STAGE_TRANSFER_BIT, 0, 0, nullptr, 0, nullptr, 1, &toSource); + if (!direct) context.Function("vkCmdPipelineBarrier")(commands, VK_PIPELINE_STAGE_ALL_COMMANDS_BIT, VK_PIPELINE_STAGE_TRANSFER_BIT, 0, 0, nullptr, 0, nullptr, 1, &toSource); if (!seeds.empty() || !padding.copies.empty()) { // The seeded units' import bytes (every earlier writer of them, host stores included) // precede the seed copies; the previous command's trailing barrier may have covered that. @@ -2325,22 +2434,37 @@ std::uint64_t StorageTexture::writeBackWindows(const HostImport& import, std::sp // orders the scratch writes). for (const auto& [source, copies] : padding.copies) context.Resolved(&DeviceFunctions::cmdCopyBuffer, "vkCmdCopyBuffer")(commands, source, tiledScratch->Handle(), static_cast(copies.size()), copies.data()); } - std::vector regions; - for (std::size_t i = 0; i < windows.size(); ++i) { - for (auto region : windows[i].regions) { - region.bufferOffset += linearPositions[i]; - regions.push_back(region); - } - } - context.Function("vkCmdCopyImageToBuffer")(commands, image, VK_IMAGE_LAYOUT_TRANSFER_SRC_OPTIMAL, linear->Handle(), static_cast(regions.size()), regions.data()); - const auto linearRead = WholeBufferBarrier(linear->Handle(), VK_ACCESS_TRANSFER_WRITE_BIT, VK_ACCESS_SHADER_READ_BIT); const VkMemoryBarrier importReady{VK_STRUCTURE_TYPE_MEMORY_BARRIER, nullptr, VK_ACCESS_SHADER_READ_BIT | VK_ACCESS_SHADER_WRITE_BIT | VK_ACCESS_TRANSFER_READ_BIT | VK_ACCESS_TRANSFER_WRITE_BIT | VK_ACCESS_HOST_WRITE_BIT, VK_ACCESS_SHADER_WRITE_BIT | VK_ACCESS_SHADER_READ_BIT}; - context.Function("vkCmdPipelineBarrier")(commands, VK_PIPELINE_STAGE_ALL_COMMANDS_BIT | VK_PIPELINE_STAGE_HOST_BIT, VK_PIPELINE_STAGE_COMPUTE_SHADER_BIT, 0, 1, &importReady, 1, &linearRead, 0, nullptr); - for (std::size_t i = 0; i < windows.size(); ++i) { - const auto& window = windows[i]; - auto retile = window.window; - retile.pipeBankXor = descriptor.pipeBankXor; - detiler.Dispatch(commands, descriptor.tileMode, elementBytes, linear->Handle(), linearPositions[i], tiledScratch->Handle(), scratchPositions[i], mips[window.level], true, window.layer, false, retile); + if (direct) { + VkImageMemoryBarrier toRead = toSource; + toRead.srcAccessMask = VK_ACCESS_SHADER_WRITE_BIT | VK_ACCESS_COLOR_ATTACHMENT_WRITE_BIT | VK_ACCESS_TRANSFER_WRITE_BIT; + toRead.dstAccessMask = VK_ACCESS_SHADER_READ_BIT; + toRead.newLayout = VK_IMAGE_LAYOUT_GENERAL; + context.Function("vkCmdPipelineBarrier")(commands, VK_PIPELINE_STAGE_ALL_COMMANDS_BIT | VK_PIPELINE_STAGE_HOST_BIT, VK_PIPELINE_STAGE_COMPUTE_SHADER_BIT, 0, 1, &importReady, 0, nullptr, 1, &toRead); + for (std::size_t i = 0; i < windows.size(); ++i) { + const auto& window = windows[i]; + auto retile = window.window; + retile.pipeBankXor = descriptor.pipeBankXor; + detiler.DispatchImage(commands, descriptor.tileMode, elementBytes, tiledScratch->Handle(), scratchPositions[i], elementLayerView(window.level, window.layer), mips[window.level], true, window.layer, retile); + } + singlePassMoves.fetch_add(1, std::memory_order_relaxed); + } else { + std::vector regions; + for (std::size_t i = 0; i < windows.size(); ++i) { + for (auto region : windows[i].regions) { + region.bufferOffset += linearPositions[i]; + regions.push_back(region); + } + } + context.Function("vkCmdCopyImageToBuffer")(commands, image, VK_IMAGE_LAYOUT_TRANSFER_SRC_OPTIMAL, linear->Handle(), static_cast(regions.size()), regions.data()); + const auto linearRead = WholeBufferBarrier(linear->Handle(), VK_ACCESS_TRANSFER_WRITE_BIT, VK_ACCESS_SHADER_READ_BIT); + context.Function("vkCmdPipelineBarrier")(commands, VK_PIPELINE_STAGE_ALL_COMMANDS_BIT | VK_PIPELINE_STAGE_HOST_BIT, VK_PIPELINE_STAGE_COMPUTE_SHADER_BIT, 0, 1, &importReady, 1, &linearRead, 0, nullptr); + for (std::size_t i = 0; i < windows.size(); ++i) { + const auto& window = windows[i]; + auto retile = window.window; + retile.pipeBankXor = descriptor.pipeBankXor; + detiler.Dispatch(commands, descriptor.tileMode, elementBytes, linear->Handle(), linearPositions[i], tiledScratch->Handle(), scratchPositions[i], mips[window.level], true, window.layer, false, retile); + } } { // The retiled scratch (and a seed's slab bytes, which the scratch copies overwrite in @@ -2360,7 +2484,7 @@ std::uint64_t StorageTexture::writeBackWindows(const HostImport& import, std::sp // merges its leading barrier. constexpr VkAccessFlags storedAccess = VK_ACCESS_SHADER_READ_BIT | VK_ACCESS_SHADER_WRITE_BIT | VK_ACCESS_TRANSFER_READ_BIT | VK_ACCESS_TRANSFER_WRITE_BIT | VK_ACCESS_HOST_READ_BIT; const VkMemoryBarrier stored{VK_STRUCTURE_TYPE_MEMORY_BARRIER, nullptr, VK_ACCESS_TRANSFER_WRITE_BIT | VK_ACCESS_SHADER_WRITE_BIT, storedAccess}; - context.Function("vkCmdPipelineBarrier")(commands, VK_PIPELINE_STAGE_COMPUTE_SHADER_BIT | VK_PIPELINE_STAGE_TRANSFER_BIT, VK_PIPELINE_STAGE_ALL_COMMANDS_BIT | VK_PIPELINE_STAGE_HOST_BIT, 0, 1, &stored, 0, nullptr, 1, &backToGeneral); + context.Function("vkCmdPipelineBarrier")(commands, VK_PIPELINE_STAGE_COMPUTE_SHADER_BIT | VK_PIPELINE_STAGE_TRANSFER_BIT, VK_PIPELINE_STAGE_ALL_COMMANDS_BIT | VK_PIPELINE_STAGE_HOST_BIT, 0, 1, &stored, 0, nullptr, direct ? 0u : 1u, &backToGeneral); if (batch) { batch->SubmitAndWait(); } else { @@ -3733,7 +3857,7 @@ void StorageTexture::writeBackLayers(const std::vector& layers) { recorder->FlushStoresOverlapping(firstStored, static_cast(lastStored - firstStored)); commands = recorder->Commands(); timing = recorder->BeginGpuTiming(Recorder::CommandClass::StorageWriteBack); - recorder->Keep(linear, linear->Size()); + if (linear) recorder->Keep(linear, linear->Size()); recorder->Keep(tiledScratch, tiledScratch->Size()); for (const auto& slab : padding.slabs) recorder->Keep(slab); // The image itself must outlive the recorded retile: the cache may evict it right after. @@ -3965,6 +4089,8 @@ void StorageTexture::release() noexcept { uintViews.clear(); if (elementView) context.Function("vkDestroyImageView")(context.device, elementView, nullptr); elementView = VK_NULL_HANDLE; + for (const auto& [key, element] : elementLayerViews) context.Function("vkDestroyImageView")(context.device, element, nullptr); + elementLayerViews.clear(); for (const auto& [format, attachment] : attachmentViews) context.Function("vkDestroyImageView")(context.device, attachment, nullptr); attachmentViews.clear(); if (proxyView) context.Function("vkDestroyImageView")(context.device, proxyView, nullptr); diff --git a/core/libs/prx/libSceAgcDriver/Graphics/src/TextureDetiler.cpp b/core/libs/prx/libSceAgcDriver/Graphics/src/TextureDetiler.cpp index eea951d96..bed858e7a 100644 --- a/core/libs/prx/libSceAgcDriver/Graphics/src/TextureDetiler.cpp +++ b/core/libs/prx/libSceAgcDriver/Graphics/src/TextureDetiler.cpp @@ -4,6 +4,7 @@ #include "prx/libSceAgcDriver/Graphics/include/TextureTiling.hpp" #include "prx/libSceAgcDriver/Graphics/shaders/TextureDetile_spv.h" #include "prx/libSceAgcDriver/Graphics/shaders/CmaskClear_spv.h" +#include "prx/libSceAgcDriver/Graphics/shaders/TextureDetileImage_spv.h" #include #include #include @@ -58,8 +59,8 @@ std::uint32_t BlockBytesFor(TextureTileMode tileMode) { throw std::runtime_error("AGC graphics: TextureDetiler encountered an unknown tile mode"); } -std::uint32_t PipelineKey(TextureTileMode tileMode, std::uint32_t elementBytes, bool retile, bool thick) { - return (thick ? 1u << 17 : 0u) | (retile ? 1u << 16 : 0u) | (static_cast(tileMode) << 8) | elementBytes; +std::uint32_t PipelineKey(TextureTileMode tileMode, std::uint32_t elementBytes, bool retile, bool thick, bool image) { + return (image ? 1u << 18 : 0u) | (thick ? 1u << 17 : 0u) | (retile ? 1u << 16 : 0u) | (static_cast(tileMode) << 8) | elementBytes; } } @@ -91,6 +92,10 @@ TextureDetiler::TextureDetiler(const Context& context) : context(context) { moduleInfo.codeSize = sizeof(TEXTURE_DETILE_SPV); moduleInfo.pCode = TEXTURE_DETILE_SPV; Check(context.Function("vkCreateShaderModule")(context.device, &moduleInfo, nullptr, &module), "vkCreateShaderModule"); + bindings[1].descriptorType = VK_DESCRIPTOR_TYPE_STORAGE_IMAGE; + Check(context.Function("vkCreateDescriptorSetLayout")(context.device, &layoutInfo, nullptr, &imageDescriptorLayout), "vkCreateDescriptorSetLayout"); + pipelineLayoutInfo.pSetLayouts = &imageDescriptorLayout; + Check(context.Function("vkCreatePipelineLayout")(context.device, &pipelineLayoutInfo, nullptr, &imagePipelineLayout), "vkCreatePipelineLayout"); } catch (...) { release(); throw; @@ -108,15 +113,30 @@ void TextureDetiler::release() noexcept { if (cmaskDescriptorLayout) context.Function("vkDestroyDescriptorSetLayout")(context.device, cmaskDescriptorLayout, nullptr); cmaskErrors.reset(); for (const auto pool : descriptorPools) context.Function("vkDestroyDescriptorPool")(context.device, pool, nullptr); + for (const auto pool : imageDescriptorPools) context.Function("vkDestroyDescriptorPool")(context.device, pool, nullptr); + if (imageModule) context.Function("vkDestroyShaderModule")(context.device, imageModule, nullptr); + if (imagePipelineLayout) context.Function("vkDestroyPipelineLayout")(context.device, imagePipelineLayout, nullptr); + if (imageDescriptorLayout) context.Function("vkDestroyDescriptorSetLayout")(context.device, imageDescriptorLayout, nullptr); for (const auto& entry : pipelines) context.Function("vkDestroyPipeline")(context.device, entry.second, nullptr); if (module) context.Function("vkDestroyShaderModule")(context.device, module, nullptr); if (pipelineLayout) context.Function("vkDestroyPipelineLayout")(context.device, pipelineLayout, nullptr); if (descriptorLayout) context.Function("vkDestroyDescriptorSetLayout")(context.device, descriptorLayout, nullptr); } -VkPipeline TextureDetiler::pipeline(TextureTileMode tileMode, std::uint32_t elementBytes, bool retile, bool thick) { +VkFormat TextureDetiler::ImageElementFormat(std::uint32_t elementBytes) { + switch (elementBytes) { + case 1: return VK_FORMAT_R8_UINT; + case 2: return VK_FORMAT_R16_UINT; + case 4: return VK_FORMAT_R32_UINT; + case 8: return VK_FORMAT_R32G32_UINT; + case 16: return VK_FORMAT_R32G32B32A32_UINT; + default: return VK_FORMAT_UNDEFINED; + } +} + +VkPipeline TextureDetiler::pipeline(TextureTileMode tileMode, std::uint32_t elementBytes, bool retile, bool thick, bool image) { Require(std::has_single_bit(elementBytes) && elementBytes <= 16u, "unsupported element size for texture detiling"); - const auto key = PipelineKey(tileMode, elementBytes, retile, thick); + const auto key = PipelineKey(tileMode, elementBytes, retile, thick, image); for (const auto& entry : pipelines) { if (entry.first == key) return entry.second; } @@ -139,6 +159,12 @@ VkPipeline TextureDetiler::pipeline(TextureTileMode tileMode, std::uint32_t elem values[2] = 2u; std::copy(equation->bits.begin(), equation->bits.end(), values.begin() + 4); } + if (image && imageModule == VK_NULL_HANDLE) { + VkShaderModuleCreateInfo moduleInfo{VK_STRUCTURE_TYPE_SHADER_MODULE_CREATE_INFO}; + moduleInfo.codeSize = sizeof(TEXTURE_DETILE_IMAGE_SPV); + moduleInfo.pCode = TEXTURE_DETILE_IMAGE_SPV; + Check(context.Function("vkCreateShaderModule")(context.device, &moduleInfo, nullptr, &imageModule), "vkCreateShaderModule"); + } std::array entries{}; for (std::uint32_t index = 0; index < entries.size(); ++index) entries[index] = {index, index * 4u, 4}; VkSpecializationInfo specialization{}; @@ -148,27 +174,32 @@ VkPipeline TextureDetiler::pipeline(TextureTileMode tileMode, std::uint32_t elem specialization.pData = values.data(); VkPipelineShaderStageCreateInfo stage{VK_STRUCTURE_TYPE_PIPELINE_SHADER_STAGE_CREATE_INFO}; stage.stage = VK_SHADER_STAGE_COMPUTE_BIT; - stage.module = module; + stage.module = image ? imageModule : module; stage.pName = "main"; stage.pSpecializationInfo = &specialization; VkComputePipelineCreateInfo createInfo{VK_STRUCTURE_TYPE_COMPUTE_PIPELINE_CREATE_INFO}; createInfo.stage = stage; - createInfo.layout = pipelineLayout; + createInfo.layout = image ? imagePipelineLayout : pipelineLayout; VkPipeline result = VK_NULL_HANDLE; Check(context.Function("vkCreateComputePipelines")(context.device, context.pipelineCache, 1, &createInfo, nullptr, &result), "vkCreateComputePipelines"); pipelines.emplace_back(key, result); return result; } +void TextureDetiler::checkWindow(const TileMipLayout& layout, const DetileWindow& window, std::uint32_t& columnEnd, std::uint32_t& rowEnd) const { + Require(layout.width != 0 && layout.height != 0, "texture detiling requires a non-empty mip layout"); + Require(layout.tiledSize != 0 && layout.linearSize != 0, "texture detiling requires a non-empty mip layout"); + columnEnd = window.columnEnd != 0 ? std::min(window.columnEnd, layout.width) : layout.width; + rowEnd = window.rowEnd != 0 ? std::min(window.rowEnd, layout.height) : layout.height; + if (!(window.columnBegin < columnEnd && window.rowBegin < rowEnd)) Require(false, "texture detiling window lies outside the mip: columns " + std::to_string(window.columnBegin) + ".." + std::to_string(columnEnd) + ", rows " + std::to_string(window.rowBegin) + ".." + std::to_string(rowEnd) + " of " + std::to_string(layout.width) + "x" + std::to_string(layout.height)); +} + void TextureDetiler::Dispatch(VkCommandBuffer commands, TextureTileMode tileMode, std::uint32_t elementBytes, VkBuffer source, std::uint64_t sourceOffset, VkBuffer destination, std::uint64_t destinationOffset, const TileMipLayout& layout, bool retile, std::uint32_t slice, bool thick, const DetileWindow& window) { Require(commands != VK_NULL_HANDLE, "texture detiling requires an active command buffer"); Require(window.rangeBegin < window.rangeEnd && window.tiledBase <= window.rangeBegin, "texture detiling window is empty"); Require(source != VK_NULL_HANDLE && destination != VK_NULL_HANDLE, "texture detiling requires source and destination buffers"); - Require(layout.width != 0 && layout.height != 0, "texture detiling requires a non-empty mip layout"); - Require(layout.tiledSize != 0 && layout.linearSize != 0, "texture detiling requires a non-empty mip layout"); - const auto columnEnd = window.columnEnd != 0 ? std::min(window.columnEnd, layout.width) : layout.width; - const auto rowEnd = window.rowEnd != 0 ? std::min(window.rowEnd, layout.height) : layout.height; - if (!(window.columnBegin < columnEnd && window.rowBegin < rowEnd)) Require(false, "texture detiling window lies outside the mip: columns " + std::to_string(window.columnBegin) + ".." + std::to_string(columnEnd) + ", rows " + std::to_string(window.rowBegin) + ".." + std::to_string(rowEnd) + " of " + std::to_string(layout.width) + "x" + std::to_string(layout.height)); + std::uint32_t columnEnd = 0, rowEnd = 0; + checkWindow(layout, window, columnEnd, rowEnd); const auto target = pipeline(tileMode, elementBytes, retile, thick); const auto alignment = std::max(context.limits.minStorageBufferOffsetAlignment, 4); const auto sourceDescriptorOffset = sourceOffset - sourceOffset % alignment; @@ -332,4 +363,63 @@ void TextureDetiler::DispatchCmaskClear(VkCommandBuffer commands, VkBuffer cmask barrier(VK_PIPELINE_STAGE_TRANSFER_BIT, VK_PIPELINE_STAGE_ALL_COMMANDS_BIT | VK_PIPELINE_STAGE_HOST_BIT, VK_ACCESS_TRANSFER_WRITE_BIT, anyAccess); } +void TextureDetiler::DispatchImage(VkCommandBuffer commands, TextureTileMode tileMode, std::uint32_t elementBytes, VkBuffer tiled, std::uint64_t tiledOffset, VkImageView view, const TileMipLayout& layout, bool retile, std::uint32_t slice, const DetileWindow& window) { + Require(commands != VK_NULL_HANDLE, "texture detiling requires an active command buffer"); + Require(window.rangeBegin < window.rangeEnd && window.tiledBase <= window.rangeBegin, "texture detiling window is empty"); + Require(tiled != VK_NULL_HANDLE && view != VK_NULL_HANDLE, "texture detiling requires a tiled buffer and an image view"); + Require(ImageElementFormat(elementBytes) != VK_FORMAT_UNDEFINED, "unsupported element size for texture detiling through an image"); + std::uint32_t columnEnd = 0, rowEnd = 0; + checkWindow(layout, window, columnEnd, rowEnd); + const auto target = pipeline(tileMode, elementBytes, retile, false, true); + const auto alignment = std::max(context.limits.minStorageBufferOffsetAlignment, 4); + const auto descriptorOffset = tiledOffset - tiledOffset % alignment; + const auto base = tiledOffset - descriptorOffset; + Require(base <= UINT32_MAX, "texture detiling buffer offset exceeds addressable range"); + const std::uint64_t tiledBytes = window.rangeEnd == 0xffffffffu ? layout.tiledSize : std::min(layout.tiledSize, window.rangeEnd) - window.tiledBase; + const auto range = (base + tiledBytes + 3) / 4 * 4; + Require(range <= context.limits.maxStorageBufferRange, "texture detiling buffer range exceeds device limits"); + const auto set = allocateSet(true); + const VkDescriptorBufferInfo bufferInfo{tiled, descriptorOffset, range}; + const VkDescriptorImageInfo imageInfo{VK_NULL_HANDLE, view, VK_IMAGE_LAYOUT_GENERAL}; + std::array writes{}; + writes[0].sType = VK_STRUCTURE_TYPE_WRITE_DESCRIPTOR_SET; + writes[0].dstSet = set; + writes[0].dstBinding = 0; + writes[0].descriptorCount = 1; + writes[0].descriptorType = VK_DESCRIPTOR_TYPE_STORAGE_BUFFER; + writes[0].pBufferInfo = &bufferInfo; + writes[1].sType = VK_STRUCTURE_TYPE_WRITE_DESCRIPTOR_SET; + writes[1].dstSet = set; + writes[1].dstBinding = 1; + writes[1].descriptorCount = 1; + writes[1].descriptorType = VK_DESCRIPTOR_TYPE_STORAGE_IMAGE; + writes[1].pImageInfo = &imageInfo; + context.Function("vkUpdateDescriptorSets")(context.device, static_cast(writes.size()), writes.data(), 0, nullptr); + context.Function("vkCmdBindPipeline")(commands, VK_PIPELINE_BIND_POINT_COMPUTE, target); + context.Function("vkCmdBindDescriptorSets")(commands, VK_PIPELINE_BIND_POINT_COMPUTE, imagePipelineLayout, 0, 1, &set, 0, nullptr); + Push push{}; + push.srcBase = static_cast(retile ? 0 : base); + push.dstBase = static_cast(retile ? base : 0); + push.width = layout.width; + push.height = layout.height; + push.pitchBytes = layout.pitchBytes; + push.blocksPerRow = layout.blocksPerRow; + push.tail = layout.tail ? 1u : 0u; + push.tailX = layout.tailX; + push.tailY = layout.tailY; + push.elementBytes = elementBytes; + push.slice = slice; + push.rangeBegin = window.rangeBegin; + push.rangeEnd = window.rangeEnd; + push.tiledBase = window.tiledBase; + push.linearBase = 0; + push.columnBegin = window.columnBegin; + push.rowBegin = window.rowBegin; + push.pipeBankXor = window.pipeBankXor; + context.Function("vkCmdPushConstants")(commands, imagePipelineLayout, VK_SHADER_STAGE_COMPUTE_BIT, 0, sizeof(Push), &push); + const auto groupsX = (columnEnd - window.columnBegin + 7u) / 8u; + const auto groupsY = (rowEnd - window.rowBegin + 7u) / 8u; + context.Function("vkCmdDispatch")(commands, groupsX, groupsY, 1); +} + } diff --git a/core/libs/prx/libSceAgcDriver/Graphics/src/TextureDetilerDescriptors.cpp b/core/libs/prx/libSceAgcDriver/Graphics/src/TextureDetilerDescriptors.cpp index cea243e2e..4d482528c 100644 --- a/core/libs/prx/libSceAgcDriver/Graphics/src/TextureDetilerDescriptors.cpp +++ b/core/libs/prx/libSceAgcDriver/Graphics/src/TextureDetilerDescriptors.cpp @@ -1,6 +1,7 @@ #include "prx/libSceAgcDriver/Graphics/include/Recorder.hpp" #include "prx/libSceAgcDriver/Graphics/include/TextureDetiler.hpp" #include "prx/libSceAgcDriver/Execution/include/GuestMemory.hpp" +#include namespace AgcDriver::Graphics { @@ -17,29 +18,37 @@ void TextureDetiler::BeginBatch() { Check(context.Function("vkResetDescriptorPool")(context.device, pool, 0), "vkResetDescriptorPool CMASK clear"); } allocatedCmaskSets = 0; + for (const auto pool : imageDescriptorPools) { + Check(context.Function("vkResetDescriptorPool")(context.device, pool, 0), "vkResetDescriptorPool texture detiler"); + } + allocatedImageSets = 0; } -VkDescriptorSet TextureDetiler::allocateSet() { +VkDescriptorSet TextureDetiler::allocateSet(bool image) { constexpr std::uint32_t setsPerPool = 64; - const auto poolIndex = allocatedSets / setsPerPool; - if (poolIndex == descriptorPools.size()) { - const VkDescriptorPoolSize size{VK_DESCRIPTOR_TYPE_STORAGE_BUFFER, setsPerPool * 2}; + auto& pools = image ? imageDescriptorPools : descriptorPools; + auto& allocated = image ? allocatedImageSets : allocatedSets; + const auto poolIndex = allocated / setsPerPool; + if (poolIndex == pools.size()) { + std::array sizes{}; + sizes[0] = {VK_DESCRIPTOR_TYPE_STORAGE_BUFFER, image ? setsPerPool : setsPerPool * 2}; + sizes[1] = {VK_DESCRIPTOR_TYPE_STORAGE_IMAGE, setsPerPool}; VkDescriptorPoolCreateInfo info{VK_STRUCTURE_TYPE_DESCRIPTOR_POOL_CREATE_INFO}; info.maxSets = setsPerPool; - info.poolSizeCount = 1; - info.pPoolSizes = &size; - descriptorPools.reserve(descriptorPools.size() + 1); + info.poolSizeCount = image ? 2u : 1u; + info.pPoolSizes = sizes.data(); + pools.reserve(pools.size() + 1); VkDescriptorPool pool = VK_NULL_HANDLE; Check(context.Function("vkCreateDescriptorPool")(context.device, &info, nullptr, &pool), "vkCreateDescriptorPool texture detiler"); - descriptorPools.push_back(pool); + pools.push_back(pool); } VkDescriptorSetAllocateInfo allocation{VK_STRUCTURE_TYPE_DESCRIPTOR_SET_ALLOCATE_INFO}; - allocation.descriptorPool = descriptorPools.at(poolIndex); + allocation.descriptorPool = pools.at(poolIndex); allocation.descriptorSetCount = 1; - allocation.pSetLayouts = &descriptorLayout; + allocation.pSetLayouts = image ? &imageDescriptorLayout : &descriptorLayout; VkDescriptorSet set = VK_NULL_HANDLE; Check(context.Function("vkAllocateDescriptorSets")(context.device, &allocation, &set), "vkAllocateDescriptorSets texture detiler"); - ++allocatedSets; + ++allocated; return set; } diff --git a/core/libs/prx/libSceAgcDriver/tests/Recorder.cpp b/core/libs/prx/libSceAgcDriver/tests/Recorder.cpp index 34dc7009f..b0d503dac 100644 --- a/core/libs/prx/libSceAgcDriver/tests/Recorder.cpp +++ b/core/libs/prx/libSceAgcDriver/tests/Recorder.cpp @@ -167,6 +167,9 @@ public: function("vkGetPhysicalDeviceFeatures")(context.physical, &supported); enabled.occlusionQueryPrecise = supported.occlusionQueryPrecise; context.occlusionQueryPrecise = supported.occlusionQueryPrecise == VK_TRUE; + enabled.shaderStorageImageReadWithoutFormat = supported.shaderStorageImageReadWithoutFormat; + enabled.shaderStorageImageWriteWithoutFormat = supported.shaderStorageImageWriteWithoutFormat; + context.singlePassStorage = supported.shaderStorageImageReadWithoutFormat == VK_TRUE && supported.shaderStorageImageWriteWithoutFormat == VK_TRUE; address.pNext = &bytes; std::vector extensionsEnabled{VK_KHR_BUFFER_DEVICE_ADDRESS_EXTENSION_NAME, VK_KHR_8BIT_STORAGE_EXTENSION_NAME}; VkPhysicalDeviceDriverProperties driverProperties{VK_STRUCTURE_TYPE_PHYSICAL_DEVICE_DRIVER_PROPERTIES}; @@ -2366,6 +2369,201 @@ void writeBackPaddingTests(const Device& device, Recorder& recorder, bool watche std::cout << "write-back padding" << (watched ? " (watched)" : "") << ": ok\n"; } +void singlePassTests(const Device& device, Recorder& recorder, bool watched) { + const auto& base = device.GetContext(); + if (base.hostImportAlignment == 0 || !base.singlePassStorage) { + std::cout << "host imports or storage access without a format unavailable: single-pass storage moves not tested\n"; + return; + } + struct Case { + const char* name; + std::uint32_t format; + TextureTileMode tileMode; + std::uint32_t width; + std::uint32_t height; + std::uint32_t mipCount; + std::uint32_t pipeBankXor = 0; + }; + constexpr Case cases[] = { + {"R8 64 KiB R_X 200x150", 1, TextureTileMode::kR64KBX, 200, 150, 1}, + {"R16 64 KiB R_X 200x150, 3 mips", 7, TextureTileMode::kR64KBX, 200, 150, 3}, + {"RGBA8 64 KiB R_X 520x260, 6 mips", 56, TextureTileMode::kR64KBX, 520, 260, 6}, + {"RGBA16F 64 KiB R_X 384x256", 71, TextureTileMode::kR64KBX, 384, 256, 1}, + {"RGBA16F 64 KiB R_X 256x200, 4 mips", 71, TextureTileMode::kR64KBX, 256, 200, 4}, + {"RGBA32F 64 KiB R_X 136x72, 2 mips", 77, TextureTileMode::kR64KBX, 136, 72, 2}, + {"RGBA8 64 KiB S 300x170", 56, TextureTileMode::kStandard64KB, 300, 170, 1}, + {"RGBA16F 4 KiB S 100x70, 2 mips", 71, TextureTileMode::kStandard4KB, 100, 70, 2}, + {"R8 256 B S 90x40", 1, TextureTileMode::kStandard256B, 90, 40, 1}, + {"RGBA8 64 KiB R_X 520x260, 2 mips, pipe/bank XOR 5", 56, TextureTileMode::kR64KBX, 520, 260, 2, 5}, + }; + std::string failures; + std::size_t tested = 0; + for (const auto& test : cases) { + if (!StorageFormatAvailable(base, test.format)) continue; + const std::string what = std::string(watched ? "(watched) " : "") + test.name; + GuestTextureResource resource{}; + resource.width = test.width; + resource.height = test.height; + resource.mipCount = test.mipCount; + resource.tileMode = test.tileMode; + resource.dimension = TextureDimension::k2D; + resource.format = test.format; + resource.pipeBankXor = test.pipeBankXor; + resource.dstSelX = 4; + resource.dstSelY = 5; + resource.dstSelZ = 6; + resource.dstSelW = 7; + const auto geometry = DescribeSurface(resource); + const auto elementBytes = BytesPerElement(test.format); + const auto bytes = (static_cast(geometry.guestBytes) + 65535) / 65536 * 65536 + 65536; + struct Result { + std::vector uploaded, memory, reuploaded; + std::uint64_t moves = 0; + }; + std::array results; + bool skipped = false; + for (int mode = 0; mode < 2 && !skipped; ++mode) { + auto context = base; + context.singlePassStorage = mode == 1; + void* block = nullptr; + if (watched) { + block = AllocateWatched(bytes, 65536); + } else { +#ifdef _WIN32 + block = VirtualAlloc(nullptr, bytes, MEM_RESERVE | MEM_COMMIT, PAGE_READWRITE); +#else + block = std::aligned_alloc(65536, bytes); +#endif + } + if (block == nullptr) { + std::cout << "no write watching: single-pass storage moves in watched memory not tested\n"; + return; + } + auto* memory = static_cast(block); + std::uint32_t seed = 0x13579bdfu; + const auto random = [&] { + seed ^= seed << 13; + seed ^= seed >> 17; + seed ^= seed << 5; + return static_cast(seed >> 9); + }; + for (std::size_t i = 0; i < bytes; ++i) memory[i] = random(); + const auto address = reinterpret_cast(block); + { + GuestAllocations::Mutation mutation; + mutation.Add(block, bytes, true, true, true); + } + struct Unregister { + const Context& context; + void* block; + std::uint64_t address; + std::size_t bytes; + bool watched; + ~Unregister() { + { + GuestAllocations::Mutation mutation; + mutation.Remove(block); + } + HostImportFor(context, address, bytes); + if (watched) ReleaseWatched(block, bytes); +#ifdef _WIN32 + else VirtualFree(block, 0, MEM_RELEASE); +#else + else std::free(block); +#endif + } + } unregister{base, block, address, bytes, watched}; + if (HostImportFor(base, address, bytes) == nullptr) { + std::cout << "host import refused: single-pass storage moves not tested\n"; + return; + } + TextureDetiler detiler(base); + context.detiler = &detiler; + resource.baseAddress = address; + std::size_t linearBytes = 0; + for (const auto& mip : geometry.mips) linearBytes += static_cast(mip.width) * mip.height * elementBytes; + const auto readImage = [&](const StorageTexture& image) { + Buffer readback(context, linearBytes, VK_BUFFER_USAGE_TRANSFER_DST_BIT); + const auto commands = recorder.Commands(); + RecordMemoryBarrier(context, commands, VK_PIPELINE_STAGE_ALL_COMMANDS_BIT, VK_PIPELINE_STAGE_TRANSFER_BIT, VK_ACCESS_SHADER_WRITE_BIT | VK_ACCESS_TRANSFER_WRITE_BIT | VK_ACCESS_COLOR_ATTACHMENT_WRITE_BIT, VK_ACCESS_TRANSFER_READ_BIT); + std::vector copies; + std::size_t at = 0; + for (std::uint32_t level = 0; level < geometry.mips.size(); ++level) { + VkBufferImageCopy copy{}; + copy.bufferOffset = at; + copy.imageSubresource = {VK_IMAGE_ASPECT_COLOR_BIT, level, 0, 1}; + copy.imageExtent = {geometry.mips[level].width, geometry.mips[level].height, 1}; + copies.push_back(copy); + at += static_cast(geometry.mips[level].width) * geometry.mips[level].height * elementBytes; + } + context.Function("vkCmdCopyImageToBuffer")(commands, image.Image(), VK_IMAGE_LAYOUT_GENERAL, readback.Handle(), static_cast(copies.size()), copies.data()); + RecordMemoryBarrier(context, commands, VK_PIPELINE_STAGE_TRANSFER_BIT, VK_PIPELINE_STAGE_ALL_COMMANDS_BIT | VK_PIPELINE_STAGE_HOST_BIT, VK_ACCESS_TRANSFER_READ_BIT | VK_ACCESS_TRANSFER_WRITE_BIT, VK_ACCESS_HOST_READ_BIT | VK_ACCESS_SHADER_READ_BIT | VK_ACCESS_SHADER_WRITE_BIT | VK_ACCESS_TRANSFER_WRITE_BIT); + recorder.Submit(); + device.WaitQueue(); + recorder.Sync(); + const auto read = readback.Bytes(); + return std::vector(reinterpret_cast(read.data()), reinterpret_cast(read.data()) + linearBytes); + }; + std::vector texels(linearBytes); + for (auto& value : texels) value = random(); + const auto movesBefore = StorageTexture::SinglePassMoves(); + { + auto image = std::make_shared(context, detiler, resource, 0); + recorder.Keep(image); + results[mode].uploaded = readImage(*image); + Buffer staging(context, linearBytes, VK_BUFFER_USAGE_TRANSFER_SRC_BIT); + std::memcpy(staging.Bytes().data(), texels.data(), linearBytes); + const auto commands = recorder.Commands(); + RecordMemoryBarrier(context, commands, VK_PIPELINE_STAGE_ALL_COMMANDS_BIT, VK_PIPELINE_STAGE_TRANSFER_BIT, VK_ACCESS_SHADER_WRITE_BIT | VK_ACCESS_TRANSFER_WRITE_BIT | VK_ACCESS_COLOR_ATTACHMENT_WRITE_BIT, VK_ACCESS_TRANSFER_WRITE_BIT); + std::vector copies; + std::size_t at = 0; + for (std::uint32_t level = 0; level < geometry.mips.size(); ++level) { + VkBufferImageCopy copy{}; + copy.bufferOffset = at; + copy.imageSubresource = {VK_IMAGE_ASPECT_COLOR_BIT, level, 0, 1}; + copy.imageExtent = {geometry.mips[level].width, geometry.mips[level].height, 1}; + copies.push_back(copy); + at += static_cast(geometry.mips[level].width) * geometry.mips[level].height * elementBytes; + } + context.Function("vkCmdCopyBufferToImage")(commands, staging.Handle(), image->Image(), VK_IMAGE_LAYOUT_GENERAL, static_cast(copies.size()), copies.data()); + RecordMemoryBarrier(context, commands, VK_PIPELINE_STAGE_TRANSFER_BIT, VK_PIPELINE_STAGE_ALL_COMMANDS_BIT, VK_ACCESS_TRANSFER_WRITE_BIT, VK_ACCESS_SHADER_READ_BIT | VK_ACCESS_SHADER_WRITE_BIT | VK_ACCESS_TRANSFER_READ_BIT | VK_ACCESS_TRANSFER_WRITE_BIT); + image->MarkDirty(); + image->WriteBack(); + recorder.Submit(); + device.WaitQueue(); + recorder.Sync(); + auto again = std::make_shared(context, detiler, resource, 0); + recorder.Keep(again); + results[mode].reuploaded = readImage(*again); + std::vector read(bytes); + if (watched) AgcDriver::GuestMemory::Read(address, read); + else { + StorageTexture::FlushPending(address, bytes, nullptr, "test", PublishScope::Whole); + recorder.Submit(); + device.WaitQueue(); + recorder.Sync(); + std::memcpy(read.data(), memory, bytes); + } + results[mode].memory.assign(reinterpret_cast(read.data()), reinterpret_cast(read.data()) + bytes); + } + recorder.Sync(); + results[mode].moves = StorageTexture::SinglePassMoves() - movesBefore; + if (results[mode].reuploaded != texels) failures += what + (mode == 1 ? " (single pass)" : " (copies)") + ": the second image does not hold the texels written back\n"; + } + if (results[0].moves != 0 || results[1].moves < 3) failures += what + ": single-pass moves " + std::to_string(results[0].moves) + " with the copies and " + std::to_string(results[1].moves) + " in single pass (want 0 and at least 3)\n"; + if (results[0].uploaded != results[1].uploaded) failures += what + ": the first upload differs between the copies and single pass\n"; + if (results[0].memory != results[1].memory) { + std::size_t differ = 0; + for (std::size_t i = 0; i < results[0].memory.size(); ++i) differ += results[0].memory[i] != results[1].memory[i]; + failures += what + ": guest memory after the write-back differs in " + std::to_string(differ) + " bytes between the copies and single pass\n"; + } + ++tested; + } + if (!failures.empty()) throw std::runtime_error("single-pass storage moves:\n" + failures); + std::cout << "single-pass storage moves" << (watched ? " (watched)" : "") << ": ok (" << tested << " surfaces)\n"; +} + + void unchangedCpuStampTests(const Device& device, Recorder& recorder) { const auto& base = device.GetContext(); constexpr std::uint32_t side = 256; @@ -3906,6 +4104,8 @@ int main(int argc, char** argv) { storageRefreshTests(device, recorder, true); writeBackPaddingTests(device, recorder, false); writeBackPaddingTests(device, recorder, true); + singlePassTests(device, recorder, false); + singlePassTests(device, recorder, true); unchangedCpuStampTests(device, recorder); importWatchTests(device); staleGenerationTests(device, recorder);