diff --git a/core/libs/prx/libSceAgcDriver/CMakeLists.txt b/core/libs/prx/libSceAgcDriver/CMakeLists.txt index 9e57ff47e..9c492e71b 100644 --- a/core/libs/prx/libSceAgcDriver/CMakeLists.txt +++ b/core/libs/prx/libSceAgcDriver/CMakeLists.txt @@ -59,6 +59,19 @@ add_custom_command( VERBATIM ) +set(agcCmaskClearShaderSource ${CMAKE_CURRENT_SOURCE_DIR}/Graphics/shaders/CmaskClear.comp) +set(agcCmaskClearShaderSpv ${agcTextureDetileGeneratedDir}/CmaskClear.spv) +set(agcCmaskClearShaderHeader ${agcTextureDetileGeneratedDir}/CmaskClear_spv.h) +add_custom_command( + OUTPUT "${agcCmaskClearShaderHeader}" + COMMAND ${CMAKE_COMMAND} -E make_directory "${agcTextureDetileGeneratedDir}" + COMMAND "$" -V --target-env vulkan1.1 -o "${agcCmaskClearShaderSpv}" "${agcCmaskClearShaderSource}" + COMMAND ${CMAKE_COMMAND} -DINPUT=${agcCmaskClearShaderSpv} -DOUTPUT=${agcCmaskClearShaderHeader} + -DSYMBOL=CMASK_CLEAR_SPV -P "${CMAKE_CURRENT_SOURCE_DIR}/embed_spirv.cmake" + DEPENDS "${agcCmaskClearShaderSource}" glslang-standalone + VERBATIM +) + set(agcSampleCounterShaderSource ${CMAKE_CURRENT_SOURCE_DIR}/Graphics/shaders/SampleCounter.comp) set(agcSampleCounterShaderSpv ${agcTextureDetileGeneratedDir}/SampleCounter.spv) set(agcSampleCounterShaderHeader ${agcTextureDetileGeneratedDir}/SampleCounter_spv.h) @@ -171,6 +184,7 @@ set(agcDriverSources ${CMAKE_CURRENT_SOURCE_DIR}/Graphics/src/UnitShadow.cpp ${CMAKE_CURRENT_SOURCE_DIR}/Graphics/src/Sampler.cpp ${agcTextureDetileShaderHeader} + ${agcCmaskClearShaderHeader} ${agcMeshArgumentsShaderHeader} ${agcSampleCounterShaderHeader} ${agcSampleDumpsShaderHeader} @@ -546,6 +560,27 @@ foreach(agcCoverStage IN ITEMS vert frag) list(APPEND agcCoverShaderHeaders ${agcCoverShaderHeader}) endforeach() +add_test_executable(agc_driver_cmask_fast_clear_tests EXCLUDE_FROM_ALL + tests/CmaskFastClear.cpp + ${agcCoverShaderHeaders} +) + +target_include_directories(agc_driver_cmask_fast_clear_tests PRIVATE + ${LIBS_INCLUDE_DIR} + ${agcTestShaderGeneratedRoot} + ${CMAKE_SOURCE_DIR}/core/shader/recompiler + ${CMAKE_SOURCE_DIR}/3rdparty/Vulkan-Headers/include +) + +target_link_libraries(agc_driver_cmask_fast_clear_tests PRIVATE + libSceAgcDriver + SDL2-static + libc +) + +set_target_properties(agc_driver_cmask_fast_clear_tests PROPERTIES CXX_EXTENSIONS OFF) +configure_windows_unwind(agc_driver_cmask_fast_clear_tests) + add_test_executable(agc_driver_recorder_tests EXCLUDE_FROM_ALL tests/Recorder.cpp tests/ResidentPresent.cpp @@ -649,6 +684,7 @@ add_test_executable(agc_driver_graphics_tests EXCLUDE_FROM_ALL Graphics/src/Recorder.cpp Graphics/src/ShaderInputState.cpp ${agcTextureDetileShaderHeader} + ${agcCmaskClearShaderHeader} ${agcMeshArgumentsShaderHeader} ${agcSampleCounterShaderHeader} ${agcSampleDumpsShaderHeader} @@ -745,7 +781,7 @@ if(BUILD_TESTING) if(WIN32) set_tests_properties(agc_shader_bindless PROPERTIES ENVIRONMENT_MODIFICATION "PATH=path_list_prepend:$") endif() - foreach(agcTest IN ITEMS agc_shared_write_tests agc_driver_tests agc_shader_memory_tests agc_structurizer_tests agc_spirv_module_tests agc_dpp_write_tests agc_masked_select_tests agc_vop1_control_decode_tests agc_validate_program_tests agc_shader_disk_cache_tests agc_shader_device_profile_tests agc_shader_control_flow_tests agc_driver_flip_tests agc_driver_pm4_tests agc_driver_graphics_tests agc_driver_bda_device_tests agc_driver_mesh_tests ${agcExecutionTests}) + foreach(agcTest IN ITEMS agc_shared_write_tests agc_driver_tests agc_shader_memory_tests agc_structurizer_tests agc_spirv_module_tests agc_dpp_write_tests agc_masked_select_tests agc_vop1_control_decode_tests agc_validate_program_tests agc_shader_disk_cache_tests agc_shader_device_profile_tests agc_shader_control_flow_tests agc_driver_flip_tests agc_driver_pm4_tests agc_driver_graphics_tests agc_driver_bda_device_tests agc_driver_mesh_tests agc_driver_cmask_fast_clear_tests ${agcExecutionTests}) if(agcTest STREQUAL "agc_driver_constant_store_tests") continue() endif() diff --git a/core/libs/prx/libSceAgcDriver/Graphics/include/ColorTargetLayout.hpp b/core/libs/prx/libSceAgcDriver/Graphics/include/ColorTargetLayout.hpp index 78112575a..866a9d44a 100644 --- a/core/libs/prx/libSceAgcDriver/Graphics/include/ColorTargetLayout.hpp +++ b/core/libs/prx/libSceAgcDriver/Graphics/include/ColorTargetLayout.hpp @@ -48,6 +48,23 @@ private: const std::uint32_t* yOffsets = nullptr; }; +class CmaskLayout { +public: + static constexpr std::size_t Alignment = 4096; + CmaskLayout(std::uint32_t width, std::uint32_t height); + std::size_t Bytes() const { return bytes; } + std::uint32_t BlocksPerRow() const { return blocksPerRow; } + std::uint32_t TilesX() const { return (width + 7u) / 8u; } + std::uint32_t TilesY() const { return (height + 7u) / 8u; } + std::size_t Nibble(std::uint32_t tileX, std::uint32_t tileY) const; + +private: + std::uint32_t width; + std::uint32_t height; + std::uint32_t blocksPerRow; + std::size_t bytes; +}; + } #endif diff --git a/core/libs/prx/libSceAgcDriver/Graphics/include/State.hpp b/core/libs/prx/libSceAgcDriver/Graphics/include/State.hpp index 704b9ca7d..aef06d5bd 100644 --- a/core/libs/prx/libSceAgcDriver/Graphics/include/State.hpp +++ b/core/libs/prx/libSceAgcDriver/Graphics/include/State.hpp @@ -152,7 +152,7 @@ struct DrawKeyRange { std::uint32_t first; std::uint32_t count; }; -inline constexpr std::array DrawKeyRegisters{{ +inline constexpr std::array DrawKeyRegisters{{ {RegisterBank::Context, 0x000, 1}, {RegisterBank::Context, 0x002, 1}, {RegisterBank::Context, 0x007, 7}, {RegisterBank::Context, 0x010, 6}, {RegisterBank::Context, 0x01a, 4}, {RegisterBank::Context, 0x080, 4}, {RegisterBank::Context, 0x08c, 4}, {RegisterBank::Context, 0x090, 2}, {RegisterBank::Context, 0x094, 2}, {RegisterBank::Context, 0x0b4, 2}, {RegisterBank::Context, 0x105, 4}, {RegisterBank::Context, 0x10b, 3}, {RegisterBank::Context, 0x10f, 6}, // SPI_PS_INPUT_CNTL_0..31, SPI_PS_INPUT_ENA/ADDR, SPI_PS_IN_CONTROL, SPI_SHADER_POS/Z/COL_FORMAT, @@ -162,7 +162,7 @@ inline constexpr std::array DrawKeyRegisters{{ // PA_SU_VTX_CNTL, the sample masks, PA_SC_CONSERVATIVE_RASTERIZATION_CNTL. {RegisterBank::Context, 0x200, 8}, {RegisterBank::Context, 0x292, 2}, {RegisterBank::Context, 0x29b, 1}, {RegisterBank::Context, 0x2ab, 1}, {RegisterBank::Context, 0x2ce, 1}, {RegisterBank::Context, 0x2d5, 2}, {RegisterBank::Context, 0x2db, 2}, {RegisterBank::Context, 0x2de, 6}, {RegisterBank::Context, 0x2f8, 2}, {RegisterBank::Context, 0x30e, 2}, {RegisterBank::Context, 0x313, 1}, // CB_COLOR0..7_BASE .. DCC_BASE (15 words a slot), CB_COLOR0..7_BASE_EXT, DCC_BASE_EXT, ATTRIB2, ATTRIB3. - {RegisterBank::Context, 0x318, 0x78}, {RegisterBank::Context, 0x390, 8}, {RegisterBank::Context, 0x3a8, 0x18}, + {RegisterBank::Context, 0x318, 0x78}, {RegisterBank::Context, 0x390, 8}, {RegisterBank::Context, 0x398, 8}, {RegisterBank::Context, 0x3a8, 0x18}, // The pixel program address, RSRC2 and user words; the geometry-back user pointer and program // address; the vertex/geometry-front RSRC1/RSRC2 and user words; the vertex program address; // the hull user pointer, program address, RSRC2 and user words; the local program address. diff --git a/core/libs/prx/libSceAgcDriver/Graphics/include/Texture.hpp b/core/libs/prx/libSceAgcDriver/Graphics/include/Texture.hpp index a14c015ef..4bbd11575 100644 --- a/core/libs/prx/libSceAgcDriver/Graphics/include/Texture.hpp +++ b/core/libs/prx/libSceAgcDriver/Graphics/include/Texture.hpp @@ -129,6 +129,7 @@ public: VkImageView View(std::uint32_t mip); VkImageView FirstLayerView(std::uint32_t mip); VkImageView StorageView(std::uint32_t mip, bool firstLayer); + VkImageView ElementView(); VkImageView AtomicView(std::uint32_t mip, bool firstLayer); VkImageView Atomic64View(std::uint32_t mip, bool firstLayer); // Render targets live in the same images: draws attach mip 0 through a view of the color @@ -477,6 +478,7 @@ private: std::map firstLayerViews; std::map, VkImageView> atomicViews; std::map, VkImageView> uintViews; + VkImageView elementView = VK_NULL_HANDLE; bool attachable = false; std::map, VkImageView> attachmentViews; VkImage proxyImage = VK_NULL_HANDLE; diff --git a/core/libs/prx/libSceAgcDriver/Graphics/include/TextureDetiler.hpp b/core/libs/prx/libSceAgcDriver/Graphics/include/TextureDetiler.hpp index 7ffb40425..78759fa52 100644 --- a/core/libs/prx/libSceAgcDriver/Graphics/include/TextureDetiler.hpp +++ b/core/libs/prx/libSceAgcDriver/Graphics/include/TextureDetiler.hpp @@ -5,12 +5,16 @@ #include "prx/libSceAgcDriver/Graphics/include/Context.hpp" #include "prx/libSceAgcDriver/Graphics/include/GuestTextureResource.hpp" #include "prx/libSceAgcDriver/Graphics/include/TextureTiling.hpp" +#include #include +#include #include #include namespace AgcDriver::Graphics { + class Buffer; + // The part of a mip one Dispatch moves: only elements whose tiled offset (relative to the mip's // tiled base; the linear offset of a linear surface) lies in [rangeBegin, rangeEnd), with the // tiled buffer holding the mip from tiledBase and the linear buffer holding `linearBytes` of @@ -45,6 +49,8 @@ namespace AgcDriver::Graphics { void Dispatch(VkCommandBuffer commands, TextureTileMode tileMode, std::uint32_t elementBytes, VkBuffer source, std::uint64_t sourceOffset, VkBuffer destination, std::uint64_t destinationOffset, const TileMipLayout& layout, bool retile = false, std::uint32_t slice = 0, bool thick = false, const DetileWindow& window = {}); // Recycles the descriptor sets of the previous batch; call before recording a new command batch. void BeginBatch(); + void DispatchCmaskClear(VkCommandBuffer commands, VkBuffer cmask, std::uint64_t cmaskOffset, std::size_t cmaskBytes, VkImageView view, std::uint32_t width, std::uint32_t height, std::uint32_t elementBytes, const std::array& clearWords, bool write); + std::uint32_t CmaskErrors(); private: VkPipeline pipeline(TextureTileMode tileMode, std::uint32_t elementBytes, bool retile, bool thick); @@ -58,6 +64,12 @@ namespace AgcDriver::Graphics { std::vector> pipelines; std::vector descriptorPools; std::size_t allocatedSets = 0; + VkDescriptorSetLayout cmaskDescriptorLayout = VK_NULL_HANDLE; + VkPipelineLayout cmaskPipelineLayout = VK_NULL_HANDLE; + VkPipeline cmaskPipeline = VK_NULL_HANDLE; + std::vector cmaskDescriptorPools; + std::size_t allocatedCmaskSets = 0; + std::unique_ptr cmaskErrors; }; } diff --git a/core/libs/prx/libSceAgcDriver/Graphics/shaders/CmaskClear.comp b/core/libs/prx/libSceAgcDriver/Graphics/shaders/CmaskClear.comp new file mode 100644 index 000000000..0b2dde1f1 --- /dev/null +++ b/core/libs/prx/libSceAgcDriver/Graphics/shaders/CmaskClear.comp @@ -0,0 +1,38 @@ +#version 450 + +layout(local_size_x = 8, local_size_y = 8) in; + +layout(set = 0, binding = 0, std430) readonly buffer Cmask { uint words[]; } cmask; +layout(set = 0, binding = 1, std430) buffer Errors { uint codes; } errors; +layout(set = 0, binding = 2) writeonly uniform uimage2D target; + +layout(push_constant) uniform Push { + uint base; + uint width; + uint height; + uint blocksPerRow; + uint clear0; + uint clear1; + uint write; +} push; + +const uint xMasks[13] = uint[](0x008u, 0x000u, 0x040u, 0x000u, 0x080u, 0x000u, 0x100u, 0x000u, 0x200u, 0x008u, 0x010u, 0x040u, 0x020u); +const uint yMasks[13] = uint[](0x000u, 0x010u, 0x000u, 0x040u, 0x000u, 0x080u, 0x000u, 0x100u, 0x000u, 0x008u, 0x010u, 0x020u, 0x040u); + +void main() { + uvec2 pixel = gl_GlobalInvocationID.xy; + if (pixel.x >= push.width || pixel.y >= push.height) return; + uint x = pixel.x & ~7u; + uint y = pixel.y & ~7u; + uint offset = 0u; + for (uint bit = 0u; bit < 13u; ++bit) offset |= (uint(bitCount((x & xMasks[bit]) ^ (y & yMasks[bit]))) & 1u) << bit; + uint nibble = ((y / 512u) * push.blocksPerRow + x / 1024u) * 8192u + offset; + uint byteAddress = push.base + nibble / 2u; + uint code = (cmask.words[byteAddress / 4u] >> ((byteAddress % 4u) * 8u + (nibble % 2u) * 4u)) & 0xfu; + if (code == 0xfu) return; + if (code == 0u && push.write != 0u) { + imageStore(target, ivec2(pixel), uvec4(push.clear0, push.clear1, 0u, 0u)); + return; + } + if ((pixel.x & 7u) == 0u && (pixel.y & 7u) == 0u) atomicOr(errors.codes, 1u << code); +} diff --git a/core/libs/prx/libSceAgcDriver/Graphics/src/ColorTargetLayout.cpp b/core/libs/prx/libSceAgcDriver/Graphics/src/ColorTargetLayout.cpp index 5c1895606..e9c5fe3bc 100644 --- a/core/libs/prx/libSceAgcDriver/Graphics/src/ColorTargetLayout.cpp +++ b/core/libs/prx/libSceAgcDriver/Graphics/src/ColorTargetLayout.cpp @@ -1,10 +1,12 @@ #include "prx/libSceAgcDriver/Graphics/include/ColorTargetLayout.hpp" #include "prx/libSceAgcDriver/Graphics/include/TextureSwizzleEquations.hpp" +#include #include #include #include #include #include +#include namespace AgcDriver::Graphics { namespace { @@ -167,4 +169,23 @@ void ColorTargetLayout::Tile(std::span source, std::span(blocksPerRow) * ((height + 511u) / 512u) * Alignment; +} + +std::size_t CmaskLayout::Nibble(std::uint32_t tileX, std::uint32_t tileY) const { + require(tileX < TilesX() && tileY < TilesY(), "AGC graphics: CMASK tile out of range"); + static constexpr std::array, 13> equation{{ + {0x008u, 0x000u}, {0x000u, 0x010u}, {0x040u, 0x000u}, {0x000u, 0x040u}, {0x080u, 0x000u}, {0x000u, 0x080u}, {0x100u, 0x000u}, + {0x000u, 0x100u}, {0x200u, 0x000u}, {0x008u, 0x008u}, {0x010u, 0x010u}, {0x040u, 0x020u}, {0x020u, 0x040u}, + }}; + const auto x = tileX * 8u; + const auto y = tileY * 8u; + std::size_t offset = 0; + for (std::size_t bit = 0; bit < equation.size(); ++bit) offset |= static_cast(parity((x & equation[bit].first) ^ (y & equation[bit].second))) << bit; + const auto block = static_cast(y / 512u) * blocksPerRow + x / 1024u; + return block * Alignment * 2u + offset; +} + } diff --git a/core/libs/prx/libSceAgcDriver/Graphics/src/Draw.cpp b/core/libs/prx/libSceAgcDriver/Graphics/src/Draw.cpp index 591f7fc52..fe5ddf120 100644 --- a/core/libs/prx/libSceAgcDriver/Graphics/src/Draw.cpp +++ b/core/libs/prx/libSceAgcDriver/Graphics/src/Draw.cpp @@ -8,6 +8,8 @@ #include "prx/libSceAgcDriver/Graphics/include/ShaderResources.hpp" #include "prx/libSceAgcDriver/Graphics/include/Recorder.hpp" #include "prx/libSceAgcDriver/Graphics/include/TextureFormat.hpp" +#include "prx/libSceAgcDriver/Graphics/include/UnitShadow.hpp" +#include "prx/libSceAgcDriver/Graphics/include/GuestBufferMemory.hpp" #include "prx/libSceAgcDriver/Execution/include/GuestMemory.hpp" #include "prx/libSceAgcDriver/Execution/include/CaptureTrace.hpp" #include "prx/libSceAgcDriver/Execution/include/PerformanceTimer.hpp" @@ -29,6 +31,7 @@ #include #include #include +#include #include namespace AgcDriver::Graphics { @@ -134,22 +137,156 @@ void materializeRegisterClear(const Context& context, const ColorTarget& color, resident.Refresh(); } -bool materializeCmaskClear(const Context& context, const ColorTarget& color, StorageTexture* resident) { +enum class CmaskState { Expanded, Cleared, Partial }; + +struct CmaskProof { + std::size_t bytes = 0; + std::uint64_t generation = 0; +}; + +std::mutex& cmaskProofMutex() { + static std::mutex mutex; + return mutex; +} + +std::unordered_map& cmaskProofs() { + static std::unordered_map proofs; + return proofs; +} + +bool cmaskProved(std::uint64_t address, std::size_t bytes) { + if (GuestMemory::CollectWrites(address, bytes) == 0) return false; + std::lock_guard lock(cmaskProofMutex()); + const auto found = cmaskProofs().find(address); + return found != cmaskProofs().end() && found->second.bytes == bytes && !GuestMemory::StoredOver(address, bytes, found->second.generation); +} + +void proveCmask(std::uint64_t address, std::size_t bytes, std::uint64_t generation) { + std::lock_guard lock(cmaskProofMutex()); + if (generation == 0) cmaskProofs().erase(address); + else cmaskProofs()[address] = {bytes, generation}; +} + +std::string hexAddress(std::uint64_t address) { + char text[24]; + std::snprintf(text, sizeof(text), "0x%llx", static_cast(address)); + return text; +} + +void requireCmaskCodes(std::uint32_t codes) { + if (codes == 0) return; + Require((codes & 1u) == 0, "a CMASK fast clear of a DCC color target, whose clear belongs in its DCC keys, was recorded"); + std::string listed; + for (std::uint32_t code = 1; code < 15; ++code) { + if ((codes >> code) & 1u) listed += (listed.empty() ? "" : ", ") + std::to_string(code); + } + Require(false, "CMASK codes " + listed + " were recorded, which are not a single-sample fast-clear code (0 cleared, 15 expanded)"); +} + +CmaskState readCmask(const ColorTarget& color, const CmaskLayout& layout, std::vector& cleared) { + const auto address = color.cmaskAddress; + const auto bytes = layout.Bytes(); + const bool locked = GuestMemory::GpuMutex().HeldByThisThread(); + auto* recorder = Recorder::Active(); + if (locked && recorder != nullptr && recorder->PendingWriteOverlaps(address, bytes)) { + Recorder::CountSync(2); + recorder->SyncThrough(address, bytes); + } + if (AnyShadowedOverlaps(address, bytes)) { + Require(locked, "a CMASK under a unit shadow must be read under the device lock"); + PublishShadow(address, bytes, PublishScope::Whole, PublishReason::Keys); + if (recorder != nullptr) { + Recorder::CountSync(2); + recorder->Sync(); + } + } + std::vector mask(bytes); + GuestMemory::Read(address, mask, CmaskLayout::Alignment); + const auto uniform = [&](std::byte value) { return std::all_of(mask.begin(), mask.end(), [&](std::byte entry) { return entry == value; }); }; + if (uniform(std::byte{0xff})) return CmaskState::Expanded; + if (uniform(std::byte{0})) return CmaskState::Cleared; + for (std::uint32_t tileY = 0; tileY < layout.TilesY(); ++tileY) { + for (std::uint32_t tileX = 0; tileX < layout.TilesX(); ++tileX) { + const auto nibble = layout.Nibble(tileX, tileY); + const auto code = (std::to_integer(mask[nibble / 2u]) >> ((nibble % 2u) * 4u)) & 0xfu; + if (code == 0u) cleared.push_back(tileY * layout.TilesX() + tileX); + else Require(code == 0xfu, "CMASK code " + std::to_string(code) + " of color target " + hexAddress(color.address) + " is not a single-sample fast-clear code (0 cleared, 15 expanded)"); + } + } + if (cleared.empty()) return CmaskState::Expanded; + if (cleared.size() == static_cast(layout.TilesX()) * layout.TilesY()) return CmaskState::Cleared; + return CmaskState::Partial; +} + +void storeCmaskClearTexels(const ColorTarget& color, const CmaskLayout& layout, const std::vector* tiles, const std::array& texel) { + StorageTexture::FlushPending(color.address, color.bytes, nullptr, "fast-clear materialization"); + const ColorTargetLayout surface(color.extent.width, color.extent.height, color.tileMode, color.elementBytes); + Require(surface.Bytes() == color.bytes, "CMASK fast clear over a color target whose layout differs from its surface"); + std::vector texels(surface.Bytes()); + GuestMemory::Read(color.address, texels, surface.Alignment()); + const auto fill = [&](std::uint32_t tile) { + const auto x0 = (tile % layout.TilesX()) * 8u; + const auto y0 = (tile / layout.TilesX()) * 8u; + for (auto y = y0; y < std::min(y0 + 8u, color.extent.height); ++y) { + for (auto x = x0; x < std::min(x0 + 8u, color.extent.width); ++x) std::memcpy(texels.data() + surface.Offset(x, y), texel.data(), color.elementBytes); + } + }; + if (tiles != nullptr) { + for (const auto tile : *tiles) fill(tile); + } else { + for (std::uint32_t tile = 0; tile < layout.TilesX() * layout.TilesY(); ++tile) fill(tile); + } + GuestMemory::Write(color.address, texels, surface.Alignment()); +} + +void recordCmaskClear(const Context& context, Recorder& recorder, const HostImport& import, const ColorTarget& color, const CmaskLayout& layout, const std::shared_ptr& resident, VkImageView view) { + const auto address = color.cmaskAddress; + const auto bytes = layout.Bytes(); + StorageTexture::FlushPending(address, bytes, nullptr, "fast-clear materialization"); + if (AnyShadowedOverlaps(address, bytes)) PublishShadow(address, bytes, PublishScope::Whole, PublishReason::Keys); + recorder.FlushKeyStoresOverlapping(address, bytes); + const auto commands = recorder.Commands(); + recorder.Keep(resident); + context.detiler->DispatchCmaskClear(commands, import.buffer, address - import.base, bytes, view, color.extent.width, color.extent.height, color.elementBytes, color.clearWords, color.dccAddress == 0); + recorder.NotePendingWrite(address, bytes); + proveCmask(address, bytes, GuestMemory::MarkWritten(address, bytes)); +} + +bool materializeCmaskClear(const Context& context, const ColorTarget& color, const std::shared_ptr& resident) { if (color.cmaskAddress == 0) return false; - const auto metadataBytes = static_cast(color.cmaskBytes) * 256u; - const auto state = CurrentDccKeys(color.cmaskAddress, metadataBytes); - if (state == DccKeys::Uncompressed) return false; - Require(color.dccAddress == 0, std::string("CMASK of a DCC color target that is not all expanded is not modeled (") + DccKeysName(state) + ")"); - Require(state == DccKeys::Clear0000, std::string("CMASK whose tiles are not all fast-cleared or all expanded is not modeled (") + DccKeysName(state) + ")"); + if (context.detiler != nullptr) requireCmaskCodes(context.detiler->CmaskErrors()); + const CmaskLayout layout(color.extent.width, color.extent.height); + const auto bytes = layout.Bytes(); + Require(color.cmaskBytes == bytes, "CMASK size differs from the color target's CMASK layout"); + if (cmaskProved(color.cmaskAddress, bytes)) return false; + auto* recorder = Recorder::Active(); + if (resident != nullptr && context.detiler != nullptr && recorder != nullptr && GuestMemory::GpuMutex().HeldByThisThread()) { + const auto* import = HostImportFor(context, color.cmaskAddress, bytes); + const auto view = import != nullptr ? resident->ElementView() : VK_NULL_HANDLE; + if (view != VK_NULL_HANDLE) { + recordCmaskClear(context, *recorder, *import, color, layout, resident, view); + return true; + } + } + const auto collected = GuestMemory::CollectWrites(color.cmaskAddress, bytes); + std::vector cleared; + const auto state = readCmask(color, layout, cleared); + if (state == CmaskState::Expanded) { + proveCmask(color.cmaskAddress, bytes, collected); + return false; + } + Require(color.dccAddress == 0, "CMASK fast clear of a DCC color target, whose clear belongs in its DCC keys"); const auto texel = clearTexel(color, DccKeys::ClearRegister); const char* refusal = nullptr; - if (resident == nullptr || !clearToTexel(*resident, texel, color.elementBytes, refusal)) { - StorageTexture::FlushPending(color.address, color.bytes, nullptr, "fast-clear materialization"); - writeTexels(color, texel); + const bool filled = state == CmaskState::Cleared && resident != nullptr && clearToTexel(*resident, texel, color.elementBytes, refusal); + if (!filled) { + storeCmaskClearTexels(color, layout, state == CmaskState::Cleared ? nullptr : &cleared, texel); if (resident != nullptr) resident->Refresh(); } - MarkDccUncompressed(context, color.cmaskAddress, metadataBytes); - return true; + const std::vector expanded(bytes, std::byte{0xff}); + GuestMemory::Write(color.cmaskAddress, expanded, CmaskLayout::Alignment); + proveCmask(color.cmaskAddress, bytes, GuestMemory::CollectWrites(color.cmaskAddress, bytes)); + return filled; } void imageBarrier(const Context& context, VkCommandBuffer commands, VkImage image, VkImageLayout oldLayout, VkImageLayout newLayout, VkPipelineStageFlags sourceStage, VkPipelineStageFlags destinationStage, VkAccessFlags sourceAccess, VkAccessFlags destinationAccess) { @@ -943,7 +1080,7 @@ std::shared_ptr refreshResidentTarget(const Context& context, co std::lock_guard lock(reportMutex); if (reported.insert(color.address).second) std::fprintf(stderr, "[gpu] color target 0x%llx stays non-resident: %s\n", static_cast(color.address), error.what()); } - if (resident != nullptr) materializeCmaskClear(context, color, resident.get()); + if (resident != nullptr) materializeCmaskClear(context, color, resident); if (profile) { const auto lookupUs = std::chrono::duration(std::chrono::steady_clock::now() - lookupStart).count(); ++outcome.targetLookups; @@ -2176,7 +2313,7 @@ void RunColorMetadataPass(const Context& context, const ColorMetadataPass& pass) for (const auto& color : pass.targets) { if (color.cmaskAddress != 0) { const auto resident = metadataPassResident(context, color); - if (materializeCmaskClear(context, color, resident.get()) && resident != nullptr) resident->MarkDirty(); + if (materializeCmaskClear(context, color, resident) && resident != nullptr) resident->MarkDirty(); } if (color.dccAddress == 0) continue; auto keys = CurrentDccKeys(color.dccAddress, color.bytes); diff --git a/core/libs/prx/libSceAgcDriver/Graphics/src/State.cpp b/core/libs/prx/libSceAgcDriver/Graphics/src/State.cpp index a074fabc2..30b02529f 100644 --- a/core/libs/prx/libSceAgcDriver/Graphics/src/State.cpp +++ b/core/libs/prx/libSceAgcDriver/Graphics/src/State.cpp @@ -734,10 +734,7 @@ std::array ExportMappings(const State& state) { } std::size_t CmaskBytes(std::uint32_t width, std::uint32_t height) { - constexpr std::size_t metablockWidth = 1024; - constexpr std::size_t metablockHeight = 512; - constexpr std::size_t metablockBytes = 4096; - return ((width + metablockWidth - 1) / metablockWidth) * ((height + metablockHeight - 1) / metablockHeight) * metablockBytes; + return CmaskLayout(width, height).Bytes(); } std::uint32_t ColorWriteMask(const Registers& context) { @@ -813,10 +810,13 @@ ColorTarget DecodeColorBuffer(const Registers& cx, std::uint32_t slot) { } if ((info & 0x2000u) != 0) { Require(maxMip == 0 && !volume && slice == 0, "CMASK fast clears of a mipmapped, 3D or array color target are unsupported"); + Require(((attrib3 >> 19u) & 0x1fu) == 0x18u && (attrib3 & 0x4000000u) != 0, "CMASK fast clears need pipe-aligned SW_64KB_Z_X metadata (CB_COLOR_ATTRIB3 FMASK_SW_MODE 24, CMASK_PIPE_ALIGNED)"); + Require(color.elementBytes <= 8, "CMASK fast clears of texels over 64 bits are unsupported"); const auto cmaskHigh = find(cx, 0x398 + slot); color.cmaskAddress = ((cmaskHigh == cx.end() ? 0ull : static_cast(cmaskHigh->second & 0xffu)) << 40u) | (static_cast(read(cx, 0x31f + stride)) << 8u); Require(color.cmaskAddress != 0, "CMASK fast clears without a CMASK address are unsupported"); color.cmaskBytes = CmaskBytes(color.extent.width, color.extent.height); + GuestMemory::CheckRange(reinterpret_cast(color.cmaskAddress), color.cmaskBytes, CmaskLayout::Alignment, true); } if ((info & 0x10000000u) != 0) { if (maxMip == 0) { diff --git a/core/libs/prx/libSceAgcDriver/Graphics/src/Texture.cpp b/core/libs/prx/libSceAgcDriver/Graphics/src/Texture.cpp index 26911542c..4697f2fe8 100644 --- a/core/libs/prx/libSceAgcDriver/Graphics/src/Texture.cpp +++ b/core/libs/prx/libSceAgcDriver/Graphics/src/Texture.cpp @@ -1159,6 +1159,16 @@ VkImageView StorageTexture::StorageView(std::uint32_t mip, bool firstLayer) { return created; } +VkImageView StorageTexture::ElementView() { + if (elementView != VK_NULL_HANDLE) return elementView; + if (descriptor.dimension != TextureDimension::k2D || geometry.imageLayers != 1) return VK_NULL_HANDLE; + const auto bytes = BytesPerElement(descriptor.format); + const auto format = bytes == 1 ? VK_FORMAT_R8_UINT : bytes == 2 ? VK_FORMAT_R16_UINT : bytes == 4 ? VK_FORMAT_R32_UINT : bytes == 8 ? VK_FORMAT_R32G32_UINT : VK_FORMAT_UNDEFINED; + if (format == VK_FORMAT_UNDEFINED || StorageFormatOrUndefined(context, format) != format) return VK_NULL_HANDLE; + elementView = createView(0, false, format); + return elementView; +} + VkImageView StorageTexture::AtomicView(std::uint32_t mip, bool firstLayer) { if (storageFormat == VK_FORMAT_R32_UINT) return firstLayer ? FirstLayerView(mip) : View(mip); Require(storageFormat == VK_FORMAT_R32_SINT || storageFormat == VK_FORMAT_R32_SFLOAT, "storage image atomics need a surface of one 32-bit component"); @@ -3833,6 +3843,8 @@ void StorageTexture::release() noexcept { atomicViews.clear(); for (const auto& [key, uint] : uintViews) context.Function("vkDestroyImageView")(context.device, uint, nullptr); uintViews.clear(); + if (elementView) context.Function("vkDestroyImageView")(context.device, elementView, nullptr); + elementView = VK_NULL_HANDLE; for (const auto& [format, attachment] : attachmentViews) context.Function("vkDestroyImageView")(context.device, attachment, nullptr); attachmentViews.clear(); if (proxyView) context.Function("vkDestroyImageView")(context.device, proxyView, nullptr); diff --git a/core/libs/prx/libSceAgcDriver/Graphics/src/TextureDetiler.cpp b/core/libs/prx/libSceAgcDriver/Graphics/src/TextureDetiler.cpp index 428c464f9..294c4c685 100644 --- a/core/libs/prx/libSceAgcDriver/Graphics/src/TextureDetiler.cpp +++ b/core/libs/prx/libSceAgcDriver/Graphics/src/TextureDetiler.cpp @@ -1,11 +1,14 @@ #include "prx/libSceAgcDriver/Graphics/include/TextureDetiler.hpp" +#include "prx/libSceAgcDriver/Graphics/include/Resources.hpp" #include "prx/libSceAgcDriver/Graphics/include/TextureSwizzleEquations.hpp" #include "prx/libSceAgcDriver/Graphics/include/TextureTiling.hpp" #include "prx/libSceAgcDriver/Graphics/shaders/TextureDetile_spv.h" +#include "prx/libSceAgcDriver/Graphics/shaders/CmaskClear_spv.h" #include #include #include #include +#include #include #include @@ -98,6 +101,11 @@ TextureDetiler::~TextureDetiler() { } void TextureDetiler::release() noexcept { + for (const auto pool : cmaskDescriptorPools) context.Function("vkDestroyDescriptorPool")(context.device, pool, nullptr); + if (cmaskPipeline) context.Function("vkDestroyPipeline")(context.device, cmaskPipeline, nullptr); + if (cmaskPipelineLayout) context.Function("vkDestroyPipelineLayout")(context.device, cmaskPipelineLayout, nullptr); + if (cmaskDescriptorLayout) context.Function("vkDestroyDescriptorSetLayout")(context.device, cmaskDescriptorLayout, nullptr); + cmaskErrors.reset(); for (const auto pool : descriptorPools) context.Function("vkDestroyDescriptorPool")(context.device, pool, nullptr); for (const auto& entry : pipelines) context.Function("vkDestroyPipeline")(context.device, entry.second, nullptr); if (module) context.Function("vkDestroyShaderModule")(context.device, module, nullptr); @@ -217,4 +225,109 @@ void TextureDetiler::Dispatch(VkCommandBuffer commands, TextureTileMode tileMode context.Function("vkCmdDispatch")(commands, groupsX, groupsY, 1); } +std::uint32_t TextureDetiler::CmaskErrors() { + if (cmaskErrors == nullptr) return 0; + std::uint32_t codes = 0; + std::memcpy(&codes, cmaskErrors->Bytes().data(), sizeof(codes)); + return codes; +} + +void TextureDetiler::DispatchCmaskClear(VkCommandBuffer commands, VkBuffer cmask, std::uint64_t cmaskOffset, std::size_t cmaskBytes, VkImageView view, std::uint32_t width, std::uint32_t height, std::uint32_t elementBytes, const std::array& clearWords, bool write) { + Require(commands != VK_NULL_HANDLE && cmask != VK_NULL_HANDLE && view != VK_NULL_HANDLE, "a CMASK clear needs a command buffer, the CMASK buffer and the target view"); + Require(elementBytes == 1 || elementBytes == 2 || elementBytes == 4 || elementBytes == 8, "a CMASK clear of this element size is not modeled"); + Require(cmaskOffset % 4 == 0 && cmaskBytes % 4 == 0 && cmaskBytes != 0, "a CMASK clear needs a word-aligned CMASK"); + if (cmaskPipeline == VK_NULL_HANDLE) { + std::array bindings{}; + for (std::uint32_t index = 0; index < bindings.size(); ++index) { + bindings[index].binding = index; + bindings[index].descriptorType = index == 2 ? VK_DESCRIPTOR_TYPE_STORAGE_IMAGE : VK_DESCRIPTOR_TYPE_STORAGE_BUFFER; + bindings[index].descriptorCount = 1; + bindings[index].stageFlags = VK_SHADER_STAGE_COMPUTE_BIT; + } + VkDescriptorSetLayoutCreateInfo layoutInfo{VK_STRUCTURE_TYPE_DESCRIPTOR_SET_LAYOUT_CREATE_INFO}; + layoutInfo.bindingCount = static_cast(bindings.size()); + layoutInfo.pBindings = bindings.data(); + Check(context.Function("vkCreateDescriptorSetLayout")(context.device, &layoutInfo, nullptr, &cmaskDescriptorLayout), "vkCreateDescriptorSetLayout CMASK clear"); + const VkPushConstantRange pushRange{VK_SHADER_STAGE_COMPUTE_BIT, 0, 7 * sizeof(std::uint32_t)}; + VkPipelineLayoutCreateInfo pipelineLayoutInfo{VK_STRUCTURE_TYPE_PIPELINE_LAYOUT_CREATE_INFO}; + pipelineLayoutInfo.setLayoutCount = 1; + pipelineLayoutInfo.pSetLayouts = &cmaskDescriptorLayout; + pipelineLayoutInfo.pushConstantRangeCount = 1; + pipelineLayoutInfo.pPushConstantRanges = &pushRange; + Check(context.Function("vkCreatePipelineLayout")(context.device, &pipelineLayoutInfo, nullptr, &cmaskPipelineLayout), "vkCreatePipelineLayout CMASK clear"); + VkShaderModuleCreateInfo moduleInfo{VK_STRUCTURE_TYPE_SHADER_MODULE_CREATE_INFO}; + moduleInfo.codeSize = sizeof(CMASK_CLEAR_SPV); + moduleInfo.pCode = CMASK_CLEAR_SPV; + VkShaderModule cmaskModule = VK_NULL_HANDLE; + Check(context.Function("vkCreateShaderModule")(context.device, &moduleInfo, nullptr, &cmaskModule), "vkCreateShaderModule CMASK clear"); + VkComputePipelineCreateInfo pipelineInfo{VK_STRUCTURE_TYPE_COMPUTE_PIPELINE_CREATE_INFO}; + pipelineInfo.stage = {VK_STRUCTURE_TYPE_PIPELINE_SHADER_STAGE_CREATE_INFO}; + pipelineInfo.stage.stage = VK_SHADER_STAGE_COMPUTE_BIT; + pipelineInfo.stage.module = cmaskModule; + pipelineInfo.stage.pName = "main"; + pipelineInfo.layout = cmaskPipelineLayout; + const auto created = context.Function("vkCreateComputePipelines")(context.device, context.pipelineCache, 1, &pipelineInfo, nullptr, &cmaskPipeline); + context.Function("vkDestroyShaderModule")(context.device, cmaskModule, nullptr); + Check(created, "vkCreateComputePipelines CMASK clear"); + cmaskErrors = std::make_unique(context, 4, VK_BUFFER_USAGE_STORAGE_BUFFER_BIT); + std::memset(cmaskErrors->Bytes().data(), 0, 4); + } + constexpr std::uint32_t setsPerPool = 64; + const auto poolIndex = allocatedCmaskSets / setsPerPool; + if (poolIndex == cmaskDescriptorPools.size()) { + const std::array sizes{{{VK_DESCRIPTOR_TYPE_STORAGE_BUFFER, setsPerPool * 2}, {VK_DESCRIPTOR_TYPE_STORAGE_IMAGE, setsPerPool}}}; + VkDescriptorPoolCreateInfo info{VK_STRUCTURE_TYPE_DESCRIPTOR_POOL_CREATE_INFO}; + info.maxSets = setsPerPool; + info.poolSizeCount = static_cast(sizes.size()); + info.pPoolSizes = sizes.data(); + cmaskDescriptorPools.reserve(cmaskDescriptorPools.size() + 1); + VkDescriptorPool pool = VK_NULL_HANDLE; + Check(context.Function("vkCreateDescriptorPool")(context.device, &info, nullptr, &pool), "vkCreateDescriptorPool CMASK clear"); + cmaskDescriptorPools.push_back(pool); + } + VkDescriptorSetAllocateInfo allocation{VK_STRUCTURE_TYPE_DESCRIPTOR_SET_ALLOCATE_INFO}; + allocation.descriptorPool = cmaskDescriptorPools.at(poolIndex); + allocation.descriptorSetCount = 1; + allocation.pSetLayouts = &cmaskDescriptorLayout; + VkDescriptorSet set = VK_NULL_HANDLE; + Check(context.Function("vkAllocateDescriptorSets")(context.device, &allocation, &set), "vkAllocateDescriptorSets CMASK clear"); + ++allocatedCmaskSets; + const auto alignment = std::max(context.limits.minStorageBufferOffsetAlignment, 4); + const auto descriptorOffset = cmaskOffset - cmaskOffset % alignment; + const auto base = cmaskOffset - descriptorOffset; + const VkDescriptorBufferInfo cmaskInfo{cmask, descriptorOffset, base + cmaskBytes}; + const VkDescriptorBufferInfo errorInfo{cmaskErrors->Handle(), 0, 4}; + const VkDescriptorImageInfo imageInfo{VK_NULL_HANDLE, view, VK_IMAGE_LAYOUT_GENERAL}; + std::array writes{}; + for (std::uint32_t index = 0; index < writes.size(); ++index) { + writes[index].sType = VK_STRUCTURE_TYPE_WRITE_DESCRIPTOR_SET; + writes[index].dstSet = set; + writes[index].dstBinding = index; + writes[index].descriptorCount = 1; + writes[index].descriptorType = index == 2 ? VK_DESCRIPTOR_TYPE_STORAGE_IMAGE : VK_DESCRIPTOR_TYPE_STORAGE_BUFFER; + } + writes[0].pBufferInfo = &cmaskInfo; + writes[1].pBufferInfo = &errorInfo; + writes[2].pImageInfo = &imageInfo; + context.Function("vkUpdateDescriptorSets")(context.device, static_cast(writes.size()), writes.data(), 0, nullptr); + const auto barrier = [&](VkPipelineStageFlags source, VkPipelineStageFlags destination, VkAccessFlags sourceAccess, VkAccessFlags destinationAccess) { + VkMemoryBarrier memory{VK_STRUCTURE_TYPE_MEMORY_BARRIER}; + memory.srcAccessMask = sourceAccess; + memory.dstAccessMask = destinationAccess; + context.Function("vkCmdPipelineBarrier")(commands, source, destination, 0, 1, &memory, 0, nullptr, 0, nullptr); + }; + constexpr VkAccessFlags anyWrite = VK_ACCESS_SHADER_WRITE_BIT | VK_ACCESS_TRANSFER_WRITE_BIT | VK_ACCESS_COLOR_ATTACHMENT_WRITE_BIT | VK_ACCESS_HOST_WRITE_BIT; + constexpr VkAccessFlags anyAccess = anyWrite | VK_ACCESS_SHADER_READ_BIT | VK_ACCESS_TRANSFER_READ_BIT | VK_ACCESS_COLOR_ATTACHMENT_READ_BIT | VK_ACCESS_HOST_READ_BIT | VK_ACCESS_INDIRECT_COMMAND_READ_BIT; + barrier(VK_PIPELINE_STAGE_ALL_COMMANDS_BIT, VK_PIPELINE_STAGE_COMPUTE_SHADER_BIT, anyWrite, VK_ACCESS_SHADER_READ_BIT | VK_ACCESS_SHADER_WRITE_BIT); + context.Function("vkCmdBindPipeline")(commands, VK_PIPELINE_BIND_POINT_COMPUTE, cmaskPipeline); + context.Function("vkCmdBindDescriptorSets")(commands, VK_PIPELINE_BIND_POINT_COMPUTE, cmaskPipelineLayout, 0, 1, &set, 0, nullptr); + const std::uint32_t mask = elementBytes >= 4 ? 0xffffffffu : (1u << (elementBytes * 8u)) - 1u; + const std::array push{static_cast(base), width, height, (width + 1023u) / 1024u, clearWords[0] & mask, elementBytes == 8 ? clearWords[1] : 0u, write ? 1u : 0u}; + context.Function("vkCmdPushConstants")(commands, cmaskPipelineLayout, VK_SHADER_STAGE_COMPUTE_BIT, 0, sizeof(push), push.data()); + context.Function("vkCmdDispatch")(commands, (width + 7u) / 8u, (height + 7u) / 8u, 1); + barrier(VK_PIPELINE_STAGE_COMPUTE_SHADER_BIT, VK_PIPELINE_STAGE_ALL_COMMANDS_BIT, VK_ACCESS_SHADER_WRITE_BIT | VK_ACCESS_SHADER_READ_BIT, anyAccess); + context.Function("vkCmdFillBuffer")(commands, cmask, cmaskOffset, cmaskBytes, 0xffffffffu); + barrier(VK_PIPELINE_STAGE_TRANSFER_BIT, VK_PIPELINE_STAGE_ALL_COMMANDS_BIT | VK_PIPELINE_STAGE_HOST_BIT, VK_ACCESS_TRANSFER_WRITE_BIT, anyAccess); +} + } diff --git a/core/libs/prx/libSceAgcDriver/Graphics/src/TextureDetilerDescriptors.cpp b/core/libs/prx/libSceAgcDriver/Graphics/src/TextureDetilerDescriptors.cpp index ed3b905e4..cea243e2e 100644 --- a/core/libs/prx/libSceAgcDriver/Graphics/src/TextureDetilerDescriptors.cpp +++ b/core/libs/prx/libSceAgcDriver/Graphics/src/TextureDetilerDescriptors.cpp @@ -13,6 +13,10 @@ void TextureDetiler::BeginBatch() { Check(context.Function("vkResetDescriptorPool")(context.device, pool, 0), "vkResetDescriptorPool texture detiler"); } allocatedSets = 0; + for (const auto pool : cmaskDescriptorPools) { + Check(context.Function("vkResetDescriptorPool")(context.device, pool, 0), "vkResetDescriptorPool CMASK clear"); + } + allocatedCmaskSets = 0; } VkDescriptorSet TextureDetiler::allocateSet() { diff --git a/core/libs/prx/libSceAgcDriver/tests/CmaskFastClear.cpp b/core/libs/prx/libSceAgcDriver/tests/CmaskFastClear.cpp new file mode 100644 index 000000000..c430e3784 --- /dev/null +++ b/core/libs/prx/libSceAgcDriver/tests/CmaskFastClear.cpp @@ -0,0 +1,199 @@ +#include "prx/libSceAgcDriver/Execution/include/GuestMemory.hpp" +#include "prx/libSceAgcDriver/Execution/include/VulkanDevice.hpp" +#include "prx/libSceAgcDriver/Graphics/include/ColorTargetLayout.hpp" +#include "prx/libSceAgcDriver/Graphics/include/Texture.hpp" +#include "prx/libc/include/GuestAllocations.hpp" +#include "Cover_vert_spv.h" +#include "Cover_frag_spv.h" +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include + +namespace { + +using namespace AgcDriver::Graphics; + +constexpr int VulkanTestSkipped = 77; +constexpr std::uint32_t Width = 61; +constexpr std::uint32_t Height = 13; +constexpr std::uint32_t Stale = 0x5a5a5a5au; +constexpr std::uint32_t Clear = 0x11223344u; +constexpr std::uint32_t Drawn = 0xffffffffu; + +alignas(4096) std::array SurfaceStorage{}; +const std::span Surface{reinterpret_cast((reinterpret_cast(SurfaceStorage.data()) + 65535u) & ~std::uintptr_t{65535u}), 65536}; +alignas(4096) std::array Cmask{}; + +ColorTarget target(ColorTileMode mode) { + const ColorTargetLayout layout(Width, Height, mode, 4); + ColorTarget color{}; + color.address = reinterpret_cast(Surface.data()); + color.extent = {Width, Height}; + color.format = VK_FORMAT_R8G8B8A8_UNORM; + color.bytes = layout.Bytes(); + color.componentMapping = 0xe4u; + color.tileMode = mode; + color.elementBytes = 4; + color.clearWords = {Clear, 0}; + color.surfaceAddress = color.address; + color.surfaceExtent = color.extent; + color.cmaskAddress = reinterpret_cast(Cmask.data()); + color.cmaskBytes = CmaskLayout(Width, Height).Bytes(); + return color; +} + +void prepare(const ColorTarget& color, const std::function& cleared, std::uint8_t code = 0) { + std::lock_guard lock(AgcDriver::GuestMemory::GpuMutex()); + StorageTexture::FlushPending(color.address, Surface.size(), nullptr, "test"); + std::vector stale(Surface.size()); + for (std::size_t offset = 0; offset < stale.size(); offset += 4) std::memcpy(stale.data() + offset, &Stale, 4); + AgcDriver::GuestMemory::Write(color.address, stale); + const CmaskLayout layout(Width, Height); + std::vector mask(Cmask.size(), std::byte{0xff}); + for (std::uint32_t tileY = 0; tileY < layout.TilesY(); ++tileY) { + for (std::uint32_t tileX = 0; tileX < layout.TilesX(); ++tileX) { + if (!cleared(tileX, tileY)) continue; + const auto nibble = layout.Nibble(tileX, tileY); + mask[nibble / 2u] &= std::byte{static_cast((nibble % 2u) == 0 ? 0xf0u : 0x0fu)}; + mask[nibble / 2u] |= std::byte{static_cast((nibble % 2u) == 0 ? code : code << 4u)}; + } + } + AgcDriver::GuestMemory::Write(color.cmaskAddress, mask); +} + +void draw(AgcDriver::VulkanDevice& device, const ColorTarget& color, VkRect2D scissor) { + ShaderRecompiler::RecompileResult vertex; + vertex.spirv = std::vector(std::begin(COVER_VERT_SPV), std::end(COVER_VERT_SPV)); + ShaderRecompiler::RecompileResult fragment; + fragment.spirv = std::vector(std::begin(COVER_FRAG_SPV), std::end(COVER_FRAG_SPV)); + const std::array shaders{{ + {ShaderRecompiler::ShaderStage::Vertex, &vertex, 0}, + {ShaderRecompiler::ShaderStage::Fragment, &fragment, 0} + }}; + State state{}; + state.stages = {ShaderPath::Vertex, 0u, 64u, 64u, std::nullopt, std::nullopt}; + state.color = color; + state.colors = {color}; + state.hasColorTarget = true; + state.renderExtent = {Width, Height}; + state.topology = VK_PRIMITIVE_TOPOLOGY_TRIANGLE_LIST; + state.viewport = {0, 0, static_cast(Width), static_cast(Height), 0, 1}; + state.negativeOneToOne = false; + state.scissor = scissor; + state.cullMode = VK_CULL_MODE_NONE; + state.frontFace = VK_FRONT_FACE_COUNTER_CLOCKWISE; + state.blend.colorWriteMask = 15; + state.blends = {state.blend}; + state.blendConstants = {}; + std::lock_guard lock(AgcDriver::GuestMemory::GpuMutex()); + device.Draw(state, {0, 3, 0, 1, 0, false}, shaders); + device.WaitIdle(); +} + +void expect(AgcDriver::VulkanDevice& device, const ColorTarget& color, const std::function& texel, const std::string& what) { + std::lock_guard lock(AgcDriver::GuestMemory::GpuMutex()); + StorageTexture::FlushPending(color.address, color.bytes, nullptr, "test"); + device.WaitIdle(); + const ColorTargetLayout layout(Width, Height, color.tileMode, 4); + for (std::uint32_t y = 0; y < Height; ++y) { + for (std::uint32_t x = 0; x < Width; ++x) { + std::uint32_t stored = 0; + std::memcpy(&stored, Surface.data() + layout.Offset(x, y), 4); + Require(stored == texel(x, y), what + ": texel (" + std::to_string(x) + ", " + std::to_string(y) + ") holds " + std::to_string(stored)); + } + } + Require(std::all_of(Cmask.begin(), Cmask.end(), [](std::uint8_t value) { return value == 0xffu; }), what + ": the CMASK was not left expanded"); +} + +void eliminate(AgcDriver::VulkanDevice& device, const ColorTarget& color) { + std::lock_guard lock(AgcDriver::GuestMemory::GpuMutex()); + device.ColorMetadataPass({ColorMetadataPass::Mode::EliminateFastClear, {color}}); +} + +void run(AgcDriver::VulkanDevice& device, ColorTileMode mode, const std::string& name) { + const auto color = target(mode); + prepare(color, [](std::uint32_t, std::uint32_t) { return true; }); + draw(device, color, {{0, 0}, {20, Height}}); + expect(device, color, [](std::uint32_t x, std::uint32_t) { return x < 20 ? Drawn : Clear; }, name + " fast-cleared target drawn over its left part"); + draw(device, color, {{40, 0}, {Width - 40, Height}}); + expect(device, color, [](std::uint32_t x, std::uint32_t) { return x < 20 || x >= 40 ? Drawn : Clear; }, name + " fast-cleared target drawn again over its right part"); + prepare(color, [](std::uint32_t tileX, std::uint32_t tileY) { return (tileX == 0 && tileY == 0) || (tileX == 7 && tileY == 1); }); + draw(device, color, {{0, 0}, {20, Height}}); + expect(device, color, [](std::uint32_t x, std::uint32_t y) { return x < 20 ? Drawn : x >= 56 && y >= 8 ? Clear : Stale; }, name + " partly fast-cleared target drawn over its left part"); + prepare(color, [](std::uint32_t, std::uint32_t) { return true; }); + eliminate(device, color); + expect(device, color, [](std::uint32_t, std::uint32_t) { return Clear; }, name + " fast-cleared target eliminated without a draw"); + prepare(color, [](std::uint32_t tileX, std::uint32_t tileY) { return tileX == 7 && tileY == 1; }); + eliminate(device, color); + expect(device, color, [](std::uint32_t x, std::uint32_t y) { return x >= 56 && y >= 8 ? Clear : Stale; }, name + " partly fast-cleared edge tile eliminated"); + prepare(color, [](std::uint32_t tileX, std::uint32_t tileY) { return (tileX + tileY) % 2u == 0; }); + eliminate(device, color); + expect(device, color, [](std::uint32_t x, std::uint32_t y) { return (x / 8u + y / 8u) % 2u == 0 ? Clear : Stale; }, name + " checkerboard of fast-cleared tiles eliminated"); + prepare(color, [](std::uint32_t tileX, std::uint32_t tileY) { return tileX >= 3 && tileY == 1; }); + draw(device, color, {{0, 0}, {Width, 4}}); + expect(device, color, [](std::uint32_t x, std::uint32_t y) { return y < 4 ? Drawn : y >= 8 && x >= 24 ? Clear : Stale; }, name + " fast-cleared bottom edge tiles under a draw of the top rows"); +} + +void invalidCode(AgcDriver::VulkanDevice& device) { + const auto color = target(ColorTileMode::RenderTarget); + prepare(color, [](std::uint32_t tileX, std::uint32_t tileY) { return tileX == 2 && tileY == 0; }, 3); + eliminate(device, color); + { + std::lock_guard lock(AgcDriver::GuestMemory::GpuMutex()); + device.WaitIdle(); + } + prepare(color, [](std::uint32_t, std::uint32_t) { return true; }); + try { + eliminate(device, color); + } catch (const std::runtime_error& error) { + Require(std::string(error.what()).find("were recorded, which are not a single-sample fast-clear code") != std::string::npos, std::string("unexpected failure: ") + error.what()); + return; + } + throw std::runtime_error("a CMASK code 3 read by the GPU pass did not throw"); +} + +} + +int main() { + try { + std::unique_ptr device; + try { + device = std::make_unique(); + } catch (const std::exception& error) { + if (std::getenv("ANYPS5_REQUIRE_VULKAN") != nullptr) throw; + std::printf("skipped, no usable Vulkan device: %s\n", error.what()); + return VulkanTestSkipped; + } + { + GuestAllocations::Mutation mutation; + mutation.Add(SurfaceStorage.data(), SurfaceStorage.size(), true, true); + mutation.Add(Cmask.data(), Cmask.size(), true, true); + } + run(*device, ColorTileMode::Linear, "linear"); + run(*device, ColorTileMode::RenderTarget, "SW_64KB_R_X"); + invalidCode(*device); + device->WaitIdle(); + device.reset(); + { + GuestAllocations::Mutation mutation; + mutation.Remove(SurfaceStorage.data()); + mutation.Remove(Cmask.data()); + } + std::cout << "CMASK fast clear draw and eliminate tests passed\n"; + return 0; + } catch (const std::exception& error) { + std::cerr << error.what() << '\n'; + return 1; + } +} diff --git a/core/libs/prx/libSceAgcDriver/tests/ColorTargetLayout.cpp b/core/libs/prx/libSceAgcDriver/tests/ColorTargetLayout.cpp index 5671eb457..5d02bebda 100644 --- a/core/libs/prx/libSceAgcDriver/tests/ColorTargetLayout.cpp +++ b/core/libs/prx/libSceAgcDriver/tests/ColorTargetLayout.cpp @@ -6,6 +6,7 @@ #include #include #include +#include #include namespace { @@ -77,6 +78,33 @@ void RunColorTargetLayoutTests() { } Require(DecodeColorTileMode(0x4dc24000) == ColorTileMode::Standard64KB, "64 KiB standard color descriptor was rejected"); reject([&] { layout.Offset(257, 0); }); + Require(CmaskLayout(1, 1).Bytes() == 4096 && CmaskLayout(1024, 512).Bytes() == 4096 && CmaskLayout(1025, 513).Bytes() == 16384 && CmaskLayout(1920, 1080).Bytes() == 24576 && CmaskLayout(2432, 1368).Bytes() == 36864 && CmaskLayout(3328, 1872).Bytes() == 65536 && CmaskLayout(3840, 2160).Bytes() == 81920, "CMASK size differs from addrlib's pipe-aligned SW_64KB_Z_X CMASK"); + struct CmaskReference { + std::uint32_t width; + std::uint32_t height; + std::uint32_t tileX; + std::uint32_t tileY; + std::size_t nibble; + }; + constexpr std::array cmaskReferences{{ + {20, 12, 2, 1, 0x600u}, {61, 13, 7, 1, 0x1401u}, {61, 13, 5, 0, 0x1201u}, {1920, 1080, 239, 134, 0xb347u}, {1920, 1080, 128, 64, 0x6000u}, + {1920, 1080, 1, 1, 0x1u}, {3840, 2160, 479, 269, 0x2651du}, {3840, 2160, 200, 77, 0xb30cu}, {2432, 1368, 303, 170, 0x10acfu}, {1025, 513, 128, 64, 0x6000u}, + }}; + for (const auto& reference : cmaskReferences) Require(CmaskLayout(reference.width, reference.height).Nibble(reference.tileX, reference.tileY) == reference.nibble, "CMASK nibble address differs from addrlib's"); + for (const auto [width, height] : {std::pair{61u, 13u}, std::pair{1920u, 1080u}}) { + const CmaskLayout cmask(width, height); + std::vector seen(cmask.Bytes() * 2u); + for (std::uint32_t tileY = 0; tileY < cmask.TilesY(); ++tileY) { + for (std::uint32_t tileX = 0; tileX < cmask.TilesX(); ++tileX) { + const auto nibble = cmask.Nibble(tileX, tileY); + Require(nibble < seen.size() && !seen[nibble], "CMASK tiles alias or leave the CMASK"); + seen[nibble] = true; + } + } + } + reject([] { CmaskLayout(0, 1); }); + reject([] { CmaskLayout(61, 13).Nibble(8, 0); }); + reject([] { CmaskLayout(61, 13).Nibble(0, 2); }); std::vector tiled(layout.Bytes(), std::byte{0x5a}); std::vector linear(layout.LinearBytes()); std::vector restored(linear.size()); diff --git a/core/libs/prx/libSceAgcDriver/tests/Graphics.cpp b/core/libs/prx/libSceAgcDriver/tests/Graphics.cpp index e3f213aa0..d2c2e9160 100644 --- a/core/libs/prx/libSceAgcDriver/tests/Graphics.cpp +++ b/core/libs/prx/libSceAgcDriver/tests/Graphics.cpp @@ -1171,7 +1171,7 @@ void metadataPassTests() { } } -alignas(256) std::array cmaskMemory{}; +alignas(4096) std::array cmaskMemory{}; void cmaskTests() { using AgcDriver::Graphics::DecodeColorBuffer; @@ -1182,8 +1182,15 @@ void cmaskTests() { queue.context[0x398] = static_cast(cmaskAddress >> 40u); queue.context[0x323] = 0x11223344; queue.context[0x324] = 0; + queue.context[0x3b8] = 0x0dc00000u; const auto target = DecodeColorBuffer(queue.context, 0); Require(target.cmaskAddress == cmaskAddress && target.cmaskBytes == cmaskMemory.size(), "the fast-clear target lost its CMASK"); + std::vector log; + AgcDriver::Graphics::RegisterReadLog() = &log; + static_cast(DecodeColorBuffer(queue.context, 0)); + AgcDriver::Graphics::RegisterReadLog() = nullptr; + Require(!log.empty(), "the register facade recorded nothing for a fast-clear target"); + for (const auto read : log) Require(AgcDriver::Graphics::DrawKeyCovers(read), "DrawKeyRegisters lacks a register the fast-clear target decode reads: " + std::to_string(read.offset)); Require(DecodeColorBuffer(makeState().context, 0).cmaskAddress == 0, "a target without FAST_CLEAR got a CMASK"); using AgcDriver::Graphics::CmaskBytes; Require(CmaskBytes(1920, 1080) == 0x6000 && CmaskBytes(960, 544) == 0x2000 && CmaskBytes(800, 450) == 0x1000 && CmaskBytes(1024, 512) == 0x1000 && CmaskBytes(3840, 2160) == 0x14000, "a CMASK does not span whole 1024x512 metablocks"); @@ -1194,6 +1201,16 @@ void cmaskTests() { unaddressed.context[0x31f] = 0; unaddressed.context[0x398] = 0; expectFailure([&] { DecodeColorBuffer(unaddressed.context, 0); }, "without a CMASK address"); + auto unaligned = queue; + unaligned.context[0x3b8] = 0x09c00000u; + expectFailure([&] { DecodeColorBuffer(unaligned.context, 0); }, "pipe-aligned SW_64KB_Z_X"); + unaligned.context[0x3b8] = 0x0d000000u; + expectFailure([&] { DecodeColorBuffer(unaligned.context, 0); }, "pipe-aligned SW_64KB_Z_X"); + unaligned = queue; + unaligned.context[0x31f] += 1u; + expectFailure([&] { DecodeColorBuffer(unaligned.context, 0); }, "misaligned"); + using AgcDriver::Graphics::CmaskLayout; + Require(CmaskBytes(61, 13) == CmaskLayout(61, 13).Bytes() && CmaskBytes(3840, 2160) == CmaskLayout(3840, 2160).Bytes(), "the CMASK size and layout disagree"); queue.context[0x0] = 0; queue.context[0x202] = 0xcc0020; @@ -1216,9 +1233,42 @@ void cmaskTests() { std::memset(colorMemory.data(), 0x5a, colorMemory.size()); AgcDriver::Graphics::RunColorMetadataPass(context, *pass); Require(texels(0x5a5a5a5au) && cmaskIs(0xff), "a pass over an expanded CMASK changed the texels"); + const CmaskLayout layout(64, 4); + const auto setTile = [&](std::uint32_t tileX, std::uint8_t code) { + const auto nibble = layout.Nibble(tileX, 0); + auto& entry = cmaskMemory[nibble / 2u]; + const auto shift = (nibble % 2u) * 4u; + entry = static_cast((entry & ~(0xfu << shift)) | (code << shift)); + }; + const auto tiles = [&](const auto& cleared) { + for (std::size_t offset = 0; offset < colorMemory.size(); offset += 4) { + std::uint32_t texel = 0; + std::memcpy(&texel, colorMemory.data() + offset, 4); + if (texel != (cleared((offset / 4u) % 64u / 8u) ? 0x11223344u : 0x5a5a5a5au)) return false; + } + return true; + }; + std::memset(colorMemory.data(), 0x5a, colorMemory.size()); cmaskMemory.fill(0); cmaskMemory[cmaskMemory.size() - 1] = 0xff; - expectFailure([&] { AgcDriver::Graphics::RunColorMetadataPass(context, *pass); }, "not all fast-cleared or all expanded"); + AgcDriver::Graphics::RunColorMetadataPass(context, *pass); + Require(texels(0x11223344) && cmaskIs(0xff), "CMASK nibbles past the target's tiles changed what its tiles hold"); + std::memset(colorMemory.data(), 0x5a, colorMemory.size()); + cmaskMemory.fill(0xff); + setTile(1, 0); + setTile(7, 0); + AgcDriver::Graphics::RunColorMetadataPass(context, *pass); + Require(tiles([](std::uint32_t tile) { return tile == 1 || tile == 7; }) && cmaskIs(0xff), "a partly fast-cleared CMASK did not clear exactly its cleared tiles"); + std::memset(colorMemory.data(), 0x5a, colorMemory.size()); + cmaskMemory.fill(0xff); + setTile(4, 0); + AgcDriver::Graphics::RunColorMetadataPass(context, {AgcDriver::Graphics::ColorMetadataPass::Mode::DccDecompress, pass->targets}); + Require(tiles([](std::uint32_t tile) { return tile == 4; }) && cmaskIs(0xff), "a DCC decompress pass did not eliminate the CMASK fast clear"); + std::memset(colorMemory.data(), 0x5a, colorMemory.size()); + cmaskMemory.fill(0xff); + setTile(2, 0); + setTile(3, 3); + expectFailure([&] { AgcDriver::Graphics::RunColorMetadataPass(context, *pass); }, "not a single-sample fast-clear code"); Require(texels(0x5a5a5a5au), "a refused pass changed the texels"); queue.context[0x31c] |= 0x10000000; @@ -1234,7 +1284,7 @@ void cmaskTests() { Require(texels(0x11223344) && cmaskIs(0xff) && std::ranges::all_of(dccKeys, [](std::uint8_t key) { return key == 0xff; }), "a DCC fast clear under an expanded CMASK was not eliminated through its keys"); std::memset(colorMemory.data(), 0x5a, colorMemory.size()); cmaskMemory.fill(0); - expectFailure([&] { AgcDriver::Graphics::RunColorMetadataPass(context, *pass); }, "DCC color target that is not all expanded"); + expectFailure([&] { AgcDriver::Graphics::RunColorMetadataPass(context, *pass); }, "DCC color target"); Require(texels(0x5a5a5a5au), "a refused pass changed the texels of a DCC target"); } diff --git a/docs/dev/TechnicalDebt.md b/docs/dev/TechnicalDebt.md index ca6231cc9..07bce621d 100644 --- a/docs/dev/TechnicalDebt.md +++ b/docs/dev/TechnicalDebt.md @@ -341,7 +341,7 @@ Throughout the project, every function at every stage either **does exactly what - The Linux placement of the `--to-intel` stubs in the [ELF patcher](../../core/relinker/elfpatcher/src/linux/LinuxElfPatcher.cpp) is covered only by a synthetic test. - The libc [SSE4a trap emulation](../../core/libs/prx/libc/src/specifics/windows/Sse4aEmulation.hpp) on Windows is superseded by `--to-intel` and remains only until the relinked title has been verified without it. - [Descriptor pool chain](../../core/libs/prx/libSceAgcDriver/Graphics/src/ShaderResources.cpp) (libSceAgcDriver) - each pool holds two storage buffer descriptors more than a set can take, as a workaround: RADV (Mesa 26.0 and later) lets a `FREE_DESCRIPTOR_SET` pool place a set up to 32 bytes past the pool's buffer ([`util_vma_heap_init`](https://gitlab.freedesktop.org/mesa/mesa/-/blob/main/src/amd/vulkan/radv_descriptor_pool.c) is given `bo_size + RADV_POOL_HEAP_OFFSET` as a size), and with a page-multiple pool size those bytes are an unmapped page -- [CMASK fast clears](../../core/libs/prx/libSceAgcDriver/Graphics/src/Draw.cpp) (libSceAgcDriver) - only a CMASK whose nibbles are all fast-cleared (0) or all expanded (0xF) is modeled, checked over the target's whole CMASK, 4 KiB per 1024x512-pixel metablock (the size PS5 titles clear, for every color format seen); the addressing within a metablock is not decoded. The first draw or fast-clear eliminate over a fast-cleared CMASK fills the whole target with `CB_COLOR_CLEAR_WORD` and stores those bytes as expanded, so a draw that covers only part of the target reads the clear value in the rest as the eliminate would leave it. On a DCC target the clear is in the DCC keys, so its CMASK must be all expanded. A mixed CMASK throws, as does a DCC target's CMASK that is not all expanded and a fast clear of a mipmapped, 3D or array color target +- [CMASK fast clears](../../core/libs/prx/libSceAgcDriver/Graphics/src/Draw.cpp) (libSceAgcDriver) - each fast-cleared CMASK tile (code 0) is stored as `CB_COLOR_CLEAR_WORD` into its 8x8 texels at the first draw into the target or at its fast-clear eliminate or DCC decompress pass, not as the color block writes the tile, and the CMASK is then stored as expanded (0xF): a texture or copy read of the target after that draw and before the title's eliminate sees the clear value in tiles no draw wrote, where the console keeps their previous bytes. Tile addresses follow the pipe-aligned SW_64KB_Z_X CMASK equation for 16 pipes; other CMASK swizzle modes, codes other than 0 and 0xF, a cleared tile of a DCC target and a fast clear of a mipmapped, 3D or array color target throw. When the clear runs as a compute pass (a resident target whose CMASK is host-imported), an invalid code throws at the next fast-clear materialization, not at the one that read it - [Host imports](../../core/libs/prx/libSceAgcDriver/Graphics/src/GuestBufferMemory.cpp) (libSceAgcDriver) on Linux skip ranges that are not mapped read-write, which are copied instead. The NVIDIA driver (595.91.07) refuses `VK_EXT_external_memory_host` imports of read-only or executable mappings (and shared mappings of regular files) with `VK_ERROR_OUT_OF_DEVICE_MEMORY`, after which every `vkAllocateMemory` on the device fails the same way. Read-only ranges are never imported, even on drivers that accept them. Shared mappings of regular files are not checked for. - [Cube texture descriptors](../../core/libs/prx/libSceAgcDriver/Graphics/src/GuestTextureResource.cpp) with equal `BASE_ARRAY` and `DEPTH` expose six faces starting at `BASE_ARRAY`. Measured with `image_load`, `image_sample` and `image_get_resinfo` on RX 6600 (gfx1032), using 64x64 R32_UINT faces and base slices 0, 5 and 6. Other ranges containing fewer than six faces or incomplete cube arrays still throw: hardware load bounds and filtered-sample addressing differ. Seamless cube filtering and out-of-range cube-array sampling remain unimplemented. PS5 equivalence has not been measured on console hardware. - [DCC display buffers](../../core/libs/prx/libSceAgcDriver/Execution/src/VulkanDevice.cpp) are presented only when all keys of the console's DCC extent agree on uncompressed or a fast-clear code, padding keys included; other keys, and register-clear keys over a pending image, throw.