diff --git a/src/CMakeLists.txt b/src/CMakeLists.txt index 718cc838..def9299d 100644 --- a/src/CMakeLists.txt +++ b/src/CMakeLists.txt @@ -19,6 +19,7 @@ set(CMAKE_CXX_STANDARD_REQUIRED ON) set(CMAKE_CXX_EXTENSIONS OFF) include(utils.cmake) +include(CTest) set(KYTY_THIRD_PARTY_DIR "${CMAKE_CURRENT_SOURCE_DIR}/../3rdparty") @@ -183,6 +184,9 @@ set(gpu_tiler_shader_names prt_3d render_target depth + promote_d16 + demote_d16 + swap_bgra16 ) file(GLOB gpu_tiler_shader_includes CONFIGURE_DEPENDS "${gpu_tiler_shader_dir}/gpu_tiler_*.inc") foreach(shader_name IN LISTS gpu_tiler_shader_names) @@ -204,6 +208,30 @@ foreach(shader_name IN LISTS gpu_tiler_shader_names) endforeach() list(APPEND kyty_emulator_src ${gpu_tiler_shader_headers}) +set(gpu_blit_generated_dir "${PROJECT_BINARY_DIR}/gpu_blit_shaders") +set(gpu_blit_shader_sources + "${gpu_tiler_shader_dir}/gpu_blit_fs_triangle.vert" + "${gpu_tiler_shader_dir}/gpu_blit_color_to_ms_depth.frag" +) +foreach(shader_source IN LISTS gpu_blit_shader_sources) + get_filename_component(shader_name "${shader_source}" NAME_WE) + set(shader_spv "${gpu_blit_generated_dir}/${shader_name}.spv") + set(shader_header "${gpu_blit_generated_dir}/${shader_name}_spv.h") + string(TOUPPER "${shader_name}_SPV" shader_symbol) + add_custom_command( + OUTPUT "${shader_header}" + COMMAND ${CMAKE_COMMAND} -E make_directory "${gpu_blit_generated_dir}" + COMMAND "${KYTY_GLSLANG_VALIDATOR}" -V --target-env vulkan1.0 -Os + "-I${gpu_tiler_shader_dir}" -o "${shader_spv}" "${shader_source}" + COMMAND ${CMAKE_COMMAND} -DINPUT=${shader_spv} -DOUTPUT=${shader_header} + -DSYMBOL=${shader_symbol} -P "${CMAKE_CURRENT_SOURCE_DIR}/embed_spirv.cmake" + DEPENDS "${shader_source}" + VERBATIM + ) + list(APPEND gpu_blit_shader_headers "${shader_header}") +endforeach() +list(APPEND kyty_emulator_src ${gpu_blit_shader_headers}) + list(APPEND kyty_emulator_src emulator.h emulator.cpp @@ -318,6 +346,14 @@ add_executable(resource_mutex_tests EXCLUDE_FROM_ALL target_link_libraries(resource_mutex_tests common) target_include_directories(resource_mutex_tests PRIVATE ${inc_headers}) +add_executable(event_queue_lifetime_tests EXCLUDE_FROM_ALL + ../tests/EventQueueLifetimeTests.cpp + kernel/eventQueue.cpp + loader/timer.cpp +) +target_link_libraries(event_queue_lifetime_tests common fmt::fmt) +target_include_directories(event_queue_lifetime_tests PRIVATE ${inc_headers}) + add_executable(image_page_table_tests EXCLUDE_FROM_ALL ../tests/ImagePageTableTests.cpp ) @@ -325,10 +361,67 @@ target_include_directories(image_page_table_tests PRIVATE ${inc_headers}) add_kyty_full_emulator_test(shader_recompiler_compute_tests ../tests/ShaderRecompilerComputeTests.cpp) +set(gpu_test_generated_dir "${PROJECT_BINARY_DIR}/gpu_test_shaders") +set(gpu_test_ms_depth_source + "${gpu_tiler_shader_dir}/gpu_test_ms_depth.comp") +set(gpu_test_ms_depth_spv + "${gpu_test_generated_dir}/gpu_test_ms_depth.spv") +set(gpu_test_ms_depth_header + "${gpu_test_generated_dir}/gpu_test_ms_depth_spv.h") +add_custom_command( + OUTPUT "${gpu_test_ms_depth_header}" + COMMAND ${CMAKE_COMMAND} -E make_directory "${gpu_test_generated_dir}" + COMMAND "${KYTY_GLSLANG_VALIDATOR}" -V --target-env vulkan1.0 -Os + -o "${gpu_test_ms_depth_spv}" "${gpu_test_ms_depth_source}" + COMMAND ${CMAKE_COMMAND} -DINPUT=${gpu_test_ms_depth_spv} + -DOUTPUT=${gpu_test_ms_depth_header} -DSYMBOL=GPU_TEST_MS_DEPTH_SPV + -P "${CMAKE_CURRENT_SOURCE_DIR}/embed_spirv.cmake" + DEPENDS "${gpu_test_ms_depth_source}" + VERBATIM +) +target_sources(shader_recompiler_compute_tests PRIVATE + "${gpu_test_ms_depth_header}") + add_kyty_full_emulator_test(virtual_memory_allocation_tests ../tests/VirtualMemoryAllocationTests.cpp) target_compile_definitions(virtual_memory_allocation_tests PRIVATE KYTY_VIRTUAL_MEMORY_ALLOCATION_TESTS=1) +if(BUILD_TESTING) + add_test(NAME image_page_table COMMAND $) + add_test(NAME memory_tracker COMMAND $) + add_test(NAME page_manager COMMAND $) + add_test(NAME resource_mutex COMMAND $) + add_test(NAME event_queue_lifetime COMMAND $) + add_test(NAME shader_recompiler_compute COMMAND $) + add_test(NAME command_scheduler_timeline + COMMAND $ --scheduler-only) + add_test(NAME stream_buffer_ring + COMMAND $ --stream-buffer-only) + add_test(NAME gpu_command_lane + COMMAND $ --gpu-command-lane-only) + add_test(NAME gpu_tiler + COMMAND $ --gpu-tiler-only) + + if(WIN32) + add_test(NAME texture_cache_image_overlap + COMMAND $ --image-overlap-only) + add_test(NAME texture_cache_htile_clear + COMMAND $ --htile-clear-only) + add_test(NAME texture_cache_layered_image + COMMAND $ --layered-image-only) + add_test(NAME texture_cache_image_views + COMMAND $ --image-view-cache-only) + add_test(NAME texture_cache_storage_sampled + COMMAND $ --storage-sampled-only) + add_test(NAME texture_cache_depth_readback + COMMAND $ --depth-readback-only) + add_test(NAME buffer_cache_ranges + COMMAND $ --buffer-cache-range-only) + add_test(NAME buffer_cache_dirty_gc + COMMAND $ --buffer-cache-gc-only) + endif() +endif() + add_executable(kyty_emulator main.cpp ${kyty_emulator_src}) diff --git a/src/common/emulatorConfig.cpp b/src/common/emulatorConfig.cpp index 418cfda4..d40df0f1 100644 --- a/src/common/emulatorConfig.cpp +++ b/src/common/emulatorConfig.cpp @@ -93,4 +93,8 @@ bool NggRectlistDrawEnabled() { return g_config->ngg_rectlist_draw_enabled; } +bool ReadbackLinearImagesEnabled() { + return g_config->readback_linear_images; +} + } // namespace Config diff --git a/src/common/emulatorConfig.h b/src/common/emulatorConfig.h index f2426366..8407f0a8 100644 --- a/src/common/emulatorConfig.h +++ b/src/common/emulatorConfig.h @@ -36,6 +36,7 @@ struct ConfigOptions { bool spirv_debug_printf_enabled = false; bool renderdoc_enabled = false; bool ngg_rectlist_draw_enabled = true; + bool readback_linear_images = false; }; void Load(const ConfigOptions& cfg); @@ -64,6 +65,7 @@ bool SpirvDebugPrintfEnabled(); bool RenderDocEnabled(); bool NggRectlistDrawEnabled(); +bool ReadbackLinearImagesEnabled(); } // namespace Config diff --git a/src/common/lruCache.h b/src/common/lruCache.h new file mode 100644 index 00000000..7110c807 --- /dev/null +++ b/src/common/lruCache.h @@ -0,0 +1,119 @@ +#ifndef EMULATOR_SRC_COMMON_LRUCACHE_H_ +#define EMULATOR_SRC_COMMON_LRUCACHE_H_ + +#include +#include +#include +#include + +namespace Common { + +template +class LeastRecentlyUsedCache { + struct Item { + Object object {}; + Tick tick {}; + Item* next = nullptr; + Item* prev = nullptr; + }; + +public: + [[nodiscard]] size_t Insert(Object object, Tick tick) { + const auto id = Build(); + auto& item = m_items[id]; + item.object = std::move(object); + item.tick = tick; + Attach(item); + return id; + } + + void Touch(size_t id, Tick tick) { + auto& item = m_items[id]; + if (item.tick >= tick) { + return; + } + item.tick = tick; + if (&item != m_last) { + Detach(item); + Attach(item); + } + } + + void Free(size_t id) { + auto& item = m_items[id]; + Detach(item); + item.next = nullptr; + item.prev = nullptr; + m_free.push_back(id); + } + + template + void ForEachItemBelow(Tick tick, Function&& function) { + constexpr bool ReturnsBool = + std::is_same_v, bool>; + for (auto* item = m_first; item != nullptr;) { + if (item->tick > tick) { + return; + } + auto* next = item->next; + if constexpr (ReturnsBool) { + if (function(item->object)) { + return; + } + } else { + function(item->object); + } + item = next; + } + } + +private: + [[nodiscard]] size_t Build() { + if (m_free.empty()) { + const auto id = m_items.size(); + m_items.emplace_back(); + return id; + } + const auto id = m_free.front(); + m_free.pop_front(); + return id; + } + + void Attach(Item& item) { + if (m_first == nullptr) { + m_first = &item; + } + if (m_last == nullptr) { + m_last = &item; + return; + } + item.prev = m_last; + m_last->next = &item; + item.next = nullptr; + m_last = &item; + } + + void Detach(Item& item) { + if (item.prev != nullptr) { + item.prev->next = item.next; + } + if (item.next != nullptr) { + item.next->prev = item.prev; + } + if (m_first == &item) { + m_first = item.next; + } + if (m_last == &item) { + m_last = item.prev; + } + } + + std::deque m_items; + std::deque m_free; + Item* m_first = nullptr; + Item* m_last = nullptr; +}; + +} // namespace Common + +#endif // EMULATOR_SRC_COMMON_LRUCACHE_H_ diff --git a/src/common/uniqueFunction.h b/src/common/uniqueFunction.h new file mode 100644 index 00000000..c72e6733 --- /dev/null +++ b/src/common/uniqueFunction.h @@ -0,0 +1,56 @@ +#ifndef KYTY_COMMON_UNIQUEFUNCTION_H_ +#define KYTY_COMMON_UNIQUEFUNCTION_H_ + +#include +#include +#include + +namespace Common { + +template +class UniqueFunction { + class CallableBase { + public: + virtual ~CallableBase() = default; + virtual Result Invoke(Args&&... args) = 0; + }; + + template + class Callable final: public CallableBase { + public: + explicit Callable(Function function): m_function(std::move(function)) {} + + Result Invoke(Args&&... args) override { + return m_function(std::forward(args)...); + } + + private: + Function m_function; + }; + +public: + UniqueFunction() = default; + + template + UniqueFunction(Function&& function) + : m_callable(std::make_unique>>( + std::forward(function))) {} + + UniqueFunction(UniqueFunction&&) noexcept = default; + UniqueFunction& operator=(UniqueFunction&&) noexcept = default; + UniqueFunction(const UniqueFunction&) = delete; + UniqueFunction& operator=(const UniqueFunction&) = delete; + + Result operator()(Args... args) const { + return m_callable->Invoke(std::forward(args)...); + } + + explicit operator bool() const noexcept { return m_callable != nullptr; } + +private: + std::unique_ptr m_callable; +}; + +} // namespace Common + +#endif // KYTY_COMMON_UNIQUEFUNCTION_H_ diff --git a/src/graphics/guest_gpu/command_processor/commandProcessor.h b/src/graphics/guest_gpu/command_processor/commandProcessor.h index c685988e..32261345 100644 --- a/src/graphics/guest_gpu/command_processor/commandProcessor.h +++ b/src/graphics/guest_gpu/command_processor/commandProcessor.h @@ -11,8 +11,6 @@ namespace Libs::Graphics { -inline constexpr uint32_t AcquireGcrGl2Writeback = 1u << 15u; - bool TestWaitRegMemValue(uint64_t value, uint64_t ref, uint64_t mask, uint32_t func); enum class Pm4ProcessResult { Complete, Blocked }; @@ -45,7 +43,7 @@ public: int64_t flip_arg = 0; }; - CommandProcessor() = default; + explicit CommandProcessor(RenderContext& renderer): m_renderer(renderer) {} ~CommandProcessor() = default; KYTY_CLASS_NO_COPY(CommandProcessor); @@ -60,16 +58,13 @@ public: if (m_readback_active) { EXIT("nested command-processor readback transaction\n"); } - m_readback_active = true; - m_readback_finished = false; + m_readback_active = true; } - void FinishReadbackTransaction(); void EndReadbackTransaction() { if (!m_readback_active) { EXIT("command-processor readback transaction is not active\n"); } - m_readback_active = false; - m_readback_finished = false; + m_readback_active = false; } HW::Context& GetCtx() { return m_ctx; } @@ -106,12 +101,10 @@ public: void Flip(void* dst_gpu_addr, uint32_t value); void FlipWithInterrupt(uint32_t eop_event_type, uint32_t cache_action, void* dst_gpu_addr, uint32_t value); - void PrepareCpuFlip(); + void PrepareCpuFlip(uint64_t request_id); void SynchronizeGpu(); - void MemoryBarrier(); + void EmitGlobalBarrier(); void TriggerEopEventAtEndOfPipe(uint32_t interrupt_context_id); - void RenderTextureBarrier(uint64_t vaddr, uint64_t size); - void DepthStencilBarrier(uint64_t vaddr, uint64_t size); void DispatchDirect(uint32_t thread_group_x, uint32_t thread_group_y, uint32_t thread_group_z, uint32_t mode); void DispatchIndirect(uint32_t data_offset, uint32_t mode); @@ -160,11 +153,12 @@ private: void ProcessPm4(Pm4Execution& execution, size_t stop_depth); void SuspendPm4(); - CommandScheduler& GetScheduler() const { return GetRenderContext().GetCommandScheduler(); } + CommandScheduler& GetScheduler() const { return m_renderer.GetCommandScheduler(); } RenderCommandBuffer& CurrentBuffer() { return GetScheduler().Current(); } void CheckBuffer() const { GetScheduler().CheckActive(); } - GpuResourceManager& GetGpuResources() const { return GetRenderContext().GetGpuResources(); } + GpuResourceManager& GetGpuResources() const { return m_renderer.GetGpuResources(); } + RenderContext& m_renderer; HW::Context m_ctx; HW::UserConfig m_ucfg; HW::Shader m_sh_ctx; @@ -179,8 +173,7 @@ private: uint32_t m_de_count = 0; uint32_t m_ce_count = 0; bool m_ce_complete = false; - bool m_readback_active = false; - bool m_readback_finished = false; + bool m_readback_active = false; uint32_t m_const_ram[0x3000] = {0}; diff --git a/src/graphics/guest_gpu/command_processor/pm4Handlers.cpp b/src/graphics/guest_gpu/command_processor/pm4Handlers.cpp index 7a3fef8a..e7a27f38 100644 --- a/src/graphics/guest_gpu/command_processor/pm4Handlers.cpp +++ b/src/graphics/guest_gpu/command_processor/pm4Handlers.cpp @@ -6,7 +6,6 @@ #include "graphics/guest_gpu/command_processor/pm4Dispatch.h" #include "graphics/guest_gpu/graphicsRun.h" #include "graphics/host_gpu/graphicContext.h" -#include "graphics/host_gpu/objects/label.h" #include "graphics/host_gpu/renderer/render.h" #include "graphics/host_gpu/renderer/renderContext.h" #include "graphics/presentation/videoOut.h" @@ -1764,230 +1763,11 @@ static bool HwUcTrySetFakeRegisterRange(uint32_t cmd_offset, const uint32_t* buf return true; } -// NOLINTNEXTLINE(readability-function-cognitive-complexity) KYTY_CP_OP_PARSER(CpOpAcquireMem) { KYTY_PROFILER_FUNCTION(); EXIT_NOT_IMPLEMENTED(cmd_id != 0xC0055800 && cmd_id != 0xc0061050); - - bool custom = (cmd_id == 0xc0061050); - - uint32_t engine = buffer[0] >> 31u; - uint32_t stall_mode = (custom ? 1u : engine); - uint32_t cache_action = buffer[0] & 0x7fffffffu; - uint64_t size_lo = buffer[1]; - uint32_t size_hi = buffer[2]; - uint64_t base_lo = buffer[3]; - uint32_t base_hi = buffer[4]; - uint32_t poll = buffer[5]; - [[maybe_unused]] uint32_t gcr_cntl = (custom ? buffer[6] : 0); - - uint32_t target_mask = cache_action & 0x00007FC0u; - uint32_t extended_action = cache_action & 0x2E000000u; - uint32_t action = - ((cache_action & 0x00C00000u) >> 0x12u) | ((cache_action & 0x00058000u) >> 0xfu); - - if (custom && engine > 1) { - LOGF("\t warning: custom acquire_mem unsupported engine: %" PRIu32 "\n", engine); - } - - // EXIT_NOT_IMPLEMENTED(stall_mode != 1); - EXIT_NOT_IMPLEMENTED(size_hi != 0); - EXIT_NOT_IMPLEMENTED(base_hi != 0); - if (poll != 10) { - LOGF("\t warning: acquire_mem unexpected poll interval: %" PRIu32 "\n", poll); - } - - switch (cache_action) { - case 0x00000000: { - if (custom && gcr_cntl != 0) { - LOGF("\t custom acquire_mem GCR-only barrier, gcr_cntl = 0x%08" PRIx32 - ", base = 0x%016" PRIx64 ", size = 0x%016" PRIx64 "\n", - gcr_cntl, base_lo << 8u, size_lo << 8u); - - cp.MemoryBarrier(); - if ((gcr_cntl & AcquireGcrGl2Writeback) != 0) { - cp.SynchronizeGpu(); - } - } - } break; - case 0x00000040: - case 0x00003fc0: - case 0x00004000: - case 0x00007fc0: { - // target_mask set, no CB/DB action bits. Treat as an ordering barrier. - EXIT_IF(target_mask != cache_action); - EXIT_IF(extended_action != 0x00000000); - EXIT_IF(action != 0x00); - - LOGF("\t temporary: acquire_mem target-mask-only barrier, target_mask = 0x%08" PRIx32 - ", gcr_cntl = 0x%08" PRIx32 ", base = 0x%016" PRIx64 ", size = 0x%016" PRIx64 "\n", - target_mask, gcr_cntl, base_lo << 8u, size_lo << 8u); - - cp.MemoryBarrier(); - } break; - case 0x02000000: { - // target_mask: 0x00000000 (none) - // extended_action: 0x02000000 (FlushAndInvalidateCbCache) - // action: 0x00 (none) - EXIT_IF(target_mask != 0x00000000); - EXIT_IF(extended_action != 0x02000000); - EXIT_IF(action != 0x00); - - LOGF("\t temporary: acquire_mem CB-cache-only barrier, gcr_cntl = 0x%08" PRIx32 - ", base = 0x%016" PRIx64 ", size = 0x%016" PRIx64 "\n", - gcr_cntl, base_lo << 8u, size_lo << 8u); - - cp.MemoryBarrier(); - } break; - case 0x04000000: { - // target_mask: 0x00000000 (none) - // extended_action: 0x04000000 (FlushAndInvalidateDbCache) - // action: 0x00 (none) - EXIT_IF(target_mask != 0x00000000); - EXIT_IF(extended_action != 0x04000000); - EXIT_IF(action != 0x00); - - LOGF("\t temporary: acquire_mem DB-cache-only barrier, gcr_cntl = 0x%08" PRIx32 - ", base = 0x%016" PRIx64 ", size = 0x%016" PRIx64 "\n", - gcr_cntl, base_lo << 8u, size_lo << 8u); - - cp.MemoryBarrier(); - } break; - case 0x04004000: - case 0x04007fc0: { - // target_mask: 0x00004000 (Depth Target), 0x00007fc0 (all rt and depth) - // extended_action: 0x04000000 (FlushAndInvalidateDbCache) - // action: 0x00 (none) - EXIT_IF(target_mask != 0x00004000 && target_mask != 0x00007FC0); - EXIT_IF(extended_action != 0x04000000); - EXIT_IF(action != 0x00); - - LOGF("\t temporary: acquire_mem DB target barrier, target_mask = 0x%08" PRIx32 - ", gcr_cntl = 0x%08" PRIx32 ", base = 0x%016" PRIx64 ", size = 0x%016" PRIx64 "\n", - target_mask, gcr_cntl, base_lo << 8u, size_lo << 8u); - - if (size_lo != 0) { - cp.DepthStencilBarrier(base_lo << 8u, size_lo << 8u); - } else { - cp.MemoryBarrier(); - } - } break; - case 0x02c40040: - case 0x02c43fc0: - case 0x02c47fc0: { - // target_mask: 0x00000040 (rt0), 0x00003fc0 (all rt), 0x00007fc0 (all rt and depth) - // extended_action: 0x02000000 (FlushAndInvalidateCbCache) - // action: 0x38 (WriteBackAndInvalidateL1andL2) - EXIT_IF(target_mask != 0x00000040 && target_mask != 0x00003FC0 && - target_mask != 0x00007FC0); - EXIT_IF(extended_action != 0x02000000); - EXIT_IF(action != 0x38); - EXIT_NOT_IMPLEMENTED(size_lo == 0); - EXIT_NOT_IMPLEMENTED(base_lo == 0); - - cp.RenderTextureBarrier(base_lo << 8u, size_lo << 8u); - cp.SynchronizeGpu(); - } break; - case 0x02003fc0: - case 0x02007fc0: { - // target_mask: 0x00003FC0 (all rt), 0x00007fc0 (all rt and depth) - // extended_action: 0x02000000 (FlushAndInvalidateCbCache) - // action: 0x00 (none) - EXIT_IF(target_mask != 0x00003FC0 && target_mask != 0x00007fc0); - EXIT_IF(extended_action != 0x02000000); - EXIT_IF(action != 0x00); - - if (size_lo == 0) { - if (base_lo != 0) { - LOGF("\t warning: acquire_mem CB-cache barrier with non-zero base"); - } - - cp.MemoryBarrier(); - } else { - EXIT_NOT_IMPLEMENTED(base_lo == 0); - - cp.RenderTextureBarrier(base_lo << 8u, size_lo << 8u); - } - } break; - case 0x00C40000: { - // target_mask: 0x00000000 (none) - // extended_action: 0x00000000 (none) - // action: 0x38 (WriteBackAndInvalidateL1andL2) - EXIT_IF(target_mask != 0x00000000); - EXIT_IF(extended_action != 0x00000000); - EXIT_IF(action != 0x38); - EXIT_NOT_IMPLEMENTED(size_lo != 1); - EXIT_NOT_IMPLEMENTED(base_lo != 0); - - cp.MemoryBarrier(); - cp.SynchronizeGpu(); - } break; - case 0x00400000: { - // target_mask: 0x00000000 (none) - // extended_action: 0x00000000 (none) - // action: 0x10 (InvalidateL1) - EXIT_IF(target_mask != 0x00000000); - EXIT_IF(extended_action != 0x00000000); - EXIT_IF(action != 0x10); - EXIT_NOT_IMPLEMENTED(size_lo != 1); - EXIT_NOT_IMPLEMENTED(base_lo != 0); - - cp.MemoryBarrier(); - } break; - case 0x04c44000: { - // target_mask: 0x00004000 (Depth Target) - // extended_action: 0x04000000 (FlushAndInvalidateDbCache) - // action: 0x38 (WriteBackAndInvalidateL1andL2) - EXIT_IF(target_mask != 0x00004000); - EXIT_IF(extended_action != 0x04000000); - EXIT_IF(action != 0x38); - - cp.DepthStencilBarrier(base_lo << 8u, size_lo << 8u); - cp.SynchronizeGpu(); - } break; - - case 0x06000040: - case 0x06000080: - case 0x06003fc0: - case 0x06007fc0: { - // target_mask: 0x00000040 (rt0), 0x00000080 (rt1), 0x00003fc0 (all rt), 0x00007fc0 - // (all rt and depth) extended_action: 0x06000000 (Flush Cb & Db) action: 0x00 - // (none) - if (gcr_cntl != 0 && gcr_cntl != 0x280 && gcr_cntl != 0x300) { - LOGF("\t temporary: acquire_mem CB+DB barrier with unhandled GCR control " - "0x%08" PRIx32 "\n", - gcr_cntl); - } - - EXIT_IF(target_mask != 0x00000040 && target_mask != 0x00000080 && - target_mask != 0x00003fc0 && target_mask != 0x00007fc0); - EXIT_IF(extended_action != 0x06000000); - EXIT_IF(action != 0x00); - - if (size_lo != 0) { - if ((target_mask & 0x00003fc0) != 0) { - cp.RenderTextureBarrier(base_lo << 8u, size_lo << 8u); - } - if ((target_mask & 0x00004000) != 0) { - cp.DepthStencilBarrier(base_lo << 8u, size_lo << 8u); - } - } else { - cp.MemoryBarrier(); - } - } break; - - default: - EXIT("unknown barrier: 0x%08" PRIx32 ", 0x%08" PRIx32 ", 0x%08" PRIx32 ", 0x%08" PRIx32 - "\n", - cache_action, target_mask, extended_action, action); - } - - if (stall_mode == 0) { - cp.BufferWait(); - } - - return (custom ? 7 : 6); + return (cmd_id == 0xc0061050 ? 7 : 6); } KYTY_CP_OP_PARSER(CpOpDispatchDirect) { @@ -2930,7 +2710,7 @@ KYTY_CP_OP_PARSER(CpOpReleaseMem) { if (data_sel == 0 || interrupt_selector == 4) { if (eop_event_type != 0x28 || gcr_cntl != 0) { - cp.MemoryBarrier(); + cp.EmitGlobalBarrier(); } if (gl2_writeback) { @@ -2944,7 +2724,7 @@ KYTY_CP_OP_PARSER(CpOpReleaseMem) { if (release_dst == ReleaseMemDstMemory && dst_gpu_addr == nullptr) { if (eop_event_type != 0x28 || gcr_cntl != 0) { - cp.MemoryBarrier(); + cp.EmitGlobalBarrier(); } if (gl2_writeback) { @@ -2957,7 +2737,7 @@ KYTY_CP_OP_PARSER(CpOpReleaseMem) { } if (ReleaseMemGcrNeedsBarrier(eop_event_type, gcr_cntl)) { - cp.MemoryBarrier(); + cp.EmitGlobalBarrier(); } auto cache_action = ReleaseMemCacheActionFromGcr(gcr_cntl); diff --git a/src/graphics/guest_gpu/graphicsRun.cpp b/src/graphics/guest_gpu/graphicsRun.cpp index 599bf4fb..15144a33 100644 --- a/src/graphics/guest_gpu/graphicsRun.cpp +++ b/src/graphics/guest_gpu/graphicsRun.cpp @@ -10,11 +10,9 @@ #include "graphics/guest_gpu/command_processor/pm4Dispatch.h" #include "graphics/guest_gpu/hardwareContext.h" #include "graphics/guest_gpu/pm4.h" -#include "graphics/host_gpu/objects/label.h" #include "graphics/host_gpu/renderer/render.h" #include "graphics/host_gpu/renderer/renderContext.h" #include "graphics/host_gpu/renderer/sync.h" -#include "graphics/presentation/displayBuffer.h" #include "graphics/presentation/videoOut.h" #include "graphics/presentation/window.h" #include "graphics/shader/shader.h" @@ -28,6 +26,8 @@ #include #include #include +#include +#include #include namespace Libs::Graphics { @@ -36,6 +36,7 @@ static thread_local CommandProcessor* g_current_processor = nullptr; static thread_local Pm4Execution* g_current_execution = nullptr; static thread_local uint32_t g_submission_pause_depth = 0; static thread_local bool g_gpu_mutex_owned = false; +static thread_local bool g_gpu_thread = false; class GpuMutexLock final { public: @@ -75,32 +76,37 @@ private: std::vector m_words; }; -class Gpu { +class GpuState { public: static constexpr uint32_t ComputePipeCount = 7; static constexpr uint32_t QueuesPerComputePipe = 8; static constexpr uint32_t ComputeQueueCount = ComputePipeCount * QueuesPerComputePipe; static constexpr uint32_t QueueCount = 1 + ComputeQueueCount; - Gpu() { + explicit GpuState(RenderContext& renderer): m_renderer(renderer) { EXIT_NOT_IMPLEMENTED(!Common::Thread::IsMainThread()); - m_gfx_cp = std::make_unique(); - Common::Thread thread(ThreadRun, this); - thread.Detach(); + m_gfx_cp = std::make_unique(renderer); + m_thread = std::jthread(ThreadRun, this); } - ~Gpu() { KYTY_NOT_IMPLEMENTED; } + ~GpuState(); - KYTY_CLASS_NO_COPY(Gpu); + KYTY_CLASS_NO_COPY(GpuState); void Submit(uint32_t* cmd_draw_buffer, uint32_t num_draw_dw, uint32_t* cmd_const_buffer, uint32_t num_const_dw, bool trigger_agc_interrupt_on_done); void SubmitCompute(uint32_t queue, uint32_t* cmd_buffer, uint32_t num_dw, bool trigger_agc_interrupt_on_done); - void SubmitFlipPreparation(); + void SubmitFlipPreparation(uint64_t request_id); void Done(); void PauseSubmissions(); void ResumeSubmissions(); + void Shutdown(); + [[nodiscard]] bool IsStopping(); + void SendCommand(Common::UniqueFunction&& command); + void SendCommandSync(Common::UniqueFunction&& command); + void SendCommandSyncWithProcessor(Common::UniqueFunction&& command); int GetFrameNum(); + [[nodiscard]] static bool IsGpuThread() noexcept { return g_gpu_thread; } private: enum class SubmissionType { Graphics, Compute, FlipPreparation }; @@ -118,6 +124,7 @@ private: bool command_complete = false; bool constant_complete = false; bool blocked = false; + uint64_t flip_request_id = 0; }; void WaitLocked(); @@ -127,42 +134,100 @@ private: static void ThreadRun(void* data); CommandProcessor& GetProcessor(uint32_t queue_id); + RenderContext& m_renderer; Common::Mutex m_submission_mutex; Common::Mutex m_queue_mutex; + std::mutex m_shutdown_mutex; Common::CondVar m_work_available; Common::CondVar m_idle; std::array, QueueCount> m_queues; - uint32_t m_next_queue = 0; - uint32_t m_submission_count = 0; - bool m_processing = false; - bool m_graphics_done = true; + std::deque> m_commands; + uint32_t m_next_queue = 0; + uint32_t m_submission_count = 0; + bool m_processing = false; + bool m_graphics_done = true; + bool m_accepting = true; + bool m_stopping = false; + bool m_shutdown_complete = false; std::unique_ptr m_gfx_cp; std::array, ComputeQueueCount> m_compute_cp; uint64_t m_submit_id = 0; std::atomic_int m_done_num = 0; + std::jthread m_thread; }; -static Gpu* g_gpu = nullptr; - static bool GraphicsRunDebugDumpEnabled() { return Config::GraphicsDebugDumpEnabled() && Config::GetPrintfDirection() != Config::OutputDirection::Silent; } -void GraphicsRunInit() { - EXIT_NOT_IMPLEMENTED(!Common::Thread::IsMainThread()); - - EXIT_IF(g_gpu != nullptr); - - GraphicsInitJmpTables(); - - g_gpu = new Gpu; +GpuState::~GpuState() { + Shutdown(); } -void Gpu::Submit(uint32_t* cmd_draw_buffer, uint32_t num_draw_dw, uint32_t* cmd_const_buffer, - uint32_t num_const_dw, bool trigger_agc_interrupt_on_done) { +void GpuState::Shutdown() { + std::lock_guard shutdown_lock(m_shutdown_mutex); + if (m_shutdown_complete) { + return; + } + { + Common::LockGuard lock(m_queue_mutex); + m_accepting = false; + m_stopping = true; + m_work_available.SignalAll(); + } + if (m_thread.joinable()) { + m_thread.join(); + } + m_shutdown_complete = true; +} + +bool GpuState::IsStopping() { + Common::LockGuard lock(m_queue_mutex); + return m_stopping; +} + +void GpuState::SendCommand(Common::UniqueFunction&& command) { + EXIT_IF(!command); + if (IsGpuThread()) { + command(); + return; + } + Common::LockGuard lock(m_queue_mutex); + EXIT_IF(!m_accepting); + m_commands.push_back(std::move(command)); + m_work_available.Signal(); +} + +void GpuState::SendCommandSync(Common::UniqueFunction&& command) { + EXIT_IF(!command); + if (IsGpuThread()) { + command(); + return; + } + std::binary_semaphore done {0}; + SendCommand([operation = std::move(command), &done]() mutable { + operation(); + done.release(); + }); + done.acquire(); +} + +void GpuState::SendCommandSyncWithProcessor( + Common::UniqueFunction&& command) { + EXIT_IF(!command); + SendCommandSync([this, operation = std::move(command)]() mutable { + EXIT_IF(g_current_processor != nullptr); + g_current_processor = m_gfx_cp.get(); + operation(*m_gfx_cp); + g_current_processor = nullptr; + }); +} + +void GpuState::Submit(uint32_t* cmd_draw_buffer, uint32_t num_draw_dw, uint32_t* cmd_const_buffer, + uint32_t num_const_dw, bool trigger_agc_interrupt_on_done) { GpuMutexLock lock(m_submission_mutex); Submission submission; submission.type = SubmissionType::Graphics; @@ -175,8 +240,8 @@ void Gpu::Submit(uint32_t* cmd_draw_buffer, uint32_t num_draw_dw, uint32_t* cmd_ Enqueue(std::move(submission)); } -void Gpu::SubmitCompute(uint32_t queue, uint32_t* cmd_buffer, uint32_t num_dw, - bool trigger_agc_interrupt_on_done) { +void GpuState::SubmitCompute(uint32_t queue, uint32_t* cmd_buffer, uint32_t num_dw, + bool trigger_agc_interrupt_on_done) { GpuMutexLock lock(m_submission_mutex); constexpr uint32_t compute_queue_base = 0x20u; @@ -192,60 +257,52 @@ void Gpu::SubmitCompute(uint32_t queue, uint32_t* cmd_buffer, uint32_t num_dw, Enqueue(std::move(submission)); } -void Gpu::SubmitFlipPreparation() { +void GpuState::SubmitFlipPreparation(uint64_t request_id) { GpuMutexLock lock(m_submission_mutex); Submission submission; submission.type = SubmissionType::FlipPreparation; submission.queue_id = 0; submission.reset_processor = m_graphics_done; + submission.flip_request_id = request_id; m_graphics_done = false; Enqueue(std::move(submission)); } -void Gpu::Done() { +void GpuState::Done() { GpuMutexLock lock(m_submission_mutex); - WaitLocked(); + if (IsGpuThread()) { + m_gfx_cp->BufferWait(); + } else { + WaitLocked(); + } m_graphics_done = true; m_done_num++; } -int Gpu::GetFrameNum() { +int GpuState::GetFrameNum() { return m_done_num; } -void Gpu::WaitLocked() { +void GpuState::WaitLocked() { WaitForIdle(); - m_gfx_cp->BufferWait(); + SendCommandSync([this] { m_gfx_cp->BufferWait(); }); } -CommandProcessor& Gpu::GetProcessor(uint32_t queue_id) { +CommandProcessor& GpuState::GetProcessor(uint32_t queue_id) { EXIT_IF(queue_id >= QueueCount); if (queue_id == 0) { return *m_gfx_cp; } auto& processor = m_compute_cp[queue_id - 1]; if (processor == nullptr) { - processor = std::make_unique(); + processor = std::make_unique(m_renderer); } return *processor; } -void CommandProcessor::FinishReadbackTransaction() { - if (GraphicsRunCurrentCommandProcessor() != this || !m_readback_active) { - EXIT("GPU readback finish requires a command-processor thread\n"); - } - if (m_readback_finished) { - return; - } - GetScheduler().FinishCurrent(); - m_readback_finished = true; -} - void CommandProcessor::Reset() { BufferWait(); - Sync::DeleteBuffers(); - m_sh_ctx.Reset(); m_ucfg.Reset(); m_ctx.Reset(); @@ -396,56 +453,83 @@ void CommandProcessor::DmaData(uint8_t engine, uint8_t dst_sel, uint8_t dst_cach EXIT_NOT_IMPLEMENTED(wait_for_previous > 1); EXIT_NOT_IMPLEMENTED(write_confirm > 1); EXIT_NOT_IMPLEMENTED(block_engine > 1); - const bool dst_memory = dst_sel == 0 || dst_sel == 3; - const bool src_memory = src_sel == 0 || src_sel == 3; - if (!dst_memory) { + if (static_cast(dst_address_or_offset) == 0x3022cu) { + return; + } + auto decode_gds = [](uint8_t selector, bool& is_gds) { + switch (selector) { + case 0: + case 3: is_gds = false; return true; + case 1: is_gds = true; return true; + default: return false; + } + }; + bool dst_gds = false; + if (!decode_gds(dst_sel, dst_gds)) { EXIT("unsupported dmaData destination selector 0x%02" PRIx8 "\n", dst_sel); } + auto& buffer_cache = GetGpuResources().GetBufferCache(); if (src_sel == 2) { - GetGpuResources().FillBuffer( - CurrentBuffer(), dst_address_or_offset, num_bytes, - static_cast(src_address_or_offset_or_immediate & 0xffffffffu)); + buffer_cache.FillBuffer( + dst_address_or_offset, num_bytes, + static_cast(src_address_or_offset_or_immediate & 0xffffffffu), dst_gds); return; } - if (src_memory) { - GetGpuResources().CopyBuffer(CurrentBuffer(), dst_address_or_offset, - src_address_or_offset_or_immediate, num_bytes); - return; + bool src_gds = false; + if (!decode_gds(src_sel, src_gds)) { + EXIT("unsupported dmaData source selector 0x%02" PRIx8 "\n", src_sel); } - EXIT("unsupported dmaData source selector 0x%02" PRIx8 "\n", src_sel); + if (src_gds && dst_gds) { + EXIT("unsupported dmaData GDS-to-GDS copy\n"); + } + buffer_cache.CopyBuffer(dst_address_or_offset, src_address_or_offset_or_immediate, num_bytes, + dst_gds, src_gds); } -void Gpu::Enqueue(Submission submission) { +void GpuState::Enqueue(Submission submission) { EXIT_IF(submission.queue_id >= QueueCount); Common::LockGuard lock(m_queue_mutex); + EXIT_IF(!m_accepting); m_queues[submission.queue_id].push_back(std::move(submission)); m_submission_count++; m_work_available.Signal(); } -void Gpu::WaitForIdle() { +void GpuState::WaitForIdle() { Common::LockGuard lock(m_queue_mutex); - while (m_processing || m_submission_count != 0) { + while (m_processing || !m_commands.empty() || m_submission_count != 0) { m_idle.Wait(&m_queue_mutex); } } -void Gpu::ThreadRun(void* data) { - auto* gpu = static_cast(data); +void GpuState::ThreadRun(void* data) { + auto* gpu = static_cast(data); EXIT_IF(gpu == nullptr); KYTY_PROFILER_THREAD("Thread_Gpu"); + g_gpu_thread = true; for (;;) { - Submission submission; + Submission submission; + Common::UniqueFunction command; + bool has_submission = false; + bool should_stop = false; { Common::LockGuard lock(gpu->m_queue_mutex); - int selected_queue = -1; - while (selected_queue < 0) { - while (gpu->m_submission_count == 0) { - gpu->m_processing = false; - gpu->m_idle.Signal(); - gpu->m_work_available.Wait(&gpu->m_queue_mutex); - } + while (gpu->m_commands.empty() && gpu->m_submission_count == 0 && !gpu->m_stopping) { + gpu->m_processing = false; + gpu->m_idle.Signal(); + gpu->m_work_available.Wait(&gpu->m_queue_mutex); + } + if (gpu->m_stopping && gpu->m_commands.empty() && gpu->m_submission_count == 0) { + gpu->m_processing = false; + gpu->m_idle.SignalAll(); + should_stop = true; + } else if (!gpu->m_commands.empty()) { + command = std::move(gpu->m_commands.front()); + gpu->m_commands.pop_front(); + gpu->m_processing = true; + } else { + int selected_queue = -1; for (uint32_t offset = 0; offset < QueueCount; offset++) { const auto id = (gpu->m_next_queue + offset) % QueueCount; if (!gpu->m_queues[id].empty() && !gpu->m_queues[id].front().blocked) { @@ -461,16 +545,36 @@ void Gpu::ThreadRun(void* data) { queue.front().blocked = false; } } + continue; } + auto& queue = gpu->m_queues[static_cast(selected_queue)]; + submission = std::move(queue.front()); + queue.pop_front(); + gpu->m_submission_count--; + gpu->m_next_queue = (static_cast(selected_queue) + 1) % QueueCount; + gpu->m_processing = true; + has_submission = true; } - auto& queue = gpu->m_queues[static_cast(selected_queue)]; - submission = std::move(queue.front()); - queue.pop_front(); - gpu->m_submission_count--; - gpu->m_next_queue = (static_cast(selected_queue) + 1) % QueueCount; - gpu->m_processing = true; + } + if (should_stop) { + gpu->m_gfx_cp->BufferWait(); + g_gpu_thread = false; + return; } + if (command) { + EXIT_IF(g_current_processor != nullptr); + command(); + + Common::LockGuard lock(gpu->m_queue_mutex); + gpu->m_processing = false; + if (gpu->m_commands.empty() && gpu->m_submission_count == 0) { + gpu->m_idle.SignalAll(); + } + continue; + } + + EXIT_IF(!has_submission); const bool complete = gpu->Process(submission); Common::LockGuard lock(gpu->m_queue_mutex); @@ -486,13 +590,13 @@ void Gpu::ThreadRun(void* data) { } } gpu->m_processing = false; - if (gpu->m_submission_count == 0) { - gpu->m_idle.Signal(); + if (gpu->m_commands.empty() && gpu->m_submission_count == 0) { + gpu->m_idle.SignalAll(); } } } -bool Gpu::Process(Submission& submission) { +bool GpuState::Process(Submission& submission) { auto& cp = GetProcessor(submission.queue_id); const bool first_slice = !submission.started; @@ -537,10 +641,15 @@ bool Gpu::Process(Submission& submission) { } } if (progressed) { + if (complete) { + m_renderer.GetGpuResources().RunGarbageCollector(); + } cp.BufferFlush(); + } else if (complete) { + m_renderer.GetGpuResources().RunGarbageCollector(); } if (complete && submission.trigger_agc_interrupt_on_done) { - Sync::TriggerEopEvent(0); + m_renderer.TriggerEopEvent(0); } break; } @@ -561,30 +670,40 @@ bool Gpu::Process(Submission& submission) { complete = cp.Process(submission.command_execution, buffer, num_dw) == Pm4ProcessResult::Complete; if (submission.command_execution.MadeProgress()) { + if (complete) { + m_renderer.GetGpuResources().RunGarbageCollector(); + } cp.BufferFlush(); + } else if (complete) { + m_renderer.GetGpuResources().RunGarbageCollector(); } if (complete && submission.trigger_agc_interrupt_on_done) { Sync::TriggerAgcUserInterrupt(); } break; } - case SubmissionType::FlipPreparation: cp.PrepareCpuFlip(); break; + case SubmissionType::FlipPreparation: + m_renderer.GetGpuResources().RunGarbageCollector(); + cp.PrepareCpuFlip(submission.flip_request_id); + break; } return complete; } -void Gpu::PauseSubmissions() { +void GpuState::PauseSubmissions() { if (g_gpu_mutex_owned) { EXIT("GPU submissions are already paused by this thread\n"); } g_gpu_mutex_owned = true; m_submission_mutex.Lock(); - WaitLocked(); - LabelDrain(); + if (!IsGpuThread()) { + WaitLocked(); + } + m_renderer.GetCommandScheduler().DrainPriorityOperations(); } -void Gpu::ResumeSubmissions() { +void GpuState::ResumeSubmissions() { if (!g_gpu_mutex_owned) { EXIT("GPU submissions resumed without an active pause\n"); } @@ -823,9 +942,9 @@ void CommandProcessor::DrawIndex(uint32_t index_count, const void* index_addr, u "\n", vertex_offset_add, first_instance); } - RenderDrawIndex(m_submit_id, CurrentBuffer(), m_index_type_and_size, index_count, index_addr, - flags, type, instance_count, render_target_slice_offset, vertex_offset_add, - first_instance); + m_renderer.GetRenderExecutor().DrawIndex( + m_submit_id, CurrentBuffer(), m_index_type_and_size, index_count, index_addr, flags, type, + instance_count, render_target_slice_offset, vertex_offset_add, first_instance); } void CommandProcessor::DrawIndexOffset(uint32_t index_offset, uint32_t index_count, @@ -843,8 +962,9 @@ void CommandProcessor::DrawIndexOffset(uint32_t index_offset, uint32_t index_cou auto* index_addr = reinterpret_cast( m_index_base_addr + static_cast(index_offset) * index_size); - RenderDrawIndex(m_submit_id, CurrentBuffer(), m_index_type_and_size, index_count, index_addr, - flags, 1, m_num_instances); + m_renderer.GetRenderExecutor().DrawIndex(m_submit_id, CurrentBuffer(), + m_index_type_and_size, index_count, index_addr, + flags, 1, m_num_instances); } void CommandProcessor::DrawIndirect(uint32_t data_offset, uint32_t draw_initiator, bool indexed) { @@ -1036,7 +1156,7 @@ void CommandProcessor::DispatchDirect(uint32_t thread_group_x, uint32_t thread_g { CheckBuffer(); - frame_num = GraphicsRunGetFrameNum(); + frame_num = m_renderer.GetGpu().GetFrameNum(); if (GraphicsRunDebugDumpEnabled()) { static std::atomic log_count {0}; if (log_count.fetch_add(1, std::memory_order_relaxed) < 1024) { @@ -1070,8 +1190,8 @@ void CommandProcessor::DispatchDirect(uint32_t thread_group_x, uint32_t thread_g } } - RenderDispatchDirect(m_submit_id, CurrentBuffer(), thread_group_x, thread_group_y, - thread_group_z, mode); + m_renderer.GetRenderExecutor().DispatchDirect( + m_submit_id, CurrentBuffer(), thread_group_x, thread_group_y, thread_group_z, mode); } constexpr uint32_t DispatchInitiatorUseThreadDimensions = 1u << 5u; @@ -1117,15 +1237,16 @@ void CommandProcessor::DrawIndexAuto(uint32_t index_count, uint32_t flags, uint32_t first_vertex, uint32_t first_instance) { CheckBuffer(); - RenderDrawIndexAuto(m_submit_id, CurrentBuffer(), index_count, flags, - render_target_slice_offset, instance_count, first_vertex, first_instance); + m_renderer.GetRenderExecutor().DrawAuto( + m_submit_id, CurrentBuffer(), index_count, flags, render_target_slice_offset, + instance_count, first_vertex, first_instance); } void CommandProcessor::WaitFlipDone(uint32_t video_out_handle, uint32_t display_buffer_index) { BufferFlush(); - VideoOut::VideoOutWaitFlipDone(static_cast(video_out_handle), - static_cast(display_buffer_index)); + m_renderer.GetVideoOut().WaitFlipDone(static_cast(video_out_handle), + static_cast(display_buffer_index)); } template @@ -1196,7 +1317,8 @@ void CommandProcessor::WriteAtEndOfPipe(uint32_t cache_policy, uint32_t event_wr if (eop_event_type == 0x2f && cache_action == 0x00 && event_index == 0x06) { auto* dst = static_cast(dst_gpu_addr); SynchronizeGpu(); - Sync::ReadGds(dst, value & 0xffffu, value >> 16u); + Sync::ReadGds(m_renderer.GetBufferCache().GetGdsBuffer(), dst, + value & 0xffffu, value >> 16u); Sync::WriteAtEndOfPipeGds32(m_submit_id, CurrentBuffer(), dst, value & 0xffffu, value >> 16u); return; @@ -1355,10 +1477,23 @@ void CommandProcessor::WriteAtEndOfPipe64(uint32_t cache_policy, uint32_t event_ interrupt_context_id); } -void CommandProcessor::MemoryBarrier() { +void CommandProcessor::EmitGlobalBarrier() { CheckBuffer(); - GraphicsRenderMemoryBarrier(CurrentBuffer()); + Common::LockGuard lock(m_renderer.GetMutex()); + + vk::MemoryBarrier2 barrier {}; + barrier.srcStageMask = vk::PipelineStageFlagBits2::eAllCommands; + barrier.srcAccessMask = vk::AccessFlagBits2::eMemoryWrite; + barrier.dstStageMask = vk::PipelineStageFlagBits2::eAllCommands; + barrier.dstAccessMask = + vk::AccessFlagBits2::eMemoryRead | vk::AccessFlagBits2::eMemoryWrite; + + vk::DependencyInfo dependency {}; + dependency.memoryBarrierCount = 1; + dependency.pMemoryBarriers = &barrier; + GetScheduler().EndRendering(); + CurrentBuffer().Handle().pipelineBarrier2(dependency); } void CommandProcessor::TriggerEopEventAtEndOfPipe(uint32_t interrupt_context_id) { @@ -1367,18 +1502,6 @@ void CommandProcessor::TriggerEopEventAtEndOfPipe(uint32_t interrupt_context_id) Sync::TriggerEopEventAtEndOfPipe(CurrentBuffer(), interrupt_context_id); } -void CommandProcessor::RenderTextureBarrier(uint64_t vaddr, uint64_t size) { - CheckBuffer(); - - GraphicsRenderTextureBarrier(CurrentBuffer(), vaddr, size); -} - -void CommandProcessor::DepthStencilBarrier(uint64_t vaddr, uint64_t size) { - CheckBuffer(); - - GraphicsRenderDepthStencilBarrier(CurrentBuffer(), vaddr, size); -} - void CommandProcessor::TriggerEvent(uint32_t event_type, uint32_t event_index) { if (GraphicsRunDebugDumpEnabled()) { LOGF("CommandProcessor::TriggerEvent()\n" @@ -1392,7 +1515,7 @@ void CommandProcessor::TriggerEvent(uint32_t event_type, uint32_t event_index) { // CsPartialFlush, GsPartialFlush, PsPartialFlush. case 0x00000007: case 0x0000000f: - case 0x00000010: MemoryBarrier(); break; + case 0x00000010: EmitGlobalBarrier(); break; // CbDbDataWritebackInvalidate, CbDataWritebackInvalidate. case 0x00000016: case 0x00000031: @@ -1400,7 +1523,7 @@ void CommandProcessor::TriggerEvent(uint32_t event_type, uint32_t event_index) { EXIT("unknown event type: 0x%08" PRIx32 ", 0x%08" PRIx32 "\n", event_type, event_index); } - MemoryBarrier(); + EmitGlobalBarrier(); SynchronizeGpu(); break; // DbDataWritebackInvalidate, DbMetadataWritebackInvalidate, CbMetadataWritebackInvalidate. @@ -1411,7 +1534,7 @@ void CommandProcessor::TriggerEvent(uint32_t event_type, uint32_t event_index) { EXIT("unknown event type: 0x%08" PRIx32 ", 0x%08" PRIx32 "\n", event_type, event_index); } - MemoryBarrier(); + EmitGlobalBarrier(); break; case 0x0000000d: case 0x0000000e: @@ -1441,8 +1564,8 @@ void CommandProcessor::Flip() { } auto& command = CurrentBuffer(); - auto request = Sync::PrepareDisplayBufferFlip(command, m_flip.handle, m_flip.index, - m_flip.flip_mode, m_flip.flip_arg); + auto request = Sync::PrepareVideoOutFlip(command, m_flip.handle, m_flip.index, m_flip.flip_mode, + m_flip.flip_arg); Sync::WriteAtEndOfPipeOnlyFlip(m_submit_id, command, m_flip.handle, m_flip.index, m_flip.flip_mode, m_flip.flip_arg, request); GetScheduler().Flush(); @@ -1460,8 +1583,8 @@ void CommandProcessor::Flip(void* dst_gpu_addr, uint32_t value) { std::memcpy(dst_gpu_addr, &value, sizeof(value)); auto& command = CurrentBuffer(); - auto request = Sync::PrepareDisplayBufferFlip(command, m_flip.handle, m_flip.index, - m_flip.flip_mode, m_flip.flip_arg); + auto request = Sync::PrepareVideoOutFlip(command, m_flip.handle, m_flip.index, m_flip.flip_mode, + m_flip.flip_arg); Sync::WriteAtEndOfPipeWithFlip32(m_submit_id, command, static_cast(dst_gpu_addr), value, m_flip.handle, m_flip.index, m_flip.flip_mode, m_flip.flip_arg, request); @@ -1486,15 +1609,15 @@ void CommandProcessor::FlipWithInterrupt(uint32_t eop_event_type, uint32_t cache } std::memcpy(dst_gpu_addr, &value, sizeof(value)); auto& command = CurrentBuffer(); - auto request = Sync::PrepareDisplayBufferFlip(command, m_flip.handle, m_flip.index, - m_flip.flip_mode, m_flip.flip_arg); + auto request = Sync::PrepareVideoOutFlip(command, m_flip.handle, m_flip.index, m_flip.flip_mode, + m_flip.flip_arg); Sync::WriteAtEndOfPipeWithInterruptWriteBackFlip32( m_submit_id, command, static_cast(dst_gpu_addr), value, m_flip.handle, m_flip.index, m_flip.flip_mode, m_flip.flip_arg, request); GetScheduler().Flush(); } -void CommandProcessor::PrepareCpuFlip() { +void CommandProcessor::PrepareCpuFlip(uint64_t request_id) { CheckBuffer(); if (g_current_processor != nullptr) { EXIT("invalid graphics-thread CPU flip preparation\n"); @@ -1505,90 +1628,107 @@ void CommandProcessor::PrepareCpuFlip() { }; ProcessorScope processor_scope(*this); - auto prepared_id = Presentation::DisplayBufferPrepareNextFlipOnGpu(CurrentBuffer()); + m_renderer.GetVideoOut().PrepareFlip(request_id, CurrentBuffer()); GetScheduler().Flush(); - Presentation::DisplayBufferCompleteFlipFromGpu(prepared_id); + m_renderer.GetVideoOut().CompleteFlip(request_id); } void CommandProcessor::SynchronizeGpu() { GetScheduler().FinishCurrent(); } -void GraphicsRunSubmit(uint32_t* cmd_draw_buffer, uint32_t num_draw_dw, uint32_t* cmd_const_buffer, - uint32_t num_const_dw, bool trigger_agc_interrupt_on_done) { - EXIT_IF(cmd_draw_buffer == nullptr); - EXIT_IF(num_draw_dw == 0); - EXIT_IF(g_gpu == nullptr); - - g_gpu->Submit(cmd_draw_buffer, num_draw_dw, cmd_const_buffer, num_const_dw, - trigger_agc_interrupt_on_done); +Gpu::Gpu(RenderContext& renderer) { + EXIT_NOT_IMPLEMENTED(!Common::Thread::IsMainThread()); + GraphicsInitJmpTables(); + m_state = std::make_unique(renderer); } -void GraphicsRunSubmitCompute(uint32_t queue, uint32_t* cmd_buffer, uint32_t num_dw, - bool trigger_agc_interrupt_on_done) { - EXIT_IF(cmd_buffer == nullptr); - EXIT_IF(num_dw == 0); - EXIT_IF(g_gpu == nullptr); +Gpu::~Gpu() = default; - g_gpu->SubmitCompute(queue, cmd_buffer, num_dw, trigger_agc_interrupt_on_done); +void Gpu::Shutdown() { + m_state->Shutdown(); } -void GraphicsRunSubmitFlipPreparation() { - EXIT_IF(g_gpu == nullptr); - g_gpu->SubmitFlipPreparation(); +bool Gpu::IsStopping() { + return m_state->IsStopping(); } -GraphicsRunSubmissionLock::GraphicsRunSubmissionLock() { - if (g_gpu == nullptr || g_current_processor != nullptr || - g_submission_pause_depth == UINT32_MAX) { +void Gpu::SendCommand(Common::UniqueFunction&& command) { + m_state->SendCommand(std::move(command)); +} + +void Gpu::SendCommandSync(Common::UniqueFunction&& command) { + m_state->SendCommandSync(std::move(command)); +} + +void Gpu::SendCommandSyncWithProcessor(Common::UniqueFunction&& command) { + m_state->SendCommandSyncWithProcessor(std::move(command)); +} + +void Gpu::Submit(uint32_t* draw_commands, uint32_t draw_size_dw, uint32_t* constant_commands, + uint32_t constant_size_dw, bool trigger_agc_interrupt_on_done) { + EXIT_IF(draw_commands == nullptr || draw_size_dw == 0); + m_state->Submit(draw_commands, draw_size_dw, constant_commands, constant_size_dw, + trigger_agc_interrupt_on_done); +} + +void Gpu::SubmitCompute(uint32_t queue, uint32_t* commands, uint32_t size_dw, + bool trigger_agc_interrupt_on_done) { + EXIT_IF(commands == nullptr || size_dw == 0); + m_state->SubmitCompute(queue, commands, size_dw, trigger_agc_interrupt_on_done); +} + +void Gpu::SubmitFlipPreparation(uint64_t request_id) { + m_state->SubmitFlipPreparation(request_id); +} + +void Gpu::Done() { + m_state->Done(); +} + +int Gpu::GetFrameNum() const { + return m_state->GetFrameNum(); +} + +void Gpu::PauseSubmissions() { + m_state->PauseSubmissions(); +} + +void Gpu::ResumeSubmissions() { + m_state->ResumeSubmissions(); +} + +Gpu::SubmissionLock::SubmissionLock(Gpu& gpu): m_gpu(gpu) { + if (g_current_processor != nullptr || g_submission_pause_depth == UINT32_MAX) { EXIT("cannot acquire GPU submission lock in the current state\n"); } if (g_submission_pause_depth++ == 0) { - g_gpu->PauseSubmissions(); + m_gpu.PauseSubmissions(); } } -GraphicsRunSubmissionLock::~GraphicsRunSubmissionLock() { - if (g_gpu == nullptr || g_submission_pause_depth == 0) { +Gpu::SubmissionLock::~SubmissionLock() { + if (g_submission_pause_depth == 0) { EXIT("GPU submission lock released without ownership\n"); } if (--g_submission_pause_depth == 0) { - g_gpu->ResumeSubmissions(); + m_gpu.ResumeSubmissions(); } } -void GraphicsRunDone() { - EXIT_IF(g_gpu == nullptr); - - g_gpu->Done(); -} - -int GraphicsRunGetFrameNum() { - EXIT_IF(g_gpu == nullptr); - - return g_gpu->GetFrameNum(); -} - -bool GraphicsRunIsCommandProcessorThread() noexcept { +bool Gpu::IsCommandProcessorThread() noexcept { return g_current_processor != nullptr; } -CommandProcessor* GraphicsRunCurrentCommandProcessor() noexcept { +CommandProcessor* Gpu::CurrentCommandProcessor() noexcept { return g_current_processor; } -void GraphicsRunFinishScheduler() { - if (g_current_processor == nullptr) { - EXIT("GPU readback finish requires a command-processor thread\n"); - } - g_current_processor->FinishReadbackTransaction(); -} - -bool GraphicsRunSubmissionLockHeld() noexcept { +bool Gpu::SubmissionLockHeld() noexcept { return g_submission_pause_depth != 0; } -bool GraphicsRunGpuLockHeld() noexcept { +bool Gpu::MutexHeld() noexcept { return g_gpu_mutex_owned; } diff --git a/src/graphics/guest_gpu/graphicsRun.h b/src/graphics/guest_gpu/graphicsRun.h index cc1b4a41..aa6210da 100644 --- a/src/graphics/guest_gpu/graphicsRun.h +++ b/src/graphics/guest_gpu/graphicsRun.h @@ -3,32 +3,59 @@ #include "common/abi.h" #include "common/common.h" +#include "common/uniqueFunction.h" + +#include namespace Libs::Graphics { class CommandProcessor; +class GpuState; +class RenderContext; -class GraphicsRunSubmissionLock final { +class Gpu final { public: - GraphicsRunSubmissionLock(); - ~GraphicsRunSubmissionLock(); - KYTY_CLASS_NO_COPY(GraphicsRunSubmissionLock); + explicit Gpu(RenderContext& renderer); + ~Gpu(); + KYTY_CLASS_NO_COPY(Gpu); + + void Shutdown(); + [[nodiscard]] bool IsStopping(); + void SendCommand(Common::UniqueFunction&& command); + void SendCommandSync(Common::UniqueFunction&& command); + void SendCommandSyncWithProcessor(Common::UniqueFunction&& command); + + void Submit(uint32_t* draw_commands, uint32_t draw_size_dw, uint32_t* constant_commands, + uint32_t constant_size_dw, bool trigger_agc_interrupt_on_done = false); + void SubmitCompute(uint32_t queue, uint32_t* commands, uint32_t size_dw, + bool trigger_agc_interrupt_on_done = false); + void SubmitFlipPreparation(uint64_t request_id); + void Done(); + [[nodiscard]] int GetFrameNum() const; + + [[nodiscard]] static bool IsCommandProcessorThread() noexcept; + [[nodiscard]] static CommandProcessor* CurrentCommandProcessor() noexcept; + [[nodiscard]] static bool SubmissionLockHeld() noexcept; + [[nodiscard]] static bool MutexHeld() noexcept; + + class SubmissionLock final { + public: + explicit SubmissionLock(Gpu& gpu); + ~SubmissionLock(); + KYTY_CLASS_NO_COPY(SubmissionLock); + + private: + Gpu& m_gpu; + }; + +private: + friend class SubmissionLock; + + void PauseSubmissions(); + void ResumeSubmissions(); + + std::unique_ptr m_state; }; - -void GraphicsRunInit(); - -void GraphicsRunSubmit(uint32_t* cmd_draw_buffer, uint32_t num_draw_dw, uint32_t* cmd_const_buffer, - uint32_t num_const_dw, bool trigger_agc_interrupt_on_done = false); -void GraphicsRunSubmitCompute(uint32_t queue, uint32_t* cmd_buffer, uint32_t num_dw, - bool trigger_agc_interrupt_on_done = false); -void GraphicsRunSubmitFlipPreparation(); -void GraphicsRunDone(); -int GraphicsRunGetFrameNum(); -[[nodiscard]] bool GraphicsRunIsCommandProcessorThread() noexcept; -[[nodiscard]] CommandProcessor* GraphicsRunCurrentCommandProcessor() noexcept; -void GraphicsRunFinishScheduler(); -[[nodiscard]] bool GraphicsRunSubmissionLockHeld() noexcept; -[[nodiscard]] bool GraphicsRunGpuLockHeld() noexcept; } // namespace Libs::Graphics #endif /* EMULATOR_INCLUDE_EMULATOR_GRAPHICS_GRAPHICSRUN_H_ */ diff --git a/src/graphics/host_gpu/gpuTiler.cpp b/src/graphics/host_gpu/gpuTiler.cpp deleted file mode 100644 index 59481f16..00000000 --- a/src/graphics/host_gpu/gpuTiler.cpp +++ /dev/null @@ -1,537 +0,0 @@ -#include "graphics/host_gpu/gpuTiler.h" - -#include "common/assert.h" -#include "common/threads.h" -#include "gpu_tiler_shaders/gpu_tiler_depth_spv.h" -#include "gpu_tiler_shaders/gpu_tiler_prt_3d_spv.h" -#include "gpu_tiler_shaders/gpu_tiler_prt_spv.h" -#include "gpu_tiler_shaders/gpu_tiler_render_target_spv.h" -#include "gpu_tiler_shaders/gpu_tiler_standard256_spv.h" -#include "gpu_tiler_shaders/gpu_tiler_standard4_3d_spv.h" -#include "gpu_tiler_shaders/gpu_tiler_standard4_spv.h" -#include "gpu_tiler_shaders/gpu_tiler_standard64_3d_spv.h" -#include "gpu_tiler_shaders/gpu_tiler_standard64_spv.h" -#include "graphics/host_gpu/graphicContext.h" -#include "graphics/host_gpu/renderer/render.h" -#include "graphics/host_gpu/renderer/renderContext.h" -#include "graphics/host_gpu/vma.h" -#include "graphics/host_gpu/vulkanCommon.h" - -#include -#include -#include -#include -#include -#include - -namespace Libs::Graphics { -namespace { - -constexpr uint32_t FAMILY_COUNT = static_cast(TileBlockFamily::Count); -constexpr uint32_t BYTES_PER_ELEMENT_COUNT = 5; -constexpr uint32_t DIRECTION_COUNT = 2; -constexpr uint32_t PIPELINE_COUNT = FAMILY_COUNT * BYTES_PER_ELEMENT_COUNT * DIRECTION_COUNT; -static_assert(FAMILY_COUNT == 9); - -struct Push { - uint32_t src_base; - uint32_t dst_base; - uint32_t width; - uint32_t height; - uint32_t depth; - uint32_t surface_z; - uint32_t pitch_bytes; - uint32_t slice_bytes; - uint32_t blocks_per_row; - uint32_t blocks_per_slice; - uint32_t tail_x; - uint32_t tail_y; - uint32_t tail; -}; -static_assert(sizeof(Push) == 52); - -struct Shader { - const uint32_t* code; - size_t words; -}; - -constexpr std::array SHADERS {{ - {GPU_TILER_STANDARD256_SPV, std::size(GPU_TILER_STANDARD256_SPV)}, - {GPU_TILER_STANDARD4_SPV, std::size(GPU_TILER_STANDARD4_SPV)}, - {GPU_TILER_STANDARD4_3D_SPV, std::size(GPU_TILER_STANDARD4_3D_SPV)}, - {GPU_TILER_STANDARD64_SPV, std::size(GPU_TILER_STANDARD64_SPV)}, - {GPU_TILER_STANDARD64_3D_SPV, std::size(GPU_TILER_STANDARD64_3D_SPV)}, - {GPU_TILER_PRT_SPV, std::size(GPU_TILER_PRT_SPV)}, - {GPU_TILER_PRT_3D_SPV, std::size(GPU_TILER_PRT_3D_SPV)}, - {GPU_TILER_RENDER_TARGET_SPV, std::size(GPU_TILER_RENDER_TARGET_SPV)}, - {GPU_TILER_DEPTH_SPV, std::size(GPU_TILER_DEPTH_SPV)}, -}}; - -struct Dispatch { - Push push {}; - uint32_t pipeline_slot = 0; -}; - -struct Resources { - vk::DescriptorSetLayout descriptor_layout = nullptr; - vk::PipelineLayout pipeline_layout = nullptr; - vk::DescriptorPool descriptor_pool = nullptr; - vk::DescriptorSet descriptor_set = nullptr; - std::array pipelines {}; - VulkanBuffer staging; - VulkanBuffer linear; - void* mapped = nullptr; -}; - -bool CheckedAdd(uint64_t a, uint64_t b, uint64_t& result) { - return b <= UINT64_MAX - a && (result = a + b, true); -} - -bool CheckedMultiply(uint64_t a, uint64_t b, uint64_t& result) { - return (a == 0 || b <= UINT64_MAX / a) && (result = a * b, true); -} - -bool CheckedAddProduct(uint64_t& value, uint64_t count, uint64_t stride) { - uint64_t bytes = 0; - return CheckedMultiply(count, stride, bytes) && CheckedAdd(value, bytes, value); -} - -bool IsRangeValid(uint64_t offset, uint64_t size, uint64_t capacity) { - return size != 0 && offset <= capacity && size <= capacity - offset; -} - -uint64_t AlignToDword(uint64_t value) { - return (value + 3u) & ~uint64_t {3}; -} - -uint32_t GetPipelineSlot(bool to_tiled, TileBlockFamily family, uint32_t bytes_per_element) { - const uint32_t direction_index = to_tiled ? 1u : 0u; - const uint32_t family_index = static_cast(family); - const uint32_t element_size_index = std::countr_zero(bytes_per_element); - return (direction_index * FAMILY_COUNT + family_index) * BYTES_PER_ELEMENT_COUNT + - element_size_index; -} - -void Barrier(vk::CommandBuffer command, vk::Buffer buffer, vk::AccessFlags src_access, - vk::AccessFlags dst_access, vk::PipelineStageFlags src_stage, - vk::PipelineStageFlags dst_stage) { - vk::BufferMemoryBarrier barrier {}; - barrier.sType = vk::StructureType::eBufferMemoryBarrier; - barrier.srcAccessMask = src_access; - barrier.dstAccessMask = dst_access; - barrier.srcQueueFamilyIndex = VK_QUEUE_FAMILY_IGNORED; - barrier.dstQueueFamilyIndex = VK_QUEUE_FAMILY_IGNORED; - barrier.buffer = buffer; - barrier.size = VK_WHOLE_SIZE; - command.pipelineBarrier(src_stage, dst_stage, {}, 0, nullptr, 1, &barrier, 0, nullptr); -} - -class TileCompute final { -public: - explicit TileCompute(GraphicContext& graphics): graphics(graphics) {} - void Run(bool to_tiled, const void* input, void* output, uint64_t tiled_capacity, - uint64_t linear_capacity, std::span infos, - const GpuTileRecord& record); - void Release(); - -private: - void Prepare(bool to_tiled, uint64_t tiled_capacity, uint64_t linear_capacity, - std::span infos, std::vector& dispatches) const; - void Init(); - void CreatePipelines(std::span dispatches); - void CreatePipeline(uint32_t pipeline_slot); - void Resize(uint64_t staging_size, uint64_t linear_size); - void CreateBuffer(uint64_t size, bool mapped, VulkanBuffer& buffer, void** data) const; - void Execute(bool to_tiled, const void* input, void* output, uint64_t tiled_capacity, - uint64_t linear_capacity, std::span dispatches, - const GpuTileRecord& record); - void Destroy(Resources& target) const; - - GraphicContext& graphics; - Resources resources; -}; - -Common::Mutex g_tiler_mutex; -std::unique_ptr g_tiler; - -void TileCompute::Prepare(bool to_tiled, uint64_t tiled_capacity, uint64_t linear_capacity, - std::span infos, - std::vector& dispatches) const { - EXIT_IF(infos.empty() || tiled_capacity == 0 || linear_capacity == 0); - const auto& limits = graphics.GetPhysicalDeviceProperties().limits; - EXIT_NOT_IMPLEMENTED(tiled_capacity > UINT32_MAX || linear_capacity > UINT32_MAX || - AlignToDword(tiled_capacity) > limits.maxStorageBufferRange || - AlignToDword(linear_capacity) > limits.maxStorageBufferRange); - - dispatches.clear(); - dispatches.reserve(infos.size()); - for (const auto& info: infos) { - TileBlockLayout block {}; - const uint32_t tiled_width = info.tiled_width != 0 ? info.tiled_width : info.pitch; - const uint32_t tiled_height = info.tiled_height != 0 ? info.tiled_height : info.height; - const uint64_t groups_x = (static_cast(info.width) + 7u) / 8u; - const uint64_t groups_y = (static_cast(info.height) + 7u) / 8u; - EXIT_NOT_IMPLEMENTED( - !TileGetBlockLayout(info.family, info.bytes_per_element, block) || info.width == 0 || - info.height == 0 || info.depth == 0 || info.pitch < info.width || - groups_x > limits.maxComputeWorkGroupCount[0] || - groups_y > limits.maxComputeWorkGroupCount[1] || - info.depth > limits.maxComputeWorkGroupCount[2] || - (!info.tail && (tiled_width < info.width || tiled_height < info.height)) || - !IsRangeValid(info.linear_offset, info.linear_size, linear_capacity) || - !IsRangeValid(info.tiled_offset, info.tiled_size, tiled_capacity) || - (block.block_depth == 1 && info.depth != 1)); - - uint64_t pitch_bytes = 0; - EXIT_NOT_IMPLEMENTED(!CheckedMultiply(info.pitch, info.bytes_per_element, pitch_bytes) || - pitch_bytes > UINT32_MAX); - uint64_t slice_bytes = info.linear_slice_stride; - EXIT_NOT_IMPLEMENTED(slice_bytes == 0 && - !CheckedMultiply(pitch_bytes, info.height, slice_bytes)); - uint64_t linear_used = 0, minimum_slice = 0; - EXIT_NOT_IMPLEMENTED(!CheckedMultiply(pitch_bytes, info.height, minimum_slice) || - (info.depth > 1 && slice_bytes < minimum_slice) || - !CheckedAddProduct(linear_used, info.depth - 1u, slice_bytes) || - !CheckedAddProduct(linear_used, info.height - 1u, pitch_bytes) || - !CheckedAddProduct(linear_used, info.width, info.bytes_per_element) || - linear_used > info.linear_size || slice_bytes > UINT32_MAX); - - const uint64_t columns = - (static_cast(tiled_width) + block.block_width - 1u) / block.block_width; - const uint64_t rows = - (static_cast(tiled_height) + block.block_height - 1u) / block.block_height; - uint64_t blocks_per_slice = 0; - EXIT_NOT_IMPLEMENTED(!CheckedMultiply(columns, rows, blocks_per_slice) || - columns > UINT32_MAX || blocks_per_slice > UINT32_MAX || - rows * block.block_height > UINT32_MAX); - if (info.tail) { - const bool supported = info.family != TileBlockFamily::Standard256B; - EXIT_NOT_IMPLEMENTED( - !supported || info.depth > block.block_depth || info.tail_x >= block.block_width || - info.width > block.block_width - info.tail_x || info.tail_y >= block.block_height || - info.height > block.block_height - info.tail_y || - info.tiled_size < block.block_size); - } else { - const uint64_t slices = - (static_cast(info.depth) + block.block_depth - 1u) / block.block_depth; - uint64_t tiled_used = 0; - EXIT_NOT_IMPLEMENTED(!CheckedMultiply(blocks_per_slice, slices, tiled_used) || - !CheckedMultiply(tiled_used, block.block_size, tiled_used) || - tiled_used > info.tiled_size); - } - const uint32_t alignment = std::min(info.bytes_per_element, 4u); - EXIT_NOT_IMPLEMENTED(((info.linear_offset | info.tiled_offset | pitch_bytes | slice_bytes) & - (alignment - 1u)) != 0); - - Dispatch dispatch {}; - dispatch.pipeline_slot = GetPipelineSlot(to_tiled, info.family, info.bytes_per_element); - dispatch.push.src_base = - static_cast(to_tiled ? info.linear_offset : info.tiled_offset); - dispatch.push.dst_base = - static_cast(to_tiled ? info.tiled_offset : info.linear_offset); - dispatch.push.width = info.width; - dispatch.push.height = info.height; - dispatch.push.depth = info.depth; - dispatch.push.surface_z = info.surface_z; - dispatch.push.pitch_bytes = static_cast(pitch_bytes); - dispatch.push.slice_bytes = static_cast(slice_bytes); - dispatch.push.blocks_per_row = static_cast(columns); - dispatch.push.blocks_per_slice = static_cast(blocks_per_slice); - dispatch.push.tail_x = info.tail_x; - dispatch.push.tail_y = info.tail_y; - dispatch.push.tail = info.tail; - dispatches.push_back(dispatch); - } -} - -void TileCompute::Destroy(Resources& target) const { - if (target.mapped != nullptr) { - graphics.UnmapMemory(target.staging.memory); - } - if (target.staging.buffer != nullptr) { - graphics.DeleteBuffer(target.staging); - } - if (target.linear.buffer != nullptr) { - graphics.DeleteBuffer(target.linear); - } - for (auto pipeline: target.pipelines) { - if (pipeline != nullptr) { - graphics.device.destroyPipeline(pipeline, nullptr); - } - } - if (target.descriptor_pool != nullptr) { - graphics.device.destroyDescriptorPool(target.descriptor_pool, nullptr); - } - if (target.pipeline_layout != nullptr) { - graphics.device.destroyPipelineLayout(target.pipeline_layout, nullptr); - } - if (target.descriptor_layout != nullptr) { - graphics.device.destroyDescriptorSetLayout(target.descriptor_layout, nullptr); - } - target = {}; -} - -void TileCompute::Init() { - if (resources.pipeline_layout != nullptr) { - return; - } - std::array bindings {}; - for (uint32_t i = 0; i < bindings.size(); i++) { - bindings[i] = {i, vk::DescriptorType::eStorageBuffer, 1, vk::ShaderStageFlagBits::eCompute, - nullptr}; - } - vk::DescriptorSetLayoutCreateInfo descriptor_info {}; - descriptor_info.sType = vk::StructureType::eDescriptorSetLayoutCreateInfo; - descriptor_info.bindingCount = static_cast(bindings.size()); - descriptor_info.pBindings = bindings.data(); - RequireVulkanSuccess(graphics.device.createDescriptorSetLayout(&descriptor_info, nullptr, - &resources.descriptor_layout), - "create GPU tiler descriptor layout"); - - vk::PushConstantRange push_range {vk::ShaderStageFlagBits::eCompute, 0, sizeof(Push)}; - vk::PipelineLayoutCreateInfo layout_info {}; - layout_info.sType = vk::StructureType::ePipelineLayoutCreateInfo; - layout_info.setLayoutCount = 1; - layout_info.pSetLayouts = &resources.descriptor_layout; - layout_info.pushConstantRangeCount = 1; - layout_info.pPushConstantRanges = &push_range; - RequireVulkanSuccess( - graphics.device.createPipelineLayout(&layout_info, nullptr, &resources.pipeline_layout), - "create GPU tiler pipeline layout"); - vk::DescriptorPoolSize pool_size {vk::DescriptorType::eStorageBuffer, 2}; - vk::DescriptorPoolCreateInfo pool_info {}; - pool_info.sType = vk::StructureType::eDescriptorPoolCreateInfo; - pool_info.maxSets = 1; - pool_info.poolSizeCount = 1; - pool_info.pPoolSizes = &pool_size; - RequireVulkanSuccess( - graphics.device.createDescriptorPool(&pool_info, nullptr, &resources.descriptor_pool), - "create GPU tiler descriptor pool"); - vk::DescriptorSetAllocateInfo set_info {}; - set_info.sType = vk::StructureType::eDescriptorSetAllocateInfo; - set_info.descriptorPool = resources.descriptor_pool; - set_info.descriptorSetCount = 1; - set_info.pSetLayouts = &resources.descriptor_layout; - RequireVulkanSuccess( - graphics.device.allocateDescriptorSets(&set_info, &resources.descriptor_set), - "allocate GPU tiler descriptor set"); -} - -void TileCompute::CreatePipeline(uint32_t pipeline_slot) { - const uint32_t element_size_index = pipeline_slot % BYTES_PER_ELEMENT_COUNT; - const uint32_t family_direction_index = pipeline_slot / BYTES_PER_ELEMENT_COUNT; - const uint32_t family_index = family_direction_index % FAMILY_COUNT; - const uint32_t direction_index = family_direction_index / FAMILY_COUNT; - const uint32_t specialization_values[] {1u << element_size_index, direction_index}; - const vk::SpecializationMapEntry entries[] {{0, 0, 4}, {1, 4, 4}}; - vk::SpecializationInfo specialization {2, entries, sizeof(specialization_values), - specialization_values}; - vk::ShaderModuleCreateInfo module_info {}; - module_info.sType = vk::StructureType::eShaderModuleCreateInfo; - module_info.codeSize = SHADERS[family_index].words * sizeof(uint32_t); - module_info.pCode = SHADERS[family_index].code; - vk::ShaderModule module = nullptr; - RequireVulkanSuccess(graphics.device.createShaderModule(&module_info, nullptr, &module), - "create GPU tiler shader module"); - vk::PipelineShaderStageCreateInfo stage {}; - stage.sType = vk::StructureType::ePipelineShaderStageCreateInfo; - stage.stage = vk::ShaderStageFlagBits::eCompute; - stage.module = module; - stage.pName = "main"; - stage.pSpecializationInfo = &specialization; - vk::ComputePipelineCreateInfo info {}; - info.sType = vk::StructureType::eComputePipelineCreateInfo; - info.stage = stage; - info.layout = resources.pipeline_layout; - vk::Pipeline pipeline = nullptr; - const auto result = - graphics.device.createComputePipelines(nullptr, 1, &info, nullptr, &pipeline); - graphics.device.destroyShaderModule(module, nullptr); - RequireVulkanSuccess(result, "create GPU tiler pipeline"); - resources.pipelines[pipeline_slot] = pipeline; -} - -void TileCompute::CreatePipelines(std::span dispatches) { - for (const auto& dispatch: dispatches) { - if (resources.pipelines[dispatch.pipeline_slot] == nullptr) { - CreatePipeline(dispatch.pipeline_slot); - } - } -} - -void TileCompute::CreateBuffer(uint64_t size, bool mapped, VulkanBuffer& buffer, - void** data) const { - buffer.usage = vk::BufferUsageFlagBits::eStorageBuffer | vk::BufferUsageFlagBits::eTransferSrc | - vk::BufferUsageFlagBits::eTransferDst; - buffer.memory.property = - mapped - ? vk::MemoryPropertyFlagBits::eHostVisible | vk::MemoryPropertyFlagBits::eHostCoherent - : vk::MemoryPropertyFlags(vk::MemoryPropertyFlagBits::eDeviceLocal); - graphics.CreateBuffer(size, buffer); - if (mapped) graphics.MapMemory(buffer.memory, *data); -} - -void TileCompute::Resize(uint64_t staging_size, uint64_t linear_size) { - if (resources.staging.buffer_size >= staging_size && - resources.linear.buffer_size >= linear_size) { - return; - } - staging_size = std::max(staging_size, resources.staging.buffer_size); - linear_size = std::max(linear_size, resources.linear.buffer_size); - VulkanBuffer staging {}, linear {}; - void* mapped = nullptr; - CreateBuffer(staging_size, true, staging, &mapped); - CreateBuffer(linear_size, false, linear, nullptr); - if (resources.mapped != nullptr) graphics.UnmapMemory(resources.staging.memory); - if (resources.staging.buffer != nullptr) graphics.DeleteBuffer(resources.staging); - if (resources.linear.buffer != nullptr) graphics.DeleteBuffer(resources.linear); - resources.staging = staging; - resources.linear = linear; - resources.mapped = mapped; -} - -void TileCompute::Execute(bool to_tiled, const void* input, void* output, uint64_t tiled_capacity, - uint64_t linear_capacity, std::span dispatches, - const GpuTileRecord& record) { - const uint64_t tiled_size = AlignToDword(tiled_capacity); - const uint64_t linear_size = AlignToDword(linear_capacity); - const uint64_t input_size = to_tiled ? linear_capacity : tiled_capacity; - if (input != nullptr) { - std::memcpy(resources.mapped, input, static_cast(input_size)); - std::memset(static_cast(resources.mapped) + input_size, 0, - static_cast(AlignToDword(input_size) - input_size)); - } - - std::array buffer_info {{ - {to_tiled ? resources.linear.buffer : resources.staging.buffer, 0, - to_tiled ? linear_size : tiled_size}, - {to_tiled ? resources.staging.buffer : resources.linear.buffer, 0, - to_tiled ? tiled_size : linear_size}, - }}; - std::array writes {}; - for (uint32_t i = 0; i < writes.size(); i++) { - writes[i].sType = vk::StructureType::eWriteDescriptorSet; - writes[i].dstSet = resources.descriptor_set; - writes[i].dstBinding = i; - writes[i].descriptorCount = 1; - writes[i].descriptorType = vk::DescriptorType::eStorageBuffer; - writes[i].pBufferInfo = &buffer_info[i]; - } - graphics.device.updateDescriptorSets(static_cast(writes.size()), writes.data(), 0, - nullptr); - - CommandBuffer command; - command.Begin(); - auto vk_command = command.Handle(); - if (input != nullptr) { - Barrier(vk_command, resources.staging.buffer, vk::AccessFlagBits::eHostWrite, - vk::AccessFlagBits::eShaderRead | vk::AccessFlagBits::eTransferRead, - vk::PipelineStageFlagBits::eHost, - vk::PipelineStageFlagBits::eComputeShader | vk::PipelineStageFlagBits::eTransfer); - } - if (to_tiled && input != nullptr) { - const vk::BufferCopy copy {0, 0, linear_size}; - vk_command.copyBuffer(resources.staging.buffer, resources.linear.buffer, 1, ©); - Barrier(vk_command, resources.linear.buffer, vk::AccessFlagBits::eTransferWrite, - vk::AccessFlagBits::eShaderRead, vk::PipelineStageFlagBits::eTransfer, - vk::PipelineStageFlagBits::eComputeShader); - Barrier(vk_command, resources.staging.buffer, vk::AccessFlagBits::eTransferRead, - vk::AccessFlagBits::eTransferWrite, vk::PipelineStageFlagBits::eTransfer, - vk::PipelineStageFlagBits::eTransfer); - } - if (to_tiled && record) { - record(command, resources.linear); - Barrier(vk_command, resources.linear.buffer, - vk::AccessFlagBits::eTransferWrite | vk::AccessFlagBits::eMemoryWrite, - vk::AccessFlagBits::eShaderRead, vk::PipelineStageFlagBits::eAllCommands, - vk::PipelineStageFlagBits::eComputeShader); - } - - const auto output_buffer = to_tiled ? resources.staging.buffer : resources.linear.buffer; - const auto output_size = to_tiled ? tiled_size : linear_size; - vk_command.fillBuffer(output_buffer, 0, output_size, 0); - Barrier(vk_command, output_buffer, vk::AccessFlagBits::eTransferWrite, - vk::AccessFlagBits::eShaderRead | vk::AccessFlagBits::eShaderWrite, - vk::PipelineStageFlagBits::eTransfer, vk::PipelineStageFlagBits::eComputeShader); - vk_command.bindDescriptorSets(vk::PipelineBindPoint::eCompute, resources.pipeline_layout, 0, 1, - &resources.descriptor_set, 0, nullptr); - for (const auto& dispatch: dispatches) { - vk_command.bindPipeline(vk::PipelineBindPoint::eCompute, - resources.pipelines[dispatch.pipeline_slot]); - vk_command.pushConstants(resources.pipeline_layout, vk::ShaderStageFlagBits::eCompute, 0, - sizeof(dispatch.push), &dispatch.push); - vk_command.dispatch((dispatch.push.width + 7u) / 8u, (dispatch.push.height + 7u) / 8u, - dispatch.push.depth); - } - Barrier(vk_command, output_buffer, vk::AccessFlagBits::eShaderWrite, - vk::AccessFlagBits::eTransferRead | vk::AccessFlagBits::eHostRead, - vk::PipelineStageFlagBits::eComputeShader, - vk::PipelineStageFlagBits::eTransfer | vk::PipelineStageFlagBits::eHost); - if (!to_tiled && record) { - record(command, resources.linear); - } - if (!to_tiled && output != nullptr) { - const vk::BufferCopy copy {0, 0, linear_size}; - vk_command.copyBuffer(resources.linear.buffer, resources.staging.buffer, 1, ©); - Barrier(vk_command, resources.staging.buffer, vk::AccessFlagBits::eTransferWrite, - vk::AccessFlagBits::eHostRead, vk::PipelineStageFlagBits::eTransfer, - vk::PipelineStageFlagBits::eHost); - } - command.End(); - command.Execute(); - command.WaitForFence(); - if (output != nullptr) { - std::memcpy(output, resources.mapped, - static_cast(to_tiled ? tiled_capacity : linear_capacity)); - } -} - -void TileCompute::Run(bool to_tiled, const void* input, void* output, uint64_t tiled_capacity, - uint64_t linear_capacity, std::span infos, - const GpuTileRecord& record) { - EXIT_IF((to_tiled && (output == nullptr || (input == nullptr && !record))) || - (!to_tiled && (input == nullptr || (output == nullptr && !record)))); - std::vector dispatches; - Prepare(to_tiled, tiled_capacity, linear_capacity, infos, dispatches); - Init(); - CreatePipelines(dispatches); - const uint64_t staging_size = - std::max(AlignToDword(tiled_capacity), AlignToDword(linear_capacity)); - const uint64_t linear_size = AlignToDword(linear_capacity); - Resize(staging_size, linear_size); - Execute(to_tiled, input, output, tiled_capacity, linear_capacity, dispatches, record); -} - -void TileCompute::Release() { - Destroy(resources); -} - -} // namespace - -void GpuDetile(const void* tiled, void* linear, uint64_t tiled_capacity, uint64_t linear_capacity, - std::span infos, const GpuTileRecord& after) { - Common::LockGuard lock(g_tiler_mutex); - if (!g_tiler) { - g_tiler = std::make_unique(GetRenderContext().GetGraphics()); - } - g_tiler->Run(false, tiled, linear, tiled_capacity, linear_capacity, infos, after); -} - -void GpuTile(const void* linear, void* tiled, uint64_t tiled_capacity, uint64_t linear_capacity, - std::span infos, const GpuTileRecord& before) { - Common::LockGuard lock(g_tiler_mutex); - if (!g_tiler) { - g_tiler = std::make_unique(GetRenderContext().GetGraphics()); - } - g_tiler->Run(true, linear, tiled, tiled_capacity, linear_capacity, infos, before); -} - -void GpuTileRelease() { - Common::LockGuard lock(g_tiler_mutex); - if (g_tiler) { - g_tiler->Release(); - g_tiler.reset(); - } -} - -} // namespace Libs::Graphics diff --git a/src/graphics/host_gpu/gpuTiler.h b/src/graphics/host_gpu/gpuTiler.h deleted file mode 100644 index ad2af34f..00000000 --- a/src/graphics/host_gpu/gpuTiler.h +++ /dev/null @@ -1,45 +0,0 @@ -#pragma once - -#include "graphics/guest_gpu/tile.h" - -#include -#include -#include - -namespace Libs::Graphics { - -struct GraphicContext; -struct VulkanBuffer; -class CommandBuffer; - -struct GpuTileInfo { - TileBlockFamily family = TileBlockFamily::Count; - uint32_t bytes_per_element = 0; - uint64_t linear_offset = 0; - uint64_t linear_size = 0; - uint64_t tiled_offset = 0; - uint64_t tiled_size = 0; - uint64_t linear_slice_stride = 0; - uint32_t width = 0; - uint32_t height = 0; - uint32_t depth = 1; - uint32_t pitch = 0; - uint32_t tail_x = 0; - uint32_t tail_y = 0; - bool tail = false; - uint32_t tiled_width = 0; - uint32_t tiled_height = 0; - uint32_t surface_z = 0; -}; - -using GpuTileRecord = std::function; - -void GpuDetile(const void* tiled, void* linear, uint64_t tiled_capacity, - uint64_t linear_capacity, std::span infos, - const GpuTileRecord& after = {}); -void GpuTile(const void* linear, void* tiled, uint64_t tiled_capacity, - uint64_t linear_capacity, std::span infos, - const GpuTileRecord& before = {}); -void GpuTileRelease(); - -} // namespace Libs::Graphics diff --git a/src/graphics/host_gpu/graphicContext.h b/src/graphics/host_gpu/graphicContext.h index fdc0441f..7aed2d80 100644 --- a/src/graphics/host_gpu/graphicContext.h +++ b/src/graphics/host_gpu/graphicContext.h @@ -6,8 +6,6 @@ #include "graphics/host_gpu/vulkanCommon.h" // IWYU pragma: export #include "graphics/host_gpu/vulkanInstance.h" -#include -#include #include #include @@ -18,32 +16,20 @@ struct VulkanBuffer; struct VulkanImage; struct VulkanMemory; -struct VulkanSwapchain { - ~VulkanSwapchain(); - - vk::SwapchainKHR swapchain = nullptr; - vk::Format swapchain_format = vk::Format::eUndefined; - vk::Extent2D swapchain_extent = {}; - std::unique_ptr swapchain_images; - std::unique_ptr swapchain_image_views; - uint32_t swapchain_images_count = 0; - std::unique_ptr image_acquired_semaphores; - std::unique_ptr render_complete_semaphores; - uint32_t current_index = 0; - uint32_t present_frame = 0; -}; - struct GraphicContext: public VulkanInstance { [[nodiscard]] bool CreateAllocator(); void DestroyAllocator(); void LogMemoryBudget() const; - void CreateBuffer(uint64_t size, VulkanBuffer& buffer); - void DeleteBuffer(VulkanBuffer& buffer); - [[nodiscard]] bool CreateImage(const vk::ImageCreateInfo& info, VulkanImage& image); - void DeleteImage(VulkanImage& image); - void MapMemory(VulkanMemory& memory, void*& data); - void UnmapMemory(VulkanMemory& memory); - void AppendHardwareRayTracingDeviceExtensions( + [[nodiscard]] bool CanReportMemoryUsage() const noexcept { return memory_budget_ext_enabled; } + [[nodiscard]] uint64_t GetDeviceMemoryUsage() const; + [[nodiscard]] uint64_t GetTotalMemoryBudget() const; + void CreateBuffer(uint64_t size, VulkanBuffer& buffer); + void DeleteBuffer(VulkanBuffer& buffer); + [[nodiscard]] bool CreateImage(const vk::ImageCreateInfo& info, VulkanImage& image); + void DeleteImage(VulkanImage& image); + void MapMemory(VulkanMemory& memory, void*& data); + void UnmapMemory(VulkanMemory& memory); + void AppendHardwareRayTracingDeviceExtensions( const std::vector& available_extensions, std::vector& device_extensions); void LoadHardwareRayTracingFunctions() const; @@ -64,92 +50,29 @@ struct VulkanMemory { uint64_t unique_id = 0; }; -enum class VulkanImageType { - Unknown, - VideoOut, - DepthStencil, - Texture, - StorageTexture, - RenderTexture -}; - -struct ImageViewInfo { - vk::Format format = vk::Format::eUndefined; - vk::ImageViewType type = vk::ImageViewType::e2D; - vk::ImageAspectFlags aspect = {}; - uint32_t base_level = 0; - uint32_t level_count = 0; - uint32_t base_layer = 0; - uint32_t layer_count = 1; - uint32_t swizzle = 0; - vk::ImageUsageFlags usage = vk::ImageUsageFlagBits::eSampled; - - bool operator==(const ImageViewInfo&) const = default; -}; - -struct CachedImageView { - ImageViewInfo info; - vk::ImageView view = nullptr; -}; - -struct ImageViewCache { - std::mutex mutex; - std::vector views; - - ImageViewCache() = default; - KYTY_CLASS_NO_COPY(ImageViewCache); +struct VulkanImageState { + vk::PipelineStageFlags2 pl_stage = vk::PipelineStageFlagBits2::eAllCommands; + vk::AccessFlags2 access_mask = vk::AccessFlagBits2::eNone; + vk::ImageLayout layout = vk::ImageLayout::eUndefined; }; struct VulkanImage { - static constexpr int VIEW_MAX = 4; - static constexpr int VIEW_DEFAULT = 0; - static constexpr int VIEW_DEFAULT_ARRAY = 1; - static constexpr int VIEW_STORAGE = 2; - static constexpr int VIEW_STORAGE_ARRAY = 3; - - explicit VulkanImage(VulkanImageType type): type(type) {} + VulkanImage() = default; KYTY_CLASS_NO_COPY(VulkanImage); - VulkanImageType type = VulkanImageType::Unknown; - vk::Format format = vk::Format::eUndefined; - vk::Extent2D extent = {}; - uint32_t guest_pitch = 0; - uint32_t layers = 1; - uint32_t mip_levels = 1; - uint32_t samples = 1; - vk::Image image = nullptr; - vk::ImageView image_view[VIEW_MAX] = {}; - vk::ImageLayout layout = vk::ImageLayout::eUndefined; + vk::Format format = vk::Format::eUndefined; + vk::ImageType image_type = vk::ImageType::e2D; + vk::Extent3D extent = {1, 1, 1}; + uint32_t guest_pitch = 0; + uint32_t layers = 1; + uint32_t mip_levels = 1; + uint32_t samples = 1; + vk::ImageUsageFlags usage = {}; + vk::ImageCreateFlags flags = {}; + vk::Image image = nullptr; + VulkanImageState state; + std::vector subresource_states; Graphics::VulkanMemory memory; - ImageViewCache view_cache; -}; - -struct VideoOutVulkanImage: public VulkanImage { - VideoOutVulkanImage(): VulkanImage(VulkanImageType::VideoOut) {} -}; - -struct DepthStencilVulkanImage: public VulkanImage { - DepthStencilVulkanImage(): VulkanImage(VulkanImageType::DepthStencil) {} - bool compressed = false; - bool initial_depth_clear_pending = false; - bool initial_stencil_clear_pending = false; -}; - -struct GpuTextureVulkanImage: public VulkanImage { - explicit GpuTextureVulkanImage(VulkanImageType type): VulkanImage(type) {} -}; - -struct TextureVulkanImage: public GpuTextureVulkanImage { - TextureVulkanImage(): GpuTextureVulkanImage(VulkanImageType::Texture) {} -}; - -struct StorageTextureVulkanImage: public GpuTextureVulkanImage { - StorageTextureVulkanImage(): GpuTextureVulkanImage(VulkanImageType::StorageTexture) {} -}; - -struct RenderTextureVulkanImage: public VulkanImage { - RenderTextureVulkanImage(): VulkanImage(VulkanImageType::RenderTexture) {} - bool initial_clear_pending = false; }; struct VulkanBuffer { @@ -159,8 +82,6 @@ struct VulkanBuffer { uint64_t buffer_size = 0; }; -struct StorageVulkanBuffer: public VulkanBuffer {}; - } // namespace Libs::Graphics #endif /* EMULATOR_INCLUDE_EMULATOR_GRAPHICS_GRAPHICCONTEXT_H_ */ diff --git a/src/graphics/host_gpu/memoryTracker.cpp b/src/graphics/host_gpu/memoryTracker.cpp index 9479c37e..fd78122b 100644 --- a/src/graphics/host_gpu/memoryTracker.cpp +++ b/src/graphics/host_gpu/memoryTracker.cpp @@ -31,6 +31,44 @@ MemoryTracker::MemoryTracker(PageManager& page_manager, PageWatchMode gpu_watch_ MemoryTracker::~MemoryTracker() = default; +void MemoryTracker::ValidateGpuDirtyPages(const RangeSet& dirty, uint64_t vaddr, uint64_t size, + const char* operation) const noexcept { + if (vaddr == 0 || size == 0 || size > UINT64_MAX - vaddr || + (vaddr & (TRACKER_PAGE_SIZE - 1)) != 0 || (size & (TRACKER_PAGE_SIZE - 1)) != 0) { + EXIT("MemoryTracker: invalid dirty-page validation range\n"); + } + for (auto page = vaddr; page < vaddr + size; page += TRACKER_PAGE_SIZE) { + bool found = false; + dirty.ForEachIntersection(page, TRACKER_PAGE_SIZE, + [&found](RangeSet::Range) { found = true; }); + if (!found) { + EXIT("MemoryTracker: GPU-dirty tracker page has no dirty bytes, operation=%s " + "addr=0x%016" PRIx64 "\n", + operation, page); + } + } +} + +void MemoryTracker::ValidateGpuDirtyOwnership(const RangeSet& dirty, uint64_t vaddr, uint64_t size, + const char* operation) { + ValidateRange(vaddr, size); + if (vaddr + size > UINT64_MAX - (TRACKER_PAGE_SIZE - 1)) { + EXIT("MemoryTracker: dirty ownership range alignment overflow\n"); + } + const auto begin = vaddr & ~(TRACKER_PAGE_SIZE - 1); + const auto end = (vaddr + size + TRACKER_PAGE_SIZE - 1) & ~(TRACKER_PAGE_SIZE - 1); + for (auto page = begin; page < end; page += TRACKER_PAGE_SIZE) { + bool has_dirty_bytes = false; + dirty.ForEachIntersection(page, TRACKER_PAGE_SIZE, + [&has_dirty_bytes](RangeSet::Range) { has_dirty_bytes = true; }); + if (IsRegionGpuModified(page, TRACKER_PAGE_SIZE) != has_dirty_bytes) { + EXIT("MemoryTracker: tracker and byte ownership disagree, operation=%s " + "addr=0x%016" PRIx64 "\n", + operation, page); + } + } +} + void MemoryTracker::ValidateRange(uint64_t vaddr, uint64_t size) { if (vaddr == 0 || size == 0 || vaddr >= TRACKER_ADDRESS_SIZE || size > TRACKER_ADDRESS_SIZE - vaddr) { diff --git a/src/graphics/host_gpu/memoryTracker.h b/src/graphics/host_gpu/memoryTracker.h index 08fab1d1..9798ba25 100644 --- a/src/graphics/host_gpu/memoryTracker.h +++ b/src/graphics/host_gpu/memoryTracker.h @@ -3,6 +3,7 @@ #include "common/assert.h" #include "graphics/host_gpu/pageManager.h" +#include "graphics/host_gpu/rangeSet.h" #include "graphics/host_gpu/regionManager.h" #include @@ -10,6 +11,7 @@ #include #include #include +#include #include namespace Libs::Graphics { @@ -38,6 +40,10 @@ public: PageFaultPhase phase) noexcept; [[nodiscard]] bool InvalidateVirtualGpuWrite(PageFaultAccess access, uint64_t vaddr, uint64_t size, PageFaultPhase phase) noexcept; + void ValidateGpuDirtyPages(const RangeSet& dirty, uint64_t vaddr, uint64_t size, + const char* operation) const noexcept; + void ValidateGpuDirtyOwnership(const RangeSet& dirty, uint64_t vaddr, uint64_t size, + const char* operation); template void ForEachDownloadRange(uint64_t vaddr, uint64_t size, Preflight&& preflight, Func&& func) { @@ -99,7 +105,7 @@ public: std::unique_lock access(m_access_mutex); RequireMapped(vaddr, size); Iterate(vaddr, size, [](RegionManager*, uint64_t, uint64_t) {}); - s_upload_owner = this; + const auto* previous_upload_owner = std::exchange(s_upload_owner, this); Iterate(vaddr, size, [&](RegionManager* manager, uint64_t offset, uint64_t bytes) { manager->lock.lock(); manager->Track(manager->GetCpuAddr() + offset, bytes); @@ -119,15 +125,15 @@ public: manager->lock.unlock(); }); } - s_upload_owner = nullptr; + s_upload_owner = previous_upload_owner; } private: static constexpr size_t REGION_COUNT = TRACKER_ADDRESS_SIZE / TRACKER_REGION_SIZE; inline static thread_local const MemoryTracker* s_upload_owner = nullptr; - static void CheckNotInUploadCallback() noexcept { - if (s_upload_owner != nullptr) { + void CheckNotInUploadCallback() const noexcept { + if (s_upload_owner == this) { EXIT("memory tracker re-entered from upload callback\n"); } } @@ -167,7 +173,9 @@ private: void RequireMapped(uint64_t vaddr, uint64_t size) const { ValidateRange(vaddr, size); if (!m_page_manager.IsMapped(vaddr, size)) { - EXIT("memory tracker range is not mapped\n"); + EXIT("memory tracker range [0x%llx, 0x%llx) is not mapped\n", + static_cast(vaddr), + static_cast(vaddr + size)); } } RegionManager* GetOrCreateRegion(uint64_t index); diff --git a/src/graphics/host_gpu/objects/label.cpp b/src/graphics/host_gpu/objects/label.cpp deleted file mode 100644 index 162838c2..00000000 --- a/src/graphics/host_gpu/objects/label.cpp +++ /dev/null @@ -1,348 +0,0 @@ -#include "graphics/host_gpu/objects/label.h" - -#include "common/assert.h" -#include "common/threads.h" -#include "graphics/host_gpu/graphicContext.h" -#include "graphics/host_gpu/renderer/render.h" -#include "graphics/host_gpu/renderer/renderContext.h" -#include "graphics/host_gpu/vulkanCommon.h" - -#include -#include -#include - -namespace Libs::Graphics { - -enum LabelStatus { - New, - Active, - ActiveDeleted, - NotActive, -}; - -struct LabelCallbacks { - LabelCallback callback_1 = nullptr; - LabelCallback callback_2 = nullptr; - uint64_t args[LABEL_ARGS_MAX] = {}; -}; - -struct LabelEvent final { - vk::Device device = nullptr; - vk::Event event = nullptr; - - ~LabelEvent() { - if (event != nullptr) { - device.destroyEvent(event, nullptr); - } - } -}; - -struct LabelSubmission { - std::shared_ptr completion; - LabelCallbacks callbacks; -}; - -struct Label { - vk::Device device = nullptr; - LabelStatus status = LabelStatus::New; - LabelCallbacks callbacks; - std::vector submissions; -}; - -class LabelManager { -public: - LabelManager() { - EXIT_NOT_IMPLEMENTED(!Common::Thread::IsMainThread()); - Common::Thread t(ThreadRun, this); - t.Detach(); - } - ~LabelManager() { KYTY_NOT_IMPLEMENTED; } - KYTY_CLASS_NO_COPY(LabelManager); - - Label* Create(LabelCallback callback_1, LabelCallback callback_2, const uint64_t* args); - void Delete(Label& label); - void Set(CommandBuffer& buffer, Label& label); - void Drain(); - -private: - static void ThreadRun(void* data); - - bool Remove(Label& label); - static void Destroy(Label& label); - - Common::Mutex m_mutex; - Common::CondVar m_cond_var; - std::vector m_labels; - uint64_t m_callbacks_in_flight = 0; -}; - -static LabelManager* g_label_manager = nullptr; -static thread_local bool g_in_label_callback = false; - -class LabelCallbackScope final { -public: - LabelCallbackScope() { - if (g_in_label_callback) { - EXIT("recursive GPU label callback\n"); - } - g_in_label_callback = true; - } - ~LabelCallbackScope() { - if (!g_in_label_callback) { - EXIT("GPU label callback scope is not active\n"); - } - g_in_label_callback = false; - } -}; - -void LabelManager::ThreadRun(void* data) { - auto* manager = static_cast(data); - - for (;;) { - manager->m_mutex.Lock(); - - uint64_t active_count = 0; - - std::vector deleted_labels; - std::vector fired_labels; - std::vector finished_submissions; - deleted_labels.reserve(manager->m_labels.size()); - - for (auto& label: manager->m_labels) { - for (auto it = label->submissions.begin(); it != label->submissions.end();) { - active_count++; - - if (it->completion == nullptr || it->completion->device == nullptr || - it->completion->event == nullptr) { - EXIT("GPU label submission has no completion event\n"); - } - const auto status = it->completion->device.getEventStatus(it->completion->event); - switch (status) { - case vk::Result::eEventSet: - fired_labels.push_back(it->callbacks); - finished_submissions.push_back(*it); - it = label->submissions.erase(it); - break; - case vk::Result::eEventReset: ++it; break; - default: EXIT("vkGetEventStatus returned an unexpected result\n"); - } - } - - if (label->submissions.empty()) { - switch (label->status) { - case LabelStatus::ActiveDeleted: deleted_labels.push_back(label); break; - case LabelStatus::Active: label->status = LabelStatus::NotActive; break; - default: break; - } - } - } - - if (active_count == 0) { - manager->m_cond_var.Wait(&manager->m_mutex); - } - if (fired_labels.size() > UINT64_MAX - manager->m_callbacks_in_flight) { - EXIT("GPU label callback count overflow\n"); - } - manager->m_callbacks_in_flight += fired_labels.size(); - - for (auto& label: deleted_labels) { - bool removed = manager->Remove(*label); - EXIT_NOT_IMPLEMENTED(!removed); - } - - manager->m_mutex.Unlock(); - - // Each completion event is shared with the recording command buffer's fence retainer. - // The event is destroyed only after both the label thread observed it and that command - // buffer completed, even when either side wins the race. - (void)finished_submissions; - - for (auto& label: deleted_labels) { - Destroy(*label); - } - - for (auto& label: fired_labels) { - LabelCallbackScope callback_scope; - - if (label.callback_1 != nullptr) { - (void)label.callback_1(label.args); - } - - if (label.callback_2 != nullptr) { - label.callback_2(label.args); - } - } - - if (!fired_labels.empty()) { - Common::LockGuard lock(manager->m_mutex); - if (manager->m_callbacks_in_flight < fired_labels.size()) { - EXIT("GPU label callback count underflow\n"); - } - manager->m_callbacks_in_flight -= fired_labels.size(); - manager->m_cond_var.SignalAll(); - } - - Common::Thread::SleepMicro(100); - } -} - -Label* LabelManager::Create(LabelCallback callback_1, LabelCallback callback_2, - const uint64_t* args) { - Common::LockGuard lock(m_mutex); - - auto* label = new Label; - - label->status = LabelStatus::New; - label->device = GetRenderContext().GetGraphics().device; - label->callbacks.callback_1 = callback_1; - label->callbacks.callback_2 = callback_2; - - if (args != nullptr) { - for (int i = 0; i < LABEL_ARGS_MAX; i++) { - label->callbacks.args[i] = args[i]; - } - } - - m_labels.push_back(label); - - return label; -} - -bool LabelManager::Remove(Label& label) { - EXIT_IF(label.device == nullptr); - - Common::LockGuard lock(m_mutex); - - const auto it = std::find(m_labels.begin(), m_labels.end(), &label); - EXIT_NOT_IMPLEMENTED(it == m_labels.end()); - - EXIT_NOT_IMPLEMENTED(label.status != LabelStatus::NotActive && - label.status != LabelStatus::Active && - label.status != LabelStatus::ActiveDeleted); - - if (!label.submissions.empty()) { - label.status = LabelStatus::ActiveDeleted; - - return false; - } - - m_labels.erase(it); - - return true; -} - -void LabelManager::Destroy(Label& label) { - EXIT_IF(label.device == nullptr); - - EXIT_NOT_IMPLEMENTED(!label.submissions.empty()); - - delete &label; -} - -void LabelManager::Delete(Label& label) { - if (Remove(label)) { - Destroy(label); - } -} - -void LabelManager::Set(CommandBuffer& buffer, Label& label) { - EXIT_IF(buffer.IsInvalid()); - EXIT_IF(label.device == nullptr); - - Common::LockGuard lock(m_mutex); - - const auto it = std::find(m_labels.begin(), m_labels.end(), &label); - EXIT_NOT_IMPLEMENTED(it == m_labels.end()); - - EXIT_NOT_IMPLEMENTED(label.status != LabelStatus::New && - label.status != LabelStatus::NotActive && - label.status != LabelStatus::Active); - - label.status = LabelStatus::Active; - - LabelSubmission submission {}; - submission.callbacks = label.callbacks; - submission.completion = std::make_shared(); - submission.completion->device = label.device; - - auto vk_buffer = buffer.Handle(); - - EXIT_NOT_IMPLEMENTED(vk_buffer == nullptr); - - vk::EventCreateInfo create_info {}; - create_info.sType = vk::StructureType::eEventCreateInfo; - create_info.pNext = nullptr; - create_info.flags = {}; - - const auto create_result = - label.device.createEvent(&create_info, nullptr, &submission.completion->event); - if (create_result != vk::Result::eSuccess || submission.completion->event == nullptr) { - EXIT("failed to create label event: %s (%d)\n", VulkanToString(create_result).c_str(), - static_cast(create_result)); - } - buffer.RetainResourceUntilFence(submission.completion); - - // Labels can be reused before an earlier end-of-pipe event has been - // observed by the polling thread. Capture a separate Vulkan event and - // callback snapshot for each set so older writes are not lost. - const auto reset_result = label.device.resetEvent(submission.completion->event); - if (reset_result != vk::Result::eSuccess) { - EXIT("failed to reset label event: %s (%d)\n", VulkanToString(reset_result).c_str(), - static_cast(reset_result)); - } - vk_buffer.setEvent(submission.completion->event, vk::PipelineStageFlagBits::eBottomOfPipe); - - label.submissions.push_back(submission); - - m_cond_var.SignalAll(); -} - -void LabelManager::Drain() { - m_mutex.Lock(); - for (;;) { - const bool pending = std::any_of(m_labels.begin(), m_labels.end(), [](const Label* label) { - return !label->submissions.empty(); - }); - if (!pending && m_callbacks_in_flight == 0) { - m_mutex.Unlock(); - return; - } - m_cond_var.SignalAll(); - m_cond_var.Wait(&m_mutex); - } -} - -void LabelInit() { - EXIT_IF(g_label_manager != nullptr); - - g_label_manager = new LabelManager; -} - -Label* LabelCreate(LabelCallback callback_1, LabelCallback callback_2, const uint64_t* args) { - EXIT_IF(g_label_manager == nullptr); - - return g_label_manager->Create(callback_1, callback_2, args); -} - -void LabelDelete(Label& label) { - EXIT_IF(g_label_manager == nullptr); - - g_label_manager->Delete(label); -} - -void LabelSet(CommandBuffer& buffer, Label& label) { - EXIT_IF(g_label_manager == nullptr); - - g_label_manager->Set(buffer, label); -} - -void LabelDrain() { - EXIT_IF(g_label_manager == nullptr); - g_label_manager->Drain(); -} - -bool LabelInCallback() noexcept { - return g_in_label_callback; -} - -} // namespace Libs::Graphics diff --git a/src/graphics/host_gpu/objects/label.h b/src/graphics/host_gpu/objects/label.h deleted file mode 100644 index 19db369e..00000000 --- a/src/graphics/host_gpu/objects/label.h +++ /dev/null @@ -1,26 +0,0 @@ -#ifndef EMULATOR_INCLUDE_EMULATOR_GRAPHICS_OBJECTS_LABEL_H_ -#define EMULATOR_INCLUDE_EMULATOR_GRAPHICS_OBJECTS_LABEL_H_ - -#include "common/abi.h" -#include "common/common.h" - -namespace Libs::Graphics { - -struct Label; -class CommandBuffer; -struct GraphicContext; - -void LabelInit(); - -constexpr int LABEL_ARGS_MAX = 5; -using LabelCallback = bool (*)(const uint64_t* args); - -Label* LabelCreate(LabelCallback callback_1, LabelCallback callback_2, const uint64_t* args); -void LabelDelete(Label& label); -void LabelSet(CommandBuffer& buffer, Label& label); -void LabelDrain(); -[[nodiscard]] bool LabelInCallback() noexcept; - -} // namespace Libs::Graphics - -#endif /* EMULATOR_INCLUDE_EMULATOR_GRAPHICS_OBJECTS_LABEL_H_ */ diff --git a/src/graphics/host_gpu/objects/textureCommon.cpp b/src/graphics/host_gpu/objects/textureCommon.cpp index bbd0c500..391e27b4 100644 --- a/src/graphics/host_gpu/objects/textureCommon.cpp +++ b/src/graphics/host_gpu/objects/textureCommon.cpp @@ -1,17 +1,12 @@ #include "graphics/host_gpu/objects/textureCommon.h" #include "common/assert.h" -#include "common/logging/log.h" #include "graphics/guest_gpu/gpu_defs.h" #include "graphics/guest_gpu/gpu_format.h" -#include "graphics/host_gpu/graphicContext.h" -#include "graphics/host_gpu/renderer/renderContext.h" -#include "graphics/host_gpu/transfer.h" -#include "graphics/host_gpu/vma.h" +#include "graphics/host_gpu/renderer/tiler.h" #include "graphics/host_gpu/vulkanCommon.h" #include -#include #include #include #include @@ -159,6 +154,8 @@ RenderTargetFormatInfo TextureGetRenderTargetFormat(uint32_t raw_layout, uint32_ raw_type, raw_order); } +namespace { + static uint64_t GetLevelSrcOffset(const TileSizeOffset& level_size) { return (level_size.src_size != 0 ? level_size.src_offset : level_size.offset); } @@ -193,28 +190,6 @@ uint64_t TextureUploadSliceSourceOffset(const TextureUploadLayout& layout, uint3 return level_offset + static_cast(slice) * slice_stride; } -uint64_t TextureCalcUploadSize(const TextureUploadLayout& layout, - const std::vector& regions, uint64_t levels, - uint32_t depth) { - uint64_t size = 0; - - for (const auto& r: regions) { - size = std::max(size, static_cast(r.offset) + - layout.level_sizes[r.dst_level].size); - } - - for (uint32_t level = 0; level < levels; level++) { - const auto src_size = GetLevelSrcSize(layout.level_sizes[level]); - const auto mip_depth = GetTextureLevelDepth(depth, level, layout.volume_texture); - for (uint32_t z = 0; z < mip_depth; z++) { - size = std::max(size, - TextureUploadSliceSourceOffset(layout, level, z) + src_size); - } - } - - return size; -} - vk::ComponentSwizzle TextureGetComponentSwizzle(uint8_t s) { switch (static_cast(s)) { case Prospero::CompSwizzle::kZero: return vk::ComponentSwizzle::eZero; @@ -232,6 +207,8 @@ static uint32_t TextureGetDstSel(uint32_t swizzle, uint32_t channel) { return (swizzle >> (channel * 3u)) & 0x7u; } +} // namespace + vk::ComponentMapping TextureGetComponentMapping(uint32_t swizzle) { vk::ComponentMapping components {}; components.r = TextureGetComponentSwizzle(static_cast(TextureGetDstSel(swizzle, 0))); @@ -241,94 +218,6 @@ vk::ComponentMapping TextureGetComponentMapping(uint32_t swizzle) { return components; } -bool TextureCheckFormat(vk::ImageCreateInfo& image_info) { - auto& graphics = GetRenderContext().GetGraphics(); - vk::ImageFormatProperties props {}; - if (graphics.GetImageFormatProperties(image_info.format, image_info.imageType, - image_info.tiling, image_info.usage, image_info.flags, - &props) == vk::Result::eErrorFormatNotSupported) { - auto apply_fallback = [&](vk::Format replacement, const char* message) { - image_info.format = replacement; - const bool result = TextureCheckFormat(image_info); - LOGF("%s [%s]\n", message, (!result ? "FAIL" : "SUCCESS")); - return result; - }; - - if (image_info.format == vk::Format::eR8G8B8A8Srgb) { - // TODO() convert SRGB -> LINEAR in shader - return apply_fallback( - vk::Format::eR8G8B8A8Unorm, - "replace vk::Format::eR8G8B8A8Srgb => vk::Format::eR8G8B8A8Unorm"); - } - if (image_info.format == vk::Format::eB8G8R8A8Srgb) { - // TODO() convert SRGB -> LINEAR in shader - return apply_fallback( - vk::Format::eB8G8R8A8Unorm, - "replace vk::Format::eB8G8R8A8Srgb => vk::Format::eB8G8R8A8Unorm"); - } - return false; - } - return true; -} - -static bool TextureCheckFormatExact(const vk::ImageCreateInfo& image_info) { - auto& graphics = GetRenderContext().GetGraphics(); - vk::ImageFormatProperties props {}; - return graphics.GetImageFormatProperties(image_info.format, image_info.imageType, - image_info.tiling, image_info.usage, image_info.flags, - &props) != vk::Result::eErrorFormatNotSupported; -} - -bool TextureCheckStorageSwizzle(vk::ImageCreateInfo& image_info, vk::ComponentMapping& components) { - if (image_info.usage & vk::ImageUsageFlagBits::eStorage) { - if (components.r == vk::ComponentSwizzle::eR && components.g == vk::ComponentSwizzle::eG && - components.b == vk::ComponentSwizzle::eB && components.a == vk::ComponentSwizzle::eA) { - return true; - } - - if (components.r == vk::ComponentSwizzle::eB && components.g == vk::ComponentSwizzle::eG && - components.b == vk::ComponentSwizzle::eR && components.a == vk::ComponentSwizzle::eA && - image_info.format == vk::Format::eR8G8B8A8Srgb) { - LOGF("replace vk::Format::eR8G8B8A8Srgb => vk::Format::eB8G8R8A8Srgb\n"); - - components.r = vk::ComponentSwizzle::eR; - components.g = vk::ComponentSwizzle::eG; - components.b = vk::ComponentSwizzle::eB; - components.a = vk::ComponentSwizzle::eA; - image_info.format = vk::Format::eB8G8R8A8Srgb; - return true; - } - - // TODO() swizzle channels in shader - - return false; - } - return true; -} - -vk::ImageUsageFlags TextureGetUsage(TextureFormatUsage usage) { - vk::ImageUsageFlags vk_usage = - vk::ImageUsageFlagBits::eTransferDst | vk::ImageUsageFlagBits::eTransferSrc; - if (TextureHasFormatUsage(usage, TextureFormatUsage::Sampled)) { - vk_usage |= vk::ImageUsageFlagBits::eSampled; - } - if (TextureHasFormatUsage(usage, TextureFormatUsage::Storage)) { - vk_usage |= vk::ImageUsageFlagBits::eStorage; - } - return vk_usage; -} - -vk::ImageUsageFlags TextureGetViewUsage(TextureFormatUsage usage) { - vk::ImageUsageFlags vk_usage = {}; - if (TextureHasFormatUsage(usage, TextureFormatUsage::Sampled)) { - vk_usage |= vk::ImageUsageFlagBits::eSampled; - } - if (TextureHasFormatUsage(usage, TextureFormatUsage::Storage)) { - vk_usage |= vk::ImageUsageFlagBits::eStorage; - } - return vk_usage; -} - vk::Format TextureGetFormat(uint32_t fmt) { const auto vk_format = VulkanFormat(fmt); if (vk_format != vk::Format::eUndefined) { @@ -338,261 +227,10 @@ vk::Format TextureGetFormat(uint32_t fmt) { return vk::Format::eUndefined; } -static uint32_t AlignUpU32(uint32_t value, uint32_t alignment) { - return (value + alignment - 1u) & ~(alignment - 1u); -} +namespace { -uint32_t TextureGetAtlasSliceYStride(vk::Format format, uint32_t mip_height, uint32_t depth, - uint64_t levels) { - return (depth > 1 && levels > 1 && Transfer::IsBlockCompressedFormat(format) - ? AlignUpU32(mip_height, 4u) - : mip_height); -} - -uint32_t TextureCalcStackedImageHeight(vk::Format format, uint32_t height, uint32_t depth, - uint64_t levels) { - auto image_height = height * depth; - if (depth <= 1 || levels <= 1 || !Transfer::IsBlockCompressedFormat(format)) { - return image_height; - } - - uint32_t mip_height = height; - for (uint32_t level = 0; level < levels; level++) { - const auto mip_image_height = - TextureGetAtlasSliceYStride(format, mip_height, depth, levels) * depth; - image_height = std::max(image_height, mip_image_height << level); - if (mip_height > 1) { - mip_height /= 2; - } - } - - return image_height; -} - -uint32_t TextureCalcMipmapAtlasImageHeight(vk::Format format, uint32_t width, uint32_t height, - uint32_t depth, uint64_t levels) { - auto image_height = height * depth; - if (levels <= 1) { - return image_height; - } - - uint32_t mip_height = height; - for (uint32_t level = 0; level < levels; level++) { - const auto mipmap_offset = Transfer::MipmapAtlasOffset(level, width, height); - const auto mip_bottom = - static_cast(mipmap_offset.second) + - TextureGetAtlasSliceYStride(format, mip_height, depth, levels) * depth; - image_height = std::max(image_height, mip_bottom); - if (mip_height > 1) { - mip_height /= 2; - } - } - - return image_height; -} - -bool TextureIs3DTexture(uint64_t type) { - return static_cast(type) == Prospero::ImageType::kColor3D; -} - -bool TextureIsCubeTexture(uint64_t type) { - return static_cast(type) == Prospero::ImageType::kCube; -} - -bool TextureIsLayeredTexture(uint64_t type) { - const auto image_type = static_cast(type); - return image_type == Prospero::ImageType::kCube || - image_type == Prospero::ImageType::kColor1DArray || - image_type == Prospero::ImageType::kColor2DArray || - image_type == Prospero::ImageType::kColor2DMsaaArray; -} - -bool TextureCanCreateCubeView(uint64_t type, uint32_t base_array, uint32_t layer_count) { - return TextureIsCubeTexture(type) && base_array % 6u == 0 && layer_count >= 6 && - layer_count % 6u == 0; -} - -vk::ComponentMapping TextureCreateImage(VulkanImage& vk_obj, - const TextureImageCreateParams& params) { - auto& graphics = GetRenderContext().GetGraphics(); - EXIT_IF(params.owner == nullptr); - - const bool array_texture = TextureIsLayeredTexture(params.type); - const bool volume_texture = TextureIs3DTexture(params.type); - - auto pixel_format = TextureGetFormat(params.fmt); - EXIT_NOT_IMPLEMENTED(pixel_format == vk::Format::eUndefined); - EXIT_NOT_IMPLEMENTED(params.width == 0); - EXIT_NOT_IMPLEMENTED(params.height == 0); - EXIT_NOT_IMPLEMENTED(params.depth == 0); - EXIT_NOT_IMPLEMENTED(params.levels == 0 || params.levels > 16); - - uint32_t image_height = 0; - uint32_t image_mips = 0; - if (params.image_layout == TextureUploadDestination::MipAtlas) { - EXIT_NOT_IMPLEMENTED(params.base_level != 0); - const auto atlas_depth = (array_texture || volume_texture ? 1u : params.depth); - image_height = TextureCalcMipmapAtlasImageHeight( - pixel_format, static_cast(params.width), static_cast(params.height), - atlas_depth, params.levels); - image_mips = 1; - } else { - image_height = - (array_texture || volume_texture - ? static_cast(params.height) - : TextureCalcStackedImageHeight(pixel_format, static_cast(params.height), - params.depth, params.levels)); - image_mips = static_cast(params.levels); - } - - vk::ComponentMapping components = - TextureGetComponentMapping(static_cast(params.swizzle)); - - vk::ImageCreateInfo image_info {}; - image_info.sType = vk::StructureType::eImageCreateInfo; - image_info.pNext = nullptr; - image_info.flags = - (params.allow_cube_view && TextureCanCreateCubeView(params.type, 0, params.depth) - ? vk::ImageCreateFlagBits::eCubeCompatible - : vk::ImageCreateFlags {}) | - (volume_texture ? vk::ImageCreateFlagBits::e2DArrayCompatible : vk::ImageCreateFlags {}) | - (params.compatible_format_views ? vk::ImageCreateFlagBits::eMutableFormat - : vk::ImageCreateFlags {}); - image_info.imageType = (volume_texture ? vk::ImageType::e3D : vk::ImageType::e2D); - image_info.extent.width = static_cast(params.width); - image_info.extent.height = image_height; - image_info.extent.depth = (volume_texture ? params.depth : 1); - image_info.mipLevels = image_mips; - image_info.arrayLayers = (array_texture ? params.depth : 1); - image_info.format = pixel_format; - image_info.tiling = vk::ImageTiling::eOptimal; - image_info.initialLayout = vk::ImageLayout::eUndefined; - image_info.usage = TextureGetUsage(params.format_usage); - image_info.sharingMode = vk::SharingMode::eExclusive; - image_info.samples = vk::SampleCountFlagBits::e1; - - const bool storage_view = TextureHasFormatUsage(params.view_usage, TextureFormatUsage::Storage); - if (storage_view && !TextureCheckStorageSwizzle(image_info, components)) { - if (!params.storage_swizzle_fallback) { - EXIT("swizzle is not supported"); - } - - static std::atomic_uint log_count {0}; - if (log_count.fetch_add(1, std::memory_order_relaxed) < 16) { - LOGF("\t %s swizzle 0x%08" PRIx64 " is not supported, using identity mapping\n", - params.owner, params.swizzle); - } - components.r = vk::ComponentSwizzle::eR; - components.g = vk::ComponentSwizzle::eG; - components.b = vk::ComponentSwizzle::eB; - components.a = vk::ComponentSwizzle::eA; - } - - const auto view_checked_format = image_info.format; - const auto requested_usage = params.format_usage; - const auto required_usage = params.required_format_usage; - const bool has_optional_usage = static_cast(requested_usage & ~required_usage) != 0; - if (!TextureCheckFormatExact(image_info) && has_optional_usage) { - auto required_info = image_info; - required_info.usage = TextureGetUsage(required_usage); - required_info.format = view_checked_format; - if (TextureCheckFormatExact(required_info)) { - static std::atomic_uint log_count {0}; - if (log_count.fetch_add(1, std::memory_order_relaxed) < 16) { - LOGF("\t %s usage 0x%08x is not supported for format %d, using required " - "usage 0x%08x\n", - params.owner, static_cast(TextureGetUsage(requested_usage)), - static_cast(image_info.format), - static_cast(TextureGetUsage(required_usage))); - } - image_info = required_info; - } - } - - if (!TextureCheckFormat(image_info)) { - if (has_optional_usage) { - image_info.format = view_checked_format; - image_info.usage = TextureGetUsage(required_usage); - } - if (!has_optional_usage || !TextureCheckFormat(image_info)) { - EXIT("format is not supported"); - } - } - - vk_obj.extent.width = image_info.extent.width; - vk_obj.extent.height = image_info.extent.height; - vk_obj.layers = image_info.arrayLayers; - vk_obj.mip_levels = image_info.mipLevels; - vk_obj.format = image_info.format; - vk_obj.image = nullptr; - vk_obj.layout = image_info.initialLayout; - - vk_obj.memory.property = vk::MemoryPropertyFlagBits::eDeviceLocal; - - bool created = graphics.CreateImage(image_info, vk_obj); - EXIT_NOT_IMPLEMENTED(!created); - - return components; -} - -void TextureCreateImageViews(VulkanImage& vk_obj, - vk::ComponentMapping components, uint64_t type, uint32_t base_array, - uint32_t base_level, uint32_t level_count, uint32_t depth, - bool allow_cube_view, TextureFormatUsage view_usage) { - auto& graphics = GetRenderContext().GetGraphics(); - EXIT_IF(level_count == 0 || base_level + level_count > vk_obj.mip_levels); - - const bool layered_texture = TextureIsLayeredTexture(type); - const bool volume_texture = TextureIs3DTexture(type); - const auto layer_count = (layered_texture && base_array < depth ? depth - base_array : 1u); - const auto volume_slices = std::max(depth >> base_level, 1u); - - vk::ImageViewUsageCreateInfo usage_info {}; - usage_info.sType = vk::StructureType::eImageViewUsageCreateInfo; - usage_info.pNext = nullptr; - usage_info.usage = TextureGetViewUsage(view_usage); - - vk::ImageViewCreateInfo create_info {}; - create_info.sType = vk::StructureType::eImageViewCreateInfo; - create_info.pNext = (usage_info.usage ? &usage_info : nullptr); - create_info.flags = {}; - create_info.image = vk_obj.image; - create_info.viewType = vk::ImageViewType::e2D; - create_info.format = vk_obj.format; - create_info.components = components; - create_info.subresourceRange.aspectMask = vk::ImageAspectFlagBits::eColor; - create_info.subresourceRange.baseArrayLayer = (layered_texture ? base_array : 0); - create_info.subresourceRange.baseMipLevel = base_level; - create_info.subresourceRange.layerCount = layer_count; - create_info.subresourceRange.levelCount = level_count; - if (volume_texture) { - create_info.viewType = vk::ImageViewType::e3D; - create_info.subresourceRange.baseArrayLayer = 0; - create_info.subresourceRange.layerCount = 1; - } else if (allow_cube_view && TextureCanCreateCubeView(type, base_array, layer_count)) { - create_info.viewType = - (layer_count > 6 ? vk::ImageViewType::eCubeArray : vk::ImageViewType::eCube); - } else if (layered_texture) { - create_info.viewType = vk::ImageViewType::e2DArray; - } - - auto create_view = [&](int index) { - const auto result = - graphics.device.createImageView(&create_info, nullptr, &vk_obj.image_view[index]); - if (result != vk::Result::eSuccess || vk_obj.image_view[index] == nullptr) { - EXIT("failed to create texture image view: result=%d format=%d index=%d\n", - static_cast(result), static_cast(create_info.format), index); - } - }; - create_view(VulkanImage::VIEW_DEFAULT); - - create_info.viewType = vk::ImageViewType::e2DArray; - create_info.subresourceRange.layerCount = volume_texture ? volume_slices : layer_count; - create_view(VulkanImage::VIEW_DEFAULT_ARRAY); -} - -static uint64_t CalcTextureSliceStride(const TileSizeOffset* level_sizes, uint64_t levels, - uint64_t total_size, uint32_t depth) { +uint64_t CalcTextureSliceStride(const TileSizeOffset* level_sizes, uint64_t levels, + uint64_t total_size, uint32_t depth) { uint64_t stride = 0; for (uint32_t i = 0; i < levels; i++) { stride = @@ -609,7 +247,7 @@ static uint64_t CalcTextureSliceStride(const TileSizeOffset* level_sizes, uint64 return stride; } -static uint64_t CalcLinearUploadLevelSize(uint32_t fmt, uint32_t pitch, uint32_t height) { +uint64_t CalcLinearUploadLevelSize(uint32_t fmt, uint32_t pitch, uint32_t height) { if (const uint32_t bytes_per_element = Prospero::NumBytesPerElement(fmt); bytes_per_element != 0) { return static_cast(pitch) * height * bytes_per_element; @@ -625,8 +263,8 @@ static uint64_t CalcLinearUploadLevelSize(uint32_t fmt, uint32_t pitch, uint32_t return 0; } -static uint64_t SetLinearUploadLevels(TileSizeOffset* level_sizes, uint32_t fmt, uint64_t height, - uint64_t levels, uint32_t base_pitch) { +uint64_t SetLinearUploadLevels(TileSizeOffset* level_sizes, uint32_t fmt, uint64_t height, + uint64_t levels, uint32_t base_pitch) { uint64_t offset = 0; auto pitch = base_pitch; auto h = static_cast(height); @@ -655,6 +293,8 @@ static uint64_t SetLinearUploadLevels(TileSizeOffset* level_sizes, uint32_t fmt, return offset; } +} // namespace + TextureUploadLayout TextureCalcUploadLayout(uint32_t fmt, uint64_t width, uint64_t height, uint64_t levels, uint32_t depth, uint64_t pitch, uint64_t tile, uint64_t upload_size, @@ -663,6 +303,7 @@ TextureUploadLayout TextureCalcUploadLayout(uint32_t fmt, uint64_t width, uint64 TextureUploadLayout layout {}; layout.tile = static_cast(tile); layout.pitch = static_cast(pitch); + layout.texel_block = Prospero::BlockCompressedBytesPerBlock(fmt) != 0 ? 4u : 1u; layout.volume_texture = volume_texture; if (fmt != 0) { @@ -752,12 +393,10 @@ TextureUploadLayout TextureCalcUploadLayout(uint32_t fmt, uint64_t width, uint64 return layout; } -std::vector TextureBuildUploadRegions(const TextureUploadLayout& layout, - vk::Format image_format, uint32_t width, - uint32_t height, uint32_t depth, - uint64_t levels, bool array_texture, - bool volume_texture, - TextureUploadDestination destination) { +std::vector +TextureBuildImageCopies(const TextureUploadLayout& layout, uint32_t width, uint32_t height, + uint32_t depth, uint64_t levels, bool array_texture, + bool volume_texture) { uint32_t mip_width = width; uint32_t mip_height = height; uint32_t mip_pitch = volume_texture && static_cast(layout.tile) != @@ -765,53 +404,34 @@ std::vector TextureBuildUploadRegions(const TextureUploadLayout ? width : layout.pitch; - std::vector regions; + std::vector regions; regions.reserve(GetTextureRegionCount(depth, levels, volume_texture)); for (uint32_t i = 0; i < levels; i++) { EXIT_NOT_IMPLEMENTED(layout.level_sizes[i].size == 0); - const auto mipmap_offset = Transfer::MipmapAtlasOffset(i, width, height); - const auto mip_depth = GetTextureLevelDepth(depth, i, volume_texture); + const auto mip_depth = GetTextureLevelDepth(depth, i, volume_texture); for (uint32_t z = 0; z < mip_depth; z++) { - const auto slice_offset = z * layout.slice_stride; - BufferImageCopy region {}; - - region.offset = static_cast(layout.level_sizes[i].offset + slice_offset); - region.width = mip_width; - region.height = mip_height; - region.copy_height = - (!array_texture && !volume_texture && depth > 1 && levels > 1 && - Transfer::IsBlockCompressedFormat(image_format) - ? TextureGetAtlasSliceYStride(image_format, mip_height, depth, levels) - : 0); - region.dst_layer = (array_texture ? z : 0); - region.dst_z = (volume_texture ? static_cast(z) : 0); - if (!layout.volume_texture && - static_cast(layout.tile) == Prospero::TileMode::kLinear && - layout.padded_sizes[i].width != 0) { - region.pitch = layout.padded_sizes[i].width; + const auto slice_offset = z * layout.slice_stride; + vk::BufferImageCopy region {}; + region.bufferOffset = + layout.level_sizes[i].offset + slice_offset; + region.imageSubresource = {vk::ImageAspectFlagBits::eColor, i, + array_texture ? z : 0, 1}; + region.imageOffset.z = volume_texture ? static_cast(z) : 0; + region.imageExtent = {mip_width, mip_height, 1}; + const bool linear = + static_cast(layout.tile) == Prospero::TileMode::kLinear; + if (linear) { + region.bufferRowLength = layout.padded_sizes[i].width; + region.bufferImageHeight = layout.padded_sizes[i].height; } else { - region.pitch = mip_pitch; - } - - if (destination == TextureUploadDestination::MipLevels) { - region.dst_level = i; - region.dst_x = 0; - region.dst_y = - (array_texture || volume_texture - ? 0 - : static_cast(z * TextureGetAtlasSliceYStride( - image_format, mip_height, depth, levels))); - } else { - region.dst_level = 0; - region.dst_x = mipmap_offset.first; - region.dst_y = - (array_texture || volume_texture - ? mipmap_offset.second - : mipmap_offset.second + - static_cast(z * TextureGetAtlasSliceYStride( - image_format, mip_height, depth, levels))); + const auto align = [](uint32_t value, uint32_t block) { + return ((value + block - 1u) / block) * block; + }; + const auto pitch = align(mip_pitch, layout.texel_block); + region.bufferRowLength = + pitch > align(mip_width, layout.texel_block) ? pitch : 0; } regions.push_back(region); } @@ -830,55 +450,6 @@ std::vector TextureBuildUploadRegions(const TextureUploadLayout return regions; } -std::vector -TextureBuildDownloadRegions(const std::vector& upload_regions) { - std::vector regions; - regions.reserve(upload_regions.size()); - for (const auto& region: upload_regions) { - regions.push_back({region.offset, region.pitch, region.dst_level, region.width, - region.height, region.copy_height, region.dst_layer, region.dst_x, - region.dst_y, region.dst_z, region.aspect}); - } - return regions; -} - -static uint64_t FmaskRegionCopySize(const BufferImageCopy& region) { - const uint64_t row_length = (region.pitch != 0 ? region.pitch : region.width); - if (region.height == 0 || region.width == 0) { - return 0; - } - return ((static_cast(region.height - 1u) * row_length) + region.width) * - sizeof(uint32_t); -} - -static void UploadFmaskIdentity(VulkanImage& vk_obj, - const std::vector& regions, - vk::ImageLayout dst_layout, const char* owner) { - constexpr uint32_t kIdentityFmaskPattern = 0x76543210u; - - std::vector upload_regions = regions; - uint64_t upload_size = 0; - for (auto& region: upload_regions) { - upload_size = (upload_size + 255u) & ~uint64_t {255}; - EXIT_NOT_IMPLEMENTED(upload_size > UINT32_MAX); - region.offset = static_cast(upload_size); - upload_size += FmaskRegionCopySize(region); - } - - LOGF("%s: temporary: decoding PS5 FMASK8_S4_F4 as identity pattern 0x%08" PRIx32 - ", upload_size=%" PRIu64 " regions=%zu\n", - owner, kIdentityFmaskPattern, upload_size, upload_regions.size()); - - Transfer::ScratchBuffer temp_buf(upload_size); - auto* words = static_cast(temp_buf.Data()); - for (uint64_t i = 0; i < upload_size / sizeof(uint32_t); i++) { - words[i] = kIdentityFmaskPattern; - } - - Transfer::UploadImage(vk_obj, temp_buf.Data(), upload_size, upload_regions, - dst_layout); -} - struct GpuTileElementLayout { uint32_t bytes = 0; uint32_t wide = 1; @@ -905,7 +476,8 @@ static bool SetGpuTileSize(uint64_t offset, uint64_t length, uint64_t capacity, return true; } -bool TextureBuildGpuTileInfos(uint64_t size, const std::vector& regions, +bool TextureBuildGpuTileInfos(uint64_t size, + const std::vector& regions, const TextureUploadLayout& layout, uint32_t fmt, uint32_t depth, uint64_t levels, std::vector& out_infos) { if (size == 0 || levels == 0 || levels > 16 || depth == 0 || @@ -929,7 +501,8 @@ bool TextureBuildGpuTileInfos(uint64_t size, const std::vector& infos.reserve(regions.size()); if (layout.volume_texture) { TileVolumeLayout volume {}; - if (!TileGetTextureVolumeLayout(fmt, regions[0].width, regions[0].height, depth, + if (!TileGetTextureVolumeLayout(fmt, regions[0].imageExtent.width, + regions[0].imageExtent.height, depth, static_cast(levels), layout.tile, volume)) { return false; } @@ -945,10 +518,16 @@ bool TextureBuildGpuTileInfos(uint64_t size, const std::vector& for (uint32_t z = 0; z < mip_depth; z += block.block_depth) { const uint32_t copy_depth = std::min(block.block_depth, mip_depth - z); const auto& region = regions[region_base + z]; + const auto pitch = region.bufferRowLength != 0 + ? region.bufferRowLength + : region.imageExtent.width; + const auto logical_height = region.bufferImageHeight != 0 + ? region.bufferImageHeight + : region.imageExtent.height; GpuTileInfo info {}; info.family = block.family; info.bytes_per_element = block.bytes_per_element; - info.linear_offset = region.offset; + info.linear_offset = region.bufferOffset; info.tiled_offset = static_cast(z / block.block_depth) * volume.block_slice_size + volume.level_offsets[level]; @@ -961,11 +540,16 @@ bool TextureBuildGpuTileInfos(uint64_t size, const std::vector& return false; } info.linear_slice_stride = linear_stride; - info.width = std::max((region.width + element.wide - 1u) / element.wide, 1u); - info.height = std::max((region.height + element.tall - 1u) / element.tall, 1u); + info.width = std::max( + (region.imageExtent.width + element.wide - 1u) / element.wide, 1u); + info.height = std::max( + (logical_height + element.tall - 1u) / element.tall, 1u); info.depth = copy_depth; - info.surface_z = block.block_depth == 1 ? static_cast(region.dst_z) : 0; - info.pitch = std::max((region.pitch + element.wide - 1u) / element.wide, 1u); + info.surface_z = block.block_depth == 1 + ? static_cast(region.imageOffset.z) + : 0; + info.pitch = + std::max((pitch + element.wide - 1u) / element.wide, 1u); info.tail_x = tail ? volume.tail_x[level] : 0; info.tail_y = tail ? volume.tail_y[level] : 0; info.tail = tail; @@ -993,23 +577,32 @@ bool TextureBuildGpuTileInfos(uint64_t size, const std::vector& const auto level_depth = GetTextureLevelDepth(depth, level, layout.volume_texture); for (uint32_t z = 0; z < level_depth; z++) { const auto& region = regions[region_index++]; + const auto pitch = region.bufferRowLength != 0 + ? region.bufferRowLength + : region.imageExtent.width; + const auto logical_height = region.bufferImageHeight != 0 + ? region.bufferImageHeight + : region.imageExtent.height; GpuTileInfo info {}; info.family = block.family; info.bytes_per_element = block.bytes_per_element; - info.linear_offset = region.offset; + info.linear_offset = region.bufferOffset; info.tiled_offset = TextureUploadSliceSourceOffset(layout, level, z); if (!SetGpuTileSize(info.linear_offset, level_size.size, size, info.linear_size) || !SetGpuTileSize(info.tiled_offset, GetLevelSrcSize(level_size), size, info.tiled_size)) { return false; } - info.width = std::max((region.width + element.wide - 1u) / element.wide, 1u); - info.height = std::max((region.height + element.tall - 1u) / element.tall, 1u); + info.width = std::max( + (region.imageExtent.width + element.wide - 1u) / element.wide, 1u); + info.height = std::max( + (logical_height + element.tall - 1u) / element.tall, 1u); info.surface_z = base_family == TileBlockFamily::RenderTarget64KB || base_family == TileBlockFamily::Depth64KB - ? region.dst_layer + ? region.imageSubresource.baseArrayLayer : 0; - info.pitch = std::max((region.pitch + element.wide - 1u) / element.wide, 1u); + info.pitch = + std::max((pitch + element.wide - 1u) / element.wide, 1u); info.tail = tail; info.tail_x = tail ? level_size.x : 0; info.tail_y = tail ? level_size.y : 0; @@ -1037,32 +630,4 @@ bool TextureBuildGpuTileInfos(uint64_t size, const std::vector& return true; } -void TextureUploadGuestImage(VulkanImage& vk_obj, const void* src_data, - uint64_t size, const std::vector& regions, - const TextureUploadLayout& layout, uint32_t fmt, uint64_t width, - uint64_t height, uint32_t depth, uint64_t levels, const char* owner, - vk::ImageLayout dst_layout) { - if (fmt == 0) { - EXIT("%s: texture upload format unsupported: fmt=0 tile=%u size=%" PRIu64 " extent=%" PRIu64 - "x%" PRIu64 " depth=%u pitch=%u levels=%" PRIu64 "\n", - owner, layout.tile, size, width, height, depth, layout.pitch, levels); - } - if (static_cast(layout.tile) == Prospero::TileMode::kLinear) { - Transfer::UploadImage(vk_obj, src_data, size, regions, dst_layout); - return; - } - if (layout.tile_family == TileBlockFamily::Depth64KB && Prospero::IsFmaskTextureFormat(fmt)) { - UploadFmaskIdentity(vk_obj, regions, dst_layout, owner); - return; - } - - std::vector infos; - if (!TextureBuildGpuTileInfos(size, regions, layout, fmt, depth, levels, infos)) { - EXIT("%s: GPU tiled upload unsupported: fmt=%u tile=%u size=%" PRIu64 " extent=%" PRIu64 - "x%" PRIu64 " depth=%u pitch=%u levels=%" PRIu64 "\n", - owner, fmt, layout.tile, size, width, height, depth, layout.pitch, levels); - } - Transfer::UploadTiledImage(vk_obj, src_data, size, size, infos, regions, dst_layout); -} - } // namespace Libs::Graphics diff --git a/src/graphics/host_gpu/objects/textureCommon.h b/src/graphics/host_gpu/objects/textureCommon.h index 63ff5801..69f76f88 100644 --- a/src/graphics/host_gpu/objects/textureCommon.h +++ b/src/graphics/host_gpu/objects/textureCommon.h @@ -5,44 +5,13 @@ #include "common/common.h" #include "graphics/guest_gpu/gpu_defs.h" #include "graphics/guest_gpu/tile.h" -#include "graphics/host_gpu/transfer.h" #include "graphics/host_gpu/vulkanCommon.h" #include namespace Libs::Graphics { -struct GraphicContext; -struct VulkanImage; - -enum class TextureFormatUsage : uint32_t { - None = 0, - Sampled = 1u << 0u, - Storage = 1u << 1u, -}; - -constexpr TextureFormatUsage operator|(TextureFormatUsage lhs, TextureFormatUsage rhs) { - return static_cast(static_cast(lhs) | static_cast(rhs)); -} - -constexpr TextureFormatUsage operator&(TextureFormatUsage lhs, TextureFormatUsage rhs) { - return static_cast(static_cast(lhs) & static_cast(rhs)); -} - -constexpr TextureFormatUsage operator~(TextureFormatUsage usage) { - return static_cast(~static_cast(usage)); -} - -constexpr TextureFormatUsage& operator|=(TextureFormatUsage& lhs, TextureFormatUsage rhs) { - lhs = lhs | rhs; - return lhs; -} - -constexpr bool TextureHasFormatUsage(TextureFormatUsage usage, TextureFormatUsage flag) { - return (static_cast(usage & flag) == static_cast(flag)); -} - -enum class TextureUploadDestination { MipLevels, MipAtlas }; +struct GpuTileInfo; struct RenderTargetFormatInfo { vk::Format format = vk::Format::eUndefined; @@ -53,6 +22,7 @@ struct RenderTargetFormatInfo { struct TextureUploadLayout { uint32_t tile = 0; uint32_t pitch = 0; + uint32_t texel_block = 1; uint64_t slice_stride = 0; uint64_t source_slice_stride = 0; uint32_t first_tail_level = 16; @@ -62,75 +32,22 @@ struct TextureUploadLayout { TilePaddedSize padded_sizes[16] = {}; }; -struct TextureImageCreateParams { - uint32_t fmt = 0; - uint64_t width = 0; - uint64_t height = 0; - uint32_t base_level = 0; - uint64_t levels = 1; - uint32_t depth = 1; - uint64_t type = 0; - uint64_t swizzle = 0; - TextureFormatUsage format_usage = TextureFormatUsage::Sampled; - TextureFormatUsage required_format_usage = TextureFormatUsage::Sampled; - TextureFormatUsage view_usage = TextureFormatUsage::Sampled; - TextureUploadDestination image_layout = TextureUploadDestination::MipLevels; - bool allow_cube_view = false; - bool compatible_format_views = false; - bool storage_swizzle_fallback = false; - const char* owner = nullptr; -}; - -vk::ComponentSwizzle TextureGetComponentSwizzle(uint8_t s); vk::ComponentMapping TextureGetComponentMapping(uint32_t swizzle); -bool TextureCheckFormat(vk::ImageCreateInfo& image_info); -bool TextureCheckStorageSwizzle(vk::ImageCreateInfo& image_info, vk::ComponentMapping& components); -vk::ImageUsageFlags TextureGetUsage(TextureFormatUsage usage); -vk::ImageUsageFlags TextureGetViewUsage(TextureFormatUsage usage); vk::Format TextureGetFormat(uint32_t fmt); RenderTargetFormatInfo TextureGetRenderTargetFormat(uint32_t layout, uint32_t type, uint32_t order); -uint32_t TextureGetAtlasSliceYStride(vk::Format format, uint32_t mip_height, uint32_t depth, - uint64_t levels); -uint32_t TextureCalcStackedImageHeight(vk::Format format, uint32_t height, uint32_t depth, - uint64_t levels); -uint32_t TextureCalcMipmapAtlasImageHeight(vk::Format format, uint32_t width, uint32_t height, - uint32_t depth, uint64_t levels); -bool TextureIs3DTexture(uint64_t type); -bool TextureIsCubeTexture(uint64_t type); -bool TextureIsLayeredTexture(uint64_t type); -bool TextureCanCreateCubeView(uint64_t type, uint32_t base_array, uint32_t layer_count); -vk::ComponentMapping TextureCreateImage(VulkanImage& image, - const TextureImageCreateParams& params); -void TextureCreateImageViews(VulkanImage& vk_obj, - vk::ComponentMapping components, uint64_t type, uint32_t base_array, - uint32_t base_level, uint32_t level_count, uint32_t depth, - bool allow_cube_view, TextureFormatUsage view_usage); TextureUploadLayout TextureCalcUploadLayout(uint32_t fmt, uint64_t width, uint64_t height, uint64_t levels, uint32_t depth, uint64_t pitch, uint64_t tile, uint64_t upload_size, bool allow_depth_tile, bool volume_texture, const char* owner); -uint64_t TextureUploadSliceSourceOffset(const TextureUploadLayout& layout, uint32_t level, - uint32_t slice); -uint64_t TextureCalcUploadSize(const TextureUploadLayout& layout, - const std::vector& regions, uint64_t levels, - uint32_t depth); -std::vector TextureBuildUploadRegions(const TextureUploadLayout& layout, - vk::Format image_format, uint32_t width, - uint32_t height, uint32_t depth, - uint64_t levels, bool array_texture, - bool volume_texture, - TextureUploadDestination destination); -std::vector -TextureBuildDownloadRegions(const std::vector& upload_regions); -bool TextureBuildGpuTileInfos(uint64_t size, const std::vector& regions, +std::vector +TextureBuildImageCopies(const TextureUploadLayout& layout, uint32_t width, uint32_t height, + uint32_t depth, uint64_t levels, bool array_texture, + bool volume_texture); +bool TextureBuildGpuTileInfos(uint64_t size, + const std::vector& regions, const TextureUploadLayout& layout, uint32_t fmt, uint32_t depth, uint64_t levels, std::vector& infos); -void TextureUploadGuestImage(VulkanImage& vk_obj, const void* src_data, - uint64_t size, const std::vector& regions, - const TextureUploadLayout& layout, uint32_t fmt, uint64_t width, - uint64_t height, uint32_t depth, uint64_t levels, const char* owner, - vk::ImageLayout dst_layout); } // namespace Libs::Graphics diff --git a/src/graphics/host_gpu/pageManager.cpp b/src/graphics/host_gpu/pageManager.cpp index e2c58aa7..510b92c4 100644 --- a/src/graphics/host_gpu/pageManager.cpp +++ b/src/graphics/host_gpu/pageManager.cpp @@ -297,8 +297,7 @@ bool PageManager::IsTracked(uint64_t vaddr) const noexcept { } bool PageManager::IsMapped(uint64_t vaddr, uint64_t size) const noexcept { - if (g_in_fault_resolution || vaddr == 0 || size == 0 || vaddr >= ADDRESS_SIZE || - size > ADDRESS_SIZE - vaddr) { + if (vaddr == 0 || size == 0 || vaddr >= ADDRESS_SIZE || size > ADDRESS_SIZE - vaddr) { return false; } const auto end = PageStart(vaddr + size - 1) + PAGE_SIZE; @@ -363,9 +362,6 @@ bool PageManager::HasGpuAccess(uint64_t vaddr, uint64_t size, GpuAccess access) void PageManager::UpdatePageWatchers(bool track, uint64_t vaddr, uint64_t size, PageWatchMode mode) { - if (g_in_fault_resolution) { - FailFast("page watchers changed during fault resolution"); - } if (mode != PageWatchMode::Write && mode != PageWatchMode::ReadWrite) { Fatal("invalid watcher mode"); } @@ -500,6 +496,37 @@ PageManager::BackingWrite::~BackingWrite() { m_manager.EndBackingWrite(m_vaddr, m_size); } +std::vector> +PageManager::ReserveBackingWrites(std::span ranges) { + if (ranges.empty()) { + Fatal("cannot reserve empty backing-write ranges"); + } + std::vector> writes; + writes.reserve(ranges.size()); + uint64_t begin = 0; + uint64_t end = 0; + for (const auto& range: ranges) { + if (range.address == 0 || range.size == 0 || range.size > UINT64_MAX - range.address || + range.address + range.size > UINT64_MAX - (PAGE_SIZE - 1)) { + Fatal("invalid backing-write range"); + } + const auto page_begin = PageStart(range.address); + const auto page_end = PageStart(range.address + range.size + PAGE_SIZE - 1); + if (begin != 0 && page_begin > end) { + writes.push_back(std::make_unique(*this, begin, end - begin)); + begin = 0; + } + if (begin == 0) { + begin = page_begin; + end = page_end; + } else { + end = std::max(end, page_end); + } + } + writes.push_back(std::make_unique(*this, begin, end - begin)); + return writes; +} + void PageManager::BeginBackingWrite(uint64_t vaddr, uint64_t size) noexcept { if (g_in_fault_resolution) { FailFast("backing write began during fault resolution"); diff --git a/src/graphics/host_gpu/pageManager.h b/src/graphics/host_gpu/pageManager.h index cf5214f4..50af34b8 100644 --- a/src/graphics/host_gpu/pageManager.h +++ b/src/graphics/host_gpu/pageManager.h @@ -2,8 +2,11 @@ #define EMULATOR_SRC_GRAPHICS_HOST_GPU_PAGEMANAGER_H_ #include "common/common.h" +#include "graphics/host_gpu/rangeSet.h" #include +#include +#include namespace Libs::Graphics { @@ -48,6 +51,8 @@ public: [[nodiscard]] bool HandleFault(PageFaultAccess access, uint64_t fault_vaddr) noexcept; [[nodiscard]] bool HandleWriteRange(uint64_t vaddr, uint64_t size) noexcept; + [[nodiscard]] std::vector> + ReserveBackingWrites(std::span ranges); private: void BeginBackingWrite(uint64_t vaddr, uint64_t size) noexcept; diff --git a/src/graphics/host_gpu/renderer/blitHelper.cpp b/src/graphics/host_gpu/renderer/blitHelper.cpp new file mode 100644 index 00000000..354083a4 --- /dev/null +++ b/src/graphics/host_gpu/renderer/blitHelper.cpp @@ -0,0 +1,231 @@ +#include "graphics/host_gpu/renderer/blitHelper.h" + +#include "common/assert.h" +#include "gpu_blit_shaders/gpu_blit_color_to_ms_depth_spv.h" +#include "gpu_blit_shaders/gpu_blit_fs_triangle_spv.h" +#include "graphics/host_gpu/graphicContext.h" +#include "graphics/host_gpu/renderer/commandScheduler.h" +#include "graphics/host_gpu/renderer/image.h" +#include "graphics/host_gpu/renderer/renderTarget.h" + +#include +#include +#include + +namespace Libs::Graphics { + +BlitHelper::BlitHelper(GraphicContext& graphics, CommandScheduler& scheduler) + : m_graphics(graphics), m_scheduler(scheduler) { + vk::DescriptorSetLayoutBinding texture_binding {}; + texture_binding.binding = 0; + texture_binding.descriptorType = vk::DescriptorType::eSampledImage; + texture_binding.descriptorCount = 1; + texture_binding.stageFlags = vk::ShaderStageFlagBits::eFragment; + + vk::DescriptorSetLayoutCreateInfo descriptor_info {}; + descriptor_info.sType = vk::StructureType::eDescriptorSetLayoutCreateInfo; + descriptor_info.flags = vk::DescriptorSetLayoutCreateFlagBits::ePushDescriptorKHR; + descriptor_info.bindingCount = 1; + descriptor_info.pBindings = &texture_binding; + RequireVulkanSuccess(m_graphics.device.createDescriptorSetLayout(&descriptor_info, nullptr, + &m_descriptor_layout), + "create BlitHelper descriptor layout"); + + vk::PipelineLayoutCreateInfo layout_info {}; + layout_info.sType = vk::StructureType::ePipelineLayoutCreateInfo; + layout_info.setLayoutCount = 1; + layout_info.pSetLayouts = &m_descriptor_layout; + RequireVulkanSuccess( + m_graphics.device.createPipelineLayout(&layout_info, nullptr, &m_pipeline_layout), + "create BlitHelper pipeline layout"); + + m_vertex_shader = CreateShader(GPU_BLIT_FS_TRIANGLE_SPV, std::size(GPU_BLIT_FS_TRIANGLE_SPV)); + m_fragment_shader = + CreateShader(GPU_BLIT_COLOR_TO_MS_DEPTH_SPV, std::size(GPU_BLIT_COLOR_TO_MS_DEPTH_SPV)); +} + +BlitHelper::~BlitHelper() { + for (const auto& pipeline: m_pipelines) { + m_graphics.device.destroyPipeline(pipeline.handle, nullptr); + } + if (m_fragment_shader != nullptr) { + m_graphics.device.destroyShaderModule(m_fragment_shader, nullptr); + } + if (m_vertex_shader != nullptr) { + m_graphics.device.destroyShaderModule(m_vertex_shader, nullptr); + } + if (m_pipeline_layout != nullptr) { + m_graphics.device.destroyPipelineLayout(m_pipeline_layout, nullptr); + } + if (m_descriptor_layout != nullptr) { + m_graphics.device.destroyDescriptorSetLayout(m_descriptor_layout, nullptr); + } +} + +vk::ShaderModule BlitHelper::CreateShader(const uint32_t* code, size_t words) const { + EXIT_IF(code == nullptr || words == 0); + vk::ShaderModuleCreateInfo create {}; + create.sType = vk::StructureType::eShaderModuleCreateInfo; + create.codeSize = words * sizeof(uint32_t); + create.pCode = code; + vk::ShaderModule module = nullptr; + RequireVulkanSuccess(m_graphics.device.createShaderModule(&create, nullptr, &module), + "create BlitHelper shader module"); + return module; +} + +vk::Pipeline BlitHelper::GetPipeline(PipelineKey key) { + const auto cached = std::ranges::find(m_pipelines, key, &Pipeline::key); + if (cached != m_pipelines.end()) { + return cached->handle; + } + + const auto samples = vulkan_sample_count(key.samples); + EXIT_IF(samples == vk::SampleCountFlagBits {} || key.format == vk::Format::eUndefined); + + std::array stages {}; + stages[0].sType = vk::StructureType::ePipelineShaderStageCreateInfo; + stages[0].stage = vk::ShaderStageFlagBits::eVertex; + stages[0].module = m_vertex_shader; + stages[0].pName = "main"; + stages[1].sType = vk::StructureType::ePipelineShaderStageCreateInfo; + stages[1].stage = vk::ShaderStageFlagBits::eFragment; + stages[1].module = m_fragment_shader; + stages[1].pName = "main"; + + vk::PipelineVertexInputStateCreateInfo vertex_input {}; + vertex_input.sType = vk::StructureType::ePipelineVertexInputStateCreateInfo; + vk::PipelineInputAssemblyStateCreateInfo input_assembly {}; + input_assembly.sType = vk::StructureType::ePipelineInputAssemblyStateCreateInfo; + input_assembly.topology = vk::PrimitiveTopology::eTriangleList; + vk::PipelineViewportStateCreateInfo viewport {}; + viewport.sType = vk::StructureType::ePipelineViewportStateCreateInfo; + viewport.viewportCount = 1; + viewport.scissorCount = 1; + vk::PipelineRasterizationStateCreateInfo rasterization {}; + rasterization.sType = vk::StructureType::ePipelineRasterizationStateCreateInfo; + rasterization.polygonMode = vk::PolygonMode::eFill; + rasterization.cullMode = vk::CullModeFlagBits::eNone; + rasterization.lineWidth = 1.0f; + vk::PipelineMultisampleStateCreateInfo multisample {}; + multisample.sType = vk::StructureType::ePipelineMultisampleStateCreateInfo; + multisample.rasterizationSamples = samples; + vk::PipelineDepthStencilStateCreateInfo depth {}; + depth.sType = vk::StructureType::ePipelineDepthStencilStateCreateInfo; + depth.depthTestEnable = VK_TRUE; + depth.depthWriteEnable = VK_TRUE; + depth.depthCompareOp = vk::CompareOp::eAlways; + vk::PipelineColorBlendStateCreateInfo color_blend {}; + color_blend.sType = vk::StructureType::ePipelineColorBlendStateCreateInfo; + const std::array dynamic_states {vk::DynamicState::eViewport, vk::DynamicState::eScissor}; + vk::PipelineDynamicStateCreateInfo dynamic {}; + dynamic.sType = vk::StructureType::ePipelineDynamicStateCreateInfo; + dynamic.dynamicStateCount = static_cast(dynamic_states.size()); + dynamic.pDynamicStates = dynamic_states.data(); + + vk::PipelineRenderingCreateInfo rendering {}; + rendering.sType = vk::StructureType::ePipelineRenderingCreateInfo; + rendering.depthAttachmentFormat = key.format; + + vk::GraphicsPipelineCreateInfo create {}; + create.sType = vk::StructureType::eGraphicsPipelineCreateInfo; + create.pNext = &rendering; + create.stageCount = static_cast(stages.size()); + create.pStages = stages.data(); + create.pVertexInputState = &vertex_input; + create.pInputAssemblyState = &input_assembly; + create.pViewportState = &viewport; + create.pRasterizationState = &rasterization; + create.pMultisampleState = &multisample; + create.pDepthStencilState = &depth; + create.pColorBlendState = &color_blend; + create.pDynamicState = &dynamic; + create.layout = m_pipeline_layout; + + vk::Pipeline pipeline = nullptr; + RequireVulkanSuccess( + m_graphics.device.createGraphicsPipelines(nullptr, 1, &create, nullptr, &pipeline), + "create color-to-MS-depth pipeline"); + m_pipelines.push_back({key, pipeline}); + return pipeline; +} + +void BlitHelper::ReinterpretColorAsMsDepth(Image& source, Image& destination) { + const auto& source_info = source.info; + const auto& destination_info = destination.info; + EXIT_IF(DepthAspectTransferFormat(source_info.pixel_format) != vk::Format::eUndefined || + DepthAspectTransferFormat(destination_info.pixel_format) == vk::Format::eUndefined || + source_info.samples != 1 || destination_info.samples <= 1 || + destination_info.samples > 4 || source.backing.image_type != vk::ImageType::e2D || + destination.backing.image_type != vk::ImageType::e2D || + source_info.extent.width != destination_info.extent.width || + source_info.extent.height != destination_info.extent.height || + source_info.extent.depth != 1 || destination_info.extent.depth != 1 || + source.backing.image == nullptr || destination.backing.image == nullptr); + m_scheduler.EndRendering(); + + ImageViewInfo source_view_info {}; + source_view_info.format = source_info.pixel_format; + source_view_info.type = vk::ImageViewType::e2D; + source_view_info.aspect = vk::ImageAspectFlagBits::eColor; + source_view_info.usage = vk::ImageUsageFlagBits::eSampled; + const auto source_view = source.FindView(source_view_info); + + ImageViewInfo destination_view_info {}; + destination_view_info.format = destination_info.pixel_format; + destination_view_info.type = vk::ImageViewType::e2D; + destination_view_info.aspect = vk::ImageAspectFlagBits::eDepth; + destination_view_info.usage = vk::ImageUsageFlagBits::eDepthStencilAttachment; + const auto destination_view = destination.FindView(destination_view_info); + + auto& command_buffer = m_scheduler.Current(); + auto command = command_buffer.Handle(); + source.Transit(vk::ImageLayout::eShaderReadOnlyOptimal, vk::AccessFlagBits2::eShaderRead, {}, + command); + destination.Transit(ColorToMsDepthLayout, + vk::AccessFlagBits2::eDepthStencilAttachmentWrite, {}, command); + + vk::RenderingAttachmentInfo depth_attachment {}; + depth_attachment.sType = vk::StructureType::eRenderingAttachmentInfo; + depth_attachment.imageView = destination_view; + depth_attachment.imageLayout = ColorToMsDepthLayout; + depth_attachment.loadOp = vk::AttachmentLoadOp::eClear; + depth_attachment.storeOp = vk::AttachmentStoreOp::eStore; + depth_attachment.clearValue.depthStencil = {0.0f, 0}; + + vk::RenderingInfo rendering {}; + rendering.sType = vk::StructureType::eRenderingInfo; + rendering.renderArea.extent = {destination_info.extent.width, destination_info.extent.height}; + rendering.layerCount = 1; + rendering.pDepthAttachment = &depth_attachment; + command.beginRendering(&rendering); + + vk::DescriptorImageInfo descriptor_image {}; + descriptor_image.imageView = source_view; + descriptor_image.imageLayout = vk::ImageLayout::eShaderReadOnlyOptimal; + vk::WriteDescriptorSet descriptor_write {}; + descriptor_write.sType = vk::StructureType::eWriteDescriptorSet; + descriptor_write.dstBinding = 0; + descriptor_write.descriptorCount = 1; + descriptor_write.descriptorType = vk::DescriptorType::eSampledImage; + descriptor_write.pImageInfo = &descriptor_image; + command.pushDescriptorSetKHR(vk::PipelineBindPoint::eGraphics, m_pipeline_layout, 0, 1, + &descriptor_write); + command.bindPipeline(vk::PipelineBindPoint::eGraphics, + GetPipeline({destination_info.samples, destination_info.pixel_format})); + + const vk::Viewport viewport {0.0f, + 0.0f, + static_cast(destination_info.extent.width), + static_cast(destination_info.extent.height), + 0.0f, + 1.0f}; + const vk::Rect2D scissor {{0, 0}, + {destination_info.extent.width, destination_info.extent.height}}; + command.setViewport(0, 1, &viewport); + command.setScissor(0, 1, &scissor); + command.draw(3, 1, 0, 0); + command.endRendering(); +} + +} // namespace Libs::Graphics diff --git a/src/graphics/host_gpu/renderer/blitHelper.h b/src/graphics/host_gpu/renderer/blitHelper.h new file mode 100644 index 00000000..1c23ef78 --- /dev/null +++ b/src/graphics/host_gpu/renderer/blitHelper.h @@ -0,0 +1,54 @@ +#ifndef EMULATOR_SRC_GRAPHICS_HOST_GPU_RENDERER_BLITHELPER_H_ +#define EMULATOR_SRC_GRAPHICS_HOST_GPU_RENDERER_BLITHELPER_H_ + +#include "common/common.h" +#include "graphics/host_gpu/vulkanCommon.h" + +#include +#include + +namespace Libs::Graphics { + +class CommandScheduler; +class Image; +struct GraphicContext; + +class BlitHelper final { +public: + inline static constexpr auto ColorToMsDepthLayout = + vk::ImageLayout::eDepthStencilAttachmentOptimal; + + BlitHelper(GraphicContext& graphics, CommandScheduler& scheduler); + ~BlitHelper(); + KYTY_CLASS_NO_COPY(BlitHelper); + + void ReinterpretColorAsMsDepth(Image& source, Image& destination); + +private: + struct PipelineKey { + uint32_t samples = 1; + vk::Format format = vk::Format::eUndefined; + + auto operator<=>(const PipelineKey&) const = default; + }; + + struct Pipeline { + PipelineKey key; + vk::Pipeline handle = nullptr; + }; + + [[nodiscard]] vk::ShaderModule CreateShader(const uint32_t* code, size_t words) const; + [[nodiscard]] vk::Pipeline GetPipeline(PipelineKey key); + + GraphicContext& m_graphics; + CommandScheduler& m_scheduler; + vk::DescriptorSetLayout m_descriptor_layout = nullptr; + vk::PipelineLayout m_pipeline_layout = nullptr; + vk::ShaderModule m_vertex_shader = nullptr; + vk::ShaderModule m_fragment_shader = nullptr; + std::vector m_pipelines; +}; + +} // namespace Libs::Graphics + +#endif // EMULATOR_SRC_GRAPHICS_HOST_GPU_RENDERER_BLITHELPER_H_ diff --git a/src/graphics/host_gpu/renderer/bufferCache.cpp b/src/graphics/host_gpu/renderer/bufferCache.cpp index 5465f2b1..2d34b3d3 100644 --- a/src/graphics/host_gpu/renderer/bufferCache.cpp +++ b/src/graphics/host_gpu/renderer/bufferCache.cpp @@ -3,122 +3,38 @@ #include "common/assert.h" #include "common/logging/log.h" #include "common/profiler.h" -#include "graphics/guest_gpu/graphicsRun.h" #include "graphics/host_gpu/graphicContext.h" -#include "graphics/host_gpu/objects/label.h" +#include "graphics/host_gpu/renderer/commandScheduler.h" #include "graphics/host_gpu/renderer/render.h" -#include "graphics/host_gpu/renderer/renderContext.h" #include "graphics/host_gpu/renderer/resourceMutex.h" #include "graphics/host_gpu/renderer/textureCache.h" -#include "graphics/host_gpu/transfer.h" -#include "graphics/host_gpu/vma.h" #include "kernel/memory.h" #include #include -#include -#include #include -#include -#include -#include +#include #include namespace Libs::Graphics { namespace { + thread_local const void* g_cache_lock_owner = nullptr; -constexpr uint32_t READBACK_MAX_RANGES = 256; -constexpr uint64_t READBACK_COPY_ALIGNMENT = 64; -constexpr uint64_t READBACK_CAPACITY = - TRACKER_PAGE_SIZE + READBACK_MAX_RANGES * (READBACK_COPY_ALIGNMENT - 1); - -constexpr uint64_t AlignReadbackCopySize(uint64_t size) noexcept { - return (size + READBACK_COPY_ALIGNMENT - 1) & ~(READBACK_COPY_ALIGNMENT - 1); -} - -struct SharedVulkanBufferOwner { - explicit SharedVulkanBufferOwner(GraphicContext& graphics): graphics(graphics) {} - - ~SharedVulkanBufferOwner() { - if (buffer.buffer != nullptr) { - graphics.DeleteBuffer(buffer); - } - } - - GraphicContext& graphics; - VulkanBuffer buffer; -}; - -std::shared_ptr MakeSharedVulkanBuffer() { - auto owner = std::make_shared(GetRenderContext().GetGraphics()); - return {owner, &owner->buffer}; -} - -std::vector> -ReserveBackingWrites(PageManager& page_manager, const std::vector& ranges) { - if (ranges.empty()) { - EXIT("BufferCache: cannot reserve empty backing-write ranges\n"); - } - std::vector> writes; - writes.reserve(ranges.size()); - uint64_t begin = 0; - uint64_t end = 0; - for (const auto& range: ranges) { - if (range.address == 0 || range.size == 0 || range.size > UINT64_MAX - range.address || - range.address + range.size > UINT64_MAX - (TRACKER_PAGE_SIZE - 1)) { - EXIT("BufferCache: invalid backing-write range, addr=0x%016" PRIx64 - " size=0x%016" PRIx64 "\n", - range.address, range.size); - } - const auto page_begin = range.address & ~(TRACKER_PAGE_SIZE - 1); - const auto page_end = - (range.address + range.size + TRACKER_PAGE_SIZE - 1) & ~(TRACKER_PAGE_SIZE - 1); - if (begin != 0 && page_begin > end) { - writes.push_back( - std::make_unique(page_manager, begin, end - begin)); - begin = 0; - } - if (begin == 0) { - begin = page_begin; - end = page_end; - } else { - end = std::max(end, page_end); - } - } - writes.push_back(std::make_unique(page_manager, begin, end - begin)); - return writes; -} - -void ValidateDirtyPages(const RangeSet& dirty, uint64_t vaddr, uint64_t size, - const char* operation) noexcept { - if (vaddr == 0 || size == 0 || size > UINT64_MAX - vaddr || - (vaddr & (TRACKER_PAGE_SIZE - 1)) != 0 || (size & (TRACKER_PAGE_SIZE - 1)) != 0) { - EXIT("BufferCache: invalid dirty-page validation range\n"); - } - for (auto page = vaddr; page < vaddr + size; page += TRACKER_PAGE_SIZE) { - bool found = false; - dirty.ForEachIntersection(page, TRACKER_PAGE_SIZE, - [&found](RangeSet::Range) { found = true; }); - if (!found) { - EXIT("BufferCache: GPU-dirty tracker page has no dirty bytes, operation=%s " - "addr=0x%016" PRIx64 "\n", - operation, page); - } - } -} +constexpr uint64_t MiB = 1024 * 1024; +constexpr uint64_t GdsBufferSize = 64 * 1024; class FaultSafeCacheLock final { public: FaultSafeCacheLock(const void* owner, Common::Mutex& mutex): m_mutex(mutex) { if (g_cache_lock_owner != nullptr) { - EXIT("BufferCache: recursive cache lock acquisition, current=%p\n", - static_cast(g_cache_lock_owner)); + EXIT("BufferCache: recursive cache lock acquisition\n"); } g_cache_lock_owner = owner; m_mutex.Lock(); } + ~FaultSafeCacheLock() { m_mutex.Unlock(); g_cache_lock_owner = nullptr; @@ -128,28 +44,43 @@ private: Common::Mutex& m_mutex; }; -uint64_t AlignDown(uint64_t value) { - return value & ~(BufferCache::CACHING_PAGE_SIZE - 1); +} // namespace + +uint64_t BufferCache::AlignDown(uint64_t value) noexcept { + return value & ~(CACHING_PAGE_SIZE - 1); } -uint64_t AlignUp(uint64_t value) { - if (value > UINT64_MAX - (BufferCache::CACHING_PAGE_SIZE - 1)) { - EXIT("BufferCache: address alignment overflow, value=0x%016" PRIx64 "\n", value); +uint64_t BufferCache::AlignUp(uint64_t value) { + if (value > UINT64_MAX - (CACHING_PAGE_SIZE - 1)) { + EXIT("BufferCache: address alignment overflow\n"); } - return (value + BufferCache::CACHING_PAGE_SIZE - 1) & ~(BufferCache::CACHING_PAGE_SIZE - 1); + return (value + CACHING_PAGE_SIZE - 1) & ~(CACHING_PAGE_SIZE - 1); } -bool PageOverlaps(uint64_t left, uint64_t left_size, uint64_t right, uint64_t right_size) { +bool BufferCache::PageOverlaps(uint64_t left, uint64_t left_size, uint64_t right, + uint64_t right_size) noexcept { const auto left_begin = left & ~(TRACKER_PAGE_SIZE - 1); const auto left_end = (left + left_size + TRACKER_PAGE_SIZE - 1) & ~(TRACKER_PAGE_SIZE - 1); const auto right_begin = right & ~(TRACKER_PAGE_SIZE - 1); const auto right_end = (right + right_size + TRACKER_PAGE_SIZE - 1) & ~(TRACKER_PAGE_SIZE - 1); return left_begin < right_end && right_begin < left_end; } -} // namespace -bool MergeOverlappingBufferCacheRange(BufferCacheRange& merged, - BufferCacheRange candidate) noexcept { +void BufferCache::Upload(CommandBuffer& command, Buffer& destination, uint64_t destination_offset, + const void* source, uint64_t size) { + auto* bytes = static_cast(source); + while (size != 0) { + const auto chunk = std::min(size, m_staging_buffer.Size()); + const auto stage_offset = m_staging_buffer.Copy(bytes, chunk, 4); + destination.CopyFrom(command, m_staging_buffer, stage_offset, destination_offset, chunk, + vk::AccessFlagBits::eHostWrite); + bytes += chunk; + destination_offset += chunk; + size -= chunk; + } +} + +bool BufferCache::ResolveOverlap(CacheRange& merged, CacheRange candidate) noexcept { if (merged.address == 0 || merged.size == 0 || candidate.address == 0 || candidate.size == 0 || merged.size > UINT64_MAX - merged.address || candidate.size > UINT64_MAX - candidate.address) { @@ -167,1006 +98,959 @@ bool MergeOverlappingBufferCacheRange(BufferCacheRange& merged, } struct BufferCache::CachedBuffer { - uint64_t vaddr = 0; - uint64_t size = 0; - std::shared_ptr buffer; + uint64_t vaddr = 0; + uint64_t size = 0; + std::shared_ptr buffer; + uint64_t tick_accessed_last = 0; }; -struct BufferCache::ReadbackWorker { - static constexpr uint32_t MAX_RANGES = READBACK_MAX_RANGES; - enum class State : uint32_t { - Uninitialized, - InitRequested, - Idle, - Claimed, - Requested, - Ready, - Installed, - Completed, - Stopping, - Stopped - }; - static_assert(std::atomic::is_always_lock_free); - - struct Range { - uint64_t address = 0; - uint32_t size = 0; - uint32_t offset = 0; - }; - - explicit ReadbackWorker(BufferCache& owner): cache(owner), thread([this] { Run(); }) {} - - ~ReadbackWorker() { - auto expected = State::Idle; - if (!state.compare_exchange_strong(expected, State::Stopping, std::memory_order_acq_rel)) { - expected = State::Uninitialized; - if (!state.compare_exchange_strong(expected, State::Stopping, - std::memory_order_acq_rel)) { - EXIT("BufferCache: cannot stop readback worker from state %u\n", - static_cast(expected)); - } - } - state.notify_all(); - thread.join(); - if (state.load(std::memory_order_acquire) != State::Stopped) { - EXIT("BufferCache: readback worker did not reach stopped state\n"); - } - } - - void Prepare() { - auto current = state.load(std::memory_order_acquire); - while (current == State::InitRequested || - (current == State::Claimed && - (command == nullptr || mapped == nullptr || readback.buffer == nullptr))) { - state.wait(current, std::memory_order_acquire); - current = state.load(std::memory_order_acquire); - } - if (current != State::Uninitialized) { - if (current == State::Stopping || current == State::Stopped) { - EXIT("BufferCache: cannot prepare a stopping readback worker, state=%u\n", - static_cast(current)); - } - if (command == nullptr || mapped == nullptr || readback.buffer == nullptr) { - EXIT("BufferCache: initialized readback worker has invalid resources, state=%u " - "command=%p mapped=%p buffer=%p\n", - static_cast(current), static_cast(command.get()), - static_cast(mapped), static_cast(readback.buffer)); - } - return; - } - auto expected = State::Uninitialized; - if (!state.compare_exchange_strong(expected, State::Claimed, std::memory_order_acq_rel)) { - EXIT("BufferCache: readback prepare requires uninitialized state, state=%u\n", - static_cast(expected)); - } - state.store(State::InitRequested, std::memory_order_release); - state.notify_all(); - while ((current = state.load(std::memory_order_acquire)) != State::Idle) { - if (current != State::InitRequested) { - EXIT("BufferCache: invalid readback initialization state %u\n", - static_cast(current)); - } - state.wait(current, std::memory_order_acquire); - } - if (command == nullptr || mapped == nullptr || readback.buffer == nullptr) { - EXIT("BufferCache: readback initialization produced invalid resources, command=%p " - "mapped=%p buffer=%p\n", - static_cast(command.get()), static_cast(mapped), - static_cast(readback.buffer)); - } - } - - void Request(PageFaultAccess fault_access, uint64_t fault_vaddr, uint64_t fault_size) noexcept { - const bool command_thread = GraphicsRunIsCommandProcessorThread(); - const bool submissions_prepaused_now = GraphicsRunSubmissionLockHeld() || command_thread; - const bool unsafe_gpu_lock = GraphicsRunGpuLockHeld() && !submissions_prepaused_now; - if (unsafe_gpu_lock || LabelInCallback() || g_cache_lock_owner != nullptr || - command == nullptr || mapped == nullptr || readback.buffer == nullptr) { - EXIT("BufferCache: unsafe readback request context, command_thread=%d " - "submission_lock=%d " - "gpu_lock=%d label_callback=%d cache_lock=%p command=%p mapped=%p " - "buffer=%p\n", - command_thread, GraphicsRunSubmissionLockHeld(), GraphicsRunGpuLockHeld(), - LabelInCallback(), static_cast(g_cache_lock_owner), - static_cast(command.get()), static_cast(mapped), - static_cast(readback.buffer)); - } - State expected = State::Idle; - while (!state.compare_exchange_weak(expected, State::Claimed, std::memory_order_acq_rel)) { - if (expected == State::Stopping || expected == State::Stopped) { - EXIT("BufferCache: readback requested while worker is stopping, state=%u\n", - static_cast(expected)); - } - state.wait(expected, std::memory_order_acquire); - expected = State::Idle; - } - access = fault_access; - vaddr = fault_vaddr; - size = fault_size; - range_count = 0; - submissions_prepaused = submissions_prepaused_now; - state.store(State::Requested, std::memory_order_release); - state.notify_all(); - while (true) { - const auto current = state.load(std::memory_order_acquire); - if (current == State::Ready) { - break; - } - if (current != State::Requested) { - EXIT("BufferCache: invalid state while waiting for readback, state=%u\n", - static_cast(current)); - } - state.wait(current, std::memory_order_acquire); - } - } - - [[nodiscard]] bool Complete(PageFaultAccess fault_access, uint64_t fault_vaddr, - uint64_t fault_size) noexcept { - const auto current = state.load(std::memory_order_acquire); - if (current == State::Uninitialized || current == State::InitRequested || - current == State::Idle || current == State::Stopping || current == State::Stopped) { - return false; - } - if (current != State::Ready) { - EXIT("BufferCache: active readback has invalid completion state %u\n", - static_cast(current)); - } - if (access != fault_access || vaddr != fault_vaddr || size != fault_size) { - EXIT("BufferCache: mismatched active readback completion\n"); - } - if (range_count == 0 || range_count > MAX_RANGES) { - EXIT("BufferCache: invalid completed readback range count %u\n", range_count); - } - for (uint32_t i = 0; i < range_count; i++) { - const auto& range = ranges[i]; - Libs::LibKernel::Memory::WriteBacking(range.address, data.data() + range.offset, - range.size); - } - if (!cache.m_memory_tracker.CompleteCpuFault(vaddr, size, access, true)) { - EXIT("BufferCache: failed to complete downloaded CPU fault, addr=0x%016" PRIx64 - " size=0x%016" PRIx64 " access=%u\n", - vaddr, size, static_cast(access)); - } - state.store(State::Installed, std::memory_order_release); - state.notify_all(); - return true; - } - - void Release(PageFaultAccess fault_access, uint64_t fault_vaddr, uint64_t fault_size) noexcept { - const auto current = state.load(std::memory_order_acquire); - if (current == State::Uninitialized || current == State::InitRequested || - current == State::Idle || current == State::Stopping || current == State::Stopped) { - return; - } - if (current != State::Installed) { - EXIT("BufferCache: active readback has invalid release state %u\n", - static_cast(current)); - } - if (access != fault_access || vaddr != fault_vaddr || size != fault_size) { - EXIT("BufferCache: mismatched active readback release\n"); - } - state.store(State::Completed, std::memory_order_release); - state.notify_all(); - while (true) { - const auto current = state.load(std::memory_order_acquire); - if (current == State::Idle) { - break; - } - if (current != State::Completed) { - EXIT("BufferCache: invalid state while releasing readback, state=%u\n", - static_cast(current)); - } - state.wait(current, std::memory_order_acquire); - } - } - - void Run() noexcept { - while (true) { - auto current = state.load(std::memory_order_acquire); - while (current == State::Uninitialized || current == State::Idle) { - state.wait(current, std::memory_order_acquire); - current = state.load(std::memory_order_acquire); - } - if (current == State::Stopping) { - command.reset(); - if (mapped != nullptr) { - cache.m_graphics.UnmapMemory(readback.memory); - mapped = nullptr; - } - if (readback.buffer != nullptr) { - cache.m_graphics.DeleteBuffer(readback); - } - state.store(State::Stopped, std::memory_order_release); - state.notify_all(); - return; - } - if (current == State::InitRequested) { - if (command != nullptr || mapped != nullptr || readback.buffer != nullptr) { - EXIT("BufferCache: invalid resources before readback initialization, " - "command=%p mapped=%p buffer=%p\n", - static_cast(command.get()), static_cast(mapped), - static_cast(readback.buffer)); - } - readback.usage = vk::BufferUsageFlagBits::eTransferDst; - readback.memory.property = vk::MemoryPropertyFlagBits::eHostVisible | - vk::MemoryPropertyFlagBits::eHostCoherent | - vk::MemoryPropertyFlagBits::eHostCached; - cache.m_graphics.CreateBuffer(READBACK_CAPACITY, readback); - cache.m_graphics.MapMemory(readback.memory, mapped); - command = std::make_unique(); - state.store(State::Idle, std::memory_order_release); - state.notify_all(); - continue; - } - if (current != State::Requested) { - EXIT("BufferCache: worker received unsupported state %u\n", - static_cast(current)); - } - - std::optional submissions; - if (!submissions_prepaused) { - submissions.emplace(); - } - const auto page = vaddr & ~(TRACKER_PAGE_SIZE - 1); - FaultSafeCacheLock lock(&cache, cache.m_mutex); - auto it = cache.m_buffers.upper_bound(vaddr); - if (it == cache.m_buffers.begin()) { - EXIT("BufferCache: readback address has no cached buffer, addr=0x%016" PRIx64 "\n", - vaddr); - } - --it; - auto& cached = *it->second; - if (vaddr < cached.vaddr || vaddr >= cached.vaddr + cached.size || - page < cached.vaddr || TRACKER_PAGE_SIZE > cached.size - (page - cached.vaddr)) { - EXIT("BufferCache: readback page is outside cached buffer, fault=0x%016" PRIx64 - " page=0x%016" PRIx64 " buffer=0x%016" PRIx64 " size=0x%016" PRIx64 "\n", - vaddr, page, cached.vaddr, cached.size); - } - uint32_t data_offset = 0; - std::array copies {}; - std::array barriers {}; - cache.m_gpu_modified_ranges.ForEachIntersection( - page, TRACKER_PAGE_SIZE, [&](RangeSet::Range range) { - if (range_count == MAX_RANGES || (range.address - cached.vaddr) % 4 != 0 || - range.size % 4 != 0) { - EXIT("BufferCache: invalid GPU-modified readback range, addr=0x%016" PRIx64 - " size=0x%016" PRIx64 " offset=%u count=%u\n", - range.address, range.size, data_offset, range_count); - } - auto& out = ranges[range_count]; - out.address = range.address; - out.size = static_cast(range.size); - out.offset = data_offset; - copies[range_count] = {.srcOffset = range.address - cached.vaddr, - .dstOffset = data_offset, - .size = range.size}; - auto& barrier = barriers[range_count]; - barrier.sType = vk::StructureType::eBufferMemoryBarrier; - barrier.srcAccessMask = vk::AccessFlagBits::eMemoryWrite; - barrier.dstAccessMask = vk::AccessFlagBits::eTransferRead; - barrier.srcQueueFamilyIndex = VK_QUEUE_FAMILY_IGNORED; - barrier.dstQueueFamilyIndex = VK_QUEUE_FAMILY_IGNORED; - barrier.buffer = cached.buffer->buffer; - barrier.offset = copies[range_count].srcOffset; - barrier.size = range.size; - range_count++; - const auto slot_size = AlignReadbackCopySize(out.size); - if (slot_size > data.size() - data_offset) { - EXIT("BufferCache: aligned readback range exceeds persistent capacity, " - "offset=%u size=%" PRIu64 " capacity=%" PRIu64 "\n", - data_offset, slot_size, static_cast(data.size())); - } - data_offset += static_cast(slot_size); - }); - if (range_count == 0 || data_offset == 0) { - EXIT( - "BufferCache: GPU-modified page produced no readback ranges, page=0x%016" PRIx64 - " count=%u bytes=%u\n", - page, range_count, data_offset); - } - auto vk_buffer = command->Handle(); - command->Begin(); - vk_buffer.pipelineBarrier(vk::PipelineStageFlagBits::eAllCommands, - vk::PipelineStageFlagBits::eTransfer, vk::DependencyFlags {}, - 0, nullptr, range_count, barriers.data(), 0, nullptr); - vk_buffer.copyBuffer(cached.buffer->buffer, readback.buffer, range_count, - copies.data()); - command->End(); - command->Execute(); - command->WaitForFenceAndReset(); - std::memcpy(data.data(), mapped, data_offset); - state.store(State::Ready, std::memory_order_release); - state.notify_all(); - while ((current = state.load(std::memory_order_acquire)) != State::Completed) { - if (current != State::Ready && current != State::Installed) { - EXIT("BufferCache: invalid readback completion state %u\n", - static_cast(current)); - } - state.wait(current, std::memory_order_acquire); - } - cache.m_gpu_modified_ranges.Subtract(page, TRACKER_PAGE_SIZE); - submissions_prepaused = false; - state.store(State::Idle, std::memory_order_release); - state.notify_all(); - } - } - - BufferCache& cache; - std::atomic state {State::Uninitialized}; - VulkanBuffer readback {}; - void* mapped = nullptr; - std::unique_ptr command; - PageFaultAccess access = PageFaultAccess::Unknown; - uint64_t vaddr = 0; - uint64_t size = 0; - uint32_t range_count = 0; - bool submissions_prepaused = false; - std::array ranges {}; - std::array data {}; - std::thread thread; +struct BufferCache::DownloadCopy { + std::shared_ptr owner; + uint64_t source_offset = 0; + uint64_t address = 0; + uint64_t size = 0; }; -BufferCache::BufferCache(GraphicContext& graphics, PageManager& page_manager, +struct BufferCache::DownloadRange { + uint64_t address = 0; + uint64_t size = 0; + uint64_t offset = 0; +}; + +struct BufferCache::RetiredBuffer { + uint64_t address = 0; + uint64_t size = 0; + std::shared_ptr owner; +}; + +struct BufferCache::FaultReadback { + PageFaultAccess access = PageFaultAccess::Unknown; + uint64_t vaddr = 0; + uint64_t size = 0; + std::vector ranges; + bool installed = false; + + [[nodiscard]] bool Active() const noexcept { return !ranges.empty(); } + + void Reset() { + access = PageFaultAccess::Unknown; + vaddr = 0; + size = 0; + installed = false; + ranges.clear(); + } +}; + +struct BufferCache::PendingBackingPublication { + uint64_t address = 0; + uint64_t size = 0; + uint64_t tick = 0; +}; + +std::pair BufferCache::DownloadEnvelope(const DownloadCopy& copy) { + if (copy.owner == nullptr || copy.size == 0 || copy.source_offset > copy.owner->Size() || + copy.size > copy.owner->Size() - copy.source_offset) { + EXIT("BufferCache: invalid download copy\n"); + } + const auto begin = copy.source_offset & ~uint64_t {3}; + if (copy.source_offset > UINT64_MAX - copy.size || + copy.source_offset + copy.size > UINT64_MAX - 3) { + EXIT("BufferCache: download copy alignment overflow\n"); + } + const auto end = (copy.source_offset + copy.size + 3) & ~uint64_t {3}; + if (end > copy.owner->Size()) { + EXIT("BufferCache: aligned download copy exceeds its owner\n"); + } + return {begin, end - begin}; +} + +std::vector +BufferCache::RecordDownloads(std::span copies) { + uint64_t reservation_size = 0; + for (const auto& copy: copies) { + const auto [source_begin, envelope_size] = DownloadEnvelope(copy); + (void)source_begin; + if (envelope_size > UINT64_MAX - (DOWNLOAD_ALIGNMENT - 1)) { + EXIT("BufferCache: download batch alignment overflow\n"); + } + const auto aligned_size = AlignDownload(envelope_size); + if (aligned_size > UINT64_MAX - reservation_size) { + EXIT("BufferCache: download batch overflow\n"); + } + reservation_size += aligned_size; + } + if (reservation_size == 0) { + return {}; + } + + auto& download = m_download_buffer; + const auto [mapped, base_offset] = + download.Map(reservation_size, DOWNLOAD_ALIGNMENT); + if (mapped == nullptr) { + EXIT("BufferCache: download batch could not reserve the shared stream\n"); + } + + std::vector downloads; + downloads.reserve(copies.size()); + uint64_t cursor = 0; + for (const auto& copy: copies) { + const auto [source_begin, envelope_size] = DownloadEnvelope(copy); + const auto prefix = copy.source_offset - source_begin; + download.CopyFrom(m_scheduler.Current(), *copy.owner, source_begin, base_offset + cursor, + envelope_size, vk::AccessFlagBits::eMemoryWrite, vk::AccessFlags {}, + vk::AccessFlagBits::eMemoryRead | vk::AccessFlagBits::eMemoryWrite, + vk::AccessFlagBits::eHostRead); + downloads.push_back({copy.address, copy.size, base_offset + cursor + prefix}); + cursor += AlignDownload(envelope_size); + } + download.Commit(); + return downloads; +} + +void BufferCache::PublishDownloads(std::span downloads) { + for (const auto& range: downloads) { + m_download_buffer.Invalidate(range.offset, range.size); + Libs::LibKernel::Memory::WriteBacking( + range.address, m_download_buffer.Mapped().data() + range.offset, range.size); + } +} + +void BufferCache::QueueGarbageDownload(std::span copies, + RetiredBuffer retire) { + if (copies.empty()) { + return; + } + auto downloads = RecordDownloads(copies); + const auto tick = m_scheduler.CurrentTick(); + BeginBackingPublication(retire.address, retire.size, tick); + m_scheduler.DeferOperation( + [this, downloads = std::move(downloads), retire = std::move(retire), tick]() mutable { + PublishDownloads(downloads); + { + FaultSafeCacheLock lock(this, m_mutex); + if (m_memory_tracker.IsRegionGpuModified(retire.address, retire.size)) { + m_memory_tracker.ForEachDownloadRange( + retire.address, retire.size, + [&](uint64_t address, uint64_t size) noexcept { + m_memory_tracker.ValidateGpuDirtyPages( + m_gpu_modified_ranges, address, size, + "asynchronous garbage retirement"); + }, + [](uint64_t, uint64_t) noexcept {}); + } + for (const auto& range: downloads) { + m_gpu_modified_ranges.Subtract(range.address, range.size); + } + if (m_memory_tracker.IsRegionGpuModified(retire.address, retire.size) || + !m_gpu_modified_ranges.Intersections(retire.address, retire.size).empty()) { + EXIT("BufferCache: asynchronous garbage collection retained GPU ownership\n"); + } + m_memory_tracker.UntrackMemory(retire.address, retire.size); + } + CompleteBackingPublication(retire.address, retire.size, tick); + }); +} + +BufferCache::BufferCache(GraphicContext& graphics, CommandScheduler& scheduler, + PageManager& page_manager, TextureCache& texture_cache, ResourceMutex& resource_mutex) - : m_graphics(graphics), m_memory_tracker(page_manager), m_page_manager(page_manager), + : m_graphics(graphics), m_scheduler(scheduler), + m_gds_buffer(graphics, scheduler, MemoryUsage::Stream, 0, AllFlags, GdsBufferSize), + m_fault_readback(std::make_unique()), m_memory_tracker(page_manager), + m_staging_buffer(graphics, scheduler, MemoryUsage::Upload, 512 * MiB), + m_stream_buffer(graphics, scheduler, MemoryUsage::Stream, 64 * MiB), + m_download_buffer(graphics, scheduler, MemoryUsage::Download, 32 * MiB), + m_device_buffer(graphics, scheduler, MemoryUsage::DeviceLocal, 128 * MiB), + m_page_manager(page_manager), m_texture_cache(texture_cache), m_resource_mutex(resource_mutex) { - EXIT_NOT_IMPLEMENTED(!Common::Thread::IsMainThread()); - m_readback = std::make_unique(*this); + std::memset(m_gds_buffer.Mapped().data(), 0, static_cast(m_gds_buffer.Size())); + m_gds_buffer.Flush(0, m_gds_buffer.Size()); + if (!m_graphics.CanReportMemoryUsage()) { + return; + } + constexpr int64_t GiB = 1024ll * 1024 * 1024; + constexpr int64_t target_threshold = 8 * GiB; + const auto budget = + static_cast(std::min(m_graphics.GetTotalMemoryBudget(), INT64_MAX)); + const auto threshold = std::min(budget, target_threshold); + const auto expected = std::min(budget - 6 * threshold / 10, budget - GiB); + const auto critical = std::min(budget - 2 * threshold / 10, budget - GiB / 2); + m_trigger_gc_memory = static_cast(std::max(expected, GiB)); + m_critical_gc_memory = static_cast(std::max(critical, 2 * GiB)); } BufferCache::~BufferCache() { - m_readback.reset(); + if (m_fault_readback->Active()) { + EXIT("BufferCache: destroyed with an active fault readback\n"); + } if (!m_gpu_modified_ranges.Empty()) { EXIT("BufferCache: destroyed with pending GPU-modified ranges\n"); } + if (!m_pending_backing_publications.empty()) { + EXIT("BufferCache: destroyed with pending backing publications\n"); + } for (const auto& [vaddr, cached]: m_buffers) { (void)vaddr; if (m_memory_tracker.IsRegionGpuModified(cached->vaddr, cached->size)) { - EXIT("BufferCache: destroyed with GPU-modified buffer, addr=0x%016" PRIx64 - " size=0x%016" PRIx64 "\n", - cached->vaddr, cached->size); + EXIT("BufferCache: destroyed with GPU-modified buffer\n"); } } - if (!m_buffers.empty()) { - Transfer::WaitForQueueIdle(); - } m_buffers.clear(); } +bool BufferCache::SynchronizeBacking(uint64_t vaddr, uint64_t size) { + bool waited = false; + for (;;) { + uint64_t tick = 0; + const auto page_begin = vaddr & ~(TRACKER_PAGE_SIZE - 1); + const auto page_end = + (vaddr + size + TRACKER_PAGE_SIZE - 1) & ~(TRACKER_PAGE_SIZE - 1); + CacheRange affected {.address = page_begin, .size = page_end - page_begin}; + { + FaultSafeCacheLock lock(this, m_mutex); + bool changed = true; + while (changed) { + changed = false; + for (const auto& [address, cached]: m_buffers) { + const CacheRange previous = affected; + if (ResolveOverlap(affected, {address, cached->size}) && + (previous.address != affected.address || previous.size != affected.size)) { + changed = true; + } + } + } + } + { + std::lock_guard lock(m_publication_mutex); + for (const auto& publication: m_pending_backing_publications) { + if (publication.address < affected.address + affected.size && + affected.address < publication.address + publication.size) { + tick = std::max(tick, publication.tick); + } + } + } + if (tick == 0) { + return waited; + } + waited = true; + m_scheduler.Wait(tick); + m_scheduler.WaitPriorityOperations(tick); + } +} + +void BufferCache::RefreshInvalidatedRanges(CommandBuffer& command, CachedBuffer& cached, + uint64_t vaddr, uint64_t size, bool upload) { + const auto invalidated = m_image_invalidated_ranges.Intersections(vaddr, size); + if (upload) { + std::array bytes; + for (const auto& range: invalidated) { + for (uint64_t copied = 0; copied < range.size;) { + const auto chunk = std::min(range.size - copied, bytes.size()); + if (!Libs::LibKernel::Memory::TryReadBacking(range.address + copied, bytes.data(), + chunk)) { + EXIT("BufferCache: failed to refresh an invalidated image alias\n"); + } + Upload(command, *cached.buffer, cached.buffer->Offset(range.address + copied), + bytes.data(), chunk); + copied += chunk; + } + } + } + if (!invalidated.empty()) { + m_image_invalidated_ranges.Subtract(vaddr, size); + } +} + +StreamBuffer& BufferCache::GetUtilityBuffer(MemoryUsage usage) noexcept { + switch (usage) { + case MemoryUsage::Upload: return m_staging_buffer; + case MemoryUsage::Stream: return m_stream_buffer; + case MemoryUsage::Download: return m_download_buffer; + case MemoryUsage::DeviceLocal: return m_device_buffer; + } + EXIT("BufferCache: invalid utility-buffer usage\n"); +} + +BufferBinding BufferCache::UploadTransient(const void* data, uint64_t size, uint64_t alignment) { + EXIT_IF(data == nullptr || size == 0); + if (auto [mapped, offset] = m_stream_buffer.Map(size, alignment, false); mapped != nullptr) { + std::memcpy(mapped, data, static_cast(size)); + m_stream_buffer.Commit(); + return {{}, m_stream_buffer.Handle(), offset}; + } + auto owner = + std::make_shared(m_graphics, m_scheduler, MemoryUsage::Upload, 0, AllFlags, size); + owner->Write(0, data, size); + return {owner, owner->Handle(), 0}; +} + bool BufferCache::InvalidateMemory(PageFaultAccess access, uint64_t vaddr, uint64_t size, PageFaultPhase phase) noexcept { const auto page = vaddr & ~(TRACKER_PAGE_SIZE - 1); if (size == 0 || size > page + TRACKER_PAGE_SIZE - vaddr) { - EXIT("BufferCache: invalid page-fault range, addr=0x%016" PRIx64 " size=0x%016" PRIx64 "\n", - vaddr, size); + EXIT("BufferCache: invalid page-fault range\n"); } - switch (phase) { - case PageFaultPhase::Invalidate: break; - case PageFaultPhase::Complete: - return m_readback->Complete(access, vaddr, size) || - m_memory_tracker.CompleteCpuFault(vaddr, size, access, false); - case PageFaultPhase::Release: m_readback->Release(access, vaddr, size); return true; - default: - EXIT("BufferCache: unsupported page-fault phase %u\n", static_cast(phase)); + + if (phase == PageFaultPhase::Complete) { + FaultSafeCacheLock lock(this, m_mutex); + auto& fault = *m_fault_readback; + if (!fault.Active()) { + return m_memory_tracker.CompleteCpuFault(vaddr, size, access, false); + } + if (fault.access != access || fault.vaddr != vaddr || fault.size != size || + fault.installed) { + EXIT("BufferCache: mismatched fault readback completion\n"); + } + PublishDownloads(fault.ranges); + if (!m_memory_tracker.CompleteCpuFault(vaddr, size, access, true)) { + EXIT("BufferCache: failed to complete downloaded CPU fault\n"); + } + fault.installed = true; + return true; } + + if (phase == PageFaultPhase::Release) { + FaultSafeCacheLock lock(this, m_mutex); + auto& fault = *m_fault_readback; + if (fault.Active()) { + if (fault.access != access || fault.vaddr != vaddr || fault.size != size || + !fault.installed) { + EXIT("BufferCache: mismatched fault readback release\n"); + } + for (const auto& range: fault.ranges) { + m_gpu_modified_ranges.Subtract(range.address, range.size); + } + fault.Reset(); + } + return true; + } + + if (phase != PageFaultPhase::Invalidate) { + EXIT("BufferCache: unsupported page-fault phase\n"); + } + const auto action = m_memory_tracker.BeginCpuFault(vaddr, size, access); if (action != CpuFaultAction::Download) { return action == CpuFaultAction::Continue; } - if (GraphicsRunIsCommandProcessorThread()) { - GraphicsRunFinishScheduler(); + + auto& fault = *m_fault_readback; + std::vector copies; + { + FaultSafeCacheLock lock(this, m_mutex); + if (fault.Active()) { + EXIT("BufferCache: nested fault readback\n"); + } + fault.access = access; + fault.vaddr = vaddr; + fault.size = size; + + m_gpu_modified_ranges.ForEachIntersection( + page, TRACKER_PAGE_SIZE, [&](RangeSet::Range range) { + auto owner = m_buffers.upper_bound(range.address); + if (owner == m_buffers.begin()) { + EXIT("BufferCache: fault readback has no buffer owner\n"); + } + --owner; + auto& cached = *owner->second; + if (!cached.buffer->IsInBounds(range.address, range.size)) { + EXIT("BufferCache: fault readback is outside its buffer owner\n"); + } + copies.push_back({cached.buffer, cached.buffer->Offset(range.address), + range.address, range.size}); + }); + if (copies.empty()) { + EXIT("BufferCache: GPU-dirty fault page has no dirty byte ranges\n"); + } } - m_readback->Request(access, vaddr, size); + fault.ranges = RecordDownloads(copies); + if (!fault.Active()) { + EXIT("BufferCache: GPU-dirty fault page has no dirty byte ranges\n"); + } + m_scheduler.FinishCurrent(); return true; } void BufferCache::UnmapMemory(uint64_t vaddr, uint64_t size) { - GraphicsRunSubmissionLock submissions; - FaultSafeCacheLock lock(this, m_mutex); - for (const auto& [begin, cached]: m_buffers) { - if (vaddr < begin + cached->size && begin < vaddr + size && - (vaddr > begin || size < cached->size || vaddr + size < begin + cached->size)) { - EXIT("BufferCache: partial buffer unmap is unsupported, unmap=0x%016" PRIx64 - "+0x%016" PRIx64 " buffer=0x%016" PRIx64 "+0x%016" PRIx64 "\n", - vaddr, size, begin, cached->size); + if (vaddr == 0 || size == 0 || size > UINT64_MAX - vaddr) { + EXIT("BufferCache: invalid unmap range\n"); + } + (void)SynchronizeBacking(vaddr, size); + + std::vector copies; + std::vector dirty_ranges; + std::vector> modified_buffers; + std::vector> backing_writes; + std::vector> retired_buffers; + { + FaultSafeCacheLock lock(this, m_mutex); + for (const auto& [begin, cached]: m_buffers) { + if (vaddr < begin + cached->size && begin < vaddr + size) { + retired_buffers.emplace_back(begin, cached->size); + } + } + for (const auto& [begin, cached]: m_buffers) { + if (vaddr >= begin + cached->size || begin >= vaddr + size || + !m_memory_tracker.IsRegionGpuModified(begin, cached->size)) { + continue; + } + const auto dirty = m_gpu_modified_ranges.Intersections(begin, cached->size); + if (dirty.empty()) { + EXIT("BufferCache: GPU-modified buffer has no dirty ranges\n"); + } + dirty_ranges.insert(dirty_ranges.end(), dirty.begin(), dirty.end()); + modified_buffers.emplace_back(begin, cached->size); + } + if (!dirty_ranges.empty()) { + backing_writes = m_page_manager.ReserveBackingWrites(dirty_ranges); + } + for (const auto& [begin, bytes]: modified_buffers) { + auto owner = m_buffers.find(begin); + if (owner == m_buffers.end() || owner->second->size != bytes) { + EXIT("BufferCache: unmap owner changed during collection\n"); + } + auto& cached = *owner->second; + m_memory_tracker.ForEachDownloadRange( + begin, cached.size, + [&](uint64_t address, uint64_t bytes) noexcept { + m_memory_tracker.ValidateGpuDirtyPages(m_gpu_modified_ranges, address, bytes, + "unmap"); + }, + [&](uint64_t address, uint64_t bytes) noexcept { + for (const auto& range: m_gpu_modified_ranges.Intersections(address, bytes)) { + copies.push_back( + {cached.buffer, range.address - begin, range.address, range.size}); + } + }); } } - for (const auto& [begin, cached]: m_buffers) { - const auto offset = begin >= vaddr ? begin - vaddr : UINT64_MAX; - if (offset > size || cached->size > size - offset || - !m_memory_tracker.IsRegionGpuModified(begin, cached->size)) { - continue; - } - const auto dirty = m_gpu_modified_ranges.Intersections(begin, cached->size); - if (dirty.empty()) { - EXIT("BufferCache: GPU-modified buffer has no dirty ranges, addr=0x%016" PRIx64 - " size=0x%016" PRIx64 "\n", - begin, cached->size); - } - auto backing_writes = ReserveBackingWrites(m_page_manager, dirty); - uint64_t downloaded = 0; - m_memory_tracker.ForEachDownloadRange( - begin, cached->size, - [&](uint64_t address, uint64_t bytes) noexcept { - ValidateDirtyPages(m_gpu_modified_ranges, address, bytes, "unmap"); - }, - [&](uint64_t address, uint64_t bytes) noexcept { - const auto ranges = m_gpu_modified_ranges.Intersections(address, bytes); - for (const auto& range: ranges) { - std::vector data(range.size); - Transfer::DownloadBuffer(*cached->buffer, range.address - begin, data.data(), - range.size); - Libs::LibKernel::Memory::WriteBacking(range.address, data.data(), data.size()); - downloaded += range.size; - } - }); - if (downloaded == 0) { - EXIT("BufferCache: GPU-modified buffer downloaded no bytes, addr=0x%016" PRIx64 - " size=0x%016" PRIx64 "\n", - begin, cached->size); - } - m_memory_tracker.MarkRegionAsCpuModified(begin, cached->size); - m_gpu_modified_ranges.Subtract(begin, cached->size); + if (!copies.empty()) { + auto downloads = RecordDownloads(copies); + m_scheduler.FinishCurrent(); + PublishDownloads(downloads); + } else if (!retired_buffers.empty()) { + // Image uploads can reference a clean cached buffer without owning it. Submit the active + // command stream before removing such backing. + m_scheduler.FinishCurrent(); } - if (!m_gpu_modified_ranges.Intersections(vaddr, size).empty()) { - EXIT("BufferCache: unmap retained dirty byte ranges, addr=0x%016" PRIx64 - " size=0x%016" PRIx64 "\n", - vaddr, size); - } - m_memory_tracker.UntrackMemory(vaddr, size); - for (auto it = m_buffers.begin(); it != m_buffers.end();) { - const auto offset = (it->first >= vaddr ? it->first - vaddr : UINT64_MAX); - if (offset <= size && it->second->size <= size - offset) { - it = m_buffers.erase(it); - } else { - ++it; + backing_writes.clear(); + + { + FaultSafeCacheLock lock(this, m_mutex); + for (const auto& [begin, bytes]: modified_buffers) { + if (!m_memory_tracker.IsRegionGpuModified(begin, bytes)) { + continue; + } + m_memory_tracker.ForEachDownloadRange( + begin, bytes, + [&](uint64_t address, uint64_t download_size) noexcept { + m_memory_tracker.ValidateGpuDirtyPages( + m_gpu_modified_ranges, address, download_size, "unmap retirement"); + }, + [](uint64_t, uint64_t) noexcept {}); + } + for (const auto& [begin, bytes]: modified_buffers) { + m_gpu_modified_ranges.Subtract(begin, bytes); + } + for (const auto& [begin, bytes]: retired_buffers) { + m_memory_tracker.MarkRegionAsCpuModified(begin, bytes); + } + if (!m_gpu_modified_ranges.Intersections(vaddr, size).empty()) { + EXIT("BufferCache: unmap retained dirty byte ranges\n"); + } + m_image_invalidated_ranges.Subtract(vaddr, size); + m_memory_tracker.UntrackMemory(vaddr, size); + for (auto it = m_buffers.begin(); it != m_buffers.end();) { + if (vaddr < it->first + it->second->size && it->first < vaddr + size) { + if (it->second->size > m_total_used_memory) { + EXIT("BufferCache: allocation accounting underflow\n"); + } + m_total_used_memory -= it->second->size; + it = m_buffers.erase(it); + } else { + ++it; + } } } } -BufferBinding BufferCache::ObtainBuffer(CommandBuffer& command, uint64_t vaddr, uint64_t size, - bool is_written, bool is_read, bool is_formatted) { - if (command.IsInvalid() || command.IsExecute() || vaddr == 0 || size == 0 || - size > UINT64_MAX - vaddr) { - EXIT("BufferCache: invalid buffer request, addr=0x%016" PRIx64 " size=0x%016" PRIx64 "\n", - vaddr, size); - } - ValidateGpuAccess(vaddr, size, is_read, is_written); - const auto begin = AlignDown(vaddr); - const auto end = AlignUp(vaddr + size); - std::lock_guard transaction(m_resource_mutex); - const auto texture_region = m_texture_cache->QueryRegion(vaddr, size); - // Snapshot small CPU-current reads now and upload them only during the final non-resetting - // commit. This keeps unaligned guest addresses valid without tying preparation to a command-local - // stream allocation. - if (is_read && !is_written && size <= CACHING_PAGE_SIZE && - !m_memory_tracker.IsRegionGpuModified(vaddr, size) && - m_memory_tracker.IsRegionCpuModified(vaddr, size) && !texture_region.gpu_image_bytes && - !texture_region.gpu_metadata_bytes) { - std::vector guest_data(size); - if (Libs::LibKernel::Memory::TryReadBacking(vaddr, guest_data.data(), size)) { - return {nullptr, 0, std::move(guest_data)}; - } - } - const auto texture_pages = begin == vaddr && end - begin == size - ? texture_region - : m_texture_cache->QueryRegion(begin, end - begin); - if (texture_pages.metadata_pages) { - EXIT("BufferCache: buffer aliases metadata pages, request=0x%016" PRIx64 "+0x%016" PRIx64 - " aligned=0x%016" PRIx64 "+0x%016" PRIx64 " read=%d written=%d formatted=%d" - " request_meta=%d/%d/%d aligned_meta=%d/%d/%d\n", - vaddr, size, begin, end - begin, is_read, is_written, is_formatted, - texture_region.metadata_pages, texture_region.metadata_bytes, - texture_region.gpu_metadata_bytes, texture_pages.metadata_pages, - texture_pages.metadata_bytes, texture_pages.gpu_metadata_bytes); - } - // Cache allocations are tracker-page aligned, but byte-disjoint buffers and images may share - // an edge page. Clean read-only buffer and image views may coexist; Kyty retains a hard failure - // when either cache owns newer GPU bytes. Writable buffers delegate the ownership transition - // to TextureCache, which distinguishes raw texture-data writes from formatted target paths. - if (texture_pages.image_pages && texture_region.image_bytes) { - const bool coherent_read = is_read && !is_written && - !m_memory_tracker.IsRegionGpuModified(vaddr, size) && - !texture_region.gpu_image_bytes; - if (!coherent_read) { - if (!is_written) { - EXIT("BufferCache: unsupported buffer/image alias, addr=0x%016" PRIx64 - " size=0x%016" PRIx64 " read=%d written=%d formatted=%d\n", - vaddr, size, is_read, is_written, is_formatted); - } - (void)m_texture_cache->InvalidateMemoryFromGPU(vaddr, size, is_formatted); - } - } - if (is_written) { - m_readback->Prepare(); - } - FaultSafeCacheLock lock(this, m_mutex); - - auto it = m_buffers.upper_bound(vaddr); +BufferCache::CachedBuffer& BufferCache::GetOrCreateBuffer(CommandBuffer& command, uint64_t vaddr, + uint64_t size) { + const auto begin = AlignDown(vaddr); + const auto end = AlignUp(vaddr + size); + auto it = m_buffers.upper_bound(vaddr); if (it != m_buffers.begin()) { - auto previous = std::prev(it); - const auto offset = vaddr - previous->second->vaddr; - if (offset <= previous->second->size && size <= previous->second->size - offset) { + auto previous = std::prev(it); + if (previous->second->buffer->IsInBounds(vaddr, size)) { it = previous; } } - if (it == m_buffers.end() || it->second->vaddr > vaddr || - size > it->second->size - (vaddr - it->second->vaddr)) { - BufferCacheRange merged {.address = begin, .size = end - begin}; - using BufferIterator = decltype(m_buffers.begin()); - std::vector overlaps; - auto first = m_buffers.lower_bound(begin); - if (first != m_buffers.begin()) { - auto previous = std::prev(first); - if (MergeOverlappingBufferCacheRange( - merged, {previous->second->vaddr, previous->second->size})) { - first = previous; - } - } - for (auto candidate = first; candidate != m_buffers.end(); ++candidate) { - if (candidate->first >= merged.address + merged.size) { - break; - } - if (MergeOverlappingBufferCacheRange( - merged, {candidate->second->vaddr, candidate->second->size})) { - overlaps.push_back(candidate); - } - } - - if (!overlaps.empty()) { - for (const auto& overlap: overlaps) { - auto& old = *overlap->second; - if (old.buffer == nullptr || old.buffer->buffer == nullptr) { - EXIT("BufferCache: invalid overlapping buffer owner, addr=0x%016" PRIx64 - " size=0x%016" PRIx64 " buffer=%p\n", - old.vaddr, old.size, static_cast(old.buffer.get())); - } - std::vector> uploads; - m_memory_tracker.ForEachUploadRange( - old.vaddr, old.size, false, - [&](uint64_t upload_vaddr, uint64_t upload_size) noexcept { - uploads.emplace_back(upload_vaddr, upload_size); - }, - [&]() noexcept { - for (const auto& [upload_vaddr, upload_size]: uploads) { - Transfer::UploadBuffer(Transfer::StagingBufferType::Vertex, *old.buffer, - upload_vaddr - old.vaddr, - reinterpret_cast(upload_vaddr), - upload_size); - } - }); - } - } - auto cached = std::make_unique(); - cached->vaddr = merged.address; - cached->size = merged.size; - cached->buffer = MakeSharedVulkanBuffer(); - cached->buffer->usage = - vk::BufferUsageFlagBits::eTransferSrc | vk::BufferUsageFlagBits::eTransferDst | - vk::BufferUsageFlagBits::eVertexBuffer | vk::BufferUsageFlagBits::eIndexBuffer | - vk::BufferUsageFlagBits::eStorageBuffer; - cached->buffer->memory.property = vk::MemoryPropertyFlagBits::eDeviceLocal; - m_graphics.CreateBuffer(cached->size, *cached->buffer); - if (!overlaps.empty()) { - auto vk_buffer = command.Handle(); - std::vector before; - before.reserve(overlaps.size() + 1); - for (const auto& overlap: overlaps) { - vk::BufferMemoryBarrier barrier {}; - barrier.sType = vk::StructureType::eBufferMemoryBarrier; - barrier.srcAccessMask = vk::AccessFlagBits::eMemoryWrite; - barrier.dstAccessMask = vk::AccessFlagBits::eTransferRead; - barrier.srcQueueFamilyIndex = VK_QUEUE_FAMILY_IGNORED; - barrier.dstQueueFamilyIndex = VK_QUEUE_FAMILY_IGNORED; - barrier.buffer = overlap->second->buffer->buffer; - barrier.offset = 0; - barrier.size = overlap->second->size; - before.push_back(barrier); - } - vk::BufferMemoryBarrier destination {}; - destination.sType = vk::StructureType::eBufferMemoryBarrier; - destination.dstAccessMask = vk::AccessFlagBits::eTransferWrite; - destination.srcQueueFamilyIndex = VK_QUEUE_FAMILY_IGNORED; - destination.dstQueueFamilyIndex = VK_QUEUE_FAMILY_IGNORED; - destination.buffer = cached->buffer->buffer; - destination.offset = 0; - destination.size = cached->size; - before.push_back(destination); - vk_buffer.pipelineBarrier( - vk::PipelineStageFlagBits::eAllCommands, vk::PipelineStageFlagBits::eTransfer, - vk::DependencyFlagBits::eByRegion, 0, nullptr, static_cast(before.size()), - before.data(), 0, nullptr); - for (const auto& overlap: overlaps) { - const auto& old = *overlap->second; - const vk::BufferCopy copy { - .srcOffset = 0, .dstOffset = old.vaddr - cached->vaddr, .size = old.size}; - vk_buffer.copyBuffer(old.buffer->buffer, cached->buffer->buffer, 1, ©); - } - vk::BufferMemoryBarrier after {}; - after.sType = vk::StructureType::eBufferMemoryBarrier; - after.srcAccessMask = vk::AccessFlagBits::eTransferWrite; - after.dstAccessMask = - vk::AccessFlagBits::eMemoryRead | vk::AccessFlagBits::eMemoryWrite; - after.srcQueueFamilyIndex = VK_QUEUE_FAMILY_IGNORED; - after.dstQueueFamilyIndex = VK_QUEUE_FAMILY_IGNORED; - after.buffer = cached->buffer->buffer; - after.offset = 0; - after.size = cached->size; - vk_buffer.pipelineBarrier( - vk::PipelineStageFlagBits::eTransfer, vk::PipelineStageFlagBits::eAllCommands, - vk::DependencyFlagBits::eByRegion, 0, nullptr, 1, &after, 0, nullptr); - for (const auto& overlap: overlaps) { - command.RetainResourceUntilFence(overlap->second->buffer); - m_buffers.erase(overlap); - } - } - it = m_buffers.emplace(cached->vaddr, std::move(cached)).first; + if (it != m_buffers.end() && it->second->buffer->IsInBounds(vaddr, size)) { + it->second->tick_accessed_last = m_gc_tick; + return *it->second; } - auto& cached = *it->second; - std::vector> ranges; - ranges.reserve((size + 2 * TRACKER_PAGE_SIZE - 2) / TRACKER_PAGE_SIZE); - m_memory_tracker.ForEachUploadRange( - vaddr, size, is_written, - [&](uint64_t upload_vaddr, uint64_t upload_size) noexcept { - ranges.emplace_back(upload_vaddr, upload_size); - }, - [&]() noexcept { - for (const auto& [upload_vaddr, upload_size]: ranges) { - Transfer::UploadBuffer(Transfer::StagingBufferType::Vertex, *cached.buffer, - upload_vaddr - cached.vaddr, - reinterpret_cast(upload_vaddr), upload_size); - } - }); - if (is_written) { - m_gpu_modified_ranges.Add(vaddr, size); - } - return {cached.buffer, vaddr - cached.vaddr}; -} - -bool BufferCache::UploadHostData(CommandBuffer& command, const void* src, uint64_t size, - uint64_t alignment, VulkanBuffer*& out_buffer, - uint64_t& out_offset, uint64_t& out_range) { - if (command.IsInvalid() || command.IsExecute()) { - EXIT("BufferCache: host stream upload requires a recording command buffer\n"); - } - return command.m_host_stream.Copy(src, size, alignment, out_buffer, out_offset, out_range); -} - -std::shared_ptr BufferCache::ObtainNullBuffer() { - FaultSafeCacheLock lock(this, m_mutex); - if (m_null_buffer == nullptr) { - // robustBufferAccess makes every fetch safe; TODO: Use a - // persistent 16-byte fallback when Vulkan null vertex descriptors are unavailable. - auto buffer = MakeSharedVulkanBuffer(); - buffer->usage = vk::BufferUsageFlagBits::eStorageBuffer | - vk::BufferUsageFlagBits::eVertexBuffer | - vk::BufferUsageFlagBits::eIndexBuffer; - buffer->memory.property = vk::MemoryPropertyFlagBits::eHostVisible | - vk::MemoryPropertyFlagBits::eHostCoherent | - vk::MemoryPropertyFlagBits::eHostCached; - m_graphics.CreateBuffer(16, *buffer); - void* data = nullptr; - m_graphics.MapMemory(buffer->memory, data); - std::memset(data, 0, 16); - m_graphics.UnmapMemory(buffer->memory); - m_null_buffer = std::move(buffer); - } - - return m_null_buffer; -} - -BufferImageCopySource BufferCache::ObtainBufferForImage(uint64_t vaddr, uint64_t size) { - if (vaddr == 0 || size == 0 || vaddr >= TRACKER_ADDRESS_SIZE || - size > TRACKER_ADDRESS_SIZE - vaddr || (vaddr & (TRACKER_PAGE_SIZE - 1)) != 0 || - (size & (TRACKER_PAGE_SIZE - 1)) != 0) { - EXIT("BufferCache: invalid image source, addr=0x%016" PRIx64 " size=0x%016" PRIx64 - " page_aligned=%d\n", - vaddr, size, - (vaddr & (TRACKER_PAGE_SIZE - 1)) == 0 && (size & (TRACKER_PAGE_SIZE - 1)) == 0); - } - FaultSafeCacheLock lock(this, m_mutex); - const bool cpu_modified = m_memory_tracker.IsRegionCpuModified(vaddr, size); - const bool gpu_modified = m_memory_tracker.IsRegionGpuModified(vaddr, size); - const auto dirty_ranges = m_gpu_modified_ranges.Intersections(vaddr, size); - if (gpu_modified != !dirty_ranges.empty()) { - EXIT("BufferCache: image-source tracker and byte ownership disagree, addr=0x%016" PRIx64 - " size=0x%016" PRIx64 " tracker_dirty=%d byte_ranges=%zu\n", - vaddr, size, gpu_modified, dirty_ranges.size()); - } - auto find_owner = [&](uint64_t address, uint64_t bytes) noexcept -> CachedBuffer& { - auto owner = m_buffers.upper_bound(address); - if (owner == m_buffers.begin()) { - EXIT("BufferCache: GPU image-source bytes have no cached owner\n"); + CacheRange merged {.address = begin, .size = end - begin}; + using Iterator = decltype(m_buffers.begin()); + std::vector overlaps; + auto first = m_buffers.lower_bound(begin); + if (first != m_buffers.begin()) { + auto previous = std::prev(first); + if (ResolveOverlap(merged, {previous->second->vaddr, previous->second->size})) { + first = previous; } - --owner; - auto& cached = *owner->second; - const auto offset = address - cached.vaddr; - if (offset > cached.size || bytes > cached.size - offset || cached.buffer == nullptr || - cached.buffer->buffer == nullptr) { - EXIT("BufferCache: GPU image-source bytes have no containing native buffer\n"); + } + for (auto candidate = first; candidate != m_buffers.end(); ++candidate) { + if (candidate->first >= merged.address + merged.size) { + break; } - return cached; - }; - if (gpu_modified) { - if (!GraphicsRunIsCommandProcessorThread() && !GraphicsRunSubmissionLockHeld() && - !LabelInCallback()) { - EXIT("BufferCache: GPU-dirty image source requires ordered GPU context, " - "addr=0x%016" PRIx64 " size=0x%016" PRIx64 - " command_thread=%d submission_lock=%d completion_callback=%d\n", - vaddr, size, GraphicsRunIsCommandProcessorThread(), - GraphicsRunSubmissionLockHeld(), LabelInCallback()); + if (ResolveOverlap(merged, {candidate->second->vaddr, candidate->second->size})) { + overlaps.push_back(candidate); } - Transfer::WaitForQueueIdle(); - auto backing_writes = ReserveBackingWrites(m_page_manager, dirty_ranges); - uint64_t downloaded = 0; - m_memory_tracker.ForEachDownloadRange( - vaddr, size, - [&](uint64_t address, uint64_t bytes) noexcept { - ValidateDirtyPages(m_gpu_modified_ranges, address, bytes, "image"); - }, - [&](uint64_t address, uint64_t bytes) noexcept { - const auto dirty = m_gpu_modified_ranges.Intersections(address, bytes); - if (dirty.empty()) { - EXIT("BufferCache: GPU-dirty image pages have no dirty byte ranges, " - "addr=0x%016" PRIx64 " size=0x%016" PRIx64 "\n", - address, bytes); - } - for (const auto& range: dirty) { - auto& owner = find_owner(range.address, range.size); - const auto buffer_offset = range.address - owner.vaddr; - std::vector data(range.size); - Transfer::DownloadBuffer(*owner.buffer, buffer_offset, data.data(), range.size); - Libs::LibKernel::Memory::WriteBacking(range.address, data.data(), data.size()); - downloaded += range.size; - } - }); - if (downloaded == 0) { - EXIT("BufferCache: image source cleared no tracked GPU pages, addr=0x%016" PRIx64 - " size=0x%016" PRIx64 "\n", - vaddr, size); - } - m_gpu_modified_ranges.Subtract(vaddr, size); } - auto owner = m_buffers.upper_bound(vaddr); - if (owner == m_buffers.begin()) { - return {nullptr, 0, vaddr, size, true, cpu_modified}; - } - --owner; - auto& cached = *owner->second; - const auto offset = vaddr - cached.vaddr; - if (offset > cached.size || size > cached.size - offset) { - return {nullptr, 0, vaddr, size, true, cpu_modified}; - } - if (cached.buffer == nullptr || cached.buffer->buffer == nullptr) { - EXIT("BufferCache: containing image source is inconsistent\n"); - } - if (cpu_modified) { + for (const auto overlap: overlaps) { + auto& old = *overlap->second; std::vector> uploads; m_memory_tracker.ForEachUploadRange( - vaddr, size, false, + old.vaddr, old.size, false, [&](uint64_t address, uint64_t bytes) noexcept { uploads.emplace_back(address, bytes); }, [&]() noexcept { for (const auto& [address, bytes]: uploads) { - Transfer::UploadBuffer(Transfer::StagingBufferType::Vertex, *cached.buffer, - address - cached.vaddr, - reinterpret_cast(address), bytes); + Upload(command, *old.buffer, old.buffer->Offset(address), + reinterpret_cast(address), bytes); } }); - if (uploads.empty()) { - EXIT("BufferCache: CPU-modified image source produced no uploads, addr=0x%016" PRIx64 - " size=0x%016" PRIx64 "\n", - vaddr, size); + } + + auto cached = std::make_unique(); + cached->vaddr = merged.address; + cached->size = merged.size; + cached->tick_accessed_last = m_gc_tick; + cached->buffer = std::make_shared(m_graphics, m_scheduler, MemoryUsage::DeviceLocal, + merged.address, AllFlags, merged.size); + for (const auto overlap: overlaps) { + const auto& old = *overlap->second; + cached->buffer->CopyFrom(command, *old.buffer, 0, old.vaddr - cached->vaddr, old.size); + command.RetainResourceUntilFence(old.buffer); + } + for (const auto overlap: overlaps) { + if (overlap->second->size > m_total_used_memory) { + EXIT("BufferCache: allocation accounting underflow\n"); + } + m_total_used_memory -= overlap->second->size; + m_buffers.erase(overlap); + } + m_total_used_memory += cached->size; + return *m_buffers.emplace(cached->vaddr, std::move(cached)).first->second; +} + +BufferBinding BufferCache::ObtainBuffer(CommandBuffer& command, uint64_t vaddr, uint64_t size, + bool is_written, bool is_read, bool is_formatted) { + if (command.IsInvalid() || command.IsExecute()) { + EXIT("BufferCache: buffer request requires a recording command buffer\n"); + } + ValidateGpuAccess(vaddr, size, is_read, is_written); + std::lock_guard transaction(m_resource_mutex); + (void)SynchronizeBacking(vaddr, size); + + if (is_read && !is_written && size <= CACHING_PAGE_SIZE && + !m_memory_tracker.IsRegionGpuModified(vaddr, size) && + m_memory_tracker.IsRegionCpuModified(vaddr, size)) { + std::vector data(size); + if (Libs::LibKernel::Memory::TryReadBacking(vaddr, data.data(), size)) { + return UploadTransient(data.data(), size, 16); } } - if (m_memory_tracker.IsRegionCpuModified(vaddr, size) || - m_memory_tracker.IsRegionGpuModified(vaddr, size) || - !m_gpu_modified_ranges.Intersections(vaddr, size).empty()) { - EXIT("BufferCache: image source did not become coherent, addr=0x%016" PRIx64 - " size=0x%016" PRIx64 "\n", - vaddr, size); + + if (is_formatted && is_read && !is_written) { + (void)m_texture_cache.SynchronizeImageToBuffer(vaddr, size); + } else if (is_formatted && is_written) { + (void)m_texture_cache.InvalidateMemoryFromGPU(vaddr, size, true); } - return {cached.buffer.get(), offset, vaddr, size, true, cpu_modified}; + + FaultSafeCacheLock lock(this, m_mutex); + auto& cached = GetOrCreateBuffer(command, vaddr, size); + std::vector> uploads; + m_memory_tracker.ForEachUploadRange( + vaddr, size, is_written, + [&](uint64_t address, uint64_t bytes) noexcept { uploads.emplace_back(address, bytes); }, + [&]() noexcept { + for (const auto& [address, bytes]: uploads) { + Upload(command, *cached.buffer, cached.buffer->Offset(address), + reinterpret_cast(address), bytes); + } + }); + RefreshInvalidatedRanges(command, cached, vaddr, size, is_read); + if (is_written) { + m_gpu_modified_ranges.Add(vaddr, size); + } + return {cached.buffer, cached.buffer->Handle(), cached.buffer->Offset(vaddr)}; } -namespace { -vk::BufferMemoryBarrier MakeDmaBarrier(VulkanBuffer& buffer, uint64_t offset, uint64_t size, - vk::AccessFlags source, vk::AccessFlags destination) { - if (buffer.buffer == nullptr || size == 0 || offset > buffer.buffer_size || - size > buffer.buffer_size - offset) { - EXIT("BufferCache: invalid DMA barrier, buffer=%p handle=%p offset=0x%016" PRIx64 - " size=0x%016" PRIx64 " buffer_size=0x%016" PRIx64 "\n", - static_cast(&buffer), static_cast(buffer.buffer), offset, - size, buffer.buffer_size); +std::shared_ptr BufferCache::ObtainNullBuffer() { + std::shared_ptr buffer; + { + FaultSafeCacheLock lock(this, m_mutex); + if (m_null_buffer != nullptr) { + return m_null_buffer; + } + m_null_buffer = std::make_shared(m_graphics, m_scheduler, MemoryUsage::DeviceLocal, + 0, AllFlags, 16); + buffer = m_null_buffer; } - vk::BufferMemoryBarrier barrier {}; - barrier.sType = vk::StructureType::eBufferMemoryBarrier; - barrier.srcAccessMask = source; - barrier.dstAccessMask = destination; - barrier.srcQueueFamilyIndex = VK_QUEUE_FAMILY_IGNORED; - barrier.dstQueueFamilyIndex = VK_QUEUE_FAMILY_IGNORED; - barrier.buffer = buffer.buffer; - barrier.offset = offset; - barrier.size = size; - return barrier; + const std::array zeros {}; + Upload(m_scheduler.Current(), *buffer, 0, zeros.data(), zeros.size()); + return buffer; } -} // namespace -void BufferCache::FillBuffer(CommandBuffer* command, uint64_t vaddr, uint64_t size, - uint32_t value) { - if ((vaddr & 3u) != 0 || (size & 3u) != 0) { - EXIT("BufferCache: fill range must be dword aligned, addr=0x%016" PRIx64 - " size=0x%016" PRIx64 "\n", - vaddr, size); +ImageBufferSource BufferCache::ObtainBufferForImage(uint64_t vaddr, uint64_t size) { + if (vaddr == 0 || size == 0 || vaddr >= TRACKER_ADDRESS_SIZE || + size > TRACKER_ADDRESS_SIZE - vaddr) { + EXIT("BufferCache: invalid image source\n"); + } + (void)SynchronizeBacking(vaddr, size); + auto find_owner = [&]() { + auto owner = m_buffers.upper_bound(vaddr); + if (owner == m_buffers.begin()) { + return m_buffers.end(); + } + --owner; + return owner->second->buffer->IsInBounds(vaddr, size) ? owner : m_buffers.end(); + }; + + { + FaultSafeCacheLock lock(this, m_mutex); + const bool cpu_modified = m_memory_tracker.IsRegionCpuModified(vaddr, size); + const bool gpu_modified = m_memory_tracker.IsRegionGpuModified(vaddr, size); + const auto dirty = m_gpu_modified_ranges.Intersections(vaddr, size); + const bool invalidated = + !m_image_invalidated_ranges.Intersections(vaddr, size).empty(); + const bool requested_gpu_owned = !dirty.empty(); + m_memory_tracker.ValidateGpuDirtyOwnership(m_gpu_modified_ranges, vaddr, size, + "image source"); + + auto owner = find_owner(); + if (requested_gpu_owned && owner == m_buffers.end()) { + CacheRange merged {.address = AlignDown(vaddr), + .size = AlignUp(vaddr + size) - AlignDown(vaddr)}; + using Iterator = decltype(m_buffers.begin()); + std::vector overlaps; + auto first = m_buffers.lower_bound(merged.address); + if (first != m_buffers.begin()) { + auto previous = std::prev(first); + if (ResolveOverlap(merged, {previous->second->vaddr, previous->second->size})) { + first = previous; + } + } + for (auto candidate = first; candidate != m_buffers.end(); ++candidate) { + if (candidate->first >= merged.address + merged.size) { + break; + } + if (ResolveOverlap(merged, {candidate->second->vaddr, candidate->second->size})) { + overlaps.push_back(candidate); + } + } + if (overlaps.empty()) { + EXIT("BufferCache: GPU-dirty image source has no native buffer\n"); + } + + auto cached = std::make_unique(); + cached->vaddr = merged.address; + cached->size = merged.size; + cached->tick_accessed_last = m_gc_tick; + cached->buffer = + std::make_shared(m_graphics, m_scheduler, MemoryUsage::DeviceLocal, + merged.address, AllFlags, merged.size); + for (const auto overlap: overlaps) { + const auto& old = *overlap->second; + cached->buffer->CopyFrom(m_scheduler.Current(), *old.buffer, 0, + old.vaddr - cached->vaddr, old.size); + m_scheduler.Current().RetainResourceUntilFence(old.buffer); + } + for (const auto overlap: overlaps) { + if (overlap->second->size > m_total_used_memory) { + EXIT("BufferCache: allocation accounting underflow\n"); + } + m_total_used_memory -= overlap->second->size; + m_buffers.erase(overlap); + } + m_total_used_memory += cached->size; + owner = m_buffers.emplace(cached->vaddr, std::move(cached)).first; + if (!owner->second->buffer->IsInBounds(vaddr, size)) { + EXIT("BufferCache: merged image source does not contain the requested range\n"); + } + } + if (owner != m_buffers.end() && !cpu_modified && !invalidated && + (!gpu_modified || requested_gpu_owned)) { + DiscardGpuDirtyBytesLocked(vaddr, size, "image source transfer"); + owner->second->tick_accessed_last = m_gc_tick; + return {owner->second->buffer.get(), owner->second->buffer->Offset(vaddr), + requested_gpu_owned}; + } + if (requested_gpu_owned && owner == m_buffers.end()) { + EXIT("BufferCache: GPU-dirty image source could not resolve its native owner\n"); + } + } + + // Direct-memory backing remains readable while PageManager protects the guest mapping. The + // fallback exists for plain host mappings used by standalone renderer tests and is deliberately + // performed outside the cache lock so a page fault cannot recurse into BufferCache. + const auto stage_address = vaddr & ~(TRACKER_PAGE_SIZE - 1); + const auto stage_end = (vaddr + size + TRACKER_PAGE_SIZE - 1) & ~(TRACKER_PAGE_SIZE - 1); + const auto stage_size = stage_end - stage_address; + (void)SynchronizeBacking(stage_address, stage_size); + std::vector bytes(stage_size); + if (!Libs::LibKernel::Memory::TryReadBacking(stage_address, bytes.data(), stage_size)) { + EXIT("BufferCache: failed to read mapped guest image backing\n"); + } + + FaultSafeCacheLock lock(this, m_mutex); + const auto dirty = m_gpu_modified_ranges.Intersections(vaddr, size); + const bool invalidated = !m_image_invalidated_ranges.Intersections(vaddr, size).empty(); + const bool requested_gpu_owned = !dirty.empty(); + auto owner = find_owner(); + if (requested_gpu_owned && owner == m_buffers.end()) { + EXIT("BufferCache: GPU-dirty image source lost its native owner\n"); + } + const auto stage_offset = m_staging_buffer.Copy(bytes.data(), stage_size, 16); + if (owner == m_buffers.end() || invalidated || + (m_memory_tracker.IsRegionGpuModified(vaddr, size) && !requested_gpu_owned)) { + return {&m_staging_buffer, stage_offset + vaddr - stage_address, false}; + } + + auto& cached = *owner->second; + cached.tick_accessed_last = m_gc_tick; + std::vector> uploads; + m_memory_tracker.ForEachUploadRange( + vaddr, size, false, + [&](uint64_t address, uint64_t upload_size) noexcept { + uploads.emplace_back(address, upload_size); + }, + [&]() noexcept { + for (const auto& [address, upload_size]: uploads) { + cached.buffer->CopyFrom( + m_scheduler.Current(), m_staging_buffer, + stage_offset + address - stage_address, cached.buffer->Offset(address), + upload_size, vk::AccessFlagBits::eHostWrite); + } + }); + DiscardGpuDirtyBytesLocked(vaddr, size, "staged image source transfer"); + return {cached.buffer.get(), cached.buffer->Offset(vaddr), requested_gpu_owned}; +} + +void BufferCache::DiscardGpuDirtyBytesLocked(uint64_t vaddr, uint64_t size, const char* operation) { + m_memory_tracker.ValidateGpuDirtyOwnership(m_gpu_modified_ranges, vaddr, size, operation); + m_gpu_modified_ranges.Subtract(vaddr, size); + const auto page_begin = vaddr & ~(TRACKER_PAGE_SIZE - 1); + const auto page_end = (vaddr + size + TRACKER_PAGE_SIZE - 1) & ~(TRACKER_PAGE_SIZE - 1); + for (auto page = page_begin; page < page_end; page += TRACKER_PAGE_SIZE) { + if (m_gpu_modified_ranges.Intersections(page, TRACKER_PAGE_SIZE).empty() && + m_memory_tracker.IsRegionGpuModified(page, TRACKER_PAGE_SIZE)) { + m_memory_tracker.UnmarkRegionAsGpuModified(page, TRACKER_PAGE_SIZE); + } + } + m_memory_tracker.ValidateGpuDirtyOwnership(m_gpu_modified_ranges, vaddr, size, operation); +} + +void BufferCache::DiscardGpuDirtyBytes(uint64_t vaddr, uint64_t size) { + if (vaddr == 0 || size == 0 || vaddr >= TRACKER_ADDRESS_SIZE || + size > TRACKER_ADDRESS_SIZE - vaddr) { + EXIT("BufferCache: invalid dirty-byte discard range\n"); + } + FaultSafeCacheLock lock(this, m_mutex); + DiscardGpuDirtyBytesLocked(vaddr, size, "image output supersession"); +} + +void BufferCache::WriteHostMemory(uint64_t vaddr, std::span data) { + if (vaddr == 0 || data.empty() || data.size() > UINT64_MAX - vaddr) { + EXIT("BufferCache: invalid host DMA write\n"); + } + (void)SynchronizeBacking(vaddr, data.size()); + Libs::LibKernel::Memory::WriteBacking(vaddr, data.data(), data.size()); + + FaultSafeCacheLock lock(this, m_mutex); + const auto end = vaddr + data.size(); + for (auto& [address, cached]: m_buffers) { + const auto cached_end = address + cached->size; + const auto begin = std::max(vaddr, address); + const auto range_end = std::min(end, cached_end); + if (begin >= range_end) { + continue; + } + Upload(m_scheduler.Current(), *cached->buffer, cached->buffer->Offset(begin), + data.data() + begin - vaddr, range_end - begin); + cached->tick_accessed_last = m_gc_tick; + } + m_image_invalidated_ranges.Subtract(vaddr, data.size()); +} + +std::pair, uint64_t> BufferCache::ObtainBufferForImageWrite(uint64_t vaddr, + uint64_t size) { + if (vaddr == 0 || size == 0 || vaddr >= TRACKER_ADDRESS_SIZE || + size > TRACKER_ADDRESS_SIZE - vaddr) { + EXIT("BufferCache: invalid image destination\n"); + } + const auto stage_address = vaddr & ~(TRACKER_PAGE_SIZE - 1); + const auto stage_end = (vaddr + size + TRACKER_PAGE_SIZE - 1) & ~(TRACKER_PAGE_SIZE - 1); + const auto stage_size = stage_end - stage_address; + (void)SynchronizeBacking(stage_address, stage_size); + std::vector bytes(stage_size); + if (!Libs::LibKernel::Memory::TryReadBacking(stage_address, bytes.data(), stage_size)) { + EXIT("BufferCache: failed to preserve guest bytes around an image mirror\n"); + } + FaultSafeCacheLock lock(this, m_mutex); + auto& cached = GetOrCreateBuffer(m_scheduler.Current(), vaddr, size); + m_memory_tracker.ValidateGpuDirtyOwnership(m_gpu_modified_ranges, vaddr, size, + "image destination"); + if (!m_gpu_modified_ranges.Intersections(vaddr, size).empty()) { + EXIT("BufferCache: image destination aliases GPU-owned buffer bytes\n"); + } + const auto stage_offset = m_staging_buffer.Copy(bytes.data(), stage_size, 16); + std::vector> uploads; + m_memory_tracker.ForEachUploadRange( + vaddr, size, false, + [&](uint64_t address, uint64_t upload_size) noexcept { + uploads.emplace_back(address, upload_size); + }, + [&]() noexcept { + for (const auto& [address, upload_size]: uploads) { + cached.buffer->CopyFrom( + m_scheduler.Current(), m_staging_buffer, + stage_offset + address - stage_address, cached.buffer->Offset(address), + upload_size, vk::AccessFlagBits::eHostWrite); + } + }); + return {cached.buffer, cached.buffer->Offset(vaddr)}; +} + +void BufferCache::FillBuffer(uint64_t vaddr, uint64_t size, uint32_t value, bool is_gds) { + if ((vaddr & 3u) != 0 || size == 0 || (size & 3u) != 0 || size > UINT64_MAX - vaddr) { + EXIT("BufferCache: fill range must be dword aligned\n"); + } + if (is_gds) { + if (vaddr > m_gds_buffer.Size() || size > m_gds_buffer.Size() - vaddr) { + EXIT("BufferCache: GDS fill range is out of bounds\n"); + } + m_gds_buffer.Fill(vaddr, size, value); + return; + } + if (vaddr == 0) { + EXIT("BufferCache: invalid fill memory address\n"); } ValidateGpuAccess(vaddr, size, false, true); + (void)m_texture_cache.ClearMeta(vaddr); { std::lock_guard transaction(m_resource_mutex); - const auto texture_region = m_texture_cache->QueryRegion(vaddr, size); - const bool image_overlap = texture_region.image_bytes; - const bool buffer_overlap = HasPageOverlap(vaddr, size); - const bool buffer_gpu_modified = IsRegionGpuModified(vaddr, size); - if (!buffer_overlap && !buffer_gpu_modified) { - if (image_overlap) { - m_texture_cache->PrepareHostWrite(vaddr, size); + const auto region = m_texture_cache.QueryRegion(vaddr, size); + if (!HasGpuDirtyBytes(vaddr, size) && !region.gpu_image_bytes) { + if (region.image_bytes) { + m_texture_cache.PrepareHostWrite(vaddr, size); + } + std::array values; + values.fill(value); + const std::span bytes { + reinterpret_cast(values.data()), sizeof(values)}; + for (uint64_t offset = 0; offset < size;) { + const auto chunk = std::min(size - offset, bytes.size()); + WriteHostMemory(vaddr + offset, bytes.first(chunk)); + offset += chunk; } - auto* dst = reinterpret_cast(vaddr); - std::fill(dst, dst + size / sizeof(uint32_t), value); return; } - if (image_overlap) { - EXIT("BufferCache: GPU fill aliases image pages, addr=0x%016" PRIx64 - " size=0x%016" PRIx64 "\n", - vaddr, size); - } - if (texture_region.metadata_bytes) { - LOGF("BufferCache: GPU fill overlaps virtual metadata, addr=0x%016" PRIx64 - " size=0x%016" PRIx64 "\n", - vaddr, size); - EXIT("BufferCache: GPU fill of virtual metadata is unsupported, addr=0x%016" PRIx64 - " size=0x%016" PRIx64 "\n", - vaddr, size); - } } - EXIT_IF(command == nullptr); - auto dst = ObtainBuffer(*command, vaddr, size, true, false); - EXIT_IF(dst.buffer == nullptr || !dst.host_data.empty()); - command->RetainResourceUntilFence(dst.buffer); - const auto before = MakeDmaBarrier( - *dst.buffer, dst.offset, size, - vk::AccessFlagBits::eMemoryRead | vk::AccessFlagBits::eMemoryWrite, - vk::AccessFlagBits::eTransferWrite); - const auto vk_buffer = command->Handle(); - vk_buffer.pipelineBarrier( - vk::PipelineStageFlagBits::eAllCommands, vk::PipelineStageFlagBits::eTransfer, - vk::DependencyFlagBits::eByRegion, 0, nullptr, 1, &before, 0, nullptr); - vk_buffer.fillBuffer(dst.buffer->buffer, dst.offset, size, value); - const auto after = - MakeDmaBarrier(*dst.buffer, dst.offset, size, vk::AccessFlagBits::eTransferWrite, - vk::AccessFlagBits::eMemoryRead | vk::AccessFlagBits::eMemoryWrite); - vk_buffer.pipelineBarrier(vk::PipelineStageFlagBits::eTransfer, - vk::PipelineStageFlagBits::eAllCommands, - vk::DependencyFlagBits::eByRegion, 0, nullptr, 1, &after, 0, nullptr); + + auto& command = m_scheduler.Current(); + auto dst = ObtainBuffer(command, vaddr, size, true, false, true); + EXIT_IF(dst.buffer == nullptr || dst.owner == nullptr); + command.RetainResourceUntilFence(dst.owner); + auto owner = std::static_pointer_cast(dst.owner); + owner->Fill(dst.offset, size, value); } -void BufferCache::CopyBuffer(CommandBuffer* command, uint64_t dst_vaddr, uint64_t src_vaddr, - uint64_t size) { - if (dst_vaddr == 0 || src_vaddr == 0 || size == 0 || +void BufferCache::CopyBuffer(uint64_t dst_vaddr, uint64_t src_vaddr, uint64_t size, bool dst_gds, + bool src_gds) { + const bool dst_memory = !dst_gds; + const bool src_memory = !src_gds; + if ((dst_memory && dst_vaddr == 0) || (src_memory && src_vaddr == 0) || size == 0 || ((dst_vaddr | src_vaddr | size) & 3u) != 0 || size > UINT64_MAX - dst_vaddr || - size > UINT64_MAX - src_vaddr || - (src_vaddr < dst_vaddr + size && dst_vaddr < src_vaddr + size)) { - EXIT("BufferCache: invalid or overlapping copy range, src=0x%016" PRIx64 - " dst=0x%016" PRIx64 " size=0x%016" PRIx64 "\n", - src_vaddr, dst_vaddr, size); + size > UINT64_MAX - src_vaddr || (dst_gds && src_gds) || + (dst_gds == src_gds && src_vaddr < dst_vaddr + size && dst_vaddr < src_vaddr + size) || + (dst_gds && (dst_vaddr > m_gds_buffer.Size() || size > m_gds_buffer.Size() - dst_vaddr)) || + (src_gds && (src_vaddr > m_gds_buffer.Size() || size > m_gds_buffer.Size() - src_vaddr))) { + EXIT("BufferCache: invalid or overlapping copy range\n"); } - ValidateGpuAccess(src_vaddr, size, true, false); - ValidateGpuAccess(dst_vaddr, size, false, true); - bool dst_image_transition = false; - { + if (src_memory) { + ValidateGpuAccess(src_vaddr, size, true, false); + } + if (dst_memory) { + ValidateGpuAccess(dst_vaddr, size, false, true); + } + if (src_memory || dst_memory) { std::lock_guard transaction(m_resource_mutex); - const auto src_region = m_texture_cache->QueryRegion(src_vaddr, size); - const auto dst_region = m_texture_cache->QueryRegion(dst_vaddr, size); - const bool src_image_gpu = src_region.gpu_image_bytes; - const bool src_meta = src_region.metadata_bytes; - if (src_region.non_sampled_pages) { - EXIT("BufferCache: GPU copy aliases target pages, src=0x%016" PRIx64 - " dst=0x%016" PRIx64 " size=0x%016" PRIx64 "\n", - src_vaddr, dst_vaddr, size); + if (src_memory) { + (void)SynchronizeBacking(src_vaddr, size); } - if (!HasPageOverlap(dst_vaddr, size) && !IsRegionGpuModified(src_vaddr, size) && - !IsRegionGpuModified(dst_vaddr, size) && !src_image_gpu) { - if (src_region.gpu_metadata_bytes) { - LOGF("BufferCache: host copy reads virtual metadata, src=0x%016" PRIx64 - " size=0x%016" PRIx64 "\n", - src_vaddr, size); - EXIT("BufferCache: host copy from GPU-modified metadata is unsupported, " - "src=0x%016" PRIx64 " size=0x%016" PRIx64 "\n", - src_vaddr, size); - } + const auto src_region = + src_memory ? m_texture_cache.QueryRegion(src_vaddr, size) : TextureCache::RegionInfo {}; + const auto dst_region = + dst_memory ? m_texture_cache.QueryRegion(dst_vaddr, size) : TextureCache::RegionInfo {}; + if (src_memory && src_region.gpu_image_bytes && + !m_texture_cache.SynchronizeImageToBuffer(src_vaddr, size)) { + EXIT("BufferCache: GPU copy source image could not be synchronized\n"); + } + if (src_memory && dst_memory && !HasGpuDirtyBytes(src_vaddr, size) && + !HasGpuDirtyBytes(dst_vaddr, size) && !src_region.gpu_image_bytes && + !dst_region.gpu_image_bytes) { if (dst_region.image_bytes) { - m_texture_cache->PrepareHostWrite(dst_vaddr, size); + m_texture_cache.PrepareHostWrite(dst_vaddr, size); + } + std::array bytes; + for (uint64_t offset = 0; offset < size;) { + const auto chunk = std::min(size - offset, bytes.size()); + if (!Libs::LibKernel::Memory::TryReadBacking(src_vaddr + offset, bytes.data(), + chunk)) { + EXIT("BufferCache: host DMA source has no direct backing\n"); + } + WriteHostMemory(dst_vaddr + offset, std::span {bytes}.first(chunk)); + offset += chunk; } - std::memcpy(reinterpret_cast(dst_vaddr), - reinterpret_cast(src_vaddr), size); return; } - if (src_image_gpu) { - EXIT("BufferCache: GPU copy source aliases GPU-modified image, src=0x%016" PRIx64 - " dst=0x%016" PRIx64 " size=0x%016" PRIx64 "\n", - src_vaddr, dst_vaddr, size); - } - dst_image_transition = m_texture_cache->InvalidateMemoryFromGPU(dst_vaddr, size); - const auto transitioned_dst = m_texture_cache->QueryRegion(dst_vaddr, size); - // A clean target destination is handled above like a protected host write. Target - // aliases that require an actual GPU buffer copy remain unsupported. - if (transitioned_dst.non_sampled_pages) { - EXIT("BufferCache: GPU copy aliases target pages, src=0x%016" PRIx64 - " dst=0x%016" PRIx64 " size=0x%016" PRIx64 "\n", - src_vaddr, dst_vaddr, size); - } - if (src_meta || transitioned_dst.metadata_bytes) { - LOGF("BufferCache: GPU copy overlaps virtual metadata, src=0x%016" PRIx64 - " dst=0x%016" PRIx64 " size=0x%016" PRIx64 "\n", - src_vaddr, dst_vaddr, size); - EXIT("BufferCache: GPU copy involving virtual metadata is unsupported, " - "src=0x%016" PRIx64 " dst=0x%016" PRIx64 " size=0x%016" PRIx64 "\n", - src_vaddr, dst_vaddr, size); - } } - EXIT_IF(command == nullptr); - auto src = ObtainBuffer(*command, src_vaddr, size, false, true); - auto dst = ObtainBuffer(*command, dst_vaddr, size, true, false, dst_image_transition); - EXIT_IF(dst.buffer == nullptr || !dst.host_data.empty()); - VulkanBuffer* src_buffer = src.buffer.get(); - vk::DeviceSize src_offset = src.offset; - if (!src.host_data.empty()) { - vk::DeviceSize range = 0; - EXIT_IF(!UploadHostData(*command, src.host_data.data(), src.host_data.size(), 16, - src_buffer, src_offset, range) || - range != size); - } else { - command->RetainResourceUntilFence(src.buffer); + + auto& command = m_scheduler.Current(); + auto src = src_memory ? ObtainBuffer(command, src_vaddr, size, false, true) + : BufferBinding {.buffer = m_gds_buffer.Handle(), .offset = src_vaddr}; + auto dst = dst_memory ? ObtainBuffer(command, dst_vaddr, size, true, false, true) + : BufferBinding {.buffer = m_gds_buffer.Handle(), .offset = dst_vaddr}; + EXIT_IF(src.buffer == nullptr || dst.buffer == nullptr || (dst_memory && dst.owner == nullptr)); + if (src.owner != nullptr) { + command.RetainResourceUntilFence(src.owner); + } + if (dst.owner != nullptr) { + command.RetainResourceUntilFence(dst.owner); } - command->RetainResourceUntilFence(dst.buffer); - EXIT_IF(src_buffer == nullptr); if (src.buffer == dst.buffer && src.offset < dst.offset + size && dst.offset < src.offset + size) { - EXIT("BufferCache: resolved Vulkan copy ranges overlap, src_offset=0x%016" PRIx64 - " dst_offset=0x%016" PRIx64 " size=0x%016" PRIx64 "\n", - src.offset, dst.offset, size); + EXIT("BufferCache: resolved Vulkan copy ranges overlap\n"); } - const vk::BufferMemoryBarrier before[] = { - MakeDmaBarrier(*dst.buffer, dst.offset, size, - vk::AccessFlagBits::eMemoryRead | vk::AccessFlagBits::eMemoryWrite, - vk::AccessFlagBits::eTransferWrite), - MakeDmaBarrier(*src_buffer, src_offset, size, vk::AccessFlagBits::eMemoryWrite, - vk::AccessFlagBits::eTransferRead), - }; - const auto vk_buffer = command->Handle(); - vk_buffer.pipelineBarrier(vk::PipelineStageFlagBits::eAllCommands, - vk::PipelineStageFlagBits::eTransfer, - vk::DependencyFlagBits::eByRegion, 0, nullptr, 2, before, 0, nullptr); - const vk::BufferCopy copy {src_offset, dst.offset, size}; - vk_buffer.copyBuffer(src_buffer->buffer, dst.buffer->buffer, 1, ©); - const vk::BufferMemoryBarrier after[] = { - MakeDmaBarrier(*dst.buffer, dst.offset, size, vk::AccessFlagBits::eTransferWrite, - vk::AccessFlagBits::eMemoryRead | vk::AccessFlagBits::eMemoryWrite), - MakeDmaBarrier(*src_buffer, src_offset, size, vk::AccessFlagBits::eTransferRead, - vk::AccessFlagBits::eMemoryWrite), - }; - vk_buffer.pipelineBarrier(vk::PipelineStageFlagBits::eTransfer, - vk::PipelineStageFlagBits::eAllCommands, - vk::DependencyFlagBits::eByRegion, 0, nullptr, 2, after, 0, nullptr); + auto& source = src.owner != nullptr ? *std::static_pointer_cast(src.owner) + : src_gds ? m_gds_buffer + : m_stream_buffer; + auto& destination = dst.owner != nullptr ? *std::static_pointer_cast(dst.owner) + : m_gds_buffer; + if (source.Handle() != src.buffer || destination.Handle() != dst.buffer) { + EXIT("BufferCache: resolved copy owner does not match its Vulkan handle\n"); + } + destination.CopyFrom(command, source, src.offset, dst.offset, size); } bool BufferCache::HasPageOverlap(uint64_t vaddr, uint64_t size) { if (vaddr == 0 || size == 0 || vaddr >= TRACKER_ADDRESS_SIZE || size > TRACKER_ADDRESS_SIZE - vaddr) { - EXIT("BufferCache: invalid page-overlap query, addr=0x%016" PRIx64 " size=0x%016" PRIx64 - "\n", - vaddr, size); + EXIT("BufferCache: invalid page-overlap query\n"); } FaultSafeCacheLock lock(this, m_mutex); for (const auto& [address, cached]: m_buffers) { @@ -1181,72 +1065,179 @@ bool BufferCache::IsRegionGpuModified(uint64_t vaddr, uint64_t size) { return m_memory_tracker.IsRegionGpuModified(vaddr, size); } +bool BufferCache::HasGpuDirtyBytes(uint64_t vaddr, uint64_t size) { + FaultSafeCacheLock lock(this, m_mutex); + return !m_gpu_modified_ranges.Intersections(vaddr, size).empty(); +} + bool BufferCache::IsRegionCpuModified(uint64_t vaddr, uint64_t size) { return m_memory_tracker.IsRegionCpuModified(vaddr, size); } -void BufferCache::PublishImageBacking(uint64_t vaddr, uint64_t size) { +void BufferCache::InvalidateImageAliases(uint64_t vaddr, uint64_t size) { + if (vaddr == 0 || size == 0 || vaddr >= TRACKER_ADDRESS_SIZE || + size > TRACKER_ADDRESS_SIZE - vaddr) { + EXIT("BufferCache: invalid image-alias invalidation\n"); + } + FaultSafeCacheLock lock(this, m_mutex); + const auto end = vaddr + size; + for (const auto& [address, cached]: m_buffers) { + const auto cached_end = address + cached->size; + const auto begin = std::max(vaddr, address); + const auto range_end = std::min(end, cached_end); + if (begin >= range_end) { + continue; + } + const auto bytes = range_end - begin; + if (!m_gpu_modified_ranges.Intersections(begin, bytes).empty()) { + EXIT("BufferCache: image ownership overlaps exact dirty buffer bytes\n"); + } + m_image_invalidated_ranges.Add(begin, bytes); + } +} + +void BufferCache::BeginBackingPublication(uint64_t vaddr, uint64_t size, uint64_t tick) { + if (vaddr == 0 || size == 0 || tick == 0 || vaddr >= TRACKER_ADDRESS_SIZE || + size > TRACKER_ADDRESS_SIZE - vaddr) { + EXIT("BufferCache: invalid pending backing publication\n"); + } + std::lock_guard lock(m_publication_mutex); + m_pending_backing_publications.push_back({vaddr, size, tick}); +} + +void BufferCache::CompleteBackingPublication(uint64_t vaddr, uint64_t size, uint64_t tick) { + std::lock_guard lock(m_publication_mutex); + const auto publication = + std::ranges::find_if(m_pending_backing_publications, [&](const auto& pending) { + return pending.address == vaddr && pending.size == size && pending.tick == tick; + }); + if (publication == m_pending_backing_publications.end()) { + EXIT("BufferCache: completed an unknown backing publication\n"); + } + m_pending_backing_publications.erase(publication); +} + +void BufferCache::PublishImageBuffer(uint64_t vaddr, uint64_t size) { FaultSafeCacheLock lock(this, m_mutex); auto owner = m_buffers.end(); for (auto it = m_buffers.begin(); it != m_buffers.end(); ++it) { if (!PageOverlaps(vaddr, size, it->second->vaddr, it->second->size)) { continue; } - const auto offset = vaddr >= it->second->vaddr ? vaddr - it->second->vaddr : UINT64_MAX; - if (owner != m_buffers.end() || offset > it->second->size || - size > it->second->size - offset) { - EXIT("BufferCache: image backing aliases a non-containing cached buffer, " - "addr=0x%016" PRIx64 " size=0x%016" PRIx64 " buffer=0x%016" PRIx64 "+0x%016" PRIx64 - "\n", - vaddr, size, it->second->vaddr, it->second->size); + if (owner != m_buffers.end() || !it->second->buffer->IsInBounds(vaddr, size)) { + EXIT("BufferCache: image destination aliases a non-containing cached buffer\n"); } owner = it; } - if ((owner != m_buffers.end() && m_memory_tracker.IsRegionCpuModified(vaddr, size)) || - m_memory_tracker.IsRegionGpuModified(vaddr, size) || + m_memory_tracker.ValidateGpuDirtyOwnership(m_gpu_modified_ranges, vaddr, size, + "image destination publication"); + if (owner == m_buffers.end() || m_memory_tracker.IsRegionCpuModified(vaddr, size) || !m_gpu_modified_ranges.Intersections(vaddr, size).empty()) { - EXIT("BufferCache: image backing requires clean buffer ownership, addr=0x%016" PRIx64 - " size=0x%016" PRIx64 " cached=%d\n", - vaddr, size, owner != m_buffers.end()); + EXIT("BufferCache: image destination requires clean buffer ownership\n"); } - // A fresh tracker is CPU-dirty by construction when no cached buffer exists. Keep the range - // CPU-dirty so subsequently-created buffer uploads the just-published image backing. - m_memory_tracker.MarkRegionAsCpuModified(vaddr, size); + m_memory_tracker.MarkRegionAsGpuModified(vaddr, size); + m_gpu_modified_ranges.Add(vaddr, size); + m_image_invalidated_ranges.Subtract(vaddr, size); + m_memory_tracker.ValidateGpuDirtyOwnership(m_gpu_modified_ranges, vaddr, size, + "published image destination"); + owner->second->tick_accessed_last = m_gc_tick; } void BufferCache::ValidateGpuAccess(uint64_t vaddr, uint64_t size, bool is_read, bool is_written) const { if ((!is_read && !is_written) || vaddr == 0 || size == 0 || size > UINT64_MAX - vaddr) { - EXIT("BufferCache: invalid GPU access request, addr=0x%016" PRIx64 " size=0x%016" PRIx64 - " read=%d write=%d\n", - vaddr, size, is_read, is_written); + EXIT("BufferCache: invalid GPU access request\n"); } if (is_read && !m_page_manager.HasGpuAccess(vaddr, size, GpuAccess::Read)) { - LOGF("BufferCache: GPU-read access denied, addr=0x%016" PRIx64 " size=0x%016" PRIx64 "\n", - vaddr, size); - EXIT("BufferCache: GPU-read access denied, addr=0x%016" PRIx64 " size=0x%016" PRIx64 "\n", - vaddr, size); + EXIT("BufferCache: GPU-read access denied\n"); } if (is_written && !m_page_manager.HasGpuAccess(vaddr, size, GpuAccess::Write)) { - LOGF("BufferCache: GPU-write access denied, addr=0x%016" PRIx64 " size=0x%016" PRIx64 "\n", - vaddr, size); - EXIT("BufferCache: GPU-write access denied, addr=0x%016" PRIx64 " size=0x%016" PRIx64 "\n", - vaddr, size); + EXIT("BufferCache: GPU-write access denied\n"); } } -void BufferCache::SetTextureCache(TextureCache& texture_cache) { - if (m_texture_cache != nullptr) { - EXIT("BufferCache: texture cache already connected\n"); +void BufferCache::RunGarbageCollector() { + std::lock_guard transaction(m_resource_mutex); + const auto tick = m_gc_tick++; + if (m_graphics.CanReportMemoryUsage()) { + m_total_used_memory = m_graphics.GetDeviceMemoryUsage(); + } + if (m_total_used_memory < m_trigger_gc_memory || m_fault_readback->Active()) { + return; } - m_texture_cache = &texture_cache; -} -void BufferCache::ResetNullBuffer() { - KYTY_PROFILER_BLOCK("BufferCache::ResetNullBuffer"); + const bool aggressive = m_total_used_memory >= m_critical_gc_memory; + const uint64_t age = std::min(aggressive ? 80 : 160, tick); + const size_t limit = aggressive ? 64 : 32; + + std::vector retires; + std::vector>> dirty_retires; + { + FaultSafeCacheLock lock(this, m_mutex); + std::vector candidates; + for (const auto& [address, owner]: m_buffers) { + const auto& cached = *owner; + if (tick - std::min(tick, cached.tick_accessed_last) < age) { + continue; + } + candidates.push_back(address); + } + std::ranges::sort(candidates, [&](uint64_t left, uint64_t right) { + return m_buffers.at(left)->tick_accessed_last < m_buffers.at(right)->tick_accessed_last; + }); + if (candidates.size() > limit) { + candidates.resize(limit); + } + for (const auto address: candidates) { + auto& cached = *m_buffers.at(address); + m_memory_tracker.ValidateGpuDirtyOwnership( + m_gpu_modified_ranges, cached.vaddr, cached.size, "garbage collection"); + retires.push_back({address, cached.size, cached.buffer}); + // GC runs immediately before submission. Preserve every source referenced by commands + // already recorded in the active batch. + m_scheduler.Current().RetainResourceUntilFence(cached.buffer); + } + for (const auto& retire: retires) { + if (!m_memory_tracker.IsRegionGpuModified(retire.address, retire.size)) { + continue; + } + auto& copies = dirty_retires.emplace_back(retire, std::vector {}).second; + m_memory_tracker.ForEachDownloadRange( + retire.address, retire.size, + [&](uint64_t address, uint64_t size) noexcept { + m_memory_tracker.ValidateGpuDirtyPages( + m_gpu_modified_ranges, address, size, "garbage collection"); + }, + [&](uint64_t address, uint64_t size) noexcept { + for (const auto range: m_gpu_modified_ranges.Intersections(address, size)) { + copies.push_back({retire.owner, range.address - retire.address, range.address, + range.size}); + } + }); + } + } + + for (auto& [retire, copies]: dirty_retires) { + QueueGarbageDownload(copies, std::move(retire)); + } FaultSafeCacheLock lock(this, m_mutex); - m_null_buffer.reset(); + for (const auto& retire: retires) { + auto found = m_buffers.find(retire.address); + if (found == m_buffers.end() || found->second->size != retire.size || + found->second->buffer != retire.owner) { + EXIT("BufferCache: garbage-collection owner changed during download\n"); + } + if (!m_memory_tracker.IsRegionGpuModified(retire.address, retire.size)) { + m_memory_tracker.UntrackMemory(retire.address, retire.size); + } + m_image_invalidated_ranges.Subtract(retire.address, retire.size); + if (retire.size > m_total_used_memory) { + EXIT("BufferCache: allocation accounting underflow\n"); + } + m_total_used_memory -= retire.size; + m_buffers.erase(found); + } } } // namespace Libs::Graphics diff --git a/src/graphics/host_gpu/renderer/bufferCache.h b/src/graphics/host_gpu/renderer/bufferCache.h index b887c467..9a9c1179 100644 --- a/src/graphics/host_gpu/renderer/bufferCache.h +++ b/src/graphics/host_gpu/renderer/bufferCache.h @@ -6,44 +6,34 @@ #include "common/threads.h" #include "graphics/host_gpu/memoryTracker.h" #include "graphics/host_gpu/rangeSet.h" +#include "graphics/host_gpu/renderer/streamBuffer.h" #include #include #include +#include +#include #include namespace Libs::Graphics { struct GraphicContext; -struct VulkanBuffer; class CommandBuffer; +class CommandScheduler; class TextureCache; class ResourceMutex; -struct BufferImageCopySource { - VulkanBuffer* buffer = nullptr; - uint64_t offset = 0; - uint64_t address = 0; - uint64_t size = 0; - bool cpu_current = false; - // True when the guest range was CPU-dirty before coherence resolution. ObtainBufferForImage - // may consume that tracker state while publishing the same bytes to a cached buffer. - bool cpu_dirty = false; -}; - -struct BufferCacheRange { - uint64_t address = 0; - uint64_t size = 0; -}; - struct BufferBinding { - std::shared_ptr buffer; - uint64_t offset = 0; - std::vector host_data; + std::shared_ptr owner; + vk::Buffer buffer = nullptr; + uint64_t offset = 0; }; -[[nodiscard]] bool MergeOverlappingBufferCacheRange(BufferCacheRange& merged, - BufferCacheRange candidate) noexcept; +struct ImageBufferSource { + Buffer* buffer = nullptr; + uint64_t offset = 0; + bool gpu_owned = false; +}; class BufferCache { public: @@ -52,7 +42,8 @@ public: return vaddr & (CACHING_PAGE_SIZE - 1); } - BufferCache(GraphicContext& graphics, PageManager& page_manager, ResourceMutex& resource_mutex); + BufferCache(GraphicContext& graphics, CommandScheduler& scheduler, PageManager& page_manager, + TextureCache& texture_cache, ResourceMutex& resource_mutex); ~BufferCache(); KYTY_CLASS_NO_COPY(BufferCache); @@ -62,39 +53,91 @@ public: [[nodiscard]] BufferBinding ObtainBuffer(CommandBuffer& command, uint64_t vaddr, uint64_t size, bool is_written = false, bool is_read = true, bool is_formatted = false); - // Emulator-owned, CPU-current scratch only. Guest ranges must use ObtainBuffer so page - // ownership is resolved before any CPU access. - [[nodiscard]] bool UploadHostData(CommandBuffer& command, const void* src, uint64_t size, - uint64_t alignment, VulkanBuffer*& out_buffer, - uint64_t& out_offset, uint64_t& out_range); - [[nodiscard]] std::shared_ptr ObtainNullBuffer(); - [[nodiscard]] BufferImageCopySource ObtainBufferForImage(uint64_t vaddr, uint64_t size); - void FillBuffer(CommandBuffer* command, uint64_t vaddr, uint64_t size, uint32_t value); - void CopyBuffer(CommandBuffer* command, uint64_t dst_vaddr, uint64_t src_vaddr, uint64_t size); + [[nodiscard]] StreamBuffer& GetUtilityBuffer(MemoryUsage usage) noexcept; + [[nodiscard]] Buffer& GetGdsBuffer() noexcept { return m_gds_buffer; } + [[nodiscard]] const Buffer& GetGdsBuffer() const noexcept { return m_gds_buffer; } + [[nodiscard]] BufferBinding UploadTransient(const void* data, uint64_t size, + uint64_t alignment); + [[nodiscard]] std::shared_ptr ObtainNullBuffer(); + [[nodiscard]] ImageBufferSource ObtainBufferForImage(uint64_t vaddr, uint64_t size); + [[nodiscard]] std::pair, uint64_t> + ObtainBufferForImageWrite(uint64_t vaddr, uint64_t size); + void DiscardGpuDirtyBytes(uint64_t vaddr, uint64_t size); + void FillBuffer(uint64_t vaddr, uint64_t size, uint32_t value, bool is_gds = false); + void CopyBuffer(uint64_t dst_vaddr, uint64_t src_vaddr, uint64_t size, bool dst_gds = false, + bool src_gds = false); [[nodiscard]] bool HasPageOverlap(uint64_t vaddr, uint64_t size); + [[nodiscard]] bool HasGpuDirtyBytes(uint64_t vaddr, uint64_t size); [[nodiscard]] bool IsRegionCpuModified(uint64_t vaddr, uint64_t size); [[nodiscard]] bool IsRegionGpuModified(uint64_t vaddr, uint64_t size); - void PublishImageBacking(uint64_t vaddr, uint64_t size); + void InvalidateImageAliases(uint64_t vaddr, uint64_t size); + void BeginBackingPublication(uint64_t vaddr, uint64_t size, uint64_t tick); + void CompleteBackingPublication(uint64_t vaddr, uint64_t size, uint64_t tick); + [[nodiscard]] bool SynchronizeBacking(uint64_t vaddr, uint64_t size); + void PublishImageBuffer(uint64_t vaddr, uint64_t size); void ValidateGpuAccess(uint64_t vaddr, uint64_t size, bool is_read, bool is_written) const; - void SetTextureCache(TextureCache& texture_cache); - - void ResetNullBuffer(); + void RunGarbageCollector(); private: - struct CachedBuffer; - struct ReadbackWorker; + friend struct BufferCacheTestAccess; - GraphicContext& m_graphics; - Common::Mutex m_mutex; - std::shared_ptr m_null_buffer; - // TODO: add LRU cache + struct CacheRange { + uint64_t address = 0; + uint64_t size = 0; + }; + struct CachedBuffer; + struct DownloadCopy; + struct DownloadRange; + struct RetiredBuffer; + struct FaultReadback; + struct PendingBackingPublication; + static constexpr uint64_t DOWNLOAD_ALIGNMENT = 64; + [[nodiscard]] static uint64_t AlignDown(uint64_t value) noexcept; + [[nodiscard]] static uint64_t AlignUp(uint64_t value); + [[nodiscard]] static constexpr uint64_t AlignDownload(uint64_t size) noexcept { + return (size + DOWNLOAD_ALIGNMENT - 1) & ~(DOWNLOAD_ALIGNMENT - 1); + } + [[nodiscard]] static bool PageOverlaps(uint64_t left, uint64_t left_size, uint64_t right, + uint64_t right_size) noexcept; + [[nodiscard]] static std::pair + DownloadEnvelope(const DownloadCopy& copy); + [[nodiscard]] static bool ResolveOverlap(CacheRange& merged, CacheRange candidate) noexcept; + void Upload(CommandBuffer& command, Buffer& destination, uint64_t destination_offset, + const void* source, uint64_t size); + [[nodiscard]] CachedBuffer& GetOrCreateBuffer(CommandBuffer& command, uint64_t vaddr, + uint64_t size); + [[nodiscard]] std::vector + RecordDownloads(std::span copies); + void PublishDownloads(std::span downloads); + void QueueGarbageDownload(std::span copies, RetiredBuffer retire); + void RefreshInvalidatedRanges(CommandBuffer& command, CachedBuffer& cached, uint64_t vaddr, + uint64_t size, bool upload); + void DiscardGpuDirtyBytesLocked(uint64_t vaddr, uint64_t size, const char* operation); + void WriteHostMemory(uint64_t vaddr, std::span data); + + GraphicContext& m_graphics; + CommandScheduler& m_scheduler; + Buffer m_gds_buffer; + Common::Mutex m_mutex; + std::shared_ptr m_null_buffer; std::map> m_buffers; - std::unique_ptr m_readback; + std::unique_ptr m_fault_readback; RangeSet m_gpu_modified_ranges; + RangeSet m_image_invalidated_ranges; + std::mutex m_publication_mutex; + std::vector m_pending_backing_publications; MemoryTracker m_memory_tracker; + StreamBuffer m_staging_buffer; + StreamBuffer m_stream_buffer; + StreamBuffer m_download_buffer; + StreamBuffer m_device_buffer; PageManager& m_page_manager; - TextureCache* m_texture_cache = nullptr; + TextureCache& m_texture_cache; ResourceMutex& m_resource_mutex; + uint64_t m_total_used_memory = 0; + uint64_t m_trigger_gc_memory = 1ull * 1024 * 1024 * 1024; + uint64_t m_critical_gc_memory = 2ull * 1024 * 1024 * 1024; + uint64_t m_gc_tick = 0; }; } // namespace Libs::Graphics diff --git a/src/graphics/host_gpu/renderer/colorRenderTarget.cpp b/src/graphics/host_gpu/renderer/colorRenderTarget.cpp index 17e7f680..c2839109 100644 --- a/src/graphics/host_gpu/renderer/colorRenderTarget.cpp +++ b/src/graphics/host_gpu/renderer/colorRenderTarget.cpp @@ -10,12 +10,9 @@ #include "graphics/host_gpu/objects/textureCommon.h" #include "graphics/host_gpu/renderer/debug.h" #include "graphics/host_gpu/renderer/descriptorCache.h" -#include "graphics/host_gpu/renderer/framebufferCache.h" #include "graphics/host_gpu/renderer/render.h" #include "graphics/host_gpu/renderer/renderContext.h" -#include "graphics/host_gpu/transfer.h" #include "graphics/host_gpu/vulkanCommon.h" -#include "graphics/presentation/displayBuffer.h" #include #include @@ -25,9 +22,11 @@ namespace Libs::Graphics { static std::atomic g_render_color_log_count = 0; // NOLINTNEXTLINE(readability-function-cognitive-complexity) -void ResolveRenderColorTarget(uint64_t submit_id, RenderCommandBuffer& buffer, RenderColorInfo& r, - uint32_t render_target_slice_offset, uint32_t render_target_slot, - bool ignore_target_mask, bool reuse_existing_render_texture) { +void RenderExecutor::ResolveRenderColorTarget(uint64_t submit_id, RenderCommandBuffer& buffer, + RenderColorInfo& r, + uint32_t render_target_slice_offset, + uint32_t render_target_slot, bool ignore_target_mask, + bool exact_format) { KYTY_PROFILER_FUNCTION(); const auto& hw = buffer.GetRegisters(); @@ -57,14 +56,17 @@ void ResolveRenderColorTarget(uint64_t submit_id, RenderCommandBuffer& buffer, R // No color output r.type = RenderColorType::NoColorOutput; + r.desc = {}; r.base_addr = 0; - r.vulkan_buffer = nullptr; - r.vulkan_view = nullptr; + r.image_id = {}; + r.image_view = nullptr; r.format = vk::Format::eUndefined; r.extent = {}; r.base_mip_level = 0; + r.base_array_layer = 0; r.buffer_size = 0; r.samples = 1; + r.export_mapping = {}; r.color_clear_enable = false; r.color_clear_value = {}; return; @@ -176,26 +178,33 @@ void ResolveRenderColorTarget(uint64_t submit_id, RenderCommandBuffer& buffer, R pitch = width; } + TileSizeOffset mip_sizes[16] {}; + TilePaddedSize mip_padded[16] {}; if (tile) { TileSizeAlign layout {}; bool valid_layout = false; if (standard64) { TileGetTextureSize(Prospero::GpuEnumValue(Prospero::BufferFormat::k32Float), width, - height, pitch, levels, rt.attrib3.tile_mode, &layout, nullptr, - nullptr); + height, pitch, levels, rt.attrib3.tile_mode, &layout, mip_sizes, + mip_padded); valid_layout = layout.size != 0 && layout.align == 65536; } else { valid_layout = levels == 1 ? TileGetRenderTargetSize(width, height, pitch, bytes_per_element, layout, rt.attrib.num_fragments) : TileGetRenderTargetMipLayout(width, height, pitch, bytes_per_element, - levels, layout, nullptr, nullptr); + levels, layout, mip_sizes, mip_padded); } if (!valid_layout) { EXIT("unsupported render-target layout: %ux%u pitch=%u bytes=%u levels=%u\n", width, height, pitch, bytes_per_element, levels); } size = layout.size; + EXIT_IF(size > UINT32_MAX); + if (levels == 1) { + mip_sizes[0] = {static_cast(size), 0, 0, 0, 0, 0}; + mip_padded[0] = {pitch, height}; + } if (rt.slice.slice_div64_minus1 != 0 && (static_cast(rt.slice.slice_div64_minus1) + 1u) * 64u != size) { EXIT("render-target slice span mismatch: encoded=0x%016" PRIx64 " derived=0x%016" PRIx64 @@ -204,6 +213,11 @@ void ResolveRenderColorTarget(uint64_t submit_id, RenderCommandBuffer& buffer, R } } else { size = static_cast(pitch) * height * bytes_per_element * samples; + if (size > UINT32_MAX) { + EXIT("linear render-target slice exceeds the supported layout size\n"); + } + mip_sizes[0] = {static_cast(size), 0, 0, 0, 0, 0}; + mip_padded[0] = {pitch, height}; } if (size == 0 || size > UINT64_MAX / view.image_layers) { EXIT("render-target memory footprint is invalid\n"); @@ -213,119 +227,69 @@ void ResolveRenderColorTarget(uint64_t submit_id, RenderCommandBuffer& buffer, R EXIT("render-target backing range is invalid\n"); } - auto video_image = Presentation::DisplayBufferFind(rt.base.addr, true); - if (video_image.image != nullptr && - !IsSupportedDisplayRenderTargetTileMode(rt.attrib3.tile_mode)) { - EXIT("unsupported display render-target tile mode: tile=%u expected=%u addr=0x%010" PRIx64 - " backing_size=0x%016" PRIx64 " video_size=0x%016" PRIx64 "\n", - rt.attrib3.tile_mode, Prospero::GpuEnumValue(Prospero::TileMode::kRenderTarget), - rt.base.addr, backing_size, video_image.size); - } - bool render_to_texture = view.base_layer != 0 || video_image.image == nullptr; - if (!render_to_texture && (levels != 1 || rt.view.current_mip_level != 0)) { - EXIT("mipmapped display render targets are unsupported\n"); - } const vk::Extent2D view_extent = {std::max(width >> rt.view.current_mip_level, 1u), std::max(height >> rt.view.current_mip_level, 1u)}; auto decision_log_id = g_render_color_log_count.fetch_add(1); - if (decision_log_id < 128 || !render_to_texture) { + if (decision_log_id < 128) { LOGF("RenderColorTarget: slot=%" PRIu32 " addr=0x%010" PRIx64 " size=0x%016" PRIx64 " extent=%ux%u view_mip=%u view_extent=%ux%u levels=%u pitch=%u" - " fmt=0x%08" PRIx32 " nfmt=0x%08" PRIx32 " order=0x%08" PRIx32 - " samples=%u tile=%s target=%s video_size=0x%016" PRIx64 " video_pitch=%" PRIu64 "\n", + " fmt=0x%08" PRIx32 " nfmt=0x%08" PRIx32 " order=0x%08" PRIx32 " samples=%u tile=%s\n", rt_slot, rt.base.addr, backing_size, width, height, rt.view.current_mip_level, view_extent.width, view_extent.height, levels, pitch, rt.info.format, - rt.info.channel_type, rt.info.channel_order, samples, tile ? "tiled" : "linear", - render_to_texture ? "RenderTexture" : "DisplayBuffer", video_image.size, - video_image.pitch); + rt.info.channel_type, rt.info.channel_order, samples, tile ? "tiled" : "linear"); } - if (render_to_texture) { - (void)reuse_existing_render_texture; - RenderTargetInfo target {}; - target.address = rt.base.addr; - target.size = backing_size; - target.format = target_format.format; - target.width = width; - target.height = height; - target.pitch = pitch; - target.bytes_per_element = target_format.bytes_per_element; - target.tile_mode = rt.attrib3.tile_mode; - target.levels = levels; - target.layers = view.image_layers; - target.samples = samples; - auto& texture_cache = GetRenderContext().GetTextureCache(); - auto& buffer_vulkan = texture_cache.FindRenderTarget(buffer, target); - r.type = RenderColorType::RenderTexture; - r.base_addr = rt.base.addr; - r.vulkan_buffer = &buffer_vulkan; - r.vulkan_view = texture_cache.GetRenderTargetAttachmentView( - buffer_vulkan, target.format, rt.view.current_mip_level, view.base_layer, - view.layer_count); - r.format = target.format; - r.extent = view_extent; - r.base_mip_level = rt.view.current_mip_level; - r.buffer_size = backing_size; - r.samples = samples; - r.export_mapping = target_format.export_mapping; - r.color_clear_enable = buffer_vulkan.initial_clear_pending; - r.color_clear_value = {}; - } else { - if (samples != 1) { - EXIT("multisampled display render targets are unsupported\n"); - } - const auto layout = static_cast(rt.info.format); - const auto type = static_cast(rt.info.channel_type); - const auto order = static_cast(rt.info.channel_order); - - bool supported_display_format = - (layout == Prospero::ChannelLayout::k8_8_8_8 && - (type == Prospero::ChannelType::kSrgb || type == Prospero::ChannelType::kUNorm) && - (order == Prospero::ChannelOrder::kStandard || - order == Prospero::ChannelOrder::kAlt)) || - (layout == Prospero::ChannelLayout::k10_10_10_2 && - type == Prospero::ChannelType::kUNorm && - (order == Prospero::ChannelOrder::kStandard || - order == Prospero::ChannelOrder::kAlt)) || - (layout == Prospero::ChannelLayout::k16_16_16_16 && - type == Prospero::ChannelType::kFloat && - (order == Prospero::ChannelOrder::kStandard || order == Prospero::ChannelOrder::kAlt)); - - EXIT_NOT_IMPLEMENTED(!supported_display_format); - - // Display buffer - if (video_image.size != size) { - LOGF("RenderColorTarget: display buffer size differs from render target span, " - "video_size=0x%016" PRIx64 " render_size=0x%016" PRIx64 "\n", - video_image.size, size); - } - EXIT_NOT_IMPLEMENTED(video_image.size < size); - EXIT_NOT_IMPLEMENTED(video_image.pitch != pitch); - r.type = RenderColorType::DisplayBuffer; - r.base_addr = rt.base.addr; - r.vulkan_buffer = video_image.image; - r.vulkan_view = video_image.image->image_view[VulkanImage::VIEW_DEFAULT]; - r.format = video_image.image->format; - r.extent = video_image.image->extent; - r.base_mip_level = 0; - r.buffer_size = video_image.size; - r.samples = 1; - r.export_mapping = target_format.export_mapping; - } -} - -void MarkRenderTargetGpuWritten(const RenderColorInfo& target) { - const bool with_color = target.vulkan_buffer != nullptr; - - if (with_color) { - if (target.type == RenderColorType::RenderTexture || - target.type == RenderColorType::DisplayBuffer) { - GetRenderContext().GetTextureCache().MarkGpuWritten(*target.vulkan_buffer); - } else { - EXIT("unknown writable render-color resource type\n"); - } + TextureCache::ImageDesc desc {}; + desc.type = TextureCache::BindingType::RenderTarget; + desc.info.data = {rt.base.addr, backing_size}; + desc.info.pixel_format = target_format.format; + desc.info.guest_format = ImageOps::RenderTargetTransferFormat(bytes_per_element); + desc.info.type = Prospero::ImageType::kColor2D; + desc.info.extent = {width, height, 1}; + desc.info.resources = {levels, view.image_layers}; + desc.info.pitch = pitch; + desc.info.bytes_per_block = bytes_per_element; + desc.info.samples = samples; + desc.info.tile_mode = rt.attrib3.tile_mode; + for (uint32_t level = 0; level < levels; level++) { + const auto level_offset = + mip_sizes[level].src_size != 0 ? mip_sizes[level].src_offset : mip_sizes[level].offset; + const auto level_size = + static_cast(mip_sizes[level].src_size != 0 ? mip_sizes[level].src_size + : mip_sizes[level].size) * + view.image_layers; + desc.info.mip_layout[level] = { + level_offset, + level_size, + mip_padded[level].width, + mip_padded[level].height, + }; } + desc.view_info.format = target_format.format; + desc.view_info.type = + view.layer_count == 1 ? vk::ImageViewType::e2D : vk::ImageViewType::e2DArray; + desc.view_info.aspect = vk::ImageAspectFlagBits::eColor; + desc.view_info.base_level = rt.view.current_mip_level; + desc.view_info.level_count = 1; + desc.view_info.base_layer = view.base_layer; + desc.view_info.layer_count = view.layer_count; + desc.view_info.usage = vk::ImageUsageFlagBits::eColorAttachment; + auto& texture_cache = m_context.GetTextureCache(); + r.desc = std::move(desc); + r.image_id = texture_cache.FindImage(r.desc, exact_format); + r.type = RenderColorType::RenderTexture; + r.base_addr = rt.base.addr; + r.image_view = nullptr; + r.format = r.desc.view_info.format; + r.extent = view_extent; + r.base_mip_level = rt.view.current_mip_level; + r.buffer_size = backing_size; + r.samples = samples; + r.export_mapping = target_format.export_mapping; + r.color_clear_enable = false; + r.color_clear_value = {}; + BindRenderTarget(r.image_id); } } // namespace Libs::Graphics diff --git a/src/graphics/host_gpu/renderer/colorRenderTarget.h b/src/graphics/host_gpu/renderer/colorRenderTarget.h index d97aa9d6..2ddc0091 100644 --- a/src/graphics/host_gpu/renderer/colorRenderTarget.h +++ b/src/graphics/host_gpu/renderer/colorRenderTarget.h @@ -3,6 +3,7 @@ #include "graphics/guest_gpu/gpu_defs.h" #include "graphics/host_gpu/renderer/renderTarget.h" +#include "graphics/host_gpu/renderer/textureCache.h" #include "graphics/host_gpu/vulkanCommon.h" #include @@ -10,18 +11,17 @@ namespace Libs::Graphics { class RenderCommandBuffer; -struct VulkanImage; enum class RenderColorType { NoColorOutput, - DisplayBuffer, RenderTexture, }; struct RenderColorInfo { - RenderColorType type = RenderColorType::NoColorOutput; - VulkanImage* vulkan_buffer = nullptr; - vk::ImageView vulkan_view = nullptr; + RenderColorType type = RenderColorType::NoColorOutput; + TextureCache::ImageDesc desc; + ImageId image_id; + vk::ImageView image_view = nullptr; vk::Format format = vk::Format::eUndefined; vk::Extent2D extent = {}; uint32_t base_mip_level = 0; @@ -35,13 +35,6 @@ struct RenderColorInfo { vk::ClearColorValue color_clear_value {}; }; -void ResolveRenderColorTarget(uint64_t submit_id, RenderCommandBuffer& buffer, RenderColorInfo& r, - uint32_t render_target_slice_offset = 0, - uint32_t render_target_slot = UINT32_MAX, - bool ignore_target_mask = false, - bool reuse_existing_render_texture = false); -void MarkRenderTargetGpuWritten(const RenderColorInfo& target); - } // namespace Libs::Graphics #endif // EMULATOR_SRC_GRAPHICS_HOST_GPU_RENDERER_COLORRENDERTARGET_H_ diff --git a/src/graphics/host_gpu/renderer/commandScheduler.cpp b/src/graphics/host_gpu/renderer/commandScheduler.cpp index 45e0f1f6..c6f3b5e1 100644 --- a/src/graphics/host_gpu/renderer/commandScheduler.cpp +++ b/src/graphics/host_gpu/renderer/commandScheduler.cpp @@ -1,18 +1,161 @@ #include "graphics/host_gpu/renderer/commandScheduler.h" #include "common/assert.h" +#include "graphics/host_gpu/graphicContext.h" + +#include namespace Libs::Graphics { +static thread_local CommandScheduler* g_deferred_callback_scheduler = nullptr; + +void CommandSlot::Reset() { + EXIT_IF(buffer == nullptr); + const auto result = buffer.reset(vk::CommandBufferResetFlagBits::eReleaseResources); + if (result != vk::Result::eSuccess) { + EXIT("failed to reset Vulkan command buffer: %s (%d)\n", VulkanToString(result).c_str(), + static_cast(result)); + } +} + +CommandScheduler::CommandPool::~CommandPool() { + Destroy(); +} + +void CommandScheduler::CommandPool::Create(GraphicContext& graphics) { + EXIT_IF(m_pool != nullptr || m_graphics != nullptr || + graphics.queue_family == static_cast(-1)); + m_graphics = &graphics; + + vk::CommandPoolCreateInfo create {}; + create.sType = vk::StructureType::eCommandPoolCreateInfo; + create.queueFamilyIndex = graphics.queue_family; + create.flags = vk::CommandPoolCreateFlagBits::eResetCommandBuffer; + const auto result = graphics.device.createCommandPool(&create, nullptr, &m_pool); + EXIT_NOT_IMPLEMENTED(result != vk::Result::eSuccess || m_pool == nullptr); +} + +CommandSlot* CommandScheduler::CommandPool::CreateSlot() { + EXIT_IF(m_graphics == nullptr); + auto& graphics = *m_graphics; + + vk::CommandBufferAllocateInfo allocate {}; + allocate.sType = vk::StructureType::eCommandBufferAllocateInfo; + allocate.commandPool = m_pool; + allocate.level = vk::CommandBufferLevel::ePrimary; + allocate.commandBufferCount = 1; + vk::CommandBuffer buffer = nullptr; + EXIT_IF(graphics.device.allocateCommandBuffers(&allocate, &buffer) != vk::Result::eSuccess); + + vk::FenceCreateInfo fence_create {}; + fence_create.sType = vk::StructureType::eFenceCreateInfo; + fence_create.flags = vk::FenceCreateFlagBits::eSignaled; + vk::Fence fence = nullptr; + if (graphics.device.createFence(&fence_create, nullptr, &fence) != vk::Result::eSuccess) { + graphics.device.freeCommandBuffers(m_pool, 1, &buffer); + EXIT("failed to create command-buffer fence\n"); + } + + auto& slot = m_slots.emplace_back(); + slot.pool_mutex = &m_mutex; + slot.id = static_cast(m_slots.size() - 1); + slot.buffer = buffer; + slot.fence = fence; + return &slot; +} + +CommandSlot* CommandScheduler::CommandPool::Allocate(GraphicContext& graphics) { + Common::LockGuard lock(m_mutex); + if (m_pool == nullptr) { + Create(graphics); + } + EXIT_IF(m_graphics != &graphics); + auto found = std::ranges::find_if(m_slots, [](const auto& slot) { return !slot.busy; }); + auto* slot = found != m_slots.end() ? &*found : CreateSlot(); + slot->busy = true; + slot->Reset(); + return slot; +} + +void CommandScheduler::CommandPool::Destroy() { + Common::LockGuard lock(m_mutex); + if (m_pool == nullptr) { + return; + } + EXIT_IF(std::ranges::any_of(m_slots, [](const auto& slot) { return slot.busy; })); + EXIT_IF(m_graphics == nullptr); + for (const auto& slot: m_slots) { + m_graphics->device.destroyFence(slot.fence, nullptr); + } + m_graphics->device.destroyCommandPool(m_pool, nullptr); + m_slots.clear(); + m_pool = nullptr; + m_graphics = nullptr; +} + +bool CommandScheduler::InDeferredOperation() noexcept { + return g_deferred_callback_scheduler != nullptr; +} + +CommandScheduler::CommandScheduler(RenderContext& context, GraphicContext& graphics) + : m_master(graphics), m_context(context), m_graphics(graphics), + m_priority_thread([this](std::stop_token stop) { PriorityOperationsThread(stop); }) {} + +CommandScheduler::~CommandScheduler() { + Shutdown(); +} + +void CommandScheduler::Shutdown() { + { + std::unique_lock lock(m_operation_mutex); + if (m_operation_state == OperationState::Closed) { + return; + } + if (g_deferred_callback_scheduler == this) { + EXIT_IF(m_operation_state == OperationState::Open); + // A priority callback cannot join its own runner, while a normal callback can be + // executing inside the shutdown owner's final PopPendingOperations. The owning + // thread will finish shutdown after this callback returns. + return; + } + if (m_operation_state == OperationState::Draining) { + m_operation_available.wait( + lock, [this] { return m_operation_state == OperationState::Closed; }); + return; + } + m_operation_state = OperationState::Draining; + } + if (Active() && m_recording) { + Finish(); + } + DrainPriorityOperations(); + m_priority_thread.request_stop(); + m_operation_available.notify_all(); + if (m_priority_thread.joinable()) { + m_priority_thread.join(); + } + { + std::lock_guard lock(m_operation_mutex); + EXIT_IF(!m_pending_operations.empty() || !m_priority_operations.empty() || + m_priority_active); + m_operation_state = OperationState::Closed; + } + m_operation_available.notify_all(); +} + void CommandScheduler::Begin(HW::Context& registers, HW::UserConfig& user_config, HW::Shader& shaders) { + { + std::lock_guard lock(m_operation_mutex); + EXIT_IF(m_operation_state != OperationState::Open); + } m_registers = ®isters; m_user_config = &user_config; m_shaders = &shaders; if (!Active()) { for (auto& buffer: m_buffers) { - buffer = std::make_unique(); + buffer = std::make_unique(*this); } m_current = 0; } @@ -24,37 +167,201 @@ void CommandScheduler::Begin(HW::Context& registers, HW::UserConfig& user_config } } +void CommandScheduler::BeginRendering(const RenderState& state) { + Current().BeginRendering(state); +} + +void CommandScheduler::EndRendering() { + if (Active() && m_recording) { + Current().EndRendering(); + } +} + void CommandScheduler::Flush() { - SubmitCurrent(); + SubmitInfo submit; + Flush(submit); +} + +void CommandScheduler::Flush(SubmitInfo& submit) { + SubmitCurrent(submit); BeginNext(); } CommandBuffer& CommandScheduler::FlushAndGetSubmitted() { - auto& submitted = SubmitCurrent(); + SubmitInfo submit; + auto& submitted = SubmitCurrent(submit); BeginNext(); return submitted; } void CommandScheduler::Finish() { CheckActive(); + const auto tick = CurrentTick(); if (m_recording) { - SubmitCurrent(); + SubmitInfo submit; + SubmitCurrent(submit); } for (auto& buffer: m_buffers) { buffer->WaitForFenceAndReset(); } + m_master.Wait(tick); + PopPendingOperations(); BindCurrent(); Current().Begin(); m_recording = true; } void CommandScheduler::FinishCurrent() { - auto& submitted = SubmitCurrent(); + SubmitInfo submit; + auto& submitted = SubmitCurrent(submit); submitted.WaitForFenceAndReset(); + m_master.Refresh(); + PopPendingOperations(); submitted.Begin(); m_recording = true; } +void CommandScheduler::Wait(uint64_t tick) { + CheckActive(); + EXIT_IF(tick > CurrentTick()); + if (tick >= CurrentTick()) { + // A stream-buffer wrap can wait while a draw is being prepared through a reference to + // Current(). Recycle the same command object so that reference remains valid. + FinishCurrent(); + return; + } + m_master.Wait(tick); + PopPendingOperations(); +} + +void CommandScheduler::PopPendingOperations() { + m_master.Refresh(); + for (;;) { + Common::UniqueFunction callback; + { + std::lock_guard lock(m_operation_mutex); + if (m_pending_operations.empty() || + !m_master.IsFree(m_pending_operations.front().tick)) { + return; + } + callback = std::move(m_pending_operations.front().callback); + m_pending_operations.pop(); + } + RunOperation(std::move(callback)); + } +} + +void CommandScheduler::DeferOperation(Common::UniqueFunction&& operation) { + CheckActive(); + EXIT_IF(!operation); + std::unique_lock lock(m_operation_mutex); + if (m_operation_state == OperationState::Open) { + m_pending_operations.push({std::move(operation), CurrentTick()}); + return; + } + if (g_deferred_callback_scheduler == this) { + lock.unlock(); + operation(); + return; + } + m_operation_available.wait(lock, + [this] { return m_operation_state == OperationState::Closed; }); + lock.unlock(); + operation(); +} + +void CommandScheduler::DeferPriorityOperation(Common::UniqueFunction&& operation) { + CheckActive(); + EXIT_IF(!operation); + std::unique_lock lock(m_operation_mutex); + if (m_operation_state == OperationState::Open) { + m_priority_operations.push({std::move(operation), CurrentTick()}); + lock.unlock(); + m_operation_available.notify_one(); + return; + } + if (g_deferred_callback_scheduler == this) { + lock.unlock(); + operation(); + return; + } + m_operation_available.wait(lock, + [this] { return m_operation_state == OperationState::Closed; }); + lock.unlock(); + operation(); +} + +void CommandScheduler::PriorityOperationsThread(std::stop_token stop) { + while (!stop.stop_requested()) { + PendingOperation operation; + { + std::unique_lock lock(m_operation_mutex); + m_operation_available.wait(lock, [this, &stop] { + return stop.stop_requested() || !m_priority_operations.empty(); + }); + if (stop.stop_requested()) { + return; + } + operation = std::move(m_priority_operations.front()); + m_priority_operations.pop(); + m_priority_active = true; + m_priority_active_tick = operation.tick; + } + m_master.Wait(operation.tick); + if (!stop.stop_requested()) { + RunOperation(std::move(operation.callback)); + } + { + std::lock_guard lock(m_operation_mutex); + m_priority_active = false; + m_priority_active_tick = 0; + } + m_operation_available.notify_all(); + } +} + +void CommandScheduler::DrainPriorityOperations() { + EXIT_IF(g_deferred_callback_scheduler == this); + std::unique_lock lock(m_operation_mutex); + m_operation_available.wait( + lock, [this] { return m_priority_operations.empty() && !m_priority_active; }); +} + +void CommandScheduler::WaitPriorityOperations(uint64_t tick) { + EXIT_IF(g_deferred_callback_scheduler == this); + std::unique_lock lock(m_operation_mutex); + m_operation_available.wait(lock, [this, tick] { + const bool active_before_or_at = + m_priority_active && m_priority_active_tick <= tick; + const bool queued_before_or_at = + !m_priority_operations.empty() && m_priority_operations.front().tick <= tick; + return !active_before_or_at && !queued_before_or_at; + }); +} + +void CommandScheduler::RunOperation(Common::UniqueFunction&& operation) { + auto* previous = g_deferred_callback_scheduler; + g_deferred_callback_scheduler = this; + operation(); + g_deferred_callback_scheduler = previous; +} + +bool CommandScheduler::IsFree(uint64_t tick) { + if (m_master.IsFree(tick)) { + return true; + } + m_master.Refresh(); + return m_master.IsFree(tick); +} + +CommandSlot* CommandScheduler::AllocateCommandBuffer() { + return m_command_pool.Allocate(m_graphics); +} + +uint64_t CommandScheduler::NextSubmitSequence() noexcept { + return m_submit_sequence.fetch_add(1, std::memory_order_relaxed) + 1; +} + void CommandScheduler::CheckActive() const { EXIT_IF(!Active() || m_current >= BufferCount); } @@ -70,12 +377,14 @@ void CommandScheduler::BindCurrent() const { Current().Bind(*m_registers, *m_user_config, *m_shaders); } -CommandBuffer& CommandScheduler::SubmitCurrent() { +CommandBuffer& CommandScheduler::SubmitCurrent(SubmitInfo& submit) { CheckActive(); EXIT_IF(!m_recording); auto& submitted = Current(); submitted.End(); - submitted.Execute(); + const auto signal_tick = m_master.NextTick(); + submit.AddSignal(m_master.Handle(), signal_tick); + submitted.Execute(submit); m_recording = false; return submitted; } @@ -84,6 +393,7 @@ void CommandScheduler::BeginNext() { EXIT_IF(m_recording); m_current = (m_current + 1) % BufferCount; Current().WaitForFenceAndReset(); + PopPendingOperations(); BindCurrent(); Current().Begin(); m_recording = true; diff --git a/src/graphics/host_gpu/renderer/commandScheduler.h b/src/graphics/host_gpu/renderer/commandScheduler.h index 3dc2add9..79b75e54 100644 --- a/src/graphics/host_gpu/renderer/commandScheduler.h +++ b/src/graphics/host_gpu/renderer/commandScheduler.h @@ -2,42 +2,124 @@ #define EMULATOR_SRC_GRAPHICS_HOST_GPU_RENDERER_COMMANDSCHEDULER_H_ #include "common/common.h" +#include "common/uniqueFunction.h" +#include "graphics/host_gpu/renderer/masterSemaphore.h" #include "graphics/host_gpu/renderer/render.h" #include +#include +#include +#include #include +#include + +#include + +#include namespace Libs::Graphics { +struct CommandSlot { + Common::Mutex* pool_mutex = nullptr; + uint32_t id = 0; + vk::CommandBuffer buffer = nullptr; + vk::Fence fence = nullptr; + bool busy = false; + + void Reset(); +}; + class CommandScheduler { public: static constexpr int BufferCount = 8; - CommandScheduler() = default; - ~CommandScheduler() = default; + CommandScheduler(RenderContext& context, GraphicContext& graphics); + ~CommandScheduler(); KYTY_CLASS_NO_COPY(CommandScheduler); void Begin(HW::Context& registers, HW::UserConfig& user_config, HW::Shader& shaders); + void BeginRendering(const RenderState& state); + void EndRendering(); void Flush(); + void Flush(SubmitInfo& submit); CommandBuffer& FlushAndGetSubmitted(); void Finish(); void FinishCurrent(); + // Deferred callbacks can observe an externally owned drain, but cannot initiate shutdown: + // the priority runner cannot join itself. + void Shutdown(); + void Wait(uint64_t tick); + void PopPendingOperations(); + void DrainPriorityOperations(); + void WaitPriorityOperations(uint64_t tick); + void DeferOperation(Common::UniqueFunction&& operation); + void DeferPriorityOperation(Common::UniqueFunction&& operation); + [[nodiscard]] static bool InDeferredOperation() noexcept; - [[nodiscard]] bool Active() const noexcept { return m_current >= 0; } - void CheckActive() const; - RenderCommandBuffer& Current() const; + [[nodiscard]] bool Active() const noexcept { return m_current >= 0; } + void CheckActive() const; + RenderCommandBuffer& Current() const; + [[nodiscard]] uint64_t CurrentTick() const noexcept { return m_master.CurrentTick(); } + [[nodiscard]] bool IsFree(uint64_t tick); + [[nodiscard]] RenderContext& Context() const noexcept { return m_context; } + [[nodiscard]] GraphicContext& Graphics() const noexcept { return m_graphics; } private: - void BindCurrent() const; - CommandBuffer& SubmitCurrent(); - void BeginNext(); + class CommandPool { + public: + CommandPool() = default; + ~CommandPool(); + KYTY_CLASS_NO_COPY(CommandPool); + CommandSlot* Allocate(GraphicContext& graphics); + + private: + void Create(GraphicContext& graphics); + CommandSlot* CreateSlot(); + void Destroy(); + + GraphicContext* m_graphics = nullptr; + Common::Mutex m_mutex; + vk::CommandPool m_pool = nullptr; + std::deque m_slots; + }; + + enum class OperationState { Open, Draining, Closed }; + + struct PendingOperation { + Common::UniqueFunction callback; + uint64_t tick = 0; + }; + + void BindCurrent() const; + CommandBuffer& SubmitCurrent(SubmitInfo& submit); + void BeginNext(); + void PriorityOperationsThread(std::stop_token stop); + void RunOperation(Common::UniqueFunction&& operation); + [[nodiscard]] CommandSlot* AllocateCommandBuffer(); + [[nodiscard]] uint64_t NextSubmitSequence() noexcept; + + MasterSemaphore m_master; + RenderContext& m_context; + GraphicContext& m_graphics; + CommandPool m_command_pool; std::array, BufferCount> m_buffers; - int m_current = -1; - bool m_recording = false; - HW::Context* m_registers = nullptr; - HW::UserConfig* m_user_config = nullptr; - HW::Shader* m_shaders = nullptr; + std::queue m_pending_operations; + std::queue m_priority_operations; + std::mutex m_operation_mutex; + std::condition_variable m_operation_available; + std::jthread m_priority_thread; + bool m_priority_active = false; + uint64_t m_priority_active_tick = 0; + OperationState m_operation_state = OperationState::Open; + int m_current = -1; + bool m_recording = false; + HW::Context* m_registers = nullptr; + HW::UserConfig* m_user_config = nullptr; + HW::Shader* m_shaders = nullptr; + std::atomic m_submit_sequence = 0; + + friend class CommandBuffer; }; } // namespace Libs::Graphics diff --git a/src/graphics/host_gpu/renderer/context.cpp b/src/graphics/host_gpu/renderer/context.cpp index b28e8a34..0648641e 100644 --- a/src/graphics/host_gpu/renderer/context.cpp +++ b/src/graphics/host_gpu/renderer/context.cpp @@ -9,63 +9,17 @@ #include "graphics/host_gpu/renderer/debug.h" #include "graphics/host_gpu/renderer/depthRenderTarget.h" #include "graphics/host_gpu/renderer/descriptorCache.h" -#include "graphics/host_gpu/renderer/framebufferCache.h" #include "graphics/host_gpu/renderer/imageView.h" #include "graphics/host_gpu/renderer/render.h" #include "graphics/host_gpu/renderer/renderContext.h" -#include "graphics/host_gpu/transfer.h" #include "graphics/host_gpu/vma.h" #include "graphics/host_gpu/vulkanCommon.h" #include -#include +#include #include -#include #include namespace Libs::Graphics { -static std::atomic g_command_buffer_submit_seq = 0; - -static void ResetNativeCommandBuffer(vk::CommandBuffer buffer) { - EXIT_IF(buffer == nullptr); - const auto result = buffer.reset(vk::CommandBufferResetFlagBits::eReleaseResources); - if (result != vk::Result::eSuccess) { - EXIT("failed to reset Vulkan command buffer: %s (%d)\n", VulkanToString(result).c_str(), - static_cast(result)); - } -} - -struct CommandSlot { - Common::Mutex* pool_mutex = nullptr; - uint32_t id = 0; - vk::CommandBuffer buffer = nullptr; - vk::Fence fence = nullptr; - bool busy = false; -}; - -class ThreadCommandPool { -public: - ThreadCommandPool() = default; - - KYTY_CLASS_NO_COPY(ThreadCommandPool); - - CommandSlot* Allocate(); - void Destroy(); - -private: - void Create(); - CommandSlot* CreateSlot(); - - Common::Mutex m_mutex; - vk::CommandPool m_pool = nullptr; - std::deque m_slots; -}; - -static RenderContext* g_render_ctx = nullptr; -static thread_local ThreadCommandPool g_command_pool; - -RenderContext& GetRenderContext() noexcept { - return *g_render_ctx; -} FenceResourceRetainer::~FenceResourceRetainer() { if (!m_resources.empty()) { @@ -88,93 +42,14 @@ void FenceResourceRetainer::ReleaseAfterFence() noexcept { m_resources.clear(); } -void GraphicsRenderInit(GraphicContext& graphics) { - g_render_ctx = new RenderContext(graphics); -} - -void GraphicsRenderReleaseThreadCommandPool() { - g_command_pool.Destroy(); -} - -CommandBuffer::CommandBuffer() - : m_graphics(GetRenderContext().GetGraphics()), m_slot(g_command_pool.Allocate()), - m_host_stream(m_graphics) {} +CommandBuffer::CommandBuffer(CommandScheduler& scheduler) + : m_context(scheduler.Context()), m_scheduler(scheduler), m_graphics(scheduler.Graphics()), + m_slot(scheduler.AllocateCommandBuffer()) {} CommandBuffer::~CommandBuffer() { Release(); } -void ThreadCommandPool::Create() { - auto& graphics = GetRenderContext().GetGraphics(); - EXIT_IF(m_pool != nullptr || graphics.queue_family == static_cast(-1)); - - vk::CommandPoolCreateInfo pool_info {}; - pool_info.sType = vk::StructureType::eCommandPoolCreateInfo; - pool_info.pNext = nullptr; - pool_info.queueFamilyIndex = graphics.queue_family; - pool_info.flags = vk::CommandPoolCreateFlagBits::eResetCommandBuffer; - - const auto result = graphics.device.createCommandPool(&pool_info, nullptr, &m_pool); - EXIT_NOT_IMPLEMENTED(result != vk::Result::eSuccess || m_pool == nullptr); -} - -CommandSlot* ThreadCommandPool::CreateSlot() { - auto& graphics = GetRenderContext().GetGraphics(); - vk::CommandBufferAllocateInfo alloc_info {}; - alloc_info.sType = vk::StructureType::eCommandBufferAllocateInfo; - alloc_info.commandPool = m_pool; - alloc_info.level = vk::CommandBufferLevel::ePrimary; - alloc_info.commandBufferCount = 1; - - vk::CommandBuffer buffer = nullptr; - if (graphics.device.allocateCommandBuffers(&alloc_info, &buffer) != vk::Result::eSuccess) { - EXIT("Can't allocate command buffers"); - } - - vk::FenceCreateInfo fence_info {}; - fence_info.sType = vk::StructureType::eFenceCreateInfo; - fence_info.flags = vk::FenceCreateFlagBits::eSignaled; - - vk::Fence fence = nullptr; - if (graphics.device.createFence(&fence_info, nullptr, &fence) != vk::Result::eSuccess) { - graphics.device.freeCommandBuffers(m_pool, 1, &buffer); - EXIT("Can't create fence"); - } - auto& slot = m_slots.emplace_back(); - slot.pool_mutex = &m_mutex; - slot.id = static_cast(m_slots.size() - 1); - slot.buffer = buffer; - slot.fence = fence; - return &slot; -} - -CommandSlot* ThreadCommandPool::Allocate() { - Common::LockGuard lock(m_mutex); - if (m_pool == nullptr) { - Create(); - } - auto it = std::ranges::find_if(m_slots, [](const auto& slot) { return !slot.busy; }); - auto* slot = it != m_slots.end() ? &*it : CreateSlot(); - slot->busy = true; - ResetNativeCommandBuffer(slot->buffer); - return slot; -} - -void ThreadCommandPool::Destroy() { - Common::LockGuard lock(m_mutex); - if (m_pool == nullptr) { - return; - } - EXIT_IF(std::ranges::any_of(m_slots, [](const auto& slot) { return slot.busy; })); - auto& graphics = GetRenderContext().GetGraphics(); - for (const auto& slot: m_slots) { - graphics.device.destroyFence(slot.fence, nullptr); - } - graphics.device.destroyCommandPool(m_pool, nullptr); - m_slots.clear(); - m_pool = nullptr; -} - bool CommandBuffer::IsInvalid() const { return m_slot == nullptr; } @@ -194,10 +69,8 @@ void CommandBuffer::Release() { WaitForFence(); - m_host_stream.Release(); - m_slot->busy = false; - ResetNativeCommandBuffer(m_slot->buffer); + m_slot->Reset(); ReleaseResourcesAfterFence(); m_slot = nullptr; @@ -224,12 +97,13 @@ void CommandBuffer::RecycleDescriptorAfterFence(VulkanDescriptorSet& set) { void CommandBuffer::RecycleDescriptorsAfterFence() { for (auto* set: m_descriptor_sets_after_fence) { - GetRenderContext().GetDescriptorCache().Recycle(*set); + m_context.GetDescriptorCache().Recycle(*set); } m_descriptor_sets_after_fence.clear(); } void CommandBuffer::Begin() const { + EXIT_IF(m_rendering); auto buffer = Handle(); vk::CommandBufferBeginInfo begin_info {}; @@ -244,6 +118,7 @@ void CommandBuffer::Begin() const { } void CommandBuffer::End() const { + EndRendering(); auto buffer = Handle(); auto result = buffer.end(); @@ -262,39 +137,34 @@ void CommandBuffer::SetDebugInfo(uint32_t op, uint64_t submit_id, uint32_t arg0, m_debug_arg4 = arg4; } -void CommandBuffer::Execute() { - Submit(nullptr, {}, nullptr); -} - -void CommandBuffer::ExecuteWithSemaphore(vk::Semaphore wait_semaphore, - vk::PipelineStageFlags wait_stage, - vk::Semaphore signal_semaphore) { - EXIT_IF(wait_semaphore == nullptr || signal_semaphore == nullptr); - Submit(wait_semaphore, wait_stage, signal_semaphore); -} - -void CommandBuffer::Submit(vk::Semaphore wait_semaphore, vk::PipelineStageFlags wait_stage, - vk::Semaphore signal_semaphore) { +void CommandBuffer::Execute(const SubmitInfo& submit) { EXIT_IF(IsInvalid()); EXIT_IF(m_execute); + EXIT_IF(submit.num_wait_semaphores > SubmitInfo::MaxSemaphores || + submit.num_signal_semaphores > SubmitInfo::MaxSemaphores); - const bool has_wait = wait_semaphore != nullptr; - const bool has_signal = signal_semaphore != nullptr; - auto buffer = Handle(); - auto fence = m_slot->fence; + auto buffer = Handle(); + auto fence = m_slot->fence; + + vk::TimelineSemaphoreSubmitInfo timeline_info {}; + timeline_info.sType = vk::StructureType::eTimelineSemaphoreSubmitInfo; + timeline_info.waitSemaphoreValueCount = submit.num_wait_semaphores; + timeline_info.pWaitSemaphoreValues = submit.wait_ticks.data(); + timeline_info.signalSemaphoreValueCount = submit.num_signal_semaphores; + timeline_info.pSignalSemaphoreValues = submit.signal_ticks.data(); vk::SubmitInfo submit_info {}; submit_info.sType = vk::StructureType::eSubmitInfo; - submit_info.pNext = nullptr; - submit_info.waitSemaphoreCount = has_wait ? 1u : 0u; - submit_info.pWaitSemaphores = has_wait ? &wait_semaphore : nullptr; - submit_info.pWaitDstStageMask = has_wait ? &wait_stage : nullptr; + submit_info.pNext = &timeline_info; + submit_info.waitSemaphoreCount = submit.num_wait_semaphores; + submit_info.pWaitSemaphores = submit.wait_semaphores.data(); + submit_info.pWaitDstStageMask = submit.wait_stages.data(); submit_info.commandBufferCount = 1; submit_info.pCommandBuffers = &buffer; - submit_info.signalSemaphoreCount = has_signal ? 1u : 0u; - submit_info.pSignalSemaphores = has_signal ? &signal_semaphore : nullptr; + submit_info.signalSemaphoreCount = submit.num_signal_semaphores; + submit_info.pSignalSemaphores = submit.signal_semaphores.data(); - auto& graphics = GetRenderContext().GetGraphics(); + auto& graphics = m_graphics; EXIT_IF(graphics.queue == nullptr); auto result = graphics.device.resetFences(1, &fence); @@ -305,16 +175,16 @@ void CommandBuffer::Submit(vk::Semaphore wait_semaphore, vk::PipelineStageFlags EXIT_NOT_IMPLEMENTED(result != vk::Result::eSuccess); if (Config::GraphicsDebugDumpEnabled()) { - LOGF("vkQueueSubmit begin: slot=%u wait_semaphore=%p signal_semaphore=%p" - " debug_op=%u debug_submit=%" PRIu64 " args=%u,%u,%u,%u,0x%016" PRIx64 "\n", - m_slot->id, static_cast(wait_semaphore), static_cast(signal_semaphore), - m_debug_op, m_debug_submit_id, m_debug_arg0, m_debug_arg1, m_debug_arg2, m_debug_arg3, + LOGF("vkQueueSubmit begin: slot=%u waits=%u signals=%u debug_op=%u debug_submit=%" PRIu64 + " args=%u,%u,%u,%u,0x%016" PRIx64 "\n", + m_slot->id, submit.num_wait_semaphores, submit.num_signal_semaphores, m_debug_op, + m_debug_submit_id, m_debug_arg0, m_debug_arg1, m_debug_arg2, m_debug_arg3, m_debug_arg4); } { Common::LockGuard lock(graphics.queue_mutex); - m_submit_seq = g_command_buffer_submit_seq.fetch_add(1, std::memory_order_relaxed) + 1; + m_submit_seq = m_scheduler.NextSubmitSequence(); result = graphics.queue.submit(1, &submit_info, fence); } @@ -340,7 +210,7 @@ void CommandBuffer::WaitForFenceOnly() { if (!m_execute || m_fence_waited) { return; } - auto device = GetRenderContext().GetGraphics().device; + auto device = m_graphics.device; auto result = device.waitForFences(1, &m_slot->fence, VK_TRUE, UINT64_MAX); if (result != vk::Result::eSuccess) { LOGF("vkWaitForFences failed: %s (%d), slot=%u submit_seq=%" PRIu64 @@ -364,12 +234,10 @@ void CommandBuffer::FinalizeFence(bool reset_recording) { m_execute = false; m_fence_waited = false; if (reset_recording) { - ResetNativeCommandBuffer(m_slot->buffer); + Common::LockGuard lock(*m_slot->pool_mutex); + m_slot->Reset(); } } - if (reset_recording) { - m_host_stream.Reset(); - } if (was_executed) { ReleaseResourcesAfterFence(); } @@ -383,142 +251,70 @@ void CommandBuffer::ReleaseResourcesAfterFence() { void CommandBuffer::DeleteBuffersAfterFence() { for (const auto& buffer: m_retired_buffers) { - GetRenderContext().GetGraphics().DeleteBuffer(*buffer); + m_graphics.DeleteBuffer(*buffer); } m_retired_buffers.clear(); } -void CommandBuffer::BeginRenderPass(VulkanFramebuffer& framebuffer, RenderColorInfo* colors, - uint32_t requested_color_count, RenderDepthInfo& depth) const { - auto buffer = Handle(); - - EXIT_IF(colors == nullptr); - EXIT_IF(requested_color_count > RENDER_COLOR_ATTACHMENTS_MAX); - - bool with_depth = (depth.format != vk::Format::eUndefined && depth.vulkan_buffer != nullptr); - uint32_t color_count = 0; - for (uint32_t i = 0; i < requested_color_count; i++) { - if (colors[i].vulkan_buffer == nullptr) { - break; - } - color_count++; +void CommandBuffer::BeginRendering(const RenderState& state) const { + EXIT_IF(state.width == 0 || state.height == 0 || state.num_layers == 0 || + state.num_color_attachments > RENDER_COLOR_ATTACHMENTS_MAX); + if (m_rendering && m_render_state == state) { + return; } - bool with_color = (color_count != 0); + EndRendering(); - EXIT_NOT_IMPLEMENTED(!with_depth && !with_color); - - vk::ClearValue clears[RENDER_COLOR_ATTACHMENTS_MAX + 1] = {}; - for (uint32_t i = 0; i < color_count; i++) { - clears[i].color = colors[i].color_clear_value; - } - clears[color_count].depthStencil = {depth.depth_clear_value, depth.stencil_clear_value}; - - vk::Extent2D extent = (with_color ? colors[0].extent : depth.vulkan_buffer->extent); - - vk::RenderPassBeginInfo render_pass_info {}; - render_pass_info.sType = vk::StructureType::eRenderPassBeginInfo; - render_pass_info.pNext = nullptr; - render_pass_info.renderPass = framebuffer.render_pass; - render_pass_info.framebuffer = framebuffer.framebuffer; - render_pass_info.renderArea.offset = {0, 0}; - render_pass_info.renderArea.extent = extent; - render_pass_info.clearValueCount = color_count + (with_depth ? 1u : 0u); - render_pass_info.pClearValues = clears; - - for (uint32_t i = 0; i < color_count; i++) { - const auto color_initial_layout = framebuffer.color_layout[i]; - if (colors[i].vulkan_buffer->layout != color_initial_layout) { - if (graphics_debug_dump_enabled()) { - LOGF("BeginRenderPass: color%u initial barrier image=%p mem=%" PRIu64 " %s -> %s\n", - i, VulkanHandleToPointer(colors[i].vulkan_buffer->image), - colors[i].vulkan_buffer->memory.unique_id, - VulkanToString(colors[i].vulkan_buffer->layout).c_str(), - VulkanToString(color_initial_layout).c_str()); - } - - vk::ImageMemoryBarrier image_memory_barrier {}; - image_memory_barrier.sType = vk::StructureType::eImageMemoryBarrier; - image_memory_barrier.pNext = nullptr; - image_memory_barrier.srcAccessMask = {}; - image_memory_barrier.dstAccessMask = vk::AccessFlagBits::eColorAttachmentRead | - vk::AccessFlagBits::eColorAttachmentWrite; - image_memory_barrier.oldLayout = colors[i].vulkan_buffer->layout; - image_memory_barrier.newLayout = color_initial_layout; - image_memory_barrier.srcQueueFamilyIndex = VK_QUEUE_FAMILY_IGNORED; - image_memory_barrier.dstQueueFamilyIndex = VK_QUEUE_FAMILY_IGNORED; - image_memory_barrier.image = colors[i].vulkan_buffer->image; - image_memory_barrier.subresourceRange.aspectMask = vk::ImageAspectFlagBits::eColor; - image_memory_barrier.subresourceRange.baseMipLevel = 0; - image_memory_barrier.subresourceRange.levelCount = VK_REMAINING_MIP_LEVELS; - image_memory_barrier.subresourceRange.baseArrayLayer = 0; - image_memory_barrier.subresourceRange.layerCount = colors[i].vulkan_buffer->layers; - - buffer.pipelineBarrier(vk::PipelineStageFlagBits::eTopOfPipe, - vk::PipelineStageFlagBits::eColorAttachmentOutput, - vk::DependencyFlags {}, 0, nullptr, 0, nullptr, 1, - &image_memory_barrier); - - colors[i].vulkan_buffer->layout = image_memory_barrier.newLayout; - } else if (graphics_debug_dump_enabled()) { - LOGF("BeginRenderPass: color%u initial image=%p mem=%" PRIu64 " layout=%s\n", i, - VulkanHandleToPointer(colors[i].vulkan_buffer->image), - colors[i].vulkan_buffer->memory.unique_id, - VulkanToString(colors[i].vulkan_buffer->layout).c_str()); - } + std::array colors {}; + for (uint32_t i = 0; i < state.num_color_attachments; i++) { + const auto& attachment = state.color_attachments[i]; + colors[i].sType = vk::StructureType::eRenderingAttachmentInfo; + colors[i].imageView = attachment.image_view; + colors[i].imageLayout = attachment.image_layout; + colors[i].loadOp = attachment.is_clear ? vk::AttachmentLoadOp::eClear + : vk::AttachmentLoadOp::eLoad; + colors[i].storeOp = vk::AttachmentStoreOp::eStore; + colors[i].clearValue.color.uint32 = attachment.clear_value; } - const auto depth_layout = - with_depth ? framebuffer.depth_layout : vk::ImageLayout::eDepthStencilAttachmentOptimal; + const auto& depth_stencil = state.depth_stencil_attachment; + vk::RenderingAttachmentInfo depth {}; + depth.sType = vk::StructureType::eRenderingAttachmentInfo; + depth.imageView = depth_stencil.image_view; + depth.imageLayout = depth_stencil.image_layout; + depth.loadOp = depth_stencil.depth_clear ? vk::AttachmentLoadOp::eClear + : vk::AttachmentLoadOp::eLoad; + depth.storeOp = vk::AttachmentStoreOp::eStore; + depth.clearValue.depthStencil.depth = std::bit_cast(depth_stencil.clear_value[0]); - if (with_depth && depth.vulkan_buffer->layout != depth_layout) { - vk::ImageMemoryBarrier image_memory_barrier {}; - image_memory_barrier.sType = vk::StructureType::eImageMemoryBarrier; - image_memory_barrier.pNext = nullptr; - image_memory_barrier.srcAccessMask = - vk::AccessFlagBits::eMemoryRead | vk::AccessFlagBits::eMemoryWrite; - image_memory_barrier.dstAccessMask = - (depth_layout == vk::ImageLayout::eDepthStencilReadOnlyOptimal - ? vk::AccessFlagBits::eMemoryRead - : vk::AccessFlagBits::eMemoryWrite); - image_memory_barrier.oldLayout = depth.vulkan_buffer->layout; - image_memory_barrier.newLayout = depth_layout; - image_memory_barrier.srcQueueFamilyIndex = VK_QUEUE_FAMILY_IGNORED; - image_memory_barrier.dstQueueFamilyIndex = VK_QUEUE_FAMILY_IGNORED; - image_memory_barrier.image = depth.vulkan_buffer->image; - image_memory_barrier.subresourceRange.aspectMask = - ImageViewOps::DepthAspectMask(depth.vulkan_buffer->format); - image_memory_barrier.subresourceRange.baseMipLevel = 0; - image_memory_barrier.subresourceRange.levelCount = 1; - image_memory_barrier.subresourceRange.baseArrayLayer = 0; - image_memory_barrier.subresourceRange.layerCount = depth.vulkan_buffer->layers; + vk::RenderingAttachmentInfo stencil {}; + stencil.sType = vk::StructureType::eRenderingAttachmentInfo; + stencil.imageView = depth_stencil.image_view; + stencil.imageLayout = depth_stencil.image_layout; + stencil.loadOp = depth_stencil.stencil_clear ? vk::AttachmentLoadOp::eClear + : vk::AttachmentLoadOp::eLoad; + stencil.storeOp = vk::AttachmentStoreOp::eStore; + stencil.clearValue.depthStencil.stencil = depth_stencil.clear_value[1]; - buffer.pipelineBarrier( - vk::PipelineStageFlagBits::eAllGraphics | vk::PipelineStageFlagBits::eComputeShader, - vk::PipelineStageFlagBits::eAllGraphics | vk::PipelineStageFlagBits::eComputeShader, - vk::DependencyFlags {}, 0, nullptr, 0, nullptr, 1, &image_memory_barrier); - - depth.vulkan_buffer->layout = image_memory_barrier.newLayout; - } - - buffer.beginRenderPass(&render_pass_info, vk::SubpassContents::eInline); - - for (uint32_t i = 0; i < color_count; i++) { - colors[i].vulkan_buffer->layout = RENDER_COLOR_IMAGE_LAYOUT; - if (colors[i].vulkan_buffer->type == VulkanImageType::RenderTexture) { - static_cast(colors[i].vulkan_buffer)->initial_clear_pending = - false; - } - } - if (with_depth) { - depth.vulkan_buffer->initial_depth_clear_pending = false; - depth.vulkan_buffer->initial_stencil_clear_pending = false; - } + vk::RenderingInfo rendering {}; + rendering.sType = vk::StructureType::eRenderingInfo; + rendering.renderArea.extent = {state.width, state.height}; + rendering.layerCount = state.num_layers; + rendering.colorAttachmentCount = state.num_color_attachments; + rendering.pColorAttachments = colors.data(); + rendering.pDepthAttachment = depth_stencil.has_depth ? &depth : nullptr; + rendering.pStencilAttachment = depth_stencil.has_stencil ? &stencil : nullptr; + Handle().beginRendering(rendering); + m_render_state = state; + m_rendering = true; } -void CommandBuffer::EndRenderPass() const { - auto buffer = Handle(); - - buffer.endRenderPass(); +void CommandBuffer::EndRendering() const { + if (!m_rendering) { + return; + } + Handle().endRendering(); + m_rendering = false; + m_render_state = {}; } } // namespace Libs::Graphics diff --git a/src/graphics/host_gpu/renderer/depthRenderTarget.cpp b/src/graphics/host_gpu/renderer/depthRenderTarget.cpp index f1ab6b3b..c3d556e9 100644 --- a/src/graphics/host_gpu/renderer/depthRenderTarget.cpp +++ b/src/graphics/host_gpu/renderer/depthRenderTarget.cpp @@ -13,12 +13,10 @@ #include "graphics/host_gpu/objects/textureCommon.h" #include "graphics/host_gpu/renderer/debug.h" #include "graphics/host_gpu/renderer/descriptorCache.h" -#include "graphics/host_gpu/renderer/framebufferCache.h" +#include "graphics/host_gpu/renderer/imageView.h" #include "graphics/host_gpu/renderer/render.h" #include "graphics/host_gpu/renderer/renderContext.h" -#include "graphics/host_gpu/transfer.h" #include "graphics/host_gpu/vulkanCommon.h" -#include "graphics/presentation/displayBuffer.h" #include #include @@ -88,7 +86,8 @@ static bool UsesStencilOpValue(uint8_t fail, uint8_t pass, uint8_t depth_fail) { } // NOLINTNEXTLINE(readability-function-cognitive-complexity) -void ResolveRenderDepthTarget(uint64_t submit_id, RenderCommandBuffer& buffer, RenderDepthInfo& r) { +void RenderExecutor::ResolveRenderDepthTarget(uint64_t submit_id, RenderCommandBuffer& buffer, + RenderDepthInfo& r) { KYTY_PROFILER_FUNCTION(); (void)submit_id; const auto& hw = buffer.GetRegisters(); @@ -282,18 +281,9 @@ void ResolveRenderDepthTarget(uint64_t submit_id, RenderCommandBuffer& buffer, R r.stencil_buffer_vaddr = has_stencil ? z.stencil_read_base_addr : 0; r.htile_buffer_size = has_htile ? htile_backing_size : 0; r.htile_buffer_vaddr = has_htile ? z.htile_data_base_addr : 0; - auto& cache = GetRenderContext().GetTextureCache(); - if (has_htile) { - cache.RegisterMeta(r.htile_buffer_vaddr, r.htile_buffer_size, view.image_layers); - } - if (has_htile && rc.depth_clear_enable && !cache.ClearMeta(z.htile_data_base_addr)) { - DepthFatal("failed to acquire HTile metadata for a depth clear"); - } - const bool meta_clear = - has_htile && cache.IsMetaCleared(z.htile_data_base_addr, z.depth_view.slice_start); r.depth_clear_enable = rc.depth_clear_enable; - r.depth_meta_clear_enable = meta_clear; - r.depth_load_clear_enable = r.depth_clear_enable || r.depth_meta_clear_enable; + r.depth_meta_clear_enable = false; + r.depth_load_clear_enable = r.depth_clear_enable; r.depth_clear_value = hw.GetDepthClearValue(); r.depth_test_enable = dc.z_enable; r.depth_write_enable = dc.z_write_enable && !z.depth_view.depth_write_disable; @@ -343,55 +333,94 @@ void ResolveRenderDepthTarget(uint64_t submit_id, RenderCommandBuffer& buffer, R r.vaddr[1] = r.stencil_buffer_vaddr; r.size[1] = r.stencil_buffer_size; } - DepthTargetInfo info {}; - info.address = r.depth_buffer_vaddr; - info.size = r.depth_buffer_size; - info.stencil_address = r.stencil_buffer_vaddr; - info.stencil_size = r.stencil_buffer_size; - info.htile_address = r.htile_buffer_vaddr; - info.htile_size = r.htile_buffer_size; - info.format = r.format; - info.guest_format = guest_format; - info.width = width; - info.height = height; - info.pitch = pitch; - info.bytes_per_element = bytes; - info.tile_mode = Prospero::GpuEnumValue(Prospero::TileMode::kDepth); - info.layers = view.image_layers; - info.samples = samples; - info.depth_load_clear = r.depth_load_clear_enable; - info.depth_access = depth_active; - info.stencil_load_clear = rc.stencil_clear_enable; - info.stencil_access = - r.stencil_clear_enable || - (r.stencil_test_enable && - (stencil_face_accesses_attachment(r.stencil_static_front, r.stencil_dynamic_front) || - stencil_face_accesses_attachment(r.stencil_static_back, r.stencil_dynamic_back))); - info.stencil_htile_compressed = + TextureCache::ImageDesc desc {}; + desc.type = TextureCache::BindingType::DepthTarget; + desc.info.data = {r.depth_buffer_vaddr, r.depth_buffer_size}; + desc.info.stencil = {r.stencil_buffer_vaddr, r.stencil_buffer_size}; + desc.info.pixel_format = r.format; + desc.info.guest_format = guest_format; + desc.info.type = Prospero::ImageType::kColor2D; + desc.info.extent = {width, height, 1}; + desc.info.resources = {1, view.image_layers}; + desc.info.pitch = pitch; + desc.info.bytes_per_block = bytes; + desc.info.samples = samples; + desc.info.tile_mode = Prospero::GpuEnumValue(Prospero::TileMode::kDepth); + desc.info.mip_layout[0] = {0, r.depth_buffer_size, pitch, height}; + desc.info.metadata.range = {r.htile_buffer_vaddr, r.htile_buffer_size}; + desc.info.metadata.kind = has_htile ? ImageMetadataKind::Htile : ImageMetadataKind::None; + desc.info.metadata.stencil_compressed = has_stencil && has_htile && !z.stencil_info.tile_stencil_disable; - r.vulkan_buffer = &cache.FindDepthTarget(buffer, info); - r.vulkan_view = - cache.GetDepthTargetAttachmentView(*r.vulkan_buffer, view.base_layer, view.layer_count); - if (r.vulkan_buffer->initial_depth_clear_pending) { - r.depth_load_clear_enable = true; - r.depth_clear_value = 0.0f; - } - if (r.vulkan_buffer->initial_stencil_clear_pending) { - r.stencil_clear_enable = true; - r.stencil_clear_value = 0; - } - if (meta_clear && !cache.TouchMeta(z.htile_data_base_addr, z.depth_view.slice_start, false)) { - DepthFatal("failed to consume HTile clear state"); - } + desc.view_info.format = r.format; + desc.view_info.type = + view.layer_count == 1 ? vk::ImageViewType::e2D : vk::ImageViewType::e2DArray; + desc.view_info.aspect = ImageViewOps::DepthAspectMask(r.format); + desc.view_info.base_level = 0; + desc.view_info.level_count = 1; + desc.view_info.base_layer = view.base_layer; + desc.view_info.layer_count = view.layer_count; + desc.view_info.usage = vk::ImageUsageFlagBits::eDepthStencilAttachment; + r.desc = std::move(desc); + auto& cache = m_context.GetTextureCache(); + r.image_id = cache.FindImage(r.desc); + r.image_view = nullptr; + BindRenderTarget(r.image_id); } -void MarkRenderTargetGpuWritten(const RenderDepthInfo& target) { - const bool with_depth = - target.format != vk::Format::eUndefined && target.vulkan_buffer != nullptr; - - if (with_depth && !depth_attachment_read_only(target)) { - GetRenderContext().GetTextureCache().MarkGpuWritten(*target.vulkan_buffer); +vk::ImageAspectFlags RenderDepthInfo::AttachmentWriteAspects() const { + if (format == vk::Format::eUndefined) { + return {}; } + + const auto available = ImageViewOps::DepthAspectMask(format); + vk::ImageAspectFlags writes {}; + if ((available & vk::ImageAspectFlagBits::eDepth) && + (depth_load_clear_enable || (depth_test_enable && depth_write_enable))) { + writes |= vk::ImageAspectFlagBits::eDepth; + } + if (!(available & vk::ImageAspectFlagBits::eStencil)) { + return writes; + } + + const auto face_writes = [&](const PipelineStencilStaticState& state, + const PipelineStencilDynamicState& dynamic) { + if (dynamic.writeMask == 0) { + return false; + } + bool can_pass = state.compareOp != vk::CompareOp::eNever; + bool can_fail = state.compareOp != vk::CompareOp::eAlways; + if (dynamic.compareMask == 0) { + switch (state.compareOp) { + case vk::CompareOp::eEqual: + case vk::CompareOp::eLessOrEqual: + case vk::CompareOp::eGreaterOrEqual: + case vk::CompareOp::eAlways: + can_pass = true; + can_fail = false; + break; + case vk::CompareOp::eNever: + case vk::CompareOp::eLess: + case vk::CompareOp::eGreater: + case vk::CompareOp::eNotEqual: + can_pass = false; + can_fail = true; + break; + default: break; + } + } + const bool depth_pass = !depth_test_enable || depth_compare_op != vk::CompareOp::eNever; + const bool depth_fail = depth_test_enable && depth_compare_op != vk::CompareOp::eAlways; + return (can_fail && state.failOp != vk::StencilOp::eKeep) || + (can_pass && depth_pass && state.passOp != vk::StencilOp::eKeep) || + (can_pass && depth_fail && state.depthFailOp != vk::StencilOp::eKeep); + }; + if (stencil_clear_enable || + (stencil_test_enable && + (face_writes(stencil_static_front, stencil_dynamic_front) || + face_writes(stencil_static_back, stencil_dynamic_back)))) { + writes |= vk::ImageAspectFlagBits::eStencil; + } + return writes; } } // namespace Libs::Graphics diff --git a/src/graphics/host_gpu/renderer/depthRenderTarget.h b/src/graphics/host_gpu/renderer/depthRenderTarget.h index e7ac0987..c5f1ca29 100644 --- a/src/graphics/host_gpu/renderer/depthRenderTarget.h +++ b/src/graphics/host_gpu/renderer/depthRenderTarget.h @@ -2,7 +2,9 @@ #define EMULATOR_SRC_GRAPHICS_HOST_GPU_RENDERER_DEPTHRENDERTARGET_H_ #include "common/assert.h" +#include "graphics/host_gpu/renderer/imageView.h" #include "graphics/host_gpu/renderer/renderTarget.h" +#include "graphics/host_gpu/renderer/textureCache.h" #include "graphics/host_gpu/vulkanCommon.h" #include @@ -10,7 +12,6 @@ namespace Libs::Graphics { class RenderCommandBuffer; -struct DepthStencilVulkanImage; inline constexpr bool depth_htile_stencil_acceleration_compatible(bool has_stencil, bool has_htile, bool acceleration_disabled) { @@ -18,6 +19,7 @@ inline constexpr bool depth_htile_stencil_acceleration_compatible(bool has_stenc } struct RenderDepthInfo { + TextureCache::ImageDesc desc; vk::Format format = vk::Format::eUndefined; uint32_t width = 0; uint32_t height = 0; @@ -49,29 +51,46 @@ struct RenderDepthInfo { PipelineStencilStaticState stencil_static_back; PipelineStencilDynamicState stencil_dynamic_front; PipelineStencilDynamicState stencil_dynamic_back; - DepthStencilVulkanImage* vulkan_buffer = nullptr; - vk::ImageView vulkan_view = nullptr; - uint64_t vaddr[3] = {}; - uint64_t size[3] = {}; - int vaddr_num = 0; + ImageId image_id; + vk::ImageView image_view = nullptr; + uint64_t vaddr[3] = {}; + uint64_t size[3] = {}; + int vaddr_num = 0; + + [[nodiscard]] vk::ImageAspectFlags AttachmentWriteAspects() const; }; inline bool depth_attachment_read_only(const RenderDepthInfo& depth) { - const bool stencil_write = - depth.stencil_test_enable && - (depth.stencil_dynamic_front.writeMask != 0 || depth.stencil_dynamic_back.writeMask != 0); - return !depth.depth_load_clear_enable && !depth.stencil_clear_enable && - !depth.depth_write_enable && !stencil_write; + return !depth.AttachmentWriteAspects(); } inline vk::ImageLayout depth_attachment_layout(const RenderDepthInfo& depth) { - return depth_attachment_read_only(depth) ? vk::ImageLayout::eDepthStencilReadOnlyOptimal - : vk::ImageLayout::eDepthStencilAttachmentOptimal; + const auto available = ImageViewOps::DepthAspectMask(depth.format); + const auto writes = depth.AttachmentWriteAspects(); + const bool has_depth = static_cast(available & vk::ImageAspectFlagBits::eDepth); + const bool has_stencil = static_cast(available & vk::ImageAspectFlagBits::eStencil); + const bool depth_write = static_cast(writes & vk::ImageAspectFlagBits::eDepth); + const bool stencil_write = static_cast(writes & vk::ImageAspectFlagBits::eStencil); + if (!has_stencil) { + return depth_write ? vk::ImageLayout::eDepthAttachmentOptimal + : vk::ImageLayout::eDepthReadOnlyOptimal; + } + if (!has_depth) { + return stencil_write ? vk::ImageLayout::eStencilAttachmentOptimal + : vk::ImageLayout::eStencilReadOnlyOptimal; + } + if (depth_write && stencil_write) { + return vk::ImageLayout::eDepthStencilAttachmentOptimal; + } + if (depth_write) { + return vk::ImageLayout::eDepthAttachmentStencilReadOnlyOptimal; + } + if (stencil_write) { + return vk::ImageLayout::eDepthReadOnlyStencilAttachmentOptimal; + } + return vk::ImageLayout::eDepthStencilReadOnlyOptimal; } -void ResolveRenderDepthTarget(uint64_t submit_id, RenderCommandBuffer& buffer, RenderDepthInfo& r); -void MarkRenderTargetGpuWritten(const RenderDepthInfo& target); - } // namespace Libs::Graphics #endif // EMULATOR_SRC_GRAPHICS_HOST_GPU_RENDERER_DEPTHRENDERTARGET_H_ diff --git a/src/graphics/host_gpu/renderer/descriptorCache.cpp b/src/graphics/host_gpu/renderer/descriptorCache.cpp index 7a9d1429..e4c58106 100644 --- a/src/graphics/host_gpu/renderer/descriptorCache.cpp +++ b/src/graphics/host_gpu/renderer/descriptorCache.cpp @@ -2,7 +2,6 @@ #include "common/assert.h" #include "common/profiler.h" -#include "graphics/host_gpu/renderer/renderContext.h" #include "graphics/shader/recompiler/ShaderIR.h" #include @@ -76,31 +75,35 @@ std::vector LayoutKey(DescriptorCache::Stage stage, return key; } -vk::ImageLayout SampledLayout(const VulkanImage& image) { - switch (image.layout) { - case vk::ImageLayout::eTransferDstOptimal: - case vk::ImageLayout::eTransferSrcOptimal: - case vk::ImageLayout::eColorAttachmentOptimal: return vk::ImageLayout::eGeneral; - default: return image.layout; - } -} - vk::DescriptorBufferInfo BufferInfo(const BufferView& view) { - EXIT_IF(view.buffer == nullptr || view.buffer->buffer == nullptr); - return {view.buffer->buffer, view.offset, view.range}; -} - -vk::DescriptorImageInfo MakeDescriptorImageInfo(const DescriptorCache::TextureBinding& texture, - vk::ImageLayout layout) { - EXIT_IF(texture.image == nullptr || texture.view < 0 || texture.view >= VulkanImage::VIEW_MAX); - const auto view = texture.image_view != nullptr ? texture.image_view - : texture.image->image_view[texture.view]; - EXIT_IF(view == nullptr); - return {nullptr, view, layout}; + EXIT_IF(view.buffer == nullptr); + return {view.buffer, view.offset, view.range}; } } // namespace +vk::DescriptorImageInfo DescriptorCache::MakeImageInfo(const TextureBinding& texture) { + EXIT_IF(!texture.image_id || texture.image_view == nullptr || + texture.layout == vk::ImageLayout::eUndefined); + return {nullptr, texture.image_view, texture.layout}; +} + +DescriptorCache::~DescriptorCache() { + for (auto& [layout, sets]: m_free_sets_by_layout) { + (void)layout; + for (auto* set: sets) { + delete set; + } + } + for (const auto& [key, layout]: m_descriptor_set_layouts) { + (void)key; + m_graphics.device.destroyDescriptorSetLayout(layout, nullptr); + } + for (const auto& pool: m_pools) { + m_graphics.device.destroyDescriptorPool(pool.pool, nullptr); + } +} + vk::DescriptorSetLayout DescriptorCache::GetDescriptorSetLayoutInternal(Stage stage, const ShaderRecompiler::IR::Program& program) { @@ -256,13 +259,9 @@ VulkanDescriptorSet& DescriptorCache::GetDescriptor(Stage } break; default: { - const auto layout = IsStorageImage(binding.kind) ? vk::ImageLayout::eGeneral - : vk::ImageLayout::eUndefined; for (const auto resource: binding.resources) { const auto& texture = data.images.at(resource); - image_infos.push_back(MakeDescriptorImageInfo( - texture, - IsSampledImage(binding.kind) ? SampledLayout(*texture.image) : layout)); + image_infos.push_back(MakeImageInfo(texture)); } break; } diff --git a/src/graphics/host_gpu/renderer/descriptorCache.h b/src/graphics/host_gpu/renderer/descriptorCache.h index 65fa6d49..9511e3e1 100644 --- a/src/graphics/host_gpu/renderer/descriptorCache.h +++ b/src/graphics/host_gpu/renderer/descriptorCache.h @@ -6,6 +6,7 @@ #include "common/common.h" #include "common/threads.h" #include "graphics/host_gpu/graphicContext.h" +#include "graphics/host_gpu/renderer/textureCache.h" #include "graphics/host_gpu/vulkanCommon.h" #include "graphics/shader/shaderBindings.h" @@ -18,11 +19,14 @@ namespace Libs::Graphics { namespace ShaderRecompiler::IR { +struct DescriptorValue; +struct ImageResource; struct Program; struct ResourceSnapshot; -} +} // namespace ShaderRecompiler::IR class CommandBuffer; +struct DescriptorCacheTestAccess; struct ShaderStageRuntime; struct VulkanDescriptorSet { @@ -32,11 +36,10 @@ struct VulkanDescriptorSet { }; struct BufferView { - std::shared_ptr owner; - VulkanBuffer* buffer = nullptr; - vk::DeviceSize offset = 0; - vk::DeviceSize range = VK_WHOLE_SIZE; - std::vector host_data; + std::shared_ptr owner; + vk::Buffer buffer = nullptr; + vk::DeviceSize offset = 0; + vk::DeviceSize range = VK_WHOLE_SIZE; }; class DescriptorCache { @@ -44,19 +47,10 @@ public: enum class Stage { Unknown, Vertex, Pixel, Compute }; struct TextureBinding { - VulkanImage* image = nullptr; - int view = VulkanImage::VIEW_DEFAULT; - vk::ImageView image_view = nullptr; - std::shared_ptr owner; - }; - - enum class TextureVariant : int { - Float2D = 0, - Uint2D, - FloatArray, - UintArray, - Float3D, - Uint3D, + ImageId image_id; + vk::ImageView image_view = nullptr; + TextureCache::ImageDesc desc; + vk::ImageLayout layout = vk::ImageLayout::eUndefined; }; struct NativeDescriptors { @@ -70,20 +64,20 @@ public: }; struct PreparedBindings { - std::shared_ptr program; + std::shared_ptr program; std::shared_ptr snapshot; - NativeDescriptors resources; - std::vector flattened_srt; - std::vector user_data; - vk::ShaderStageFlags shader_stage; - Stage stage = Stage::Unknown; - bool committed = false; + NativeDescriptors resources; + std::vector flattened_srt; + std::vector user_data; + vk::ShaderStageFlags shader_stage; + Stage stage = Stage::Unknown; + bool committed = false; }; explicit DescriptorCache(GraphicContext& graphics): m_graphics(graphics) { EXIT_NOT_IMPLEMENTED(!Common::Thread::IsMainThread()); } - ~DescriptorCache() { KYTY_NOT_IMPLEMENTED; } + ~DescriptorCache(); KYTY_CLASS_NO_COPY(DescriptorCache); vk::DescriptorSetLayout GetDescriptorSetLayout(Stage stage, @@ -93,11 +87,14 @@ public: const NativeDescriptors& descriptors); private: + friend struct DescriptorCacheTestAccess; + struct Pool { vk::DescriptorPool pool = nullptr; int next_free_pool = -1; }; + static vk::DescriptorImageInfo MakeImageInfo(const TextureBinding& texture); void CreatePool(); VulkanDescriptorSet* Allocate(Stage stage, const ShaderRecompiler::IR::Program& program); vk::DescriptorSetLayout @@ -112,15 +109,6 @@ private: std::map, vk::DescriptorSetLayout> m_descriptor_set_layouts; }; -[[nodiscard]] DescriptorCache::PreparedBindings -PrepareBindings(CommandBuffer& buffer, const ShaderStageRuntime& runtime, - vk::ShaderStageFlags shader_stage, DescriptorCache::Stage stage); -void RebindBuffers(CommandBuffer& buffer, DescriptorCache::PreparedBindings& bindings); -void RebindImages(CommandBuffer& buffer, DescriptorCache::PreparedBindings& bindings); -void ActivateImageWrites(std::span bindings); -void CommitBindings(CommandBuffer& buffer, vk::PipelineBindPoint pipeline_bind_point, - vk::PipelineLayout layout, DescriptorCache::PreparedBindings& bindings); - } // namespace Libs::Graphics #endif // EMULATOR_SRC_GRAPHICS_HOST_GPU_RENDERER_DESCRIPTORCACHE_H_ diff --git a/src/graphics/host_gpu/renderer/descriptors.cpp b/src/graphics/host_gpu/renderer/descriptors.cpp index c81dacec..6d792332 100644 --- a/src/graphics/host_gpu/renderer/descriptors.cpp +++ b/src/graphics/host_gpu/renderer/descriptors.cpp @@ -17,16 +17,12 @@ #include "graphics/host_gpu/objects/textureCommon.h" #include "graphics/host_gpu/renderer/debug.h" #include "graphics/host_gpu/renderer/descriptorCache.h" -#include "graphics/host_gpu/renderer/framebufferCache.h" #include "graphics/host_gpu/renderer/imageView.h" #include "graphics/host_gpu/renderer/render.h" #include "graphics/host_gpu/renderer/renderContext.h" -#include "graphics/host_gpu/renderer/renderTargetBarriers.h" #include "graphics/host_gpu/renderer/shaderResourceBarrier.h" -#include "graphics/host_gpu/transfer.h" #include "graphics/host_gpu/vma.h" #include "graphics/host_gpu/vulkanCommon.h" -#include "graphics/presentation/displayBuffer.h" #include "graphics/shader/recompiler/BindingLayout.h" #include "graphics/shader/recompiler/ResourceMaterialization.h" #include "graphics/shader/recompiler/ShaderIR.h" @@ -37,7 +33,7 @@ #include #include #include -#include +#include #ifdef min #undef min @@ -48,92 +44,37 @@ namespace Libs::Graphics { -using TextureVariant = DescriptorCache::TextureVariant; - -ImageWriteActivation ClassifyImageWriteActivation(VulkanImageType type) noexcept { - switch (type) { - case VulkanImageType::StorageTexture: return ImageWriteActivation::Implicit; - case VulkanImageType::VideoOut: - case VulkanImageType::DepthStencil: - case VulkanImageType::RenderTexture: return ImageWriteActivation::Explicit; - case VulkanImageType::Unknown: - case VulkanImageType::Texture: return ImageWriteActivation::Unsupported; - } - return ImageWriteActivation::Unsupported; -} - -static void BindNullStorageBuffer(BufferView& dst) { - dst.owner = GetRenderContext().GetBufferCache().ObtainNullBuffer(); - dst.buffer = dst.owner.get(); +static void BindNullStorageBuffer(RenderContext& context, BufferView& dst) { + auto owner = context.GetBufferCache().ObtainNullBuffer(); + dst.buffer = owner->Handle(); + dst.owner = std::move(owner); dst.offset = 0; dst.range = 16; } -// NOLINTNEXTLINE(readability-function-cognitive-complexity) -static const char* VulkanImageTypeName(VulkanImageType type) { - switch (type) { - case VulkanImageType::VideoOut: return "VideoOut"; - case VulkanImageType::DepthStencil: return "DepthStencil"; - case VulkanImageType::Texture: return "Texture"; - case VulkanImageType::StorageTexture: return "StorageTexture"; - case VulkanImageType::RenderTexture: return "RenderTexture"; - case VulkanImageType::Unknown: - default: return "Unknown"; - } -} - -static int SampledArrayViewIndex(const VulkanImage& image, int view_index) { - switch (view_index) { - case VulkanImage::VIEW_DEFAULT: return VulkanImage::VIEW_DEFAULT_ARRAY; - default: return view_index; - } -} - -static int SelectSampledTextureArrayView(const VulkanImage& image, int base_view) { - const int array_view = SampledArrayViewIndex(image, base_view); - if (image.image_view[array_view] == nullptr) { - EXIT("missing sampled array image view: image_type=%s base_view=%d array_view=%d " - "layers=%u\n", - VulkanImageTypeName(image.type), base_view, array_view, image.layers); - } - return array_view; -} - -static bool TextureVariantIsUint(TextureVariant variant) { - return variant == TextureVariant::Uint2D || variant == TextureVariant::UintArray || - variant == TextureVariant::Uint3D; -} - -static bool TextureVariantIsArray(TextureVariant variant) { - return variant == TextureVariant::FloatArray || variant == TextureVariant::UintArray; -} - -static bool TextureVariantIs3D(TextureVariant variant) { - return variant == TextureVariant::Float3D || variant == TextureVariant::Uint3D; -} - -static int TextureVariantDefaultView(TextureVariant variant) { - return TextureVariantIsArray(variant) ? VulkanImage::VIEW_DEFAULT_ARRAY - : VulkanImage::VIEW_DEFAULT; -} - -static VulkanImage& GetDummySampledTexture(TextureVariant variant) { - return GetRenderContext().GetTextureCache().GetDummySampledTexture( - TextureVariantIsUint(variant), TextureVariantIs3D(variant)); -} - -static VulkanImage& GetDummyStorageTexture(TextureVariant variant) { - return GetRenderContext().GetTextureCache().GetDummyStorageTexture( - TextureVariantIsUint(variant), TextureVariantIs3D(variant)); -} - static void CopyNativeDescriptor(const ShaderRecompiler::IR::DescriptorValue& source, std::span destination) { EXIT_IF(source.dword_count != destination.size()); std::copy_n(source.dwords.begin(), destination.size(), destination.begin()); } -static BufferView NativeStorageBuffer(CommandBuffer& command_buffer, +static Prospero::ImageType TextureType(const ShaderTextureResource& descriptor) { + const auto type = static_cast(descriptor.Type()); + return type == Prospero::ImageType::kCube ? Prospero::ImageType::kColor2DArray : type; +} + +static Prospero::ImageType TextureBaseType(Prospero::ImageType type) { + switch (type) { + case Prospero::ImageType::kColor1DArray: return Prospero::ImageType::kColor1D; + case Prospero::ImageType::kColor2DArray: + case Prospero::ImageType::kColor2DMsaa: + case Prospero::ImageType::kColor2DMsaaArray: return Prospero::ImageType::kColor2D; + default: return type; + } +} + +static BufferView NativeStorageBuffer(RenderContext& context, + CommandBuffer& command_buffer, const ShaderBufferResource& descriptor, const ShaderRecompiler::IR::BufferResource& resource) { BufferView result; @@ -146,36 +87,36 @@ static BufferView NativeStorageBuffer(CommandBuffer& command_buffer, } const auto size = stride != 0 ? static_cast(stride) * records : records; if (address == 0 || size == 0) { - BindNullStorageBuffer(result); + BindNullStorageBuffer(context, result); return result; } - const auto& graphics = GetRenderContext().GetGraphics(); + const auto& graphics = context.GetGraphics(); const auto alignment = graphics.StorageMinAlignment(); if (alignment == 0 || size > graphics.GetPhysicalDeviceProperties().limits.maxStorageBufferRange || BufferCache::CACHING_PAGE_SIZE % alignment != 0) { EXIT("storage buffer range or device alignment is unsupported\n"); } - auto binding = GetRenderContext().GetBufferCache().ObtainBuffer( + auto binding = context.GetBufferCache().ObtainBuffer( command_buffer, address, size, resource.written, resource.read, resource.formatted); - if (binding.host_data.empty() && binding.offset % alignment != 0) { + if (binding.offset % alignment != 0) { EXIT("storage buffer binding is not device-aligned\n"); } - result.owner = std::move(binding.buffer); - result.buffer = result.owner.get(); - result.offset = binding.offset; - result.range = static_cast(size); - result.host_data = std::move(binding.host_data); + result.owner = std::move(binding.owner); + result.buffer = binding.buffer; + result.offset = binding.offset; + result.range = static_cast(size); return result; } static BufferView -NativeAddressBuffer(CommandBuffer& command_buffer, +NativeAddressBuffer(RenderContext& context, + CommandBuffer& command_buffer, const ShaderRecompiler::IR::AddressResource& resource, const ShaderRecompiler::IR::ResourceSnapshot::Address& address) { BufferView result; if (address.binding_base == 0) { - BindNullStorageBuffer(result); + BindNullStorageBuffer(context, result); return result; } if (resource.written) { @@ -183,8 +124,7 @@ NativeAddressBuffer(CommandBuffer& command_buffer, } const auto limit = resource.kind == ShaderRecompiler::IR::ResourceKind::Flat ? ShaderRecompiler::IR::FlatAddressWindowSize - : static_cast(GetRenderContext() - .GetGraphics() + : static_cast(context.GetGraphics() .GetPhysicalDeviceProperties() .limits.maxStorageBufferRange); uint64_t size = 0; @@ -193,35 +133,22 @@ NativeAddressBuffer(CommandBuffer& command_buffer, EXIT("address resource is not host-accessible: base=0x%016" PRIx64 "\n", address.binding_base); } - const auto& graphics = GetRenderContext().GetGraphics(); + const auto& graphics = context.GetGraphics(); const auto alignment = graphics.StorageMinAlignment(); if (alignment == 0 || size > graphics.GetPhysicalDeviceProperties().limits.maxStorageBufferRange || BufferCache::GetBufferOffset(address.binding_base) % alignment != 0) { EXIT("address resource range or alignment is unsupported\n"); } - auto binding = GetRenderContext().GetBufferCache().ObtainBuffer(command_buffer, - address.binding_base, size); - result.owner = std::move(binding.buffer); - result.buffer = result.owner.get(); - result.offset = binding.offset; - result.range = static_cast(size); - result.host_data = std::move(binding.host_data); + auto binding = + context.GetBufferCache().ObtainBuffer(command_buffer, address.binding_base, size); + result.owner = std::move(binding.owner); + result.buffer = binding.buffer; + result.offset = binding.offset; + result.range = static_cast(size); return result; } -static TextureVariant NativeTextureVariant(const ShaderRecompiler::IR::ImageResource& resource) { - const bool uint_image = resource.kind == ShaderRecompiler::IR::ResourceKind::ImageUint || - resource.kind == ShaderRecompiler::IR::ResourceKind::StorageImageUint; - switch (resource.dimension) { - case ShaderRecompiler::Decoder::ImageDimension::Dim3D: - return uint_image ? TextureVariant::Uint3D : TextureVariant::Float3D; - case ShaderRecompiler::Decoder::ImageDimension::Dim2DArray: - return uint_image ? TextureVariant::UintArray : TextureVariant::FloatArray; - default: return uint_image ? TextureVariant::Uint2D : TextureVariant::Float2D; - } -} - static bool IsSupportedSampledColorResource(const ShaderRecompiler::IR::ImageResource& resource) { bool supported_dimension = false; switch (resource.dimension) { @@ -268,35 +195,35 @@ TargetTextureViewInfo ResolveTargetTextureView(const ShaderRecompiler::IR::Image } bool IsSupportedSampledVideoOutView(const ShaderRecompiler::IR::ImageResource& resource, - const ShaderTextureResource& descriptor, - const VulkanImage& image) { - return image.type == VulkanImageType::VideoOut && image.layers == 1 && + const ShaderTextureResource& descriptor, const Image& image) { + return image.usage.video_out && image.info.resources.layers == 1 && IsSupportedSampledColorResource(resource) && resource.dimension == ShaderRecompiler::Decoder::ImageDimension::Dim2D && descriptor.Type() == Prospero::GpuEnumValue(Prospero::ImageType::kColor2D) && descriptor.Depth() == 0 && descriptor.BaseArray5() == 0; } -bool IsSupportedDepthTargetDescriptor(const ShaderTextureResource& descriptor, - const VulkanImage& image) { +bool IsSupportedDepthTargetDescriptor(const ShaderTextureResource& descriptor, const Image& image) { const auto width = static_cast(descriptor.Width5()) + 1u; const auto height = static_cast(descriptor.Height5()) + 1u; const auto pitch = TileGetTexturePitch(descriptor.Format(), width, 1, descriptor.TileMode()); const auto type = static_cast(descriptor.Type()); const bool supported_single_layer = - image.layers == 1 && descriptor.Depth() == 0 && descriptor.BaseArray5() == 0 && + image.info.resources.layers == 1 && descriptor.Depth() == 0 && + descriptor.BaseArray5() == 0 && (type == Prospero::ImageType::kColor2D || type == Prospero::ImageType::kColor2DArray); - const bool supported_cube = type == Prospero::ImageType::kCube && width == height && - image.layers >= 6 && image.layers % 6u == 0 && - static_cast(descriptor.Depth()) + 1u == image.layers && - descriptor.BaseArray5() == 0; - return image.type == VulkanImageType::DepthStencil && width == image.extent.width && - height == image.extent.height && (supported_single_layer || supported_cube) && + const bool supported_cube = + type == Prospero::ImageType::kCube && width == height && image.info.resources.layers >= 6 && + image.info.resources.layers % 6u == 0 && + static_cast(descriptor.Depth()) + 1u == image.info.resources.layers && + descriptor.BaseArray5() == 0; + return image.info.IsDepth() && width == image.info.extent.width && + height == image.info.extent.height && (supported_single_layer || supported_cube) && descriptor.BaseLevel() == 0 && descriptor.LastLevel() == 0 && descriptor.MaxMip() == 0 && descriptor.MinLod() == 0 && descriptor.BaseArray5() == 0 && descriptor.TileMode() == Prospero::GpuEnumValue(Prospero::TileMode::kDepth) && descriptor.BCSwizzle() == 0 && !descriptor.MsaaDepth() && pitch >= width && - pitch == image.guest_pitch; + pitch == image.info.pitch; } bool IsSupportedDepthTextureEncoding(const ShaderTextureResource& descriptor) { @@ -315,15 +242,15 @@ bool IsSupportedDepthTextureEncoding(const ShaderTextureResource& descriptor) { } static void ValidateDepthTargetBinding(const ShaderRecompiler::IR::ImageResource& resource, - const ShaderTextureResource& descriptor, - const VulkanImage* image, vk::Format view_format, - uint64_t size) { + const ShaderTextureResource& descriptor, const Image* image, + vk::Format view_format, uint64_t size) { const bool resource_ok = IsSupportedSampledDepthResource(resource); const bool descriptor_ok = image != nullptr && IsSupportedDepthTargetDescriptor(descriptor, *image); const bool encoding_ok = IsSupportedDepthTextureEncoding(descriptor); - const bool format_ok = image != nullptr && IsSupportedSampledDepthFormat( - image->format, descriptor.Format(), view_format); + const bool format_ok = + image != nullptr && + IsSupportedSampledDepthFormat(image->info.pixel_format, descriptor.Format(), view_format); if (resource_ok && descriptor_ok && encoding_ok && format_ok && size != 0) { return; } @@ -341,20 +268,26 @@ static void ValidateDepthTargetBinding(const ShaderRecompiler::IR::ImageResource resource.read, resource.written, resource.atomic, resource.depth_compare, descriptor.Format(), descriptor.DstSelXYZW(), image == nullptr ? static_cast(vk::Format::eUndefined) - : static_cast(image->format), - static_cast(view_format), image == nullptr ? 0u : image->layers, descriptor.Type(), - descriptor.BaseArray5(), descriptor.Depth(), descriptor_pitch, - image == nullptr ? 0u : image->guest_pitch, descriptor.Base40(), size, - descriptor.fields[0], descriptor.fields[1], descriptor.fields[2], descriptor.fields[3], - descriptor.fields[4], descriptor.fields[5], descriptor.fields[6], descriptor.fields[7]); + : static_cast(image->info.pixel_format), + static_cast(view_format), image == nullptr ? 0u : image->info.resources.layers, + descriptor.Type(), descriptor.BaseArray5(), descriptor.Depth(), descriptor_pitch, + image == nullptr ? 0u : image->info.pitch, descriptor.Base40(), size, descriptor.fields[0], + descriptor.fields[1], descriptor.fields[2], descriptor.fields[3], descriptor.fields[4], + descriptor.fields[5], descriptor.fields[6], descriptor.fields[7]); +} + +static bool IsSupportedStorageDepthTile(const ShaderTextureResource& descriptor) { + const auto type = static_cast(descriptor.Type()); + return descriptor.Depth() == 0 && + ((descriptor.Format() == Prospero::GpuEnumValue(Prospero::BufferFormat::k8UInt) && + type == Prospero::ImageType::kColor2DArray) || + (descriptor.Format() == Prospero::GpuEnumValue(Prospero::BufferFormat::k32UInt) && + type == Prospero::ImageType::kColor2D)); } static bool IsSupportedStorageTextureDescriptor(const ShaderRecompiler::IR::ImageResource& resource, const ShaderTextureResource& descriptor) { - const auto tile = descriptor.TileMode(); - const auto width = static_cast(descriptor.Width5()) + 1u; - const auto height = static_cast(descriptor.Height5()) + 1u; - const auto depth = static_cast(descriptor.Depth()) + 1u; + const auto tile = descriptor.TileMode(); const bool is_2d = resource.dimension == ShaderRecompiler::Decoder::ImageDimension::Dim2D && descriptor.Type() == Prospero::GpuEnumValue(Prospero::ImageType::kColor2D) && descriptor.Depth() == 0; @@ -367,13 +300,15 @@ static bool IsSupportedStorageTextureDescriptor(const ShaderRecompiler::IR::Imag const bool supported_depth_tile = tile == Prospero::GpuEnumValue(Prospero::TileMode::kDepth) && !resource.read && resource.kind == ShaderRecompiler::IR::ResourceKind::StorageImageUint && - IsSupportedStorageDepthTile(descriptor.Format(), descriptor.Type(), width, height, depth); + IsSupportedStorageDepthTile(descriptor); + const bool supported_volume_tile = + is_3d && tile == Prospero::GpuEnumValue(Prospero::TileMode::kStandard4KB); const bool supported_tile = tile == Prospero::GpuEnumValue(Prospero::TileMode::kLinear) || tile == Prospero::GpuEnumValue(Prospero::TileMode::kRenderTarget) || - supported_depth_tile; - const bool supported_swizzle = - IsSupportedStorageSwizzle(descriptor.Format(), descriptor.DstSelXYZW()) && - (descriptor.DstSelXYZW() == DstSel(4, 5, 6, 7) || !resource.read); + supported_depth_tile || supported_volume_tile; + const bool supported_swizzle = IsValidImageSwizzle(descriptor.DstSelXYZW()) && + (descriptor.DstSelXYZW() == DstSel(4, 5, 6, 7) || + !resource.read); const bool supported_mip_view = descriptor.BaseLevel() == 0 || is_2d; return (is_2d || is_2d_array || is_3d) && supported_tile && supported_mip_view && descriptor.BaseLevel() == descriptor.LastLevel() && @@ -395,8 +330,7 @@ static bool IsSupportedStorageTextureEncoding(const ShaderTextureResource& descr return (descriptor.fields[1] & field1_reserved_mask) == 0 && (descriptor.fields[2] & field2_reserved_mask) == 0 && descriptor.fields[3] == expected_field3 && descriptor.fields[4] == descriptor.Depth() && - (descriptor.fields[5] & ~field5_max_mip_mask) == field5_expected && - descriptor.fields[6] == 0 && descriptor.fields[7] == 0; + (descriptor.fields[5] & ~field5_max_mip_mask) == field5_expected; } void ValidateStorageTexture(const ShaderRecompiler::IR::ImageResource& resource, @@ -423,12 +357,8 @@ void ValidateStorageTexture(const ShaderRecompiler::IR::ImageResource& resource, static_cast(resource.dimension), static_cast(resource.mip_mode), resource.atomic, resource.depth_compare, descriptor.BaseLevel(), descriptor.LastLevel(), descriptor.MaxMip(), descriptor.MinLod(), descriptor.BaseArray5(), descriptor.BCSwizzle(), - descriptor.MsaaDepth(), - IsSupportedStorageDepthTile(descriptor.Format(), descriptor.Type(), - static_cast(descriptor.Width5()) + 1u, - static_cast(descriptor.Height5()) + 1u, - static_cast(descriptor.Depth()) + 1u), - IsSupportedStorageSwizzle(descriptor.Format(), descriptor.DstSelXYZW()), + descriptor.MsaaDepth(), IsSupportedStorageDepthTile(descriptor), + IsValidImageSwizzle(descriptor.DstSelXYZW()), descriptor.Base40(), size, static_cast(descriptor.Width5()) + 1u, static_cast(descriptor.Height5()) + 1u, static_cast(descriptor.Depth()) + 1u, descriptor.Type(), format, @@ -437,90 +367,167 @@ void ValidateStorageTexture(const ShaderRecompiler::IR::ImageResource& resource, descriptor.fields[4], descriptor.fields[5], descriptor.fields[6], descriptor.fields[7]); } -void ValidateMetadataReuseTexture(const ShaderRecompiler::IR::ImageResource& resource, - const ShaderTextureResource& descriptor, uint64_t size) { - constexpr uint32_t field1_reserved = 0x200fff00u; - constexpr uint32_t field2_reserved = 0xf0003000u; - constexpr uint32_t field5_common = 0x00700000u; - const auto format = descriptor.Format(); - const bool resource_ok = IsSupportedSampledColorResource(resource); - const bool swizzle_ok = IsValidSampledColorSwizzle(descriptor.DstSelXYZW()); - const bool descriptor_ok = - resource.dimension == ShaderRecompiler::Decoder::ImageDimension::Dim2D && - descriptor.Type() == Prospero::GpuEnumValue(Prospero::ImageType::kColor2D) && - descriptor.Depth() == 0 && descriptor.BaseArray5() == 0 && - descriptor.BaseLevel() <= descriptor.LastLevel() && - descriptor.LastLevel() <= descriptor.MaxMip() && descriptor.MaxMip() < 15 && - descriptor.TileMode() == Prospero::GpuEnumValue(Prospero::TileMode::kStandard4KB) && - swizzle_ok; - const uint32_t field3_expected = descriptor.DstSelXYZW() | - (static_cast(descriptor.BaseLevel()) << 12u) | - (static_cast(descriptor.LastLevel()) << 16u) | - (static_cast(descriptor.TileMode()) << 20u) | - (static_cast(descriptor.Type()) << 28u); - const uint32_t field4_expected = descriptor.Depth() | (descriptor.BaseArray5() << 16u); - const uint32_t field5_expected = - field5_common | (static_cast(descriptor.MaxMip()) << 4u); - const bool encoding_ok = - (descriptor.fields[1] & field1_reserved) == 0 && - (descriptor.fields[2] & field2_reserved) == 0 && descriptor.fields[3] == field3_expected && - descriptor.fields[4] == field4_expected && descriptor.fields[5] == field5_expected && - descriptor.fields[6] == 0 && descriptor.fields[7] == 0; - const bool format_ok = - Prospero::IsSupportedTextureFormat(format) && !Prospero::IsUintTextureFormat(format); - if (!resource_ok || !descriptor_ok || !encoding_ok || !format_ok || size == 0) { - EXIT("unsupported metadata-reuse sampled texture: resource=%d descriptor=%d encoding=%d " - "format=%d " - "kind=%u dimension=%u mip_mode=%u read=%d written=%d atomic=%d compare=%d " - "base_level=%u last_level=%u max_mip=%u base_array=%u swizzle_ok=%d " - "addr=0x%016" PRIx64 " size=0x%016" PRIx64 - " extent=%ux%ux%u type=%u format=%u tile=%u swizzle=0x%03x " - "dwords=%08x,%08x,%08x,%08x,%08x,%08x,%08x,%08x\n", - resource_ok, descriptor_ok, encoding_ok, format_ok, - static_cast(resource.kind), static_cast(resource.dimension), - static_cast(resource.mip_mode), resource.read, resource.written, - resource.atomic, resource.depth_compare, descriptor.BaseLevel(), - descriptor.LastLevel(), descriptor.MaxMip(), descriptor.BaseArray5(), swizzle_ok, - descriptor.Base40(), size, static_cast(descriptor.Width5()) + 1u, - static_cast(descriptor.Height5()) + 1u, - static_cast(descriptor.Depth()) + 1u, descriptor.Type(), format, - descriptor.TileMode(), descriptor.DstSelXYZW(), descriptor.fields[0], - descriptor.fields[1], descriptor.fields[2], descriptor.fields[3], descriptor.fields[4], - descriptor.fields[5], descriptor.fields[6], descriptor.fields[7]); +struct NullImageSpec { + vk::Format format; + uint32_t guest_format; +}; + +static NullImageSpec NullTextureSpec(const ShaderRecompiler::IR::ImageResource& resource) { + const bool uint_image = resource.kind == ShaderRecompiler::IR::ResourceKind::ImageUint || + resource.kind == ShaderRecompiler::IR::ResourceKind::StorageImageUint; + return uint_image + ? NullImageSpec {vk::Format::eR32Uint, + Prospero::GpuEnumValue(Prospero::BufferFormat::k32UInt)} + : NullImageSpec {vk::Format::eR32Sfloat, + Prospero::GpuEnumValue(Prospero::BufferFormat::k32Float)}; +} + +static TextureCache::ImageDesc +NullTextureDesc(const ShaderRecompiler::IR::ImageResource& resource, + TextureCache::BindingType binding) { + const auto spec = NullTextureSpec(resource); + TextureCache::ImageDesc desc {}; + desc.info.pixel_format = spec.format; + desc.info.guest_format = spec.guest_format; + desc.info.type = Prospero::ImageType::kColor2D; + desc.info.extent = {1, 1, 1}; + desc.info.resources = {1, 1}; + desc.info.bytes_per_block = 4; + desc.info.samples = 1; + desc.info.mip_layout[0] = {0, 0, 1, 1}; + desc.view_info.format = desc.info.pixel_format; + desc.view_info.type = vk::ImageViewType::e2D; + desc.view_info.aspect = vk::ImageAspectFlagBits::eColor; + desc.view_info.usage = + binding == TextureCache::BindingType::Storage ? vk::ImageUsageFlagBits::eStorage + : vk::ImageUsageFlagBits::eSampled; + desc.type = binding; + return desc; +} + +static void PopulateTextureMipLayout(ImageInfo& info) { + if (info.IsVolume() && info.tile_mode != Prospero::GpuEnumValue(Prospero::TileMode::kLinear)) { + TileVolumeLayout volume {}; + if (!TileGetTextureVolumeLayout(info.guest_format, info.extent.width, info.extent.height, + info.extent.depth, info.resources.levels, info.tile_mode, + volume)) { + EXIT("unsupported normalized volume texture layout\n"); + } + for (uint32_t level = 0; level < info.resources.levels; level++) { + info.mip_layout[level] = { + volume.level_offsets[level], + volume.level_sizes[level], + volume.level_widths[level], + volume.level_heights[level], + }; + } + return; + } + + TileSizeOffset levels[16] {}; + TilePaddedSize padded[16] {}; + TileGetTextureSize(info.guest_format, info.extent.width, info.extent.height, info.pitch, + info.resources.levels, info.tile_mode, nullptr, levels, padded); + for (uint32_t level = 0; level < info.resources.levels; level++) { + const auto offset = + levels[level].src_size != 0 ? levels[level].src_offset : levels[level].offset; + auto size = static_cast(levels[level].src_size != 0 ? levels[level].src_size + : levels[level].size); + if (info.IsVolume()) { + size *= std::max(info.extent.depth >> level, 1u); + } else { + size *= info.resources.layers; + } + info.mip_layout[level] = { + offset, + size, + padded[level].width != 0 ? padded[level].width : std::max(info.pitch >> level, 1u), + padded[level].height != 0 ? padded[level].height + : std::max(info.extent.height >> level, 1u), + }; } } -static DescriptorCache::TextureBinding -NativeTexture(CommandBuffer& command_buffer, const ShaderRecompiler::IR::ImageResource& resource, - const ShaderRecompiler::IR::DescriptorValue& value) { +static ImageViewInfo TextureViewInfo(const ShaderRecompiler::IR::ImageResource& resource, + const ShaderTextureResource& descriptor, vk::Format format, + bool storage, uint32_t view_levels, + uint32_t image_layers) { + ImageViewInfo view {}; + view.format = format; + view.aspect = vk::ImageAspectFlagBits::eColor; + view.base_level = descriptor.BaseLevel(); + view.level_count = view_levels; + view.usage = storage ? vk::ImageUsageFlagBits::eStorage : vk::ImageUsageFlagBits::eSampled; + view.mapping = + storage ? vk::ComponentMapping {} : TextureGetComponentMapping(descriptor.DstSelXYZW()); + switch (resource.dimension) { + case ShaderRecompiler::Decoder::ImageDimension::Dim3D: + view.type = vk::ImageViewType::e3D; + view.base_layer = 0; + view.layer_count = 1; + break; + case ShaderRecompiler::Decoder::ImageDimension::Dim2DArray: + view.type = vk::ImageViewType::e2DArray; + view.base_layer = descriptor.BaseArray5(); + if (view.base_layer >= image_layers) { + EXIT("texture array base layer is out of bounds\n"); + } + view.layer_count = image_layers - view.base_layer; + break; + default: + view.type = vk::ImageViewType::e2D; + view.base_layer = 0; + view.layer_count = 1; + break; + } + return view; +} + +DescriptorCache::TextureBinding +RenderExecutor::ResolveTexture(const ShaderRecompiler::IR::ImageResource& resource, + const ShaderRecompiler::IR::DescriptorValue& value) { ShaderTextureResource descriptor; CopyNativeDescriptor(value, descriptor.fields); - const bool storage = resource.kind == ShaderRecompiler::IR::ResourceKind::StorageImage || - resource.kind == ShaderRecompiler::IR::ResourceKind::StorageImageUint; - const auto variant = NativeTextureVariant(resource); + const bool storage = resource.written; if (storage) { ValidateStorageImageResource(resource); } + + auto& texture_cache = m_context.GetTextureCache(); if (descriptor.IsNull()) { - return storage ? DescriptorCache::TextureBinding {&GetDummyStorageTexture(variant), - TextureVariantDefaultView(variant)} - : DescriptorCache::TextureBinding {&GetDummySampledTexture(variant), - TextureVariantDefaultView(variant)}; + auto desc = NullTextureDesc(resource, storage ? TextureCache::BindingType::Storage + : TextureCache::BindingType::Texture); + const auto id = texture_cache.FindImage(desc); + return {id, nullptr, std::move(desc)}; } + const auto address = descriptor.Base40(); const auto width = static_cast(descriptor.Width5()) + 1u; const auto height = static_cast(descriptor.Height5()) + 1u; const auto base_level = descriptor.BaseLevel(); const auto last_level = descriptor.LastLevel(); - const auto levels = static_cast(descriptor.MaxMip()) + 1u; - if (base_level > last_level || last_level >= levels) { + const auto type = TextureType(descriptor); + const bool multisampled = + type == Prospero::ImageType::kColor2DMsaa || + type == Prospero::ImageType::kColor2DMsaaArray; + const auto levels = multisampled ? 1u : static_cast(descriptor.MaxMip()) + 1u; + const auto tile = descriptor.TileMode(); + const bool msaa_tile = + tile == Prospero::GpuEnumValue(Prospero::TileMode::kRenderTarget); + const bool msaa_array = type == Prospero::ImageType::kColor2DMsaaArray; + if ((!multisampled && (base_level > last_level || last_level >= levels)) || + (multisampled && + (base_level != 0 || last_level == 0 || last_level > 3 || + descriptor.MaxMip() != last_level || !msaa_tile || descriptor.MsaaDepth() || + (!msaa_array && (descriptor.Depth() != 0 || descriptor.BaseArray5() != 0))))) { EXIT("unsupported texture mip view: base=%u last=%u levels=%u\n", base_level, last_level, levels); } - const auto view_levels = last_level - base_level + 1u; - const auto depth = static_cast(descriptor.Depth()) + 1u; - const auto tile = descriptor.TileMode(); - const auto format = descriptor.Format(); + const auto samples = multisampled ? 1u << last_level : 1u; + const auto view_levels = + multisampled ? 1u : static_cast(last_level - base_level) + 1u; + const auto depth = static_cast(descriptor.Depth()) + 1u; + const auto format = descriptor.Format(); const bool sampled_numeric_class = storage || ((resource.kind == ShaderRecompiler::IR::ResourceKind::ImageUint) == Prospero::IsUintTextureFormat(format)); @@ -531,207 +538,84 @@ NativeTexture(CommandBuffer& command_buffer, const ShaderRecompiler::IR::ImageRe EXIT("sampled image numeric class mismatch: kind=%u format=%u addr=0x%016" PRIx64 "\n", static_cast(resource.kind), format, address); } - const auto view_format = TextureGetFormat(format); - const auto type = static_cast(descriptor.Type()); - const auto target_view = - ResolveTargetTextureView(resource, type, descriptor.BaseArray5(), depth); - const auto pitch = TileGetTexturePitch(format, width, levels, tile); - const auto swizzle = descriptor.DstSelXYZW(); - TileSizeAlign size; - TileGetTextureTotalSize(format, width, height, depth, pitch, levels, tile, - type == Prospero::ImageType::kColor3D, size); - EXIT_NOT_IMPLEMENTED(size.size == 0 || + + const bool volume = type == Prospero::ImageType::kColor3D; + const bool layered = type == Prospero::ImageType::kColor1DArray || + type == Prospero::ImageType::kColor2DArray || + type == Prospero::ImageType::kColor2DMsaaArray; + const auto image_layers = layered ? depth : 1u; + uint32_t pitch = 0; + TileSizeAlign size {}; + if (multisampled) { + const auto bytes = Prospero::NumBytesPerElement(format); + pitch = TileGetRenderTargetPitch(width, bytes, last_level); + if (pitch == 0 || + !TileGetRenderTargetSize(width, height, pitch, bytes, size, last_level) || + size.size > UINT32_MAX / image_layers) { + EXIT("unsupported multisample texture layout\n"); + } + size.size *= image_layers; + } else { + pitch = TileGetTexturePitch(format, width, levels, tile); + TileGetTextureTotalSize(format, width, height, volume ? depth : image_layers, pitch, + levels, tile, volume, size); + } + EXIT_NOT_IMPLEMENTED(size.size == 0 || size.align == 0 || (address & (static_cast(size.align) - 1u)) != 0); if (storage) { ValidateStorageTexture(resource, descriptor, size.size); - GetRenderContext().GetBufferCache().ValidateGpuAccess(address, size.size, resource.read, - resource.written); + m_context.GetBufferCache().ValidateGpuAccess(address, size.size, resource.read, + resource.written); } - VulkanImage* image = nullptr; - int view = VulkanImage::VIEW_DEFAULT; - vk::ImageView image_view = nullptr; - bool external = false; - const bool check_depth = static_cast(tile) == Prospero::TileMode::kDepth || - descriptor.MsaaDepth(); - if (image == nullptr) { - if (check_depth) { - image = GetRenderContext().GetTextureCache().FindDepthTargetByRange( - command_buffer, address, size.size, true); - } else { - image = GetRenderContext().GetTextureCache().FindRenderTargetByRange( - command_buffer, address, size.size); - } - if (image != nullptr) { - if (check_depth) { - const bool uint_reinterpret = - IsSupportedSampledDepthUintResource(resource) && - IsSupportedDepthTargetDescriptor(descriptor, *image) && - IsSupportedDepthTextureEncoding(descriptor) && - IsDepthUintTextureReinterpretation(image->format, descriptor.Format(), - view_format); - const bool uint_storage_reinterpret = - storage && IsSupportedStorageImageResource(resource) && - IsSupportedStorageTextureDescriptor(resource, descriptor) && - IsSupportedStorageTextureEncoding(descriptor) && - IsDepthUintTextureReinterpretation(image->format, descriptor.Format(), - view_format); - if (uint_reinterpret || uint_storage_reinterpret) { - image = nullptr; - } else { - const auto depth_view = ResolveTargetTextureView( - resource, type, descriptor.BaseArray5(), image->layers); - ValidateDepthTargetBinding(resource, descriptor, image, view_format, size.size); - if (depth_view.type == - static_cast(VK_IMAGE_VIEW_TYPE_MAX_ENUM)) { - EXIT("unsupported sampled depth target view: dimension=%u " - "descriptor_type=%u " - "base_array=%u image_layers=%u\n", - static_cast(resource.dimension), descriptor.Type(), - descriptor.BaseArray5(), image->layers); - } - image_view = GetRenderContext().GetTextureCache().GetDepthTargetSampledView( - *static_cast(image), view_format, swizzle, 0, 1, - depth_view.type, depth_view.base_layer, depth_view.layer_count); - } - } else { - if (!(storage ? IsSupportedStorageImageResource(resource) - : IsSupportedSampledColorResource(resource)) || - image->type != VulkanImageType::RenderTexture || width != image->extent.width || - height != image->extent.height || - (storage ? levels != image->mip_levels || base_level != 0 - : levels != image->mip_levels || base_level >= levels) || - target_view.type == - static_cast(VK_IMAGE_VIEW_TYPE_MAX_ENUM) || - target_view.base_layer >= image->layers || - target_view.layer_count > image->layers - target_view.base_layer) { - EXIT("unsupported cached render-target image view: storage=%d resource=%u " - "dimension=%u" - " image_type=%u layers=%u extent=%ux%u/%ux%u depth=%u" - " levels=%u/%u base_level=%u base_array=%u descriptor_type=%u\n", - storage, static_cast(resource.kind), - static_cast(resource.dimension), - static_cast(image->type), image->layers, width, height, - image->extent.width, image->extent.height, depth, levels, - image->mip_levels, base_level, descriptor.BaseArray5(), - static_cast(type)); - } - if (storage) { - view = SelectStorageColorView(image->format, view_format, swizzle); - image_view = GetRenderContext().GetTextureCache().GetRenderTargetStorageView( - *static_cast(image), view_format, base_level, - view_levels, target_view.type, target_view.base_layer, - target_view.layer_count); - } else { - image_view = GetRenderContext().GetTextureCache().GetSampledColorView( - *image, view_format, swizzle, base_level, view_levels, target_view.type, - target_view.base_layer, target_view.layer_count); - } - } - if (image != nullptr && image_view == nullptr && image->image_view[view] == nullptr) { - EXIT("required cached texture image view is missing\n"); - } - } + const auto pixel_format = TextureGetFormat(format); + const auto block_bytes = Prospero::BlockCompressedBytesPerBlock(format); + TextureCache::ImageDesc desc {}; + desc.info.data = {address, size.size}; + desc.info.pixel_format = pixel_format; + desc.info.guest_format = format; + desc.info.type = TextureBaseType(type); + desc.info.extent = {width, height, volume ? depth : 1u}; + desc.info.resources = {levels, image_layers}; + desc.info.pitch = pitch; + desc.info.bytes_per_block = + block_bytes != 0 ? block_bytes : Prospero::NumBytesPerElement(format); + desc.info.samples = samples; + desc.info.tile_mode = tile; + if (samples > 1) { + desc.info.mip_layout[0] = {0, size.size, pitch, height}; + } else { + PopulateTextureMipLayout(desc.info); } - if (image == nullptr) { - const auto video = Presentation::DisplayBufferFind(address); - if (video.image != nullptr) { - external = true; - if (storage) { - const bool exact = - resource.kind == ShaderRecompiler::IR::ResourceKind::StorageImageUint && - resource.dimension == ShaderRecompiler::Decoder::ImageDimension::Dim2D && - !resource.read && resource.written && !resource.atomic && - format == Prospero::GpuEnumValue(Prospero::BufferFormat::k8_8_8_8UInt) && - view_format == vk::Format::eR8G8B8A8Uint && swizzle == DstSel(6, 5, 4, 7) && - width == video.image->extent.width && height == video.image->extent.height && - depth == 1 && levels == 1 && base_level == 0 && view_levels == 1 && - type == Prospero::ImageType::kColor2D && video.size == size.size && - video.pitch == pitch && - (video.image->format == vk::Format::eR8G8B8A8Srgb || - video.image->format == vk::Format::eB8G8R8A8Srgb) && - video.image->image_view[VulkanImage::VIEW_STORAGE] != nullptr; - if (!exact) { - EXIT("unsupported storage access to video-out surface: format=%u view=%d" - " extent=%ux%u size=0x%016" PRIx64 " pitch=%u\n", - format, static_cast(view_format), width, height, size.size, pitch); - } - image = video.image; - view = VulkanImage::VIEW_STORAGE; - } else { - const bool exact = - IsSupportedSampledVideoOutView(resource, descriptor, *video.image) && - width == video.image->extent.width && height == video.image->extent.height && - levels == 1 && base_level == 0 && view_levels == 1 && video.size == size.size && - video.pitch == pitch; - if (!exact) { - EXIT("unsupported sampled access to video-out surface: resource=%u dimension=%u" - " image_format=%d view_format=%d swizzle=0x%03x extent=%ux%u/%ux%u" - " depth=%u levels=%u base=%u count=%u type=%u size=0x%016" PRIx64 - "/0x%016" PRIx64 " pitch=%u/%u\n", - static_cast(resource.kind), - static_cast(resource.dimension), - static_cast(video.image->format), static_cast(view_format), - swizzle, width, height, video.image->extent.width, - video.image->extent.height, depth, levels, base_level, view_levels, - static_cast(type), size.size, video.size, pitch, video.pitch); - } - image = video.image; - image_view = GetRenderContext().GetTextureCache().GetSampledColorView( - *video.image, view_format, swizzle, 0, 1, vk::ImageViewType::e2D, 0, 1); - } - } - } - if (image == nullptr) { - auto& texture_cache = GetRenderContext().GetTextureCache(); - const bool metadata_read = texture_cache.QueryRegion(address, size.size).metadata_pages; - if (storage && metadata_read) { - EXIT("storage texture overlaps surface metadata\n"); - } - if (!storage && metadata_read) { - ValidateMetadataReuseTexture(resource, descriptor, size.size); - } - (void)command_buffer; - ImageInfo info {}; - info.address = address; - info.size = size.size; - info.format = format; - info.width = width; - info.height = height; - info.pitch = pitch; - info.base_level = base_level; - info.levels = levels; - info.view_levels = view_levels; - info.tile = tile; - info.swizzle = swizzle; - info.depth = depth; - info.type = descriptor.Type(); - info.base_array = descriptor.BaseArray5(); + desc.view_info = TextureViewInfo(resource, descriptor, pixel_format, storage, view_levels, + desc.info.resources.layers); + desc.type = storage ? TextureCache::BindingType::Storage + : TextureCache::BindingType::Texture; + + auto id = texture_cache.FindImage(desc); + auto* image = &texture_cache.GetImage(id); + const bool stencil_association = static_cast(image->depth_id); + if (stencil_association) { + id = image->depth_id; + image = &texture_cache.GetImage(id); + } else if (image->info.IsDepth()) { if (storage) { - image = &texture_cache.FindStorageTexture(command_buffer, info); - view = VulkanImage::VIEW_DEFAULT; - image_view = texture_cache.GetStorageTextureStorageView( - *static_cast(image), base_level); - } else { - image = &texture_cache.FindTexture(command_buffer, info, metadata_read); - if (image->type == VulkanImageType::StorageTexture) { - image_view = texture_cache.GetStorageTextureSampledView( - *static_cast(image), info); - } + EXIT("depth target cannot be bound as a storage image\n"); } + ValidateDepthTargetBinding(resource, descriptor, image, pixel_format, size.size); + (void)SelectSampledDepthView(image->info.pixel_format, pixel_format, + descriptor.DstSelXYZW()); + } else if (storage) { + ValidateStorageColorView(image->info.pixel_format, pixel_format, descriptor.DstSelXYZW()); + } else { + (void)SelectSampledColorView(image->info.pixel_format, pixel_format, + descriptor.DstSelXYZW()); } - EXIT_NOT_IMPLEMENTED(image == nullptr); - if (NeedsStaticSampledArrayView(resource.dimension == - ShaderRecompiler::Decoder::ImageDimension::Dim2DArray, - image_view != nullptr)) { - view = SelectSampledTextureArrayView(*image, view); - } - return {image, view, image_view, - external ? std::shared_ptr {} - : GetRenderContext().GetTextureCache().GetImageOwner(*image)}; + return {id, nullptr, std::move(desc)}; } -static vk::Sampler NativeSampler(const ShaderRecompiler::IR::Program& program, uint32_t index, +static vk::Sampler NativeSampler(RenderContext& context, + const ShaderRecompiler::IR::Program& program, uint32_t index, const ShaderRecompiler::IR::DescriptorValue& value) { ShaderSamplerResource descriptor; CopyNativeDescriptor(value, descriptor.fields); @@ -744,21 +628,57 @@ static vk::Sampler NativeSampler(const ShaderRecompiler::IR::Program& program, u if (!depth_compare) { descriptor.fields[0] &= ~(0x7u << 12u); } - return GetRenderContext().GetSamplerCache().GetSampler(descriptor); + return context.GetSamplerCache().GetSampler(descriptor); } -static BufferView NativeUpload(CommandBuffer& command_buffer, std::span data) { +static BufferView NativeUpload(RenderContext& context, CommandBuffer& command_buffer, + std::span data) { EXIT_IF(data.empty()); - BufferView result; - EXIT_IF(!GetRenderContext().GetBufferCache().UploadHostData( - command_buffer, data.data(), data.size_bytes(), 256, result.buffer, result.offset, - result.range)); - return result; + EXIT_IF(command_buffer.IsInvalid() || command_buffer.IsExecute()); + auto binding = + context.GetBufferCache().UploadTransient(data.data(), data.size_bytes(), 256); + return {.owner = std::move(binding.owner), + .buffer = binding.buffer, + .offset = binding.offset, + .range = data.size_bytes()}; +} + +void RenderExecutor::TrackImageBinding(ImageId id) { + auto image = m_context.GetTextureCache().ResolveOwner(id); + EXIT_IF(image == nullptr); + if (std::ranges::find(m_bound_images, image) == m_bound_images.end()) { + m_bound_images.push_back(std::move(image)); + } +} + +void RenderExecutor::BindImage(ImageId id, bool storage) { + auto& image = m_context.GetTextureCache().GetImage(id); + if (image.info.data.Empty()) { + return; + } + if (image.binding.is_bound) { + image.binding.force_general |= storage; + } + image.binding.is_bound = true; + TrackImageBinding(id); +} + +void RenderExecutor::BindRenderTarget(ImageId id) { + auto& image = m_context.GetTextureCache().GetImage(id); + image.binding.is_target = true; + TrackImageBinding(id); +} + +void RenderExecutor::ResetBindings() { + for (const auto& image: m_bound_images) { + image->binding = {}; + } + m_bound_images.clear(); } DescriptorCache::PreparedBindings -PrepareBindings(CommandBuffer& buffer, const ShaderStageRuntime& runtime, - vk::ShaderStageFlags shader_stage, DescriptorCache::Stage stage) { +RenderExecutor::PrepareBindings(CommandBuffer& buffer, const ShaderStageRuntime& runtime, + vk::ShaderStageFlags shader_stage, DescriptorCache::Stage stage) { KYTY_PROFILER_FUNCTION(); EXIT_IF(!runtime); const auto& program = *runtime.program; @@ -779,27 +699,23 @@ PrepareBindings(CommandBuffer& buffer, const ShaderStageRuntime& runtime, ShaderBufferResource descriptor; CopyNativeDescriptor(snapshot.buffers[i], descriptor.fields); descriptors.buffers.push_back( - NativeStorageBuffer(buffer, descriptor, program.info.buffers[i])); + NativeStorageBuffer(m_context, buffer, descriptor, program.info.buffers[i])); } descriptors.images.reserve(program.info.images.size()); for (uint32_t i = 0; i < program.info.images.size(); i++) { - const auto kind = program.info.images[i].kind; - if ((kind == ShaderRecompiler::IR::ResourceKind::StorageImage || - kind == ShaderRecompiler::IR::ResourceKind::StorageImageUint) && - shader_stage != vk::ShaderStageFlagBits::eCompute) { - EXIT("storage images are unsupported outside compute shaders\n"); - } - descriptors.images.push_back( - NativeTexture(buffer, program.info.images[i], snapshot.images[i])); + auto binding = ResolveTexture(program.info.images[i], snapshot.images[i]); + BindImage(binding.image_id, binding.desc.type == TextureCache::BindingType::Storage); + descriptors.images.push_back(binding); } descriptors.samplers.reserve(program.info.samplers.size()); for (uint32_t i = 0; i < program.info.samplers.size(); i++) { - descriptors.samplers.push_back(NativeSampler(program, i, snapshot.samplers[i])); + descriptors.samplers.push_back( + NativeSampler(m_context, program, i, snapshot.samplers[i])); } descriptors.addresses.reserve(program.info.addresses.size()); for (uint32_t i = 0; i < program.info.addresses.size(); i++) { - descriptors.addresses.push_back( - NativeAddressBuffer(buffer, program.info.addresses[i], snapshot.addresses[i])); + descriptors.addresses.push_back(NativeAddressBuffer( + m_context, buffer, program.info.addresses[i], snapshot.addresses[i])); } if (ShaderRecompiler::IR::FindBinding( program.bindings, ShaderRecompiler::IR::DescriptorBindingKind::FlattenedSrt) != @@ -813,16 +729,17 @@ PrepareBindings(CommandBuffer& buffer, const ShaderStageRuntime& runtime, } if (ShaderRecompiler::IR::FindBinding( program.bindings, ShaderRecompiler::IR::DescriptorBindingKind::Gds) != nullptr) { - descriptors.gds.buffer = &GetRenderContext().GetGdsBuffer().GetBuffer(); + descriptors.gds.buffer = m_context.GetBufferCache().GetGdsBuffer().Handle(); } return prepared; } -void RebindBuffers(CommandBuffer& buffer, DescriptorCache::PreparedBindings& prepared) { +void RenderExecutor::RebindBuffers(CommandBuffer& buffer, + DescriptorCache::PreparedBindings& prepared) { KYTY_PROFILER_FUNCTION(); EXIT_IF(prepared.program == nullptr || prepared.snapshot == nullptr); - const auto& program = *prepared.program; - const auto& snapshot = *prepared.snapshot; + const auto& program = *prepared.program; + const auto& snapshot = *prepared.snapshot; auto& resources = prepared.resources; resources.buffers.clear(); @@ -831,62 +748,72 @@ void RebindBuffers(CommandBuffer& buffer, DescriptorCache::PreparedBindings& pre ShaderBufferResource descriptor; CopyNativeDescriptor(snapshot.buffers[i], descriptor.fields); resources.buffers.push_back( - NativeStorageBuffer(buffer, descriptor, program.info.buffers[i])); + NativeStorageBuffer(m_context, buffer, descriptor, program.info.buffers[i])); } resources.addresses.clear(); resources.addresses.reserve(program.info.addresses.size()); for (uint32_t i = 0; i < program.info.addresses.size(); i++) { - resources.addresses.push_back( - NativeAddressBuffer(buffer, program.info.addresses[i], snapshot.addresses[i])); + resources.addresses.push_back(NativeAddressBuffer( + m_context, buffer, program.info.addresses[i], snapshot.addresses[i])); } } -void RebindImages(CommandBuffer& buffer, DescriptorCache::PreparedBindings& prepared) { +void RenderExecutor::RebindImages(CommandBuffer& buffer, + DescriptorCache::PreparedBindings& prepared) { KYTY_PROFILER_FUNCTION(); EXIT_IF(prepared.program == nullptr || prepared.snapshot == nullptr); const auto& program = *prepared.program; const auto& snapshot = *prepared.snapshot; auto& images = prepared.resources.images; - images.clear(); - images.reserve(program.info.images.size()); + EXIT_IF(images.size() != program.info.images.size()); + auto& texture_cache = m_context.GetTextureCache(); for (uint32_t i = 0; i < program.info.images.size(); i++) { - images.push_back(NativeTexture(buffer, program.info.images[i], snapshot.images[i])); + const auto old_image = texture_cache.ResolveOwner(images[i].image_id); + if (old_image == nullptr || + (!old_image->registered && !old_image->info.data.Empty()) || + old_image->binding.needs_rebind) { + if (old_image != nullptr) { + old_image->binding = {}; + } + images[i] = ResolveTexture(program.info.images[i], snapshot.images[i]); + BindImage(images[i].image_id, + images[i].desc.type == TextureCache::BindingType::Storage); + } + auto& binding = images[i]; + binding.image_view = texture_cache.FindTexture(binding.image_id, binding.desc); + auto& image = texture_cache.GetImage(binding.image_id); + const bool storage = binding.desc.type == TextureCache::BindingType::Storage; + image.usage.storage |= storage; + image.usage.texture |= !storage; } } -void ActivateImageWrites(std::span bindings) { - std::unordered_set writable; - for (const auto* prepared: bindings) { - if (prepared == nullptr) { - continue; - } - EXIT_IF(prepared->program == nullptr || - prepared->resources.images.size() != prepared->program->info.images.size()); - for (uint32_t i = 0; i < prepared->program->info.images.size(); i++) { - const auto& binding = prepared->resources.images[i]; - EXIT_IF(binding.image == nullptr); - if (binding.owner != nullptr) { - auto current = GetRenderContext().GetTextureCache().GetImageOwner(*binding.image); - EXIT_IF(current.get() != binding.owner.get()); - } - if (prepared->program->info.images[i].written) { - switch (ClassifyImageWriteActivation(binding.image->type)) { - case ImageWriteActivation::Implicit: break; - case ImageWriteActivation::Explicit: writable.insert(binding.image); break; - case ImageWriteActivation::Unsupported: - EXIT("unsupported writable image type: %u\n", - static_cast(binding.image->type)); - } - } - } - } - for (auto* image: writable) { - GetRenderContext().GetTextureCache().MarkGpuWritten(*image); +RenderExecutor::GraphicsBindings +RenderExecutor::PrepareGraphicsBindings(CommandBuffer& buffer, const ShaderStageRuntime& vertex, + const ShaderStageRuntime& pixel, bool pixel_active) { + GraphicsBindings bindings { + .vertex = PrepareBindings(buffer, vertex, vk::ShaderStageFlagBits::eVertex, + DescriptorCache::Stage::Vertex), + }; + RebindBuffers(buffer, bindings.vertex); + RebindImages(buffer, bindings.vertex); + if (pixel_active) { + bindings.pixel.emplace(PrepareBindings(buffer, pixel, vk::ShaderStageFlagBits::eFragment, + DescriptorCache::Stage::Pixel)); + RebindBuffers(buffer, *bindings.pixel); + RebindImages(buffer, *bindings.pixel); } + return bindings; } -static void RetainBindings(CommandBuffer& buffer, +static void RetainBindings(CommandBuffer& buffer, const DescriptorCache::NativeDescriptors& resources) { + if (resources.flattened_srt.owner != nullptr) { + buffer.RetainResourceUntilFence(resources.flattened_srt.owner); + } + if (resources.user_data.owner != nullptr) { + buffer.RetainResourceUntilFence(resources.user_data.owner); + } for (const auto& view: resources.buffers) { if (view.owner != nullptr) { buffer.RetainResourceUntilFence(view.owner); @@ -897,51 +824,31 @@ static void RetainBindings(CommandBuffer& buffer, buffer.RetainResourceUntilFence(view.owner); } } - for (const auto& image: resources.images) { - if (image.owner != nullptr) { - buffer.RetainResourceUntilFence(image.owner); - } - } } -static void CommitHostBuffers(CommandBuffer& buffer, std::vector& views) { - const auto alignment = GetRenderContext().GetGraphics().StorageMinAlignment(); - EXIT_IF(alignment == 0); - for (auto& view: views) { - if (view.host_data.empty()) { - continue; - } - EXIT_IF(view.owner != nullptr || view.buffer != nullptr); - vk::DeviceSize uploaded_range = 0; - EXIT_IF(!GetRenderContext().GetBufferCache().UploadHostData( - buffer, view.host_data.data(), view.host_data.size(), alignment, view.buffer, - view.offset, uploaded_range)); - EXIT_IF(uploaded_range != view.range); - } -} - -void CommitBindings(CommandBuffer& buffer, vk::PipelineBindPoint pipeline_bind_point, - vk::PipelineLayout layout, DescriptorCache::PreparedBindings& prepared) { +void RenderExecutor::CommitBindings(CommandBuffer& buffer, + vk::PipelineBindPoint pipeline_bind_point, + vk::PipelineLayout layout, + DescriptorCache::PreparedBindings& prepared) { KYTY_PROFILER_FUNCTION(); EXIT_IF(prepared.program == nullptr || prepared.stage == DescriptorCache::Stage::Unknown || prepared.committed); const auto& program = *prepared.program; auto& descriptors = prepared.resources; - CommitHostBuffers(buffer, descriptors.buffers); - CommitHostBuffers(buffer, descriptors.addresses); if (!prepared.flattened_srt.empty()) { - descriptors.flattened_srt = NativeUpload(buffer, prepared.flattened_srt); + descriptors.flattened_srt = NativeUpload(m_context, buffer, prepared.flattened_srt); } if (ShaderRecompiler::IR::FindBinding( program.bindings, ShaderRecompiler::IR::DescriptorBindingKind::UserData) != nullptr) { - descriptors.user_data = NativeUpload(buffer, prepared.user_data); + descriptors.user_data = NativeUpload(m_context, buffer, prepared.user_data); } RetainBindings(buffer, descriptors); auto vk_buffer = buffer.Handle(); const auto shader_stages = ShaderPipelineStages(prepared.shader_stage); if (descriptors.gds.buffer != nullptr) { - const auto barrier = MakeGdsDependency(*descriptors.gds.buffer); + buffer.EndRendering(); + const auto barrier = MakeGdsDependency(descriptors.gds.buffer); vk_buffer.pipelineBarrier( vk::PipelineStageFlagBits::eHost | vk::PipelineStageFlagBits::eTransfer | vk::PipelineStageFlagBits::eAllGraphics | vk::PipelineStageFlagBits::eComputeShader, @@ -949,35 +856,40 @@ void CommitBindings(CommandBuffer& buffer, vk::PipelineBindPoint pipeline_bind_p } for (uint32_t i = 0; i < program.info.images.size(); i++) { - auto* image = descriptors.images[i].image; - const auto& resource = program.info.images[i]; - if (resource.kind == ShaderRecompiler::IR::ResourceKind::Image || - resource.kind == ShaderRecompiler::IR::ResourceKind::ImageUint) { - switch (image->type) { - case VulkanImageType::DepthStencil: - GraphicsRenderDepthStencilBarrier(vk_buffer, *image); - break; - case VulkanImageType::RenderTexture: - case VulkanImageType::StorageTexture: - GraphicsRenderTextureBarrier(vk_buffer, *image); - break; - case VulkanImageType::VideoOut: - GraphicsRenderColorImageBarrier(vk_buffer, *image, RENDER_COLOR_IMAGE_LAYOUT); - break; - default: break; - } + auto& image = m_context.GetTextureCache().GetImage(descriptors.images[i].image_id); + auto& binding = descriptors.images[i]; + const auto& view = binding.desc.view_info; + const ImageSubresourceRange range {view.base_level, view.level_count, view.base_layer, + view.layer_count}; + const bool storage = binding.desc.type == TextureCache::BindingType::Storage; + if (image.info.data.Empty()) { + image.Transit(vk::ImageLayout::eGeneral, + storage ? vk::AccessFlagBits2::eShaderRead | + vk::AccessFlagBits2::eShaderWrite + : vk::AccessFlagBits2::eShaderRead, + range, vk_buffer); + } else if ((image.binding.force_general || image.binding.is_target) && + !image.info.IsDepth()) { + image.Transit(vk::ImageLayout::eGeneral, + vk::AccessFlagBits2::eShaderRead | + vk::AccessFlagBits2::eColorAttachmentRead | + vk::AccessFlagBits2::eColorAttachmentWrite, + {}, vk_buffer); + } else if (storage) { + image.Transit(vk::ImageLayout::eGeneral, + vk::AccessFlagBits2::eShaderRead | vk::AccessFlagBits2::eShaderWrite, + range, vk_buffer); } else { - const auto barrier = - MakeStorageImageDependency(*image, resource.read, resource.written); - vk_buffer.pipelineBarrier(vk::PipelineStageFlagBits::eAllCommands, shader_stages, - vk::DependencyFlags {}, 0, nullptr, 0, nullptr, 1, &barrier); - image->layout = vk::ImageLayout::eGeneral; + image.Transit(image.info.IsDepth() ? vk::ImageLayout::eDepthStencilReadOnlyOptimal + : vk::ImageLayout::eShaderReadOnlyOptimal, + vk::AccessFlagBits2::eShaderRead, range, vk_buffer); } + binding.layout = image.backing.state.layout; } if (!program.bindings.descriptors.empty()) { - auto& set = GetRenderContext().GetDescriptorCache().GetDescriptor( - prepared.stage, program, descriptors); + auto& set = + m_context.GetDescriptorCache().GetDescriptor(prepared.stage, program, descriptors); vk_buffer.bindDescriptorSets(pipeline_bind_point, layout, program.bindings.descriptor_set, 1, &set.set, 0, nullptr); buffer.RecycleDescriptorAfterFence(set); @@ -985,7 +897,8 @@ void CommitBindings(CommandBuffer& buffer, vk::PipelineBindPoint pipeline_bind_p if (program.bindings.push_constant_size != 0) { EXIT_IF(program.bindings.push_constant_size != prepared.user_data.size() * sizeof(uint32_t)); - vk_buffer.pushConstants(layout, prepared.shader_stage, program.bindings.push_constant_offset, + vk_buffer.pushConstants(layout, prepared.shader_stage, + program.bindings.push_constant_offset, program.bindings.push_constant_size, prepared.user_data.data()); } prepared.committed = true; diff --git a/src/graphics/host_gpu/renderer/descriptors.h b/src/graphics/host_gpu/renderer/descriptors.h index 830378cc..77dc2902 100644 --- a/src/graphics/host_gpu/renderer/descriptors.h +++ b/src/graphics/host_gpu/renderer/descriptors.h @@ -2,6 +2,7 @@ #define EMULATOR_SRC_GRAPHICS_HOST_GPU_RENDERER_DESCRIPTORS_H_ #include "common/assert.h" +#include "graphics/host_gpu/renderer/image.h" #include "graphics/host_gpu/vulkanCommon.h" #include "graphics/shader/recompiler/ShaderIR.h" #include "graphics/shader/shaderBindings.h" @@ -12,14 +13,6 @@ namespace Libs::Graphics { -struct VulkanImage; -enum class VulkanImageType; - -enum class ImageWriteActivation { Unsupported, Implicit, Explicit }; - -[[nodiscard]] ImageWriteActivation -ClassifyImageWriteActivation(VulkanImageType type) noexcept; - template [[nodiscard]] T DecodeNativeDescriptor(const ShaderRecompiler::IR::DescriptorValue& value) { static_assert(std::is_trivially_copyable_v); @@ -41,13 +34,11 @@ ResolveTargetTextureView(const ShaderRecompiler::IR::ImageResource& resource, Prospero::ImageType type, uint32_t base_layer, uint32_t image_layers); [[nodiscard]] bool IsSupportedDepthTargetDescriptor(const ShaderTextureResource& descriptor, - const VulkanImage& image); + const Image& image); [[nodiscard]] bool IsSupportedDepthTextureEncoding(const ShaderTextureResource& descriptor); [[nodiscard]] bool IsSupportedSampledVideoOutView(const ShaderRecompiler::IR::ImageResource& resource, - const ShaderTextureResource& descriptor, const VulkanImage& image); -void ValidateMetadataReuseTexture(const ShaderRecompiler::IR::ImageResource& resource, - const ShaderTextureResource& descriptor, uint64_t size); + const ShaderTextureResource& descriptor, const Image& image); void ValidateStorageTexture(const ShaderRecompiler::IR::ImageResource& resource, const ShaderTextureResource& descriptor, uint64_t size); diff --git a/src/graphics/host_gpu/renderer/dummyTextureCache.cpp b/src/graphics/host_gpu/renderer/dummyTextureCache.cpp deleted file mode 100644 index 98f23ea2..00000000 --- a/src/graphics/host_gpu/renderer/dummyTextureCache.cpp +++ /dev/null @@ -1,50 +0,0 @@ -#include "graphics/host_gpu/renderer/dummyTextureCache.h" - -#include "common/assert.h" -#include "graphics/host_gpu/renderer/image.h" -#include "graphics/host_gpu/transfer.h" - -#include - -namespace Libs::Graphics { - -namespace { - -[[nodiscard]] constexpr size_t DummyTextureIndex(bool uint_format, bool image_3d) noexcept { - return (image_3d ? 2u : 0u) + (uint_format ? 1u : 0u); -} - -} // namespace - -DummyTextureCache::~DummyTextureCache() { - Common::LockGuard lock(m_mutex); - const auto populated = [](const auto& slots) { - return std::ranges::any_of(slots, [](const auto& slot) { return slot.image != nullptr; }); - }; - if (!populated(m_sampled) && !populated(m_storage)) { - return; - } - Transfer::WaitForQueueIdle(); - const auto destroy = [](auto& slots) { - for (auto& slot: slots) { - if (slot.image != nullptr) { - ImageOps::Destroy(*slot.image); - slot.image = nullptr; - } - } - }; - destroy(m_sampled); - destroy(m_storage); -} - -VulkanImage& DummyTextureCache::Get(Usage usage, bool uint_format, bool image_3d) { - Common::LockGuard lock(m_mutex); - auto& slots = usage == Usage::Storage ? m_storage : m_sampled; - auto& slot = slots[DummyTextureIndex(uint_format, image_3d)]; - if (slot.image == nullptr) { - slot.image = ImageOps::CreateDummyTexture(uint_format, image_3d, usage == Usage::Storage); - } - return *slot.image; -} - -} // namespace Libs::Graphics diff --git a/src/graphics/host_gpu/renderer/dummyTextureCache.h b/src/graphics/host_gpu/renderer/dummyTextureCache.h deleted file mode 100644 index fe697fd1..00000000 --- a/src/graphics/host_gpu/renderer/dummyTextureCache.h +++ /dev/null @@ -1,35 +0,0 @@ -#ifndef EMULATOR_SRC_GRAPHICS_HOST_GPU_RENDERER_DUMMYTEXTURECACHE_H_ -#define EMULATOR_SRC_GRAPHICS_HOST_GPU_RENDERER_DUMMYTEXTURECACHE_H_ - -#include "common/abi.h" -#include "common/threads.h" -#include "graphics/host_gpu/graphicContext.h" - -#include -#include - -namespace Libs::Graphics { - -class DummyTextureCache final { -public: - enum class Usage : uint8_t { Sampled, Storage }; - - DummyTextureCache() = default; - ~DummyTextureCache(); - KYTY_CLASS_NO_COPY(DummyTextureCache); - - [[nodiscard]] VulkanImage& Get(Usage usage, bool uint_format, bool image_3d); - -private: - struct Slot { - GpuTextureVulkanImage* image = nullptr; - }; - - Common::Mutex m_mutex; - std::array m_sampled {}; - std::array m_storage {}; -}; - -} // namespace Libs::Graphics - -#endif // EMULATOR_SRC_GRAPHICS_HOST_GPU_RENDERER_DUMMYTEXTURECACHE_H_ diff --git a/src/graphics/host_gpu/renderer/framebufferCache.cpp b/src/graphics/host_gpu/renderer/framebufferCache.cpp deleted file mode 100644 index 629a8eed..00000000 --- a/src/graphics/host_gpu/renderer/framebufferCache.cpp +++ /dev/null @@ -1,357 +0,0 @@ -#include "graphics/host_gpu/renderer/framebufferCache.h" - -#include "common/assert.h" -#include "common/logging/log.h" -#include "common/profiler.h" -#include "graphics/host_gpu/renderer/colorRenderTarget.h" -#include "graphics/host_gpu/renderer/depthRenderTarget.h" -#include "graphics/host_gpu/renderer/renderContext.h" -#include "graphics/host_gpu/vulkanCommon.h" - -#include -#include - -namespace Libs::Graphics { - -VulkanFramebuffer* FramebufferCache::CreateFramebuffer(RenderColorInfo* colors, - uint32_t requested_color_count, - RenderDepthInfo& depth) { - KYTY_PROFILER_FUNCTION(); - - Common::LockGuard lock(m_mutex); - - EXIT_IF(colors == nullptr); - EXIT_IF(requested_color_count > RENDER_COLOR_ATTACHMENTS_MAX); - - bool with_depth = (depth.format != vk::Format::eUndefined && depth.vulkan_buffer != nullptr); - bool with_color[RENDER_COLOR_ATTACHMENTS_MAX] = {}; - uint32_t color_count = 0; - VulkanImage* first_color = nullptr; - vk::Extent2D first_color_extent = {}; - uint32_t attachment_samples = 0; - for (uint32_t i = 0; i < requested_color_count; i++) { - with_color[i] = (colors[i].vulkan_buffer != nullptr); - if (!with_color[i]) { - break; - } - if (first_color == nullptr) { - first_color = colors[i].vulkan_buffer; - first_color_extent = colors[i].extent; - attachment_samples = colors[i].samples; - } else if (colors[i].extent.width != first_color_extent.width || - colors[i].extent.height != first_color_extent.height) { - LOGF("Framebuffer: temporary: dropping mismatched MRT%u attachment color0=%ux%u " - "color%u=%ux%u\n", - i, first_color_extent.width, first_color_extent.height, i, colors[i].extent.width, - colors[i].extent.height); - with_color[i] = false; - break; - } - if (colors[i].samples != attachment_samples || - colors[i].vulkan_buffer->samples != colors[i].samples) { - EXIT("Framebuffer: mismatched color attachment samples at slot %u, expected=%u " - "requested=%u image=%u\n", - i, attachment_samples, colors[i].samples, colors[i].vulkan_buffer->samples); - } - color_count++; - } - if (with_depth) { - if (depth.samples != depth.vulkan_buffer->samples) { - EXIT("Framebuffer: depth attachment sample identity mismatch, requested=%u image=%u\n", - depth.samples, depth.vulkan_buffer->samples); - } - if (attachment_samples == 0) { - attachment_samples = depth.samples; - } else if (attachment_samples != depth.samples) { - EXIT( - "Framebuffer: mixed color/depth sample counts are unsupported, color=%u depth=%u\n", - attachment_samples, depth.samples); - } - } - if (!with_depth && color_count == 0) { - LOGF("Framebuffer: warning: no color or depth attachment\n"); - return nullptr; - } - if (vulkan_sample_count(attachment_samples) == vk::SampleCountFlagBits {}) { - EXIT("Framebuffer: invalid attachment sample count %u\n", attachment_samples); - } - vk::ImageLayout color_layout[RENDER_COLOR_ATTACHMENTS_MAX] = {}; - for (auto& layout: color_layout) { - layout = RENDER_COLOR_IMAGE_LAYOUT; - } - auto depth_layout = (with_depth ? depth_attachment_layout(depth) - : vk::ImageLayout::eDepthStencilAttachmentOptimal); - auto depth_read_only = - (with_depth && depth_layout == vk::ImageLayout::eDepthStencilReadOnlyOptimal); - - if (with_depth && first_color != nullptr && - (first_color_extent.width != depth.vulkan_buffer->extent.width || - first_color_extent.height != depth.vulkan_buffer->extent.height)) { - static std::atomic log_count {0}; - if (log_count.fetch_add(1, std::memory_order_relaxed) < 16) { - LOGF("Framebuffer: temporary: dropping mismatched PS5 depth attachment color=%ux%u " - "depth=%ux%u format=%s\n", - first_color_extent.width, first_color_extent.height, - depth.vulkan_buffer->extent.width, depth.vulkan_buffer->extent.height, - VulkanToString(depth.format).c_str()); - } - depth.format = vk::Format::eUndefined; - depth.vulkan_buffer = nullptr; - depth.vulkan_view = nullptr; - depth.depth_test_enable = false; - depth.depth_write_enable = false; - depth.depth_bounds_test_enable = false; - depth.stencil_test_enable = false; - depth.depth_clear_enable = false; - depth.depth_load_clear_enable = false; - depth.stencil_clear_enable = false; - with_depth = false; - depth_layout = vk::ImageLayout::eDepthStencilAttachmentOptimal; - depth_read_only = false; - } - - for (auto& f: m_framebuffers) { - bool color_match = (f.framebuffer != nullptr); - for (uint32_t i = 0; color_match && i < RENDER_COLOR_ATTACHMENTS_MAX; i++) { - const uint64_t image_id = - (i < color_count && with_color[i] ? colors[i].vulkan_buffer->memory.unique_id : 0); - color_match = - color_match && f.image_id[i] == image_id && - f.color_view[i] == - (i < color_count && with_color[i] ? colors[i].vulkan_view : nullptr) && - f.color_clear_enable[i] == - (i < color_count && with_color[i] && colors[i].color_clear_enable) && - f.color_layout[i] == color_layout[i]; - } - if (color_match && f.depth_id == (with_depth ? depth.vulkan_buffer->memory.unique_id : 0) && - f.depth_view == (with_depth ? depth.vulkan_view : nullptr) && - f.depth_clear_enable == depth.depth_load_clear_enable && - f.stencil_clear_enable == depth.stencil_clear_enable && - f.depth_read_only == depth_read_only) { - return f.framebuffer; - } - } - - EXIT_NOT_IMPLEMENTED(with_depth && first_color != nullptr && - (first_color_extent.width != depth.vulkan_buffer->extent.width || - first_color_extent.height != depth.vulkan_buffer->extent.height)); - - if (first_color == nullptr) { - first_color_extent = depth.vulkan_buffer->extent; - } - - auto* framebuffer = new VulkanFramebuffer; - framebuffer->render_pass = nullptr; - framebuffer->framebuffer = nullptr; - framebuffer->samples = attachment_samples; - for (uint32_t i = 0; i < RENDER_COLOR_ATTACHMENTS_MAX; i++) { - framebuffer->color_layout[i] = color_layout[i]; - } - framebuffer->depth_layout = depth_layout; - - vk::AttachmentDescription attachments[RENDER_COLOR_ATTACHMENTS_MAX + 1] = {}; - for (uint32_t i = 0; i < color_count; i++) { - attachments[i].flags = {}; - attachments[i].format = colors[i].format; - attachments[i].samples = vulkan_sample_count(attachment_samples); - attachments[i].loadOp = (colors[i].color_clear_enable ? vk::AttachmentLoadOp::eClear - : vk::AttachmentLoadOp::eLoad); - attachments[i].storeOp = vk::AttachmentStoreOp::eStore; - attachments[i].stencilLoadOp = vk::AttachmentLoadOp::eDontCare; - attachments[i].stencilStoreOp = vk::AttachmentStoreOp::eDontCare; - attachments[i].initialLayout = color_layout[i]; - attachments[i].finalLayout = RENDER_COLOR_IMAGE_LAYOUT; - } - - const uint32_t depth_attachment = color_count; - attachments[depth_attachment].flags = {}; - attachments[depth_attachment].format = depth.format; - attachments[depth_attachment].samples = vulkan_sample_count(attachment_samples); - attachments[depth_attachment].loadOp = - (depth.depth_load_clear_enable ? vk::AttachmentLoadOp::eClear - : vk::AttachmentLoadOp::eLoad); - attachments[depth_attachment].storeOp = vk::AttachmentStoreOp::eStore; - attachments[depth_attachment].stencilLoadOp = - (depth.stencil_clear_enable ? vk::AttachmentLoadOp::eClear : vk::AttachmentLoadOp::eLoad); - attachments[depth_attachment].stencilStoreOp = vk::AttachmentStoreOp::eStore; - attachments[depth_attachment].initialLayout = depth_layout; - attachments[depth_attachment].finalLayout = depth_layout; - - vk::AttachmentReference color_attachment_ref[RENDER_COLOR_ATTACHMENTS_MAX] = {}; - for (uint32_t i = 0; i < RENDER_COLOR_ATTACHMENTS_MAX; i++) { - color_attachment_ref[i].attachment = (i < color_count ? i : VK_ATTACHMENT_UNUSED); - color_attachment_ref[i].layout = RENDER_COLOR_IMAGE_LAYOUT; - } - - vk::AttachmentReference depth_attachment_ref {}; - depth_attachment_ref.attachment = depth_attachment; - depth_attachment_ref.layout = depth_layout; - - vk::SubpassDescription subpass {}; - subpass.flags = {}; - subpass.pipelineBindPoint = vk::PipelineBindPoint::eGraphics; - subpass.inputAttachmentCount = 0; - subpass.pInputAttachments = nullptr; - subpass.colorAttachmentCount = color_count; - subpass.pColorAttachments = (color_count > 0 ? color_attachment_ref : nullptr); - subpass.pResolveAttachments = nullptr; - subpass.pDepthStencilAttachment = (with_depth ? &depth_attachment_ref : nullptr); - subpass.preserveAttachmentCount = 0; - subpass.pPreserveAttachments = nullptr; - - const auto attachment_stage_mask = - static_cast(vk::PipelineStageFlagBits::eColorAttachmentOutput | - vk::PipelineStageFlagBits::eEarlyFragmentTests | - vk::PipelineStageFlagBits::eLateFragmentTests); - const auto attachment_access_mask = static_cast( - vk::AccessFlagBits::eColorAttachmentRead | vk::AccessFlagBits::eColorAttachmentWrite | - vk::AccessFlagBits::eDepthStencilAttachmentRead | - vk::AccessFlagBits::eDepthStencilAttachmentWrite); - - vk::SubpassDependency dependencies[2] = {}; - dependencies[0].srcSubpass = VK_SUBPASS_EXTERNAL; - dependencies[0].dstSubpass = 0; - dependencies[0].srcStageMask = vk::PipelineStageFlagBits::eAllCommands; - dependencies[0].dstStageMask = attachment_stage_mask; - dependencies[0].srcAccessMask = - vk::AccessFlagBits::eMemoryRead | vk::AccessFlagBits::eMemoryWrite; - dependencies[0].dstAccessMask = attachment_access_mask; - dependencies[0].dependencyFlags = vk::DependencyFlagBits::eByRegion; - - dependencies[1].srcSubpass = 0; - dependencies[1].dstSubpass = VK_SUBPASS_EXTERNAL; - dependencies[1].srcStageMask = attachment_stage_mask; - dependencies[1].dstStageMask = vk::PipelineStageFlagBits::eAllCommands; - dependencies[1].srcAccessMask = attachment_access_mask; - dependencies[1].dstAccessMask = - vk::AccessFlagBits::eMemoryRead | vk::AccessFlagBits::eMemoryWrite; - dependencies[1].dependencyFlags = vk::DependencyFlagBits::eByRegion; - - vk::RenderPassCreateInfo render_pass_info {}; - render_pass_info.sType = vk::StructureType::eRenderPassCreateInfo; - render_pass_info.pNext = nullptr; - render_pass_info.flags = {}; - render_pass_info.attachmentCount = color_count + (with_depth ? 1u : 0u); - render_pass_info.pAttachments = attachments; - render_pass_info.subpassCount = 1; - render_pass_info.pSubpasses = &subpass; - render_pass_info.dependencyCount = 2; - render_pass_info.pDependencies = dependencies; - - auto result = - m_graphics.device.createRenderPass(&render_pass_info, nullptr, &framebuffer->render_pass); - EXIT_NOT_IMPLEMENTED(result != vk::Result::eSuccess); - - vk::Format color_formats[RENDER_COLOR_ATTACHMENTS_MAX] = {}; - for (uint32_t i = 0; i < color_count; i++) { - color_formats[i] = colors[i].format; - } - framebuffer->render_pass_id = render_pass_compat_id( - color_count, color_formats, with_depth, depth.format, depth_layout, attachment_samples); - - EXIT_NOT_IMPLEMENTED(framebuffer->render_pass == nullptr); - - vk::ImageView views[RENDER_COLOR_ATTACHMENTS_MAX + 1] = {}; - for (uint32_t i = 0; i < color_count; i++) { - if (colors[i].vulkan_view == nullptr) { - EXIT("Framebuffer: color attachment view is missing at slot %u\n", i); - } - views[i] = colors[i].vulkan_view; - } - if (with_depth) { - if (depth.vulkan_view == nullptr) { - EXIT("Framebuffer: depth attachment view is missing\n"); - } - views[color_count] = depth.vulkan_view; - } - - vk::FramebufferCreateInfo framebuffer_info {}; - framebuffer_info.sType = vk::StructureType::eFramebufferCreateInfo; - framebuffer_info.pNext = nullptr; - framebuffer_info.flags = {}; - framebuffer_info.renderPass = framebuffer->render_pass; - framebuffer_info.attachmentCount = color_count + (with_depth ? 1u : 0u); - framebuffer_info.pAttachments = views; - framebuffer_info.width = first_color_extent.width; - framebuffer_info.height = first_color_extent.height; - framebuffer_info.layers = 1; - - result = - m_graphics.device.createFramebuffer(&framebuffer_info, nullptr, &framebuffer->framebuffer); - EXIT_NOT_IMPLEMENTED(result != vk::Result::eSuccess); - - EXIT_NOT_IMPLEMENTED(framebuffer->framebuffer == nullptr); - - Framebuffer fnew; - fnew.framebuffer = framebuffer; - for (uint32_t i = 0; i < RENDER_COLOR_ATTACHMENTS_MAX; i++) { - fnew.image_id[i] = - (i < color_count && with_color[i] ? colors[i].vulkan_buffer->memory.unique_id : 0); - fnew.color_view[i] = (i < color_count && with_color[i] ? colors[i].vulkan_view : nullptr); - fnew.color_clear_enable[i] = - (i < color_count && with_color[i] && colors[i].color_clear_enable); - fnew.color_layout[i] = color_layout[i]; - } - fnew.depth_id = (with_depth ? depth.vulkan_buffer->memory.unique_id : 0); - fnew.depth_view = (with_depth ? depth.vulkan_view : nullptr); - fnew.depth_clear_enable = depth.depth_load_clear_enable; - fnew.stencil_clear_enable = depth.stencil_clear_enable; - fnew.depth_read_only = depth_read_only; - - bool updated = false; - - for (auto& f: m_framebuffers) { - if (f.framebuffer == nullptr) { - f = fnew; - updated = true; - break; - } - } - - if (!updated) { - m_framebuffers.push_back(fnew); - } - - return framebuffer; -} - -void FramebufferCache::FreeFramebufferByColor(VulkanImage& image) { - Common::LockGuard lock(m_mutex); - - for (auto& f: m_framebuffers) { - bool uses_image = false; - for (auto image_id: f.image_id) { - if (image_id == image.memory.unique_id) { - uses_image = true; - break; - } - } - if (f.framebuffer != nullptr && uses_image) { - m_graphics.device.destroyFramebuffer(f.framebuffer->framebuffer, nullptr); - - m_graphics.device.destroyRenderPass(f.framebuffer->render_pass, nullptr); - - delete f.framebuffer; - - f.framebuffer = nullptr; - } - } -} - -void FramebufferCache::FreeFramebufferByDepth(DepthStencilVulkanImage& image) { - Common::LockGuard lock(m_mutex); - - for (auto& f: m_framebuffers) { - if (f.framebuffer != nullptr && f.depth_id == image.memory.unique_id) { - m_graphics.device.destroyFramebuffer(f.framebuffer->framebuffer, nullptr); - - m_graphics.device.destroyRenderPass(f.framebuffer->render_pass, nullptr); - - delete f.framebuffer; - - f.framebuffer = nullptr; - } - } -} - -} // namespace Libs::Graphics diff --git a/src/graphics/host_gpu/renderer/framebufferCache.h b/src/graphics/host_gpu/renderer/framebufferCache.h deleted file mode 100644 index 6ebbf4d3..00000000 --- a/src/graphics/host_gpu/renderer/framebufferCache.h +++ /dev/null @@ -1,85 +0,0 @@ -#ifndef EMULATOR_SRC_GRAPHICS_HOST_GPU_RENDERER_FRAMEBUFFERCACHE_H_ -#define EMULATOR_SRC_GRAPHICS_HOST_GPU_RENDERER_FRAMEBUFFERCACHE_H_ - -#include "common/abi.h" -#include "common/assert.h" -#include "common/common.h" -#include "common/threads.h" -#include "graphics/host_gpu/graphicContext.h" -#include "graphics/host_gpu/renderer/renderTarget.h" -#include "graphics/host_gpu/vulkanCommon.h" - -#include - -namespace Libs::Graphics { - -struct RenderColorInfo; -struct RenderDepthInfo; - -static constexpr vk::ImageLayout RENDER_COLOR_IMAGE_LAYOUT = vk::ImageLayout::eGeneral; - -struct VulkanFramebuffer { - vk::RenderPass render_pass = nullptr; - uint64_t render_pass_id = 0; - vk::Framebuffer framebuffer = nullptr; - uint32_t samples = 1; - vk::ImageLayout color_layout[RENDER_COLOR_ATTACHMENTS_MAX] = {}; - vk::ImageLayout depth_layout = vk::ImageLayout::eDepthStencilAttachmentOptimal; -}; - -inline uint64_t render_pass_compat_id(uint32_t color_count, const vk::Format* color_formats, - bool with_depth, vk::Format depth_format, - vk::ImageLayout depth_layout, uint32_t samples) { - uint64_t id = 0xcbf29ce484222325ull; - auto mix = [&id](uint64_t v) { - id ^= v; - id *= 0x100000001b3ull; - }; - - mix(color_count); - for (uint32_t i = 0; i < color_count; i++) { - mix(static_cast(color_formats[i])); - } - mix(with_depth ? 1u : 0u); - mix(static_cast(depth_format)); - mix(static_cast(depth_layout)); - mix(samples); - - return id; -} - -class FramebufferCache { -public: - explicit FramebufferCache(GraphicContext& graphics): m_graphics(graphics) { - EXIT_NOT_IMPLEMENTED(!Common::Thread::IsMainThread()); - } - ~FramebufferCache() { KYTY_NOT_IMPLEMENTED; } - KYTY_CLASS_NO_COPY(FramebufferCache); - - VulkanFramebuffer* CreateFramebuffer(RenderColorInfo* colors, uint32_t color_count, - RenderDepthInfo& depth); - void FreeFramebufferByColor(VulkanImage& image); - void FreeFramebufferByDepth(DepthStencilVulkanImage& image); - -private: - struct Framebuffer { - VulkanFramebuffer* framebuffer = nullptr; - uint64_t image_id[RENDER_COLOR_ATTACHMENTS_MAX] = {}; - vk::ImageView color_view[RENDER_COLOR_ATTACHMENTS_MAX] = {}; - uint64_t depth_id = 0; - vk::ImageView depth_view = nullptr; - bool color_clear_enable[RENDER_COLOR_ATTACHMENTS_MAX] = {}; - vk::ImageLayout color_layout[RENDER_COLOR_ATTACHMENTS_MAX] = {}; - bool depth_clear_enable = false; - bool stencil_clear_enable = false; - bool depth_read_only = false; - }; - - GraphicContext& m_graphics; - Common::Mutex m_mutex; - std::vector m_framebuffers; -}; - -} // namespace Libs::Graphics - -#endif // EMULATOR_SRC_GRAPHICS_HOST_GPU_RENDERER_FRAMEBUFFERCACHE_H_ diff --git a/src/graphics/host_gpu/renderer/gdsBuffer.cpp b/src/graphics/host_gpu/renderer/gdsBuffer.cpp deleted file mode 100644 index 9c9ce504..00000000 --- a/src/graphics/host_gpu/renderer/gdsBuffer.cpp +++ /dev/null @@ -1,67 +0,0 @@ -#include "graphics/host_gpu/renderer/gdsBuffer.h" - -#include "common/assert.h" -#include "graphics/host_gpu/transfer.h" -#include "graphics/host_gpu/vma.h" - -namespace Libs::Graphics { - -void GdsBuffer::Init() { - if (m_buffer == nullptr) { - m_buffer = std::make_unique(); - - m_buffer->usage = vk::BufferUsageFlagBits::eStorageBuffer; - m_buffer->memory.property = vk::MemoryPropertyFlagBits::eHostVisible | - vk::MemoryPropertyFlagBits::eHostCoherent | - vk::MemoryPropertyFlagBits::eHostCached; - m_graphics.CreateBuffer(DW_SIZE * 4, *m_buffer); - } -} - -void GdsBuffer::Clear(uint64_t dw_offset, uint32_t dw_num, uint32_t clear_value) { - Common::LockGuard lock(m_mutex); - - Init(); - - EXIT_NOT_IMPLEMENTED(dw_offset >= DW_SIZE); - EXIT_NOT_IMPLEMENTED(dw_offset + dw_num > DW_SIZE); - - void* data = nullptr; - m_graphics.MapMemory(m_buffer->memory, data); - - for (uint32_t i = 0; i < dw_num; i++) { - static_cast(data)[dw_offset + i] = clear_value; - } - - m_graphics.UnmapMemory(m_buffer->memory); -} - -void GdsBuffer::Read(uint32_t* dst, uint32_t dw_offset, uint32_t dw_size) { - EXIT_IF(dst == nullptr); - - Common::LockGuard lock(m_mutex); - - Init(); - - EXIT_NOT_IMPLEMENTED(dw_offset >= DW_SIZE); - EXIT_NOT_IMPLEMENTED(dw_offset + dw_size > DW_SIZE); - - void* data = nullptr; - m_graphics.MapMemory(m_buffer->memory, data); - - for (uint32_t i = 0; i < dw_size; i++) { - dst[i] = static_cast(data)[dw_offset + i]; - } - - m_graphics.UnmapMemory(m_buffer->memory); -} - -VulkanBuffer& GdsBuffer::GetBuffer() { - Common::LockGuard lock(m_mutex); - - Init(); - - return *m_buffer; -} - -} // namespace Libs::Graphics diff --git a/src/graphics/host_gpu/renderer/gdsBuffer.h b/src/graphics/host_gpu/renderer/gdsBuffer.h deleted file mode 100644 index 754d2b35..00000000 --- a/src/graphics/host_gpu/renderer/gdsBuffer.h +++ /dev/null @@ -1,39 +0,0 @@ -#ifndef EMULATOR_SRC_GRAPHICS_HOST_GPU_RENDERER_GDSBUFFER_H_ -#define EMULATOR_SRC_GRAPHICS_HOST_GPU_RENDERER_GDSBUFFER_H_ - -#include "common/abi.h" -#include "common/assert.h" -#include "common/common.h" -#include "common/threads.h" -#include "graphics/host_gpu/graphicContext.h" - -#include - -namespace Libs::Graphics { - -class GdsBuffer { -public: - explicit GdsBuffer(GraphicContext& graphics): m_graphics(graphics) { - EXIT_NOT_IMPLEMENTED(!Common::Thread::IsMainThread()); - } - ~GdsBuffer() { KYTY_NOT_IMPLEMENTED; } - KYTY_CLASS_NO_COPY(GdsBuffer); - - void Clear(uint64_t dw_offset, uint32_t dw_num, uint32_t clear_value); - void Read(uint32_t* dst, uint32_t dw_offset, uint32_t dw_size); - - VulkanBuffer& GetBuffer(); - -private: - static constexpr uint64_t DW_SIZE = 0x3000; - - void Init(); - - GraphicContext& m_graphics; - Common::Mutex m_mutex; - std::unique_ptr m_buffer; -}; - -} // namespace Libs::Graphics - -#endif // EMULATOR_SRC_GRAPHICS_HOST_GPU_RENDERER_GDSBUFFER_H_ diff --git a/src/graphics/host_gpu/renderer/gpuResourceManager.cpp b/src/graphics/host_gpu/renderer/gpuResourceManager.cpp index 682389f3..25f7cc49 100644 --- a/src/graphics/host_gpu/renderer/gpuResourceManager.cpp +++ b/src/graphics/host_gpu/renderer/gpuResourceManager.cpp @@ -3,17 +3,14 @@ #include "common/assert.h" #include "graphics/guest_gpu/command_processor/commandProcessor.h" #include "graphics/guest_gpu/graphicsRun.h" -#include "graphics/host_gpu/objects/label.h" -#include "graphics/host_gpu/renderer/render.h" -#include "graphics/host_gpu/renderer/renderContext.h" +#include "graphics/host_gpu/renderer/commandScheduler.h" namespace Libs::Graphics { -GpuResourceManager::GpuResourceManager(GraphicContext& graphics) - : m_page_manager(FaultThunk, this), m_buffer_cache(graphics, m_page_manager, m_resource_mutex), - m_texture_cache(graphics, m_page_manager, m_buffer_cache, m_resource_mutex) { - m_buffer_cache.SetTextureCache(m_texture_cache); -} +GpuResourceManager::GpuResourceManager(GraphicContext& graphics, CommandScheduler& scheduler) + : m_page_manager(FaultThunk, this), + m_buffer_cache(graphics, scheduler, m_page_manager, m_texture_cache, m_resource_mutex), + m_texture_cache(graphics, scheduler, m_page_manager, m_buffer_cache, m_resource_mutex) {} GpuResourceManager::~GpuResourceManager() = default; @@ -24,6 +21,15 @@ bool GpuResourceManager::FaultThunk(void* context, PageFaultAccess access, uint6 bool GpuResourceManager::InvalidateMemory(PageFaultAccess access, uint64_t vaddr, uint64_t size, PageFaultPhase phase) noexcept { + // Let the authoritative image materialize first. A clean overlapping buffer marks a write + // fault CPU-dirty when it begins ownership transfer; doing that before image preflight would + // make the image appear to race a real CPU write. Completion and release retain buffer-first + // ordering so its pending fault is gone before TextureCache publishes the downloaded backing. + if (phase == PageFaultPhase::Invalidate) { + const bool image_handled = m_texture_cache.InvalidateMemory(access, vaddr, size, phase); + const bool buffer_handled = m_buffer_cache.InvalidateMemory(access, vaddr, size, phase); + return buffer_handled || image_handled; + } const bool buffer_handled = m_buffer_cache.InvalidateMemory(access, vaddr, size, phase); const bool image_handled = m_texture_cache.InvalidateMemory(access, vaddr, size, phase); return buffer_handled || image_handled; @@ -33,19 +39,23 @@ bool GpuResourceManager::HandleFault(PageFaultAccess access, uint64_t fault_vadd if (!m_page_manager.IsMapped(fault_vaddr, 1)) { return false; } - if (LabelInCallback()) { - EXIT("unsupported guest-memory fault from an asynchronous GPU label callback, " + if (CommandScheduler::InDeferredOperation()) { + EXIT("unsupported guest-memory fault from an asynchronous GPU completion, " "addr=0x%016" PRIx64 " access=%u\n", fault_vaddr, static_cast(access)); } - if (auto* cp = GraphicsRunCurrentCommandProcessor(); cp != nullptr) { - cp->BeginReadbackTransaction(); - bool handled = false; + bool handled = false; + const auto resolve = [this, access, fault_vaddr, &handled](CommandProcessor& cp) { + cp.BeginReadbackTransaction(); + (void)m_buffer_cache.SynchronizeBacking(fault_vaddr, 1); { ResourceMutex::FaultScope fault(m_resource_mutex); handled = m_page_manager.HandleFault(access, fault_vaddr); } - cp->EndReadbackTransaction(); + cp.EndReadbackTransaction(); + }; + if (auto* cp = Gpu::CurrentCommandProcessor(); cp != nullptr) { + resolve(*cp); return handled; } if (m_resource_mutex.IsOwnedByCurrentThread()) { @@ -53,36 +63,36 @@ bool GpuResourceManager::HandleFault(PageFaultAccess access, uint64_t fault_vadd " access=%u\n", fault_vaddr, static_cast(access)); } - // Stop command-processor jobs before taking the shared cache transaction. External readback - // workers inherit this paused state and therefore never form resource -> submission lock - // inversion. - GraphicsRunSubmissionLock submissions; - ResourceMutex::FaultScope fault(m_resource_mutex); - return m_page_manager.HandleFault(access, fault_vaddr); + EXIT_IF(m_gpu == nullptr); + m_gpu->SendCommandSyncWithProcessor(resolve); + return handled; } void GpuResourceManager::PrepareHostWrite(uint64_t vaddr, uint64_t size) { if (!m_page_manager.HasAnyMapping(vaddr, size)) { return; } - if (LabelInCallback()) { - EXIT("unsupported host write from an asynchronous GPU label callback, addr=0x%016" PRIx64 + if (CommandScheduler::InDeferredOperation()) { + EXIT("unsupported host write from an asynchronous GPU completion, addr=0x%016" PRIx64 " size=0x%016" PRIx64 "\n", vaddr, size); } - const auto handle_range = [this, vaddr, size]() { + const auto handle_range = [this, vaddr, size] { if (!m_page_manager.HandleWriteRange(vaddr, size)) { EXIT("failed to prepare host write, addr=0x%016" PRIx64 " size=0x%016" PRIx64 "\n", vaddr, size); } }; - if (auto* cp = GraphicsRunCurrentCommandProcessor(); cp != nullptr) { - cp->BeginReadbackTransaction(); + const auto resolve = [this, &handle_range](CommandProcessor& cp) { + cp.BeginReadbackTransaction(); { ResourceMutex::FaultScope fault(m_resource_mutex); handle_range(); } - cp->EndReadbackTransaction(); + cp.EndReadbackTransaction(); + }; + if (auto* cp = Gpu::CurrentCommandProcessor(); cp != nullptr) { + resolve(*cp); return; } if (m_resource_mutex.IsOwnedByCurrentThread()) { @@ -90,9 +100,8 @@ void GpuResourceManager::PrepareHostWrite(uint64_t vaddr, uint64_t size) { " size=0x%016" PRIx64 "\n", vaddr, size); } - GraphicsRunSubmissionLock submissions; - ResourceMutex::FaultScope fault(m_resource_mutex); - handle_range(); + EXIT_IF(m_gpu == nullptr); + m_gpu->SendCommandSyncWithProcessor(resolve); } bool GpuResourceManager::IsMapped(uint64_t vaddr, uint64_t size) const noexcept { @@ -107,27 +116,26 @@ void GpuResourceManager::UnmapMemory(uint64_t vaddr, uint64_t size, GpuAccess ac if (!IsMapped(vaddr, size)) { EXIT("cannot unmap an unmapped GPU resource range\n"); } - m_texture_cache.UnmapMemory(vaddr, size); - m_buffer_cache.UnmapMemory(vaddr, size); - m_page_manager.OnGpuUnmap(vaddr, size, access); + const auto unmap = [this, vaddr, size, access] { + m_texture_cache.UnmapMemory(vaddr, size); + m_buffer_cache.UnmapMemory(vaddr, size); + m_page_manager.OnGpuUnmap(vaddr, size, access); + }; + if (m_gpu == nullptr) { + if (m_resource_mutex.IsOwnedByCurrentThread()) { + EXIT("cannot synchronously unmap from a resource transaction\n"); + } + unmap(); + return; + } + Gpu::SubmissionLock submissions(*m_gpu); + m_gpu->SendCommandSync(unmap); } -void GpuResourceManager::FillBuffer(CommandBuffer& command, uint64_t vaddr, uint64_t size, - uint32_t value) { - if (command.IsInvalid()) { - EXIT("cannot fill a buffer without a valid render command context\n"); - } - Common::LockGuard lock(GetRenderContext().GetMutex()); - m_buffer_cache.FillBuffer(&command, vaddr, size, value); -} - -void GpuResourceManager::CopyBuffer(CommandBuffer& command, uint64_t dst_vaddr, uint64_t src_vaddr, - uint64_t size) { - if (command.IsInvalid()) { - EXIT("cannot copy a buffer without a valid render command context\n"); - } - Common::LockGuard lock(GetRenderContext().GetMutex()); - m_buffer_cache.CopyBuffer(&command, dst_vaddr, src_vaddr, size); +void GpuResourceManager::RunGarbageCollector() { + m_texture_cache.ProcessDownloadImages(); + m_texture_cache.RunGarbageCollector(); + m_buffer_cache.RunGarbageCollector(); } } // namespace Libs::Graphics diff --git a/src/graphics/host_gpu/renderer/gpuResourceManager.h b/src/graphics/host_gpu/renderer/gpuResourceManager.h index 2ef0b6fa..24d32fec 100644 --- a/src/graphics/host_gpu/renderer/gpuResourceManager.h +++ b/src/graphics/host_gpu/renderer/gpuResourceManager.h @@ -12,24 +12,25 @@ namespace Libs::Graphics { -class CommandBuffer; +class CommandScheduler; +class Gpu; class GpuResourceManager { public: - explicit GpuResourceManager(GraphicContext& graphics); + GpuResourceManager(GraphicContext& graphics, CommandScheduler& scheduler); ~GpuResourceManager(); KYTY_CLASS_NO_COPY(GpuResourceManager); [[nodiscard]] BufferCache& GetBufferCache() { return m_buffer_cache; } [[nodiscard]] TextureCache& GetTextureCache() { return m_texture_cache; } + void SetGpu(Gpu* gpu) noexcept { m_gpu = gpu; } [[nodiscard]] bool HandleFault(PageFaultAccess access, uint64_t fault_vaddr) noexcept; void PrepareHostWrite(uint64_t vaddr, uint64_t size); [[nodiscard]] bool IsMapped(uint64_t vaddr, uint64_t size) const noexcept; void MapMemory(uint64_t vaddr, uint64_t size, GpuAccess access); void UnmapMemory(uint64_t vaddr, uint64_t size, GpuAccess access); - void FillBuffer(CommandBuffer& command, uint64_t vaddr, uint64_t size, uint32_t value); - void CopyBuffer(CommandBuffer& command, uint64_t dst_vaddr, uint64_t src_vaddr, uint64_t size); + void RunGarbageCollector(); private: static bool FaultThunk(void* context, PageFaultAccess access, uint64_t vaddr, uint64_t size, @@ -41,6 +42,7 @@ private: ResourceMutex m_resource_mutex; BufferCache m_buffer_cache; TextureCache m_texture_cache; + Gpu* m_gpu = nullptr; }; } // namespace Libs::Graphics diff --git a/src/graphics/host_gpu/renderer/image.cpp b/src/graphics/host_gpu/renderer/image.cpp index 87ae7cdc..ee0809b5 100644 --- a/src/graphics/host_gpu/renderer/image.cpp +++ b/src/graphics/host_gpu/renderer/image.cpp @@ -2,481 +2,781 @@ #include "common/assert.h" #include "common/profiler.h" -#include "graphics/guest_gpu/gpu_defs.h" -#include "graphics/guest_gpu/tile.h" -#include "graphics/host_gpu/gpuTiler.h" -#include "graphics/host_gpu/graphicContext.h" -#include "graphics/host_gpu/objects/textureCommon.h" -#include "graphics/host_gpu/regionDefinitions.h" -#include "graphics/host_gpu/renderer/framebufferCache.h" +#include "graphics/host_gpu/renderer/commandScheduler.h" #include "graphics/host_gpu/renderer/imageView.h" -#include "graphics/host_gpu/renderer/renderContext.h" #include "graphics/host_gpu/renderer/renderTarget.h" -#include "graphics/host_gpu/transfer.h" -#include "graphics/host_gpu/vma.h" -#include "graphics/shader/shader.h" +#include "graphics/host_gpu/renderer/streamBuffer.h" +#include "kernel/memory.h" #include +#include +#include +#include namespace Libs::Graphics { namespace { -TextureImageCreateParams MakeImageParams(const ImageInfo& info, bool storage) { - TextureImageCreateParams params {}; - params.fmt = info.format; - params.width = info.width; - params.height = info.height; - params.base_level = SelectImageBackingBaseLevel(storage, info.base_level); - params.levels = info.levels; - params.depth = info.depth; - params.type = info.type; - // Storage image views use identity component mapping. The guest storage write mapping is - // validated before this point and intentionally does not become a Vulkan view swizzle. - params.swizzle = storage ? DstSel(4, 5, 6, 7) : info.swizzle; - params.format_usage = TextureFormatUsage::Sampled | TextureFormatUsage::Storage; - params.required_format_usage = storage - ? TextureFormatUsage::Sampled | TextureFormatUsage::Storage - : TextureFormatUsage::Sampled; - params.view_usage = storage ? TextureFormatUsage::Sampled | TextureFormatUsage::Storage - : TextureFormatUsage::Sampled; - params.image_layout = TextureUploadDestination::MipLevels; - params.allow_cube_view = !storage; - params.compatible_format_views = - storage && (IsRgba8SrgbViewFormat(TextureGetFormat(info.format)) || - info.format == Prospero::GpuEnumValue(Prospero::BufferFormat::k32UInt) || - info.format == Prospero::GpuEnumValue(Prospero::BufferFormat::k32Float)); - params.owner = storage ? "StorageTextureCache" : "TextureCache"; - return params; +[[nodiscard]] vk::ImageType HostImageType(Prospero::ImageType type) { + switch (type) { + case Prospero::ImageType::kColor1D: return vk::ImageType::e1D; + case Prospero::ImageType::kColor3D: return vk::ImageType::e3D; + case Prospero::ImageType::kColor2D: return vk::ImageType::e2D; + default: EXIT("non-base image type: %u\n", static_cast(type)); + } } -bool RenderTargetSupportsStorage(vk::Format format, vk::ImageCreateFlags flags) { - const auto compatible = SrgbStorageViewFormat(format); - const auto required_flags = - vk::ImageCreateFlagBits::eMutableFormat | vk::ImageCreateFlagBits::eExtendedUsage; - const bool compatible_views = (flags & required_flags) == required_flags; - return ImageViewOps::FormatSupportsStorage(format) || - (compatible_views && compatible != vk::Format::eUndefined && - ImageViewOps::FormatSupportsStorage(compatible)); -} - -vk::ImageCreateFlags RenderTargetCreateFlags(vk::Format format) { - const bool compatible_format_view = - IsRgba8SrgbViewFormat(format) || - BgraToRgbaSampledViewFormat(format) != vk::Format::eUndefined || - format == vk::Format::eR8G8B8A8Uint || format == vk::Format::eR16G16B16A16Sfloat || - format == vk::Format::eR16G16B16A16Uint; - return compatible_format_view - ? vk::ImageCreateFlagBits::eMutableFormat | vk::ImageCreateFlagBits::eExtendedUsage - : vk::ImageCreateFlags {0}; -} - -vk::ImageUsageFlags RenderTargetUsage(vk::Format format, vk::ImageCreateFlags flags, - uint32_t samples) { - auto& graphics = GetRenderContext().GetGraphics(); - auto usage = static_cast(vk::ImageUsageFlagBits::eColorAttachment) | - static_cast(vk::ImageUsageFlagBits::eTransferSrc) | - static_cast(vk::ImageUsageFlagBits::eTransferDst); - if (samples == 1) { - usage |= vk::ImageUsageFlagBits::eSampled; - if (RenderTargetSupportsStorage(format, flags)) { - usage |= vk::ImageUsageFlagBits::eStorage; +[[nodiscard]] vk::ImageCreateFlags ImageCreateFlags(const ImageInfo& info) { + vk::ImageCreateFlags flags {}; + if (DepthAspectTransferFormat(info.pixel_format) == vk::Format::eUndefined) { + flags |= vk::ImageCreateFlagBits::eMutableFormat; + flags |= vk::ImageCreateFlagBits::eExtendedUsage; + if (Prospero::BlockCompressedBytesPerBlock(info.guest_format) != 0) { + flags |= vk::ImageCreateFlagBits::eBlockTexelViewCompatible; } } - vk::ImageFormatProperties properties {}; - if (graphics.GetImageFormatProperties(format, vk::ImageType::e2D, vk::ImageTiling::eOptimal, - usage, flags, &properties) != vk::Result::eSuccess || - !static_cast(properties.sampleCounts & vulkan_sample_count(samples))) { - EXIT("TextureCache: render-target format does not support required usage, format=%d " - "usage=0x%x samples=%u supported=0x%x\n", - static_cast(format), static_cast(usage), samples, - static_cast(properties.sampleCounts)); + if (info.IsVolume()) { + flags |= vk::ImageCreateFlagBits::e2DArrayCompatible; + } + return flags; +} + +[[nodiscard]] bool HasFormatFeature(vk::FormatProperties properties, + vk::FormatFeatureFlagBits feature) { + return static_cast(properties.optimalTilingFeatures & feature); +} + +[[nodiscard]] vk::ImageUsageFlags ImageUsageFlags(GraphicContext& graphics, const ImageInfo& info) { + const auto properties = graphics.GetFormatProperties(info.pixel_format); + auto usage = vk::ImageUsageFlagBits::eTransferSrc | vk::ImageUsageFlagBits::eTransferDst; + if (HasFormatFeature(properties, vk::FormatFeatureFlagBits::eSampledImage)) { + usage |= vk::ImageUsageFlagBits::eSampled; + } + if (DepthAspectTransferFormat(info.pixel_format) != vk::Format::eUndefined) { + usage |= vk::ImageUsageFlagBits::eDepthStencilAttachment; + return usage; + } + if (HasFormatFeature(properties, vk::FormatFeatureFlagBits::eColorAttachment)) { + usage |= vk::ImageUsageFlagBits::eColorAttachment; + } + if (info.samples == 1 && + HasFormatFeature(properties, vk::FormatFeatureFlagBits::eStorageImage)) { + usage |= vk::ImageUsageFlagBits::eStorage; + } else if (info.samples == 1) { + const auto compatible = SrgbStorageViewFormat(info.pixel_format); + if (compatible != vk::Format::eUndefined && + HasFormatFeature(graphics.GetFormatProperties(compatible), + vk::FormatFeatureFlagBits::eStorageImage)) { + usage |= vk::ImageUsageFlagBits::eStorage; + } } return usage; } -[[nodiscard]] uint32_t RenderTargetTransferFormatImpl(uint32_t bytes_per_element) { +void ValidateRange(GuestRange range, const char* name) { + if ((range.address == 0) != (range.size == 0) || + (range.address != 0 && (range.address >= TRACKER_ADDRESS_SIZE || + range.size > TRACKER_ADDRESS_SIZE - range.address))) { + EXIT("invalid %s image range: address=0x%016llx size=0x%016llx\n", name, + static_cast(range.address), + static_cast(range.size)); + } +} + +} // namespace + +vk::ImageAspectFlags Image::FullAspectMask(vk::Format format) noexcept { + switch (format) { + case vk::Format::eD16Unorm: + case vk::Format::eX8D24UnormPack32: + case vk::Format::eD32Sfloat: return vk::ImageAspectFlagBits::eDepth; + case vk::Format::eS8Uint: return vk::ImageAspectFlagBits::eStencil; + case vk::Format::eD16UnormS8Uint: + case vk::Format::eD24UnormS8Uint: + case vk::Format::eD32SfloatS8Uint: + return vk::ImageAspectFlagBits::eDepth | vk::ImageAspectFlagBits::eStencil; + default: return vk::ImageAspectFlagBits::eColor; + } +} + +Image::Barriers Image::GetBarriers(vk::ImageLayout destination_layout, + vk::AccessFlags2 destination_access, + vk::PipelineStageFlags2 destination_stage, + std::optional range) { + auto& state = backing.state; + auto& subresource_states = backing.subresource_states; + + const bool partial = + range && (range->base_level != 0 || range->level_count != info.resources.levels || + range->base_layer != 0 || range->layer_count != info.resources.layers); + const bool has_subresource_states = !subresource_states.empty(); + + Barriers barriers; + if (partial || has_subresource_states) { + if (!has_subresource_states) { + subresource_states.resize(info.resources.levels * info.resources.layers, state); + } + + const uint32_t base_level = partial ? range->base_level : 0; + const uint32_t level_count = partial ? range->level_count : info.resources.levels; + const uint32_t base_layer = partial ? range->base_layer : 0; + const uint32_t layer_count = partial ? range->layer_count : info.resources.layers; + for (uint32_t level = base_level; level < base_level + level_count; level++) { + for (uint32_t layer = base_layer; layer < base_layer + layer_count; layer++) { + const auto index = level * info.resources.layers + layer; + EXIT_IF(index >= subresource_states.size()); + auto& subresource_state = subresource_states[index]; + + constexpr auto write_access = vk::AccessFlagBits2::eTransferWrite | + vk::AccessFlagBits2::eShaderWrite | + vk::AccessFlagBits2::eMemoryWrite; + const bool repeated_write = + static_cast(subresource_state.access_mask & write_access); + if (subresource_state.layout != destination_layout || + subresource_state.access_mask != destination_access || repeated_write) { + vk::ImageMemoryBarrier2 barrier {}; + barrier.srcStageMask = subresource_state.pl_stage; + barrier.srcAccessMask = subresource_state.access_mask; + barrier.dstStageMask = destination_stage; + barrier.dstAccessMask = destination_access; + barrier.oldLayout = subresource_state.layout; + barrier.newLayout = destination_layout; + barrier.srcQueueFamilyIndex = VK_QUEUE_FAMILY_IGNORED; + barrier.dstQueueFamilyIndex = VK_QUEUE_FAMILY_IGNORED; + barrier.image = backing.image; + barrier.subresourceRange.aspectMask = FullAspectMask(backing.format); + barrier.subresourceRange.baseMipLevel = level; + barrier.subresourceRange.levelCount = 1; + barrier.subresourceRange.baseArrayLayer = layer; + barrier.subresourceRange.layerCount = 1; + barriers.push_back(barrier); + subresource_state = {destination_stage, destination_access, destination_layout}; + } + } + } + + if (!partial) { + subresource_states.clear(); + } + } else { + constexpr auto write_access = vk::AccessFlagBits2::eTransferWrite | + vk::AccessFlagBits2::eShaderWrite | + vk::AccessFlagBits2::eMemoryWrite; + const bool repeated_write = static_cast(state.access_mask & write_access); + if (state.layout == destination_layout && state.access_mask == destination_access && + !repeated_write) { + return {}; + } + + vk::ImageMemoryBarrier2 barrier {}; + barrier.srcStageMask = state.pl_stage; + barrier.srcAccessMask = state.access_mask; + barrier.dstStageMask = destination_stage; + barrier.dstAccessMask = destination_access; + barrier.oldLayout = state.layout; + barrier.newLayout = destination_layout; + barrier.srcQueueFamilyIndex = VK_QUEUE_FAMILY_IGNORED; + barrier.dstQueueFamilyIndex = VK_QUEUE_FAMILY_IGNORED; + barrier.image = backing.image; + barrier.subresourceRange.aspectMask = FullAspectMask(backing.format); + barrier.subresourceRange.baseMipLevel = 0; + barrier.subresourceRange.levelCount = VK_REMAINING_MIP_LEVELS; + barrier.subresourceRange.baseArrayLayer = 0; + barrier.subresourceRange.layerCount = VK_REMAINING_ARRAY_LAYERS; + barriers.push_back(barrier); + } + + state = {destination_stage, destination_access, destination_layout}; + return barriers; +} + +void Image::Transit(vk::ImageLayout destination_layout, vk::AccessFlags2 destination_access, + std::optional range, + vk::CommandBuffer command_buffer) { + const auto transfer_access = + vk::AccessFlagBits2::eTransferRead | vk::AccessFlagBits2::eTransferWrite; + vk::PipelineStageFlags2 destination_stage {}; + if (static_cast(destination_access & transfer_access)) { + destination_stage |= vk::PipelineStageFlagBits2::eTransfer; + } + if (!destination_access || + static_cast(destination_access & ~vk::AccessFlags2 {transfer_access})) { + destination_stage |= vk::PipelineStageFlagBits2::eAllGraphics | + vk::PipelineStageFlagBits2::eComputeShader; + } + const auto barriers = + GetBarriers(destination_layout, destination_access, destination_stage, range); + if (barriers.empty()) { + return; + } + if (m_scheduler != nullptr) { + m_scheduler->EndRendering(); + } + vk::DependencyInfo dependency {}; + dependency.imageMemoryBarrierCount = static_cast(barriers.size()); + dependency.pImageMemoryBarriers = barriers.data(); + command_buffer.pipelineBarrier2(dependency); +} + +void Image::Upload(std::span copies, vk::Buffer buffer, + uint64_t offset, uint64_t size) { + EXIT_IF(m_scheduler == nullptr || copies.empty() || buffer == nullptr || + size == 0); + m_scheduler->EndRendering(); + vk::BufferMemoryBarrier2 buffer_barrier {}; + buffer_barrier.srcStageMask = vk::PipelineStageFlagBits2::eAllCommands; + buffer_barrier.srcAccessMask = vk::AccessFlagBits2::eMemoryWrite; + buffer_barrier.dstStageMask = vk::PipelineStageFlagBits2::eTransfer; + buffer_barrier.dstAccessMask = vk::AccessFlagBits2::eTransferRead; + buffer_barrier.srcQueueFamilyIndex = VK_QUEUE_FAMILY_IGNORED; + buffer_barrier.dstQueueFamilyIndex = VK_QUEUE_FAMILY_IGNORED; + buffer_barrier.buffer = buffer; + buffer_barrier.offset = offset; + buffer_barrier.size = size; + const auto image_barriers = + GetBarriers(vk::ImageLayout::eTransferDstOptimal, + vk::AccessFlagBits2::eTransferWrite, + vk::PipelineStageFlagBits2::eCopy, {}); + vk::DependencyInfo dependency {}; + dependency.dependencyFlags = vk::DependencyFlagBits::eByRegion; + dependency.bufferMemoryBarrierCount = 1; + dependency.pBufferMemoryBarriers = &buffer_barrier; + dependency.imageMemoryBarrierCount = static_cast(image_barriers.size()); + dependency.pImageMemoryBarriers = image_barriers.data(); + auto command = m_scheduler->Current().Handle(); + command.pipelineBarrier2(dependency); + command.copyBufferToImage(buffer, backing.image, vk::ImageLayout::eTransferDstOptimal, + static_cast(copies.size()), copies.data()); + buffer_barrier.srcStageMask = vk::PipelineStageFlagBits2::eTransfer; + buffer_barrier.srcAccessMask = vk::AccessFlagBits2::eTransferRead; + buffer_barrier.dstStageMask = vk::PipelineStageFlagBits2::eAllCommands; + buffer_barrier.dstAccessMask = + vk::AccessFlagBits2::eMemoryRead | vk::AccessFlagBits2::eMemoryWrite; + dependency.imageMemoryBarrierCount = 0; + dependency.pImageMemoryBarriers = nullptr; + command.pipelineBarrier2(dependency); + Transit(vk::ImageLayout::eGeneral, + vk::AccessFlagBits2::eShaderRead | vk::AccessFlagBits2::eTransferRead, {}, + command); +} + +void Image::Download(std::span copies, vk::Buffer buffer, + uint64_t offset, uint64_t size) { + EXIT_IF(m_scheduler == nullptr || copies.empty() || buffer == nullptr || size == 0); + m_scheduler->EndRendering(); + vk::BufferMemoryBarrier2 buffer_barrier {}; + buffer_barrier.srcStageMask = vk::PipelineStageFlagBits2::eAllCommands; + buffer_barrier.srcAccessMask = + vk::AccessFlagBits2::eMemoryRead | vk::AccessFlagBits2::eMemoryWrite; + buffer_barrier.dstStageMask = vk::PipelineStageFlagBits2::eCopy; + buffer_barrier.dstAccessMask = vk::AccessFlagBits2::eTransferWrite; + buffer_barrier.srcQueueFamilyIndex = VK_QUEUE_FAMILY_IGNORED; + buffer_barrier.dstQueueFamilyIndex = VK_QUEUE_FAMILY_IGNORED; + buffer_barrier.buffer = buffer; + buffer_barrier.offset = offset; + buffer_barrier.size = size; + const auto image_barriers = + GetBarriers(vk::ImageLayout::eTransferSrcOptimal, + vk::AccessFlagBits2::eTransferRead, + vk::PipelineStageFlagBits2::eCopy, {}); + vk::DependencyInfo dependency {}; + dependency.dependencyFlags = vk::DependencyFlagBits::eByRegion; + dependency.bufferMemoryBarrierCount = 1; + dependency.pBufferMemoryBarriers = &buffer_barrier; + dependency.imageMemoryBarrierCount = static_cast(image_barriers.size()); + dependency.pImageMemoryBarriers = image_barriers.data(); + auto command = m_scheduler->Current().Handle(); + command.pipelineBarrier2(dependency); + command.copyImageToBuffer(backing.image, vk::ImageLayout::eTransferSrcOptimal, buffer, + static_cast(copies.size()), copies.data()); + buffer_barrier.srcStageMask = vk::PipelineStageFlagBits2::eCopy; + buffer_barrier.srcAccessMask = vk::AccessFlagBits2::eTransferWrite; + buffer_barrier.dstStageMask = vk::PipelineStageFlagBits2::eAllCommands; + buffer_barrier.dstAccessMask = + vk::AccessFlagBits2::eMemoryRead | vk::AccessFlagBits2::eMemoryWrite; + dependency.imageMemoryBarrierCount = 0; + dependency.pImageMemoryBarriers = nullptr; + command.pipelineBarrier2(dependency); +} + +std::pair +Image::SanitizeCopyLayers(const Image& source, const Image& destination, uint32_t depth) { + const auto source_type = source.backing.image_type; + const auto destination_type = destination.backing.image_type; + uint32_t source_layers = source.backing.layers; + uint32_t destination_layers = destination.backing.layers; + if (source_type == vk::ImageType::e3D) { + source_layers = 1; + } + if (destination_type == vk::ImageType::e3D) { + destination_layers = 1; + } + if (source_type == destination_type) { + source_layers = destination_layers = + std::min(source_layers, destination_layers); + } else if (source_type == vk::ImageType::e2D && + destination_type == vk::ImageType::e3D) { + source_layers = depth; + } else if (source_type == vk::ImageType::e3D && + destination_type == vk::ImageType::e2D) { + destination_layers = depth; + } + return {source_layers, destination_layers}; +} + +void Image::CopyImage(Image& source) { + EXIT_IF(m_scheduler == nullptr || source.backing.samples != backing.samples); + m_scheduler->EndRendering(); + const uint32_t levels = + std::min(source.backing.mip_levels, backing.mip_levels); + const uint32_t base_depth = backing.image_type == vk::ImageType::e3D + ? backing.extent.depth + : source.backing.extent.depth; + const auto source_aspect = + FullAspectMask(source.backing.format) & ~vk::ImageAspectFlagBits::eStencil; + const auto destination_aspect = + FullAspectMask(backing.format) & ~vk::ImageAspectFlagBits::eStencil; + std::vector copies; + copies.reserve(levels); + for (uint32_t level = 0; level < levels; level++) { + const auto width = std::max(source.backing.extent.width >> level, 1u); + const auto height = std::max(source.backing.extent.height >> level, 1u); + const auto depth = std::max(base_depth >> level, 1u); + const auto [source_layers, destination_layers] = + SanitizeCopyLayers(source, *this, depth); + vk::ImageCopy copy {}; + copy.srcSubresource = {source_aspect, level, 0, 1}; + copy.dstSubresource = {destination_aspect, level, 0, 1}; + if (source.backing.image_type == backing.image_type) { + if (source.backing.image_type == vk::ImageType::e3D) { + copy.extent = {width, height, depth}; + } else { + copy.srcSubresource.layerCount = + std::min(source_layers, destination_layers); + copy.dstSubresource.layerCount = copy.srcSubresource.layerCount; + copy.extent = {width, height, 1}; + } + } else if (source.backing.image_type == vk::ImageType::e2D) { + copy.srcSubresource.layerCount = source_layers; + copy.extent = {width, height, source_layers}; + } else { + copy.dstSubresource.layerCount = destination_layers; + copy.extent = {width, height, destination_layers}; + } + copies.push_back(copy); + } + if (copies.empty()) { + return; + } + auto command = m_scheduler->Current().Handle(); + source.Transit(vk::ImageLayout::eTransferSrcOptimal, + vk::AccessFlagBits2::eTransferRead, {}, command); + Transit(vk::ImageLayout::eTransferDstOptimal, + vk::AccessFlagBits2::eTransferWrite, {}, command); + command.copyImage(source.backing.image, vk::ImageLayout::eTransferSrcOptimal, + backing.image, vk::ImageLayout::eTransferDstOptimal, + static_cast(copies.size()), copies.data()); + Transit(vk::ImageLayout::eGeneral, + vk::AccessFlagBits2::eShaderRead | vk::AccessFlagBits2::eTransferRead, {}, + command); +} + +void Image::Resolve(Image& source, const ImageSubresourceRange& source_range, + const ImageSubresourceRange& destination_range) { + EXIT_IF(m_scheduler == nullptr || backing.samples != 1 || + source.backing.image_type != vk::ImageType::e2D || + backing.image_type != vk::ImageType::e2D || + source_range.level_count != 1 || destination_range.level_count != 1 || + source_range.base_level >= source.backing.mip_levels || + destination_range.base_level >= backing.mip_levels || + source_range.base_layer >= source.backing.layers || + destination_range.base_layer >= backing.layers); + const auto layers = std::min( + {source_range.layer_count, destination_range.layer_count, + source.backing.layers - source_range.base_layer, + backing.layers - destination_range.base_layer}); + const auto source_width = + std::max(source.backing.extent.width >> source_range.base_level, 1u); + const auto source_height = + std::max(source.backing.extent.height >> source_range.base_level, 1u); + const auto destination_width = + std::max(backing.extent.width >> destination_range.base_level, 1u); + const auto destination_height = + std::max(backing.extent.height >> destination_range.base_level, 1u); + const bool copy = source.backing.samples == 1; + EXIT_IF(layers == 0 || info.extent.width > source_width || + info.extent.height > source_height || info.extent.width > destination_width || + info.extent.height > destination_height || + (copy ? !ImageViewOps::FormatsCompatible(source.backing.format, backing.format) + : source.backing.format != backing.format)); + auto resolved_source_range = source_range; + auto resolved_destination_range = destination_range; + resolved_source_range.layer_count = layers; + resolved_destination_range.layer_count = layers; + const vk::Extent3D resolve_extent {info.extent.width, info.extent.height, 1}; + + m_scheduler->EndRendering(); + auto command = m_scheduler->Current().Handle(); + source.Transit(vk::ImageLayout::eTransferSrcOptimal, + vk::AccessFlagBits2::eTransferRead, resolved_source_range, command); + Transit(vk::ImageLayout::eTransferDstOptimal, + vk::AccessFlagBits2::eTransferWrite, resolved_destination_range, command); + if (copy) { + vk::ImageCopy region {}; + region.srcSubresource = {vk::ImageAspectFlagBits::eColor, + resolved_source_range.base_level, + resolved_source_range.base_layer, layers}; + region.dstSubresource = {vk::ImageAspectFlagBits::eColor, + resolved_destination_range.base_level, + resolved_destination_range.base_layer, layers}; + region.extent = resolve_extent; + command.copyImage(source.backing.image, vk::ImageLayout::eTransferSrcOptimal, + backing.image, vk::ImageLayout::eTransferDstOptimal, region); + } else { + vk::ImageResolve region {}; + region.srcSubresource = {vk::ImageAspectFlagBits::eColor, + resolved_source_range.base_level, + resolved_source_range.base_layer, layers}; + region.dstSubresource = {vk::ImageAspectFlagBits::eColor, + resolved_destination_range.base_level, + resolved_destination_range.base_layer, layers}; + region.extent = resolve_extent; + command.resolveImage(source.backing.image, vk::ImageLayout::eTransferSrcOptimal, + backing.image, vk::ImageLayout::eTransferDstOptimal, region); + } +} + +uint32_t Image::CopyRows(uint64_t row_size, uint32_t rows, uint64_t capacity) noexcept { + if (row_size == 0 || rows == 0 || row_size > capacity) { + return 0; + } + return static_cast(std::min(rows, capacity / row_size)); +} + +void Image::CopyImageWithBuffer(Image& source, Buffer& buffer) { + EXIT_IF(m_scheduler == nullptr || buffer.Handle() == nullptr || + source.backing.samples != 1 || backing.samples != 1); + m_scheduler->EndRendering(); + const uint32_t levels = + std::min(source.backing.mip_levels, backing.mip_levels); + const auto source_aspect = + FullAspectMask(source.backing.format) & ~vk::ImageAspectFlagBits::eStencil; + const auto destination_aspect = + FullAspectMask(backing.format) & ~vk::ImageAspectFlagBits::eStencil; + const auto source_bytes = DepthAspectTransferBytes(source.backing.format) != 0 + ? DepthAspectTransferBytes(source.backing.format) + : source.info.bytes_per_block; + const auto destination_bytes = DepthAspectTransferBytes(backing.format) != 0 + ? DepthAspectTransferBytes(backing.format) + : info.bytes_per_block; + const uint32_t source_block = source.info.IsBlock() ? 4u : 1u; + const uint32_t destination_block = info.IsBlock() ? 4u : 1u; + EXIT_IF(levels == 0 || source_bytes == 0 || source_bytes != destination_bytes || + source_block != destination_block); + + vk::BufferMemoryBarrier2 barrier {}; + barrier.srcStageMask = vk::PipelineStageFlagBits2::eTransfer; + barrier.srcAccessMask = vk::AccessFlagBits2::eTransferRead; + barrier.dstStageMask = vk::PipelineStageFlagBits2::eTransfer; + barrier.dstAccessMask = vk::AccessFlagBits2::eTransferWrite; + barrier.srcQueueFamilyIndex = VK_QUEUE_FAMILY_IGNORED; + barrier.dstQueueFamilyIndex = VK_QUEUE_FAMILY_IGNORED; + barrier.buffer = buffer.Handle(); + barrier.offset = 0; + vk::DependencyInfo dependency {}; + dependency.dependencyFlags = vk::DependencyFlagBits::eByRegion; + dependency.bufferMemoryBarrierCount = 1; + dependency.pBufferMemoryBarriers = &barrier; + auto command = m_scheduler->Current().Handle(); + source.Transit(vk::ImageLayout::eTransferSrcOptimal, + vk::AccessFlagBits2::eTransferRead, {}, command); + Transit(vk::ImageLayout::eTransferDstOptimal, + vk::AccessFlagBits2::eTransferWrite, {}, command); + for (uint32_t level = 0; level < levels; level++) { + const auto width = std::max(source.backing.extent.width >> level, 1u); + const auto height = std::max(source.backing.extent.height >> level, 1u); + const auto source_depth = source.backing.image_type == vk::ImageType::e3D + ? std::max(source.backing.extent.depth >> level, 1u) + : source.backing.layers; + const auto destination_depth = backing.image_type == vk::ImageType::e3D + ? std::max(backing.extent.depth >> level, 1u) + : backing.layers; + const auto slices = std::min(source_depth, destination_depth); + const auto block_rows = (height + source_block - 1) / source_block; + const auto row_size = + static_cast((width + source_block - 1) / source_block) * source_bytes; + const auto rows_per_copy = CopyRows(row_size, block_rows, buffer.Size()); + EXIT_IF(slices == 0 || rows_per_copy == 0); + for (uint32_t slice = 0; slice < slices; slice++) { + for (uint32_t block_row = 0; block_row < block_rows; + block_row += rows_per_copy) { + const auto copy_rows = std::min(rows_per_copy, block_rows - block_row); + const auto y = block_row * source_block; + const auto copy_height = + std::min(copy_rows * source_block, height - y); + const auto copy_size = row_size * copy_rows; + vk::BufferImageCopy source_copy {}; + source_copy.imageSubresource = { + source_aspect, level, + source.backing.image_type == vk::ImageType::e3D ? 0u : slice, 1}; + source_copy.imageOffset = { + 0, static_cast(y), + source.backing.image_type == vk::ImageType::e3D + ? static_cast(slice) + : 0}; + source_copy.imageExtent = {width, copy_height, 1}; + auto destination_copy = source_copy; + destination_copy.imageSubresource = { + destination_aspect, level, + backing.image_type == vk::ImageType::e3D ? 0u : slice, 1}; + destination_copy.imageOffset.z = + backing.image_type == vk::ImageType::e3D + ? static_cast(slice) + : 0; + barrier.size = copy_size; + barrier.srcAccessMask = vk::AccessFlagBits2::eTransferRead; + barrier.dstAccessMask = vk::AccessFlagBits2::eTransferWrite; + command.pipelineBarrier2(dependency); + command.copyImageToBuffer(source.backing.image, + vk::ImageLayout::eTransferSrcOptimal, + buffer.Handle(), source_copy); + barrier.srcAccessMask = vk::AccessFlagBits2::eTransferWrite; + barrier.dstAccessMask = vk::AccessFlagBits2::eTransferRead; + command.pipelineBarrier2(dependency); + command.copyBufferToImage(buffer.Handle(), backing.image, + vk::ImageLayout::eTransferDstOptimal, + destination_copy); + } + } + } + Transit(vk::ImageLayout::eGeneral, + vk::AccessFlagBits2::eShaderRead | vk::AccessFlagBits2::eTransferRead, {}, + command); +} + +void Image::CopyMip(Image& source, uint32_t mip, uint32_t layer) { + EXIT_IF(m_scheduler == nullptr || source.backing.samples != backing.samples || + mip >= backing.mip_levels || layer >= backing.layers); + m_scheduler->EndRendering(); + const auto width = std::max(backing.extent.width >> mip, 1u); + const auto height = std::max(backing.extent.height >> mip, 1u); + const auto depth = std::max(backing.extent.depth >> mip, 1u); + EXIT_IF(width != source.backing.extent.width || + height != source.backing.extent.height); + const auto [source_layers, destination_layers] = + SanitizeCopyLayers(source, *this, depth); + const auto aspects = FullAspectMask(source.backing.format); + EXIT_IF(aspects != FullAspectMask(backing.format)); + std::array copies {}; + uint32_t copy_count = 0; + for (const auto aspect: {vk::ImageAspectFlagBits::eColor, vk::ImageAspectFlagBits::eDepth, + vk::ImageAspectFlagBits::eStencil}) { + if (!static_cast(aspects & aspect)) { + continue; + } + auto& copy = copies[copy_count++]; + copy.srcSubresource = {aspect, 0, 0, source_layers}; + copy.dstSubresource = {aspect, mip, layer, destination_layers}; + copy.extent = {width, height, depth}; + } + auto command = m_scheduler->Current().Handle(); + Transit(vk::ImageLayout::eTransferDstOptimal, + vk::AccessFlagBits2::eTransferWrite, {}, command); + source.Transit(vk::ImageLayout::eTransferSrcOptimal, + vk::AccessFlagBits2::eTransferRead, {}, command); + command.copyImage(source.backing.image, vk::ImageLayout::eTransferSrcOptimal, + backing.image, vk::ImageLayout::eTransferDstOptimal, copy_count, + copies.data()); + Transit(vk::ImageLayout::eGeneral, + vk::AccessFlagBits2::eShaderRead | vk::AccessFlagBits2::eTransferRead, {}, + command); +} + +namespace ImageOps { + +void Validate(const ImageInfo& info) { + ValidateRange(info.data, "data"); + ValidateRange(info.stencil, "stencil"); + + if (info.pixel_format == vk::Format::eUndefined) { + const bool metadata_empty = + info.metadata.range.address == 0 && info.metadata.range.size == 0 && + info.metadata.kind == ImageMetadataKind::None && + info.metadata.control == 0 && + info.metadata.compression == VideoOutCompression::Uncompressed && + !info.metadata.stencil_compressed; + if (info.data.Empty() || info.HasStencil() || !metadata_empty || info.extent.width == 0 || + info.extent.height == 0 || info.extent.depth == 0 || info.resources.levels != 1 || + info.resources.layers != 1 || info.samples != 1 || info.pitch != 0 || + info.bytes_per_block != 0) { + EXIT("invalid stencil association image\n"); + } + return; + } + + if (info.extent.width == 0 || info.extent.height == 0 || info.extent.depth == 0 || + info.resources.levels == 0 || + info.resources.levels > info.mip_layout.size() || info.resources.layers == 0 || + info.samples == 0 || vulkan_sample_count(info.samples) == vk::SampleCountFlagBits {} || + info.bytes_per_block == 0 || (info.data.address != 0 && info.pitch == 0)) { + EXIT("invalid image geometry or format\n"); + } + + switch (info.type) { + case Prospero::ImageType::kColor1D: + if (info.extent.height != 1 || info.extent.depth != 1) { + EXIT("invalid 1D image shape\n"); + } + break; + case Prospero::ImageType::kColor3D: + if (info.resources.layers != 1) { + EXIT("3D images cannot have array layers\n"); + } + break; + case Prospero::ImageType::kColor2D: + if (info.extent.depth != 1) { + EXIT("invalid 2D image shape\n"); + } + break; + default: EXIT("non-base image type: %u\n", static_cast(info.type)); + } + if (info.samples > 1 && info.resources.levels != 1) { + EXIT("multisampled images cannot have mip levels\n"); + } + + if (info.metadata.stencil_compressed && !info.HasStencil()) { + EXIT("compressed stencil metadata requires a stencil plane\n"); + } + switch (info.metadata.kind) { + case ImageMetadataKind::None: + if (info.metadata.range.address != 0 || info.metadata.range.size != 0 || + info.metadata.control != 0 || + info.metadata.compression != VideoOutCompression::Uncompressed || + info.metadata.stencil_compressed) { + EXIT("metadata-free image has metadata state\n"); + } + break; + case ImageMetadataKind::Htile: + ValidateRange(info.metadata.range, "HTILE"); + if (info.metadata.range.Empty() || + info.metadata.compression != VideoOutCompression::Uncompressed) { + EXIT("invalid HTILE metadata\n"); + } + break; + case ImageMetadataKind::Dcc: + if (info.metadata.range.address == 0 || + info.metadata.range.address >= TRACKER_ADDRESS_SIZE || + (info.metadata.range.size != 0 && + info.metadata.range.size > TRACKER_ADDRESS_SIZE - info.metadata.range.address) || + info.metadata.compression == VideoOutCompression::Unsupported) { + EXIT("invalid DCC metadata\n"); + } + break; + } +} + +uint32_t RenderTargetTransferFormat(uint32_t bytes_per_element) { switch (bytes_per_element) { case 1: return Prospero::GpuEnumValue(Prospero::BufferFormat::k8UNorm); case 2: return Prospero::GpuEnumValue(Prospero::BufferFormat::k16UNorm); case 4: return Prospero::GpuEnumValue(Prospero::BufferFormat::k32Float); case 8: return Prospero::GpuEnumValue(Prospero::BufferFormat::k16_16_16_16Float); case 16: return Prospero::GpuEnumValue(Prospero::BufferFormat::k32_32_32_32Float); - default: - EXIT("TextureCache: unsupported render-target element size: %u\n", bytes_per_element); - } -} - -static constexpr uint32_t DummyTextureSwizzle() { - return Prospero::GpuEnumValue(Prospero::CompSwizzle::kRed) | - (Prospero::GpuEnumValue(Prospero::CompSwizzle::kGreen) << 3u) | - (Prospero::GpuEnumValue(Prospero::CompSwizzle::kBlue) << 6u) | - (Prospero::GpuEnumValue(Prospero::CompSwizzle::kAlpha) << 9u); -} - -TextureImageCreateParams MakeDummyTextureParams(bool uint_format, bool image_3d, - TextureFormatUsage usage, const char* owner) { - TextureImageCreateParams params {}; - params.fmt = static_cast( - Prospero::GpuEnumValue(uint_format ? Prospero::BufferFormat::k8_8_8_8UInt - : Prospero::BufferFormat::k8_8_8_8UNorm)); - params.width = 1; - params.height = 1; - params.base_level = 0; - params.levels = 1; - params.depth = 1; - params.type = Prospero::GpuEnumValue(image_3d ? Prospero::ImageType::kColor3D - : Prospero::ImageType::kColor2D); - params.swizzle = DummyTextureSwizzle(); - params.format_usage = usage; - params.required_format_usage = usage; - params.view_usage = usage; - params.image_layout = TextureUploadDestination::MipLevels; - params.allow_cube_view = true; - params.storage_swizzle_fallback = TextureHasFormatUsage(usage, TextureFormatUsage::Storage); - params.owner = owner; - return params; -} - -} // namespace - -namespace ImageOps { - -uint32_t RenderTargetTransferFormat(uint32_t bytes_per_element) { - return RenderTargetTransferFormatImpl(bytes_per_element); -} - -GpuTextureVulkanImage* CreateTexture(const ImageInfo& info, bool storage, - vk::ComponentMapping& components) { - auto* image = storage ? static_cast(new StorageTextureVulkanImage) - : new TextureVulkanImage; - components = TextureCreateImage(*image, MakeImageParams(info, storage)); - return image; -} - -void CreateTextureViews(GpuTextureVulkanImage& image, const ImageInfo& info, bool storage, - vk::ComponentMapping components) { - if (storage) { - TextureCreateImageViews(image, components, info.type, 0, 0, 1, info.depth, false, - TextureFormatUsage::Sampled | TextureFormatUsage::Storage); - } else { - TextureCreateImageViews(image, components, info.type, info.base_array, info.base_level, - info.view_levels, info.depth, true, TextureFormatUsage::Sampled); - } -} - -void UploadRenderTargetLayers(RenderTextureVulkanImage& image, const RenderTargetInfo& info, - uint32_t base_layer, uint32_t layer_count, bool refresh) { - if (info.layers == 0 || info.size % info.layers != 0 || layer_count == 0 || - base_layer >= info.layers || layer_count > info.layers - base_layer || - base_layer >= image.layers || layer_count > image.layers - base_layer) { - EXIT("TextureCache: invalid render-target layer upload, base=%u count=%u " - "info_layers=%u image_layers=%u size=0x%016" PRIx64 "\n", - base_layer, layer_count, info.layers, image.layers, info.size); - } - if (info.samples != 1 || image.samples != 1) { - EXIT("TextureCache: multisampled render-target upload is unsupported, samples=%u/%u\n", - info.samples, image.samples); - } - if (refresh) { - Transfer::WaitForQueueIdle(); - } - const auto slice_size = info.size / info.layers; - const auto upload_size = slice_size * layer_count; - const bool standard64 = IsSupportedStandard64RenderTarget(info); - if (standard64 || info.levels > 1 || info.layers > 1) { - const auto format = RenderTargetTransferFormat(info.bytes_per_element); - auto layout = TextureCalcUploadLayout(format, info.width, info.height, info.levels, - layer_count, info.pitch, info.tile_mode, upload_size, - false, false, "TextureCache render target"); - const bool render_target_tiled = - info.tile_mode == Prospero::GpuEnumValue(Prospero::TileMode::kRenderTarget); - if (!standard64 && - ((render_target_tiled && layout.tile_family != TileBlockFamily::RenderTarget64KB) || - layout.pitch != info.pitch)) { - EXIT("TextureCache: unsupported render-target mip upload layout, pitch=%u/%u tile=%u\n", - info.pitch, layout.pitch, info.tile_mode); - } - auto regions = TextureBuildUploadRegions(layout, info.format, info.width, info.height, - layer_count, info.levels, true, false, - TextureUploadDestination::MipLevels); - for (auto& region: regions) { - region.dst_layer += base_layer; - } - const auto source_address = info.address + slice_size * base_layer; - TextureUploadGuestImage(image, reinterpret_cast(source_address), upload_size, - regions, layout, format, info.width, info.height, layer_count, - info.levels, "TextureCache render target", - vk::ImageLayout::eGeneral); - return; - } - if (info.tile_mode == Prospero::GpuEnumValue(Prospero::TileMode::kRenderTarget) && - Transfer::GuestBufferIsTiled(info.address, slice_size)) { - const auto format = RenderTargetTransferFormat(info.bytes_per_element); - auto layout = TextureCalcUploadLayout(format, info.width, info.height, 1, 1, info.pitch, - info.tile_mode, slice_size, false, false, - "TextureCache render target"); - auto regions = TextureBuildUploadRegions(layout, info.format, info.width, info.height, 1, 1, - true, false, TextureUploadDestination::MipLevels); - TextureUploadGuestImage(image, reinterpret_cast(info.address), slice_size, - regions, layout, format, info.width, info.height, 1, 1, - "TextureCache render target", vk::ImageLayout::eGeneral); - } else { - Transfer::UploadImage(image, reinterpret_cast(info.address), slice_size, - info.pitch, vk::ImageLayout::eGeneral); - } -} - -void UploadRenderTarget(RenderTextureVulkanImage& image, const RenderTargetInfo& info, - bool refresh) { - UploadRenderTargetLayers(image, info, 0, info.layers, refresh); -} - -RenderTextureVulkanImage* CreateRenderTarget(const RenderTargetInfo& info) { - auto& graphics = GetRenderContext().GetGraphics(); - auto* image = new RenderTextureVulkanImage; - image->extent.width = info.width; - image->extent.height = info.height; - image->format = info.format; - image->mip_levels = info.levels; - image->layers = info.layers; - image->samples = info.samples; - image->layout = vk::ImageLayout::eUndefined; - vk::ImageCreateInfo create {}; - create.sType = vk::StructureType::eImageCreateInfo; - create.flags = RenderTargetCreateFlags(info.format); - create.imageType = vk::ImageType::e2D; - create.extent = {info.width, info.height, 1}; - create.mipLevels = info.levels; - create.arrayLayers = info.layers; - create.format = info.format; - create.tiling = vk::ImageTiling::eOptimal; - create.initialLayout = vk::ImageLayout::eUndefined; - create.usage = RenderTargetUsage(info.format, create.flags, info.samples); - create.sharingMode = vk::SharingMode::eExclusive; - create.samples = vulkan_sample_count(info.samples); - image->memory.property = vk::MemoryPropertyFlagBits::eDeviceLocal; - if (!graphics.CreateImage(create, *image)) { - EXIT("TextureCache: failed to create render target, addr=0x%016" PRIx64 - " extent=%ux%u format=%d\n", - info.address, info.width, info.height, static_cast(info.format)); - } - ImageViewOps::CreateRenderTargetViews(*image); - return image; -} - -DepthStencilVulkanImage* CreateDepthTarget(const DepthTargetInfo& info) { - auto& graphics = GetRenderContext().GetGraphics(); - vk::ImageCreateInfo create {}; - create.sType = vk::StructureType::eImageCreateInfo; - create.imageType = vk::ImageType::e2D; - create.extent = {info.width, info.height, 1}; - create.mipLevels = 1; - create.arrayLayers = info.layers; - create.format = info.format; - create.tiling = vk::ImageTiling::eOptimal; - create.initialLayout = vk::ImageLayout::eUndefined; - create.usage = DepthTargetImageUsage(); - create.sharingMode = vk::SharingMode::eExclusive; - create.samples = vulkan_sample_count(info.samples); - vk::ImageFormatProperties properties {}; - if (graphics.GetImageFormatProperties( - info.format, vk::ImageType::e2D, vk::ImageTiling::eOptimal, create.usage, - vk::ImageCreateFlags {}, &properties) != vk::Result::eSuccess || - !static_cast(properties.sampleCounts & create.samples)) { - EXIT("TextureCache: depth format does not support required usage, format=%d usage=0x%x " - "samples=%u supported=0x%x\n", - static_cast(info.format), - static_cast(create.usage), info.samples, - static_cast(properties.sampleCounts)); - } - auto* image = new DepthStencilVulkanImage; - image->extent.width = info.width; - image->extent.height = info.height; - image->guest_pitch = info.pitch; - image->layers = info.layers; - image->samples = info.samples; - image->format = info.format; - image->layout = vk::ImageLayout::eUndefined; - image->compressed = false; - image->memory.property = vk::MemoryPropertyFlagBits::eDeviceLocal; - if (!graphics.CreateImage(create, *image)) { - EXIT("TextureCache: failed to create depth target, addr=0x%016" PRIx64 - " extent=%ux%u format=%d\n", - info.address, info.width, info.height, static_cast(info.format)); - } - ImageViewOps::CreateDepthViews(*image); - return image; -} - -void ValidateVideoOut(const VideoOutInfo& info) { - const auto compression = - ClassifyVideoOutCompression(info.compression != VideoOutCompression::Uncompressed, - info.metadata_address, info.dcc_control, 0); - const bool metadata_invalid = compression != VideoOutCompression::Uncompressed && - compression != VideoOutCompression::Unsupported && - (info.metadata_address >= TRACKER_ADDRESS_SIZE || - (info.metadata_address >= info.address && - info.metadata_address < info.address + info.size)); - if (info.address == 0 || info.size == 0 || info.address >= TRACKER_ADDRESS_SIZE || - info.size > TRACKER_ADDRESS_SIZE - info.address || (info.address & 0xffffu) != 0 || - info.width == 0 || info.height == 0 || info.width > 16384 || info.height > 16384 || - info.pitch < info.width || - info.tile_mode != Prospero::GpuEnumValue(Prospero::TileMode::kRenderTarget) || - compression == VideoOutCompression::Unsupported || compression != info.compression || - metadata_invalid || !IsSupportedVideoOutFormat(info)) { - EXIT("TextureCache: unsupported video-out surface, addr=0x%016" PRIx64 " size=0x%016" PRIx64 - " metadata=0x%016" PRIx64 " dcc=0x%08" PRIx32 - " extent=%ux%u pitch=%u tile=%u guest_format=%u bpe=%u vk_format=%d\n", - info.address, info.size, info.metadata_address, info.dcc_control, info.width, - info.height, info.pitch, info.tile_mode, info.guest_format, info.bytes_per_element, - static_cast(info.format)); - } - TileSizeAlign exact {}; - TileGetTextureTotalSize(info.guest_format, info.width, info.height, 1, info.pitch, 1, - info.tile_mode, false, exact); - if (exact.align != 65536 || exact.size != info.size || - TileGetTexturePitch(info.guest_format, info.width, 1, info.tile_mode) != info.pitch) { - EXIT("TextureCache: video-out tile layout mismatch, addr=0x%016" PRIx64 - " size=0x%016" PRIx64 " expected_size=0x%016" PRIx64 " align=0x%016" PRIx64 - " pitch=%u\n", - info.address, info.size, exact.size, exact.align, info.pitch); - } - (void)RenderTargetUsage(info.format, vk::ImageCreateFlags {}, 1); -} - -VideoOutVulkanImage* CreateVideoOut(const VideoOutInfo& info) { - auto& graphics = GetRenderContext().GetGraphics(); - auto* image = new VideoOutVulkanImage; - image->extent.width = info.width; - image->extent.height = info.height; - image->format = info.format; - image->layout = vk::ImageLayout::eUndefined; - vk::ImageCreateInfo create {}; - create.sType = vk::StructureType::eImageCreateInfo; - create.imageType = vk::ImageType::e2D; - create.extent = {info.width, info.height, 1}; - create.mipLevels = 1; - create.arrayLayers = 1; - create.format = info.format; - create.tiling = vk::ImageTiling::eOptimal; - create.initialLayout = vk::ImageLayout::eUndefined; - create.flags = RenderTargetCreateFlags(info.format); - create.usage = RenderTargetUsage(info.format, create.flags, 1); - create.sharingMode = vk::SharingMode::eExclusive; - create.samples = vk::SampleCountFlagBits::e1; - image->memory.property = vk::MemoryPropertyFlagBits::eDeviceLocal; - if (!graphics.CreateImage(create, *image)) { - EXIT("TextureCache: failed to create video-out image, addr=0x%016" PRIx64 - " extent=%ux%u format=%d\n", - info.address, info.width, info.height, static_cast(info.format)); - } - ImageViewOps::CreateVideoOutViews(*image); - return image; -} - -void UploadVideoOut(VideoOutVulkanImage& image, const VideoOutInfo& info, bool refresh) { - if (info.compression != VideoOutCompression::Uncompressed) { - EXIT("TextureCache: compressed video-out guest upload is unsupported, " - "addr=0x%016" PRIx64 " metadata=0x%016" PRIx64 " dcc=0x%08" PRIx32 "\n", - info.address, info.metadata_address, info.dcc_control); - } - if (refresh) { - Transfer::WaitForQueueIdle(); - } - image.layout = vk::ImageLayout::eUndefined; - if (!info.bgra16) { - auto layout = - TextureCalcUploadLayout(info.guest_format, info.width, info.height, 1, 1, info.pitch, - info.tile_mode, info.size, false, false, "VideoOut"); - auto regions = TextureBuildUploadRegions(layout, info.format, info.width, info.height, 1, 1, - false, false, TextureUploadDestination::MipLevels); - TextureUploadGuestImage(image, reinterpret_cast(info.address), info.size, - regions, layout, info.guest_format, info.width, info.height, 1, 1, - "VideoOut", vk::ImageLayout::eGeneral); - return; - } - Transfer::ScratchBuffer scratch(info.size); - TileBlockLayout block {}; - EXIT_NOT_IMPLEMENTED( - !TileGetBlockLayout(TileBlockFamily::RenderTarget64KB, info.bytes_per_element, block)); - const GpuTileInfo tile_info {block.family, - block.bytes_per_element, - 0, - info.size, - 0, - info.size, - 0, - info.width, - info.height, - 1, - info.pitch}; - GpuDetile(reinterpret_cast(info.address), scratch.Data(), info.size, info.size, - std::span(&tile_info, 1)); - SwapVideoOutBgra16(scratch.Data(), info.size); - Transfer::UploadImage(image, scratch.Data(), info.size, info.pitch, vk::ImageLayout::eGeneral); -} - -void SwapVideoOutBgra16(void* data, uint64_t size) { - auto* pixels = static_cast(data); - for (uint64_t i = 0; i < size / sizeof(uint16_t); i += 4) { - std::swap(pixels[i], pixels[i + 2]); - } -} - -GpuTextureVulkanImage* CreateDummyTexture(bool uint_format, bool image_3d, bool storage) { - auto* image = storage ? static_cast(new StorageTextureVulkanImage) - : new TextureVulkanImage; - auto usage = storage ? TextureFormatUsage::Storage : TextureFormatUsage::Sampled; - auto layout = storage ? vk::ImageLayout::eGeneral : vk::ImageLayout::eShaderReadOnlyOptimal; - auto owner = storage ? "DummyStorageTexture" : "DummySampledTexture"; - - auto params = MakeDummyTextureParams(uint_format, image_3d, usage, owner); - auto components = TextureCreateImage(*image, params); - - static constexpr uint32_t zero = 0; - Transfer::UploadImage(*image, &zero, sizeof(zero), 1, layout); - TextureCreateImageViews(*image, components, params.type, 0, params.base_level, params.levels, - params.depth, params.allow_cube_view, params.view_usage); - return image; -} - -void Destroy(VulkanImage& image) { - auto& graphics = GetRenderContext().GetGraphics(); - KYTY_PROFILER_BLOCK("TextureCache::DeleteImage"); - switch (image.type) { - case VulkanImageType::RenderTexture: - case VulkanImageType::VideoOut: - GetRenderContext().GetFramebufferCache().FreeFramebufferByColor(image); - break; - case VulkanImageType::DepthStencil: - GetRenderContext().GetFramebufferCache().FreeFramebufferByDepth( - static_cast(image)); - break; - case VulkanImageType::Texture: - case VulkanImageType::StorageTexture: break; - case VulkanImageType::Unknown: EXIT("cannot destroy an untyped Vulkan image\n"); - } - - ImageViewOps::DestroyViews(image); - graphics.DeleteImage(image); - - switch (image.type) { - case VulkanImageType::Texture: delete &static_cast(image); break; - case VulkanImageType::StorageTexture: - delete &static_cast(image); - break; - case VulkanImageType::RenderTexture: - delete &static_cast(image); - break; - case VulkanImageType::DepthStencil: - delete &static_cast(image); - break; - case VulkanImageType::VideoOut: delete &static_cast(image); break; - case VulkanImageType::Unknown: EXIT("cannot delete an untyped Vulkan image\n"); + default: EXIT("unsupported render-target element size: %u\n", bytes_per_element); } } } // namespace ImageOps +Image::Image(GraphicContext& graphics, CommandScheduler& scheduler, + const ImageInfo& image_info) + : info(image_info), m_graphics(&graphics), m_scheduler(&scheduler) { + KYTY_PROFILER_FUNCTION(); + ImageOps::Validate(info); + if (info.pixel_format == vk::Format::eUndefined) { + return; + } + + backing.format = info.pixel_format; + backing.image_type = HostImageType(info.type); + backing.extent = info.extent; + backing.guest_pitch = info.pitch; + backing.layers = info.IsVolume() ? 1u : info.resources.layers; + backing.mip_levels = info.resources.levels; + backing.samples = info.samples; + backing.flags = ImageCreateFlags(info); + backing.usage = ImageUsageFlags(graphics, info); + + vk::ImageCreateInfo create {}; + create.sType = vk::StructureType::eImageCreateInfo; + create.flags = backing.flags; + create.imageType = backing.image_type; + create.extent = backing.extent; + create.mipLevels = backing.mip_levels; + create.arrayLayers = backing.layers; + create.format = backing.format; + create.tiling = vk::ImageTiling::eOptimal; + create.initialLayout = backing.state.layout; + create.usage = backing.usage; + create.sharingMode = vk::SharingMode::eExclusive; + create.samples = vulkan_sample_count(backing.samples); + + vk::ImageFormatProperties properties {}; + if (graphics.GetImageFormatProperties(create.format, create.imageType, create.tiling, + create.usage, create.flags, + &properties) != vk::Result::eSuccess || + !static_cast(properties.sampleCounts & create.samples)) { + EXIT("image format does not support required usage: format=%d type=%d usage=0x%x " + "flags=0x%x samples=%u\n", + static_cast(create.format), static_cast(create.imageType), + static_cast(create.usage), + static_cast(create.flags), backing.samples); + } + + backing.memory.property = vk::MemoryPropertyFlagBits::eDeviceLocal; + if (!graphics.CreateImage(create, backing)) { + EXIT("failed to create image: extent=%ux%ux%u format=%d layers=%u levels=%u\n", + create.extent.width, create.extent.height, create.extent.depth, + static_cast(create.format), create.arrayLayers, create.mipLevels); + } +} + +uint64_t Image::HashGuestEdges() const { + constexpr uint64_t page_mask = TRACKER_PAGE_SIZE - 1; + std::array bytes {}; + const auto range = info.data; + const uint64_t head_end = std::min(range.End(), (range.address + page_mask) & ~page_mask); + const uint64_t tail_begin = std::max(range.address, range.End() & ~page_mask); + const uint64_t head_size = head_end - range.address; + const uint64_t tail_address = tail_begin < head_end ? head_end : tail_begin; + const uint64_t tail_size = range.End() - tail_address; + if ((head_size != 0 && + !LibKernel::Memory::TryReadBacking(range.address, bytes.data(), head_size)) || + (tail_size != 0 && + !LibKernel::Memory::TryReadBacking(tail_address, bytes.data() + head_size, tail_size))) { + EXIT("Image: failed to hash guest backing\n"); + } + return XXH3_64bits(bytes.data(), static_cast(head_size + tail_size)); +} + +Image::~Image() { + KYTY_PROFILER_FUNCTION(); + if (m_graphics == nullptr) { + return; + } + { + std::lock_guard lock(views.mutex); + for (auto& cached: views.views) { + if (cached.view != nullptr) { + m_graphics->device.destroyImageView(cached.view, nullptr); + cached.view = nullptr; + } + } + views.views.clear(); + } + if (backing.image != nullptr) { + m_graphics->DeleteImage(backing); + } +} + } // namespace Libs::Graphics diff --git a/src/graphics/host_gpu/renderer/image.h b/src/graphics/host_gpu/renderer/image.h index 57d8b4e3..24e66b2f 100644 --- a/src/graphics/host_gpu/renderer/image.h +++ b/src/graphics/host_gpu/renderer/image.h @@ -2,49 +2,93 @@ #define EMULATOR_SRC_GRAPHICS_HOST_GPU_RENDERER_IMAGE_H_ #include "common/assert.h" +#include "graphics/host_gpu/graphicContext.h" #include "graphics/host_gpu/renderer/imageInfo.h" -#include -#include +#include +#include +#include +#include #include +#include +#include namespace Libs::Graphics { -struct DepthStencilVulkanImage; -struct GpuTextureVulkanImage; -struct GraphicContext; -struct RenderTextureVulkanImage; -struct VideoOutVulkanImage; -struct VulkanImage; +class Buffer; +class CommandScheduler; +struct ImageTestAccess; -struct Image final: ImageInfo { - Image& operator=(const ImageInfo& value) { - if (IsCpuDirty()) { - EXIT("dirty sampled image cannot be reassigned\n"); - } - static_cast(*this) = value; - m_track_begin = address; - m_track_end = address + size; - m_maybe_cpu_hash_valid = false; - return *this; +struct ImageId { + uint32_t index = std::numeric_limits::max(); + uint32_t generation = 0; + + [[nodiscard]] explicit operator bool() const noexcept { + return index != std::numeric_limits::max(); } + auto operator<=>(const ImageId&) const = default; +}; + +struct CachedImageView { + ImageViewInfo info; + vk::ImageView view = nullptr; +}; + +struct ImageViewCache { + ImageViewCache() = default; + KYTY_CLASS_NO_COPY(ImageViewCache); + + std::mutex mutex; + std::vector views; +}; + +struct ImageUsage { + bool texture = false; + bool storage = false; + bool render_target = false; + bool depth_target = false; + bool video_out = false; +}; + +struct ImageBinding { + bool is_bound = false; + bool is_target = false; + bool needs_rebind = false; + bool force_general = false; +}; + +class Image final { +public: + Image(GraphicContext& graphics, CommandScheduler& scheduler, const ImageInfo& info); + ~Image(); + KYTY_CLASS_NO_COPY(Image); + + [[nodiscard]] vk::ImageView FindView(const ImageViewInfo& view_info); + void AssociateDepth(ImageId image_id) { depth_id = image_id; } + using Barriers = std::vector; + [[nodiscard]] Barriers + GetBarriers(vk::ImageLayout destination_layout, vk::AccessFlags2 destination_access, + vk::PipelineStageFlags2 destination_stage, + std::optional range); + void Transit(vk::ImageLayout destination_layout, vk::AccessFlags2 destination_access, + std::optional range, vk::CommandBuffer command_buffer); + void Upload(std::span copies, vk::Buffer buffer, + uint64_t offset, uint64_t size); + void Download(std::span copies, vk::Buffer buffer, + uint64_t offset, uint64_t size); + void CopyImage(Image& source); + void Resolve(Image& source, const ImageSubresourceRange& source_range, + const ImageSubresourceRange& destination_range); + void CopyImageWithBuffer(Image& source, Buffer& buffer); + void CopyMip(Image& source, uint32_t mip, uint32_t layer); void InvalidateCpuWrite(uint64_t vaddr, uint64_t size) { - if (ImageRangeOverlaps(address, this->size, vaddr, size)) { - m_cpu_dirty = true; - m_maybe_cpu_dirty = false; - m_maybe_cpu_hash_valid = false; - m_track_begin = m_track_end; - } else if (ImagePageRangesOverlap(address, this->size, vaddr, size)) { - constexpr uint64_t page_mask = 4096 - 1; - if (vaddr + size <= address) { - const auto next_page = (address + page_mask) & ~page_mask; - m_track_begin = std::min(m_track_end, std::max(m_track_begin, next_page)); - } else if (vaddr >= address + this->size) { - const auto page = (address + this->size) & ~page_mask; - m_track_end = std::max(m_track_begin, std::min(m_track_end, page)); - } - m_maybe_cpu_dirty = m_track_begin == m_track_end; + if (ImageRangeOverlaps(info.data.address, info.data.size, vaddr, size)) { + m_cpu_dirty = true; + m_maybe_cpu_dirty = false; + m_maybe_hash_valid = false; + } else if (ImagePageRangesOverlap(info.data.address, info.data.size, vaddr, size)) { + m_maybe_cpu_dirty = true; } } @@ -52,117 +96,93 @@ struct Image final: ImageInfo { [[nodiscard]] bool IsDefinitelyCpuDirty() const { return m_cpu_dirty; } [[nodiscard]] bool IsMaybeCpuDirty() const { return m_maybe_cpu_dirty; } [[nodiscard]] bool NeedsMaybeCpuHash() const { - return m_maybe_cpu_dirty && !m_maybe_cpu_hash_valid; - } - [[nodiscard]] bool IsCpuTrackingComplete() const { - return m_track_begin == address && m_track_end == address + size; + return m_maybe_cpu_dirty && !m_maybe_hash_valid; } void SetMaybeCpuHash(uint64_t hash) { if (!NeedsMaybeCpuHash()) { - EXIT("sampled image cannot initialize maybe-dirty hash\n"); + EXIT("image cannot initialize maybe-dirty hash\n"); } - m_maybe_cpu_hash = hash; - m_maybe_cpu_hash_valid = true; + m_maybe_cpu_hash = hash; + m_maybe_hash_valid = true; } [[nodiscard]] bool ResolveMaybeCpuHash(uint64_t hash) { - if (!m_maybe_cpu_dirty || !m_maybe_cpu_hash_valid || m_cpu_dirty) { - EXIT("sampled image cannot resolve maybe-dirty hash\n"); - } - m_maybe_cpu_dirty = false; - m_maybe_cpu_hash_valid = false; - m_cpu_dirty = hash != m_maybe_cpu_hash; - if (!m_cpu_dirty) { - m_track_begin = address; - m_track_end = address + size; + if (!m_maybe_cpu_dirty || !m_maybe_hash_valid || m_cpu_dirty) { + EXIT("image cannot resolve maybe-dirty hash\n"); } + m_maybe_cpu_dirty = false; + m_maybe_hash_valid = false; + m_cpu_dirty |= hash != m_maybe_cpu_hash; return m_cpu_dirty; } void RefreshComplete() { if (!IsCpuDirty()) { - EXIT("clean sampled image cannot complete a refresh\n"); + EXIT("clean image cannot complete a refresh\n"); } - m_cpu_dirty = false; - m_maybe_cpu_dirty = false; - m_maybe_cpu_hash_valid = false; - m_track_begin = address; - m_track_end = address + size; + m_cpu_dirty = false; + m_maybe_cpu_dirty = false; + m_maybe_hash_valid = false; } + [[nodiscard]] bool IsGpuModified() const noexcept { return m_gpu_modified; } + void MarkGpuModified() noexcept { m_gpu_modified = true; } + void ClearGpuModified() noexcept { m_gpu_modified = false; } + + [[nodiscard]] bool IsBufferModified() const noexcept { return m_buffer_modified; } + void MarkBufferModified() noexcept { m_buffer_modified = true; } + void ClearBufferModified() noexcept { m_buffer_modified = false; } + + [[nodiscard]] bool Overlaps(uint64_t address, uint64_t size, bool pages = false) const noexcept { + return pages ? ImagePageRangesOverlap(info.data.address, info.data.size, address, size) + : ImageRangeOverlaps(info.data.address, info.data.size, address, size); + } + [[nodiscard]] bool GpuOverlaps(uint64_t address, uint64_t size) const noexcept { + return IsGpuModified() && Overlaps(address, size); + } + [[nodiscard]] bool SafeToDownload() const noexcept { + return IsGpuModified() && !IsBufferModified() && !IsCpuDirty(); + } + [[nodiscard]] uint64_t AccountedSize() const noexcept { + return backing.image == nullptr ? 0 : (info.data.size + 1023) & ~uint64_t {1023}; + } + [[nodiscard]] uint64_t HashGuestEdges() const; + + ImageInfo info; + VulkanImage backing; + ImageViewCache views; + ImageUsage usage; + ImageBinding binding; + bool registered = false; + ImageId depth_id {}; + uint64_t tick_accessed_last = 0; + size_t lru_id = 0; + private: - bool m_cpu_dirty = false; - bool m_maybe_cpu_dirty = false; - bool m_maybe_cpu_hash_valid = false; - uint64_t m_track_begin = 0; - uint64_t m_track_end = 0; - uint64_t m_maybe_cpu_hash = 0; + friend struct ImageTestAccess; + + [[nodiscard]] static vk::ImageAspectFlags FullAspectMask(vk::Format format) noexcept; + [[nodiscard]] static uint32_t CopyRows(uint64_t row_size, uint32_t rows, + uint64_t capacity) noexcept; + [[nodiscard]] static std::pair + SanitizeCopyLayers(const Image& source, const Image& destination, uint32_t depth); + + GraphicContext* m_graphics = nullptr; + CommandScheduler* m_scheduler = nullptr; + uint64_t m_maybe_cpu_hash = 0; + bool m_cpu_dirty = false; + bool m_maybe_cpu_dirty = false; + bool m_maybe_hash_valid = false; + bool m_gpu_modified = false; + bool m_buffer_modified = false; }; namespace ImageOps { -[[nodiscard]] GpuTextureVulkanImage* CreateTexture(const ImageInfo& info, - bool storage, vk::ComponentMapping& components); -void CreateTextureViews(GpuTextureVulkanImage& image, - const ImageInfo& info, bool storage, vk::ComponentMapping components); - -[[nodiscard]] RenderTextureVulkanImage* CreateRenderTarget( - const RenderTargetInfo& info); -[[nodiscard]] uint32_t RenderTargetTransferFormat(uint32_t bytes_per_element); -void UploadRenderTargetLayers(RenderTextureVulkanImage& image, - const RenderTargetInfo& info, uint32_t base_layer, - uint32_t layer_count, bool refresh); -void UploadRenderTarget(RenderTextureVulkanImage& image, - const RenderTargetInfo& info, bool refresh); - -[[nodiscard]] DepthStencilVulkanImage* CreateDepthTarget( - const DepthTargetInfo& info); - -void ValidateVideoOut(const VideoOutInfo& info); -[[nodiscard]] VideoOutVulkanImage* CreateVideoOut( - const VideoOutInfo& info); -void SwapVideoOutBgra16(void* data, uint64_t size); -void UploadVideoOut(VideoOutVulkanImage& image, const VideoOutInfo& info, - bool refresh); - -[[nodiscard]] GpuTextureVulkanImage* CreateDummyTexture(bool uint_format, - bool image_3d, bool storage); - -void Destroy(VulkanImage& image); +void Validate(const ImageInfo& info); +[[nodiscard]] uint32_t RenderTargetTransferFormat(uint32_t bytes_per_element); } // namespace ImageOps -struct ImageRetirementRange { - uint64_t address = 0; - uint64_t size = 0; - bool retire = false; -}; - -struct ImageRetirementConflict { - size_t retired = SIZE_MAX; - size_t retained = SIZE_MAX; - - [[nodiscard]] bool Exists() const { return retired != SIZE_MAX; } -}; - -[[nodiscard]] inline ImageRetirementConflict -FindImageRetirementConflict(std::span ranges) { - for (size_t retired = 0; retired < ranges.size(); retired++) { - if (!ranges[retired].retire) { - continue; - } - for (size_t retained = 0; retained < ranges.size(); retained++) { - if (ranges[retained].retire) { - continue; - } - if (ImageRangeOverlaps(ranges[retired].address, ranges[retired].size, - ranges[retained].address, ranges[retained].size)) { - return {retired, retained}; - } - } - } - return {}; -} - } // namespace Libs::Graphics #endif // EMULATOR_SRC_GRAPHICS_HOST_GPU_RENDERER_IMAGE_H_ diff --git a/src/graphics/host_gpu/renderer/imageInfo.h b/src/graphics/host_gpu/renderer/imageInfo.h index d8ee17f6..065e9a04 100644 --- a/src/graphics/host_gpu/renderer/imageInfo.h +++ b/src/graphics/host_gpu/renderer/imageInfo.h @@ -3,9 +3,11 @@ #include "common/assert.h" #include "graphics/guest_gpu/gpu_defs.h" +#include "graphics/guest_gpu/gpu_format.h" #include "graphics/host_gpu/regionDefinitions.h" #include "graphics/host_gpu/vulkanCommon.h" +#include #include #include #include @@ -13,87 +15,200 @@ namespace Libs::Graphics { -struct ImageInfo { - uint64_t address = 0; - uint64_t size = 0; - uint32_t format = 0; - uint32_t width = 0; - uint32_t height = 0; - uint32_t pitch = 0; +struct GuestRange { + uint64_t address = 0; + uint64_t size = 0; + + [[nodiscard]] constexpr bool Empty() const noexcept { return address == 0 || size == 0; } + [[nodiscard]] constexpr bool Valid() const noexcept { + return !Empty() && address < TRACKER_ADDRESS_SIZE && + size <= TRACKER_ADDRESS_SIZE - address; + } + [[nodiscard]] constexpr uint64_t End() const noexcept { return address + size; } + auto operator<=>(const GuestRange&) const = default; +}; + +enum class VideoOutCompression : uint8_t { Uncompressed, Dcc256_256_0, Dcc256_64_64, Unsupported }; + +enum class ImageMetadataKind : uint8_t { None, Htile, Dcc }; + +struct ImageMetadataInfo { + GuestRange range; + ImageMetadataKind kind = ImageMetadataKind::None; + uint32_t control = 0; + VideoOutCompression compression = VideoOutCompression::Uncompressed; + bool stencil_compressed = false; +}; + +struct ImageSubresources { + uint32_t levels = 1; + uint32_t layers = 1; + auto operator<=>(const ImageSubresources&) const = default; +}; + +struct ImageSubresourceRange { uint32_t base_level = 0; - uint32_t levels = 1; - uint32_t view_levels = 1; - uint32_t tile = 0; - uint32_t swizzle = 0; - uint32_t depth = 1; - uint32_t type = 0; - uint32_t base_array = 0; + uint32_t level_count = 1; + uint32_t base_layer = 0; + uint32_t layer_count = 1; + auto operator<=>(const ImageSubresourceRange&) const = default; }; -struct RenderTargetInfo { - uint64_t address = 0; - uint64_t size = 0; - vk::Format format = vk::Format::eUndefined; - uint32_t width = 0; - uint32_t height = 0; - uint32_t pitch = 0; - uint32_t bytes_per_element = 0; - uint32_t tile_mode = 0; - uint32_t levels = 1; - uint32_t layers = 1; - uint32_t samples = 1; +struct ImageMipInfo { + uint64_t offset = 0; + uint64_t size = 0; + uint32_t pitch = 0; + uint32_t height = 0; + auto operator<=>(const ImageMipInfo&) const = default; }; -// Common image-to-buffer copy description. Storage images and render targets keep distinct -// cache records today, but downloads should consume one normalized layout just as uploads do. -struct ColorImageTransferInfo { - uint64_t address = 0; - uint64_t size = 0; - vk::Format format = vk::Format::eUndefined; - uint32_t width = 0; - uint32_t height = 0; - uint32_t pitch = 0; - uint32_t bytes_per_element = 0; - uint32_t tile_mode = 0; - uint32_t levels = 1; - uint32_t samples = 1; +struct ImageInfo { + GuestRange data; + GuestRange stencil; + ImageMetadataInfo metadata; + uint32_t htile_clear_mask = UINT32_MAX; + vk::Format pixel_format = vk::Format::eUndefined; + uint32_t guest_format = 0; + Prospero::ImageType type = Prospero::ImageType::kColor2D; + vk::Extent3D extent = {1, 1, 1}; + ImageSubresources resources; + uint32_t pitch = 0; + uint32_t bytes_per_block = 0; + uint32_t samples = 1; + uint32_t tile_mode = 0; + bool bgra16 = false; + std::array mip_layout {}; + + [[nodiscard]] constexpr bool HasStencil() const noexcept { return !stencil.Empty(); } + [[nodiscard]] constexpr bool HasMetadata() const noexcept { + return metadata.kind != ImageMetadataKind::None; + } + [[nodiscard]] bool IsDepth() const noexcept; + [[nodiscard]] bool IsBlock() const noexcept { + return Prospero::BlockCompressedBytesPerBlock(guest_format) != 0; + } + [[nodiscard]] bool IsTiled() const noexcept { + return tile_mode != Prospero::GpuEnumValue(Prospero::TileMode::kLinear); + } + [[nodiscard]] constexpr bool IsVolume() const noexcept { + return type == Prospero::ImageType::kColor3D; + } + [[nodiscard]] constexpr bool IsLayered() const noexcept { + return !IsVolume() && resources.layers > 1; + } + [[nodiscard]] constexpr uint32_t TransferLayers() const noexcept { + return IsVolume() ? extent.depth : resources.layers; + } + [[nodiscard]] vk::Extent2D BlockExtent() const noexcept { + const auto shift = Prospero::BlockCompressedBytesPerBlock(guest_format) != 0 ? 2u : 0u; + return {pitch >> shift, extent.height >> shift}; + } + [[nodiscard]] bool IsCompatible(const ImageInfo& other) const noexcept { + return pixel_format == other.pixel_format && samples == other.samples && + bytes_per_block == other.bytes_per_block; + } + [[nodiscard]] int32_t MipOf(const ImageInfo& container) const noexcept { + if (!IsCompatible(container) || tile_mode != container.tile_mode || resources.levels != 1 || + container.resources.layers == 0 || + container.resources.levels > container.mip_layout.size()) { + return -1; + } + if (HasStencil() != container.HasStencil() || + (HasStencil() && (stencil.address < container.stencil.address || + stencil.End() > container.stencil.End()))) { + return -1; + } + + int32_t mip = -1; + for (uint32_t level = 0; level < container.resources.levels; level++) { + const auto& layout = container.mip_layout[level]; + if (layout.size == 0 || layout.size % container.resources.layers != 0 || + container.data.address > UINT64_MAX - layout.offset) { + continue; + } + const auto mip_base = container.data.address + layout.offset; + const auto slice_size = layout.size / container.resources.layers; + if (slice_size == 0 || mip_base > UINT64_MAX - layout.size) { + continue; + } + const auto mip_end = mip_base + layout.size; + if (data.address >= mip_base && data.address < mip_end && + (data.address - mip_base) % slice_size == 0) { + mip = static_cast(level); + break; + } + } + if (mip < 0) { + return -1; + } + + const auto level = static_cast(mip); + if (extent.width != std::max(container.extent.width >> level, 1u) || + extent.height != std::max(container.extent.height >> level, 1u)) { + return -1; + } + const auto mip_depth = std::max(container.extent.depth >> level, 1u); + if (container.type == Prospero::ImageType::kColor3D && + type == Prospero::ImageType::kColor2D) { + if (resources.layers != mip_depth) { + return -1; + } + } else if (type != container.type) { + return -1; + } + return mip; + } + [[nodiscard]] int32_t SliceOf(const ImageInfo& container, int32_t mip) const noexcept { + if (!IsCompatible(container) || type != container.type || mip < 0 || + static_cast(mip) >= container.resources.levels || + container.resources.levels > container.mip_layout.size() || + container.resources.layers == 0 || data.size == 0) { + return -1; + } + const auto level = static_cast(mip); + if (extent.width != std::max(container.extent.width >> level, 1u) || + extent.height != std::max(container.extent.height >> level, 1u)) { + return -1; + } + const auto& layout = container.mip_layout[level]; + if (layout.size == 0 || layout.size % container.resources.layers != 0 || + container.data.address > UINT64_MAX - layout.offset) { + return -1; + } + const auto slice_size = layout.size / container.resources.layers; + if (slice_size == 0 || data.size % slice_size != 0) { + return -1; + } + const auto mip_base = container.data.address + layout.offset; + if (data.address < mip_base) { + return -1; + } + const auto address_delta = data.address - mip_base; + if (address_delta % data.size != 0 || address_delta / data.size > INT32_MAX) { + return -1; + } + return static_cast(address_delta / data.size); + } }; -[[nodiscard]] inline ColorImageTransferInfo -MakeColorImageTransferInfo(const ImageInfo& info, vk::Format format, - uint32_t bytes_per_element) noexcept { - return {info.address, info.size, format, info.width, info.height, - info.pitch, bytes_per_element, info.tile, info.levels, 1}; -} +struct ImageViewInfo { + vk::Format format = vk::Format::eUndefined; + vk::ImageViewType type = vk::ImageViewType::e2D; + vk::ImageAspectFlags aspect = vk::ImageAspectFlagBits::eColor; + uint32_t base_level = 0; + uint32_t level_count = 1; + uint32_t base_layer = 0; + uint32_t layer_count = 1; + vk::ComponentMapping mapping = {}; + vk::ImageUsageFlags usage = vk::ImageUsageFlagBits::eSampled; -[[nodiscard]] inline ColorImageTransferInfo -MakeColorImageTransferInfo(const RenderTargetInfo& info) noexcept { - return { - info.address, info.size, info.format, info.width, info.height, info.pitch, - info.bytes_per_element, info.tile_mode, info.levels, info.samples}; -} - -struct DepthTargetInfo { - uint64_t address = 0; - uint64_t size = 0; - uint64_t stencil_address = 0; - uint64_t stencil_size = 0; - uint64_t htile_address = 0; - uint64_t htile_size = 0; - vk::Format format = vk::Format::eUndefined; - uint32_t guest_format = 0; - uint32_t width = 0; - uint32_t height = 0; - uint32_t pitch = 0; - uint32_t bytes_per_element = 0; - uint32_t tile_mode = 0; - uint32_t layers = 1; - uint32_t samples = 1; - bool depth_load_clear = false; - bool depth_access = false; - bool stencil_load_clear = false; - bool stencil_access = false; - bool stencil_htile_compressed = false; + [[nodiscard]] bool operator==(const ImageViewInfo& rhs) const noexcept { + return format == rhs.format && type == rhs.type && aspect == rhs.aspect && + base_level == rhs.base_level && level_count == rhs.level_count && + base_layer == rhs.base_layer && layer_count == rhs.layer_count && + mapping.r == rhs.mapping.r && mapping.g == rhs.mapping.g && + mapping.b == rhs.mapping.b && mapping.a == rhs.mapping.a && usage == rhs.usage; + } }; struct DepthFormatPolicy { @@ -183,6 +298,10 @@ FindGuestDepthFormatPolicy(uint32_t guest_format) noexcept { } } +inline bool ImageInfo::IsDepth() const noexcept { + return DepthAspectTransferFormat(pixel_format) != vk::Format::eUndefined; +} + [[nodiscard]] inline constexpr uint32_t DepthAspectTransferBytes(vk::Format format) noexcept { switch (DepthAspectTransferFormat(format)) { case vk::Format::eD16Unorm: return 2; @@ -193,7 +312,6 @@ FindGuestDepthFormatPolicy(uint32_t guest_format) noexcept { } [[nodiscard]] inline constexpr uint32_t EncodeD16AsD24(uint16_t value) noexcept { - // Preserve the guest UNORM value when widening 16 bits to the 24-bit transfer plane. return static_cast((static_cast(value) * 0x00ffffffu + 0x7fffu) / 0xffffu); } @@ -221,44 +339,28 @@ FindGuestDepthFormatPolicy(uint32_t guest_format) noexcept { return false; } -[[nodiscard]] inline constexpr bool IsSupportedDepthTargetFormat(const DepthTargetInfo& info) { - const bool has_stencil = info.stencil_address != 0 || info.stencil_size != 0; - const auto* policy = FindGuestDepthFormatPolicy(info.guest_format); - return policy != nullptr && info.bytes_per_element == policy->bytes_per_element && - (has_stencil ? IsStencilAttachmentFormat(*policy, info.format) - : info.format == policy->depth_attachment_format); +[[nodiscard]] inline constexpr bool IsSupportedDepthTargetFormat(const ImageInfo& info) { + const auto* policy = FindGuestDepthFormatPolicy(info.guest_format); + return policy != nullptr && info.bytes_per_block == policy->bytes_per_element && + (info.HasStencil() ? IsStencilAttachmentFormat(*policy, info.pixel_format) + : info.pixel_format == policy->depth_attachment_format); } -[[nodiscard]] inline constexpr bool IsSupportedDepthReadbackFormat(const DepthTargetInfo& info) { - const bool has_stencil = info.stencil_address != 0 || info.stencil_size != 0; - return IsSupportedDepthTargetFormat(info) && - DepthAspectTransferBytes(info.format) == info.bytes_per_element && - (!has_stencil || !info.stencil_htile_compressed); +[[nodiscard]] inline constexpr bool IsSupportedDepthPlaneReadback(const ImageInfo& info) { + if (!IsSupportedDepthTargetFormat(info)) { + return false; + } + const auto transfer_bytes = DepthAspectTransferBytes(info.pixel_format); + return transfer_bytes == info.bytes_per_block || + (info.bytes_per_block == sizeof(uint16_t) && + transfer_bytes == sizeof(uint32_t)); } -enum class VideoOutCompression : uint8_t { Uncompressed, Dcc256_256_0, Dcc256_64_64, Unsupported }; - -struct VideoOutInfo { - uint64_t address = 0; - uint64_t size = 0; - uint64_t metadata_address = 0; - vk::Format format = vk::Format::eUndefined; - uint32_t guest_format = 0; - uint32_t width = 0; - uint32_t height = 0; - uint32_t pitch = 0; - uint32_t bytes_per_element = 0; - uint32_t tile_mode = 0; - uint32_t dcc_control = 0; - VideoOutCompression compression = VideoOutCompression::Unsupported; - bool bgra16 = false; -}; - [[nodiscard]] inline VideoOutCompression ClassifyVideoOutCompression(bool compressed, uint64_t metadata_address, uint32_t dcc_control, uint64_t dcc_clear_color) noexcept { - constexpr uint32_t VIDEO_OUT_DCC_CONTROL_256_256_0 = 0x00000048u; - constexpr uint32_t VIDEO_OUT_DCC_CONTROL_256_64_64 = 0x00000208u; + constexpr uint32_t DCC_256_256_0 = 0x00000048u; + constexpr uint32_t DCC_256_64_64 = 0x00000208u; if (!compressed) { return metadata_address == 0 && dcc_control == 0 && dcc_clear_color == 0 ? VideoOutCompression::Uncompressed @@ -268,8 +370,8 @@ ClassifyVideoOutCompression(bool compressed, uint64_t metadata_address, uint32_t return VideoOutCompression::Unsupported; } switch (dcc_control) { - case VIDEO_OUT_DCC_CONTROL_256_256_0: return VideoOutCompression::Dcc256_256_0; - case VIDEO_OUT_DCC_CONTROL_256_64_64: return VideoOutCompression::Dcc256_64_64; + case DCC_256_256_0: return VideoOutCompression::Dcc256_256_0; + case DCC_256_64_64: return VideoOutCompression::Dcc256_64_64; default: return VideoOutCompression::Unsupported; } } @@ -316,7 +418,7 @@ inline constexpr std::array VIDEO_OUT_FORMAT_POLICIES { }}; [[nodiscard]] inline bool DecodeVideoOutPixelFormat(uint64_t pixel_format, - VideoOutPixelFormatInfo& info) { + VideoOutPixelFormatInfo& info) { for (const auto& policy: VIDEO_OUT_FORMAT_POLICIES) { if (policy.pixel_format == pixel_format) { info = policy.info; @@ -326,10 +428,11 @@ inline constexpr std::array VIDEO_OUT_FORMAT_POLICIES { return false; } -[[nodiscard]] inline bool IsSupportedVideoOutFormat(const VideoOutInfo& info) { +[[nodiscard]] inline bool IsSupportedVideoOutFormat(const ImageInfo& info) { for (const auto& policy: VIDEO_OUT_FORMAT_POLICIES) { - if (info.format == policy.info.format && info.guest_format == policy.info.guest_format && - info.bytes_per_element == policy.info.bytes_per_element && + if (info.pixel_format == policy.info.format && + info.guest_format == policy.info.guest_format && + info.bytes_per_block == policy.info.bytes_per_element && info.bgra16 == policy.info.bgra16) { return true; } @@ -337,189 +440,87 @@ inline constexpr std::array VIDEO_OUT_FORMAT_POLICIES { return false; } -enum class DepthOverlap : uint8_t { - None, - RetireSampled, - RetireStorage, - ExpandTarget, - DiscardTarget, - RecreateTarget, - Unsupported -}; -enum class DepthTransitionSource : uint8_t { None, Guest, Native }; -enum class RenderTargetOverlap : uint8_t { - None, - RetireSampled, - RetireStorage, - PreserveStorage, - ExpandTarget, - RetireTarget, - Unsupported -}; -enum class SampledOverlap : uint8_t { None, ReadOnlyAlias, Unsupported }; -enum class StorageSampledOverlap : uint8_t { None, ExactImage, RetireStorage, Unsupported }; -enum class StorageSampledViewShape : uint8_t { Image2D, Image2DArray, Image3D, Unsupported }; -enum class StorageImageOverlap : uint8_t { None, RetireSampled, PageNeighbor, Unsupported }; -enum class HostWriteOverlap : uint8_t { None, InvalidateImage, Unsupported }; -enum class BufferImageBinding : uint8_t { - Texture, - VideoOut, - RenderTarget, - StorageTexture, - DepthTarget, - Unsupported -}; -enum class BufferImageWrite : uint8_t { - None, - InvalidateTexture, - InvalidateVideoOut, - InvalidateStorageTexture, - InvalidateDepthTarget, - InvalidateRenderTarget, - SynchronizeRenderTarget, - SynchronizeStorageTexture, - SynchronizeDepthTarget, - SynchronizeVideoOut, - Unsupported -}; - -[[nodiscard]] inline constexpr bool -HasGuestCurrentImageOwnership(bool image_gpu_modified, bool buffer_modified, bool cpu_dirty, - bool tracker_gpu_modified) noexcept { - return !image_gpu_modified && !buffer_modified && !cpu_dirty && !tracker_gpu_modified; -} - -enum class StorageBufferRebind : uint8_t { Reuse, RefreshFromBacking, Unsupported }; -enum class MetaImageOverlap : uint8_t { RetainSampled, RetireImage, Unsupported }; - -[[nodiscard]] inline constexpr bool CanRetireGuestCurrentDepthForMetadataReuse( - bool depth_gpu_modified, bool depth_buffer_modified, bool depth_tracker_gpu_modified, - bool metadata_gpu_modified, bool metadata_tracker_gpu_modified, - uint32_t metadata_clear_mask) noexcept { - return !metadata_gpu_modified && !metadata_tracker_gpu_modified && metadata_clear_mask == 0 && - HasGuestCurrentImageOwnership(depth_gpu_modified, depth_buffer_modified, false, - depth_tracker_gpu_modified); -} - -[[nodiscard]] inline constexpr uint32_t SelectImageBackingBaseLevel(bool storage, - uint32_t view_base_level) { - // Storage descriptors select a per-mip view of one full allocation. Backing creation must not - // depend on which view happens to be bound first. - return storage ? 0u : view_base_level; -} - -[[nodiscard]] inline constexpr bool -IsDepthUintTextureReinterpretation(vk::Format image_format, uint32_t guest_format, - vk::Format view_format) noexcept { - switch (image_format) { - case vk::Format::eD32Sfloat: - return guest_format == Prospero::GpuEnumValue(Prospero::BufferFormat::k32UInt) && - view_format == vk::Format::eR32Uint; - default: return false; - } -} - -[[nodiscard]] inline constexpr bool NeedsStaticSampledArrayView(bool shader_array, - bool dynamic_view_selected) { - return shader_array && !dynamic_view_selected; -} - -[[nodiscard]] inline constexpr StorageSampledViewShape -SelectStorageSampledViewShape(uint32_t type, uint32_t depth, uint32_t backing_layers) noexcept { - switch (static_cast(type)) { - case Prospero::ImageType::kColor2D: - return depth == 1 && backing_layers == 1 ? StorageSampledViewShape::Image2D - : StorageSampledViewShape::Unsupported; - case Prospero::ImageType::kColor2DArray: - return depth != 0 && depth == backing_layers ? StorageSampledViewShape::Image2DArray - : StorageSampledViewShape::Unsupported; - case Prospero::ImageType::kColor3D: - return depth != 0 && backing_layers == 1 ? StorageSampledViewShape::Image3D - : StorageSampledViewShape::Unsupported; - default: return StorageSampledViewShape::Unsupported; - } -} - -[[nodiscard]] inline constexpr bool IsSupportedRenderTargetElementSize(uint32_t size) noexcept { - switch (size) { - case 1: - case 2: - case 4: - case 8: return true; - default: return false; - } -} - [[nodiscard]] inline constexpr bool IsSupportedDisplayRenderTargetTileMode(uint32_t tile_mode) noexcept { return tile_mode == Prospero::GpuEnumValue(Prospero::TileMode::kRenderTarget); } -[[nodiscard]] inline constexpr bool -IsSupportedStandard64RenderTarget(const RenderTargetInfo& info) noexcept { +[[nodiscard]] inline constexpr bool IsSupportedStandard64RenderTarget(const ImageInfo& info) { if (info.tile_mode != Prospero::GpuEnumValue(Prospero::TileMode::kStandard64KB) || - info.address == 0 || (info.address & 0xffffu) != 0 || info.width == 0 || info.height == 0 || - info.bytes_per_element != 4 || info.levels != 1 || info.layers != 1 || info.samples != 1) { + info.data.address == 0 || (info.data.address & 0xffffu) != 0 || info.extent.width == 0 || + info.extent.height == 0 || info.bytes_per_block != 4 || info.resources.levels != 1 || + info.resources.layers != 1 || info.samples != 1) { return false; } - const auto expected_pitch = (static_cast(info.width) + 127u) & ~uint64_t {127u}; - const auto padded_height = (static_cast(info.height) + 127u) & ~uint64_t {127u}; + const auto expected_pitch = + (static_cast(info.extent.width) + 127u) & ~uint64_t {127u}; + const auto padded_height = + (static_cast(info.extent.height) + 127u) & ~uint64_t {127u}; return expected_pitch <= UINT32_MAX && info.pitch == expected_pitch && - expected_pitch <= UINT64_MAX / padded_height / info.bytes_per_element && - info.size == expected_pitch * padded_height * info.bytes_per_element; + expected_pitch <= UINT64_MAX / padded_height / info.bytes_per_block && + info.data.size == expected_pitch * padded_height * info.bytes_per_block; } -[[nodiscard]] inline constexpr bool IsTiledRenderTarget(const RenderTargetInfo& info) noexcept { +[[nodiscard]] inline constexpr bool IsTiledRenderTarget(const ImageInfo& info) noexcept { return info.tile_mode == Prospero::GpuEnumValue(Prospero::TileMode::kRenderTarget) || IsSupportedStandard64RenderTarget(info); } -[[nodiscard]] inline constexpr DepthTransitionSource -SelectDepthTransitionSource(bool depth_load_clear, bool sampled_native_available, - bool sampled_cpu_dirty, bool sampled_buffer_modified, - bool buffer_overlap, bool buffer_cpu_dirty) noexcept { - if (depth_load_clear) { - return DepthTransitionSource::None; +[[nodiscard]] inline bool IsRgba8SrgbReinterpretation(vk::Format cached, + vk::Format requested) noexcept { + switch (cached) { + case vk::Format::eR8G8B8A8Unorm: return requested == vk::Format::eR8G8B8A8Srgb; + case vk::Format::eR8G8B8A8Srgb: return requested == vk::Format::eR8G8B8A8Unorm; + case vk::Format::eB8G8R8A8Unorm: return requested == vk::Format::eB8G8R8A8Srgb; + case vk::Format::eB8G8R8A8Srgb: return requested == vk::Format::eB8G8R8A8Unorm; + default: return false; } - return sampled_native_available && !sampled_cpu_dirty && !sampled_buffer_modified && - !(buffer_overlap && buffer_cpu_dirty) - ? DepthTransitionSource::Native - : DepthTransitionSource::Guest; } -[[nodiscard]] inline MetaImageOverlap ClassifyMetaImageOverlap(bool sampled, bool writable_image, - bool gpu_modified, - bool buffer_modified, - bool cpu_dirty) { - if (sampled && !gpu_modified) { - return MetaImageOverlap::RetainSampled; +[[nodiscard]] inline bool IsRgba16UintFloatReinterpretation(vk::Format cached, + vk::Format requested) noexcept { + switch (cached) { + case vk::Format::eR16G16B16A16Sfloat: return requested == vk::Format::eR16G16B16A16Uint; + case vk::Format::eR16G16B16A16Uint: return requested == vk::Format::eR16G16B16A16Sfloat; + default: return false; } - if (writable_image && !gpu_modified && !buffer_modified && !cpu_dirty) { - return MetaImageOverlap::RetireImage; - } - return MetaImageOverlap::Unsupported; } -[[nodiscard]] inline float DecodeSrgbClearComponent(uint32_t value) { - const auto encoded = static_cast(value & 0xffu) / 255.0f; - return encoded <= 0.04045f ? encoded / 12.92f : std::pow((encoded + 0.055f) / 1.055f, 2.4f); +[[nodiscard]] inline bool IsRgba8UnormUintReinterpretation(vk::Format cached, + vk::Format requested) noexcept { + switch (cached) { + case vk::Format::eR8G8B8A8Unorm: return requested == vk::Format::eR8G8B8A8Uint; + case vk::Format::eR8G8B8A8Uint: return requested == vk::Format::eR8G8B8A8Unorm; + default: return false; + } } [[nodiscard]] inline bool DecodePackedColorClear(vk::Format format, uint32_t packed, - vk::ClearColorValue& clear) { + vk::ClearColorValue& clear) { vk::ClearColorValue next {}; const auto unorm8 = [](uint32_t value) { return static_cast(value & 0xffu) / 255.0f; }; + const auto srgb8 = [](uint32_t value) { + const auto encoded = static_cast(value & 0xffu) / 255.0f; + return encoded <= 0.04045f ? encoded / 12.92f + : std::pow((encoded + 0.055f) / 1.055f, 2.4f); + }; switch (format) { + case vk::Format::eR32Uint: + next.uint32[0] = packed; + break; + case vk::Format::eR32Sint: + next.int32[0] = static_cast(packed); + break; case vk::Format::eR8G8B8A8Srgb: - next.float32[0] = DecodeSrgbClearComponent(packed); - next.float32[1] = DecodeSrgbClearComponent(packed >> 8u); - next.float32[2] = DecodeSrgbClearComponent(packed >> 16u); + next.float32[0] = srgb8(packed); + next.float32[1] = srgb8(packed >> 8u); + next.float32[2] = srgb8(packed >> 16u); next.float32[3] = unorm8(packed >> 24u); break; case vk::Format::eB8G8R8A8Srgb: - next.float32[0] = DecodeSrgbClearComponent(packed >> 16u); - next.float32[1] = DecodeSrgbClearComponent(packed >> 8u); - next.float32[2] = DecodeSrgbClearComponent(packed); + next.float32[0] = srgb8(packed >> 16u); + next.float32[1] = srgb8(packed >> 8u); + next.float32[2] = srgb8(packed); next.float32[3] = unorm8(packed >> 24u); break; case vk::Format::eR8G8B8A8Unorm: @@ -573,41 +574,6 @@ SelectDepthTransitionSource(bool depth_load_clear, bool sampled_native_available return true; } -[[nodiscard]] inline bool CanNativeClearDepthFromBuffer(const DepthTargetInfo& target, - uint64_t address, uint64_t size) { - const bool d32 = - target.format == vk::Format::eD32Sfloat || target.format == vk::Format::eD32SfloatS8Uint; - return address == target.address && size == target.size && target.layers == 1 && d32 && - target.samples == 1 && - target.guest_format == Prospero::GpuEnumValue(Prospero::BufferFormat::k32Float) && - target.bytes_per_element == 4 && - target.tile_mode == Prospero::GpuEnumValue(Prospero::TileMode::kDepth) && - target.htile_address == 0 && target.htile_size == 0; -} - -[[nodiscard]] inline bool CanLoadStencilAttachment(const DepthTargetInfo& target, - bool stencil_initialized) { - const bool has_stencil = target.stencil_address != 0 || target.stencil_size != 0; - return !has_stencil || !target.stencil_access || target.stencil_load_clear || - stencil_initialized; -} - -[[nodiscard]] inline bool CanLoadRawStencilPlane(const DepthTargetInfo& target) { - const bool has_stencil = target.stencil_address != 0 || target.stencil_size != 0; - return has_stencil && !target.stencil_htile_compressed; -} - -[[nodiscard]] inline bool IsDepthTargetRangeCompatible(const DepthTargetInfo& target, - uint64_t address, uint64_t size) { - if (address == 0 || size == 0 || address > UINT64_MAX - size) { - EXIT("invalid depth-target view range\n"); - } - const bool depth = address == target.address && size <= target.size; - const bool stencil = target.stencil_address != 0 && address == target.stencil_address && - size == target.stencil_size; - return depth || stencil; -} - [[nodiscard]] inline bool ImageRangeOverlaps(uint64_t left, uint64_t left_size, uint64_t right, uint64_t right_size) { if (left_size == 0 || right_size == 0 || left > UINT64_MAX - left_size || @@ -617,6 +583,10 @@ SelectDepthTransitionSource(bool depth_load_clear, bool sampled_native_available return left < right + right_size && right < left + left_size; } +[[nodiscard]] inline bool ImageRangeOverlaps(GuestRange left, GuestRange right) { + return ImageRangeOverlaps(left.address, left.size, right.address, right.size); +} + [[nodiscard]] inline bool ImagePageRangesOverlap(uint64_t left, uint64_t left_size, uint64_t right, uint64_t right_size) { if (left_size == 0 || right_size == 0 || left > UINT64_MAX - left_size || @@ -630,587 +600,8 @@ SelectDepthTransitionSource(bool depth_load_clear, bool sampled_native_available return left_first <= right_last && right_first <= left_last; } -[[nodiscard]] inline SampledOverlap ClassifySampledOverlap(const ImageInfo& requested, - const ImageInfo& cached, - bool cached_gpu_modified) { - if (!ImagePageRangesOverlap(requested.address, requested.size, cached.address, cached.size)) { - return SampledOverlap::None; - } - return !cached_gpu_modified ? SampledOverlap::ReadOnlyAlias : SampledOverlap::Unsupported; -} - -[[nodiscard]] inline bool IsRgba8SrgbReinterpretation(vk::Format cached, - vk::Format requested) noexcept; - -[[nodiscard]] inline bool IsR32UintFloatReinterpretation(vk::Format cached, - vk::Format requested) noexcept { - switch (cached) { - case vk::Format::eR32Uint: return requested == vk::Format::eR32Sfloat; - case vk::Format::eR32Sfloat: return requested == vk::Format::eR32Uint; - default: return false; - } -} - -[[nodiscard]] inline bool IsRgba16UintFloatReinterpretation(vk::Format cached, - vk::Format requested) noexcept { - switch (cached) { - case vk::Format::eR16G16B16A16Sfloat: return requested == vk::Format::eR16G16B16A16Uint; - case vk::Format::eR16G16B16A16Uint: return requested == vk::Format::eR16G16B16A16Sfloat; - default: return false; - } -} - -[[nodiscard]] inline bool IsRgba8UnormUintReinterpretation(vk::Format cached, - vk::Format requested) noexcept { - switch (cached) { - case vk::Format::eR8G8B8A8Unorm: return requested == vk::Format::eR8G8B8A8Uint; - case vk::Format::eR8G8B8A8Uint: return requested == vk::Format::eR8G8B8A8Unorm; - default: return false; - } -} - -[[nodiscard]] inline bool IsSampledDepthExpansion(const ImageInfo& sampled, - const DepthTargetInfo& target) noexcept { - const bool array_expansion = - sampled.type == Prospero::GpuEnumValue(Prospero::ImageType::kColor2DArray) && - sampled.depth > 1 && target.size <= UINT64_MAX / sampled.depth && - sampled.size == target.size * sampled.depth && sampled.width == target.width && - sampled.height == target.height && sampled.pitch == target.pitch; - return sampled.address == target.address && sampled.size > target.size && - sampled.format == target.guest_format && target.format == vk::Format::eD32Sfloat && - target.guest_format == Prospero::GpuEnumValue(Prospero::BufferFormat::k32Float) && - target.bytes_per_element == 4 && target.stencil_address == 0 && - target.stencil_size == 0 && target.htile_address == 0 && target.htile_size == 0 && - target.layers == 1 && array_expansion && sampled.base_level == 0 && - sampled.levels == 1 && sampled.view_levels == 1 && sampled.tile == target.tile_mode && - sampled.base_array == 0; -} - -[[nodiscard]] inline StorageSampledOverlap ClassifyStorageSampledOverlap( - const ImageInfo& requested, const ImageInfo& cached, vk::Format requested_view_format, - vk::Format cached_image_format, bool cached_gpu_modified, bool cached_cpu_dirty, - bool exact_mip_subresource = false, bool cached_buffer_modified = false, - bool tracker_gpu_modified = true) { - if (!ImagePageRangesOverlap(requested.address, requested.size, cached.address, cached.size)) { - return StorageSampledOverlap::None; - } - const bool same_backing = - requested.address == cached.address && requested.size == cached.size && - requested.width == cached.width && requested.height == cached.height && - requested.pitch == cached.pitch && requested.base_level == cached.base_level && - requested.levels == cached.levels && requested.view_levels == cached.view_levels && - requested.tile == cached.tile && requested.depth == cached.depth && - requested.type == cached.type && requested.base_array == cached.base_array; - const bool compatible_format = - (requested.format == cached.format && requested_view_format == cached_image_format) || - IsRgba8SrgbReinterpretation(cached_image_format, requested_view_format) || - IsR32UintFloatReinterpretation(cached_image_format, requested_view_format); - if (same_backing && compatible_format && cached_gpu_modified && !cached_cpu_dirty) { - return StorageSampledOverlap::ExactImage; - } - if (exact_mip_subresource && cached_gpu_modified && tracker_gpu_modified && - !cached_buffer_modified && !cached_cpu_dirty) { - return StorageSampledOverlap::RetireStorage; - } - return StorageSampledOverlap::Unsupported; -} - -[[nodiscard]] inline HostWriteOverlap -ClassifyHostWriteOverlap(uint64_t write_address, uint64_t write_size, uint64_t image_address, - uint64_t image_size, bool host_refreshable, bool gpu_modified, - bool metadata_overlap) { - if (!ImagePageRangesOverlap(write_address, write_size, image_address, image_size)) { - return HostWriteOverlap::None; - } - return host_refreshable && !gpu_modified && !metadata_overlap - ? HostWriteOverlap::InvalidateImage - : HostWriteOverlap::Unsupported; -} - -[[nodiscard]] inline BufferImageWrite -ClassifyBufferImageWrite(uint64_t buffer_address, uint64_t buffer_size, uint64_t image_address, - uint64_t image_size, BufferImageBinding binding, bool image_gpu_modified, - bool buffer_formatted, bool image_buffer_modified = false) { - if (!ImagePageRangesOverlap(buffer_address, buffer_size, image_address, image_size)) { - return BufferImageWrite::None; - } - const bool exact = buffer_address == image_address && buffer_size == image_size; - const auto offset = - buffer_address >= image_address ? buffer_address - image_address : UINT64_MAX; - const bool contained = offset <= image_size && buffer_size <= image_size - offset; - const auto image_offset = - image_address >= buffer_address ? image_address - buffer_address : UINT64_MAX; - const bool image_contained = - image_offset <= buffer_size && image_size <= buffer_size - image_offset; - const bool buffer_page_aligned = - ((buffer_address | buffer_size) & (TRACKER_PAGE_SIZE - 1)) == 0; - const bool image_page_aligned = ((image_address | image_size) & (TRACKER_PAGE_SIZE - 1)) == 0; - switch (binding) { - case BufferImageBinding::Texture: - return contained && image_page_aligned && !image_gpu_modified - ? BufferImageWrite::InvalidateTexture - : BufferImageWrite::Unsupported; - case BufferImageBinding::VideoOut: - if (!exact || !buffer_page_aligned || !buffer_formatted) { - return BufferImageWrite::Unsupported; - } - return image_gpu_modified ? BufferImageWrite::SynchronizeVideoOut - : BufferImageWrite::InvalidateVideoOut; - case BufferImageBinding::RenderTarget: - if (!exact || !buffer_page_aligned || !buffer_formatted) { - return BufferImageWrite::Unsupported; - } - if (image_gpu_modified && !image_buffer_modified) { - return BufferImageWrite::SynchronizeRenderTarget; - } - return !image_gpu_modified && image_buffer_modified - ? BufferImageWrite::InvalidateRenderTarget - : BufferImageWrite::Unsupported; - case BufferImageBinding::StorageTexture: - if (!buffer_page_aligned || !image_page_aligned || !buffer_formatted) { - return BufferImageWrite::Unsupported; - } - if (contained && image_gpu_modified && !image_buffer_modified) { - return BufferImageWrite::SynchronizeStorageTexture; - } - return image_contained && !image_gpu_modified && image_buffer_modified - ? BufferImageWrite::InvalidateStorageTexture - : BufferImageWrite::Unsupported; - case BufferImageBinding::DepthTarget: - if (!exact || !buffer_page_aligned || !buffer_formatted) { - return BufferImageWrite::Unsupported; - } - if (image_gpu_modified && !image_buffer_modified) { - return BufferImageWrite::SynchronizeDepthTarget; - } - return !image_gpu_modified && image_buffer_modified - ? BufferImageWrite::InvalidateDepthTarget - : BufferImageWrite::Unsupported; - case BufferImageBinding::Unsupported: return BufferImageWrite::Unsupported; - } - return BufferImageWrite::Unsupported; -} - -[[nodiscard]] inline StorageBufferRebind -ClassifyStorageBufferRebind(bool buffer_overlap, bool cached_gpu_modified, - bool cached_buffer_modified, bool tracker_gpu_modified, - bool tracker_cpu_modified, bool coherent_guest_source) noexcept { - if (cached_gpu_modified != tracker_gpu_modified || - (tracker_gpu_modified && tracker_cpu_modified)) { - return StorageBufferRebind::Unsupported; - } - if (!cached_buffer_modified) { - return StorageBufferRebind::Reuse; - } - return buffer_overlap && !tracker_gpu_modified && coherent_guest_source - ? StorageBufferRebind::RefreshFromBacking - : StorageBufferRebind::Unsupported; -} - -[[nodiscard]] inline DepthOverlap ClassifyDepthOverlap(const ImageInfo& sampled, - bool sampled_gpu_modified, - const DepthTargetInfo& depth) { - const bool depth_overlap = - ImageRangeOverlaps(sampled.address, sampled.size, depth.address, depth.size); - const bool stencil_overlap = - depth.stencil_address != 0 && ImageRangeOverlaps(sampled.address, sampled.size, - depth.stencil_address, depth.stencil_size); - if (!depth_overlap && !stencil_overlap) { - return DepthOverlap::None; - } - const bool has_stencil = depth.stencil_address != 0 || depth.stencil_size != 0; - const auto* depth_policy = FindGuestDepthFormatPolicy(depth.guest_format); - const bool exact_depth_format = - sampled.format == depth.guest_format && depth_policy != nullptr && - depth.bytes_per_element == depth_policy->bytes_per_element && - DepthAspectTransferBytes(depth.format) == depth.bytes_per_element && - (has_stencil ? IsStencilAttachmentFormat(*depth_policy, depth.format) - : depth.format == depth_policy->depth_attachment_format); - // Keep this equivalent to ResolveDepthOverlap: recreate the sampled color image as - // depth and preserve its depth aspect. A disjoint stencil component is initialized separately. - const bool exact_depth_load = - !stencil_overlap && !depth.depth_load_clear && sampled.address == depth.address && - sampled.size == depth.size && sampled.width == depth.width && - sampled.height == depth.height && sampled.pitch == depth.pitch && sampled.base_level == 0 && - sampled.levels == 1 && sampled.view_levels == 1 && sampled.tile == depth.tile_mode && - sampled.depth == 1 && - sampled.type == Prospero::GpuEnumValue(Prospero::ImageType::kColor2D) && - depth.layers == 1 && sampled.base_array == 0 && exact_depth_format; - if (!sampled_gpu_modified && exact_depth_load) { - return DepthOverlap::RetireSampled; - } - if (sampled.address == depth.address && !sampled_gpu_modified && depth.depth_load_clear && - (!has_stencil || - (depth.stencil_address != 0 && depth.stencil_size != 0 && depth.stencil_load_clear))) { - return DepthOverlap::RetireSampled; - } - return DepthOverlap::Unsupported; -} - -// Mirrors storage -> depth-target binding transition. An inaccessible stencil aspect -// does not require the otherwise necessary image content copy. -[[nodiscard]] inline DepthOverlap -ClassifyStorageDepthOverlap(const ImageInfo& storage, bool storage_gpu_modified, - bool storage_buffer_modified, bool storage_cpu_dirty, - bool tracker_gpu_modified, const DepthTargetInfo& depth) { - const bool overlaps_depth = - ImageRangeOverlaps(storage.address, storage.size, depth.address, depth.size); - const bool overlaps_stencil = - depth.stencil_address != 0 && ImageRangeOverlaps(storage.address, storage.size, - depth.stencil_address, depth.stencil_size); - if (!overlaps_depth && !overlaps_stencil) { - return DepthOverlap::None; - } - const bool guest_current = HasGuestCurrentImageOwnership( - storage_gpu_modified, storage_buffer_modified, storage_cpu_dirty, tracker_gpu_modified); - const bool aspects_discarded = - (!overlaps_depth || !depth.depth_access) && (!overlaps_stencil || !depth.stencil_access); - return guest_current || (!storage_buffer_modified && !storage_cpu_dirty && aspects_discarded) - ? DepthOverlap::RetireStorage - : DepthOverlap::Unsupported; -} - -[[nodiscard]] inline bool -CanRetireGuestCurrentDepthForSampled(const ImageInfo& sampled, const DepthTargetInfo& depth, - bool depth_gpu_modified, bool depth_buffer_modified, - bool depth_tracker_gpu_modified, - bool stencil_tracker_gpu_modified) noexcept { - const bool overlaps_depth = - ImageRangeOverlaps(sampled.address, sampled.size, depth.address, depth.size); - const bool overlaps_stencil = - depth.stencil_address != 0 && ImageRangeOverlaps(sampled.address, sampled.size, - depth.stencil_address, depth.stencil_size); - return (overlaps_depth || overlaps_stencil) && - HasGuestCurrentImageOwnership(depth_gpu_modified, depth_buffer_modified, false, - depth_tracker_gpu_modified || - stencil_tracker_gpu_modified); -} - -[[nodiscard]] inline bool -CanRetireGuestCurrentSampledForDepth(const ImageInfo& sampled, const DepthTargetInfo& depth, - bool sampled_gpu_modified, bool sampled_buffer_modified, - bool sampled_cpu_dirty, bool tracker_gpu_modified, - bool guest_source_current) noexcept { - const bool overlaps_depth = - ImageRangeOverlaps(sampled.address, sampled.size, depth.address, depth.size); - const bool overlaps_stencil = - depth.stencil_address != 0 && ImageRangeOverlaps(sampled.address, sampled.size, - depth.stencil_address, depth.stencil_size); - return (overlaps_depth || overlaps_stencil) && guest_source_current && - HasGuestCurrentImageOwnership(sampled_gpu_modified, sampled_buffer_modified, - sampled_cpu_dirty, tracker_gpu_modified); -} - -[[nodiscard]] inline RenderTargetOverlap -ClassifyRenderTargetOverlap(const ImageInfo& sampled, bool sampled_gpu_modified, - const RenderTargetInfo& target) { - if (!ImagePageRangesOverlap(sampled.address, sampled.size, target.address, target.size)) { - return RenderTargetOverlap::None; - } - if (!ImageRangeOverlaps(sampled.address, sampled.size, target.address, target.size)) { - return RenderTargetOverlap::None; - } - return !sampled_gpu_modified ? RenderTargetOverlap::RetireSampled - : RenderTargetOverlap::Unsupported; -} - -[[nodiscard]] inline RenderTargetOverlap -ClassifyStorageRenderTargetOverlap(const ImageInfo& storage, vk::Format storage_format, - bool storage_gpu_modified, bool storage_buffer_modified, - bool storage_cpu_dirty, bool tracker_gpu_modified, - const RenderTargetInfo& target) { - if (!ImagePageRangesOverlap(storage.address, storage.size, target.address, target.size)) { - return RenderTargetOverlap::None; - } - if (!ImageRangeOverlaps(storage.address, storage.size, target.address, target.size)) { - return RenderTargetOverlap::None; - } - const bool exact_native_image = - storage.address == target.address && storage.size == target.size && - storage_format == target.format && storage.width == target.width && - storage.height == target.height && storage.pitch == target.pitch && - storage.base_level == 0 && storage.levels == 1 && storage.view_levels == 1 && - storage.tile == target.tile_mode && storage.depth == 1 && - storage.type == Prospero::GpuEnumValue(Prospero::ImageType::kColor2D) && - storage.base_array == 0 && target.levels == 1 && target.layers == 1 && target.samples == 1; - if (exact_native_image && storage_gpu_modified && tracker_gpu_modified && - !storage_buffer_modified && !storage_cpu_dirty) { - return RenderTargetOverlap::PreserveStorage; - } - return HasGuestCurrentImageOwnership(storage_gpu_modified, storage_buffer_modified, - storage_cpu_dirty, tracker_gpu_modified) - ? RenderTargetOverlap::RetireStorage - : RenderTargetOverlap::Unsupported; -} - -[[nodiscard]] inline bool IsRgba8SrgbViewFormat(vk::Format format) noexcept { - return format == vk::Format::eR8G8B8A8Unorm || format == vk::Format::eR8G8B8A8Srgb || - format == vk::Format::eB8G8R8A8Unorm || format == vk::Format::eB8G8R8A8Srgb; -} - -[[nodiscard]] inline bool IsRgba8SrgbReinterpretation(vk::Format cached, - vk::Format requested) noexcept { - switch (cached) { - case vk::Format::eR8G8B8A8Unorm: return requested == vk::Format::eR8G8B8A8Srgb; - case vk::Format::eR8G8B8A8Srgb: return requested == vk::Format::eR8G8B8A8Unorm; - case vk::Format::eB8G8R8A8Unorm: return requested == vk::Format::eB8G8R8A8Srgb; - case vk::Format::eB8G8R8A8Srgb: return requested == vk::Format::eB8G8R8A8Unorm; - default: return false; - } -} - -[[nodiscard]] inline bool IsCompatibleRenderTargetView(const RenderTargetInfo& cached, - const RenderTargetInfo& requested) noexcept { - return cached.address == requested.address && cached.size == requested.size && - cached.width == requested.width && cached.height == requested.height && - cached.pitch == requested.pitch && - cached.bytes_per_element == requested.bytes_per_element && - cached.tile_mode == requested.tile_mode && cached.levels == requested.levels && - cached.layers == requested.layers && cached.samples == requested.samples && - IsRgba8SrgbReinterpretation(cached.format, requested.format); -} - -[[nodiscard]] inline RenderTargetOverlap -ClassifySampledRenderTargetOverlap(const ImageInfo& sampled, const RenderTargetInfo& target, - bool target_buffer_modified) { - if (!ImagePageRangesOverlap(sampled.address, sampled.size, target.address, target.size)) { - return RenderTargetOverlap::None; - } - if (!ImageRangeOverlaps(sampled.address, sampled.size, target.address, target.size)) { - return RenderTargetOverlap::None; - } - return !target_buffer_modified ? RenderTargetOverlap::RetireTarget - : RenderTargetOverlap::Unsupported; -} - -[[nodiscard]] inline StorageImageOverlap -ClassifyStorageImageOverlap(uint64_t requested_address, uint64_t requested_size, - uint64_t cached_address, uint64_t cached_size, bool sampled, - bool gpu_modified, bool buffer_modified, bool tracker_gpu_modified) { - if (!ImagePageRangesOverlap(requested_address, requested_size, cached_address, cached_size)) { - return StorageImageOverlap::None; - } - if (!ImageRangeOverlaps(requested_address, requested_size, cached_address, cached_size)) { - return StorageImageOverlap::PageNeighbor; - } - return sampled && !gpu_modified && !buffer_modified && !tracker_gpu_modified - ? StorageImageOverlap::RetireSampled - : StorageImageOverlap::Unsupported; -} - -[[nodiscard]] inline constexpr bool LayeredBackingContains(uint64_t container_size, - uint32_t container_layers, - uint64_t view_size, - uint32_t view_layers) { - return container_layers >= view_layers && container_layers != 0 && view_layers != 0 && - container_size % container_layers == 0 && view_size % view_layers == 0 && - container_size / container_layers == view_size / view_layers; -} - -[[nodiscard]] inline constexpr bool -LayeredPlaneContains(uint64_t container_address, uint64_t container_size, uint32_t container_layers, - uint64_t view_address, uint64_t view_size, uint32_t view_layers) { - return container_address == 0 ? view_address == 0 && view_size == 0 - : view_address == container_address && - LayeredBackingContains(container_size, container_layers, - view_size, view_layers); -} - -[[nodiscard]] inline bool -IsCompatibleRenderTargetBacking(const RenderTargetInfo& cached, - const RenderTargetInfo& requested) noexcept { - return cached.address == requested.address && - LayeredBackingContains(cached.size, cached.layers, requested.size, requested.layers) && - cached.width == requested.width && cached.height == requested.height && - cached.pitch == requested.pitch && - cached.bytes_per_element == requested.bytes_per_element && - cached.tile_mode == requested.tile_mode && cached.levels == requested.levels && - cached.samples == requested.samples && - (cached.format == requested.format || - IsRgba8SrgbReinterpretation(cached.format, requested.format)); -} - -[[nodiscard]] inline bool -IsCompatibleDepthTargetBacking(const DepthTargetInfo& cached, - const DepthTargetInfo& requested) noexcept { - return cached.address == requested.address && - LayeredBackingContains(cached.size, cached.layers, requested.size, requested.layers) && - LayeredPlaneContains(cached.stencil_address, cached.stencil_size, cached.layers, - requested.stencil_address, requested.stencil_size, - requested.layers) && - LayeredPlaneContains(cached.htile_address, cached.htile_size, cached.layers, - requested.htile_address, requested.htile_size, requested.layers) && - cached.format == requested.format && cached.guest_format == requested.guest_format && - cached.width == requested.width && cached.height == requested.height && - cached.pitch == requested.pitch && - cached.bytes_per_element == requested.bytes_per_element && - cached.tile_mode == requested.tile_mode && cached.samples == requested.samples && - cached.stencil_htile_compressed == requested.stencil_htile_compressed; -} - -[[nodiscard]] inline RenderTargetOverlap -ClassifyRenderTargetOverlap(const RenderTargetInfo& cached, bool cached_gpu_modified, - bool cached_buffer_modified, bool tracker_gpu_modified, - bool guest_source_current, const RenderTargetInfo& requested) { - if (!ImagePageRangesOverlap(cached.address, cached.size, requested.address, requested.size)) { - return RenderTargetOverlap::None; - } - if (!ImageRangeOverlaps(cached.address, cached.size, requested.address, requested.size)) { - return RenderTargetOverlap::None; - } - const bool expand = requested.layers > cached.layers && - IsCompatibleRenderTargetBacking(requested, cached) && - cached.format == requested.format; - if (expand && cached_gpu_modified && tracker_gpu_modified && !cached_buffer_modified) { - return RenderTargetOverlap::ExpandTarget; - } - // A clean, page-isolated overlap is allocation-pool reuse, including a contained subrange. - // Compatible views are handled before this classifier and are never retired here. - const bool page_isolated = - cached.address % TRACKER_PAGE_SIZE == 0 && cached.size % TRACKER_PAGE_SIZE == 0 && - requested.address % TRACKER_PAGE_SIZE == 0 && requested.size % TRACKER_PAGE_SIZE == 0; - const bool incompatible_format = cached.format != requested.format && - !IsRgba8SrgbReinterpretation(cached.format, requested.format); - const bool pool_storage_shape_changed = - incompatible_format || cached.width != requested.width || - cached.height != requested.height || cached.pitch != requested.pitch || - cached.bytes_per_element != requested.bytes_per_element || - cached.tile_mode != requested.tile_mode || cached.levels != requested.levels || - cached.layers != requested.layers || cached.samples != requested.samples; - const bool new_allocation = cached.address != requested.address || - cached.size != requested.size || pool_storage_shape_changed; - return page_isolated && new_allocation && guest_source_current && - HasGuestCurrentImageOwnership(cached_gpu_modified, cached_buffer_modified, false, - tracker_gpu_modified) - ? RenderTargetOverlap::RetireTarget - : RenderTargetOverlap::Unsupported; -} - -[[nodiscard]] inline DepthOverlap ClassifyDepthTargetOverlap(const DepthTargetInfo& cached, - bool cached_gpu_modified, - bool cached_buffer_modified, - const DepthTargetInfo& requested) { - const auto overlaps = [](uint64_t left, uint64_t left_size, uint64_t right, uint64_t right_size, - bool pages) { - if (left == 0 || left_size == 0 || right == 0 || right_size == 0) { - return false; - } - return pages ? ImagePageRangesOverlap(left, left_size, right, right_size) - : ImageRangeOverlaps(left, left_size, right, right_size); - }; - const auto planes_overlap = [&](bool pages) { - return overlaps(cached.address, cached.size, requested.address, requested.size, pages) || - overlaps(cached.address, cached.size, requested.stencil_address, - requested.stencil_size, pages) || - overlaps(cached.stencil_address, cached.stencil_size, requested.address, - requested.size, pages) || - overlaps(cached.stencil_address, cached.stencil_size, requested.stencil_address, - requested.stencil_size, pages); - }; - if (!planes_overlap(true) || !planes_overlap(false)) { - return DepthOverlap::None; - } - const bool expand = - requested.layers > cached.layers && IsCompatibleDepthTargetBacking(requested, cached); - if (expand && cached_gpu_modified && !cached_buffer_modified) { - return DepthOverlap::ExpandTarget; - } - const bool exact_discard = - requested.depth_load_clear && cached_gpu_modified && !cached_buffer_modified && - cached.address == requested.address && cached.size == requested.size && - cached.samples == requested.samples && cached.stencil_address == 0 && - cached.stencil_size == 0 && cached.htile_address == 0 && cached.htile_size == 0 && - requested.stencil_address == 0 && requested.stencil_size == 0 && - requested.htile_address == 0 && requested.htile_size == 0; - if (exact_discard) { - return DepthOverlap::DiscardTarget; - } - const bool exact_plane_rebind = - (cached.address == requested.address && cached.size == requested.size) || - (cached.address == requested.stencil_address && cached.size == requested.stencil_size) || - (cached.stencil_address == requested.address && cached.stencil_size == requested.size) || - (cached.stencil_address == requested.stencil_address && - cached.stencil_size == requested.stencil_size); - const bool same_shape = cached.format == requested.format && - cached.guest_format == requested.guest_format && - cached.width == requested.width && cached.height == requested.height && - cached.pitch == requested.pitch && - cached.bytes_per_element == requested.bytes_per_element && - cached.tile_mode == requested.tile_mode && cached.layers == 1 && - requested.layers == 1 && cached.samples == 1 && requested.samples == 1; - return exact_plane_rebind && same_shape && !cached_buffer_modified - ? DepthOverlap::RecreateTarget - : DepthOverlap::Unsupported; -} - -[[nodiscard]] inline bool CanRecreateDepthForRenderTarget(const DepthTargetInfo& depth, - bool gpu_modified, bool buffer_modified, - bool tracker_gpu_modified, - bool guest_source_current, - const RenderTargetInfo& target) noexcept { - const bool exact_depth = depth.address == target.address && depth.size == target.size; - const bool exact_stencil = - depth.stencil_address == target.address && depth.stencil_size == target.size; - const auto contains = [&](uint64_t address, uint64_t size) { - if (address == 0 || size == 0 || address < target.address) { - return false; - } - const auto offset = address - target.address; - return offset <= target.size && size <= target.size - offset; - }; - const bool page_isolated_rebind = - target.address % TRACKER_PAGE_SIZE == 0 && target.size % TRACKER_PAGE_SIZE == 0 && - ((contains(depth.address, depth.size) && depth.address % TRACKER_PAGE_SIZE == 0 && - depth.size % TRACKER_PAGE_SIZE == 0) || - (contains(depth.stencil_address, depth.stencil_size) && - depth.stencil_address % TRACKER_PAGE_SIZE == 0 && - depth.stencil_size % TRACKER_PAGE_SIZE == 0)); - const bool ownership_consistent = gpu_modified == tracker_gpu_modified && !buffer_modified; - const bool source_available = - ((exact_depth || exact_stencil) && gpu_modified) || guest_source_current; - return ownership_consistent && source_available && - (exact_depth || exact_stencil || page_isolated_rebind) && depth.layers == 1 && - target.levels == 1 && target.layers == 1 && depth.samples == 1 && target.samples == 1; -} - -[[nodiscard]] inline bool CanRecreateRenderTargetForDepth(const RenderTargetInfo& target, - bool gpu_modified, bool buffer_modified, - bool tracker_gpu_modified, - bool guest_source_current, - const DepthTargetInfo& depth) noexcept { - const bool exact_depth = target.address == depth.address && target.size == depth.size; - const bool exact_stencil = - target.address == depth.stencil_address && target.size == depth.stencil_size; - const auto overlaps_plane = [&](uint64_t address, uint64_t size) { - return address != 0 && size != 0 && - ImageRangeOverlaps(target.address, target.size, address, size); - }; - const bool page_isolated_rebind = - target.address % TRACKER_PAGE_SIZE == 0 && target.size % TRACKER_PAGE_SIZE == 0 && - ((overlaps_plane(depth.address, depth.size) && depth.address % TRACKER_PAGE_SIZE == 0 && - depth.size % TRACKER_PAGE_SIZE == 0) || - (overlaps_plane(depth.stencil_address, depth.stencil_size) && - depth.stencil_address % TRACKER_PAGE_SIZE == 0 && - depth.stencil_size % TRACKER_PAGE_SIZE == 0)); - const bool source_available = - (gpu_modified && (exact_depth || exact_stencil)) || guest_source_current; - return gpu_modified == tracker_gpu_modified && !buffer_modified && source_available && - (exact_depth || exact_stencil || page_isolated_rebind) && target.levels == 1 && - target.layers == 1 && depth.layers == 1 && target.samples == 1 && depth.samples == 1; -} - -[[nodiscard]] inline bool RequiresMultisampleDepthRefresh(const DepthTargetInfo& info, - bool buffer_modified, - bool depth_cpu_modified, - bool stencil_cpu_modified) noexcept { - if (info.samples == 1) { - return false; - } - const bool depth_refresh = - info.depth_access && !info.depth_load_clear && (buffer_modified || depth_cpu_modified); - const bool stencil_refresh = info.stencil_access && !info.stencil_load_clear && - (buffer_modified || stencil_cpu_modified); - return depth_refresh || stencil_refresh; +[[nodiscard]] inline bool ImagePageRangesOverlap(GuestRange left, GuestRange right) { + return ImagePageRangesOverlap(left.address, left.size, right.address, right.size); } } // namespace Libs::Graphics diff --git a/src/graphics/host_gpu/renderer/imageView.cpp b/src/graphics/host_gpu/renderer/imageView.cpp index 808c225f..b1a25f83 100644 --- a/src/graphics/host_gpu/renderer/imageView.cpp +++ b/src/graphics/host_gpu/renderer/imageView.cpp @@ -1,9 +1,8 @@ #include "graphics/host_gpu/renderer/imageView.h" #include "common/assert.h" -#include "graphics/host_gpu/objects/textureCommon.h" -#include "graphics/host_gpu/renderer/renderContext.h" -#include "graphics/host_gpu/renderer/textureCache.h" +#include "graphics/host_gpu/graphicContext.h" +#include "graphics/host_gpu/renderer/image.h" #include @@ -11,62 +10,286 @@ namespace Libs::Graphics { namespace { -void CreateView(VulkanImage& image, int view_index, - vk::ImageViewType view_type, vk::ImageAspectFlags aspect_mask, - vk::ComponentMapping components, uint32_t base_array_layer, uint32_t base_mip_level, - uint32_t layer_count, uint32_t level_count, - vk::Format view_format = vk::Format::eUndefined, - vk::ImageUsageFlags view_usage = {}) { - auto& graphics = GetRenderContext().GetGraphics(); - if (view_index < 0 || view_index >= VulkanImage::VIEW_MAX || - image.image_view[view_index] != nullptr) { - EXIT("invalid image-view creation target: image=%p index=%d current_view=%d\n", - static_cast(&image), view_index, - view_index >= 0 && view_index < VulkanImage::VIEW_MAX && - image.image_view[view_index] != nullptr); - } - - vk::ImageViewUsageCreateInfo usage_info {}; - usage_info.sType = vk::StructureType::eImageViewUsageCreateInfo; - usage_info.usage = view_usage; - - vk::ImageViewCreateInfo create_info {}; - create_info.sType = vk::StructureType::eImageViewCreateInfo; - create_info.pNext = view_usage ? &usage_info : nullptr; - create_info.image = image.image; - create_info.viewType = view_type; - create_info.format = view_format != vk::Format::eUndefined ? view_format : image.format; - create_info.components = components; - create_info.subresourceRange.aspectMask = aspect_mask; - create_info.subresourceRange.baseArrayLayer = base_array_layer; - create_info.subresourceRange.baseMipLevel = base_mip_level; - create_info.subresourceRange.layerCount = layer_count; - create_info.subresourceRange.levelCount = level_count; - - const auto result = - graphics.device.createImageView(&create_info, nullptr, &image.image_view[view_index]); - if (result != vk::Result::eSuccess || image.image_view[view_index] == nullptr) { - EXIT("failed to create image view: result=%d image_format=%d view_format=%d index=%d\n", - static_cast(result), static_cast(image.format), - static_cast(create_info.format), view_index); +[[nodiscard]] bool IsComponentSwizzle(vk::ComponentSwizzle swizzle) { + switch (swizzle) { + case vk::ComponentSwizzle::eIdentity: + case vk::ComponentSwizzle::eZero: + case vk::ComponentSwizzle::eOne: + case vk::ComponentSwizzle::eR: + case vk::ComponentSwizzle::eG: + case vk::ComponentSwizzle::eB: + case vk::ComponentSwizzle::eA: return true; + default: return false; } } -void CreateRenderTargetView(VulkanImage& image, int index, - vk::ComponentSwizzle r, vk::ComponentSwizzle g, vk::ComponentSwizzle b, - vk::ComponentSwizzle a, vk::ImageViewType type = vk::ImageViewType::e2D, - vk::Format view_format = vk::Format::eUndefined, - vk::ImageUsageFlags view_usage = {}, uint32_t level_count = 0) { - const auto layer_count = type == vk::ImageViewType::e2DArray ? image.layers : 1u; - CreateView(image, index, type, vk::ImageAspectFlagBits::eColor, {r, g, b, a}, 0, 0, - layer_count, level_count == 0 ? image.mip_levels : level_count, view_format, - view_usage); +[[nodiscard]] bool IsCompatibleViewFormat(vk::Format image_format, vk::Format view_format) { + return ImageViewOps::FormatsCompatible(image_format, view_format); +} + +[[nodiscard]] bool IsStencilViewFormat(vk::Format format) { + switch (format) { + case vk::Format::eS8Uint: + case vk::Format::eR8Uint: + case vk::Format::eR8Unorm: return true; + default: return false; + } +} + +[[nodiscard]] bool IsDepthViewFormat(vk::Format format) { + switch (format) { + case vk::Format::eD16Unorm: + case vk::Format::eR16Unorm: + case vk::Format::eD32Sfloat: + case vk::Format::eR32Sfloat: + case vk::Format::eR32Uint: return true; + default: return false; + } +} + +[[nodiscard]] bool IsValidViewType(const VulkanImage& image, const ImageViewInfo& info) { + switch (image.image_type) { + case vk::ImageType::e1D: + if (info.type != vk::ImageViewType::e1D && info.type != vk::ImageViewType::e1DArray) { + return false; + } + return info.type != vk::ImageViewType::e1D || info.layer_count == 1; + case vk::ImageType::e2D: + switch (info.type) { + case vk::ImageViewType::e2D: return info.layer_count == 1; + case vk::ImageViewType::e2DArray: return true; + case vk::ImageViewType::eCube: + return static_cast(image.flags & + vk::ImageCreateFlagBits::eCubeCompatible) && + info.base_layer % 6 == 0 && info.layer_count == 6; + case vk::ImageViewType::eCubeArray: + return static_cast(image.flags & + vk::ImageCreateFlagBits::eCubeCompatible) && + info.base_layer % 6 == 0 && info.layer_count % 6 == 0; + default: return false; + } + case vk::ImageType::e3D: + return info.type == vk::ImageViewType::e3D && info.base_layer == 0 && + info.layer_count == 1; + default: return false; + } +} + +[[nodiscard]] bool IsValidAspect(const VulkanImage& image, vk::ImageAspectFlags aspect) { + const auto depth_format = DepthAspectTransferFormat(image.format); + if (depth_format == vk::Format::eUndefined) { + return aspect == vk::ImageAspectFlagBits::eColor; + } + const auto supported = ImageViewOps::DepthAspectMask(image.format); + return static_cast(aspect) && !(aspect & ~supported); } } // namespace namespace ImageViewOps { +namespace { + +enum CompatibilityClass : uint32_t { + None = 0, + Bit8 = 1u << 0, + Bit16 = 1u << 1, + Bit24 = 1u << 2, + Bit32 = 1u << 3, + Bit48 = 1u << 4, + Bit64 = 1u << 5, + Bit96 = 1u << 6, + Bit128 = 1u << 7, + Bit192 = 1u << 8, + Bit256 = 1u << 9, + Bc1Rgb = 1u << 10, + Bc1Rgba = 1u << 11, + Bc2 = 1u << 12, + Bc3 = 1u << 13, + Bc4 = 1u << 14, + Bc5 = 1u << 15, + Bc6h = 1u << 16, + Bc7 = 1u << 17, + D16 = 1u << 18, + D16S8 = 1u << 19, + D24 = 1u << 20, + D24S8 = 1u << 21, + D32 = 1u << 22, + D32S8 = 1u << 23, + S8 = 1u << 24, +}; + +[[nodiscard]] uint32_t FormatClass(vk::Format format) noexcept { + switch (format) { + case vk::Format::eR4G4UnormPack8: + case vk::Format::eR8Sint: + case vk::Format::eR8Snorm: + case vk::Format::eR8Srgb: + case vk::Format::eR8Sscaled: + case vk::Format::eR8Uint: + case vk::Format::eR8Unorm: + case vk::Format::eR8Uscaled: return Bit8; + + case vk::Format::eA1R5G5B5UnormPack16: + case vk::Format::eA4B4G4R4UnormPack16: + case vk::Format::eA4R4G4B4UnormPack16: + case vk::Format::eB4G4R4A4UnormPack16: + case vk::Format::eB5G5R5A1UnormPack16: + case vk::Format::eB5G6R5UnormPack16: + case vk::Format::eR10X6UnormPack16: + case vk::Format::eR12X4UnormPack16: + case vk::Format::eR16Sfloat: + case vk::Format::eR16Sint: + case vk::Format::eR16Snorm: + case vk::Format::eR16Sscaled: + case vk::Format::eR16Uint: + case vk::Format::eR16Unorm: + case vk::Format::eR16Uscaled: + case vk::Format::eR4G4B4A4UnormPack16: + case vk::Format::eR5G5B5A1UnormPack16: + case vk::Format::eR5G6B5UnormPack16: + case vk::Format::eR8G8Sint: + case vk::Format::eR8G8Snorm: + case vk::Format::eR8G8Srgb: + case vk::Format::eR8G8Sscaled: + case vk::Format::eR8G8Uint: + case vk::Format::eR8G8Unorm: + case vk::Format::eR8G8Uscaled: return Bit16; + + case vk::Format::eB8G8R8Sint: + case vk::Format::eB8G8R8Snorm: + case vk::Format::eB8G8R8Srgb: + case vk::Format::eB8G8R8Sscaled: + case vk::Format::eB8G8R8Uint: + case vk::Format::eB8G8R8Unorm: + case vk::Format::eB8G8R8Uscaled: + case vk::Format::eR8G8B8Sint: + case vk::Format::eR8G8B8Snorm: + case vk::Format::eR8G8B8Srgb: + case vk::Format::eR8G8B8Sscaled: + case vk::Format::eR8G8B8Uint: + case vk::Format::eR8G8B8Unorm: + case vk::Format::eR8G8B8Uscaled: return Bit24; + + case vk::Format::eA2B10G10R10SintPack32: + case vk::Format::eA2B10G10R10SnormPack32: + case vk::Format::eA2B10G10R10SscaledPack32: + case vk::Format::eA2B10G10R10UintPack32: + case vk::Format::eA2B10G10R10UnormPack32: + case vk::Format::eA2B10G10R10UscaledPack32: + case vk::Format::eA2R10G10B10SintPack32: + case vk::Format::eA2R10G10B10SnormPack32: + case vk::Format::eA2R10G10B10SscaledPack32: + case vk::Format::eA2R10G10B10UintPack32: + case vk::Format::eA2R10G10B10UnormPack32: + case vk::Format::eA2R10G10B10UscaledPack32: + case vk::Format::eA8B8G8R8SintPack32: + case vk::Format::eA8B8G8R8SnormPack32: + case vk::Format::eA8B8G8R8SrgbPack32: + case vk::Format::eA8B8G8R8SscaledPack32: + case vk::Format::eA8B8G8R8UintPack32: + case vk::Format::eA8B8G8R8UnormPack32: + case vk::Format::eA8B8G8R8UscaledPack32: + case vk::Format::eB10G11R11UfloatPack32: + case vk::Format::eB8G8R8A8Sint: + case vk::Format::eB8G8R8A8Snorm: + case vk::Format::eB8G8R8A8Srgb: + case vk::Format::eB8G8R8A8Sscaled: + case vk::Format::eB8G8R8A8Uint: + case vk::Format::eB8G8R8A8Unorm: + case vk::Format::eB8G8R8A8Uscaled: + case vk::Format::eE5B9G9R9UfloatPack32: + case vk::Format::eR10X6G10X6Unorm2Pack16: + case vk::Format::eR12X4G12X4Unorm2Pack16: + case vk::Format::eR16G16Sfloat: + case vk::Format::eR16G16Sint: + case vk::Format::eR16G16Snorm: + case vk::Format::eR16G16Sscaled: + case vk::Format::eR16G16Uint: + case vk::Format::eR16G16Unorm: + case vk::Format::eR16G16Uscaled: + case vk::Format::eR32Sfloat: + case vk::Format::eR32Sint: + case vk::Format::eR32Uint: + case vk::Format::eR8G8B8A8Sint: + case vk::Format::eR8G8B8A8Snorm: + case vk::Format::eR8G8B8A8Srgb: + case vk::Format::eR8G8B8A8Sscaled: + case vk::Format::eR8G8B8A8Uint: + case vk::Format::eR8G8B8A8Unorm: + case vk::Format::eR8G8B8A8Uscaled: return Bit32; + + case vk::Format::eR16G16B16Sfloat: + case vk::Format::eR16G16B16Sint: + case vk::Format::eR16G16B16Snorm: + case vk::Format::eR16G16B16Sscaled: + case vk::Format::eR16G16B16Uint: + case vk::Format::eR16G16B16Unorm: + case vk::Format::eR16G16B16Uscaled: return Bit48; + + case vk::Format::eR16G16B16A16Sfloat: + case vk::Format::eR16G16B16A16Sint: + case vk::Format::eR16G16B16A16Snorm: + case vk::Format::eR16G16B16A16Sscaled: + case vk::Format::eR16G16B16A16Uint: + case vk::Format::eR16G16B16A16Unorm: + case vk::Format::eR16G16B16A16Uscaled: + case vk::Format::eR32G32Sfloat: + case vk::Format::eR32G32Sint: + case vk::Format::eR32G32Uint: + case vk::Format::eR64Sfloat: + case vk::Format::eR64Sint: + case vk::Format::eR64Uint: return Bit64; + + case vk::Format::eR32G32B32Sfloat: + case vk::Format::eR32G32B32Sint: + case vk::Format::eR32G32B32Uint: return Bit96; + + case vk::Format::eR32G32B32A32Sfloat: + case vk::Format::eR32G32B32A32Sint: + case vk::Format::eR32G32B32A32Uint: + case vk::Format::eR64G64Sfloat: + case vk::Format::eR64G64Sint: + case vk::Format::eR64G64Uint: return Bit128; + + case vk::Format::eR64G64B64Sfloat: + case vk::Format::eR64G64B64Sint: + case vk::Format::eR64G64B64Uint: return Bit192; + + case vk::Format::eR64G64B64A64Sfloat: + case vk::Format::eR64G64B64A64Sint: + case vk::Format::eR64G64B64A64Uint: return Bit256; + + case vk::Format::eBc1RgbSrgbBlock: + case vk::Format::eBc1RgbUnormBlock: return Bc1Rgb | Bit64; + case vk::Format::eBc1RgbaSrgbBlock: + case vk::Format::eBc1RgbaUnormBlock: return Bc1Rgba | Bit64; + case vk::Format::eBc2SrgbBlock: + case vk::Format::eBc2UnormBlock: return Bc2 | Bit128; + case vk::Format::eBc3SrgbBlock: + case vk::Format::eBc3UnormBlock: return Bc3 | Bit128; + case vk::Format::eBc4SnormBlock: + case vk::Format::eBc4UnormBlock: return Bc4 | Bit64; + case vk::Format::eBc5SnormBlock: + case vk::Format::eBc5UnormBlock: return Bc5 | Bit128; + case vk::Format::eBc6HSfloatBlock: + case vk::Format::eBc6HUfloatBlock: return Bc6h | Bit128; + case vk::Format::eBc7SrgbBlock: + case vk::Format::eBc7UnormBlock: return Bc7 | Bit128; + + case vk::Format::eD16Unorm: return D16; + case vk::Format::eD16UnormS8Uint: return D16S8; + case vk::Format::eX8D24UnormPack32: return D24; + case vk::Format::eD24UnormS8Uint: return D24S8; + case vk::Format::eD32Sfloat: return D32; + case vk::Format::eD32SfloatS8Uint: return D32S8; + case vk::Format::eS8Uint: return S8; + default: return None; + } +} + +} // namespace + vk::ImageAspectFlags DepthAspectMask(vk::Format format) { switch (format) { case vk::Format::eD16Unorm: @@ -79,314 +302,97 @@ vk::ImageAspectFlags DepthAspectMask(vk::Format format) { } } -bool FormatSupportsStorage(vk::Format format) { - auto& graphics = GetRenderContext().GetGraphics(); - const auto properties = graphics.GetFormatProperties(format); - return static_cast(properties.optimalTilingFeatures & - vk::FormatFeatureFlagBits::eStorageImage); -} - -void CreateRenderTargetViews(RenderTextureVulkanImage& image) { - CreateRenderTargetView(image, VulkanImage::VIEW_DEFAULT, - vk::ComponentSwizzle::eIdentity, vk::ComponentSwizzle::eIdentity, - vk::ComponentSwizzle::eIdentity, vk::ComponentSwizzle::eIdentity); - if (image.layers > 1) { - CreateRenderTargetView(image, VulkanImage::VIEW_DEFAULT_ARRAY, - vk::ComponentSwizzle::eIdentity, vk::ComponentSwizzle::eIdentity, - vk::ComponentSwizzle::eIdentity, vk::ComponentSwizzle::eIdentity, - vk::ImageViewType::e2DArray); - } - if (image.samples == 1 && FormatSupportsStorage(image.format)) { - CreateRenderTargetView(image, VulkanImage::VIEW_STORAGE, - vk::ComponentSwizzle::eIdentity, vk::ComponentSwizzle::eIdentity, - vk::ComponentSwizzle::eIdentity, vk::ComponentSwizzle::eIdentity, - vk::ImageViewType::e2D, vk::Format::eUndefined, - vk::ImageUsageFlags {}, 1); - if (image.layers > 1) { - CreateRenderTargetView(image, VulkanImage::VIEW_STORAGE_ARRAY, - vk::ComponentSwizzle::eIdentity, vk::ComponentSwizzle::eIdentity, - vk::ComponentSwizzle::eIdentity, vk::ComponentSwizzle::eIdentity, - vk::ImageViewType::e2DArray, vk::Format::eUndefined, - vk::ImageUsageFlags {}, 1); - } - } -} - -void CreateDepthViews(DepthStencilVulkanImage& image) { - CreateView(image, VulkanImage::VIEW_DEFAULT, vk::ImageViewType::e2D, - DepthAspectMask(image.format), - {vk::ComponentSwizzle::eIdentity, vk::ComponentSwizzle::eIdentity, - vk::ComponentSwizzle::eIdentity, vk::ComponentSwizzle::eIdentity}, - 0, 0, 1, 1); -} - -void CreateVideoOutViews(VideoOutVulkanImage& image) { - CreateRenderTargetView(image, VulkanImage::VIEW_DEFAULT, - vk::ComponentSwizzle::eIdentity, vk::ComponentSwizzle::eIdentity, - vk::ComponentSwizzle::eIdentity, vk::ComponentSwizzle::eIdentity); - if ((image.format == vk::Format::eR8G8B8A8Srgb || image.format == vk::Format::eB8G8R8A8Srgb) && - FormatSupportsStorage(vk::Format::eR8G8B8A8Uint)) { - CreateRenderTargetView(image, VulkanImage::VIEW_STORAGE, - vk::ComponentSwizzle::eIdentity, vk::ComponentSwizzle::eIdentity, - vk::ComponentSwizzle::eIdentity, vk::ComponentSwizzle::eIdentity, - vk::ImageViewType::e2D, vk::Format::eR8G8B8A8Uint, - vk::ImageUsageFlagBits::eStorage, 1); - } -} - -void DestroyViews(VulkanImage& image) { - auto& graphics = GetRenderContext().GetGraphics(); - for (auto& cached: image.view_cache.views) { - if (cached.view != nullptr) { - graphics.device.destroyImageView(cached.view, nullptr); - cached.view = nullptr; - } - } - image.view_cache.views.clear(); - for (auto& view: image.image_view) { - if (view != nullptr) { - graphics.device.destroyImageView(view, nullptr); - view = nullptr; - } +bool FormatsCompatible(vk::Format base, vk::Format view) noexcept { + if (base == view) { + return true; } + const auto base_class = FormatClass(base); + const auto view_class = FormatClass(view); + return view_class != None && (base_class & view_class) == view_class; } } // namespace ImageViewOps -vk::ImageView TextureCache::GetRenderTargetAttachmentView(RenderTextureVulkanImage& image, - vk::Format format, uint32_t level, - uint32_t base_layer, - uint32_t layer_count) { - if (image.format == vk::Format::eUndefined || level >= image.mip_levels || level >= 16 || - layer_count == 0 || base_layer >= image.layers || layer_count > image.layers - base_layer) { - EXIT("TextureCache: invalid render-target attachment view, image=%p format=%d" - " level=%u image_levels=%u base_layer=%u layer_count=%u image_layers=%u\n", - static_cast(&image), static_cast(format), level, image.mip_levels, - base_layer, layer_count, image.layers); +vk::ImageView Image::FindView(const ImageViewInfo& view_info) { + const auto& image = backing; + auto normalized = view_info; + const bool is_storage = + static_cast(normalized.usage & vk::ImageUsageFlagBits::eStorage); + normalized.aspect = FullAspectMask(image.format); + if (normalized.aspect & vk::ImageAspectFlagBits::eDepth && + IsDepthViewFormat(normalized.format)) { + normalized.format = image.format; + normalized.aspect = vk::ImageAspectFlagBits::eDepth; } - if (format != image.format && !IsRgba8SrgbReinterpretation(image.format, format)) { - EXIT("TextureCache: incompatible render-target attachment view, image_format=%d" - " view_format=%d level=%u\n", - static_cast(image.format), static_cast(format), level); + if (normalized.aspect & vk::ImageAspectFlagBits::eStencil && + IsStencilViewFormat(normalized.format)) { + normalized.format = image.format; + normalized.aspect = vk::ImageAspectFlagBits::eStencil; + } + normalized.usage = + is_storage ? vk::ImageUsageFlagBits::eStorage : vk::ImageUsageFlags {}; + const bool format_compatible = normalized.format != vk::Format::eUndefined && + IsCompatibleViewFormat(image.format, normalized.format); + const bool ranges_valid = normalized.level_count != 0 && + normalized.base_level < image.mip_levels && + normalized.level_count <= image.mip_levels - normalized.base_level && + normalized.layer_count != 0 && normalized.base_layer < image.layers && + normalized.layer_count <= image.layers - normalized.base_layer; + const bool mapping_valid = + IsComponentSwizzle(normalized.mapping.r) && IsComponentSwizzle(normalized.mapping.g) && + IsComponentSwizzle(normalized.mapping.b) && IsComponentSwizzle(normalized.mapping.a); + if (image.image == nullptr || !format_compatible || !ranges_valid || !mapping_valid || + !IsValidViewType(image, normalized) || + !IsValidAspect(image, normalized.aspect)) { + EXIT("invalid image view: image_format=%d view_format=%d type=%d aspect=0x%x " + "mip=%u+%u layer=%u+%u usage=0x%x image_levels=%u image_layers=%u\n", + static_cast(image.format), static_cast(normalized.format), + static_cast(normalized.type), + static_cast(normalized.aspect), normalized.base_level, + normalized.level_count, normalized.base_layer, normalized.layer_count, + static_cast(normalized.usage), image.mip_levels, + image.layers); } - return GetImageView( - image, {format, layer_count == 1 ? vk::ImageViewType::e2D : vk::ImageViewType::e2DArray, - vk::ImageAspectFlagBits::eColor, level, 1, base_layer, layer_count, - DstSel(4, 5, 6, 7), vk::ImageUsageFlagBits::eColorAttachment}); -} - -vk::ImageView TextureCache::GetDepthTargetAttachmentView(DepthStencilVulkanImage& image, - uint32_t base_layer, - uint32_t layer_count) { - if (layer_count == 0 || base_layer >= image.layers || layer_count > image.layers - base_layer) { - EXIT("TextureCache: invalid depth-target attachment view, image=%p base_layer=%u " - "layer_count=%u image_layers=%u\n", - static_cast(&image), base_layer, layer_count, image.layers); - } - return GetImageView(image, - {image.format, - layer_count == 1 ? vk::ImageViewType::e2D : vk::ImageViewType::e2DArray, - ImageViewOps::DepthAspectMask(image.format), 0, 1, base_layer, layer_count, - DstSel(4, 5, 6, 7), vk::ImageUsageFlagBits::eDepthStencilAttachment}); -} - -vk::ImageView TextureCache::GetImageView(VulkanImage& image, const ImageViewInfo& info) { - const bool supported_type = info.type == vk::ImageViewType::e2D || - info.type == vk::ImageViewType::e2DArray || - info.type == vk::ImageViewType::e3D; - const bool supported_usage = info.usage == vk::ImageUsageFlagBits::eSampled || - info.usage == vk::ImageUsageFlagBits::eStorage || - info.usage == vk::ImageUsageFlagBits::eColorAttachment || - info.usage == vk::ImageUsageFlagBits::eDepthStencilAttachment; - const bool valid_shape = - (info.type == vk::ImageViewType::e2D && info.layer_count == 1) || - info.type == vk::ImageViewType::e2DArray || - (info.type == vk::ImageViewType::e3D && info.base_layer == 0 && info.layer_count == 1); - if (info.format == vk::Format::eUndefined || !info.aspect || info.level_count == 0 || - info.base_level >= (image.mip_levels) || - info.level_count > image.mip_levels - info.base_level || info.layer_count == 0 || - info.base_layer >= image.layers || info.layer_count > image.layers - info.base_layer || - !supported_type || !valid_shape || !supported_usage) { - EXIT("TextureCache: invalid dynamic image view, image=%p format=%d aspect=0x%x" - " swizzle=0x%03x mip=%u+%u layer=%u+%u type=%d usage=0x%x" - " image_levels=%u image_layers=%u\n", - static_cast(&image), static_cast(info.format), - static_cast(info.aspect), info.swizzle, - info.base_level, info.level_count, info.base_layer, info.layer_count, - static_cast(info.type), static_cast(info.usage), - image.mip_levels, image.layers); - } - - auto& cache = image.view_cache; - std::lock_guard lock(cache.mutex); - for (const auto& cached: cache.views) { - if (cached.info == info) { + std::lock_guard lock(views.mutex); + for (const auto& cached: views.views) { + if (cached.info == normalized) { return cached.view; } } vk::ImageViewUsageCreateInfo usage {}; usage.sType = vk::StructureType::eImageViewUsageCreateInfo; - usage.usage = info.usage; + usage.usage = image.usage; + if (!is_storage) { + usage.usage &= ~vk::ImageUsageFlagBits::eStorage; + } vk::ImageViewCreateInfo create {}; create.sType = vk::StructureType::eImageViewCreateInfo; create.pNext = &usage; create.image = image.image; - create.viewType = info.type; - create.format = info.format; - create.components = info.usage == vk::ImageUsageFlagBits::eSampled - ? TextureGetComponentMapping(info.swizzle) - : vk::ComponentMapping {}; - create.subresourceRange.aspectMask = info.aspect; - create.subresourceRange.baseMipLevel = info.base_level; - create.subresourceRange.levelCount = info.level_count; - create.subresourceRange.baseArrayLayer = info.base_layer; - create.subresourceRange.layerCount = info.layer_count; - vk::ImageView view = nullptr; - const auto result = m_graphics.device.createImageView(&create, nullptr, &view); + create.viewType = normalized.type; + create.format = normalized.format; + create.components = normalized.mapping; + create.subresourceRange.aspectMask = normalized.aspect; + create.subresourceRange.baseMipLevel = normalized.base_level; + create.subresourceRange.levelCount = normalized.level_count; + create.subresourceRange.baseArrayLayer = normalized.base_layer; + create.subresourceRange.layerCount = normalized.layer_count; + + vk::ImageView view = nullptr; + const auto result = m_graphics->device.createImageView(&create, nullptr, &view); if (result != vk::Result::eSuccess || view == nullptr) { - EXIT("TextureCache: failed to create dynamic image view, result=%d format=%d" - " aspect=0x%x swizzle=0x%03x mip=%u+%u layer=%u+%u type=%d usage=0x%x\n", - static_cast(result), static_cast(info.format), - static_cast(info.aspect), info.swizzle, - info.base_level, info.level_count, info.base_layer, info.layer_count, - static_cast(info.type), static_cast(info.usage)); + EXIT("failed to create image view: result=%d image_format=%d view_format=%d type=%d " + "aspect=0x%x mip=%u+%u layer=%u+%u usage=0x%x\n", + static_cast(result), static_cast(image.format), + static_cast(view_info.format), static_cast(view_info.type), + static_cast(view_info.aspect), view_info.base_level, + view_info.level_count, view_info.base_layer, view_info.layer_count, + static_cast(view_info.usage)); } - cache.views.push_back({info, view}); + views.views.push_back({normalized, view}); return view; } -vk::ImageView TextureCache::GetDepthTargetSampledView(DepthStencilVulkanImage& image, - vk::Format view_format, uint32_t swizzle, - uint32_t base_level, uint32_t level_count, - vk::ImageViewType type, uint32_t base_layer, - uint32_t layer_count) { - if (view_format == vk::Format::eUndefined || - !IsSupportedSampledDepthView(image.format, view_format, swizzle)) { - EXIT("TextureCache: invalid sampled depth-target view, image=%p image_format=%d" - " view_format=%d swizzle=0x%03x mip=%u+%u layer=%u+%u type=%d" - " image_levels=%u image_layers=%u\n", - static_cast(&image), static_cast(image.format), - static_cast(view_format), swizzle, base_level, level_count, base_layer, - layer_count, static_cast(type), image.mip_levels, image.layers); - } - return GetImageView(image, {image.format, type, vk::ImageAspectFlagBits::eDepth, base_level, - level_count, base_layer, layer_count, swizzle}); -} - -vk::ImageView TextureCache::GetSampledColorView(VulkanImage& image, vk::Format view_format, - uint32_t swizzle, uint32_t base_level, - uint32_t level_count, vk::ImageViewType type, - uint32_t base_layer, uint32_t layer_count) { - if (view_format == vk::Format::eUndefined || base_level >= 16 || - (type != vk::ImageViewType::e2D && type != vk::ImageViewType::e2DArray) || - !IsSupportedSampledColorView(image.format, view_format, swizzle)) { - EXIT("TextureCache: invalid sampled color view, image=%p swizzle=0x%03x" - " view_format=%d mip=%u+%u layer=%u+%u type=%d image_levels=%u image_layers=%u\n", - static_cast(&image), swizzle, static_cast(view_format), base_level, - level_count, base_layer, layer_count, static_cast(type), image.mip_levels, - image.layers); - } - const auto precreated_view = type == vk::ImageViewType::e2DArray - ? VulkanImage::VIEW_DEFAULT_ARRAY - : VulkanImage::VIEW_DEFAULT; - const bool full_view = base_level == 0 && level_count == image.mip_levels && base_layer == 0 && - layer_count == (type == vk::ImageViewType::e2DArray ? image.layers : 1u); - if (view_format == image.format && swizzle == DstSel(4, 5, 6, 7) && full_view && - image.image_view[precreated_view] != nullptr) { - return image.image_view[precreated_view]; - } - return GetImageView(image, {view_format, type, vk::ImageAspectFlagBits::eColor, base_level, - level_count, base_layer, layer_count, swizzle}); -} - -vk::ImageView TextureCache::GetRenderTargetStorageView(RenderTextureVulkanImage& image, - vk::Format view_format, uint32_t base_level, - uint32_t level_count, vk::ImageViewType type, - uint32_t base_layer, uint32_t layer_count) { - if (view_format == vk::Format::eUndefined || - (type != vk::ImageViewType::e2D && type != vk::ImageViewType::e2DArray)) { - EXIT("TextureCache: invalid render-target storage view, image=%p view_format=%d" - " mip=%u+%u layer=%u+%u type=%d image_levels=%u image_layers=%u\n", - static_cast(&image), static_cast(view_format), base_level, - level_count, base_layer, layer_count, static_cast(type), image.mip_levels, - image.layers); - } - const bool exact = view_format == image.format; - const bool compatible = view_format == BgraSrgbStorageViewFormat(image.format); - if (!exact && !compatible) { - EXIT("TextureCache: incompatible render-target storage view, image_format=%d" - " view_format=%d base=%u count=%u\n", - static_cast(image.format), static_cast(view_format), base_level, - level_count); - } - if (exact) { - const auto index = type == vk::ImageViewType::e2DArray ? VulkanImage::VIEW_STORAGE_ARRAY - : VulkanImage::VIEW_STORAGE; - const bool full_view = - base_level == 0 && level_count == 1 && base_layer == 0 && - layer_count == (type == vk::ImageViewType::e2DArray ? image.layers : 1u); - if (full_view && image.image_view[index] != nullptr) { - return image.image_view[index]; - } - } - - if (compatible && !ImageViewOps::FormatSupportsStorage(view_format)) { - EXIT("TextureCache: compatible render-target storage format lacks storage support," - " image_format=%d view_format=%d base=%u count=%u\n", - static_cast(image.format), static_cast(view_format), base_level, - level_count); - } - return GetImageView(image, {view_format, type, vk::ImageAspectFlagBits::eColor, base_level, - level_count, base_layer, layer_count, DstSel(4, 5, 6, 7), - vk::ImageUsageFlagBits::eStorage}); -} - -vk::ImageView TextureCache::GetStorageTextureSampledView(StorageTextureVulkanImage& image, - const ImageInfo& info) { - const auto shape = SelectStorageSampledViewShape(info.type, info.depth, image.layers); - if (image.image == nullptr || shape == StorageSampledViewShape::Unsupported || - info.base_array != 0 || info.levels != image.mip_levels || info.base_level >= info.levels || - info.view_levels == 0 || info.base_level + info.view_levels > info.levels) { - EXIT("TextureCache: invalid sampled view of storage texture, image=%p type=%u depth=%u" - " base=%u levels=%u view_levels=%u image_levels=%u base_array=%u\n", - static_cast(&image), info.type, info.depth, info.base_level, info.levels, - info.view_levels, image.mip_levels, info.base_array); - } - const auto view_format = TextureGetFormat(info.format); - if (view_format != image.format && !IsRgba8SrgbReinterpretation(image.format, view_format) && - !IsR32UintFloatReinterpretation(image.format, view_format)) { - EXIT("TextureCache: incompatible sampled view of storage texture, image_format=%d" - " view_format=%d swizzle=0x%03x\n", - static_cast(image.format), static_cast(view_format), info.swizzle); - } - - vk::ImageViewType type = static_cast(VK_IMAGE_VIEW_TYPE_MAX_ENUM); - switch (shape) { - case StorageSampledViewShape::Image2D: type = vk::ImageViewType::e2D; break; - case StorageSampledViewShape::Image2DArray: type = vk::ImageViewType::e2DArray; break; - case StorageSampledViewShape::Image3D: type = vk::ImageViewType::e3D; break; - case StorageSampledViewShape::Unsupported: - EXIT("TextureCache: unsupported sampled storage-image view shape\n"); - } - const auto layer_count = shape == StorageSampledViewShape::Image2DArray ? info.depth : 1u; - return GetImageView(image, {view_format, type, vk::ImageAspectFlagBits::eColor, info.base_level, - info.view_levels, 0, layer_count, info.swizzle}); -} - -vk::ImageView TextureCache::GetStorageTextureStorageView(StorageTextureVulkanImage& image, - uint32_t base_level) { - if (image.image == nullptr || base_level >= (image.mip_levels)) { - EXIT("TextureCache: invalid storage-texture mip view, image=%p level=%u levels=%u\n", - static_cast(&image), base_level, image.mip_levels); - } - if (base_level == 0) { - return image.image_view[VulkanImage::VIEW_DEFAULT]; - } - return GetImageView(image, {image.format, vk::ImageViewType::e2D, - vk::ImageAspectFlagBits::eColor, base_level, 1, 0, 1, - DstSel(4, 5, 6, 7), vk::ImageUsageFlagBits::eStorage}); -} - } // namespace Libs::Graphics diff --git a/src/graphics/host_gpu/renderer/imageView.h b/src/graphics/host_gpu/renderer/imageView.h index 41a90ca1..4bbf728e 100644 --- a/src/graphics/host_gpu/renderer/imageView.h +++ b/src/graphics/host_gpu/renderer/imageView.h @@ -2,42 +2,28 @@ #define EMULATOR_SRC_GRAPHICS_HOST_GPU_RENDERER_IMAGEVIEW_H_ #include "common/assert.h" -#include "graphics/host_gpu/graphicContext.h" #include "graphics/host_gpu/renderer/imageInfo.h" #include "graphics/shader/recompiler/ShaderIR.h" #include "graphics/shader/shader.h" namespace Libs::Graphics { -[[nodiscard]] inline bool IsSupportedStorageSwizzle(uint32_t format, uint32_t swizzle) noexcept { - const bool single_channel = - format == Prospero::GpuEnumValue(Prospero::BufferFormat::k8UNorm) || - format == Prospero::GpuEnumValue(Prospero::BufferFormat::k8UInt) || - format == Prospero::GpuEnumValue(Prospero::BufferFormat::k16UInt) || - format == Prospero::GpuEnumValue(Prospero::BufferFormat::k32UInt) || - format == Prospero::GpuEnumValue(Prospero::BufferFormat::k16Float) || - format == Prospero::GpuEnumValue(Prospero::BufferFormat::k32Float); - return swizzle == DstSel(4, 5, 6, 7) || - (single_channel && (swizzle == DstSel(4, 0, 0, 0) || swizzle == DstSel(4, 0, 0, 1) || - swizzle == DstSel(4, 4, 4, 4))) || - (format == Prospero::GpuEnumValue(Prospero::BufferFormat::k32_32UInt) && - swizzle == DstSel(4, 5, 0, 1)) || - ((format == Prospero::GpuEnumValue(Prospero::BufferFormat::k8_8_8_8UNorm) || - format == Prospero::GpuEnumValue(Prospero::BufferFormat::k8_8_8_8UInt)) && - (swizzle == DstSel(4, 5, 6, 1) || swizzle == DstSel(6, 5, 4, 7))) || - (format == Prospero::GpuEnumValue(Prospero::BufferFormat::k32_32_32_32Float) && - swizzle == DstSel(5, 6, 7, 4)); -} - -[[nodiscard]] inline bool IsSupportedStorageDepthTile(uint32_t format, uint32_t type, - uint32_t width, uint32_t height, - uint32_t depth) noexcept { - return (format == Prospero::GpuEnumValue(Prospero::BufferFormat::k8UInt) && - type == Prospero::GpuEnumValue(Prospero::ImageType::kColor2DArray) && width != 0 && - height != 0 && depth == 1) || - (format == Prospero::GpuEnumValue(Prospero::BufferFormat::k32UInt) && - type == Prospero::GpuEnumValue(Prospero::ImageType::kColor2D) && width != 0 && - height != 0 && depth == 1); +[[nodiscard]] inline bool IsValidImageSwizzle(uint32_t swizzle) noexcept { + if ((swizzle & ~0xfffu) != 0) { + return false; + } + for (uint32_t channel = 0; channel < 4; channel++) { + switch (GetDstSel(swizzle, channel)) { + case 0: + case 1: + case 4: + case 5: + case 6: + case 7: break; + default: return false; + } + } + return true; } [[noreturn]] inline void UnsupportedColorView(const char* usage, vk::Format image_format, @@ -60,11 +46,8 @@ namespace Libs::Graphics { switch (image_format) { case vk::Format::eB8G8R8A8Unorm: case vk::Format::eB8G8R8A8Srgb: - switch (view_format) { - case vk::Format::eR8G8B8A8Unorm: - case vk::Format::eR8G8B8A8Srgb: return true; - default: return false; - } + return view_format == vk::Format::eR8G8B8A8Unorm || + view_format == vk::Format::eR8G8B8A8Srgb; case vk::Format::eA2R10G10B10UnormPack32: return view_format == vk::Format::eA2B10G10R10UnormPack32; default: return false; @@ -86,28 +69,10 @@ namespace Libs::Graphics { return view_format == BgraSrgbStorageViewFormat(image_format) && swizzle == DstSel(6, 5, 4, 7); } -[[nodiscard]] inline bool IsValidSampledColorSwizzle(uint32_t swizzle) noexcept { - if ((swizzle & ~0xfffu) != 0) { - return false; - } - for (uint32_t channel = 0; channel < 4; channel++) { - switch (GetDstSel(swizzle, channel)) { - case 0: - case 1: - case 4: - case 5: - case 6: - case 7: break; - default: return false; - } - } - return true; -} - [[nodiscard]] inline bool IsSupportedSampledColorView(vk::Format image_format, vk::Format view_format, uint32_t swizzle) noexcept { - if (!IsValidSampledColorSwizzle(swizzle)) { + if (!IsValidImageSwizzle(swizzle)) { return false; } if (image_format == view_format || IsRgba8SrgbReinterpretation(image_format, view_format)) { @@ -169,26 +134,13 @@ IsSupportedSampledDepthUintResource(const ShaderRecompiler::IR::ImageResource& r !resource.written && !resource.atomic && !resource.depth_compare; } -[[nodiscard]] inline int SelectStorageColorView(vk::Format image_format, vk::Format view_format, - uint32_t swizzle) noexcept { - const bool single_channel = - view_format == vk::Format::eR8Unorm || view_format == vk::Format::eR8Uint || - view_format == vk::Format::eR16Uint || view_format == vk::Format::eR32Uint || - view_format == vk::Format::eR16Sfloat || view_format == vk::Format::eR32Sfloat; - const bool swizzle_ok = - swizzle == DstSel(4, 5, 6, 7) || - (single_channel && (swizzle == DstSel(4, 0, 0, 0) || swizzle == DstSel(4, 0, 0, 1) || - swizzle == DstSel(4, 4, 4, 4))) || - (view_format == vk::Format::eR32G32Uint && swizzle == DstSel(4, 5, 0, 1)) || - ((view_format == vk::Format::eR8G8B8A8Unorm || view_format == vk::Format::eR8G8B8A8Uint) && - (swizzle == DstSel(4, 5, 6, 1) || swizzle == DstSel(6, 5, 4, 7))) || - (view_format == vk::Format::eR32G32B32A32Sfloat && swizzle == DstSel(5, 6, 7, 4)); +inline void ValidateStorageColorView(vk::Format image_format, vk::Format view_format, + uint32_t swizzle) noexcept { if ((image_format != view_format && !IsBgraSrgbStorageView(image_format, view_format, swizzle)) || - !swizzle_ok) { + !IsValidImageSwizzle(swizzle)) { UnsupportedColorView("storage", image_format, view_format, swizzle); } - return VulkanImage::VIEW_STORAGE; } [[nodiscard]] inline bool @@ -216,13 +168,7 @@ ValidateStorageImageResource(const ShaderRecompiler::IR::ImageResource& resource namespace ImageViewOps { [[nodiscard]] vk::ImageAspectFlags DepthAspectMask(vk::Format format); -[[nodiscard]] bool FormatSupportsStorage(vk::Format format); - -void CreateRenderTargetViews(RenderTextureVulkanImage& image); -void CreateDepthViews(DepthStencilVulkanImage& image); -void CreateVideoOutViews(VideoOutVulkanImage& image); -void DestroyViews(VulkanImage& image); - +[[nodiscard]] bool FormatsCompatible(vk::Format base, vk::Format view) noexcept; } // namespace ImageViewOps } // namespace Libs::Graphics diff --git a/src/graphics/host_gpu/renderer/masterSemaphore.cpp b/src/graphics/host_gpu/renderer/masterSemaphore.cpp new file mode 100644 index 00000000..7d125336 --- /dev/null +++ b/src/graphics/host_gpu/renderer/masterSemaphore.cpp @@ -0,0 +1,60 @@ +#include "graphics/host_gpu/renderer/masterSemaphore.h" + +#include "common/assert.h" +#include "graphics/host_gpu/graphicContext.h" + +namespace Libs::Graphics { + +MasterSemaphore::MasterSemaphore(GraphicContext& graphics): m_graphics(graphics) { + vk::SemaphoreTypeCreateInfo type_info {}; + type_info.sType = vk::StructureType::eSemaphoreTypeCreateInfo; + type_info.semaphoreType = vk::SemaphoreType::eTimeline; + type_info.initialValue = 0; + + vk::SemaphoreCreateInfo create_info {}; + create_info.sType = vk::StructureType::eSemaphoreCreateInfo; + create_info.pNext = &type_info; + + const auto result = m_graphics.device.createSemaphore(&create_info, nullptr, &m_semaphore); + EXIT_NOT_IMPLEMENTED(result != vk::Result::eSuccess || m_semaphore == nullptr); +} + +MasterSemaphore::~MasterSemaphore() { + if (m_semaphore != nullptr) { + m_graphics.device.destroySemaphore(m_semaphore, nullptr); + } +} + +void MasterSemaphore::Refresh() { + uint64_t counter = 0; + const auto result = m_graphics.device.getSemaphoreCounterValue(m_semaphore, &counter); + EXIT_NOT_IMPLEMENTED(result != vk::Result::eSuccess); + + auto known = m_gpu_tick.load(std::memory_order_acquire); + while (known < counter && + !m_gpu_tick.compare_exchange_weak(known, counter, std::memory_order_release, + std::memory_order_relaxed)) { + } +} + +void MasterSemaphore::Wait(uint64_t tick) { + if (IsFree(tick)) { + return; + } + Refresh(); + if (IsFree(tick)) { + return; + } + + vk::SemaphoreWaitInfo wait_info {}; + wait_info.sType = vk::StructureType::eSemaphoreWaitInfo; + wait_info.semaphoreCount = 1; + wait_info.pSemaphores = &m_semaphore; + wait_info.pValues = &tick; + + const auto result = m_graphics.device.waitSemaphores(&wait_info, UINT64_MAX); + EXIT_NOT_IMPLEMENTED(result != vk::Result::eSuccess); + Refresh(); +} + +} // namespace Libs::Graphics diff --git a/src/graphics/host_gpu/renderer/masterSemaphore.h b/src/graphics/host_gpu/renderer/masterSemaphore.h new file mode 100644 index 00000000..6b96cffa --- /dev/null +++ b/src/graphics/host_gpu/renderer/masterSemaphore.h @@ -0,0 +1,43 @@ +#ifndef EMULATOR_SRC_GRAPHICS_HOST_GPU_RENDERER_MASTERSEMAPHORE_H_ +#define EMULATOR_SRC_GRAPHICS_HOST_GPU_RENDERER_MASTERSEMAPHORE_H_ + +#include "common/common.h" +#include "graphics/host_gpu/vulkanCommon.h" + +#include + +namespace Libs::Graphics { + +struct GraphicContext; + +class MasterSemaphore { +public: + explicit MasterSemaphore(GraphicContext& graphics); + ~MasterSemaphore(); + KYTY_CLASS_NO_COPY(MasterSemaphore); + + [[nodiscard]] uint64_t CurrentTick() const noexcept { + return m_current_tick.load(std::memory_order_acquire); + } + [[nodiscard]] uint64_t KnownGpuTick() const noexcept { + return m_gpu_tick.load(std::memory_order_acquire); + } + [[nodiscard]] bool IsFree(uint64_t tick) const noexcept { return KnownGpuTick() >= tick; } + [[nodiscard]] uint64_t NextTick() noexcept { + return m_current_tick.fetch_add(1, std::memory_order_release); + } + [[nodiscard]] vk::Semaphore Handle() const noexcept { return m_semaphore; } + + void Refresh(); + void Wait(uint64_t tick); + +private: + GraphicContext& m_graphics; + vk::Semaphore m_semaphore = nullptr; + std::atomic m_gpu_tick {0}; + std::atomic m_current_tick {1}; +}; + +} // namespace Libs::Graphics + +#endif // EMULATOR_SRC_GRAPHICS_HOST_GPU_RENDERER_MASTERSEMAPHORE_H_ diff --git a/src/graphics/host_gpu/renderer/pipelineCache.cpp b/src/graphics/host_gpu/renderer/pipelineCache.cpp index ae7c4bd0..9e808018 100644 --- a/src/graphics/host_gpu/renderer/pipelineCache.cpp +++ b/src/graphics/host_gpu/renderer/pipelineCache.cpp @@ -7,7 +7,7 @@ #include "graphics/host_gpu/renderer/colorRenderTarget.h" #include "graphics/host_gpu/renderer/debug.h" #include "graphics/host_gpu/renderer/depthRenderTarget.h" -#include "graphics/host_gpu/renderer/framebufferCache.h" +#include "graphics/host_gpu/renderer/imageView.h" #include "graphics/host_gpu/renderer/render.h" #include "graphics/host_gpu/renderer/renderContext.h" @@ -36,13 +36,25 @@ void NormalizeStaticParamsForDynamicState(PipelineStaticParameters& static_param } // namespace +PipelineCache::~PipelineCache() { + auto destroy = [this](const auto& pipelines) { + for (const auto& [key, pipeline]: pipelines) { + (void)key; + m_graphics.device.destroyPipeline(pipeline->pipeline, nullptr); + m_graphics.device.destroyPipelineLayout(pipeline->pipeline_layout, nullptr); + } + }; + destroy(m_graphics_pipelines); + destroy(m_compute_pipelines); +} + bool PipelineStaticParameters::operator==(const PipelineStaticParameters& other) const noexcept { return std::memcmp(this, &other, sizeof(*this)) == 0; } PipelineCache::GraphicsPipeline& PipelineCache::CreateGraphicsPipeline( - VulkanFramebuffer& framebuffer, RenderColorInfo* colors, uint32_t color_count, - RenderDepthInfo& depth, ShaderVertexInputInfo& vs_input_info, RenderCommandBuffer& command, + RenderColorInfo* colors, uint32_t color_count, RenderDepthInfo& depth, + ShaderVertexInputInfo& vs_input_info, RenderCommandBuffer& command, ShaderPixelInputInfo* ps_input_info, vk::PrimitiveTopology topology, bool ps_active, std::span vs_spirv, std::span ps_spirv) { KYTY_PROFILER_BLOCK("PipelineCache::CreatePipeline(Gfx)", profiler::colors::DeepOrangeA200); @@ -61,10 +73,10 @@ PipelineCache::GraphicsPipeline& PipelineCache::CreateGraphicsPipeline( const HW::BlendColor& bclr = ctx.GetBlendColor(); uint32_t color_mask[RENDER_COLOR_ATTACHMENTS_MAX] = {}; for (uint32_t i = 0; i < color_count; i++) { - color_mask[i] = (colors[i].vulkan_buffer != nullptr - ? colors[i].export_mapping.ApplyMask(render_target_mask_slot( - ctx.GetRenderTargetMask(), colors[i].target_slot)) - : 0); + color_mask[i] = + (colors[i].image_id ? colors[i].export_mapping.ApplyMask(render_target_mask_slot( + ctx.GetRenderTargetMask(), colors[i].target_slot)) + : 0); } const HW::ModeControl& mc = ctx.GetModeControl(); @@ -76,11 +88,40 @@ PipelineCache::GraphicsPipeline& PipelineCache::CreateGraphicsPipeline( PipelineStaticParameters static_params {}; GraphicsPipeline p {}; - p.render_pass_id = framebuffer.render_pass_id; p.ps_shader_id = ps_id; p.vs_shader_id = vs_id; static_params.color_count = color_count; + PipelineRenderingState rendering {}; + rendering.color_count = color_count; + uint32_t attachment_samples = 0; + for (uint32_t i = 0; i < color_count; i++) { + EXIT_IF(!colors[i].image_id || colors[i].format == vk::Format::eUndefined); + rendering.color_formats[i] = colors[i].format; + if (attachment_samples == 0) { + attachment_samples = colors[i].samples; + } else if (attachment_samples != colors[i].samples) { + EXIT("mixed color attachment sample counts are unsupported: %u and %u\n", + attachment_samples, colors[i].samples); + } + } + const bool with_depth = + depth.format != vk::Format::eUndefined && static_cast(depth.image_id); + if (with_depth) { + const auto aspects = ImageViewOps::DepthAspectMask(depth.format); + rendering.depth_format = + aspects & vk::ImageAspectFlagBits::eDepth ? depth.format : vk::Format::eUndefined; + rendering.stencil_format = + aspects & vk::ImageAspectFlagBits::eStencil ? depth.format : vk::Format::eUndefined; + if (attachment_samples == 0) { + attachment_samples = depth.samples; + } else if (attachment_samples != depth.samples) { + EXIT("mixed color/depth sample counts are unsupported: %u and %u\n", + attachment_samples, depth.samples); + } + } + EXIT_IF(attachment_samples == 0 || + vulkan_sample_count(attachment_samples) == vk::SampleCountFlagBits {}); if (ps_active && depth.depth_test_enable && ps_input_info->ps_execute_on_noop) { static std::atomic log_count {0}; @@ -94,14 +135,13 @@ PipelineCache::GraphicsPipeline& PipelineCache::CreateGraphicsPipeline( static_params.negative_one_to_one = !clip_control.dx_clip_space; static_params.depth_clip_enable = clip_control.IsZClipEnabled(); static_params.topology = topology; - static_params.samples = framebuffer.samples; + static_params.samples = attachment_samples; static_params.sample_shading_enable = - ps_active && framebuffer.samples > 1 && ps_input_info->ps_sample_shading; + ps_active && attachment_samples > 1 && ps_input_info->ps_sample_shading; if (static_params.sample_shading_enable && !m_graphics.sample_rate_shading_enabled) { EXIT("Pipeline: sample-rate shading is required but unsupported by the host\n"); } - static_params.with_depth = - (depth.format != vk::Format::eUndefined && depth.vulkan_buffer != nullptr); + static_params.with_depth = with_depth; static_params.depth_test_enable = depth.depth_test_enable; static_params.depth_write_enable = (depth.depth_write_enable && !depth.depth_clear_enable); static_params.depth_compare_op = depth.depth_compare_op; @@ -139,7 +179,7 @@ PipelineCache::GraphicsPipeline& PipelineCache::CreateGraphicsPipeline( NormalizeStaticParamsForDynamicState(static_params); GraphicsPipelineKey key {}; - key.render_pass_id = p.render_pass_id; + key.rendering = rendering; key.vs_shader_id = p.vs_shader_id; key.ps_shader_id = p.ps_shader_id; key.static_params = static_params; @@ -162,7 +202,8 @@ PipelineCache::GraphicsPipeline& PipelineCache::CreateGraphicsPipeline( auto cached = std::make_unique(p); LogPipelineTrace("CreatePipelineInternal begin", vs_id.hash0, vs_id.crc32, ps_id.hash0, ps_id.crc32); - CreatePipelineInternal(*cached, framebuffer.render_pass, vs_input_info, vs_spirv, ps_input_info, + CreatePipelineInternal(m_graphics, m_descriptor_cache, *cached, rendering, vs_input_info, + vs_spirv, ps_input_info, ps_spirv, static_params, vs_id.hash0, vs_id.crc32, ps_id.hash0, ps_id.crc32, ps_active); LogPipelineTrace("CreatePipelineInternal done", vs_id.hash0, vs_id.crc32, ps_id.hash0, @@ -204,7 +245,7 @@ PipelineCache::CreateComputePipeline(ShaderComputeInputInfo& input_info, } auto cached = std::make_unique(p); - CreatePipelineInternal(*cached, input_info, cs_spirv); + CreatePipelineInternal(m_graphics, m_descriptor_cache, *cached, input_info, cs_spirv); EXIT_NOT_IMPLEMENTED(cached->pipeline == nullptr); EXIT_NOT_IMPLEMENTED(cached->pipeline_layout == nullptr); diff --git a/src/graphics/host_gpu/renderer/pipelineCache.h b/src/graphics/host_gpu/renderer/pipelineCache.h index 6dc5a8e1..3c80f0e5 100644 --- a/src/graphics/host_gpu/renderer/pipelineCache.h +++ b/src/graphics/host_gpu/renderer/pipelineCache.h @@ -20,8 +20,8 @@ namespace Libs::Graphics { struct GraphicContext; struct RenderColorInfo; struct RenderDepthInfo; -struct VulkanFramebuffer; class RenderCommandBuffer; +class DescriptorCache; namespace HW { class Context; @@ -86,12 +86,22 @@ static_assert(sizeof(PipelineStaticParameters) == sizeof(uint8_t[RENDER_COLOR_ATTACHMENTS_MAX]) * 6 + sizeof(bool[RENDER_COLOR_ATTACHMENTS_MAX]) * 3 + sizeof(float) * 4); +struct PipelineRenderingState { + std::array color_formats {}; + vk::Format depth_format = vk::Format::eUndefined; + vk::Format stencil_format = vk::Format::eUndefined; + uint32_t color_count = 0; + + bool operator==(const PipelineRenderingState&) const = default; +}; + class PipelineCache { public: - explicit PipelineCache(GraphicContext& graphics): m_graphics(graphics) { + PipelineCache(GraphicContext& graphics, DescriptorCache& descriptor_cache) + : m_graphics(graphics), m_descriptor_cache(descriptor_cache) { EXIT_NOT_IMPLEMENTED(!Common::Thread::IsMainThread()); } - ~PipelineCache() { KYTY_NOT_IMPLEMENTED; } + ~PipelineCache(); KYTY_CLASS_NO_COPY(PipelineCache); struct Pipeline { @@ -100,7 +110,6 @@ public: }; struct GraphicsPipeline: Pipeline { - uint64_t render_pass_id = 0; ShaderId vs_shader_id; ShaderId ps_shader_id; }; @@ -110,8 +119,8 @@ public: }; GraphicsPipeline& CreateGraphicsPipeline( - VulkanFramebuffer& framebuffer, RenderColorInfo* colors, uint32_t color_count, - RenderDepthInfo& depth, ShaderVertexInputInfo& vs_input_info, RenderCommandBuffer& command, + RenderColorInfo* colors, uint32_t color_count, RenderDepthInfo& depth, + ShaderVertexInputInfo& vs_input_info, RenderCommandBuffer& command, ShaderPixelInputInfo* ps_input_info, vk::PrimitiveTopology topology, bool ps_active, std::span vs_spirv, std::span ps_spirv); ComputePipeline& CreateComputePipeline(ShaderComputeInputInfo& input_info, @@ -120,13 +129,13 @@ public: private: struct GraphicsPipelineKey { - uint64_t render_pass_id = 0; + PipelineRenderingState rendering; ShaderId vs_shader_id; ShaderId ps_shader_id; PipelineStaticParameters static_params; bool operator==(const GraphicsPipelineKey& other) const { - return render_pass_id == other.render_pass_id && vs_shader_id == other.vs_shader_id && + return rendering == other.rendering && vs_shader_id == other.vs_shader_id && ps_shader_id == other.ps_shader_id && static_params == other.static_params; } }; @@ -160,12 +169,21 @@ private: Mix(hash, bytes[i]); } } + + static void MixRendering(std::size_t& hash, const PipelineRenderingState& rendering) { + Mix(hash, rendering.color_count); + for (uint32_t i = 0; i < rendering.color_count; i++) { + Mix(hash, static_cast(rendering.color_formats[i])); + } + Mix(hash, static_cast(rendering.depth_format)); + Mix(hash, static_cast(rendering.stencil_format)); + } }; struct GraphicsPipelineKeyHash { std::size_t operator()(const GraphicsPipelineKey& key) const { std::size_t hash = 0; - PipelineKeyHash::Mix(hash, key.render_pass_id); + PipelineKeyHash::MixRendering(hash, key.rendering); PipelineKeyHash::MixShaderId(hash, key.vs_shader_id); PipelineKeyHash::MixShaderId(hash, key.ps_shader_id); PipelineKeyHash::MixStaticParams(hash, key.static_params); @@ -182,6 +200,7 @@ private: }; GraphicContext& m_graphics; + DescriptorCache& m_descriptor_cache; std::unordered_map, GraphicsPipelineKeyHash> m_graphics_pipelines; @@ -192,7 +211,9 @@ private: void LogPipelineTrace(const char* phase, uint32_t vs_hash0, uint32_t vs_crc32, uint32_t ps_hash0, uint32_t ps_crc32); -void CreatePipelineInternal(PipelineCache::GraphicsPipeline& pipeline, vk::RenderPass render_pass, +void CreatePipelineInternal(GraphicContext& graphics, DescriptorCache& descriptor_cache, + PipelineCache::GraphicsPipeline& pipeline, + const PipelineRenderingState& rendering, const ShaderVertexInputInfo& vs_input_info, std::span vs_shader, const ShaderPixelInputInfo* ps_input_info, @@ -200,7 +221,8 @@ void CreatePipelineInternal(PipelineCache::GraphicsPipeline& pipeline, vk::Rende const PipelineStaticParameters& static_params, uint32_t vs_hash0, uint32_t vs_crc32, uint32_t ps_hash0, uint32_t ps_crc32, bool ps_active); -void CreatePipelineInternal(PipelineCache::ComputePipeline& pipeline, +void CreatePipelineInternal(GraphicContext& graphics, DescriptorCache& descriptor_cache, + PipelineCache::ComputePipeline& pipeline, const ShaderComputeInputInfo& input_info, std::span cs_shader); diff --git a/src/graphics/host_gpu/renderer/render.h b/src/graphics/host_gpu/renderer/render.h index 35cc5694..264b71bc 100644 --- a/src/graphics/host_gpu/renderer/render.h +++ b/src/graphics/host_gpu/renderer/render.h @@ -2,10 +2,13 @@ #define EMULATOR_INCLUDE_EMULATOR_GRAPHICS_GRAPHICSRENDER_H_ #include "common/abi.h" +#include "common/assert.h" #include "common/common.h" -#include "graphics/host_gpu/renderer/streamBuffer.h" +#include "graphics/host_gpu/renderer/descriptorCache.h" +#include "graphics/host_gpu/renderer/renderTarget.h" #include "graphics/host_gpu/vulkanCommon.h" +#include #include #include @@ -15,29 +18,23 @@ namespace HW { class Context; class UserConfig; class Shader; -struct DepthRenderTarget; } // namespace HW struct GraphicContext; struct ShaderBufferResource; struct ShaderComputeInputInfo; -struct VideoOutVulkanImage; -struct DepthStencilVulkanImage; -struct TextureVulkanImage; -struct StorageTextureVulkanImage; -struct RenderTextureVulkanImage; struct CommandSlot; struct VulkanBuffer; struct VulkanDescriptorSet; -struct VulkanFramebuffer; struct RenderDepthInfo; struct RenderColorInfo; -class BufferCache; - -struct HtileClearTarget { - uint64_t address = 0; - uint64_t size = 0; -}; +struct DrawCallInfo; +struct DrawEmitInfo; +struct DrawIndexBufferSource; +struct DrawRenderState; +class RenderContext; +class CommandScheduler; +struct RenderExecutorTestAccess; enum class CommandBufferDebugOp : uint32_t { DispatchDirect, @@ -66,9 +63,35 @@ private: std::vector> m_resources; }; +struct SubmitInfo { + static constexpr uint32_t MaxSemaphores = 3; + + std::array wait_semaphores {}; + std::array wait_ticks {}; + std::array wait_stages {}; + std::array signal_semaphores {}; + std::array signal_ticks {}; + uint32_t num_wait_semaphores = 0; + uint32_t num_signal_semaphores = 0; + + void AddWait(vk::Semaphore semaphore, uint64_t tick = 1, + vk::PipelineStageFlags stage = vk::PipelineStageFlagBits::eAllCommands) { + EXIT_IF(semaphore == nullptr || num_wait_semaphores >= MaxSemaphores); + wait_semaphores[num_wait_semaphores] = semaphore; + wait_ticks[num_wait_semaphores] = tick; + wait_stages[num_wait_semaphores++] = stage; + } + + void AddSignal(vk::Semaphore semaphore, uint64_t tick = 1) { + EXIT_IF(semaphore == nullptr || num_signal_semaphores >= MaxSemaphores); + signal_semaphores[num_signal_semaphores] = semaphore; + signal_ticks[num_signal_semaphores++] = tick; + } +}; + class CommandBuffer { public: - CommandBuffer(); + explicit CommandBuffer(CommandScheduler& scheduler); ~CommandBuffer(); KYTY_CLASS_NO_COPY(CommandBuffer); @@ -77,14 +100,11 @@ public: void Begin() const; void End() const; - void Execute(); - void ExecuteWithSemaphore(vk::Semaphore wait_semaphore, vk::PipelineStageFlags wait_stage, - vk::Semaphore signal_semaphore); + void Execute(const SubmitInfo& submit = {}); void SetDebugInfo(uint32_t op, uint64_t submit_id, uint32_t arg0 = 0, uint32_t arg1 = 0, uint32_t arg2 = 0, uint32_t arg3 = 0, uint64_t arg4 = 0); - void BeginRenderPass(VulkanFramebuffer& framebuffer, RenderColorInfo* colors, - uint32_t color_count, RenderDepthInfo& depth) const; - void EndRenderPass() const; + void BeginRendering(const RenderState& state) const; + void EndRendering() const; void WaitForFenceOnly(); void WaitForFence(); void WaitForFenceAndReset(); @@ -94,19 +114,18 @@ public: [[nodiscard]] vk::CommandBuffer Handle() const; [[nodiscard]] GraphicContext& GetGraphics() const noexcept { return m_graphics; } + [[nodiscard]] RenderContext& GetContext() const noexcept { return m_context; } [[nodiscard]] bool IsExecute() const { return m_execute; } private: - friend class BufferCache; - - void Submit(vk::Semaphore wait_semaphore, vk::PipelineStageFlags wait_stage, - vk::Semaphore signal_semaphore); void Release(); void FinalizeFence(bool reset_recording); void ReleaseResourcesAfterFence(); void DeleteBuffersAfterFence(); void RecycleDescriptorsAfterFence(); + RenderContext& m_context; + CommandScheduler& m_scheduler; GraphicContext& m_graphics; CommandSlot* m_slot = nullptr; bool m_execute = false; @@ -122,12 +141,13 @@ private: std::vector> m_retired_buffers; FenceResourceRetainer m_fence_resources; std::vector m_descriptor_sets_after_fence; - HostStreamBuffer m_host_stream; + mutable RenderState m_render_state; + mutable bool m_rendering = false; }; class RenderCommandBuffer final: public CommandBuffer { public: - RenderCommandBuffer() = default; + explicit RenderCommandBuffer(CommandScheduler& scheduler): CommandBuffer(scheduler) {} void Bind(HW::Context& registers, HW::UserConfig& user_config, HW::Shader& shaders) noexcept { m_registers = ®isters; @@ -145,30 +165,79 @@ private: HW::Shader* m_shaders = nullptr; }; -void RenderDrawIndex(uint64_t submit_id, RenderCommandBuffer& buffer, uint32_t index_type_and_size, - uint32_t index_count, const void* index_addr, uint32_t flags, uint32_t type, - uint32_t instance_count = 1, uint32_t render_target_slice_offset = 0, - int32_t vertex_offset_add = 0, uint32_t first_instance = 0); -void RenderDrawIndexAuto(uint64_t submit_id, RenderCommandBuffer& buffer, uint32_t index_count, - uint32_t flags, uint32_t render_target_slice_offset = 0, - uint32_t instance_count = 1, uint32_t first_vertex = 0, - uint32_t first_instance = 0); -void RenderDispatchDirect(uint64_t submit_id, RenderCommandBuffer& buffer, uint32_t thread_group_x, - uint32_t thread_group_y, uint32_t thread_group_z, uint32_t mode); +class RenderExecutor { +public: + explicit RenderExecutor(RenderContext& context): m_context(context) {} + KYTY_CLASS_NO_COPY(RenderExecutor); -void GraphicsRenderInit(GraphicContext& graphics); -void GraphicsRenderReleaseThreadCommandPool(); + void DrawIndex(uint64_t submit_id, RenderCommandBuffer& buffer, uint32_t index_type_and_size, + uint32_t index_count, const void* index_addr, uint32_t flags, uint32_t type, + uint32_t instance_count = 1, uint32_t render_target_slice_offset = 0, + int32_t vertex_offset_add = 0, uint32_t first_instance = 0); + void DrawAuto(uint64_t submit_id, RenderCommandBuffer& buffer, uint32_t index_count, + uint32_t flags, uint32_t render_target_slice_offset = 0, + uint32_t instance_count = 1, uint32_t first_vertex = 0, + uint32_t first_instance = 0); + void DispatchDirect(uint64_t submit_id, RenderCommandBuffer& buffer, uint32_t thread_group_x, + uint32_t thread_group_y, uint32_t thread_group_z, uint32_t mode); + + [[nodiscard]] DescriptorCache::PreparedBindings + PrepareBindings(CommandBuffer& buffer, const ShaderStageRuntime& runtime, + vk::ShaderStageFlags shader_stage, DescriptorCache::Stage stage); + void RebindBuffers(CommandBuffer& buffer, DescriptorCache::PreparedBindings& bindings); + void RebindImages(CommandBuffer& buffer, DescriptorCache::PreparedBindings& bindings); + void CommitBindings(CommandBuffer& buffer, vk::PipelineBindPoint pipeline_bind_point, + vk::PipelineLayout layout, DescriptorCache::PreparedBindings& bindings); + +private: + struct GraphicsBindings { + DescriptorCache::PreparedBindings vertex; + std::optional pixel; + }; + + [[nodiscard]] DescriptorCache::TextureBinding + ResolveTexture(const ShaderRecompiler::IR::ImageResource& resource, + const ShaderRecompiler::IR::DescriptorValue& value); + [[nodiscard]] GraphicsBindings PrepareGraphicsBindings(CommandBuffer& buffer, + const ShaderStageRuntime& vertex, + const ShaderStageRuntime& pixel, + bool pixel_active); + void ResolveRenderColorTarget(uint64_t submit_id, RenderCommandBuffer& buffer, + RenderColorInfo& target, uint32_t render_target_slice_offset = 0, + uint32_t render_target_slot = UINT32_MAX, + bool ignore_target_mask = false, bool exact_format = false); + void ResolveRenderDepthTarget(uint64_t submit_id, RenderCommandBuffer& buffer, + RenderDepthInfo& target); + [[nodiscard]] bool PrepareDrawRenderState(uint64_t submit_id, RenderCommandBuffer& buffer, + const DrawCallInfo& draw, + uint32_t render_target_slice_offset, + bool log_setup_phases, DrawRenderState& state); + void ExecutePreparedDraw(uint64_t submit_id, RenderCommandBuffer& buffer, + const DrawCallInfo& draw, DrawRenderState& state, + vk::PrimitiveTopology topology, const DrawEmitInfo& emit, + const DrawIndexBufferSource& index_source, bool log_pipeline_phase, + bool set_bind_debug, bool set_auto_debug); + [[nodiscard]] RenderState AcquireRenderTargets(CommandBuffer& buffer, RenderColorInfo* colors, + uint32_t color_count, RenderDepthInfo& depth); + [[nodiscard]] bool ResolveColorTargets(uint64_t submit_id, RenderCommandBuffer& buffer, + uint32_t render_target_slice_offset); + void BindImage(ImageId id, bool storage); + void BindRenderTarget(ImageId id); + void TrackImageBinding(ImageId id); + void ResetBindings(); + [[nodiscard]] bool TryConsumeComputeMetaClear(const ShaderComputeInputInfo& input, + const RenderCommandBuffer& buffer); + + RenderContext& m_context; + std::vector> m_bound_images; + + friend struct RenderExecutorTestAccess; +}; [[nodiscard]] bool ResolveComputeImageClear(const ShaderComputeInputInfo& input, uint32_t group_x, uint32_t group_y, uint32_t group_z, uint32_t mode, ShaderBufferResource& descriptor, uint32_t& packed_clear, uint64_t& size); -[[nodiscard]] bool ResolveHtileClearTarget(const HW::DepthRenderTarget& target, - uint64_t descriptor_size, HtileClearTarget& resolved); - -void GraphicsRenderMemoryBarrier(CommandBuffer& buffer); -void GraphicsRenderTextureBarrier(CommandBuffer& buffer, uint64_t vaddr, uint64_t size); -void GraphicsRenderDepthStencilBarrier(CommandBuffer& buffer, uint64_t vaddr, uint64_t size); } // namespace Libs::Graphics diff --git a/src/graphics/host_gpu/renderer/renderCompute.cpp b/src/graphics/host_gpu/renderer/renderCompute.cpp index 0c361f58..649f8c44 100644 --- a/src/graphics/host_gpu/renderer/renderCompute.cpp +++ b/src/graphics/host_gpu/renderer/renderCompute.cpp @@ -9,20 +9,15 @@ #include "graphics/guest_gpu/gpu_defs.h" #include "graphics/guest_gpu/graphicsRun.h" #include "graphics/guest_gpu/hardwareContext.h" -#include "graphics/guest_gpu/tile.h" #include "graphics/host_gpu/graphicContext.h" -#include "graphics/host_gpu/objects/label.h" -#include "graphics/host_gpu/renderer/depthRenderTarget.h" #include "graphics/host_gpu/renderer/descriptorCache.h" #include "graphics/host_gpu/renderer/descriptors.h" -#include "graphics/host_gpu/renderer/framebufferCache.h" #include "graphics/host_gpu/renderer/imageInfo.h" #include "graphics/host_gpu/renderer/pipelineCache.h" #include "graphics/host_gpu/renderer/render.h" #include "graphics/host_gpu/renderer/renderContext.h" #include "graphics/host_gpu/renderer/shaderResourceBarrier.h" #include "graphics/host_gpu/renderer/shaderSubgroup.h" -#include "graphics/host_gpu/transfer.h" #include "graphics/host_gpu/vulkanCommon.h" #include "graphics/shader/recompiler/ResourceMaterialization.h" #include "graphics/shader/recompiler/ShaderIR.h" @@ -50,217 +45,35 @@ static uint64_t BufferDescriptorSize(const ShaderBufferResource& descriptor) { return stride == 0 ? records : records * stride; } -bool ResolveHtileClearTarget(const HW::DepthRenderTarget& z, uint64_t descriptor_size, - HtileClearTarget& resolved) { - resolved = {}; - const bool has_stencil = - z.stencil_info.format != Prospero::GpuEnumValue(Prospero::StencilFormat::kInvalid); - const auto* depth_policy = FindDepthFormatPolicy(z.z_info.format); - const bool supported_depth_state = - z.z_info.tile_surface_enable && depth_policy != nullptr && z.z_info.tile_mode_index == 0 && - z.z_info.num_samples <= 3 && z.z_info.zrange_precision <= 1 && !z.z_info.expclear_enabled && - !z.z_info.embedded_sample_locations && !z.z_info.partially_resident && - z.z_info.num_mip_levels == 0 && z.z_info.plane_compression == 0 && - z.depth_view.current_mip_level == 0 && z.depth_view.slice_start == 0 && - z.depth_view.slice_max == 0 && z.depth_info.addr5_swizzle_mask == 0 && - z.depth_info.array_mode == 0 && z.depth_info.pipe_config == 0 && - z.depth_info.bank_width == 0 && z.depth_info.bank_height == 0 && - z.depth_info.macro_tile_aspect == 0 && z.depth_info.num_banks == 0; - const bool supported_stencil_state = - z.stencil_info.tile_mode_index == 0 && z.stencil_info.tile_split == 0 && - !z.stencil_info.expclear_enabled && - depth_htile_stencil_acceleration_compatible(has_stencil, true, - z.stencil_info.tile_stencil_disable) && - !z.stencil_info.texture_compatible_stencil && !z.stencil_info.partially_resident && - (!has_stencil || - (z.stencil_info.format == Prospero::GpuEnumValue(Prospero::StencilFormat::k8UInt) && - !z.depth_view.stencil_write_disable)); - const bool supported_htile_state = - z.htile_surface.linear == 0 && z.htile_surface.full_cache == 0 && - z.htile_surface.htile_uses_preload_win == 0 && z.htile_surface.preload == 0 && - z.htile_surface.prefetch_width == 0 && z.htile_surface.prefetch_height == 0 && - z.htile_surface.dst_outside_zero_to_one == 0; - const bool supported_addresses = - z.z_read_base_addr != 0 && z.z_write_base_addr == z.z_read_base_addr && - (z.z_read_base_addr & 0xffffu) == 0 && - (has_stencil ? z.stencil_read_base_addr != 0 && - z.stencil_write_base_addr == z.stencil_read_base_addr && - (z.stencil_read_base_addr & 0xffffu) == 0 - : z.stencil_read_base_addr == 0 && z.stencil_write_base_addr == 0) && - z.htile_data_base_addr != 0 && (z.htile_data_base_addr & 0x7fffu) == 0 && - descriptor_size != 0 && (descriptor_size & 0x7fffu) == 0 && - z.htile_data_base_addr < TRACKER_ADDRESS_SIZE && - descriptor_size <= TRACKER_ADDRESS_SIZE - z.htile_data_base_addr; - if (!supported_depth_state || !supported_stencil_state || !supported_htile_state || - !supported_addresses) { - return false; - } - - const bool size_xy_valid = z.size.valid; - const bool wh_valid = z.width_height_valid && z.width != 0 && z.height != 0; - if (!size_xy_valid && !wh_valid) { - // Prospero emits an exact full-surface metadata descriptor even - // when the compute context omits depth extent registers. Admit only that wholly absent - // layout state; partially programmed layouts remain unsupported. - const bool descriptor_backed_state = - !z.size.valid && z.size.x_max == 0 && z.size.y_max == 0 && !z.width_height_valid && - z.width == 0 && z.height == 0 && !z.pitch_height_valid && z.pitch_div8_minus1 == 0 && - z.height_div8_minus1 == 0 && z.slice_div64_minus1 == 0; - if (!descriptor_backed_state) { - return false; - } - resolved = {.address = z.htile_data_base_addr, .size = descriptor_size}; - return true; - } - const uint32_t width = size_xy_valid ? static_cast(z.size.x_max) + 1u : z.width; - const uint32_t height = size_xy_valid ? static_cast(z.size.y_max) + 1u : z.height; - if (width > 16384 || height > 16384 || - (size_xy_valid && wh_valid && (width != z.width || height != z.height)) || - (!z.pitch_height_valid && - (z.pitch_div8_minus1 != 0 || z.height_div8_minus1 != 0 || z.slice_div64_minus1 != 0))) { - return false; - } - TileSizeAlign htile_size {}; - if (!TileGetHtileSize(width, height, htile_size) || htile_size.size != descriptor_size) { - return false; - } - resolved = {.address = z.htile_data_base_addr, .size = htile_size.size}; - return true; -} - -static void ValidateFullHtileClearDispatch(const ShaderComputeInputInfo& input, - const ShaderBufferResource& metadata, uint32_t group_x, - uint32_t group_y, uint32_t group_z, uint32_t mode) { - const bool thread_dimensions = input.dispatch_thread_dimensions; - const bool supported_shape = - input.threads_num[0] == 64 && input.threads_num[1] == 1 && input.threads_num[2] == 1 && - group_x != 0 && group_y == 1 && group_z == 1 && input.group_id[0] && !input.group_id[1] && - !input.group_id[2] && input.thread_ids_num == 1 && input.wave_size == 32 && - !input.tg_size_en && mode == (thread_dimensions ? 0x61u : 0x41u); - const bool dimensions_match = - thread_dimensions - ? input.dispatch_threads_num[0] == group_x && input.dispatch_threads_num[1] == 1 && - input.dispatch_threads_num[2] == 1 && group_x % input.threads_num[0] == 0 - : input.dispatch_threads_num[0] == 0 && input.dispatch_threads_num[1] == 0 && - input.dispatch_threads_num[2] == 0; - const uint64_t launched_threads = - thread_dimensions ? group_x : static_cast(group_x) * input.threads_num[0]; - if (!supported_shape || !dimensions_match || metadata.Stride() == 0 || - launched_threads != metadata.NumRecords()) { - EXIT("HTile compute clear does not cover the complete metadata surface\n"); - } -} - -static bool TryConsumeComputeMetaClear(const ShaderComputeInputInfo& input, - const RenderCommandBuffer& buffer, uint32_t group_x, - uint32_t group_y, uint32_t group_z, uint32_t mode) { - const auto& ctx = buffer.GetRegisters(); +bool RenderExecutor::TryConsumeComputeMetaClear(const ShaderComputeInputInfo& input, + const RenderCommandBuffer& buffer) { const auto& program = *input.stage.program; const auto& resources = *input.stage.resources; if (resources.buffers.size() != program.info.buffers.size()) { EXIT("compute runtime buffer count does not match shader metadata\n"); } - const auto& z = ctx.GetDepthRenderTarget(); - const uint64_t meta_addr = z.htile_data_base_addr; - auto& cache = GetRenderContext().GetTextureCache(); - uint32_t current_references = 0; - uint32_t registered_writes = 0; - uint64_t described_meta_size = 0; - HtileClearTarget registered_target {}; - TextureCache::MetaRangeInfo registered_meta {}; + auto& cache = buffer.GetContext().GetTextureCache(); for (uint32_t i = 0; i < program.info.buffers.size(); i++) { const auto& resource = program.info.buffers[i]; const auto descriptor = DecodeNativeDescriptor(resources.buffers[i]); - const auto descriptor_size = BufferDescriptorSize(descriptor); - if (meta_addr != 0 && descriptor.Base48() == meta_addr) { - current_references++; - described_meta_size = descriptor_size; - } - // An exact registered metadata range remains - // identifiable even when it is no longer the currently bound depth target. - TextureCache::MetaRangeInfo resolved_meta {}; - if (resource.written && - cache.ResolveMetaRange(descriptor.Base48(), descriptor_size, resolved_meta)) { - registered_writes++; - registered_target = {.address = descriptor.Base48(), .size = descriptor_size}; - registered_meta = resolved_meta; + if (!resource.written && cache.IsMeta(descriptor.Base48())) { + return false; } } - if (current_references == 0 && registered_writes == 0) { - return false; - } - if (current_references > 1 || (current_references == 0 && registered_writes > 1)) { - EXIT("HTile clear has ambiguous metadata descriptors: current=%u registered=%u\n", - current_references, registered_writes); - } - HtileClearTarget target {}; - if (current_references != 0) { - if (!ResolveHtileClearTarget(z, described_meta_size, target)) { - EXIT("unsupported HTile compute-clear target state: current=%u registered=%u " - "meta=0x%016" PRIx64 "+0x%016" PRIx64 " depth=0x%016" PRIx64 "/0x%016" PRIx64 - " stencil=0x%016" PRIx64 "/0x%016" PRIx64 - " extent=%d:%ux%u wh=%d:%ux%u pitch=%d:%u/%u/%u zfmt=%u sfmt=%u samples=%u\n", - current_references, registered_writes, meta_addr, described_meta_size, - z.z_read_base_addr, z.z_write_base_addr, z.stencil_read_base_addr, - z.stencil_write_base_addr, z.size.valid, static_cast(z.size.x_max) + 1u, - static_cast(z.size.y_max) + 1u, z.width_height_valid, z.width, z.height, - z.pitch_height_valid, z.pitch_div8_minus1, z.height_div8_minus1, - z.slice_div64_minus1, z.z_info.format, z.stencil_info.format, - z.z_info.num_samples); - } - cache.RegisterMeta(target.address, target.size); - if (!cache.ResolveMetaRange(target.address, target.size, registered_meta)) { - EXIT("failed to resolve registered HTile compute-clear range\n"); - } - } else { - target = registered_target; - } - GetRenderContext().GetBufferCache().ValidateGpuAccess(target.address, target.size, false, true); - uint32_t metadata_writes = 0; - ShaderBufferResource metadata_descriptor {}; - if (!program.info.images.empty() || !program.info.samplers.empty() || - !program.info.addresses.empty()) { - EXIT("HTile clear with non-buffer resources is unsupported: images=%zu samplers=%zu " - "addresses=%zu\n", - program.info.images.size(), program.info.samplers.size(), - program.info.addresses.size()); - } - for (uint32_t i = 0; i < program.info.buffers.size(); i++) { - const auto& resource = program.info.buffers[i]; - const auto descriptor = DecodeNativeDescriptor(resources.buffers[i]); - const auto descriptor_size = BufferDescriptorSize(descriptor); - if (descriptor.Base48() == target.address) { - if (!resource.written || resource.read || resource.atomic || - descriptor_size != target.size || descriptor.SwizzleEnabled() || - descriptor.IndexStride() != 0 || descriptor.AddTid() || - resource.packed_stride != descriptor.PackedStride()) { - EXIT("unsupported HTile compute metadata access\n"); + if (!program.info.has_bitwise_xor) { + for (uint32_t i = 0; i < program.info.buffers.size(); i++) { + const auto& resource = program.info.buffers[i]; + if (resource.written) { + const auto descriptor = + DecodeNativeDescriptor(resources.buffers[i]); + if (cache.ClearMeta(descriptor.Base48())) { + return true; + } } - metadata_descriptor = descriptor; - metadata_writes++; - continue; } - if (resource.written || !resource.read || resource.atomic || descriptor.Base48() == 0 || - descriptor_size == 0 || - cache.QueryRegion(descriptor.Base48(), descriptor_size).metadata_pages) { - EXIT("unsupported HTile clear side-buffer access\n"); - } - GetRenderContext().GetBufferCache().ValidateGpuAccess(descriptor.Base48(), descriptor_size, - true, false); } - if (metadata_writes != 1) { - EXIT("HTile clear requires exactly one write-only metadata buffer, writes=%u\n", - metadata_writes); - } - ValidateFullHtileClearDispatch(input, metadata_descriptor, group_x, group_y, group_z, mode); - const bool recorded = registered_meta.full ? cache.ClearMeta(registered_meta.metadata_address) - : cache.TouchMeta(registered_meta.metadata_address, - registered_meta.slice, true); - if (!recorded) { - EXIT("failed to record HTile compute clear\n"); - } - return true; + return false; } bool ResolveComputeImageClear(const ShaderComputeInputInfo& input, uint32_t group_x, @@ -324,7 +137,7 @@ static bool TryConsumeComputeImageClear(const ShaderComputeInputInfo& input, Com size)) { return false; } - auto& cache = GetRenderContext().GetTextureCache(); + auto& cache = command.GetContext().GetTextureCache(); if (!cache.ClearImageFromBuffer(command, descriptor.Base48(), size, packed_clear)) { return false; } @@ -337,8 +150,9 @@ static bool TryConsumeComputeImageClear(const ShaderComputeInputInfo& input, Com return true; } -void RenderDispatchDirect(uint64_t submit_id, RenderCommandBuffer& buffer, uint32_t thread_group_x, - uint32_t thread_group_y, uint32_t thread_group_z, uint32_t mode) { +void RenderExecutor::DispatchDirect(uint64_t submit_id, RenderCommandBuffer& buffer, + uint32_t thread_group_x, uint32_t thread_group_y, + uint32_t thread_group_z, uint32_t mode) { EXIT_IF(buffer.IsInvalid()); auto& ctx = buffer.GetRegisters(); auto& sh_ctx = buffer.GetShaders(); @@ -347,8 +161,7 @@ void RenderDispatchDirect(uint64_t submit_id, RenderCommandBuffer& buffer, uint3 thread_group_x, thread_group_y, thread_group_z, mode, sh_ctx.GetCs().cs_regs.data_addr); - Common::LockGuard lock(GetRenderContext().GetMutex()); - + Common::LockGuard lock(m_context.GetMutex()); if (sh_ctx.GetCs().cs_regs.data_addr == 0) { LOGF("GraphicsRenderDispatchDirect: temporary: ignoring dispatch with null CS shader, " "groups=%ux%ux%u mode=%u\n", @@ -396,17 +209,18 @@ void RenderDispatchDirect(uint64_t submit_id, RenderCommandBuffer& buffer, uint3 input_info.dispatch_threads_num[2] = thread_group_z; } - const uint32_t frame_num = GraphicsRunGetFrameNum(); + const uint32_t frame_num = static_cast(m_context.GetGpu().GetFrameNum()); const bool large_workgroup = (input_info.threads_num[0] * input_info.threads_num[1] * input_info.threads_num[2] >= 512); const auto& program = *input_info.stage.program; const auto& resources = *input_info.stage.resources; - if (TryConsumeComputeMetaClear(input_info, buffer, thread_group_x, thread_group_y, - thread_group_z, mode)) { + if (TryConsumeComputeMetaClear(input_info, buffer)) { + ResetBindings(); return; } if (TryConsumeComputeImageClear(input_info, buffer, thread_group_x, thread_group_y, thread_group_z, mode)) { + ResetBindings(); return; } const auto sampled_images = std::count_if( @@ -501,15 +315,13 @@ void RenderDispatchDirect(uint64_t submit_id, RenderCommandBuffer& buffer, uint3 return; } - auto& pipeline = GetRenderContext().GetPipelineCache().CreateComputePipeline( - input_info, sh_ctx.GetCs(), cs_shader); - auto bindings = PrepareBindings(buffer, input_info.stage, - vk::ShaderStageFlagBits::eCompute, + buffer.EndRendering(); + auto& pipeline = + m_context.GetPipelineCache().CreateComputePipeline(input_info, sh_ctx.GetCs(), cs_shader); + auto bindings = PrepareBindings(buffer, input_info.stage, vk::ShaderStageFlagBits::eCompute, DescriptorCache::Stage::Compute); RebindBuffers(buffer, bindings); RebindImages(buffer, bindings); - DescriptorCache::PreparedBindings* binding_sets[] = {&bindings}; - ActivateImageWrites(binding_sets); auto vk_buffer = buffer.Handle(); CommitBindings(buffer, vk::PipelineBindPoint::eCompute, pipeline.pipeline_layout, bindings); @@ -518,17 +330,17 @@ void RenderDispatchDirect(uint64_t submit_id, RenderCommandBuffer& buffer, uint3 bool has_storage_writes = HasShaderBufferWrites(input_info.stage); has_storage_writes = - std::any_of( - program.info.images.begin(), program.info.images.end(), - [](const auto& image) { - return image.written && - (image.kind == ShaderRecompiler::IR::ResourceKind::StorageImage || - image.kind == ShaderRecompiler::IR::ResourceKind::StorageImageUint); - }) || + std::any_of(program.info.images.begin(), program.info.images.end(), + [](const auto& image) { + return image.written && + (image.kind == ShaderRecompiler::IR::ResourceKind::StorageImage || + image.kind == ShaderRecompiler::IR::ResourceKind::StorageImageUint); + }) || has_storage_writes; if (has_storage_writes) { ShaderWriteBarrier(vk_buffer, vk::PipelineStageFlagBits::eComputeShader); } + ResetBindings(); } } // namespace Libs::Graphics diff --git a/src/graphics/host_gpu/renderer/renderContext.cpp b/src/graphics/host_gpu/renderer/renderContext.cpp index 67ab9e47..046d06a2 100644 --- a/src/graphics/host_gpu/renderer/renderContext.cpp +++ b/src/graphics/host_gpu/renderer/renderContext.cpp @@ -2,6 +2,8 @@ #include "common/assert.h" #include "common/logging/log.h" +#include "graphics/guest_gpu/graphicsRun.h" +#include "graphics/presentation/videoOut.h" #include "kernel/pthread.h" #include "libs/errno.h" @@ -10,24 +12,66 @@ namespace Libs::Graphics { RenderContext::RenderContext(GraphicContext& graphics) - : m_graphics(graphics), m_pipeline_cache(graphics), m_descriptor_cache(graphics), - m_framebuffer_cache(graphics), m_sampler_cache(graphics), m_gds_buffer(graphics), - m_gpu_resources(graphics) { + : m_graphics(graphics), m_render_executor(*this), m_command_scheduler(*this, graphics), + m_descriptor_cache(graphics), m_pipeline_cache(graphics, m_descriptor_cache), + m_sampler_cache(graphics), + m_gpu_resources(graphics, m_command_scheduler) { EXIT_NOT_IMPLEMENTED(!Common::Thread::IsMainThread()); } +RenderContext::~RenderContext() { + ShutdownGpu(); + m_command_scheduler.Shutdown(); +} + +void RenderContext::InitializeGpu(VideoOut::VideoOutDriver* video_out) { + EXIT_IF(m_gpu != nullptr); + m_video_out = video_out; + m_gpu = std::make_unique(*this); + m_gpu_resources.SetGpu(m_gpu.get()); +} + +void RenderContext::ShutdownGpu() { + if (m_gpu != nullptr) { + m_gpu_resources.SetGpu(nullptr); + m_gpu->Shutdown(); + m_gpu.reset(); + } + if (m_video_out != nullptr) { + if (m_command_scheduler.Active()) { + m_command_scheduler.FinishCurrent(); + } + m_command_scheduler.DrainPriorityOperations(); + m_video_out = nullptr; + } +} + +Gpu& RenderContext::GetGpu() const { + EXIT_IF(m_gpu == nullptr); + return *m_gpu; +} + +VideoOut::VideoOutDriver& RenderContext::GetVideoOut() const { + EXIT_IF(m_video_out == nullptr); + return *m_video_out; +} + void RenderContext::AddEopEq(LibKernel::EventQueue::KernelEqueue eq, int id) { + auto queue = LibKernel::EventQueue::KernelPinEqueue(eq); + if (!queue) { + return; + } + Common::LockGuard lock(m_eop_mutex); auto it = std::find_if(m_eop_eqs.begin(), m_eop_eqs.end(), [eq, id](const auto& entry) { return entry.eq == eq && entry.id == id; }); if (it != m_eop_eqs.end()) { - it->count++; return; } - m_eop_eqs.push_back({eq, id, 1}); + m_eop_eqs.push_back({eq, std::move(queue), id}); } void RenderContext::DeleteEopEq(LibKernel::EventQueue::KernelEqueue eq, int id) { @@ -40,22 +84,27 @@ void RenderContext::DeleteEopEq(LibKernel::EventQueue::KernelEqueue eq, int id) return; } - if (--it->count == 0) { - m_eop_eqs.erase(it); - } + m_eop_eqs.erase(it); } void RenderContext::TriggerEopEvent(uint32_t context_id) { - Common::LockGuard lock(m_eop_mutex); + std::vector registrations; + { + Common::LockGuard lock(m_eop_mutex); + registrations = m_eop_eqs; + } - for (auto& eop_entry: m_eop_eqs) { - if (eop_entry.eq != nullptr) { - const auto id = static_cast(eop_entry.id); - auto result = LibKernel::EventQueue::KernelTriggerEvent( - eop_entry.eq, id, LibKernel::EventQueue::KERNEL_EVFILT_GRAPHICS, - reinterpret_cast(static_cast(context_id))); - EXIT_NOT_IMPLEMENTED(result != OK && result != LibKernel::KERNEL_ERROR_ENOENT); + for (const auto& registration: registrations) { + const auto result = LibKernel::EventQueue::KernelTriggerEvent( + registration.eq, static_cast(registration.id), + LibKernel::EventQueue::KERNEL_EVFILT_GRAPHICS, + reinterpret_cast(static_cast(context_id))); + if (result == LibKernel::KERNEL_ERROR_EBADF || + result == LibKernel::KERNEL_ERROR_ENOENT) { + DeleteEopEq(registration.eq, registration.id); + continue; } + EXIT_NOT_IMPLEMENTED(result != OK); } auto tsc = LibKernel::KernelReadTsc(); diff --git a/src/graphics/host_gpu/renderer/renderContext.h b/src/graphics/host_gpu/renderer/renderContext.h index 6de6bbad..69c0c6d9 100644 --- a/src/graphics/host_gpu/renderer/renderContext.h +++ b/src/graphics/host_gpu/renderer/renderContext.h @@ -8,38 +8,45 @@ #include "graphics/host_gpu/renderer/bufferCache.h" #include "graphics/host_gpu/renderer/commandScheduler.h" #include "graphics/host_gpu/renderer/descriptorCache.h" -#include "graphics/host_gpu/renderer/framebufferCache.h" -#include "graphics/host_gpu/renderer/gdsBuffer.h" #include "graphics/host_gpu/renderer/gpuResourceManager.h" #include "graphics/host_gpu/renderer/pipelineCache.h" #include "graphics/host_gpu/renderer/samplerCache.h" #include "graphics/host_gpu/renderer/textureCache.h" #include "kernel/eventQueue.h" +#include #include +namespace Libs::VideoOut { +class VideoOutDriver; +} + namespace Libs::Graphics { constexpr int AGC_USER_INTERRUPT_EVENT = 0x1800; +class Gpu; class RenderContext { public: explicit RenderContext(GraphicContext& graphics); - ~RenderContext() { KYTY_NOT_IMPLEMENTED; } + ~RenderContext(); KYTY_CLASS_NO_COPY(RenderContext); [[nodiscard]] GraphicContext& GetGraphics() const noexcept { return m_graphics; } + void InitializeGpu(VideoOut::VideoOutDriver* video_out); + void ShutdownGpu(); + [[nodiscard]] Gpu& GetGpu() const; + [[nodiscard]] VideoOut::VideoOutDriver& GetVideoOut() const; Common::Mutex& GetMutex() { return m_mutex; } CommandScheduler& GetCommandScheduler() { return m_command_scheduler; } PipelineCache& GetPipelineCache() { return m_pipeline_cache; } DescriptorCache& GetDescriptorCache() { return m_descriptor_cache; } - FramebufferCache& GetFramebufferCache() { return m_framebuffer_cache; } SamplerCache& GetSamplerCache() { return m_sampler_cache; } - GdsBuffer& GetGdsBuffer() { return m_gds_buffer; } GpuResourceManager& GetGpuResources() { return m_gpu_resources; } BufferCache& GetBufferCache() { return m_gpu_resources.GetBufferCache(); } TextureCache& GetTextureCache() { return m_gpu_resources.GetTextureCache(); } + RenderExecutor& GetRenderExecutor() { return m_render_executor; } void AddEopEq(LibKernel::EventQueue::KernelEqueue eq, int id); void DeleteEopEq(LibKernel::EventQueue::KernelEqueue eq, int id); @@ -47,27 +54,26 @@ public: private: struct EopEqRegistration { - LibKernel::EventQueue::KernelEqueue eq = nullptr; - int id = 0; - uint32_t count = 0; + LibKernel::EventQueue::KernelEqueue eq = LibKernel::EventQueue::KERNEL_EQUEUE_INVALID; + LibKernel::EventQueue::KernelEqueueRef queue; + int id = 0; }; - GraphicContext& m_graphics; - Common::Mutex m_mutex; - CommandScheduler m_command_scheduler; - PipelineCache m_pipeline_cache; - DescriptorCache m_descriptor_cache; - FramebufferCache m_framebuffer_cache; - SamplerCache m_sampler_cache; - GdsBuffer m_gds_buffer; - GpuResourceManager m_gpu_resources; + GraphicContext& m_graphics; + Common::Mutex m_mutex; + RenderExecutor m_render_executor; + CommandScheduler m_command_scheduler; + DescriptorCache m_descriptor_cache; + PipelineCache m_pipeline_cache; + SamplerCache m_sampler_cache; + GpuResourceManager m_gpu_resources; + std::unique_ptr m_gpu; + VideoOut::VideoOutDriver* m_video_out = nullptr; Common::Mutex m_eop_mutex; std::vector m_eop_eqs; }; -[[nodiscard]] RenderContext& GetRenderContext() noexcept; - } // namespace Libs::Graphics #endif // EMULATOR_SRC_GRAPHICS_HOST_GPU_RENDERER_RENDERCONTEXT_H_ diff --git a/src/graphics/host_gpu/renderer/renderDraw.cpp b/src/graphics/host_gpu/renderer/renderDraw.cpp index d946ab73..26f6e813 100644 --- a/src/graphics/host_gpu/renderer/renderDraw.cpp +++ b/src/graphics/host_gpu/renderer/renderDraw.cpp @@ -13,18 +13,15 @@ #include "graphics/guest_gpu/hardwareContext.h" #include "graphics/guest_gpu/tile.h" #include "graphics/host_gpu/graphicContext.h" -#include "graphics/host_gpu/objects/label.h" #include "graphics/host_gpu/renderer/colorRenderTarget.h" #include "graphics/host_gpu/renderer/debug.h" #include "graphics/host_gpu/renderer/depthRenderTarget.h" #include "graphics/host_gpu/renderer/descriptorCache.h" -#include "graphics/host_gpu/renderer/framebufferCache.h" #include "graphics/host_gpu/renderer/pipelineCache.h" #include "graphics/host_gpu/renderer/render.h" #include "graphics/host_gpu/renderer/renderContext.h" #include "graphics/host_gpu/renderer/shaderResourceBarrier.h" #include "graphics/host_gpu/renderer/shaderSubgroup.h" -#include "graphics/host_gpu/transfer.h" #include "graphics/host_gpu/vulkanCommon.h" #include "graphics/shader/recompiler/ResourceMaterialization.h" #include "graphics/shader/recompiler/ShaderIR.h" @@ -36,6 +33,7 @@ #include #include #include +#include #include #include #include @@ -72,13 +70,9 @@ static std::atomic g_mrt_state_log_count = 0; static std::atomic g_shader_stage_log_count = 0; static std::atomic g_framebuffer_skip_log_count = 0; -static bool ResolveColorTargets(uint64_t submit_id, RenderCommandBuffer& buffer, - uint32_t render_target_slice_offset); - static const char* RenderColorTypeName(RenderColorType type) { switch (type) { case RenderColorType::NoColorOutput: return "NoColorOutput"; - case RenderColorType::DisplayBuffer: return "DisplayBuffer"; case RenderColorType::RenderTexture: return "RenderTexture"; default: return "Unknown"; } @@ -107,8 +101,8 @@ static void LogFramebufferSkip(const char* draw_name, const RenderColorInfo& col " color_image=%s depth_format=%s depth_image=%s depth_vaddr_num=%d target_mask=0x%08" PRIx32 " prim=%u index_count=%u flags=0x%08" PRIx32 "\n", log_id, draw_name, RenderColorTypeName(color.type), color.base_addr, color.buffer_size, - color.vulkan_buffer != nullptr ? "yes" : "no", VulkanToString(depth.format).c_str(), - depth.vulkan_buffer != nullptr ? "yes" : "no", depth.vaddr_num, ctx.GetRenderTargetMask(), + color.image_id ? "yes" : "no", VulkanToString(depth.format).c_str(), + depth.image_id ? "yes" : "no", depth.vaddr_num, ctx.GetRenderTargetMask(), ucfg.GetPrimType(), index_count, flags); } @@ -183,7 +177,7 @@ static void LogDrawTargetState(const char* draw_name, const RenderColorInfo& col } auto log_id = g_draw_state_log_count.fetch_add(1); - if (log_id >= 192 && color.type != RenderColorType::DisplayBuffer) { + if (log_id >= 192) { return; } @@ -199,7 +193,7 @@ static void LogDrawTargetState(const char* draw_name, const RenderColorInfo& col image.kind == ShaderRecompiler::IR::ResourceKind::ImageUint; }); - vk::Extent2D extent = color.vulkan_buffer != nullptr ? color.extent : vk::Extent2D {}; + vk::Extent2D extent = color.image_id ? color.extent : vk::Extent2D {}; auto sc = calc_final_scissor(vp, ctx.GetScanModeControl(), extent); LOGF( @@ -209,12 +203,13 @@ static void LogDrawTargetState(const char* draw_name, const RenderColorInfo& col " blend=%s src=%u dst=%u comb=%u ps_tex=%d sampled=%d storage=%d ps_kill=%s target_mode0=%u" " depth_test=%s depth_write=%s depth_func=%u depth_clear=%s viewport=(%.1f,%.1f %.1fx%.1f) " "scissor=(%d,%d)-(%d,%d)\n", - log_id, GraphicsRunGetFrameNum(), draw_name, RenderColorTypeName(color.type), - color.base_addr, extent.width, extent.height, ucfg.GetPrimType(), index_count, flags, - ctx.GetRenderTargetMask(), color.color_clear_enable ? "true" : "false", - color.color_clear_value.float32[0], color.color_clear_value.float32[1], - color.color_clear_value.float32[2], color.color_clear_value.float32[3], cc.mode, cc.op, - bc.enable ? "true" : "false", bc.color_srcblend, bc.color_destblend, bc.color_comb_fcn, + log_id, buffer.GetContext().GetGpu().GetFrameNum(), draw_name, + RenderColorTypeName(color.type), color.base_addr, extent.width, extent.height, + ucfg.GetPrimType(), index_count, flags, ctx.GetRenderTargetMask(), + color.color_clear_enable ? "true" : "false", color.color_clear_value.float32[0], + color.color_clear_value.float32[1], color.color_clear_value.float32[2], + color.color_clear_value.float32[3], cc.mode, cc.op, bc.enable ? "true" : "false", + bc.color_srcblend, bc.color_destblend, bc.color_comb_fcn, static_cast(ps_resources.images.size()), static_cast(sampled_images), static_cast(ps_resources.images.size() - sampled_images), ps_input_info.ps_pixel_kill_enable ? "true" : "false", ps_input_info.target_output_mode[0], @@ -226,20 +221,21 @@ static void LogDrawTargetState(const char* draw_name, const RenderColorInfo& col LogMrtState(draw_name, buffer, ps_input_info); } -static void LogDrawInputState(const RenderColorInfo& color, +static void LogDrawInputState(const RenderCommandBuffer& buffer, + const RenderColorInfo& color, const ShaderVertexInputInfo& vs_input_info, uint32_t index_type_and_size, uint32_t index_count, const void* index_addr) { auto log_id = g_draw_input_log_count.fetch_add(1); - if (log_id >= 64 && color.type != RenderColorType::DisplayBuffer) { + if (log_id >= 64) { return; } LOGF("DrawInputState[%u]: frame=%d target=%s addr=0x%010" PRIx64 " index_type=%u index_count=%u index_addr=0x%016" PRIx64 " vs_resources=%d vs_buffers=%d\n", - log_id, GraphicsRunGetFrameNum(), RenderColorTypeName(color.type), color.base_addr, - index_type_and_size, index_count, reinterpret_cast(index_addr), + log_id, buffer.GetContext().GetGpu().GetFrameNum(), RenderColorTypeName(color.type), + color.base_addr, index_type_and_size, index_count, reinterpret_cast(index_addr), vs_input_info.resources_num, vs_input_info.buffers_num); for (int bi = 0; bi < vs_input_info.buffers_num; bi++) { @@ -305,15 +301,6 @@ static void LogDrawInputState(const RenderColorInfo& color, } } -static void VulkanCmdSetColorWriteEnableEXT(vk::CommandBuffer command_buffer, - uint32_t attachment_count, - const vk::Bool32* p_color_write_enables) { - if (VULKAN_HPP_DEFAULT_DISPATCHER.vkCmdSetColorWriteEnableEXT == nullptr) { - EXIT("vkCmdSetColorWriteEnableEXT not present\n"); - } - command_buffer.setColorWriteEnableEXT(attachment_count, p_color_write_enables); -} - static PipelineDynamicParameters BuildGraphicsDynamicParams(const RenderCommandBuffer& buffer, const RenderColorInfo* colors, uint32_t color_count, @@ -327,10 +314,10 @@ static PipelineDynamicParameters BuildGraphicsDynamicParams(const RenderCommandB const auto& vp = ctx.GetScreenViewport(); vk::Extent2D framebuffer_extent {}; - if (color_count > 0 && colors[0].vulkan_buffer != nullptr) { + if (color_count > 0 && colors[0].image_id) { framebuffer_extent = colors[0].extent; - } else if (depth.vulkan_buffer != nullptr) { - framebuffer_extent = depth.vulkan_buffer->extent; + } else if (depth.image_id) { + framebuffer_extent = {depth.width, depth.height}; } const auto final_scissor = calc_final_scissor(vp, ctx.GetScanModeControl(), framebuffer_extent); @@ -412,7 +399,9 @@ static void SetDynamicParams(const RenderCommandBuffer& buffer, vk::CommandBuffe for (uint32_t i = 0; i < dynamic_params.color_write_count; i++) { enable[i] = (dynamic_params.color_write_enable[i] ? VK_TRUE : VK_FALSE); } - VulkanCmdSetColorWriteEnableEXT(vk_buffer, dynamic_params.color_write_count, enable); + if (dynamic_params.color_write_count != 0) { + vk_buffer.setColorWriteEnableEXT(dynamic_params.color_write_count, enable); + } } static bool DrawHasValidVertexShader(const HW::Shader& sh_ctx) { @@ -488,12 +477,11 @@ struct DrawRenderState { RenderColorInfo color_info[RENDER_COLOR_ATTACHMENTS_MAX] = {}; uint32_t color_count = 0; bool ps_active = true; - VulkanFramebuffer* framebuffer = nullptr; + RenderState rendering; ShaderVertexInputInfo vs_input_info; ShaderPixelInputInfo ps_input_info; std::span vs_shader; std::span ps_shader; - std::vector> target_owners; }; struct DrawCallInfo { @@ -505,6 +493,123 @@ struct DrawCallInfo { uint32_t first_instance = 0; }; +RenderState RenderExecutor::AcquireRenderTargets(CommandBuffer& buffer, RenderColorInfo* colors, + uint32_t color_count, RenderDepthInfo& depth) { + EXIT_IF(colors == nullptr || color_count > RENDER_COLOR_ATTACHMENTS_MAX); + auto& cache = m_context.GetTextureCache(); + RenderState state {}; + state.width = std::numeric_limits::max(); + state.height = std::numeric_limits::max(); + state.num_layers = std::numeric_limits::max(); + state.num_color_attachments = color_count; + uint32_t attachment_samples = 0; + for (uint32_t i = 0; i < color_count; i++) { + auto& target = colors[i]; + EXIT_IF(!target.image_id); + const auto old_image = cache.ResolveOwner(target.image_id); + if (old_image == nullptr || + (!old_image->registered && !old_image->info.data.Empty()) || + old_image->binding.needs_rebind) { + if (old_image != nullptr) { + old_image->binding = {}; + } + target.image_id = cache.FindImage(target.desc); + BindRenderTarget(target.image_id); + } + target.image_view = cache.FindRenderTarget(target.image_id, target.desc); + auto& image = cache.GetImage(target.image_id); + EXIT_IF(image.backing.samples != target.samples || target.image_view == nullptr); + if (attachment_samples == 0) { + attachment_samples = target.samples; + } else if (attachment_samples != target.samples) { + EXIT("mixed color attachment sample counts are unsupported: %u and %u\n", + attachment_samples, target.samples); + } + const auto& view = target.desc.view_info; + const auto layout = + image.binding.is_bound ? vk::ImageLayout::eGeneral + : vk::ImageLayout::eColorAttachmentOptimal; + image.Transit(layout, + vk::AccessFlagBits2::eColorAttachmentRead | + vk::AccessFlagBits2::eColorAttachmentWrite, + ImageSubresourceRange {view.base_level, view.level_count, view.base_layer, + view.layer_count}, + buffer.Handle()); + state.width = std::min(state.width, target.extent.width); + state.height = std::min(state.height, target.extent.height); + state.num_layers = std::min(state.num_layers, view.layer_count); + auto& attachment = state.color_attachments[i]; + attachment.image_view = target.image_view; + attachment.image_layout = layout; + attachment.clear_value = target.color_clear_value.uint32; + attachment.is_clear = target.color_clear_enable; + } + if (depth.image_id) { + const auto owner = cache.ResolveOwner(depth.image_id); + if (owner == nullptr || !owner->registered || owner->binding.needs_rebind) { + EXIT("depth target changed after render-state discovery\n"); + } + depth.image_view = cache.FindDepthTarget(depth.image_id, depth.desc); + if (depth.htile && depth.depth_clear_enable && !cache.ClearMeta(depth.htile_buffer_vaddr)) { + EXIT("failed to acquire HTile metadata for a depth clear\n"); + } + depth.depth_meta_clear_enable = + depth.htile && + cache.IsMetaCleared(depth.htile_buffer_vaddr, depth.desc.view_info.base_layer); + depth.depth_load_clear_enable = + depth.depth_clear_enable || depth.depth_meta_clear_enable; + if (depth.depth_meta_clear_enable && + !cache.TouchMeta(depth.htile_buffer_vaddr, depth.desc.view_info.base_layer, false)) { + EXIT("failed to consume HTile clear state\n"); + } + auto& image = cache.GetImage(depth.image_id); + EXIT_IF(depth.image_view == nullptr || image.backing.samples != depth.samples); + if (attachment_samples == 0) { + attachment_samples = depth.samples; + } else if (attachment_samples != depth.samples) { + EXIT("mixed color/depth sample counts are unsupported: %u and %u\n", + attachment_samples, depth.samples); + } + const auto layout = depth_attachment_layout(depth); + const auto writes = depth.AttachmentWriteAspects(); + auto access = vk::AccessFlags2 {vk::AccessFlagBits2::eDepthStencilAttachmentRead}; + if (writes) { + access |= vk::AccessFlagBits2::eDepthStencilAttachmentWrite; + } + const auto& view = depth.desc.view_info; + image.Transit(layout, access, + ImageSubresourceRange {view.base_level, view.level_count, view.base_layer, + view.layer_count}, + buffer.Handle()); + state.width = std::min(state.width, depth.width); + state.height = std::min(state.height, depth.height); + state.num_layers = std::min(state.num_layers, view.layer_count); + const auto aspects = ImageViewOps::DepthAspectMask(depth.format); + auto& attachment = state.depth_stencil_attachment; + attachment.image_view = depth.image_view; + attachment.image_layout = layout; + attachment.clear_value[0] = std::bit_cast(depth.depth_clear_value); + attachment.clear_value[1] = depth.stencil_clear_value; + attachment.has_depth = + static_cast(aspects & vk::ImageAspectFlagBits::eDepth); + attachment.depth_clear = depth.depth_load_clear_enable; + attachment.has_stencil = + static_cast(aspects & vk::ImageAspectFlagBits::eStencil); + attachment.stencil_clear = depth.stencil_clear_enable; + } + if (attachment_samples == 0 || + vulkan_sample_count(attachment_samples) == vk::SampleCountFlagBits {}) { + EXIT("render state has no valid attachments\n"); + } + if (state.num_layers == std::numeric_limits::max()) { + state.num_layers = 1; + } + EXIT_IF(state.width == 0 || state.height == 0 || state.num_layers == 0 || + state.width == std::numeric_limits::max() || + state.height == std::numeric_limits::max()); + return state; +} + static bool DrawHasActivePixelShader(const RenderCommandBuffer& buffer, const DrawRenderState& state, const DrawCallInfo& draw) { EXIT_IF(draw.name == nullptr); @@ -512,7 +617,7 @@ static bool DrawHasActivePixelShader(const RenderCommandBuffer& buffer, const auto& sh_ctx = buffer.GetShaders(); const bool with_depth = (state.depth_info.format != vk::Format::eUndefined && - state.depth_info.vulkan_buffer != nullptr); + static_cast(state.depth_info.image_id)); if (state.color_count != 0 || !with_depth) { return true; } @@ -561,13 +666,13 @@ struct DrawIndexBufferSource { }; struct PreparedIndexBuffer { - std::shared_ptr buffer; - const void* host_data = nullptr; - std::vector owned_data; - uint64_t address = 0; - uint64_t size = 0; - vk::DeviceSize offset = 0; - vk::IndexType type = vk::IndexType::eUint16; + std::shared_ptr owner; + vk::Buffer buffer = nullptr; + uint64_t address = 0; + uint64_t size = 0; + vk::DeviceSize offset = 0; + vk::IndexType type = vk::IndexType::eUint16; + bool streamed = false; }; static uint64_t VertexBufferDescriptorSize(const ShaderVertexInputBuffer& buffer) { @@ -625,26 +730,16 @@ static bool GetDrawTopology(const HW::UserConfig& ucfg, bool auto_draw, bool use return true; } -static bool PrepareDrawRenderState(uint64_t submit_id, RenderCommandBuffer& buffer, - const DrawCallInfo& draw, uint32_t render_target_slice_offset, - bool skip_null_framebuffer, bool log_setup_phases, - DrawRenderState& state) { +bool RenderExecutor::PrepareDrawRenderState(uint64_t submit_id, RenderCommandBuffer& buffer, + const DrawCallInfo& draw, + uint32_t render_target_slice_offset, + bool log_setup_phases, DrawRenderState& state) { EXIT_IF(draw.name == nullptr); auto& ctx = buffer.GetRegisters(); - if (log_setup_phases) { - LogDrawPhase(draw.name, "ResolveRenderDepthTarget"); - } - ResolveRenderDepthTarget(submit_id, buffer, state.depth_info); - if (ResolveColorTargets(submit_id, buffer, render_target_slice_offset)) { return false; } - if (state.depth_info.vulkan_buffer != nullptr) { - state.target_owners.push_back( - GetRenderContext().GetTextureCache().GetImageOwner(*state.depth_info.vulkan_buffer)); - } - if (log_setup_phases) { LogDrawPhase(draw.name, "ResolveRenderColorTarget"); } @@ -653,20 +748,18 @@ static bool PrepareDrawRenderState(uint64_t submit_id, RenderCommandBuffer& buff ctx.GetRenderTarget(slot).base.addr != 0)) { ResolveRenderColorTarget(submit_id, buffer, state.color_info[state.color_count], render_target_slice_offset, slot); - if (state.color_info[state.color_count].vulkan_buffer != nullptr) { - if (state.color_info[state.color_count].type == RenderColorType::RenderTexture) { - state.target_owners.push_back(GetRenderContext() - .GetTextureCache() - .GetImageOwner(*state.color_info[state.color_count] - .vulkan_buffer)); - } + if (state.color_info[state.color_count].image_id) { state.color_count++; } } } + if (log_setup_phases) { + LogDrawPhase(draw.name, "ResolveRenderDepthTarget"); + } + ResolveRenderDepthTarget(submit_id, buffer, state.depth_info); const bool with_depth = (state.depth_info.format != vk::Format::eUndefined && - state.depth_info.vulkan_buffer != nullptr); + static_cast(state.depth_info.image_id)); if (state.color_count == 0 && !with_depth) { LogFramebufferSkip(draw.name, state.color_info[0], state.depth_info, buffer, draw.index_count, draw.flags); @@ -674,20 +767,6 @@ static bool PrepareDrawRenderState(uint64_t submit_id, RenderCommandBuffer& buff } state.ps_active = DrawHasActivePixelShader(buffer, state, draw); - if (log_setup_phases) { - LogDrawPhase(draw.name, "CreateFramebuffer"); - } - state.framebuffer = GetRenderContext().GetFramebufferCache().CreateFramebuffer( - state.color_info, state.color_count, state.depth_info); - - if (state.framebuffer == nullptr && skip_null_framebuffer) { - LogFramebufferSkip(draw.name, state.color_info[0], state.depth_info, buffer, - draw.index_count, draw.flags); - return false; - } - EXIT_NOT_IMPLEMENTED(state.framebuffer == nullptr); - EXIT_NOT_IMPLEMENTED(state.framebuffer->render_pass == nullptr); - return true; } @@ -731,9 +810,10 @@ static void RefreshShaders(RenderCommandBuffer& buffer, const DrawCallInfo& draw } } -static std::vector -PrepareVertexBuffers(uint64_t submit_id, RenderCommandBuffer& buffer, const DrawCallInfo& draw, - const ShaderVertexInputInfo& vs_input_info) { +static std::vector PrepareVertexBuffers(uint64_t submit_id, + RenderCommandBuffer& buffer, + const DrawCallInfo& draw, + const ShaderVertexInputInfo& vs_input_info) { EXIT_IF(draw.name == nullptr); (void)submit_id; @@ -744,100 +824,80 @@ PrepareVertexBuffers(uint64_t submit_id, RenderCommandBuffer& buffer, const Draw const auto& b = vs_input_info.buffers[i]; const auto size = VertexBufferDescriptorSize(b); if (size == 0) { - bindings.push_back({GetRenderContext().GetBufferCache().ObtainNullBuffer(), 0}); + auto owner = buffer.GetContext().GetBufferCache().ObtainNullBuffer(); + bindings.push_back({owner, owner->Handle(), 0}); } else { bindings.push_back( - GetRenderContext().GetBufferCache().ObtainBuffer(buffer, b.addr, size)); + buffer.GetContext().GetBufferCache().ObtainBuffer(buffer, b.addr, size)); } } return bindings; } -static void RebindVertexBuffers(RenderCommandBuffer& buffer, +static void RebindVertexBuffers(RenderCommandBuffer& buffer, const ShaderVertexInputInfo& vs_input_info, - std::vector& bindings) { + std::vector& bindings) { EXIT_IF(bindings.size() != static_cast(vs_input_info.buffers_num)); for (int i = 0; i < vs_input_info.buffers_num; i++) { const auto& vertex = vs_input_info.buffers[i]; const auto size = VertexBufferDescriptorSize(vertex); - bindings[i] = size == 0 - ? BufferBinding {GetRenderContext().GetBufferCache().ObtainNullBuffer(), 0} - : GetRenderContext().GetBufferCache().ObtainBuffer(buffer, vertex.addr, size); + if (size == 0) { + auto owner = buffer.GetContext().GetBufferCache().ObtainNullBuffer(); + bindings[i] = {owner, owner->Handle(), 0}; + } else { + bindings[i] = + buffer.GetContext().GetBufferCache().ObtainBuffer(buffer, vertex.addr, size); + } } } -static PreparedIndexBuffer PrepareIndexBuffer(RenderCommandBuffer& buffer, +static PreparedIndexBuffer PrepareIndexBuffer(RenderCommandBuffer& buffer, const DrawIndexBufferSource& source) { PreparedIndexBuffer prepared; if (!source.enabled) { return prepared; } EXIT_IF(source.size == 0); - prepared.host_data = source.host_data; - prepared.address = source.address; - prepared.size = source.size; - prepared.type = source.type; + prepared.address = source.address; + prepared.size = source.size; + prepared.type = source.type; if (source.host_data != nullptr) { - return prepared; + auto binding = + buffer.GetContext().GetBufferCache().UploadTransient(source.host_data, source.size, 16); + prepared.owner = std::move(binding.owner); + prepared.buffer = binding.buffer; + prepared.offset = binding.offset; + prepared.streamed = true; } else { - auto binding = GetRenderContext().GetBufferCache().ObtainBuffer( - buffer, source.address, source.size); - prepared.buffer = std::move(binding.buffer); - prepared.owned_data = std::move(binding.host_data); - prepared.offset = binding.offset; + auto binding = + buffer.GetContext().GetBufferCache().ObtainBuffer(buffer, source.address, source.size); + prepared.owner = std::move(binding.owner); + prepared.buffer = binding.buffer; + prepared.offset = binding.offset; } return prepared; } static void RebindIndexBuffer(RenderCommandBuffer& buffer, PreparedIndexBuffer& prepared) { - if (prepared.size == 0 || prepared.host_data != nullptr) { + if (prepared.size == 0 || prepared.streamed) { return; } - auto binding = GetRenderContext().GetBufferCache().ObtainBuffer( - buffer, prepared.address, prepared.size); - prepared.buffer = std::move(binding.buffer); - prepared.owned_data = std::move(binding.host_data); - prepared.offset = binding.offset; -} - -static void RevalidateDrawTargets(DrawRenderState& state) { - auto old_owners = std::move(state.target_owners); - std::vector> current_owners; - auto validate = [&](VulkanImage& image) { - auto owner = GetRenderContext().GetTextureCache().GetImageOwner(image); - const auto found = std::find_if(old_owners.begin(), old_owners.end(), [&](const auto& old) { - return old.get() == owner.get(); - }); - EXIT_IF(found == old_owners.end()); - current_owners.push_back(std::move(owner)); - }; - if (state.depth_info.vulkan_buffer != nullptr) { - validate(*state.depth_info.vulkan_buffer); - } - for (uint32_t i = 0; i < state.color_count; i++) { - if (state.color_info[i].type == RenderColorType::RenderTexture) { - validate(*state.color_info[i].vulkan_buffer); - } - } - state.target_owners = std::move(current_owners); + auto binding = + buffer.GetContext().GetBufferCache().ObtainBuffer(buffer, prepared.address, prepared.size); + prepared.owner = std::move(binding.owner); + prepared.buffer = binding.buffer; + prepared.offset = binding.offset; } static void CommitVertexBuffers(RenderCommandBuffer& buffer, vk::CommandBuffer vk_buffer, std::vector& bindings) { for (uint32_t slot = 0; slot < bindings.size(); slot++) { - auto& binding = bindings[slot]; - VulkanBuffer* vertex_buffer = binding.buffer.get(); - vk::DeviceSize vertex_offset = binding.offset; - if (!binding.host_data.empty()) { - vk::DeviceSize range = 0; - EXIT_IF(!GetRenderContext().GetBufferCache().UploadHostData( - buffer, binding.host_data.data(), binding.host_data.size(), 16, vertex_buffer, - vertex_offset, range)); - } else { - buffer.RetainResourceUntilFence(binding.buffer); + auto& binding = bindings[slot]; + if (binding.owner != nullptr) { + buffer.RetainResourceUntilFence(binding.owner); } - EXIT_IF(vertex_buffer == nullptr); - vk_buffer.bindVertexBuffers(slot, 1, &vertex_buffer->buffer, &vertex_offset); + EXIT_IF(binding.buffer == nullptr); + vk_buffer.bindVertexBuffers(slot, 1, &binding.buffer, &binding.offset); } } @@ -846,23 +906,11 @@ static void CommitIndexBuffer(RenderCommandBuffer& buffer, vk::CommandBuffer vk_ if (prepared.size == 0) { return; } - VulkanBuffer* index_buffer = prepared.buffer.get(); - vk::DeviceSize index_offset = prepared.offset; - const void* host_data = prepared.host_data; - if (host_data == nullptr && !prepared.owned_data.empty()) { - host_data = prepared.owned_data.data(); + if (prepared.owner != nullptr) { + buffer.RetainResourceUntilFence(prepared.owner); } - if (host_data != nullptr) { - vk::DeviceSize range = 0; - if (!GetRenderContext().GetBufferCache().UploadHostData( - buffer, host_data, prepared.size, 16, index_buffer, index_offset, range)) { - EXIT("failed to upload host index buffer\n"); - } - } else { - buffer.RetainResourceUntilFence(prepared.buffer); - } - EXIT_IF(index_buffer == nullptr); - vk_buffer.bindIndexBuffer(index_buffer->buffer, index_offset, prepared.type); + EXIT_IF(prepared.buffer == nullptr); + vk_buffer.bindIndexBuffer(prepared.buffer, prepared.offset, prepared.type); } static void LogDrawStateIfNeeded(const RenderCommandBuffer& buffer, const DrawCallInfo& draw, @@ -883,7 +931,7 @@ static void LogDrawStateIfNeeded(const RenderCommandBuffer& buffer, const DrawCa LogDrawTargetState(draw.name, state.color_info[0], state.depth_info, buffer, state.ps_input_info, draw.index_count, draw.flags); } - LogDrawInputState(state.color_info[0], state.vs_input_info, index_type_and_size, + LogDrawInputState(buffer, state.color_info[0], state.vs_input_info, index_type_and_size, draw.index_count, index_addr); // LogDrawTextureState(draw.name, state.color_info[0], state.ps_input_info); } @@ -957,63 +1005,35 @@ static void EmitDrawPrimitives(const HW::UserConfig& ucfg, vk::CommandBuffer vk_ } } -static void ExecutePreparedDraw(uint64_t submit_id, RenderCommandBuffer& buffer, - const DrawCallInfo& draw, DrawRenderState& state, - vk::PrimitiveTopology topology, const DrawEmitInfo& emit, - const DrawIndexBufferSource& index_source, bool log_pipeline_phase, - bool set_bind_debug, bool set_auto_debug) { +void RenderExecutor::ExecutePreparedDraw(uint64_t submit_id, RenderCommandBuffer& buffer, + const DrawCallInfo& draw, DrawRenderState& state, + vk::PrimitiveTopology topology, const DrawEmitInfo& emit, + const DrawIndexBufferSource& index_source, + bool log_pipeline_phase, bool set_bind_debug, + bool set_auto_debug) { EXIT_IF(draw.name == nullptr); auto& ucfg = buffer.GetUserConfig(); + + LogDrawPhase(draw.name, "PrepareBindings"); + auto bindings = PrepareGraphicsBindings(buffer, state.vs_input_info.stage, + state.ps_input_info.stage, state.ps_active); + auto vertex_bindings = PrepareVertexBuffers(submit_id, buffer, draw, state.vs_input_info); + auto index_binding = PrepareIndexBuffer(buffer, index_source); + RebindVertexBuffers(buffer, state.vs_input_info, vertex_bindings); + RebindIndexBuffer(buffer, index_binding); + state.rendering = + AcquireRenderTargets(buffer, state.color_info, state.color_count, state.depth_info); + if (log_pipeline_phase) { LogDrawPhase(draw.name, "CreatePipeline"); } - auto& pipeline = GetRenderContext().GetPipelineCache().CreateGraphicsPipeline( - *state.framebuffer, state.color_info, state.color_count, state.depth_info, - state.vs_input_info, buffer, &state.ps_input_info, topology, state.ps_active, - state.vs_shader, state.ps_shader); - - LogDrawPhase(draw.name, "PrepareBindingsVS"); - auto vs_bindings = PrepareBindings(buffer, state.vs_input_info.stage, - vk::ShaderStageFlagBits::eVertex, - DescriptorCache::Stage::Vertex); - std::optional ps_bindings; - if (state.ps_active) { - LogDrawPhase(draw.name, "PrepareBindingsPS"); - ps_bindings.emplace(PrepareBindings(buffer, state.ps_input_info.stage, - vk::ShaderStageFlagBits::eFragment, - DescriptorCache::Stage::Pixel)); - } - auto vertex_bindings = PrepareVertexBuffers(submit_id, buffer, draw, state.vs_input_info); - auto index_binding = PrepareIndexBuffer(buffer, index_source); - - // The discovery pass above can merge cache allocations. Rebind every buffer only after the - // complete draw resource set is known. - RebindBuffers(buffer, vs_bindings); - if (ps_bindings.has_value()) { - RebindBuffers(buffer, *ps_bindings); - } - RebindVertexBuffers(buffer, state.vs_input_info, vertex_bindings); - RebindIndexBuffer(buffer, index_binding); - RebindImages(buffer, vs_bindings); - if (ps_bindings.has_value()) { - RebindImages(buffer, *ps_bindings); - } + auto& pipeline = m_context.GetPipelineCache().CreateGraphicsPipeline( + state.color_info, state.color_count, state.depth_info, state.vs_input_info, buffer, + &state.ps_input_info, topology, state.ps_active, state.vs_shader, state.ps_shader); // Resource preparation above may synchronously finish and restart the scheduler. From this // point onward, every operation targets the current command buffer and cannot touch guest // memory. - RevalidateDrawTargets(state); - DescriptorCache::PreparedBindings* binding_sets[] = { - &vs_bindings, ps_bindings.has_value() ? &*ps_bindings : nullptr}; - ActivateImageWrites(binding_sets); - MarkRenderTargetGpuWritten(state.depth_info); - for (uint32_t i = 0; i < state.color_count; i++) { - MarkRenderTargetGpuWritten(state.color_info[i]); - } - for (const auto& owner: state.target_owners) { - buffer.RetainResourceUntilFence(owner); - } - auto vk_buffer = buffer.Handle(); if (set_bind_debug) { SetDrawDebugPhase(buffer, submit_id, draw, 0x100u); @@ -1023,26 +1043,25 @@ static void ExecutePreparedDraw(uint64_t submit_id, RenderCommandBuffer& buffer, } CommitVertexBuffers(buffer, vk_buffer, vertex_bindings); CommitBindings(buffer, vk::PipelineBindPoint::eGraphics, pipeline.pipeline_layout, - vs_bindings); - if (ps_bindings.has_value()) { + bindings.vertex); + if (bindings.pixel.has_value()) { if (set_auto_debug) { SetDrawDebugPhase(buffer, submit_id, draw, 0x300u); } CommitBindings(buffer, vk::PipelineBindPoint::eGraphics, pipeline.pipeline_layout, - *ps_bindings); + *bindings.pixel); } CommitIndexBuffer(buffer, vk_buffer, index_binding); - const auto dynamic_params = BuildGraphicsDynamicParams(buffer, state.color_info, - state.color_count, state.depth_info); + const auto dynamic_params = + BuildGraphicsDynamicParams(buffer, state.color_info, state.color_count, state.depth_info); SetDynamicParams(buffer, vk_buffer, dynamic_params); - LogDrawPhase(draw.name, "BeginRenderPass"); + LogDrawPhase(draw.name, "BeginRendering"); if (set_auto_debug) { SetDrawDebugPhase(buffer, submit_id, draw, 0x400u); } - buffer.BeginRenderPass(*state.framebuffer, state.color_info, state.color_count, - state.depth_info); + m_context.GetCommandScheduler().BeginRendering(state.rendering); vk_buffer.bindPipeline(vk::PipelineBindPoint::eGraphics, pipeline.pipeline); if (set_auto_debug) { SetDrawDebugPhase(buffer, submit_id, draw, 0x500u); @@ -1052,7 +1071,6 @@ static void ExecutePreparedDraw(uint64_t submit_id, RenderCommandBuffer& buffer, if (set_auto_debug) { SetDrawDebugPhase(buffer, submit_id, draw, 0x600u); } - buffer.EndRenderPass(); vk::PipelineStageFlags shader_write_stages = {}; if (HasShaderBufferWrites(state.vs_input_info.stage)) { shader_write_stages |= vk::PipelineStageFlagBits::eVertexShader; @@ -1061,18 +1079,20 @@ static void ExecutePreparedDraw(uint64_t submit_id, RenderCommandBuffer& buffer, shader_write_stages |= vk::PipelineStageFlagBits::eFragmentShader; } if (shader_write_stages) { + m_context.GetCommandScheduler().EndRendering(); ShaderWriteBarrier(vk_buffer, shader_write_stages); } - LogDrawPhase(draw.name, "EndRenderPass"); + LogDrawPhase(draw.name, "DrawComplete"); if (set_auto_debug) { SetDrawDebugPhase(buffer, submit_id, draw, 0x700u); } } -void RenderDrawIndex(uint64_t submit_id, RenderCommandBuffer& buffer, uint32_t index_type_and_size, - uint32_t index_count, const void* index_addr, uint32_t flags, uint32_t type, - uint32_t instance_count, uint32_t render_target_slice_offset, - int32_t vertex_offset_add, uint32_t first_instance) { +void RenderExecutor::DrawIndex(uint64_t submit_id, RenderCommandBuffer& buffer, + uint32_t index_type_and_size, uint32_t index_count, + const void* index_addr, uint32_t flags, uint32_t type, + uint32_t instance_count, uint32_t render_target_slice_offset, + int32_t vertex_offset_add, uint32_t first_instance) { KYTY_PROFILER_FUNCTION(); EXIT_IF(buffer.IsInvalid()); @@ -1083,13 +1103,13 @@ void RenderDrawIndex(uint64_t submit_id, RenderCommandBuffer& buffer, uint32_t i index_count, flags, type, instance_count, reinterpret_cast(index_addr)); - Common::LockGuard lock(GetRenderContext().GetMutex()); - + Common::LockGuard lock(m_context.GetMutex()); if (index_count == 0) { return; } if (ConsumeMetadataColorOperation(buffer)) { + ResetBindings(); return; } @@ -1181,8 +1201,8 @@ void RenderDrawIndex(uint64_t submit_id, RenderCommandBuffer& buffer, uint32_t i index_source.type = index_type; DrawRenderState state {}; - if (!PrepareDrawRenderState(submit_id, buffer, draw, render_target_slice_offset, false, true, - state)) { + if (!PrepareDrawRenderState(submit_id, buffer, draw, render_target_slice_offset, true, state)) { + ResetBindings(); return; } @@ -1199,12 +1219,15 @@ void RenderDrawIndex(uint64_t submit_id, RenderCommandBuffer& buffer, uint32_t i ExecutePreparedDraw(submit_id, buffer, draw, state, topology, emit, index_source, true, true, false); + ResetBindings(); } // NOLINTNEXTLINE(readability-function-cognitive-complexity) -void RenderDrawIndexAuto(uint64_t submit_id, RenderCommandBuffer& buffer, uint32_t index_count, - uint32_t flags, uint32_t render_target_slice_offset, - uint32_t instance_count, uint32_t first_vertex, uint32_t first_instance) { +void RenderExecutor::DrawAuto(uint64_t submit_id, RenderCommandBuffer& buffer, + uint32_t index_count, + uint32_t flags, uint32_t render_target_slice_offset, + uint32_t instance_count, uint32_t first_vertex, + uint32_t first_instance) { KYTY_PROFILER_FUNCTION(); EXIT_IF(buffer.IsInvalid()); @@ -1214,13 +1237,13 @@ void RenderDrawIndexAuto(uint64_t submit_id, RenderCommandBuffer& buffer, uint32 buffer.SetDebugInfo(static_cast(CommandBufferDebugOp::DrawIndexAuto), submit_id, index_count, flags, first_vertex, instance_count, first_instance); - Common::LockGuard lock(GetRenderContext().GetMutex()); - + Common::LockGuard lock(m_context.GetMutex()); if (index_count == 0) { return; } if (ConsumeMetadataColorOperation(buffer)) { + ResetBindings(); return; } @@ -1262,8 +1285,8 @@ void RenderDrawIndexAuto(uint64_t submit_id, RenderCommandBuffer& buffer, uint32 instance_count, first_instance}; DrawRenderState state {}; - if (!PrepareDrawRenderState(submit_id, buffer, draw, render_target_slice_offset, true, false, - state)) { + if (!PrepareDrawRenderState(submit_id, buffer, draw, render_target_slice_offset, false, state)) { + ResetBindings(); return; } @@ -1271,6 +1294,7 @@ void RenderDrawIndexAuto(uint64_t submit_id, RenderCommandBuffer& buffer, uint32 const bool use_ngg_rectlist_draw = Config::NggRectlistDrawEnabled(); if (!GetDrawTopology(ucfg, true, use_ngg_rectlist_draw, topology)) { + ResetBindings(); return; } const bool draw_prim7_as_ngg = @@ -1287,6 +1311,7 @@ void RenderDrawIndexAuto(uint64_t submit_id, RenderCommandBuffer& buffer, uint32 state.ps_input_info.input_num, sh_ctx.GetPs().ps_regs.chksum, sh_ctx.GetVs().es_regs.data_addr, sh_ctx.GetVs().gs_regs.data_addr); } + ResetBindings(); return; } @@ -1306,27 +1331,11 @@ void RenderDrawIndexAuto(uint64_t submit_id, RenderCommandBuffer& buffer, uint32 DrawIndexBufferSource index_source {}; ExecutePreparedDraw(submit_id, buffer, draw, state, topology, emit, index_source, false, false, true); + ResetBindings(); } -bool IsSameColorResolveSubresource(const RenderColorInfo& src, const RenderColorInfo& dst) { - return src.base_addr == dst.base_addr && src.base_mip_level == dst.base_mip_level && - src.base_array_layer == dst.base_array_layer; -} - -ImageImageCopy MakeColorResolveCopy(const RenderColorInfo& src, const RenderColorInfo& dst, - uint32_t width, uint32_t height) { - ImageImageCopy region {*src.vulkan_buffer}; - region.src_level = src.base_mip_level; - region.dst_level = dst.base_mip_level; - region.width = width; - region.height = height; - region.src_layer = src.base_array_layer; - region.dst_layer = dst.base_array_layer; - return region; -} - -static bool ResolveColorTargets(uint64_t submit_id, RenderCommandBuffer& buffer, - uint32_t render_target_slice_offset) { +bool RenderExecutor::ResolveColorTargets(uint64_t submit_id, RenderCommandBuffer& buffer, + uint32_t render_target_slice_offset) { const auto& hw = buffer.GetRegisters(); if (hw.GetColorControl().mode != 3) { return false; @@ -1341,25 +1350,23 @@ static bool ResolveColorTargets(uint64_t submit_id, RenderCommandBuffer& buffer, RenderColorInfo src {}; RenderColorInfo dst {}; ResolveRenderColorTarget(submit_id, buffer, src, render_target_slice_offset, 0, true, true); - ResolveRenderColorTarget(submit_id, buffer, dst, render_target_slice_offset, 1, true); - if (src.vulkan_buffer == nullptr || dst.vulkan_buffer == nullptr || - src.type == RenderColorType::NoColorOutput || dst.type == RenderColorType::NoColorOutput) { + ResolveRenderColorTarget(submit_id, buffer, dst, render_target_slice_offset, 1, true, true); + if (!src.image_id || !dst.image_id || src.type == RenderColorType::NoColorOutput || + dst.type == RenderColorType::NoColorOutput) { return false; } - if (IsSameColorResolveSubresource(src, dst)) { + if (src.base_addr == dst.base_addr && src.base_mip_level == dst.base_mip_level && + src.base_array_layer == dst.base_array_layer) { return true; } - const uint32_t width = std::min(src.extent.width, dst.extent.width); - const uint32_t height = std::min(src.extent.height, dst.extent.height); - if (width == 0 || height == 0) { - return false; - } - - const std::array regions {MakeColorResolveCopy(src, dst, width, height)}; - - MarkRenderTargetGpuWritten(dst); - Transfer::CopyImage(buffer, regions, *dst.vulkan_buffer, dst.vulkan_buffer->layout); + auto& cache = m_context.GetTextureCache(); + cache.MarkGpuWritten(dst.image_id); + auto& source = cache.GetImage(src.image_id); + auto& destination = cache.GetImage(dst.image_id); + destination.Resolve(source, + {src.base_mip_level, 1, src.base_array_layer, 1}, + {dst.base_mip_level, 1, dst.base_array_layer, 1}); return true; } diff --git a/src/graphics/host_gpu/renderer/renderDraw.h b/src/graphics/host_gpu/renderer/renderDraw.h index 6ff662cf..f17af2d4 100644 --- a/src/graphics/host_gpu/renderer/renderDraw.h +++ b/src/graphics/host_gpu/renderer/renderDraw.h @@ -8,7 +8,6 @@ namespace Libs::Graphics { -struct ImageImageCopy; struct RenderColorInfo; struct ShaderVertexInputInfo; @@ -39,13 +38,8 @@ static_assert(sizeof(PipelineDynamicParameters) == sizeof(uint32_t) + sizeof(bool[RENDER_COLOR_ATTACHMENTS_MAX]) + sizeof(PipelineStencilDynamicState) * 2); -[[nodiscard]] bool IsSameColorResolveSubresource(const RenderColorInfo& src, - const RenderColorInfo& dst); -[[nodiscard]] ImageImageCopy MakeColorResolveCopy(const RenderColorInfo& src, - const RenderColorInfo& dst, uint32_t width, - uint32_t height); -[[nodiscard]] int32_t ResolveVertexOffset(uint32_t index_offset, - const ShaderVertexInputInfo& vs_input_info); +[[nodiscard]] int32_t ResolveVertexOffset(uint32_t index_offset, + const ShaderVertexInputInfo& vs_input_info); } // namespace Libs::Graphics diff --git a/src/graphics/host_gpu/renderer/renderTarget.h b/src/graphics/host_gpu/renderer/renderTarget.h index 23a79f3c..37f9efdb 100644 --- a/src/graphics/host_gpu/renderer/renderTarget.h +++ b/src/graphics/host_gpu/renderer/renderTarget.h @@ -3,6 +3,7 @@ #include "graphics/host_gpu/vulkanCommon.h" +#include #include #include @@ -10,6 +11,30 @@ namespace Libs::Graphics { static constexpr uint32_t RENDER_COLOR_ATTACHMENTS_MAX = 8; +struct RenderAttachment { + vk::ImageView image_view = nullptr; + vk::ImageLayout image_layout = vk::ImageLayout::eUndefined; + std::array clear_value = {}; + bool is_clear = false; + bool has_depth = false; + bool depth_clear = false; + bool has_stencil = false; + bool stencil_clear = false; + + bool operator==(const RenderAttachment&) const = default; +}; + +struct RenderState { + std::array color_attachments; + RenderAttachment depth_stencil_attachment; + uint32_t width = 0; + uint32_t height = 0; + uint32_t num_layers = 1; + uint32_t num_color_attachments = 0; + + bool operator==(const RenderState&) const = default; +}; + [[nodiscard]] inline constexpr uint32_t render_sample_count(uint32_t encoded_samples) { return encoded_samples <= 3 ? 1u << encoded_samples : 0; } diff --git a/src/graphics/host_gpu/renderer/renderTargetBarriers.cpp b/src/graphics/host_gpu/renderer/renderTargetBarriers.cpp deleted file mode 100644 index 2e39edf3..00000000 --- a/src/graphics/host_gpu/renderer/renderTargetBarriers.cpp +++ /dev/null @@ -1,221 +0,0 @@ -#include "graphics/host_gpu/renderer/renderTargetBarriers.h" - -#include "common/assert.h" -#include "common/common.h" -#include "common/logging/log.h" -#include "common/profiler.h" -#include "common/stringUtils.h" -#include "common/threads.h" -#include "graphics/guest_gpu/gpu_defs.h" -#include "graphics/guest_gpu/hardwareContext.h" -#include "graphics/guest_gpu/tile.h" -#include "graphics/host_gpu/graphicContext.h" -#include "graphics/host_gpu/objects/textureCommon.h" -#include "graphics/host_gpu/renderer/framebufferCache.h" -#include "graphics/host_gpu/renderer/imageView.h" -#include "graphics/host_gpu/renderer/render.h" -#include "graphics/host_gpu/renderer/renderContext.h" -#include "graphics/host_gpu/transfer.h" -#include "graphics/host_gpu/vulkanCommon.h" -#include "graphics/presentation/displayBuffer.h" - -#include -#include -#include -#include -#include -#include - -namespace Libs::Graphics { - -void GraphicsRenderMemoryBarrier(CommandBuffer& buffer) { - EXIT_IF(buffer.IsInvalid()); - - Common::LockGuard lock(GetRenderContext().GetMutex()); - - auto vk_buffer = buffer.Handle(); - - VulkanMemoryBarrier mem_barrier {}; - mem_barrier.sType = vk::StructureType::eMemoryBarrier; - mem_barrier.pNext = nullptr; - mem_barrier.srcAccessMask = vk::AccessFlagBits::eMemoryWrite; - mem_barrier.dstAccessMask = vk::AccessFlagBits::eMemoryRead | vk::AccessFlagBits::eMemoryWrite; - - vk_buffer.pipelineBarrier(vk::PipelineStageFlagBits::eAllCommands, - vk::PipelineStageFlagBits::eAllCommands, vk::DependencyFlags {}, 1, - &mem_barrier, 0, nullptr, 0, nullptr); -} - -void GraphicsRenderTextureBarrier(vk::CommandBuffer vk_buffer, VulkanImage& image) { - - vk::ImageMemoryBarrier image_memory_barrier {}; - image_memory_barrier.sType = vk::StructureType::eImageMemoryBarrier; - image_memory_barrier.pNext = nullptr; - image_memory_barrier.srcAccessMask = - vk::AccessFlagBits::eColorAttachmentWrite | vk::AccessFlagBits::eTransferWrite | - vk::AccessFlagBits::eShaderWrite | vk::AccessFlagBits::eMemoryRead; - image_memory_barrier.dstAccessMask = vk::AccessFlagBits::eShaderRead; - image_memory_barrier.oldLayout = image.layout; - image_memory_barrier.newLayout = RENDER_COLOR_IMAGE_LAYOUT; - image_memory_barrier.srcQueueFamilyIndex = VK_QUEUE_FAMILY_IGNORED; - image_memory_barrier.dstQueueFamilyIndex = VK_QUEUE_FAMILY_IGNORED; - image_memory_barrier.image = image.image; - image_memory_barrier.subresourceRange.aspectMask = vk::ImageAspectFlagBits::eColor; - image_memory_barrier.subresourceRange.baseMipLevel = 0; - image_memory_barrier.subresourceRange.levelCount = VK_REMAINING_MIP_LEVELS; - image_memory_barrier.subresourceRange.baseArrayLayer = 0; - image_memory_barrier.subresourceRange.layerCount = image.layers; - - vk_buffer.pipelineBarrier( - vk::PipelineStageFlagBits::eColorAttachmentOutput | - vk::PipelineStageFlagBits::eComputeShader | vk::PipelineStageFlagBits::eTransfer | - vk::PipelineStageFlagBits::eFragmentShader, - vk::PipelineStageFlagBits::eVertexShader | vk::PipelineStageFlagBits::eFragmentShader | - vk::PipelineStageFlagBits::eComputeShader, - vk::DependencyFlags {}, 0, nullptr, 0, nullptr, 1, &image_memory_barrier); - - image.layout = image_memory_barrier.newLayout; -} - -void GraphicsRenderColorImageBarrier(vk::CommandBuffer vk_buffer, VulkanImage& image, - vk::ImageLayout new_layout) { - if (image.layout != new_layout) { - if (image.type == VulkanImageType::VideoOut && - image.layout == vk::ImageLayout::eTransferDstOptimal) { - static std::atomic log_count {0}; - if (log_count.fetch_add(1, std::memory_order_relaxed) < 128) { - LOGF("GraphicsRenderColorImageBarrier: image=%p type=%d layout=%s -> %s\n", - VulkanHandleToPointer(image.image), static_cast(image.type), - VulkanToString(image.layout).c_str(), VulkanToString(new_layout).c_str()); - } - } - vk::ImageMemoryBarrier image_memory_barrier {}; - image_memory_barrier.sType = vk::StructureType::eImageMemoryBarrier; - image_memory_barrier.pNext = nullptr; - image_memory_barrier.srcAccessMask = - vk::AccessFlagBits::eMemoryWrite | vk::AccessFlagBits::eMemoryRead; - image_memory_barrier.dstAccessMask = - vk::AccessFlagBits::eMemoryWrite | vk::AccessFlagBits::eMemoryRead; - image_memory_barrier.oldLayout = image.layout; - image_memory_barrier.newLayout = new_layout; - image_memory_barrier.srcQueueFamilyIndex = VK_QUEUE_FAMILY_IGNORED; - image_memory_barrier.dstQueueFamilyIndex = VK_QUEUE_FAMILY_IGNORED; - image_memory_barrier.image = image.image; - image_memory_barrier.subresourceRange.aspectMask = vk::ImageAspectFlagBits::eColor; - image_memory_barrier.subresourceRange.baseMipLevel = 0; - image_memory_barrier.subresourceRange.levelCount = VK_REMAINING_MIP_LEVELS; - image_memory_barrier.subresourceRange.baseArrayLayer = 0; - image_memory_barrier.subresourceRange.layerCount = image.layers; - - const auto stages = static_cast( - vk::PipelineStageFlagBits::eAllGraphics | vk::PipelineStageFlagBits::eComputeShader | - vk::PipelineStageFlagBits::eTransfer); - vk_buffer.pipelineBarrier(stages, stages, vk::DependencyFlags {}, 0, nullptr, 0, nullptr, 1, - &image_memory_barrier); - - image.layout = image_memory_barrier.newLayout; - } -} - -void GraphicsRenderDepthStencilImageBarrier(vk::CommandBuffer vk_buffer, VulkanImage& image, - vk::ImageLayout new_layout) { - EXIT_IF(image.type != VulkanImageType::DepthStencil); - if (new_layout == vk::ImageLayout::eUndefined || - new_layout == vk::ImageLayout::ePreinitialized) { - EXIT("invalid destination depth/stencil image layout: %s\n", - VulkanToString(new_layout).c_str()); - } - if (image.layout == new_layout) { - return; - } - vk::ImageMemoryBarrier barrier {}; - barrier.sType = vk::StructureType::eImageMemoryBarrier; - barrier.srcAccessMask = vk::AccessFlagBits::eMemoryRead | vk::AccessFlagBits::eMemoryWrite; - barrier.dstAccessMask = vk::AccessFlagBits::eMemoryRead | vk::AccessFlagBits::eMemoryWrite; - barrier.oldLayout = image.layout; - barrier.newLayout = new_layout; - barrier.srcQueueFamilyIndex = VK_QUEUE_FAMILY_IGNORED; - barrier.dstQueueFamilyIndex = VK_QUEUE_FAMILY_IGNORED; - barrier.image = image.image; - barrier.subresourceRange = {ImageViewOps::DepthAspectMask(image.format), 0, - VK_REMAINING_MIP_LEVELS, 0, image.layers}; - const auto stages = static_cast( - vk::PipelineStageFlagBits::eAllGraphics | vk::PipelineStageFlagBits::eComputeShader | - vk::PipelineStageFlagBits::eTransfer); - vk_buffer.pipelineBarrier(stages, stages, vk::DependencyFlags {}, 0, nullptr, 0, nullptr, 1, - &barrier); - image.layout = new_layout; -} - -void GraphicsRenderDepthStencilBarrier(vk::CommandBuffer vk_buffer, VulkanImage& image) { - EXIT_IF(image.type != VulkanImageType::DepthStencil); - - auto& depth = static_cast(image); - if (depth.compressed) { - static std::atomic log_count {0}; - if (log_count.fetch_add(1, std::memory_order_relaxed) < 16) { - LOGF( - "DepthTexture: decompressing depth target for shader read format=%s extent=%ux%u\n", - VulkanToString(depth.format).c_str(), depth.extent.width, depth.extent.height); - } - depth.compressed = false; - } - - if (image.layout != vk::ImageLayout::eDepthStencilReadOnlyOptimal) { - vk::ImageMemoryBarrier image_memory_barrier {}; - image_memory_barrier.sType = vk::StructureType::eImageMemoryBarrier; - image_memory_barrier.pNext = nullptr; - image_memory_barrier.srcAccessMask = - vk::AccessFlagBits::eMemoryRead | vk::AccessFlagBits::eMemoryWrite; - image_memory_barrier.dstAccessMask = vk::AccessFlagBits::eShaderRead; - image_memory_barrier.oldLayout = image.layout; - image_memory_barrier.newLayout = vk::ImageLayout::eDepthStencilReadOnlyOptimal; - image_memory_barrier.srcQueueFamilyIndex = VK_QUEUE_FAMILY_IGNORED; - image_memory_barrier.dstQueueFamilyIndex = VK_QUEUE_FAMILY_IGNORED; - image_memory_barrier.image = image.image; - image_memory_barrier.subresourceRange.aspectMask = - ImageViewOps::DepthAspectMask(image.format); - image_memory_barrier.subresourceRange.baseMipLevel = 0; - image_memory_barrier.subresourceRange.levelCount = VK_REMAINING_MIP_LEVELS; - image_memory_barrier.subresourceRange.baseArrayLayer = 0; - image_memory_barrier.subresourceRange.layerCount = image.layers; - - vk_buffer.pipelineBarrier( - vk::PipelineStageFlagBits::eAllGraphics | vk::PipelineStageFlagBits::eComputeShader | - vk::PipelineStageFlagBits::eTransfer, - vk::PipelineStageFlagBits::eVertexShader | vk::PipelineStageFlagBits::eFragmentShader | - vk::PipelineStageFlagBits::eComputeShader, - vk::DependencyFlags {}, 0, nullptr, 0, nullptr, 1, &image_memory_barrier); - - image.layout = image_memory_barrier.newLayout; - } -} - -void GraphicsRenderTextureBarrier(CommandBuffer& buffer, uint64_t vaddr, uint64_t size) { - EXIT_IF(buffer.IsInvalid()); - - Common::LockGuard lock(GetRenderContext().GetMutex()); - - auto vk_buffer = buffer.Handle(); - - auto* native = GetRenderContext().GetTextureCache().FindRenderTargetByRange(buffer, vaddr, size); - if (native == nullptr) { - EXIT("render-target barrier range has no cached image\n"); - } - GraphicsRenderTextureBarrier(vk_buffer, *native); -} - -void GraphicsRenderDepthStencilBarrier(CommandBuffer& buffer, uint64_t vaddr, uint64_t size) { - EXIT_IF(buffer.IsInvalid()); - - Common::LockGuard lock(GetRenderContext().GetMutex()); - - auto vk_buffer = buffer.Handle(); - auto* native = GetRenderContext().GetTextureCache().FindDepthTargetByRange(buffer, vaddr, size); - if (native == nullptr) { - EXIT("depth-target barrier range has no cached image\n"); - } - GraphicsRenderDepthStencilBarrier(vk_buffer, *native); -} - -} // namespace Libs::Graphics diff --git a/src/graphics/host_gpu/renderer/renderTargetBarriers.h b/src/graphics/host_gpu/renderer/renderTargetBarriers.h deleted file mode 100644 index 3e4056f7..00000000 --- a/src/graphics/host_gpu/renderer/renderTargetBarriers.h +++ /dev/null @@ -1,19 +0,0 @@ -#ifndef EMULATOR_SRC_GRAPHICS_HOST_GPU_RENDERER_RENDERTARGETBARRIERS_H_ -#define EMULATOR_SRC_GRAPHICS_HOST_GPU_RENDERER_RENDERTARGETBARRIERS_H_ - -#include "graphics/host_gpu/vulkanCommon.h" - -namespace Libs::Graphics { - -struct VulkanImage; - -void GraphicsRenderTextureBarrier(vk::CommandBuffer vk_buffer, VulkanImage& image); -void GraphicsRenderColorImageBarrier(vk::CommandBuffer vk_buffer, VulkanImage& image, - vk::ImageLayout new_layout); -void GraphicsRenderDepthStencilImageBarrier(vk::CommandBuffer vk_buffer, VulkanImage& image, - vk::ImageLayout new_layout); -void GraphicsRenderDepthStencilBarrier(vk::CommandBuffer vk_buffer, VulkanImage& image); - -} // namespace Libs::Graphics - -#endif // EMULATOR_SRC_GRAPHICS_HOST_GPU_RENDERER_RENDERTARGETBARRIERS_H_ diff --git a/src/graphics/host_gpu/renderer/samplerCache.cpp b/src/graphics/host_gpu/renderer/samplerCache.cpp index 54cf4890..34d37fe6 100644 --- a/src/graphics/host_gpu/renderer/samplerCache.cpp +++ b/src/graphics/host_gpu/renderer/samplerCache.cpp @@ -7,6 +7,13 @@ namespace Libs::Graphics { +SamplerCache::~SamplerCache() { + for (const auto& [key, sampler]: m_samplers) { + (void)key; + m_graphics.device.destroySampler(sampler, nullptr); + } +} + vk::Sampler SamplerCache::GetSampler(const ShaderSamplerResource& r) { Common::LockGuard lock(m_mutex); diff --git a/src/graphics/host_gpu/renderer/samplerCache.h b/src/graphics/host_gpu/renderer/samplerCache.h index d7ad4d83..4121c29f 100644 --- a/src/graphics/host_gpu/renderer/samplerCache.h +++ b/src/graphics/host_gpu/renderer/samplerCache.h @@ -21,7 +21,7 @@ public: explicit SamplerCache(GraphicContext& graphics): m_graphics(graphics) { EXIT_NOT_IMPLEMENTED(!Common::Thread::IsMainThread()); } - ~SamplerCache() { KYTY_NOT_IMPLEMENTED; } + ~SamplerCache(); KYTY_CLASS_NO_COPY(SamplerCache); vk::Sampler GetSampler(const ShaderSamplerResource& r); diff --git a/src/graphics/host_gpu/renderer/shaderResourceBarrier.cpp b/src/graphics/host_gpu/renderer/shaderResourceBarrier.cpp index 1e824cf7..d65e7a73 100644 --- a/src/graphics/host_gpu/renderer/shaderResourceBarrier.cpp +++ b/src/graphics/host_gpu/renderer/shaderResourceBarrier.cpp @@ -36,36 +36,8 @@ VulkanMemoryBarrier MakeShaderWriteDependency() { return barrier; } -vk::ImageMemoryBarrier MakeStorageImageDependency(const VulkanImage& image, bool read, - bool written) { - EXIT_IF(image.image == nullptr || image.type == VulkanImageType::DepthStencil); - - vk::ImageMemoryBarrier barrier {}; - barrier.sType = vk::StructureType::eImageMemoryBarrier; - barrier.srcAccessMask = vk::AccessFlagBits::eMemoryWrite; - barrier.dstAccessMask = {}; - if (read) { - barrier.dstAccessMask |= vk::AccessFlagBits::eShaderRead; - } - if (written) { - barrier.dstAccessMask |= vk::AccessFlagBits::eShaderWrite; - } - barrier.oldLayout = image.layout; - barrier.newLayout = vk::ImageLayout::eGeneral; - barrier.srcQueueFamilyIndex = VK_QUEUE_FAMILY_IGNORED; - barrier.dstQueueFamilyIndex = VK_QUEUE_FAMILY_IGNORED; - barrier.image = image.image; - barrier.subresourceRange.aspectMask = vk::ImageAspectFlagBits::eColor; - barrier.subresourceRange.baseMipLevel = 0; - barrier.subresourceRange.levelCount = VK_REMAINING_MIP_LEVELS; - barrier.subresourceRange.baseArrayLayer = 0; - barrier.subresourceRange.layerCount = image.layers; - EXIT_IF(!barrier.dstAccessMask || barrier.subresourceRange.layerCount == 0); - return barrier; -} - -vk::BufferMemoryBarrier MakeGdsDependency(const VulkanBuffer& buffer) { - EXIT_IF(buffer.buffer == nullptr); +vk::BufferMemoryBarrier MakeGdsDependency(vk::Buffer buffer) { + EXIT_IF(buffer == nullptr); vk::BufferMemoryBarrier barrier {}; barrier.sType = vk::StructureType::eBufferMemoryBarrier; @@ -74,7 +46,7 @@ vk::BufferMemoryBarrier MakeGdsDependency(const VulkanBuffer& buffer) { barrier.dstAccessMask = vk::AccessFlagBits::eShaderRead | vk::AccessFlagBits::eShaderWrite; barrier.srcQueueFamilyIndex = VK_QUEUE_FAMILY_IGNORED; barrier.dstQueueFamilyIndex = VK_QUEUE_FAMILY_IGNORED; - barrier.buffer = buffer.buffer; + barrier.buffer = buffer; barrier.offset = 0; barrier.size = VK_WHOLE_SIZE; return barrier; diff --git a/src/graphics/host_gpu/renderer/shaderResourceBarrier.h b/src/graphics/host_gpu/renderer/shaderResourceBarrier.h index 6c57cc6c..3f7a5314 100644 --- a/src/graphics/host_gpu/renderer/shaderResourceBarrier.h +++ b/src/graphics/host_gpu/renderer/shaderResourceBarrier.h @@ -19,9 +19,7 @@ struct ShaderBufferWriteRange { vk::PipelineStageFlags ShaderPipelineStages(vk::ShaderStageFlags stages); VulkanMemoryBarrier MakeShaderWriteDependency(); -vk::ImageMemoryBarrier MakeStorageImageDependency(const VulkanImage& image, bool read, - bool written); -vk::BufferMemoryBarrier MakeGdsDependency(const VulkanBuffer& buffer); +vk::BufferMemoryBarrier MakeGdsDependency(vk::Buffer buffer); std::vector CollectShaderBufferWrites(const ShaderRecompiler::IR::Program& program, const ShaderRecompiler::IR::ResourceSnapshot& resources); diff --git a/src/graphics/host_gpu/renderer/shaders.cpp b/src/graphics/host_gpu/renderer/shaders.cpp index dd959456..2eb88dcc 100644 --- a/src/graphics/host_gpu/renderer/shaders.cpp +++ b/src/graphics/host_gpu/renderer/shaders.cpp @@ -385,7 +385,9 @@ static vk::BlendOp GetBlendOp(uint32_t op) { return vk::BlendOp::eAdd; } -static void CreateLayout(std::span set_layouts, uint32_t& set_layouts_num, +static void CreateLayout(DescriptorCache& descriptor_cache, + std::span set_layouts, + uint32_t& set_layouts_num, std::span push_constant_info, uint32_t& push_constant_info_num, const ShaderRecompiler::IR::Program& program, @@ -405,17 +407,16 @@ static void CreateLayout(std::span set_layouts, uint32_ if (need_descriptor) { EXIT_IF(bindings.descriptor_set != set_layouts_num); - set_layouts[set_layouts_num] = - GetRenderContext().GetDescriptorCache().GetDescriptorSetLayout(stage, program); + set_layouts[set_layouts_num] = descriptor_cache.GetDescriptorSetLayout(stage, program); set_layouts_num++; } } -static void ConfigureSubgroupSize(vk::ShaderStageFlagBits vk_stage, +static void ConfigureSubgroupSize(const GraphicContext& graphics, + vk::ShaderStageFlagBits vk_stage, const ShaderRecompiler::IR::Program& program, vk::PipelineShaderStageRequiredSubgroupSizeCreateInfo& required, vk::PipelineShaderStageCreateInfo& stage) { - const auto& graphics = GetRenderContext().GetGraphics(); const auto config = ConfigureShaderSubgroup(ShaderSubgroupCapabilities {graphics}, vk_stage, program); switch (config.mode) { @@ -455,7 +456,9 @@ static void ConfigureSubgroupSize(vk::ShaderStageFlagBits } // NOLINTNEXTLINE(readability-function-cognitive-complexity) -void CreatePipelineInternal(PipelineCache::GraphicsPipeline& pipeline, vk::RenderPass render_pass, +void CreatePipelineInternal(GraphicContext& graphics, DescriptorCache& descriptor_cache, + PipelineCache::GraphicsPipeline& pipeline, + const PipelineRenderingState& rendering, const ShaderVertexInputInfo& vs_input_info, std::span vs_shader, const ShaderPixelInputInfo* ps_input_info, @@ -463,11 +466,8 @@ void CreatePipelineInternal(PipelineCache::GraphicsPipeline& pipeline, vk::Rende const PipelineStaticParameters& static_params, uint32_t vs_hash0, uint32_t vs_crc32, uint32_t ps_hash0, uint32_t ps_crc32, bool ps_active) { - EXIT_IF(render_pass == nullptr); EXIT_IF(ps_active && ps_input_info == nullptr); - auto& graphics = GetRenderContext().GetGraphics(); - vk::ShaderModule vert_shader_module = nullptr; vk::ShaderModule frag_shader_module = nullptr; @@ -511,7 +511,8 @@ void CreatePipelineInternal(PipelineCache::GraphicsPipeline& pipeline, vk::Rende vert_shader_stage_info.pName = "main"; vert_shader_stage_info.pSpecializationInfo = nullptr; EXIT_IF(!vs_input_info.stage); - ConfigureSubgroupSize(vk::ShaderStageFlagBits::eVertex, *vs_input_info.stage.program, + ConfigureSubgroupSize(graphics, vk::ShaderStageFlagBits::eVertex, + *vs_input_info.stage.program, vert_subgroup_size, vert_shader_stage_info); vk::PipelineShaderStageCreateInfo frag_shader_stage_info {}; @@ -525,7 +526,8 @@ void CreatePipelineInternal(PipelineCache::GraphicsPipeline& pipeline, vk::Rende frag_shader_stage_info.pSpecializationInfo = nullptr; if (ps_active) { EXIT_IF(!ps_input_info->stage); - ConfigureSubgroupSize(vk::ShaderStageFlagBits::eFragment, *ps_input_info->stage.program, + ConfigureSubgroupSize(graphics, vk::ShaderStageFlagBits::eFragment, + *ps_input_info->stage.program, frag_subgroup_size, frag_shader_stage_info); } @@ -823,12 +825,14 @@ void CreatePipelineInternal(PipelineCache::GraphicsPipeline& pipeline, vk::Rende uint32_t push_constant_info_num = 0; EXIT_IF(!vs_input_info.stage); - CreateLayout(set_layouts, set_layouts_num, push_constant_info, push_constant_info_num, + CreateLayout(descriptor_cache, set_layouts, set_layouts_num, push_constant_info, + push_constant_info_num, *vs_input_info.stage.program, vk::ShaderStageFlagBits::eVertex, DescriptorCache::Stage::Vertex); if (ps_active) { EXIT_IF(!ps_input_info->stage); - CreateLayout(set_layouts, set_layouts_num, push_constant_info, push_constant_info_num, + CreateLayout(descriptor_cache, set_layouts, set_layouts_num, push_constant_info, + push_constant_info_num, *ps_input_info->stage.program, vk::ShaderStageFlagBits::eFragment, DescriptorCache::Stage::Pixel); } @@ -902,8 +906,14 @@ void CreatePipelineInternal(PipelineCache::GraphicsPipeline& pipeline, vk::Rende dynamic_state.pDynamicStates = dynamic_states; vk::GraphicsPipelineCreateInfo pipeline_info {}; + vk::PipelineRenderingCreateInfo rendering_info {}; + rendering_info.sType = vk::StructureType::ePipelineRenderingCreateInfo; + rendering_info.colorAttachmentCount = rendering.color_count; + rendering_info.pColorAttachmentFormats = rendering.color_formats.data(); + rendering_info.depthAttachmentFormat = rendering.depth_format; + rendering_info.stencilAttachmentFormat = rendering.stencil_format; pipeline_info.sType = vk::StructureType::eGraphicsPipelineCreateInfo; - pipeline_info.pNext = nullptr; + pipeline_info.pNext = &rendering_info; pipeline_info.flags = {}; pipeline_info.stageCount = shader_stage_count; pipeline_info.pStages = shader_stages; @@ -917,7 +927,7 @@ void CreatePipelineInternal(PipelineCache::GraphicsPipeline& pipeline, vk::Rende pipeline_info.pColorBlendState = &color_blending; pipeline_info.pDynamicState = &dynamic_state; pipeline_info.layout = pipeline.pipeline_layout; - pipeline_info.renderPass = render_pass; + pipeline_info.renderPass = nullptr; pipeline_info.subpass = 0; pipeline_info.basePipelineHandle = nullptr; pipeline_info.basePipelineIndex = -1; @@ -952,11 +962,10 @@ void CreatePipelineInternal(PipelineCache::GraphicsPipeline& pipeline, vk::Rende } // NOLINTNEXTLINE(readability-function-cognitive-complexity) -void CreatePipelineInternal(PipelineCache::ComputePipeline& pipeline, +void CreatePipelineInternal(GraphicContext& graphics, DescriptorCache& descriptor_cache, + PipelineCache::ComputePipeline& pipeline, const ShaderComputeInputInfo& input_info, std::span cs_shader) { - auto& graphics = GetRenderContext().GetGraphics(); - vk::ShaderModule comp_shader_module = nullptr; vk::ShaderModuleCreateInfo create_info {}; @@ -985,7 +994,8 @@ void CreatePipelineInternal(PipelineCache::ComputePipeline& pipeline, comp_shader_stage_info.pName = "main"; comp_shader_stage_info.pSpecializationInfo = nullptr; EXIT_IF(!input_info.stage); - ConfigureSubgroupSize(vk::ShaderStageFlagBits::eCompute, *input_info.stage.program, + ConfigureSubgroupSize(graphics, vk::ShaderStageFlagBits::eCompute, + *input_info.stage.program, comp_subgroup_size, comp_shader_stage_info); vk::DescriptorSetLayout set_layouts[1] = {}; @@ -995,7 +1005,8 @@ void CreatePipelineInternal(PipelineCache::ComputePipeline& pipeline, uint32_t push_constant_info_num = 0; EXIT_IF(!input_info.stage); - CreateLayout(set_layouts, set_layouts_num, push_constant_info, push_constant_info_num, + CreateLayout(descriptor_cache, set_layouts, set_layouts_num, push_constant_info, + push_constant_info_num, *input_info.stage.program, vk::ShaderStageFlagBits::eCompute, DescriptorCache::Stage::Compute); diff --git a/src/graphics/host_gpu/renderer/streamBuffer.cpp b/src/graphics/host_gpu/renderer/streamBuffer.cpp index 66f45bc9..0517bf04 100644 --- a/src/graphics/host_gpu/renderer/streamBuffer.cpp +++ b/src/graphics/host_gpu/renderer/streamBuffer.cpp @@ -1,95 +1,360 @@ #include "graphics/host_gpu/renderer/streamBuffer.h" #include "common/assert.h" -#include "common/logging/log.h" #include "common/profiler.h" #include "graphics/host_gpu/graphicContext.h" -#include "graphics/host_gpu/transfer.h" +#include "graphics/host_gpu/renderer/commandScheduler.h" #include "graphics/host_gpu/vma.h" -#include #include +#include +#include +#include namespace Libs::Graphics { namespace { -std::atomic g_stream_buffer_count {0}; -} // namespace -HostStreamBuffer::HostStreamBuffer(GraphicContext& graphics): m_graphics(graphics) {} +constexpr size_t WATCHES_INITIAL_RESERVE = 0x4000; +constexpr size_t WATCHES_RESERVE_CHUNK = 0x1000; -HostStreamBuffer::~HostStreamBuffer() { - Release(); +[[nodiscard]] VmaAllocationCreateFlags AllocationFlags(MemoryUsage usage) { + switch (usage) { + case MemoryUsage::Upload: + case MemoryUsage::Stream: + return VMA_ALLOCATION_CREATE_MAPPED_BIT | + VMA_ALLOCATION_CREATE_HOST_ACCESS_SEQUENTIAL_WRITE_BIT; + case MemoryUsage::Download: + return VMA_ALLOCATION_CREATE_MAPPED_BIT | VMA_ALLOCATION_CREATE_HOST_ACCESS_RANDOM_BIT; + case MemoryUsage::DeviceLocal: return {}; + } + return {}; } -void HostStreamBuffer::EnsureBuffer() { - if (m_buffer != nullptr) { - if (m_mapped == nullptr) { - EXIT("host stream buffer lost its mapped storage\n"); - } - return; +[[nodiscard]] VmaMemoryUsage AllocationUsage(MemoryUsage usage) { + switch (usage) { + case MemoryUsage::DeviceLocal: + case MemoryUsage::Stream: return VMA_MEMORY_USAGE_AUTO_PREFER_DEVICE; + case MemoryUsage::Upload: + case MemoryUsage::Download: return VMA_MEMORY_USAGE_AUTO_PREFER_HOST; } - KYTY_PROFILER_BLOCK("HostStreamBuffer::create"); - m_buffer = std::make_unique(); - m_buffer->usage = vk::BufferUsageFlagBits::eStorageBuffer | - vk::BufferUsageFlagBits::eVertexBuffer | - vk::BufferUsageFlagBits::eIndexBuffer | - vk::BufferUsageFlagBits::eTransferSrc | vk::BufferUsageFlagBits::eTransferDst; - m_buffer->memory.property = vk::MemoryPropertyFlagBits::eHostVisible | - vk::MemoryPropertyFlagBits::eHostCoherent | - vk::MemoryPropertyFlagBits::eHostCached; - m_graphics.CreateBuffer(CAPACITY, *m_buffer); - m_graphics.MapMemory(m_buffer->memory, m_mapped); - const auto count = g_stream_buffer_count.fetch_add(1, std::memory_order_relaxed) + 1; - LOGF("HostStreamBuffer: created serial=%" PRIu64 " capacity=%" PRIu64 "\n", count, CAPACITY); + return VMA_MEMORY_USAGE_AUTO_PREFER_DEVICE; } -bool HostStreamBuffer::Copy(const void* src, uint64_t size, uint64_t alignment, - VulkanBuffer*& out_buffer, uint64_t& out_offset, uint64_t& out_range) { - KYTY_PROFILER_FUNCTION(); - if (src == nullptr) { - EXIT("invalid host stream upload\n"); - } - std::lock_guard lock(m_mutex); - if (size == 0) { - return false; - } +[[nodiscard]] bool AlignUp(uint64_t value, uint64_t alignment, uint64_t& result) { if (alignment == 0) { - alignment = 1; + result = value; + return true; } - if ((alignment & (alignment - 1)) != 0 || m_offset > UINT64_MAX - (alignment - 1)) { + const auto remainder = value % alignment; + if (remainder == 0) { + result = value; + return true; + } + const auto increment = alignment - remainder; + if (value > std::numeric_limits::max() - increment) { return false; } - const auto offset = (m_offset + alignment - 1) & ~(alignment - 1); - if (offset > CAPACITY || size > CAPACITY - offset) { - return false; - } - EnsureBuffer(); - std::memcpy(static_cast(m_mapped) + offset, src, static_cast(size)); - m_offset = offset + size; - out_buffer = m_buffer.get(); - out_offset = offset; - out_range = size; + result = value + increment; return true; } -void HostStreamBuffer::Reset() noexcept { - std::lock_guard lock(m_mutex); - m_offset = 0; +} // namespace + +Buffer::Buffer(GraphicContext& graphics, CommandScheduler& scheduler, MemoryUsage usage, + uint64_t cpu_address, vk::BufferUsageFlags flags, uint64_t size) + : m_graphics(&graphics), m_scheduler(&scheduler), m_usage(usage), m_cpu_address(cpu_address), + m_size(size), m_buffer(std::make_unique()) { + KYTY_PROFILER_FUNCTION(); + EXIT_IF(graphics.allocator == nullptr || size == 0); + + vk::BufferCreateInfo buffer_info {}; + buffer_info.size = size; + buffer_info.usage = flags; + buffer_info.sharingMode = vk::SharingMode::eExclusive; + + VmaAllocationCreateInfo allocation_info {}; + allocation_info.flags = VMA_ALLOCATION_CREATE_WITHIN_BUDGET_BIT | AllocationFlags(usage); + allocation_info.usage = AllocationUsage(usage); + allocation_info.preferredFlags = usage == MemoryUsage::DeviceLocal + ? VkMemoryPropertyFlags {} + : VK_MEMORY_PROPERTY_HOST_COHERENT_BIT; + + VmaAllocationInfo allocation_result {}; + VkBuffer native_buffer = VK_NULL_HANDLE; + const auto result = static_cast(vmaCreateBuffer( + graphics.allocator, static_cast(buffer_info), &allocation_info, + &native_buffer, &m_buffer->memory.allocation, &allocation_result)); + if (result != vk::Result::eSuccess) { + graphics.LogMemoryBudget(); + } + EXIT_NOT_IMPLEMENTED(result != vk::Result::eSuccess); + + m_buffer->buffer = native_buffer; + m_buffer->usage = flags; + m_buffer->buffer_size = size; + m_buffer->memory.allocation_info = allocation_result; + m_buffer->memory.memory = allocation_result.deviceMemory; + m_buffer->memory.offset = allocation_result.offset; + m_buffer->memory.type = allocation_result.memoryType; + m_buffer->memory.unique_id = VulkanNextMemoryUniqueId(); + graphics.device.getBufferMemoryRequirements(m_buffer->buffer, &m_buffer->memory.requirements); + + VkMemoryPropertyFlags properties = 0; + vmaGetAllocationMemoryProperties(graphics.allocator, m_buffer->memory.allocation, &properties); + m_buffer->memory.property = vk::MemoryPropertyFlags(properties); + m_is_coherent = (properties & VK_MEMORY_PROPERTY_HOST_COHERENT_BIT) != 0; + if (allocation_result.pMappedData != nullptr) { + m_mapped = {static_cast(allocation_result.pMappedData), + static_cast(size)}; + } + VulkanTrackAllocation(m_buffer->memory); } -void HostStreamBuffer::Release() noexcept { - std::lock_guard lock(m_mutex); - if (m_buffer != nullptr) { - if (m_mapped == nullptr) { - EXIT("invalid host stream buffer release state\n"); - } - m_graphics.UnmapMemory(m_buffer->memory); - m_graphics.DeleteBuffer(*m_buffer); - m_buffer.reset(); +Buffer::~Buffer() { + if (m_buffer->buffer != nullptr) { + VulkanUntrackAllocation(m_buffer->memory); + vmaDestroyBuffer(m_graphics->allocator, m_buffer->buffer, m_buffer->memory.allocation); } - m_mapped = nullptr; - m_offset = 0; +} + +vk::Buffer Buffer::Handle() const noexcept { + return m_buffer->buffer; +} + +bool Buffer::IsInBounds(uint64_t address, uint64_t size) const noexcept { + return address >= m_cpu_address && size <= m_size && address - m_cpu_address <= m_size - size; +} + +void Buffer::Write(uint64_t offset, const void* source, uint64_t size) { + EXIT_IF(source == nullptr || m_mapped.empty() || offset > m_size || size > m_size - offset); + std::memcpy(m_mapped.data() + offset, source, static_cast(size)); + Flush(offset, size); +} + +void Buffer::Flush(uint64_t offset, uint64_t size) { + EXIT_IF(m_mapped.empty() || offset > m_size || size > m_size - offset); + if (!m_is_coherent && size != 0) { + const auto result = vmaFlushAllocation(m_graphics->allocator, m_buffer->memory.allocation, + offset, size); + EXIT_NOT_IMPLEMENTED(static_cast(result) != vk::Result::eSuccess); + } +} + +vk::BufferMemoryBarrier Buffer::Barrier(uint64_t offset, uint64_t size, vk::AccessFlags source, + vk::AccessFlags destination) const { + if (Handle() == nullptr || size == 0 || offset > m_size || size > m_size - offset) { + EXIT("Buffer: invalid DMA barrier, handle=%p offset=0x%016" PRIx64 + " size=0x%016" PRIx64 " capacity=0x%016" PRIx64 "\n", + static_cast(Handle()), offset, size, m_size); + } + vk::BufferMemoryBarrier barrier {}; + barrier.sType = vk::StructureType::eBufferMemoryBarrier; + barrier.srcAccessMask = source; + barrier.dstAccessMask = destination; + barrier.srcQueueFamilyIndex = VK_QUEUE_FAMILY_IGNORED; + barrier.dstQueueFamilyIndex = VK_QUEUE_FAMILY_IGNORED; + barrier.buffer = Handle(); + barrier.offset = offset; + barrier.size = size; + return barrier; +} + +void Buffer::CopyFrom(CommandBuffer& command, const Buffer& source, uint64_t source_offset, + uint64_t destination_offset, uint64_t size, vk::AccessFlags source_before, + vk::AccessFlags destination_before, vk::AccessFlags source_after, + vk::AccessFlags destination_after) { + if (size == 0 || source_offset > source.m_size || size > source.m_size - source_offset || + destination_offset > m_size || size > m_size - destination_offset) { + EXIT("Buffer: invalid copy range\n"); + } + if (source.Handle() == Handle() && source_offset < destination_offset + size && + destination_offset < source_offset + size) { + EXIT("Buffer: overlapping self-copy\n"); + } + command.EndRendering(); + const vk::BufferMemoryBarrier before[] = { + source.Barrier(source_offset, size, source_before, vk::AccessFlagBits::eTransferRead), + Barrier(destination_offset, size, destination_before, + vk::AccessFlagBits::eTransferWrite), + }; + const auto host_access = vk::AccessFlagBits::eHostRead | vk::AccessFlagBits::eHostWrite; + auto before_stage = vk::PipelineStageFlags {vk::PipelineStageFlagBits::eAllCommands}; + if (static_cast((source_before | destination_before) & host_access)) { + before_stage |= vk::PipelineStageFlagBits::eHost; + } + const auto native = command.Handle(); + native.pipelineBarrier(before_stage, vk::PipelineStageFlagBits::eTransfer, + vk::DependencyFlagBits::eByRegion, 0, nullptr, 2, before, 0, nullptr); + const vk::BufferCopy copy {source_offset, destination_offset, size}; + native.copyBuffer(source.Handle(), Handle(), 1, ©); + const vk::BufferMemoryBarrier after[] = { + source.Barrier(source_offset, size, vk::AccessFlagBits::eTransferRead, source_after), + Barrier(destination_offset, size, vk::AccessFlagBits::eTransferWrite, destination_after), + }; + auto after_stage = vk::PipelineStageFlags {vk::PipelineStageFlagBits::eAllCommands}; + if (static_cast((source_after | destination_after) & host_access)) { + after_stage |= vk::PipelineStageFlagBits::eHost; + } + native.pipelineBarrier(vk::PipelineStageFlagBits::eTransfer, after_stage, + vk::DependencyFlagBits::eByRegion, 0, nullptr, 2, after, 0, nullptr); +} + +void Buffer::Fill(uint64_t offset, uint64_t size, uint32_t value) { + if (((offset | size) & 3u) != 0) { + EXIT("Buffer: fill range must be dword aligned\n"); + } + auto& command = Scheduler().Current(); + command.EndRendering(); + const auto before = + Barrier(offset, size, vk::AccessFlagBits::eMemoryRead | vk::AccessFlagBits::eMemoryWrite, + vk::AccessFlagBits::eTransferWrite); + const auto native = command.Handle(); + native.pipelineBarrier(vk::PipelineStageFlagBits::eAllCommands, + vk::PipelineStageFlagBits::eTransfer, vk::DependencyFlagBits::eByRegion, + 0, nullptr, 1, &before, 0, nullptr); + native.fillBuffer(Handle(), offset, size, value); + const auto after = + Barrier(offset, size, vk::AccessFlagBits::eTransferWrite, + vk::AccessFlagBits::eMemoryRead | vk::AccessFlagBits::eMemoryWrite); + native.pipelineBarrier(vk::PipelineStageFlagBits::eTransfer, + vk::PipelineStageFlagBits::eAllCommands, + vk::DependencyFlagBits::eByRegion, 0, nullptr, 1, &after, 0, nullptr); +} + +StreamBuffer::StreamBuffer(GraphicContext& graphics, CommandScheduler& scheduler, MemoryUsage usage, + uint64_t size) + : Buffer(graphics, scheduler, usage, 0, AllFlags, size) { + ReserveWatches(m_current_watches, WATCHES_INITIAL_RESERVE); + ReserveWatches(m_previous_watches, WATCHES_INITIAL_RESERVE); +} + +bool StreamBuffer::NormalizeReservation(bool coherent, uint64_t atom, uint64_t& size, + uint64_t& alignment) { + if (coherent) { + return true; + } + if (!AlignUp(size, atom, size)) { + return false; + } + const auto divisor = std::gcd(alignment, atom); + if (alignment != 0 && alignment / divisor > UINT64_MAX / atom) { + return false; + } + alignment = alignment == 0 ? atom : alignment / divisor * atom; + return true; +} + +std::pair StreamBuffer::Map(uint64_t size, uint64_t alignment, + bool allow_wait) { + if (Mapped().empty()) { + return {nullptr, 0}; + } + uint64_t mapped_size = size; + const auto atom = Graphics().physical_device_properties.limits.nonCoherentAtomSize; + if (!NormalizeReservation(IsCoherent(), atom, mapped_size, alignment)) { + return {nullptr, 0}; + } + if (mapped_size > Size()) { + return {nullptr, 0}; + } + + uint64_t aligned_offset = 0; + if (!AlignUp(m_offset, alignment, aligned_offset)) { + return {nullptr, 0}; + } + + const bool wrap = aligned_offset > Size() - mapped_size; + if (wrap) { + aligned_offset = 0; + } + + auto wait_cursor = wrap ? size_t {0} : m_wait_cursor; + auto wait_bound = wrap ? uint64_t {0} : m_wait_bound; + auto invalidation_mark = + wrap ? std::optional {m_current_watch_cursor} : m_invalidation_mark; + auto& pending_watches = wrap ? m_current_watches : m_previous_watches; + if (!WaitPendingOperations(pending_watches, invalidation_mark, aligned_offset + mapped_size, + allow_wait, wait_cursor, wait_bound)) { + return {nullptr, 0}; + } + + if (wrap) { + m_invalidation_mark = invalidation_mark; + m_current_watch_cursor = 0; + std::swap(m_previous_watches, m_current_watches); + } + m_wait_cursor = wait_cursor; + m_wait_bound = wait_bound; + m_offset = aligned_offset; + m_mapped_size = mapped_size; + return {Mapped().data() + m_offset, m_offset}; +} + +void StreamBuffer::Commit() { + if (!IsCoherent() && Usage() != MemoryUsage::Download && m_mapped_size != 0) { + const auto result = vmaFlushAllocation( + Graphics().allocator, NativeBuffer().memory.allocation, m_offset, m_mapped_size); + EXIT_NOT_IMPLEMENTED(static_cast(result) != vk::Result::eSuccess); + } + + m_offset += m_mapped_size; + const auto tick = Scheduler().CurrentTick(); + if (m_current_watch_cursor != 0 && m_current_watches[m_current_watch_cursor - 1].tick == tick) { + m_current_watches[m_current_watch_cursor - 1].upper_bound = m_offset; + return; + } + if (m_current_watch_cursor + 1 >= m_current_watches.size()) { + ReserveWatches(m_current_watches, WATCHES_RESERVE_CHUNK); + } + auto& watch = m_current_watches[m_current_watch_cursor++]; + watch.upper_bound = m_offset; + watch.tick = tick; +} + +void StreamBuffer::Invalidate(uint64_t offset, uint64_t size) { + EXIT_IF(Usage() != MemoryUsage::Download || offset > Size() || size > Size() - offset); + if (IsCoherent() || size == 0) { + return; + } + const auto result = vmaInvalidateAllocation(Graphics().allocator, + NativeBuffer().memory.allocation, offset, size); + EXIT_NOT_IMPLEMENTED(static_cast(result) != vk::Result::eSuccess); +} + +uint64_t StreamBuffer::Copy(const void* source, uint64_t size, uint64_t alignment) { + EXIT_IF(source == nullptr); + const auto [data, offset] = Map(size, alignment); + EXIT_IF(data == nullptr); + std::memcpy(data, source, static_cast(size)); + Commit(); + return offset; +} + +void StreamBuffer::ReserveWatches(std::vector& watches, size_t grow_size) { + watches.resize(watches.size() + grow_size); +} + +bool StreamBuffer::WaitPendingOperations(const std::vector& watches, + std::optional invalidation_mark, + uint64_t requested_upper_bound, bool allow_wait, + size_t& wait_cursor, uint64_t& wait_bound) { + if (!invalidation_mark.has_value()) { + return true; + } + while (requested_upper_bound > wait_bound && wait_cursor < *invalidation_mark) { + const auto& watch = watches[wait_cursor]; + if (!Scheduler().IsFree(watch.tick) && !allow_wait) { + return false; + } + Scheduler().Wait(watch.tick); + if (Usage() == MemoryUsage::Download) { + Scheduler().WaitPriorityOperations(watch.tick); + } + wait_bound = watch.upper_bound; + ++wait_cursor; + } + return true; } } // namespace Libs::Graphics diff --git a/src/graphics/host_gpu/renderer/streamBuffer.h b/src/graphics/host_gpu/renderer/streamBuffer.h index 9bda8061..00de6162 100644 --- a/src/graphics/host_gpu/renderer/streamBuffer.h +++ b/src/graphics/host_gpu/renderer/streamBuffer.h @@ -3,43 +3,126 @@ #include "common/abi.h" #include "common/common.h" +#include "graphics/host_gpu/vulkanCommon.h" #include #include -#include +#include +#include +#include +#include namespace Libs::Graphics { +class CommandBuffer; +class CommandScheduler; +struct StreamBufferTestAccess; struct GraphicContext; struct VulkanBuffer; -// Host-only mapped stream mechanism. BufferCache owns the upload policy/API; one backend remains -// attached to each CommandBuffer so Reset is provably after that command's fence. Replacing this -// backend with scheduler watches will not change descriptor/render call sites. -class HostStreamBuffer final { -public: - explicit HostStreamBuffer(GraphicContext& graphics); - ~HostStreamBuffer(); - KYTY_CLASS_NO_COPY(HostStreamBuffer); +enum class MemoryUsage : uint8_t { + DeviceLocal, + Upload, + Download, + Stream, +}; - [[nodiscard]] bool Copy(const void* src, uint64_t size, uint64_t alignment, - VulkanBuffer*& out_buffer, uint64_t& out_offset, uint64_t& out_range); - void Reset() noexcept; - void Release() noexcept; +inline constexpr vk::BufferUsageFlags ReadFlags = + vk::BufferUsageFlagBits::eTransferSrc | vk::BufferUsageFlagBits::eUniformBuffer | + vk::BufferUsageFlagBits::eIndexBuffer | vk::BufferUsageFlagBits::eVertexBuffer | + vk::BufferUsageFlagBits::eIndirectBuffer; + +inline constexpr vk::BufferUsageFlags AllFlags = + ReadFlags | vk::BufferUsageFlagBits::eTransferDst | vk::BufferUsageFlagBits::eStorageBuffer; + +class Buffer { +public: + Buffer(GraphicContext& graphics, CommandScheduler& scheduler, MemoryUsage usage, + uint64_t cpu_address, vk::BufferUsageFlags flags, uint64_t size); + ~Buffer(); + KYTY_CLASS_NO_COPY(Buffer); + + [[nodiscard]] vk::Buffer Handle() const noexcept; + [[nodiscard]] uint64_t Size() const noexcept { return m_size; } + [[nodiscard]] std::span Mapped() const noexcept { return m_mapped; } + [[nodiscard]] bool IsCoherent() const noexcept { return m_is_coherent; } + [[nodiscard]] MemoryUsage Usage() const noexcept { return m_usage; } + [[nodiscard]] uint64_t CpuAddress() const noexcept { return m_cpu_address; } + [[nodiscard]] uint64_t Offset(uint64_t address) const noexcept { + return address - m_cpu_address; + } + [[nodiscard]] bool IsInBounds(uint64_t address, uint64_t size) const noexcept; + void Write(uint64_t offset, const void* source, uint64_t size); + void Flush(uint64_t offset, uint64_t size); + void CopyFrom( + CommandBuffer& command, const Buffer& source, uint64_t source_offset, + uint64_t destination_offset, uint64_t size, + vk::AccessFlags source_before = vk::AccessFlagBits::eMemoryWrite, + vk::AccessFlags destination_before = + vk::AccessFlagBits::eMemoryRead | vk::AccessFlagBits::eMemoryWrite, + vk::AccessFlags source_after = + vk::AccessFlagBits::eMemoryRead | vk::AccessFlagBits::eMemoryWrite, + vk::AccessFlags destination_after = + vk::AccessFlagBits::eMemoryRead | vk::AccessFlagBits::eMemoryWrite); + void Fill(uint64_t offset, uint64_t size, uint32_t value); + +protected: + [[nodiscard]] GraphicContext& Graphics() const noexcept { return *m_graphics; } + [[nodiscard]] CommandScheduler& Scheduler() const noexcept { return *m_scheduler; } + [[nodiscard]] VulkanBuffer& NativeBuffer() noexcept { return *m_buffer; } private: - void EnsureBuffer(); + [[nodiscard]] vk::BufferMemoryBarrier Barrier(uint64_t offset, uint64_t size, + vk::AccessFlags source, + vk::AccessFlags destination) const; - // Interim fence-scoped backend: one command processor's eight live command buffers reserve up - // to 128 MiB lazily. Additional active processors scale that aggregate until this backend is - // replaced by a single 64 MiB scheduler-watched ring; the BufferCache seam stays fixed. - static constexpr uint64_t CAPACITY = 16ull * 1024ull * 1024ull; - - mutable std::mutex m_mutex; - GraphicContext& m_graphics; + GraphicContext* m_graphics = nullptr; + CommandScheduler* m_scheduler = nullptr; + MemoryUsage m_usage = MemoryUsage::DeviceLocal; + uint64_t m_cpu_address = 0; + uint64_t m_size = 0; std::unique_ptr m_buffer; - void* m_mapped = nullptr; - uint64_t m_offset = 0; + std::span m_mapped; + bool m_is_coherent = false; +}; + +class StreamBuffer final: public Buffer { +public: + StreamBuffer(GraphicContext& graphics, CommandScheduler& scheduler, MemoryUsage usage, + uint64_t size); + + [[nodiscard]] std::pair Map(uint64_t size, uint64_t alignment = 0, + bool allow_wait = true); + void Commit(); + // Download mappings become visible to the CPU only after their GPU completion tick is free. + // Call this from the scheduler's deferred completion operation before reading Mapped(). + void Invalidate(uint64_t offset, uint64_t size); + [[nodiscard]] uint64_t Copy(const void* source, uint64_t size, uint64_t alignment = 0); + +private: + friend struct StreamBufferTestAccess; + + struct Watch { + uint64_t tick = 0; + uint64_t upper_bound = 0; + }; + + void ReserveWatches(std::vector& watches, size_t grow_size); + [[nodiscard]] static bool NormalizeReservation(bool coherent, uint64_t atom, uint64_t& size, + uint64_t& alignment); + [[nodiscard]] bool WaitPendingOperations(const std::vector& watches, + std::optional invalidation_mark, + uint64_t requested_upper_bound, bool allow_wait, + size_t& wait_cursor, uint64_t& wait_bound); + + uint64_t m_offset = 0; + uint64_t m_mapped_size = 0; + std::vector m_current_watches; + size_t m_current_watch_cursor = 0; + std::optional m_invalidation_mark; + std::vector m_previous_watches; + size_t m_wait_cursor = 0; + uint64_t m_wait_bound = 0; }; } // namespace Libs::Graphics diff --git a/src/graphics/host_gpu/renderer/sync.cpp b/src/graphics/host_gpu/renderer/sync.cpp index 90bd46a3..b0907093 100644 --- a/src/graphics/host_gpu/renderer/sync.cpp +++ b/src/graphics/host_gpu/renderer/sync.cpp @@ -5,16 +5,16 @@ #include "common/logging/log.h" #include "common/threads.h" #include "graphics/host_gpu/graphicContext.h" -#include "graphics/host_gpu/objects/label.h" #include "graphics/host_gpu/renderer/bufferCache.h" #include "graphics/host_gpu/renderer/render.h" #include "graphics/host_gpu/renderer/renderContext.h" -#include "graphics/presentation/displayBuffer.h" +#include "graphics/presentation/videoOut.h" #include "kernel/eventQueue.h" #include "kernel/pthread.h" #include "libs/errno.h" #include +#include #include #include @@ -57,19 +57,6 @@ uint64_t ReadReferenceClock() { return value; } -static void SubmitLabel(CommandBuffer& buffer, LabelCallback callback_1 = nullptr, - LabelCallback callback_2 = nullptr, const uint64_t* args = nullptr) { - auto* label = LabelCreate(callback_1, callback_2, args); - LabelSet(buffer, *label); - LabelDelete(*label); -} - -static bool CompleteDisplayBufferFlip(const uint64_t* args) { - EXIT_IF(args == nullptr); - Presentation::DisplayBufferCompleteFlipFromGpu(args[0]); - return true; -} - enum class EndOfPipeCompletion { None, Interrupt, Flip, FlipAndInterrupt }; struct EndOfPipeSignal { @@ -86,17 +73,6 @@ struct EndOfPipeSignal { enum class EndOfPipeWriteSize : uint32_t { Dword = 4, Qword = 8 }; enum class EndOfPipeWriteAction { Write, WriteBack, Interrupt, InterruptWriteBack }; -static bool TriggerEopEventCallback(const uint64_t* args) { - EXIT_IF(args == nullptr); - GetRenderContext().TriggerEopEvent(static_cast(args[0])); - return true; -} - -static bool TriggerDefaultEopEventCallback(const uint64_t* /*args*/) { - GetRenderContext().TriggerEopEvent(0); - return true; -} - static void ValidateEndOfPipeSignal(const EndOfPipeSignal& signal) { if (signal.destination.has_value()) { EXIT_IF(*signal.destination == 0); @@ -111,19 +87,33 @@ static void RecordEndOfPipeSignal(const EndOfPipeSignal& signal) { signal.debug_args[0], signal.debug_args[1], signal.debug_args[2], signal.debug_args[3], signal.debug_data); - const uint64_t args[LABEL_ARGS_MAX] = {signal.completion_data}; + auto& renderer = signal.buffer->GetContext(); + auto& scheduler = renderer.GetCommandScheduler(); + if (signal.completion != EndOfPipeCompletion::None) { + EXIT_IF(!scheduler.Active() || signal.buffer != &scheduler.Current()); + } switch (signal.completion) { case EndOfPipeCompletion::None: return; - case EndOfPipeCompletion::Interrupt: - SubmitLabel(*signal.buffer, nullptr, TriggerEopEventCallback, args); + case EndOfPipeCompletion::Interrupt: { + const auto context_id = static_cast(signal.completion_data); + scheduler.DeferPriorityOperation( + [&renderer, context_id] { renderer.TriggerEopEvent(context_id); }); return; - case EndOfPipeCompletion::Flip: - SubmitLabel(*signal.buffer, CompleteDisplayBufferFlip, nullptr, args); + } + case EndOfPipeCompletion::Flip: { + const auto request_id = signal.completion_data; + scheduler.DeferPriorityOperation( + [&renderer, request_id] { renderer.GetVideoOut().CompleteFlip(request_id); }); return; - case EndOfPipeCompletion::FlipAndInterrupt: - SubmitLabel(*signal.buffer, CompleteDisplayBufferFlip, TriggerDefaultEopEventCallback, - args); + } + case EndOfPipeCompletion::FlipAndInterrupt: { + const auto request_id = signal.completion_data; + scheduler.DeferPriorityOperation([&renderer, request_id] { + renderer.GetVideoOut().CompleteFlip(request_id); + renderer.TriggerEopEvent(0); + }); return; + } } } @@ -172,10 +162,6 @@ void TriggerAgcUserInterrupt() { EXIT_NOT_IMPLEMENTED(result != OK && result != LibKernel::KERNEL_ERROR_ENOENT); } -void TriggerEopEvent(uint32_t context_id) { - GetRenderContext().TriggerEopEvent(context_id); -} - void WriteAtEndOfPipe32(uint64_t submit_id, CommandBuffer& buffer, uint32_t* dst_gpu_addr, uint32_t value) { RecordEndOfPipeWrite(submit_id, buffer, reinterpret_cast(dst_gpu_addr), value, @@ -261,11 +247,12 @@ void WriteAtEndOfPipeWithInterrupt32(uint64_t submit_id, CommandBuffer& buffer, EndOfPipeWriteSize::Dword, EndOfPipeWriteAction::Interrupt, context_id); } -uint64_t PrepareDisplayBufferFlip(CommandBuffer& buffer, int handle, int index, int flip_mode, - int64_t flip_arg) { +uint64_t PrepareVideoOutFlip(CommandBuffer& buffer, int handle, int index, int flip_mode, + int64_t flip_arg) { for (;;) { uint64_t request_id = 0; - const auto result = Presentation::DisplayBufferSubmitFlipFromGpu( + auto& video_out = buffer.GetContext().GetVideoOut(); + const auto result = video_out.SubmitFlipFromGpu( buffer, handle, index, flip_mode, flip_arg, request_id); if (result == OK) { EXIT_IF(request_id == 0); @@ -276,7 +263,7 @@ uint64_t PrepareDisplayBufferFlip(CommandBuffer& buffer, int handle, int index, "\n", result, handle, index, flip_mode, flip_arg); } - Presentation::DisplayBufferWaitForFlipQueueSlot(); + video_out.WaitForSubmitSlot(); } } @@ -331,9 +318,11 @@ void WriteAtEndOfPipeOnlyFlip(uint64_t submit_id, CommandBuffer& buffer, int han void TriggerEopEventAtEndOfPipe(CommandBuffer& buffer, uint32_t context_id) { ValidateEndOfPipeSignal({.buffer = &buffer}); - - uint64_t args[LABEL_ARGS_MAX] = {static_cast(context_id)}; - SubmitLabel(buffer, nullptr, TriggerEopEventCallback, args); + auto& renderer = buffer.GetContext(); + auto& scheduler = renderer.GetCommandScheduler(); + EXIT_IF(!scheduler.Active() || &buffer != &scheduler.Current()); + scheduler.DeferPriorityOperation( + [&renderer, context_id] { renderer.TriggerEopEvent(context_id); }); } static void EopEventResetFunc(LibKernel::EventQueue::KernelEqueueEvent* event) { @@ -349,7 +338,9 @@ static void EopEventDeleteFunc(LibKernel::EventQueue::KernelEqueue eq, EXIT_NOT_IMPLEMENTED(event->event.filter != LibKernel::EventQueue::KERNEL_EVFILT_GRAPHICS); if (event->event.ident == GRAPHICS_EVENT_QUEUED_GRAPHICS_INTERRUPT || event->event.ident == GRAPHICS_EVENT_EOP) { - GetRenderContext().DeleteEopEq(eq, static_cast(event->event.ident)); + auto* renderer = static_cast(event->filter.data); + EXIT_IF(renderer == nullptr); + renderer->DeleteEopEq(eq, static_cast(event->event.ident)); } } @@ -368,7 +359,8 @@ static void EopEventTriggerFunc(LibKernel::EventQueue::KernelEqueueEvent* event, } } -int AddEqEvent(LibKernel::EventQueue::KernelEqueue eq, int id, void* udata) { +int AddEqEvent(RenderContext& renderer, LibKernel::EventQueue::KernelEqueue eq, int id, + void* udata) { LibKernel::EventQueue::KernelEqueueEvent event; event.triggered = false; event.event.ident = static_cast(id); @@ -379,13 +371,13 @@ int AddEqEvent(LibKernel::EventQueue::KernelEqueue eq, int id, void* udata) { event.filter.delete_event_func = EopEventDeleteFunc; event.filter.reset_func = EopEventResetFunc; event.filter.trigger_func = EopEventTriggerFunc; - event.filter.data = nullptr; + event.filter.data = &renderer; int result = LibKernel::EventQueue::KernelAddEvent(eq, event); if (result == 0 && (id == GRAPHICS_EVENT_QUEUED_GRAPHICS_INTERRUPT || id == GRAPHICS_EVENT_EOP)) { - GetRenderContext().AddEopEq(eq, id); + renderer.AddEopEq(eq, id); } return result; @@ -398,12 +390,12 @@ int DeleteEqEvent(LibKernel::EventQueue::KernelEqueue eq, int id) { return result; } -void ReadGds(uint32_t* dst, uint32_t dw_offset, uint32_t dw_size) { - GetRenderContext().GetGdsBuffer().Read(dst, dw_offset, dw_size); -} - -void DeleteBuffers() { - GetRenderContext().GetBufferCache().ResetNullBuffer(); +void ReadGds(Buffer& gds, uint32_t* dst, uint32_t dw_offset, uint32_t dw_size) { + const auto offset = uint64_t {dw_offset} * sizeof(uint32_t); + const auto size = uint64_t {dw_size} * sizeof(uint32_t); + EXIT_IF(dst == nullptr || offset > gds.Size() || size > gds.Size() - offset || + gds.Mapped().empty()); + std::memcpy(dst, gds.Mapped().data() + offset, static_cast(size)); } } // namespace Libs::Graphics::Sync diff --git a/src/graphics/host_gpu/renderer/sync.h b/src/graphics/host_gpu/renderer/sync.h index 29068e9a..1271a5ac 100644 --- a/src/graphics/host_gpu/renderer/sync.h +++ b/src/graphics/host_gpu/renderer/sync.h @@ -8,6 +8,8 @@ namespace Libs::Graphics { class CommandBuffer; +class Buffer; +class RenderContext; namespace Sync { @@ -16,7 +18,6 @@ namespace Sync { [[nodiscard]] uint64_t ReadReferenceClock(); void TriggerAgcUserInterrupt(); -void TriggerEopEvent(uint32_t context_id); void TriggerEopEventAtEndOfPipe(CommandBuffer& buffer, uint32_t context_id); void WriteAtEndOfPipe32(uint64_t submit_id, CommandBuffer& buffer, uint32_t* dst_gpu_addr, @@ -46,8 +47,8 @@ void WriteAtEndOfPipeWithInterruptWriteBack64(uint64_t submit_id, CommandBuffer& uint64_t* dst_gpu_addr, uint64_t value, uint32_t context_id = 0); -[[nodiscard]] uint64_t PrepareDisplayBufferFlip(CommandBuffer& buffer, int handle, int index, - int flip_mode, int64_t flip_arg); +[[nodiscard]] uint64_t PrepareVideoOutFlip(CommandBuffer& buffer, int handle, int index, + int flip_mode, int64_t flip_arg); void WriteAtEndOfPipeOnlyFlip(uint64_t submit_id, CommandBuffer& buffer, int handle, int index, int flip_mode, int64_t flip_arg, uint64_t request_id); void WriteAtEndOfPipeWithFlip32(uint64_t submit_id, CommandBuffer& buffer, uint32_t* dst_gpu_addr, @@ -58,10 +59,10 @@ void WriteAtEndOfPipeWithInterruptWriteBackFlip32(uint64_t submit_id, CommandBuf int handle, int index, int flip_mode, int64_t flip_arg, uint64_t request_id); -int AddEqEvent(LibKernel::EventQueue::KernelEqueue eq, int id, void* udata); +int AddEqEvent(RenderContext& renderer, LibKernel::EventQueue::KernelEqueue eq, int id, + void* udata); int DeleteEqEvent(LibKernel::EventQueue::KernelEqueue eq, int id); -void ReadGds(uint32_t* dst, uint32_t dw_offset, uint32_t dw_size); -void DeleteBuffers(); +void ReadGds(Buffer& gds, uint32_t* dst, uint32_t dw_offset, uint32_t dw_size); } // namespace Sync } // namespace Libs::Graphics diff --git a/src/graphics/host_gpu/renderer/textureCache.cpp b/src/graphics/host_gpu/renderer/textureCache.cpp index 010acaef..4fe7ccd4 100644 --- a/src/graphics/host_gpu/renderer/textureCache.cpp +++ b/src/graphics/host_gpu/renderer/textureCache.cpp @@ -1,3859 +1,1882 @@ #include "graphics/host_gpu/renderer/textureCache.h" #include "common/assert.h" +#include "common/emulatorConfig.h" #include "common/logging/log.h" #include "common/profiler.h" -#include "graphics/guest_gpu/gpu_defs.h" #include "graphics/guest_gpu/gpu_format.h" -#include "graphics/guest_gpu/graphicsRun.h" #include "graphics/guest_gpu/tile.h" -#include "graphics/host_gpu/gpuTiler.h" #include "graphics/host_gpu/graphicContext.h" -#include "graphics/host_gpu/objects/label.h" #include "graphics/host_gpu/objects/textureCommon.h" #include "graphics/host_gpu/renderer/bufferCache.h" -#include "graphics/host_gpu/renderer/dummyTextureCache.h" -#include "graphics/host_gpu/renderer/image.h" +#include "graphics/host_gpu/renderer/commandScheduler.h" #include "graphics/host_gpu/renderer/imageView.h" #include "graphics/host_gpu/renderer/render.h" -#include "graphics/host_gpu/renderer/renderContext.h" -#include "graphics/host_gpu/renderer/renderTargetBarriers.h" #include "graphics/host_gpu/renderer/resourceMutex.h" -#include "graphics/host_gpu/transfer.h" -#include "graphics/host_gpu/vulkanCommon.h" +#include "graphics/host_gpu/renderer/tiler.h" #include "kernel/memory.h" #include -#include +#include #include -#include +#include #include +#include #include -#include -#include -#include +#include +#include namespace Libs::Graphics { namespace { -thread_local const void* g_texture_cache_lock_owner = nullptr; -thread_local const void* g_texture_fault_owner = nullptr; +constexpr uint64_t NumFramesBeforeRemoval = 32; -[[nodiscard]] bool GuestRangeIsZero(uint64_t address, uint64_t size) noexcept { - const auto* bytes = reinterpret_cast(address); - while (size >= sizeof(uint64_t)) { - uint64_t word = 0; - std::memcpy(&word, bytes, sizeof(word)); - if (word != 0) { - return false; +thread_local const TextureCache* g_locked_cache = nullptr; + +class CacheLock final { +public: + CacheLock(const TextureCache& owner, TrackingSpinLock& lock): m_lock(lock) { + if (g_locked_cache != nullptr) { + EXIT("TextureCache: recursive cache lock\n"); } - bytes += sizeof(word); - size -= sizeof(word); + g_locked_cache = &owner; + m_lock.lock(); } - while (size-- != 0) { - if (*bytes++ != 0) { - return false; + ~CacheLock() { + m_lock.unlock(); + g_locked_cache = nullptr; + } + +private: + TrackingSpinLock& m_lock; +}; + +} // namespace + +TextureCache::TextureCache(GraphicContext& graphics, CommandScheduler& scheduler, + PageManager& page_manager, BufferCache& buffer_cache, + ResourceMutex& resource_mutex) + : m_graphics(graphics), m_scheduler(scheduler), + m_memory_tracker(page_manager, PageWatchMode::Write), m_blit_helper(graphics, scheduler), + m_tiler(std::make_unique(graphics, scheduler, + buffer_cache.GetUtilityBuffer(MemoryUsage::Stream))), + m_buffer_cache(buffer_cache), m_resource_mutex(resource_mutex), + m_readback_linear_images(Config::ReadbackLinearImagesEnabled()) { + if (m_graphics.CanReportMemoryUsage()) { + constexpr int64_t GiB = 1024ll * 1024 * 1024; + const auto budget = + static_cast(std::min(m_graphics.GetTotalMemoryBudget(), INT64_MAX)); + const auto threshold = std::min(budget, 8 * GiB); + m_pressure_gc_memory = static_cast( + std::max(std::min(budget - 6 * threshold / 10, budget - GiB), GiB + GiB / 2)); + m_critical_gc_memory = static_cast( + std::max(std::min(budget - 2 * threshold / 10, budget - GiB / 2), 3 * GiB)); + m_trigger_gc_memory = static_cast(std::max((budget - threshold) / 2, 0)); + } +} + +TextureCache::~TextureCache() { + for (uint32_t index = 0; index < m_slots.size(); index++) { + if (m_slots[index].image != nullptr && m_slots[index].image->registered) { + UnregisterImage({index, m_slots[index].generation}, false); } + m_slots[index].image.reset(); + } +} + +bool TextureCache::SameBacking(const ImageInfo& cached, const ImageInfo& requested, + bool exact_format) { + const bool unit_extent = + requested.extent.width == 1 && requested.extent.height == 1 && requested.extent.depth == 1; + return cached.data == requested.data && cached.extent == requested.extent && + cached.samples == requested.samples && + cached.bytes_per_block == requested.bytes_per_block && + (cached.type == requested.type || unit_extent) && + (exact_format + ? cached.pixel_format == requested.pixel_format + : ImageViewOps::FormatsCompatible(cached.pixel_format, requested.pixel_format)); +} + +TextureCache::BindingType TextureCache::UploadBinding(const Image& image) { + if (image.info.IsDepth()) { + return BindingType::DepthTarget; + } + if (image.usage.render_target) { + return BindingType::RenderTarget; + } + if (image.usage.video_out) { + return BindingType::VideoOut; + } + return image.usage.storage ? BindingType::Storage : BindingType::Texture; +} + +bool TextureCache::SafeToDownload(const Image& image) { + if (!image.SafeToDownload()) { + return false; + } + const auto range = image.info.data; + return !m_buffer_cache.HasGpuDirtyBytes(range.address, range.size) && + !m_memory_tracker.IsRegionCpuModified(range.address, range.size); +} + +Image& TextureCache::ResolveImage(ImageId id) { + if (!id || id.index >= m_slots.size()) { + EXIT("TextureCache: invalid image id\n"); + } + auto& slot = m_slots[id.index]; + if (slot.generation != id.generation || slot.image == nullptr) { + EXIT("TextureCache: stale image id\n"); + } + return *slot.image; +} + +const Image& TextureCache::ResolveImage(ImageId id) const { + if (!id || id.index >= m_slots.size()) { + EXIT("TextureCache: invalid image id\n"); + } + const auto& slot = m_slots[id.index]; + if (slot.generation != id.generation || slot.image == nullptr) { + EXIT("TextureCache: stale image id\n"); + } + return *slot.image; +} + +std::shared_ptr TextureCache::ResolveOwner(ImageId id) const { + if (!id || id.index >= m_slots.size()) { + return {}; + } + const auto& slot = m_slots[id.index]; + return slot.generation == id.generation ? slot.image : nullptr; +} + +ImageId TextureCache::InsertImage(const ImageInfo& info) { + uint32_t index = 0; + if (m_free_slots.empty()) { + index = static_cast(m_slots.size()); + m_slots.emplace_back(); + } else { + index = m_free_slots.back(); + m_free_slots.pop_back(); + } + auto& slot = m_slots[index]; + if (slot.image != nullptr) { + EXIT("TextureCache: occupied free image slot\n"); + } + slot.image = std::make_shared(m_graphics, m_scheduler, info); + const ImageId id {index, slot.generation}; + if (!info.data.Empty()) { + RegisterImage(id); + } + return id; +} + +void TextureCache::RegisterImage(ImageId id) { + auto& image = ResolveImage(id); + if (image.registered || image.info.data.Empty()) { + EXIT("TextureCache: invalid image registration\n"); + } + std::vector ranges; + ranges.push_back({image.info.data.address, image.info.data.size}); + if (!m_image_owner_index.Register(id, ranges)) { + EXIT("TextureCache: duplicate or invalid image registration\n"); + } + image.registered = true; + image.lru_id = m_lru_cache.Insert(id, m_gc_tick); + m_total_used_memory += image.AccountedSize(); +} + +void TextureCache::UnregisterImage(ImageId id, bool release_tracking) { + auto& image = ResolveImage(id); + if (!image.registered) { + return; + } + std::vector releases; + if (!m_image_owner_index.Unregister(id, releases)) { + EXIT("TextureCache: image missing from owner index\n"); + } + m_lru_cache.Free(image.lru_id); + if (release_tracking) { + for (const auto& range: releases) { + m_memory_tracker.UntrackMemory(range.address, range.size); + } + } + const auto accounted = image.AccountedSize(); + if (accounted > m_total_used_memory) { + EXIT("TextureCache: image accounting underflow\n"); + } + m_total_used_memory -= accounted; + image.registered = false; +} + +void TextureCache::DeleteImage(ImageId id, bool release_tracking) { + auto owner = ResolveOwner(id); + if (owner == nullptr || !owner->registered) { + return; + } + if (!owner->depth_id) { + std::vector associations; + for (uint32_t index = 0; index < m_slots.size(); index++) { + const auto& slot = m_slots[index]; + if (slot.image != nullptr && slot.image->depth_id == id) { + associations.push_back({index, slot.generation}); + } + } + for (const auto association: associations) { + ReleaseGpuTracking(association); + DeleteImage(association); + } + } + if (owner->IsGpuModified()) { + EXIT("TextureCache: deleting a GPU-modified image without resolving its contents\n"); + } + m_download_images.erase(id); + if (owner->info.metadata.kind == ImageMetadataKind::Htile) { + m_surface_metas.erase(owner->info.metadata.range.address); + } + UnregisterImage(id, release_tracking); + const auto erase_slot = [this, id, retained = owner] { + auto& slot = m_slots[id.index]; + if (slot.generation != id.generation || slot.image != retained) { + EXIT("TextureCache: retired image slot changed before deferred erasure\n"); + } + slot.image.reset(); + if (++slot.generation == 0) { + slot.generation = 1; + } + m_free_slots.push_back(id.index); + }; + if (m_scheduler.Active()) { + m_scheduler.DeferOperation(erase_slot); + } else { + erase_slot(); + } +} + +void TextureCache::DeleteImages(std::span ids, + std::optional native_source) { + std::set> unique; + for (const auto id: ids) { + if (!id || !unique.emplace(id.index, id.generation).second) { + continue; + } + auto owner = ResolveOwner(id); + if (owner == nullptr) { + continue; + } + if (native_source == id) { + ReleaseGpuTracking(id); + } else if (owner->IsGpuModified()) { + DownloadImage(id); + ReleaseGpuTracking(id); + } + DeleteImage(id); + } +} + +void TextureCache::RetainImage(CommandBuffer& command, ImageId id) { + auto owner = ResolveOwner(id); + if (owner == nullptr) { + EXIT("TextureCache: retaining a stale image\n"); + } + if (owner->depth_id) { + auto depth = ResolveOwner(owner->depth_id); + if (depth == nullptr) { + EXIT("TextureCache: stencil association points to a stale depth image\n"); + } + command.RetainResourceUntilFence(std::move(depth)); + } + command.RetainResourceUntilFence(std::move(owner)); +} + +void TextureCache::TouchImage(Image& image) { + if (image.registered) { + m_lru_cache.Touch(image.lru_id, m_gc_tick); + } +} + +void TextureCache::TrackImageDownload(ImageId id) { + std::lock_guard transaction(m_resource_mutex); + CacheLock lock(*this, m_lock); + auto& image = ResolveImage(id); + TrackImageDownloadLocked(id, image); +} + +void TextureCache::TrackImageDownloadLocked(ImageId id, Image& image) { + if (m_readback_linear_images && !image.info.IsTiled() && !image.info.data.Empty()) { + if (!image.IsGpuModified()) { + EXIT("TextureCache: cannot enroll a non-GPU-owned image for download\n"); + } + m_download_images.insert(id); + } +} + +Image& TextureCache::GetImage(ImageId id) { + auto& image = ResolveImage(id); + TouchImage(image); + return image; +} + +const Image& TextureCache::GetImage(ImageId id) const { + return ResolveImage(id); +} + +std::vector TextureCache::FindImagesInRegion(uint64_t address, uint64_t size, + bool page_overlap) const { + return page_overlap ? m_image_owner_index.QueryCandidates(address, size) + : m_image_owner_index.Query(address, size); +} + +ImageId TextureCache::GetNullImage(const ImageDesc& desc) { + auto& command = m_scheduler.Current(); + const auto format = desc.info.pixel_format; + if (const auto found = m_null_images.find(format); found != m_null_images.end()) { + RetainImage(command, found->second); + return found->second; + } + ImageInfo info {}; + info.pixel_format = desc.info.pixel_format; + info.guest_format = desc.info.guest_format; + info.type = Prospero::ImageType::kColor2D; + info.extent = {1, 1, 1}; + info.resources = {1, 1}; + info.pitch = 1; + info.bytes_per_block = std::max(desc.info.bytes_per_block, 1u); + info.samples = 1; + info.tile_mode = Prospero::GpuEnumValue(Prospero::TileMode::kLinear); + info.mip_layout[0] = {0, info.bytes_per_block, 1, 1}; + const auto id = InsertImage(info); + m_null_images.emplace(format, id); + RetainImage(command, id); + return id; +} + +void TextureCache::ValidateImageDesc(const ImageDesc& desc) const { + ImageOps::Validate(desc.info); + if (desc.view_info.format == vk::Format::eUndefined || desc.view_info.level_count == 0 || + desc.view_info.layer_count == 0 || + desc.view_info.base_level >= desc.info.resources.levels || + desc.view_info.level_count > desc.info.resources.levels - desc.view_info.base_level || + (!desc.info.IsVolume() && + (desc.view_info.base_layer >= desc.info.resources.layers || + desc.view_info.layer_count > desc.info.resources.layers - desc.view_info.base_layer))) { + EXIT("TextureCache: invalid image view description\n"); + } + if (desc.type == BindingType::DepthTarget && !IsSupportedDepthTargetFormat(desc.info)) { + EXIT("TextureCache: unsupported depth image description\n"); + } + if (desc.type == BindingType::VideoOut && !IsSupportedVideoOutFormat(desc.info)) { + EXIT("TextureCache: unsupported video-out image description\n"); + } + if (desc.type == BindingType::VideoOut && + desc.info.metadata.compression == VideoOutCompression::Unsupported) { + EXIT("TextureCache: unsupported compressed video-out description\n"); + } +} + +void TextureCache::PrepareImageCopy(Image& image) { + const auto range = image.info.data; + m_memory_tracker.ForEachUploadRange( + range.address, range.size, false, [](uint64_t, uint64_t) noexcept {}, []() noexcept {}); + if (image.IsCpuDirty()) { + image.RefreshComplete(); + } +} + +void TextureCache::RefreshCopySource(ImageId id) { + auto& image = ResolveImage(id); + RefreshImage(id, ImageDesc {.info = image.info, .view_info = {}, .type = UploadBinding(image)}); + if (image.IsDefinitelyCpuDirty()) { + EXIT("TextureCache: image copy source remained CPU-dirty after refresh\n"); + } +} + +bool TextureCache::CopyD16(Image& destination, Image& source) { + const bool source_depth = source.info.IsDepth(); + const bool destination_depth = destination.info.IsDepth(); + if (source_depth == destination_depth) { + return false; + } + auto& depth = source_depth ? source : destination; + auto& color = source_depth ? destination : source; + const auto transfer_bytes = DepthAspectTransferBytes(depth.backing.format); + if (depth.info.bytes_per_block != sizeof(uint16_t) || + color.info.bytes_per_block != sizeof(uint16_t) || transfer_bytes != sizeof(uint32_t)) { + return false; + } + EXIT_IF(source.backing.samples != 1 || destination.backing.samples != 1 || + source.info.resources.levels != 1 || destination.info.resources.levels != 1 || + source.info.extent != destination.info.extent || + source.info.resources.layers != destination.info.resources.layers); + + const auto layers = depth.info.resources.layers; + const uint64_t depth_slice = + static_cast(depth.info.pitch) * depth.info.extent.height * transfer_bytes; + const uint64_t color_slice = + static_cast(color.info.pitch) * color.info.extent.height * sizeof(uint16_t); + EXIT_IF(layers == 0 || depth_slice > UINT64_MAX / layers || color_slice > UINT64_MAX / layers); + const auto depth_size = depth_slice * layers; + const auto color_size = color_slice * layers; + std::vector depth_copies(layers); + std::vector color_copies(layers); + for (uint32_t layer = 0; layer < layers; layer++) { + depth_copies[layer].bufferOffset = depth_slice * layer; + depth_copies[layer].bufferRowLength = depth.info.pitch; + depth_copies[layer].bufferImageHeight = depth.info.extent.height; + depth_copies[layer].imageSubresource = {vk::ImageAspectFlagBits::eDepth, 0, layer, 1}; + depth_copies[layer].imageExtent = depth.info.extent; + color_copies[layer].bufferOffset = color_slice * layer; + color_copies[layer].bufferRowLength = color.info.pitch; + color_copies[layer].bufferImageHeight = color.info.extent.height; + color_copies[layer].imageSubresource = {vk::ImageAspectFlagBits::eColor, 0, layer, 1}; + color_copies[layer].imageExtent = color.info.extent; + } + + auto depth_buffer = m_tiler->GetScratchBuffer(depth_size); + auto color_buffer = m_tiler->GetScratchBuffer(color_size); + const TileManager::D16Layout promote_layout { + .width = depth.info.extent.width, + .height = depth.info.extent.height, + .layers = layers, + .source_row_stride = static_cast(color.info.pitch) * sizeof(uint16_t), + .target_row_stride = static_cast(depth.info.pitch) * transfer_bytes, + .source_slice_stride = color_slice, + .target_slice_stride = depth_slice, + }; + const bool d32 = DepthAspectTransferFormat(depth.backing.format) == vk::Format::eD32Sfloat; + if (source_depth) { + source.Download(depth_copies, depth_buffer.buffer, depth_buffer.offset, depth_buffer.size); + m_tiler->ConvertD16(depth_buffer, color_buffer, TileManager::D16Direction::Demote, d32, + {.width = promote_layout.width, + .height = promote_layout.height, + .layers = promote_layout.layers, + .source_row_stride = promote_layout.target_row_stride, + .target_row_stride = promote_layout.source_row_stride, + .source_slice_stride = promote_layout.target_slice_stride, + .target_slice_stride = promote_layout.source_slice_stride}); + destination.Upload(color_copies, color_buffer.buffer, color_buffer.offset, + color_buffer.size); + } else { + source.Download(color_copies, color_buffer.buffer, color_buffer.offset, color_buffer.size); + m_tiler->ConvertD16(color_buffer, depth_buffer, TileManager::D16Direction::Promote, d32, + promote_layout); + destination.Upload(depth_copies, depth_buffer.buffer, depth_buffer.offset, + depth_buffer.size); } return true; } -[[nodiscard]] uint64_t HashSampledImageEdges(const Image& image) { - constexpr uint64_t page_mask = TRACKER_PAGE_SIZE - 1; - const uint64_t begin = image.address; - const uint64_t end = image.address + image.size; - const uint64_t head_end = std::min(end, (begin + page_mask) & ~page_mask); - const uint64_t tail_begin = std::max(begin, end & ~page_mask); - const uint64_t head_size = head_end - begin; - const uint64_t tail_size = tail_begin < head_end ? end - head_end : end - tail_begin; - std::array bytes {}; - if (head_size + tail_size > bytes.size()) { - EXIT("TextureCache: sampled-image edge hash range overflow\n"); +void TextureCache::CopyImage(ImageId destination_id, ImageId source_id) { + RefreshCopySource(source_id); + auto& destination = ResolveImage(destination_id); + auto& source = ResolveImage(source_id); + if (source.backing.samples != destination.backing.samples) { + EXIT("TextureCache: cannot issue an unequal-sample image copy\n"); } - if ((head_size != 0 && !LibKernel::Memory::TryReadBacking(begin, bytes.data(), head_size)) || - (tail_size != 0 && - !LibKernel::Memory::TryReadBacking(tail_begin < head_end ? head_end : tail_begin, - bytes.data() + head_size, tail_size))) { - EXIT("TextureCache: failed to hash sampled-image edge backing, addr=0x%016" PRIx64 - " size=0x%016" PRIx64 "\n", - image.address, image.size); + PrepareImageCopy(destination); + if (source.IsBufferModified()) { + if (source.info.data == destination.info.data) { + destination.MarkBufferModified(); + } + RestoreGpuTracking(destination); + return; } - return XXH3_64bits(bytes.data(), static_cast(head_size + tail_size)); + const bool source_depth = source.info.IsDepth(); + const bool dest_depth = destination.info.IsDepth(); + const bool direct_copy = + source.backing.format == destination.backing.format || + (!source_depth && !dest_depth && + ImageViewOps::FormatsCompatible(source.backing.format, destination.backing.format)); + if (direct_copy) { + destination.CopyImage(source); + } else if (!CopyD16(destination, source)) { + if (source.backing.samples != 1 || destination.backing.samples != 1) { + EXIT("TextureCache: cross-format multisample image copy is unsupported\n"); + } + auto& copy_buffer = m_buffer_cache.GetUtilityBuffer(MemoryUsage::DeviceLocal); + destination.CopyImageWithBuffer(source, copy_buffer); + } + RetainImage(m_scheduler.Current(), source_id); + RetainImage(m_scheduler.Current(), destination_id); + if (source.IsGpuModified()) { + destination.MarkGpuModified(); + } + destination.ClearBufferModified(); + RestoreGpuTracking(destination); } -class FaultSafeTextureLock final { -public: - FaultSafeTextureLock(const void* owner, TrackingSpinLock& mutex): m_mutex(mutex) { - if (g_texture_cache_lock_owner != nullptr) { - EXIT("TextureCache: recursive cache lock acquisition, current=%p\n", - g_texture_cache_lock_owner); - } - g_texture_cache_lock_owner = owner; - m_mutex.lock(); +void TextureCache::CopyImageMip(ImageId destination_id, ImageId source_id, uint32_t mip, + uint32_t layer) { + RefreshCopySource(source_id); + auto& destination = ResolveImage(destination_id); + auto& source = ResolveImage(source_id); + if (source.IsBufferModified() || source.backing.samples != destination.backing.samples) { + EXIT("TextureCache: invalid mip-copy ownership or sample count\n"); } - ~FaultSafeTextureLock() { - m_mutex.unlock(); - g_texture_cache_lock_owner = nullptr; + destination.CopyMip(source, mip, layer); + RetainImage(m_scheduler.Current(), source_id); + RetainImage(m_scheduler.Current(), destination_id); + if (source.IsGpuModified()) { + destination.MarkGpuModified(); + } + RestoreGpuTracking(destination); +} + +ImageId TextureCache::ResolveDepthOverlap(const ImageInfo& requested, BindingType binding, + ImageId cached_id) { + auto& cached = ResolveImage(cached_id); + if (!cached.info.IsDepth() && !requested.IsDepth()) { + return {}; + } + const bool stencil_match = requested.HasStencil() == cached.info.HasStencil(); + const bool bpp_match = requested.bytes_per_block == cached.info.bytes_per_block; + bool recreate = cached.info.resources < requested.resources; + switch (binding) { + case BindingType::Texture: recreate |= requested.IsDepth() && !cached.info.IsDepth(); break; + case BindingType::Storage: recreate |= cached.info.IsDepth(); break; + case BindingType::RenderTarget: recreate |= cached.info.IsDepth(); break; + case BindingType::DepthTarget: + recreate |= !cached.info.IsDepth(); + recreate |= cached.info.IsDepth() && !(stencil_match && bpp_match); + break; + case BindingType::VideoOut: recreate |= cached.info.IsDepth(); break; + } + if (!recreate) { + return cached_id; + } + RefreshImage(cached_id, + ImageDesc {.info = cached.info, .view_info = {}, .type = UploadBinding(cached)}); + auto info = requested; + info.resources = std::max(requested.resources, cached.info.resources); + info.htile_clear_mask = 0; + const auto replacement_id = InsertImage(info); + auto& replacement = ResolveImage(replacement_id); + replacement.usage = cached.usage; + if (cached.binding.is_bound || cached.binding.is_target) { + cached.binding.needs_rebind = true; + } + bool copied = false; + if (cached.backing.samples == replacement.backing.samples) { + const bool copy_supported = + cached.backing.samples == 1 || cached.backing.format == replacement.backing.format || + (!cached.info.IsDepth() && !replacement.info.IsDepth() && + ImageViewOps::FormatsCompatible(cached.backing.format, replacement.backing.format)); + if (copy_supported) { + CopyImage(replacement_id, cached_id); + copied = true; + } else { + LOGF_COLOR(Log::Color::BrightYellow, + "TextureCache: unsupported cross-format multisample depth copy\n"); + } + } else if (cached.backing.samples == 1 && replacement.backing.samples > 1 && + replacement.info.IsDepth()) { + RefreshCopySource(cached_id); + if (cached.IsBufferModified() || cached.IsDefinitelyCpuDirty()) { + EXIT("TextureCache: multisample depth conversion source is not native-current\n"); + } + PrepareImageCopy(replacement); + m_blit_helper.ReinterpretColorAsMsDepth(cached, replacement); + auto& command = m_scheduler.Current(); + RetainImage(command, cached_id); + RetainImage(command, replacement_id); + CommitGpuWrite(replacement); + copied = true; + } else { + LOGF_COLOR(Log::Color::BrightYellow, + "TextureCache: unsupported unequal-sample depth overlap copy (%u -> %u)\n", + cached.backing.samples, replacement.backing.samples); + } + if (copied) { + DeleteImages(std::array {cached_id}, cached_id); + } else { + ReleaseGpuTracking(cached_id); + DeleteImage(cached_id); + } + return replacement_id; +} + +TextureCache::OverlapResult TextureCache::ResolveOverlap(const ImageInfo& requested, + BindingType binding, ImageId cached_id, + ImageId merged_id) { + auto owner = ResolveOwner(cached_id); + if (owner == nullptr) { + return {merged_id}; + } + auto& cached = *owner; + const auto current_tick = m_scheduler.CurrentTick(); + const bool safe_to_delete = + current_tick - std::min(current_tick, cached.tick_accessed_last) > NumFramesBeforeRemoval; + + if (requested.data.address == cached.info.data.address) { + const uint32_t requested_block = requested.bytes_per_block * requested.samples; + const uint32_t cached_block = cached.info.bytes_per_block * cached.info.samples; + if (requested.BlockExtent() != cached.info.BlockExtent() || + requested_block != cached_block) { + if (safe_to_delete) { + DeleteImages(std::array {cached_id}, cached_id); + } + return {merged_id}; + } + + if (const auto depth_id = ResolveDepthOverlap(requested, binding, cached_id)) { + return {depth_id}; + } + if (requested.IsBlock() && !cached.info.IsBlock()) { + return {ExpandImage(requested, cached_id)}; + } + if (requested.data.size == cached.info.data.size && + (requested.IsVolume() || cached.info.IsVolume())) { + return {ExpandImage(requested, cached_id)}; + } + if (requested.pixel_format != cached.info.pixel_format || + requested.data.size <= cached.info.data.size) { + const auto result_id = merged_id ? merged_id : cached_id; + const auto result = ResolveOwner(result_id); + return {result != nullptr && ImageViewOps::FormatsCompatible(result->info.pixel_format, + requested.pixel_format) + ? result_id + : ImageId {}}; + } + if (requested.type == cached.info.type && requested.resources > cached.info.resources) { + return {ExpandImage(requested, cached_id)}; + } + if (requested.tile_mode != cached.info.tile_mode) { + if (safe_to_delete) { + DeleteImages(std::array {cached_id}, cached_id); + } + return {merged_id}; + } + EXIT("TextureCache: unresolvable equal-address image overlap, address=0x%016" PRIx64 + " requested=%ux%u " + "cached=%ux%u requested_size=0x%016" PRIx64 " cached_size=0x%016" PRIx64 + " type=%u/%u tile=%u/%u\n", + requested.data.address, requested.resources.levels, requested.resources.layers, + cached.info.resources.levels, cached.info.resources.layers, requested.data.size, + cached.info.data.size, static_cast(requested.type), + static_cast(cached.info.type), requested.tile_mode, cached.info.tile_mode); } -private: - TrackingSpinLock& m_mutex; + if (requested.data.address > cached.info.data.address) { + const int32_t mip = requested.MipOf(cached.info); + if (mip >= 0) { + const int32_t layer = requested.SliceOf(cached.info, mip); + if (layer >= 0) { + return {cached_id, mip, layer}; + } + } + if (safe_to_delete) { + DeleteImages(std::array {cached_id}, cached_id); + } + return {}; + } + + const int32_t mip = cached.info.MipOf(requested); + if (mip >= 0) { + const int32_t layer = cached.info.SliceOf(requested, mip); + if (layer >= 0) { + if (cached.binding.is_target) { + cached.binding.needs_rebind = true; + if (merged_id) { + ResolveImage(merged_id).binding.is_target = true; + } + DeleteImages(std::array {cached_id}, cached_id); + return {merged_id}; + } + if (merged_id) { + CopyImageMip(merged_id, cached_id, static_cast(mip), + static_cast(layer)); + DeleteImages(std::array {cached_id}, cached_id); + } + } + } + return {merged_id}; +} + +ImageId TextureCache::ExpandImage(const ImageInfo& info, ImageId source_id) { + RefreshCopySource(source_id); + const auto expanded_id = InsertImage(info); + auto& expanded = ResolveImage(expanded_id); + auto& source = ResolveImage(source_id); + expanded.usage = source.usage; + if (source.binding.is_bound || source.binding.is_target) { + source.binding.needs_rebind = true; + } + InitializeImage(expanded_id, + ImageDesc {.info = info, .view_info = {}, .type = UploadBinding(source)}); + CopyImage(expanded_id, source_id); + DeleteImages(std::array {source_id}, source_id); + return expanded_id; +} + +struct TextureCache::ColorTransferPlan { + TextureUploadLayout layout; + std::vector regions; + std::vector tiles; + bool tiled = false; + bool swap_bgra16 = false; + bool valid = false; }; -enum class ImageRangeOverlap : uint8_t { None, PageOnly, Bytes }; - -ImageRangeOverlap ClassifyImageRangeOverlap(uint64_t left, uint64_t left_size, uint64_t right, - uint64_t right_size) { - if (ImageRangeOverlaps(left, left_size, right, right_size)) { - return ImageRangeOverlap::Bytes; - } - return ImagePageRangesOverlap(left, left_size, right, right_size) ? ImageRangeOverlap::PageOnly - : ImageRangeOverlap::None; -} - -} // namespace - -bool IsExactRenderTargetMipStorage(const ImageInfo& sampled, const ImageInfo& storage, - vk::Format sampled_view_format, - vk::Format storage_image_format) noexcept { - const auto render_target = Prospero::GpuEnumValue(Prospero::TileMode::kRenderTarget); - const auto image_2d = Prospero::GpuEnumValue(Prospero::ImageType::kColor2D); - if (sampled.address == 0 || sampled.size == 0 || sampled.width == 0 || sampled.height == 0 || - (sampled.address & 0xffffu) != 0 || sampled.levels <= 1 || sampled.levels > 16 || - sampled.base_level != 0 || sampled.view_levels != sampled.levels || - sampled.tile != render_target || sampled.depth != 1 || sampled.type != image_2d || - sampled.base_array != 0 || storage.address == 0 || storage.size == 0 || - (storage.address & 0xffffu) != 0 || storage.width == 0 || storage.height == 0 || - storage.base_level != 0 || storage.levels != 1 || storage.view_levels != 1 || - storage.tile != render_target || storage.depth != 1 || storage.type != image_2d || - storage.base_array != 0 || sampled.format != storage.format || - sampled_view_format != storage_image_format) { - return false; - } - const auto bytes_per_element = Prospero::NumBytesPerElement(sampled.format); - if (bytes_per_element == 0) { - return false; - } - TileSizeAlign sampled_layout {}; - std::array level_layouts {}; - std::array level_padded {}; - if (!TileGetRenderTargetMipLayout(sampled.width, sampled.height, sampled.pitch, - bytes_per_element, sampled.levels, sampled_layout, - level_layouts.data(), level_padded.data()) || - sampled_layout.align != 65536 || sampled_layout.size != sampled.size) { - return false; - } - TileSizeAlign storage_layout {}; - if (!TileGetRenderTargetSize(storage.width, storage.height, storage.pitch, bytes_per_element, - storage_layout) || - storage_layout.align != 65536 || storage_layout.size != storage.size) { - return false; - } - for (uint32_t level = 0; level < sampled.levels; level++) { - const uint64_t divisor = 1ull << level; - const auto level_width = - static_cast((static_cast(sampled.width) + divisor - 1) / divisor); - const auto level_height = - static_cast((static_cast(sampled.height) + divisor - 1) / divisor); - const auto& layout = level_layouts[level]; - const bool dedicated_allocation = layout.size == layout.src_size && - layout.offset == layout.src_offset && layout.x == 0 && - layout.y == 0; - const auto& padded = level_padded[level]; - const bool packed_tail_allocation = - layout.src_size == 65536 && !dedicated_allocation && padded.width != 0 && - padded.height != 0 && storage.width == padded.width && - storage.height == padded.height && storage.pitch == padded.width && - storage.size == layout.src_size && storage_layout.size == layout.src_size && - layout.src_offset <= sampled.size && - layout.src_size <= sampled.size - layout.src_offset && - sampled.address <= UINT64_MAX - layout.src_offset && - storage.address == sampled.address + layout.src_offset; - if (packed_tail_allocation) { - // A PS5 mip tail is one physical thin-64 KiB tile. The storage descriptor exposes - // that complete padded block so its shader can address the packed mip locations. - return true; - } - if (!dedicated_allocation || storage.width != level_width || - storage.height != level_height || - storage.pitch != TileGetRenderTargetPitch(level_width, bytes_per_element) || - storage.size != layout.size || storage_layout.size != layout.size || - sampled.address > UINT64_MAX - layout.offset || - storage.address != sampled.address + layout.offset) { - continue; - } - return true; - } - return false; -} - -struct TextureCache::CachedImage { - explicit CachedImage(GraphicContext& graphics): graphics(graphics) {} - - enum class Kind { - Texture, - StorageTexture, - RenderTarget, - DepthTarget, - VideoOut - } kind = Kind::Texture; - Image info; - RenderTargetInfo target; - DepthTargetInfo depth; - VideoOutInfo video_out; - GraphicContext& graphics; - VulkanImage* image = nullptr; - bool gpu_modified = false; - bool buffer_modified = false; - bool stencil_initialized = false; - bool registered = false; - - ~CachedImage() { - if (image == nullptr || registered) { - EXIT("TextureCache: cached image destroyed with invalid resources, image=%p " - "kind=%u registered=%d\n", - static_cast(image), static_cast(kind), registered); - } - ImageOps::Destroy(*image); - image = nullptr; - } - - [[nodiscard]] uint32_t RangeCount() const { - return kind == Kind::DepthTarget && depth.stencil_address != 0 ? 2u : 1u; - } - [[nodiscard]] BufferImageBinding BufferBinding() const { - switch (kind) { - case Kind::Texture: return BufferImageBinding::Texture; - case Kind::RenderTarget: return BufferImageBinding::RenderTarget; - case Kind::VideoOut: return BufferImageBinding::VideoOut; - case Kind::StorageTexture: return BufferImageBinding::StorageTexture; - case Kind::DepthTarget: return BufferImageBinding::DepthTarget; - } - return BufferImageBinding::Unsupported; - } - [[nodiscard]] uint64_t Address(uint32_t index = 0) const { - if (index >= RangeCount()) { - EXIT("TextureCache: image address range index out of bounds, index=%u count=%u\n", - index, RangeCount()); - } - if (index == 1) { - return depth.stencil_address; - } - switch (kind) { - case Kind::Texture: return info.address; - case Kind::StorageTexture: return info.address; - case Kind::RenderTarget: return target.address; - case Kind::DepthTarget: return depth.address; - case Kind::VideoOut: return video_out.address; - } - EXIT("TextureCache: unsupported cached image kind %u for address\n", - static_cast(kind)); - } - [[nodiscard]] uint64_t Size(uint32_t index = 0) const { - if (index >= RangeCount()) { - EXIT("TextureCache: image size range index out of bounds, index=%u count=%u\n", index, - RangeCount()); - } - if (index == 1) { - return depth.stencil_size; - } - switch (kind) { - case Kind::Texture: return info.size; - case Kind::StorageTexture: return info.size; - case Kind::RenderTarget: return target.size; - case Kind::DepthTarget: return depth.size; - case Kind::VideoOut: return video_out.size; - } - EXIT("TextureCache: unsupported cached image kind %u for size\n", - static_cast(kind)); - } - [[nodiscard]] bool OverlapsRange(uint64_t address, uint64_t size, bool page) const { - for (uint32_t i = 0; i < RangeCount(); i++) { - if (page ? ImagePageRangesOverlap(address, size, Address(i), Size(i)) - : ImageRangeOverlaps(address, size, Address(i), Size(i))) { - return true; - } - } - return false; - } - // A page-expanded region lookup identifies candidates before exact byte classification. Kyty's - // tracker retains GPU ownership per 4 KiB page, so an edge-page - // candidate must remain coherent even when the triggering byte lies just outside the image. - [[nodiscard]] bool IsGpuReadbackPageCandidate(uint64_t address, uint64_t size) const { - return gpu_modified && OverlapsRange(address, size, true); - } - [[nodiscard]] bool HasExactRange(uint64_t address, uint64_t size) const { - for (uint32_t i = 0; i < RangeCount(); i++) { - if (address == Address(i) && size == Size(i)) { - return true; - } - } - return false; - } +struct TextureCache::DownloadPlan { + ColorTransferPlan color; + bool depth = false; + bool valid = false; }; -void TextureCache::RegisterImageLocked(CachedImage& image) { - if (image.registered) { - EXIT("TextureCache: registering an already registered image\n"); - } - std::vector ranges; - ranges.reserve(image.RangeCount()); - for (uint32_t range = 0; range < image.RangeCount(); range++) { - ranges.push_back({image.Address(range), image.Size(range)}); - } - if (!m_image_owner_index.Register(&image, ranges)) { - EXIT("TextureCache: invalid or duplicate image registration\n"); - } - image.registered = true; -} +TextureCache::ColorTransferPlan +TextureCache::BuildColorTransfer(const Image& image, BindingType binding, + TransferDirection direction) const { + const auto& info = image.info; + uint32_t format = info.guest_format; + uint32_t layers = info.TransferLayers(); + bool volume = info.IsVolume(); + bool layered = info.IsLayered(); + bool allow_depth_tile = direction == TransferDirection::Upload; + const char* owner = + direction == TransferDirection::Upload ? "TextureCache" : "TextureCache readback"; -void TextureCache::UnregisterImageLocked(CachedImage& image, bool release_tracking) { - if (!image.registered) { - EXIT("TextureCache: unregistering an unregistered image\n"); + ColorTransferPlan plan; + if (direction == TransferDirection::Upload) { + switch (binding) { + case BindingType::Texture: break; + case BindingType::Storage: owner = "StorageTextureCache"; break; + case BindingType::RenderTarget: + case BindingType::VideoOut: + if (info.resources.layers == 0 || info.data.size % info.resources.layers != 0 || + info.samples != 1 || image.backing.samples != 1 || + (binding == BindingType::VideoOut && + info.metadata.compression != VideoOutCompression::Uncompressed)) { + EXIT("TextureCache: invalid color-attachment upload\n"); + } + format = binding == BindingType::RenderTarget + ? ImageOps::RenderTargetTransferFormat(info.bytes_per_block) + : info.guest_format; + layers = info.resources.layers; + volume = false; + layered = layers > 1; + allow_depth_tile = false; + plan.swap_bgra16 = info.bgra16; + owner = binding == BindingType::RenderTarget ? "RenderTarget" : "VideoOut"; + break; + case BindingType::DepthTarget: return plan; + } + } else { + if (binding == BindingType::DepthTarget) { + return plan; + } + format = binding == BindingType::RenderTarget + ? ImageOps::RenderTargetTransferFormat(info.bytes_per_block) + : info.guest_format; + allow_depth_tile = binding == BindingType::Storage; + plan.swap_bgra16 = info.bgra16; } - std::vector final_releases; - if (!m_image_owner_index.Unregister(&image, final_releases)) { - EXIT("TextureCache: image missing from owner index\n"); - } - if (release_tracking) { - for (const auto& release: final_releases) { - m_memory_tracker.UntrackMemory(release.address, release.size); + + plan.layout = TextureCalcUploadLayout(format, info.extent.width, info.extent.height, + info.resources.levels, layers, info.pitch, info.tile_mode, + info.data.size, allow_depth_tile, volume, owner); + plan.regions = TextureBuildImageCopies(plan.layout, info.extent.width, info.extent.height, + layers, info.resources.levels, layered, volume); + plan.tiled = static_cast(plan.layout.tile) != Prospero::TileMode::kLinear; + if (plan.tiled) { + // CMASK/FMASK decoding is intentionally deferred. + if (plan.layout.tile_family == TileBlockFamily::Depth64KB && + Prospero::IsFmaskTextureFormat(format)) { + return plan; + } + if (!TextureBuildGpuTileInfos(info.data.size, plan.regions, plan.layout, format, layers, + info.resources.levels, plan.tiles)) { + return plan; } } - image.registered = false; + plan.valid = true; + return plan; } -VulkanImage& TextureCache::PublishImage(CommandBuffer& command, - std::shared_ptr image) { - if (image == nullptr || image->image == nullptr || image->registered) { - EXIT("TextureCache: invalid image publication, record=%p image=%p " - "registered=%d\n", - static_cast(image.get()), - image != nullptr ? static_cast(image->image) : nullptr, - image != nullptr && image->registered); +TextureCache::DownloadPlan TextureCache::BuildDownload(const Image& image) const { + const auto& info = image.info; + DownloadPlan plan {.depth = info.IsDepth()}; + if (info.samples != 1 || image.backing.samples != 1) { + return plan; } - auto& result = *image->image; - command.RetainResourceUntilFence(image); - m_images.push_back(std::move(image)); - RegisterImageLocked(*m_images.back()); - return result; -} - -std::shared_ptr TextureCache::GetImageOwner(const VulkanImage& image) { - FaultSafeTextureLock lock(this, m_lock); - const auto owner = std::ranges::find_if( - m_images, [&](const auto& cached) { return cached->image == ℑ }); - if (owner == m_images.end()) { - EXIT("TextureCache: image has no cache owner, image=%p\n", - static_cast(&image)); + if (plan.depth) { + plan.valid = IsSupportedDepthPlaneReadback(info) && info.resources.layers != 0 && + info.data.size % info.resources.layers == 0 && + Prospero::NumBytesPerElement(info.guest_format) == info.bytes_per_block; + return plan; } - return *owner; + if (info.metadata.compression != VideoOutCompression::Uncompressed) { + return plan; + } + plan.color = BuildColorTransfer(image, UploadBinding(image), TransferDirection::Download); + plan.valid = plan.color.valid; + return plan; } -std::vector -TextureCache::FindImagesInRegionLocked(uint64_t vaddr, uint64_t size, bool page_overlap) { - return page_overlap ? m_image_owner_index.QueryCandidates(vaddr, size) - : m_image_owner_index.Query(vaddr, size); -} - -struct TextureCache::ReadbackWorker { - enum class State : uint32_t { - Idle, - Claimed, - Requested, - Ready, - Installed, - Completed, - Stopping, - Stopped +void TextureCache::UploadImage(Image& image, const ImageDesc& desc, Buffer& source, + uint64_t source_offset) { + const auto& info = image.info; + const auto upload = [&](std::vector& copies, TileManager::Result linear) { + for (auto& copy: copies) { + copy.bufferOffset += linear.offset; + } + image.Upload(copies, linear.buffer, linear.offset, linear.size); }; - struct ReadbackRange { - uint64_t address; - uint64_t size; - }; - struct ReadbackTransfer { - std::array ranges {}; - uint32_t count = 0; - void Add(uint64_t address, uint64_t size) { - if (address == 0 || size == 0 || count == ranges.size()) { - EXIT("TextureCache: invalid image readback transfer range\n"); - } - ranges[count++] = {address, size}; + if (desc.type != BindingType::DepthTarget) { + auto plan = BuildColorTransfer(image, desc.type, TransferDirection::Upload); + EXIT_NOT_IMPLEMENTED(!plan.valid); + TileManager::Result linear {source.Handle(), source_offset, info.data.size}; + if (plan.tiled) { + linear = m_tiler->Detile(source.Handle(), source_offset, info.data.size, info.data.size, + plan.tiles); } - [[nodiscard]] std::span Ranges() const { - return {ranges.data(), count}; - } - }; - static_assert(std::atomic::is_always_lock_free); - - explicit ReadbackWorker(TextureCache& owner): cache(owner), thread([this] { Run(); }) {} - - ~ReadbackWorker() { - auto expected = State::Idle; - if (!state.compare_exchange_strong(expected, State::Stopping, std::memory_order_acq_rel)) { - EXIT("TextureCache: cannot stop readback worker from state %u\n", - static_cast(expected)); - } - state.notify_all(); - thread.join(); - if (state.load(std::memory_order_acquire) != State::Stopped) { - EXIT("TextureCache: readback worker did not reach stopped state\n"); + if (plan.swap_bgra16) { + linear = m_tiler->SwapBgra16(linear); } + upload(plan.regions, linear); + return; } - void Request(PageFaultAccess fault_access, uint64_t fault_vaddr, uint64_t fault_size) noexcept { - const bool command_thread = GraphicsRunIsCommandProcessorThread(); - const bool submissions_prepaused_now = GraphicsRunSubmissionLockHeld() || command_thread; - const bool unsafe_gpu_lock = GraphicsRunGpuLockHeld() && !submissions_prepaused_now; - if ((fault_access != PageFaultAccess::Read && fault_access != PageFaultAccess::Write) || - unsafe_gpu_lock || LabelInCallback() || g_texture_cache_lock_owner != nullptr || - g_texture_fault_owner != &cache) { - EXIT("TextureCache: unsafe image readback request, access=%u command_thread=%d " - "submission_lock=%d gpu_lock=%d label_callback=%d cache_lock=%p fault_owner=%p\n", - static_cast(fault_access), command_thread, - GraphicsRunSubmissionLockHeld(), GraphicsRunGpuLockHeld(), LabelInCallback(), - g_texture_cache_lock_owner, g_texture_fault_owner); - } - State expected = State::Idle; - while (!state.compare_exchange_weak(expected, State::Claimed, std::memory_order_acq_rel)) { - if (expected == State::Stopping || expected == State::Stopped) { - EXIT("TextureCache: image readback requested while worker is stopping, state=%u\n", - static_cast(expected)); - } - state.wait(expected, std::memory_order_acquire); - expected = State::Idle; - } - access = fault_access; - vaddr = fault_vaddr; - size = fault_size; - submissions_prepaused = submissions_prepaused_now; - state.store(State::Requested, std::memory_order_release); - state.notify_all(); - while (true) { - const auto current = state.load(std::memory_order_acquire); - if (current == State::Ready) { - return; - } - if (current != State::Requested) { - EXIT("TextureCache: invalid state while waiting for image readback, state=%u\n", - static_cast(current)); - } - state.wait(current, std::memory_order_acquire); + if (desc.type != BindingType::DepthTarget || info.samples != 1 || image.backing.samples != 1 || + info.resources.layers == 0 || info.data.size % info.resources.layers != 0 || + Prospero::NumBytesPerElement(info.guest_format) != info.bytes_per_block) { + EXIT("TextureCache: invalid depth upload\n"); + } + TileBlockLayout block {}; + EXIT_NOT_IMPLEMENTED( + !TileGetBlockLayout(TileBlockFamily::Depth64KB, info.bytes_per_block, block)); + const auto layers = info.resources.layers; + const auto full_slice_size = info.data.size / layers; + std::vector tiles; + std::vector copies(layers); + tiles.reserve(layers); + for (uint32_t layer = 0; layer < layers; layer++) { + const uint64_t offset = full_slice_size * layer; + auto& copy = copies[layer]; + copy.bufferOffset = offset; + copy.bufferRowLength = info.pitch; + copy.bufferImageHeight = info.extent.height; + copy.imageSubresource = {vk::ImageAspectFlagBits::eDepth, 0, layer, 1}; + copy.imageExtent = {info.extent.width, info.extent.height, 1}; + if (static_cast(info.tile_mode) != Prospero::TileMode::kLinear) { + tiles.push_back({block.family, block.bytes_per_element, offset, full_slice_size, offset, + full_slice_size, 0, info.extent.width, info.extent.height, 1, + info.pitch}); + tiles.back().surface_z = layer; } } - - [[nodiscard]] bool Complete(PageFaultAccess fault_access, uint64_t fault_vaddr, - uint64_t fault_size) noexcept { - const auto current = state.load(std::memory_order_acquire); - if (current == State::Idle || current == State::Stopping || current == State::Stopped) { - return false; - } - if (current != State::Ready) { - EXIT("TextureCache: active image readback has invalid completion state %u\n", - static_cast(current)); - } - if (access != fault_access || vaddr != fault_vaddr || size != fault_size) { - EXIT("TextureCache: mismatched active image readback completion\n"); - } - state.store(State::Installed, std::memory_order_release); - state.notify_all(); - return true; + TileManager::Result linear {source.Handle(), source_offset, source.Size() - source_offset}; + if (!tiles.empty()) { + linear = + m_tiler->Detile(source.Handle(), source_offset, info.data.size, info.data.size, tiles); } - - [[nodiscard]] bool IsReady(PageFaultAccess fault_access, uint64_t fault_vaddr, - uint64_t fault_size) const noexcept { - return state.load(std::memory_order_acquire) == State::Ready && access == fault_access && - vaddr == fault_vaddr && size == fault_size; + const auto transfer_bytes = DepthAspectTransferBytes(info.pixel_format); + if (transfer_bytes != info.bytes_per_block) { + const uint64_t texels_per_slice = static_cast(info.pitch) * info.extent.height; + EXIT_NOT_IMPLEMENTED(info.bytes_per_block != sizeof(uint16_t) || + transfer_bytes != sizeof(uint32_t) || texels_per_slice > UINT32_MAX || + texels_per_slice > UINT64_MAX / transfer_bytes); + const uint64_t transfer_slice = texels_per_slice * transfer_bytes; + EXIT_NOT_IMPLEMENTED(transfer_slice > UINT64_MAX / layers); + auto promoted = m_tiler->GetScratchBuffer(transfer_slice * layers); + m_tiler->ConvertD16( + linear, promoted, TileManager::D16Direction::Promote, + info.pixel_format == vk::Format::eD32SfloatS8Uint, + {.width = info.extent.width, + .height = info.extent.height, + .layers = layers, + .source_row_stride = static_cast(info.pitch) * sizeof(uint16_t), + .target_row_stride = static_cast(info.pitch) * sizeof(uint32_t), + .source_slice_stride = full_slice_size, + .target_slice_stride = transfer_slice}); + linear = promoted; + for (uint32_t layer = 0; layer < layers; layer++) { + copies[layer].bufferOffset = transfer_slice * layer; + } } + upload(copies, linear); +} - void Release(PageFaultAccess fault_access, uint64_t fault_vaddr, uint64_t fault_size) noexcept { - const auto current = state.load(std::memory_order_acquire); - if (current == State::Idle || current == State::Stopping || current == State::Stopped) { +void TextureCache::InitializeImage(ImageId id, const ImageDesc& desc) { + auto& image = ResolveImage(id); + if (image.info.data.Empty()) { + return; + } + if (image.info.metadata.compression != VideoOutCompression::Uncompressed) { + m_memory_tracker.ForEachUploadRange( + image.info.data.address, image.info.data.size, false, + [](uint64_t, uint64_t) noexcept {}, []() noexcept {}); + if (image.IsCpuDirty()) { + image.RefreshComplete(); + } + return; + } + if (image.info.samples > 1) { + RestoreGpuTracking(image); + return; + } + bool data_gpu_owned = false; + bool data_imported = false; + bool uploaded = false; + m_memory_tracker.ForEachUploadRange( + image.info.data.address, image.info.data.size, false, + [&](uint64_t, uint64_t) noexcept { uploaded = true; }, + [&]() noexcept { + uploaded |= image.IsBufferModified() || image.IsDefinitelyCpuDirty(); + if (!uploaded) { + return; + } + const auto source = + m_buffer_cache.ObtainBufferForImage(image.info.data.address, image.info.data.size); + if (source.buffer == nullptr) { + EXIT("TextureCache: failed to obtain image upload source\n"); + } + data_gpu_owned |= source.gpu_owned; + data_imported = true; + UploadImage(image, desc, *source.buffer, source.offset); + }); + if (data_imported) { + image.ClearBufferModified(); + } + if (data_gpu_owned) { + image.MarkGpuModified(); + } + if (image.IsCpuDirty()) { + image.RefreshComplete(); + } + RestoreGpuTracking(image); +} + +void TextureCache::RefreshImage(ImageId id, const ImageDesc& desc) { + auto& image = ResolveImage(id); + bool unchanged_maybe = false; + if (image.IsMaybeCpuDirty()) { + const auto hash = image.HashGuestEdges(); + if (image.NeedsMaybeCpuHash()) { + image.SetMaybeCpuHash(hash); return; } - if (current != State::Installed) { - EXIT("TextureCache: active image readback has invalid release state %u\n", - static_cast(current)); - } - if (access != fault_access || vaddr != fault_vaddr || size != fault_size) { - EXIT("TextureCache: mismatched active image readback release\n"); - } - state.store(State::Completed, std::memory_order_release); - state.notify_all(); - while (true) { - const auto current = state.load(std::memory_order_acquire); - if (current == State::Idle) { - break; - } - if (current != State::Completed) { - EXIT("TextureCache: invalid state while releasing image readback, state=%u\n", - static_cast(current)); - } - state.wait(current, std::memory_order_acquire); + unchanged_maybe = !image.ResolveMaybeCpuHash(hash); + if (unchanged_maybe) { + m_memory_tracker.ForEachUploadRange( + image.info.data.address, image.info.data.size, false, + [](uint64_t, uint64_t) noexcept {}, []() noexcept {}); } } - - [[nodiscard]] ReadbackTransfer DownloadDepthTarget(CachedImage& cached, - bool require_fault_range = true) { - const auto& info = cached.depth; - if (info.samples != 1 || cached.image->samples != 1) { - EXIT("TextureCache: multisampled depth-image readback is unsupported, samples=%u/%u\n", - info.samples, cached.image->samples); - } - const bool has_stencil = info.stencil_address != 0 || info.stencil_size != 0; - const bool has_htile = info.htile_address != 0 || info.htile_size != 0; - const bool fault_in_depth = vaddr >= info.address && vaddr < info.address + info.size && - size <= info.address + info.size - vaddr; - const bool fault_in_stencil = has_stencil && vaddr >= info.stencil_address && - vaddr < info.stencil_address + info.stencil_size && - size <= info.stencil_address + info.stencil_size - vaddr; - const bool d16 = - info.guest_format == Prospero::GpuEnumValue(Prospero::BufferFormat::k16UNorm) && - info.bytes_per_element == 2; - const bool d32 = - info.guest_format == Prospero::GpuEnumValue(Prospero::BufferFormat::k32Float) && - info.bytes_per_element == 4; - TileSizeAlign expected_stencil {}; - TileSizeAlign expected_htile {}; - TileSizeAlign expected_depth {}; - const bool prospero_layout = - (d16 || d32) && IsSupportedDepthReadbackFormat(info) && - TileGetDepthSize(info.width, info.height, 0, - Prospero::GpuEnumValue(d16 ? Prospero::DepthFormat::kZ16 - : Prospero::DepthFormat::kZ32F), - Prospero::GpuEnumValue(has_stencil - ? Prospero::StencilFormat::k8UInt - : Prospero::StencilFormat::kInvalid), - has_htile, expected_stencil, expected_htile, expected_depth); - const auto expected_pitch = - prospero_layout - ? TileGetTexturePitch(info.guest_format, info.width, 1, - Prospero::GpuEnumValue(Prospero::TileMode::kDepth)) - : 0u; - const auto expected_stencil_pitch = - prospero_layout && has_stencil - ? TileGetTexturePitch(Prospero::GpuEnumValue(Prospero::BufferFormat::k8UInt), - info.width, 1, - Prospero::GpuEnumValue(Prospero::TileMode::kDepth)) - : 0u; - const bool layered_sizes = - info.layers != 0 && info.size <= UINT32_MAX && - expected_depth.size <= UINT64_MAX / info.layers && - info.size == expected_depth.size * info.layers && - (!has_stencil || (info.stencil_size <= UINT32_MAX && - expected_stencil.size <= UINT64_MAX / info.layers && - info.stencil_size == expected_stencil.size * info.layers)) && - (!has_htile || (expected_htile.size <= UINT64_MAX / info.layers && - info.htile_size == expected_htile.size * info.layers)); - if ((require_fault_range && !fault_in_depth && !fault_in_stencil) || info.address == 0 || - info.size == 0 || info.width == 0 || info.height == 0 || info.pitch < info.width || - info.tile_mode != Prospero::GpuEnumValue(Prospero::TileMode::kDepth) || - !prospero_layout || info.pitch != expected_pitch || !layered_sizes || - expected_depth.align != 65536u || cached.image->layers != info.layers || - (has_stencil && (info.stencil_address == 0 || expected_stencil_pitch < info.width || - expected_stencil.align != 65536u)) || - (has_htile && expected_htile.align != 32768u)) { - EXIT("TextureCache: unsupported depth-image readback layout, fault=0x%016" PRIx64 - "+0x%016" PRIx64 " depth=0x%016" PRIx64 "+0x%016" PRIx64 " stencil=0x%016" PRIx64 - "+0x%016" PRIx64 " htile=0x%016" PRIx64 "+0x%016" PRIx64 - " extent=%ux%u pitch=%u/%u layers=%u/%u tile=%u format=%d guest_format=%u bpe=%u " - "expected_depth=0x%016" PRIx64 " expected_stencil=0x%016" PRIx64 - " expected_htile=0x%016" PRIx64 "\n", - vaddr, size, info.address, info.size, info.stencil_address, info.stencil_size, - info.htile_address, info.htile_size, info.width, info.height, info.pitch, - expected_pitch, info.layers, cached.image->layers, info.tile_mode, - static_cast(info.format), info.guest_format, info.bytes_per_element, - expected_depth.size, expected_stencil.size, expected_htile.size); - } - const auto rows = static_cast(info.height - 1); - if (rows > (UINT64_MAX - info.width) / info.pitch || - (has_stencil && rows > (UINT64_MAX - info.width) / expected_stencil_pitch)) { - EXIT("TextureCache: depth-image readback size overflow\n"); - } - const auto depth_linear_elements = rows * info.pitch + info.width; - const auto stencil_linear_elements = rows * expected_stencil_pitch + info.width; - if (depth_linear_elements > UINT64_MAX / info.bytes_per_element) { - EXIT("TextureCache: depth-image readback size overflow\n"); - } - const auto depth_linear_size = depth_linear_elements * info.bytes_per_element; - const auto depth_slice_size = info.size / info.layers; - const auto stencil_slice_size = has_stencil ? info.stencil_size / info.layers : 0; - const bool meta_overlap = - cache.HasMetaOverlapLocked(info.address, info.size) || - (has_stencil && cache.HasMetaOverlapLocked(info.stencil_address, info.stencil_size)); - const bool buffer_overlap = cache.m_buffer_cache.HasPageOverlap(info.address, info.size) || - (has_stencil && cache.m_buffer_cache.HasPageOverlap( - info.stencil_address, info.stencil_size)); - const auto transfer_size = info.size + info.stencil_size; - if (depth_linear_size > depth_slice_size || - (has_stencil && stencil_linear_elements > stencil_slice_size) || - transfer_size > UINT32_MAX || cached.image->format != info.format || - cached.image->extent.width != info.width || - cached.image->extent.height != info.height || meta_overlap || buffer_overlap) { - EXIT("TextureCache: depth-image readback storage is unsupported, " - "depth=0x%016" PRIx64 "+0x%016" PRIx64 " linear=0x%016" PRIx64 - " format=%d/%d extent=%ux%u/%ux%u meta=%d buffer=%d\n", - info.address, info.size, depth_linear_size, static_cast(cached.image->format), - static_cast(info.format), cached.image->extent.width, - cached.image->extent.height, info.width, info.height, meta_overlap, - buffer_overlap); - } - auto regions = Transfer::MakeLayeredImageBufferCopies(info.layers, depth_slice_size, - info.pitch, info.width, info.height, - vk::ImageAspectFlagBits::eDepth); - if (has_stencil) { - auto stencil_regions = Transfer::MakeLayeredImageBufferCopies( - info.layers, stencil_slice_size, expected_stencil_pitch, info.width, info.height, - vk::ImageAspectFlagBits::eStencil); - for (auto& region: stencil_regions) { - region.offset += static_cast(info.size); - } - regions.insert(regions.end(), stencil_regions.begin(), stencil_regions.end()); - } - std::vector gpu_infos; - TileBlockLayout depth_block {}; - TileBlockLayout stencil_block {}; - EXIT_NOT_IMPLEMENTED( - !TileGetBlockLayout(TileBlockFamily::Depth64KB, info.bytes_per_element, depth_block) || - (has_stencil && !TileGetBlockLayout(TileBlockFamily::Depth64KB, 1, stencil_block))); - gpu_infos.reserve(regions.size()); - for (uint32_t layer = 0; layer < info.layers; layer++) { - const uint64_t offset = depth_slice_size * layer; - GpuTileInfo tile {depth_block.family, - depth_block.bytes_per_element, - offset, - depth_slice_size, - offset, - depth_slice_size, - 0, - info.width, - info.height, - 1, - info.pitch}; - tile.surface_z = layer; - gpu_infos.push_back(tile); - } - if (has_stencil) { - for (uint32_t layer = 0; layer < info.layers; layer++) { - const uint64_t offset = info.size + stencil_slice_size * layer; - GpuTileInfo tile {stencil_block.family, - stencil_block.bytes_per_element, - offset, - stencil_slice_size, - offset, - stencil_slice_size, - 0, - info.width, - info.height, - 1, - expected_stencil_pitch}; - tile.surface_z = layer; - gpu_infos.push_back(tile); - } - } - guest.resize(transfer_size); - Transfer::DownloadTiledImage(guest.data(), transfer_size, transfer_size, gpu_infos, regions, - *cached.image, cached.image->layout); - Libs::LibKernel::Memory::WriteBacking(info.address, guest.data(), info.size); - ReadbackTransfer transfer; - transfer.Add(info.address, info.size); - if (has_stencil) { - Libs::LibKernel::Memory::WriteBacking(info.stencil_address, guest.data() + info.size, - info.stencil_size); - transfer.Add(info.stencil_address, info.stencil_size); - } - return transfer; + bool cpu_dirty = image.IsBufferModified() || image.IsDefinitelyCpuDirty(); + if (!unchanged_maybe) { + cpu_dirty |= + m_memory_tracker.IsRegionCpuModified(image.info.data.address, image.info.data.size); } - - [[nodiscard]] ReadbackTransfer DownloadColorImage(CachedImage& cached) { - const bool storage = cached.kind == CachedImage::Kind::StorageTexture; - const bool target = cached.kind == CachedImage::Kind::RenderTarget; - const auto info = - storage ? MakeColorImageTransferInfo(cached.info, VulkanFormat(cached.info.format), - Prospero::NumBytesPerElement(cached.info.format)) - : MakeColorImageTransferInfo(cached.target); - const bool linear = info.tile_mode == Prospero::GpuEnumValue(Prospero::TileMode::kLinear); - const bool tiled_target = target && IsTiledRenderTarget(cached.target); - const bool tiled_storage = - storage && info.tile_mode == Prospero::GpuEnumValue(Prospero::TileMode::kRenderTarget); - bool single_layer_storage = false; - if (storage) { - switch (static_cast(cached.info.type)) { - case Prospero::ImageType::kColor2D: - case Prospero::ImageType::kColor2DArray: - single_layer_storage = cached.info.depth == 1; - break; - default: break; - } + if (image.info.metadata.compression != VideoOutCompression::Uncompressed) { + if (cpu_dirty) { + EXIT("TextureCache: compressed guest image refresh is unsupported\n"); } - const bool basic_storage = - !storage || - (single_layer_storage && cached.info.base_level == 0 && cached.info.levels == 1 && - cached.info.base_array == 0 && (linear || tiled_storage)); - const auto layers = target ? cached.target.layers : 1u; - TileSizeAlign target_mip_layout {}; - const bool target_mip_chain = - target && info.levels > 1 && layers == 1 && tiled_target && - TileGetRenderTargetMipLayout(info.width, info.height, info.pitch, - info.bytes_per_element, info.levels, target_mip_layout, - nullptr, nullptr) && - target_mip_layout.align == 65536 && target_mip_layout.size == info.size && - cached.image != nullptr && cached.image->format == info.format && - cached.image->extent.width == info.width && - cached.image->extent.height == info.height && cached.image->layers == 1 && - cached.image->mip_levels == info.levels && cached.image->samples == 1; - if (info.samples != 1 || (!linear && !tiled_target && !tiled_storage) || !basic_storage || - (info.levels != 1 && !target_mip_chain) || layers == 0 || info.size > UINT32_MAX) { - EXIT("TextureCache: unsupported color-image readback layout, addr=0x%016" PRIx64 - " size=0x%016" PRIx64 - " extent=%ux%u pitch=%u bpe=%u levels=%u samples=%u tile=%u kind=%u\n", - info.address, info.size, info.width, info.height, info.pitch, - info.bytes_per_element, info.levels, info.samples, info.tile_mode, - static_cast(cached.kind)); - } - const auto slice_size = info.size / layers; - const bool meta_overlap = cache.HasMetaOverlapLocked(info.address, info.size); - const bool buffer_overlap = cache.m_buffer_cache.HasPageOverlap(info.address, info.size); - if (meta_overlap || buffer_overlap) { - EXIT("TextureCache: color-image readback storage is unsupported, addr=0x%016" PRIx64 - " size=0x%016" PRIx64 " meta=%d buffer=%d kind=%u\n", - info.address, info.size, meta_overlap, buffer_overlap, - static_cast(cached.kind)); - } - std::vector regions; - std::vector tiled_regions; - TextureUploadLayout tiled_layout {}; - const bool gpu_tiled = tiled_target || tiled_storage; - if (gpu_tiled) { - const auto format = storage - ? cached.info.format - : ImageOps::RenderTargetTransferFormat(info.bytes_per_element); - tiled_layout = TextureCalcUploadLayout(format, info.width, info.height, info.levels, - layers, info.pitch, info.tile_mode, info.size, - false, false, "RenderTargetReadback"); - if (target_mip_chain && - (tiled_layout.tile_family != TileBlockFamily::RenderTarget64KB || - tiled_layout.pitch != info.pitch)) { - EXIT("TextureCache: inconsistent render-target readback layout, addr=0x%016" PRIx64 - " size=0x%016" PRIx64 " pitch=%u/%u levels=%u\n", - info.address, info.size, info.pitch, tiled_layout.pitch, info.levels); - } - tiled_regions = TextureBuildUploadRegions(tiled_layout, info.format, info.width, - info.height, layers, info.levels, layers > 1, - false, TextureUploadDestination::MipLevels); - regions = TextureBuildDownloadRegions(tiled_regions); - } else { - regions = Transfer::MakeLayeredImageBufferCopies(layers, slice_size, info.pitch, - info.width, info.height); - } - if (gpu_tiled) { - std::vector infos; - const auto format = storage - ? cached.info.format - : ImageOps::RenderTargetTransferFormat(info.bytes_per_element); - guest.resize(info.size); - EXIT_NOT_IMPLEMENTED(!TextureBuildGpuTileInfos(info.size, tiled_regions, tiled_layout, - format, layers, info.levels, infos)); - Transfer::DownloadTiledImage(guest.data(), info.size, info.size, infos, regions, - *cached.image, cached.image->layout); - Libs::LibKernel::Memory::WriteBacking(info.address, guest.data(), info.size); - } else { - download.resize(info.size); - std::fill(download.begin(), download.end(), 0); - Transfer::DownloadImage(download.data(), info.size, regions, *cached.image, - cached.image->layout); - Libs::LibKernel::Memory::WriteBacking(info.address, download.data(), info.size); - } - ReadbackTransfer transfer; - transfer.Add(info.address, info.size); - return transfer; - } - - void Run() noexcept { - while (true) { - auto current = state.load(std::memory_order_acquire); - while (current == State::Idle) { - state.wait(current, std::memory_order_acquire); - current = state.load(std::memory_order_acquire); - } - if (current == State::Stopping) { - state.store(State::Stopped, std::memory_order_release); - state.notify_all(); - return; - } - if (current != State::Requested) { - EXIT("TextureCache: readback worker received unsupported state %u\n", - static_cast(current)); - } - - std::optional submissions; - if (!submissions_prepaused) { - submissions.emplace(); - } - FaultSafeTextureLock lock(&cache, cache.m_lock); - CachedImage* selected = cache.FindGpuReadbackPageCandidateLocked(vaddr, size); - const bool render_target = - selected != nullptr && selected->kind == CachedImage::Kind::RenderTarget; - const bool storage_texture = - selected != nullptr && selected->kind == CachedImage::Kind::StorageTexture; - const bool depth_target = - selected != nullptr && selected->kind == CachedImage::Kind::DepthTarget; - if ((!render_target && !storage_texture && !depth_target) || !selected->gpu_modified || - selected->buffer_modified || selected->image == nullptr) { - EXIT("TextureCache: unsupported GPU image readback owner, addr=0x%016" PRIx64 - " size=0x%016" PRIx64 " access=%u image=%p kind=%u gpu_modified=%d " - "buffer_modified=%d vulkan_image=%p\n", - vaddr, size, static_cast(access), static_cast(selected), - selected != nullptr ? static_cast(selected->kind) : UINT32_MAX, - selected != nullptr && selected->gpu_modified, - selected != nullptr && selected->buffer_modified, - selected != nullptr ? static_cast(selected->image) : nullptr); - } - - const auto transfer = - depth_target ? DownloadDepthTarget(*selected) : DownloadColorImage(*selected); - - if (!cache.m_memory_tracker.IsRegionGpuModified(vaddr, size)) { - EXIT("TextureCache: readback fault page is not GPU-modified, addr=0x%016" PRIx64 - " size=0x%016" PRIx64 "\n", - vaddr, size); - } - const auto fault_page = vaddr & ~(TRACKER_PAGE_SIZE - 1); - const auto page_end = fault_page + TRACKER_PAGE_SIZE; - for (const auto& range: transfer.Ranges()) { - const auto range_end = range.address + range.size; - if (fault_page < range_end && page_end > range.address) { - if (fault_page > range.address) { - cache.m_memory_tracker.ForEachDownloadRange( - range.address, fault_page - range.address, - [](uint64_t, uint64_t) noexcept {}); - } - if (page_end < range_end) { - cache.m_memory_tracker.ForEachDownloadRange( - page_end, range_end - page_end, [](uint64_t, uint64_t) noexcept {}); - } - } else { - cache.m_memory_tracker.ForEachDownloadRange( - range.address, range.size, [](uint64_t, uint64_t) noexcept {}); - } - } - - state.store(State::Ready, std::memory_order_release); - state.notify_all(); - while ((current = state.load(std::memory_order_acquire)) != State::Completed) { - if (current != State::Ready && current != State::Installed) { - EXIT("TextureCache: invalid image readback completion state %u\n", - static_cast(current)); - } - state.wait(current, std::memory_order_acquire); - } - for (const auto& range: transfer.Ranges()) { - if (cache.m_memory_tracker.IsRegionGpuModified(range.address, range.size)) { - EXIT("TextureCache: completed image readback retained GPU ownership, " - "addr=0x%016" PRIx64 " size=0x%016" PRIx64 "\n", - range.address, range.size); - } - } - selected->gpu_modified = false; - submissions_prepaused = false; - state.store(State::Idle, std::memory_order_release); - state.notify_all(); - } - } - - TextureCache& cache; - std::atomic state {State::Idle}; - PageFaultAccess access = PageFaultAccess::Unknown; - uint64_t vaddr = 0; - uint64_t size = 0; - bool submissions_prepaused = false; - std::vector download; - std::vector guest; - std::thread thread; -}; - -void TextureCache::RequireRetirementIsolation(const std::vector& retire, - const char* operation, uint64_t address, - uint64_t size) const { - if (retire.empty()) { + RestoreGpuTracking(image); return; } - std::vector ranges; - ranges.reserve(m_images.size() * 2); - for (const auto& cached: m_images) { - const bool retiring = std::find(retire.begin(), retire.end(), cached.get()) != retire.end(); - for (uint32_t range = 0; range < cached->RangeCount(); range++) { - ranges.push_back({cached->Address(range), cached->Size(range), retiring}); - } - } - const auto conflict = FindImageRetirementConflict(ranges); - if (conflict.Exists()) { - const auto& retired = ranges[conflict.retired]; - const auto& retained = ranges[conflict.retained]; - EXIT("TextureCache: %s retirement leaves a tracked page alias, request=0x%016" PRIx64 - "+0x%016" PRIx64 " retired=0x%016" PRIx64 "+0x%016" PRIx64 " retained=0x%016" PRIx64 - "+0x%016" PRIx64 "\n", - operation, address, size, retired.address, retired.size, retained.address, - retained.size); - } -} - -void TextureCache::RetireImages(const std::vector& retire, - const CachedImage* native_image_source) { - if (retire.empty()) { + if (!cpu_dirty) { + RestoreGpuTracking(image); return; } - // FindTexture and FindRenderTarget retain cache records on their recording commands. Erasing - // the lookup owner therefore defers Vulkan destruction until the last referencing fence. - size_t removed = 0; - bool native_image_retired = false; - for (auto it = m_images.begin(); it != m_images.end();) { - if (std::find(retire.begin(), retire.end(), it->get()) == retire.end()) { - ++it; - continue; + InitializeImage(id, desc); +} + +void TextureCache::AssociateStencil(ImageId depth_id, GuestRange stencil) { + std::lock_guard transaction(m_resource_mutex); + CacheLock lock(*this, m_lock); + AssociateStencilLocked(depth_id, stencil); +} + +void TextureCache::AssociateStencilLocked(ImageId depth_id, GuestRange stencil) { + if (!stencil.Valid()) { + EXIT("TextureCache: invalid stencil association range\n"); + } + auto& depth = ResolveImage(depth_id); + if (!depth.info.IsDepth() || !depth.info.HasStencil()) { + EXIT("TextureCache: stencil association requires a depth/stencil image\n"); + } + + ImageId association {}; + for (const auto id: FindImagesInRegion(stencil.address, stencil.size, false)) { + const auto owner = ResolveOwner(id); + if (owner != nullptr && owner->info.data.address == stencil.address) { + association = id; } - const bool sampled = (*it)->kind == CachedImage::Kind::Texture; - const bool storage = (*it)->kind == CachedImage::Kind::StorageTexture; - const bool target = (*it)->kind == CachedImage::Kind::RenderTarget || - (*it)->kind == CachedImage::Kind::DepthTarget; - const bool native_image = it->get() == native_image_source; - if (native_image) { - bool source_valid = (*it)->gpu_modified && !(*it)->buffer_modified; - switch ((*it)->kind) { - case CachedImage::Kind::StorageTexture: - source_valid = source_valid && !(*it)->info.IsCpuDirty(); - break; - case CachedImage::Kind::RenderTarget: - case CachedImage::Kind::DepthTarget: break; - case CachedImage::Kind::Texture: - case CachedImage::Kind::VideoOut: source_valid = false; break; + } + if (!association) { + ImageInfo info {}; + info.data = stencil; + info.extent = depth.info.extent; + association = InsertImage(info); + } + auto& record = ResolveImage(association); + TouchImage(record); + record.AssociateDepth(depth_id); +} + +ImageId TextureCache::FindImage(ImageDesc& desc, bool exact_format) { + auto& command = m_scheduler.Current(); + if (command.IsInvalid()) { + EXIT("TextureCache: image lookup requires a valid command buffer\n"); + } + ValidateImageDesc(desc); + if (desc.info.data.Empty()) { + CacheLock lock(*this, m_lock); + return GetNullImage(desc); + } + + ImageId result {}; + bool replacement_buffer = false; + bool replacing_image = false; + { + std::lock_guard transaction(m_resource_mutex); + CacheLock lock(*this, m_lock); + const std::vector candidates = + FindImagesInRegion(desc.info.data.address, desc.info.data.size, false); + + for (const auto id: candidates) { + const auto owner = ResolveOwner(id); + if (owner == nullptr || owner->info.data != desc.info.data) { + continue; } - for (uint32_t range = 0; source_valid && range < (*it)->RangeCount(); range++) { - source_valid = - m_memory_tracker.IsRegionGpuModified((*it)->Address(range), (*it)->Size(range)); - } - if (!source_valid) { - EXIT("TextureCache: invalid native image retirement, kind=%u gpu_modified=%d " - "buffer_modified=%d cpu_dirty=%d\n", - static_cast((*it)->kind), (*it)->gpu_modified, - (*it)->buffer_modified, - (*it)->kind == CachedImage::Kind::StorageTexture && (*it)->info.IsCpuDirty()); - } - (*it)->gpu_modified = false; - native_image_retired = true; - } - if ((!sampled && !storage && !target && !native_image) || (*it)->gpu_modified || - (storage && ((*it)->buffer_modified || (*it)->info.IsCpuDirty())) || - (target && (*it)->buffer_modified)) { - EXIT("TextureCache: invalid image retirement, kind=%u gpu_modified=%d " - "buffer_modified=%d\n", - static_cast((*it)->kind), (*it)->gpu_modified, (*it)->buffer_modified); - } - for (uint32_t range = 0; range < (*it)->RangeCount(); range++) { - if (target && !native_image && - m_memory_tracker.IsRegionGpuModified((*it)->Address(range), (*it)->Size(range))) { - EXIT("TextureCache: clean target retirement retained tracker GPU ownership, " - "addr=0x%016" PRIx64 " size=0x%016" PRIx64 "\n", - (*it)->Address(range), (*it)->Size(range)); + if (SameBacking(owner->info, desc.info, exact_format)) { + result = id; } } - // A native transition keeps tracker ownership continuous until the replacement is - // published. Normal retirement only releases 4 KiB pages with no remaining image owner. - UnregisterImageLocked(**it, !native_image); - it = m_images.erase(it); - removed++; - } - if (removed != retire.size()) { - EXIT("TextureCache: image retirement set mismatch, requested=%zu removed=%zu\n", - retire.size(), removed); - } - if (native_image_source != nullptr && !native_image_retired) { - EXIT("TextureCache: native image retirement source was not retired\n"); - } -} -void TextureCache::RetireDepthMetadataLocked(const std::vector& retire, - uint64_t preserve_address) { - std::vector addresses; - for (auto* cached: retire) { - if (cached->kind != CachedImage::Kind::DepthTarget || cached->depth.htile_address == 0) { - continue; + int32_t view_mip = -1; + int32_t view_layer = -1; + if (!result) { + for (const auto candidate: candidates) { + view_mip = -1; + view_layer = -1; + const auto owner = ResolveOwner(candidate); + if (owner == nullptr) { + continue; + } + const auto merged_info = result ? ResolveImage(result).info : desc.info; + const auto overlap = ResolveOverlap(merged_info, desc.type, candidate, result); + if (overlap.image) { + result = overlap.image; + view_mip = overlap.mip; + view_layer = overlap.layer; + } + } } - const bool retained_owner = - std::any_of(m_images.begin(), m_images.end(), [&](const auto& other) { - return std::find(retire.begin(), retire.end(), other.get()) == retire.end() && - other->kind == CachedImage::Kind::DepthTarget && - other->depth.htile_address == cached->depth.htile_address && - other->depth.htile_size == cached->depth.htile_size; - }); - if (!retained_owner && cached->depth.htile_address != preserve_address && - std::find(addresses.begin(), addresses.end(), cached->depth.htile_address) == - addresses.end()) { - addresses.push_back(cached->depth.htile_address); + + if (result) { + auto& resolved = ResolveImage(result); + if (exact_format && resolved.info.pixel_format != desc.info.pixel_format) { + result = {}; + } else if (resolved.info.resources < desc.info.resources) { + ImageDesc refresh { + .info = resolved.info, .view_info = {}, .type = UploadBinding(resolved)}; + RefreshImage(result, refresh); + if (resolved.IsGpuModified() && !SynchronizeImageToBuffer(result)) { + EXIT("TextureCache: cannot preserve an unsupported replacement image\n"); + } + replacement_buffer = resolved.IsBufferModified(); + DeleteImage(result); + result = {}; + replacing_image = true; + } } - } - for (const auto address: addresses) { - auto metadata = m_surface_metas.find(address); - auto owner = std::find_if(retire.begin(), retire.end(), [&](const auto* cached) { - return cached->kind == CachedImage::Kind::DepthTarget && - cached->depth.htile_address == address; - }); - if (metadata == m_surface_metas.end() || owner == retire.end() || - metadata->second.size != (*owner)->depth.htile_size) { - EXIT("TextureCache: retiring depth target has invalid HTile metadata, " - "addr=0x%016" PRIx64 " size=0x%016" PRIx64 "\n", - address, owner != retire.end() ? (*owner)->depth.htile_size : 0); + if (!result) { + result = InsertImage(desc.info); + auto& inserted = ResolveImage(result); + if (replacement_buffer || m_buffer_cache.HasGpuDirtyBytes(inserted.info.data.address, + inserted.info.data.size)) { + inserted.MarkBufferModified(); + } else if (replacing_image) { + m_memory_tracker.MarkRegionAsCpuModified(inserted.info.data.address, + inserted.info.data.size); + } + InitializeImage(result, desc); + } else { + RefreshImage(result, desc); } - if (metadata->second.gpu_modified) { - m_metadata_tracker.ForEachDownloadRange(metadata->first, metadata->second.size, - [](uint64_t, uint64_t) noexcept {}); + + auto& image = ResolveImage(result); + if (desc.type == BindingType::VideoOut && + desc.info.metadata.compression != VideoOutCompression::Uncompressed) { + const bool guest_dirty = + image.IsBufferModified() || image.IsCpuDirty() || + m_memory_tracker.IsRegionCpuModified(image.info.data.address, image.info.data.size); + const bool native_current = + (image.usage.render_target || image.IsGpuModified()) && !guest_dirty; + if (!native_current) { + EXIT("TextureCache: compressed video-out read requires clean native GPU " + "contents\n"); + } } - m_metadata_tracker.UntrackMemory(metadata->first, metadata->second.size); - m_surface_metas.erase(metadata); - } -} - -void TextureCache::MaterializeImagesToGuestLocked( - const std::vector>& images) { - if (std::any_of(images.begin(), images.end(), - [](const auto& cached) { return cached->gpu_modified; })) { - Transfer::WaitForQueueIdle(); - } - for (const auto& cached: images) { - if (!cached->gpu_modified) { - continue; + if (view_mip >= 0) { + desc.view_info.base_level = static_cast(view_mip); } - if (cached->buffer_modified) { - EXIT("TextureCache: image recreation has invalid ownership, kind=%u " - "buffer_modified=%d\n", - static_cast(cached->kind), cached->buffer_modified); + if (view_layer >= 0) { + desc.view_info.base_layer = static_cast(view_layer); } - if (cached->kind != CachedImage::Kind::DepthTarget && - cached->kind != CachedImage::Kind::RenderTarget && - cached->kind != CachedImage::Kind::StorageTexture) { - EXIT("TextureCache: unsupported image recreation source kind %u\n", - static_cast(cached->kind)); - } - const auto transfer = cached->kind == CachedImage::Kind::DepthTarget - ? m_readback->DownloadDepthTarget(*cached, false) - : m_readback->DownloadColorImage(*cached); - for (const auto& range: transfer.Ranges()) { - m_memory_tracker.ForEachDownloadRange(range.address, range.size, - [](uint64_t, uint64_t) noexcept {}); - } - cached->gpu_modified = false; - } -} - -namespace { - -bool Equal(const ImageInfo& left, const ImageInfo& right) { - return left.address == right.address && left.size == right.size && - left.format == right.format && left.width == right.width && - left.height == right.height && left.pitch == right.pitch && - left.base_level == right.base_level && left.levels == right.levels && - left.view_levels == right.view_levels && left.tile == right.tile && - left.swizzle == right.swizzle && left.depth == right.depth && left.type == right.type && - left.base_array == right.base_array; -} - -bool EqualStorageBacking(const ImageInfo& left, const ImageInfo& right) { - return left.address == right.address && left.size == right.size && - left.format == right.format && left.width == right.width && - left.height == right.height && left.pitch == right.pitch && - left.levels == right.levels && left.tile == right.tile && left.depth == right.depth && - left.type == right.type && left.base_array == right.base_array; -} - -bool Equal(const RenderTargetInfo& left, const RenderTargetInfo& right) { - return left.address == right.address && left.size == right.size && - left.format == right.format && left.width == right.width && - left.height == right.height && left.pitch == right.pitch && - left.bytes_per_element == right.bytes_per_element && left.tile_mode == right.tile_mode && - left.levels == right.levels && left.layers == right.layers && - left.samples == right.samples; -} - -bool Equal(const DepthTargetInfo& left, const DepthTargetInfo& right) { - return left.address == right.address && left.size == right.size && - left.stencil_address == right.stencil_address && - left.stencil_size == right.stencil_size && left.htile_address == right.htile_address && - left.htile_size == right.htile_size && left.format == right.format && - left.guest_format == right.guest_format && left.width == right.width && - left.height == right.height && left.pitch == right.pitch && - left.bytes_per_element == right.bytes_per_element && left.tile_mode == right.tile_mode && - left.layers == right.layers && left.samples == right.samples && - left.stencil_htile_compressed == right.stencil_htile_compressed; -} - -[[nodiscard]] bool IsCoherentGuestImageSource(const BufferImageCopySource& source, uint64_t address, - uint64_t size) { - // The GPU tiler currently consumes coherent guest backing directly. - return source.cpu_current && source.address == address && source.size == size && - (source.buffer != nullptr || source.offset == 0); -} - -[[nodiscard]] BufferImageCopySource SelectSourceRange(const BufferImageCopySource& source, - uint64_t address, uint64_t size) { - if (address < source.address || size > source.size || - address - source.address > source.size - size || - (source.buffer != nullptr && source.offset > UINT64_MAX - (address - source.address))) { - EXIT("TextureCache: image source subrange is invalid\n"); - } - const auto offset = source.buffer != nullptr ? source.offset + address - source.address : 0; - return {source.buffer, offset, address, size, source.cpu_current}; -} - -[[nodiscard]] DepthTargetInfo SelectDepthLayers(const DepthTargetInfo& info, uint32_t base_layer, - uint32_t layer_count) { - if (info.layers == 0 || base_layer >= info.layers || layer_count == 0 || - layer_count > info.layers - base_layer || info.size % info.layers != 0 || - info.stencil_size % info.layers != 0 || info.htile_size % info.layers != 0) { - EXIT("TextureCache: depth layer range is invalid\n"); - } - auto result = info; - result.layers = layer_count; - for (auto [address, size]: {std::pair {&result.address, &result.size}, - std::pair {&result.stencil_address, &result.stencil_size}, - std::pair {&result.htile_address, &result.htile_size}}) { - const auto slice_size = *size / info.layers; - *address += slice_size * base_layer; - *size = slice_size * layer_count; + image.tick_accessed_last = m_scheduler.CurrentTick(); + TouchImage(image); + RetainImage(command, result); } return result; } -void AppendLayerCopies(std::vector& regions, VulkanImage& source, - vk::ImageAspectFlags aspect) { - for (uint32_t layer = 0; layer < source.layers; layer++) { - for (uint32_t level = 0; level < source.mip_levels; level++) { - ImageImageCopy region {source}; - region.src_aspect = aspect; - region.dst_aspect = aspect; - region.src_level = level; - region.dst_level = level; - region.src_layer = layer; - region.dst_layer = layer; - region.width = std::max(source.extent.width >> level, 1u); - region.height = std::max(source.extent.height >> level, 1u); - regions.push_back(region); - } +ImageId TextureCache::FindImageFromRange(uint64_t address, uint64_t size, bool ensure_valid) { + if (!GuestRange {address, size}.Valid()) { + return {}; } -} - -} // namespace - -TextureCache::TextureCache(GraphicContext& graphics, PageManager& page_manager, - BufferCache& buffer_cache, ResourceMutex& resource_mutex) - : m_graphics(graphics), m_memory_tracker(page_manager), - m_metadata_tracker(page_manager, PageWatchMode::Write), m_buffer_cache(buffer_cache), - m_resource_mutex(resource_mutex) { - if (!Common::Thread::IsMainThread()) { - EXIT("TextureCache: construction is restricted to the main thread\n"); - } - m_dummy_textures = std::make_unique(); - m_readback = std::make_unique(*this); -} - -TextureCache::CachedImage* TextureCache::FindGpuReadbackPageCandidateLocked(uint64_t vaddr, - uint64_t size) { - CachedImage* selected = nullptr; - for (auto* cached: FindImagesInRegionLocked(vaddr, size, true)) { - if (!cached->IsGpuReadbackPageCandidate(vaddr, size)) { + CacheLock lock(*this, m_lock); + std::vector matches; + for (const auto id: FindImagesInRegion(address, size, false)) { + auto owner = ResolveOwner(id); + if (owner == nullptr || owner->info.data.address != address) { continue; } - if (selected != nullptr) { - EXIT("TextureCache: CPU fault has multiple GPU-modified image page candidates, " - "addr=0x%016" PRIx64 " size=0x%016" PRIx64 " first=%p second=%p\n", - vaddr, size, static_cast(selected), static_cast(cached)); + if (ensure_valid && owner->depth_id) { + owner = ResolveOwner(owner->depth_id); } - selected = cached; + if (owner == nullptr || (ensure_valid && !owner->SafeToDownload())) { + continue; + } + matches.push_back(id); + } + ImageId selected {}; + if (matches.size() == 1) { + selected = matches.front(); + } else { + for (const auto id: matches) { + const auto& image = ResolveImage(id); + if (image.info.data.size == size) { + selected = id; + break; + } + } + } + if (selected) { + if (ensure_valid) { + const auto owner = ResolveOwner(selected); + if (owner != nullptr && owner->depth_id) { + selected = owner->depth_id; + } + } + RetainImage(m_scheduler.Current(), selected); } return selected; } -void TextureCache::MarkSampledAliasesCpuDirtyLocked(uint64_t vaddr, uint64_t size) { - for (auto* cached: FindImagesInRegionLocked(vaddr, size, true)) { - if (cached->kind != CachedImage::Kind::Texture) { - continue; - } - if (cached->gpu_modified && - ImageRangeOverlaps(cached->info.address, cached->info.size, vaddr, size)) { - EXIT("TextureCache: CPU write overlaps a GPU-modified sampled texture, " - "write=0x%016" PRIx64 "+0x%016" PRIx64 " image=0x%016" PRIx64 "+0x%016" PRIx64 - "\n", - vaddr, size, cached->info.address, cached->info.size); - } - cached->info.InvalidateCpuWrite(vaddr, size); - if (cached->info.NeedsMaybeCpuHash()) { - cached->info.SetMaybeCpuHash(HashSampledImageEdges(cached->info)); +vk::ImageView TextureCache::FindTexture(ImageId id, const ImageDesc& desc) { + std::lock_guard transaction(m_resource_mutex); + CacheLock lock(*this, m_lock); + auto& image = ResolveImage(id); + TouchImage(image); + if (!image.info.data.Empty()) { + if (!image.registered || image.depth_id || image.binding.needs_rebind) { + EXIT("TextureCache: texture requires rediscovery before final acquisition\n"); } + RefreshImage(id, desc); } + switch (desc.type) { + case BindingType::Texture: break; + case BindingType::Storage: + if (image.info.data.Empty()) { + image.MarkGpuModified(); + } else { + if (!image.registered || image.depth_id) { + EXIT("TextureCache: cannot acquire an unavailable storage image\n"); + } + CommitGpuWrite(image); + } + TrackImageDownloadLocked(id, image); + break; + default: EXIT("TextureCache: invalid texture binding\n"); + } + const auto view = image.FindView(desc.view_info); + RetainImage(m_scheduler.Current(), id); + return view; } -void TextureCache::RetireSampledTargetAliases(const ImageInfo& requested) { - std::vector retire; - bool wait_idle = false; - for (auto* cached: FindImagesInRegionLocked(requested.address, requested.size, true)) { - if (cached->kind != CachedImage::Kind::RenderTarget) { - continue; - } - switch (ClassifySampledRenderTargetOverlap(requested, cached->target, - cached->buffer_modified)) { - case RenderTargetOverlap::None: continue; - case RenderTargetOverlap::RetireTarget: - wait_idle |= cached->gpu_modified; - retire.push_back(cached); - break; - case RenderTargetOverlap::RetireSampled: - case RenderTargetOverlap::RetireStorage: - case RenderTargetOverlap::PreserveStorage: - case RenderTargetOverlap::ExpandTarget: - case RenderTargetOverlap::Unsupported: - EXIT("TextureCache: unsupported sampled/render-target alias, sampled=0x%016" PRIx64 - "+0x%016" PRIx64 " target=0x%016" PRIx64 "+0x%016" PRIx64 - " gpu=%d buffer=%d\n", - requested.address, requested.size, cached->target.address, cached->target.size, - cached->gpu_modified, cached->buffer_modified); - } - } - for (auto* cached: FindImagesInRegionLocked(requested.address, requested.size, true)) { - if (cached->kind != CachedImage::Kind::DepthTarget) { - continue; - } - const bool overlaps_depth = ImageRangeOverlaps(requested.address, requested.size, - cached->depth.address, cached->depth.size); - const bool overlaps_stencil = - cached->depth.stencil_address != 0 && - ImageRangeOverlaps(requested.address, requested.size, cached->depth.stencil_address, - cached->depth.stencil_size); - if (!overlaps_depth && !overlaps_stencil) { - continue; - } - const auto delta = cached->depth.address >= requested.address - ? cached->depth.address - requested.address - : UINT64_MAX; - const bool contained = - delta <= requested.size && cached->depth.size <= requested.size - delta; - const bool sampled_expansion = IsSampledDepthExpansion(requested, cached->depth); - const bool exact_range = - requested.address == cached->depth.address && requested.size == cached->depth.size; - const bool depth_tracker_gpu = - m_memory_tracker.IsRegionGpuModified(cached->depth.address, cached->depth.size); - const bool stencil_tracker_gpu = - cached->depth.stencil_address != 0 && - m_memory_tracker.IsRegionGpuModified(cached->depth.stencil_address, - cached->depth.stencil_size); - const bool clean_pool_alias = CanRetireGuestCurrentDepthForSampled( - requested, cached->depth, cached->gpu_modified, cached->buffer_modified, - depth_tracker_gpu, stencil_tracker_gpu); - const bool native_transition = - !cached->buffer_modified && cached->depth.stencil_address == 0 && - cached->depth.stencil_size == 0 && cached->depth.layers == 1 && contained && - (exact_range || sampled_expansion); - const bool supported = clean_pool_alias || native_transition; - if (!supported) { - EXIT("TextureCache: unsupported sampled/depth-target alias, sampled=0x%016" PRIx64 - "+0x%016" PRIx64 " depth=0x%016" PRIx64 "+0x%016" PRIx64 - " overlaps=%d/%d contained=%d gpu=%d/%d/%d buffer=%d" - " stencil=0x%016" PRIx64 "+0x%016" PRIx64 " layers=%u expansion=%d" - " sampled_info={fmt=%u extent=%ux%u pitch=%u levels=%u/%u tile=%u type=%u}" - " depth_info={fmt=%u host=%d extent=%ux%u pitch=%u tile=%u htile=0x%016" PRIx64 - "+0x%016" PRIx64 "}\n", - requested.address, requested.size, cached->depth.address, cached->depth.size, - overlaps_depth, overlaps_stencil, contained, cached->gpu_modified, - depth_tracker_gpu, stencil_tracker_gpu, cached->buffer_modified, - cached->depth.stencil_address, cached->depth.stencil_size, cached->depth.layers, - sampled_expansion, requested.format, requested.width, requested.height, - requested.pitch, requested.levels, requested.view_levels, requested.tile, - requested.type, cached->depth.guest_format, static_cast(cached->depth.format), - cached->depth.width, cached->depth.height, cached->depth.pitch, - cached->depth.tile_mode, cached->depth.htile_address, cached->depth.htile_size); - } - wait_idle |= cached->gpu_modified; - retire.push_back(cached); - } - if (retire.empty()) { - return; - } - RequireRetirementIsolation(retire, "sampled target", requested.address, requested.size); - if (wait_idle) { - Transfer::WaitForQueueIdle(); - } - for (auto* cached: retire) { - if (!cached->gpu_modified) { - continue; - } - const auto transfer = cached->kind == CachedImage::Kind::DepthTarget - ? m_readback->DownloadDepthTarget(*cached, false) - : m_readback->DownloadColorImage(*cached); - for (const auto& range: transfer.Ranges()) { - m_memory_tracker.ForEachDownloadRange(range.address, range.size, - [](uint64_t, uint64_t) noexcept {}); - } - cached->gpu_modified = false; - } - RetireDepthMetadataLocked(retire); - RetireImages(retire); -} - -void TextureCache::ResolveStorageImageOverlaps(const ImageInfo& requested) { - std::vector retire; - for (auto* cached: FindImagesInRegionLocked(requested.address, requested.size, true)) { - const bool tracker_gpu = - m_memory_tracker.IsRegionGpuModified(cached->Address(), cached->Size()); - switch (ClassifyStorageImageOverlap( - requested.address, requested.size, cached->Address(), cached->Size(), - cached->kind == CachedImage::Kind::Texture, cached->gpu_modified, - cached->buffer_modified, tracker_gpu)) { - case StorageImageOverlap::None: continue; - case StorageImageOverlap::RetireSampled: retire.push_back(cached); continue; - case StorageImageOverlap::PageNeighbor: continue; - case StorageImageOverlap::Unsupported: - EXIT("TextureCache: unsupported storage-image byte alias, requested=0x%016" PRIx64 - "+0x%016" PRIx64 " existing=0x%016" PRIx64 "+0x%016" PRIx64 - " kind=%u gpu=%d/%d buffer=%d\n", - requested.address, requested.size, cached->Address(), cached->Size(), - static_cast(cached->kind), cached->gpu_modified, tracker_gpu, - cached->buffer_modified); - } - } - // Every exact byte overlap was classified above: only clean sampled images are retired and all - // retained byte aliases remain fatal. A retired full mip chain can still contain a cached, - // byte-disjoint subresource outside this storage request. The multi-owner index keeps those - // retained pages tracked and UnregisterImageLocked releases only pages whose final owner left. - for (auto* cached: retire) { - if (!cached->gpu_modified) { - continue; - } - const auto transfer = m_readback->DownloadColorImage(*cached); - for (const auto& range: transfer.Ranges()) { - m_memory_tracker.ForEachDownloadRange(range.address, range.size, - [](uint64_t, uint64_t) noexcept {}); - } - cached->gpu_modified = false; - } - RetireImages(retire); -} - -void TextureCache::RetireStorageDepthAliasLocked(const ImageInfo& requested) { - CachedImage* selected = nullptr; - for (auto* entry: FindImagesInRegionLocked(requested.address, requested.size, true)) { - auto& cached = *entry; - if (cached.kind != CachedImage::Kind::DepthTarget || - !cached.OverlapsRange(requested.address, requested.size, true)) { - continue; - } - const auto& depth = cached.depth; - const bool exact_d32_uint = - cached.gpu_modified && !cached.buffer_modified && depth.address == requested.address && - depth.size == requested.size && depth.stencil_address == 0 && depth.stencil_size == 0 && - depth.htile_address == 0 && depth.htile_size == 0 && depth.layers == 1 && - depth.format == vk::Format::eD32Sfloat && - depth.guest_format == Prospero::GpuEnumValue(Prospero::BufferFormat::k32Float) && - depth.bytes_per_element == 4 && depth.width == requested.width && - depth.height == requested.height && depth.pitch == requested.pitch && - requested.format == Prospero::GpuEnumValue(Prospero::BufferFormat::k32UInt) && - requested.tile == Prospero::GpuEnumValue(Prospero::TileMode::kDepth) && - requested.type == Prospero::GpuEnumValue(Prospero::ImageType::kColor2D) && - requested.depth == 1 && requested.levels == 1; - if (!exact_d32_uint || selected != nullptr) { - EXIT("TextureCache: unsupported storage/depth-target alias, requested=0x%016" PRIx64 - "+0x%016" PRIx64 " depth=0x%016" PRIx64 "+0x%016" PRIx64 - " exact=%d ambiguous=%d\n", - requested.address, requested.size, depth.address, depth.size, exact_d32_uint, - selected != nullptr); - } - selected = &cached; - } - if (selected == nullptr) { - return; - } - Transfer::WaitForQueueIdle(); - const auto transfer = m_readback->DownloadDepthTarget(*selected, false); - for (const auto& range: transfer.Ranges()) { - m_memory_tracker.ForEachDownloadRange(range.address, range.size, - [](uint64_t, uint64_t) noexcept {}); - } - selected->gpu_modified = false; - RetireImages({selected}); -} - -TextureCache::~TextureCache() { - m_readback.reset(); - if (!m_images.empty()) { - Transfer::WaitForQueueIdle(); - } - for (const auto& image: m_images) { - UnregisterImageLocked(*image, false); - } - m_images.clear(); - m_dummy_textures.reset(); -} - -VulkanImage& TextureCache::FindTexture(CommandBuffer& command, const ImageInfo& info, - bool metadata_read) { - if (info.address == 0 || info.size == 0 || info.address >= TRACKER_ADDRESS_SIZE || - info.size > TRACKER_ADDRESS_SIZE - info.address || info.width == 0 || info.height == 0 || - info.depth == 0 || info.levels == 0 || info.levels >= 16 || info.view_levels == 0 || - info.base_level + info.view_levels > info.levels) { - EXIT("TextureCache: invalid sampled-image request, command=%p addr=0x%016" PRIx64 - " size=0x%016" PRIx64 " extent=%ux%ux%u levels=%u\n", - static_cast(&command), info.address, info.size, info.width, info.height, - info.depth, info.levels); +vk::ImageView TextureCache::FindRenderTarget(ImageId id, const ImageDesc& desc) { + if (desc.type != BindingType::RenderTarget) { + EXIT("TextureCache: invalid color-target binding\n"); } std::lock_guard transaction(m_resource_mutex); - { - FaultSafeTextureLock lock(this, m_lock); - RetireSampledTargetAliases(info); + CacheLock lock(*this, m_lock); + auto& image = ResolveImage(id); + if (!image.registered || image.depth_id || image.binding.needs_rebind) { + EXIT("TextureCache: color target requires rediscovery before final acquisition\n"); } - BufferImageCopySource source {nullptr, 0, info.address, info.size, true}; - if (m_buffer_cache.HasPageOverlap(info.address, info.size)) { - // ObtainBufferForImage publishes dirty native-buffer bytes into coherent guest backing. - source = m_buffer_cache.ObtainBufferForImage(info.address, info.size); - if (!IsCoherentGuestImageSource(source, info.address, info.size)) { - EXIT("TextureCache: sampled-image buffer source is inconsistent, addr=0x%016" PRIx64 - " size=0x%016" PRIx64 " source=%p source_addr=0x%016" PRIx64 - " source_size=0x%016" PRIx64 " current=%d\n", - info.address, info.size, static_cast(source.buffer), source.address, - source.size, source.cpu_current); - } + TouchImage(image); + RefreshImage(id, desc); + CommitGpuWrite(image); + image.usage.render_target = true; + TrackImageDownloadLocked(id, image); + const auto view = image.FindView(desc.view_info); + RetainImage(m_scheduler.Current(), id); + return view; +} + +vk::ImageView TextureCache::FindDepthTarget(ImageId id, const ImageDesc& desc) { + if (desc.type != BindingType::DepthTarget) { + EXIT("TextureCache: invalid depth-target binding\n"); } - FaultSafeTextureLock lock(this, m_lock); - if (HasMetaOverlapLocked(info.address, info.size) != metadata_read) { - EXIT("TextureCache: sampled-image metadata classification mismatch, addr=0x%016" PRIx64 - " size=0x%016" PRIx64 " requested=%d\n", - info.address, info.size, metadata_read); + std::lock_guard transaction(m_resource_mutex); + CacheLock lock(*this, m_lock); + auto& image = ResolveImage(id); + if (!image.registered || image.depth_id || image.binding.needs_rebind) { + EXIT("TextureCache: depth target requires rediscovery before final acquisition\n"); } - std::shared_ptr storage_match; - std::vector storage_retire; - const auto requested_view_format = TextureGetFormat(info.format); - for (auto& cached: m_images) { - if (cached->kind != CachedImage::Kind::StorageTexture) { + TouchImage(image); + RefreshImage(id, desc); + if (desc.info.HasMetadata()) { + image.info.metadata = desc.info.metadata; + m_surface_metas.try_emplace(desc.info.metadata.range.address, + MetaDataInfo {.clear_mask = image.info.htile_clear_mask}); + } + CommitGpuWrite(image); + image.usage.depth_target = true; + if (desc.info.HasStencil()) { + AssociateStencilLocked(id, desc.info.stencil); + } + const auto view = image.FindView(desc.view_info); + RetainImage(m_scheduler.Current(), id); + return view; +} + +void TextureCache::MarkGpuWritten(ImageId id) { + std::lock_guard transaction(m_resource_mutex); + CacheLock lock(*this, m_lock); + auto& image = ResolveImage(id); + if (!image.registered || image.depth_id) { + EXIT("TextureCache: cannot mark an unavailable image GPU-written\n"); + } + CommitGpuWrite(image); +} + +void TextureCache::CommitGpuWrite(Image& image) { + if (image.depth_id || image.backing.image == nullptr) { + EXIT("TextureCache: stencil association cannot own image contents\n"); + } + const auto range = image.info.data; + if (m_buffer_cache.HasGpuDirtyBytes(range.address, range.size)) { + m_buffer_cache.DiscardGpuDirtyBytes(range.address, range.size); + } + m_buffer_cache.InvalidateImageAliases(range.address, range.size); + image.ClearBufferModified(); + m_memory_tracker.ForEachUploadRange( + range.address, range.size, true, [](uint64_t, uint64_t) noexcept {}, []() noexcept {}); + if (image.IsCpuDirty()) { + image.RefreshComplete(); + } + image.MarkGpuModified(); + RestoreGpuTracking(image); +} + +bool TextureCache::ClearImageFromBuffer(CommandBuffer& command, uint64_t address, uint64_t size, + uint32_t packed_clear) { + if (command.IsInvalid() || !GuestRange {address, size}.Valid()) { + EXIT("TextureCache: invalid image clear\n"); + } + m_buffer_cache.ValidateGpuAccess(address, size, false, true); + std::lock_guard transaction(m_resource_mutex); + CacheLock lock(*this, m_lock); + ImageId selected {}; + vk::ImageAspectFlags aspect {}; + for (const auto id: FindImagesInRegion(address, size, false)) { + auto owner = ResolveOwner(id); + if (owner == nullptr) { continue; } - const bool tracker_gpu = - m_memory_tracker.IsRegionGpuModified(cached->info.address, cached->info.size); - const bool cpu_dirty = - cached->info.IsCpuDirty() || - m_memory_tracker.IsRegionCpuModified(cached->info.address, cached->info.size); - const bool exact_mip = IsExactRenderTargetMipStorage( - info, cached->info, requested_view_format, cached->image->format); - switch (ClassifyStorageSampledOverlap( - info, cached->info, requested_view_format, cached->image->format, cached->gpu_modified, - cpu_dirty, exact_mip, cached->buffer_modified, tracker_gpu)) { - case StorageSampledOverlap::None: break; - case StorageSampledOverlap::ExactImage: - if (storage_match != nullptr) { - EXIT("TextureCache: duplicate exact storage image for sampled binding, " - "addr=0x%016" PRIx64 " size=0x%016" PRIx64 "\n", - info.address, info.size); - } - storage_match = cached; - break; - case StorageSampledOverlap::RetireStorage: - storage_retire.push_back(cached.get()); - break; - case StorageSampledOverlap::Unsupported: - EXIT("TextureCache: unsupported sampled/storage image alias, " - "requested=0x%016" PRIx64 "+0x%016" PRIx64 " storage=0x%016" PRIx64 - "+0x%016" PRIx64 - " gpu_modified=%d buffer_modified=%d tracker_gpu=%d cpu_dirty=%d" - " exact_mip=%d" - " requested_info={format=%u extent=%ux%ux%u pitch=%u base=%u levels=%u" - " view_levels=%u tile=%u swizzle=0x%03x type=%u base_array=%u}" - " storage_info={format=%u extent=%ux%ux%u pitch=%u base=%u levels=%u" - " view_levels=%u tile=%u swizzle=0x%03x type=%u base_array=%u}\n", - info.address, info.size, cached->info.address, cached->info.size, - cached->gpu_modified, cached->buffer_modified, tracker_gpu, cpu_dirty, - exact_mip, info.format, info.width, info.height, info.depth, info.pitch, - info.base_level, info.levels, info.view_levels, info.tile, info.swizzle, - info.type, info.base_array, cached->info.format, cached->info.width, - cached->info.height, cached->info.depth, cached->info.pitch, - cached->info.base_level, cached->info.levels, cached->info.view_levels, - cached->info.tile, cached->info.swizzle, cached->info.type, - cached->info.base_array); - } - } - if (storage_match != nullptr && !storage_retire.empty()) { - EXIT("TextureCache: sampled binding has both exact and mip storage owners, " - "addr=0x%016" PRIx64 " size=0x%016" PRIx64 " mip_owners=%zu\n", - info.address, info.size, storage_retire.size()); - } - if (storage_match != nullptr) { - if (m_memory_tracker.IsRegionCpuModified(info.address, info.size) || - !m_memory_tracker.IsRegionGpuModified(info.address, info.size) || - storage_match->image->type != VulkanImageType::StorageTexture || - storage_match->image->image_view[VulkanImage::VIEW_DEFAULT] == nullptr) { - EXIT("TextureCache: sampled storage-image ownership or view is inconsistent, " - "addr=0x%016" PRIx64 " size=0x%016" PRIx64 "\n", - info.address, info.size); - } - command.RetainResourceUntilFence(storage_match); - return *storage_match->image; - } - if (!storage_retire.empty()) { - // Resolve a modified cached mip before replacing it with the containing image. - // Kyty does not yet copy between those independently allocated Vulkan images, so use the - // existing synchronized tiled readback seam and rebuild the complete chain from coherent - // guest backing. This is an uncommon ownership transition, not a frame lookup fast path. - Transfer::WaitForQueueIdle(); - for (auto* cached: storage_retire) { - if (!cached->gpu_modified || cached->buffer_modified || cached->info.IsCpuDirty() || - !m_memory_tracker.IsRegionGpuModified(cached->info.address, cached->info.size) || - m_memory_tracker.IsRegionCpuModified(cached->info.address, cached->info.size)) { - EXIT("TextureCache: sampled mip-storage ownership changed during transition, " - "addr=0x%016" PRIx64 " size=0x%016" PRIx64 " gpu=%d buffer=%d dirty=%d\n", - cached->info.address, cached->info.size, cached->gpu_modified, - cached->buffer_modified, cached->info.IsCpuDirty()); - } - const auto transfer = m_readback->DownloadColorImage(*cached); - for (const auto& range: transfer.Ranges()) { - m_memory_tracker.ForEachDownloadRange(range.address, range.size, - [](uint64_t, uint64_t) noexcept {}); - } - cached->gpu_modified = false; - } - RetireImages(storage_retire); - } - if (m_memory_tracker.IsRegionCpuModified(info.address, info.size)) { - MarkSampledAliasesCpuDirtyLocked(info.address, info.size); - } - std::shared_ptr match; - for (auto& cached: m_images) { - if (cached->kind != CachedImage::Kind::Texture || !Equal(info, cached->info)) { - continue; - } - if (match != nullptr || cached->gpu_modified) { - EXIT("TextureCache: invalid exact sampled-image cache match, addr=0x%016" PRIx64 - " size=0x%016" PRIx64 " duplicate=%d gpu_modified=%d\n", - info.address, info.size, match != nullptr, cached->gpu_modified); - } - match = cached; - } - if (match != nullptr) { - if (match->info.IsMaybeCpuDirty() && !match->info.IsDefinitelyCpuDirty()) { - const bool changed = - match->info.ResolveMaybeCpuHash(HashSampledImageEdges(match->info)); - if (!changed) { - m_memory_tracker.ForEachUploadRange( - info.address, info.size, false, [](uint64_t, uint64_t) noexcept {}, - []() noexcept {}); - } - } - const bool buffer_dirty = match->buffer_modified; - if (buffer_dirty) { - if (match->gpu_modified || - !IsCoherentGuestImageSource(source, info.address, info.size)) { - EXIT("TextureCache: sampled-image refresh has invalid buffer ownership, " - "addr=0x%016" PRIx64 " size=0x%016" PRIx64 - " source=%p current=%d gpu_modified=%d\n", - info.address, info.size, static_cast(source.buffer), - source.cpu_current, match->gpu_modified); - } - m_memory_tracker.MarkRegionAsCpuModified(info.address, info.size); - } - if (match->info.IsCpuDirty() || buffer_dirty) { - m_memory_tracker.ForEachUploadRange( - info.address, info.size, false, [](uint64_t, uint64_t) noexcept {}, - [&]() noexcept { - m_tiler.DetileImage(*static_cast(match->image), - match->info, source, true, false); - }); - if (match->info.IsCpuDirty()) { - match->info.RefreshComplete(); - } - match->buffer_modified = false; - } - command.RetainResourceUntilFence(match); - return *match->image; - } - for (const auto& cached: m_images) { - if (cached->kind != CachedImage::Kind::Texture) { - if (!cached->OverlapsRange(info.address, info.size, true)) { + vk::ImageAspectFlags candidate {}; + ImageId candidate_id = id; + if (owner->depth_id && owner->info.data.address == address && + owner->info.data.size == size) { + candidate = vk::ImageAspectFlagBits::eStencil; + candidate_id = owner->depth_id; + owner = ResolveOwner(candidate_id); + if (owner == nullptr || owner->backing.image == nullptr || !owner->info.HasStencil()) { continue; } - EXIT("TextureCache: sampled image overlaps GPU target, requested=0x%016" PRIx64 - "+0x%016" PRIx64 " target=0x%016" PRIx64 "+0x%016" PRIx64 - " target_kind=%u requested_info={format=%u extent=%ux%ux%u pitch=%u levels=%u" - " tile=%u type=%u}\n", - info.address, info.size, cached->Address(), cached->Size(), - static_cast(cached->kind), info.format, info.width, info.height, - info.depth, info.pitch, info.levels, info.tile, info.type); + } else if (!owner->depth_id && owner->info.data.address == address && + owner->info.data.size == size) { + candidate = owner->info.IsDepth() ? vk::ImageAspectFlagBits::eDepth + : vk::ImageAspectFlagBits::eColor; } - const auto overlap = ClassifySampledOverlap(info, cached->info, cached->gpu_modified); - if (overlap == SampledOverlap::Unsupported) { - EXIT("TextureCache: unsupported sampled-texture alias, requested=0x%016" PRIx64 - "+0x%016" PRIx64 " existing=0x%016" PRIx64 "+0x%016" PRIx64 " gpu_modified=%d\n", - info.address, info.size, cached->info.address, cached->info.size, - cached->gpu_modified); - } - } - m_images.reserve(m_images.size() + 1); - auto cached = std::make_shared(m_graphics); - cached->kind = CachedImage::Kind::Texture; - cached->info = info; - vk::ComponentMapping components {}; - cached->image = ImageOps::CreateTexture(info, false, components); - m_memory_tracker.ForEachUploadRange( - info.address, info.size, false, [](uint64_t, uint64_t) noexcept {}, - [&]() noexcept { - m_tiler.DetileImage(*static_cast(cached->image), cached->info, - source, false, false); - }); - ImageOps::CreateTextureViews(*static_cast(cached->image), info, false, - components); - return PublishImage(command, std::move(cached)); -} - -StorageTextureVulkanImage& TextureCache::FindStorageTexture(CommandBuffer& command, - const ImageInfo& info) { - const bool supported_type = - info.type == Prospero::GpuEnumValue(Prospero::ImageType::kColor2D) || - info.type == Prospero::GpuEnumValue(Prospero::ImageType::kColor2DArray) || - info.type == Prospero::GpuEnumValue(Prospero::ImageType::kColor3D); - const bool supported_depth_tile = - info.tile == Prospero::GpuEnumValue(Prospero::TileMode::kDepth) && - IsSupportedStorageDepthTile(info.format, info.type, info.width, info.height, info.depth); - if (info.address == 0 || info.size == 0 || info.address >= TRACKER_ADDRESS_SIZE || - info.size > TRACKER_ADDRESS_SIZE - info.address || info.width == 0 || info.height == 0 || - info.depth == 0 || info.levels == 0 || info.levels > 16 || info.base_level >= info.levels || - info.view_levels != 1 || info.base_array != 0 || !supported_type || - (info.tile != Prospero::GpuEnumValue(Prospero::TileMode::kLinear) && - info.tile != Prospero::GpuEnumValue(Prospero::TileMode::kRenderTarget) && - !supported_depth_tile) || - !IsSupportedStorageSwizzle(info.format, info.swizzle)) { - EXIT("TextureCache: unsupported storage-image request, command=%p " - "addr=0x%016" PRIx64 " size=0x%016" PRIx64 - " extent=%ux%ux%u levels=%u base_level=%u base_array=%u type=%u tile=%u " - "swizzle=0x%03x\n", - static_cast(&command), info.address, info.size, info.width, info.height, - info.depth, info.levels, info.base_level, info.base_array, info.type, info.tile, - info.swizzle); - } - if ((info.type == Prospero::GpuEnumValue(Prospero::ImageType::kColor2D) && info.depth != 1) || - (info.type == Prospero::GpuEnumValue(Prospero::ImageType::kColor3D) && info.depth == 0) || - (info.type == Prospero::GpuEnumValue(Prospero::ImageType::kColor2DArray) && - info.base_array >= info.depth)) { - EXIT("TextureCache: storage-image type and depth disagree, type=%u depth=%u\n", info.type, - info.depth); - } - - std::lock_guard transaction(m_resource_mutex); - const bool buffer_overlap = m_buffer_cache.HasPageOverlap(info.address, info.size); - BufferImageCopySource source {nullptr, 0, info.address, info.size, true}; - if (buffer_overlap) { - source = m_buffer_cache.ObtainBufferForImage(info.address, info.size); - if (!IsCoherentGuestImageSource(source, info.address, info.size)) { - EXIT("TextureCache: storage-image buffer source is inconsistent, addr=0x%016" PRIx64 - " size=0x%016" PRIx64 " source=%p source_addr=0x%016" PRIx64 - " source_size=0x%016" PRIx64 " current=%d\n", - info.address, info.size, static_cast(source.buffer), source.address, - source.size, source.cpu_current); - } - } - FaultSafeTextureLock lock(this, m_lock); - ResolveImageMetadataOverlapsLocked(info.address, info.size); - if (supported_depth_tile && - info.format == Prospero::GpuEnumValue(Prospero::BufferFormat::k32UInt)) { - RetireStorageDepthAliasLocked(info); - } - - std::shared_ptr match; - for (auto& cached: m_images) { - if (cached->kind != CachedImage::Kind::StorageTexture || - !EqualStorageBacking(info, cached->info)) { + if (!candidate) { continue; } - if (match != nullptr || cached->info.IsCpuDirty()) { - EXIT("TextureCache: invalid exact storage-image cache match, addr=0x%016" PRIx64 - " size=0x%016" PRIx64 " duplicate=%d cpu_dirty=%d\n", - info.address, info.size, match != nullptr, cached->info.IsCpuDirty()); - } - match = cached; - } - if (match != nullptr) { - const bool cpu_modified = m_memory_tracker.IsRegionCpuModified(info.address, info.size); - const bool gpu_modified = m_memory_tracker.IsRegionGpuModified(info.address, info.size); - const auto rebind = ClassifyStorageBufferRebind( - buffer_overlap, match->gpu_modified, match->buffer_modified, gpu_modified, cpu_modified, - IsCoherentGuestImageSource(source, info.address, info.size)); - if (rebind == StorageBufferRebind::Unsupported) { - EXIT("TextureCache: storage-image ownership is inconsistent, addr=0x%016" PRIx64 - " size=0x%016" PRIx64 - " cached_gpu_modified=%d buffer_modified=%d tracker_gpu_modified=%d " - "cpu_modified=%d buffer_overlap=%d source_current=%d\n", - info.address, info.size, match->gpu_modified, match->buffer_modified, gpu_modified, - cpu_modified, buffer_overlap, source.cpu_current); - } - if (rebind == StorageBufferRebind::RefreshFromBacking) { - // The formatted buffer may cover only one page of a packed mip chain. The - // synchronization path published the complete tiled backing before that write, - // and ObtainBufferForImage has now folded its dirty subrange back into it. - m_memory_tracker.MarkRegionAsCpuModified(info.address, info.size); - m_memory_tracker.ForEachUploadRange( - info.address, info.size, true, [](uint64_t, uint64_t) noexcept {}, - [&]() noexcept { - m_tiler.DetileImage(*static_cast(match->image), - match->info, source, true, true); - }); - match->buffer_modified = false; - match->gpu_modified = true; - } else if (!gpu_modified) { - if (cpu_modified) { - m_memory_tracker.ForEachUploadRange( - info.address, info.size, true, [](uint64_t, uint64_t) noexcept {}, - [&]() noexcept { - m_tiler.DetileImage( - *static_cast(match->image), match->info, - {nullptr, 0, match->info.address, match->info.size, true}, true, true); - }); - } else { - // A readback leaves both copies current. Reclaim ownership without an unnecessary - // upload through the clean UpdateImage storage-binding path. - m_memory_tracker.MarkRegionAsGpuModified(info.address, info.size); - } - match->gpu_modified = true; - } - command.RetainResourceUntilFence(match); - return *static_cast(match->image); - } - - ResolveStorageImageOverlaps(info); - m_images.reserve(m_images.size() + 1); - auto cached = std::make_shared(m_graphics); - cached->kind = CachedImage::Kind::StorageTexture; - cached->info = info; - vk::ComponentMapping components {}; - cached->image = ImageOps::CreateTexture(info, true, components); - m_memory_tracker.ForEachUploadRange( - info.address, info.size, true, [](uint64_t, uint64_t) noexcept {}, - [&]() noexcept { - m_tiler.DetileImage(*static_cast(cached->image), cached->info, - source, false, true); - }); - cached->gpu_modified = true; - ImageOps::CreateTextureViews(*static_cast(cached->image), info, true, - components); - return static_cast(PublishImage(command, std::move(cached))); -} - -RenderTextureVulkanImage& TextureCache::FindRenderTarget(CommandBuffer& command, - const RenderTargetInfo& info) { - const bool standard64 = IsSupportedStandard64RenderTarget(info); - const bool valid_samples = - info.samples == 1 || info.samples == 2 || info.samples == 4 || info.samples == 8; - if (info.address == 0 || info.size == 0 || info.address >= TRACKER_ADDRESS_SIZE || - info.size > TRACKER_ADDRESS_SIZE - info.address || info.format == vk::Format::eUndefined || - info.width == 0 || info.height == 0 || info.pitch < info.width || - info.bytes_per_element == 0 || info.levels == 0 || info.levels > 16 || info.layers == 0 || - info.size % info.layers != 0 || !valid_samples || - (info.samples > 1 && - (info.levels != 1 || info.layers != 1 || - info.tile_mode != Prospero::GpuEnumValue(Prospero::TileMode::kRenderTarget))) || - (info.tile_mode != Prospero::GpuEnumValue(Prospero::TileMode::kLinear) && - info.tile_mode != Prospero::GpuEnumValue(Prospero::TileMode::kRenderTarget) && - !standard64)) { - EXIT("TextureCache: invalid render-target request, addr=0x%016" PRIx64 " size=0x%016" PRIx64 - " extent=%ux%u pitch=%u bpe=%u samples=%u tile=%u format=%d\n", - info.address, info.size, info.width, info.height, info.pitch, info.bytes_per_element, - info.samples, info.tile_mode, static_cast(info.format)); - } - if (info.levels == 1 && - info.tile_mode == Prospero::GpuEnumValue(Prospero::TileMode::kRenderTarget)) { - TileSizeAlign layout {}; - const auto encoded_samples = static_cast(std::countr_zero(info.samples)); - if (!TileGetRenderTargetSize(info.width, info.height, info.pitch, info.bytes_per_element, - layout, encoded_samples) || - layout.size > UINT64_MAX / info.layers || - static_cast(layout.size) * info.layers != info.size) { - EXIT("TextureCache: invalid render-target block layout, size=0x%016" PRIx64 - " expected=0x%08x extent=%ux%u pitch=%u bpe=%u samples=%u\n", - info.size, layout.size, info.width, info.height, info.pitch, - info.bytes_per_element, info.samples); - } - } - if (info.levels > 1) { - TileSizeAlign layout {}; - if (info.tile_mode != Prospero::GpuEnumValue(Prospero::TileMode::kRenderTarget) || - !TileGetRenderTargetMipLayout(info.width, info.height, info.pitch, - info.bytes_per_element, info.levels, layout, nullptr, - nullptr) || - layout.size > UINT64_MAX / info.layers || - static_cast(layout.size) * info.layers != info.size) { - EXIT("TextureCache: unsupported render-target mip layout, size=0x%016" PRIx64 - " expected=0x%08x extent=%ux%u pitch=%u bpe=%u levels=%u tile=%u\n", - info.size, layout.size, info.width, info.height, info.pitch, - info.bytes_per_element, info.levels, info.tile_mode); - } - } - if (standard64) { - const auto format = ImageOps::RenderTargetTransferFormat(info.bytes_per_element); - const auto expected_pitch = - TileGetTexturePitch(format, info.width, info.levels, info.tile_mode); - TileSizeAlign layout {}; - TileGetTextureSize(format, info.width, info.height, expected_pitch, info.levels, - info.tile_mode, &layout, nullptr, nullptr); - if (expected_pitch != info.pitch || layout.align != 65536 || layout.size != info.size) { - EXIT("TextureCache: invalid Standard64KB render-target layout," - " size=0x%016" PRIx64 " expected=0x%08x align=0x%08x pitch=%u/%u\n", - info.size, layout.size, layout.align, info.pitch, expected_pitch); - } - } - const auto rows = static_cast(info.height - 1); - if (rows > (UINT64_MAX - info.width) / info.pitch) { - EXIT("TextureCache: render-target element count overflow, extent=%ux%u pitch=%u\n", - info.width, info.height, info.pitch); - } - const auto elements = rows * info.pitch + info.width; - if (elements > UINT64_MAX / info.bytes_per_element || - elements * info.bytes_per_element > UINT64_MAX / info.layers || - info.size < elements * info.bytes_per_element * info.layers) { - EXIT("TextureCache: invalid render-target storage, size=0x%016" PRIx64 - " elements=0x%016" PRIx64 " bpe=%u\n", - info.size, elements, info.bytes_per_element); - } - std::lock_guard transaction(m_resource_mutex); - BufferImageCopySource target_source {nullptr, 0, info.address, info.size, true}; - const bool target_buffer_overlap = m_buffer_cache.HasPageOverlap(info.address, info.size); - if (target_buffer_overlap) { - // A render target may use a containing native buffer after dirty bytes are published or - // coherent guest backing for a clean partial view. ObtainBufferForImage keeps GPU-dirty - // partial ownership and inconsistent state as hard failures. - target_source = m_buffer_cache.ObtainBufferForImage(info.address, info.size); - } - FaultSafeTextureLock lock(this, m_lock); - std::shared_ptr match; - for (auto& cached: m_images) { - if (cached->kind != CachedImage::Kind::RenderTarget || - (!Equal(info, cached->target) && !IsCompatibleRenderTargetView(cached->target, info) && - !IsCompatibleRenderTargetBacking(cached->target, info))) { - continue; - } - match = cached; - } - if (match != nullptr) { - ResolveImageMetadataOverlapsLocked(info.address, info.size); - if (info.samples > 1 && (match->buffer_modified || - m_memory_tracker.IsRegionCpuModified(info.address, info.size))) { - EXIT("TextureCache: multisampled render-target refresh is unsupported, " - "addr=0x%016" PRIx64 " size=0x%016" PRIx64 " samples=%u\n", - info.address, info.size, info.samples); - } - if (match->buffer_modified) { - if (match->gpu_modified || - !IsCoherentGuestImageSource(target_source, info.address, info.size)) { - EXIT("TextureCache: render-target refresh has invalid buffer ownership, " - "addr=0x%016" PRIx64 " size=0x%016" PRIx64 - " source=%p current=%d gpu_modified=%d\n", - info.address, info.size, static_cast(target_source.buffer), - target_source.cpu_current, match->gpu_modified); - } - m_memory_tracker.MarkRegionAsCpuModified(info.address, info.size); - } - if (!match->gpu_modified && m_memory_tracker.IsRegionCpuModified(info.address, info.size)) { - m_memory_tracker.ForEachUploadRange( - info.address, info.size, false, [](uint64_t, uint64_t) noexcept {}, - [&]() noexcept { - ImageOps::UploadRenderTarget( - *static_cast(match->image), info, true); - }); - match->buffer_modified = false; - } - command.RetainResourceUntilFence(match); - return *static_cast(match->image); - } - std::vector retire; - std::vector> recreated_depth_sources; - std::shared_ptr native_image_source; - for (const auto& entry: m_images) { - auto& cached = *entry; - if (!cached.OverlapsRange(info.address, info.size, true)) { - continue; - } - RenderTargetOverlap overlap = RenderTargetOverlap::Unsupported; - switch (cached.kind) { - case CachedImage::Kind::Texture: - overlap = ClassifyRenderTargetOverlap(cached.info, cached.gpu_modified, info); - break; - case CachedImage::Kind::StorageTexture: - overlap = ClassifyStorageRenderTargetOverlap( - cached.info, cached.image->format, cached.gpu_modified, cached.buffer_modified, - cached.info.IsCpuDirty() || - m_memory_tracker.IsRegionCpuModified(cached.info.address, cached.info.size), - m_memory_tracker.IsRegionGpuModified(cached.info.address, cached.info.size), - info); - break; - case CachedImage::Kind::RenderTarget: - overlap = ClassifyRenderTargetOverlap( - cached.target, cached.gpu_modified, cached.buffer_modified, - m_memory_tracker.IsRegionGpuModified(cached.target.address, cached.target.size), - IsCoherentGuestImageSource(target_source, info.address, info.size), info); - break; - case CachedImage::Kind::DepthTarget: { - bool tracker_gpu_modified = false; - for (uint32_t range = 0; range < cached.RangeCount(); range++) { - tracker_gpu_modified |= m_memory_tracker.IsRegionGpuModified( - cached.Address(range), cached.Size(range)); - } - if (CanRecreateDepthForRenderTarget( - cached.depth, cached.gpu_modified, cached.buffer_modified, - tracker_gpu_modified, - IsCoherentGuestImageSource(target_source, info.address, info.size), info)) { - overlap = RenderTargetOverlap::RetireTarget; - } - } break; - case CachedImage::Kind::VideoOut: break; - } - if (overlap == RenderTargetOverlap::None) { - continue; - } - bool supported = false; - switch (overlap) { - case RenderTargetOverlap::RetireSampled: - supported = cached.kind == CachedImage::Kind::Texture; - break; - case RenderTargetOverlap::RetireStorage: - supported = cached.kind == CachedImage::Kind::StorageTexture; - break; - case RenderTargetOverlap::PreserveStorage: - supported = cached.kind == CachedImage::Kind::StorageTexture && - native_image_source == nullptr; - if (supported) { - native_image_source = entry; - } - break; - case RenderTargetOverlap::ExpandTarget: - supported = cached.kind == CachedImage::Kind::RenderTarget && - native_image_source == nullptr; - if (supported) { - native_image_source = entry; - } - break; - case RenderTargetOverlap::RetireTarget: - supported = cached.kind == CachedImage::Kind::RenderTarget || - cached.kind == CachedImage::Kind::DepthTarget; - if (supported && cached.kind == CachedImage::Kind::DepthTarget) { - recreated_depth_sources.push_back(entry); - } - break; - case RenderTargetOverlap::None: - case RenderTargetOverlap::Unsupported: break; - } - if (!supported) { - EXIT("TextureCache: unsupported render-target alias, requested=0x%016" PRIx64 - "+0x%016" PRIx64 " existing_kind=%u existing=0x%016" PRIx64 "+0x%016" PRIx64 - " gpu_modified=%d" - " sampled_format=%u extent=%ux%ux%u pitch=%u levels=%u base_level=%u" - " tile=%u type=%u base_array=%u\n", - info.address, info.size, static_cast(cached.kind), cached.Address(), - cached.Size(), cached.gpu_modified, cached.info.format, cached.info.width, - cached.info.height, cached.info.depth, cached.info.pitch, cached.info.levels, - cached.info.base_level, cached.info.tile, cached.info.type, - cached.info.base_array); - } - retire.push_back(&cached); - } - RequireRetirementIsolation(retire, "render target", info.address, info.size); - MaterializeImagesToGuestLocked(recreated_depth_sources); - for (const auto& cached: recreated_depth_sources) { - // The exact guest range owns the current bytes. Clear a stale buffer marker only after - // source-coherence validation so retirement can replace the obsolete Vulkan shape. - cached->buffer_modified = false; - } - RetireDepthMetadataLocked(retire); - RetireImages(retire, native_image_source.get()); - ResolveImageMetadataOverlapsLocked(info.address, info.size); - auto cached = std::make_shared(m_graphics); - cached->kind = CachedImage::Kind::RenderTarget; - cached->target = info; - cached->image = ImageOps::CreateRenderTarget(info); - const bool preserve_native = native_image_source != nullptr; - if (preserve_native) { - command.RetainResourceUntilFence(native_image_source); - if (native_image_source->kind == CachedImage::Kind::RenderTarget) { - const auto& old = native_image_source->target; - const auto tail_address = info.address + old.size; - const auto tail_size = info.size - old.size; - if (old.layers >= info.layers || tail_size == 0 || - !IsCoherentGuestImageSource(target_source, info.address, info.size)) { - EXIT("TextureCache: invalid render-target expansion source, old_layers=%u " - "new_layers=%u old_size=0x%016" PRIx64 " new_size=0x%016" PRIx64 "\n", - old.layers, info.layers, old.size, info.size); - } - m_memory_tracker.ForEachUploadRange( - tail_address, tail_size, true, [](uint64_t, uint64_t) noexcept {}, - [&]() noexcept { - ImageOps::UploadRenderTargetLayers( - *static_cast(cached->image), info, old.layers, - info.layers - old.layers, false); - }); - } - std::vector regions; - regions.reserve(static_cast(native_image_source->image->layers) * - native_image_source->image->mip_levels); - AppendLayerCopies(regions, *native_image_source->image, vk::ImageAspectFlagBits::eColor); - Transfer::CopyImage(command, regions, *cached->image, RENDER_COLOR_IMAGE_LAYOUT); - } else { - if (info.samples > 1) { - if (!IsCoherentGuestImageSource(target_source, info.address, info.size) || - !GuestRangeIsZero(info.address, info.size)) { - EXIT("TextureCache: nonzero multisampled render-target upload is unsupported, " - "addr=0x%016" PRIx64 " size=0x%016" PRIx64 " samples=%u\n", - info.address, info.size, info.samples); - } - m_memory_tracker.ForEachUploadRange( - info.address, info.size, false, [](uint64_t, uint64_t) noexcept {}, - []() noexcept {}); - static_cast(cached->image)->initial_clear_pending = true; - } else { - m_memory_tracker.ForEachUploadRange( - info.address, info.size, false, [](uint64_t, uint64_t) noexcept {}, - [&]() noexcept { - ImageOps::UploadRenderTarget( - *static_cast(cached->image), info, false); - }); - } - } - cached->gpu_modified = preserve_native; - return static_cast(PublishImage(command, std::move(cached))); -} - -DepthStencilVulkanImage& TextureCache::FindDepthTarget(CommandBuffer& command, - const DepthTargetInfo& info) { - const bool has_stencil = info.stencil_address != 0 || info.stencil_size != 0; - const bool has_htile = info.htile_address != 0 || info.htile_size != 0; - const bool valid_samples = - info.samples == 1 || info.samples == 2 || info.samples == 4 || info.samples == 8; - if (info.address == 0 || info.size == 0 || info.address >= TRACKER_ADDRESS_SIZE || - info.size > TRACKER_ADDRESS_SIZE - info.address || (info.address & 0xffffu) != 0 || - info.width == 0 || info.height == 0 || info.pitch < info.width || info.layers == 0 || - !valid_samples || (info.samples > 1 && info.layers != 1) || info.size % info.layers != 0 || - info.size > UINT32_MAX || info.stencil_size > UINT32_MAX || - info.tile_mode != Prospero::GpuEnumValue(Prospero::TileMode::kDepth) || - (has_stencil && - (info.stencil_address == 0 || info.stencil_size == 0 || - info.stencil_size % info.layers != 0 || info.stencil_address >= TRACKER_ADDRESS_SIZE || - info.stencil_size > TRACKER_ADDRESS_SIZE - info.stencil_address || - (info.stencil_address & 0xffffu) != 0 || - ImagePageRangesOverlap(info.address, info.size, info.stencil_address, - info.stencil_size))) || - ((info.htile_address == 0) != (info.htile_size == 0)) || - (has_htile && - (info.htile_size % info.layers != 0 || info.htile_address >= TRACKER_ADDRESS_SIZE || - info.htile_size > TRACKER_ADDRESS_SIZE - info.htile_address || - (info.htile_address & 0x7fffu) != 0 || - ImagePageRangesOverlap(info.address, info.size, info.htile_address, info.htile_size) || - (has_stencil && ImagePageRangesOverlap(info.stencil_address, info.stencil_size, - info.htile_address, info.htile_size)))) || - (info.stencil_htile_compressed && (!has_stencil || !has_htile)) || - !IsSupportedDepthTargetFormat(info)) { - EXIT("TextureCache: unsupported depth target, command=%p depth=0x%016" PRIx64 - "+0x%016" PRIx64 " stencil=0x%016" PRIx64 "+0x%016" PRIx64 " htile=0x%016" PRIx64 - "+0x%016" PRIx64 - " extent=%ux%u pitch=%u samples=%u tile=%u format=%d guest_format=%u bpe=%u\n", - static_cast(&command), info.address, info.size, info.stencil_address, - info.stencil_size, info.htile_address, info.htile_size, info.width, info.height, - info.pitch, info.samples, info.tile_mode, static_cast(info.format), - info.guest_format, info.bytes_per_element); - } - const auto* depth_policy = FindGuestDepthFormatPolicy(info.guest_format); - TileSizeAlign expected_depth {}; - TileSizeAlign expected_stencil {}; - TileSizeAlign expected_htile {}; - const auto encoded_samples = static_cast(std::countr_zero(info.samples)); - if (depth_policy == nullptr || depth_policy->bytes_per_element != info.bytes_per_element || - info.pitch != TileGetDepthPitch(info.width, info.bytes_per_element, encoded_samples) || - !TileGetDepthSize( - info.width, info.height, 0, Prospero::GpuEnumValue(depth_policy->depth_format), - Prospero::GpuEnumValue(has_stencil ? Prospero::StencilFormat::k8UInt - : Prospero::StencilFormat::kInvalid), - has_htile, expected_stencil, expected_htile, expected_depth, encoded_samples) || - expected_depth.size > UINT64_MAX / info.layers || - static_cast(expected_depth.size) * info.layers != info.size || - (has_stencil && - (expected_stencil.size > UINT64_MAX / info.layers || - static_cast(expected_stencil.size) * info.layers != info.stencil_size)) || - (has_htile && - (expected_htile.size > UINT64_MAX / info.layers || - static_cast(expected_htile.size) * info.layers != info.htile_size))) { - EXIT("TextureCache: invalid depth-target block layout, depth=0x%016" PRIx64 "+0x%016" PRIx64 - " stencil=0x%016" PRIx64 "+0x%016" PRIx64 " htile=0x%016" PRIx64 "+0x%016" PRIx64 - " extent=%ux%u pitch=%u samples=%u\n", - info.address, info.size, info.stencil_address, info.stencil_size, info.htile_address, - info.htile_size, info.width, info.height, info.pitch, info.samples); - } - const auto rows = static_cast(info.height - 1); - if (rows > (UINT64_MAX - info.width) / info.pitch) { - EXIT("TextureCache: depth-target element count overflow, extent=%ux%u pitch=%u\n", - info.width, info.height, info.pitch); - } - const auto elements = rows * info.pitch + info.width; - if (elements > UINT64_MAX / info.bytes_per_element / info.samples || - elements * info.bytes_per_element * info.samples > UINT64_MAX / info.layers || - info.size < elements * info.bytes_per_element * info.samples * info.layers) { - EXIT("TextureCache: depth storage is too small, size=0x%016" PRIx64 - " elements=0x%016" PRIx64 " bpe=%u\n", - info.size, elements, info.bytes_per_element); - } - if (has_stencil && (elements > UINT64_MAX / info.samples / info.layers || - info.stencil_size < elements * info.samples * info.layers)) { - EXIT("TextureCache: stencil storage is too small, size=0x%016" PRIx64 - " elements=0x%016" PRIx64 "\n", - info.stencil_size, elements); - } - if (has_stencil && !info.stencil_load_clear && !CanLoadRawStencilPlane(info)) { - EXIT("TextureCache: HTile-compressed stencil load is unsupported, addr=0x%016" PRIx64 - " size=0x%016" PRIx64 " htile=0x%016" PRIx64 "+0x%016" PRIx64 "\n", - info.stencil_address, info.stencil_size, info.htile_address, info.htile_size); - } - if (has_htile) { - RegisterMeta(info.htile_address, info.htile_size, info.layers); - } - std::lock_guard transaction(m_resource_mutex); - // BufferCache treats an untracked guest range as CPU-current. That is useful when - // uploading from guest memory, but it is not evidence that a clean native image is - // stale. Only a genuinely overlapping buffer participates in transition selection. - const bool depth_buffer_overlap = m_buffer_cache.HasPageOverlap(info.address, info.size); - const auto depth_source = m_buffer_cache.ObtainBufferForImage(info.address, info.size); - const auto stencil_source = - has_stencil ? m_buffer_cache.ObtainBufferForImage(info.stencil_address, info.stencil_size) - : BufferImageCopySource {}; - FaultSafeTextureLock lock(this, m_lock); - ResolveImageMetadataOverlapsLocked(info.address, info.size); - if (has_stencil) { - ResolveImageMetadataOverlapsLocked(info.stencil_address, info.stencil_size); - } - std::shared_ptr match; - for (auto& cached: m_images) { - if (cached->kind != CachedImage::Kind::DepthTarget || - (!Equal(info, cached->depth) && !IsCompatibleDepthTargetBacking(cached->depth, info))) { - continue; - } - match = cached; - } - if (match != nullptr) { - const bool depth_cpu_modified = - m_memory_tracker.IsRegionCpuModified(info.address, info.size); - const bool stencil_cpu_modified = - has_stencil && - m_memory_tracker.IsRegionCpuModified(info.stencil_address, info.stencil_size); - if (RequiresMultisampleDepthRefresh(info, match->buffer_modified, depth_cpu_modified, - stencil_cpu_modified)) { - EXIT("TextureCache: multisampled depth-target refresh is unsupported, " - "addr=0x%016" PRIx64 " size=0x%016" PRIx64 - " samples=%u access=%d/%d clear=%d/%d buffer=%d cpu=%d/%d\n", - info.address, info.size, info.samples, info.depth_access, info.stencil_access, - info.depth_load_clear, info.stencil_load_clear, match->buffer_modified, - depth_cpu_modified, stencil_cpu_modified); - } - if (!CanLoadStencilAttachment(info, match->stencil_initialized)) { - EXIT("TextureCache: stencil load requires initialized contents, addr=0x%016" PRIx64 - " size=0x%016" PRIx64 "\n", - info.stencil_address, info.stencil_size); - } - if (has_stencil && info.stencil_load_clear) { - match->stencil_initialized = true; - } - if (match->buffer_modified) { - if (match->gpu_modified || !depth_buffer_overlap || - !IsCoherentGuestImageSource(depth_source, info.address, info.size)) { - EXIT("TextureCache: depth-target refresh has invalid buffer ownership, " - "addr=0x%016" PRIx64 " size=0x%016" PRIx64 - " gpu_modified=%d overlap=%d source_current=%d\n", - info.address, info.size, match->gpu_modified, depth_buffer_overlap, - depth_source.cpu_current); - } - m_memory_tracker.MarkRegionAsCpuModified(info.address, info.size); - m_memory_tracker.ForEachUploadRange( - info.address, info.size, true, [](uint64_t, uint64_t) noexcept {}, - [&]() noexcept { - m_tiler.DetileImage(*static_cast(match->image), info, - depth_source, true); - }); - match->buffer_modified = false; - match->gpu_modified = true; - } else if (!match->gpu_modified && depth_source.cpu_dirty) { - m_memory_tracker.ForEachUploadRange( - info.address, info.size, false, [](uint64_t, uint64_t) noexcept {}, - [&]() noexcept { - m_tiler.DetileImage(*static_cast(match->image), info, - depth_source, true); - }); - } - if (!match->gpu_modified && has_stencil && stencil_source.cpu_dirty) { - m_memory_tracker.ForEachUploadRange( - info.stencil_address, info.stencil_size, false, [](uint64_t, uint64_t) noexcept {}, - [&]() noexcept { - if (!info.stencil_load_clear) { - m_tiler.DetileStencil(*static_cast(match->image), - info, stencil_source, true); - match->stencil_initialized = true; - } - }); - } - command.RetainResourceUntilFence(match); - return *static_cast(match->image); - } - std::vector retire; - std::shared_ptr sampled_depth_source; - std::shared_ptr native_depth_source; - std::shared_ptr discarded_depth_source; - std::shared_ptr retired_storage_source; - std::vector> recreated_target_sources; - for (const auto& entry: m_images) { - auto& cached = *entry; - const bool overlaps = - cached.OverlapsRange(info.address, info.size, true) || - (has_stencil && cached.OverlapsRange(info.stencil_address, info.stencil_size, true)); - if (!overlaps) { - continue; - } - DepthOverlap overlap = DepthOverlap::Unsupported; - switch (cached.kind) { - case CachedImage::Kind::Texture: { - const auto native_overlap = - ClassifyDepthOverlap(cached.info, cached.gpu_modified, info); - const bool overlaps_depth = ImageRangeOverlaps( - cached.info.address, cached.info.size, info.address, info.size); - const bool overlaps_stencil = - has_stencil && ImageRangeOverlaps(cached.info.address, cached.info.size, - info.stencil_address, info.stencil_size); - const bool guest_source_current = - (!overlaps_depth || info.depth_load_clear || - IsCoherentGuestImageSource(depth_source, info.address, info.size)) && - (!overlaps_stencil || info.stencil_load_clear || - IsCoherentGuestImageSource(stencil_source, info.stencil_address, - info.stencil_size)); - overlap = native_overlap; - if (overlap == DepthOverlap::Unsupported && - CanRetireGuestCurrentSampledForDepth( - cached.info, info, cached.gpu_modified, cached.buffer_modified, - cached.info.IsCpuDirty(), - m_memory_tracker.IsRegionGpuModified(cached.info.address, cached.info.size), - guest_source_current)) { - overlap = DepthOverlap::RetireSampled; - } - if (overlap == DepthOverlap::RetireSampled && !info.depth_load_clear && - native_overlap == DepthOverlap::RetireSampled && - sampled_depth_source == nullptr) { - sampled_depth_source = entry; - } - } break; - case CachedImage::Kind::DepthTarget: - overlap = ClassifyDepthTargetOverlap(cached.depth, cached.gpu_modified, - cached.buffer_modified, info); - break; - case CachedImage::Kind::StorageTexture: - overlap = ClassifyStorageDepthOverlap( - cached.info, cached.gpu_modified, cached.buffer_modified, - cached.info.IsCpuDirty() || - m_memory_tracker.IsRegionCpuModified(cached.info.address, cached.info.size), - m_memory_tracker.IsRegionGpuModified(cached.info.address, cached.info.size), - info); - break; - case CachedImage::Kind::RenderTarget: { - const bool overlaps_depth = ImageRangeOverlaps( - cached.target.address, cached.target.size, info.address, info.size); - const bool overlaps_stencil = - has_stencil && ImageRangeOverlaps(cached.target.address, cached.target.size, - info.stencil_address, info.stencil_size); - const bool guest_source_current = - (!overlaps_depth || info.depth_load_clear || - IsCoherentGuestImageSource(depth_source, info.address, info.size)) && - (!overlaps_stencil || info.stencil_load_clear || - IsCoherentGuestImageSource(stencil_source, info.stencil_address, - info.stencil_size)); - overlap = CanRecreateRenderTargetForDepth( - cached.target, cached.gpu_modified, cached.buffer_modified, - m_memory_tracker.IsRegionGpuModified(cached.target.address, - cached.target.size), - guest_source_current, info) - ? DepthOverlap::RecreateTarget - : DepthOverlap::Unsupported; - } break; - case CachedImage::Kind::VideoOut: break; - } - bool supported = false; - switch (overlap) { - case DepthOverlap::RetireSampled: - supported = cached.kind == CachedImage::Kind::Texture; - break; - case DepthOverlap::RetireStorage: - supported = cached.kind == CachedImage::Kind::StorageTexture && - retired_storage_source == nullptr && native_depth_source == nullptr && - discarded_depth_source == nullptr && recreated_target_sources.empty(); - if (supported) { - retired_storage_source = entry; - } - break; - case DepthOverlap::ExpandTarget: - supported = cached.kind == CachedImage::Kind::DepthTarget && - native_depth_source == nullptr && retired_storage_source == nullptr && - recreated_target_sources.empty(); - if (supported) { - native_depth_source = entry; - } - break; - case DepthOverlap::DiscardTarget: - supported = cached.kind == CachedImage::Kind::DepthTarget && - discarded_depth_source == nullptr && native_depth_source == nullptr; - supported = supported && retired_storage_source == nullptr && - recreated_target_sources.empty(); - if (supported) { - discarded_depth_source = entry; - } - break; - case DepthOverlap::RecreateTarget: - supported = (cached.kind == CachedImage::Kind::DepthTarget || - cached.kind == CachedImage::Kind::RenderTarget) && - native_depth_source == nullptr && discarded_depth_source == nullptr && - retired_storage_source == nullptr; - if (supported) { - recreated_target_sources.push_back(entry); - } - break; - case DepthOverlap::None: - case DepthOverlap::Unsupported: break; - } - if (!supported) { - EXIT("TextureCache: unsupported depth-target alias, depth=0x%016" PRIx64 - "+0x%016" PRIx64 " existing_kind=%u existing=0x%016" PRIx64 "+0x%016" PRIx64 "\n", - info.address, info.size, static_cast(cached.kind), cached.Address(), - cached.Size()); - } - retire.push_back(&cached); - } - RequireRetirementIsolation(retire, "depth target", info.address, info.size); - MaterializeImagesToGuestLocked(recreated_target_sources); - if (retired_storage_source != nullptr) { - if (retired_storage_source->gpu_modified) { - m_memory_tracker.UnmarkRegionAsGpuModified(retired_storage_source->info.address, - retired_storage_source->info.size); - retired_storage_source->gpu_modified = false; - } - } - const auto* transition_source = - native_depth_source != nullptr - ? native_depth_source.get() - : (discarded_depth_source != nullptr ? discarded_depth_source.get() : nullptr); - RetireDepthMetadataLocked(retire, info.htile_address); - RetireImages(retire, transition_source); - const bool coherent_guest_stencil = - has_stencil && - IsCoherentGuestImageSource(stencil_source, info.stencil_address, info.stencil_size); - // Native expansion must preserve old layers. Guest data initializes only a wholly new image. - const bool stencil_contents_available = native_depth_source != nullptr - ? native_depth_source->stencil_initialized - : coherent_guest_stencil; - if (!CanLoadStencilAttachment(info, stencil_contents_available)) { - EXIT("TextureCache: new stencil target requires a clear before stencil access, " - "addr=0x%016" PRIx64 " size=0x%016" PRIx64 "\n", - info.stencil_address, info.stencil_size); - } - if (info.samples > 1 && (native_depth_source != nullptr || sampled_depth_source != nullptr || - retired_storage_source != nullptr)) { - EXIT("TextureCache: multisampled depth native transition is unsupported, " - "addr=0x%016" PRIx64 " size=0x%016" PRIx64 " samples=%u\n", - info.address, info.size, info.samples); - } - const bool initial_depth_clear = info.samples > 1 && !info.depth_load_clear; - const bool initial_stencil_clear = info.samples > 1 && has_stencil && !info.stencil_load_clear; - if (initial_depth_clear && - (!IsCoherentGuestImageSource(depth_source, info.address, info.size) || - !GuestRangeIsZero(info.address, info.size))) { - EXIT("TextureCache: nonzero multisampled depth upload is unsupported, " - "addr=0x%016" PRIx64 " size=0x%016" PRIx64 " samples=%u\n", - info.address, info.size, info.samples); - } - if (initial_stencil_clear && - (!coherent_guest_stencil || !GuestRangeIsZero(info.stencil_address, info.stencil_size))) { - EXIT("TextureCache: nonzero multisampled stencil upload is unsupported, " - "addr=0x%016" PRIx64 " size=0x%016" PRIx64 " samples=%u\n", - info.stencil_address, info.stencil_size, info.samples); - } - auto cached = std::make_shared(m_graphics); - cached->kind = CachedImage::Kind::DepthTarget; - cached->depth = info; - cached->stencil_initialized = !has_stencil || info.stencil_load_clear || initial_stencil_clear; - cached->image = ImageOps::CreateDepthTarget(info); - auto* depth_image = static_cast(cached->image); - depth_image->initial_depth_clear_pending = initial_depth_clear; - depth_image->initial_stencil_clear_pending = initial_stencil_clear; - if (discarded_depth_source != nullptr) { - command.RetainResourceUntilFence(discarded_depth_source); - } - if (retired_storage_source != nullptr) { - command.RetainResourceUntilFence(retired_storage_source); - } - if (native_depth_source != nullptr) { - const auto& old = native_depth_source->depth; - if (old.layers >= info.layers || native_depth_source->image->layers != old.layers) { - EXIT("TextureCache: invalid depth expansion source, old_layers=%u new_layers=%u\n", - old.layers, info.layers); - } - command.RetainResourceUntilFence(native_depth_source); - const auto tail = SelectDepthLayers(info, old.layers, info.layers - old.layers); - const auto depth_tail = SelectSourceRange(depth_source, tail.address, tail.size); - if (!info.depth_load_clear && - !IsCoherentGuestImageSource(depth_tail, tail.address, tail.size)) { - EXIT("TextureCache: depth expansion tail is not CPU-current\n"); - } - m_memory_tracker.ForEachUploadRange( - tail.address, tail.size, true, [](uint64_t, uint64_t) noexcept {}, - [&]() noexcept { - if (!info.depth_load_clear) { - m_tiler.DetileImage(*static_cast(cached->image), tail, - depth_tail, false, old.layers); - } - }); - if (has_stencil) { - const auto stencil_tail = - SelectSourceRange(stencil_source, tail.stencil_address, tail.stencil_size); - if (!info.stencil_load_clear && - !IsCoherentGuestImageSource(stencil_tail, tail.stencil_address, - tail.stencil_size)) { - EXIT("TextureCache: stencil expansion tail is not CPU-current\n"); - } - m_memory_tracker.ForEachUploadRange( - tail.stencil_address, tail.stencil_size, true, [](uint64_t, uint64_t) noexcept {}, - [&]() noexcept { - if (!info.stencil_load_clear) { - m_tiler.DetileStencil(*static_cast(cached->image), - tail, stencil_tail, false, old.layers); - } - }); - cached->stencil_initialized = true; - } - std::vector regions; - regions.reserve(native_depth_source->image->layers * (has_stencil ? 2u : 1u)); - AppendLayerCopies(regions, *native_depth_source->image, vk::ImageAspectFlagBits::eDepth); - if (has_stencil) { - AppendLayerCopies(regions, *native_depth_source->image, - vk::ImageAspectFlagBits::eStencil); - } - Transfer::CopyImage(command, regions, *cached->image, - vk::ImageLayout::eDepthStencilAttachmentOptimal); - cached->gpu_modified = true; - } else { - if (info.samples > 1) { - // Multisample guest planes cannot be copied through Vulkan buffer-image commands. - // Their verified-zero initial contents are materialized by the first render pass. - m_memory_tracker.ForEachUploadRange( - info.address, info.size, false, [](uint64_t, uint64_t) noexcept {}, - []() noexcept {}); - if (has_stencil) { - m_memory_tracker.ForEachUploadRange( - info.stencil_address, info.stencil_size, false, - [](uint64_t, uint64_t) noexcept {}, []() noexcept {}); - } - return static_cast(PublishImage(command, std::move(cached))); - } - if (sampled_depth_source != nullptr && - (sampled_depth_source->image->type != VulkanImageType::Texture || - sampled_depth_source->image->format != VulkanFormat(info.guest_format) || - sampled_depth_source->image->extent.width != info.width || - sampled_depth_source->image->extent.height != info.height)) { - EXIT("TextureCache: sampled-depth native source is inconsistent\n"); - } - const auto transition_source = SelectDepthTransitionSource( - info.depth_load_clear, sampled_depth_source != nullptr, - sampled_depth_source != nullptr && sampled_depth_source->info.IsCpuDirty(), - sampled_depth_source != nullptr && sampled_depth_source->buffer_modified, - depth_buffer_overlap, depth_source.cpu_dirty); - m_memory_tracker.ForEachUploadRange( - info.address, info.size, false, [](uint64_t, uint64_t) noexcept {}, - [&]() noexcept { - switch (transition_source) { - case DepthTransitionSource::None: break; - case DepthTransitionSource::Guest: - m_tiler.DetileImage(*static_cast(cached->image), - info, depth_source, false); - break; - case DepthTransitionSource::Native: - command.RetainResourceUntilFence(sampled_depth_source); - Transfer::CopyImageViaBuffer( - command, *sampled_depth_source->image, vk::ImageAspectFlagBits::eColor, - *cached->image, vk::ImageAspectFlagBits::eDepth, info.bytes_per_element, - vk::ImageLayout::eDepthStencilAttachmentOptimal); - break; - } - }); - if (has_stencil) { - m_memory_tracker.ForEachUploadRange( - info.stencil_address, info.stencil_size, false, [](uint64_t, uint64_t) noexcept {}, - [&]() noexcept { - if (!info.stencil_load_clear) { - m_tiler.DetileStencil(*static_cast(cached->image), - info, stencil_source, false); - } - }); - cached->stencil_initialized = true; - } - } - return static_cast(PublishImage(command, std::move(cached))); -} - -std::vector -TextureCache::RegisterVideoOutSurfaces(const std::vector& infos) { - if (infos.empty()) { - EXIT("TextureCache: video-out registration requires surfaces\n"); - } - for (const auto& info: infos) { - ImageOps::ValidateVideoOut(info); - } - for (size_t i = 0; i < infos.size(); i++) { - for (size_t j = i + 1; j < infos.size(); j++) { - if (ImagePageRangesOverlap(infos[i].address, infos[i].size, infos[j].address, - infos[j].size)) { - EXIT("TextureCache: video-out surfaces share pages, first=%zu second=%zu\n", i, j); - } - } - } - std::lock_guard transaction(m_resource_mutex); - for (const auto& info: infos) { - if (m_buffer_cache.HasPageOverlap(info.address, info.size)) { - EXIT("TextureCache: video-out surface aliases buffer pages, addr=0x%016" PRIx64 - " size=0x%016" PRIx64 "\n", - info.address, info.size); - } - } - FaultSafeTextureLock lock(this, m_lock); - for (const auto& info: infos) { - ResolveImageMetadataOverlapsLocked(info.address, info.size); - for (const auto& cached: m_images) { - if (cached->OverlapsRange(info.address, info.size, true)) { - EXIT( - "TextureCache: video-out surface aliases cached image pages, addr=0x%016" PRIx64 - " size=0x%016" PRIx64 " existing_kind=%u\n", - info.address, info.size, static_cast(cached->kind)); - } - } - } - m_images.reserve(m_images.size() + infos.size()); - std::vector result; - result.reserve(infos.size()); - for (const auto& info: infos) { - auto cached = std::make_shared(m_graphics); - cached->kind = CachedImage::Kind::VideoOut; - cached->video_out = info; - cached->image = ImageOps::CreateVideoOut(info); - if (info.compression == VideoOutCompression::Uncompressed) { - m_memory_tracker.ForEachUploadRange( - info.address, info.size, false, [](uint64_t, uint64_t) noexcept {}, - [&]() noexcept { - ImageOps::UploadVideoOut(*static_cast(cached->image), - info, false); - }); - } else { - // A compressed guest surface cannot be decoded without its DCC metadata. Establish the - // normal tracked range, but leave the shared native image to be initialized by a GPU - // render/clear before any sampled or presentation read. - m_memory_tracker.ForEachUploadRange( - info.address, info.size, false, [](uint64_t, uint64_t) noexcept {}, - []() noexcept {}); - } - result.push_back(static_cast(cached->image)); - m_images.push_back(std::move(cached)); - RegisterImageLocked(*m_images.back()); - } - return result; -} - -void TextureCache::RefreshVideoOut(VideoOutVulkanImage& image, bool render_target) { - std::lock_guard transaction(m_resource_mutex); - FaultSafeTextureLock lock(this, m_lock); - const auto it = std::find_if(m_images.begin(), m_images.end(), - [&image](const auto& cached) { return cached->image == ℑ }); - if (it == m_images.end() || (*it)->kind != CachedImage::Kind::VideoOut) { - EXIT("TextureCache: video-out image is not registered, image=%p\n", - static_cast(&image)); - } - auto& cached = **it; - if (cached.gpu_modified) { - return; - } - const auto& info = cached.video_out; - const bool image_dirty = m_memory_tracker.IsRegionCpuModified(info.address, info.size); - const bool buffer_overlap = m_buffer_cache.HasPageOverlap(info.address, info.size); - const bool buffer_dirty = - cached.buffer_modified || - (buffer_overlap && (m_buffer_cache.IsRegionCpuModified(info.address, info.size) || - m_buffer_cache.IsRegionGpuModified(info.address, info.size))); - if (!image_dirty && !buffer_dirty) { - if (info.compression == VideoOutCompression::Uncompressed || - CanUseVideoOutNativeWithoutUpload(info.compression, render_target, false, false)) { - return; - } - EXIT("TextureCache: compressed video-out read requires native GPU contents, " - "addr=0x%016" PRIx64 " size=0x%016" PRIx64 "\n", - info.address, info.size); - } - if (info.compression != VideoOutCompression::Uncompressed) { - EXIT("TextureCache: compressed video-out guest refresh is unsupported, " - "addr=0x%016" PRIx64 " size=0x%016" PRIx64 - " image_dirty=%d buffer_dirty=%d render_target=%d\n", - info.address, info.size, image_dirty, buffer_dirty, render_target); - } - if (buffer_dirty) { - const auto source = m_buffer_cache.ObtainBufferForImage(info.address, info.size); - if (!IsCoherentGuestImageSource(source, info.address, info.size)) { - EXIT("TextureCache: invalid video-out source, addr=0x%016" PRIx64 " size=0x%016" PRIx64 - " buffer=%p current=%d source=0x%016" PRIx64 "+0x%016" PRIx64 - " buffer_modified=%d\n", - info.address, info.size, static_cast(source.buffer), - source.cpu_current, source.address, source.size, cached.buffer_modified); - } - cached.buffer_modified = false; - } - m_memory_tracker.ForEachUploadRange( - info.address, info.size, false, [](uint64_t, uint64_t) noexcept {}, - [&]() noexcept { ImageOps::UploadVideoOut(image, info, true); }); -} - -void TextureCache::UnregisterVideoOutSurfaces(const std::vector& images) { - if (images.empty()) { - EXIT("TextureCache: video-out unregistration requires surfaces\n"); - } - std::lock_guard transaction(m_resource_mutex); - FaultSafeTextureLock lock(this, m_lock); - std::vector selected; - selected.reserve(images.size()); - for (size_t i = 0; i < images.size(); i++) { - auto* image = images[i]; - if (image == nullptr || - std::find(images.begin(), images.begin() + i, image) != images.begin() + i) { - EXIT("TextureCache: invalid or duplicate video-out image at index %zu, image=%p\n", i, - static_cast(image)); - } - auto it = std::find_if(m_images.begin(), m_images.end(), - [image](const auto& cached) { return cached->image == image; }); - if (it == m_images.end() || (*it)->kind != CachedImage::Kind::VideoOut) { - EXIT("TextureCache: video-out image is not registered, index=%zu image=%p\n", i, - static_cast(image)); - } - selected.push_back(it->get()); - } - for (auto* cached: selected) { - if (cached->buffer_modified) { - EXIT("TextureCache: cannot unregister a buffer-dirty video-out surface, " - "addr=0x%016" PRIx64 " size=0x%016" PRIx64 "\n", - cached->Address(), cached->Size()); - } - } - Transfer::WaitForQueueIdle(); - for (auto* cached: selected) { - if (cached->gpu_modified) { - m_memory_tracker.UnmarkRegionAsGpuModified(cached->Address(), cached->Size()); - cached->gpu_modified = false; - } - UnregisterImageLocked(*cached, true); - } - for (auto* image: images) { - auto it = std::find_if(m_images.begin(), m_images.end(), - [image](const auto& cached) { return cached->image == image; }); - m_images.erase(it); - } -} - -bool TextureCache::ClearImageFromBuffer(CommandBuffer& command, uint64_t vaddr, uint64_t size, - uint32_t packed_clear) { - if (command.IsInvalid() || vaddr == 0 || size == 0 || vaddr >= TRACKER_ADDRESS_SIZE || - size > TRACKER_ADDRESS_SIZE - vaddr) { - EXIT("TextureCache: invalid compute image clear, command=%p addr=0x%016" PRIx64 - " size=0x%016" PRIx64 "\n", - static_cast(&command), vaddr, size); - } - m_buffer_cache.ValidateGpuAccess(vaddr, size, false, true); - std::lock_guard transaction(m_resource_mutex); - FaultSafeTextureLock lock(this, m_lock); - enum class ClearAspect : uint8_t { None, Color, Depth, Stencil }; - auto classify = [vaddr, size](const CachedImage& cached) { - const bool color = cached.kind == CachedImage::Kind::VideoOut || - cached.kind == CachedImage::Kind::RenderTarget; - if (color && cached.Address() == vaddr && cached.Size() == size) { - return ClearAspect::Color; - } - if (cached.kind != CachedImage::Kind::DepthTarget) { - return ClearAspect::None; - } - if (CanNativeClearDepthFromBuffer(cached.depth, vaddr, size)) { - return ClearAspect::Depth; - } - if (cached.RangeCount() == 2 && cached.Address(1) == vaddr && cached.Size(1) == size) { - return ClearAspect::Stencil; - } - return ClearAspect::None; - }; - - std::shared_ptr match; - ClearAspect aspect = ClearAspect::None; - bool incompatible = false; - uint32_t matches = 0; - for (const auto& cached: m_images) { - if (!cached->OverlapsRange(vaddr, size, true)) { - continue; - } - const auto candidate = classify(*cached); - if (candidate == ClearAspect::None) { - incompatible = true; - continue; - } - if (aspect == ClearAspect::None) { - aspect = candidate; - match = cached; - } else if (candidate != aspect) { - incompatible = true; - continue; - } - matches++; - } - if (matches > 1) { - EXIT("TextureCache: compute image clear has ambiguous exact aliases, " - "addr=0x%016" PRIx64 " size=0x%016" PRIx64 " aspect=%u matches=%u\n", - vaddr, size, static_cast(aspect), matches); - } - if (incompatible || match == nullptr) { - return false; - } - - float depth_clear = 0.0f; - if (aspect == ClearAspect::Depth && - !DecodePackedDepthClear(match->image->format, packed_clear, depth_clear)) { - return false; - } - uint8_t stencil_clear = 0; - if (aspect == ClearAspect::Stencil && !DecodePackedStencilClear(packed_clear, stencil_clear)) { - return false; - } - - RequireNoMetaOverlapLocked(vaddr, size); - const bool buffer_overlap = m_buffer_cache.HasPageOverlap(vaddr, size); - const bool buffer_cpu_modified = - buffer_overlap && m_buffer_cache.IsRegionCpuModified(vaddr, size); - const bool buffer_gpu_modified = - buffer_overlap && m_buffer_cache.IsRegionGpuModified(vaddr, size); - BufferImageCopySource source {nullptr, 0, vaddr, size, true}; - if (buffer_overlap && !buffer_cpu_modified && !buffer_gpu_modified) { - source = m_buffer_cache.ObtainBufferForImage(vaddr, size); - } - const bool buffer_source_valid = - !buffer_overlap || IsCoherentGuestImageSource(source, vaddr, size); - const bool image_cpu_modified = m_memory_tracker.IsRegionCpuModified(vaddr, size); - const bool invalid_common = !buffer_source_valid || buffer_cpu_modified || - buffer_gpu_modified || match->buffer_modified || image_cpu_modified; - if (aspect == ClearAspect::Color && invalid_common) { - EXIT("TextureCache: compute image clear requires exclusive GPU image ownership, " - "addr=0x%016" PRIx64 " size=0x%016" PRIx64 - " buffer_overlap=%d source_valid=%d buffer_cpu_modified=%d" - " buffer_gpu_modified=%d buffer_modified=%d image_cpu_modified=%d\n", - vaddr, size, buffer_overlap, buffer_source_valid, buffer_cpu_modified, - buffer_gpu_modified, match->buffer_modified, image_cpu_modified); - } - if (aspect != ClearAspect::Color && (!match->gpu_modified || invalid_common)) { - const bool depth = aspect == ClearAspect::Depth; - const char* plane = depth ? "depth" : "stencil"; - const char* source_name = depth ? "buffer" : "stencil"; - EXIT("TextureCache: compute %s clear requires a GPU-owned depth image and a clean " - "%s source, addr=0x%016" PRIx64 " size=0x%016" PRIx64 - " gpu_modified=%d buffer_overlap=%d source_valid=%d buffer_cpu_modified=%d" - " buffer_gpu_modified=%d buffer_modified=%d image_cpu_modified=%d\n", - plane, source_name, vaddr, size, match->gpu_modified, buffer_overlap, - buffer_source_valid, buffer_cpu_modified, buffer_gpu_modified, match->buffer_modified, - image_cpu_modified); - } - - auto vk_buffer = command.Handle(); - if (aspect == ClearAspect::Color) { - vk::ClearColorValue clear {}; - if (!DecodePackedColorClear(match->image->format, packed_clear, clear)) { + if (selected && selected != candidate_id) { return false; } - GraphicsRenderColorImageBarrier(vk_buffer, *match->image, - vk::ImageLayout::eTransferDstOptimal); - const vk::ImageSubresourceRange range {vk::ImageAspectFlagBits::eColor, 0, - VK_REMAINING_MIP_LEVELS, 0, match->image->layers}; - vk_buffer.clearColorImage(match->image->image, match->image->layout, &clear, 1, &range); - GraphicsRenderColorImageBarrier(vk_buffer, *match->image, RENDER_COLOR_IMAGE_LAYOUT); - if (!match->gpu_modified) { - m_memory_tracker.MarkRegionAsGpuModified(vaddr, size); - match->gpu_modified = true; + selected = candidate_id; + aspect = candidate; + } + if (!selected) { + return false; + } + auto& image = ResolveImage(selected); + vk::ClearColorValue color_clear {}; + float depth_clear = 0.0f; + uint8_t stencil_clear = 0; + if (aspect == vk::ImageAspectFlagBits::eColor) { + if (!DecodePackedColorClear(image.info.pixel_format, packed_clear, color_clear)) { + return false; + } + } else { + if ((aspect == vk::ImageAspectFlagBits::eDepth && + !DecodePackedDepthClear(image.info.pixel_format, packed_clear, depth_clear)) || + (aspect == vk::ImageAspectFlagBits::eStencil && + !DecodePackedStencilClear(packed_clear, stencil_clear))) { + return false; } - command.RetainResourceUntilFence(match); - return true; } - - if (match->image->layout == vk::ImageLayout::eUndefined || - match->image->layout == vk::ImageLayout::ePreinitialized) { - EXIT("TextureCache: compute %s clear has invalid source layout %u\n", - aspect == ClearAspect::Depth ? "depth" : "stencil", - static_cast(match->image->layout)); + if (m_buffer_cache.HasGpuDirtyBytes(address, size)) { + m_buffer_cache.DiscardGpuDirtyBytes(address, size); } - const auto old_layout = match->image->layout; - GraphicsRenderDepthStencilImageBarrier(vk_buffer, *match->image, - vk::ImageLayout::eTransferDstOptimal); - const vk::ClearDepthStencilValue clear {depth_clear, stencil_clear}; - const auto clear_aspect = static_cast( - aspect == ClearAspect::Depth ? vk::ImageAspectFlagBits::eDepth - : vk::ImageAspectFlagBits::eStencil); - const vk::ImageSubresourceRange range {clear_aspect, 0, VK_REMAINING_MIP_LEVELS, 0, - match->image->layers}; - vk_buffer.clearDepthStencilImage(match->image->image, match->image->layout, &clear, 1, &range); - GraphicsRenderDepthStencilImageBarrier(vk_buffer, *match->image, old_layout); - if (aspect == ClearAspect::Stencil) { - match->stencil_initialized = true; + if (image.IsBufferModified() || image.IsCpuDirty() || + m_memory_tracker.IsRegionCpuModified(image.info.data.address, image.info.data.size)) { + ImageDesc refresh {.info = image.info, .view_info = {}, .type = UploadBinding(image)}; + InitializeImage(selected, refresh); + if (image.info.samples == 1 && (image.IsBufferModified() || image.IsCpuDirty())) { + EXIT("TextureCache: image clear retained guest ownership\n"); + } } - command.RetainResourceUntilFence(match); + command.EndRendering(); + image.Transit(vk::ImageLayout::eTransferDstOptimal, vk::AccessFlagBits2::eTransferWrite, {}, + command.Handle()); + const vk::ImageSubresourceRange range {aspect, 0, VK_REMAINING_MIP_LEVELS, 0, + image.backing.layers}; + if (aspect == vk::ImageAspectFlagBits::eColor) { + command.Handle().clearColorImage(image.backing.image, vk::ImageLayout::eTransferDstOptimal, + &color_clear, 1, &range); + } else { + const vk::ClearDepthStencilValue clear {depth_clear, stencil_clear}; + command.Handle().clearDepthStencilImage( + image.backing.image, vk::ImageLayout::eTransferDstOptimal, &clear, 1, &range); + } + CommitGpuWrite(image); + RetainImage(command, selected); return true; } -void TextureCache::MarkGpuWritten(VulkanImage& image) { - std::lock_guard transaction(m_resource_mutex); - FaultSafeTextureLock lock(this, m_lock); - for (auto& cached: m_images) { - if (cached->image != &image) { - continue; - } - if (cached->kind != CachedImage::Kind::RenderTarget && - cached->kind != CachedImage::Kind::DepthTarget && - cached->kind != CachedImage::Kind::VideoOut) { - EXIT("TextureCache: sampled texture cannot be marked GPU-written, image=%p kind=%u\n", - static_cast(&image), static_cast(cached->kind)); - } - for (uint32_t i = 0; i < cached->RangeCount(); i++) { - if (m_buffer_cache.HasPageOverlap(cached->Address(i), cached->Size(i))) { - const bool cpu_modified = - m_buffer_cache.IsRegionCpuModified(cached->Address(i), cached->Size(i)); - const bool gpu_modified = - m_buffer_cache.IsRegionGpuModified(cached->Address(i), cached->Size(i)); - if (cpu_modified || gpu_modified) { - EXIT( - "TextureCache: GPU-written image aliases a dirty buffer, addr=0x%016" PRIx64 - " size=0x%016" PRIx64 " cpu_modified=%d gpu_modified=%d\n", - cached->Address(i), cached->Size(i), cpu_modified, gpu_modified); - } - } - if (m_memory_tracker.IsRegionCpuModified(cached->Address(i), cached->Size(i))) { - EXIT("TextureCache: GPU-write begins while image range is CPU-modified, " - "addr=0x%016" PRIx64 " size=0x%016" PRIx64 "\n", - cached->Address(i), cached->Size(i)); +void TextureCache::PrepareHostWrite(uint64_t address, uint64_t size) { + if (!GuestRange {address, size}.Valid()) { + EXIT("TextureCache: invalid host-write range\n"); + } + CacheLock lock(*this, m_lock); + InvalidateCpuAliases(address, size); + m_memory_tracker.ForEachDownloadRange(address, size, [](uint64_t, uint64_t) noexcept {}); + m_memory_tracker.MarkRegionAsCpuModified(address, size); +} + +void TextureCache::DownloadDepth(Image& image, Buffer& destination, uint64_t destination_offset) { + const auto& info = image.info; + const auto layers = info.resources.layers; + const auto full_slice_size = info.data.size / layers; + const auto transfer_bytes = DepthAspectTransferBytes(info.pixel_format); + const uint64_t texels_per_slice = static_cast(info.pitch) * info.extent.height; + EXIT_NOT_IMPLEMENTED(transfer_bytes == 0 || texels_per_slice > UINT32_MAX || + texels_per_slice > UINT64_MAX / transfer_bytes || + texels_per_slice > UINT64_MAX / info.bytes_per_block); + const uint64_t transfer_slice = texels_per_slice * transfer_bytes; + const uint64_t guest_slice = texels_per_slice * info.bytes_per_block; + EXIT_NOT_IMPLEMENTED(transfer_slice > UINT64_MAX / layers); + const uint64_t transfer_size = transfer_slice * layers; + EXIT_NOT_IMPLEMENTED(guest_slice > full_slice_size); + std::vector copies(layers); + for (uint32_t layer = 0; layer < layers; layer++) { + auto& copy = copies[layer]; + copy.bufferOffset = full_slice_size * layer; + copy.bufferRowLength = info.pitch; + copy.bufferImageHeight = info.extent.height; + copy.imageSubresource = {vk::ImageAspectFlagBits::eDepth, 0, layer, 1}; + copy.imageExtent = {info.extent.width, info.extent.height, 1}; + } + if (transfer_bytes == info.bytes_per_block) { + if (!info.IsTiled()) { + for (auto& copy: copies) { + copy.bufferOffset += destination_offset; } + image.Download(copies, destination.Handle(), destination_offset, info.data.size); + return; } - if (cached->kind == CachedImage::Kind::DepthTarget) { - if ((cached->depth.htile_address == 0) != (cached->depth.htile_size == 0)) { - EXIT("TextureCache: depth target has incomplete HTile range, addr=0x%016" PRIx64 - " size=0x%016" PRIx64 "\n", - cached->depth.htile_address, cached->depth.htile_size); - } - if (cached->depth.htile_address != 0) { - auto meta = m_surface_metas.find(cached->depth.htile_address); - if (meta == m_surface_metas.end() || - meta->second.size != cached->depth.htile_size) { - EXIT("TextureCache: depth target HTile metadata is missing or mismatched, " - "addr=0x%016" PRIx64 " size=0x%016" PRIx64 "\n", - cached->depth.htile_address, cached->depth.htile_size); - } - m_buffer_cache.ValidateGpuAccess(cached->depth.htile_address, - cached->depth.htile_size, false, true); - m_metadata_tracker.ForEachUploadRange( - cached->depth.htile_address, cached->depth.htile_size, true, - [](uint64_t, uint64_t) noexcept {}, []() noexcept {}); - meta->second.gpu_modified = true; - } - } - if (!cached->gpu_modified) { - for (uint32_t i = 0; i < cached->RangeCount(); i++) { - m_memory_tracker.MarkRegionAsGpuModified(cached->Address(i), cached->Size(i)); - } - cached->gpu_modified = true; + TileBlockLayout block {}; + EXIT_NOT_IMPLEMENTED( + !TileGetBlockLayout(TileBlockFamily::Depth64KB, info.bytes_per_block, block)); + std::vector tiles; + tiles.reserve(layers); + for (uint32_t layer = 0; layer < layers; layer++) { + const uint64_t offset = full_slice_size * layer; + tiles.push_back({block.family, block.bytes_per_element, offset, full_slice_size, offset, + full_slice_size, 0, info.extent.width, info.extent.height, 1, + info.pitch}); + tiles.back().surface_z = layer; } + m_tiler->TileImage(image, copies, destination.Handle(), destination_offset, info.data.size, + info.data.size, tiles); return; } - EXIT("TextureCache: GPU-written image is not registered, image=%p\n", - static_cast(&image)); -} - -void TextureCache::PrepareHostWrite(uint64_t vaddr, uint64_t size) { - if (vaddr == 0 || size == 0 || vaddr >= TRACKER_ADDRESS_SIZE || - size > TRACKER_ADDRESS_SIZE - vaddr) { - EXIT("TextureCache: invalid host-write range, addr=0x%016" PRIx64 " size=0x%016" PRIx64 - "\n", - vaddr, size); + EXIT_NOT_IMPLEMENTED(info.bytes_per_block != sizeof(uint16_t) || + transfer_bytes != sizeof(uint32_t)); + for (uint32_t layer = 0; layer < layers; layer++) { + copies[layer].bufferOffset = transfer_slice * layer; } - FaultSafeTextureLock lock(this, m_lock); - const bool metadata_overlap = HasMetaOverlapLocked(vaddr, size); - bool found = false; - for (const auto& cached: m_images) { - for (uint32_t range = 0; range < cached->RangeCount(); range++) { - // CPU-current depth targets use the same tracked guest refresh path as sampled and - // color images. GPU-owned targets and metadata aliases remain unsupported below. - const bool host_refreshable = (cached->kind == CachedImage::Kind::Texture || - cached->kind == CachedImage::Kind::RenderTarget || - cached->kind == CachedImage::Kind::DepthTarget) && - !cached->buffer_modified; - switch (ClassifyHostWriteOverlap(vaddr, size, cached->Address(range), - cached->Size(range), host_refreshable, - cached->gpu_modified, metadata_overlap)) { - case HostWriteOverlap::None: break; - case HostWriteOverlap::InvalidateImage: found = true; break; - case HostWriteOverlap::Unsupported: - EXIT("TextureCache: host write aliases unsupported image, " - "write=0x%016" PRIx64 "+0x%016" PRIx64 " image=0x%016" PRIx64 - "+0x%016" PRIx64 - " kind=%u gpu_modified=%d buffer_modified=%d metadata_overlap=%d\n", - vaddr, size, cached->Address(range), cached->Size(range), - static_cast(cached->kind), cached->gpu_modified, - cached->buffer_modified, metadata_overlap); - } - } + auto host_linear = m_tiler->GetScratchBuffer(transfer_size); + image.Download(copies, host_linear.buffer, 0, host_linear.size); + const bool tiled = info.IsTiled(); + auto guest_linear = tiled ? m_tiler->GetScratchBuffer(info.data.size) + : TileManager::Result {destination.Handle(), destination_offset, + destination.Size() - destination_offset}; + m_tiler->ConvertD16(host_linear, guest_linear, TileManager::D16Direction::Demote, + DepthAspectTransferFormat(info.pixel_format) == vk::Format::eD32Sfloat, + {.width = info.extent.width, + .height = info.extent.height, + .layers = layers, + .source_row_stride = static_cast(info.pitch) * sizeof(uint32_t), + .target_row_stride = static_cast(info.pitch) * sizeof(uint16_t), + .source_slice_stride = transfer_slice, + .target_slice_stride = full_slice_size}); + if (!tiled) { + return; } - if (!found) { - EXIT("TextureCache: host write expected a refreshable image overlap, addr=0x%016" PRIx64 - " size=0x%016" PRIx64 "\n", - vaddr, size); - } - // Explicitly invalidate on a write fault. The resource transaction is already held here, so - // writing first would recursively enter the cache. - m_memory_tracker.MarkRegionAsCpuModified(vaddr, size); - MarkSampledAliasesCpuDirtyLocked(vaddr, size); -} - -void TextureCache::SynchronizeColorImageToBufferLocked(CachedImage& cached, uint64_t write_address, - uint64_t write_size) { - const bool render_target = cached.kind == CachedImage::Kind::RenderTarget; - const bool video_out = cached.kind == CachedImage::Kind::VideoOut; - const bool storage = cached.kind == CachedImage::Kind::StorageTexture; - RenderTargetInfo target = cached.target; - if (storage) { - const auto& info = cached.info; - target.address = info.address; - target.size = info.size; - target.format = VulkanFormat(info.format); - target.width = info.width; - target.height = info.height; - target.pitch = info.pitch; - target.bytes_per_element = Prospero::RenderTargetBytesPerElement(info.format); - target.tile_mode = info.tile; - target.levels = info.levels; - } - if (video_out) { - const auto& info = cached.video_out; - target.address = info.address; - target.size = info.size; - target.format = info.format; - target.width = info.width; - target.height = info.height; - target.pitch = info.pitch; - target.bytes_per_element = info.bytes_per_element; - target.tile_mode = info.tile_mode; - } - const bool linear = target.tile_mode == Prospero::GpuEnumValue(Prospero::TileMode::kLinear); - const bool tiled = IsTiledRenderTarget(target); - const bool bgra16 = video_out && cached.video_out.bgra16; - TileSizeAlign exact {}; - bool single_slice = false; - if (IsSupportedStandard64RenderTarget(target)) { - exact = {static_cast(target.size), 65536}; - single_slice = true; - } else { - single_slice = TileGetRenderTargetSize(target.width, target.height, target.pitch, - target.bytes_per_element, exact); - } - const bool layered_size = - single_slice && static_cast(exact.size) * target.layers == target.size; - if (storage && target.levels > 1) { - single_slice = TileGetRenderTargetMipLayout(target.width, target.height, target.pitch, - target.bytes_per_element, target.levels, exact, - nullptr, nullptr); - } - const bool exact_tiled = tiled && exact.align == 65536 && - (storage ? single_slice && exact.size == target.size : layered_size); - const bool valid_kind = - render_target || storage || - (video_out && cached.video_out.compression == VideoOutCompression::Uncompressed); - if (!valid_kind || !cached.gpu_modified || cached.buffer_modified || - (!storage && target.levels != 1) || target.size > UINT32_MAX || (!linear && !exact_tiled) || - HasMetaOverlapLocked(target.address, target.size)) { - EXIT("TextureCache: unsupported color-image buffer synchronization, " - "addr=0x%016" PRIx64 " size=0x%016" PRIx64 - " extent=%ux%u pitch=%u bpe=%u levels=%u layers=%u tile=%u" - " kind=%u compression=%u gpu_modified=%d buffer_modified=%d\n", - target.address, target.size, target.width, target.height, target.pitch, - target.bytes_per_element, target.levels, target.layers, target.tile_mode, - static_cast(cached.kind), - video_out ? static_cast(cached.video_out.compression) : 0, - cached.gpu_modified, cached.buffer_modified); - } - if (write_address < target.address || write_size == 0 || - write_address - target.address > target.size || - write_size > target.size - (write_address - target.address)) { - EXIT("TextureCache: image synchronization write is outside backing, " - "write=0x%016" PRIx64 "+0x%016" PRIx64 " image=0x%016" PRIx64 "+0x%016" PRIx64 "\n", - write_address, write_size, target.address, target.size); - } - const auto slice_size = target.size / target.layers; - if (cached.image->format != target.format || cached.image->extent.width != target.width || - cached.image->extent.height != target.height || - (tiled && !IsSupportedRenderTargetElementSize(target.bytes_per_element)) || - HasMetaOverlapLocked(target.address, target.size)) { - EXIT("TextureCache: color-image buffer synchronization storage mismatch, " - "addr=0x%016" PRIx64 " size=0x%016" PRIx64 " linear=0x%016" PRIx64 "\n", - target.address, target.size); - } - - Transfer::WaitForQueueIdle(); - std::vector regions; - std::vector tiled_regions; - TextureUploadLayout tiled_layout {}; - uint32_t tiled_format = 0; - if (storage) { - const bool array_texture = TextureIsLayeredTexture(cached.info.type); - const bool volume_texture = TextureIs3DTexture(cached.info.type); - tiled_format = cached.info.format; - tiled_layout = TextureCalcUploadLayout( - cached.info.format, cached.info.width, cached.info.height, cached.info.levels, - cached.info.depth, cached.info.pitch, cached.info.tile, cached.info.size, true, - volume_texture, "StorageTextureReadback"); - tiled_regions = TextureBuildUploadRegions( - tiled_layout, cached.image->format, cached.info.width, cached.info.height, - cached.info.depth, cached.info.levels, array_texture, volume_texture, - TextureUploadDestination::MipLevels); - regions = TextureBuildDownloadRegions(tiled_regions); - } else if (tiled && !bgra16) { - tiled_format = ImageOps::RenderTargetTransferFormat(target.bytes_per_element); - tiled_layout = TextureCalcUploadLayout( - tiled_format, target.width, target.height, target.levels, target.layers, target.pitch, - target.tile_mode, target.size, false, false, "ColorBufferTransition"); - tiled_regions = TextureBuildUploadRegions( - tiled_layout, target.format, target.width, target.height, target.layers, target.levels, - target.layers > 1, false, TextureUploadDestination::MipLevels); - regions = TextureBuildDownloadRegions(tiled_regions); - } else { - regions = Transfer::MakeLayeredImageBufferCopies(target.layers, slice_size, target.pitch, - target.width, target.height); - } - if (tiled && !bgra16) { - std::vector infos; - const uint32_t depth = storage ? cached.info.depth : target.layers; - EXIT_NOT_IMPLEMENTED(!TextureBuildGpuTileInfos(target.size, tiled_regions, tiled_layout, - tiled_format, depth, target.levels, infos)); - m_buffer_transition_guest.resize(target.size); - Transfer::DownloadTiledImage(m_buffer_transition_guest.data(), target.size, target.size, - infos, regions, *cached.image, cached.image->layout); - Libs::LibKernel::Memory::WriteBacking(target.address, m_buffer_transition_guest.data(), - target.size); - } else if (tiled) { - std::vector linear_data(target.size); - Transfer::DownloadImage(linear_data.data(), target.size, regions, *cached.image, - cached.image->layout); - ImageOps::SwapVideoOutBgra16(linear_data.data(), target.size); - TileBlockLayout block {}; - EXIT_NOT_IMPLEMENTED(!TileGetBlockLayout(TileBlockFamily::RenderTarget64KB, - target.bytes_per_element, block)); - const GpuTileInfo info { - block.family, block.bytes_per_element, 0, target.size, 0, target.size, 0, - target.width, target.height, 1, target.pitch}; - m_buffer_transition_guest.resize(target.size); - GpuTile(linear_data.data(), m_buffer_transition_guest.data(), target.size, target.size, - std::span(&info, 1)); - Libs::LibKernel::Memory::WriteBacking(target.address, m_buffer_transition_guest.data(), - target.size); - } else { - Transfer::ProcessDownloadedImage(target.size, regions, *cached.image, cached.image->layout, - [&](std::span linear_data) { - Libs::LibKernel::Memory::WriteBacking( - target.address, linear_data.data(), target.size); - }); - } - m_memory_tracker.ForEachDownloadRange(target.address, target.size, - [](uint64_t, uint64_t) noexcept {}); - // Only the impending buffer-write range needs publication into BufferCache ownership. The - // complete image backing was reconstructed above so a later partial-buffer rebind can detile - // the coherent mip chain without requiring one cached buffer to contain the whole image. - m_buffer_cache.PublishImageBacking(write_address, write_size); - cached.gpu_modified = false; - cached.buffer_modified = true; -} - -void TextureCache::SynchronizeDepthImageToBufferLocked(CachedImage& cached, uint64_t write_address, - uint64_t write_size) { - const auto& info = cached.depth; - const bool has_stencil = info.stencil_address != 0 || info.stencil_size != 0; - const bool has_htile = info.htile_address != 0 || info.htile_size != 0; - TileSizeAlign expected_stencil {}; - TileSizeAlign expected_htile {}; - TileSizeAlign expected_depth {}; - const bool d16 = - info.guest_format == Prospero::GpuEnumValue(Prospero::BufferFormat::k16UNorm) && - info.format == vk::Format::eD16Unorm && info.bytes_per_element == 2; - const bool d32 = - info.guest_format == Prospero::GpuEnumValue(Prospero::BufferFormat::k32Float) && - info.format == vk::Format::eD32Sfloat && info.bytes_per_element == 4; - const bool layout = - (d16 || d32) && TileGetDepthSize(info.width, info.height, 0, - Prospero::GpuEnumValue(d16 ? Prospero::DepthFormat::kZ16 - : Prospero::DepthFormat::kZ32F), - Prospero::GpuEnumValue(Prospero::StencilFormat::kInvalid), - false, expected_stencil, expected_htile, expected_depth); - if (cached.kind != CachedImage::Kind::DepthTarget || !cached.gpu_modified || - cached.buffer_modified || write_address != info.address || write_size != info.size || - has_stencil || has_htile || info.layers != 1 || !layout || expected_depth.align != 65536 || - expected_depth.size != info.size || cached.image->format != info.format || - cached.image->extent.width != info.width || cached.image->extent.height != info.height || - HasMetaOverlapLocked(info.address, info.size)) { - EXIT("TextureCache: unsupported depth-image buffer synchronization, " - "write=0x%016" PRIx64 "+0x%016" PRIx64 " depth=0x%016" PRIx64 "+0x%016" PRIx64 - " extent=%ux%u layers=%u format=%d guest=%u bpe=%u" - " stencil=%d htile=%d gpu_modified=%d buffer_modified=%d\n", - write_address, write_size, info.address, info.size, info.width, info.height, - info.layers, static_cast(info.format), info.guest_format, info.bytes_per_element, - has_stencil, has_htile, cached.gpu_modified, cached.buffer_modified); - } - Transfer::WaitForQueueIdle(); - const auto regions = Transfer::MakeLayeredImageBufferCopies( - 1, info.size, info.pitch, info.width, info.height, vk::ImageAspectFlagBits::eDepth); TileBlockLayout block {}; EXIT_NOT_IMPLEMENTED( - !TileGetBlockLayout(TileBlockFamily::Depth64KB, info.bytes_per_element, block)); - const GpuTileInfo tile_info {block.family, - block.bytes_per_element, - 0, - info.size, - 0, - info.size, - 0, - info.width, - info.height, - 1, - info.pitch}; - m_buffer_transition_guest.resize(info.size); - Transfer::DownloadTiledImage(m_buffer_transition_guest.data(), info.size, info.size, - std::span(&tile_info, 1), regions, - *cached.image, cached.image->layout); - Libs::LibKernel::Memory::WriteBacking(info.address, m_buffer_transition_guest.data(), - info.size); - m_memory_tracker.ForEachDownloadRange(info.address, info.size, - [](uint64_t, uint64_t) noexcept {}); - m_buffer_cache.PublishImageBacking(write_address, write_size); - cached.gpu_modified = false; - cached.buffer_modified = true; + !TileGetBlockLayout(TileBlockFamily::Depth64KB, info.bytes_per_block, block)); + std::vector tiles; + tiles.reserve(layers); + for (uint32_t layer = 0; layer < layers; layer++) { + const uint64_t offset = full_slice_size * layer; + tiles.push_back({block.family, block.bytes_per_element, offset, full_slice_size, offset, + full_slice_size, 0, info.extent.width, info.extent.height, 1, info.pitch}); + tiles.back().surface_z = layer; + } + m_tiler->Tile(guest_linear.buffer, guest_linear.offset, info.data.size, destination.Handle(), + destination_offset, info.data.size, tiles); } -bool TextureCache::InvalidateMemoryFromGPU(uint64_t vaddr, uint64_t size, - bool formatted_buffer_write) { - if (vaddr == 0 || size == 0 || vaddr >= TRACKER_ADDRESS_SIZE || - size > TRACKER_ADDRESS_SIZE - vaddr) { - EXIT("TextureCache: invalid GPU invalidation range, addr=0x%016" PRIx64 - " size=0x%016" PRIx64 "\n", - vaddr, size); +void TextureCache::DownloadImageData(Image& image, Buffer& destination, uint64_t destination_offset, + DownloadPlan plan) { + if (!plan.valid) { + EXIT("TextureCache: invalid image download plan\n"); } - FaultSafeTextureLock lock(this, m_lock); - for (auto it = m_surface_metas.begin(); it != m_surface_metas.end();) { - switch (ClassifyImageRangeOverlap(vaddr, size, it->first, it->second.size)) { - case ImageRangeOverlap::None: - case ImageRangeOverlap::PageOnly: ++it; continue; - case ImageRangeOverlap::Bytes: break; - } - LOGF("TextureCache: discarding overwritten virtual metadata, write=0x%016" PRIx64 - "+0x%016" PRIx64 " metadata=0x%016" PRIx64 "+0x%016" PRIx64 "\n", - vaddr, size, it->first, it->second.size); - if (it->second.gpu_modified) { - m_metadata_tracker.ForEachDownloadRange(it->first, it->second.size, - [](uint64_t, uint64_t) noexcept {}); - } - m_metadata_tracker.UntrackMemory(it->first, it->second.size); - it = m_surface_metas.erase(it); + if (plan.depth) { + DownloadDepth(image, destination, destination_offset); + return; } - auto match = m_images.end(); - BufferImageWrite action = BufferImageWrite::None; - for (auto it = m_images.begin(); it != m_images.end(); ++it) { - auto& cached = **it; - if (!cached.OverlapsRange(vaddr, size, false)) { - continue; + + auto& color = plan.color; + const auto transform = color.swap_bgra16 ? TileManager::ColorTransform::SwapBgra16 + : TileManager::ColorTransform::None; + if (!color.tiled) { + if (transform == TileManager::ColorTransform::SwapBgra16) { + auto linear = m_tiler->GetScratchBuffer(image.info.data.size); + image.Download(color.regions, linear.buffer, 0, linear.size); + m_tiler->SwapBgra16(linear, + {destination.Handle(), destination_offset, image.info.data.size}); + return; } - const auto next = ClassifyBufferImageWrite(vaddr, size, cached.Address(), cached.Size(), - cached.BufferBinding(), cached.gpu_modified, - formatted_buffer_write, cached.buffer_modified); - if (match != m_images.end() || next == BufferImageWrite::None || - next == BufferImageWrite::Unsupported) { - EXIT("TextureCache: unsupported GPU invalidation alias, addr=0x%016" PRIx64 - " size=0x%016" PRIx64 " cached_kind=%u cached=0x%016" PRIx64 "+0x%016" PRIx64 - " gpu_modified=%d buffer_modified=%d formatted=%d ambiguous=%d\n", - vaddr, size, static_cast(cached.kind), cached.Address(), cached.Size(), - cached.gpu_modified, cached.buffer_modified, formatted_buffer_write, - match != m_images.end()); + for (auto& copy: color.regions) { + copy.bufferOffset += destination_offset; } - match = it; - action = next; + image.Download(color.regions, destination.Handle(), destination_offset, + image.info.data.size); + return; } - if (match == m_images.end()) { + + m_tiler->TileImage(image, color.regions, destination.Handle(), destination_offset, + image.info.data.size, image.info.data.size, color.tiles, transform); +} + +std::pair TextureCache::MapDownload(uint64_t size, uint64_t alignment) { + if (size == 0) { + EXIT("TextureCache: cannot map an empty image download\n"); + } + auto& download = m_buffer_cache.GetUtilityBuffer(MemoryUsage::Download); + auto mapping = download.Map(size, std::max(alignment, 4)); + if (mapping.first == nullptr) { + EXIT("TextureCache: failed to map reusable download buffer\n"); + } + download.Commit(); + return mapping; +} + +void TextureCache::QueueDownload(GuestRange range, StreamBuffer& download, uint8_t* mapped, + uint64_t offset) { + vk::BufferMemoryBarrier barrier {}; + barrier.sType = vk::StructureType::eBufferMemoryBarrier; + barrier.srcAccessMask = vk::AccessFlagBits::eMemoryWrite | vk::AccessFlagBits::eTransferWrite | + vk::AccessFlagBits::eShaderWrite; + barrier.dstAccessMask = vk::AccessFlagBits::eHostRead; + barrier.srcQueueFamilyIndex = VK_QUEUE_FAMILY_IGNORED; + barrier.dstQueueFamilyIndex = VK_QUEUE_FAMILY_IGNORED; + barrier.buffer = download.Handle(); + barrier.offset = offset; + barrier.size = range.size; + m_scheduler.EndRendering(); + m_scheduler.Current().Handle().pipelineBarrier(vk::PipelineStageFlagBits::eAllCommands, + vk::PipelineStageFlagBits::eHost, {}, 0, nullptr, + 1, &barrier, 0, nullptr); + const auto tick = m_scheduler.CurrentTick(); + m_buffer_cache.BeginBackingPublication(range.address, range.size, tick); + m_scheduler.DeferPriorityOperation([this, &download, range, mapped, offset, tick] { + download.Invalidate(offset, range.size); + LibKernel::Memory::WriteBacking(range.address, mapped, range.size); + m_buffer_cache.CompleteBackingPublication(range.address, range.size, tick); + }); +} + +bool TextureCache::TryDownloadImage(ImageId id) { + auto& image = ResolveImage(id); + if (image.depth_id) { return false; } - auto& cached = **match; - switch (action) { - case BufferImageWrite::InvalidateTexture: - case BufferImageWrite::InvalidateVideoOut: - case BufferImageWrite::InvalidateStorageTexture: - case BufferImageWrite::InvalidateDepthTarget: - case BufferImageWrite::InvalidateRenderTarget: cached.buffer_modified = true; return true; - case BufferImageWrite::SynchronizeRenderTarget: - case BufferImageWrite::SynchronizeStorageTexture: - SynchronizeColorImageToBufferLocked(cached, vaddr, size); - return true; - case BufferImageWrite::SynchronizeDepthTarget: - SynchronizeDepthImageToBufferLocked(cached, vaddr, size); - return true; - case BufferImageWrite::SynchronizeVideoOut: - SynchronizeColorImageToBufferLocked(cached, vaddr, size); - return true; - case BufferImageWrite::None: - case BufferImageWrite::Unsupported: - EXIT("TextureCache: invalid GPU invalidation action %u\n", - static_cast(action)); + auto plan = BuildDownload(image); + if (!plan.valid || !SafeToDownload(image)) { + return false; } - return false; + const auto range = image.info.data; + auto [mapped, offset] = MapDownload(range.size, image.info.bytes_per_block); + auto& download = m_buffer_cache.GetUtilityBuffer(MemoryUsage::Download); + if (!LibKernel::Memory::TryReadBacking(range.address, mapped, range.size)) { + return false; + } + download.Flush(offset, range.size); + + DownloadImageData(image, download, offset, std::move(plan)); + + QueueDownload(range, download, mapped, offset); + return true; } -DepthStencilVulkanImage* TextureCache::FindDepthTargetByRange(CommandBuffer& command, - uint64_t vaddr, uint64_t size, - bool allow_containing_sampled) { - if (vaddr == 0 || size == 0 || vaddr >= TRACKER_ADDRESS_SIZE || - size > TRACKER_ADDRESS_SIZE - vaddr) { - EXIT("TextureCache: invalid depth-target range query, addr=0x%016" PRIx64 - " size=0x%016" PRIx64 "\n", - vaddr, size); +void TextureCache::DownloadImage(ImageId id) { + if (!TryDownloadImage(id)) { + EXIT("TextureCache: unsupported image readback\n"); } - FaultSafeTextureLock lock(this, m_lock); - CachedImage* found = nullptr; - for (auto* cached: FindImagesInRegionLocked(vaddr, size, false)) { - if (cached->kind != CachedImage::Kind::DepthTarget || - !cached->OverlapsRange(vaddr, size, false)) { - continue; - } - const bool containing_sampled = - allow_containing_sampled && vaddr == cached->depth.address && size > cached->depth.size; - if ((!IsDepthTargetRangeCompatible(cached->depth, vaddr, size) && !containing_sampled) || - found != nullptr) { - EXIT("TextureCache: incompatible or ambiguous depth-target range, addr=0x%016" PRIx64 - " size=0x%016" PRIx64 " cached=0x%016" PRIx64 "+0x%016" PRIx64 " previous=%p\n", - vaddr, size, cached->depth.address, cached->depth.size, - static_cast(found)); - } - if (containing_sampled) { - return nullptr; - } - const bool stencil_range = - vaddr == cached->depth.stencil_address && size == cached->depth.stencil_size; - if (stencil_range && !cached->stencil_initialized) { - EXIT("TextureCache: sampled stencil range is uninitialized, addr=0x%016" PRIx64 - " size=0x%016" PRIx64 "\n", - vaddr, size); - } - found = cached; - } - if (found == nullptr) { - return nullptr; - } - const auto owner = std::find_if(m_images.begin(), m_images.end(), - [found](const auto& image) { return image.get() == found; }); - if (owner == m_images.end()) { - EXIT("TextureCache: page-table depth target has no cache owner\n"); - } - command.RetainResourceUntilFence(*owner); - return static_cast(found->image); + m_scheduler.FinishCurrent(); + m_scheduler.DrainPriorityOperations(); } -RenderTextureVulkanImage* TextureCache::FindRenderTargetByRange(CommandBuffer& command, - uint64_t vaddr, uint64_t size) { - if (vaddr == 0 || size == 0 || vaddr >= TRACKER_ADDRESS_SIZE || - size > TRACKER_ADDRESS_SIZE - vaddr) { - EXIT("TextureCache: invalid render-target range query, addr=0x%016" PRIx64 - " size=0x%016" PRIx64 "\n", - vaddr, size); +bool TextureCache::SynchronizeImageToBuffer(ImageId id) { + auto& image = ResolveImage(id); + if (image.depth_id) { + return true; } - FaultSafeTextureLock lock(this, m_lock); - CachedImage* found = nullptr; - for (auto* cached: FindImagesInRegionLocked(vaddr, size, false)) { - if (cached->kind != CachedImage::Kind::RenderTarget || - !ImageRangeOverlaps(vaddr, size, cached->Address(), cached->Size())) { - continue; - } - const auto slice_size = - cached->target.layers != 0 ? cached->target.size / cached->target.layers : 0; - const bool contained = vaddr == cached->target.address && slice_size != 0 && - cached->target.size % cached->target.layers == 0 && - size <= cached->target.size && size % slice_size == 0; - if (!contained) { - continue; - } - if (found != nullptr) { - EXIT("TextureCache: incompatible or ambiguous render-target range, addr=0x%016" PRIx64 - " size=0x%016" PRIx64 " cached=0x%016" PRIx64 "+0x%016" PRIx64 " previous=%p\n", - vaddr, size, cached->Address(), cached->Size(), static_cast(found)); - } - found = cached; + auto plan = BuildDownload(image); + if (!plan.valid) { + return false; } - if (found == nullptr) { - return nullptr; + const auto range = image.info.data; + const bool refresh = image.IsDefinitelyCpuDirty() || + m_memory_tracker.IsRegionCpuModified(range.address, range.size); + if (refresh) { + RefreshImage(id, + ImageDesc {.info = image.info, .view_info = {}, .type = UploadBinding(image)}); } - const auto owner = std::find_if(m_images.begin(), m_images.end(), - [found](const auto& image) { return image.get() == found; }); - if (owner == m_images.end()) { - EXIT("TextureCache: page-table render target has no cache owner\n"); + if (!image.IsGpuModified()) { + return true; } - command.RetainResourceUntilFence(*owner); - return static_cast(found->image); + if (image.IsDefinitelyCpuDirty() || image.IsBufferModified() || + m_memory_tracker.IsRegionCpuModified(range.address, range.size)) { + EXIT("TextureCache: image mirror source is not native-current\n"); + } + auto [destination, offset] = + m_buffer_cache.ObtainBufferForImageWrite(range.address, range.size); + if (destination == nullptr) { + EXIT("TextureCache: failed to allocate image mirror\n"); + } + DownloadImageData(image, *destination, offset, std::move(plan)); + m_scheduler.Current().RetainResourceUntilFence(destination); + m_buffer_cache.PublishImageBuffer(range.address, range.size); + image.MarkBufferModified(); + RetainImage(m_scheduler.Current(), id); + ReleaseGpuTracking(id); + return true; } -TextureCache::RegionInfo TextureCache::QueryRegion(uint64_t vaddr, uint64_t size) { - if (vaddr == 0 || size == 0 || vaddr >= TRACKER_ADDRESS_SIZE || - size > TRACKER_ADDRESS_SIZE - vaddr) { - EXIT("TextureCache: invalid region query, addr=0x%016" PRIx64 " size=0x%016" PRIx64 "\n", - vaddr, size); +bool TextureCache::SynchronizeImageToBuffer(uint64_t address, uint64_t size) { + if (!GuestRange {address, size}.Valid()) { + return false; } - FaultSafeTextureLock lock(this, m_lock); - RegionInfo result; - const auto candidates = FindImagesInRegionLocked(vaddr, size, true); - result.image_pages = !candidates.empty(); - for (const auto* cached: candidates) { - const bool bytes = cached->OverlapsRange(vaddr, size, false); - result.image_bytes |= bytes; - result.gpu_image_bytes |= bytes && cached->gpu_modified; - result.non_sampled_pages |= cached->kind != CachedImage::Kind::Texture; + CacheLock lock(*this, m_lock); + ImageId selected {}; + for (const auto id: FindImagesInRegion(address, size, true)) { + auto owner = ResolveOwner(id); + if (owner == nullptr || !owner->GpuOverlaps(address, size)) { + continue; + } + if (selected) { + EXIT("TextureCache: ambiguous image-to-buffer synchronization\n"); + } + selected = id; } - for (const auto& [address, metadata]: m_surface_metas) { - result.metadata_pages |= ImagePageRangesOverlap(vaddr, size, address, metadata.size); - result.metadata_bytes |= ImageRangeOverlaps(vaddr, size, address, metadata.size); + if (!selected) { + return false; + } + return SynchronizeImageToBuffer(selected); +} + +bool TextureCache::InvalidateMemoryFromGPU(uint64_t address, uint64_t size, + bool formatted_buffer_write) { + if (!GuestRange {address, size}.Valid()) { + return false; + } + CacheLock lock(*this, m_lock); + bool found = false; + for (const auto id: FindImagesInRegion(address, size, true)) { + auto owner = ResolveOwner(id); + if (owner == nullptr || owner->depth_id || !owner->Overlaps(address, size)) { + continue; + } + if (owner->IsGpuModified()) { + if (!formatted_buffer_write) { + EXIT("TextureCache: buffer write aliases GPU-modified image\n"); + } + ReleaseGpuTracking(id); + } + owner->MarkBufferModified(); + found = true; + } + return found; +} + +TextureCache::RegionInfo TextureCache::QueryRegion(uint64_t address, uint64_t size) { + RegionInfo result {}; + if (!GuestRange {address, size}.Valid()) { + return result; + } + CacheLock lock(*this, m_lock); + for (const auto id: FindImagesInRegion(address, size, true)) { + auto owner = ResolveOwner(id); + if (owner == nullptr || owner->depth_id || !owner->Overlaps(address, size, true)) { + continue; + } + result.image_pages = true; + result.image_bytes |= owner->Overlaps(address, size); + result.gpu_image_bytes |= owner->GpuOverlaps(address, size); } - result.gpu_metadata_bytes = - result.metadata_bytes && m_metadata_tracker.IsRegionGpuModified(vaddr, size); return result; } -bool TextureCache::ResolveMetaRange(uint64_t vaddr, uint64_t size, MetaRangeInfo& info) { - if (vaddr == 0 || size == 0 || vaddr >= TRACKER_ADDRESS_SIZE || - size > TRACKER_ADDRESS_SIZE - vaddr) { - return false; - } - FaultSafeTextureLock lock(this, m_lock); - MetaRangeInfo found {}; - bool matched = false; - for (const auto& [address, metadata]: m_surface_metas) { - const auto slice_size = metadata.size / metadata.layers; - const bool full = vaddr == address && size == metadata.size; - const auto offset = vaddr >= address ? vaddr - address : UINT64_MAX; - const bool slice = - !full && size == slice_size && offset < metadata.size && offset % slice_size == 0; - if (!full && !slice) { +void TextureCache::InvalidateCpuAliases(uint64_t address, uint64_t size) { + for (const auto id: FindImagesInRegion(address, size, true)) { + auto owner = ResolveOwner(id); + if (owner == nullptr || owner->depth_id) { continue; } - MetaRangeInfo candidate {.metadata_address = address, - .metadata_size = metadata.size, - .slice = full ? 0u : static_cast(offset / slice_size), - .full = full}; - if (matched && (candidate.metadata_address != found.metadata_address || - candidate.metadata_size != found.metadata_size || - candidate.slice != found.slice || candidate.full != found.full)) { - EXIT("TextureCache: ambiguous exact metadata range, request=0x%016" PRIx64 - "+0x%016" PRIx64 " first=0x%016" PRIx64 "+0x%016" PRIx64 - " slice=%u full=%d second=0x%016" PRIx64 "+0x%016" PRIx64 " slice=%u full=%d\n", - vaddr, size, found.metadata_address, found.metadata_size, found.slice, found.full, - candidate.metadata_address, candidate.metadata_size, candidate.slice, - candidate.full); + owner->InvalidateCpuWrite(address, size); + if (owner->NeedsMaybeCpuHash()) { + owner->SetMaybeCpuHash(owner->HashGuestEdges()); } - found = candidate; - matched = true; } - if (matched) { - info = found; - } - return matched; } -void TextureCache::RegisterMeta(uint64_t vaddr, uint64_t size, uint32_t layers) { - if (vaddr == 0 || size == 0 || vaddr >= TRACKER_ADDRESS_SIZE || - size > TRACKER_ADDRESS_SIZE - vaddr || (vaddr & 0x7fffu) != 0 || layers == 0 || - layers > 32 || size % layers != 0) { - EXIT("TextureCache: invalid metadata registration, addr=0x%016" PRIx64 " size=0x%016" PRIx64 - " layers=%u\n", - vaddr, size, layers); +void TextureCache::RestoreGpuTracking(const Image& image) { + if (!image.IsGpuModified()) { + return; } + constexpr uint64_t page_mask = TRACKER_PAGE_SIZE - 1; + const auto range = image.info.data; + const auto begin = range.address & ~page_mask; + const auto end = (range.End() + page_mask) & ~page_mask; + for (auto page = begin; page < end; page += TRACKER_PAGE_SIZE) { + if (!m_memory_tracker.IsRegionGpuModified(page, TRACKER_PAGE_SIZE) && + !m_memory_tracker.IsRegionCpuModified(page, TRACKER_PAGE_SIZE)) { + m_memory_tracker.MarkRegionAsGpuModified(page, TRACKER_PAGE_SIZE); + } + } +} + +void TextureCache::ReleaseGpuTracking(ImageId id) { + auto owner = ResolveOwner(id); + if (owner == nullptr || !owner->IsGpuModified()) { + return; + } + const auto released = owner->info.data; + owner->ClearGpuModified(); + m_memory_tracker.ForEachDownloadRange(released.address, released.size, + [](uint64_t, uint64_t) noexcept {}); + for (const auto candidate: FindImagesInRegion(released.address, released.size, true)) { + const auto survivor = ResolveOwner(candidate); + if (survivor != nullptr && survivor.get() != owner.get() && !survivor->depth_id) { + RestoreGpuTracking(*survivor); + } + } + RestoreGpuTracking(*owner); +} + +bool TextureCache::IsMeta(uint64_t address) { + CacheLock lock(*this, m_lock); + return m_surface_metas.contains(address); +} + +bool TextureCache::IsMetaCleared(uint64_t address, uint32_t slice) { + CacheLock lock(*this, m_lock); + const auto found = m_surface_metas.find(address); + if (found == m_surface_metas.end()) { + return false; + } + return (found->second.clear_mask & (1u << slice)) != 0; +} + +bool TextureCache::ClearMeta(uint64_t address) { std::lock_guard transaction(m_resource_mutex); - if (m_buffer_cache.HasPageOverlap(vaddr, size)) { - // Register virtual surface metadata independently of an earlier buffer view. Kyty's split - // caches first publish any dirty buffer bytes; clean partial views use guest backing. - const auto source = m_buffer_cache.ObtainBufferForImage(vaddr, size); - if (!IsCoherentGuestImageSource(source, vaddr, size)) { - EXIT("TextureCache: metadata buffer source is inconsistent, addr=0x%016" PRIx64 - " size=0x%016" PRIx64 " source=%p source_addr=0x%016" PRIx64 - " source_size=0x%016" PRIx64 " current=%d\n", - vaddr, size, static_cast(source.buffer), source.address, source.size, - source.cpu_current); - } - } - FaultSafeTextureLock lock(this, m_lock); - auto existing = m_surface_metas.find(vaddr); - uint64_t range_vaddr = vaddr; - uint64_t range_size = size; - if (existing != m_surface_metas.end()) { - const auto slice_size = size / layers; - const auto old_slice_size = existing->second.size / existing->second.layers; - if (slice_size != old_slice_size || - (size > existing->second.size) != (layers > existing->second.layers)) { - EXIT("TextureCache: incompatible metadata backing growth\n"); - } - if (layers <= existing->second.layers) { - return; - } - range_vaddr += existing->second.size; - range_size -= existing->second.size; - } - for (const auto& [address, meta]: m_surface_metas) { - if (address != vaddr && - ImagePageRangesOverlap(range_vaddr, range_size, address, meta.size)) { - LOGF("TextureCache: registering overlapping metadata views, requested=0x%016" PRIx64 - "+0x%016" PRIx64 " existing=0x%016" PRIx64 "+0x%016" PRIx64 "\n", - range_vaddr, range_size, address, meta.size); - } - } - std::vector retire; - for (const auto& cached: m_images) { - if (!cached->OverlapsRange(range_vaddr, range_size, true)) { - continue; - } - const bool sampled = cached->kind == CachedImage::Kind::Texture; - const bool writable_image = cached->kind == CachedImage::Kind::StorageTexture || - cached->kind == CachedImage::Kind::RenderTarget || - cached->kind == CachedImage::Kind::DepthTarget; - const bool gpu_modified = cached->gpu_modified || m_memory_tracker.IsRegionGpuModified( - cached->Address(), cached->Size()); - const bool cpu_dirty = - cached->kind == CachedImage::Kind::StorageTexture && - (cached->info.IsCpuDirty() || - m_memory_tracker.IsRegionCpuModified(cached->info.address, cached->info.size)); - const auto overlap = ClassifyMetaImageOverlap(sampled, writable_image, gpu_modified, - cached->buffer_modified, cpu_dirty); - if (overlap == MetaImageOverlap::Unsupported) { - EXIT("TextureCache: metadata aliases image pages, addr=0x%016" PRIx64 - " size=0x%016" PRIx64 " image_kind=%u image=0x%016" PRIx64 "+0x%016" PRIx64 - " gpu_modified=%d buffer_modified=%d\n", - range_vaddr, range_size, static_cast(cached->kind), cached->Address(), - cached->Size(), cached->gpu_modified, cached->buffer_modified); - } - if (overlap == MetaImageOverlap::RetireImage) { - retire.push_back(cached.get()); - continue; - } - LOGF("TextureCache: metadata aliases a CPU-current sampled image, metadata=0x%016" PRIx64 - "+0x%016" PRIx64 " image=0x%016" PRIx64 "+0x%016" PRIx64 "\n", - range_vaddr, range_size, cached->Address(), cached->Size()); - } - if (!retire.empty()) { - RequireRetirementIsolation(retire, "metadata", range_vaddr, range_size); - for (const auto* cached: retire) { - LOGF("TextureCache: retiring a guest-current image for metadata reuse, " - "metadata=0x%016" PRIx64 "+0x%016" PRIx64 " image=0x%016" PRIx64 "+0x%016" PRIx64 - " kind=%u\n", - range_vaddr, range_size, cached->Address(), cached->Size(), - static_cast(cached->kind)); - } - RetireDepthMetadataLocked(retire, vaddr); - RetireImages(retire); - } - if (existing == m_surface_metas.end()) { - m_surface_metas.emplace(vaddr, MetaDataInfo {.size = size, .layers = layers}); - } else { - existing->second.size = size; - existing->second.layers = layers; - } -} - -bool TextureCache::IsMeta(uint64_t vaddr) { - FaultSafeTextureLock lock(this, m_lock); - return m_surface_metas.contains(vaddr); -} - -bool TextureCache::IsMetaRange(uint64_t vaddr, uint64_t size) { - if (vaddr == 0 || size == 0 || vaddr >= TRACKER_ADDRESS_SIZE || - size > TRACKER_ADDRESS_SIZE - vaddr) { + CacheLock lock(*this, m_lock); + const auto found = m_surface_metas.find(address); + if (found == m_surface_metas.end()) { return false; } - FaultSafeTextureLock lock(this, m_lock); - const auto it = m_surface_metas.find(vaddr); - return it != m_surface_metas.end() && it->second.size == size; -} - -bool TextureCache::HasMetaOverlapLocked(uint64_t vaddr, uint64_t size) const { - for (const auto& [address, meta]: m_surface_metas) { - if (ImagePageRangesOverlap(vaddr, size, address, meta.size)) { - return true; - } - } - return false; -} - -void TextureCache::RequireNoMetaOverlapLocked(uint64_t vaddr, uint64_t size) const { - for (const auto& [address, meta]: m_surface_metas) { - if (!ImagePageRangesOverlap(vaddr, size, address, meta.size)) { - continue; - } - const auto owner = std::find_if(m_images.begin(), m_images.end(), [&](const auto& cached) { - return cached->kind == CachedImage::Kind::DepthTarget && - cached->depth.htile_address == address && cached->depth.htile_size == meta.size; - }); - EXIT("TextureCache: image range overlaps virtual metadata, image=0x%016" PRIx64 - "+0x%016" PRIx64 " metadata=0x%016" PRIx64 "+0x%016" PRIx64 - " layers=%u gpu=%d clear=0x%08x owner=%p owner_depth=0x%016" PRIx64 "+0x%016" PRIx64 - " owner_gpu=%d owner_buffer=%d\n", - vaddr, size, address, meta.size, meta.layers, meta.gpu_modified, meta.clear_mask, - owner != m_images.end() ? static_cast(owner->get()) : nullptr, - owner != m_images.end() ? (*owner)->depth.address : 0, - owner != m_images.end() ? (*owner)->depth.size : 0, - owner != m_images.end() && (*owner)->gpu_modified, - owner != m_images.end() && (*owner)->buffer_modified); - } -} - -void TextureCache::ResolveImageMetadataOverlapsLocked(uint64_t vaddr, uint64_t size) { - std::vector retire; - for (const auto& [address, meta]: m_surface_metas) { - if (!ImagePageRangesOverlap(vaddr, size, address, meta.size)) { - continue; - } - bool found_owner = false; - for (const auto& cached: m_images) { - if (cached->kind != CachedImage::Kind::DepthTarget || - cached->depth.htile_address != address) { - continue; - } - found_owner = true; - bool tracker_gpu_modified = false; - for (uint32_t range = 0; range < cached->RangeCount(); range++) { - tracker_gpu_modified |= m_memory_tracker.IsRegionGpuModified(cached->Address(range), - cached->Size(range)); - } - const bool metadata_tracker_gpu_modified = - m_metadata_tracker.IsRegionGpuModified(address, meta.size); - if (cached->depth.htile_size != meta.size || - !CanRetireGuestCurrentDepthForMetadataReuse( - cached->gpu_modified, cached->buffer_modified, tracker_gpu_modified, - meta.gpu_modified, metadata_tracker_gpu_modified, meta.clear_mask)) { - EXIT("TextureCache: image range overlaps owned metadata, image=0x%016" PRIx64 - "+0x%016" PRIx64 " metadata=0x%016" PRIx64 "+0x%016" PRIx64 - " layers=%u clear=0x%08x gpu=%d/%d/%d buffer=%d\n", - vaddr, size, address, meta.size, meta.layers, meta.clear_mask, - cached->gpu_modified, tracker_gpu_modified, metadata_tracker_gpu_modified, - cached->buffer_modified); - } - if (std::find(retire.begin(), retire.end(), cached.get()) == retire.end()) { - retire.push_back(cached.get()); - } - } - if (!found_owner) { - EXIT("TextureCache: image range overlaps unowned metadata, image=0x%016" PRIx64 - "+0x%016" PRIx64 " metadata=0x%016" PRIx64 "+0x%016" PRIx64 - " layers=%u gpu=%d clear=0x%08x\n", - vaddr, size, address, meta.size, meta.layers, meta.gpu_modified, meta.clear_mask); - } - } - RequireRetirementIsolation(retire, "image metadata", vaddr, size); - RetireDepthMetadataLocked(retire); - RetireImages(retire); - RequireNoMetaOverlapLocked(vaddr, size); -} - -bool TextureCache::IsMetaCleared(uint64_t vaddr, uint32_t slice) { - FaultSafeTextureLock lock(this, m_lock); - const auto it = m_surface_metas.find(vaddr); - if (it == m_surface_metas.end()) { - return false; - } - if (slice >= it->second.layers) { - EXIT("TextureCache: metadata clear slice out of range, addr=0x%016" PRIx64 - " slice=%u layers=%u\n", - vaddr, slice, it->second.layers); - } - if ((it->second.clear_mask & (1u << slice)) == 0) { - return false; - } - const auto slice_size = it->second.size / it->second.layers; - const auto slice_addr = vaddr + slice_size * slice; - if (!it->second.gpu_modified || - !m_metadata_tracker.IsRegionGpuModified(slice_addr, slice_size) || - m_metadata_tracker.IsRegionCpuModified(slice_addr, slice_size)) { - EXIT("TextureCache: cleared metadata slice is not GPU-owned\n"); - } + found->second.clear_mask = UINT32_MAX; return true; } -bool TextureCache::ClearMeta(uint64_t vaddr) { - std::lock_guard transaction(m_resource_mutex); - FaultSafeTextureLock lock(this, m_lock); - const auto it = m_surface_metas.find(vaddr); - if (it == m_surface_metas.end()) { +bool TextureCache::TouchMeta(uint64_t address, uint32_t slice, bool is_clear) { + CacheLock lock(*this, m_lock); + const auto found = m_surface_metas.find(address); + if (found == m_surface_metas.end()) { return false; } - m_metadata_tracker.ForEachUploadRange( - vaddr, it->second.size, true, [](uint64_t, uint64_t) noexcept {}, []() noexcept {}); - it->second.gpu_modified = true; - it->second.clear_mask = it->second.layers == 32 ? UINT32_MAX : (1u << it->second.layers) - 1u; - return true; -} - -bool TextureCache::TouchMeta(uint64_t vaddr, uint32_t slice, bool is_clear) { - FaultSafeTextureLock lock(this, m_lock); - const auto it = m_surface_metas.find(vaddr); - if (it == m_surface_metas.end()) { - return false; - } - if (slice >= it->second.layers) { - EXIT("TextureCache: metadata update slice out of range\n"); - } - const auto slice_size = it->second.size / it->second.layers; - const auto slice_addr = vaddr + slice_size * slice; - if (!it->second.gpu_modified || - !m_metadata_tracker.IsRegionGpuModified(slice_addr, slice_size)) { - EXIT("TextureCache: metadata update requires GPU ownership, addr=0x%016" PRIx64 "\n", - vaddr); - } - if (m_metadata_tracker.IsRegionCpuModified(slice_addr, slice_size)) { - EXIT("TextureCache: metadata update races CPU modification, addr=0x%016" PRIx64 - " size=0x%016" PRIx64 "\n", - slice_addr, slice_size); - } if (is_clear) { - it->second.clear_mask |= 1u << slice; + found->second.clear_mask |= 1u << slice; } else { - it->second.clear_mask &= ~(1u << slice); + found->second.clear_mask &= ~(1u << slice); } return true; } -bool TextureCache::InvalidateMemory(PageFaultAccess access, uint64_t vaddr, uint64_t size, +bool TextureCache::InvalidateMemory(PageFaultAccess access, uint64_t address, uint64_t size, PageFaultPhase phase) noexcept { - if (access != PageFaultAccess::Read && access != PageFaultAccess::Write) { - EXIT("TextureCache: unsupported page-fault access %u, addr=0x%016" PRIx64 - " size=0x%016" PRIx64 " phase=%u\n", - static_cast(access), vaddr, size, static_cast(phase)); + if ((access != PageFaultAccess::Read && access != PageFaultAccess::Write) || + !GuestRange {address, size}.Valid()) { + return false; + } + if (access == PageFaultAccess::Read) { + return false; } if (phase == PageFaultPhase::Invalidate) { - if (g_texture_fault_owner != nullptr) { - EXIT("TextureCache: recursive page-fault invalidation, owner=%p\n", - g_texture_fault_owner); - } - m_fault_mutex.lock(); - g_texture_fault_owner = this; - - const bool metadata = - access == PageFaultAccess::Write && - m_metadata_tracker.InvalidateVirtualGpuWrite(access, vaddr, size, phase); - CpuFaultAction action = CpuFaultAction::Untracked; - bool needs_readback = false; + const bool gpu_image = + m_memory_tracker.InvalidateVirtualGpuWrite(access, address, size, phase); + CpuFaultAction action = gpu_image ? CpuFaultAction::Download + : m_memory_tracker.BeginCpuFault(address, size, access); { - FaultSafeTextureLock lock(this, m_lock); - if (access == PageFaultAccess::Write) { - MarkSampledAliasesCpuDirtyLocked(vaddr, size); - } - needs_readback = FindGpuReadbackPageCandidateLocked(vaddr, size) != nullptr; - if (!needs_readback) { - action = m_memory_tracker.BeginCpuFault(vaddr, size, access); - } + CacheLock lock(*this, m_lock); + InvalidateCpuAliases(address, size); } - - if (needs_readback) { - if (GraphicsRunIsCommandProcessorThread()) { - GraphicsRunFinishScheduler(); - } - m_readback->Request(access, vaddr, size); - action = m_memory_tracker.BeginCpuFault(vaddr, size, access); - if (action != CpuFaultAction::Download) { - EXIT("TextureCache: downloaded image did not retain the active fault page " - "tracking, addr=0x%016" PRIx64 " size=0x%016" PRIx64 " action=%u\n", - vaddr, size, static_cast(action)); - } - } else if (action == CpuFaultAction::Download) { - EXIT("generic region invalidation cannot download GPU-dirty memory\n"); - } - return metadata || action != CpuFaultAction::Untracked; + return action != CpuFaultAction::Untracked; } - if (g_texture_fault_owner != this) { - EXIT("TextureCache: page-fault phase has no matching invalidation, phase=%u owner=%p\n", - static_cast(phase), g_texture_fault_owner); - } if (phase == PageFaultPhase::Complete) { - const bool metadata = - access == PageFaultAccess::Write && - m_metadata_tracker.InvalidateVirtualGpuWrite(access, vaddr, size, phase); - const bool downloaded = m_readback->IsReady(access, vaddr, size); - const bool image = m_memory_tracker.CompleteCpuFault(vaddr, size, access, downloaded); - const bool readback = m_readback->Complete(access, vaddr, size); - if (readback && !image) { - EXIT("TextureCache: image readback completed without a matching tracker fault\n"); - } - return metadata || image || readback; + const bool gpu_image = m_memory_tracker.IsRegionGpuModified(address, size); + return gpu_image ? m_memory_tracker.InvalidateVirtualGpuWrite(access, address, size, phase) + : m_memory_tracker.CompleteCpuFault(address, size, access, false); } if (phase != PageFaultPhase::Release) { - EXIT("TextureCache: unsupported page-fault phase %u\n", static_cast(phase)); + return false; } - if (access == PageFaultAccess::Write) { - (void)m_metadata_tracker.InvalidateVirtualGpuWrite(access, vaddr, size, phase); - } - m_readback->Release(access, vaddr, size); - g_texture_fault_owner = nullptr; - m_fault_mutex.unlock(); + (void)m_memory_tracker.InvalidateVirtualGpuWrite(access, address, size, phase); return true; } -void TextureCache::UnmapMemory(uint64_t vaddr, uint64_t size) { - if (vaddr == 0 || size == 0 || vaddr >= TRACKER_ADDRESS_SIZE || - size > TRACKER_ADDRESS_SIZE - vaddr) { - EXIT("TextureCache: invalid unmap range, addr=0x%016" PRIx64 " size=0x%016" PRIx64 "\n", - vaddr, size); +void TextureCache::UnmapMemory(uint64_t address, uint64_t size) { + if (!GuestRange {address, size}.Valid()) { + EXIT("TextureCache: invalid unmap range\n"); } - std::lock_guard transaction(m_resource_mutex); - FaultSafeTextureLock lock(this, m_lock); - const auto end = vaddr + size; - for (const auto& [address, meta]: m_surface_metas) { - if (ImagePageRangesOverlap(vaddr, size, address, meta.size) && - (vaddr > address || end < address + meta.size)) { - EXIT("TextureCache: partial metadata unmap is unsupported, unmap=0x%016" PRIx64 - "+0x%016" PRIx64 " metadata=0x%016" PRIx64 "+0x%016" PRIx64 "\n", - vaddr, size, address, meta.size); + std::lock_guard transaction(m_resource_mutex); + CacheLock lock(*this, m_lock); + auto images = FindImagesInRegion(address, size, false); + if (!images.empty()) { + m_scheduler.Finish(); + } + for (const auto id: images) { + auto owner = ResolveOwner(id); + if (owner == nullptr) { + continue; + } + if (owner->IsGpuModified()) { + ReleaseGpuTracking(id); + } + DeleteImage(id); + } + m_memory_tracker.UntrackMemory(address, size); + for (const auto id: FindImagesInRegion(address, size, true)) { + if (const auto survivor = ResolveOwner(id); survivor != nullptr) { + RestoreGpuTracking(*survivor); } } - for (const auto& cached: m_images) { - for (uint32_t i = 0; i < cached->RangeCount(); i++) { - if (ImagePageRangesOverlap(vaddr, size, cached->Address(i), cached->Size(i)) && - (vaddr > cached->Address(i) || end < cached->Address(i) + cached->Size(i))) { - EXIT("TextureCache: partial image unmap is unsupported, unmap=0x%016" PRIx64 - "+0x%016" PRIx64 " image=0x%016" PRIx64 "+0x%016" PRIx64 "\n", - vaddr, size, cached->Address(i), cached->Size(i)); +} + +void TextureCache::RunGarbageCollector() { + std::lock_guard transaction(m_resource_mutex); + CacheLock lock(*this, m_lock); + const uint64_t tick = m_gc_tick++; + if (m_graphics.CanReportMemoryUsage()) { + m_total_used_memory = m_graphics.GetDeviceMemoryUsage(); + } + if (m_total_used_memory < m_trigger_gc_memory) { + return; + } + const auto collect = [&](bool allow_aggressive) { + bool pressured = m_total_used_memory >= m_pressure_gc_memory; + bool aggressive = allow_aggressive && m_total_used_memory >= m_critical_gc_memory; + const uint64_t age = std::min(aggressive ? 160 : pressured ? 80 : 16, tick); + size_t deletions = aggressive ? 40 : pressured ? 20 : 10; + std::vector candidates; + candidates.reserve(deletions); + // Deleting depth recursively deletes its stencil association, so finish LRU traversal + // first. + m_lru_cache.ForEachItemBelow(tick - age, [&](ImageId id) { + candidates.push_back(id); + return candidates.size() == deletions; + }); + for (const auto id: candidates) { + if (deletions == 0) { + break; + } + --deletions; + auto owner = ResolveOwner(id); + if (owner == nullptr || !owner->registered || owner->depth_id) { + continue; + } + if (owner->IsGpuModified()) { + const bool safe = SafeToDownload(*owner); + if (safe && owner->info.IsTiled()) { + continue; + } + if (safe && !pressured) { + continue; + } + if (safe && !TryDownloadImage(id)) { + continue; + } + ReleaseGpuTracking(id); + } + DeleteImage(id); + if (m_total_used_memory < m_critical_gc_memory && aggressive) { + deletions >>= 2; + aggressive = false; + } + if (m_total_used_memory < m_pressure_gc_memory && pressured) { + deletions >>= 1; + pressured = false; } } - if (cached->OverlapsRange(vaddr, size, false) && - cached->kind == CachedImage::Kind::VideoOut) { - EXIT("TextureCache: registered video-out surface must be unregistered before unmap, " - "unmap=0x%016" PRIx64 "+0x%016" PRIx64 " image=0x%016" PRIx64 "+0x%016" PRIx64 - " buffer_modified=%d\n", - vaddr, size, cached->Address(), cached->Size(), cached->buffer_modified); - } - } - std::vector retire_depth_metadata; - for (const auto& cached: m_images) { - if (cached->kind != CachedImage::Kind::DepthTarget || - !cached->OverlapsRange(vaddr, size, false) || cached->depth.htile_address == 0) { - continue; - } - const bool retained_owner = - std::any_of(m_images.begin(), m_images.end(), [&](const auto& other) { - return other.get() != cached.get() && - other->kind == CachedImage::Kind::DepthTarget && - !other->OverlapsRange(vaddr, size, false) && - other->depth.htile_address == cached->depth.htile_address && - other->depth.htile_size == cached->depth.htile_size; - }); - if (!retained_owner && - std::find(retire_depth_metadata.begin(), retire_depth_metadata.end(), - cached->depth.htile_address) == retire_depth_metadata.end()) { - retire_depth_metadata.push_back(cached->depth.htile_address); - } - } - for (const auto address: retire_depth_metadata) { - const auto meta = m_surface_metas.find(address); - const auto owner = std::find_if(m_images.begin(), m_images.end(), [&](const auto& cached) { - return cached->kind == CachedImage::Kind::DepthTarget && - cached->OverlapsRange(vaddr, size, false) && - cached->depth.htile_address == address; - }); - if (meta == m_surface_metas.end() || owner == m_images.end() || - meta->second.size != (*owner)->depth.htile_size) { - EXIT("TextureCache: retiring depth image has invalid HTile registration, " - "unmap=0x%016" PRIx64 "+0x%016" PRIx64 " metadata=0x%016" PRIx64 - " registered=0x%016" PRIx64 " expected=0x%016" PRIx64 "\n", - vaddr, size, address, meta != m_surface_metas.end() ? meta->second.size : 0, - owner != m_images.end() ? (*owner)->depth.htile_size : 0); - } - } - std::vector metadata_ranges; - metadata_ranges.reserve(m_surface_metas.size()); - for (const auto& [address, meta]: m_surface_metas) { - const bool retiring = ImageRangeOverlaps(vaddr, size, address, meta.size) || - std::find(retire_depth_metadata.begin(), retire_depth_metadata.end(), - address) != retire_depth_metadata.end(); - metadata_ranges.push_back({address, meta.size, retiring}); - } - const auto metadata_conflict = FindImageRetirementConflict(metadata_ranges); - if (metadata_conflict.Exists()) { - const auto& retired = metadata_ranges[metadata_conflict.retired]; - const auto& retained = metadata_ranges[metadata_conflict.retained]; - EXIT("TextureCache: depth-image retirement leaves a tracked metadata alias, " - "unmap=0x%016" PRIx64 "+0x%016" PRIx64 " retired=0x%016" PRIx64 "+0x%016" PRIx64 - " retained=0x%016" PRIx64 "+0x%016" PRIx64 "\n", - vaddr, size, retired.address, retired.size, retained.address, retained.size); - } - bool wait_idle = false; - for (auto& cached: m_images) { - if (cached->OverlapsRange(vaddr, size, false)) { - wait_idle = true; - } - } - if (wait_idle) { - Transfer::WaitForQueueIdle(); - } - for (auto& cached: m_images) { - if (!cached->OverlapsRange(vaddr, size, false) || !cached->gpu_modified) { - continue; - } - for (uint32_t i = 0; i < cached->RangeCount(); i++) { - m_memory_tracker.UnmarkRegionAsGpuModified(cached->Address(i), cached->Size(i)); - } - cached->gpu_modified = false; - } - for (auto it = m_surface_metas.begin(); it != m_surface_metas.end();) { - const bool allocation_unmapped = - ImageRangeOverlaps(vaddr, size, it->first, it->second.size); - const bool depth_owner_retired = - std::find(retire_depth_metadata.begin(), retire_depth_metadata.end(), it->first) != - retire_depth_metadata.end(); - if (!allocation_unmapped && !depth_owner_retired) { - ++it; - continue; - } - if (depth_owner_retired) { - LOGF("TextureCache: retiring HTile metadata with depth image, " - "unmap=0x%016" PRIx64 "+0x%016" PRIx64 " metadata=0x%016" PRIx64 "+0x%016" PRIx64 - "\n", - vaddr, size, it->first, it->second.size); - } - if (it->second.gpu_modified) { - m_metadata_tracker.ForEachDownloadRange(it->first, it->second.size, - [](uint64_t, uint64_t) noexcept {}); - } - m_metadata_tracker.UntrackMemory(it->first, it->second.size); - it = m_surface_metas.erase(it); - } - m_memory_tracker.UntrackMemory(vaddr, size); - for (auto it = m_images.begin(); it != m_images.end();) { - if (!(**it).OverlapsRange(vaddr, size, false)) { - ++it; - continue; - } - UnregisterImageLocked(**it, false); - it = m_images.erase(it); + }; + collect(false); + if (m_total_used_memory >= m_critical_gc_memory) { + collect(true); } } -VulkanImage& TextureCache::GetDummySampledTexture(bool uint_format, bool image_3d) { - KYTY_PROFILER_BLOCK("TextureCache::GetDummySampledTexture"); - return m_dummy_textures->Get(DummyTextureCache::Usage::Sampled, uint_format, image_3d); -} - -VulkanImage& TextureCache::GetDummyStorageTexture(bool uint_format, bool image_3d) { - KYTY_PROFILER_BLOCK("TextureCache::GetDummyStorageTexture"); - return m_dummy_textures->Get(DummyTextureCache::Usage::Storage, uint_format, image_3d); +void TextureCache::ProcessDownloadImages() { + std::lock_guard transaction(m_resource_mutex); + CacheLock lock(*this, m_lock); + for (const auto id: m_download_images) { + const auto owner = ResolveOwner(id); + if (owner != nullptr && owner->registered && owner->IsGpuModified()) { + (void)TryDownloadImage(id); + } + } + m_download_images.clear(); } } // namespace Libs::Graphics diff --git a/src/graphics/host_gpu/renderer/textureCache.h b/src/graphics/host_gpu/renderer/textureCache.h index 53ce4414..6dc8c3bc 100644 --- a/src/graphics/host_gpu/renderer/textureCache.h +++ b/src/graphics/host_gpu/renderer/textureCache.h @@ -3,174 +3,184 @@ #include "common/abi.h" #include "common/common.h" -#include "common/threads.h" +#include "common/lruCache.h" #include "graphics/host_gpu/memoryTracker.h" -#include "graphics/host_gpu/renderer/imageInfo.h" +#include "graphics/host_gpu/renderer/blitHelper.h" +#include "graphics/host_gpu/renderer/image.h" #include "graphics/host_gpu/renderer/multiLevelPageTable.h" -#include "graphics/host_gpu/renderer/tiler.h" -#include "graphics/host_gpu/vulkanCommon.h" +#include #include #include -#include +#include +#include +#include +#include #include namespace Libs::Graphics { -struct DepthStencilVulkanImage; -struct GpuTextureVulkanImage; struct GraphicContext; -struct ImageViewInfo; -struct RenderTextureVulkanImage; -struct StorageTextureVulkanImage; -struct VideoOutVulkanImage; -struct VulkanImage; -struct VulkanMemory; +class Buffer; class BufferCache; class CommandBuffer; -class DummyTextureCache; +class CommandScheduler; class ResourceMutex; - -[[nodiscard]] bool IsExactRenderTargetMipStorage(const ImageInfo& sampled, const ImageInfo& storage, - vk::Format sampled_view_format, - vk::Format storage_image_format) noexcept; +class RenderExecutor; +class StreamBuffer; +class TileManager; +struct TextureCacheTestAccess; class TextureCache { public: - struct RegionInfo { - bool image_pages = false; - bool image_bytes = false; - bool gpu_image_bytes = false; - bool non_sampled_pages = false; - bool metadata_pages = false; - bool metadata_bytes = false; - bool gpu_metadata_bytes = false; - }; - struct MetaRangeInfo { - uint64_t metadata_address = 0; - uint64_t metadata_size = 0; - uint32_t slice = 0; - bool full = false; + enum class BindingType : uint8_t { Texture, Storage, RenderTarget, DepthTarget, VideoOut }; + + struct ImageDesc { + ImageInfo info; + ImageViewInfo view_info; + BindingType type = BindingType::Texture; }; - TextureCache(GraphicContext& graphics, PageManager& page_manager, BufferCache& buffer_cache, - ResourceMutex& resource_mutex); + struct RegionInfo { + bool image_pages = false; + bool image_bytes = false; + bool gpu_image_bytes = false; + }; + + TextureCache(GraphicContext& graphics, CommandScheduler& scheduler, PageManager& page_manager, + BufferCache& buffer_cache, ResourceMutex& resource_mutex); ~TextureCache(); KYTY_CLASS_NO_COPY(TextureCache); - [[nodiscard]] VulkanImage& FindTexture(CommandBuffer& command, const ImageInfo& info, - bool metadata_read); - [[nodiscard]] StorageTextureVulkanImage& FindStorageTexture(CommandBuffer& command, - const ImageInfo& info); - [[nodiscard]] RenderTextureVulkanImage& FindRenderTarget(CommandBuffer& command, - const RenderTargetInfo& info); - [[nodiscard]] DepthStencilVulkanImage& FindDepthTarget(CommandBuffer& command, - const DepthTargetInfo& info); - [[nodiscard]] std::vector - RegisterVideoOutSurfaces(const std::vector& infos); - void RefreshVideoOut(VideoOutVulkanImage& image, bool render_target = false); - void UnregisterVideoOutSurfaces(const std::vector& images); - [[nodiscard]] bool ClearImageFromBuffer(CommandBuffer& command, uint64_t vaddr, uint64_t size, - uint32_t packed_clear); - void MarkGpuWritten(VulkanImage& image); - void PrepareHostWrite(uint64_t vaddr, uint64_t size); - [[nodiscard]] bool InvalidateMemoryFromGPU(uint64_t vaddr, uint64_t size, - bool formatted_buffer_write = false); - [[nodiscard]] RenderTextureVulkanImage* FindRenderTargetByRange(CommandBuffer& command, - uint64_t vaddr, uint64_t size); - [[nodiscard]] vk::ImageView GetRenderTargetAttachmentView(RenderTextureVulkanImage& image, - vk::Format format, uint32_t level, - uint32_t base_layer, - uint32_t layer_count); - [[nodiscard]] vk::ImageView GetDepthTargetAttachmentView(DepthStencilVulkanImage& image, - uint32_t base_layer, - uint32_t layer_count); - [[nodiscard]] vk::ImageView - GetDepthTargetSampledView(DepthStencilVulkanImage& image, vk::Format view_format, - uint32_t swizzle, uint32_t base_level, uint32_t level_count, - vk::ImageViewType type, uint32_t base_layer, uint32_t layer_count); - [[nodiscard]] vk::ImageView GetSampledColorView(VulkanImage& image, vk::Format view_format, - uint32_t swizzle, uint32_t base_level, - uint32_t level_count, vk::ImageViewType type, - uint32_t base_layer, uint32_t layer_count); - [[nodiscard]] vk::ImageView - GetRenderTargetStorageView(RenderTextureVulkanImage& image, vk::Format view_format, - uint32_t base_level, uint32_t level_count, vk::ImageViewType type, - uint32_t base_layer, uint32_t layer_count); - [[nodiscard]] vk::ImageView GetStorageTextureSampledView(StorageTextureVulkanImage& image, - const ImageInfo& info); - [[nodiscard]] vk::ImageView GetStorageTextureStorageView(StorageTextureVulkanImage& image, - uint32_t base_level); - [[nodiscard]] DepthStencilVulkanImage* - FindDepthTargetByRange(CommandBuffer& command, uint64_t vaddr, uint64_t size, - bool allow_containing_sampled = false); - [[nodiscard]] RegionInfo QueryRegion(uint64_t vaddr, uint64_t size); - [[nodiscard]] bool ResolveMetaRange(uint64_t vaddr, uint64_t size, MetaRangeInfo& info); - void RegisterMeta(uint64_t vaddr, uint64_t size, uint32_t layers = 1); - [[nodiscard]] bool IsMeta(uint64_t vaddr); - [[nodiscard]] bool IsMetaRange(uint64_t vaddr, uint64_t size); - [[nodiscard]] bool IsMetaCleared(uint64_t vaddr, uint32_t slice); - [[nodiscard]] bool ClearMeta(uint64_t vaddr); - [[nodiscard]] bool TouchMeta(uint64_t vaddr, uint32_t slice, bool is_clear); - [[nodiscard]] bool InvalidateMemory(PageFaultAccess access, uint64_t vaddr, uint64_t size, - PageFaultPhase phase) noexcept; - void UnmapMemory(uint64_t vaddr, uint64_t size); + [[nodiscard]] ImageId FindImage(ImageDesc& desc, bool exact_format = false); + [[nodiscard]] ImageId FindImageFromRange(uint64_t address, uint64_t size, + bool ensure_valid = true); + [[nodiscard]] vk::ImageView FindTexture(ImageId id, const ImageDesc& desc); + [[nodiscard]] vk::ImageView FindRenderTarget(ImageId id, const ImageDesc& desc); + [[nodiscard]] vk::ImageView FindDepthTarget(ImageId id, const ImageDesc& desc); + [[nodiscard]] Image& GetImage(ImageId id); + [[nodiscard]] const Image& GetImage(ImageId id) const; + void MarkGpuWritten(ImageId id); - VulkanImage& GetDummySampledTexture(bool uint_format, bool image_3d); - VulkanImage& GetDummyStorageTexture(bool uint_format, bool image_3d); - [[nodiscard]] std::shared_ptr GetImageOwner(const VulkanImage& image); + [[nodiscard]] bool ClearImageFromBuffer(CommandBuffer& command, uint64_t address, uint64_t size, + uint32_t packed_clear); + void PrepareHostWrite(uint64_t address, uint64_t size); + [[nodiscard]] bool SynchronizeImageToBuffer(uint64_t address, uint64_t size); + [[nodiscard]] bool InvalidateMemoryFromGPU(uint64_t address, uint64_t size, + bool formatted_buffer_write = false); + [[nodiscard]] RegionInfo QueryRegion(uint64_t address, uint64_t size); + + [[nodiscard]] bool IsMeta(uint64_t address); + [[nodiscard]] bool IsMetaCleared(uint64_t address, uint32_t slice); + [[nodiscard]] bool ClearMeta(uint64_t address); + [[nodiscard]] bool TouchMeta(uint64_t address, uint32_t slice, bool is_clear); + + [[nodiscard]] bool InvalidateMemory(PageFaultAccess access, uint64_t address, uint64_t size, + PageFaultPhase phase) noexcept; + void UnmapMemory(uint64_t address, uint64_t size); + void ProcessDownloadImages(); + void RunGarbageCollector(); private: - struct CachedImage; - using ImageOwnerIndex = MultiRangePageOwnerIndex; - struct ReadbackWorker; - struct MetaDataInfo { - uint64_t size = 0; - uint32_t layers = 1; - uint32_t clear_mask = 0; - bool gpu_modified = false; - }; - [[nodiscard]] vk::ImageView GetImageView(VulkanImage& image, const ImageViewInfo& info); - [[nodiscard]] bool HasMetaOverlapLocked(uint64_t vaddr, uint64_t size) const; - [[nodiscard]] CachedImage* FindGpuReadbackPageCandidateLocked(uint64_t vaddr, uint64_t size); - void RequireNoMetaOverlapLocked(uint64_t vaddr, uint64_t size) const; - void ResolveImageMetadataOverlapsLocked(uint64_t vaddr, uint64_t size); - void MarkSampledAliasesCpuDirtyLocked(uint64_t vaddr, uint64_t size); - void RetireSampledTargetAliases(const ImageInfo& requested); - void ResolveStorageImageOverlaps(const ImageInfo& requested); - void RetireStorageDepthAliasLocked(const ImageInfo& requested); - void RegisterImageLocked(CachedImage& image); - void UnregisterImageLocked(CachedImage& image, bool release_tracking); - [[nodiscard]] VulkanImage& PublishImage(CommandBuffer& command, - std::shared_ptr image); - [[nodiscard]] std::vector FindImagesInRegionLocked(uint64_t vaddr, uint64_t size, - bool page_overlap); - void RequireRetirementIsolation(const std::vector& retire, const char* operation, - uint64_t address, uint64_t size) const; - void RetireImages(const std::vector& retire, - const CachedImage* native_image_source = nullptr); - void RetireDepthMetadataLocked(const std::vector& retire, - uint64_t preserve_address = 0); - void MaterializeImagesToGuestLocked(const std::vector>& images); - void SynchronizeColorImageToBufferLocked(CachedImage& cached, uint64_t write_address, - uint64_t write_size); - void SynchronizeDepthImageToBufferLocked(CachedImage& cached, uint64_t write_address, - uint64_t write_size); + enum class TransferDirection { Upload, Download }; + struct ColorTransferPlan; + struct DownloadPlan; - GraphicContext& m_graphics; - std::unique_ptr m_dummy_textures; - TrackingSpinLock m_lock; - std::mutex m_fault_mutex; - MemoryTracker m_memory_tracker; - MemoryTracker m_metadata_tracker; - Tiler m_tiler; - BufferCache& m_buffer_cache; - ResourceMutex& m_resource_mutex; - std::vector> m_images; - ImageOwnerIndex m_image_owner_index; - std::map m_surface_metas; - std::unique_ptr m_readback; - std::vector m_buffer_transition_guest; + struct Slot { + std::shared_ptr image; + uint32_t generation = 1; + }; + + struct MetaDataInfo { + uint32_t clear_mask = 0; + }; + + struct OverlapResult { + ImageId image; + int32_t mip = -1; + int32_t layer = -1; + }; + + using ImageOwnerIndex = MultiRangePageOwnerIndex; + + [[nodiscard]] Image& ResolveImage(ImageId id); + [[nodiscard]] const Image& ResolveImage(ImageId id) const; + [[nodiscard]] std::shared_ptr ResolveOwner(ImageId id) const; + [[nodiscard]] ImageId InsertImage(const ImageInfo& info); + [[nodiscard]] ImageId GetNullImage(const ImageDesc& desc); + void RegisterImage(ImageId id); + void UnregisterImage(ImageId id, bool release_tracking); + void DeleteImage(ImageId id, bool release_tracking = true); + void DeleteImages(std::span ids, std::optional native_source = {}); + void RetainImage(CommandBuffer& command, ImageId id); + void TouchImage(Image& image); + void TrackImageDownload(ImageId id); + void TrackImageDownloadLocked(ImageId id, Image& image); + [[nodiscard]] static bool SameBacking(const ImageInfo& cached, const ImageInfo& requested, + bool exact_format); + [[nodiscard]] static BindingType UploadBinding(const Image& image); + [[nodiscard]] bool SafeToDownload(const Image& image); + + [[nodiscard]] std::vector FindImagesInRegion(uint64_t address, uint64_t size, + bool page_overlap) const; + [[nodiscard]] OverlapResult ResolveOverlap(const ImageInfo& requested, BindingType binding, + ImageId cached, ImageId merged); + [[nodiscard]] ImageId ResolveDepthOverlap(const ImageInfo& requested, BindingType binding, + ImageId cached); + [[nodiscard]] ImageId ExpandImage(const ImageInfo& info, ImageId source); + void RefreshImage(ImageId id, const ImageDesc& desc); + void InitializeImage(ImageId id, const ImageDesc& desc); + [[nodiscard]] ColorTransferPlan BuildColorTransfer(const Image& image, BindingType binding, + TransferDirection direction) const; + [[nodiscard]] DownloadPlan BuildDownload(const Image& image) const; + void UploadImage(Image& image, const ImageDesc& desc, Buffer& source, uint64_t source_offset); + void DownloadImageData(Image& image, Buffer& destination, uint64_t destination_offset, + DownloadPlan plan); + void DownloadDepth(Image& image, Buffer& destination, uint64_t destination_offset); + void CommitGpuWrite(Image& image); + void PrepareImageCopy(Image& image); + void RefreshCopySource(ImageId id); + [[nodiscard]] bool CopyD16(Image& destination, Image& source); + void CopyImage(ImageId destination, ImageId source); + void AssociateStencil(ImageId depth, GuestRange stencil); + void AssociateStencilLocked(ImageId depth, GuestRange stencil); + void CopyImageMip(ImageId destination, ImageId source, uint32_t mip, uint32_t layer); + void ValidateImageDesc(const ImageDesc& desc) const; + + void InvalidateCpuAliases(uint64_t address, uint64_t size); + void RestoreGpuTracking(const Image& image); + void ReleaseGpuTracking(ImageId id); + + [[nodiscard]] bool SynchronizeImageToBuffer(ImageId id); + void DownloadImage(ImageId id); + [[nodiscard]] bool TryDownloadImage(ImageId id); + [[nodiscard]] std::pair MapDownload(uint64_t size, uint64_t alignment); + void QueueDownload(GuestRange range, StreamBuffer& download, uint8_t* mapped, uint64_t offset); + + GraphicContext& m_graphics; + CommandScheduler& m_scheduler; + TrackingSpinLock m_lock; + MemoryTracker m_memory_tracker; + BlitHelper m_blit_helper; + std::unique_ptr m_tiler; + BufferCache& m_buffer_cache; + ResourceMutex& m_resource_mutex; + std::vector m_slots; + std::vector m_free_slots; + ImageOwnerIndex m_image_owner_index; + std::map m_null_images; + Common::LeastRecentlyUsedCache m_lru_cache; + std::set m_download_images; + std::map m_surface_metas; + uint64_t m_total_used_memory = 0; + uint64_t m_trigger_gc_memory = 0; + uint64_t m_pressure_gc_memory = 1536ull * 1024 * 1024; + uint64_t m_critical_gc_memory = 3ull * 1024 * 1024 * 1024; + uint64_t m_gc_tick = 0; + bool m_readback_linear_images = false; + + friend struct TextureCacheTestAccess; + friend class RenderExecutor; }; } // namespace Libs::Graphics diff --git a/src/graphics/host_gpu/renderer/tiler.cpp b/src/graphics/host_gpu/renderer/tiler.cpp index d01a79b9..dacaa5d0 100644 --- a/src/graphics/host_gpu/renderer/tiler.cpp +++ b/src/graphics/host_gpu/renderer/tiler.cpp @@ -1,161 +1,718 @@ #include "graphics/host_gpu/renderer/tiler.h" #include "common/assert.h" -#include "graphics/guest_gpu/gpu_defs.h" -#include "graphics/guest_gpu/gpu_format.h" -#include "graphics/guest_gpu/tile.h" -#include "graphics/host_gpu/gpuTiler.h" +#include "gpu_tiler_shaders/gpu_tiler_demote_d16_spv.h" +#include "gpu_tiler_shaders/gpu_tiler_depth_spv.h" +#include "gpu_tiler_shaders/gpu_tiler_promote_d16_spv.h" +#include "gpu_tiler_shaders/gpu_tiler_prt_3d_spv.h" +#include "gpu_tiler_shaders/gpu_tiler_prt_spv.h" +#include "gpu_tiler_shaders/gpu_tiler_render_target_spv.h" +#include "gpu_tiler_shaders/gpu_tiler_standard256_spv.h" +#include "gpu_tiler_shaders/gpu_tiler_standard4_3d_spv.h" +#include "gpu_tiler_shaders/gpu_tiler_standard4_spv.h" +#include "gpu_tiler_shaders/gpu_tiler_standard64_3d_spv.h" +#include "gpu_tiler_shaders/gpu_tiler_standard64_spv.h" +#include "gpu_tiler_shaders/gpu_tiler_swap_bgra16_spv.h" #include "graphics/host_gpu/graphicContext.h" -#include "graphics/host_gpu/objects/textureCommon.h" +#include "graphics/host_gpu/renderer/commandScheduler.h" #include "graphics/host_gpu/renderer/image.h" -#include "graphics/host_gpu/transfer.h" +#include "graphics/host_gpu/renderer/streamBuffer.h" -#include +#include +#include +#include +#include +#include namespace Libs::Graphics { -namespace { -struct DepthTransfer { - std::vector infos; - std::vector regions; -}; - -DepthTransfer MakeDepthTransfer(uint64_t size, uint32_t layers, uint32_t format, - uint32_t bytes_per_element, uint32_t width, uint32_t height, - uint32_t pitch, uint32_t base_layer, vk::ImageAspectFlags aspect) { - EXIT_IF(size == 0 || layers == 0 || size % layers != 0); - TileBlockLayout block {}; - EXIT_NOT_IMPLEMENTED( - !TileGetBlockLayout(TileBlockFamily::Depth64KB, bytes_per_element, block) || - Prospero::NumBytesPerElement(format) != bytes_per_element); - - const uint64_t slice_size = size / layers; - DepthTransfer transfer; - transfer.infos.reserve(layers); - transfer.regions.reserve(layers); - for (uint32_t layer = 0; layer < layers; layer++) { - const uint64_t offset = slice_size * layer; - GpuTileInfo info {block.family, - block.bytes_per_element, - offset, - slice_size, - offset, - slice_size, - 0, - width, - height, - 1, - pitch}; - info.surface_z = base_layer + layer; - transfer.infos.push_back(info); - - BufferImageCopy region {}; - region.offset = static_cast(offset); - region.pitch = pitch; - region.width = width; - region.height = height; - region.dst_layer = base_layer + layer; - region.aspect = aspect; - transfer.regions.push_back(region); +TileManager::TileManager(GraphicContext& graphics, CommandScheduler& scheduler, + StreamBuffer& stream_buffer) + : m_graphics(graphics), m_scheduler(scheduler), m_stream_buffer(stream_buffer) { + static_assert(FamilyCount == 9); + static_assert(sizeof(Push) == 52); + std::array bindings {}; + for (uint32_t index = 0; index < 2; index++) { + bindings[index] = {index, vk::DescriptorType::eStorageBuffer, 1, + vk::ShaderStageFlagBits::eCompute, nullptr}; } - return transfer; + bindings[2] = {2, vk::DescriptorType::eUniformBuffer, 1, vk::ShaderStageFlagBits::eCompute, + nullptr}; + + vk::DescriptorSetLayoutCreateInfo descriptor_info {}; + descriptor_info.sType = vk::StructureType::eDescriptorSetLayoutCreateInfo; + descriptor_info.flags = vk::DescriptorSetLayoutCreateFlagBits::ePushDescriptorKHR; + descriptor_info.bindingCount = static_cast(bindings.size()); + descriptor_info.pBindings = bindings.data(); + RequireVulkanSuccess(m_graphics.device.createDescriptorSetLayout(&descriptor_info, nullptr, + &m_descriptor_layout), + "create TileManager descriptor layout"); + + const vk::PushConstantRange push_range {vk::ShaderStageFlagBits::eCompute, 0, sizeof(Push)}; + vk::PipelineLayoutCreateInfo layout_info {}; + layout_info.sType = vk::StructureType::ePipelineLayoutCreateInfo; + layout_info.setLayoutCount = 1; + layout_info.pSetLayouts = &m_descriptor_layout; + layout_info.pushConstantRangeCount = 1; + layout_info.pPushConstantRanges = &push_range; + RequireVulkanSuccess( + m_graphics.device.createPipelineLayout(&layout_info, nullptr, &m_pipeline_layout), + "create TileManager pipeline layout"); } -void UploadDepth(DepthStencilVulkanImage& image, uint64_t source_address, uint64_t size, - uint32_t layers, uint32_t format, uint32_t bytes_per_element, uint32_t width, - uint32_t height, uint32_t pitch, uint32_t base_layer, - vk::ImageAspectFlags aspect) { - auto transfer = MakeDepthTransfer(size, layers, format, bytes_per_element, width, height, pitch, - base_layer, aspect); - Transfer::UploadTiledImage(image, reinterpret_cast(source_address), size, size, - transfer.infos, transfer.regions, - vk::ImageLayout::eDepthStencilAttachmentOptimal); -} - -template -void UploadPromotedD16Depth(DepthStencilVulkanImage& image, const DepthTargetInfo& info, - const BufferImageCopySource& source, uint32_t base_layer) { - const uint64_t guest_slice_size = info.size / info.layers; - const uint64_t texels = static_cast(info.pitch) * info.height; - const uint64_t host_slice_size = texels * sizeof(uint32_t); - const uint64_t host_upload_size = host_slice_size * info.layers; - EXIT_IF(host_upload_size > UINT32_MAX); - - auto transfer = MakeDepthTransfer(info.size, info.layers, info.guest_format, - info.bytes_per_element, info.width, info.height, info.pitch, - base_layer, vk::ImageAspectFlagBits::eDepth); - std::vector guest_linear(info.size / sizeof(uint16_t)); - GpuDetile(reinterpret_cast(source.address), guest_linear.data(), info.size, - info.size, transfer.infos); - - Transfer::ScratchBuffer host_linear(host_upload_size); - for (uint32_t layer = 0; layer < info.layers; layer++) { - const auto* guest = guest_linear.data() + guest_slice_size / sizeof(uint16_t) * layer; - auto* host = reinterpret_cast(static_cast(host_linear.Data()) + - host_slice_size * layer); - for (uint64_t texel = 0; texel < texels; texel++) { - host[texel] = Encode(guest[texel]); - } - transfer.regions[layer].offset = static_cast(host_slice_size * layer); - } - Transfer::UploadImage(image, host_linear.Data(), host_upload_size, transfer.regions, - vk::ImageLayout::eDepthStencilAttachmentOptimal); -} - -} // namespace - -void Tiler::DetileImage(GpuTextureVulkanImage& image, const ImageInfo& info, - const BufferImageCopySource& source, bool refresh, bool storage) const { - if (refresh) Transfer::WaitForQueueIdle(); - - const bool array_texture = TextureIsLayeredTexture(info.type); - const bool volume_texture = TextureIs3DTexture(info.type); - auto layout = TextureCalcUploadLayout( - info.format, info.width, info.height, info.levels, info.depth, info.pitch, info.tile, - info.size, true, volume_texture, storage ? "StorageTextureCache" : "TextureCache"); - auto regions = TextureBuildUploadRegions(layout, image.format, info.width, info.height, - info.depth, info.levels, array_texture, volume_texture, - TextureUploadDestination::MipLevels); - TextureUploadGuestImage(image, reinterpret_cast(source.address), info.size, - regions, layout, info.format, info.width, info.height, info.depth, - info.levels, storage ? "StorageTextureCache" : "TextureCache", - storage ? vk::ImageLayout::eGeneral - : vk::ImageLayout::eShaderReadOnlyOptimal); -} - -void Tiler::DetileImage(DepthStencilVulkanImage& image, const DepthTargetInfo& info, - const BufferImageCopySource& source, bool refresh, - uint32_t base_layer) const { - EXIT_NOT_IMPLEMENTED(info.samples != 1 || image.samples != 1); - if (refresh) Transfer::WaitForQueueIdle(); - - if (DepthAspectTransferBytes(info.format) != info.bytes_per_element) { - switch (info.format) { - case vk::Format::eD24UnormS8Uint: - UploadPromotedD16Depth(image, info, source, base_layer); - return; - case vk::Format::eD32SfloatS8Uint: - UploadPromotedD16Depth(image, info, source, base_layer); - return; - default: EXIT_NOT_IMPLEMENTED(true); +TileManager::~TileManager() { + for (auto pipeline: m_pipelines) { + if (pipeline != nullptr) { + m_graphics.device.destroyPipeline(pipeline, nullptr); } } - UploadDepth(image, source.address, info.size, info.layers, info.guest_format, - info.bytes_per_element, info.width, info.height, info.pitch, base_layer, - vk::ImageAspectFlagBits::eDepth); + if (m_d16_to_d24 != nullptr) { + m_graphics.device.destroyPipeline(m_d16_to_d24, nullptr); + } + if (m_d16_to_d32 != nullptr) { + m_graphics.device.destroyPipeline(m_d16_to_d32, nullptr); + } + if (m_d24_to_d16 != nullptr) { + m_graphics.device.destroyPipeline(m_d24_to_d16, nullptr); + } + if (m_d32_to_d16 != nullptr) { + m_graphics.device.destroyPipeline(m_d32_to_d16, nullptr); + } + if (m_swap_bgra16 != nullptr) { + m_graphics.device.destroyPipeline(m_swap_bgra16, nullptr); + } + if (m_pipeline_layout != nullptr) { + m_graphics.device.destroyPipelineLayout(m_pipeline_layout, nullptr); + } + if (m_descriptor_layout != nullptr) { + m_graphics.device.destroyDescriptorSetLayout(m_descriptor_layout, nullptr); + } } -void Tiler::DetileStencil(DepthStencilVulkanImage& image, const DepthTargetInfo& info, - const BufferImageCopySource& source, bool refresh, - uint32_t base_layer) const { - EXIT_NOT_IMPLEMENTED(info.samples != 1 || image.samples != 1); - if (refresh) Transfer::WaitForQueueIdle(); +TileManager::Scratch TileManager::AllocateScratch(uint64_t size) { + EXIT_IF(size == 0); + vk::BufferCreateInfo create {}; + create.sType = vk::StructureType::eBufferCreateInfo; + create.size = size; + create.usage = vk::BufferUsageFlagBits::eStorageBuffer | vk::BufferUsageFlagBits::eTransferSrc | + vk::BufferUsageFlagBits::eTransferDst; + create.sharingMode = vk::SharingMode::eExclusive; - const auto format = Prospero::GpuEnumValue(Prospero::BufferFormat::k8UInt); - const auto pitch = TileGetTexturePitch(format, info.width, 1, - Prospero::GpuEnumValue(Prospero::TileMode::kDepth)); - UploadDepth(image, source.address, info.stencil_size, info.layers, format, 1, info.width, - info.height, pitch, base_layer, vk::ImageAspectFlagBits::eStencil); + VmaAllocationCreateInfo allocate {}; + allocate.usage = VMA_MEMORY_USAGE_AUTO_PREFER_DEVICE; + VkBuffer buffer = VK_NULL_HANDLE; + VmaAllocation memory = nullptr; + const auto raw = static_cast(create); + RequireVulkanSuccess(static_cast(vmaCreateBuffer( + m_graphics.allocator, &raw, &allocate, &buffer, &memory, nullptr)), + "allocate TileManager scratch buffer"); + return {buffer, memory, size}; +} + +void TileManager::DeferDestroy(Scratch scratch) { + auto allocator = m_graphics.allocator; + m_scheduler.DeferOperation( + [allocator, scratch] { vmaDestroyBuffer(allocator, scratch.buffer, scratch.allocation); }); +} + +void TileManager::Prepare(bool tile, uint64_t tiled_capacity, uint64_t linear_capacity, + std::span infos, uint64_t source_base, + uint64_t target_base, std::vector& dispatches) { + EXIT_IF(infos.empty() || tiled_capacity == 0 || linear_capacity == 0); + const auto& limits = m_graphics.GetPhysicalDeviceProperties().limits; + EXIT_NOT_IMPLEMENTED(tiled_capacity > UINT32_MAX || linear_capacity > UINT32_MAX); + + const auto checked_multiply = [](uint64_t left, uint64_t right, uint64_t& result) { + return (left == 0 || right <= UINT64_MAX / left) && (result = left * right, true); + }; + const auto checked_add = [](uint64_t left, uint64_t right, uint64_t& result) { + return right <= UINT64_MAX - left && (result = left + right, true); + }; + const auto valid_range = [](uint64_t offset, uint64_t size, uint64_t capacity) { + return size != 0 && offset <= capacity && size <= capacity - offset; + }; + + dispatches.clear(); + dispatches.reserve(infos.size()); + for (const auto& info: infos) { + TileBlockLayout block {}; + const uint32_t tiled_width = info.tiled_width != 0 ? info.tiled_width : info.pitch; + const uint32_t tiled_height = info.tiled_height != 0 ? info.tiled_height : info.height; + const uint64_t groups_x = (static_cast(info.width) + 7u) / 8u; + const uint64_t groups_y = (static_cast(info.height) + 7u) / 8u; + EXIT_NOT_IMPLEMENTED( + !TileGetBlockLayout(info.family, info.bytes_per_element, block) || info.width == 0 || + info.height == 0 || info.depth == 0 || info.pitch < info.width || + groups_x > limits.maxComputeWorkGroupCount[0] || + groups_y > limits.maxComputeWorkGroupCount[1] || + info.depth > limits.maxComputeWorkGroupCount[2] || + (!info.tail && (tiled_width < info.width || tiled_height < info.height)) || + !valid_range(info.linear_offset, info.linear_size, linear_capacity) || + !valid_range(info.tiled_offset, info.tiled_size, tiled_capacity) || + (block.block_depth == 1 && info.depth != 1)); + + uint64_t pitch_bytes = 0; + EXIT_NOT_IMPLEMENTED(!checked_multiply(info.pitch, info.bytes_per_element, pitch_bytes) || + pitch_bytes > UINT32_MAX); + uint64_t slice_bytes = info.linear_slice_stride; + uint64_t minimum_slice = 0; + EXIT_NOT_IMPLEMENTED(!checked_multiply(pitch_bytes, info.height, minimum_slice)); + if (slice_bytes == 0) { + slice_bytes = minimum_slice; + } + uint64_t linear_used = 0; + uint64_t bytes = 0; + EXIT_NOT_IMPLEMENTED((info.depth > 1 && slice_bytes < minimum_slice) || + !checked_multiply(info.depth - 1u, slice_bytes, bytes) || + !checked_add(linear_used, bytes, linear_used) || + !checked_multiply(info.height - 1u, pitch_bytes, bytes) || + !checked_add(linear_used, bytes, linear_used) || + !checked_multiply(info.width, info.bytes_per_element, bytes) || + !checked_add(linear_used, bytes, linear_used) || + linear_used > info.linear_size || slice_bytes > UINT32_MAX); + + const uint64_t columns = + (static_cast(tiled_width) + block.block_width - 1u) / block.block_width; + const uint64_t rows = + (static_cast(tiled_height) + block.block_height - 1u) / block.block_height; + uint64_t blocks_per_slice = 0; + EXIT_NOT_IMPLEMENTED(!checked_multiply(columns, rows, blocks_per_slice) || + columns > UINT32_MAX || blocks_per_slice > UINT32_MAX); + if (info.tail) { + EXIT_NOT_IMPLEMENTED( + info.family == TileBlockFamily::Standard256B || info.depth > block.block_depth || + info.tail_x >= block.block_width || info.width > block.block_width - info.tail_x || + info.tail_y >= block.block_height || + info.height > block.block_height - info.tail_y || + info.tiled_size < block.block_size); + } else { + const uint64_t slices = + (static_cast(info.depth) + block.block_depth - 1u) / block.block_depth; + uint64_t tiled_used = 0; + EXIT_NOT_IMPLEMENTED(!checked_multiply(blocks_per_slice, slices, tiled_used) || + !checked_multiply(tiled_used, block.block_size, tiled_used) || + tiled_used > info.tiled_size); + } + + const uint32_t alignment = std::min(info.bytes_per_element, 4u); + EXIT_NOT_IMPLEMENTED(((info.linear_offset | info.tiled_offset | pitch_bytes | slice_bytes) & + (alignment - 1u)) != 0); + const uint64_t src = source_base + (tile ? info.linear_offset : info.tiled_offset); + const uint64_t dst = target_base + (tile ? info.tiled_offset : info.linear_offset); + EXIT_NOT_IMPLEMENTED(src > UINT32_MAX || dst > UINT32_MAX); + + const uint32_t family_index = static_cast(info.family); + const uint32_t element_index = std::countr_zero(info.bytes_per_element); + EXIT_NOT_IMPLEMENTED(family_index >= FamilyCount || element_index >= BytesPerElementCount); + Dispatch dispatch {}; + dispatch.pipeline_slot = + ((tile ? FamilyCount : 0u) + family_index) * BytesPerElementCount + element_index; + dispatch.push.src_base = static_cast(src); + dispatch.push.dst_base = static_cast(dst); + dispatch.push.width = info.width; + dispatch.push.height = info.height; + dispatch.push.depth = info.depth; + dispatch.push.surface_z = info.surface_z; + dispatch.push.pitch_bytes = static_cast(pitch_bytes); + dispatch.push.slice_bytes = static_cast(slice_bytes); + dispatch.push.blocks_per_row = static_cast(columns); + dispatch.push.blocks_per_slice = static_cast(blocks_per_slice); + dispatch.push.tail_x = info.tail_x; + dispatch.push.tail_y = info.tail_y; + dispatch.push.tail = info.tail; + dispatches.push_back(dispatch); + } + + const uint64_t uniform_alignment = + std::max(limits.minUniformBufferOffsetAlignment, 1); + const uint64_t stride = (sizeof(Push) + uniform_alignment - 1) & ~(uniform_alignment - 1); + EXIT_NOT_IMPLEMENTED(dispatches.size() > UINT64_MAX / stride); + const uint64_t bytes = dispatches.size() * stride; + auto [mapped, offset] = m_stream_buffer.Map(bytes, uniform_alignment); + EXIT_IF(mapped == nullptr); + for (size_t index = 0; index < dispatches.size(); index++) { + std::memcpy(mapped + index * stride, &dispatches[index].push, sizeof(Push)); + dispatches[index].params_offset = offset + index * stride; + } + m_stream_buffer.Commit(); +} + +vk::Pipeline TileManager::GetPipeline(uint32_t slot) { + EXIT_IF(slot >= m_pipelines.size()); + if (m_pipelines[slot] != nullptr) { + return m_pipelines[slot]; + } + struct Shader { + const uint32_t* code; + size_t words; + }; + static constexpr std::array shaders {{ + {GPU_TILER_STANDARD256_SPV, std::size(GPU_TILER_STANDARD256_SPV)}, + {GPU_TILER_STANDARD4_SPV, std::size(GPU_TILER_STANDARD4_SPV)}, + {GPU_TILER_STANDARD4_3D_SPV, std::size(GPU_TILER_STANDARD4_3D_SPV)}, + {GPU_TILER_STANDARD64_SPV, std::size(GPU_TILER_STANDARD64_SPV)}, + {GPU_TILER_STANDARD64_3D_SPV, std::size(GPU_TILER_STANDARD64_3D_SPV)}, + {GPU_TILER_PRT_SPV, std::size(GPU_TILER_PRT_SPV)}, + {GPU_TILER_PRT_3D_SPV, std::size(GPU_TILER_PRT_3D_SPV)}, + {GPU_TILER_RENDER_TARGET_SPV, std::size(GPU_TILER_RENDER_TARGET_SPV)}, + {GPU_TILER_DEPTH_SPV, std::size(GPU_TILER_DEPTH_SPV)}, + }}; + const uint32_t element_index = slot % BytesPerElementCount; + const uint32_t direction_index = slot / (FamilyCount * BytesPerElementCount); + const uint32_t family_index = (slot / BytesPerElementCount) % FamilyCount; + const uint32_t values[] {1u << element_index, direction_index}; + const vk::SpecializationMapEntry entries[] {{0, 0, 4}, {1, 4, 4}}; + const vk::SpecializationInfo specialization {2, entries, sizeof(values), values}; + vk::ShaderModuleCreateInfo module_info {}; + module_info.sType = vk::StructureType::eShaderModuleCreateInfo; + module_info.codeSize = shaders[family_index].words * sizeof(uint32_t); + module_info.pCode = shaders[family_index].code; + vk::ShaderModule module = nullptr; + RequireVulkanSuccess(m_graphics.device.createShaderModule(&module_info, nullptr, &module), + "create TileManager shader module"); + vk::PipelineShaderStageCreateInfo stage {}; + stage.sType = vk::StructureType::ePipelineShaderStageCreateInfo; + stage.stage = vk::ShaderStageFlagBits::eCompute; + stage.module = module; + stage.pName = "main"; + stage.pSpecializationInfo = &specialization; + vk::ComputePipelineCreateInfo create {}; + create.sType = vk::StructureType::eComputePipelineCreateInfo; + create.stage = stage; + create.layout = m_pipeline_layout; + const auto result = + m_graphics.device.createComputePipelines(nullptr, 1, &create, nullptr, &m_pipelines[slot]); + m_graphics.device.destroyShaderModule(module, nullptr); + RequireVulkanSuccess(result, "create TileManager pipeline"); + return m_pipelines[slot]; +} + +void TileManager::Record(bool tile, vk::Buffer source, uint64_t source_offset, + uint64_t source_capacity, vk::Buffer target, uint64_t target_offset, + uint64_t target_capacity, std::span dispatches, + bool clear_target) { + const auto& limits = m_graphics.GetPhysicalDeviceProperties().limits; + const uint64_t descriptor_alignment = + std::max(limits.minStorageBufferOffsetAlignment, 4); + const uint64_t source_descriptor_offset = source_offset & ~(descriptor_alignment - 1); + const uint64_t target_descriptor_offset = target_offset & ~(descriptor_alignment - 1); + const uint64_t source_base = source_offset - source_descriptor_offset; + const uint64_t target_base = target_offset - target_descriptor_offset; + const uint64_t source_range = (source_base + source_capacity + 3u) & ~uint64_t {3}; + const uint64_t target_range = (target_base + target_capacity + 3u) & ~uint64_t {3}; + EXIT_NOT_IMPLEMENTED(source_range > limits.maxStorageBufferRange || + target_range > limits.maxStorageBufferRange || target_offset % 4 != 0 || + target_capacity % 4 != 0); + + m_scheduler.EndRendering(); + auto command = m_scheduler.Current().Handle(); + vk::BufferMemoryBarrier barriers[3] {}; + barriers[0].sType = vk::StructureType::eBufferMemoryBarrier; + barriers[0].srcAccessMask = vk::AccessFlagBits::eMemoryWrite | vk::AccessFlagBits::eHostWrite; + barriers[0].dstAccessMask = vk::AccessFlagBits::eShaderRead; + barriers[0].srcQueueFamilyIndex = VK_QUEUE_FAMILY_IGNORED; + barriers[0].dstQueueFamilyIndex = VK_QUEUE_FAMILY_IGNORED; + barriers[0].buffer = source; + barriers[0].offset = source_offset; + barriers[0].size = source_capacity; + barriers[1] = barriers[0]; + barriers[1].srcAccessMask = vk::AccessFlagBits::eMemoryWrite | vk::AccessFlagBits::eHostWrite; + barriers[1].dstAccessMask = + clear_target ? vk::AccessFlagBits::eTransferWrite + : vk::AccessFlagBits::eShaderRead | vk::AccessFlagBits::eShaderWrite; + barriers[1].buffer = target; + barriers[1].offset = target_offset; + barriers[1].size = target_capacity; + barriers[2] = barriers[0]; + barriers[2].srcAccessMask = vk::AccessFlagBits::eHostWrite; + barriers[2].dstAccessMask = vk::AccessFlagBits::eUniformRead; + barriers[2].buffer = m_stream_buffer.Handle(); + barriers[2].offset = dispatches.front().params_offset; + barriers[2].size = + dispatches.back().params_offset - dispatches.front().params_offset + sizeof(Push); + command.pipelineBarrier( + vk::PipelineStageFlagBits::eAllCommands | vk::PipelineStageFlagBits::eHost, + vk::PipelineStageFlagBits::eComputeShader | vk::PipelineStageFlagBits::eTransfer, {}, 0, + nullptr, 3, barriers, 0, nullptr); + if (clear_target) { + command.fillBuffer(target, target_offset, target_capacity, 0); + barriers[1].srcAccessMask = vk::AccessFlagBits::eTransferWrite; + barriers[1].dstAccessMask = + vk::AccessFlagBits::eShaderRead | vk::AccessFlagBits::eShaderWrite; + command.pipelineBarrier(vk::PipelineStageFlagBits::eTransfer, + vk::PipelineStageFlagBits::eComputeShader, {}, 0, nullptr, 1, + &barriers[1], 0, nullptr); + } + + const vk::DescriptorBufferInfo source_info {source, source_descriptor_offset, source_range}; + const vk::DescriptorBufferInfo target_info {target, target_descriptor_offset, target_range}; + for (auto& dispatch: dispatches) { + const vk::DescriptorBufferInfo params_info {m_stream_buffer.Handle(), + dispatch.params_offset, sizeof(Push)}; + const vk::DescriptorBufferInfo infos[] {source_info, target_info, params_info}; + std::array writes {}; + for (uint32_t index = 0; index < writes.size(); index++) { + writes[index].sType = vk::StructureType::eWriteDescriptorSet; + writes[index].dstBinding = index; + writes[index].descriptorCount = 1; + writes[index].descriptorType = index == 2 ? vk::DescriptorType::eUniformBuffer + : vk::DescriptorType::eStorageBuffer; + writes[index].pBufferInfo = &infos[index]; + } + command.pushDescriptorSetKHR(vk::PipelineBindPoint::eCompute, m_pipeline_layout, 0, + static_cast(writes.size()), writes.data()); + command.bindPipeline(vk::PipelineBindPoint::eCompute, GetPipeline(dispatch.pipeline_slot)); + command.dispatch((dispatch.push.width + 7u) / 8u, (dispatch.push.height + 7u) / 8u, + dispatch.push.depth); + } + + barriers[1].srcAccessMask = vk::AccessFlagBits::eShaderWrite; + barriers[1].dstAccessMask = vk::AccessFlagBits::eTransferRead | vk::AccessFlagBits::eMemoryRead; + command.pipelineBarrier(vk::PipelineStageFlagBits::eComputeShader, + vk::PipelineStageFlagBits::eAllCommands, {}, 0, nullptr, 1, + &barriers[1], 0, nullptr); +} + +TileManager::Result TileManager::Detile(vk::Buffer tiled, uint64_t tiled_offset, + uint64_t tiled_capacity, uint64_t linear_capacity, + std::span infos) { + const auto& limits = m_graphics.GetPhysicalDeviceProperties().limits; + const uint64_t descriptor_alignment = + std::max(limits.minStorageBufferOffsetAlignment, 4); + const uint64_t source_base = tiled_offset & (descriptor_alignment - 1); + std::vector dispatches; + Prepare(false, tiled_capacity, linear_capacity, infos, source_base, 0, dispatches); + auto scratch = AllocateScratch((linear_capacity + 3u) & ~uint64_t {3}); + DeferDestroy(scratch); + Record(false, tiled, tiled_offset, tiled_capacity, scratch.buffer, 0, scratch.size, dispatches, + true); + return {scratch.buffer, 0, linear_capacity}; +} + +void TileManager::Tile(vk::Buffer linear, uint64_t linear_offset, uint64_t linear_capacity, + vk::Buffer tiled, uint64_t tiled_offset, uint64_t tiled_capacity, + std::span infos) { + const auto& limits = m_graphics.GetPhysicalDeviceProperties().limits; + const uint64_t descriptor_alignment = + std::max(limits.minStorageBufferOffsetAlignment, 4); + const uint64_t source_base = linear_offset & (descriptor_alignment - 1); + const uint64_t target_base = tiled_offset & (descriptor_alignment - 1); + std::vector dispatches; + Prepare(true, tiled_capacity, linear_capacity, infos, source_base, target_base, dispatches); + Record(true, linear, linear_offset, linear_capacity, tiled, tiled_offset, tiled_capacity, + dispatches, false); +} + +void TileManager::TileImage(Image& image, std::span regions, + vk::Buffer tiled, uint64_t tiled_offset, uint64_t tiled_capacity, + uint64_t linear_capacity, std::span infos, + ColorTransform transform) { + EXIT_IF(regions.empty()); + const auto& limits = m_graphics.GetPhysicalDeviceProperties().limits; + const uint64_t descriptor_alignment = + std::max(limits.minStorageBufferOffsetAlignment, 4); + const uint64_t target_base = tiled_offset & (descriptor_alignment - 1); + std::vector dispatches; + // Reserve all stream parameters before creating a scheduler-lived scratch dependency: + // StreamBuffer::Map is allowed to submit the current tick when it wraps. + Prepare(true, tiled_capacity, linear_capacity, infos, 0, target_base, dispatches); + auto linear = AllocateScratch((linear_capacity + 3u) & ~uint64_t {3}); + DeferDestroy(linear); + image.Download(regions, linear.buffer, 0, linear.size); + Result source {linear.buffer, 0, linear.size}; + if (transform == ColorTransform::SwapBgra16) { + source = SwapBgra16(source); + } + Record(true, source.buffer, source.offset, linear_capacity, tiled, tiled_offset, tiled_capacity, + dispatches, false); +} + +TileManager::Result TileManager::GetScratchBuffer(uint64_t size) { + auto scratch = AllocateScratch((size + 3u) & ~uint64_t {3}); + DeferDestroy(scratch); + return {scratch.buffer, 0, scratch.size}; +} + +TileManager::StorageBinding TileManager::BindStorage(Result buffer, uint64_t size) const { + const auto& limits = m_graphics.GetPhysicalDeviceProperties().limits; + const auto alignment = std::max(limits.minStorageBufferOffsetAlignment, 4); + const auto descriptor_offset = buffer.offset - buffer.offset % alignment; + const auto base = buffer.offset - descriptor_offset; + EXIT_IF(buffer.buffer == nullptr || size == 0 || buffer.size < size || base > UINT32_MAX || + size > UINT64_MAX - base || base + size > UINT64_MAX - 3); + const auto range = (base + size + 3) & ~uint64_t {3}; + EXIT_IF(range > limits.maxStorageBufferRange || range > UINT32_MAX); + return {{buffer.buffer, descriptor_offset, range}, static_cast(base)}; +} + +uint32_t TileManager::ConversionRows(uint64_t offset, uint64_t row_stride, uint64_t active, + uint32_t remaining, uint64_t alignment, uint64_t max_range, + uint32_t max_groups) noexcept { + if (row_stride == 0 || active == 0 || remaining == 0 || alignment == 0 || max_groups == 0) { + return 0; + } + const auto prefix = offset % alignment; + if (prefix >= max_range || active > max_range - prefix) { + return 0; + } + const auto descriptor_rows = 1 + (max_range - prefix - active) / row_stride; + return static_cast(std::min({remaining, descriptor_rows, max_groups})); +} + +void TileManager::ConvertD16(Result source, Result target, D16Direction direction, bool d32, + const D16Layout& layout) { + vk::Pipeline* pipeline_pointer = nullptr; + if (direction == D16Direction::Promote) { + pipeline_pointer = d32 ? &m_d16_to_d32 : &m_d16_to_d24; + } else { + pipeline_pointer = d32 ? &m_d32_to_d16 : &m_d24_to_d16; + } + auto& pipeline = *pipeline_pointer; + if (pipeline == nullptr) { + const uint32_t value = d32 ? 1u : 0u; + const vk::SpecializationMapEntry entry {0, 0, sizeof(value)}; + const vk::SpecializationInfo specialization {1, &entry, sizeof(value), &value}; + const uint32_t* code = nullptr; + size_t words = 0; + if (direction == D16Direction::Promote) { + code = GPU_TILER_PROMOTE_D16_SPV; + words = std::size(GPU_TILER_PROMOTE_D16_SPV); + } else { + code = GPU_TILER_DEMOTE_D16_SPV; + words = std::size(GPU_TILER_DEMOTE_D16_SPV); + } + vk::ShaderModuleCreateInfo module_info {}; + module_info.sType = vk::StructureType::eShaderModuleCreateInfo; + module_info.codeSize = words * sizeof(uint32_t); + module_info.pCode = code; + vk::ShaderModule module = nullptr; + RequireVulkanSuccess(m_graphics.device.createShaderModule(&module_info, nullptr, &module), + "create D16 conversion shader module"); + vk::PipelineShaderStageCreateInfo stage {}; + stage.sType = vk::StructureType::ePipelineShaderStageCreateInfo; + stage.stage = vk::ShaderStageFlagBits::eCompute; + stage.module = module; + stage.pName = "main"; + stage.pSpecializationInfo = &specialization; + vk::ComputePipelineCreateInfo create {}; + create.sType = vk::StructureType::eComputePipelineCreateInfo; + create.stage = stage; + create.layout = m_pipeline_layout; + const auto result = + m_graphics.device.createComputePipelines(nullptr, 1, &create, nullptr, &pipeline); + m_graphics.device.destroyShaderModule(module, nullptr); + RequireVulkanSuccess(result, "create D16 conversion pipeline"); + } + + const uint64_t source_element = + direction == D16Direction::Promote ? sizeof(uint16_t) : sizeof(uint32_t); + const uint64_t target_element = + direction == D16Direction::Promote ? sizeof(uint32_t) : sizeof(uint16_t); + const uint64_t source_active = static_cast(layout.width) * source_element; + const uint64_t target_active = static_cast(layout.width) * target_element; + const auto required = [](uint32_t height, uint32_t layers, uint64_t row_stride, + uint64_t slice_stride, uint64_t active) { + EXIT_IF(height == 0 || layers == 0 || row_stride < active || + (height - 1) > (UINT64_MAX - active) / row_stride); + const auto slice = static_cast(height - 1) * row_stride + active; + EXIT_IF(slice_stride < slice || (layers - 1) > (UINT64_MAX - slice) / slice_stride); + return static_cast(layers - 1) * slice_stride + slice; + }; + EXIT_IF(layout.width == 0 || layout.source_row_stride > UINT32_MAX || + layout.target_row_stride > UINT32_MAX); + const auto source_required = required(layout.height, layout.layers, layout.source_row_stride, + layout.source_slice_stride, source_active); + const auto target_required = required(layout.height, layout.layers, layout.target_row_stride, + layout.target_slice_stride, target_active); + EXIT_IF(source_required > UINT64_MAX - 3 || target_required > UINT64_MAX - 3); + const auto source_barrier_size = (source_required + 3) & ~uint64_t {3}; + const auto target_barrier_size = (target_required + 3) & ~uint64_t {3}; + EXIT_IF(source.size < source_barrier_size || target.size < target_barrier_size); + + m_scheduler.EndRendering(); + auto command = m_scheduler.Current().Handle(); + vk::BufferMemoryBarrier barriers[2] {}; + barriers[0].sType = vk::StructureType::eBufferMemoryBarrier; + barriers[0].srcQueueFamilyIndex = VK_QUEUE_FAMILY_IGNORED; + barriers[0].dstQueueFamilyIndex = VK_QUEUE_FAMILY_IGNORED; + barriers[0].buffer = source.buffer; + barriers[0].offset = source.offset; + barriers[0].size = source_barrier_size; + barriers[0].srcAccessMask = vk::AccessFlagBits::eMemoryWrite | vk::AccessFlagBits::eHostWrite | + vk::AccessFlagBits::eTransferWrite | + vk::AccessFlagBits::eShaderWrite; + barriers[0].dstAccessMask = vk::AccessFlagBits::eShaderRead; + barriers[1].sType = vk::StructureType::eBufferMemoryBarrier; + barriers[1].srcQueueFamilyIndex = VK_QUEUE_FAMILY_IGNORED; + barriers[1].dstQueueFamilyIndex = VK_QUEUE_FAMILY_IGNORED; + barriers[1].buffer = target.buffer; + barriers[1].offset = target.offset; + barriers[1].size = target_barrier_size; + barriers[1].srcAccessMask = vk::AccessFlagBits::eMemoryRead | vk::AccessFlagBits::eMemoryWrite | + vk::AccessFlagBits::eHostWrite | + vk::AccessFlagBits::eTransferWrite | + vk::AccessFlagBits::eShaderWrite; + barriers[1].dstAccessMask = vk::AccessFlagBits::eShaderRead | vk::AccessFlagBits::eShaderWrite; + command.pipelineBarrier( + vk::PipelineStageFlagBits::eAllCommands | vk::PipelineStageFlagBits::eHost, + vk::PipelineStageFlagBits::eComputeShader, {}, 0, nullptr, 2, barriers, 0, nullptr); + command.bindPipeline(vk::PipelineBindPoint::eCompute, pipeline); + const auto& limits = m_graphics.GetPhysicalDeviceProperties().limits; + const auto descriptor_alignment = std::max(limits.minStorageBufferOffsetAlignment, 4); + const auto rows_for = [&](Result buffer, uint64_t relative, uint64_t stride, uint64_t active, + uint32_t remaining) { + EXIT_IF(relative > buffer.size || buffer.offset > UINT64_MAX - relative); + const auto offset = buffer.offset + relative; + return ConversionRows(offset, stride, active, remaining, descriptor_alignment, + limits.maxStorageBufferRange, limits.maxComputeWorkGroupCount[1]); + }; + const auto groups_x = (static_cast(layout.width) + 63u) / 64u; + EXIT_IF(groups_x == 0 || groups_x > limits.maxComputeWorkGroupCount[0]); + for (uint32_t layer = 0; layer < layout.layers; layer++) { + for (uint32_t row = 0; row < layout.height;) { + const auto source_relative = + layout.source_slice_stride * layer + layout.source_row_stride * row; + const auto target_relative = + layout.target_slice_stride * layer + layout.target_row_stride * row; + const auto remaining = layout.height - row; + const auto rows = std::min(rows_for(source, source_relative, layout.source_row_stride, + source_active, remaining), + rows_for(target, target_relative, layout.target_row_stride, + target_active, remaining)); + EXIT_IF(rows == 0); + const auto source_span = + static_cast(rows - 1) * layout.source_row_stride + source_active; + const auto target_span = + static_cast(rows - 1) * layout.target_row_stride + target_active; + const auto source_binding = BindStorage( + {source.buffer, source.offset + source_relative, source.size - source_relative}, + source_span); + const auto target_binding = BindStorage( + {target.buffer, target.offset + target_relative, target.size - target_relative}, + target_span); + const vk::DescriptorBufferInfo infos[] { + source_binding.info, + target_binding.info, + }; + std::array writes {}; + for (uint32_t index = 0; index < writes.size(); index++) { + writes[index].sType = vk::StructureType::eWriteDescriptorSet; + writes[index].dstBinding = index; + writes[index].descriptorCount = 1; + writes[index].descriptorType = vk::DescriptorType::eStorageBuffer; + writes[index].pBufferInfo = &infos[index]; + } + command.pushDescriptorSetKHR(vk::PipelineBindPoint::eCompute, m_pipeline_layout, 0, + static_cast(writes.size()), writes.data()); + Push push {}; + push.src_base = source_binding.base; + push.dst_base = target_binding.base; + push.width = layout.width; + push.height = rows; + push.pitch_bytes = static_cast(layout.source_row_stride); + push.slice_bytes = static_cast(layout.target_row_stride); + command.pushConstants(m_pipeline_layout, vk::ShaderStageFlagBits::eCompute, 0, + sizeof(push), &push); + command.dispatch(static_cast(groups_x), rows, 1); + row += rows; + } + } + barriers[1].srcAccessMask = vk::AccessFlagBits::eShaderWrite; + barriers[1].dstAccessMask = vk::AccessFlagBits::eTransferRead | vk::AccessFlagBits::eMemoryRead; + command.pipelineBarrier(vk::PipelineStageFlagBits::eComputeShader, + vk::PipelineStageFlagBits::eAllCommands, {}, 0, nullptr, 1, + &barriers[1], 0, nullptr); +} + +void TileManager::SwapBgra16(Result input, Result output, uint32_t pixels) { + if (m_swap_bgra16 == nullptr) { + vk::ShaderModuleCreateInfo module_info {}; + module_info.sType = vk::StructureType::eShaderModuleCreateInfo; + module_info.codeSize = std::size(GPU_TILER_SWAP_BGRA16_SPV) * sizeof(uint32_t); + module_info.pCode = GPU_TILER_SWAP_BGRA16_SPV; + vk::ShaderModule module = nullptr; + RequireVulkanSuccess(m_graphics.device.createShaderModule(&module_info, nullptr, &module), + "create BGRA16 swap shader module"); + vk::PipelineShaderStageCreateInfo stage {}; + stage.sType = vk::StructureType::ePipelineShaderStageCreateInfo; + stage.stage = vk::ShaderStageFlagBits::eCompute; + stage.module = module; + stage.pName = "main"; + vk::ComputePipelineCreateInfo create {}; + create.sType = vk::StructureType::eComputePipelineCreateInfo; + create.stage = stage; + create.layout = m_pipeline_layout; + const auto result = + m_graphics.device.createComputePipelines(nullptr, 1, &create, nullptr, &m_swap_bgra16); + m_graphics.device.destroyShaderModule(module, nullptr); + RequireVulkanSuccess(result, "create BGRA16 swap pipeline"); + } + const uint64_t bytes = static_cast(pixels) * 8u; + EXIT_IF(pixels == 0); + const auto input_binding = BindStorage(input, bytes); + const auto output_binding = BindStorage(output, bytes); + + const vk::DescriptorBufferInfo infos[] { + input_binding.info, + output_binding.info, + }; + std::array writes {}; + for (uint32_t index = 0; index < writes.size(); index++) { + writes[index].sType = vk::StructureType::eWriteDescriptorSet; + writes[index].dstBinding = index; + writes[index].descriptorCount = 1; + writes[index].descriptorType = vk::DescriptorType::eStorageBuffer; + writes[index].pBufferInfo = &infos[index]; + } + vk::BufferMemoryBarrier barriers[2] {}; + for (uint32_t index = 0; index < 2; index++) { + barriers[index].sType = vk::StructureType::eBufferMemoryBarrier; + barriers[index].srcQueueFamilyIndex = VK_QUEUE_FAMILY_IGNORED; + barriers[index].dstQueueFamilyIndex = VK_QUEUE_FAMILY_IGNORED; + barriers[index].buffer = infos[index].buffer; + barriers[index].offset = infos[index].offset; + barriers[index].size = infos[index].range; + } + barriers[0].srcAccessMask = vk::AccessFlagBits::eMemoryWrite | vk::AccessFlagBits::eHostWrite | + vk::AccessFlagBits::eShaderWrite; + barriers[0].dstAccessMask = vk::AccessFlagBits::eShaderRead; + barriers[1].srcAccessMask = vk::AccessFlagBits::eMemoryRead; + barriers[1].dstAccessMask = vk::AccessFlagBits::eShaderWrite; + m_scheduler.EndRendering(); + auto command = m_scheduler.Current().Handle(); + command.pipelineBarrier( + vk::PipelineStageFlagBits::eAllCommands | vk::PipelineStageFlagBits::eHost, + vk::PipelineStageFlagBits::eComputeShader, {}, 0, nullptr, 2, barriers, 0, nullptr); + command.bindPipeline(vk::PipelineBindPoint::eCompute, m_swap_bgra16); + command.pushDescriptorSetKHR(vk::PipelineBindPoint::eCompute, m_pipeline_layout, 0, + static_cast(writes.size()), writes.data()); + Push push {}; + push.src_base = input_binding.base; + push.dst_base = output_binding.base; + push.width = pixels; + command.pushConstants(m_pipeline_layout, vk::ShaderStageFlagBits::eCompute, 0, sizeof(push), + &push); + command.dispatch((pixels + 63u) / 64u, 1, 1); + barriers[1].srcAccessMask = vk::AccessFlagBits::eShaderWrite; + barriers[1].dstAccessMask = vk::AccessFlagBits::eTransferRead; + command.pipelineBarrier(vk::PipelineStageFlagBits::eComputeShader, + vk::PipelineStageFlagBits::eTransfer, {}, 0, nullptr, 1, &barriers[1], + 0, nullptr); +} + +TileManager::Result TileManager::SwapBgra16(Result input) { + EXIT_NOT_IMPLEMENTED(input.size == 0 || input.size % 8u != 0 || input.size / 8u > UINT32_MAX); + auto output = AllocateScratch(input.size); + DeferDestroy(output); + Result result {output.buffer, 0, output.size}; + SwapBgra16(input, result, static_cast(input.size / 8u)); + return result; +} + +void TileManager::SwapBgra16(Result input, Result output) { + EXIT_NOT_IMPLEMENTED(input.size == 0 || input.size % 8u != 0 || input.size / 8u > UINT32_MAX || + output.size < input.size); + SwapBgra16(input, output, static_cast(input.size / 8u)); } } // namespace Libs::Graphics diff --git a/src/graphics/host_gpu/renderer/tiler.h b/src/graphics/host_gpu/renderer/tiler.h index a45d5670..ec8f9ede 100644 --- a/src/graphics/host_gpu/renderer/tiler.h +++ b/src/graphics/host_gpu/renderer/tiler.h @@ -2,26 +2,146 @@ #define EMULATOR_SRC_GRAPHICS_HOST_GPU_RENDERER_TILER_H_ #include "common/common.h" -#include "graphics/host_gpu/renderer/bufferCache.h" -#include "graphics/host_gpu/renderer/imageInfo.h" +#include "graphics/guest_gpu/tile.h" +#include "graphics/host_gpu/vulkanCommon.h" + +#include +#include +#include +#include namespace Libs::Graphics { -struct DepthStencilVulkanImage; -struct GpuTextureVulkanImage; -class Tiler final { -public: - Tiler() = default; - KYTY_CLASS_NO_COPY(Tiler); +class CommandScheduler; +class Image; +class StreamBuffer; +struct GraphicContext; +struct TileManagerTestAccess; - void DetileImage(GpuTextureVulkanImage& image, const ImageInfo& info, - const BufferImageCopySource& source, bool refresh, bool storage) const; - void DetileImage(DepthStencilVulkanImage& image, const DepthTargetInfo& info, - const BufferImageCopySource& source, bool refresh, - uint32_t base_layer = 0) const; - void DetileStencil(DepthStencilVulkanImage& image, const DepthTargetInfo& info, - const BufferImageCopySource& source, bool refresh, - uint32_t base_layer = 0) const; +struct GpuTileInfo { + TileBlockFamily family = TileBlockFamily::Count; + uint32_t bytes_per_element = 0; + uint64_t linear_offset = 0; + uint64_t linear_size = 0; + uint64_t tiled_offset = 0; + uint64_t tiled_size = 0; + uint64_t linear_slice_stride = 0; + uint32_t width = 0; + uint32_t height = 0; + uint32_t depth = 1; + uint32_t pitch = 0; + uint32_t tail_x = 0; + uint32_t tail_y = 0; + bool tail = false; + uint32_t tiled_width = 0; + uint32_t tiled_height = 0; + uint32_t surface_z = 0; +}; + +class TileManager final { +public: + enum class D16Direction { Promote, Demote }; + enum class ColorTransform { None, SwapBgra16 }; + + struct Result { + vk::Buffer buffer = nullptr; + uint64_t offset = 0; + uint64_t size = 0; + }; + struct D16Layout { + uint32_t width = 0; + uint32_t height = 0; + uint32_t layers = 0; + uint64_t source_row_stride = 0; + uint64_t target_row_stride = 0; + uint64_t source_slice_stride = 0; + uint64_t target_slice_stride = 0; + }; + + TileManager(GraphicContext& graphics, CommandScheduler& scheduler, StreamBuffer& stream_buffer); + ~TileManager(); + KYTY_CLASS_NO_COPY(TileManager); + + // The returned device-local buffer remains alive through the current scheduler tick. + [[nodiscard]] Result Detile(vk::Buffer tiled, uint64_t tiled_offset, uint64_t tiled_capacity, + uint64_t linear_capacity, std::span infos); + void Tile(vk::Buffer linear, uint64_t linear_offset, uint64_t linear_capacity, vk::Buffer tiled, + uint64_t tiled_offset, uint64_t tiled_capacity, std::span infos); + void TileImage(Image& image, std::span regions, vk::Buffer tiled, + uint64_t tiled_offset, uint64_t tiled_capacity, uint64_t linear_capacity, + std::span infos, + ColorTransform transform = ColorTransform::None); + [[nodiscard]] Result GetScratchBuffer(uint64_t size); + void ConvertD16(Result source, Result target, D16Direction direction, bool d32, + const D16Layout& layout); + [[nodiscard]] Result SwapBgra16(Result input); + void SwapBgra16(Result input, Result output); + +private: + friend struct TileManagerTestAccess; + + static constexpr uint32_t FamilyCount = static_cast(TileBlockFamily::Count); + static constexpr uint32_t BytesPerElementCount = 5; + static constexpr uint32_t DirectionCount = 2; + static constexpr uint32_t PipelineCount = FamilyCount * BytesPerElementCount * DirectionCount; + + struct Push { + uint32_t src_base; + uint32_t dst_base; + uint32_t width; + uint32_t height; + uint32_t depth; + uint32_t surface_z; + uint32_t pitch_bytes; + uint32_t slice_bytes; + uint32_t blocks_per_row; + uint32_t blocks_per_slice; + uint32_t tail_x; + uint32_t tail_y; + uint32_t tail; + }; + struct Dispatch { + Push push {}; + uint32_t pipeline_slot = 0; + uint64_t params_offset = 0; + }; + struct Scratch { + vk::Buffer buffer = nullptr; + VmaAllocation allocation = nullptr; + uint64_t size = 0; + }; + struct StorageBinding { + vk::DescriptorBufferInfo info; + uint32_t base = 0; + }; + + [[nodiscard]] Scratch AllocateScratch(uint64_t size); + [[nodiscard]] StorageBinding BindStorage(Result buffer, uint64_t size) const; + [[nodiscard]] static uint32_t ConversionRows(uint64_t offset, uint64_t row_stride, + uint64_t active, uint32_t remaining, + uint64_t alignment, uint64_t max_range, + uint32_t max_groups) noexcept; + void DeferDestroy(Scratch scratch); + void Prepare(bool tile, uint64_t tiled_capacity, uint64_t linear_capacity, + std::span infos, uint64_t source_base, uint64_t target_base, + std::vector& dispatches); + void Record(bool tile, vk::Buffer source, uint64_t source_offset, uint64_t source_capacity, + vk::Buffer target, uint64_t target_offset, uint64_t target_capacity, + std::span dispatches, bool clear_target); + [[nodiscard]] vk::Pipeline GetPipeline(uint32_t slot); + void SwapBgra16(Result input, Result output, uint32_t pixels); + + GraphicContext& m_graphics; + CommandScheduler& m_scheduler; + StreamBuffer& m_stream_buffer; + vk::DescriptorSetLayout m_descriptor_layout = nullptr; + vk::PipelineLayout m_pipeline_layout = nullptr; + std::array m_pipelines {}; + vk::Pipeline m_d16_to_d24 = nullptr; + vk::Pipeline m_d16_to_d32 = nullptr; + vk::Pipeline m_d24_to_d16 = nullptr; + vk::Pipeline m_d32_to_d16 = nullptr; + vk::Pipeline m_swap_bgra16 = nullptr; }; } // namespace Libs::Graphics diff --git a/src/graphics/host_gpu/shaders/gpu_blit_color_to_ms_depth.frag b/src/graphics/host_gpu/shaders/gpu_blit_color_to_ms_depth.frag new file mode 100644 index 00000000..e9fcdcaa --- /dev/null +++ b/src/graphics/host_gpu/shaders/gpu_blit_color_to_ms_depth.frag @@ -0,0 +1,10 @@ +#version 450 core +#extension GL_EXT_samplerless_texture_functions : require + +layout(binding = 0, set = 0) uniform texture2D color; +layout(location = 0) in vec2 uv; + +void main() { + ivec2 coord = ivec2(uv * vec2(textureSize(color, 0))); + gl_FragDepth = texelFetch(color, coord, 0)[gl_SampleID]; +} diff --git a/src/graphics/host_gpu/shaders/gpu_blit_fs_triangle.vert b/src/graphics/host_gpu/shaders/gpu_blit_fs_triangle.vert new file mode 100644 index 00000000..f9bf430d --- /dev/null +++ b/src/graphics/host_gpu/shaders/gpu_blit_fs_triangle.vert @@ -0,0 +1,9 @@ +#version 450 + +layout(location = 0) out vec2 uv; + +void main() { + vec2 position = vec2(float((gl_VertexIndex & 1u) << 2u), float((gl_VertexIndex & 2u) << 1u)); + gl_Position = vec4(position - vec2(1.0), 0.0, 1.0); + uv = position * 0.5; +} diff --git a/src/graphics/host_gpu/shaders/gpu_test_ms_depth.comp b/src/graphics/host_gpu/shaders/gpu_test_ms_depth.comp new file mode 100644 index 00000000..9dd24329 --- /dev/null +++ b/src/graphics/host_gpu/shaders/gpu_test_ms_depth.comp @@ -0,0 +1,23 @@ +#version 450 +#extension GL_EXT_samplerless_texture_functions : require + +layout(local_size_x = 4) in; + +layout(set = 0, binding = 0) uniform texture2DMS input_depth; +layout(set = 0, binding = 1, std430) buffer Output { + uint value[]; +} +output_data; + +layout(push_constant) uniform Push { + uint samples; +} +push; + +void main() { + const uint sample_id = gl_LocalInvocationID.x; + if (sample_id < push.samples) { + output_data.value[sample_id] = + floatBitsToUint(texelFetch(input_depth, ivec2(0), int(sample_id)).x); + } +} diff --git a/src/graphics/host_gpu/shaders/gpu_tiler_common.inc b/src/graphics/host_gpu/shaders/gpu_tiler_common.inc index 426250bb..1941223f 100644 --- a/src/graphics/host_gpu/shaders/gpu_tiler_common.inc +++ b/src/graphics/host_gpu/shaders/gpu_tiler_common.inc @@ -3,7 +3,7 @@ layout(local_size_x = 8, local_size_y = 8) in; layout(set = 0, binding = 0, std430) readonly buffer Input { uint data[]; } input_buffer; layout(set = 0, binding = 1, std430) buffer Output { uint data[]; } output_buffer; -layout(push_constant) uniform Push { +layout(set = 0, binding = 2, std140) uniform Push { uint src_base; uint dst_base; uint width; @@ -27,7 +27,9 @@ void copy_element(uint src, uint dst) { } else { uint mask = ELEMENT_BYTES == 1 ? 0xffu : 0xffffu; uint value = (input_buffer.data[src >> 2] >> ((src & 3u) * 8u)) & mask; - atomicOr(output_buffer.data[dst >> 2], value << ((dst & 3u) * 8u)); + uint shift = (dst & 3u) * 8u; + atomicAnd(output_buffer.data[dst >> 2], ~(mask << shift)); + atomicOr(output_buffer.data[dst >> 2], value << shift); } } diff --git a/src/graphics/host_gpu/shaders/gpu_tiler_demote_d16.comp b/src/graphics/host_gpu/shaders/gpu_tiler_demote_d16.comp new file mode 100644 index 00000000..66638af1 --- /dev/null +++ b/src/graphics/host_gpu/shaders/gpu_tiler_demote_d16.comp @@ -0,0 +1,45 @@ +#version 450 + +layout(constant_id = 0) const uint D32 = 0u; +layout(local_size_x = 64) in; + +layout(set = 0, binding = 0, std430) readonly buffer Input { uint data[]; } input_buffer; +layout(set = 0, binding = 1, std430) buffer Output { uint data[]; } output_buffer; + +layout(push_constant, std430) uniform Push { + uint src_base; + uint dst_base; + uint width; + uint height; + uint depth; + uint surface_z; + uint pitch_bytes; + uint slice_bytes; + uint blocks_per_row; + uint blocks_per_slice; + uint tail_x; + uint tail_y; + uint tail; +} params; + +uint decode(uint value) { + float normalized = D32 != 0u + ? clamp(uintBitsToFloat(value), 0.0, 1.0) + : float(value & 0x00ffffffu) / 16777215.0; + return uint(round(normalized * 65535.0)); +} + +void main() { + uint x = gl_GlobalInvocationID.x; + uint y = gl_GlobalInvocationID.y; + if (x >= params.width || y >= params.height) { + return; + } + uint src = params.src_base + y * params.pitch_bytes + x * 4u; + uint dst = params.dst_base + y * params.slice_bytes + x * 2u; + uint shift = (dst & 2u) * 8u; + uint mask = 0xffffu << shift; + uint word = dst >> 2u; + atomicAnd(output_buffer.data[word], ~mask); + atomicOr(output_buffer.data[word], decode(input_buffer.data[src >> 2u]) << shift); +} diff --git a/src/graphics/host_gpu/shaders/gpu_tiler_promote_d16.comp b/src/graphics/host_gpu/shaders/gpu_tiler_promote_d16.comp new file mode 100644 index 00000000..003857a6 --- /dev/null +++ b/src/graphics/host_gpu/shaders/gpu_tiler_promote_d16.comp @@ -0,0 +1,39 @@ +#version 450 + +layout(constant_id = 0) const uint D32 = 0u; +layout(local_size_x = 64) in; + +layout(set = 0, binding = 0, std430) readonly buffer Input { uint data[]; } input_buffer; +layout(set = 0, binding = 1, std430) buffer Output { uint data[]; } output_buffer; + +layout(push_constant, std430) uniform Push { + uint src_base; + uint dst_base; + uint width; + uint height; + uint depth; + uint surface_z; + uint pitch_bytes; + uint slice_bytes; + uint blocks_per_row; + uint blocks_per_slice; + uint tail_x; + uint tail_y; + uint tail; +} params; + +void main() { + uint x = gl_GlobalInvocationID.x; + uint y = gl_GlobalInvocationID.y; + if (x >= params.width || y >= params.height) { + return; + } + uint src = params.src_base + y * params.pitch_bytes + x * 2u; + uint packed = input_buffer.data[src >> 2u]; + uint value = (packed >> ((src & 2u) * 8u)) & 0xffffu; + uint encoded = D32 != 0u + ? floatBitsToUint(float(value) / 65535.0) + : value * 256u + (value + 128u) / 257u; + uint dst = params.dst_base + y * params.slice_bytes + x * 4u; + output_buffer.data[dst >> 2u] = encoded; +} diff --git a/src/graphics/host_gpu/shaders/gpu_tiler_swap_bgra16.comp b/src/graphics/host_gpu/shaders/gpu_tiler_swap_bgra16.comp new file mode 100644 index 00000000..fb5f833e --- /dev/null +++ b/src/graphics/host_gpu/shaders/gpu_tiler_swap_bgra16.comp @@ -0,0 +1,35 @@ +#version 450 + +layout(local_size_x = 64) in; + +layout(set = 0, binding = 0, std430) readonly buffer Input { uint data[]; } input_buffer; +layout(set = 0, binding = 1, std430) buffer Output { uint data[]; } output_buffer; + +layout(push_constant, std430) uniform Push { + uint src_base; + uint dst_base; + uint width; + uint height; + uint depth; + uint surface_z; + uint pitch_bytes; + uint slice_bytes; + uint blocks_per_row; + uint blocks_per_slice; + uint tail_x; + uint tail_y; + uint tail; +} params; + +void main() { + uint pixel = gl_GlobalInvocationID.x; + if (pixel >= params.width) { + return; + } + uint src = (params.src_base >> 2u) + pixel * 2u; + uint dst = (params.dst_base >> 2u) + pixel * 2u; + uint bg = input_buffer.data[src]; + uint ra = input_buffer.data[src + 1u]; + output_buffer.data[dst] = (ra & 0x0000ffffu) | (bg & 0xffff0000u); + output_buffer.data[dst + 1u] = (bg & 0x0000ffffu) | (ra & 0xffff0000u); +} diff --git a/src/graphics/host_gpu/transfer.cpp b/src/graphics/host_gpu/transfer.cpp deleted file mode 100644 index 5ce81240..00000000 --- a/src/graphics/host_gpu/transfer.cpp +++ /dev/null @@ -1,1083 +0,0 @@ -#include "graphics/host_gpu/transfer.h" - -#include "common/assert.h" -#include "common/logging/log.h" -#include "common/profiler.h" -#include "common/threads.h" -#include "graphics/host_gpu/gpuTiler.h" -#include "graphics/host_gpu/graphicContext.h" -#include "graphics/host_gpu/renderer/imageView.h" -#include "graphics/host_gpu/renderer/render.h" -#include "graphics/host_gpu/renderer/renderContext.h" -#include "graphics/host_gpu/vma.h" -#include "graphics/host_gpu/vulkanCommon.h" -#include "graphics/presentation/window.h" - -#include -#include -#include -#include - -namespace Libs::Graphics::Transfer { - -static Common::Mutex g_scratch_buffer_mutex; -static std::unique_ptr g_scratch_buffer; -static uint64_t g_scratch_buffer_capacity = 0; - -std::vector MakeLayeredImageBufferCopies(uint32_t layers, uint64_t slice_size, - uint32_t pitch, uint32_t width, - uint32_t height, - vk::ImageAspectFlags aspect) { - if (layers == 0 || slice_size == 0 || slice_size > UINT32_MAX / layers || pitch < width || - width == 0 || height == 0) { - EXIT("invalid layered image-buffer copy layout\n"); - } - std::vector regions(layers); - for (uint32_t layer = 0; layer < layers; layer++) { - auto& region = regions[layer]; - region.offset = static_cast(slice_size * layer); - region.pitch = pitch; - region.width = width; - region.height = height; - region.src_layer = layer; - region.aspect = aspect; - } - return regions; -} - -ScratchBuffer::ScratchBuffer(uint64_t size) { - EXIT_IF(size == 0); - - g_scratch_buffer_mutex.Lock(); - - if (g_scratch_buffer_capacity < size) { - g_scratch_buffer = std::make_unique(size); - g_scratch_buffer_capacity = size; - } - - m_data = g_scratch_buffer.get(); -} - -ScratchBuffer::~ScratchBuffer() { - g_scratch_buffer_mutex.Unlock(); -} - -bool GuestBufferIsTiled(uint64_t vaddr, uint64_t size) { - if ((size & 0x7u) == 0) { - if (size == 0) { - return false; - } - - const auto* ptr = reinterpret_cast(vaddr); - const auto* ptr_end = reinterpret_cast(vaddr + size); - const auto element = *ptr; - - for (; ptr < ptr_end; ptr++) { - if (element != *ptr) { - return true; - } - } - return false; - } - return true; -} - -void WaitForQueueIdle() { - auto& graphics = GetRenderContext().GetGraphics(); - EXIT_IF(graphics.queue == nullptr); - Common::LockGuard lock(graphics.queue_mutex); - const auto result = graphics.queue.waitIdle(); - - if (result != vk::Result::eSuccess) { - LOGF("vkQueueWaitIdle failed: %s (%d)\n", VulkanToString(result).c_str(), - static_cast(result)); - } - EXIT_NOT_IMPLEMENTED(result != vk::Result::eSuccess); -} - -static void SetImageLayout(vk::CommandBuffer buffer, VulkanImage& dst_image, uint32_t base_level, - uint32_t levels, vk::ImageAspectFlags aspect_mask, - vk::ImageLayout old_image_layout, vk::ImageLayout new_image_layout); - -template -static void ExecuteImmediateCommands(const Recorder& recorder) { - CommandBuffer command; - command.Begin(); - recorder(command, command.Handle()); - command.End(); - command.Execute(); - command.WaitForFence(); -} - -[[nodiscard]] static bool IsSingleImageAspect(vk::ImageAspectFlags aspect) { - return aspect == vk::ImageAspectFlagBits::eColor || aspect == vk::ImageAspectFlagBits::eDepth || - aspect == vk::ImageAspectFlagBits::eStencil; -} - -[[nodiscard]] static vk::BufferImageCopy -MakeBufferImageCopy(vk::DeviceSize buffer_offset, uint32_t buffer_pitch, - vk::ImageAspectFlags aspect, uint32_t mip_level, uint32_t array_layer, - vk::Offset3D image_offset, vk::Extent3D image_extent) { - vk::BufferImageCopy copy {}; - copy.bufferOffset = buffer_offset; - copy.bufferRowLength = buffer_pitch != image_extent.width ? buffer_pitch : 0; - copy.bufferImageHeight = 0; - copy.imageSubresource.aspectMask = aspect; - copy.imageSubresource.mipLevel = mip_level; - copy.imageSubresource.baseArrayLayer = array_layer; - copy.imageSubresource.layerCount = 1; - copy.imageOffset = image_offset; - copy.imageExtent = image_extent; - return copy; -} - -static void SetBufferMemoryBarrier(vk::CommandBuffer command, vk::Buffer buffer, - vk::DeviceSize offset, vk::DeviceSize size, - vk::AccessFlags src_access, vk::AccessFlags dst_access, - vk::PipelineStageFlags src_stage, - vk::PipelineStageFlags dst_stage) { - vk::BufferMemoryBarrier barrier {}; - barrier.sType = vk::StructureType::eBufferMemoryBarrier; - barrier.srcAccessMask = src_access; - barrier.dstAccessMask = dst_access; - barrier.srcQueueFamilyIndex = VK_QUEUE_FAMILY_IGNORED; - barrier.dstQueueFamilyIndex = VK_QUEUE_FAMILY_IGNORED; - barrier.buffer = buffer; - barrier.offset = offset; - barrier.size = size; - command.pipelineBarrier(src_stage, dst_stage, vk::DependencyFlags {}, 0, nullptr, 1, &barrier, - 0, nullptr); -} - -[[nodiscard]] static vk::ImageAspectFlags GetTransferAspects(const VulkanImage& image, - vk::ImageAspectFlags copy_aspect) { - if (copy_aspect == vk::ImageAspectFlagBits::eColor) { - return vk::ImageAspectFlagBits::eColor; - } - if (copy_aspect == vk::ImageAspectFlagBits::eDepth || - copy_aspect == vk::ImageAspectFlagBits::eStencil) { - const auto aspects = ImageViewOps::DepthAspectMask(image.format); - if (!(copy_aspect & aspects)) { - EXIT("image transfer aspect is unavailable: format=%d copy=0x%x available=0x%x\n", - static_cast(image.format), - static_cast(copy_aspect), - static_cast(aspects)); - } - return aspects; - } - EXIT("unsupported image transfer aspect: 0x%x\n", - static_cast(copy_aspect)); -} - -static void RecordBufferToImageCopy(CommandBuffer& command, VulkanBuffer& src_buffer, - VulkanImage& dst_image, - std::span regions, - vk::ImageAspectFlags transition_aspects, - vk::ImageLayout initial_layout, vk::ImageLayout final_layout) { - auto vk_command = command.Handle(); - SetBufferMemoryBarrier(vk_command, src_buffer.buffer, 0, VK_WHOLE_SIZE, - vk::AccessFlagBits::eMemoryWrite, vk::AccessFlagBits::eTransferRead, - vk::PipelineStageFlagBits::eAllCommands, - vk::PipelineStageFlagBits::eTransfer); - SetImageLayout(vk_command, dst_image, 0, VK_REMAINING_MIP_LEVELS, transition_aspects, - initial_layout, vk::ImageLayout::eTransferDstOptimal); - vk_command.copyBufferToImage(src_buffer.buffer, dst_image.image, - vk::ImageLayout::eTransferDstOptimal, - static_cast(regions.size()), regions.data()); - SetImageLayout(vk_command, dst_image, 0, VK_REMAINING_MIP_LEVELS, transition_aspects, - vk::ImageLayout::eTransferDstOptimal, final_layout); -} - -[[nodiscard]] static std::span -ConvertBufferImageCopies(std::span regions) { - static thread_local std::vector vk_regions; - vk_regions.resize(regions.size()); - - for (size_t i = 0; i < regions.size(); i++) { - const auto& r = regions[i]; - vk_regions[i] = MakeBufferImageCopy( - r.offset, r.pitch, r.aspect, r.dst_level, r.dst_layer, {r.dst_x, r.dst_y, r.dst_z}, - {r.width, (r.copy_height != 0 ? r.copy_height : r.height), 1}); - } - return vk_regions; -} - -[[nodiscard]] static std::span -ConvertImageBufferCopies(std::span regions) { - static thread_local std::vector vk_regions; - vk_regions.resize(regions.size()); - for (size_t i = 0; i < regions.size(); i++) { - const auto& r = regions[i]; - vk_regions[i] = MakeBufferImageCopy( - r.offset, r.pitch, r.aspect, r.src_level, r.src_layer, {r.src_x, r.src_y, r.src_z}, - {r.width, r.copy_height != 0 ? r.copy_height : r.height, 1}); - } - return vk_regions; -} - -static void -RecordImageToBuffer(CommandBuffer& command, VulkanImage& src_image, VulkanBuffer& dst_buffer, - std::span regions, vk::ImageLayout final_layout, - vk::AccessFlags final_access = vk::AccessFlagBits::eHostRead, - vk::PipelineStageFlags final_stage = vk::PipelineStageFlagBits::eHost) { - auto vk_command = command.Handle(); - vk::ImageAspectFlags aspects = {}; - for (const auto& region: regions) { - aspects |= GetTransferAspects(src_image, region.aspect); - } - SetImageLayout(vk_command, src_image, 0, VK_REMAINING_MIP_LEVELS, aspects, src_image.layout, - vk::ImageLayout::eTransferSrcOptimal); - SetBufferMemoryBarrier(vk_command, dst_buffer.buffer, 0, VK_WHOLE_SIZE, - vk::AccessFlagBits::eMemoryRead, vk::AccessFlagBits::eTransferWrite, - vk::PipelineStageFlagBits::eAllCommands, - vk::PipelineStageFlagBits::eTransfer); - const auto copies = ConvertImageBufferCopies(regions); - vk_command.copyImageToBuffer(src_image.image, vk::ImageLayout::eTransferSrcOptimal, - dst_buffer.buffer, static_cast(copies.size()), - copies.data()); - SetBufferMemoryBarrier(vk_command, dst_buffer.buffer, 0, VK_WHOLE_SIZE, - vk::AccessFlagBits::eTransferWrite, final_access, - vk::PipelineStageFlagBits::eTransfer, final_stage); - SetImageLayout(vk_command, src_image, 0, VK_REMAINING_MIP_LEVELS, aspects, - vk::ImageLayout::eTransferSrcOptimal, final_layout); -} - -static void RecordImageToBuffer(CommandBuffer& command, VulkanImage& src_image, - VulkanBuffer& dst_buffer, uint32_t dst_pitch, - vk::ImageLayout final_layout, vk::ImageAspectFlags aspect) { - const ImageBufferCopy region { - 0, dst_pitch, 0, src_image.extent.width, src_image.extent.height, 0, 0, 0, 0, 0, aspect}; - RecordImageToBuffer(command, src_image, dst_buffer, - std::span(®ion, 1), final_layout); -} - -class ReusableStagingBuffer { -public: - explicit ReusableStagingBuffer(GraphicContext& graphics): m_graphics(graphics) {} - ~ReusableStagingBuffer() = default; - KYTY_CLASS_NO_COPY(ReusableStagingBuffer); - - void UploadToBuffer(VulkanBuffer& dst_buffer, const void* src_data, uint64_t size, - uint64_t dst_offset) { - RecordUpload(src_data, size, [&](CommandBuffer&, vk::CommandBuffer vk_command) { - SetBufferMemoryBarrier( - vk_command, m_buffer.buffer, 0, size, vk::AccessFlagBits::eMemoryWrite, - vk::AccessFlagBits::eTransferRead, vk::PipelineStageFlagBits::eAllCommands, - vk::PipelineStageFlagBits::eTransfer); - SetBufferMemoryBarrier( - vk_command, dst_buffer.buffer, dst_offset, size, - vk::AccessFlagBits::eMemoryRead | vk::AccessFlagBits::eMemoryWrite, - vk::AccessFlagBits::eTransferWrite, vk::PipelineStageFlagBits::eAllCommands, - vk::PipelineStageFlagBits::eTransfer); - const vk::BufferCopy copy_region { - .srcOffset = 0, .dstOffset = dst_offset, .size = size}; - vk_command.copyBuffer(m_buffer.buffer, dst_buffer.buffer, 1, ©_region); - SetBufferMemoryBarrier( - vk_command, dst_buffer.buffer, dst_offset, size, vk::AccessFlagBits::eTransferWrite, - vk::AccessFlagBits::eVertexAttributeRead | vk::AccessFlagBits::eIndexRead | - vk::AccessFlagBits::eShaderRead | vk::AccessFlagBits::eShaderWrite, - vk::PipelineStageFlagBits::eTransfer, - vk::PipelineStageFlagBits::eVertexInput | vk::PipelineStageFlagBits::eVertexShader | - vk::PipelineStageFlagBits::eFragmentShader | - vk::PipelineStageFlagBits::eComputeShader); - }); - } - - void UploadToImage(VulkanImage& image, const void* src_data, uint64_t size, uint32_t src_pitch, - vk::ImageAspectFlags copy_aspect, vk::ImageLayout initial_layout, - vk::ImageLayout final_layout) { - const auto transition_aspects = GetTransferAspects(image, copy_aspect); - RecordUpload(src_data, size, [&](CommandBuffer& command, vk::CommandBuffer) { - const auto region = MakeBufferImageCopy(0, src_pitch, copy_aspect, 0, 0, {0, 0, 0}, - {image.extent.width, image.extent.height, 1}); - RecordBufferToImageCopy(command, m_buffer, image, - std::span(®ion, 1), - transition_aspects, initial_layout, final_layout); - }); - } - - void UploadToImage(VulkanImage& dst_image, const void* src_data, uint64_t size, - std::span regions, vk::ImageLayout dst_layout) { - RecordUpload(src_data, size, [&](CommandBuffer& command, vk::CommandBuffer) { - vk::ImageAspectFlags transition_aspects = {}; - for (const auto& region: regions) { - transition_aspects |= GetTransferAspects(dst_image, region.aspect); - } - RecordBufferToImageCopy(command, m_buffer, dst_image, ConvertBufferImageCopies(regions), - transition_aspects, dst_image.layout, dst_layout); - }); - } - - void DownloadFromImage(void* dst_data, uint64_t size, uint32_t dst_pitch, - VulkanImage& src_image, vk::ImageLayout src_layout, - vk::ImageAspectFlags aspect) { - Common::LockGuard lock(m_mutex); - - EnsureBuffer(size, vk::BufferUsageFlagBits::eTransferDst); - - ExecuteImmediateCommands([&](CommandBuffer& command, vk::CommandBuffer) { - RecordImageToBuffer(command, src_image, m_buffer, dst_pitch, src_layout, aspect); - }); - - std::memcpy(dst_data, m_mapped_data, size); - } - - void DownloadFromImage(void* dst_data, uint64_t size, std::span regions, - VulkanImage& src_image, vk::ImageLayout src_layout) { - WithDownloadedImage(size, regions, src_image, src_layout, - [&](std::span data) { - std::memcpy(dst_data, data.data(), data.size()); - }); - } - - void ProcessDownloadedImage(uint64_t size, std::span regions, - VulkanImage& src_image, vk::ImageLayout src_layout, - const DownloadedImageConsumer& consumer) { - WithDownloadedImage(size, regions, src_image, src_layout, - [&](std::span data) { - if (m_host_cached) { - consumer(data); - return; - } - // Read uncached mappings sequentially before CPU consumers inspect - // them. - m_cached_readback.resize(data.size()); - std::memcpy(m_cached_readback.data(), data.data(), data.size()); - consumer(m_cached_readback); - }); - } - - void Release() { - Common::LockGuard lock(m_mutex); - if (m_buffer.buffer != nullptr) { - EXIT_IF(m_mapped_data == nullptr); - m_graphics.UnmapMemory(m_buffer.memory); - m_graphics.DeleteBuffer(m_buffer); - } - m_capacity = 0; - m_mapped_data = nullptr; - m_host_cached = false; - } - -private: - template - void WithDownloadedImage(uint64_t size, std::span regions, - VulkanImage& src_image, vk::ImageLayout src_layout, - const Consumer& consumer) { - Common::LockGuard lock(m_mutex); - EnsureBuffer(size, vk::BufferUsageFlagBits::eTransferDst); - ExecuteImmediateCommands([&](CommandBuffer& command, vk::CommandBuffer) { - RecordImageToBuffer(command, src_image, m_buffer, regions, src_layout); - }); - consumer(std::span(static_cast(m_mapped_data), - static_cast(size))); - } - - template - void RecordUpload(const void* src_data, uint64_t size, const Recorder& recorder) { - Common::LockGuard lock(m_mutex); - CopyFromHost(src_data, size, vk::BufferUsageFlagBits::eTransferSrc); - ExecuteImmediateCommands(recorder); - } - - void CopyFromHost(const void* src_data, uint64_t size, vk::BufferUsageFlags usage) { - EnsureBuffer(size, usage); - - std::memcpy(m_mapped_data, src_data, size); - } - - void EnsureBuffer(uint64_t size, vk::BufferUsageFlags usage) { - if (m_capacity < size || m_buffer.usage != usage) { - if (m_buffer.buffer != nullptr) { - m_graphics.UnmapMemory(m_buffer.memory); - m_mapped_data = nullptr; - m_graphics.DeleteBuffer(m_buffer); - } - - m_buffer.usage = usage; - m_buffer.memory.property = vk::MemoryPropertyFlagBits::eHostVisible | - vk::MemoryPropertyFlagBits::eHostCoherent; - // CPU readback benefits from cached host memory. Keep it optional so Vulkan devices - // without a coherent cached type can fall back to the original sequential-copy path. - m_buffer.memory.preferred_property = - usage & vk::BufferUsageFlagBits::eTransferDst - ? vk::MemoryPropertyFlags(vk::MemoryPropertyFlagBits::eHostCached) - : vk::MemoryPropertyFlags {}; - m_graphics.CreateBuffer(size, m_buffer); - m_capacity = size; - const auto& properties = m_graphics.GetPhysicalDeviceMemoryProperties(); - EXIT_IF(m_buffer.memory.type >= properties.memoryTypeCount); - m_host_cached = - static_cast(properties.memoryTypes[m_buffer.memory.type].propertyFlags & - vk::MemoryPropertyFlagBits::eHostCached); - - m_graphics.MapMemory(m_buffer.memory, m_mapped_data); - } - } - - GraphicContext& m_graphics; - Common::Mutex m_mutex; - VulkanBuffer m_buffer; - uint64_t m_capacity = 0; - void* m_mapped_data = nullptr; - bool m_host_cached = false; - std::vector m_cached_readback; -}; - -struct TransferResources { - explicit TransferResources(GraphicContext& graphics) - : graphics(graphics), texture(graphics), vertex(graphics), readback(graphics) {} - - GraphicContext& graphics; - ReusableStagingBuffer texture; - ReusableStagingBuffer vertex; - ReusableStagingBuffer readback; -}; - -static Common::Mutex g_resources_mutex; -static std::unique_ptr g_resources; - -static TransferResources& GetResources() { - Common::LockGuard lock(g_resources_mutex); - if (!g_resources) { - g_resources = std::make_unique(GetRenderContext().GetGraphics()); - } - return *g_resources; -} - -bool IsBlockCompressedFormat(vk::Format format) { - return format == vk::Format::eBc1RgbUnormBlock || format == vk::Format::eBc1RgbSrgbBlock || - format == vk::Format::eBc1RgbaUnormBlock || format == vk::Format::eBc1RgbaSrgbBlock || - format == vk::Format::eBc2UnormBlock || format == vk::Format::eBc2SrgbBlock || - format == vk::Format::eBc3UnormBlock || format == vk::Format::eBc3SrgbBlock || - format == vk::Format::eBc4UnormBlock || format == vk::Format::eBc4SnormBlock || - format == vk::Format::eBc5UnormBlock || format == vk::Format::eBc5SnormBlock || - format == vk::Format::eBc6HUfloatBlock || format == vk::Format::eBc6HSfloatBlock || - format == vk::Format::eBc7UnormBlock || format == vk::Format::eBc7SrgbBlock; -} - -uint32_t BlockCompressedBytesPerBlock(vk::Format format) { - switch (format) { - case vk::Format::eBc1RgbUnormBlock: - case vk::Format::eBc1RgbSrgbBlock: - case vk::Format::eBc1RgbaUnormBlock: - case vk::Format::eBc1RgbaSrgbBlock: - case vk::Format::eBc4UnormBlock: - case vk::Format::eBc4SnormBlock: return 8; - case vk::Format::eBc2UnormBlock: - case vk::Format::eBc2SrgbBlock: - case vk::Format::eBc3UnormBlock: - case vk::Format::eBc3SrgbBlock: - case vk::Format::eBc5UnormBlock: - case vk::Format::eBc5SnormBlock: - case vk::Format::eBc6HUfloatBlock: - case vk::Format::eBc6HSfloatBlock: - case vk::Format::eBc7UnormBlock: - case vk::Format::eBc7SrgbBlock: return 16; - default: return 0; - } -} - -static uint32_t GetFormatTexelSize(vk::Format format) { - switch (format) { - case vk::Format::eR32G32Sfloat: - case vk::Format::eR32G32Uint: return 8; - case vk::Format::eR32G32B32A32Sfloat: - case vk::Format::eR32G32B32A32Uint: return 16; - default: return 0; - } -} - -static ReusableStagingBuffer& GetStagingBuffer(StagingBufferType type) { - auto& resources = GetResources(); - switch (type) { - case StagingBufferType::Texture: return resources.texture; - case StagingBufferType::Vertex: return resources.vertex; - case StagingBufferType::ReadBack: return resources.readback; - default: EXIT("unknown staging buffer type\n"); - } - return resources.texture; -} - -void ReleaseCachedResources() { - GpuTileRelease(); - Common::LockGuard lock(g_resources_mutex); - if (g_resources) { - g_resources->texture.Release(); - g_resources->vertex.Release(); - g_resources->readback.Release(); - } -} - -void UploadTiledImage(VulkanImage& dst_image, const void* tiled_data, uint64_t tiled_size, - uint64_t linear_size, std::span infos, - std::span regions, vk::ImageLayout dst_layout) { - EXIT_IF(tiled_data == nullptr || regions.empty()); - GpuDetile(tiled_data, nullptr, tiled_size, linear_size, infos, - [&](CommandBuffer& command, VulkanBuffer& linear) { - vk::ImageAspectFlags aspects {}; - for (const auto& region: regions) { - aspects |= GetTransferAspects(dst_image, region.aspect); - } - RecordBufferToImageCopy(command, linear, dst_image, - ConvertBufferImageCopies(regions), aspects, - dst_image.layout, dst_layout); - }); -} - -void DownloadTiledImage(void* tiled_data, uint64_t tiled_size, uint64_t linear_size, - std::span infos, - std::span regions, VulkanImage& src_image, - vk::ImageLayout src_layout) { - EXIT_IF(tiled_data == nullptr || regions.empty()); - GpuTile(nullptr, tiled_data, tiled_size, linear_size, infos, - [&](CommandBuffer& command, VulkanBuffer& linear) { - RecordImageToBuffer(command, src_image, linear, regions, src_layout, - vk::AccessFlagBits::eShaderRead, - vk::PipelineStageFlagBits::eComputeShader); - }); -} - -static void SetImageLayout(vk::CommandBuffer buffer, VulkanImage& dst_image, uint32_t base_level, - uint32_t levels, vk::ImageAspectFlags aspect_mask, - vk::ImageLayout old_image_layout, vk::ImageLayout new_image_layout) { - if ((old_image_layout == vk::ImageLayout::eTransferDstOptimal && - new_image_layout == vk::ImageLayout::eTransferSrcOptimal) || - (old_image_layout == vk::ImageLayout::eTransferDstOptimal && - new_image_layout == vk::ImageLayout::eGeneral) || - (old_image_layout == vk::ImageLayout::eTransferDstOptimal && - new_image_layout == vk::ImageLayout::ePresentSrcKHR)) { - static std::atomic log_count {0}; - if (log_count.fetch_add(1, std::memory_order_relaxed) < 64) { - LOGF("set_image_layout: image=%p type=%d tracked=%d requested=%d->%d base=%u " - "levels=%u\n", - VulkanHandleToPointer(dst_image.image), static_cast(dst_image.type), - static_cast(dst_image.layout), static_cast(old_image_layout), - static_cast(new_image_layout), base_level, levels); - } - } - - if (old_image_layout == new_image_layout) { - dst_image.layout = new_image_layout; - return; - } - - vk::ImageMemoryBarrier image_memory_barrier {}; - image_memory_barrier.sType = vk::StructureType::eImageMemoryBarrier; - image_memory_barrier.pNext = nullptr; - image_memory_barrier.srcAccessMask = {}; - image_memory_barrier.dstAccessMask = {}; - image_memory_barrier.oldLayout = old_image_layout; - image_memory_barrier.newLayout = new_image_layout; - image_memory_barrier.srcQueueFamilyIndex = VK_QUEUE_FAMILY_IGNORED; - image_memory_barrier.dstQueueFamilyIndex = VK_QUEUE_FAMILY_IGNORED; - image_memory_barrier.image = dst_image.image; - image_memory_barrier.subresourceRange.aspectMask = aspect_mask; - image_memory_barrier.subresourceRange.baseMipLevel = base_level; - image_memory_barrier.subresourceRange.levelCount = levels; - image_memory_barrier.subresourceRange.baseArrayLayer = 0; - image_memory_barrier.subresourceRange.layerCount = dst_image.layers; - - switch (old_image_layout) { - case vk::ImageLayout::eColorAttachmentOptimal: - image_memory_barrier.srcAccessMask = vk::AccessFlagBits::eColorAttachmentWrite; - break; - case vk::ImageLayout::eDepthStencilAttachmentOptimal: - image_memory_barrier.srcAccessMask = vk::AccessFlagBits::eDepthStencilAttachmentRead | - vk::AccessFlagBits::eDepthStencilAttachmentWrite; - break; - case vk::ImageLayout::eGeneral: - image_memory_barrier.srcAccessMask = - vk::AccessFlagBits::eShaderRead | vk::AccessFlagBits::eShaderWrite | - vk::AccessFlagBits::eColorAttachmentRead | - vk::AccessFlagBits::eColorAttachmentWrite | vk::AccessFlagBits::eTransferRead | - vk::AccessFlagBits::eTransferWrite; - break; - case vk::ImageLayout::eTransferDstOptimal: - image_memory_barrier.srcAccessMask = vk::AccessFlagBits::eTransferWrite; - break; - case vk::ImageLayout::eTransferSrcOptimal: - image_memory_barrier.srcAccessMask = vk::AccessFlagBits::eTransferRead; - break; - case vk::ImageLayout::ePreinitialized: - image_memory_barrier.srcAccessMask = vk::AccessFlagBits::eHostWrite; - break; - default: break; - } - - switch (new_image_layout) { - case vk::ImageLayout::eTransferDstOptimal: - image_memory_barrier.dstAccessMask = vk::AccessFlagBits::eTransferWrite; - break; - case vk::ImageLayout::eTransferSrcOptimal: - image_memory_barrier.dstAccessMask = vk::AccessFlagBits::eTransferRead; - break; - case vk::ImageLayout::eShaderReadOnlyOptimal: - image_memory_barrier.srcAccessMask |= - vk::AccessFlagBits::eTransferWrite | vk::AccessFlagBits::eHostWrite; - image_memory_barrier.dstAccessMask = vk::AccessFlagBits::eShaderRead; - break; - case vk::ImageLayout::eGeneral: - image_memory_barrier.dstAccessMask = - vk::AccessFlagBits::eShaderRead | vk::AccessFlagBits::eShaderWrite; - break; - case vk::ImageLayout::eColorAttachmentOptimal: - image_memory_barrier.dstAccessMask = vk::AccessFlagBits::eColorAttachmentWrite; - break; - case vk::ImageLayout::eDepthStencilAttachmentOptimal: - image_memory_barrier.dstAccessMask = vk::AccessFlagBits::eDepthStencilAttachmentRead | - vk::AccessFlagBits::eDepthStencilAttachmentWrite; - break; - default: break; - } - - auto src_stages = static_cast( - vk::PipelineStageFlagBits::eTopOfPipe | vk::PipelineStageFlagBits::eTransfer | - vk::PipelineStageFlagBits::eColorAttachmentOutput | - vk::PipelineStageFlagBits::eComputeShader | vk::PipelineStageFlagBits::eEarlyFragmentTests | - vk::PipelineStageFlagBits::eLateFragmentTests | vk::PipelineStageFlagBits::eHost); - auto dest_stages = static_cast( - vk::PipelineStageFlagBits::eTransfer | vk::PipelineStageFlagBits::eVertexShader | - vk::PipelineStageFlagBits::eFragmentShader | vk::PipelineStageFlagBits::eComputeShader | - vk::PipelineStageFlagBits::eColorAttachmentOutput | - vk::PipelineStageFlagBits::eEarlyFragmentTests | - vk::PipelineStageFlagBits::eLateFragmentTests); - - buffer.pipelineBarrier(src_stages, dest_stages, vk::DependencyFlags {}, 0, nullptr, 0, nullptr, - 1, &image_memory_barrier); - - dst_image.layout = new_image_layout; -} - -static uint64_t CompressedImageCopyBufferSize(const ImageImageCopy& r, - const VulkanImage& dst_image) { - if (r.src_aspect != vk::ImageAspectFlagBits::eColor || - r.dst_aspect != vk::ImageAspectFlagBits::eColor || - !IsBlockCompressedFormat(dst_image.format)) { - return 0; - } - - auto src_texel_size = GetFormatTexelSize(r.src_image.format); - auto dst_block_size = BlockCompressedBytesPerBlock(dst_image.format); - if (src_texel_size == 0 || src_texel_size != dst_block_size) { - return 0; - } - - auto block_width = (r.width + 3u) / 4u; - auto block_height = (r.height + 3u) / 4u; - if (block_width > r.src_image.extent.width || block_height > r.src_image.extent.height) { - return 0; - } - return static_cast(block_width) * block_height * dst_block_size; -} - -static void CopyBlockImageToCompressedImage(vk::CommandBuffer vk_buffer, const ImageImageCopy& r, - VulkanImage& dst_image, VulkanBuffer& copy_buffer) { - const auto copy_size = CompressedImageCopyBufferSize(r, dst_image); - EXIT_IF(copy_size == 0 || copy_buffer.buffer_size < copy_size); - - const auto block_width = (r.width + 3u) / 4u; - const auto block_height = (r.height + 3u) / 4u; - - const auto src_region = - MakeBufferImageCopy(0, 0, vk::ImageAspectFlagBits::eColor, r.src_level, r.src_layer, - {r.src_x, r.src_y, 0}, {block_width, block_height, 1}); - - auto src_layout = r.src_image.layout; - SetImageLayout(vk_buffer, r.src_image, r.src_level, 1, vk::ImageAspectFlagBits::eColor, - src_layout, vk::ImageLayout::eTransferSrcOptimal); - - SetBufferMemoryBarrier(vk_buffer, copy_buffer.buffer, 0, VK_WHOLE_SIZE, - vk::AccessFlagBits::eTransferRead, vk::AccessFlagBits::eTransferWrite, - vk::PipelineStageFlagBits::eTransfer, - vk::PipelineStageFlagBits::eTransfer); - vk_buffer.copyImageToBuffer(r.src_image.image, vk::ImageLayout::eTransferSrcOptimal, - copy_buffer.buffer, 1, &src_region); - SetBufferMemoryBarrier(vk_buffer, copy_buffer.buffer, 0, VK_WHOLE_SIZE, - vk::AccessFlagBits::eTransferWrite, vk::AccessFlagBits::eTransferRead, - vk::PipelineStageFlagBits::eTransfer, - vk::PipelineStageFlagBits::eTransfer); - - const auto dst_region = - MakeBufferImageCopy(0, 0, vk::ImageAspectFlagBits::eColor, r.dst_level, r.dst_layer, - {r.dst_x, r.dst_y, 0}, {r.width, r.height, 1}); - - vk_buffer.copyBufferToImage(copy_buffer.buffer, dst_image.image, - vk::ImageLayout::eTransferDstOptimal, 1, &dst_region); - - SetImageLayout(vk_buffer, r.src_image, r.src_level, 1, vk::ImageAspectFlagBits::eColor, - vk::ImageLayout::eTransferSrcOptimal, src_layout); -} - -void CopyImage(CommandBuffer& buffer, std::span regions, - VulkanImage& dst_image, vk::ImageLayout dst_layout) { - auto vk_buffer = buffer.Handle(); - - bool same_image_copy = false; - vk::ImageAspectFlags dst_transition_aspects = {}; - for (const auto& r: regions) { - dst_transition_aspects |= GetTransferAspects(dst_image, r.dst_aspect); - if (&r.src_image == &dst_image) { - same_image_copy = true; - break; - } - } - - if (same_image_copy) { - SetImageLayout(vk_buffer, dst_image, 0, VK_REMAINING_MIP_LEVELS, dst_transition_aspects, - dst_image.layout, vk::ImageLayout::eGeneral); - - for (const auto& r: regions) { - vk::ImageCopy region; - - region.srcSubresource.aspectMask = r.src_aspect; - region.srcSubresource.mipLevel = r.src_level; - region.srcSubresource.baseArrayLayer = r.src_layer; - region.srcSubresource.layerCount = 1; - region.srcOffset = {r.src_x, r.src_y, r.src_z}; - region.dstSubresource.aspectMask = r.dst_aspect; - region.dstSubresource.mipLevel = r.dst_level; - region.dstSubresource.baseArrayLayer = r.dst_layer; - region.dstSubresource.layerCount = 1; - region.dstOffset = {r.dst_x, r.dst_y, r.dst_z}; - region.extent = {r.width, r.height, 1}; - - auto src_layout = vk::ImageLayout::eGeneral; - auto restore_layout = vk::ImageLayout::eGeneral; - if (&r.src_image != &dst_image) { - restore_layout = r.src_image.layout; - SetImageLayout(vk_buffer, r.src_image, r.src_level, 1, - GetTransferAspects(r.src_image, r.src_aspect), restore_layout, - vk::ImageLayout::eTransferSrcOptimal); - src_layout = vk::ImageLayout::eTransferSrcOptimal; - } - - vk_buffer.copyImage(r.src_image.image, src_layout, dst_image.image, - vk::ImageLayout::eGeneral, 1, ®ion); - - if (&r.src_image != &dst_image) { - SetImageLayout(vk_buffer, r.src_image, r.src_level, 1, - GetTransferAspects(r.src_image, r.src_aspect), - vk::ImageLayout::eTransferSrcOptimal, restore_layout); - } - } - - SetImageLayout(vk_buffer, dst_image, 0, VK_REMAINING_MIP_LEVELS, dst_transition_aspects, - vk::ImageLayout::eGeneral, dst_layout); - return; - } - - SetImageLayout(vk_buffer, dst_image, 0, VK_REMAINING_MIP_LEVELS, dst_transition_aspects, - dst_image.layout, vk::ImageLayout::eTransferDstOptimal); - uint64_t compressed_copy_size = 0; - for (const auto& r: regions) { - compressed_copy_size = - std::max(compressed_copy_size, CompressedImageCopyBufferSize(r, dst_image)); - } - std::unique_ptr compressed_copy_owner; - VulkanBuffer* compressed_copy_buffer = nullptr; - if (compressed_copy_size != 0) { - compressed_copy_owner = std::make_unique(); - compressed_copy_buffer = compressed_copy_owner.get(); - compressed_copy_buffer->usage = - vk::BufferUsageFlagBits::eTransferSrc | vk::BufferUsageFlagBits::eTransferDst; - compressed_copy_buffer->memory.property = {}; - buffer.GetGraphics().CreateBuffer(compressed_copy_size, *compressed_copy_buffer); - buffer.RetireBufferAfterFence(std::move(compressed_copy_owner)); - } - - for (const auto& r: regions) { - vk::ImageCopy region; - - auto src_layout = r.src_image.layout; - - if (CompressedImageCopyBufferSize(r, dst_image) != 0) { - CopyBlockImageToCompressedImage(vk_buffer, r, dst_image, *compressed_copy_buffer); - continue; - } - - region.srcSubresource.aspectMask = r.src_aspect; - region.srcSubresource.mipLevel = r.src_level; - region.srcSubresource.baseArrayLayer = r.src_layer; - region.srcSubresource.layerCount = 1; - region.srcOffset = {r.src_x, r.src_y, r.src_z}; - region.dstSubresource.aspectMask = r.dst_aspect; - region.dstSubresource.mipLevel = r.dst_level; - region.dstSubresource.baseArrayLayer = r.dst_layer; - region.dstSubresource.layerCount = 1; - region.dstOffset = {r.dst_x, r.dst_y, r.dst_z}; - region.extent = {r.width, r.height, 1}; - - SetImageLayout(vk_buffer, r.src_image, r.src_level, 1, - GetTransferAspects(r.src_image, r.src_aspect), src_layout, - vk::ImageLayout::eTransferSrcOptimal); - - vk_buffer.copyImage(r.src_image.image, vk::ImageLayout::eTransferSrcOptimal, - dst_image.image, vk::ImageLayout::eTransferDstOptimal, 1, ®ion); - - SetImageLayout(vk_buffer, r.src_image, r.src_level, 1, - GetTransferAspects(r.src_image, r.src_aspect), - vk::ImageLayout::eTransferSrcOptimal, src_layout); - } - - SetImageLayout(vk_buffer, dst_image, 0, VK_REMAINING_MIP_LEVELS, dst_transition_aspects, - vk::ImageLayout::eTransferDstOptimal, dst_layout); -} - -void CopyImageViaBuffer(CommandBuffer& buffer, VulkanImage& src_image, - vk::ImageAspectFlags src_aspect, VulkanImage& dst_image, - vk::ImageAspectFlags dst_aspect, uint32_t bytes_per_element, - vk::ImageLayout dst_layout) { - if (!IsSingleImageAspect(src_aspect) || !IsSingleImageAspect(dst_aspect) || - (bytes_per_element != 2 && bytes_per_element != 4) || src_image.layers != 1 || - dst_image.layers != 1 || src_image.mip_levels != 1 || dst_image.mip_levels != 1 || - src_image.extent.width == 0 || src_image.extent.height == 0 || - src_image.extent.width != dst_image.extent.width || - src_image.extent.height != dst_image.extent.height) { - EXIT("unsupported image-buffer-image copy, src_aspect=0x%x dst_aspect=0x%x bpe=%u " - "src=%ux%u/%u/%u dst=%ux%u/%u/%u\n", - static_cast(src_aspect), - static_cast(dst_aspect), bytes_per_element, - src_image.extent.width, src_image.extent.height, src_image.layers, - src_image.mip_levels, dst_image.extent.width, dst_image.extent.height, - dst_image.layers, dst_image.mip_levels); - } - - const uint64_t row_bytes = static_cast(src_image.extent.width) * bytes_per_element; - constexpr uint64_t MAX_COPY_BUFFER_SIZE = 16ull * 1024ull * 1024ull; - if (row_bytes == 0 || row_bytes > MAX_COPY_BUFFER_SIZE) { - EXIT("unsupported image-buffer-image row size: 0x%016" PRIx64 "\n", row_bytes); - } - const auto rows_per_chunk = static_cast( - std::min(src_image.extent.height, MAX_COPY_BUFFER_SIZE / row_bytes)); - const auto copy_buffer_size = row_bytes * rows_per_chunk; - - auto& graphics = buffer.GetGraphics(); - auto copy_buffer_owner = std::make_unique(); - auto* copy_buffer = copy_buffer_owner.get(); - copy_buffer->usage = - vk::BufferUsageFlagBits::eTransferSrc | vk::BufferUsageFlagBits::eTransferDst; - copy_buffer->memory.property = vk::MemoryPropertyFlagBits::eDeviceLocal; - graphics.CreateBuffer(copy_buffer_size, *copy_buffer); - buffer.RetireBufferAfterFence(std::move(copy_buffer_owner)); - - auto vk_buffer = buffer.Handle(); - const auto src_layout = src_image.layout; - const auto final_layout = dst_layout; - SetImageLayout(vk_buffer, src_image, 0, 1, src_aspect, src_layout, - vk::ImageLayout::eTransferSrcOptimal); - SetImageLayout(vk_buffer, dst_image, 0, 1, dst_aspect, dst_image.layout, - vk::ImageLayout::eTransferDstOptimal); - - for (uint32_t row = 0; row < src_image.extent.height; row += rows_per_chunk) { - const auto rows = std::min(rows_per_chunk, src_image.extent.height - row); - auto copy = MakeBufferImageCopy(0, 0, src_aspect, 0, 0, {0, static_cast(row), 0}, - {src_image.extent.width, rows, 1}); - vk_buffer.copyImageToBuffer(src_image.image, vk::ImageLayout::eTransferSrcOptimal, - copy_buffer->buffer, 1, ©); - - SetBufferMemoryBarrier( - vk_buffer, copy_buffer->buffer, 0, copy_buffer_size, vk::AccessFlagBits::eTransferWrite, - vk::AccessFlagBits::eTransferRead, vk::PipelineStageFlagBits::eTransfer, - vk::PipelineStageFlagBits::eTransfer); - - copy.imageSubresource.aspectMask = dst_aspect; - vk_buffer.copyBufferToImage(copy_buffer->buffer, dst_image.image, - vk::ImageLayout::eTransferDstOptimal, 1, ©); - if (row + rows < src_image.extent.height) { - SetBufferMemoryBarrier( - vk_buffer, copy_buffer->buffer, 0, copy_buffer_size, - vk::AccessFlagBits::eTransferRead, vk::AccessFlagBits::eTransferWrite, - vk::PipelineStageFlagBits::eTransfer, vk::PipelineStageFlagBits::eTransfer); - } - } - - SetImageLayout(vk_buffer, src_image, 0, 1, src_aspect, vk::ImageLayout::eTransferSrcOptimal, - src_layout); - SetImageLayout(vk_buffer, dst_image, 0, 1, dst_aspect, vk::ImageLayout::eTransferDstOptimal, - final_layout); -} - -void BlitToSwapchain(CommandBuffer& buffer, VulkanImage& src_image, - VulkanSwapchain& dst_swapchain) { - auto vk_buffer = buffer.Handle(); - if (src_image.layout != vk::ImageLayout::eTransferSrcOptimal) { - EXIT("invalid prepared presentation image, vk_image=%p layout=%d\n", - static_cast(src_image.image), static_cast(src_image.layout)); - } - - VulkanImage swapchain_image(VulkanImageType::Unknown); - - swapchain_image.image = dst_swapchain.swapchain_images[dst_swapchain.current_index]; - swapchain_image.layout = vk::ImageLayout::eUndefined; - - SetImageLayout(vk_buffer, swapchain_image, 0, 1, vk::ImageAspectFlagBits::eColor, - vk::ImageLayout::eUndefined, vk::ImageLayout::eTransferDstOptimal); - - vk::ImageBlit region {}; - region.srcSubresource.aspectMask = vk::ImageAspectFlagBits::eColor; - region.srcSubresource.mipLevel = 0; - region.srcSubresource.baseArrayLayer = 0; - region.srcSubresource.layerCount = 1; - region.srcOffsets[0].x = 0; - region.srcOffsets[0].y = 0; - region.srcOffsets[0].z = 0; - region.srcOffsets[1].x = static_cast(src_image.extent.width); - region.srcOffsets[1].y = static_cast(src_image.extent.height); - region.srcOffsets[1].z = 1; - region.dstSubresource.aspectMask = vk::ImageAspectFlagBits::eColor; - region.dstSubresource.mipLevel = 0; - region.dstSubresource.baseArrayLayer = 0; - region.dstSubresource.layerCount = 1; - region.dstOffsets[0].x = 0; - region.dstOffsets[0].y = 0; - region.dstOffsets[0].z = 0; - region.dstOffsets[1].x = static_cast(dst_swapchain.swapchain_extent.width); - region.dstOffsets[1].y = static_cast(dst_swapchain.swapchain_extent.height); - region.dstOffsets[1].z = 1; - - vk_buffer.blitImage(src_image.image, vk::ImageLayout::eTransferSrcOptimal, - swapchain_image.image, vk::ImageLayout::eTransferDstOptimal, 1, ®ion, - vk::Filter::eLinear); -} - -void ClearColorImage(CommandBuffer& buffer, VulkanImage& image, const vk::ClearColorValue& color) { - auto vk_buffer = buffer.Handle(); - SetImageLayout(vk_buffer, image, 0, 1, vk::ImageAspectFlagBits::eColor, image.layout, - vk::ImageLayout::eTransferDstOptimal); - const vk::ImageSubresourceRange range {vk::ImageAspectFlagBits::eColor, 0, 1, 0, 1}; - vk_buffer.clearColorImage(image.image, vk::ImageLayout::eTransferDstOptimal, &color, 1, &range); - SetImageLayout(vk_buffer, image, 0, 1, vk::ImageAspectFlagBits::eColor, - vk::ImageLayout::eTransferDstOptimal, vk::ImageLayout::eTransferSrcOptimal); -} - -void UploadImage(VulkanImage& dst_image, const void* src_data, uint64_t size, uint32_t src_pitch, - vk::ImageLayout dst_layout) { - KYTY_PROFILER_FUNCTION(); - - EXIT_IF(size == 0); - - GetStagingBuffer(StagingBufferType::Texture) - .UploadToImage(dst_image, src_data, size, src_pitch, vk::ImageAspectFlagBits::eColor, - vk::ImageLayout::eUndefined, dst_layout); -} - -void DownloadImage(void* dst_data, uint64_t size, uint32_t dst_pitch, VulkanImage& src_image, - vk::ImageLayout src_layout, vk::ImageAspectFlags aspect) { - KYTY_PROFILER_FUNCTION(); - - EXIT_IF(size == 0); - - GetStagingBuffer(StagingBufferType::ReadBack) - .DownloadFromImage(dst_data, size, dst_pitch, src_image, src_layout, aspect); -} - -void DownloadImage(void* dst_data, uint64_t size, std::span regions, - VulkanImage& src_image, vk::ImageLayout src_layout) { - KYTY_PROFILER_FUNCTION(); - EXIT_IF(size == 0 || regions.empty()); - GetStagingBuffer(StagingBufferType::ReadBack) - .DownloadFromImage(dst_data, size, regions, src_image, src_layout); -} - -void ProcessDownloadedImage(uint64_t size, std::span regions, - VulkanImage& src_image, vk::ImageLayout src_layout, - const DownloadedImageConsumer& consumer) { - KYTY_PROFILER_FUNCTION(); - EXIT_IF(size == 0 || regions.empty() || !consumer); - GetStagingBuffer(StagingBufferType::ReadBack) - .ProcessDownloadedImage(size, regions, src_image, src_layout, consumer); -} - -void UploadImage(VulkanImage& image, const void* src_data, uint64_t size, - std::span regions, vk::ImageLayout dst_layout) { - EXIT_IF(size == 0 || regions.empty()); - - GetStagingBuffer(StagingBufferType::Texture) - .UploadToImage(image, src_data, size, regions, dst_layout); -} - -void CopyImageImmediate(std::span regions, VulkanImage& dst_image, - vk::ImageLayout dst_layout) { - ExecuteImmediateCommands([&](CommandBuffer& command, vk::CommandBuffer) { - CopyImage(command, regions, dst_image, dst_layout); - }); -} - -void UploadImage(DepthStencilVulkanImage& image, const void* src_data, uint64_t size, - uint32_t src_pitch, vk::ImageAspectFlags aspect) { - EXIT_IF(size == 0); - GetStagingBuffer(StagingBufferType::Texture) - .UploadToImage(image, src_data, size, src_pitch, aspect, image.layout, - vk::ImageLayout::eDepthStencilAttachmentOptimal); -} - -void UploadBuffer(StagingBufferType type, VulkanBuffer& dst_buffer, uint64_t dst_offset, - const void* src_data, uint64_t size) { - EXIT_IF(size == 0); - EXIT_IF(dst_offset > dst_buffer.buffer_size || size > dst_buffer.buffer_size - dst_offset); - GetStagingBuffer(type).UploadToBuffer(dst_buffer, src_data, size, dst_offset); -} - -void CopyBuffer(VulkanBuffer& src_buffer, VulkanBuffer& dst_buffer, uint64_t size) { - EXIT_IF(size == 0 || size > src_buffer.buffer_size || size > dst_buffer.buffer_size); - - ExecuteImmediateCommands([&](CommandBuffer&, vk::CommandBuffer vk_command) { - SetBufferMemoryBarrier(vk_command, src_buffer.buffer, 0, size, - vk::AccessFlagBits::eMemoryWrite, vk::AccessFlagBits::eTransferRead, - vk::PipelineStageFlagBits::eAllCommands, - vk::PipelineStageFlagBits::eTransfer); - SetBufferMemoryBarrier(vk_command, dst_buffer.buffer, 0, size, - vk::AccessFlagBits::eMemoryRead | vk::AccessFlagBits::eMemoryWrite, - vk::AccessFlagBits::eTransferWrite, - vk::PipelineStageFlagBits::eAllCommands, - vk::PipelineStageFlagBits::eTransfer); - const vk::BufferCopy copy_region {.srcOffset = 0, .dstOffset = 0, .size = size}; - vk_command.copyBuffer(src_buffer.buffer, dst_buffer.buffer, 1, ©_region); - SetBufferMemoryBarrier( - vk_command, dst_buffer.buffer, 0, size, vk::AccessFlagBits::eTransferWrite, - vk::AccessFlagBits::eMemoryRead | vk::AccessFlagBits::eMemoryWrite, - vk::PipelineStageFlagBits::eTransfer, vk::PipelineStageFlagBits::eAllCommands); - }); -} - -void DownloadBuffer(VulkanBuffer& src_buffer, uint64_t src_offset, void* dst_data, uint64_t size) { - auto& graphics = GetRenderContext().GetGraphics(); - EXIT_IF(size == 0); - EXIT_IF(src_offset > src_buffer.buffer_size || size > src_buffer.buffer_size - src_offset); - if (src_buffer.memory.property & vk::MemoryPropertyFlagBits::eHostVisible) { - void* mapped = nullptr; - graphics.MapMemory(src_buffer.memory, mapped); - std::memcpy(dst_data, static_cast(mapped) + src_offset, size); - graphics.UnmapMemory(src_buffer.memory); - return; - } - - VulkanBuffer readback {}; - readback.usage = vk::BufferUsageFlagBits::eTransferDst; - readback.memory.property = vk::MemoryPropertyFlagBits::eHostVisible | - vk::MemoryPropertyFlagBits::eHostCoherent | - vk::MemoryPropertyFlagBits::eHostCached; - graphics.CreateBuffer(size, readback); - - ExecuteImmediateCommands([&](CommandBuffer&, vk::CommandBuffer vk_command) { - SetBufferMemoryBarrier(vk_command, src_buffer.buffer, src_offset, size, - vk::AccessFlagBits::eMemoryWrite, vk::AccessFlagBits::eTransferRead, - vk::PipelineStageFlagBits::eAllCommands, - vk::PipelineStageFlagBits::eTransfer); - const vk::BufferCopy copy {.srcOffset = src_offset, .dstOffset = 0, .size = size}; - vk_command.copyBuffer(src_buffer.buffer, readback.buffer, 1, ©); - SetBufferMemoryBarrier(vk_command, readback.buffer, 0, size, - vk::AccessFlagBits::eTransferWrite, vk::AccessFlagBits::eHostRead, - vk::PipelineStageFlagBits::eTransfer, - vk::PipelineStageFlagBits::eHost); - }); - - void* mapped = nullptr; - graphics.MapMemory(readback.memory, mapped); - std::memcpy(dst_data, mapped, size); - graphics.UnmapMemory(readback.memory); - graphics.DeleteBuffer(readback); -} - -} // namespace Libs::Graphics::Transfer diff --git a/src/graphics/host_gpu/transfer.h b/src/graphics/host_gpu/transfer.h deleted file mode 100644 index a1e174da..00000000 --- a/src/graphics/host_gpu/transfer.h +++ /dev/null @@ -1,165 +0,0 @@ -#ifndef EMULATOR_SRC_GRAPHICS_HOST_GPU_TRANSFER_H_ -#define EMULATOR_SRC_GRAPHICS_HOST_GPU_TRANSFER_H_ - -#include "common/abi.h" -#include "common/common.h" -#include "graphics/host_gpu/gpuTiler.h" -#include "graphics/host_gpu/vulkanCommon.h" - -#include -#include -#include -#include - -namespace Libs::Graphics { - -class CommandBuffer; -struct GraphicContext; -struct VulkanBuffer; -struct VulkanImage; -struct DepthStencilVulkanImage; -struct VulkanSwapchain; - -struct BufferImageCopy { - uint32_t offset; - uint32_t pitch; - uint32_t dst_level; - uint32_t width; - uint32_t height; - uint32_t copy_height = 0; - uint32_t dst_layer = 0; - int dst_x; - int dst_y; - int dst_z = 0; - vk::ImageAspectFlags aspect = vk::ImageAspectFlagBits::eColor; -}; - -struct ImageBufferCopy { - uint32_t offset; - uint32_t pitch; - uint32_t src_level; - uint32_t width; - uint32_t height; - uint32_t copy_height = 0; - uint32_t src_layer = 0; - int src_x; - int src_y; - int src_z = 0; - vk::ImageAspectFlags aspect = vk::ImageAspectFlagBits::eColor; -}; - -struct ImageImageCopy { - explicit ImageImageCopy(VulkanImage& source): src_image(source) {} - - VulkanImage& src_image; - uint32_t src_level = 0; - uint32_t dst_level = 0; - uint32_t width = 0; - uint32_t height = 0; - uint32_t src_layer = 0; - uint32_t dst_layer = 0; - vk::ImageAspectFlags src_aspect = vk::ImageAspectFlagBits::eColor; - vk::ImageAspectFlags dst_aspect = vk::ImageAspectFlagBits::eColor; - int src_x = 0; - int src_y = 0; - int src_z = 0; - int dst_x = 0; - int dst_y = 0; - int dst_z = 0; -}; - -namespace Transfer { - -[[nodiscard]] std::vector -MakeLayeredImageBufferCopies(uint32_t layers, uint64_t slice_size, uint32_t pitch, uint32_t width, - uint32_t height, - vk::ImageAspectFlags aspect = vk::ImageAspectFlagBits::eColor); - -enum class StagingBufferType { Texture, Vertex, ReadBack }; - -using DownloadedImageConsumer = std::function)>; - -class ScratchBuffer { -public: - explicit ScratchBuffer(uint64_t size); - ~ScratchBuffer(); - KYTY_CLASS_NO_COPY(ScratchBuffer); - - [[nodiscard]] void* Data() const { return m_data; } - -private: - void* m_data = nullptr; -}; - -void CopyImage(CommandBuffer& buffer, std::span regions, - VulkanImage& dst_image, vk::ImageLayout dst_layout); -void CopyImageViaBuffer(CommandBuffer& buffer, VulkanImage& src_image, - vk::ImageAspectFlags src_aspect, VulkanImage& dst_image, - vk::ImageAspectFlags dst_aspect, uint32_t bytes_per_element, - vk::ImageLayout dst_layout); -void BlitToSwapchain(CommandBuffer& buffer, VulkanImage& src_image, VulkanSwapchain& dst_swapchain); -void ClearColorImage(CommandBuffer& buffer, VulkanImage& image, const vk::ClearColorValue& color); -void UploadImage(VulkanImage& dst_image, const void* src_data, uint64_t size, uint32_t src_pitch, - vk::ImageLayout dst_layout); -void UploadImage(DepthStencilVulkanImage& dst_image, const void* src_data, uint64_t size, - uint32_t src_pitch, vk::ImageAspectFlags aspect); -void UploadImage(VulkanImage& dst_image, const void* src_data, uint64_t size, - std::span regions, vk::ImageLayout dst_layout); -void UploadTiledImage(VulkanImage& dst_image, const void* tiled_data, uint64_t tiled_size, - uint64_t linear_size, std::span infos, - std::span regions, vk::ImageLayout dst_layout); -void CopyImageImmediate(std::span regions, VulkanImage& dst_image, - vk::ImageLayout dst_layout); -void DownloadImage(void* dst_data, uint64_t size, uint32_t dst_pitch, VulkanImage& src_image, - vk::ImageLayout src_layout, - vk::ImageAspectFlags aspect = vk::ImageAspectFlagBits::eColor); -void DownloadImage(void* dst_data, uint64_t size, std::span regions, - VulkanImage& src_image, vk::ImageLayout src_layout); -// Invokes consumer synchronously after the image copy fence while the readback staging buffer is -// reserved. The supplied span is valid only for the call; the consumer must not retain it or -// re-enter Transfer. -void ProcessDownloadedImage(uint64_t size, std::span regions, - VulkanImage& src_image, vk::ImageLayout src_layout, - const DownloadedImageConsumer& consumer); -void DownloadTiledImage(void* tiled_data, uint64_t tiled_size, uint64_t linear_size, - std::span infos, - std::span regions, VulkanImage& src_image, - vk::ImageLayout src_layout); -void UploadBuffer(StagingBufferType type, VulkanBuffer& dst_buffer, uint64_t dst_offset, - const void* src_data, uint64_t size); -void CopyBuffer(VulkanBuffer& src_buffer, VulkanBuffer& dst_buffer, uint64_t size); -void DownloadBuffer(VulkanBuffer& src_buffer, uint64_t src_offset, void* dst_data, uint64_t size); -void ReleaseCachedResources(); -bool GuestBufferIsTiled(uint64_t vaddr, uint64_t size); -bool IsBlockCompressedFormat(vk::Format format); -uint32_t BlockCompressedBytesPerBlock(vk::Format format); - -void WaitForQueueIdle(); - -inline std::pair MipmapAtlasOffset(uint32_t lod, uint32_t width, uint32_t height) { - uint32_t mip_width = width; - uint32_t mip_height = height; - int mip_x = 0; - int mip_y = 0; - - for (uint32_t i = 0; i < 16; i++) { - if (i == lod) { - return {mip_x, mip_y}; - } - - bool odd = ((i & 1u) != 0); - mip_x += static_cast(odd ? mip_width : 0); - mip_y += static_cast(odd ? 0 : mip_height); - - mip_width >>= (mip_width > 1 ? 1u : 0u); - mip_height >>= (mip_height > 1 ? 1u : 0u); - } - - return {mip_x, mip_y}; -} - -} // namespace Transfer - -} // namespace Libs::Graphics - -#endif // EMULATOR_SRC_GRAPHICS_HOST_GPU_TRANSFER_H_ diff --git a/src/graphics/host_gpu/vma.cpp b/src/graphics/host_gpu/vma.cpp index 4da78fdd..32c0eca3 100644 --- a/src/graphics/host_gpu/vma.cpp +++ b/src/graphics/host_gpu/vma.cpp @@ -20,6 +20,7 @@ #include "graphics/host_gpu/graphicContext.h" #include "graphics/host_gpu/vma.h" +#include #include #include @@ -111,6 +112,58 @@ void GraphicContext::LogMemoryBudget() const { } } +uint64_t GraphicContext::GetDeviceMemoryUsage() const { + if (!CanReportMemoryUsage() || allocator == nullptr) { + return 0; + } + VmaBudget budgets[VK_MAX_MEMORY_HEAPS] {}; + vmaGetHeapBudgets(allocator, budgets); + const bool discrete = + physical_device_properties.deviceType == vk::PhysicalDeviceType::eDiscreteGpu; + uint64_t usage = 0; + for (uint32_t heap = 0; heap < physical_device_memory_properties.memoryHeapCount; heap++) { + const bool device_local = static_cast( + physical_device_memory_properties.memoryHeaps[heap].flags & + vk::MemoryHeapFlagBits::eDeviceLocal); + if (!discrete || device_local) { + usage += budgets[heap].usage; + } + } + return usage; +} + +uint64_t GraphicContext::GetTotalMemoryBudget() const { + if (allocator == nullptr) { + return 0; + } + VmaBudget budgets[VK_MAX_MEMORY_HEAPS] {}; + vmaGetHeapBudgets(allocator, budgets); + const bool discrete = + physical_device_properties.deviceType == vk::PhysicalDeviceType::eDiscreteGpu; + uint64_t budget = 0; + uint64_t local = 0; + uint64_t usage = 0; + for (uint32_t heap = 0; heap < physical_device_memory_properties.memoryHeapCount; heap++) { + const auto& properties = physical_device_memory_properties.memoryHeaps[heap]; + const bool device_local = + static_cast(properties.flags & vk::MemoryHeapFlagBits::eDeviceLocal); + if (device_local) { + local += properties.size; + } + if (!discrete || device_local) { + budget += CanReportMemoryUsage() ? budgets[heap].budget : properties.size; + usage += CanReportMemoryUsage() ? budgets[heap].usage : 0; + } + } + if (discrete) { + return budget - std::min(budget / 8, 1024ull * 1024 * 1024); + } + constexpr uint64_t system_reserve = 8ull * 1024 * 1024 * 1024; + const auto available = budget > usage ? budget - usage : uint64_t {0}; + return std::max(local, + available > system_reserve ? available - system_reserve : uint64_t {0}); +} + void GraphicContext::CreateBuffer(uint64_t size, VulkanBuffer& buffer) { KYTY_PROFILER_FUNCTION(); EXIT_IF(allocator == nullptr || buffer.buffer != nullptr || diff --git a/src/graphics/presentation/displayBuffer.h b/src/graphics/presentation/displayBuffer.h deleted file mode 100644 index bd7726ed..00000000 --- a/src/graphics/presentation/displayBuffer.h +++ /dev/null @@ -1,29 +0,0 @@ -#ifndef EMULATOR_INCLUDE_EMULATOR_GRAPHICS_PRESENTATION_DISPLAYBUFFER_H_ -#define EMULATOR_INCLUDE_EMULATOR_GRAPHICS_PRESENTATION_DISPLAYBUFFER_H_ - -#include "common/common.h" - -namespace Libs::Graphics { -class CommandBuffer; -struct VideoOutVulkanImage; -} // namespace Libs::Graphics - -namespace Libs::Presentation { - -struct DisplayBufferImage { - Graphics::VideoOutVulkanImage* image = nullptr; - uint32_t index = static_cast(-1); - uint64_t size = 0; - uint64_t pitch = 0; -}; - -DisplayBufferImage DisplayBufferFind(uint64_t addr, bool render_target = false); -int DisplayBufferSubmitFlipFromGpu(Graphics::CommandBuffer& buffer, int handle, int index, - int flip_mode, int64_t flip_arg, uint64_t& request_id); -uint64_t DisplayBufferPrepareNextFlipOnGpu(Graphics::CommandBuffer& buffer); -void DisplayBufferCompleteFlipFromGpu(uint64_t request_id); -void DisplayBufferWaitForFlipQueueSlot(); - -} // namespace Libs::Presentation - -#endif // EMULATOR_INCLUDE_EMULATOR_GRAPHICS_PRESENTATION_DISPLAYBUFFER_H_ diff --git a/src/graphics/presentation/presenter.h b/src/graphics/presentation/presenter.h new file mode 100644 index 00000000..10f54fb8 --- /dev/null +++ b/src/graphics/presentation/presenter.h @@ -0,0 +1,38 @@ +#ifndef EMULATOR_SRC_GRAPHICS_PRESENTATION_PRESENTER_H_ +#define EMULATOR_SRC_GRAPHICS_PRESENTATION_PRESENTER_H_ + +#include "common/common.h" + +#include + +namespace Libs::Graphics { + +class CommandBuffer; +class RenderContext; +struct ImageInfo; +struct WindowContext; + +class Presenter final { +public: + struct Frame; + + explicit Presenter(WindowContext& window); + ~Presenter(); + KYTY_CLASS_NO_COPY(Presenter); + + [[nodiscard]] Frame& PrepareFrame(CommandBuffer& command, const ImageInfo& info); + [[nodiscard]] Frame& PrepareBlankFrame(uint32_t width, uint32_t height, bool opaque, + CommandBuffer* producer = nullptr); + [[nodiscard]] Frame* PrepareLastFrame(); + [[nodiscard]] bool IsGuestPaused() const noexcept; + [[nodiscard]] RenderContext& Renderer() const noexcept; + void Present(Frame& frame, bool reuse = false); + void Discard(Frame& frame); + +private: + struct Impl; + std::unique_ptr m_impl; +}; +} // namespace Libs::Graphics + +#endif // EMULATOR_SRC_GRAPHICS_PRESENTATION_PRESENTER_H_ diff --git a/src/graphics/presentation/videoOut.cpp b/src/graphics/presentation/videoOut.cpp index 0e2a8bbc..03208e11 100644 --- a/src/graphics/presentation/videoOut.cpp +++ b/src/graphics/presentation/videoOut.cpp @@ -12,18 +12,18 @@ #include "graphics/guest_gpu/gpu_defs.h" #include "graphics/guest_gpu/graphicsRun.h" #include "graphics/guest_gpu/tile.h" +#include "graphics/host_gpu/renderer/imageInfo.h" #include "graphics/host_gpu/renderer/render.h" #include "graphics/host_gpu/renderer/renderContext.h" -#include "graphics/host_gpu/renderer/textureCache.h" -#include "graphics/presentation/displayBuffer.h" -#include "graphics/presentation/window.h" +#include "graphics/presentation/presenter.h" #include "kernel/pthread.h" #include "libs/errno.h" #include "libs/libs.h" #include -#include +#include #include +#include #include namespace Libs::Graphics { @@ -66,6 +66,26 @@ enum class VideoOutEventKind : uintptr_t { enum class FlipRequestSource { Cpu, GpuEop }; +struct VideoOutEventState; + +struct VideoOutEventRegistration { + EventQueue::KernelEqueue handle = EventQueue::KERNEL_EQUEUE_INVALID; + std::shared_ptr state; + uint64_t generation = 0; + VideoOutEventKind kind = VideoOutEventKind::Flip; +}; + +using VideoOutEventRegistrationRef = std::shared_ptr; +using VideoOutEventQueues = std::vector; + +struct VideoOutEventState { + Common::Mutex mutex; + VideoOutEventQueues flip; + VideoOutEventQueues pre_vblank; + VideoOutEventQueues vblank; + VideoOutEventQueues output_mode; +}; + struct VideoOutBufferAttribute2 { uint32_t reserved0; uint32_t tiling_mode; @@ -131,140 +151,143 @@ struct VideoOutBuffers { const void* reserved[2]; }; -struct VideoOutBufferSet { - int start_index = 0; - int num = 0; - int set_id = 0; +struct VideoOutBuffer { + int group_index = -1; + uint64_t data_address = 0; + uint64_t metadata_address = 0; + + [[nodiscard]] bool Occupied() const noexcept { return group_index >= 0; } }; -struct VideoOutBufferInfo { - const void* buffer = nullptr; - Graphics::VideoOutVulkanImage* buffer_vulkan = nullptr; - uint64_t buffer_size = 0; - uint64_t buffer_pitch = 0; - int set_id = 0; +struct BufferAttributeGroup { + VideoOutBufferAttribute2 attribute {}; + int category = VIDEO_OUT_BUFFER_ATTRIBUTE_CATEGORY_UNCOMPRESSED; + bool occupied = false; + + [[nodiscard]] Graphics::ImageInfo ImageInfo(const VideoOutBuffer& buffer) const; }; struct VideoOutConfig { - Common::Mutex mutex; - uint32_t width = 0; - uint32_t height = 0; - bool opened = false; - bool closing = false; - bool unregistering[16] = {}; - int flip_rate = 0; - std::vector flip_eqs; - std::vector pre_vblank_eqs; - std::vector vblank_eqs; - std::vector output_mode_eqs; - uint64_t output_mode = VIDEO_OUT_OUTPUT_MODE_DEFAULT; - float gamma = 1.0f; - VideoOutFlipStatus flip_status; - VideoOutVblankStatus pre_vblank_status; - VideoOutVblankStatus vblank_status; - VideoOutBufferInfo buffers[16]; - std::vector buffers_sets; + Common::Mutex mutex; + Common::CondVar vblank_cond; + std::shared_ptr events = std::make_shared(); + uint32_t width = 0; + uint32_t height = 0; + uint64_t generation = 0; + bool opened = false; + bool closing = false; + int flip_rate = 0; + uint64_t output_mode = VIDEO_OUT_OUTPUT_MODE_DEFAULT; + float gamma = 1.0f; + VideoOutFlipStatus flip_status; + VideoOutVblankStatus pre_vblank_status; + VideoOutVblankStatus vblank_status; + std::array buffers; + std::array groups; }; class FlipQueue { public: - FlipQueue() { EXIT_NOT_IMPLEMENTED(!Common::Thread::IsMainThread()); } - ~FlipQueue() { KYTY_NOT_IMPLEMENTED; } + explicit FlipQueue(Graphics::Presenter& presenter): m_presenter(presenter) { + EXIT_NOT_IMPLEMENTED(!Common::Thread::IsMainThread()); + } + ~FlipQueue(); KYTY_CLASS_NO_COPY(FlipQueue); - bool Reserve(VideoOutConfig& cfg, int index, int64_t flip_arg, FlipRequestSource source, - uint64_t& request_id); - void Prepare(uint64_t request_id, Graphics::CommandBuffer& buffer); - uint64_t PrepareNextCpu(Graphics::CommandBuffer& buffer); - void Complete(uint64_t request_id); - void WaitForSubmitSlot(); - bool Flip(uint32_t micros); - bool HasPending(VideoOutConfig& cfg, int start_index, int count); - void GetFlipStatus(VideoOutConfig& cfg, VideoOutFlipStatus& out); - void Wait(VideoOutConfig& cfg, int index); + bool Reserve(VideoOutConfig& cfg, int index, int64_t flip_arg, FlipRequestSource source, + uint64_t& request_id); + void Cancel(VideoOutConfig& cfg); + void Prepare(uint64_t request_id, Graphics::CommandBuffer& buffer); + void Complete(uint64_t request_id); + void WaitForSubmitSlot(); + bool Flip(uint32_t micros); + void GetFlipStatus(VideoOutConfig& cfg, VideoOutFlipStatus& out); + void Wait(VideoOutConfig& cfg, int index); private: enum class RequestState { Reserved, Recording, Ready, Presenting }; struct Request { - uint64_t id; - VideoOutConfig* cfg; - int index; - int64_t flip_arg; - uint64_t submit_ptc; - FlipRequestSource source; - RequestState state; - Graphics::PreparedFrame* frame; + uint64_t id; + VideoOutConfig* cfg; + uint64_t generation; + int index; + int64_t flip_arg; + uint64_t submit_ptc; + FlipRequestSource source; + RequestState state; + Graphics::Presenter::Frame* frame; }; - Common::Mutex m_mutex; - Common::CondVar m_submit_cond_var; - Common::CondVar m_submit_slot_cond_var; - Common::CondVar m_done_cond_var; - std::list m_requests; - std::list m_cpu_requests; - bool m_processing = false; - uint64_t m_next_request_id = 1; + Graphics::Presenter& m_presenter; + Common::Mutex m_mutex; + Common::CondVar m_submit_cond_var; + Common::CondVar m_submit_slot_cond_var; + Common::CondVar m_done_cond_var; + std::list m_requests; + std::list m_cpu_requests; + std::list m_cancelled_requests; + bool m_processing = false; + uint64_t m_next_request_id = 1; }; -class VideoOutContext { +struct VideoOutDriver::Impl { public: static constexpr int VIDEO_OUT_NUM_MAX = 2; - VideoOutContext() { EXIT_NOT_IMPLEMENTED(!Common::Thread::IsMainThread()); } - ~VideoOutContext() { KYTY_NOT_IMPLEMENTED; } - - KYTY_CLASS_NO_COPY(VideoOutContext); + Impl(uint32_t width, uint32_t height, Graphics::Presenter& presenter) + : m_renderer(presenter.Renderer()), m_presenter(presenter), m_flip_queue(presenter) { + EXIT_NOT_IMPLEMENTED(!Common::Thread::IsMainThread()); + Init(width, height); + m_present_thread = std::jthread([this](std::stop_token token) { PresentThread(token); }); + } + ~Impl(); + KYTY_CLASS_NO_COPY(Impl); int Open(); - void Close(int handle); + bool Close(int handle); VideoOutConfig* Get(int handle); + VideoOutConfig* Get(int handle, uint64_t& generation); bool IsOpened(int handle); - Presentation::DisplayBufferImage FindImage(const void* buffer, bool render_target); - void Init(uint32_t width, uint32_t height); - FlipQueue& GetFlipQueue() { return m_flip_queue; } + Graphics::RenderContext& Renderer() const noexcept { return m_renderer; } void VblankBegin(); void VblankEnd(); + void PresentThread(std::stop_token token); private: - Common::Mutex m_mutex; - VideoOutConfig m_video_out_ctx[VIDEO_OUT_NUM_MAX]; - FlipQueue m_flip_queue; + Common::Mutex m_mutex; + VideoOutConfig m_video_out_ctx[VIDEO_OUT_NUM_MAX]; + Graphics::RenderContext& m_renderer; + Graphics::Presenter& m_presenter; + FlipQueue m_flip_queue; + std::jthread m_present_thread; }; -static VideoOutContext* g_video_out_context = nullptr; +static std::unique_ptr g_video_out_driver; -using VideoOutEventQueues = std::vector; +static VideoOutDriver::Impl& DriverState() { + EXIT_IF(g_video_out_driver == nullptr); + return g_video_out_driver->State(); +} static uintptr_t VideoOutEventId(VideoOutEventKind kind) { return static_cast(kind); } -static VideoOutEventKind GetVideoOutEventKind(uintptr_t event_id) { - switch (event_id) { - case VIDEO_OUT_EVENT_FLIP: return VideoOutEventKind::Flip; - case VIDEO_OUT_EVENT_VBLANK: return VideoOutEventKind::Vblank; - case VIDEO_OUT_EVENT_PRE_VBLANK_START: return VideoOutEventKind::PreVblankStart; - case VIDEO_OUT_EVENT_SET_MODE: return VideoOutEventKind::OutputMode; - default: EXIT("unsupported video-out event id=%" PRIuPTR "\n", event_id); - } - return VideoOutEventKind::Flip; -} - -static VideoOutEventQueues& VideoOutEventQueuesFor(VideoOutConfig& video_out, - VideoOutEventKind kind) { +static VideoOutEventQueues& VideoOutEventQueuesFor(VideoOutEventState& state, + VideoOutEventKind kind) { switch (kind) { - case VideoOutEventKind::Flip: return video_out.flip_eqs; - case VideoOutEventKind::Vblank: return video_out.vblank_eqs; - case VideoOutEventKind::PreVblankStart: return video_out.pre_vblank_eqs; - case VideoOutEventKind::OutputMode: return video_out.output_mode_eqs; + case VideoOutEventKind::Flip: return state.flip; + case VideoOutEventKind::Vblank: return state.vblank; + case VideoOutEventKind::PreVblankStart: return state.pre_vblank; + case VideoOutEventKind::OutputMode: return state.output_mode; } EXIT("unsupported video-out event kind\n"); - return video_out.flip_eqs; + return state.flip; } static intptr_t MakeVideoOutEventData(intptr_t current_data, void* trigger_data) { @@ -305,64 +328,84 @@ static void TriggerVideoOutEvent(EventQueue::KernelEqueueEvent* event, void* tri static void RemoveVideoOutEventQueue(EventQueue::KernelEqueue eq, EventQueue::KernelEqueueEvent* event) { - EXIT_IF(event == nullptr); - EXIT_IF(event->filter.data == nullptr); - EXIT_NOT_IMPLEMENTED(event->event.filter != EventQueue::KERNEL_EVFILT_VIDEO_OUT); + if (event == nullptr || event->filter.data == nullptr) { + return; + } - auto* video_out = static_cast(event->filter.data); - auto& queues = VideoOutEventQueuesFor(*video_out, GetVideoOutEventKind(event->event.ident)); - Common::LockGuard lock(video_out->mutex); - EXIT_IF(queues.empty()); - const auto entry = std::find(queues.begin(), queues.end(), eq); - EXIT_NOT_IMPLEMENTED(entry == queues.end()); - *entry = nullptr; + auto* registration = static_cast(event->filter.data); + auto state = registration->state; + if (registration->handle != eq || !state) { + return; + } + auto& queues = VideoOutEventQueuesFor(*state, registration->kind); + Common::LockGuard lock(state->mutex); + const auto entry = + std::find_if(queues.begin(), queues.end(), [registration](const auto& candidate) { + return candidate.get() == registration; + }); + if (entry != queues.end()) { + queues.erase(entry); + } } -static void TriggerVideoOutEventsLocked(const VideoOutEventQueues& queues, VideoOutEventKind kind, - void* trigger_data) { - for (auto eq: queues) { - if (eq == nullptr) { +static void TriggerVideoOutEvents(VideoOutConfig& video_out, VideoOutEventKind kind, + void* trigger_data) { + VideoOutEventQueues queues; + { + Common::LockGuard lock(video_out.events->mutex); + queues = VideoOutEventQueuesFor(*video_out.events, kind); + } + for (const auto& registration: queues) { + if (!registration || registration->generation != video_out.generation) { continue; } const auto result = EventQueue::KernelTriggerEvent( - eq, VideoOutEventId(kind), EventQueue::KERNEL_EVFILT_VIDEO_OUT, trigger_data); - EXIT_NOT_IMPLEMENTED(result != OK); + registration->handle, VideoOutEventId(kind), EventQueue::KERNEL_EVFILT_VIDEO_OUT, + trigger_data); + EXIT_NOT_IMPLEMENTED(result != OK && result != LibKernel::KERNEL_ERROR_EBADF && + result != LibKernel::KERNEL_ERROR_ENOENT); } } -static void DeleteVideoOutEventsLocked(VideoOutEventQueues& queues, VideoOutEventKind kind) { - for (auto eq: queues) { - if (eq == nullptr) { +static void DeleteVideoOutEvents(const VideoOutEventQueues& queues, VideoOutEventKind kind) { + for (const auto& registration: queues) { + if (!registration) { continue; } - const auto result = EventQueue::KernelDeleteEvent(eq, VideoOutEventId(kind), - EventQueue::KERNEL_EVFILT_VIDEO_OUT); - EXIT_NOT_IMPLEMENTED(result != OK); + const auto result = + EventQueue::KernelDeleteEvent(registration->handle, VideoOutEventId(kind), + EventQueue::KERNEL_EVFILT_VIDEO_OUT); + EXIT_NOT_IMPLEMENTED(result != OK && result != LibKernel::KERNEL_ERROR_EBADF && + result != LibKernel::KERNEL_ERROR_ENOENT); } - queues.clear(); } static int RegisterVideoOutEvent(int handle, EventQueue::KernelEqueue eq, VideoOutEventKind kind, void* udata) { - EXIT_IF(g_video_out_context == nullptr); - - auto* video_out = g_video_out_context->Get(handle); + uint64_t generation = 0; + auto* video_out = DriverState().Get(handle, generation); if (video_out == nullptr) { return VIDEO_OUT_ERROR_INVALID_HANDLE; } Common::LockGuard lock(video_out->mutex); + if (!video_out->opened || video_out->closing || video_out->generation != generation) { + return VIDEO_OUT_ERROR_INVALID_HANDLE; + } if (kind == VideoOutEventKind::OutputMode) { LOGF("\t eq = 0x%016" PRIx64 "\n" "\t handle = %d\n" "\t udata = 0x%016" PRIx64 "\n", - reinterpret_cast(eq), handle, reinterpret_cast(udata)); + static_cast(eq), handle, reinterpret_cast(udata)); } - if (eq == nullptr) { + if (eq == EventQueue::KERNEL_EQUEUE_INVALID) { return VIDEO_OUT_ERROR_INVALID_EVENT_QUEUE; } - auto& queues = VideoOutEventQueuesFor(*video_out, kind); - const bool add_queue = std::find(queues.begin(), queues.end(), eq) == queues.end(); + if (!EventQueue::KernelPinEqueue(eq)) { + return VIDEO_OUT_ERROR_INVALID_EVENT_QUEUE; + } + auto event_state = video_out->events; + auto& queues = VideoOutEventQueuesFor(*event_state, kind); const bool initially_triggered = kind == VideoOutEventKind::OutputMode; void* const initial_trigger_data = initially_triggered ? reinterpret_cast(video_out->output_mode) : nullptr; @@ -377,59 +420,64 @@ static int RegisterVideoOutEvent(int handle, EventQueue::KernelEqueue eq, VideoO event.filter.delete_event_func = RemoveVideoOutEventQueue; event.filter.reset_func = ResetVideoOutEvent; event.filter.trigger_func = TriggerVideoOutEvent; - event.filter.data = video_out; - const int result = EventQueue::KernelAddEvent(eq, event); - if (result == OK && add_queue) { - queues.push_back(eq); + VideoOutEventRegistrationRef registration; + bool add_queue = false; + { + Common::LockGuard event_lock(event_state->mutex); + const auto existing = std::find_if(queues.begin(), queues.end(), [&](const auto& candidate) { + return candidate->handle == eq && candidate->generation == generation; + }); + if (existing != queues.end()) { + registration = *existing; + } else { + registration = std::make_shared( + VideoOutEventRegistration {.handle = eq, + .state = event_state, + .generation = generation, + .kind = kind}); + queues.push_back(registration); + add_queue = true; + } } - return result; + event.filter.data = registration.get(); + event.filter.owner = registration; + const int result = EventQueue::KernelAddEvent(eq, event); + if (result != OK && add_queue) { + Common::LockGuard event_lock(event_state->mutex); + const auto added = std::find(queues.begin(), queues.end(), registration); + if (added != queues.end()) { + queues.erase(added); + } + } + return result == LibKernel::KERNEL_ERROR_EBADF ? VIDEO_OUT_ERROR_INVALID_EVENT_QUEUE : result; } static int DeleteVideoOutEvent(int handle, EventQueue::KernelEqueue eq, VideoOutEventKind kind) { - EXIT_IF(g_video_out_context == nullptr); - - if (g_video_out_context->Get(handle) == nullptr) { + uint64_t generation = 0; + auto* video_out = DriverState().Get(handle, generation); + if (video_out == nullptr) { return VIDEO_OUT_ERROR_INVALID_HANDLE; } - if (eq == nullptr) { + Common::LockGuard lock(video_out->mutex); + if (!video_out->opened || video_out->closing || video_out->generation != generation) { + return VIDEO_OUT_ERROR_INVALID_HANDLE; + } + if (!EventQueue::KernelPinEqueue(eq)) { return VIDEO_OUT_ERROR_INVALID_EVENT_QUEUE; } - return EventQueue::KernelDeleteEvent(eq, VideoOutEventId(kind), - EventQueue::KERNEL_EVFILT_VIDEO_OUT); + const int result = EventQueue::KernelDeleteEvent(eq, VideoOutEventId(kind), + EventQueue::KERNEL_EVFILT_VIDEO_OUT); + if (result == LibKernel::KERNEL_ERROR_EBADF) { + return VIDEO_OUT_ERROR_INVALID_EVENT_QUEUE; + } + return result == LibKernel::KERNEL_ERROR_ENOENT ? OK : result; } -static void WaitForNextVblank() { - static uint64_t next_vblank_ticks = 0; - - const uint64_t frequency = Common::Timer::QueryPerformanceFrequency(); - const uint64_t period = frequency / Config::GetVblankFrequency(); - uint64_t now = Common::Timer::QueryPerformanceCounter(); - - if (next_vblank_ticks == 0) { - next_vblank_ticks = now; +static bool IsFlipDueLocked(const VideoOutConfig& cfg, uint64_t generation) { + if (!cfg.opened || cfg.closing || cfg.generation != generation) { + return false; } - - if (now < next_vblank_ticks) { - const uint64_t wait_ticks = next_vblank_ticks - now; - const uint64_t wait_micros = (wait_ticks * 1000000u) / frequency; - - if (wait_micros > 0) { - Common::Thread::SleepMicro( - static_cast(std::min(wait_micros, UINT32_MAX))); - } - - now = Common::Timer::QueryPerformanceCounter(); - } - - do { - next_vblank_ticks += period; - } while (next_vblank_ticks <= now); -} - -static bool IsFlipDue(VideoOutConfig& cfg) { - Common::LockGuard lock(cfg.mutex); - const int interval = cfg.flip_rate + 1; return interval <= 1 || (cfg.vblank_status.count % static_cast(interval)) == 0; @@ -447,11 +495,9 @@ static bool IsValidFlipMode(int mode) { return mode >= VIDEO_OUT_FLIP_MODE_VSYNC && mode <= VIDEO_OUT_FLIP_MODE_VSYNC_MULTI; } -static int ReserveFlipRequest(int handle, int index, int flip_mode, int64_t flip_arg, - FlipRequestSource source, uint64_t& request_id) { - EXIT_IF(g_video_out_context == nullptr); - - auto* video_out = g_video_out_context->Get(handle); +static int ReserveFlipRequest(VideoOutDriver::Impl& driver, int handle, int index, int flip_mode, + int64_t flip_arg, FlipRequestSource source, uint64_t& request_id) { + auto* video_out = driver.Get(handle); if (video_out == nullptr) { return VIDEO_OUT_ERROR_INVALID_HANDLE; } @@ -464,27 +510,26 @@ static int ReserveFlipRequest(int handle, int index, int flip_mode, int64_t flip Common::LockGuard lock(video_out->mutex); if (video_out->closing || - (!IsSpecialBufferIndex(index) && - (video_out->unregistering[index] || video_out->buffers[index].buffer_vulkan == nullptr))) { + (!IsSpecialBufferIndex(index) && !video_out->buffers[index].Occupied())) { return VIDEO_OUT_ERROR_INVALID_INDEX; } - if (!g_video_out_context->GetFlipQueue().Reserve(*video_out, index, flip_arg, source, - request_id)) { + if (!driver.GetFlipQueue().Reserve(*video_out, index, flip_arg, source, request_id)) { return VIDEO_OUT_ERROR_FLIP_QUEUE_FULL; } return OK; } -static Graphics::VideoOutInfo MakeVideoOutInfo(const VideoOutBufferAttribute2& attribute, - uint64_t address, uint64_t metadata_address, - Graphics::VideoOutCompression compression) { +Graphics::ImageInfo BufferAttributeGroup::ImageInfo(const VideoOutBuffer& buffer) const { + const auto compression = Graphics::ClassifyVideoOutCompression( + category == VIDEO_OUT_BUFFER_ATTRIBUTE_CATEGORY_COMPRESSED, buffer.metadata_address, + attribute.dcc_control, attribute.dcc_cb_register_clear_color); if (attribute.reserved0 != 0 || attribute.aspect_ratio != 0 || attribute.width == 0 || attribute.height == 0 || attribute.width > 16384 || attribute.height > 16384 || attribute.pitch_in_pixel != 0 || (attribute.option != 0 && attribute.option != VIDEO_OUT_BUFFER_ATTRIBUTE_OPTION_STRICT_COLORIMETRY) || attribute.tiling_mode != 0 || attribute.pad0 != 0 || attribute.reserved1[0] != 0 || - attribute.reserved1[1] != 0 || attribute.reserved1[2] != 0 || address == 0 || + attribute.reserved1[1] != 0 || attribute.reserved1[2] != 0 || buffer.data_address == 0 || compression == Graphics::VideoOutCompression::Unsupported) { EXIT("unsupported or invalid video-out surface attributes\n"); } @@ -499,42 +544,73 @@ static Graphics::VideoOutInfo MakeVideoOutInfo(const VideoOutBufferAttribute2& a Graphics::TileSizeAlign total {}; Graphics::TileGetTextureTotalSize(pixel_format.guest_format, attribute.width, attribute.height, 1, pitch, 1, tile_mode, false, total); - if (total.size == 0 || total.align != 65536 || (address & (total.align - 1u)) != 0) { + if (total.size == 0 || total.align != 65536 || + (buffer.data_address & (total.align - 1u)) != 0) { EXIT("invalid video-out surface footprint or alignment\n"); } - Graphics::VideoOutInfo info {}; - info.address = address; - info.size = total.size; - info.metadata_address = metadata_address; - info.format = pixel_format.format; - info.guest_format = pixel_format.guest_format; - info.width = attribute.width; - info.height = attribute.height; - info.pitch = pitch; - info.bytes_per_element = pixel_format.bytes_per_element; - info.tile_mode = tile_mode; - info.dcc_control = attribute.dcc_control; - info.compression = compression; - info.bgra16 = pixel_format.bgra16; + Graphics::ImageInfo info {}; + info.data = {buffer.data_address, total.size}; + info.pixel_format = pixel_format.format; + info.guest_format = pixel_format.guest_format; + info.type = Graphics::Prospero::ImageType::kColor2D; + info.extent = {attribute.width, attribute.height, 1}; + info.resources = {1, 1}; + info.pitch = pitch; + info.bytes_per_block = pixel_format.bytes_per_element; + info.samples = 1; + info.tile_mode = tile_mode; + info.bgra16 = pixel_format.bgra16; + info.mip_layout[0] = {0, total.size, pitch, attribute.height}; + if (compression != Graphics::VideoOutCompression::Uncompressed) { + info.metadata.range = {buffer.metadata_address, 0}; + info.metadata.kind = Graphics::ImageMetadataKind::Dcc; + info.metadata.control = attribute.dcc_control; + info.metadata.compression = compression; + } + Graphics::ImageOps::Validate(info); + if (!Graphics::IsSupportedVideoOutFormat(info)) { + EXIT("unsupported normalized video-out format\n"); + } return info; } -void VideoOutInit(uint32_t width, uint32_t height) { - EXIT_IF(g_video_out_context != nullptr); +VideoOutDriver::VideoOutDriver(uint32_t width, uint32_t height, Graphics::Presenter& presenter) + : m_impl(std::make_unique(width, height, presenter)) {} - g_video_out_context = new VideoOutContext; +VideoOutDriver::~VideoOutDriver() = default; - g_video_out_context->Init(width, height); +VideoOutDriver::Impl& VideoOutDriver::State() noexcept { + return *m_impl; } -void VideoOutContext::Init(uint32_t width, uint32_t height) { +VideoOutDriver& VideoOutInit(uint32_t width, uint32_t height, Graphics::Presenter& presenter) { + EXIT_IF(g_video_out_driver != nullptr); + g_video_out_driver = std::make_unique(width, height, presenter); + return *g_video_out_driver; +} + +void VideoOutShutdown() { + g_video_out_driver.reset(); +} + +VideoOutDriver::Impl::~Impl() { + if (m_present_thread.joinable()) { + m_present_thread.request_stop(); + m_present_thread.join(); + } + for (int handle = 1; handle < VIDEO_OUT_NUM_MAX; handle++) { + (void)Close(handle); + } +} + +void VideoOutDriver::Impl::Init(uint32_t width, uint32_t height) { for (auto& ctx: m_video_out_ctx) { ctx.width = width; ctx.height = height; } } -int VideoOutContext::Open() { +int VideoOutDriver::Impl::Open() { Common::LockGuard lock(m_mutex); int handle = -1; @@ -552,22 +628,26 @@ int VideoOutContext::Open() { auto& config = m_video_out_ctx[handle]; Common::LockGuard config_lock(config.mutex); - EXIT_IF(!config.flip_eqs.empty()); - EXIT_IF(!config.pre_vblank_eqs.empty()); - EXIT_IF(!config.vblank_eqs.empty()); - EXIT_IF(!config.output_mode_eqs.empty()); - EXIT_IF(config.flip_rate != 0); - EXIT_IF(!config.buffers_sets.empty()); - for (const auto& buffer: config.buffers) { - EXIT_IF(buffer.buffer != nullptr || buffer.buffer_vulkan != nullptr || - buffer.buffer_size != 0 || buffer.buffer_pitch != 0); + { + Common::LockGuard event_lock(config.events->mutex); + EXIT_IF(!config.events->flip.empty()); + EXIT_IF(!config.events->pre_vblank.empty()); + EXIT_IF(!config.events->vblank.empty()); + EXIT_IF(!config.events->output_mode.empty()); } - for (bool unregistering: config.unregistering) { - EXIT_IF(unregistering); + EXIT_IF(config.flip_rate != 0); + for (const auto& buffer: config.buffers) { + EXIT_IF(buffer.Occupied()); + } + for (const auto& group: config.groups) { + EXIT_IF(group.occupied); } - config.closing = false; - config.opened = true; + config.closing = false; + config.opened = true; + if (++config.generation == 0) { + EXIT("video-out port generation wrapped\n"); + } config.output_mode = VIDEO_OUT_OUTPUT_MODE_DEFAULT; config.flip_status = VideoOutFlipStatus(); config.flip_status.flipArg = -1; @@ -579,59 +659,62 @@ int VideoOutContext::Open() { return handle; } -void VideoOutContext::Close(int handle) { +bool VideoOutDriver::Impl::Close(int handle) { Common::LockGuard lock(m_mutex); - EXIT_NOT_IMPLEMENTED(handle >= VIDEO_OUT_NUM_MAX); - EXIT_NOT_IMPLEMENTED(!m_video_out_ctx[handle].opened); - - auto& config = m_video_out_ctx[handle]; - config.opened = false; - - config.mutex.Lock(); - if (config.closing) { - EXIT("video-out handle is already closing\n"); + if (handle <= 0 || handle >= VIDEO_OUT_NUM_MAX || !m_video_out_ctx[handle].opened) { + return false; } - config.closing = true; - if (m_flip_queue.HasPending(config, VIDEO_OUT_BUFFER_INDEX_BLACK, - VIDEO_OUT_BUFFER_NUM_MAX - VIDEO_OUT_BUFFER_INDEX_BLACK)) { - EXIT("cannot close video-out handle with pending flips\n"); - } - DeleteVideoOutEventsLocked(config.flip_eqs, VideoOutEventKind::Flip); - DeleteVideoOutEventsLocked(config.pre_vblank_eqs, VideoOutEventKind::PreVblankStart); - DeleteVideoOutEventsLocked(config.vblank_eqs, VideoOutEventKind::Vblank); - DeleteVideoOutEventsLocked(config.output_mode_eqs, VideoOutEventKind::OutputMode); - config.flip_rate = 0; - - std::vector images; - for (const auto& buffer: config.buffers) { - if ((buffer.buffer == nullptr) != (buffer.buffer_vulkan == nullptr) || - (buffer.buffer_vulkan != nullptr && - (buffer.buffer_size == 0 || buffer.buffer_pitch == 0))) { - EXIT("inconsistent registered video-out buffer state\n"); + auto& config = m_video_out_ctx[handle]; + VideoOutEventQueues flip_events; + VideoOutEventQueues pre_vblank_events; + VideoOutEventQueues vblank_events; + VideoOutEventQueues output_mode_events; + { + Common::LockGuard config_lock(config.mutex); + if (config.closing) { + return false; } - if (buffer.buffer_vulkan != nullptr) { - images.push_back(buffer.buffer_vulkan); + config.opened = false; + config.closing = true; + if (++config.generation == 0) { + EXIT("video-out port generation wrapped\n"); } - } - if (!images.empty()) { - Graphics::GetRenderContext().GetTextureCache().UnregisterVideoOutSurfaces(images); - } - for (auto& buffer: config.buffers) { - buffer = VideoOutBufferInfo {}; + { + Common::LockGuard event_lock(config.events->mutex); + flip_events = std::move(config.events->flip); + pre_vblank_events = std::move(config.events->pre_vblank); + vblank_events = std::move(config.events->vblank); + output_mode_events = std::move(config.events->output_mode); + } + config.flip_rate = 0; + + for (const auto& buffer: config.buffers) { + if (buffer.Occupied() && + (buffer.group_index >= VIDEO_OUT_BUFFER_ATTRIBUTE_NUM_MAX || + buffer.data_address == 0 || !config.groups[buffer.group_index].occupied)) { + EXIT("inconsistent registered video-out buffer state\n"); + } + } + for (auto& buffer: config.buffers) { + buffer = VideoOutBuffer {}; + } + for (auto& group: config.groups) { + group = BufferAttributeGroup {}; + } + config.vblank_cond.SignalAll(); } - config.buffers_sets.clear(); - for (bool unregistering: config.unregistering) { - if (unregistering) { - EXIT("video-out close raced with buffer unregistration\n"); - } - } - config.mutex.Unlock(); + m_flip_queue.Cancel(config); + DeleteVideoOutEvents(flip_events, VideoOutEventKind::Flip); + DeleteVideoOutEvents(pre_vblank_events, VideoOutEventKind::PreVblankStart); + DeleteVideoOutEvents(vblank_events, VideoOutEventKind::Vblank); + DeleteVideoOutEvents(output_mode_events, VideoOutEventKind::OutputMode); + return true; } -VideoOutConfig* VideoOutContext::Get(int handle) { +VideoOutConfig* VideoOutDriver::Impl::Get(int handle) { Common::LockGuard lock(m_mutex); if (handle <= 0 || handle >= VIDEO_OUT_NUM_MAX || !m_video_out_ctx[handle].opened) { return nullptr; @@ -640,13 +723,28 @@ VideoOutConfig* VideoOutContext::Get(int handle) { return m_video_out_ctx + handle; } -bool VideoOutContext::IsOpened(int handle) { +VideoOutConfig* VideoOutDriver::Impl::Get(int handle, uint64_t& generation) { + Common::LockGuard lock(m_mutex); + if (handle <= 0 || handle >= VIDEO_OUT_NUM_MAX || !m_video_out_ctx[handle].opened) { + return nullptr; + } + + auto* config = m_video_out_ctx + handle; + Common::LockGuard config_lock(config->mutex); + if (!config->opened || config->closing) { + return nullptr; + } + generation = config->generation; + return config; +} + +bool VideoOutDriver::Impl::IsOpened(int handle) { Common::LockGuard lock(m_mutex); return handle > 0 && handle < VIDEO_OUT_NUM_MAX && m_video_out_ctx[handle].opened; } -void VideoOutContext::VblankBegin() { +void VideoOutDriver::Impl::VblankBegin() { Common::LockGuard lock(m_mutex); for (int i = 1; i < VIDEO_OUT_NUM_MAX; i++) { @@ -658,14 +756,14 @@ void VideoOutContext::VblankBegin() { ctx.pre_vblank_status.reserved = LibKernel::KernelReadTsc(); ctx.pre_vblank_status.processTimeCounter = LibKernel::KernelGetProcessTimeCounter(); - TriggerVideoOutEventsLocked(ctx.pre_vblank_eqs, VideoOutEventKind::PreVblankStart, - reinterpret_cast(ctx.pre_vblank_status.count)); + TriggerVideoOutEvents(ctx, VideoOutEventKind::PreVblankStart, + reinterpret_cast(ctx.pre_vblank_status.count)); ctx.mutex.Unlock(); } } } -void VideoOutContext::VblankEnd() { +void VideoOutDriver::Impl::VblankEnd() { Common::LockGuard lock(m_mutex); for (int i = 1; i < VIDEO_OUT_NUM_MAX; i++) { @@ -677,43 +775,75 @@ void VideoOutContext::VblankEnd() { ctx.vblank_status.reserved = LibKernel::KernelReadTsc(); ctx.vblank_status.processTimeCounter = LibKernel::KernelGetProcessTimeCounter(); - TriggerVideoOutEventsLocked(ctx.vblank_eqs, VideoOutEventKind::Vblank, - reinterpret_cast(ctx.vblank_status.count)); + TriggerVideoOutEvents(ctx, VideoOutEventKind::Vblank, + reinterpret_cast(ctx.vblank_status.count)); + ctx.vblank_cond.SignalAll(); ctx.mutex.Unlock(); } } } -Presentation::DisplayBufferImage VideoOutContext::FindImage(const void* buffer, - bool render_target) { - Presentation::DisplayBufferImage ret; - Common::LockGuard lock(m_mutex); - for (auto& ctx: m_video_out_ctx) { - if (!ctx.opened) { - continue; - } - Common::LockGuard config_lock(ctx.mutex); - if (ctx.closing) { - EXIT("display-buffer lookup raced with video-out close\n"); - } - for (const auto& set: ctx.buffers_sets) { - for (int j = set.start_index; j < set.start_index + set.num; j++) { - if (ctx.buffers[j].buffer == buffer) { - if (ctx.unregistering[j] || ctx.buffers[j].buffer_vulkan == nullptr) { - EXIT("display-buffer lookup found an unavailable video-out surface\n"); - } - ret.image = ctx.buffers[j].buffer_vulkan; - ret.size = ctx.buffers[j].buffer_size; - ret.pitch = ctx.buffers[j].buffer_pitch; - ret.index = j - set.start_index; - Graphics::GetRenderContext().GetTextureCache().RefreshVideoOut(*ret.image, - render_target); - return ret; - } +void VideoOutDriver::Impl::PresentThread(std::stop_token token) { + const auto frequency = Common::Timer::QueryPerformanceFrequency(); + EXIT_IF(frequency == 0); + + int64_t total_wait = 0; + while (!token.stop_requested()) { + const auto sleep_begin = Common::Timer::QueryPerformanceCounter(); + if (total_wait > 0) { + const auto sleep_end = sleep_begin + static_cast(total_wait); + auto now = sleep_begin; + while (!token.stop_requested() && now < sleep_end) { + const auto remaining_us = (sleep_end - now) * 1000000u / frequency; + Common::Thread::SleepMicro( + static_cast(std::clamp(remaining_us, 1, 1000))); + now = Common::Timer::QueryPerformanceCounter(); } } + if (token.stop_requested()) { + break; + } + const auto frame_begin = Common::Timer::QueryPerformanceCounter(); + total_wait -= static_cast(frame_begin - sleep_begin); + + const auto refresh = std::max(Config::GetVblankFrequency(), 1u); + const auto period = std::max(frequency / refresh, uint64_t {1}); + + if (m_presenter.IsGuestPaused()) { + if (auto* frame = m_presenter.PrepareLastFrame(); frame != nullptr) { + m_presenter.Present(*frame, true); + } + const auto frame_end = Common::Timer::QueryPerformanceCounter(); + total_wait += static_cast(period) - + static_cast(frame_end - frame_begin); + continue; + } + + VblankBegin(); + const bool presented = m_flip_queue.Flip(0); + if (!presented && total_wait < 0) { + bool any_open = false; + uint32_t width = 0; + uint32_t height = 0; + { + Common::LockGuard lock(m_mutex); + width = m_video_out_ctx[0].width; + height = m_video_out_ctx[0].height; + for (int handle = 1; handle < VIDEO_OUT_NUM_MAX; handle++) { + any_open |= m_video_out_ctx[handle].opened; + } + } + if (!any_open) { + auto& blank = m_presenter.PrepareBlankFrame(width, height, true); + m_presenter.Present(blank); + } + } + VblankEnd(); + + const auto frame_end = Common::Timer::QueryPerformanceCounter(); + total_wait += static_cast(period) - + static_cast(frame_end - frame_begin); } - return ret; } bool FlipQueue::Reserve(VideoOutConfig& cfg, int index, int64_t flip_arg, FlipRequestSource source, @@ -728,6 +858,7 @@ bool FlipQueue::Reserve(VideoOutConfig& cfg, int index, int64_t flip_arg, FlipRe Request r {}; r.id = m_next_request_id++; r.cfg = &cfg; + r.generation = cfg.generation; r.index = index; r.flip_arg = flip_arg; r.submit_ptc = LibKernel::KernelGetProcessTimeCounter(); @@ -746,9 +877,58 @@ bool FlipQueue::Reserve(VideoOutConfig& cfg, int index, int64_t flip_arg, FlipRe return true; } +FlipQueue::~FlipQueue() { + for (auto* queue: {&m_requests, &m_cpu_requests, &m_cancelled_requests}) { + for (auto& request: *queue) { + if (request.frame != nullptr) { + m_presenter.Discard(*request.frame); + } + } + } +} + +void FlipQueue::Cancel(VideoOutConfig& cfg) { + std::vector frames; + m_mutex.Lock(); + while (m_processing && !m_requests.empty() && m_requests.front().cfg == &cfg) { + m_done_cond_var.Wait(&m_mutex); + } + for (auto* queue: {&m_requests, &m_cpu_requests}) { + for (auto it = queue->begin(); it != queue->end();) { + if (it->cfg != &cfg) { + ++it; + continue; + } + if (it->state == RequestState::Reserved || it->state == RequestState::Recording) { + auto cancelled = it++; + m_cancelled_requests.splice(m_cancelled_requests.end(), *queue, cancelled); + continue; + } + if (it->state == RequestState::Presenting) { + EXIT("video-out cancellation retained a presenting request\n"); + } + if (it->frame != nullptr) { + frames.push_back(it->frame); + } + it = queue->erase(it); + } + } + m_done_cond_var.SignalAll(); + m_submit_slot_cond_var.SignalAll(); + m_submit_cond_var.SignalAll(); + m_mutex.Unlock(); + for (auto* frame: frames) { + m_presenter.Discard(*frame); + } + Common::LockGuard lock(cfg.mutex); + cfg.flip_status.flipPendingNum = 0; + cfg.flip_status.gcQueueNum = 0; +} + void FlipQueue::Prepare(uint64_t request_id, Graphics::CommandBuffer& buffer) { - VideoOutConfig* cfg = nullptr; - int index = 0; + VideoOutConfig* cfg = nullptr; + uint64_t generation = 0; + int index = 0; { Common::LockGuard lock(m_mutex); auto request = std::find_if(m_requests.begin(), m_requests.end(), @@ -757,7 +937,15 @@ void FlipQueue::Prepare(uint64_t request_id, Graphics::CommandBuffer& buffer) { auto pending = std::find_if(m_cpu_requests.begin(), m_cpu_requests.end(), [request_id](const auto& r) { return r.id == request_id; }); if (pending == m_cpu_requests.end()) { - EXIT("cannot prepare video-out request id=%" PRIu64 "\n", request_id); + auto cancelled = + std::find_if(m_cancelled_requests.begin(), m_cancelled_requests.end(), + [request_id](const auto& r) { return r.id == request_id; }); + if (cancelled == m_cancelled_requests.end() || + cancelled->state != RequestState::Reserved) { + EXIT("cannot prepare video-out request id=%" PRIu64 "\n", request_id); + } + cancelled->state = RequestState::Recording; + return; } request = m_requests.insert(m_requests.end(), *pending); m_cpu_requests.erase(pending); @@ -767,70 +955,104 @@ void FlipQueue::Prepare(uint64_t request_id, Graphics::CommandBuffer& buffer) { } request->state = RequestState::Recording; cfg = request->cfg; + generation = request->generation; index = request->index; } - const bool special = IsSpecialBufferIndex(index); - Graphics::VideoOutVulkanImage* source = nullptr; - uint32_t width = 0; - uint32_t height = 0; + const bool special = IsSpecialBufferIndex(index); + Graphics::ImageInfo source_info; + uint32_t width = 0; + uint32_t height = 0; + bool current = false; { Common::LockGuard lock(cfg->mutex); - if (cfg->closing) { - EXIT("cannot prepare flip for a closing video-out, id=%" PRIu64 "\n", request_id); - } - if (special) { - width = cfg->width; - height = cfg->height; - } else { - if (cfg->unregistering[index]) { - EXIT("cannot prepare flip from an unavailable surface, id=%" PRIu64 " index=%d\n", - request_id, index); - } - source = cfg->buffers[index].buffer_vulkan; - if (source == nullptr) { - EXIT("cannot prepare flip without a native surface, id=%" PRIu64 " index=%d\n", - request_id, index); + current = cfg->opened && !cfg->closing && cfg->generation == generation; + if (current) { + if (special) { + width = cfg->width; + height = cfg->height; + } else { + const auto& surface = cfg->buffers[index]; + if (!surface.Occupied()) { + EXIT("cannot prepare flip from an unregistered surface, id=%" PRIu64 + " index=%d\n", + request_id, index); + } + if (surface.group_index >= VIDEO_OUT_BUFFER_ATTRIBUTE_NUM_MAX || + !cfg->groups[surface.group_index].occupied) { + EXIT("video-out surface references an unavailable attribute group, id=%" PRIu64 + " index=%d group=%d\n", + request_id, index, surface.group_index); + } + source_info = cfg->groups[surface.group_index].ImageInfo(surface); } } } - auto& frame = special ? Graphics::WindowPrepareBlankFrame(buffer, width, height, - index == VIDEO_OUT_BUFFER_INDEX_BLACK) - : Graphics::WindowPrepareFrame(buffer, *source); + if (!current) { + Common::LockGuard lock(m_mutex); + const auto request = + std::find_if(m_requests.begin(), m_requests.end(), + [request_id](const auto& r) { return r.id == request_id; }); + if (request != m_requests.end()) { + m_cancelled_requests.splice(m_cancelled_requests.end(), m_requests, request); + } + return; + } + Graphics::Presenter::Frame* frame = nullptr; + if (special) { + frame = &m_presenter.PrepareBlankFrame(width, height, + index == VIDEO_OUT_BUFFER_INDEX_BLACK, &buffer); + } else { + frame = &m_presenter.PrepareFrame(buffer, source_info); + } Common::LockGuard lock(m_mutex); - auto request = std::find_if(m_requests.begin(), m_requests.end(), + Request* prepared = nullptr; + if (const auto request = + std::find_if(m_requests.begin(), m_requests.end(), + [request_id](const auto& r) { return r.id == request_id; }); + request != m_requests.end()) { + prepared = &*request; + } else if (const auto cancelled = + std::find_if(m_cancelled_requests.begin(), m_cancelled_requests.end(), [request_id](const auto& r) { return r.id == request_id; }); - if (request == m_requests.end() || request->state != RequestState::Recording || - request->frame != nullptr) { + cancelled != m_cancelled_requests.end()) { + prepared = &*cancelled; + } + if (prepared == nullptr || prepared->state != RequestState::Recording || + prepared->frame != nullptr) { EXIT("video-out request changed while recording, id=%" PRIu64 "\n", request_id); } - request->frame = &frame; -} - -uint64_t FlipQueue::PrepareNextCpu(Graphics::CommandBuffer& buffer) { - uint64_t request_id = 0; - { - Common::LockGuard lock(m_mutex); - if (m_cpu_requests.empty()) { - EXIT("CPU flip preparation has no accepted request\n"); - } - request_id = m_cpu_requests.front().id; - } - Prepare(request_id, buffer); - return request_id; + prepared->frame = frame; } void FlipQueue::Complete(uint64_t request_id) { - Common::LockGuard lock(m_mutex); + m_mutex.Lock(); auto request = std::find_if(m_requests.begin(), m_requests.end(), [request_id](const auto& r) { return r.id == request_id; }); - if (request == m_requests.end() || request->state != RequestState::Recording || - request->frame == nullptr) { + if (request != m_requests.end()) { + if (request->state != RequestState::Recording || request->frame == nullptr) { + m_mutex.Unlock(); + EXIT("completed GPU flip has no prepared recording, id=%" PRIu64 "\n", request_id); + } + request->state = RequestState::Ready; + m_submit_cond_var.Signal(); + m_mutex.Unlock(); + return; + } + auto cancelled = std::find_if(m_cancelled_requests.begin(), m_cancelled_requests.end(), + [request_id](const auto& r) { return r.id == request_id; }); + if (cancelled == m_cancelled_requests.end() || cancelled->state != RequestState::Recording) { + m_mutex.Unlock(); EXIT("completed GPU flip has no prepared recording, id=%" PRIu64 "\n", request_id); } - request->state = RequestState::Ready; - m_submit_cond_var.Signal(); + auto* frame = cancelled->frame; + m_cancelled_requests.erase(cancelled); + m_done_cond_var.SignalAll(); + m_mutex.Unlock(); + if (frame != nullptr) { + m_presenter.Discard(*frame); + } } void FlipQueue::WaitForSubmitSlot() { @@ -856,19 +1078,6 @@ void FlipQueue::Wait(VideoOutConfig& cfg, int index) { } } -bool FlipQueue::HasPending(VideoOutConfig& cfg, int start_index, int count) { - if (count <= 0 || start_index > INT_MAX - count) { - EXIT("invalid video-out pending-flip query range\n"); - } - Common::LockGuard lock(m_mutex); - auto matches = [&](const auto& request) { - return request.cfg == &cfg && request.index >= start_index && - request.index < start_index + count; - }; - return std::any_of(m_requests.begin(), m_requests.end(), matches) || - std::any_of(m_cpu_requests.begin(), m_cpu_requests.end(), matches); -} - bool FlipQueue::Flip(uint32_t micros) { KYTY_PROFILER_BLOCK("FlipQueue::Flip"); @@ -891,9 +1100,13 @@ bool FlipQueue::Flip(uint32_t micros) { m_processing = true; auto r = m_requests.front(); m_mutex.Unlock(); - if (!IsFlipDue(*r.cfg)) { - Common::LockGuard lock(m_mutex); + + r.cfg->mutex.Lock(); + if (!IsFlipDueLocked(*r.cfg, r.generation)) { + r.cfg->mutex.Unlock(); + Common::LockGuard queue_lock(m_mutex); m_processing = false; + m_done_cond_var.SignalAll(); return false; } @@ -905,7 +1118,7 @@ bool FlipQueue::Flip(uint32_t micros) { m_requests.front().state = RequestState::Presenting; m_mutex.Unlock(); - Graphics::WindowPresentFrame(*r.frame); + m_presenter.Present(*r.frame); m_mutex.Lock(); if (m_requests.empty() || m_requests.front().id != r.id || @@ -924,15 +1137,12 @@ bool FlipQueue::Flip(uint32_t micros) { if (r.source == FlipRequestSource::GpuEop && r.cfg->flip_status.gcQueueNum > 0) { r.cfg->flip_status.gcQueueNum--; } + TriggerVideoOutEvents(*r.cfg, VideoOutEventKind::Flip, reinterpret_cast(r.flip_arg)); m_processing = false; m_done_cond_var.SignalAll(); m_submit_slot_cond_var.Signal(); m_mutex.Unlock(); - - r.cfg->mutex.Lock(); - TriggerVideoOutEventsLocked(r.cfg->flip_eqs, VideoOutEventKind::Flip, - reinterpret_cast(r.flip_arg)); r.cfg->mutex.Unlock(); if (Config::GraphicsDebugDumpEnabled() && @@ -944,43 +1154,21 @@ bool FlipQueue::Flip(uint32_t micros) { } void FlipQueue::GetFlipStatus(VideoOutConfig& cfg, VideoOutFlipStatus& out) { - Common::LockGuard lock(m_mutex); + Common::LockGuard lock(cfg.mutex); out = cfg.flip_status; } -bool VideoOutFlipWindow(uint32_t micros) { - EXIT_IF(g_video_out_context == nullptr); - - WaitForNextVblank(); - - return g_video_out_context->GetFlipQueue().Flip(micros); -} - -void VideoOutBeginVblank() { - EXIT_IF(g_video_out_context == nullptr); - - g_video_out_context->VblankBegin(); -} - -void VideoOutEndVblank() { - EXIT_IF(g_video_out_context == nullptr); - - g_video_out_context->VblankEnd(); -} - KYTY_SYSV_ABI int VideoOutOpen(int user_id, int bus_type, int index, const void* param) { PRINT_NAME(); - EXIT_IF(g_video_out_context == nullptr); - EXIT_NOT_IMPLEMENTED(user_id != 255 && user_id != 0); EXIT_NOT_IMPLEMENTED(bus_type != 0); EXIT_NOT_IMPLEMENTED(index != 0); LOGF("\t param = 0x%016" PRIx64 "\n", reinterpret_cast(param)); - int handle = g_video_out_context->Open(); + int handle = DriverState().Open(); if (handle < 0) { return VIDEO_OUT_ERROR_RESOURCE_BUSY; @@ -992,15 +1180,7 @@ KYTY_SYSV_ABI int VideoOutOpen(int user_id, int bus_type, int index, const void* KYTY_SYSV_ABI int VideoOutClose(int handle) { PRINT_NAME(); - EXIT_IF(g_video_out_context == nullptr); - - if (!g_video_out_context->IsOpened(handle)) { - return VIDEO_OUT_ERROR_INVALID_HANDLE; - } - - g_video_out_context->Close(handle); - - return OK; + return DriverState().Close(handle) ? OK : VIDEO_OUT_ERROR_INVALID_HANDLE; } KYTY_SYSV_ABI void VideoOutSetBufferAttribute2(VideoOutBufferAttribute2* attribute, @@ -1038,11 +1218,9 @@ KYTY_SYSV_ABI void VideoOutSetBufferAttribute2(VideoOutBufferAttribute2* attribu KYTY_SYSV_ABI int VideoOutSetFlipRate(int handle, int rate) { PRINT_NAME(); - EXIT_IF(g_video_out_context == nullptr); - LOGF("\trate = %d\n", rate); - auto* ctx = g_video_out_context->Get(handle); + auto* ctx = DriverState().Get(handle); if (ctx == nullptr) { return VIDEO_OUT_ERROR_INVALID_HANDLE; } @@ -1051,6 +1229,7 @@ KYTY_SYSV_ABI int VideoOutSetFlipRate(int handle, int rate) { return VIDEO_OUT_ERROR_INVALID_VALUE; } + Common::LockGuard lock(ctx->mutex); ctx->flip_rate = rate; return OK; @@ -1094,61 +1273,13 @@ KYTY_SYSV_ABI int VideoOutAddOutputModeEvent(LibKernel::EventQueue::KernelEqueue return RegisterVideoOutEvent(handle, eq, VideoOutEventKind::OutputMode, udata); } -static int RegisterBuffersInternal(VideoOutConfig& ctx, int set_id, int start_index, - const void* const* addresses, int buffer_num, - const std::vector& infos) { - if (addresses == nullptr || buffer_num <= 0 || - infos.size() != static_cast(buffer_num)) { - EXIT("invalid internal video-out buffer registration arguments\n"); - } - if (set_id < 0 || set_id >= VIDEO_OUT_BUFFER_ATTRIBUTE_NUM_MAX) { - EXIT("internal video-out buffer set identifier is out of range\n"); - } - Common::LockGuard lock(ctx.mutex); - if (ctx.closing) { - EXIT("cannot register buffers on a closing video-out handle\n"); - } - if (std::any_of(ctx.buffers_sets.begin(), ctx.buffers_sets.end(), - [set_id](const auto& set) { return set.set_id == set_id; })) { - return VIDEO_OUT_ERROR_INVALID_INDEX; - } - for (int i = 0; i < buffer_num; i++) { - if (ctx.unregistering[start_index + i]) { - EXIT("video-out buffer registration raced with unregistration\n"); - } - if (ctx.buffers[start_index + i].buffer != nullptr) { - return VIDEO_OUT_ERROR_SLOT_OCCUPIED; - } - } - auto images = Graphics::GetRenderContext().GetTextureCache().RegisterVideoOutSurfaces(infos); - if (images.size() != infos.size()) { - EXIT("video-out texture cache returned an incomplete surface set\n"); - } - ctx.buffers_sets.push_back({start_index, buffer_num, set_id}); - for (int i = 0; i < buffer_num; i++) { - auto& dst = ctx.buffers[i + start_index]; - dst.set_id = set_id; - dst.buffer = addresses[i]; - dst.buffer_size = infos[i].size; - dst.buffer_pitch = infos[i].pitch; - dst.buffer_vulkan = images[i]; - LOGF("\tbuffers[%d] = %016" PRIx64 " metadata = %016" PRIx64 " dcc = %08" PRIx32 "\n", - i + start_index, reinterpret_cast(addresses[i]), infos[i].metadata_address, - infos[i].dcc_control); - } - - return OK; -} - KYTY_SYSV_ABI int VideoOutRegisterBuffers2(int handle, int set_index, int buffer_index_start, const VideoOutBuffers* buffers, int buffer_num, const VideoOutBufferAttribute2* attribute, int category, void* option) { PRINT_NAME(); - EXIT_IF(g_video_out_context == nullptr); - - auto* ctx = g_video_out_context->Get(handle); + auto* ctx = DriverState().Get(handle); if (ctx == nullptr) { return VIDEO_OUT_ERROR_INVALID_HANDLE; } @@ -1184,16 +1315,20 @@ KYTY_SYSV_ABI int VideoOutRegisterBuffers2(int handle, int set_index, int buffer attribute->option, category); if (option != nullptr) { - EXIT("video-out buffer registration options are unsupported\n"); + return VIDEO_OUT_ERROR_INVALID_OPTION; } if (category != VIDEO_OUT_BUFFER_ATTRIBUTE_CATEGORY_UNCOMPRESSED && category != VIDEO_OUT_BUFFER_ATTRIBUTE_CATEGORY_COMPRESSED) { return VIDEO_OUT_ERROR_INVALID_CATEGORY; } - const bool compressed = category == VIDEO_OUT_BUFFER_ATTRIBUTE_CATEGORY_COMPRESSED; - std::vector addresses(static_cast(buffer_num)); - std::vector infos; - infos.reserve(static_cast(buffer_num)); + + BufferAttributeGroup group { + .attribute = *attribute, + .category = category, + .occupied = true, + }; + std::vector registrations; + registrations.reserve(static_cast(buffer_num)); for (int i = 0; i < buffer_num; i++) { LOGF("\t buffers[%d]: data=%p metadata=%p\n", i, buffers[i].data, buffers[i].metadata); @@ -1203,22 +1338,37 @@ KYTY_SYSV_ABI int VideoOutRegisterBuffers2(int handle, int set_index, int buffer } const auto data_address = reinterpret_cast(buffers[i].data); const auto metadata_address = reinterpret_cast(buffers[i].metadata); - const auto compression = Graphics::ClassifyVideoOutCompression( - compressed, metadata_address, attribute->dcc_control, - attribute->dcc_cb_register_clear_color); - if (compression == Graphics::VideoOutCompression::Unsupported) { - EXIT("unsupported video-out compression, category=%d data=0x%016" PRIx64 - " metadata=0x%016" PRIx64 " dcc_control=0x%08" PRIx32 " clear_color=0x%016" PRIx64 - "\n", - category, data_address, metadata_address, attribute->dcc_control, - attribute->dcc_cb_register_clear_color); - } - addresses[static_cast(i)] = buffers[i].data; - infos.push_back(MakeVideoOutInfo(*attribute, data_address, metadata_address, compression)); + registrations.push_back({ + .group_index = set_index, + .data_address = data_address, + .metadata_address = metadata_address, + }); + (void)group.ImageInfo(registrations.back()); } - return RegisterBuffersInternal(*ctx, set_index, buffer_index_start, addresses.data(), - buffer_num, infos); + Common::LockGuard lock(ctx->mutex); + if (ctx->closing) { + return VIDEO_OUT_ERROR_INVALID_HANDLE; + } + if (ctx->groups[set_index].occupied) { + return VIDEO_OUT_ERROR_INVALID_INDEX; + } + for (int i = 0; i < buffer_num; i++) { + if (ctx->buffers[buffer_index_start + i].Occupied()) { + return VIDEO_OUT_ERROR_SLOT_OCCUPIED; + } + } + + ctx->groups[set_index] = group; + for (int i = 0; i < buffer_num; i++) { + ctx->buffers[buffer_index_start + i] = registrations[static_cast(i)]; + const auto& buffer = registrations[static_cast(i)]; + LOGF("\tbuffers[%d] = %016" PRIx64 " metadata = %016" PRIx64 " dcc = %08" PRIx32 "\n", + buffer_index_start + i, buffer.data_address, buffer.metadata_address, + attribute->dcc_control); + } + + return OK; } KYTY_SYSV_ABI int VideoOutSubmitChangeBufferAttribute2(int handle, int set_index, @@ -1226,9 +1376,7 @@ KYTY_SYSV_ABI int VideoOutSubmitChangeBufferAttribute2(int handle, int set_index void* option) { PRINT_NAME(); - EXIT_IF(g_video_out_context == nullptr); - - auto* ctx = g_video_out_context->Get(handle); + auto* ctx = DriverState().Get(handle); if (ctx == nullptr) { return VIDEO_OUT_ERROR_INVALID_HANDLE; } @@ -1243,17 +1391,32 @@ KYTY_SYSV_ABI int VideoOutSubmitChangeBufferAttribute2(int handle, int set_index if (option != nullptr) { return VIDEO_OUT_ERROR_INVALID_OPTION; } - (void)ctx; - (void)set_index; - EXIT("video-out buffer attribute query is unsupported\n"); + + Common::LockGuard lock(ctx->mutex); + const auto& current = ctx->groups[set_index]; + if (ctx->closing || !current.occupied) { + return VIDEO_OUT_ERROR_INVALID_INDEX; + } + + BufferAttributeGroup replacement { + .attribute = *attribute, + .category = current.category, + .occupied = true, + }; + for (const auto& buffer: ctx->buffers) { + if (buffer.group_index == set_index) { + (void)replacement.ImageInfo(buffer); + } + } + ctx->groups[set_index] = replacement; + + return OK; } KYTY_SYSV_ABI int VideoOutUnregisterBuffers(int handle, int set_index) { PRINT_NAME(); - EXIT_IF(g_video_out_context == nullptr); - - auto* ctx = g_video_out_context->Get(handle); + auto* ctx = DriverState().Get(handle); if (ctx == nullptr) { return VIDEO_OUT_ERROR_INVALID_HANDLE; } @@ -1263,132 +1426,89 @@ KYTY_SYSV_ABI int VideoOutUnregisterBuffers(int handle, int set_index) { Common::LockGuard lock(ctx->mutex); if (ctx->closing) { - EXIT("cannot unregister buffers from a closing video-out handle\n"); + return VIDEO_OUT_ERROR_INVALID_HANDLE; } - auto set_it = std::find_if(ctx->buffers_sets.begin(), ctx->buffers_sets.end(), - [set_index](const auto& set) { return set.set_id == set_index; }); - if (set_it == ctx->buffers_sets.end()) { + if (!ctx->groups[set_index].occupied) { return VIDEO_OUT_ERROR_INVALID_INDEX; } - std::vector images; - images.reserve(static_cast(set_it->num)); - for (int i = set_it->start_index; i < set_it->start_index + set_it->num; i++) { - if (ctx->unregistering[i]) { - EXIT("video-out buffer is already being unregistered\n"); + ctx->groups[set_index] = BufferAttributeGroup {}; + for (auto& buffer: ctx->buffers) { + if (buffer.group_index == set_index) { + buffer = VideoOutBuffer {}; } - ctx->unregistering[i] = true; - if (ctx->buffers[i].set_id != set_index || ctx->buffers[i].buffer == nullptr || - ctx->buffers[i].buffer_vulkan == nullptr) { - EXIT("video-out buffer set contains inconsistent surface state\n"); - } - images.push_back(ctx->buffers[i].buffer_vulkan); } - if (g_video_out_context->GetFlipQueue().HasPending(*ctx, set_it->start_index, set_it->num)) { - EXIT("cannot unregister video-out buffers with pending flips\n"); - } - Graphics::GetRenderContext().GetTextureCache().UnregisterVideoOutSurfaces(images); - for (int i = set_it->start_index; i < set_it->start_index + set_it->num; i++) { - ctx->buffers[i] = VideoOutBufferInfo {}; - ctx->unregistering[i] = false; - } - ctx->buffers_sets.erase(set_it); return OK; } -} // namespace Libs::VideoOut - -namespace Libs::Presentation { - -DisplayBufferImage DisplayBufferFind(uint64_t addr, bool render_target) { - EXIT_IF(VideoOut::g_video_out_context == nullptr); - - return VideoOut::g_video_out_context->FindImage(reinterpret_cast(addr), render_target); -} - -} // namespace Libs::Presentation - -namespace Libs::VideoOut { - KYTY_SYSV_ABI int VideoOutSubmitFlip(int handle, int index, int flip_mode, int64_t flip_arg) { PRINT_NAME(); uint64_t request_id = 0; - const int result = - ReserveFlipRequest(handle, index, flip_mode, flip_arg, FlipRequestSource::Cpu, request_id); + const int result = ReserveFlipRequest(DriverState(), handle, index, flip_mode, flip_arg, + FlipRequestSource::Cpu, request_id); if (result == VIDEO_OUT_ERROR_INVALID_VALUE) { LOGF("\t unsupported flip_mode = %d\n", flip_mode); } if (result != OK) { return result; } - Graphics::GraphicsRunSubmitFlipPreparation(); + g_video_out_driver->SubmitFlipPreparation(request_id); return OK; } -} // namespace Libs::VideoOut +int VideoOutDriver::SubmitFlipFromGpu(Graphics::CommandBuffer& buffer, int handle, int index, + int flip_mode, int64_t flip_arg, uint64_t& request_id) { + EXIT_IF(buffer.IsInvalid()); -namespace Libs::Presentation { - -int DisplayBufferSubmitFlipFromGpu(Graphics::CommandBuffer& buffer, int handle, int index, - int flip_mode, int64_t flip_arg, uint64_t& request_id) { - EXIT_IF(VideoOut::g_video_out_context == nullptr || buffer.IsInvalid()); - - const int result = VideoOut::ReserveFlipRequest( - handle, index, flip_mode, flip_arg, VideoOut::FlipRequestSource::GpuEop, request_id); + const int result = ReserveFlipRequest(*m_impl, handle, index, flip_mode, flip_arg, + FlipRequestSource::GpuEop, request_id); if (result != OK) { return result; } - VideoOut::g_video_out_context->GetFlipQueue().Prepare(request_id, buffer); + m_impl->GetFlipQueue().Prepare(request_id, buffer); return OK; } -uint64_t DisplayBufferPrepareNextFlipOnGpu(Graphics::CommandBuffer& buffer) { - EXIT_IF(VideoOut::g_video_out_context == nullptr); - return VideoOut::g_video_out_context->GetFlipQueue().PrepareNextCpu(buffer); +void VideoOutDriver::SubmitFlipPreparation(uint64_t request_id) { + m_impl->Renderer().GetGpu().SubmitFlipPreparation(request_id); } -void DisplayBufferCompleteFlipFromGpu(uint64_t request_id) { - EXIT_IF(VideoOut::g_video_out_context == nullptr); - VideoOut::g_video_out_context->GetFlipQueue().Complete(request_id); +void VideoOutDriver::PrepareFlip(uint64_t request_id, Graphics::CommandBuffer& buffer) { + m_impl->GetFlipQueue().Prepare(request_id, buffer); } -void DisplayBufferWaitForFlipQueueSlot() { - EXIT_IF(VideoOut::g_video_out_context == nullptr); - VideoOut::g_video_out_context->GetFlipQueue().WaitForSubmitSlot(); +void VideoOutDriver::CompleteFlip(uint64_t request_id) { + m_impl->GetFlipQueue().Complete(request_id); } -} // namespace Libs::Presentation +void VideoOutDriver::WaitForSubmitSlot() { + m_impl->GetFlipQueue().WaitForSubmitSlot(); +} -namespace Libs::VideoOut { - -void VideoOutWaitFlipDone(int handle, int index) { - EXIT_IF(g_video_out_context == nullptr); - - auto* ctx = g_video_out_context->Get(handle); +void VideoOutDriver::WaitFlipDone(int handle, int index) { + auto* ctx = m_impl->Get(handle); EXIT_IF(ctx == nullptr); EXIT_NOT_IMPLEMENTED(!IsValidBufferIndex(index)); - g_video_out_context->GetFlipQueue().Wait(*ctx, index); + m_impl->GetFlipQueue().Wait(*ctx, index); } KYTY_SYSV_ABI int VideoOutGetFlipStatus(int handle, VideoOutFlipStatus* status) { PRINT_NAME(); - EXIT_IF(g_video_out_context == nullptr); - if (status == nullptr) { return VIDEO_OUT_ERROR_INVALID_ADDRESS; } - auto* ctx = g_video_out_context->Get(handle); + auto* ctx = DriverState().Get(handle); if (ctx == nullptr) { return VIDEO_OUT_ERROR_INVALID_HANDLE; } - g_video_out_context->GetFlipQueue().GetFlipStatus(*ctx, *status); + DriverState().GetFlipQueue().GetFlipStatus(*ctx, *status); LOGF("\t count = %" PRIu64 "\n" "\t processTime = %" PRIu64 "\n" @@ -1408,15 +1528,13 @@ KYTY_SYSV_ABI int VideoOutGetFlipStatus(int handle, VideoOutFlipStatus* status) KYTY_SYSV_ABI int VideoOutIsFlipPending(int handle) { PRINT_NAME(); - EXIT_IF(g_video_out_context == nullptr); - - auto* ctx = g_video_out_context->Get(handle); + auto* ctx = DriverState().Get(handle); if (ctx == nullptr) { return VIDEO_OUT_ERROR_INVALID_HANDLE; } VideoOutFlipStatus status {}; - g_video_out_context->GetFlipQueue().GetFlipStatus(*ctx, status); + DriverState().GetFlipQueue().GetFlipStatus(*ctx, status); LOGF("\t flipPendingNum = %d\n", status.flipPendingNum); @@ -1426,13 +1544,11 @@ KYTY_SYSV_ABI int VideoOutIsFlipPending(int handle) { KYTY_SYSV_ABI int VideoOutGetVblankStatus(int handle, VideoOutVblankStatus* status) { PRINT_NAME(); - EXIT_IF(g_video_out_context == nullptr); - if (status == nullptr) { return VIDEO_OUT_ERROR_INVALID_ADDRESS; } - auto* ctx = g_video_out_context->Get(handle); + auto* ctx = DriverState().Get(handle); if (ctx == nullptr) { return VIDEO_OUT_ERROR_INVALID_HANDLE; } @@ -1508,15 +1624,16 @@ KYTY_SYSV_ABI int VideoOutGetEventCount(const EventQueue::KernelEvent* ev) { KYTY_SYSV_ABI int VideoOutWaitVblank(int handle) { PRINT_NAME(); - EXIT_IF(g_video_out_context == nullptr); - - [[maybe_unused]] auto* ctx = g_video_out_context->Get(handle); + auto* ctx = DriverState().Get(handle); if (ctx == nullptr) { return VIDEO_OUT_ERROR_INVALID_HANDLE; } - WaitForNextVblank(); - g_video_out_context->VblankEnd(); + Common::LockGuard lock(ctx->mutex); + const auto count = ctx->vblank_status.count; + while (ctx->opened && ctx->vblank_status.count == count) { + ctx->vblank_cond.Wait(&ctx->mutex); + } return OK; } @@ -1524,13 +1641,11 @@ KYTY_SYSV_ABI int VideoOutWaitVblank(int handle) { KYTY_SYSV_ABI int VideoOutGetOutputStatus(int handle, VideoOutOutputStatus* status) { PRINT_NAME(); - EXIT_IF(g_video_out_context == nullptr); - if (status == nullptr) { return VIDEO_OUT_ERROR_INVALID_ADDRESS; } - auto* ctx = g_video_out_context->Get(handle); + auto* ctx = DriverState().Get(handle); if (ctx == nullptr) { return VIDEO_OUT_ERROR_INVALID_HANDLE; } @@ -1553,9 +1668,7 @@ KYTY_SYSV_ABI int VideoOutGetOutputStatus(int handle, VideoOutOutputStatus* stat static int ValidateOutputConfig(int handle, uint64_t mode, const VideoOutOutputOptions* options, void* reserved_ptr, uint64_t reserved) { - EXIT_IF(g_video_out_context == nullptr); - - if (!g_video_out_context->IsOpened(handle)) { + if (!DriverState().IsOpened(handle)) { return VIDEO_OUT_ERROR_INVALID_HANDLE; } @@ -1624,15 +1737,15 @@ KYTY_SYSV_ABI int VideoOutConfigureOutput(int handle, uint64_t mode, return VIDEO_OUT_ERROR_UNAVAILABLE_OUTPUT_MODE; } - auto* ctx = g_video_out_context->Get(handle); + auto* ctx = DriverState().Get(handle); if (ctx == nullptr) { return VIDEO_OUT_ERROR_INVALID_HANDLE; } ctx->mutex.Lock(); ctx->output_mode = mode; - TriggerVideoOutEventsLocked(ctx->output_mode_eqs, VideoOutEventKind::OutputMode, - reinterpret_cast(ctx->output_mode)); + TriggerVideoOutEvents(*ctx, VideoOutEventKind::OutputMode, + reinterpret_cast(ctx->output_mode)); ctx->mutex.Unlock(); return OK; @@ -1641,9 +1754,7 @@ KYTY_SYSV_ABI int VideoOutConfigureOutput(int handle, uint64_t mode, KYTY_SYSV_ABI int VideoOutSetWindowModeMargins(int handle, int top, int bottom) { PRINT_NAME(); - EXIT_IF(g_video_out_context == nullptr); - - [[maybe_unused]] auto* ctx = g_video_out_context->Get(handle); + [[maybe_unused]] auto* ctx = DriverState().Get(handle); if (ctx == nullptr) { return VIDEO_OUT_ERROR_INVALID_HANDLE; } @@ -1658,9 +1769,7 @@ KYTY_SYSV_ABI int VideoOutSetWindowModeMargins(int handle, int top, int bottom) KYTY_SYSV_ABI int VideoOutLatencyControlWaitBeforeInput(int handle) { PRINT_NAME(); - EXIT_IF(g_video_out_context == nullptr); - - [[maybe_unused]] auto* ctx = g_video_out_context->Get(handle); + [[maybe_unused]] auto* ctx = DriverState().Get(handle); if (ctx == nullptr) { return VIDEO_OUT_ERROR_INVALID_HANDLE; } @@ -1671,9 +1780,7 @@ KYTY_SYSV_ABI int VideoOutLatencyControlWaitBeforeInput(int handle) { KYTY_SYSV_ABI int VideoOutLatencyMeasureSetStartPoint(int handle, uint32_t point) { PRINT_NAME(); - EXIT_IF(g_video_out_context == nullptr); - - [[maybe_unused]] auto* ctx = g_video_out_context->Get(handle); + [[maybe_unused]] auto* ctx = DriverState().Get(handle); if (ctx == nullptr) { return VIDEO_OUT_ERROR_INVALID_HANDLE; } @@ -1701,17 +1808,15 @@ KYTY_SYSV_ABI int VideoOutColorSettingsSetGamma(VideoOutColorSettings* settings, KYTY_SYSV_ABI int VideoOutAdjustColor(int handle, const VideoOutColorSettings* settings) { PRINT_NAME(); - EXIT_IF(g_video_out_context == nullptr); - if (settings == nullptr) { return VIDEO_OUT_ERROR_INVALID_ADDRESS; } - if (!g_video_out_context->IsOpened(handle)) { + if (!DriverState().IsOpened(handle)) { return VIDEO_OUT_ERROR_INVALID_HANDLE; } - auto* ctx = g_video_out_context->Get(handle); + auto* ctx = DriverState().Get(handle); if (ctx == nullptr) { return VIDEO_OUT_ERROR_INVALID_HANDLE; } diff --git a/src/graphics/presentation/videoOut.h b/src/graphics/presentation/videoOut.h index 04666dc8..a9a598af 100644 --- a/src/graphics/presentation/videoOut.h +++ b/src/graphics/presentation/videoOut.h @@ -7,9 +7,11 @@ #include "common/abi.h" #include "kernel/eventQueue.h" +#include + namespace Libs::Graphics { -// struct VulkanSwapchain; -struct VideoOutVulkanImage; +class CommandBuffer; +class Presenter; } // namespace Libs::Graphics namespace Libs::VideoOut { @@ -22,8 +24,31 @@ struct VideoOutOutputOptions; struct VideoOutBuffers; struct VideoOutColorSettings; -void VideoOutInit(uint32_t width, uint32_t height); -void VideoOutWaitFlipDone(int handle, int index); +class VideoOutDriver final { +public: + struct Impl; + + VideoOutDriver(uint32_t width, uint32_t height, Graphics::Presenter& presenter); + ~VideoOutDriver(); + KYTY_CLASS_NO_COPY(VideoOutDriver); + + int SubmitFlipFromGpu(Graphics::CommandBuffer& buffer, int handle, int index, int flip_mode, + int64_t flip_arg, uint64_t& request_id); + void PrepareFlip(uint64_t request_id, Graphics::CommandBuffer& buffer); + void CompleteFlip(uint64_t request_id); + void SubmitFlipPreparation(uint64_t request_id); + void WaitForSubmitSlot(); + void WaitFlipDone(int handle, int index); + + [[nodiscard]] Impl& State() noexcept; + +private: + std::unique_ptr m_impl; +}; + +[[nodiscard]] VideoOutDriver& VideoOutInit(uint32_t width, uint32_t height, + Graphics::Presenter& presenter); +void VideoOutShutdown(); KYTY_SYSV_ABI int VideoOutOpen(int user_id, int bus_type, int index, const void* param); KYTY_SYSV_ABI int VideoOutClose(int handle); @@ -75,10 +100,6 @@ KYTY_SYSV_ABI int VideoOutLatencyMeasureSetStartPoint(int handle, uint32_t point KYTY_SYSV_ABI int VideoOutColorSettingsSetGamma(VideoOutColorSettings* settings, float gamma); KYTY_SYSV_ABI int VideoOutAdjustColor(int handle, const VideoOutColorSettings* settings); -void VideoOutBeginVblank(); -void VideoOutEndVblank(); -bool VideoOutFlipWindow(uint32_t micros); - } // namespace Libs::VideoOut #endif /* EMULATOR_INCLUDE_EMULATOR_GRAPHICS_VIDEOOUT_H_ */ diff --git a/src/graphics/presentation/window.h b/src/graphics/presentation/window.h index 88ff6793..f70995aa 100644 --- a/src/graphics/presentation/window.h +++ b/src/graphics/presentation/window.h @@ -6,16 +6,11 @@ namespace Libs::Graphics { -class CommandBuffer; -struct VideoOutVulkanImage; -struct PreparedFrame; +class Presenter; -void WindowInit(uint32_t width, uint32_t height); -void WindowRun(); -PreparedFrame& WindowPrepareFrame(CommandBuffer& buffer, VideoOutVulkanImage& image); -PreparedFrame& WindowPrepareBlankFrame(CommandBuffer& buffer, uint32_t width, uint32_t height, - bool opaque); -void WindowPresentFrame(PreparedFrame& frame); +[[nodiscard]] Presenter& WindowInit(uint32_t width, uint32_t height); +void WindowRun(); +void WindowShutdown(); } // namespace Libs::Graphics diff --git a/src/graphics/presentation/window/swapchain.cpp b/src/graphics/presentation/window/swapchain.cpp index df979228..f2f6a517 100644 --- a/src/graphics/presentation/window/swapchain.cpp +++ b/src/graphics/presentation/window/swapchain.cpp @@ -29,12 +29,11 @@ #include "graphics/host_gpu/graphicContext.h" #include "graphics/host_gpu/renderer/render.h" #include "graphics/host_gpu/renderer/renderContext.h" -#include "graphics/host_gpu/transfer.h" #include "graphics/host_gpu/vma.h" #include "graphics/host_gpu/vulkanCommon.h" +#include "graphics/presentation/presenter.h" #include "graphics/presentation/renderDoc.h" #include "graphics/presentation/videoOut.h" -#include "graphics/presentation/window.h" #include "graphics/presentation/window/windowInternal.h" #include "libs/controller.h" #include "loader/systemContent.h" @@ -46,6 +45,7 @@ #include #include #include +#include #include #include #include @@ -58,27 +58,58 @@ namespace Libs::Graphics { -struct PreparedFrame { - VulkanImage image {VulkanImageType::Unknown}; +struct Presenter::Frame { + VulkanImage image; std::unique_ptr present_commands; bool busy = false; + bool reusing_last = false; + + void Configure(GraphicContext& graphics, vk::Extent2D extent, vk::Format format); + void Transit(vk::CommandBuffer command, vk::ImageLayout layout, vk::AccessFlags2 access); + void CopyFrom(CommandBuffer& command, Image& source); + void Clear(CommandBuffer& command, const vk::ClearColorValue& color); }; -class PreparedFramePool { +class FramePool final { public: - void EnsureCapacity(uint32_t count, vk::Format format) { + explicit FramePool(WindowContext& window): m_window(window) {} + ~FramePool() { + for (auto& frame: m_frames) { + if (frame->present_commands != nullptr) { + frame->present_commands->WaitForFenceOnly(); + } + if (frame->image.image != nullptr) { + m_window.graphic_ctx.DeleteImage(frame->image); + } + } + } + KYTY_CLASS_NO_COPY(FramePool); + + void Initialize(uint32_t count, vk::Format format) { if (count == 0 || format == vk::Format::eUndefined) { EXIT("prepared-frame pool requires at least one frame\n"); } Common::LockGuard lock(m_mutex); + if (!m_frames.empty()) { + EXIT("prepared-frame pool was initialized twice\n"); + } m_format = format; - while (m_frames.size() < count) { - auto frame = std::make_unique(); + m_frames.reserve(count); + for (uint32_t i = 0; i < count; i++) { + auto frame = std::make_unique(); m_free.push_back(frame.get()); m_frames.push_back(std::move(frame)); } } + void SetFormat(vk::Format format) { + if (format == vk::Format::eUndefined) { + EXIT("prepared-frame pool requires a presentation format\n"); + } + Common::LockGuard lock(m_mutex); + m_format = format; + } + vk::Format GetFormat() { Common::LockGuard lock(m_mutex); if (m_format == vk::Format::eUndefined) { @@ -87,7 +118,7 @@ public: return m_format; } - PreparedFrame* Acquire() { + Presenter::Frame* Acquire() { m_mutex.Lock(); if (m_frames.empty()) { EXIT("prepared-frame pool was used before swapchain initialization\n"); @@ -100,19 +131,47 @@ public: if (frame->busy) { EXIT("prepared-frame pool returned an invalid frame\n"); } - frame->busy = true; + if (m_last_frame == frame) { + m_last_frame = nullptr; + } + frame->busy = true; + frame->reusing_last = false; m_mutex.Unlock(); - // The producer only waits here. Reset stays on the presentation thread that owns the - // allocating Vulkan command pool. - if (frame->present_commands != nullptr) { - frame->present_commands->WaitForFenceOnly(); - } - + WaitForFrame(*frame); return frame; } - void Release(PreparedFrame* frame) { + Presenter::Frame* AcquireLast() { + m_mutex.Lock(); + auto* frame = m_last_frame; + if (frame == nullptr) { + m_mutex.Unlock(); + return nullptr; + } + auto free = std::find(m_free.begin(), m_free.end(), frame); + if (free == m_free.end() || frame->busy) { + m_mutex.Unlock(); + EXIT("last submitted frame is not available for reuse\n"); + } + m_free.erase(free); + m_last_frame = nullptr; + frame->busy = true; + frame->reusing_last = true; + m_mutex.Unlock(); + + WaitForFrame(*frame); + return frame; + } + + void ValidateForPresent(Presenter::Frame* frame, bool reuse) { + Common::LockGuard lock(m_mutex); + if (frame == nullptr || !frame->busy || frame->reusing_last != reuse) { + EXIT("prepared frame has invalid presentation ownership\n"); + } + } + + void Release(Presenter::Frame* frame, bool make_last = false) { if (frame == nullptr) { EXIT("cannot release a null prepared frame\n"); } @@ -120,48 +179,67 @@ public: if (!frame->busy) { EXIT("prepared frame was released twice\n"); } - frame->busy = false; + frame->busy = false; + frame->reusing_last = false; + if (make_last) { + m_last_frame = frame; + } m_free.push_back(frame); m_available.Signal(); } private: + static void WaitForFrame(Presenter::Frame& frame) { + // The producer only waits here. Reset stays on the presentation thread that owns the + // allocating Vulkan command pool. + if (frame.present_commands != nullptr) { + frame.present_commands->WaitForFenceOnly(); + } + } + + WindowContext& m_window; Common::Mutex m_mutex; Common::CondVar m_available; - std::vector> m_frames; - std::deque m_free; + std::vector> m_frames; + std::deque m_free; + Presenter::Frame* m_last_frame = nullptr; vk::Format m_format = vk::Format::eUndefined; }; -static PreparedFramePool* GetPreparedFramePool() { - static auto* pool = new PreparedFramePool; - return pool; -} - -static void ConfigurePreparedFrame(PreparedFrame& frame, vk::Extent2D extent, vk::Format format) { +void Presenter::Frame::Configure(GraphicContext& graphics, vk::Extent2D extent, + vk::Format format) { if (extent.width == 0 || extent.height == 0 || format == vk::Format::eUndefined) { EXIT("unsupported prepared frame, extent=%ux%u format=%d\n", extent.width, extent.height, static_cast(format)); } + const auto features = graphics.GetFormatProperties(format).optimalTilingFeatures; + const auto required = vk::FormatFeatureFlagBits::eBlitSrc | + vk::FormatFeatureFlagBits::eSampledImageFilterLinear | + vk::FormatFeatureFlagBits::eTransferSrc | + vk::FormatFeatureFlagBits::eTransferDst; + if ((features & required) != required) { + EXIT("prepared presentation format lacks optimal blit support: format=%d features=0x%x\n", + static_cast(format), + static_cast(features)); + } - auto& graphics = g_window_ctx->graphic_ctx; - auto& dst = frame.image; + auto& dst = image; const bool compatible = dst.image != nullptr && dst.extent.width == extent.width && dst.extent.height == extent.height && dst.format == format; if (compatible) { return; } if (dst.image != nullptr) { - EXIT_IF(!dst.view_cache.views.empty()); graphics.DeleteImage(dst); dst.memory = {}; } - dst.extent = extent; + dst.extent = {extent.width, extent.height, 1}; dst.format = format; dst.layers = 1; dst.mip_levels = 1; - dst.layout = vk::ImageLayout::eUndefined; + dst.state = {}; + dst.subresource_states.clear(); dst.memory.property = vk::MemoryPropertyFlagBits::eDeviceLocal; vk::ImageCreateInfo create {}; @@ -182,347 +260,588 @@ static void ConfigurePreparedFrame(PreparedFrame& frame, vk::Extent2D extent, vk } } -VulkanSwapchain::~VulkanSwapchain() = default; +void Presenter::Frame::Transit(vk::CommandBuffer command, vk::ImageLayout layout, + vk::AccessFlags2 access) { + const auto stage = access == vk::AccessFlagBits2::eTransferRead || + access == vk::AccessFlagBits2::eTransferWrite + ? vk::PipelineStageFlagBits2::eTransfer + : vk::PipelineStageFlagBits2::eAllCommands; + constexpr auto writes = vk::AccessFlagBits2::eTransferWrite | + vk::AccessFlagBits2::eShaderWrite | + vk::AccessFlagBits2::eMemoryWrite; + if (image.state.layout == layout && image.state.access_mask == access && + !static_cast(image.state.access_mask & writes)) { + return; + } + vk::ImageMemoryBarrier2 barrier {}; + barrier.srcStageMask = image.state.pl_stage; + barrier.srcAccessMask = image.state.access_mask; + barrier.dstStageMask = stage; + barrier.dstAccessMask = access; + barrier.oldLayout = image.state.layout; + barrier.newLayout = layout; + barrier.srcQueueFamilyIndex = VK_QUEUE_FAMILY_IGNORED; + barrier.dstQueueFamilyIndex = VK_QUEUE_FAMILY_IGNORED; + barrier.image = image.image; + barrier.subresourceRange.aspectMask = vk::ImageAspectFlagBits::eColor; + barrier.subresourceRange.baseMipLevel = 0; + barrier.subresourceRange.levelCount = VK_REMAINING_MIP_LEVELS; + barrier.subresourceRange.baseArrayLayer = 0; + barrier.subresourceRange.layerCount = VK_REMAINING_ARRAY_LAYERS; + vk::DependencyInfo dependency {}; + dependency.imageMemoryBarrierCount = 1; + dependency.pImageMemoryBarriers = &barrier; + command.pipelineBarrier2(dependency); + image.state = {stage, access, layout}; + image.subresource_states.clear(); +} -[[maybe_unused]] static vk::SwapchainKHR VulkanCreateSwapchainInternal( - vk::Device device, vk::SurfaceKHR surface, uint32_t width, uint32_t height, - uint32_t image_count, SurfaceCapabilities& r, vk::Format& swapchain_format, - vk::Extent2D& swapchain_extent, std::unique_ptr& swapchain_images, - std::unique_ptr& swapchain_image_views, uint32_t& swapchain_images_count) { - EXIT_IF(device == nullptr); - EXIT_IF(surface == nullptr); +void Presenter::Frame::CopyFrom(CommandBuffer& command_buffer, Image& source) { + command_buffer.EndRendering(); + auto command = command_buffer.Handle(); + source.Transit(vk::ImageLayout::eTransferSrcOptimal, + vk::AccessFlagBits2::eTransferRead, {}, command); + Transit(command, vk::ImageLayout::eTransferDstOptimal, + vk::AccessFlagBits2::eTransferWrite); + vk::ImageCopy copy {}; + copy.srcSubresource = {vk::ImageAspectFlagBits::eColor, 0, 0, + source.backing.layers}; + copy.dstSubresource = {vk::ImageAspectFlagBits::eColor, 0, 0, image.layers}; + copy.extent = {std::min(source.backing.extent.width, image.extent.width), + std::min(source.backing.extent.height, image.extent.height), 1}; + EXIT_IF(copy.srcSubresource.layerCount != copy.dstSubresource.layerCount); + command.copyImage(source.backing.image, vk::ImageLayout::eTransferSrcOptimal, + image.image, vk::ImageLayout::eTransferDstOptimal, copy); + Transit(command, vk::ImageLayout::eTransferSrcOptimal, + vk::AccessFlagBits2::eTransferRead); +} - EXIT_NOT_IMPLEMENTED(r.formats.empty()); +void Presenter::Frame::Clear(CommandBuffer& command_buffer, + const vk::ClearColorValue& color) { + command_buffer.EndRendering(); + auto command = command_buffer.Handle(); + Transit(command, vk::ImageLayout::eTransferDstOptimal, + vk::AccessFlagBits2::eTransferWrite); + const vk::ImageSubresourceRange range { + vk::ImageAspectFlagBits::eColor, 0, 1, 0, 1}; + command.clearColorImage(image.image, vk::ImageLayout::eTransferDstOptimal, &color, 1, + &range); + Transit(command, vk::ImageLayout::eTransferSrcOptimal, + vk::AccessFlagBits2::eTransferRead); +} - vk::Extent2D extent {}; - extent.width = - std::clamp(width, r.capabilities.minImageExtent.width, r.capabilities.maxImageExtent.width); - extent.height = std::clamp(height, r.capabilities.minImageExtent.height, - r.capabilities.maxImageExtent.height); +class Swapchain final { +public: + enum class Status : uint8_t { Success, Recreate, SurfaceLost }; - image_count = - std::clamp(image_count, r.capabilities.minImageCount, r.capabilities.maxImageCount); + explicit Swapchain(WindowContext& window): m_window(window) {} + ~Swapchain(); + KYTY_CLASS_NO_COPY(Swapchain); - vk::SwapchainCreateInfoKHR create_info {}; - create_info.sType = vk::StructureType::eSwapchainCreateInfoKHR; - create_info.pNext = nullptr; - create_info.flags = {}; - create_info.surface = surface; - create_info.minImageCount = image_count; + void Create(); + void Recreate(bool surface_lost = false); + [[nodiscard]] Status AcquireNextImage(); + void RecordPresentCommands(CommandBuffer& command, VulkanImage& source); + void Submit(CommandBuffer& command); + [[nodiscard]] Status Present(); - if (r.format_unorm_bgra32) { - create_info.imageFormat = vk::Format::eB8G8R8A8Unorm; - create_info.imageColorSpace = vk::ColorSpaceKHR::eSrgbNonlinear; - } else if (r.format_srgb_bgra32) { - create_info.imageFormat = vk::Format::eB8G8R8A8Srgb; - create_info.imageColorSpace = vk::ColorSpaceKHR::eSrgbNonlinear; - } else { - const auto& format = r.formats[0]; - create_info.imageFormat = format.format; - create_info.imageColorSpace = format.colorSpace; + [[nodiscard]] uint32_t ImageCount() const noexcept { + return static_cast(m_images.size()); + } + [[nodiscard]] vk::Format Format() const noexcept { return m_format; } + +private: + void Destroy(); + void RefreshSurfaceSize(); + + WindowContext& m_window; + vk::SwapchainKHR m_handle = nullptr; + vk::Format m_format = vk::Format::eUndefined; + vk::Extent2D m_extent {}; + std::vector m_images; + std::vector m_image_views; + std::vector m_image_acquired; + std::vector m_render_complete; + uint32_t m_image_index = static_cast(-1); + uint32_t m_frame_index = 0; +}; + +struct Presenter::Impl { + explicit Impl(WindowContext& owner) + : renderer(*owner.render_context), window(owner), swapchain(owner), + present_scheduler(renderer, owner.graphic_ctx), frames(owner) { + EXIT_IF(owner.render_context == nullptr); + swapchain.Create(); + frames.Initialize(swapchain.ImageCount(), swapchain.Format()); } - create_info.imageExtent = extent; + void RecoverSwapchain(Swapchain::Status status) { + LOGF("Recovering Vulkan swapchain%s\n", + status == Swapchain::Status::SurfaceLost ? " and surface" : ""); + swapchain.Recreate(status == Swapchain::Status::SurfaceLost); + frames.SetFormat(swapchain.Format()); + } + + Image& ResolveSurface(const ImageInfo& info) { + TextureCache::ImageDesc desc {}; + desc.info = info; + desc.view_info.format = info.pixel_format; + desc.view_info.type = vk::ImageViewType::e2D; + desc.view_info.aspect = vk::ImageAspectFlagBits::eColor; + desc.view_info.base_level = 0; + desc.view_info.level_count = 1; + desc.view_info.base_layer = 0; + desc.view_info.layer_count = 1; + desc.view_info.usage = vk::ImageUsageFlagBits::eTransferSrc; + desc.type = TextureCache::BindingType::VideoOut; + + auto& cache = renderer.GetTextureCache(); + auto& image = cache.GetImage(cache.FindImage(desc)); + image.usage.video_out = true; + return image; + } + + RenderContext& renderer; + WindowContext& window; + Swapchain swapchain; + CommandScheduler present_scheduler; + FramePool frames; +}; + +void Swapchain::Create() { + auto& graphics = m_window.graphic_ctx; + EXIT_IF(graphics.screen_width == 0); + EXIT_IF(graphics.screen_height == 0); + EXIT_IF(graphics.device == nullptr); + EXIT_IF(m_window.surface == nullptr); + + Common::LockGuard lock(m_window.mutex); + const auto& surface = m_window.surface_capabilities; + EXIT_NOT_IMPLEMENTED(surface.formats.empty()); + + m_extent = surface.capabilities.currentExtent; + if (m_extent.width == std::numeric_limits::max()) { + m_extent.width = + std::clamp(graphics.screen_width, surface.capabilities.minImageExtent.width, + surface.capabilities.maxImageExtent.width); + m_extent.height = + std::clamp(graphics.screen_height, surface.capabilities.minImageExtent.height, + surface.capabilities.maxImageExtent.height); + } + uint32_t image_count = surface.capabilities.minImageCount + 1; + if (surface.capabilities.maxImageCount != 0) { + image_count = std::min(image_count, surface.capabilities.maxImageCount); + } + const auto transform = + surface.capabilities.supportedTransforms & vk::SurfaceTransformFlagBitsKHR::eIdentity + ? vk::SurfaceTransformFlagBitsKHR::eIdentity + : surface.capabilities.currentTransform; + const auto composite = + surface.capabilities.supportedCompositeAlpha & vk::CompositeAlphaFlagBitsKHR::eOpaque + ? vk::CompositeAlphaFlagBitsKHR::eOpaque + : vk::CompositeAlphaFlagBitsKHR::eInherit; + + vk::SurfaceFormatKHR format {vk::Format::eR8G8B8A8Unorm, + vk::ColorSpaceKHR::eSrgbNonlinear}; + if (surface.formats.size() != 1 || + surface.formats.front().format != vk::Format::eUndefined) { + const auto it = std::find_if(surface.formats.begin(), surface.formats.end(), + [](const vk::SurfaceFormatKHR& candidate) { + return candidate.format == + vk::Format::eB8G8R8A8Unorm || + candidate.format == + vk::Format::eR8G8B8A8Unorm; + }); + if (it == surface.formats.end()) { + EXIT("no supported UNORM swapchain format\n"); + } + format = *it; + } + m_format = format.format; + const auto swapchain_features = + graphics.GetFormatProperties(m_format).optimalTilingFeatures; + if (!static_cast(swapchain_features & vk::FormatFeatureFlagBits::eBlitDst)) { + EXIT("swapchain format cannot be a blit destination: format=%d\n", + static_cast(m_format)); + } + + vk::SwapchainCreateInfoKHR create_info {}; + create_info.sType = vk::StructureType::eSwapchainCreateInfoKHR; + create_info.surface = m_window.surface; + create_info.minImageCount = image_count; + create_info.imageFormat = format.format; + create_info.imageColorSpace = format.colorSpace; + create_info.imageExtent = m_extent; create_info.imageArrayLayers = 1; create_info.imageUsage = vk::ImageUsageFlagBits::eColorAttachment | vk::ImageUsageFlagBits::eTransferDst; create_info.imageSharingMode = vk::SharingMode::eExclusive; - create_info.preTransform = r.capabilities.currentTransform; - create_info.compositeAlpha = vk::CompositeAlphaFlagBitsKHR::eOpaque; + create_info.preTransform = transform; + create_info.compositeAlpha = composite; create_info.presentMode = vk::PresentModeKHR::eFifo; create_info.clipped = VK_TRUE; - create_info.oldSwapchain = nullptr; - - swapchain_format = create_info.imageFormat; - swapchain_extent = extent; - - vk::SwapchainKHR swapchain = nullptr; - - RequireVulkanSuccess(device.createSwapchainKHR(&create_info, nullptr, &swapchain), + RequireVulkanSuccess(graphics.device.createSwapchainKHR(&create_info, nullptr, &m_handle), "vkCreateSwapchainKHR"); - EXIT_IF(swapchain == nullptr); + EXIT_IF(m_handle == nullptr); - auto images = EnumerateVulkan( - "vkGetSwapchainImagesKHR", [&](uint32_t* count, vk::Image* values) { - return device.getSwapchainImagesKHR(swapchain, count, values); + m_images = EnumerateVulkan( + "vkGetSwapchainImagesKHR", [&](uint32_t* count, vk::Image* images) { + return graphics.device.getSwapchainImagesKHR(m_handle, count, images); }); - EXIT_NOT_IMPLEMENTED(images.empty()); - - swapchain_images_count = static_cast(images.size()); - swapchain_images = std::make_unique(images.size()); - std::copy(images.begin(), images.end(), swapchain_images.get()); - - swapchain_image_views = std::make_unique(swapchain_images_count); - for (uint32_t i = 0; i < swapchain_images_count; i++) { - vk::ImageViewCreateInfo create_info {}; - create_info.sType = vk::StructureType::eImageViewCreateInfo; - create_info.pNext = nullptr; - create_info.flags = {}; - create_info.image = (swapchain_images)[i]; - create_info.viewType = vk::ImageViewType::e2D; - create_info.format = swapchain_format; - create_info.components.r = vk::ComponentSwizzle::eIdentity; - create_info.components.g = vk::ComponentSwizzle::eIdentity; - create_info.components.b = vk::ComponentSwizzle::eIdentity; - create_info.components.a = vk::ComponentSwizzle::eIdentity; - create_info.subresourceRange.aspectMask = vk::ImageAspectFlagBits::eColor; - create_info.subresourceRange.baseArrayLayer = 0; - create_info.subresourceRange.baseMipLevel = 0; - create_info.subresourceRange.layerCount = 1; - create_info.subresourceRange.levelCount = 1; + EXIT_NOT_IMPLEMENTED(m_images.empty()); + m_image_views.resize(m_images.size()); + for (size_t i = 0; i < m_images.size(); i++) { + vk::ImageViewCreateInfo view {}; + view.sType = vk::StructureType::eImageViewCreateInfo; + view.image = m_images[i]; + view.viewType = vk::ImageViewType::e2D; + view.format = m_format; + view.components = {}; + view.subresourceRange.aspectMask = vk::ImageAspectFlagBits::eColor; + view.subresourceRange.baseArrayLayer = 0; + view.subresourceRange.baseMipLevel = 0; + view.subresourceRange.layerCount = 1; + view.subresourceRange.levelCount = 1; RequireVulkanSuccess( - device.createImageView(&create_info, nullptr, &((swapchain_image_views)[i])), + graphics.device.createImageView(&view, nullptr, &m_image_views[i]), "vkCreateImageView"); - EXIT_IF((swapchain_image_views)[i] == nullptr); + EXIT_IF(m_image_views[i] == nullptr); } - return swapchain; -} - -VulkanSwapchain* VulkanCreateSwapchain(uint32_t image_count) { - auto& graphics = g_window_ctx->graphic_ctx; - EXIT_IF(graphics.screen_width == 0); - EXIT_IF(graphics.screen_height == 0); - - Common::LockGuard lock(g_window_ctx->mutex); - - auto swapchain_owner = std::make_unique(); - auto* s = swapchain_owner.get(); - - s->swapchain = VulkanCreateSwapchainInternal( - graphics.device, g_window_ctx->surface, graphics.screen_width, graphics.screen_height, - image_count, *g_window_ctx->surface_capabilities, s->swapchain_format, s->swapchain_extent, - s->swapchain_images, s->swapchain_image_views, s->swapchain_images_count); - if (s->swapchain == nullptr) { - EXIT("Could not create swapchain"); - } - - s->current_index = static_cast(-1); - s->present_frame = 0; - vk::SemaphoreCreateInfo semaphore_info {}; semaphore_info.sType = vk::StructureType::eSemaphoreCreateInfo; - semaphore_info.pNext = nullptr; - semaphore_info.flags = {}; - - s->image_acquired_semaphores = std::make_unique(s->swapchain_images_count); - s->render_complete_semaphores = std::make_unique(s->swapchain_images_count); - for (uint32_t i = 0; i < s->swapchain_images_count; i++) { - auto result = graphics.device.createSemaphore(&semaphore_info, nullptr, - &s->image_acquired_semaphores[i]); - EXIT_NOT_IMPLEMENTED(result != vk::Result::eSuccess); - - result = graphics.device.createSemaphore(&semaphore_info, nullptr, - &s->render_complete_semaphores[i]); - EXIT_NOT_IMPLEMENTED(result != vk::Result::eSuccess); + m_image_acquired.resize(m_images.size()); + m_render_complete.resize(m_images.size()); + for (size_t i = 0; i < m_images.size(); i++) { + RequireVulkanSuccess( + graphics.device.createSemaphore(&semaphore_info, nullptr, &m_image_acquired[i]), + "create swapchain image-acquired semaphore"); + RequireVulkanSuccess( + graphics.device.createSemaphore(&semaphore_info, nullptr, &m_render_complete[i]), + "create swapchain render-complete semaphore"); } - GetPreparedFramePool()->EnsureCapacity(s->swapchain_images_count, s->swapchain_format); - - return swapchain_owner.release(); + m_image_index = static_cast(-1); + m_frame_index = 0; } -static void VulkanDeleteSwapchain(VulkanSwapchain* s) { - if (s == nullptr) { +Swapchain::~Swapchain() { + Destroy(); +} + +void Swapchain::Destroy() { + if (m_handle == nullptr && m_image_acquired.empty() && m_render_complete.empty() && + m_image_views.empty()) { return; } - auto swapchain_owner = std::unique_ptr(s); - auto& graphics = g_window_ctx->graphic_ctx; + auto& graphics = m_window.graphic_ctx; - Transfer::WaitForQueueIdle(); - - if (s->image_acquired_semaphores != nullptr) { - for (uint32_t i = 0; i < s->swapchain_images_count; i++) { - if (s->image_acquired_semaphores[i] != nullptr) { - graphics.device.destroySemaphore(s->image_acquired_semaphores[i], nullptr); - } - } - } - if (s->render_complete_semaphores != nullptr) { - for (uint32_t i = 0; i < s->swapchain_images_count; i++) { - if (s->render_complete_semaphores[i] != nullptr) { - graphics.device.destroySemaphore(s->render_complete_semaphores[i], nullptr); - } - } - } - if (s->swapchain_image_views != nullptr) { - for (uint32_t i = 0; i < s->swapchain_images_count; i++) { - if (s->swapchain_image_views[i] != nullptr) { - graphics.device.destroyImageView(s->swapchain_image_views[i], nullptr); - } - } + { + Common::LockGuard queue_lock(graphics.queue_mutex); + RequireVulkanSuccess(graphics.queue.waitIdle(), "wait for swapchain queue"); } - if (s->swapchain != nullptr) { - graphics.device.destroySwapchainKHR(s->swapchain, nullptr); + for (const auto semaphore: m_image_acquired) { + if (semaphore != nullptr) { + graphics.device.destroySemaphore(semaphore, nullptr); + } } + for (const auto semaphore: m_render_complete) { + if (semaphore != nullptr) { + graphics.device.destroySemaphore(semaphore, nullptr); + } + } + for (const auto view: m_image_views) { + if (view != nullptr) { + graphics.device.destroyImageView(view, nullptr); + } + } + if (m_handle != nullptr) { + graphics.device.destroySwapchainKHR(m_handle, nullptr); + } + + m_handle = nullptr; + m_format = vk::Format::eUndefined; + m_extent = {}; + m_image_index = static_cast(-1); + m_frame_index = 0; + m_images.clear(); + m_image_views.clear(); + m_image_acquired.clear(); + m_render_complete.clear(); } -static void VulkanRefreshSurfaceSize() { +void Swapchain::RefreshSurfaceSize() { int width = 0; int height = 0; - SDL_Vulkan_GetDrawableSize(g_window_ctx->window, &width, &height); + SDL_Vulkan_GetDrawableSize(m_window.window, &width, &height); if (width > 0 && height > 0) { - g_window_ctx->graphic_ctx.screen_width = static_cast(width); - g_window_ctx->graphic_ctx.screen_height = static_cast(height); + m_window.graphic_ctx.screen_width = static_cast(width); + m_window.graphic_ctx.screen_height = static_cast(height); } - VulkanGetSurfaceCapabilities(g_window_ctx->graphic_ctx.physical_device, g_window_ctx->surface, - *g_window_ctx->surface_capabilities); + m_window.RefreshSurfaceCapabilities(); } -static void VulkanRecreateSwapchain() { - LOGF("Recreating Vulkan swapchain\n"); - VulkanRefreshSurfaceSize(); - VulkanDeleteSwapchain(g_window_ctx->swapchain); - g_window_ctx->swapchain = VulkanCreateSwapchain(2); -} - -PreparedFrame& WindowPrepareFrame(CommandBuffer& buffer, VideoOutVulkanImage& image) { - KYTY_PROFILER_FUNCTION(); - EXIT_IF(buffer.IsInvalid()); - if (image.format == vk::Format::eUndefined) { - EXIT("unsupported presentation source, image=%p\n", static_cast(&image)); +void Swapchain::Recreate(bool surface_lost) { + Destroy(); + if (surface_lost) { + m_window.RecreateSurface(); } - - auto* frame = GetPreparedFramePool()->Acquire(); - Common::LockGuard render_lock(GetRenderContext().GetMutex()); - GetRenderContext().GetTextureCache().RefreshVideoOut(image); - ConfigurePreparedFrame(*frame, image.extent, image.format); - ImageImageCopy copy {image}; - copy.width = image.extent.width; - copy.height = image.extent.height; - const std::array copies {copy}; - Transfer::CopyImage(buffer, copies, frame->image, vk::ImageLayout::eTransferSrcOptimal); - return *frame; + RefreshSurfaceSize(); + Create(); } -PreparedFrame& WindowPrepareBlankFrame(CommandBuffer& buffer, uint32_t width, uint32_t height, - bool opaque) { - KYTY_PROFILER_FUNCTION(); - EXIT_IF(buffer.IsInvalid()); - auto* pool = GetPreparedFramePool(); - auto format = pool->GetFormat(); - auto* frame = pool->Acquire(); - Common::LockGuard render_lock(GetRenderContext().GetMutex()); - ConfigurePreparedFrame(*frame, {width, height}, format); - vk::ClearColorValue clear {}; - clear.float32[3] = opaque ? 1.0f : 0.0f; - Transfer::ClearColorImage(buffer, frame->image, clear); - return *frame; -} - -void WindowPresentFrame(PreparedFrame& frame) { - KYTY_PROFILER_FUNCTION(); - - struct ReleaseScope { - PreparedFrame* frame; - ~ReleaseScope() { GetPreparedFramePool()->Release(frame); } - }; - ReleaseScope release {&frame}; - - if (g_window_ctx->window_hidden) { - WindowUpdateIcon(); - - SDL_ShowWindow(g_window_ctx->window); - - g_window_ctx->window_hidden = false; - VulkanRecreateSwapchain(); - } - - auto* swapchain = g_window_ctx->swapchain; - - const auto present_frame = swapchain->present_frame; - EXIT_IF(present_frame >= swapchain->swapchain_images_count); - - swapchain->current_index = static_cast(-1); - - auto result = g_window_ctx->graphic_ctx.device.acquireNextImageKHR( - swapchain->swapchain, UINT64_MAX, swapchain->image_acquired_semaphores[present_frame], - nullptr, &swapchain->current_index); - +Swapchain::Status Swapchain::AcquireNextImage() { + EXIT_IF(m_handle == nullptr || m_frame_index >= m_image_acquired.size()); + m_image_index = static_cast(-1); + const auto result = m_window.graphic_ctx.device.acquireNextImageKHR( + m_handle, std::numeric_limits::max(), m_image_acquired[m_frame_index], nullptr, + &m_image_index); switch (result) { case vk::Result::eSuccess: break; case vk::Result::eSuboptimalKHR: LOGF("vkAcquireNextImageKHR returned vk::Result::eSuboptimalKHR\n"); - break; + return Status::Recreate; case vk::Result::eErrorOutOfDateKHR: LOGF("vkAcquireNextImageKHR returned vk::Result::eErrorOutOfDateKHR\n"); - VulkanRecreateSwapchain(); - return; + return Status::Recreate; + case vk::Result::eErrorUnknown: + LOGF("vkAcquireNextImageKHR returned vk::Result::eErrorUnknown\n"); + return Status::Recreate; + case vk::Result::eErrorSurfaceLostKHR: + LOGF("vkAcquireNextImageKHR returned vk::Result::eErrorSurfaceLostKHR\n"); + return Status::SurfaceLost; default: EXIT("vkAcquireNextImageKHR failed: %s\n", VulkanToString(result).c_str()); } - EXIT_NOT_IMPLEMENTED(swapchain->current_index == static_cast(-1)); - if (frame.present_commands == nullptr) { - frame.present_commands = std::make_unique(); + EXIT_IF(m_image_index >= m_images.size()); + return Status::Success; +} + +void Swapchain::RecordPresentCommands(CommandBuffer& command, VulkanImage& source) { + if (source.state.layout != vk::ImageLayout::eTransferSrcOptimal) { + EXIT("invalid prepared presentation image, vk_image=%p layout=%d\n", + static_cast(source.image), static_cast(source.state.layout)); } - frame.present_commands->WaitForFenceAndReset(); - auto& buffer = *frame.present_commands; + EXIT_IF(m_image_index >= m_images.size()); + auto vk_command = command.Handle(); + command.Begin(); - auto vk_buffer = buffer.Handle(); + vk::ImageMemoryBarrier to_transfer {}; + to_transfer.sType = vk::StructureType::eImageMemoryBarrier; + to_transfer.dstAccessMask = vk::AccessFlagBits::eTransferWrite; + to_transfer.oldLayout = vk::ImageLayout::eUndefined; + to_transfer.newLayout = vk::ImageLayout::eTransferDstOptimal; + to_transfer.srcQueueFamilyIndex = VK_QUEUE_FAMILY_IGNORED; + to_transfer.dstQueueFamilyIndex = VK_QUEUE_FAMILY_IGNORED; + to_transfer.image = m_images[m_image_index]; + to_transfer.subresourceRange.aspectMask = vk::ImageAspectFlagBits::eColor; + to_transfer.subresourceRange.baseMipLevel = 0; + to_transfer.subresourceRange.levelCount = 1; + to_transfer.subresourceRange.baseArrayLayer = 0; + to_transfer.subresourceRange.layerCount = 1; + vk_command.pipelineBarrier(vk::PipelineStageFlagBits::eTopOfPipe, + vk::PipelineStageFlagBits::eTransfer, vk::DependencyFlags {}, 0, + nullptr, 0, nullptr, 1, &to_transfer); - buffer.Begin(); + vk::ImageBlit region {}; + region.srcSubresource.aspectMask = vk::ImageAspectFlagBits::eColor; + region.srcSubresource.mipLevel = 0; + region.srcSubresource.baseArrayLayer = 0; + region.srcSubresource.layerCount = 1; + region.srcOffsets[1].x = static_cast(source.extent.width); + region.srcOffsets[1].y = static_cast(source.extent.height); + region.srcOffsets[1].z = 1; + region.dstSubresource.aspectMask = vk::ImageAspectFlagBits::eColor; + region.dstSubresource.mipLevel = 0; + region.dstSubresource.baseArrayLayer = 0; + region.dstSubresource.layerCount = 1; + region.dstOffsets[1].x = static_cast(m_extent.width); + region.dstOffsets[1].y = static_cast(m_extent.height); + region.dstOffsets[1].z = 1; + vk_command.blitImage(source.image, vk::ImageLayout::eTransferSrcOptimal, + m_images[m_image_index], vk::ImageLayout::eTransferDstOptimal, 1, ®ion, + vk::Filter::eLinear); - Transfer::BlitToSwapchain(buffer, frame.image, *swapchain); + vk::ImageMemoryBarrier to_present {}; + to_present.sType = vk::StructureType::eImageMemoryBarrier; + to_present.srcAccessMask = vk::AccessFlagBits::eTransferWrite; + to_present.dstAccessMask = vk::AccessFlagBits::eMemoryRead; + to_present.oldLayout = vk::ImageLayout::eTransferDstOptimal; + to_present.newLayout = vk::ImageLayout::ePresentSrcKHR; + to_present.srcQueueFamilyIndex = VK_QUEUE_FAMILY_IGNORED; + to_present.dstQueueFamilyIndex = VK_QUEUE_FAMILY_IGNORED; + to_present.image = m_images[m_image_index]; + to_present.subresourceRange.aspectMask = vk::ImageAspectFlagBits::eColor; + to_present.subresourceRange.baseMipLevel = 0; + to_present.subresourceRange.levelCount = 1; + to_present.subresourceRange.baseArrayLayer = 0; + to_present.subresourceRange.layerCount = 1; + vk_command.pipelineBarrier(vk::PipelineStageFlagBits::eAllCommands, + vk::PipelineStageFlagBits::eAllCommands, + vk::DependencyFlagBits::eByRegion, 0, + nullptr, 0, nullptr, 1, &to_present); + command.End(); +} - vk::ImageMemoryBarrier pre_present_barrier {}; - pre_present_barrier.sType = vk::StructureType::eImageMemoryBarrier; - pre_present_barrier.pNext = nullptr; - pre_present_barrier.srcAccessMask = vk::AccessFlagBits::eTransferWrite; - pre_present_barrier.dstAccessMask = vk::AccessFlagBits::eMemoryRead; - pre_present_barrier.oldLayout = vk::ImageLayout::eTransferDstOptimal; - pre_present_barrier.newLayout = vk::ImageLayout::ePresentSrcKHR; - pre_present_barrier.srcQueueFamilyIndex = VK_QUEUE_FAMILY_IGNORED; - pre_present_barrier.dstQueueFamilyIndex = VK_QUEUE_FAMILY_IGNORED; - pre_present_barrier.subresourceRange.aspectMask = vk::ImageAspectFlagBits::eColor; - pre_present_barrier.subresourceRange.baseMipLevel = 0; - pre_present_barrier.subresourceRange.levelCount = 1; - pre_present_barrier.subresourceRange.baseArrayLayer = 0; - pre_present_barrier.subresourceRange.layerCount = 1; - pre_present_barrier.image = swapchain->swapchain_images[swapchain->current_index]; - vk_buffer.pipelineBarrier(vk::PipelineStageFlagBits::eAllCommands, - vk::PipelineStageFlagBits::eBottomOfPipe, vk::DependencyFlags {}, 0, - nullptr, 0, nullptr, 1, &pre_present_barrier); +void Swapchain::Submit(CommandBuffer& command) { + EXIT_IF(m_frame_index >= m_image_acquired.size() || m_image_index >= m_render_complete.size()); + SubmitInfo submit; + submit.AddWait(m_image_acquired[m_frame_index], 1, vk::PipelineStageFlagBits::eTransfer); + submit.AddSignal(m_render_complete[m_image_index]); + command.Execute(submit); +} - buffer.End(); - - auto render_complete_semaphore = - swapchain->render_complete_semaphores[swapchain->current_index]; - const vk::PipelineStageFlags wait_stage = vk::PipelineStageFlagBits::eTransfer; - buffer.ExecuteWithSemaphore(swapchain->image_acquired_semaphores[present_frame], wait_stage, - render_complete_semaphore); - - vk::PresentInfoKHR present; +Swapchain::Status Swapchain::Present() { + EXIT_IF(m_image_index >= m_render_complete.size()); + const auto ready = m_render_complete[m_image_index]; + vk::PresentInfoKHR present {}; present.sType = vk::StructureType::ePresentInfoKHR; - present.pNext = nullptr; present.swapchainCount = 1; - present.pSwapchains = &swapchain->swapchain; - present.pImageIndices = &swapchain->current_index; - present.pWaitSemaphores = &render_complete_semaphore; + present.pSwapchains = &m_handle; + present.pImageIndices = &m_image_index; + present.pWaitSemaphores = &ready; present.waitSemaphoreCount = 1; - present.pResults = nullptr; - auto& graphics = g_window_ctx->graphic_ctx; + vk::Result result; { - Common::LockGuard lock(graphics.queue_mutex); - result = graphics.queue.presentKHR(&present); + Common::LockGuard lock(m_window.graphic_ctx.queue_mutex); + result = m_window.graphic_ctx.queue.presentKHR(&present); } switch (result) { case vk::Result::eSuccess: break; case vk::Result::eSuboptimalKHR: LOGF("vkQueuePresentKHR returned vk::Result::eSuboptimalKHR\n"); - break; + return Status::Recreate; case vk::Result::eErrorOutOfDateKHR: LOGF("vkQueuePresentKHR returned vk::Result::eErrorOutOfDateKHR\n"); - VulkanRecreateSwapchain(); - return; + return Status::Recreate; + case vk::Result::eErrorSurfaceLostKHR: + LOGF("vkQueuePresentKHR returned vk::Result::eErrorSurfaceLostKHR\n"); + return Status::SurfaceLost; default: EXIT("vkQueuePresentKHR failed: %s\n", VulkanToString(result).c_str()); } - - swapchain->present_frame = (present_frame + 1u) % swapchain->swapchain_images_count; - - RenderDocOnPresent(); - WindowUpdateTitle(); + m_frame_index = (m_frame_index + 1u) % static_cast(m_images.size()); + return Status::Success; } +Presenter::Presenter(WindowContext& window): m_impl(std::make_unique(window)) {} + +Presenter::~Presenter() = default; + +Presenter::Frame& Presenter::PrepareFrame(CommandBuffer& buffer, const ImageInfo& info) { + KYTY_PROFILER_FUNCTION(); + EXIT_IF(buffer.IsInvalid()); + auto* frame = m_impl->frames.Acquire(); + Common::LockGuard render_lock(m_impl->renderer.GetMutex()); + auto& image = m_impl->ResolveSurface(info); + if (image.backing.format == vk::Format::eUndefined) { + EXIT("unsupported presentation source, image=%p\n", static_cast(&image)); + } + + auto frame_format = info.pixel_format; + switch (frame_format) { + case vk::Format::eR8G8B8A8Srgb: frame_format = vk::Format::eR8G8B8A8Unorm; break; + case vk::Format::eB8G8R8A8Srgb: frame_format = vk::Format::eB8G8R8A8Unorm; break; + default: break; + } + frame->Configure(m_impl->window.graphic_ctx, + {image.backing.extent.width, image.backing.extent.height}, + frame_format); + frame->CopyFrom(buffer, image); + return *frame; +} + +Presenter::Frame& Presenter::PrepareBlankFrame(uint32_t width, uint32_t height, bool opaque, + CommandBuffer* producer) { + KYTY_PROFILER_FUNCTION(); + auto format = m_impl->frames.GetFormat(); + auto* frame = m_impl->frames.Acquire(); + Common::LockGuard render_lock(m_impl->renderer.GetMutex()); + frame->Configure(m_impl->window.graphic_ctx, {width, height}, format); + vk::ClearColorValue clear {}; + clear.float32[3] = opaque ? 1.0f : 0.0f; + if (producer != nullptr) { + EXIT_IF(producer->IsInvalid()); + frame->Clear(*producer, clear); + } else { + if (frame->present_commands == nullptr) { + frame->present_commands = + std::make_unique(m_impl->present_scheduler); + } + auto& command = *frame->present_commands; + command.WaitForFenceAndReset(); + command.Begin(); + frame->Clear(command, clear); + command.End(); + command.Execute(); + } + return *frame; +} + +Presenter::Frame* Presenter::PrepareLastFrame() { + return m_impl->frames.AcquireLast(); +} + +bool Presenter::IsGuestPaused() const noexcept { + return m_impl->window.loop.paused.load(std::memory_order_acquire); +} + +RenderContext& Presenter::Renderer() const noexcept { + return m_impl->renderer; +} + +void Presenter::Present(Frame& frame, bool reuse) { + KYTY_PROFILER_FUNCTION(); + m_impl->frames.ValidateForPresent(&frame, reuse); + + auto& window = m_impl->window; + if (window.window_hidden) { + window.UpdateIcon(); + + SDL_ShowWindow(window.window); + + window.window_hidden = false; + m_impl->RecoverSwapchain(Swapchain::Status::Recreate); + } + + auto& swapchain = m_impl->swapchain; + for (uint32_t attempt = 0; attempt < 2; attempt++) { + auto status = swapchain.AcquireNextImage(); + if (status != Swapchain::Status::Success) { + m_impl->RecoverSwapchain(status); + continue; + } + if (frame.present_commands == nullptr) { + frame.present_commands = + std::make_unique(m_impl->present_scheduler); + } + { + Common::LockGuard render_lock(m_impl->renderer.GetMutex()); + frame.present_commands->WaitForFenceAndReset(); + auto& command = *frame.present_commands; + swapchain.RecordPresentCommands(command, frame.image); + swapchain.Submit(command); + } + status = swapchain.Present(); + if (status != Swapchain::Status::Success) { + m_impl->RecoverSwapchain(status); + continue; + } + + RenderDocOnPresent(); + window.UpdateTitle(); + m_impl->frames.Release(&frame, true); + return; + } + LOGF("Vulkan presentation retry exhausted; dropping frame\n"); + m_impl->frames.Release(&frame, reuse); +} + +void Presenter::Discard(Frame& frame) { + m_impl->frames.Release(&frame); +} + +WindowContext::WindowContext() = default; + } // namespace Libs::Graphics diff --git a/src/graphics/presentation/window/vulkanWindow.cpp b/src/graphics/presentation/window/vulkanWindow.cpp index 2df2a4bc..c4df2780 100644 --- a/src/graphics/presentation/window/vulkanWindow.cpp +++ b/src/graphics/presentation/window/vulkanWindow.cpp @@ -28,9 +28,11 @@ #include "common/timer.h" #include "graphics/host_gpu/graphicContext.h" #include "graphics/host_gpu/renderer/render.h" -#include "graphics/host_gpu/transfer.h" +#include "graphics/host_gpu/renderer/renderContext.h" #include "graphics/host_gpu/vma.h" #include "graphics/host_gpu/vulkanCommon.h" +#include "graphics/presentation/presenter.h" +#include "kernel/memory.h" #include "graphics/presentation/renderDoc.h" #include "graphics/presentation/videoOut.h" #include "graphics/presentation/window.h" @@ -40,7 +42,6 @@ #include #include -#include #include #include #include @@ -64,6 +65,14 @@ struct VulkanExtensions { std::vector available_layers; }; +vk::PhysicalDeviceVulkan13Features WindowContext::RequiredVulkan13Features() noexcept { + vk::PhysicalDeviceVulkan13Features features {}; + features.sType = vk::StructureType::ePhysicalDeviceVulkan13Features; + features.dynamicRendering = VK_TRUE; + features.synchronization2 = VK_TRUE; + return features; +} + static bool HasExtension(const std::vector& extensions, const char* name) { return std::any_of(extensions.begin(), extensions.end(), [name](const auto& ext) { return strcmp(ext.extensionName, name) == 0; }); @@ -79,8 +88,8 @@ static bool HasLayer(const std::vector& layers, const char* [name](const auto& layer) { return strcmp(layer.layerName, name) == 0; }); } -void VulkanGetSurfaceCapabilities(vk::PhysicalDevice physical_device, vk::SurfaceKHR surface, - SurfaceCapabilities& r) { +static void GetSurfaceCapabilities(vk::PhysicalDevice physical_device, vk::SurfaceKHR surface, + SurfaceCapabilities& r) { RequireVulkanSuccess(physical_device.getSurfaceCapabilitiesKHR(surface, &r.capabilities), "vkGetPhysicalDeviceSurfaceCapabilitiesKHR"); @@ -96,22 +105,6 @@ void VulkanGetSurfaceCapabilities(vk::PhysicalDevice physical_device, vk::Surfac return physical_device.getSurfacePresentModesKHR(surface, count, values); }); EXIT_NOT_IMPLEMENTED(r.present_modes.empty()); - - r.format_srgb_bgra32 = false; - r.format_unorm_bgra32 = false; - for (const auto& f: r.formats) { - if (f.format == vk::Format::eB8G8R8A8Srgb && - f.colorSpace == vk::ColorSpaceKHR::eSrgbNonlinear) { - r.format_srgb_bgra32 = true; - } - if (f.format == vk::Format::eB8G8R8A8Unorm && - f.colorSpace == vk::ColorSpaceKHR::eSrgbNonlinear) { - r.format_unorm_bgra32 = true; - } - if (r.format_srgb_bgra32 && r.format_unorm_bgra32) { - break; - } - } } static bool CheckFormat(vk::PhysicalDevice device, vk::Format format, bool tile, @@ -214,6 +207,7 @@ static void VulkanFindPhysicalDevice(vk::Instance instance, vk::SurfaceKHR surfa device_features2.pNext = &features13; device.getFeatures2(&device_features2); + const auto required_features13 = WindowContext::RequiredVulkan13Features(); const auto queue_family = VulkanFindQueueFamily(device, surface); if (queue_family == static_cast(-1)) { @@ -239,10 +233,28 @@ static void VulkanFindPhysicalDevice(vk::Instance instance, vk::SurfaceKHR surfa LOGF("samplerMirrorClampToEdge is not supported\n"); skip_device = true; } + if (features12.timelineSemaphore != VK_TRUE) { + LOGF("timelineSemaphore is not supported\n"); + skip_device = true; + } if (features13.robustImageAccess != VK_TRUE) { LOGF("robustImageAccess is not supported\n"); skip_device = true; } + if (required_features13.dynamicRendering == VK_TRUE && + features13.dynamicRendering != VK_TRUE) { + LOGF("dynamicRendering is not supported\n"); + skip_device = true; + } + if (required_features13.synchronization2 == VK_TRUE && + features13.synchronization2 != VK_TRUE) { + LOGF("synchronization2 is not supported\n"); + skip_device = true; + } + if (device_features2.features.sampleRateShading != VK_TRUE) { + LOGF("sampleRateShading is not supported\n"); + skip_device = true; + } if (device_features2.features.fragmentStoresAndAtomics != VK_TRUE) { LOGF("fragmentStoresAndAtomics is not supported\n"); @@ -309,7 +321,7 @@ static void VulkanFindPhysicalDevice(vk::Instance instance, vk::SurfaceKHR surfa SurfaceCapabilities candidate_capabilities; if (!skip_device) { - VulkanGetSurfaceCapabilities(device, surface, candidate_capabilities); + GetSurfaceCapabilities(device, surface, candidate_capabilities); if (!(candidate_capabilities.capabilities.supportedUsageFlags & vk::ImageUsageFlagBits::eTransferDst)) { @@ -318,12 +330,6 @@ static void VulkanFindPhysicalDevice(vk::Instance instance, vk::SurfaceKHR surfa } } - if (!skip_device && !CheckFormat(device, vk::Format::eR8G8B8A8Srgb, false, - vk::FormatFeatureFlagBits::eBlitSrc)) { - LOGF("Format vk::Format::eR8G8B8A8Srgb cannot be used as transfer source\n"); - skip_device = true; - } - if (!skip_device && !CheckFormat(device, vk::Format::eD32Sfloat, true, vk::FormatFeatureFlagBits::eDepthStencilAttachment)) { LOGF("Format vk::Format::eD32Sfloat cannot be used as depth buffer\n"); @@ -420,9 +426,9 @@ static void VulkanFindPhysicalDevice(vk::Instance instance, vk::SurfaceKHR surfa } } -static void VulkanInitSubgroupSizeControl(vk::PhysicalDevice physical_device) { +static void VulkanInitSubgroupSizeControl(vk::PhysicalDevice physical_device, + GraphicContext& graphics) { EXIT_IF(physical_device == nullptr); - auto& graphics = g_window_ctx->graphic_ctx; vk::PhysicalDeviceSubgroupSizeControlProperties subgroup_size_control {}; subgroup_size_control.sType = vk::StructureType::ePhysicalDeviceSubgroupSizeControlProperties; @@ -463,10 +469,10 @@ static void VulkanInitSubgroupSizeControl(vk::PhysicalDevice physical_device) { } static vk::Device VulkanCreateDevice(vk::PhysicalDevice physical_device, const VulkanExtensions& r, - uint32_t queue_family, - const std::vector& device_extensions) { + uint32_t queue_family, + const std::vector& device_extensions, + GraphicContext& graphics) { EXIT_IF(physical_device == nullptr); - auto& graphics = g_window_ctx->graphic_ctx; EXIT_IF(queue_family == static_cast(-1)); const float queue_priority = 1.0f; @@ -502,7 +508,11 @@ static vk::Device VulkanCreateDevice(vk::PhysicalDevice physical_device, const V vk::PhysicalDeviceVulkan13Features supported_features13 {}; supported_features13.sType = vk::StructureType::ePhysicalDeviceVulkan13Features; - supported_features13.pNext = nullptr; + + vk::PhysicalDeviceVulkan12Features supported_features12 {}; + supported_features12.sType = vk::StructureType::ePhysicalDeviceVulkan12Features; + supported_features12.pNext = nullptr; + supported_features13.pNext = &supported_features12; const auto robustness2_ext_enabled = HasExtension(device_extensions, VK_EXT_ROBUSTNESS_2_EXTENSION_NAME); @@ -511,13 +521,21 @@ static vk::Device VulkanCreateDevice(vk::PhysicalDevice physical_device, const V supported_robustness2.sType = vk::StructureType::ePhysicalDeviceRobustness2FeaturesEXT; supported_robustness2.pNext = nullptr; if (robustness2_ext_enabled) { - supported_features13.pNext = &supported_robustness2; + supported_features12.pNext = &supported_robustness2; } vk::PhysicalDeviceFeatures2 supported_features2 {}; supported_features2.sType = vk::StructureType::ePhysicalDeviceFeatures2; supported_features2.pNext = &supported_features13; physical_device.getFeatures2(&supported_features2); + const auto required_features13 = WindowContext::RequiredVulkan13Features(); + EXIT_NOT_IMPLEMENTED(supported_features12.timelineSemaphore != VK_TRUE); + EXIT_NOT_IMPLEMENTED(required_features13.dynamicRendering == VK_TRUE && + supported_features13.dynamicRendering != VK_TRUE); + EXIT_NOT_IMPLEMENTED(required_features13.synchronization2 == VK_TRUE && + supported_features13.synchronization2 != VK_TRUE); + EXIT_NOT_IMPLEMENTED(supported_features2.features.sampleRateShading != VK_TRUE); + features12.timelineSemaphore = VK_TRUE; vk::PhysicalDeviceFeatures device_features {}; device_features.fragmentStoresAndAtomics = VK_TRUE; @@ -529,8 +547,8 @@ static vk::Device VulkanCreateDevice(vk::PhysicalDevice physical_device, const V device_features.shaderImageGatherExtended = VK_TRUE; device_features.independentBlend = VK_TRUE; device_features.tessellationShader = VK_TRUE; - device_features.sampleRateShading = supported_features2.features.sampleRateShading; - graphics.sample_rate_shading_enabled = device_features.sampleRateShading == VK_TRUE; + device_features.sampleRateShading = VK_TRUE; + graphics.sample_rate_shading_enabled = true; device_features.vertexPipelineStoresAndAtomics = supported_features2.features.vertexPipelineStoresAndAtomics; @@ -552,8 +570,7 @@ static vk::Device VulkanCreateDevice(vk::PhysicalDevice physical_device, const V robustness2.nullDescriptor = supported_robustness2.nullDescriptor; } - vk::PhysicalDeviceVulkan13Features features13 {}; - features13.sType = vk::StructureType::ePhysicalDeviceVulkan13Features; + auto features13 = required_features13; features13.pNext = robustness2_ext_enabled ? &robustness2 : const_cast(base_feature_chain); features13.robustImageAccess = supported_features13.robustImageAccess; @@ -773,12 +790,12 @@ static void VulkanCheckInstanceVersion() { } } -void VulkanCreate(WindowContext& window) { - EXIT_IF(window.window == nullptr); - EXIT_IF(window.graphic_ctx.instance != nullptr); - EXIT_IF(window.graphic_ctx.physical_device != nullptr); - EXIT_IF(window.graphic_ctx.device != nullptr); - EXIT_IF(window.surface_capabilities != nullptr); +void WindowContext::CreateVulkan() { + EXIT_IF(window == nullptr); + EXIT_IF(graphic_ctx.instance != nullptr); + EXIT_IF(graphic_ctx.physical_device != nullptr); + EXIT_IF(graphic_ctx.device != nullptr); + EXIT_IF(surface != nullptr); auto get_instance_proc_addr = reinterpret_cast(SDL_Vulkan_GetVkGetInstanceProcAddr()); @@ -788,7 +805,7 @@ void VulkanCreate(WindowContext& window) { VULKAN_HPP_DEFAULT_DISPATCHER.init(get_instance_proc_addr); VulkanExtensions r; - VulkanGetExtensions(window.window, r); + VulkanGetExtensions(window, r); VulkanCheckInstanceVersion(); vk::ApplicationInfo app_info {}; @@ -852,36 +869,35 @@ void VulkanCreate(WindowContext& window) { inst_info.ppEnabledLayerNames = (r.enable_validation_layers ? r.required_layers.data() : nullptr); - const vk::Result result = vk::createInstance(&inst_info, nullptr, &window.graphic_ctx.instance); + const vk::Result result = vk::createInstance(&inst_info, nullptr, &graphic_ctx.instance); switch (result) { case vk::Result::eSuccess: break; case vk::Result::eErrorIncompatibleDriver: EXIT("Unable to find a compatible Vulkan Driver"); default: EXIT("Could not create a Vulkan instance (for unknown reasons)"); } - VULKAN_HPP_DEFAULT_DISPATCHER.init(window.graphic_ctx.instance); + VULKAN_HPP_DEFAULT_DISPATCHER.init(graphic_ctx.instance); if (r.enable_validation_layers) { dbg_create_info.pNext = nullptr; - if (VulkanCreateDebugUtilsMessengerEXT(window.graphic_ctx.instance, &dbg_create_info, - nullptr, &window.graphic_ctx.debug_messenger) != + if (VulkanCreateDebugUtilsMessengerEXT(graphic_ctx.instance, &dbg_create_info, nullptr, + &graphic_ctx.debug_messenger) != vk::Result::eSuccess) { EXIT("Could not create debug messenger"); } } vk::SurfaceKHR::CType native_surface = VK_NULL_HANDLE; - if (SDL_Vulkan_CreateSurface(window.window, - static_cast(window.graphic_ctx.instance), + if (SDL_Vulkan_CreateSurface(window, static_cast(graphic_ctx.instance), &native_surface) == SDL_FALSE) { EXIT("Could not create a Vulkan surface"); } - window.surface = native_surface; + surface = native_surface; std::vector device_extensions = { VK_KHR_SWAPCHAIN_EXTENSION_NAME, VK_EXT_DEPTH_CLIP_ENABLE_EXTENSION_NAME, VK_EXT_DEPTH_CLIP_CONTROL_EXTENSION_NAME, VK_EXT_COLOR_WRITE_ENABLE_EXTENSION_NAME, - "VK_KHR_maintenance1"}; + VK_KHR_PUSH_DESCRIPTOR_EXTENSION_NAME, "VK_KHR_maintenance1"}; #ifdef KYTY_ENABLE_DEBUG_PRINTF if (Config::SpirvDebugPrintfEnabled()) { @@ -889,23 +905,18 @@ void VulkanCreate(WindowContext& window) { } #endif - window.surface_capabilities = new SurfaceCapabilities {}; - uint32_t queue_family = static_cast(-1); - VulkanFindPhysicalDevice(window.graphic_ctx.instance, window.surface, device_extensions, - *window.surface_capabilities, window.graphic_ctx.physical_device, - queue_family); + VulkanFindPhysicalDevice(graphic_ctx.instance, surface, device_extensions, + surface_capabilities, graphic_ctx.physical_device, queue_family); - if (window.graphic_ctx.physical_device == nullptr) { + if (graphic_ctx.physical_device == nullptr) { EXIT("Could not find suitable device"); } - window.graphic_ctx.physical_device.getProperties( - &window.graphic_ctx.physical_device_properties); - window.graphic_ctx.physical_device.getMemoryProperties( - &window.graphic_ctx.physical_device_memory_properties); - const auto& device_properties = window.graphic_ctx.GetPhysicalDeviceProperties(); + graphic_ctx.physical_device.getProperties(&graphic_ctx.physical_device_properties); + graphic_ctx.physical_device.getMemoryProperties(&graphic_ctx.physical_device_memory_properties); + const auto& device_properties = graphic_ctx.GetPhysicalDeviceProperties(); LOGF("Select device: %s\n", device_properties.deviceName.data()); @@ -913,41 +924,96 @@ void VulkanCreate(WindowContext& window) { auto available_extensions = EnumerateVulkan( "vkEnumerateDeviceExtensionProperties", [&](uint32_t* count, vk::ExtensionProperties* values) { - return window.graphic_ctx.physical_device.enumerateDeviceExtensionProperties( - nullptr, count, values); + return graphic_ctx.physical_device.enumerateDeviceExtensionProperties(nullptr, + count, + values); }); if (HasExtension(available_extensions, VK_EXT_MEMORY_BUDGET_EXTENSION_NAME)) { device_extensions.push_back(VK_EXT_MEMORY_BUDGET_EXTENSION_NAME); - window.graphic_ctx.memory_budget_ext_enabled = true; + graphic_ctx.memory_budget_ext_enabled = true; } if (HasExtension(available_extensions, VK_EXT_ROBUSTNESS_2_EXTENSION_NAME)) { device_extensions.push_back(VK_EXT_ROBUSTNESS_2_EXTENSION_NAME); } } - memcpy(window.device_name, device_properties.deviceName, sizeof(window.device_name)); - std::snprintf(window.processor_name, sizeof(window.processor_name), "%s", + memcpy(device_name, device_properties.deviceName, sizeof(device_name)); + std::snprintf(processor_name, sizeof(processor_name), "%s", Common::GetSystemInfo().ProcessorName.c_str()); - VulkanInitSubgroupSizeControl(window.graphic_ctx.physical_device); + VulkanInitSubgroupSizeControl(graphic_ctx.physical_device, graphic_ctx); - window.graphic_ctx.device = - VulkanCreateDevice(window.graphic_ctx.physical_device, r, queue_family, device_extensions); - if (window.graphic_ctx.device == nullptr) { + graphic_ctx.device = VulkanCreateDevice(graphic_ctx.physical_device, r, queue_family, + device_extensions, graphic_ctx); + if (graphic_ctx.device == nullptr) { EXIT("Could not create device"); } - VULKAN_HPP_DEFAULT_DISPATCHER.init(window.graphic_ctx.device); - window.graphic_ctx.queue_family = queue_family; - window.graphic_ctx.device.getQueue(queue_family, 0, &window.graphic_ctx.queue); - EXIT_IF(window.graphic_ctx.queue == nullptr); + VULKAN_HPP_DEFAULT_DISPATCHER.init(graphic_ctx.device); + graphic_ctx.queue_family = queue_family; + graphic_ctx.device.getQueue(queue_family, 0, &graphic_ctx.queue); + EXIT_IF(graphic_ctx.queue == nullptr); - if (!window.graphic_ctx.CreateAllocator()) { + if (!graphic_ctx.CreateAllocator()) { EXIT("Could not create Vulkan memory allocator"); } - window.swapchain = VulkanCreateSwapchain(2); - RenderDocSetActiveWindow(window.graphic_ctx.instance, window.window); + render_context = std::make_unique(graphic_ctx); + LibKernel::Memory::InstallGpuResources(&render_context->GetGpuResources()); + presenter = std::make_unique(*this); + RenderDocSetActiveWindow(graphic_ctx.instance, window); +} + +void WindowContext::RefreshSurfaceCapabilities() { + EXIT_IF(graphic_ctx.physical_device == nullptr || surface == nullptr); + GetSurfaceCapabilities(graphic_ctx.physical_device, surface, surface_capabilities); +} + +void WindowContext::RecreateSurface() { + EXIT_IF(window == nullptr || graphic_ctx.instance == nullptr); + if (surface != nullptr) { + graphic_ctx.instance.destroySurfaceKHR(surface, nullptr); + surface = nullptr; + } + vk::SurfaceKHR::CType native_surface = VK_NULL_HANDLE; + if (SDL_Vulkan_CreateSurface(window, static_cast(graphic_ctx.instance), + &native_surface) == SDL_FALSE) { + EXIT("Could not recreate the Vulkan surface: %s\n", SDL_GetError()); + } + surface = native_surface; + RefreshSurfaceCapabilities(); +} + +WindowContext::~WindowContext() { + presenter.reset(); + LibKernel::Memory::InstallGpuResources(nullptr); + render_context.reset(); + + if (graphic_ctx.device != nullptr) { + RequireVulkanSuccess(graphic_ctx.device.waitIdle(), "wait for Vulkan device shutdown"); + graphic_ctx.DestroyAllocator(); + graphic_ctx.device.destroy(nullptr); + graphic_ctx.device = nullptr; + graphic_ctx.queue = nullptr; + } + if (surface != nullptr) { + graphic_ctx.instance.destroySurfaceKHR(surface, nullptr); + surface = nullptr; + } + if (graphic_ctx.debug_messenger != nullptr) { + graphic_ctx.instance.destroyDebugUtilsMessengerEXT(graphic_ctx.debug_messenger, nullptr); + graphic_ctx.debug_messenger = nullptr; + } + if (graphic_ctx.instance != nullptr) { + graphic_ctx.instance.destroy(nullptr); + graphic_ctx.instance = nullptr; + graphic_ctx.physical_device = nullptr; + } + if (window != nullptr) { + SDL_DestroyWindow(window); + window = nullptr; + } + SDL_QuitSubSystem(SDL_INIT_VIDEO | SDL_INIT_GAMECONTROLLER); } } // namespace Libs::Graphics diff --git a/src/graphics/presentation/window/window.cpp b/src/graphics/presentation/window/window.cpp index 455df4a0..a241c362 100644 --- a/src/graphics/presentation/window/window.cpp +++ b/src/graphics/presentation/window/window.cpp @@ -24,24 +24,20 @@ #include "common/logging/log.h" #include "common/profiler.h" #include "common/stringUtils.h" -#include "common/subsystems.h" #include "common/systemInfo.h" #include "common/threads.h" #include "common/timer.h" #include "graphics/host_gpu/graphicContext.h" #include "graphics/host_gpu/renderer/render.h" -#include "graphics/host_gpu/transfer.h" #include "graphics/host_gpu/vma.h" #include "graphics/host_gpu/vulkanCommon.h" #include "graphics/presentation/renderDoc.h" -#include "graphics/presentation/videoOut.h" #include "graphics/presentation/window/windowInternal.h" #include "libs/controller.h" #include "loader/systemContent.h" #include #include -#include #include #include #include @@ -61,7 +57,6 @@ namespace Libs::Graphics { -constexpr float FPS_UPDATE_TIME = 1.0f; constexpr int KEYBOARD_CONTROLLER_ID = -1000; struct EventKeyboard { @@ -213,163 +208,36 @@ constexpr uint32_t KYTY_SDL_BUTTON_RMASK = SDL_BUTTON_RMASK; // NOLINT(hicpp-s constexpr uint32_t KYTY_SDL_BUTTON_X1MASK = SDL_BUTTON_X1MASK; // NOLINT(hicpp-signed-bitwise) constexpr uint32_t KYTY_SDL_BUTTON_X2MASK = SDL_BUTTON_X2MASK; // NOLINT(hicpp-signed-bitwise) -struct WindowGame { - void* private_data = nullptr; - void* event = nullptr; +namespace { - bool m_game_need_exit = {false}; - bool m_game_is_paused = {false}; - uint32_t m_screen_width = {0}; - uint32_t m_screen_height = {0}; - double m_current_time_seconds = {0.0}; - double m_previous_time_seconds = {0.0}; - int m_update_num = {0}; - int m_frame_num = {0}; - double m_update_time_seconds = {0.0}; - double m_current_fps = {0.0}; - int m_max_updates_per_frame = {4}; - double m_update_fixed_time = 1.0 / 60.0; - int m_fps_frames_num = {0}; - double m_fps_start_time = {0}; -}; +std::unique_ptr g_window; -struct WindowGamePrivate { - explicit WindowGamePrivate(GraphicContext& graphics): graphics(graphics) {} - - Common::Mutex mutex; - int skip_frames = 0; - GraphicContext& graphics; -}; - -WindowContext* g_window_ctx = nullptr; -static WindowGame g_window_game; +} // namespace constexpr const char* KYTY_SDL_WINDOW_CAPTION = "Game"; constexpr uint32_t KYTY_SDL_WINDOW_FLAGS = (static_cast(SDL_WINDOW_HIDDEN) | static_cast(SDL_WINDOW_VULKAN)); constexpr int KYTY_SDL_WINDOWPOS_CENTERED = SDL_WINDOWPOS_CENTERED; /*NOLINT(hicpp-signed-bitwise)*/ -static void CalcFrameTime(WindowGame& game, double game_time_s) { - game.m_previous_time_seconds = game.m_current_time_seconds; - game.m_current_time_seconds = game_time_s; - - game.m_frame_num++; - game.m_fps_frames_num++; - - const auto fps_time = game.m_current_time_seconds - game.m_fps_start_time; - if (fps_time > FPS_UPDATE_TIME) { - game.m_current_fps = static_cast(game.m_fps_frames_num) / fps_time; - game.m_fps_frames_num = 0; - game.m_fps_start_time = game.m_current_time_seconds; - } -} - -static bool Init(WindowGame& /*game*/) { - return true; -} -static bool Update(WindowGame& /*game*/) { - return true; -} -static bool Render(WindowGame& /*game*/) { - return true; -} -static bool Close(WindowGame& /*game*/) { - return true; -} -static void SetPause(WindowGame& game, bool flag) { +static void SetPause(WindowLoopState& game, bool flag) { LOGF("Pause: %s\n", flag ? "true" : "false"); - game.m_game_is_paused = flag; + game.paused.store(flag, std::memory_order_release); } -static bool RenderAndUpdate(WindowGame& game) { - static double lag = 0.0; - - lag += game.m_current_time_seconds - game.m_previous_time_seconds; - - int num = 0; - - bool ok = true; - - while (lag >= game.m_update_fixed_time) { - if (num < game.m_max_updates_per_frame) { - ok = ok && Update(game); - - game.m_update_num++; - num++; - game.m_update_time_seconds = game.m_update_num * game.m_update_fixed_time; - } - - lag -= game.m_update_fixed_time; - } - - ok = ok && Render(game); - - return ok; -} - -bool GameInit(WindowGame& game, const Common::Timer& timer) { - EXIT_IF(game.private_data || game.event); - auto& graphics = g_window_ctx->graphic_ctx; - - EXIT_IF(graphics.screen_width == 0 || graphics.screen_height == 0); - - auto* pdata = new WindowGamePrivate(graphics); - - game.private_data = pdata; - game.event = new SDL_Event; - - game.m_screen_width = graphics.screen_width; - game.m_screen_height = graphics.screen_height; - - CalcFrameTime(game, timer.GetTimeS()); - - return Init(game); -} - -bool GameClose(WindowGame& game) { - EXIT_IF(!game.private_data || !game.event); - - delete (static_cast(game.private_data)); - delete (static_cast(game.event)); - - return Close(game); -} - -void GameShowWindow(WindowGame& game, const Common::Timer& timer) { - auto* p = static_cast(game.private_data); - - EXIT_IF(!p); - - p->mutex.Lock(); - { - if (p->skip_frames > 0) { - p->skip_frames--; - LOGF("skip frame %d\n", p->skip_frames); - } else { - VideoOut::VideoOutBeginVblank(); - if (VideoOut::VideoOutFlipWindow(0)) { - CalcFrameTime(game, timer.GetTimeS()); - } - VideoOut::VideoOutEndVblank(); - } - } - p->mutex.Unlock(); -} - -void GameEventQuit(WindowGame& game) { +static void GameEventQuit(WindowLoopState& game) { LOGF("Event: quit\n"); - game.m_game_need_exit = true; + game.need_exit = true; } -void GameEventTerminate(WindowGame& game) { +static void GameEventTerminate(WindowLoopState& game) { LOGF("Event: terminate\n"); - game.m_game_need_exit = true; + game.need_exit = true; } -void GameEventKeyboard(WindowGame& game, const EventKeyboard& key) { +static void GameEventKeyboard(WindowLoopState& game, const EventKeyboard& key) { #ifdef KYTY_DBG_INPUT LOGF("Key: time = %.04f, %s%s, %s%s, %s, scan = %d, key = %d, mod = %04" PRIx16 "\n", key.timestamp_seconds, (key.down ? "down" : ""), (key.up ? "up" : ""), @@ -380,8 +248,10 @@ void GameEventKeyboard(WindowGame& game, const EventKeyboard& key) { #if KYTY_PLATFORM == KYTY_PLATFORM_WINDOWS || KYTY_PLATFORM == KYTY_PLATFORM_LINUX if (key.down) { switch (key.key_code) { - case SDLK_ESCAPE: game.m_game_need_exit = true; break; - case SDLK_SPACE: SetPause(game, !game.m_game_is_paused); break; + case SDLK_ESCAPE: game.need_exit = true; break; + case SDLK_SPACE: + SetPause(game, !game.paused.load(std::memory_order_acquire)); + break; case SDLK_F1: if (!key.repeat) { RenderDocRequestCapture(); @@ -403,7 +273,7 @@ void GameEventKeyboard(WindowGame& game, const EventKeyboard& key) { #endif } -void GameEventMouse([[maybe_unused]] WindowGame& game, [[maybe_unused]] const EventMouse& mb) { +static void GameEventMouse([[maybe_unused]] const EventMouse& mb) { #ifdef KYTY_DBG_INPUT if (mb.wheel) { LOGF("Mouse wheel: time = %.04f, %s[%d, %d]\n", mb.timestamp_seconds, @@ -424,7 +294,7 @@ void GameEventMouse([[maybe_unused]] WindowGame& game, [[maybe_unused]] const Ev #endif } -void GameEventFinger([[maybe_unused]] WindowGame& game, [[maybe_unused]] const EventFinger& f) { +static void GameEventFinger([[maybe_unused]] const EventFinger& f) { #ifdef KYTY_DBG_INPUT if (f.motion) { LOGF("Finger motion: time = %.04f, %d, %d, (x,y) = [%f, %f], (dx,dy) = [%f, %f], pressure " @@ -439,8 +309,7 @@ void GameEventFinger([[maybe_unused]] WindowGame& game, [[maybe_unused]] const E #endif } -void GameEventController([[maybe_unused]] WindowGame& game, - [[maybe_unused]] const EventController& f) { +static void GameEventController([[maybe_unused]] const EventController& f) { EXIT_NOT_IMPLEMENTED(f.remapped); #ifdef KYTY_DBG_INPUT @@ -486,116 +355,104 @@ void GameEventController([[maybe_unused]] WindowGame& game, } } -void GameEventDisplay([[maybe_unused]] WindowGame& game) { - auto* p = static_cast(game.private_data); - - p->mutex.Lock(); - game.m_screen_width = p->graphics.screen_width; - game.m_screen_height = p->graphics.screen_height; - p->mutex.Unlock(); -} - -void GameEventLowMemory(WindowGame& /*game*/) { +static void GameEventLowMemory() { LOGF("Event: low_memory\n"); } -void GameEventWillEnterBackground(WindowGame& game) { +static void GameEventWillEnterBackground(WindowLoopState& game) { LOGF("Event: will_enter_background\n"); SetPause(game, true); } -void GameEventDidEnterBackground(WindowGame& /*game*/) { +static void GameEventDidEnterBackground() { LOGF("Event: did_enter_background\n"); } -void GameEventWillEnterForeground(WindowGame& /*game*/) { +static void GameEventWillEnterForeground() { LOGF("Event: will_enter_foreground\n"); } -void GameEventDidEnterForeground(WindowGame& game) { +static void GameEventDidEnterForeground(WindowLoopState& game) { LOGF("Event: did_enter_foreground\n"); SetPause(game, false); } -void GameEventResize(WindowGame& game, uint32_t new_width, uint32_t new_height) { +void WindowContext::Resize(uint32_t new_width, uint32_t new_height) { EXIT_IF(new_width == 0 || new_height == 0); - - auto* p = static_cast(game.private_data); - EXIT_IF(p == nullptr); - - p->mutex.Lock(); - { - p->skip_frames++; - p->graphics.screen_width = new_width; - p->graphics.screen_height = new_height; - - game.m_screen_width = p->graphics.screen_width; - game.m_screen_height = p->graphics.screen_height; - } - p->mutex.Unlock(); + graphic_ctx.screen_width = new_width; + graphic_ctx.screen_height = new_height; } -static void ProcessWindowEvent(WindowGame& game, SDL_WindowEvent window) { - switch (window.event) { - case SDL_WINDOWEVENT_SHOWN: LOGF("Window %" PRIu32 " shown\n", window.windowID); break; +void WindowContext::ProcessWindowEvent(const SDL_WindowEvent& event) { + const auto& window_event = event; + switch (window_event.event) { + case SDL_WINDOWEVENT_SHOWN: LOGF("Window %" PRIu32 " shown\n", window_event.windowID); break; - case SDL_WINDOWEVENT_HIDDEN: LOGF("Window %" PRIu32 " hidden\n", window.windowID); break; + case SDL_WINDOWEVENT_HIDDEN: + LOGF("Window %" PRIu32 " hidden\n", window_event.windowID); + break; - case SDL_WINDOWEVENT_EXPOSED: LOGF("Window %" PRIu32 " exposed\n", window.windowID); break; + case SDL_WINDOWEVENT_EXPOSED: + LOGF("Window %" PRIu32 " exposed\n", window_event.windowID); + break; case SDL_WINDOWEVENT_MOVED: - LOGF("Window %" PRIu32 " moved to %" PRId32 ",%" PRId32 "\n", window.windowID, - window.data1, window.data2); + LOGF("Window %" PRIu32 " moved to %" PRId32 ",%" PRId32 "\n", + window_event.windowID, window_event.data1, window_event.data2); break; case SDL_WINDOWEVENT_RESIZED: - LOGF("Window %" PRIu32 " resized to %" PRId32 "x%" PRId32 "\n", window.windowID, - window.data1, window.data2); + LOGF("Window %" PRIu32 " resized to %" PRId32 "x%" PRId32 "\n", + window_event.windowID, window_event.data1, window_event.data2); LOGF("m: %d\n", static_cast(SDL_ThreadID())); - GameEventResize(game, window.data1, window.data2); + Resize(window_event.data1, window_event.data2); break; case SDL_WINDOWEVENT_SIZE_CHANGED: - LOGF("Window %" PRIu32 " size changed to %" PRId32 "x%" PRId32 "\n", window.windowID, - window.data1, window.data2); + LOGF("Window %" PRIu32 " size changed to %" PRId32 "x%" PRId32 "\n", + window_event.windowID, window_event.data1, window_event.data2); LOGF("m: %d\n", static_cast(SDL_ThreadID())); - GameEventResize(game, window.data1, window.data2); + Resize(window_event.data1, window_event.data2); break; case SDL_WINDOWEVENT_MINIMIZED: - LOGF("Window %" PRIu32 " minimized\n", window.windowID); + LOGF("Window %" PRIu32 " minimized\n", window_event.windowID); break; case SDL_WINDOWEVENT_MAXIMIZED: - LOGF("Window %" PRIu32 " maximized\n", window.windowID); + LOGF("Window %" PRIu32 " maximized\n", window_event.windowID); break; case SDL_WINDOWEVENT_RESTORED: - LOGF("Window %" PRIu32 " restored\n", window.windowID); + LOGF("Window %" PRIu32 " restored\n", window_event.windowID); break; case SDL_WINDOWEVENT_ENTER: - LOGF("Mouse entered window %" PRIu32 "\n", window.windowID); + LOGF("Mouse entered window %" PRIu32 "\n", window_event.windowID); + break; + case SDL_WINDOWEVENT_LEAVE: + LOGF("Mouse left window %" PRIu32 "\n", window_event.windowID); break; - case SDL_WINDOWEVENT_LEAVE: LOGF("Mouse left window %" PRIu32 "\n", window.windowID); break; case SDL_WINDOWEVENT_FOCUS_GAINED: - LOGF("Window %" PRIu32 " gained keyboard focus\n", window.windowID); + LOGF("Window %" PRIu32 " gained keyboard focus\n", window_event.windowID); break; case SDL_WINDOWEVENT_FOCUS_LOST: - LOGF("Window %" PRIu32 " lost keyboard focus\n", window.windowID); + LOGF("Window %" PRIu32 " lost keyboard focus\n", window_event.windowID); + break; + case SDL_WINDOWEVENT_CLOSE: + LOGF("Window %" PRIu32 " closed\n", window_event.windowID); break; - case SDL_WINDOWEVENT_CLOSE: LOGF("Window %" PRIu32 " closed\n", window.windowID); break; default: - LOGF("Window %" PRIu32 " got unknown event %" PRIu8 "\n", window.windowID, - window.event); + LOGF("Window %" PRIu32 " got unknown event %" PRIu8 "\n", window_event.windowID, + window_event.event); break; } } -static void ProcessDisplayEvent(WindowGame& game, SDL_DisplayEvent display) { +void WindowContext::ProcessDisplayEvent(const SDL_DisplayEvent& display) { bool sdl = false; switch (display.event) { @@ -613,10 +470,6 @@ static void ProcessDisplayEvent(WindowGame& game, SDL_DisplayEvent display) { default: LOGF("???\n"); } - if (!sdl) { - GameEventDisplay(game); - } - break; } default: @@ -626,27 +479,9 @@ static void ProcessDisplayEvent(WindowGame& game, SDL_DisplayEvent display) { } } -int GamePollEvent(WindowGame& game) { - auto* event = static_cast(game.event); - - EXIT_IF(!event); - - return SDL_PollEvent(event); -} - -int GameWaitEvent(WindowGame& game) { - auto* event = static_cast(game.event); - - EXIT_IF(!event); - - return SDL_WaitEvent(event); -} - -void GameProcessEvent(WindowGame& game, double time_s) { - auto* event = static_cast(game.event); - - EXIT_IF(!event); - +void WindowContext::ProcessEvent(double time_s) { + auto& game = loop; + auto* event = &game.event; EXIT_IF(SDL_GetEventState(SDL_DISPLAYEVENT) != SDL_ENABLE); switch (event->type) { @@ -654,13 +489,13 @@ void GameProcessEvent(WindowGame& game, double time_s) { case SDL_APP_TERMINATING: GameEventTerminate(game); break; - case SDL_APP_LOWMEMORY: GameEventLowMemory(game); break; + case SDL_APP_LOWMEMORY: GameEventLowMemory(); break; case SDL_APP_WILLENTERBACKGROUND: GameEventWillEnterBackground(game); break; - case SDL_APP_DIDENTERBACKGROUND: GameEventDidEnterBackground(game); break; + case SDL_APP_DIDENTERBACKGROUND: GameEventDidEnterBackground(); break; - case SDL_APP_WILLENTERFOREGROUND: GameEventWillEnterForeground(game); break; + case SDL_APP_WILLENTERFOREGROUND: GameEventWillEnterForeground(); break; case SDL_APP_DIDENTERFOREGROUND: GameEventDidEnterForeground(game); break; @@ -683,9 +518,9 @@ void GameProcessEvent(WindowGame& game, double time_s) { break; } - case SDL_WINDOWEVENT: ProcessWindowEvent(game, event->window); break; + case SDL_WINDOWEVENT: ProcessWindowEvent(event->window); break; - case SDL_DISPLAYEVENT: ProcessDisplayEvent(game, event->display); break; + case SDL_DISPLAYEVENT: ProcessDisplayEvent(event->display); break; case SDL_MOUSEBUTTONDOWN: case SDL_MOUSEBUTTONUP: { @@ -710,7 +545,7 @@ void GameProcessEvent(WindowGame& game, double time_s) { mb.motion_y = 0; mb.timestamp_seconds = time_s; - GameEventMouse(game, mb); + GameEventMouse(mb); break; } @@ -737,7 +572,7 @@ void GameProcessEvent(WindowGame& game, double time_s) { mb.motion_y = 0; mb.timestamp_seconds = time_s; - GameEventMouse(game, mb); + GameEventMouse(mb); break; } @@ -764,7 +599,7 @@ void GameProcessEvent(WindowGame& game, double time_s) { mb.motion_y = event->motion.yrel; mb.timestamp_seconds = time_s; - GameEventMouse(game, mb); + GameEventMouse(mb); break; } @@ -786,7 +621,7 @@ void GameProcessEvent(WindowGame& game, double time_s) { f.pressure = event->tfinger.pressure; f.timestamp_seconds = time_s; - GameEventFinger(game, f); + GameEventFinger(f); break; } @@ -808,7 +643,7 @@ void GameProcessEvent(WindowGame& game, double time_s) { c.released = false; c.timestamp_seconds = time_s; - GameEventController(game, c); + GameEventController(c); break; } @@ -831,7 +666,7 @@ void GameProcessEvent(WindowGame& game, double time_s) { c.released = (event->cbutton.state == SDL_RELEASED); c.timestamp_seconds = time_s; - GameEventController(game, c); + GameEventController(c); break; } @@ -855,67 +690,43 @@ void GameProcessEvent(WindowGame& game, double time_s) { c.released = false; c.timestamp_seconds = time_s; - GameEventController(game, c); + GameEventController(c); break; } } } -void GameMainLoop(WindowGame& game) { - bool need_exit = false; - +void WindowContext::Run() { Common::Timer timer; timer.Start(); - if (!GameInit(game, timer)) { - need_exit = true; - } + loop.event = {}; + loop.need_exit = false; + loop.paused.store(false, std::memory_order_release); - for (;;) { - if (need_exit) { - break; - } - - if (GamePollEvent(game) != 0) { - GameProcessEvent(game, timer.GetTimeS()); + while (!loop.need_exit) { + if (SDL_PollEvent(&loop.event) != 0) { + ProcessEvent(timer.GetTimeS()); continue; } - if (game.m_game_is_paused) { + if (loop.paused.load(std::memory_order_acquire)) { if (!timer.IsPaused()) { timer.Pause(); } - - GameWaitEvent(game); - - GameProcessEvent(game, timer.GetTimeS()); - need_exit = game.m_game_need_exit; + if (SDL_WaitEvent(&loop.event) == 0) { + EXIT("%s\n", SDL_GetError()); + } + ProcessEvent(timer.GetTimeS()); continue; } - need_exit = game.m_game_need_exit; - - if (game.m_game_is_paused) { - if (!timer.IsPaused()) { - timer.Pause(); - } - } else { - if (timer.IsPaused()) { - timer.Resume(); - } - - if (!need_exit) { - need_exit = !RenderAndUpdate(game); - } - - if (!need_exit) { - GameShowWindow(game, timer); - } + if (timer.IsPaused()) { + timer.Resume(); } + Common::Thread::SleepMicro(1000); } - - GameClose(game); } static void WindowCreate(WindowContext& context) { @@ -949,29 +760,33 @@ static void WindowCreate(WindowContext& context) { SDL_SetWindowResizable(context.window, SDL_FALSE); } -void WindowInit(uint32_t width, uint32_t height) { +Presenter& WindowInit(uint32_t width, uint32_t height) { EXIT_NOT_IMPLEMENTED(!Common::Thread::IsMainThread()); - EXIT_IF(g_window_ctx != nullptr); + EXIT_IF(g_window != nullptr); - g_window_ctx = new WindowContext; + auto window = std::make_unique(); - g_window_ctx->graphic_ctx.screen_width = width; - g_window_ctx->graphic_ctx.screen_height = height; + window->graphic_ctx.screen_width = width; + window->graphic_ctx.screen_height = height; - WindowCreate(*g_window_ctx); - VulkanCreate(*g_window_ctx); - GraphicsRenderInit(g_window_ctx->graphic_ctx); + WindowCreate(*window); + window->CreateVulkan(); + auto& presenter = *window->presenter; + g_window = std::move(window); + return presenter; } void WindowRun() { KYTY_PROFILER_THREAD("Thread_Window"); + EXIT_IF(g_window == nullptr); - GameMainLoop(g_window_game); + g_window->Run(); +} - // TODO: replace std::_Exit shutdown with full Vulkan teardown, then destroy - // the VMA allocator immediately before vkDestroyDevice. - Common::SubsystemsListSingleton::Instance()->ShutdownAll(); - std::_Exit(0); +void WindowShutdown() { + if (g_window != nullptr) { + g_window.reset(); + } } static int WindowIconRead(void* user, char* data, int size) { @@ -1033,7 +848,7 @@ static void WindowLoadPngIcon(const std::string& path, WindowIcon* icon) { EXIT_NOT_IMPLEMENTED(icon->surface == nullptr); } -void WindowUpdateIcon() { +void WindowContext::UpdateIcon() { static WindowIcon icon; static bool icon_loaded = false; @@ -1046,11 +861,11 @@ void WindowUpdateIcon() { } if (icon.surface != nullptr) { - SDL_SetWindowIcon(g_window_ctx->window, icon.surface); + SDL_SetWindowIcon(window, icon.surface); } } -void WindowUpdateTitle() { +void WindowContext::UpdateTitle() { static char title[128]; static char title_id[12]; static char app_ver[12]; @@ -1059,15 +874,29 @@ void WindowUpdateTitle() { Loader::SystemContentParamSfoGetString("TITLE_ID", title_id, sizeof(title_id)); static bool has_app_ver = Loader::SystemContentParamSfoGetString("APP_VER", app_ver, sizeof(app_ver)); + static uint64_t fps_start = Common::Timer::QueryPerformanceCounter(); + static uint64_t frame_num = 0; + static uint64_t fps_frames = 0; + static double current_fps = 0.0; + + const auto now = Common::Timer::QueryPerformanceCounter(); + const auto frequency = Common::Timer::QueryPerformanceFrequency(); + frame_num++; + fps_frames++; + if (now - fps_start >= frequency) { + current_fps = static_cast(fps_frames) * static_cast(frequency) / + static_cast(now - fps_start); + fps_start = now; + fps_frames = 0; + } auto fps = fmt::format("{}{}{}{}{}{}[{}] [{}], frame: {}, fps: {:f}", (has_title ? title : ""), (has_title ? ", " : ""), (has_title_id ? title_id : ""), (has_title_id ? ", " : ""), (has_app_ver ? app_ver : ""), - (has_app_ver ? " " : ""), g_window_ctx->device_name, - g_window_ctx->processor_name, g_window_game.m_frame_num, - g_window_game.m_current_fps); + (has_app_ver ? " " : ""), device_name, processor_name, + frame_num, current_fps); - SDL_SetWindowTitle(g_window_ctx->window, fps.c_str()); + SDL_SetWindowTitle(window, fps.c_str()); } } // namespace Libs::Graphics diff --git a/src/graphics/presentation/window/windowInternal.h b/src/graphics/presentation/window/windowInternal.h index 6846a5c9..cb1c3fc2 100644 --- a/src/graphics/presentation/window/windowInternal.h +++ b/src/graphics/presentation/window/windowInternal.h @@ -1,31 +1,60 @@ #ifndef EMULATOR_SRC_GRAPHICS_PRESENTATION_WINDOW_WINDOWINTERNAL_H_ #define EMULATOR_SRC_GRAPHICS_PRESENTATION_WINDOW_WINDOWINTERNAL_H_ +#include "SDL_events.h" #include "SDL_video.h" #include "common/threads.h" #include "graphics/host_gpu/graphicContext.h" #include "graphics/host_gpu/vulkanCommon.h" +#include #include +#include #include namespace Libs::Graphics { +class Presenter; +class RenderContext; + struct SurfaceCapabilities { vk::SurfaceCapabilitiesKHR capabilities {}; std::vector formats; std::vector present_modes; - bool format_srgb_bgra32 = false; - bool format_unorm_bgra32 = false; +}; + +struct WindowLoopState { + SDL_Event event {}; + bool need_exit = false; + std::atomic_bool paused = false; }; struct WindowContext { - GraphicContext graphic_ctx; - VulkanSwapchain* swapchain = nullptr; - SDL_Window* window = nullptr; - bool window_hidden = true; - vk::SurfaceKHR surface = nullptr; - SurfaceCapabilities* surface_capabilities = nullptr; + WindowContext(); + ~WindowContext(); + KYTY_CLASS_NO_COPY(WindowContext); + + [[nodiscard]] static vk::PhysicalDeviceVulkan13Features + RequiredVulkan13Features() noexcept; + void CreateVulkan(); + void RecreateSurface(); + void RefreshSurfaceCapabilities(); + void UpdateIcon(); + void UpdateTitle(); + void Resize(uint32_t width, uint32_t height); + void ProcessWindowEvent(const SDL_WindowEvent& event); + void ProcessDisplayEvent(const SDL_DisplayEvent& event); + void ProcessEvent(double time_seconds); + void Run(); + + GraphicContext graphic_ctx; + SDL_Window* window = nullptr; + bool window_hidden = true; + vk::SurfaceKHR surface = nullptr; + SurfaceCapabilities surface_capabilities; + std::unique_ptr render_context; + std::unique_ptr presenter; + WindowLoopState loop; char device_name[VK_MAX_PHYSICAL_DEVICE_NAME_SIZE] = {0}; char processor_name[64] = {0}; @@ -33,16 +62,6 @@ struct WindowContext { Common::Mutex mutex; }; -extern WindowContext* g_window_ctx; - -void VulkanGetSurfaceCapabilities(vk::PhysicalDevice physical_device, vk::SurfaceKHR surface, - SurfaceCapabilities& capabilities); -VulkanSwapchain* VulkanCreateSwapchain(uint32_t image_count); -void VulkanCreate(WindowContext& window); - -void WindowUpdateIcon(); -void WindowUpdateTitle(); - } // namespace Libs::Graphics #endif // EMULATOR_SRC_GRAPHICS_PRESENTATION_WINDOW_WINDOWINTERNAL_H_ diff --git a/src/graphics/shader/recompiler/ShaderIR.h b/src/graphics/shader/recompiler/ShaderIR.h index 417abf49..4b86e71f 100644 --- a/src/graphics/shader/recompiler/ShaderIR.h +++ b/src/graphics/shader/recompiler/ShaderIR.h @@ -726,6 +726,7 @@ struct ShaderInfo { std::vector inputs; std::vector outputs; int32_t vertex_offset_sgpr = -1; + bool has_bitwise_xor = false; bool operator==(const ShaderInfo& other) const = default; }; diff --git a/src/graphics/shader/recompiler/ShaderInfoCollection.cpp b/src/graphics/shader/recompiler/ShaderInfoCollection.cpp index 6e89075a..67d8fbce 100644 --- a/src/graphics/shader/recompiler/ShaderInfoCollection.cpp +++ b/src/graphics/shader/recompiler/ShaderInfoCollection.cpp @@ -87,7 +87,7 @@ bool ValidateOptions(const Program& program, const ShaderInfoOptions& options, s } void CollectVertexInputs(const Program& program, const ShaderVertexInputInfo* vertex, - ShaderInfo& info) { + ShaderInfo& info) { AddInput(info, StageInputKind::VertexIndex, 0, 1, "gl_VertexIndex"); AddInput(info, StageInputKind::InstanceIndex, 0, 1, "gl_InstanceIndex"); if (vertex == nullptr) { @@ -130,7 +130,7 @@ void CollectPixelInputs(const ShaderPixelInputInfo* pixel, ShaderInfo& info) { } void CollectComputeInputs(const Program& program, const ShaderComputeInputInfo* compute, - ShaderInfo& info) { + ShaderInfo& info) { if (compute != nullptr) { if (compute->group_id[0] || compute->group_id[1] || compute->group_id[2]) { AddInput(info, StageInputKind::WorkgroupId, 0, 3, "gl_WorkGroupID"); @@ -197,7 +197,7 @@ bool CollectShaderInfo(Program& program, const ShaderInfoOptions& options, std:: if (!program.resource_tracking_complete || program.shader_info_complete) { if (error != nullptr) { *error = !program.resource_tracking_complete ? "shader resources were not tracked" - : "shader info already collected"; + : "shader info already collected"; } return false; } @@ -208,6 +208,18 @@ bool CollectShaderInfo(Program& program, const ShaderInfoOptions& options, std:: auto next = program.info; next.inputs.clear(); next.outputs.clear(); + next.has_bitwise_xor = + std::any_of(program.blocks.begin(), program.blocks.end(), [](const auto& block) { + return std::any_of(block.instructions.begin(), block.instructions.end(), + [](const auto& inst) { + switch (inst.op) { + case Opcode::BitwiseXorU32: + case Opcode::BitwiseXor3U32: + case Opcode::XorAddU32: return true; + default: return false; + } + }); + }); switch (program.stage) { case ShaderType::Vertex: CollectVertexInputs(program, options.vertex, next); break; case ShaderType::Pixel: CollectPixelInputs(options.pixel, next); break; diff --git a/src/graphics/shader/recompiler/spirvEmitter/spirvEmitterControlFlow.cpp b/src/graphics/shader/recompiler/spirvEmitter/spirvEmitterControlFlow.cpp index b37fff98..0c3f55c9 100644 --- a/src/graphics/shader/recompiler/spirvEmitter/spirvEmitterControlFlow.cpp +++ b/src/graphics/shader/recompiler/spirvEmitter/spirvEmitterControlFlow.cpp @@ -139,9 +139,8 @@ uint32_t EmitVsharpDwordLoad(EmitterState& state, uint32_t dword_index) { const auto pointer = state.builder.AllocateId(); const auto value = state.builder.AllocateId(); state.builder.AddFunction({OpAccessChain, state.ptr_push_constant_uint, pointer, - state.push_constant_variable, ConstantU32(state, 0), - ConstantU32(state, dword_index / 4u), - ConstantU32(state, dword_index % 4u)}); + state.push_constant_variable, ConstantU32(state, 0), + ConstantU32(state, dword_index)}); state.builder.AddFunction({OpLoad, state.uint_type, value, pointer}); return value; } @@ -149,8 +148,8 @@ uint32_t EmitVsharpDwordLoad(EmitterState& state, uint32_t dword_index) { const auto pointer = state.builder.AllocateId(); const auto value = state.builder.AllocateId(); state.builder.AddFunction({OpAccessChain, state.ptr_storage_buffer_uint, pointer, - state.vsharp_storage_variable, ConstantU32(state, 0), - ConstantU32(state, dword_index)}); + state.vsharp_storage_variable, ConstantU32(state, 0), + ConstantU32(state, dword_index)}); state.builder.AddFunction({OpLoad, state.uint_type, value, pointer}); return value; } @@ -192,8 +191,8 @@ void EmitRegisterVariables(EmitterState& state) { {OpVariable, state.ptr_func_uint, state.dispatch_pc_variable, StorageClassFunction}); } if (state.pixel_valid_mask_variable != 0) { - state.builder.AddFunction({OpVariable, state.ptr_func_uint, - state.pixel_valid_mask_variable, StorageClassFunction}); + state.builder.AddFunction({OpVariable, state.ptr_func_uint, state.pixel_valid_mask_variable, + StorageClassFunction}); state.builder.AddFunction( {OpStore, state.pixel_valid_mask_variable, ConstantU32(state, 1)}); } @@ -226,9 +225,9 @@ void EmitComputeInputRegisters(EmitterState& state) { if (!cs->group_id[i]) { continue; } - const auto pointer = PointerForRegister( - state, - {IR::RegisterFile::Scalar, static_cast(cs->workgroup_register) + reg_offset}); + const auto pointer = + PointerForRegister(state, {IR::RegisterFile::Scalar, + static_cast(cs->workgroup_register) + reg_offset}); if (pointer != 0) { state.builder.AddFunction( {OpStore, pointer, @@ -238,9 +237,9 @@ void EmitComputeInputRegisters(EmitterState& state) { } if (cs->tg_size_en) { - const auto pointer = PointerForRegister( - state, - {IR::RegisterFile::Scalar, static_cast(cs->workgroup_register) + reg_offset}); + const auto pointer = + PointerForRegister(state, {IR::RegisterFile::Scalar, + static_cast(cs->workgroup_register) + reg_offset}); if (pointer != 0) { const uint32_t wave_size = cs->wave_size != 0 ? cs->wave_size : 64u; const uint32_t total_threads = std::max(cs->threads_num[0], 1u) * @@ -263,7 +262,7 @@ void EmitComputeInputRegisters(EmitterState& state) { state.builder.AddFunction( {OpIEqual, state.bool_type, is_first, wave_id, ConstantU32(state, 0)}); state.builder.AddFunction({OpSelect, state.uint_type, first_bit, is_first, - ConstantU32(state, 0x80000000u), ConstantU32(state, 0)}); + ConstantU32(state, 0x80000000u), ConstantU32(state, 0)}); state.builder.AddFunction( {OpBitwiseOr, state.uint_type, base, wave_bits, ConstantU32(state, waves)}); state.builder.AddFunction({OpBitwiseOr, state.uint_type, packed, base, first_bit}); @@ -360,7 +359,7 @@ void EmitVertexInputRegisters(EmitterState& state) { const auto instance_index = PointerForRegister(state, {IR::RegisterFile::Vector, 8}); if (instance_index != 0) { state.builder.AddFunction({OpStore, instance_index, - EmitInputScalarU32(state, IR::StageInputKind::InstanceIndex)}); + EmitInputScalarU32(state, IR::StageInputKind::InstanceIndex)}); } } @@ -861,7 +860,7 @@ uint32_t DispatcherTargetValue(EmitterState& state, uint32_t block_id) { void EmitDispatcherStoreTarget(EmitterState& state, uint32_t block_id) { state.builder.AddFunction({OpStore, state.dispatch_pc_variable, - ConstantU32(state, DispatcherTargetValue(state, block_id))}); + ConstantU32(state, DispatcherTargetValue(state, block_id))}); } void EmitDispatcherExit(EmitterState& state); @@ -912,7 +911,7 @@ void EmitDispatcherStoreSelectorTarget(EmitterState& state, const CFG::Terminato for (uint32_t i = 0; i < term.indirect_selector_values.size(); i++) { const auto match = state.builder.AllocateId(); state.builder.AddFunction({OpIEqual, state.bool_type, match, selector_value, - ConstantU32(state, term.indirect_selector_values[i])}); + ConstantU32(state, term.indirect_selector_values[i])}); const auto next = state.builder.AllocateId(); state.builder.AddFunction( {OpSelect, state.uint_type, next, match, @@ -949,7 +948,7 @@ void EmitDispatcherStoreIndirectTarget(EmitterState& state, const CFG::Terminato for (uint32_t i = 0; i < count; i++) { const auto match = state.builder.AllocateId(); state.builder.AddFunction({OpIEqual, state.bool_type, match, pc_value, - ConstantU32(state, term.indirect_target_pcs[i])}); + ConstantU32(state, term.indirect_target_pcs[i])}); const auto next = state.builder.AllocateId(); state.builder.AddFunction( {OpSelect, state.uint_type, next, match, @@ -1001,8 +1000,8 @@ void EmitDispatcherSwitch(EmitterState& state, const IR::Program& program) { const auto done = state.builder.AllocateId(); state.builder.AddFunction( {OpIEqual, state.bool_type, done, pc_value, ConstantU32(state, UINT32_MAX)}); - state.builder.AddFunction({OpLoopMerge, state.dispatch_merge_label, - state.dispatch_continue_label, LoopControlNone}); + state.builder.AddFunction( + {OpLoopMerge, state.dispatch_merge_label, state.dispatch_continue_label, LoopControlNone}); state.builder.AddFunction( {OpBranchConditional, done, state.dispatch_merge_label, state.dispatch_select_label}); diff --git a/src/graphics/shader/recompiler/spirvEmitter/spirvEmitterInternal.h b/src/graphics/shader/recompiler/spirvEmitter/spirvEmitterInternal.h index 1494e27c..096019f5 100644 --- a/src/graphics/shader/recompiler/spirvEmitter/spirvEmitterInternal.h +++ b/src/graphics/shader/recompiler/spirvEmitter/spirvEmitterInternal.h @@ -304,73 +304,72 @@ struct EmitterState { Builder builder; const IR::Program& program; const IR::ResourceSnapshot& resources; - const ShaderVertexInputInfo* vertex_input_info = nullptr; - const ShaderPixelInputInfo* pixel_input_info = nullptr; - const ShaderComputeInputInfo* compute_input_info = nullptr; - ShaderType stage = ShaderType::Unknown; - uint32_t wave_size = 64; - bool exact_subgroup_operations = false; - bool per_invocation_masks = false; - uint32_t void_type = 0; - uint32_t bool_type = 0; - uint32_t uint_type = 0; - uint32_t uint_pair_type = 0; - uint32_t int_pair_type = 0; - uint32_t int_type = 0; - uint32_t float_type = 0; - uint32_t vec2_uint_type = 0; - uint32_t vec3_uint_type = 0; - uint32_t vec4_uint_type = 0; - uint32_t vec2_int_type = 0; - uint32_t vec3_int_type = 0; - uint32_t vec4_int_type = 0; - uint32_t vec2_float_type = 0; - uint32_t vec3_float_type = 0; - uint32_t vec4_float_type = 0; - uint32_t ptr_func_uint = 0; - uint32_t ptr_input_float = 0; - uint32_t ptr_input_bool = 0; - uint32_t ptr_input_int = 0; - uint32_t ptr_input_uint = 0; - uint32_t ptr_input_vec2_float = 0; - uint32_t ptr_input_vec3_float = 0; - uint32_t ptr_input_vec2_int = 0; - uint32_t ptr_input_vec3_int = 0; - uint32_t ptr_input_vec4_int = 0; - uint32_t ptr_input_vec2_uint = 0; - uint32_t ptr_input_vec3_uint = 0; - uint32_t ptr_input_vec4_uint = 0; - uint32_t ptr_input_vec4_float = 0; - uint32_t sample_mask_array_type = 0; - uint32_t ptr_output_int = 0; - uint32_t ptr_output_sample_mask_array = 0; - uint32_t ptr_output_float = 0; - uint32_t ptr_output_vec4_float = 0; - uint32_t per_vertex_type = 0; - uint32_t ptr_output_per_vertex = 0; - uint32_t storage_runtime_array_type = 0; - uint32_t storage_buffer_type = 0; - uint32_t ptr_storage_buffer = 0; - uint32_t ptr_storage_buffer_uint = 0; - uint32_t storage_buffer_array_type = 0; - uint32_t ptr_storage_buffer_array = 0; - uint32_t storage_buffer_variable = 0; - uint32_t address_memory_array_type = 0; - uint32_t ptr_address_memory_array = 0; - uint32_t address_memory_variable = 0; - uint32_t gds_variable = 0; - uint32_t push_constant_u32x4_array_type = 0; - uint32_t push_constant_rows_array_type = 0; - uint32_t push_constant_block_type = 0; - uint32_t ptr_push_constant_block = 0; - uint32_t ptr_push_constant_uint = 0; - uint32_t push_constant_variable = 0; - uint32_t vsharp_storage_variable = 0; - uint32_t flattened_srt_variable = 0; - uint32_t lds_array_type = 0; - uint32_t ptr_workgroup_array = 0; - uint32_t ptr_workgroup_uint = 0; - uint32_t lds_variable = 0; + const ShaderVertexInputInfo* vertex_input_info = nullptr; + const ShaderPixelInputInfo* pixel_input_info = nullptr; + const ShaderComputeInputInfo* compute_input_info = nullptr; + ShaderType stage = ShaderType::Unknown; + uint32_t wave_size = 64; + bool exact_subgroup_operations = false; + bool per_invocation_masks = false; + uint32_t void_type = 0; + uint32_t bool_type = 0; + uint32_t uint_type = 0; + uint32_t uint_pair_type = 0; + uint32_t int_pair_type = 0; + uint32_t int_type = 0; + uint32_t float_type = 0; + uint32_t vec2_uint_type = 0; + uint32_t vec3_uint_type = 0; + uint32_t vec4_uint_type = 0; + uint32_t vec2_int_type = 0; + uint32_t vec3_int_type = 0; + uint32_t vec4_int_type = 0; + uint32_t vec2_float_type = 0; + uint32_t vec3_float_type = 0; + uint32_t vec4_float_type = 0; + uint32_t ptr_func_uint = 0; + uint32_t ptr_input_float = 0; + uint32_t ptr_input_bool = 0; + uint32_t ptr_input_int = 0; + uint32_t ptr_input_uint = 0; + uint32_t ptr_input_vec2_float = 0; + uint32_t ptr_input_vec3_float = 0; + uint32_t ptr_input_vec2_int = 0; + uint32_t ptr_input_vec3_int = 0; + uint32_t ptr_input_vec4_int = 0; + uint32_t ptr_input_vec2_uint = 0; + uint32_t ptr_input_vec3_uint = 0; + uint32_t ptr_input_vec4_uint = 0; + uint32_t ptr_input_vec4_float = 0; + uint32_t sample_mask_array_type = 0; + uint32_t ptr_output_int = 0; + uint32_t ptr_output_sample_mask_array = 0; + uint32_t ptr_output_float = 0; + uint32_t ptr_output_vec4_float = 0; + uint32_t per_vertex_type = 0; + uint32_t ptr_output_per_vertex = 0; + uint32_t storage_runtime_array_type = 0; + uint32_t storage_buffer_type = 0; + uint32_t ptr_storage_buffer = 0; + uint32_t ptr_storage_buffer_uint = 0; + uint32_t storage_buffer_array_type = 0; + uint32_t ptr_storage_buffer_array = 0; + uint32_t storage_buffer_variable = 0; + uint32_t address_memory_array_type = 0; + uint32_t ptr_address_memory_array = 0; + uint32_t address_memory_variable = 0; + uint32_t gds_variable = 0; + uint32_t push_constant_array_type = 0; + uint32_t push_constant_block_type = 0; + uint32_t ptr_push_constant_block = 0; + uint32_t ptr_push_constant_uint = 0; + uint32_t push_constant_variable = 0; + uint32_t vsharp_storage_variable = 0; + uint32_t flattened_srt_variable = 0; + uint32_t lds_array_type = 0; + uint32_t ptr_workgroup_array = 0; + uint32_t ptr_workgroup_uint = 0; + uint32_t lds_variable = 0; std::array sampled_images; uint32_t sampler_type = 0; uint32_t sampler_array_type = 0; diff --git a/src/graphics/shader/recompiler/spirvEmitter/spirvEmitterModule.cpp b/src/graphics/shader/recompiler/spirvEmitter/spirvEmitterModule.cpp index 8e5483d2..7fa6bf22 100644 --- a/src/graphics/shader/recompiler/spirvEmitter/spirvEmitterModule.cpp +++ b/src/graphics/shader/recompiler/spirvEmitter/spirvEmitterModule.cpp @@ -237,9 +237,9 @@ uint32_t BuiltInForInput(IR::StageInputKind kind) { void AddInputAnnotationsAndNames(EmitterState& state) { if (state.subgroup_local_invocation_id_variable != 0) { state.builder.AddName(state.subgroup_local_invocation_id_variable, - "gl_SubgroupInvocationID"); + "gl_SubgroupInvocationID"); state.builder.AddAnnotation({OpDecorate, state.subgroup_local_invocation_id_variable, - DecorationBuiltIn, BuiltInSubgroupLocalInvocationId}); + DecorationBuiltIn, BuiltInSubgroupLocalInvocationId}); if (state.stage == ShaderType::Pixel) { state.builder.AddAnnotation( {OpDecorate, state.subgroup_local_invocation_id_variable, DecorationFlat}); @@ -373,13 +373,10 @@ void AddVsharpAnnotationsAndNames(EmitterState& state) { state.builder.AddName(state.push_constant_block_type, "BufferResource"); state.builder.AddName(state.push_constant_variable, "vsharp"); state.builder.AddAnnotation( - {OpDecorate, state.push_constant_u32x4_array_type, DecorationArrayStride, 4}); - state.builder.AddAnnotation( - {OpDecorate, state.push_constant_rows_array_type, DecorationArrayStride, 16}); + {OpDecorate, state.push_constant_array_type, DecorationArrayStride, 4}); state.builder.AddAnnotation({OpMemberDecorate, state.push_constant_block_type, 0, - DecorationOffset, bind.push_constant_offset}); - state.builder.AddAnnotation( - {OpDecorate, state.push_constant_block_type, DecorationBlock}); + DecorationOffset, bind.push_constant_offset}); + state.builder.AddAnnotation({OpDecorate, state.push_constant_block_type, DecorationBlock}); } if (state.vsharp_storage_variable != 0) { const auto* storage = DescriptorBinding(state, IR::DescriptorBindingKind::UserData); @@ -439,11 +436,10 @@ void EmitHeaderAndTypes(EmitterState& state) { state.ptr_address_memory_array = state.builder.AllocateId(); } if (state.push_constant_variable != 0) { - state.push_constant_u32x4_array_type = state.builder.AllocateId(); - state.push_constant_rows_array_type = state.builder.AllocateId(); - state.push_constant_block_type = state.builder.AllocateId(); - state.ptr_push_constant_block = state.builder.AllocateId(); - state.ptr_push_constant_uint = state.builder.AllocateId(); + state.push_constant_array_type = state.builder.AllocateId(); + state.push_constant_block_type = state.builder.AllocateId(); + state.ptr_push_constant_block = state.builder.AllocateId(); + state.ptr_push_constant_uint = state.builder.AllocateId(); } state.lds_array_type = state.builder.AllocateId(); state.ptr_workgroup_array = state.builder.AllocateId(); @@ -517,7 +513,7 @@ void EmitHeaderAndTypes(EmitterState& state) { state.builder.AddExtInstImport(state.glsl_std450, "GLSL.std.450"); state.builder.AddMemoryModel({AddressingModelLogical, MemoryModelGLSL450}); state.builder.AddEntryPoint(ExecutionModelForStage(state.stage), state.main_func, "main", - state.interface_variables); + state.interface_variables); if (state.stage == ShaderType::Compute) { uint32_t local_x = state.needs_compute_derivatives ? 2u : 1u; uint32_t local_y = state.needs_compute_derivatives ? 2u : 1u; @@ -563,8 +559,7 @@ void EmitHeaderAndTypes(EmitterState& state) { state.builder.AddType({OpTypeVoid, state.void_type}); state.builder.AddType({OpTypeBool, state.bool_type}); state.builder.AddType({OpTypeInt, state.uint_type, 32, 0}); - state.builder.AddType( - {OpTypeStruct, state.uint_pair_type, state.uint_type, state.uint_type}); + state.builder.AddType({OpTypeStruct, state.uint_pair_type, state.uint_type, state.uint_type}); state.builder.AddType({OpTypeInt, state.int_type, 32, 1}); state.builder.AddType({OpTypeStruct, state.int_pair_type, state.int_type, state.int_type}); state.builder.AddType({OpTypeFloat, state.float_type, 32}); @@ -583,8 +578,7 @@ void EmitHeaderAndTypes(EmitterState& state) { {OpTypePointer, state.ptr_input_float, StorageClassInput, state.float_type}); state.builder.AddType( {OpTypePointer, state.ptr_input_bool, StorageClassInput, state.bool_type}); - state.builder.AddType( - {OpTypePointer, state.ptr_input_int, StorageClassInput, state.int_type}); + state.builder.AddType({OpTypePointer, state.ptr_input_int, StorageClassInput, state.int_type}); state.builder.AddType( {OpTypePointer, state.ptr_input_uint, StorageClassInput, state.uint_type}); state.builder.AddType( @@ -607,7 +601,7 @@ void EmitHeaderAndTypes(EmitterState& state) { {OpTypePointer, state.ptr_input_vec4_float, StorageClassInput, state.vec4_float_type}); if (state.subgroup_local_invocation_id_variable != 0) { state.builder.AddType({OpVariable, state.ptr_input_uint, - state.subgroup_local_invocation_id_variable, StorageClassInput}); + state.subgroup_local_invocation_id_variable, StorageClassInput}); } for (const auto& input: state.inputs) { uint32_t ptr_type = state.ptr_input_uint; @@ -643,15 +637,15 @@ void EmitHeaderAndTypes(EmitterState& state) { if (state.per_vertex_variable != 0) { state.builder.AddType({OpTypeStruct, state.per_vertex_type, state.vec4_float_type}); state.builder.AddType({OpTypePointer, state.ptr_output_per_vertex, StorageClassOutput, - state.per_vertex_type}); - state.builder.AddType({OpVariable, state.ptr_output_per_vertex, - state.per_vertex_variable, StorageClassOutput}); + state.per_vertex_type}); + state.builder.AddType({OpVariable, state.ptr_output_per_vertex, state.per_vertex_variable, + StorageClassOutput}); } for (const auto& binding: state.outputs) { if (binding.kind == IR::StageOutputKind::Parameter || binding.kind == IR::StageOutputKind::Mrt) { - state.builder.AddType({OpVariable, state.ptr_output_vec4_float, binding.variable_id, - StorageClassOutput}); + state.builder.AddType( + {OpVariable, state.ptr_output_vec4_float, binding.variable_id, StorageClassOutput}); } } if (state.depth_variable != 0) { @@ -662,52 +656,48 @@ void EmitHeaderAndTypes(EmitterState& state) { state.builder.AddType( {OpTypeArray, state.sample_mask_array_type, state.int_type, ConstantU32(state, 1)}); state.builder.AddType({OpTypePointer, state.ptr_output_sample_mask_array, - StorageClassOutput, state.sample_mask_array_type}); + StorageClassOutput, state.sample_mask_array_type}); state.builder.AddType({OpVariable, state.ptr_output_sample_mask_array, - state.sample_mask_variable, StorageClassOutput}); + state.sample_mask_variable, StorageClassOutput}); } - state.builder.AddType( - {OpTypeRuntimeArray, state.storage_runtime_array_type, state.uint_type}); + state.builder.AddType({OpTypeRuntimeArray, state.storage_runtime_array_type, state.uint_type}); state.builder.AddType( {OpTypeStruct, state.storage_buffer_type, state.storage_runtime_array_type}); state.builder.AddType({OpTypePointer, state.ptr_storage_buffer, StorageClassStorageBuffer, - state.storage_buffer_type}); - state.builder.AddType({OpTypePointer, state.ptr_storage_buffer_uint, - StorageClassStorageBuffer, state.uint_type}); + state.storage_buffer_type}); + state.builder.AddType( + {OpTypePointer, state.ptr_storage_buffer_uint, StorageClassStorageBuffer, state.uint_type}); if (state.storage_buffer_variable != 0) { const auto count = ConstantU32(state, DescriptorCount(state, IR::DescriptorBindingKind::Buffers)); state.builder.AddType( {OpTypeArray, state.storage_buffer_array_type, state.storage_buffer_type, count}); state.builder.AddType({OpTypePointer, state.ptr_storage_buffer_array, - StorageClassStorageBuffer, state.storage_buffer_array_type}); + StorageClassStorageBuffer, state.storage_buffer_array_type}); state.builder.AddType({OpVariable, state.ptr_storage_buffer_array, - state.storage_buffer_variable, StorageClassStorageBuffer}); + state.storage_buffer_variable, StorageClassStorageBuffer}); } if (state.gds_variable != 0) { - state.builder.AddType({OpVariable, state.ptr_storage_buffer, state.gds_variable, - StorageClassStorageBuffer}); + state.builder.AddType( + {OpVariable, state.ptr_storage_buffer, state.gds_variable, StorageClassStorageBuffer}); } if (state.push_constant_variable != 0) { - const auto row_width = ConstantU32(state, 4); - const auto row_count = ConstantU32( - state, std::max((state.program.bindings.push_constant_size + 15u) / 16u, 1u)); + const auto dword_count = + ConstantU32(state, state.program.bindings.push_constant_size / sizeof(uint32_t)); state.builder.AddType( - {OpTypeArray, state.push_constant_u32x4_array_type, state.uint_type, row_width}); - state.builder.AddType({OpTypeArray, state.push_constant_rows_array_type, - state.push_constant_u32x4_array_type, row_count}); + {OpTypeArray, state.push_constant_array_type, state.uint_type, dword_count}); state.builder.AddType( - {OpTypeStruct, state.push_constant_block_type, state.push_constant_rows_array_type}); + {OpTypeStruct, state.push_constant_block_type, state.push_constant_array_type}); state.builder.AddType({OpTypePointer, state.ptr_push_constant_block, - StorageClassPushConstant, state.push_constant_block_type}); + StorageClassPushConstant, state.push_constant_block_type}); state.builder.AddType({OpTypePointer, state.ptr_push_constant_uint, - StorageClassPushConstant, state.uint_type}); + StorageClassPushConstant, state.uint_type}); state.builder.AddType({OpVariable, state.ptr_push_constant_block, - state.push_constant_variable, StorageClassPushConstant}); + state.push_constant_variable, StorageClassPushConstant}); } if (state.vsharp_storage_variable != 0) { - state.builder.AddType({OpVariable, state.ptr_storage_buffer, - state.vsharp_storage_variable, StorageClassStorageBuffer}); + state.builder.AddType({OpVariable, state.ptr_storage_buffer, state.vsharp_storage_variable, + StorageClassStorageBuffer}); } if (state.address_memory_variable != 0) { const auto count = @@ -715,13 +705,13 @@ void EmitHeaderAndTypes(EmitterState& state) { state.builder.AddType( {OpTypeArray, state.address_memory_array_type, state.storage_buffer_type, count}); state.builder.AddType({OpTypePointer, state.ptr_address_memory_array, - StorageClassStorageBuffer, state.address_memory_array_type}); + StorageClassStorageBuffer, state.address_memory_array_type}); state.builder.AddType({OpVariable, state.ptr_address_memory_array, - state.address_memory_variable, StorageClassStorageBuffer}); + state.address_memory_variable, StorageClassStorageBuffer}); } if (state.flattened_srt_variable != 0) { - state.builder.AddType({OpVariable, state.ptr_storage_buffer, - state.flattened_srt_variable, StorageClassStorageBuffer}); + state.builder.AddType({OpVariable, state.ptr_storage_buffer, state.flattened_srt_variable, + StorageClassStorageBuffer}); } if (state.stage == ShaderType::Compute || state.needs_function_lds) { const auto storage_class = @@ -733,8 +723,8 @@ void EmitHeaderAndTypes(EmitterState& state) { state.builder.AddType( {OpTypePointer, state.ptr_workgroup_uint, storage_class, state.uint_type}); if (state.stage == ShaderType::Compute) { - state.builder.AddType({OpVariable, state.ptr_workgroup_array, state.lds_variable, - StorageClassWorkgroup}); + state.builder.AddType( + {OpVariable, state.ptr_workgroup_array, state.lds_variable, StorageClassWorkgroup}); } } for (uint32_t i = 0; i < state.sampled_images.size(); i++) { @@ -744,7 +734,7 @@ void EmitHeaderAndTypes(EmitterState& state) { const auto dimension = view == ImageViewKind::Dim3D ? Dim3D : Dim2D; const auto arrayed = view == ImageViewKind::Dim2DArray ? 1u : 0u; state.builder.AddType({OpTypeImage, image.image_type, component, dimension, 0, arrayed, 0, - 1, ImageFormatUnknown}); + 1, ImageFormatUnknown}); state.builder.AddType({OpTypeSampledImage, image.sampled_image_type, image.image_type}); state.builder.AddType( {OpTypePointer, image.pointer_type, StorageClassUniformConstant, image.image_type}); @@ -753,113 +743,110 @@ void EmitHeaderAndTypes(EmitterState& state) { const auto count = ConstantU32(state, DescriptorCount(state, kind)); state.builder.AddType({OpTypeArray, image.array_type, image.image_type, count}); state.builder.AddType({OpTypePointer, image.array_pointer_type, - StorageClassUniformConstant, image.array_type}); + StorageClassUniformConstant, image.array_type}); state.builder.AddType({OpVariable, image.array_pointer_type, image.variable, - StorageClassUniformConstant}); + StorageClassUniformConstant}); } } state.builder.AddType({OpTypeSampler, state.sampler_type}); state.builder.AddType({OpTypePointer, state.ptr_uniform_sampler, StorageClassUniformConstant, - state.sampler_type}); + state.sampler_type}); if (state.sampler_variable != 0) { const auto count = ConstantU32(state, DescriptorCount(state, IR::DescriptorBindingKind::Samplers)); - state.builder.AddType( - {OpTypeArray, state.sampler_array_type, state.sampler_type, count}); + state.builder.AddType({OpTypeArray, state.sampler_array_type, state.sampler_type, count}); state.builder.AddType({OpTypePointer, state.ptr_uniform_sampler_array, - StorageClassUniformConstant, state.sampler_array_type}); - state.builder.AddType({OpVariable, state.ptr_uniform_sampler_array, - state.sampler_variable, StorageClassUniformConstant}); + StorageClassUniformConstant, state.sampler_array_type}); + state.builder.AddType({OpVariable, state.ptr_uniform_sampler_array, state.sampler_variable, + StorageClassUniformConstant}); } - state.builder.AddType({OpTypeImage, state.storage_image_type, state.float_type, Dim2D, 0, 0, - 0, 2, ImageFormatUnknown}); + state.builder.AddType({OpTypeImage, state.storage_image_type, state.float_type, Dim2D, 0, 0, 0, + 2, ImageFormatUnknown}); state.builder.AddType({OpTypePointer, state.ptr_uniform_storage_image, - StorageClassUniformConstant, state.storage_image_type}); + StorageClassUniformConstant, state.storage_image_type}); if (state.storage_image_variable != 0) { const auto count = ConstantU32(state, DescriptorCount(state, IR::DescriptorBindingKind::Storage2D)); state.builder.AddType( {OpTypeArray, state.storage_image_array_type, state.storage_image_type, count}); state.builder.AddType({OpTypePointer, state.ptr_uniform_storage_image_array, - StorageClassUniformConstant, state.storage_image_array_type}); + StorageClassUniformConstant, state.storage_image_array_type}); state.builder.AddType({OpVariable, state.ptr_uniform_storage_image_array, - state.storage_image_variable, StorageClassUniformConstant}); + state.storage_image_variable, StorageClassUniformConstant}); } - state.builder.AddType({OpTypeImage, state.storage_image_2d_array_type, state.float_type, - Dim2D, 0, 1, 0, 2, ImageFormatUnknown}); + state.builder.AddType({OpTypeImage, state.storage_image_2d_array_type, state.float_type, Dim2D, + 0, 1, 0, 2, ImageFormatUnknown}); state.builder.AddType({OpTypePointer, state.ptr_uniform_storage_image_2d_array, - StorageClassUniformConstant, state.storage_image_2d_array_type}); + StorageClassUniformConstant, state.storage_image_2d_array_type}); if (state.storage_image_2d_array_variable != 0) { const auto count = ConstantU32(state, DescriptorCount(state, IR::DescriptorBindingKind::Storage2DArray)); state.builder.AddType({OpTypeArray, state.storage_image_2d_array_array_type, - state.storage_image_2d_array_type, count}); + state.storage_image_2d_array_type, count}); state.builder.AddType({OpTypePointer, state.ptr_uniform_storage_image_2d_array_array, - StorageClassUniformConstant, - state.storage_image_2d_array_array_type}); + StorageClassUniformConstant, + state.storage_image_2d_array_array_type}); state.builder.AddType({OpVariable, state.ptr_uniform_storage_image_2d_array_array, - state.storage_image_2d_array_variable, - StorageClassUniformConstant}); + state.storage_image_2d_array_variable, StorageClassUniformConstant}); } - state.builder.AddType({OpTypeImage, state.storage_image_3d_type, state.float_type, Dim3D, 0, - 0, 0, 2, ImageFormatUnknown}); + state.builder.AddType({OpTypeImage, state.storage_image_3d_type, state.float_type, Dim3D, 0, 0, + 0, 2, ImageFormatUnknown}); state.builder.AddType({OpTypePointer, state.ptr_uniform_storage_image_3d, - StorageClassUniformConstant, state.storage_image_3d_type}); + StorageClassUniformConstant, state.storage_image_3d_type}); if (state.storage_image_3d_variable != 0) { const auto count = ConstantU32(state, DescriptorCount(state, IR::DescriptorBindingKind::Storage3D)); state.builder.AddType( {OpTypeArray, state.storage_image_3d_array_type, state.storage_image_3d_type, count}); state.builder.AddType({OpTypePointer, state.ptr_uniform_storage_image_3d_array, - StorageClassUniformConstant, state.storage_image_3d_array_type}); + StorageClassUniformConstant, state.storage_image_3d_array_type}); state.builder.AddType({OpVariable, state.ptr_uniform_storage_image_3d_array, - state.storage_image_3d_variable, StorageClassUniformConstant}); + state.storage_image_3d_variable, StorageClassUniformConstant}); } - state.builder.AddType({OpTypeImage, state.storage_image_uint_type, state.uint_type, Dim2D, 0, - 0, 0, 2, ImageFormatR32ui}); + state.builder.AddType({OpTypeImage, state.storage_image_uint_type, state.uint_type, Dim2D, 0, 0, + 0, 2, ImageFormatR32ui}); state.builder.AddType({OpTypePointer, state.ptr_uniform_storage_image_uint, - StorageClassUniformConstant, state.storage_image_uint_type}); + StorageClassUniformConstant, state.storage_image_uint_type}); if (state.storage_image_uint_variable != 0) { const auto count = ConstantU32(state, DescriptorCount(state, IR::DescriptorBindingKind::StorageUint2D)); state.builder.AddType({OpTypeArray, state.storage_image_uint_array_type, - state.storage_image_uint_type, count}); + state.storage_image_uint_type, count}); state.builder.AddType({OpTypePointer, state.ptr_uniform_storage_image_uint_array, - StorageClassUniformConstant, state.storage_image_uint_array_type}); + StorageClassUniformConstant, state.storage_image_uint_array_type}); state.builder.AddType({OpVariable, state.ptr_uniform_storage_image_uint_array, - state.storage_image_uint_variable, StorageClassUniformConstant}); + state.storage_image_uint_variable, StorageClassUniformConstant}); } state.builder.AddType({OpTypeImage, state.storage_image_uint_2d_array_type, state.uint_type, - Dim2D, 0, 1, 0, 2, ImageFormatR32ui}); + Dim2D, 0, 1, 0, 2, ImageFormatR32ui}); state.builder.AddType({OpTypePointer, state.ptr_uniform_storage_image_uint_2d_array, - StorageClassUniformConstant, state.storage_image_uint_2d_array_type}); + StorageClassUniformConstant, state.storage_image_uint_2d_array_type}); if (state.storage_image_uint_2d_array_variable != 0) { const auto count = ConstantU32( state, DescriptorCount(state, IR::DescriptorBindingKind::StorageUint2DArray)); state.builder.AddType({OpTypeArray, state.storage_image_uint_2d_array_array_type, - state.storage_image_uint_2d_array_type, count}); + state.storage_image_uint_2d_array_type, count}); state.builder.AddType({OpTypePointer, state.ptr_uniform_storage_image_uint_2d_array_array, - StorageClassUniformConstant, - state.storage_image_uint_2d_array_array_type}); + StorageClassUniformConstant, + state.storage_image_uint_2d_array_array_type}); state.builder.AddType({OpVariable, state.ptr_uniform_storage_image_uint_2d_array_array, - state.storage_image_uint_2d_array_variable, - StorageClassUniformConstant}); + state.storage_image_uint_2d_array_variable, + StorageClassUniformConstant}); } - state.builder.AddType({OpTypeImage, state.storage_image_uint_3d_type, state.uint_type, Dim3D, - 0, 0, 0, 2, ImageFormatR32ui}); + state.builder.AddType({OpTypeImage, state.storage_image_uint_3d_type, state.uint_type, Dim3D, 0, + 0, 0, 2, ImageFormatR32ui}); state.builder.AddType({OpTypePointer, state.ptr_uniform_storage_image_uint_3d, - StorageClassUniformConstant, state.storage_image_uint_3d_type}); + StorageClassUniformConstant, state.storage_image_uint_3d_type}); if (state.storage_image_uint_3d_variable != 0) { const auto count = ConstantU32(state, DescriptorCount(state, IR::DescriptorBindingKind::StorageUint3D)); state.builder.AddType({OpTypeArray, state.storage_image_uint_3d_array_type, - state.storage_image_uint_3d_type, count}); + state.storage_image_uint_3d_type, count}); state.builder.AddType({OpTypePointer, state.ptr_uniform_storage_image_uint_3d_array, - StorageClassUniformConstant, - state.storage_image_uint_3d_array_type}); + StorageClassUniformConstant, + state.storage_image_uint_3d_array_type}); state.builder.AddType({OpVariable, state.ptr_uniform_storage_image_uint_3d_array, - state.storage_image_uint_3d_variable, - StorageClassUniformConstant}); + state.storage_image_uint_3d_variable, StorageClassUniformConstant}); } state.builder.AddType( {OpTypePointer, state.ptr_image_uint, StorageClassImage, state.uint_type}); diff --git a/src/graphics/shader/shader.cpp b/src/graphics/shader/shader.cpp index c8680d14..dc18047a 100644 --- a/src/graphics/shader/shader.cpp +++ b/src/graphics/shader/shader.cpp @@ -12,6 +12,7 @@ #include "graphics/guest_gpu/gpu_defs.h" #include "graphics/guest_gpu/graphicsRun.h" #include "graphics/guest_gpu/hardwareContext.h" +#include "graphics/host_gpu/renderer/renderContext.h" #include "graphics/shader/recompiler/ShaderDecoder.h" #include "graphics/shader/recompiler/ShaderRecompiler.h" #include "graphics/shader/shaderVertexMetadata.h" @@ -825,6 +826,11 @@ static void ShaderGetStaticInputInfoPS( vs_info.stage.program != nullptr && !vs_info.stage.program->bindings.descriptors.empty() ? 1 : 0; + ps_info.push_constant_offset = + vs_info.stage.program != nullptr + ? vs_info.stage.program->bindings.push_constant_offset + + vs_info.stage.program->bindings.push_constant_size + : 0; for (int i = 0; i < 8; i++) { ps_info.target_output_mode[i] = sh.target_output_mode[i]; @@ -991,7 +997,7 @@ static void ShaderAppendNativeSpecialization(std::vector& ids.push_back(program.bindings.push_constant_size); ids.push_back(static_cast(program.bindings.user_data_registers.size())); ids.insert(ids.end(), program.bindings.user_data_registers.begin(), - program.bindings.user_data_registers.end()); + program.bindings.user_data_registers.end()); ids.push_back(static_cast(program.bindings.descriptors.size())); for (const auto& binding: program.bindings.descriptors) { ids.push_back(static_cast(binding.kind)); @@ -1284,9 +1290,9 @@ static void DumpShaderRecompilerSpirv(const char* type, uint64_t shader_hash, static std::atomic_int id = 0; - const auto base_name = Config::GetShaderLogFolder() / - fmt::format("{:04d}_{:04d}_new_shader_{}_{:016x}", - GraphicsRunGetFrameNum(), id++, type, shader_hash); + const auto base_name = + Config::GetShaderLogFolder() / + fmt::format("{:04d}_new_shader_{}_{:016x}", id++, type, shader_hash); Common::File::CreateDirectories(base_name.parent_path()); Common::File spv_file; @@ -1335,9 +1341,9 @@ static void DumpShaderRecompilerOriginal(const char* type, uint64_t shader_hash, static std::atomic_int id = 0; - const auto base_name = Config::GetShaderLogFolder() / "original" / - fmt::format("{:04d}_{:04d}_new_shader_{}_{:016x}", - GraphicsRunGetFrameNum(), id++, type, shader_hash); + const auto base_name = + Config::GetShaderLogFolder() / "original" / + fmt::format("{:04d}_new_shader_{}_{:016x}", id++, type, shader_hash); Common::File::CreateDirectories(base_name.parent_path()); Common::File bin_file; @@ -1437,7 +1443,7 @@ bool ShaderCompileSpirvPS(const HW::PixelShaderInfo& regs, const HW::ShaderRegis options.user_data_count = regs.ps_regs.rsrc2.user_sgpr; options.user_data = regs.ps_user_sgpr.value; options.descriptor_set = input_info.descriptor_set; - options.push_constant_offset = 0; + options.push_constant_offset = input_info.push_constant_offset; options.pixel_input_info = &input_info; options.dump_ir = ShaderRecompilerTextDumpEnabled(); options.early_dump = options.dump_ir; @@ -1591,6 +1597,7 @@ ShaderId ShaderGetIdPS(const HW::PixelShaderInfo& regs, const ShaderPixelInputIn ret.crc32 = regs.ps_regs.chksum & 0xffffffffu; ret.ids.push_back(input_info.descriptor_set); + ret.ids.push_back(input_info.push_constant_offset); ret.ids.push_back(input_info.input_num); ret.ids.push_back(input_info.ps_system_input_base); ret.ids.push_back(static_cast(input_info.ps_pos_x)); diff --git a/src/graphics/shader/shader.h b/src/graphics/shader/shader.h index 753c1274..5c6c5b1a 100644 --- a/src/graphics/shader/shader.h +++ b/src/graphics/shader/shader.h @@ -103,6 +103,7 @@ struct ShaderPixelInputInfo { std::array target_export_mapping = {}; uint32_t mrt_output_mask = 0; uint32_t descriptor_set = 0; + uint32_t push_constant_offset = 0; bool ps_pos_x = false; bool ps_pos_y = false; bool ps_pos_xy = false; diff --git a/src/kernel/eventQueue.cpp b/src/kernel/eventQueue.cpp index e27d0457..afdc8653 100644 --- a/src/kernel/eventQueue.cpp +++ b/src/kernel/eventQueue.cpp @@ -12,7 +12,10 @@ #include #include #include +#include #include +#include +#include namespace Libs::LibKernel::EventQueue { @@ -29,12 +32,13 @@ static uint64_t MonotonicTimeNs() { .count()); } -static std::list g_equeues; -static Common::Mutex g_equeues_mutex; +static std::unordered_map g_equeues; +static Common::Mutex g_equeues_mutex; +static uint64_t g_next_equeue = 1; class KernelEqueuePrivate { public: - KernelEqueuePrivate() = default; + explicit KernelEqueuePrivate(KernelEqueue handle): m_handle(handle) {} virtual ~KernelEqueuePrivate(); KYTY_CLASS_NO_COPY(KernelEqueuePrivate); @@ -42,12 +46,13 @@ public: [[nodiscard]] const std::string& GetName() const { return m_name; } void SetName(const std::string& m_name) { this->m_name = m_name; } - void AddEvent(const KernelEqueueEvent& event); - bool TriggerEvent(uintptr_t ident, int16_t filter, void* trigger_data); - bool DeleteEvent(uintptr_t ident, int16_t filter); + int AddEvent(const KernelEqueueEvent& event); + int TriggerEvent(uintptr_t ident, int16_t filter, void* trigger_data); + int DeleteEvent(uintptr_t ident, int16_t filter); - int GetTriggeredEvents(KernelEvent* ev, int num); - int WaitForEvents(KernelEvent* ev, int num, uint32_t micros); + int GetTriggeredEvents(KernelEvent* ev, int num); + int WaitForEvents(KernelEvent* ev, int num, uint32_t micros); + void Close(); private: void TriggerExpiredTimers(uint64_t now_ns); @@ -57,22 +62,38 @@ private: Common::Mutex m_mutex; Common::CondVar m_cond_var; std::string m_name; + KernelEqueue m_handle = KERNEL_EQUEUE_INVALID; + bool m_closed = false; }; KernelEqueuePrivate::~KernelEqueuePrivate() { + Close(); +} + +void KernelEqueuePrivate::Close() { Common::LockGuard lock(m_mutex); + if (m_closed) { + return; + } + m_closed = true; for (auto& event: m_events) { if (event.filter.delete_event_func != nullptr) { - event.filter.delete_event_func(this, &event); + auto owner = event.filter.owner; + event.filter.delete_event_func(m_handle, &event); } } + m_events.clear(); + m_cond_var.SignalAll(); } int KernelEqueuePrivate::GetTriggeredEvents(KernelEvent* ev, int num) { Common::LockGuard lock(m_mutex); EXIT_IF(num < 1); + if (m_closed) { + return KERNEL_ERROR_EBADF; + } TriggerExpiredTimers(MonotonicTimeNs()); @@ -146,6 +167,9 @@ int KernelEqueuePrivate::WaitForEvents(KernelEvent* ev, int num, uint32_t micros Common::LockGuard lock(m_mutex); EXIT_IF(num < 1); + if (m_closed) { + return KERNEL_ERROR_EBADF; + } uint32_t elapsed = 0; Common::Timer t; @@ -154,7 +178,7 @@ int KernelEqueuePrivate::WaitForEvents(KernelEvent* ev, int num, uint32_t micros for (;;) { int ret = GetTriggeredEvents(ev, num); - if (ret > 0 || (elapsed >= micros && micros != 0)) { + if (ret != 0 || (elapsed >= micros && micros != 0)) { return ret; } @@ -174,25 +198,36 @@ int KernelEqueuePrivate::WaitForEvents(KernelEvent* ev, int num, uint32_t micros return 0; } -void KernelEqueuePrivate::AddEvent(const KernelEqueueEvent& event) { +int KernelEqueuePrivate::AddEvent(const KernelEqueueEvent& event) { Common::LockGuard lock(m_mutex); + if (m_closed) { + return KERNEL_ERROR_EBADF; + } auto it = std::find_if(m_events.begin(), m_events.end(), [ident = event.event.ident, filter = event.event.filter](const auto& e) { return e.event.ident == ident && e.event.filter == filter; }); if (it != m_events.end()) { - *it = event; + it->deadline_ns = event.deadline_ns; + it->event.udata = event.event.udata; + for (auto& pending: it->pending_events) { + pending.udata = event.event.udata; + } } else { m_events.push_back(event); } m_cond_var.Signal(); + return OK; } -bool KernelEqueuePrivate::TriggerEvent(uintptr_t ident, int16_t filter, void* trigger_data) { +int KernelEqueuePrivate::TriggerEvent(uintptr_t ident, int16_t filter, void* trigger_data) { Common::LockGuard lock(m_mutex); + if (m_closed) { + return KERNEL_ERROR_EBADF; + } auto it = std::find_if(m_events.begin(), m_events.end(), [ident, filter](const auto& e) { return e.event.ident == ident && e.event.filter == filter; }); @@ -207,10 +242,10 @@ bool KernelEqueuePrivate::TriggerEvent(uintptr_t ident, int16_t filter, void* tr m_cond_var.Signal(); - return true; + return OK; } - return false; + return KERNEL_ERROR_ENOENT; } static void UserEventTriggerFunc(KernelEqueueEvent* event, void* trigger_data) { @@ -241,9 +276,12 @@ static void UserEventResetFunc(KernelEqueueEvent* event) { } } -bool KernelEqueuePrivate::DeleteEvent(uintptr_t ident, int16_t filter) { +int KernelEqueuePrivate::DeleteEvent(uintptr_t ident, int16_t filter) { Common::LockGuard lock(m_mutex); + if (m_closed) { + return KERNEL_ERROR_EBADF; + } auto it = std::find_if(m_events.begin(), m_events.end(), [ident, filter](const auto& e) { return e.event.ident == ident && e.event.filter == filter; }); @@ -251,15 +289,26 @@ bool KernelEqueuePrivate::DeleteEvent(uintptr_t ident, int16_t filter) { auto& event = *it; if (event.filter.delete_event_func != nullptr) { - event.filter.delete_event_func(this, &event); + auto owner = event.filter.owner; + event.filter.delete_event_func(m_handle, &event); } m_events.erase(it); - return true; + return OK; } - return false; + return KERNEL_ERROR_ENOENT; +} + +KernelEqueueRef KernelPinEqueue(KernelEqueue eq) { + if (eq == KERNEL_EQUEUE_INVALID) { + return {}; + } + + Common::LockGuard lock(g_equeues_mutex); + const auto entry = g_equeues.find(eq); + return entry != g_equeues.end() ? entry->second : KernelEqueueRef {}; } int KYTY_SYSV_ABI KernelCreateEqueue(KernelEqueue* eq, const char* name) { @@ -269,13 +318,15 @@ int KYTY_SYSV_ABI KernelCreateEqueue(KernelEqueue* eq, const char* name) { return KERNEL_ERROR_EINVAL; } - *eq = new KernelEqueuePrivate; - - (*eq)->SetName(std::string(name)); - { Common::LockGuard lock(g_equeues_mutex); - g_equeues.push_back(*eq); + if (g_next_equeue > static_cast(std::numeric_limits::max())) { + EXIT("event queue handle space exhausted\n"); + } + *eq = static_cast(g_next_equeue++); + auto owner = std::make_shared(*eq); + owner->SetName(std::string(name)); + g_equeues.emplace(*eq, std::move(owner)); } LOGF("\tEqueue create: %s\n", name); @@ -284,55 +335,50 @@ int KYTY_SYSV_ABI KernelCreateEqueue(KernelEqueue* eq, const char* name) { } int KYTY_SYSV_ABI KernelAddEvent(KernelEqueue eq, const KernelEqueueEvent& event) { - if (eq == nullptr) { + auto owner = KernelPinEqueue(eq); + if (!owner) { return KERNEL_ERROR_EBADF; } - eq->AddEvent(event); - - return OK; + return owner->AddEvent(event); } int KYTY_SYSV_ABI KernelTriggerEvent(KernelEqueue eq, uintptr_t ident, int16_t filter, void* trigger_data) { - if (eq == nullptr) { + auto owner = KernelPinEqueue(eq); + if (!owner) { return KERNEL_ERROR_EBADF; } - if (!eq->TriggerEvent(ident, filter, trigger_data)) { - return KERNEL_ERROR_ENOENT; - } - - return OK; + return owner->TriggerEvent(ident, filter, trigger_data); } int KYTY_SYSV_ABI KernelDeleteEvent(KernelEqueue eq, uintptr_t ident, int16_t filter) { - if (eq == nullptr) { + auto owner = KernelPinEqueue(eq); + if (!owner) { return KERNEL_ERROR_EBADF; } - if (!eq->DeleteEvent(ident, filter)) { - return KERNEL_ERROR_ENOENT; - } - - return OK; + return owner->DeleteEvent(ident, filter); } int KYTY_SYSV_ABI KernelDeleteEqueue(KernelEqueue eq) { PRINT_NAME(); - if (eq == nullptr) { - return KERNEL_ERROR_EBADF; - } - - LOGF("\tEqueue delete: %s\n", eq->GetName().c_str()); + KernelEqueueRef owner; { Common::LockGuard lock(g_equeues_mutex); - g_equeues.remove(eq); + const auto entry = g_equeues.find(eq); + if (entry == g_equeues.end()) { + return KERNEL_ERROR_EBADF; + } + owner = std::move(entry->second); + g_equeues.erase(entry); } - delete eq; + LOGF("\tEqueue delete: %s\n", owner->GetName().c_str()); + owner->Close(); return OK; } @@ -341,7 +387,8 @@ int KYTY_SYSV_ABI KernelWaitEqueue(KernelEqueue eq, KernelEvent* ev, int num, in const KernelUseconds* timo) { PRINT_NAME(); - if (eq == nullptr) { + auto owner = KernelPinEqueue(eq); + if (!owner) { return KERNEL_ERROR_EBADF; } @@ -357,25 +404,28 @@ int KYTY_SYSV_ABI KernelWaitEqueue(KernelEqueue eq, KernelEvent* ev, int num, in LOGF("\tEqueue wait: %s, caller = 0x%016" PRIx64 ", eq = 0x%016" PRIx64 ", ev = 0x%016" PRIx64 ", num = %d, timo = %s, thread_id = %d\n", - eq->GetName().c_str(), reinterpret_cast(__builtin_return_address(0)), - reinterpret_cast(eq), reinterpret_cast(ev), num, + owner->GetName().c_str(), reinterpret_cast(__builtin_return_address(0)), + static_cast(eq), reinterpret_cast(ev), num, (timo == nullptr ? "inf" : fmt::format("{}", *timo).c_str()), Common::Thread::GetThreadIdUnique()); if (timo == nullptr) { - *out = eq->WaitForEvents(ev, num, 0); + *out = owner->WaitForEvents(ev, num, 0); } if (timo != nullptr) { if (*timo == 0) { - *out = eq->GetTriggeredEvents(ev, num); + *out = owner->GetTriggeredEvents(ev, num); } else { - *out = eq->WaitForEvents(ev, num, *timo); + *out = owner->WaitForEvents(ev, num, *timo); } } + if (*out == KERNEL_ERROR_EBADF) { + return KERNEL_ERROR_EBADF; + } if (*out == 0) { - LOGF("\tEqueue wait timedout: %s\n", eq->GetName().c_str()); + LOGF("\tEqueue wait timedout: %s\n", owner->GetName().c_str()); return KERNEL_ERROR_ETIMEDOUT; } @@ -391,11 +441,7 @@ int KYTY_SYSV_ABI KernelWaitEqueue(KernelEqueue eq, KernelEvent* ev, int num, in int KYTY_SYSV_ABI KernelAddUserEvent(KernelEqueue eq, int id) { PRINT_NAME(); - LOGF("\t user event add: eq = 0x%016" PRIx64 ", id = %d\n", reinterpret_cast(eq), id); - - if (eq == nullptr) { - return KERNEL_ERROR_EBADF; - } + LOGF("\t user event add: eq = 0x%016" PRIx64 ", id = %d\n", static_cast(eq), id); KernelEqueueEvent event {}; event.event.ident = static_cast(id); @@ -407,21 +453,15 @@ int KYTY_SYSV_ABI KernelAddUserEvent(KernelEqueue eq, int id) { event.filter.trigger_func = UserEventTriggerFunc; event.filter.reset_func = UserEventResetFunc; - eq->AddEvent(event); - - return OK; + return KernelAddEvent(eq, event); } int KYTY_SYSV_ABI KernelAddUserEventEdge(KernelEqueue eq, int id) { PRINT_NAME(); - LOGF("\t user event edge add: eq = 0x%016" PRIx64 ", id = %d\n", reinterpret_cast(eq), + LOGF("\t user event edge add: eq = 0x%016" PRIx64 ", id = %d\n", static_cast(eq), id); - if (eq == nullptr) { - return KERNEL_ERROR_EBADF; - } - KernelEqueueEvent event {}; event.event.ident = static_cast(id); event.event.filter = KERNEL_EVFILT_USER; @@ -432,28 +472,31 @@ int KYTY_SYSV_ABI KernelAddUserEventEdge(KernelEqueue eq, int id) { event.filter.trigger_func = UserEventTriggerFunc; event.filter.reset_func = UserEventResetFunc; - eq->AddEvent(event); - - return OK; + return KernelAddEvent(eq, event); } int KYTY_SYSV_ABI KernelTriggerUserEvent(KernelEqueue eq, int id, void* udata) { PRINT_NAME(); LOGF("\t user event trigger: eq = 0x%016" PRIx64 ", id = %d, udata = 0x%016" PRIx64 "\n", - reinterpret_cast(eq), id, reinterpret_cast(udata)); + static_cast(eq), id, reinterpret_cast(udata)); return KernelTriggerEvent(eq, static_cast(id), KERNEL_EVFILT_USER, udata); } int KYTY_SYSV_ABI KernelTriggerUserEventForAll(int id, void* udata) { - int triggered = 0; + int triggered = 0; + std::vector queues; - Common::LockGuard lock(g_equeues_mutex); - - for (auto* eq: g_equeues) { - if (eq != nullptr && - eq->TriggerEvent(static_cast(id), KERNEL_EVFILT_USER, udata)) { + { + Common::LockGuard lock(g_equeues_mutex); + queues.reserve(g_equeues.size()); + for (const auto& [handle, queue]: g_equeues) { + queues.push_back(queue); + } + } + for (const auto& eq: queues) { + if (eq->TriggerEvent(static_cast(id), KERNEL_EVFILT_USER, udata) == OK) { triggered++; } } @@ -464,7 +507,7 @@ int KYTY_SYSV_ABI KernelTriggerUserEventForAll(int id, void* udata) { int KYTY_SYSV_ABI KernelDeleteUserEvent(KernelEqueue eq, int id) { PRINT_NAME(); - LOGF("\t user event delete: eq = 0x%016" PRIx64 ", id = %d\n", reinterpret_cast(eq), + LOGF("\t user event delete: eq = 0x%016" PRIx64 ", id = %d\n", static_cast(eq), id); return KernelDeleteEvent(eq, static_cast(id), KERNEL_EVFILT_USER); @@ -472,9 +515,6 @@ int KYTY_SYSV_ABI KernelDeleteUserEvent(KernelEqueue eq, int id) { int KYTY_SYSV_ABI KernelAddHRTimerEvent(KernelEqueue eq, int id, const KernelTimespec* ts, void* udata) { - if (eq == nullptr) { - return KERNEL_ERROR_EBADF; - } if (ts == nullptr) { return KERNEL_ERROR_EFAULT; } @@ -496,9 +536,7 @@ int KYTY_SYSV_ABI KernelAddHRTimerEvent(KernelEqueue eq, int id, const KernelTim event.event.fflags = 0; event.event.data = 0; event.event.udata = udata; - eq->AddEvent(event); - - return OK; + return KernelAddEvent(eq, event); } int KYTY_SYSV_ABI KernelDeleteHRTimerEvent(KernelEqueue eq, int id) { @@ -509,9 +547,9 @@ int KYTY_SYSV_ABI KernelAddAmprEvent(KernelEqueue eq, int id, void* udata) { PRINT_NAME(); LOGF("\t AMPR event add: eq = 0x%016" PRIx64 ", id = %d, udata = 0x%016" PRIx64 "\n", - reinterpret_cast(eq), id, reinterpret_cast(udata)); + static_cast(eq), id, reinterpret_cast(udata)); - if (eq != nullptr) { + if (eq != KERNEL_EQUEUE_INVALID) { KernelEqueueEvent event {}; event.event.ident = static_cast(id); event.event.filter = KERNEL_EVFILT_USER; @@ -521,7 +559,7 @@ int KYTY_SYSV_ABI KernelAddAmprEvent(KernelEqueue eq, int id, void* udata) { event.event.udata = udata; event.filter.trigger_func = AmprEventTriggerFunc; event.filter.reset_func = UserEventResetFunc; - eq->AddEvent(event); + (void)KernelAddEvent(eq, event); } return OK; @@ -531,7 +569,7 @@ int KYTY_SYSV_ABI KernelAddAmprSystemEvent(KernelEqueue eq, int id, void* udata) PRINT_NAME(); LOGF("\t AMPR system event add: eq = 0x%016" PRIx64 ", id = %d, udata = 0x%016" PRIx64 "\n", - reinterpret_cast(eq), id, reinterpret_cast(udata)); + static_cast(eq), id, reinterpret_cast(udata)); return KernelAddAmprEvent(eq, id, udata); } @@ -539,11 +577,11 @@ int KYTY_SYSV_ABI KernelAddAmprSystemEvent(KernelEqueue eq, int id, void* udata) int KYTY_SYSV_ABI KernelDeleteAmprEvent(KernelEqueue eq, int id) { PRINT_NAME(); - LOGF("\t AMPR event delete: eq = 0x%016" PRIx64 ", id = %d\n", reinterpret_cast(eq), + LOGF("\t AMPR event delete: eq = 0x%016" PRIx64 ", id = %d\n", static_cast(eq), id); - if (eq != nullptr) { - eq->DeleteEvent(static_cast(id), KERNEL_EVFILT_USER); + if (eq != KERNEL_EQUEUE_INVALID) { + (void)KernelDeleteEvent(eq, static_cast(id), KERNEL_EVFILT_USER); } return OK; @@ -553,7 +591,7 @@ int KYTY_SYSV_ABI KernelDeleteAmprSystemEvent(KernelEqueue eq, int id) { PRINT_NAME(); LOGF("\t AMPR system event delete: eq = 0x%016" PRIx64 ", id = %d\n", - reinterpret_cast(eq), id); + static_cast(eq), id); return KernelDeleteAmprEvent(eq, id); } diff --git a/src/kernel/eventQueue.h b/src/kernel/eventQueue.h index 564d2276..808ffd4b 100644 --- a/src/kernel/eventQueue.h +++ b/src/kernel/eventQueue.h @@ -6,6 +6,7 @@ #include "kernel/pthread.h" #include +#include namespace Libs::LibKernel::EventQueue { @@ -21,7 +22,10 @@ constexpr int16_t KERNEL_EVFILT_HRTIMER = -15; class KernelEqueuePrivate; struct KernelEqueueEvent; -using KernelEqueue = KernelEqueuePrivate*; +using KernelEqueue = int64_t; +using KernelEqueueRef = std::shared_ptr; + +constexpr KernelEqueue KERNEL_EQUEUE_INVALID = 0; using trigger_func_t = void (*)(KernelEqueueEvent* event, void* trigger_data); using reset_func_t = void (*)(KernelEqueueEvent* event); @@ -37,10 +41,11 @@ struct KernelEvent { }; struct KernelFilter { - void* data = nullptr; - trigger_func_t trigger_func = nullptr; - reset_func_t reset_func = nullptr; - delete_func_t delete_event_func = nullptr; + void* data = nullptr; + std::shared_ptr owner; + trigger_func_t trigger_func = nullptr; + reset_func_t reset_func = nullptr; + delete_func_t delete_event_func = nullptr; }; struct KernelEqueueEvent { @@ -51,6 +56,8 @@ struct KernelEqueueEvent { std::deque pending_events; }; +[[nodiscard]] KernelEqueueRef KernelPinEqueue(KernelEqueue eq); + int KYTY_SYSV_ABI KernelAddEvent(KernelEqueue eq, const KernelEqueueEvent& event); int KYTY_SYSV_ABI KernelTriggerEvent(KernelEqueue eq, uintptr_t ident, int16_t filter, void* trigger_data); diff --git a/src/kernel/memory.cpp b/src/kernel/memory.cpp index 46f9dd36..8fce1f9f 100644 --- a/src/kernel/memory.cpp +++ b/src/kernel/memory.cpp @@ -7,7 +7,7 @@ #include "common/threads.h" #include "common/virtualMemory.h" #include "graphics/guest_gpu/graphicsRun.h" -#include "graphics/host_gpu/renderer/renderContext.h" +#include "graphics/host_gpu/renderer/gpuResourceManager.h" #include "libs/errno.h" #include "libs/libs.h" @@ -66,9 +66,11 @@ constexpr int PAGE_TABLE_POOL_ENTRIES = constexpr uint64_t DEFAULT_FLEXIBLE_MEMORY_SIZE = 4ull * 1024ull * 1024ull * 1024ull; static uint64_t g_flexible_memory_size = DEFAULT_FLEXIBLE_MEMORY_SIZE; +static Graphics::GpuResourceManager* g_gpu_resources = nullptr; static Graphics::GpuResourceManager& GetGpuResources() { - return Graphics::GetRenderContext().GetGpuResources(); + EXIT_IF(g_gpu_resources == nullptr); + return *g_gpu_resources; } static bool IsGpuAddressRange(uint64_t vaddr, uint64_t size) { @@ -97,12 +99,10 @@ static void UnmapGpuRange(uint64_t vaddr, uint64_t size, GpuAccessMode mode) { if (!IsGpuAddressRange(vaddr, size)) { EXIT("invalid GPU unmap range: addr=0x%016" PRIx64 " size=0x%016" PRIx64 "\n", vaddr, size); } - auto& resources = GetGpuResources(); - Graphics::GraphicsRunSubmissionLock submission_lock; const auto access = mode == GpuAccessMode::Read ? Graphics::GpuAccess::Read : mode == GpuAccessMode::Write ? Graphics::GpuAccess::Write : Graphics::GpuAccess::ReadWrite; - resources.UnmapMemory(vaddr, size, access); + GetGpuResources().UnmapMemory(vaddr, size, access); } static bool DecodeMemoryProtection(int prot, VirtualMemory::Mode* mode, GpuAccessMode* gpu_mode) { @@ -867,11 +867,13 @@ static void MemoryPoolSubtractCommitted(uint64_t len); static std::recursive_mutex g_memory_operation_mutex; bool TryWriteBacking(uint64_t vaddr, const void* data, uint64_t size) { - return g_direct_memory_backing->TryWriteBacking(vaddr, data, size); + return g_direct_memory_backing != nullptr && + g_direct_memory_backing->TryWriteBacking(vaddr, data, size); } bool TryReadBacking(uint64_t vaddr, void* data, uint64_t size) { - return g_direct_memory_backing->TryReadBacking(vaddr, data, size); + return g_direct_memory_backing != nullptr && + g_direct_memory_backing->TryReadBacking(vaddr, data, size); } void WriteBacking(uint64_t vaddr, const void* data, uint64_t size) noexcept { @@ -886,7 +888,16 @@ void PrepareHostWrite(uint64_t vaddr, uint64_t size) { if (size == 0) { return; } - Graphics::GetRenderContext().GetGpuResources().PrepareHostWrite(vaddr, size); + GetGpuResources().PrepareHostWrite(vaddr, size); +} + +void InstallGpuResources(Graphics::GpuResourceManager* resources) noexcept { + EXIT_IF(resources != nullptr && g_gpu_resources != nullptr); + g_gpu_resources = resources; +} + +bool HandleGpuFault(Graphics::PageFaultAccess access, uint64_t fault_vaddr) noexcept { + return g_gpu_resources != nullptr && g_gpu_resources->HandleFault(access, fault_vaddr); } struct PrtAperture { diff --git a/src/kernel/memory.h b/src/kernel/memory.h index a297e44f..8b31a7e3 100644 --- a/src/kernel/memory.h +++ b/src/kernel/memory.h @@ -6,6 +6,11 @@ #include "common/subsystems.h" #include "common/virtualMemory.h" +namespace Libs::Graphics { +class GpuResourceManager; +enum class PageFaultAccess; +} // namespace Libs::Graphics + namespace Libs::LibKernel::Memory { KYTY_SUBSYSTEM_DEFINE(Memory); @@ -100,6 +105,8 @@ bool TryWriteBacking(uint64_t vaddr, const void* data, uint64_t size); bool TryReadBacking(uint64_t vaddr, void* data, uint64_t size); void WriteBacking(uint64_t vaddr, const void* data, uint64_t size) noexcept; void PrepareHostWrite(uint64_t vaddr, uint64_t size); +void InstallGpuResources(Graphics::GpuResourceManager* resources) noexcept; +[[nodiscard]] bool HandleGpuFault(Graphics::PageFaultAccess access, uint64_t fault_vaddr) noexcept; int KYTY_SYSV_ABI KernelMapNamedFlexibleMemory(void** addr_in_out, size_t len, int prot, int flags, const char* name); diff --git a/src/libs/agc.cpp b/src/libs/agc.cpp index 5e4ec94c..2e3c0476 100644 --- a/src/libs/agc.cpp +++ b/src/libs/agc.cpp @@ -11,8 +11,10 @@ #include "graphics/guest_gpu/hardwareContext.h" #include "graphics/guest_gpu/pm4.h" #include "graphics/guest_gpu/tile.h" -#include "graphics/host_gpu/objects/label.h" +#include "graphics/host_gpu/renderer/render.h" +#include "graphics/host_gpu/renderer/renderContext.h" #include "graphics/host_gpu/renderer/sync.h" +#include "graphics/presentation/presenter.h" #include "graphics/presentation/renderDoc.h" #include "graphics/presentation/videoOut.h" #include "graphics/presentation/window.h" @@ -37,6 +39,8 @@ namespace Libs::Graphics { +static RenderContext* g_renderer = nullptr; + KYTY_SUBSYSTEM_INIT(Graphics) { // Some games lock up if this is not called first if (Config::RenderDocEnabled()) { @@ -46,16 +50,22 @@ KYTY_SUBSYSTEM_INIT(Graphics) { auto width = Config::GetScreenWidth(); auto height = Config::GetScreenHeight(); - WindowInit(width, height); - VideoOut::VideoOutInit(width, height); - GraphicsRunInit(); - LabelInit(); + auto& presenter = WindowInit(width, height); + auto& video_out = VideoOut::VideoOutInit(width, height, presenter); + g_renderer = &presenter.Renderer(); + g_renderer->InitializeGpu(&video_out); ShaderInit(); } KYTY_SUBSYSTEM_UNEXPECTED_SHUTDOWN(Graphics) {} -KYTY_SUBSYSTEM_DESTROY(Graphics) {} +KYTY_SUBSYSTEM_DESTROY(Graphics) { + EXIT_IF(g_renderer == nullptr); + g_renderer->ShutdownGpu(); + VideoOut::VideoOutShutdown(); + WindowShutdown(); + g_renderer = nullptr; +} void GraphicsDbgDumpDcb(const char* type, uint32_t num_dw, uint32_t* cmd_buffer) { EXIT_IF(type == nullptr); @@ -64,9 +74,9 @@ void GraphicsDbgDumpDcb(const char* type, uint32_t num_dw, uint32_t* cmd_buffer) if (Config::CommandBufferDumpEnabled() && num_dw > 0 && cmd_buffer != nullptr) { Common::File f; - auto file_name = - Config::GetCommandBufferDumpFolder() / - fmt::format("{:04d}_{:04d}_buffer_{}.log", GraphicsRunGetFrameNum(), id++, type); + auto file_name = Config::GetCommandBufferDumpFolder() / + fmt::format("{:04d}_{:04d}_buffer_{}.log", + g_renderer->GetGpu().GetFrameNum(), id++, type); Common::File::CreateDirectories(file_name.parent_path()); f.Create(file_name); if (f.IsInvalid()) { @@ -1382,7 +1392,8 @@ int KYTY_SYSV_ABI GraphicsJumpPatchSetTarget(uint32_t* cmd, const volatile uint3 int KYTY_SYSV_ABI GraphicsSuspendPoint() { PRINT_NAME(); - GraphicsRunDone(); + EXIT_IF(g_renderer == nullptr); + g_renderer->GetGpu().Done(); return OK; } @@ -3690,8 +3701,9 @@ static bool dcb_has_queued_interrupt(const uint32_t* dcb, uint32_t size_in_dword static void submit_dcb(uint32_t* dcb, uint32_t size_in_dwords) { GraphicsDbgDumpDcb("d", size_in_dwords, dcb); - GraphicsRunSubmit(dcb, size_in_dwords, nullptr, 0, - !dcb_has_queued_interrupt(dcb, size_in_dwords)); + EXIT_IF(g_renderer == nullptr); + g_renderer->GetGpu().Submit(dcb, size_in_dwords, nullptr, 0, + !dcb_has_queued_interrupt(dcb, size_in_dwords)); Gen5::track_pending_graphics_segment_after_submit(dcb, size_in_dwords); } @@ -3956,7 +3968,8 @@ static void submit_acb(uint32_t queue, uint32_t* acb, uint32_t size_in_dwords) { GraphicsDbgDumpDcb("a", size_in_dwords, acb); const bool trigger_interrupt_on_done = !dcb_has_queued_interrupt(acb, size_in_dwords); - GraphicsRunSubmitCompute(queue, acb, size_in_dwords, trigger_interrupt_on_done); + EXIT_IF(g_renderer == nullptr); + g_renderer->GetGpu().SubmitCompute(queue, acb, size_in_dwords, trigger_interrupt_on_done); } int KYTY_SYSV_ABI GraphicsDriverSubmitAcb(uint32_t queue, const Packet* packet) { @@ -4005,17 +4018,18 @@ int KYTY_SYSV_ABI GraphicsDriverAddEqEvent(LibKernel::EventQueue::KernelEqueue e void* udata) { PRINT_NAME(); - if (eq == nullptr) { + if (eq == LibKernel::EventQueue::KERNEL_EQUEUE_INVALID) { return LibKernel::KERNEL_ERROR_EBADF; } - return Sync::AddEqEvent(eq, id, udata); + EXIT_IF(g_renderer == nullptr); + return Sync::AddEqEvent(*g_renderer, eq, id, udata); } int KYTY_SYSV_ABI GraphicsDriverDeleteEqEvent(LibKernel::EventQueue::KernelEqueue eq, int id) { PRINT_NAME(); - if (eq == nullptr) { + if (eq == LibKernel::EventQueue::KERNEL_EQUEUE_INVALID) { return LibKernel::KERNEL_ERROR_EBADF; } diff --git a/src/libs/libAmpr.cpp b/src/libs/libAmpr.cpp index 74696de0..0d87bed0 100644 --- a/src/libs/libAmpr.cpp +++ b/src/libs/libAmpr.cpp @@ -1511,8 +1511,8 @@ static int ExecuteAprCommandBuffer(uint64_t command_buffer, int32_t* execution_r } break; case CommandKind::KernelEvent: { const auto& command = state.kernel_event_commands[entry.index]; - auto* eq = reinterpret_cast(command.eq); - auto result = LibKernel::EventQueue::KernelTriggerUserEvent( + const auto eq = static_cast(command.eq); + auto result = LibKernel::EventQueue::KernelTriggerUserEvent( eq, command.id, reinterpret_cast(command.data)); if (result != OK) { LOGF("\tAPR submit event failed: eq=0x%016" PRIx64 ", id=%" PRId32 diff --git a/src/loader/runtimeLinker.cpp b/src/loader/runtimeLinker.cpp index b54ffafa..98195546 100644 --- a/src/loader/runtimeLinker.cpp +++ b/src/loader/runtimeLinker.cpp @@ -13,7 +13,7 @@ #include "common/stringUtils.h" #include "common/threads.h" #include "common/virtualMemory.h" -#include "graphics/host_gpu/renderer/renderContext.h" +#include "graphics/host_gpu/pageManager.h" #include "kernel/memory.h" #include "kernel/pthread.h" #include "loader/elf.h" @@ -515,8 +515,7 @@ static bool KytyExceptionHandler(const Common::HostException::ExceptionInfo& exc EXIT("invalid access type for page fault at 0x%016" PRIx64 "\n", info->access_violation_vaddr); }(); - if (Libs::Graphics::GetRenderContext().GetGpuResources().HandleFault( - access, info->access_violation_vaddr)) { + if (Libs::LibKernel::Memory::HandleGpuFault(access, info->access_violation_vaddr)) { return true; } diff --git a/src/main.cpp b/src/main.cpp index 88992fa1..6d0aa713 100644 --- a/src/main.cpp +++ b/src/main.cpp @@ -59,6 +59,7 @@ static void PrintUsage() { ::printf(" --spirv-debug-printf Enable SPIR-V debug printf.\n"); ::printf(" --ngg-rectlist-draw Draw rect-list auto draws using the NGG " "4-vertex path.\n"); + ::printf(" --readback-linear-images Read back writable linear images on submit.\n"); ::printf(" --rd Enable RenderDoc capture.\n"); } @@ -210,6 +211,11 @@ static bool ParseArgs(int argc, char* argv[], RunOptions& options, bool& show_he ::printf("invalid boolean for %s: %s\n", arg.c_str(), value.c_str()); return false; } + } else if (arg == "--readback-linear-images") { + if (!ParseBool(value, options.config.readback_linear_images)) { + ::printf("invalid boolean for %s: %s\n", arg.c_str(), value.c_str()); + return false; + } } else { ::printf("unknown option: %s\n", arg.c_str()); return false; diff --git a/tests/EventQueueLifetimeTests.cpp b/tests/EventQueueLifetimeTests.cpp new file mode 100644 index 00000000..17a70ffd --- /dev/null +++ b/tests/EventQueueLifetimeTests.cpp @@ -0,0 +1,500 @@ +#include "kernel/eventQueue.h" + +#include "libs/errno.h" + +#include +#include +#include +#include +#include +#include +#include +#include + +namespace { + +namespace EventQueue = Libs::LibKernel::EventQueue; +using Libs::LibKernel::KERNEL_ERROR_EBADF; +using Libs::LibKernel::KERNEL_ERROR_ENOENT; + +void Check(bool value, const char *text) { + if (!value) { + std::fprintf(stderr, "EventQueueLifetimeTests: failed: %s\n", text); + std::abort(); + } +} + +void CheckConcurrentResult(int result, const char *text) { + Check(result == OK || result == KERNEL_ERROR_EBADF || + result == KERNEL_ERROR_ENOENT, + text); +} + +void CountDeletedEvent(EventQueue::KernelEqueue, + EventQueue::KernelEqueueEvent *event) { + auto *count = static_cast(event->filter.data); + count->fetch_add(1, std::memory_order_relaxed); +} + +struct DuplicateEventOwner { + std::atomic_uint32_t delete_count{0}; +}; + +void QueueDuplicateEvent(EventQueue::KernelEqueueEvent *event, + void *trigger_data) { + auto next = event->event; + next.data = reinterpret_cast(trigger_data); + if (event->triggered) { + event->pending_events.push_back(next); + } else { + event->event = next; + event->triggered = true; + } +} + +void ResetDuplicateEvent(EventQueue::KernelEqueueEvent *event) { + event->triggered = false; + event->event.data = 0; +} + +void DeleteDuplicateEvent(EventQueue::KernelEqueue, + EventQueue::KernelEqueueEvent *event) { + auto *owner = static_cast(event->filter.data); + owner->delete_count.fetch_add(1, std::memory_order_relaxed); +} + +void PoisonDuplicateEvent(EventQueue::KernelEqueueEvent *, void *) { + Check(false, "duplicate add replaced trigger callback"); +} + +void TestDuplicateAddPreservesEventState() { + EventQueue::KernelEqueue queue = EventQueue::KERNEL_EQUEUE_INVALID; + Check(EventQueue::KernelCreateEqueue(&queue, "duplicate-add") == OK, + "create duplicate add queue"); + + auto original_owner = std::make_shared(); + std::weak_ptr weak_original = original_owner; + EventQueue::KernelEqueueEvent original{}; + original.event.ident = 17; + original.event.filter = EventQueue::KERNEL_EVFILT_VIDEO_OUT; + original.event.udata = reinterpret_cast(0x1111); + original.filter.data = original_owner.get(); + original.filter.owner = original_owner; + original.filter.trigger_func = QueueDuplicateEvent; + original.filter.reset_func = ResetDuplicateEvent; + original.filter.delete_event_func = DeleteDuplicateEvent; + Check(EventQueue::KernelAddEvent(queue, original) == OK, + "add original duplicate event"); + Check(EventQueue::KernelTriggerEvent(queue, 17, + EventQueue::KERNEL_EVFILT_VIDEO_OUT, + reinterpret_cast(0x1234)) == OK, + "queue first trigger"); + Check(EventQueue::KernelTriggerEvent(queue, 17, + EventQueue::KERNEL_EVFILT_VIDEO_OUT, + reinterpret_cast(0x5678)) == OK, + "queue pending trigger"); + + auto replacement_owner = std::make_shared(); + std::weak_ptr weak_replacement = replacement_owner; + EventQueue::KernelEqueueEvent duplicate{}; + duplicate.triggered = false; + duplicate.deadline_ns = 1; + duplicate.event.ident = 17; + duplicate.event.filter = EventQueue::KERNEL_EVFILT_VIDEO_OUT; + duplicate.event.data = 0x7fffffff; + duplicate.event.udata = reinterpret_cast(0x2222); + duplicate.filter.data = replacement_owner.get(); + duplicate.filter.owner = replacement_owner; + duplicate.filter.trigger_func = PoisonDuplicateEvent; + Check(EventQueue::KernelAddEvent(queue, duplicate) == OK, + "update duplicate event"); + + duplicate.filter.owner.reset(); + replacement_owner.reset(); + Check(weak_replacement.expired(), "duplicate owner is not retained"); + original.filter.owner.reset(); + original_owner.reset(); + Check(!weak_original.expired(), "original event owner remains retained"); + + EventQueue::KernelEvent events[2]{}; + int out = 0; + Libs::LibKernel::KernelUseconds timeout = 0; + Check(EventQueue::KernelWaitEqueue(queue, events, 2, &out, &timeout) == OK, + "read queued duplicate triggers"); + Check(out == 2, "duplicate add preserves pending event count"); + Check(events[0].data == 0x1234 && events[1].data == 0x5678, + "duplicate add preserves current and pending event data"); + Check(events[0].udata == reinterpret_cast(0x2222), + "duplicate add updates current user data"); + Check(events[1].udata == reinterpret_cast(0x2222), + "duplicate add updates pending user data"); + + EventQueue::KernelEvent timer_event{}; + Check(EventQueue::KernelWaitEqueue(queue, &timer_event, 1, &out, &timeout) == + OK && + out == 1, + "duplicate add updates deadline metadata"); + Check(timer_event.data == 0 && + timer_event.udata == reinterpret_cast(0x2222), + "deadline trigger retains updated duplicate metadata"); + + auto retained_owner = weak_original.lock(); + Check(retained_owner != nullptr, "original owner alive before delete"); + Check(EventQueue::KernelDeleteEvent(queue, 17, + EventQueue::KERNEL_EVFILT_VIDEO_OUT) == + OK, + "delete duplicate event"); + Check(retained_owner->delete_count.load(std::memory_order_relaxed) == 1, + "duplicate add preserves delete callback"); + retained_owner.reset(); + Check(weak_original.expired(), "original owner released on delete"); + Check(EventQueue::KernelDeleteEqueue(queue) == OK, + "delete duplicate add queue"); +} + +struct SimulatedVideoOutEventState; + +struct SimulatedVideoOutRegistration { + EventQueue::KernelEqueue handle = EventQueue::KERNEL_EQUEUE_INVALID; + std::shared_ptr state; + uint64_t marker = 0x123456789abcdef0ull; +}; + +struct SimulatedVideoOutEventState { + SimulatedVideoOutEventState(std::atomic_uint32_t &stage, + std::atomic_uint32_t &destroy_count) + : stage(stage), destroy_count(destroy_count) {} + + ~SimulatedVideoOutEventState() { + destroy_count.fetch_add(1, std::memory_order_relaxed); + } + + std::mutex mutex; + std::vector> queues; + std::atomic_uint32_t &stage; + std::atomic_uint32_t &destroy_count; + uint64_t marker = 0xfedcba9876543210ull; +}; + +void DetachSimulatedVideoOutEvent(EventQueue::KernelEqueue queue, + EventQueue::KernelEqueueEvent *event) { + auto *registration = + static_cast(event->filter.data); + Check(registration != nullptr && registration->handle == queue, + "simulated registration identity"); + auto state = registration->state; + Check(state != nullptr, "simulated event owns shared state"); + state->stage.store(1, std::memory_order_release); + while (state->stage.load(std::memory_order_acquire) != 2) { + std::this_thread::yield(); + } + + { + std::lock_guard lock(state->mutex); + const auto entry = + std::find_if(state->queues.begin(), state->queues.end(), + [registration](const auto &candidate) { + return candidate.get() == registration; + }); + if (entry != state->queues.end()) { + state->queues.erase(entry); + } + } + event->filter.owner.reset(); + Check(state->marker == 0xfedcba9876543210ull && + registration->marker == 0x123456789abcdef0ull, + "callback state survives simulated port destruction"); +} + +void TestCallbackStateOutlivesPort() { + EventQueue::KernelEqueue queue = EventQueue::KERNEL_EQUEUE_INVALID; + Check(EventQueue::KernelCreateEqueue(&queue, "shared-port-state") == OK, + "create shared port state queue"); + + std::atomic_uint32_t stage{0}; + std::atomic_uint32_t destroy_count{0}; + auto port_state = + std::make_shared(stage, destroy_count); + std::weak_ptr weak_state = port_state; + auto registration = std::make_shared(); + std::weak_ptr weak_registration = + registration; + registration->handle = queue; + registration->state = port_state; + port_state->queues.push_back(registration); + + EventQueue::KernelEqueueEvent event{}; + event.event.ident = 8; + event.event.filter = EventQueue::KERNEL_EVFILT_VIDEO_OUT; + event.filter.data = registration.get(); + event.filter.owner = registration; + event.filter.delete_event_func = DetachSimulatedVideoOutEvent; + Check(EventQueue::KernelAddEvent(queue, event) == OK, + "add shared port state event"); + event.filter.owner.reset(); + + std::jthread close([&] { + Check(EventQueue::KernelDeleteEqueue(queue) == OK, + "delete shared port state queue"); + }); + while (stage.load(std::memory_order_acquire) != 1) { + std::this_thread::yield(); + } + + std::vector> detached; + { + std::lock_guard lock(port_state->mutex); + detached = std::move(port_state->queues); + } + registration.reset(); + detached.clear(); + port_state.reset(); + Check(!weak_state.expired(), + "callback state outlives simulated port object"); + Check(!weak_registration.expired(), + "registration outlives simulated port object"); + + stage.store(2, std::memory_order_release); + close.join(); + Check(weak_registration.expired(), "detached registration is released"); + Check(weak_state.expired(), "detached event state is released"); + Check(destroy_count.load(std::memory_order_relaxed) == 1, + "shared event state is destroyed exactly once"); +} + +struct OwnedCallbackPayload { + OwnedCallbackPayload(std::atomic_uint32_t &stage, + std::atomic_uint32_t &delete_count, + std::atomic_uint32_t &destroy_count) + : stage(stage), delete_count(delete_count), destroy_count(destroy_count) {} + + std::atomic_uint32_t &stage; + std::atomic_uint32_t &delete_count; + std::atomic_uint32_t &destroy_count; + uint64_t marker = 0xc0dec0dec0dec0deull; + + ~OwnedCallbackPayload() { + destroy_count.fetch_add(1, std::memory_order_relaxed); + } +}; + +void DeleteOwnedEvent(EventQueue::KernelEqueue queue, + EventQueue::KernelEqueueEvent *event) { + auto *payload = static_cast(event->filter.data); + Check(payload != nullptr, "owned callback payload"); + Check(!EventQueue::KernelPinEqueue(queue), + "owned callback runs after registry removal"); + payload->delete_count.fetch_add(1, std::memory_order_relaxed); + event->filter.owner.reset(); + payload->stage.store(1, std::memory_order_release); + while (payload->stage.load(std::memory_order_acquire) != 2) { + std::this_thread::yield(); + } + Check(payload->marker == 0xc0dec0dec0dec0deull, + "owned callback payload remains valid"); +} + +void TestCallbackOwnsPayload() { + EventQueue::KernelEqueue queue = EventQueue::KERNEL_EQUEUE_INVALID; + Check(EventQueue::KernelCreateEqueue(&queue, "owned-callback") == OK, + "create owned callback queue"); + + std::atomic_uint32_t stage{0}; + std::atomic_uint32_t delete_count{0}; + std::atomic_uint32_t destroy_count{0}; + auto registration = + std::make_shared(stage, delete_count, destroy_count); + std::weak_ptr weak_registration = registration; + std::vector> port_registrations{ + registration}; + { + EventQueue::KernelEqueueEvent event{}; + event.event.ident = 2; + event.event.filter = EventQueue::KERNEL_EVFILT_VIDEO_OUT; + event.filter.data = registration.get(); + event.filter.owner = registration; + event.filter.delete_event_func = DeleteOwnedEvent; + Check(EventQueue::KernelAddEvent(queue, event) == OK, + "add owned callback event"); + } + + std::jthread close([&] { + Check(EventQueue::KernelDeleteEqueue(queue) == OK, + "delete owned callback queue"); + }); + while (stage.load(std::memory_order_acquire) != 1) { + std::this_thread::yield(); + } + + Check(!EventQueue::KernelPinEqueue(queue), + "owned callback queue removed while callback blocked"); + port_registrations.clear(); + registration.reset(); + Check(!weak_registration.expired(), + "delete callback retains detached payload"); + + stage.store(2, std::memory_order_release); + close.join(); + Check(delete_count.load(std::memory_order_relaxed) == 1, + "owned callback runs exactly once"); + Check(weak_registration.expired(), + "owned callback payload released with event"); + Check(destroy_count.load(std::memory_order_relaxed) == 1, + "owned callback payload destroyed exactly once"); +} + +void TestPinnedClose() { + EventQueue::KernelEqueue queue = EventQueue::KERNEL_EQUEUE_INVALID; + Check(EventQueue::KernelCreateEqueue(&queue, "pinned-close") == OK, + "create pinned queue"); + + std::atomic_uint32_t delete_count{0}; + EventQueue::KernelEqueueEvent event{}; + event.event.ident = 1; + event.event.filter = EventQueue::KERNEL_EVFILT_VIDEO_OUT; + event.filter.data = &delete_count; + event.filter.delete_event_func = CountDeletedEvent; + Check(EventQueue::KernelAddEvent(queue, event) == OK, "add callback event"); + + auto owner = EventQueue::KernelPinEqueue(queue); + Check(owner != nullptr, "pin live queue"); + Check(EventQueue::KernelDeleteEqueue(queue) == OK, "delete pinned queue"); + Check(delete_count.load(std::memory_order_relaxed) == 1, + "close invokes callback once"); + Check(!EventQueue::KernelPinEqueue(queue), "deleted queue leaves registry"); + Check(EventQueue::KernelTriggerEvent(queue, 1, + EventQueue::KERNEL_EVFILT_VIDEO_OUT, + nullptr) == KERNEL_ERROR_EBADF, + "stale trigger rejected"); + Check(EventQueue::KernelDeleteEqueue(queue) == KERNEL_ERROR_EBADF, + "second queue delete rejected"); + + owner.reset(); + Check(delete_count.load(std::memory_order_relaxed) == 1, + "deferred destruction does not repeat callback"); +} + +void TestStaleHandleNeverAliasesNewQueue() { + EventQueue::KernelEqueue stale = EventQueue::KERNEL_EQUEUE_INVALID; + Check(EventQueue::KernelCreateEqueue(&stale, "stale-handle") == OK, + "create stale queue"); + Check(EventQueue::KernelDeleteEqueue(stale) == OK, "delete stale queue"); + + EventQueue::KernelEqueue replacement = EventQueue::KERNEL_EQUEUE_INVALID; + Check(EventQueue::KernelCreateEqueue(&replacement, "replacement") == OK, + "create replacement queue"); + Check(stale != replacement, "queue handles are never recycled"); + Check(!EventQueue::KernelPinEqueue(stale), "stale handle does not pin"); + Check(EventQueue::KernelAddUserEvent(stale, 11) == KERNEL_ERROR_EBADF, + "stale handle cannot mutate replacement"); + Check(EventQueue::KernelAddUserEvent(replacement, 11) == OK, + "replacement handle remains valid"); + Check(EventQueue::KernelTriggerUserEvent(stale, 11, nullptr) == + KERNEL_ERROR_EBADF, + "stale handle cannot trigger replacement"); + Check(EventQueue::KernelTriggerUserEvent(replacement, 11, nullptr) == OK, + "replacement event triggers"); + Check(EventQueue::KernelDeleteEqueue(replacement) == OK, + "delete replacement queue"); +} + +void TestConcurrentCloseCallback() { + for (uint32_t iteration = 0; iteration < 64; iteration++) { + EventQueue::KernelEqueue queue = EventQueue::KERNEL_EQUEUE_INVALID; + Check(EventQueue::KernelCreateEqueue(&queue, "callback-race") == OK, + "create callback race queue"); + + std::atomic_uint32_t delete_count{0}; + EventQueue::KernelEqueueEvent callback_event{}; + callback_event.event.ident = 9; + callback_event.event.filter = EventQueue::KERNEL_EVFILT_GRAPHICS; + callback_event.filter.data = &delete_count; + callback_event.filter.delete_event_func = CountDeletedEvent; + Check(EventQueue::KernelAddEvent(queue, callback_event) == OK, + "add callback race event"); + + std::atomic_bool start{false}; + std::jthread trigger([&] { + while (!start.load(std::memory_order_acquire)) { + std::this_thread::yield(); + } + for (uint32_t i = 0; i < 256; i++) { + CheckConcurrentResult( + EventQueue::KernelTriggerEvent( + queue, 9, EventQueue::KERNEL_EVFILT_GRAPHICS, nullptr), + "callback race trigger result"); + } + }); + + start.store(true, std::memory_order_release); + Check(EventQueue::KernelDeleteEqueue(queue) == OK, + "callback race queue delete"); + trigger.join(); + Check(delete_count.load(std::memory_order_relaxed) == 1, + "concurrent close invokes callback exactly once"); + } +} + +void TestConcurrentDelete() { + for (uint32_t iteration = 0; iteration < 64; iteration++) { + EventQueue::KernelEqueue queue = EventQueue::KERNEL_EQUEUE_INVALID; + Check(EventQueue::KernelCreateEqueue(&queue, "concurrent-delete") == OK, + "create concurrent queue"); + EventQueue::KernelEqueueEvent event{}; + event.event.ident = 7; + event.event.filter = EventQueue::KERNEL_EVFILT_USER; + Check(EventQueue::KernelAddEvent(queue, event) == OK, + "add concurrent event"); + + std::atomic_bool start{false}; + std::jthread mutate([&] { + while (!start.load(std::memory_order_acquire)) { + std::this_thread::yield(); + } + for (uint32_t i = 0; i < 64; i++) { + CheckConcurrentResult(EventQueue::KernelAddEvent(queue, event), + "concurrent add result"); + CheckConcurrentResult( + EventQueue::KernelTriggerEvent( + queue, 7, EventQueue::KERNEL_EVFILT_USER, nullptr), + "concurrent trigger result"); + CheckConcurrentResult(EventQueue::KernelDeleteEvent( + queue, 7, EventQueue::KERNEL_EVFILT_USER), + "concurrent event delete result"); + } + }); + std::jthread trigger([&] { + while (!start.load(std::memory_order_acquire)) { + std::this_thread::yield(); + } + for (uint32_t i = 0; i < 128; i++) { + CheckConcurrentResult( + EventQueue::KernelTriggerEvent( + queue, 7, EventQueue::KERNEL_EVFILT_USER, nullptr), + "parallel trigger result"); + } + }); + + start.store(true, std::memory_order_release); + Check(EventQueue::KernelDeleteEqueue(queue) == OK, + "concurrent queue delete"); + mutate.join(); + trigger.join(); + Check(!EventQueue::KernelPinEqueue(queue), + "concurrent queue removed from registry"); + } +} + +} // namespace + +int main() { + TestDuplicateAddPreservesEventState(); + TestCallbackStateOutlivesPort(); + TestCallbackOwnsPayload(); + TestPinnedClose(); + TestStaleHandleNeverAliasesNewQueue(); + TestConcurrentCloseCallback(); + TestConcurrentDelete(); + std::printf("EventQueueLifetimeTests: all cases passed\n"); + return 0; +} diff --git a/tests/ResourceTrackingTests.cpp b/tests/ResourceTrackingTests.cpp index de6b139a..8b3e0679 100644 --- a/tests/ResourceTrackingTests.cpp +++ b/tests/ResourceTrackingTests.cpp @@ -1,3 +1,4 @@ +#include "graphics/guest_gpu/gpu_defs.h" #include "graphics/shader/recompiler/BindingLayout.h" #include "graphics/shader/recompiler/ResourceMaterialization.h" #include "graphics/shader/recompiler/ResourceTracking.h" @@ -5,7 +6,6 @@ #include "graphics/shader/recompiler/ShaderInfoCollection.h" #include "graphics/shader/recompiler/SrtPatcher.h" #include "graphics/shader/recompiler/SrtWalker.h" -#include "graphics/guest_gpu/gpu_defs.h" #include "graphics/shader/shaderBindings.h" #include @@ -164,7 +164,7 @@ const StageOutput *FindOutput(const ShaderInfo &info, StageOutputKind kind, return nullptr; } -void Prepare(Program& program) { +void Prepare(Program &program) { std::string error; if (!BuildScalarProvenance(program, &error) || !BuildSrtPlan(program, &error) || !PatchSrtReads(program, &error) || @@ -230,16 +230,17 @@ void TestScalarAndVectorBufferAlias() { void TestBufferImageAliasIsLinkedDuringTracking() { Program program; program.blocks.resize(1); - program.blocks[0].instructions = { - BufferUse(4, 0), BufferUse(8, 8), - ImageUse(12, Opcode::ImageLoad, ResourceKind::Image, - Decoder::ImageDimension::Dim2D)}; + program.blocks[0].instructions = {BufferUse(4, 0), BufferUse(8, 8), + ImageUse(12, Opcode::ImageLoad, + ResourceKind::Image, + Decoder::ImageDimension::Dim2D)}; Prepare(program); Check(program.info.buffers.size() == 2 && program.info.images.size() == 1 && program.info.buffers[0].image_alias == 0 && program.info.buffers[1].image_alias == BufferResource::NoImageAlias, - "buffer/image descriptor provenance aliases were not linked during tracking"); + "buffer/image descriptor provenance aliases were not linked during " + "tracking"); } void TestImagesAndSamplers() { @@ -462,9 +463,12 @@ void TestComputeShaderInfoCollection() { error.c_str()); Check(program.shader_info_complete && program.info.inputs.size() == 3 && FindInput(program.info, StageInputKind::WorkgroupId) != nullptr && - FindInput(program.info, StageInputKind::LocalInvocationId) == nullptr && - FindInput(program.info, StageInputKind::LocalInvocationIndex) != nullptr && - FindInput(program.info, StageInputKind::GlobalInvocationId) != nullptr && + FindInput(program.info, StageInputKind::LocalInvocationId) == + nullptr && + FindInput(program.info, StageInputKind::LocalInvocationIndex) != + nullptr && + FindInput(program.info, StageInputKind::GlobalInvocationId) != + nullptr && program.info.buffers == buffers && program.info.images == images && program.info.samplers == samplers && program.info.sampled_pairs == pairs, @@ -473,16 +477,20 @@ void TestComputeShaderInfoCollection() { Program metadata_program; metadata_program.stage = ShaderType::Compute; metadata_program.blocks.resize(1); + Instruction xor_address; + xor_address.op = Opcode::BitwiseXor3U32; + metadata_program.blocks[0].instructions = {xor_address}; Prepare(metadata_program); compute = {}; compute.thread_ids_num = 2; Check(CollectShaderInfo(metadata_program, {.compute = &compute}, &error), error.c_str()); Check(FindInput(metadata_program.info, StageInputKind::LocalInvocationId) != - nullptr && + nullptr && FindInput(metadata_program.info, - StageInputKind::LocalInvocationIndex) != nullptr, - "compute thread metadata did not request local invocation inputs"); + StageInputKind::LocalInvocationIndex) != nullptr && + metadata_program.info.has_bitwise_xor, + "compute thread metadata or XOR-address heuristic was not collected"); } void TestVertexShaderInfoCollection() { @@ -496,16 +504,15 @@ void TestVertexShaderInfoCollection() { Instruction attr2 = attr0; attr2.input_info.attr = 2; attr2.input_info.chan = 0; - program.blocks[0].instructions = { - attr0, attr2, Export(8, ExportTargetKind::Position), - Export(12, ExportTargetKind::Parameter, 2)}; + program.blocks[0].instructions = {attr0, attr2, + Export(8, ExportTargetKind::Position), + Export(12, ExportTargetKind::Parameter, 2)}; Prepare(program); ShaderVertexInputInfo vertex; vertex.resources_num = 3; std::string error; - Check(CollectShaderInfo(program, {.vertex = &vertex}, &error), - error.c_str()); + Check(CollectShaderInfo(program, {.vertex = &vertex}, &error), error.c_str()); const auto *input0 = FindInput(program.info, StageInputKind::Parameter, 0); const auto *input2 = FindInput(program.info, StageInputKind::Parameter, 2); Check(program.info.inputs.size() == 4 && input0 != nullptr && @@ -534,11 +541,10 @@ void TestPixelShaderInfoCollection() { Program program; program.stage = ShaderType::Pixel; program.blocks.resize(1); - program.blocks[0].instructions = { - Export(4, ExportTargetKind::Mrt, 3), - Export(8, ExportTargetKind::MrtZ, 0, 0x4), - Export(12, ExportTargetKind::Mrt, 3), - Export(16, ExportTargetKind::Mrt, 7, 0)}; + program.blocks[0].instructions = {Export(4, ExportTargetKind::Mrt, 3), + Export(8, ExportTargetKind::MrtZ, 0, 0x4), + Export(12, ExportTargetKind::Mrt, 3), + Export(16, ExportTargetKind::Mrt, 7, 0)}; Prepare(program); ShaderPixelInputInfo pixel; @@ -548,8 +554,7 @@ void TestPixelShaderInfoCollection() { pixel.ps_depth_export_enable = true; pixel.ps_sample_mask_export_enable = true; std::string error; - Check(CollectShaderInfo(program, {.pixel = &pixel}, &error), - error.c_str()); + Check(CollectShaderInfo(program, {.pixel = &pixel}, &error), error.c_str()); Check(program.info.inputs.size() == 4 && FindInput(program.info, StageInputKind::FragCoord) != nullptr && FindInput(program.info, StageInputKind::FrontFacing) != nullptr && @@ -560,7 +565,8 @@ void TestPixelShaderInfoCollection() { FindOutput(program.info, StageOutputKind::Mrt, 7) == nullptr && FindOutput(program.info, StageOutputKind::Depth) == nullptr && FindOutput(program.info, StageOutputKind::SampleMask) != nullptr, - "pixel inputs, disabled exports, or sample-mask-only MRTZ collection was wrong"); + "pixel inputs, disabled exports, or sample-mask-only MRTZ collection " + "was wrong"); Program depth_program; depth_program.stage = ShaderType::Pixel; @@ -572,7 +578,8 @@ void TestPixelShaderInfoCollection() { error.c_str()); Check(depth_program.info.outputs.size() == 1 && FindOutput(depth_program.info, StageOutputKind::Depth) != nullptr && - FindOutput(depth_program.info, StageOutputKind::SampleMask) == nullptr, + FindOutput(depth_program.info, StageOutputKind::SampleMask) == + nullptr, "depth-only MRTZ export incorrectly enabled sample-mask output"); } @@ -670,13 +677,13 @@ void TestTrackingRequiresSrtPatching() { program.blocks.resize(1); program.blocks[0].instructions = {BufferUse(4, 0)}; std::string error; - Check(BuildScalarProvenance(program, &error) && - BuildSrtPlan(program, &error), + Check(BuildScalarProvenance(program, &error) && BuildSrtPlan(program, &error), error.c_str()); const auto source = program.blocks[0].instructions[0].memory.resource_source; Check(!TrackResources(program, &error) && error.find("SRT reads were not patched") != std::string::npos && - program.blocks[0].instructions[0].memory.resource_source == source && + program.blocks[0].instructions[0].memory.resource_source == + source && !program.resource_tracking_complete, "resource tracking bypassed SRT patch completion"); Check(PatchSrtReads(program, &error) && TrackResources(program, &error), @@ -693,8 +700,7 @@ void TestDynamicSrtReadRemainsExplicit() { load.src_count = 1; insts.push_back(load); } - insts.push_back(ImageUse(0x40, Opcode::ImageStore, - ResourceKind::StorageImage, + insts.push_back(ImageUse(0x40, Opcode::ImageStore, ResourceKind::StorageImage, Decoder::ImageDimension::Dim2D)); std::string error; Check(BuildScalarProvenance(program, &error) && @@ -783,12 +789,13 @@ void TestScalarMemoryGroupsSnapshotOperands() { const auto value = insts[i].scalar_value; const auto &node = program.provenance.values[value]; const auto offset_arg = buffer ? 4u : 2u; - Check(program.provenance.values[node.args[offset_arg]].op == - ScalarValueOp::UserData && - program.provenance.values[node.args[offset_arg]].imm == 20 && - insts[i].op == - (buffer ? Opcode::SBufferLoadDword : Opcode::SLoadDword), - "overlapping scalar-memory offset was read after a component write"); + Check( + program.provenance.values[node.args[offset_arg]].op == + ScalarValueOp::UserData && + program.provenance.values[node.args[offset_arg]].imm == 20 && + insts[i].op == + (buffer ? Opcode::SBufferLoadDword : Opcode::SLoadDword), + "overlapping scalar-memory offset was read after a component write"); } }; CheckOffsetOverlap(false); @@ -801,8 +808,7 @@ void TestSrtPatchingHandlesGvnAndMoveForwarding() { auto &insts = program.blocks[0].instructions; for (uint32_t copy = 0; copy < 2; copy++) { for (uint32_t i = 0; i < 4; i++) { - insts.push_back(ScalarLoad(copy * 0x20 + i * 4, copy * 4 + i, 16, - i * 4)); + insts.push_back(ScalarLoad(copy * 0x20 + i * 4, copy * 4 + i, 16, i * 4)); } } for (uint32_t i = 0; i < 4; i++) { @@ -817,8 +823,7 @@ void TestSrtPatchingHandlesGvnAndMoveForwarding() { PatchSrtReads(program, &error), error.c_str()); for (uint32_t i = 0; i < 8; i++) { - Check(insts[i].op == Opcode::LoadSrtDword && - insts[i].src[0].imm == i % 4, + Check(insts[i].op == Opcode::LoadSrtDword && insts[i].src[0].imm == i % 4, "all producers of a GVN'd SRT read were not patched"); } for (uint32_t i = 8; i < 12; i++) { @@ -867,25 +872,25 @@ void TestSrtPatchPlanValidation() { } insts.push_back(BufferUse(0x20, 0)); std::string error; - Check(BuildScalarProvenance(program, &error) && - BuildSrtPlan(program, &error), + Check(BuildScalarProvenance(program, &error) && BuildSrtPlan(program, &error), error.c_str()); program.srt.reads = {program.srt.reads[0], program.srt.reads[0]}; const auto instructions = insts; Check(!PatchSrtReads(program, &error) && - error.find("dense value-to-offset bijection") != std::string::npos && + error.find("dense value-to-offset bijection") != + std::string::npos && insts == instructions && !program.srt_patching_complete, "duplicate SRT flat slots were accepted or partially patched"); Program dynamic; dynamic.blocks.resize(1); dynamic.blocks[0].instructions = {BufferUse(4, 0)}; - Check(BuildScalarProvenance(dynamic, &error) && - BuildSrtPlan(dynamic, &error), + Check(BuildScalarProvenance(dynamic, &error) && BuildSrtPlan(dynamic, &error), error.c_str()); dynamic.srt.dynamic_sources = {ScalarProvenance::Undefined}; Check(!PatchSrtReads(dynamic, &error) && - error.find("invalid dynamic descriptor source") != std::string::npos && + error.find("invalid dynamic descriptor source") != + std::string::npos && !dynamic.srt_patching_complete, "undefined dynamic descriptor source was accepted"); } @@ -904,8 +909,7 @@ void TestMaterializationSharesReadConstEvaluation() { } insts.push_back(ImageUse(0x40, Opcode::ImageSample, ResourceKind::Image, Decoder::ImageDimension::Dim2D)); - insts.push_back(ImageUse(0x44, Opcode::ImageStore, - ResourceKind::StorageImage, + insts.push_back(ImageUse(0x44, Opcode::ImageStore, ResourceKind::StorageImage, Decoder::ImageDimension::Dim2D)); Prepare(program); Check(program.info.images.size() == 2 && program.info.samplers.size() == 1, @@ -928,8 +932,8 @@ void TestMaterializationSharesReadConstEvaluation() { for (uint32_t i = 0; i < memory.words.size(); i++) { memory.words[i] = 0x100 + i; } - memory.words[3] |= - Prospero::GpuEnumValue(Prospero::ImageType::kColor2D) << 28u; + memory.words[3] |= Prospero::GpuEnumValue(Prospero::ImageType::kColor2D) + << 28u; std::array user_data{}; user_data[16] = static_cast(memory.base); user_data[17] = static_cast(memory.base >> 32u); @@ -943,12 +947,13 @@ void TestMaterializationSharesReadConstEvaluation() { snapshot.images[0] == snapshot.images[1], "dense runtime snapshot did not preserve resource order or aliases"); Check(snapshot.flattened_srt.size() == 8 && - std::equal(snapshot.flattened_srt.begin(), snapshot.flattened_srt.end(), - memory.words.begin()) && - snapshot.user_data == std::vector(user_data.begin(), user_data.end()), + std::equal(snapshot.flattened_srt.begin(), + snapshot.flattened_srt.end(), memory.words.begin()) && + snapshot.user_data == + std::vector(user_data.begin(), user_data.end()), "runtime snapshot omitted flattened SRT or current user data"); - Check(memory.reads == 8, - "aliased image views repeated ReadConst evaluation instead of sharing it"); + Check(memory.reads == 8, "aliased image views repeated ReadConst evaluation " + "instead of sharing it"); for (uint32_t i = 0; i < 8; i++) { Check(snapshot.images[0].dwords[i] == memory.words[i], "materialized image descriptor contains the wrong dword"); @@ -960,16 +965,16 @@ void TestInvalidImagesMaterializeAsNull() { sampled.stage = ShaderType::Pixel; sampled.user_data_count = 8; sampled.blocks.resize(1); - sampled.blocks[0].instructions = { - ImageUse(0x40, Opcode::ImageLoad, ResourceKind::Image, - Decoder::ImageDimension::Dim2D)}; + sampled.blocks[0].instructions = {ImageUse(0x40, Opcode::ImageLoad, + ResourceKind::Image, + Decoder::ImageDimension::Dim2D)}; Prepare(sampled); Check(sampled.info.images.size() == 1 && sampled.info.samplers.empty(), "sampled-image normalization test has unexpected resource topology"); - constexpr std::array stale = { - 0x00000004, 0x00000004, 0xc0061060, 0x06000514, - 0x20010000, 0xa4580290, 0x00000004, 0x00000001}; + constexpr std::array stale = {0x00000004, 0x00000004, 0xc0061060, + 0x06000514, 0x20010000, 0xa4580290, + 0x00000004, 0x00000001}; std::string error; for (uint32_t type = 0; type < 8; type++) { auto descriptor = stale; @@ -997,9 +1002,9 @@ void TestInvalidImagesMaterializeAsNull() { storage.stage = ShaderType::Compute; storage.user_data_count = 8; storage.blocks.resize(1); - storage.blocks[0].instructions = { - ImageUse(0x40, Opcode::ImageStore, ResourceKind::StorageImage, - Decoder::ImageDimension::Dim2D)}; + storage.blocks[0].instructions = {ImageUse(0x40, Opcode::ImageStore, + ResourceKind::StorageImage, + Decoder::ImageDimension::Dim2D)}; Prepare(storage); ResourceSnapshot storage_snapshot; Check(MaterializeResources(storage, {stale}, storage_snapshot, &error) && @@ -1016,8 +1021,7 @@ void TestMaterializationFailureIsTransactional() { for (uint32_t i = 0; i < 8; i++) { insts.push_back(ScalarLoad(i * 4, i, 16, i * 4)); } - insts.push_back(ImageUse(0x40, Opcode::ImageStore, - ResourceKind::StorageImage, + insts.push_back(ImageUse(0x40, Opcode::ImageStore, ResourceKind::StorageImage, Decoder::ImageDimension::Dim2D)); Prepare(program); @@ -1056,18 +1060,20 @@ void TestResourceSpecializationIsTypedAndTransactional() { null_snapshot.images[0].dwords[2] = 0x89abcdefu; null_snapshot.images[0].dwords[3] = 0x01234567u; std::string error; - Check(SpecializeResources(null_program, null_snapshot, &error) && - ValidateResourceSpecialization(null_program, null_snapshot, &error) && - null_program.info.images[0].kind == ResourceKind::Image && - null_program.info.images[0].dimension == Decoder::ImageDimension::Dim3D, - "zero-base image descriptor did not preserve tracked null-image shape"); + Check( + SpecializeResources(null_program, null_snapshot, &error) && + ValidateResourceSpecialization(null_program, null_snapshot, &error) && + null_program.info.images[0].kind == ResourceKind::Image && + null_program.info.images[0].dimension == + Decoder::ImageDimension::Dim3D, + "zero-base image descriptor did not preserve tracked null-image shape"); Program program; program.stage = ShaderType::Compute; program.blocks.resize(1); - program.blocks[0].instructions = { - ImageUse(0x20, Opcode::ImageStore, ResourceKind::StorageImage, - Decoder::ImageDimension::Dim3D)}; + program.blocks[0].instructions = {ImageUse(0x20, Opcode::ImageStore, + ResourceKind::StorageImage, + Decoder::ImageDimension::Dim3D)}; Prepare(program); ResourceSnapshot snapshot; @@ -1098,7 +1104,8 @@ void TestResourceSpecializationIsTypedAndTransactional() { image.storage_swizzle == 0x3acu && inst.memory.kind == ResourceKind::StorageImageUint && inst.memory.image_dimension == Decoder::ImageDimension::Dim2D, - "runtime descriptor shape and integer format did not specialize dense IR"); + "runtime descriptor shape and integer format did not specialize dense " + "IR"); auto stale_swizzle = snapshot; stale_swizzle.images[0].dwords[3] = @@ -1115,9 +1122,12 @@ void TestResourceSpecializationIsTypedAndTransactional() { error.c_str()); const auto *binding = FindBinding(program.bindings, DescriptorBindingKind::StorageUint2D); - Check(binding != nullptr && binding->resources == std::vector({0}) && - FindBinding(program.bindings, DescriptorBindingKind::Storage3D) == nullptr, - "specialized image topology did not reach the exact native binding group"); + Check(binding != nullptr && + binding->resources == std::vector({0}) && + FindBinding(program.bindings, DescriptorBindingKind::Storage3D) == + nullptr, + "specialized image topology did not reach the exact native binding " + "group"); } void TestRuntimeSpecializationCoversBakedBufferAndAddressFields() { @@ -1132,11 +1142,12 @@ void TestRuntimeSpecializationCoversBakedBufferAndAddressFields() { buffer_snapshot.buffers[0].dword_count = 4; buffer_snapshot.buffers[0].dwords[1] = (16u << 16u) | (1u << 31u); buffer_snapshot.buffers[0].dwords[3] = - (Prospero::GpuEnumValue(Prospero::BufferFormat::k32UInt) << 12u) | (2u << 21u) | - (1u << 23u); + (Prospero::GpuEnumValue(Prospero::BufferFormat::k32UInt) << 12u) | + (2u << 21u) | (1u << 23u); std::string error; Check(SpecializeResources(buffer_program, buffer_snapshot, &error) && - ValidateResourceSpecialization(buffer_program, buffer_snapshot, &error), + ValidateResourceSpecialization(buffer_program, buffer_snapshot, + &error), error.c_str()); Check(buffer_program.info.buffers[0].packed_stride == (16u | (1u << 14u) | (2u << 16u) | (1u << 20u)) && @@ -1162,9 +1173,10 @@ void TestRuntimeSpecializationCoversBakedBufferAndAddressFields() { error.c_str()); auto truncated_user_data = buffer_snapshot; truncated_user_data.user_data.resize(3); - Check(!ValidateResourceSnapshot(buffer_program, truncated_user_data, &error) && - error.find("user SGPR 3") != std::string::npos, - "truncated cache snapshot user-data window was accepted"); + Check( + !ValidateResourceSnapshot(buffer_program, truncated_user_data, &error) && + error.find("user SGPR 3") != std::string::npos, + "truncated cache snapshot user-data window was accepted"); Program based_program; based_program.stage = ShaderType::Compute; @@ -1173,8 +1185,8 @@ void TestRuntimeSpecializationCoversBakedBufferAndAddressFields() { raw.memory.offset = static_cast(-4); raw.src[0] = Sgpr(20); raw.src_count = 1; - based_program.blocks[0].instructions = { - MoveImmediate(0, 16, 0x1000), MoveImmediate(4, 17, 0), raw}; + based_program.blocks[0].instructions = {MoveImmediate(0, 16, 0x1000), + MoveImmediate(4, 17, 0), raw}; Prepare(based_program); ResourceSnapshot based_snapshot; Check(MaterializeResources(based_program, {}, based_snapshot, &error) && @@ -1184,7 +1196,8 @@ void TestRuntimeSpecializationCoversBakedBufferAndAddressFields() { relocated.addresses[0].guest_base += 0x1000; relocated.addresses[0].binding_base += 0x1000; Check(ValidateResourceSpecialization(based_program, relocated, &error), - "relocated based address with identical relative bias changed specialization"); + "relocated based address with identical relative bias changed " + "specialization"); relocated.addresses[0].binding_base++; Check(!ValidateResourceSpecialization(based_program, relocated, &error) && error.find("address resource 0") != std::string::npos, @@ -1199,19 +1212,22 @@ void TestRuntimeSpecializationCoversBakedBufferAndAddressFields() { flat_program.blocks[0].instructions = {flat}; Prepare(flat_program); ResourceSnapshot flat_snapshot; - flat_snapshot.addresses = {{0x11u, 0x10u}}; + flat_snapshot.addresses = {{0x11u, 0x10u}}; flat_snapshot.user_data = {0xdeadbeefu}; const auto prior_flat_snapshot = flat_snapshot; - Check(!MaterializeResources(flat_program, {}, flat_snapshot, &error) && - error.find("requires runtime guest-address translation") != std::string::npos && - flat_snapshot.addresses == prior_flat_snapshot.addresses && - flat_snapshot.user_data == prior_flat_snapshot.user_data, - "unbased flat memory without a translator did not fail transactionally"); + Check( + !MaterializeResources(flat_program, {}, flat_snapshot, &error) && + error.find("requires runtime guest-address translation") != + std::string::npos && + flat_snapshot.addresses == prior_flat_snapshot.addresses && + flat_snapshot.user_data == prior_flat_snapshot.user_data, + "unbased flat memory without a translator did not fail transactionally"); SrtRuntime flat_runtime; flat_runtime.flat_memory_base = 0x100000000ull; - Check(MaterializeResources(flat_program, flat_runtime, flat_snapshot, &error) && - SpecializeResources(flat_program, flat_snapshot, &error), - error.c_str()); + Check( + MaterializeResources(flat_program, flat_runtime, flat_snapshot, &error) && + SpecializeResources(flat_program, flat_snapshot, &error), + error.c_str()); auto stale_flat = flat_snapshot; stale_flat.addresses[0].guest_base += 0x1000; stale_flat.addresses[0].binding_base += 0x1000; @@ -1232,8 +1248,7 @@ void TestTrackedProgramIsImmutable() { } insts.push_back(ImageUse(0x50, Opcode::ImageSample, ResourceKind::Image, Decoder::ImageDimension::Dim2D)); - insts.push_back(ImageUse(0x54, Opcode::ImageStore, - ResourceKind::StorageImage, + insts.push_back(ImageUse(0x54, Opcode::ImageStore, ResourceKind::StorageImage, Decoder::ImageDimension::Dim2D)); insts.push_back(BufferUse(0x58, 12)); Prepare(program); @@ -1263,8 +1278,9 @@ void TestTrackedProgramIsImmutable() { insts.size() == memory.size(), "rejected post-track pass mutated immutable program state"); for (uint32_t i = 0; i < memory.size(); i++) { - Check(insts[i].memory == memory[i], - "rejected post-track pass mutated a dense operand or source handle"); + Check( + insts[i].memory == memory[i], + "rejected post-track pass mutated a dense operand or source handle"); } }; @@ -1289,8 +1305,7 @@ void TestNativeBindingLayout() { Decoder::ImageDimension::Dim2D, 0, 2)); insts.push_back(ImageUse(12, Opcode::ImageSample, ResourceKind::Image, Decoder::ImageDimension::Dim2DArray, 0, 2)); - insts.push_back(ImageUse(16, Opcode::ImageStore, - ResourceKind::StorageImage, + insts.push_back(ImageUse(16, Opcode::ImageStore, ResourceKind::StorageImage, Decoder::ImageDimension::Dim3D, 4)); insts.push_back(ImageUse(20, Opcode::ImageStore, ResourceKind::StorageImageUint, @@ -1315,25 +1330,26 @@ void TestNativeBindingLayout() { FindBinding(program.bindings, DescriptorBindingKind::StorageUint2DArray); const auto *samplers = FindBinding(program.bindings, DescriptorBindingKind::Samplers); - Check(program.bindings.descriptor_set == 3 && buffers != nullptr && - sampled2d != nullptr && sampled_array != nullptr && - storage3d != nullptr && storage_uint_array != nullptr && - samplers != nullptr && buffers->binding == 0 && - sampled2d->binding == 1 && sampled_array->binding == 2 && - storage3d->binding == 3 && storage_uint_array->binding == 4 && - samplers->binding == 5 && buffers->resources == - std::vector{0} && - sampled2d->resources == std::vector{0} && - sampled_array->resources == std::vector{1} && - storage3d->resources == std::vector{2} && - storage_uint_array->resources == std::vector{3} && - samplers->resources == std::vector{0} && - !program.bindings.user_data_registers.empty() && - program.bindings.push_constant_size != 0 && - FindBinding(program.bindings, - DescriptorBindingKind::FlattenedSrt) == nullptr && - program.binding_layout_complete, - "native binding allocator did not preserve dense typed resource groups"); + Check( + program.bindings.descriptor_set == 3 && buffers != nullptr && + sampled2d != nullptr && sampled_array != nullptr && + storage3d != nullptr && storage_uint_array != nullptr && + samplers != nullptr && buffers->binding == 0 && + sampled2d->binding == 1 && sampled_array->binding == 2 && + storage3d->binding == 3 && storage_uint_array->binding == 4 && + samplers->binding == 5 && + buffers->resources == std::vector{0} && + sampled2d->resources == std::vector{0} && + sampled_array->resources == std::vector{1} && + storage3d->resources == std::vector{2} && + storage_uint_array->resources == std::vector{3} && + samplers->resources == std::vector{0} && + !program.bindings.user_data_registers.empty() && + program.bindings.push_constant_size != 0 && + FindBinding(program.bindings, DescriptorBindingKind::FlattenedSrt) == + nullptr && + program.binding_layout_complete, + "native binding allocator did not preserve dense typed resource groups"); } void TestNativeBindingLayoutSrtAndUserDataOverflow() { @@ -1341,8 +1357,7 @@ void TestNativeBindingLayoutSrtAndUserDataOverflow() { srt.stage = ShaderType::Compute; srt.blocks.resize(1); for (uint32_t i = 0; i < 4; i++) { - srt.blocks[0].instructions.push_back( - ScalarLoad(i * 4, i, 16, i * 4)); + srt.blocks[0].instructions.push_back(ScalarLoad(i * 4, i, 16, i * 4)); } srt.blocks[0].instructions.push_back(BufferUse(0x20, 0)); Prepare(srt); @@ -1392,7 +1407,7 @@ void TestNativeBindingLayoutGds() { program.blocks.resize(1); Instruction append; append.op = Opcode::DsAppend; - append.memory.kind = ResourceKind::Gds; + append.memory.kind = ResourceKind::Gds; program.blocks[0].instructions.push_back(append); ShaderComputeInputInfo compute; @@ -1407,18 +1422,18 @@ void TestNativeBindingLayoutGds() { Check(gds != nullptr && gds->binding == 0 && gds->resources.empty(), "GDS append/consume did not allocate one native storage binding"); - Program lds; - lds.stage = ShaderType::Compute; - lds.blocks.resize(1); - append.memory.kind = ResourceKind::Lds; - lds.blocks[0].instructions.push_back(append); - Check(BuildScalarProvenance(lds, &error) && BuildSrtPlan(lds, &error) && - PatchSrtReads(lds, &error) && TrackResources(lds, &error) && - CollectShaderInfo(lds, {.compute = &compute}, &error) && - AllocateBindings(lds, {}, &error), - error.c_str()); - Check(FindBinding(lds.bindings, DescriptorBindingKind::Gds) == nullptr, - "LDS append/consume incorrectly allocated a GDS binding"); + Program lds; + lds.stage = ShaderType::Compute; + lds.blocks.resize(1); + append.memory.kind = ResourceKind::Lds; + lds.blocks[0].instructions.push_back(append); + Check(BuildScalarProvenance(lds, &error) && BuildSrtPlan(lds, &error) && + PatchSrtReads(lds, &error) && TrackResources(lds, &error) && + CollectShaderInfo(lds, {.compute = &compute}, &error) && + AllocateBindings(lds, {}, &error), + error.c_str()); + Check(FindBinding(lds.bindings, DescriptorBindingKind::Gds) == nullptr, + "LDS append/consume incorrectly allocated a GDS binding"); } void TestNativeBindingLayoutIsTransactional() { @@ -1466,7 +1481,8 @@ void TestNativeBindingLayoutIsTransactional() { CollectShaderInfo(tail, {.compute = &compute}, &error) && AllocateBindings(tail, {.push_constant_offset = 124}, &error) && tail.bindings.push_constant_size == 4 && - FindBinding(tail.bindings, DescriptorBindingKind::UserData) == nullptr, + FindBinding(tail.bindings, DescriptorBindingKind::UserData) == + nullptr, "valid final dword of the guaranteed push-constant range was rejected"); } @@ -1529,12 +1545,11 @@ void TestNativeBindingLayoutRejectsUnknownShapeAndBadProvenance() { Program image; image.stage = ShaderType::Compute; image.blocks.resize(1); - image.blocks[0].instructions = { - ImageUse(4, Opcode::ImageStore, ResourceKind::StorageImage, - Decoder::ImageDimension::Dim2D)}; + image.blocks[0].instructions = {ImageUse(4, Opcode::ImageStore, + ResourceKind::StorageImage, + Decoder::ImageDimension::Dim2D)}; Prepare(image); - Check(CollectShaderInfo(image, {.compute = &compute}, &error), - error.c_str()); + Check(CollectShaderInfo(image, {.compute = &compute}, &error), error.c_str()); image.info.images[0].dimension = Decoder::ImageDimension::Unknown; image.bindings.descriptor_set = 77; const auto image_layout = image.bindings; @@ -1618,8 +1633,8 @@ void TestNativeBindingLayoutTracksRawScalarMemoryBase() { compute.thread_ids_num = 1; Check(BuildScalarProvenance(program, &error) && BuildSrtPlan(program, &error) && program.srt.reads.empty() && - program.srt.dynamic_reads.empty() && PatchSrtReads(program, &error) && - TrackResources(program, &error) && + program.srt.dynamic_reads.empty() && + PatchSrtReads(program, &error) && TrackResources(program, &error) && CollectShaderInfo(program, {.compute = &compute}, &error) && AllocateBindings(program, {}, &error), error.c_str()); @@ -1649,23 +1664,27 @@ void TestRawScalarMemoryTracksReachingBaseIdentity() { second.src[0] = Sgpr(20); second.src_count = 1; program.blocks[0].instructions = { - MoveImmediate(0, 16, 0x1000), MoveImmediate(4, 17, 0), first, + MoveImmediate(0, 16, 0x1000), MoveImmediate(4, 17, 0), first, MoveImmediate(12, 16, 0x2000), MoveImmediate(16, 17, 0), second}; std::string error; - Check(BuildScalarProvenance(program, &error) && BuildSrtPlan(program, &error) && - PatchSrtReads(program, &error) && TrackResources(program, &error), + Check(BuildScalarProvenance(program, &error) && + BuildSrtPlan(program, &error) && PatchSrtReads(program, &error) && + TrackResources(program, &error), error.c_str()); Check(program.info.addresses.size() == 2 && - program.info.addresses[0].source != program.info.addresses[1].source && + program.info.addresses[0].source != + program.info.addresses[1].source && program.blocks[0].instructions[2].memory.resource == 0 && program.blocks[0].instructions[5].memory.resource == 1, - "raw scalar loads with redefined SBASE collapsed to one address resource"); + "raw scalar loads with redefined SBASE collapsed to one address " + "resource"); std::array user_data{}; user_data[20] = 4; ResourceSnapshot snapshot; - Check(MaterializeResources(program, {user_data}, snapshot, &error), error.c_str()); + Check(MaterializeResources(program, {user_data}, snapshot, &error), + error.c_str()); Check(snapshot.addresses.size() == 2 && snapshot.addresses[0].guest_base == 0x1000 && snapshot.addresses[0].binding_base == 0x0ffc && @@ -1692,11 +1711,13 @@ void TestNativeBindingLayoutUsesExplicitFlatMemory() { error.c_str()); const auto *flat = FindBinding(program.bindings, DescriptorBindingKind::AddressMemory); - Check(flat != nullptr && flat->resources == std::vector({0}) && - program.info.addresses.size() == 1 && - program.info.addresses[0].kind == ResourceKind::Flat && - FindBinding(program.bindings, DescriptorBindingKind::Buffers) == nullptr, - "flat address space was aliased to an ordinary buffer descriptor group"); + Check( + flat != nullptr && flat->resources == std::vector({0}) && + program.info.addresses.size() == 1 && + program.info.addresses[0].kind == ResourceKind::Flat && + FindBinding(program.bindings, DescriptorBindingKind::Buffers) == + nullptr, + "flat address space was aliased to an ordinary buffer descriptor group"); ResourceSnapshot snapshot; SrtRuntime runtime; @@ -1727,8 +1748,8 @@ void TestNativeBindingLayoutHonorsUserDataCount() { ShaderComputeInputInfo compute; compute.thread_ids_num = 1; Check(BuildScalarProvenance(program, &error) && - program.provenance.values[program.blocks[0].instructions[0] - .scalar_sources[0]] + program.provenance + .values[program.blocks[0].instructions[0].scalar_sources[0]] .op == ScalarValueOp::UserData && program.blocks[0].instructions[1].scalar_sources[0] == ScalarProvenance::Unknown && @@ -1753,28 +1774,29 @@ void TestNativeBindingLayoutHonorsUserDataBase() { program.user_data_base = 8; program.user_data_count = 8; program.blocks.resize(1); - program.blocks[0].instructions = { - Move(0, 20, 7), Move(4, 21, 8), Move(8, 22, 15), Move(12, 23, 16), - BufferUse(16, 8)}; + program.blocks[0].instructions = {Move(0, 20, 7), Move(4, 21, 8), + Move(8, 22, 15), Move(12, 23, 16), + BufferUse(16, 8)}; std::string error; Check(BuildScalarProvenance(program, &error) && program.blocks[0].instructions[0].scalar_sources[0] == ScalarProvenance::Unknown && - program.provenance.values[ - program.blocks[0].instructions[1].scalar_sources[0]].op == - ScalarValueOp::UserData && + program.provenance + .values[program.blocks[0].instructions[1].scalar_sources[0]] + .op == ScalarValueOp::UserData && program.blocks[0].instructions[3].scalar_sources[0] == ScalarProvenance::Unknown && BuildSrtPlan(program, &error), error.c_str()); const std::array user_data = { - 0x11111111, 0x22222222, 0x33333333, 0x44444444, - 0, 0, 0, 0xaaaaaaaa}; + 0x11111111, 0x22222222, 0x33333333, 0x44444444, 0, 0, 0, 0xaaaaaaaa}; DescriptorValue descriptor; - const auto source = program.blocks[0].instructions.back().memory.resource_source; - Check(EvaluateDescriptorSource(program, source, 16, {user_data}, descriptor, &error) && + const auto source = + program.blocks[0].instructions.back().memory.resource_source; + Check(EvaluateDescriptorSource(program, source, 16, {user_data}, descriptor, + &error) && descriptor.dwords[0] == user_data[0] && descriptor.dwords[3] == user_data[3], "vertex user-data base was not translated to runtime-local indices"); @@ -1784,18 +1806,21 @@ void TestNativeBindingLayoutHonorsUserDataBase() { CollectShaderInfo(program, {.vertex = &vertex}, &error) && AllocateBindings(program, {}, &error), error.c_str()); - Check(program.bindings.user_data_registers == - std::vector({8, 9, 10, 11, 15}), - "native binding plan did not retain physical shifted user-SGPR indices"); + Check( + program.bindings.user_data_registers == + std::vector({8, 9, 10, 11, 15}), + "native binding plan did not retain physical shifted user-SGPR indices"); } void TestTextureNullDescriptorUsesAddressBits() { ShaderTextureResource texture; const uint32_t captured[8] = {0x00000000, 0xc3800000, 0x0059c09f, 0x91b00fac, 0x00000000, 0x00700000, 0x00000000, 0x00000000}; - std::copy(std::begin(captured), std::end(captured), std::begin(texture.fields)); - Check(texture.IsNull(), - "zero-address image with populated metadata was not classified as null"); + std::copy(std::begin(captured), std::end(captured), + std::begin(texture.fields)); + Check( + texture.IsNull(), + "zero-address image with populated metadata was not classified as null"); texture.fields[0] = 1; Check(!texture.IsNull(), "nonzero image base address was classified as null"); } @@ -1803,8 +1828,10 @@ void TestTextureNullDescriptorUsesAddressBits() { } // namespace int main() { - const char* current = "startup"; -#define RUN(test) current = #test; test() + const char *current = "startup"; +#define RUN(test) \ + current = #test; \ + test() try { RUN(TestDenseBufferPatching); RUN(TestScalarAndVectorBufferAlias); @@ -1849,7 +1876,8 @@ int main() { std::cout << "ResourceTrackingTests: all cases passed\n"; return 0; } catch (const std::exception &e) { - std::cerr << "ResourceTrackingTests: " << current << " failed: " << e.what() << '\n'; + std::cerr << "ResourceTrackingTests: " << current << " failed: " << e.what() + << '\n'; return 1; } #undef RUN diff --git a/tests/ShaderRecompilerComputeTests.cpp b/tests/ShaderRecompilerComputeTests.cpp index e36190ed..440877d5 100644 --- a/tests/ShaderRecompilerComputeTests.cpp +++ b/tests/ShaderRecompilerComputeTests.cpp @@ -7,20 +7,23 @@ #include "graphics/guest_gpu/command_processor/pm4Dispatch.h" #include "graphics/guest_gpu/gpu_defs.h" #include "graphics/guest_gpu/gpu_format.h" +#include "graphics/guest_gpu/graphicsRun.h" #include "graphics/guest_gpu/hardwareContext.h" #include "graphics/guest_gpu/pm4.h" #include "graphics/guest_gpu/tile.h" -#include "graphics/host_gpu/gpuTiler.h" #include "graphics/host_gpu/hostMemory.h" #include "graphics/host_gpu/memoryTracker.h" #include "graphics/host_gpu/objects/textureCommon.h" #include "graphics/host_gpu/pageManager.h" +#include "graphics/host_gpu/renderer/blitHelper.h" #include "graphics/host_gpu/renderer/bufferCache.h" #include "graphics/host_gpu/renderer/colorRenderTarget.h" #include "graphics/host_gpu/renderer/depthRenderTarget.h" #include "graphics/host_gpu/renderer/descriptors.h" +#include "graphics/host_gpu/renderer/gpuResourceManager.h" #include "graphics/host_gpu/renderer/image.h" #include "graphics/host_gpu/renderer/imageView.h" +#include "graphics/host_gpu/renderer/pipelineCache.h" #include "graphics/host_gpu/renderer/render.h" #include "graphics/host_gpu/renderer/renderContext.h" #include "graphics/host_gpu/renderer/renderDraw.h" @@ -29,16 +32,30 @@ #include "graphics/host_gpu/renderer/sync.h" #include "graphics/host_gpu/renderer/textureCache.h" #include "graphics/host_gpu/renderer/tiler.h" -#include "graphics/host_gpu/transfer.h" #include "graphics/host_gpu/vulkanCommon.h" +#include "graphics/presentation/window/windowInternal.h" #include "graphics/shader/recompiler/BindingLayout.h" #include "graphics/shader/recompiler/ShaderDecoder.h" #include "graphics/shader/recompiler/ShaderRecompiler.h" #include "graphics/shader/recompiler/SpirvBuilder.h" #include "graphics/shader/shader.h" +#include "kernel/memory.h" +#include "gpu_test_shaders/gpu_test_ms_depth_spv.h" #include "spirv-tools/libspirv.hpp" +#if __has_include("graphics/host_gpu/renderer/renderTargetBarriers.h") +#error "legacy render-target barrier API must remain deleted" +#endif + +#if __has_include("graphics/host_gpu/transfer.h") +#error "legacy transfer facade must remain deleted" +#endif + +#if __has_include("graphics/host_gpu/renderer/framebufferCache.h") +#error "classic framebuffer/render-pass cache must remain deleted" +#endif + #include #include #include @@ -49,9 +66,13 @@ #include #include #include +#include #include +#include #include +#include #include +#include #include #include #include @@ -66,9 +87,195 @@ #endif namespace Libs::Graphics { + +template +concept HasGetDownloadBuffer = + requires(Cache &cache) { cache.GetDownloadBuffer(uint64_t{1}); }; +static_assert(!HasGetDownloadBuffer); + +template +concept HasLegacyImageLayout = requires(Backing &backing) { backing.layout; }; +static_assert(!HasLegacyImageLayout); + +template +concept HasPolicyDetileImage = requires { &Manager::DetileImage; }; +template +concept HasPolicyTileImageBacking = requires { &Manager::TileImageBacking; }; +static_assert(!HasPolicyDetileImage); +static_assert(!HasPolicyTileImageBacking); +static_assert(BlitHelper::ColorToMsDepthLayout == + vk::ImageLayout::eDepthStencilAttachmentOptimal); + +struct BufferCacheTestAccess { + static void SetGarbageCollectionThresholds(BufferCache &cache, + uint64_t trigger, + uint64_t critical) { + cache.m_trigger_gc_memory = trigger; + cache.m_critical_gc_memory = critical; + } + + static StreamBuffer &DownloadBuffer(BufferCache &cache) { + return cache.m_download_buffer; + } +}; + +struct StreamBufferTestAccess { + static bool NormalizeReservation(bool coherent, uint64_t atom, uint64_t &size, + uint64_t &alignment) { + return StreamBuffer::NormalizeReservation(coherent, atom, size, alignment); + } +}; + +struct ImageTestAccess { + static uint32_t CopyRows(uint64_t row_size, uint32_t rows, + uint64_t capacity) { + return Image::CopyRows(row_size, rows, capacity); + } +}; + +struct TileManagerTestAccess { + static uint32_t ConversionRows(uint64_t offset, uint64_t row_stride, + uint64_t active, uint32_t remaining, + uint64_t alignment, uint64_t max_range, + uint32_t max_groups) { + return TileManager::ConversionRows(offset, row_stride, active, remaining, + alignment, max_range, max_groups); + } +}; + +struct TextureCacheTestAccess { + static void ConfigureGarbageCollection(TextureCache &cache, + std::span oldest, + uint64_t tick, uint64_t pressure) { + cache.m_trigger_gc_memory = 0; + cache.m_pressure_gc_memory = pressure; + cache.m_critical_gc_memory = UINT64_MAX; + cache.m_gc_tick = tick; + std::vector live; + cache.m_lru_cache = {}; + for (auto &slot : cache.m_slots) { + if (slot.image != nullptr && slot.image->registered) { + slot.image->tick_accessed_last = cache.m_scheduler.CurrentTick(); + live.push_back({static_cast(&slot - cache.m_slots.data()), + slot.generation}); + } + } + for (const auto id : oldest) { + const auto owner = cache.ResolveOwner(id); + if (owner != nullptr && owner->registered) { + owner->tick_accessed_last = 0; + owner->lru_id = cache.m_lru_cache.Insert(id, 0); + } + } + for (const auto id : live) { + if (std::ranges::find(oldest, id) == oldest.end()) { + cache.ResolveImage(id).lru_id = cache.m_lru_cache.Insert(id, tick); + } + } + } + + static bool Contains(const TextureCache &cache, ImageId id) { + const auto owner = cache.ResolveOwner(id); + return owner != nullptr && owner->registered; + } + + static std::shared_ptr Owner(const TextureCache &cache, ImageId id) { + return cache.ResolveOwner(id); + } + + static bool PendingDownload(const TextureCache &cache, ImageId id) { + return cache.m_download_images.contains(id); + } + + static size_t NullImageCount(const TextureCache &cache) { + return cache.m_null_images.size(); + } + + static void TrackDownload(TextureCache &cache, ImageId id) { + cache.TrackImageDownload(id); + } + + static void AssociateStencil(TextureCache &cache, ImageId depth, + GuestRange stencil) { + cache.AssociateStencil(depth, stencil); + } + + static void SetLinearReadback(TextureCache &cache, bool enabled) { + cache.m_readback_linear_images = enabled; + } + + static std::pair + MapDownload(TextureCache &cache, uint64_t size, uint64_t alignment) { + return cache.MapDownload(size, alignment); + } + + static bool TryDownload(TextureCache &cache, ImageId id) { + return cache.TryDownloadImage(id); + } + + static TileManager &Tiler(TextureCache &cache) { return *cache.m_tiler; } +}; + +struct RenderExecutorTestAccess { + static auto PrepareGraphicsBindings(RenderExecutor &executor, + CommandBuffer &buffer, + const ShaderStageRuntime &vertex, + const ShaderStageRuntime &pixel, + bool pixel_active) { + return executor.PrepareGraphicsBindings(buffer, vertex, pixel, + pixel_active); + } + + static void CommitBindings(RenderExecutor &executor, CommandBuffer &buffer, + DescriptorCache::PreparedBindings &bindings) { + executor.CommitBindings(buffer, vk::PipelineBindPoint::eGraphics, nullptr, + bindings); + } + + static void ResolveRenderDepthTarget(RenderExecutor &executor, + uint64_t submit_id, + RenderCommandBuffer &buffer, + RenderDepthInfo &depth) { + executor.ResolveRenderDepthTarget(submit_id, buffer, depth); + } + + static void BindRenderTarget(RenderExecutor &executor, ImageId id) { + executor.BindRenderTarget(id); + } + + static RenderState AcquireRenderTargets(RenderExecutor &executor, + CommandBuffer &buffer, + RenderColorInfo *colors, + uint32_t color_count, + RenderDepthInfo &depth) { + return executor.AcquireRenderTargets(buffer, colors, color_count, depth); + } + + static void ResetBindings(RenderExecutor &executor) { + executor.ResetBindings(); + } + + static bool BoundImagesInOrder(const RenderExecutor &executor, + const std::shared_ptr &first, + const std::shared_ptr &second) { + return executor.m_bound_images.size() == 2 && + executor.m_bound_images[0] == first && + executor.m_bound_images[1] == second; + } +}; + +struct DescriptorCacheTestAccess { + static vk::DescriptorImageInfo + MakeImageInfo(const DescriptorCache::TextureBinding &binding) { + return DescriptorCache::MakeImageInfo(binding); + } +}; + namespace { using u32 = uint32_t; +using BindingType = TextureCache::BindingType; +using ImageDesc = TextureCache::ImageDesc; using ShaderOpcode = ShaderRecompiler::Decoder::Opcode; constexpr u32 InlineU32(u32 value) { return 128u + value; } @@ -447,6 +654,7 @@ void EnsureConfigInitialized() { options.printf_direction = Config::OutputDirection::Silent; Config::Load(options); Log::LogSubsystem::Instance()->Init(nullptr); + Libs::LibKernel::Memory::MemorySubsystem::Instance()->Init(nullptr); config_initialized = true; } } @@ -466,9 +674,23 @@ void Require(const char *shader_name, const char *stage, bool value, } } -bool RejectUnexpectedPageFault(void *, PageFaultAccess, uint64_t, uint64_t, - PageFaultPhase) noexcept { - return false; +void CheckLeastRecentlyUsedCacheOrdering() { + Common::LeastRecentlyUsedCache cache; + const auto first = cache.Insert(1, 1); + const auto middle = cache.Insert(2, 2); + const auto last = cache.Insert(3, 3); + (void)first; + (void)last; + cache.Touch(middle, 4); + std::vector visited; + cache.ForEachItemBelow(4, [&](uint32_t value) { + visited.push_back(value); + return visited.size() > 3; + }); + Require("LeastRecentlyUsedCache", "non-tail touch traversal", + visited == std::vector{1, 3, 2}, + "touching a non-tail item left a cycle or changed LRU order"); + std::printf("[host] %-32s ok\n", "LeastRecentlyUsedCache"); } struct TestCase { @@ -857,563 +1079,4810 @@ public: }; [[nodiscard]] vk::Device Device() const { return m_device; } + [[nodiscard]] GraphicContext &RuntimeContext() { + EnsureRuntimeContext(); + return m_runtime_context; + } + [[nodiscard]] RenderContext &RuntimeRenderer() { + EnsureRuntimeContext(); + return Renderer(); + } void CheckCommandPoolGrowth() { EnsureRuntimeContext(); - std::array commands; - for (auto &command : commands) { - Require("CommandPoolGrowth", "allocation", !command.IsInvalid(), - "unified command pool failed to grow"); - command.Begin(); - command.End(); - command.Execute(); + std::vector> commands; + for (uint32_t i = 0; i < 12; i++) { + commands.push_back( + std::make_unique(Renderer().GetCommandScheduler())); } for (auto &command : commands) { - command.WaitForFence(); + Require("CommandPoolGrowth", "allocation", !command->IsInvalid(), + "unified command pool failed to grow"); + command->Begin(); + command->End(); + command->Execute(); + } + for (auto &command : commands) { + command->WaitForFence(); } std::printf("[host] %-32s ok\n", "CommandPoolGrowth"); } - void CheckMutableStorageSrgbView() { - constexpr const char *name = "StorageTextureMutableSrgbView"; - vk::ImageCreateInfo image_info{}; - image_info.sType = vk::StructureType::eImageCreateInfo; - image_info.flags = vk::ImageCreateFlagBits::eMutableFormat; - image_info.imageType = vk::ImageType::e2D; - image_info.format = vk::Format::eR8G8B8A8Srgb; - image_info.extent = {16, 16, 1}; - image_info.mipLevels = 1; - image_info.arrayLayers = 1; - image_info.samples = vk::SampleCountFlagBits::e1; - image_info.tiling = vk::ImageTiling::eOptimal; - image_info.usage = vk::ImageUsageFlagBits::eTransferSrc | - vk::ImageUsageFlagBits::eTransferDst | - vk::ImageUsageFlagBits::eSampled | - vk::ImageUsageFlagBits::eStorage; - image_info.sharingMode = vk::SharingMode::eExclusive; - image_info.initialLayout = vk::ImageLayout::eUndefined; - - vk::ImageFormatProperties exact_properties{}; - const bool srgb_storage_supported = - m_physical_device.getImageFormatProperties( - image_info.format, image_info.imageType, image_info.tiling, - image_info.usage, image_info.flags, - &exact_properties) == vk::Result::eSuccess; - GraphicContext context{}; - context.physical_device = m_physical_device; - Require(name, "format fallback", TextureCheckFormat(image_info), - "mutable sampled/storage RGBA8 image is unsupported"); - Require(name, "format fallback", - image_info.format == vk::Format::eR8G8B8A8Unorm || - (srgb_storage_supported && - image_info.format == vk::Format::eR8G8B8A8Srgb), - "unsupported sRGB storage format did not fall back to UNORM"); - - vk::Image image = nullptr; - RequireVk(name, "backing", - m_device.createImage(&image_info, nullptr, &image), - "vkCreateImage"); - vk::MemoryRequirements requirements{}; - m_device.getImageMemoryRequirements(image, &requirements); - u32 memory_type = 0; - Require(name, "backing", - FindMemoryType(requirements.memoryTypeBits, - vk::MemoryPropertyFlagBits::eDeviceLocal, - &memory_type) || - FindMemoryType(requirements.memoryTypeBits, {}, &memory_type), - "no memory type for mutable storage image"); - vk::MemoryAllocateInfo allocation{}; - allocation.sType = vk::StructureType::eMemoryAllocateInfo; - allocation.allocationSize = requirements.size; - allocation.memoryTypeIndex = memory_type; - vk::DeviceMemory memory = nullptr; - RequireVk(name, "backing", - m_device.allocateMemory(&allocation, nullptr, &memory), - "vkAllocateMemory"); - RequireVk(name, "backing", m_device.bindImageMemory(image, memory, 0), - "vkBindImageMemory"); - - vk::ImageViewUsageCreateInfo view_usage{}; - view_usage.sType = vk::StructureType::eImageViewUsageCreateInfo; - view_usage.usage = - vk::ImageUsageFlagBits::eSampled | vk::ImageUsageFlagBits::eStorage; - vk::ImageViewCreateInfo view_info{}; - view_info.sType = vk::StructureType::eImageViewCreateInfo; - view_info.pNext = &view_usage; - view_info.image = image; - view_info.viewType = vk::ImageViewType::e2D; - view_info.format = image_info.format; - view_info.subresourceRange.aspectMask = vk::ImageAspectFlagBits::eColor; - view_info.subresourceRange.levelCount = 1; - view_info.subresourceRange.layerCount = 1; - vk::ImageView default_view = nullptr; - RequireVk(name, "default view", - m_device.createImageView(&view_info, nullptr, &default_view), - "vk::Device::createImageView(default sampled/storage)"); - - view_usage.usage = vk::ImageUsageFlagBits::eSampled; - view_info.format = vk::Format::eR8G8B8A8Srgb; - vk::ImageView srgb_view = nullptr; - RequireVk(name, "sRGB view", - m_device.createImageView(&view_info, nullptr, &srgb_view), - "vk::Device::createImageView(sRGB sampled)"); - - m_device.destroyImageView(srgb_view, nullptr); - m_device.destroyImageView(default_view, nullptr); - m_device.destroyImage(image, nullptr); - m_device.freeMemory(memory, nullptr); - std::printf("[host] %-32s ok (backing=%d)\n", name, - static_cast(image_info.format)); - } - - void CheckMutableRenderTargetBgraStorageView() { - constexpr const char *name = "RenderTargetMutableBgraStorage"; - vk::ImageCreateInfo image_info{}; - image_info.sType = vk::StructureType::eImageCreateInfo; - image_info.flags = vk::ImageCreateFlagBits::eMutableFormat | - vk::ImageCreateFlagBits::eExtendedUsage; - image_info.imageType = vk::ImageType::e2D; - image_info.format = vk::Format::eB8G8R8A8Srgb; - image_info.extent = {8, 8, 1}; - image_info.mipLevels = 1; - image_info.arrayLayers = 1; - image_info.samples = vk::SampleCountFlagBits::e1; - image_info.tiling = vk::ImageTiling::eOptimal; - image_info.usage = vk::ImageUsageFlagBits::eColorAttachment | - vk::ImageUsageFlagBits::eTransferSrc | - vk::ImageUsageFlagBits::eTransferDst | - vk::ImageUsageFlagBits::eSampled | - vk::ImageUsageFlagBits::eStorage; - image_info.sharingMode = vk::SharingMode::eExclusive; - image_info.initialLayout = vk::ImageLayout::eUndefined; - - vk::ImageFormatProperties properties{}; - RequireVk(name, "backing query", - m_physical_device.getImageFormatProperties( - image_info.format, image_info.imageType, image_info.tiling, - image_info.usage, image_info.flags, &properties), - "mutable extended-usage BGRA sRGB backing"); - - vk::Image image = nullptr; - RequireVk(name, "backing", - m_device.createImage(&image_info, nullptr, &image), - "vkCreateImage"); - vk::MemoryRequirements requirements{}; - m_device.getImageMemoryRequirements(image, &requirements); - u32 memory_type = 0; - Require(name, "backing", - FindMemoryType(requirements.memoryTypeBits, - vk::MemoryPropertyFlagBits::eDeviceLocal, - &memory_type) || - FindMemoryType(requirements.memoryTypeBits, {}, &memory_type), - "no memory type for mutable render target"); - vk::MemoryAllocateInfo allocation{}; - allocation.sType = vk::StructureType::eMemoryAllocateInfo; - allocation.allocationSize = requirements.size; - allocation.memoryTypeIndex = memory_type; - vk::DeviceMemory memory = nullptr; - RequireVk(name, "backing", - m_device.allocateMemory(&allocation, nullptr, &memory), - "vkAllocateMemory"); - RequireVk(name, "backing", m_device.bindImageMemory(image, memory, 0), - "vkBindImageMemory"); - - vk::ImageViewUsageCreateInfo view_usage{}; - view_usage.sType = vk::StructureType::eImageViewUsageCreateInfo; - view_usage.usage = vk::ImageUsageFlagBits::eStorage; - vk::ImageViewCreateInfo view_info{}; - view_info.sType = vk::StructureType::eImageViewCreateInfo; - view_info.pNext = &view_usage; - view_info.image = image; - view_info.viewType = vk::ImageViewType::e2D; - view_info.format = vk::Format::eR8G8B8A8Unorm; - view_info.components = { - vk::ComponentSwizzle::eIdentity, vk::ComponentSwizzle::eIdentity, - vk::ComponentSwizzle::eIdentity, vk::ComponentSwizzle::eIdentity}; - view_info.subresourceRange.aspectMask = vk::ImageAspectFlagBits::eColor; - view_info.subresourceRange.levelCount = 1; - view_info.subresourceRange.layerCount = 1; - vk::ImageView storage_view = nullptr; - RequireVk(name, "storage view", - m_device.createImageView(&view_info, nullptr, &storage_view), - "vk::Device::createImageView(RGBA UNORM storage)"); - - m_device.destroyImageView(storage_view, nullptr); - m_device.destroyImage(image, nullptr); - m_device.freeMemory(memory, nullptr); - std::printf("[host] %-32s ok (backing=%d view=%d)\n", name, - static_cast(image_info.format), - static_cast(view_info.format)); - } - - void CheckRenderTargetViewCache() { - constexpr const char *name = "RenderTargetViewCache"; - vk::ImageCreateInfo image_info{}; - image_info.sType = vk::StructureType::eImageCreateInfo; - image_info.flags = vk::ImageCreateFlagBits::eMutableFormat; - image_info.imageType = vk::ImageType::e2D; - image_info.format = vk::Format::eR8G8B8A8Unorm; - image_info.extent = {8, 8, 1}; - image_info.mipLevels = 2; - image_info.arrayLayers = 2; - image_info.samples = vk::SampleCountFlagBits::e1; - image_info.tiling = vk::ImageTiling::eOptimal; - image_info.usage = vk::ImageUsageFlagBits::eColorAttachment | - vk::ImageUsageFlagBits::eSampled | - vk::ImageUsageFlagBits::eStorage; - image_info.sharingMode = vk::SharingMode::eExclusive; - - vk::Image image_handle = nullptr; - RequireVk(name, "image", - m_device.createImage(&image_info, nullptr, &image_handle), - "vkCreateImage"); - vk::MemoryRequirements requirements{}; - m_device.getImageMemoryRequirements(image_handle, &requirements); - u32 memory_type = 0; - Require(name, "memory", - FindMemoryType(requirements.memoryTypeBits, - vk::MemoryPropertyFlagBits::eDeviceLocal, - &memory_type) || - FindMemoryType(requirements.memoryTypeBits, {}, &memory_type), - "no memory type for sampled render target"); - vk::MemoryAllocateInfo allocation{}; - allocation.sType = vk::StructureType::eMemoryAllocateInfo; - allocation.allocationSize = requirements.size; - allocation.memoryTypeIndex = memory_type; - vk::DeviceMemory memory = nullptr; - RequireVk(name, "memory", - m_device.allocateMemory(&allocation, nullptr, &memory), - "vkAllocateMemory"); - RequireVk(name, "memory", m_device.bindImageMemory(image_handle, memory, 0), - "vkBindImageMemory"); - + void CheckSchedulerTimeline() { EnsureRuntimeContext(); - ResourceMutex resource_mutex; - PageManager page_manager(RejectUnexpectedPageFault, nullptr); - BufferCache buffer_cache(m_runtime_context, page_manager, resource_mutex); - TextureCache texture_cache(m_runtime_context, page_manager, buffer_cache, - resource_mutex); - buffer_cache.SetTextureCache(texture_cache); - RenderTextureVulkanImage image; - image.image = image_handle; - image.format = image_info.format; - image.extent = {image_info.extent.width, image_info.extent.height}; - image.layers = 2; - image.mip_levels = 2; + CommandScheduler scheduler(Renderer(), m_runtime_context); + HW::Context registers{}; + HW::UserConfig user_config{}; + HW::Shader shaders{}; + scheduler.Begin(registers, user_config, shaders); - constexpr auto first_swizzle = DstSel(5, 1, 7, 0); - constexpr auto second_swizzle = DstSel(7, 4, 0, 1); - const auto first = texture_cache.GetSampledColorView( - image, image.format, first_swizzle, 0, 1, vk::ImageViewType::e2D, 0, 1); - const auto first_again = texture_cache.GetSampledColorView( - image, image.format, first_swizzle, 0, 1, vk::ImageViewType::e2D, 0, 1); - const auto second = - texture_cache.GetSampledColorView(image, image.format, second_swizzle, - 0, 1, vk::ImageViewType::e2D, 0, 1); - const auto different_mip = texture_cache.GetSampledColorView( - image, image.format, first_swizzle, 1, 1, vk::ImageViewType::e2D, 0, 1); - const auto different_layer = texture_cache.GetSampledColorView( - image, image.format, first_swizzle, 0, 1, vk::ImageViewType::e2D, 1, 1); - const auto array_view = - texture_cache.GetSampledColorView(image, image.format, first_swizzle, 0, - 1, vk::ImageViewType::e2DArray, 0, 2); - constexpr auto identity = DstSel(4, 5, 6, 7); - const auto native_format = texture_cache.GetSampledColorView( - image, image.format, identity, 0, 1, vk::ImageViewType::e2D, 0, 1); - const auto reinterpreted_format = texture_cache.GetSampledColorView( - image, vk::Format::eR8G8B8A8Uint, identity, 0, 1, - vk::ImageViewType::e2D, 0, 1); - const auto storage = texture_cache.GetRenderTargetStorageView( - image, image.format, 0, 1, vk::ImageViewType::e2D, 0, 1); - const auto storage_again = texture_cache.GetRenderTargetStorageView( - image, image.format, 0, 1, vk::ImageViewType::e2D, 0, 1); - const auto attachment = texture_cache.GetRenderTargetAttachmentView( - image, image.format, 0, 0, 1); - const auto attachment_again = texture_cache.GetRenderTargetAttachmentView( - image, image.format, 0, 0, 1); - const auto attachment_mip = texture_cache.GetRenderTargetAttachmentView( - image, image.format, 1, 0, 1); - const auto attachment_array = texture_cache.GetRenderTargetAttachmentView( - image, image.format, 0, 0, 2); - Require(name, "cache identity", - first != nullptr && first_again == first && second != nullptr && - second != first && different_mip != nullptr && - different_mip != first && different_layer != nullptr && - different_layer != first && array_view != nullptr && - array_view != first && native_format != nullptr && - reinterpreted_format != nullptr && - reinterpreted_format != native_format && storage != nullptr && - storage != native_format && storage_again == storage && - attachment != nullptr && attachment_again == attachment && - attachment != native_format && attachment != storage && - attachment_mip != nullptr && attachment_mip != attachment && - attachment_array != nullptr && attachment_array != attachment && - image.view_cache.views.size() == 11, - "view cache omitted attachment usage, swizzle, format, type, mip, " - "or layer identity"); + const auto first_tick = scheduler.CurrentTick(); + uint32_t completed = 0; + scheduler.DeferOperation([owned = std::make_unique(1), + &completed] { completed = *owned; }); + scheduler.Flush(); + scheduler.Wait(first_tick); + Require("SchedulerTimeline", "first tick", + scheduler.CurrentTick() == first_tick + 1 && + scheduler.IsFree(first_tick) && completed == 1, + "timeline wait did not release its deferred operation"); - ImageViewOps::DestroyViews(image); - Require(name, "view teardown", image.view_cache.views.empty(), - "dynamic render-target views survived DestroyViews"); - m_device.destroyImage(image_handle, nullptr); - m_device.freeMemory(memory, nullptr); - std::printf("[host] %-32s ok\n", name); + const auto second_tick = scheduler.CurrentTick(); + std::atomic priority_in_callback{false}; + scheduler.DeferPriorityOperation([owned = std::make_unique(2), + &completed, &priority_in_callback] { + priority_in_callback = CommandScheduler::InDeferredOperation(); + completed = *owned; + }); + scheduler.Finish(); + scheduler.DrainPriorityOperations(); + Require("SchedulerTimeline", "finish", + scheduler.CurrentTick() == second_tick + 1 && + scheduler.IsFree(second_tick) && completed == 2 && + priority_in_callback.load() && + !CommandScheduler::InDeferredOperation(), + "scheduler completion drain lost its tick or callback context"); + + const auto implicit_flush_tick = scheduler.CurrentTick(); + scheduler.DeferOperation([owned = std::make_unique(3), + &completed] { completed = *owned; }); + scheduler.Wait(implicit_flush_tick); + Require("SchedulerTimeline", "implicit flush", + scheduler.CurrentTick() == implicit_flush_tick + 1 && + scheduler.IsFree(implicit_flush_tick) && completed == 3, + "waiting for the current tick did not flush it"); + + vk::SemaphoreTypeCreateInfo timeline_type{}; + timeline_type.sType = vk::StructureType::eSemaphoreTypeCreateInfo; + timeline_type.semaphoreType = vk::SemaphoreType::eTimeline; + vk::SemaphoreCreateInfo timeline_create{}; + timeline_create.sType = vk::StructureType::eSemaphoreCreateInfo; + timeline_create.pNext = &timeline_type; + vk::Semaphore external_timeline = nullptr; + Require("SchedulerTimeline", "external timeline create", + m_runtime_context.device.createSemaphore(&timeline_create, nullptr, + &external_timeline) == + vk::Result::eSuccess && + external_timeline != nullptr, + "failed to create the external timeline semaphore"); + + constexpr uint64_t external_wait_value = 9; + const auto external_wait_tick = scheduler.CurrentTick(); + SubmitInfo external_wait; + external_wait.AddWait(external_timeline, external_wait_value); + scheduler.Flush(external_wait); + Require("SchedulerTimeline", "external timeline blocked", + !scheduler.IsFree(external_wait_tick), + "submission ignored the external timeline wait value"); + vk::SemaphoreSignalInfo signal_info{}; + signal_info.sType = vk::StructureType::eSemaphoreSignalInfo; + signal_info.semaphore = external_timeline; + signal_info.value = external_wait_value; + Require("SchedulerTimeline", "external timeline signal", + m_runtime_context.device.signalSemaphore(&signal_info) == + vk::Result::eSuccess, + "failed to signal the external timeline semaphore"); + scheduler.Wait(external_wait_tick); + m_runtime_context.device.destroySemaphore(external_timeline, nullptr); + scheduler.Shutdown(); + + CommandScheduler draining(Renderer(), m_runtime_context); + HW::Context drain_registers{}; + HW::UserConfig drain_user_config{}; + HW::Shader drain_shaders{}; + draining.Begin(drain_registers, drain_user_config, drain_shaders); + uint32_t reentrant = 0; + draining.DeferOperation([&draining, &reentrant] { + reentrant++; + draining.DeferOperation([&reentrant] { reentrant++; }); + }); + + std::binary_semaphore priority_entered{0}; + std::binary_semaphore release_priority{0}; + draining.DeferPriorityOperation( + [&draining, &priority_entered, &release_priority] { + priority_entered.release(); + draining.Shutdown(); + release_priority.acquire(); + }); + std::jthread shutdown_thread([&draining] { draining.Shutdown(); }); + priority_entered.acquire(); + std::jthread second_shutdown_thread([&draining] { draining.Shutdown(); }); + + std::atomic concurrent_completed{false}; + std::jthread concurrent_defer([&draining, &concurrent_completed] { + draining.DeferOperation( + [&concurrent_completed] { concurrent_completed = true; }); + }); + release_priority.release(); + shutdown_thread.join(); + second_shutdown_thread.join(); + concurrent_defer.join(); + Require("SchedulerTimeline", "shutdown drain", + reentrant == 2 && concurrent_completed.load(), + "shutdown lost reentrant or concurrent deferred work"); + std::printf("[host] %-32s ok\n", "SchedulerTimeline"); } - void CheckDepthTargetSampledViewCache() { - constexpr const char *name = "DepthTargetSampledViewCache"; - vk::ImageCreateInfo image_info{}; - image_info.sType = vk::StructureType::eImageCreateInfo; - image_info.imageType = vk::ImageType::e2D; - image_info.format = vk::Format::eD32Sfloat; - image_info.extent = {8, 8, 1}; - image_info.mipLevels = 1; - image_info.arrayLayers = 2; - image_info.samples = vk::SampleCountFlagBits::e1; - image_info.tiling = vk::ImageTiling::eOptimal; - image_info.usage = vk::ImageUsageFlagBits::eDepthStencilAttachment | - vk::ImageUsageFlagBits::eSampled; - image_info.sharingMode = vk::SharingMode::eExclusive; - - vk::Image image_handle = nullptr; - RequireVk(name, "image", - m_device.createImage(&image_info, nullptr, &image_handle), - "vkCreateImage"); - vk::MemoryRequirements requirements{}; - m_device.getImageMemoryRequirements(image_handle, &requirements); - u32 memory_type = 0; - Require(name, "memory", - FindMemoryType(requirements.memoryTypeBits, - vk::MemoryPropertyFlagBits::eDeviceLocal, - &memory_type) || - FindMemoryType(requirements.memoryTypeBits, {}, &memory_type), - "no memory type for sampled depth target"); - vk::MemoryAllocateInfo allocation{}; - allocation.sType = vk::StructureType::eMemoryAllocateInfo; - allocation.allocationSize = requirements.size; - allocation.memoryTypeIndex = memory_type; - vk::DeviceMemory memory = nullptr; - RequireVk(name, "memory", - m_device.allocateMemory(&allocation, nullptr, &memory), - "vkAllocateMemory"); - RequireVk(name, "memory", m_device.bindImageMemory(image_handle, memory, 0), - "vkBindImageMemory"); - + void CheckStreamBufferRing() { EnsureRuntimeContext(); - ResourceMutex resource_mutex; - PageManager page_manager(RejectUnexpectedPageFault, nullptr); - BufferCache buffer_cache(m_runtime_context, page_manager, resource_mutex); - TextureCache texture_cache(m_runtime_context, page_manager, buffer_cache, - resource_mutex); - buffer_cache.SetTextureCache(texture_cache); - DepthStencilVulkanImage image; - image.image = image_handle; - image.format = image_info.format; - image.extent = {image_info.extent.width, image_info.extent.height}; - image.layers = image_info.arrayLayers; - image.mip_levels = image_info.mipLevels; + CommandScheduler scheduler(Renderer(), m_runtime_context); + HW::Context registers{}; + HW::UserConfig user_config{}; + HW::Shader shaders{}; + scheduler.Begin(registers, user_config, shaders); + GpuResourceManager resources(m_runtime_context, scheduler); + auto &cache = resources.GetBufferCache(); + constexpr std::array, 4> utilities{{ + {MemoryUsage::Upload, 512ull << 20}, + {MemoryUsage::Stream, 64ull << 20}, + {MemoryUsage::Download, 32ull << 20}, + {MemoryUsage::DeviceLocal, 128ull << 20}, + }}; + std::array handles{}; + for (size_t i = 0; i < utilities.size(); i++) { + auto &utility = cache.GetUtilityBuffer(utilities[i].first); + handles[i] = utility.Handle(); + Require("StreamBufferRing", "utility ownership", + utility.Usage() == utilities[i].first && + utility.Size() == utilities[i].second && + utility.Handle() != nullptr, + "BufferCache utility buffer does not match the fixed layout"); + } + for (size_t i = 0; i < handles.size(); i++) { + for (size_t j = i + 1; j < handles.size(); j++) { + Require("StreamBufferRing", "utility identity", + handles[i] != handles[j], + "utility usages alias the same native buffer"); + } + } + auto *fixed_download = &cache.GetUtilityBuffer(MemoryUsage::Download); + const auto fixed_handle = fixed_download->Handle(); + const auto [oversized_download, oversized_offset] = + fixed_download->Map(fixed_download->Size() + 4, 64, false); + const auto [download_probe, download_probe_offset] = + fixed_download->Map(64, 64, false); + Require("StreamBufferRing", "fixed download utility", + oversized_download == nullptr && oversized_offset == 0 && + download_probe != nullptr && download_probe_offset == 0 && + &cache.GetUtilityBuffer(MemoryUsage::Download) == + fixed_download && + fixed_download->Size() == (32ull << 20) && + fixed_download->Handle() == fixed_handle, + "oversized download replaced or corrupted the fixed shared ring"); + fixed_download->Commit(); - constexpr auto replicated = DstSel(4, 4, 4, 4); - constexpr auto r001 = DstSel(4, 0, 0, 1); - const auto first = texture_cache.GetDepthTargetSampledView( - image, vk::Format::eR32Sfloat, replicated, 0, 1, vk::ImageViewType::e2D, - 0, 1); - const auto first_again = texture_cache.GetDepthTargetSampledView( - image, vk::Format::eR32Sfloat, replicated, 0, 1, vk::ImageViewType::e2D, - 0, 1); - const auto different_swizzle = texture_cache.GetDepthTargetSampledView( - image, vk::Format::eR32Sfloat, r001, 0, 1, vk::ImageViewType::e2D, 0, - 1); - const auto different_layer = texture_cache.GetDepthTargetSampledView( - image, vk::Format::eR32Sfloat, replicated, 0, 1, vk::ImageViewType::e2D, - 1, 1); - const auto array_view = texture_cache.GetDepthTargetSampledView( - image, vk::Format::eR32Sfloat, replicated, 0, 1, - vk::ImageViewType::e2DArray, 0, 2); - const auto attachment = - texture_cache.GetDepthTargetAttachmentView(image, 0, 1); - const auto attachment_again = - texture_cache.GetDepthTargetAttachmentView(image, 0, 1); - const auto attachment_layer = - texture_cache.GetDepthTargetAttachmentView(image, 1, 1); - const auto attachment_array = - texture_cache.GetDepthTargetAttachmentView(image, 0, 2); + std::vector full_stream(64ull << 20, 0x5a); + const auto utility_tick = scheduler.CurrentTick(); + auto full_binding = + cache.UploadTransient(full_stream.data(), full_stream.size(), 16); + const std::array overflow_data{ + 0xa5, 0xa5, 0xa5, 0xa5, 0xa5, 0xa5, 0xa5, 0xa5, + 0xa5, 0xa5, 0xa5, 0xa5, 0xa5, 0xa5, 0xa5, 0xa5}; + auto overflow_binding = + cache.UploadTransient(overflow_data.data(), overflow_data.size(), 16); + auto overflow_owner = std::static_pointer_cast( + overflow_binding.owner); Require( - name, "cache identity", - first != nullptr && first_again == first && - different_swizzle != nullptr && different_swizzle != first && - different_layer != nullptr && different_layer != first && - array_view != nullptr && array_view != first && - attachment != nullptr && attachment_again == attachment && - attachment != first && attachment_layer != nullptr && - attachment_layer != attachment && attachment_array != nullptr && - attachment_array != attachment && - image.view_cache.views.size() == 7, - "depth view cache omitted attachment usage, swizzle, type, or layer " - "identity"); + "StreamBufferRing", "current tick overflow", + full_binding.owner == nullptr && + full_binding.buffer == + cache.GetUtilityBuffer(MemoryUsage::Stream).Handle() && + overflow_owner != nullptr && + overflow_binding.buffer == overflow_owner->Handle() && + overflow_binding.buffer != full_binding.buffer && + overflow_owner->Mapped().size() == overflow_data.size() && + std::memcmp(overflow_owner->Mapped().data(), overflow_data.data(), + overflow_data.size()) == 0 && + scheduler.CurrentTick() == utility_tick, + "current-tick stream overflow submitted partial state or lost bytes"); - ImageViewOps::DestroyViews(image); - Require(name, "view teardown", image.view_cache.views.empty(), - "dynamic depth-target views survived DestroyViews"); - m_device.destroyImage(image_handle, nullptr); - m_device.freeMemory(memory, nullptr); - std::printf("[host] %-32s ok\n", name); + const auto atom = + m_runtime_context.physical_device_properties.limits.nonCoherentAtomSize; + uint64_t policy_size = 17; + uint64_t policy_alignment = 12; + uint64_t coherent_size = policy_size; + uint64_t coherent_alignment = policy_alignment; + Require("StreamBufferRing", "non-coherent reservation policy", + StreamBufferTestAccess::NormalizeReservation( + false, 256, policy_size, policy_alignment) && + policy_size == 256 && policy_alignment == 768 && + StreamBufferTestAccess::NormalizeReservation( + true, 256, coherent_size, coherent_alignment) && + coherent_size == 17 && coherent_alignment == 12, + "non-coherent reservations are not isolated at the atom/texel LCM"); + const auto small_ring_size = std::max(64, atom * 2); + StreamBuffer stream(m_runtime_context, scheduler, MemoryUsage::Upload, + small_ring_size); + const auto first_size = + stream.IsCoherent() ? small_ring_size - 16 : small_ring_size - atom + 1; + constexpr uint64_t blocked_size = 24; + const auto [first, first_offset] = stream.Map(first_size, 16, false); + Require("StreamBufferRing", "first allocation", + first != nullptr && first_offset == 0, + "initial stream allocation failed"); + std::memset(first, 0x5a, static_cast(first_size)); + stream.Commit(); + + const auto [oversize, oversize_offset] = + stream.Map(small_ring_size + 1, 1, false); + Require("StreamBufferRing", "oversize", + oversize == nullptr && oversize_offset == 0, + "oversize allocation was accepted"); + const auto [blocked, blocked_offset] = stream.Map(blocked_size, 1, false); + const auto [blocked_again, blocked_again_offset] = + stream.Map(blocked_size, 1, false); + Require("StreamBufferRing", "transactional failure", + blocked == nullptr && blocked_offset == 0 && + blocked_again == nullptr && blocked_again_offset == 0, + "failed wrap mutated stream state"); + + scheduler.Flush(); + const auto [wrapped, wrapped_offset] = stream.Map(blocked_size, 1); + Require("StreamBufferRing", "watched wrap", + wrapped != nullptr && wrapped_offset == 0, + "completed watch did not release the wrapped allocation"); + stream.Commit(); + + const auto download_alignment = atom / std::gcd(atom, 12) * 12; + const auto download_ring_size = std::max( + 64, download_alignment + std::max(atom, 16)); + StreamBuffer download(m_runtime_context, scheduler, MemoryUsage::Download, + download_ring_size); + const auto [download_data, download_offset] = download.Map(16, 8); + Require("StreamBufferRing", "download allocation", + download_data != nullptr && download_offset == 0, + "download stream is not host visible"); + download.Commit(); + scheduler.Flush(); + const auto [second_download, second_download_offset] = download.Map(16, 12); + Require("StreamBufferRing", "download atom isolation", + second_download != nullptr && + (download.IsCoherent() || + (download_offset % atom == 0 && + second_download_offset % atom == 0 && + second_download_offset >= download_offset + atom)) && + second_download_offset % 12 == 0, + "download reservations share a non-coherent atom or lost caller " + "alignment"); + download.Commit(); + + scheduler.Finish(); + download.Invalidate(download_offset, 16); + std::printf("[host] %-32s ok\n", "StreamBufferRing"); } - void CheckVideoOutSampledViewCache() { - constexpr const char *name = "VideoOutSampledViewCache"; - auto backing = - CreateImage2D(name, 8, 8, vk::Format::eR8G8B8A8Unorm, - vk::ImageUsageFlagBits::eColorAttachment | - vk::ImageUsageFlagBits::eSampled, - {}, 1, vk::ImageLayout::eShaderReadOnlyOptimal); - + void CheckGpuCommandLane() { EnsureRuntimeContext(); - ResourceMutex resource_mutex; - PageManager page_manager(RejectUnexpectedPageFault, nullptr); - BufferCache buffer_cache(m_runtime_context, page_manager, resource_mutex); - TextureCache texture_cache(m_runtime_context, page_manager, buffer_cache, - resource_mutex); - buffer_cache.SetTextureCache(texture_cache); - VideoOutVulkanImage image; - image.image = backing.image; - image.format = backing.format; - image.extent = {backing.width, backing.height}; - image.layers = 1; - image.mip_levels = 1; - image.image_view[VulkanImage::VIEW_DEFAULT] = backing.view; + auto &context = Renderer(); + context.InitializeGpu(nullptr); + auto &gpu = context.GetGpu(); - constexpr auto identity = DstSel(4, 5, 6, 7); - constexpr auto bgra = DstSel(6, 5, 4, 7); - constexpr auto uncommon = DstSel(5, 1, 7, 0); - const auto default_view = texture_cache.GetSampledColorView( - image, image.format, identity, 0, 1, vk::ImageViewType::e2D, 0, 1); - const auto bgra_view = texture_cache.GetSampledColorView( - image, image.format, bgra, 0, 1, vk::ImageViewType::e2D, 0, 1); - const auto uncommon_view = texture_cache.GetSampledColorView( - image, image.format, uncommon, 0, 1, vk::ImageViewType::e2D, 0, 1); - const auto uncommon_again = texture_cache.GetSampledColorView( - image, image.format, uncommon, 0, 1, vk::ImageViewType::e2D, 0, 1); - Require(name, "cache identity", - default_view == backing.view && bgra_view != nullptr && - bgra_view != default_view && uncommon_view != nullptr && - uncommon_view != bgra_view && uncommon_again == uncommon_view && - image.view_cache.views.size() == 2, - "video-out sampled views did not use identity fast path and lazy " - "mappings"); + const auto caller_thread = std::this_thread::get_id(); + std::thread::id gpu_thread; + std::thread::id nested_thread; + uint32_t order = 0; + gpu.SendCommand([&order] { order = 1; }); + gpu.SendCommandSync([&] { + gpu_thread = std::this_thread::get_id(); + Require("GpuCommandLane", "FIFO", order == 1, + "synchronous command overtook an older host command"); + Require("GpuCommandLane", "generic context", + !Gpu::IsCommandProcessorThread() && + Gpu::CurrentCommandProcessor() == nullptr, + "generic host command manufactured a PM4 processor context"); + gpu.SendCommandSync( + [&nested_thread] { nested_thread = std::this_thread::get_id(); }); + order = 2; + }); + gpu.SendCommandSyncWithProcessor([&](CommandProcessor &processor) { + Require( + "GpuCommandLane", "processor context", + Gpu::IsCommandProcessorThread() && + Gpu::CurrentCommandProcessor() == &processor, + "resource command did not receive its explicit processor context"); + }); + Require( + "GpuCommandLane", "dispatch thread", + order == 2 && gpu_thread != caller_thread && + nested_thread == gpu_thread, + "host command did not run on the GPU thread or nested sync deadlocked"); - ImageViewOps::DestroyViews(image); - Require(name, "view teardown", - image.view_cache.views.empty() && - std::all_of(std::begin(image.image_view), - std::end(image.image_view), - [](vk::ImageView view) { return view == nullptr; }), - "dynamic or fixed video-out views survived DestroyViews"); - backing.view = nullptr; - DestroyImage(&backing); + uint32_t label = 0; + uint32_t prefix = 0; + uint32_t suffix = 0; + const auto address = [](const void *value) { + return reinterpret_cast(value); + }; + std::array commands{}; + commands[0] = KYTY_PM4(5, Pm4::IT_WRITE_DATA, 0); + commands[1] = 0; + commands[2] = static_cast(address(&prefix)); + commands[3] = static_cast(address(&prefix) >> 32u); + commands[4] = 11; + commands[5] = KYTY_PM4(7, Pm4::IT_NOP, Pm4::R_WAIT_MEM_32); + commands[6] = static_cast(address(&label)); + commands[7] = static_cast(address(&label) >> 32u); + commands[8] = UINT32_MAX; + commands[9] = 1; + commands[10] = 0x10u | 3u; + commands[12] = KYTY_PM4(5, Pm4::IT_WRITE_DATA, 0); + commands[13] = 0; + commands[14] = static_cast(address(&suffix)); + commands[15] = static_cast(address(&suffix) >> 32u); + commands[16] = 22; + gpu.Submit(commands.data(), static_cast(commands.size()), nullptr, + 0); + + std::binary_semaphore ordered_started{0}; + std::atomic ordered_finished{false}; + uint32_t ordered_suffix = 0; + std::jthread ordered([&] { + ordered_started.release(); + Gpu::SubmissionLock submissions(gpu); + gpu.SendCommandSync([&] { + ordered_suffix = suffix; + ordered_finished = true; + }); + }); + ordered_started.acquire(); + gpu.SendCommandSync([&] { + Require("GpuCommandLane", "ordered barrier", !ordered_finished.load(), + "ordered host command overtook a queued submission"); + label = 1; + }); + ordered.join(); + Require("GpuCommandLane", "ordered completion", + prefix == 11 && suffix == 22 && ordered_suffix == 22 && + ordered_finished.load(), + "submission barrier did not drain prior PM4 work"); + +#if KYTY_PLATFORM == KYTY_PLATFORM_WINDOWS + constexpr uintptr_t fault_base = 0x0000000200500000ull; + constexpr uint64_t fault_size = 0x10000; + int64_t fault_direct_offset = -1; + Require("GpuCommandLane", "processor fault direct allocation", + Libs::LibKernel::Memory::KernelAllocateDirectMemory( + 0, Libs::LibKernel::Memory::KernelGetDirectMemorySize(), + fault_size, fault_size, 0, &fault_direct_offset) == 0, + "processor-fault direct-memory allocation failed"); + void *fault_memory = reinterpret_cast(fault_base); + Require("GpuCommandLane", "processor fault direct mapping", + Libs::LibKernel::Memory::KernelMapDirectMemory( + &fault_memory, fault_size, 0x3, 0x10, fault_direct_offset, + fault_size) == 0, + "processor-fault fixed direct-memory mapping failed"); + Require("GpuCommandLane", "processor fault allocation", + fault_memory == reinterpret_cast(fault_base), + "fixed processor-fault allocation failed"); + auto &resources = context.GetGpuResources(); + resources.MapMemory(fault_base, fault_size, GpuAccess::ReadWrite); + + constexpr uint64_t immediate_dst = fault_base + 0x1000; + constexpr uint64_t immediate_memory_dst = fault_base + 0x2000; + constexpr uint64_t memory_src = fault_base + 0x4000; + constexpr uint64_t memory_copy_dst = fault_base + 0x6000; + constexpr uint64_t memory_dst = fault_base + 0x8000; + constexpr uint64_t l2_copy_dst = fault_base + 0xa000; + constexpr uint64_t clean_cached_fill = fault_base + 0x3000; + constexpr uint64_t clean_cached_copy = fault_base + 0x3008; + constexpr uint64_t clean_cached_source = fault_base + 0x3010; + constexpr uint64_t clean_cached_readback = fault_base + 0xb000; + constexpr std::array source_words{0x11223344u, 0x55667788u}; + constexpr std::array clean_copy_words{0x13579bdfu, + 0x2468ace0u}; + std::memcpy(reinterpret_cast(memory_src), source_words.data(), + sizeof(source_words)); + std::memcpy(reinterpret_cast(clean_cached_source), + clean_copy_words.data(), sizeof(clean_copy_words)); + + std::array dma_commands{}; + size_t dma_cursor = 0; + const auto append_dma = [&](uint8_t dst_sel, uint64_t dst, uint8_t src_sel, + uint64_t src, uint32_t bytes) { + auto *packet = dma_commands.data() + dma_cursor; + dma_cursor += 7; + packet[0] = KYTY_PM4(7, Pm4::IT_DMA_DATA, 0); + packet[1] = (static_cast(dst_sel) << 20u) | + (static_cast(src_sel) << 29u); + packet[2] = static_cast(src); + packet[3] = static_cast(src >> 32u); + packet[4] = static_cast(dst); + packet[5] = static_cast(dst >> 32u); + packet[6] = bytes; + }; + constexpr uint64_t gds_immediate_offset = 0; + constexpr uint64_t gds_memory_offset = 0x10; + constexpr uint32_t immediate_value = 0xa1b2c3d4u; + append_dma(1, gds_immediate_offset, 2, immediate_value, + sizeof(source_words)); + append_dma(0, immediate_dst, 1, gds_immediate_offset, sizeof(source_words)); + append_dma(1, gds_memory_offset, 0, memory_src, sizeof(source_words)); + append_dma(0, memory_dst, 1, gds_memory_offset, sizeof(source_words)); + append_dma(0, immediate_memory_dst, 2, immediate_value, + sizeof(source_words)); + append_dma(0, memory_copy_dst, 0, memory_src, sizeof(source_words)); + append_dma(3, l2_copy_dst, 3, memory_src, sizeof(source_words)); + Require("GpuCommandLane", "DMA_DATA packet assembly", + dma_cursor == dma_commands.size(), + "DMA_DATA GDS packet stream has the wrong size"); + gpu.Submit(dma_commands.data(), static_cast(dma_commands.size()), + nullptr, 0); + { + Gpu::SubmissionLock submissions(gpu); + } + constexpr uint32_t clean_fill_value = 0xdecafbad; + gpu.SendCommandSyncWithProcessor([&](CommandProcessor &) { + auto &buffer_cache = resources.GetBufferCache(); + Require("GpuCommandLane", "clean cached setup", + buffer_cache.HasPageOverlap(clean_cached_fill, + sizeof(source_words)) && + buffer_cache.HasGpuDirtyBytes(immediate_dst, + sizeof(source_words)), + "DMA did not establish a cached page with a dirty sibling"); + + buffer_cache.FillBuffer(clean_cached_fill, sizeof(source_words), + clean_fill_value); + buffer_cache.CopyBuffer(clean_cached_copy, clean_cached_source, + sizeof(clean_copy_words)); + Require("GpuCommandLane", "clean cached DMA ownership", + !buffer_cache.HasGpuDirtyBytes(clean_cached_fill, + sizeof(source_words)) && + !buffer_cache.HasGpuDirtyBytes(clean_cached_copy, + sizeof(clean_copy_words)) && + buffer_cache.HasGpuDirtyBytes(immediate_dst, + sizeof(source_words)), + "clean cached DMA acquired ownership or released a byte-disjoint " + "dirty sibling"); + + constexpr uint64_t gds_clean_mirror = 0x20; + buffer_cache.CopyBuffer(gds_clean_mirror, clean_cached_fill, + 2 * sizeof(source_words), true, false); + buffer_cache.CopyBuffer(clean_cached_readback, gds_clean_mirror, + 2 * sizeof(source_words), false, true); + }); + Require("GpuCommandLane", "DMA_DATA immediate GDS readback", + resources.HandleFault(PageFaultAccess::Read, immediate_dst), + "immediate-to-GDS-to-memory copy did not publish GPU bytes"); + Require("GpuCommandLane", "DMA_DATA memory GDS readback", + resources.HandleFault(PageFaultAccess::Read, memory_dst), + "memory-to-GDS-to-memory copy did not publish GPU bytes"); + Require("GpuCommandLane", "DMA_DATA immediate memory readback", + resources.HandleFault(PageFaultAccess::Read, immediate_memory_dst), + "immediate-to-memory copy did not publish GPU bytes"); + Require("GpuCommandLane", "DMA_DATA memory copy readback", + resources.HandleFault(PageFaultAccess::Read, memory_copy_dst), + "memory-to-memory copy did not publish GPU bytes"); + Require("GpuCommandLane", "DMA_DATA L2 readback", + resources.HandleFault(PageFaultAccess::Read, l2_copy_dst), + "MemoryUsingL2 copy did not publish GPU bytes"); + Require("GpuCommandLane", "clean cached mirror readback", + resources.HandleFault(PageFaultAccess::Read, clean_cached_readback), + "clean host DMA was not reflected in the cached buffer"); + std::array immediate_words{}; + std::array copied_words{}; + std::array immediate_memory_words{}; + std::array memory_copy_words{}; + std::array l2_copy_words{}; + std::array clean_cached_words{}; + std::memcpy(immediate_words.data(), + reinterpret_cast(immediate_dst), + sizeof(immediate_words)); + std::memcpy(copied_words.data(), reinterpret_cast(memory_dst), + sizeof(copied_words)); + std::memcpy(immediate_memory_words.data(), + reinterpret_cast(immediate_memory_dst), + sizeof(immediate_memory_words)); + std::memcpy(memory_copy_words.data(), + reinterpret_cast(memory_copy_dst), + sizeof(memory_copy_words)); + std::memcpy(l2_copy_words.data(), + reinterpret_cast(l2_copy_dst), + sizeof(l2_copy_words)); + std::memcpy(clean_cached_words.data(), + reinterpret_cast(clean_cached_readback), + sizeof(clean_cached_words)); + Require("GpuCommandLane", "DMA_DATA immediate GDS contents", + immediate_words == + std::array{immediate_value, immediate_value}, + "immediate-to-GDS-to-memory bytes do not match"); + Require("GpuCommandLane", "DMA_DATA memory GDS contents", + copied_words == source_words, + "memory-to-GDS-to-memory bytes do not match"); + Require("GpuCommandLane", "DMA_DATA immediate memory contents", + immediate_memory_words == + std::array{immediate_value, immediate_value}, + "immediate-to-memory bytes do not match"); + Require("GpuCommandLane", "DMA_DATA memory copy contents", + memory_copy_words == source_words, + "memory-to-memory bytes do not match"); + Require("GpuCommandLane", "DMA_DATA L2 contents", + l2_copy_words == source_words, "MemoryUsingL2 bytes do not match"); + Require("GpuCommandLane", "clean cached DMA contents", + clean_cached_words == std::array{clean_fill_value, + clean_fill_value, + clean_copy_words[0], + clean_copy_words[1]}, + "host-memory fill/copy did not update the clean cached mirror"); + + gpu.SendCommandSyncWithProcessor([&](CommandProcessor &processor) { + Require("GpuCommandLane", "processor fault context", + Gpu::CurrentCommandProcessor() == &processor, + "processor resource test lost its command context"); + resources.PrepareHostWrite(fault_base, sizeof(uint32_t)); + }); + resources.UnmapMemory(fault_base, fault_size, GpuAccess::ReadWrite); + Require("GpuCommandLane", "processor fault unmap", + Libs::LibKernel::Memory::KernelMunmap(fault_base, fault_size) == 0, + "processor-fault direct-memory mapping release failed"); + Require("GpuCommandLane", "processor fault free", + Libs::LibKernel::Memory::KernelReleaseDirectMemory( + fault_direct_offset, fault_size) == 0, + "processor-fault direct-memory allocation release failed"); +#endif + + std::binary_semaphore command_entered{0}; + std::binary_semaphore release_command{0}; + std::atomic shutdown_complete{false}; + std::atomic submission_lane_entered{false}; + gpu.SendCommand([&] { + command_entered.release(); + release_command.acquire(); + gpu.Done(); + submission_lane_entered = true; + }); + command_entered.acquire(); + std::jthread shutdown_thread([&] { + gpu.Shutdown(); + shutdown_complete = true; + }); + while (!gpu.IsStopping()) { + std::this_thread::yield(); + } + Require("GpuCommandLane", "owned shutdown", !shutdown_complete.load(), + "GPU owner returned before an in-flight command completed"); + release_command.release(); + shutdown_thread.join(); + Require( + "GpuCommandLane", "owned shutdown completion", + shutdown_complete.load() && submission_lane_entered.load(), + "GPU owner did not drain a command that entered the submission lane"); + context.ShutdownGpu(); + std::printf("[host] %-32s ok\n", "GpuCommandLane"); + } + + void CheckUnifiedImageViewCache() { + EnsureRuntimeContext(); + constexpr const char *name = "UnifiedImageViewCache"; + CommandScheduler scheduler(Renderer(), m_runtime_context); + + ImageInfo color_info{}; + color_info.pixel_format = vk::Format::eR8G8B8A8Unorm; + color_info.guest_format = + Prospero::GpuEnumValue(Prospero::BufferFormat::k8_8_8_8UNorm); + color_info.type = Prospero::ImageType::kColor2D; + color_info.extent = {8, 8, 1}; + color_info.resources = {2, 2}; + color_info.pitch = 8; + color_info.bytes_per_block = 4; + color_info.samples = 1; + color_info.tile_mode = Prospero::GpuEnumValue(Prospero::TileMode::kLinear); + color_info.mip_layout[0] = {0, 512, 8, 8}; + color_info.mip_layout[1] = {512, 128, 4, 4}; + + Libs::Graphics::Image color(m_runtime_context, scheduler, color_info); + const vk::ComponentMapping identity{}; + const vk::ComponentMapping bgra{ + vk::ComponentSwizzle::eB, vk::ComponentSwizzle::eG, + vk::ComponentSwizzle::eR, vk::ComponentSwizzle::eA}; + + ImageViewInfo sampled{}; + sampled.format = color_info.pixel_format; + sampled.type = vk::ImageViewType::e2D; + sampled.aspect = vk::ImageAspectFlagBits::eColor; + sampled.base_level = 0; + sampled.level_count = 1; + sampled.base_layer = 0; + sampled.layer_count = 1; + sampled.mapping = identity; + sampled.usage = vk::ImageUsageFlagBits::eSampled; + + const auto first = color.FindView(sampled); + const auto first_again = color.FindView(sampled); + auto swizzled_info = sampled; + swizzled_info.mapping = bgra; + const auto swizzled = color.FindView(swizzled_info); + auto mip_info = sampled; + mip_info.base_level = 1; + const auto mip = color.FindView(mip_info); + auto layer_info = sampled; + layer_info.base_layer = 1; + const auto layer = color.FindView(layer_info); + auto array_info = sampled; + array_info.type = vk::ImageViewType::e2DArray; + array_info.layer_count = 2; + const auto array = color.FindView(array_info); + auto reinterpreted_info = sampled; + reinterpreted_info.format = vk::Format::eR8G8B8A8Uint; + const auto reinterpreted = color.FindView(reinterpreted_info); + auto storage_info = sampled; + storage_info.usage = vk::ImageUsageFlagBits::eStorage; + const auto storage = color.FindView(storage_info); + auto attachment_info = sampled; + attachment_info.usage = vk::ImageUsageFlagBits::eColorAttachment; + const auto attachment = color.FindView(attachment_info); + + Require(name, "color views", + first != nullptr && first_again == first && swizzled != nullptr && + swizzled != first && mip != nullptr && mip != first && + layer != nullptr && layer != first && array != nullptr && + array != first && reinterpreted != nullptr && + reinterpreted != first && storage != nullptr && + storage != first && attachment == first && + color.views.views.size() == 7, + "dynamic view identity omitted mapping, format, mip, layer, type, " + "or storage usage"); + + ImageInfo depth_info{}; + depth_info.pixel_format = vk::Format::eD32Sfloat; + depth_info.guest_format = + Prospero::GpuEnumValue(Prospero::BufferFormat::k32Float); + depth_info.type = Prospero::ImageType::kColor2D; + depth_info.extent = {8, 8, 1}; + depth_info.resources = {1, 2}; + depth_info.pitch = 8; + depth_info.bytes_per_block = 4; + depth_info.samples = 1; + depth_info.tile_mode = Prospero::GpuEnumValue(Prospero::TileMode::kDepth); + depth_info.mip_layout[0] = {0, 512, 8, 8}; + + Libs::Graphics::Image depth(m_runtime_context, scheduler, depth_info); + ImageViewInfo depth_sampled{}; + depth_sampled.format = depth_info.pixel_format; + depth_sampled.type = vk::ImageViewType::e2D; + depth_sampled.aspect = vk::ImageAspectFlagBits::eDepth; + depth_sampled.base_level = 0; + depth_sampled.level_count = 1; + depth_sampled.base_layer = 0; + depth_sampled.layer_count = 1; + depth_sampled.mapping = {vk::ComponentSwizzle::eR, vk::ComponentSwizzle::eR, + vk::ComponentSwizzle::eR, + vk::ComponentSwizzle::eR}; + depth_sampled.usage = vk::ImageUsageFlagBits::eSampled; + const auto depth_first = depth.FindView(depth_sampled); + auto depth_alias_info = depth_sampled; + depth_alias_info.format = vk::Format::eR32Uint; + const auto depth_alias = depth.FindView(depth_alias_info); + const auto depth_again = depth.FindView(depth_sampled); + auto depth_array_info = depth_sampled; + depth_array_info.type = vk::ImageViewType::e2DArray; + depth_array_info.layer_count = 2; + const auto depth_array = depth.FindView(depth_array_info); + auto depth_attachment_info = depth_array_info; + depth_attachment_info.mapping = {}; + depth_attachment_info.usage = + vk::ImageUsageFlagBits::eDepthStencilAttachment; + const auto depth_attachment = depth.FindView(depth_attachment_info); + Require(name, "depth views", + depth_first != nullptr && depth_again == depth_first && + depth_alias == depth_first && + depth_array != nullptr && depth_array != depth_first && + depth_attachment != nullptr && + depth_attachment != depth_array && + depth.views.views.size() == 3, + "unified depth view cache lost sampled/attachment identity"); + Require(name, "role-free backing", + color.backing.image_type == vk::ImageType::e2D && + color.backing.layers == 2 && + depth.backing.image_type == vk::ImageType::e2D && + ImageViewOps::FormatsCompatible(vk::Format::eR8G8B8A8Unorm, + vk::Format::eR8G8B8A8Uint) && + !ImageViewOps::FormatsCompatible(vk::Format::eD32Sfloat, + vk::Format::eR32Sfloat), + "role-free image backing or host format compatibility diverged"); std::printf("[host] %-32s ok\n", name); } #if KYTY_PLATFORM == KYTY_PLATFORM_WINDOWS - void CheckQueryRegionImageClassification() { - constexpr const char *name = "QueryRegionImageClassification"; - constexpr uintptr_t base = 0x0000000200100000ull; - constexpr uint64_t allocation_size = 0x4000; - auto *memory = static_cast( - VirtualAlloc(reinterpret_cast(base), allocation_size, - MEM_RESERVE | MEM_COMMIT, PAGE_READWRITE)); - Require(name, "allocation", memory == reinterpret_cast(base), - "fixed VirtualAlloc failed"); - std::memset(memory, 0, allocation_size); + void CheckBufferCacheDirtyGarbageCollection() { + constexpr const char *name = "BufferCacheDirtyGarbageCollection"; + constexpr uintptr_t base = 0x0000000200700000ull; + constexpr uint64_t allocation_size = 0x2400000; + constexpr uint64_t allocation_alignment = 0x10000; + constexpr uint64_t first_offset = 0x100; + constexpr uint64_t second_offset = 0x200; + constexpr uint64_t clean_offset = 0x300; + constexpr uint64_t unmap_offset = 0x4100; + constexpr uint64_t partial_unmap_offset = 0x8100; + constexpr uint64_t partial_unmap_survivor_offset = 0xc100; + constexpr uint64_t ring_fault_first_offset = 0x2100; + constexpr uint64_t ring_fault_second_offset = 0x2110; + constexpr uint32_t first_value = 0x10203040u; + constexpr uint32_t second_value = 0x50607080u; + constexpr uint32_t first_stale = 0x0badf00du; + constexpr uint32_t second_stale = 0xdeadbeefu; + constexpr uint32_t clean_value = 0xaabbccddu; + constexpr uint32_t unmap_value = 0x91a2b3c4u; + constexpr uint32_t partial_unmap_value = 0x62738495u; + constexpr uint32_t partial_unmap_survivor_value = 0xa6b7c8d9u; + constexpr uint32_t ring_fault_first_value = 0x0a1b2c3du; + constexpr uint32_t ring_fault_second_value = 0x4e5f6071u; EnsureRuntimeContext(); - ResourceMutex resource_mutex; - PageManager page_manager(RejectUnexpectedPageFault, nullptr); - BufferCache buffer_cache(m_runtime_context, page_manager, resource_mutex); - TextureCache texture_cache(m_runtime_context, page_manager, buffer_cache, - resource_mutex); - buffer_cache.SetTextureCache(texture_cache); - page_manager.OnGpuMap(base, allocation_size); + CommandScheduler scheduler(Renderer(), m_runtime_context); + HW::Context registers{}; + HW::UserConfig user_config{}; + HW::Shader shaders{}; + scheduler.Begin(registers, user_config, shaders); + Gpu gpu(Renderer()); - constexpr auto format = - Prospero::GpuEnumValue(Prospero::BufferFormat::k8_8_8_8UNorm); - constexpr auto linear = Prospero::GpuEnumValue(Prospero::TileMode::kLinear); - ImageInfo sampled_info{}; - sampled_info.address = base + 0x100; - sampled_info.format = format; - sampled_info.width = 4; - sampled_info.height = 4; - sampled_info.pitch = - TileGetTexturePitch(format, sampled_info.width, 1, linear); - sampled_info.tile = linear; - sampled_info.type = Prospero::GpuEnumValue(Prospero::ImageType::kColor2D); - TileSizeAlign sampled_size{}; - TileGetTextureTotalSize(format, sampled_info.width, sampled_info.height, - sampled_info.depth, sampled_info.pitch, - sampled_info.levels, linear, false, sampled_size); - sampled_info.size = sampled_size.size; - Require(name, "sampled fixture", - sampled_info.size != 0 && sampled_size.align != 0 && - sampled_info.size <= 0x600 && - sampled_info.address % sampled_size.align == 0, - "sampled image does not fit the intended sub-page fixture"); - - RenderTargetInfo target_info{}; - target_info.address = base + 0x1100; - target_info.size = 0x100; - target_info.format = vk::Format::eR8G8B8A8Unorm; - target_info.width = 4; - target_info.height = 4; - target_info.pitch = 4; - target_info.bytes_per_element = 4; - target_info.tile_mode = linear; + int64_t direct_offset = -1; + Require(name, "direct allocation", + Libs::LibKernel::Memory::KernelAllocateDirectMemory( + 0, Libs::LibKernel::Memory::KernelGetDirectMemorySize(), + allocation_size, allocation_alignment, 0, &direct_offset) == 0, + "dirty-GC direct-memory allocation failed"); + void *mapped = reinterpret_cast(base); + Require(name, "direct mapping", + Libs::LibKernel::Memory::KernelMapDirectMemory( + &mapped, allocation_size, 0x3, 0x10, direct_offset, + allocation_alignment) == 0 && + mapped == reinterpret_cast(base), + "dirty-GC fixed direct-memory mapping failed"); + auto *memory = static_cast(mapped); + std::memcpy(memory + first_offset, &first_stale, sizeof(first_stale)); + std::memcpy(memory + second_offset, &second_stale, sizeof(second_stale)); + std::memcpy(memory + clean_offset, &clean_value, sizeof(clean_value)); { - CommandBuffer command; - (void)texture_cache.FindTexture(command, sampled_info, false); - auto& target = texture_cache.FindRenderTarget(command, target_info); + GpuResourceManager resources(m_runtime_context, scheduler); + resources.SetGpu(&gpu); + auto &cache = resources.GetBufferCache(); + resources.MapMemory(base, allocation_size, GpuAccess::ReadWrite); - const auto sampled_bytes = - texture_cache.QueryRegion(sampled_info.address + 0x20, 0x20); - const auto sampled_page = texture_cache.QueryRegion(base + 0x800, 0x20); - Require(name, "sampled classification", - sampled_bytes.image_pages && sampled_bytes.image_bytes && - !sampled_bytes.gpu_image_bytes && - !sampled_bytes.non_sampled_pages && - sampled_page.image_pages && !sampled_page.image_bytes && - !sampled_page.gpu_image_bytes && - !sampled_page.non_sampled_pages, - "sampled image page and byte ownership were not separated"); + const auto MarkGpuWrite = [&](uint64_t address, uint64_t size) { + auto allocation = + cache.ObtainBuffer(scheduler.Current(), address, size, true, false); + Require(name, "dirty allocation", allocation.owner != nullptr, + "dirty-GC buffer allocation failed"); + scheduler.Current().RetainResourceUntilFence(allocation.owner); + }; + MarkGpuWrite(base + first_offset, sizeof(first_value)); + MarkGpuWrite(base + second_offset, sizeof(second_value)); + cache.FillBuffer(base + first_offset, sizeof(first_value), first_value); + cache.FillBuffer(base + second_offset, sizeof(second_value), + second_value); + cache.DiscardGpuDirtyBytes(base + first_offset, sizeof(first_value)); + Require( + name, "exact dirty discard", + !cache.HasGpuDirtyBytes(base + first_offset, sizeof(first_value)) && + cache.HasGpuDirtyBytes(base + second_offset, + sizeof(second_value)) && + cache.IsRegionGpuModified(base, TRACKER_PAGE_SIZE), + "discarding one exact range released its dirty page sibling"); + cache.DiscardGpuDirtyBytes(base + second_offset, sizeof(second_value)); + Require( + name, "last dirty discard", + !cache.HasGpuDirtyBytes(base + second_offset, sizeof(second_value)) && + !cache.IsRegionGpuModified(base, TRACKER_PAGE_SIZE), + "discarding the last exact range retained page ownership"); + MarkGpuWrite(base + first_offset, sizeof(first_value)); + MarkGpuWrite(base + second_offset, sizeof(second_value)); + cache.FillBuffer(base + first_offset, sizeof(first_value), first_value); + cache.FillBuffer(base + second_offset, sizeof(second_value), + second_value); + auto &download = BufferCacheTestAccess::DownloadBuffer(cache); + auto *fixed_download = &download; + const auto fixed_download_handle = download.Handle(); + auto [ring_tail, ring_tail_offset] = + download.Map(download.Size() - sizeof(uint32_t), 4); + Require(name, "fault-ring tail reservation", + ring_tail != nullptr && ring_tail_offset == 0, + "failed to position the reusable download ring near wrap"); + download.Commit(); + MarkGpuWrite(base + ring_fault_first_offset, + sizeof(ring_fault_first_value)); + MarkGpuWrite(base + ring_fault_second_offset, + sizeof(ring_fault_second_value)); + cache.FillBuffer(base + ring_fault_first_offset, + sizeof(ring_fault_first_value), ring_fault_first_value); + cache.FillBuffer(base + ring_fault_second_offset, + sizeof(ring_fault_second_value), + ring_fault_second_value); + Require(name, "fault-ring wrapped batch", + resources.HandleFault(PageFaultAccess::Read, + base + ring_fault_first_offset), + "fault readback could not wrap a live download-ring tick"); + uint64_t expected_packing_offset = 128; + uint64_t expected_packing_alignment = 1; + if (!download.IsCoherent()) { + Require(name, "fault-ring atom policy", + StreamBufferTestAccess::NormalizeReservation( + false, + m_runtime_context.physical_device_properties.limits + .nonCoherentAtomSize, + expected_packing_offset, expected_packing_alignment), + "failed to calculate non-coherent fault-ring stride"); + } + auto [packing_probe, packing_offset] = download.Map(1, 1, false); + Require(name, "adjacent download reservation stride", + packing_probe != nullptr && + packing_offset == expected_packing_offset, + "adjacent fault downloads did not reserve an atom-safe stride"); + download.Commit(); + uint32_t ring_fault_first_backing = 0; + uint32_t ring_fault_second_backing = 0; + std::memcpy(&ring_fault_first_backing, memory + ring_fault_first_offset, + sizeof(ring_fault_first_backing)); + std::memcpy(&ring_fault_second_backing, memory + ring_fault_second_offset, + sizeof(ring_fault_second_backing)); + Require(name, "fault-ring wrapped contents", + ring_fault_first_backing == ring_fault_first_value && + ring_fault_second_backing == ring_fault_second_value && + &BufferCacheTestAccess::DownloadBuffer(cache) == + fixed_download && + download.Handle() == fixed_download_handle && + download.Size() == (32ull << 20), + "wrapped fault batch published incorrect disjoint ranges"); - const auto target_bytes = - texture_cache.QueryRegion(target_info.address + 0x20, 0x20); - const auto target_page = - texture_cache.QueryRegion(target_info.address + 0x400, 0x20); - Require(name, "non-sampled classification", - target_bytes.image_pages && target_bytes.image_bytes && - !target_bytes.gpu_image_bytes && - target_bytes.non_sampled_pages && target_page.image_pages && - !target_page.image_bytes && !target_page.gpu_image_bytes && - target_page.non_sampled_pages, - "non-sampled image page and byte ownership were not separated"); + for (uint32_t tick = 0; tick < 160; tick++) { + cache.RunGarbageCollector(); + } + Require(name, "age before pressure", + cache.HasPageOverlap(base, allocation_size), + "buffer was reclaimed without memory pressure"); + BufferCacheTestAccess::SetGarbageCollectionThresholds( + cache, 0, std::numeric_limits::max()); + const auto gc_submission_tick = scheduler.CurrentTick(); + cache.RunGarbageCollector(); + Require(name, "dirty retirement", + !cache.HasPageOverlap(base, allocation_size), + "aged GPU-dirty buffer survived pressured collection"); - texture_cache.MarkGpuWritten(target); - const auto gpu_bytes = - texture_cache.QueryRegion(target_info.address + 0x20, 0x20); - const auto gpu_page = - texture_cache.QueryRegion(target_info.address + 0x400, 0x20); - Require(name, "GPU ownership", - gpu_bytes.image_pages && gpu_bytes.image_bytes && - gpu_bytes.gpu_image_bytes && gpu_bytes.non_sampled_pages && - gpu_page.image_pages && !gpu_page.image_bytes && - !gpu_page.gpu_image_bytes && gpu_page.non_sampled_pages, - "GPU ownership escaped the target's exact byte range"); + uint32_t first_before_completion = 0; + uint32_t second_before_completion = 0; + Libs::LibKernel::Memory::TryReadBacking(base + first_offset, + &first_before_completion, + sizeof(first_before_completion)); + Libs::LibKernel::Memory::TryReadBacking(base + second_offset, + &second_before_completion, + sizeof(second_before_completion)); + Require(name, "deferred dirty retirement", + first_before_completion == first_stale && + second_before_completion == second_stale && + scheduler.CurrentTick() == gc_submission_tick, + "buffer GC synchronously submitted or published its download"); + Require(name, "publication-gated CPU fault", + resources.HandleFault(PageFaultAccess::Read, base + first_offset), + "CPU fault did not wait for the deferred buffer publication"); - texture_cache.UnmapMemory(base, allocation_size); - page_manager.OnGpuUnmap(base, allocation_size); + uint32_t first_backing = 0; + uint32_t second_backing = 0; + uint32_t clean_backing = 0; + std::memcpy(&first_backing, memory + first_offset, sizeof(first_backing)); + std::memcpy(&second_backing, memory + second_offset, + sizeof(second_backing)); + std::memcpy(&clean_backing, memory + clean_offset, sizeof(clean_backing)); + Require(name, "downloaded contents", + first_backing == first_value && second_backing == second_value && + clean_backing == clean_value, + "dirty GC did not publish exact buffer ranges"); + + auto unmap_allocation = + cache.ObtainBuffer(scheduler.Current(), base + unmap_offset, + sizeof(unmap_value), true, false); + Require(name, "direct-unmap allocation", + unmap_allocation.owner != nullptr, + "direct-unmap buffer allocation failed"); + scheduler.Current().RetainResourceUntilFence(unmap_allocation.owner); + cache.FillBuffer(base + unmap_offset, sizeof(unmap_value), unmap_value); + cache.UnmapMemory(base + 0x4000, 0x4000); + uint32_t unmap_backing = 0; + std::memcpy(&unmap_backing, memory + unmap_offset, sizeof(unmap_backing)); + Require(name, "direct-unmap contents", + unmap_backing == unmap_value && + !cache.HasPageOverlap(base + 0x4000, 0x4000), + "direct dirty unmap cleared tracking before publishing bytes"); + + auto partial_unmap_allocation = cache.ObtainBuffer( + scheduler.Current(), base + 0x8000, 0x8000, true, false); + Require(name, "partial-unmap allocation", + partial_unmap_allocation.owner != nullptr, + "cross-page cached buffer allocation failed"); + scheduler.Current().RetainResourceUntilFence( + partial_unmap_allocation.owner); + cache.FillBuffer(base + partial_unmap_offset, sizeof(partial_unmap_value), + partial_unmap_value); + cache.FillBuffer(base + partial_unmap_survivor_offset, + sizeof(partial_unmap_survivor_value), + partial_unmap_survivor_value); + cache.UnmapMemory(base + 0x8000, 0x4000); + auto [survivor, survivor_offset] = + cache.ObtainBufferForImageWrite(base + partial_unmap_survivor_offset, + sizeof(partial_unmap_survivor_value)); + Require(name, "partial-unmap survivor", survivor != nullptr, + "still-mapped cached-buffer remainder could not be recreated"); + scheduler.Current().RetainResourceUntilFence(survivor); + auto partial_unmap_readback = + CreateHostBuffer(name, sizeof(partial_unmap_survivor_value), + vk::BufferUsageFlagBits::eTransferDst, {0}); + const vk::BufferCopy survivor_copy{survivor_offset, 0, + sizeof(partial_unmap_survivor_value)}; + scheduler.Current().Handle().copyBuffer( + survivor->Handle(), partial_unmap_readback.buffer, 1, &survivor_copy); + vk::BufferMemoryBarrier survivor_barrier{}; + survivor_barrier.sType = vk::StructureType::eBufferMemoryBarrier; + survivor_barrier.srcAccessMask = vk::AccessFlagBits::eTransferWrite; + survivor_barrier.dstAccessMask = vk::AccessFlagBits::eHostRead; + survivor_barrier.srcQueueFamilyIndex = VK_QUEUE_FAMILY_IGNORED; + survivor_barrier.dstQueueFamilyIndex = VK_QUEUE_FAMILY_IGNORED; + survivor_barrier.buffer = partial_unmap_readback.buffer; + survivor_barrier.size = partial_unmap_readback.size; + scheduler.Current().Handle().pipelineBarrier( + vk::PipelineStageFlagBits::eTransfer, + vk::PipelineStageFlagBits::eHost, {}, 0, nullptr, 1, + &survivor_barrier, 0, nullptr); + scheduler.Finish(); + Require(name, "partial-unmap survivor contents", + ReadBuffer(name, partial_unmap_readback, 1) == + std::vector{partial_unmap_survivor_value}, + "partial unmap recreated its still-mapped remainder from " + "uninitialized native bytes"); + DestroyBuffer(&partial_unmap_readback); + + constexpr uint64_t large_offset = 0x10000; + constexpr uint64_t large_size = 31ull * 1024 * 1024; + constexpr uint32_t large_value = 0x5aa55aa5u; + constexpr uint32_t large_stale = 0x12345678u; + std::memcpy(memory + large_offset, &large_stale, sizeof(large_stale)); + std::memcpy(memory + large_offset + large_size - sizeof(large_stale), + &large_stale, sizeof(large_stale)); + auto large_allocation = cache.ObtainBuffer( + scheduler.Current(), base + large_offset, large_size, true, false); + Require(name, "near-capacity dirty allocation", + large_allocation.owner != nullptr, + "failed to allocate the near-capacity dirty native buffer"); + scheduler.Current().RetainResourceUntilFence(large_allocation.owner); + cache.FillBuffer(base + large_offset, large_size, large_value); + for (uint32_t tick = 0; tick <= 160; tick++) { + cache.RunGarbageCollector(); + } + Require(name, "near-capacity deferred retirement", + !cache.HasPageOverlap(base + large_offset, large_size), + "near-capacity dirty Buffer survived pressured collection"); + uint32_t large_before_completion = 0; + Libs::LibKernel::Memory::TryReadBacking(base + large_offset, + &large_before_completion, + sizeof(large_before_completion)); + Require(name, "near-capacity backing remained deferred", + large_before_completion == large_stale, + "near-capacity Buffer GC published before its scheduler tick"); + const auto large_image_source = + cache.ObtainBufferForImage(base + large_offset, sizeof(large_value)); + Require(name, "fixed download during Buffer publication", + large_image_source.buffer != nullptr && + &BufferCacheTestAccess::DownloadBuffer(cache) == + fixed_download && + fixed_download->Handle() == fixed_download_handle && + fixed_download->Size() == (32ull << 20), + "image acquisition replaced the shared Buffer download stream"); + uint32_t large_first = 0; + uint32_t large_last = 0; + Libs::LibKernel::Memory::TryReadBacking(base + large_offset, &large_first, + sizeof(large_first)); + Libs::LibKernel::Memory::TryReadBacking(base + large_offset + large_size - + sizeof(large_last), + &large_last, sizeof(large_last)); + Require(name, "near-capacity Buffer publication contents", + large_first == large_value && large_last == large_value, + "near-capacity Buffer GC did not publish its complete transfer"); + + constexpr uint64_t grouped_first_offset = 0x10000; + constexpr uint64_t grouped_second_offset = 0x1200000; + constexpr uint64_t grouped_owner_size = 17ull * 1024 * 1024; + constexpr uint32_t grouped_first_value = 0x1122aabbu; + constexpr uint32_t grouped_second_value = 0x3344ccddu; + constexpr uint32_t grouped_stale = 0; + static_assert(grouped_owner_size < (32ull << 20) && + grouped_owner_size * 2 > (32ull << 20)); + Libs::LibKernel::Memory::WriteBacking( + base + grouped_first_offset, &grouped_stale, sizeof(grouped_stale)); + Libs::LibKernel::Memory::WriteBacking( + base + grouped_second_offset, &grouped_stale, sizeof(grouped_stale)); + auto grouped_first = + cache.ObtainBuffer(scheduler.Current(), base + grouped_first_offset, + grouped_owner_size, true, false); + auto grouped_second = + cache.ObtainBuffer(scheduler.Current(), base + grouped_second_offset, + grouped_owner_size, true, false); + Require(name, "per-owner GC allocation", + grouped_first.owner != nullptr && + grouped_second.owner != nullptr && + grouped_first.owner != grouped_second.owner, + "disjoint GC candidates merged into one Buffer owner"); + scheduler.Current().RetainResourceUntilFence(grouped_first.owner); + scheduler.Current().RetainResourceUntilFence(grouped_second.owner); + cache.FillBuffer(base + grouped_first_offset, grouped_owner_size, + grouped_first_value); + cache.FillBuffer(base + grouped_second_offset, grouped_owner_size, + grouped_second_value); + for (uint32_t tick = 0; tick <= 160; tick++) { + cache.RunGarbageCollector(); + } + Require(name, "per-owner fixed-ring retirement", + !cache.HasPageOverlap(base + grouped_first_offset, + grouped_owner_size) && + !cache.HasPageOverlap(base + grouped_second_offset, + grouped_owner_size) && + &BufferCacheTestAccess::DownloadBuffer(cache) == + fixed_download && + fixed_download->Handle() == fixed_download_handle, + "GC aggregated disjoint retirees or replaced the download ring"); + scheduler.FinishCurrent(); + uint32_t grouped_first_backing = 0; + uint32_t grouped_second_backing = 0; + Libs::LibKernel::Memory::TryReadBacking(base + grouped_first_offset, + &grouped_first_backing, + sizeof(grouped_first_backing)); + Libs::LibKernel::Memory::TryReadBacking(base + grouped_second_offset, + &grouped_second_backing, + sizeof(grouped_second_backing)); + Require(name, "per-owner fixed-ring contents", + grouped_first_backing == grouped_first_value && + grouped_second_backing == grouped_second_value, + "per-owner GC transfers lost data while wrapping the fixed ring"); + + constexpr uint64_t disjoint_owner_offset = 0x2140000; + constexpr uint64_t disjoint_owner_size = 0x8000; + constexpr uint64_t disjoint_dirty_offset = disjoint_owner_offset + 0x100; + constexpr uint64_t disjoint_new_owner_offset = + disjoint_owner_offset + 0x5000; + constexpr uint32_t disjoint_value = 0x89abcdefu; + constexpr uint32_t disjoint_stale = 0x76543210u; + std::memcpy(memory + disjoint_dirty_offset, &disjoint_stale, + sizeof(disjoint_stale)); + auto wide_owner = + cache.ObtainBuffer(scheduler.Current(), base + disjoint_owner_offset, + disjoint_owner_size, false, true); + Require(name, "disjoint retirement owner", wide_owner.owner != nullptr, + "failed to create the clean cross-page retirement owner"); + scheduler.Current().RetainResourceUntilFence(wide_owner.owner); + auto dirty_alias = + cache.ObtainBuffer(scheduler.Current(), base + disjoint_dirty_offset, + sizeof(disjoint_value), true, false); + if (dirty_alias.owner != nullptr) { + scheduler.Current().RetainResourceUntilFence(dirty_alias.owner); + } + cache.FillBuffer(base + disjoint_dirty_offset, sizeof(disjoint_value), + disjoint_value); + for (uint32_t tick = 0; tick <= 160; tick++) { + cache.RunGarbageCollector(); + } + Require(name, "disjoint deferred retirement", + !cache.HasPageOverlap(base + disjoint_owner_offset, + disjoint_owner_size), + "cross-page owner survived pressured collection"); + uint32_t disjoint_before_unmap = 0; + Libs::LibKernel::Memory::TryReadBacking(base + disjoint_dirty_offset, + &disjoint_before_unmap, + sizeof(disjoint_before_unmap)); + Require( + name, "disjoint retirement remained deferred", + disjoint_before_unmap == disjoint_stale, + "whole-owner Buffer publication completed before synchronization"); + cache.UnmapMemory(base + disjoint_owner_offset + 0x4000, 0x4000); + uint32_t disjoint_after_unmap = 0; + Libs::LibKernel::Memory::TryReadBacking(base + disjoint_dirty_offset, + &disjoint_after_unmap, + sizeof(disjoint_after_unmap)); + Require(name, "disjoint publication-gated unmap", + disjoint_after_unmap == disjoint_value && + !cache.HasGpuDirtyBytes(base + disjoint_dirty_offset, + sizeof(disjoint_value)), + "disjoint unmap did not wait for the pending whole-owner " + "publication"); + auto disjoint_new_owner = cache.ObtainBuffer( + scheduler.Current(), base + disjoint_new_owner_offset, + sizeof(disjoint_value), true, false); + Require(name, "disjoint publication-gated reacquire", + disjoint_new_owner.owner != nullptr, + "disjoint acquisition did not wait for whole-owner " + "publication"); + scheduler.Current().RetainResourceUntilFence(disjoint_new_owner.owner); + uint32_t disjoint_backing = 0; + Libs::LibKernel::Memory::TryReadBacking(base + disjoint_dirty_offset, + &disjoint_backing, + sizeof(disjoint_backing)); + Require(name, "disjoint retirement contents", + disjoint_backing == disjoint_value, + "old retirement callback lost dirty bytes or retained ownership " + "after a disjoint reacquire"); + cache.DiscardGpuDirtyBytes(base + disjoint_new_owner_offset, + sizeof(disjoint_value)); + + constexpr uint64_t reacquire_owner_offset = 0x2200000; + constexpr uint64_t reacquire_owner_size = 0x8000; + constexpr uint64_t reacquire_dirty_offset = + reacquire_owner_offset + 0x100; + constexpr uint64_t reacquire_disjoint_offset = + reacquire_owner_offset + 0x5000; + constexpr uint32_t reacquire_value = 0xc001d00du; + constexpr uint32_t reacquire_stale = 0x0badf00du; + std::memcpy(memory + reacquire_dirty_offset, &reacquire_stale, + sizeof(reacquire_stale)); + auto reacquire_owner = + cache.ObtainBuffer(scheduler.Current(), base + reacquire_owner_offset, + reacquire_owner_size, false, true); + Require(name, "reacquire retirement owner", + reacquire_owner.owner != nullptr, + "failed to create a fresh cross-page retirement owner"); + scheduler.Current().RetainResourceUntilFence(reacquire_owner.owner); + auto reacquire_dirty = + cache.ObtainBuffer(scheduler.Current(), base + reacquire_dirty_offset, + sizeof(reacquire_value), true, false); + if (reacquire_dirty.owner != nullptr) { + scheduler.Current().RetainResourceUntilFence(reacquire_dirty.owner); + } + cache.FillBuffer(base + reacquire_dirty_offset, sizeof(reacquire_value), + reacquire_value); + for (uint32_t tick = 0; tick <= 160; tick++) { + cache.RunGarbageCollector(); + } + Require(name, "reacquire deferred retirement", + !cache.HasPageOverlap(base + reacquire_owner_offset, + reacquire_owner_size), + "fresh cross-page owner survived pressured collection"); + uint32_t reacquire_before = 0; + Libs::LibKernel::Memory::TryReadBacking(base + reacquire_dirty_offset, + &reacquire_before, + sizeof(reacquire_before)); + Require(name, "reacquire publication remained deferred", + reacquire_before == reacquire_stale, + "fresh whole-owner publication completed before reacquisition"); + auto reacquired = cache.ObtainBuffer( + scheduler.Current(), base + reacquire_disjoint_offset, + sizeof(reacquire_value), true, false); + Require(name, "independent disjoint publication-gated reacquire", + reacquired.owner != nullptr, + "clean disjoint-half acquisition did not synchronize the " + "pending whole-owner publication"); + scheduler.Current().RetainResourceUntilFence(reacquired.owner); + uint32_t reacquire_after = 0; + Libs::LibKernel::Memory::TryReadBacking(base + reacquire_dirty_offset, + &reacquire_after, + sizeof(reacquire_after)); + Require(name, "independent disjoint reacquire contents", + reacquire_after == reacquire_value && + !cache.HasGpuDirtyBytes(base + reacquire_dirty_offset, + sizeof(reacquire_value)), + "disjoint-half acquisition failed to publish the retired " + "owner's dirty prefix"); + cache.DiscardGpuDirtyBytes(base + reacquire_disjoint_offset, + sizeof(reacquire_value)); + + resources.SetGpu(nullptr); + resources.UnmapMemory(base, allocation_size, GpuAccess::ReadWrite); + scheduler.Finish(); } - Require(name, "free", VirtualFree(memory, 0, MEM_RELEASE) != 0, - "VirtualFree failed"); + gpu.Shutdown(); + Require(name, "unmap direct backing", + Libs::LibKernel::Memory::KernelMunmap(base, allocation_size) == 0, + "dirty-GC direct-memory mapping release failed"); + Require(name, "release direct backing", + Libs::LibKernel::Memory::KernelReleaseDirectMemory( + direct_offset, allocation_size) == 0, + "dirty-GC direct-memory allocation release failed"); + std::printf("[host] %-32s ok\n", name); + } + + void CheckUnifiedTextureCacheFlow() { + constexpr const char *name = "UnifiedTextureCacheFlow"; + constexpr uintptr_t base = 0x0000000200600000ull; + constexpr uint64_t allocation_size = 0x2800000; + constexpr uint64_t allocation_alignment = 0x200000; + EnsureRuntimeContext(); + CommandScheduler scheduler(Renderer(), m_runtime_context); + HW::Context registers{}; + HW::UserConfig user_config{}; + HW::Shader shaders{}; + scheduler.Begin(registers, user_config, shaders); + Gpu gpu(Renderer()); + int64_t direct_offset = -1; + Require(name, "direct allocation", + Libs::LibKernel::Memory::KernelAllocateDirectMemory( + 0, Libs::LibKernel::Memory::KernelGetDirectMemorySize(), + allocation_size, allocation_alignment, 0, &direct_offset) == 0, + "cache test direct-memory allocation failed"); + void *mapped = reinterpret_cast(base); + Require(name, "direct mapping", + Libs::LibKernel::Memory::KernelMapDirectMemory( + &mapped, allocation_size, 0x3, 0x10, direct_offset, + allocation_alignment) == 0 && + mapped == reinterpret_cast(base), + "fixed cache direct-memory mapping failed"); + auto *memory = static_cast(mapped); + const uint32_t initial = 0x44332211u; + std::memcpy(memory, &initial, sizeof(initial)); + + { + GpuResourceManager resources(m_runtime_context, scheduler); + resources.SetGpu(&gpu); + auto &texture_cache = resources.GetTextureCache(); + const auto [narrow_download, narrow_download_offset] = + TextureCacheTestAccess::MapDownload(texture_cache, 4, 4); + const auto [wide_download, wide_download_offset] = + TextureCacheTestAccess::MapDownload(texture_cache, 1, 16); + Require(name, "image download alignment", + narrow_download != nullptr && narrow_download_offset % 4 == 0 && + wide_download != nullptr && wide_download_offset % 16 == 0, + "wide/block image readback was not aligned to its texel block"); + resources.MapMemory(base, allocation_size, GpuAccess::ReadWrite); + + ImageDesc sampled{}; + sampled.type = BindingType::Texture; + sampled.info.data = {base, sizeof(initial)}; + sampled.info.pixel_format = vk::Format::eR8G8B8A8Srgb; + sampled.info.guest_format = + Prospero::GpuEnumValue(Prospero::BufferFormat::k8_8_8_8Srgb); + sampled.info.type = Prospero::ImageType::kColor2D; + sampled.info.extent = {1, 1, 1}; + sampled.info.resources = {1, 1}; + sampled.info.pitch = 1; + sampled.info.bytes_per_block = 4; + sampled.info.samples = 1; + sampled.info.tile_mode = + Prospero::GpuEnumValue(Prospero::TileMode::kLinear); + sampled.info.mip_layout[0] = {0, 4, 1, 1}; + sampled.view_info.format = sampled.info.pixel_format; + sampled.view_info.type = vk::ImageViewType::e2D; + sampled.view_info.aspect = vk::ImageAspectFlagBits::eColor; + sampled.view_info.usage = vk::ImageUsageFlagBits::eSampled; + + auto &command = scheduler.Current(); + auto first_desc = sampled; + const auto first = texture_cache.FindImage(first_desc); + auto repeated_desc = sampled; + const auto repeated = texture_cache.FindImage(repeated_desc); + auto compatible_desc = sampled; + compatible_desc.info.pixel_format = vk::Format::eR8G8B8A8Uint; + compatible_desc.info.guest_format = + Prospero::GpuEnumValue(Prospero::BufferFormat::k8_8_8_8UInt); + compatible_desc.view_info.format = compatible_desc.info.pixel_format; + const auto compatible = texture_cache.FindImage(compatible_desc); + Require(name, "normalized FindImage", + first && repeated == first && compatible == first && + texture_cache.GetImage(first).info.pixel_format == + vk::Format::eR8G8B8A8Srgb, + "registered compatible backing did not reuse one ImageId"); + + auto &image = texture_cache.GetImage(first); + constexpr uint32_t final_sampled_value = 0x88776655u; + Require(name, "sampled write between discovery and acquisition", + resources.HandleFault(PageFaultAccess::Write, + sampled.info.data.address), + "sampled image did not accept a CPU update after discovery"); + std::memcpy(memory, &final_sampled_value, sizeof(final_sampled_value)); + Require(name, "sampled final-acquisition precondition", + image.IsCpuDirty(), + "the between-phase CPU write did not dirty the discovered image"); + const auto first_view = texture_cache.FindTexture(first, first_desc); + const auto repeated_view = + texture_cache.FindTexture(repeated, repeated_desc); + auto mapped_view_info = sampled.view_info; + mapped_view_info.mapping.r = vk::ComponentSwizzle::eB; + mapped_view_info.mapping.b = vk::ComponentSwizzle::eR; + const auto mapped_view = image.FindView(mapped_view_info); + Require( + name, "dynamic views", + first_view != nullptr && repeated_view == first_view && + mapped_view != nullptr && mapped_view != first_view && + image.views.views.size() == 2 && !image.usage.texture && + !image.IsGpuModified() && !image.IsCpuDirty() && + !TextureCacheTestAccess::PendingDownload(texture_cache, first), + "TextureCache::FindTexture did not preserve sampled view " + "identity, final refresh, and cache-only ownership"); + + ImageInfo chain = sampled.info; + chain.data = {0x10000, 0x8000}; + chain.extent = {8, 8, 1}; + chain.resources = {2, 4}; + chain.pitch = 8; + chain.mip_layout[0] = {0, 0x4000, 8, 8}; + chain.mip_layout[1] = {0x4000, 0x4000, 4, 4}; + ImageInfo subresource = chain; + subresource.data = {0x16000, 0x1000}; + subresource.extent = {4, 4, 1}; + subresource.resources = {1, 1}; + subresource.pitch = 4; + subresource.mip_layout[0] = {0, 0x1000, 4, 4}; + const auto mip = subresource.MipOf(chain); + Require(name, "overlap resolution", + mip == 1 && subresource.SliceOf(chain, mip) == 2 && + ImageRangeOverlaps(chain.data, subresource.data) && + ImagePageRangesOverlap(chain.data, subresource.data), + "normalized mip/slice overlap did not resolve"); + + auto exact_desc = compatible_desc; + const auto exact = texture_cache.FindImage(exact_desc, true); + Require(name, "exact-format coexistence", + exact && exact != first && + TextureCacheTestAccess::Contains(texture_cache, first) && + texture_cache.GetImage(first).info.pixel_format == + sampled.info.pixel_format && + texture_cache.GetImage(exact).info.pixel_format == + compatible_desc.info.pixel_format, + "exact-format lookup replaced or synchronized its compatible " + "cache record"); + auto compatible_after_exact_desc = compatible_desc; + const auto compatible_after_exact = + texture_cache.FindImage(compatible_after_exact_desc); + Require(name, "last compatible format winner", + compatible_after_exact == exact, + "non-exact lookup did not retain the last compatible " + "registration"); + + auto null_desc = sampled; + null_desc.info.data = {}; + auto null_repeat_desc = null_desc; + const auto null_image = texture_cache.FindImage(null_desc); + const auto null_repeat = texture_cache.FindImage(null_repeat_desc); + Require(name, "null image", + null_image && null_repeat == null_image && null_image != exact, + "typed null-image lookup was not stable"); + + auto MakeLinearDesc = [&](uint64_t address, uint64_t size, + vk::Format format, uint32_t guest_format, + Prospero::ImageType type, vk::Extent3D extent, + uint32_t layers, uint32_t bytes_per_block, + uint32_t samples) { + ImageDesc desc{}; + desc.type = BindingType::Texture; + desc.info.data = {address, size}; + desc.info.pixel_format = format; + desc.info.guest_format = guest_format; + desc.info.type = type; + desc.info.extent = extent; + desc.info.resources = {1, layers}; + desc.info.pitch = extent.width; + desc.info.bytes_per_block = bytes_per_block; + desc.info.samples = samples; + desc.info.tile_mode = + Prospero::GpuEnumValue(Prospero::TileMode::kLinear); + desc.info.mip_layout[0] = {0, size, extent.width, extent.height}; + desc.view_info.format = format; + desc.view_info.type = type == Prospero::ImageType::kColor3D + ? vk::ImageViewType::e3D + : layers > 1 ? vk::ImageViewType::e2DArray + : vk::ImageViewType::e2D; + desc.view_info.aspect = vk::ImageAspectFlagBits::eColor; + desc.view_info.layer_count = layers; + desc.view_info.usage = vk::ImageUsageFlagBits::eSampled; + return desc; + }; + const auto HostReadBarrier = [&](vk::Buffer buffer, uint64_t size, + vk::PipelineStageFlags source_stage, + vk::AccessFlags source_access) { + vk::BufferMemoryBarrier barrier{}; + barrier.sType = vk::StructureType::eBufferMemoryBarrier; + barrier.srcAccessMask = source_access; + barrier.dstAccessMask = vk::AccessFlagBits::eHostRead; + barrier.srcQueueFamilyIndex = VK_QUEUE_FAMILY_IGNORED; + barrier.dstQueueFamilyIndex = VK_QUEUE_FAMILY_IGNORED; + barrier.buffer = buffer; + barrier.offset = 0; + barrier.size = size; + scheduler.Current().Handle().pipelineBarrier( + source_stage, vk::PipelineStageFlagBits::eHost, {}, 0, nullptr, 1, + &barrier, 0, nullptr); + }; + const std::array volume_values{0x10203040u, 0x50607080u}; + std::memcpy(memory + 0x1000, volume_values.data(), sizeof(volume_values)); + auto array_desc = MakeLinearDesc( + base + 0x1000, sizeof(volume_values), vk::Format::eR32Uint, + Prospero::GpuEnumValue(Prospero::BufferFormat::k32UInt), + Prospero::ImageType::kColor2D, {1, 1, 1}, 2, 4, 1); + const auto array_image = texture_cache.FindImage(array_desc); + texture_cache.MarkGpuWritten(array_image); + auto volume_desc = array_desc; + volume_desc.info.type = Prospero::ImageType::kColor3D; + volume_desc.info.extent = {1, 1, 2}; + volume_desc.info.resources.layers = 1; + volume_desc.view_info.type = vk::ImageViewType::e3D; + volume_desc.view_info.layer_count = 1; + const auto volume_image = texture_cache.FindImage(volume_desc); + Require(name, "2D-array/3D overlap copy", + volume_image && volume_image != array_image && + texture_cache.GetImage(volume_image).backing.image_type == + vk::ImageType::e3D && + texture_cache.GetImage(volume_image).IsGpuModified(), + "array-to-volume expansion lost slices or GPU ownership"); + + auto native_array_info = array_desc.info; + native_array_info.data = {}; + auto native_volume_info = volume_desc.info; + native_volume_info.data = {}; + Libs::Graphics::Image native_array(m_runtime_context, scheduler, + native_array_info); + Libs::Graphics::Image native_volume(m_runtime_context, scheduler, + native_volume_info); + native_volume.CopyImage(native_array); + native_array.CopyImage(native_volume); + Require( + name, "Image-owned 2D-array/3D copy state", + native_volume.backing.state.layout == + vk::ImageLayout::eTransferSrcOptimal && + native_volume.backing.state.access_mask == + vk::AccessFlagBits2::eTransferRead && + native_array.backing.state.layout == vk::ImageLayout::eGeneral && + native_array.backing.state.access_mask == + (vk::AccessFlagBits2::eShaderRead | + vk::AccessFlagBits2::eTransferRead), + "Image::CopyImage did not retain pinned source/destination states"); + + ImageInfo resolve_source_info{}; + resolve_source_info.pixel_format = vk::Format::eR8G8B8A8Unorm; + resolve_source_info.guest_format = + Prospero::GpuEnumValue(Prospero::BufferFormat::k8_8_8_8UNorm); + resolve_source_info.type = Prospero::ImageType::kColor2D; + resolve_source_info.extent = {4, 4, 1}; + resolve_source_info.resources = {1, 1}; + resolve_source_info.pitch = 4; + resolve_source_info.bytes_per_block = 4; + resolve_source_info.samples = 2; + resolve_source_info.tile_mode = + Prospero::GpuEnumValue(Prospero::TileMode::kRenderTarget); + ImageInfo resolve_destination_info = resolve_source_info; + resolve_destination_info.type = Prospero::ImageType::kColor2D; + resolve_destination_info.samples = 1; + Libs::Graphics::Image resolve_source(m_runtime_context, scheduler, + resolve_source_info); + Libs::Graphics::Image resolve_destination(m_runtime_context, scheduler, + resolve_destination_info); + resolve_destination.Resolve(resolve_source, {}, {}); + Require( + name, "Image-owned MSAA resolve", + resolve_source.backing.state.layout == + vk::ImageLayout::eTransferSrcOptimal && + resolve_destination.backing.state.layout == + vk::ImageLayout::eTransferDstOptimal, + "Image::Resolve did not issue the MSAA-to-single-sample transfer"); + Libs::Graphics::Image copy_destination(m_runtime_context, scheduler, + resolve_destination_info); + copy_destination.Resolve(resolve_destination, {}, {}); + Require(name, "Image-owned single-sample resolve copy", + resolve_destination.backing.state.layout == + vk::ImageLayout::eTransferSrcOptimal && + copy_destination.backing.state.layout == + vk::ImageLayout::eTransferDstOptimal, + "Image::Resolve did not retain the single-sample copy path"); + + const std::array multisample_source{0x4000u, 0xc000u}; + std::memcpy(memory + 0x2000, &multisample_source, + sizeof(multisample_source)); + auto color_desc = MakeLinearDesc( + base + 0x2000, sizeof(multisample_source), vk::Format::eR16G16Unorm, + Prospero::GpuEnumValue(Prospero::BufferFormat::k16_16UNorm), + Prospero::ImageType::kColor2D, {1, 1, 1}, 1, 4, 1); + const auto color_image = texture_cache.FindImage(color_desc); + texture_cache.MarkGpuWritten(color_image); + const std::array refreshed_multisample_source{0x2000u, + 0xe000u}; + Require(name, "unequal-sample source CPU write", + resources.HandleFault(PageFaultAccess::Write, + color_desc.info.data.address), + "color-to-MS-depth source did not accept a CPU update"); + std::memcpy(memory + 0x2000, refreshed_multisample_source.data(), + sizeof(refreshed_multisample_source)); + constexpr uint64_t ms_stencil_offset = 0x80000; + constexpr uint64_t ms_stencil_size = 0x10000; + auto [ms_stencil_owner, ms_stencil_owner_offset] = + resources.GetBufferCache().ObtainBufferForImageWrite( + base + ms_stencil_offset, ms_stencil_size); + (void)ms_stencil_owner_offset; + Require(name, "MS stencil buffer allocation", ms_stencil_owner != nullptr, + "failed to create an unequal-sample stencil source"); + command.RetainResourceUntilFence(ms_stencil_owner); + resources.GetBufferCache().FillBuffer(base + ms_stencil_offset, + ms_stencil_size, 0x41414141u); + auto ms_depth_desc = color_desc; + ms_depth_desc.type = BindingType::DepthTarget; + ms_depth_desc.info.stencil = {base + ms_stencil_offset, ms_stencil_size}; + ms_depth_desc.info.pixel_format = vk::Format::eD24UnormS8Uint; + ms_depth_desc.info.guest_format = + Prospero::GpuEnumValue(Prospero::BufferFormat::k16UNorm); + ms_depth_desc.info.bytes_per_block = 2; + ms_depth_desc.info.samples = 2; + ms_depth_desc.info.type = Prospero::ImageType::kColor2D; + ms_depth_desc.view_info.format = vk::Format::eD24UnormS8Uint; + ms_depth_desc.view_info.aspect = + vk::ImageAspectFlagBits::eDepth | vk::ImageAspectFlagBits::eStencil; + ms_depth_desc.view_info.usage = + vk::ImageUsageFlagBits::eDepthStencilAttachment; + const auto ms_depth_image = texture_cache.FindImage(ms_depth_desc); + constexpr uint64_t ms_htile_address = base + 0x2600000; + constexpr uint64_t ms_htile_size = 0x10000; + ms_depth_desc.info.metadata.range = {ms_htile_address, ms_htile_size}; + ms_depth_desc.info.metadata.kind = ImageMetadataKind::Htile; + const auto ms_depth_with_htile = texture_cache.FindImage(ms_depth_desc); + const auto ms_depth_view = + texture_cache.FindDepthTarget(ms_depth_with_htile, ms_depth_desc); + auto sampled_ms_depth = ms_depth_desc; + sampled_ms_depth.type = BindingType::Texture; + sampled_ms_depth.info.metadata = {}; + sampled_ms_depth.view_info.usage = vk::ImageUsageFlagBits::eSampled; + const auto sampled_ms_depth_image = + texture_cache.FindImage(sampled_ms_depth); + Require( + name, "unequal-sample depth overlap", + ms_depth_image && ms_depth_with_htile == ms_depth_image && + sampled_ms_depth_image == ms_depth_image && + ms_depth_image != color_image && ms_depth_view != nullptr && + texture_cache.GetImage(ms_depth_image).backing.samples == 2 && + texture_cache.GetImage(ms_depth_image).backing.state.layout == + vk::ImageLayout::eDepthStencilAttachmentOptimal && + texture_cache.GetImage(ms_depth_image).IsGpuModified() && + texture_cache.GetImage(ms_depth_image).info.htile_clear_mask == + 0 && + texture_cache.GetImage(ms_depth_image).info.metadata.range == + ms_depth_desc.info.metadata.range && + texture_cache.IsMeta(ms_htile_address) && + !texture_cache.IsMetaCleared(ms_htile_address, 0) && + !resources.GetBufferCache().HasGpuDirtyBytes( + base + ms_stencil_offset, ms_stencil_size) && + !texture_cache + .QueryRegion(base + ms_stencil_offset, ms_stencil_size) + .gpu_image_bytes, + "unequal-sample overlap did not run the color-to-MS-depth pass " + "without manufacturing stencil ownership"); + auto &oversized_ms = texture_cache.GetImage(ms_depth_image); + const auto ms_data_size = oversized_ms.info.data.size; + oversized_ms.info.data.size = (32ull << 20) + 4; + const bool oversized_ms_readback = + !TextureCacheTestAccess::TryDownload(texture_cache, ms_depth_image); + const bool oversized_ms_mirror = !texture_cache.SynchronizeImageToBuffer( + ms_depth_desc.info.data.address, ms_data_size); + oversized_ms.info.data.size = ms_data_size; + Require(name, "oversized multisample download rejection", + oversized_ms_readback && oversized_ms_mirror && + oversized_ms.IsGpuModified() && + !oversized_ms.IsBufferModified() && + !resources.GetBufferCache().HasGpuDirtyBytes( + ms_depth_desc.info.data.address, ms_data_size), + "an unsupported multisample source reserved download storage or " + "published Buffer ownership"); + + const std::array multisample_source_4x{0x0000u, 0x4000u, + 0x8000u, 0xffffu}; + std::memcpy(memory + 0x7000, multisample_source_4x.data(), + sizeof(multisample_source_4x)); + auto color_desc_4x = MakeLinearDesc( + base + 0x7000, sizeof(multisample_source_4x), + vk::Format::eR16G16B16A16Unorm, + Prospero::GpuEnumValue(Prospero::BufferFormat::k16_16_16_16UNorm), + Prospero::ImageType::kColor2D, {1, 1, 1}, 1, 8, 1); + const auto color_image_4x = texture_cache.FindImage(color_desc_4x); + auto ms_depth_desc_4x = color_desc_4x; + ms_depth_desc_4x.type = BindingType::DepthTarget; + ms_depth_desc_4x.info.pixel_format = vk::Format::eD16Unorm; + ms_depth_desc_4x.info.guest_format = + Prospero::GpuEnumValue(Prospero::BufferFormat::k16UNorm); + ms_depth_desc_4x.info.bytes_per_block = 2; + ms_depth_desc_4x.info.samples = 4; + ms_depth_desc_4x.info.type = Prospero::ImageType::kColor2D; + ms_depth_desc_4x.view_info.format = vk::Format::eD16Unorm; + ms_depth_desc_4x.view_info.aspect = vk::ImageAspectFlagBits::eDepth; + ms_depth_desc_4x.view_info.usage = + vk::ImageUsageFlagBits::eDepthStencilAttachment; + const auto ms_depth_image_4x = texture_cache.FindImage(ms_depth_desc_4x); + Require(name, "four-sample depth overlap", + ms_depth_image_4x && ms_depth_image_4x != color_image_4x && + texture_cache.GetImage(ms_depth_image_4x).backing.samples == + 4, + "four-sample color packing did not produce a depth image"); + + const auto ms_depth_backing = + texture_cache.GetImage(ms_depth_image).backing.image; + const auto first_stencil_association = texture_cache.FindImageFromRange( + base + ms_stencil_offset, ms_stencil_size, false); + Require( + name, "stencil association shape", + first_stencil_association && + first_stencil_association != ms_depth_image && + texture_cache.GetImage(first_stencil_association) + .info.pixel_format == vk::Format::eUndefined && + texture_cache.GetImage(first_stencil_association).backing.image == + nullptr && + texture_cache.GetImage(first_stencil_association).depth_id == + ms_depth_image, + "stencil address did not create a lightweight depth association"); + Require( + name, "stencil association download rejection", + !TextureCacheTestAccess::TryDownload(texture_cache, + first_stencil_association), + "a lightweight stencil association entered image download planning"); + + auto exact_ms_depth_alias = ms_depth_desc; + exact_ms_depth_alias.type = BindingType::Texture; + exact_ms_depth_alias.info.stencil = {}; + exact_ms_depth_alias.info.pixel_format = vk::Format::eR8G8B8A8Unorm; + exact_ms_depth_alias.info.guest_format = + Prospero::GpuEnumValue(Prospero::BufferFormat::k8_8_8_8UNorm); + exact_ms_depth_alias.info.bytes_per_block = 4; + exact_ms_depth_alias.view_info.format = vk::Format::eR8G8B8A8Unorm; + exact_ms_depth_alias.view_info.aspect = vk::ImageAspectFlagBits::eColor; + exact_ms_depth_alias.view_info.usage = vk::ImageUsageFlagBits::eSampled; + const auto exact_ms_depth_image = + texture_cache.FindImage(exact_ms_depth_alias, true); + Require( + name, "exact multisample depth/stencil coexistence", + exact_ms_depth_image && exact_ms_depth_image != ms_depth_image && + TextureCacheTestAccess::Contains(texture_cache, ms_depth_image) && + texture_cache.GetImage(ms_depth_image).backing.image == + ms_depth_backing && + texture_cache.GetImage(first_stencil_association).depth_id == + ms_depth_image, + "exact multisample depth/stencil lookup replaced the old image " + "or stole its stencil association"); + + constexpr uint64_t second_stencil_offset = 0x90000; + auto switched_ms_depth = ms_depth_desc; + switched_ms_depth.info.stencil = {base + second_stencil_offset, + ms_stencil_size}; + const auto switched_ms_depth_id = + texture_cache.FindImage(switched_ms_depth); + TextureCacheTestAccess::AssociateStencil( + texture_cache, switched_ms_depth_id, switched_ms_depth.info.stencil); + const auto second_stencil_association = texture_cache.FindImageFromRange( + base + second_stencil_offset, ms_stencil_size, false); + Require(name, "stencil association switch", + switched_ms_depth_id == ms_depth_image && + texture_cache.GetImage(switched_ms_depth_id).backing.image == + ms_depth_backing && + texture_cache.FindImageFromRange(base + ms_stencil_offset, + ms_stencil_size, false) == + first_stencil_association && + texture_cache.GetImage(first_stencil_association).depth_id == + ms_depth_image && + second_stencil_association && + texture_cache.GetImage(second_stencil_association).depth_id == + ms_depth_image, + "changing stencil address recreated depth or discarded an " + "existing lightweight association"); + + constexpr uint64_t added_stencil_depth_offset = 0x60000; + constexpr uint64_t added_stencil_offset = 0x70000; + constexpr uint64_t added_stencil_size = 0x10000; + constexpr float added_stencil_depth_value = 0.75f; + std::memcpy(memory + added_stencil_depth_offset, + &added_stencil_depth_value, + sizeof(added_stencil_depth_value)); + auto depth_only_source = MakeLinearDesc( + base + added_stencil_depth_offset, 0x10000, vk::Format::eD32Sfloat, + Prospero::GpuEnumValue(Prospero::BufferFormat::k32Float), + Prospero::ImageType::kColor2D, {1, 1, 1}, 1, 4, 1); + depth_only_source.type = BindingType::DepthTarget; + depth_only_source.view_info.format = vk::Format::eD32Sfloat; + depth_only_source.view_info.aspect = vk::ImageAspectFlagBits::eDepth; + depth_only_source.view_info.usage = + vk::ImageUsageFlagBits::eDepthStencilAttachment; + const auto depth_only_source_image = + texture_cache.FindImage(depth_only_source); + + auto combined_destination = depth_only_source; + combined_destination.info.stencil = {base + added_stencil_offset, + added_stencil_size}; + combined_destination.info.pixel_format = vk::Format::eD32SfloatS8Uint; + combined_destination.view_info.format = vk::Format::eD32SfloatS8Uint; + combined_destination.view_info.aspect = + vk::ImageAspectFlagBits::eDepth | vk::ImageAspectFlagBits::eStencil; + const auto combined_destination_image = + texture_cache.FindImage(combined_destination); + TextureCacheTestAccess::AssociateStencil( + texture_cache, combined_destination_image, + combined_destination.info.stencil); + const auto combined_stencil_association = + texture_cache.FindImageFromRange(base + added_stencil_offset, + added_stencil_size, false); + Require( + name, "depth-only to combined association", + combined_destination_image != depth_only_source_image && + combined_stencil_association && + texture_cache.GetImage(combined_stencil_association).depth_id == + combined_destination_image && + texture_cache.GetImage(combined_stencil_association) + .backing.image == nullptr, + "stencil addition did not recreate depth and register its " + "lightweight association"); + + const uint32_t mirror_value = 0xa1b2c3d4u; + std::memcpy(memory + 0x5000, &mirror_value, sizeof(mirror_value)); + auto mirror_desc = MakeLinearDesc( + base + 0x5000, sizeof(mirror_value), vk::Format::eR32Uint, + Prospero::GpuEnumValue(Prospero::BufferFormat::k32UInt), + Prospero::ImageType::kColor2D, {1, 1, 1}, 1, 4, 1); + const auto mirror_image = texture_cache.FindImage(mirror_desc); + texture_cache.MarkGpuWritten(mirror_image); + constexpr uint32_t mirror_cpu_value = 0x0f1e2d3cu; + Require(name, "CPU-dirty formatted mirror fault", + resources.HandleFault(PageFaultAccess::Write, + mirror_desc.info.data.address), + "GPU image did not accept a CPU write before Buffer mirroring"); + std::memcpy(memory + 0x5000, &mirror_cpu_value, sizeof(mirror_cpu_value)); + Require(name, "image-to-buffer mirror", + texture_cache.SynchronizeImageToBuffer(base + 0x5000, + sizeof(mirror_value)) && + texture_cache.GetImage(mirror_image).IsBufferModified() && + !texture_cache.GetImage(mirror_image).IsGpuModified(), + "image-to-buffer synchronization did not transfer ownership"); + auto mirror_binding = resources.GetBufferCache().ObtainBuffer( + command, base + 0x5000, sizeof(mirror_value), false, true); + Require(name, "CPU-dirty formatted mirror source", + mirror_binding.buffer != nullptr && + mirror_binding.owner != nullptr, + "formatted mirror did not expose its native Buffer source"); + command.RetainResourceUntilFence(mirror_binding.owner); + auto mirror_cpu_readback = CreateHostBuffer( + name, sizeof(mirror_cpu_value), vk::BufferUsageFlagBits::eTransferDst, + std::vector{0}); + const vk::BufferCopy mirror_cpu_copy{mirror_binding.offset, 0, + sizeof(mirror_cpu_value)}; + command.Handle().copyBuffer(mirror_binding.buffer, + mirror_cpu_readback.buffer, 1, + &mirror_cpu_copy); + HostReadBarrier(mirror_cpu_readback.buffer, mirror_cpu_readback.size, + vk::PipelineStageFlagBits::eTransfer, + vk::AccessFlagBits::eTransferWrite); + auto mirror_refresh_desc = mirror_desc; + const auto mirror_refresh = texture_cache.FindImage(mirror_refresh_desc); + Require(name, "buffer-to-image ownership", + mirror_refresh == mirror_image && + !texture_cache.GetImage(mirror_refresh).IsBufferModified() && + texture_cache.GetImage(mirror_refresh).IsGpuModified(), + "buffer-backed refresh did not atomically reclaim image " + "ownership"); + texture_cache.MarkGpuWritten(mirror_refresh); + + constexpr uint64_t exact_buffer_offset = 0x90000; + constexpr uint32_t exact_buffer_value = 0x3f234567u; + std::memcpy(memory + exact_buffer_offset, &exact_buffer_value, + sizeof(exact_buffer_value)); + auto exact_buffer_desc = MakeLinearDesc( + base + exact_buffer_offset, sizeof(exact_buffer_value), + vk::Format::eR32Uint, + Prospero::GpuEnumValue(Prospero::BufferFormat::k32UInt), + Prospero::ImageType::kColor2D, {1, 1, 1}, 1, 4, 1); + const auto exact_buffer_image = + texture_cache.FindImage(exact_buffer_desc); + texture_cache.MarkGpuWritten(exact_buffer_image); + Require(name, "exact replacement Buffer publication", + texture_cache.SynchronizeImageToBuffer( + exact_buffer_desc.info.data.address, + exact_buffer_desc.info.data.size) && + texture_cache.GetImage(exact_buffer_image).IsBufferModified(), + "exact replacement source did not transfer to Buffer ownership"); + auto exact_float_desc = exact_buffer_desc; + exact_float_desc.info.pixel_format = vk::Format::eR32Sfloat; + exact_float_desc.info.guest_format = + Prospero::GpuEnumValue(Prospero::BufferFormat::k32Float); + exact_float_desc.view_info.format = vk::Format::eR32Sfloat; + const auto exact_float_image = + texture_cache.FindImage(exact_float_desc, true); + Require( + name, "Buffer-superseded exact coexistence", + exact_float_image != exact_buffer_image && + TextureCacheTestAccess::Contains(texture_cache, + exact_buffer_image) && + texture_cache.GetImage(exact_buffer_image).IsBufferModified() && + !texture_cache.GetImage(exact_float_image).IsBufferModified() && + texture_cache.GetImage(exact_float_image).IsGpuModified(), + "exact-format lookup retired its old record or failed to import " + "the shared Buffer source"); + auto exact_buffer_readback = CreateHostBuffer( + name, sizeof(exact_buffer_value), + vk::BufferUsageFlagBits::eTransferDst, std::vector{0}); + auto &exact_float_native = texture_cache.GetImage(exact_float_image); + exact_float_native.Transit(vk::ImageLayout::eTransferSrcOptimal, + vk::AccessFlagBits2::eTransferRead, {}, + command.Handle()); + vk::BufferImageCopy exact_float_copy{}; + exact_float_copy.imageSubresource.aspectMask = + vk::ImageAspectFlagBits::eColor; + exact_float_copy.imageSubresource.layerCount = 1; + exact_float_copy.imageExtent = {1, 1, 1}; + command.Handle().copyImageToBuffer(exact_float_native.backing.image, + vk::ImageLayout::eTransferSrcOptimal, + exact_buffer_readback.buffer, 1, + &exact_float_copy); + HostReadBarrier(exact_buffer_readback.buffer, exact_buffer_readback.size, + vk::PipelineStageFlagBits::eTransfer, + vk::AccessFlagBits::eTransferWrite); + + const uint32_t clear_source = 0x01010101u; + std::memcpy(memory + 0x6000, &clear_source, sizeof(clear_source)); + auto clear_desc = MakeLinearDesc( + base + 0x6000, sizeof(clear_source), vk::Format::eR8G8B8A8Unorm, + Prospero::GpuEnumValue(Prospero::BufferFormat::k8_8_8_8UNorm), + Prospero::ImageType::kColor2D, {1, 1, 1}, 1, 4, 1); + const auto clear_image = texture_cache.FindImage(clear_desc); + auto buffer_write = resources.GetBufferCache().ObtainBuffer( + command, base + 0x6000, sizeof(clear_source), true, false, true); + Require(name, "buffer-write ownership", + buffer_write.buffer != nullptr && + texture_cache.GetImage(clear_image).IsBufferModified(), + "formatted buffer write did not supersede the image"); + if (buffer_write.owner != nullptr) { + command.RetainResourceUntilFence(buffer_write.owner); + } + Require(name, "buffer-owned full image clear", + texture_cache.ClearImageFromBuffer( + command, base + 0x6000, sizeof(clear_source), 0xaabbccddu) && + !texture_cache.GetImage(clear_image).IsBufferModified() && + texture_cache.GetImage(clear_image).IsGpuModified(), + "full image clear retained dual buffer/image GPU ownership"); + texture_cache.MarkGpuWritten(clear_image); + auto clear_read_desc = clear_desc; + Require(name, "post-clear image reuse", + texture_cache.FindImage(clear_read_desc) == clear_image, + "cleared image could not be read or written after ownership " + "transfer"); + + constexpr uint64_t partial_image_offset = 0xa000; + constexpr uint64_t partial_buffer_offset = 0xa010; + constexpr uint64_t partial_clean_offset = 0xa020; + constexpr uint32_t partial_image_value = 0x31415926u; + constexpr uint32_t partial_buffer_value = 0x27182818u; + constexpr uint32_t partial_clean_value = 0xabcdef01u; + std::memcpy(memory + partial_clean_offset, &partial_clean_value, + sizeof(partial_clean_value)); + auto partial_write = resources.GetBufferCache().ObtainBuffer( + command, base + partial_image_offset, sizeof(partial_image_value), + true, false); + Require(name, "partial-page buffer allocation", + partial_write.owner != nullptr, + "partial-page GPU write did not create a native buffer"); + command.RetainResourceUntilFence(partial_write.owner); + resources.GetBufferCache().FillBuffer(base + partial_image_offset, + sizeof(partial_image_value), + partial_image_value); + resources.GetBufferCache().FillBuffer(base + partial_buffer_offset, + sizeof(partial_buffer_value), + partial_buffer_value); + auto partial_desc = MakeLinearDesc( + base + partial_image_offset, sizeof(partial_image_value), + vk::Format::eR32Uint, + Prospero::GpuEnumValue(Prospero::BufferFormat::k32UInt), + Prospero::ImageType::kColor2D, {1, 1, 1}, 1, 4, 1); + const auto partial_image = texture_cache.FindImage(partial_desc); + Require(name, "partial-page image ownership", + texture_cache.GetImage(partial_image).IsGpuModified(), + "image did not consume its exact GPU-owned buffer bytes"); + auto partial_image_mirror = resources.GetBufferCache().ObtainBuffer( + command, base + partial_image_offset, sizeof(partial_image_value), + false, true, true); + Require(name, "partial-page image mirror", + partial_image_mirror.buffer != nullptr && + partial_image_mirror.owner != nullptr && + texture_cache.GetImage(partial_image).IsBufferModified() && + !texture_cache.GetImage(partial_image).IsGpuModified(), + "same-page image bytes did not transfer back to their buffer"); + command.RetainResourceUntilFence(partial_image_mirror.owner); + const auto partial_clean_source = + resources.GetBufferCache().ObtainBufferForImage( + base + partial_clean_offset, sizeof(partial_clean_value)); + Require(name, "partial-page clean source", + partial_clean_source.buffer != nullptr && + !partial_clean_source.gpu_owned, + "clean same-page bytes inherited unrelated buffer ownership"); + Require(name, "partial-page remaining fault", + resources.HandleFault(PageFaultAccess::Read, + base + partial_buffer_offset), + "same-page buffer ownership was lost after image transfer"); + uint32_t partial_buffer_backing = 0; + std::memcpy(&partial_buffer_backing, memory + partial_buffer_offset, + sizeof(partial_buffer_backing)); + uint32_t partial_image_backing = 0; + std::memcpy(&partial_image_backing, memory + partial_image_offset, + sizeof(partial_image_backing)); + Require(name, "partial-page readback values", + partial_image_backing == partial_image_value && + partial_buffer_backing == partial_buffer_value, + "same-page image/buffer readback published incorrect bytes"); + constexpr uint32_t partial_cpu_refresh_value = 0x13579bdfu; + Require(name, "partial-page CPU write fault", + resources.HandleFault(PageFaultAccess::Write, + base + partial_clean_offset), + "same-page CPU write did not invalidate cached ownership"); + std::memcpy(memory + partial_clean_offset, &partial_cpu_refresh_value, + sizeof(partial_cpu_refresh_value)); + const auto partial_cpu_refresh_source = + resources.GetBufferCache().ObtainBufferForImage( + base + partial_clean_offset, sizeof(partial_cpu_refresh_value)); + Require(name, "partial-page CPU refresh source", + partial_cpu_refresh_source.buffer != nullptr && + !partial_cpu_refresh_source.gpu_owned, + "CPU-dirty same-page bytes did not resolve through the cached " + "buffer"); + auto partial_cpu_refresh_readback = + CreateHostBuffer(name, sizeof(partial_cpu_refresh_value), + vk::BufferUsageFlagBits::eTransferDst, {0}); + const vk::BufferCopy partial_cpu_refresh_copy{ + partial_cpu_refresh_source.offset, 0, + sizeof(partial_cpu_refresh_value)}; + command.Handle().copyBuffer(partial_cpu_refresh_source.buffer->Handle(), + partial_cpu_refresh_readback.buffer, 1, + &partial_cpu_refresh_copy); + vk::BufferMemoryBarrier partial_cpu_refresh_barrier{}; + partial_cpu_refresh_barrier.sType = + vk::StructureType::eBufferMemoryBarrier; + partial_cpu_refresh_barrier.srcAccessMask = + vk::AccessFlagBits::eTransferWrite; + partial_cpu_refresh_barrier.dstAccessMask = vk::AccessFlagBits::eHostRead; + partial_cpu_refresh_barrier.srcQueueFamilyIndex = VK_QUEUE_FAMILY_IGNORED; + partial_cpu_refresh_barrier.dstQueueFamilyIndex = VK_QUEUE_FAMILY_IGNORED; + partial_cpu_refresh_barrier.buffer = partial_cpu_refresh_readback.buffer; + partial_cpu_refresh_barrier.size = partial_cpu_refresh_readback.size; + command.Handle().pipelineBarrier(vk::PipelineStageFlagBits::eTransfer, + vk::PipelineStageFlagBits::eHost, {}, 0, + nullptr, 1, &partial_cpu_refresh_barrier, + 0, nullptr); + + const uint32_t fault_a = 0x01020304u; + const uint32_t fault_b = 0x11121314u; + std::memcpy(memory + 0x8000, &fault_a, sizeof(fault_a)); + std::memcpy(memory + 0x8010, &fault_b, sizeof(fault_b)); + auto fault_a_desc = MakeLinearDesc( + base + 0x8000, sizeof(fault_a), vk::Format::eR32Uint, + Prospero::GpuEnumValue(Prospero::BufferFormat::k32UInt), + Prospero::ImageType::kColor2D, {1, 1, 1}, 1, 4, 1); + auto fault_b_desc = fault_a_desc; + fault_b_desc.info.data.address = base + 0x8010; + const auto fault_a_image = texture_cache.FindImage(fault_a_desc); + const auto fault_b_image = texture_cache.FindImage(fault_b_desc); + texture_cache.MarkGpuWritten(fault_a_image); + texture_cache.MarkGpuWritten(fault_b_image); + constexpr uint64_t padding_fault_offset = 0x8080; + uint32_t write_only_read_a = 0; + uint32_t write_only_read_b = 0; + std::memcpy(&write_only_read_a, memory + 0x8000, + sizeof(write_only_read_a)); + std::memcpy(&write_only_read_b, memory + 0x8010, + sizeof(write_only_read_b)); + Require(name, "write-only image read policy", + write_only_read_a == fault_a && write_only_read_b == fault_b && + texture_cache.GetImage(fault_a_image).IsGpuModified() && + texture_cache.GetImage(fault_b_image).IsGpuModified(), + "TextureCache claimed a CPU read fault instead of preserving " + "GPU image ownership"); + Require(name, "same-page image write invalidation", + resources.HandleFault(PageFaultAccess::Write, + base + padding_fault_offset) && + texture_cache.GetImage(fault_a_image).IsGpuModified() && + texture_cache.GetImage(fault_b_image).IsGpuModified(), + "a byte-disjoint CPU write discarded authoritative images"); + Require(name, "same-page image re-track", + texture_cache.FindImage(fault_a_desc) == fault_a_image && + texture_cache.FindImage(fault_b_desc) == fault_b_image && + texture_cache.SynchronizeImageToBuffer(base + 0x8000, + sizeof(fault_a)) && + !texture_cache.GetImage(fault_a_image).IsGpuModified() && + texture_cache.GetImage(fault_b_image).IsGpuModified(), + "retiring one same-page image lost the surviving owner"); + Require(name, "same-page survivor write fault", + resources.HandleFault(PageFaultAccess::Write, base + 0x8010) && + texture_cache.GetImage(fault_b_image).IsGpuModified() && + texture_cache.GetImage(fault_b_image).IsDefinitelyCpuDirty(), + "the surviving image was not protected after its alias retired"); + constexpr uint32_t fault_b_cpu = 0xa5a6a7a8u; + std::memcpy(memory + 0x8010, &fault_b_cpu, sizeof(fault_b_cpu)); + Require( + name, "same-page survivor refresh", + texture_cache.FindImage(fault_b_desc) == fault_b_image && + !texture_cache.GetImage(fault_b_image).IsDefinitelyCpuDirty() && + texture_cache.GetImage(fault_b_image).IsGpuModified(), + "the surviving image could not reconcile its CPU write"); + + constexpr uint64_t publish_image_offset = 0x26000; + constexpr uint64_t publish_buffer_offset = 0x26010; + constexpr uint32_t publish_image_value = 0x31415926u; + constexpr uint32_t publish_buffer_value = 0x27182818u; + std::memcpy(memory + publish_image_offset, &publish_image_value, + sizeof(publish_image_value)); + auto publish_image_desc = MakeLinearDesc( + base + publish_image_offset, sizeof(publish_image_value), + vk::Format::eR32Uint, + Prospero::GpuEnumValue(Prospero::BufferFormat::k32UInt), + Prospero::ImageType::kColor2D, {1, 1, 1}, 1, 4, 1); + const auto publish_image = texture_cache.FindImage(publish_image_desc); + texture_cache.MarkGpuWritten(publish_image); + auto [publish_buffer_owner, publish_buffer_native_offset] = + resources.GetBufferCache().ObtainBufferForImageWrite( + base + publish_buffer_offset, sizeof(publish_buffer_value)); + (void)publish_buffer_native_offset; + Require(name, "exact-disjoint buffer allocation", + publish_buffer_owner != nullptr, + "failed to create the neighboring native buffer owner"); + command.RetainResourceUntilFence(publish_buffer_owner); + resources.GetBufferCache().FillBuffer(base + publish_buffer_offset, + sizeof(publish_buffer_value), + publish_buffer_value); + auto publish_replacement_desc = publish_image_desc; + publish_replacement_desc.info.pixel_format = vk::Format::eR32Sfloat; + publish_replacement_desc.info.guest_format = + Prospero::GpuEnumValue(Prospero::BufferFormat::k32Float); + publish_replacement_desc.view_info.format = vk::Format::eR32Sfloat; + const auto publish_replacement = + texture_cache.FindImage(publish_replacement_desc, true); + Require(name, "exact-disjoint image replacement", + publish_replacement && publish_replacement != publish_image, + "exact-format lookup did not replace the image"); + Require(name, "exact-disjoint image publication", + !resources.GetBufferCache().HasGpuDirtyBytes( + base + publish_buffer_offset, sizeof(publish_buffer_value)), + "clean neighboring buffer unexpectedly became GPU-owned"); + uint32_t published_image_backing = 0; + uint32_t published_buffer_backing = 0; + std::memcpy(&published_image_backing, memory + publish_image_offset, + sizeof(published_image_backing)); + std::memcpy(&published_buffer_backing, memory + publish_buffer_offset, + sizeof(published_buffer_backing)); + Require(name, "exact-disjoint publication contents", + published_image_backing == publish_image_value && + published_buffer_backing == publish_buffer_value, + "image publication or neighboring buffer readback changed bytes"); + + for (const auto [offset, samples] : + std::array, 2>{ + {{0x10000, 2}, {0x11000, 4}}}) { + std::memset(memory + offset, 0, samples * sizeof(uint16_t)); + auto standalone_ms = MakeLinearDesc( + base + offset, samples * sizeof(uint16_t), vk::Format::eD16Unorm, + Prospero::GpuEnumValue(Prospero::BufferFormat::k16UNorm), + Prospero::ImageType::kColor2D, {1, 1, 1}, 1, 2, samples); + standalone_ms.type = BindingType::DepthTarget; + standalone_ms.view_info.aspect = vk::ImageAspectFlagBits::eDepth; + standalone_ms.view_info.usage = + vk::ImageUsageFlagBits::eDepthStencilAttachment; + const auto image = texture_cache.FindImage(standalone_ms); + texture_cache.MarkGpuWritten(image); + uint16_t standalone_cpu_read = 1; + std::memcpy(&standalone_cpu_read, memory + offset, + sizeof(standalone_cpu_read)); + Require( + name, "standalone MSAA ownership", + texture_cache.GetImage(image).IsGpuModified() && + standalone_cpu_read == 0 && + texture_cache.GetImage(image).IsGpuModified() && + resources.HandleFault(PageFaultAccess::Write, base + offset) && + texture_cache.GetImage(image).IsGpuModified() && + texture_cache.GetImage(image).IsDefinitelyCpuDirty(), + "fresh multisample target violated write-only ownership"); + } + + constexpr uint64_t metadata_data_a = 0x12000; + constexpr uint64_t metadata_data_b = 0x12100; + constexpr uint64_t metadata_a = 0x13000; + constexpr uint64_t metadata_b = 0x13100; + auto MakeMetadataDepth = [&](uint64_t data, uint64_t metadata) { + auto desc = MakeLinearDesc( + base + data, sizeof(uint32_t), vk::Format::eD32Sfloat, + Prospero::GpuEnumValue(Prospero::BufferFormat::k32Float), + Prospero::ImageType::kColor2D, {1, 1, 1}, 1, 4, 1); + desc.type = BindingType::DepthTarget; + desc.info.metadata.kind = ImageMetadataKind::Htile; + desc.info.metadata.range = {base + metadata, 0x80}; + desc.view_info.aspect = vk::ImageAspectFlagBits::eDepth; + desc.view_info.usage = vk::ImageUsageFlagBits::eDepthStencilAttachment; + return desc; + }; + auto metadata_depth_a = MakeMetadataDepth(metadata_data_a, metadata_a); + auto metadata_depth_b = MakeMetadataDepth(metadata_data_b, metadata_b); + const auto metadata_depth_a_id = + texture_cache.FindImage(metadata_depth_a); + const auto metadata_depth_b_id = + texture_cache.FindImage(metadata_depth_b); + const auto metadata_depth_a_view = + texture_cache.FindDepthTarget(metadata_depth_a_id, metadata_depth_a); + const auto metadata_depth_b_view = + texture_cache.FindDepthTarget(metadata_depth_b_id, metadata_depth_b); + Require(name, "shared-page metadata state", + metadata_depth_a_view != nullptr && + metadata_depth_b_view != nullptr && + texture_cache.ClearMeta(base + metadata_a) && + texture_cache.ClearMeta(base + metadata_b), + "shared-page metadata did not retain logical clear state"); + auto metadata_alias = MakeLinearDesc( + base + metadata_a, sizeof(uint32_t), vk::Format::eR32Sfloat, + Prospero::GpuEnumValue(Prospero::BufferFormat::k32Float), + Prospero::ImageType::kColor2D, {1, 1, 1}, 1, 4, 1); + metadata_alias.type = BindingType::RenderTarget; + metadata_alias.view_info.usage = vk::ImageUsageFlagBits::eColorAttachment; + const auto metadata_alias_id = + texture_cache.FindImage(metadata_alias, true); + const auto metadata_alias_view = + texture_cache.FindRenderTarget(metadata_alias_id, metadata_alias); + Require( + name, "exact alias metadata coexistence", + metadata_alias_view != nullptr && + texture_cache.GetImage(metadata_depth_a_id).IsGpuModified() && + texture_cache.GetImage(metadata_depth_b_id).IsGpuModified() && + texture_cache.IsMeta(base + metadata_a) && + texture_cache.IsMetaCleared(base + metadata_a, 0) && + texture_cache.IsMeta(base + metadata_b) && + texture_cache.IsMetaCleared(base + metadata_b, 0), + "image discovery retired aliased metadata or its live depth image"); + Require( + name, "metadata first-touch state", + texture_cache.TouchMeta(base + metadata_b, 0, true) && + texture_cache.IsMetaCleared(base + metadata_b, 0) && + texture_cache.TouchMeta(base + metadata_b, 0, false) && + !texture_cache.IsMetaCleared(base + metadata_b, 0), + "per-slice metadata update incorrectly required prior GPU ownership"); + resources.GetBufferCache().FillBuffer(base + metadata_b, sizeof(uint32_t), + 0); + Require( + name, "metadata buffer fill state", + texture_cache.IsMetaCleared(base + metadata_b, 0), + "BufferCache fill did not publish an exact-address metadata clear"); + + constexpr uint64_t partial_unmap_image_offset = 0x2700000; + auto partial_unmap_image = MakeLinearDesc( + base + partial_unmap_image_offset, 0x2000, vk::Format::eR32Uint, + Prospero::GpuEnumValue(Prospero::BufferFormat::k32UInt), + Prospero::ImageType::kColor2D, {2048, 1, 1}, 1, 4, 1); + const auto partial_unmap_image_id = + texture_cache.FindImage(partial_unmap_image); + texture_cache.UnmapMemory(partial_unmap_image.info.data.address, 0x1000); + Require(name, "partial image unmap tracking", + partial_unmap_image_id && + !texture_cache.FindImageFromRange( + partial_unmap_image.info.data.address, 0x2000, false) && + !texture_cache.InvalidateMemory( + PageFaultAccess::Write, + partial_unmap_image.info.data.address + 0x1000, 0x1000, + PageFaultPhase::Invalidate), + "partial unmap left the deleted image's mapped tail tracked"); + + constexpr uint64_t unformatted_alias_offset = 0x2500000; + auto unformatted_alias = MakeLinearDesc( + base + unformatted_alias_offset, sizeof(uint32_t), + vk::Format::eR32Uint, + Prospero::GpuEnumValue(Prospero::BufferFormat::k32UInt), + Prospero::ImageType::kColor2D, {1, 1, 1}, 1, 4, 1); + const auto unformatted_alias_image = + texture_cache.FindImage(unformatted_alias); + texture_cache.MarkGpuWritten(unformatted_alias_image); + auto unformatted_alias_buffer = resources.GetBufferCache().ObtainBuffer( + command, unformatted_alias.info.data.address, + unformatted_alias.info.data.size, false, true, false); + Require( + name, "unformatted buffer/image alias", + unformatted_alias_buffer.buffer != nullptr && + texture_cache.GetImage(unformatted_alias_image).IsGpuModified() && + !texture_cache.GetImage(unformatted_alias_image).IsBufferModified(), + "read-only unformatted buffer alias did not follow ordinary BufferCache acquisition"); + if (unformatted_alias_buffer.owner != nullptr) { + command.RetainResourceUntilFence(unformatted_alias_buffer.owner); + } + + constexpr uint64_t layered_offset = 0x14000; + constexpr uint64_t layered_guest_size = 0x800; + const std::array layered_values{0.0f, 0.125f, 0.25f, 0.375f, + 0.5f, 0.625f, 0.75f, 0.875f, + 1.0f, 0.0625f}; + std::memset(memory + layered_offset, 0, layered_guest_size); + const auto layered_layout = TextureCalcUploadLayout( + Prospero::GpuEnumValue(Prospero::BufferFormat::k32Float), 2, 2, 2, 2, + 2, Prospero::GpuEnumValue(Prospero::TileMode::kLinear), + layered_guest_size, true, false, "UnifiedTextureCacheFlow"); + const auto layered_upload_regions = + TextureBuildImageCopies(layered_layout, 2, 2, 2, 2, true, false); + Require(name, "layered guest layout", layered_upload_regions.size() == 4, + "unexpected layered/mipped upload-region count"); + for (const auto ®ion : layered_upload_regions) { + for (uint32_t y = 0; y < region.imageExtent.height; y++) { + for (uint32_t x = 0; x < region.imageExtent.width; x++) { + const uint32_t logical = + region.imageSubresource.mipLevel == 0 + ? region.imageSubresource.baseArrayLayer * 4 + y * 2 + x + : 8 + region.imageSubresource.baseArrayLayer; + const uint64_t byte_offset = + region.bufferOffset + + (static_cast(y) * region.bufferRowLength + x) * + sizeof(float); + Require(name, "layered guest layout bounds", + byte_offset + sizeof(float) <= layered_guest_size, + "layered guest texel lies outside its backing range"); + std::memcpy(memory + layered_offset + byte_offset, + &layered_values[logical], sizeof(float)); + } + } + } + auto layered_color = MakeLinearDesc( + base + layered_offset, layered_guest_size, vk::Format::eR32Sfloat, + Prospero::GpuEnumValue(Prospero::BufferFormat::k32Float), + Prospero::ImageType::kColor2D, {2, 2, 1}, 2, 4, 1); + layered_color.info.resources.levels = 2; + layered_color.info.mip_layout[0] = {0, 32, 2, 2}; + layered_color.info.mip_layout[1] = {32, 8, 1, 1}; + layered_color.view_info.level_count = 2; + const auto layered_color_image = texture_cache.FindImage(layered_color); + texture_cache.MarkGpuWritten(layered_color_image); + auto layered_depth = layered_color; + layered_depth.type = BindingType::DepthTarget; + layered_depth.info.resources = {1, 4}; + layered_depth.view_info.level_count = 1; + layered_depth.info.pixel_format = vk::Format::eD32Sfloat; + layered_depth.view_info.format = vk::Format::eD32Sfloat; + layered_depth.view_info.aspect = vk::ImageAspectFlagBits::eDepth; + layered_depth.view_info.usage = + vk::ImageUsageFlagBits::eDepthStencilAttachment; + const auto layered_depth_image = texture_cache.FindImage(layered_depth); + const auto &layered_native = texture_cache.GetImage(layered_depth_image); + Require(name, "layered mipped depth alias", + layered_depth_image != layered_color_image && + layered_native.backing.layers == 2 && + layered_native.backing.mip_levels == 2 && + layered_native.info.resources == + layered_color.info.resources && + layered_native.IsGpuModified(), + "depth/color conversion did not use the lexicographic " + "resource maximum"); + const volatile auto layered_guest_byte = + *reinterpret_cast(memory + layered_offset); + (void)layered_guest_byte; + Require(name, "layered depth write-only read policy", + texture_cache.GetImage(layered_depth_image).IsGpuModified(), + "a CPU read invalidated layered depth GPU ownership"); + + constexpr uint64_t replacement_offset = 0x18000; + constexpr uint64_t replacement_size = 0x2000; + constexpr uint32_t replacement_pixel = 0xcafebabeu; + constexpr uint32_t replacement_tail = 0x0badf00du; + std::memset(memory + replacement_offset, 0, replacement_size); + std::memcpy(memory + replacement_offset, &replacement_pixel, + sizeof(replacement_pixel)); + std::memcpy(memory + replacement_offset + replacement_size - + sizeof(replacement_tail), + &replacement_tail, sizeof(replacement_tail)); + auto replacement_source = MakeLinearDesc( + base + replacement_offset, sizeof(uint32_t), vk::Format::eR32Uint, + Prospero::GpuEnumValue(Prospero::BufferFormat::k32UInt), + Prospero::ImageType::kColor2D, {1, 1, 1}, 1, 4, 1); + const auto replacement_source_image = + texture_cache.FindImage(replacement_source); + texture_cache.MarkGpuWritten(replacement_source_image); + auto replacement_desc = MakeLinearDesc( + base + replacement_offset, replacement_size, vk::Format::eR32Uint, + Prospero::GpuEnumValue(Prospero::BufferFormat::k32UInt), + Prospero::ImageType::kColor2D, {1, 1, 1}, 2, 4, 1); + const auto replacement = texture_cache.FindImage(replacement_desc); + const bool replacement_changed = replacement != replacement_source_image; + const bool replacement_gpu_before = + texture_cache.GetImage(replacement).IsGpuModified(); + const bool replacement_fault = resources.HandleFault( + PageFaultAccess::Write, base + replacement_offset + 0x1000); + const bool replacement_gpu_after = + texture_cache.GetImage(replacement).IsGpuModified(); + uint32_t replacement_pixel_after = 0; + uint32_t replacement_tail_after = 0; + std::memcpy(&replacement_pixel_after, memory + replacement_offset, + sizeof(replacement_pixel_after)); + std::memcpy(&replacement_tail_after, + memory + replacement_offset + replacement_size - + sizeof(replacement_tail_after), + sizeof(replacement_tail_after)); + Require(name, "replacement tracking", + replacement_changed && replacement_gpu_before && + replacement_fault && replacement_gpu_after && + texture_cache.GetImage(replacement).IsDefinitelyCpuDirty() && + replacement_pixel_after == replacement_pixel && + replacement_tail_after == replacement_tail, + "replacement write invalidation lost native authority or guest " + "backing"); + + const uint32_t compressed_value = 0x55667788u; + std::memcpy(memory + 0xc000, &compressed_value, sizeof(compressed_value)); + auto compressed_desc = MakeLinearDesc( + base + 0xc000, sizeof(compressed_value), vk::Format::eR8G8B8A8Srgb, + Prospero::GpuEnumValue(Prospero::BufferFormat::k8_8_8_8Srgb), + Prospero::ImageType::kColor2D, {1, 1, 1}, 1, 4, 1); + compressed_desc.type = BindingType::RenderTarget; + compressed_desc.info.metadata.kind = ImageMetadataKind::Dcc; + compressed_desc.info.metadata.range = {base + 0xd000, 0}; + compressed_desc.info.metadata.compression = + VideoOutCompression::Dcc256_256_0; + compressed_desc.view_info.usage = + vk::ImageUsageFlagBits::eColorAttachment; + const auto compressed_image = texture_cache.FindImage(compressed_desc); + texture_cache.MarkGpuWritten(compressed_image); + uint32_t compressed_cpu_read = 0; + std::memcpy(&compressed_cpu_read, memory + 0xc000, + sizeof(compressed_cpu_read)); + Require(name, "compressed write-only read policy", + compressed_cpu_read == compressed_value && + texture_cache.GetImage(compressed_image).IsGpuModified(), + "compressed image incorrectly claimed a CPU read fault"); + Require(name, "compressed download rejection", + !TextureCacheTestAccess::TryDownload(texture_cache, + compressed_image) && + !texture_cache.SynchronizeImageToBuffer( + compressed_desc.info.data.address, + compressed_desc.info.data.size) && + texture_cache.GetImage(compressed_image).IsGpuModified() && + !texture_cache.GetImage(compressed_image).IsBufferModified(), + "a compressed image escaped the unified download guard"); + auto compressed_video_desc = compressed_desc; + compressed_video_desc.type = BindingType::VideoOut; + compressed_video_desc.view_info.usage = vk::ImageUsageFlagBits::eSampled; + const auto compressed_video = + texture_cache.FindImage(compressed_video_desc); + const auto &compressed_video_image = + texture_cache.GetImage(compressed_video); + Require( + name, "cache-only video-out discovery", + compressed_video == compressed_image && + !compressed_video_image.binding.is_bound && + !compressed_video_image.binding.is_target && + !compressed_video_image.binding.needs_rebind && + !compressed_video_image.usage.video_out, + "FindImage claimed caller-owned video-out usage or binding state"); + + constexpr uint64_t mixed_source_offset = 0x20000; + constexpr uint64_t mixed_source_size = 0x1004; + constexpr uint32_t mixed_source_width = 1025; + constexpr uint32_t mixed_cpu_value = 0x1234abcdu; + constexpr uint32_t mixed_gpu_value = 0x9876fedcu; + auto [mixed_owner, mixed_owner_offset] = + resources.GetBufferCache().ObtainBufferForImageWrite( + base + mixed_source_offset, mixed_source_size); + (void)mixed_owner_offset; + Require(name, "mixed-page source allocation", mixed_owner != nullptr, + "mixed CPU/GPU image source did not create a containing buffer"); + command.RetainResourceUntilFence(mixed_owner); + Require(name, "mixed-page CPU write fault", + resources.HandleFault(PageFaultAccess::Write, + base + mixed_source_offset), + "mixed image source could not dirty its first page"); + std::memcpy(memory + mixed_source_offset, &mixed_cpu_value, + sizeof(mixed_cpu_value)); + resources.GetBufferCache().FillBuffer(base + mixed_source_offset + 0x1000, + sizeof(mixed_gpu_value), + mixed_gpu_value); + auto mixed_source_desc = MakeLinearDesc( + base + mixed_source_offset, mixed_source_size, vk::Format::eR32Uint, + Prospero::GpuEnumValue(Prospero::BufferFormat::k32UInt), + Prospero::ImageType::kColor2D, {mixed_source_width, 1, 1}, 1, 4, 1); + const auto mixed_source_image = + texture_cache.FindImage(mixed_source_desc); + Require(name, "mixed-page image upload", + !texture_cache.GetImage(mixed_source_image).IsGpuModified(), + "clean Buffer source manufactured GPU image ownership"); + auto mixed_source_readback = CreateHostBuffer( + name, mixed_source_size, vk::BufferUsageFlagBits::eTransferDst, + std::vector(mixed_source_size / sizeof(u32), 0)); + auto &mixed_source_native = texture_cache.GetImage(mixed_source_image); + mixed_source_native.Transit(vk::ImageLayout::eTransferSrcOptimal, + vk::AccessFlagBits2::eTransferRead, {}, + command.Handle()); + vk::BufferImageCopy mixed_source_copy{}; + mixed_source_copy.bufferRowLength = mixed_source_width; + mixed_source_copy.imageSubresource.aspectMask = + vk::ImageAspectFlagBits::eColor; + mixed_source_copy.imageSubresource.layerCount = 1; + mixed_source_copy.imageExtent = {mixed_source_width, 1, 1}; + command.Handle().copyImageToBuffer(mixed_source_native.backing.image, + vk::ImageLayout::eTransferSrcOptimal, + mixed_source_readback.buffer, 1, + &mixed_source_copy); + HostReadBarrier(mixed_source_readback.buffer, mixed_source_readback.size, + vk::PipelineStageFlagBits::eTransfer, + vk::AccessFlagBits::eTransferWrite); + + constexpr uint64_t byte_mirror_page_offset = 0x24000; + constexpr uint64_t byte_mirror_offset = byte_mirror_page_offset + 1; + const std::array byte_mirror_guest{0xacu, 0x5au, 0xbdu, + 0xceu}; + std::memcpy(memory + byte_mirror_page_offset, byte_mirror_guest.data(), + byte_mirror_guest.size()); + auto byte_mirror_desc = MakeLinearDesc( + base + byte_mirror_offset, 1, vk::Format::eR8Unorm, + Prospero::GpuEnumValue(Prospero::BufferFormat::k8UNorm), + Prospero::ImageType::kColor2D, {1, 1, 1}, 1, 1, 1); + const auto byte_mirror_image = texture_cache.FindImage(byte_mirror_desc); + texture_cache.MarkGpuWritten(byte_mirror_image); + auto byte_mirror = resources.GetBufferCache().ObtainBuffer( + command, base + byte_mirror_offset, 1, false, true, true); + Require(name, "byte image mirror", + byte_mirror.owner != nullptr && byte_mirror.buffer != nullptr && + texture_cache.GetImage(byte_mirror_image).IsBufferModified(), + "one-byte image did not transfer exact ownership to BufferCache"); + command.RetainResourceUntilFence(byte_mirror.owner); + Require(name, "byte image fault", + resources.HandleFault(PageFaultAccess::Read, + base + byte_mirror_offset), + "one-byte image mirror could not be downloaded"); + std::array byte_mirror_backing{}; + std::memcpy(byte_mirror_backing.data(), memory + byte_mirror_page_offset, + byte_mirror_backing.size()); + Require(name, "byte image fault contents", + byte_mirror_backing == byte_mirror_guest, + "aligned one-byte download changed neighboring guest sentinels"); + + const std::array bgra16_guest{0x3c00u, 0x4000u, 0x4200u, + 0x4400u}; + std::memcpy(memory + 0xb000, bgra16_guest.data(), sizeof(bgra16_guest)); + auto bgra16_desc = MakeLinearDesc( + base + 0xb000, sizeof(bgra16_guest), vk::Format::eR16G16B16A16Sfloat, + Prospero::GpuEnumValue(Prospero::BufferFormat::k16_16_16_16Float), + Prospero::ImageType::kColor2D, {1, 1, 1}, 1, 8, 1); + bgra16_desc.type = BindingType::RenderTarget; + bgra16_desc.info.bgra16 = true; + bgra16_desc.view_info.usage = vk::ImageUsageFlagBits::eColorAttachment; + const auto bgra16_image = texture_cache.FindImage(bgra16_desc); + auto bgra16_readback = + CreateHostBuffer(name, sizeof(bgra16_guest), + vk::BufferUsageFlagBits::eTransferDst, {0, 0}); + auto &bgra16_native = texture_cache.GetImage(bgra16_image); + bgra16_native.Transit(vk::ImageLayout::eTransferSrcOptimal, + vk::AccessFlagBits2::eTransferRead, {}, + command.Handle()); + vk::BufferImageCopy bgra16_copy{}; + bgra16_copy.imageSubresource.aspectMask = vk::ImageAspectFlagBits::eColor; + bgra16_copy.imageSubresource.layerCount = 1; + bgra16_copy.imageExtent = {1, 1, 1}; + command.Handle().copyImageToBuffer( + bgra16_native.backing.image, vk::ImageLayout::eTransferSrcOptimal, + bgra16_readback.buffer, 1, &bgra16_copy); + HostReadBarrier(bgra16_readback.buffer, bgra16_readback.size, + vk::PipelineStageFlagBits::eTransfer, + vk::AccessFlagBits::eTransferWrite); + + auto layered_readback = CreateHostBuffer( + name, sizeof(layered_values), vk::BufferUsageFlagBits::eTransferDst, + std::vector(sizeof(layered_values) / sizeof(u32), 0)); + auto &layered_depth_native = texture_cache.GetImage(layered_depth_image); + layered_depth_native.Transit(vk::ImageLayout::eTransferSrcOptimal, + vk::AccessFlagBits2::eTransferRead, {}, + command.Handle()); + std::array layered_copies{}; + layered_copies[0].bufferOffset = 0; + layered_copies[0].imageSubresource.aspectMask = + vk::ImageAspectFlagBits::eDepth; + layered_copies[0].imageSubresource.mipLevel = 0; + layered_copies[0].imageSubresource.layerCount = 2; + layered_copies[0].imageExtent = {2, 2, 1}; + layered_copies[1].bufferOffset = 32; + layered_copies[1].imageSubresource.aspectMask = + vk::ImageAspectFlagBits::eDepth; + layered_copies[1].imageSubresource.mipLevel = 1; + layered_copies[1].imageSubresource.layerCount = 2; + layered_copies[1].imageExtent = {1, 1, 1}; + command.Handle().copyImageToBuffer( + layered_depth_native.backing.image, + vk::ImageLayout::eTransferSrcOptimal, layered_readback.buffer, + static_cast(layered_copies.size()), layered_copies.data()); + HostReadBarrier(layered_readback.buffer, layered_readback.size, + vk::PipelineStageFlagBits::eTransfer, + vk::AccessFlagBits::eTransferWrite); + + vk::ShaderModuleCreateInfo module_info{}; + module_info.sType = vk::StructureType::eShaderModuleCreateInfo; + module_info.codeSize = sizeof(GPU_TEST_MS_DEPTH_SPV); + module_info.pCode = GPU_TEST_MS_DEPTH_SPV; + vk::ShaderModule ms_depth_module = nullptr; + RequireVk( + name, "MS depth content", + m_device.createShaderModule(&module_info, nullptr, &ms_depth_module), + "vkCreateShaderModule"); + std::array observer_bindings{}; + observer_bindings[0].binding = 0; + observer_bindings[0].descriptorType = vk::DescriptorType::eSampledImage; + observer_bindings[0].descriptorCount = 1; + observer_bindings[0].stageFlags = vk::ShaderStageFlagBits::eCompute; + observer_bindings[1].binding = 1; + observer_bindings[1].descriptorType = vk::DescriptorType::eStorageBuffer; + observer_bindings[1].descriptorCount = 1; + observer_bindings[1].stageFlags = vk::ShaderStageFlagBits::eCompute; + vk::DescriptorSetLayoutCreateInfo observer_layout_info{}; + observer_layout_info.sType = + vk::StructureType::eDescriptorSetLayoutCreateInfo; + observer_layout_info.bindingCount = + static_cast(observer_bindings.size()); + observer_layout_info.pBindings = observer_bindings.data(); + vk::DescriptorSetLayout observer_set_layout = nullptr; + RequireVk(name, "MS depth content", + m_device.createDescriptorSetLayout( + &observer_layout_info, nullptr, &observer_set_layout), + "vkCreateDescriptorSetLayout"); + const vk::PushConstantRange observer_push{ + vk::ShaderStageFlagBits::eCompute, 0, sizeof(uint32_t)}; + vk::PipelineLayoutCreateInfo observer_pipeline_layout_info{}; + observer_pipeline_layout_info.sType = + vk::StructureType::ePipelineLayoutCreateInfo; + observer_pipeline_layout_info.setLayoutCount = 1; + observer_pipeline_layout_info.pSetLayouts = &observer_set_layout; + observer_pipeline_layout_info.pushConstantRangeCount = 1; + observer_pipeline_layout_info.pPushConstantRanges = &observer_push; + vk::PipelineLayout observer_pipeline_layout = nullptr; + RequireVk(name, "MS depth content", + m_device.createPipelineLayout(&observer_pipeline_layout_info, + nullptr, + &observer_pipeline_layout), + "vkCreatePipelineLayout"); + vk::PipelineShaderStageCreateInfo observer_stage{}; + observer_stage.sType = vk::StructureType::ePipelineShaderStageCreateInfo; + observer_stage.stage = vk::ShaderStageFlagBits::eCompute; + observer_stage.module = ms_depth_module; + observer_stage.pName = "main"; + vk::ComputePipelineCreateInfo observer_pipeline_info{}; + observer_pipeline_info.sType = + vk::StructureType::eComputePipelineCreateInfo; + observer_pipeline_info.stage = observer_stage; + observer_pipeline_info.layout = observer_pipeline_layout; + vk::Pipeline observer_depth_pipeline = nullptr; + RequireVk( + name, "MS depth content", + m_device.createComputePipelines(nullptr, 1, &observer_pipeline_info, + nullptr, &observer_depth_pipeline), + "vkCreateComputePipelines"); + const std::array observer_pool_sizes{{ + {vk::DescriptorType::eSampledImage, 2}, + {vk::DescriptorType::eStorageBuffer, 2}, + }}; + vk::DescriptorPoolCreateInfo observer_pool_info{}; + observer_pool_info.sType = vk::StructureType::eDescriptorPoolCreateInfo; + observer_pool_info.maxSets = 2; + observer_pool_info.poolSizeCount = + static_cast(observer_pool_sizes.size()); + observer_pool_info.pPoolSizes = observer_pool_sizes.data(); + vk::DescriptorPool observer_pool = nullptr; + RequireVk(name, "MS depth content", + m_device.createDescriptorPool(&observer_pool_info, nullptr, + &observer_pool), + "vkCreateDescriptorPool"); + const std::array observer_layouts{ + observer_set_layout, observer_set_layout}; + std::array observer_sets{}; + vk::DescriptorSetAllocateInfo observer_allocate{}; + observer_allocate.sType = vk::StructureType::eDescriptorSetAllocateInfo; + observer_allocate.descriptorPool = observer_pool; + observer_allocate.descriptorSetCount = + static_cast(observer_sets.size()); + observer_allocate.pSetLayouts = observer_layouts.data(); + RequireVk(name, "MS depth content", + m_device.allocateDescriptorSets(&observer_allocate, + observer_sets.data()), + "vkAllocateDescriptorSets"); + std::array ms_observer_outputs{ + CreateHostBuffer(name, 4 * sizeof(u32), + vk::BufferUsageFlagBits::eStorageBuffer, + std::vector(4, 0)), + CreateHostBuffer(name, 4 * sizeof(u32), + vk::BufferUsageFlagBits::eStorageBuffer, + std::vector(4, 0))}; + const std::array observed_images{ms_depth_image, + ms_depth_image_4x}; + const std::array observed_samples{2, 4}; + for (uint32_t index = 0; index < observed_images.size(); index++) { + auto &observed = texture_cache.GetImage(observed_images[index]); + observed.Transit(vk::ImageLayout::eShaderReadOnlyOptimal, + vk::AccessFlagBits2::eShaderRead, {}, + command.Handle()); + ImageViewInfo view_info{}; + view_info.format = observed.backing.format; + view_info.type = vk::ImageViewType::e2D; + view_info.aspect = vk::ImageAspectFlagBits::eDepth; + view_info.mapping = {vk::ComponentSwizzle::eR, vk::ComponentSwizzle::eR, + vk::ComponentSwizzle::eR, + vk::ComponentSwizzle::eR}; + view_info.usage = vk::ImageUsageFlagBits::eSampled; + const auto view = observed.FindView(view_info); + const vk::DescriptorImageInfo image_info{ + nullptr, view, vk::ImageLayout::eShaderReadOnlyOptimal}; + const vk::DescriptorBufferInfo buffer_info{ + ms_observer_outputs[index].buffer, 0, + ms_observer_outputs[index].size}; + std::array writes{}; + writes[0].sType = vk::StructureType::eWriteDescriptorSet; + writes[0].dstSet = observer_sets[index]; + writes[0].dstBinding = 0; + writes[0].descriptorCount = 1; + writes[0].descriptorType = vk::DescriptorType::eSampledImage; + writes[0].pImageInfo = &image_info; + writes[1].sType = vk::StructureType::eWriteDescriptorSet; + writes[1].dstSet = observer_sets[index]; + writes[1].dstBinding = 1; + writes[1].descriptorCount = 1; + writes[1].descriptorType = vk::DescriptorType::eStorageBuffer; + writes[1].pBufferInfo = &buffer_info; + m_device.updateDescriptorSets(static_cast(writes.size()), + writes.data(), 0, nullptr); + command.Handle().bindPipeline(vk::PipelineBindPoint::eCompute, + observer_depth_pipeline); + command.Handle().bindDescriptorSets(vk::PipelineBindPoint::eCompute, + observer_pipeline_layout, 0, 1, + &observer_sets[index], 0, nullptr); + command.Handle().pushConstants( + observer_pipeline_layout, vk::ShaderStageFlagBits::eCompute, 0, + sizeof(observed_samples[index]), &observed_samples[index]); + command.Handle().dispatch(1, 1, 1); + HostReadBarrier(ms_observer_outputs[index].buffer, + ms_observer_outputs[index].size, + vk::PipelineStageFlagBits::eComputeShader, + vk::AccessFlagBits::eShaderWrite); + } + + scheduler.Finish(); + + Require(name, "CPU-dirty formatted mirror content", + ReadBuffer(name, mirror_cpu_readback, 1) == + std::vector{mirror_cpu_value}, + "formatted Buffer mirror published stale native image bytes"); + Require(name, "Buffer-superseded exact content", + ReadBuffer(name, exact_buffer_readback, 1) == + std::vector{exact_buffer_value}, + "exact-format recreation initialized from stale guest bytes"); + Require( + name, "partial-page CPU refresh content", + ReadBuffer(name, partial_cpu_refresh_readback, 1) == + std::vector{partial_cpu_refresh_value}, + "cached buffer uploaded bytes outside the exact staged guest range"); + const auto mixed_source_words = ReadBuffer( + name, mixed_source_readback, mixed_source_size / sizeof(u32)); + Require(name, "mixed-page image content", + mixed_source_words.front() == mixed_cpu_value && + mixed_source_words[0x1000 / sizeof(u32)] == mixed_gpu_value, + "mixed CPU/GPU source upload lost one ownership domain"); + Require(name, "BGRA16 content", + ReadBuffer(name, bgra16_readback, 2) == + std::vector{0x40004200u, 0x44003c00u}, + "GPU BGRA16 swap did not publish RGBA half-word order"); + scheduler.Finish(); + const auto layered_words = + ReadBuffer(name, layered_readback, layered_values.size()); + const std::array layered_expected{ + layered_values[0], layered_values[1], layered_values[2], + layered_values[3], layered_values[4], layered_values[5], + layered_values[6], layered_values[7], layered_values[8], + layered_values[9]}; + bool layered_content = layered_words.size() == layered_expected.size(); + for (uint32_t index = 0; + layered_content && index < layered_expected.size(); index++) { + layered_content &= + layered_words[index] == std::bit_cast(layered_expected[index]); + } + Require(name, "layered mipped depth content", layered_content, + "depth/color conversion changed a mip or array-layer value"); + const std::array, 2> expected_ms_depth{{ + {0x2000 / 65535.0f, 0xe000 / 65535.0f, 0.0f, 0.0f}, + {0.0f, 0x4000 / 65535.0f, 0x8000 / 65535.0f, 1.0f}, + }}; + for (uint32_t image = 0; image < expected_ms_depth.size(); image++) { + const auto words = ReadBuffer(name, ms_observer_outputs[image], 4); + bool content = words.size() == 4; + for (uint32_t sample = 0; content && sample < observed_samples[image]; + sample++) { + const float actual = std::bit_cast(words[sample]); + content &= std::abs(actual - expected_ms_depth[image][sample]) <= + 1.5f / 65535.0f; + } + Require(name, + image == 0 ? "two-sample depth content" + : "four-sample depth content", + content, + "color-to-multisample-depth changed a packed sample value"); + } + + auto unrelated_stencil_alias = MakeLinearDesc( + base + second_stencil_offset, ms_stencil_size, vk::Format::eR32Uint, + Prospero::GpuEnumValue(Prospero::BufferFormat::k32UInt), + Prospero::ImageType::kColor2D, + {static_cast(ms_stencil_size / sizeof(uint32_t)), 1, 1}, 1, + 4, 1); + const auto unrelated_stencil_image = + texture_cache.FindImage(unrelated_stencil_alias); + Require(name, "stencil association filtering", + unrelated_stencil_image && + unrelated_stencil_image != ms_depth_image && + !texture_cache.GetImage(unrelated_stencil_image).depth_id, + "an unrelated color alias was hijacked by the stencil " + "association record"); + + texture_cache.UnmapMemory(base + second_stencil_offset, ms_stencil_size); + Require(name, "stencil proxy unmap", + !texture_cache.FindImageFromRange(base + second_stencil_offset, + ms_stencil_size, false) && + texture_cache.GetImage(ms_depth_image).backing.image == + ms_depth_backing, + "stencil unmap removed the depth backing or retained its proxy"); + const auto reassociated_depth = + texture_cache.FindImage(switched_ms_depth); + TextureCacheTestAccess::AssociateStencil( + texture_cache, reassociated_depth, switched_ms_depth.info.stencil); + const auto reassociated_proxy = texture_cache.FindImageFromRange( + base + second_stencil_offset, ms_stencil_size, false); + Require(name, "stencil proxy reassociation", + reassociated_depth == ms_depth_image && reassociated_proxy && + texture_cache.GetImage(reassociated_proxy).depth_id == + ms_depth_image, + "stencil proxy did not re-associate with the surviving depth"); + texture_cache.UnmapMemory(ms_depth_desc.info.data.address, + ms_depth_desc.info.data.size); + Require(name, "depth unmap proxy retirement", + !texture_cache.FindImageFromRange(base + second_stencil_offset, + ms_stencil_size, false), + "unmapping depth retained its re-associated stencil proxy"); + + constexpr uint64_t exact_image_offset = 0x334100; + constexpr uint64_t dirty_sibling_offset = 0x334200; + constexpr uint32_t dirty_sibling_value = 0xc001d00du; + auto dirty_sibling = resources.GetBufferCache().ObtainBuffer( + scheduler.Current(), base + dirty_sibling_offset, + sizeof(dirty_sibling_value), true, false); + Require(name, "same-page dirty sibling allocation", + dirty_sibling.owner != nullptr, + "failed to create the disjoint same-page Buffer owner"); + scheduler.Current().RetainResourceUntilFence(dirty_sibling.owner); + resources.GetBufferCache().FillBuffer(base + dirty_sibling_offset, + sizeof(dirty_sibling_value), + dirty_sibling_value); + auto exact_image_desc = MakeLinearDesc( + base + exact_image_offset, sizeof(uint32_t), vk::Format::eR32Uint, + Prospero::GpuEnumValue(Prospero::BufferFormat::k32UInt), + Prospero::ImageType::kColor2D, {1, 1, 1}, 1, 4, 1); + const auto exact_image = texture_cache.FindImage(exact_image_desc); + texture_cache.MarkGpuWritten(exact_image); + Require(name, "same-page exact alias ownership", + resources.GetBufferCache().HasGpuDirtyBytes( + base + dirty_sibling_offset, sizeof(dirty_sibling_value)) && + !resources.GetBufferCache().HasGpuDirtyBytes( + base + exact_image_offset, sizeof(uint32_t)) && + texture_cache.GetImage(exact_image).IsGpuModified(), + "image ownership discarded or conflicted with disjoint dirty " + "Buffer bytes on the same tracker page"); + resources.GetBufferCache().DiscardGpuDirtyBytes( + base + dirty_sibling_offset, sizeof(dirty_sibling_value)); + + constexpr std::array gc_image_offsets{0x330000, 0x332000}; + constexpr std::array gc_image_values{0x76543210u, + 0x89abcdefu}; + constexpr std::array gc_stale_values{0x10293847u, + 0x56473829u}; + for (size_t index = 0; index < gc_image_offsets.size(); index++) { + std::memcpy(memory + gc_image_offsets[index], &gc_image_values[index], + sizeof(uint32_t)); + } + auto gc_image_desc_a = MakeLinearDesc( + base + gc_image_offsets[0], sizeof(uint32_t), vk::Format::eR32Uint, + Prospero::GpuEnumValue(Prospero::BufferFormat::k32UInt), + Prospero::ImageType::kColor2D, {1, 1, 1}, 1, 4, 1); + auto gc_image_desc_b = gc_image_desc_a; + gc_image_desc_b.info.data.address = base + gc_image_offsets[1]; + auto clean_buffer_alias = resources.GetBufferCache().ObtainBuffer( + scheduler.Current(), gc_image_desc_a.info.data.address, + gc_image_desc_a.info.data.size, true, false); + Require(name, "clean pre-image buffer alias", + clean_buffer_alias.owner != nullptr && + clean_buffer_alias.buffer != nullptr, + "failed to create the clean cached Buffer alias"); + scheduler.Current().RetainResourceUntilFence(clean_buffer_alias.owner); + resources.GetBufferCache().DiscardGpuDirtyBytes( + gc_image_desc_a.info.data.address, gc_image_desc_a.info.data.size); + const std::array gc_images{texture_cache.FindImage(gc_image_desc_a), + texture_cache.FindImage(gc_image_desc_b)}; + Require( + name, "non-GPU image range validity", + !texture_cache.FindImageFromRange(gc_image_desc_a.info.data.address, + gc_image_desc_a.info.data.size), + "FindImageFromRange accepted an image without current GPU " + "contents"); + for (const auto image : gc_images) { + texture_cache.MarkGpuWritten(image); + } + Require(name, "GPU image range validity", + texture_cache.FindImageFromRange( + gc_image_desc_a.info.data.address, + gc_image_desc_a.info.data.size) == gc_images[0], + "FindImageFromRange rejected a clean GPU-current image"); + texture_cache.GetImage(gc_images[0]) + .InvalidateCpuWrite(gc_image_desc_a.info.data.address, + gc_image_desc_a.info.data.size); + Require( + name, "CPU-dirty image range validity", + !texture_cache.FindImageFromRange(gc_image_desc_a.info.data.address, + gc_image_desc_a.info.data.size), + "FindImageFromRange accepted CPU-dirty native contents"); + texture_cache.MarkGpuWritten(gc_images[0]); + for (size_t index = 0; index < gc_image_offsets.size(); index++) { + Libs::LibKernel::Memory::WriteBacking(base + gc_image_offsets[index], + &gc_stale_values[index], + sizeof(uint32_t)); + } + TextureCacheTestAccess::ConfigureGarbageCollection( + texture_cache, gc_images, 17, UINT64_MAX); + texture_cache.RunGarbageCollector(); + Require(name, "downloadable image pre-pressure retention", + std::ranges::all_of(gc_images, + [&](ImageId image) { + return TextureCacheTestAccess::Contains( + texture_cache, image); + }), + "GC retired safely downloadable GPU images before pressure"); + const auto gc_batch_tick = scheduler.CurrentTick(); + TextureCacheTestAccess::ConfigureGarbageCollection(texture_cache, + gc_images, 81, 0); + texture_cache.RunGarbageCollector(); + std::array gc_before_completion{}; + for (size_t index = 0; index < gc_image_offsets.size(); index++) { + Libs::LibKernel::Memory::TryReadBacking(base + gc_image_offsets[index], + &gc_before_completion[index], + sizeof(uint32_t)); + } + Require(name, "batched image pressure retirement", + std::ranges::none_of(gc_images, + [&](ImageId image) { + return TextureCacheTestAccess::Contains( + texture_cache, image); + }) && + scheduler.CurrentTick() == gc_batch_tick && + gc_before_completion == gc_stale_values, + "GC submitted per image or published a readback before GPU " + "completion"); + auto refreshed_buffer_alias = resources.GetBufferCache().ObtainBuffer( + scheduler.Current(), gc_image_desc_a.info.data.address, + gc_image_desc_a.info.data.size, false, true); + Require(name, "publication-gated Buffer reacquire", + refreshed_buffer_alias.buffer != nullptr, + "Buffer lookup did not wait for the retired image publication"); + if (refreshed_buffer_alias.owner != nullptr) { + scheduler.Current().RetainResourceUntilFence( + refreshed_buffer_alias.owner); + } + std::array gc_after_completion{}; + for (size_t index = 0; index < gc_image_offsets.size(); index++) { + Libs::LibKernel::Memory::TryReadBacking(base + gc_image_offsets[index], + &gc_after_completion[index], + sizeof(uint32_t)); + } + Require(name, "batched image readback publication", + scheduler.CurrentTick() == gc_batch_tick + 1 && + gc_after_completion == gc_image_values, + "one submission did not publish both deferred image readbacks"); + Require(name, "refreshed post-image buffer alias", + refreshed_buffer_alias.buffer != nullptr, + "failed to reacquire the cached Buffer alias after image " + "publication"); + if (refreshed_buffer_alias.owner != nullptr) { + scheduler.Current().RetainResourceUntilFence( + refreshed_buffer_alias.owner); + } + auto alias_readback = CreateHostBuffer( + name, sizeof(uint32_t), vk::BufferUsageFlagBits::eTransferDst, {0}); + const vk::BufferCopy alias_copy{refreshed_buffer_alias.offset, 0, + sizeof(uint32_t)}; + scheduler.Current().Handle().copyBuffer( + refreshed_buffer_alias.buffer, alias_readback.buffer, 1, &alias_copy); + HostReadBarrier(alias_readback.buffer, alias_readback.size, + vk::PipelineStageFlagBits::eTransfer, + vk::AccessFlagBits::eTransferWrite); + scheduler.FinishCurrent(); + Require(name, "post-image Buffer alias content", + ReadBuffer(name, alias_readback, 1) == + std::vector{gc_image_values[0]}, + "a clean cached Buffer alias survived image ownership with " + "stale native bytes"); + DestroyBuffer(&alias_readback); + + constexpr uint64_t submit_readback_offset = 0x336000; + constexpr uint32_t submit_readback_value = 0x13579bdfu; + constexpr uint32_t submit_readback_stale = 0x2468ace0u; + auto submit_readback_desc = MakeLinearDesc( + base + submit_readback_offset, sizeof(uint32_t), vk::Format::eR32Uint, + Prospero::GpuEnumValue(Prospero::BufferFormat::k32UInt), + Prospero::ImageType::kColor2D, {1, 1, 1}, 1, 4, 1); + submit_readback_desc.type = BindingType::Storage; + submit_readback_desc.view_info.usage = vk::ImageUsageFlagBits::eStorage; + TextureCacheTestAccess::SetLinearReadback(texture_cache, true); + const auto submit_readback_image = + texture_cache.FindImage(submit_readback_desc); + Require(name, "linear submit image clear", + texture_cache.ClearImageFromBuffer( + command, submit_readback_desc.info.data.address, + submit_readback_desc.info.data.size, submit_readback_value), + "failed to create GPU-current linear storage contents"); + TextureCacheTestAccess::TrackDownload(texture_cache, + submit_readback_image); + Libs::LibKernel::Memory::WriteBacking( + submit_readback_desc.info.data.address, &submit_readback_stale, + sizeof(submit_readback_stale)); + const auto submit_readback_tick = scheduler.CurrentTick(); + texture_cache.ProcessDownloadImages(); + TextureCacheTestAccess::SetLinearReadback(texture_cache, false); + uint32_t submit_before_completion = 0; + Libs::LibKernel::Memory::TryReadBacking( + submit_readback_desc.info.data.address, &submit_before_completion, + sizeof(submit_before_completion)); + Require(name, "linear submit deferred publication", + submit_before_completion == submit_readback_stale && + scheduler.CurrentTick() == submit_readback_tick, + "submit-time linear readback published synchronously"); + scheduler.FinishCurrent(); + scheduler.DrainPriorityOperations(); + uint32_t submit_after_completion = 0; + Libs::LibKernel::Memory::TryReadBacking( + submit_readback_desc.info.data.address, &submit_after_completion, + sizeof(submit_after_completion)); + Require(name, "linear submit readback reuse", + submit_after_completion == submit_readback_value && + TextureCacheTestAccess::Contains(texture_cache, + submit_readback_image) && + texture_cache.GetImage(submit_readback_image).IsGpuModified(), + "submit-time readback did not publish or retained no reusable " + "GPU-current image"); + + constexpr uint64_t linear_depth_offset = 0x33a000; + constexpr uint64_t linear_stencil_offset = 0x33b000; + constexpr uint32_t linear_depth_width = 3; + constexpr uint32_t linear_depth_height = 2; + constexpr uint32_t linear_depth_pitch = 4; + constexpr uint32_t linear_depth_layers = 2; + constexpr uint32_t linear_depth_words = + linear_depth_pitch * linear_depth_height * linear_depth_layers; + constexpr uint32_t linear_depth_clear = 0x3f400000u; + std::array linear_depth_guest{}; + for (uint32_t index = 0; index < linear_depth_words; index++) { + linear_depth_guest[index] = 0x51000000u + index; + } + constexpr std::array linear_stencil_guest{ + 0x91, 0x82, 0x73, 0x64, 0x55, 0x46, 0x37, 0x28}; + std::memcpy(memory + linear_depth_offset, linear_depth_guest.data(), + sizeof(linear_depth_guest)); + std::memcpy(memory + linear_stencil_offset, linear_stencil_guest.data(), + sizeof(linear_stencil_guest)); + auto linear_depth_desc = MakeLinearDesc( + base + linear_depth_offset, sizeof(linear_depth_guest), + vk::Format::eD32SfloatS8Uint, + Prospero::GpuEnumValue(Prospero::BufferFormat::k32Float), + Prospero::ImageType::kColor2D, + {linear_depth_width, linear_depth_height, 1}, linear_depth_layers, 4, + 1); + linear_depth_desc.type = BindingType::DepthTarget; + linear_depth_desc.info.pitch = linear_depth_pitch; + linear_depth_desc.info.stencil = {base + linear_stencil_offset, + sizeof(linear_stencil_guest)}; + linear_depth_desc.info.mip_layout[0] = {0, sizeof(linear_depth_guest), + linear_depth_pitch, + linear_depth_height}; + linear_depth_desc.view_info.aspect = + vk::ImageAspectFlagBits::eDepth | vk::ImageAspectFlagBits::eStencil; + linear_depth_desc.view_info.usage = + vk::ImageUsageFlagBits::eDepthStencilAttachment; + const auto linear_depth_image = + texture_cache.FindImage(linear_depth_desc); + Require(name, "linear depth native clear", + texture_cache.ClearImageFromBuffer( + command, linear_depth_desc.info.data.address, + linear_depth_desc.info.data.size, linear_depth_clear), + "failed to create GPU-current padded linear depth contents"); + TextureCacheTestAccess::SetLinearReadback(texture_cache, true); + TextureCacheTestAccess::TrackDownload(texture_cache, linear_depth_image); + const auto linear_depth_tick = scheduler.CurrentTick(); + texture_cache.ProcessDownloadImages(); + TextureCacheTestAccess::SetLinearReadback(texture_cache, false); + std::array linear_depth_before{}; + std::memcpy(linear_depth_before.data(), memory + linear_depth_offset, + sizeof(linear_depth_before)); + Require(name, "linear depth deferred publication", + linear_depth_before == linear_depth_guest && + scheduler.CurrentTick() == linear_depth_tick, + "linear depth readback published before GPU completion"); + scheduler.FinishCurrent(); + scheduler.DrainPriorityOperations(); + std::array linear_depth_after{}; + std::array linear_stencil_after{}; + std::memcpy(linear_depth_after.data(), memory + linear_depth_offset, + sizeof(linear_depth_after)); + std::memcpy(linear_stencil_after.data(), memory + linear_stencil_offset, + sizeof(linear_stencil_after)); + bool linear_depth_matches = true; + uint32_t linear_depth_mismatch = UINT32_MAX; + for (uint32_t layer = 0; layer < linear_depth_layers; layer++) { + for (uint32_t y = 0; y < linear_depth_height; y++) { + for (uint32_t x = 0; x < linear_depth_pitch; x++) { + const auto index = + layer * linear_depth_pitch * linear_depth_height + + y * linear_depth_pitch + x; + const auto expected = x < linear_depth_width + ? linear_depth_clear + : linear_depth_guest[index]; + if (linear_depth_after[index] != expected && + linear_depth_mismatch == UINT32_MAX) { + linear_depth_mismatch = index; + } + linear_depth_matches &= linear_depth_after[index] == expected; + } + } + } + Require(name, "linear depth native contents", + linear_depth_matches && + linear_stencil_after == linear_stencil_guest && + TextureCacheTestAccess::Contains(texture_cache, + linear_depth_image) && + texture_cache.GetImage(linear_depth_image).IsGpuModified(), + fmt::format( + "linear depth mismatch={} actual=0x{:08x} expected=0x{:08x} " + "stencil={} contains={} gpu={}", + linear_depth_mismatch, + linear_depth_mismatch == UINT32_MAX + ? 0 + : linear_depth_after[linear_depth_mismatch], + linear_depth_mismatch == UINT32_MAX + ? 0 + : (linear_depth_mismatch % linear_depth_pitch < + linear_depth_width + ? linear_depth_clear + : linear_depth_guest[linear_depth_mismatch]), + linear_stencil_after == linear_stencil_guest, + TextureCacheTestAccess::Contains(texture_cache, + linear_depth_image), + texture_cache.GetImage(linear_depth_image).IsGpuModified()) + .c_str()); + + constexpr uint64_t tiled_depth_offset = 0x380000; + constexpr uint32_t tiled_depth_width = 3; + constexpr uint32_t tiled_depth_height = 2; + constexpr uint32_t tiled_depth_pitch = 4; + constexpr uint32_t tiled_depth_layers = 2; + constexpr uint64_t tiled_depth_slice = 0x10000; + constexpr uint64_t tiled_depth_size = + tiled_depth_slice * tiled_depth_layers; + constexpr uint32_t tiled_depth_clear = 0x3f200000u; + constexpr uint32_t tiled_depth_stale = 0xdeadbeefu; + std::vector tiled_depth_guest( + tiled_depth_size / sizeof(uint32_t), tiled_depth_stale); + std::memcpy(memory + tiled_depth_offset, tiled_depth_guest.data(), + tiled_depth_size); + auto tiled_depth_desc = MakeLinearDesc( + base + tiled_depth_offset, tiled_depth_size, vk::Format::eD32Sfloat, + Prospero::GpuEnumValue(Prospero::BufferFormat::k32Float), + Prospero::ImageType::kColor2D, + {tiled_depth_width, tiled_depth_height, 1}, tiled_depth_layers, 4, 1); + tiled_depth_desc.type = BindingType::DepthTarget; + tiled_depth_desc.info.pitch = tiled_depth_pitch; + tiled_depth_desc.info.tile_mode = + Prospero::GpuEnumValue(Prospero::TileMode::kDepth); + tiled_depth_desc.info.mip_layout[0] = { + 0, tiled_depth_size, tiled_depth_pitch, tiled_depth_height}; + tiled_depth_desc.view_info.aspect = vk::ImageAspectFlagBits::eDepth; + tiled_depth_desc.view_info.usage = + vk::ImageUsageFlagBits::eDepthStencilAttachment; + const auto tiled_depth_image = texture_cache.FindImage(tiled_depth_desc); + Require(name, "tiled depth native clear", + texture_cache.ClearImageFromBuffer( + command, tiled_depth_desc.info.data.address, + tiled_depth_desc.info.data.size, tiled_depth_clear), + "failed to create GPU-current layered tiled depth contents"); + const auto [tiled_prefix, tiled_prefix_offset] = + TextureCacheTestAccess::MapDownload(texture_cache, 64, 64); + Require(name, "tiled depth download prefix", + tiled_prefix != nullptr && tiled_prefix_offset != 0, + "failed to force a nonzero reusable-download offset"); + const auto tiled_depth_tick = scheduler.CurrentTick(); + Require( + name, "tiled depth download queue", + TextureCacheTestAccess::TryDownload(texture_cache, tiled_depth_image), + "layered tiled depth readback was rejected"); + std::vector tiled_depth_before(tiled_depth_guest.size()); + std::memcpy(tiled_depth_before.data(), memory + tiled_depth_offset, + tiled_depth_size); + Require(name, "tiled depth deferred publication", + tiled_depth_before == tiled_depth_guest && + scheduler.CurrentTick() == tiled_depth_tick, + "tiled depth readback published before GPU completion"); + scheduler.FinishCurrent(); + scheduler.DrainPriorityOperations(); + std::vector tiled_depth_after(tiled_depth_guest.size()); + std::memcpy(tiled_depth_after.data(), memory + tiled_depth_offset, + tiled_depth_size); + const auto active_tiled_words = + tiled_depth_width * tiled_depth_height * tiled_depth_layers; + const auto clear_tiled_words = + std::ranges::count(tiled_depth_after, tiled_depth_clear); + const auto stale_tiled_words = + std::ranges::count(tiled_depth_after, tiled_depth_stale); + Require( + name, "tiled depth backing preservation", + clear_tiled_words == active_tiled_words && + stale_tiled_words == + tiled_depth_after.size() - active_tiled_words, + fmt::format("tiled depth readback changed clear={}/{} stale={}/{}", + clear_tiled_words, active_tiled_words, stale_tiled_words, + tiled_depth_after.size() - active_tiled_words) + .c_str()); + + TileBlockLayout depth_block{}; + Require(name, "tiled depth block layout", + TileGetBlockLayout(TileBlockFamily::Depth64KB, sizeof(uint32_t), + depth_block), + "failed to describe the tiled depth test surface"); + std::array tiled_depth_tiles{}; + for (uint32_t layer = 0; layer < tiled_depth_layers; layer++) { + const uint64_t offset = tiled_depth_slice * layer; + tiled_depth_tiles[layer] = {depth_block.family, + depth_block.bytes_per_element, + offset, + tiled_depth_slice, + offset, + tiled_depth_slice, + 0, + tiled_depth_width, + tiled_depth_height, + 1, + tiled_depth_pitch}; + tiled_depth_tiles[layer].surface_z = layer; + } + auto tiled_depth_input = + CreateHostBuffer(name, tiled_depth_size, AllFlags, tiled_depth_after); + auto tiled_depth_linear = + TextureCacheTestAccess::Tiler(texture_cache) + .Detile(tiled_depth_input.buffer, 0, tiled_depth_size, + tiled_depth_size, tiled_depth_tiles); + auto tiled_depth_output = + CreateHostBuffer(name, tiled_depth_size, AllFlags, + std::vector(tiled_depth_guest.size(), 0)); + const vk::BufferCopy tiled_depth_copy{tiled_depth_linear.offset, 0, + tiled_depth_size}; + scheduler.Current().Handle().copyBuffer(tiled_depth_linear.buffer, + tiled_depth_output.buffer, 1, + &tiled_depth_copy); + HostReadBarrier(tiled_depth_output.buffer, tiled_depth_size, + vk::PipelineStageFlagBits::eTransfer, + vk::AccessFlagBits::eTransferWrite); + scheduler.FinishCurrent(); + const auto tiled_depth_words = + ReadBuffer(name, tiled_depth_output, tiled_depth_guest.size()); + bool tiled_depth_matches = true; + for (uint32_t layer = 0; layer < tiled_depth_layers; layer++) { + for (uint32_t y = 0; y < tiled_depth_height; y++) { + for (uint32_t x = 0; x < tiled_depth_width; x++) { + const auto index = layer * (tiled_depth_slice / sizeof(uint32_t)) + + y * tiled_depth_pitch + x; + tiled_depth_matches &= + tiled_depth_words[index] == tiled_depth_clear; + } + } + } + Require(name, "tiled depth layered contents", + tiled_depth_matches && + TextureCacheTestAccess::Contains(texture_cache, + tiled_depth_image) && + texture_cache.GetImage(tiled_depth_image).IsGpuModified(), + "tiled depth readback lost a padded slice or retired its image"); + DestroyBuffer(&tiled_depth_output); + DestroyBuffer(&tiled_depth_input); + + constexpr uint64_t tiled_d16_offset = 0x3a0000; + constexpr uint64_t tiled_d16_stencil_offset = 0x3c0000; + constexpr uint16_t tiled_d16_clear = 0x8000u; + constexpr uint16_t tiled_d16_stale = 0xbeefu; + std::vector tiled_d16_guest(tiled_depth_size / sizeof(uint16_t), + tiled_d16_stale); + constexpr std::array tiled_d16_stencil{ + 0x10, 0x21, 0x32, 0x43, 0x54, 0x65, 0x76, 0x87, + 0x98, 0xa9, 0xba, 0xcb, 0xdc, 0xed, 0xfe, 0x0f}; + std::memcpy(memory + tiled_d16_offset, tiled_d16_guest.data(), + tiled_depth_size); + std::memcpy(memory + tiled_d16_stencil_offset, tiled_d16_stencil.data(), + tiled_d16_stencil.size()); + auto tiled_d16_desc = MakeLinearDesc( + base + tiled_d16_offset, tiled_depth_size, + vk::Format::eD32SfloatS8Uint, + Prospero::GpuEnumValue(Prospero::BufferFormat::k16UNorm), + Prospero::ImageType::kColor2D, + {tiled_depth_width, tiled_depth_height, 1}, tiled_depth_layers, 2, 1); + tiled_d16_desc.type = BindingType::DepthTarget; + tiled_d16_desc.info.pitch = tiled_depth_pitch; + tiled_d16_desc.info.tile_mode = + Prospero::GpuEnumValue(Prospero::TileMode::kDepth); + tiled_d16_desc.info.stencil = {base + tiled_d16_stencil_offset, + tiled_d16_stencil.size()}; + tiled_d16_desc.info.mip_layout[0] = { + 0, tiled_depth_size, tiled_depth_pitch, tiled_depth_height}; + tiled_d16_desc.view_info.aspect = + vk::ImageAspectFlagBits::eDepth | vk::ImageAspectFlagBits::eStencil; + tiled_d16_desc.view_info.usage = + vk::ImageUsageFlagBits::eDepthStencilAttachment; + const auto tiled_d16_image = texture_cache.FindImage(tiled_d16_desc); + Require(name, "tiled D16 native clear", + texture_cache.ClearImageFromBuffer( + command, tiled_d16_desc.info.data.address, + tiled_d16_desc.info.data.size, + EncodeD16AsD32(tiled_d16_clear)), + "failed to clear a layered tiled D16 fallback image"); + Require( + name, "tiled D16 download queue", + TextureCacheTestAccess::TryDownload(texture_cache, tiled_d16_image), + "layered tiled D16 fallback readback was rejected"); + scheduler.FinishCurrent(); + scheduler.DrainPriorityOperations(); + std::vector tiled_d16_after(tiled_d16_guest.size()); + std::array tiled_d16_stencil_after{}; + std::memcpy(tiled_d16_after.data(), memory + tiled_d16_offset, + tiled_depth_size); + std::memcpy(tiled_d16_stencil_after.data(), + memory + tiled_d16_stencil_offset, + tiled_d16_stencil_after.size()); + Require(name, "tiled D16 backing preservation", + std::ranges::count(tiled_d16_after, tiled_d16_clear) == + active_tiled_words && + std::ranges::count(tiled_d16_after, tiled_d16_stale) == + tiled_d16_after.size() - active_tiled_words && + tiled_d16_stencil_after == tiled_d16_stencil, + "tiled D16 fallback overwrote inactive depth or stencil backing"); + + std::vector tiled_d16_words(tiled_depth_size / + sizeof(uint32_t)); + std::memcpy(tiled_d16_words.data(), tiled_d16_after.data(), + tiled_depth_size); + std::array tiled_d16_tiles{}; + Require(name, "tiled D16 block layout", + TileGetBlockLayout(TileBlockFamily::Depth64KB, sizeof(uint16_t), + depth_block), + "failed to describe the tiled D16 test surface"); + for (uint32_t layer = 0; layer < tiled_depth_layers; layer++) { + const uint64_t offset = tiled_depth_slice * layer; + tiled_d16_tiles[layer] = {depth_block.family, + depth_block.bytes_per_element, + offset, + tiled_depth_slice, + offset, + tiled_depth_slice, + 0, + tiled_depth_width, + tiled_depth_height, + 1, + tiled_depth_pitch}; + tiled_d16_tiles[layer].surface_z = layer; + } + auto tiled_d16_input = + CreateHostBuffer(name, tiled_depth_size, AllFlags, tiled_d16_words); + auto tiled_d16_linear = + TextureCacheTestAccess::Tiler(texture_cache) + .Detile(tiled_d16_input.buffer, 0, tiled_depth_size, + tiled_depth_size, tiled_d16_tiles); + auto tiled_d16_output = + CreateHostBuffer(name, tiled_depth_size, AllFlags, + std::vector(tiled_d16_words.size(), 0)); + const vk::BufferCopy tiled_d16_copy{tiled_d16_linear.offset, 0, + tiled_depth_size}; + scheduler.Current().Handle().copyBuffer( + tiled_d16_linear.buffer, tiled_d16_output.buffer, 1, &tiled_d16_copy); + HostReadBarrier(tiled_d16_output.buffer, tiled_depth_size, + vk::PipelineStageFlagBits::eTransfer, + vk::AccessFlagBits::eTransferWrite); + scheduler.FinishCurrent(); + const auto tiled_d16_linear_words = + ReadBuffer(name, tiled_d16_output, tiled_d16_words.size()); + std::vector tiled_d16_linear_values(tiled_d16_guest.size()); + std::memcpy(tiled_d16_linear_values.data(), tiled_d16_linear_words.data(), + tiled_depth_size); + bool tiled_d16_matches = true; + for (uint32_t layer = 0; layer < tiled_depth_layers; layer++) { + for (uint32_t y = 0; y < tiled_depth_height; y++) { + for (uint32_t x = 0; x < tiled_depth_width; x++) { + const auto index = layer * (tiled_depth_slice / sizeof(uint16_t)) + + y * tiled_depth_pitch + x; + tiled_d16_matches &= + tiled_d16_linear_values[index] == tiled_d16_clear; + } + } + } + Require(name, "tiled D16 layered contents", + tiled_d16_matches && + TextureCacheTestAccess::Contains(texture_cache, + tiled_d16_image) && + texture_cache.GetImage(tiled_d16_image).IsGpuModified(), + "tiled D16 fallback lost a converted layer or retired its image"); + DestroyBuffer(&tiled_d16_output); + DestroyBuffer(&tiled_d16_input); + + constexpr uint64_t d16_fallback_offset = 0x33c000; + constexpr uint64_t d16_fallback_stencil_offset = 0x33d000; + constexpr std::array d16_fallback_values{0x0000u, 0x2468u, + 0xabcdu, 0xffffu}; + std::memcpy(memory + d16_fallback_offset, d16_fallback_values.data(), + sizeof(d16_fallback_values)); + std::memset(memory + d16_fallback_stencil_offset, 0x6d, 4); + auto d16_depth_desc = MakeLinearDesc( + base + d16_fallback_offset, sizeof(d16_fallback_values), + vk::Format::eD24UnormS8Uint, + Prospero::GpuEnumValue(Prospero::BufferFormat::k16UNorm), + Prospero::ImageType::kColor2D, {4, 1, 1}, 1, 2, 1); + d16_depth_desc.type = BindingType::DepthTarget; + d16_depth_desc.info.stencil = {base + d16_fallback_stencil_offset, 4}; + d16_depth_desc.view_info.aspect = + vk::ImageAspectFlagBits::eDepth | vk::ImageAspectFlagBits::eStencil; + d16_depth_desc.view_info.usage = + vk::ImageUsageFlagBits::eDepthStencilAttachment; + const auto d16_depth_image = texture_cache.FindImage(d16_depth_desc); + texture_cache.MarkGpuWritten(d16_depth_image); + auto d16_storage_desc = d16_depth_desc; + d16_storage_desc.type = BindingType::Storage; + d16_storage_desc.info.stencil = {}; + d16_storage_desc.info.pixel_format = vk::Format::eR16Unorm; + d16_storage_desc.view_info.format = vk::Format::eR16Unorm; + d16_storage_desc.view_info.aspect = vk::ImageAspectFlagBits::eColor; + d16_storage_desc.view_info.usage = vk::ImageUsageFlagBits::eStorage; + const auto d16_storage_image = + texture_cache.FindImage(d16_storage_desc, true); + auto d16_storage_readback = CreateHostBuffer( + name, sizeof(d16_fallback_values), + vk::BufferUsageFlagBits::eTransferDst, std::vector(2, 0)); + auto &d16_storage_native = texture_cache.GetImage(d16_storage_image); + d16_storage_native.Transit(vk::ImageLayout::eTransferSrcOptimal, + vk::AccessFlagBits2::eTransferRead, {}, + scheduler.Current().Handle()); + vk::BufferImageCopy d16_storage_copy{}; + d16_storage_copy.bufferRowLength = 4; + d16_storage_copy.imageSubresource = {vk::ImageAspectFlagBits::eColor, 0, + 0, 1}; + d16_storage_copy.imageExtent = {4, 1, 1}; + scheduler.Current().Handle().copyImageToBuffer( + d16_storage_native.backing.image, + vk::ImageLayout::eTransferSrcOptimal, d16_storage_readback.buffer, 1, + &d16_storage_copy); + HostReadBarrier(d16_storage_readback.buffer, d16_storage_readback.size, + vk::PipelineStageFlagBits::eTransfer, + vk::AccessFlagBits::eTransferWrite); + scheduler.FinishCurrent(); + const auto d16_storage_words = ReadBuffer(name, d16_storage_readback, 2); + std::array d16_storage_values{}; + std::memcpy(d16_storage_values.data(), d16_storage_words.data(), + sizeof(d16_storage_values)); + Require( + name, "D16 fallback depth/storage copy", + d16_storage_values == d16_fallback_values && + d16_storage_image != d16_depth_image && + texture_cache.GetImage(d16_storage_image).IsGpuModified(), + "D16 fallback reinterpreted four-byte host depth as two-byte color"); + DestroyBuffer(&d16_storage_readback); + + if (!texture_cache.GetImage(combined_destination_image).IsGpuModified()) { + texture_cache.MarkGpuWritten(combined_destination_image); + } + constexpr float stale_added_stencil_depth = 0.125f; + Libs::LibKernel::Memory::WriteBacking( + combined_destination.info.data.address, &stale_added_stencil_depth, + sizeof(stale_added_stencil_depth)); + const auto depth_gc_tick = scheduler.CurrentTick(); + TextureCacheTestAccess::ConfigureGarbageCollection( + texture_cache, std::array{combined_destination_image}, 81, 0); + texture_cache.RunGarbageCollector(); + float depth_before_completion = 0.0f; + Libs::LibKernel::Memory::TryReadBacking( + combined_destination.info.data.address, &depth_before_completion, + sizeof(depth_before_completion)); + const bool depth_image_retired = !TextureCacheTestAccess::Contains( + texture_cache, combined_destination_image); + const bool depth_proxy_retired = !texture_cache.FindImageFromRange( + base + added_stencil_offset, added_stencil_size, false); + Require( + name, "depth/stencil deferred pressure retirement", + depth_image_retired && depth_proxy_retired && + scheduler.CurrentTick() == depth_gc_tick && + depth_before_completion == stale_added_stencil_depth, + fmt::format( + "GC failed to retire/defer depth: image={} proxy={} tick={}/{} " + "depth={}/{}", + depth_image_retired, depth_proxy_retired, scheduler.CurrentTick(), + depth_gc_tick, depth_before_completion, stale_added_stencil_depth) + .c_str()); + scheduler.FinishCurrent(); + scheduler.DrainPriorityOperations(); + float depth_after_completion = 0.0f; + Libs::LibKernel::Memory::TryReadBacking( + combined_destination.info.data.address, &depth_after_completion, + sizeof(depth_after_completion)); + Require(name, "depth/stencil depth-plane preservation", + depth_after_completion == added_stencil_depth_value, + "GC discarded the current depth plane of a depth/stencil " + "image"); + + constexpr size_t gc_depth_pair_count = 6; + constexpr uint64_t gc_depth_pair_offset = 0x350000; + constexpr uint64_t gc_depth_pair_stride = 0x1000; + std::array gc_depth_images{}; + std::array gc_stencil_images{}; + std::array gc_depth_lru{}; + for (size_t index = 0; index < gc_depth_pair_count; index++) { + auto depth = MakeLinearDesc( + base + gc_depth_pair_offset + index * gc_depth_pair_stride, + sizeof(float), vk::Format::eD32SfloatS8Uint, + Prospero::GpuEnumValue(Prospero::BufferFormat::k32Float), + Prospero::ImageType::kColor2D, {1, 1, 1}, 1, 4, 1); + depth.type = BindingType::DepthTarget; + depth.info.stencil = {depth.info.data.address + + gc_depth_pair_stride / 2, + sizeof(uint32_t)}; + depth.view_info.format = vk::Format::eD32SfloatS8Uint; + depth.view_info.aspect = + vk::ImageAspectFlagBits::eDepth | vk::ImageAspectFlagBits::eStencil; + depth.view_info.usage = vk::ImageUsageFlagBits::eDepthStencilAttachment; + gc_depth_images[index] = texture_cache.FindImage(depth); + TextureCacheTestAccess::AssociateStencil( + texture_cache, gc_depth_images[index], depth.info.stencil); + gc_stencil_images[index] = texture_cache.FindImageFromRange( + depth.info.stencil.address, depth.info.stencil.size, false); + gc_depth_lru[index * 2] = gc_depth_images[index]; + gc_depth_lru[index * 2 + 1] = gc_stencil_images[index]; + } + Require( + name, "depth/stencil GC fixtures", + std::ranges::all_of(gc_depth_images, + [](ImageId image) { return bool{image}; }) && + std::ranges::all_of(gc_stencil_images, + [](ImageId image) { return bool{image}; }), + "failed to create six depth/stencil association pairs"); + TextureCacheTestAccess::ConfigureGarbageCollection( + texture_cache, gc_depth_lru, 81, UINT64_MAX); + texture_cache.RunGarbageCollector(); + bool gc_depth_budget = true; + for (size_t index = 0; index < gc_depth_pair_count; index++) { + const bool expected_live = index == gc_depth_pair_count - 1; + gc_depth_budget &= + TextureCacheTestAccess::Contains( + texture_cache, gc_depth_images[index]) == expected_live && + TextureCacheTestAccess::Contains( + texture_cache, gc_stencil_images[index]) == expected_live; + } + Require(name, "depth/stencil GC traversal budget", gc_depth_budget, + "recursive association deletion stopped LRU traversal or " + "exceeded the ten-entry deletion budget"); + + constexpr uint64_t large_offset = 0x400000; + constexpr uint32_t large_width = 4096; + constexpr uint32_t large_height = 2047; + constexpr uint64_t large_size = + uint64_t{large_width} * large_height * sizeof(uint32_t); + static_assert(large_size < (32ull << 20)); + std::memset(memory + large_offset, 0, static_cast(large_size)); + auto large_desc = MakeLinearDesc( + base + large_offset, large_size, vk::Format::eR8G8B8A8Unorm, + Prospero::GpuEnumValue(Prospero::BufferFormat::k8_8_8_8UNorm), + Prospero::ImageType::kColor2D, {large_width, large_height, 1}, 1, 4, + 1); + const auto RunLargeReadback = [&](uint32_t clear_value) { + const auto image = texture_cache.FindImage(large_desc); + Require(name, "near-capacity image clear", + texture_cache.ClearImageFromBuffer( + command, large_desc.info.data.address, + large_desc.info.data.size, clear_value), + "failed to write the near-capacity readback source image"); + const std::array sample_offsets{ + 0, large_size / 2, large_size - sizeof(uint32_t)}; + constexpr uint32_t stale = 0; + for (const auto offset : sample_offsets) { + Libs::LibKernel::Memory::WriteBacking(base + large_offset + offset, + &stale, sizeof(stale)); + } + TextureCacheTestAccess::ConfigureGarbageCollection( + texture_cache, std::array{image}, 81, 0); + const auto tick = scheduler.CurrentTick(); + texture_cache.RunGarbageCollector(); + const auto handle = + BufferCacheTestAccess::DownloadBuffer(resources.GetBufferCache()) + .Handle(); + Require( + name, "near-capacity deferred retirement", + !TextureCacheTestAccess::Contains(texture_cache, image) && + scheduler.CurrentTick() == tick, + "near-capacity readback was rejected or synchronously submitted"); + scheduler.FinishCurrent(); + scheduler.DrainPriorityOperations(); + bool content = true; + for (const auto offset : sample_offsets) { + uint32_t value = 0; + Libs::LibKernel::Memory::TryReadBacking(base + large_offset + offset, + &value, sizeof(value)); + content &= value == clear_value; + } + Require( + name, "near-capacity readback content", content, + "near-capacity image readback did not publish its GPU contents"); + return handle; + }; + const auto large_download = RunLargeReadback(0xa5a5a5a5u); + const auto reused_large_download = RunLargeReadback(0x5a5a5a5au); + Require(name, "near-capacity shared download reuse", + large_download == reused_large_download, + "successive near-capacity image transfers replaced the shared " + "download buffer"); + + for (auto &output : ms_observer_outputs) { + DestroyBuffer(&output); + } + DestroyBuffer(&layered_readback); + DestroyBuffer(&bgra16_readback); + DestroyBuffer(&mixed_source_readback); + DestroyBuffer(&partial_cpu_refresh_readback); + DestroyBuffer(&mirror_cpu_readback); + DestroyBuffer(&exact_buffer_readback); + m_device.destroyDescriptorPool(observer_pool, nullptr); + m_device.destroyPipeline(observer_depth_pipeline, nullptr); + m_device.destroyPipelineLayout(observer_pipeline_layout, nullptr); + m_device.destroyDescriptorSetLayout(observer_set_layout, nullptr); + m_device.destroyShaderModule(ms_depth_module, nullptr); + + resources.SetGpu(nullptr); + resources.UnmapMemory(base, allocation_size, GpuAccess::ReadWrite); + scheduler.Finish(); + } + gpu.Shutdown(); + Require(name, "unmap direct backing", + Libs::LibKernel::Memory::KernelMunmap(base, allocation_size) == 0, + "cache direct-memory mapping release failed"); + Require(name, "release direct backing", + Libs::LibKernel::Memory::KernelReleaseDirectMemory( + direct_offset, allocation_size) == 0, + "cache direct-memory allocation release failed"); + std::printf("[host] %-32s ok\n", name); + } + + void CheckBgra16Readback() { + constexpr const char *name = "Bgra16Readback"; + constexpr uintptr_t base = 0x0000000204000000ull; + constexpr uint64_t allocation_size = 0x200000; + constexpr uint32_t format = + Prospero::GpuEnumValue(Prospero::BufferFormat::k16_16_16_16Float); + constexpr uint32_t tile = + Prospero::GpuEnumValue(Prospero::TileMode::kRenderTarget); + EnsureRuntimeContext(); + + int64_t direct_offset = -1; + Require(name, "direct allocation", + Libs::LibKernel::Memory::KernelAllocateDirectMemory( + 0, Libs::LibKernel::Memory::KernelGetDirectMemorySize(), + allocation_size, allocation_size, 0, &direct_offset) == 0, + "BGRA16 direct-memory allocation failed"); + void *mapped = reinterpret_cast(base); + Require(name, "direct mapping", + Libs::LibKernel::Memory::KernelMapDirectMemory( + &mapped, allocation_size, 0x3, 0x10, direct_offset, + allocation_size) == 0 && + mapped == reinterpret_cast(base), + "BGRA16 fixed mapping failed"); + + CommandScheduler scheduler(Renderer(), m_runtime_context); + HW::Context registers{}; + HW::UserConfig user_config{}; + HW::Shader shaders{}; + scheduler.Begin(registers, user_config, shaders); + { + GpuResourceManager resources(m_runtime_context, scheduler); + resources.MapMemory(base, allocation_size, GpuAccess::ReadWrite); + const uint32_t pitch = TileGetTexturePitch(format, 1, 1, tile); + TileSizeAlign total{}; + TileSizeOffset mip{}; + TilePaddedSize padded{}; + TileGetTextureSize(format, 1, 1, pitch, 1, tile, &total, &mip, &padded); + Require(name, "tiled layout", + total.size >= 8 && total.size <= allocation_size, + "BGRA16 tiled layout exceeds its guest allocation"); + std::memset(mapped, 0x5a, total.size); + + ImageDesc desc{}; + desc.type = BindingType::RenderTarget; + desc.info.data = {base, total.size}; + desc.info.pixel_format = vk::Format::eR16G16B16A16Sfloat; + desc.info.guest_format = format; + desc.info.type = Prospero::ImageType::kColor2D; + desc.info.extent = {1, 1, 1}; + desc.info.resources = {1, 1}; + desc.info.pitch = pitch; + desc.info.bytes_per_block = 8; + desc.info.samples = 1; + desc.info.tile_mode = tile; + desc.info.bgra16 = true; + desc.info.mip_layout[0] = {mip.offset, mip.size, pitch, 1}; + desc.view_info.format = desc.info.pixel_format; + desc.view_info.type = vk::ImageViewType::e2D; + desc.view_info.aspect = vk::ImageAspectFlagBits::eColor; + desc.view_info.usage = vk::ImageUsageFlagBits::eColorAttachment; + + auto &cache = resources.GetTextureCache(); + const auto id = cache.FindImage(desc); + auto &image = cache.GetImage(id); + image.Transit(vk::ImageLayout::eTransferDstOptimal, + vk::AccessFlagBits2::eTransferWrite, {}, + scheduler.Current().Handle()); + vk::ClearColorValue clear{}; + clear.float32[0] = 1.0f; + clear.float32[1] = 2.0f; + clear.float32[2] = 3.0f; + clear.float32[3] = 4.0f; + const vk::ImageSubresourceRange range{vk::ImageAspectFlagBits::eColor, 0, + 1, 0, 1}; + scheduler.Current().Handle().clearColorImage( + image.backing.image, vk::ImageLayout::eTransferDstOptimal, clear, + range); + cache.MarkGpuWritten(id); + Require(name, "guest readback queue", + TextureCacheTestAccess::TryDownload(cache, id), + "tiled BGRA16 guest readback was rejected"); + Require(name, "Buffer mirror", + cache.SynchronizeImageToBuffer(base, total.size), + "tiled BGRA16 Buffer mirror was rejected"); + + auto mirror = resources.GetBufferCache().ObtainBuffer( + scheduler.Current(), base, 8, false, true); + Require(name, "mirror owner", + mirror.buffer != nullptr && mirror.owner != nullptr, + "tiled BGRA16 mirror has no BufferCache owner"); + scheduler.Current().RetainResourceUntilFence(mirror.owner); + auto mirror_readback = CreateHostBuffer( + name, 8, vk::BufferUsageFlagBits::eTransferDst, {0, 0}); + const vk::BufferCopy copy{mirror.offset, 0, 8}; + scheduler.Current().Handle().copyBuffer(mirror.buffer, + mirror_readback.buffer, 1, ©); + vk::BufferMemoryBarrier barrier{}; + barrier.sType = vk::StructureType::eBufferMemoryBarrier; + barrier.srcAccessMask = vk::AccessFlagBits::eTransferWrite; + barrier.dstAccessMask = vk::AccessFlagBits::eHostRead; + barrier.srcQueueFamilyIndex = VK_QUEUE_FAMILY_IGNORED; + barrier.dstQueueFamilyIndex = VK_QUEUE_FAMILY_IGNORED; + barrier.buffer = mirror_readback.buffer; + barrier.size = mirror_readback.size; + scheduler.Current().Handle().pipelineBarrier( + vk::PipelineStageFlagBits::eTransfer, + vk::PipelineStageFlagBits::eHost, {}, 0, nullptr, 1, &barrier, 0, + nullptr); + scheduler.Finish(); + scheduler.DrainPriorityOperations(); + + std::vector backing(total.size); + Require(name, "guest backing", + Libs::LibKernel::Memory::TryReadBacking(base, backing.data(), + backing.size()), + "tiled BGRA16 guest backing is unreadable"); + const std::array expected{0x4200u, 0x4000u, 0x3c00u, + 0x4400u}; + std::array observed{}; + std::memcpy(observed.data(), backing.data(), sizeof(observed)); + Require(name, "guest component order", + observed == expected && + std::all_of(backing.begin() + sizeof(observed), backing.end(), + [](uint8_t value) { return value == 0x5a; }), + "tiled BGRA16 guest readback changed component order or padding"); + Require(name, "mirror component order", + ReadBuffer(name, mirror_readback, 2) == + std::vector{0x40004200u, 0x44003c00u}, + "tiled BGRA16 Buffer mirror changed guest component order"); + DestroyBuffer(&mirror_readback); + resources.UnmapMemory(base, allocation_size, GpuAccess::ReadWrite); + scheduler.Finish(); + } + Require(name, "unmap", + Libs::LibKernel::Memory::KernelMunmap(base, allocation_size) == 0, + "BGRA16 fixed mapping release failed"); + Require(name, "release", + Libs::LibKernel::Memory::KernelReleaseDirectMemory( + direct_offset, allocation_size) == 0, + "BGRA16 direct-memory release failed"); + std::printf("[gpu] %-32s ok\n", name); + } + + void CheckRenderExecutorStencilBindingDiscovery() { + constexpr const char *name = "RenderExecutorStencilBindingDiscovery"; + constexpr uintptr_t base = 0x0000000203600000ull; + constexpr uint64_t allocation_size = 0x180000; + constexpr uint64_t allocation_alignment = 0x10000; + constexpr uint64_t depth_address = base + 0x40000; + constexpr uint64_t stencil_address = base + 0x70000; + EnsureRuntimeContext(); + + int64_t direct_offset = -1; + Require(name, "direct allocation", + Libs::LibKernel::Memory::KernelAllocateDirectMemory( + 0, Libs::LibKernel::Memory::KernelGetDirectMemorySize(), + allocation_size, allocation_alignment, 0, &direct_offset) == 0, + "descriptor discovery direct-memory allocation failed"); + void *mapped = reinterpret_cast(base); + Require(name, "direct mapping", + Libs::LibKernel::Memory::KernelMapDirectMemory( + &mapped, allocation_size, 0x3, 0x10, direct_offset, + allocation_alignment) == 0 && + mapped == reinterpret_cast(base), + "descriptor discovery fixed mapping failed"); + std::memset(mapped, 0, allocation_size); + + { + RenderContext context(m_runtime_context); + auto &scheduler = context.GetCommandScheduler(); + HW::Context registers{}; + HW::UserConfig user_config{}; + HW::Shader shaders{}; + scheduler.Begin(registers, user_config, shaders); + auto &resources = context.GetGpuResources(); + auto &texture_cache = resources.GetTextureCache(); + auto &executor = context.GetRenderExecutor(); + resources.MapMemory(base, allocation_size, GpuAccess::ReadWrite); + + constexpr auto stencil_format = + Prospero::GpuEnumValue(Prospero::BufferFormat::k8UInt); + constexpr auto linear = + Prospero::GpuEnumValue(Prospero::TileMode::kLinear); + const auto stencil_pitch = + TileGetTexturePitch(stencil_format, 1, 1, linear); + TileSizeAlign stencil_layout{}; + TileGetTextureTotalSize(stencil_format, 1, 1, 1, stencil_pitch, 1, linear, + false, stencil_layout); + Require(name, "stencil footprint", + stencil_layout.size != 0 && stencil_layout.align != 0 && + (stencil_address & (stencil_layout.align - 1u)) == 0, + "linear stencil descriptor produced an invalid footprint"); + + ShaderRecompiler::IR::Program null_program{}; + null_program.stage = ShaderType::Compute; + null_program.resource_tracking_complete = true; + ShaderRecompiler::IR::ImageResource null_resource{}; + null_resource.kind = ShaderRecompiler::IR::ResourceKind::ImageUint; + null_resource.dimension = + ShaderRecompiler::Decoder::ImageDimension::Dim2D; + null_resource.read = true; + null_program.info.images.push_back(null_resource); + auto null_volume = null_resource; + null_volume.dimension = ShaderRecompiler::Decoder::ImageDimension::Dim3D; + null_program.info.images.push_back(null_volume); + auto null_storage = null_resource; + null_storage.kind = ShaderRecompiler::IR::ResourceKind::StorageImageUint; + null_storage.read = false; + null_storage.written = true; + null_program.info.images.push_back(null_storage); + ShaderRecompiler::IR::ResourceSnapshot null_snapshot{}; + ShaderRecompiler::IR::DescriptorValue null_descriptor{}; + null_descriptor.dword_count = 8; + null_snapshot.images.assign(3, null_descriptor); + ShaderStageRuntime null_runtime{ + std::make_shared( + std::move(null_program)), + std::make_shared( + std::move(null_snapshot))}; + auto null_bindings = executor.PrepareBindings( + scheduler.Current(), null_runtime, vk::ShaderStageFlagBits::eCompute, + DescriptorCache::Stage::Compute); + executor.RebindImages(scheduler.Current(), null_bindings); + Require(name, "null descriptor count", + null_bindings.resources.images.size() == 3, + "null descriptor preparation lost an image binding"); + const auto null_image_id = null_bindings.resources.images[0].image_id; + const auto &null_image = texture_cache.GetImage(null_image_id); + Require(name, "one null image per format", + null_bindings.resources.images[1].image_id == null_image_id && + null_bindings.resources.images[2].image_id == null_image_id && + TextureCacheTestAccess::NullImageCount(texture_cache) == 1, + "null view dimension or usage created another image allocation"); + Require(name, "null descriptors share final cache acquisition", + null_image.info.data.Empty() && !null_image.binding.is_bound && + !null_image.binding.is_target && + !null_image.binding.needs_rebind && + null_image.usage.texture && null_image.usage.storage && + null_image.IsGpuModified() && + !TextureCacheTestAccess::PendingDownload(texture_cache, + null_image_id), + "the shared null image did not preserve texture/storage " + "acquisition without guest readback or RenderExecutor ownership"); + RenderExecutorTestAccess::CommitBindings(executor, scheduler.Current(), + null_bindings); + Require(name, "null descriptor general layouts", + std::ranges::all_of( + null_bindings.resources.images, + [](const auto &binding) { + const auto info = + DescriptorCacheTestAccess::MakeImageInfo(binding); + return binding.layout == vk::ImageLayout::eGeneral && + info.imageView == binding.image_view && + info.imageLayout == binding.layout; + }) && + null_image.backing.state.layout == vk::ImageLayout::eGeneral, + "shared sampled/storage null descriptors did not retain " + "the general layout through descriptor generation"); + + constexpr uint64_t target_mip_size = 0x10000; + const auto make_target_desc = [&](uint64_t address, uint64_t size, + vk::Extent3D extent) { + ImageDesc desc{}; + desc.type = BindingType::RenderTarget; + desc.info.data = {address, size}; + desc.info.pixel_format = vk::Format::eR32Uint; + desc.info.guest_format = + Prospero::GpuEnumValue(Prospero::BufferFormat::k32UInt); + desc.info.type = Prospero::ImageType::kColor2D; + desc.info.extent = extent; + desc.info.resources = {1, 1}; + desc.info.pitch = extent.width; + desc.info.bytes_per_block = 4; + desc.info.samples = 1; + desc.info.tile_mode = linear; + desc.info.mip_layout[0] = {0, size, extent.width, extent.height}; + desc.view_info.format = desc.info.pixel_format; + desc.view_info.type = vk::ImageViewType::e2D; + desc.view_info.aspect = vk::ImageAspectFlagBits::eColor; + desc.view_info.usage = vk::ImageUsageFlagBits::eColorAttachment; + return desc; + }; + + TextureCacheTestAccess::SetLinearReadback(texture_cache, true); + ShaderTextureResource storage{}; + const uint64_t storage_address = base + 0x60000; + constexpr uint32_t storage_native_value = 0x13579bdfu; + constexpr uint32_t storage_stale_value = 0x2468ace0u; + std::memcpy(reinterpret_cast(mapped) + 0x60000, + &storage_native_value, sizeof(storage_native_value)); + const uint64_t encoded_storage_address = storage_address >> 8u; + storage.fields[0] = static_cast(encoded_storage_address); + storage.fields[1] = + static_cast(encoded_storage_address >> 32u) | + (stencil_format << 20u); + storage.fields[3] = + DstSel(4, 5, 6, 7) | (linear << 20u) | + (Prospero::GpuEnumValue(Prospero::ImageType::kColor2D) << 28u); + storage.fields[5] = 0x00700000u; + ShaderRecompiler::IR::Program storage_program{}; + storage_program.stage = ShaderType::Vertex; + storage_program.resource_tracking_complete = true; + ShaderRecompiler::IR::ImageResource storage_resource{}; + storage_resource.kind = + ShaderRecompiler::IR::ResourceKind::StorageImageUint; + storage_resource.dimension = + ShaderRecompiler::Decoder::ImageDimension::Dim2D; + storage_resource.written = true; + storage_program.info.images.push_back(storage_resource); + ShaderRecompiler::IR::DescriptorValue storage_descriptor{}; + std::copy(std::begin(storage.fields), std::end(storage.fields), + storage_descriptor.dwords.begin()); + storage_descriptor.dword_count = 8; + ShaderRecompiler::IR::ResourceSnapshot storage_snapshot{}; + storage_snapshot.images.push_back(storage_descriptor); + ShaderStageRuntime storage_runtime{ + std::make_shared( + std::move(storage_program)), + std::make_shared( + std::move(storage_snapshot))}; + auto storage_discovery = executor.PrepareBindings( + scheduler.Current(), storage_runtime, + vk::ShaderStageFlagBits::eVertex, DescriptorCache::Stage::Vertex); + const auto storage_id = storage_discovery.resources.images[0].image_id; + Require(name, "storage prefetch purity", + storage_discovery.resources.images[0].image_view == nullptr && + texture_cache.GetImage(storage_id).binding.is_bound && + !texture_cache.GetImage(storage_id).usage.storage && + !TextureCacheTestAccess::PendingDownload(texture_cache, + storage_id), + "the first descriptor pass performed final storage acquisition"); + RenderExecutorTestAccess::ResetBindings(executor); + + ShaderRecompiler::IR::Program sampled_program{}; + sampled_program.stage = ShaderType::Pixel; + sampled_program.resource_tracking_complete = true; + ShaderRecompiler::IR::ImageResource sampled_resource{}; + sampled_resource.kind = ShaderRecompiler::IR::ResourceKind::ImageUint; + sampled_resource.dimension = + ShaderRecompiler::Decoder::ImageDimension::Dim2D; + sampled_resource.read = true; + sampled_program.info.images.push_back(sampled_resource); + ShaderRecompiler::IR::ResourceSnapshot sampled_snapshot{}; + sampled_snapshot.images.push_back(storage_descriptor); + ShaderStageRuntime sampled_runtime{ + std::make_shared( + std::move(sampled_program)), + std::make_shared( + std::move(sampled_snapshot))}; + + constexpr uint64_t ordered_sampled_address = base + 0x10000; + const uint32_t ordered_sampled_value = 0x89abcdefu; + std::memcpy(reinterpret_cast(mapped) + 0x10000, + &ordered_sampled_value, sizeof(ordered_sampled_value)); + auto ordered_sampled = storage; + const auto encoded_ordered_address = ordered_sampled_address >> 8u; + ordered_sampled.fields[0] = + static_cast(encoded_ordered_address); + ordered_sampled.fields[1] = + static_cast(encoded_ordered_address >> 32u) | + (stencil_format << 20u); + ShaderRecompiler::IR::DescriptorValue ordered_descriptor{}; + std::copy(std::begin(ordered_sampled.fields), + std::end(ordered_sampled.fields), + ordered_descriptor.dwords.begin()); + ordered_descriptor.dword_count = 8; + ShaderRecompiler::IR::ResourceSnapshot ordered_snapshot{}; + ordered_snapshot.images.push_back(ordered_descriptor); + ShaderStageRuntime ordered_sampled_runtime{ + sampled_runtime.program, + std::make_shared( + std::move(ordered_snapshot))}; + auto ordered_bindings = RenderExecutorTestAccess::PrepareGraphicsBindings( + executor, scheduler.Current(), storage_runtime, + ordered_sampled_runtime, true); + const auto ordered_sampled_id = + ordered_bindings.pixel->resources.images[0].image_id; + Require(name, "VS-before-PS retained-owner order", + ordered_bindings.vertex.resources.images[0].image_id == + storage_id && + ordered_sampled_id != storage_id && + RenderExecutorTestAccess::BoundImagesInOrder( + executor, + TextureCacheTestAccess::Owner(texture_cache, storage_id), + TextureCacheTestAccess::Owner(texture_cache, + ordered_sampled_id)), + "production graphics binding did not retain vertex resources " + "before pixel resources"); + RenderExecutorTestAccess::ResetBindings(executor); + + auto graphics_bindings = + RenderExecutorTestAccess::PrepareGraphicsBindings( + executor, scheduler.Current(), storage_runtime, sampled_runtime, + true); + const auto &storage_binding = + graphics_bindings.vertex.resources.images[0]; + const auto &sampled_binding = + graphics_bindings.pixel->resources.images[0]; + Require(name, "storage final acquisition", + storage_binding.image_view != nullptr && + texture_cache.GetImage(storage_id).usage.storage && + texture_cache.GetImage(storage_id).IsGpuModified() && + TextureCacheTestAccess::PendingDownload(texture_cache, + storage_id), + "the production graphics binding path did not establish GPU " + "ownership before storage readback"); + Require(name, "VS-to-PS image acquisition order", + storage_binding.image_id == storage_id && + sampled_binding.image_id == storage_id && + texture_cache.GetImage(storage_id).IsGpuModified() && + texture_cache.GetImage(storage_id).usage.storage && + texture_cache.GetImage(storage_id).usage.texture, + "the production graphics binding path did not complete vertex " + "storage acquisition before pixel sampling"); + RenderExecutorTestAccess::ResetBindings(executor); + + auto vertex_sampled_program = + std::make_shared( + *sampled_runtime.program); + vertex_sampled_program->stage = ShaderType::Vertex; + ShaderStageRuntime vertex_sampled_runtime{ + std::move(vertex_sampled_program), sampled_runtime.resources}; + auto pixel_storage_program = + std::make_shared( + *storage_runtime.program); + pixel_storage_program->stage = ShaderType::Pixel; + ShaderStageRuntime pixel_storage_runtime{std::move(pixel_storage_program), + storage_runtime.resources}; + auto writable_alias_bindings = + RenderExecutorTestAccess::PrepareGraphicsBindings( + executor, scheduler.Current(), vertex_sampled_runtime, + pixel_storage_runtime, true); + Require(name, "late writable alias promotion", + texture_cache.GetImage(storage_id).binding.force_general, + "an already sampled image was not promoted when a later " + "storage alias bound the same backing"); + RenderExecutorTestAccess::CommitBindings(executor, scheduler.Current(), + writable_alias_bindings.vertex); + RenderExecutorTestAccess::CommitBindings(executor, scheduler.Current(), + *writable_alias_bindings.pixel); + Require(name, "forced-general descriptor capture", + writable_alias_bindings.vertex.resources.images[0].layout == + vk::ImageLayout::eGeneral && + writable_alias_bindings.pixel->resources.images[0].layout == + vk::ImageLayout::eGeneral && + DescriptorCacheTestAccess::MakeImageInfo( + writable_alias_bindings.vertex.resources.images[0]) + .imageLayout == vk::ImageLayout::eGeneral && + DescriptorCacheTestAccess::MakeImageInfo( + writable_alias_bindings.pixel->resources.images[0]) + .imageLayout == vk::ImageLayout::eGeneral && + texture_cache.GetImage(storage_id).backing.state.layout == + vk::ImageLayout::eGeneral, + "sampled/storage aliases did not retain the promoted general " + "layout in both generated descriptors"); + RenderExecutorTestAccess::ResetBindings(executor); + + const std::array split_mip_data{0x01020304u, 0x05060708u, + 0x090a0b0cu}; + std::memcpy(reinterpret_cast(mapped) + 0x50000, + split_mip_data.data(), sizeof(split_mip_data)); + ImageDesc split_desc{}; + split_desc.type = BindingType::Texture; + split_desc.info.data = {base + 0x50000, sizeof(split_mip_data)}; + split_desc.info.pixel_format = vk::Format::eR32Uint; + split_desc.info.guest_format = + Prospero::GpuEnumValue(Prospero::BufferFormat::k32UInt); + split_desc.info.type = Prospero::ImageType::kColor2D; + split_desc.info.extent = {2, 1, 1}; + split_desc.info.resources = {2, 1}; + split_desc.info.pitch = 2; + split_desc.info.bytes_per_block = 4; + split_desc.info.samples = 1; + split_desc.info.tile_mode = linear; + split_desc.info.mip_layout[0] = {0, 8, 2, 1}; + split_desc.info.mip_layout[1] = {8, 4, 1, 1}; + split_desc.view_info.format = split_desc.info.pixel_format; + split_desc.view_info.type = vk::ImageViewType::e2D; + split_desc.view_info.aspect = vk::ImageAspectFlagBits::eColor; + split_desc.view_info.level_count = 2; + split_desc.view_info.layer_count = 1; + split_desc.view_info.usage = vk::ImageUsageFlagBits::eSampled; + const auto split_id = texture_cache.FindImage(split_desc); + + auto split_storage_desc = split_desc; + split_storage_desc.type = BindingType::Storage; + split_storage_desc.view_info.base_level = 0; + split_storage_desc.view_info.level_count = 1; + split_storage_desc.view_info.usage = vk::ImageUsageFlagBits::eStorage; + auto split_sampled_desc = split_desc; + split_sampled_desc.view_info.base_level = 1; + split_sampled_desc.view_info.level_count = 1; + split_sampled_desc.view_info.usage = vk::ImageUsageFlagBits::eSampled; + + auto split_program = std::make_shared(); + split_program->stage = ShaderType::Vertex; + split_program->resource_tracking_complete = true; + split_program->info.images = {storage_resource, sampled_resource}; + DescriptorCache::PreparedBindings split_bindings{}; + split_bindings.program = split_program; + split_bindings.snapshot = + std::make_shared(); + split_bindings.shader_stage = vk::ShaderStageFlagBits::eVertex; + split_bindings.stage = DescriptorCache::Stage::Vertex; + split_bindings.resources.images.push_back( + {split_id, texture_cache.FindTexture(split_id, split_storage_desc), + split_storage_desc}); + split_bindings.resources.images.push_back( + {split_id, texture_cache.FindTexture(split_id, split_sampled_desc), + split_sampled_desc}); + RenderExecutorTestAccess::CommitBindings(executor, scheduler.Current(), + split_bindings); + const auto &split_image = texture_cache.GetImage(split_id); + Require(name, "per-binding subresource layouts", + split_bindings.resources.images[0].layout == + vk::ImageLayout::eGeneral && + split_bindings.resources.images[1].layout == + vk::ImageLayout::eShaderReadOnlyOptimal && + DescriptorCacheTestAccess::MakeImageInfo( + split_bindings.resources.images[0]) + .imageLayout == vk::ImageLayout::eGeneral && + DescriptorCacheTestAccess::MakeImageInfo( + split_bindings.resources.images[1]) + .imageLayout == + vk::ImageLayout::eShaderReadOnlyOptimal && + split_image.backing.subresource_states.size() == 2 && + split_image.backing.subresource_states[0].layout == + vk::ImageLayout::eGeneral && + split_image.backing.subresource_states[1].layout == + vk::ImageLayout::eShaderReadOnlyOptimal, + "descriptor layouts were queried after a later mip transition " + "instead of being captured at each binding"); + + Libs::LibKernel::Memory::WriteBacking( + storage_address, &storage_stale_value, sizeof(storage_stale_value)); + texture_cache.ProcessDownloadImages(); + Require( + name, "storage acquisition download consumption", + !TextureCacheTestAccess::PendingDownload(texture_cache, storage_id), + "submit-time processing retained an acquired storage request"); + RenderExecutorTestAccess::ResetBindings(executor); + scheduler.FinishCurrent(); + scheduler.DrainPriorityOperations(); + uint32_t storage_downloaded_value = 0; + Libs::LibKernel::Memory::TryReadBacking(storage_address, + &storage_downloaded_value, + sizeof(storage_downloaded_value)); + const uint32_t storage_expected_value = + (storage_stale_value & 0xffffff00u) | (storage_native_value & 0xffu); + Require(name, "storage acquisition readback window", + storage_downloaded_value == storage_expected_value && + texture_cache.GetImage(storage_id).IsGpuModified(), + "readback enrollment was consumed before final storage " + "ownership became visible"); + + constexpr uint64_t phased_depth_address = base + 0x90000; + constexpr uint64_t phased_htile_address = base + 0xb0000; + HW::DepthRenderTarget phased_depth_target{}; + phased_depth_target.z_info.format = + Prospero::GpuEnumValue(Prospero::DepthFormat::kZ32F); + phased_depth_target.z_info.tile_surface_enable = true; + phased_depth_target.z_info.zrange_precision = 1; + phased_depth_target.stencil_info.tile_stencil_disable = true; + phased_depth_target.z_read_base_addr = phased_depth_address; + phased_depth_target.z_write_base_addr = phased_depth_address; + phased_depth_target.htile_data_base_addr = phased_htile_address; + phased_depth_target.size.valid = true; + registers.SetDepthRenderTarget(phased_depth_target); + HW::DepthControl phased_depth_control{}; + phased_depth_control.z_enable = true; + phased_depth_control.z_write_enable = true; + registers.SetDepthControl(phased_depth_control); + HW::RenderControl phased_render_control{}; + registers.SetRenderControl(phased_render_control); + + RenderDepthInfo phased_depth{}; + RenderExecutorTestAccess::ResolveRenderDepthTarget( + executor, 1, scheduler.Current(), phased_depth); + Require( + name, "HTile discovery purity", + phased_depth.image_id && phased_depth.htile && + !phased_depth.depth_clear_enable && + !phased_depth.depth_meta_clear_enable && + !texture_cache.IsMeta(phased_depth.htile_buffer_vaddr) && + !texture_cache.GetImage(phased_depth.image_id).IsGpuModified() && + !texture_cache.GetImage(phased_depth.image_id).usage.depth_target, + "depth discovery registered, consumed, or GPU-owned HTile " + "state before final acquisition"); + RenderColorInfo no_color{}; + auto phased_rendering = RenderExecutorTestAccess::AcquireRenderTargets( + executor, scheduler.Current(), &no_color, 0, phased_depth); + Require( + name, "HTile final acquisition", + phased_depth.image_view != nullptr && + phased_rendering.num_color_attachments == 0 && + phased_rendering.depth_stencil_attachment.image_view == + phased_depth.image_view && + phased_rendering.depth_stencil_attachment.has_depth && + phased_depth.depth_meta_clear_enable && + phased_depth.depth_load_clear_enable && + texture_cache.IsMeta(phased_depth.htile_buffer_vaddr) && + !texture_cache.IsMetaCleared( + phased_depth.htile_buffer_vaddr, + phased_depth.desc.view_info.base_layer) && + texture_cache.GetImage(phased_depth.image_id).IsGpuModified() && + texture_cache.GetImage(phased_depth.image_id).usage.depth_target, + "final depth acquisition did not publish and consume HTile " + "state at the final acquisition boundary"); + RenderExecutorTestAccess::ResetBindings(executor); + + auto video_subresource = + make_target_desc(base + 0x20000, target_mip_size, {1, 1, 1}); + video_subresource.type = BindingType::VideoOut; + video_subresource.info.pixel_format = vk::Format::eR8G8B8A8Srgb; + video_subresource.info.guest_format = + Prospero::GpuEnumValue(Prospero::BufferFormat::k8_8_8_8Srgb); + video_subresource.view_info.format = video_subresource.info.pixel_format; + video_subresource.view_info.usage = vk::ImageUsageFlagBits::eSampled; + const auto video_subresource_id = + texture_cache.FindImage(video_subresource); + const auto video_subresource_owner = + TextureCacheTestAccess::Owner(texture_cache, video_subresource_id); + auto video_parent = + make_target_desc(base + 0x20000, target_mip_size * 2, {2, 2, 1}); + video_parent.type = BindingType::VideoOut; + video_parent.info.pixel_format = vk::Format::eR8G8B8A8Srgb; + video_parent.info.guest_format = + Prospero::GpuEnumValue(Prospero::BufferFormat::k8_8_8_8Srgb); + video_parent.info.resources.levels = 2; + video_parent.info.mip_layout[0] = {0, target_mip_size, 2, 2}; + video_parent.info.mip_layout[1] = {target_mip_size, target_mip_size, 1, + 1}; + video_parent.view_info.format = video_parent.info.pixel_format; + video_parent.view_info.usage = vk::ImageUsageFlagBits::eSampled; + const auto video_parent_id = texture_cache.FindImage(video_parent); + const auto video_parent_owner = + TextureCacheTestAccess::Owner(texture_cache, video_parent_id); + Require(name, "video-out replacement remains cache-only", + video_subresource_owner != nullptr && + video_parent_owner != nullptr && + video_parent_id != video_subresource_id && + !video_subresource_owner->binding.is_bound && + !video_subresource_owner->binding.is_target && + !video_subresource_owner->binding.needs_rebind && + !video_subresource_owner->usage.video_out && + !video_parent_owner->binding.is_bound && + !video_parent_owner->binding.is_target && + !video_parent_owner->binding.needs_rebind && + !video_parent_owner->usage.video_out, + "FindImage leaked caller-owned state across a video-out " + "replacement"); + + constexpr uint64_t array_target_address = base + 0xd0000; + auto array_target = + make_target_desc(array_target_address, target_mip_size, {128, 128, 1}); + const auto array_target_id = texture_cache.FindImage(array_target); + const auto array_target_owner = + TextureCacheTestAccess::Owner(texture_cache, array_target_id); + RenderExecutorTestAccess::BindRenderTarget(executor, array_target_id); + const auto array_target_view = + texture_cache.FindRenderTarget(array_target_id, array_target); + + ShaderTextureResource array_texture{}; + const auto encoded_array_address = array_target_address >> 8u; + constexpr auto array_format = + Prospero::GpuEnumValue(Prospero::BufferFormat::k32UInt); + array_texture.fields[0] = + static_cast(encoded_array_address); + array_texture.fields[1] = + static_cast(encoded_array_address >> 32u) | + (array_format << 20u) | (3u << 30u); + array_texture.fields[2] = (127u >> 2u) | (127u << 14u); + array_texture.fields[3] = + DstSel(4, 5, 6, 7) | (linear << 20u) | + (Prospero::GpuEnumValue(Prospero::ImageType::kColor2DArray) + << 28u); + array_texture.fields[4] = 1; + + ShaderRecompiler::IR::Program array_program{}; + array_program.stage = ShaderType::Compute; + array_program.resource_tracking_complete = true; + ShaderRecompiler::IR::ImageResource array_resource{}; + array_resource.kind = ShaderRecompiler::IR::ResourceKind::ImageUint; + array_resource.dimension = + ShaderRecompiler::Decoder::ImageDimension::Dim2DArray; + array_resource.read = true; + array_program.info.images.push_back(array_resource); + ShaderRecompiler::IR::DescriptorValue array_descriptor{}; + std::copy(std::begin(array_texture.fields), + std::end(array_texture.fields), + array_descriptor.dwords.begin()); + array_descriptor.dword_count = 8; + ShaderRecompiler::IR::ResourceSnapshot array_snapshot{}; + array_snapshot.images.push_back(array_descriptor); + ShaderStageRuntime array_runtime{ + std::make_shared( + std::move(array_program)), + std::make_shared( + std::move(array_snapshot))}; + + auto array_binding = executor.PrepareBindings( + scheduler.Current(), array_runtime, + vk::ShaderStageFlagBits::eCompute, DescriptorCache::Stage::Compute); + executor.RebindImages(scheduler.Current(), array_binding); + const auto expanded_array_id = + array_binding.resources.images[0].image_id; + const auto &expanded_array = + texture_cache.GetImage(expanded_array_id); + Require( + name, "2D target to array backing expansion", + array_target_view != nullptr && array_target_owner != nullptr && + array_target_owner->binding.needs_rebind && + expanded_array_id != array_target_id && + array_binding.resources.images[0].image_view != nullptr && + array_binding.resources.images[0].desc.info.type == + Prospero::ImageType::kColor2D && + array_binding.resources.images[0].desc.info.resources == + ImageSubresources{1, 2} && + array_binding.resources.images[0].desc.view_info.type == + vk::ImageViewType::e2DArray && + array_binding.resources.images[0].desc.view_info.layer_count == + 2 && + expanded_array.backing.layers == 2 && + expanded_array.IsGpuModified() && expanded_array.usage.texture, + "raw array view did not expand and reuse the Color2D backing"); + + RenderColorInfo rebound_array_target{}; + rebound_array_target.type = RenderColorType::RenderTexture; + rebound_array_target.desc = array_target; + rebound_array_target.image_id = array_target_id; + rebound_array_target.format = array_target.view_info.format; + rebound_array_target.extent = {128, 128}; + rebound_array_target.samples = 1; + RenderDepthInfo no_array_depth{}; + auto array_rendering = RenderExecutorTestAccess::AcquireRenderTargets( + executor, scheduler.Current(), &rebound_array_target, 1, + no_array_depth); + Require( + name, "expanded array target rebind", + rebound_array_target.image_id == expanded_array_id && + rebound_array_target.image_view != nullptr && + array_rendering.num_color_attachments == 1 && + array_rendering.color_attachments[0].image_view == + rebound_array_target.image_view && + array_rendering.width == 128 && array_rendering.height == 128 && + array_rendering.num_layers == 1 && + texture_cache.GetImage(expanded_array_id).binding.is_bound && + texture_cache.GetImage(expanded_array_id).binding.is_target, + "the retained single-layer target did not rebind to the expanded " + "backing"); + RenderExecutorTestAccess::ResetBindings(executor); + + auto colliding_msaa_texture = array_texture; + colliding_msaa_texture.fields[3] = + DstSel(4, 5, 6, 7) | (1u << 16u) | + (Prospero::GpuEnumValue(Prospero::TileMode::kRenderTarget) << 20u) | + (Prospero::GpuEnumValue(Prospero::ImageType::kColor2DMsaa) + << 28u); + colliding_msaa_texture.fields[4] = 0; + colliding_msaa_texture.fields[5] = (7u << 20u) | (1u << 4u); + ShaderRecompiler::IR::DescriptorValue colliding_msaa_descriptor{}; + std::copy(std::begin(colliding_msaa_texture.fields), + std::end(colliding_msaa_texture.fields), + colliding_msaa_descriptor.dwords.begin()); + colliding_msaa_descriptor.dword_count = 8; + auto colliding_msaa_program = + std::make_shared( + *array_runtime.program); + colliding_msaa_program->info.images[0].dimension = + ShaderRecompiler::Decoder::ImageDimension::Dim2D; + auto colliding_msaa_snapshot = + std::make_shared(); + colliding_msaa_snapshot->images.push_back(colliding_msaa_descriptor); + ShaderStageRuntime colliding_msaa_runtime{ + std::move(colliding_msaa_program), + std::move(colliding_msaa_snapshot)}; + auto colliding_msaa_binding = executor.PrepareBindings( + scheduler.Current(), colliding_msaa_runtime, + vk::ShaderStageFlagBits::eCompute, DescriptorCache::Stage::Compute); + executor.RebindImages(scheduler.Current(), colliding_msaa_binding); + const auto &resolved_colliding_msaa = + colliding_msaa_binding.resources.images[0]; + Require( + name, "equal-footprint sample-count identity", + resolved_colliding_msaa.image_id != expanded_array_id && + resolved_colliding_msaa.image_view != nullptr && + resolved_colliding_msaa.desc.info.type == + Prospero::ImageType::kColor2D && + resolved_colliding_msaa.desc.info.resources == + ImageSubresources{1, 1} && + resolved_colliding_msaa.desc.info.samples == 2 && + resolved_colliding_msaa.desc.info.data.size == 0x20000 && + texture_cache.GetImage(resolved_colliding_msaa.image_id) + .backing.samples == 2 && + texture_cache.GetImage(expanded_array_id).backing.samples == 1 && + texture_cache.GetImage(expanded_array_id).backing.layers == 2, + "equal-size 1x array and 2x image shared an incompatible backing"); + RenderExecutorTestAccess::ResetBindings(executor); + + constexpr uint64_t msaa_target_address = base + 0x100000; + constexpr uint32_t msaa_samples = 4; + auto msaa_target = + make_target_desc(msaa_target_address, 0x40000, {128, 128, 1}); + msaa_target.info.samples = msaa_samples; + msaa_target.info.tile_mode = + Prospero::GpuEnumValue(Prospero::TileMode::kRenderTarget); + msaa_target.info.mip_layout[0].size = msaa_target.info.data.size; + const auto msaa_target_id = texture_cache.FindImage(msaa_target); + RenderExecutorTestAccess::BindRenderTarget(executor, msaa_target_id); + const auto msaa_target_view = + texture_cache.FindRenderTarget(msaa_target_id, msaa_target); + + auto msaa_texture = array_texture; + const auto encoded_msaa_address = msaa_target_address >> 8u; + msaa_texture.fields[0] = + static_cast(encoded_msaa_address); + msaa_texture.fields[1] = + static_cast(encoded_msaa_address >> 32u) | + (array_format << 20u) | (3u << 30u); + msaa_texture.fields[3] = + DstSel(4, 5, 6, 7) | (2u << 16u) | + (Prospero::GpuEnumValue(Prospero::TileMode::kRenderTarget) << 20u) | + (Prospero::GpuEnumValue(Prospero::ImageType::kColor2DMsaa) + << 28u); + msaa_texture.fields[4] = 0; + msaa_texture.fields[5] = (7u << 20u) | (2u << 4u); + ShaderRecompiler::IR::DescriptorValue msaa_descriptor{}; + std::copy(std::begin(msaa_texture.fields), + std::end(msaa_texture.fields), + msaa_descriptor.dwords.begin()); + msaa_descriptor.dword_count = 8; + auto msaa_program = + std::make_shared( + *array_runtime.program); + msaa_program->info.images[0].dimension = + ShaderRecompiler::Decoder::ImageDimension::Dim2D; + auto msaa_snapshot = + std::make_shared(); + msaa_snapshot->images.push_back(msaa_descriptor); + ShaderStageRuntime msaa_runtime{std::move(msaa_program), + std::move(msaa_snapshot)}; + auto msaa_binding = executor.PrepareBindings( + scheduler.Current(), msaa_runtime, + vk::ShaderStageFlagBits::eCompute, DescriptorCache::Stage::Compute); + executor.RebindImages(scheduler.Current(), msaa_binding); + const auto &resolved_msaa = msaa_binding.resources.images[0]; + Require( + name, "MSAA descriptor backing reuse", + msaa_target_view != nullptr && + resolved_msaa.image_id == msaa_target_id && + resolved_msaa.image_view != nullptr && + resolved_msaa.desc.info.type == + Prospero::ImageType::kColor2D && + resolved_msaa.desc.info.resources == + ImageSubresources{1, 1} && + resolved_msaa.desc.info.samples == msaa_samples && + resolved_msaa.desc.info.data.size == 0x40000 && + resolved_msaa.desc.view_info.level_count == 1 && + texture_cache.GetImage(msaa_target_id).backing.samples == + msaa_samples, + "MSAA view fields did not resolve to the matching multisample " + "backing"); + RenderExecutorTestAccess::ResetBindings(executor); + + auto msaa_array_texture = msaa_texture; + msaa_array_texture.fields[3] = + (msaa_array_texture.fields[3] & ~(0xfu << 28u)) | + (Prospero::GpuEnumValue(Prospero::ImageType::kColor2DMsaaArray) + << 28u); + msaa_array_texture.fields[4] = 1; + ShaderRecompiler::IR::DescriptorValue msaa_array_descriptor{}; + std::copy(std::begin(msaa_array_texture.fields), + std::end(msaa_array_texture.fields), + msaa_array_descriptor.dwords.begin()); + msaa_array_descriptor.dword_count = 8; + auto msaa_array_program = + std::make_shared( + *msaa_runtime.program); + msaa_array_program->info.images[0].dimension = + ShaderRecompiler::Decoder::ImageDimension::Dim2DArray; + auto msaa_array_snapshot = + std::make_shared(); + msaa_array_snapshot->images.push_back(msaa_array_descriptor); + ShaderStageRuntime msaa_array_runtime{std::move(msaa_array_program), + std::move(msaa_array_snapshot)}; + auto msaa_array_binding = executor.PrepareBindings( + scheduler.Current(), msaa_array_runtime, + vk::ShaderStageFlagBits::eCompute, DescriptorCache::Stage::Compute); + executor.RebindImages(scheduler.Current(), msaa_array_binding); + const auto &resolved_msaa_array = + msaa_array_binding.resources.images[0]; + Require( + name, "MSAA array backing expansion", + resolved_msaa_array.image_id != msaa_target_id && + resolved_msaa_array.image_view != nullptr && + resolved_msaa_array.desc.info.type == + Prospero::ImageType::kColor2D && + resolved_msaa_array.desc.info.resources == + ImageSubresources{1, 2} && + resolved_msaa_array.desc.info.samples == msaa_samples && + resolved_msaa_array.desc.info.data.size == 0x80000 && + resolved_msaa_array.desc.view_info.type == + vk::ImageViewType::e2DArray && + resolved_msaa_array.desc.view_info.layer_count == 2 && + texture_cache.GetImage(resolved_msaa_array.image_id) + .backing.layers == 2 && + texture_cache.GetImage(resolved_msaa_array.image_id) + .backing.samples == msaa_samples, + "MSAA array view did not expand the matching Color2D backing"); + RenderExecutorTestAccess::ResetBindings(executor); + + ImageDesc depth{}; + depth.type = BindingType::DepthTarget; + depth.info.data = {depth_address, 0x10000}; + depth.info.stencil = {stencil_address, stencil_layout.size}; + depth.info.pixel_format = vk::Format::eD32SfloatS8Uint; + depth.info.guest_format = + Prospero::GpuEnumValue(Prospero::BufferFormat::k32Float); + depth.info.type = Prospero::ImageType::kColor2D; + depth.info.extent = {1, 1, 1}; + depth.info.resources = {1, 1}; + depth.info.pitch = 1; + depth.info.bytes_per_block = 4; + depth.info.samples = 1; + depth.info.tile_mode = linear; + depth.info.mip_layout[0] = {0, depth.info.data.size, 1, 1}; + depth.view_info.format = depth.info.pixel_format; + depth.view_info.type = vk::ImageViewType::e2D; + depth.view_info.aspect = vk::ImageAspectFlagBits::eDepth; + depth.view_info.usage = vk::ImageUsageFlagBits::eDepthStencilAttachment; + const auto depth_id = texture_cache.FindImage(depth); + Require(name, "cache-only depth discovery", + !texture_cache.FindImageFromRange(stencil_address, + stencil_layout.size, false) && + !texture_cache.GetImage(depth_id).binding.is_bound && + !texture_cache.GetImage(depth_id).binding.is_target && + !texture_cache.GetImage(depth_id).usage.depth_target, + "FindImage created depth acquisition state or a stencil proxy"); + RenderExecutorTestAccess::BindRenderTarget(executor, depth_id); + Require(name, "depth prefetch purity", + texture_cache.GetImage(depth_id).binding.is_target && + !texture_cache.GetImage(depth_id).usage.depth_target && + !texture_cache.FindImageFromRange(stencil_address, + stencil_layout.size, false), + "depth prefetch performed final target acquisition"); + + auto target_parent = + make_target_desc(base, target_mip_size * 2, {2, 2, 1}); + target_parent.type = BindingType::RenderTarget; + target_parent.info.resources.levels = 2; + target_parent.info.mip_layout[0] = {0, target_mip_size, 2, 2}; + target_parent.info.mip_layout[1] = {target_mip_size, target_mip_size, 1, + 1}; + target_parent.view_info.usage = vk::ImageUsageFlagBits::eColorAttachment; + auto target_base_subresource = + make_target_desc(base, target_mip_size, {2, 2, 1}); + const auto target_base_subresource_id = + texture_cache.FindImage(target_base_subresource); + auto target_subresource = + make_target_desc(base + target_mip_size, target_mip_size, {1, 1, 1}); + const auto target_subresource_id = + texture_cache.FindImage(target_subresource); + auto target_subresource_owner = + TextureCacheTestAccess::Owner(texture_cache, target_subresource_id); + std::weak_ptr retired_target = + target_subresource_owner; + const auto &discovered_target = + texture_cache.GetImage(target_subresource_id); + Require(name, "cache-only target discovery", + !discovered_target.binding.is_bound && + !discovered_target.binding.is_target && + !discovered_target.binding.needs_rebind && + !discovered_target.usage.render_target && + !TextureCacheTestAccess::PendingDownload( + texture_cache, target_subresource_id), + "FindImage claimed RenderExecutor-owned render-target state"); + RenderExecutorTestAccess::BindRenderTarget(executor, + target_subresource_id); + Require(name, "target prefetch purity", + texture_cache.GetImage(target_subresource_id).binding.is_target && + !texture_cache.GetImage(target_subresource_id) + .usage.render_target && + !TextureCacheTestAccess::PendingDownload( + texture_cache, target_subresource_id), + "target prefetch performed final target acquisition"); + const auto target_parent_id = texture_cache.FindImage(target_parent); + Require(name, "active target overlap", + target_parent_id && + target_parent_id != target_base_subresource_id && + target_parent_id != target_subresource_id && + target_subresource_owner != nullptr && + target_subresource_owner->binding.needs_rebind && + texture_cache.GetImage(target_parent_id).binding.is_target && + !TextureCacheTestAccess::PendingDownload(texture_cache, + target_parent_id), + "target overlap did not transfer target state to the merged " + "owner"); + target_subresource_owner.reset(); + + RenderColorInfo rebound_color{}; + rebound_color.type = RenderColorType::RenderTexture; + rebound_color.desc = target_subresource; + rebound_color.image_id = target_subresource_id; + rebound_color.format = target_subresource.view_info.format; + rebound_color.extent = {1, 1}; + rebound_color.samples = 1; + RenderDepthInfo rebound_depth{}; + rebound_depth.desc = depth; + rebound_depth.format = depth.info.pixel_format; + rebound_depth.width = 1; + rebound_depth.height = 1; + rebound_depth.samples = 1; + rebound_depth.image_id = depth_id; + scheduler.FinishCurrent(); + auto retired_owner = retired_target.lock(); + Require( + name, "deferred target slot erasure", + TextureCacheTestAccess::Owner(texture_cache, target_subresource_id) == + nullptr && + retired_owner != nullptr && retired_owner->binding.needs_rebind, + "the displaced target did not survive only through RenderExecutor " + "ownership"); + auto rebound_rendering = RenderExecutorTestAccess::AcquireRenderTargets( + executor, scheduler.Current(), &rebound_color, 1, rebound_depth); + const auto stencil_proxy_id = texture_cache.FindImageFromRange( + stencil_address, stencil_layout.size, false); + Require( + name, "production target rebind", + rebound_color.image_id == target_parent_id && + rebound_color.image_view != nullptr && + rebound_depth.image_view != nullptr && + rebound_rendering.num_color_attachments == 1 && + rebound_rendering.color_attachments[0].image_view == + rebound_color.image_view && + rebound_rendering.depth_stencil_attachment.image_view == + rebound_depth.image_view && + rebound_rendering.width == 1 && rebound_rendering.height == 1 && + texture_cache.GetImage(target_parent_id).binding.is_target && + texture_cache.GetImage(target_parent_id).usage.render_target && + texture_cache.GetImage(target_parent_id).IsGpuModified() && + texture_cache.GetImage(depth_id).usage.depth_target && + texture_cache.GetImage(depth_id).IsGpuModified() && + TextureCacheTestAccess::PendingDownload(texture_cache, + target_parent_id) && + stencil_proxy_id && + texture_cache.GetImage(stencil_proxy_id).depth_id == depth_id, + "the final target pass did not acquire color before depth and " + "build the dynamic render state"); + scheduler.BeginRendering(rebound_rendering); + scheduler.EndRendering(); + RenderExecutorTestAccess::ResetBindings(executor); + Require(name, "draw-scoped target reset", + !retired_owner->binding.is_target && + !retired_owner->binding.needs_rebind && + !texture_cache.GetImage(target_parent_id).binding.is_target && + !texture_cache.GetImage(depth_id).binding.is_target, + "RenderExecutor retained target binding state after reset"); + retired_owner.reset(); + Require(name, "RenderExecutor-owned retired lifetime", + retired_target.expired(), + "a retired target owner survived after RenderExecutor reset"); + + constexpr uint64_t ordered_color_address = base + 0x80000; + auto ordered_color_desc = + make_target_desc(ordered_color_address, target_mip_size, {1, 1, 1}); + const auto stale_ordered_color = + texture_cache.FindImage(ordered_color_desc); + RenderExecutorTestAccess::BindRenderTarget(executor, stale_ordered_color); + resources.UnmapMemory(ordered_color_address, target_mip_size, + GpuAccess::ReadWrite); + resources.MapMemory(ordered_color_address, target_mip_size, + GpuAccess::ReadWrite); + + auto ordered_depth_desc = depth; + ordered_depth_desc.info.stencil = {ordered_color_address, + target_mip_size}; + const auto ordered_depth_id = texture_cache.FindImage(ordered_depth_desc); + RenderExecutorTestAccess::BindRenderTarget(executor, ordered_depth_id); + RenderColorInfo ordered_color{}; + ordered_color.type = RenderColorType::RenderTexture; + ordered_color.desc = ordered_color_desc; + ordered_color.image_id = stale_ordered_color; + ordered_color.format = ordered_color_desc.view_info.format; + ordered_color.extent = {1, 1}; + ordered_color.samples = 1; + RenderDepthInfo ordered_depth{}; + ordered_depth.desc = ordered_depth_desc; + ordered_depth.format = ordered_depth_desc.info.pixel_format; + ordered_depth.width = 1; + ordered_depth.height = 1; + ordered_depth.samples = 1; + ordered_depth.image_id = ordered_depth_id; + auto ordered_rendering = RenderExecutorTestAccess::AcquireRenderTargets( + executor, scheduler.Current(), &ordered_color, 1, ordered_depth); + Require(name, "color-before-depth acquisition order", + ordered_color.image_id != stale_ordered_color && + ordered_color.image_view != nullptr && + ordered_depth.image_view != nullptr && + ordered_rendering.color_attachments[0].image_view == + ordered_color.image_view && + ordered_rendering.depth_stencil_attachment.image_view == + ordered_depth.image_view && + texture_cache.GetImage(ordered_color.image_id).depth_id == + ordered_depth.image_id, + "depth acquisition ran before the stale color target was " + "recreated and associated"); + RenderExecutorTestAccess::ResetBindings(executor); + TextureCacheTestAccess::SetLinearReadback(texture_cache, false); + + ShaderTextureResource stencil{}; + const uint64_t encoded_address = stencil_address >> 8u; + stencil.fields[0] = static_cast(encoded_address); + stencil.fields[1] = static_cast(encoded_address >> 32u) | + (stencil_format << 20u); + stencil.fields[3] = + DstSel(4, 5, 6, 7) | (linear << 20u) | + (Prospero::GpuEnumValue(Prospero::ImageType::kColor2D) << 28u); + + ShaderRecompiler::IR::Program program{}; + program.stage = ShaderType::Compute; + program.resource_tracking_complete = true; + ShaderRecompiler::IR::ImageResource stencil_resource{}; + stencil_resource.kind = ShaderRecompiler::IR::ResourceKind::ImageUint; + stencil_resource.dimension = + ShaderRecompiler::Decoder::ImageDimension::Dim2D; + stencil_resource.read = true; + program.info.images.push_back(stencil_resource); + + ShaderRecompiler::IR::ResourceSnapshot snapshot{}; + ShaderRecompiler::IR::DescriptorValue descriptor{}; + std::copy(std::begin(stencil.fields), std::end(stencil.fields), + descriptor.dwords.begin()); + descriptor.dword_count = 8; + snapshot.images.push_back(descriptor); + ShaderStageRuntime runtime{ + std::make_shared( + std::move(program)), + std::make_shared( + std::move(snapshot))}; + + auto prepared = context.GetRenderExecutor().PrepareBindings( + scheduler.Current(), runtime, vk::ShaderStageFlagBits::eCompute, + DescriptorCache::Stage::Compute); + const auto sampled_stencil_id = prepared.resources.images[0].image_id; + Require(name, "first stencil discovery", + prepared.resources.images.size() == 1 && + sampled_stencil_id != depth_id && + prepared.resources.images[0].image_view == nullptr && + texture_cache.GetImage(sampled_stencil_id).binding.is_bound && + !texture_cache.GetImage(stencil_proxy_id).binding.is_bound && + !texture_cache.GetImage(stencil_proxy_id).binding.is_target, + "the first sampled-stencil lookup did not remain an ordinary " + "discovery before final depth acquisition"); + + context.GetRenderExecutor().RebindImages(scheduler.Current(), prepared); + Require(name, "first stencil acquisition", + prepared.resources.images[0].image_id == sampled_stencil_id && + prepared.resources.images[0].image_view != nullptr && + texture_cache.GetImage(sampled_stencil_id).usage.texture, + "the first sampled-stencil image was not finally acquired"); + + RenderExecutorTestAccess::ResetBindings(executor); + RenderDepthInfo reassociated_depth = rebound_depth; + reassociated_depth.image_view = nullptr; + RenderColorInfo no_reassociated_color{}; + (void)RenderExecutorTestAccess::AcquireRenderTargets( + executor, scheduler.Current(), &no_reassociated_color, 0, + reassociated_depth); + Require(name, "existing stencil association selection", + texture_cache.GetImage(sampled_stencil_id).depth_id == + reassociated_depth.image_id, + "final depth acquisition did not associate the existing image " + "at the stencil guest address"); + RenderExecutorTestAccess::ResetBindings(executor); + + auto redirected = context.GetRenderExecutor().PrepareBindings( + scheduler.Current(), runtime, vk::ShaderStageFlagBits::eCompute, + DescriptorCache::Stage::Compute); + Require(name, "redirected owner discovery", + redirected.resources.images.size() == 1 && + redirected.resources.images[0].image_id == depth_id && + redirected.resources.images[0].image_view == nullptr && + texture_cache.GetImage(depth_id).binding.is_bound && + !texture_cache.GetImage(sampled_stencil_id).binding.is_bound, + "the established stencil association did not redirect the next " + "discovery to the depth owner"); + context.GetRenderExecutor().RebindImages(scheduler.Current(), redirected); + Require(name, "second-pass stencil acquisition", + redirected.resources.images[0].image_id == depth_id && + redirected.resources.images[0].image_view != nullptr && + texture_cache.GetImage(depth_id).usage.texture, + "RebindImages did not acquire the associated depth owner"); + RenderExecutorTestAccess::ResetBindings(executor); + + auto stencil_storage_program = + std::make_shared(*runtime.program); + stencil_storage_program->info.images[0].kind = + ShaderRecompiler::IR::ResourceKind::StorageImageUint; + stencil_storage_program->info.images[0].read = false; + stencil_storage_program->info.images[0].written = true; + auto storage_stencil = stencil; + storage_stencil.fields[5] = 0x00700000u; + ShaderRecompiler::IR::DescriptorValue stencil_storage_descriptor{}; + std::copy(std::begin(storage_stencil.fields), + std::end(storage_stencil.fields), + stencil_storage_descriptor.dwords.begin()); + stencil_storage_descriptor.dword_count = 8; + ShaderRecompiler::IR::ResourceSnapshot stencil_storage_snapshot{}; + stencil_storage_snapshot.images.push_back(stencil_storage_descriptor); + ShaderStageRuntime stencil_storage_runtime{ + std::move(stencil_storage_program), + std::make_shared( + std::move(stencil_storage_snapshot))}; + auto storage_redirected = executor.PrepareBindings( + scheduler.Current(), stencil_storage_runtime, + vk::ShaderStageFlagBits::eCompute, + DescriptorCache::Stage::Compute); + executor.RebindImages(scheduler.Current(), storage_redirected); + Require(name, "storage stencil acquisition", + storage_redirected.resources.images[0].image_id == depth_id && + storage_redirected.resources.images[0].image_view != nullptr && + texture_cache.GetImage(depth_id).usage.storage, + "storage stencil binding did not acquire the associated depth owner"); + RenderExecutorTestAccess::ResetBindings(executor); + resources.UnmapMemory(base, allocation_size, GpuAccess::ReadWrite); + scheduler.Finish(); + } + + Require(name, "unmap direct backing", + Libs::LibKernel::Memory::KernelMunmap(base, allocation_size) == 0, + "descriptor discovery direct mapping release failed"); + Require(name, "release direct backing", + Libs::LibKernel::Memory::KernelReleaseDirectMemory( + direct_offset, allocation_size) == 0, + "descriptor discovery direct-memory allocation release failed"); std::printf("[host] %-32s ok\n", name); } #endif @@ -2152,51 +6621,6 @@ public: vk::ShaderModule vertex_module = make_module(vertex_spirv); vk::ShaderModule fragment_module = make_module(fragment.spirv); - vk::AttachmentDescription attachment{}; - attachment.format = vk::Format::eR32G32B32A32Sfloat; - attachment.samples = vk::SampleCountFlagBits::e1; - attachment.loadOp = vk::AttachmentLoadOp::eClear; - attachment.storeOp = vk::AttachmentStoreOp::eStore; - attachment.stencilLoadOp = vk::AttachmentLoadOp::eDontCare; - attachment.stencilStoreOp = vk::AttachmentStoreOp::eDontCare; - attachment.initialLayout = vk::ImageLayout::eGeneral; - attachment.finalLayout = vk::ImageLayout::eGeneral; - - vk::AttachmentReference color_ref{}; - color_ref.attachment = 0; - color_ref.layout = vk::ImageLayout::eColorAttachmentOptimal; - - vk::SubpassDescription subpass{}; - subpass.pipelineBindPoint = vk::PipelineBindPoint::eGraphics; - subpass.colorAttachmentCount = 1; - subpass.pColorAttachments = &color_ref; - - vk::RenderPassCreateInfo render_pass_info{}; - render_pass_info.sType = vk::StructureType::eRenderPassCreateInfo; - render_pass_info.attachmentCount = 1; - render_pass_info.pAttachments = &attachment; - render_pass_info.subpassCount = 1; - render_pass_info.pSubpasses = &subpass; - vk::RenderPass render_pass = nullptr; - RequireVk( - test.name, "graphics", - m_device.createRenderPass(&render_pass_info, nullptr, &render_pass), - "vkCreateRenderPass"); - - vk::FramebufferCreateInfo framebuffer_info{}; - framebuffer_info.sType = vk::StructureType::eFramebufferCreateInfo; - framebuffer_info.renderPass = render_pass; - framebuffer_info.attachmentCount = 1; - framebuffer_info.pAttachments = &target.view; - framebuffer_info.width = 1; - framebuffer_info.height = 1; - framebuffer_info.layers = 1; - vk::Framebuffer framebuffer = nullptr; - RequireVk( - test.name, "graphics", - m_device.createFramebuffer(&framebuffer_info, nullptr, &framebuffer), - "vkCreateFramebuffer"); - const auto &fragment_bind = fragment.program.bindings; vk::PushConstantRange push_constant_range{}; if (fragment_bind.push_constant_size > 0) { @@ -2294,7 +6718,13 @@ public: color_blend.pAttachments = &color_attachment; vk::GraphicsPipelineCreateInfo pipeline_info{}; + const vk::Format color_format = vk::Format::eR32G32B32A32Sfloat; + vk::PipelineRenderingCreateInfo rendering_pipeline{}; + rendering_pipeline.sType = vk::StructureType::ePipelineRenderingCreateInfo; + rendering_pipeline.colorAttachmentCount = 1; + rendering_pipeline.pColorAttachmentFormats = &color_format; pipeline_info.sType = vk::StructureType::eGraphicsPipelineCreateInfo; + pipeline_info.pNext = &rendering_pipeline; pipeline_info.stageCount = 2; pipeline_info.pStages = stages; pipeline_info.pVertexInputState = &vertex_input; @@ -2304,7 +6734,6 @@ public: pipeline_info.pMultisampleState = &multisample; pipeline_info.pColorBlendState = &color_blend; pipeline_info.layout = pipeline_layout; - pipeline_info.renderPass = render_pass; vk::Pipeline pipeline = nullptr; RequireVk(test.name, "graphics", m_device.createGraphicsPipelines(nullptr, 1, &pipeline_info, @@ -2312,16 +6741,19 @@ public: "vkCreateGraphicsPipelines"); vk::CommandBuffer cmd = BeginCommands(test.name, "graphics"); - vk::ClearValue clear{}; - vk::RenderPassBeginInfo begin{}; - begin.sType = vk::StructureType::eRenderPassBeginInfo; - begin.renderPass = render_pass; - begin.framebuffer = framebuffer; - begin.renderArea.extent.width = 1; - begin.renderArea.extent.height = 1; - begin.clearValueCount = 1; - begin.pClearValues = &clear; - cmd.beginRenderPass(&begin, vk::SubpassContents::eInline); + vk::RenderingAttachmentInfo color{}; + color.sType = vk::StructureType::eRenderingAttachmentInfo; + color.imageView = target.view; + color.imageLayout = vk::ImageLayout::eGeneral; + color.loadOp = vk::AttachmentLoadOp::eClear; + color.storeOp = vk::AttachmentStoreOp::eStore; + vk::RenderingInfo rendering{}; + rendering.sType = vk::StructureType::eRenderingInfo; + rendering.renderArea.extent = {1, 1}; + rendering.layerCount = 1; + rendering.colorAttachmentCount = 1; + rendering.pColorAttachments = &color; + cmd.beginRendering(rendering); cmd.bindPipeline(vk::PipelineBindPoint::eGraphics, pipeline); if (push_constant_range.size != 0) { cmd.pushConstants(pipeline_layout, vk::ShaderStageFlagBits::eFragment, @@ -2332,7 +6764,7 @@ public: vk::DeviceSize offset = 0; cmd.bindVertexBuffers(0, 1, &vertex_buffer.buffer, &offset); cmd.draw(3, 1, 0, 0); - cmd.endRenderPass(); + cmd.endRendering(); EndSubmitAndFree(test.name, "graphics", cmd); target.layout = vk::ImageLayout::eGeneral; @@ -2341,8 +6773,6 @@ public: m_device.destroyPipeline(pipeline, nullptr); m_device.destroyPipelineLayout(pipeline_layout, nullptr); - m_device.destroyFramebuffer(framebuffer, nullptr); - m_device.destroyRenderPass(render_pass, nullptr); m_device.destroyShaderModule(fragment_module, nullptr); m_device.destroyShaderModule(vertex_module, nullptr); DestroyBuffer(&vertex_buffer); @@ -2354,6 +6784,200 @@ public: constexpr const char *name = "GpuTilerCpuParity"; EnsureRuntimeContext(); + CommandScheduler scheduler(Renderer(), m_runtime_context); + HW::Context registers{}; + HW::UserConfig user_config{}; + HW::Shader shaders{}; + scheduler.Begin(registers, user_config, shaders); + StreamBuffer parameters(m_runtime_context, scheduler, MemoryUsage::Stream, + 1u << 20); + TileManager tile_manager(m_runtime_context, scheduler, parameters); + constexpr uint64_t conversion_limit = 128ull << 20; + Require(name, "D16 conversion chunk boundaries", + TileManagerTestAccess::ConversionRows(0, 32ull << 10, 32ull << 10, + 4096, 256, conversion_limit, + 65535) == 4096 && + TileManagerTestAccess::ConversionRows( + 0, 32ull << 10, 32ull << 10, 4097, 256, conversion_limit, + 65535) == 4096 && + TileManagerTestAccess::ConversionRows( + 257, 32ull << 10, 32ull << 10, 4097, 256, conversion_limit, + 65535) == 4095, + "D16 conversion planner exceeds descriptor or dispatch limits"); + + const auto align_dword = [](uint64_t size) { + return (size + sizeof(u32) - 1u) & ~(uint64_t{sizeof(u32) - 1u}); + }; + const auto to_dwords = [&](const std::vector &bytes, + uint64_t padded_size) { + std::vector words(static_cast(padded_size / sizeof(u32)), 0); + std::memcpy(words.data(), bytes.data(), bytes.size()); + return words; + }; + const auto read_bytes = [&](const Buffer &buffer, uint64_t size) { + const auto words = ReadBuffer( + name, buffer, static_cast(align_dword(size) / sizeof(u32))); + std::vector bytes(static_cast(size)); + std::memcpy(bytes.data(), words.data(), bytes.size()); + return bytes; + }; + const auto host_barrier = [&](vk::Buffer buffer, uint64_t size, + vk::PipelineStageFlags source_stage, + vk::AccessFlags source_access) { + vk::BufferMemoryBarrier barrier{}; + barrier.sType = vk::StructureType::eBufferMemoryBarrier; + barrier.srcAccessMask = source_access; + barrier.dstAccessMask = vk::AccessFlagBits::eHostRead; + barrier.srcQueueFamilyIndex = VK_QUEUE_FAMILY_IGNORED; + barrier.dstQueueFamilyIndex = VK_QUEUE_FAMILY_IGNORED; + barrier.buffer = buffer; + barrier.offset = 0; + barrier.size = size; + scheduler.Current().Handle().pipelineBarrier( + source_stage, vk::PipelineStageFlagBits::eHost, {}, 0, nullptr, 1, + &barrier, 0, nullptr); + }; + const auto gpu_detile = + [&](const std::vector &tiled, std::vector *linear, + uint64_t tiled_capacity, uint64_t linear_capacity, + std::span infos) { + const uint64_t padded_tiled = align_dword(tiled_capacity); + const uint64_t padded_linear = align_dword(linear_capacity); + auto input = CreateHostBuffer(name, padded_tiled, AllFlags, + to_dwords(tiled, padded_tiled)); + auto output = CreateHostBuffer( + name, padded_linear, AllFlags, + std::vector(static_cast(padded_linear / sizeof(u32)), + 0xabababab)); + const auto result = tile_manager.Detile(input.buffer, 0, padded_tiled, + padded_linear, infos); + const vk::BufferCopy copy{result.offset, 0, padded_linear}; + scheduler.Current().Handle().copyBuffer(result.buffer, output.buffer, + 1, ©); + host_barrier(output.buffer, padded_linear, + vk::PipelineStageFlagBits::eTransfer, + vk::AccessFlagBits::eTransferWrite); + scheduler.Finish(); + *linear = read_bytes(output, linear_capacity); + DestroyBuffer(&output); + DestroyBuffer(&input); + }; + const auto gpu_tile = [&](const std::vector &linear, + std::vector *tiled, + uint64_t tiled_capacity, uint64_t linear_capacity, + std::span infos) { + const uint64_t padded_tiled = align_dword(tiled_capacity); + const uint64_t padded_linear = align_dword(linear_capacity); + auto input = CreateHostBuffer(name, padded_linear, AllFlags, + to_dwords(linear, padded_linear)); + auto output = CreateHostBuffer( + name, padded_tiled, AllFlags, + std::vector(static_cast(padded_tiled / sizeof(u32)), + 0xabababab)); + tile_manager.Tile(input.buffer, 0, padded_linear, output.buffer, 0, + padded_tiled, infos); + host_barrier(output.buffer, padded_tiled, + vk::PipelineStageFlagBits::eComputeShader, + vk::AccessFlagBits::eShaderWrite); + scheduler.Finish(); + *tiled = read_bytes(output, tiled_capacity); + DestroyBuffer(&output); + DestroyBuffer(&input); + }; + + for (const bool d32 : std::array{true, false}) { + constexpr uint32_t width = 5; + constexpr uint32_t height = 3; + constexpr uint32_t pitch = 8; + constexpr uint32_t layers = 2; + constexpr uint64_t d16_row = pitch * sizeof(uint16_t); + constexpr uint64_t d32_row = pitch * sizeof(uint32_t); + constexpr uint64_t d16_slice = d16_row * height; + constexpr uint64_t d32_slice = d32_row * height; + constexpr uint64_t d16_size = d16_slice * layers; + constexpr uint64_t d32_size = d32_slice * layers; + StreamBuffer conversion_upload(m_runtime_context, scheduler, + MemoryUsage::Upload, 1u << 20); + auto [prefix, prefix_offset] = conversion_upload.Map(4096, 256); + Require(name, "D16 conversion prefix", + prefix != nullptr && prefix_offset == 0, + "failed to advance the D16 conversion upload ring"); + std::memset(prefix, 0x7b, 4096); + conversion_upload.Commit(); + auto [source_bytes, source_offset] = conversion_upload.Map(d16_size, 4); + Require(name, "D16 conversion source", + source_bytes != nullptr && source_offset != 0, + "failed to reserve a nonzero D16 conversion source"); + std::memset(source_bytes, 0x3c, d16_size); + for (uint32_t layer = 0; layer < layers; layer++) { + for (uint32_t y = 0; y < height; y++) { + auto *row = reinterpret_cast( + source_bytes + layer * d16_slice + y * d16_row); + for (uint32_t x = 0; x < width; x++) { + row[x] = static_cast( + 0x1111u * (1u + x + y * width + layer * width * height)); + } + } + } + conversion_upload.Commit(); + auto promoted = CreateHostBuffer( + name, d32_size, AllFlags, + std::vector(static_cast(d32_size / sizeof(u32)), 0)); + auto round_trip = CreateHostBuffer( + name, d16_size, AllFlags, + std::vector(static_cast(d16_size / sizeof(u32)), + 0xa5a5a5a5u)); + const TileManager::D16Layout promote_layout{ + .width = width, + .height = height, + .layers = layers, + .source_row_stride = d16_row, + .target_row_stride = d32_row, + .source_slice_stride = d16_slice, + .target_slice_stride = d32_slice, + }; + tile_manager.ConvertD16({conversion_upload.Handle(), source_offset, + conversion_upload.Size() - source_offset}, + {promoted.buffer, 0, d32_size}, + TileManager::D16Direction::Promote, d32, + promote_layout); + tile_manager.ConvertD16({promoted.buffer, 0, d32_size}, + {round_trip.buffer, 0, d16_size}, + TileManager::D16Direction::Demote, d32, + {.width = width, + .height = height, + .layers = layers, + .source_row_stride = d32_row, + .target_row_stride = d16_row, + .source_slice_stride = d32_slice, + .target_slice_stride = d16_slice}); + host_barrier(round_trip.buffer, d16_size, + vk::PipelineStageFlagBits::eComputeShader, + vk::AccessFlagBits::eShaderWrite); + scheduler.Finish(); + const auto result = read_bytes(round_trip, d16_size); + bool conversion_matches = true; + for (uint32_t layer = 0; layer < layers; layer++) { + for (uint32_t y = 0; y < height; y++) { + for (uint32_t x = 0; x < pitch; x++) { + const auto offset = + layer * d16_slice + y * d16_row + x * sizeof(uint16_t); + conversion_matches &= + x < width + ? std::memcmp(result.data() + offset, source_bytes + offset, + sizeof(uint16_t)) == 0 + : result[offset] == 0xa5 && result[offset + 1] == 0xa5; + } + } + } + Require(name, + d32 ? "D32 conversion round trip" : "D24 conversion round trip", + conversion_matches, + "D16 conversion lost rows, layers, active values, or padding"); + DestroyBuffer(&round_trip); + DestroyBuffer(&promoted); + } + auto fill = [](std::vector *bytes, u32 salt) { for (size_t i = 0; i < bytes->size(); i++) { uint64_t value = @@ -2406,7 +7030,8 @@ public: u32 local = 0, block_xor = 0; Require(name, "reference offset", TileGetBlockOffset(block, lx, ly, lz, local) && - TileGetBlockXor(block, bx, by, info.surface_z + bz, block_xor), + TileGetBlockXor(block, bx, by, info.surface_z + bz, + block_xor), "CPU reference address lookup failed"); const uint64_t block_index = static_cast(bz) * columns * rows + by * columns + bx; @@ -2506,18 +7131,17 @@ public: for (const auto &info : infos) { convert_reference(false, &cpu, tiled, info); } - GpuDetile(tiled.data(), gpu.data(), size, size, infos); + gpu_detile(tiled, &gpu, size, size, infos); compare((std::string(stage) + " detile bytes").c_str(), cpu, gpu); std::vector linear(size); - std::vector cpu_tiled(size, 0); + std::vector cpu_tiled(size, 0xab); std::vector gpu_tiled(size, 0xab); fill(&linear, 0x280u + case_index); for (const auto &info : infos) { convert_reference(true, &cpu_tiled, linear, info); } - GpuTile(linear.data(), gpu_tiled.data(), size, size, - infos); + gpu_tile(linear, &gpu_tiled, size, size, infos); compare((std::string(stage) + " tile bytes").c_str(), cpu_tiled, gpu_tiled); }; @@ -2565,8 +7189,8 @@ public: ? 3 : 0; convert_reference(false, &cpu, tiled, info); - GpuDetile(tiled.data(), gpu.data(), storage_size, - storage_size, std::span(&info, 1)); + gpu_detile(tiled, &gpu, storage_size, storage_size, + std::span(&info, 1)); const auto family_label = [&](const char *operation) { std::ostringstream out; out << operation << " family=" << static_cast(family.family) @@ -2577,13 +7201,12 @@ public: { std::vector linear(storage_size); - std::vector cpu_tiled(storage_size, 0); + std::vector cpu_tiled(storage_size, 0xab); std::vector gpu_tiled(storage_size, 0xab); fill(&linear, 0x80u + case_index); convert_reference(true, &cpu_tiled, linear, info); - GpuTile(linear.data(), gpu_tiled.data(), - storage_size, storage_size, - std::span(&info, 1)); + gpu_tile(linear, &gpu_tiled, storage_size, storage_size, + std::span(&info, 1)); compare(family_label("tile bytes").c_str(), cpu_tiled, gpu_tiled); } } @@ -2641,9 +7264,8 @@ public: const auto layout = TextureCalcUploadLayout(format, width, height, 1, 1, pitch, mode.tile, total.size, false, false, name); - const auto regions = TextureBuildUploadRegions( - layout, TextureGetFormat(format), width, height, 1, 1, false, false, - TextureUploadDestination::MipLevels); + const auto regions = + TextureBuildImageCopies(layout, width, height, 1, 1, false, false); std::vector infos; if (!TextureBuildGpuTileInfos(total.size, regions, layout, format, 1, 1, infos)) { @@ -2662,8 +7284,7 @@ public: std::vector gpu(total.size, 0xab); fill(&tiled, ++case_index); convert_reference(false, &cpu, tiled, infos[0]); - GpuDetile(tiled.data(), gpu.data(), total.size, - total.size, infos); + gpu_detile(tiled, &gpu, total.size, total.size, infos); compare("format bytes", cpu, gpu); ++format_cases; } @@ -2684,9 +7305,8 @@ public: const auto layout = TextureCalcUploadLayout(format, width, height, 1, layers, pitch, tile, total.size, true, false, name); - const auto regions = TextureBuildUploadRegions( - layout, vk::Format::eR32Sfloat, width, height, layers, 1, true, false, - TextureUploadDestination::MipLevels); + const auto regions = TextureBuildImageCopies(layout, width, height, + layers, 1, true, false); std::vector infos; const bool built = TextureBuildGpuTileInfos(total.size, regions, layout, format, layers, 1, infos); @@ -2713,9 +7333,8 @@ public: const auto layout = TextureCalcUploadLayout(format, width, height, levels, 1, pitch, mode.tile, total.size, false, false, name); - const auto regions = TextureBuildUploadRegions( - layout, vk::Format::eR32Sfloat, width, height, 1, levels, false, - false, TextureUploadDestination::MipLevels); + const auto regions = TextureBuildImageCopies(layout, width, height, 1, + levels, false, false); std::vector infos; Require(name, "odd mip infos", TextureBuildGpuTileInfos(total.size, regions, layout, format, 1, @@ -2761,9 +7380,8 @@ public: const auto layout = TextureCalcUploadLayout(format, width, height, levels, 1, pitch, mode.tile, total.size, true, false, name); - const auto regions = TextureBuildUploadRegions( - layout, vk::Format::eR32Sfloat, width, height, 1, levels, false, - false, TextureUploadDestination::MipLevels); + const auto regions = TextureBuildImageCopies(layout, width, height, 1, + levels, false, false); std::vector infos; Require(name, "2D mip tail seam", layout.first_tail_level == 2 && @@ -2792,9 +7410,8 @@ public: const auto layout = TextureCalcUploadLayout(format, width, height, levels, depth, pitch, tile, total.size, false, true, name); - const auto regions = TextureBuildUploadRegions( - layout, vk::Format::eR32Sfloat, width, height, depth, levels, false, - true, TextureUploadDestination::MipLevels); + const auto regions = TextureBuildImageCopies(layout, width, height, depth, + levels, false, true); std::vector infos; const bool built = TextureBuildGpuTileInfos(total.size, regions, layout, format, depth, levels, infos); @@ -2828,9 +7445,8 @@ public: const auto layout = TextureCalcUploadLayout(format, width, height, levels, depth, pitch, tile, total.size, false, true, name); - const auto regions = TextureBuildUploadRegions( - layout, TextureGetFormat(format), width, height, depth, levels, false, - true, TextureUploadDestination::MipLevels); + const auto regions = TextureBuildImageCopies(layout, width, height, depth, + levels, false, true); std::vector infos; Require(name, "3D BC mip infos", TextureBuildGpuTileInfos(total.size, regions, layout, format, @@ -2842,6 +7458,34 @@ public: check_round_trip("3D BC mip", total.size, infos); } + { + constexpr u32 format = + Prospero::GpuEnumValue(Prospero::BufferFormat::kBc3UNorm); + constexpr u32 tile = Prospero::GpuEnumValue(Prospero::TileMode::kLinear); + constexpr u32 width = 8, height = 8, levels = 4; + const u32 pitch = TileGetTexturePitch(format, width, levels, tile); + TileSizeAlign total{}; + TileGetTextureSize(format, width, height, pitch, levels, tile, &total, + nullptr, nullptr); + const auto layout = + TextureCalcUploadLayout(format, width, height, levels, 1, pitch, tile, + total.size, false, false, name); + const auto regions = TextureBuildImageCopies(layout, width, height, 1, + levels, false, false); + Require(name, "linear BC native regions", + regions.size() == levels && + std::all_of(regions.begin(), regions.end(), + [](const auto ®ion) { + return region.bufferRowLength >= + region.imageExtent.width && + region.bufferImageHeight >= + region.imageExtent.height && + region.bufferRowLength % 4 == 0 && + region.bufferImageHeight % 4 == 0; + }), + "linear BC mip tails emitted invalid Vulkan copy strides"); + } + struct VolumeModeCase { u32 tile; TileBlockFamily family; @@ -2877,9 +7521,8 @@ public: const auto layout = TextureCalcUploadLayout( test.format, width, height, levels, depth, pitch, test.tile, total.size, true, true, name); - const auto regions = TextureBuildUploadRegions( - layout, TextureGetFormat(test.format), width, height, depth, levels, - false, true, TextureUploadDestination::MipLevels); + const auto regions = TextureBuildImageCopies(layout, width, height, depth, + levels, false, true); std::vector infos; const bool built = TextureBuildGpuTileInfos( total.size, regions, layout, test.format, depth, levels, infos); @@ -2924,9 +7567,10 @@ public: Prospero::GpuEnumValue(Prospero::TileMode::kStandard4KB); constexpr u32 levels = 5; TileBlockLayout block{}; - Require(name, "3D tail block", - TileGetBlockLayout(TileBlockFamily::Standard4KB3D, tail.bytes, block), - "Standard4KB3D tail format has no block layout"); + Require( + name, "3D tail block", + TileGetBlockLayout(TileBlockFamily::Standard4KB3D, tail.bytes, block), + "Standard4KB3D tail format has no block layout"); const bool compressed = Prospero::BlockCompressedBytesPerBlock(tail.format) != 0; const u32 width = block.block_width * (compressed ? 4u : 1u); @@ -2939,9 +7583,8 @@ public: const auto layout = TextureCalcUploadLayout(tail.format, width, height, levels, depth, pitch, tile, total.size, false, true, name); - const auto regions = TextureBuildUploadRegions( - layout, TextureGetFormat(tail.format), width, height, depth, levels, - false, true, TextureUploadDestination::MipLevels); + const auto regions = TextureBuildImageCopies(layout, width, height, depth, + levels, false, true); std::vector infos; bool valid = total.size == 8192 && @@ -3007,18 +7650,17 @@ public: ? 2 : 0; convert_reference(false, &cpu, tiled, info); - GpuDetile(tiled.data(), gpu.data(), block.block_size, - linear_size, std::span(&info, 1)); + gpu_detile(tiled, &gpu, block.block_size, linear_size, + std::span(&info, 1)); compare("tail bytes", cpu, gpu); std::vector linear(linear_size); - std::vector cpu_tiled(block.block_size, 0); + std::vector cpu_tiled(block.block_size, 0xab); std::vector gpu_tiled(block.block_size, 0xab); fill(&linear, 0x400u + case_index); convert_reference(true, &cpu_tiled, linear, info); - GpuTile(linear.data(), gpu_tiled.data(), - block.block_size, linear_size, - std::span(&info, 1)); + gpu_tile(linear, &gpu_tiled, block.block_size, linear_size, + std::span(&info, 1)); compare("tail tile bytes", cpu_tiled, gpu_tiled); } } @@ -3040,22 +7682,107 @@ public: small_info.height = 1; small_info.pitch = small_block.block_width; convert_reference(false, &small_expected, small_input, small_info); - GpuDetile(small_input.data(), small_output.data(), - small_input.size(), small_output.size(), - std::span(&small_info, 1)); + gpu_detile(small_input, &small_output, small_input.size(), + small_output.size(), + std::span(&small_info, 1)); compare("small detile", small_expected, small_output); - GpuTileRelease(); std::fill(small_output.begin(), small_output.end(), 0xab); - GpuDetile(small_input.data(), small_output.data(), - small_input.size(), small_output.size(), - std::span(&small_info, 1)); - compare("release recovery", small_expected, small_output); + gpu_detile(small_input, &small_output, small_input.size(), + small_output.size(), + std::span(&small_info, 1)); + compare("scheduler-owned reuse", small_expected, small_output); + + constexpr u32 volume_format = + Prospero::GpuEnumValue(Prospero::BufferFormat::k32UInt); + constexpr u32 volume_tile = + Prospero::GpuEnumValue(Prospero::TileMode::kLinear); + constexpr u32 volume_width = 8, volume_height = 4, volume_depth = 5; + constexpr u32 volume_levels = 3; + const u32 volume_pitch = TileGetTexturePitch(volume_format, volume_width, + volume_levels, volume_tile); + TileSizeAlign volume_size{}; + TileGetTextureTotalSize(volume_format, volume_width, volume_height, + volume_depth, volume_pitch, volume_levels, + volume_tile, true, volume_size); + const auto volume_layout = TextureCalcUploadLayout( + volume_format, volume_width, volume_height, volume_levels, volume_depth, + volume_pitch, volume_tile, volume_size.size, true, true, name); + const auto volume_copies = + TextureBuildImageCopies(volume_layout, volume_width, volume_height, + volume_depth, volume_levels, false, true); + std::vector volume_source(volume_size.size / sizeof(u32), 0); + std::vector> volume_probes; + for (const auto © : volume_copies) { + for (u32 y = 0; y < copy.imageExtent.height; y++) { + for (u32 x = 0; x < copy.imageExtent.width; x++) { + const auto index = + (copy.bufferOffset + + (static_cast(y) * copy.bufferRowLength + x) * + sizeof(u32)) / + sizeof(u32); + const u32 value = + 0xa0000000u | (copy.imageSubresource.mipLevel << 24u) | + (static_cast(copy.imageOffset.z) << 16u) | (y << 8u) | x; + Require(name, "linear volume native bounds", + index < volume_source.size(), + "linear volume copy escaped its padded guest layout"); + volume_source[index] = value; + volume_probes.emplace_back(static_cast(index), value); + } + } + } + auto volume_upload = + CreateHostBuffer(name, volume_size.size, + vk::BufferUsageFlagBits::eTransferSrc, volume_source); + auto volume_download = CreateHostBuffer( + name, volume_size.size, vk::BufferUsageFlagBits::eTransferDst, + std::vector(volume_source.size(), 0)); + ImageInfo volume_info{}; + volume_info.data = {0x10000, volume_size.size}; + volume_info.pixel_format = vk::Format::eR32Uint; + volume_info.guest_format = volume_format; + volume_info.type = Prospero::ImageType::kColor3D; + volume_info.extent = {volume_width, volume_height, volume_depth}; + volume_info.resources = {volume_levels, 1}; + volume_info.pitch = volume_pitch; + volume_info.bytes_per_block = sizeof(u32); + volume_info.tile_mode = volume_tile; + for (u32 level = 0; level < volume_levels; level++) { + volume_info.mip_layout[level] = { + volume_layout.level_sizes[level].offset, + volume_layout.level_sizes[level].size, + volume_layout.padded_sizes[level].width, + volume_layout.padded_sizes[level].height}; + } + Libs::Graphics::Image volume_image(m_runtime_context, scheduler, + volume_info); + volume_image.Upload(volume_copies, volume_upload.buffer, 0, + volume_size.size); + volume_image.Download(volume_copies, volume_download.buffer, 0, + volume_size.size); + scheduler.Finish(); + const auto volume_observed = + ReadBuffer(name, volume_download, volume_source.size()); + Require(name, "linear volume native content", + std::ranges::all_of(volume_probes, + [&](const auto &probe) { + return volume_observed[probe.first] == + probe.second; + }), + "native upload/download did not preserve padded 3D mip rows"); + DestroyBuffer(&volume_download); + DestroyBuffer(&volume_upload); std::printf("[gpu] %-32s ok (%u cases, %u format/mode pairs)\n", name, case_index, format_cases); } private: + RenderContext &Renderer() { + EXIT_IF(m_renderer == nullptr); + return *m_renderer; + } + void EnsureRuntimeContext() { if (m_runtime_context.allocator != nullptr) { return; @@ -3080,12 +7807,12 @@ private: allocator_info.physicalDevice = m_physical_device; allocator_info.device = m_device; allocator_info.pVulkanFunctions = &functions; - allocator_info.vulkanApiVersion = VK_API_VERSION_1_2; + allocator_info.vulkanApiVersion = VK_API_VERSION_1_3; RequireVk("VulkanHarness", "runtime context", static_cast(vmaCreateAllocator( &allocator_info, &m_runtime_context.allocator)), "vmaCreateAllocator"); - GraphicsRenderInit(m_runtime_context); + m_renderer = std::make_unique(m_runtime_context); } void Init() { @@ -3100,7 +7827,7 @@ private: vk::ApplicationInfo app{}; app.sType = vk::StructureType::eApplicationInfo; app.pApplicationName = "ShaderRecompilerComputeTests"; - app.apiVersion = VK_API_VERSION_1_2; + app.apiVersion = VK_API_VERSION_1_3; vk::InstanceCreateInfo instance_info{}; instance_info.sType = vk::StructureType::eInstanceCreateInfo; @@ -3146,12 +7873,35 @@ private: vk::PhysicalDeviceFeatures available_features{}; m_physical_device.getFeatures(&available_features); + vk::PhysicalDeviceVulkan12Features available_features12{}; + available_features12.sType = + vk::StructureType::ePhysicalDeviceVulkan12Features; + vk::PhysicalDeviceVulkan13Features available_features13{}; + available_features13.sType = + vk::StructureType::ePhysicalDeviceVulkan13Features; + available_features13.pNext = &available_features12; + vk::PhysicalDeviceFeatures2 available_features2{}; + available_features2.sType = vk::StructureType::ePhysicalDeviceFeatures2; + available_features2.pNext = &available_features13; + m_physical_device.getFeatures2(&available_features2); Require("VulkanHarness", "dispatch", available_features.shaderStorageImageWriteWithoutFormat == true, "shaderStorageImageWriteWithoutFormat is not supported"); Require("VulkanHarness", "dispatch", available_features.shaderStorageImageReadWithoutFormat == true, "shaderStorageImageReadWithoutFormat is not supported"); + Require("VulkanHarness", "dispatch", + available_features12.timelineSemaphore == true, + "timeline semaphores are not supported"); + Require("VulkanHarness", "dispatch", + available_features13.dynamicRendering == true, + "dynamic rendering is not supported"); + Require("VulkanHarness", "dispatch", + available_features13.synchronization2 == true, + "synchronization2 is not supported"); + Require("VulkanHarness", "dispatch", + available_features.sampleRateShading == true, + "sample-rate shading is not supported"); float priority = 1.0f; vk::DeviceQueueCreateInfo queue_info{}; @@ -3164,10 +7914,26 @@ private: device_info.sType = vk::StructureType::eDeviceCreateInfo; device_info.queueCreateInfoCount = 1; device_info.pQueueCreateInfos = &queue_info; + vk::PhysicalDeviceVulkan12Features device_features12{}; + device_features12.sType = + vk::StructureType::ePhysicalDeviceVulkan12Features; + device_features12.timelineSemaphore = true; + vk::PhysicalDeviceVulkan13Features device_features13{}; + device_features13.sType = + vk::StructureType::ePhysicalDeviceVulkan13Features; + device_features13.pNext = &device_features12; + device_features13.dynamicRendering = true; + device_features13.synchronization2 = true; + device_info.pNext = &device_features13; vk::PhysicalDeviceFeatures device_features{}; device_features.shaderStorageImageWriteWithoutFormat = true; device_features.shaderStorageImageReadWithoutFormat = true; + device_features.sampleRateShading = true; device_info.pEnabledFeatures = &device_features; + constexpr const char *device_extensions[] = { + VK_KHR_PUSH_DESCRIPTOR_EXTENSION_NAME}; + device_info.enabledExtensionCount = std::size(device_extensions); + device_info.ppEnabledExtensionNames = device_extensions; RequireVk("VulkanHarness", "dispatch", m_physical_device.createDevice(&device_info, nullptr, &m_device), "vkCreateDevice"); @@ -3187,8 +7953,7 @@ private: if (m_device != nullptr) { RequireVulkanSuccess(m_device.waitIdle(), "vkDeviceWaitIdle"); if (m_runtime_context.allocator != nullptr) { - GraphicsRenderReleaseThreadCommandPool(); - Transfer::ReleaseCachedResources(); + m_renderer.reset(); vmaDestroyAllocator(m_runtime_context.allocator); m_runtime_context.allocator = nullptr; } @@ -3429,6 +8194,7 @@ private: u32 m_queue_family = 0; vk::PhysicalDeviceMemoryProperties m_memory_properties{}; GraphicContext m_runtime_context{}; + std::unique_ptr m_renderer; }; void CompareWords(const TestCase &test, const char *stage, @@ -4029,6 +8795,31 @@ TestCase Shifts() { O::BufferStoreDword, O::SEndpgm}}; } +TestCase ExactPushConstantExtent() { + using O = ShaderOpcode; + + constexpr std::array values{ + 0x10203040u, 0x21314151u, 0x32425262u, 0x43536373u, 0x54647484u, + 0x65758595u, 0x768696a6u, 0x8797a7b7u, 0x98a8b8c8u, 0xa9b9c9d9u}; + std::vector code; + for (u32 i = 0; i < values.size(); i++) { + AppendStoreSgpr(&code, 4u + i, i); + } + AppendEnd(&code); + + TestCase test; + test.name = "ExactPushConstantExtent"; + test.code = std::move(code); + test.initial.resize(values.size()); + test.expected.assign(values.begin(), values.end()); + test.opcodes = {O::VMovB32, O::BufferStoreDword, O::SEndpgm}; + test.user_data = + MakeStructuredStorageBufferData(0, values.size() * sizeof(u32)); + std::copy(values.begin(), values.end(), test.user_data.begin() + 4); + test.has_user_data = true; + return test; +} + TestCase ScalarShiftCountsMaskLowBits() { using O = ShaderOpcode; @@ -8930,6 +13721,14 @@ TestCase ImageStoreRgbOneUsesInverseSwizzle() { return test; } +TestCase ImageStoreDuplicateSelectorUsesInverseSwizzle() { + auto test = ImageStoreRgbOneUsesInverseSwizzle(); + test.name = "ImageStoreDuplicateSelectorUsesInverseSwizzle"; + test.image_descriptor_swizzle = DstSel(4, 4, 6, 7); + test.expected_storage_image_rgba[2 * 4 + 1] = 0xc04000ffu; + return test; +} + TestCase ImageStoreBgraUsesInverseSwizzle() { using O = ShaderOpcode; @@ -9432,6 +14231,7 @@ std::vector MakeCases() { AddCase(IntegerAddSubMul); AddCase(BitwiseOps); AddCase(Shifts); + AddCase(ExactPushConstantExtent); AddCase(ScalarShiftCountsMaskLowBits); AddCase(Rdna2ScalarOpcodes); AddCase(ScalarExtendedArithmetic); @@ -9590,6 +14390,7 @@ std::vector MakeCases() { AddCase(ImageGatherCompareOpcodes); AddCase(ImageStoreVariants); AddCase(ImageStoreRgbOneUsesInverseSwizzle); + AddCase(ImageStoreDuplicateSelectorUsesInverseSwizzle); AddCase(ImageStoreBgraUsesInverseSwizzle); AddCase(ImageStoreYzwxUsesInverseSwizzle); AddCase(ImageStoreR32FloatUsesFormatlessStorageImage); @@ -9927,22 +14728,6 @@ void CheckReverseRenderTargetFormatContract() { } else if (std::strcmp(kind, "sampled-depth-swizzle") == 0) { (void)SelectSampledDepthView(vk::Format::eD32SfloatS8Uint, vk::Format::eR32Sfloat, DstSel(4, 5, 6, 7)); - } else if (std::strcmp(kind, "storage") == 0) { - (void)SelectStorageColorView(vk::Format::eR8G8B8A8Unorm, - vk::Format::eR8G8B8A8Unorm, - DstSel(4, 5, 6, 0)); - } else if (std::strcmp(kind, "storage-format") == 0) { - (void)SelectStorageColorView(vk::Format::eB8G8R8A8Unorm, - vk::Format::eR8G8B8A8Unorm, - DstSel(4, 5, 6, 7)); - } else if (std::strcmp(kind, "storage-compatible-swizzle") == 0) { - (void)SelectStorageColorView(vk::Format::eB8G8R8A8Srgb, - vk::Format::eR8G8B8A8Unorm, - DstSel(4, 5, 6, 7)); - } else if (std::strcmp(kind, "storage-abgr") == 0) { - (void)SelectStorageColorView(vk::Format::eR16G16B16A16Sfloat, - vk::Format::eR16G16B16A16Sfloat, - DstSel(7, 6, 5, 4)); } else { ShaderRecompiler::IR::ImageResource resource{}; resource.kind = ShaderRecompiler::IR::ResourceKind::StorageImage; @@ -10022,7 +14807,7 @@ void CheckSampledColorViews() { expected = false; } } - const bool valid = IsValidSampledColorSwizzle(swizzle); + const bool valid = IsValidImageSwizzle(swizzle); const bool supported = IsSupportedSampledColorView( vk::Format::eR8G8B8A8Unorm, vk::Format::eR8G8B8A8Unorm, swizzle); Require("SampledColorViews", "exhaustive read swizzle domain", @@ -10032,9 +14817,9 @@ void CheckSampledColorViews() { } Require("SampledColorViews", "all PS5 read swizzles", valid_swizzles == 1296 && - !IsValidSampledColorSwizzle(DstSel(4, 5, 6, 2)) && - !IsValidSampledColorSwizzle(DstSel(4, 5, 6, 3)) && - !IsValidSampledColorSwizzle(0x1000), + !IsValidImageSwizzle(DstSel(4, 5, 6, 2)) && + !IsValidImageSwizzle(DstSel(4, 5, 6, 3)) && + !IsValidImageSwizzle(0x1000), "valid PS5 sampled mappings were rejected or reserved selectors were " "admitted"); const auto arbitrary = DstSel(5, 1, 7, 0); @@ -10147,62 +14932,6 @@ void CheckSampledColorViews() { vk::Format::eR32Sfloat, DstSel(4, 0, 0, 1)) == DstSel(4, 0, 0, 1), "D32S8 depth target did not select its R001 depth-aspect view"); - Require("SampledColorViews", "storage identity", - SelectStorageColorView( - vk::Format::eR8G8B8A8Unorm, vk::Format::eR8G8B8A8Unorm, - DstSel(4, 5, 6, 7)) == VulkanImage::VIEW_STORAGE, - "RGBA storage did not select the identity storage view"); - Require("SampledColorViews", "storage RGB1 write mapping", - SelectStorageColorView( - vk::Format::eR8G8B8A8Unorm, vk::Format::eR8G8B8A8Unorm, - DstSel(4, 5, 6, 1)) == VulkanImage::VIEW_STORAGE, - "RGBA8 RGB1 storage did not select the identity storage view"); - Require("SampledColorViews", "storage BGRA write mapping", - SelectStorageColorView( - vk::Format::eR8G8B8A8Unorm, vk::Format::eR8G8B8A8Unorm, - DstSel(6, 5, 4, 7)) == VulkanImage::VIEW_STORAGE, - "RGBA8 BGRA storage did not select the identity storage view"); - Require("SampledColorViews", "storage uint BGRA write mapping", - SelectStorageColorView( - vk::Format::eR8G8B8A8Uint, vk::Format::eR8G8B8A8Uint, - DstSel(6, 5, 4, 7)) == VulkanImage::VIEW_STORAGE, - "RGBA8_UINT BGRA storage did not select the identity storage view"); - Require( - "SampledColorViews", "mutable BGRA sRGB storage view", - SelectStorageColorView(vk::Format::eB8G8R8A8Srgb, - vk::Format::eR8G8B8A8Unorm, - DstSel(6, 5, 4, 7)) == VulkanImage::VIEW_STORAGE, - "BGRA sRGB target did not select its RGBA UNORM identity storage view"); - Require("SampledColorViews", "storage YZWX write mapping", - SelectStorageColorView( - vk::Format::eR32G32B32A32Sfloat, vk::Format::eR32G32B32A32Sfloat, - DstSel(5, 6, 7, 4)) == VulkanImage::VIEW_STORAGE, - "RGBA32F YZWX storage did not select the identity storage view"); - Require("SampledColorViews", "storage R32 uint R001 write mapping", - SelectStorageColorView(vk::Format::eR32Uint, vk::Format::eR32Uint, - DstSel(4, 0, 0, 1)) == - VulkanImage::VIEW_STORAGE, - "R32_UINT R001 storage did not select the identity storage view"); - Require("SampledColorViews", "storage R16 uint R000 write mapping", - SelectStorageColorView(vk::Format::eR16Uint, vk::Format::eR16Uint, - DstSel(4, 0, 0, 0)) == - VulkanImage::VIEW_STORAGE, - "R16_UINT R000 storage did not select the identity storage view"); - Require("SampledColorViews", "storage R16 float R001 write mapping", - SelectStorageColorView(vk::Format::eR16Sfloat, vk::Format::eR16Sfloat, - DstSel(4, 0, 0, 1)) == - VulkanImage::VIEW_STORAGE, - "R16_SFLOAT R001 storage did not select the identity storage view"); - Require("SampledColorViews", "storage R32 float R001 write mapping", - SelectStorageColorView(vk::Format::eR32Sfloat, vk::Format::eR32Sfloat, - DstSel(4, 0, 0, 1)) == - VulkanImage::VIEW_STORAGE, - "R32_SFLOAT R001 storage did not select the identity storage view"); - Require("SampledColorViews", "storage R8 unorm R001 write mapping", - SelectStorageColorView(vk::Format::eR8Unorm, vk::Format::eR8Unorm, - DstSel(4, 0, 0, 1)) == - VulkanImage::VIEW_STORAGE, - "R8_UNORM R001 storage did not select the identity storage view"); ShaderRecompiler::IR::ImageResource storage_resource{}; storage_resource.kind = ShaderRecompiler::IR::ResourceKind::StorageImage; storage_resource.dimension = ShaderRecompiler::Decoder::ImageDimension::Dim2D; @@ -10230,26 +14959,13 @@ void CheckSampledColorViews() { Require("SampledColorViews", "host", GetModuleFileNameA(nullptr, path, MAX_PATH) != 0, "GetModuleFileName failed"); - for (const char *kind : {"sampled", - "sampled-compatible-swizzle", - "sampled-compatible-reverse", - "sampled-compatible-colorspace", - "sampled-compatible-snorm", - "sampled-rgb10-swizzle", - "sampled-rgb10-reverse", - "sampled-invalid-high", - "sampled-depth-format", - "sampled-depth-swizzle", - "storage", - "storage-format", - "storage-compatible-swizzle", - "storage-abgr", - "storage-kind", - "storage-no-write", - "storage-atomic", - "storage-compare", - "storage-mip", - "storage-dimension"}) { + for (const char *kind : + {"sampled", "sampled-compatible-swizzle", "sampled-compatible-reverse", + "sampled-compatible-colorspace", "sampled-compatible-snorm", + "sampled-rgb10-swizzle", "sampled-rgb10-reverse", + "sampled-invalid-high", "sampled-depth-format", "sampled-depth-swizzle", + "storage-kind", "storage-no-write", "storage-atomic", "storage-compare", + "storage-mip", "storage-dimension"}) { std::string command = std::string("\"") + path + "\" --image-view-death " + kind; std::vector mutable_command(command.begin(), command.end()); @@ -10336,20 +15052,12 @@ void CheckSampledDepthResource() { Require("SampledDepthResource", "integer reinterpret resource", IsSupportedSampledDepthUintResource(resource), "read-only uint depth reinterpretation resource was rejected"); - Require("SampledDepthResource", "D32 to R32_UINT", - IsDepthUintTextureReinterpretation( - vk::Format::eD32Sfloat, - Prospero::GpuEnumValue(Prospero::BufferFormat::k32UInt), - vk::Format::eR32Uint) && - !IsDepthUintTextureReinterpretation( - vk::Format::eD16Unorm, - Prospero::GpuEnumValue(Prospero::BufferFormat::k32UInt), - vk::Format::eR32Uint), - "depth uint reinterpretation format boundary changed"); std::printf("[host] %-32s ok\n", "SampledDepthResource"); } -void CheckSampledVideoOutView() { +void CheckSampledVideoOutView(RenderContext &renderer) { + auto &context = renderer.GetGraphics(); + CommandScheduler scheduler(renderer, context); ShaderRecompiler::IR::ImageResource resource{}; resource.kind = ShaderRecompiler::IR::ResourceKind::Image; resource.dimension = ShaderRecompiler::Decoder::ImageDimension::Dim2D; @@ -10358,8 +15066,20 @@ void CheckSampledVideoOutView() { ShaderTextureResource descriptor{}; descriptor.fields[3] = Prospero::GpuEnumValue(Prospero::ImageType::kColor2D) << 28u; - VulkanImage image{VulkanImageType::VideoOut}; - image.layers = 1; + ImageInfo info{}; + info.pixel_format = vk::Format::eR8G8B8A8Unorm; + info.guest_format = + Prospero::GpuEnumValue(Prospero::BufferFormat::k8_8_8_8UNorm); + info.type = Prospero::ImageType::kColor2D; + info.extent = {1, 1, 1}; + info.resources = {1, 1}; + info.pitch = 1; + info.bytes_per_block = 4; + info.samples = 1; + info.tile_mode = Prospero::GpuEnumValue(Prospero::TileMode::kLinear); + info.mip_layout[0] = {0, 4, 1, 1}; + Image image(context, scheduler, info); + image.usage.video_out = true; Require("SampledVideoOutView", "basic 2D", IsSupportedSampledVideoOutView(resource, descriptor, image), "basic 2D video-out view was rejected"); @@ -10382,7 +15102,7 @@ void CheckSampledVideoOutView() { const bool rejects_layer_count = !IsSupportedSampledVideoOutView(resource, descriptor, image); descriptor.fields[4] = 0; - image.layers = 2; + image.info.resources.layers = 2; const bool rejects_layered_image = !IsSupportedSampledVideoOutView(resource, descriptor, image); Require("SampledVideoOutView", "array hard failures", @@ -10393,29 +15113,348 @@ void CheckSampledVideoOutView() { std::printf("[host] %-32s ok\n", "SampledVideoOutView"); } -void CheckSampledDepthDescriptor() { +void CheckImageTransitionState(RenderContext &renderer) { + constexpr const char *name = "ImageTransitionState"; + auto &context = renderer.GetGraphics(); + CommandScheduler scheduler(renderer, context); + const auto MakeInfo = [](vk::Format format, uint32_t levels, + uint32_t layers) { + ImageInfo info{}; + info.pixel_format = format; + info.guest_format = Prospero::GpuEnumValue(Prospero::BufferFormat::k8UNorm); + info.type = Prospero::ImageType::kColor2D; + info.extent = {4, 4, 1}; + info.resources = {levels, layers}; + info.pitch = 4; + info.bytes_per_block = 1; + info.samples = 1; + info.tile_mode = Prospero::GpuEnumValue(Prospero::TileMode::kLinear); + return info; + }; + const auto graphics_stage = vk::PipelineStageFlagBits2::eAllGraphics | + vk::PipelineStageFlagBits2::eComputeShader; + constexpr uint64_t copy_capacity = 128ull << 20; + Require( + name, "buffered-copy capacity bands", + ImageTestAccess::CopyRows(32ull << 10, 4096, copy_capacity) == 4096 && + ImageTestAccess::CopyRows(32ull << 10, 4097, copy_capacity) == 4096 && + ImageTestAccess::CopyRows(copy_capacity + 4, 1, copy_capacity) == 0, + "buffered image copy does not split at the fixed scratch capacity"); + + Image image(context, scheduler, MakeInfo(vk::Format::eR8Unorm, 2, 3)); + auto barriers = + image.GetBarriers(vk::ImageLayout::eShaderReadOnlyOptimal, + vk::AccessFlagBits2::eShaderRead, graphics_stage, {}); + Require( + name, "initial full state", + barriers.size() == 1 && + barriers[0].srcStageMask == + vk::PipelineStageFlagBits2::eAllCommands && + barriers[0].srcAccessMask == vk::AccessFlagBits2::eNone && + barriers[0].dstStageMask == graphics_stage && + barriers[0].dstAccessMask == vk::AccessFlagBits2::eShaderRead && + barriers[0].oldLayout == vk::ImageLayout::eUndefined && + barriers[0].newLayout == vk::ImageLayout::eShaderReadOnlyOptimal && + barriers[0].subresourceRange.aspectMask == + vk::ImageAspectFlagBits::eColor && + barriers[0].subresourceRange.levelCount == VK_REMAINING_MIP_LEVELS && + barriers[0].subresourceRange.layerCount == + VK_REMAINING_ARRAY_LAYERS && + image.backing.state.layout == + vk::ImageLayout::eShaderReadOnlyOptimal && + image.backing.state.access_mask == vk::AccessFlagBits2::eShaderRead && + image.backing.state.pl_stage == graphics_stage, + "initial transition did not record or retain pinned state"); + Require(name, "repeat read elision", + image + .GetBarriers(vk::ImageLayout::eShaderReadOnlyOptimal, + vk::AccessFlagBits2::eShaderRead, graphics_stage, {}) + .empty(), + "identical read-only transition emitted a barrier"); + barriers = image.GetBarriers(vk::ImageLayout::eShaderReadOnlyOptimal, + vk::AccessFlagBits2::eShaderRead | + vk::AccessFlagBits2::eTransferRead, + graphics_stage, {}); + Require(name, "access-only transition", + barriers.size() == 1 && + barriers[0].oldLayout == barriers[0].newLayout && + barriers[0].srcAccessMask == vk::AccessFlagBits2::eShaderRead, + "same-layout access change was lost"); + + const auto CheckRepeatedWrite = [&](vk::AccessFlagBits2 access, + const char *label) { + image.backing.state = {}; + image.backing.subresource_states.clear(); + (void)image.GetBarriers(vk::ImageLayout::eGeneral, access, graphics_stage, + {}); + Require( + name, label, + image.GetBarriers(vk::ImageLayout::eGeneral, access, graphics_stage, {}) + .size() == 1, + "pinned repeated-write dependency was elided"); + }; + CheckRepeatedWrite(vk::AccessFlagBits2::eTransferWrite, + "repeat transfer write"); + CheckRepeatedWrite(vk::AccessFlagBits2::eShaderWrite, "repeat shader write"); + CheckRepeatedWrite(vk::AccessFlagBits2::eMemoryWrite, "repeat memory write"); + image.backing.state = {}; + image.backing.subresource_states.clear(); + const auto attachment_access = vk::AccessFlagBits2::eColorAttachmentRead | + vk::AccessFlagBits2::eColorAttachmentWrite; + (void)image.GetBarriers(vk::ImageLayout::eColorAttachmentOptimal, + attachment_access, graphics_stage, {}); + Require(name, "pinned attachment-write quirk", + image + .GetBarriers(vk::ImageLayout::eColorAttachmentOptimal, + attachment_access, graphics_stage, {}) + .empty(), + "attachment writes diverged from the pinned repeated-write set"); + + image.backing.state = {}; + image.backing.subresource_states.clear(); + (void)image.GetBarriers(vk::ImageLayout::eShaderReadOnlyOptimal, + vk::AccessFlagBits2::eShaderRead, graphics_stage, {}); + constexpr ImageSubresourceRange mip1_layer0{1, 1, 0, 1}; + barriers = image.GetBarriers(vk::ImageLayout::eGeneral, + vk::AccessFlagBits2::eShaderWrite, + graphics_stage, mip1_layer0); + Require(name, "first partial", + barriers.size() == 1 && + barriers[0].subresourceRange.baseMipLevel == 1 && + barriers[0].subresourceRange.baseArrayLayer == 0 && + image.backing.subresource_states.size() == 6 && + image.backing.subresource_states[3].layout == + vk::ImageLayout::eGeneral && + image.backing.subresource_states[3].access_mask == + vk::AccessFlagBits2::eShaderWrite && + image.backing.subresource_states[1].layout == + vk::ImageLayout::eShaderReadOnlyOptimal, + "partial state did not use mip-major indexing"); + constexpr ImageSubresourceRange mip0_layer1{0, 1, 1, 1}; + barriers = image.GetBarriers( + vk::ImageLayout::eTransferDstOptimal, vk::AccessFlagBits2::eTransferWrite, + vk::PipelineStageFlagBits2::eTransfer, mip0_layer1); + Require(name, "second partial", + barriers.size() == 1 && + barriers[0].subresourceRange.baseMipLevel == 0 && + barriers[0].subresourceRange.baseArrayLayer == 1 && + image.backing.subresource_states[1].layout == + vk::ImageLayout::eTransferDstOptimal && + image.backing.subresource_states[1].access_mask == + vk::AccessFlagBits2::eTransferWrite && + image.backing.subresource_states[3].layout == + vk::ImageLayout::eGeneral, + "second partial transition changed the wrong cell"); + const ImageSubresourceRange explicit_full{0, 2, 0, 3}; + barriers = image.GetBarriers(vk::ImageLayout::eShaderReadOnlyOptimal, + vk::AccessFlagBits2::eShaderRead, graphics_stage, + explicit_full); + bool saw_mip0_layer1 = false; + bool saw_mip1_layer0 = false; + for (const auto &barrier : barriers) { + saw_mip0_layer1 |= + barrier.subresourceRange.baseMipLevel == 0 && + barrier.subresourceRange.baseArrayLayer == 1 && + barrier.oldLayout == vk::ImageLayout::eTransferDstOptimal && + barrier.srcStageMask == vk::PipelineStageFlagBits2::eTransfer; + saw_mip1_layer0 |= + barrier.subresourceRange.baseMipLevel == 1 && + barrier.subresourceRange.baseArrayLayer == 0 && + barrier.oldLayout == vk::ImageLayout::eGeneral && + barrier.srcAccessMask == vk::AccessFlagBits2::eShaderWrite; + } + Require(name, "partial normalization", + barriers.size() == 2 && saw_mip0_layer1 && saw_mip1_layer0 && + image.backing.subresource_states.empty() && + image.backing.state.layout == + vk::ImageLayout::eShaderReadOnlyOptimal && + image.backing.state.access_mask == + vk::AccessFlagBits2::eShaderRead && + image.backing.state.pl_stage == graphics_stage, + "full transition did not reconcile and collapse split state"); + barriers = image.GetBarriers(vk::ImageLayout::eTransferSrcOptimal, + vk::AccessFlagBits2::eTransferRead, + vk::PipelineStageFlagBits2::eTransfer, {}); + Require(name, "normalized full transition", + barriers.size() == 1 && + barriers[0].subresourceRange.levelCount == + VK_REMAINING_MIP_LEVELS && + barriers[0].subresourceRange.layerCount == + VK_REMAINING_ARRAY_LAYERS && + barriers[0].srcStageMask == graphics_stage && + barriers[0].dstStageMask == vk::PipelineStageFlagBits2::eTransfer, + "normalized state did not return to one full barrier"); + + auto depth_info = MakeInfo(vk::Format::eD32SfloatS8Uint, 1, 1); + depth_info.guest_format = + Prospero::GpuEnumValue(Prospero::BufferFormat::k32Float); + depth_info.bytes_per_block = 4; + depth_info.tile_mode = Prospero::GpuEnumValue(Prospero::TileMode::kDepth); + Image depth(context, scheduler, depth_info); + barriers = + depth.GetBarriers(vk::ImageLayout::eDepthStencilReadOnlyOptimal, + vk::AccessFlagBits2::eShaderRead, graphics_stage, {}); + Require(name, "depth stencil aspects", + barriers.size() == 1 && barriers[0].subresourceRange.aspectMask == + (vk::ImageAspectFlagBits::eDepth | + vk::ImageAspectFlagBits::eStencil), + "depth/stencil transition did not use the full image aspect"); + + HW::Context registers{}; + HW::UserConfig user_config{}; + HW::Shader shaders{}; + scheduler.Begin(registers, user_config, shaders); + image.backing.state = {}; + image.backing.subresource_states.clear(); + image.Transit(vk::ImageLayout::eGeneral, + vk::AccessFlagBits2::eShaderRead | + vk::AccessFlagBits2::eTransferRead, + {}, scheduler.Current().Handle()); + Require(name, "mixed access stages", + image.backing.state.pl_stage == + (graphics_stage | vk::PipelineStageFlagBits2::eTransfer), + "mixed shader/transfer access omitted a required pipeline stage"); + + auto source_info = depth_info; + source_info.extent = {2, 2, 1}; + source_info.pitch = 2; + source_info.stencil = {0x1000, 4}; + auto destination_info = depth_info; + destination_info.resources.levels = 2; + destination_info.stencil = {0x2000, 16}; + Image source(context, scheduler, source_info); + Image destination(context, scheduler, destination_info); + StreamBuffer upload(context, scheduler, MemoryUsage::Upload, 4096); + StreamBuffer download(context, scheduler, MemoryUsage::Download, 4096); + const auto [upload_data, upload_offset] = upload.Map(128, 4); + Require(name, "depth/stencil upload map", upload_data != nullptr, + "combined depth/stencil upload allocation failed"); + constexpr std::array expected_depth{0x3e800000u, 0x3f000000u, + 0x3f400000u, 0x3f800000u}; + constexpr std::array expected_stencil{0x12, 0x34, 0x56, 0x78}; + std::memcpy(upload_data, expected_depth.data(), sizeof(expected_depth)); + std::memcpy(upload_data + 64, expected_stencil.data(), + sizeof(expected_stencil)); + upload.Commit(); + std::array source_copies{}; + source_copies[0].bufferOffset = upload_offset; + source_copies[0].imageSubresource = {vk::ImageAspectFlagBits::eDepth, 0, 0, + 1}; + source_copies[0].imageExtent = {2, 2, 1}; + source_copies[1].bufferOffset = upload_offset + 64; + source_copies[1].imageSubresource = {vk::ImageAspectFlagBits::eStencil, 0, 0, + 1}; + source_copies[1].imageExtent = {2, 2, 1}; + source.Upload(source_copies, upload.Handle(), upload_offset, 128); + destination.CopyMip(source, 1, 0); + + const auto [download_data, download_offset] = download.Map(128, 4); + Require(name, "depth/stencil download map", download_data != nullptr, + "combined depth/stencil download allocation failed"); + download.Commit(); + auto destination_copies = source_copies; + destination_copies[0].bufferOffset = download_offset; + destination_copies[0].imageSubresource.mipLevel = 1; + destination_copies[1].bufferOffset = download_offset + 64; + destination_copies[1].imageSubresource.mipLevel = 1; + destination.Download(destination_copies, download.Handle(), download_offset, + 128); + + auto buffered_info = MakeInfo(vk::Format::eR8Unorm, 1, 1); + buffered_info.extent = {8, 4, 1}; + buffered_info.pitch = 8; + buffered_info.mip_layout[0] = {0, 32, 8, 4}; + Image buffered_source(context, scheduler, buffered_info); + Image buffered_destination(context, scheduler, buffered_info); + constexpr std::array buffered_expected{ + 0x00, 0x11, 0x22, 0x33, 0x44, 0x55, 0x66, 0x77, 0x88, 0x99, 0xaa, + 0xbb, 0xcc, 0xdd, 0xee, 0xff, 0x10, 0x21, 0x32, 0x43, 0x54, 0x65, + 0x76, 0x87, 0x98, 0xa9, 0xba, 0xcb, 0xdc, 0xed, 0xfe, 0x0f}; + const auto [buffered_upload_data, buffered_upload_offset] = + upload.Map(buffered_expected.size(), 4); + Require(name, "buffered-copy upload map", buffered_upload_data != nullptr, + "buffered-copy source allocation failed"); + std::memcpy(buffered_upload_data, buffered_expected.data(), + buffered_expected.size()); + upload.Commit(); + vk::BufferImageCopy buffered_region{}; + buffered_region.bufferOffset = buffered_upload_offset; + buffered_region.bufferRowLength = buffered_info.pitch; + buffered_region.bufferImageHeight = buffered_info.extent.height; + buffered_region.imageSubresource = {vk::ImageAspectFlagBits::eColor, 0, 0, 1}; + buffered_region.imageExtent = buffered_info.extent; + buffered_source.Upload(std::span{&buffered_region, 1}, upload.Handle(), + buffered_upload_offset, buffered_expected.size()); + Buffer copy_scratch(context, scheduler, MemoryUsage::DeviceLocal, 0, AllFlags, + 16); + buffered_destination.CopyImageWithBuffer(buffered_source, copy_scratch); + const auto [buffered_download_data, buffered_download_offset] = + download.Map(buffered_expected.size(), 4); + Require(name, "buffered-copy download map", buffered_download_data != nullptr, + "buffered-copy destination allocation failed"); + download.Commit(); + buffered_region.bufferOffset = buffered_download_offset; + buffered_destination.Download(std::span{&buffered_region, 1}, + download.Handle(), buffered_download_offset, + buffered_expected.size()); + scheduler.Finish(); + download.Invalidate(download_offset, 128); + download.Invalidate(buffered_download_offset, buffered_expected.size()); + Require(name, "depth/stencil mip contents", + std::memcmp(download_data, expected_depth.data(), + sizeof(expected_depth)) == 0 && + std::memcmp(download_data + 64, expected_stencil.data(), + sizeof(expected_stencil)) == 0, + "combined depth/stencil mip copy lost an aspect"); + Require(name, "buffered-copy chunk contents", + std::memcmp(buffered_download_data, buffered_expected.data(), + buffered_expected.size()) == 0, + "real buffered image copy lost a row across its scratch boundary"); + + std::printf("[host] %-32s ok\n", name); +} + +void CheckSampledDepthDescriptor(RenderContext &renderer) { + auto &context = renderer.GetGraphics(); + CommandScheduler scheduler(renderer, context); + const auto make_info = [](uint32_t width, uint32_t height, uint32_t pitch, + uint32_t layers, vk::Format format, + Prospero::ImageType type) { + ImageInfo info{}; + info.pixel_format = format; + info.guest_format = + Prospero::GpuEnumValue(Prospero::BufferFormat::k32Float); + info.type = type; + info.extent = {width, height, 1}; + info.resources = {1, layers}; + info.pitch = pitch; + info.bytes_per_block = 4; + info.samples = 1; + info.tile_mode = Prospero::GpuEnumValue(Prospero::TileMode::kDepth); + info.mip_layout[0] = {0, static_cast(pitch) * height * layers * 4, + pitch, height}; + return info; + }; + ShaderTextureResource descriptor{{0x00eb0900u, 0xc1600000u, 0x00bcc14fu, 0x91800924u, 0x00000000u, 0x00700000u, 0x00000000u, 0x00000000u}}; - DepthStencilVulkanImage image; - image.extent = {1344, 756}; - image.guest_pitch = 1408; - image.layers = 1; - image.format = vk::Format::eD32SfloatS8Uint; - Require("SampledDepthDescriptor", "PPSA04319 padded pitch", - descriptor.Width5() + 1u == 1344 && - descriptor.Height5() + 1u == 756 && - TileGetTexturePitch(descriptor.Format(), 1344, 1, - descriptor.TileMode()) == 1408 && + Image image(context, scheduler, + make_info(1344, 756, 1408, 1, vk::Format::eD32SfloatS8Uint, + Prospero::ImageType::kColor2D)); + image.usage.depth_target = true; + Require("SampledDepthDescriptor", "normalized padded pitch", + descriptor.Width5() + 1u == image.info.extent.width && + descriptor.Height5() + 1u == image.info.extent.height && IsSupportedDepthTargetDescriptor(descriptor, image), - "valid padded-pitch depth texture was rejected"); + "normalized depth image rejected a valid padded descriptor"); descriptor.fields[3] = (descriptor.fields[3] & ~(0xfu << 28u)) | (Prospero::GpuEnumValue(Prospero::ImageType::kColor2DArray) << 28u); Require("SampledDepthDescriptor", "singleton array descriptor", IsSupportedDepthTargetDescriptor(descriptor, image), - "valid singleton-array depth texture was rejected"); + "normalized singleton-array depth view was rejected"); descriptor.fields[3] = (descriptor.fields[3] & ~(0xfu << 28u)) | (Prospero::GpuEnumValue(Prospero::ImageType::kColor2D) << 28u); @@ -10423,11 +15462,10 @@ void CheckSampledDepthDescriptor() { ShaderTextureResource cube_descriptor{{0x01267d00u, 0xc0700000u, 0x00ffc0ffu, 0xb1800924u, 0x00000005u, 0x00700000u, 0x00000000u, 0x00000000u}}; - DepthStencilVulkanImage cube_image; - cube_image.extent = {1024, 1024}; - cube_image.guest_pitch = 1024; - cube_image.layers = 6; - cube_image.format = vk::Format::eD32Sfloat; + Image cube_image(context, scheduler, + make_info(1024, 1024, 1024, 6, vk::Format::eD32Sfloat, + Prospero::ImageType::kColor2D)); + cube_image.usage.depth_target = true; ShaderRecompiler::IR::ImageResource cube_resource{}; cube_resource.kind = ShaderRecompiler::IR::ResourceKind::Image; cube_resource.dimension = @@ -10435,13 +15473,15 @@ void CheckSampledDepthDescriptor() { cube_resource.read = true; cube_resource.depth_compare = true; const auto cube_view = ResolveTargetTextureView( - cube_resource, Prospero::ImageType::kCube, 0, cube_image.layers); - Require("SampledDepthDescriptor", "PPSA06084 six-face depth cubemap", + cube_resource, Prospero::ImageType::kCube, 0, + cube_image.info.resources.layers); + Require("SampledDepthDescriptor", "normalized depth cube", IsSupportedDepthTargetDescriptor(cube_descriptor, cube_image) && IsSupportedDepthTextureEncoding(cube_descriptor) && cube_view.type == vk::ImageViewType::e2DArray && - cube_view.base_layer == 0 && cube_view.layer_count == 6, - "captured depth cubemap did not resolve to its layered target view"); + cube_view.layer_count == 6, + "normalized depth cube did not preserve its six-face view"); + auto partial_cube = cube_descriptor; partial_cube.fields[4] = 4; auto based_cube = cube_descriptor; @@ -10451,162 +15491,20 @@ void CheckSampledDepthDescriptor() { auto non_square_cube = cube_descriptor; non_square_cube.fields[2] = (non_square_cube.fields[2] & ~(0x3fffu << 14u)) | (511u << 14u); - DepthStencilVulkanImage non_square_image; - non_square_image.extent = cube_image.extent; - non_square_image.guest_pitch = cube_image.guest_pitch; - non_square_image.layers = cube_image.layers; - non_square_image.format = cube_image.format; - non_square_image.extent.height = 512; - Require( - "SampledDepthDescriptor", "cubemap hard guards", - !IsSupportedDepthTargetDescriptor(partial_cube, cube_image) && - !IsSupportedDepthTargetDescriptor(based_cube, cube_image) && - !IsSupportedDepthTextureEncoding(reserved_cube) && - !IsSupportedDepthTargetDescriptor(non_square_cube, non_square_image), - "partial, based, non-square, or reserved-bit depth cubemap was accepted"); - - image.guest_pitch = 1344; - Require("SampledDepthDescriptor", "target pitch mismatch", - !IsSupportedDepthTargetDescriptor(descriptor, image), - "mismatched unpadded target pitch was accepted"); - image.guest_pitch = 1472; - Require("SampledDepthDescriptor", "different padded pitch", - !IsSupportedDepthTargetDescriptor(descriptor, image), - "different padded target pitch was accepted"); - image.guest_pitch = 1408; - descriptor.fields[3] = - (descriptor.fields[3] & ~(0x1fu << 20u)) | - (Prospero::GpuEnumValue(Prospero::TileMode::kStandard4KB) << 20u); - Require("SampledDepthDescriptor", "non-depth tile", - !IsSupportedDepthTargetDescriptor(descriptor, image), - "non-depth texture descriptor was accepted"); - - DepthTargetInfo target{}; - target.address = 0x4a8c00000; - target.size = 0x100000; - target.width = 512; - target.height = 512; - target.pitch = 512; - target.layers = 1; - target.bytes_per_element = 4; - target.guest_format = - Prospero::GpuEnumValue(Prospero::BufferFormat::k32Float); - target.format = vk::Format::eD32Sfloat; - target.tile_mode = Prospero::GpuEnumValue(Prospero::TileMode::kDepth); - ImageInfo expanded{}; - expanded.address = target.address; - expanded.size = 0x400000; - expanded.format = target.guest_format; - expanded.width = target.width; - expanded.height = target.height; - expanded.pitch = target.pitch; - expanded.levels = 1; - expanded.view_levels = 1; - expanded.depth = 4; - expanded.tile = target.tile_mode; - expanded.type = Prospero::GpuEnumValue(Prospero::ImageType::kColor2DArray); - Require("SampledDepthDescriptor", "PPSA01530 layer expansion", - IsSampledDepthExpansion(expanded, target), - "exact D32 per-layer expansion was rejected"); - auto invalid_expansion = expanded; - invalid_expansion.pitch *= 2; + image.info.pitch = 1344; const bool rejects_pitch = - !IsSampledDepthExpansion(invalid_expansion, target); - invalid_expansion = expanded; - invalid_expansion.address++; - const bool rejects_address = - !IsSampledDepthExpansion(invalid_expansion, target); - invalid_expansion = expanded; - invalid_expansion.levels = 2; - const bool rejects_mips = !IsSampledDepthExpansion(invalid_expansion, target); - invalid_expansion = expanded; - invalid_expansion.type = - Prospero::GpuEnumValue(Prospero::ImageType::kColor2D); - const bool rejects_topology = - !IsSampledDepthExpansion(invalid_expansion, target); - auto metadata_target = target; - metadata_target.htile_address = 0x4a9c00000; - metadata_target.htile_size = 0x10000; - const bool rejects_metadata = - !IsSampledDepthExpansion(expanded, metadata_target); - Require("SampledDepthDescriptor", "expansion boundaries", - rejects_pitch && rejects_address && rejects_mips && - rejects_topology && rejects_metadata, - "unsupported depth expansion topology was accepted"); + !IsSupportedDepthTargetDescriptor(descriptor, image); + image.info.pitch = 1408; + Require("SampledDepthDescriptor", "normalized hard guards", + rejects_pitch && + !IsSupportedDepthTargetDescriptor(partial_cube, cube_image) && + !IsSupportedDepthTargetDescriptor(based_cube, cube_image) && + !IsSupportedDepthTextureEncoding(reserved_cube) && + !IsSupportedDepthTargetDescriptor(non_square_cube, cube_image), + "normalized depth descriptor accepted an incompatible image view"); std::printf("[host] %-32s ok\n", "SampledDepthDescriptor"); } -void CheckBufferCacheRangeMerge() { - BufferCacheRange merged{0x10000, 0x4000}; - Require("BufferCacheRangeMerge", "disjoint", - !MergeOverlappingBufferCacheRange(merged, {0x14000, 0x4000}) && - merged.address == 0x10000 && merged.size == 0x4000, - "adjacent range was merged"); - Require("BufferCacheRangeMerge", "right growth", - MergeOverlappingBufferCacheRange(merged, {0x12000, 0x8000}) && - merged.address == 0x10000 && merged.size == 0xa000, - "right overlap did not grow the cache range"); - Require("BufferCacheRangeMerge", "left growth", - MergeOverlappingBufferCacheRange(merged, {0xc000, 0x8000}) && - merged.address == 0xc000 && merged.size == 0xe000, - "left overlap did not grow the cache range"); - Require("BufferCacheRangeMerge", "contained", - MergeOverlappingBufferCacheRange(merged, {0x10000, 0x1000}) && - merged.address == 0xc000 && merged.size == 0xe000, - "contained range changed the cache union"); - std::printf("[host] %-32s ok\n", "BufferCacheRangeMerge"); -} - -void CheckBufferBindingStabilization() { - constexpr std::array requests{{ - {0x10000, 0x4000}, - {0x12000, 0x6000}, - {0x17000, 0x5000}, - {0x11000, 0x1000}, - }}; - BufferCacheRange allocation = requests.front(); - for (const auto request : std::span{requests}.subspan(1)) { - Require("BufferBindingStabilization", "discovery", - MergeOverlappingBufferCacheRange(allocation, request), - "overlapping discovery request did not join the cache allocation"); - } - Require("BufferBindingStabilization", "canonical range", - allocation.address == 0x10000 && allocation.size == 0xc000, - "discovery did not produce the complete cache allocation"); - - for (const auto request : requests) { - const auto before = allocation; - Require("BufferBindingStabilization", "second pass", - MergeOverlappingBufferCacheRange(allocation, request) && - allocation.address == before.address && - allocation.size == before.size, - "second-pass binding changed a fully discovered allocation"); - } - std::printf("[host] %-32s ok\n", "BufferBindingStabilization"); -} - -void CheckImageWriteActivationPolicy() { - Require("ImageWriteActivationPolicy", "standalone storage", - ClassifyImageWriteActivation(VulkanImageType::StorageTexture) == - ImageWriteActivation::Implicit, - "standalone storage image was sent through target write activation"); - for (const auto type : {VulkanImageType::VideoOut, - VulkanImageType::DepthStencil, - VulkanImageType::RenderTexture}) { - Require("ImageWriteActivationPolicy", "cache target", - ClassifyImageWriteActivation(type) == - ImageWriteActivation::Explicit, - "writable cache target skipped explicit write activation"); - } - for (const auto type : {VulkanImageType::Unknown, VulkanImageType::Texture}) { - Require("ImageWriteActivationPolicy", "unsupported", - ClassifyImageWriteActivation(type) == - ImageWriteActivation::Unsupported, - "unsupported writable image type received an activation policy"); - } - std::printf("[host] %-32s ok\n", "ImageWriteActivationPolicy"); -} - ShaderRecompiler::IR::ImageResource BasicStorageTextureResource() { ShaderRecompiler::IR::ImageResource resource{}; resource.kind = ShaderRecompiler::IR::ResourceKind::StorageImage; @@ -10645,6 +15543,11 @@ ShaderTextureResource BasicBgraStorageTextureDescriptor() { 0x00700000u, 0x00000000u, 0x00000000u}}; } +ShaderTextureResource Ppsa06228R11G11B10StorageTextureDescriptor() { + return {{0x10c6b500u, 0xc2400000u, 0x010dc1dfu, 0x91b003acu, 0x00000000u, + 0x00700000u, 0x00000000u, 0x00000000u}}; +} + ShaderTextureResource Ppsa01530MaxMipStorageTextureDescriptor() { return {{0x04a42900u, 0xc3e00000u, 0x000fc00fu, 0x91b0022cu, 0x00000000u, 0x00700050u, 0x00000000u, 0x00000000u}}; @@ -10742,13 +15645,11 @@ ShaderTextureResource BasicUintVolumeStorageTextureDescriptor() { resource = BasicLinearStorageTextureResource(); descriptor = BasicYzwxStorageTextureDescriptor(); resource.read = true; - } else if (std::strcmp(kind, "yzwx-format") == 0) { + } else if (std::strcmp(kind, "reserved-swizzle") == 0) { resource = BasicLinearStorageTextureResource(); - descriptor = BasicYzwxStorageTextureDescriptor(); - descriptor.fields[1] = - (descriptor.fields[1] & ~0x1ff00000u) | - (Prospero::GpuEnumValue(Prospero::BufferFormat::k16_16_16_16Float) - << 20u); + descriptor = BasicLinearStorageTextureDescriptor(); + descriptor.fields[3] = + (descriptor.fields[3] & ~0xfffu) | DstSel(4, 5, 6, 2); } else if (std::strcmp(kind, "resource") == 0) { resource.written = false; } else if (std::strcmp(kind, "type") == 0) { @@ -10758,7 +15659,7 @@ ShaderTextureResource BasicUintVolumeStorageTextureDescriptor() { } else if (std::strcmp(kind, "tile") == 0) { descriptor.fields[3] = (descriptor.fields[3] & ~(0x1fu << 20u)) | - (Prospero::GpuEnumValue(Prospero::TileMode::kStandard4KB) << 20u); + (Prospero::GpuEnumValue(Prospero::TileMode::kStandard256B) << 20u); } else if (std::strcmp(kind, "mip") == 0) { descriptor.fields[3] |= 1u << 16u; } else if (std::strcmp(kind, "swizzle") == 0) { @@ -10775,8 +15676,6 @@ ShaderTextureResource BasicUintVolumeStorageTextureDescriptor() { descriptor.fields[5] |= 1u << 4u; } else if (std::strcmp(kind, "reserved") == 0) { descriptor.fields[1] |= 1u << 29u; - } else if (std::strcmp(kind, "metadata") == 0) { - descriptor.fields[6] |= 1u << 21u; } else if (std::strcmp(kind, "uint-format") == 0) { descriptor.fields[1] = (descriptor.fields[1] & ~0x1ff00000u) | @@ -10808,6 +15707,16 @@ void CheckBasicStorageTextureDescriptor() { "basic 3D storage descriptor fixture is malformed"); ValidateStorageTexture(BasicStorageTextureResource(), descriptor, 0x10000); + const ShaderTextureResource extended{{ + 0x204aca00u, 0xc4700000u, 0x000fc00fu, 0xa1b00facu, + 0x0000003fu, 0x00700000u, 0x006b0000u, 0x00204b0au}}; + auto extended_resource = BasicStorageTextureResource(); + extended_resource.read = false; + Require("BasicStorageTexture", "extended descriptor", + extended.fields[6] != 0 && extended.fields[7] != 0, + "extended 3D storage descriptor fixture is malformed"); + ValidateStorageTexture(extended_resource, extended, 0x400000); + const auto linear = BasicLinearStorageTextureDescriptor(); Require("BasicStorageTexture", "linear descriptor", linear.Base40() == 0x4bcc40100ull && linear.Width5() + 1u == 3840 && @@ -10832,6 +15741,25 @@ void CheckBasicStorageTextureDescriptor() { "PPSA02604 BGRA 2D storage descriptor fixture is malformed"); ValidateStorageTexture(BasicBgraStorageTextureResource(), bgra, 0x870000); + const auto r11g11b10 = Ppsa06228R11G11B10StorageTextureDescriptor(); + Require("BasicStorageTexture", "PPSA06228 R11G11B10 descriptor", + r11g11b10.Base40() == 0x10c6b50000ull && + r11g11b10.Width5() + 1u == 1920 && + r11g11b10.Height5() + 1u == 1080 && + r11g11b10.Depth() + 1u == 1 && + r11g11b10.Format() == + Prospero::GpuEnumValue( + Prospero::BufferFormat::k11_11_10Float) && + r11g11b10.TileMode() == + Prospero::GpuEnumValue(Prospero::TileMode::kRenderTarget) && + r11g11b10.DstSelXYZW() == DstSel(4, 5, 6, 1), + "PPSA06228 R11G11B10 storage descriptor fixture is malformed"); + ValidateStorageTexture(BasicBgraStorageTextureResource(), r11g11b10, + 0x870000); + ValidateStorageColorView(vk::Format::eB10G11R11UfloatPack32, + vk::Format::eB10G11R11UfloatPack32, + r11g11b10.DstSelXYZW()); + const auto max_mip = Ppsa01530MaxMipStorageTextureDescriptor(); Require("BasicStorageTexture", "PPSA01530 max-mip descriptor", max_mip.Base40() == 0x4a4290000ull && max_mip.Width5() + 1u == 64 && @@ -10906,6 +15834,27 @@ void CheckBasicStorageTextureDescriptor() { yzwx.DstSelXYZW() == DstSel(5, 6, 7, 4), "PPSA04181 linear YZWX storage descriptor fixture is malformed"); ValidateStorageTexture(BasicLinearStorageTextureResource(), yzwx, 0x800); + auto all_swizzles = yzwx; + all_swizzles.fields[1] = + (all_swizzles.fields[1] & ~0x1ff00000u) | + (Prospero::GpuEnumValue(Prospero::BufferFormat::k16_16_16_16Float) + << 20u); + uint32_t valid_storage_swizzles = 0; + for (uint32_t swizzle = 0; swizzle <= 0xfffu; swizzle++) { + if (!IsValidImageSwizzle(swizzle)) { + continue; + } + all_swizzles.fields[3] = + (all_swizzles.fields[3] & ~0xfffu) | swizzle; + ValidateStorageTexture(BasicLinearStorageTextureResource(), all_swizzles, + 0x800); + ValidateStorageColorView(vk::Format::eR16G16B16A16Sfloat, + vk::Format::eR16G16B16A16Sfloat, swizzle); + valid_storage_swizzles++; + } + Require("BasicStorageTexture", "all write swizzles", + valid_storage_swizzles == 1296, + "valid write-only storage image mappings were rejected"); const auto array = BasicArrayStorageTextureDescriptor(); Require("BasicStorageTexture", "2D-array descriptor", @@ -10955,6 +15904,21 @@ void CheckBasicStorageTextureDescriptor() { ValidateStorageTexture(BasicUintVolumeStorageTextureResource(), uint_volume, 0x10000); + const ShaderTextureResource tiled_uint_volume{{ + 0x1ac0e530u, 0xc0b00000u, 0x0003c003u, 0xa0500004u, + 0x0000000fu, 0x00700000u, 0x00000000u, 0x00000000u}}; + Require("BasicStorageTexture", "tiled uint 3D descriptor", + tiled_uint_volume.Width5() + 1u == 16 && + tiled_uint_volume.Height5() + 1u == 16 && + tiled_uint_volume.Depth() + 1u == 16 && + tiled_uint_volume.Type() == + Prospero::GpuEnumValue(Prospero::ImageType::kColor3D) && + tiled_uint_volume.TileMode() == + Prospero::GpuEnumValue(Prospero::TileMode::kStandard4KB), + "captured tiled uint 3D storage descriptor fixture is malformed"); + ValidateStorageTexture(BasicUintVolumeStorageTextureResource(), + tiled_uint_volume, 0x2000); + const auto depth_tile = Ppsa14053DepthTileStorageTextureDescriptor(); Require("BasicStorageTexture", "PPSA14053 depth-tile descriptor", depth_tile.Base40() == 0x20144c0000ull && @@ -10971,15 +15935,19 @@ void CheckBasicStorageTextureDescriptor() { "malformed"); ValidateStorageTexture(Ppsa14053DepthTileStorageTextureResource(), depth_tile, 0x10000); - Require("BasicStorageTexture", "D32-compatible R32_UINT depth tile", - IsSupportedStorageDepthTile( - Prospero::GpuEnumValue(Prospero::BufferFormat::k32UInt), - Prospero::GpuEnumValue(Prospero::ImageType::kColor2D), 64, 64, 1), - "write-only R32_UINT depth-tile recreation was rejected"); + auto depth_tile_r32 = depth_tile; + depth_tile_r32.fields[1] = + (depth_tile_r32.fields[1] & ~(0x1ffu << 20u)) | + (Prospero::GpuEnumValue(Prospero::BufferFormat::k32UInt) << 20u); + depth_tile_r32.fields[3] = + (depth_tile_r32.fields[3] & ~(0xfu << 28u)) | + (Prospero::GpuEnumValue(Prospero::ImageType::kColor2D) << 28u); + auto depth_tile_r32_resource = Ppsa14053DepthTileStorageTextureResource(); + depth_tile_r32_resource.dimension = + ShaderRecompiler::Decoder::ImageDimension::Dim2D; + ValidateStorageTexture(depth_tile_r32_resource, depth_tile_r32, 0x10000); Require("BasicStorageTexture", "R32_UINT replicated write mapping", - IsSupportedStorageSwizzle( - Prospero::GpuEnumValue(Prospero::BufferFormat::k32UInt), - DstSel(4, 4, 4, 4)), + IsValidImageSwizzle(DstSel(4, 4, 4, 4)), "single-channel replicated destination selection was rejected"); char path[MAX_PATH]{}; @@ -10989,8 +15957,8 @@ void CheckBasicStorageTextureDescriptor() { for (const char *kind : {"resource", "type", "tile", "mip", "swizzle", "linear-rgb1-read", "bgra-read", "r16-float-read", "r8-unorm-read", "yzwx-read", - "yzwx-format", "array-base-view", "array-mip-view", "reserved", - "metadata", "uint-format", "uint-resource-float-format", + "reserved-swizzle", "array-base-view", "array-mip-view", "reserved", + "uint-format", "uint-resource-float-format", "depth-tile-read", "depth-tile-extent"}) { std::string command = std::string("\"") + path + "\" --storage-texture-descriptor-death " + kind; @@ -11026,20 +15994,21 @@ void CheckStorageTextureLinearUploadLayout() { constexpr uint32_t tile = Prospero::GpuEnumValue(Prospero::TileMode::kLinear); const auto pitch = TileGetTexturePitch(format, width, 1, tile); TileSizeAlign total{}; - TileGetTextureTotalSize(format, width, height, depth, pitch, 1, tile, false, total); + TileGetTextureTotalSize(format, width, height, depth, pitch, 1, tile, false, + total); const auto layout = TextureCalcUploadLayout( format, width, height, 1, depth, pitch, tile, total.size, true, false, "StorageTextureLinearTest"); - const auto regions = TextureBuildUploadRegions( - layout, vk::Format::eR8G8B8A8Unorm, width, height, depth, 1, false, false, - TextureUploadDestination::MipLevels); + const auto regions = + TextureBuildImageCopies(layout, width, height, depth, 1, false, false); Require("StorageTextureLinearUpload", "layout", pitch == width && total.size == 0x1fa4000 && total.align == 256 && layout.tile == tile && layout.pitch == width && layout.slice_stride == total.size && regions.size() == 1 && - regions[0].offset == 0 && regions[0].width == width && - regions[0].height == height && regions[0].pitch == width && - TextureCalcUploadSize(layout, regions, 1, depth) == total.size, + regions[0].bufferOffset == 0 && + regions[0].imageExtent.width == width && + regions[0].imageExtent.height == height && + regions[0].bufferRowLength == width, "linear RGBA8 storage upload lost Prospero pitch or allocation size"); std::printf("[host] %-32s ok\n", "StorageTextureLinearUpload"); } @@ -11058,13 +16027,13 @@ void CheckStorageTextureDepthTileUploadLayout() { TilePaddedSize padded{}; TileGetTextureSize(format, width, height, pitch, 1, tile, &slice, &level, &padded); - TileGetTextureTotalSize(format, width, height, depth, pitch, 1, tile, false, total); + TileGetTextureTotalSize(format, width, height, depth, pitch, 1, tile, false, + total); const auto layout = TextureCalcUploadLayout( format, width, height, 1, depth, pitch, tile, total.size, true, false, "StorageTextureDepthTileTest"); - const auto regions = TextureBuildUploadRegions( - layout, vk::Format::eR8Uint, width, height, depth, 1, true, false, - TextureUploadDestination::MipLevels); + const auto regions = + TextureBuildImageCopies(layout, width, height, depth, 1, true, false); Require("StorageTextureDepthTileUpload", "PPSA14053 layout", pitch == 256 && padded.width == 256 && padded.height == 256 && slice.size == 0x10000 && slice.align == 0x10000 && @@ -11076,43 +16045,13 @@ void CheckStorageTextureDepthTileUploadLayout() { layout.source_slice_stride == total.size && layout.level_sizes[0].size == pitch && layout.level_sizes[0].src_size == total.size && - regions.size() == 1 && regions[0].offset == 0 && - regions[0].width == width && regions[0].height == height && - regions[0].pitch == pitch && - TextureCalcUploadSize(layout, regions, 1, depth) == total.size, + regions.size() == 1 && regions[0].bufferOffset == 0 && + regions[0].imageExtent.width == width && + regions[0].imageExtent.height == height && + regions[0].bufferRowLength == pitch, "1x1 R8_UINT depth tile lost its 64 KiB source footprint"); std::printf("[host] %-32s ok\n", "StorageTextureDepthTileUpload"); } - -void CheckStorageTextureLinearReadbackLayout() { - ImageInfo info{}; - info.address = 0x4bcc40100ull; - info.size = 0x1fa4000; - info.format = Prospero::GpuEnumValue(Prospero::BufferFormat::k8_8_8_8UNorm); - info.width = 3840; - info.height = 2160; - info.pitch = 3840; - info.levels = 1; - info.tile = Prospero::GpuEnumValue(Prospero::TileMode::kLinear); - info.swizzle = DstSel(4, 5, 6, 1); - info.depth = 1; - info.type = Prospero::GpuEnumValue(Prospero::ImageType::kColor2D); - const auto transfer = - MakeColorImageTransferInfo(info, vk::Format::eR8G8B8A8Unorm, 4); - const auto linear_size = - ((static_cast(transfer.height) - 1) * transfer.pitch + - transfer.width) * - transfer.bytes_per_element; - Require("StorageTextureLinearReadback", "layout", - transfer.address == info.address && transfer.size == info.size && - transfer.format == vk::Format::eR8G8B8A8Unorm && - transfer.width == info.width && transfer.height == info.height && - transfer.pitch == info.pitch && transfer.bytes_per_element == 4 && - transfer.tile_mode == info.tile && linear_size == info.size, - "normalized storage readback lost the exact PS5 linear image layout"); - std::printf("[host] %-32s ok\n", "StorageTextureLinearReadback"); -} - void CheckStorageImageSwizzleSpecializationId() { std::array code{}; HW::ComputeShaderInfo regs{}; @@ -11163,17 +16102,14 @@ void CheckStorageTextureVolumeUploadLayout() { format, width, height, 1, depth, pitch, Prospero::GpuEnumValue(Prospero::TileMode::kRenderTarget), total.size, true, true, "StorageTextureVolumeTest"); - const auto regions = TextureBuildUploadRegions( - layout, vk::Format::eR16G16B16A16Sfloat, width, height, depth, 1, false, - true, TextureUploadDestination::MipLevels); + const auto regions = + TextureBuildImageCopies(layout, width, height, depth, 1, false, true); Require("StorageTextureVolumeUpload", "layout", pitch == 128 && total.size == 0x210000 && layout.slice_stride == 0x2208 && layout.source_slice_stride == 0 && layout.level_sizes[0].size == 0x2208 && - layout.level_sizes[0].src_size == 0 && - TextureCalcUploadSize(layout, regions, 1, depth) == - layout.slice_stride * depth, + layout.level_sizes[0].src_size == 0 && regions.size() == depth, "3D render-target upload did not preserve its compact linear layout"); std::vector infos; @@ -11207,71 +16143,33 @@ void CheckStorageTextureVolumeMipRegions() { const auto layout = TextureCalcUploadLayout( format, width, height, levels, depth, pitch, tile, total.size, true, true, "StorageTextureVolumeMipTest"); - const auto uploads = TextureBuildUploadRegions( - layout, vk::Format::eR8G8B8A8Unorm, width, height, depth, levels, false, - true, TextureUploadDestination::MipLevels); - const auto downloads = TextureBuildDownloadRegions(uploads); + const auto copies = TextureBuildImageCopies(layout, width, height, depth, + levels, false, true); - bool valid = uploads.size() == 8 && downloads.size() == uploads.size(); + bool valid = copies.size() == 8; size_t index = 0; for (uint32_t level = 0; level < levels && valid; level++) { const uint32_t mip_depth = std::max(depth >> level, 1u); const uint32_t mip_width = std::max(width >> level, 1u); const uint32_t mip_height = std::max(height >> level, 1u); for (uint32_t z = 0; z < mip_depth; z++, index++) { - const auto &upload = uploads[index]; - const auto &download = downloads[index]; - valid &= - upload.dst_level == level && upload.dst_z == static_cast(z) && - upload.width == mip_width && upload.height == mip_height && - upload.offset == - layout.level_sizes[level].offset + z * layout.slice_stride && - download.src_level == upload.dst_level && - download.src_z == upload.dst_z && download.offset == upload.offset && - download.pitch == upload.pitch; + const auto © = copies[index]; + valid &= copy.imageSubresource.mipLevel == level && + copy.imageOffset.z == static_cast(z) && + copy.imageExtent.width == mip_width && + copy.imageExtent.height == mip_height && + copy.bufferOffset == + layout.level_sizes[level].offset + z * layout.slice_stride && + copy.bufferRowLength == layout.padded_sizes[level].width && + copy.bufferImageHeight == layout.padded_sizes[level].height; } } - valid &= index == uploads.size(); + valid &= index == copies.size(); Require("StorageTextureVolumeMipRegions", "per-mip depth", valid, - "3D upload/readback regions did not shrink depth or preserve Vulkan " - "Z coordinates"); - const auto upload_size = - TextureCalcUploadSize(layout, uploads, levels, depth); - if (upload_size != total.size) { - std::ostringstream out; - out << "calculated=" << upload_size << " total=" << total.size - << " stride=" << layout.slice_stride; - Fail("StorageTextureVolumeMipRegions", "allocation size", out.str()); - } + "direction-neutral image copies did not shrink depth or preserve " + "Vulkan Z coordinates"); std::printf("[host] %-32s ok\n", "StorageTextureVolumeMipRegions"); } - -void CheckColorResolveLayers() { - RenderColorInfo src{}; - RenderColorInfo dst{}; - src.base_addr = dst.base_addr = 0x12340000; - src.base_mip_level = dst.base_mip_level = 2; - src.base_array_layer = 1; - dst.base_array_layer = 3; - src.vulkan_buffer = reinterpret_cast(uintptr_t{1}); - - Require( - "ColorResolveLayers", "identity", - !IsSameColorResolveSubresource(src, dst), - "different array layers were treated as the same resolve subresource"); - const auto copy = MakeColorResolveCopy(src, dst, 128, 64); - Require("ColorResolveLayers", "region", - ©.src_image == src.vulkan_buffer && copy.src_level == 2 && - copy.dst_level == 2 && copy.src_layer == 1 && - copy.dst_layer == 3 && copy.width == 128 && copy.height == 64, - "color resolve dropped its source or destination array layer"); - dst.base_array_layer = src.base_array_layer; - Require("ColorResolveLayers", "same subresource", - IsSameColorResolveSubresource(src, dst), - "matching color resolve subresources were not recognized"); - std::printf("[host] %-32s ok\n", "ColorResolveLayers"); -} - void CheckStandard64RenderTargetTileRoundTrip() { constexpr uint32_t format = Prospero::GpuEnumValue(Prospero::BufferFormat::k32Float); @@ -11300,15 +16198,18 @@ void CheckStandard64RenderTargetTileRoundTrip() { pitch == 384 && storage.size == 0x60000 && storage.align == 0x10000, "partial Standard64KB footprint was not padded in 128x128 blocks"); - RenderTargetInfo info{}; - info.address = 0x10000; - info.size = storage.size; - info.format = vk::Format::eR8G8B8A8Unorm; - info.width = width; - info.height = height; + ImageInfo info{}; + info.data = {0x10000, storage.size}; + info.pixel_format = vk::Format::eR8G8B8A8Unorm; + info.guest_format = format; + info.type = Prospero::ImageType::kColor2D; + info.extent = {width, height, 1}; + info.resources = {1, 1}; info.pitch = pitch; - info.bytes_per_element = 4; + info.bytes_per_block = 4; + info.samples = 1; info.tile_mode = tile; + info.mip_layout[0] = {0, storage.size, pitch, height}; Require("Standard64RenderTarget", "support boundary", IsSupportedStandard64RenderTarget(info) && IsTiledRenderTarget(info), "exact Standard64KB render target was not classified as tiled"); @@ -11318,12 +16219,12 @@ void CheckStandard64RenderTargetTileRoundTrip() { !IsSupportedDisplayRenderTargetTileMode(tile), "Standard64KB render target could alias a mode-27 display image"); auto unsupported = info; - unsupported.address += 4; + unsupported.data.address += 4; Require("Standard64RenderTarget", "address guard", !IsSupportedStandard64RenderTarget(unsupported), "unaligned Standard64KB backing was accepted"); unsupported = info; - unsupported.bytes_per_element = 8; + unsupported.bytes_per_block = 8; Require("Standard64RenderTarget", "element guard", !IsSupportedStandard64RenderTarget(unsupported), "unimplemented Standard64KB element size was accepted"); @@ -11333,17 +16234,17 @@ void CheckStandard64RenderTargetTileRoundTrip() { !IsSupportedStandard64RenderTarget(unsupported), "non-minimal Standard64KB pitch was accepted"); unsupported = info; - unsupported.size += 0x10000; + unsupported.data.size += 0x10000; Require("Standard64RenderTarget", "size guard", !IsSupportedStandard64RenderTarget(unsupported), "non-exact Standard64KB allocation was accepted"); unsupported = info; - unsupported.levels = 2; + unsupported.resources.levels = 2; Require("Standard64RenderTarget", "mip guard", !IsSupportedStandard64RenderTarget(unsupported), "unimplemented Standard64KB mip chain was accepted"); unsupported = info; - unsupported.layers = 2; + unsupported.resources.layers = 2; Require("Standard64RenderTarget", "layer guard", !IsSupportedStandard64RenderTarget(unsupported), "unimplemented Standard64KB array was accepted"); @@ -11372,8 +16273,7 @@ void CheckStorageTextureGpuOwnedRebindState() { "StorageTextureGpuOwnedRebind", "owned", tracker.IsRegionGpuModified(base, size) && page_manager.IsMapped(base, size) && - (!HostMemoryQueryReadable(base, size, readable) || - readable < size) && + (!HostMemoryQueryReadable(base, size, readable) || readable < size) && HostMemoryQueryRange(base, size, HostMemoryAccess::Mapped, mapped) && mapped == size && VirtualQuery(memory, &protection, sizeof(protection)) != 0 && @@ -11386,8 +16286,7 @@ void CheckStorageTextureGpuOwnedRebindState() { Require("StorageTextureGpuOwnedRebind", "clean readback", !tracker.IsRegionGpuModified(base, size) && !tracker.IsRegionCpuModified(base, size) && - HostMemoryQueryReadable(base, size, readable) && - readable == size, + HostMemoryQueryReadable(base, size, readable) && readable == size, "clean storage readback did not publish readable coherent backing"); tracker.MarkRegionAsGpuModified(base, size); readable = 0; @@ -11423,2116 +16322,6 @@ void CheckStorageTextureGpuOwnedRebindState() { std::printf("[host] %-32s ok\n", "StorageTextureGpuOwnedRebind"); } -void CheckStorageTextureSampledReuse() { - const auto image_2d = Prospero::GpuEnumValue(Prospero::ImageType::kColor2D); - const auto image_2d_array = - Prospero::GpuEnumValue(Prospero::ImageType::kColor2DArray); - const auto image_3d = Prospero::GpuEnumValue(Prospero::ImageType::kColor3D); - Require( - "StorageTextureSampledReuse", "view shapes", - SelectStorageSampledViewShape(image_2d, 1, 1) == - StorageSampledViewShape::Image2D && - SelectStorageSampledViewShape(image_2d_array, 16, 16) == - StorageSampledViewShape::Image2DArray && - SelectStorageSampledViewShape(image_3d, 16, 1) == - StorageSampledViewShape::Image3D && - SelectStorageSampledViewShape(image_2d, 16, 1) == - StorageSampledViewShape::Unsupported && - SelectStorageSampledViewShape(image_2d_array, 16, 1) == - StorageSampledViewShape::Unsupported && - SelectStorageSampledViewShape(image_3d, 16, 16) == - StorageSampledViewShape::Unsupported, - "sampled storage view shape or backing-layer validation is incorrect"); - ImageInfo storage{}; - storage.address = 0x78650000; - storage.size = 0x210000; - storage.format = - Prospero::GpuEnumValue(Prospero::BufferFormat::k16_16_16_16Float); - storage.width = 33; - storage.height = 33; - storage.pitch = 128; - storage.levels = 1; - storage.tile = Prospero::GpuEnumValue(Prospero::TileMode::kRenderTarget); - storage.swizzle = DstSel(4, 5, 6, 7); - storage.depth = 33; - storage.type = Prospero::GpuEnumValue(Prospero::ImageType::kColor3D); - - Require("StorageTextureSampledReuse", "exact", - ClassifyStorageSampledOverlap( - storage, storage, vk::Format::eR16G16B16A16Sfloat, - vk::Format::eR16G16B16A16Sfloat, true, - false) == StorageSampledOverlap::ExactImage, - "exact GPU-owned storage image was not reusable for sampling"); - auto incompatible = storage; - incompatible.format = - Prospero::GpuEnumValue(Prospero::BufferFormat::k16_16_16_16UNorm); - Require("StorageTextureSampledReuse", "incompatible", - ClassifyStorageSampledOverlap( - incompatible, storage, vk::Format::eR16G16B16A16Unorm, - vk::Format::eR16G16B16A16Sfloat, true, - false) == StorageSampledOverlap::Unsupported && - ClassifyStorageSampledOverlap( - storage, storage, vk::Format::eR16G16B16A16Sfloat, - vk::Format::eR16G16B16A16Sfloat, false, - false) == StorageSampledOverlap::Unsupported && - ClassifyStorageSampledOverlap( - storage, storage, vk::Format::eR16G16B16A16Sfloat, - vk::Format::eR16G16B16A16Sfloat, true, - true) == StorageSampledOverlap::Unsupported, - "incompatible storage-image state was accepted for sampling"); - auto separate = storage; - separate.address += 0x400000; - Require("StorageTextureSampledReuse", "separate", - ClassifyStorageSampledOverlap(separate, storage, - vk::Format::eR16G16B16A16Sfloat, - vk::Format::eR16G16B16A16Sfloat, true, - false) == StorageSampledOverlap::None, - "disjoint storage image was classified as an alias"); - - ImageInfo mip_chain{}; - mip_chain.address = 0x10eb50000ull; - mip_chain.size = 0x2b0000; - mip_chain.format = - Prospero::GpuEnumValue(Prospero::BufferFormat::k16_16_16_16Float); - mip_chain.width = 512; - mip_chain.height = 512; - mip_chain.pitch = 512; - mip_chain.levels = 10; - mip_chain.view_levels = 10; - mip_chain.tile = Prospero::GpuEnumValue(Prospero::TileMode::kRenderTarget); - mip_chain.swizzle = DstSel(4, 5, 6, 7); - mip_chain.type = image_2d; - auto mip_storage = mip_chain; - mip_storage.address = mip_chain.address + 0x30000; - mip_storage.size = 0x80000; - mip_storage.width = 256; - mip_storage.height = 256; - mip_storage.pitch = 256; - mip_storage.levels = 1; - mip_storage.view_levels = 1; - Require("StorageTextureSampledReuse", "captured mip transition", - IsExactRenderTargetMipStorage(mip_chain, mip_storage, - vk::Format::eR16G16B16A16Sfloat, - vk::Format::eR16G16B16A16Sfloat) && - ClassifyStorageSampledOverlap( - mip_chain, mip_storage, vk::Format::eR16G16B16A16Sfloat, - vk::Format::eR16G16B16A16Sfloat, true, false, true, false, - true) == StorageSampledOverlap::RetireStorage, - "captured GPU-owned level-1 storage allocation was not materialized " - "before " - "rebuilding its sampled chain"); - auto tail_storage = mip_chain; - tail_storage.size = 0x10000; - tail_storage.width = 128; - tail_storage.height = 64; - tail_storage.pitch = 128; - tail_storage.levels = 1; - tail_storage.view_levels = 1; - Require("StorageTextureSampledReuse", "captured mip-tail transition", - IsExactRenderTargetMipStorage(mip_chain, tail_storage, - vk::Format::eR16G16B16A16Sfloat, - vk::Format::eR16G16B16A16Sfloat) && - ClassifyStorageSampledOverlap( - mip_chain, tail_storage, vk::Format::eR16G16B16A16Sfloat, - vk::Format::eR16G16B16A16Sfloat, true, false, true, false, - true) == StorageSampledOverlap::RetireStorage, - "captured GPU-owned physical mip-tail block was not materialized " - "before rebuilding its sampled chain"); - auto malformed_mip = mip_storage; - malformed_mip.address += 0x10000; - auto malformed_tail = tail_storage; - malformed_tail.width = 64; - auto misaligned_chain = mip_chain; - misaligned_chain.address += 0x1000; - auto misaligned_tail = tail_storage; - misaligned_tail.address += 0x1000; - Require("StorageTextureSampledReuse", "mip transition guards", - !IsExactRenderTargetMipStorage(mip_chain, malformed_mip, - vk::Format::eR16G16B16A16Sfloat, - vk::Format::eR16G16B16A16Sfloat) && - !IsExactRenderTargetMipStorage(mip_chain, malformed_tail, - vk::Format::eR16G16B16A16Sfloat, - vk::Format::eR16G16B16A16Sfloat) && - !IsExactRenderTargetMipStorage(misaligned_chain, misaligned_tail, - vk::Format::eR16G16B16A16Sfloat, - vk::Format::eR16G16B16A16Sfloat) && - ClassifyStorageSampledOverlap( - mip_chain, mip_storage, vk::Format::eR16G16B16A16Sfloat, - vk::Format::eR16G16B16A16Sfloat, true, false, true, true, - true) == StorageSampledOverlap::Unsupported && - ClassifyStorageSampledOverlap( - mip_chain, mip_storage, vk::Format::eR16G16B16A16Sfloat, - vk::Format::eR16G16B16A16Sfloat, true, false, true, false, - false) == StorageSampledOverlap::Unsupported && - ClassifyStorageSampledOverlap( - mip_chain, mip_storage, vk::Format::eR16G16B16A16Sfloat, - vk::Format::eR16G16B16A16Sfloat, true, true, true, false, - true) == StorageSampledOverlap::Unsupported, - "malformed or unsafe mip-storage ownership was accepted"); - - ImageInfo ppsa02604_storage{}; - ppsa02604_storage.address = 0x7c690000; - ppsa02604_storage.size = 0x870000; - ppsa02604_storage.format = - Prospero::GpuEnumValue(Prospero::BufferFormat::k8_8_8_8UNorm); - ppsa02604_storage.width = 1920; - ppsa02604_storage.height = 1080; - ppsa02604_storage.pitch = 1920; - ppsa02604_storage.tile = - Prospero::GpuEnumValue(Prospero::TileMode::kRenderTarget); - ppsa02604_storage.swizzle = DstSel(6, 5, 4, 7); - ppsa02604_storage.type = - Prospero::GpuEnumValue(Prospero::ImageType::kColor2D); - auto ppsa02604_sampled = ppsa02604_storage; - ppsa02604_sampled.format = - Prospero::GpuEnumValue(Prospero::BufferFormat::k8_8_8_8Srgb); - Require("StorageTextureSampledReuse", "PPSA02604 sRGB view", - ClassifyStorageSampledOverlap( - ppsa02604_sampled, ppsa02604_storage, vk::Format::eR8G8B8A8Srgb, - vk::Format::eR8G8B8A8Unorm, true, - false) == StorageSampledOverlap::ExactImage, - "PPSA02604 GPU-owned UNORM storage image did not accept its exact " - "sRGB view"); - auto alternate_swizzle = ppsa02604_storage; - alternate_swizzle.swizzle = DstSel(4, 5, 6, 7); - Require( - "StorageTextureSampledReuse", "descriptor swizzle view", - ClassifyStorageSampledOverlap(alternate_swizzle, ppsa02604_storage, - vk::Format::eR8G8B8A8Unorm, - vk::Format::eR8G8B8A8Unorm, true, - false) == StorageSampledOverlap::ExactImage, - "exact storage backing rejected a distinct sampled descriptor swizzle"); - ImageInfo depth_uint_storage{}; - depth_uint_storage.address = 0x4a5c90000; - depth_uint_storage.size = 0x10000; - depth_uint_storage.format = - Prospero::GpuEnumValue(Prospero::BufferFormat::k32UInt); - depth_uint_storage.width = 64; - depth_uint_storage.height = 64; - depth_uint_storage.pitch = 128; - depth_uint_storage.levels = 1; - depth_uint_storage.view_levels = 1; - depth_uint_storage.depth = 1; - depth_uint_storage.tile = Prospero::GpuEnumValue(Prospero::TileMode::kDepth); - depth_uint_storage.swizzle = DstSel(4, 4, 4, 4); - depth_uint_storage.type = image_2d; - auto depth_float_sampled = depth_uint_storage; - depth_float_sampled.format = - Prospero::GpuEnumValue(Prospero::BufferFormat::k32Float); - depth_float_sampled.swizzle = DstSel(4, 0, 0, 1); - Require("StorageTextureSampledReuse", "PPSA01530 R32 view", - ClassifyStorageSampledOverlap(depth_float_sampled, depth_uint_storage, - vk::Format::eR32Sfloat, - vk::Format::eR32Uint, true, false) == - StorageSampledOverlap::ExactImage, - "PPSA01530 GPU-owned R32_UINT storage image did not accept its exact " - "R32_SFLOAT view"); - const auto usage = TextureFormatUsage::Sampled | TextureFormatUsage::Storage; - Require( - "StorageTextureSampledReuse", "usage", - (TextureGetUsage(usage) & - (vk::ImageUsageFlagBits::eSampled | vk::ImageUsageFlagBits::eStorage)) == - (vk::ImageUsageFlagBits::eSampled | - vk::ImageUsageFlagBits::eStorage) && - (TextureGetViewUsage(usage) & (vk::ImageUsageFlagBits::eSampled | - vk::ImageUsageFlagBits::eStorage)) == - (vk::ImageUsageFlagBits::eSampled | - vk::ImageUsageFlagBits::eStorage), - "shared storage image or view is missing sampled/storage usage"); - std::printf("[host] %-32s ok\n", "StorageTextureSampledReuse"); -} - -void CheckStorageTextureDepthAlias() { - ImageInfo storage{}; - storage.address = 0x201ae60000ull; - storage.size = 0x870000; - DepthTargetInfo depth{}; - depth.address = 0x2046ca0000ull; - depth.size = 0x1fe0000; - depth.stencil_address = storage.address; - depth.stencil_size = storage.size; - Require("StorageTextureDepthAlias", "inactive stencil", - ClassifyStorageDepthOverlap(storage, true, false, false, true, - depth) == DepthOverlap::RetireStorage, - "inactive stencil storage ownership was not retired"); - depth.stencil_access = true; - Require("StorageTextureDepthAlias", "active stencil", - ClassifyStorageDepthOverlap(storage, true, false, false, true, - depth) == DepthOverlap::Unsupported, - "active stencil storage contents were discarded"); - depth.stencil_access = false; - storage.address = depth.address; - depth.depth_access = true; - Require("StorageTextureDepthAlias", "depth aspect", - ClassifyStorageDepthOverlap(storage, true, false, false, true, - depth) == DepthOverlap::Unsupported, - "depth-aspect storage contents were discarded"); - Require("StorageTextureDepthAlias", "clean depth transition", - ClassifyStorageDepthOverlap(storage, false, false, false, false, - depth) == DepthOverlap::RetireStorage, - "guest-current storage image was not retired for a depth target"); - Require("StorageTextureDepthAlias", "depth ownership guards", - ClassifyStorageDepthOverlap(storage, false, true, false, false, - depth) == DepthOverlap::Unsupported && - ClassifyStorageDepthOverlap(storage, false, false, true, false, - depth) == DepthOverlap::Unsupported && - ClassifyStorageDepthOverlap(storage, false, false, false, true, - depth) == DepthOverlap::Unsupported, - "incoherent storage-to-depth transition was admitted"); - storage.address = 0x2000000000ull; - Require("StorageTextureDepthAlias", "disjoint", - ClassifyStorageDepthOverlap(storage, true, true, true, true, depth) == - DepthOverlap::None, - "disjoint storage image was classified as an alias"); - std::printf("[host] %-32s ok\n", "StorageTextureDepthAlias"); -} - -[[noreturn]] void RunStorageTextureAccessDeathCase(const char *kind) { - GraphicContext context{}; - constexpr uintptr_t base = 0x0000000200300000ull; - constexpr uint64_t size = 0x10000; - auto *memory = VirtualAlloc(reinterpret_cast(base), size, - MEM_RESERVE | MEM_COMMIT, PAGE_READWRITE); - if (memory != reinterpret_cast(base)) { - EXIT("storage access child could not reserve its fixed test address\n"); - } - ResourceMutex resource_mutex; - PageManager page_manager(CacheFault, nullptr); - BufferCache buffer_cache(context, page_manager, resource_mutex); - if (std::strcmp(kind, "read") == 0) { - page_manager.OnGpuMap(base, size, GpuAccess::Write); - buffer_cache.ValidateGpuAccess(base, size, true, true); - } else if (std::strcmp(kind, "write") == 0) { - page_manager.OnGpuMap(base, size, GpuAccess::Read); - buffer_cache.ValidateGpuAccess(base, size, true, true); - } else { - std::_Exit(0x7e); - } - std::_Exit(0x7f); -} - -void CheckStorageTextureAccessPermissions() { - GraphicContext context{}; - constexpr uintptr_t base = 0x0000000200300000ull; - constexpr uint64_t size = 0x10000; - auto *memory = VirtualAlloc(reinterpret_cast(base), size, - MEM_RESERVE | MEM_COMMIT, PAGE_READWRITE); - Require("StorageTextureAccess", "allocation", - memory == reinterpret_cast(base), - "fixed VirtualAlloc failed"); - ResourceMutex resource_mutex; - PageManager page_manager(CacheFault, nullptr); - BufferCache buffer_cache(context, page_manager, resource_mutex); - page_manager.OnGpuMap(base, size, GpuAccess::ReadWrite); - buffer_cache.ValidateGpuAccess(base, size, true, true); - page_manager.OnGpuUnmap(base, size, GpuAccess::ReadWrite); - Require("StorageTextureAccess", "free", - VirtualFree(memory, 0, MEM_RELEASE) != 0, "VirtualFree failed"); - - char path[MAX_PATH]{}; - Require("StorageTextureAccess", "host", - GetModuleFileNameA(nullptr, path, MAX_PATH) != 0, - "GetModuleFileName failed"); - for (const char *kind : {"read", "write"}) { - std::string command = - std::string("\"") + path + "\" --storage-texture-access-death " + kind; - std::vector mutable_command(command.begin(), command.end()); - mutable_command.push_back('\0'); - STARTUPINFOA startup{sizeof(startup)}; - PROCESS_INFORMATION process{}; - Require("StorageTextureAccess", "host", - CreateProcessA(nullptr, mutable_command.data(), nullptr, nullptr, - FALSE, CREATE_NO_WINDOW, nullptr, nullptr, &startup, - &process) != 0, - "CreateProcess failed"); - Require("StorageTextureAccess", "host", - WaitForSingleObject(process.hProcess, 10000) == WAIT_OBJECT_0, - "access death case timed out"); - DWORD exit_code = 0; - const bool exited = GetExitCodeProcess(process.hProcess, &exit_code) != 0; - CloseHandle(process.hThread); - CloseHandle(process.hProcess); - Require("StorageTextureAccess", "host", exited && exit_code == 321, - std::string(kind) + " access denial did not report a fatal error"); - } - std::printf("[host] %-32s ok\n", "StorageTextureAccessPermissions"); -} - -[[noreturn]] void RunMetaOverlapDeathCase(const char *kind) { - GraphicContext context{}; - constexpr uint64_t address = 0x0000000200010000ull; - constexpr uint64_t allocation_size = 0x10000; - auto *memory = static_cast( - VirtualAlloc(reinterpret_cast(address), allocation_size, - MEM_RESERVE | MEM_COMMIT, PAGE_READWRITE)); - if (memory != reinterpret_cast(address)) { - EXIT("metadata-overlap child could not reserve its fixed test address\n"); - } - ResourceMutex resource_mutex; - PageManager page_manager(CacheFault, nullptr); - BufferCache buffer_cache(context, page_manager, resource_mutex); - TextureCache texture_cache(context, page_manager, buffer_cache, - resource_mutex); - buffer_cache.SetTextureCache(texture_cache); - page_manager.OnGpuMap(address, allocation_size); - texture_cache.RegisterMeta(address, 0x8000); - - if (std::strcmp(kind, "metadata-size") == 0) { - texture_cache.RegisterMeta(address, 0x10000); - } else if (std::strcmp(kind, "texture") == 0) { - ImageInfo info{}; - info.address = address; - info.size = 0x1000; - info.width = 1; - info.height = 1; - (void)texture_cache.FindTexture(*reinterpret_cast(1), info, - false); - } else if (std::strcmp(kind, "render-target") == 0) { - RenderTargetInfo info{}; - info.address = address; - info.size = 0x1000; - info.format = vk::Format::eR8G8B8A8Unorm; - info.width = 1; - info.height = 1; - info.pitch = 1; - info.bytes_per_element = 4; - info.tile_mode = Prospero::GpuEnumValue(Prospero::TileMode::kLinear); - (void)texture_cache.FindRenderTarget(*reinterpret_cast(1), - info); - } else if (std::strcmp(kind, "depth-target") == 0) { - DepthTargetInfo info{}; - info.address = address; - info.size = 0x1000; - info.format = vk::Format::eD16Unorm; - info.guest_format = - Prospero::GpuEnumValue(Prospero::BufferFormat::k16UNorm); - info.width = 1; - info.height = 1; - info.pitch = 1; - info.bytes_per_element = 2; - info.tile_mode = Prospero::GpuEnumValue(Prospero::TileMode::kDepth); - (void)texture_cache.FindDepthTarget(*reinterpret_cast(1), - info); - } else if (std::strcmp(kind, "copy-stale") == 0) { - if (!texture_cache.ClearMeta(address)) { - std::_Exit(0x7f); - } - buffer_cache.CopyBuffer(nullptr, address + 0x9000, address, 0x1000); - } else { - EXIT("metadata-reuse child received an unknown mode\n"); - } - std::_Exit(0x7f); -} - -ShaderRecompiler::IR::ImageResource BasicMetadataReuseResource() { - ShaderRecompiler::IR::ImageResource resource{}; - resource.kind = ShaderRecompiler::IR::ResourceKind::Image; - resource.dimension = ShaderRecompiler::Decoder::ImageDimension::Dim2D; - resource.read = true; - return resource; -} - -ShaderTextureResource BasicMetadataReuseDescriptor() { - return {{0x0062b100u, 0xcad00000u, 0x003fc03fu, 0x90500facu, 0x00000000u, - 0x00700000u, 0x00000000u, 0x00000000u}}; -} - -ShaderTextureResource Ppsa06084MetadataReuseDescriptor() { - return {{0x00722290u, 0xc0100000u, 0x000fc00fu, 0x90560800u, 0x00000000u, - 0x00700060u, 0x00000000u, 0x00000000u}}; -} - -[[noreturn]] void RunMetadataDescriptorDeathCase(const char *kind) { - auto resource = BasicMetadataReuseResource(); - auto descriptor = BasicMetadataReuseDescriptor(); - if (std::strcmp(kind, "field1-reserved") == 0) { - descriptor.fields[1] |= 0x20000000u; - } else if (std::strcmp(kind, "field2-low-reserved") == 0) { - descriptor.fields[2] |= 0x00001000u; - } else if (std::strcmp(kind, "field2-high-reserved") == 0) { - descriptor.fields[2] |= 0x10000000u; - } else if (std::strcmp(kind, "unsupported-format") == 0) { - descriptor.fields[1] = - (descriptor.fields[1] & ~0x1ff00000u) | - (Prospero::GpuEnumValue(Prospero::BufferFormat::k8_8_8_8SInt) << 20u); - } else if (std::strcmp(kind, "uint-format") == 0) { - descriptor.fields[1] = - (descriptor.fields[1] & ~0x1ff00000u) | - (Prospero::GpuEnumValue(Prospero::BufferFormat::k8_8_8_8UInt) << 20u); - } else if (std::strcmp(kind, "invalid-swizzle") == 0) { - descriptor = Ppsa06084MetadataReuseDescriptor(); - descriptor.fields[3] = - (descriptor.fields[3] & ~0xfffu) | DstSel(2, 0, 0, 4); - } else if (std::strcmp(kind, "mip-view-outside-allocation") == 0) { - descriptor = Ppsa06084MetadataReuseDescriptor(); - descriptor.fields[3] = - (descriptor.fields[3] & ~(0xfu << 16u)) | (7u << 16u); - } else { - std::_Exit(0x7e); - } - ValidateMetadataReuseTexture(resource, descriptor, 0x10000); - std::_Exit(0x7f); -} - -void CheckMetaOverlapDeaths() { - char path[MAX_PATH]{}; - Require("MetaOverlap", "host", - GetModuleFileNameA(nullptr, path, MAX_PATH) != 0, - "GetModuleFileName failed"); - for (const char *kind : {"metadata-size", "texture", "render-target", - "depth-target", "copy-stale"}) { - std::string command = - std::string("\"") + path + "\" --meta-overlap-death " + kind; - std::vector mutable_command(command.begin(), command.end()); - mutable_command.push_back('\0'); - STARTUPINFOA startup{sizeof(startup)}; - PROCESS_INFORMATION process{}; - Require("MetaOverlap", "host", - CreateProcessA(nullptr, mutable_command.data(), nullptr, nullptr, - FALSE, CREATE_NO_WINDOW, nullptr, nullptr, &startup, - &process) != 0, - "CreateProcess failed"); - Require("MetaOverlap", "host", - WaitForSingleObject(process.hProcess, 10000) == WAIT_OBJECT_0, - "death case timed out"); - DWORD exit_code = 0; - const bool exited = GetExitCodeProcess(process.hProcess, &exit_code) != 0; - CloseHandle(process.hThread); - CloseHandle(process.hProcess); - Require("MetaOverlap", "host", exited && exit_code == 321, - std::string(kind) + " did not reject registered metadata overlap"); - } - std::printf("[host] %-32s ok\n", "MetaOverlapRegistration"); -} - -void CheckOverlappingMetadataViews() { - GraphicContext context{}; - constexpr uintptr_t base = 0x0000000200010000ull; - constexpr uint64_t allocation_size = 0x60000; - constexpr uint64_t metadata_size = 0x30000; - constexpr uint64_t second = base + 0x18000; - auto *memory = static_cast( - VirtualAlloc(reinterpret_cast(base), allocation_size, - MEM_RESERVE | MEM_COMMIT, PAGE_READWRITE)); - Require("OverlappingMetadataViews", "allocation", - memory == reinterpret_cast(base), - "fixed VirtualAlloc failed"); - - ResourceMutex resource_mutex; - CacheFaultContext fault_context; - PageManager page_manager(CacheFault, &fault_context); - BufferCache buffer_cache(context, page_manager, resource_mutex); - TextureCache texture_cache(context, page_manager, buffer_cache, - resource_mutex); - fault_context.texture = &texture_cache; - buffer_cache.SetTextureCache(texture_cache); - page_manager.OnGpuMap(base, allocation_size); - - texture_cache.RegisterMeta(base, metadata_size); - texture_cache.RegisterMeta(second, metadata_size); - Require("OverlappingMetadataViews", "registration", - texture_cache.IsMeta(base) && texture_cache.IsMeta(second) && - texture_cache.IsMetaRange(base, metadata_size) && - texture_cache.IsMetaRange(second, metadata_size) && - !texture_cache.IsMetaRange(base, metadata_size - 0x8000) && - !texture_cache.IsMetaRange(base + 0x8000, metadata_size), - "overlapping metadata identities were not retained"); - Require("OverlappingMetadataViews", "clear", - texture_cache.ClearMeta(base) && texture_cache.ClearMeta(second) && - texture_cache.IsMetaCleared(base, 0) && - texture_cache.IsMetaCleared(second, 0), - "overlapping metadata clears were not retained independently"); - Require("OverlappingMetadataViews", "fault", - page_manager.HandleFault(PageFaultAccess::Write, second), - "shared metadata page did not transfer to CPU ownership"); - Require("OverlappingMetadataViews", "ownership", - !texture_cache.QueryRegion(second, 0x1000).gpu_metadata_bytes, - "shared metadata page retained GPU ownership after a CPU fault"); - - texture_cache.UnmapMemory(base, allocation_size); - page_manager.OnGpuUnmap(base, allocation_size); - Require("OverlappingMetadataViews", "free", - VirtualFree(memory, 0, MEM_RELEASE) != 0, "VirtualFree failed"); - std::printf("[host] %-32s ok\n", "OverlappingMetadataViews"); -} - -void CheckQueryRegionAggregation() { - GraphicContext context{}; - constexpr uintptr_t base = 0x0000000200010000ull; - constexpr uint64_t allocation_size = 0x10000; - constexpr uint64_t metadata_size = 0x180; - auto *memory = static_cast( - VirtualAlloc(reinterpret_cast(base), allocation_size, - MEM_RESERVE | MEM_COMMIT, PAGE_READWRITE)); - Require("QueryRegionAggregation", "allocation", - memory == reinterpret_cast(base), - "fixed VirtualAlloc failed"); - - ResourceMutex resource_mutex; - CacheFaultContext fault_context; - PageManager page_manager(CacheFault, &fault_context); - BufferCache buffer_cache(context, page_manager, resource_mutex); - TextureCache texture_cache(context, page_manager, buffer_cache, - resource_mutex); - fault_context.texture = &texture_cache; - buffer_cache.SetTextureCache(texture_cache); - page_manager.OnGpuMap(base, allocation_size); - - const auto empty = texture_cache.QueryRegion(base + 0x20, 0x40); - Require("QueryRegionAggregation", "empty", - !empty.image_pages && !empty.image_bytes && !empty.gpu_image_bytes && - !empty.non_sampled_pages && !empty.metadata_pages && - !empty.metadata_bytes && !empty.gpu_metadata_bytes, - "empty region reported cached ownership"); - - texture_cache.RegisterMeta(base, metadata_size); - const auto bytes = texture_cache.QueryRegion(base + 0x20, 0x40); - const auto page_only = texture_cache.QueryRegion(base + 0x400, 0x40); - const auto disjoint = texture_cache.QueryRegion(base + 0x1000, 0x40); - Require( - "QueryRegionAggregation", "page and byte overlap", - bytes.metadata_pages && bytes.metadata_bytes && - !bytes.gpu_metadata_bytes && page_only.metadata_pages && - !page_only.metadata_bytes && !page_only.gpu_metadata_bytes && - !disjoint.metadata_pages && !disjoint.metadata_bytes && - !bytes.image_pages && !bytes.image_bytes && !bytes.gpu_image_bytes && - !bytes.non_sampled_pages, - "metadata page candidates were not separated from exact byte overlap"); - - Require("QueryRegionAggregation", "clear", texture_cache.ClearMeta(base), - "metadata clear setup failed"); - const auto gpu_bytes = texture_cache.QueryRegion(base + 0x20, 0x40); - const auto gpu_page_only = texture_cache.QueryRegion(base + 0x400, 0x40); - Require("QueryRegionAggregation", "GPU ownership", - gpu_bytes.metadata_pages && gpu_bytes.metadata_bytes && - gpu_bytes.gpu_metadata_bytes && gpu_page_only.metadata_pages && - !gpu_page_only.metadata_bytes && - !gpu_page_only.gpu_metadata_bytes, - "GPU metadata ownership escaped its exact registered byte range"); - - texture_cache.UnmapMemory(base, allocation_size); - page_manager.OnGpuUnmap(base, allocation_size); - Require("QueryRegionAggregation", "free", - VirtualFree(memory, 0, MEM_RELEASE) != 0, "VirtualFree failed"); - std::printf("[host] %-32s ok\n", "QueryRegionAggregation"); -} - -void CheckGpuMetadataReuse() { - GraphicContext context{}; - constexpr uintptr_t base = 0x0000000200010000ull; - constexpr uint64_t allocation_size = 0x20000; - constexpr uint64_t metadata_size = 0x18000; - constexpr uint32_t layers = 3; - auto *memory = static_cast( - VirtualAlloc(reinterpret_cast(base), allocation_size, - MEM_RESERVE | MEM_COMMIT, PAGE_READWRITE)); - Require("GpuMetadataReuse", "allocation", - memory == reinterpret_cast(base), - "fixed VirtualAlloc failed"); - - ResourceMutex resource_mutex; - CacheFaultContext fault_context; - PageManager page_manager(CacheFault, &fault_context); - BufferCache buffer_cache(context, page_manager, resource_mutex); - TextureCache texture_cache(context, page_manager, buffer_cache, - resource_mutex); - fault_context.texture = &texture_cache; - buffer_cache.SetTextureCache(texture_cache); - page_manager.OnGpuMap(base, allocation_size); - - texture_cache.RegisterMeta(base, metadata_size, layers); - TextureCache::MetaRangeInfo full_meta{}; - TextureCache::MetaRangeInfo slice_meta{}; - TextureCache::MetaRangeInfo invalid_meta{}; - constexpr uint64_t slice_size = metadata_size / layers; - Require("GpuMetadataReuse", "exact metadata ranges", - texture_cache.ResolveMetaRange(base, metadata_size, full_meta) && - full_meta.metadata_address == base && - full_meta.metadata_size == metadata_size && full_meta.full && - full_meta.slice == 0 && - texture_cache.ResolveMetaRange(base + slice_size, slice_size, - slice_meta) && - slice_meta.metadata_address == base && - slice_meta.metadata_size == metadata_size && !slice_meta.full && - slice_meta.slice == 1 && - !texture_cache.ResolveMetaRange(base + 0x1000, slice_size, - invalid_meta) && - !texture_cache.ResolveMetaRange(base + slice_size, slice_size / 2, - invalid_meta), - "whole and per-slice metadata ranges were not classified exactly"); - Require("GpuMetadataReuse", "clear", texture_cache.ClearMeta(base), - "metadata clear setup failed"); - const bool full_image_transition = - texture_cache.InvalidateMemoryFromGPU(base, allocation_size); - Require("GpuMetadataReuse", "discard", - !full_image_transition && !texture_cache.IsMeta(base) && - !texture_cache.QueryRegion(base, allocation_size).metadata_bytes, - "metadata-only overwrite retained identity or claimed an image " - "transition"); - texture_cache.RegisterMeta(base, metadata_size, layers); - Require("GpuMetadataReuse", "re-register", - texture_cache.IsMetaRange(base, metadata_size) && - texture_cache.ClearMeta(base), - "metadata identity could not be reused after a GPU overwrite"); - Require("GpuMetadataReuse", "layered clear", - texture_cache.IsMetaCleared(base, 0) && - texture_cache.IsMetaCleared(base, 1) && - texture_cache.IsMetaCleared(base, 2) && - texture_cache.TouchMeta(base, 1, false) && - texture_cache.IsMetaCleared(base, 0) && - !texture_cache.IsMetaCleared(base, 1) && - texture_cache.IsMetaCleared(base, 2), - "consuming one metadata slice erased another slice's lazy clear"); - const bool partial_image_transition = - texture_cache.InvalidateMemoryFromGPU(base + 0x1000, 0x1000); - Require("GpuMetadataReuse", "partial discard", - !partial_image_transition && !texture_cache.IsMeta(base), - "partial metadata overwrite retained identity or claimed an image " - "transition"); - - texture_cache.UnmapMemory(base, allocation_size); - page_manager.OnGpuUnmap(base, allocation_size); - Require("GpuMetadataReuse", "free", VirtualFree(memory, 0, MEM_RELEASE) != 0, - "VirtualFree failed"); - std::printf("[host] %-32s ok\n", "GpuMetadataReuse"); -} - -void CheckMetadataReuseDescriptors() { - ValidateMetadataReuseTexture(BasicMetadataReuseResource(), - BasicMetadataReuseDescriptor(), 0x10000); - const auto captured = Ppsa06084MetadataReuseDescriptor(); - ValidateMetadataReuseTexture(BasicMetadataReuseResource(), captured, 0x2000); - Require("MetadataReuseDescriptor", "PPSA06084 sampled mip chain", - captured.Format() == - Prospero::GpuEnumValue(Prospero::BufferFormat::k8UNorm) && - captured.Type() == - Prospero::GpuEnumValue(Prospero::ImageType::kColor2D) && - captured.BaseLevel() == 0 && captured.LastLevel() == 6 && - captured.MaxMip() == 6 && - captured.TileMode() == - Prospero::GpuEnumValue(Prospero::TileMode::kStandard4KB) && - captured.DstSelXYZW() == DstSel(0, 0, 0, 4), - "captured metadata-reuse descriptor fields decoded unexpectedly"); - char path[MAX_PATH]{}; - Require("MetadataReuseDescriptor", "host", - GetModuleFileNameA(nullptr, path, MAX_PATH) != 0, - "GetModuleFileName failed"); - for (const char *kind : - {"field1-reserved", "field2-low-reserved", "field2-high-reserved", - "unsupported-format", "uint-format", "invalid-swizzle", - "mip-view-outside-allocation"}) { - std::string command = - std::string("\"") + path + "\" --metadata-descriptor-death " + kind; - std::vector mutable_command(command.begin(), command.end()); - mutable_command.push_back('\0'); - STARTUPINFOA startup{sizeof(startup)}; - PROCESS_INFORMATION process{}; - Require("MetadataReuseDescriptor", "host", - CreateProcessA(nullptr, mutable_command.data(), nullptr, nullptr, - FALSE, CREATE_NO_WINDOW, nullptr, nullptr, &startup, - &process) != 0, - "CreateProcess failed"); - Require("MetadataReuseDescriptor", "host", - WaitForSingleObject(process.hProcess, 10000) == WAIT_OBJECT_0, - "death case timed out"); - DWORD exit_code = 0; - const bool exited = GetExitCodeProcess(process.hProcess, &exit_code) != 0; - CloseHandle(process.hThread); - CloseHandle(process.hProcess); - Require("MetadataReuseDescriptor", "host", exited && exit_code == 321, - std::string(kind) + " descriptor was admitted"); - } - std::printf("[host] %-32s ok\n", "MetadataReuseDescriptors"); -} - -void CheckBufferImageWrites() { - constexpr uint64_t sampled = 0xe80e0000ull; - constexpr uint64_t sampled_size = 0x870000ull; - constexpr uint64_t video = 0x1000c10000ull; - constexpr uint64_t video_size = 0x1fe0000ull; - constexpr uint64_t target = 0x1158d80000ull; - constexpr uint64_t target_size = 0x870000ull; - constexpr uint64_t storage_address = 0x4a4290000ull; - constexpr uint64_t storage_size = 0x20000ull; - struct Case { - const char *name; - uint64_t buffer_address; - uint64_t buffer_size; - uint64_t image_address; - uint64_t image_size; - BufferImageBinding binding; - bool gpu_modified; - bool formatted; - BufferImageWrite expected; - }; - constexpr std::array cases{ - Case{"sampled exact", sampled, sampled_size, sampled, sampled_size, - BufferImageBinding::Texture, false, false, - BufferImageWrite::InvalidateTexture}, - Case{"sampled contained prefix", sampled, - sampled_size - TRACKER_PAGE_SIZE, sampled, sampled_size, - BufferImageBinding::Texture, false, false, - BufferImageWrite::InvalidateTexture}, - Case{"sampled contained raw", sampled + 0x23030, 0xfa0, sampled, - sampled_size, BufferImageBinding::Texture, false, false, - BufferImageWrite::InvalidateTexture}, - Case{"sampled prefix crossing", sampled - 0x10, 0x20, sampled, - sampled_size, BufferImageBinding::Texture, false, false, - BufferImageWrite::Unsupported}, - Case{"sampled suffix crossing", sampled + sampled_size - 0x10, 0x20, - sampled, sampled_size, BufferImageBinding::Texture, false, false, - BufferImageWrite::Unsupported}, - Case{"sampled contained GPU-owned", sampled + 0x23030, 0xfa0, sampled, - sampled_size, BufferImageBinding::Texture, true, false, - BufferImageWrite::Unsupported}, - Case{"sampled contained unsupported", sampled + 0x23030, 0xfa0, sampled, - sampled_size, BufferImageBinding::Unsupported, false, false, - BufferImageWrite::Unsupported}, - Case{"sampled unaligned", sampled + 1, sampled_size, sampled + 1, - sampled_size, BufferImageBinding::Texture, false, false, - BufferImageWrite::Unsupported}, - Case{"sampled GPU-owned", sampled, sampled_size, sampled, sampled_size, - BufferImageBinding::Texture, true, false, - BufferImageWrite::Unsupported}, - Case{"sampled disjoint", sampled, sampled_size, sampled + sampled_size, - sampled_size, BufferImageBinding::Texture, false, false, - BufferImageWrite::None}, - Case{"video exact", video, video_size, video, video_size, - BufferImageBinding::VideoOut, false, true, - BufferImageWrite::InvalidateVideoOut}, - Case{"video disjoint", video, video_size, video + video_size, video_size, - BufferImageBinding::VideoOut, false, true, BufferImageWrite::None}, - Case{"video partial", video, video_size - 4, video, video_size, - BufferImageBinding::VideoOut, false, true, - BufferImageWrite::Unsupported}, - Case{"video wrong binding", video, video_size, video, video_size, - BufferImageBinding::Unsupported, false, true, - BufferImageWrite::Unsupported}, - Case{"video GPU-owned", video, video_size, video, video_size, - BufferImageBinding::VideoOut, true, true, - BufferImageWrite::SynchronizeVideoOut}, - Case{"video unformatted", video, video_size, video, video_size, - BufferImageBinding::VideoOut, false, false, - BufferImageWrite::Unsupported}, - Case{"video GPU-owned unformatted", video, video_size, video, video_size, - BufferImageBinding::VideoOut, true, false, - BufferImageWrite::Unsupported}, - Case{"video GPU-owned partial", video, video_size - 4, video, video_size, - BufferImageBinding::VideoOut, true, true, - BufferImageWrite::Unsupported}, - Case{"video unaligned", video + 0x100, video_size, video + 0x100, - video_size, BufferImageBinding::VideoOut, true, true, - BufferImageWrite::Unsupported}, - Case{"target exact", target, target_size, target, target_size, - BufferImageBinding::RenderTarget, true, true, - BufferImageWrite::SynchronizeRenderTarget}, - Case{"target partial", target, target_size - 4, target, target_size, - BufferImageBinding::RenderTarget, true, true, - BufferImageWrite::Unsupported}, - Case{"target CPU-owned", target, target_size, target, target_size, - BufferImageBinding::RenderTarget, false, true, - BufferImageWrite::Unsupported}, - Case{"target unformatted", target, target_size, target, target_size, - BufferImageBinding::RenderTarget, true, false, - BufferImageWrite::Unsupported}, - Case{"target unaligned", target + 0x100, target_size, target + 0x100, - target_size, BufferImageBinding::RenderTarget, true, true, - BufferImageWrite::Unsupported}, - Case{"storage mip page", storage_address + 0x10000, 0x10000, - storage_address, storage_size, BufferImageBinding::StorageTexture, - true, true, BufferImageWrite::SynchronizeStorageTexture}, - Case{"storage unformatted", storage_address + 0x10000, 0x10000, - storage_address, storage_size, BufferImageBinding::StorageTexture, - true, false, BufferImageWrite::Unsupported}, - Case{"storage unaligned", storage_address + 0x100, 0x10000, - storage_address, storage_size, BufferImageBinding::StorageTexture, - true, true, BufferImageWrite::Unsupported}, - Case{"depth exact", 0x4a5c80000ull, 0x10000, 0x4a5c80000ull, 0x10000, - BufferImageBinding::DepthTarget, true, true, - BufferImageWrite::SynchronizeDepthTarget}, - Case{"depth partial", 0x4a5c80000ull, 0x8000, 0x4a5c80000ull, 0x10000, - BufferImageBinding::DepthTarget, true, true, - BufferImageWrite::Unsupported}, - Case{"depth unformatted", 0x4a5c80000ull, 0x10000, 0x4a5c80000ull, - 0x10000, BufferImageBinding::DepthTarget, true, false, - BufferImageWrite::Unsupported}, - }; - for (const auto &test : cases) { - const auto actual = ClassifyBufferImageWrite( - test.buffer_address, test.buffer_size, test.image_address, - test.image_size, test.binding, test.gpu_modified, test.formatted); - Require("BufferImageWrite", test.name, actual == test.expected, - "buffer/image write classification mismatch"); - } - Require("BufferImageWrite", "storage containing overwrite", - ClassifyBufferImageWrite( - storage_address - 0x80000, 0x100000, storage_address, - storage_size, BufferImageBinding::StorageTexture, false, true, - true) == BufferImageWrite::InvalidateStorageTexture, - "formatted containing overwrite did not retain refreshable storage " - "ownership"); - Require("BufferImageWrite", "storage containing overwrite GPU-owned", - ClassifyBufferImageWrite( - storage_address - 0x80000, 0x100000, storage_address, - storage_size, BufferImageBinding::StorageTexture, true, true, - false) == BufferImageWrite::Unsupported, - "containing overwrite bypassed GPU-owned storage synchronization"); - Require("BufferImageWrite", "depth repeated buffer overwrite", - ClassifyBufferImageWrite(0x4a5cd0000ull, 0x10000, 0x4a5cd0000ull, - 0x10000, BufferImageBinding::DepthTarget, - false, true, true) == - BufferImageWrite::InvalidateDepthTarget, - "buffer-owned exact depth overwrite was rejected"); - - // PPSA01530's packed mip0 page is first synchronized out of a GPU-owned - // storage image, written through a persistent formatted buffer, and then - // rebound as the same storage image. - const auto partial_refresh = - ClassifyStorageBufferRebind(true, false, true, false, false, true); - const auto refreshed_reuse = - ClassifyStorageBufferRebind(true, true, false, true, false, true); - Require("BufferImageWrite", "storage partial lifecycle", - partial_refresh == StorageBufferRebind::RefreshFromBacking && - refreshed_reuse == StorageBufferRebind::Reuse, - "storage-to-partial-buffer-to-storage lifecycle was rejected"); - Require("BufferImageWrite", "storage partial incoherent", - ClassifyStorageBufferRebind(true, false, true, false, false, false) == - StorageBufferRebind::Unsupported, - "incoherent partial storage backing was admitted"); - Require("BufferImageWrite", "storage contradictory ownership", - ClassifyStorageBufferRebind(true, false, true, true, false, true) == - StorageBufferRebind::Unsupported, - "contradictory storage tracker ownership was admitted"); - Require( - "BufferImageWrite", "storage mip-one cache miss", - SelectImageBackingBaseLevel(true, 1) == 0 && - SelectImageBackingBaseLevel(false, 1) == 1, - "storage backing creation remained dependent on first-bound view mip"); - Require("BufferImageWrite", "dynamic one-layer array view", - !NeedsStaticSampledArrayView(true, true) && - NeedsStaticSampledArrayView(true, false) && - !NeedsStaticSampledArrayView(false, false), - "dynamic sampled array view was replaced by a missing static view"); - - DepthTargetInfo old_depth{}; - old_depth.address = 0x4a5c90000ull; - old_depth.size = 0x10000; - old_depth.width = old_depth.height = 32; - old_depth.pitch = 32; - old_depth.bytes_per_element = 4; - old_depth.layers = 1; - DepthTargetInfo cleared_depth = old_depth; - cleared_depth.width = cleared_depth.height = 64; - cleared_depth.depth_load_clear = true; - Require( - "BufferImageWrite", "cleared depth reinterpretation", - ClassifyDepthTargetOverlap(old_depth, true, false, cleared_depth) == - DepthOverlap::DiscardTarget, - "exact cleared depth allocation could not discard its old native shape"); - cleared_depth.depth_load_clear = false; - Require("BufferImageWrite", "loaded depth reinterpretation", - ClassifyDepthTargetOverlap(old_depth, true, false, cleared_depth) == - DepthOverlap::Unsupported, - "loaded depth reinterpretation discarded live native contents"); - RenderTargetInfo old_color{}; - old_color.address = 0x4a5cb0000ull; - old_color.size = 0x10000; - old_color.width = old_color.height = old_color.pitch = 128; - old_color.bytes_per_element = 4; - old_color.format = vk::Format::eR8G8B8A8Unorm; - auto reinterpreted_color = old_color; - reinterpreted_color.format = vk::Format::eR32Uint; - Require("BufferImageWrite", "clean color format recreation", - ClassifyRenderTargetOverlap(old_color, false, false, false, true, - reinterpreted_color) == - RenderTargetOverlap::RetireTarget && - ClassifyRenderTargetOverlap(old_color, true, false, true, true, - reinterpreted_color) == - RenderTargetOverlap::Unsupported, - "clean equal-base color allocation did not preserve GPU-owned format " - "failures"); - auto pool_depth = old_depth; - pool_depth.stencil_address = old_depth.address + 0x20000; - pool_depth.stencil_size = old_depth.size; - auto shifted_depth = pool_depth; - shifted_depth.address = old_depth.address - 0x20000; - shifted_depth.stencil_address = old_depth.address; - Require( - "BufferImageWrite", "shifted depth-plane recreation", - ClassifyDepthTargetOverlap(pool_depth, true, false, shifted_depth) == - DepthOverlap::RecreateTarget && - ClassifyDepthTargetOverlap(pool_depth, true, true, shifted_depth) == - DepthOverlap::Unsupported, - "exact stencil/depth pool reuse did not require clean single-sample " - "recreation"); - shifted_depth.samples = 8; - Require( - "BufferImageWrite", "multisampled depth-plane recreation", - ClassifyDepthTargetOverlap(pool_depth, true, false, shifted_depth) == - DepthOverlap::Unsupported, - "multisampled depth-plane reuse bypassed the unsupported readback guard"); - RenderTargetInfo pool_color{}; - pool_color.address = shifted_depth.stencil_address; - pool_color.size = shifted_depth.stencil_size; - shifted_depth.samples = 1; - Require( - "BufferImageWrite", "color/depth pool recreation", - CanRecreateRenderTargetForDepth(pool_color, false, false, false, true, - shifted_depth) && - !CanRecreateRenderTargetForDepth(pool_color, false, true, false, true, - shifted_depth), - "exact color-plane reuse did not require clean single-sample recreation"); - pool_color.size *= 2; - Require( - "BufferImageWrite", "larger color/depth pool recreation", - CanRecreateRenderTargetForDepth(pool_color, false, false, false, true, - shifted_depth), - "equal-base page-isolated color allocation was not recreated for depth"); - auto contained_depth = shifted_depth; - contained_depth.address = pool_color.address + TRACKER_PAGE_SIZE; - contained_depth.size = TRACKER_PAGE_SIZE; - contained_depth.stencil_address = 0; - contained_depth.stencil_size = 0; - Require( - "BufferImageWrite", "contained color/depth pool recreation", - CanRecreateRenderTargetForDepth(pool_color, false, false, false, true, - contained_depth) && - !CanRecreateRenderTargetForDepth(pool_color, false, false, false, - false, contained_depth) && - !CanRecreateRenderTargetForDepth(pool_color, true, false, false, true, - contained_depth), - "contained depth allocation bypassed source or tracker ownership guards"); - pool_color.samples = 8; - Require( - "BufferImageWrite", "multisampled color/depth recreation", - !CanRecreateRenderTargetForDepth(pool_color, false, false, false, true, - shifted_depth), - "multisampled color-plane reuse bypassed the unsupported readback guard"); - auto multisample_depth = old_depth; - multisample_depth.samples = 8; - multisample_depth.depth_access = true; - multisample_depth.stencil_access = true; - Require("BufferImageWrite", "multisampled depth refresh state", - !RequiresMultisampleDepthRefresh(multisample_depth, false, false, - false) && - RequiresMultisampleDepthRefresh(multisample_depth, false, true, - false) && - RequiresMultisampleDepthRefresh(multisample_depth, false, false, - true) && - RequiresMultisampleDepthRefresh(multisample_depth, true, false, - false), - "multisampled refresh used consumed source history instead of " - "current plane ownership"); - RenderTargetInfo color_alias{}; - color_alias.address = old_depth.address; - color_alias.size = old_depth.size; - color_alias.width = color_alias.height = 32; - color_alias.pitch = 32; - color_alias.bytes_per_element = 4; - Require("BufferImageWrite", "guest-backed depth to color", - CanRecreateDepthForRenderTarget(old_depth, false, false, false, true, - color_alias) && - CanRecreateDepthForRenderTarget(old_depth, true, false, true, - false, color_alias) && - !CanRecreateDepthForRenderTarget(old_depth, true, true, true, - true, color_alias), - "depth-to-color recreation did not require coherent guest ownership"); - Require("BufferImageWrite", "guest-current depth to color", - !CanRecreateDepthForRenderTarget(old_depth, false, false, false, - false, color_alias), - "stale guest backing authorized depth-to-color recreation"); - auto mismatched_color = color_alias; - mismatched_color.address += 0x1000; - Require("BufferImageWrite", "mismatched depth to color", - !CanRecreateDepthForRenderTarget(old_depth, false, false, false, true, - mismatched_color), - "mismatched depth shape was discarded as a color allocation"); - color_alias.address = pool_depth.stencil_address; - Require( - "BufferImageWrite", "stencil-plane depth to color", - CanRecreateDepthForRenderTarget(pool_depth, false, false, false, true, - color_alias), - "exact stencil-plane reuse was not treated as whole-image recreation"); - auto containing_color = color_alias; - containing_color.address = pool_depth.stencil_address - 0x40000; - containing_color.size = 0x80000; - Require("BufferImageWrite", "contained depth allocation to color", - CanRecreateDepthForRenderTarget(pool_depth, false, false, false, true, - containing_color) && - !CanRecreateDepthForRenderTarget(pool_depth, false, false, false, - false, containing_color) && - !CanRecreateDepthForRenderTarget(pool_depth, false, false, true, - true, containing_color), - "contained guest-current depth allocation bypassed ownership guards"); - - TileSizeAlign storage{}; - Require("BufferImageWrite", "target layout", - TileGetRenderTargetPitch(1920, 4) == 1920 && - TileGetRenderTargetSize(1920, 1080, 1920, 4, storage) && - storage.align == 0x10000 && storage.size == target_size, - "render-target write fixture has an invalid tiled layout"); - Require("BufferImageWrite", "PPSA02721 video layout", - TileGetRenderTargetPitch(3840, 4) == 3840 && - TileGetRenderTargetSize(3840, 2160, 3840, 4, storage) && - storage.align == 0x10000 && storage.size == video_size, - "4K video-out ownership fixture lost its exact tiled layout"); - std::printf("[host] %-32s ok\n", "BufferImageWrite"); -} - -void CheckImageOverlapResolution() { - CheckBufferImageWrites(); - VideoOutPixelFormatInfo video_format{}; - Require( - "ImageOverlapResolution", "existing video-out formats", - DecodeVideoOutPixelFormat(0x8000000022000000ull, video_format) && - video_format.format == vk::Format::eR8G8B8A8Srgb && - video_format.guest_format == - Prospero::GpuEnumValue(Prospero::BufferFormat::k8_8_8_8Srgb) && - DecodeVideoOutPixelFormat(0x8000000000000000ull, video_format) && - video_format.format == vk::Format::eB8G8R8A8Srgb && - video_format.guest_format == - Prospero::GpuEnumValue(Prospero::BufferFormat::k8_8_8_8Srgb) && - DecodeVideoOutPixelFormat(0x8100000022000000ull, video_format) && - video_format.format == vk::Format::eA2B10G10R10UnormPack32 && - video_format.guest_format == - Prospero::GpuEnumValue(Prospero::BufferFormat::k10_10_10_2UNorm), - "pre-existing PS5 video-out format mappings changed during decoder " - "centralization"); - Require( - "ImageOverlapResolution", "B10G10R10A2 video-out format", - DecodeVideoOutPixelFormat(0x8100000000000000ull, video_format) && - video_format.format == vk::Format::eA2R10G10B10UnormPack32 && - video_format.guest_format == - Prospero::GpuEnumValue(Prospero::BufferFormat::k10_10_10_2UNorm), - "PS5 B10G10R10A2 sRGB video-out format did not preserve alternate " - "channel order"); - Require( - "ImageOverlapResolution", "RGBA16 float video-out formats", - DecodeVideoOutPixelFormat(0xc001000622000000ull, video_format) && - video_format.format == vk::Format::eR16G16B16A16Sfloat && - video_format.guest_format == - Prospero::GpuEnumValue( - Prospero::BufferFormat::k16_16_16_16Float) && - video_format.bytes_per_element == 8 && - DecodeVideoOutPixelFormat(0xc001000600000000ull, video_format) && - video_format.bgra16, - "PS5 RGBA16-float video-out formats lost their 64-bpp storage contract"); - VideoOutInfo supported_video_out{}; - supported_video_out.format = video_format.format; - supported_video_out.guest_format = video_format.guest_format; - supported_video_out.bytes_per_element = video_format.bytes_per_element; - supported_video_out.bgra16 = video_format.bgra16; - Require( - "ImageOverlapResolution", "video-out format policy", - IsSupportedVideoOutFormat(supported_video_out), - "decoded PS5 video-out format was rejected by the centralized policy"); - std::array bgra16_pixels{1, 2, 3, 4, 5, 6, 7, 8}; - ImageOps::SwapVideoOutBgra16(bgra16_pixels.data(), sizeof(bgra16_pixels)); - Require("ImageOverlapResolution", "BGRA16 channel conversion", - bgra16_pixels == std::array{3, 2, 1, 4, 7, 6, 5, 8}, - "BGRA16 video-out conversion did not swap red and blue"); - ImageOps::SwapVideoOutBgra16(bgra16_pixels.data(), sizeof(bgra16_pixels)); - Require("ImageOverlapResolution", "BGRA16 channel round trip", - bgra16_pixels == std::array{1, 2, 3, 4, 5, 6, 7, 8}, - "BGRA16 video-out conversion was not reversible for readback"); - Require( - "ImageOverlapResolution", "video-out byte-size guards", ([&] { - auto mismatched = supported_video_out; - mismatched.bytes_per_element = 4; - return !IsSupportedVideoOutFormat(mismatched); - })(), - "video-out format validation admitted a mismatched storage element size"); - Require( - "ImageOverlapResolution", "video-out format guards", - !DecodeVideoOutPixelFormat(0x8100070400000000ull, video_format) && - !DecodeVideoOutPixelFormat(0xc001070700000000ull, video_format) && !([&] { - auto mismatched = supported_video_out; - mismatched.guest_format = - Prospero::GpuEnumValue(Prospero::BufferFormat::k8_8_8_8Srgb); - return IsSupportedVideoOutFormat(mismatched); - })(), - "unsupported PS5 video-out formats or mismatched guest/host pairs were " - "admitted"); - TileSizeAlign render_target_layout{}; - Require("ImageOverlapResolution", "R8 render-target pitch", - TileGetRenderTargetPitch(1920, 1) == 2048, - "R8 target did not use native 256-pixel block width"); - Require("ImageOverlapResolution", "R8 render-target size", - TileGetRenderTargetSize(1920, 1080, 2048, 1, render_target_layout) && - render_target_layout.size == 0x280000 && - render_target_layout.align == 0x10000, - "R8 target did not use native 64-KiB block geometry"); - Require( - "ImageOverlapResolution", "RGBA8 render-target size", - TileGetRenderTargetPitch(1920, 4) == 1920 && - TileGetRenderTargetSize(1920, 1080, 1920, 4, render_target_layout) && - render_target_layout.size == 0x870000, - "RGBA8 target layout regressed"); - Require("ImageOverlapResolution", "PPSA19268 render-target edge", - TileGetRenderTargetPitch(120, 8) == 128 && - TileGetRenderTargetSize(120, 67, 128, 8, render_target_layout) && - render_target_layout.size == 0x20000 && - ImageRangeOverlaps(0x11029fff8ull, 1, 0x110280000ull, - render_target_layout.size), - "title write did not land in the final native render-target block"); - Require( - "ImageOverlapResolution", "render-target layout rejection", - TileGetRenderTargetPitch(1920, 3) == 0 && - !TileGetRenderTargetSize(1920, 1080, 1920, 1, render_target_layout), - "unsupported element size or mismatched pitch was admitted"); - - TileSizeOffset mip_levels[16]{}; - TilePaddedSize mip_padded[16]{}; - Require( - "ImageOverlapResolution", "PPSA09076 mip-chain layout", - TileGetRenderTargetMipLayout(960, 540, 1024, 8, 10, render_target_layout, - mip_levels, mip_padded) && - render_target_layout.size == 0x650000 && - render_target_layout.align == 0x10000 && - mip_levels[0].offset == 0x1d0000 && mip_levels[0].size == 0x480000 && - mip_levels[1].offset == 0x90000 && mip_levels[1].size == 0x140000 && - mip_padded[1].width == 512 && mip_padded[1].height == 320, - "native render-target mip allocation or subresource geometry regressed"); - Require( - "ImageOverlapResolution", "render-target mip-chain rejection", - !TileGetRenderTargetMipLayout(960, 540, 960, 8, 10, render_target_layout, - nullptr, nullptr) && - !TileGetRenderTargetMipLayout( - 960, 540, 1024, 3, 10, render_target_layout, nullptr, nullptr) && - !TileGetRenderTargetMipLayout( - 960, 540, 1024, 8, 0, render_target_layout, nullptr, nullptr) && - !TileGetRenderTargetMipLayout( - 960, 540, 1024, 8, 17, render_target_layout, nullptr, nullptr) && - !TileGetRenderTargetMipLayout(4, 4, TileGetRenderTargetPitch(4, 4), 4, - 4, render_target_layout, nullptr, - nullptr), - "invalid render-target mip topology was admitted"); - - ImageInfo sampled{}; - sampled.address = 0x200000; - sampled.size = 0x8000; - ImageInfo sampled_alias = sampled; - sampled_alias.address += 0x1000; - Require("ImageOverlapResolution", "sampled alias", - ClassifySampledOverlap(sampled_alias, sampled, false) == - SampledOverlap::ReadOnlyAlias, - "read-only sampled pool alias was rejected"); - Require("ImageOverlapResolution", "sampled GPU owner", - ClassifySampledOverlap(sampled_alias, sampled, true) == - SampledOverlap::Unsupported, - "GPU-modified sampled alias was admitted"); - ImageInfo dirty_info{}; - dirty_info.address = 0x500100; - dirty_info.size = 0x100; - Image dirty_image; - dirty_image = dirty_info; - dirty_image.InvalidateCpuWrite(0x500300, 0x10); - Require( - "ImageOverlapResolution", "sampled edge-page maybe dirty", - dirty_image.IsCpuDirty() && dirty_image.IsMaybeCpuDirty() && - !dirty_image.IsDefinitelyCpuDirty(), - "byte-disjoint write sharing a tracker page was not kept maybe-dirty"); - Require("ImageOverlapResolution", "sampled edge hash required", - dirty_image.NeedsMaybeCpuHash(), - "collapsed edge tracking did not request a baseline hash"); - dirty_image.SetMaybeCpuHash(0x1234); - Require("ImageOverlapResolution", "sampled unchanged edge hash", - !dirty_image.ResolveMaybeCpuHash(0x1234) && - !dirty_image.IsCpuDirty() && dirty_image.IsCpuTrackingComplete(), - "unchanged edge bytes forced an image upload"); - dirty_image.InvalidateCpuWrite(0x500300, 0x10); - dirty_image.SetMaybeCpuHash(0x1234); - Require("ImageOverlapResolution", "sampled changed edge hash", - dirty_image.ResolveMaybeCpuHash(0x5678) && - dirty_image.IsDefinitelyCpuDirty(), - "changed edge bytes were not promoted to definitely dirty"); - dirty_image.RefreshComplete(); - dirty_image.InvalidateCpuWrite(dirty_info.address, 1); - Require("ImageOverlapResolution", "sampled byte overlap dirty", - dirty_image.IsCpuDirty() && dirty_image.IsDefinitelyCpuDirty() && - !dirty_image.IsMaybeCpuDirty(), - "true byte overlap was not promoted to definitely dirty"); - ImageInfo two_page_info{}; - two_page_info.address = 0x600100; - two_page_info.size = 0x1800; - Image two_page_image; - two_page_image = two_page_info; - two_page_image.InvalidateCpuWrite(0x600000, 1); - Require("ImageOverlapResolution", "sampled head tracking shrink", - !two_page_image.IsCpuDirty(), - "a single disjoint edge fault prematurely dirtied a two-page image"); - two_page_image.InvalidateCpuWrite(0x601a00, 1); - Require("ImageOverlapResolution", "sampled head-tail collapse", - two_page_image.IsMaybeCpuDirty() && - two_page_image.NeedsMaybeCpuHash(), - "both disjoint edge faults did not collapse logical tracking"); - constexpr uint64_t storage_subrange = 0x649b0100; - constexpr uint64_t storage_subrange_size = 0x800; - constexpr uint64_t sampled_backing = 0x649a3000; - constexpr uint64_t sampled_backing_size = 0x24000; - Require("ImageOverlapResolution", "storage retires clean sampled backing", - ClassifyStorageImageOverlap(storage_subrange, storage_subrange_size, - sampled_backing, sampled_backing_size, - true, false, false, false) == - StorageImageOverlap::RetireSampled, - "clean sampled subrange was not retired before storage creation"); - Require("ImageOverlapResolution", "storage preserves dirty sampled backing", - ClassifyStorageImageOverlap(storage_subrange, storage_subrange_size, - sampled_backing, sampled_backing_size, - true, true, false, - true) == StorageImageOverlap::Unsupported, - "GPU-owned sampled subrange was admitted as storage backing"); - ImageInfo page_left = sampled; - page_left.size = TRACKER_PAGE_SIZE / 2; - ImageInfo same_page = page_left; - same_page.address += page_left.size; - Require( - "ImageOverlapResolution", "sampled shared page", - !ImageRangeOverlaps(same_page.address, same_page.size, page_left.address, - page_left.size) && - ClassifySampledOverlap(same_page, page_left, false) == - SampledOverlap::ReadOnlyAlias, - "byte-disjoint sampled images sharing a tracker page were not aliased"); - ImageInfo separate_page = same_page; - separate_page.address = page_left.address + TRACKER_PAGE_SIZE; - Require("ImageOverlapResolution", "sampled separate page", - ClassifySampledOverlap(separate_page, page_left, false) == - SampledOverlap::None, - "sampled images on separate pages were aliased"); - constexpr uint64_t edge_target_address = 0x108ad00100ull; - constexpr uint64_t edge_target_size = 0x1fa400; - constexpr uint64_t edge_fault_address = - edge_target_address + edge_target_size + 0x1b8; - Require( - "ImageOverlapResolution", "GPU target shared edge page", - !ImageRangeOverlaps(edge_fault_address, 1, edge_target_address, - edge_target_size) && - ImagePageRangesOverlap(edge_fault_address, 1, edge_target_address, - edge_target_size), - "byte-disjoint CPU access lost the GPU owner of its shared tracker page"); - Image sampled_image{}; - sampled_image = page_left; - Image same_page_image{}; - same_page_image = same_page; - sampled_image.InvalidateCpuWrite(page_left.address + 1, 1); - same_page_image.InvalidateCpuWrite(page_left.address + 1, 1); - Require("ImageOverlapResolution", "sampled dirty fanout", - sampled_image.IsCpuDirty() && same_page_image.IsCpuDirty(), - "one guest write did not dirty every sampled image sharing its page"); - sampled_image.RefreshComplete(); - Require("ImageOverlapResolution", "sampled dirty independence", - !sampled_image.IsCpuDirty() && same_page_image.IsCpuDirty(), - "refreshing one sampled alias cleaned another alias"); - - constexpr uint64_t fill_address = 0x73771080; - constexpr uint64_t fill_size = sizeof(uint32_t); - constexpr uint64_t texture_address = 0x73771000; - constexpr uint64_t texture_size = 0x4000; - Require("ImageOverlapResolution", "host fill sampled image", - ClassifyHostWriteOverlap(fill_address, fill_size, texture_address, - texture_size, true, false, - false) == HostWriteOverlap::InvalidateImage, - "host-backed fill did not invalidate its CPU-current sampled image"); - Require("ImageOverlapResolution", "host fill GPU owner", - ClassifyHostWriteOverlap(fill_address, fill_size, texture_address, - texture_size, true, true, - false) == HostWriteOverlap::Unsupported, - "host-backed fill was admitted over a GPU-owned sampled image"); - Require("ImageOverlapResolution", "host fill clean render target", - ClassifyHostWriteOverlap(fill_address, fill_size, texture_address, - texture_size, true, false, - false) == HostWriteOverlap::InvalidateImage, - "host-backed fill did not invalidate its clean render target"); - Require("ImageOverlapResolution", "host fill non-refreshable image", - ClassifyHostWriteOverlap(fill_address, fill_size, texture_address, - texture_size, false, false, - false) == HostWriteOverlap::Unsupported, - "host-backed fill was admitted over a non-refreshable image"); - constexpr uint64_t dma_address = 0x10f562000ull; - constexpr uint64_t dma_size = 0x40000; - constexpr uint64_t target_address = 0x10f570000ull; - constexpr uint64_t target_size = 0x20000; - Require( - "ImageOverlapResolution", "host copy clean render target", - ClassifyHostWriteOverlap(dma_address, dma_size, target_address, - target_size, true, false, - false) == HostWriteOverlap::InvalidateImage, - "host-backed copy did not invalidate the covered clean render target"); - Require("ImageOverlapResolution", "host copy GPU render target", - ClassifyHostWriteOverlap(dma_address, dma_size, target_address, - target_size, true, true, - false) == HostWriteOverlap::Unsupported, - "host-backed copy was admitted over a GPU-owned render target"); - Require("ImageOverlapResolution", "host copy partial clean render target", - ClassifyHostWriteOverlap(target_address + 0x1000, 0x1000, - target_address, target_size, true, false, - false) == HostWriteOverlap::InvalidateImage, - "partial host copy did not invalidate its clean render target"); - Require("ImageOverlapResolution", "host copy shared target page", - ClassifyHostWriteOverlap(target_address + target_size - 1, 1, - target_address, target_size - 1, true, false, - false) == HostWriteOverlap::InvalidateImage, - "shared-page host copy did not conservatively invalidate the clean " - "target"); - Require("ImageOverlapResolution", "host fill metadata", - ClassifyHostWriteOverlap(fill_address, fill_size, texture_address, - texture_size, true, false, - true) == HostWriteOverlap::Unsupported, - "host-backed sampled-image fill was admitted over metadata pages"); - Require("ImageOverlapResolution", "host fill disjoint page", - ClassifyHostWriteOverlap(texture_address + texture_size, fill_size, - texture_address, texture_size, true, false, - true) == HostWriteOverlap::None, - "host-backed fill aliased a sampled image on another page"); - - constexpr uint64_t video_metadata = 0x1020c10000ull; - Require("ImageOverlapResolution", "video-out DCC 256/256/0", - ClassifyVideoOutCompression(true, video_metadata, 0x48u, 0) == - VideoOutCompression::Dcc256_256_0, - "Prospero video-out DCC 256/256/0 mode was rejected"); - Require("ImageOverlapResolution", "video-out DCC 256/64/64", - ClassifyVideoOutCompression(true, video_metadata, 0x208u, 0) == - VideoOutCompression::Dcc256_64_64, - "Prospero video-out DCC 256/64/64 mode was rejected"); - Require( - "ImageOverlapResolution", "video-out compression guards", - ClassifyVideoOutCompression(false, 0, 0, 0) == - VideoOutCompression::Uncompressed && - ClassifyVideoOutCompression(true, 0, 0x208u, 0) == - VideoOutCompression::Unsupported && - ClassifyVideoOutCompression(true, video_metadata + 1, 0x208u, 0) == - VideoOutCompression::Unsupported && - ClassifyVideoOutCompression(true, video_metadata, 0x204u, 0) == - VideoOutCompression::Unsupported && - ClassifyVideoOutCompression(true, video_metadata, 0x208u, 1) == - VideoOutCompression::Unsupported && - ClassifyVideoOutCompression(false, video_metadata, 0, 0) == - VideoOutCompression::Unsupported, - "unsupported video-out compression state was admitted"); - Require( - "ImageOverlapResolution", "compressed video-out native ownership", - CanUseVideoOutNativeWithoutUpload(VideoOutCompression::Dcc256_64_64, true, - false, false) && - CanUseVideoOutNativeWithoutUpload(VideoOutCompression::Dcc256_64_64, - false, true, false) && - !CanUseVideoOutNativeWithoutUpload(VideoOutCompression::Dcc256_64_64, - false, false, false) && - CanUseVideoOutNativeWithoutUpload(VideoOutCompression::Dcc256_256_0, - true, false, false) && - !CanUseVideoOutNativeWithoutUpload(VideoOutCompression::Dcc256_256_0, - false, false, true) && - !CanUseVideoOutNativeWithoutUpload(VideoOutCompression::Dcc256_64_64, - true, false, true) && - !CanUseVideoOutNativeWithoutUpload(VideoOutCompression::Uncompressed, - true, true, false), - "compressed video-out upload/read ownership boundary regressed"); - - vk::ClearColorValue clear{}; - Require( - "ImageOverlapResolution", "RGBA8 compute clear decode", - DecodePackedColorClear(vk::Format::eR8G8B8A8Unorm, 0x80402010u, clear) && - std::abs(clear.float32[0] - 16.0f / 255.0f) < 0.000001f && - std::abs(clear.float32[1] - 32.0f / 255.0f) < 0.000001f && - std::abs(clear.float32[2] - 64.0f / 255.0f) < 0.000001f && - std::abs(clear.float32[3] - 128.0f / 255.0f) < 0.000001f, - "packed RGBA8 clear did not preserve component values"); - Require( - "ImageOverlapResolution", "BGRA8 compute clear decode", - DecodePackedColorClear(vk::Format::eB8G8R8A8Unorm, 0x80402010u, clear) && - std::abs(clear.float32[0] - 64.0f / 255.0f) < 0.000001f && - std::abs(clear.float32[1] - 32.0f / 255.0f) < 0.000001f && - std::abs(clear.float32[2] - 16.0f / 255.0f) < 0.000001f, - "packed BGRA8 clear did not account for memory channel order"); - const uint32_t rgb10_clear = - 1023u | (512u << 10u) | (1u << 20u) | (3u << 30u); - Require("ImageOverlapResolution", "RGB10 compute clear decode", - DecodePackedColorClear(vk::Format::eA2B10G10R10UnormPack32, - rgb10_clear, clear) && - clear.float32[0] == 1.0f && - std::abs(clear.float32[1] - 512.0f / 1023.0f) < 0.000001f && - std::abs(clear.float32[2] - 1.0f / 1023.0f) < 0.000001f && - clear.float32[3] == 1.0f, - "packed RGB10 clear did not preserve component values"); - Require( - "ImageOverlapResolution", "sRGB compute clear decode", - DecodePackedColorClear(vk::Format::eR8G8B8A8Srgb, 0xff808080u, clear) && - std::abs(clear.float32[0] - 0.215861f) < 0.00001f && - std::abs(clear.float32[1] - 0.215861f) < 0.00001f && - std::abs(clear.float32[2] - 0.215861f) < 0.00001f && - clear.float32[3] == 1.0f && - !DecodePackedColorClear(vk::Format::eR16G16Unorm, 0, clear), - "sRGB clear decode or unsupported-format guard regressed"); - uint8_t stencil_clear = 0; - Require("ImageOverlapResolution", "stencil compute clear decode", - DecodePackedStencilClear(0x7f7f7f7fu, stencil_clear) && - stencil_clear == 0x7fu && - DecodePackedStencilClear(0, stencil_clear) && - stencil_clear == 0 && - !DecodePackedStencilClear(0x0000007fu, stencil_clear), - "packed stencil clear admitted non-uniform byte values"); - float depth_clear = -1.0f; - Require( - "ImageOverlapResolution", "D32 compute clear decode", - DecodePackedDepthClear(vk::Format::eD32Sfloat, 0u, depth_clear) && - depth_clear == 0.0f && - DecodePackedDepthClear(vk::Format::eD32SfloatS8Uint, 0x3f800000u, - depth_clear) && - depth_clear == 1.0f && - !DecodePackedDepthClear(vk::Format::eD16Unorm, 0u, depth_clear) && - !DecodePackedDepthClear(vk::Format::eD32Sfloat, 0xbf800000u, - depth_clear) && - !DecodePackedDepthClear(vk::Format::eD32Sfloat, 0x40000000u, - depth_clear) && - !DecodePackedDepthClear(vk::Format::eD32Sfloat, 0x7f800000u, - depth_clear), - "D32 compute clear decode admitted an unsupported format or value"); - DepthTargetInfo native_clear_target{}; - native_clear_target.address = 0x1158d80000ull; - native_clear_target.size = 0x870000ull; - native_clear_target.format = vk::Format::eD32Sfloat; - native_clear_target.guest_format = - Prospero::GpuEnumValue(Prospero::BufferFormat::k32Float); - native_clear_target.bytes_per_element = 4; - native_clear_target.tile_mode = - Prospero::GpuEnumValue(Prospero::TileMode::kDepth); - Require("ImageOverlapResolution", "native D32 buffer clear target", - CanNativeClearDepthFromBuffer(native_clear_target, - native_clear_target.address, - native_clear_target.size), - "exact uncompressed PS5 D32 target was rejected"); - auto compressed_depth = native_clear_target; - compressed_depth.htile_address = 0x1161480000ull; - compressed_depth.htile_size = 0x10000ull; - auto d16_depth = native_clear_target; - d16_depth.format = vk::Format::eD16Unorm; - d16_depth.guest_format = - Prospero::GpuEnumValue(Prospero::BufferFormat::k16UNorm); - d16_depth.bytes_per_element = 2; - Require("ImageOverlapResolution", "native depth buffer clear guards", - !CanNativeClearDepthFromBuffer(native_clear_target, - native_clear_target.address, - native_clear_target.size - 4) && - !CanNativeClearDepthFromBuffer(native_clear_target, - native_clear_target.address + 4, - native_clear_target.size) && - !CanNativeClearDepthFromBuffer(compressed_depth, - compressed_depth.address, - compressed_depth.size) && - !CanNativeClearDepthFromBuffer(d16_depth, d16_depth.address, - d16_depth.size), - "partial, offset, HTile-backed, or D16 depth clear was admitted"); - - RenderTargetInfo target{}; - target.address = sampled.address; - target.size = sampled.size; - Require( - "ImageOverlapResolution", "render target reuse", - ClassifyRenderTargetOverlap(sampled, false, target) == - RenderTargetOverlap::RetireSampled, - "exact CPU-owned sampled allocation was not retired for a render target"); - Require("ImageOverlapResolution", "render target GPU owner", - ClassifyRenderTargetOverlap(sampled, true, target) == - RenderTargetOverlap::Unsupported, - "GPU-owned sampled allocation was retired for a render target"); - ImageInfo partial_target_sample{}; - partial_target_sample.address = 0x79b01000; - partial_target_sample.size = 0x20000; - RenderTargetInfo layered_target{}; - layered_target.address = 0x79b10000; - layered_target.size = 0x60000; - Require("ImageOverlapResolution", "sampled partial target transition", - ClassifySampledRenderTargetOverlap(partial_target_sample, - layered_target, false) == - RenderTargetOverlap::RetireTarget, - "sampled partial target alias was not routed through readback"); - Require("ImageOverlapResolution", "sampled partial target rejection", - ClassifySampledRenderTargetOverlap(partial_target_sample, - layered_target, true) == - RenderTargetOverlap::Unsupported, - "buffer-owned target alias was admitted"); - - ImageInfo storage{}; - storage.address = 0x112cd0000ull; - storage.size = 0xff0000; - storage.format = - Prospero::GpuEnumValue(Prospero::BufferFormat::k16_16_16_16Float); - storage.width = 1920; - storage.height = 1080; - storage.pitch = 1920; - storage.tile = Prospero::GpuEnumValue(Prospero::TileMode::kRenderTarget); - storage.type = Prospero::GpuEnumValue(Prospero::ImageType::kColor2D); - RenderTargetInfo storage_target{}; - storage_target.address = storage.address; - storage_target.size = storage.size; - storage_target.format = vk::Format::eR16G16B16A16Sfloat; - storage_target.width = storage.width; - storage_target.height = storage.height; - storage_target.pitch = storage.pitch; - storage_target.bytes_per_element = 8; - storage_target.tile_mode = storage.tile; - Require("ImageOverlapResolution", "storage render-target native transition", - ClassifyStorageRenderTargetOverlap( - storage, storage_target.format, true, false, false, true, - storage_target) == RenderTargetOverlap::PreserveStorage, - "exact GPU-owned RGBA16F storage image was not preserved for a " - "render target"); - auto partial_storage_target = storage_target; - partial_storage_target.address += 0x10000; - partial_storage_target.size = 0x10000; - Require("ImageOverlapResolution", "clean storage render-target transition", - ClassifyStorageRenderTargetOverlap( - storage, storage_target.format, false, false, false, false, - partial_storage_target) == RenderTargetOverlap::RetireStorage, - "clean storage allocation was not retired for an overlapping render " - "target"); - auto mismatched_storage_target = storage_target; - mismatched_storage_target.width--; - Require("ImageOverlapResolution", "storage render-target guards", - ClassifyStorageRenderTargetOverlap( - storage, storage_target.format, true, true, false, true, - storage_target) == RenderTargetOverlap::Unsupported && - ClassifyStorageRenderTargetOverlap( - storage, storage_target.format, true, false, true, true, - storage_target) == RenderTargetOverlap::Unsupported && - ClassifyStorageRenderTargetOverlap( - storage, vk::Format::eR32G32B32A32Sfloat, true, false, false, - true, storage_target) == RenderTargetOverlap::Unsupported && - ClassifyStorageRenderTargetOverlap(storage, storage_target.format, - true, false, false, true, - mismatched_storage_target) == - RenderTargetOverlap::Unsupported && - ClassifyStorageRenderTargetOverlap( - storage, storage_target.format, false, false, false, true, - storage_target) == RenderTargetOverlap::Unsupported && - ClassifyStorageRenderTargetOverlap( - storage, storage_target.format, true, false, false, false, - storage_target) == RenderTargetOverlap::Unsupported, - "unsupported storage-to-render-target transition was admitted"); - partial_storage_target.address = storage.address + storage.size; - Require("ImageOverlapResolution", "storage render-target adjacency", - ClassifyStorageRenderTargetOverlap( - storage, storage_target.format, false, false, false, false, - partial_storage_target) == RenderTargetOverlap::None, - "byte-disjoint storage and render-target allocations were treated as " - "aliases"); - target.address += 0x2000; - target.size = 0x2000; - Require("ImageOverlapResolution", "render target pool reuse", - ClassifyRenderTargetOverlap(sampled, false, target) == - RenderTargetOverlap::RetireSampled, - "page-contained CPU-owned sampled pool was not retired for a render " - "target"); - target.address = sampled.address + 0x6000; - target.size = 0x4000; - Require( - "ImageOverlapResolution", "render target chance overlap", - ClassifyRenderTargetOverlap(sampled, false, target) == - RenderTargetOverlap::RetireSampled, - "page-isolated sampled pool overlap was not retired for a render target"); - target.address++; - Require("ImageOverlapResolution", "render target partial page", - ClassifyRenderTargetOverlap(sampled, false, target) == - RenderTargetOverlap::RetireSampled, - "true byte overlap within a partial tracker page was not retired"); - target.address = sampled.address + 0x6101; - target.size = 0x4000; - Require("ImageOverlapResolution", "render target unaligned chance overlap", - ClassifyRenderTargetOverlap(sampled, false, target) == - RenderTargetOverlap::RetireSampled, - "unaligned clean sampled chance overlap was not retired"); - target.address = page_left.address + page_left.size; - target.size = page_left.size; - Require("ImageOverlapResolution", "render target shared page", - ClassifyRenderTargetOverlap(page_left, false, target) == - RenderTargetOverlap::None, - "byte-disjoint allocation sharing a tracker page was treated as an " - "alias"); - Require("ImageOverlapResolution", "sampled render-target shared page", - ClassifySampledRenderTargetOverlap(page_left, target, false) == - RenderTargetOverlap::None, - "reverse byte-disjoint tracker-page relationship was treated as an " - "alias"); - target.address = sampled.address + sampled.size; - target.size = sampled.size; - Require( - "ImageOverlapResolution", "render target adjacent", - ClassifyRenderTargetOverlap(sampled, false, target) == - RenderTargetOverlap::None, - "adjacent sampled and render-target allocations were treated as aliases"); - - RenderTargetInfo old_target{}; - old_target.address = 0x108d80000ull; - old_target.size = 0x10000; - old_target.format = vk::Format::eR8G8B8A8Unorm; - old_target.width = 48; - old_target.height = 48; - old_target.pitch = 128; - old_target.bytes_per_element = 4; - old_target.tile_mode = - Prospero::GpuEnumValue(Prospero::TileMode::kRenderTarget); - RenderTargetInfo replacement_target = old_target; - replacement_target.size = 0x20000; - replacement_target.format = vk::Format::eR16G16B16A16Sfloat; - replacement_target.width = 128; - replacement_target.height = 128; - replacement_target.bytes_per_element = 8; - Require("ImageOverlapResolution", "render target allocation-pool replacement", - ClassifyRenderTargetOverlap(old_target, false, false, false, true, - replacement_target) == - RenderTargetOverlap::RetireTarget, - "clean same-base render-target pool allocation was not retired"); - Require("ImageOverlapResolution", "render target allocation-pool GPU owner", - ClassifyRenderTargetOverlap(old_target, true, false, true, true, - replacement_target) == - RenderTargetOverlap::Unsupported, - "GPU-owned render-target pool allocation was retired"); - Require("ImageOverlapResolution", - "render target allocation-pool buffer owner", - ClassifyRenderTargetOverlap(old_target, false, true, false, true, - replacement_target) == - RenderTargetOverlap::Unsupported, - "buffer-dirty render-target pool allocation was retired"); - auto offset_target = replacement_target; - offset_target.address += TRACKER_PAGE_SIZE; - Require("ImageOverlapResolution", "render target allocation-pool offset", - ClassifyRenderTargetOverlap(old_target, false, false, false, true, - offset_target) == - RenderTargetOverlap::RetireTarget, - "clean offset render-target allocation was not retired"); - auto contained_target = old_target; - contained_target.address += TRACKER_PAGE_SIZE; - contained_target.size = TRACKER_PAGE_SIZE; - Require("ImageOverlapResolution", - "render target contained allocation-pool reuse", - ClassifyRenderTargetOverlap(old_target, false, false, false, true, - contained_target) == - RenderTargetOverlap::RetireTarget && - ClassifyRenderTargetOverlap(old_target, false, false, false, - false, contained_target) == - RenderTargetOverlap::Unsupported, - "contained render-target allocation bypassed guest ownership guards"); - auto partial_page_target = replacement_target; - partial_page_target.address++; - Require("ImageOverlapResolution", - "render target allocation-pool partial page", - ClassifyRenderTargetOverlap(old_target, false, false, false, true, - partial_page_target) == - RenderTargetOverlap::Unsupported, - "partially shared tracker page was treated as allocation-pool " - "replacement"); - auto same_shape_target = old_target; - same_shape_target.format = vk::Format::eR8G8B8A8Srgb; - Require("ImageOverlapResolution", "render target RGBA8 sRGB reinterpretation", - IsCompatibleRenderTargetView(old_target, same_shape_target), - "exact RGBA8 UNORM-to-sRGB target was not recognized as a compatible " - "view"); - Require("ImageOverlapResolution", "render target sRGB no retirement", - ClassifyRenderTargetOverlap(old_target, false, false, false, true, - same_shape_target) == - RenderTargetOverlap::Unsupported, - "compatible render-target view fell through to target retirement"); - auto incompatible_same_shape_target = same_shape_target; - incompatible_same_shape_target.format = vk::Format::eR8G8B8A8Uint; - Require("ImageOverlapResolution", "render target incompatible same shape", - !IsCompatibleRenderTargetView(old_target, - incompatible_same_shape_target) && - ClassifyRenderTargetOverlap(old_target, false, false, false, true, - incompatible_same_shape_target) == - RenderTargetOverlap::RetireTarget, - "clean incompatible render-target format was not recreated"); - - RenderTargetInfo ppsa02604_unorm_target{}; - ppsa02604_unorm_target.address = 0x79c50000ull; - ppsa02604_unorm_target.size = 0x870000; - ppsa02604_unorm_target.format = vk::Format::eB8G8R8A8Unorm; - ppsa02604_unorm_target.width = 1920; - ppsa02604_unorm_target.height = 1080; - ppsa02604_unorm_target.pitch = 1920; - ppsa02604_unorm_target.bytes_per_element = 4; - ppsa02604_unorm_target.tile_mode = - Prospero::GpuEnumValue(Prospero::TileMode::kRenderTarget); - auto ppsa02604_srgb_target = ppsa02604_unorm_target; - ppsa02604_srgb_target.format = vk::Format::eB8G8R8A8Srgb; - Require("ImageOverlapResolution", "PPSA02604 BGRA8 sRGB reinterpretation", - IsCompatibleRenderTargetView(ppsa02604_unorm_target, - ppsa02604_srgb_target), - "PPSA02604 exact BGRA8 UNORM-to-sRGB target was not recognized as a " - "view"); - auto adjacent_target = replacement_target; - adjacent_target.address = old_target.address + old_target.size; - Require( - "ImageOverlapResolution", "render target allocation-pool adjacent", - ClassifyRenderTargetOverlap(old_target, false, false, false, true, - adjacent_target) == RenderTargetOverlap::None, - "adjacent render targets were treated as allocation-pool replacement"); - - Require("ImageOverlapResolution", "metadata retains sampled image", - ClassifyMetaImageOverlap(true, false, false, false, false) == - MetaImageOverlap::RetainSampled, - "CPU-current sampled metadata alias was rejected"); - Require("ImageOverlapResolution", "metadata retires writable image", - ClassifyMetaImageOverlap(false, true, false, false, false) == - MetaImageOverlap::RetireImage, - "guest-current writable image was not retired for metadata reuse"); - Require("ImageOverlapResolution", "metadata rejects GPU target", - ClassifyMetaImageOverlap(false, true, true, false, false) == - MetaImageOverlap::Unsupported && - ClassifyMetaImageOverlap(false, true, false, true, false) == - MetaImageOverlap::Unsupported && - ClassifyMetaImageOverlap(false, true, false, false, true) == - MetaImageOverlap::Unsupported, - "unordered or dirty writable image was admitted for metadata " - "reuse"); - Require("ImageOverlapResolution", "metadata rejects unsupported image", - ClassifyMetaImageOverlap(false, false, false, false, false) == - MetaImageOverlap::Unsupported, - "unsupported cached image kind was admitted for metadata reuse"); - Require( - "ImageOverlapResolution", "guest-current depth metadata pool reuse", - CanRetireGuestCurrentDepthForMetadataReuse(false, false, false, false, - false, 0), - "guest-current depth owner was not retired when its HTile allocation was " - "reused"); - Require("ImageOverlapResolution", "depth metadata pool ownership guards", - !CanRetireGuestCurrentDepthForMetadataReuse(true, false, true, false, - false, 0) && - !CanRetireGuestCurrentDepthForMetadataReuse(false, true, false, - false, false, 0) && - !CanRetireGuestCurrentDepthForMetadataReuse(false, false, false, - true, true, 1), - "GPU-owned, buffer-owned, or cleared HTile metadata was " - "admitted for pool reuse"); - - ImageInfo pooled_sampled{}; - pooled_sampled.address = 0x71141000; - pooled_sampled.size = 0x40000; - DepthTargetInfo pooled_depth{}; - pooled_depth.address = 0x71150000; - pooled_depth.size = 0x10000; - pooled_depth.stencil_address = 0x71170000; - pooled_depth.stencil_size = 0x10000; - Require("ImageOverlapResolution", "guest-current sampled/depth pool alias", - CanRetireGuestCurrentDepthForSampled(pooled_sampled, pooled_depth, - false, false, false, false), - "guest-current contained depth target was not retired for a sampled " - "allocation"); - Require("ImageOverlapResolution", "sampled/depth pool ownership guards", - !CanRetireGuestCurrentDepthForSampled(pooled_sampled, pooled_depth, - true, false, true, true) && - !CanRetireGuestCurrentDepthForSampled( - pooled_sampled, pooled_depth, false, true, false, false), - "GPU-owned or buffer-owned sampled/depth alias was " - "admitted"); - ImageInfo offset_sampled{}; - offset_sampled.address = 0x6c6cd000; - offset_sampled.size = 0x10000; - DepthTargetInfo offset_depth{}; - offset_depth.address = 0x6c6d0000; - offset_depth.size = 0x10000; - Require("ImageOverlapResolution", "guest-current sampled to depth pool alias", - CanRetireGuestCurrentSampledForDepth( - offset_sampled, offset_depth, false, false, false, false, true), - "offset guest-current sampled allocation was not retired for depth " - "reuse"); - Require( - "ImageOverlapResolution", "sampled to depth pool ownership guards", - !CanRetireGuestCurrentSampledForDepth(offset_sampled, offset_depth, true, - false, false, true, true) && - !CanRetireGuestCurrentSampledForDepth( - offset_sampled, offset_depth, false, true, false, false, true) && - !CanRetireGuestCurrentSampledForDepth( - offset_sampled, offset_depth, false, false, true, false, true) && - !CanRetireGuestCurrentSampledForDepth( - offset_sampled, offset_depth, false, false, false, false, false), - "dirty or stale sampled allocation was admitted for " - "depth reuse"); - - const std::array isolated_retirement{ - ImageRetirementRange{sampled.address, TRACKER_PAGE_SIZE, true}, - ImageRetirementRange{sampled.address + 2 * TRACKER_PAGE_SIZE, - TRACKER_PAGE_SIZE, false}}; - Require("ImageOverlapResolution", "isolated retirement", - !FindImageRetirementConflict(isolated_retirement).Exists(), - "page-isolated sampled retirement was rejected"); - const std::array tail_alias_retirement{ - ImageRetirementRange{sampled.address, 3 * TRACKER_PAGE_SIZE, true}, - ImageRetirementRange{sampled.address + 2 * TRACKER_PAGE_SIZE, - TRACKER_PAGE_SIZE, false}}; - const auto retirement_conflict = - FindImageRetirementConflict(tail_alias_retirement); - Require("ImageOverlapResolution", "retained tail alias", - retirement_conflict.Exists() && retirement_conflict.retired == 0 && - retirement_conflict.retained == 1, - "retiring a wide image silently untracked a retained tail alias"); - using MipOwnerIndex = MultiRangePageOwnerIndex; - constexpr uint64_t mip_chain_address = 0x110a10000ull; - constexpr uint64_t mip_chain_size = 0x2b0000; - constexpr uint64_t mip_zero_address = 0x110ac0000ull; - constexpr uint64_t mip_zero_size = 0x200000; - constexpr uint64_t mip_one_address = 0x110a40000ull; - constexpr uint64_t mip_one_size = 0x80000; - MipOwnerIndex mip_owners; - Require("ImageOverlapResolution", "PPSA06084 mip alias registration", - mip_owners.Register(1, {{mip_chain_address, mip_chain_size}}) && - mip_owners.Register(2, {{mip_zero_address, mip_zero_size}}) && - ClassifyStorageImageOverlap(mip_one_address, mip_one_size, - mip_chain_address, mip_chain_size, - true, false, false, false) == - StorageImageOverlap::RetireSampled && - ClassifyStorageImageOverlap(mip_one_address, mip_one_size, - mip_zero_address, mip_zero_size, true, - false, false, - false) == StorageImageOverlap::None, - "captured full-chain and disjoint mip owners were misclassified"); - std::vector mip_releases; - Require( - "ImageOverlapResolution", "PPSA06084 retained mip ownership", - mip_owners.Unregister(1, mip_releases) && mip_releases.size() == 1 && - mip_releases[0].address == mip_chain_address && - mip_releases[0].size == mip_zero_address - mip_chain_address && - mip_owners.Query(mip_zero_address, mip_zero_size) == - std::vector{2} && - mip_owners.Register(3, {{mip_one_address, mip_one_size}}) && - mip_owners.Query(mip_one_address, mip_one_size) == - std::vector{3} && - mip_owners.Query(mip_zero_address, mip_zero_size) == - std::vector{2}, - "retiring the full chain released or conflated a disjoint cached mip"); - DepthTargetInfo depth{}; - depth.address = sampled.address; - depth.size = 0x6000; - depth.depth_load_clear = true; - Require( - "ImageOverlapResolution", "clear depth reuse", - ClassifyDepthOverlap(sampled, false, depth) == - DepthOverlap::RetireSampled, - "equal-base CPU-owned sampled image was not classified as pool reuse"); - Require("ImageOverlapResolution", "gpu owner", - ClassifyDepthOverlap(sampled, true, depth) == - DepthOverlap::Unsupported, - "GPU-owned sampled image was admitted for retirement"); - depth.depth_load_clear = false; - Require("ImageOverlapResolution", "incompatible depth load", - ClassifyDepthOverlap(sampled, false, depth) == - DepthOverlap::Unsupported, - "layout-unknown depth load was admitted for retirement"); - - ImageInfo ppsa01880_sampled_depth{}; - ppsa01880_sampled_depth.address = 0x1095200000ull; - ppsa01880_sampled_depth.size = 0x10000000ull; - ppsa01880_sampled_depth.format = - Prospero::GpuEnumValue(Prospero::BufferFormat::k32Float); - ppsa01880_sampled_depth.width = 8192; - ppsa01880_sampled_depth.height = 8192; - ppsa01880_sampled_depth.pitch = 8192; - ppsa01880_sampled_depth.tile = - Prospero::GpuEnumValue(Prospero::TileMode::kDepth); - ppsa01880_sampled_depth.type = - Prospero::GpuEnumValue(Prospero::ImageType::kColor2D); - DepthTargetInfo ppsa01880_depth{}; - ppsa01880_depth.address = ppsa01880_sampled_depth.address; - ppsa01880_depth.size = ppsa01880_sampled_depth.size; - ppsa01880_depth.format = vk::Format::eD32Sfloat; - ppsa01880_depth.guest_format = ppsa01880_sampled_depth.format; - ppsa01880_depth.width = ppsa01880_sampled_depth.width; - ppsa01880_depth.height = ppsa01880_sampled_depth.height; - ppsa01880_depth.pitch = ppsa01880_sampled_depth.pitch; - ppsa01880_depth.bytes_per_element = 4; - ppsa01880_depth.tile_mode = ppsa01880_sampled_depth.tile; - Require("ImageOverlapResolution", "PPSA01880 exact depth load", - ClassifyDepthOverlap(ppsa01880_sampled_depth, false, - ppsa01880_depth) == DepthOverlap::RetireSampled, - "captured CPU-owned R32F/D32F depth layout was not retired"); - - ImageInfo ppsa09477_sampled_depth{}; - ppsa09477_sampled_depth.address = 0x10adc00000ull; - ppsa09477_sampled_depth.size = 0x870000; - ppsa09477_sampled_depth.format = - Prospero::GpuEnumValue(Prospero::BufferFormat::k32Float); - ppsa09477_sampled_depth.width = 1920; - ppsa09477_sampled_depth.height = 1080; - ppsa09477_sampled_depth.pitch = 2048; - ppsa09477_sampled_depth.tile = - Prospero::GpuEnumValue(Prospero::TileMode::kDepth); - ppsa09477_sampled_depth.type = - Prospero::GpuEnumValue(Prospero::ImageType::kColor2D); - DepthTargetInfo ppsa09477_depth{}; - ppsa09477_depth.address = ppsa09477_sampled_depth.address; - ppsa09477_depth.size = ppsa09477_sampled_depth.size; - ppsa09477_depth.stencil_address = 0x10aec00000ull; - ppsa09477_depth.stencil_size = 0x220000; - ppsa09477_depth.format = vk::Format::eD32SfloatS8Uint; - ppsa09477_depth.guest_format = ppsa09477_sampled_depth.format; - ppsa09477_depth.width = ppsa09477_sampled_depth.width; - ppsa09477_depth.height = ppsa09477_sampled_depth.height; - ppsa09477_depth.pitch = ppsa09477_sampled_depth.pitch; - ppsa09477_depth.bytes_per_element = 4; - ppsa09477_depth.tile_mode = ppsa09477_sampled_depth.tile; - ppsa09477_depth.stencil_access = true; - Require("ImageOverlapResolution", "PPSA09477 disjoint stencil depth load", - ClassifyDepthOverlap(ppsa09477_sampled_depth, false, - ppsa09477_depth) == DepthOverlap::RetireSampled, - "captured sampled depth was not preserved beside its disjoint " - "stencil plane"); - auto ppsa09477_overlapping_stencil = ppsa09477_depth; - ppsa09477_overlapping_stencil.stencil_address = - ppsa09477_sampled_depth.address + 0x800000; - Require("ImageOverlapResolution", "PPSA09477 overlapping stencil guard", - ClassifyDepthOverlap(ppsa09477_sampled_depth, false, - ppsa09477_overlapping_stencil) == - DepthOverlap::Unsupported, - "sampled depth was retired while also aliasing the stencil plane"); - auto promoted_d16_sampled = ppsa09477_sampled_depth; - promoted_d16_sampled.format = - Prospero::GpuEnumValue(Prospero::BufferFormat::k16UNorm); - auto promoted_d16_depth = ppsa09477_depth; - promoted_d16_depth.guest_format = promoted_d16_sampled.format; - promoted_d16_depth.format = vk::Format::eD24UnormS8Uint; - promoted_d16_depth.bytes_per_element = 2; - Require( - "ImageOverlapResolution", "promoted D16 native-copy guard", - ClassifyDepthOverlap(promoted_d16_sampled, false, promoted_d16_depth) == - DepthOverlap::Unsupported, - "sampled D16 depth was admitted without a native transfer-width match"); - Require("ImageOverlapResolution", "PPSA01880 GPU depth load", - ClassifyDepthOverlap(ppsa01880_sampled_depth, true, - ppsa01880_depth) == DepthOverlap::Unsupported, - "captured GPU-owned sampled depth was admitted for retirement"); - auto mismatched_sampled_depth = ppsa01880_sampled_depth; - mismatched_sampled_depth.pitch++; - auto mismatched_depth = ppsa01880_depth; - mismatched_depth.stencil_address = - ppsa01880_depth.address + ppsa01880_depth.size; - mismatched_depth.stencil_size = 0x4000000; - Require( - "ImageOverlapResolution", "PPSA01880 depth layout guards", - ClassifyDepthOverlap(mismatched_sampled_depth, false, ppsa01880_depth) == - DepthOverlap::Unsupported && - ClassifyDepthOverlap(ppsa01880_sampled_depth, false, - mismatched_depth) == DepthOverlap::Unsupported, - "pitch-mismatched or stencil-bearing depth load was admitted"); - mismatched_sampled_depth = ppsa01880_sampled_depth; - mismatched_sampled_depth.levels = 2; - mismatched_sampled_depth.view_levels = 2; - mismatched_depth = ppsa01880_depth; - mismatched_depth.format = vk::Format::eD16Unorm; - Require( - "ImageOverlapResolution", "PPSA01880 depth topology guards", - ClassifyDepthOverlap(mismatched_sampled_depth, false, ppsa01880_depth) == - DepthOverlap::Unsupported && - ClassifyDepthOverlap(ppsa01880_sampled_depth, false, - mismatched_depth) == DepthOverlap::Unsupported, - "mipmapped or format-mismatched depth load was admitted"); - struct DepthTransitionCase { - const char *name; - bool clear; - bool native; - bool sampled_cpu_dirty; - bool sampled_buffer_modified; - bool buffer_overlap; - bool buffer_cpu_dirty; - DepthTransitionSource expected; - }; - constexpr DepthTransitionCase transition_cases[] = { - {"clear", true, true, false, false, false, true, - DepthTransitionSource::None}, - {"clean native", false, true, false, false, false, true, - DepthTransitionSource::Native}, - {"missing native", false, false, false, false, false, false, - DepthTransitionSource::Guest}, - {"CPU-dirty native", false, true, true, false, false, false, - DepthTransitionSource::Guest}, - {"buffer-modified native", false, true, false, true, false, false, - DepthTransitionSource::Guest}, - {"dirty overlapping buffer", false, true, false, false, true, true, - DepthTransitionSource::Guest}, - }; - for (const auto &test : transition_cases) { - Require("ImageOverlapResolution", test.name, - SelectDepthTransitionSource( - test.clear, test.native, test.sampled_cpu_dirty, - test.sampled_buffer_modified, test.buffer_overlap, - test.buffer_cpu_dirty) == test.expected, - "depth preservation source changed"); - } - depth.depth_load_clear = true; - depth.stencil_address = sampled.address + 0x6000; - depth.stencil_size = 0x2000; - Require("ImageOverlapResolution", "stencil load", - ClassifyDepthOverlap(sampled, false, depth) == - DepthOverlap::Unsupported, - "load-preserving stencil use was admitted for retirement"); - depth.stencil_load_clear = true; - Require("ImageOverlapResolution", "clear stencil reuse", - ClassifyDepthOverlap(sampled, false, depth) == - DepthOverlap::RetireSampled, - "fully cleared depth/stencil reuse was rejected"); - depth.stencil_load_clear = false; - depth.stencil_access = false; - Require("ImageOverlapResolution", "unused stencil first load", - CanLoadStencilAttachment(depth, false), - "unused stencil unnecessarily required initialized contents"); - depth.stencil_access = true; - Require("ImageOverlapResolution", "uninitialized stencil access", - !CanLoadStencilAttachment(depth, false), - "uninitialized stencil access was silently admitted"); - Require("ImageOverlapResolution", "initialized stencil load", - CanLoadStencilAttachment(depth, true), - "initialized stencil contents could not be loaded"); - depth.stencil_load_clear = true; - Require("ImageOverlapResolution", "stencil clear initializes", - CanLoadStencilAttachment(depth, false), - "stencil clear did not initialize the attachment"); - depth.stencil_load_clear = false; - depth.stencil_htile_compressed = false; - Require("ImageOverlapResolution", "raw stencil load", - CanLoadRawStencilPlane(depth), - "uncompressed guest stencil plane was rejected"); - depth.stencil_htile_compressed = true; - Require("ImageOverlapResolution", "compressed stencil load", - !CanLoadRawStencilPlane(depth), - "HTile-compressed guest stencil plane was silently admitted"); - depth.stencil_htile_compressed = false; - depth.address = sampled.address; - depth.size = 0xff0000; - depth.stencil_address = sampled.address + 0x1000000; - depth.stencil_size = 0x200000; - Require("ImageOverlapResolution", "logical depth view", - IsDepthTargetRangeCompatible(depth, depth.address, 0xfd2000), - "same-base logical depth range did not reuse its padded target"); - Require("ImageOverlapResolution", "complete depth view", - IsDepthTargetRangeCompatible(depth, depth.address, depth.size), - "complete depth range was rejected"); - Require("ImageOverlapResolution", "oversized depth view", - !IsDepthTargetRangeCompatible(depth, depth.address, depth.size + 1), - "oversized depth range was silently admitted"); - Require( - "ImageOverlapResolution", "offset depth view", - !IsDepthTargetRangeCompatible(depth, depth.address + 1, depth.size - 1), - "offset depth range was silently admitted"); - Require("ImageOverlapResolution", "complete stencil view", - IsDepthTargetRangeCompatible(depth, depth.stencil_address, - depth.stencil_size), - "complete stencil range was rejected"); - Require("ImageOverlapResolution", "partial stencil view", - !IsDepthTargetRangeCompatible(depth, depth.stencil_address, - depth.stencil_size - 1), - "partial stencil range was silently admitted"); - depth.address = sampled.address + 0x1000; - Require("ImageOverlapResolution", "offset alias", - ClassifyDepthOverlap(sampled, false, depth) == - DepthOverlap::Unsupported, - "offset image overlap was admitted for retirement"); - depth.address = sampled.address + sampled.size; - depth.stencil_address = 0; - depth.stencil_size = 0; - depth.stencil_load_clear = false; - Require("ImageOverlapResolution", "adjacent", - ClassifyDepthOverlap(sampled, false, depth) == DepthOverlap::None, - "byte-disjoint adjacent image ranges were treated as aliases"); - std::printf("[host] %-32s ok\n", "ImageOverlapResolution"); -} - void CheckNativeMsaaState() { Require("NativeMsaaState", "sample encoding", render_sample_count(0) == 1 && render_sample_count(1) == 2 && @@ -13563,7 +16352,8 @@ void CheckNativeMsaaState() { TileGetDepthSize( 1920, 1080, 0, Prospero::GpuEnumValue(Prospero::DepthFormat::kZ32F), - Prospero::GpuEnumValue(Prospero::StencilFormat::k8UInt), true, stencil, htile, depth, 3) && + Prospero::GpuEnumValue(Prospero::StencilFormat::k8UInt), true, + stencil, htile, depth, 3) && depth.align == 0x10000 && depth.size == 0x03fc0000 && stencil.align == 0x10000 && stencil.size == 0x010e0000 && htile.align == 0x8000 && htile.size == 0x00030000, @@ -13611,6 +16401,132 @@ void CheckStencilAttachmentAccess() { std::printf("[host] %-32s ok\n", "StencilAttachmentAccess"); } +void CheckDepthAttachmentWrites() { + RenderDepthInfo target{}; + target.format = vk::Format::eD32SfloatS8Uint; + target.depth_write_enable = true; + Require("DepthAttachmentWrites", "disabled depth test", + !target.AttachmentWriteAspects(), + "disabled depth testing claimed a depth write"); + + target.depth_test_enable = true; + target.depth_compare_op = vk::CompareOp::eLess; + Require("DepthAttachmentWrites", "depth write", + target.AttachmentWriteAspects() == vk::ImageAspectFlagBits::eDepth, + "write-enabled depth testing did not claim depth"); + + target.depth_test_enable = false; + target.depth_write_enable = false; + target.depth_load_clear_enable = true; + Require("DepthAttachmentWrites", "depth clear", + target.AttachmentWriteAspects() == vk::ImageAspectFlagBits::eDepth, + "depth attachment clear did not claim depth"); + + target.depth_load_clear_enable = false; + target.stencil_test_enable = true; + target.stencil_dynamic_front = {0xff, 0xff, 0}; + target.stencil_dynamic_back = target.stencil_dynamic_front; + target.stencil_static_front = {vk::StencilOp::eKeep, vk::StencilOp::eKeep, + vk::StencilOp::eKeep, vk::CompareOp::eAlways}; + target.stencil_static_back = target.stencil_static_front; + Require("DepthAttachmentWrites", "stencil keep", + !target.AttachmentWriteAspects(), + "KEEP-only stencil state claimed a stencil write"); + + target.stencil_static_front.failOp = vk::StencilOp::eZero; + Require("DepthAttachmentWrites", "unreachable stencil fail", + !target.AttachmentWriteAspects(), + "ALWAYS comparison claimed an unreachable fail operation"); + + target.stencil_static_front.failOp = vk::StencilOp::eKeep; + target.stencil_static_front.passOp = vk::StencilOp::eReplace; + Require("DepthAttachmentWrites", "stencil pass write", + target.AttachmentWriteAspects() == vk::ImageAspectFlagBits::eStencil, + "write-capable stencil pass did not claim stencil"); + + target.stencil_dynamic_front.writeMask = 0; + Require("DepthAttachmentWrites", "back-face keep", + !target.AttachmentWriteAspects(), + "masked front write or KEEP-only back face claimed stencil"); + + target.stencil_test_enable = false; + target.stencil_clear_enable = true; + Require("DepthAttachmentWrites", "stencil clear", + target.AttachmentWriteAspects() == vk::ImageAspectFlagBits::eStencil, + "stencil attachment clear did not claim stencil"); + + target.format = vk::Format::eD32Sfloat; + Require("DepthAttachmentWrites", "missing stencil aspect", + !target.AttachmentWriteAspects(), + "depth-only format claimed a stencil write"); + std::printf("[host] %-32s ok\n", "DepthAttachmentWrites"); +} + +void CheckDynamicRenderingState() { + RenderState first{}; + first.width = 64; + first.height = 32; + first.num_layers = 2; + first.num_color_attachments = 1; + first.color_attachments[0].image_layout = + vk::ImageLayout::eColorAttachmentOptimal; + first.color_attachments[0].clear_value = {1, 2, 3, 4}; + first.color_attachments[0].is_clear = true; + auto changed = first; + Require("DynamicRenderingState", "identical state", first == changed, + "fieldwise RenderState identity rejected an identical scope"); + changed.color_attachments[0].is_clear = false; + Require("DynamicRenderingState", "clear identity", first != changed, + "clear/load state did not participate in rendering identity"); + changed = first; + changed.num_layers = 1; + Require("DynamicRenderingState", "layer identity", first != changed, + "layer count did not participate in rendering identity"); + changed = first; + changed.color_attachments[0].image_layout = vk::ImageLayout::eGeneral; + Require("DynamicRenderingState", "layout identity", first != changed, + "attachment layout did not participate in rendering identity"); + + PipelineRenderingState rgba{}; + rgba.color_count = 1; + rgba.color_formats[0] = vk::Format::eR8G8B8A8Unorm; + auto uint_color = rgba; + uint_color.color_formats[0] = vk::Format::eR8G8B8A8Uint; + Require("DynamicRenderingState", "pipeline color format identity", + rgba != uint_color, + "distinct dynamic-rendering formats can alias one pipeline key"); + auto depth = rgba; + depth.depth_format = vk::Format::eD32SfloatS8Uint; + depth.stencil_format = vk::Format::eD32SfloatS8Uint; + Require("DynamicRenderingState", "pipeline depth/stencil identity", + rgba != depth, + "depth/stencil formats did not participate in pipeline identity"); + + RenderDepthInfo attachment{}; + attachment.format = vk::Format::eD32SfloatS8Uint; + Require("DynamicRenderingState", "read-only depth/stencil layout", + depth_attachment_layout(attachment) == + vk::ImageLayout::eDepthStencilReadOnlyOptimal, + "fully read-only depth/stencil used a writable layout"); + attachment.depth_load_clear_enable = true; + Require("DynamicRenderingState", "depth-write stencil-read layout", + depth_attachment_layout(attachment) == + vk::ImageLayout::eDepthAttachmentStencilReadOnlyOptimal, + "depth-only writes did not retain read-only stencil"); + attachment.depth_load_clear_enable = false; + attachment.stencil_clear_enable = true; + Require("DynamicRenderingState", "depth-read stencil-write layout", + depth_attachment_layout(attachment) == + vk::ImageLayout::eDepthReadOnlyStencilAttachmentOptimal, + "stencil-only writes did not retain read-only depth"); + attachment.depth_load_clear_enable = true; + Require("DynamicRenderingState", "writable depth/stencil layout", + depth_attachment_layout(attachment) == + vk::ImageLayout::eDepthStencilAttachmentOptimal, + "combined depth/stencil writes did not use the writable layout"); + std::printf("[host] %-32s ok\n", "DynamicRenderingState"); +} + void CheckDepthTargetFootprints() { TileSizeAlign stencil{}; TileSizeAlign htile{}; @@ -13674,10 +16590,10 @@ void CheckDepthTargetFootprints() { true}, {"D16 via D24S8", vk::Format::eD24UnormS8Uint, Prospero::GpuEnumValue(Prospero::BufferFormat::k16UNorm), 2, true, true, - false}, + true}, {"D16 via D32S8", vk::Format::eD32SfloatS8Uint, Prospero::GpuEnumValue(Prospero::BufferFormat::k16UNorm), 2, true, true, - false}, + true}, {"D32", vk::Format::eD32Sfloat, Prospero::GpuEnumValue(Prospero::BufferFormat::k32Float), 4, false, true, true}, @@ -13698,30 +16614,32 @@ void CheckDepthTargetFootprints() { false, false}, }; for (const auto &test : target_cases) { - DepthTargetInfo target{}; - target.format = test.host_format; + ImageInfo target{}; + target.pixel_format = test.host_format; target.guest_format = test.guest_format; - target.bytes_per_element = test.bytes_per_element; - target.stencil_address = test.stencil ? 0x10000 : 0; - target.stencil_size = test.stencil ? 0x10000 : 0; + target.bytes_per_block = test.bytes_per_element; + target.stencil = test.stencil ? GuestRange{0x10000, 0x10000} : GuestRange{}; Require("DepthTargetFootprints", test.name, IsSupportedDepthTargetFormat(target) == test.supported && - IsSupportedDepthReadbackFormat(target) == test.readback, + IsSupportedDepthPlaneReadback(target) == test.readback, "host/guest depth format or exact readback policy changed"); } - DepthTargetInfo compressed_stencil{}; - compressed_stencil.format = vk::Format::eD32SfloatS8Uint; + ImageInfo compressed_stencil{}; + compressed_stencil.pixel_format = vk::Format::eD32SfloatS8Uint; compressed_stencil.guest_format = Prospero::GpuEnumValue(Prospero::BufferFormat::k32Float); - compressed_stencil.bytes_per_element = 4; - compressed_stencil.stencil_address = 0x10000; - compressed_stencil.stencil_size = 0x10000; - compressed_stencil.stencil_htile_compressed = true; - Require( - "DepthTargetFootprints", "compressed stencil readback hard guard", - IsSupportedDepthTargetFormat(compressed_stencil) && - !IsSupportedDepthReadbackFormat(compressed_stencil), - "compressed stencil unexpectedly entered the raw-plane readback path"); + compressed_stencil.bytes_per_block = 4; + compressed_stencil.stencil = {0x10000, 0x10000}; + compressed_stencil.metadata.stencil_compressed = true; + Require("DepthTargetFootprints", "compressed stencil depth-plane readback", + IsSupportedDepthTargetFormat(compressed_stencil) && + IsSupportedDepthPlaneReadback(compressed_stencil), + "stencil metadata incorrectly blocked independent depth-plane " + "readback"); + Require("DepthTargetFootprints", "lexicographic subresource comparison", + ImageSubresources{2, 1} > ImageSubresources{1, 4} && + ImageSubresources{1, 4} < ImageSubresources{2, 1}, + "subresource ordering diverged from required overlap semantics"); struct TransferPlaneCase { const char *name; vk::Format attachment; @@ -13762,7 +16680,8 @@ void CheckDepthTargetFootprints() { Require("DepthTargetFootprints", "640x360 Z32S8 without HTile", TileGetDepthSize( 640, 360, 0, Prospero::GpuEnumValue(Prospero::DepthFormat::kZ32F), - Prospero::GpuEnumValue(Prospero::StencilFormat::k8UInt), false, stencil, htile, depth), + Prospero::GpuEnumValue(Prospero::StencilFormat::k8UInt), false, + stencil, htile, depth), "valid non-HTile depth/stencil footprint was rejected"); Require( "DepthTargetFootprints", "640x360 Prospero block sizes", @@ -13777,7 +16696,8 @@ void CheckDepthTargetFootprints() { TileSizeAlign texture_depth{}; TileGetTextureTotalSize( Prospero::GpuEnumValue(Prospero::BufferFormat::k32Float), 640, 360, 1, - depth_pitch, 1, Prospero::GpuEnumValue(Prospero::TileMode::kDepth), false, texture_depth); + depth_pitch, 1, Prospero::GpuEnumValue(Prospero::TileMode::kDepth), false, + texture_depth); Require("DepthTargetFootprints", "640x360 generic depth tile", depth_pitch == 640 && texture_depth.size == 0xf0000 && texture_depth.align == 0x10000, @@ -13827,132 +16747,17 @@ void CheckDepthTargetFootprints() { Require("DepthTargetFootprints", "invalid extent", !TileGetDepthSize( 0, 540, 0, Prospero::GpuEnumValue(Prospero::DepthFormat::kZ32F), - Prospero::GpuEnumValue(Prospero::StencilFormat::k8UInt), true, stencil, htile, depth) && + Prospero::GpuEnumValue(Prospero::StencilFormat::k8UInt), true, + stencil, htile, depth) && !TileGetDepthSize( 16385, 540, 0, Prospero::GpuEnumValue(Prospero::DepthFormat::kZ32F), - Prospero::GpuEnumValue(Prospero::StencilFormat::k8UInt), true, stencil, htile, depth), + Prospero::GpuEnumValue(Prospero::StencilFormat::k8UInt), true, + stencil, htile, depth), "invalid HTile extent was silently admitted"); std::printf("[host] %-32s ok\n", "DepthTargetFootprints"); } -void CheckHtileClearTargetResolution() { - HW::DepthDepthSizeXY one_pixel{}; - one_pixel.valid = true; - Require("HtileClearTargetResolution", "1x1 encoded depth extent", - one_pixel.valid && one_pixel.x_max + 1u == 1u && - one_pixel.y_max + 1u == 1u, - "zero DB_DEPTH_SIZE_XY fields were mistaken for an absent register"); - HW::DepthDepthSizeXY absent{}; - Require("HtileClearTargetResolution", "absent depth extent", !absent.valid, - "an unwritten DB_DEPTH_SIZE_XY register was admitted"); - HW::DepthRenderTarget descriptor_backed{}; - descriptor_backed.z_info.format = - Prospero::GpuEnumValue(Prospero::DepthFormat::kZ16); - descriptor_backed.z_info.tile_surface_enable = true; - descriptor_backed.z_info.zrange_precision = 1; - descriptor_backed.stencil_info.tile_stencil_disable = true; - descriptor_backed.z_read_base_addr = 0xc1a80000; - descriptor_backed.z_write_base_addr = descriptor_backed.z_read_base_addr; - descriptor_backed.htile_data_base_addr = 0xc1a98000; - - HtileClearTarget resolved{}; - Require("HtileClearTargetResolution", - "PPSA09076 descriptor-backed Z16 target", - ResolveHtileClearTarget(descriptor_backed, 0x8000, resolved) && - resolved.address == 0xc1a98000 && resolved.size == 0x8000, - "valid descriptor-proven Z16 HTile clear target was rejected"); - - auto partial_extent = descriptor_backed; - partial_extent.size.x_max = 1; - auto missing_stencil = descriptor_backed; - missing_stencil.stencil_info.format = - Prospero::GpuEnumValue(Prospero::StencilFormat::k8UInt); - auto mismatched_write = descriptor_backed; - mismatched_write.z_write_base_addr += 0x10000; - Require( - "HtileClearTargetResolution", "descriptor-backed rejection boundaries", - !ResolveHtileClearTarget(descriptor_backed, 0x4000, resolved) && - !ResolveHtileClearTarget(partial_extent, 0x8000, resolved) && - !ResolveHtileClearTarget(missing_stencil, 0x8000, resolved) && - !ResolveHtileClearTarget(mismatched_write, 0x8000, resolved), - "ambiguous or inconsistent descriptor-backed HTile target was admitted"); - - HW::DepthRenderTarget derived{}; - derived.z_info.format = Prospero::GpuEnumValue(Prospero::DepthFormat::kZ32F); - derived.z_info.tile_surface_enable = true; - derived.z_info.zrange_precision = 1; - derived.stencil_info.format = - Prospero::GpuEnumValue(Prospero::StencilFormat::k8UInt); - derived.stencil_info.tile_stencil_disable = true; - derived.z_read_base_addr = 0x100000; - derived.z_write_base_addr = derived.z_read_base_addr; - derived.stencil_read_base_addr = 0x500000; - derived.stencil_write_base_addr = derived.stencil_read_base_addr; - derived.htile_data_base_addr = 0x700000; - derived.size.valid = true; - derived.size.x_max = 959; - derived.size.y_max = 539; - TileSizeAlign depth_size{}; - TileSizeAlign stencil_size{}; - TileSizeAlign htile_size{}; - Require( - "HtileClearTargetResolution", "derived Z32S8 target", - TileGetDepthSize(960, 540, 0, derived.z_info.format, - derived.stencil_info.format, true, stencil_size, htile_size, depth_size) && - ResolveHtileClearTarget(derived, htile_size.size, resolved) && - resolved.address == derived.htile_data_base_addr && - resolved.size == htile_size.size && - !ResolveHtileClearTarget(derived, htile_size.size + 0x8000, - resolved), - "extent-derived HTile validation or descriptor cross-check regressed"); - - auto derived_z16 = derived; - derived_z16.z_info.format = - Prospero::GpuEnumValue(Prospero::DepthFormat::kZ16); - derived_z16.stencil_info.format = - Prospero::GpuEnumValue(Prospero::StencilFormat::kInvalid); - derived_z16.stencil_read_base_addr = 0; - derived_z16.stencil_write_base_addr = 0; - Require( - "HtileClearTargetResolution", "derived Z16 depth-only target", - TileGetDepthSize(960, 540, 0, derived_z16.z_info.format, - derived_z16.stencil_info.format, true, stencil_size, htile_size, depth_size) && - stencil_size.align == 0 && stencil_size.size == 0 && - ResolveHtileClearTarget(derived_z16, htile_size.size, resolved) && - resolved.address == derived_z16.htile_data_base_addr && - resolved.size == htile_size.size, - "extent-derived Z16 depth-only HTile target was rejected"); - - auto derived_z16s8 = derived; - derived_z16s8.z_info.format = - Prospero::GpuEnumValue(Prospero::DepthFormat::kZ16); - Require( - "HtileClearTargetResolution", "derived Z16S8 target", - TileGetDepthSize(960, 540, 0, derived_z16s8.z_info.format, - derived_z16s8.stencil_info.format, true, stencil_size, htile_size, depth_size) && - stencil_size.size != 0 && stencil_size.align == 0x10000 && - ResolveHtileClearTarget(derived_z16s8, htile_size.size, resolved) && - resolved.address == derived_z16s8.htile_data_base_addr && - resolved.size == htile_size.size, - "extent-derived Z16S8 HTile target was rejected"); - - auto derived_8x = derived; - derived_8x.z_info.num_samples = 3; - auto malformed_fragments = derived; - malformed_fragments.z_info.num_samples = 4; - Require("HtileClearTargetResolution", "fragment-independent metadata plane", - ResolveHtileClearTarget(derived_8x, htile_size.size, resolved) && - resolved.address == derived_8x.htile_data_base_addr && - resolved.size == htile_size.size && - render_sample_count(derived_8x.z_info.num_samples) == 8 && - !ResolveHtileClearTarget(malformed_fragments, htile_size.size, - resolved), - "HTile clear was coupled to attachment MSAA support or admitted an " - "invalid fragment field"); - std::printf("[host] %-32s ok\n", "HtileClearTargetResolution"); -} - struct FenceLifetimeProbe { explicit FenceLifetimeProbe(bool *destroyed) : destroyed(destroyed) { if (destroyed == nullptr || *destroyed) { @@ -13987,65 +16792,6 @@ void CheckSharedFenceResourceLifetime() { std::printf("[host] %-32s ok\n", "SharedFenceResourceLifetime"); } -void CheckHostDmaMetadataReuse() { - GraphicContext context{}; - constexpr uintptr_t base = 0x0000000200010000ull; - constexpr uint64_t allocation_size = 0x10000; - constexpr uint64_t metadata_size = 0x8000; - auto *memory = static_cast( - VirtualAlloc(reinterpret_cast(base), allocation_size, - MEM_RESERVE | MEM_COMMIT, PAGE_READWRITE)); - Require("HostDmaMetadataReuse", "allocation", - memory == reinterpret_cast(base), - "fixed VirtualAlloc failed"); - std::memset(memory + metadata_size, 0x5a, 0x1000); - - ResourceMutex resource_mutex; - CacheFaultContext fault_context; - PageManager page_manager(CacheFault, &fault_context); - BufferCache buffer_cache(context, page_manager, resource_mutex); - TextureCache texture_cache(context, page_manager, buffer_cache, - resource_mutex); - fault_context.texture = &texture_cache; - buffer_cache.SetTextureCache(texture_cache); - page_manager.OnGpuMap(base, allocation_size); - texture_cache.RegisterMeta(base, metadata_size); - - Require("HostDmaMetadataReuse", "clear", texture_cache.ClearMeta(base), - "metadata clear setup failed"); - MEMORY_BASIC_INFORMATION protection{}; - Require("HostDmaMetadataReuse", "protected", - VirtualQuery(memory, &protection, sizeof(protection)) != 0 && - protection.Protect == PAGE_READONLY, - "virtual metadata was not write-watched"); - Require("HostDmaMetadataReuse", "copy fault", - page_manager.HandleFault(PageFaultAccess::Write, base), - "copy destination did not transfer to CPU ownership"); - buffer_cache.CopyBuffer(nullptr, base, base + metadata_size, 0x1000); - Require("HostDmaMetadataReuse", "copy", - std::memcmp(memory, memory + metadata_size, 0x1000) == 0, - "post-clear CPU DMA copy did not publish backing"); - Require("HostDmaMetadataReuse", "fill fault", - page_manager.HandleFault(PageFaultAccess::Write, base + 0x1000), - "fill destination did not transfer to CPU ownership"); - buffer_cache.FillBuffer(nullptr, base + 0x1000, 0x1000, 0x11223344); - Require("HostDmaMetadataReuse", "fill", - reinterpret_cast(memory + 0x1000)[0] == 0x11223344 && - reinterpret_cast(memory + 0x1ffc)[0] == 0x11223344, - "post-clear CPU DMA fill did not publish backing"); - Require("HostDmaMetadataReuse", "identity", - texture_cache.QueryRegion(base, metadata_size).metadata_pages && - !texture_cache.QueryRegion(base, 0x2000).gpu_metadata_bytes && - VirtualQuery(memory, &protection, sizeof(protection)) != 0 && - protection.Protect == PAGE_READWRITE, - "CPU writes erased metadata identity or retained virtual ownership"); - - texture_cache.UnmapMemory(base, allocation_size); - page_manager.OnGpuUnmap(base, allocation_size); - Require("HostDmaMetadataReuse", "free", - VirtualFree(memory, 0, MEM_RELEASE) != 0, "VirtualFree failed"); - std::printf("[host] %-32s ok\n", "HostDmaMetadataReuse"); -} #endif void CheckEmbeddedFetchLaneSpill() { @@ -14080,8 +16826,7 @@ void CheckEmbeddedFetchLaneSpill() { ShaderRecompiler::CompileResult result; std::string error; Require("EmbeddedFetchLaneSpill", "compile", - ShaderRecompiler::TryRecompile(code, options, result, &error), - error); + ShaderRecompiler::TryRecompile(code, options, result, &error), error); Require("EmbeddedFetchLaneSpill", "fetch rewrite", vertex.resource_fetch_components[0] == 4, "lane-spilled fetch descriptor was not recognized and rewritten"); @@ -14136,9 +16881,44 @@ void CheckClipControlDepthClipState() { std::printf("[host] %-32s ok\n", "ClipControlDepthClipState"); } -void CheckPm4WaitResume() { +void CheckVulkan13FeatureRequirements() { + const auto features = WindowContext::RequiredVulkan13Features(); + Require( + "Vulkan13FeatureRequirements", "production requirements", + features.sType == vk::StructureType::ePhysicalDeviceVulkan13Features && + features.pNext == nullptr && features.dynamicRendering == VK_TRUE && + features.synchronization2 == VK_TRUE, + "production device creation did not require dynamic rendering and " + "synchronization2 from one feature declaration"); + std::printf("[host] %-32s ok\n", "Vulkan13FeatureRequirements"); +} + +void CheckPm4AcquireMemNoOp(RenderContext &renderer) { GraphicsInitJmpTables(); - CommandProcessor processor; + CommandProcessor processor(renderer); + const std::array standard_payload{0x00400000u, 1u, 0u, + 0u, 0u, 10u}; + const std::array custom_payload{ + 0xdeadbeefu, 0xffffffffu, 0x80000000u, 0x13579bdfu, + 0x2468ace0u, 0xaaaaaaaau, 0x55555555u}; + const auto standard_before = standard_payload; + const auto custom_before = custom_payload; + Require("Pm4AcquireMemNoOp", "recognized packets", + CpOpAcquireMem(processor, 0xC0055800u, standard_payload.data(), 0, + 0) == 6 && + CpOpAcquireMem(processor, 0xc0061050u, custom_payload.data(), 0, + 0) == 7 && + standard_payload == standard_before && + custom_payload == custom_before, + "ACQUIRE_MEM did not consume both pinned packet forms as " + "side-effect-free " + "no-ops"); + std::printf("[host] %-32s ok\n", "Pm4AcquireMemNoOp"); +} + +void CheckPm4WaitResume(RenderContext &renderer) { + GraphicsInitJmpTables(); + CommandProcessor processor(renderer); uint32_t label = 0; uint32_t prefix = 0; @@ -14192,17 +16972,18 @@ void CheckPm4WaitResume() { label = 1; child[4] = 1; - Require("Pm4WaitResume", "resume", - processor.Process(execution, commands.data(), commands.size()) == - Pm4ProcessResult::Complete && - prefix == 11 && child_observation == 0 && suffix == 22, - "resumed indirect wait replayed a child or did not finish its parent"); + Require( + "Pm4WaitResume", "resume", + processor.Process(execution, commands.data(), commands.size()) == + Pm4ProcessResult::Complete && + prefix == 11 && child_observation == 0 && suffix == 22, + "resumed indirect wait replayed a child or did not finish its parent"); std::printf("[host] %-32s ok\n", "Pm4WaitResume"); } -void CheckPm4CeCompletion() { +void CheckPm4CeCompletion(RenderContext &renderer) { GraphicsInitJmpTables(); - CommandProcessor processor; + CommandProcessor processor(renderer); uint32_t suffix = 0; const auto address = reinterpret_cast(&suffix); @@ -14238,7 +17019,9 @@ void CheckPm4CeCompletion() { int main(int argc, char **argv) { using namespace Libs::Graphics; + std::setvbuf(stdout, nullptr, _IONBF, 0); EnsureConfigInitialized(); + CheckLeastRecentlyUsedCacheOrdering(); if (argc == 2 && std::strcmp(argv[1], "--clip-control-only") == 0) { CheckClipControlDepthClipState(); return 0; @@ -14247,6 +17030,26 @@ int main(int argc, char **argv) { CheckReferenceClockScale(); return 0; } + if (argc == 2 && std::strcmp(argv[1], "--scheduler-only") == 0) { + VulkanHarness vulkan; + vulkan.CheckSchedulerTimeline(); + return 0; + } + if (argc == 2 && std::strcmp(argv[1], "--stream-buffer-only") == 0) { + VulkanHarness vulkan; + vulkan.CheckStreamBufferRing(); + return 0; + } + if (argc == 2 && std::strcmp(argv[1], "--gpu-tiler-only") == 0) { + VulkanHarness vulkan; + vulkan.CheckGpuTilerCpuParity(); + return 0; + } + if (argc == 2 && std::strcmp(argv[1], "--gpu-command-lane-only") == 0) { + VulkanHarness vulkan; + vulkan.CheckGpuCommandLane(); + return 0; + } #if KYTY_PLATFORM == KYTY_PLATFORM_WINDOWS if (argc == 2 && std::strcmp(argv[1], "--reverse-rt-death") == 0) { RunReverseRenderTargetDeathCase(); @@ -14260,50 +17063,61 @@ int main(int argc, char **argv) { return 0; } if (argc == 2 && std::strcmp(argv[1], "--image-overlap-only") == 0) { - CheckImageOverlapResolution(); - CheckQueryRegionAggregation(); + CheckDepthAttachmentWrites(); + CheckDynamicRenderingState(); VulkanHarness vulkan; - vulkan.CheckQueryRegionImageClassification(); + vulkan.CheckRenderExecutorStencilBindingDiscovery(); + vulkan.CheckUnifiedTextureCacheFlow(); + vulkan.CheckBgra16Readback(); return 0; } if (argc == 2 && std::strcmp(argv[1], "--htile-clear-only") == 0) { - CheckHtileClearTargetResolution(); - CheckOverlappingMetadataViews(); + VulkanHarness vulkan; + vulkan.CheckUnifiedTextureCacheFlow(); return 0; } if (argc == 2 && std::strcmp(argv[1], "--layered-image-only") == 0) { - CheckColorResolveLayers(); - CheckGpuMetadataReuse(); + VulkanHarness vulkan; + vulkan.CheckUnifiedImageViewCache(); return 0; } if (argc == 2 && std::strcmp(argv[1], "--image-view-only") == 0) { + VulkanHarness vulkan; CheckSampledColorViews(); - CheckSampledVideoOutView(); + CheckSampledVideoOutView(vulkan.RuntimeRenderer()); return 0; } if (argc == 2 && std::strcmp(argv[1], "--image-view-cache-only") == 0) { VulkanHarness vulkan; - vulkan.CheckRenderTargetViewCache(); - vulkan.CheckDepthTargetSampledViewCache(); - vulkan.CheckVideoOutSampledViewCache(); + vulkan.CheckUnifiedImageViewCache(); + return 0; + } + if (argc == 2 && std::strcmp(argv[1], "--image-transition-only") == 0) { + VulkanHarness vulkan; + CheckImageTransitionState(vulkan.RuntimeRenderer()); return 0; } if (argc == 2 && std::strcmp(argv[1], "--sampled-depth-resource-only") == 0) { + VulkanHarness vulkan; CheckSampledDepthResource(); - CheckSampledDepthDescriptor(); + CheckSampledDepthDescriptor(vulkan.RuntimeRenderer()); return 0; } if (argc == 2 && std::strcmp(argv[1], "--buffer-cache-range-only") == 0) { - CheckBufferCacheRangeMerge(); - CheckBufferBindingStabilization(); - CheckImageWriteActivationPolicy(); + VulkanHarness vulkan; + vulkan.CheckUnifiedTextureCacheFlow(); + return 0; + } + if (argc == 2 && std::strcmp(argv[1], "--buffer-cache-gc-only") == 0) { + VulkanHarness vulkan; + vulkan.CheckBufferCacheDirtyGarbageCollection(); return 0; } if (argc == 2 && std::strcmp(argv[1], "--storage-bgra-only") == 0) { CheckSampledColorViews(); CheckBasicStorageTextureDescriptor(); VulkanHarness vulkan; - vulkan.CheckMutableRenderTargetBgraStorageView(); + vulkan.CheckUnifiedImageViewCache(); RunCase(&vulkan, ImageStoreBgraUsesInverseSwizzle()); return 0; } @@ -14316,9 +17130,8 @@ int main(int argc, char **argv) { return 0; } if (argc == 2 && std::strcmp(argv[1], "--storage-sampled-only") == 0) { - CheckStorageTextureSampledReuse(); VulkanHarness vulkan; - vulkan.CheckMutableStorageSrgbView(); + vulkan.CheckUnifiedImageViewCache(); return 0; } if (argc == 2 && std::strcmp(argv[1], "--depth-readback-only") == 0) { @@ -14332,70 +17145,59 @@ int main(int argc, char **argv) { std::strcmp(argv[1], "--storage-texture-descriptor-death") == 0) { RunStorageTextureDescriptorDeathCase(argv[2]); } - if (argc == 3 && - std::strcmp(argv[1], "--storage-texture-access-death") == 0) { - RunStorageTextureAccessDeathCase(argv[2]); - } - if (argc == 3 && std::strcmp(argv[1], "--meta-overlap-death") == 0) { - RunMetaOverlapDeathCase(argv[2]); - } - if (argc == 3 && std::strcmp(argv[1], "--metadata-descriptor-death") == 0) { - RunMetadataDescriptorDeathCase(argv[2]); + if (argc != 1) { + std::fprintf(stderr, "unknown test selector: %s\n", argv[1]); + return 2; } + VulkanHarness vulkan; CheckReverseRenderTargetFormatContract(); CheckSampledColorViews(); - CheckSampledVideoOutView(); + CheckSampledVideoOutView(vulkan.RuntimeRenderer()); + CheckImageTransitionState(vulkan.RuntimeRenderer()); CheckSampledDepthResource(); - CheckSampledDepthDescriptor(); - CheckBufferCacheRangeMerge(); - CheckBufferBindingStabilization(); - CheckImageWriteActivationPolicy(); + CheckSampledDepthDescriptor(vulkan.RuntimeRenderer()); CheckBasicStorageTextureDescriptor(); CheckStorageTextureLinearUploadLayout(); CheckStorageTextureDepthTileUploadLayout(); - CheckStorageTextureLinearReadbackLayout(); CheckStorageImageSwizzleSpecializationId(); - CheckColorResolveLayers(); CheckStandard64RenderTargetTileRoundTrip(); CheckStorageTextureVolumeUploadLayout(); CheckStorageTextureVolumeMipRegions(); CheckStorageTextureGpuOwnedRebindState(); - CheckStorageTextureSampledReuse(); - CheckStorageTextureDepthAlias(); - CheckStorageTextureAccessPermissions(); - CheckMetaOverlapDeaths(); - CheckOverlappingMetadataViews(); - CheckGpuMetadataReuse(); - CheckMetadataReuseDescriptors(); - CheckImageOverlapResolution(); - CheckQueryRegionAggregation(); CheckNativeMsaaState(); CheckDepthHtileStencilCompatibility(); CheckStencilAttachmentAccess(); + CheckDepthAttachmentWrites(); + CheckDynamicRenderingState(); CheckDepthTargetFootprints(); - CheckHtileClearTargetResolution(); CheckSharedFenceResourceLifetime(); - CheckHostDmaMetadataReuse(); #else - (void)argc; - (void)argv; + if (argc != 1) { + std::fprintf(stderr, "unknown test selector: %s\n", argv[1]); + return 2; + } + VulkanHarness vulkan; #endif CheckClipControlDepthClipState(); CheckReferenceClockScale(); - CheckPm4WaitResume(); - CheckPm4CeCompletion(); + CheckVulkan13FeatureRequirements(); + CheckPm4AcquireMemNoOp(vulkan.RuntimeRenderer()); + CheckPm4WaitResume(vulkan.RuntimeRenderer()); + CheckPm4CeCompletion(vulkan.RuntimeRenderer()); CheckEmbeddedFetchVertexOffset(); CheckEmbeddedFetchLaneSpill(); CheckPs5GameExampleImageClearRuntimeShape(); - VulkanHarness vulkan; + vulkan.CheckSchedulerTimeline(); + vulkan.CheckStreamBufferRing(); vulkan.CheckCommandPoolGrowth(); vulkan.CheckGpuTilerCpuParity(); - vulkan.CheckQueryRegionImageClassification(); - vulkan.CheckMutableStorageSrgbView(); - vulkan.CheckMutableRenderTargetBgraStorageView(); - vulkan.CheckRenderTargetViewCache(); - vulkan.CheckDepthTargetSampledViewCache(); - vulkan.CheckVideoOutSampledViewCache(); +#if KYTY_PLATFORM == KYTY_PLATFORM_WINDOWS + vulkan.CheckRenderExecutorStencilBindingDiscovery(); + vulkan.CheckUnifiedTextureCacheFlow(); + vulkan.CheckBgra16Readback(); + vulkan.CheckBufferCacheDirtyGarbageCollection(); +#endif + vulkan.CheckUnifiedImageViewCache(); const auto tests = MakeCases(); const auto graphics_tests = MakeGraphicsCases(); CheckOpcodeCoverage(tests, graphics_tests); @@ -14408,6 +17210,7 @@ int main(int argc, char **argv) { for (const auto &test : graphics_tests) { RunGraphicsCase(&vulkan, test); } + vulkan.CheckGpuCommandLane(); std::printf("ShaderRecompilerComputeTests: all cases passed\n"); return 0; } diff --git a/tests/shaderCfgTests.cpp b/tests/shaderCfgTests.cpp index 8d7e2561..a17261be 100644 --- a/tests/shaderCfgTests.cpp +++ b/tests/shaderCfgTests.cpp @@ -5,6 +5,7 @@ #include "graphics/guest_gpu/hardwareContext.h" #include "graphics/guest_gpu/pm4.h" #include "graphics/host_gpu/objects/textureCommon.h" +#include "graphics/host_gpu/renderer/imageView.h" #include "graphics/host_gpu/renderer/shaderResourceBarrier.h" #include "graphics/host_gpu/renderer/shaderSubgroup.h" #include "graphics/shader/recompiler/ExecMask.h" @@ -446,23 +447,9 @@ void TestNativeShaderResourceDependencies() { Check(writes == std::vector({{0x1000, 48}, {0x2000, 64}}), "graphics/compute write collector lost or invented a storage-buffer range"); - VulkanImage image {VulkanImageType::StorageTexture}; - image.image = reinterpret_cast(uintptr_t {1}); - image.layout = vk::ImageLayout::eGeneral; - image.layers = 3; - const auto image_barrier = MakeStorageImageDependency(image, true, true); - Check(image_barrier.oldLayout == vk::ImageLayout::eGeneral && - image_barrier.newLayout == vk::ImageLayout::eGeneral && - (image_barrier.srcAccessMask & vk::AccessFlagBits::eMemoryWrite) && - (image_barrier.dstAccessMask & vk::AccessFlagBits::eShaderRead) && - (image_barrier.dstAccessMask & vk::AccessFlagBits::eShaderWrite) && - image_barrier.subresourceRange.levelCount == VK_REMAINING_MIP_LEVELS && - image_barrier.subresourceRange.layerCount == image.layers, - "GENERAL storage-image dependency lacks a complete memory barrier"); - VulkanBuffer buffer; buffer.buffer = reinterpret_cast(uintptr_t {1}); - const auto gds_barrier = MakeGdsDependency(buffer); + const auto gds_barrier = MakeGdsDependency(buffer.buffer); Check((gds_barrier.srcAccessMask & vk::AccessFlagBits::eHostWrite) && (gds_barrier.srcAccessMask & vk::AccessFlagBits::eTransferWrite) && (gds_barrier.srcAccessMask & vk::AccessFlagBits::eShaderWrite) && @@ -472,6 +459,60 @@ void TestNativeShaderResourceDependencies() { "GDS dependency does not order host/transfer/shader writes"); } +void TestNormalizedImageContracts() { + ImageInfo container {}; + container.data = {0x10000, 0x15000}; + container.pixel_format = vk::Format::eR8G8B8A8Unorm; + container.guest_format = + Prospero::GpuEnumValue(Prospero::BufferFormat::k8_8_8_8UNorm); + container.type = Prospero::ImageType::kColor2D; + container.extent = {64, 64, 1}; + container.resources = {3, 4}; + container.pitch = 64; + container.bytes_per_block = 4; + container.samples = 1; + container.tile_mode = Prospero::GpuEnumValue(Prospero::TileMode::kStandard64KB); + container.mip_layout[0] = {0, 0x10000, 64, 64}; + container.mip_layout[1] = {0x10000, 0x4000, 32, 32}; + container.mip_layout[2] = {0x14000, 0x1000, 16, 16}; + + ImageInfo subresource = container; + subresource.data = {0x22000, 0x1000}; + subresource.extent = {32, 32, 1}; + subresource.resources = {1, 1}; + subresource.pitch = 32; + subresource.mip_layout = {}; + subresource.mip_layout[0] = {0, 0x1000, 32, 32}; + + Check(container.BlockExtent() == vk::Extent2D {64, 64}, + "normalized image block extent changed"); + Check(subresource.IsCompatible(container), "normalized compatible image was rejected"); + Check(subresource.MipOf(container) == 1, "normalized mip lookup missed a subresource"); + Check(subresource.SliceOf(container, 1) == 2, + "normalized slice lookup missed a subresource"); + + auto incompatible = subresource; + incompatible.samples = 2; + Check(!incompatible.IsCompatible(container) && incompatible.MipOf(container) == -1, + "sample-count mismatch was accepted as a compatible image"); + + auto compressed = container; + compressed.guest_format = + Prospero::GpuEnumValue(Prospero::BufferFormat::kBc3UNorm); + compressed.pitch = 128; + compressed.extent.height = 64; + Check(compressed.BlockExtent() == vk::Extent2D {32, 16}, + "block-compressed extent was not expressed in blocks"); + + Check(ImageViewOps::FormatsCompatible(vk::Format::eR8G8B8A8Unorm, + vk::Format::eR8G8B8A8Uint) && + !ImageViewOps::FormatsCompatible(vk::Format::eD32Sfloat, + vk::Format::eR32Sfloat) && + ImageViewOps::FormatsCompatible(vk::Format::eBc3UnormBlock, + vk::Format::eR32G32B32A32Uint), + "Vulkan image-view compatibility classes diverged from production"); +} + void TestNativeSubgroupPolicy() { GraphicContext context; context.subgroup_size = 32; @@ -6743,6 +6784,7 @@ int main() { TestResourceDescriptorClassification(); TestNativeShaderResourceDependencies(); + TestNormalizedImageContracts(); TestNativeSubgroupPolicy(); TestNewShaderRecompilerSMovB32(); TestNewShaderRecompilerSoppMarkers();