diff --git a/src/graphics/guest_gpu/command_processor/commandProcessor.h b/src/graphics/guest_gpu/command_processor/commandProcessor.h index 90280cb7..c85894ab 100644 --- a/src/graphics/guest_gpu/command_processor/commandProcessor.h +++ b/src/graphics/guest_gpu/command_processor/commandProcessor.h @@ -12,8 +12,6 @@ namespace Libs::Graphics { -class CommandProcessorGuestAccessScope; - inline constexpr uint32_t AcquireGcrGl2Writeback = 1u << 15u; bool TestWaitRegMemValue(uint64_t value, uint64_t ref, uint64_t mask, uint32_t func); @@ -72,6 +70,21 @@ public: } } + void SubmitForReadback() { + if (!Active()) { + return; + } + SubmitCurrent(); + } + + void ResumeAfterReadback() { + if (!Active()) { + return; + } + Current()->WaitForFenceAndReset(); + Current()->Begin(); + } + private: CommandBuffer* SubmitCurrent() { auto* submitted = Current(); @@ -107,22 +120,26 @@ public: void Reset(); - void BufferInit(); - void BufferFlush(); - void BufferFlushAndWait(); - void BufferWait(); - [[nodiscard]] int BeginReadbackTransaction(bool mutex_preowned) { - if (!mutex_preowned) { - m_mutex.Lock(); + void BufferInit(); + void BufferFlush(); + void BufferFlushAndWait(); + void BufferWait(); + void BeginReadbackTransaction() { + m_mutex.Lock(); + if (m_readback_active) { + EXIT("nested command-processor readback transaction\n"); } - m_scheduler.Flush(); - m_scheduler.WaitAll(); - return m_scheduler.Queue(); + m_readback_active = true; + m_readback_finished = false; } - void EndReadbackTransaction(bool mutex_preowned) { - if (!mutex_preowned) { - m_mutex.Unlock(); + void FinishReadbackTransaction(); + void EndReadbackTransaction() { + if (!m_readback_active) { + EXIT("command-processor readback transaction is not active\n"); } + m_readback_active = false; + m_readback_finished = false; + m_mutex.Unlock(); } void RunLock() { m_run_mutex.Lock(); } @@ -164,7 +181,6 @@ public: uint32_t value); void PrepareCpuFlip(); void SynchronizeGpu(); - void SynchronizeGpuRange(uint64_t vaddr, uint64_t size); void MemoryBarrier(); void TriggerEopEventAtEndOfPipe(uint32_t interrupt_context_id); void RenderTextureBarrier(uint64_t vaddr, uint64_t size); @@ -182,9 +198,6 @@ public: void PopMarker() {} void PrefetchL2(void* addr, uint32_t size) {} - void ClearGds(uint64_t dw_offset, uint32_t dw_num, uint32_t clear_value); - void ReadGds(uint32_t* dst, uint32_t dw_offset, uint32_t dw_size); - void ResetDeCe(); void WaitCe(); void WaitDeDiff(uint32_t diff); @@ -208,7 +221,7 @@ public: void Run(uint32_t* data, uint32_t num_dw); - void SetQueue(int queue) { m_scheduler.SetQueue(queue); } + void SetQueue(int queue); [[nodiscard]] int GetQueue() const { return m_scheduler.Queue(); } [[nodiscard]] const FlipInfo& GetFlip() const { return m_flip; } @@ -218,8 +231,6 @@ public: void SetSubmitId(uint64_t submit_id) { m_submit_id = submit_id; } private: - friend class CommandProcessorGuestAccessScope; - struct Counter { Common::Mutex mutex; Common::CondVar cond_var; @@ -231,6 +242,7 @@ private: uint32_t cache_action, uint32_t event_index, uint32_t event_write_source, void* dst_gpu_addr, T value, uint32_t interrupt_selector, uint32_t interrupt_context_id); + void FinishCommandProcessors(); CommandBuffer* CurrentBuffer() { return m_scheduler.Current(); } void CheckBuffer() const { m_scheduler.CheckActive(); } @@ -250,8 +262,11 @@ private: uint64_t m_dispatch_indirect_args_base_addr = 0; uint32_t m_num_instances = 1; - Common::Mutex m_mutex; - Common::Mutex m_run_mutex; + inline static Common::Mutex m_mutex; + inline static std::array m_processors {}; + inline static bool m_readback_active = false; + inline static bool m_readback_finished = false; + Common::Mutex m_run_mutex; CommandScheduler m_scheduler; diff --git a/src/graphics/guest_gpu/command_processor/pm4Handlers.cpp b/src/graphics/guest_gpu/command_processor/pm4Handlers.cpp index 5e5d3f59..fec2b9a5 100644 --- a/src/graphics/guest_gpu/command_processor/pm4Handlers.cpp +++ b/src/graphics/guest_gpu/command_processor/pm4Handlers.cpp @@ -2233,7 +2233,7 @@ KYTY_CP_OP_PARSER(CpOpCopyData) { cp->WriteReferenceClock(dst, num_bytes); return 5; } - const auto dma_src = CopyDataSrcToDma(src_sel); + const auto dma_src = CopyDataSrcToDma(src_sel); if (dma_src == 2 && num_bytes == 8) { EXIT("unsupported 64-bit immediate copyData\n"); } @@ -2636,7 +2636,6 @@ KYTY_CP_OP_PARSER(CpOpIndirectCxRegs) { if (indirect_buffer == nullptr) { EXIT("indirect CX registers have null address, num_regs = %" PRIu32 "\n", indirect_num_dw); } - for (uint32_t i = 0; i < indirect_num_dw; i++, indirect_buffer += 2) { auto cmd_offset = indirect_buffer[0]; auto value = indirect_buffer[1]; @@ -2695,13 +2694,14 @@ KYTY_CP_OP_PARSER(CpOpIndirectShRegs) { if (indirect_buffer == nullptr) { EXIT("indirect SH registers have null address, num_regs = %" PRIu32 "\n", indirect_num_dw); } + const auto indirect_address = reinterpret_cast(indirect_buffer); for (uint32_t i = 0; i < indirect_num_dw; i++, indirect_buffer += 2) { auto raw_cmd_offset = indirect_buffer[0]; auto cmd_offset = NormalizeRegisterOffset(raw_cmd_offset); auto value = indirect_buffer[1]; - //Not sure if this is correct + // Not sure if this is correct if (raw_cmd_offset != cmd_offset) { LOGF_COLOR(Log::Color::Red, "\t temporary: normalized indirect SH register offset 0x%08" PRIx32 @@ -2709,7 +2709,7 @@ KYTY_CP_OP_PARSER(CpOpIndirectShRegs) { raw_cmd_offset, cmd_offset); } - //Not sure if this is correct + // Not sure if this is correct if (cmd_offset == Pm4::SH_NOP || raw_cmd_offset == 0xffffffffu) { continue; } @@ -2725,8 +2725,7 @@ KYTY_CP_OP_PARSER(CpOpIndirectShRegs) { if (pfunc == nullptr) { LOGF("unknown indirect SH register: index=%" PRIu32 "/%" PRIu32 ", regs=0x%016" PRIx64 ", offset=0x%08" PRIx32 ", value=0x%08" PRIx32 "\n", - i, indirect_num_dw, reinterpret_cast(indirect_buffer - i * 2), - cmd_offset, value); + i, indirect_num_dw, indirect_address, cmd_offset, value); auto* dump_regs = indirect_buffer - i * 2; for (uint32_t j = 0; j < indirect_num_dw && j < 16; j++) { LOGF("\t sh_indirect[%" PRIu32 "] offset=0x%08" PRIx32 ", value=0x%08" PRIx32 "\n", @@ -2758,13 +2757,12 @@ KYTY_CP_OP_PARSER(CpOpIndirectUcRegs) { if (indirect_buffer == nullptr) { EXIT("indirect UC registers have null address, num_regs = %" PRIu32 "\n", indirect_num_dw); } - for (uint32_t i = 0; i < indirect_num_dw; i++, indirect_buffer += 2) { auto raw_cmd_offset = indirect_buffer[0]; auto cmd_offset = NormalizeRegisterOffset(raw_cmd_offset); auto value = indirect_buffer[1]; - //Not sure if this is correct + // Not sure if this is correct if (raw_cmd_offset != cmd_offset) { LOGF_COLOR(Log::Color::Red, "\t temporary: normalized indirect UC register offset 0x%08" PRIx32 diff --git a/src/graphics/guest_gpu/graphicsRun.cpp b/src/graphics/guest_gpu/graphicsRun.cpp index c688a552..28652815 100644 --- a/src/graphics/guest_gpu/graphicsRun.cpp +++ b/src/graphics/guest_gpu/graphicsRun.cpp @@ -33,19 +33,9 @@ namespace Libs::Graphics { -enum class CommandProcessorGuestAccess : uint32_t { - None, - WaitPoll, - DirectFence, -}; - -static thread_local CommandProcessor* g_current_run_cp = nullptr; -static thread_local uint32_t g_submission_pause_depth = 0; -static thread_local bool g_gpu_mutex_owned = false; -static thread_local bool g_cp_readback_owned = false; -static thread_local bool g_cp_readback_preowned = false; -static thread_local CommandProcessorGuestAccess g_guest_memory_access = - CommandProcessorGuestAccess::None; +static thread_local CommandProcessor* g_current_run_cp = nullptr; +static thread_local uint32_t g_submission_pause_depth = 0; +static thread_local bool g_gpu_mutex_owned = false; class GpuMutexLock final { public: @@ -68,54 +58,13 @@ private: Common::Mutex& m_mutex; }; -class CommandProcessorGuestAccessScope final { -public: - CommandProcessorGuestAccessScope(CommandProcessorGuestAccess access, uint64_t vaddr, - uint64_t size) { - if (access == CommandProcessorGuestAccess::None || - g_guest_memory_access != CommandProcessorGuestAccess::None || - g_current_run_cp == nullptr) { - EXIT("invalid nested command-processor guest-memory access\n"); - } - if (g_render_ctx == nullptr) { - EXIT("command-processor guest-memory access has no render context\n"); - } - m_cp = g_current_run_cp; - if (access == CommandProcessorGuestAccess::WaitPoll) { - // Preserve serialized scheduler semantics: a poll owns the command processor before it - // enters the shared buffer/texture transaction, and releases both before yielding. - m_cp->m_mutex.Lock(); - m_cp_mutex_owned = true; - } - m_resource_lock = - g_render_ctx->GetGpuResources()->LockCommandProcessorGuestAccess(vaddr, size); - g_guest_memory_access = access; - } - ~CommandProcessorGuestAccessScope() { - if (g_guest_memory_access == CommandProcessorGuestAccess::None || - !m_resource_lock.owns_lock() || m_cp == nullptr) { - EXIT("command-processor guest-memory access scope is not active\n"); - } - g_guest_memory_access = CommandProcessorGuestAccess::None; - m_resource_lock.unlock(); - if (m_cp_mutex_owned) { - m_cp->m_mutex.Unlock(); - } - } - -private: - CommandProcessor* m_cp = nullptr; - bool m_cp_mutex_owned = false; - std::unique_lock m_resource_lock; - KYTY_CLASS_NO_COPY(CommandProcessorGuestAccessScope); -}; - struct OwnedCmdBuffer { std::vector storage; uint32_t* data = nullptr; uint32_t num_dw = 0; OwnedCmdBuffer() = default; + OwnedCmdBuffer(const uint32_t* src, uint32_t count) { Assign(src, count); } OwnedCmdBuffer(const OwnedCmdBuffer& other): storage(other.storage), num_dw(other.num_dw) { data = (storage.empty() ? other.data : storage.data()); @@ -165,9 +114,8 @@ public: KYTY_CLASS_NO_COPY(GraphicsRing); - void Submit(uint32_t* cmd_draw_buffer, uint32_t num_draw_dw, uint32_t* cmd_const_buffer, - uint32_t num_const_dw, int handle, int index, int flip_mode, int64_t flip_arg, - bool trigger_agc_interrupt_on_done); + void Submit(OwnedCmdBuffer draw_buffer, OwnedCmdBuffer const_buffer, int handle, int index, + int flip_mode, int64_t flip_arg, bool trigger_agc_interrupt_on_done); void SubmitFlipPreparation(); void Done(); void WaitForIdle(); @@ -217,7 +165,7 @@ public: KYTY_CLASS_NO_COPY(ComputeRing); - void Submit(uint32_t* cmd_buffer, uint32_t num_dw, bool trigger_agc_interrupt_on_done); + void Submit(OwnedCmdBuffer buffer, bool trigger_agc_interrupt_on_done); void Done(); void WaitForIdle(); bool IsIdle(); @@ -311,15 +259,18 @@ void GraphicsRunInit() { void Gpu::Submit(uint32_t* cmd_draw_buffer, uint32_t num_draw_dw, uint32_t* cmd_const_buffer, uint32_t num_const_dw, bool trigger_agc_interrupt_on_done) { - GpuMutexLock lock(m_mutex); + OwnedCmdBuffer draw_buffer(cmd_draw_buffer, num_draw_dw); + OwnedCmdBuffer const_buffer(cmd_const_buffer, num_const_dw); + GpuMutexLock lock(m_mutex); - m_gfx_ring->Submit(cmd_draw_buffer, num_draw_dw, cmd_const_buffer, num_const_dw, 0, 0, 0, 0, + m_gfx_ring->Submit(std::move(draw_buffer), std::move(const_buffer), 0, 0, 0, 0, trigger_agc_interrupt_on_done); } void Gpu::SubmitCompute(uint32_t queue, uint32_t* cmd_buffer, uint32_t num_dw, bool trigger_agc_interrupt_on_done) { - GpuMutexLock lock(m_mutex); + OwnedCmdBuffer buffer(cmd_buffer, num_dw); + GpuMutexLock lock(m_mutex); constexpr uint32_t compute_queue_base = 0x20u; constexpr uint32_t compute_queue_num = 7u * 8u; @@ -336,7 +287,7 @@ void Gpu::SubmitCompute(uint32_t queue, uint32_t* cmd_buffer, uint32_t num_dw, auto* ring = GetRing(ring_id); - ring->Submit(cmd_buffer, num_dw, trigger_agc_interrupt_on_done); + ring->Submit(std::move(buffer), trigger_agc_interrupt_on_done); } void Gpu::SubmitFlipPreparation() { @@ -454,6 +405,45 @@ ComputeRing* Gpu::GetRing(uint32_t ring_id) { return m_compute_ring[v]; } +void CommandProcessor::SetQueue(int queue) { + Common::LockGuard lock(m_mutex); + if (queue < 0 || queue >= GraphicContext::QUEUES_NUM || + (m_processors[queue] != nullptr && m_processors[queue] != this)) { + EXIT("invalid command-processor queue registration: queue=%d owner=%p\n", queue, + static_cast(m_processors[queue])); + } + m_scheduler.SetQueue(queue); + m_processors[queue] = this; +} + +void CommandProcessor::FinishReadbackTransaction() { + if (GraphicsRunCurrentCommandProcessor() == nullptr || !m_readback_active) { + EXIT("GPU readback finish requires a command-processor thread\n"); + } + if (m_readback_finished) { + return; + } + FinishCommandProcessors(); + m_readback_finished = true; +} + +void CommandProcessor::FinishCommandProcessors() { + std::array processors {}; + uint32_t processor_count = 0; + for (auto* processor: m_processors) { + if (processor == nullptr || + std::find(processors.begin(), processors.begin() + processor_count, processor) != + processors.begin() + processor_count) { + continue; + } + processors[processor_count++] = processor; + processor->m_scheduler.SubmitForReadback(); + } + for (uint32_t i = 0; i < processor_count; i++) { + processors[i]->m_scheduler.ResumeAfterReadback(); + } +} + void CommandProcessor::Reset() { BufferWait(); @@ -598,13 +588,6 @@ bool TestWaitRegMemValue(uint64_t value, uint64_t ref, uint64_t mask, uint32_t f return false; } -template -static T LoadGuestWaitValue(const T* addr) { - CommandProcessorGuestAccessScope guest_access(CommandProcessorGuestAccess::WaitPoll, - reinterpret_cast(addr), sizeof(*addr)); - return *addr; -} - static void YieldCommandProcessorWait(uint32_t poll_interval_cycles) noexcept { // PS5 specifies GPU poll cycles, not host microseconds. Yield at the scheduler boundary now; // a future resumable guest GPU scheduler can replace this one function without touching polling @@ -625,12 +608,9 @@ void CommandProcessor::WaitRegMem(uint32_t func, const T* addr, T ref, T mask, u const auto log_width = static_cast(sizeof(T) * 2u); const auto bits = static_cast(sizeof(T) * 8u); - if (!TestWaitRegMemValue(LoadGuestWaitValue(addr), ref, mask, func)) { - SynchronizeGpuRange(addr_value, sizeof(*addr)); - } uint64_t spin_count = 0; for (;;) { - const auto value = LoadGuestWaitValue(addr); + const auto value = *addr; if (TestWaitRegMemValue(value, ref, mask, func)) { break; } @@ -670,10 +650,7 @@ void CommandProcessor::WriteData(uint32_t* dst, const uint32_t* src, uint32_t dw return; } - const auto size = static_cast(dw_num) * sizeof(uint32_t); - CommandProcessorGuestAccessScope guest_access(CommandProcessorGuestAccess::DirectFence, - reinterpret_cast(dst), size); - memcpy(dst, src, static_cast(size)); + memcpy(dst, src, static_cast(dw_num) * sizeof(uint32_t)); } void CommandProcessor::WriteReferenceClock(uint64_t dst_address, uint32_t num_bytes) { @@ -684,11 +661,8 @@ void CommandProcessor::WriteReferenceClock(uint64_t dst_address, uint32_t num_by num_bytes); } const auto value = GraphicsRenderReadReferenceClock(); - CommandProcessorGuestAccessScope guest_access(CommandProcessorGuestAccess::DirectFence, - dst_address, num_bytes); std::memcpy(reinterpret_cast(dst_address), &value, num_bytes); - LOGF("\t copy_data reference clock: dst=0x%016" PRIx64 " value=0x%016" PRIx64 - " size=%u\n", + LOGF("\t copy_data reference clock: dst=0x%016" PRIx64 " value=0x%016" PRIx64 " size=%u\n", dst_address, value, num_bytes); } @@ -729,9 +703,9 @@ void CommandProcessor::DmaData(uint8_t engine, uint8_t dst_sel, uint8_t dst_cach EXIT("unsupported dmaData source selector 0x%02" PRIx8 "\n", src_sel); } -void GraphicsRing::Submit(uint32_t* cmd_draw_buffer, uint32_t num_draw_dw, - uint32_t* cmd_const_buffer, uint32_t num_const_dw, int handle, int index, - int flip_mode, int64_t flip_arg, bool trigger_agc_interrupt_on_done) { +void GraphicsRing::Submit(OwnedCmdBuffer draw_buffer, OwnedCmdBuffer const_buffer, int handle, + int index, int flip_mode, int64_t flip_arg, + bool trigger_agc_interrupt_on_done) { EXIT_IF(m_cp == nullptr); Common::LockGuard lock(m_mutex); @@ -748,9 +722,9 @@ void GraphicsRing::Submit(uint32_t* cmd_draw_buffer, uint32_t num_draw_dw, m_cp->Reset(); } - auto& buf = m_cmd_batches.emplace_back(); - buf.draw_buffer.Assign(cmd_draw_buffer, num_draw_dw); - buf.const_buffer.Assign(cmd_const_buffer, num_const_dw); + auto& buf = m_cmd_batches.emplace_back(); + buf.draw_buffer = std::move(draw_buffer); + buf.const_buffer = std::move(const_buffer); buf.flip.handle = handle; buf.flip.index = index; buf.flip.flip_mode = flip_mode; @@ -942,12 +916,11 @@ void ComputeRing::ThreadRun(void* data) { } } -void ComputeRing::Submit(uint32_t* cmd_buffer, uint32_t num_dw, - bool trigger_agc_interrupt_on_done) { +void ComputeRing::Submit(OwnedCmdBuffer buffer, bool trigger_agc_interrupt_on_done) { Common::LockGuard lock(m_mutex); - EXIT_IF(cmd_buffer == nullptr); - EXIT_IF(num_dw == 0); + EXIT_IF(buffer.data == nullptr); + EXIT_IF(buffer.num_dw == 0); WindowWaitForGraphicInitialized(); GraphicsRenderCreateContext(); @@ -957,7 +930,7 @@ void ComputeRing::Submit(uint32_t* cmd_buffer, uint32_t num_dw, } DirectBatch batch {}; - batch.buffer.Assign(cmd_buffer, num_dw); + batch.buffer = std::move(buffer); batch.trigger_agc_interrupt_on_done = trigger_agc_interrupt_on_done; m_direct_batches.push_back(std::move(batch)); @@ -1229,35 +1202,38 @@ void CommandProcessor::DrawIndirect(uint32_t data_offset, uint32_t draw_initiato EXIT_NOT_IMPLEMENTED((draw_initiator & ~0x20u) != 2u); EXIT_NOT_IMPLEMENTED(m_draw_indirect_args_base_addr == 0); - const auto args_addr = m_draw_indirect_args_base_addr + data_offset; + const auto* args_addr = + reinterpret_cast(m_draw_indirect_args_base_addr + data_offset); if (!indexed) { - auto* args = reinterpret_cast(args_addr); - if (args->instance_count != 1u || args->start_vertex_location != 0u || - args->start_instance_location != 0u) { + DrawIndirectArgs args {}; + std::memcpy(&args, args_addr, sizeof(args)); + if (args.instance_count != 1u || args.start_vertex_location != 0u || + args.start_instance_location != 0u) { static std::atomic log_count {0}; if (log_count.fetch_add(1) < 64) { LOGF("\t warning: partial DrawIndirect args: vertex_count=%" PRIu32 ", instance_count=%" PRIu32 ", start_vertex=%" PRIu32 ", start_instance=%" PRIu32 "\n", - args->vertex_count_per_instance, args->instance_count, - args->start_vertex_location, args->start_instance_location); + args.vertex_count_per_instance, args.instance_count, + args.start_vertex_location, args.start_instance_location); } } - DrawIndexAuto(args->vertex_count_per_instance, 0, 0, args->instance_count, - args->start_vertex_location, args->start_instance_location); + DrawIndexAuto(args.vertex_count_per_instance, 0, 0, args.instance_count, + args.start_vertex_location, args.start_instance_location); return; } - auto* args = reinterpret_cast(args_addr); - if (args->base_vertex_location != 0u || args->start_instance_location != 0u) { + DrawIndexedIndirectArgs args {}; + std::memcpy(&args, args_addr, sizeof(args)); + if (args.base_vertex_location != 0u || args.start_instance_location != 0u) { static std::atomic log_count {0}; if (log_count.fetch_add(1) < 64) { LOGF("\t warning: partial DrawIndexIndirect args: index_count=%" PRIu32 ", instance_count=%" PRIu32 ", start_index=%" PRIu32 ", base_vertex=%" PRIu32 ", start_instance=%" PRIu32 "\n", - args->index_count_per_instance, args->instance_count, args->start_index_location, - args->base_vertex_location, args->start_instance_location); + args.index_count_per_instance, args.instance_count, args.start_index_location, + args.base_vertex_location, args.start_instance_location); } } @@ -1270,22 +1246,22 @@ void CommandProcessor::DrawIndirect(uint32_t data_offset, uint32_t draw_initiato } auto* index_addr = reinterpret_cast( - m_index_base_addr + static_cast(args->start_index_location) * index_size); + m_index_base_addr + static_cast(args.start_index_location) * index_size); const uint32_t index_count = - (m_index_buffer_size != 0 ? std::min(args->index_count_per_instance, m_index_buffer_size) - : args->index_count_per_instance); - if (GraphicsRunDebugDumpEnabled() && index_count != args->index_count_per_instance) { + (m_index_buffer_size != 0 ? std::min(args.index_count_per_instance, m_index_buffer_size) + : args.index_count_per_instance); + if (GraphicsRunDebugDumpEnabled() && index_count != args.index_count_per_instance) { static std::atomic log_count {0}; if (log_count.fetch_add(1, std::memory_order_relaxed) < 64) { LOGF("\t DrawIndexIndirect: clamped index_count from %" PRIu32 " to %" PRIu32 " using INDEX_BUFFER_SIZE\n", - args->index_count_per_instance, index_count); + args.index_count_per_instance, index_count); } } - DrawIndex(index_count, index_addr, 0, 1, args->instance_count, nullptr, 0, - static_cast(args->base_vertex_location), args->start_instance_location); + DrawIndex(index_count, index_addr, 0, 1, args.instance_count, nullptr, 0, + static_cast(args.base_vertex_location), args.start_instance_location); } void CommandProcessor::DrawIndirectMulti(uint32_t data_offset, uint32_t max_count_or_count, @@ -1503,18 +1479,6 @@ void CommandProcessor::DrawIndexAuto(uint32_t index_count, uint32_t flags, first_instance); } -void CommandProcessor::ClearGds(uint64_t dw_offset, uint32_t dw_num, uint32_t clear_value) { - Common::LockGuard lock(m_mutex); - - GraphicsRenderClearGds(dw_offset, dw_num, clear_value); -} - -void CommandProcessor::ReadGds(uint32_t* dst, uint32_t dw_offset, uint32_t dw_size) { - Common::LockGuard lock(m_mutex); - - GraphicsRenderReadGds(dst, dw_offset, dw_size); -} - void CommandProcessor::WaitFlipDone(uint32_t video_out_handle, uint32_t display_buffer_index) { BufferFlush(); @@ -1569,11 +1533,9 @@ void CommandProcessor::WriteAtEndOfPipe(uint32_t cache_policy, uint32_t event_wr } auto write32 = [&](bool with_writeback) { - auto* dst = static_cast(dst_gpu_addr); - auto data = static_cast(value); - CommandProcessorGuestAccessScope guest_access(CommandProcessorGuestAccess::DirectFence, - reinterpret_cast(dst), - sizeof(*dst)); + auto* dst = static_cast(dst_gpu_addr); + auto data = static_cast(value); + std::memcpy(dst, &data, sizeof(data)); if (with_interrupt) { if (with_writeback) { @@ -1594,8 +1556,10 @@ void CommandProcessor::WriteAtEndOfPipe(uint32_t cache_policy, uint32_t event_wr case 0x01: if constexpr (sizeof(T) == sizeof(uint32_t)) { if (eop_event_type == 0x2f && cache_action == 0x00 && event_index == 0x06) { - GraphicsRenderWriteAtEndOfPipeGds32(m_submit_id, CurrentBuffer(), - static_cast(dst_gpu_addr), + auto* dst = static_cast(dst_gpu_addr); + SynchronizeGpu(); + GraphicsRenderReadGds(dst, value & 0xffffu, value >> 16u); + GraphicsRenderWriteAtEndOfPipeGds32(m_submit_id, CurrentBuffer(), dst, value & 0xffffu, value >> 16u); return; } @@ -1615,10 +1579,8 @@ void CommandProcessor::WriteAtEndOfPipe(uint32_t cache_policy, uint32_t event_wr } } else { auto write64 = [&](bool with_writeback) { - auto* dst = static_cast(dst_gpu_addr); - CommandProcessorGuestAccessScope guest_access( - CommandProcessorGuestAccess::DirectFence, reinterpret_cast(dst), - sizeof(*dst)); + auto* dst = static_cast(dst_gpu_addr); + std::memcpy(dst, &value, sizeof(value)); if (with_interrupt) { if (with_writeback) { @@ -1699,16 +1661,16 @@ void CommandProcessor::WriteAtEndOfPipe(uint32_t cache_policy, uint32_t event_wr break; case 0x04: if constexpr (sizeof(T) == sizeof(uint64_t)) { - CommandProcessorGuestAccessScope guest_access( - CommandProcessorGuestAccess::DirectFence, - reinterpret_cast(dst_gpu_addr), sizeof(uint64_t)); + const auto clock = GraphicsRenderReadReferenceClock(); + std::memcpy(dst_gpu_addr, &clock, sizeof(clock)); switch (cache_action) { case 0x00: if (((eop_event_type == 0x04 && event_index == 0x05) || (eop_event_type == 0x28 && event_index == 0x00)) && !with_interrupt) { GraphicsRenderWriteAtEndOfPipeClockCounter( - m_submit_id, CurrentBuffer(), static_cast(dst_gpu_addr)); + m_submit_id, CurrentBuffer(), static_cast(dst_gpu_addr), + clock); return; } break; @@ -1718,7 +1680,8 @@ void CommandProcessor::WriteAtEndOfPipe(uint32_t cache_policy, uint32_t event_wr (eop_event_type == 0x28 && event_index == 0x00)) && !with_interrupt) { GraphicsRenderWriteAtEndOfPipeClockCounterWithWriteBack( - m_submit_id, CurrentBuffer(), static_cast(dst_gpu_addr)); + m_submit_id, CurrentBuffer(), static_cast(dst_gpu_addr), + clock); return; } break; @@ -1850,8 +1813,8 @@ void CommandProcessor::Flip() { } auto* command = CurrentBuffer(); - auto request = GraphicsRenderPrepareDisplayBufferFlip( - command, m_flip.handle, m_flip.index, m_flip.flip_mode, m_flip.flip_arg); + auto request = GraphicsRenderPrepareDisplayBufferFlip(command, m_flip.handle, m_flip.index, + m_flip.flip_mode, m_flip.flip_arg); GraphicsRenderWriteAtEndOfPipeOnlyFlip(m_submit_id, command, m_flip.handle, m_flip.index, m_flip.flip_mode, m_flip.flip_arg, request); m_scheduler.Flush(); @@ -1869,12 +1832,10 @@ void CommandProcessor::Flip(void* dst_gpu_addr, uint32_t value) { reinterpret_cast(dst_gpu_addr), value); } + std::memcpy(dst_gpu_addr, &value, sizeof(value)); auto* command = CurrentBuffer(); - auto request = GraphicsRenderPrepareDisplayBufferFlip( - command, m_flip.handle, m_flip.index, m_flip.flip_mode, m_flip.flip_arg); - CommandProcessorGuestAccessScope guest_access(CommandProcessorGuestAccess::DirectFence, - reinterpret_cast(dst_gpu_addr), - sizeof(uint32_t)); + auto request = GraphicsRenderPrepareDisplayBufferFlip(command, m_flip.handle, m_flip.index, + m_flip.flip_mode, m_flip.flip_arg); GraphicsRenderWriteAtEndOfPipeWithFlip32( m_submit_id, command, static_cast(dst_gpu_addr), value, m_flip.handle, m_flip.index, m_flip.flip_mode, m_flip.flip_arg, request); @@ -1899,12 +1860,10 @@ void CommandProcessor::FlipWithInterrupt(uint32_t eop_event_type, uint32_t cache if (eop_event_type != 0x00000004 || cache_action != 0x00000038) { EXIT("unknown event type\n"); } + std::memcpy(dst_gpu_addr, &value, sizeof(value)); auto* command = CurrentBuffer(); auto request = GraphicsRenderPrepareDisplayBufferFlip(command, m_flip.handle, m_flip.index, m_flip.flip_mode, m_flip.flip_arg); - CommandProcessorGuestAccessScope guest_access(CommandProcessorGuestAccess::DirectFence, - reinterpret_cast(dst_gpu_addr), - sizeof(uint32_t)); GraphicsRenderWriteAtEndOfPipeWithInterruptWriteBackFlip32( m_submit_id, command, static_cast(dst_gpu_addr), value, m_flip.handle, m_flip.index, m_flip.flip_mode, m_flip.flip_arg, request); @@ -1929,15 +1888,8 @@ void CommandProcessor::PrepareCpuFlip() { } void CommandProcessor::SynchronizeGpu() { - BufferFlush(); - BufferWait(); -} - -void CommandProcessor::SynchronizeGpuRange(uint64_t vaddr, uint64_t size) { - if (vaddr == 0 || size == 0 || size > UINT64_MAX - vaddr) { - EXIT("invalid GPU synchronization range\n"); - } - SynchronizeGpu(); + Common::LockGuard lock(m_mutex); + FinishCommandProcessors(); } void GraphicsRunSubmit(uint32_t* cmd_draw_buffer, uint32_t num_draw_dw, uint32_t* cmd_const_buffer, @@ -2002,14 +1954,15 @@ bool GraphicsRunIsCommandProcessorThread() noexcept { return g_current_run_cp != nullptr; } -bool GraphicsRunIsGuestMemoryPoll() noexcept { - return g_current_run_cp != nullptr && - g_guest_memory_access == CommandProcessorGuestAccess::WaitPoll; +CommandProcessor* GraphicsRunCurrentCommandProcessor() noexcept { + return g_current_run_cp; } -bool GraphicsRunIsCommandProcessorGuestAccess() noexcept { - return g_current_run_cp != nullptr && - g_guest_memory_access != CommandProcessorGuestAccess::None; +void GraphicsRunFinishCommandProcessors() { + if (g_current_run_cp == nullptr) { + EXIT("GPU readback finish requires a command-processor thread\n"); + } + g_current_run_cp->FinishReadbackTransaction(); } bool GraphicsRunSubmissionLockHeld() noexcept { @@ -2020,35 +1973,4 @@ bool GraphicsRunGpuLockHeld() noexcept { return g_gpu_mutex_owned; } -int GraphicsRunBeginCommandProcessorReadback() { - if (g_current_run_cp == nullptr || g_submission_pause_depth != 0 || g_gpu_mutex_owned || - g_cp_readback_owned || g_guest_memory_access == CommandProcessorGuestAccess::None) { - EXIT("GPU command-thread readback requires an unpaused command processor\n"); - } - const auto queue = g_current_run_cp->GetQueue(); - if (queue < 0 || queue >= GraphicContext::QUEUES_NUM) { - EXIT("GPU command-thread readback has invalid queue %d\n", queue); - } - g_cp_readback_owned = true; - // Direct fences enter from an already serialized command handler. Wait polls acquire the - // same processor mutex in CommandProcessorGuestAccessScope before the resource transaction. - g_cp_readback_preowned = true; - const auto synchronized_queue = - g_current_run_cp->BeginReadbackTransaction(g_cp_readback_preowned); - if (synchronized_queue != queue) { - EXIT("GPU command-thread queue changed during readback\n"); - } - return synchronized_queue; -} - -void GraphicsRunEndCommandProcessorReadback() { - if (g_current_run_cp == nullptr || !g_cp_readback_owned) { - EXIT("GPU command-thread readback ended without ownership\n"); - } - const auto preowned = g_cp_readback_preowned; - g_cp_readback_owned = false; - g_cp_readback_preowned = false; - g_current_run_cp->EndReadbackTransaction(preowned); -} - } // namespace Libs::Graphics diff --git a/src/graphics/guest_gpu/graphicsRun.h b/src/graphics/guest_gpu/graphicsRun.h index f2c0eebd..c4da95fc 100644 --- a/src/graphics/guest_gpu/graphicsRun.h +++ b/src/graphics/guest_gpu/graphicsRun.h @@ -6,6 +6,8 @@ namespace Libs::Graphics { +class CommandProcessor; + class GraphicsRunSubmissionLock final { public: GraphicsRunSubmissionLock(); @@ -23,17 +25,11 @@ void GraphicsRunSubmitFlipPreparation(); void GraphicsRunWait(); void GraphicsRunDone(); int GraphicsRunGetFrameNum(); -[[nodiscard]] bool GraphicsRunIsCommandProcessorThread() noexcept; -[[nodiscard]] bool GraphicsRunIsGuestMemoryPoll() noexcept; -[[nodiscard]] bool GraphicsRunIsCommandProcessorGuestAccess() noexcept; -[[nodiscard]] bool GraphicsRunSubmissionLockHeld() noexcept; -[[nodiscard]] bool GraphicsRunGpuLockHeld() noexcept; -// Begin/end an exclusive same-command-processor readback transaction. The begin call finishes -// work already recorded on that queue and retains the processor lock so its peer draw/constant -// job cannot record new work until BufferCache publishes the faulted page. -[[nodiscard]] int GraphicsRunBeginCommandProcessorReadback(); -void GraphicsRunEndCommandProcessorReadback(); - +[[nodiscard]] bool GraphicsRunIsCommandProcessorThread() noexcept; +[[nodiscard]] CommandProcessor* GraphicsRunCurrentCommandProcessor() noexcept; +void GraphicsRunFinishCommandProcessors(); +[[nodiscard]] bool GraphicsRunSubmissionLockHeld() noexcept; +[[nodiscard]] bool GraphicsRunGpuLockHeld() noexcept; } // namespace Libs::Graphics #endif /* EMULATOR_INCLUDE_EMULATOR_GRAPHICS_GRAPHICSRUN_H_ */ diff --git a/src/graphics/host_gpu/hostMemory.cpp b/src/graphics/host_gpu/hostMemory.cpp index c440728d..292252d3 100644 --- a/src/graphics/host_gpu/hostMemory.cpp +++ b/src/graphics/host_gpu/hostMemory.cpp @@ -2,7 +2,6 @@ #include #include -#include #if KYTY_PLATFORM == KYTY_PLATFORM_WINDOWS #ifndef NOMINMAX @@ -21,10 +20,10 @@ bool IsAccessible(DWORD protect, HostMemoryAccess access) { constexpr DWORD blocked = PAGE_NOACCESS | PAGE_GUARD; constexpr DWORD readable = PAGE_READONLY | PAGE_READWRITE | PAGE_WRITECOPY | PAGE_EXECUTE_READ | PAGE_EXECUTE_READWRITE | PAGE_EXECUTE_WRITECOPY; - constexpr DWORD writable = - PAGE_READWRITE | PAGE_WRITECOPY | PAGE_EXECUTE_READWRITE | PAGE_EXECUTE_WRITECOPY; - return (protect & blocked) == 0 && - (protect & (access == HostMemoryAccess::Write ? writable : readable)) != 0; + if (access == HostMemoryAccess::Mapped) { + return (protect & PAGE_GUARD) == 0; + } + return (protect & blocked) == 0 && (protect & readable) != 0; } #endif @@ -77,8 +76,7 @@ bool HostMemoryQueryRange(uint64_t addr, uint64_t requested_size, HostMemoryAcce if (begin > current) { break; } - const bool allowed = - access == HostMemoryAccess::Write ? permissions[1] == 'w' : permissions[0] == 'r'; + const bool allowed = access == HostMemoryAccess::Mapped || permissions[0] == 'r'; if (!allowed) { break; } @@ -93,39 +91,10 @@ bool HostMemoryQueryRange(uint64_t addr, uint64_t requested_size, HostMemoryAcce return *accessible_size != 0; } -bool HostMemoryQueryEnd(uint64_t addr, HostMemoryAccess access, uint64_t* end) { - if (addr == 0 || end == nullptr) { - return false; - } - uint64_t accessible_size = 0; - if (!HostMemoryQueryRange(addr, UINT64_MAX - addr, access, &accessible_size) || - accessible_size == 0 || UINT64_MAX - addr < accessible_size) { - return false; - } - *end = addr + accessible_size; - return true; -} - bool HostMemoryQueryReadable(uint64_t addr, uint64_t requested_size, uint64_t* readable_size) { return HostMemoryQueryRange(addr, requested_size, HostMemoryAccess::Read, readable_size); } -bool HostMemoryQueryWritable(uint64_t addr, uint64_t requested_size, uint64_t* writable_size) { - return HostMemoryQueryRange(addr, requested_size, HostMemoryAccess::Write, writable_size); -} - -bool HostMemoryReadDword(void*, uint64_t addr, uint32_t* value) { - if (value == nullptr) { - return false; - } - uint64_t readable = 0; - if (!HostMemoryQueryReadable(addr, sizeof(*value), &readable) || readable < sizeof(*value)) { - return false; - } - std::memcpy(value, reinterpret_cast(static_cast(addr)), sizeof(*value)); - return true; -} - bool HostMemoryIsReadable(uint64_t addr) { return HostMemoryRangeIsReadable(addr, 1); } diff --git a/src/graphics/host_gpu/hostMemory.h b/src/graphics/host_gpu/hostMemory.h index 7463d501..eca0b155 100644 --- a/src/graphics/host_gpu/hostMemory.h +++ b/src/graphics/host_gpu/hostMemory.h @@ -5,14 +5,11 @@ namespace Libs::Graphics { -enum class HostMemoryAccess { Read, Write }; +enum class HostMemoryAccess { Read, Mapped }; bool HostMemoryQueryRange(uint64_t addr, uint64_t requested_size, HostMemoryAccess access, uint64_t* accessible_size); -bool HostMemoryQueryEnd(uint64_t addr, HostMemoryAccess access, uint64_t* end); bool HostMemoryQueryReadable(uint64_t addr, uint64_t requested_size, uint64_t* readable_size); -bool HostMemoryQueryWritable(uint64_t addr, uint64_t requested_size, uint64_t* writable_size); -bool HostMemoryReadDword(void* userdata, uint64_t addr, uint32_t* value); bool HostMemoryIsReadable(uint64_t addr); bool HostMemoryRangeIsReadable(uint64_t addr, uint64_t size); diff --git a/src/graphics/host_gpu/objects/label.cpp b/src/graphics/host_gpu/objects/label.cpp index b44b9089..27ccf616 100644 --- a/src/graphics/host_gpu/objects/label.cpp +++ b/src/graphics/host_gpu/objects/label.cpp @@ -1,14 +1,11 @@ #include "graphics/host_gpu/objects/label.h" #include "common/assert.h" -#include "common/logging/log.h" -#include "kernel/memory.h" #include "common/threads.h" #include "graphics/host_gpu/graphicContext.h" #include "graphics/host_gpu/renderer/render.h" #include -#include #include #include @@ -22,10 +19,6 @@ enum LabelStatus { }; struct LabelCallbacks { - uint64_t* dst_gpu_addr64 = nullptr; - uint64_t value64 = 0; - uint32_t* dst_gpu_addr32 = nullptr; - uint32_t value32 = 0; LabelCallback callback_1 = nullptr; LabelCallback callback_2 = nullptr; uint64_t args[LABEL_ARGS_MAX] = {}; @@ -64,10 +57,8 @@ public: ~LabelManager() { KYTY_NOT_IMPLEMENTED; } KYTY_CLASS_NO_COPY(LabelManager); - Label* Create64(GraphicContext* ctx, uint64_t* dst_gpu_addr, uint64_t value, - LabelCallback callback_1, LabelCallback callback_2, const uint64_t* args); - Label* Create32(GraphicContext* ctx, uint32_t* dst_gpu_addr, uint32_t value, - LabelCallback callback_1, LabelCallback callback_2, const uint64_t* args); + Label* Create(GraphicContext* ctx, LabelCallback callback_1, LabelCallback callback_2, + const uint64_t* args); void Delete(Label* label); void Set(CommandBuffer* buffer, Label* label); void Drain(); @@ -75,9 +66,6 @@ public: private: static void ThreadRun(void* data); - template - Label* Create(GraphicContext* ctx, T* dst_gpu_addr, T value, LabelCallback callback_1, - LabelCallback callback_2, const uint64_t* args); bool Remove(Label* label); static void Destroy(Label* label); @@ -90,11 +78,6 @@ private: static LabelManager* g_label_manager = nullptr; static thread_local bool g_in_label_callback = false; -template -void WriteGuestLabel(T* address, T value) { - LabelWriteGuestMemory(address, &value, sizeof(value)); -} - class LabelCallbackScope final { public: LabelCallbackScope() { @@ -132,8 +115,7 @@ void LabelManager::ThreadRun(void* data) { it->completion->event == nullptr) { EXIT("GPU label submission has no completion event\n"); } - const auto status = - vkGetEventStatus(it->completion->device, it->completion->event); + const auto status = vkGetEventStatus(it->completion->device, it->completion->event); switch (status) { case VK_EVENT_SET: fired_labels.push_back(it->callbacks); @@ -180,35 +162,9 @@ void LabelManager::ThreadRun(void* data) { for (auto& label: fired_labels) { LabelCallbackScope callback_scope; - bool write = true; if (label.callback_1 != nullptr) { - write = label.callback_1(label.args); - } - - if (write && label.dst_gpu_addr64 != nullptr) { - // Publishing EOP fences through physical backing keeps a fence word coherent even - // when unrelated bytes on its host page are - // protected for BufferCache GPU ownership. - WriteGuestLabel(label.dst_gpu_addr64, label.value64); - - static std::atomic log_count {0}; - if (log_count.fetch_add(1) < 256) { - LOGF_COLOR(Log::Color::BrightGreen, - "EndOfPipe Signal!!! [0x%016" PRIx64 "] <- 0x%016" PRIx64 "\n", - reinterpret_cast(label.dst_gpu_addr64), label.value64); - } - } - - if (write && label.dst_gpu_addr32 != nullptr) { - WriteGuestLabel(label.dst_gpu_addr32, label.value32); - - static std::atomic log_count {0}; - if (log_count.fetch_add(1) < 256) { - LOGF_COLOR(Log::Color::BrightGreen, - "EndOfPipe Signal!!! [0x%016" PRIx64 "] <- 0x%08" PRIx32 "\n", - reinterpret_cast(label.dst_gpu_addr32), label.value32); - } + (void)label.callback_1(label.args); } if (label.callback_2 != nullptr) { @@ -229,29 +185,15 @@ void LabelManager::ThreadRun(void* data) { } } -template -Label* LabelManager::Create(GraphicContext* ctx, T* dst_gpu_addr, T value, LabelCallback callback_1, - LabelCallback callback_2, const uint64_t* args) { - static_assert(sizeof(T) == sizeof(uint32_t) || sizeof(T) == sizeof(uint64_t)); - +Label* LabelManager::Create(GraphicContext* ctx, LabelCallback callback_1, LabelCallback callback_2, + const uint64_t* args) { EXIT_IF(ctx == nullptr); Common::LockGuard lock(m_mutex); auto* label = new Label; - label->status = LabelStatus::New; - if constexpr (sizeof(T) == sizeof(uint64_t)) { - label->callbacks.dst_gpu_addr64 = dst_gpu_addr; - label->callbacks.value64 = value; - label->callbacks.dst_gpu_addr32 = nullptr; - label->callbacks.value32 = 0; - } else { - label->callbacks.dst_gpu_addr32 = dst_gpu_addr; - label->callbacks.value32 = value; - label->callbacks.dst_gpu_addr64 = nullptr; - label->callbacks.value64 = 0; - } + label->status = LabelStatus::New; label->device = ctx->device; label->callbacks.callback_1 = callback_1; label->callbacks.callback_2 = callback_2; @@ -267,18 +209,6 @@ Label* LabelManager::Create(GraphicContext* ctx, T* dst_gpu_addr, T value, Label return label; } -Label* LabelManager::Create64(GraphicContext* ctx, uint64_t* dst_gpu_addr, uint64_t value, - LabelCallback callback_1, LabelCallback callback_2, - const uint64_t* args) { - return Create(ctx, dst_gpu_addr, value, callback_1, callback_2, args); -} - -Label* LabelManager::Create32(GraphicContext* ctx, uint32_t* dst_gpu_addr, uint32_t value, - LabelCallback callback_1, LabelCallback callback_2, - const uint64_t* args) { - return Create(ctx, dst_gpu_addr, value, callback_1, callback_2, args); -} - bool LabelManager::Remove(Label* label) { EXIT_IF(label == nullptr); EXIT_IF(label->device == nullptr); @@ -336,8 +266,8 @@ void LabelManager::Set(CommandBuffer* buffer, Label* label) { label->status = LabelStatus::Active; LabelSubmission submission {}; - submission.callbacks = label->callbacks; - submission.completion = std::make_shared(); + submission.callbacks = label->callbacks; + submission.completion = std::make_shared(); submission.completion->device = label->device; auto* vk_buffer = buffer->GetPool()->buffers[buffer->GetIndex()]; @@ -385,18 +315,11 @@ void LabelInit() { g_label_manager = new LabelManager; } -Label* LabelCreate64(GraphicContext* ctx, uint64_t* dst_gpu_addr, uint64_t value, - LabelCallback callback_1, LabelCallback callback_2, const uint64_t* args) { +Label* LabelCreate(GraphicContext* ctx, LabelCallback callback_1, LabelCallback callback_2, + const uint64_t* args) { EXIT_IF(g_label_manager == nullptr); - return g_label_manager->Create64(ctx, dst_gpu_addr, value, callback_1, callback_2, args); -} - -Label* LabelCreate32(GraphicContext* ctx, uint32_t* dst_gpu_addr, uint32_t value, - LabelCallback callback_1, LabelCallback callback_2, const uint64_t* args) { - EXIT_IF(g_label_manager == nullptr); - - return g_label_manager->Create32(ctx, dst_gpu_addr, value, callback_1, callback_2, args); + return g_label_manager->Create(ctx, callback_1, callback_2, args); } void LabelDelete(Label* label) { @@ -416,15 +339,6 @@ void LabelDrain() { g_label_manager->Drain(); } -void LabelWriteGuestMemory(void* address, const void* data, uint64_t size) { - if (address == nullptr || data == nullptr || size == 0) { - EXIT("invalid guest label write\n"); - } - if (!Libs::LibKernel::Memory::TryWriteBacking(reinterpret_cast(address), data, size)) { - std::memcpy(address, data, static_cast(size)); - } -} - bool LabelInCallback() noexcept { return g_in_label_callback; } diff --git a/src/graphics/host_gpu/objects/label.h b/src/graphics/host_gpu/objects/label.h index 2c1bf624..033903c7 100644 --- a/src/graphics/host_gpu/objects/label.h +++ b/src/graphics/host_gpu/objects/label.h @@ -15,14 +15,11 @@ void LabelInit(); constexpr int LABEL_ARGS_MAX = 5; using LabelCallback = bool (*)(const uint64_t* args); -Label* LabelCreate64(GraphicContext* ctx, uint64_t* dst_gpu_addr, uint64_t value, - LabelCallback callback_1, LabelCallback callback_2, const uint64_t* args); -Label* LabelCreate32(GraphicContext* ctx, uint32_t* dst_gpu_addr, uint32_t value, - LabelCallback callback_1, LabelCallback callback_2, const uint64_t* args); +Label* LabelCreate(GraphicContext* ctx, LabelCallback callback_1, LabelCallback callback_2, + const uint64_t* args); void LabelDelete(Label* label); void LabelSet(CommandBuffer* buffer, Label* label); void LabelDrain(); -void LabelWriteGuestMemory(void* address, const void* data, uint64_t size); [[nodiscard]] bool LabelInCallback() noexcept; } // namespace Libs::Graphics diff --git a/src/graphics/host_gpu/renderer/bufferCache.cpp b/src/graphics/host_gpu/renderer/bufferCache.cpp index 0cdfe790..42dcec14 100644 --- a/src/graphics/host_gpu/renderer/bufferCache.cpp +++ b/src/graphics/host_gpu/renderer/bufferCache.cpp @@ -16,7 +16,6 @@ #include #include #include -#include #include #include #include @@ -37,27 +36,6 @@ constexpr uint64_t AlignReadbackCopySize(uint64_t size) noexcept { return (size + READBACK_COPY_ALIGNMENT - 1) & ~(READBACK_COPY_ALIGNMENT - 1); } -constexpr bool ShouldLogCommandProcessorReadback(uint64_t serial) noexcept { - return serial <= 32 || (serial % 256) == 0; -} - -struct CommandProcessorReadbackState { - BufferCache* cache = nullptr; - PageFaultAccess access = PageFaultAccess::Unknown; - uint64_t vaddr = 0; - uint64_t size = 0; - uint64_t page = 0; - uint64_t serial = 0; - uint64_t dirty_bytes = 0; - uint64_t packed_bytes = 0; - uint32_t dirty_count = 0; - std::chrono::steady_clock::time_point started {}; - bool installed = false; -}; - -thread_local CommandProcessorReadbackState g_command_processor_readback; -std::atomic g_command_processor_readback_serial {0}; - struct SharedVulkanBufferOwner { explicit SharedVulkanBufferOwner(GraphicContext* context): ctx(context) { EXIT_IF(ctx == nullptr); @@ -192,10 +170,6 @@ bool CanMergeBufferCacheQueueMask(uint64_t queue_mask, uint32_t queue) noexcept return queue < 64 && (queue_mask & ~(uint64_t {1} << queue)) == 0; } -bool CanReadbackBufferCacheQueueMask(uint64_t queue_mask, uint32_t queue) noexcept { - return queue < 64 && queue_mask == (uint64_t {1} << queue); -} - struct BufferCache::CachedBuffer { uint64_t vaddr = 0; uint64_t size = 0; @@ -204,14 +178,6 @@ struct BufferCache::CachedBuffer { std::shared_ptr buffer; }; -struct BufferCache::CommandProcessorReadbackResources { - GraphicContext* ctx = nullptr; - int queue = -1; - VulkanBuffer buffer {}; - void* mapped = nullptr; - std::unique_ptr command; -}; - struct BufferCache::ReadbackWorker { static constexpr uint32_t MAX_RANGES = READBACK_MAX_RANGES; enum class State : uint32_t { @@ -305,20 +271,20 @@ struct BufferCache::ReadbackWorker { } void Request(PageFaultAccess fault_access, uint64_t fault_vaddr, uint64_t fault_size) noexcept { - const bool submissions_prepaused_now = GraphicsRunSubmissionLockHeld(); + const bool command_thread = GraphicsRunIsCommandProcessorThread(); + const bool submissions_prepaused_now = GraphicsRunSubmissionLockHeld() || command_thread; const bool unsafe_gpu_lock = GraphicsRunGpuLockHeld() && !submissions_prepaused_now; - if (GraphicsRunIsCommandProcessorThread() || unsafe_gpu_lock || LabelInCallback() || - g_cache_lock_owner != nullptr || ctx == nullptr || command == nullptr || - mapped == nullptr || readback.buffer == nullptr) { + if (unsafe_gpu_lock || LabelInCallback() || g_cache_lock_owner != nullptr || + ctx == nullptr || command == nullptr || mapped == nullptr || + readback.buffer == nullptr) { EXIT("BufferCache: unsafe readback request context, command_thread=%d " "submission_lock=%d " "gpu_lock=%d label_callback=%d cache_lock=%p ctx=%p command=%p mapped=%p " "buffer=%p\n", - GraphicsRunIsCommandProcessorThread(), GraphicsRunSubmissionLockHeld(), - GraphicsRunGpuLockHeld(), LabelInCallback(), - static_cast(g_cache_lock_owner), static_cast(ctx), - static_cast(command.get()), static_cast(mapped), - static_cast(readback.buffer)); + command_thread, GraphicsRunSubmissionLockHeld(), GraphicsRunGpuLockHeld(), + LabelInCallback(), static_cast(g_cache_lock_owner), + static_cast(ctx), static_cast(command.get()), + static_cast(mapped), static_cast(readback.buffer)); } State expected = State::Idle; while (!state.compare_exchange_weak(expected, State::Claimed, std::memory_order_acq_rel)) { @@ -584,18 +550,6 @@ BufferCache::BufferCache(PageManager& page_manager, ResourceMutex& resource_mute BufferCache::~BufferCache() { m_readback.reset(); - if (m_cp_readback_resources != nullptr) { - auto& resources = *m_cp_readback_resources; - resources.command.reset(); - if (resources.mapped != nullptr) { - VulkanUnmapMemory(resources.ctx, &resources.buffer.memory); - resources.mapped = nullptr; - } - if (resources.buffer.buffer != nullptr) { - VulkanDeleteBuffer(resources.ctx, &resources.buffer); - } - m_cp_readback_resources.reset(); - } if (!m_gpu_modified_ranges.Empty()) { EXIT("BufferCache: destroyed with pending GPU-modified ranges\n"); } @@ -623,14 +577,9 @@ bool BufferCache::InvalidateMemory(PageFaultAccess access, uint64_t vaddr, uint6 switch (phase) { case PageFaultPhase::Invalidate: break; case PageFaultPhase::Complete: - return CompleteCommandProcessorReadback(access, vaddr, size) || - m_readback->Complete(access, vaddr, size) || + return m_readback->Complete(access, vaddr, size) || m_memory_tracker.CompleteCpuFault(vaddr, size, access, false); - case PageFaultPhase::Release: - if (!ReleaseCommandProcessorReadback(access, vaddr, size)) { - m_readback->Release(access, vaddr, size); - } - return true; + case PageFaultPhase::Release: m_readback->Release(access, vaddr, size); return true; default: EXIT("BufferCache: unsupported page-fault phase %u\n", static_cast(phase)); } @@ -639,242 +588,9 @@ bool BufferCache::InvalidateMemory(PageFaultAccess access, uint64_t vaddr, uint6 return action == CpuFaultAction::Continue; } if (GraphicsRunIsCommandProcessorThread()) { - if (!GraphicsRunIsCommandProcessorGuestAccess()) { - EXIT("BufferCache: command-processor fault outside a coherent guest-memory access, " - "addr=0x%016" PRIx64 " size=0x%016" PRIx64 "\n", - vaddr, size); - } - RequestCommandProcessorReadback(access, vaddr, size); - } else { - m_readback->Request(access, vaddr, size); - } - return true; -} - -void BufferCache::RequestCommandProcessorReadback(PageFaultAccess access, uint64_t vaddr, - uint64_t size) { - if (!GraphicsRunIsCommandProcessorThread() || g_command_processor_readback.cache != nullptr) { - EXIT("BufferCache: invalid nested command-processor readback\n"); - } - const auto serial = - g_command_processor_readback_serial.fetch_add(1, std::memory_order_relaxed) + 1; - if (serial == 0) { - EXIT("BufferCache: command-processor readback serial overflow\n"); - } - const bool log_detail = ShouldLogCommandProcessorReadback(serial); - if (log_detail) { - LOGF("BufferCache: command-processor readback request serial=%" PRIu64 " addr=0x%016" PRIx64 - " size=0x%016" PRIx64 " access=%u poll=%d\n", - serial, vaddr, size, static_cast(access), GraphicsRunIsGuestMemoryPoll()); - Log::Flush(); - } - const auto page = vaddr & ~(TRACKER_PAGE_SIZE - 1); - const auto queue = static_cast(GraphicsRunBeginCommandProcessorReadback()); - if (log_detail) { - LOGF("BufferCache: command-processor queue synchronized serial=%" PRIu64 - " queue=%u page=0x%016" PRIx64 "\n", - serial, queue, page); - Log::Flush(); - } - if (g_cache_lock_owner != nullptr) { - EXIT("BufferCache: command-processor readback entered with a cache lock\n"); - } - g_cache_lock_owner = this; - m_mutex.Lock(); - if (log_detail) { - LOGF("BufferCache: command-processor cache locked serial=%" PRIu64 " page=0x%016" PRIx64 - "\n", - serial, page); - Log::Flush(); - } - - auto it = m_buffers.upper_bound(vaddr); - if (it == m_buffers.begin()) { - EXIT("BufferCache: command-processor readback address has no cached buffer, " - "addr=0x%016" PRIx64 "\n", - vaddr); - } - --it; - auto& cached = *it->second; - if (vaddr < cached.vaddr || vaddr >= cached.vaddr + cached.size || page < cached.vaddr || - TRACKER_PAGE_SIZE > cached.size - (page - cached.vaddr) || cached.ctx == nullptr || - cached.buffer == nullptr || cached.buffer->buffer == nullptr) { - EXIT("BufferCache: invalid command-processor readback owner, fault=0x%016" PRIx64 - " page=0x%016" PRIx64 " buffer=0x%016" PRIx64 "+0x%016" PRIx64 "\n", - vaddr, page, cached.vaddr, cached.size); - } - if (!CanReadbackBufferCacheQueueMask(cached.queue_mask, queue)) { - EXIT("BufferCache: command-processor readback cannot synchronize cross-queue ownership, " - "addr=0x%016" PRIx64 " used_queues=0x%016" PRIx64 " current_queue=%u\n", - vaddr, cached.queue_mask, queue); - } - const auto dirty = m_gpu_modified_ranges.Intersections(page, TRACKER_PAGE_SIZE); - if (dirty.empty()) { - EXIT("BufferCache: command-processor GPU-dirty page has no dirty byte ranges, " - "page=0x%016" PRIx64 "\n", - page); - } - ValidateDirtyPages(m_gpu_modified_ranges, page, TRACKER_PAGE_SIZE, "command-processor"); - if (m_cp_readback_resources == nullptr) { - auto resources = std::make_unique(); - resources->ctx = cached.ctx; - resources->buffer.usage = VK_BUFFER_USAGE_TRANSFER_DST_BIT; - resources->buffer.memory.property = VK_MEMORY_PROPERTY_HOST_VISIBLE_BIT | - VK_MEMORY_PROPERTY_HOST_COHERENT_BIT | - VK_MEMORY_PROPERTY_HOST_CACHED_BIT; - VulkanCreateBuffer(cached.ctx, READBACK_CAPACITY, &resources->buffer); - VulkanMapMemory(cached.ctx, &resources->buffer.memory, &resources->mapped); - m_cp_readback_resources = std::move(resources); - } - auto& resources = *m_cp_readback_resources; - if (resources.ctx != cached.ctx || resources.mapped == nullptr || - resources.buffer.buffer == nullptr) { - EXIT("BufferCache: invalid command-processor readback resources\n"); - } - if (queue >= GraphicContext::QUEUES_NUM || cached.ctx->queues[queue].family == UINT32_MAX || - cached.ctx->queues[queue].vk_queue == nullptr) { - EXIT("BufferCache: command-processor readback has invalid Vulkan queue %u\n", queue); - } - if (resources.command == nullptr || resources.queue != static_cast(queue)) { - resources.command.reset(); - resources.command = std::make_unique(static_cast(queue)); - resources.queue = static_cast(queue); - } - if (resources.command->IsInvalid() || - resources.command->GetQueue() != static_cast(queue)) { - EXIT("BufferCache: failed to initialize owning-queue readback command buffer, queue=%u\n", - queue); - } - if (dirty.size() > ReadbackWorker::MAX_RANGES) { - EXIT("BufferCache: too many command-processor dirty ranges, page=0x%016" PRIx64 - " count=%" PRIu64 " limit=%u\n", - page, static_cast(dirty.size()), ReadbackWorker::MAX_RANGES); - } - std::array copies {}; - std::array barriers {}; - uint64_t dirty_bytes = 0; - uint64_t packed_bytes = 0; - uint32_t dirty_count = 0; - for (const auto& range: dirty) { - if ((range.address - cached.vaddr) % 4 != 0 || range.size % 4 != 0) { - EXIT("BufferCache: invalid command-processor dirty range, addr=0x%016" PRIx64 - " size=0x%016" PRIx64 " page=0x%016" PRIx64 " packed=0x%016" PRIx64 "\n", - range.address, range.size, page, dirty_bytes); - } - copies[dirty_count] = {.srcOffset = range.address - cached.vaddr, - .dstOffset = packed_bytes, - .size = range.size}; - auto& barrier = barriers[dirty_count]; - barrier.sType = VK_STRUCTURE_TYPE_BUFFER_MEMORY_BARRIER; - barrier.srcAccessMask = VK_ACCESS_MEMORY_WRITE_BIT; - barrier.dstAccessMask = VK_ACCESS_TRANSFER_READ_BIT; - barrier.srcQueueFamilyIndex = VK_QUEUE_FAMILY_IGNORED; - barrier.dstQueueFamilyIndex = VK_QUEUE_FAMILY_IGNORED; - barrier.buffer = cached.buffer->buffer; - barrier.offset = copies[dirty_count].srcOffset; - barrier.size = range.size; - const auto slot_size = AlignReadbackCopySize(range.size); - if (packed_bytes > READBACK_CAPACITY || slot_size > READBACK_CAPACITY - packed_bytes) { - EXIT("BufferCache: aligned command-processor readback exceeds persistent capacity, " - "packed=%" PRIu64 " size=%" PRIu64 " capacity=%" PRIu64 "\n", - packed_bytes, slot_size, READBACK_CAPACITY); - } - packed_bytes += slot_size; - dirty_bytes += range.size; - dirty_count++; - } - if (dirty_count == 0 || dirty_bytes == 0) { - EXIT("BufferCache: command-processor readback produced no packed ranges\n"); - } - const auto started = std::chrono::steady_clock::now(); - if (log_detail) { - LOGF("BufferCache: command-processor readback begin serial=%" PRIu64 - " queue=%u page=0x%016" PRIx64 " ranges=%u bytes=%" PRIu64 " packed=%" PRIu64 "\n", - serial, queue, page, dirty_count, dirty_bytes, packed_bytes); - Log::Flush(); - } - auto* vk_command = resources.command->GetPool()->buffers[resources.command->GetIndex()]; - resources.command->Begin(); - vkCmdPipelineBarrier(vk_command, VK_PIPELINE_STAGE_ALL_COMMANDS_BIT, - VK_PIPELINE_STAGE_TRANSFER_BIT, 0, 0, nullptr, dirty_count, - barriers.data(), 0, nullptr); - vkCmdCopyBuffer(vk_command, cached.buffer->buffer, resources.buffer.buffer, dirty_count, - copies.data()); - resources.command->End(); - resources.command->Execute(); - resources.command->WaitForFenceAndReset(); - if (log_detail) { - LOGF("BufferCache: command-processor GPU copy complete serial=%" PRIu64 "\n", serial); - Log::Flush(); - } - for (uint32_t i = 0; i < dirty_count; i++) { - Libs::LibKernel::Memory::WriteBacking( - dirty[i].address, static_cast(resources.mapped) + copies[i].dstOffset, - dirty[i].size); - } - g_command_processor_readback = {.cache = this, - .access = access, - .vaddr = vaddr, - .size = size, - .page = page, - .serial = serial, - .dirty_bytes = dirty_bytes, - .packed_bytes = packed_bytes, - .dirty_count = dirty_count, - .started = started}; -} - -bool BufferCache::CompleteCommandProcessorReadback(PageFaultAccess access, uint64_t vaddr, - uint64_t size) noexcept { - auto& pending = g_command_processor_readback; - if (pending.cache == nullptr) { - return false; - } - if (pending.cache != this || pending.access != access || pending.vaddr != vaddr || - pending.size != size || pending.installed) { - EXIT("BufferCache: mismatched active command-processor readback completion\n"); - } - if (!m_memory_tracker.CompleteCpuFault(vaddr, size, access, true)) { - EXIT("BufferCache: failed to complete command-processor readback, addr=0x%016" PRIx64 - " size=0x%016" PRIx64 " access=%u\n", - vaddr, size, static_cast(access)); - } - m_gpu_modified_ranges.Subtract(pending.page, TRACKER_PAGE_SIZE); - pending.installed = true; - if (ShouldLogCommandProcessorReadback(pending.serial)) { - LOGF("BufferCache: command-processor readback complete serial=%" PRIu64 - " page=0x%016" PRIx64 " ranges=%u bytes=%" PRIu64 "\n", - pending.serial, pending.page, pending.dirty_count, pending.dirty_bytes); - } - return true; -} - -bool BufferCache::ReleaseCommandProcessorReadback(PageFaultAccess access, uint64_t vaddr, - uint64_t size) noexcept { - auto& pending = g_command_processor_readback; - if (pending.cache == nullptr) { - return false; - } - if (pending.cache != this || pending.access != access || pending.vaddr != vaddr || - pending.size != size || !pending.installed) { - EXIT("BufferCache: mismatched active command-processor readback release\n"); - } - const auto serial = pending.serial; - const auto page = pending.page; - const auto dirty_count = pending.dirty_count; - const auto dirty_bytes = pending.dirty_bytes; - const auto elapsed_micro = std::chrono::duration_cast( - std::chrono::steady_clock::now() - pending.started) - .count(); - pending = {}; - m_mutex.Unlock(); - g_cache_lock_owner = nullptr; - GraphicsRunEndCommandProcessorReadback(); - if (ShouldLogCommandProcessorReadback(serial)) { - LOGF("BufferCache: command-processor readback release serial=%" PRIu64 " page=0x%016" PRIx64 - " ranges=%u bytes=%" PRIu64 " elapsed_us=%" PRIi64 "\n", - serial, page, dirty_count, dirty_bytes, static_cast(elapsed_micro)); + GraphicsRunFinishCommandProcessors(); } + m_readback->Request(access, vaddr, size); return true; } diff --git a/src/graphics/host_gpu/renderer/bufferCache.h b/src/graphics/host_gpu/renderer/bufferCache.h index 116d330c..8ce33c9e 100644 --- a/src/graphics/host_gpu/renderer/bufferCache.h +++ b/src/graphics/host_gpu/renderer/bufferCache.h @@ -39,7 +39,6 @@ struct BufferCacheRange { [[nodiscard]] bool MergeOverlappingBufferCacheRange(BufferCacheRange* merged, BufferCacheRange candidate) noexcept; [[nodiscard]] bool CanMergeBufferCacheQueueMask(uint64_t queue_mask, uint32_t queue) noexcept; -[[nodiscard]] bool CanReadbackBufferCacheQueueMask(uint64_t queue_mask, uint32_t queue) noexcept; class BufferCache { public: @@ -61,11 +60,9 @@ public: // Emulator-owned, CPU-current scratch only. Guest ranges must use ObtainBuffer so page // ownership is resolved before any CPU access. [[nodiscard]] bool UploadHostData(CommandBuffer* command, GraphicContext* ctx, const void* src, - uint64_t size, uint64_t alignment, - VulkanBuffer** out_buffer, uint64_t* out_offset, - uint64_t* out_range); - [[nodiscard]] VulkanBuffer* ObtainNullBuffer(CommandBuffer* command, - GraphicContext* ctx); + uint64_t size, uint64_t alignment, VulkanBuffer** out_buffer, + uint64_t* out_offset, uint64_t* out_range); + [[nodiscard]] VulkanBuffer* ObtainNullBuffer(CommandBuffer* command, GraphicContext* ctx); [[nodiscard]] BufferImageCopySource ObtainBufferForImage(uint64_t vaddr, uint64_t size); void FillBuffer(CommandBuffer* command, GraphicContext* ctx, uint64_t vaddr, uint64_t size, uint32_t value); @@ -84,19 +81,12 @@ public: private: struct CachedBuffer; struct ReadbackWorker; - struct CommandProcessorReadbackResources; - void RequestCommandProcessorReadback(PageFaultAccess access, uint64_t vaddr, uint64_t size); - [[nodiscard]] bool CompleteCommandProcessorReadback(PageFaultAccess access, uint64_t vaddr, - uint64_t size) noexcept; - [[nodiscard]] bool ReleaseCommandProcessorReadback(PageFaultAccess access, uint64_t vaddr, - uint64_t size) noexcept; Common::Mutex m_mutex; std::vector m_delete_later; std::shared_ptr m_null_buffer; std::map> m_buffers; std::unique_ptr m_readback; - std::unique_ptr m_cp_readback_resources; RangeSet m_gpu_modified_ranges; MemoryTracker m_memory_tracker; PageManager& m_page_manager; diff --git a/src/graphics/host_gpu/renderer/context.cpp b/src/graphics/host_gpu/renderer/context.cpp index 29d677d9..6fa8348f 100644 --- a/src/graphics/host_gpu/renderer/context.cpp +++ b/src/graphics/host_gpu/renderer/context.cpp @@ -332,7 +332,7 @@ void CommandBuffer::Execute() { m_execute = true; m_fence_waited = false; - m_submit_seq = g_command_buffer_submit_seq.fetch_add(1, std::memory_order_relaxed) + 1; + m_submit_seq = g_command_buffer_submit_seq.fetch_add(1, std::memory_order_relaxed) + 1; if (result != VK_SUCCESS) { LOGF("vkQueueSubmit failed: %s (%d), queue=%d index=%u submit_seq=%" PRIu64 @@ -396,7 +396,7 @@ void CommandBuffer::ExecuteWithSemaphore(VkSemaphore signal_semaphore) { m_execute = true; m_fence_waited = false; - m_submit_seq = g_command_buffer_submit_seq.fetch_add(1, std::memory_order_relaxed) + 1; + m_submit_seq = g_command_buffer_submit_seq.fetch_add(1, std::memory_order_relaxed) + 1; if (result != VK_SUCCESS) { LOGF("vkQueueSubmit failed: %s (%d), queue=%d index=%u submit_seq=%" PRIu64 @@ -464,7 +464,7 @@ void CommandBuffer::ExecuteWithSemaphore(VkSemaphore wait_semaphore, m_execute = true; m_fence_waited = false; - m_submit_seq = g_command_buffer_submit_seq.fetch_add(1, std::memory_order_relaxed) + 1; + m_submit_seq = g_command_buffer_submit_seq.fetch_add(1, std::memory_order_relaxed) + 1; if (result != VK_SUCCESS) { LOGF("vkQueueSubmit failed: %s (%d), queue=%d index=%u submit_seq=%" PRIu64 @@ -518,6 +518,7 @@ void CommandBuffer::WaitForFenceAndReset() { m_fence_waited = false; vkResetCommandBuffer(m_pool->buffers[m_index], VK_COMMAND_BUFFER_RESET_RELEASE_RESOURCES_BIT); + m_recording_generation++; } m_host_stream.Reset(); if (was_executed) { diff --git a/src/graphics/host_gpu/renderer/descriptors.cpp b/src/graphics/host_gpu/renderer/descriptors.cpp index 1d5e6849..7e3fa7ba 100644 --- a/src/graphics/host_gpu/renderer/descriptors.cpp +++ b/src/graphics/host_gpu/renderer/descriptors.cpp @@ -205,7 +205,7 @@ NativeAddressBuffer(uint64_t submit_id, CommandBuffer* command_buffer, ->GetPhysicalDeviceProperties() .limits.maxStorageBufferRange); uint64_t size = 0; - const auto access = HostMemoryAccess::Read; + const auto access = HostMemoryAccess::Mapped; if (!HostMemoryQueryRange(address.binding_base, limit, access, &size)) { EXIT("address resource is not host-accessible: base=0x%016" PRIx64 "\n", address.binding_base); diff --git a/src/graphics/host_gpu/renderer/gpuResourceManager.cpp b/src/graphics/host_gpu/renderer/gpuResourceManager.cpp index cb27702a..a48a213e 100644 --- a/src/graphics/host_gpu/renderer/gpuResourceManager.cpp +++ b/src/graphics/host_gpu/renderer/gpuResourceManager.cpp @@ -1,6 +1,7 @@ #include "graphics/host_gpu/renderer/gpuResourceManager.h" #include "common/assert.h" +#include "graphics/guest_gpu/command_processor/commandProcessor.h" #include "graphics/guest_gpu/graphicsRun.h" #include "graphics/host_gpu/objects/label.h" #include "graphics/host_gpu/renderer/render.h" @@ -37,17 +38,24 @@ bool GpuResourceManager::HandleFault(PageFaultAccess access, uint64_t fault_vadd "addr=0x%016" PRIx64 " access=%u\n", fault_vaddr, static_cast(access)); } - if (m_resource_mutex.IsOwnedByCurrentThread()) { - if (!GraphicsRunIsCommandProcessorGuestAccess()) { - EXIT("unsupported page fault from a pre-owned resource transaction, addr=0x%016" PRIx64 - " access=%u\n", - fault_vaddr, static_cast(access)); + if (auto* cp = GraphicsRunCurrentCommandProcessor(); cp != nullptr) { + cp->BeginReadbackTransaction(); + bool handled = false; + { + ResourceMutex::FaultScope fault(m_resource_mutex); + handled = m_page_manager.HandleFault(access, fault_vaddr); } - ResourceMutex::FaultScope fault(m_resource_mutex); - return m_page_manager.HandleFault(access, fault_vaddr); + cp->EndReadbackTransaction(); + return handled; + } + if (m_resource_mutex.IsOwnedByCurrentThread()) { + EXIT("unsupported page fault from a pre-owned resource transaction, addr=0x%016" PRIx64 + " access=%u\n", + fault_vaddr, static_cast(access)); } // Stop command-processor jobs before taking the shared cache transaction. External readback - // workers inherit this paused state and therefore never form resource -> submission lock inversion. + // workers inherit this paused state and therefore never form resource -> submission lock + // inversion. GraphicsRunSubmissionLock submissions; ResourceMutex::FaultScope fault(m_resource_mutex); return m_page_manager.HandleFault(access, fault_vaddr); @@ -57,35 +65,6 @@ bool GpuResourceManager::IsMapped(uint64_t vaddr, uint64_t size) const noexcept return m_page_manager.IsMapped(vaddr, size); } -std::unique_lock GpuResourceManager::LockCommandProcessorGuestAccess( - uint64_t vaddr, uint64_t size) { - if (vaddr == 0 || size == 0 || vaddr >= TRACKER_ADDRESS_SIZE || - size > TRACKER_ADDRESS_SIZE - vaddr) { - EXIT("invalid command-processor guest-memory access, addr=0x%016" PRIx64 - " size=0x%016" PRIx64 "\n", - vaddr, size); - } - return std::unique_lock(m_resource_mutex); -} - -std::unique_lock GpuResourceManager::LockAsynchronousGuestWrite(uint64_t vaddr, - uint64_t size) { - if (vaddr == 0 || size == 0 || vaddr >= TRACKER_ADDRESS_SIZE || - size > TRACKER_ADDRESS_SIZE - vaddr) { - EXIT("invalid asynchronous guest-memory write, addr=0x%016" PRIx64 - " size=0x%016" PRIx64 "\n", - vaddr, size); - } - std::unique_lock transaction(m_resource_mutex); - if (m_buffer_cache.HasPageOverlap(vaddr, size) || m_texture_cache.HasPageOverlap(vaddr, size) || - m_texture_cache.HasMetaOverlap(vaddr, size)) { - EXIT("asynchronous guest-memory write to a cached buffer/image range is unsupported, " - "addr=0x%016" PRIx64 " size=0x%016" PRIx64 "\n", - vaddr, size); - } - return transaction; -} - void GpuResourceManager::MapMemory(uint64_t vaddr, uint64_t size, GpuAccess access) { m_page_manager.OnGpuMap(vaddr, size, access); } diff --git a/src/graphics/host_gpu/renderer/gpuResourceManager.h b/src/graphics/host_gpu/renderer/gpuResourceManager.h index ead9e5e9..c968e200 100644 --- a/src/graphics/host_gpu/renderer/gpuResourceManager.h +++ b/src/graphics/host_gpu/renderer/gpuResourceManager.h @@ -9,7 +9,6 @@ #include "graphics/host_gpu/renderer/textureCache.h" #include -#include namespace Libs::Graphics { @@ -26,14 +25,6 @@ public: [[nodiscard]] bool HandleFault(PageFaultAccess access, uint64_t fault_vaddr) noexcept; [[nodiscard]] bool IsMapped(uint64_t vaddr, uint64_t size) const noexcept; - // Serializes synchronous command-processor guest-memory access against buffer/image ownership - // publication. Address metadata keeps this seam ready for a future page-range lock. - [[nodiscard]] std::unique_lock - LockCommandProcessorGuestAccess(uint64_t vaddr, uint64_t size); - // Keeps an asynchronous host callback's validation and guest write in one cache transaction. - // Cached buffer/image destinations are unsupported and fail before the callback can fault. - [[nodiscard]] std::unique_lock LockAsynchronousGuestWrite(uint64_t vaddr, - uint64_t size); void MapMemory(uint64_t vaddr, uint64_t size, GpuAccess access); void UnmapMemory(uint64_t vaddr, uint64_t size, GpuAccess access); void FillBuffer(CommandBuffer* command, uint64_t vaddr, uint64_t size, uint32_t value); @@ -48,7 +39,7 @@ private: PageManager m_page_manager; ResourceMutex m_resource_mutex; BufferCache m_buffer_cache; - TextureCache m_texture_cache; + TextureCache m_texture_cache; }; } // namespace Libs::Graphics diff --git a/src/graphics/host_gpu/renderer/render.h b/src/graphics/host_gpu/renderer/render.h index 24a85b54..d55c7b3c 100644 --- a/src/graphics/host_gpu/renderer/render.h +++ b/src/graphics/host_gpu/renderer/render.h @@ -87,25 +87,27 @@ public: [[nodiscard]] int GetQueue() const { return m_queue; } VulkanCommandPool* GetPool() { return m_pool; } [[nodiscard]] bool IsExecute() const { return m_execute; } + [[nodiscard]] uint64_t GetRecordingGeneration() const { return m_recording_generation; } private: friend class BufferCache; void RecycleDescriptorsAfterFence(); - VulkanCommandPool* m_pool = nullptr; - uint32_t m_index = static_cast(-1); - int m_queue = -1; - bool m_execute = false; - bool m_fence_waited = false; - uint64_t m_submit_seq = 0; - uint32_t m_debug_op = 0; - uint64_t m_debug_submit_id = 0; - uint32_t m_debug_arg0 = 0; - uint32_t m_debug_arg1 = 0; - uint32_t m_debug_arg2 = 0; - uint32_t m_debug_arg3 = 0; - uint64_t m_debug_arg4 = 0; + VulkanCommandPool* m_pool = nullptr; + uint32_t m_index = static_cast(-1); + int m_queue = -1; + bool m_execute = false; + bool m_fence_waited = false; + uint64_t m_submit_seq = 0; + uint64_t m_recording_generation = 0; + uint32_t m_debug_op = 0; + uint64_t m_debug_submit_id = 0; + uint32_t m_debug_arg0 = 0; + uint32_t m_debug_arg1 = 0; + uint32_t m_debug_arg2 = 0; + uint32_t m_debug_arg3 = 0; + uint64_t m_debug_arg4 = 0; std::vector m_delete_after_fence; FenceResourceRetainer m_fence_resources; std::vector m_descriptor_sets_after_fence; @@ -129,8 +131,8 @@ void RenderDispatchDirect(uint64_t submit_id, CommandBuffer* buffer, HW::Context void GraphicsRenderInit(); void GraphicsRenderCreateContext(); -[[nodiscard]] bool GraphicsScaleReferenceClock(uint64_t host_ticks, uint64_t host_frequency, - uint64_t* value); +[[nodiscard]] bool GraphicsScaleReferenceClock(uint64_t host_ticks, uint64_t host_frequency, + uint64_t* value); [[nodiscard]] uint64_t GraphicsRenderReadReferenceClock(); [[nodiscard]] bool ResolveComputeImageClear(const ShaderComputeInputInfo& input, uint32_t group_x, @@ -140,21 +142,22 @@ void GraphicsRenderCreateContext(); [[nodiscard]] bool ResolveHtileClearTarget(const HW::DepthRenderTarget& target, uint64_t descriptor_size, HtileClearTarget* resolved); -void GraphicsRenderWriteAtEndOfPipe64(uint64_t submit_id, CommandBuffer* buffer, - uint64_t* dst_gpu_addr, uint64_t value); -void GraphicsRenderWriteAtEndOfPipeClockCounter(uint64_t submit_id, CommandBuffer* buffer, - uint64_t* dst_gpu_addr); -void GraphicsRenderWriteAtEndOfPipeClockCounterWithWriteBack(uint64_t submit_id, - CommandBuffer* buffer, - uint64_t* dst_gpu_addr); -void GraphicsRenderWriteAtEndOfPipe32(uint64_t submit_id, CommandBuffer* buffer, - uint32_t* dst_gpu_addr, uint32_t value); -void GraphicsRenderWriteAtEndOfPipeGds32(uint64_t submit_id, CommandBuffer* buffer, - uint32_t* dst_gpu_addr, uint32_t dw_offset, - uint32_t dw_num); +void GraphicsRenderWriteAtEndOfPipe64(uint64_t submit_id, CommandBuffer* buffer, + uint64_t* dst_gpu_addr, uint64_t value); +void GraphicsRenderWriteAtEndOfPipeClockCounter(uint64_t submit_id, CommandBuffer* buffer, + uint64_t* dst_gpu_addr, uint64_t value); +void GraphicsRenderWriteAtEndOfPipeClockCounterWithWriteBack(uint64_t submit_id, + CommandBuffer* buffer, + uint64_t* dst_gpu_addr, + uint64_t value); +void GraphicsRenderWriteAtEndOfPipe32(uint64_t submit_id, CommandBuffer* buffer, + uint32_t* dst_gpu_addr, uint32_t value); +void GraphicsRenderWriteAtEndOfPipeGds32(uint64_t submit_id, CommandBuffer* buffer, + uint32_t* dst_gpu_addr, uint32_t dw_offset, + uint32_t dw_num); uint64_t GraphicsRenderPrepareDisplayBufferFlip(CommandBuffer* buffer, int handle, int index, int flip_mode, int64_t flip_arg); -void GraphicsRenderWriteAtEndOfPipeWithInterruptWriteBackFlip32( +void GraphicsRenderWriteAtEndOfPipeWithInterruptWriteBackFlip32( uint64_t submit_id, CommandBuffer* buffer, uint32_t* dst_gpu_addr, uint32_t value, int handle, int index, int flip_mode, int64_t flip_arg, uint64_t request_id); void GraphicsRenderWriteAtEndOfPipeWithFlip32(uint64_t submit_id, CommandBuffer* buffer, @@ -193,7 +196,6 @@ int GraphicsRenderDeleteEqEvent(LibKernel::EventQueue::KernelEqueue eq, int id) void GraphicsRenderTriggerAgcUserInterrupt(); void GraphicsRenderTriggerEopEvent(uint32_t context_id); -void GraphicsRenderClearGds(uint64_t dw_offset, uint32_t dw_num, uint32_t clear_value); void GraphicsRenderReadGds(uint32_t* dst, uint32_t dw_offset, uint32_t dw_size); } // namespace Libs::Graphics diff --git a/src/graphics/host_gpu/renderer/renderCompute.cpp b/src/graphics/host_gpu/renderer/renderCompute.cpp index 8988e6bd..72bd98d6 100644 --- a/src/graphics/host_gpu/renderer/renderCompute.cpp +++ b/src/graphics/host_gpu/renderer/renderCompute.cpp @@ -1,7 +1,3 @@ -#include "graphics/host_gpu/renderer/render.h" - -#include "graphics/host_gpu/renderer/renderInternal.h" - #include "common/assert.h" #include "common/common.h" #include "common/emulatorConfig.h" @@ -19,7 +15,9 @@ #include "graphics/host_gpu/renderer/descriptorCache.h" #include "graphics/host_gpu/renderer/framebufferCache.h" #include "graphics/host_gpu/renderer/pipelineCache.h" +#include "graphics/host_gpu/renderer/render.h" #include "graphics/host_gpu/renderer/renderContext.h" +#include "graphics/host_gpu/renderer/renderInternal.h" #include "graphics/host_gpu/renderer/renderState.h" #include "graphics/host_gpu/renderer/shaderResourceBarrier.h" #include "graphics/host_gpu/renderer/shaderSubgroup.h" @@ -547,30 +545,38 @@ void RenderDispatchDirect(uint64_t submit_id, CommandBuffer* buffer, HW::Context return; } - auto* vk_buffer = buffer->GetPool()->buffers[buffer->GetIndex()]; - auto* pipeline = g_render_ctx->GetPipelineCache()->CreateComputePipeline( - &input_info, &sh_ctx->GetCs(), cs_shader); + for (;;) { + const auto recording_generation = buffer->GetRecordingGeneration(); + auto* vk_buffer = buffer->GetPool()->buffers[buffer->GetIndex()]; + auto* pipeline = g_render_ctx->GetPipelineCache()->CreateComputePipeline( + &input_info, &sh_ctx->GetCs(), cs_shader); - vkCmdBindPipeline(vk_buffer, VK_PIPELINE_BIND_POINT_COMPUTE, pipeline->pipeline); + vkCmdBindPipeline(vk_buffer, VK_PIPELINE_BIND_POINT_COMPUTE, pipeline->pipeline); - const auto address_writes = BindDescriptors( - submit_id, buffer, VK_PIPELINE_BIND_POINT_COMPUTE, pipeline->pipeline_layout, - input_info.stage, VK_SHADER_STAGE_COMPUTE_BIT, DescriptorCache::Stage::Compute); + const auto address_writes = BindDescriptors( + submit_id, buffer, VK_PIPELINE_BIND_POINT_COMPUTE, pipeline->pipeline_layout, + input_info.stage, VK_SHADER_STAGE_COMPUTE_BIT, DescriptorCache::Stage::Compute); + if (buffer->GetRecordingGeneration() != recording_generation) { + continue; + } - vkCmdDispatch(vk_buffer, thread_group_x, thread_group_y, thread_group_z); + vkCmdDispatch(vk_buffer, thread_group_x, thread_group_y, thread_group_z); - bool has_storage_writes = HasShaderBufferWrites(input_info.stage); - has_storage_writes = MarkShaderAddressWrites(address_writes) || has_storage_writes; - has_storage_writes = - std::any_of(program.info.images.begin(), program.info.images.end(), - [](const auto& image) { - return image.written && - (image.kind == ShaderRecompiler::IR::ResourceKind::StorageImage || - image.kind == ShaderRecompiler::IR::ResourceKind::StorageImageUint); - }) || - has_storage_writes; - if (has_storage_writes) { - ShaderWriteBarrier(vk_buffer, VK_PIPELINE_STAGE_COMPUTE_SHADER_BIT); + bool has_storage_writes = HasShaderBufferWrites(input_info.stage); + has_storage_writes = MarkShaderAddressWrites(address_writes) || has_storage_writes; + has_storage_writes = + std::any_of( + program.info.images.begin(), program.info.images.end(), + [](const auto& image) { + return image.written && + (image.kind == ShaderRecompiler::IR::ResourceKind::StorageImage || + image.kind == ShaderRecompiler::IR::ResourceKind::StorageImageUint); + }) || + has_storage_writes; + if (has_storage_writes) { + ShaderWriteBarrier(vk_buffer, VK_PIPELINE_STAGE_COMPUTE_SHADER_BIT); + } + break; } } diff --git a/src/graphics/host_gpu/renderer/renderDraw.cpp b/src/graphics/host_gpu/renderer/renderDraw.cpp index 53bf5933..f69b16a0 100644 --- a/src/graphics/host_gpu/renderer/renderDraw.cpp +++ b/src/graphics/host_gpu/renderer/renderDraw.cpp @@ -1,7 +1,3 @@ -#include "graphics/host_gpu/renderer/render.h" - -#include "graphics/host_gpu/renderer/renderInternal.h" - #include "common/assert.h" #include "common/common.h" #include "common/emulatorConfig.h" @@ -19,7 +15,9 @@ #include "graphics/host_gpu/renderer/descriptorCache.h" #include "graphics/host_gpu/renderer/framebufferCache.h" #include "graphics/host_gpu/renderer/pipelineCache.h" +#include "graphics/host_gpu/renderer/render.h" #include "graphics/host_gpu/renderer/renderContext.h" +#include "graphics/host_gpu/renderer/renderInternal.h" #include "graphics/host_gpu/renderer/renderState.h" #include "graphics/host_gpu/renderer/shaderResourceBarrier.h" #include "graphics/host_gpu/renderer/shaderSubgroup.h" @@ -603,11 +601,12 @@ struct DrawEmitInfo { uint32_t first_vertex = 0; }; -struct DrawIndexBufferBind { - bool enabled = false; - VulkanBuffer* buffer = nullptr; - VkDeviceSize offset = 0; - VkIndexType type = VK_INDEX_TYPE_UINT16; +struct DrawIndexBufferSource { + bool enabled = false; + uint64_t address = 0; + const void* host_data = nullptr; + uint64_t size = 0; + VkIndexType type = VK_INDEX_TYPE_UINT16; }; static uint64_t VertexBufferDescriptorSize(const ShaderVertexInputBuffer& buffer) { @@ -802,6 +801,32 @@ static void BindDrawVertexBuffers(uint64_t submit_id, CommandBuffer* buffer, } } +static void BindDrawIndexBuffer(CommandBuffer* buffer, VkCommandBuffer vk_buffer, + const DrawIndexBufferSource& source) { + if (!source.enabled) { + return; + } + EXIT_IF(source.size == 0); + + VulkanBuffer* index_buffer = nullptr; + VkDeviceSize index_offset = 0; + if (source.host_data != nullptr) { + VkDeviceSize range = 0; + if (!g_render_ctx->GetBufferCache()->UploadHostData( + buffer, g_render_ctx->GetGraphicCtx(), source.host_data, source.size, 16, + &index_buffer, &index_offset, &range)) { + EXIT("failed to upload host index buffer\n"); + } + } else { + auto binding = g_render_ctx->GetBufferCache()->ObtainBuffer( + buffer, g_render_ctx->GetGraphicCtx(), source.address, source.size); + index_buffer = binding.first; + index_offset = binding.second; + } + EXIT_IF(index_buffer == nullptr); + vkCmdBindIndexBuffer(vk_buffer, index_buffer->buffer, index_offset, source.type); +} + static void LogDrawStateIfNeeded(HW::Context* ctx, HW::UserConfig* ucfg, const DrawCallInfo& draw, const DrawRenderState& state, bool always_log, bool force_legacy_rect_log, uint32_t index_type_and_size, @@ -898,7 +923,7 @@ static void EmitDrawPrimitives(const HW::UserConfig* ucfg, VkCommandBuffer vk_bu static void ExecutePreparedDraw(uint64_t submit_id, CommandBuffer* buffer, HW::Context* ctx, HW::UserConfig* ucfg, HW::Shader* sh_ctx, const DrawCallInfo& draw, DrawRenderState* state, VkPrimitiveTopology topology, - const DrawEmitInfo& emit, const DrawIndexBufferBind& index_bind, + const DrawEmitInfo& emit, const DrawIndexBufferSource& index_source, bool log_pipeline_phase, bool set_bind_debug, bool set_auto_debug) { EXIT_IF(buffer == nullptr); EXIT_IF(ctx == nullptr); @@ -907,86 +932,93 @@ static void ExecutePreparedDraw(uint64_t submit_id, CommandBuffer* buffer, HW::C EXIT_IF(draw.name == nullptr); EXIT_IF(state == nullptr); - if (log_pipeline_phase) { - LogDrawPhase(draw.name, "CreatePipeline"); - } - auto* pipeline = g_render_ctx->GetPipelineCache()->CreateGraphicsPipeline( - state->framebuffer, state->color_info, state->color_count, &state->depth_info, - &state->vs_input_info, ctx, sh_ctx, &state->ps_input_info, topology, state->ps_active, - state->vs_shader, state->ps_shader); + for (;;) { + const auto recording_generation = buffer->GetRecordingGeneration(); + if (log_pipeline_phase) { + LogDrawPhase(draw.name, "CreatePipeline"); + } + auto* pipeline = g_render_ctx->GetPipelineCache()->CreateGraphicsPipeline( + state->framebuffer, state->color_info, state->color_count, &state->depth_info, + &state->vs_input_info, ctx, sh_ctx, &state->ps_input_info, topology, state->ps_active, + state->vs_shader, state->ps_shader); - if (set_bind_debug) { - SetDrawDebugPhase(buffer, submit_id, draw, 0x100u); - } - vkCmdBindPipeline(state->vk_buffer, VK_PIPELINE_BIND_POINT_GRAPHICS, pipeline->pipeline); - const auto dynamic_params = - BuildGraphicsDynamicParams(*ctx, state->color_info, state->color_count, state->depth_info); - SetDynamicParams(state->vk_buffer, dynamic_params); + if (set_bind_debug) { + SetDrawDebugPhase(buffer, submit_id, draw, 0x100u); + } + vkCmdBindPipeline(state->vk_buffer, VK_PIPELINE_BIND_POINT_GRAPHICS, pipeline->pipeline); + const auto dynamic_params = BuildGraphicsDynamicParams( + *ctx, state->color_info, state->color_count, state->depth_info); + SetDynamicParams(state->vk_buffer, dynamic_params); - // EXIT_NOT_IMPLEMENTED(vs_input_info.buffers_num > 1); - BindDrawVertexBuffers(submit_id, buffer, draw, state->vk_buffer, state->vs_input_info); + // EXIT_NOT_IMPLEMENTED(vs_input_info.buffers_num > 1); + BindDrawVertexBuffers(submit_id, buffer, draw, state->vk_buffer, state->vs_input_info); - LogDrawPhase(draw.name, "BindDescriptorsVS"); - if (set_auto_debug) { - SetDrawDebugPhase(buffer, submit_id, draw, 0x200u); - } - const auto vs_address_writes = BindDescriptors( - submit_id, buffer, VK_PIPELINE_BIND_POINT_GRAPHICS, pipeline->pipeline_layout, - state->vs_input_info.stage, VK_SHADER_STAGE_VERTEX_BIT, DescriptorCache::Stage::Vertex); - - std::vector ps_address_writes; - if (state->ps_active) { - LogDrawPhase(draw.name, "BindDescriptorsPS"); + LogDrawPhase(draw.name, "BindDescriptorsVS"); if (set_auto_debug) { - SetDrawDebugPhase(buffer, submit_id, draw, 0x300u); + SetDrawDebugPhase(buffer, submit_id, draw, 0x200u); } - ps_address_writes = - BindDescriptors(submit_id, buffer, VK_PIPELINE_BIND_POINT_GRAPHICS, - pipeline->pipeline_layout, state->ps_input_info.stage, - VK_SHADER_STAGE_FRAGMENT_BIT, DescriptorCache::Stage::Pixel); - } + const auto vs_address_writes = BindDescriptors( + submit_id, buffer, VK_PIPELINE_BIND_POINT_GRAPHICS, pipeline->pipeline_layout, + state->vs_input_info.stage, VK_SHADER_STAGE_VERTEX_BIT, DescriptorCache::Stage::Vertex); - if (index_bind.enabled) { - EXIT_NOT_IMPLEMENTED(index_bind.buffer == nullptr); - vkCmdBindIndexBuffer(state->vk_buffer, index_bind.buffer->buffer, index_bind.offset, - index_bind.type); - } - - LogDrawPhase(draw.name, "BeginRenderPass"); - if (set_auto_debug) { - SetDrawDebugPhase(buffer, submit_id, draw, 0x400u); - } - buffer->BeginRenderPass(state->framebuffer, state->color_info, state->color_count, - &state->depth_info); - if (set_auto_debug) { - SetDrawDebugPhase(buffer, submit_id, draw, 0x500u); - } - - EmitDrawPrimitives(ucfg, state->vk_buffer, state->vs_input_info, draw, emit); - - if (set_auto_debug) { - SetDrawDebugPhase(buffer, submit_id, draw, 0x600u); - } - buffer->EndRenderPass(); - VkPipelineStageFlags shader_write_stages = 0; - const bool vs_wrote_buffers = HasShaderBufferWrites(state->vs_input_info.stage); - const bool vs_wrote_addresses = MarkShaderAddressWrites(vs_address_writes); - if (vs_wrote_buffers || vs_wrote_addresses) { - shader_write_stages |= VK_PIPELINE_STAGE_VERTEX_SHADER_BIT; - } - if (state->ps_active) { - const bool ps_wrote_buffers = HasShaderBufferWrites(state->ps_input_info.stage); - const bool ps_wrote_addresses = MarkShaderAddressWrites(ps_address_writes); - if (ps_wrote_buffers || ps_wrote_addresses) { - shader_write_stages |= VK_PIPELINE_STAGE_FRAGMENT_SHADER_BIT; + std::vector ps_address_writes; + if (state->ps_active) { + LogDrawPhase(draw.name, "BindDescriptorsPS"); + if (set_auto_debug) { + SetDrawDebugPhase(buffer, submit_id, draw, 0x300u); + } + ps_address_writes = + BindDescriptors(submit_id, buffer, VK_PIPELINE_BIND_POINT_GRAPHICS, + pipeline->pipeline_layout, state->ps_input_info.stage, + VK_SHADER_STAGE_FRAGMENT_BIT, DescriptorCache::Stage::Pixel); } - } - if (shader_write_stages != 0) { - ShaderWriteBarrier(state->vk_buffer, shader_write_stages); - } - LogDrawPhase(draw.name, "EndRenderPass"); - if (set_auto_debug) { - SetDrawDebugPhase(buffer, submit_id, draw, 0x700u); + if (buffer->GetRecordingGeneration() != recording_generation) { + continue; + } + // Index data may use this command buffer's host stream. Resolve it only after the other + // fault-capable bindings, and rebuild it whenever a fault reset changes the generation. + BindDrawIndexBuffer(buffer, state->vk_buffer, index_source); + if (buffer->GetRecordingGeneration() != recording_generation) { + continue; + } + + LogDrawPhase(draw.name, "BeginRenderPass"); + if (set_auto_debug) { + SetDrawDebugPhase(buffer, submit_id, draw, 0x400u); + } + buffer->BeginRenderPass(state->framebuffer, state->color_info, state->color_count, + &state->depth_info); + if (set_auto_debug) { + SetDrawDebugPhase(buffer, submit_id, draw, 0x500u); + } + + EmitDrawPrimitives(ucfg, state->vk_buffer, state->vs_input_info, draw, emit); + + if (set_auto_debug) { + SetDrawDebugPhase(buffer, submit_id, draw, 0x600u); + } + buffer->EndRenderPass(); + VkPipelineStageFlags shader_write_stages = 0; + const bool vs_wrote_buffers = HasShaderBufferWrites(state->vs_input_info.stage); + const bool vs_wrote_addresses = MarkShaderAddressWrites(vs_address_writes); + if (vs_wrote_buffers || vs_wrote_addresses) { + shader_write_stages |= VK_PIPELINE_STAGE_VERTEX_SHADER_BIT; + } + if (state->ps_active) { + const bool ps_wrote_buffers = HasShaderBufferWrites(state->ps_input_info.stage); + const bool ps_wrote_addresses = MarkShaderAddressWrites(ps_address_writes); + if (ps_wrote_buffers || ps_wrote_addresses) { + shader_write_stages |= VK_PIPELINE_STAGE_FRAGMENT_SHADER_BIT; + } + } + if (shader_write_stages != 0) { + ShaderWriteBarrier(state->vk_buffer, shader_write_stages); + } + LogDrawPhase(draw.name, "EndRenderPass"); + if (set_auto_debug) { + SetDrawDebugPhase(buffer, submit_id, draw, 0x700u); + } + break; } } @@ -1068,7 +1100,7 @@ void RenderDrawIndex(uint64_t submit_id, CommandBuffer* buffer, HW::Context* ctx index_type = VK_INDEX_TYPE_UINT32; index_size = 4 * static_cast(index_count); break; - // Some games use it + // Some games use it - need vulkan extension case Prospero::IndexType::kIndex8: index_type = VK_INDEX_TYPE_UINT16; index_size = static_cast(index_count); @@ -1086,6 +1118,25 @@ void RenderDrawIndex(uint64_t submit_id, CommandBuffer* buffer, HW::Context* ctx const DrawCallInfo draw {"DrawIndex", CommandBufferDebugOp::DrawIndex, index_count, flags, instance_count, first_instance}; + std::vector expanded_indices; + if (expand_index8_to_u16) { + EXIT_NOT_IMPLEMENTED(index_addr == nullptr); + const auto* src = static_cast(index_addr); + expanded_indices.resize(index_count); + for (uint32_t i = 0; i < index_count; i++) { + expanded_indices[i] = src[i]; + } + } + + DrawIndexBufferSource index_source {}; + index_source.enabled = true; + index_source.address = reinterpret_cast(index_addr); + index_source.host_data = + expanded_indices.empty() ? nullptr : static_cast(expanded_indices.data()); + index_source.size = expanded_indices.empty() + ? index_size + : expanded_indices.size() * sizeof(uint16_t); + index_source.type = index_type; DrawRenderState state {}; if (!PrepareDrawRenderState(submit_id, buffer, ctx, ucfg, sh_ctx, draw, @@ -1100,43 +1151,12 @@ void RenderDrawIndex(uint64_t submit_id, CommandBuffer* buffer, HW::Context* ctx const auto vertex_offset = ResolveVertexOffset(ucfg->GetIndexOffset(), state.vs_input_info) + vertex_offset_add; - VkDeviceSize index_offset = 0; - VulkanBuffer* indices = nullptr; - if (expand_index8_to_u16) { - EXIT_NOT_IMPLEMENTED(index_addr == nullptr); - const auto* src = static_cast(index_addr); - std::vector expanded_indices(index_count); - for (uint32_t i = 0; i < index_count; i++) { - expanded_indices[i] = src[i]; - } - VkDeviceSize range = 0; - if (!g_render_ctx->GetBufferCache()->UploadHostData( - buffer, g_render_ctx->GetGraphicCtx(), expanded_indices.data(), - expanded_indices.size() * sizeof(uint16_t), 16, &indices, &index_offset, &range)) { - EXIT("failed to upload expanded 8-bit index buffer\n"); - } - } else { - auto binding = g_render_ctx->GetBufferCache()->ObtainBuffer( - buffer, g_render_ctx->GetGraphicCtx(), reinterpret_cast(index_addr), - index_size); - indices = binding.first; - index_offset = binding.second; - } - - EXIT_NOT_IMPLEMENTED(indices == nullptr); - - DrawIndexBufferBind index_bind {}; - index_bind.enabled = true; - index_bind.buffer = indices; - index_bind.offset = index_offset; - index_bind.type = index_type; - DrawEmitInfo emit {}; emit.indexed = true; emit.vertex_offset = vertex_offset; ExecutePreparedDraw(submit_id, buffer, ctx, ucfg, sh_ctx, draw, &state, topology, emit, - index_bind, true, true, false); + index_source, true, true, false); } // NOLINTNEXTLINE(readability-function-cognitive-complexity) @@ -1243,9 +1263,9 @@ void RenderDrawIndexAuto(uint64_t submit_id, CommandBuffer* buffer, HW::Context* emit.draw_vertex_count = draw_vertex_count; emit.first_vertex = static_cast(vertex_offset); - DrawIndexBufferBind index_bind {}; + DrawIndexBufferSource index_source {}; ExecutePreparedDraw(submit_id, buffer, ctx, ucfg, sh_ctx, draw, &state, topology, emit, - index_bind, false, false, true); + index_source, false, false, true); } bool IsSameColorResolveSubresource(const RenderColorInfo& src, const RenderColorInfo& dst) { diff --git a/src/graphics/host_gpu/renderer/resourceMutex.h b/src/graphics/host_gpu/renderer/resourceMutex.h index b6ea1c1a..c717b776 100644 --- a/src/graphics/host_gpu/renderer/resourceMutex.h +++ b/src/graphics/host_gpu/renderer/resourceMutex.h @@ -8,8 +8,8 @@ namespace Libs::Graphics { -// Owner-tracked shared buffer/image transaction. External faults pause GPU submissions before -// taking this mutex; synchronous command-processor accesses reuse their already-owned transaction. +// Owner-tracked shared buffer/image transaction. External faults pause GPU submissions first; +// command-processor faults drain their queue before entering this transaction. class ResourceMutex final { public: class FaultScope final { @@ -27,8 +27,8 @@ public: ~ResourceMutex(); KYTY_CLASS_NO_COPY(ResourceMutex); - void lock(); - void unlock(); + void lock(); + void unlock(); [[nodiscard]] bool IsOwnedByCurrentThread(); private: diff --git a/src/graphics/host_gpu/renderer/sync.cpp b/src/graphics/host_gpu/renderer/sync.cpp index 4fb952b8..c20f11c1 100644 --- a/src/graphics/host_gpu/renderer/sync.cpp +++ b/src/graphics/host_gpu/renderer/sync.cpp @@ -13,31 +13,29 @@ #include "kernel/pthread.h" #include "libs/errno.h" -#include #include namespace Libs::Graphics { -constexpr int GRAPHICS_EVENT_QUEUED_GRAPHICS_INTERRUPT = 0x00; -constexpr int GRAPHICS_EVENT_EOP = 0x40; -constexpr uint64_t GRAPHICS_REFERENCE_CLOCK_FREQUENCY = 100000000; +constexpr int GRAPHICS_EVENT_QUEUED_GRAPHICS_INTERRUPT = 0x00; +constexpr int GRAPHICS_EVENT_EOP = 0x40; +constexpr uint64_t GRAPHICS_REFERENCE_CLOCK_FREQUENCY = 100000000; bool GraphicsScaleReferenceClock(uint64_t host_ticks, uint64_t host_frequency, uint64_t* value) { if (host_frequency == 0 || value == nullptr) { return false; } - const auto whole_seconds = host_ticks / host_frequency; - const auto remainder = host_ticks % host_frequency; - constexpr auto MAX_VALUE = std::numeric_limits::max(); + const auto whole_seconds = host_ticks / host_frequency; + const auto remainder = host_ticks % host_frequency; + constexpr auto MAX_VALUE = std::numeric_limits::max(); if (whole_seconds > MAX_VALUE / GRAPHICS_REFERENCE_CLOCK_FREQUENCY || remainder > MAX_VALUE / GRAPHICS_REFERENCE_CLOCK_FREQUENCY) { return false; } - const auto whole_value = whole_seconds * GRAPHICS_REFERENCE_CLOCK_FREQUENCY; - const auto fractional_value = - (remainder * GRAPHICS_REFERENCE_CLOCK_FREQUENCY) / host_frequency; + const auto whole_value = whole_seconds * GRAPHICS_REFERENCE_CLOCK_FREQUENCY; + const auto fractional_value = (remainder * GRAPHICS_REFERENCE_CLOCK_FREQUENCY) / host_frequency; if (whole_value > MAX_VALUE - fractional_value) { return false; } @@ -50,40 +48,19 @@ uint64_t GraphicsRenderReadReferenceClock() { const auto host_ticks = LibKernel::KernelReadTsc(); uint64_t value = 0; if (!GraphicsScaleReferenceClock(host_ticks, host_frequency, &value)) { - EXIT("cannot scale host clock, ticks=0x%016" PRIx64 " frequency=%" PRIu64 "\n", - host_ticks, host_frequency); + EXIT("cannot scale host clock, ticks=0x%016" PRIx64 " frequency=%" PRIu64 "\n", host_ticks, + host_frequency); } return value; } -static void SubmitLabel32(CommandBuffer* buffer, uint32_t* dst, uint32_t value, - LabelCallback callback_1 = nullptr, LabelCallback callback_2 = nullptr, - const uint64_t* args = nullptr) { - auto* label = - LabelCreate32(g_render_ctx->GetGraphicCtx(), dst, value, callback_1, callback_2, args); +static void SubmitLabel(CommandBuffer* buffer, LabelCallback callback_1 = nullptr, + LabelCallback callback_2 = nullptr, const uint64_t* args = nullptr) { + auto* label = LabelCreate(g_render_ctx->GetGraphicCtx(), callback_1, callback_2, args); LabelSet(buffer, label); LabelDelete(label); } -static void SubmitLabel64(CommandBuffer* buffer, uint64_t* dst, uint64_t value, - LabelCallback callback_1 = nullptr, LabelCallback callback_2 = nullptr, - const uint64_t* args = nullptr) { - auto* label = - LabelCreate64(g_render_ctx->GetGraphicCtx(), dst, value, callback_1, callback_2, args); - LabelSet(buffer, label); - LabelDelete(label); -} - -template -static void PublishImmediateFence(T* dst, T value) { - static_assert(sizeof(T) == sizeof(uint32_t) || sizeof(T) == sizeof(uint64_t)); - EXIT_IF(dst == nullptr); - // EOP/EOS fence payloads are constant and published while parsing the packet. Kyty performs the - // direct mapped write under CommandProcessorGuestAccessScope: a protected access is first resolved by - // BufferCache inline, then this instruction retries so the EOP payload wins over prior GPU bytes. - std::memcpy(dst, &value, sizeof(value)); -} - static bool CompleteDisplayBufferFlip(const uint64_t* args) { if (g_render_ctx == nullptr || args == nullptr) { EXIT("GPU flip completion has invalid state, render_ctx=%p args=%p\n", @@ -114,8 +91,6 @@ void GraphicsRenderWriteAtEndOfPipe32(uint64_t submit_id, CommandBuffer* buffer, buffer->SetDebugInfo(static_cast(CommandBufferDebugOp::EopWrite), submit_id, 4, value, 0, 0, reinterpret_cast(dst_gpu_addr)); - - PublishImmediateFence(dst_gpu_addr, value); } void GraphicsRenderWriteAtEndOfPipeGds32(uint64_t submit_id, CommandBuffer* buffer, @@ -128,28 +103,6 @@ void GraphicsRenderWriteAtEndOfPipeGds32(uint64_t submit_id, CommandBuffer* buff buffer->SetDebugInfo(static_cast(CommandBufferDebugOp::EopWrite), submit_id, dw_offset, dw_num, 0, 0, reinterpret_cast(dst_gpu_addr)); - - uint64_t args[LABEL_ARGS_MAX] = {static_cast(dw_offset), - static_cast(dw_num), - reinterpret_cast(dst_gpu_addr), 0}; - - SubmitLabel32( - buffer, dst_gpu_addr, 0, - [](const uint64_t* args) { - auto dw_offset = static_cast(args[0]); - auto dw_num = static_cast(args[1]); - auto* dst_gpu_addr = reinterpret_cast(args[2]); - if (dst_gpu_addr == nullptr || dw_num == 0) { - EXIT("invalid asynchronous EOP GDS write\n"); - } - const auto size = static_cast(dw_num) * sizeof(uint32_t); - auto transaction = g_render_ctx->GetGpuResources()->LockAsynchronousGuestWrite( - reinterpret_cast(dst_gpu_addr), size); - g_render_ctx->GetGdsBuffer()->Read(g_render_ctx->GetGraphicCtx(), dst_gpu_addr, - dw_offset, dw_num); - return false; - }, - nullptr, args); } void GraphicsRenderWriteAtEndOfPipe64(uint64_t submit_id, CommandBuffer* buffer, @@ -162,12 +115,10 @@ void GraphicsRenderWriteAtEndOfPipe64(uint64_t submit_id, CommandBuffer* buffer, buffer->SetDebugInfo(static_cast(CommandBufferDebugOp::EopWrite), submit_id, 8, static_cast(value), static_cast(value >> 32u), 0, reinterpret_cast(dst_gpu_addr)); - - PublishImmediateFence(dst_gpu_addr, value); } void GraphicsRenderWriteAtEndOfPipeClockCounter(uint64_t submit_id, CommandBuffer* buffer, - uint64_t* dst_gpu_addr) { + uint64_t* dst_gpu_addr, uint64_t value) { EXIT_IF(g_render_ctx == nullptr); EXIT_IF(dst_gpu_addr == nullptr); EXIT_IF(buffer == nullptr); @@ -176,8 +127,6 @@ void GraphicsRenderWriteAtEndOfPipeClockCounter(uint64_t submit_id, CommandBuffe buffer->SetDebugInfo(static_cast(CommandBufferDebugOp::EopWrite), submit_id, 8, 0, 0, 0, reinterpret_cast(dst_gpu_addr)); - const auto value = GraphicsRenderReadReferenceClock(); - PublishImmediateFence(dst_gpu_addr, value); LOGF_COLOR(Log::Color::BrightGreen, "EndOfPipe Signal!!! [0x%016" PRIx64 "] <- Clock: 0x%016" PRIx64 "\n", reinterpret_cast(dst_gpu_addr), value); @@ -185,7 +134,8 @@ void GraphicsRenderWriteAtEndOfPipeClockCounter(uint64_t submit_id, CommandBuffe void GraphicsRenderWriteAtEndOfPipeClockCounterWithWriteBack(uint64_t submit_id, CommandBuffer* buffer, - uint64_t* dst_gpu_addr) { + uint64_t* dst_gpu_addr, + uint64_t value) { EXIT_IF(g_render_ctx == nullptr); EXIT_IF(dst_gpu_addr == nullptr); EXIT_IF(buffer == nullptr); @@ -194,8 +144,6 @@ void GraphicsRenderWriteAtEndOfPipeClockCounterWithWriteBack(uint64_t subm buffer->SetDebugInfo(static_cast(CommandBufferDebugOp::EopWriteBack), submit_id, 8, 0, 0, 0, reinterpret_cast(dst_gpu_addr)); - const auto value = GraphicsRenderReadReferenceClock(); - PublishImmediateFence(dst_gpu_addr, value); LOGF_COLOR(Log::Color::BrightGreen, "EndOfPipe Signal!!! [0x%016" PRIx64 "] <- Clock: 0x%016" PRIx64 "\n", reinterpret_cast(dst_gpu_addr), value); @@ -211,8 +159,6 @@ void GraphicsRenderWriteAtEndOfPipeWithWriteBack64(uint64_t submit_id, CommandBu buffer->SetDebugInfo(static_cast(CommandBufferDebugOp::EopWriteBack), submit_id, 8, static_cast(value), static_cast(value >> 32u), 0, reinterpret_cast(dst_gpu_addr)); - - PublishImmediateFence(dst_gpu_addr, value); } void GraphicsRenderWriteAtEndOfPipeWithWriteBack32(uint64_t submit_id, CommandBuffer* buffer, @@ -224,8 +170,6 @@ void GraphicsRenderWriteAtEndOfPipeWithWriteBack32(uint64_t submit_id, CommandBu buffer->SetDebugInfo(static_cast(CommandBufferDebugOp::EopWriteBack), submit_id, 4, value, 0, 0, reinterpret_cast(dst_gpu_addr)); - - PublishImmediateFence(dst_gpu_addr, value); } void GraphicsRenderWriteAtEndOfPipeWithInterruptWriteBack64(uint64_t submit_id, @@ -241,12 +185,10 @@ void GraphicsRenderWriteAtEndOfPipeWithInterruptWriteBack64(uint64_t submi context_id, static_cast(value), static_cast(value >> 32u), reinterpret_cast(dst_gpu_addr)); - PublishImmediateFence(dst_gpu_addr, value); - uint64_t args[LABEL_ARGS_MAX] = {context_id != 0 ? context_id : value}; - SubmitLabel64( - buffer, nullptr, 0, nullptr, + SubmitLabel( + buffer, nullptr, [](const uint64_t* args) { EXIT_IF(g_render_ctx == nullptr); g_render_ctx->TriggerEopEvent(static_cast(args[0])); @@ -266,12 +208,10 @@ void GraphicsRenderWriteAtEndOfPipeWithInterruptWriteBack32(uint64_t submi buffer->SetDebugInfo(static_cast(CommandBufferDebugOp::EopWriteBack), submit_id, 4, context_id, value, 0, reinterpret_cast(dst_gpu_addr)); - PublishImmediateFence(dst_gpu_addr, value); - uint64_t args[LABEL_ARGS_MAX] = {context_id != 0 ? context_id : value}; - SubmitLabel32( - buffer, nullptr, 0, nullptr, + SubmitLabel( + buffer, nullptr, [](const uint64_t* args) { EXIT_IF(g_render_ctx == nullptr); g_render_ctx->TriggerEopEvent(static_cast(args[0])); @@ -292,12 +232,10 @@ void GraphicsRenderWriteAtEndOfPipeWithInterrupt64(uint64_t submit_id, CommandBu context_id, static_cast(value), static_cast(value >> 32u), reinterpret_cast(dst_gpu_addr)); - PublishImmediateFence(dst_gpu_addr, value); - uint64_t args[LABEL_ARGS_MAX] = {context_id != 0 ? context_id : value}; - SubmitLabel64( - buffer, nullptr, 0, nullptr, + SubmitLabel( + buffer, nullptr, [](const uint64_t* args) { EXIT_IF(g_render_ctx == nullptr); g_render_ctx->TriggerEopEvent(static_cast(args[0])); @@ -316,12 +254,10 @@ void GraphicsRenderWriteAtEndOfPipeWithInterrupt32(uint64_t submit_id, CommandBu buffer->SetDebugInfo(static_cast(CommandBufferDebugOp::EopInterrupt), submit_id, 4, context_id, value, 0, reinterpret_cast(dst_gpu_addr)); - PublishImmediateFence(dst_gpu_addr, value); - uint64_t args[LABEL_ARGS_MAX] = {context_id != 0 ? context_id : value}; - SubmitLabel32( - buffer, nullptr, 0, nullptr, + SubmitLabel( + buffer, nullptr, [](const uint64_t* args) { EXIT_IF(g_render_ctx == nullptr); g_render_ctx->TriggerEopEvent(static_cast(args[0])); @@ -360,11 +296,9 @@ void GraphicsRenderWriteAtEndOfPipeWithInterruptWriteBackFlip32( buffer->SetDebugInfo(static_cast(CommandBufferDebugOp::EopWriteBackFlip), submit_id, static_cast(handle), static_cast(index), static_cast(flip_mode), value, static_cast(flip_arg)); - PublishImmediateFence(dst_gpu_addr, value); - const uint64_t args[LABEL_ARGS_MAX] = {request_id}; - SubmitLabel32( - buffer, nullptr, 0, CompleteDisplayBufferFlip, + SubmitLabel( + buffer, CompleteDisplayBufferFlip, [](const uint64_t* /*args*/) { EXIT_IF(g_render_ctx == nullptr); g_render_ctx->TriggerEopEvent(0); @@ -385,10 +319,8 @@ void GraphicsRenderWriteAtEndOfPipeWithFlip32(uint64_t submit_id, CommandBuffer* buffer->SetDebugInfo(static_cast(CommandBufferDebugOp::EopFlip), submit_id, static_cast(handle), static_cast(index), static_cast(flip_mode), value, static_cast(flip_arg)); - PublishImmediateFence(dst_gpu_addr, value); - const uint64_t args[LABEL_ARGS_MAX] = {request_id}; - SubmitLabel32(buffer, nullptr, 0, CompleteDisplayBufferFlip, nullptr, args); + SubmitLabel(buffer, CompleteDisplayBufferFlip, nullptr, args); } void GraphicsRenderWriteAtEndOfPipeOnlyFlip(uint64_t submit_id, CommandBuffer* buffer, int handle, @@ -403,7 +335,7 @@ void GraphicsRenderWriteAtEndOfPipeOnlyFlip(uint64_t submit_id, CommandBuffer* b static_cast(flip_mode), 0, static_cast(flip_arg)); const uint64_t args[LABEL_ARGS_MAX] = {request_id}; - SubmitLabel32(buffer, nullptr, 0, CompleteDisplayBufferFlip, nullptr, args); + SubmitLabel(buffer, CompleteDisplayBufferFlip, nullptr, args); } void GraphicsRenderTriggerEopEventAtEndOfPipe(CommandBuffer* buffer, uint32_t context_id) { @@ -413,8 +345,8 @@ void GraphicsRenderTriggerEopEventAtEndOfPipe(CommandBuffer* buffer, uint32_t co uint64_t args[LABEL_ARGS_MAX] = {static_cast(context_id)}; - SubmitLabel32( - buffer, nullptr, 0, nullptr, + SubmitLabel( + buffer, nullptr, [](const uint64_t* args) { EXIT_IF(g_render_ctx == nullptr); g_render_ctx->TriggerEopEvent(static_cast(args[0])); @@ -489,13 +421,6 @@ int GraphicsRenderDeleteEqEvent(LibKernel::EventQueue::KernelEqueue eq, int id) return result; } -void GraphicsRenderClearGds(uint64_t dw_offset, uint32_t dw_num, uint32_t clear_value) { - EXIT_IF(g_render_ctx == nullptr); - - g_render_ctx->GetGdsBuffer()->Clear(g_render_ctx->GetGraphicCtx(), dw_offset, dw_num, - clear_value); -} - void GraphicsRenderReadGds(uint32_t* dst, uint32_t dw_offset, uint32_t dw_size) { EXIT_IF(g_render_ctx == nullptr); diff --git a/src/graphics/host_gpu/renderer/textureCache.cpp b/src/graphics/host_gpu/renderer/textureCache.cpp index f4d0ff52..58b2cb88 100644 --- a/src/graphics/host_gpu/renderer/textureCache.cpp +++ b/src/graphics/host_gpu/renderer/textureCache.cpp @@ -8,7 +8,6 @@ #include "graphics/guest_gpu/graphicsRun.h" #include "graphics/guest_gpu/tile.h" #include "graphics/host_gpu/graphicContext.h" -#include "graphics/host_gpu/hostMemory.h" #include "graphics/host_gpu/objects/label.h" #include "graphics/host_gpu/objects/textureCommon.h" #include "graphics/host_gpu/renderer/bufferCache.h" @@ -232,14 +231,15 @@ struct TextureCache::ReadbackWorker { } void Request(PageFaultAccess fault_access, uint64_t fault_vaddr, uint64_t fault_size) noexcept { - const bool submissions_prepaused_now = GraphicsRunSubmissionLockHeld(); + const bool command_thread = GraphicsRunIsCommandProcessorThread(); + const bool submissions_prepaused_now = GraphicsRunSubmissionLockHeld() || command_thread; const bool unsafe_gpu_lock = GraphicsRunGpuLockHeld() && !submissions_prepaused_now; if ((fault_access != PageFaultAccess::Read && fault_access != PageFaultAccess::Write) || - GraphicsRunIsCommandProcessorThread() || unsafe_gpu_lock || LabelInCallback() || - g_texture_cache_lock_owner != nullptr || g_texture_fault_owner != &cache) { + unsafe_gpu_lock || LabelInCallback() || g_texture_cache_lock_owner != nullptr || + g_texture_fault_owner != &cache) { EXIT("TextureCache: unsafe image readback request, access=%u command_thread=%d " "submission_lock=%d gpu_lock=%d label_callback=%d cache_lock=%p fault_owner=%p\n", - static_cast(fault_access), GraphicsRunIsCommandProcessorThread(), + static_cast(fault_access), command_thread, GraphicsRunSubmissionLockHeld(), GraphicsRunGpuLockHeld(), LabelInCallback(), g_texture_cache_lock_owner, g_texture_fault_owner); } @@ -1557,13 +1557,12 @@ VkImageView TextureCache::GetRenderTargetStorageView(GraphicContext* c VkImageView TextureCache::GetStorageTextureSampledView(GraphicContext* ctx, StorageTextureVulkanImage* image, const ImageInfo& info) { - const auto shape = SelectStorageSampledViewShape( - info.type, info.depth, image != nullptr ? image->layers : 0); + const auto shape = + SelectStorageSampledViewShape(info.type, info.depth, image != nullptr ? image->layers : 0); if (ctx == nullptr || image == nullptr || image->image == nullptr || - shape == StorageSampledViewShape::Unsupported || - info.base_array != 0 || info.levels != image->mip_levels || - info.base_level >= info.levels || info.view_levels == 0 || - info.base_level + info.view_levels > info.levels) { + shape == StorageSampledViewShape::Unsupported || info.base_array != 0 || + info.levels != image->mip_levels || info.base_level >= info.levels || + info.view_levels == 0 || info.base_level + info.view_levels > info.levels) { EXIT("TextureCache: invalid sampled view of storage texture, image=%p type=%u depth=%u" " base=%u levels=%u view_levels=%u image_levels=%u base_array=%u\n", static_cast(image), info.type, info.depth, info.base_level, info.levels, @@ -1589,23 +1588,21 @@ VkImageView TextureCache::GetStorageTextureSampledView(GraphicContext* usage.sType = VK_STRUCTURE_TYPE_IMAGE_VIEW_USAGE_CREATE_INFO; usage.usage = VK_IMAGE_USAGE_SAMPLED_BIT; VkImageViewCreateInfo create {}; - create.sType = VK_STRUCTURE_TYPE_IMAGE_VIEW_CREATE_INFO; - create.pNext = &usage; - create.image = image->image; - create.viewType = shape == StorageSampledViewShape::Image3D - ? VK_IMAGE_VIEW_TYPE_3D - : shape == StorageSampledViewShape::Image2DArray - ? VK_IMAGE_VIEW_TYPE_2D_ARRAY - : VK_IMAGE_VIEW_TYPE_2D; - create.format = view_format; - create.components = TextureGetComponentMapping(info.swizzle); - create.subresourceRange.aspectMask = VK_IMAGE_ASPECT_COLOR_BIT; - create.subresourceRange.baseMipLevel = info.base_level; - create.subresourceRange.levelCount = info.view_levels; + create.sType = VK_STRUCTURE_TYPE_IMAGE_VIEW_CREATE_INFO; + create.pNext = &usage; + create.image = image->image; + create.viewType = shape == StorageSampledViewShape::Image3D ? VK_IMAGE_VIEW_TYPE_3D + : shape == StorageSampledViewShape::Image2DArray ? VK_IMAGE_VIEW_TYPE_2D_ARRAY + : VK_IMAGE_VIEW_TYPE_2D; + create.format = view_format; + create.components = TextureGetComponentMapping(info.swizzle); + create.subresourceRange.aspectMask = VK_IMAGE_ASPECT_COLOR_BIT; + create.subresourceRange.baseMipLevel = info.base_level; + create.subresourceRange.levelCount = info.view_levels; create.subresourceRange.baseArrayLayer = 0; create.subresourceRange.layerCount = shape == StorageSampledViewShape::Image2DArray ? info.depth : 1; - VkImageView view = nullptr; + VkImageView view = nullptr; const auto result = vkCreateImageView(ctx->device, &create, nullptr, &view); if (result != VK_SUCCESS || view == nullptr) { EXIT("TextureCache: failed to create sampled view of storage texture, result=%d" @@ -1926,15 +1923,6 @@ VulkanImage* TextureCache::FindTexture(CommandBuffer* command, GraphicContext* c cached->gpu_modified, cached->ctx == ctx); } } - uint64_t readable = 0; - if (!HostMemoryQueryReadable(info.address, info.size, &readable) || readable < info.size) { - EXIT("TextureCache: new sampled image has unreadable guest backing, addr=0x%016" PRIx64 - " size=0x%016" PRIx64 " readable=0x%016" PRIx64 - " format=%u extent=%ux%ux%u pitch=%u levels=%u tile=%u type=%u\n", - info.address, info.size, readable, info.format, info.width, info.height, info.depth, - info.pitch, info.levels, info.tile, info.type); - } - m_images.reserve(m_images.size() + 1); auto cached = std::make_shared(); cached->kind = CachedImage::Kind::Texture; @@ -2057,13 +2045,6 @@ StorageTextureVulkanImage* TextureCache::FindStorageTexture(CommandBuffer* com } RetireStoragePageNeighbors(ctx, info); - uint64_t readable = 0; - if (!HostMemoryQueryReadable(info.address, info.size, &readable) || readable < info.size) { - EXIT("TextureCache: new storage image has unreadable guest backing, addr=0x%016" PRIx64 - " size=0x%016" PRIx64 " readable=0x%016" PRIx64 "\n", - info.address, info.size, readable); - } - m_images.reserve(m_images.size() + 1); auto cached = std::make_shared(); cached->kind = CachedImage::Kind::StorageTexture; @@ -3639,6 +3620,9 @@ bool TextureCache::InvalidateMemory(PageFaultAccess access, uint64_t vaddr, uint } if (needs_readback) { + if (GraphicsRunIsCommandProcessorThread()) { + GraphicsRunFinishCommandProcessors(); + } m_readback->Request(access, vaddr, size); action = m_memory_tracker.BeginCpuFault(vaddr, size, access); if (action != CpuFaultAction::Download) { diff --git a/src/graphics/shader/recompiler/SrtWalker.cpp b/src/graphics/shader/recompiler/SrtWalker.cpp index 1f41bbeb..8cf66357 100644 --- a/src/graphics/shader/recompiler/SrtWalker.cpp +++ b/src/graphics/shader/recompiler/SrtWalker.cpp @@ -1,9 +1,9 @@ #include "graphics/shader/recompiler/SrtWalker.h" -#include "graphics/host_gpu/hostMemory.h" #include "graphics/shader/recompiler/ScalarProvenance.h" #include +#include #include namespace Libs::Graphics::ShaderRecompiler::IR { @@ -485,12 +485,14 @@ private: value.pc, base_aligned, relative)); } } - const auto read_memory = - m_runtime.read_memory != nullptr ? m_runtime.read_memory : HostMemoryReadDword; - if (!read_memory(m_runtime.userdata, address, result)) { + if (m_runtime.read_memory != nullptr && + !m_runtime.read_memory(m_runtime.userdata, address, result)) { return Fail( error, fmt::format("ReadConst pc=0x{:08x} failed at 0x{:016x}", value.pc, address)); } + if (m_runtime.read_memory == nullptr) { + std::memcpy(result, reinterpret_cast(address), sizeof(*result)); + } return true; } diff --git a/src/graphics/shader/shader.cpp b/src/graphics/shader/shader.cpp index ccfed3e0..09ab3a1e 100644 --- a/src/graphics/shader/shader.cpp +++ b/src/graphics/shader/shader.cpp @@ -12,7 +12,6 @@ #include "graphics/guest_gpu/graphicsRun.h" #include "graphics/guest_gpu/hardwareContext.h" #include "graphics/guest_gpu/gpu_defs.h" -#include "graphics/host_gpu/hostMemory.h" #include "graphics/shader/recompiler/ShaderDecoder.h" #include "graphics/shader/recompiler/ShaderRecompiler.h" #include "graphics/shader/shaderVertexMetadata.h" @@ -785,31 +784,6 @@ static bool ShaderGetStaticInputInfoVS(const HW::VertexShaderInfo* regs, shader_addr); return false; } - uint32_t max_semantic = 0; - for (uint32_t i = 0; i < metadata.input_semantics_count; i++) { - max_semantic = - std::max(max_semantic, static_cast(metadata.input_semantics[i].semantic)); - } - if (!HostMemoryRangeIsReadable(reinterpret_cast(attrib), - static_cast(max_semantic + 1) * - sizeof(uint32_t))) { - LOGF("ShaderGetInputInfoVS(): unreadable vertex attribute table shader=0x%016" PRIx64 - "\n", - shader_addr); - return false; - } - uint32_t max_buffer = 0; - for (uint32_t i = 0; i < metadata.input_semantics_count; i++) { - max_buffer = std::max(max_buffer, attrib[metadata.input_semantics[i].semantic] & 0x1fu); - } - if (!HostMemoryRangeIsReadable(reinterpret_cast(buffer), - static_cast(max_buffer + 1) * 4 * - sizeof(uint32_t))) { - LOGF("ShaderGetInputInfoVS(): unreadable vertex buffer table shader=0x%016" PRIx64 "\n", - shader_addr); - return false; - } - ShaderApplyAttribSemantics(info, metadata.input_semantics.data(), metadata.input_semantics_count, attrib, buffer); ShaderDetectBuffers(info); diff --git a/src/graphics/shader/shaderVertexMetadata.cpp b/src/graphics/shader/shaderVertexMetadata.cpp index 6f1f56b6..66a8b330 100644 --- a/src/graphics/shader/shaderVertexMetadata.cpp +++ b/src/graphics/shader/shaderVertexMetadata.cpp @@ -1,7 +1,5 @@ #include "graphics/shader/shaderVertexMetadata.h" -#include "graphics/host_gpu/hostMemory.h" - #include #include @@ -16,10 +14,6 @@ bool Fail(std::string* error, const char* message) { return false; } -bool Readable(const void* data, uint64_t size) { - return data != nullptr && HostMemoryRangeIsReadable(reinterpret_cast(data), size); -} - } // namespace bool ShaderReadVertexMetadata(const ShaderMappedData& data, uint32_t max_user_sgprs, @@ -27,8 +21,8 @@ bool ShaderReadVertexMetadata(const ShaderMappedData& data, uint32_t max_user_sg if (metadata == nullptr) { return Fail(error, "missing vertex metadata output"); } - if (!Readable(data.user_data, sizeof(ShaderUserData))) { - return Fail(error, "unreadable AGC user-data header"); + if (data.user_data == nullptr) { + return Fail(error, "missing AGC user-data header"); } ShaderUserData user_data {}; @@ -44,8 +38,8 @@ bool ShaderReadVertexMetadata(const ShaderMappedData& data, uint32_t max_user_sg const auto direct_size = static_cast(user_data.direct_resource_count) * sizeof(uint16_t); if (direct_size != 0) { - if (!Readable(user_data.direct_resource_offset, direct_size)) { - return Fail(error, "unreadable AGC direct-resource offsets"); + if (user_data.direct_resource_offset == nullptr) { + return Fail(error, "missing AGC direct-resource offsets"); } std::memcpy(direct_offsets.data(), user_data.direct_resource_offset, direct_size); } @@ -84,8 +78,8 @@ bool ShaderReadVertexMetadata(const ShaderMappedData& data, uint32_t max_user_sg const auto semantic_size = static_cast(data.num_input_semantics) * sizeof(ShaderSemantic); - if (!Readable(data.input_semantics, semantic_size)) { - return Fail(error, "unreadable vertex input semantics"); + if (data.input_semantics == nullptr) { + return Fail(error, "missing vertex input semantics"); } std::memcpy(next.input_semantics.data(), data.input_semantics, semantic_size); next.input_semantics_count = data.num_input_semantics; diff --git a/src/libs/agc.cpp b/src/libs/agc.cpp index 9258c6ab..d695952f 100644 --- a/src/libs/agc.cpp +++ b/src/libs/agc.cpp @@ -11,7 +11,6 @@ #include "graphics/guest_gpu/pm4.h" #include "graphics/guest_gpu/gpu_defs.h" #include "graphics/guest_gpu/tile.h" -#include "graphics/host_gpu/hostMemory.h" #include "graphics/host_gpu/objects/label.h" #include "graphics/host_gpu/renderer/render.h" #include "graphics/presentation/renderDoc.h" @@ -373,11 +372,6 @@ struct Label { uint64_t m_reserved[3]; }; -static bool is_readable_command_packet_range(const uint32_t* cmd, uint32_t num_dw) { - return HostMemoryRangeIsReadable(reinterpret_cast(cmd), - static_cast(num_dw) * sizeof(uint32_t)); -} - int KYTY_SYSV_ABI GraphicsInit(uint32_t* state, uint32_t ver) { PRINT_NAME(); @@ -3750,8 +3744,7 @@ static void flush_pending_graphics_segment_before_acb(const uint32_t* acb, if (!wait_addresses.empty() && Gen5::g_pending_graphics_segment.start != nullptr) { auto* scan = Gen5::g_pending_graphics_segment.start; auto* matched_end = Gen5::g_pending_graphics_segment.start; - while (scan < Gen5::g_pending_graphics_segment.end && - Gen5::is_readable_command_packet_range(scan, 1)) { + while (scan < Gen5::g_pending_graphics_segment.end) { auto cmd_id = *scan; if (cmd_id == 0x80000000u) { scan++; @@ -3763,8 +3756,7 @@ static void flush_pending_graphics_segment_before_acb(const uint32_t* acb, auto len = KYTY_PM4_LEN(cmd_id); if (len == 0 || - len > static_cast(Gen5::g_pending_graphics_segment.end - scan) || - !Gen5::is_readable_command_packet_range(scan, len)) { + len > static_cast(Gen5::g_pending_graphics_segment.end - scan)) { break; } @@ -3789,8 +3781,7 @@ static void flush_pending_graphics_segment_before_acb(const uint32_t* acb, Gen5::g_pending_graphics_segment.end > Gen5::g_pending_graphics_segment.start) { auto* scan = Gen5::g_pending_graphics_segment.start; auto* valid_end = Gen5::g_pending_graphics_segment.start; - while (scan < Gen5::g_pending_graphics_segment.end && - Gen5::is_readable_command_packet_range(scan, 1)) { + while (scan < Gen5::g_pending_graphics_segment.end) { auto cmd_id = *scan; if (cmd_id == 0x80000000u) { scan++; @@ -3803,8 +3794,7 @@ static void flush_pending_graphics_segment_before_acb(const uint32_t* acb, auto len = KYTY_PM4_LEN(cmd_id); if (len == 0 || - len > static_cast(Gen5::g_pending_graphics_segment.end - scan) || - !Gen5::is_readable_command_packet_range(scan, len)) { + len > static_cast(Gen5::g_pending_graphics_segment.end - scan)) { break; } @@ -3931,16 +3921,14 @@ int KYTY_SYSV_ABI GraphicsDriverSubmitMultiCommandBuffers(uint32_t queue, uint32 } static void submit_acb(uint32_t queue, uint32_t* acb, uint32_t size_in_dwords) { - if (acb != nullptr && size_in_dwords >= 5 && Gen5::is_readable_command_packet_range(acb, 5)) { + if (acb != nullptr && size_in_dwords >= 5) { auto descriptor_addr = static_cast(acb[0]) | (static_cast(acb[1]) << 32u); auto descriptor_size = acb[2]; auto descriptor_flags = acb[3]; auto descriptor_magic = acb[4]; if (descriptor_addr != 0 && descriptor_size != 0 && descriptor_flags == 0 && - descriptor_magic == 0x5533ccaau && - Gen5::is_readable_command_packet_range(reinterpret_cast(descriptor_addr), - descriptor_size)) { + descriptor_magic == 0x5533ccaau) { LOGF("\t descriptor addr = 0x%016" PRIx64 "\n" "\t descriptor size = 0x%08" PRIx32 "\n" "\t descriptor magic = 0x%08" PRIx32 "\n", @@ -3976,11 +3964,6 @@ int KYTY_SYSV_ABI GraphicsDriverSubmitAcb(uint32_t queue, const Packet* packet) if (packet == nullptr) { return OK; } - if (!Gen5::is_readable_command_packet_range(reinterpret_cast(packet), - sizeof(Packet) / sizeof(uint32_t))) { - return OK; - } - LOGF("\t acb = 0x%016" PRIx64 "\n" "\t size = 0x%08" PRIx32 "\n" "\t flags = 0x%02" PRIx8 "\n", diff --git a/tests/ShaderRecompilerComputeTests.cpp b/tests/ShaderRecompilerComputeTests.cpp index 457c475a..a7b6f906 100644 --- a/tests/ShaderRecompilerComputeTests.cpp +++ b/tests/ShaderRecompilerComputeTests.cpp @@ -9025,14 +9025,6 @@ void CheckBufferCacheRangeMerge() { 3) && !CanMergeBufferCacheQueueMask(0, 64), "cross-queue or invalid queue ownership was accepted"); - Require("BufferCacheRangeMerge", "readback queue ownership", - !CanReadbackBufferCacheQueueMask(0, 3) && - CanReadbackBufferCacheQueueMask(uint64_t{1} << 3u, 3) && - !CanReadbackBufferCacheQueueMask((uint64_t{1} << 2u) | - (uint64_t{1} << 3u), - 3) && - !CanReadbackBufferCacheQueueMask(uint64_t{1} << 3u, 64), - "zero, cross-queue, or invalid readback ownership was accepted"); std::printf("[host] %-32s ok\n", "BufferCacheRangeMerge"); } @@ -9855,10 +9847,13 @@ void CheckStorageTextureGpuOwnedRebindState() { tracker.ForEachUploadRange(base, size, true, [](uint64_t, uint64_t) noexcept {}, []() noexcept {}); uint64_t readable = 0; + uint64_t mapped = 0; MEMORY_BASIC_INFORMATION protection{}; Require("StorageTextureGpuOwnedRebind", "owned", tracker.IsRegionGpuModified(base, size) && page_manager.IsMapped(base, size) && (!HostMemoryQueryReadable(base, size, &readable) || readable < size) && + HostMemoryQueryRange(base, size, HostMemoryAccess::Mapped, &mapped) && + mapped == size && VirtualQuery(memory, &protection, sizeof(protection)) != 0 && protection.Protect == PAGE_NOACCESS, "GPU-owned storage pages remained host-readable or lost tracker identity"); diff --git a/tests/ShaderStageRuntimeTests.cpp b/tests/ShaderStageRuntimeTests.cpp index b120240a..746e20ca 100644 --- a/tests/ShaderStageRuntimeTests.cpp +++ b/tests/ShaderStageRuntimeTests.cpp @@ -3,7 +3,6 @@ #include "graphics/shader/recompiler/ResourceMaterialization.h" #include "graphics/shader/recompiler/ShaderIR.h" -#include #include #include #include @@ -53,22 +52,7 @@ std::shared_ptr UnbasedFlat return std::make_shared(std::move(program)); } -void TestFailurePreservesPriorStage() { - using namespace Libs::Graphics; - auto cached_program = SrtProgram(0x1001); - auto prior_program = std::make_shared(); - auto prior_resources = std::make_shared(); - ShaderStageRuntime stage {prior_program, prior_resources}; - const std::array user_data = {}; - std::string error; - Check(!ShaderMaterializeStageRuntime(cached_program, user_data, 0, &stage, &error) && - error.find("failed at 0x0000000000001000") != std::string::npos, - "unmapped cache ReadConst did not fail through the checked production reader"); - Check(stage.program == prior_program && stage.resources == prior_resources, - "failed cache rematerialization replaced the prior stage snapshot"); -} - -void TestMappedSrtUsesCheckedReaderByDefault() { +void TestMappedSrtUsesDirectReaderByDefault() { using namespace Libs::Graphics; const uint32_t dword = 0x12345678; auto cached_program = SrtProgram(reinterpret_cast(&dword)); @@ -79,7 +63,7 @@ void TestMappedSrtUsesCheckedReaderByDefault() { "cache rematerialization did not publish the mapped stage"); Check(stage.resources->flattened_srt.size() == 1 && stage.resources->flattened_srt[0] == dword, - "cache rematerialization did not use the checked reader by default"); + "cache rematerialization did not use the direct reader by default"); } void TestUnbasedFlatCacheHitFailsClosed() { @@ -99,8 +83,7 @@ void TestUnbasedFlatCacheHitFailsClosed() { } // namespace int main() { - TestFailurePreservesPriorStage(); - TestMappedSrtUsesCheckedReaderByDefault(); + TestMappedSrtUsesDirectReaderByDefault(); TestUnbasedFlatCacheHitFailsClosed(); std::puts("ShaderStageRuntimeTests: all cases passed"); return 0; diff --git a/tests/ShaderVertexMetadataTests.cpp b/tests/ShaderVertexMetadataTests.cpp index 94af86ab..d6a710a7 100644 --- a/tests/ShaderVertexMetadataTests.cpp +++ b/tests/ShaderVertexMetadataTests.cpp @@ -1,23 +1,10 @@ #include "graphics/shader/shaderVertexMetadata.h" -#include #include #include #include #include -#if KYTY_PLATFORM == KYTY_PLATFORM_WINDOWS -#ifndef NOMINMAX -#define NOMINMAX -#endif -#include -#undef min -#undef max -#elif KYTY_PLATFORM == KYTY_PLATFORM_LINUX -#include -#include -#endif - namespace { using namespace Libs::Graphics; @@ -60,7 +47,7 @@ void CheckRejected(const ShaderMappedData& data, const char* text) { "metadata rejection changed the prior output"); } -void TestValidAndHostileMetadata() { +void TestValidAndInvalidMetadata() { Fixture fixture; ShaderVertexMetadata output; std::string error; @@ -70,9 +57,13 @@ void TestValidAndHostileMetadata() { output.input_semantics_count == 1, "valid AGC vertex metadata was decoded incorrectly"); - auto unreadable = fixture.mapped; - unreadable.user_data = reinterpret_cast(uintptr_t {1}); - CheckRejected(unreadable, "unreadable AGC user-data header was accepted"); + auto missing_header = fixture.mapped; + missing_header.user_data = nullptr; + CheckRejected(missing_header, "missing AGC user-data header was accepted"); + + Fixture missing_offsets; + missing_offsets.user_data.direct_resource_offset = nullptr; + CheckRejected(missing_offsets.mapped, "missing direct-resource offsets were accepted"); Fixture excessive_resources; excessive_resources.user_data.direct_resource_count = @@ -88,65 +79,15 @@ void TestValidAndHostileMetadata() { static_cast(AgcDirectResourceType::PtrVertexBufferTable)] = 63; CheckRejected(excessive_register.mapped, "out-of-domain vertex table SGPR was accepted"); - Fixture bad_semantics; - bad_semantics.mapped.input_semantics = - reinterpret_cast(uintptr_t {1}); - CheckRejected(bad_semantics.mapped, "unreadable vertex semantic array was accepted"); -} - -void TestCrossPageMetadata() { -#if KYTY_PLATFORM == KYTY_PLATFORM_WINDOWS - SYSTEM_INFO system_info {}; - GetSystemInfo(&system_info); - const auto page_size = static_cast(system_info.dwPageSize); - auto* pages = static_cast( - VirtualAlloc(nullptr, page_size * 2, MEM_RESERVE | MEM_COMMIT, PAGE_READWRITE)); - Check(pages != nullptr, "failed to allocate metadata test pages"); - DWORD old_protect = 0; - Check(VirtualProtect(pages + page_size, page_size, PAGE_NOACCESS, &old_protect) != 0, - "failed to protect metadata test page"); -#elif KYTY_PLATFORM == KYTY_PLATFORM_LINUX - const auto native_page_size = sysconf(_SC_PAGESIZE); - Check(native_page_size > 0, "failed to query page size"); - const auto page_size = static_cast(native_page_size); - auto* pages = static_cast( - mmap(nullptr, page_size * 2, PROT_READ | PROT_WRITE, MAP_PRIVATE | MAP_ANONYMOUS, -1, 0)); - Check(pages != MAP_FAILED, "failed to allocate metadata test pages"); - Check(mprotect(pages + page_size, page_size, PROT_NONE) == 0, - "failed to protect metadata test page"); -#else - return; -#endif - - Fixture fixture; - auto header_crossing = fixture.mapped; - header_crossing.user_data = reinterpret_cast( - pages + page_size - sizeof(ShaderUserData) + 1); - CheckRejected(header_crossing, "cross-page AGC user-data header was accepted"); - - auto offsets_crossing = fixture.mapped; - fixture.user_data.direct_resource_offset = - reinterpret_cast(pages + page_size - sizeof(uint16_t)); - CheckRejected(offsets_crossing, "cross-page direct-resource offsets were accepted"); - - fixture.user_data.direct_resource_offset = fixture.offsets.data(); - auto semantics_crossing = fixture.mapped; - semantics_crossing.input_semantics = - reinterpret_cast(pages + page_size - sizeof(uint16_t)); - CheckRejected(semantics_crossing, "cross-page vertex semantics were accepted"); - -#if KYTY_PLATFORM == KYTY_PLATFORM_WINDOWS - Check(VirtualFree(pages, 0, MEM_RELEASE) != 0, "failed to free metadata test pages"); -#elif KYTY_PLATFORM == KYTY_PLATFORM_LINUX - Check(munmap(pages, page_size * 2) == 0, "failed to free metadata test pages"); -#endif + Fixture missing_semantics; + missing_semantics.mapped.input_semantics = nullptr; + CheckRejected(missing_semantics.mapped, "missing vertex semantic array was accepted"); } } // namespace int main() { - TestValidAndHostileMetadata(); - TestCrossPageMetadata(); + TestValidAndInvalidMetadata(); std::puts("ShaderVertexMetadataTests: all cases passed"); return 0; }