shifathosn
/
KytyPS5
1
0
Fork 0
Code Issues Pull Requests Packages Projects Releases Wiki Activity

remove readback hacks, allows to read/write guest memory directly

This commit is contained in:
nmzik 2026-07-17 05:16:00 +02:00
parent 013e9a4485
commit e903c93848
27 changed files with 522 additions and 1228 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

63
src/graphics/guest_gpu/command_processor/commandProcessor.h
View File

@ -12,8 +12,6 @@
namespace Libs::Graphics {
class CommandProcessorGuestAccessScope;
inline constexpr uint32_t AcquireGcrGl2Writeback = 1u << 15u;
bool TestWaitRegMemValue(uint64_t value, uint64_t ref, uint64_t mask, uint32_t func);
@ -72,6 +70,21 @@ public:
}
}
void SubmitForReadback() {
if (!Active()) {
return;
}
SubmitCurrent();
}
void ResumeAfterReadback() {
if (!Active()) {
return;
}
Current()->WaitForFenceAndReset();
Current()->Begin();
}
private:
CommandBuffer* SubmitCurrent() {
auto* submitted = Current();
@ -107,22 +120,26 @@ public:
void Reset();
void BufferInit();
void BufferFlush();
void BufferFlushAndWait();
void BufferWait();
[[nodiscard]] int BeginReadbackTransaction(bool mutex_preowned) {
if (!mutex_preowned) {
m_mutex.Lock();
void BufferInit();
void BufferFlush();
void BufferFlushAndWait();
void BufferWait();
void BeginReadbackTransaction() {
m_mutex.Lock();
if (m_readback_active) {
EXIT("nested command-processor readback transaction\n");
}
m_scheduler.Flush();
m_scheduler.WaitAll();
return m_scheduler.Queue();
m_readback_active = true;
m_readback_finished = false;
}
void EndReadbackTransaction(bool mutex_preowned) {
if (!mutex_preowned) {
m_mutex.Unlock();
void FinishReadbackTransaction();
void EndReadbackTransaction() {
if (!m_readback_active) {
EXIT("command-processor readback transaction is not active\n");
}
m_readback_active = false;
m_readback_finished = false;
m_mutex.Unlock();
}
void RunLock() { m_run_mutex.Lock(); }
@ -164,7 +181,6 @@ public:
uint32_t value);
void PrepareCpuFlip();
void SynchronizeGpu();
void SynchronizeGpuRange(uint64_t vaddr, uint64_t size);
void MemoryBarrier();
void TriggerEopEventAtEndOfPipe(uint32_t interrupt_context_id);
void RenderTextureBarrier(uint64_t vaddr, uint64_t size);
@ -182,9 +198,6 @@ public:
void PopMarker() {}
void PrefetchL2(void* addr, uint32_t size) {}
void ClearGds(uint64_t dw_offset, uint32_t dw_num, uint32_t clear_value);
void ReadGds(uint32_t* dst, uint32_t dw_offset, uint32_t dw_size);
void ResetDeCe();
void WaitCe();
void WaitDeDiff(uint32_t diff);
@ -208,7 +221,7 @@ public:
void Run(uint32_t* data, uint32_t num_dw);
void SetQueue(int queue) { m_scheduler.SetQueue(queue); }
void SetQueue(int queue);
[[nodiscard]] int GetQueue() const { return m_scheduler.Queue(); }
[[nodiscard]] const FlipInfo& GetFlip() const { return m_flip; }
@ -218,8 +231,6 @@ public:
void SetSubmitId(uint64_t submit_id) { m_submit_id = submit_id; }
private:
friend class CommandProcessorGuestAccessScope;
struct Counter {
Common::Mutex mutex;
Common::CondVar cond_var;
@ -231,6 +242,7 @@ private:
uint32_t cache_action, uint32_t event_index, uint32_t event_write_source,
void* dst_gpu_addr, T value, uint32_t interrupt_selector,
uint32_t interrupt_context_id);
void FinishCommandProcessors();
CommandBuffer* CurrentBuffer() { return m_scheduler.Current(); }
void CheckBuffer() const { m_scheduler.CheckActive(); }
@ -250,8 +262,11 @@ private:
uint64_t m_dispatch_indirect_args_base_addr = 0;
uint32_t m_num_instances = 1;
Common::Mutex m_mutex;
Common::Mutex m_run_mutex;
inline static Common::Mutex m_mutex;
inline static std::array<CommandProcessor*, GraphicContext::QUEUES_NUM> m_processors {};
inline static bool m_readback_active = false;
inline static bool m_readback_finished = false;
Common::Mutex m_run_mutex;
CommandScheduler m_scheduler;

14
src/graphics/guest_gpu/command_processor/pm4Handlers.cpp
View File

@ -2233,7 +2233,7 @@ KYTY_CP_OP_PARSER(CpOpCopyData) {
cp->WriteReferenceClock(dst, num_bytes);
return 5;
}
const auto dma_src = CopyDataSrcToDma(src_sel);
const auto dma_src = CopyDataSrcToDma(src_sel);
if (dma_src == 2 && num_bytes == 8) {
EXIT("unsupported 64-bit immediate copyData\n");
}
@ -2636,7 +2636,6 @@ KYTY_CP_OP_PARSER(CpOpIndirectCxRegs) {
if (indirect_buffer == nullptr) {
EXIT("indirect CX registers have null address, num_regs = %" PRIu32 "\n", indirect_num_dw);
}
for (uint32_t i = 0; i < indirect_num_dw; i++, indirect_buffer += 2) {
auto cmd_offset = indirect_buffer[0];
auto value = indirect_buffer[1];
@ -2695,13 +2694,14 @@ KYTY_CP_OP_PARSER(CpOpIndirectShRegs) {
if (indirect_buffer == nullptr) {
EXIT("indirect SH registers have null address, num_regs = %" PRIu32 "\n", indirect_num_dw);
}
const auto indirect_address = reinterpret_cast<uint64_t>(indirect_buffer);
for (uint32_t i = 0; i < indirect_num_dw; i++, indirect_buffer += 2) {
auto raw_cmd_offset = indirect_buffer[0];
auto cmd_offset = NormalizeRegisterOffset(raw_cmd_offset);
auto value = indirect_buffer[1];
//Not sure if this is correct
// Not sure if this is correct
if (raw_cmd_offset != cmd_offset) {
LOGF_COLOR(Log::Color::Red,
"\t temporary: normalized indirect SH register offset 0x%08" PRIx32
@ -2709,7 +2709,7 @@ KYTY_CP_OP_PARSER(CpOpIndirectShRegs) {
raw_cmd_offset, cmd_offset);
}
//Not sure if this is correct
// Not sure if this is correct
if (cmd_offset == Pm4::SH_NOP || raw_cmd_offset == 0xffffffffu) {
continue;
}
@ -2725,8 +2725,7 @@ KYTY_CP_OP_PARSER(CpOpIndirectShRegs) {
if (pfunc == nullptr) {
LOGF("unknown indirect SH register: index=%" PRIu32 "/%" PRIu32 ", regs=0x%016" PRIx64
", offset=0x%08" PRIx32 ", value=0x%08" PRIx32 "\n",
i, indirect_num_dw, reinterpret_cast<uint64_t>(indirect_buffer - i * 2),
cmd_offset, value);
i, indirect_num_dw, indirect_address, cmd_offset, value);
auto* dump_regs = indirect_buffer - i * 2;
for (uint32_t j = 0; j < indirect_num_dw && j < 16; j++) {
LOGF("\t sh_indirect[%" PRIu32 "] offset=0x%08" PRIx32 ", value=0x%08" PRIx32 "\n",
@ -2758,13 +2757,12 @@ KYTY_CP_OP_PARSER(CpOpIndirectUcRegs) {
if (indirect_buffer == nullptr) {
EXIT("indirect UC registers have null address, num_regs = %" PRIu32 "\n", indirect_num_dw);
}
for (uint32_t i = 0; i < indirect_num_dw; i++, indirect_buffer += 2) {
auto raw_cmd_offset = indirect_buffer[0];
auto cmd_offset = NormalizeRegisterOffset(raw_cmd_offset);
auto value = indirect_buffer[1];
//Not sure if this is correct
// Not sure if this is correct
if (raw_cmd_offset != cmd_offset) {
LOGF_COLOR(Log::Color::Red,
"\t temporary: normalized indirect UC register offset 0x%08" PRIx32

314
src/graphics/guest_gpu/graphicsRun.cpp
View File

@ -33,19 +33,9 @@
namespace Libs::Graphics {
enum class CommandProcessorGuestAccess : uint32_t {
None,
WaitPoll,
DirectFence,
};
static thread_local CommandProcessor* g_current_run_cp = nullptr;
static thread_local uint32_t g_submission_pause_depth = 0;
static thread_local bool g_gpu_mutex_owned = false;
static thread_local bool g_cp_readback_owned = false;
static thread_local bool g_cp_readback_preowned = false;
static thread_local CommandProcessorGuestAccess g_guest_memory_access =
CommandProcessorGuestAccess::None;
static thread_local CommandProcessor* g_current_run_cp = nullptr;
static thread_local uint32_t g_submission_pause_depth = 0;
static thread_local bool g_gpu_mutex_owned = false;
class GpuMutexLock final {
public:
@ -68,54 +58,13 @@ private:
Common::Mutex& m_mutex;
};
class CommandProcessorGuestAccessScope final {
public:
CommandProcessorGuestAccessScope(CommandProcessorGuestAccess access, uint64_t vaddr,
uint64_t size) {
if (access == CommandProcessorGuestAccess::None ||
g_guest_memory_access != CommandProcessorGuestAccess::None ||
g_current_run_cp == nullptr) {
EXIT("invalid nested command-processor guest-memory access\n");
}
if (g_render_ctx == nullptr) {
EXIT("command-processor guest-memory access has no render context\n");
}
m_cp = g_current_run_cp;
if (access == CommandProcessorGuestAccess::WaitPoll) {
// Preserve serialized scheduler semantics: a poll owns the command processor before it
// enters the shared buffer/texture transaction, and releases both before yielding.
m_cp->m_mutex.Lock();
m_cp_mutex_owned = true;
}
m_resource_lock =
g_render_ctx->GetGpuResources()->LockCommandProcessorGuestAccess(vaddr, size);
g_guest_memory_access = access;
}
~CommandProcessorGuestAccessScope() {
if (g_guest_memory_access == CommandProcessorGuestAccess::None ||
!m_resource_lock.owns_lock() || m_cp == nullptr) {
EXIT("command-processor guest-memory access scope is not active\n");
}
g_guest_memory_access = CommandProcessorGuestAccess::None;
m_resource_lock.unlock();
if (m_cp_mutex_owned) {
m_cp->m_mutex.Unlock();
}
}
private:
CommandProcessor* m_cp = nullptr;
bool m_cp_mutex_owned = false;
std::unique_lock<ResourceMutex> m_resource_lock;
KYTY_CLASS_NO_COPY(CommandProcessorGuestAccessScope);
};
struct OwnedCmdBuffer {
std::vector<uint32_t> storage;
uint32_t* data = nullptr;
uint32_t num_dw = 0;
OwnedCmdBuffer() = default;
OwnedCmdBuffer(const uint32_t* src, uint32_t count) { Assign(src, count); }
OwnedCmdBuffer(const OwnedCmdBuffer& other): storage(other.storage), num_dw(other.num_dw) {
data = (storage.empty() ? other.data : storage.data());
@ -165,9 +114,8 @@ public:
KYTY_CLASS_NO_COPY(GraphicsRing);
void Submit(uint32_t* cmd_draw_buffer, uint32_t num_draw_dw, uint32_t* cmd_const_buffer,
uint32_t num_const_dw, int handle, int index, int flip_mode, int64_t flip_arg,
bool trigger_agc_interrupt_on_done);
void Submit(OwnedCmdBuffer draw_buffer, OwnedCmdBuffer const_buffer, int handle, int index,
int flip_mode, int64_t flip_arg, bool trigger_agc_interrupt_on_done);
void SubmitFlipPreparation();
void Done();
void WaitForIdle();
@ -217,7 +165,7 @@ public:
KYTY_CLASS_NO_COPY(ComputeRing);
void Submit(uint32_t* cmd_buffer, uint32_t num_dw, bool trigger_agc_interrupt_on_done);
void Submit(OwnedCmdBuffer buffer, bool trigger_agc_interrupt_on_done);
void Done();
void WaitForIdle();
bool IsIdle();
@ -311,15 +259,18 @@ void GraphicsRunInit() {
void Gpu::Submit(uint32_t* cmd_draw_buffer, uint32_t num_draw_dw, uint32_t* cmd_const_buffer,
uint32_t num_const_dw, bool trigger_agc_interrupt_on_done) {
GpuMutexLock lock(m_mutex);
OwnedCmdBuffer draw_buffer(cmd_draw_buffer, num_draw_dw);
OwnedCmdBuffer const_buffer(cmd_const_buffer, num_const_dw);
GpuMutexLock lock(m_mutex);
m_gfx_ring->Submit(cmd_draw_buffer, num_draw_dw, cmd_const_buffer, num_const_dw, 0, 0, 0, 0,
m_gfx_ring->Submit(std::move(draw_buffer), std::move(const_buffer), 0, 0, 0, 0,
trigger_agc_interrupt_on_done);
}
void Gpu::SubmitCompute(uint32_t queue, uint32_t* cmd_buffer, uint32_t num_dw,
bool trigger_agc_interrupt_on_done) {
GpuMutexLock lock(m_mutex);
OwnedCmdBuffer buffer(cmd_buffer, num_dw);
GpuMutexLock lock(m_mutex);
constexpr uint32_t compute_queue_base = 0x20u;
constexpr uint32_t compute_queue_num = 7u * 8u;
@ -336,7 +287,7 @@ void Gpu::SubmitCompute(uint32_t queue, uint32_t* cmd_buffer, uint32_t num_dw,
auto* ring = GetRing(ring_id);
ring->Submit(cmd_buffer, num_dw, trigger_agc_interrupt_on_done);
ring->Submit(std::move(buffer), trigger_agc_interrupt_on_done);
}
void Gpu::SubmitFlipPreparation() {
@ -454,6 +405,45 @@ ComputeRing* Gpu::GetRing(uint32_t ring_id) {
return m_compute_ring[v];
}
void CommandProcessor::SetQueue(int queue) {
Common::LockGuard lock(m_mutex);
if (queue < 0 || queue >= GraphicContext::QUEUES_NUM ||
(m_processors[queue] != nullptr && m_processors[queue] != this)) {
EXIT("invalid command-processor queue registration: queue=%d owner=%p\n", queue,
static_cast<const void*>(m_processors[queue]));
}
m_scheduler.SetQueue(queue);
m_processors[queue] = this;
}
void CommandProcessor::FinishReadbackTransaction() {
if (GraphicsRunCurrentCommandProcessor() == nullptr || !m_readback_active) {
EXIT("GPU readback finish requires a command-processor thread\n");
}
if (m_readback_finished) {
return;
}
FinishCommandProcessors();
m_readback_finished = true;
}
void CommandProcessor::FinishCommandProcessors() {
std::array<CommandProcessor*, GraphicContext::QUEUES_NUM> processors {};
uint32_t processor_count = 0;
for (auto* processor: m_processors) {
if (processor == nullptr ||
std::find(processors.begin(), processors.begin() + processor_count, processor) !=
processors.begin() + processor_count) {
continue;
}
processors[processor_count++] = processor;
processor->m_scheduler.SubmitForReadback();
}
for (uint32_t i = 0; i < processor_count; i++) {
processors[i]->m_scheduler.ResumeAfterReadback();
}
}
void CommandProcessor::Reset() {
BufferWait();
@ -598,13 +588,6 @@ bool TestWaitRegMemValue(uint64_t value, uint64_t ref, uint64_t mask, uint32_t f
return false;
}
template <typename T>
static T LoadGuestWaitValue(const T* addr) {
CommandProcessorGuestAccessScope guest_access(CommandProcessorGuestAccess::WaitPoll,
reinterpret_cast<uint64_t>(addr), sizeof(*addr));
return *addr;
}
static void YieldCommandProcessorWait(uint32_t poll_interval_cycles) noexcept {
// PS5 specifies GPU poll cycles, not host microseconds. Yield at the scheduler boundary now;
// a future resumable guest GPU scheduler can replace this one function without touching polling
@ -625,12 +608,9 @@ void CommandProcessor::WaitRegMem(uint32_t func, const T* addr, T ref, T mask, u
const auto log_width = static_cast<int>(sizeof(T) * 2u);
const auto bits = static_cast<unsigned>(sizeof(T) * 8u);
if (!TestWaitRegMemValue(LoadGuestWaitValue(addr), ref, mask, func)) {
SynchronizeGpuRange(addr_value, sizeof(*addr));
}
uint64_t spin_count = 0;
for (;;) {
const auto value = LoadGuestWaitValue(addr);
const auto value = *addr;
if (TestWaitRegMemValue(value, ref, mask, func)) {
break;
}
@ -670,10 +650,7 @@ void CommandProcessor::WriteData(uint32_t* dst, const uint32_t* src, uint32_t dw
return;
}
const auto size = static_cast<uint64_t>(dw_num) * sizeof(uint32_t);
CommandProcessorGuestAccessScope guest_access(CommandProcessorGuestAccess::DirectFence,
reinterpret_cast<uint64_t>(dst), size);
memcpy(dst, src, static_cast<size_t>(size));
memcpy(dst, src, static_cast<size_t>(dw_num) * sizeof(uint32_t));
}
void CommandProcessor::WriteReferenceClock(uint64_t dst_address, uint32_t num_bytes) {
@ -684,11 +661,8 @@ void CommandProcessor::WriteReferenceClock(uint64_t dst_address, uint32_t num_by
num_bytes);
}
const auto value = GraphicsRenderReadReferenceClock();
CommandProcessorGuestAccessScope guest_access(CommandProcessorGuestAccess::DirectFence,
dst_address, num_bytes);
std::memcpy(reinterpret_cast<void*>(dst_address), &value, num_bytes);
LOGF("\t copy_data reference clock: dst=0x%016" PRIx64 " value=0x%016" PRIx64
" size=%u\n",
LOGF("\t copy_data reference clock: dst=0x%016" PRIx64 " value=0x%016" PRIx64 " size=%u\n",
dst_address, value, num_bytes);
}
@ -729,9 +703,9 @@ void CommandProcessor::DmaData(uint8_t engine, uint8_t dst_sel, uint8_t dst_cach
EXIT("unsupported dmaData source selector 0x%02" PRIx8 "\n", src_sel);
}
void GraphicsRing::Submit(uint32_t* cmd_draw_buffer, uint32_t num_draw_dw,
uint32_t* cmd_const_buffer, uint32_t num_const_dw, int handle, int index,
int flip_mode, int64_t flip_arg, bool trigger_agc_interrupt_on_done) {
void GraphicsRing::Submit(OwnedCmdBuffer draw_buffer, OwnedCmdBuffer const_buffer, int handle,
int index, int flip_mode, int64_t flip_arg,
bool trigger_agc_interrupt_on_done) {
EXIT_IF(m_cp == nullptr);
Common::LockGuard lock(m_mutex);
@ -748,9 +722,9 @@ void GraphicsRing::Submit(uint32_t* cmd_draw_buffer, uint32_t num_draw_dw,
m_cp->Reset();
}
auto& buf = m_cmd_batches.emplace_back();
buf.draw_buffer.Assign(cmd_draw_buffer, num_draw_dw);
buf.const_buffer.Assign(cmd_const_buffer, num_const_dw);
auto& buf = m_cmd_batches.emplace_back();
buf.draw_buffer = std::move(draw_buffer);
buf.const_buffer = std::move(const_buffer);
buf.flip.handle = handle;
buf.flip.index = index;
buf.flip.flip_mode = flip_mode;
@ -942,12 +916,11 @@ void ComputeRing::ThreadRun(void* data) {
}
}
void ComputeRing::Submit(uint32_t* cmd_buffer, uint32_t num_dw,
bool trigger_agc_interrupt_on_done) {
void ComputeRing::Submit(OwnedCmdBuffer buffer, bool trigger_agc_interrupt_on_done) {
Common::LockGuard lock(m_mutex);
EXIT_IF(cmd_buffer == nullptr);
EXIT_IF(num_dw == 0);
EXIT_IF(buffer.data == nullptr);
EXIT_IF(buffer.num_dw == 0);
WindowWaitForGraphicInitialized();
GraphicsRenderCreateContext();
@ -957,7 +930,7 @@ void ComputeRing::Submit(uint32_t* cmd_buffer, uint32_t num_dw,
}
DirectBatch batch {};
batch.buffer.Assign(cmd_buffer, num_dw);
batch.buffer = std::move(buffer);
batch.trigger_agc_interrupt_on_done = trigger_agc_interrupt_on_done;
m_direct_batches.push_back(std::move(batch));
@ -1229,35 +1202,38 @@ void CommandProcessor::DrawIndirect(uint32_t data_offset, uint32_t draw_initiato
EXIT_NOT_IMPLEMENTED((draw_initiator & ~0x20u) != 2u);
EXIT_NOT_IMPLEMENTED(m_draw_indirect_args_base_addr == 0);
const auto args_addr = m_draw_indirect_args_base_addr + data_offset;
const auto* args_addr =
reinterpret_cast<const void*>(m_draw_indirect_args_base_addr + data_offset);
if (!indexed) {
auto* args = reinterpret_cast<const DrawIndirectArgs*>(args_addr);
if (args->instance_count != 1u || args->start_vertex_location != 0u ||
args->start_instance_location != 0u) {
DrawIndirectArgs args {};
std::memcpy(&args, args_addr, sizeof(args));
if (args.instance_count != 1u || args.start_vertex_location != 0u ||
args.start_instance_location != 0u) {
static std::atomic<uint32_t> log_count {0};
if (log_count.fetch_add(1) < 64) {
LOGF("\t warning: partial DrawIndirect args: vertex_count=%" PRIu32
", instance_count=%" PRIu32 ", start_vertex=%" PRIu32
", start_instance=%" PRIu32 "\n",
args->vertex_count_per_instance, args->instance_count,
args->start_vertex_location, args->start_instance_location);
args.vertex_count_per_instance, args.instance_count,
args.start_vertex_location, args.start_instance_location);
}
}
DrawIndexAuto(args->vertex_count_per_instance, 0, 0, args->instance_count,
args->start_vertex_location, args->start_instance_location);
DrawIndexAuto(args.vertex_count_per_instance, 0, 0, args.instance_count,
args.start_vertex_location, args.start_instance_location);
return;
}
auto* args = reinterpret_cast<const DrawIndexedIndirectArgs*>(args_addr);
if (args->base_vertex_location != 0u || args->start_instance_location != 0u) {
DrawIndexedIndirectArgs args {};
std::memcpy(&args, args_addr, sizeof(args));
if (args.base_vertex_location != 0u || args.start_instance_location != 0u) {
static std::atomic<uint32_t> log_count {0};
if (log_count.fetch_add(1) < 64) {
LOGF("\t warning: partial DrawIndexIndirect args: index_count=%" PRIu32
", instance_count=%" PRIu32 ", start_index=%" PRIu32 ", base_vertex=%" PRIu32
", start_instance=%" PRIu32 "\n",
args->index_count_per_instance, args->instance_count, args->start_index_location,
args->base_vertex_location, args->start_instance_location);
args.index_count_per_instance, args.instance_count, args.start_index_location,
args.base_vertex_location, args.start_instance_location);
}
}
@ -1270,22 +1246,22 @@ void CommandProcessor::DrawIndirect(uint32_t data_offset, uint32_t draw_initiato
}
auto* index_addr = reinterpret_cast<const void*>(
m_index_base_addr + static_cast<uint64_t>(args->start_index_location) * index_size);
m_index_base_addr + static_cast<uint64_t>(args.start_index_location) * index_size);
const uint32_t index_count =
(m_index_buffer_size != 0 ? std::min(args->index_count_per_instance, m_index_buffer_size)
: args->index_count_per_instance);
if (GraphicsRunDebugDumpEnabled() && index_count != args->index_count_per_instance) {
(m_index_buffer_size != 0 ? std::min(args.index_count_per_instance, m_index_buffer_size)
: args.index_count_per_instance);
if (GraphicsRunDebugDumpEnabled() && index_count != args.index_count_per_instance) {
static std::atomic<uint32_t> log_count {0};
if (log_count.fetch_add(1, std::memory_order_relaxed) < 64) {
LOGF("\t DrawIndexIndirect: clamped index_count from %" PRIu32 " to %" PRIu32
" using INDEX_BUFFER_SIZE\n",
args->index_count_per_instance, index_count);
args.index_count_per_instance, index_count);
}
}
DrawIndex(index_count, index_addr, 0, 1, args->instance_count, nullptr, 0,
static_cast<int32_t>(args->base_vertex_location), args->start_instance_location);
DrawIndex(index_count, index_addr, 0, 1, args.instance_count, nullptr, 0,
static_cast<int32_t>(args.base_vertex_location), args.start_instance_location);
}
void CommandProcessor::DrawIndirectMulti(uint32_t data_offset, uint32_t max_count_or_count,
@ -1503,18 +1479,6 @@ void CommandProcessor::DrawIndexAuto(uint32_t index_count, uint32_t flags,
first_instance);
}
void CommandProcessor::ClearGds(uint64_t dw_offset, uint32_t dw_num, uint32_t clear_value) {
Common::LockGuard lock(m_mutex);
GraphicsRenderClearGds(dw_offset, dw_num, clear_value);
}
void CommandProcessor::ReadGds(uint32_t* dst, uint32_t dw_offset, uint32_t dw_size) {
Common::LockGuard lock(m_mutex);
GraphicsRenderReadGds(dst, dw_offset, dw_size);
}
void CommandProcessor::WaitFlipDone(uint32_t video_out_handle, uint32_t display_buffer_index) {
BufferFlush();
@ -1569,11 +1533,9 @@ void CommandProcessor::WriteAtEndOfPipe(uint32_t cache_policy, uint32_t event_wr
}
auto write32 = [&](bool with_writeback) {
auto* dst = static_cast<uint32_t*>(dst_gpu_addr);
auto data = static_cast<uint32_t>(value);
CommandProcessorGuestAccessScope guest_access(CommandProcessorGuestAccess::DirectFence,
reinterpret_cast<uint64_t>(dst),
sizeof(*dst));
auto* dst = static_cast<uint32_t*>(dst_gpu_addr);
auto data = static_cast<uint32_t>(value);
std::memcpy(dst, &data, sizeof(data));
if (with_interrupt) {
if (with_writeback) {
@ -1594,8 +1556,10 @@ void CommandProcessor::WriteAtEndOfPipe(uint32_t cache_policy, uint32_t event_wr
case 0x01:
if constexpr (sizeof(T) == sizeof(uint32_t)) {
if (eop_event_type == 0x2f && cache_action == 0x00 && event_index == 0x06) {
GraphicsRenderWriteAtEndOfPipeGds32(m_submit_id, CurrentBuffer(),
static_cast<uint32_t*>(dst_gpu_addr),
auto* dst = static_cast<uint32_t*>(dst_gpu_addr);
SynchronizeGpu();
GraphicsRenderReadGds(dst, value & 0xffffu, value >> 16u);
GraphicsRenderWriteAtEndOfPipeGds32(m_submit_id, CurrentBuffer(), dst,
value & 0xffffu, value >> 16u);
return;
}
@ -1615,10 +1579,8 @@ void CommandProcessor::WriteAtEndOfPipe(uint32_t cache_policy, uint32_t event_wr
}
} else {
auto write64 = [&](bool with_writeback) {
auto* dst = static_cast<uint64_t*>(dst_gpu_addr);
CommandProcessorGuestAccessScope guest_access(
CommandProcessorGuestAccess::DirectFence, reinterpret_cast<uint64_t>(dst),
sizeof(*dst));
auto* dst = static_cast<uint64_t*>(dst_gpu_addr);
std::memcpy(dst, &value, sizeof(value));
if (with_interrupt) {
if (with_writeback) {
@ -1699,16 +1661,16 @@ void CommandProcessor::WriteAtEndOfPipe(uint32_t cache_policy, uint32_t event_wr
break;
case 0x04:
if constexpr (sizeof(T) == sizeof(uint64_t)) {
CommandProcessorGuestAccessScope guest_access(
CommandProcessorGuestAccess::DirectFence,
reinterpret_cast<uint64_t>(dst_gpu_addr), sizeof(uint64_t));
const auto clock = GraphicsRenderReadReferenceClock();
std::memcpy(dst_gpu_addr, &clock, sizeof(clock));
switch (cache_action) {
case 0x00:
if (((eop_event_type == 0x04 && event_index == 0x05) ||
(eop_event_type == 0x28 && event_index == 0x00)) &&
!with_interrupt) {
GraphicsRenderWriteAtEndOfPipeClockCounter(
m_submit_id, CurrentBuffer(), static_cast<uint64_t*>(dst_gpu_addr));
m_submit_id, CurrentBuffer(), static_cast<uint64_t*>(dst_gpu_addr),
clock);
return;
}
break;
@ -1718,7 +1680,8 @@ void CommandProcessor::WriteAtEndOfPipe(uint32_t cache_policy, uint32_t event_wr
(eop_event_type == 0x28 && event_index == 0x00)) &&
!with_interrupt) {
GraphicsRenderWriteAtEndOfPipeClockCounterWithWriteBack(
m_submit_id, CurrentBuffer(), static_cast<uint64_t*>(dst_gpu_addr));
m_submit_id, CurrentBuffer(), static_cast<uint64_t*>(dst_gpu_addr),
clock);
return;
}
break;
@ -1850,8 +1813,8 @@ void CommandProcessor::Flip() {
}
auto* command = CurrentBuffer();
auto request = GraphicsRenderPrepareDisplayBufferFlip(
command, m_flip.handle, m_flip.index, m_flip.flip_mode, m_flip.flip_arg);
auto request = GraphicsRenderPrepareDisplayBufferFlip(command, m_flip.handle, m_flip.index,
m_flip.flip_mode, m_flip.flip_arg);
GraphicsRenderWriteAtEndOfPipeOnlyFlip(m_submit_id, command, m_flip.handle, m_flip.index,
m_flip.flip_mode, m_flip.flip_arg, request);
m_scheduler.Flush();
@ -1869,12 +1832,10 @@ void CommandProcessor::Flip(void* dst_gpu_addr, uint32_t value) {
reinterpret_cast<uint64_t>(dst_gpu_addr), value);
}
std::memcpy(dst_gpu_addr, &value, sizeof(value));
auto* command = CurrentBuffer();
auto request = GraphicsRenderPrepareDisplayBufferFlip(
command, m_flip.handle, m_flip.index, m_flip.flip_mode, m_flip.flip_arg);
CommandProcessorGuestAccessScope guest_access(CommandProcessorGuestAccess::DirectFence,
reinterpret_cast<uint64_t>(dst_gpu_addr),
sizeof(uint32_t));
auto request = GraphicsRenderPrepareDisplayBufferFlip(command, m_flip.handle, m_flip.index,
m_flip.flip_mode, m_flip.flip_arg);
GraphicsRenderWriteAtEndOfPipeWithFlip32(
m_submit_id, command, static_cast<uint32_t*>(dst_gpu_addr), value, m_flip.handle,
m_flip.index, m_flip.flip_mode, m_flip.flip_arg, request);
@ -1899,12 +1860,10 @@ void CommandProcessor::FlipWithInterrupt(uint32_t eop_event_type, uint32_t cache
if (eop_event_type != 0x00000004 || cache_action != 0x00000038) {
EXIT("unknown event type\n");
}
std::memcpy(dst_gpu_addr, &value, sizeof(value));
auto* command = CurrentBuffer();
auto request = GraphicsRenderPrepareDisplayBufferFlip(command, m_flip.handle, m_flip.index,
m_flip.flip_mode, m_flip.flip_arg);
CommandProcessorGuestAccessScope guest_access(CommandProcessorGuestAccess::DirectFence,
reinterpret_cast<uint64_t>(dst_gpu_addr),
sizeof(uint32_t));
GraphicsRenderWriteAtEndOfPipeWithInterruptWriteBackFlip32(
m_submit_id, command, static_cast<uint32_t*>(dst_gpu_addr), value, m_flip.handle,
m_flip.index, m_flip.flip_mode, m_flip.flip_arg, request);
@ -1929,15 +1888,8 @@ void CommandProcessor::PrepareCpuFlip() {
}
void CommandProcessor::SynchronizeGpu() {
BufferFlush();
BufferWait();
}
void CommandProcessor::SynchronizeGpuRange(uint64_t vaddr, uint64_t size) {
if (vaddr == 0 || size == 0 || size > UINT64_MAX - vaddr) {
EXIT("invalid GPU synchronization range\n");
}
SynchronizeGpu();
Common::LockGuard lock(m_mutex);
FinishCommandProcessors();
}
void GraphicsRunSubmit(uint32_t* cmd_draw_buffer, uint32_t num_draw_dw, uint32_t* cmd_const_buffer,
@ -2002,14 +1954,15 @@ bool GraphicsRunIsCommandProcessorThread() noexcept {
return g_current_run_cp != nullptr;
}
bool GraphicsRunIsGuestMemoryPoll() noexcept {
return g_current_run_cp != nullptr &&
g_guest_memory_access == CommandProcessorGuestAccess::WaitPoll;
CommandProcessor* GraphicsRunCurrentCommandProcessor() noexcept {
return g_current_run_cp;
}
bool GraphicsRunIsCommandProcessorGuestAccess() noexcept {
return g_current_run_cp != nullptr &&
g_guest_memory_access != CommandProcessorGuestAccess::None;
void GraphicsRunFinishCommandProcessors() {
if (g_current_run_cp == nullptr) {
EXIT("GPU readback finish requires a command-processor thread\n");
}
g_current_run_cp->FinishReadbackTransaction();
}
bool GraphicsRunSubmissionLockHeld() noexcept {
@ -2020,35 +1973,4 @@ bool GraphicsRunGpuLockHeld() noexcept {
return g_gpu_mutex_owned;
}
int GraphicsRunBeginCommandProcessorReadback() {
if (g_current_run_cp == nullptr || g_submission_pause_depth != 0 || g_gpu_mutex_owned ||
g_cp_readback_owned || g_guest_memory_access == CommandProcessorGuestAccess::None) {
EXIT("GPU command-thread readback requires an unpaused command processor\n");
}
const auto queue = g_current_run_cp->GetQueue();
if (queue < 0 || queue >= GraphicContext::QUEUES_NUM) {
EXIT("GPU command-thread readback has invalid queue %d\n", queue);
}
g_cp_readback_owned = true;
// Direct fences enter from an already serialized command handler. Wait polls acquire the
// same processor mutex in CommandProcessorGuestAccessScope before the resource transaction.
g_cp_readback_preowned = true;
const auto synchronized_queue =
g_current_run_cp->BeginReadbackTransaction(g_cp_readback_preowned);
if (synchronized_queue != queue) {
EXIT("GPU command-thread queue changed during readback\n");
}
return synchronized_queue;
}
void GraphicsRunEndCommandProcessorReadback() {
if (g_current_run_cp == nullptr || !g_cp_readback_owned) {
EXIT("GPU command-thread readback ended without ownership\n");
}
const auto preowned = g_cp_readback_preowned;
g_cp_readback_owned = false;
g_cp_readback_preowned = false;
g_current_run_cp->EndReadbackTransaction(preowned);
}
} // namespace Libs::Graphics

18
src/graphics/guest_gpu/graphicsRun.h
View File

@ -6,6 +6,8 @@
namespace Libs::Graphics {
class CommandProcessor;
class GraphicsRunSubmissionLock final {
public:
GraphicsRunSubmissionLock();
@ -23,17 +25,11 @@ void GraphicsRunSubmitFlipPreparation();
void GraphicsRunWait();
void GraphicsRunDone();
int GraphicsRunGetFrameNum();
[[nodiscard]] bool GraphicsRunIsCommandProcessorThread() noexcept;
[[nodiscard]] bool GraphicsRunIsGuestMemoryPoll() noexcept;
[[nodiscard]] bool GraphicsRunIsCommandProcessorGuestAccess() noexcept;
[[nodiscard]] bool GraphicsRunSubmissionLockHeld() noexcept;
[[nodiscard]] bool GraphicsRunGpuLockHeld() noexcept;
// Begin/end an exclusive same-command-processor readback transaction. The begin call finishes
// work already recorded on that queue and retains the processor lock so its peer draw/constant
// job cannot record new work until BufferCache publishes the faulted page.
[[nodiscard]] int GraphicsRunBeginCommandProcessorReadback();
void GraphicsRunEndCommandProcessorReadback();
[[nodiscard]] bool GraphicsRunIsCommandProcessorThread() noexcept;
[[nodiscard]] CommandProcessor* GraphicsRunCurrentCommandProcessor() noexcept;
void GraphicsRunFinishCommandProcessors();
[[nodiscard]] bool GraphicsRunSubmissionLockHeld() noexcept;
[[nodiscard]] bool GraphicsRunGpuLockHeld() noexcept;
} // namespace Libs::Graphics
#endif /* EMULATOR_INCLUDE_EMULATOR_GRAPHICS_GRAPHICSRUN_H_ */

41
src/graphics/host_gpu/hostMemory.cpp
View File

@ -2,7 +2,6 @@
#include <cinttypes>
#include <cstdio>
#include <cstring>
#if KYTY_PLATFORM == KYTY_PLATFORM_WINDOWS
#ifndef NOMINMAX
@ -21,10 +20,10 @@ bool IsAccessible(DWORD protect, HostMemoryAccess access) {
constexpr DWORD blocked = PAGE_NOACCESS | PAGE_GUARD;
constexpr DWORD readable = PAGE_READONLY | PAGE_READWRITE | PAGE_WRITECOPY | PAGE_EXECUTE_READ |
PAGE_EXECUTE_READWRITE | PAGE_EXECUTE_WRITECOPY;
constexpr DWORD writable =
PAGE_READWRITE | PAGE_WRITECOPY | PAGE_EXECUTE_READWRITE | PAGE_EXECUTE_WRITECOPY;
return (protect & blocked) == 0 &&
(protect & (access == HostMemoryAccess::Write ? writable : readable)) != 0;
if (access == HostMemoryAccess::Mapped) {
return (protect & PAGE_GUARD) == 0;
}
return (protect & blocked) == 0 && (protect & readable) != 0;
}
#endif
@ -77,8 +76,7 @@ bool HostMemoryQueryRange(uint64_t addr, uint64_t requested_size, HostMemoryAcce
if (begin > current) {
break;
}
const bool allowed =
access == HostMemoryAccess::Write ? permissions[1] == 'w' : permissions[0] == 'r';
const bool allowed = access == HostMemoryAccess::Mapped || permissions[0] == 'r';
if (!allowed) {
break;
}
@ -93,39 +91,10 @@ bool HostMemoryQueryRange(uint64_t addr, uint64_t requested_size, HostMemoryAcce
return *accessible_size != 0;
}
bool HostMemoryQueryEnd(uint64_t addr, HostMemoryAccess access, uint64_t* end) {
if (addr == 0 || end == nullptr) {
return false;
}
uint64_t accessible_size = 0;
if (!HostMemoryQueryRange(addr, UINT64_MAX - addr, access, &accessible_size) ||
accessible_size == 0 || UINT64_MAX - addr < accessible_size) {
return false;
}
*end = addr + accessible_size;
return true;
}
bool HostMemoryQueryReadable(uint64_t addr, uint64_t requested_size, uint64_t* readable_size) {
return HostMemoryQueryRange(addr, requested_size, HostMemoryAccess::Read, readable_size);
}
bool HostMemoryQueryWritable(uint64_t addr, uint64_t requested_size, uint64_t* writable_size) {
return HostMemoryQueryRange(addr, requested_size, HostMemoryAccess::Write, writable_size);
}
bool HostMemoryReadDword(void*, uint64_t addr, uint32_t* value) {
if (value == nullptr) {
return false;
}
uint64_t readable = 0;
if (!HostMemoryQueryReadable(addr, sizeof(*value), &readable) || readable < sizeof(*value)) {
return false;
}
std::memcpy(value, reinterpret_cast<const void*>(static_cast<uintptr_t>(addr)), sizeof(*value));
return true;
}
bool HostMemoryIsReadable(uint64_t addr) {
return HostMemoryRangeIsReadable(addr, 1);
}

5
src/graphics/host_gpu/hostMemory.h
View File

@ -5,14 +5,11 @@
namespace Libs::Graphics {
enum class HostMemoryAccess { Read, Write };
enum class HostMemoryAccess { Read, Mapped };
bool HostMemoryQueryRange(uint64_t addr, uint64_t requested_size, HostMemoryAccess access,
uint64_t* accessible_size);
bool HostMemoryQueryEnd(uint64_t addr, HostMemoryAccess access, uint64_t* end);
bool HostMemoryQueryReadable(uint64_t addr, uint64_t requested_size, uint64_t* readable_size);
bool HostMemoryQueryWritable(uint64_t addr, uint64_t requested_size, uint64_t* writable_size);
bool HostMemoryReadDword(void* userdata, uint64_t addr, uint32_t* value);
bool HostMemoryIsReadable(uint64_t addr);
bool HostMemoryRangeIsReadable(uint64_t addr, uint64_t size);

110
src/graphics/host_gpu/objects/label.cpp
View File

@ -1,14 +1,11 @@
#include "graphics/host_gpu/objects/label.h"
#include "common/assert.h"
#include "common/logging/log.h"
#include "kernel/memory.h"
#include "common/threads.h"
#include "graphics/host_gpu/graphicContext.h"
#include "graphics/host_gpu/renderer/render.h"
#include <algorithm>
#include <atomic>
#include <memory>
#include <vector>
@ -22,10 +19,6 @@ enum LabelStatus {
};
struct LabelCallbacks {
uint64_t* dst_gpu_addr64 = nullptr;
uint64_t value64 = 0;
uint32_t* dst_gpu_addr32 = nullptr;
uint32_t value32 = 0;
LabelCallback callback_1 = nullptr;
LabelCallback callback_2 = nullptr;
uint64_t args[LABEL_ARGS_MAX] = {};
@ -64,10 +57,8 @@ public:
~LabelManager() { KYTY_NOT_IMPLEMENTED; }
KYTY_CLASS_NO_COPY(LabelManager);
Label* Create64(GraphicContext* ctx, uint64_t* dst_gpu_addr, uint64_t value,
LabelCallback callback_1, LabelCallback callback_2, const uint64_t* args);
Label* Create32(GraphicContext* ctx, uint32_t* dst_gpu_addr, uint32_t value,
LabelCallback callback_1, LabelCallback callback_2, const uint64_t* args);
Label* Create(GraphicContext* ctx, LabelCallback callback_1, LabelCallback callback_2,
const uint64_t* args);
void Delete(Label* label);
void Set(CommandBuffer* buffer, Label* label);
void Drain();
@ -75,9 +66,6 @@ public:
private:
static void ThreadRun(void* data);
template <typename T>
Label* Create(GraphicContext* ctx, T* dst_gpu_addr, T value, LabelCallback callback_1,
LabelCallback callback_2, const uint64_t* args);
bool Remove(Label* label);
static void Destroy(Label* label);
@ -90,11 +78,6 @@ private:
static LabelManager* g_label_manager = nullptr;
static thread_local bool g_in_label_callback = false;
template <typename T>
void WriteGuestLabel(T* address, T value) {
LabelWriteGuestMemory(address, &value, sizeof(value));
}
class LabelCallbackScope final {
public:
LabelCallbackScope() {
@ -132,8 +115,7 @@ void LabelManager::ThreadRun(void* data) {
it->completion->event == nullptr) {
EXIT("GPU label submission has no completion event\n");
}
const auto status =
vkGetEventStatus(it->completion->device, it->completion->event);
const auto status = vkGetEventStatus(it->completion->device, it->completion->event);
switch (status) {
case VK_EVENT_SET:
fired_labels.push_back(it->callbacks);
@ -180,35 +162,9 @@ void LabelManager::ThreadRun(void* data) {
for (auto& label: fired_labels) {
LabelCallbackScope callback_scope;
bool write = true;
if (label.callback_1 != nullptr) {
write = label.callback_1(label.args);
}
if (write && label.dst_gpu_addr64 != nullptr) {
// Publishing EOP fences through physical backing keeps a fence word coherent even
// when unrelated bytes on its host page are
// protected for BufferCache GPU ownership.
WriteGuestLabel(label.dst_gpu_addr64, label.value64);
static std::atomic<uint32_t> log_count {0};
if (log_count.fetch_add(1) < 256) {
LOGF_COLOR(Log::Color::BrightGreen,
"EndOfPipe Signal!!! [0x%016" PRIx64 "] <- 0x%016" PRIx64 "\n",
reinterpret_cast<uint64_t>(label.dst_gpu_addr64), label.value64);
}
}
if (write && label.dst_gpu_addr32 != nullptr) {
WriteGuestLabel(label.dst_gpu_addr32, label.value32);
static std::atomic<uint32_t> log_count {0};
if (log_count.fetch_add(1) < 256) {
LOGF_COLOR(Log::Color::BrightGreen,
"EndOfPipe Signal!!! [0x%016" PRIx64 "] <- 0x%08" PRIx32 "\n",
reinterpret_cast<uint64_t>(label.dst_gpu_addr32), label.value32);
}
(void)label.callback_1(label.args);
}
if (label.callback_2 != nullptr) {
@ -229,29 +185,15 @@ void LabelManager::ThreadRun(void* data) {
}
}
template <typename T>
Label* LabelManager::Create(GraphicContext* ctx, T* dst_gpu_addr, T value, LabelCallback callback_1,
LabelCallback callback_2, const uint64_t* args) {
static_assert(sizeof(T) == sizeof(uint32_t) || sizeof(T) == sizeof(uint64_t));
Label* LabelManager::Create(GraphicContext* ctx, LabelCallback callback_1, LabelCallback callback_2,
const uint64_t* args) {
EXIT_IF(ctx == nullptr);
Common::LockGuard lock(m_mutex);
auto* label = new Label;
label->status = LabelStatus::New;
if constexpr (sizeof(T) == sizeof(uint64_t)) {
label->callbacks.dst_gpu_addr64 = dst_gpu_addr;
label->callbacks.value64 = value;
label->callbacks.dst_gpu_addr32 = nullptr;
label->callbacks.value32 = 0;
} else {
label->callbacks.dst_gpu_addr32 = dst_gpu_addr;
label->callbacks.value32 = value;
label->callbacks.dst_gpu_addr64 = nullptr;
label->callbacks.value64 = 0;
}
label->status = LabelStatus::New;
label->device = ctx->device;
label->callbacks.callback_1 = callback_1;
label->callbacks.callback_2 = callback_2;
@ -267,18 +209,6 @@ Label* LabelManager::Create(GraphicContext* ctx, T* dst_gpu_addr, T value, Label
return label;
}
Label* LabelManager::Create64(GraphicContext* ctx, uint64_t* dst_gpu_addr, uint64_t value,
LabelCallback callback_1, LabelCallback callback_2,
const uint64_t* args) {
return Create(ctx, dst_gpu_addr, value, callback_1, callback_2, args);
}
Label* LabelManager::Create32(GraphicContext* ctx, uint32_t* dst_gpu_addr, uint32_t value,
LabelCallback callback_1, LabelCallback callback_2,
const uint64_t* args) {
return Create(ctx, dst_gpu_addr, value, callback_1, callback_2, args);
}
bool LabelManager::Remove(Label* label) {
EXIT_IF(label == nullptr);
EXIT_IF(label->device == nullptr);
@ -336,8 +266,8 @@ void LabelManager::Set(CommandBuffer* buffer, Label* label) {
label->status = LabelStatus::Active;
LabelSubmission submission {};
submission.callbacks = label->callbacks;
submission.completion = std::make_shared<LabelEvent>();
submission.callbacks = label->callbacks;
submission.completion = std::make_shared<LabelEvent>();
submission.completion->device = label->device;
auto* vk_buffer = buffer->GetPool()->buffers[buffer->GetIndex()];
@ -385,18 +315,11 @@ void LabelInit() {
g_label_manager = new LabelManager;
}
Label* LabelCreate64(GraphicContext* ctx, uint64_t* dst_gpu_addr, uint64_t value,
LabelCallback callback_1, LabelCallback callback_2, const uint64_t* args) {
Label* LabelCreate(GraphicContext* ctx, LabelCallback callback_1, LabelCallback callback_2,
const uint64_t* args) {
EXIT_IF(g_label_manager == nullptr);
return g_label_manager->Create64(ctx, dst_gpu_addr, value, callback_1, callback_2, args);
}
Label* LabelCreate32(GraphicContext* ctx, uint32_t* dst_gpu_addr, uint32_t value,
LabelCallback callback_1, LabelCallback callback_2, const uint64_t* args) {
EXIT_IF(g_label_manager == nullptr);
return g_label_manager->Create32(ctx, dst_gpu_addr, value, callback_1, callback_2, args);
return g_label_manager->Create(ctx, callback_1, callback_2, args);
}
void LabelDelete(Label* label) {
@ -416,15 +339,6 @@ void LabelDrain() {
g_label_manager->Drain();
}
void LabelWriteGuestMemory(void* address, const void* data, uint64_t size) {
if (address == nullptr || data == nullptr || size == 0) {
EXIT("invalid guest label write\n");
}
if (!Libs::LibKernel::Memory::TryWriteBacking(reinterpret_cast<uint64_t>(address), data, size)) {
std::memcpy(address, data, static_cast<size_t>(size));
}
}
bool LabelInCallback() noexcept {
return g_in_label_callback;
}

7
src/graphics/host_gpu/objects/label.h
View File

@ -15,14 +15,11 @@ void LabelInit();
constexpr int LABEL_ARGS_MAX = 5;
using LabelCallback = bool (*)(const uint64_t* args);
Label* LabelCreate64(GraphicContext* ctx, uint64_t* dst_gpu_addr, uint64_t value,
LabelCallback callback_1, LabelCallback callback_2, const uint64_t* args);
Label* LabelCreate32(GraphicContext* ctx, uint32_t* dst_gpu_addr, uint32_t value,
LabelCallback callback_1, LabelCallback callback_2, const uint64_t* args);
Label* LabelCreate(GraphicContext* ctx, LabelCallback callback_1, LabelCallback callback_2,
const uint64_t* args);
void LabelDelete(Label* label);
void LabelSet(CommandBuffer* buffer, Label* label);
void LabelDrain();
void LabelWriteGuestMemory(void* address, const void* data, uint64_t size);
[[nodiscard]] bool LabelInCallback() noexcept;
} // namespace Libs::Graphics

310
src/graphics/host_gpu/renderer/bufferCache.cpp
View File

@ -16,7 +16,6 @@
#include <algorithm>
#include <array>
#include <atomic>
#include <chrono>
#include <cstdlib>
#include <cstring>
#include <memory>
@ -37,27 +36,6 @@ constexpr uint64_t AlignReadbackCopySize(uint64_t size) noexcept {
return (size + READBACK_COPY_ALIGNMENT - 1) & ~(READBACK_COPY_ALIGNMENT - 1);
}
constexpr bool ShouldLogCommandProcessorReadback(uint64_t serial) noexcept {
return serial <= 32 || (serial % 256) == 0;
}
struct CommandProcessorReadbackState {
BufferCache* cache = nullptr;
PageFaultAccess access = PageFaultAccess::Unknown;
uint64_t vaddr = 0;
uint64_t size = 0;
uint64_t page = 0;
uint64_t serial = 0;
uint64_t dirty_bytes = 0;
uint64_t packed_bytes = 0;
uint32_t dirty_count = 0;
std::chrono::steady_clock::time_point started {};
bool installed = false;
};
thread_local CommandProcessorReadbackState g_command_processor_readback;
std::atomic<uint64_t> g_command_processor_readback_serial {0};
struct SharedVulkanBufferOwner {
explicit SharedVulkanBufferOwner(GraphicContext* context): ctx(context) {
EXIT_IF(ctx == nullptr);
@ -192,10 +170,6 @@ bool CanMergeBufferCacheQueueMask(uint64_t queue_mask, uint32_t queue) noexcept
return queue < 64 && (queue_mask & ~(uint64_t {1} << queue)) == 0;
}
bool CanReadbackBufferCacheQueueMask(uint64_t queue_mask, uint32_t queue) noexcept {
return queue < 64 && queue_mask == (uint64_t {1} << queue);
}
struct BufferCache::CachedBuffer {
uint64_t vaddr = 0;
uint64_t size = 0;
@ -204,14 +178,6 @@ struct BufferCache::CachedBuffer {
std::shared_ptr<VulkanBuffer> buffer;
};
struct BufferCache::CommandProcessorReadbackResources {
GraphicContext* ctx = nullptr;
int queue = -1;
VulkanBuffer buffer {};
void* mapped = nullptr;
std::unique_ptr<CommandBuffer> command;
};
struct BufferCache::ReadbackWorker {
static constexpr uint32_t MAX_RANGES = READBACK_MAX_RANGES;
enum class State : uint32_t {
@ -305,20 +271,20 @@ struct BufferCache::ReadbackWorker {
}
void Request(PageFaultAccess fault_access, uint64_t fault_vaddr, uint64_t fault_size) noexcept {
const bool submissions_prepaused_now = GraphicsRunSubmissionLockHeld();
const bool command_thread = GraphicsRunIsCommandProcessorThread();
const bool submissions_prepaused_now = GraphicsRunSubmissionLockHeld() || command_thread;
const bool unsafe_gpu_lock = GraphicsRunGpuLockHeld() && !submissions_prepaused_now;
if (GraphicsRunIsCommandProcessorThread() || unsafe_gpu_lock || LabelInCallback() ||
g_cache_lock_owner != nullptr || ctx == nullptr || command == nullptr ||
mapped == nullptr || readback.buffer == nullptr) {
if (unsafe_gpu_lock || LabelInCallback() || g_cache_lock_owner != nullptr ||
ctx == nullptr || command == nullptr || mapped == nullptr ||
readback.buffer == nullptr) {
EXIT("BufferCache: unsafe readback request context, command_thread=%d "
"submission_lock=%d "
"gpu_lock=%d label_callback=%d cache_lock=%p ctx=%p command=%p mapped=%p "
"buffer=%p\n",
GraphicsRunIsCommandProcessorThread(), GraphicsRunSubmissionLockHeld(),
GraphicsRunGpuLockHeld(), LabelInCallback(),
static_cast<const void*>(g_cache_lock_owner), static_cast<const void*>(ctx),
static_cast<const void*>(command.get()), static_cast<const void*>(mapped),
static_cast<const void*>(readback.buffer));
command_thread, GraphicsRunSubmissionLockHeld(), GraphicsRunGpuLockHeld(),
LabelInCallback(), static_cast<const void*>(g_cache_lock_owner),
static_cast<const void*>(ctx), static_cast<const void*>(command.get()),
static_cast<const void*>(mapped), static_cast<const void*>(readback.buffer));
}
State expected = State::Idle;
while (!state.compare_exchange_weak(expected, State::Claimed, std::memory_order_acq_rel)) {
@ -584,18 +550,6 @@ BufferCache::BufferCache(PageManager& page_manager, ResourceMutex& resource_mute
BufferCache::~BufferCache() {
m_readback.reset();
if (m_cp_readback_resources != nullptr) {
auto& resources = *m_cp_readback_resources;
resources.command.reset();
if (resources.mapped != nullptr) {
VulkanUnmapMemory(resources.ctx, &resources.buffer.memory);
resources.mapped = nullptr;
}
if (resources.buffer.buffer != nullptr) {
VulkanDeleteBuffer(resources.ctx, &resources.buffer);
}
m_cp_readback_resources.reset();
}
if (!m_gpu_modified_ranges.Empty()) {
EXIT("BufferCache: destroyed with pending GPU-modified ranges\n");
}
@ -623,14 +577,9 @@ bool BufferCache::InvalidateMemory(PageFaultAccess access, uint64_t vaddr, uint6
switch (phase) {
case PageFaultPhase::Invalidate: break;
case PageFaultPhase::Complete:
return CompleteCommandProcessorReadback(access, vaddr, size) ||
m_readback->Complete(access, vaddr, size) ||
return m_readback->Complete(access, vaddr, size) ||
m_memory_tracker.CompleteCpuFault(vaddr, size, access, false);
case PageFaultPhase::Release:
if (!ReleaseCommandProcessorReadback(access, vaddr, size)) {
m_readback->Release(access, vaddr, size);
}
return true;
case PageFaultPhase::Release: m_readback->Release(access, vaddr, size); return true;
default:
EXIT("BufferCache: unsupported page-fault phase %u\n", static_cast<uint32_t>(phase));
}
@ -639,242 +588,9 @@ bool BufferCache::InvalidateMemory(PageFaultAccess access, uint64_t vaddr, uint6
return action == CpuFaultAction::Continue;
}
if (GraphicsRunIsCommandProcessorThread()) {
if (!GraphicsRunIsCommandProcessorGuestAccess()) {
EXIT("BufferCache: command-processor fault outside a coherent guest-memory access, "
"addr=0x%016" PRIx64 " size=0x%016" PRIx64 "\n",
vaddr, size);
}
RequestCommandProcessorReadback(access, vaddr, size);
} else {
m_readback->Request(access, vaddr, size);
}
return true;
}
void BufferCache::RequestCommandProcessorReadback(PageFaultAccess access, uint64_t vaddr,
uint64_t size) {
if (!GraphicsRunIsCommandProcessorThread() || g_command_processor_readback.cache != nullptr) {
EXIT("BufferCache: invalid nested command-processor readback\n");
}
const auto serial =
g_command_processor_readback_serial.fetch_add(1, std::memory_order_relaxed) + 1;
if (serial == 0) {
EXIT("BufferCache: command-processor readback serial overflow\n");
}
const bool log_detail = ShouldLogCommandProcessorReadback(serial);
if (log_detail) {
LOGF("BufferCache: command-processor readback request serial=%" PRIu64 " addr=0x%016" PRIx64
" size=0x%016" PRIx64 " access=%u poll=%d\n",
serial, vaddr, size, static_cast<uint32_t>(access), GraphicsRunIsGuestMemoryPoll());
Log::Flush();
}
const auto page = vaddr & ~(TRACKER_PAGE_SIZE - 1);
const auto queue = static_cast<uint32_t>(GraphicsRunBeginCommandProcessorReadback());
if (log_detail) {
LOGF("BufferCache: command-processor queue synchronized serial=%" PRIu64
" queue=%u page=0x%016" PRIx64 "\n",
serial, queue, page);
Log::Flush();
}
if (g_cache_lock_owner != nullptr) {
EXIT("BufferCache: command-processor readback entered with a cache lock\n");
}
g_cache_lock_owner = this;
m_mutex.Lock();
if (log_detail) {
LOGF("BufferCache: command-processor cache locked serial=%" PRIu64 " page=0x%016" PRIx64
"\n",
serial, page);
Log::Flush();
}
auto it = m_buffers.upper_bound(vaddr);
if (it == m_buffers.begin()) {
EXIT("BufferCache: command-processor readback address has no cached buffer, "
"addr=0x%016" PRIx64 "\n",
vaddr);
}
--it;
auto& cached = *it->second;
if (vaddr < cached.vaddr || vaddr >= cached.vaddr + cached.size || page < cached.vaddr ||
TRACKER_PAGE_SIZE > cached.size - (page - cached.vaddr) || cached.ctx == nullptr ||
cached.buffer == nullptr || cached.buffer->buffer == nullptr) {
EXIT("BufferCache: invalid command-processor readback owner, fault=0x%016" PRIx64
" page=0x%016" PRIx64 " buffer=0x%016" PRIx64 "+0x%016" PRIx64 "\n",
vaddr, page, cached.vaddr, cached.size);
}
if (!CanReadbackBufferCacheQueueMask(cached.queue_mask, queue)) {
EXIT("BufferCache: command-processor readback cannot synchronize cross-queue ownership, "
"addr=0x%016" PRIx64 " used_queues=0x%016" PRIx64 " current_queue=%u\n",
vaddr, cached.queue_mask, queue);
}
const auto dirty = m_gpu_modified_ranges.Intersections(page, TRACKER_PAGE_SIZE);
if (dirty.empty()) {
EXIT("BufferCache: command-processor GPU-dirty page has no dirty byte ranges, "
"page=0x%016" PRIx64 "\n",
page);
}
ValidateDirtyPages(m_gpu_modified_ranges, page, TRACKER_PAGE_SIZE, "command-processor");
if (m_cp_readback_resources == nullptr) {
auto resources = std::make_unique<CommandProcessorReadbackResources>();
resources->ctx = cached.ctx;
resources->buffer.usage = VK_BUFFER_USAGE_TRANSFER_DST_BIT;
resources->buffer.memory.property = VK_MEMORY_PROPERTY_HOST_VISIBLE_BIT |
VK_MEMORY_PROPERTY_HOST_COHERENT_BIT |
VK_MEMORY_PROPERTY_HOST_CACHED_BIT;
VulkanCreateBuffer(cached.ctx, READBACK_CAPACITY, &resources->buffer);
VulkanMapMemory(cached.ctx, &resources->buffer.memory, &resources->mapped);
m_cp_readback_resources = std::move(resources);
}
auto& resources = *m_cp_readback_resources;
if (resources.ctx != cached.ctx || resources.mapped == nullptr ||
resources.buffer.buffer == nullptr) {
EXIT("BufferCache: invalid command-processor readback resources\n");
}
if (queue >= GraphicContext::QUEUES_NUM || cached.ctx->queues[queue].family == UINT32_MAX ||
cached.ctx->queues[queue].vk_queue == nullptr) {
EXIT("BufferCache: command-processor readback has invalid Vulkan queue %u\n", queue);
}
if (resources.command == nullptr || resources.queue != static_cast<int>(queue)) {
resources.command.reset();
resources.command = std::make_unique<CommandBuffer>(static_cast<int>(queue));
resources.queue = static_cast<int>(queue);
}
if (resources.command->IsInvalid() ||
resources.command->GetQueue() != static_cast<int>(queue)) {
EXIT("BufferCache: failed to initialize owning-queue readback command buffer, queue=%u\n",
queue);
}
if (dirty.size() > ReadbackWorker::MAX_RANGES) {
EXIT("BufferCache: too many command-processor dirty ranges, page=0x%016" PRIx64
" count=%" PRIu64 " limit=%u\n",
page, static_cast<uint64_t>(dirty.size()), ReadbackWorker::MAX_RANGES);
}
std::array<VkBufferCopy, ReadbackWorker::MAX_RANGES> copies {};
std::array<VkBufferMemoryBarrier, ReadbackWorker::MAX_RANGES> barriers {};
uint64_t dirty_bytes = 0;
uint64_t packed_bytes = 0;
uint32_t dirty_count = 0;
for (const auto& range: dirty) {
if ((range.address - cached.vaddr) % 4 != 0 || range.size % 4 != 0) {
EXIT("BufferCache: invalid command-processor dirty range, addr=0x%016" PRIx64
" size=0x%016" PRIx64 " page=0x%016" PRIx64 " packed=0x%016" PRIx64 "\n",
range.address, range.size, page, dirty_bytes);
}
copies[dirty_count] = {.srcOffset = range.address - cached.vaddr,
.dstOffset = packed_bytes,
.size = range.size};
auto& barrier = barriers[dirty_count];
barrier.sType = VK_STRUCTURE_TYPE_BUFFER_MEMORY_BARRIER;
barrier.srcAccessMask = VK_ACCESS_MEMORY_WRITE_BIT;
barrier.dstAccessMask = VK_ACCESS_TRANSFER_READ_BIT;
barrier.srcQueueFamilyIndex = VK_QUEUE_FAMILY_IGNORED;
barrier.dstQueueFamilyIndex = VK_QUEUE_FAMILY_IGNORED;
barrier.buffer = cached.buffer->buffer;
barrier.offset = copies[dirty_count].srcOffset;
barrier.size = range.size;
const auto slot_size = AlignReadbackCopySize(range.size);
if (packed_bytes > READBACK_CAPACITY || slot_size > READBACK_CAPACITY - packed_bytes) {
EXIT("BufferCache: aligned command-processor readback exceeds persistent capacity, "
"packed=%" PRIu64 " size=%" PRIu64 " capacity=%" PRIu64 "\n",
packed_bytes, slot_size, READBACK_CAPACITY);
}
packed_bytes += slot_size;
dirty_bytes += range.size;
dirty_count++;
}
if (dirty_count == 0 || dirty_bytes == 0) {
EXIT("BufferCache: command-processor readback produced no packed ranges\n");
}
const auto started = std::chrono::steady_clock::now();
if (log_detail) {
LOGF("BufferCache: command-processor readback begin serial=%" PRIu64
" queue=%u page=0x%016" PRIx64 " ranges=%u bytes=%" PRIu64 " packed=%" PRIu64 "\n",
serial, queue, page, dirty_count, dirty_bytes, packed_bytes);
Log::Flush();
}
auto* vk_command = resources.command->GetPool()->buffers[resources.command->GetIndex()];
resources.command->Begin();
vkCmdPipelineBarrier(vk_command, VK_PIPELINE_STAGE_ALL_COMMANDS_BIT,
VK_PIPELINE_STAGE_TRANSFER_BIT, 0, 0, nullptr, dirty_count,
barriers.data(), 0, nullptr);
vkCmdCopyBuffer(vk_command, cached.buffer->buffer, resources.buffer.buffer, dirty_count,
copies.data());
resources.command->End();
resources.command->Execute();
resources.command->WaitForFenceAndReset();
if (log_detail) {
LOGF("BufferCache: command-processor GPU copy complete serial=%" PRIu64 "\n", serial);
Log::Flush();
}
for (uint32_t i = 0; i < dirty_count; i++) {
Libs::LibKernel::Memory::WriteBacking(
dirty[i].address, static_cast<const uint8_t*>(resources.mapped) + copies[i].dstOffset,
dirty[i].size);
}
g_command_processor_readback = {.cache = this,
.access = access,
.vaddr = vaddr,
.size = size,
.page = page,
.serial = serial,
.dirty_bytes = dirty_bytes,
.packed_bytes = packed_bytes,
.dirty_count = dirty_count,
.started = started};
}
bool BufferCache::CompleteCommandProcessorReadback(PageFaultAccess access, uint64_t vaddr,
uint64_t size) noexcept {
auto& pending = g_command_processor_readback;
if (pending.cache == nullptr) {
return false;
}
if (pending.cache != this || pending.access != access || pending.vaddr != vaddr ||
pending.size != size || pending.installed) {
EXIT("BufferCache: mismatched active command-processor readback completion\n");
}
if (!m_memory_tracker.CompleteCpuFault(vaddr, size, access, true)) {
EXIT("BufferCache: failed to complete command-processor readback, addr=0x%016" PRIx64
" size=0x%016" PRIx64 " access=%u\n",
vaddr, size, static_cast<uint32_t>(access));
}
m_gpu_modified_ranges.Subtract(pending.page, TRACKER_PAGE_SIZE);
pending.installed = true;
if (ShouldLogCommandProcessorReadback(pending.serial)) {
LOGF("BufferCache: command-processor readback complete serial=%" PRIu64
" page=0x%016" PRIx64 " ranges=%u bytes=%" PRIu64 "\n",
pending.serial, pending.page, pending.dirty_count, pending.dirty_bytes);
}
return true;
}
bool BufferCache::ReleaseCommandProcessorReadback(PageFaultAccess access, uint64_t vaddr,
uint64_t size) noexcept {
auto& pending = g_command_processor_readback;
if (pending.cache == nullptr) {
return false;
}
if (pending.cache != this || pending.access != access || pending.vaddr != vaddr ||
pending.size != size || !pending.installed) {
EXIT("BufferCache: mismatched active command-processor readback release\n");
}
const auto serial = pending.serial;
const auto page = pending.page;
const auto dirty_count = pending.dirty_count;
const auto dirty_bytes = pending.dirty_bytes;
const auto elapsed_micro = std::chrono::duration_cast<std::chrono::microseconds>(
std::chrono::steady_clock::now() - pending.started)
.count();
pending = {};
m_mutex.Unlock();
g_cache_lock_owner = nullptr;
GraphicsRunEndCommandProcessorReadback();
if (ShouldLogCommandProcessorReadback(serial)) {
LOGF("BufferCache: command-processor readback release serial=%" PRIu64 " page=0x%016" PRIx64
" ranges=%u bytes=%" PRIu64 " elapsed_us=%" PRIi64 "\n",
serial, page, dirty_count, dirty_bytes, static_cast<int64_t>(elapsed_micro));
GraphicsRunFinishCommandProcessors();
}
m_readback->Request(access, vaddr, size);
return true;
}

16
src/graphics/host_gpu/renderer/bufferCache.h
View File

@ -39,7 +39,6 @@ struct BufferCacheRange {
[[nodiscard]] bool MergeOverlappingBufferCacheRange(BufferCacheRange* merged,
BufferCacheRange candidate) noexcept;
[[nodiscard]] bool CanMergeBufferCacheQueueMask(uint64_t queue_mask, uint32_t queue) noexcept;
[[nodiscard]] bool CanReadbackBufferCacheQueueMask(uint64_t queue_mask, uint32_t queue) noexcept;
class BufferCache {
public:
@ -61,11 +60,9 @@ public:
// Emulator-owned, CPU-current scratch only. Guest ranges must use ObtainBuffer so page
// ownership is resolved before any CPU access.
[[nodiscard]] bool UploadHostData(CommandBuffer* command, GraphicContext* ctx, const void* src,
uint64_t size, uint64_t alignment,
VulkanBuffer** out_buffer, uint64_t* out_offset,
uint64_t* out_range);
[[nodiscard]] VulkanBuffer* ObtainNullBuffer(CommandBuffer* command,
GraphicContext* ctx);
uint64_t size, uint64_t alignment, VulkanBuffer** out_buffer,
uint64_t* out_offset, uint64_t* out_range);
[[nodiscard]] VulkanBuffer* ObtainNullBuffer(CommandBuffer* command, GraphicContext* ctx);
[[nodiscard]] BufferImageCopySource ObtainBufferForImage(uint64_t vaddr, uint64_t size);
void FillBuffer(CommandBuffer* command, GraphicContext* ctx, uint64_t vaddr, uint64_t size,
uint32_t value);
@ -84,19 +81,12 @@ public:
private:
struct CachedBuffer;
struct ReadbackWorker;
struct CommandProcessorReadbackResources;
void RequestCommandProcessorReadback(PageFaultAccess access, uint64_t vaddr, uint64_t size);
[[nodiscard]] bool CompleteCommandProcessorReadback(PageFaultAccess access, uint64_t vaddr,
uint64_t size) noexcept;
[[nodiscard]] bool ReleaseCommandProcessorReadback(PageFaultAccess access, uint64_t vaddr,
uint64_t size) noexcept;
Common::Mutex m_mutex;
std::vector<VulkanBuffer*> m_delete_later;
std::shared_ptr<VulkanBuffer> m_null_buffer;
std::map<uint64_t, std::unique_ptr<CachedBuffer>> m_buffers;
std::unique_ptr<ReadbackWorker> m_readback;
std::unique_ptr<CommandProcessorReadbackResources> m_cp_readback_resources;
RangeSet m_gpu_modified_ranges;
MemoryTracker m_memory_tracker;
PageManager& m_page_manager;

7
src/graphics/host_gpu/renderer/context.cpp
View File

@ -332,7 +332,7 @@ void CommandBuffer::Execute() {
m_execute = true;
m_fence_waited = false;
m_submit_seq = g_command_buffer_submit_seq.fetch_add(1, std::memory_order_relaxed) + 1;
m_submit_seq = g_command_buffer_submit_seq.fetch_add(1, std::memory_order_relaxed) + 1;
if (result != VK_SUCCESS) {
LOGF("vkQueueSubmit failed: %s (%d), queue=%d index=%u submit_seq=%" PRIu64
@ -396,7 +396,7 @@ void CommandBuffer::ExecuteWithSemaphore(VkSemaphore signal_semaphore) {
m_execute = true;
m_fence_waited = false;
m_submit_seq = g_command_buffer_submit_seq.fetch_add(1, std::memory_order_relaxed) + 1;
m_submit_seq = g_command_buffer_submit_seq.fetch_add(1, std::memory_order_relaxed) + 1;
if (result != VK_SUCCESS) {
LOGF("vkQueueSubmit failed: %s (%d), queue=%d index=%u submit_seq=%" PRIu64
@ -464,7 +464,7 @@ void CommandBuffer::ExecuteWithSemaphore(VkSemaphore wait_semaphore,
m_execute = true;
m_fence_waited = false;
m_submit_seq = g_command_buffer_submit_seq.fetch_add(1, std::memory_order_relaxed) + 1;
m_submit_seq = g_command_buffer_submit_seq.fetch_add(1, std::memory_order_relaxed) + 1;
if (result != VK_SUCCESS) {
LOGF("vkQueueSubmit failed: %s (%d), queue=%d index=%u submit_seq=%" PRIu64
@ -518,6 +518,7 @@ void CommandBuffer::WaitForFenceAndReset() {
m_fence_waited = false;
vkResetCommandBuffer(m_pool->buffers[m_index],
VK_COMMAND_BUFFER_RESET_RELEASE_RESOURCES_BIT);
m_recording_generation++;
}
m_host_stream.Reset();
if (was_executed) {

2
src/graphics/host_gpu/renderer/descriptors.cpp
View File

@ -205,7 +205,7 @@ NativeAddressBuffer(uint64_t submit_id, CommandBuffer* command_buffer,
->GetPhysicalDeviceProperties()
.limits.maxStorageBufferRange);
uint64_t size = 0;
const auto access = HostMemoryAccess::Read;
const auto access = HostMemoryAccess::Mapped;
if (!HostMemoryQueryRange(address.binding_base, limit, access, &size)) {
EXIT("address resource is not host-accessible: base=0x%016" PRIx64 "\n",
address.binding_base);

53
src/graphics/host_gpu/renderer/gpuResourceManager.cpp
View File

@ -1,6 +1,7 @@
#include "graphics/host_gpu/renderer/gpuResourceManager.h"
#include "common/assert.h"
#include "graphics/guest_gpu/command_processor/commandProcessor.h"
#include "graphics/guest_gpu/graphicsRun.h"
#include "graphics/host_gpu/objects/label.h"
#include "graphics/host_gpu/renderer/render.h"
@ -37,17 +38,24 @@ bool GpuResourceManager::HandleFault(PageFaultAccess access, uint64_t fault_vadd
"addr=0x%016" PRIx64 " access=%u\n",
fault_vaddr, static_cast<uint32_t>(access));
}
if (m_resource_mutex.IsOwnedByCurrentThread()) {
if (!GraphicsRunIsCommandProcessorGuestAccess()) {
EXIT("unsupported page fault from a pre-owned resource transaction, addr=0x%016" PRIx64
" access=%u\n",
fault_vaddr, static_cast<uint32_t>(access));
if (auto* cp = GraphicsRunCurrentCommandProcessor(); cp != nullptr) {
cp->BeginReadbackTransaction();
bool handled = false;
{
ResourceMutex::FaultScope fault(m_resource_mutex);
handled = m_page_manager.HandleFault(access, fault_vaddr);
}
ResourceMutex::FaultScope fault(m_resource_mutex);
return m_page_manager.HandleFault(access, fault_vaddr);
cp->EndReadbackTransaction();
return handled;
}
if (m_resource_mutex.IsOwnedByCurrentThread()) {
EXIT("unsupported page fault from a pre-owned resource transaction, addr=0x%016" PRIx64
" access=%u\n",
fault_vaddr, static_cast<uint32_t>(access));
}
// Stop command-processor jobs before taking the shared cache transaction. External readback
// workers inherit this paused state and therefore never form resource -> submission lock inversion.
// workers inherit this paused state and therefore never form resource -> submission lock
// inversion.
GraphicsRunSubmissionLock submissions;
ResourceMutex::FaultScope fault(m_resource_mutex);
return m_page_manager.HandleFault(access, fault_vaddr);
@ -57,35 +65,6 @@ bool GpuResourceManager::IsMapped(uint64_t vaddr, uint64_t size) const noexcept
return m_page_manager.IsMapped(vaddr, size);
}
std::unique_lock<ResourceMutex> GpuResourceManager::LockCommandProcessorGuestAccess(
uint64_t vaddr, uint64_t size) {
if (vaddr == 0 || size == 0 || vaddr >= TRACKER_ADDRESS_SIZE ||
size > TRACKER_ADDRESS_SIZE - vaddr) {
EXIT("invalid command-processor guest-memory access, addr=0x%016" PRIx64
" size=0x%016" PRIx64 "\n",
vaddr, size);
}
return std::unique_lock(m_resource_mutex);
}
std::unique_lock<ResourceMutex> GpuResourceManager::LockAsynchronousGuestWrite(uint64_t vaddr,
uint64_t size) {
if (vaddr == 0 || size == 0 || vaddr >= TRACKER_ADDRESS_SIZE ||
size > TRACKER_ADDRESS_SIZE - vaddr) {
EXIT("invalid asynchronous guest-memory write, addr=0x%016" PRIx64
" size=0x%016" PRIx64 "\n",
vaddr, size);
}
std::unique_lock transaction(m_resource_mutex);
if (m_buffer_cache.HasPageOverlap(vaddr, size) || m_texture_cache.HasPageOverlap(vaddr, size) ||
m_texture_cache.HasMetaOverlap(vaddr, size)) {
EXIT("asynchronous guest-memory write to a cached buffer/image range is unsupported, "
"addr=0x%016" PRIx64 " size=0x%016" PRIx64 "\n",
vaddr, size);
}
return transaction;
}
void GpuResourceManager::MapMemory(uint64_t vaddr, uint64_t size, GpuAccess access) {
m_page_manager.OnGpuMap(vaddr, size, access);
}

11
src/graphics/host_gpu/renderer/gpuResourceManager.h
View File

@ -9,7 +9,6 @@
#include "graphics/host_gpu/renderer/textureCache.h"
#include <cstdint>
#include <mutex>
namespace Libs::Graphics {
@ -26,14 +25,6 @@ public:
[[nodiscard]] bool HandleFault(PageFaultAccess access, uint64_t fault_vaddr) noexcept;
[[nodiscard]] bool IsMapped(uint64_t vaddr, uint64_t size) const noexcept;
// Serializes synchronous command-processor guest-memory access against buffer/image ownership
// publication. Address metadata keeps this seam ready for a future page-range lock.
[[nodiscard]] std::unique_lock<ResourceMutex>
LockCommandProcessorGuestAccess(uint64_t vaddr, uint64_t size);
// Keeps an asynchronous host callback's validation and guest write in one cache transaction.
// Cached buffer/image destinations are unsupported and fail before the callback can fault.
[[nodiscard]] std::unique_lock<ResourceMutex> LockAsynchronousGuestWrite(uint64_t vaddr,
uint64_t size);
void MapMemory(uint64_t vaddr, uint64_t size, GpuAccess access);
void UnmapMemory(uint64_t vaddr, uint64_t size, GpuAccess access);
void FillBuffer(CommandBuffer* command, uint64_t vaddr, uint64_t size, uint32_t value);
@ -48,7 +39,7 @@ private:
PageManager m_page_manager;
ResourceMutex m_resource_mutex;
BufferCache m_buffer_cache;
TextureCache m_texture_cache;
TextureCache m_texture_cache;
};
} // namespace Libs::Graphics

60
src/graphics/host_gpu/renderer/render.h
View File

@ -87,25 +87,27 @@ public:
[[nodiscard]] int GetQueue() const { return m_queue; }
VulkanCommandPool* GetPool() { return m_pool; }
[[nodiscard]] bool IsExecute() const { return m_execute; }
[[nodiscard]] uint64_t GetRecordingGeneration() const { return m_recording_generation; }
private:
friend class BufferCache;
void RecycleDescriptorsAfterFence();
VulkanCommandPool* m_pool = nullptr;
uint32_t m_index = static_cast<uint32_t>(-1);
int m_queue = -1;
bool m_execute = false;
bool m_fence_waited = false;
uint64_t m_submit_seq = 0;
uint32_t m_debug_op = 0;
uint64_t m_debug_submit_id = 0;
uint32_t m_debug_arg0 = 0;
uint32_t m_debug_arg1 = 0;
uint32_t m_debug_arg2 = 0;
uint32_t m_debug_arg3 = 0;
uint64_t m_debug_arg4 = 0;
VulkanCommandPool* m_pool = nullptr;
uint32_t m_index = static_cast<uint32_t>(-1);
int m_queue = -1;
bool m_execute = false;
bool m_fence_waited = false;
uint64_t m_submit_seq = 0;
uint64_t m_recording_generation = 0;
uint32_t m_debug_op = 0;
uint64_t m_debug_submit_id = 0;
uint32_t m_debug_arg0 = 0;
uint32_t m_debug_arg1 = 0;
uint32_t m_debug_arg2 = 0;
uint32_t m_debug_arg3 = 0;
uint64_t m_debug_arg4 = 0;
std::vector<VulkanBuffer*> m_delete_after_fence;
FenceResourceRetainer m_fence_resources;
std::vector<VulkanDescriptorSet*> m_descriptor_sets_after_fence;
@ -129,8 +131,8 @@ void RenderDispatchDirect(uint64_t submit_id, CommandBuffer* buffer, HW::Context
void GraphicsRenderInit();
void GraphicsRenderCreateContext();
[[nodiscard]] bool GraphicsScaleReferenceClock(uint64_t host_ticks, uint64_t host_frequency,
uint64_t* value);
[[nodiscard]] bool GraphicsScaleReferenceClock(uint64_t host_ticks, uint64_t host_frequency,
uint64_t* value);
[[nodiscard]] uint64_t GraphicsRenderReadReferenceClock();
[[nodiscard]] bool ResolveComputeImageClear(const ShaderComputeInputInfo& input, uint32_t group_x,
@ -140,21 +142,22 @@ void GraphicsRenderCreateContext();
[[nodiscard]] bool ResolveHtileClearTarget(const HW::DepthRenderTarget& target,
uint64_t descriptor_size, HtileClearTarget* resolved);
void GraphicsRenderWriteAtEndOfPipe64(uint64_t submit_id, CommandBuffer* buffer,
uint64_t* dst_gpu_addr, uint64_t value);
void GraphicsRenderWriteAtEndOfPipeClockCounter(uint64_t submit_id, CommandBuffer* buffer,
uint64_t* dst_gpu_addr);
void GraphicsRenderWriteAtEndOfPipeClockCounterWithWriteBack(uint64_t submit_id,
CommandBuffer* buffer,
uint64_t* dst_gpu_addr);
void GraphicsRenderWriteAtEndOfPipe32(uint64_t submit_id, CommandBuffer* buffer,
uint32_t* dst_gpu_addr, uint32_t value);
void GraphicsRenderWriteAtEndOfPipeGds32(uint64_t submit_id, CommandBuffer* buffer,
uint32_t* dst_gpu_addr, uint32_t dw_offset,
uint32_t dw_num);
void GraphicsRenderWriteAtEndOfPipe64(uint64_t submit_id, CommandBuffer* buffer,
uint64_t* dst_gpu_addr, uint64_t value);
void GraphicsRenderWriteAtEndOfPipeClockCounter(uint64_t submit_id, CommandBuffer* buffer,
uint64_t* dst_gpu_addr, uint64_t value);
void GraphicsRenderWriteAtEndOfPipeClockCounterWithWriteBack(uint64_t submit_id,
CommandBuffer* buffer,
uint64_t* dst_gpu_addr,
uint64_t value);
void GraphicsRenderWriteAtEndOfPipe32(uint64_t submit_id, CommandBuffer* buffer,
uint32_t* dst_gpu_addr, uint32_t value);
void GraphicsRenderWriteAtEndOfPipeGds32(uint64_t submit_id, CommandBuffer* buffer,
uint32_t* dst_gpu_addr, uint32_t dw_offset,
uint32_t dw_num);
uint64_t GraphicsRenderPrepareDisplayBufferFlip(CommandBuffer* buffer, int handle, int index,
int flip_mode, int64_t flip_arg);
void GraphicsRenderWriteAtEndOfPipeWithInterruptWriteBackFlip32(
void GraphicsRenderWriteAtEndOfPipeWithInterruptWriteBackFlip32(
uint64_t submit_id, CommandBuffer* buffer, uint32_t* dst_gpu_addr, uint32_t value, int handle,
int index, int flip_mode, int64_t flip_arg, uint64_t request_id);
void GraphicsRenderWriteAtEndOfPipeWithFlip32(uint64_t submit_id, CommandBuffer* buffer,
@ -193,7 +196,6 @@ int GraphicsRenderDeleteEqEvent(LibKernel::EventQueue::KernelEqueue eq, int id)
void GraphicsRenderTriggerAgcUserInterrupt();
void GraphicsRenderTriggerEopEvent(uint32_t context_id);
void GraphicsRenderClearGds(uint64_t dw_offset, uint32_t dw_num, uint32_t clear_value);
void GraphicsRenderReadGds(uint32_t* dst, uint32_t dw_offset, uint32_t dw_size);
} // namespace Libs::Graphics

54
src/graphics/host_gpu/renderer/renderCompute.cpp
View File

@ -1,7 +1,3 @@
#include "graphics/host_gpu/renderer/render.h"
#include "graphics/host_gpu/renderer/renderInternal.h"
#include "common/assert.h"
#include "common/common.h"
#include "common/emulatorConfig.h"
@ -19,7 +15,9 @@
#include "graphics/host_gpu/renderer/descriptorCache.h"
#include "graphics/host_gpu/renderer/framebufferCache.h"
#include "graphics/host_gpu/renderer/pipelineCache.h"
#include "graphics/host_gpu/renderer/render.h"
#include "graphics/host_gpu/renderer/renderContext.h"
#include "graphics/host_gpu/renderer/renderInternal.h"
#include "graphics/host_gpu/renderer/renderState.h"
#include "graphics/host_gpu/renderer/shaderResourceBarrier.h"
#include "graphics/host_gpu/renderer/shaderSubgroup.h"
@ -547,30 +545,38 @@ void RenderDispatchDirect(uint64_t submit_id, CommandBuffer* buffer, HW::Context
return;
}
auto* vk_buffer = buffer->GetPool()->buffers[buffer->GetIndex()];
auto* pipeline = g_render_ctx->GetPipelineCache()->CreateComputePipeline(
&input_info, &sh_ctx->GetCs(), cs_shader);
for (;;) {
const auto recording_generation = buffer->GetRecordingGeneration();
auto* vk_buffer = buffer->GetPool()->buffers[buffer->GetIndex()];
auto* pipeline = g_render_ctx->GetPipelineCache()->CreateComputePipeline(
&input_info, &sh_ctx->GetCs(), cs_shader);
vkCmdBindPipeline(vk_buffer, VK_PIPELINE_BIND_POINT_COMPUTE, pipeline->pipeline);
vkCmdBindPipeline(vk_buffer, VK_PIPELINE_BIND_POINT_COMPUTE, pipeline->pipeline);
const auto address_writes = BindDescriptors(
submit_id, buffer, VK_PIPELINE_BIND_POINT_COMPUTE, pipeline->pipeline_layout,
input_info.stage, VK_SHADER_STAGE_COMPUTE_BIT, DescriptorCache::Stage::Compute);
const auto address_writes = BindDescriptors(
submit_id, buffer, VK_PIPELINE_BIND_POINT_COMPUTE, pipeline->pipeline_layout,
input_info.stage, VK_SHADER_STAGE_COMPUTE_BIT, DescriptorCache::Stage::Compute);
if (buffer->GetRecordingGeneration() != recording_generation) {
continue;
}
vkCmdDispatch(vk_buffer, thread_group_x, thread_group_y, thread_group_z);
vkCmdDispatch(vk_buffer, thread_group_x, thread_group_y, thread_group_z);
bool has_storage_writes = HasShaderBufferWrites(input_info.stage);
has_storage_writes = MarkShaderAddressWrites(address_writes) || has_storage_writes;
has_storage_writes =
std::any_of(program.info.images.begin(), program.info.images.end(),
[](const auto& image) {
return image.written &&
(image.kind == ShaderRecompiler::IR::ResourceKind::StorageImage ||
image.kind == ShaderRecompiler::IR::ResourceKind::StorageImageUint);
}) ||
has_storage_writes;
if (has_storage_writes) {
ShaderWriteBarrier(vk_buffer, VK_PIPELINE_STAGE_COMPUTE_SHADER_BIT);
bool has_storage_writes = HasShaderBufferWrites(input_info.stage);
has_storage_writes = MarkShaderAddressWrites(address_writes) || has_storage_writes;
has_storage_writes =
std::any_of(
program.info.images.begin(), program.info.images.end(),
[](const auto& image) {
return image.written &&
(image.kind == ShaderRecompiler::IR::ResourceKind::StorageImage ||
image.kind == ShaderRecompiler::IR::ResourceKind::StorageImageUint);
}) ||
has_storage_writes;
if (has_storage_writes) {
ShaderWriteBarrier(vk_buffer, VK_PIPELINE_STAGE_COMPUTE_SHADER_BIT);
}
break;
}
}

256
src/graphics/host_gpu/renderer/renderDraw.cpp
View File

@ -1,7 +1,3 @@
#include "graphics/host_gpu/renderer/render.h"
#include "graphics/host_gpu/renderer/renderInternal.h"
#include "common/assert.h"
#include "common/common.h"
#include "common/emulatorConfig.h"
@ -19,7 +15,9 @@
#include "graphics/host_gpu/renderer/descriptorCache.h"
#include "graphics/host_gpu/renderer/framebufferCache.h"
#include "graphics/host_gpu/renderer/pipelineCache.h"
#include "graphics/host_gpu/renderer/render.h"
#include "graphics/host_gpu/renderer/renderContext.h"
#include "graphics/host_gpu/renderer/renderInternal.h"
#include "graphics/host_gpu/renderer/renderState.h"
#include "graphics/host_gpu/renderer/shaderResourceBarrier.h"
#include "graphics/host_gpu/renderer/shaderSubgroup.h"
@ -603,11 +601,12 @@ struct DrawEmitInfo {
uint32_t first_vertex = 0;
};
struct DrawIndexBufferBind {
bool enabled = false;
VulkanBuffer* buffer = nullptr;
VkDeviceSize offset = 0;
VkIndexType type = VK_INDEX_TYPE_UINT16;
struct DrawIndexBufferSource {
bool enabled = false;
uint64_t address = 0;
const void* host_data = nullptr;
uint64_t size = 0;
VkIndexType type = VK_INDEX_TYPE_UINT16;
};
static uint64_t VertexBufferDescriptorSize(const ShaderVertexInputBuffer& buffer) {
@ -802,6 +801,32 @@ static void BindDrawVertexBuffers(uint64_t submit_id, CommandBuffer* buffer,
}
}
static void BindDrawIndexBuffer(CommandBuffer* buffer, VkCommandBuffer vk_buffer,
const DrawIndexBufferSource& source) {
if (!source.enabled) {
return;
}
EXIT_IF(source.size == 0);
VulkanBuffer* index_buffer = nullptr;
VkDeviceSize index_offset = 0;
if (source.host_data != nullptr) {
VkDeviceSize range = 0;
if (!g_render_ctx->GetBufferCache()->UploadHostData(
buffer, g_render_ctx->GetGraphicCtx(), source.host_data, source.size, 16,
&index_buffer, &index_offset, &range)) {
EXIT("failed to upload host index buffer\n");
}
} else {
auto binding = g_render_ctx->GetBufferCache()->ObtainBuffer(
buffer, g_render_ctx->GetGraphicCtx(), source.address, source.size);
index_buffer = binding.first;
index_offset = binding.second;
}
EXIT_IF(index_buffer == nullptr);
vkCmdBindIndexBuffer(vk_buffer, index_buffer->buffer, index_offset, source.type);
}
static void LogDrawStateIfNeeded(HW::Context* ctx, HW::UserConfig* ucfg, const DrawCallInfo& draw,
const DrawRenderState& state, bool always_log,
bool force_legacy_rect_log, uint32_t index_type_and_size,
@ -898,7 +923,7 @@ static void EmitDrawPrimitives(const HW::UserConfig* ucfg, VkCommandBuffer vk_bu
static void ExecutePreparedDraw(uint64_t submit_id, CommandBuffer* buffer, HW::Context* ctx,
HW::UserConfig* ucfg, HW::Shader* sh_ctx, const DrawCallInfo& draw,
DrawRenderState* state, VkPrimitiveTopology topology,
const DrawEmitInfo& emit, const DrawIndexBufferBind& index_bind,
const DrawEmitInfo& emit, const DrawIndexBufferSource& index_source,
bool log_pipeline_phase, bool set_bind_debug, bool set_auto_debug) {
EXIT_IF(buffer == nullptr);
EXIT_IF(ctx == nullptr);
@ -907,86 +932,93 @@ static void ExecutePreparedDraw(uint64_t submit_id, CommandBuffer* buffer, HW::C
EXIT_IF(draw.name == nullptr);
EXIT_IF(state == nullptr);
if (log_pipeline_phase) {
LogDrawPhase(draw.name, "CreatePipeline");
}
auto* pipeline = g_render_ctx->GetPipelineCache()->CreateGraphicsPipeline(
state->framebuffer, state->color_info, state->color_count, &state->depth_info,
&state->vs_input_info, ctx, sh_ctx, &state->ps_input_info, topology, state->ps_active,
state->vs_shader, state->ps_shader);
for (;;) {
const auto recording_generation = buffer->GetRecordingGeneration();
if (log_pipeline_phase) {
LogDrawPhase(draw.name, "CreatePipeline");
}
auto* pipeline = g_render_ctx->GetPipelineCache()->CreateGraphicsPipeline(
state->framebuffer, state->color_info, state->color_count, &state->depth_info,
&state->vs_input_info, ctx, sh_ctx, &state->ps_input_info, topology, state->ps_active,
state->vs_shader, state->ps_shader);
if (set_bind_debug) {
SetDrawDebugPhase(buffer, submit_id, draw, 0x100u);
}
vkCmdBindPipeline(state->vk_buffer, VK_PIPELINE_BIND_POINT_GRAPHICS, pipeline->pipeline);
const auto dynamic_params =
BuildGraphicsDynamicParams(*ctx, state->color_info, state->color_count, state->depth_info);
SetDynamicParams(state->vk_buffer, dynamic_params);
if (set_bind_debug) {
SetDrawDebugPhase(buffer, submit_id, draw, 0x100u);
}
vkCmdBindPipeline(state->vk_buffer, VK_PIPELINE_BIND_POINT_GRAPHICS, pipeline->pipeline);
const auto dynamic_params = BuildGraphicsDynamicParams(
*ctx, state->color_info, state->color_count, state->depth_info);
SetDynamicParams(state->vk_buffer, dynamic_params);
// EXIT_NOT_IMPLEMENTED(vs_input_info.buffers_num > 1);
BindDrawVertexBuffers(submit_id, buffer, draw, state->vk_buffer, state->vs_input_info);
// EXIT_NOT_IMPLEMENTED(vs_input_info.buffers_num > 1);
BindDrawVertexBuffers(submit_id, buffer, draw, state->vk_buffer, state->vs_input_info);
LogDrawPhase(draw.name, "BindDescriptorsVS");
if (set_auto_debug) {
SetDrawDebugPhase(buffer, submit_id, draw, 0x200u);
}
const auto vs_address_writes = BindDescriptors(
submit_id, buffer, VK_PIPELINE_BIND_POINT_GRAPHICS, pipeline->pipeline_layout,
state->vs_input_info.stage, VK_SHADER_STAGE_VERTEX_BIT, DescriptorCache::Stage::Vertex);
std::vector<ShaderAddressWriteRange> ps_address_writes;
if (state->ps_active) {
LogDrawPhase(draw.name, "BindDescriptorsPS");
LogDrawPhase(draw.name, "BindDescriptorsVS");
if (set_auto_debug) {
SetDrawDebugPhase(buffer, submit_id, draw, 0x300u);
SetDrawDebugPhase(buffer, submit_id, draw, 0x200u);
}
ps_address_writes =
BindDescriptors(submit_id, buffer, VK_PIPELINE_BIND_POINT_GRAPHICS,
pipeline->pipeline_layout, state->ps_input_info.stage,
VK_SHADER_STAGE_FRAGMENT_BIT, DescriptorCache::Stage::Pixel);
}
const auto vs_address_writes = BindDescriptors(
submit_id, buffer, VK_PIPELINE_BIND_POINT_GRAPHICS, pipeline->pipeline_layout,
state->vs_input_info.stage, VK_SHADER_STAGE_VERTEX_BIT, DescriptorCache::Stage::Vertex);
if (index_bind.enabled) {
EXIT_NOT_IMPLEMENTED(index_bind.buffer == nullptr);
vkCmdBindIndexBuffer(state->vk_buffer, index_bind.buffer->buffer, index_bind.offset,
index_bind.type);
}
LogDrawPhase(draw.name, "BeginRenderPass");
if (set_auto_debug) {
SetDrawDebugPhase(buffer, submit_id, draw, 0x400u);
}
buffer->BeginRenderPass(state->framebuffer, state->color_info, state->color_count,
&state->depth_info);
if (set_auto_debug) {
SetDrawDebugPhase(buffer, submit_id, draw, 0x500u);
}
EmitDrawPrimitives(ucfg, state->vk_buffer, state->vs_input_info, draw, emit);
if (set_auto_debug) {
SetDrawDebugPhase(buffer, submit_id, draw, 0x600u);
}
buffer->EndRenderPass();
VkPipelineStageFlags shader_write_stages = 0;
const bool vs_wrote_buffers = HasShaderBufferWrites(state->vs_input_info.stage);
const bool vs_wrote_addresses = MarkShaderAddressWrites(vs_address_writes);
if (vs_wrote_buffers || vs_wrote_addresses) {
shader_write_stages |= VK_PIPELINE_STAGE_VERTEX_SHADER_BIT;
}
if (state->ps_active) {
const bool ps_wrote_buffers = HasShaderBufferWrites(state->ps_input_info.stage);
const bool ps_wrote_addresses = MarkShaderAddressWrites(ps_address_writes);
if (ps_wrote_buffers || ps_wrote_addresses) {
shader_write_stages |= VK_PIPELINE_STAGE_FRAGMENT_SHADER_BIT;
std::vector<ShaderAddressWriteRange> ps_address_writes;
if (state->ps_active) {
LogDrawPhase(draw.name, "BindDescriptorsPS");
if (set_auto_debug) {
SetDrawDebugPhase(buffer, submit_id, draw, 0x300u);
}
ps_address_writes =
BindDescriptors(submit_id, buffer, VK_PIPELINE_BIND_POINT_GRAPHICS,
pipeline->pipeline_layout, state->ps_input_info.stage,
VK_SHADER_STAGE_FRAGMENT_BIT, DescriptorCache::Stage::Pixel);
}
}
if (shader_write_stages != 0) {
ShaderWriteBarrier(state->vk_buffer, shader_write_stages);
}
LogDrawPhase(draw.name, "EndRenderPass");
if (set_auto_debug) {
SetDrawDebugPhase(buffer, submit_id, draw, 0x700u);
if (buffer->GetRecordingGeneration() != recording_generation) {
continue;
}
// Index data may use this command buffer's host stream. Resolve it only after the other
// fault-capable bindings, and rebuild it whenever a fault reset changes the generation.
BindDrawIndexBuffer(buffer, state->vk_buffer, index_source);
if (buffer->GetRecordingGeneration() != recording_generation) {
continue;
}
LogDrawPhase(draw.name, "BeginRenderPass");
if (set_auto_debug) {
SetDrawDebugPhase(buffer, submit_id, draw, 0x400u);
}
buffer->BeginRenderPass(state->framebuffer, state->color_info, state->color_count,
&state->depth_info);
if (set_auto_debug) {
SetDrawDebugPhase(buffer, submit_id, draw, 0x500u);
}
EmitDrawPrimitives(ucfg, state->vk_buffer, state->vs_input_info, draw, emit);
if (set_auto_debug) {
SetDrawDebugPhase(buffer, submit_id, draw, 0x600u);
}
buffer->EndRenderPass();
VkPipelineStageFlags shader_write_stages = 0;
const bool vs_wrote_buffers = HasShaderBufferWrites(state->vs_input_info.stage);
const bool vs_wrote_addresses = MarkShaderAddressWrites(vs_address_writes);
if (vs_wrote_buffers || vs_wrote_addresses) {
shader_write_stages |= VK_PIPELINE_STAGE_VERTEX_SHADER_BIT;
}
if (state->ps_active) {
const bool ps_wrote_buffers = HasShaderBufferWrites(state->ps_input_info.stage);
const bool ps_wrote_addresses = MarkShaderAddressWrites(ps_address_writes);
if (ps_wrote_buffers || ps_wrote_addresses) {
shader_write_stages |= VK_PIPELINE_STAGE_FRAGMENT_SHADER_BIT;
}
}
if (shader_write_stages != 0) {
ShaderWriteBarrier(state->vk_buffer, shader_write_stages);
}
LogDrawPhase(draw.name, "EndRenderPass");
if (set_auto_debug) {
SetDrawDebugPhase(buffer, submit_id, draw, 0x700u);
}
break;
}
}
@ -1068,7 +1100,7 @@ void RenderDrawIndex(uint64_t submit_id, CommandBuffer* buffer, HW::Context* ctx
index_type = VK_INDEX_TYPE_UINT32;
index_size = 4 * static_cast<uint64_t>(index_count);
break;
// Some games use it
// Some games use it - need vulkan extension
case Prospero::IndexType::kIndex8:
index_type = VK_INDEX_TYPE_UINT16;
index_size = static_cast<uint64_t>(index_count);
@ -1086,6 +1118,25 @@ void RenderDrawIndex(uint64_t submit_id, CommandBuffer* buffer, HW::Context* ctx
const DrawCallInfo draw {"DrawIndex", CommandBufferDebugOp::DrawIndex,
index_count, flags,
instance_count, first_instance};
std::vector<uint16_t> expanded_indices;
if (expand_index8_to_u16) {
EXIT_NOT_IMPLEMENTED(index_addr == nullptr);
const auto* src = static_cast<const uint8_t*>(index_addr);
expanded_indices.resize(index_count);
for (uint32_t i = 0; i < index_count; i++) {
expanded_indices[i] = src[i];
}
}
DrawIndexBufferSource index_source {};
index_source.enabled = true;
index_source.address = reinterpret_cast<uint64_t>(index_addr);
index_source.host_data =
expanded_indices.empty() ? nullptr : static_cast<const void*>(expanded_indices.data());
index_source.size = expanded_indices.empty()
? index_size
: expanded_indices.size() * sizeof(uint16_t);
index_source.type = index_type;
DrawRenderState state {};
if (!PrepareDrawRenderState(submit_id, buffer, ctx, ucfg, sh_ctx, draw,
@ -1100,43 +1151,12 @@ void RenderDrawIndex(uint64_t submit_id, CommandBuffer* buffer, HW::Context* ctx
const auto vertex_offset =
ResolveVertexOffset(ucfg->GetIndexOffset(), state.vs_input_info) + vertex_offset_add;
VkDeviceSize index_offset = 0;
VulkanBuffer* indices = nullptr;
if (expand_index8_to_u16) {
EXIT_NOT_IMPLEMENTED(index_addr == nullptr);
const auto* src = static_cast<const uint8_t*>(index_addr);
std::vector<uint16_t> expanded_indices(index_count);
for (uint32_t i = 0; i < index_count; i++) {
expanded_indices[i] = src[i];
}
VkDeviceSize range = 0;
if (!g_render_ctx->GetBufferCache()->UploadHostData(
buffer, g_render_ctx->GetGraphicCtx(), expanded_indices.data(),
expanded_indices.size() * sizeof(uint16_t), 16, &indices, &index_offset, &range)) {
EXIT("failed to upload expanded 8-bit index buffer\n");
}
} else {
auto binding = g_render_ctx->GetBufferCache()->ObtainBuffer(
buffer, g_render_ctx->GetGraphicCtx(), reinterpret_cast<uint64_t>(index_addr),
index_size);
indices = binding.first;
index_offset = binding.second;
}
EXIT_NOT_IMPLEMENTED(indices == nullptr);
DrawIndexBufferBind index_bind {};
index_bind.enabled = true;
index_bind.buffer = indices;
index_bind.offset = index_offset;
index_bind.type = index_type;
DrawEmitInfo emit {};
emit.indexed = true;
emit.vertex_offset = vertex_offset;
ExecutePreparedDraw(submit_id, buffer, ctx, ucfg, sh_ctx, draw, &state, topology, emit,
index_bind, true, true, false);
index_source, true, true, false);
}
// NOLINTNEXTLINE(readability-function-cognitive-complexity)
@ -1243,9 +1263,9 @@ void RenderDrawIndexAuto(uint64_t submit_id, CommandBuffer* buffer, HW::Context*
emit.draw_vertex_count = draw_vertex_count;
emit.first_vertex = static_cast<uint32_t>(vertex_offset);
DrawIndexBufferBind index_bind {};
DrawIndexBufferSource index_source {};
ExecutePreparedDraw(submit_id, buffer, ctx, ucfg, sh_ctx, draw, &state, topology, emit,
index_bind, false, false, true);
index_source, false, false, true);
}
bool IsSameColorResolveSubresource(const RenderColorInfo& src, const RenderColorInfo& dst) {

8
src/graphics/host_gpu/renderer/resourceMutex.h
View File

@ -8,8 +8,8 @@
namespace Libs::Graphics {
// Owner-tracked shared buffer/image transaction. External faults pause GPU submissions before
// taking this mutex; synchronous command-processor accesses reuse their already-owned transaction.
// Owner-tracked shared buffer/image transaction. External faults pause GPU submissions first;
// command-processor faults drain their queue before entering this transaction.
class ResourceMutex final {
public:
class FaultScope final {
@ -27,8 +27,8 @@ public:
~ResourceMutex();
KYTY_CLASS_NO_COPY(ResourceMutex);
void lock();
void unlock();
void lock();
void unlock();
[[nodiscard]] bool IsOwnedByCurrentThread();
private:

135
src/graphics/host_gpu/renderer/sync.cpp
View File

@ -13,31 +13,29 @@
#include "kernel/pthread.h"
#include "libs/errno.h"
#include <cstring>
#include <limits>
namespace Libs::Graphics {
constexpr int GRAPHICS_EVENT_QUEUED_GRAPHICS_INTERRUPT = 0x00;
constexpr int GRAPHICS_EVENT_EOP = 0x40;
constexpr uint64_t GRAPHICS_REFERENCE_CLOCK_FREQUENCY = 100000000;
constexpr int GRAPHICS_EVENT_QUEUED_GRAPHICS_INTERRUPT = 0x00;
constexpr int GRAPHICS_EVENT_EOP = 0x40;
constexpr uint64_t GRAPHICS_REFERENCE_CLOCK_FREQUENCY = 100000000;
bool GraphicsScaleReferenceClock(uint64_t host_ticks, uint64_t host_frequency, uint64_t* value) {
if (host_frequency == 0 || value == nullptr) {
return false;
}
const auto whole_seconds = host_ticks / host_frequency;
const auto remainder = host_ticks % host_frequency;
constexpr auto MAX_VALUE = std::numeric_limits<uint64_t>::max();
const auto whole_seconds = host_ticks / host_frequency;
const auto remainder = host_ticks % host_frequency;
constexpr auto MAX_VALUE = std::numeric_limits<uint64_t>::max();
if (whole_seconds > MAX_VALUE / GRAPHICS_REFERENCE_CLOCK_FREQUENCY ||
remainder > MAX_VALUE / GRAPHICS_REFERENCE_CLOCK_FREQUENCY) {
return false;
}
const auto whole_value = whole_seconds * GRAPHICS_REFERENCE_CLOCK_FREQUENCY;
const auto fractional_value =
(remainder * GRAPHICS_REFERENCE_CLOCK_FREQUENCY) / host_frequency;
const auto whole_value = whole_seconds * GRAPHICS_REFERENCE_CLOCK_FREQUENCY;
const auto fractional_value = (remainder * GRAPHICS_REFERENCE_CLOCK_FREQUENCY) / host_frequency;
if (whole_value > MAX_VALUE - fractional_value) {
return false;
}
@ -50,40 +48,19 @@ uint64_t GraphicsRenderReadReferenceClock() {
const auto host_ticks = LibKernel::KernelReadTsc();
uint64_t value = 0;
if (!GraphicsScaleReferenceClock(host_ticks, host_frequency, &value)) {
EXIT("cannot scale host clock, ticks=0x%016" PRIx64 " frequency=%" PRIu64 "\n",
host_ticks, host_frequency);
EXIT("cannot scale host clock, ticks=0x%016" PRIx64 " frequency=%" PRIu64 "\n", host_ticks,
host_frequency);
}
return value;
}
static void SubmitLabel32(CommandBuffer* buffer, uint32_t* dst, uint32_t value,
LabelCallback callback_1 = nullptr, LabelCallback callback_2 = nullptr,
const uint64_t* args = nullptr) {
auto* label =
LabelCreate32(g_render_ctx->GetGraphicCtx(), dst, value, callback_1, callback_2, args);
static void SubmitLabel(CommandBuffer* buffer, LabelCallback callback_1 = nullptr,
LabelCallback callback_2 = nullptr, const uint64_t* args = nullptr) {
auto* label = LabelCreate(g_render_ctx->GetGraphicCtx(), callback_1, callback_2, args);
LabelSet(buffer, label);
LabelDelete(label);
}
static void SubmitLabel64(CommandBuffer* buffer, uint64_t* dst, uint64_t value,
LabelCallback callback_1 = nullptr, LabelCallback callback_2 = nullptr,
const uint64_t* args = nullptr) {
auto* label =
LabelCreate64(g_render_ctx->GetGraphicCtx(), dst, value, callback_1, callback_2, args);
LabelSet(buffer, label);
LabelDelete(label);
}
template <typename T>
static void PublishImmediateFence(T* dst, T value) {
static_assert(sizeof(T) == sizeof(uint32_t) || sizeof(T) == sizeof(uint64_t));
EXIT_IF(dst == nullptr);
// EOP/EOS fence payloads are constant and published while parsing the packet. Kyty performs the
// direct mapped write under CommandProcessorGuestAccessScope: a protected access is first resolved by
// BufferCache inline, then this instruction retries so the EOP payload wins over prior GPU bytes.
std::memcpy(dst, &value, sizeof(value));
}
static bool CompleteDisplayBufferFlip(const uint64_t* args) {
if (g_render_ctx == nullptr || args == nullptr) {
EXIT("GPU flip completion has invalid state, render_ctx=%p args=%p\n",
@ -114,8 +91,6 @@ void GraphicsRenderWriteAtEndOfPipe32(uint64_t submit_id, CommandBuffer* buffer,
buffer->SetDebugInfo(static_cast<uint32_t>(CommandBufferDebugOp::EopWrite), submit_id, 4, value,
0, 0, reinterpret_cast<uint64_t>(dst_gpu_addr));
PublishImmediateFence(dst_gpu_addr, value);
}
void GraphicsRenderWriteAtEndOfPipeGds32(uint64_t submit_id, CommandBuffer* buffer,
@ -128,28 +103,6 @@ void GraphicsRenderWriteAtEndOfPipeGds32(uint64_t submit_id, CommandBuffer* buff
buffer->SetDebugInfo(static_cast<uint32_t>(CommandBufferDebugOp::EopWrite), submit_id,
dw_offset, dw_num, 0, 0, reinterpret_cast<uint64_t>(dst_gpu_addr));
uint64_t args[LABEL_ARGS_MAX] = {static_cast<uint64_t>(dw_offset),
static_cast<uint64_t>(dw_num),
reinterpret_cast<uint64_t>(dst_gpu_addr), 0};
SubmitLabel32(
buffer, dst_gpu_addr, 0,
[](const uint64_t* args) {
auto dw_offset = static_cast<uint32_t>(args[0]);
auto dw_num = static_cast<uint32_t>(args[1]);
auto* dst_gpu_addr = reinterpret_cast<uint32_t*>(args[2]);
if (dst_gpu_addr == nullptr || dw_num == 0) {
EXIT("invalid asynchronous EOP GDS write\n");
}
const auto size = static_cast<uint64_t>(dw_num) * sizeof(uint32_t);
auto transaction = g_render_ctx->GetGpuResources()->LockAsynchronousGuestWrite(
reinterpret_cast<uint64_t>(dst_gpu_addr), size);
g_render_ctx->GetGdsBuffer()->Read(g_render_ctx->GetGraphicCtx(), dst_gpu_addr,
dw_offset, dw_num);
return false;
},
nullptr, args);
}
void GraphicsRenderWriteAtEndOfPipe64(uint64_t submit_id, CommandBuffer* buffer,
@ -162,12 +115,10 @@ void GraphicsRenderWriteAtEndOfPipe64(uint64_t submit_id, CommandBuffer* buffer,
buffer->SetDebugInfo(static_cast<uint32_t>(CommandBufferDebugOp::EopWrite), submit_id, 8,
static_cast<uint32_t>(value), static_cast<uint32_t>(value >> 32u), 0,
reinterpret_cast<uint64_t>(dst_gpu_addr));
PublishImmediateFence(dst_gpu_addr, value);
}
void GraphicsRenderWriteAtEndOfPipeClockCounter(uint64_t submit_id, CommandBuffer* buffer,
uint64_t* dst_gpu_addr) {
uint64_t* dst_gpu_addr, uint64_t value) {
EXIT_IF(g_render_ctx == nullptr);
EXIT_IF(dst_gpu_addr == nullptr);
EXIT_IF(buffer == nullptr);
@ -176,8 +127,6 @@ void GraphicsRenderWriteAtEndOfPipeClockCounter(uint64_t submit_id, CommandBuffe
buffer->SetDebugInfo(static_cast<uint32_t>(CommandBufferDebugOp::EopWrite), submit_id, 8, 0, 0,
0, reinterpret_cast<uint64_t>(dst_gpu_addr));
const auto value = GraphicsRenderReadReferenceClock();
PublishImmediateFence(dst_gpu_addr, value);
LOGF_COLOR(Log::Color::BrightGreen,
"EndOfPipe Signal!!! [0x%016" PRIx64 "] <- Clock: 0x%016" PRIx64 "\n",
reinterpret_cast<uint64_t>(dst_gpu_addr), value);
@ -185,7 +134,8 @@ void GraphicsRenderWriteAtEndOfPipeClockCounter(uint64_t submit_id, CommandBuffe
void GraphicsRenderWriteAtEndOfPipeClockCounterWithWriteBack(uint64_t submit_id,
CommandBuffer* buffer,
uint64_t* dst_gpu_addr) {
uint64_t* dst_gpu_addr,
uint64_t value) {
EXIT_IF(g_render_ctx == nullptr);
EXIT_IF(dst_gpu_addr == nullptr);
EXIT_IF(buffer == nullptr);
@ -194,8 +144,6 @@ void GraphicsRenderWriteAtEndOfPipeClockCounterWithWriteBack(uint64_t subm
buffer->SetDebugInfo(static_cast<uint32_t>(CommandBufferDebugOp::EopWriteBack), submit_id, 8, 0,
0, 0, reinterpret_cast<uint64_t>(dst_gpu_addr));
const auto value = GraphicsRenderReadReferenceClock();
PublishImmediateFence(dst_gpu_addr, value);
LOGF_COLOR(Log::Color::BrightGreen,
"EndOfPipe Signal!!! [0x%016" PRIx64 "] <- Clock: 0x%016" PRIx64 "\n",
reinterpret_cast<uint64_t>(dst_gpu_addr), value);
@ -211,8 +159,6 @@ void GraphicsRenderWriteAtEndOfPipeWithWriteBack64(uint64_t submit_id, CommandBu
buffer->SetDebugInfo(static_cast<uint32_t>(CommandBufferDebugOp::EopWriteBack), submit_id, 8,
static_cast<uint32_t>(value), static_cast<uint32_t>(value >> 32u), 0,
reinterpret_cast<uint64_t>(dst_gpu_addr));
PublishImmediateFence(dst_gpu_addr, value);
}
void GraphicsRenderWriteAtEndOfPipeWithWriteBack32(uint64_t submit_id, CommandBuffer* buffer,
@ -224,8 +170,6 @@ void GraphicsRenderWriteAtEndOfPipeWithWriteBack32(uint64_t submit_id, CommandBu
buffer->SetDebugInfo(static_cast<uint32_t>(CommandBufferDebugOp::EopWriteBack), submit_id, 4,
value, 0, 0, reinterpret_cast<uint64_t>(dst_gpu_addr));
PublishImmediateFence(dst_gpu_addr, value);
}
void GraphicsRenderWriteAtEndOfPipeWithInterruptWriteBack64(uint64_t submit_id,
@ -241,12 +185,10 @@ void GraphicsRenderWriteAtEndOfPipeWithInterruptWriteBack64(uint64_t submi
context_id, static_cast<uint32_t>(value),
static_cast<uint32_t>(value >> 32u),
reinterpret_cast<uint64_t>(dst_gpu_addr));
PublishImmediateFence(dst_gpu_addr, value);
uint64_t args[LABEL_ARGS_MAX] = {context_id != 0 ? context_id : value};
SubmitLabel64(
buffer, nullptr, 0, nullptr,
SubmitLabel(
buffer, nullptr,
[](const uint64_t* args) {
EXIT_IF(g_render_ctx == nullptr);
g_render_ctx->TriggerEopEvent(static_cast<uint32_t>(args[0]));
@ -266,12 +208,10 @@ void GraphicsRenderWriteAtEndOfPipeWithInterruptWriteBack32(uint64_t submi
buffer->SetDebugInfo(static_cast<uint32_t>(CommandBufferDebugOp::EopWriteBack), submit_id, 4,
context_id, value, 0, reinterpret_cast<uint64_t>(dst_gpu_addr));
PublishImmediateFence(dst_gpu_addr, value);
uint64_t args[LABEL_ARGS_MAX] = {context_id != 0 ? context_id : value};
SubmitLabel32(
buffer, nullptr, 0, nullptr,
SubmitLabel(
buffer, nullptr,
[](const uint64_t* args) {
EXIT_IF(g_render_ctx == nullptr);
g_render_ctx->TriggerEopEvent(static_cast<uint32_t>(args[0]));
@ -292,12 +232,10 @@ void GraphicsRenderWriteAtEndOfPipeWithInterrupt64(uint64_t submit_id, CommandBu
context_id, static_cast<uint32_t>(value),
static_cast<uint32_t>(value >> 32u),
reinterpret_cast<uint64_t>(dst_gpu_addr));
PublishImmediateFence(dst_gpu_addr, value);
uint64_t args[LABEL_ARGS_MAX] = {context_id != 0 ? context_id : value};
SubmitLabel64(
buffer, nullptr, 0, nullptr,
SubmitLabel(
buffer, nullptr,
[](const uint64_t* args) {
EXIT_IF(g_render_ctx == nullptr);
g_render_ctx->TriggerEopEvent(static_cast<uint32_t>(args[0]));
@ -316,12 +254,10 @@ void GraphicsRenderWriteAtEndOfPipeWithInterrupt32(uint64_t submit_id, CommandBu
buffer->SetDebugInfo(static_cast<uint32_t>(CommandBufferDebugOp::EopInterrupt), submit_id, 4,
context_id, value, 0, reinterpret_cast<uint64_t>(dst_gpu_addr));
PublishImmediateFence(dst_gpu_addr, value);
uint64_t args[LABEL_ARGS_MAX] = {context_id != 0 ? context_id : value};
SubmitLabel32(
buffer, nullptr, 0, nullptr,
SubmitLabel(
buffer, nullptr,
[](const uint64_t* args) {
EXIT_IF(g_render_ctx == nullptr);
g_render_ctx->TriggerEopEvent(static_cast<uint32_t>(args[0]));
@ -360,11 +296,9 @@ void GraphicsRenderWriteAtEndOfPipeWithInterruptWriteBackFlip32(
buffer->SetDebugInfo(static_cast<uint32_t>(CommandBufferDebugOp::EopWriteBackFlip), submit_id,
static_cast<uint32_t>(handle), static_cast<uint32_t>(index),
static_cast<uint32_t>(flip_mode), value, static_cast<uint64_t>(flip_arg));
PublishImmediateFence(dst_gpu_addr, value);
const uint64_t args[LABEL_ARGS_MAX] = {request_id};
SubmitLabel32(
buffer, nullptr, 0, CompleteDisplayBufferFlip,
SubmitLabel(
buffer, CompleteDisplayBufferFlip,
[](const uint64_t* /*args*/) {
EXIT_IF(g_render_ctx == nullptr);
g_render_ctx->TriggerEopEvent(0);
@ -385,10 +319,8 @@ void GraphicsRenderWriteAtEndOfPipeWithFlip32(uint64_t submit_id, CommandBuffer*
buffer->SetDebugInfo(static_cast<uint32_t>(CommandBufferDebugOp::EopFlip), submit_id,
static_cast<uint32_t>(handle), static_cast<uint32_t>(index),
static_cast<uint32_t>(flip_mode), value, static_cast<uint64_t>(flip_arg));
PublishImmediateFence(dst_gpu_addr, value);
const uint64_t args[LABEL_ARGS_MAX] = {request_id};
SubmitLabel32(buffer, nullptr, 0, CompleteDisplayBufferFlip, nullptr, args);
SubmitLabel(buffer, CompleteDisplayBufferFlip, nullptr, args);
}
void GraphicsRenderWriteAtEndOfPipeOnlyFlip(uint64_t submit_id, CommandBuffer* buffer, int handle,
@ -403,7 +335,7 @@ void GraphicsRenderWriteAtEndOfPipeOnlyFlip(uint64_t submit_id, CommandBuffer* b
static_cast<uint32_t>(flip_mode), 0, static_cast<uint64_t>(flip_arg));
const uint64_t args[LABEL_ARGS_MAX] = {request_id};
SubmitLabel32(buffer, nullptr, 0, CompleteDisplayBufferFlip, nullptr, args);
SubmitLabel(buffer, CompleteDisplayBufferFlip, nullptr, args);
}
void GraphicsRenderTriggerEopEventAtEndOfPipe(CommandBuffer* buffer, uint32_t context_id) {
@ -413,8 +345,8 @@ void GraphicsRenderTriggerEopEventAtEndOfPipe(CommandBuffer* buffer, uint32_t co
uint64_t args[LABEL_ARGS_MAX] = {static_cast<uint64_t>(context_id)};
SubmitLabel32(
buffer, nullptr, 0, nullptr,
SubmitLabel(
buffer, nullptr,
[](const uint64_t* args) {
EXIT_IF(g_render_ctx == nullptr);
g_render_ctx->TriggerEopEvent(static_cast<uint32_t>(args[0]));
@ -489,13 +421,6 @@ int GraphicsRenderDeleteEqEvent(LibKernel::EventQueue::KernelEqueue eq, int id)
return result;
}
void GraphicsRenderClearGds(uint64_t dw_offset, uint32_t dw_num, uint32_t clear_value) {
EXIT_IF(g_render_ctx == nullptr);
g_render_ctx->GetGdsBuffer()->Clear(g_render_ctx->GetGraphicCtx(), dw_offset, dw_num,
clear_value);
}
void GraphicsRenderReadGds(uint32_t* dst, uint32_t dw_offset, uint32_t dw_size) {
EXIT_IF(g_render_ctx == nullptr);

66
src/graphics/host_gpu/renderer/textureCache.cpp
View File

@ -8,7 +8,6 @@
#include "graphics/guest_gpu/graphicsRun.h"
#include "graphics/guest_gpu/tile.h"
#include "graphics/host_gpu/graphicContext.h"
#include "graphics/host_gpu/hostMemory.h"
#include "graphics/host_gpu/objects/label.h"
#include "graphics/host_gpu/objects/textureCommon.h"
#include "graphics/host_gpu/renderer/bufferCache.h"
@ -232,14 +231,15 @@ struct TextureCache::ReadbackWorker {
}
void Request(PageFaultAccess fault_access, uint64_t fault_vaddr, uint64_t fault_size) noexcept {
const bool submissions_prepaused_now = GraphicsRunSubmissionLockHeld();
const bool command_thread = GraphicsRunIsCommandProcessorThread();
const bool submissions_prepaused_now = GraphicsRunSubmissionLockHeld() || command_thread;
const bool unsafe_gpu_lock = GraphicsRunGpuLockHeld() && !submissions_prepaused_now;
if ((fault_access != PageFaultAccess::Read && fault_access != PageFaultAccess::Write) ||
GraphicsRunIsCommandProcessorThread() || unsafe_gpu_lock || LabelInCallback() ||
g_texture_cache_lock_owner != nullptr || g_texture_fault_owner != &cache) {
unsafe_gpu_lock || LabelInCallback() || g_texture_cache_lock_owner != nullptr ||
g_texture_fault_owner != &cache) {
EXIT("TextureCache: unsafe image readback request, access=%u command_thread=%d "
"submission_lock=%d gpu_lock=%d label_callback=%d cache_lock=%p fault_owner=%p\n",
static_cast<uint32_t>(fault_access), GraphicsRunIsCommandProcessorThread(),
static_cast<uint32_t>(fault_access), command_thread,
GraphicsRunSubmissionLockHeld(), GraphicsRunGpuLockHeld(), LabelInCallback(),
g_texture_cache_lock_owner, g_texture_fault_owner);
}
@ -1557,13 +1557,12 @@ VkImageView TextureCache::GetRenderTargetStorageView(GraphicContext* c
VkImageView TextureCache::GetStorageTextureSampledView(GraphicContext* ctx,
StorageTextureVulkanImage* image,
const ImageInfo& info) {
const auto shape = SelectStorageSampledViewShape(
info.type, info.depth, image != nullptr ? image->layers : 0);
const auto shape =
SelectStorageSampledViewShape(info.type, info.depth, image != nullptr ? image->layers : 0);
if (ctx == nullptr || image == nullptr || image->image == nullptr ||
shape == StorageSampledViewShape::Unsupported ||
info.base_array != 0 || info.levels != image->mip_levels ||
info.base_level >= info.levels || info.view_levels == 0 ||
info.base_level + info.view_levels > info.levels) {
shape == StorageSampledViewShape::Unsupported || info.base_array != 0 ||
info.levels != image->mip_levels || info.base_level >= info.levels ||
info.view_levels == 0 || info.base_level + info.view_levels > info.levels) {
EXIT("TextureCache: invalid sampled view of storage texture, image=%p type=%u depth=%u"
" base=%u levels=%u view_levels=%u image_levels=%u base_array=%u\n",
static_cast<const void*>(image), info.type, info.depth, info.base_level, info.levels,
@ -1589,23 +1588,21 @@ VkImageView TextureCache::GetStorageTextureSampledView(GraphicContext*
usage.sType = VK_STRUCTURE_TYPE_IMAGE_VIEW_USAGE_CREATE_INFO;
usage.usage = VK_IMAGE_USAGE_SAMPLED_BIT;
VkImageViewCreateInfo create {};
create.sType = VK_STRUCTURE_TYPE_IMAGE_VIEW_CREATE_INFO;
create.pNext = &usage;
create.image = image->image;
create.viewType = shape == StorageSampledViewShape::Image3D
? VK_IMAGE_VIEW_TYPE_3D
: shape == StorageSampledViewShape::Image2DArray
? VK_IMAGE_VIEW_TYPE_2D_ARRAY
: VK_IMAGE_VIEW_TYPE_2D;
create.format = view_format;
create.components = TextureGetComponentMapping(info.swizzle);
create.subresourceRange.aspectMask = VK_IMAGE_ASPECT_COLOR_BIT;
create.subresourceRange.baseMipLevel = info.base_level;
create.subresourceRange.levelCount = info.view_levels;
create.sType = VK_STRUCTURE_TYPE_IMAGE_VIEW_CREATE_INFO;
create.pNext = &usage;
create.image = image->image;
create.viewType = shape == StorageSampledViewShape::Image3D ? VK_IMAGE_VIEW_TYPE_3D
: shape == StorageSampledViewShape::Image2DArray ? VK_IMAGE_VIEW_TYPE_2D_ARRAY
: VK_IMAGE_VIEW_TYPE_2D;
create.format = view_format;
create.components = TextureGetComponentMapping(info.swizzle);
create.subresourceRange.aspectMask = VK_IMAGE_ASPECT_COLOR_BIT;
create.subresourceRange.baseMipLevel = info.base_level;
create.subresourceRange.levelCount = info.view_levels;
create.subresourceRange.baseArrayLayer = 0;
create.subresourceRange.layerCount =
shape == StorageSampledViewShape::Image2DArray ? info.depth : 1;
VkImageView view = nullptr;
VkImageView view = nullptr;
const auto result = vkCreateImageView(ctx->device, &create, nullptr, &view);
if (result != VK_SUCCESS || view == nullptr) {
EXIT("TextureCache: failed to create sampled view of storage texture, result=%d"
@ -1926,15 +1923,6 @@ VulkanImage* TextureCache::FindTexture(CommandBuffer* command, GraphicContext* c
cached->gpu_modified, cached->ctx == ctx);
}
}
uint64_t readable = 0;
if (!HostMemoryQueryReadable(info.address, info.size, &readable) || readable < info.size) {
EXIT("TextureCache: new sampled image has unreadable guest backing, addr=0x%016" PRIx64
" size=0x%016" PRIx64 " readable=0x%016" PRIx64
" format=%u extent=%ux%ux%u pitch=%u levels=%u tile=%u type=%u\n",
info.address, info.size, readable, info.format, info.width, info.height, info.depth,
info.pitch, info.levels, info.tile, info.type);
}
m_images.reserve(m_images.size() + 1);
auto cached = std::make_shared<CachedImage>();
cached->kind = CachedImage::Kind::Texture;
@ -2057,13 +2045,6 @@ StorageTextureVulkanImage* TextureCache::FindStorageTexture(CommandBuffer* com
}
RetireStoragePageNeighbors(ctx, info);
uint64_t readable = 0;
if (!HostMemoryQueryReadable(info.address, info.size, &readable) || readable < info.size) {
EXIT("TextureCache: new storage image has unreadable guest backing, addr=0x%016" PRIx64
" size=0x%016" PRIx64 " readable=0x%016" PRIx64 "\n",
info.address, info.size, readable);
}
m_images.reserve(m_images.size() + 1);
auto cached = std::make_shared<CachedImage>();
cached->kind = CachedImage::Kind::StorageTexture;
@ -3639,6 +3620,9 @@ bool TextureCache::InvalidateMemory(PageFaultAccess access, uint64_t vaddr, uint
}
if (needs_readback) {
if (GraphicsRunIsCommandProcessorThread()) {
GraphicsRunFinishCommandProcessors();
}
m_readback->Request(access, vaddr, size);
action = m_memory_tracker.BeginCpuFault(vaddr, size, access);
if (action != CpuFaultAction::Download) {

10
src/graphics/shader/recompiler/SrtWalker.cpp
View File

@ -1,9 +1,9 @@
#include "graphics/shader/recompiler/SrtWalker.h"
#include "graphics/host_gpu/hostMemory.h"
#include "graphics/shader/recompiler/ScalarProvenance.h"
#include <algorithm>
#include <cstring>
#include <fmt/format.h>
namespace Libs::Graphics::ShaderRecompiler::IR {
@ -485,12 +485,14 @@ private:
value.pc, base_aligned, relative));
}
}
const auto read_memory =
m_runtime.read_memory != nullptr ? m_runtime.read_memory : HostMemoryReadDword;
if (!read_memory(m_runtime.userdata, address, result)) {
if (m_runtime.read_memory != nullptr &&
!m_runtime.read_memory(m_runtime.userdata, address, result)) {
return Fail(
error, fmt::format("ReadConst pc=0x{:08x} failed at 0x{:016x}", value.pc, address));
}
if (m_runtime.read_memory == nullptr) {
std::memcpy(result, reinterpret_cast<const void*>(address), sizeof(*result));
}
return true;
}

26
src/graphics/shader/shader.cpp
View File

@ -12,7 +12,6 @@
#include "graphics/guest_gpu/graphicsRun.h"
#include "graphics/guest_gpu/hardwareContext.h"
#include "graphics/guest_gpu/gpu_defs.h"
#include "graphics/host_gpu/hostMemory.h"
#include "graphics/shader/recompiler/ShaderDecoder.h"
#include "graphics/shader/recompiler/ShaderRecompiler.h"
#include "graphics/shader/shaderVertexMetadata.h"
@ -785,31 +784,6 @@ static bool ShaderGetStaticInputInfoVS(const HW::VertexShaderInfo* regs,
shader_addr);
return false;
}
uint32_t max_semantic = 0;
for (uint32_t i = 0; i < metadata.input_semantics_count; i++) {
max_semantic =
std::max(max_semantic, static_cast<uint32_t>(metadata.input_semantics[i].semantic));
}
if (!HostMemoryRangeIsReadable(reinterpret_cast<uint64_t>(attrib),
static_cast<uint64_t>(max_semantic + 1) *
sizeof(uint32_t))) {
LOGF("ShaderGetInputInfoVS(): unreadable vertex attribute table shader=0x%016" PRIx64
"\n",
shader_addr);
return false;
}
uint32_t max_buffer = 0;
for (uint32_t i = 0; i < metadata.input_semantics_count; i++) {
max_buffer = std::max(max_buffer, attrib[metadata.input_semantics[i].semantic] & 0x1fu);
}
if (!HostMemoryRangeIsReadable(reinterpret_cast<uint64_t>(buffer),
static_cast<uint64_t>(max_buffer + 1) * 4 *
sizeof(uint32_t))) {
LOGF("ShaderGetInputInfoVS(): unreadable vertex buffer table shader=0x%016" PRIx64 "\n",
shader_addr);
return false;
}
ShaderApplyAttribSemantics(info, metadata.input_semantics.data(),
metadata.input_semantics_count, attrib, buffer);
ShaderDetectBuffers(info);

18
src/graphics/shader/shaderVertexMetadata.cpp
View File

@ -1,7 +1,5 @@
#include "graphics/shader/shaderVertexMetadata.h"
#include "graphics/host_gpu/hostMemory.h"
#include <array>
#include <cstring>
@ -16,10 +14,6 @@ bool Fail(std::string* error, const char* message) {
return false;
}
bool Readable(const void* data, uint64_t size) {
return data != nullptr && HostMemoryRangeIsReadable(reinterpret_cast<uint64_t>(data), size);
}
} // namespace
bool ShaderReadVertexMetadata(const ShaderMappedData& data, uint32_t max_user_sgprs,
@ -27,8 +21,8 @@ bool ShaderReadVertexMetadata(const ShaderMappedData& data, uint32_t max_user_sg
if (metadata == nullptr) {
return Fail(error, "missing vertex metadata output");
}
if (!Readable(data.user_data, sizeof(ShaderUserData))) {
return Fail(error, "unreadable AGC user-data header");
if (data.user_data == nullptr) {
return Fail(error, "missing AGC user-data header");
}
ShaderUserData user_data {};
@ -44,8 +38,8 @@ bool ShaderReadVertexMetadata(const ShaderMappedData& data, uint32_t max_user_sg
const auto direct_size =
static_cast<uint64_t>(user_data.direct_resource_count) * sizeof(uint16_t);
if (direct_size != 0) {
if (!Readable(user_data.direct_resource_offset, direct_size)) {
return Fail(error, "unreadable AGC direct-resource offsets");
if (user_data.direct_resource_offset == nullptr) {
return Fail(error, "missing AGC direct-resource offsets");
}
std::memcpy(direct_offsets.data(), user_data.direct_resource_offset, direct_size);
}
@ -84,8 +78,8 @@ bool ShaderReadVertexMetadata(const ShaderMappedData& data, uint32_t max_user_sg
const auto semantic_size =
static_cast<uint64_t>(data.num_input_semantics) * sizeof(ShaderSemantic);
if (!Readable(data.input_semantics, semantic_size)) {
return Fail(error, "unreadable vertex input semantics");
if (data.input_semantics == nullptr) {
return Fail(error, "missing vertex input semantics");
}
std::memcpy(next.input_semantics.data(), data.input_semantics, semantic_size);
next.input_semantics_count = data.num_input_semantics;

29
src/libs/agc.cpp
View File

@ -11,7 +11,6 @@
#include "graphics/guest_gpu/pm4.h"
#include "graphics/guest_gpu/gpu_defs.h"
#include "graphics/guest_gpu/tile.h"
#include "graphics/host_gpu/hostMemory.h"
#include "graphics/host_gpu/objects/label.h"
#include "graphics/host_gpu/renderer/render.h"
#include "graphics/presentation/renderDoc.h"
@ -373,11 +372,6 @@ struct Label {
uint64_t m_reserved[3];
};
static bool is_readable_command_packet_range(const uint32_t* cmd, uint32_t num_dw) {
return HostMemoryRangeIsReadable(reinterpret_cast<uintptr_t>(cmd),
static_cast<uint64_t>(num_dw) * sizeof(uint32_t));
}
int KYTY_SYSV_ABI GraphicsInit(uint32_t* state, uint32_t ver) {
PRINT_NAME();
@ -3750,8 +3744,7 @@ static void flush_pending_graphics_segment_before_acb(const uint32_t* acb,
if (!wait_addresses.empty() && Gen5::g_pending_graphics_segment.start != nullptr) {
auto* scan = Gen5::g_pending_graphics_segment.start;
auto* matched_end = Gen5::g_pending_graphics_segment.start;
while (scan < Gen5::g_pending_graphics_segment.end &&
Gen5::is_readable_command_packet_range(scan, 1)) {
while (scan < Gen5::g_pending_graphics_segment.end) {
auto cmd_id = *scan;
if (cmd_id == 0x80000000u) {
scan++;
@ -3763,8 +3756,7 @@ static void flush_pending_graphics_segment_before_acb(const uint32_t* acb,
auto len = KYTY_PM4_LEN(cmd_id);
if (len == 0 ||
len > static_cast<uint32_t>(Gen5::g_pending_graphics_segment.end - scan) ||
!Gen5::is_readable_command_packet_range(scan, len)) {
len > static_cast<uint32_t>(Gen5::g_pending_graphics_segment.end - scan)) {
break;
}
@ -3789,8 +3781,7 @@ static void flush_pending_graphics_segment_before_acb(const uint32_t* acb,
Gen5::g_pending_graphics_segment.end > Gen5::g_pending_graphics_segment.start) {
auto* scan = Gen5::g_pending_graphics_segment.start;
auto* valid_end = Gen5::g_pending_graphics_segment.start;
while (scan < Gen5::g_pending_graphics_segment.end &&
Gen5::is_readable_command_packet_range(scan, 1)) {
while (scan < Gen5::g_pending_graphics_segment.end) {
auto cmd_id = *scan;
if (cmd_id == 0x80000000u) {
scan++;
@ -3803,8 +3794,7 @@ static void flush_pending_graphics_segment_before_acb(const uint32_t* acb,
auto len = KYTY_PM4_LEN(cmd_id);
if (len == 0 ||
len > static_cast<uint32_t>(Gen5::g_pending_graphics_segment.end - scan) ||
!Gen5::is_readable_command_packet_range(scan, len)) {
len > static_cast<uint32_t>(Gen5::g_pending_graphics_segment.end - scan)) {
break;
}
@ -3931,16 +3921,14 @@ int KYTY_SYSV_ABI GraphicsDriverSubmitMultiCommandBuffers(uint32_t queue, uint32
}
static void submit_acb(uint32_t queue, uint32_t* acb, uint32_t size_in_dwords) {
if (acb != nullptr && size_in_dwords >= 5 && Gen5::is_readable_command_packet_range(acb, 5)) {
if (acb != nullptr && size_in_dwords >= 5) {
auto descriptor_addr =
static_cast<uint64_t>(acb[0]) | (static_cast<uint64_t>(acb[1]) << 32u);
auto descriptor_size = acb[2];
auto descriptor_flags = acb[3];
auto descriptor_magic = acb[4];
if (descriptor_addr != 0 && descriptor_size != 0 && descriptor_flags == 0 &&
descriptor_magic == 0x5533ccaau &&
Gen5::is_readable_command_packet_range(reinterpret_cast<uint32_t*>(descriptor_addr),
descriptor_size)) {
descriptor_magic == 0x5533ccaau) {
LOGF("\t descriptor addr = 0x%016" PRIx64 "\n"
"\t descriptor size = 0x%08" PRIx32 "\n"
"\t descriptor magic = 0x%08" PRIx32 "\n",
@ -3976,11 +3964,6 @@ int KYTY_SYSV_ABI GraphicsDriverSubmitAcb(uint32_t queue, const Packet* packet)
if (packet == nullptr) {
return OK;
}
if (!Gen5::is_readable_command_packet_range(reinterpret_cast<const uint32_t*>(packet),
sizeof(Packet) / sizeof(uint32_t))) {
return OK;
}
LOGF("\t acb = 0x%016" PRIx64 "\n"
"\t size = 0x%08" PRIx32 "\n"
"\t flags = 0x%02" PRIx8 "\n",

11
tests/ShaderRecompilerComputeTests.cpp
View File

@ -9025,14 +9025,6 @@ void CheckBufferCacheRangeMerge() {
3) &&
!CanMergeBufferCacheQueueMask(0, 64),
"cross-queue or invalid queue ownership was accepted");
Require("BufferCacheRangeMerge", "readback queue ownership",
!CanReadbackBufferCacheQueueMask(0, 3) &&
CanReadbackBufferCacheQueueMask(uint64_t{1} << 3u, 3) &&
!CanReadbackBufferCacheQueueMask((uint64_t{1} << 2u) |
(uint64_t{1} << 3u),
3) &&
!CanReadbackBufferCacheQueueMask(uint64_t{1} << 3u, 64),
"zero, cross-queue, or invalid readback ownership was accepted");
std::printf("[host] %-32s ok\n", "BufferCacheRangeMerge");
}
@ -9855,10 +9847,13 @@ void CheckStorageTextureGpuOwnedRebindState() {
tracker.ForEachUploadRange(base, size, true,
[](uint64_t, uint64_t) noexcept {}, []() noexcept {});
uint64_t readable = 0;
uint64_t mapped = 0;
MEMORY_BASIC_INFORMATION protection{};
Require("StorageTextureGpuOwnedRebind", "owned",
tracker.IsRegionGpuModified(base, size) && page_manager.IsMapped(base, size) &&
(!HostMemoryQueryReadable(base, size, &readable) || readable < size) &&
HostMemoryQueryRange(base, size, HostMemoryAccess::Mapped, &mapped) &&
mapped == size &&
VirtualQuery(memory, &protection, sizeof(protection)) != 0 &&
protection.Protect == PAGE_NOACCESS,
"GPU-owned storage pages remained host-readable or lost tracker identity");

23
tests/ShaderStageRuntimeTests.cpp
View File

@ -3,7 +3,6 @@
#include "graphics/shader/recompiler/ResourceMaterialization.h"
#include "graphics/shader/recompiler/ShaderIR.h"
#include <array>
#include <cstdio>
#include <cstdlib>
#include <memory>
@ -53,22 +52,7 @@ std::shared_ptr<const Libs::Graphics::ShaderRecompiler::IR::Program> UnbasedFlat
return std::make_shared<const Program>(std::move(program));
}
void TestFailurePreservesPriorStage() {
using namespace Libs::Graphics;
auto cached_program = SrtProgram(0x1001);
auto prior_program = std::make_shared<const ShaderRecompiler::IR::Program>();
auto prior_resources = std::make_shared<const ShaderRecompiler::IR::ResourceSnapshot>();
ShaderStageRuntime stage {prior_program, prior_resources};
const std::array<uint32_t, 1> user_data = {};
std::string error;
Check(!ShaderMaterializeStageRuntime(cached_program, user_data, 0, &stage, &error) &&
error.find("failed at 0x0000000000001000") != std::string::npos,
"unmapped cache ReadConst did not fail through the checked production reader");
Check(stage.program == prior_program && stage.resources == prior_resources,
"failed cache rematerialization replaced the prior stage snapshot");
}
void TestMappedSrtUsesCheckedReaderByDefault() {
void TestMappedSrtUsesDirectReaderByDefault() {
using namespace Libs::Graphics;
const uint32_t dword = 0x12345678;
auto cached_program = SrtProgram(reinterpret_cast<uint64_t>(&dword));
@ -79,7 +63,7 @@ void TestMappedSrtUsesCheckedReaderByDefault() {
"cache rematerialization did not publish the mapped stage");
Check(stage.resources->flattened_srt.size() == 1 &&
stage.resources->flattened_srt[0] == dword,
"cache rematerialization did not use the checked reader by default");
"cache rematerialization did not use the direct reader by default");
}
void TestUnbasedFlatCacheHitFailsClosed() {
@ -99,8 +83,7 @@ void TestUnbasedFlatCacheHitFailsClosed() {
} // namespace
int main() {
TestFailurePreservesPriorStage();
TestMappedSrtUsesCheckedReaderByDefault();
TestMappedSrtUsesDirectReaderByDefault();
TestUnbasedFlatCacheHitFailsClosed();
std::puts("ShaderStageRuntimeTests: all cases passed");
return 0;

83
tests/ShaderVertexMetadataTests.cpp
View File

@ -1,23 +1,10 @@
#include "graphics/shader/shaderVertexMetadata.h"
#include <algorithm>
#include <array>
#include <cstdint>
#include <cstdio>
#include <cstdlib>
#if KYTY_PLATFORM == KYTY_PLATFORM_WINDOWS
#ifndef NOMINMAX
#define NOMINMAX
#endif
#include <windows.h>
#undef min
#undef max
#elif KYTY_PLATFORM == KYTY_PLATFORM_LINUX
#include <sys/mman.h>
#include <unistd.h>
#endif
namespace {
using namespace Libs::Graphics;
@ -60,7 +47,7 @@ void CheckRejected(const ShaderMappedData& data, const char* text) {
"metadata rejection changed the prior output");
}
void TestValidAndHostileMetadata() {
void TestValidAndInvalidMetadata() {
Fixture fixture;
ShaderVertexMetadata output;
std::string error;
@ -70,9 +57,13 @@ void TestValidAndHostileMetadata() {
output.input_semantics_count == 1,
"valid AGC vertex metadata was decoded incorrectly");
auto unreadable = fixture.mapped;
unreadable.user_data = reinterpret_cast<ShaderUserData*>(uintptr_t {1});
CheckRejected(unreadable, "unreadable AGC user-data header was accepted");
auto missing_header = fixture.mapped;
missing_header.user_data = nullptr;
CheckRejected(missing_header, "missing AGC user-data header was accepted");
Fixture missing_offsets;
missing_offsets.user_data.direct_resource_offset = nullptr;
CheckRejected(missing_offsets.mapped, "missing direct-resource offsets were accepted");
Fixture excessive_resources;
excessive_resources.user_data.direct_resource_count =
@ -88,65 +79,15 @@ void TestValidAndHostileMetadata() {
static_cast<size_t>(AgcDirectResourceType::PtrVertexBufferTable)] = 63;
CheckRejected(excessive_register.mapped, "out-of-domain vertex table SGPR was accepted");
Fixture bad_semantics;
bad_semantics.mapped.input_semantics =
reinterpret_cast<ShaderSemantic*>(uintptr_t {1});
CheckRejected(bad_semantics.mapped, "unreadable vertex semantic array was accepted");
}
void TestCrossPageMetadata() {
#if KYTY_PLATFORM == KYTY_PLATFORM_WINDOWS
SYSTEM_INFO system_info {};
GetSystemInfo(&system_info);
const auto page_size = static_cast<uint64_t>(system_info.dwPageSize);
auto* pages = static_cast<uint8_t*>(
VirtualAlloc(nullptr, page_size * 2, MEM_RESERVE | MEM_COMMIT, PAGE_READWRITE));
Check(pages != nullptr, "failed to allocate metadata test pages");
DWORD old_protect = 0;
Check(VirtualProtect(pages + page_size, page_size, PAGE_NOACCESS, &old_protect) != 0,
"failed to protect metadata test page");
#elif KYTY_PLATFORM == KYTY_PLATFORM_LINUX
const auto native_page_size = sysconf(_SC_PAGESIZE);
Check(native_page_size > 0, "failed to query page size");
const auto page_size = static_cast<uint64_t>(native_page_size);
auto* pages = static_cast<uint8_t*>(
mmap(nullptr, page_size * 2, PROT_READ | PROT_WRITE, MAP_PRIVATE | MAP_ANONYMOUS, -1, 0));
Check(pages != MAP_FAILED, "failed to allocate metadata test pages");
Check(mprotect(pages + page_size, page_size, PROT_NONE) == 0,
"failed to protect metadata test page");
#else
return;
#endif
Fixture fixture;
auto header_crossing = fixture.mapped;
header_crossing.user_data = reinterpret_cast<ShaderUserData*>(
pages + page_size - sizeof(ShaderUserData) + 1);
CheckRejected(header_crossing, "cross-page AGC user-data header was accepted");
auto offsets_crossing = fixture.mapped;
fixture.user_data.direct_resource_offset =
reinterpret_cast<uint16_t*>(pages + page_size - sizeof(uint16_t));
CheckRejected(offsets_crossing, "cross-page direct-resource offsets were accepted");
fixture.user_data.direct_resource_offset = fixture.offsets.data();
auto semantics_crossing = fixture.mapped;
semantics_crossing.input_semantics =
reinterpret_cast<ShaderSemantic*>(pages + page_size - sizeof(uint16_t));
CheckRejected(semantics_crossing, "cross-page vertex semantics were accepted");
#if KYTY_PLATFORM == KYTY_PLATFORM_WINDOWS
Check(VirtualFree(pages, 0, MEM_RELEASE) != 0, "failed to free metadata test pages");
#elif KYTY_PLATFORM == KYTY_PLATFORM_LINUX
Check(munmap(pages, page_size * 2) == 0, "failed to free metadata test pages");
#endif
Fixture missing_semantics;
missing_semantics.mapped.input_semantics = nullptr;
CheckRejected(missing_semantics.mapped, "missing vertex semantic array was accepted");
}
} // namespace
int main() {
TestValidAndHostileMetadata();
TestCrossPageMetadata();
TestValidAndInvalidMetadata();
std::puts("ShaderVertexMetadataTests: all cases passed");
return 0;
}