raphiaura/KytyPS5
1
0
Fork 0
mirror of https://github.com/KytyPS5/KytyPS5.git synced 2026-10-26 21:06:53 +01:00
Code Issues Packages Projects Releases Wiki Activity

remove readback hacks, allows to read/write guest memory directly

Browse Source
This commit is contained in:
nmzik
2026-07-17 05:16:00 +02:00
parent 013e9a4485
commit e903c93848
27 changed files with 522 additions and 1228 deletions
Download Patch File Download Diff File Expand all files Collapse all files
src/graphics/guest_gpu/command_processor/commandProcessor.h
+39 -24
View File
@@ -12,8 +12,6 @@
namespace Libs::Graphics { namespace Libs::Graphics {
class CommandProcessorGuestAccessScope;
inline constexpr uint32_t AcquireGcrGl2Writeback = 1u << 15u; inline constexpr uint32_t AcquireGcrGl2Writeback = 1u << 15u;
bool TestWaitRegMemValue(uint64_t value, uint64_t ref, uint64_t mask, uint32_t func); bool TestWaitRegMemValue(uint64_t value, uint64_t ref, uint64_t mask, uint32_t func);
@@ -72,6 +70,21 @@ public:
} }
} }
void SubmitForReadback() {
if (!Active()) {
return;
}
SubmitCurrent();
}
void ResumeAfterReadback() {
if (!Active()) {
return;
}
Current()->WaitForFenceAndReset();
Current()->Begin();
}
private: private:
CommandBuffer* SubmitCurrent() { CommandBuffer* SubmitCurrent() {
auto* submitted = Current(); auto* submitted = Current();
@@ -107,22 +120,26 @@ public:
void Reset(); void Reset();
void BufferInit(); void BufferInit();
void BufferFlush(); void BufferFlush();
void BufferFlushAndWait(); void BufferFlushAndWait();
void BufferWait(); void BufferWait();
[[nodiscard]] int BeginReadbackTransaction(bool mutex_preowned) { void BeginReadbackTransaction() {
if (!mutex_preowned) { m_mutex.Lock();
m_mutex.Lock(); if (m_readback_active) {
EXIT("nested command-processor readback transaction\n");
} }
m_scheduler.Flush(); m_readback_active = true;
m_scheduler.WaitAll(); m_readback_finished = false;
return m_scheduler.Queue();
} }
void EndReadbackTransaction(bool mutex_preowned) { void FinishReadbackTransaction();
if (!mutex_preowned) { void EndReadbackTransaction() {
m_mutex.Unlock(); if (!m_readback_active) {
EXIT("command-processor readback transaction is not active\n");
} }
m_readback_active = false;
m_readback_finished = false;
m_mutex.Unlock();
} }
void RunLock() { m_run_mutex.Lock(); } void RunLock() { m_run_mutex.Lock(); }
@@ -164,7 +181,6 @@ public:
uint32_t value); uint32_t value);
void PrepareCpuFlip(); void PrepareCpuFlip();
void SynchronizeGpu(); void SynchronizeGpu();
void SynchronizeGpuRange(uint64_t vaddr, uint64_t size);
void MemoryBarrier(); void MemoryBarrier();
void TriggerEopEventAtEndOfPipe(uint32_t interrupt_context_id); void TriggerEopEventAtEndOfPipe(uint32_t interrupt_context_id);
void RenderTextureBarrier(uint64_t vaddr, uint64_t size); void RenderTextureBarrier(uint64_t vaddr, uint64_t size);
@@ -182,9 +198,6 @@ public:
void PopMarker() {} void PopMarker() {}
void PrefetchL2(void* addr, uint32_t size) {} void PrefetchL2(void* addr, uint32_t size) {}
void ClearGds(uint64_t dw_offset, uint32_t dw_num, uint32_t clear_value);
void ReadGds(uint32_t* dst, uint32_t dw_offset, uint32_t dw_size);
void ResetDeCe(); void ResetDeCe();
void WaitCe(); void WaitCe();
void WaitDeDiff(uint32_t diff); void WaitDeDiff(uint32_t diff);
@@ -208,7 +221,7 @@ public:
void Run(uint32_t* data, uint32_t num_dw); void Run(uint32_t* data, uint32_t num_dw);
void SetQueue(int queue) { m_scheduler.SetQueue(queue); } void SetQueue(int queue);
[[nodiscard]] int GetQueue() const { return m_scheduler.Queue(); } [[nodiscard]] int GetQueue() const { return m_scheduler.Queue(); }
[[nodiscard]] const FlipInfo& GetFlip() const { return m_flip; } [[nodiscard]] const FlipInfo& GetFlip() const { return m_flip; }
@@ -218,8 +231,6 @@ public:
void SetSubmitId(uint64_t submit_id) { m_submit_id = submit_id; } void SetSubmitId(uint64_t submit_id) { m_submit_id = submit_id; }
private: private:
friend class CommandProcessorGuestAccessScope;
struct Counter { struct Counter {
Common::Mutex mutex; Common::Mutex mutex;
Common::CondVar cond_var; Common::CondVar cond_var;
@@ -231,6 +242,7 @@ private:
uint32_t cache_action, uint32_t event_index, uint32_t event_write_source, uint32_t cache_action, uint32_t event_index, uint32_t event_write_source,
void* dst_gpu_addr, T value, uint32_t interrupt_selector, void* dst_gpu_addr, T value, uint32_t interrupt_selector,
uint32_t interrupt_context_id); uint32_t interrupt_context_id);
void FinishCommandProcessors();
CommandBuffer* CurrentBuffer() { return m_scheduler.Current(); } CommandBuffer* CurrentBuffer() { return m_scheduler.Current(); }
void CheckBuffer() const { m_scheduler.CheckActive(); } void CheckBuffer() const { m_scheduler.CheckActive(); }
@@ -250,8 +262,11 @@ private:
uint64_t m_dispatch_indirect_args_base_addr = 0; uint64_t m_dispatch_indirect_args_base_addr = 0;
uint32_t m_num_instances = 1; uint32_t m_num_instances = 1;
Common::Mutex m_mutex; inline static Common::Mutex m_mutex;
Common::Mutex m_run_mutex; inline static std::array<CommandProcessor*, GraphicContext::QUEUES_NUM> m_processors {};
inline static bool m_readback_active = false;
inline static bool m_readback_finished = false;
Common::Mutex m_run_mutex;
CommandScheduler m_scheduler; CommandScheduler m_scheduler;
src/graphics/guest_gpu/command_processor/pm4Handlers.cpp
+6 -8
View File
@@ -2233,7 +2233,7 @@ KYTY_CP_OP_PARSER(CpOpCopyData) {
cp->WriteReferenceClock(dst, num_bytes); cp->WriteReferenceClock(dst, num_bytes);
return 5; return 5;
} }
const auto dma_src = CopyDataSrcToDma(src_sel); const auto dma_src = CopyDataSrcToDma(src_sel);
if (dma_src == 2 && num_bytes == 8) { if (dma_src == 2 && num_bytes == 8) {
EXIT("unsupported 64-bit immediate copyData\n"); EXIT("unsupported 64-bit immediate copyData\n");
} }
@@ -2636,7 +2636,6 @@ KYTY_CP_OP_PARSER(CpOpIndirectCxRegs) {
if (indirect_buffer == nullptr) { if (indirect_buffer == nullptr) {
EXIT("indirect CX registers have null address, num_regs = %" PRIu32 "\n", indirect_num_dw); EXIT("indirect CX registers have null address, num_regs = %" PRIu32 "\n", indirect_num_dw);
} }
for (uint32_t i = 0; i < indirect_num_dw; i++, indirect_buffer += 2) { for (uint32_t i = 0; i < indirect_num_dw; i++, indirect_buffer += 2) {
auto cmd_offset = indirect_buffer[0]; auto cmd_offset = indirect_buffer[0];
auto value = indirect_buffer[1]; auto value = indirect_buffer[1];
@@ -2695,13 +2694,14 @@ KYTY_CP_OP_PARSER(CpOpIndirectShRegs) {
if (indirect_buffer == nullptr) { if (indirect_buffer == nullptr) {
EXIT("indirect SH registers have null address, num_regs = %" PRIu32 "\n", indirect_num_dw); EXIT("indirect SH registers have null address, num_regs = %" PRIu32 "\n", indirect_num_dw);
} }
const auto indirect_address = reinterpret_cast<uint64_t>(indirect_buffer);
for (uint32_t i = 0; i < indirect_num_dw; i++, indirect_buffer += 2) { for (uint32_t i = 0; i < indirect_num_dw; i++, indirect_buffer += 2) {
auto raw_cmd_offset = indirect_buffer[0]; auto raw_cmd_offset = indirect_buffer[0];
auto cmd_offset = NormalizeRegisterOffset(raw_cmd_offset); auto cmd_offset = NormalizeRegisterOffset(raw_cmd_offset);
auto value = indirect_buffer[1]; auto value = indirect_buffer[1];
//Not sure if this is correct // Not sure if this is correct
if (raw_cmd_offset != cmd_offset) { if (raw_cmd_offset != cmd_offset) {
LOGF_COLOR(Log::Color::Red, LOGF_COLOR(Log::Color::Red,
"\t temporary: normalized indirect SH register offset 0x%08" PRIx32 "\t temporary: normalized indirect SH register offset 0x%08" PRIx32
@@ -2709,7 +2709,7 @@ KYTY_CP_OP_PARSER(CpOpIndirectShRegs) {
raw_cmd_offset, cmd_offset); raw_cmd_offset, cmd_offset);
} }
//Not sure if this is correct // Not sure if this is correct
if (cmd_offset == Pm4::SH_NOP || raw_cmd_offset == 0xffffffffu) { if (cmd_offset == Pm4::SH_NOP || raw_cmd_offset == 0xffffffffu) {
continue; continue;
} }
@@ -2725,8 +2725,7 @@ KYTY_CP_OP_PARSER(CpOpIndirectShRegs) {
if (pfunc == nullptr) { if (pfunc == nullptr) {
LOGF("unknown indirect SH register: index=%" PRIu32 "/%" PRIu32 ", regs=0x%016" PRIx64 LOGF("unknown indirect SH register: index=%" PRIu32 "/%" PRIu32 ", regs=0x%016" PRIx64
", offset=0x%08" PRIx32 ", value=0x%08" PRIx32 "\n", ", offset=0x%08" PRIx32 ", value=0x%08" PRIx32 "\n",
i, indirect_num_dw, reinterpret_cast<uint64_t>(indirect_buffer - i * 2), i, indirect_num_dw, indirect_address, cmd_offset, value);
cmd_offset, value);
auto* dump_regs = indirect_buffer - i * 2; auto* dump_regs = indirect_buffer - i * 2;
for (uint32_t j = 0; j < indirect_num_dw && j < 16; j++) { for (uint32_t j = 0; j < indirect_num_dw && j < 16; j++) {
LOGF("\t sh_indirect[%" PRIu32 "] offset=0x%08" PRIx32 ", value=0x%08" PRIx32 "\n", LOGF("\t sh_indirect[%" PRIu32 "] offset=0x%08" PRIx32 ", value=0x%08" PRIx32 "\n",
@@ -2758,13 +2757,12 @@ KYTY_CP_OP_PARSER(CpOpIndirectUcRegs) {
if (indirect_buffer == nullptr) { if (indirect_buffer == nullptr) {
EXIT("indirect UC registers have null address, num_regs = %" PRIu32 "\n", indirect_num_dw); EXIT("indirect UC registers have null address, num_regs = %" PRIu32 "\n", indirect_num_dw);
} }
for (uint32_t i = 0; i < indirect_num_dw; i++, indirect_buffer += 2) { for (uint32_t i = 0; i < indirect_num_dw; i++, indirect_buffer += 2) {
auto raw_cmd_offset = indirect_buffer[0]; auto raw_cmd_offset = indirect_buffer[0];
auto cmd_offset = NormalizeRegisterOffset(raw_cmd_offset); auto cmd_offset = NormalizeRegisterOffset(raw_cmd_offset);
auto value = indirect_buffer[1]; auto value = indirect_buffer[1];
//Not sure if this is correct // Not sure if this is correct
if (raw_cmd_offset != cmd_offset) { if (raw_cmd_offset != cmd_offset) {
LOGF_COLOR(Log::Color::Red, LOGF_COLOR(Log::Color::Red,
"\t temporary: normalized indirect UC register offset 0x%08" PRIx32 "\t temporary: normalized indirect UC register offset 0x%08" PRIx32
src/graphics/guest_gpu/graphicsRun.cpp
+118 -196
View File
@@ -33,19 +33,9 @@
namespace Libs::Graphics { namespace Libs::Graphics {
enum class CommandProcessorGuestAccess : uint32_t { static thread_local CommandProcessor* g_current_run_cp = nullptr;
None, static thread_local uint32_t g_submission_pause_depth = 0;
WaitPoll, static thread_local bool g_gpu_mutex_owned = false;
DirectFence,
};
static thread_local CommandProcessor* g_current_run_cp = nullptr;
static thread_local uint32_t g_submission_pause_depth = 0;
static thread_local bool g_gpu_mutex_owned = false;
static thread_local bool g_cp_readback_owned = false;
static thread_local bool g_cp_readback_preowned = false;
static thread_local CommandProcessorGuestAccess g_guest_memory_access =
CommandProcessorGuestAccess::None;
class GpuMutexLock final { class GpuMutexLock final {
public: public:
@@ -68,54 +58,13 @@ private:
Common::Mutex& m_mutex; Common::Mutex& m_mutex;
}; };
class CommandProcessorGuestAccessScope final {
public:
CommandProcessorGuestAccessScope(CommandProcessorGuestAccess access, uint64_t vaddr,
uint64_t size) {
if (access == CommandProcessorGuestAccess::None ||
g_guest_memory_access != CommandProcessorGuestAccess::None ||
g_current_run_cp == nullptr) {
EXIT("invalid nested command-processor guest-memory access\n");
}
if (g_render_ctx == nullptr) {
EXIT("command-processor guest-memory access has no render context\n");
}
m_cp = g_current_run_cp;
if (access == CommandProcessorGuestAccess::WaitPoll) {
// Preserve serialized scheduler semantics: a poll owns the command processor before it
// enters the shared buffer/texture transaction, and releases both before yielding.
m_cp->m_mutex.Lock();
m_cp_mutex_owned = true;
}
m_resource_lock =
g_render_ctx->GetGpuResources()->LockCommandProcessorGuestAccess(vaddr, size);
g_guest_memory_access = access;
}
~CommandProcessorGuestAccessScope() {
if (g_guest_memory_access == CommandProcessorGuestAccess::None ||
!m_resource_lock.owns_lock() || m_cp == nullptr) {
EXIT("command-processor guest-memory access scope is not active\n");
}
g_guest_memory_access = CommandProcessorGuestAccess::None;
m_resource_lock.unlock();
if (m_cp_mutex_owned) {
m_cp->m_mutex.Unlock();
}
}
private:
CommandProcessor* m_cp = nullptr;
bool m_cp_mutex_owned = false;
std::unique_lock<ResourceMutex> m_resource_lock;
KYTY_CLASS_NO_COPY(CommandProcessorGuestAccessScope);
};
struct OwnedCmdBuffer { struct OwnedCmdBuffer {
std::vector<uint32_t> storage; std::vector<uint32_t> storage;
uint32_t* data = nullptr; uint32_t* data = nullptr;
uint32_t num_dw = 0; uint32_t num_dw = 0;
OwnedCmdBuffer() = default; OwnedCmdBuffer() = default;
OwnedCmdBuffer(const uint32_t* src, uint32_t count) { Assign(src, count); }
OwnedCmdBuffer(const OwnedCmdBuffer& other): storage(other.storage), num_dw(other.num_dw) { OwnedCmdBuffer(const OwnedCmdBuffer& other): storage(other.storage), num_dw(other.num_dw) {
data = (storage.empty() ? other.data : storage.data()); data = (storage.empty() ? other.data : storage.data());
@@ -165,9 +114,8 @@ public:
KYTY_CLASS_NO_COPY(GraphicsRing); KYTY_CLASS_NO_COPY(GraphicsRing);
void Submit(uint32_t* cmd_draw_buffer, uint32_t num_draw_dw, uint32_t* cmd_const_buffer, void Submit(OwnedCmdBuffer draw_buffer, OwnedCmdBuffer const_buffer, int handle, int index,
uint32_t num_const_dw, int handle, int index, int flip_mode, int64_t flip_arg, int flip_mode, int64_t flip_arg, bool trigger_agc_interrupt_on_done);
bool trigger_agc_interrupt_on_done);
void SubmitFlipPreparation(); void SubmitFlipPreparation();
void Done(); void Done();
void WaitForIdle(); void WaitForIdle();
@@ -217,7 +165,7 @@ public:
KYTY_CLASS_NO_COPY(ComputeRing); KYTY_CLASS_NO_COPY(ComputeRing);
void Submit(uint32_t* cmd_buffer, uint32_t num_dw, bool trigger_agc_interrupt_on_done); void Submit(OwnedCmdBuffer buffer, bool trigger_agc_interrupt_on_done);
void Done(); void Done();
void WaitForIdle(); void WaitForIdle();
bool IsIdle(); bool IsIdle();
@@ -311,15 +259,18 @@ void GraphicsRunInit() {
void Gpu::Submit(uint32_t* cmd_draw_buffer, uint32_t num_draw_dw, uint32_t* cmd_const_buffer, void Gpu::Submit(uint32_t* cmd_draw_buffer, uint32_t num_draw_dw, uint32_t* cmd_const_buffer,
uint32_t num_const_dw, bool trigger_agc_interrupt_on_done) { uint32_t num_const_dw, bool trigger_agc_interrupt_on_done) {
GpuMutexLock lock(m_mutex); OwnedCmdBuffer draw_buffer(cmd_draw_buffer, num_draw_dw);
OwnedCmdBuffer const_buffer(cmd_const_buffer, num_const_dw);
GpuMutexLock lock(m_mutex);
m_gfx_ring->Submit(cmd_draw_buffer, num_draw_dw, cmd_const_buffer, num_const_dw, 0, 0, 0, 0, m_gfx_ring->Submit(std::move(draw_buffer), std::move(const_buffer), 0, 0, 0, 0,
trigger_agc_interrupt_on_done); trigger_agc_interrupt_on_done);
} }
void Gpu::SubmitCompute(uint32_t queue, uint32_t* cmd_buffer, uint32_t num_dw, void Gpu::SubmitCompute(uint32_t queue, uint32_t* cmd_buffer, uint32_t num_dw,
bool trigger_agc_interrupt_on_done) { bool trigger_agc_interrupt_on_done) {
GpuMutexLock lock(m_mutex); OwnedCmdBuffer buffer(cmd_buffer, num_dw);
GpuMutexLock lock(m_mutex);
constexpr uint32_t compute_queue_base = 0x20u; constexpr uint32_t compute_queue_base = 0x20u;
constexpr uint32_t compute_queue_num = 7u * 8u; constexpr uint32_t compute_queue_num = 7u * 8u;
@@ -336,7 +287,7 @@ void Gpu::SubmitCompute(uint32_t queue, uint32_t* cmd_buffer, uint32_t num_dw,
auto* ring = GetRing(ring_id); auto* ring = GetRing(ring_id);
ring->Submit(cmd_buffer, num_dw, trigger_agc_interrupt_on_done); ring->Submit(std::move(buffer), trigger_agc_interrupt_on_done);
} }
void Gpu::SubmitFlipPreparation() { void Gpu::SubmitFlipPreparation() {
@@ -454,6 +405,45 @@ ComputeRing* Gpu::GetRing(uint32_t ring_id) {
return m_compute_ring[v]; return m_compute_ring[v];
} }
void CommandProcessor::SetQueue(int queue) {
Common::LockGuard lock(m_mutex);
if (queue < 0 || queue >= GraphicContext::QUEUES_NUM ||
(m_processors[queue] != nullptr && m_processors[queue] != this)) {
EXIT("invalid command-processor queue registration: queue=%d owner=%p\n", queue,
static_cast<const void*>(m_processors[queue]));
}
m_scheduler.SetQueue(queue);
m_processors[queue] = this;
}
void CommandProcessor::FinishReadbackTransaction() {
if (GraphicsRunCurrentCommandProcessor() == nullptr || !m_readback_active) {
EXIT("GPU readback finish requires a command-processor thread\n");
}
if (m_readback_finished) {
return;
}
FinishCommandProcessors();
m_readback_finished = true;
}
void CommandProcessor::FinishCommandProcessors() {
std::array<CommandProcessor*, GraphicContext::QUEUES_NUM> processors {};
uint32_t processor_count = 0;
for (auto* processor: m_processors) {
if (processor == nullptr ||
std::find(processors.begin(), processors.begin() + processor_count, processor) !=
processors.begin() + processor_count) {
continue;
}
processors[processor_count++] = processor;
processor->m_scheduler.SubmitForReadback();
}
for (uint32_t i = 0; i < processor_count; i++) {
processors[i]->m_scheduler.ResumeAfterReadback();
}
}
void CommandProcessor::Reset() { void CommandProcessor::Reset() {
BufferWait(); BufferWait();
@@ -598,13 +588,6 @@ bool TestWaitRegMemValue(uint64_t value, uint64_t ref, uint64_t mask, uint32_t f
return false; return false;
} }
template <typename T>
static T LoadGuestWaitValue(const T* addr) {
CommandProcessorGuestAccessScope guest_access(CommandProcessorGuestAccess::WaitPoll,
reinterpret_cast<uint64_t>(addr), sizeof(*addr));
return *addr;
}
static void YieldCommandProcessorWait(uint32_t poll_interval_cycles) noexcept { static void YieldCommandProcessorWait(uint32_t poll_interval_cycles) noexcept {
// PS5 specifies GPU poll cycles, not host microseconds. Yield at the scheduler boundary now; // PS5 specifies GPU poll cycles, not host microseconds. Yield at the scheduler boundary now;
// a future resumable guest GPU scheduler can replace this one function without touching polling // a future resumable guest GPU scheduler can replace this one function without touching polling
@@ -625,12 +608,9 @@ void CommandProcessor::WaitRegMem(uint32_t func, const T* addr, T ref, T mask, u
const auto log_width = static_cast<int>(sizeof(T) * 2u); const auto log_width = static_cast<int>(sizeof(T) * 2u);
const auto bits = static_cast<unsigned>(sizeof(T) * 8u); const auto bits = static_cast<unsigned>(sizeof(T) * 8u);
if (!TestWaitRegMemValue(LoadGuestWaitValue(addr), ref, mask, func)) {
SynchronizeGpuRange(addr_value, sizeof(*addr));
}
uint64_t spin_count = 0; uint64_t spin_count = 0;
for (;;) { for (;;) {
const auto value = LoadGuestWaitValue(addr); const auto value = *addr;
if (TestWaitRegMemValue(value, ref, mask, func)) { if (TestWaitRegMemValue(value, ref, mask, func)) {
break; break;
} }
@@ -670,10 +650,7 @@ void CommandProcessor::WriteData(uint32_t* dst, const uint32_t* src, uint32_t dw
return; return;
} }
const auto size = static_cast<uint64_t>(dw_num) * sizeof(uint32_t); memcpy(dst, src, static_cast<size_t>(dw_num) * sizeof(uint32_t));
CommandProcessorGuestAccessScope guest_access(CommandProcessorGuestAccess::DirectFence,
reinterpret_cast<uint64_t>(dst), size);
memcpy(dst, src, static_cast<size_t>(size));
} }
void CommandProcessor::WriteReferenceClock(uint64_t dst_address, uint32_t num_bytes) { void CommandProcessor::WriteReferenceClock(uint64_t dst_address, uint32_t num_bytes) {
@@ -684,11 +661,8 @@ void CommandProcessor::WriteReferenceClock(uint64_t dst_address, uint32_t num_by
num_bytes); num_bytes);
} }
const auto value = GraphicsRenderReadReferenceClock(); const auto value = GraphicsRenderReadReferenceClock();
CommandProcessorGuestAccessScope guest_access(CommandProcessorGuestAccess::DirectFence,
dst_address, num_bytes);
std::memcpy(reinterpret_cast<void*>(dst_address), &value, num_bytes); std::memcpy(reinterpret_cast<void*>(dst_address), &value, num_bytes);
LOGF("\t copy_data reference clock: dst=0x%016" PRIx64 " value=0x%016" PRIx64 LOGF("\t copy_data reference clock: dst=0x%016" PRIx64 " value=0x%016" PRIx64 " size=%u\n",
" size=%u\n",
dst_address, value, num_bytes); dst_address, value, num_bytes);
} }
@@ -729,9 +703,9 @@ void CommandProcessor::DmaData(uint8_t engine, uint8_t dst_sel, uint8_t dst_cach
EXIT("unsupported dmaData source selector 0x%02" PRIx8 "\n", src_sel); EXIT("unsupported dmaData source selector 0x%02" PRIx8 "\n", src_sel);
} }
void GraphicsRing::Submit(uint32_t* cmd_draw_buffer, uint32_t num_draw_dw, void GraphicsRing::Submit(OwnedCmdBuffer draw_buffer, OwnedCmdBuffer const_buffer, int handle,
uint32_t* cmd_const_buffer, uint32_t num_const_dw, int handle, int index, int index, int flip_mode, int64_t flip_arg,
int flip_mode, int64_t flip_arg, bool trigger_agc_interrupt_on_done) { bool trigger_agc_interrupt_on_done) {
EXIT_IF(m_cp == nullptr); EXIT_IF(m_cp == nullptr);
Common::LockGuard lock(m_mutex); Common::LockGuard lock(m_mutex);
@@ -748,9 +722,9 @@ void GraphicsRing::Submit(uint32_t* cmd_draw_buffer, uint32_t num_draw_dw,
m_cp->Reset(); m_cp->Reset();
} }
auto& buf = m_cmd_batches.emplace_back(); auto& buf = m_cmd_batches.emplace_back();
buf.draw_buffer.Assign(cmd_draw_buffer, num_draw_dw); buf.draw_buffer = std::move(draw_buffer);
buf.const_buffer.Assign(cmd_const_buffer, num_const_dw); buf.const_buffer = std::move(const_buffer);
buf.flip.handle = handle; buf.flip.handle = handle;
buf.flip.index = index; buf.flip.index = index;
buf.flip.flip_mode = flip_mode; buf.flip.flip_mode = flip_mode;
@@ -942,12 +916,11 @@ void ComputeRing::ThreadRun(void* data) {
} }
} }
void ComputeRing::Submit(uint32_t* cmd_buffer, uint32_t num_dw, void ComputeRing::Submit(OwnedCmdBuffer buffer, bool trigger_agc_interrupt_on_done) {
bool trigger_agc_interrupt_on_done) {
Common::LockGuard lock(m_mutex); Common::LockGuard lock(m_mutex);
EXIT_IF(cmd_buffer == nullptr); EXIT_IF(buffer.data == nullptr);
EXIT_IF(num_dw == 0); EXIT_IF(buffer.num_dw == 0);
WindowWaitForGraphicInitialized(); WindowWaitForGraphicInitialized();
GraphicsRenderCreateContext(); GraphicsRenderCreateContext();
@@ -957,7 +930,7 @@ void ComputeRing::Submit(uint32_t* cmd_buffer, uint32_t num_dw,
} }
DirectBatch batch {}; DirectBatch batch {};
batch.buffer.Assign(cmd_buffer, num_dw); batch.buffer = std::move(buffer);
batch.trigger_agc_interrupt_on_done = trigger_agc_interrupt_on_done; batch.trigger_agc_interrupt_on_done = trigger_agc_interrupt_on_done;
m_direct_batches.push_back(std::move(batch)); m_direct_batches.push_back(std::move(batch));
@@ -1229,35 +1202,38 @@ void CommandProcessor::DrawIndirect(uint32_t data_offset, uint32_t draw_initiato
EXIT_NOT_IMPLEMENTED((draw_initiator & ~0x20u) != 2u); EXIT_NOT_IMPLEMENTED((draw_initiator & ~0x20u) != 2u);
EXIT_NOT_IMPLEMENTED(m_draw_indirect_args_base_addr == 0); EXIT_NOT_IMPLEMENTED(m_draw_indirect_args_base_addr == 0);
const auto args_addr = m_draw_indirect_args_base_addr + data_offset; const auto* args_addr =
reinterpret_cast<const void*>(m_draw_indirect_args_base_addr + data_offset);
if (!indexed) { if (!indexed) {
auto* args = reinterpret_cast<const DrawIndirectArgs*>(args_addr); DrawIndirectArgs args {};
if (args->instance_count != 1u || args->start_vertex_location != 0u || std::memcpy(&args, args_addr, sizeof(args));
args->start_instance_location != 0u) { if (args.instance_count != 1u || args.start_vertex_location != 0u ||
args.start_instance_location != 0u) {
static std::atomic<uint32_t> log_count {0}; static std::atomic<uint32_t> log_count {0};
if (log_count.fetch_add(1) < 64) { if (log_count.fetch_add(1) < 64) {
LOGF("\t warning: partial DrawIndirect args: vertex_count=%" PRIu32 LOGF("\t warning: partial DrawIndirect args: vertex_count=%" PRIu32
", instance_count=%" PRIu32 ", start_vertex=%" PRIu32 ", instance_count=%" PRIu32 ", start_vertex=%" PRIu32
", start_instance=%" PRIu32 "\n", ", start_instance=%" PRIu32 "\n",
args->vertex_count_per_instance, args->instance_count, args.vertex_count_per_instance, args.instance_count,
args->start_vertex_location, args->start_instance_location); args.start_vertex_location, args.start_instance_location);
} }
} }
DrawIndexAuto(args->vertex_count_per_instance, 0, 0, args->instance_count, DrawIndexAuto(args.vertex_count_per_instance, 0, 0, args.instance_count,
args->start_vertex_location, args->start_instance_location); args.start_vertex_location, args.start_instance_location);
return; return;
} }
auto* args = reinterpret_cast<const DrawIndexedIndirectArgs*>(args_addr); DrawIndexedIndirectArgs args {};
if (args->base_vertex_location != 0u || args->start_instance_location != 0u) { std::memcpy(&args, args_addr, sizeof(args));
if (args.base_vertex_location != 0u || args.start_instance_location != 0u) {
static std::atomic<uint32_t> log_count {0}; static std::atomic<uint32_t> log_count {0};
if (log_count.fetch_add(1) < 64) { if (log_count.fetch_add(1) < 64) {
LOGF("\t warning: partial DrawIndexIndirect args: index_count=%" PRIu32 LOGF("\t warning: partial DrawIndexIndirect args: index_count=%" PRIu32
", instance_count=%" PRIu32 ", start_index=%" PRIu32 ", base_vertex=%" PRIu32 ", instance_count=%" PRIu32 ", start_index=%" PRIu32 ", base_vertex=%" PRIu32
", start_instance=%" PRIu32 "\n", ", start_instance=%" PRIu32 "\n",
args->index_count_per_instance, args->instance_count, args->start_index_location, args.index_count_per_instance, args.instance_count, args.start_index_location,
args->base_vertex_location, args->start_instance_location); args.base_vertex_location, args.start_instance_location);
} }
} }
@@ -1270,22 +1246,22 @@ void CommandProcessor::DrawIndirect(uint32_t data_offset, uint32_t draw_initiato
} }
auto* index_addr = reinterpret_cast<const void*>( auto* index_addr = reinterpret_cast<const void*>(
m_index_base_addr + static_cast<uint64_t>(args->start_index_location) * index_size); m_index_base_addr + static_cast<uint64_t>(args.start_index_location) * index_size);
const uint32_t index_count = const uint32_t index_count =
(m_index_buffer_size != 0 ? std::min(args->index_count_per_instance, m_index_buffer_size) (m_index_buffer_size != 0 ? std::min(args.index_count_per_instance, m_index_buffer_size)
: args->index_count_per_instance); : args.index_count_per_instance);
if (GraphicsRunDebugDumpEnabled() && index_count != args->index_count_per_instance) { if (GraphicsRunDebugDumpEnabled() && index_count != args.index_count_per_instance) {
static std::atomic<uint32_t> log_count {0}; static std::atomic<uint32_t> log_count {0};
if (log_count.fetch_add(1, std::memory_order_relaxed) < 64) { if (log_count.fetch_add(1, std::memory_order_relaxed) < 64) {
LOGF("\t DrawIndexIndirect: clamped index_count from %" PRIu32 " to %" PRIu32 LOGF("\t DrawIndexIndirect: clamped index_count from %" PRIu32 " to %" PRIu32
" using INDEX_BUFFER_SIZE\n", " using INDEX_BUFFER_SIZE\n",
args->index_count_per_instance, index_count); args.index_count_per_instance, index_count);
} }
} }
DrawIndex(index_count, index_addr, 0, 1, args->instance_count, nullptr, 0, DrawIndex(index_count, index_addr, 0, 1, args.instance_count, nullptr, 0,
static_cast<int32_t>(args->base_vertex_location), args->start_instance_location); static_cast<int32_t>(args.base_vertex_location), args.start_instance_location);
} }
void CommandProcessor::DrawIndirectMulti(uint32_t data_offset, uint32_t max_count_or_count, void CommandProcessor::DrawIndirectMulti(uint32_t data_offset, uint32_t max_count_or_count,
@@ -1503,18 +1479,6 @@ void CommandProcessor::DrawIndexAuto(uint32_t index_count, uint32_t flags,
first_instance); first_instance);
} }
void CommandProcessor::ClearGds(uint64_t dw_offset, uint32_t dw_num, uint32_t clear_value) {
Common::LockGuard lock(m_mutex);
GraphicsRenderClearGds(dw_offset, dw_num, clear_value);
}
void CommandProcessor::ReadGds(uint32_t* dst, uint32_t dw_offset, uint32_t dw_size) {
Common::LockGuard lock(m_mutex);
GraphicsRenderReadGds(dst, dw_offset, dw_size);
}
void CommandProcessor::WaitFlipDone(uint32_t video_out_handle, uint32_t display_buffer_index) { void CommandProcessor::WaitFlipDone(uint32_t video_out_handle, uint32_t display_buffer_index) {
BufferFlush(); BufferFlush();
@@ -1569,11 +1533,9 @@ void CommandProcessor::WriteAtEndOfPipe(uint32_t cache_policy, uint32_t event_wr
} }
auto write32 = [&](bool with_writeback) { auto write32 = [&](bool with_writeback) {
auto* dst = static_cast<uint32_t*>(dst_gpu_addr); auto* dst = static_cast<uint32_t*>(dst_gpu_addr);
auto data = static_cast<uint32_t>(value); auto data = static_cast<uint32_t>(value);
CommandProcessorGuestAccessScope guest_access(CommandProcessorGuestAccess::DirectFence, std::memcpy(dst, &data, sizeof(data));
reinterpret_cast<uint64_t>(dst),
sizeof(*dst));
if (with_interrupt) { if (with_interrupt) {
if (with_writeback) { if (with_writeback) {
@@ -1594,8 +1556,10 @@ void CommandProcessor::WriteAtEndOfPipe(uint32_t cache_policy, uint32_t event_wr
case 0x01: case 0x01:
if constexpr (sizeof(T) == sizeof(uint32_t)) { if constexpr (sizeof(T) == sizeof(uint32_t)) {
if (eop_event_type == 0x2f && cache_action == 0x00 && event_index == 0x06) { if (eop_event_type == 0x2f && cache_action == 0x00 && event_index == 0x06) {
GraphicsRenderWriteAtEndOfPipeGds32(m_submit_id, CurrentBuffer(), auto* dst = static_cast<uint32_t*>(dst_gpu_addr);
static_cast<uint32_t*>(dst_gpu_addr), SynchronizeGpu();
GraphicsRenderReadGds(dst, value & 0xffffu, value >> 16u);
GraphicsRenderWriteAtEndOfPipeGds32(m_submit_id, CurrentBuffer(), dst,
value & 0xffffu, value >> 16u); value & 0xffffu, value >> 16u);
return; return;
} }
@@ -1615,10 +1579,8 @@ void CommandProcessor::WriteAtEndOfPipe(uint32_t cache_policy, uint32_t event_wr
} }
} else { } else {
auto write64 = [&](bool with_writeback) { auto write64 = [&](bool with_writeback) {
auto* dst = static_cast<uint64_t*>(dst_gpu_addr); auto* dst = static_cast<uint64_t*>(dst_gpu_addr);
CommandProcessorGuestAccessScope guest_access( std::memcpy(dst, &value, sizeof(value));
CommandProcessorGuestAccess::DirectFence, reinterpret_cast<uint64_t>(dst),
sizeof(*dst));
if (with_interrupt) { if (with_interrupt) {
if (with_writeback) { if (with_writeback) {
@@ -1699,16 +1661,16 @@ void CommandProcessor::WriteAtEndOfPipe(uint32_t cache_policy, uint32_t event_wr
break; break;
case 0x04: case 0x04:
if constexpr (sizeof(T) == sizeof(uint64_t)) { if constexpr (sizeof(T) == sizeof(uint64_t)) {
CommandProcessorGuestAccessScope guest_access( const auto clock = GraphicsRenderReadReferenceClock();
CommandProcessorGuestAccess::DirectFence, std::memcpy(dst_gpu_addr, &clock, sizeof(clock));
reinterpret_cast<uint64_t>(dst_gpu_addr), sizeof(uint64_t));
switch (cache_action) { switch (cache_action) {
case 0x00: case 0x00:
if (((eop_event_type == 0x04 && event_index == 0x05) || if (((eop_event_type == 0x04 && event_index == 0x05) ||
(eop_event_type == 0x28 && event_index == 0x00)) && (eop_event_type == 0x28 && event_index == 0x00)) &&
!with_interrupt) { !with_interrupt) {
GraphicsRenderWriteAtEndOfPipeClockCounter( GraphicsRenderWriteAtEndOfPipeClockCounter(
m_submit_id, CurrentBuffer(), static_cast<uint64_t*>(dst_gpu_addr)); m_submit_id, CurrentBuffer(), static_cast<uint64_t*>(dst_gpu_addr),
clock);
return; return;
} }
break; break;
@@ -1718,7 +1680,8 @@ void CommandProcessor::WriteAtEndOfPipe(uint32_t cache_policy, uint32_t event_wr
(eop_event_type == 0x28 && event_index == 0x00)) && (eop_event_type == 0x28 && event_index == 0x00)) &&
!with_interrupt) { !with_interrupt) {
GraphicsRenderWriteAtEndOfPipeClockCounterWithWriteBack( GraphicsRenderWriteAtEndOfPipeClockCounterWithWriteBack(
m_submit_id, CurrentBuffer(), static_cast<uint64_t*>(dst_gpu_addr)); m_submit_id, CurrentBuffer(), static_cast<uint64_t*>(dst_gpu_addr),
clock);
return; return;
} }
break; break;
@@ -1850,8 +1813,8 @@ void CommandProcessor::Flip() {
} }
auto* command = CurrentBuffer(); auto* command = CurrentBuffer();
auto request = GraphicsRenderPrepareDisplayBufferFlip( auto request = GraphicsRenderPrepareDisplayBufferFlip(command, m_flip.handle, m_flip.index,
command, m_flip.handle, m_flip.index, m_flip.flip_mode, m_flip.flip_arg); m_flip.flip_mode, m_flip.flip_arg);
GraphicsRenderWriteAtEndOfPipeOnlyFlip(m_submit_id, command, m_flip.handle, m_flip.index, GraphicsRenderWriteAtEndOfPipeOnlyFlip(m_submit_id, command, m_flip.handle, m_flip.index,
m_flip.flip_mode, m_flip.flip_arg, request); m_flip.flip_mode, m_flip.flip_arg, request);
m_scheduler.Flush(); m_scheduler.Flush();
@@ -1869,12 +1832,10 @@ void CommandProcessor::Flip(void* dst_gpu_addr, uint32_t value) {
reinterpret_cast<uint64_t>(dst_gpu_addr), value); reinterpret_cast<uint64_t>(dst_gpu_addr), value);
} }
std::memcpy(dst_gpu_addr, &value, sizeof(value));
auto* command = CurrentBuffer(); auto* command = CurrentBuffer();
auto request = GraphicsRenderPrepareDisplayBufferFlip( auto request = GraphicsRenderPrepareDisplayBufferFlip(command, m_flip.handle, m_flip.index,
command, m_flip.handle, m_flip.index, m_flip.flip_mode, m_flip.flip_arg); m_flip.flip_mode, m_flip.flip_arg);
CommandProcessorGuestAccessScope guest_access(CommandProcessorGuestAccess::DirectFence,
reinterpret_cast<uint64_t>(dst_gpu_addr),
sizeof(uint32_t));
GraphicsRenderWriteAtEndOfPipeWithFlip32( GraphicsRenderWriteAtEndOfPipeWithFlip32(
m_submit_id, command, static_cast<uint32_t*>(dst_gpu_addr), value, m_flip.handle, m_submit_id, command, static_cast<uint32_t*>(dst_gpu_addr), value, m_flip.handle,
m_flip.index, m_flip.flip_mode, m_flip.flip_arg, request); m_flip.index, m_flip.flip_mode, m_flip.flip_arg, request);
@@ -1899,12 +1860,10 @@ void CommandProcessor::FlipWithInterrupt(uint32_t eop_event_type, uint32_t cache
if (eop_event_type != 0x00000004 || cache_action != 0x00000038) { if (eop_event_type != 0x00000004 || cache_action != 0x00000038) {
EXIT("unknown event type\n"); EXIT("unknown event type\n");
} }
std::memcpy(dst_gpu_addr, &value, sizeof(value));
auto* command = CurrentBuffer(); auto* command = CurrentBuffer();
auto request = GraphicsRenderPrepareDisplayBufferFlip(command, m_flip.handle, m_flip.index, auto request = GraphicsRenderPrepareDisplayBufferFlip(command, m_flip.handle, m_flip.index,
m_flip.flip_mode, m_flip.flip_arg); m_flip.flip_mode, m_flip.flip_arg);
CommandProcessorGuestAccessScope guest_access(CommandProcessorGuestAccess::DirectFence,
reinterpret_cast<uint64_t>(dst_gpu_addr),
sizeof(uint32_t));
GraphicsRenderWriteAtEndOfPipeWithInterruptWriteBackFlip32( GraphicsRenderWriteAtEndOfPipeWithInterruptWriteBackFlip32(
m_submit_id, command, static_cast<uint32_t*>(dst_gpu_addr), value, m_flip.handle, m_submit_id, command, static_cast<uint32_t*>(dst_gpu_addr), value, m_flip.handle,
m_flip.index, m_flip.flip_mode, m_flip.flip_arg, request); m_flip.index, m_flip.flip_mode, m_flip.flip_arg, request);
@@ -1929,15 +1888,8 @@ void CommandProcessor::PrepareCpuFlip() {
} }
void CommandProcessor::SynchronizeGpu() { void CommandProcessor::SynchronizeGpu() {
BufferFlush(); Common::LockGuard lock(m_mutex);
BufferWait(); FinishCommandProcessors();
}
void CommandProcessor::SynchronizeGpuRange(uint64_t vaddr, uint64_t size) {
if (vaddr == 0 || size == 0 || size > UINT64_MAX - vaddr) {
EXIT("invalid GPU synchronization range\n");
}
SynchronizeGpu();
} }
void GraphicsRunSubmit(uint32_t* cmd_draw_buffer, uint32_t num_draw_dw, uint32_t* cmd_const_buffer, void GraphicsRunSubmit(uint32_t* cmd_draw_buffer, uint32_t num_draw_dw, uint32_t* cmd_const_buffer,
@@ -2002,14 +1954,15 @@ bool GraphicsRunIsCommandProcessorThread() noexcept {
return g_current_run_cp != nullptr; return g_current_run_cp != nullptr;
} }
bool GraphicsRunIsGuestMemoryPoll() noexcept { CommandProcessor* GraphicsRunCurrentCommandProcessor() noexcept {
return g_current_run_cp != nullptr && return g_current_run_cp;
g_guest_memory_access == CommandProcessorGuestAccess::WaitPoll;
} }
bool GraphicsRunIsCommandProcessorGuestAccess() noexcept { void GraphicsRunFinishCommandProcessors() {
return g_current_run_cp != nullptr && if (g_current_run_cp == nullptr) {
g_guest_memory_access != CommandProcessorGuestAccess::None; EXIT("GPU readback finish requires a command-processor thread\n");
}
g_current_run_cp->FinishReadbackTransaction();
} }
bool GraphicsRunSubmissionLockHeld() noexcept { bool GraphicsRunSubmissionLockHeld() noexcept {
@@ -2020,35 +1973,4 @@ bool GraphicsRunGpuLockHeld() noexcept {
return g_gpu_mutex_owned; return g_gpu_mutex_owned;
} }
int GraphicsRunBeginCommandProcessorReadback() {
if (g_current_run_cp == nullptr || g_submission_pause_depth != 0 || g_gpu_mutex_owned ||
g_cp_readback_owned || g_guest_memory_access == CommandProcessorGuestAccess::None) {
EXIT("GPU command-thread readback requires an unpaused command processor\n");
}
const auto queue = g_current_run_cp->GetQueue();
if (queue < 0 || queue >= GraphicContext::QUEUES_NUM) {
EXIT("GPU command-thread readback has invalid queue %d\n", queue);
}
g_cp_readback_owned = true;
// Direct fences enter from an already serialized command handler. Wait polls acquire the
// same processor mutex in CommandProcessorGuestAccessScope before the resource transaction.
g_cp_readback_preowned = true;
const auto synchronized_queue =
g_current_run_cp->BeginReadbackTransaction(g_cp_readback_preowned);
if (synchronized_queue != queue) {
EXIT("GPU command-thread queue changed during readback\n");
}
return synchronized_queue;
}
void GraphicsRunEndCommandProcessorReadback() {
if (g_current_run_cp == nullptr || !g_cp_readback_owned) {
EXIT("GPU command-thread readback ended without ownership\n");
}
const auto preowned = g_cp_readback_preowned;
g_cp_readback_owned = false;
g_cp_readback_preowned = false;
g_current_run_cp->EndReadbackTransaction(preowned);
}
} // namespace Libs::Graphics } // namespace Libs::Graphics
src/graphics/guest_gpu/graphicsRun.h
+7 -11
View File
@@ -6,6 +6,8 @@
namespace Libs::Graphics { namespace Libs::Graphics {
class CommandProcessor;
class GraphicsRunSubmissionLock final { class GraphicsRunSubmissionLock final {
public: public:
GraphicsRunSubmissionLock(); GraphicsRunSubmissionLock();
@@ -23,17 +25,11 @@ void GraphicsRunSubmitFlipPreparation();
void GraphicsRunWait(); void GraphicsRunWait();
void GraphicsRunDone(); void GraphicsRunDone();
int GraphicsRunGetFrameNum(); int GraphicsRunGetFrameNum();
[[nodiscard]] bool GraphicsRunIsCommandProcessorThread() noexcept; [[nodiscard]] bool GraphicsRunIsCommandProcessorThread() noexcept;
[[nodiscard]] bool GraphicsRunIsGuestMemoryPoll() noexcept; [[nodiscard]] CommandProcessor* GraphicsRunCurrentCommandProcessor() noexcept;
[[nodiscard]] bool GraphicsRunIsCommandProcessorGuestAccess() noexcept; void GraphicsRunFinishCommandProcessors();
[[nodiscard]] bool GraphicsRunSubmissionLockHeld() noexcept; [[nodiscard]] bool GraphicsRunSubmissionLockHeld() noexcept;
[[nodiscard]] bool GraphicsRunGpuLockHeld() noexcept; [[nodiscard]] bool GraphicsRunGpuLockHeld() noexcept;
// Begin/end an exclusive same-command-processor readback transaction. The begin call finishes
// work already recorded on that queue and retains the processor lock so its peer draw/constant
// job cannot record new work until BufferCache publishes the faulted page.
[[nodiscard]] int GraphicsRunBeginCommandProcessorReadback();
void GraphicsRunEndCommandProcessorReadback();
} // namespace Libs::Graphics } // namespace Libs::Graphics
#endif /* EMULATOR_INCLUDE_EMULATOR_GRAPHICS_GRAPHICSRUN_H_ */ #endif /* EMULATOR_INCLUDE_EMULATOR_GRAPHICS_GRAPHICSRUN_H_ */
src/graphics/host_gpu/hostMemory.cpp
+5 -36
View File
@@ -2,7 +2,6 @@
#include <cinttypes> #include <cinttypes>
#include <cstdio> #include <cstdio>
#include <cstring>
#if KYTY_PLATFORM == KYTY_PLATFORM_WINDOWS #if KYTY_PLATFORM == KYTY_PLATFORM_WINDOWS
#ifndef NOMINMAX #ifndef NOMINMAX
@@ -21,10 +20,10 @@ bool IsAccessible(DWORD protect, HostMemoryAccess access) {
constexpr DWORD blocked = PAGE_NOACCESS | PAGE_GUARD; constexpr DWORD blocked = PAGE_NOACCESS | PAGE_GUARD;
constexpr DWORD readable = PAGE_READONLY | PAGE_READWRITE | PAGE_WRITECOPY | PAGE_EXECUTE_READ | constexpr DWORD readable = PAGE_READONLY | PAGE_READWRITE | PAGE_WRITECOPY | PAGE_EXECUTE_READ |
PAGE_EXECUTE_READWRITE | PAGE_EXECUTE_WRITECOPY; PAGE_EXECUTE_READWRITE | PAGE_EXECUTE_WRITECOPY;
constexpr DWORD writable = if (access == HostMemoryAccess::Mapped) {
PAGE_READWRITE | PAGE_WRITECOPY | PAGE_EXECUTE_READWRITE | PAGE_EXECUTE_WRITECOPY; return (protect & PAGE_GUARD) == 0;
return (protect & blocked) == 0 && }
(protect & (access == HostMemoryAccess::Write ? writable : readable)) != 0; return (protect & blocked) == 0 && (protect & readable) != 0;
} }
#endif #endif
@@ -77,8 +76,7 @@ bool HostMemoryQueryRange(uint64_t addr, uint64_t requested_size, HostMemoryAcce
if (begin > current) { if (begin > current) {
break; break;
} }
const bool allowed = const bool allowed = access == HostMemoryAccess::Mapped || permissions[0] == 'r';
access == HostMemoryAccess::Write ? permissions[1] == 'w' : permissions[0] == 'r';
if (!allowed) { if (!allowed) {
break; break;
} }
@@ -93,39 +91,10 @@ bool HostMemoryQueryRange(uint64_t addr, uint64_t requested_size, HostMemoryAcce
return *accessible_size != 0; return *accessible_size != 0;
} }
bool HostMemoryQueryEnd(uint64_t addr, HostMemoryAccess access, uint64_t* end) {
if (addr == 0 || end == nullptr) {
return false;
}
uint64_t accessible_size = 0;
if (!HostMemoryQueryRange(addr, UINT64_MAX - addr, access, &accessible_size) ||
accessible_size == 0 || UINT64_MAX - addr < accessible_size) {
return false;
}
*end = addr + accessible_size;
return true;
}
bool HostMemoryQueryReadable(uint64_t addr, uint64_t requested_size, uint64_t* readable_size) { bool HostMemoryQueryReadable(uint64_t addr, uint64_t requested_size, uint64_t* readable_size) {
return HostMemoryQueryRange(addr, requested_size, HostMemoryAccess::Read, readable_size); return HostMemoryQueryRange(addr, requested_size, HostMemoryAccess::Read, readable_size);
} }
bool HostMemoryQueryWritable(uint64_t addr, uint64_t requested_size, uint64_t* writable_size) {
return HostMemoryQueryRange(addr, requested_size, HostMemoryAccess::Write, writable_size);
}
bool HostMemoryReadDword(void*, uint64_t addr, uint32_t* value) {
if (value == nullptr) {
return false;
}
uint64_t readable = 0;
if (!HostMemoryQueryReadable(addr, sizeof(*value), &readable) || readable < sizeof(*value)) {
return false;
}
std::memcpy(value, reinterpret_cast<const void*>(static_cast<uintptr_t>(addr)), sizeof(*value));
return true;
}
bool HostMemoryIsReadable(uint64_t addr) { bool HostMemoryIsReadable(uint64_t addr) {
return HostMemoryRangeIsReadable(addr, 1); return HostMemoryRangeIsReadable(addr, 1);
} }
src/graphics/host_gpu/hostMemory.h
+1 -4
View File
@@ -5,14 +5,11 @@
namespace Libs::Graphics { namespace Libs::Graphics {
enum class HostMemoryAccess { Read, Write }; enum class HostMemoryAccess { Read, Mapped };
bool HostMemoryQueryRange(uint64_t addr, uint64_t requested_size, HostMemoryAccess access, bool HostMemoryQueryRange(uint64_t addr, uint64_t requested_size, HostMemoryAccess access,
uint64_t* accessible_size); uint64_t* accessible_size);
bool HostMemoryQueryEnd(uint64_t addr, HostMemoryAccess access, uint64_t* end);
bool HostMemoryQueryReadable(uint64_t addr, uint64_t requested_size, uint64_t* readable_size); bool HostMemoryQueryReadable(uint64_t addr, uint64_t requested_size, uint64_t* readable_size);
bool HostMemoryQueryWritable(uint64_t addr, uint64_t requested_size, uint64_t* writable_size);
bool HostMemoryReadDword(void* userdata, uint64_t addr, uint32_t* value);
bool HostMemoryIsReadable(uint64_t addr); bool HostMemoryIsReadable(uint64_t addr);
bool HostMemoryRangeIsReadable(uint64_t addr, uint64_t size); bool HostMemoryRangeIsReadable(uint64_t addr, uint64_t size);
src/graphics/host_gpu/objects/label.cpp
+12 -98
View File
@@ -1,14 +1,11 @@
#include "graphics/host_gpu/objects/label.h" #include "graphics/host_gpu/objects/label.h"
#include "common/assert.h" #include "common/assert.h"
#include "common/logging/log.h"
#include "kernel/memory.h"
#include "common/threads.h" #include "common/threads.h"
#include "graphics/host_gpu/graphicContext.h" #include "graphics/host_gpu/graphicContext.h"
#include "graphics/host_gpu/renderer/render.h" #include "graphics/host_gpu/renderer/render.h"
#include <algorithm> #include <algorithm>
#include <atomic>
#include <memory> #include <memory>
#include <vector> #include <vector>
@@ -22,10 +19,6 @@ enum LabelStatus {
}; };
struct LabelCallbacks { struct LabelCallbacks {
uint64_t* dst_gpu_addr64 = nullptr;
uint64_t value64 = 0;
uint32_t* dst_gpu_addr32 = nullptr;
uint32_t value32 = 0;
LabelCallback callback_1 = nullptr; LabelCallback callback_1 = nullptr;
LabelCallback callback_2 = nullptr; LabelCallback callback_2 = nullptr;
uint64_t args[LABEL_ARGS_MAX] = {}; uint64_t args[LABEL_ARGS_MAX] = {};
@@ -64,10 +57,8 @@ public:
~LabelManager() { KYTY_NOT_IMPLEMENTED; } ~LabelManager() { KYTY_NOT_IMPLEMENTED; }
KYTY_CLASS_NO_COPY(LabelManager); KYTY_CLASS_NO_COPY(LabelManager);
Label* Create64(GraphicContext* ctx, uint64_t* dst_gpu_addr, uint64_t value, Label* Create(GraphicContext* ctx, LabelCallback callback_1, LabelCallback callback_2,
LabelCallback callback_1, LabelCallback callback_2, const uint64_t* args); const uint64_t* args);
Label* Create32(GraphicContext* ctx, uint32_t* dst_gpu_addr, uint32_t value,
LabelCallback callback_1, LabelCallback callback_2, const uint64_t* args);
void Delete(Label* label); void Delete(Label* label);
void Set(CommandBuffer* buffer, Label* label); void Set(CommandBuffer* buffer, Label* label);
void Drain(); void Drain();
@@ -75,9 +66,6 @@ public:
private: private:
static void ThreadRun(void* data); static void ThreadRun(void* data);
template <typename T>
Label* Create(GraphicContext* ctx, T* dst_gpu_addr, T value, LabelCallback callback_1,
LabelCallback callback_2, const uint64_t* args);
bool Remove(Label* label); bool Remove(Label* label);
static void Destroy(Label* label); static void Destroy(Label* label);
@@ -90,11 +78,6 @@ private:
static LabelManager* g_label_manager = nullptr; static LabelManager* g_label_manager = nullptr;
static thread_local bool g_in_label_callback = false; static thread_local bool g_in_label_callback = false;
template <typename T>
void WriteGuestLabel(T* address, T value) {
LabelWriteGuestMemory(address, &value, sizeof(value));
}
class LabelCallbackScope final { class LabelCallbackScope final {
public: public:
LabelCallbackScope() { LabelCallbackScope() {
@@ -132,8 +115,7 @@ void LabelManager::ThreadRun(void* data) {
it->completion->event == nullptr) { it->completion->event == nullptr) {
EXIT("GPU label submission has no completion event\n"); EXIT("GPU label submission has no completion event\n");
} }
const auto status = const auto status = vkGetEventStatus(it->completion->device, it->completion->event);
vkGetEventStatus(it->completion->device, it->completion->event);
switch (status) { switch (status) {
case VK_EVENT_SET: case VK_EVENT_SET:
fired_labels.push_back(it->callbacks); fired_labels.push_back(it->callbacks);
@@ -180,35 +162,9 @@ void LabelManager::ThreadRun(void* data) {
for (auto& label: fired_labels) { for (auto& label: fired_labels) {
LabelCallbackScope callback_scope; LabelCallbackScope callback_scope;
bool write = true;
if (label.callback_1 != nullptr) { if (label.callback_1 != nullptr) {
write = label.callback_1(label.args); (void)label.callback_1(label.args);
}
if (write && label.dst_gpu_addr64 != nullptr) {
// Publishing EOP fences through physical backing keeps a fence word coherent even
// when unrelated bytes on its host page are
// protected for BufferCache GPU ownership.
WriteGuestLabel(label.dst_gpu_addr64, label.value64);
static std::atomic<uint32_t> log_count {0};
if (log_count.fetch_add(1) < 256) {
LOGF_COLOR(Log::Color::BrightGreen,
"EndOfPipe Signal!!! [0x%016" PRIx64 "] <- 0x%016" PRIx64 "\n",
reinterpret_cast<uint64_t>(label.dst_gpu_addr64), label.value64);
}
}
if (write && label.dst_gpu_addr32 != nullptr) {
WriteGuestLabel(label.dst_gpu_addr32, label.value32);
static std::atomic<uint32_t> log_count {0};
if (log_count.fetch_add(1) < 256) {
LOGF_COLOR(Log::Color::BrightGreen,
"EndOfPipe Signal!!! [0x%016" PRIx64 "] <- 0x%08" PRIx32 "\n",
reinterpret_cast<uint64_t>(label.dst_gpu_addr32), label.value32);
}
} }
if (label.callback_2 != nullptr) { if (label.callback_2 != nullptr) {
@@ -229,29 +185,15 @@ void LabelManager::ThreadRun(void* data) {
} }
} }
template <typename T> Label* LabelManager::Create(GraphicContext* ctx, LabelCallback callback_1, LabelCallback callback_2,
Label* LabelManager::Create(GraphicContext* ctx, T* dst_gpu_addr, T value, LabelCallback callback_1, const uint64_t* args) {
LabelCallback callback_2, const uint64_t* args) {
static_assert(sizeof(T) == sizeof(uint32_t) || sizeof(T) == sizeof(uint64_t));
EXIT_IF(ctx == nullptr); EXIT_IF(ctx == nullptr);
Common::LockGuard lock(m_mutex); Common::LockGuard lock(m_mutex);
auto* label = new Label; auto* label = new Label;
label->status = LabelStatus::New; label->status = LabelStatus::New;
if constexpr (sizeof(T) == sizeof(uint64_t)) {
label->callbacks.dst_gpu_addr64 = dst_gpu_addr;
label->callbacks.value64 = value;
label->callbacks.dst_gpu_addr32 = nullptr;
label->callbacks.value32 = 0;
} else {
label->callbacks.dst_gpu_addr32 = dst_gpu_addr;
label->callbacks.value32 = value;
label->callbacks.dst_gpu_addr64 = nullptr;
label->callbacks.value64 = 0;
}
label->device = ctx->device; label->device = ctx->device;
label->callbacks.callback_1 = callback_1; label->callbacks.callback_1 = callback_1;
label->callbacks.callback_2 = callback_2; label->callbacks.callback_2 = callback_2;
@@ -267,18 +209,6 @@ Label* LabelManager::Create(GraphicContext* ctx, T* dst_gpu_addr, T value, Label
return label; return label;
} }
Label* LabelManager::Create64(GraphicContext* ctx, uint64_t* dst_gpu_addr, uint64_t value,
LabelCallback callback_1, LabelCallback callback_2,
const uint64_t* args) {
return Create(ctx, dst_gpu_addr, value, callback_1, callback_2, args);
}
Label* LabelManager::Create32(GraphicContext* ctx, uint32_t* dst_gpu_addr, uint32_t value,
LabelCallback callback_1, LabelCallback callback_2,
const uint64_t* args) {
return Create(ctx, dst_gpu_addr, value, callback_1, callback_2, args);
}
bool LabelManager::Remove(Label* label) { bool LabelManager::Remove(Label* label) {
EXIT_IF(label == nullptr); EXIT_IF(label == nullptr);
EXIT_IF(label->device == nullptr); EXIT_IF(label->device == nullptr);
@@ -336,8 +266,8 @@ void LabelManager::Set(CommandBuffer* buffer, Label* label) {
label->status = LabelStatus::Active; label->status = LabelStatus::Active;
LabelSubmission submission {}; LabelSubmission submission {};
submission.callbacks = label->callbacks; submission.callbacks = label->callbacks;
submission.completion = std::make_shared<LabelEvent>(); submission.completion = std::make_shared<LabelEvent>();
submission.completion->device = label->device; submission.completion->device = label->device;
auto* vk_buffer = buffer->GetPool()->buffers[buffer->GetIndex()]; auto* vk_buffer = buffer->GetPool()->buffers[buffer->GetIndex()];
@@ -385,18 +315,11 @@ void LabelInit() {
g_label_manager = new LabelManager; g_label_manager = new LabelManager;
} }
Label* LabelCreate64(GraphicContext* ctx, uint64_t* dst_gpu_addr, uint64_t value, Label* LabelCreate(GraphicContext* ctx, LabelCallback callback_1, LabelCallback callback_2,
LabelCallback callback_1, LabelCallback callback_2, const uint64_t* args) { const uint64_t* args) {
EXIT_IF(g_label_manager == nullptr); EXIT_IF(g_label_manager == nullptr);
return g_label_manager->Create64(ctx, dst_gpu_addr, value, callback_1, callback_2, args); return g_label_manager->Create(ctx, callback_1, callback_2, args);
}
Label* LabelCreate32(GraphicContext* ctx, uint32_t* dst_gpu_addr, uint32_t value,
LabelCallback callback_1, LabelCallback callback_2, const uint64_t* args) {
EXIT_IF(g_label_manager == nullptr);
return g_label_manager->Create32(ctx, dst_gpu_addr, value, callback_1, callback_2, args);
} }
void LabelDelete(Label* label) { void LabelDelete(Label* label) {
@@ -416,15 +339,6 @@ void LabelDrain() {
g_label_manager->Drain(); g_label_manager->Drain();
} }
void LabelWriteGuestMemory(void* address, const void* data, uint64_t size) {
if (address == nullptr || data == nullptr || size == 0) {
EXIT("invalid guest label write\n");
}
if (!Libs::LibKernel::Memory::TryWriteBacking(reinterpret_cast<uint64_t>(address), data, size)) {
std::memcpy(address, data, static_cast<size_t>(size));
}
}
bool LabelInCallback() noexcept { bool LabelInCallback() noexcept {
return g_in_label_callback; return g_in_label_callback;
} }
src/graphics/host_gpu/objects/label.h
+2 -5
View File
@@ -15,14 +15,11 @@ void LabelInit();
constexpr int LABEL_ARGS_MAX = 5; constexpr int LABEL_ARGS_MAX = 5;
using LabelCallback = bool (*)(const uint64_t* args); using LabelCallback = bool (*)(const uint64_t* args);
Label* LabelCreate64(GraphicContext* ctx, uint64_t* dst_gpu_addr, uint64_t value, Label* LabelCreate(GraphicContext* ctx, LabelCallback callback_1, LabelCallback callback_2,
LabelCallback callback_1, LabelCallback callback_2, const uint64_t* args); const uint64_t* args);
Label* LabelCreate32(GraphicContext* ctx, uint32_t* dst_gpu_addr, uint32_t value,
LabelCallback callback_1, LabelCallback callback_2, const uint64_t* args);
void LabelDelete(Label* label); void LabelDelete(Label* label);
void LabelSet(CommandBuffer* buffer, Label* label); void LabelSet(CommandBuffer* buffer, Label* label);
void LabelDrain(); void LabelDrain();
void LabelWriteGuestMemory(void* address, const void* data, uint64_t size);
[[nodiscard]] bool LabelInCallback() noexcept; [[nodiscard]] bool LabelInCallback() noexcept;
} // namespace Libs::Graphics } // namespace Libs::Graphics
src/graphics/host_gpu/renderer/bufferCache.cpp
+13 -297
View File
@@ -16,7 +16,6 @@
#include <algorithm> #include <algorithm>
#include <array> #include <array>
#include <atomic> #include <atomic>
#include <chrono>
#include <cstdlib> #include <cstdlib>
#include <cstring> #include <cstring>
#include <memory> #include <memory>
@@ -37,27 +36,6 @@ constexpr uint64_t AlignReadbackCopySize(uint64_t size) noexcept {
return (size + READBACK_COPY_ALIGNMENT - 1) & ~(READBACK_COPY_ALIGNMENT - 1); return (size + READBACK_COPY_ALIGNMENT - 1) & ~(READBACK_COPY_ALIGNMENT - 1);
} }
constexpr bool ShouldLogCommandProcessorReadback(uint64_t serial) noexcept {
return serial <= 32 || (serial % 256) == 0;
}
struct CommandProcessorReadbackState {
BufferCache* cache = nullptr;
PageFaultAccess access = PageFaultAccess::Unknown;
uint64_t vaddr = 0;
uint64_t size = 0;
uint64_t page = 0;
uint64_t serial = 0;
uint64_t dirty_bytes = 0;
uint64_t packed_bytes = 0;
uint32_t dirty_count = 0;
std::chrono::steady_clock::time_point started {};
bool installed = false;
};
thread_local CommandProcessorReadbackState g_command_processor_readback;
std::atomic<uint64_t> g_command_processor_readback_serial {0};
struct SharedVulkanBufferOwner { struct SharedVulkanBufferOwner {
explicit SharedVulkanBufferOwner(GraphicContext* context): ctx(context) { explicit SharedVulkanBufferOwner(GraphicContext* context): ctx(context) {
EXIT_IF(ctx == nullptr); EXIT_IF(ctx == nullptr);
@@ -192,10 +170,6 @@ bool CanMergeBufferCacheQueueMask(uint64_t queue_mask, uint32_t queue) noexcept
return queue < 64 && (queue_mask & ~(uint64_t {1} << queue)) == 0; return queue < 64 && (queue_mask & ~(uint64_t {1} << queue)) == 0;
} }
bool CanReadbackBufferCacheQueueMask(uint64_t queue_mask, uint32_t queue) noexcept {
return queue < 64 && queue_mask == (uint64_t {1} << queue);
}
struct BufferCache::CachedBuffer { struct BufferCache::CachedBuffer {
uint64_t vaddr = 0; uint64_t vaddr = 0;
uint64_t size = 0; uint64_t size = 0;
@@ -204,14 +178,6 @@ struct BufferCache::CachedBuffer {
std::shared_ptr<VulkanBuffer> buffer; std::shared_ptr<VulkanBuffer> buffer;
}; };
struct BufferCache::CommandProcessorReadbackResources {
GraphicContext* ctx = nullptr;
int queue = -1;
VulkanBuffer buffer {};
void* mapped = nullptr;
std::unique_ptr<CommandBuffer> command;
};
struct BufferCache::ReadbackWorker { struct BufferCache::ReadbackWorker {
static constexpr uint32_t MAX_RANGES = READBACK_MAX_RANGES; static constexpr uint32_t MAX_RANGES = READBACK_MAX_RANGES;
enum class State : uint32_t { enum class State : uint32_t {
@@ -305,20 +271,20 @@ struct BufferCache::ReadbackWorker {
} }
void Request(PageFaultAccess fault_access, uint64_t fault_vaddr, uint64_t fault_size) noexcept { void Request(PageFaultAccess fault_access, uint64_t fault_vaddr, uint64_t fault_size) noexcept {
const bool submissions_prepaused_now = GraphicsRunSubmissionLockHeld(); const bool command_thread = GraphicsRunIsCommandProcessorThread();
const bool submissions_prepaused_now = GraphicsRunSubmissionLockHeld() || command_thread;
const bool unsafe_gpu_lock = GraphicsRunGpuLockHeld() && !submissions_prepaused_now; const bool unsafe_gpu_lock = GraphicsRunGpuLockHeld() && !submissions_prepaused_now;
if (GraphicsRunIsCommandProcessorThread() || unsafe_gpu_lock || LabelInCallback() || if (unsafe_gpu_lock || LabelInCallback() || g_cache_lock_owner != nullptr ||
g_cache_lock_owner != nullptr || ctx == nullptr || command == nullptr || ctx == nullptr || command == nullptr || mapped == nullptr ||
mapped == nullptr || readback.buffer == nullptr) { readback.buffer == nullptr) {
EXIT("BufferCache: unsafe readback request context, command_thread=%d " EXIT("BufferCache: unsafe readback request context, command_thread=%d "
"submission_lock=%d " "submission_lock=%d "
"gpu_lock=%d label_callback=%d cache_lock=%p ctx=%p command=%p mapped=%p " "gpu_lock=%d label_callback=%d cache_lock=%p ctx=%p command=%p mapped=%p "
"buffer=%p\n", "buffer=%p\n",
GraphicsRunIsCommandProcessorThread(), GraphicsRunSubmissionLockHeld(), command_thread, GraphicsRunSubmissionLockHeld(), GraphicsRunGpuLockHeld(),
GraphicsRunGpuLockHeld(), LabelInCallback(), LabelInCallback(), static_cast<const void*>(g_cache_lock_owner),
static_cast<const void*>(g_cache_lock_owner), static_cast<const void*>(ctx), static_cast<const void*>(ctx), static_cast<const void*>(command.get()),
static_cast<const void*>(command.get()), static_cast<const void*>(mapped), static_cast<const void*>(mapped), static_cast<const void*>(readback.buffer));
static_cast<const void*>(readback.buffer));
} }
State expected = State::Idle; State expected = State::Idle;
while (!state.compare_exchange_weak(expected, State::Claimed, std::memory_order_acq_rel)) { while (!state.compare_exchange_weak(expected, State::Claimed, std::memory_order_acq_rel)) {
@@ -584,18 +550,6 @@ BufferCache::BufferCache(PageManager& page_manager, ResourceMutex& resource_mute
BufferCache::~BufferCache() { BufferCache::~BufferCache() {
m_readback.reset(); m_readback.reset();
if (m_cp_readback_resources != nullptr) {
auto& resources = *m_cp_readback_resources;
resources.command.reset();
if (resources.mapped != nullptr) {
VulkanUnmapMemory(resources.ctx, &resources.buffer.memory);
resources.mapped = nullptr;
}
if (resources.buffer.buffer != nullptr) {
VulkanDeleteBuffer(resources.ctx, &resources.buffer);
}
m_cp_readback_resources.reset();
}
if (!m_gpu_modified_ranges.Empty()) { if (!m_gpu_modified_ranges.Empty()) {
EXIT("BufferCache: destroyed with pending GPU-modified ranges\n"); EXIT("BufferCache: destroyed with pending GPU-modified ranges\n");
} }
@@ -623,14 +577,9 @@ bool BufferCache::InvalidateMemory(PageFaultAccess access, uint64_t vaddr, uint6
switch (phase) { switch (phase) {
case PageFaultPhase::Invalidate: break; case PageFaultPhase::Invalidate: break;
case PageFaultPhase::Complete: case PageFaultPhase::Complete:
return CompleteCommandProcessorReadback(access, vaddr, size) || return m_readback->Complete(access, vaddr, size) ||
m_readback->Complete(access, vaddr, size) ||
m_memory_tracker.CompleteCpuFault(vaddr, size, access, false); m_memory_tracker.CompleteCpuFault(vaddr, size, access, false);
case PageFaultPhase::Release: case PageFaultPhase::Release: m_readback->Release(access, vaddr, size); return true;
if (!ReleaseCommandProcessorReadback(access, vaddr, size)) {
m_readback->Release(access, vaddr, size);
}
return true;
default: default:
EXIT("BufferCache: unsupported page-fault phase %u\n", static_cast<uint32_t>(phase)); EXIT("BufferCache: unsupported page-fault phase %u\n", static_cast<uint32_t>(phase));
} }
@@ -639,242 +588,9 @@ bool BufferCache::InvalidateMemory(PageFaultAccess access, uint64_t vaddr, uint6
return action == CpuFaultAction::Continue; return action == CpuFaultAction::Continue;
} }
if (GraphicsRunIsCommandProcessorThread()) { if (GraphicsRunIsCommandProcessorThread()) {
if (!GraphicsRunIsCommandProcessorGuestAccess()) { GraphicsRunFinishCommandProcessors();
EXIT("BufferCache: command-processor fault outside a coherent guest-memory access, "
"addr=0x%016" PRIx64 " size=0x%016" PRIx64 "\n",
vaddr, size);
}
RequestCommandProcessorReadback(access, vaddr, size);
} else {
m_readback->Request(access, vaddr, size);
}
return true;
}
void BufferCache::RequestCommandProcessorReadback(PageFaultAccess access, uint64_t vaddr,
uint64_t size) {
if (!GraphicsRunIsCommandProcessorThread() || g_command_processor_readback.cache != nullptr) {
EXIT("BufferCache: invalid nested command-processor readback\n");
}
const auto serial =
g_command_processor_readback_serial.fetch_add(1, std::memory_order_relaxed) + 1;
if (serial == 0) {
EXIT("BufferCache: command-processor readback serial overflow\n");
}
const bool log_detail = ShouldLogCommandProcessorReadback(serial);
if (log_detail) {
LOGF("BufferCache: command-processor readback request serial=%" PRIu64 " addr=0x%016" PRIx64
" size=0x%016" PRIx64 " access=%u poll=%d\n",
serial, vaddr, size, static_cast<uint32_t>(access), GraphicsRunIsGuestMemoryPoll());
Log::Flush();
}
const auto page = vaddr & ~(TRACKER_PAGE_SIZE - 1);
const auto queue = static_cast<uint32_t>(GraphicsRunBeginCommandProcessorReadback());
if (log_detail) {
LOGF("BufferCache: command-processor queue synchronized serial=%" PRIu64
" queue=%u page=0x%016" PRIx64 "\n",
serial, queue, page);
Log::Flush();
}
if (g_cache_lock_owner != nullptr) {
EXIT("BufferCache: command-processor readback entered with a cache lock\n");
}
g_cache_lock_owner = this;
m_mutex.Lock();
if (log_detail) {
LOGF("BufferCache: command-processor cache locked serial=%" PRIu64 " page=0x%016" PRIx64
"\n",
serial, page);
Log::Flush();
}
auto it = m_buffers.upper_bound(vaddr);
if (it == m_buffers.begin()) {
EXIT("BufferCache: command-processor readback address has no cached buffer, "
"addr=0x%016" PRIx64 "\n",
vaddr);
}
--it;
auto& cached = *it->second;
if (vaddr < cached.vaddr || vaddr >= cached.vaddr + cached.size || page < cached.vaddr ||
TRACKER_PAGE_SIZE > cached.size - (page - cached.vaddr) || cached.ctx == nullptr ||
cached.buffer == nullptr || cached.buffer->buffer == nullptr) {
EXIT("BufferCache: invalid command-processor readback owner, fault=0x%016" PRIx64
" page=0x%016" PRIx64 " buffer=0x%016" PRIx64 "+0x%016" PRIx64 "\n",
vaddr, page, cached.vaddr, cached.size);
}
if (!CanReadbackBufferCacheQueueMask(cached.queue_mask, queue)) {
EXIT("BufferCache: command-processor readback cannot synchronize cross-queue ownership, "
"addr=0x%016" PRIx64 " used_queues=0x%016" PRIx64 " current_queue=%u\n",
vaddr, cached.queue_mask, queue);
}
const auto dirty = m_gpu_modified_ranges.Intersections(page, TRACKER_PAGE_SIZE);
if (dirty.empty()) {
EXIT("BufferCache: command-processor GPU-dirty page has no dirty byte ranges, "
"page=0x%016" PRIx64 "\n",
page);
}
ValidateDirtyPages(m_gpu_modified_ranges, page, TRACKER_PAGE_SIZE, "command-processor");
if (m_cp_readback_resources == nullptr) {
auto resources = std::make_unique<CommandProcessorReadbackResources>();
resources->ctx = cached.ctx;
resources->buffer.usage = VK_BUFFER_USAGE_TRANSFER_DST_BIT;
resources->buffer.memory.property = VK_MEMORY_PROPERTY_HOST_VISIBLE_BIT |
VK_MEMORY_PROPERTY_HOST_COHERENT_BIT |
VK_MEMORY_PROPERTY_HOST_CACHED_BIT;
VulkanCreateBuffer(cached.ctx, READBACK_CAPACITY, &resources->buffer);
VulkanMapMemory(cached.ctx, &resources->buffer.memory, &resources->mapped);
m_cp_readback_resources = std::move(resources);
}
auto& resources = *m_cp_readback_resources;
if (resources.ctx != cached.ctx || resources.mapped == nullptr ||
resources.buffer.buffer == nullptr) {
EXIT("BufferCache: invalid command-processor readback resources\n");
}
if (queue >= GraphicContext::QUEUES_NUM || cached.ctx->queues[queue].family == UINT32_MAX ||
cached.ctx->queues[queue].vk_queue == nullptr) {
EXIT("BufferCache: command-processor readback has invalid Vulkan queue %u\n", queue);
}
if (resources.command == nullptr || resources.queue != static_cast<int>(queue)) {
resources.command.reset();
resources.command = std::make_unique<CommandBuffer>(static_cast<int>(queue));
resources.queue = static_cast<int>(queue);
}
if (resources.command->IsInvalid() ||
resources.command->GetQueue() != static_cast<int>(queue)) {
EXIT("BufferCache: failed to initialize owning-queue readback command buffer, queue=%u\n",
queue);
}
if (dirty.size() > ReadbackWorker::MAX_RANGES) {
EXIT("BufferCache: too many command-processor dirty ranges, page=0x%016" PRIx64
" count=%" PRIu64 " limit=%u\n",
page, static_cast<uint64_t>(dirty.size()), ReadbackWorker::MAX_RANGES);
}
std::array<VkBufferCopy, ReadbackWorker::MAX_RANGES> copies {};
std::array<VkBufferMemoryBarrier, ReadbackWorker::MAX_RANGES> barriers {};
uint64_t dirty_bytes = 0;
uint64_t packed_bytes = 0;
uint32_t dirty_count = 0;
for (const auto& range: dirty) {
if ((range.address - cached.vaddr) % 4 != 0 || range.size % 4 != 0) {
EXIT("BufferCache: invalid command-processor dirty range, addr=0x%016" PRIx64
" size=0x%016" PRIx64 " page=0x%016" PRIx64 " packed=0x%016" PRIx64 "\n",
range.address, range.size, page, dirty_bytes);
}
copies[dirty_count] = {.srcOffset = range.address - cached.vaddr,
.dstOffset = packed_bytes,
.size = range.size};
auto& barrier = barriers[dirty_count];
barrier.sType = VK_STRUCTURE_TYPE_BUFFER_MEMORY_BARRIER;
barrier.srcAccessMask = VK_ACCESS_MEMORY_WRITE_BIT;
barrier.dstAccessMask = VK_ACCESS_TRANSFER_READ_BIT;
barrier.srcQueueFamilyIndex = VK_QUEUE_FAMILY_IGNORED;
barrier.dstQueueFamilyIndex = VK_QUEUE_FAMILY_IGNORED;
barrier.buffer = cached.buffer->buffer;
barrier.offset = copies[dirty_count].srcOffset;
barrier.size = range.size;
const auto slot_size = AlignReadbackCopySize(range.size);
if (packed_bytes > READBACK_CAPACITY || slot_size > READBACK_CAPACITY - packed_bytes) {
EXIT("BufferCache: aligned command-processor readback exceeds persistent capacity, "
"packed=%" PRIu64 " size=%" PRIu64 " capacity=%" PRIu64 "\n",
packed_bytes, slot_size, READBACK_CAPACITY);
}
packed_bytes += slot_size;
dirty_bytes += range.size;
dirty_count++;
}
if (dirty_count == 0 || dirty_bytes == 0) {
EXIT("BufferCache: command-processor readback produced no packed ranges\n");
}
const auto started = std::chrono::steady_clock::now();
if (log_detail) {
LOGF("BufferCache: command-processor readback begin serial=%" PRIu64
" queue=%u page=0x%016" PRIx64 " ranges=%u bytes=%" PRIu64 " packed=%" PRIu64 "\n",
serial, queue, page, dirty_count, dirty_bytes, packed_bytes);
Log::Flush();
}
auto* vk_command = resources.command->GetPool()->buffers[resources.command->GetIndex()];
resources.command->Begin();
vkCmdPipelineBarrier(vk_command, VK_PIPELINE_STAGE_ALL_COMMANDS_BIT,
VK_PIPELINE_STAGE_TRANSFER_BIT, 0, 0, nullptr, dirty_count,
barriers.data(), 0, nullptr);
vkCmdCopyBuffer(vk_command, cached.buffer->buffer, resources.buffer.buffer, dirty_count,
copies.data());
resources.command->End();
resources.command->Execute();
resources.command->WaitForFenceAndReset();
if (log_detail) {
LOGF("BufferCache: command-processor GPU copy complete serial=%" PRIu64 "\n", serial);
Log::Flush();
}
for (uint32_t i = 0; i < dirty_count; i++) {
Libs::LibKernel::Memory::WriteBacking(
dirty[i].address, static_cast<const uint8_t*>(resources.mapped) + copies[i].dstOffset,
dirty[i].size);
}
g_command_processor_readback = {.cache = this,
.access = access,
.vaddr = vaddr,
.size = size,
.page = page,
.serial = serial,
.dirty_bytes = dirty_bytes,
.packed_bytes = packed_bytes,
.dirty_count = dirty_count,
.started = started};
}
bool BufferCache::CompleteCommandProcessorReadback(PageFaultAccess access, uint64_t vaddr,
uint64_t size) noexcept {
auto& pending = g_command_processor_readback;
if (pending.cache == nullptr) {
return false;
}
if (pending.cache != this || pending.access != access || pending.vaddr != vaddr ||
pending.size != size || pending.installed) {
EXIT("BufferCache: mismatched active command-processor readback completion\n");
}
if (!m_memory_tracker.CompleteCpuFault(vaddr, size, access, true)) {
EXIT("BufferCache: failed to complete command-processor readback, addr=0x%016" PRIx64
" size=0x%016" PRIx64 " access=%u\n",
vaddr, size, static_cast<uint32_t>(access));
}
m_gpu_modified_ranges.Subtract(pending.page, TRACKER_PAGE_SIZE);
pending.installed = true;
if (ShouldLogCommandProcessorReadback(pending.serial)) {
LOGF("BufferCache: command-processor readback complete serial=%" PRIu64
" page=0x%016" PRIx64 " ranges=%u bytes=%" PRIu64 "\n",
pending.serial, pending.page, pending.dirty_count, pending.dirty_bytes);
}
return true;
}
bool BufferCache::ReleaseCommandProcessorReadback(PageFaultAccess access, uint64_t vaddr,
uint64_t size) noexcept {
auto& pending = g_command_processor_readback;
if (pending.cache == nullptr) {
return false;
}
if (pending.cache != this || pending.access != access || pending.vaddr != vaddr ||
pending.size != size || !pending.installed) {
EXIT("BufferCache: mismatched active command-processor readback release\n");
}
const auto serial = pending.serial;
const auto page = pending.page;
const auto dirty_count = pending.dirty_count;
const auto dirty_bytes = pending.dirty_bytes;
const auto elapsed_micro = std::chrono::duration_cast<std::chrono::microseconds>(
std::chrono::steady_clock::now() - pending.started)
.count();
pending = {};
m_mutex.Unlock();
g_cache_lock_owner = nullptr;
GraphicsRunEndCommandProcessorReadback();
if (ShouldLogCommandProcessorReadback(serial)) {
LOGF("BufferCache: command-processor readback release serial=%" PRIu64 " page=0x%016" PRIx64
" ranges=%u bytes=%" PRIu64 " elapsed_us=%" PRIi64 "\n",
serial, page, dirty_count, dirty_bytes, static_cast<int64_t>(elapsed_micro));
} }
m_readback->Request(access, vaddr, size);
return true; return true;
} }
src/graphics/host_gpu/renderer/bufferCache.h
+3 -13
View File
@@ -39,7 +39,6 @@ struct BufferCacheRange {
[[nodiscard]] bool MergeOverlappingBufferCacheRange(BufferCacheRange* merged, [[nodiscard]] bool MergeOverlappingBufferCacheRange(BufferCacheRange* merged,
BufferCacheRange candidate) noexcept; BufferCacheRange candidate) noexcept;
[[nodiscard]] bool CanMergeBufferCacheQueueMask(uint64_t queue_mask, uint32_t queue) noexcept; [[nodiscard]] bool CanMergeBufferCacheQueueMask(uint64_t queue_mask, uint32_t queue) noexcept;
[[nodiscard]] bool CanReadbackBufferCacheQueueMask(uint64_t queue_mask, uint32_t queue) noexcept;
class BufferCache { class BufferCache {
public: public:
@@ -61,11 +60,9 @@ public:
// Emulator-owned, CPU-current scratch only. Guest ranges must use ObtainBuffer so page // Emulator-owned, CPU-current scratch only. Guest ranges must use ObtainBuffer so page
// ownership is resolved before any CPU access. // ownership is resolved before any CPU access.
[[nodiscard]] bool UploadHostData(CommandBuffer* command, GraphicContext* ctx, const void* src, [[nodiscard]] bool UploadHostData(CommandBuffer* command, GraphicContext* ctx, const void* src,
uint64_t size, uint64_t alignment, uint64_t size, uint64_t alignment, VulkanBuffer** out_buffer,
VulkanBuffer** out_buffer, uint64_t* out_offset, uint64_t* out_offset, uint64_t* out_range);
uint64_t* out_range); [[nodiscard]] VulkanBuffer* ObtainNullBuffer(CommandBuffer* command, GraphicContext* ctx);
[[nodiscard]] VulkanBuffer* ObtainNullBuffer(CommandBuffer* command,
GraphicContext* ctx);
[[nodiscard]] BufferImageCopySource ObtainBufferForImage(uint64_t vaddr, uint64_t size); [[nodiscard]] BufferImageCopySource ObtainBufferForImage(uint64_t vaddr, uint64_t size);
void FillBuffer(CommandBuffer* command, GraphicContext* ctx, uint64_t vaddr, uint64_t size, void FillBuffer(CommandBuffer* command, GraphicContext* ctx, uint64_t vaddr, uint64_t size,
uint32_t value); uint32_t value);
@@ -84,19 +81,12 @@ public:
private: private:
struct CachedBuffer; struct CachedBuffer;
struct ReadbackWorker; struct ReadbackWorker;
struct CommandProcessorReadbackResources;
void RequestCommandProcessorReadback(PageFaultAccess access, uint64_t vaddr, uint64_t size);
[[nodiscard]] bool CompleteCommandProcessorReadback(PageFaultAccess access, uint64_t vaddr,
uint64_t size) noexcept;
[[nodiscard]] bool ReleaseCommandProcessorReadback(PageFaultAccess access, uint64_t vaddr,
uint64_t size) noexcept;
Common::Mutex m_mutex; Common::Mutex m_mutex;
std::vector<VulkanBuffer*> m_delete_later; std::vector<VulkanBuffer*> m_delete_later;
std::shared_ptr<VulkanBuffer> m_null_buffer; std::shared_ptr<VulkanBuffer> m_null_buffer;
std::map<uint64_t, std::unique_ptr<CachedBuffer>> m_buffers; std::map<uint64_t, std::unique_ptr<CachedBuffer>> m_buffers;
std::unique_ptr<ReadbackWorker> m_readback; std::unique_ptr<ReadbackWorker> m_readback;
std::unique_ptr<CommandProcessorReadbackResources> m_cp_readback_resources;
RangeSet m_gpu_modified_ranges; RangeSet m_gpu_modified_ranges;
MemoryTracker m_memory_tracker; MemoryTracker m_memory_tracker;
PageManager& m_page_manager; PageManager& m_page_manager;
src/graphics/host_gpu/renderer/context.cpp
+4 -3
View File
@@ -332,7 +332,7 @@ void CommandBuffer::Execute() {
m_execute = true; m_execute = true;
m_fence_waited = false; m_fence_waited = false;
m_submit_seq = g_command_buffer_submit_seq.fetch_add(1, std::memory_order_relaxed) + 1; m_submit_seq = g_command_buffer_submit_seq.fetch_add(1, std::memory_order_relaxed) + 1;
if (result != VK_SUCCESS) { if (result != VK_SUCCESS) {
LOGF("vkQueueSubmit failed: %s (%d), queue=%d index=%u submit_seq=%" PRIu64 LOGF("vkQueueSubmit failed: %s (%d), queue=%d index=%u submit_seq=%" PRIu64
@@ -396,7 +396,7 @@ void CommandBuffer::ExecuteWithSemaphore(VkSemaphore signal_semaphore) {
m_execute = true; m_execute = true;
m_fence_waited = false; m_fence_waited = false;
m_submit_seq = g_command_buffer_submit_seq.fetch_add(1, std::memory_order_relaxed) + 1; m_submit_seq = g_command_buffer_submit_seq.fetch_add(1, std::memory_order_relaxed) + 1;
if (result != VK_SUCCESS) { if (result != VK_SUCCESS) {
LOGF("vkQueueSubmit failed: %s (%d), queue=%d index=%u submit_seq=%" PRIu64 LOGF("vkQueueSubmit failed: %s (%d), queue=%d index=%u submit_seq=%" PRIu64
@@ -464,7 +464,7 @@ void CommandBuffer::ExecuteWithSemaphore(VkSemaphore wait_semaphore,
m_execute = true; m_execute = true;
m_fence_waited = false; m_fence_waited = false;
m_submit_seq = g_command_buffer_submit_seq.fetch_add(1, std::memory_order_relaxed) + 1; m_submit_seq = g_command_buffer_submit_seq.fetch_add(1, std::memory_order_relaxed) + 1;
if (result != VK_SUCCESS) { if (result != VK_SUCCESS) {
LOGF("vkQueueSubmit failed: %s (%d), queue=%d index=%u submit_seq=%" PRIu64 LOGF("vkQueueSubmit failed: %s (%d), queue=%d index=%u submit_seq=%" PRIu64
@@ -518,6 +518,7 @@ void CommandBuffer::WaitForFenceAndReset() {
m_fence_waited = false; m_fence_waited = false;
vkResetCommandBuffer(m_pool->buffers[m_index], vkResetCommandBuffer(m_pool->buffers[m_index],
VK_COMMAND_BUFFER_RESET_RELEASE_RESOURCES_BIT); VK_COMMAND_BUFFER_RESET_RELEASE_RESOURCES_BIT);
m_recording_generation++;
} }
m_host_stream.Reset(); m_host_stream.Reset();
if (was_executed) { if (was_executed) {
src/graphics/host_gpu/renderer/descriptors.cpp
+1 -1
View File
@@ -205,7 +205,7 @@ NativeAddressBuffer(uint64_t submit_id, CommandBuffer* command_buffer,
->GetPhysicalDeviceProperties() ->GetPhysicalDeviceProperties()
.limits.maxStorageBufferRange); .limits.maxStorageBufferRange);
uint64_t size = 0; uint64_t size = 0;
const auto access = HostMemoryAccess::Read; const auto access = HostMemoryAccess::Mapped;
if (!HostMemoryQueryRange(address.binding_base, limit, access, &size)) { if (!HostMemoryQueryRange(address.binding_base, limit, access, &size)) {
EXIT("address resource is not host-accessible: base=0x%016" PRIx64 "\n", EXIT("address resource is not host-accessible: base=0x%016" PRIx64 "\n",
address.binding_base); address.binding_base);
src/graphics/host_gpu/renderer/gpuResourceManager.cpp
+16 -37
View File
@@ -1,6 +1,7 @@
#include "graphics/host_gpu/renderer/gpuResourceManager.h" #include "graphics/host_gpu/renderer/gpuResourceManager.h"
#include "common/assert.h" #include "common/assert.h"
#include "graphics/guest_gpu/command_processor/commandProcessor.h"
#include "graphics/guest_gpu/graphicsRun.h" #include "graphics/guest_gpu/graphicsRun.h"
#include "graphics/host_gpu/objects/label.h" #include "graphics/host_gpu/objects/label.h"
#include "graphics/host_gpu/renderer/render.h" #include "graphics/host_gpu/renderer/render.h"
@@ -37,17 +38,24 @@ bool GpuResourceManager::HandleFault(PageFaultAccess access, uint64_t fault_vadd
"addr=0x%016" PRIx64 " access=%u\n", "addr=0x%016" PRIx64 " access=%u\n",
fault_vaddr, static_cast<uint32_t>(access)); fault_vaddr, static_cast<uint32_t>(access));
} }
if (m_resource_mutex.IsOwnedByCurrentThread()) { if (auto* cp = GraphicsRunCurrentCommandProcessor(); cp != nullptr) {
if (!GraphicsRunIsCommandProcessorGuestAccess()) { cp->BeginReadbackTransaction();
EXIT("unsupported page fault from a pre-owned resource transaction, addr=0x%016" PRIx64 bool handled = false;
" access=%u\n", {
fault_vaddr, static_cast<uint32_t>(access)); ResourceMutex::FaultScope fault(m_resource_mutex);
handled = m_page_manager.HandleFault(access, fault_vaddr);
} }
ResourceMutex::FaultScope fault(m_resource_mutex); cp->EndReadbackTransaction();
return m_page_manager.HandleFault(access, fault_vaddr); return handled;
}
if (m_resource_mutex.IsOwnedByCurrentThread()) {
EXIT("unsupported page fault from a pre-owned resource transaction, addr=0x%016" PRIx64
" access=%u\n",
fault_vaddr, static_cast<uint32_t>(access));
} }
// Stop command-processor jobs before taking the shared cache transaction. External readback // Stop command-processor jobs before taking the shared cache transaction. External readback
// workers inherit this paused state and therefore never form resource -> submission lock inversion. // workers inherit this paused state and therefore never form resource -> submission lock
// inversion.
GraphicsRunSubmissionLock submissions; GraphicsRunSubmissionLock submissions;
ResourceMutex::FaultScope fault(m_resource_mutex); ResourceMutex::FaultScope fault(m_resource_mutex);
return m_page_manager.HandleFault(access, fault_vaddr); return m_page_manager.HandleFault(access, fault_vaddr);
@@ -57,35 +65,6 @@ bool GpuResourceManager::IsMapped(uint64_t vaddr, uint64_t size) const noexcept
return m_page_manager.IsMapped(vaddr, size); return m_page_manager.IsMapped(vaddr, size);
} }
std::unique_lock<ResourceMutex> GpuResourceManager::LockCommandProcessorGuestAccess(
uint64_t vaddr, uint64_t size) {
if (vaddr == 0 || size == 0 || vaddr >= TRACKER_ADDRESS_SIZE ||
size > TRACKER_ADDRESS_SIZE - vaddr) {
EXIT("invalid command-processor guest-memory access, addr=0x%016" PRIx64
" size=0x%016" PRIx64 "\n",
vaddr, size);
}
return std::unique_lock(m_resource_mutex);
}
std::unique_lock<ResourceMutex> GpuResourceManager::LockAsynchronousGuestWrite(uint64_t vaddr,
uint64_t size) {
if (vaddr == 0 || size == 0 || vaddr >= TRACKER_ADDRESS_SIZE ||
size > TRACKER_ADDRESS_SIZE - vaddr) {
EXIT("invalid asynchronous guest-memory write, addr=0x%016" PRIx64
" size=0x%016" PRIx64 "\n",
vaddr, size);
}
std::unique_lock transaction(m_resource_mutex);
if (m_buffer_cache.HasPageOverlap(vaddr, size) || m_texture_cache.HasPageOverlap(vaddr, size) ||
m_texture_cache.HasMetaOverlap(vaddr, size)) {
EXIT("asynchronous guest-memory write to a cached buffer/image range is unsupported, "
"addr=0x%016" PRIx64 " size=0x%016" PRIx64 "\n",
vaddr, size);
}
return transaction;
}
void GpuResourceManager::MapMemory(uint64_t vaddr, uint64_t size, GpuAccess access) { void GpuResourceManager::MapMemory(uint64_t vaddr, uint64_t size, GpuAccess access) {
m_page_manager.OnGpuMap(vaddr, size, access); m_page_manager.OnGpuMap(vaddr, size, access);
} }
src/graphics/host_gpu/renderer/gpuResourceManager.h
+1 -10
View File
@@ -9,7 +9,6 @@
#include "graphics/host_gpu/renderer/textureCache.h" #include "graphics/host_gpu/renderer/textureCache.h"
#include <cstdint> #include <cstdint>
#include <mutex>
namespace Libs::Graphics { namespace Libs::Graphics {
@@ -26,14 +25,6 @@ public:
[[nodiscard]] bool HandleFault(PageFaultAccess access, uint64_t fault_vaddr) noexcept; [[nodiscard]] bool HandleFault(PageFaultAccess access, uint64_t fault_vaddr) noexcept;
[[nodiscard]] bool IsMapped(uint64_t vaddr, uint64_t size) const noexcept; [[nodiscard]] bool IsMapped(uint64_t vaddr, uint64_t size) const noexcept;
// Serializes synchronous command-processor guest-memory access against buffer/image ownership
// publication. Address metadata keeps this seam ready for a future page-range lock.
[[nodiscard]] std::unique_lock<ResourceMutex>
LockCommandProcessorGuestAccess(uint64_t vaddr, uint64_t size);
// Keeps an asynchronous host callback's validation and guest write in one cache transaction.
// Cached buffer/image destinations are unsupported and fail before the callback can fault.
[[nodiscard]] std::unique_lock<ResourceMutex> LockAsynchronousGuestWrite(uint64_t vaddr,
uint64_t size);
void MapMemory(uint64_t vaddr, uint64_t size, GpuAccess access); void MapMemory(uint64_t vaddr, uint64_t size, GpuAccess access);
void UnmapMemory(uint64_t vaddr, uint64_t size, GpuAccess access); void UnmapMemory(uint64_t vaddr, uint64_t size, GpuAccess access);
void FillBuffer(CommandBuffer* command, uint64_t vaddr, uint64_t size, uint32_t value); void FillBuffer(CommandBuffer* command, uint64_t vaddr, uint64_t size, uint32_t value);
@@ -48,7 +39,7 @@ private:
PageManager m_page_manager; PageManager m_page_manager;
ResourceMutex m_resource_mutex; ResourceMutex m_resource_mutex;
BufferCache m_buffer_cache; BufferCache m_buffer_cache;
TextureCache m_texture_cache; TextureCache m_texture_cache;
}; };
} // namespace Libs::Graphics } // namespace Libs::Graphics
src/graphics/host_gpu/renderer/render.h
+31 -29
View File
@@ -87,25 +87,27 @@ public:
[[nodiscard]] int GetQueue() const { return m_queue; } [[nodiscard]] int GetQueue() const { return m_queue; }
VulkanCommandPool* GetPool() { return m_pool; } VulkanCommandPool* GetPool() { return m_pool; }
[[nodiscard]] bool IsExecute() const { return m_execute; } [[nodiscard]] bool IsExecute() const { return m_execute; }
[[nodiscard]] uint64_t GetRecordingGeneration() const { return m_recording_generation; }
private: private:
friend class BufferCache; friend class BufferCache;
void RecycleDescriptorsAfterFence(); void RecycleDescriptorsAfterFence();
VulkanCommandPool* m_pool = nullptr; VulkanCommandPool* m_pool = nullptr;
uint32_t m_index = static_cast<uint32_t>(-1); uint32_t m_index = static_cast<uint32_t>(-1);
int m_queue = -1; int m_queue = -1;
bool m_execute = false; bool m_execute = false;
bool m_fence_waited = false; bool m_fence_waited = false;
uint64_t m_submit_seq = 0; uint64_t m_submit_seq = 0;
uint32_t m_debug_op = 0; uint64_t m_recording_generation = 0;
uint64_t m_debug_submit_id = 0; uint32_t m_debug_op = 0;
uint32_t m_debug_arg0 = 0; uint64_t m_debug_submit_id = 0;
uint32_t m_debug_arg1 = 0; uint32_t m_debug_arg0 = 0;
uint32_t m_debug_arg2 = 0; uint32_t m_debug_arg1 = 0;
uint32_t m_debug_arg3 = 0; uint32_t m_debug_arg2 = 0;
uint64_t m_debug_arg4 = 0; uint32_t m_debug_arg3 = 0;
uint64_t m_debug_arg4 = 0;
std::vector<VulkanBuffer*> m_delete_after_fence; std::vector<VulkanBuffer*> m_delete_after_fence;
FenceResourceRetainer m_fence_resources; FenceResourceRetainer m_fence_resources;
std::vector<VulkanDescriptorSet*> m_descriptor_sets_after_fence; std::vector<VulkanDescriptorSet*> m_descriptor_sets_after_fence;
@@ -129,8 +131,8 @@ void RenderDispatchDirect(uint64_t submit_id, CommandBuffer* buffer, HW::Context
void GraphicsRenderInit(); void GraphicsRenderInit();
void GraphicsRenderCreateContext(); void GraphicsRenderCreateContext();
[[nodiscard]] bool GraphicsScaleReferenceClock(uint64_t host_ticks, uint64_t host_frequency, [[nodiscard]] bool GraphicsScaleReferenceClock(uint64_t host_ticks, uint64_t host_frequency,
uint64_t* value); uint64_t* value);
[[nodiscard]] uint64_t GraphicsRenderReadReferenceClock(); [[nodiscard]] uint64_t GraphicsRenderReadReferenceClock();
[[nodiscard]] bool ResolveComputeImageClear(const ShaderComputeInputInfo& input, uint32_t group_x, [[nodiscard]] bool ResolveComputeImageClear(const ShaderComputeInputInfo& input, uint32_t group_x,
@@ -140,21 +142,22 @@ void GraphicsRenderCreateContext();
[[nodiscard]] bool ResolveHtileClearTarget(const HW::DepthRenderTarget& target, [[nodiscard]] bool ResolveHtileClearTarget(const HW::DepthRenderTarget& target,
uint64_t descriptor_size, HtileClearTarget* resolved); uint64_t descriptor_size, HtileClearTarget* resolved);
void GraphicsRenderWriteAtEndOfPipe64(uint64_t submit_id, CommandBuffer* buffer, void GraphicsRenderWriteAtEndOfPipe64(uint64_t submit_id, CommandBuffer* buffer,
uint64_t* dst_gpu_addr, uint64_t value); uint64_t* dst_gpu_addr, uint64_t value);
void GraphicsRenderWriteAtEndOfPipeClockCounter(uint64_t submit_id, CommandBuffer* buffer, void GraphicsRenderWriteAtEndOfPipeClockCounter(uint64_t submit_id, CommandBuffer* buffer,
uint64_t* dst_gpu_addr); uint64_t* dst_gpu_addr, uint64_t value);
void GraphicsRenderWriteAtEndOfPipeClockCounterWithWriteBack(uint64_t submit_id, void GraphicsRenderWriteAtEndOfPipeClockCounterWithWriteBack(uint64_t submit_id,
CommandBuffer* buffer, CommandBuffer* buffer,
uint64_t* dst_gpu_addr); uint64_t* dst_gpu_addr,
void GraphicsRenderWriteAtEndOfPipe32(uint64_t submit_id, CommandBuffer* buffer, uint64_t value);
uint32_t* dst_gpu_addr, uint32_t value); void GraphicsRenderWriteAtEndOfPipe32(uint64_t submit_id, CommandBuffer* buffer,
void GraphicsRenderWriteAtEndOfPipeGds32(uint64_t submit_id, CommandBuffer* buffer, uint32_t* dst_gpu_addr, uint32_t value);
uint32_t* dst_gpu_addr, uint32_t dw_offset, void GraphicsRenderWriteAtEndOfPipeGds32(uint64_t submit_id, CommandBuffer* buffer,
uint32_t dw_num); uint32_t* dst_gpu_addr, uint32_t dw_offset,
uint32_t dw_num);
uint64_t GraphicsRenderPrepareDisplayBufferFlip(CommandBuffer* buffer, int handle, int index, uint64_t GraphicsRenderPrepareDisplayBufferFlip(CommandBuffer* buffer, int handle, int index,
int flip_mode, int64_t flip_arg); int flip_mode, int64_t flip_arg);
void GraphicsRenderWriteAtEndOfPipeWithInterruptWriteBackFlip32( void GraphicsRenderWriteAtEndOfPipeWithInterruptWriteBackFlip32(
uint64_t submit_id, CommandBuffer* buffer, uint32_t* dst_gpu_addr, uint32_t value, int handle, uint64_t submit_id, CommandBuffer* buffer, uint32_t* dst_gpu_addr, uint32_t value, int handle,
int index, int flip_mode, int64_t flip_arg, uint64_t request_id); int index, int flip_mode, int64_t flip_arg, uint64_t request_id);
void GraphicsRenderWriteAtEndOfPipeWithFlip32(uint64_t submit_id, CommandBuffer* buffer, void GraphicsRenderWriteAtEndOfPipeWithFlip32(uint64_t submit_id, CommandBuffer* buffer,
@@ -193,7 +196,6 @@ int GraphicsRenderDeleteEqEvent(LibKernel::EventQueue::KernelEqueue eq, int id)
void GraphicsRenderTriggerAgcUserInterrupt(); void GraphicsRenderTriggerAgcUserInterrupt();
void GraphicsRenderTriggerEopEvent(uint32_t context_id); void GraphicsRenderTriggerEopEvent(uint32_t context_id);
void GraphicsRenderClearGds(uint64_t dw_offset, uint32_t dw_num, uint32_t clear_value);
void GraphicsRenderReadGds(uint32_t* dst, uint32_t dw_offset, uint32_t dw_size); void GraphicsRenderReadGds(uint32_t* dst, uint32_t dw_offset, uint32_t dw_size);
} // namespace Libs::Graphics } // namespace Libs::Graphics
src/graphics/host_gpu/renderer/renderCompute.cpp
+30 -24
View File
@@ -1,7 +1,3 @@
#include "graphics/host_gpu/renderer/render.h"
#include "graphics/host_gpu/renderer/renderInternal.h"
#include "common/assert.h" #include "common/assert.h"
#include "common/common.h" #include "common/common.h"
#include "common/emulatorConfig.h" #include "common/emulatorConfig.h"
@@ -19,7 +15,9 @@
#include "graphics/host_gpu/renderer/descriptorCache.h" #include "graphics/host_gpu/renderer/descriptorCache.h"
#include "graphics/host_gpu/renderer/framebufferCache.h" #include "graphics/host_gpu/renderer/framebufferCache.h"
#include "graphics/host_gpu/renderer/pipelineCache.h" #include "graphics/host_gpu/renderer/pipelineCache.h"
#include "graphics/host_gpu/renderer/render.h"
#include "graphics/host_gpu/renderer/renderContext.h" #include "graphics/host_gpu/renderer/renderContext.h"
#include "graphics/host_gpu/renderer/renderInternal.h"
#include "graphics/host_gpu/renderer/renderState.h" #include "graphics/host_gpu/renderer/renderState.h"
#include "graphics/host_gpu/renderer/shaderResourceBarrier.h" #include "graphics/host_gpu/renderer/shaderResourceBarrier.h"
#include "graphics/host_gpu/renderer/shaderSubgroup.h" #include "graphics/host_gpu/renderer/shaderSubgroup.h"
@@ -547,30 +545,38 @@ void RenderDispatchDirect(uint64_t submit_id, CommandBuffer* buffer, HW::Context
return; return;
} }
auto* vk_buffer = buffer->GetPool()->buffers[buffer->GetIndex()]; for (;;) {
auto* pipeline = g_render_ctx->GetPipelineCache()->CreateComputePipeline( const auto recording_generation = buffer->GetRecordingGeneration();
&input_info, &sh_ctx->GetCs(), cs_shader); auto* vk_buffer = buffer->GetPool()->buffers[buffer->GetIndex()];
auto* pipeline = g_render_ctx->GetPipelineCache()->CreateComputePipeline(
&input_info, &sh_ctx->GetCs(), cs_shader);
vkCmdBindPipeline(vk_buffer, VK_PIPELINE_BIND_POINT_COMPUTE, pipeline->pipeline); vkCmdBindPipeline(vk_buffer, VK_PIPELINE_BIND_POINT_COMPUTE, pipeline->pipeline);
const auto address_writes = BindDescriptors( const auto address_writes = BindDescriptors(
submit_id, buffer, VK_PIPELINE_BIND_POINT_COMPUTE, pipeline->pipeline_layout, submit_id, buffer, VK_PIPELINE_BIND_POINT_COMPUTE, pipeline->pipeline_layout,
input_info.stage, VK_SHADER_STAGE_COMPUTE_BIT, DescriptorCache::Stage::Compute); input_info.stage, VK_SHADER_STAGE_COMPUTE_BIT, DescriptorCache::Stage::Compute);
if (buffer->GetRecordingGeneration() != recording_generation) {
continue;
}
vkCmdDispatch(vk_buffer, thread_group_x, thread_group_y, thread_group_z); vkCmdDispatch(vk_buffer, thread_group_x, thread_group_y, thread_group_z);
bool has_storage_writes = HasShaderBufferWrites(input_info.stage); bool has_storage_writes = HasShaderBufferWrites(input_info.stage);
has_storage_writes = MarkShaderAddressWrites(address_writes) || has_storage_writes; has_storage_writes = MarkShaderAddressWrites(address_writes) || has_storage_writes;
has_storage_writes = has_storage_writes =
std::any_of(program.info.images.begin(), program.info.images.end(), std::any_of(
[](const auto& image) { program.info.images.begin(), program.info.images.end(),
return image.written && [](const auto& image) {
(image.kind == ShaderRecompiler::IR::ResourceKind::StorageImage || return image.written &&
image.kind == ShaderRecompiler::IR::ResourceKind::StorageImageUint); (image.kind == ShaderRecompiler::IR::ResourceKind::StorageImage ||
}) || image.kind == ShaderRecompiler::IR::ResourceKind::StorageImageUint);
has_storage_writes; }) ||
if (has_storage_writes) { has_storage_writes;
ShaderWriteBarrier(vk_buffer, VK_PIPELINE_STAGE_COMPUTE_SHADER_BIT); if (has_storage_writes) {
ShaderWriteBarrier(vk_buffer, VK_PIPELINE_STAGE_COMPUTE_SHADER_BIT);
}
break;
} }
} }
src/graphics/host_gpu/renderer/renderDraw.cpp
+138 -118
View File
@@ -1,7 +1,3 @@
#include "graphics/host_gpu/renderer/render.h"
#include "graphics/host_gpu/renderer/renderInternal.h"
#include "common/assert.h" #include "common/assert.h"
#include "common/common.h" #include "common/common.h"
#include "common/emulatorConfig.h" #include "common/emulatorConfig.h"
@@ -19,7 +15,9 @@
#include "graphics/host_gpu/renderer/descriptorCache.h" #include "graphics/host_gpu/renderer/descriptorCache.h"
#include "graphics/host_gpu/renderer/framebufferCache.h" #include "graphics/host_gpu/renderer/framebufferCache.h"
#include "graphics/host_gpu/renderer/pipelineCache.h" #include "graphics/host_gpu/renderer/pipelineCache.h"
#include "graphics/host_gpu/renderer/render.h"
#include "graphics/host_gpu/renderer/renderContext.h" #include "graphics/host_gpu/renderer/renderContext.h"
#include "graphics/host_gpu/renderer/renderInternal.h"
#include "graphics/host_gpu/renderer/renderState.h" #include "graphics/host_gpu/renderer/renderState.h"
#include "graphics/host_gpu/renderer/shaderResourceBarrier.h" #include "graphics/host_gpu/renderer/shaderResourceBarrier.h"
#include "graphics/host_gpu/renderer/shaderSubgroup.h" #include "graphics/host_gpu/renderer/shaderSubgroup.h"
@@ -603,11 +601,12 @@ struct DrawEmitInfo {
uint32_t first_vertex = 0; uint32_t first_vertex = 0;
}; };
struct DrawIndexBufferBind { struct DrawIndexBufferSource {
bool enabled = false; bool enabled = false;
VulkanBuffer* buffer = nullptr; uint64_t address = 0;
VkDeviceSize offset = 0; const void* host_data = nullptr;
VkIndexType type = VK_INDEX_TYPE_UINT16; uint64_t size = 0;
VkIndexType type = VK_INDEX_TYPE_UINT16;
}; };
static uint64_t VertexBufferDescriptorSize(const ShaderVertexInputBuffer& buffer) { static uint64_t VertexBufferDescriptorSize(const ShaderVertexInputBuffer& buffer) {
@@ -802,6 +801,32 @@ static void BindDrawVertexBuffers(uint64_t submit_id, CommandBuffer* buffer,
} }
} }
static void BindDrawIndexBuffer(CommandBuffer* buffer, VkCommandBuffer vk_buffer,
const DrawIndexBufferSource& source) {
if (!source.enabled) {
return;
}
EXIT_IF(source.size == 0);
VulkanBuffer* index_buffer = nullptr;
VkDeviceSize index_offset = 0;
if (source.host_data != nullptr) {
VkDeviceSize range = 0;
if (!g_render_ctx->GetBufferCache()->UploadHostData(
buffer, g_render_ctx->GetGraphicCtx(), source.host_data, source.size, 16,
&index_buffer, &index_offset, &range)) {
EXIT("failed to upload host index buffer\n");
}
} else {
auto binding = g_render_ctx->GetBufferCache()->ObtainBuffer(
buffer, g_render_ctx->GetGraphicCtx(), source.address, source.size);
index_buffer = binding.first;
index_offset = binding.second;
}
EXIT_IF(index_buffer == nullptr);
vkCmdBindIndexBuffer(vk_buffer, index_buffer->buffer, index_offset, source.type);
}
static void LogDrawStateIfNeeded(HW::Context* ctx, HW::UserConfig* ucfg, const DrawCallInfo& draw, static void LogDrawStateIfNeeded(HW::Context* ctx, HW::UserConfig* ucfg, const DrawCallInfo& draw,
const DrawRenderState& state, bool always_log, const DrawRenderState& state, bool always_log,
bool force_legacy_rect_log, uint32_t index_type_and_size, bool force_legacy_rect_log, uint32_t index_type_and_size,
@@ -898,7 +923,7 @@ static void EmitDrawPrimitives(const HW::UserConfig* ucfg, VkCommandBuffer vk_bu
static void ExecutePreparedDraw(uint64_t submit_id, CommandBuffer* buffer, HW::Context* ctx, static void ExecutePreparedDraw(uint64_t submit_id, CommandBuffer* buffer, HW::Context* ctx,
HW::UserConfig* ucfg, HW::Shader* sh_ctx, const DrawCallInfo& draw, HW::UserConfig* ucfg, HW::Shader* sh_ctx, const DrawCallInfo& draw,
DrawRenderState* state, VkPrimitiveTopology topology, DrawRenderState* state, VkPrimitiveTopology topology,
const DrawEmitInfo& emit, const DrawIndexBufferBind& index_bind, const DrawEmitInfo& emit, const DrawIndexBufferSource& index_source,
bool log_pipeline_phase, bool set_bind_debug, bool set_auto_debug) { bool log_pipeline_phase, bool set_bind_debug, bool set_auto_debug) {
EXIT_IF(buffer == nullptr); EXIT_IF(buffer == nullptr);
EXIT_IF(ctx == nullptr); EXIT_IF(ctx == nullptr);
@@ -907,86 +932,93 @@ static void ExecutePreparedDraw(uint64_t submit_id, CommandBuffer* buffer, HW::C
EXIT_IF(draw.name == nullptr); EXIT_IF(draw.name == nullptr);
EXIT_IF(state == nullptr); EXIT_IF(state == nullptr);
if (log_pipeline_phase) { for (;;) {
LogDrawPhase(draw.name, "CreatePipeline"); const auto recording_generation = buffer->GetRecordingGeneration();
} if (log_pipeline_phase) {
auto* pipeline = g_render_ctx->GetPipelineCache()->CreateGraphicsPipeline( LogDrawPhase(draw.name, "CreatePipeline");
state->framebuffer, state->color_info, state->color_count, &state->depth_info, }
&state->vs_input_info, ctx, sh_ctx, &state->ps_input_info, topology, state->ps_active, auto* pipeline = g_render_ctx->GetPipelineCache()->CreateGraphicsPipeline(
state->vs_shader, state->ps_shader); state->framebuffer, state->color_info, state->color_count, &state->depth_info,
&state->vs_input_info, ctx, sh_ctx, &state->ps_input_info, topology, state->ps_active,
state->vs_shader, state->ps_shader);
if (set_bind_debug) { if (set_bind_debug) {
SetDrawDebugPhase(buffer, submit_id, draw, 0x100u); SetDrawDebugPhase(buffer, submit_id, draw, 0x100u);
} }
vkCmdBindPipeline(state->vk_buffer, VK_PIPELINE_BIND_POINT_GRAPHICS, pipeline->pipeline); vkCmdBindPipeline(state->vk_buffer, VK_PIPELINE_BIND_POINT_GRAPHICS, pipeline->pipeline);
const auto dynamic_params = const auto dynamic_params = BuildGraphicsDynamicParams(
BuildGraphicsDynamicParams(*ctx, state->color_info, state->color_count, state->depth_info); *ctx, state->color_info, state->color_count, state->depth_info);
SetDynamicParams(state->vk_buffer, dynamic_params); SetDynamicParams(state->vk_buffer, dynamic_params);
// EXIT_NOT_IMPLEMENTED(vs_input_info.buffers_num > 1); // EXIT_NOT_IMPLEMENTED(vs_input_info.buffers_num > 1);
BindDrawVertexBuffers(submit_id, buffer, draw, state->vk_buffer, state->vs_input_info); BindDrawVertexBuffers(submit_id, buffer, draw, state->vk_buffer, state->vs_input_info);
LogDrawPhase(draw.name, "BindDescriptorsVS"); LogDrawPhase(draw.name, "BindDescriptorsVS");
if (set_auto_debug) {
SetDrawDebugPhase(buffer, submit_id, draw, 0x200u);
}
const auto vs_address_writes = BindDescriptors(
submit_id, buffer, VK_PIPELINE_BIND_POINT_GRAPHICS, pipeline->pipeline_layout,
state->vs_input_info.stage, VK_SHADER_STAGE_VERTEX_BIT, DescriptorCache::Stage::Vertex);
std::vector<ShaderAddressWriteRange> ps_address_writes;
if (state->ps_active) {
LogDrawPhase(draw.name, "BindDescriptorsPS");
if (set_auto_debug) { if (set_auto_debug) {
SetDrawDebugPhase(buffer, submit_id, draw, 0x300u); SetDrawDebugPhase(buffer, submit_id, draw, 0x200u);
} }
ps_address_writes = const auto vs_address_writes = BindDescriptors(
BindDescriptors(submit_id, buffer, VK_PIPELINE_BIND_POINT_GRAPHICS, submit_id, buffer, VK_PIPELINE_BIND_POINT_GRAPHICS, pipeline->pipeline_layout,
pipeline->pipeline_layout, state->ps_input_info.stage, state->vs_input_info.stage, VK_SHADER_STAGE_VERTEX_BIT, DescriptorCache::Stage::Vertex);
VK_SHADER_STAGE_FRAGMENT_BIT, DescriptorCache::Stage::Pixel);
}
if (index_bind.enabled) { std::vector<ShaderAddressWriteRange> ps_address_writes;
EXIT_NOT_IMPLEMENTED(index_bind.buffer == nullptr); if (state->ps_active) {
vkCmdBindIndexBuffer(state->vk_buffer, index_bind.buffer->buffer, index_bind.offset, LogDrawPhase(draw.name, "BindDescriptorsPS");
index_bind.type); if (set_auto_debug) {
} SetDrawDebugPhase(buffer, submit_id, draw, 0x300u);
}
LogDrawPhase(draw.name, "BeginRenderPass"); ps_address_writes =
if (set_auto_debug) { BindDescriptors(submit_id, buffer, VK_PIPELINE_BIND_POINT_GRAPHICS,
SetDrawDebugPhase(buffer, submit_id, draw, 0x400u); pipeline->pipeline_layout, state->ps_input_info.stage,
} VK_SHADER_STAGE_FRAGMENT_BIT, DescriptorCache::Stage::Pixel);
buffer->BeginRenderPass(state->framebuffer, state->color_info, state->color_count,
&state->depth_info);
if (set_auto_debug) {
SetDrawDebugPhase(buffer, submit_id, draw, 0x500u);
}
EmitDrawPrimitives(ucfg, state->vk_buffer, state->vs_input_info, draw, emit);
if (set_auto_debug) {
SetDrawDebugPhase(buffer, submit_id, draw, 0x600u);
}
buffer->EndRenderPass();
VkPipelineStageFlags shader_write_stages = 0;
const bool vs_wrote_buffers = HasShaderBufferWrites(state->vs_input_info.stage);
const bool vs_wrote_addresses = MarkShaderAddressWrites(vs_address_writes);
if (vs_wrote_buffers || vs_wrote_addresses) {
shader_write_stages |= VK_PIPELINE_STAGE_VERTEX_SHADER_BIT;
}
if (state->ps_active) {
const bool ps_wrote_buffers = HasShaderBufferWrites(state->ps_input_info.stage);
const bool ps_wrote_addresses = MarkShaderAddressWrites(ps_address_writes);
if (ps_wrote_buffers || ps_wrote_addresses) {
shader_write_stages |= VK_PIPELINE_STAGE_FRAGMENT_SHADER_BIT;
} }
} if (buffer->GetRecordingGeneration() != recording_generation) {
if (shader_write_stages != 0) { continue;
ShaderWriteBarrier(state->vk_buffer, shader_write_stages); }
} // Index data may use this command buffer's host stream. Resolve it only after the other
LogDrawPhase(draw.name, "EndRenderPass"); // fault-capable bindings, and rebuild it whenever a fault reset changes the generation.
if (set_auto_debug) { BindDrawIndexBuffer(buffer, state->vk_buffer, index_source);
SetDrawDebugPhase(buffer, submit_id, draw, 0x700u); if (buffer->GetRecordingGeneration() != recording_generation) {
continue;
}
LogDrawPhase(draw.name, "BeginRenderPass");
if (set_auto_debug) {
SetDrawDebugPhase(buffer, submit_id, draw, 0x400u);
}
buffer->BeginRenderPass(state->framebuffer, state->color_info, state->color_count,
&state->depth_info);
if (set_auto_debug) {
SetDrawDebugPhase(buffer, submit_id, draw, 0x500u);
}
EmitDrawPrimitives(ucfg, state->vk_buffer, state->vs_input_info, draw, emit);
if (set_auto_debug) {
SetDrawDebugPhase(buffer, submit_id, draw, 0x600u);
}
buffer->EndRenderPass();
VkPipelineStageFlags shader_write_stages = 0;
const bool vs_wrote_buffers = HasShaderBufferWrites(state->vs_input_info.stage);
const bool vs_wrote_addresses = MarkShaderAddressWrites(vs_address_writes);
if (vs_wrote_buffers || vs_wrote_addresses) {
shader_write_stages |= VK_PIPELINE_STAGE_VERTEX_SHADER_BIT;
}
if (state->ps_active) {
const bool ps_wrote_buffers = HasShaderBufferWrites(state->ps_input_info.stage);
const bool ps_wrote_addresses = MarkShaderAddressWrites(ps_address_writes);
if (ps_wrote_buffers || ps_wrote_addresses) {
shader_write_stages |= VK_PIPELINE_STAGE_FRAGMENT_SHADER_BIT;
}
}
if (shader_write_stages != 0) {
ShaderWriteBarrier(state->vk_buffer, shader_write_stages);
}
LogDrawPhase(draw.name, "EndRenderPass");
if (set_auto_debug) {
SetDrawDebugPhase(buffer, submit_id, draw, 0x700u);
}
break;
} }
} }
@@ -1068,7 +1100,7 @@ void RenderDrawIndex(uint64_t submit_id, CommandBuffer* buffer, HW::Context* ctx
index_type = VK_INDEX_TYPE_UINT32; index_type = VK_INDEX_TYPE_UINT32;
index_size = 4 * static_cast<uint64_t>(index_count); index_size = 4 * static_cast<uint64_t>(index_count);
break; break;
// Some games use it // Some games use it - need vulkan extension
case Prospero::IndexType::kIndex8: case Prospero::IndexType::kIndex8:
index_type = VK_INDEX_TYPE_UINT16; index_type = VK_INDEX_TYPE_UINT16;
index_size = static_cast<uint64_t>(index_count); index_size = static_cast<uint64_t>(index_count);
@@ -1086,6 +1118,25 @@ void RenderDrawIndex(uint64_t submit_id, CommandBuffer* buffer, HW::Context* ctx
const DrawCallInfo draw {"DrawIndex", CommandBufferDebugOp::DrawIndex, const DrawCallInfo draw {"DrawIndex", CommandBufferDebugOp::DrawIndex,
index_count, flags, index_count, flags,
instance_count, first_instance}; instance_count, first_instance};
std::vector<uint16_t> expanded_indices;
if (expand_index8_to_u16) {
EXIT_NOT_IMPLEMENTED(index_addr == nullptr);
const auto* src = static_cast<const uint8_t*>(index_addr);
expanded_indices.resize(index_count);
for (uint32_t i = 0; i < index_count; i++) {
expanded_indices[i] = src[i];
}
}
DrawIndexBufferSource index_source {};
index_source.enabled = true;
index_source.address = reinterpret_cast<uint64_t>(index_addr);
index_source.host_data =
expanded_indices.empty() ? nullptr : static_cast<const void*>(expanded_indices.data());
index_source.size = expanded_indices.empty()
? index_size
: expanded_indices.size() * sizeof(uint16_t);
index_source.type = index_type;
DrawRenderState state {}; DrawRenderState state {};
if (!PrepareDrawRenderState(submit_id, buffer, ctx, ucfg, sh_ctx, draw, if (!PrepareDrawRenderState(submit_id, buffer, ctx, ucfg, sh_ctx, draw,
@@ -1100,43 +1151,12 @@ void RenderDrawIndex(uint64_t submit_id, CommandBuffer* buffer, HW::Context* ctx
const auto vertex_offset = const auto vertex_offset =
ResolveVertexOffset(ucfg->GetIndexOffset(), state.vs_input_info) + vertex_offset_add; ResolveVertexOffset(ucfg->GetIndexOffset(), state.vs_input_info) + vertex_offset_add;
VkDeviceSize index_offset = 0;
VulkanBuffer* indices = nullptr;
if (expand_index8_to_u16) {
EXIT_NOT_IMPLEMENTED(index_addr == nullptr);
const auto* src = static_cast<const uint8_t*>(index_addr);
std::vector<uint16_t> expanded_indices(index_count);
for (uint32_t i = 0; i < index_count; i++) {
expanded_indices[i] = src[i];
}
VkDeviceSize range = 0;
if (!g_render_ctx->GetBufferCache()->UploadHostData(
buffer, g_render_ctx->GetGraphicCtx(), expanded_indices.data(),
expanded_indices.size() * sizeof(uint16_t), 16, &indices, &index_offset, &range)) {
EXIT("failed to upload expanded 8-bit index buffer\n");
}
} else {
auto binding = g_render_ctx->GetBufferCache()->ObtainBuffer(
buffer, g_render_ctx->GetGraphicCtx(), reinterpret_cast<uint64_t>(index_addr),
index_size);
indices = binding.first;
index_offset = binding.second;
}
EXIT_NOT_IMPLEMENTED(indices == nullptr);
DrawIndexBufferBind index_bind {};
index_bind.enabled = true;
index_bind.buffer = indices;
index_bind.offset = index_offset;
index_bind.type = index_type;
DrawEmitInfo emit {}; DrawEmitInfo emit {};
emit.indexed = true; emit.indexed = true;
emit.vertex_offset = vertex_offset; emit.vertex_offset = vertex_offset;
ExecutePreparedDraw(submit_id, buffer, ctx, ucfg, sh_ctx, draw, &state, topology, emit, ExecutePreparedDraw(submit_id, buffer, ctx, ucfg, sh_ctx, draw, &state, topology, emit,
index_bind, true, true, false); index_source, true, true, false);
} }
// NOLINTNEXTLINE(readability-function-cognitive-complexity) // NOLINTNEXTLINE(readability-function-cognitive-complexity)
@@ -1243,9 +1263,9 @@ void RenderDrawIndexAuto(uint64_t submit_id, CommandBuffer* buffer, HW::Context*
emit.draw_vertex_count = draw_vertex_count; emit.draw_vertex_count = draw_vertex_count;
emit.first_vertex = static_cast<uint32_t>(vertex_offset); emit.first_vertex = static_cast<uint32_t>(vertex_offset);
DrawIndexBufferBind index_bind {}; DrawIndexBufferSource index_source {};
ExecutePreparedDraw(submit_id, buffer, ctx, ucfg, sh_ctx, draw, &state, topology, emit, ExecutePreparedDraw(submit_id, buffer, ctx, ucfg, sh_ctx, draw, &state, topology, emit,
index_bind, false, false, true); index_source, false, false, true);
} }
bool IsSameColorResolveSubresource(const RenderColorInfo& src, const RenderColorInfo& dst) { bool IsSameColorResolveSubresource(const RenderColorInfo& src, const RenderColorInfo& dst) {
src/graphics/host_gpu/renderer/resourceMutex.h
+4 -4
View File
@@ -8,8 +8,8 @@
namespace Libs::Graphics { namespace Libs::Graphics {
// Owner-tracked shared buffer/image transaction. External faults pause GPU submissions before // Owner-tracked shared buffer/image transaction. External faults pause GPU submissions first;
// taking this mutex; synchronous command-processor accesses reuse their already-owned transaction. // command-processor faults drain their queue before entering this transaction.
class ResourceMutex final { class ResourceMutex final {
public: public:
class FaultScope final { class FaultScope final {
@@ -27,8 +27,8 @@ public:
~ResourceMutex(); ~ResourceMutex();
KYTY_CLASS_NO_COPY(ResourceMutex); KYTY_CLASS_NO_COPY(ResourceMutex);
void lock(); void lock();
void unlock(); void unlock();
[[nodiscard]] bool IsOwnedByCurrentThread(); [[nodiscard]] bool IsOwnedByCurrentThread();
private: private:
src/graphics/host_gpu/renderer/sync.cpp
+30 -105
View File
@@ -13,31 +13,29 @@
#include "kernel/pthread.h" #include "kernel/pthread.h"
#include "libs/errno.h" #include "libs/errno.h"
#include <cstring>
#include <limits> #include <limits>
namespace Libs::Graphics { namespace Libs::Graphics {
constexpr int GRAPHICS_EVENT_QUEUED_GRAPHICS_INTERRUPT = 0x00; constexpr int GRAPHICS_EVENT_QUEUED_GRAPHICS_INTERRUPT = 0x00;
constexpr int GRAPHICS_EVENT_EOP = 0x40; constexpr int GRAPHICS_EVENT_EOP = 0x40;
constexpr uint64_t GRAPHICS_REFERENCE_CLOCK_FREQUENCY = 100000000; constexpr uint64_t GRAPHICS_REFERENCE_CLOCK_FREQUENCY = 100000000;
bool GraphicsScaleReferenceClock(uint64_t host_ticks, uint64_t host_frequency, uint64_t* value) { bool GraphicsScaleReferenceClock(uint64_t host_ticks, uint64_t host_frequency, uint64_t* value) {
if (host_frequency == 0 || value == nullptr) { if (host_frequency == 0 || value == nullptr) {
return false; return false;
} }
const auto whole_seconds = host_ticks / host_frequency; const auto whole_seconds = host_ticks / host_frequency;
const auto remainder = host_ticks % host_frequency; const auto remainder = host_ticks % host_frequency;
constexpr auto MAX_VALUE = std::numeric_limits<uint64_t>::max(); constexpr auto MAX_VALUE = std::numeric_limits<uint64_t>::max();
if (whole_seconds > MAX_VALUE / GRAPHICS_REFERENCE_CLOCK_FREQUENCY || if (whole_seconds > MAX_VALUE / GRAPHICS_REFERENCE_CLOCK_FREQUENCY ||
remainder > MAX_VALUE / GRAPHICS_REFERENCE_CLOCK_FREQUENCY) { remainder > MAX_VALUE / GRAPHICS_REFERENCE_CLOCK_FREQUENCY) {
return false; return false;
} }
const auto whole_value = whole_seconds * GRAPHICS_REFERENCE_CLOCK_FREQUENCY; const auto whole_value = whole_seconds * GRAPHICS_REFERENCE_CLOCK_FREQUENCY;
const auto fractional_value = const auto fractional_value = (remainder * GRAPHICS_REFERENCE_CLOCK_FREQUENCY) / host_frequency;
(remainder * GRAPHICS_REFERENCE_CLOCK_FREQUENCY) / host_frequency;
if (whole_value > MAX_VALUE - fractional_value) { if (whole_value > MAX_VALUE - fractional_value) {
return false; return false;
} }
@@ -50,40 +48,19 @@ uint64_t GraphicsRenderReadReferenceClock() {
const auto host_ticks = LibKernel::KernelReadTsc(); const auto host_ticks = LibKernel::KernelReadTsc();
uint64_t value = 0; uint64_t value = 0;
if (!GraphicsScaleReferenceClock(host_ticks, host_frequency, &value)) { if (!GraphicsScaleReferenceClock(host_ticks, host_frequency, &value)) {
EXIT("cannot scale host clock, ticks=0x%016" PRIx64 " frequency=%" PRIu64 "\n", EXIT("cannot scale host clock, ticks=0x%016" PRIx64 " frequency=%" PRIu64 "\n", host_ticks,
host_ticks, host_frequency); host_frequency);
} }
return value; return value;
} }
static void SubmitLabel32(CommandBuffer* buffer, uint32_t* dst, uint32_t value, static void SubmitLabel(CommandBuffer* buffer, LabelCallback callback_1 = nullptr,
LabelCallback callback_1 = nullptr, LabelCallback callback_2 = nullptr, LabelCallback callback_2 = nullptr, const uint64_t* args = nullptr) {
const uint64_t* args = nullptr) { auto* label = LabelCreate(g_render_ctx->GetGraphicCtx(), callback_1, callback_2, args);
auto* label =
LabelCreate32(g_render_ctx->GetGraphicCtx(), dst, value, callback_1, callback_2, args);
LabelSet(buffer, label); LabelSet(buffer, label);
LabelDelete(label); LabelDelete(label);
} }
static void SubmitLabel64(CommandBuffer* buffer, uint64_t* dst, uint64_t value,
LabelCallback callback_1 = nullptr, LabelCallback callback_2 = nullptr,
const uint64_t* args = nullptr) {
auto* label =
LabelCreate64(g_render_ctx->GetGraphicCtx(), dst, value, callback_1, callback_2, args);
LabelSet(buffer, label);
LabelDelete(label);
}
template <typename T>
static void PublishImmediateFence(T* dst, T value) {
static_assert(sizeof(T) == sizeof(uint32_t) || sizeof(T) == sizeof(uint64_t));
EXIT_IF(dst == nullptr);
// EOP/EOS fence payloads are constant and published while parsing the packet. Kyty performs the
// direct mapped write under CommandProcessorGuestAccessScope: a protected access is first resolved by
// BufferCache inline, then this instruction retries so the EOP payload wins over prior GPU bytes.
std::memcpy(dst, &value, sizeof(value));
}
static bool CompleteDisplayBufferFlip(const uint64_t* args) { static bool CompleteDisplayBufferFlip(const uint64_t* args) {
if (g_render_ctx == nullptr || args == nullptr) { if (g_render_ctx == nullptr || args == nullptr) {
EXIT("GPU flip completion has invalid state, render_ctx=%p args=%p\n", EXIT("GPU flip completion has invalid state, render_ctx=%p args=%p\n",
@@ -114,8 +91,6 @@ void GraphicsRenderWriteAtEndOfPipe32(uint64_t submit_id, CommandBuffer* buffer,
buffer->SetDebugInfo(static_cast<uint32_t>(CommandBufferDebugOp::EopWrite), submit_id, 4, value, buffer->SetDebugInfo(static_cast<uint32_t>(CommandBufferDebugOp::EopWrite), submit_id, 4, value,
0, 0, reinterpret_cast<uint64_t>(dst_gpu_addr)); 0, 0, reinterpret_cast<uint64_t>(dst_gpu_addr));
PublishImmediateFence(dst_gpu_addr, value);
} }
void GraphicsRenderWriteAtEndOfPipeGds32(uint64_t submit_id, CommandBuffer* buffer, void GraphicsRenderWriteAtEndOfPipeGds32(uint64_t submit_id, CommandBuffer* buffer,
@@ -128,28 +103,6 @@ void GraphicsRenderWriteAtEndOfPipeGds32(uint64_t submit_id, CommandBuffer* buff
buffer->SetDebugInfo(static_cast<uint32_t>(CommandBufferDebugOp::EopWrite), submit_id, buffer->SetDebugInfo(static_cast<uint32_t>(CommandBufferDebugOp::EopWrite), submit_id,
dw_offset, dw_num, 0, 0, reinterpret_cast<uint64_t>(dst_gpu_addr)); dw_offset, dw_num, 0, 0, reinterpret_cast<uint64_t>(dst_gpu_addr));
uint64_t args[LABEL_ARGS_MAX] = {static_cast<uint64_t>(dw_offset),
static_cast<uint64_t>(dw_num),
reinterpret_cast<uint64_t>(dst_gpu_addr), 0};
SubmitLabel32(
buffer, dst_gpu_addr, 0,
[](const uint64_t* args) {
auto dw_offset = static_cast<uint32_t>(args[0]);
auto dw_num = static_cast<uint32_t>(args[1]);
auto* dst_gpu_addr = reinterpret_cast<uint32_t*>(args[2]);
if (dst_gpu_addr == nullptr || dw_num == 0) {
EXIT("invalid asynchronous EOP GDS write\n");
}
const auto size = static_cast<uint64_t>(dw_num) * sizeof(uint32_t);
auto transaction = g_render_ctx->GetGpuResources()->LockAsynchronousGuestWrite(
reinterpret_cast<uint64_t>(dst_gpu_addr), size);
g_render_ctx->GetGdsBuffer()->Read(g_render_ctx->GetGraphicCtx(), dst_gpu_addr,
dw_offset, dw_num);
return false;
},
nullptr, args);
} }
void GraphicsRenderWriteAtEndOfPipe64(uint64_t submit_id, CommandBuffer* buffer, void GraphicsRenderWriteAtEndOfPipe64(uint64_t submit_id, CommandBuffer* buffer,
@@ -162,12 +115,10 @@ void GraphicsRenderWriteAtEndOfPipe64(uint64_t submit_id, CommandBuffer* buffer,
buffer->SetDebugInfo(static_cast<uint32_t>(CommandBufferDebugOp::EopWrite), submit_id, 8, buffer->SetDebugInfo(static_cast<uint32_t>(CommandBufferDebugOp::EopWrite), submit_id, 8,
static_cast<uint32_t>(value), static_cast<uint32_t>(value >> 32u), 0, static_cast<uint32_t>(value), static_cast<uint32_t>(value >> 32u), 0,
reinterpret_cast<uint64_t>(dst_gpu_addr)); reinterpret_cast<uint64_t>(dst_gpu_addr));
PublishImmediateFence(dst_gpu_addr, value);
} }
void GraphicsRenderWriteAtEndOfPipeClockCounter(uint64_t submit_id, CommandBuffer* buffer, void GraphicsRenderWriteAtEndOfPipeClockCounter(uint64_t submit_id, CommandBuffer* buffer,
uint64_t* dst_gpu_addr) { uint64_t* dst_gpu_addr, uint64_t value) {
EXIT_IF(g_render_ctx == nullptr); EXIT_IF(g_render_ctx == nullptr);
EXIT_IF(dst_gpu_addr == nullptr); EXIT_IF(dst_gpu_addr == nullptr);
EXIT_IF(buffer == nullptr); EXIT_IF(buffer == nullptr);
@@ -176,8 +127,6 @@ void GraphicsRenderWriteAtEndOfPipeClockCounter(uint64_t submit_id, CommandBuffe
buffer->SetDebugInfo(static_cast<uint32_t>(CommandBufferDebugOp::EopWrite), submit_id, 8, 0, 0, buffer->SetDebugInfo(static_cast<uint32_t>(CommandBufferDebugOp::EopWrite), submit_id, 8, 0, 0,
0, reinterpret_cast<uint64_t>(dst_gpu_addr)); 0, reinterpret_cast<uint64_t>(dst_gpu_addr));
const auto value = GraphicsRenderReadReferenceClock();
PublishImmediateFence(dst_gpu_addr, value);
LOGF_COLOR(Log::Color::BrightGreen, LOGF_COLOR(Log::Color::BrightGreen,
"EndOfPipe Signal!!! [0x%016" PRIx64 "] <- Clock: 0x%016" PRIx64 "\n", "EndOfPipe Signal!!! [0x%016" PRIx64 "] <- Clock: 0x%016" PRIx64 "\n",
reinterpret_cast<uint64_t>(dst_gpu_addr), value); reinterpret_cast<uint64_t>(dst_gpu_addr), value);
@@ -185,7 +134,8 @@ void GraphicsRenderWriteAtEndOfPipeClockCounter(uint64_t submit_id, CommandBuffe
void GraphicsRenderWriteAtEndOfPipeClockCounterWithWriteBack(uint64_t submit_id, void GraphicsRenderWriteAtEndOfPipeClockCounterWithWriteBack(uint64_t submit_id,
CommandBuffer* buffer, CommandBuffer* buffer,
uint64_t* dst_gpu_addr) { uint64_t* dst_gpu_addr,
uint64_t value) {
EXIT_IF(g_render_ctx == nullptr); EXIT_IF(g_render_ctx == nullptr);
EXIT_IF(dst_gpu_addr == nullptr); EXIT_IF(dst_gpu_addr == nullptr);
EXIT_IF(buffer == nullptr); EXIT_IF(buffer == nullptr);
@@ -194,8 +144,6 @@ void GraphicsRenderWriteAtEndOfPipeClockCounterWithWriteBack(uint64_t subm
buffer->SetDebugInfo(static_cast<uint32_t>(CommandBufferDebugOp::EopWriteBack), submit_id, 8, 0, buffer->SetDebugInfo(static_cast<uint32_t>(CommandBufferDebugOp::EopWriteBack), submit_id, 8, 0,
0, 0, reinterpret_cast<uint64_t>(dst_gpu_addr)); 0, 0, reinterpret_cast<uint64_t>(dst_gpu_addr));
const auto value = GraphicsRenderReadReferenceClock();
PublishImmediateFence(dst_gpu_addr, value);
LOGF_COLOR(Log::Color::BrightGreen, LOGF_COLOR(Log::Color::BrightGreen,
"EndOfPipe Signal!!! [0x%016" PRIx64 "] <- Clock: 0x%016" PRIx64 "\n", "EndOfPipe Signal!!! [0x%016" PRIx64 "] <- Clock: 0x%016" PRIx64 "\n",
reinterpret_cast<uint64_t>(dst_gpu_addr), value); reinterpret_cast<uint64_t>(dst_gpu_addr), value);
@@ -211,8 +159,6 @@ void GraphicsRenderWriteAtEndOfPipeWithWriteBack64(uint64_t submit_id, CommandBu
buffer->SetDebugInfo(static_cast<uint32_t>(CommandBufferDebugOp::EopWriteBack), submit_id, 8, buffer->SetDebugInfo(static_cast<uint32_t>(CommandBufferDebugOp::EopWriteBack), submit_id, 8,
static_cast<uint32_t>(value), static_cast<uint32_t>(value >> 32u), 0, static_cast<uint32_t>(value), static_cast<uint32_t>(value >> 32u), 0,
reinterpret_cast<uint64_t>(dst_gpu_addr)); reinterpret_cast<uint64_t>(dst_gpu_addr));
PublishImmediateFence(dst_gpu_addr, value);
} }
void GraphicsRenderWriteAtEndOfPipeWithWriteBack32(uint64_t submit_id, CommandBuffer* buffer, void GraphicsRenderWriteAtEndOfPipeWithWriteBack32(uint64_t submit_id, CommandBuffer* buffer,
@@ -224,8 +170,6 @@ void GraphicsRenderWriteAtEndOfPipeWithWriteBack32(uint64_t submit_id, CommandBu
buffer->SetDebugInfo(static_cast<uint32_t>(CommandBufferDebugOp::EopWriteBack), submit_id, 4, buffer->SetDebugInfo(static_cast<uint32_t>(CommandBufferDebugOp::EopWriteBack), submit_id, 4,
value, 0, 0, reinterpret_cast<uint64_t>(dst_gpu_addr)); value, 0, 0, reinterpret_cast<uint64_t>(dst_gpu_addr));
PublishImmediateFence(dst_gpu_addr, value);
} }
void GraphicsRenderWriteAtEndOfPipeWithInterruptWriteBack64(uint64_t submit_id, void GraphicsRenderWriteAtEndOfPipeWithInterruptWriteBack64(uint64_t submit_id,
@@ -241,12 +185,10 @@ void GraphicsRenderWriteAtEndOfPipeWithInterruptWriteBack64(uint64_t submi
context_id, static_cast<uint32_t>(value), context_id, static_cast<uint32_t>(value),
static_cast<uint32_t>(value >> 32u), static_cast<uint32_t>(value >> 32u),
reinterpret_cast<uint64_t>(dst_gpu_addr)); reinterpret_cast<uint64_t>(dst_gpu_addr));
PublishImmediateFence(dst_gpu_addr, value);
uint64_t args[LABEL_ARGS_MAX] = {context_id != 0 ? context_id : value}; uint64_t args[LABEL_ARGS_MAX] = {context_id != 0 ? context_id : value};
SubmitLabel64( SubmitLabel(
buffer, nullptr, 0, nullptr, buffer, nullptr,
[](const uint64_t* args) { [](const uint64_t* args) {
EXIT_IF(g_render_ctx == nullptr); EXIT_IF(g_render_ctx == nullptr);
g_render_ctx->TriggerEopEvent(static_cast<uint32_t>(args[0])); g_render_ctx->TriggerEopEvent(static_cast<uint32_t>(args[0]));
@@ -266,12 +208,10 @@ void GraphicsRenderWriteAtEndOfPipeWithInterruptWriteBack32(uint64_t submi
buffer->SetDebugInfo(static_cast<uint32_t>(CommandBufferDebugOp::EopWriteBack), submit_id, 4, buffer->SetDebugInfo(static_cast<uint32_t>(CommandBufferDebugOp::EopWriteBack), submit_id, 4,
context_id, value, 0, reinterpret_cast<uint64_t>(dst_gpu_addr)); context_id, value, 0, reinterpret_cast<uint64_t>(dst_gpu_addr));
PublishImmediateFence(dst_gpu_addr, value);
uint64_t args[LABEL_ARGS_MAX] = {context_id != 0 ? context_id : value}; uint64_t args[LABEL_ARGS_MAX] = {context_id != 0 ? context_id : value};
SubmitLabel32( SubmitLabel(
buffer, nullptr, 0, nullptr, buffer, nullptr,
[](const uint64_t* args) { [](const uint64_t* args) {
EXIT_IF(g_render_ctx == nullptr); EXIT_IF(g_render_ctx == nullptr);
g_render_ctx->TriggerEopEvent(static_cast<uint32_t>(args[0])); g_render_ctx->TriggerEopEvent(static_cast<uint32_t>(args[0]));
@@ -292,12 +232,10 @@ void GraphicsRenderWriteAtEndOfPipeWithInterrupt64(uint64_t submit_id, CommandBu
context_id, static_cast<uint32_t>(value), context_id, static_cast<uint32_t>(value),
static_cast<uint32_t>(value >> 32u), static_cast<uint32_t>(value >> 32u),
reinterpret_cast<uint64_t>(dst_gpu_addr)); reinterpret_cast<uint64_t>(dst_gpu_addr));
PublishImmediateFence(dst_gpu_addr, value);
uint64_t args[LABEL_ARGS_MAX] = {context_id != 0 ? context_id : value}; uint64_t args[LABEL_ARGS_MAX] = {context_id != 0 ? context_id : value};
SubmitLabel64( SubmitLabel(
buffer, nullptr, 0, nullptr, buffer, nullptr,
[](const uint64_t* args) { [](const uint64_t* args) {
EXIT_IF(g_render_ctx == nullptr); EXIT_IF(g_render_ctx == nullptr);
g_render_ctx->TriggerEopEvent(static_cast<uint32_t>(args[0])); g_render_ctx->TriggerEopEvent(static_cast<uint32_t>(args[0]));
@@ -316,12 +254,10 @@ void GraphicsRenderWriteAtEndOfPipeWithInterrupt32(uint64_t submit_id, CommandBu
buffer->SetDebugInfo(static_cast<uint32_t>(CommandBufferDebugOp::EopInterrupt), submit_id, 4, buffer->SetDebugInfo(static_cast<uint32_t>(CommandBufferDebugOp::EopInterrupt), submit_id, 4,
context_id, value, 0, reinterpret_cast<uint64_t>(dst_gpu_addr)); context_id, value, 0, reinterpret_cast<uint64_t>(dst_gpu_addr));
PublishImmediateFence(dst_gpu_addr, value);
uint64_t args[LABEL_ARGS_MAX] = {context_id != 0 ? context_id : value}; uint64_t args[LABEL_ARGS_MAX] = {context_id != 0 ? context_id : value};
SubmitLabel32( SubmitLabel(
buffer, nullptr, 0, nullptr, buffer, nullptr,
[](const uint64_t* args) { [](const uint64_t* args) {
EXIT_IF(g_render_ctx == nullptr); EXIT_IF(g_render_ctx == nullptr);
g_render_ctx->TriggerEopEvent(static_cast<uint32_t>(args[0])); g_render_ctx->TriggerEopEvent(static_cast<uint32_t>(args[0]));
@@ -360,11 +296,9 @@ void GraphicsRenderWriteAtEndOfPipeWithInterruptWriteBackFlip32(
buffer->SetDebugInfo(static_cast<uint32_t>(CommandBufferDebugOp::EopWriteBackFlip), submit_id, buffer->SetDebugInfo(static_cast<uint32_t>(CommandBufferDebugOp::EopWriteBackFlip), submit_id,
static_cast<uint32_t>(handle), static_cast<uint32_t>(index), static_cast<uint32_t>(handle), static_cast<uint32_t>(index),
static_cast<uint32_t>(flip_mode), value, static_cast<uint64_t>(flip_arg)); static_cast<uint32_t>(flip_mode), value, static_cast<uint64_t>(flip_arg));
PublishImmediateFence(dst_gpu_addr, value);
const uint64_t args[LABEL_ARGS_MAX] = {request_id}; const uint64_t args[LABEL_ARGS_MAX] = {request_id};
SubmitLabel32( SubmitLabel(
buffer, nullptr, 0, CompleteDisplayBufferFlip, buffer, CompleteDisplayBufferFlip,
[](const uint64_t* /*args*/) { [](const uint64_t* /*args*/) {
EXIT_IF(g_render_ctx == nullptr); EXIT_IF(g_render_ctx == nullptr);
g_render_ctx->TriggerEopEvent(0); g_render_ctx->TriggerEopEvent(0);
@@ -385,10 +319,8 @@ void GraphicsRenderWriteAtEndOfPipeWithFlip32(uint64_t submit_id, CommandBuffer*
buffer->SetDebugInfo(static_cast<uint32_t>(CommandBufferDebugOp::EopFlip), submit_id, buffer->SetDebugInfo(static_cast<uint32_t>(CommandBufferDebugOp::EopFlip), submit_id,
static_cast<uint32_t>(handle), static_cast<uint32_t>(index), static_cast<uint32_t>(handle), static_cast<uint32_t>(index),
static_cast<uint32_t>(flip_mode), value, static_cast<uint64_t>(flip_arg)); static_cast<uint32_t>(flip_mode), value, static_cast<uint64_t>(flip_arg));
PublishImmediateFence(dst_gpu_addr, value);
const uint64_t args[LABEL_ARGS_MAX] = {request_id}; const uint64_t args[LABEL_ARGS_MAX] = {request_id};
SubmitLabel32(buffer, nullptr, 0, CompleteDisplayBufferFlip, nullptr, args); SubmitLabel(buffer, CompleteDisplayBufferFlip, nullptr, args);
} }
void GraphicsRenderWriteAtEndOfPipeOnlyFlip(uint64_t submit_id, CommandBuffer* buffer, int handle, void GraphicsRenderWriteAtEndOfPipeOnlyFlip(uint64_t submit_id, CommandBuffer* buffer, int handle,
@@ -403,7 +335,7 @@ void GraphicsRenderWriteAtEndOfPipeOnlyFlip(uint64_t submit_id, CommandBuffer* b
static_cast<uint32_t>(flip_mode), 0, static_cast<uint64_t>(flip_arg)); static_cast<uint32_t>(flip_mode), 0, static_cast<uint64_t>(flip_arg));
const uint64_t args[LABEL_ARGS_MAX] = {request_id}; const uint64_t args[LABEL_ARGS_MAX] = {request_id};
SubmitLabel32(buffer, nullptr, 0, CompleteDisplayBufferFlip, nullptr, args); SubmitLabel(buffer, CompleteDisplayBufferFlip, nullptr, args);
} }
void GraphicsRenderTriggerEopEventAtEndOfPipe(CommandBuffer* buffer, uint32_t context_id) { void GraphicsRenderTriggerEopEventAtEndOfPipe(CommandBuffer* buffer, uint32_t context_id) {
@@ -413,8 +345,8 @@ void GraphicsRenderTriggerEopEventAtEndOfPipe(CommandBuffer* buffer, uint32_t co
uint64_t args[LABEL_ARGS_MAX] = {static_cast<uint64_t>(context_id)}; uint64_t args[LABEL_ARGS_MAX] = {static_cast<uint64_t>(context_id)};
SubmitLabel32( SubmitLabel(
buffer, nullptr, 0, nullptr, buffer, nullptr,
[](const uint64_t* args) { [](const uint64_t* args) {
EXIT_IF(g_render_ctx == nullptr); EXIT_IF(g_render_ctx == nullptr);
g_render_ctx->TriggerEopEvent(static_cast<uint32_t>(args[0])); g_render_ctx->TriggerEopEvent(static_cast<uint32_t>(args[0]));
@@ -489,13 +421,6 @@ int GraphicsRenderDeleteEqEvent(LibKernel::EventQueue::KernelEqueue eq, int id)
return result; return result;
} }
void GraphicsRenderClearGds(uint64_t dw_offset, uint32_t dw_num, uint32_t clear_value) {
EXIT_IF(g_render_ctx == nullptr);
g_render_ctx->GetGdsBuffer()->Clear(g_render_ctx->GetGraphicCtx(), dw_offset, dw_num,
clear_value);
}
void GraphicsRenderReadGds(uint32_t* dst, uint32_t dw_offset, uint32_t dw_size) { void GraphicsRenderReadGds(uint32_t* dst, uint32_t dw_offset, uint32_t dw_size) {
EXIT_IF(g_render_ctx == nullptr); EXIT_IF(g_render_ctx == nullptr);
src/graphics/host_gpu/renderer/textureCache.cpp
+25 -41
View File
@@ -8,7 +8,6 @@
#include "graphics/guest_gpu/graphicsRun.h" #include "graphics/guest_gpu/graphicsRun.h"
#include "graphics/guest_gpu/tile.h" #include "graphics/guest_gpu/tile.h"
#include "graphics/host_gpu/graphicContext.h" #include "graphics/host_gpu/graphicContext.h"
#include "graphics/host_gpu/hostMemory.h"
#include "graphics/host_gpu/objects/label.h" #include "graphics/host_gpu/objects/label.h"
#include "graphics/host_gpu/objects/textureCommon.h" #include "graphics/host_gpu/objects/textureCommon.h"
#include "graphics/host_gpu/renderer/bufferCache.h" #include "graphics/host_gpu/renderer/bufferCache.h"
@@ -232,14 +231,15 @@ struct TextureCache::ReadbackWorker {
} }
void Request(PageFaultAccess fault_access, uint64_t fault_vaddr, uint64_t fault_size) noexcept { void Request(PageFaultAccess fault_access, uint64_t fault_vaddr, uint64_t fault_size) noexcept {
const bool submissions_prepaused_now = GraphicsRunSubmissionLockHeld(); const bool command_thread = GraphicsRunIsCommandProcessorThread();
const bool submissions_prepaused_now = GraphicsRunSubmissionLockHeld() || command_thread;
const bool unsafe_gpu_lock = GraphicsRunGpuLockHeld() && !submissions_prepaused_now; const bool unsafe_gpu_lock = GraphicsRunGpuLockHeld() && !submissions_prepaused_now;
if ((fault_access != PageFaultAccess::Read && fault_access != PageFaultAccess::Write) || if ((fault_access != PageFaultAccess::Read && fault_access != PageFaultAccess::Write) ||
GraphicsRunIsCommandProcessorThread() || unsafe_gpu_lock || LabelInCallback() || unsafe_gpu_lock || LabelInCallback() || g_texture_cache_lock_owner != nullptr ||
g_texture_cache_lock_owner != nullptr || g_texture_fault_owner != &cache) { g_texture_fault_owner != &cache) {
EXIT("TextureCache: unsafe image readback request, access=%u command_thread=%d " EXIT("TextureCache: unsafe image readback request, access=%u command_thread=%d "
"submission_lock=%d gpu_lock=%d label_callback=%d cache_lock=%p fault_owner=%p\n", "submission_lock=%d gpu_lock=%d label_callback=%d cache_lock=%p fault_owner=%p\n",
static_cast<uint32_t>(fault_access), GraphicsRunIsCommandProcessorThread(), static_cast<uint32_t>(fault_access), command_thread,
GraphicsRunSubmissionLockHeld(), GraphicsRunGpuLockHeld(), LabelInCallback(), GraphicsRunSubmissionLockHeld(), GraphicsRunGpuLockHeld(), LabelInCallback(),
g_texture_cache_lock_owner, g_texture_fault_owner); g_texture_cache_lock_owner, g_texture_fault_owner);
} }
@@ -1557,13 +1557,12 @@ VkImageView TextureCache::GetRenderTargetStorageView(GraphicContext* c
VkImageView TextureCache::GetStorageTextureSampledView(GraphicContext* ctx, VkImageView TextureCache::GetStorageTextureSampledView(GraphicContext* ctx,
StorageTextureVulkanImage* image, StorageTextureVulkanImage* image,
const ImageInfo& info) { const ImageInfo& info) {
const auto shape = SelectStorageSampledViewShape( const auto shape =
info.type, info.depth, image != nullptr ? image->layers : 0); SelectStorageSampledViewShape(info.type, info.depth, image != nullptr ? image->layers : 0);
if (ctx == nullptr || image == nullptr || image->image == nullptr || if (ctx == nullptr || image == nullptr || image->image == nullptr ||
shape == StorageSampledViewShape::Unsupported || shape == StorageSampledViewShape::Unsupported || info.base_array != 0 ||
info.base_array != 0 || info.levels != image->mip_levels || info.levels != image->mip_levels || info.base_level >= info.levels ||
info.base_level >= info.levels || info.view_levels == 0 || info.view_levels == 0 || info.base_level + info.view_levels > info.levels) {
info.base_level + info.view_levels > info.levels) {
EXIT("TextureCache: invalid sampled view of storage texture, image=%p type=%u depth=%u" EXIT("TextureCache: invalid sampled view of storage texture, image=%p type=%u depth=%u"
" base=%u levels=%u view_levels=%u image_levels=%u base_array=%u\n", " base=%u levels=%u view_levels=%u image_levels=%u base_array=%u\n",
static_cast<const void*>(image), info.type, info.depth, info.base_level, info.levels, static_cast<const void*>(image), info.type, info.depth, info.base_level, info.levels,
@@ -1589,23 +1588,21 @@ VkImageView TextureCache::GetStorageTextureSampledView(GraphicContext*
usage.sType = VK_STRUCTURE_TYPE_IMAGE_VIEW_USAGE_CREATE_INFO; usage.sType = VK_STRUCTURE_TYPE_IMAGE_VIEW_USAGE_CREATE_INFO;
usage.usage = VK_IMAGE_USAGE_SAMPLED_BIT; usage.usage = VK_IMAGE_USAGE_SAMPLED_BIT;
VkImageViewCreateInfo create {}; VkImageViewCreateInfo create {};
create.sType = VK_STRUCTURE_TYPE_IMAGE_VIEW_CREATE_INFO; create.sType = VK_STRUCTURE_TYPE_IMAGE_VIEW_CREATE_INFO;
create.pNext = &usage; create.pNext = &usage;
create.image = image->image; create.image = image->image;
create.viewType = shape == StorageSampledViewShape::Image3D create.viewType = shape == StorageSampledViewShape::Image3D ? VK_IMAGE_VIEW_TYPE_3D
? VK_IMAGE_VIEW_TYPE_3D : shape == StorageSampledViewShape::Image2DArray ? VK_IMAGE_VIEW_TYPE_2D_ARRAY
: shape == StorageSampledViewShape::Image2DArray : VK_IMAGE_VIEW_TYPE_2D;
? VK_IMAGE_VIEW_TYPE_2D_ARRAY create.format = view_format;
: VK_IMAGE_VIEW_TYPE_2D; create.components = TextureGetComponentMapping(info.swizzle);
create.format = view_format; create.subresourceRange.aspectMask = VK_IMAGE_ASPECT_COLOR_BIT;
create.components = TextureGetComponentMapping(info.swizzle); create.subresourceRange.baseMipLevel = info.base_level;
create.subresourceRange.aspectMask = VK_IMAGE_ASPECT_COLOR_BIT; create.subresourceRange.levelCount = info.view_levels;
create.subresourceRange.baseMipLevel = info.base_level;
create.subresourceRange.levelCount = info.view_levels;
create.subresourceRange.baseArrayLayer = 0; create.subresourceRange.baseArrayLayer = 0;
create.subresourceRange.layerCount = create.subresourceRange.layerCount =
shape == StorageSampledViewShape::Image2DArray ? info.depth : 1; shape == StorageSampledViewShape::Image2DArray ? info.depth : 1;
VkImageView view = nullptr; VkImageView view = nullptr;
const auto result = vkCreateImageView(ctx->device, &create, nullptr, &view); const auto result = vkCreateImageView(ctx->device, &create, nullptr, &view);
if (result != VK_SUCCESS || view == nullptr) { if (result != VK_SUCCESS || view == nullptr) {
EXIT("TextureCache: failed to create sampled view of storage texture, result=%d" EXIT("TextureCache: failed to create sampled view of storage texture, result=%d"
@@ -1926,15 +1923,6 @@ VulkanImage* TextureCache::FindTexture(CommandBuffer* command, GraphicContext* c
cached->gpu_modified, cached->ctx == ctx); cached->gpu_modified, cached->ctx == ctx);
} }
} }
uint64_t readable = 0;
if (!HostMemoryQueryReadable(info.address, info.size, &readable) || readable < info.size) {
EXIT("TextureCache: new sampled image has unreadable guest backing, addr=0x%016" PRIx64
" size=0x%016" PRIx64 " readable=0x%016" PRIx64
" format=%u extent=%ux%ux%u pitch=%u levels=%u tile=%u type=%u\n",
info.address, info.size, readable, info.format, info.width, info.height, info.depth,
info.pitch, info.levels, info.tile, info.type);
}
m_images.reserve(m_images.size() + 1); m_images.reserve(m_images.size() + 1);
auto cached = std::make_shared<CachedImage>(); auto cached = std::make_shared<CachedImage>();
cached->kind = CachedImage::Kind::Texture; cached->kind = CachedImage::Kind::Texture;
@@ -2057,13 +2045,6 @@ StorageTextureVulkanImage* TextureCache::FindStorageTexture(CommandBuffer* com
} }
RetireStoragePageNeighbors(ctx, info); RetireStoragePageNeighbors(ctx, info);
uint64_t readable = 0;
if (!HostMemoryQueryReadable(info.address, info.size, &readable) || readable < info.size) {
EXIT("TextureCache: new storage image has unreadable guest backing, addr=0x%016" PRIx64
" size=0x%016" PRIx64 " readable=0x%016" PRIx64 "\n",
info.address, info.size, readable);
}
m_images.reserve(m_images.size() + 1); m_images.reserve(m_images.size() + 1);
auto cached = std::make_shared<CachedImage>(); auto cached = std::make_shared<CachedImage>();
cached->kind = CachedImage::Kind::StorageTexture; cached->kind = CachedImage::Kind::StorageTexture;
@@ -3639,6 +3620,9 @@ bool TextureCache::InvalidateMemory(PageFaultAccess access, uint64_t vaddr, uint
} }
if (needs_readback) { if (needs_readback) {
if (GraphicsRunIsCommandProcessorThread()) {
GraphicsRunFinishCommandProcessors();
}
m_readback->Request(access, vaddr, size); m_readback->Request(access, vaddr, size);
action = m_memory_tracker.BeginCpuFault(vaddr, size, access); action = m_memory_tracker.BeginCpuFault(vaddr, size, access);
if (action != CpuFaultAction::Download) { if (action != CpuFaultAction::Download) {
src/graphics/shader/recompiler/SrtWalker.cpp
+6 -4
View File
@@ -1,9 +1,9 @@
#include "graphics/shader/recompiler/SrtWalker.h" #include "graphics/shader/recompiler/SrtWalker.h"
#include "graphics/host_gpu/hostMemory.h"
#include "graphics/shader/recompiler/ScalarProvenance.h" #include "graphics/shader/recompiler/ScalarProvenance.h"
#include <algorithm> #include <algorithm>
#include <cstring>
#include <fmt/format.h> #include <fmt/format.h>
namespace Libs::Graphics::ShaderRecompiler::IR { namespace Libs::Graphics::ShaderRecompiler::IR {
@@ -485,12 +485,14 @@ private:
value.pc, base_aligned, relative)); value.pc, base_aligned, relative));
} }
} }
const auto read_memory = if (m_runtime.read_memory != nullptr &&
m_runtime.read_memory != nullptr ? m_runtime.read_memory : HostMemoryReadDword; !m_runtime.read_memory(m_runtime.userdata, address, result)) {
if (!read_memory(m_runtime.userdata, address, result)) {
return Fail( return Fail(
error, fmt::format("ReadConst pc=0x{:08x} failed at 0x{:016x}", value.pc, address)); error, fmt::format("ReadConst pc=0x{:08x} failed at 0x{:016x}", value.pc, address));
} }
if (m_runtime.read_memory == nullptr) {
std::memcpy(result, reinterpret_cast<const void*>(address), sizeof(*result));
}
return true; return true;
} }
src/graphics/shader/shader.cpp
-26
View File
@@ -12,7 +12,6 @@
#include "graphics/guest_gpu/graphicsRun.h" #include "graphics/guest_gpu/graphicsRun.h"
#include "graphics/guest_gpu/hardwareContext.h" #include "graphics/guest_gpu/hardwareContext.h"
#include "graphics/guest_gpu/gpu_defs.h" #include "graphics/guest_gpu/gpu_defs.h"
#include "graphics/host_gpu/hostMemory.h"
#include "graphics/shader/recompiler/ShaderDecoder.h" #include "graphics/shader/recompiler/ShaderDecoder.h"
#include "graphics/shader/recompiler/ShaderRecompiler.h" #include "graphics/shader/recompiler/ShaderRecompiler.h"
#include "graphics/shader/shaderVertexMetadata.h" #include "graphics/shader/shaderVertexMetadata.h"
@@ -785,31 +784,6 @@ static bool ShaderGetStaticInputInfoVS(const HW::VertexShaderInfo* regs,
shader_addr); shader_addr);
return false; return false;
} }
uint32_t max_semantic = 0;
for (uint32_t i = 0; i < metadata.input_semantics_count; i++) {
max_semantic =
std::max(max_semantic, static_cast<uint32_t>(metadata.input_semantics[i].semantic));
}
if (!HostMemoryRangeIsReadable(reinterpret_cast<uint64_t>(attrib),
static_cast<uint64_t>(max_semantic + 1) *
sizeof(uint32_t))) {
LOGF("ShaderGetInputInfoVS(): unreadable vertex attribute table shader=0x%016" PRIx64
"\n",
shader_addr);
return false;
}
uint32_t max_buffer = 0;
for (uint32_t i = 0; i < metadata.input_semantics_count; i++) {
max_buffer = std::max(max_buffer, attrib[metadata.input_semantics[i].semantic] & 0x1fu);
}
if (!HostMemoryRangeIsReadable(reinterpret_cast<uint64_t>(buffer),
static_cast<uint64_t>(max_buffer + 1) * 4 *
sizeof(uint32_t))) {
LOGF("ShaderGetInputInfoVS(): unreadable vertex buffer table shader=0x%016" PRIx64 "\n",
shader_addr);
return false;
}
ShaderApplyAttribSemantics(info, metadata.input_semantics.data(), ShaderApplyAttribSemantics(info, metadata.input_semantics.data(),
metadata.input_semantics_count, attrib, buffer); metadata.input_semantics_count, attrib, buffer);
ShaderDetectBuffers(info); ShaderDetectBuffers(info);
src/graphics/shader/shaderVertexMetadata.cpp
+6 -12
View File
@@ -1,7 +1,5 @@
#include "graphics/shader/shaderVertexMetadata.h" #include "graphics/shader/shaderVertexMetadata.h"
#include "graphics/host_gpu/hostMemory.h"
#include <array> #include <array>
#include <cstring> #include <cstring>
@@ -16,10 +14,6 @@ bool Fail(std::string* error, const char* message) {
return false; return false;
} }
bool Readable(const void* data, uint64_t size) {
return data != nullptr && HostMemoryRangeIsReadable(reinterpret_cast<uint64_t>(data), size);
}
} // namespace } // namespace
bool ShaderReadVertexMetadata(const ShaderMappedData& data, uint32_t max_user_sgprs, bool ShaderReadVertexMetadata(const ShaderMappedData& data, uint32_t max_user_sgprs,
@@ -27,8 +21,8 @@ bool ShaderReadVertexMetadata(const ShaderMappedData& data, uint32_t max_user_sg
if (metadata == nullptr) { if (metadata == nullptr) {
return Fail(error, "missing vertex metadata output"); return Fail(error, "missing vertex metadata output");
} }
if (!Readable(data.user_data, sizeof(ShaderUserData))) { if (data.user_data == nullptr) {
return Fail(error, "unreadable AGC user-data header"); return Fail(error, "missing AGC user-data header");
} }
ShaderUserData user_data {}; ShaderUserData user_data {};
@@ -44,8 +38,8 @@ bool ShaderReadVertexMetadata(const ShaderMappedData& data, uint32_t max_user_sg
const auto direct_size = const auto direct_size =
static_cast<uint64_t>(user_data.direct_resource_count) * sizeof(uint16_t); static_cast<uint64_t>(user_data.direct_resource_count) * sizeof(uint16_t);
if (direct_size != 0) { if (direct_size != 0) {
if (!Readable(user_data.direct_resource_offset, direct_size)) { if (user_data.direct_resource_offset == nullptr) {
return Fail(error, "unreadable AGC direct-resource offsets"); return Fail(error, "missing AGC direct-resource offsets");
} }
std::memcpy(direct_offsets.data(), user_data.direct_resource_offset, direct_size); std::memcpy(direct_offsets.data(), user_data.direct_resource_offset, direct_size);
} }
@@ -84,8 +78,8 @@ bool ShaderReadVertexMetadata(const ShaderMappedData& data, uint32_t max_user_sg
const auto semantic_size = const auto semantic_size =
static_cast<uint64_t>(data.num_input_semantics) * sizeof(ShaderSemantic); static_cast<uint64_t>(data.num_input_semantics) * sizeof(ShaderSemantic);
if (!Readable(data.input_semantics, semantic_size)) { if (data.input_semantics == nullptr) {
return Fail(error, "unreadable vertex input semantics"); return Fail(error, "missing vertex input semantics");
} }
std::memcpy(next.input_semantics.data(), data.input_semantics, semantic_size); std::memcpy(next.input_semantics.data(), data.input_semantics, semantic_size);
next.input_semantics_count = data.num_input_semantics; next.input_semantics_count = data.num_input_semantics;
src/libs/agc.cpp
+6 -23
View File
@@ -11,7 +11,6 @@
#include "graphics/guest_gpu/pm4.h" #include "graphics/guest_gpu/pm4.h"
#include "graphics/guest_gpu/gpu_defs.h" #include "graphics/guest_gpu/gpu_defs.h"
#include "graphics/guest_gpu/tile.h" #include "graphics/guest_gpu/tile.h"
#include "graphics/host_gpu/hostMemory.h"
#include "graphics/host_gpu/objects/label.h" #include "graphics/host_gpu/objects/label.h"
#include "graphics/host_gpu/renderer/render.h" #include "graphics/host_gpu/renderer/render.h"
#include "graphics/presentation/renderDoc.h" #include "graphics/presentation/renderDoc.h"
@@ -373,11 +372,6 @@ struct Label {
uint64_t m_reserved[3]; uint64_t m_reserved[3];
}; };
static bool is_readable_command_packet_range(const uint32_t* cmd, uint32_t num_dw) {
return HostMemoryRangeIsReadable(reinterpret_cast<uintptr_t>(cmd),
static_cast<uint64_t>(num_dw) * sizeof(uint32_t));
}
int KYTY_SYSV_ABI GraphicsInit(uint32_t* state, uint32_t ver) { int KYTY_SYSV_ABI GraphicsInit(uint32_t* state, uint32_t ver) {
PRINT_NAME(); PRINT_NAME();
@@ -3750,8 +3744,7 @@ static void flush_pending_graphics_segment_before_acb(const uint32_t* acb,
if (!wait_addresses.empty() && Gen5::g_pending_graphics_segment.start != nullptr) { if (!wait_addresses.empty() && Gen5::g_pending_graphics_segment.start != nullptr) {
auto* scan = Gen5::g_pending_graphics_segment.start; auto* scan = Gen5::g_pending_graphics_segment.start;
auto* matched_end = Gen5::g_pending_graphics_segment.start; auto* matched_end = Gen5::g_pending_graphics_segment.start;
while (scan < Gen5::g_pending_graphics_segment.end && while (scan < Gen5::g_pending_graphics_segment.end) {
Gen5::is_readable_command_packet_range(scan, 1)) {
auto cmd_id = *scan; auto cmd_id = *scan;
if (cmd_id == 0x80000000u) { if (cmd_id == 0x80000000u) {
scan++; scan++;
@@ -3763,8 +3756,7 @@ static void flush_pending_graphics_segment_before_acb(const uint32_t* acb,
auto len = KYTY_PM4_LEN(cmd_id); auto len = KYTY_PM4_LEN(cmd_id);
if (len == 0 || if (len == 0 ||
len > static_cast<uint32_t>(Gen5::g_pending_graphics_segment.end - scan) || len > static_cast<uint32_t>(Gen5::g_pending_graphics_segment.end - scan)) {
!Gen5::is_readable_command_packet_range(scan, len)) {
break; break;
} }
@@ -3789,8 +3781,7 @@ static void flush_pending_graphics_segment_before_acb(const uint32_t* acb,
Gen5::g_pending_graphics_segment.end > Gen5::g_pending_graphics_segment.start) { Gen5::g_pending_graphics_segment.end > Gen5::g_pending_graphics_segment.start) {
auto* scan = Gen5::g_pending_graphics_segment.start; auto* scan = Gen5::g_pending_graphics_segment.start;
auto* valid_end = Gen5::g_pending_graphics_segment.start; auto* valid_end = Gen5::g_pending_graphics_segment.start;
while (scan < Gen5::g_pending_graphics_segment.end && while (scan < Gen5::g_pending_graphics_segment.end) {
Gen5::is_readable_command_packet_range(scan, 1)) {
auto cmd_id = *scan; auto cmd_id = *scan;
if (cmd_id == 0x80000000u) { if (cmd_id == 0x80000000u) {
scan++; scan++;
@@ -3803,8 +3794,7 @@ static void flush_pending_graphics_segment_before_acb(const uint32_t* acb,
auto len = KYTY_PM4_LEN(cmd_id); auto len = KYTY_PM4_LEN(cmd_id);
if (len == 0 || if (len == 0 ||
len > static_cast<uint32_t>(Gen5::g_pending_graphics_segment.end - scan) || len > static_cast<uint32_t>(Gen5::g_pending_graphics_segment.end - scan)) {
!Gen5::is_readable_command_packet_range(scan, len)) {
break; break;
} }
@@ -3931,16 +3921,14 @@ int KYTY_SYSV_ABI GraphicsDriverSubmitMultiCommandBuffers(uint32_t queue, uint32
} }
static void submit_acb(uint32_t queue, uint32_t* acb, uint32_t size_in_dwords) { static void submit_acb(uint32_t queue, uint32_t* acb, uint32_t size_in_dwords) {
if (acb != nullptr && size_in_dwords >= 5 && Gen5::is_readable_command_packet_range(acb, 5)) { if (acb != nullptr && size_in_dwords >= 5) {
auto descriptor_addr = auto descriptor_addr =
static_cast<uint64_t>(acb[0]) | (static_cast<uint64_t>(acb[1]) << 32u); static_cast<uint64_t>(acb[0]) | (static_cast<uint64_t>(acb[1]) << 32u);
auto descriptor_size = acb[2]; auto descriptor_size = acb[2];
auto descriptor_flags = acb[3]; auto descriptor_flags = acb[3];
auto descriptor_magic = acb[4]; auto descriptor_magic = acb[4];
if (descriptor_addr != 0 && descriptor_size != 0 && descriptor_flags == 0 && if (descriptor_addr != 0 && descriptor_size != 0 && descriptor_flags == 0 &&
descriptor_magic == 0x5533ccaau && descriptor_magic == 0x5533ccaau) {
Gen5::is_readable_command_packet_range(reinterpret_cast<uint32_t*>(descriptor_addr),
descriptor_size)) {
LOGF("\t descriptor addr = 0x%016" PRIx64 "\n" LOGF("\t descriptor addr = 0x%016" PRIx64 "\n"
"\t descriptor size = 0x%08" PRIx32 "\n" "\t descriptor size = 0x%08" PRIx32 "\n"
"\t descriptor magic = 0x%08" PRIx32 "\n", "\t descriptor magic = 0x%08" PRIx32 "\n",
@@ -3976,11 +3964,6 @@ int KYTY_SYSV_ABI GraphicsDriverSubmitAcb(uint32_t queue, const Packet* packet)
if (packet == nullptr) { if (packet == nullptr) {
return OK; return OK;
} }
if (!Gen5::is_readable_command_packet_range(reinterpret_cast<const uint32_t*>(packet),
sizeof(Packet) / sizeof(uint32_t))) {
return OK;
}
LOGF("\t acb = 0x%016" PRIx64 "\n" LOGF("\t acb = 0x%016" PRIx64 "\n"
"\t size = 0x%08" PRIx32 "\n" "\t size = 0x%08" PRIx32 "\n"
"\t flags = 0x%02" PRIx8 "\n", "\t flags = 0x%02" PRIx8 "\n",
tests/ShaderRecompilerComputeTests.cpp
+3 -8
View File
@@ -9025,14 +9025,6 @@ void CheckBufferCacheRangeMerge() {
3) && 3) &&
!CanMergeBufferCacheQueueMask(0, 64), !CanMergeBufferCacheQueueMask(0, 64),
"cross-queue or invalid queue ownership was accepted"); "cross-queue or invalid queue ownership was accepted");
Require("BufferCacheRangeMerge", "readback queue ownership",
!CanReadbackBufferCacheQueueMask(0, 3) &&
CanReadbackBufferCacheQueueMask(uint64_t{1} << 3u, 3) &&
!CanReadbackBufferCacheQueueMask((uint64_t{1} << 2u) |
(uint64_t{1} << 3u),
3) &&
!CanReadbackBufferCacheQueueMask(uint64_t{1} << 3u, 64),
"zero, cross-queue, or invalid readback ownership was accepted");
std::printf("[host] %-32s ok\n", "BufferCacheRangeMerge"); std::printf("[host] %-32s ok\n", "BufferCacheRangeMerge");
} }
@@ -9855,10 +9847,13 @@ void CheckStorageTextureGpuOwnedRebindState() {
tracker.ForEachUploadRange(base, size, true, tracker.ForEachUploadRange(base, size, true,
[](uint64_t, uint64_t) noexcept {}, []() noexcept {}); [](uint64_t, uint64_t) noexcept {}, []() noexcept {});
uint64_t readable = 0; uint64_t readable = 0;
uint64_t mapped = 0;
MEMORY_BASIC_INFORMATION protection{}; MEMORY_BASIC_INFORMATION protection{};
Require("StorageTextureGpuOwnedRebind", "owned", Require("StorageTextureGpuOwnedRebind", "owned",
tracker.IsRegionGpuModified(base, size) && page_manager.IsMapped(base, size) && tracker.IsRegionGpuModified(base, size) && page_manager.IsMapped(base, size) &&
(!HostMemoryQueryReadable(base, size, &readable) || readable < size) && (!HostMemoryQueryReadable(base, size, &readable) || readable < size) &&
HostMemoryQueryRange(base, size, HostMemoryAccess::Mapped, &mapped) &&
mapped == size &&
VirtualQuery(memory, &protection, sizeof(protection)) != 0 && VirtualQuery(memory, &protection, sizeof(protection)) != 0 &&
protection.Protect == PAGE_NOACCESS, protection.Protect == PAGE_NOACCESS,
"GPU-owned storage pages remained host-readable or lost tracker identity"); "GPU-owned storage pages remained host-readable or lost tracker identity");
tests/ShaderStageRuntimeTests.cpp
+3 -20
View File
@@ -3,7 +3,6 @@
#include "graphics/shader/recompiler/ResourceMaterialization.h" #include "graphics/shader/recompiler/ResourceMaterialization.h"
#include "graphics/shader/recompiler/ShaderIR.h" #include "graphics/shader/recompiler/ShaderIR.h"
#include <array>
#include <cstdio> #include <cstdio>
#include <cstdlib> #include <cstdlib>
#include <memory> #include <memory>
@@ -53,22 +52,7 @@ std::shared_ptr<const Libs::Graphics::ShaderRecompiler::IR::Program> UnbasedFlat
return std::make_shared<const Program>(std::move(program)); return std::make_shared<const Program>(std::move(program));
} }
void TestFailurePreservesPriorStage() { void TestMappedSrtUsesDirectReaderByDefault() {
using namespace Libs::Graphics;
auto cached_program = SrtProgram(0x1001);
auto prior_program = std::make_shared<const ShaderRecompiler::IR::Program>();
auto prior_resources = std::make_shared<const ShaderRecompiler::IR::ResourceSnapshot>();
ShaderStageRuntime stage {prior_program, prior_resources};
const std::array<uint32_t, 1> user_data = {};
std::string error;
Check(!ShaderMaterializeStageRuntime(cached_program, user_data, 0, &stage, &error) &&
error.find("failed at 0x0000000000001000") != std::string::npos,
"unmapped cache ReadConst did not fail through the checked production reader");
Check(stage.program == prior_program && stage.resources == prior_resources,
"failed cache rematerialization replaced the prior stage snapshot");
}
void TestMappedSrtUsesCheckedReaderByDefault() {
using namespace Libs::Graphics; using namespace Libs::Graphics;
const uint32_t dword = 0x12345678; const uint32_t dword = 0x12345678;
auto cached_program = SrtProgram(reinterpret_cast<uint64_t>(&dword)); auto cached_program = SrtProgram(reinterpret_cast<uint64_t>(&dword));
@@ -79,7 +63,7 @@ void TestMappedSrtUsesCheckedReaderByDefault() {
"cache rematerialization did not publish the mapped stage"); "cache rematerialization did not publish the mapped stage");
Check(stage.resources->flattened_srt.size() == 1 && Check(stage.resources->flattened_srt.size() == 1 &&
stage.resources->flattened_srt[0] == dword, stage.resources->flattened_srt[0] == dword,
"cache rematerialization did not use the checked reader by default"); "cache rematerialization did not use the direct reader by default");
} }
void TestUnbasedFlatCacheHitFailsClosed() { void TestUnbasedFlatCacheHitFailsClosed() {
@@ -99,8 +83,7 @@ void TestUnbasedFlatCacheHitFailsClosed() {
} // namespace } // namespace
int main() { int main() {
TestFailurePreservesPriorStage(); TestMappedSrtUsesDirectReaderByDefault();
TestMappedSrtUsesCheckedReaderByDefault();
TestUnbasedFlatCacheHitFailsClosed(); TestUnbasedFlatCacheHitFailsClosed();
std::puts("ShaderStageRuntimeTests: all cases passed"); std::puts("ShaderStageRuntimeTests: all cases passed");
return 0; return 0;
tests/ShaderVertexMetadataTests.cpp
+12 -71
View File
@@ -1,23 +1,10 @@
#include "graphics/shader/shaderVertexMetadata.h" #include "graphics/shader/shaderVertexMetadata.h"
#include <algorithm>
#include <array> #include <array>
#include <cstdint> #include <cstdint>
#include <cstdio> #include <cstdio>
#include <cstdlib> #include <cstdlib>
#if KYTY_PLATFORM == KYTY_PLATFORM_WINDOWS
#ifndef NOMINMAX
#define NOMINMAX
#endif
#include <windows.h>
#undef min
#undef max
#elif KYTY_PLATFORM == KYTY_PLATFORM_LINUX
#include <sys/mman.h>
#include <unistd.h>
#endif
namespace { namespace {
using namespace Libs::Graphics; using namespace Libs::Graphics;
@@ -60,7 +47,7 @@ void CheckRejected(const ShaderMappedData& data, const char* text) {
"metadata rejection changed the prior output"); "metadata rejection changed the prior output");
} }
void TestValidAndHostileMetadata() { void TestValidAndInvalidMetadata() {
Fixture fixture; Fixture fixture;
ShaderVertexMetadata output; ShaderVertexMetadata output;
std::string error; std::string error;
@@ -70,9 +57,13 @@ void TestValidAndHostileMetadata() {
output.input_semantics_count == 1, output.input_semantics_count == 1,
"valid AGC vertex metadata was decoded incorrectly"); "valid AGC vertex metadata was decoded incorrectly");
auto unreadable = fixture.mapped; auto missing_header = fixture.mapped;
unreadable.user_data = reinterpret_cast<ShaderUserData*>(uintptr_t {1}); missing_header.user_data = nullptr;
CheckRejected(unreadable, "unreadable AGC user-data header was accepted"); CheckRejected(missing_header, "missing AGC user-data header was accepted");
Fixture missing_offsets;
missing_offsets.user_data.direct_resource_offset = nullptr;
CheckRejected(missing_offsets.mapped, "missing direct-resource offsets were accepted");
Fixture excessive_resources; Fixture excessive_resources;
excessive_resources.user_data.direct_resource_count = excessive_resources.user_data.direct_resource_count =
@@ -88,65 +79,15 @@ void TestValidAndHostileMetadata() {
static_cast<size_t>(AgcDirectResourceType::PtrVertexBufferTable)] = 63; static_cast<size_t>(AgcDirectResourceType::PtrVertexBufferTable)] = 63;
CheckRejected(excessive_register.mapped, "out-of-domain vertex table SGPR was accepted"); CheckRejected(excessive_register.mapped, "out-of-domain vertex table SGPR was accepted");
Fixture bad_semantics; Fixture missing_semantics;
bad_semantics.mapped.input_semantics = missing_semantics.mapped.input_semantics = nullptr;
reinterpret_cast<ShaderSemantic*>(uintptr_t {1}); CheckRejected(missing_semantics.mapped, "missing vertex semantic array was accepted");
CheckRejected(bad_semantics.mapped, "unreadable vertex semantic array was accepted");
}
void TestCrossPageMetadata() {
#if KYTY_PLATFORM == KYTY_PLATFORM_WINDOWS
SYSTEM_INFO system_info {};
GetSystemInfo(&system_info);
const auto page_size = static_cast<uint64_t>(system_info.dwPageSize);
auto* pages = static_cast<uint8_t*>(
VirtualAlloc(nullptr, page_size * 2, MEM_RESERVE | MEM_COMMIT, PAGE_READWRITE));
Check(pages != nullptr, "failed to allocate metadata test pages");
DWORD old_protect = 0;
Check(VirtualProtect(pages + page_size, page_size, PAGE_NOACCESS, &old_protect) != 0,
"failed to protect metadata test page");
#elif KYTY_PLATFORM == KYTY_PLATFORM_LINUX
const auto native_page_size = sysconf(_SC_PAGESIZE);
Check(native_page_size > 0, "failed to query page size");
const auto page_size = static_cast<uint64_t>(native_page_size);
auto* pages = static_cast<uint8_t*>(
mmap(nullptr, page_size * 2, PROT_READ | PROT_WRITE, MAP_PRIVATE | MAP_ANONYMOUS, -1, 0));
Check(pages != MAP_FAILED, "failed to allocate metadata test pages");
Check(mprotect(pages + page_size, page_size, PROT_NONE) == 0,
"failed to protect metadata test page");
#else
return;
#endif
Fixture fixture;
auto header_crossing = fixture.mapped;
header_crossing.user_data = reinterpret_cast<ShaderUserData*>(
pages + page_size - sizeof(ShaderUserData) + 1);
CheckRejected(header_crossing, "cross-page AGC user-data header was accepted");
auto offsets_crossing = fixture.mapped;
fixture.user_data.direct_resource_offset =
reinterpret_cast<uint16_t*>(pages + page_size - sizeof(uint16_t));
CheckRejected(offsets_crossing, "cross-page direct-resource offsets were accepted");
fixture.user_data.direct_resource_offset = fixture.offsets.data();
auto semantics_crossing = fixture.mapped;
semantics_crossing.input_semantics =
reinterpret_cast<ShaderSemantic*>(pages + page_size - sizeof(uint16_t));
CheckRejected(semantics_crossing, "cross-page vertex semantics were accepted");
#if KYTY_PLATFORM == KYTY_PLATFORM_WINDOWS
Check(VirtualFree(pages, 0, MEM_RELEASE) != 0, "failed to free metadata test pages");
#elif KYTY_PLATFORM == KYTY_PLATFORM_LINUX
Check(munmap(pages, page_size * 2) == 0, "failed to free metadata test pages");
#endif
} }
} // namespace } // namespace
int main() { int main() {
TestValidAndHostileMetadata(); TestValidAndInvalidMetadata();
TestCrossPageMetadata();
std::puts("ShaderVertexMetadataTests: all cases passed"); std::puts("ShaderVertexMetadataTests: all cases passed");
return 0; return 0;
} }