From f60007e923f9889d7e49824259b0473f3ffafbdc Mon Sep 17 00:00:00 2001 From: nmzik Date: Wed, 22 Jul 2026 23:15:03 +0200 Subject: [PATCH] refactor(gpu): use a single ordered PM4 scheduler --- src/graphics/asyncJob.h | 96 -- .../command_processor/commandProcessor.h | 148 +-- .../command_processor/pm4Handlers.cpp | 11 +- src/graphics/guest_gpu/graphicsRun.cpp | 1075 ++++++----------- src/graphics/guest_gpu/graphicsRun.h | 2 +- .../host_gpu/renderer/bufferCache.cpp | 2 +- .../host_gpu/renderer/commandScheduler.cpp | 92 ++ .../host_gpu/renderer/commandScheduler.h | 45 + src/graphics/host_gpu/renderer/render.h | 21 +- .../host_gpu/renderer/renderContext.h | 3 + .../host_gpu/renderer/textureCache.cpp | 2 +- tests/ShaderRecompilerComputeTests.cpp | 101 ++ 12 files changed, 680 insertions(+), 918 deletions(-) delete mode 100644 src/graphics/asyncJob.h create mode 100644 src/graphics/host_gpu/renderer/commandScheduler.cpp create mode 100644 src/graphics/host_gpu/renderer/commandScheduler.h diff --git a/src/graphics/asyncJob.h b/src/graphics/asyncJob.h deleted file mode 100644 index 1ea110d14..000000000 --- a/src/graphics/asyncJob.h +++ /dev/null @@ -1,96 +0,0 @@ -#ifndef EMULATOR_INCLUDE_EMULATOR_GRAPHICS_ASYNCJOB_H_ -#define EMULATOR_INCLUDE_EMULATOR_GRAPHICS_ASYNCJOB_H_ - -#include "common/common.h" -#include "common/profiler.h" -#include "common/threads.h" - -#include -#include -#include - -namespace Libs::Graphics { - -class AsyncJob final { -public: - using Task = std::function; - - explicit AsyncJob(std::string thread_name = {}) - : m_thread_name(std::move(thread_name)), m_worker_thread(WorkerEntry, this) {} - - ~AsyncJob() { - { - Common::LockGuard lock(m_mutex); - m_stop_requested = true; - m_task_available_condition.Signal(); - } - - m_worker_thread.Join(); - } - - KYTY_CLASS_NO_COPY(AsyncJob); - - void Execute(Task task) { - Common::LockGuard lock(m_mutex); - while (m_is_busy) { - m_idle_condition.Wait(&m_mutex); - } - - m_task = std::move(task); - m_is_busy = true; - m_task_available_condition.Signal(); - } - - void Wait() { - Common::LockGuard lock(m_mutex); - while (m_is_busy) { - m_idle_condition.Wait(&m_mutex); - } - } - -private: - static void WorkerEntry(void* data) { static_cast(data)->WorkerLoop(); } - - void WorkerLoop() { - if (!m_thread_name.empty()) { - KYTY_PROFILER_THREAD(m_thread_name.c_str()); - } - - for (;;) { - Task task; - { - Common::LockGuard lock(m_mutex); - while (!m_is_busy && !m_stop_requested) { - m_task_available_condition.Wait(&m_mutex); - } - - if (!m_is_busy) { - return; - } - - task = std::move(m_task); - } - - task(); - - { - Common::LockGuard lock(m_mutex); - m_is_busy = false; - m_idle_condition.SignalAll(); - } - } - } - - std::string m_thread_name; - Common::Mutex m_mutex; - Common::CondVar m_task_available_condition; - Common::CondVar m_idle_condition; - Task m_task; - bool m_is_busy = false; - bool m_stop_requested = false; - Common::Thread m_worker_thread; -}; - -} // namespace Libs::Graphics - -#endif /* EMULATOR_INCLUDE_EMULATOR_GRAPHICS_ASYNCJOB_H_ */ diff --git a/src/graphics/guest_gpu/command_processor/commandProcessor.h b/src/graphics/guest_gpu/command_processor/commandProcessor.h index f5fe4f7a3..44aabf828 100644 --- a/src/graphics/guest_gpu/command_processor/commandProcessor.h +++ b/src/graphics/guest_gpu/command_processor/commandProcessor.h @@ -2,12 +2,10 @@ #define GRAPHICS_GUEST_GPU_COMMAND_PROCESSOR_COMMAND_PROCESSOR_H #include "common/assert.h" -#include "common/threads.h" #include "graphics/guest_gpu/hardwareContext.h" #include "graphics/host_gpu/renderer/render.h" #include "graphics/host_gpu/renderer/renderContext.h" -#include #include #include @@ -17,94 +15,25 @@ inline constexpr uint32_t AcquireGcrGl2Writeback = 1u << 15u; bool TestWaitRegMemValue(uint64_t value, uint64_t ref, uint64_t mask, uint32_t func); -class CommandScheduler { +enum class Pm4ProcessResult { Complete, Blocked }; + +class Pm4Execution { public: - static constexpr int BuffersNum = 8; - - CommandScheduler(HW::Context& registers, HW::UserConfig& user_config, HW::Shader& shaders) - : m_registers(registers), m_user_config(user_config), m_shaders(shaders) {} - - bool Active() const { return m_current >= 0 && m_current < BuffersNum; } - void CheckActive() const { EXIT_IF(!Active()); } - - RenderCommandBuffer& Current() const { - CheckActive(); - EXIT_IF(m_buffers[m_current] == nullptr); - return *m_buffers[m_current]; - } - - void Init() { - if (m_current >= 0) { - return; - } - for (auto& buf: m_buffers) { - EXIT_IF(buf != nullptr); - buf = new RenderCommandBuffer(m_registers, m_user_config, m_shaders); - } - m_current = 0; - Current().Begin(); - } - - void Flush() { - SubmitCurrent(); - BeginNext(); - } - - CommandBuffer& FlushAndGetSubmitted() { - auto& submitted = SubmitCurrent(); - BeginNext(); - return submitted; - } - - void CopyBuffers(std::array& out) const { - for (int i = 0; i < BuffersNum; i++) { - auto* buf = m_buffers[i]; - EXIT_IF(buf == nullptr); - out[i] = buf; - } - } - - void WaitAll() { - for (auto* buf: m_buffers) { - EXIT_IF(buf == nullptr); - buf->WaitForFenceAndReset(); - } - } - - void SubmitForReadback() { - if (!Active()) { - return; - } - SubmitCurrent(); - } - - void ResumeAfterReadback() { - if (!Active()) { - return; - } - Current().WaitForFenceAndReset(); - Current().Begin(); - } + [[nodiscard]] bool MadeProgress() const noexcept { return m_made_progress; } private: - CommandBuffer& SubmitCurrent() { - auto& submitted = Current(); - submitted.End(); - submitted.Execute(); - return submitted; - } + friend class CommandProcessor; - void BeginNext() { - m_current = (m_current + 1) % BuffersNum; - Current().WaitForFenceAndReset(); - Current().Begin(); - } + struct BufferCursor { + uint32_t* next_packet = nullptr; + uint32_t remaining_dw = 0; + uint32_t total_dw = 0; + uint32_t deferred_advance_dw = 0; + }; - RenderCommandBuffer* m_buffers[BuffersNum] = {}; - int m_current = -1; - HW::Context& m_registers; - HW::UserConfig& m_user_config; - HW::Shader& m_shaders; + std::vector m_buffer_stack; + bool m_suspended = false; + bool m_made_progress = false; }; class CommandProcessor { @@ -116,8 +45,8 @@ public: int64_t flip_arg = 0; }; - CommandProcessor(); - ~CommandProcessor() { KYTY_NOT_IMPLEMENTED; } + CommandProcessor() = default; + ~CommandProcessor() = default; KYTY_CLASS_NO_COPY(CommandProcessor); @@ -128,7 +57,6 @@ public: void BufferFlushAndWait(); void BufferWait(); void BeginReadbackTransaction() { - m_mutex.Lock(); if (m_readback_active) { EXIT("nested command-processor readback transaction\n"); } @@ -142,12 +70,8 @@ public: } m_readback_active = false; m_readback_finished = false; - m_mutex.Unlock(); } - void RunLock() { m_run_mutex.Lock(); } - void RunUnlock() { m_run_mutex.Unlock(); } - HW::Context& GetCtx() { return m_ctx; } HW::UserConfig& GetUcfg() { return m_ucfg; } HW::Shader& GetShCtx() { return m_sh_ctx; } @@ -202,10 +126,11 @@ public: void PrefetchL2(void* addr, uint32_t size) {} void ResetDeCe(); + void SetCeComplete(bool complete) { m_ce_complete = complete; } void WaitCe(); void WaitDeDiff(uint32_t diff); - void IncremenetDe(); - void IncremenetCe(); + void IncrementDe(); + void IncrementCe(); void WriteConstRam(uint32_t offset, const uint32_t* src, uint32_t dw_num); void DumpConstRam(uint32_t* dst, uint32_t offset, uint32_t dw_num); @@ -222,30 +147,26 @@ public: const volatile void* address, uint32_t count_in_dwords); [[nodiscard]] bool ShouldSkipPredicatedPackets() const { return m_predicate_skip; } - void Run(uint32_t* data, uint32_t num_dw); + Pm4ProcessResult Process(Pm4Execution& execution, uint32_t* buffer, uint32_t size_dw); + void ProcessIndirectBuffer(uint32_t* buffer, uint32_t size_dw); - [[nodiscard]] const FlipInfo& GetFlip() const { return m_flip; } - void SetFlip(const FlipInfo& flip) { m_flip = flip; } + void SetFlip(const FlipInfo& flip) { m_flip = flip; } [[nodiscard]] uint64_t GetSubmitId() const { return m_submit_id; } void SetSubmitId(uint64_t submit_id) { m_submit_id = submit_id; } private: - struct Counter { - Common::Mutex mutex; - Common::CondVar cond_var; - uint32_t value = 0; - }; - template void WriteAtEndOfPipe(uint32_t cache_policy, uint32_t event_write_dest, uint32_t eop_event_type, uint32_t cache_action, uint32_t event_index, uint32_t event_write_source, void* dst_gpu_addr, T value, uint32_t interrupt_selector, uint32_t interrupt_context_id); - void FinishCommandProcessors(); + void ProcessPm4(Pm4Execution& execution, size_t stop_depth); + void SuspendPm4(); - RenderCommandBuffer& CurrentBuffer() { return m_scheduler.Current(); } - void CheckBuffer() const { m_scheduler.CheckActive(); } + CommandScheduler& GetScheduler() const { return GetRenderContext().GetCommandScheduler(); } + RenderCommandBuffer& CurrentBuffer() { return GetScheduler().Current(); } + void CheckBuffer() const { GetScheduler().CheckActive(); } GpuResourceManager& GetGpuResources() const { return GetRenderContext().GetGpuResources(); } HW::Context m_ctx; @@ -259,16 +180,11 @@ private: uint64_t m_dispatch_indirect_args_base_addr = 0; uint32_t m_num_instances = 1; - inline static Common::Mutex m_mutex; - inline static std::vector m_processors; - inline static bool m_readback_active = false; - inline static bool m_readback_finished = false; - Common::Mutex m_run_mutex; - - CommandScheduler m_scheduler; - - Counter m_de_counter; - Counter m_ce_counter; + uint32_t m_de_count = 0; + uint32_t m_ce_count = 0; + bool m_ce_complete = false; + bool m_readback_active = false; + bool m_readback_finished = false; uint32_t m_const_ram[0x3000] = {0}; diff --git a/src/graphics/guest_gpu/command_processor/pm4Handlers.cpp b/src/graphics/guest_gpu/command_processor/pm4Handlers.cpp index 46a7ff16d..199f9d788 100644 --- a/src/graphics/guest_gpu/command_processor/pm4Handlers.cpp +++ b/src/graphics/guest_gpu/command_processor/pm4Handlers.cpp @@ -1984,7 +1984,6 @@ KYTY_CP_OP_PARSER(CpOpAcquireMem) { } if (stall_mode == 0) { - cp.BufferFlush(); cp.BufferWait(); } @@ -2166,10 +2165,10 @@ KYTY_CP_OP_PARSER(CpOpBranch) { reinterpret_cast(else_buffer), else_num_dw); if (take_then) { - cp.Run(then_buffer, then_num_dw); + cp.ProcessIndirectBuffer(then_buffer, then_num_dw); } else if (mode == 2 && else_num_dw != 0) { EXIT_NOT_IMPLEMENTED(else_buffer == nullptr); - cp.Run(else_buffer, else_num_dw); + cp.ProcessIndirectBuffer(else_buffer, else_num_dw); } return payload_dw; @@ -2496,7 +2495,7 @@ KYTY_CP_OP_PARSER(CpOpIncrementCeCounter) { EXIT_NOT_IMPLEMENTED(cmd_id != 0xC0008400); EXIT_NOT_IMPLEMENTED(buffer[0] != 1); - cp.IncremenetCe(); + cp.IncrementCe(); return 1; } @@ -2507,7 +2506,7 @@ KYTY_CP_OP_PARSER(CpOpIncrementDeCounter) { EXIT_NOT_IMPLEMENTED(cmd_id != 0xC0008500); EXIT_NOT_IMPLEMENTED(buffer[0] != 0); - cp.IncremenetDe(); + cp.IncrementDe(); return 1; } @@ -2607,7 +2606,7 @@ KYTY_CP_OP_PARSER(CpOpIndirectBuffer) { GraphicsDbgDumpDcb("ci", indirect_num_dw, indirect_buffer); - cp.Run(indirect_buffer, indirect_num_dw); + cp.ProcessIndirectBuffer(indirect_buffer, indirect_num_dw); return 3; } diff --git a/src/graphics/guest_gpu/graphicsRun.cpp b/src/graphics/guest_gpu/graphicsRun.cpp index cb20f6016..0626276d0 100644 --- a/src/graphics/guest_gpu/graphicsRun.cpp +++ b/src/graphics/guest_gpu/graphicsRun.cpp @@ -6,7 +6,6 @@ #include "common/profiler.h" #include "common/stringUtils.h" #include "common/threads.h" -#include "graphics/asyncJob.h" #include "graphics/guest_gpu/command_processor/commandProcessor.h" #include "graphics/guest_gpu/command_processor/pm4Dispatch.h" #include "graphics/guest_gpu/hardwareContext.h" @@ -27,13 +26,14 @@ #include #include #include -#include -#include +#include +#include #include namespace Libs::Graphics { -static thread_local CommandProcessor* g_current_run_cp = nullptr; +static thread_local CommandProcessor* g_current_processor = nullptr; +static thread_local Pm4Execution* g_current_execution = nullptr; static thread_local uint32_t g_submission_pause_depth = 0; static thread_local bool g_gpu_mutex_owned = false; @@ -59,155 +59,34 @@ private: }; struct OwnedCmdBuffer { - std::vector storage; - uint32_t* data = nullptr; - uint32_t num_dw = 0; - OwnedCmdBuffer() = default; - OwnedCmdBuffer(const uint32_t* src, uint32_t count) { Assign(src, count); } - - OwnedCmdBuffer(const OwnedCmdBuffer& other): storage(other.storage), num_dw(other.num_dw) { - data = (storage.empty() ? other.data : storage.data()); - } - - OwnedCmdBuffer& operator=(const OwnedCmdBuffer& other) { - if (this != &other) { - storage = other.storage; - num_dw = other.num_dw; - data = (storage.empty() ? other.data : storage.data()); - } - return *this; - } - - OwnedCmdBuffer(OwnedCmdBuffer&& other) noexcept - : storage(std::move(other.storage)), data(other.data), num_dw(other.num_dw) { - data = (storage.empty() ? other.data : storage.data()); - } - - OwnedCmdBuffer& operator=(OwnedCmdBuffer&& other) noexcept { - if (this != &other) { - storage = std::move(other.storage); - num_dw = other.num_dw; - data = (storage.empty() ? other.data : storage.data()); - } - return *this; - } - - void Assign(const uint32_t* src, uint32_t count) { - num_dw = count; - if (src != nullptr && count != 0) { - storage.assign(src, src + count); - data = storage.data(); - } else { - storage.clear(); - data = nullptr; + explicit OwnedCmdBuffer(const uint32_t* data, uint32_t count) { + EXIT_IF(data == nullptr && count != 0); + if (count != 0) { + m_words.assign(data, data + count); } } -}; -class GraphicsRing { -public: - GraphicsRing(): m_draw_job("Thread_Gfx_Draw"), m_constant_job("Thread_Gfx_Const") { - EXIT_NOT_IMPLEMENTED(!Common::Thread::IsMainThread()); - } - ~GraphicsRing() { KYTY_NOT_IMPLEMENTED; } - - KYTY_CLASS_NO_COPY(GraphicsRing); - - void Submit(OwnedCmdBuffer draw_buffer, OwnedCmdBuffer const_buffer, int handle, int index, - int flip_mode, int64_t flip_arg, bool trigger_agc_interrupt_on_done); - void SubmitFlipPreparation(); - void Done(); - void WaitForIdle(); - bool IsIdle(); - - void SetCp(CommandProcessor& cp) { - m_cp = &cp; - Start(); - } + [[nodiscard]] bool Empty() const noexcept { return m_words.empty(); } + [[nodiscard]] uint32_t Size() const noexcept { return static_cast(m_words.size()); } + [[nodiscard]] uint32_t* Data() noexcept { return m_words.data(); } private: - void Start() { - Common::Thread t(ThreadBatchRun, this); - t.Detach(); - } - - struct CmdBatch { - OwnedCmdBuffer draw_buffer; - OwnedCmdBuffer const_buffer; - - CommandProcessor::FlipInfo flip; - bool trigger_agc_interrupt_on_done = false; - bool prepare_cpu_flip = false; - }; - - static void ThreadBatchRun(void* data); - - CmdBatch GetCmdBatch(); - - Common::Mutex m_mutex; - Common::CondVar m_cond_var; - Common::CondVar m_idle_cond_var; - std::list m_cmd_batches; - bool m_done = true; - bool m_idle = true; - - AsyncJob m_draw_job; - AsyncJob m_constant_job; - - CommandProcessor* m_cp = nullptr; -}; - -class ComputeRing { -public: - ComputeRing() = default; - ~ComputeRing() { KYTY_NOT_IMPLEMENTED; } - - KYTY_CLASS_NO_COPY(ComputeRing); - - void Submit(OwnedCmdBuffer buffer, bool trigger_agc_interrupt_on_done); - void Done(); - void WaitForIdle(); - bool IsIdle(); - - void SetCp(CommandProcessor& cp) { - m_cp = &cp; - Start(); - } - -private: - void Start() { - Common::Thread t(ThreadRun, this); - t.Detach(); - } - - static void ThreadRun(void* data); - - Common::Mutex m_mutex; - Common::CondVar m_cond_var; - Common::CondVar m_idle_cond_var; - bool m_done = true; - bool m_idle = true; - - CommandProcessor* m_cp = nullptr; - - struct DirectBatch { - OwnedCmdBuffer buffer; - bool trigger_agc_interrupt_on_done = false; - }; - - std::list m_direct_batches; + std::vector m_words; }; class Gpu { public: - static constexpr uint32_t ComputePipeCount = 7; - static constexpr uint32_t RingsPerComputePipe = 8; - static constexpr uint32_t ComputeRingCount = ComputePipeCount * RingsPerComputePipe; + static constexpr uint32_t ComputePipeCount = 7; + static constexpr uint32_t QueuesPerComputePipe = 8; + static constexpr uint32_t ComputeQueueCount = ComputePipeCount * QueuesPerComputePipe; + static constexpr uint32_t QueueCount = 1 + ComputeQueueCount; Gpu() { EXIT_NOT_IMPLEMENTED(!Common::Thread::IsMainThread()); - Init(); + m_gfx_cp = std::make_unique(); + Common::Thread thread(ThreadRun, this); + thread.Detach(); } ~Gpu() { KYTY_NOT_IMPLEMENTED; } @@ -224,20 +103,45 @@ public: int GetFrameNum(); private: - void Init(); - void WaitLocked(); + enum class SubmissionType { Graphics, Compute, FlipPreparation }; - ComputeRing* GetComputeRing(uint32_t ring_index); + struct Submission { + SubmissionType type = SubmissionType::Graphics; + uint32_t queue_id = 0; + OwnedCmdBuffer commands; + OwnedCmdBuffer constant_commands; + Pm4Execution command_execution; + Pm4Execution constant_execution; + bool trigger_agc_interrupt_on_done = false; + bool reset_processor = false; + bool started = false; + bool command_complete = false; + bool constant_complete = false; + bool blocked = false; + }; - Common::Mutex m_mutex; + void WaitLocked(); + void Enqueue(Submission submission); + void WaitForIdle(); + bool Process(Submission& submission); + static void ThreadRun(void* data); + CommandProcessor& GetProcessor(uint32_t queue_id); - CommandProcessor* m_gfx_cp = nullptr; - GraphicsRing* m_gfx_ring = nullptr; + Common::Mutex m_submission_mutex; + Common::Mutex m_queue_mutex; + Common::CondVar m_work_available; + Common::CondVar m_idle; + std::array, QueueCount> m_queues; + uint32_t m_next_queue = 0; + uint32_t m_submission_count = 0; + bool m_processing = false; + bool m_graphics_done = true; - std::array m_compute_cp {}; - std::array m_compute_ring {}; + std::unique_ptr m_gfx_cp; + std::array, ComputeQueueCount> m_compute_cp; - std::atomic_int m_done_num = 0; + uint64_t m_submit_id = 0; + std::atomic_int m_done_num = 0; }; static Gpu* g_gpu = nullptr; @@ -259,168 +163,87 @@ void GraphicsRunInit() { void Gpu::Submit(uint32_t* cmd_draw_buffer, uint32_t num_draw_dw, uint32_t* cmd_const_buffer, uint32_t num_const_dw, bool trigger_agc_interrupt_on_done) { - OwnedCmdBuffer draw_buffer(cmd_draw_buffer, num_draw_dw); - OwnedCmdBuffer const_buffer(cmd_const_buffer, num_const_dw); - GpuMutexLock lock(m_mutex); - - m_gfx_ring->Submit(std::move(draw_buffer), std::move(const_buffer), 0, 0, 0, 0, - trigger_agc_interrupt_on_done); + GpuMutexLock lock(m_submission_mutex); + Submission submission; + submission.type = SubmissionType::Graphics; + submission.queue_id = 0; + submission.commands = OwnedCmdBuffer(cmd_draw_buffer, num_draw_dw); + submission.constant_commands = OwnedCmdBuffer(cmd_const_buffer, num_const_dw); + submission.trigger_agc_interrupt_on_done = trigger_agc_interrupt_on_done; + submission.reset_processor = m_graphics_done; + m_graphics_done = false; + Enqueue(std::move(submission)); } void Gpu::SubmitCompute(uint32_t queue, uint32_t* cmd_buffer, uint32_t num_dw, bool trigger_agc_interrupt_on_done) { - OwnedCmdBuffer buffer(cmd_buffer, num_dw); - GpuMutexLock lock(m_mutex); + GpuMutexLock lock(m_submission_mutex); constexpr uint32_t compute_queue_base = 0x20u; EXIT_NOT_IMPLEMENTED(queue < compute_queue_base || - queue >= compute_queue_base + ComputeRingCount); + queue >= compute_queue_base + ComputeQueueCount); - uint32_t compute_queue = queue - compute_queue_base; - auto* ring = GetComputeRing(compute_queue); - - ring->Submit(std::move(buffer), trigger_agc_interrupt_on_done); + const auto compute_queue = queue - compute_queue_base; + Submission submission; + submission.type = SubmissionType::Compute; + submission.queue_id = 1 + compute_queue; + submission.commands = OwnedCmdBuffer(cmd_buffer, num_dw); + submission.trigger_agc_interrupt_on_done = trigger_agc_interrupt_on_done; + Enqueue(std::move(submission)); } void Gpu::SubmitFlipPreparation() { - GpuMutexLock lock(m_mutex); - m_gfx_ring->SubmitFlipPreparation(); + GpuMutexLock lock(m_submission_mutex); + Submission submission; + submission.type = SubmissionType::FlipPreparation; + submission.queue_id = 0; + submission.reset_processor = m_graphics_done; + m_graphics_done = false; + Enqueue(std::move(submission)); } void Gpu::Done() { - GraphicsRing* gfx_ring = nullptr; - CommandProcessor* gfx_cp = nullptr; - std::array compute_rings {}; - std::array compute_cps {}; - - { - GpuMutexLock lock(m_mutex); - - gfx_ring = m_gfx_ring; - gfx_cp = m_gfx_cp; - - compute_rings = m_compute_ring; - compute_cps = m_compute_cp; - - m_done_num++; - } - - if (gfx_ring != nullptr) { - gfx_ring->Done(); - } - for (auto& cr: compute_rings) { - if (cr != nullptr) { - cr->Done(); - } - } - if (gfx_ring != nullptr) { - gfx_ring->WaitForIdle(); - } - if (gfx_cp != nullptr) { - gfx_cp->BufferWait(); - } - for (auto& cr: compute_rings) { - if (cr != nullptr) { - cr->WaitForIdle(); - } - } - for (auto& cp: compute_cps) { - if (cp != nullptr) { - cp->BufferWait(); - } - } + GpuMutexLock lock(m_submission_mutex); + WaitLocked(); + m_graphics_done = true; + m_done_num++; } int Gpu::GetFrameNum() { - // Common::LockGuard lock(m_mutex); - return m_done_num; } void Gpu::WaitLocked() { - GraphicsRing* gfx_ring = nullptr; - CommandProcessor* gfx_cp = nullptr; - std::array compute_rings {}; - std::array compute_cps {}; - - gfx_ring = m_gfx_ring; - gfx_cp = m_gfx_cp; - compute_rings = m_compute_ring; - compute_cps = m_compute_cp; - - if (gfx_ring != nullptr) { - gfx_ring->WaitForIdle(); - } - if (gfx_cp != nullptr) { - gfx_cp->BufferWait(); - } - for (auto& cr: compute_rings) { - if (cr != nullptr) { - cr->WaitForIdle(); - } - } - for (auto& cp: compute_cps) { - if (cp != nullptr) { - cp->BufferWait(); - } - } + WaitForIdle(); + m_gfx_cp->BufferWait(); } -void Gpu::Init() { - EXIT_IF(m_gfx_cp != nullptr); - EXIT_IF(m_gfx_ring != nullptr); - - m_gfx_cp = new CommandProcessor; - m_gfx_ring = new GraphicsRing; - m_gfx_ring->SetCp(*m_gfx_cp); -} - -ComputeRing* Gpu::GetComputeRing(uint32_t ring_index) { - EXIT_IF(ring_index >= ComputeRingCount); - const auto pipe_id = ring_index / RingsPerComputePipe; - - if (m_compute_cp[pipe_id] == nullptr) { - m_compute_cp[pipe_id] = new CommandProcessor; +CommandProcessor& Gpu::GetProcessor(uint32_t queue_id) { + EXIT_IF(queue_id >= QueueCount); + if (queue_id == 0) { + return *m_gfx_cp; } - - if (m_compute_ring[ring_index] == nullptr) { - m_compute_ring[ring_index] = new ComputeRing; - m_compute_ring[ring_index]->SetCp(*m_compute_cp[pipe_id]); + auto& processor = m_compute_cp[queue_id - 1]; + if (processor == nullptr) { + processor = std::make_unique(); } - - return m_compute_ring[ring_index]; -} - -CommandProcessor::CommandProcessor(): m_scheduler(m_ctx, m_ucfg, m_sh_ctx) { - Common::LockGuard lock(m_mutex); - m_processors.push_back(this); + return *processor; } void CommandProcessor::FinishReadbackTransaction() { - if (GraphicsRunCurrentCommandProcessor() == nullptr || !m_readback_active) { + if (GraphicsRunCurrentCommandProcessor() != this || !m_readback_active) { EXIT("GPU readback finish requires a command-processor thread\n"); } if (m_readback_finished) { return; } - FinishCommandProcessors(); + GetScheduler().FinishCurrent(); m_readback_finished = true; } -void CommandProcessor::FinishCommandProcessors() { - for (auto* processor: m_processors) { - processor->m_scheduler.SubmitForReadback(); - } - for (auto* processor: m_processors) { - processor->m_scheduler.ResumeAfterReadback(); - } -} - void CommandProcessor::Reset() { BufferWait(); - Common::LockGuard lock(m_mutex); - Sync::DeleteBuffers(); m_sh_ctx.Reset(); @@ -436,110 +259,56 @@ void CommandProcessor::Reset() { } void CommandProcessor::BufferInit() { - Common::LockGuard lock(m_mutex); - m_scheduler.Init(); + GetScheduler().Begin(m_ctx, m_ucfg, m_sh_ctx); } void CommandProcessor::BufferFlush() { - Common::LockGuard lock(m_mutex); - m_scheduler.Flush(); + GetScheduler().Flush(); } void CommandProcessor::BufferFlushAndWait() { - auto& submitted = [&]() -> CommandBuffer& { - Common::LockGuard lock(m_mutex); - return m_scheduler.FlushAndGetSubmitted(); - }(); - - submitted.WaitForFence(); + auto& submitted = GetScheduler().FlushAndGetSubmitted(); + submitted.WaitForFenceOnly(); } void CommandProcessor::BufferWait() { - if (g_current_run_cp != this) { - m_run_mutex.Lock(); - BufferInit(); - - std::array buffers {}; - { - Common::LockGuard lock(m_mutex); - m_scheduler.CopyBuffers(buffers); - } - - for (auto* buf: buffers) { - buf->WaitForFenceAndReset(); - } - - m_run_mutex.Unlock(); - return; - } - BufferInit(); - - Common::LockGuard lock(m_mutex); - m_scheduler.WaitAll(); + GetScheduler().Finish(); } void CommandProcessor::ResetDeCe() { - m_de_counter.mutex.Lock(); - m_de_counter.value = 0; - m_de_counter.cond_var.Signal(); - m_de_counter.mutex.Unlock(); - m_ce_counter.mutex.Lock(); - m_ce_counter.value = 0; - m_ce_counter.cond_var.Signal(); - m_ce_counter.mutex.Unlock(); + m_de_count = 0; + m_ce_count = 0; + m_ce_complete = false; } void CommandProcessor::WaitCe() { - m_de_counter.mutex.Lock(); - auto de_value = m_de_counter.value; - m_de_counter.mutex.Unlock(); - - m_ce_counter.mutex.Lock(); - while (!(m_ce_counter.value > de_value)) { - m_ce_counter.cond_var.Wait(&m_ce_counter.mutex); + if (m_ce_count <= m_de_count && !m_ce_complete) { + SuspendPm4(); } - m_ce_counter.mutex.Unlock(); } void CommandProcessor::WaitDeDiff(uint32_t diff) { - m_ce_counter.mutex.Lock(); - auto ce_value = m_ce_counter.value; - m_ce_counter.mutex.Unlock(); - - m_de_counter.mutex.Lock(); - while (!(ce_value - m_de_counter.value < diff)) { - m_de_counter.cond_var.Wait(&m_de_counter.mutex); + EXIT_IF(m_de_count > m_ce_count); + if (m_ce_count - m_de_count >= diff) { + SuspendPm4(); } - m_de_counter.mutex.Unlock(); } -void CommandProcessor::IncremenetDe() { - BufferFlush(); +void CommandProcessor::IncrementDe() { BufferWait(); - - m_de_counter.mutex.Lock(); - m_de_counter.value++; - m_de_counter.cond_var.Signal(); - m_de_counter.mutex.Unlock(); + m_de_count++; } -void CommandProcessor::IncremenetCe() { - m_ce_counter.mutex.Lock(); - m_ce_counter.value++; - m_ce_counter.cond_var.Signal(); - m_ce_counter.mutex.Unlock(); +void CommandProcessor::IncrementCe() { + m_ce_count++; } void CommandProcessor::WriteConstRam(uint32_t offset, const uint32_t* src, uint32_t dw_num) { - Common::LockGuard lock(m_mutex); - memcpy(m_const_ram + offset / 4, src, static_cast(dw_num) * 4); } void CommandProcessor::DumpConstRam(uint32_t* dst, uint32_t offset, uint32_t dw_num) { - Common::LockGuard lock(m_mutex); - memcpy(dst, m_const_ram + offset / 4, static_cast(dw_num) * 4); } @@ -558,14 +327,6 @@ bool TestWaitRegMemValue(uint64_t value, uint64_t ref, uint64_t mask, uint32_t f return false; } -static void YieldCommandProcessorWait(uint32_t poll_interval_cycles) noexcept { - // PS5 specifies GPU poll cycles, not host microseconds. Yield at the scheduler boundary now; - // a future resumable guest GPU scheduler can replace this one function without touching polling - // or BufferCache coherence. - (void)poll_interval_cycles; - std::this_thread::yield(); -} - template void CommandProcessor::WaitRegMem(uint32_t func, const T* addr, T ref, T mask, uint32_t poll, uint32_t wait_op) { @@ -574,23 +335,9 @@ void CommandProcessor::WaitRegMem(uint32_t func, const T* addr, T ref, T mask, u EXIT("unsupported wait_reg_mem operation: 0x%08" PRIx32 "\n", wait_op); } - const auto addr_value = reinterpret_cast(addr); - const auto log_width = static_cast(sizeof(T) * 2u); - const auto bits = static_cast(sizeof(T) * 8u); - - uint64_t spin_count = 0; - for (;;) { - const auto value = *addr; - if (TestWaitRegMemValue(value, ref, mask, func)) { - break; - } - if ((++spin_count % 100000u) == 0) { - LOGF("\t wait_reg_mem%u still waiting: addr = 0x%016" PRIx64 ", value = 0x%0*" PRIx64 - ", ref = 0x%0*" PRIx64 ", mask = 0x%0*" PRIx64 ", func = %" PRIu32 "\n", - bits, addr_value, log_width, static_cast(value), log_width, - static_cast(ref), log_width, static_cast(mask), func); - } - YieldCommandProcessorWait(poll); + (void)poll; + if (!TestWaitRegMemValue(*addr, ref, mask, func)) { + SuspendPm4(); } } @@ -601,8 +348,6 @@ template void CommandProcessor::WaitRegMem(uint32_t, const uint64_t*, void CommandProcessor::WriteData(uint32_t* dst, const uint32_t* src, uint32_t dw_num, uint32_t write_control) { - Common::LockGuard lock(m_mutex); - const uint32_t dst_sel = ((write_control >> 30u) & 0x1u) | ((write_control >> 7u) & 0x1eu); const uint32_t cache_policy = (write_control >> 25u) & 0x3u; const uint32_t increment = (write_control >> 16u) & 0x1u; @@ -624,7 +369,6 @@ void CommandProcessor::WriteData(uint32_t* dst, const uint32_t* src, uint32_t dw } void CommandProcessor::WriteReferenceClock(uint64_t dst_address, uint32_t num_bytes) { - Common::LockGuard lock(m_mutex); if (dst_address == 0 || (num_bytes != sizeof(uint32_t) && num_bytes != sizeof(uint64_t)) || (dst_address & (num_bytes - 1u)) != 0) { EXIT("invalid reference-clock copy, dst=0x%016" PRIx64 " size=%u\n", dst_address, @@ -642,8 +386,6 @@ void CommandProcessor::DmaData(uint8_t engine, uint8_t dst_sel, uint8_t dst_cach uint64_t src_address_or_offset_or_immediate, uint32_t num_bytes, uint8_t wait_for_previous, uint8_t write_confirm, uint8_t block_engine) { - Common::LockGuard lock(m_mutex); - EXIT_NOT_IMPLEMENTED(engine > 1); if (num_bytes == 0) { return; @@ -673,138 +415,171 @@ void CommandProcessor::DmaData(uint8_t engine, uint8_t dst_sel, uint8_t dst_cach EXIT("unsupported dmaData source selector 0x%02" PRIx8 "\n", src_sel); } -void GraphicsRing::Submit(OwnedCmdBuffer draw_buffer, OwnedCmdBuffer const_buffer, int handle, - int index, int flip_mode, int64_t flip_arg, - bool trigger_agc_interrupt_on_done) { - EXIT_IF(m_cp == nullptr); - - Common::LockGuard lock(m_mutex); - - if (m_done) { - while (!m_idle) { - m_idle_cond_var.Wait(&m_mutex); - } - m_done = false; - - m_cp->Reset(); - } - - auto& buf = m_cmd_batches.emplace_back(); - buf.draw_buffer = std::move(draw_buffer); - buf.const_buffer = std::move(const_buffer); - buf.flip.handle = handle; - buf.flip.index = index; - buf.flip.flip_mode = flip_mode; - buf.flip.flip_arg = flip_arg; - buf.trigger_agc_interrupt_on_done = trigger_agc_interrupt_on_done; - - m_idle = false; - m_cond_var.Signal(); +void Gpu::Enqueue(Submission submission) { + EXIT_IF(submission.queue_id >= QueueCount); + Common::LockGuard lock(m_queue_mutex); + m_queues[submission.queue_id].push_back(std::move(submission)); + m_submission_count++; + m_work_available.Signal(); } -void GraphicsRing::SubmitFlipPreparation() { - EXIT_IF(m_cp == nullptr); - Common::LockGuard lock(m_mutex); - - if (m_done) { - while (!m_idle) { - m_idle_cond_var.Wait(&m_mutex); - } - m_done = false; - m_cp->Reset(); - } - - auto& batch = m_cmd_batches.emplace_back(); - batch.prepare_cpu_flip = true; - m_idle = false; - m_cond_var.Signal(); -} - -void GraphicsRing::Done() { - Common::LockGuard lock(m_mutex); - if (m_done) { - while (!m_idle) { - m_idle_cond_var.Wait(&m_mutex); - } - } - m_done = true; -} - -void GraphicsRing::WaitForIdle() { - Common::LockGuard lock(m_mutex); - while (!m_idle) { - m_idle_cond_var.Wait(&m_mutex); +void Gpu::WaitForIdle() { + Common::LockGuard lock(m_queue_mutex); + while (m_processing || m_submission_count != 0) { + m_idle.Wait(&m_queue_mutex); } } -bool GraphicsRing::IsIdle() { - Common::LockGuard lock(m_mutex); - return m_idle; -} - -GraphicsRing::CmdBatch GraphicsRing::GetCmdBatch() { - Common::LockGuard lock(m_mutex); - - while (m_cmd_batches.empty()) { - m_idle = true; - m_idle_cond_var.Signal(); - - m_cond_var.Wait(&m_mutex); - } - - m_idle = false; - - CmdBatch buf = std::move(m_cmd_batches.front()); - m_cmd_batches.pop_front(); - - return buf; -} - -void GraphicsRing::ThreadBatchRun(void* data) { - EXIT_IF(data == nullptr); - - static std::atomic_uint64_t seq = 0; - - auto* ring = static_cast(data); - auto* cp = ring->m_cp; - - EXIT_IF(ring == nullptr); - EXIT_IF(cp == nullptr); +void Gpu::ThreadRun(void* data) { + auto* gpu = static_cast(data); + EXIT_IF(gpu == nullptr); + KYTY_PROFILER_THREAD("Thread_Gpu"); for (;;) { - CmdBatch buf = ring->GetCmdBatch(); - - cp->RunLock(); + Submission submission; { - cp->BufferInit(); - cp->SetSubmitId(++seq); - if (buf.prepare_cpu_flip) { - cp->PrepareCpuFlip(); - } else { - cp->ResetDeCe(); - cp->SetFlip(buf.flip); - ring->m_draw_job.Execute( - [cp, buf] { cp->Run(buf.draw_buffer.data, buf.draw_buffer.num_dw); }); - ring->m_constant_job.Execute( - [cp, buf] { cp->Run(buf.const_buffer.data, buf.const_buffer.num_dw); }); - ring->m_draw_job.Wait(); - ring->m_constant_job.Wait(); - cp->BufferFlush(); - if (buf.trigger_agc_interrupt_on_done) { - Sync::TriggerEopEvent(0); + Common::LockGuard lock(gpu->m_queue_mutex); + int selected_queue = -1; + while (selected_queue < 0) { + while (gpu->m_submission_count == 0) { + gpu->m_processing = false; + gpu->m_idle.Signal(); + gpu->m_work_available.Wait(&gpu->m_queue_mutex); + } + for (uint32_t offset = 0; offset < QueueCount; offset++) { + const auto id = (gpu->m_next_queue + offset) % QueueCount; + if (!gpu->m_queues[id].empty() && !gpu->m_queues[id].front().blocked) { + selected_queue = static_cast(id); + break; + } + } + if (selected_queue < 0) { + gpu->m_processing = false; + gpu->m_work_available.WaitFor(&gpu->m_queue_mutex, 100); + for (auto& queue: gpu->m_queues) { + if (!queue.empty()) { + queue.front().blocked = false; + } + } + } + } + auto& queue = gpu->m_queues[static_cast(selected_queue)]; + submission = std::move(queue.front()); + queue.pop_front(); + gpu->m_submission_count--; + gpu->m_next_queue = (static_cast(selected_queue) + 1) % QueueCount; + gpu->m_processing = true; + } + + const bool complete = gpu->Process(submission); + + Common::LockGuard lock(gpu->m_queue_mutex); + if (!complete) { + submission.blocked = true; + gpu->m_queues[submission.queue_id].push_front(std::move(submission)); + gpu->m_submission_count++; + } else { + for (auto& queue: gpu->m_queues) { + if (!queue.empty()) { + queue.front().blocked = false; } } } - cp->RunUnlock(); + gpu->m_processing = false; + if (gpu->m_submission_count == 0) { + gpu->m_idle.Signal(); + } } } +bool Gpu::Process(Submission& submission) { + auto& cp = GetProcessor(submission.queue_id); + const bool first_slice = !submission.started; + + if (first_slice && submission.reset_processor) { + cp.Reset(); + } + + if (first_slice) { + submission.started = true; + cp.SetSubmitId(++m_submit_id); + cp.ResetDeCe(); + cp.SetFlip({}); + } + + cp.BufferInit(); + bool complete = true; + + switch (submission.type) { + case SubmissionType::Graphics: { + bool progressed = false; + submission.constant_complete |= submission.constant_commands.Empty(); + for (;;) { + bool round_progress = false; + if (!submission.constant_complete) { + submission.constant_complete = + cp.Process( + submission.constant_execution, submission.constant_commands.Data(), + submission.constant_commands.Size()) == Pm4ProcessResult::Complete; + round_progress |= submission.constant_execution.MadeProgress(); + } + cp.SetCeComplete(submission.constant_complete); + if (!submission.command_complete) { + submission.command_complete = + cp.Process(submission.command_execution, submission.commands.Data(), + submission.commands.Size()) == Pm4ProcessResult::Complete; + round_progress |= submission.command_execution.MadeProgress(); + } + progressed |= round_progress; + complete = submission.command_complete && submission.constant_complete; + if (complete || !round_progress) { + break; + } + } + if (progressed) { + cp.BufferFlush(); + } + if (complete && submission.trigger_agc_interrupt_on_done) { + Sync::TriggerEopEvent(0); + } + break; + } + case SubmissionType::Compute: { + const auto num_dw = submission.commands.Size(); + auto* buffer = submission.commands.Data(); + static uint32_t compute_batch_log_count = 0; + if (first_slice && num_dw <= 128 && compute_batch_log_count++ < 32) { + LOGF("compute direct batch: data=0x%016" PRIx64 ", num_dw=%" PRIu32 "\n", + reinterpret_cast(buffer), num_dw); + for (uint32_t i = 0; i < std::min(num_dw, 16); i++) { + LOGF("\t compute[%02" PRIu32 "] = 0x%08" PRIx32 "\n", i, buffer[i]); + } + } + if (first_slice) { + GraphicsDbgDumpDcb("cc", num_dw, buffer); + } + complete = cp.Process(submission.command_execution, buffer, num_dw) == + Pm4ProcessResult::Complete; + if (submission.command_execution.MadeProgress()) { + cp.BufferFlush(); + } + if (complete && submission.trigger_agc_interrupt_on_done) { + Sync::TriggerAgcUserInterrupt(); + } + break; + } + case SubmissionType::FlipPreparation: cp.PrepareCpuFlip(); break; + } + + return complete; +} + void Gpu::PauseSubmissions() { if (g_gpu_mutex_owned) { EXIT("GPU submissions are already paused by this thread\n"); } g_gpu_mutex_owned = true; - m_mutex.Lock(); + m_submission_mutex.Lock(); WaitLocked(); LabelDrain(); } @@ -813,230 +588,179 @@ void Gpu::ResumeSubmissions() { if (!g_gpu_mutex_owned) { EXIT("GPU submissions resumed without an active pause\n"); } - m_mutex.Unlock(); + m_submission_mutex.Unlock(); g_gpu_mutex_owned = false; } -void ComputeRing::ThreadRun(void* data) { - EXIT_IF(data == nullptr); - - auto* ring = static_cast(data); - auto* cp = ring->m_cp; - - EXIT_IF(ring == nullptr); - EXIT_IF(cp == nullptr); - - KYTY_PROFILER_THREAD("Thread_Compute"); - - ring->m_mutex.Lock(); - - for (;;) { - while (ring->m_direct_batches.empty()) { - ring->m_idle = true; - ring->m_idle_cond_var.Signal(); - ring->m_cond_var.Wait(&ring->m_mutex); - } - - ring->m_idle = false; - - uint32_t* buffer = nullptr; - uint32_t num_dw = 0; - bool trigger_agc_interrupt_on_done = false; - DirectBatch direct_batch; - - direct_batch = std::move(ring->m_direct_batches.front()); - ring->m_direct_batches.pop_front(); - buffer = direct_batch.buffer.data; - num_dw = direct_batch.buffer.num_dw; - trigger_agc_interrupt_on_done = direct_batch.trigger_agc_interrupt_on_done; - - static std::atomic compute_batch_log_count {0}; - if (num_dw <= 128 && buffer != nullptr && compute_batch_log_count.fetch_add(1) < 32) { - LOGF("compute direct batch: data=0x%016" PRIx64 ", num_dw=%" PRIu32 "\n", - reinterpret_cast(buffer), num_dw); - for (uint32_t i = 0; i < std::min(num_dw, 16); i++) { - LOGF("\t compute[%02" PRIu32 "] = 0x%08" PRIx32 "\n", i, buffer[i]); - } - } - - ring->m_mutex.Unlock(); - - cp->RunLock(); - { - cp->BufferInit(); - cp->ResetDeCe(); - - GraphicsDbgDumpDcb("cc", num_dw, buffer); - - cp->Run(buffer, num_dw); - - cp->BufferFlush(); - if (trigger_agc_interrupt_on_done) { - Sync::TriggerAgcUserInterrupt(); - } - } - cp->RunUnlock(); - - ring->m_mutex.Lock(); +Pm4ProcessResult CommandProcessor::Process(Pm4Execution& execution, uint32_t* buffer, + uint32_t size_dw) { + KYTY_PROFILER_BLOCK("CommandProcessor::Process"); + EXIT_IF(g_current_execution != nullptr); + if (execution.m_buffer_stack.empty() && size_dw != 0) { + execution.m_buffer_stack.push_back({buffer, size_dw, size_dw, 0}); } -} + execution.m_suspended = false; + execution.m_made_progress = false; -void ComputeRing::Submit(OwnedCmdBuffer buffer, bool trigger_agc_interrupt_on_done) { - Common::LockGuard lock(m_mutex); - - EXIT_IF(buffer.data == nullptr); - EXIT_IF(buffer.num_dw == 0); - - if (m_done) { - m_done = false; - } - - DirectBatch batch {}; - batch.buffer = std::move(buffer); - batch.trigger_agc_interrupt_on_done = trigger_agc_interrupt_on_done; - m_direct_batches.push_back(std::move(batch)); - - m_idle = false; - m_cond_var.Signal(); -} - -void ComputeRing::Done() { - Common::LockGuard lock(m_mutex); - m_done = true; -} - -void ComputeRing::WaitForIdle() { - Common::LockGuard lock(m_mutex); - while (!m_idle) { - m_idle_cond_var.Wait(&m_mutex); - } -} - -bool ComputeRing::IsIdle() { - Common::LockGuard lock(m_mutex); - return m_idle; -} - -void CommandProcessor::Run(uint32_t* data, uint32_t num_dw) { - KYTY_PROFILER_BLOCK("CommandProcessor::Run"); - - struct RunScope { - explicit RunScope(CommandProcessor& cp): prev(g_current_run_cp) { g_current_run_cp = &cp; } - ~RunScope() { g_current_run_cp = prev; } - - CommandProcessor* prev; - } run_scope(*this); - - auto* cmd = data; - auto dw = num_dw; - for (;;) { - if (dw == 0) { - break; + struct ExecutionScope { + ExecutionScope(CommandProcessor& processor, Pm4Execution& execution) + : previous_processor(g_current_processor), previous_execution(g_current_execution) { + g_current_processor = &processor; + g_current_execution = &execution; + } + ~ExecutionScope() { + g_current_processor = previous_processor; + g_current_execution = previous_execution; } - EXIT_NOT_IMPLEMENTED(dw > num_dw); + CommandProcessor* previous_processor; + Pm4Execution* previous_execution; + } execution_scope(*this, execution); - auto cmd_id = *cmd++; + ProcessPm4(execution, 0); + return execution.m_buffer_stack.empty() ? Pm4ProcessResult::Complete + : Pm4ProcessResult::Blocked; +} - if (cmd_id == 0x80000000u) { - dw--; +void CommandProcessor::ProcessIndirectBuffer(uint32_t* buffer, uint32_t size_dw) { + EXIT_IF(g_current_execution == nullptr); + if (size_dw == 0) { + return; + } + auto& execution = *g_current_execution; + const auto stop_depth = execution.m_buffer_stack.size(); + execution.m_buffer_stack.push_back({buffer, size_dw, size_dw, 0}); + ProcessPm4(execution, stop_depth); +} + +void CommandProcessor::SuspendPm4() { + EXIT_IF(g_current_execution == nullptr); + g_current_execution->m_suspended = true; +} + +void CommandProcessor::ProcessPm4(Pm4Execution& execution, size_t stop_depth) { + while (execution.m_buffer_stack.size() > stop_depth) { + const auto buffer_index = execution.m_buffer_stack.size() - 1; + auto& cursor = execution.m_buffer_stack[buffer_index]; + if (cursor.deferred_advance_dw != 0) { + EXIT_IF(cursor.deferred_advance_dw > cursor.remaining_dw); + cursor.next_packet += cursor.deferred_advance_dw; + cursor.remaining_dw -= cursor.deferred_advance_dw; + cursor.deferred_advance_dw = 0; + execution.m_made_progress = true; + continue; + } + if (cursor.remaining_dw == 0) { + execution.m_buffer_stack.pop_back(); continue; } - EXIT_NOT_IMPLEMENTED(dw < 2); + auto* const packet = cursor.next_packet; + const auto remaining_dw = cursor.remaining_dw; + const auto total_dw = cursor.total_dw; + const auto packet_header = packet[0]; + const auto opcode = (packet_header >> 8u) & 0xffu; + EXIT_NOT_IMPLEMENTED(remaining_dw > total_dw); + + if (packet_header == 0x80000000u) { + cursor.next_packet++; + cursor.remaining_dw--; + execution.m_made_progress = true; + continue; + } + + EXIT_NOT_IMPLEMENTED(remaining_dw < 2); - auto op = (cmd_id >> 8u) & 0xffu; if (GraphicsRunDebugDumpEnabled()) { LOGF("CP packet: offset=0x%05" PRIx32 " cmd_id=0x%08" PRIx32 " op=0x%02" PRIx32 " len=%" PRIu32 "\n", - num_dw - dw, cmd_id, op, KYTY_PM4_LEN(cmd_id)); + total_dw - remaining_dw, packet_header, opcode, KYTY_PM4_LEN(packet_header)); } - if ((cmd_id & 1u) != 0 && ShouldSkipPredicatedPackets()) { - auto packet_len = KYTY_PM4_LEN(cmd_id); - EXIT_NOT_IMPLEMENTED(packet_len == 0 || packet_len > dw); + if ((packet_header & 1u) != 0 && ShouldSkipPredicatedPackets()) { + auto packet_dw = KYTY_PM4_LEN(packet_header); + EXIT_NOT_IMPLEMENTED(packet_dw == 0 || packet_dw > remaining_dw); static std::atomic skip_log_count {0}; if (skip_log_count.fetch_add(1) < 2048) { LOGF("\t predicated skip: op=0x%02" PRIx32 ", r=0x%02" PRIx32 ", len=%" PRIu32 ", packet=0x%016" PRIx64 ", cmd_id=0x%08" PRIx32 "\n", - op, KYTY_PM4_R(cmd_id), packet_len, reinterpret_cast(cmd - 1), - cmd_id); + opcode, KYTY_PM4_R(packet_header), packet_dw, + reinterpret_cast(packet), packet_header); } - if (op == Pm4::IT_NOP && KYTY_PM4_R(cmd_id) == Pm4::R_RELEASE_MEM && packet_len >= 7) { + if (opcode == Pm4::IT_NOP && KYTY_PM4_R(packet_header) == Pm4::R_RELEASE_MEM && + packet_dw >= 7) { static std::atomic log_count {0}; if (log_count.fetch_add(1) < 128) { - const auto dst = cmd[2] | (static_cast(cmd[3]) << 32u); - const auto val = cmd[4] | (static_cast(cmd[5]) << 32u); + const auto dst = packet[3] | (static_cast(packet[4]) << 32u); + const auto val = packet[5] | (static_cast(packet[6]) << 32u); LOGF("\t predicated skip: R_RELEASE_MEM dst=0x%016" PRIx64 ", value=0x%016" PRIx64 ", action=0x%08" PRIx32 ", gcr/data/int=0x%08" PRIx32 "\n", - dst, val, cmd[0], cmd[1]); + dst, val, packet[1], packet[2]); } } - cmd += packet_len - 1u; - dw -= packet_len; + cursor.next_packet += packet_dw; + cursor.remaining_dw -= packet_dw; + execution.m_made_progress = true; continue; } - auto pfunc = g_cp_op_func[op]; + auto handler = g_cp_op_func[opcode]; - if (pfunc == nullptr) { - const auto offset = num_dw - dw; + if (handler == nullptr) { + const auto offset = total_dw - remaining_dw; LOGF("unknown PM4 packet: data=0x%016" PRIx64 ", num_dw=%" PRIu32 ", offset=0x%05" PRIx32 ", current=0x%016" PRIx64 "\n", - reinterpret_cast(data), num_dw, offset, - reinterpret_cast(cmd - 1)); - const auto dump_begin = (offset > 8 ? offset - 8 : 0); - const auto dump_end = std::min(num_dw, offset + 16); + reinterpret_cast(packet - offset), total_dw, offset, + reinterpret_cast(packet)); + const auto dump_begin = (offset > 8 ? offset - 8 : 0); + const auto dump_end = std::min(total_dw, offset + 16); + auto* const base = packet - offset; for (uint32_t i = dump_begin; i < dump_end; i++) { - LOGF("\t%05" PRIx32 "%s %08" PRIx32 "\n", i, (i == offset ? ":" : " "), data[i]); + LOGF("\t%05" PRIx32 "%s %08" PRIx32 "\n", i, (i == offset ? ":" : " "), base[i]); } - EXIT("unknown op\n\t%05" PRIx32 ":\n\tcmd_id = %08" PRIx32 "\n", num_dw - dw, cmd_id); + EXIT("unknown op\n\t%05" PRIx32 ":\n\tcmd_id = %08" PRIx32 "\n", + total_dw - remaining_dw, packet_header); } - auto s = pfunc(*this, cmd_id & ~1u, cmd, dw, num_dw); - - // LOGF("\t %05" PRIx32 ": %u\n", num_dw - dw, s); - - cmd += s; - dw -= s + 1; + const auto packet_dw = + handler(*this, packet_header & ~1u, packet + 1, remaining_dw, total_dw) + 1; + EXIT_IF(packet_dw > remaining_dw); + if (execution.m_suspended) { + if (execution.m_buffer_stack.size() > buffer_index + 1) { + execution.m_buffer_stack[buffer_index].deferred_advance_dw = packet_dw; + } + return; + } + EXIT_IF(execution.m_buffer_stack.size() != buffer_index + 1); + execution.m_buffer_stack[buffer_index].next_packet += packet_dw; + execution.m_buffer_stack[buffer_index].remaining_dw -= packet_dw; + execution.m_made_progress = true; } } void CommandProcessor::SetIndexType(uint32_t index_type_and_size) { - Common::LockGuard lock(m_mutex); - m_index_type_and_size = index_type_and_size & 0x3u; } void CommandProcessor::SetIndexBaseAddress(uint64_t index_base_addr) { - Common::LockGuard lock(m_mutex); - m_index_base_addr = index_base_addr; } void CommandProcessor::SetIndexBufferSize(uint32_t index_buffer_size) { - Common::LockGuard lock(m_mutex); - m_index_buffer_size = index_buffer_size; } void CommandProcessor::SetDrawIndirectArgsBaseAddress(uint64_t draw_indirect_args_base_addr) { - Common::LockGuard lock(m_mutex); - m_draw_indirect_args_base_addr = draw_indirect_args_base_addr; } void CommandProcessor::SetDispatchIndirectArgsBaseAddress( uint64_t dispatch_indirect_args_base_addr) { - Common::LockGuard lock(m_mutex); - m_dispatch_indirect_args_base_addr = dispatch_indirect_args_base_addr; } void CommandProcessor::SetNumInstances(uint32_t num_instances) { - Common::LockGuard lock(m_mutex); - if (num_instances == 0) { num_instances = 1; } @@ -1050,8 +774,6 @@ void CommandProcessor::SetPredication(uint32_t condition, uint32_t op, uint32_t BufferFlushAndWait(); } - Common::LockGuard lock(m_mutex); - (void)count_in_dwords; switch (op) { @@ -1084,8 +806,6 @@ void CommandProcessor::DrawIndex(uint32_t index_count, const void* index_addr, u uint32_t type, uint32_t instance_count, const void* object_ids, uint32_t render_target_slice_offset, int32_t vertex_offset_add, uint32_t first_instance) { - Common::LockGuard lock(m_mutex); - CheckBuffer(); if (instance_count == 0) { @@ -1110,8 +830,6 @@ void CommandProcessor::DrawIndex(uint32_t index_count, const void* index_addr, u void CommandProcessor::DrawIndexOffset(uint32_t index_offset, uint32_t index_count, uint32_t flags) { - Common::LockGuard lock(m_mutex); - CheckBuffer(); uint64_t index_size = 0; @@ -1317,8 +1035,6 @@ void CommandProcessor::DispatchDirect(uint32_t thread_group_x, uint32_t thread_g uint32_t local_z = 1; { - Common::LockGuard lock(m_mutex); - CheckBuffer(); frame_num = GraphicsRunGetFrameNum(); if (GraphicsRunDebugDumpEnabled()) { @@ -1399,8 +1115,6 @@ void CommandProcessor::DispatchIndirect(uint32_t data_offset, uint32_t mode) { void CommandProcessor::DrawIndexAuto(uint32_t index_count, uint32_t flags, uint32_t render_target_slice_offset, uint32_t instance_count, uint32_t first_vertex, uint32_t first_instance) { - Common::LockGuard lock(m_mutex); - CheckBuffer(); RenderDrawIndexAuto(m_submit_id, CurrentBuffer(), index_count, flags, @@ -1422,8 +1136,6 @@ void CommandProcessor::WriteAtEndOfPipe(uint32_t cache_policy, uint32_t event_wr uint32_t interrupt_context_id) { static_assert(sizeof(T) == sizeof(uint32_t) || sizeof(T) == sizeof(uint64_t)); - Common::LockGuard lock(m_mutex); - CheckBuffer(); if (GraphicsRunDebugDumpEnabled()) { @@ -1644,32 +1356,24 @@ void CommandProcessor::WriteAtEndOfPipe64(uint32_t cache_policy, uint32_t event_ } void CommandProcessor::MemoryBarrier() { - Common::LockGuard lock(m_mutex); - CheckBuffer(); GraphicsRenderMemoryBarrier(CurrentBuffer()); } void CommandProcessor::TriggerEopEventAtEndOfPipe(uint32_t interrupt_context_id) { - Common::LockGuard lock(m_mutex); - CheckBuffer(); Sync::TriggerEopEventAtEndOfPipe(CurrentBuffer(), interrupt_context_id); } void CommandProcessor::RenderTextureBarrier(uint64_t vaddr, uint64_t size) { - Common::LockGuard lock(m_mutex); - CheckBuffer(); GraphicsRenderTextureBarrier(CurrentBuffer(), vaddr, size); } void CommandProcessor::DepthStencilBarrier(uint64_t vaddr, uint64_t size) { - Common::LockGuard lock(m_mutex); - CheckBuffer(); GraphicsRenderDepthStencilBarrier(CurrentBuffer(), vaddr, size); @@ -1730,8 +1434,6 @@ void CommandProcessor::TriggerEvent(uint32_t event_type, uint32_t event_index) { } void CommandProcessor::Flip() { - Common::LockGuard lock(m_mutex); - CheckBuffer(); if (GraphicsRunDebugDumpEnabled()) { @@ -1743,12 +1445,10 @@ void CommandProcessor::Flip() { m_flip.flip_mode, m_flip.flip_arg); Sync::WriteAtEndOfPipeOnlyFlip(m_submit_id, command, m_flip.handle, m_flip.index, m_flip.flip_mode, m_flip.flip_arg, request); - m_scheduler.Flush(); + GetScheduler().Flush(); } void CommandProcessor::Flip(void* dst_gpu_addr, uint32_t value) { - Common::LockGuard lock(m_mutex); - CheckBuffer(); if (GraphicsRunDebugDumpEnabled()) { @@ -1765,13 +1465,11 @@ void CommandProcessor::Flip(void* dst_gpu_addr, uint32_t value) { Sync::WriteAtEndOfPipeWithFlip32(m_submit_id, command, static_cast(dst_gpu_addr), value, m_flip.handle, m_flip.index, m_flip.flip_mode, m_flip.flip_arg, request); - m_scheduler.Flush(); + GetScheduler().Flush(); } void CommandProcessor::FlipWithInterrupt(uint32_t eop_event_type, uint32_t cache_action, void* dst_gpu_addr, uint32_t value) { - Common::LockGuard lock(m_mutex); - CheckBuffer(); if (GraphicsRunDebugDumpEnabled()) { @@ -1793,29 +1491,27 @@ void CommandProcessor::FlipWithInterrupt(uint32_t eop_event_type, uint32_t cache Sync::WriteAtEndOfPipeWithInterruptWriteBackFlip32( m_submit_id, command, static_cast(dst_gpu_addr), value, m_flip.handle, m_flip.index, m_flip.flip_mode, m_flip.flip_arg, request); - m_scheduler.Flush(); + GetScheduler().Flush(); } void CommandProcessor::PrepareCpuFlip() { - Common::LockGuard lock(m_mutex); CheckBuffer(); - if (g_current_run_cp != nullptr) { + if (g_current_processor != nullptr) { EXIT("invalid graphics-thread CPU flip preparation\n"); } - struct RunScope { - explicit RunScope(CommandProcessor& cp) { g_current_run_cp = &cp; } - ~RunScope() { g_current_run_cp = nullptr; } + struct ProcessorScope { + explicit ProcessorScope(CommandProcessor& processor) { g_current_processor = &processor; } + ~ProcessorScope() { g_current_processor = nullptr; } }; - RunScope run_scope(*this); + ProcessorScope processor_scope(*this); auto prepared_id = Presentation::DisplayBufferPrepareNextFlipOnGpu(CurrentBuffer()); - m_scheduler.Flush(); + GetScheduler().Flush(); Presentation::DisplayBufferCompleteFlipFromGpu(prepared_id); } void CommandProcessor::SynchronizeGpu() { - Common::LockGuard lock(m_mutex); - FinishCommandProcessors(); + GetScheduler().FinishCurrent(); } void GraphicsRunSubmit(uint32_t* cmd_draw_buffer, uint32_t num_draw_dw, uint32_t* cmd_const_buffer, @@ -1847,7 +1543,8 @@ void GraphicsRunWait() { } GraphicsRunSubmissionLock::GraphicsRunSubmissionLock() { - if (g_gpu == nullptr || g_current_run_cp != nullptr || g_submission_pause_depth == UINT32_MAX) { + if (g_gpu == nullptr || g_current_processor != nullptr || + g_submission_pause_depth == UINT32_MAX) { EXIT("cannot acquire GPU submission lock in the current state\n"); } if (g_submission_pause_depth++ == 0) { @@ -1877,18 +1574,18 @@ int GraphicsRunGetFrameNum() { } bool GraphicsRunIsCommandProcessorThread() noexcept { - return g_current_run_cp != nullptr; + return g_current_processor != nullptr; } CommandProcessor* GraphicsRunCurrentCommandProcessor() noexcept { - return g_current_run_cp; + return g_current_processor; } -void GraphicsRunFinishCommandProcessors() { - if (g_current_run_cp == nullptr) { +void GraphicsRunFinishScheduler() { + if (g_current_processor == nullptr) { EXIT("GPU readback finish requires a command-processor thread\n"); } - g_current_run_cp->FinishReadbackTransaction(); + g_current_processor->FinishReadbackTransaction(); } bool GraphicsRunSubmissionLockHeld() noexcept { diff --git a/src/graphics/guest_gpu/graphicsRun.h b/src/graphics/guest_gpu/graphicsRun.h index c4da95fca..69c905676 100644 --- a/src/graphics/guest_gpu/graphicsRun.h +++ b/src/graphics/guest_gpu/graphicsRun.h @@ -27,7 +27,7 @@ void GraphicsRunDone(); int GraphicsRunGetFrameNum(); [[nodiscard]] bool GraphicsRunIsCommandProcessorThread() noexcept; [[nodiscard]] CommandProcessor* GraphicsRunCurrentCommandProcessor() noexcept; -void GraphicsRunFinishCommandProcessors(); +void GraphicsRunFinishScheduler(); [[nodiscard]] bool GraphicsRunSubmissionLockHeld() noexcept; [[nodiscard]] bool GraphicsRunGpuLockHeld() noexcept; } // namespace Libs::Graphics diff --git a/src/graphics/host_gpu/renderer/bufferCache.cpp b/src/graphics/host_gpu/renderer/bufferCache.cpp index 5b5388791..d355ef222 100644 --- a/src/graphics/host_gpu/renderer/bufferCache.cpp +++ b/src/graphics/host_gpu/renderer/bufferCache.cpp @@ -554,7 +554,7 @@ bool BufferCache::InvalidateMemory(PageFaultAccess access, uint64_t vaddr, uint6 return action == CpuFaultAction::Continue; } if (GraphicsRunIsCommandProcessorThread()) { - GraphicsRunFinishCommandProcessors(); + GraphicsRunFinishScheduler(); } m_readback->Request(access, vaddr, size); return true; diff --git a/src/graphics/host_gpu/renderer/commandScheduler.cpp b/src/graphics/host_gpu/renderer/commandScheduler.cpp new file mode 100644 index 000000000..45e0f1f6d --- /dev/null +++ b/src/graphics/host_gpu/renderer/commandScheduler.cpp @@ -0,0 +1,92 @@ +#include "graphics/host_gpu/renderer/commandScheduler.h" + +#include "common/assert.h" + +namespace Libs::Graphics { + +void CommandScheduler::Begin(HW::Context& registers, HW::UserConfig& user_config, + HW::Shader& shaders) { + m_registers = ®isters; + m_user_config = &user_config; + m_shaders = &shaders; + + if (!Active()) { + for (auto& buffer: m_buffers) { + buffer = std::make_unique(); + } + m_current = 0; + } + + BindCurrent(); + if (!m_recording) { + Current().Begin(); + m_recording = true; + } +} + +void CommandScheduler::Flush() { + SubmitCurrent(); + BeginNext(); +} + +CommandBuffer& CommandScheduler::FlushAndGetSubmitted() { + auto& submitted = SubmitCurrent(); + BeginNext(); + return submitted; +} + +void CommandScheduler::Finish() { + CheckActive(); + if (m_recording) { + SubmitCurrent(); + } + for (auto& buffer: m_buffers) { + buffer->WaitForFenceAndReset(); + } + BindCurrent(); + Current().Begin(); + m_recording = true; +} + +void CommandScheduler::FinishCurrent() { + auto& submitted = SubmitCurrent(); + submitted.WaitForFenceAndReset(); + submitted.Begin(); + m_recording = true; +} + +void CommandScheduler::CheckActive() const { + EXIT_IF(!Active() || m_current >= BufferCount); +} + +RenderCommandBuffer& CommandScheduler::Current() const { + CheckActive(); + EXIT_IF(m_buffers[m_current] == nullptr); + return *m_buffers[m_current]; +} + +void CommandScheduler::BindCurrent() const { + EXIT_IF(m_registers == nullptr || m_user_config == nullptr || m_shaders == nullptr); + Current().Bind(*m_registers, *m_user_config, *m_shaders); +} + +CommandBuffer& CommandScheduler::SubmitCurrent() { + CheckActive(); + EXIT_IF(!m_recording); + auto& submitted = Current(); + submitted.End(); + submitted.Execute(); + m_recording = false; + return submitted; +} + +void CommandScheduler::BeginNext() { + EXIT_IF(m_recording); + m_current = (m_current + 1) % BufferCount; + Current().WaitForFenceAndReset(); + BindCurrent(); + Current().Begin(); + m_recording = true; +} + +} // namespace Libs::Graphics diff --git a/src/graphics/host_gpu/renderer/commandScheduler.h b/src/graphics/host_gpu/renderer/commandScheduler.h new file mode 100644 index 000000000..3dc2add96 --- /dev/null +++ b/src/graphics/host_gpu/renderer/commandScheduler.h @@ -0,0 +1,45 @@ +#ifndef EMULATOR_SRC_GRAPHICS_HOST_GPU_RENDERER_COMMANDSCHEDULER_H_ +#define EMULATOR_SRC_GRAPHICS_HOST_GPU_RENDERER_COMMANDSCHEDULER_H_ + +#include "common/common.h" +#include "graphics/host_gpu/renderer/render.h" + +#include +#include + +namespace Libs::Graphics { + +class CommandScheduler { +public: + static constexpr int BufferCount = 8; + + CommandScheduler() = default; + ~CommandScheduler() = default; + KYTY_CLASS_NO_COPY(CommandScheduler); + + void Begin(HW::Context& registers, HW::UserConfig& user_config, HW::Shader& shaders); + void Flush(); + CommandBuffer& FlushAndGetSubmitted(); + void Finish(); + void FinishCurrent(); + + [[nodiscard]] bool Active() const noexcept { return m_current >= 0; } + void CheckActive() const; + RenderCommandBuffer& Current() const; + +private: + void BindCurrent() const; + CommandBuffer& SubmitCurrent(); + void BeginNext(); + + std::array, BufferCount> m_buffers; + int m_current = -1; + bool m_recording = false; + HW::Context* m_registers = nullptr; + HW::UserConfig* m_user_config = nullptr; + HW::Shader* m_shaders = nullptr; +}; + +} // namespace Libs::Graphics + +#endif // EMULATOR_SRC_GRAPHICS_HOST_GPU_RENDERER_COMMANDSCHEDULER_H_ diff --git a/src/graphics/host_gpu/renderer/render.h b/src/graphics/host_gpu/renderer/render.h index 44777663e..f536bb513 100644 --- a/src/graphics/host_gpu/renderer/render.h +++ b/src/graphics/host_gpu/renderer/render.h @@ -129,17 +129,22 @@ private: class RenderCommandBuffer final: public CommandBuffer { public: - RenderCommandBuffer(HW::Context& registers, HW::UserConfig& user_config, HW::Shader& shaders) - : m_registers(registers), m_user_config(user_config), m_shaders(shaders) {} + RenderCommandBuffer() = default; - [[nodiscard]] HW::Context& GetRegisters() const noexcept { return m_registers; } - [[nodiscard]] HW::UserConfig& GetUserConfig() const noexcept { return m_user_config; } - [[nodiscard]] HW::Shader& GetShaders() const noexcept { return m_shaders; } + void Bind(HW::Context& registers, HW::UserConfig& user_config, HW::Shader& shaders) noexcept { + m_registers = ®isters; + m_user_config = &user_config; + m_shaders = &shaders; + } + + [[nodiscard]] HW::Context& GetRegisters() const noexcept { return *m_registers; } + [[nodiscard]] HW::UserConfig& GetUserConfig() const noexcept { return *m_user_config; } + [[nodiscard]] HW::Shader& GetShaders() const noexcept { return *m_shaders; } private: - HW::Context& m_registers; - HW::UserConfig& m_user_config; - HW::Shader& m_shaders; + HW::Context* m_registers = nullptr; + HW::UserConfig* m_user_config = nullptr; + HW::Shader* m_shaders = nullptr; }; void RenderDrawIndex(uint64_t submit_id, RenderCommandBuffer& buffer, uint32_t index_type_and_size, diff --git a/src/graphics/host_gpu/renderer/renderContext.h b/src/graphics/host_gpu/renderer/renderContext.h index 5e3d68e3d..6de6bbadc 100644 --- a/src/graphics/host_gpu/renderer/renderContext.h +++ b/src/graphics/host_gpu/renderer/renderContext.h @@ -6,6 +6,7 @@ #include "common/common.h" #include "common/threads.h" #include "graphics/host_gpu/renderer/bufferCache.h" +#include "graphics/host_gpu/renderer/commandScheduler.h" #include "graphics/host_gpu/renderer/descriptorCache.h" #include "graphics/host_gpu/renderer/framebufferCache.h" #include "graphics/host_gpu/renderer/gdsBuffer.h" @@ -30,6 +31,7 @@ public: [[nodiscard]] GraphicContext& GetGraphics() const noexcept { return m_graphics; } Common::Mutex& GetMutex() { return m_mutex; } + CommandScheduler& GetCommandScheduler() { return m_command_scheduler; } PipelineCache& GetPipelineCache() { return m_pipeline_cache; } DescriptorCache& GetDescriptorCache() { return m_descriptor_cache; } FramebufferCache& GetFramebufferCache() { return m_framebuffer_cache; } @@ -52,6 +54,7 @@ private: GraphicContext& m_graphics; Common::Mutex m_mutex; + CommandScheduler m_command_scheduler; PipelineCache m_pipeline_cache; DescriptorCache m_descriptor_cache; FramebufferCache m_framebuffer_cache; diff --git a/src/graphics/host_gpu/renderer/textureCache.cpp b/src/graphics/host_gpu/renderer/textureCache.cpp index 903cbb6bd..f78f4a0b5 100644 --- a/src/graphics/host_gpu/renderer/textureCache.cpp +++ b/src/graphics/host_gpu/renderer/textureCache.cpp @@ -3653,7 +3653,7 @@ bool TextureCache::InvalidateMemory(PageFaultAccess access, uint64_t vaddr, uint if (needs_readback) { if (GraphicsRunIsCommandProcessorThread()) { - GraphicsRunFinishCommandProcessors(); + GraphicsRunFinishScheduler(); } m_readback->Request(access, vaddr, size); action = m_memory_tracker.BeginCpuFault(vaddr, size, access); diff --git a/tests/ShaderRecompilerComputeTests.cpp b/tests/ShaderRecompilerComputeTests.cpp index 78d906c31..81fb12301 100644 --- a/tests/ShaderRecompilerComputeTests.cpp +++ b/tests/ShaderRecompilerComputeTests.cpp @@ -3,9 +3,12 @@ #include "common/assert.h" #include "common/emulatorConfig.h" #include "common/logging/log.h" +#include "graphics/guest_gpu/command_processor/commandProcessor.h" +#include "graphics/guest_gpu/command_processor/pm4Dispatch.h" #include "graphics/guest_gpu/gpu_defs.h" #include "graphics/guest_gpu/gpu_format.h" #include "graphics/guest_gpu/hardwareContext.h" +#include "graphics/guest_gpu/pm4.h" #include "graphics/guest_gpu/tile.h" #include "graphics/host_gpu/gpuTiler.h" #include "graphics/host_gpu/hostMemory.h" @@ -14083,6 +14086,102 @@ void CheckClipControlDepthClipState() { std::printf("[host] %-32s ok\n", "ClipControlDepthClipState"); } +void CheckPm4WaitResume() { + GraphicsInitJmpTables(); + CommandProcessor processor; + + uint32_t label = 0; + uint32_t prefix = 0; + uint32_t child_observation = UINT32_MAX; + uint32_t suffix = 0; + const auto address = [](const void *value) { + return reinterpret_cast(value); + }; + + std::array child{}; + child[0] = KYTY_PM4(5, Pm4::IT_WRITE_DATA, 0); + child[1] = 0; + child[2] = static_cast(address(&child_observation)); + child[3] = static_cast(address(&child_observation) >> 32u); + child[4] = 0; + child[5] = KYTY_PM4(7, Pm4::IT_NOP, Pm4::R_WAIT_MEM_32); + child[6] = static_cast(address(&label)); + child[7] = static_cast(address(&label) >> 32u); + child[8] = UINT32_MAX; + child[9] = 1; + child[10] = 0x10u | 3u; + + std::array nested{}; + nested[0] = KYTY_PM4(4, Pm4::IT_INDIRECT_BUFFER, 0); + nested[1] = static_cast(address(child.data())); + nested[2] = static_cast(address(child.data()) >> 32u); + nested[3] = 0x0f200000u | static_cast(child.size()); + + std::array commands{}; + commands[0] = KYTY_PM4(5, Pm4::IT_WRITE_DATA, 0); + commands[1] = 0; + commands[2] = static_cast(address(&prefix)); + commands[3] = static_cast(address(&prefix) >> 32u); + commands[4] = 11; + commands[5] = KYTY_PM4(4, Pm4::IT_INDIRECT_BUFFER, 0); + commands[6] = static_cast(address(nested.data())); + commands[7] = static_cast(address(nested.data()) >> 32u); + commands[8] = 0x0f200000u | static_cast(nested.size()); + commands[9] = KYTY_PM4(5, Pm4::IT_WRITE_DATA, 0); + commands[10] = 0; + commands[11] = static_cast(address(&suffix)); + commands[12] = static_cast(address(&suffix) >> 32u); + commands[13] = 22; + + Pm4Execution execution; + Require("Pm4WaitResume", "suspend", + processor.Process(execution, commands.data(), commands.size()) == + Pm4ProcessResult::Blocked && + prefix == 11 && child_observation == 0 && suffix == 0, + "blocked indirect wait did not preserve its command position"); + + label = 1; + child[4] = 1; + Require("Pm4WaitResume", "resume", + processor.Process(execution, commands.data(), commands.size()) == + Pm4ProcessResult::Complete && + prefix == 11 && child_observation == 0 && suffix == 22, + "resumed indirect wait replayed a child or did not finish its parent"); + std::printf("[host] %-32s ok\n", "Pm4WaitResume"); +} + +void CheckPm4CeCompletion() { + GraphicsInitJmpTables(); + CommandProcessor processor; + uint32_t suffix = 0; + const auto address = reinterpret_cast(&suffix); + + std::array commands{}; + commands[0] = 0xc0008600u; + commands[1] = 1; + commands[2] = KYTY_PM4(5, Pm4::IT_WRITE_DATA, 0); + commands[3] = 0; + commands[4] = static_cast(address); + commands[5] = static_cast(address >> 32u); + commands[6] = 33; + + processor.ResetDeCe(); + Pm4Execution execution; + Require("Pm4CeCompletion", "wait", + processor.Process(execution, commands.data(), commands.size()) == + Pm4ProcessResult::Blocked && + suffix == 0, + "DE did not wait for an active CE stream"); + + processor.SetCeComplete(true); + Require("Pm4CeCompletion", "complete", + processor.Process(execution, commands.data(), commands.size()) == + Pm4ProcessResult::Complete && + suffix == 33, + "DE remained blocked after the CE stream completed"); + std::printf("[host] %-32s ok\n", "Pm4CeCompletion"); +} + } // namespace } // namespace Libs::Graphics @@ -14229,6 +14328,8 @@ int main(int argc, char **argv) { #endif CheckClipControlDepthClipState(); CheckReferenceClockScale(); + CheckPm4WaitResume(); + CheckPm4CeCompletion(); CheckEmbeddedFetchVertexOffset(); CheckEmbeddedFetchLaneSpill(); CheckPs5GameExampleImageClearRuntimeShape();