From 90bf3c92364f335f59cbe9da0bf11b932f5ae7e7 Mon Sep 17 00:00:00 2001 From: nmzik Date: Fri, 7 Aug 2026 21:02:31 +0200 Subject: [PATCH] gpu: queue AGC suspend points --- src/graphics/guest_gpu/graphicsRun.cpp | 78 ++++++++++---------------- src/graphics/guest_gpu/graphicsRun.h | 2 +- src/libs/agc.cpp | 4 +- tests/ShaderRecompilerComputeTests.cpp | 69 ++++++++++++++++------- 4 files changed, 81 insertions(+), 72 deletions(-) diff --git a/src/graphics/guest_gpu/graphicsRun.cpp b/src/graphics/guest_gpu/graphicsRun.cpp index 92c010e1c..1ede77616 100644 --- a/src/graphics/guest_gpu/graphicsRun.cpp +++ b/src/graphics/guest_gpu/graphicsRun.cpp @@ -97,7 +97,7 @@ public: void SubmitCompute(uint32_t queue, uint32_t* cmd_buffer, uint32_t num_dw, bool trigger_agc_interrupt_on_done); void SubmitFlipPreparation(uint64_t request_id); - void Done(); + void SuspendPoint(); void Shutdown(); [[nodiscard]] bool IsStopping(); void SendCommand(Common::UniqueFunction&& command); @@ -106,7 +106,7 @@ public: [[nodiscard]] static bool IsGpuThread() noexcept { return g_gpu_thread; } private: - enum class SubmissionType { Graphics, Compute, FlipPreparation }; + enum class SubmissionType { Graphics, Compute, FlipPreparation, SuspendPoint }; struct Submission { SubmissionType type = SubmissionType::Graphics; @@ -124,9 +124,7 @@ private: uint64_t flip_request_id = 0; }; - void WaitLocked(); void Enqueue(Submission submission); - void WaitForIdle(); void ProcessCommands(); bool Process(Submission& submission); static void ThreadRun(void* data); @@ -137,17 +135,16 @@ private: Common::Mutex m_queue_mutex; std::mutex m_shutdown_mutex; Common::CondVar m_work_available; - Common::CondVar m_idle; std::array, QueueCount> m_queues; std::deque> m_commands; std::atomic_uint32_t m_pending_commands {0}; uint32_t m_next_queue = 0; uint32_t m_submission_count = 0; - bool m_processing = false; - bool m_graphics_done = true; + bool m_reset_graphics = true; bool m_accepting = true; bool m_stopping = false; bool m_shutdown_complete = false; + std::binary_semaphore m_suspend_point_slot {1}; std::unique_ptr m_gfx_cp; std::array, ComputeQueueCount> m_compute_cp; @@ -241,8 +238,8 @@ void GpuState::Submit(uint32_t* cmd_draw_buffer, uint32_t num_draw_dw, uint32_t* submission.commands = OwnedCmdBuffer(cmd_draw_buffer, num_draw_dw); submission.constant_commands = OwnedCmdBuffer(cmd_const_buffer, num_const_dw); submission.trigger_agc_interrupt_on_done = trigger_agc_interrupt_on_done; - submission.reset_processor = m_graphics_done; - m_graphics_done = false; + submission.reset_processor = m_reset_graphics; + m_reset_graphics = false; Enqueue(std::move(submission)); } @@ -268,32 +265,27 @@ void GpuState::SubmitFlipPreparation(uint64_t request_id) { Submission submission; submission.type = SubmissionType::FlipPreparation; submission.queue_id = 0; - submission.reset_processor = m_graphics_done; + submission.reset_processor = m_reset_graphics; submission.flip_request_id = request_id; - m_graphics_done = false; + m_reset_graphics = false; Enqueue(std::move(submission)); } -void GpuState::Done() { +void GpuState::SuspendPoint() { + EXIT_IF(IsGpuThread()); + m_suspend_point_slot.acquire(); GpuMutexLock lock(m_submission_mutex); - if (IsGpuThread()) { - m_gfx_cp->BufferWait(); - } else { - WaitLocked(); - } - m_graphics_done = true; - m_done_num++; + Submission submission; + submission.type = SubmissionType::SuspendPoint; + submission.queue_id = 0; + m_reset_graphics = false; + Enqueue(std::move(submission)); } int GpuState::GetFrameNum() { return m_done_num; } -void GpuState::WaitLocked() { - WaitForIdle(); - SendCommandSync([this] { m_gfx_cp->BufferWait(); }); -} - CommandProcessor& GpuState::GetProcessor(uint32_t queue_id) { EXIT_IF(queue_id >= QueueCount); if (queue_id == 0) { @@ -316,9 +308,12 @@ void CommandProcessor::Reset() { m_context_state_pushed = false; m_index_type_and_size = 0; m_index_buffer_size = 0; + m_index_base_addr = 0; m_user_data_marker = HW::UserSgprType::Unknown; m_draw_indirect_args_base_addr = 0; m_dispatch_indirect_args_base_addr = 0; + m_num_instances = 1; + m_predicate_skip = false; std::memset(m_const_ram, 0, sizeof(m_const_ram)); } @@ -532,13 +527,6 @@ void GpuState::Enqueue(Submission submission) { m_work_available.Signal(); } -void GpuState::WaitForIdle() { - Common::LockGuard lock(m_queue_mutex); - while (m_processing || !m_commands.empty() || m_submission_count != 0) { - m_idle.Wait(&m_queue_mutex); - } -} - void GpuState::ThreadRun(void* data) { auto* gpu = static_cast(data); EXIT_IF(gpu == nullptr); @@ -554,19 +542,14 @@ void GpuState::ThreadRun(void* data) { { Common::LockGuard lock(gpu->m_queue_mutex); while (gpu->m_commands.empty() && gpu->m_submission_count == 0 && !gpu->m_stopping) { - gpu->m_processing = false; - gpu->m_idle.Signal(); gpu->m_work_available.Wait(&gpu->m_queue_mutex); } if (gpu->m_stopping && gpu->m_commands.empty() && gpu->m_submission_count == 0) { - gpu->m_processing = false; - gpu->m_idle.SignalAll(); should_stop = true; } else if (!gpu->m_commands.empty()) { command = std::move(gpu->m_commands.front()); gpu->m_commands.pop_front(); EXIT_IF(gpu->m_pending_commands.fetch_sub(1, std::memory_order_acq_rel) == 0); - gpu->m_processing = true; } else { int selected_queue = -1; for (uint32_t offset = 0; offset < QueueCount; offset++) { @@ -577,7 +560,6 @@ void GpuState::ThreadRun(void* data) { } } if (selected_queue < 0) { - gpu->m_processing = false; gpu->m_work_available.WaitFor(&gpu->m_queue_mutex, 100); for (auto& queue: gpu->m_queues) { if (!queue.empty()) { @@ -591,7 +573,6 @@ void GpuState::ThreadRun(void* data) { queue.pop_front(); gpu->m_submission_count--; gpu->m_next_queue = (static_cast(selected_queue) + 1) % QueueCount; - gpu->m_processing = true; has_submission = true; } } @@ -606,11 +587,6 @@ void GpuState::ThreadRun(void* data) { EXIT_IF(g_current_processor != nullptr); command(); - Common::LockGuard lock(gpu->m_queue_mutex); - gpu->m_processing = false; - if (gpu->m_commands.empty() && gpu->m_submission_count == 0) { - gpu->m_idle.SignalAll(); - } continue; } @@ -629,16 +605,19 @@ void GpuState::ThreadRun(void* data) { } } } - gpu->m_processing = false; - if (gpu->m_commands.empty() && gpu->m_submission_count == 0) { - gpu->m_idle.SignalAll(); - } } } bool GpuState::Process(Submission& submission) { auto& cp = GetProcessor(submission.queue_id); const bool first_slice = !submission.started; + if (submission.type == SubmissionType::SuspendPoint) { + EXIT_IF(!first_slice || submission.queue_id != 0); + cp.Reset(); + m_done_num++; + m_suspend_point_slot.release(); + return true; + } if (first_slice && submission.reset_processor) { cp.Reset(); @@ -726,6 +705,7 @@ bool GpuState::Process(Submission& submission) { m_renderer.GetGpuResources().RunGarbageCollector(); cp.PrepareCpuFlip(submission.flip_request_id); break; + case SubmissionType::SuspendPoint: EXIT("unreachable suspend point submission\n"); } return complete; @@ -1708,8 +1688,8 @@ void Gpu::SubmitFlipPreparation(uint64_t request_id) { m_state->SubmitFlipPreparation(request_id); } -void Gpu::Done() { - m_state->Done(); +void Gpu::SuspendPoint() { + m_state->SuspendPoint(); } int Gpu::GetFrameNum() const { diff --git a/src/graphics/guest_gpu/graphicsRun.h b/src/graphics/guest_gpu/graphicsRun.h index dc9c2cf69..c9f8f4700 100644 --- a/src/graphics/guest_gpu/graphicsRun.h +++ b/src/graphics/guest_gpu/graphicsRun.h @@ -28,7 +28,7 @@ public: void SubmitCompute(uint32_t queue, uint32_t* commands, uint32_t size_dw, bool trigger_agc_interrupt_on_done = false); void SubmitFlipPreparation(uint64_t request_id); - void Done(); + void SuspendPoint(); [[nodiscard]] int GetFrameNum() const; [[nodiscard]] static bool IsGpuThread() noexcept; diff --git a/src/libs/agc.cpp b/src/libs/agc.cpp index af23e8a0e..7cbd11918 100644 --- a/src/libs/agc.cpp +++ b/src/libs/agc.cpp @@ -1440,7 +1440,7 @@ int KYTY_SYSV_ABI GraphicsSuspendPoint() { PRINT_NAME(); EXIT_IF(g_renderer == nullptr); - g_renderer->GetGpu().Done(); + g_renderer->GetGpu().SuspendPoint(); return OK; } @@ -1466,7 +1466,7 @@ uint32_t* KYTY_SYSV_ABI GraphicsDcbContextStateOp(CommandBuffer* buf, uint32_t o buf->DbgDump(); - uint32_t* first = nullptr; + uint32_t* first = nullptr; const auto append = [&](uint32_t segment_size_dw) { auto* segment = buf->AllocateDW(segment_size_dw); if (segment == nullptr) { diff --git a/tests/ShaderRecompilerComputeTests.cpp b/tests/ShaderRecompilerComputeTests.cpp index 944fc684d..963984030 100644 --- a/tests/ShaderRecompilerComputeTests.cpp +++ b/tests/ShaderRecompilerComputeTests.cpp @@ -729,6 +729,14 @@ void Require(const char* shader_name, const char* stage, bool value, const std:: } } +void WaitForSuspendPoint(Gpu& gpu) { + const int completion = gpu.GetFrameNum() + 1; + gpu.SuspendPoint(); + while (gpu.GetFrameNum() < completion) { + std::this_thread::yield(); + } +} + void CheckLeastRecentlyUsedCacheOrdering() { Common::LeastRecentlyUsedCache cache; const auto first = cache.Insert(1, 1); @@ -1659,7 +1667,7 @@ public: }); std::this_thread::yield(); } - gpu.Done(); + WaitForSuspendPoint(gpu); Require("GpuCommandLane", "packet-boundary command polling", packet_marker_a_at_callback == 11 && packet_marker_b_at_callback == 0 && packet_marker_a == 11 && packet_marker_b == 22, @@ -1688,25 +1696,47 @@ public: commands[16] = 22; gpu.Submit(commands.data(), static_cast(commands.size()), nullptr, 0); - std::binary_semaphore ordered_started {0}; - std::atomic ordered_finished {false}; - uint32_t ordered_suffix = 0; + std::binary_semaphore first_suspend_inserted {0}; + std::binary_semaphore second_suspend_started {0}; + std::atomic second_suspend_finished {false}; + uint32_t suffix_after_first_suspend = UINT32_MAX; + const int ordered_completion = gpu.GetFrameNum() + 2; std::jthread ordered([&] { - ordered_started.release(); - gpu.Done(); - ordered_suffix = suffix; - ordered_finished = true; + gpu.SuspendPoint(); + suffix_after_first_suspend = suffix; + first_suspend_inserted.release(); + second_suspend_started.release(); + gpu.SuspendPoint(); + second_suspend_finished = true; }); - ordered_started.acquire(); + const bool first_returned = first_suspend_inserted.try_acquire_for(std::chrono::seconds(2)); + const bool second_started = + first_returned && second_suspend_started.try_acquire_for(std::chrono::seconds(2)); + if (second_started) { + const auto ordered_deadline = + std::chrono::steady_clock::now() + std::chrono::seconds(2); + while (!second_suspend_finished.load() && + std::chrono::steady_clock::now() < ordered_deadline) { + std::this_thread::yield(); + } + } + const bool second_blocked_before_release = second_started && !second_suspend_finished.load(); gpu.SendCommandSync([&] { - Require("GpuCommandLane", "submit done barrier", !ordered_finished.load(), - "submit done returned before a queued submission"); label = 1; }); ordered.join(); + while (gpu.GetFrameNum() < ordered_completion) { + std::this_thread::yield(); + } + Require("GpuCommandLane", "asynchronous suspend insertion", + first_returned && suffix_after_first_suspend == 0, + "first suspend point blocked on earlier graphics work"); + Require("GpuCommandLane", "single in-flight suspend point", + second_blocked_before_release && second_suspend_finished.load(), + "second suspend point did not wait for the first marker"); Require("GpuCommandLane", "ordered completion", - prefix == 11 && suffix == 22 && ordered_suffix == 22 && ordered_finished.load(), - "submit done did not drain prior PM4 work"); + prefix == 11 && suffix == 22, + "queued suspend point completed before earlier graphics work"); auto& resources = context.GetGpuResources(); constexpr uint64_t empty_unmap_base = 0x0000000200400000ull; @@ -1728,7 +1758,7 @@ public: unmap_complete.acquire(); } unmap_thread.join(); - gpu.Done(); + WaitForSuspendPoint(gpu); Require("GpuCommandLane", "unmap queue progress", unmap_returned && !resources.IsMapped(empty_unmap_base, empty_unmap_size) && prefix == 11 && suffix == 22, @@ -1841,7 +1871,7 @@ public: Require("GpuCommandLane", "DMA_DATA packet assembly", dma_cursor == dma_commands.size(), "DMA_DATA GDS packet stream has the wrong size"); gpu.Submit(dma_commands.data(), static_cast(dma_commands.size()), nullptr, 0); - gpu.Done(); + WaitForSuspendPoint(gpu); constexpr uint32_t clean_fill_value = 0xdecafbad; gpu.SendCommandSync([&] { auto& buffer_cache = resources.GetBufferCache(); @@ -2045,12 +2075,11 @@ public: std::binary_semaphore command_entered {0}; std::binary_semaphore release_command {0}; std::atomic shutdown_complete {false}; - std::atomic submission_lane_entered {false}; + std::atomic command_completed {false}; gpu.SendCommand([&] { command_entered.release(); release_command.acquire(); - gpu.Done(); - submission_lane_entered = true; + command_completed = true; }); command_entered.acquire(); std::jthread shutdown_thread([&] { @@ -2065,8 +2094,8 @@ public: release_command.release(); shutdown_thread.join(); Require("GpuCommandLane", "owned shutdown completion", - shutdown_complete.load() && submission_lane_entered.load(), - "GPU owner did not drain a command that entered the submission lane"); + shutdown_complete.load() && command_completed.load(), + "GPU owner did not drain an in-flight command"); context.ShutdownGpu(); std::printf("[host] %-32s ok\n", "GpuCommandLane"); }