gpu: queue AGC suspend points

src/graphics/guest_gpu/graphicsRun.cpp

@@ -97,7 +97,7 @@ public:
 	void SubmitCompute(uint32_t queue, uint32_t* cmd_buffer, uint32_t num_dw,
 	                   bool trigger_agc_interrupt_on_done);
 	void SubmitFlipPreparation(uint64_t request_id);
-	void Done();
+	void SuspendPoint();
 	void Shutdown();
 	[[nodiscard]] bool        IsStopping();
 	void                      SendCommand(Common::UniqueFunction<void>&& command);
@@ -106,7 +106,7 @@ public:
 	[[nodiscard]] static bool IsGpuThread() noexcept { return g_gpu_thread; }
 
 private:
-	enum class SubmissionType { Graphics, Compute, FlipPreparation };
+	enum class SubmissionType { Graphics, Compute, FlipPreparation, SuspendPoint };
 
 	struct Submission {
 		SubmissionType type     = SubmissionType::Graphics;
@@ -124,9 +124,7 @@ private:
 		uint64_t       flip_request_id               = 0;
 	};
 
-	void              WaitLocked();
 	void              Enqueue(Submission submission);
-	void              WaitForIdle();
 	void              ProcessCommands();
 	bool              Process(Submission& submission);
 	static void       ThreadRun(void* data);
@@ -137,17 +135,16 @@ private:
 	Common::Mutex                                  m_queue_mutex;
 	std::mutex                                     m_shutdown_mutex;
 	Common::CondVar                                m_work_available;
-	Common::CondVar                                m_idle;
 	std::array<std::deque<Submission>, QueueCount> m_queues;
 	std::deque<Common::UniqueFunction<void>>       m_commands;
 	std::atomic_uint32_t                           m_pending_commands {0};
 	uint32_t                                       m_next_queue        = 0;
 	uint32_t                                       m_submission_count  = 0;
-	bool                                           m_processing        = false;
+	bool                                           m_reset_graphics    = true;
-	bool                                           m_graphics_done     = true;
 	bool                                           m_accepting         = true;
 	bool                                           m_stopping          = false;
 	bool                                           m_shutdown_complete = false;
+	std::binary_semaphore                          m_suspend_point_slot {1};
 
 	std::unique_ptr<CommandProcessor>                                m_gfx_cp;
 	std::array<std::unique_ptr<CommandProcessor>, ComputeQueueCount> m_compute_cp;
@@ -241,8 +238,8 @@ void GpuState::Submit(uint32_t* cmd_draw_buffer, uint32_t num_draw_dw, uint32_t*
 	submission.commands                      = OwnedCmdBuffer(cmd_draw_buffer, num_draw_dw);
 	submission.constant_commands             = OwnedCmdBuffer(cmd_const_buffer, num_const_dw);
 	submission.trigger_agc_interrupt_on_done = trigger_agc_interrupt_on_done;
-	submission.reset_processor               = m_graphics_done;
+	submission.reset_processor               = m_reset_graphics;
-	m_graphics_done                          = false;
+	m_reset_graphics                         = false;
 	Enqueue(std::move(submission));
 }
 
@@ -268,32 +265,27 @@ void GpuState::SubmitFlipPreparation(uint64_t request_id) {
 	Submission   submission;
 	submission.type            = SubmissionType::FlipPreparation;
 	submission.queue_id        = 0;
-	submission.reset_processor = m_graphics_done;
+	submission.reset_processor = m_reset_graphics;
 	submission.flip_request_id = request_id;
-	m_graphics_done            = false;
+	m_reset_graphics           = false;
 	Enqueue(std::move(submission));
 }
 
-void GpuState::Done() {
+void GpuState::SuspendPoint() {
+	EXIT_IF(IsGpuThread());
+	m_suspend_point_slot.acquire();
 	GpuMutexLock lock(m_submission_mutex);
-	if (IsGpuThread()) {
+	Submission   submission;
-		m_gfx_cp->BufferWait();
+	submission.type     = SubmissionType::SuspendPoint;
-	} else {
+	submission.queue_id = 0;
-		WaitLocked();
+	m_reset_graphics    = false;
-	}
+	Enqueue(std::move(submission));
-	m_graphics_done = true;
-	m_done_num++;
 }
 
 int GpuState::GetFrameNum() {
 	return m_done_num;
 }
 
-void GpuState::WaitLocked() {
-	WaitForIdle();
-	SendCommandSync([this] { m_gfx_cp->BufferWait(); });
-}
-
 CommandProcessor& GpuState::GetProcessor(uint32_t queue_id) {
 	EXIT_IF(queue_id >= QueueCount);
 	if (queue_id == 0) {
@@ -316,9 +308,12 @@ void CommandProcessor::Reset() {
 	m_context_state_pushed             = false;
 	m_index_type_and_size              = 0;
 	m_index_buffer_size                = 0;
+	m_index_base_addr                  = 0;
 	m_user_data_marker                 = HW::UserSgprType::Unknown;
 	m_draw_indirect_args_base_addr     = 0;
 	m_dispatch_indirect_args_base_addr = 0;
+	m_num_instances                    = 1;
+	m_predicate_skip                   = false;
 
 	std::memset(m_const_ram, 0, sizeof(m_const_ram));
 }
@@ -532,13 +527,6 @@ void GpuState::Enqueue(Submission submission) {
 	m_work_available.Signal();
 }
 
-void GpuState::WaitForIdle() {
-	Common::LockGuard lock(m_queue_mutex);
-	while (m_processing || !m_commands.empty() || m_submission_count != 0) {
-		m_idle.Wait(&m_queue_mutex);
-	}
-}
-
 void GpuState::ThreadRun(void* data) {
 	auto* gpu = static_cast<GpuState*>(data);
 	EXIT_IF(gpu == nullptr);
@@ -554,19 +542,14 @@ void GpuState::ThreadRun(void* data) {
 		{
 			Common::LockGuard lock(gpu->m_queue_mutex);
 			while (gpu->m_commands.empty() && gpu->m_submission_count == 0 && !gpu->m_stopping) {
-				gpu->m_processing = false;
-				gpu->m_idle.Signal();
 				gpu->m_work_available.Wait(&gpu->m_queue_mutex);
 			}
 			if (gpu->m_stopping && gpu->m_commands.empty() && gpu->m_submission_count == 0) {
-				gpu->m_processing = false;
-				gpu->m_idle.SignalAll();
 				should_stop = true;
 			} else if (!gpu->m_commands.empty()) {
 				command = std::move(gpu->m_commands.front());
 				gpu->m_commands.pop_front();
 				EXIT_IF(gpu->m_pending_commands.fetch_sub(1, std::memory_order_acq_rel) == 0);
-				gpu->m_processing = true;
 			} else {
 				int selected_queue = -1;
 				for (uint32_t offset = 0; offset < QueueCount; offset++) {
@@ -577,7 +560,6 @@ void GpuState::ThreadRun(void* data) {
 					}
 				}
 				if (selected_queue < 0) {
-					gpu->m_processing = false;
 					gpu->m_work_available.WaitFor(&gpu->m_queue_mutex, 100);
 					for (auto& queue: gpu->m_queues) {
 						if (!queue.empty()) {
@@ -591,7 +573,6 @@ void GpuState::ThreadRun(void* data) {
 				queue.pop_front();
 				gpu->m_submission_count--;
 				gpu->m_next_queue = (static_cast<uint32_t>(selected_queue) + 1) % QueueCount;
-				gpu->m_processing = true;
 				has_submission    = true;
 			}
 		}
@@ -606,11 +587,6 @@ void GpuState::ThreadRun(void* data) {
 			EXIT_IF(g_current_processor != nullptr);
 			command();
 
-			Common::LockGuard lock(gpu->m_queue_mutex);
-			gpu->m_processing = false;
-			if (gpu->m_commands.empty() && gpu->m_submission_count == 0) {
-				gpu->m_idle.SignalAll();
-			}
 			continue;
 		}
 
@@ -629,16 +605,19 @@ void GpuState::ThreadRun(void* data) {
 				}
 			}
 		}
-		gpu->m_processing = false;
-		if (gpu->m_commands.empty() && gpu->m_submission_count == 0) {
-			gpu->m_idle.SignalAll();
-		}
 	}
 }
 
 bool GpuState::Process(Submission& submission) {
 	auto&      cp          = GetProcessor(submission.queue_id);
 	const bool first_slice = !submission.started;
+	if (submission.type == SubmissionType::SuspendPoint) {
+		EXIT_IF(!first_slice || submission.queue_id != 0);
+		cp.Reset();
+		m_done_num++;
+		m_suspend_point_slot.release();
+		return true;
+	}
 
 	if (first_slice && submission.reset_processor) {
 		cp.Reset();
@@ -726,6 +705,7 @@ bool GpuState::Process(Submission& submission) {
 			m_renderer.GetGpuResources().RunGarbageCollector();
 			cp.PrepareCpuFlip(submission.flip_request_id);
 			break;
+		case SubmissionType::SuspendPoint: EXIT("unreachable suspend point submission\n");
 	}
 
 	return complete;
@@ -1708,8 +1688,8 @@ void Gpu::SubmitFlipPreparation(uint64_t request_id) {
 	m_state->SubmitFlipPreparation(request_id);
 }
 
-void Gpu::Done() {
+void Gpu::SuspendPoint() {
-	m_state->Done();
+	m_state->SuspendPoint();
 }
 
 int Gpu::GetFrameNum() const {

src/graphics/guest_gpu/graphicsRun.h

@@ -28,7 +28,7 @@ public:
 	void SubmitCompute(uint32_t queue, uint32_t* commands, uint32_t size_dw,
 	                   bool trigger_agc_interrupt_on_done = false);
 	void SubmitFlipPreparation(uint64_t request_id);
-	void Done();
+	void SuspendPoint();
 	[[nodiscard]] int GetFrameNum() const;
 
 	[[nodiscard]] static bool IsGpuThread() noexcept;

src/libs/agc.cpp

@@ -1440,7 +1440,7 @@ int KYTY_SYSV_ABI GraphicsSuspendPoint() {
 	PRINT_NAME();
 
 	EXIT_IF(g_renderer == nullptr);
-	g_renderer->GetGpu().Done();
+	g_renderer->GetGpu().SuspendPoint();
 
 	return OK;
 }
@@ -1466,7 +1466,7 @@ uint32_t* KYTY_SYSV_ABI GraphicsDcbContextStateOp(CommandBuffer* buf, uint32_t o
 
 	buf->DbgDump();
 
-	uint32_t* first = nullptr;
+	uint32_t*  first  = nullptr;
 	const auto append = [&](uint32_t segment_size_dw) {
 		auto* segment = buf->AllocateDW(segment_size_dw);
 		if (segment == nullptr) {

tests/ShaderRecompilerComputeTests.cpp

@@ -729,6 +729,14 @@ void Require(const char* shader_name, const char* stage, bool value, const std::
 	}
 }
 
+void WaitForSuspendPoint(Gpu& gpu) {
+	const int completion = gpu.GetFrameNum() + 1;
+	gpu.SuspendPoint();
+	while (gpu.GetFrameNum() < completion) {
+		std::this_thread::yield();
+	}
+}
+
 void CheckLeastRecentlyUsedCacheOrdering() {
 	Common::LeastRecentlyUsedCache<uint32_t, uint64_t> cache;
 	const auto                                         first  = cache.Insert(1, 1);
@@ -1659,7 +1667,7 @@ public:
 			});
 			std::this_thread::yield();
 		}
-		gpu.Done();
+		WaitForSuspendPoint(gpu);
 		Require("GpuCommandLane", "packet-boundary command polling",
 		        packet_marker_a_at_callback == 11 && packet_marker_b_at_callback == 0 &&
 		            packet_marker_a == 11 && packet_marker_b == 22,
@@ -1688,25 +1696,47 @@ public:
 		commands[16] = 22;
 		gpu.Submit(commands.data(), static_cast<uint32_t>(commands.size()), nullptr, 0);
 
-		std::binary_semaphore ordered_started {0};
+		std::binary_semaphore first_suspend_inserted {0};
-		std::atomic<bool>     ordered_finished {false};
+		std::binary_semaphore second_suspend_started {0};
-		uint32_t              ordered_suffix = 0;
+		std::atomic<bool>     second_suspend_finished {false};
+		uint32_t              suffix_after_first_suspend = UINT32_MAX;
+		const int             ordered_completion         = gpu.GetFrameNum() + 2;
 		std::jthread          ordered([&] {
-			ordered_started.release();
+			gpu.SuspendPoint();
-			gpu.Done();
+			suffix_after_first_suspend = suffix;
-			ordered_suffix   = suffix;
+			first_suspend_inserted.release();
-			ordered_finished = true;
+			second_suspend_started.release();
+			gpu.SuspendPoint();
+			second_suspend_finished = true;
 		});
-		ordered_started.acquire();
+		const bool first_returned = first_suspend_inserted.try_acquire_for(std::chrono::seconds(2));
+		const bool second_started =
+		    first_returned && second_suspend_started.try_acquire_for(std::chrono::seconds(2));
+		if (second_started) {
+			const auto ordered_deadline =
+			    std::chrono::steady_clock::now() + std::chrono::seconds(2);
+			while (!second_suspend_finished.load() &&
+			       std::chrono::steady_clock::now() < ordered_deadline) {
+				std::this_thread::yield();
+			}
+		}
+		const bool second_blocked_before_release = second_started && !second_suspend_finished.load();
 		gpu.SendCommandSync([&] {
-			Require("GpuCommandLane", "submit done barrier", !ordered_finished.load(),
-			        "submit done returned before a queued submission");
 			label = 1;
 		});
 		ordered.join();
+		while (gpu.GetFrameNum() < ordered_completion) {
+			std::this_thread::yield();
+		}
+		Require("GpuCommandLane", "asynchronous suspend insertion",
+		        first_returned && suffix_after_first_suspend == 0,
+		        "first suspend point blocked on earlier graphics work");
+		Require("GpuCommandLane", "single in-flight suspend point",
+		        second_blocked_before_release && second_suspend_finished.load(),
+		        "second suspend point did not wait for the first marker");
 		Require("GpuCommandLane", "ordered completion",
-		        prefix == 11 && suffix == 22 && ordered_suffix == 22 && ordered_finished.load(),
+		        prefix == 11 && suffix == 22,
-		        "submit done did not drain prior PM4 work");
+		        "queued suspend point completed before earlier graphics work");
 
 		auto&              resources         = context.GetGpuResources();
 		constexpr uint64_t empty_unmap_base = 0x0000000200400000ull;
@@ -1728,7 +1758,7 @@ public:
 			unmap_complete.acquire();
 		}
 		unmap_thread.join();
-		gpu.Done();
+		WaitForSuspendPoint(gpu);
 		Require("GpuCommandLane", "unmap queue progress",
 		        unmap_returned && !resources.IsMapped(empty_unmap_base, empty_unmap_size) &&
 		            prefix == 11 && suffix == 22,
@@ -1841,7 +1871,7 @@ public:
 		Require("GpuCommandLane", "DMA_DATA packet assembly", dma_cursor == dma_commands.size(),
 		        "DMA_DATA GDS packet stream has the wrong size");
 		gpu.Submit(dma_commands.data(), static_cast<uint32_t>(dma_commands.size()), nullptr, 0);
-		gpu.Done();
+		WaitForSuspendPoint(gpu);
 		constexpr uint32_t clean_fill_value = 0xdecafbad;
 		gpu.SendCommandSync([&] {
 			auto& buffer_cache = resources.GetBufferCache();
@@ -2045,12 +2075,11 @@ public:
 		std::binary_semaphore command_entered {0};
 		std::binary_semaphore release_command {0};
 		std::atomic<bool>     shutdown_complete {false};
-		std::atomic<bool>     submission_lane_entered {false};
+		std::atomic<bool>     command_completed {false};
 		gpu.SendCommand([&] {
 			command_entered.release();
 			release_command.acquire();
-			gpu.Done();
+			command_completed = true;
-			submission_lane_entered = true;
 		});
 		command_entered.acquire();
 		std::jthread shutdown_thread([&] {
@@ -2065,8 +2094,8 @@ public:
 		release_command.release();
 		shutdown_thread.join();
 		Require("GpuCommandLane", "owned shutdown completion",
-		        shutdown_complete.load() && submission_lane_entered.load(),
+		        shutdown_complete.load() && command_completed.load(),
-		        "GPU owner did not drain a command that entered the submission lane");
+		        "GPU owner did not drain an in-flight command");
 		context.ShutdownGpu();
 		std::printf("[host]    %-32s ok\n", "GpuCommandLane");
 	}