perf: sync only GPU-dirty readbacks

src/graphics/guest_gpu/command_processor/commandProcessor.h

@@ -50,23 +50,10 @@ public:
 
 	void Reset();
 
-	void BufferInit();
+	void            BufferInit();
-	void BufferFlush();
+	void            BufferFlush();
-	void BufferFlushAndWait();
+	void            BufferFlushAndWait();
-	void BufferWait();
+	void            BufferWait();
-	void BeginReadbackTransaction() {
-		if (m_readback_active) {
-			EXIT("nested command-processor readback transaction\n");
-		}
-		m_readback_active = true;
-	}
-	void EndReadbackTransaction() {
-		if (!m_readback_active) {
-			EXIT("command-processor readback transaction is not active\n");
-		}
-		m_readback_active = false;
-	}
-
 	HW::Context&    GetCtx() { return m_ctx; }
 	HW::UserConfig& GetUcfg() { return m_ucfg; }
 	HW::Shader&     GetShCtx() { return m_sh_ctx; }
@@ -173,10 +160,9 @@ private:
 	// Persistent draw state: indirect draws update it for subsequent draws.
 	uint32_t m_num_instances = 1;
 
-	uint32_t m_de_count        = 0;
+	uint32_t m_de_count    = 0;
-	uint32_t m_ce_count        = 0;
+	uint32_t m_ce_count    = 0;
-	bool     m_ce_complete     = false;
+	bool     m_ce_complete = false;
-	bool     m_readback_active = false;
 
 	uint32_t m_const_ram[0x3000] = {0};
 

src/graphics/guest_gpu/graphicsRun.cpp

@@ -36,6 +36,7 @@ static thread_local CommandProcessor* g_current_processor = nullptr;
 static thread_local Pm4Execution*     g_current_execution = nullptr;
 static thread_local bool              g_gpu_mutex_owned   = false;
 static thread_local bool              g_gpu_thread        = false;
+static thread_local GpuState*         g_gpu_state         = nullptr;
 
 class GpuMutexLock final {
 public:
@@ -98,11 +99,10 @@ public:
 	void SubmitFlipPreparation(uint64_t request_id);
 	void Done();
 	void Shutdown();
-	[[nodiscard]] bool IsStopping();
+	[[nodiscard]] bool        IsStopping();
-	void               SendCommand(Common::UniqueFunction<void>&& command);
+	void                      SendCommand(Common::UniqueFunction<void>&& command);
-	void               SendCommandSync(Common::UniqueFunction<void>&& command);
+	void                      SendCommandSync(Common::UniqueFunction<void>&& command);
-	void SendCommandSyncWithProcessor(Common::UniqueFunction<void, CommandProcessor&>&& command);
+	int                       GetFrameNum();
-	int  GetFrameNum();
 	[[nodiscard]] static bool IsGpuThread() noexcept { return g_gpu_thread; }
 
 private:
@@ -127,6 +127,7 @@ private:
 	void              WaitLocked();
 	void              Enqueue(Submission submission);
 	void              WaitForIdle();
+	void              ProcessCommands();
 	bool              Process(Submission& submission);
 	static void       ThreadRun(void* data);
 	CommandProcessor& GetProcessor(uint32_t queue_id);
@@ -139,6 +140,7 @@ private:
 	Common::CondVar                                m_idle;
 	std::array<std::deque<Submission>, QueueCount> m_queues;
 	std::deque<Common::UniqueFunction<void>>       m_commands;
+	std::atomic_uint32_t                           m_pending_commands {0};
 	uint32_t                                       m_next_queue        = 0;
 	uint32_t                                       m_submission_count  = 0;
 	bool                                           m_processing        = false;
@@ -153,6 +155,8 @@ private:
 	uint64_t        m_submit_id = 0;
 	std::atomic_int m_done_num  = 0;
 	std::jthread    m_thread;
+
+	friend class CommandProcessor;
 };
 
 static bool GraphicsRunDebugDumpEnabled() {
@@ -195,9 +199,25 @@ void GpuState::SendCommand(Common::UniqueFunction<void>&& command) {
 	Common::LockGuard lock(m_queue_mutex);
 	EXIT_IF(!m_accepting);
 	m_commands.push_back(std::move(command));
+	m_pending_commands.fetch_add(1, std::memory_order_release);
 	m_work_available.Signal();
 }
 
+void GpuState::ProcessCommands() {
+	EXIT_IF(!IsGpuThread());
+	while (m_pending_commands.load(std::memory_order_acquire) != 0) {
+		Common::UniqueFunction<void> command;
+		{
+			Common::LockGuard lock(m_queue_mutex);
+			EXIT_IF(m_commands.empty());
+			command = std::move(m_commands.front());
+			m_commands.pop_front();
+			EXIT_IF(m_pending_commands.fetch_sub(1, std::memory_order_acq_rel) == 0);
+		}
+		command();
+	}
+}
+
 void GpuState::SendCommandSync(Common::UniqueFunction<void>&& command) {
 	EXIT_IF(!command);
 	if (IsGpuThread()) {
@@ -212,17 +232,6 @@ void GpuState::SendCommandSync(Common::UniqueFunction<void>&& command) {
 	done.acquire();
 }
 
-void GpuState::SendCommandSyncWithProcessor(
-    Common::UniqueFunction<void, CommandProcessor&>&& command) {
-	EXIT_IF(!command);
-	SendCommandSync([this, operation = std::move(command)]() mutable {
-		EXIT_IF(g_current_processor != nullptr);
-		g_current_processor = m_gfx_cp.get();
-		operation(*m_gfx_cp);
-		g_current_processor = nullptr;
-	});
-}
-
 void GpuState::Submit(uint32_t* cmd_draw_buffer, uint32_t num_draw_dw, uint32_t* cmd_const_buffer,
                       uint32_t num_const_dw, bool trigger_agc_interrupt_on_done) {
 	GpuMutexLock lock(m_submission_mutex);
@@ -509,6 +518,7 @@ void GpuState::ThreadRun(void* data) {
 	EXIT_IF(gpu == nullptr);
 	KYTY_PROFILER_THREAD("Thread_Gpu");
 	g_gpu_thread = true;
+	g_gpu_state  = gpu;
 
 	for (;;) {
 		Submission                   submission;
@@ -529,6 +539,7 @@ void GpuState::ThreadRun(void* data) {
 			} else if (!gpu->m_commands.empty()) {
 				command = std::move(gpu->m_commands.front());
 				gpu->m_commands.pop_front();
+				EXIT_IF(gpu->m_pending_commands.fetch_sub(1, std::memory_order_acq_rel) == 0);
 				gpu->m_processing = true;
 			} else {
 				int selected_queue = -1;
@@ -560,6 +571,7 @@ void GpuState::ThreadRun(void* data) {
 		}
 		if (should_stop) {
 			gpu->m_gfx_cp->BufferWait();
+			g_gpu_state  = nullptr;
 			g_gpu_thread = false;
 			return;
 		}
@@ -741,6 +753,9 @@ void CommandProcessor::SuspendPm4() {
 
 void CommandProcessor::ProcessPm4(Pm4Execution& execution, size_t stop_depth) {
 	while (execution.m_buffer_stack.size() > stop_depth) {
+		if (g_gpu_state != nullptr) {
+			g_gpu_state->ProcessCommands();
+		}
 		const auto buffer_index = execution.m_buffer_stack.size() - 1;
 		auto&      cursor       = execution.m_buffer_stack[buffer_index];
 		if (cursor.deferred_advance_dw != 0) {
@@ -1650,10 +1665,6 @@ void Gpu::SendCommandSync(Common::UniqueFunction<void>&& command) {
 	m_state->SendCommandSync(std::move(command));
 }
 
-void Gpu::SendCommandSyncWithProcessor(Common::UniqueFunction<void, CommandProcessor&>&& command) {
-	m_state->SendCommandSyncWithProcessor(std::move(command));
-}
-
 void Gpu::Submit(uint32_t* draw_commands, uint32_t draw_size_dw, uint32_t* constant_commands,
                  uint32_t constant_size_dw, bool trigger_agc_interrupt_on_done) {
 	EXIT_IF(draw_commands == nullptr || draw_size_dw == 0);
@@ -1679,12 +1690,8 @@ int Gpu::GetFrameNum() const {
 	return m_state->GetFrameNum();
 }
 
-bool Gpu::IsCommandProcessorThread() noexcept {
+bool Gpu::IsGpuThread() noexcept {
-	return g_current_processor != nullptr;
+	return GpuState::IsGpuThread();
-}
-
-CommandProcessor* Gpu::CurrentCommandProcessor() noexcept {
-	return g_current_processor;
 }
 
 } // namespace Libs::Graphics

src/graphics/guest_gpu/graphicsRun.h

@@ -9,7 +9,6 @@
 
 namespace Libs::Graphics {
 
-class CommandProcessor;
 class GpuState;
 class RenderContext;
 
@@ -23,7 +22,6 @@ public:
 	[[nodiscard]] bool IsStopping();
 	void               SendCommand(Common::UniqueFunction<void>&& command);
 	void               SendCommandSync(Common::UniqueFunction<void>&& command);
-	void SendCommandSyncWithProcessor(Common::UniqueFunction<void, CommandProcessor&>&& command);
 
 	void Submit(uint32_t* draw_commands, uint32_t draw_size_dw, uint32_t* constant_commands,
 	            uint32_t constant_size_dw, bool trigger_agc_interrupt_on_done = false);
@@ -33,8 +31,7 @@ public:
 	void Done();
 	[[nodiscard]] int GetFrameNum() const;
 
-	[[nodiscard]] static bool              IsCommandProcessorThread() noexcept;
+	[[nodiscard]] static bool IsGpuThread() noexcept;
-	[[nodiscard]] static CommandProcessor* CurrentCommandProcessor() noexcept;
 
 private:
 	std::unique_ptr<GpuState> m_state;

src/graphics/host_gpu/memoryTracker.h

@@ -29,45 +29,29 @@ public:
 	void               MarkRegionAsGpuModified(uint64_t vaddr, uint64_t size);
 	void               UnmarkRegionAsGpuModified(uint64_t vaddr, uint64_t size);
 	void               UntrackMemory(uint64_t vaddr, uint64_t size);
+	// Removes protection from a range and flushes GPU-owned data when required.
 	template <typename Flush>
-	void InvalidateRegion(uint64_t vaddr, uint64_t size, Flush&& on_flush) {
+	void InvalidateRegion(uint64_t vaddr, uint64_t size, Flush&& on_flush) noexcept {
 		static_assert(std::is_invocable_v<Flush&>);
 		CheckNotInUploadCallback();
 		ValidateRange(vaddr, size);
 
-		const auto update_cpu_state = [this, vaddr, size] {
+		Iterate<false>(vaddr, size, [&](RegionManager* manager, uint64_t offset, uint64_t bytes) {
-			std::lock_guard             access(m_access_mutex);
+			const bool should_flush = [&] {
-			std::vector<RegionManager*> managers;
+				// Perform both the GPU modification check and CPU state change with the lock in
-			Iterate<false>(vaddr, size, [&](RegionManager* manager, uint64_t, uint64_t) {
+				// case the GPU thread is racing to mark the page modified. If a flush is needed,
-				managers.push_back(manager);
+				// on_flush performs the CPU state change.
-			});
+				std::scoped_lock lock(manager->lock);
-			std::vector<std::unique_lock<TrackingSpinLock>> locks;
+				if (manager->IsModified<DirtySource::Gpu>(offset, bytes)) {
-			locks.reserve(managers.size());
+					return true;
-			for (auto* manager: managers) {
+				}
-				locks.emplace_back(manager->lock);
+				manager->ChangeState<DirtySource::Cpu, true>(manager->GetCpuAddr() + offset, bytes);
+				return false;
+			}();
+			if (should_flush) {
+				on_flush();
 			}
-			const bool gpu_modified = Iterate<false>(
+		});
-			    vaddr, size, [](RegionManager* manager, uint64_t offset, uint64_t bytes) {
-				    return manager->IsModified<DirtySource::Gpu>(offset, bytes);
-			    });
-			if (gpu_modified) {
-				return true;
-			}
-			Iterate<false>(vaddr, size,
-			               [](RegionManager* manager, uint64_t offset, uint64_t bytes) {
-				               manager->ChangeState<DirtySource::Cpu, true>(
-				                   manager->GetCpuAddr() + offset, bytes);
-			               });
-			return false;
-		};
-
-		if (!update_cpu_state()) {
-			return;
-		}
-		std::forward<Flush>(on_flush)();
-		if (update_cpu_state()) {
-			EXIT("memory invalidation retained GPU-owned pages\n");
-		}
 	}
 #if KYTY_BUILD == KYTY_BUILD_DEBUG
 	void ValidateGpuDirtyPages(const RangeSet& dirty, uint64_t vaddr, uint64_t size,

src/graphics/host_gpu/renderer/cache/bufferCache.cpp

@@ -3,15 +3,18 @@
 #include "common/assert.h"
 #include "common/logging/log.h"
 #include "common/profiler.h"
+#include "graphics/guest_gpu/graphicsRun.h"
 #include "graphics/host_gpu/graphicContext.h"
 #include "graphics/host_gpu/renderer/cache/resourceMutex.h"
 #include "graphics/host_gpu/renderer/cache/textureCache.h"
 #include "graphics/host_gpu/renderer/commandScheduler.h"
 #include "graphics/host_gpu/renderer/render.h"
+#include "graphics/host_gpu/renderer/renderContext.h"
 #include "kernel/memory.h"
 
 #include <algorithm>
 #include <array>
+#include <cinttypes>
 #include <cstring>
 #include <utility>
 #include <vector>
@@ -288,10 +291,29 @@ void BufferCache::InvalidateMemory(uint64_t vaddr, uint64_t size) {
 		return;
 	}
 	m_memory_tracker.InvalidateRegion(vaddr, size,
-	                                  [this, vaddr, size] { ReadMemory(vaddr, size); });
+	                                  [this, vaddr, size] { ReadMemory(vaddr, size, true); });
 }
 
-void BufferCache::ReadMemory(uint64_t vaddr, uint64_t size) {
+void BufferCache::ReadMemory(uint64_t vaddr, uint64_t size, bool is_write) {
+	if (Gpu::IsGpuThread()) {
+		ReadMemoryOnGpu(vaddr, size, is_write);
+		return;
+	}
+	if (CommandScheduler::InDeferredOperation()) {
+		EXIT("unsupported buffer readback from an asynchronous GPU completion, "
+		     "addr=0x%016" PRIx64 " size=0x%016" PRIx64 "\n",
+		     vaddr, size);
+	}
+	if (m_resource_mutex.IsOwnedByCurrentThread()) {
+		EXIT("unsupported buffer readback from a pre-owned resource transaction, "
+		     "addr=0x%016" PRIx64 " size=0x%016" PRIx64 "\n",
+		     vaddr, size);
+	}
+	m_scheduler.Context().GetGpu().SendCommandSync(
+	    [this, vaddr, size, is_write] { ReadMemoryOnGpu(vaddr, size, is_write); });
+}
+
+void BufferCache::ReadMemoryOnGpu(uint64_t vaddr, uint64_t size, bool is_write) {
 	std::vector<DownloadCopy> copies;
 	{
 		FaultSafeCacheLock lock(this, m_mutex);
@@ -322,9 +344,16 @@ void BufferCache::ReadMemory(uint64_t vaddr, uint64_t size) {
 				    }
 			    }
 		    });
-	}
+		if (copies.empty()) {
-	if (copies.empty()) {
+			if (!is_write) {
-		return;
+				return;
+			}
+			// A preceding read fault can consume the last GPU-owned copy after this write
+			// invalidation has already chosen to flush. Complete the CPU ownership handoff even
+			// though this callback no longer has bytes to download.
+			m_memory_tracker.MarkRegionAsCpuModified(vaddr, size);
+			return;
+		}
 	}
 	auto downloads = RecordDownloads(copies);
 	m_scheduler.FinishCurrent();
@@ -336,6 +365,9 @@ void BufferCache::ReadMemory(uint64_t vaddr, uint64_t size) {
 		}
 		// The enumeration above covered whole dirty pages and every exact interval on them.
 		m_memory_tracker.UnmarkRegionAsGpuModified(vaddr, size);
+		if (is_write) {
+			m_memory_tracker.MarkRegionAsCpuModified(vaddr, size);
+		}
 	}
 }
 

src/graphics/host_gpu/renderer/cache/bufferCache.h

@@ -46,7 +46,7 @@ public:
 	KYTY_CLASS_NO_COPY(BufferCache);
 
 	void                        InvalidateMemory(uint64_t vaddr, uint64_t size);
-	void                        ReadMemory(uint64_t vaddr, uint64_t size);
+	void                        ReadMemory(uint64_t vaddr, uint64_t size, bool is_write = false);
 	void                        UnmapMemory(uint64_t vaddr, uint64_t size);
 	[[nodiscard]] BufferBinding ObtainBuffer(CommandBuffer& command, uint64_t vaddr, uint64_t size,
 	                                         bool is_written = false, bool is_read = true,
@@ -97,6 +97,7 @@ private:
 	void PublishDownloads(std::span<const DownloadRange> downloads);
 	void QueueGarbageDownload(std::span<const DownloadCopy> copies, RetiredBuffer retire);
 	void WriteHostMemory(uint64_t vaddr, std::span<const uint8_t> data);
+	void ReadMemoryOnGpu(uint64_t vaddr, uint64_t size, bool is_write);
 
 	GraphicContext&                                   m_graphics;
 	CommandScheduler&                                 m_scheduler;

src/graphics/host_gpu/renderer/cache/gpuResourceManager.cpp

@@ -1,7 +1,6 @@
 #include "graphics/host_gpu/renderer/cache/gpuResourceManager.h"
 
 #include "common/assert.h"
-#include "graphics/guest_gpu/command_processor/commandProcessor.h"
 #include "graphics/guest_gpu/graphicsRun.h"
 #include "graphics/host_gpu/renderer/commandScheduler.h"
 namespace Libs::Graphics {
@@ -18,69 +17,21 @@ bool GpuResourceManager::HandleFault(PageFaultAccess access, uint64_t fault_vadd
 	if (!IsMapped(fault_vaddr, fault_size)) {
 		return false;
 	}
-	if (CommandScheduler::InDeferredOperation()) {
+	if (access == PageFaultAccess::Write) {
-		EXIT("unsupported guest-memory fault from an asynchronous GPU completion, "
+		m_buffer_cache.InvalidateMemory(fault_vaddr, fault_size);
-		     "addr=0x%016" PRIx64 " access=%u\n",
+		m_texture_cache.InvalidateMemory(fault_vaddr, fault_size);
-		     fault_vaddr, static_cast<uint32_t>(access));
+	} else {
+		m_buffer_cache.ReadMemory(fault_vaddr, fault_size);
 	}
-	bool       handled = false;
+	return true;
-	const auto resolve = [this, access, fault_vaddr, &handled](CommandProcessor& cp) {
-		cp.BeginReadbackTransaction();
-		{
-			ResourceMutex::FaultScope fault(m_resource_mutex);
-			if (access == PageFaultAccess::Write) {
-				m_buffer_cache.InvalidateMemory(fault_vaddr, fault_size);
-				m_texture_cache.InvalidateMemory(fault_vaddr, fault_size);
-			} else {
-				m_buffer_cache.ReadMemory(fault_vaddr, fault_size);
-			}
-			handled = true;
-		}
-		cp.EndReadbackTransaction();
-	};
-	if (auto* cp = Gpu::CurrentCommandProcessor(); cp != nullptr) {
-		resolve(*cp);
-		return handled;
-	}
-	if (m_resource_mutex.IsOwnedByCurrentThread()) {
-		EXIT("unsupported page fault from a pre-owned resource transaction, addr=0x%016" PRIx64
-		     " access=%u\n",
-		     fault_vaddr, static_cast<uint32_t>(access));
-	}
-	EXIT_IF(m_gpu == nullptr);
-	m_gpu->SendCommandSyncWithProcessor(resolve);
-	return handled;
 }
 
 bool GpuResourceManager::InvalidateMemory(uint64_t vaddr, uint64_t size) {
 	if (!IsMapped(vaddr, size)) {
 		return false;
 	}
-	if (CommandScheduler::InDeferredOperation()) {
+	m_buffer_cache.InvalidateMemory(vaddr, size);
-		EXIT("unsupported memory invalidation from an asynchronous GPU completion, "
+	m_texture_cache.InvalidateMemory(vaddr, size);
-		     "addr=0x%016" PRIx64 " size=0x%016" PRIx64 "\n",
-		     vaddr, size);
-	}
-	const auto resolve = [this, vaddr, size](CommandProcessor& cp) {
-		cp.BeginReadbackTransaction();
-		{
-			ResourceMutex::FaultScope fault(m_resource_mutex);
-			m_buffer_cache.InvalidateMemory(vaddr, size);
-			m_texture_cache.InvalidateMemory(vaddr, size);
-		}
-		cp.EndReadbackTransaction();
-	};
-	if (auto* cp = Gpu::CurrentCommandProcessor(); cp != nullptr) {
-		resolve(*cp);
-		return true;
-	}
-	if (m_resource_mutex.IsOwnedByCurrentThread()) {
-		EXIT("unsupported memory invalidation from a pre-owned resource transaction, "
-		     "addr=0x%016" PRIx64 " size=0x%016" PRIx64 "\n",
-		     vaddr, size);
-	}
-	EXIT_IF(m_gpu == nullptr);
-	m_gpu->SendCommandSyncWithProcessor(resolve);
 	return true;
 }
 

src/graphics/host_gpu/renderer/cache/resourceMutex.cpp

@@ -4,18 +4,10 @@
 
 namespace Libs::Graphics {
 
-ResourceMutex::FaultScope::FaultScope(ResourceMutex& mutex): m_mutex(mutex) {
-	m_resource_preowned = m_mutex.BeginFault();
-}
-
-ResourceMutex::FaultScope::~FaultScope() {
-	m_mutex.EndFault(m_resource_preowned);
-}
-
 ResourceMutex::~ResourceMutex() {
 	std::lock_guard state(m_state);
-	if (m_resource_owner != std::thread::id {} || m_fault_owner != std::thread::id {}) {
+	if (m_resource_owner != std::thread::id {}) {
-		EXIT("ResourceMutex destroyed with an active owner or fault transaction\n");
+		EXIT("ResourceMutex destroyed with an active owner\n");
 	}
 }
 
@@ -26,9 +18,6 @@ void ResourceMutex::lock() {
 		if (m_resource_owner == current) {
 			EXIT("recursive resource transaction\n");
 		}
-		if (m_fault_owner == current) {
-			EXIT("resource transaction re-entered from a page-fault callback\n");
-		}
 	}
 	m_resource.lock();
 	std::lock_guard state(m_state);
@@ -55,39 +44,4 @@ bool ResourceMutex::IsOwnedByCurrentThread() {
 	return m_resource_owner == std::this_thread::get_id();
 }
 
-bool ResourceMutex::BeginFault() {
-	const auto current = std::this_thread::get_id();
-	{
-		std::lock_guard state(m_state);
-		if (m_fault_owner == current) {
-			EXIT("nested resource page-fault transaction\n");
-		}
-		if (m_resource_owner == current) {
-			m_fault_owner = current;
-			return true;
-		}
-	}
-	lock();
-	std::lock_guard state(m_state);
-	if (m_resource_owner != current || m_fault_owner != std::thread::id {}) {
-		EXIT("resource fault transaction acquired inconsistent ownership\n");
-	}
-	m_fault_owner = current;
-	return false;
-}
-
-void ResourceMutex::EndFault(bool resource_preowned) {
-	const auto current = std::this_thread::get_id();
-	{
-		std::lock_guard state(m_state);
-		if (m_fault_owner != current || m_resource_owner != current) {
-			EXIT("resource page-fault transaction ended with inconsistent ownership\n");
-		}
-		m_fault_owner = {};
-	}
-	if (!resource_preowned) {
-		unlock();
-	}
-}
-
 } // namespace Libs::Graphics

src/graphics/host_gpu/renderer/cache/resourceMutex.h

@@ -8,21 +8,9 @@
 
 namespace Libs::Graphics {
 
-// Owner-tracked shared buffer/image transaction. External faults pause GPU submissions first;
+// Owner-tracked shared buffer/image transaction.
-// command-processor faults drain pending guest processors before entering this transaction.
 class ResourceMutex final {
 public:
-	class FaultScope final {
-	public:
-		explicit FaultScope(ResourceMutex& mutex);
-		~FaultScope();
-		KYTY_CLASS_NO_COPY(FaultScope);
-
-	private:
-		ResourceMutex& m_mutex;
-		bool           m_resource_preowned = false;
-	};
-
 	ResourceMutex() = default;
 	~ResourceMutex();
 	KYTY_CLASS_NO_COPY(ResourceMutex);
@@ -32,15 +20,9 @@ public:
 	[[nodiscard]] bool IsOwnedByCurrentThread();
 
 private:
-	friend class FaultScope;
-
-	[[nodiscard]] bool BeginFault();
-	void               EndFault(bool resource_preowned);
-
 	std::mutex      m_resource;
 	std::mutex      m_state;
 	std::thread::id m_resource_owner;
-	std::thread::id m_fault_owner;
 };
 
 } // namespace Libs::Graphics

tests/MemoryTrackerTests.cpp

@@ -6,7 +6,9 @@
 #include <cstdio>
 #include <cstdlib>
 #include <cstring>
+#include <semaphore>
 #include <string>
+#include <thread>
 #include <vector>
 
 #if KYTY_PLATFORM == KYTY_PLATFORM_WINDOWS
@@ -265,6 +267,7 @@ void TestRangeInvalidation() {
     flushes++;
     tracker.ForEachDownloadRange<true>(address + 16, size - 32,
                                        [](uint64_t, uint64_t) noexcept {});
+    tracker.MarkRegionAsCpuModified(address + 16, size - 32);
   });
   Check(flushes == 1 && !tracker.IsRegionGpuModified(address, size) &&
             tracker.IsRegionCpuModified(address, size) && IsWritable(memory) &&
@@ -277,6 +280,53 @@ void TestRangeInvalidation() {
   Release(page_manager, memory, size);
 }
 
+void TestGpuReacquisitionAfterInvalidation() {
+  TrackerHarness harness;
+  auto &tracker = harness.tracker;
+  auto &page_manager = harness.page_manager;
+  const auto page_size = page_manager.GetPageSize();
+  auto *memory = Allocate(page_manager, 1);
+  const auto address = reinterpret_cast<uint64_t>(memory);
+
+  tracker.ForEachUploadRange(
+      address, page_size, true, [](uint64_t, uint64_t) noexcept {},
+      []() noexcept {});
+  Check(tracker.IsRegionGpuModified(address, page_size) &&
+            !tracker.IsRegionCpuModified(address, page_size),
+        "reacquisition setup did not establish GPU ownership");
+
+  uint32_t flushes = 0;
+  uint32_t uploads = 0;
+  std::binary_semaphore reacquire{0};
+  std::binary_semaphore reacquired{0};
+  std::jthread publisher([&] {
+    reacquire.acquire();
+    tracker.ForEachUploadRange(
+        address + 16, 32, true,
+        [&](uint64_t, uint64_t) noexcept { uploads++; }, []() noexcept {});
+    reacquired.release();
+  });
+  tracker.InvalidateRegion(address + 16, 32, [&] {
+    flushes++;
+    tracker.ForEachDownloadRange<true>(address + 16, 32,
+                                       [](uint64_t, uint64_t) noexcept {});
+    tracker.MarkRegionAsCpuModified(address + 16, 32);
+    reacquire.release();
+    reacquired.acquire();
+  });
+  publisher.join();
+  Check(flushes == 1 && uploads == 1 &&
+            tracker.IsRegionGpuModified(address, page_size) &&
+            !tracker.IsRegionCpuModified(address, page_size) &&
+            !IsWritable(memory),
+        "invalidation rejected a new generation of GPU ownership");
+
+  tracker.UnmarkRegionAsGpuModified(address, page_size);
+  tracker.MarkRegionAsCpuModified(address, page_size);
+  tracker.UntrackMemory(address, page_size);
+  Release(page_manager, memory, page_size);
+}
+
 void TestGpuDirtyBits() {
   TrackerHarness harness;
   auto &tracker = harness.tracker;
@@ -635,6 +685,7 @@ int main(int argc, char **argv) {
   TestQueriesDoNotRequireMappedOwnership();
   TestCpuDirtyUpload();
   TestRangeInvalidation();
+  TestGpuReacquisitionAfterInvalidation();
   TestGpuDirtyBits();
   TestExactDirtyIntervalsSharingTrackerPage();
   TestGpuDownloadProtectionMirrors();

tests/ResourceMutexTests.cpp

@@ -35,171 +35,50 @@ void YieldMany() {
 	}
 }
 
-void TestFaultBlocksPublisher() {
+void TestOwnership() {
-	ResourceMutex    mutex;
+	ResourceMutex mutex;
-	std::atomic_bool publisher_started {false};
+	Check(!mutex.IsOwnedByCurrentThread(), "fresh mutex reported an owner");
-	std::atomic_bool publisher_entered {false};
-	std::thread      publisher;
 	{
-		ResourceMutex::FaultScope fault(mutex);
+		std::lock_guard lock(mutex);
-		publisher = std::thread([&] {
+		Check(mutex.IsOwnedByCurrentThread(), "owner tracking was not installed");
-			publisher_started.store(true, std::memory_order_release);
-			std::lock_guard lock(mutex);
-			publisher_entered.store(true, std::memory_order_release);
-		});
-		while (!publisher_started.load(std::memory_order_acquire)) {
-			std::this_thread::yield();
-		}
-		YieldMany();
-		Check(!publisher_entered.load(std::memory_order_acquire),
-		      "publisher entered during active fault transaction");
 	}
-	publisher.join();
+	Check(!mutex.IsOwnedByCurrentThread(), "owner tracking survived unlock");
-	Check(publisher_entered.load(std::memory_order_acquire),
-	      "publisher did not resume after fault transaction");
 }
 
-void TestFaultDrainsExistingOwner() {
+void TestSerializesTransactions() {
 	ResourceMutex    mutex;
 	std::unique_lock owner(mutex);
-	std::atomic_bool fault_started {false};
+	std::atomic_bool contender_started {false};
-	std::atomic_bool fault_entered {false};
+	std::atomic_bool contender_entered {false};
-	std::atomic_bool release_fault {false};
+	std::thread      contender([&] {
-	std::atomic_bool publisher_started {false};
+		contender_started.store(true, std::memory_order_release);
-	std::atomic_bool publisher_entered {false};
-	std::thread      fault([&] {
-		fault_started.store(true, std::memory_order_release);
-		ResourceMutex::FaultScope scope(mutex);
-		fault_entered.store(true, std::memory_order_release);
-		while (!release_fault.load(std::memory_order_acquire)) {
-			std::this_thread::yield();
-		}
-	});
-	while (!fault_started.load(std::memory_order_acquire)) {
-		std::this_thread::yield();
-	}
-	YieldMany();
-	Check(!fault_entered.load(std::memory_order_acquire),
-	      "fault transaction did not wait for existing owner");
-	owner.unlock();
-	while (!fault_entered.load(std::memory_order_acquire)) {
-		std::this_thread::yield();
-	}
-	std::thread publisher([&] {
-		publisher_started.store(true, std::memory_order_release);
 		std::lock_guard lock(mutex);
-		publisher_entered.store(true, std::memory_order_release);
+		contender_entered.store(true, std::memory_order_release);
 	});
-	while (!publisher_started.load(std::memory_order_acquire)) {
+	while (!contender_started.load(std::memory_order_acquire)) {
 		std::this_thread::yield();
 	}
 	YieldMany();
-	Check(!publisher_entered.load(std::memory_order_acquire),
+	Check(!contender_entered.load(std::memory_order_acquire),
-	      "publisher entered before drained fault completed");
+	      "contender entered an active resource transaction");
-	release_fault.store(true, std::memory_order_release);
-	fault.join();
-	publisher.join();
-	Check(publisher_entered.load(std::memory_order_acquire),
-	      "publisher did not resume after drained fault");
-}
-
-void TestFaultScopesSerialize() {
-	ResourceMutex    mutex;
-	std::atomic_bool first_entered {false};
-	std::atomic_bool second_started {false};
-	std::atomic_bool second_entered {false};
-	std::atomic_bool release_first {false};
-	std::atomic_bool release_second {false};
-	std::atomic_bool publisher_entered {false};
-	std::thread      first([&] {
-		ResourceMutex::FaultScope scope(mutex);
-		first_entered.store(true, std::memory_order_release);
-		while (!release_first.load(std::memory_order_acquire)) {
-			std::this_thread::yield();
-		}
-	});
-	while (!first_entered.load(std::memory_order_acquire)) {
-		std::this_thread::yield();
-	}
-	std::thread second([&] {
-		second_started.store(true, std::memory_order_release);
-		ResourceMutex::FaultScope scope(mutex);
-		second_entered.store(true, std::memory_order_release);
-		while (!release_second.load(std::memory_order_acquire)) {
-			std::this_thread::yield();
-		}
-	});
-	while (!second_started.load(std::memory_order_acquire)) {
-		std::this_thread::yield();
-	}
-	YieldMany();
-	Check(!second_entered.load(std::memory_order_acquire),
-	      "second fault transaction entered before first completed");
-	release_first.store(true, std::memory_order_release);
-	while (!second_entered.load(std::memory_order_acquire)) {
-		std::this_thread::yield();
-	}
-	std::thread publisher([&] {
-		std::lock_guard lock(mutex);
-		publisher_entered.store(true, std::memory_order_release);
-	});
-	YieldMany();
-	Check(!publisher_entered.load(std::memory_order_acquire),
-	      "publisher entered during second fault transaction");
-	release_second.store(true, std::memory_order_release);
-	first.join();
-	second.join();
-	publisher.join();
-	Check(publisher_entered.load(std::memory_order_acquire),
-	      "publisher did not resume after serialized faults");
-}
-
-void TestPreownedFaultKeepsResourceTransaction() {
-	ResourceMutex    mutex;
-	std::atomic_bool publisher_started {false};
-	std::atomic_bool publisher_entered {false};
-	std::thread      publisher;
-	std::unique_lock owner(mutex);
-	{
-		ResourceMutex::FaultScope fault(mutex);
-		publisher = std::thread([&] {
-			publisher_started.store(true, std::memory_order_release);
-			std::lock_guard lock(mutex);
-			publisher_entered.store(true, std::memory_order_release);
-		});
-		while (!publisher_started.load(std::memory_order_acquire)) {
-			std::this_thread::yield();
-		}
-		YieldMany();
-		Check(!publisher_entered.load(std::memory_order_acquire),
-		      "publisher entered during preowned fault transaction");
-	}
-	YieldMany();
-	Check(!publisher_entered.load(std::memory_order_acquire),
-	      "preowned fault released its outer resource transaction");
 	owner.unlock();
-	publisher.join();
+	contender.join();
-	Check(publisher_entered.load(std::memory_order_acquire),
+	Check(contender_entered.load(std::memory_order_acquire),
-	      "publisher did not resume after outer transaction");
+	      "contender did not resume after resource transaction");
 }
 
 #if KYTY_PLATFORM == KYTY_PLATFORM_WINDOWS
-[[noreturn]] void RunDeathCase(const char* name) {
+[[noreturn]] void RunDeathCase() {
 	ResourceMutex mutex;
-	if (std::strcmp(name, "recursive-lock") == 0) {
+	std::lock_guard first(mutex);
-		std::lock_guard first(mutex);
+	std::lock_guard second(mutex);
-		std::lock_guard second(mutex);
-	} else if (std::strcmp(name, "nested-fault") == 0) {
-		ResourceMutex::FaultScope first(mutex);
-		ResourceMutex::FaultScope second(mutex);
-	}
 	std::_Exit(0x7f);
 }
 
-void CheckDeathCase(const char* name) {
+void CheckDeathCase() {
 	char path[MAX_PATH] {};
 	Check(GetModuleFileNameA(nullptr, path, MAX_PATH) != 0, "GetModuleFileName failed");
-	std::string       command = std::string("\"") + path + "\" --death " + name;
+	std::string       command = std::string("\"") + path + "\" --death";
 	std::vector<char> mutable_command(command.begin(), command.end());
 	mutable_command.push_back('\0');
 	STARTUPINFOA        startup {sizeof(startup)};
@@ -224,20 +103,17 @@ void CheckDeathCase(const char* name) {
 
 int main(int argc, char** argv) {
 #if KYTY_PLATFORM == KYTY_PLATFORM_WINDOWS
-	if (argc == 3 && std::strcmp(argv[1], "--death") == 0) {
+	if (argc == 2 && std::strcmp(argv[1], "--death") == 0) {
-		RunDeathCase(argv[2]);
+		RunDeathCase();
 	}
 #else
 	(void)argc;
 	(void)argv;
 #endif
-	TestFaultBlocksPublisher();
+	TestOwnership();
-	TestFaultDrainsExistingOwner();
+	TestSerializesTransactions();
-	TestFaultScopesSerialize();
-	TestPreownedFaultKeepsResourceTransaction();
 #if KYTY_PLATFORM == KYTY_PLATFORM_WINDOWS
-	CheckDeathCase("recursive-lock");
+	CheckDeathCase();
-	CheckDeathCase("nested-fault");
 #endif
 	std::puts("ResourceMutexTests: all cases passed");
 	return 0;

tests/ShaderRecompilerComputeTests.cpp

@@ -58,6 +58,7 @@
 
 #include <algorithm>
 #include <array>
+#include <atomic>
 #include <bit>
 #include <chrono>
 #include <cinttypes>
@@ -1372,12 +1373,14 @@ public:
 
 	void CheckGpuMappedRangeLifecycle() {
 		EnsureRuntimeContext();
-		CommandScheduler scheduler(Renderer(), m_runtime_context);
+		auto&            context = Renderer();
+		CommandScheduler scheduler(context, m_runtime_context);
 		HW::Context      registers {};
 		HW::UserConfig   user_config {};
 		HW::Shader       shaders {};
 		scheduler.Begin(registers, user_config, shaders);
-		Gpu                gpu(Renderer());
+		context.InitializeGpu(nullptr);
+		auto&              gpu = context.GetGpu();
 		GpuResourceManager resources(m_runtime_context, scheduler);
 		resources.SetGpu(&gpu);
 
@@ -1416,6 +1419,7 @@ public:
 
 		resources.SetGpu(nullptr);
 		scheduler.Finish();
+		context.ShutdownGpu();
 		std::printf("[host]    %-32s ok\n", "GpuMappedRangeLifecycle");
 	}
 
@@ -1567,21 +1571,73 @@ public:
 			gpu_thread = std::this_thread::get_id();
 			Require("GpuCommandLane", "FIFO", order == 1,
 			        "synchronous command overtook an older host command");
-			Require("GpuCommandLane", "generic context",
+			Require("GpuCommandLane", "GPU context", Gpu::IsGpuThread(),
-			        !Gpu::IsCommandProcessorThread() && Gpu::CurrentCommandProcessor() == nullptr,
+			        "host command did not run on the GPU thread");
-			        "generic host command manufactured a PM4 processor context");
 			gpu.SendCommandSync([&nested_thread] { nested_thread = std::this_thread::get_id(); });
 			order = 2;
 		});
-		gpu.SendCommandSyncWithProcessor([&](CommandProcessor& processor) {
-			Require("GpuCommandLane", "processor context",
-			        Gpu::IsCommandProcessorThread() && Gpu::CurrentCommandProcessor() == &processor,
-			        "resource command did not receive its explicit processor context");
-		});
 		Require("GpuCommandLane", "dispatch thread",
 		        order == 2 && gpu_thread != caller_thread && nested_thread == gpu_thread,
 		        "host command did not run on the GPU thread or nested sync deadlocked");
 
+		alignas(uint32_t) uint32_t packet_marker_a = 0;
+		alignas(uint32_t) uint32_t packet_marker_b = 0;
+		const auto write_packet = [](uint32_t* packet, uint32_t* destination, uint32_t value) {
+			const auto destination_address = reinterpret_cast<uint64_t>(destination);
+			packet[0] = KYTY_PM4(5, Pm4::IT_WRITE_DATA, 0);
+			packet[1] = 0;
+			packet[2] = static_cast<uint32_t>(destination_address);
+			packet[3] = static_cast<uint32_t>(destination_address >> 32u);
+			packet[4] = value;
+		};
+		std::array<uint32_t, 1024> polling_loop {};
+		for (size_t i = 0; i < polling_loop.size() - 4; i += 2) {
+			polling_loop[i]     = KYTY_PM4(2, Pm4::IT_NOP, Pm4::R_ZERO);
+			polling_loop[i + 1] = 0;
+		}
+		const auto loop_address = reinterpret_cast<uint64_t>(polling_loop.data());
+		polling_loop[1020] = KYTY_PM4(4, Pm4::IT_INDIRECT_BUFFER, 0);
+		polling_loop[1021] = static_cast<uint32_t>(loop_address);
+		polling_loop[1022] = static_cast<uint32_t>(loop_address >> 32u);
+		polling_loop[1023] = 0x0f200000u | static_cast<uint32_t>(polling_loop.size());
+
+		std::array<uint32_t, 14> polling_commands {};
+		write_packet(polling_commands.data(), &packet_marker_a, 11);
+		polling_commands[5] = KYTY_PM4(4, Pm4::IT_INDIRECT_BUFFER, 0);
+		polling_commands[6] = static_cast<uint32_t>(loop_address);
+		polling_commands[7] = static_cast<uint32_t>(loop_address >> 32u);
+		polling_commands[8] = 0x0f200000u | static_cast<uint32_t>(polling_loop.size());
+		write_packet(polling_commands.data() + 9, &packet_marker_b, 22);
+		std::binary_semaphore stream_gate_entered {0};
+		std::binary_semaphore stream_gate_release {0};
+		gpu.SendCommand([&] {
+			stream_gate_entered.release();
+			stream_gate_release.acquire();
+		});
+		gpu.Submit(polling_commands.data(), polling_commands.size(), nullptr, 0);
+		stream_gate_entered.acquire();
+		stream_gate_release.release();
+		uint32_t packet_marker_a_at_callback = UINT32_MAX;
+		uint32_t packet_marker_b_at_callback = UINT32_MAX;
+		for (uint32_t attempt = 0; attempt < 8 && packet_marker_a_at_callback != 11; attempt++) {
+			gpu.SendCommandSync([&] {
+				if (packet_marker_a == 11 && packet_marker_b == 0) {
+					packet_marker_a_at_callback = packet_marker_a;
+					packet_marker_b_at_callback = packet_marker_b;
+					polling_loop[1020] = KYTY_PM4(2, Pm4::IT_NOP, Pm4::R_ZERO);
+					polling_loop[1021] = 0;
+					polling_loop[1022] = KYTY_PM4(2, Pm4::IT_NOP, Pm4::R_ZERO);
+					polling_loop[1023] = 0;
+				}
+			});
+			std::this_thread::yield();
+		}
+		gpu.Done();
+		Require("GpuCommandLane", "packet-boundary command polling",
+		        packet_marker_a_at_callback == 11 && packet_marker_b_at_callback == 0 &&
+		            packet_marker_a == 11 && packet_marker_b == 22,
+		        "host command waited for an entire non-suspended PM4 stream");
+
 		uint32_t   label   = 0;
 		uint32_t   prefix  = 0;
 		uint32_t   suffix  = 0;
@@ -1760,7 +1816,7 @@ public:
 		gpu.Submit(dma_commands.data(), static_cast<uint32_t>(dma_commands.size()), nullptr, 0);
 		gpu.Done();
 		constexpr uint32_t clean_fill_value = 0xdecafbad;
-		gpu.SendCommandSyncWithProcessor([&](CommandProcessor&) {
+		gpu.SendCommandSync([&] {
 			auto& buffer_cache = resources.GetBufferCache();
 			Require("GpuCommandLane", "clean cached setup",
 			        buffer_cache.HasPageOverlap(clean_cached_fill, sizeof(source_words)) &&
@@ -1840,14 +1896,116 @@ public:
 		                                                       clean_copy_words[1]},
 		        "host-memory fill/copy did not update the clean cached mirror");
 
-		gpu.SendCommandSyncWithProcessor([&](CommandProcessor& processor) {
+		gpu.SendCommandSync([&] {
-			Require("GpuCommandLane", "processor fault context",
+			Require("GpuCommandLane", "GPU memory invalidation",
-			        Gpu::CurrentCommandProcessor() == &processor,
-			        "processor resource test lost its command context");
-			Require("GpuCommandLane", "processor memory invalidation",
 			        resources.InvalidateMemory(fault_base, sizeof(uint32_t)),
-			        "processor memory invalidation did not find its mapped range");
+			        "GPU memory invalidation did not find its mapped range");
 		});
+
+		auto& buffer_cache = resources.GetBufferCache();
+		Require("GpuCommandLane", "clean tracked fault setup",
+		        buffer_cache.HasPageOverlap(immediate_dst, sizeof(uint32_t)) &&
+		            !buffer_cache.HasGpuDirtyBytes(immediate_dst, sizeof(uint32_t)),
+		        "clean write-fault test address is not backed by a clean cached Buffer");
+		constexpr uint64_t dirty_fault_address = fault_base + 0xd000;
+		constexpr uint32_t dirty_fault_value   = 0x5a17c0deu;
+		constexpr uint32_t dirty_fault_stale   = 0x0ddba11u;
+		std::memcpy(reinterpret_cast<void*>(dirty_fault_address), &dirty_fault_stale,
+		            sizeof(dirty_fault_stale));
+		gpu.SendCommandSync([&] {
+			auto dirty = buffer_cache.ObtainBuffer(scheduler.Current(), dirty_fault_address,
+			                                       sizeof(dirty_fault_value), true, false);
+			Require("GpuCommandLane", "dirty tracked fault setup", dirty.owner != nullptr,
+			        "dirty write-fault test could not create a cached Buffer");
+			scheduler.Current().RetainResourceUntilFence(dirty.owner);
+			buffer_cache.FillBuffer(dirty_fault_address, sizeof(dirty_fault_value),
+			                        dirty_fault_value);
+		});
+		Require("GpuCommandLane", "dirty tracked fault ownership",
+		        buffer_cache.HasGpuDirtyBytes(dirty_fault_address, sizeof(dirty_fault_value)),
+		        "dirty write-fault test address did not acquire GPU ownership");
+
+		std::binary_semaphore block_entered {0};
+		std::binary_semaphore block_release {0};
+		std::binary_semaphore clean_fault_complete {0};
+		bool                  clean_fault_handled = false;
+		gpu.SendCommand([&] {
+			block_entered.release();
+			block_release.acquire();
+		});
+		block_entered.acquire();
+		std::jthread clean_fault_thread([&] {
+			clean_fault_handled = resources.HandleFault(PageFaultAccess::Write, immediate_dst);
+			clean_fault_complete.release();
+		});
+		const bool clean_fault_was_direct =
+		    clean_fault_complete.try_acquire_for(std::chrono::seconds(1));
+		if (!clean_fault_was_direct) {
+			block_release.release();
+			clean_fault_complete.acquire();
+		}
+		clean_fault_thread.join();
+		Require("GpuCommandLane", "clean fault direct path",
+		        clean_fault_was_direct && clean_fault_handled,
+		        "a clean CPU write fault waited for the blocked GPU command lane");
+
+		std::binary_semaphore dirty_fault_complete {0};
+		bool                  dirty_fault_handled = false;
+		std::jthread dirty_fault_thread([&] {
+			dirty_fault_handled =
+			    resources.HandleFault(PageFaultAccess::Write, dirty_fault_address);
+			dirty_fault_complete.release();
+		});
+		const bool dirty_fault_bypassed_lane =
+		    dirty_fault_complete.try_acquire_for(std::chrono::milliseconds(200));
+		block_release.release();
+		if (!dirty_fault_bypassed_lane) {
+			dirty_fault_complete.acquire();
+		}
+		dirty_fault_thread.join();
+		uint32_t dirty_fault_backing = 0;
+		std::memcpy(&dirty_fault_backing, reinterpret_cast<const void*>(dirty_fault_address),
+		            sizeof(dirty_fault_backing));
+		Require("GpuCommandLane", "dirty fault synchronization",
+		        !dirty_fault_bypassed_lane && dirty_fault_handled &&
+		            dirty_fault_backing == dirty_fault_value &&
+		            !buffer_cache.HasGpuDirtyBytes(dirty_fault_address, sizeof(dirty_fault_value)),
+		        "a GPU-dirty write fault bypassed synchronization or lost the published bytes");
+
+		constexpr uint64_t empty_copy_fault_address = fault_base + 0xe000;
+		constexpr uint32_t empty_copy_fault_value   = 0x6b28d1efu;
+		gpu.SendCommandSync([&] {
+			auto dirty = buffer_cache.ObtainBuffer(scheduler.Current(), empty_copy_fault_address,
+			                                       sizeof(empty_copy_fault_value), true, false);
+			Require("GpuCommandLane", "empty-copy write setup", dirty.owner != nullptr,
+			        "empty-copy write test could not create a cached Buffer");
+			scheduler.Current().RetainResourceUntilFence(dirty.owner);
+			buffer_cache.FillBuffer(empty_copy_fault_address, sizeof(empty_copy_fault_value),
+			                        empty_copy_fault_value);
+			buffer_cache.ReadMemory(empty_copy_fault_address, sizeof(empty_copy_fault_value));
+			const bool gpu_owned = buffer_cache.IsRegionGpuModified(
+			    empty_copy_fault_address, sizeof(empty_copy_fault_value));
+			const bool cpu_owned = buffer_cache.IsRegionCpuModified(
+			    empty_copy_fault_address, sizeof(empty_copy_fault_value));
+			Require("GpuCommandLane", "empty-copy read ownership",
+			        !gpu_owned && !cpu_owned,
+			        "readback did not leave clean ownership for the queued write invalidation");
+			buffer_cache.ReadMemory(empty_copy_fault_address, sizeof(empty_copy_fault_value), true);
+		});
+		uint32_t empty_copy_fault_backing = 0;
+		std::memcpy(&empty_copy_fault_backing, reinterpret_cast<const void*>(empty_copy_fault_address),
+		            sizeof(empty_copy_fault_backing));
+		const bool empty_copy_cpu_owned = buffer_cache.IsRegionCpuModified(
+		    empty_copy_fault_address, sizeof(empty_copy_fault_value));
+		const bool empty_copy_gpu_owned = buffer_cache.IsRegionGpuModified(
+		    empty_copy_fault_address, sizeof(empty_copy_fault_value));
+		const bool empty_copy_dirty =
+		    buffer_cache.HasGpuDirtyBytes(empty_copy_fault_address, sizeof(empty_copy_fault_value));
+		Require("GpuCommandLane", "empty-copy write ownership",
+		        empty_copy_fault_backing == empty_copy_fault_value &&
+		            empty_copy_cpu_owned && !empty_copy_gpu_owned && !empty_copy_dirty,
+		        "write invalidation without a remaining copy did not establish CPU ownership");
+
 		resources.UnmapMemory(fault_base, fault_size);
 		Require("GpuCommandLane", "processor fault unmap",
 		        Libs::LibKernel::Memory::KernelMunmap(fault_base, fault_size) == 0,
@@ -2073,12 +2231,14 @@ public:
 		constexpr uint32_t    ring_fault_second_value       = 0x4e5f6071u;
 
 		EnsureRuntimeContext();
-		CommandScheduler scheduler(Renderer(), m_runtime_context);
+		auto&            context = Renderer();
+		CommandScheduler scheduler(context, m_runtime_context);
 		HW::Context      registers {};
 		HW::UserConfig   user_config {};
 		HW::Shader       shaders {};
 		scheduler.Begin(registers, user_config, shaders);
-		Gpu gpu(Renderer());
+		context.InitializeGpu(nullptr);
+		auto& gpu = context.GetGpu();
 
 		int64_t direct_offset = -1;
 		Require(name, "direct allocation",
@@ -2471,7 +2631,7 @@ public:
 			resources.UnmapMemory(base, allocation_size);
 			scheduler.Finish();
 		}
-		gpu.Shutdown();
+		context.ShutdownGpu();
 		Require(name, "unmap direct backing",
 		        Libs::LibKernel::Memory::KernelMunmap(base, allocation_size) == 0,
 		        "dirty-GC direct-memory mapping release failed");
@@ -2488,12 +2648,14 @@ public:
 		constexpr uint64_t    allocation_size      = 0x2800000;
 		constexpr uint64_t    allocation_alignment = 0x200000;
 		EnsureRuntimeContext();
-		CommandScheduler scheduler(Renderer(), m_runtime_context);
+		auto&            context = Renderer();
+		CommandScheduler scheduler(context, m_runtime_context);
 		HW::Context      registers {};
 		HW::UserConfig   user_config {};
 		HW::Shader       shaders {};
 		scheduler.Begin(registers, user_config, shaders);
-		Gpu     gpu(Renderer());
+		context.InitializeGpu(nullptr);
+		auto&   gpu = context.GetGpu();
 		int64_t direct_offset = -1;
 		Require(name, "direct allocation",
 		        Libs::LibKernel::Memory::KernelAllocateDirectMemory(
@@ -4949,7 +5111,7 @@ public:
 			resources.UnmapMemory(base, allocation_size);
 			scheduler.Finish();
 		}
-		gpu.Shutdown();
+		context.ShutdownGpu();
 		Require(name, "unmap direct backing",
 		        Libs::LibKernel::Memory::KernelMunmap(base, allocation_size) == 0,
 		        "cache direct-memory mapping release failed");