render: track pages for DMA fills and copies

src/graphics/host_gpu/renderer/cache/bufferCache.cpp

@@ -13,7 +13,6 @@
 #include "kernel/memory.h"
 
 #include <algorithm>
-#include <array>
 #include <cinttypes>
 #include <cstring>
 #include <memory>
@@ -523,26 +522,6 @@ std::pair<Buffer*, uint64_t> BufferCache::ObtainBufferForImage(uint64_t vaddr, u
 	return {&m_staging_buffer, stage_offset};
 }
 
-void BufferCache::WriteHostMemory(uint64_t vaddr, std::span<const uint8_t> data) {
-	if (vaddr == 0 || data.empty() || data.size() > UINT64_MAX - vaddr) {
-		EXIT("BufferCache: invalid host DMA write\n");
-	}
-	Libs::LibKernel::Memory::WriteBacking(vaddr, data.data(), data.size());
-
-	const auto end = vaddr + data.size();
-	for (const auto& [address, id]: m_buffers) {
-		auto&      buffer     = m_slot_buffers[id];
-		const auto buffer_end = address + buffer.Size();
-		const auto begin      = std::max(vaddr, address);
-		const auto range_end  = std::min(end, buffer_end);
-		if (begin >= range_end) {
-			continue;
-		}
-		WriteDataBuffer(buffer, begin, data.data() + begin - vaddr, range_end - begin);
-		TouchBuffer(buffer);
-	}
-}
-
 void BufferCache::FillBuffer(uint64_t vaddr, uint64_t size, uint32_t value, bool is_gds) {
 	if ((vaddr & 3u) != 0 || size == 0 || (size & 3u) != 0 || size > UINT64_MAX - vaddr) {
 		EXIT("BufferCache: fill range must be dword aligned\n");
@@ -558,23 +537,11 @@ void BufferCache::FillBuffer(uint64_t vaddr, uint64_t size, uint32_t value, bool
 		EXIT("BufferCache: invalid fill memory address\n");
 	}
 	(void)m_texture_cache.ClearMeta(vaddr);
-	{
+	if (!IsRegionGpuModified(vaddr, size)) {
-		const auto region = m_texture_cache.QueryRegion(vaddr, size);
+		// Access the guest mapping so write faults invalidate cached buffers and images.
-		if (!HasGpuDirtyBytes(vaddr, size) && !region.gpu_image_bytes) {
+		auto* destination = reinterpret_cast<uint32_t*>(vaddr);
-			if (region.image_bytes) {
+		std::fill(destination, destination + size / sizeof(uint32_t), value);
-				m_texture_cache.InvalidateMemory(vaddr, size);
+		return;
-			}
-			std::array<uint32_t, 4096> values;
-			values.fill(value);
-			const std::span<const uint8_t> bytes {reinterpret_cast<const uint8_t*>(values.data()),
-			                                      sizeof(values)};
-			for (uint64_t offset = 0; offset < size;) {
-				const auto chunk = std::min<uint64_t>(size - offset, bytes.size());
-				WriteHostMemory(vaddr + offset, bytes.first(chunk));
-				offset += chunk;
-			}
-			return;
-		}
 	}
 
 	m_texture_cache.InvalidateMemoryFromGPU(vaddr, size);
@@ -597,26 +564,10 @@ void BufferCache::CopyBuffer(uint64_t dst_vaddr, uint64_t src_vaddr, uint64_t si
 		     " size=0x%016" PRIx64 " src_gds=%d dst_gds=%d\n",
 		     src_vaddr, dst_vaddr, size, static_cast<int>(src_gds), static_cast<int>(dst_gds));
 	}
-	const auto src_region =
+	if (src_memory && dst_memory && !IsRegionGpuModified(dst_vaddr, size) &&
-	    src_memory ? m_texture_cache.QueryRegion(src_vaddr, size) : TextureCache::RegionInfo {};
+	    !IsRegionGpuModified(src_vaddr, size) && !m_texture_cache.FindImageFromRange(src_vaddr, size)) {
-	const auto dst_region =
+		std::memcpy(reinterpret_cast<void*>(dst_vaddr), reinterpret_cast<const void*>(src_vaddr),
-	    dst_memory ? m_texture_cache.QueryRegion(dst_vaddr, size) : TextureCache::RegionInfo {};
+		            size);
-	if (src_memory && dst_memory && !HasGpuDirtyBytes(src_vaddr, size) &&
-	    !HasGpuDirtyBytes(dst_vaddr, size) && !src_region.gpu_image_bytes &&
-	    !dst_region.gpu_image_bytes) {
-		if (dst_region.image_bytes) {
-			m_texture_cache.InvalidateMemory(dst_vaddr, size);
-		}
-		std::array<uint8_t, 64 * 1024> bytes;
-		for (uint64_t offset = 0; offset < size;) {
-			const auto chunk = std::min<uint64_t>(size - offset, bytes.size());
-			if (!Libs::LibKernel::Memory::TryReadBacking(src_vaddr + offset, bytes.data(),
-			                                             chunk)) {
-				EXIT("BufferCache: host DMA source has no direct backing\n");
-			}
-			WriteHostMemory(dst_vaddr + offset, std::span {bytes}.first(chunk));
-			offset += chunk;
-		}
 		return;
 	}
 

src/graphics/host_gpu/renderer/cache/bufferCache.h

@@ -107,7 +107,6 @@ private:
 	                                      uint64_t total_size);
 	[[nodiscard]] bool SynchronizeBufferFromImage(Buffer& buffer, uint64_t vaddr, uint64_t size);
 	void DownloadBufferMemory(std::span<const DownloadCopy> copies);
-	void WriteHostMemory(uint64_t vaddr, std::span<const uint8_t> data);
 	void ReadMemoryOnGpu(uint64_t vaddr, uint64_t size, bool is_write);
 
 	GraphicContext&                                   m_graphics;

src/graphics/host_gpu/renderer/cache/textureCache.cpp

@@ -1836,22 +1836,18 @@ void TextureCache::InvalidateMemoryFromGPU(uint64_t address, uint64_t size) {
 	}
 }
 
-TextureCache::RegionInfo TextureCache::QueryRegion(uint64_t address, uint64_t size) {
+bool TextureCache::IsRegionGpuModified(uint64_t address, uint64_t size) {
-	RegionInfo result {};
 	if (!GuestRange {address, size}.Valid()) {
-		return result;
+		return false;
 	}
 	std::scoped_lock lock {m_lock};
-	for (const auto id: FindImagesInRegion(address, size, true)) {
+	for (const auto id: FindImagesInRegion(address, size, false)) {
 		const auto& image = m_slot_images[id];
-		if (image.depth_id) {
+		if (!image.depth_id && image.IsGpuModified()) {
-			continue;
+			return true;
 		}
-		result.image_pages = true;
-		result.image_bytes |= image.Overlaps(address, size);
-		result.gpu_image_bytes |= image.GpuOverlaps(address, size);
 	}
-	return result;
+	return false;
 }
 
 void TextureCache::InvalidateCpuAliases(uint64_t address, uint64_t size) {

src/graphics/host_gpu/renderer/cache/textureCache.h

@@ -37,12 +37,6 @@ public:
 		BindingType   type = BindingType::Texture;
 	};
 
-	struct RegionInfo {
-		bool image_pages     = false;
-		bool image_bytes     = false;
-		bool gpu_image_bytes = false;
-	};
-
 	TextureCache(GraphicContext& graphics, CommandScheduler& scheduler, PageManager& page_manager,
 	             BufferCache& buffer_cache);
 	~TextureCache();
@@ -66,7 +60,7 @@ public:
 	                                        uint32_t packed_clear);
 	void               InvalidateMemory(uint64_t address, uint64_t size);
 	void               InvalidateMemoryFromGPU(uint64_t address, uint64_t size);
-	[[nodiscard]] RegionInfo QueryRegion(uint64_t address, uint64_t size);
+	[[nodiscard]] bool IsRegionGpuModified(uint64_t address, uint64_t size);
 
 	[[nodiscard]] bool IsMeta(uint64_t address);
 	[[nodiscard]] bool IsMetaCleared(uint64_t address, uint32_t slice,

src/graphics/host_gpu/renderer/image/image.h

@@ -128,9 +128,6 @@ public:
 		return pages ? ImagePageRangesOverlap(info.data.address, info.data.size, address, size)
 		             : ImageRangeOverlaps(info.data.address, info.data.size, address, size);
 	}
-	[[nodiscard]] bool GpuOverlaps(uint64_t address, uint64_t size) const noexcept {
-		return IsGpuModified() && Overlaps(address, size);
-	}
 	[[nodiscard]] bool SafeToDownload() const noexcept {
 		return IsGpuModified() && !IsBufferModified() && !IsCpuDirty();
 	}

src/kernel/memory.cpp

@@ -871,7 +871,7 @@ bool TryReadGpuCleanBacking(uint64_t vaddr, void* data, uint64_t size) {
 	if (g_gpu_resources != nullptr && IsGpuAddressRange(vaddr, size)) {
 		if (!Graphics::GuestGpu::IsGpuThread() ||
 		    GetGpuResources().GetBufferCache().HasGpuDirtyBytes(vaddr, size) ||
-		    GetGpuResources().GetTextureCache().QueryRegion(vaddr, size).gpu_image_bytes) {
+		    GetGpuResources().GetTextureCache().IsRegionGpuModified(vaddr, size)) {
 			return false;
 		}
 	}

tests/ShaderRecompilerComputeTests.cpp

@@ -1,5 +1,6 @@
 #include "common/assert.h"
 #include "common/emulatorConfig.h"
+#include "common/hostException.h"
 #include "common/logging/log.h"
 #include "common/subsystems.h"
 #include "common/threads.h"
@@ -4290,6 +4291,21 @@ public:
     {
       GpuResourceManager resources(m_runtime_context, scheduler);
       resources.SetGpu(&gpu);
+      namespace Exception = Common::HostException;
+      Require(name, "guest fault handler",
+              Exception::InstallHandler([](const Exception::ExceptionInfo &info) {
+                if (info.type != Exception::ExceptionType::AccessViolation ||
+                    (info.access_violation_type != Exception::AccessViolationType::Read &&
+                     info.access_violation_type != Exception::AccessViolationType::Write)) {
+                  return false;
+                }
+                const auto access =
+                    info.access_violation_type == Exception::AccessViolationType::Write
+                        ? PageFaultAccess::Write : PageFaultAccess::Read;
+                return LibKernel::Memory::HandleGpuFault(access, info.access_violation_vaddr);
+              }),
+              "failed to install the production guest-memory fault route");
+      LibKernel::Memory::InstallGpuResources(&resources);
       auto &texture_cache = resources.GetTextureCache();
       const auto [narrow_download, narrow_download_offset] =
           TextureCacheTestAccess::MapDownload(texture_cache, 4, 4);
@@ -5303,9 +5319,8 @@ public:
               !texture_cache.IsMetaCleared(ms_htile_address, 0) &&
               !resources.GetBufferCache().HasGpuDirtyBytes(
                   base + ms_stencil_offset, ms_stencil_size) &&
-              !texture_cache
+              !texture_cache.IsRegionGpuModified(base + ms_stencil_offset,
-                   .QueryRegion(base + ms_stencil_offset, ms_stencil_size)
+                                                 ms_stencil_size),
-                   .gpu_image_bytes,
           "unequal-sample overlap did not run the color-to-MS-depth pass "
           "without manufacturing stencil ownership");
       auto &oversized_ms = texture_cache.GetImage(ms_depth_image);
@@ -7558,6 +7573,7 @@ public:
       resources.SetGpu(nullptr);
       resources.UnmapMemory(base, allocation_size);
       scheduler.Finish();
+      LibKernel::Memory::InstallGpuResources(nullptr);
     }
     context.ShutdownGpu();
     Require(name, "unmap direct backing",