Optimize texture cache tracking (#123)

Optimize texture cache page tracking

src/graphics/host_gpu/pageManager.cpp

@@ -2,6 +2,7 @@
 
 #include "graphics/host_gpu/regionDefinitions.h"
 
+#include <algorithm>
 #include <array>
 #include <atomic>
 #include <cstdarg>
@@ -164,16 +165,21 @@ int ToHostProtection(uint32_t protection) {
 	}
 }
 
+struct HostMapping {
+	uint64_t end        = 0;
+	uint32_t protection = UNKNOWN_PROTECTION;
+};
+
 // Async-signal-safe lookup in the address-ordered /proc/self/maps.
-uint32_t QueryHostProtection(uint64_t vaddr) noexcept {
+HostMapping QueryHostMapping(uint64_t vaddr) noexcept {
 	int fd = ::open("/proc/self/maps", O_RDONLY | O_CLOEXEC); // NOLINT
 	if (fd < 0) {
-		return UNKNOWN_PROTECTION;
+		return {};
 	}
 
 	enum class Field { Start, End, Perms, Rest };
 
-	uint32_t result     = UNKNOWN_PROTECTION;
+	HostMapping result {};
 	auto        field      = Field::Start;
 	uint64_t    start      = 0;
 	uint64_t    end        = 0;
@@ -248,7 +254,8 @@ uint32_t QueryHostProtection(uint64_t vaddr) noexcept {
 					if (vaddr < start) {
 						done = true;
 					} else if (vaddr < end && perms_len >= 2) {
-						result = perms[1] == 'w'   ? READ_WRITE_PROTECTION
+						result.end        = end;
+						result.protection = perms[1] == 'w'   ? READ_WRITE_PROTECTION
 						                    : perms[0] == 'r' ? READ_ONLY_PROTECTION
 						                                      : NO_ACCESS_PROTECTION;
 						done              = true;
@@ -266,6 +273,10 @@ uint32_t QueryHostProtection(uint64_t vaddr) noexcept {
 	::close(fd);
 	return result;
 }
+
+uint32_t QueryHostProtection(uint64_t vaddr) noexcept {
+	return QueryHostMapping(vaddr).protection;
+}
 #endif
 
 class SpinGuard final {
@@ -323,6 +334,26 @@ struct PageManager::Impl {
 		std::array<PageState, REGION_PAGES> pages;
 	};
 
+	class PageRangeGuard final {
+	public:
+		explicit PageRangeGuard(std::span<PageState*> pages): m_pages(pages) {
+			for (auto* page: m_pages) {
+				while (page->lock.test_and_set(std::memory_order_acquire)) {
+					std::atomic_signal_fence(std::memory_order_seq_cst);
+				}
+			}
+		}
+		~PageRangeGuard() {
+			for (auto it = m_pages.rbegin(); it != m_pages.rend(); ++it) {
+				(*it)->lock.clear(std::memory_order_release);
+			}
+		}
+		KYTY_CLASS_NO_COPY(PageRangeGuard);
+
+	private:
+		std::span<PageState*> m_pages;
+	};
+
 	Impl(PageFaultHandler handler, void* context): fault_handler(handler), fault_context(context) {
 		if (fault_handler == nullptr) {
 			Fatal("null fault handler");
@@ -337,8 +368,7 @@ struct PageManager::Impl {
 #elif defined(__APPLE__)
 		// Under Rosetta the host page size is 4 KB, matching TRACKER_PAGE_SIZE.
 		if (static_cast<uint64_t>(getpagesize()) != PAGE_SIZE) {
-			Fatal("unsupported host page size 0x%08" PRIx32,
+			Fatal("unsupported host page size 0x%08" PRIx32, static_cast<uint32_t>(getpagesize()));
-			      static_cast<uint32_t>(getpagesize()));
 		}
 #else
 		const auto host_page_size = ::sysconf(_SC_PAGESIZE);
@@ -405,42 +435,57 @@ struct PageManager::Impl {
 		if (old_protection == NO_ACCESS_PROTECTION && new_protection != NO_ACCESS_PROTECTION) {
 			page.late_read_pending = true;
 		}
-		if ((old_protection == NO_ACCESS_PROTECTION ||
+		if ((old_protection == NO_ACCESS_PROTECTION || old_protection == READ_ONLY_PROTECTION) &&
-		     old_protection == READ_ONLY_PROTECTION) &&
 		    new_protection == READ_WRITE_PROTECTION) {
 			page.late_write_pending = true;
 		}
 	}
 
-	static uint32_t QueryProtection([[maybe_unused]] PageState& page, uint64_t vaddr) {
+	static void ValidateInitialProtection(std::span<PageState*> pages, uint64_t vaddr) {
+		const auto end = vaddr + pages.size() * PAGE_SIZE;
 #if KYTY_PLATFORM == KYTY_PLATFORM_WINDOWS
+		for (auto address = vaddr; address < end;) {
 			MEMORY_BASIC_INFORMATION info {};
-		if (VirtualQuery(reinterpret_cast<const void*>(static_cast<uintptr_t>(vaddr)), &info,
+			if (VirtualQuery(reinterpret_cast<const void*>(static_cast<uintptr_t>(address)), &info,
 			                 sizeof(info)) == 0 ||
 			    info.State != MEM_COMMIT || info.Protect != PAGE_READWRITE) {
 				Fatal("basic path requires PAGE_READWRITE at 0x%016" PRIx64 " (state=0x%08" PRIx32
 				      ", protection=0x%08" PRIx32 ")",
-			      vaddr, static_cast<uint32_t>(info.State), static_cast<uint32_t>(info.Protect));
+				      address, static_cast<uint32_t>(info.State),
+				      static_cast<uint32_t>(info.Protect));
+			}
+			const auto region_end = reinterpret_cast<uint64_t>(info.BaseAddress) + info.RegionSize;
+			if (region_end <= address) {
+				Fatal("VirtualQuery returned an invalid region at 0x%016" PRIx64, address);
+			}
+			address = std::min(end, region_end);
 		}
-		return info.Protect;
 #elif defined(__APPLE__)
-		const uint32_t protection = MachQueryPageProt(vaddr);
+		for (auto address = vaddr; address < end; address += PAGE_SIZE) {
+			const uint32_t protection = MachQueryPageProt(address);
 			if (protection != PAGE_READWRITE) {
-			Fatal("basic path requires PAGE_READWRITE at 0x%016" PRIx64 " (protection=0x%08" PRIx32
+				Fatal("basic path requires PAGE_READWRITE at 0x%016" PRIx64
-			      ")",
+				      " (protection=0x%08" PRIx32 ")",
-			      vaddr, protection);
+				      address, protection);
+			}
 		}
-		return protection;
 #else
-		const auto host_protection = QueryHostProtection(vaddr);
+		for (auto address = vaddr; address < end;) {
-		if (host_protection != READ_WRITE_PROTECTION) {
+			const auto mapping = QueryHostMapping(address);
+			if (mapping.protection != READ_WRITE_PROTECTION || mapping.end <= address) {
 				Fatal("basic path requires a read/write mapping at 0x%016" PRIx64
 				      " (protection=0x%08" PRIx32 ")",
-			      vaddr, host_protection);
+				      address, mapping.protection);
+			}
+			address = std::min(end, mapping.end);
+		}
+		for (auto* page: pages) {
+			page->current_protection = READ_WRITE_PROTECTION;
 		}
-		page.current_protection = host_protection;
-		return host_protection;
 #endif
+		for (auto* page: pages) {
+			page->original_protection = READ_WRITE_PROTECTION;
+		}
 	}
 
 	static bool AllowsAccess([[maybe_unused]] const PageState& page, uint64_t vaddr,
@@ -470,7 +515,8 @@ struct PageManager::Impl {
 		const auto permitted = [](uint32_t protection, PageFaultAccess wanted) {
 			switch (wanted) {
 				case PageFaultAccess::Read:
-					return protection == READ_ONLY_PROTECTION || protection == READ_WRITE_PROTECTION;
+					return protection == READ_ONLY_PROTECTION ||
+					       protection == READ_WRITE_PROTECTION;
 				case PageFaultAccess::Write: return protection == READ_WRITE_PROTECTION;
 				default: return false;
 			}
@@ -483,26 +529,84 @@ struct PageManager::Impl {
 #endif
 	}
 
-	static void Protect([[maybe_unused]] PageState& page, uint64_t vaddr, uint32_t protection,
+	static void ProtectRange(std::span<PageState*> pages, uint64_t vaddr, uint32_t protection,
-	                    uint32_t expected_old, bool fault_path) noexcept {
+	                         std::span<const uint32_t> expected_old, bool fault_path) noexcept {
+		const auto size = pages.size() * PAGE_SIZE;
+		if (pages.size() != expected_old.size()) {
+			FailFast("protection range state size mismatch");
+		}
 #if KYTY_PLATFORM == KYTY_PLATFORM_WINDOWS
+		struct HostRange {
+			uint64_t begin = 0;
+			uint64_t end   = 0;
+		};
+		std::vector<HostRange> host_ranges;
+		const auto             end = vaddr + size;
+		for (auto address = vaddr; address < end;) {
+			MEMORY_BASIC_INFORMATION info {};
+			if (VirtualQuery(reinterpret_cast<const void*>(static_cast<uintptr_t>(address)), &info,
+			                 sizeof(info)) == 0 ||
+			    info.State != MEM_COMMIT) {
+				if (fault_path) {
+					FailFast("VirtualProtect fault transition did not match expected protection");
+				}
+				Fatal("invalid protection transition at 0x%016" PRIx64 ", state=0x%08" PRIx32
+				      ", new=0x%08" PRIx32,
+				      address, static_cast<uint32_t>(info.State), protection);
+			}
+			const auto region_end = reinterpret_cast<uint64_t>(info.BaseAddress) + info.RegionSize;
+			const auto query_end  = std::min(end, region_end);
+			if (query_end <= address) {
+				if (fault_path) {
+					FailFast("VirtualQuery returned an invalid fault transition region");
+				}
+				Fatal("VirtualQuery returned an invalid region at 0x%016" PRIx64, address);
+			}
+			const auto first_page = static_cast<size_t>((address - vaddr) / PAGE_SIZE);
+			const auto last_page =
+			    static_cast<size_t>((query_end - vaddr + PAGE_SIZE - 1) / PAGE_SIZE);
+			for (auto page = first_page; page < last_page; page++) {
+				if (info.Protect != expected_old[page]) {
+					if (fault_path) {
+						FailFast(
+						    "VirtualProtect fault transition did not match expected protection");
+					}
+					Fatal("invalid protection transition at 0x%016" PRIx64 ", actual=0x%08" PRIx32
+					      ", expected=0x%08" PRIx32 ", new=0x%08" PRIx32,
+					      vaddr + page * PAGE_SIZE, static_cast<uint32_t>(info.Protect),
+					      expected_old[page], protection);
+				}
+			}
+			const auto allocation = reinterpret_cast<uint64_t>(info.AllocationBase);
+			if (host_ranges.empty() || allocation != host_ranges.back().begin) {
+				host_ranges.push_back({allocation, query_end});
+			} else {
+				host_ranges.back().end = query_end;
+			}
+			address = query_end;
+		}
+		for (auto range: host_ranges) {
+			range.begin               = std::max(range.begin, vaddr);
 			DWORD      old_protection = 0;
-		if (VirtualProtect(reinterpret_cast<void*>(static_cast<uintptr_t>(vaddr)), PAGE_SIZE,
+			const auto first_page     = static_cast<size_t>((range.begin - vaddr) / PAGE_SIZE);
-		                   protection, &old_protection) == 0 ||
+			if (VirtualProtect(reinterpret_cast<void*>(static_cast<uintptr_t>(range.begin)),
-		    old_protection != expected_old) {
+			                   range.end - range.begin, protection, &old_protection) == 0 ||
+			    old_protection != expected_old[first_page]) {
 				if (fault_path) {
 					FailFast("VirtualProtect fault transition did not match expected protection");
 				}
 				Fatal("invalid protection transition at 0x%016" PRIx64 ", old=0x%08" PRIx32
 				      ", expected=0x%08" PRIx32 ", new=0x%08" PRIx32,
-			      vaddr, static_cast<uint32_t>(old_protection), expected_old, protection);
+				      range.begin, static_cast<uint32_t>(old_protection), expected_old[first_page],
+				      protection);
+			}
 		}
 #elif defined(__APPLE__)
 		// mprotect cannot report the previous protection, so the expected_old comparison
 		// is dropped; the tracker is the sole mutator of these pages and drives the
 		// transition from its own shadow state.
 		(void)expected_old;
-		if (mprotect(reinterpret_cast<void*>(static_cast<uintptr_t>(vaddr)), PAGE_SIZE,
+		if (mprotect(reinterpret_cast<void*>(static_cast<uintptr_t>(vaddr)), size,
 		             PageProtToPosix(protection)) != 0) {
 			if (fault_path) {
 				FailFast("mprotect fault transition failed");
@@ -510,16 +614,18 @@ struct PageManager::Impl {
 			Fatal("mprotect failed at 0x%016" PRIx64 ", new=0x%08" PRIx32, vaddr, protection);
 		}
 #else
-		if (page.current_protection != UNKNOWN_PROTECTION &&
+		for (size_t i = 0; i < pages.size(); i++) {
-		    page.current_protection != expected_old) {
+			const auto actual = pages[i]->current_protection;
+			if (actual != UNKNOWN_PROTECTION && actual != expected_old[i]) {
 				if (fault_path) {
 					FailFast("mprotect fault transition did not match expected protection");
 				}
 				Fatal("invalid protection transition at 0x%016" PRIx64 ", old=0x%08" PRIx32
 				      ", expected=0x%08" PRIx32 ", new=0x%08" PRIx32,
-			      vaddr, page.current_protection, expected_old, protection);
+				      vaddr + i * PAGE_SIZE, actual, expected_old[i], protection);
 			}
-		if (::mprotect(reinterpret_cast<void*>(static_cast<uintptr_t>(vaddr)), PAGE_SIZE,
+		}
+		if (::mprotect(reinterpret_cast<void*>(static_cast<uintptr_t>(vaddr)), size,
 		               ToHostProtection(protection)) != 0) {
 			if (fault_path) {
 				FailFast("mprotect failed on the fault path");
@@ -527,10 +633,19 @@ struct PageManager::Impl {
 			Fatal("mprotect failed at 0x%016" PRIx64 ", new=0x%08" PRIx32 " (%s)", vaddr,
 			      protection, std::strerror(errno));
 		}
-		page.current_protection = protection;
+		for (auto* page: pages) {
+			page->current_protection = protection;
+		}
 #endif
 	}
 
+	static void Protect(PageState& page, uint64_t vaddr, uint32_t protection, uint32_t expected_old,
+	                    bool fault_path) noexcept {
+		PageState* pages[]    = {&page};
+		uint32_t   expected[] = {expected_old};
+		ProtectRange(pages, vaddr, protection, expected, fault_path);
+	}
+
 	std::unique_ptr<std::atomic<Region*>[]> regions;
 	std::vector<std::unique_ptr<Region>>    region_storage;
 	std::mutex                              region_mutex;
@@ -634,38 +749,119 @@ void PageManager::UpdatePageWatchers(bool track, uint64_t vaddr, uint64_t size,
 	if (mode != PageWatchMode::Write && mode != PageWatchMode::ReadWrite) {
 		Fatal("invalid watcher mode");
 	}
+	const auto begin = PageStart(vaddr);
 	const auto end   = PageEnd(vaddr, size);
-	for (auto page_vaddr = PageStart(vaddr); page_vaddr < end; page_vaddr += PAGE_SIZE) {
+	for (auto chunk_begin = begin; chunk_begin < end;) {
+		const auto chunk_end = std::min(end, (chunk_begin / REGION_SIZE + 1) * REGION_SIZE);
 		auto*      region =
-		    track ? m_impl->GetOrCreateRegion(page_vaddr) : m_impl->FindRegion(page_vaddr);
+		    track ? m_impl->GetOrCreateRegion(chunk_begin) : m_impl->FindRegion(chunk_begin);
 		if (region == nullptr) {
-			Fatal("untracking unknown page 0x%016" PRIx64, page_vaddr);
+			Fatal("untracking unknown page 0x%016" PRIx64, chunk_begin);
 		}
-		auto&     page = m_impl->GetPage(*region, page_vaddr);
+
-		SpinGuard lock(page.lock);
+		const auto page_count = static_cast<size_t>((chunk_end - chunk_begin) / PAGE_SIZE);
+		std::vector<Impl::PageState*> pages;
+		pages.reserve(page_count);
+		for (auto address = chunk_begin; address < chunk_end; address += PAGE_SIZE) {
+			pages.push_back(&m_impl->GetPage(*region, address));
+		}
+		Impl::PageRangeGuard lock(pages);
+
+		std::vector<uint8_t> first_watchers(page_count);
+		for (size_t i = 0; i < page_count; i++) {
+			auto&      page    = *pages[i];
+			const auto address = chunk_begin + i * PAGE_SIZE;
 			if (page.resolving && track) {
 				FailFast("new page watcher raced active fault resolution");
 			}
 			if (page.mappings == 0) {
-			Fatal("watching unmapped page 0x%016" PRIx64, page_vaddr);
+				Fatal("watching unmapped page 0x%016" PRIx64, address);
 			}
 			auto& watchers =
 			    (mode == PageWatchMode::ReadWrite ? page.access_watchers : page.write_watchers);
 			if (track) {
 				if (watchers == std::numeric_limits<uint32_t>::max()) {
-				Fatal("watcher overflow at 0x%016" PRIx64, page_vaddr);
+					Fatal("watcher overflow at 0x%016" PRIx64, address);
 				}
-			const bool first_watcher = page.write_watchers == 0 && page.access_watchers == 0;
+				first_watchers[i] = page.write_watchers == 0 && page.access_watchers == 0;
-			if (first_watcher) {
+			} else {
-				page.original_protection = Impl::QueryProtection(page, page_vaddr);
+				if (watchers == 0) {
+					Fatal("watcher underflow at 0x%016" PRIx64, address);
 				}
+				if (page.backing_writer != 0 && page.backing_writer != CurrentThread()) {
+					Fatal("backing write ownership changed at 0x%016" PRIx64, address);
+				}
+			}
+		}
+
+		if (track) {
+			for (size_t first = 0; first < page_count;) {
+				while (first < page_count && first_watchers[first] == 0) {
+					first++;
+				}
+				auto last = first;
+				while (last < page_count && first_watchers[last] != 0) {
+					last++;
+				}
+				if (first != last) {
+					Impl::ValidateInitialProtection(std::span {pages}.subspan(first, last - first),
+					                                chunk_begin + first * PAGE_SIZE);
+				}
+				first = last;
+			}
+		}
+
+		std::vector<uint32_t> old_protections(page_count);
+		std::vector<uint32_t> new_protections(page_count);
+		std::vector<uint8_t>  transitions(page_count);
+		for (size_t i = 0; i < page_count; i++) {
+			auto& page = *pages[i];
+			auto& watchers =
+			    (mode == PageWatchMode::ReadWrite ? page.access_watchers : page.write_watchers);
 			const auto old_protection = Impl::WatcherProtection(page);
+			if (track) {
 				watchers++;
-			const auto new_protection = Impl::WatcherProtection(page);
+			} else {
-			if (new_protection != old_protection) {
+				watchers--;
-				Impl::Protect(page, page_vaddr, new_protection, old_protection, false);
 			}
-			switch (new_protection) {
+			const auto new_protection = Impl::WatcherProtection(page);
+			old_protections[i]        = old_protection;
+			new_protections[i]        = new_protection;
+			if (new_protection != old_protection && (track || page.backing_writer == 0)) {
+				transitions[i] = 1;
+			}
+		}
+
+		for (size_t first = 0; first < page_count;) {
+			while (first < page_count && transitions[first] == 0) {
+				first++;
+			}
+			if (first == page_count) {
+				break;
+			}
+			const auto protection = new_protections[first];
+			auto       current    = first + 1;
+			auto       last       = current;
+			for (; current < page_count && new_protections[current] == protection; current++) {
+				if (old_protections[current] != new_protections[current] &&
+				    transitions[current] == 0) {
+					break;
+				}
+				if (transitions[current] != 0) {
+					last = current + 1;
+				}
+			}
+			Impl::ProtectRange(std::span {pages}.subspan(first, last - first),
+			                   chunk_begin + first * PAGE_SIZE, protection,
+			                   std::span {old_protections}.subspan(first, last - first), false);
+			first = current;
+		}
+
+		for (size_t i = 0; i < page_count; i++) {
+			auto&      page       = *pages[i];
+			const auto protection = new_protections[i];
+			if (track) {
+				switch (protection) {
 					case NO_ACCESS_PROTECTION:
 						page.late_read_pending  = false;
 						page.late_write_pending = false;
@@ -673,27 +869,15 @@ void PageManager::UpdatePageWatchers(bool track, uint64_t vaddr, uint64_t size,
 					case READ_ONLY_PROTECTION: page.late_write_pending = false; break;
 					default: break;
 				}
-		} else {
+			} else if (page.backing_writer == 0) {
-			if (watchers == 0) {
+				Impl::PublishDelayedFaults(page, old_protections[i], protection);
-				Fatal("watcher underflow at 0x%016" PRIx64, page_vaddr);
+				if (page.write_watchers == 0 && page.access_watchers == 0) {
-			}
-			if (page.backing_writer != 0 && page.backing_writer != CurrentThread()) {
-				Fatal("backing write ownership changed at 0x%016" PRIx64, page_vaddr);
-			}
-			const auto old_protection = Impl::WatcherProtection(page);
-			watchers--;
-			const auto new_protection = Impl::WatcherProtection(page);
-			if (page.backing_writer == 0 && new_protection != old_protection) {
-				Impl::Protect(page, page_vaddr, new_protection, old_protection, false);
-			}
-			if (page.backing_writer == 0) {
-				Impl::PublishDelayedFaults(page, old_protection, new_protection);
-			}
-			if (page.backing_writer == 0 && page.write_watchers == 0 && page.access_watchers == 0) {
 					page.original_protection = 0;
 				}
 			}
 		}
+		chunk_begin = chunk_end;
+	}
 }
 
 void PageManager::OnGpuMap(uint64_t vaddr, uint64_t size, GpuAccess access) {

src/graphics/host_gpu/renderer/cache/textureCache.cpp

@@ -7,13 +7,13 @@
 #include "graphics/guest_gpu/gpu_format.h"
 #include "graphics/guest_gpu/tile.h"
 #include "graphics/host_gpu/graphicContext.h"
-#include "graphics/host_gpu/renderer/image/textureCommon.h"
 #include "graphics/host_gpu/renderer/cache/bufferCache.h"
+#include "graphics/host_gpu/renderer/cache/resourceMutex.h"
 #include "graphics/host_gpu/renderer/commandScheduler.h"
 #include "graphics/host_gpu/renderer/image/imageView.h"
-#include "graphics/host_gpu/renderer/render.h"
+#include "graphics/host_gpu/renderer/image/textureCommon.h"
-#include "graphics/host_gpu/renderer/cache/resourceMutex.h"
 #include "graphics/host_gpu/renderer/image/tiler.h"
+#include "graphics/host_gpu/renderer/render.h"
 #include "kernel/memory.h"
 
 #include <algorithm>
@@ -58,8 +58,8 @@ private:
 TextureCache::TextureCache(GraphicContext& graphics, CommandScheduler& scheduler,
                            PageManager& page_manager, BufferCache& buffer_cache,
                            ResourceMutex& resource_mutex)
-    : m_graphics(graphics), m_scheduler(scheduler),
+    : m_graphics(graphics), m_scheduler(scheduler), m_page_manager(page_manager),
-      m_memory_tracker(page_manager, PageWatchMode::Write), m_blit_helper(graphics, scheduler),
+      m_blit_helper(graphics, scheduler),
       m_tiler(std::make_unique<TileManager>(graphics, scheduler,
                                             buffer_cache.GetUtilityBuffer(MemoryUsage::Stream))),
       m_buffer_cache(buffer_cache), m_resource_mutex(resource_mutex),
@@ -80,7 +80,7 @@ TextureCache::TextureCache(GraphicContext& graphics, CommandScheduler& scheduler
 TextureCache::~TextureCache() {
 	for (uint32_t index = 0; index < m_slots.size(); index++) {
 		if (m_slots[index].image != nullptr && m_slots[index].image->registered) {
-			UnregisterImage({index, m_slots[index].generation}, false);
+			UnregisterImage({index, m_slots[index].generation});
 		}
 		m_slots[index].image.reset();
 	}
@@ -117,8 +117,7 @@ bool TextureCache::SafeToDownload(const Image& image) {
 		return false;
 	}
 	const auto range = image.info.data;
-	return !m_buffer_cache.HasGpuDirtyBytes(range.address, range.size) &&
+	return !m_buffer_cache.HasGpuDirtyBytes(range.address, range.size);
-	       !m_memory_tracker.IsRegionCpuModified(range.address, range.size);
 }
 
 Image& TextureCache::ResolveImage(ImageId id) {
@@ -187,21 +186,17 @@ void TextureCache::RegisterImage(ImageId id) {
 	m_total_used_memory += image.AccountedSize();
 }
 
-void TextureCache::UnregisterImage(ImageId id, bool release_tracking) {
+void TextureCache::UnregisterImage(ImageId id) {
 	auto& image = ResolveImage(id);
 	if (!image.registered) {
 		return;
 	}
+	UntrackImage(id);
 	std::vector<ImageOwnerIndex::ByteRange> releases;
 	if (!m_image_owner_index.Unregister(id, releases)) {
 		EXIT("TextureCache: image missing from owner index\n");
 	}
 	m_lru_cache.Free(image.lru_id);
-	if (release_tracking) {
-		for (const auto& range: releases) {
-			m_memory_tracker.UntrackMemory(range.address, range.size);
-		}
-	}
 	const auto accounted = image.AccountedSize();
 	if (accounted > m_total_used_memory) {
 		EXIT("TextureCache: image accounting underflow\n");
@@ -210,7 +205,7 @@ void TextureCache::UnregisterImage(ImageId id, bool release_tracking) {
 	image.registered = false;
 }
 
-void TextureCache::DeleteImage(ImageId id, bool release_tracking) {
+void TextureCache::DeleteImage(ImageId id) {
 	auto owner = ResolveOwner(id);
 	if (owner == nullptr || !owner->registered) {
 		return;
@@ -224,7 +219,7 @@ void TextureCache::DeleteImage(ImageId id, bool release_tracking) {
 			}
 		}
 		for (const auto association: associations) {
-			ReleaseGpuTracking(association);
+			ClearGpuModified(association);
 			DeleteImage(association);
 		}
 	}
@@ -235,7 +230,7 @@ void TextureCache::DeleteImage(ImageId id, bool release_tracking) {
 	if (owner->info.metadata.kind == ImageMetadataKind::Htile) {
 		m_surface_metas.erase(owner->info.metadata.range.address);
 	}
-	UnregisterImage(id, release_tracking);
+	UnregisterImage(id);
 	const auto erase_slot = [this, id, retained = owner] {
 		auto& slot = m_slots[id.index];
 		if (slot.generation != id.generation || slot.image != retained) {
@@ -266,10 +261,10 @@ void TextureCache::DeleteImages(std::span<const ImageId> ids,
 			continue;
 		}
 		if (native_source == id) {
-			ReleaseGpuTracking(id);
+			ClearGpuModified(id);
 		} else if (owner->IsGpuModified()) {
 			DownloadImage(id);
-			ReleaseGpuTracking(id);
+			ClearGpuModified(id);
 		}
 		DeleteImage(id);
 	}
@@ -296,6 +291,121 @@ void TextureCache::TouchImage(Image& image) {
 	}
 }
 
+void TextureCache::TrackImage(ImageId id) {
+	auto& image = ResolveImage(id);
+	if (!image.registered) {
+		return;
+	}
+	const auto image_begin = image.info.data.address;
+	const auto image_end   = image.info.data.End();
+	if (image_begin == image.track_addr && image_end == image.track_addr_end) {
+		return;
+	}
+	if (!image.IsTracked()) {
+		image.track_addr     = image_begin;
+		image.track_addr_end = image_end;
+		m_page_manager.UpdatePageWatchers(true, image_begin, image.info.data.size);
+		return;
+	}
+	if (image_begin < image.track_addr) {
+		TrackImageHead(id);
+	}
+	if (image.track_addr_end < image_end) {
+		TrackImageTail(id);
+	}
+}
+
+void TextureCache::TrackImageHead(ImageId id) {
+	auto& image = ResolveImage(id);
+	if (!image.registered) {
+		return;
+	}
+	const auto image_begin = image.info.data.address;
+	if (image_begin == image.track_addr) {
+		return;
+	}
+	if (!image.IsTracked() || image_begin > image.track_addr) {
+		EXIT("TextureCache: invalid image head tracking range\n");
+	}
+	const auto size  = image.track_addr - image_begin;
+	image.track_addr = image_begin;
+	m_page_manager.UpdatePageWatchers(true, image_begin, size);
+}
+
+void TextureCache::TrackImageTail(ImageId id) {
+	auto& image = ResolveImage(id);
+	if (!image.registered) {
+		return;
+	}
+	const auto image_end = image.info.data.End();
+	if (image_end == image.track_addr_end) {
+		return;
+	}
+	if (!image.IsTracked() || image.track_addr_end > image_end) {
+		EXIT("TextureCache: invalid image tail tracking range\n");
+	}
+	const auto address   = image.track_addr_end;
+	const auto size      = image_end - address;
+	image.track_addr_end = image_end;
+	m_page_manager.UpdatePageWatchers(true, address, size);
+}
+
+void TextureCache::UntrackImage(ImageId id) {
+	auto& image = ResolveImage(id);
+	if (!image.IsTracked()) {
+		return;
+	}
+	const auto address   = image.track_addr;
+	const auto size      = image.track_addr_end - image.track_addr;
+	image.track_addr     = 0;
+	image.track_addr_end = 0;
+	if (size != 0) {
+		m_page_manager.UpdatePageWatchers(false, address, size);
+	}
+}
+
+void TextureCache::UntrackImageHead(ImageId id) {
+	auto&      image = ResolveImage(id);
+	const auto begin = image.info.data.address;
+	if (!image.IsTracked() || begin < image.track_addr) {
+		return;
+	}
+	const auto address = (begin + TRACKER_PAGE_SIZE) & ~(TRACKER_PAGE_SIZE - 1);
+	const auto size    = address - begin;
+	image.track_addr   = address;
+	if (image.track_addr == image.track_addr_end) {
+		image.MarkMaybeCpuDirty();
+		if (image.NeedsMaybeCpuHash()) {
+			image.SetMaybeCpuHash(image.HashGuestEdges());
+		}
+		UntrackImage(id);
+	}
+	if (size != 0) {
+		m_page_manager.UpdatePageWatchers(false, begin, size);
+	}
+}
+
+void TextureCache::UntrackImageTail(ImageId id) {
+	auto&      image = ResolveImage(id);
+	const auto end   = image.info.data.End();
+	if (!image.IsTracked() || image.track_addr_end < end) {
+		return;
+	}
+	const auto address   = end & ~(TRACKER_PAGE_SIZE - 1);
+	const auto size      = end - address;
+	image.track_addr_end = address;
+	if (image.track_addr == image.track_addr_end) {
+		image.MarkMaybeCpuDirty();
+		if (image.NeedsMaybeCpuHash()) {
+			image.SetMaybeCpuHash(image.HashGuestEdges());
+		}
+		UntrackImage(id);
+	}
+	if (size != 0) {
+		m_page_manager.UpdatePageWatchers(false, address, size);
+	}
+}
+
 void TextureCache::TrackImageDownload(ImageId id) {
 	std::lock_guard transaction(m_resource_mutex);
 	CacheLock       lock(*this, m_lock);
@@ -376,9 +486,6 @@ void TextureCache::ValidateImageDesc(const ImageDesc& desc) const {
 }
 
 void TextureCache::PrepareImageCopy(Image& image) {
-	const auto range = image.info.data;
-	m_memory_tracker.ForEachUploadRange(
-	    range.address, range.size, false, [](uint64_t, uint64_t) noexcept {}, []() noexcept {});
 	if (image.IsCpuDirty()) {
 		image.RefreshComplete();
 	}
@@ -471,6 +578,7 @@ void TextureCache::CopyImage(ImageId destination_id, ImageId source_id) {
 	RefreshCopySource(source_id);
 	auto& destination = ResolveImage(destination_id);
 	auto& source      = ResolveImage(source_id);
+	TrackImage(destination_id);
 	if (source.backing.samples != destination.backing.samples) {
 		EXIT("TextureCache: cannot issue an unequal-sample image copy\n");
 	}
@@ -479,7 +587,6 @@ void TextureCache::CopyImage(ImageId destination_id, ImageId source_id) {
 		if (source.info.data == destination.info.data) {
 			destination.MarkBufferModified();
 		}
-		RestoreGpuTracking(destination);
 		return;
 	}
 	const bool source_depth = source.info.IsDepth();
@@ -503,7 +610,6 @@ void TextureCache::CopyImage(ImageId destination_id, ImageId source_id) {
 		destination.MarkGpuModified();
 	}
 	destination.ClearBufferModified();
-	RestoreGpuTracking(destination);
 }
 
 void TextureCache::CopyImageMip(ImageId destination_id, ImageId source_id, uint32_t mip,
@@ -511,6 +617,7 @@ void TextureCache::CopyImageMip(ImageId destination_id, ImageId source_id, uint3
 	RefreshCopySource(source_id);
 	auto& destination = ResolveImage(destination_id);
 	auto& source      = ResolveImage(source_id);
+	TrackImage(destination_id);
 	if (source.IsBufferModified() || source.backing.samples != destination.backing.samples) {
 		EXIT("TextureCache: invalid mip-copy ownership or sample count\n");
 	}
@@ -520,7 +627,6 @@ void TextureCache::CopyImageMip(ImageId destination_id, ImageId source_id, uint3
 	if (source.IsGpuModified()) {
 		destination.MarkGpuModified();
 	}
-	RestoreGpuTracking(destination);
 }
 
 ImageId TextureCache::ResolveDepthOverlap(const ImageInfo& requested, BindingType binding,
@@ -590,7 +696,7 @@ ImageId TextureCache::ResolveDepthOverlap(const ImageInfo& requested, BindingTyp
 	if (copied) {
 		DeleteImages(std::array {cached_id}, cached_id);
 	} else {
-		ReleaseGpuTracking(cached_id);
+		ClearGpuModified(cached_id);
 		DeleteImage(cached_id);
 	}
 	return replacement_id;
@@ -903,30 +1009,20 @@ void TextureCache::InitializeImage(ImageId id, const ImageDesc& desc) {
 	if (image.info.data.Empty()) {
 		return;
 	}
+	TrackImage(id);
 	if (image.info.metadata.compression != VideoOutCompression::Uncompressed) {
-		m_memory_tracker.ForEachUploadRange(
-		    image.info.data.address, image.info.data.size, false,
-		    [](uint64_t, uint64_t) noexcept {}, []() noexcept {});
 		if (image.IsCpuDirty()) {
 			image.RefreshComplete();
 		}
 		return;
 	}
 	if (image.info.samples > 1) {
-		RestoreGpuTracking(image);
 		return;
 	}
 	bool       data_gpu_owned = false;
 	bool       data_imported  = false;
-	bool uploaded       = false;
+	const bool upload         = image.IsBufferModified() || image.IsCpuDirty();
-	m_memory_tracker.ForEachUploadRange(
+	if (upload) {
-	    image.info.data.address, image.info.data.size, false,
-	    [&](uint64_t, uint64_t) noexcept { uploaded = true; },
-	    [&]() noexcept {
-		    uploaded |= image.IsBufferModified() || image.IsDefinitelyCpuDirty();
-		    if (!uploaded) {
-			    return;
-		    }
 		const auto source =
 		    m_buffer_cache.ObtainBufferForImage(image.info.data.address, image.info.data.size);
 		if (source.buffer == nullptr) {
@@ -935,7 +1031,7 @@ void TextureCache::InitializeImage(ImageId id, const ImageDesc& desc) {
 		data_gpu_owned |= source.gpu_owned;
 		data_imported = true;
 		UploadImage(image, desc, *source.buffer, source.offset);
-	    });
+	}
 	if (data_imported) {
 		image.ClearBufferModified();
 	}
@@ -945,39 +1041,27 @@ void TextureCache::InitializeImage(ImageId id, const ImageDesc& desc) {
 	if (image.IsCpuDirty()) {
 		image.RefreshComplete();
 	}
-	RestoreGpuTracking(image);
 }
 
 void TextureCache::RefreshImage(ImageId id, const ImageDesc& desc) {
+	TrackImage(id);
 	auto& image = ResolveImage(id);
-	bool  unchanged_maybe = false;
 	if (image.IsMaybeCpuDirty()) {
 		const auto hash = image.HashGuestEdges();
 		if (image.NeedsMaybeCpuHash()) {
 			image.SetMaybeCpuHash(hash);
 			return;
 		}
-		unchanged_maybe = !image.ResolveMaybeCpuHash(hash);
+		(void)image.ResolveMaybeCpuHash(hash);
-		if (unchanged_maybe) {
-			m_memory_tracker.ForEachUploadRange(
-			    image.info.data.address, image.info.data.size, false,
-			    [](uint64_t, uint64_t) noexcept {}, []() noexcept {});
-		}
 	}
 	bool cpu_dirty = image.IsBufferModified() || image.IsDefinitelyCpuDirty();
-	if (!unchanged_maybe) {
-		cpu_dirty |=
-		    m_memory_tracker.IsRegionCpuModified(image.info.data.address, image.info.data.size);
-	}
 	if (image.info.metadata.compression != VideoOutCompression::Uncompressed) {
 		if (cpu_dirty) {
 			EXIT("TextureCache: compressed guest image refresh is unsupported\n");
 		}
-		RestoreGpuTracking(image);
 		return;
 	}
 	if (!cpu_dirty) {
-		RestoreGpuTracking(image);
 		return;
 	}
 	InitializeImage(id, desc);
@@ -1029,7 +1113,7 @@ ImageId TextureCache::FindImage(ImageDesc& desc, bool exact_format) {
 
 	ImageId result {};
 	bool    replacement_buffer = false;
-	bool    replacing_image    = false;
+	bool    inserted_new       = false;
 	{
 		std::lock_guard            transaction(m_resource_mutex);
 		CacheLock                  lock(*this, m_lock);
@@ -1080,19 +1164,18 @@ ImageId TextureCache::FindImage(ImageDesc& desc, bool exact_format) {
 				replacement_buffer = resolved.IsBufferModified();
 				DeleteImage(result);
 				result = {};
-				replacing_image = true;
 			}
 		}
 		if (!result) {
 			result         = InsertImage(desc.info);
+			inserted_new   = true;
 			auto& inserted = ResolveImage(result);
 			if (replacement_buffer || m_buffer_cache.HasGpuDirtyBytes(inserted.info.data.address,
 			                                                          inserted.info.data.size)) {
 				inserted.MarkBufferModified();
-			} else if (replacing_image) {
-				m_memory_tracker.MarkRegionAsCpuModified(inserted.info.data.address,
-				                                         inserted.info.data.size);
 			}
+		}
+		if (inserted_new) {
 			InitializeImage(result, desc);
 		} else {
 			RefreshImage(result, desc);
@@ -1101,9 +1184,7 @@ ImageId TextureCache::FindImage(ImageDesc& desc, bool exact_format) {
 		auto& image = ResolveImage(result);
 		if (desc.type == BindingType::VideoOut &&
 		    desc.info.metadata.compression != VideoOutCompression::Uncompressed) {
-			const bool guest_dirty =
+			const bool guest_dirty = image.IsBufferModified() || image.IsCpuDirty();
-			    image.IsBufferModified() || image.IsCpuDirty() ||
-			    m_memory_tracker.IsRegionCpuModified(image.info.data.address, image.info.data.size);
 			const bool native_current =
 			    (image.usage.render_target || image.IsGpuModified()) && !guest_dirty;
 			if (!native_current) {
@@ -1252,6 +1333,7 @@ void TextureCache::MarkGpuWritten(ImageId id) {
 	if (!image.registered || image.depth_id) {
 		EXIT("TextureCache: cannot mark an unavailable image GPU-written\n");
 	}
+	TrackImage(id);
 	CommitGpuWrite(image);
 }
 
@@ -1265,13 +1347,10 @@ void TextureCache::CommitGpuWrite(Image& image) {
 	}
 	m_buffer_cache.InvalidateImageAliases(range.address, range.size);
 	image.ClearBufferModified();
-	m_memory_tracker.ForEachUploadRange(
-	    range.address, range.size, true, [](uint64_t, uint64_t) noexcept {}, []() noexcept {});
 	if (image.IsCpuDirty()) {
 		image.RefreshComplete();
 	}
 	image.MarkGpuModified();
-	RestoreGpuTracking(image);
 }
 
 bool TextureCache::ClearImageFromBuffer(CommandBuffer& command, uint64_t address, uint64_t size,
@@ -1335,8 +1414,7 @@ bool TextureCache::ClearImageFromBuffer(CommandBuffer& command, uint64_t address
 	if (m_buffer_cache.HasGpuDirtyBytes(address, size)) {
 		m_buffer_cache.DiscardGpuDirtyBytes(address, size);
 	}
-	if (image.IsBufferModified() || image.IsCpuDirty() ||
+	if (image.IsBufferModified() || image.IsCpuDirty()) {
-	    m_memory_tracker.IsRegionCpuModified(image.info.data.address, image.info.data.size)) {
 		ImageDesc refresh {.info = image.info, .view_info = {}, .type = UploadBinding(image)};
 		InitializeImage(selected, refresh);
 		if (image.info.samples == 1 && (image.IsBufferModified() || image.IsCpuDirty())) {
@@ -1367,8 +1445,6 @@ void TextureCache::PrepareHostWrite(uint64_t address, uint64_t size) {
 	}
 	CacheLock lock(*this, m_lock);
 	InvalidateCpuAliases(address, size);
-	m_memory_tracker.ForEachDownloadRange<true>(address, size, [](uint64_t, uint64_t) noexcept {});
-	m_memory_tracker.MarkRegionAsCpuModified(address, size);
 }
 
 void TextureCache::DownloadDepth(Image& image, Buffer& destination, uint64_t destination_offset) {
@@ -1568,17 +1644,14 @@ bool TextureCache::SynchronizeImageToBuffer(ImageId id) {
 		return false;
 	}
 	const auto range = image.info.data;
-	const bool refresh = image.IsDefinitelyCpuDirty() ||
+	if (image.IsCpuDirty()) {
-	                     m_memory_tracker.IsRegionCpuModified(range.address, range.size);
-	if (refresh) {
 		RefreshImage(id,
 		             ImageDesc {.info = image.info, .view_info = {}, .type = UploadBinding(image)});
 	}
 	if (!image.IsGpuModified()) {
 		return true;
 	}
-	if (image.IsDefinitelyCpuDirty() || image.IsBufferModified() ||
+	if (image.IsDefinitelyCpuDirty() || image.IsBufferModified()) {
-	    m_memory_tracker.IsRegionCpuModified(range.address, range.size)) {
 		EXIT("TextureCache: image mirror source is not native-current\n");
 	}
 	auto [destination, offset] =
@@ -1591,7 +1664,7 @@ bool TextureCache::SynchronizeImageToBuffer(ImageId id) {
 	m_buffer_cache.PublishImageBuffer(range.address, range.size);
 	image.MarkBufferModified();
 	RetainImage(m_scheduler.Current(), id);
-	ReleaseGpuTracking(id);
+	ClearGpuModified(id);
 	return true;
 }
 
@@ -1633,7 +1706,7 @@ bool TextureCache::InvalidateMemoryFromGPU(uint64_t address, uint64_t size,
 			if (!formatted_buffer_write) {
 				EXIT("TextureCache: buffer write aliases GPU-modified image\n");
 			}
-			ReleaseGpuTracking(id);
+			ClearGpuModified(id);
 		}
 		owner->MarkBufferModified();
 		found = true;
@@ -1660,50 +1733,40 @@ TextureCache::RegionInfo TextureCache::QueryRegion(uint64_t address, uint64_t si
 }
 
 void TextureCache::InvalidateCpuAliases(uint64_t address, uint64_t size) {
+	const auto page_begin = address & ~(TRACKER_PAGE_SIZE - 1);
+	const auto page_end   = (address + size + TRACKER_PAGE_SIZE - 1) & ~(TRACKER_PAGE_SIZE - 1);
 	for (const auto id: FindImagesInRegion(address, size, true)) {
 		auto owner = ResolveOwner(id);
 		if (owner == nullptr || owner->depth_id) {
 			continue;
 		}
+		if (owner->Overlaps(address, size)) {
 			owner->InvalidateCpuWrite(address, size);
+			UntrackImage(id);
+			continue;
+		}
+		const auto image_begin = owner->info.data.address;
+		const auto image_end   = owner->info.data.End();
+		if (page_end < image_end) {
+			UntrackImageHead(id);
+		} else if (image_begin < page_begin) {
+			UntrackImageTail(id);
+		} else {
+			owner->MarkMaybeCpuDirty();
 			if (owner->NeedsMaybeCpuHash()) {
 				owner->SetMaybeCpuHash(owner->HashGuestEdges());
 			}
-	}
+			UntrackImage(id);
-}
-
-void TextureCache::RestoreGpuTracking(const Image& image) {
-	if (!image.IsGpuModified()) {
-		return;
-	}
-	constexpr uint64_t page_mask = TRACKER_PAGE_SIZE - 1;
-	const auto         range     = image.info.data;
-	const auto         begin     = range.address & ~page_mask;
-	const auto         end       = (range.End() + page_mask) & ~page_mask;
-	for (auto page = begin; page < end; page += TRACKER_PAGE_SIZE) {
-		if (!m_memory_tracker.IsRegionGpuModified(page, TRACKER_PAGE_SIZE) &&
-		    !m_memory_tracker.IsRegionCpuModified(page, TRACKER_PAGE_SIZE)) {
-			m_memory_tracker.MarkRegionAsGpuModified(page, TRACKER_PAGE_SIZE);
 		}
 	}
 }
 
-void TextureCache::ReleaseGpuTracking(ImageId id) {
+void TextureCache::ClearGpuModified(ImageId id) {
 	auto owner = ResolveOwner(id);
 	if (owner == nullptr || !owner->IsGpuModified()) {
 		return;
 	}
-	const auto released = owner->info.data;
 	owner->ClearGpuModified();
-	m_memory_tracker.ForEachDownloadRange<true>(released.address, released.size,
-	                                            [](uint64_t, uint64_t) noexcept {});
-	for (const auto candidate: FindImagesInRegion(released.address, released.size, true)) {
-		const auto survivor = ResolveOwner(candidate);
-		if (survivor != nullptr && survivor.get() != owner.get() && !survivor->depth_id) {
-			RestoreGpuTracking(*survivor);
-		}
-	}
-	RestoreGpuTracking(*owner);
 }
 
 bool TextureCache::IsMeta(uint64_t address) {
@@ -1755,27 +1818,25 @@ bool TextureCache::InvalidateMemory(PageFaultAccess access, uint64_t address, ui
 		return false;
 	}
 	if (phase == PageFaultPhase::Invalidate) {
-		const bool gpu_image =
-		    m_memory_tracker.InvalidateVirtualGpuWrite(access, address, size, phase);
-		CpuFaultAction action = gpu_image ? CpuFaultAction::Download
-		                                  : m_memory_tracker.BeginCpuFault(address, size, access);
-		{
 		CacheLock  lock(*this, m_lock);
+		const bool tracked =
+		    std::ranges::any_of(FindImagesInRegion(address, size, true), [&](ImageId id) {
+			    const auto owner = ResolveOwner(id);
+			    return owner != nullptr && !owner->depth_id && owner->IsTracked();
+		    });
+		if (tracked) {
 			InvalidateCpuAliases(address, size);
 		}
-		return action != CpuFaultAction::Untracked;
+		return tracked;
 	}
-
+	if (phase != PageFaultPhase::Complete && phase != PageFaultPhase::Release) {
-	if (phase == PageFaultPhase::Complete) {
-		const bool gpu_image = m_memory_tracker.IsRegionGpuModified(address, size);
-		return gpu_image ? m_memory_tracker.InvalidateVirtualGpuWrite(access, address, size, phase)
-		                 : m_memory_tracker.CompleteCpuFault(address, size, access, false);
-	}
-	if (phase != PageFaultPhase::Release) {
 		return false;
 	}
-	(void)m_memory_tracker.InvalidateVirtualGpuWrite(access, address, size, phase);
+	CacheLock lock(*this, m_lock);
-	return true;
+	return std::ranges::any_of(FindImagesInRegion(address, size, true), [&](ImageId id) {
+		const auto owner = ResolveOwner(id);
+		return owner != nullptr && !owner->depth_id;
+	});
 }
 
 void TextureCache::UnmapMemory(uint64_t address, uint64_t size) {
@@ -1794,16 +1855,10 @@ void TextureCache::UnmapMemory(uint64_t address, uint64_t size) {
 			continue;
 		}
 		if (owner->IsGpuModified()) {
-			ReleaseGpuTracking(id);
+			ClearGpuModified(id);
 		}
 		DeleteImage(id);
 	}
-	m_memory_tracker.UntrackMemory(address, size);
-	for (const auto id: FindImagesInRegion(address, size, true)) {
-		if (const auto survivor = ResolveOwner(id); survivor != nullptr) {
-			RestoreGpuTracking(*survivor);
-		}
-	}
 }
 
 void TextureCache::RunGarbageCollector() {
@@ -1849,7 +1904,7 @@ void TextureCache::RunGarbageCollector() {
 				if (safe && !TryDownloadImage(id)) {
 					continue;
 				}
-				ReleaseGpuTracking(id);
+				ClearGpuModified(id);
 			}
 			DeleteImage(id);
 			if (m_total_used_memory < m_critical_gc_memory && aggressive) {

src/graphics/host_gpu/renderer/cache/textureCache.h

@@ -4,10 +4,11 @@
 #include "common/abi.h"
 #include "common/common.h"
 #include "common/lruCache.h"
-#include "graphics/host_gpu/memoryTracker.h"
+#include "graphics/host_gpu/pageManager.h"
+#include "graphics/host_gpu/regionManager.h"
+#include "graphics/host_gpu/renderer/cache/multiLevelPageTable.h"
 #include "graphics/host_gpu/renderer/image/blitHelper.h"
 #include "graphics/host_gpu/renderer/image/image.h"
-#include "graphics/host_gpu/renderer/cache/multiLevelPageTable.h"
 
 #include <compare>
 #include <map>
@@ -109,11 +110,17 @@ private:
 	[[nodiscard]] ImageId                InsertImage(const ImageInfo& info);
 	[[nodiscard]] ImageId                GetNullImage(const ImageDesc& desc);
 	void                                 RegisterImage(ImageId id);
-	void                                 UnregisterImage(ImageId id, bool release_tracking);
+	void                                 UnregisterImage(ImageId id);
-	void                                 DeleteImage(ImageId id, bool release_tracking = true);
+	void                                 DeleteImage(ImageId id);
 	void DeleteImages(std::span<const ImageId> ids, std::optional<ImageId> native_source = {});
 	void RetainImage(CommandBuffer& command, ImageId id);
 	void TouchImage(Image& image);
+	void TrackImage(ImageId id);
+	void TrackImageHead(ImageId id);
+	void TrackImageTail(ImageId id);
+	void UntrackImage(ImageId id);
+	void UntrackImageHead(ImageId id);
+	void UntrackImageTail(ImageId id);
 	void TrackImageDownload(ImageId id);
 	void TrackImageDownloadLocked(ImageId id, Image& image);
 	[[nodiscard]] static bool SameBacking(const ImageInfo& cached, const ImageInfo& requested,
@@ -148,8 +155,7 @@ private:
 	void ValidateImageDesc(const ImageDesc& desc) const;
 
 	void InvalidateCpuAliases(uint64_t address, uint64_t size);
-	void RestoreGpuTracking(const Image& image);
+	void ClearGpuModified(ImageId id);
-	void ReleaseGpuTracking(ImageId id);
 
 	[[nodiscard]] bool                          SynchronizeImageToBuffer(ImageId id);
 	void                                        DownloadImage(ImageId id);
@@ -160,7 +166,7 @@ private:
 	GraphicContext&                                   m_graphics;
 	CommandScheduler&                                 m_scheduler;
 	TrackingSpinLock                                  m_lock;
-	MemoryTracker                                     m_memory_tracker;
+	PageManager&                                      m_page_manager;
 	BlitHelper                                        m_blit_helper;
 	std::unique_ptr<TileManager>                      m_tiler;
 	BufferCache&                                      m_buffer_cache;

src/graphics/host_gpu/renderer/image/image.cpp

@@ -2,10 +2,10 @@
 
 #include "common/assert.h"
 #include "common/profiler.h"
+#include "graphics/host_gpu/renderer/cache/streamBuffer.h"
 #include "graphics/host_gpu/renderer/commandScheduler.h"
 #include "graphics/host_gpu/renderer/image/imageView.h"
 #include "graphics/host_gpu/renderer/renderTarget.h"
-#include "graphics/host_gpu/renderer/cache/streamBuffer.h"
 #include "kernel/memory.h"
 
 #include <algorithm>
@@ -191,8 +191,7 @@ Image::Barriers Image::GetBarriers(vk::ImageLayout destination_layout,
 }
 
 void Image::Transit(vk::ImageLayout destination_layout, vk::AccessFlags2 destination_access,
-                    std::optional<ImageSubresourceRange> range,
+                    std::optional<ImageSubresourceRange> range, vk::CommandBuffer command_buffer) {
-                    vk::CommandBuffer command_buffer) {
 	const auto transfer_access =
 	    vk::AccessFlagBits2::eTransferRead | vk::AccessFlagBits2::eTransferWrite;
 	vk::PipelineStageFlags2 destination_stage {};
@@ -201,8 +200,8 @@ void Image::Transit(vk::ImageLayout destination_layout, vk::AccessFlags2 destina
 	}
 	if (!destination_access ||
 	    static_cast<bool>(destination_access & ~vk::AccessFlags2 {transfer_access})) {
-		destination_stage |= vk::PipelineStageFlagBits2::eAllGraphics |
+		destination_stage |=
-		                     vk::PipelineStageFlagBits2::eComputeShader;
+		    vk::PipelineStageFlagBits2::eAllGraphics | vk::PipelineStageFlagBits2::eComputeShader;
 	}
 	const auto barriers =
 	    GetBarriers(destination_layout, destination_access, destination_stage, range);
@@ -218,10 +217,9 @@ void Image::Transit(vk::ImageLayout destination_layout, vk::AccessFlags2 destina
 	command_buffer.pipelineBarrier2(dependency);
 }
 
-void Image::Upload(std::span<const vk::BufferImageCopy> copies, vk::Buffer buffer,
+void Image::Upload(std::span<const vk::BufferImageCopy> copies, vk::Buffer buffer, uint64_t offset,
-                   uint64_t offset, uint64_t size) {
+                   uint64_t size) {
-	EXIT_IF(m_scheduler == nullptr || copies.empty() || buffer == nullptr ||
+	EXIT_IF(m_scheduler == nullptr || copies.empty() || buffer == nullptr || size == 0);
-	        size == 0);
 	m_scheduler->EndRendering();
 	vk::BufferMemoryBarrier2 buffer_barrier {};
 	buffer_barrier.srcStageMask        = vk::PipelineStageFlagBits2::eAllCommands;
@@ -234,8 +232,7 @@ void Image::Upload(std::span<const vk::BufferImageCopy> copies, vk::Buffer buffe
 	buffer_barrier.offset              = offset;
 	buffer_barrier.size                = size;
 	const auto image_barriers =
-	    GetBarriers(vk::ImageLayout::eTransferDstOptimal,
+	    GetBarriers(vk::ImageLayout::eTransferDstOptimal, vk::AccessFlagBits2::eTransferWrite,
-	                vk::AccessFlagBits2::eTransferWrite,
 	                vk::PipelineStageFlagBits2::eCopy, {});
 	vk::DependencyInfo dependency {};
 	dependency.dependencyFlags          = vk::DependencyFlagBits::eByRegion;
@@ -256,8 +253,7 @@ void Image::Upload(std::span<const vk::BufferImageCopy> copies, vk::Buffer buffe
 	dependency.pImageMemoryBarriers    = nullptr;
 	command.pipelineBarrier2(dependency);
 	Transit(vk::ImageLayout::eGeneral,
-	        vk::AccessFlagBits2::eShaderRead | vk::AccessFlagBits2::eTransferRead, {},
+	        vk::AccessFlagBits2::eShaderRead | vk::AccessFlagBits2::eTransferRead, {}, command);
-	        command);
 }
 
 void Image::Download(std::span<const vk::BufferImageCopy> copies, vk::Buffer buffer,
@@ -276,8 +272,7 @@ void Image::Download(std::span<const vk::BufferImageCopy> copies, vk::Buffer buf
 	buffer_barrier.offset              = offset;
 	buffer_barrier.size                = size;
 	const auto image_barriers =
-	    GetBarriers(vk::ImageLayout::eTransferSrcOptimal,
+	    GetBarriers(vk::ImageLayout::eTransferSrcOptimal, vk::AccessFlagBits2::eTransferRead,
-	                vk::AccessFlagBits2::eTransferRead,
 	                vk::PipelineStageFlagBits2::eCopy, {});
 	vk::DependencyInfo dependency {};
 	dependency.dependencyFlags          = vk::DependencyFlagBits::eByRegion;
@@ -299,8 +294,8 @@ void Image::Download(std::span<const vk::BufferImageCopy> copies, vk::Buffer buf
 	command.pipelineBarrier2(dependency);
 }
 
-std::pair<uint32_t, uint32_t>
+std::pair<uint32_t, uint32_t> Image::SanitizeCopyLayers(const Image& source,
-Image::SanitizeCopyLayers(const Image& source, const Image& destination, uint32_t depth) {
+                                                        const Image& destination, uint32_t depth) {
 	const auto source_type        = source.backing.image_type;
 	const auto destination_type   = destination.backing.image_type;
 	uint32_t   source_layers      = source.backing.layers;
@@ -312,13 +307,10 @@ Image::SanitizeCopyLayers(const Image& source, const Image& destination, uint32_
 		destination_layers = 1;
 	}
 	if (source_type == destination_type) {
-		source_layers = destination_layers =
+		source_layers = destination_layers = std::min(source_layers, destination_layers);
-		    std::min(source_layers, destination_layers);
+	} else if (source_type == vk::ImageType::e2D && destination_type == vk::ImageType::e3D) {
-	} else if (source_type == vk::ImageType::e2D &&
-	           destination_type == vk::ImageType::e3D) {
 		source_layers = depth;
-	} else if (source_type == vk::ImageType::e3D &&
+	} else if (source_type == vk::ImageType::e3D && destination_type == vk::ImageType::e2D) {
-	           destination_type == vk::ImageType::e2D) {
 		destination_layers = depth;
 	}
 	return {source_layers, destination_layers};
@@ -327,8 +319,7 @@ Image::SanitizeCopyLayers(const Image& source, const Image& destination, uint32_
 void Image::CopyImage(Image& source) {
 	EXIT_IF(m_scheduler == nullptr || source.backing.samples != backing.samples);
 	m_scheduler->EndRendering();
-	const uint32_t levels =
+	const uint32_t levels     = std::min(source.backing.mip_levels, backing.mip_levels);
-	    std::min(source.backing.mip_levels, backing.mip_levels);
 	const uint32_t base_depth = backing.image_type == vk::ImageType::e3D
 	                                ? backing.extent.depth
 	                                : source.backing.extent.depth;
@@ -342,8 +333,7 @@ void Image::CopyImage(Image& source) {
 		const auto width  = std::max(source.backing.extent.width >> level, 1u);
 		const auto height = std::max(source.backing.extent.height >> level, 1u);
 		const auto depth  = std::max(base_depth >> level, 1u);
-		const auto [source_layers, destination_layers] =
+		const auto [source_layers, destination_layers] = SanitizeCopyLayers(source, *this, depth);
-		    SanitizeCopyLayers(source, *this, depth);
 		vk::ImageCopy copy {};
 		copy.srcSubresource = {source_aspect, level, 0, 1};
 		copy.dstSubresource = {destination_aspect, level, 0, 1};
@@ -351,8 +341,7 @@ void Image::CopyImage(Image& source) {
 			if (source.backing.image_type == vk::ImageType::e3D) {
 				copy.extent = {width, height, depth};
 			} else {
-				copy.srcSubresource.layerCount =
+				copy.srcSubresource.layerCount = std::min(source_layers, destination_layers);
-				    std::min(source_layers, destination_layers);
 				copy.dstSubresource.layerCount = copy.srcSubresource.layerCount;
 				copy.extent                    = {width, height, 1};
 			}
@@ -369,34 +358,30 @@ void Image::CopyImage(Image& source) {
 		return;
 	}
 	auto command = m_scheduler->Current().Handle();
-	source.Transit(vk::ImageLayout::eTransferSrcOptimal,
+	source.Transit(vk::ImageLayout::eTransferSrcOptimal, vk::AccessFlagBits2::eTransferRead, {},
-	               vk::AccessFlagBits2::eTransferRead, {}, command);
-	Transit(vk::ImageLayout::eTransferDstOptimal,
-	        vk::AccessFlagBits2::eTransferWrite, {}, command);
-	command.copyImage(source.backing.image, vk::ImageLayout::eTransferSrcOptimal,
-	                  backing.image, vk::ImageLayout::eTransferDstOptimal,
-	                  static_cast<uint32_t>(copies.size()), copies.data());
-	Transit(vk::ImageLayout::eGeneral,
-	        vk::AccessFlagBits2::eShaderRead | vk::AccessFlagBits2::eTransferRead, {},
 	               command);
+	Transit(vk::ImageLayout::eTransferDstOptimal, vk::AccessFlagBits2::eTransferWrite, {}, command);
+	command.copyImage(source.backing.image, vk::ImageLayout::eTransferSrcOptimal, backing.image,
+	                  vk::ImageLayout::eTransferDstOptimal, static_cast<uint32_t>(copies.size()),
+	                  copies.data());
+	Transit(vk::ImageLayout::eGeneral,
+	        vk::AccessFlagBits2::eShaderRead | vk::AccessFlagBits2::eTransferRead, {}, command);
 }
 
 void Image::Resolve(Image& source, const ImageSubresourceRange& source_range,
                     const ImageSubresourceRange& destination_range) {
 	EXIT_IF(m_scheduler == nullptr || backing.samples != 1 ||
 	        source.backing.image_type != vk::ImageType::e2D ||
-	        backing.image_type != vk::ImageType::e2D ||
+	        backing.image_type != vk::ImageType::e2D || source_range.level_count != 1 ||
-	        source_range.level_count != 1 || destination_range.level_count != 1 ||
+	        destination_range.level_count != 1 ||
 	        source_range.base_level >= source.backing.mip_levels ||
 	        destination_range.base_level >= backing.mip_levels ||
 	        source_range.base_layer >= source.backing.layers ||
 	        destination_range.base_layer >= backing.layers);
-	const auto layers = std::min(
+	const auto layers       = std::min({source_range.layer_count, destination_range.layer_count,
-	    {source_range.layer_count, destination_range.layer_count,
 	                                    source.backing.layers - source_range.base_layer,
 	                                    backing.layers - destination_range.base_layer});
-	const auto source_width =
+	const auto source_width = std::max(source.backing.extent.width >> source_range.base_level, 1u);
-	    std::max(source.backing.extent.width >> source_range.base_level, 1u);
 	const auto source_height =
 	    std::max(source.backing.extent.height >> source_range.base_level, 1u);
 	const auto destination_width =
@@ -404,9 +389,8 @@ void Image::Resolve(Image& source, const ImageSubresourceRange& source_range,
 	const auto destination_height =
 	    std::max(backing.extent.height >> destination_range.base_level, 1u);
 	const bool copy = source.backing.samples == 1;
-	EXIT_IF(layers == 0 || info.extent.width > source_width ||
+	EXIT_IF(layers == 0 || info.extent.width > source_width || info.extent.height > source_height ||
-	        info.extent.height > source_height || info.extent.width > destination_width ||
+	        info.extent.width > destination_width || info.extent.height > destination_height ||
-	        info.extent.height > destination_height ||
 	        (copy ? !ImageViewOps::FormatsCompatible(source.backing.format, backing.format)
 	              : source.backing.format != backing.format));
 	auto resolved_source_range             = source_range;
@@ -417,25 +401,23 @@ void Image::Resolve(Image& source, const ImageSubresourceRange& source_range,
 
 	m_scheduler->EndRendering();
 	auto command = m_scheduler->Current().Handle();
-	source.Transit(vk::ImageLayout::eTransferSrcOptimal,
+	source.Transit(vk::ImageLayout::eTransferSrcOptimal, vk::AccessFlagBits2::eTransferRead,
-	               vk::AccessFlagBits2::eTransferRead, resolved_source_range, command);
+	               resolved_source_range, command);
-	Transit(vk::ImageLayout::eTransferDstOptimal,
+	Transit(vk::ImageLayout::eTransferDstOptimal, vk::AccessFlagBits2::eTransferWrite,
-	        vk::AccessFlagBits2::eTransferWrite, resolved_destination_range, command);
+	        resolved_destination_range, command);
 	if (copy) {
 		vk::ImageCopy region {};
-		region.srcSubresource = {vk::ImageAspectFlagBits::eColor,
+		region.srcSubresource = {vk::ImageAspectFlagBits::eColor, resolved_source_range.base_level,
-		                         resolved_source_range.base_level,
 		                         resolved_source_range.base_layer, layers};
 		region.dstSubresource = {vk::ImageAspectFlagBits::eColor,
 		                         resolved_destination_range.base_level,
 		                         resolved_destination_range.base_layer, layers};
 		region.extent         = resolve_extent;
-		command.copyImage(source.backing.image, vk::ImageLayout::eTransferSrcOptimal,
+		command.copyImage(source.backing.image, vk::ImageLayout::eTransferSrcOptimal, backing.image,
-		                  backing.image, vk::ImageLayout::eTransferDstOptimal, region);
+		                  vk::ImageLayout::eTransferDstOptimal, region);
 	} else {
 		vk::ImageResolve region {};
-		region.srcSubresource = {vk::ImageAspectFlagBits::eColor,
+		region.srcSubresource = {vk::ImageAspectFlagBits::eColor, resolved_source_range.base_level,
-		                         resolved_source_range.base_level,
 		                         resolved_source_range.base_layer, layers};
 		region.dstSubresource = {vk::ImageAspectFlagBits::eColor,
 		                         resolved_destination_range.base_level,
@@ -454,11 +436,10 @@ uint32_t Image::CopyRows(uint64_t row_size, uint32_t rows, uint64_t capacity) no
 }
 
 void Image::CopyImageWithBuffer(Image& source, Buffer& buffer) {
-	EXIT_IF(m_scheduler == nullptr || buffer.Handle() == nullptr ||
+	EXIT_IF(m_scheduler == nullptr || buffer.Handle() == nullptr || source.backing.samples != 1 ||
-	        source.backing.samples != 1 || backing.samples != 1);
+	        backing.samples != 1);
 	m_scheduler->EndRendering();
-	const uint32_t levels =
+	const uint32_t levels = std::min(source.backing.mip_levels, backing.mip_levels);
-	    std::min(source.backing.mip_levels, backing.mip_levels);
 	const auto     source_aspect =
 	    FullAspectMask(source.backing.format) & ~vk::ImageAspectFlagBits::eStencil;
 	const auto destination_aspect =
@@ -488,10 +469,9 @@ void Image::CopyImageWithBuffer(Image& source, Buffer& buffer) {
 	dependency.bufferMemoryBarrierCount = 1;
 	dependency.pBufferMemoryBarriers    = &barrier;
 	auto command                        = m_scheduler->Current().Handle();
-	source.Transit(vk::ImageLayout::eTransferSrcOptimal,
+	source.Transit(vk::ImageLayout::eTransferSrcOptimal, vk::AccessFlagBits2::eTransferRead, {},
-	               vk::AccessFlagBits2::eTransferRead, {}, command);
+	               command);
-	Transit(vk::ImageLayout::eTransferDstOptimal,
+	Transit(vk::ImageLayout::eTransferDstOptimal, vk::AccessFlagBits2::eTransferWrite, {}, command);
-	        vk::AccessFlagBits2::eTransferWrite, {}, command);
 	for (uint32_t level = 0; level < levels; level++) {
 		const auto width             = std::max(source.backing.extent.width >> level, 1u);
 		const auto height            = std::max(source.backing.extent.height >> level, 1u);
@@ -508,19 +488,16 @@ void Image::CopyImageWithBuffer(Image& source, Buffer& buffer) {
 		const auto rows_per_copy = CopyRows(row_size, block_rows, buffer.Size());
 		EXIT_IF(slices == 0 || rows_per_copy == 0);
 		for (uint32_t slice = 0; slice < slices; slice++) {
-			for (uint32_t block_row = 0; block_row < block_rows;
+			for (uint32_t block_row = 0; block_row < block_rows; block_row += rows_per_copy) {
-			     block_row += rows_per_copy) {
 				const auto          copy_rows   = std::min(rows_per_copy, block_rows - block_row);
 				const auto          y           = block_row * source_block;
-				const auto copy_height =
+				const auto          copy_height = std::min(copy_rows * source_block, height - y);
-				    std::min(copy_rows * source_block, height - y);
 				const auto          copy_size   = row_size * copy_rows;
 				vk::BufferImageCopy source_copy {};
 				source_copy.imageSubresource = {
 				    source_aspect, level,
 				    source.backing.image_type == vk::ImageType::e3D ? 0u : slice, 1};
-				source_copy.imageOffset = {
+				source_copy.imageOffset           = {0, static_cast<int32_t>(y),
-				    0, static_cast<int32_t>(y),
 				                                     source.backing.image_type == vk::ImageType::e3D
 				                                         ? static_cast<int32_t>(slice)
 				                                         : 0};
@@ -530,28 +507,24 @@ void Image::CopyImageWithBuffer(Image& source, Buffer& buffer) {
 				    destination_aspect, level,
 				    backing.image_type == vk::ImageType::e3D ? 0u : slice, 1};
 				destination_copy.imageOffset.z =
-				    backing.image_type == vk::ImageType::e3D
+				    backing.image_type == vk::ImageType::e3D ? static_cast<int32_t>(slice) : 0;
-				        ? static_cast<int32_t>(slice)
-				        : 0;
 				barrier.size          = copy_size;
 				barrier.srcAccessMask = vk::AccessFlagBits2::eTransferRead;
 				barrier.dstAccessMask = vk::AccessFlagBits2::eTransferWrite;
 				command.pipelineBarrier2(dependency);
 				command.copyImageToBuffer(source.backing.image,
-				                          vk::ImageLayout::eTransferSrcOptimal,
+				                          vk::ImageLayout::eTransferSrcOptimal, buffer.Handle(),
-				                          buffer.Handle(), source_copy);
+				                          source_copy);
 				barrier.srcAccessMask = vk::AccessFlagBits2::eTransferWrite;
 				barrier.dstAccessMask = vk::AccessFlagBits2::eTransferRead;
 				command.pipelineBarrier2(dependency);
 				command.copyBufferToImage(buffer.Handle(), backing.image,
-				                          vk::ImageLayout::eTransferDstOptimal,
+				                          vk::ImageLayout::eTransferDstOptimal, destination_copy);
-				                          destination_copy);
 			}
 		}
 	}
 	Transit(vk::ImageLayout::eGeneral,
-	        vk::AccessFlagBits2::eShaderRead | vk::AccessFlagBits2::eTransferRead, {},
+	        vk::AccessFlagBits2::eShaderRead | vk::AccessFlagBits2::eTransferRead, {}, command);
-	        command);
 }
 
 void Image::CopyMip(Image& source, uint32_t mip, uint32_t layer) {
@@ -561,10 +534,8 @@ void Image::CopyMip(Image& source, uint32_t mip, uint32_t layer) {
 	const auto width  = std::max(backing.extent.width >> mip, 1u);
 	const auto height = std::max(backing.extent.height >> mip, 1u);
 	const auto depth  = std::max(backing.extent.depth >> mip, 1u);
-	EXIT_IF(width != source.backing.extent.width ||
+	EXIT_IF(width != source.backing.extent.width || height != source.backing.extent.height);
-	        height != source.backing.extent.height);
+	const auto [source_layers, destination_layers] = SanitizeCopyLayers(source, *this, depth);
-	const auto [source_layers, destination_layers] =
-	    SanitizeCopyLayers(source, *this, depth);
 	const auto aspects                             = FullAspectMask(source.backing.format);
 	EXIT_IF(aspects != FullAspectMask(backing.format));
 	std::array<vk::ImageCopy, 2> copies {};
@@ -580,16 +551,13 @@ void Image::CopyMip(Image& source, uint32_t mip, uint32_t layer) {
 		copy.extent         = {width, height, depth};
 	}
 	auto command = m_scheduler->Current().Handle();
-	Transit(vk::ImageLayout::eTransferDstOptimal,
+	Transit(vk::ImageLayout::eTransferDstOptimal, vk::AccessFlagBits2::eTransferWrite, {}, command);
-	        vk::AccessFlagBits2::eTransferWrite, {}, command);
+	source.Transit(vk::ImageLayout::eTransferSrcOptimal, vk::AccessFlagBits2::eTransferRead, {},
-	source.Transit(vk::ImageLayout::eTransferSrcOptimal,
-	               vk::AccessFlagBits2::eTransferRead, {}, command);
-	command.copyImage(source.backing.image, vk::ImageLayout::eTransferSrcOptimal,
-	                  backing.image, vk::ImageLayout::eTransferDstOptimal, copy_count,
-	                  copies.data());
-	Transit(vk::ImageLayout::eGeneral,
-	        vk::AccessFlagBits2::eShaderRead | vk::AccessFlagBits2::eTransferRead, {},
 	               command);
+	command.copyImage(source.backing.image, vk::ImageLayout::eTransferSrcOptimal, backing.image,
+	                  vk::ImageLayout::eTransferDstOptimal, copy_count, copies.data());
+	Transit(vk::ImageLayout::eGeneral,
+	        vk::AccessFlagBits2::eShaderRead | vk::AccessFlagBits2::eTransferRead, {}, command);
 }
 
 namespace ImageOps {
@@ -601,8 +569,7 @@ void Validate(const ImageInfo& info) {
 	if (info.pixel_format == vk::Format::eUndefined) {
 		const bool metadata_empty =
 		    info.metadata.range.address == 0 && info.metadata.range.size == 0 &&
-		    info.metadata.kind == ImageMetadataKind::None &&
+		    info.metadata.kind == ImageMetadataKind::None && info.metadata.control == 0 &&
-		    info.metadata.control == 0 &&
 		    info.metadata.compression == VideoOutCompression::Uncompressed &&
 		    !info.metadata.stencil_compressed;
 		if (info.data.Empty() || info.HasStencil() || !metadata_empty || info.extent.width == 0 ||
@@ -615,9 +582,9 @@ void Validate(const ImageInfo& info) {
 	}
 
 	if (info.extent.width == 0 || info.extent.height == 0 || info.extent.depth == 0 ||
-	    info.resources.levels == 0 ||
+	    info.resources.levels == 0 || info.resources.levels > info.mip_layout.size() ||
-	    info.resources.levels > info.mip_layout.size() || info.resources.layers == 0 ||
+	    info.resources.layers == 0 || info.samples == 0 ||
-	    info.samples == 0 || vulkan_sample_count(info.samples) == vk::SampleCountFlagBits {} ||
+	    vulkan_sample_count(info.samples) == vk::SampleCountFlagBits {} ||
 	    info.bytes_per_block == 0 || (info.data.address != 0 && info.pitch == 0)) {
 		EXIT("invalid image geometry or format\n");
 	}
@@ -688,11 +655,11 @@ uint32_t RenderTargetTransferFormat(uint32_t bytes_per_element) {
 
 } // namespace ImageOps
 
-Image::Image(GraphicContext& graphics, CommandScheduler& scheduler,
+Image::Image(GraphicContext& graphics, CommandScheduler& scheduler, const ImageInfo& image_info)
-             const ImageInfo& image_info)
     : info(image_info), m_graphics(&graphics), m_scheduler(&scheduler) {
 	KYTY_PROFILER_FUNCTION();
 	ImageOps::Validate(info);
+	m_cpu_dirty = !info.data.Empty();
 	if (info.pixel_format == vk::Format::eUndefined) {
 		return;
 	}

src/graphics/host_gpu/renderer/image/image.h

@@ -66,16 +66,16 @@ public:
 	[[nodiscard]] vk::ImageView FindView(const ImageViewInfo& view_info);
 	void                        AssociateDepth(ImageId image_id) { depth_id = image_id; }
 	using Barriers = std::vector<vk::ImageMemoryBarrier2>;
-	[[nodiscard]] Barriers
+	[[nodiscard]] Barriers GetBarriers(vk::ImageLayout                      destination_layout,
-	GetBarriers(vk::ImageLayout destination_layout, vk::AccessFlags2 destination_access,
+	                                   vk::AccessFlags2                     destination_access,
 	                                   vk::PipelineStageFlags2              destination_stage,
 	                                   std::optional<ImageSubresourceRange> range);
 	void Transit(vk::ImageLayout destination_layout, vk::AccessFlags2 destination_access,
 	             std::optional<ImageSubresourceRange> range, vk::CommandBuffer command_buffer);
-	void Upload(std::span<const vk::BufferImageCopy> copies, vk::Buffer buffer,
+	void Upload(std::span<const vk::BufferImageCopy> copies, vk::Buffer buffer, uint64_t offset,
-	            uint64_t offset, uint64_t size);
+	            uint64_t size);
-	void Download(std::span<const vk::BufferImageCopy> copies, vk::Buffer buffer,
+	void Download(std::span<const vk::BufferImageCopy> copies, vk::Buffer buffer, uint64_t offset,
-	              uint64_t offset, uint64_t size);
+	              uint64_t size);
 	void CopyImage(Image& source);
 	void Resolve(Image& source, const ImageSubresourceRange& source_range,
 	             const ImageSubresourceRange& destination_range);
@@ -95,6 +95,11 @@ public:
 	[[nodiscard]] bool IsCpuDirty() const { return m_cpu_dirty || m_maybe_cpu_dirty; }
 	[[nodiscard]] bool IsDefinitelyCpuDirty() const { return m_cpu_dirty; }
 	[[nodiscard]] bool IsMaybeCpuDirty() const { return m_maybe_cpu_dirty; }
+	void               MarkMaybeCpuDirty() {
+		if (!m_cpu_dirty) {
+			m_maybe_cpu_dirty = true;
+		}
+	}
 	[[nodiscard]] bool NeedsMaybeCpuHash() const {
 		return m_maybe_cpu_dirty && !m_maybe_hash_valid;
 	}
@@ -132,7 +137,8 @@ public:
 	void               MarkBufferModified() noexcept { m_buffer_modified = true; }
 	void               ClearBufferModified() noexcept { m_buffer_modified = false; }
 
-	[[nodiscard]] bool Overlaps(uint64_t address, uint64_t size, bool pages = false) const noexcept {
+	[[nodiscard]] bool Overlaps(uint64_t address, uint64_t size,
+	                            bool pages = false) const noexcept {
 		return pages ? ImagePageRangesOverlap(info.data.address, info.data.size, address, size)
 		             : ImageRangeOverlaps(info.data.address, info.data.size, address, size);
 	}
@@ -142,6 +148,7 @@ public:
 	[[nodiscard]] bool SafeToDownload() const noexcept {
 		return IsGpuModified() && !IsBufferModified() && !IsCpuDirty();
 	}
+	[[nodiscard]] bool IsTracked() const noexcept { return track_addr != 0 && track_addr_end != 0; }
 	[[nodiscard]] uint64_t AccountedSize() const noexcept {
 		return backing.image == nullptr ? 0 : (info.data.size + 1023) & ~uint64_t {1023};
 	}
@@ -153,6 +160,8 @@ public:
 	ImageUsage     usage;
 	ImageBinding   binding;
 	bool           registered     = false;
+	uint64_t       track_addr     = 0;
+	uint64_t       track_addr_end = 0;
 	ImageId        depth_id {};
 	uint64_t       tick_accessed_last = 0;
 	size_t         lru_id             = 0;

tests/PageManagerTests.cpp

@@ -577,6 +577,75 @@ void TestCrossRegionRange() {
   Check(VirtualFree(memory, 0, MEM_RELEASE) != 0, "VirtualFree failed");
 }
 
+void TestBatchedWatcherRanges() {
+  FaultContext context;
+  PageManager manager(InvalidateFault, &context);
+  context.manager = &manager;
+  const auto page_size = manager.GetPageSize();
+  constexpr uint64_t region_size = 4ull * 1024ull * 1024ull;
+  constexpr uint64_t allocation_size = region_size * 3;
+  auto *memory = Allocate(allocation_size);
+  const auto address = reinterpret_cast<uint64_t>(memory);
+
+  manager.OnGpuMap(address, allocation_size);
+
+  manager.UpdatePageWatchers(true, address + page_size, page_size);
+  manager.UpdatePageWatchers(true, address + page_size * 3, page_size);
+  manager.UpdatePageWatchers(true, address, page_size * 5);
+  for (uint64_t page = 0; page < 5; page++) {
+    Check(Protection(memory + page * page_size) == PAGE_READONLY,
+          "fragmented watch did not coalesce to read-only");
+  }
+  manager.UpdatePageWatchers(false, address, page_size * 5);
+  Check(IsWritable(memory) &&
+            Protection(memory + page_size) == PAGE_READONLY &&
+            IsWritable(memory + page_size * 2) &&
+            Protection(memory + page_size * 3) == PAGE_READONLY &&
+            IsWritable(memory + page_size * 4),
+        "fragmented unwatch lost overlapping watcher counts");
+  manager.UpdatePageWatchers(false, address + page_size, page_size);
+  manager.UpdatePageWatchers(false, address + page_size * 3, page_size);
+
+  manager.UpdatePageWatchers(true, address, allocation_size);
+  Check(!IsWritable(memory) &&
+            !IsWritable(memory + region_size) &&
+            !IsWritable(memory + region_size * 2) &&
+            !IsWritable(memory + allocation_size - page_size),
+        "large cross-region watch did not protect the full range");
+  manager.UpdatePageWatchers(false, address, allocation_size);
+  Check(IsWritable(memory) &&
+            IsWritable(memory + region_size) &&
+            IsWritable(memory + region_size * 2) &&
+            IsWritable(memory + allocation_size - page_size),
+        "large cross-region unwatch did not restore the full range");
+
+  manager.UpdatePageWatchers(true, address, page_size * 5);
+  manager.UpdatePageWatchers(true, address + page_size, page_size * 3,
+                             Libs::Graphics::PageWatchMode::ReadWrite);
+  Check(Protection(memory) == PAGE_READONLY &&
+            Protection(memory + page_size) == PAGE_NOACCESS &&
+            Protection(memory + page_size * 2) == PAGE_NOACCESS &&
+            Protection(memory + page_size * 3) == PAGE_NOACCESS &&
+            Protection(memory + page_size * 4) == PAGE_READONLY,
+        "mixed watcher modes installed incorrect protections");
+  manager.UpdatePageWatchers(false, address, page_size * 5);
+  Check(IsWritable(memory) &&
+            Protection(memory + page_size) == PAGE_NOACCESS &&
+            Protection(memory + page_size * 2) == PAGE_NOACCESS &&
+            Protection(memory + page_size * 3) == PAGE_NOACCESS &&
+            IsWritable(memory + page_size * 4),
+        "write unwatch incorrectly released read/write watchers");
+  manager.UpdatePageWatchers(false, address + page_size, page_size * 3,
+                             Libs::Graphics::PageWatchMode::ReadWrite);
+  Check(IsWritable(memory + page_size) &&
+            IsWritable(memory + page_size * 2) &&
+            IsWritable(memory + page_size * 3),
+        "read/write unwatch did not restore writable protection");
+
+  manager.OnGpuUnmap(address, allocation_size);
+  Check(VirtualFree(memory, 0, MEM_RELEASE) != 0, "VirtualFree failed");
+}
+
 [[noreturn]] void RunDeathCase(const char *name) {
   FaultContext context;
   auto manager = std::make_unique<PageManager>(InvalidateFault, &context);
@@ -795,6 +864,7 @@ int main(int argc, char **argv) {
   TestNativeAccessViolation();
   TestInvalidLateWriteTokenIsConsumed();
   TestCrossRegionRange();
+  TestBatchedWatcherRanges();
   TestConcurrentFault();
   TestExternalDirtyTransferDuringResolution();
   TestMappingDoesNotRequireCpuWriteAccess();

tests/ShaderRecompilerComputeTests.cpp

@@ -3195,6 +3195,10 @@ public:
       const auto fault_b_image = texture_cache.FindImage(fault_b_desc);
       texture_cache.MarkGpuWritten(fault_a_image);
       texture_cache.MarkGpuWritten(fault_b_image);
+      Require(name, "per-image watcher installation",
+              texture_cache.GetImage(fault_a_image).IsTracked() &&
+                  texture_cache.GetImage(fault_b_image).IsTracked(),
+              "same-page images did not install independent write watchers");
       constexpr uint64_t padding_fault_offset = 0x8080;
       uint32_t write_only_read_a = 0;
       uint32_t write_only_read_b = 0;
@@ -3212,13 +3216,24 @@ public:
               resources.HandleFault(PageFaultAccess::Write,
                                     base + padding_fault_offset) &&
                   texture_cache.GetImage(fault_a_image).IsGpuModified() &&
-                  texture_cache.GetImage(fault_b_image).IsGpuModified(),
+                  texture_cache.GetImage(fault_b_image).IsGpuModified() &&
+                  !texture_cache.GetImage(fault_a_image).IsTracked() &&
+                  !texture_cache.GetImage(fault_b_image).IsTracked() &&
+                  texture_cache.GetImage(fault_a_image).IsMaybeCpuDirty() &&
+                  texture_cache.GetImage(fault_b_image).IsMaybeCpuDirty(),
               "a byte-disjoint CPU write discarded authoritative images");
+      const auto retracked_a = texture_cache.FindImage(fault_a_desc);
+      const auto retracked_b = texture_cache.FindImage(fault_b_desc);
       Require(name, "same-page image re-track",
-              texture_cache.FindImage(fault_a_desc) == fault_a_image &&
+              retracked_a == fault_a_image && retracked_b == fault_b_image &&
-                  texture_cache.FindImage(fault_b_desc) == fault_b_image &&
+                  texture_cache.GetImage(fault_a_image).IsTracked() &&
+                  texture_cache.GetImage(fault_b_image).IsTracked() &&
+                  !texture_cache.GetImage(fault_a_image).IsCpuDirty() &&
+                  !texture_cache.GetImage(fault_b_image).IsCpuDirty() &&
                   texture_cache.SynchronizeImageToBuffer(base + 0x8000,
                                                          sizeof(fault_a)) &&
+                  texture_cache.GetImage(fault_a_image).IsTracked() &&
+                  texture_cache.GetImage(fault_b_image).IsTracked() &&
                   !texture_cache.GetImage(fault_a_image).IsGpuModified() &&
                   texture_cache.GetImage(fault_b_image).IsGpuModified(),
               "retiring one same-page image lost the surviving owner");