Implement sparse multi-image page ownership

src/CMakeLists.txt

@@ -289,6 +289,11 @@ add_executable(resource_mutex_tests EXCLUDE_FROM_ALL
 target_link_libraries(resource_mutex_tests common)
 target_include_directories(resource_mutex_tests PRIVATE ${inc_headers})
 
+add_executable(image_page_table_tests EXCLUDE_FROM_ALL
+	../tests/ImagePageTableTests.cpp
+)
+target_include_directories(image_page_table_tests PRIVATE ${inc_headers})
+
 add_kyty_full_emulator_test(shader_recompiler_compute_tests ../tests/ShaderRecompilerComputeTests.cpp)
 
 add_kyty_full_emulator_test(virtual_memory_allocation_tests ../tests/VirtualMemoryAllocationTests.cpp)

src/graphics/host_gpu/renderer/image.h

@@ -4,6 +4,7 @@
 #include "common/assert.h"
 #include "graphics/host_gpu/renderer/imageInfo.h"
 
+#include <algorithm>
 #include <cstddef>
 #include <span>
 
@@ -11,30 +12,83 @@ namespace Libs::Graphics {
 
 struct Image final: ImageInfo {
 	Image& operator=(const ImageInfo& value) {
-		if (m_cpu_dirty) {
+		if (IsCpuDirty()) {
 			EXIT("dirty sampled image cannot be reassigned\n");
 		}
 		static_cast<ImageInfo&>(*this) = value;
+		m_track_begin = address;
+		m_track_end   = address + size;
+		m_maybe_cpu_hash_valid = false;
 		return *this;
 	}
 
 	void InvalidateCpuWrite(uint64_t vaddr, uint64_t size) {
-		if (ImagePageRangesOverlap(address, this->size, vaddr, size)) {
+		if (ImageRangeOverlaps(address, this->size, vaddr, size)) {
 			m_cpu_dirty = true;
+			m_maybe_cpu_dirty = false;
+			m_maybe_cpu_hash_valid = false;
+			m_track_begin = m_track_end;
+		} else if (ImagePageRangesOverlap(address, this->size, vaddr, size)) {
+			constexpr uint64_t page_mask = 4096 - 1;
+			if (vaddr + size <= address) {
+				const auto next_page = (address + page_mask) & ~page_mask;
+				m_track_begin = std::min(m_track_end, std::max(m_track_begin, next_page));
+			} else if (vaddr >= address + this->size) {
+				const auto page = (address + this->size) & ~page_mask;
+				m_track_end = std::max(m_track_begin, std::min(m_track_end, page));
+			}
+			m_maybe_cpu_dirty = m_track_begin == m_track_end;
 		}
 	}
 
-	[[nodiscard]] bool IsCpuDirty() const { return m_cpu_dirty; }
+	[[nodiscard]] bool IsCpuDirty() const { return m_cpu_dirty || m_maybe_cpu_dirty; }
+	[[nodiscard]] bool IsDefinitelyCpuDirty() const { return m_cpu_dirty; }
+	[[nodiscard]] bool IsMaybeCpuDirty() const { return m_maybe_cpu_dirty; }
+	[[nodiscard]] bool NeedsMaybeCpuHash() const {
+		return m_maybe_cpu_dirty && !m_maybe_cpu_hash_valid;
+	}
+	[[nodiscard]] bool IsCpuTrackingComplete() const {
+		return m_track_begin == address && m_track_end == address + size;
+	}
+	void SetMaybeCpuHash(uint64_t hash) {
+		if (!NeedsMaybeCpuHash()) {
+			EXIT("sampled image cannot initialize maybe-dirty hash\n");
+		}
+		m_maybe_cpu_hash       = hash;
+		m_maybe_cpu_hash_valid = true;
+	}
+	[[nodiscard]] bool ResolveMaybeCpuHash(uint64_t hash) {
+		if (!m_maybe_cpu_dirty || !m_maybe_cpu_hash_valid || m_cpu_dirty) {
+			EXIT("sampled image cannot resolve maybe-dirty hash\n");
+		}
+		m_maybe_cpu_dirty      = false;
+		m_maybe_cpu_hash_valid = false;
+		m_cpu_dirty            = hash != m_maybe_cpu_hash;
+		if (!m_cpu_dirty) {
+			m_track_begin = address;
+			m_track_end   = address + size;
+		}
+		return m_cpu_dirty;
+	}
 
 	void RefreshComplete() {
-		if (!m_cpu_dirty) {
+		if (!IsCpuDirty()) {
 			EXIT("clean sampled image cannot complete a refresh\n");
 		}
 		m_cpu_dirty = false;
+		m_maybe_cpu_dirty = false;
+		m_maybe_cpu_hash_valid = false;
+		m_track_begin = address;
+		m_track_end   = address + size;
 	}
 
 private:
 	bool m_cpu_dirty = false;
+	bool m_maybe_cpu_dirty = false;
+	bool m_maybe_cpu_hash_valid = false;
+	uint64_t m_track_begin = 0;
+	uint64_t m_track_end = 0;
+	uint64_t m_maybe_cpu_hash = 0;
 };
 
 struct ImageRetirementRange {
@@ -60,8 +114,8 @@ FindImageRetirementConflict(std::span<const ImageRetirementRange> ranges) {
 			if (ranges[retained].retire) {
 				continue;
 			}
-			if (ImagePageRangesOverlap(ranges[retired].address, ranges[retired].size,
-			                           ranges[retained].address, ranges[retained].size)) {
+			if (ImageRangeOverlaps(ranges[retired].address, ranges[retired].size,
+			                       ranges[retained].address, ranges[retained].size)) {
 				return {retired, retained};
 			}
 		}

src/graphics/host_gpu/renderer/imageInfo.h

@@ -821,14 +821,11 @@ ClassifyRenderTargetOverlap(const ImageInfo& sampled, bool sampled_gpu_modified,
 	if (!ImagePageRangesOverlap(sampled.address, sampled.size, target.address, target.size)) {
 		return RenderTargetOverlap::None;
 	}
-	const bool exact = sampled.address == target.address && sampled.size == target.size;
-	const bool page_isolated_overlap =
-	    sampled.address % TRACKER_PAGE_SIZE == 0 && sampled.size % TRACKER_PAGE_SIZE == 0 &&
-	    target.address % TRACKER_PAGE_SIZE == 0 && target.size % TRACKER_PAGE_SIZE == 0 &&
-	    ImageRangeOverlaps(sampled.address, sampled.size, target.address, target.size);
-	return !sampled_gpu_modified && same_context && (exact || page_isolated_overlap)
-	           ? RenderTargetOverlap::RetireSampled
-	           : RenderTargetOverlap::Unsupported;
+	if (!ImageRangeOverlaps(sampled.address, sampled.size, target.address, target.size)) {
+		return RenderTargetOverlap::None;
+	}
+	return !sampled_gpu_modified && same_context ? RenderTargetOverlap::RetireSampled
+	                                              : RenderTargetOverlap::Unsupported;
 }
 
 [[nodiscard]] inline RenderTargetOverlap
@@ -886,6 +883,9 @@ ClassifySampledRenderTargetOverlap(const ImageInfo& sampled, const RenderTargetI
 	if (!ImagePageRangesOverlap(sampled.address, sampled.size, target.address, target.size)) {
 		return RenderTargetOverlap::None;
 	}
+	if (!ImageRangeOverlaps(sampled.address, sampled.size, target.address, target.size)) {
+		return RenderTargetOverlap::None;
+	}
 	return same_context && !target_buffer_modified ? RenderTargetOverlap::RetireTarget
 	                                               : RenderTargetOverlap::Unsupported;
 }

src/graphics/host_gpu/renderer/multiLevelPageTable.h

@@ -0,0 +1,342 @@
+#ifndef EMULATOR_SRC_GRAPHICS_HOST_GPU_RENDERER_MULTILEVELPAGETABLE_H_
+#define EMULATOR_SRC_GRAPHICS_HOST_GPU_RENDERER_MULTILEVELPAGETABLE_H_
+
+#include <algorithm>
+#include <array>
+#include <cstddef>
+#include <cstdint>
+#include <list>
+#include <memory>
+#include <stdexcept>
+#include <utility>
+#include <vector>
+
+namespace Libs::Graphics {
+
+// Sparse two-level lookup for texture-cache page ownership.
+// Entries are selected by guest page number; queries never allocate L1 buckets.
+template <typename EntryT, size_t PageBits = 20, size_t AddressSpaceBits = 40, size_t FirstLevelBits = 10>
+class MultiLevelPageTable final {
+public:
+	using Entry = EntryT;
+
+	static_assert(AddressSpaceBits < 64);
+	static_assert(PageBits < AddressSpaceBits);
+	static_assert(FirstLevelBits > 0 && FirstLevelBits <= AddressSpaceBits - PageBits);
+
+	static constexpr size_t kPageBits          = PageBits;
+	static constexpr size_t kAddressSpaceBits  = AddressSpaceBits;
+	static constexpr size_t kFirstLevelBits    = FirstLevelBits;
+	static constexpr size_t kSecondLevelBits   = AddressSpaceBits - FirstLevelBits - PageBits;
+	static constexpr size_t kFirstLevelEntries = size_t{1} << FirstLevelBits;
+	static constexpr size_t kBucketEntries     = size_t{1} << kSecondLevelBits;
+	static constexpr size_t kPageCount         = size_t{1} << (AddressSpaceBits - PageBits);
+	static constexpr uint64_t kAddressSpaceSize = uint64_t{1} << AddressSpaceBits;
+
+	struct PageRange final {
+		size_t first          = 0;
+		size_t last_exclusive = 0;
+	};
+
+	MultiLevelPageTable(): m_first_level(kFirstLevelEntries) {}
+
+	[[nodiscard]] Entry* Find(size_t page) {
+		if (!IsValidPage(page)) {
+			return nullptr;
+		}
+		auto& bucket = m_first_level[FirstLevelIndex(page)];
+		return bucket == nullptr ? nullptr : &(*bucket)[SecondLevelIndex(page)];
+	}
+
+	[[nodiscard]] const Entry* Find(size_t page) const {
+		if (!IsValidPage(page)) {
+			return nullptr;
+		}
+		const auto& bucket = m_first_level[FirstLevelIndex(page)];
+		return bucket == nullptr ? nullptr : &(*bucket)[SecondLevelIndex(page)];
+	}
+
+	[[nodiscard]] Entry& GetOrCreate(size_t page) {
+		if (!IsValidPage(page)) {
+			throw std::out_of_range("MultiLevelPageTable page is outside the guest address space");
+		}
+		auto& bucket = m_first_level[FirstLevelIndex(page)];
+		if (bucket == nullptr) {
+			bucket = std::make_unique<Bucket>();
+			++m_allocated_buckets;
+		}
+		return (*bucket)[SecondLevelIndex(page)];
+	}
+
+	[[nodiscard]] Entry& operator[](size_t page) { return GetOrCreate(page); }
+
+	[[nodiscard]] static constexpr bool IsValidPage(size_t page) { return page < kPageCount; }
+
+	// Returns the half-open page interval touched by a non-empty byte range.
+	// An end exactly at 2^AddressSpaceBits is valid; wrapping or crossing it is not.
+	[[nodiscard]] static constexpr bool TryGetPageRange(uint64_t address, uint64_t size, PageRange& range) {
+		if (size == 0 || address >= kAddressSpaceSize || size > kAddressSpaceSize - address) {
+			return false;
+		}
+		const uint64_t end = address + size;
+		range.first       = static_cast<size_t>(address >> PageBits);
+		range.last_exclusive = static_cast<size_t>(((end - 1) >> PageBits) + 1);
+		return true;
+	}
+
+	[[nodiscard]] size_t AllocatedBucketCount() const { return m_allocated_buckets; }
+
+private:
+	using Bucket = std::array<Entry, kBucketEntries>;
+
+	[[nodiscard]] static constexpr size_t FirstLevelIndex(size_t page) { return page >> kSecondLevelBits; }
+	[[nodiscard]] static constexpr size_t SecondLevelIndex(size_t page) { return page & (kBucketEntries - 1); }
+
+	std::vector<std::unique_ptr<Bucket>> m_first_level;
+	size_t                               m_allocated_buckets = 0;
+};
+
+// Removes only the requested owner. Other owners in the same page entry remain intact.
+template <typename Container, typename Value>
+[[nodiscard]] bool EraseExact(Container& owners, const Value& owner) {
+	const auto it = std::find(owners.begin(), owners.end(), owner);
+	if (it == owners.end()) {
+		return false;
+	}
+	owners.erase(it);
+	return true;
+}
+
+// Multi-range ownership with 1 MiB candidate buckets and precise 4 KiB
+// lifetime accounting. OwnerT only needs equality.
+template <typename OwnerT>
+class MultiRangePageOwnerIndex final {
+private:
+	struct Registration;
+	using MembershipList = std::vector<const Registration*>;
+
+public:
+	struct ByteRange final {
+		uint64_t address = 0;
+		uint64_t size    = 0;
+	};
+
+	using CoarseTable   = MultiLevelPageTable<MembershipList, 20, 40, 10>;
+	using TrackingTable = MultiLevelPageTable<MembershipList, 12, 40, 18>;
+
+	[[nodiscard]] bool Register(const OwnerT& owner, const std::vector<ByteRange>& ranges) {
+		if (FindRegistration(owner) != m_registrations.end()) {
+			return false;
+		}
+		auto normalized = Normalize(ranges);
+		if (normalized.empty()) {
+			return false;
+		}
+		m_registrations.push_back({owner, std::move(normalized)});
+		const Registration* registration = &m_registrations.back();
+		for (const size_t page: CollectPages<20>(registration->ranges)) {
+			m_coarse_pages[page].push_back(registration);
+		}
+		for (const size_t page: CollectPages<12>(registration->ranges)) {
+			m_tracking_pages[page].push_back(registration);
+		}
+		return true;
+	}
+
+	// No state changes if an expected membership is absent. Final releases are
+	// sorted and coalesced 4 KiB spans whose final owner disappeared.
+	[[nodiscard]] bool Unregister(const OwnerT& owner, std::vector<ByteRange>& final_releases) {
+		final_releases.clear();
+		auto registration = FindRegistration(owner);
+		if (registration == m_registrations.end()) {
+			return false;
+		}
+		const auto coarse_pages   = CollectPages<20>(registration->ranges);
+		const auto tracking_pages = CollectPages<12>(registration->ranges);
+		const Registration* registration_ptr = &*registration;
+		if (!HasAllMemberships(m_coarse_pages, coarse_pages, registration_ptr) ||
+		    !HasAllMemberships(m_tracking_pages, tracking_pages, registration_ptr)) {
+			return false;
+		}
+		std::vector<size_t> final_pages;
+		for (const size_t page: coarse_pages) {
+			(void)EraseExact(*m_coarse_pages.Find(page), registration_ptr);
+		}
+		for (const size_t page: tracking_pages) {
+			auto& owners = *m_tracking_pages.Find(page);
+			if (owners.size() == 1) {
+				final_pages.push_back(page);
+			}
+			(void)EraseExact(owners, registration_ptr);
+		}
+		m_registrations.erase(registration);
+		final_releases = CoalesceTrackingPages(final_pages);
+		return true;
+	}
+
+	[[nodiscard]] std::vector<OwnerT> Query(uint64_t address, uint64_t size) const {
+		return Query(address, size, [](const OwnerT&) { return true; });
+	}
+
+	template <typename Predicate>
+	[[nodiscard]] std::vector<OwnerT> Query(uint64_t address, uint64_t size, Predicate&& predicate) const {
+		return QueryImpl(address, size, true, std::forward<Predicate>(predicate));
+	}
+
+	// Fault paths use page candidates: exact byte-disjoint owners sharing a
+	// touched 4 KiB page are intentionally retained.
+	[[nodiscard]] std::vector<OwnerT> QueryCandidates(uint64_t address, uint64_t size) const {
+		return QueryCandidates(address, size, [](const OwnerT&) { return true; });
+	}
+
+	template <typename Predicate>
+	[[nodiscard]] std::vector<OwnerT> QueryCandidates(uint64_t address, uint64_t size, Predicate&& predicate) const {
+		return QueryImpl(address, size, false, std::forward<Predicate>(predicate));
+	}
+
+	[[nodiscard]] size_t CoarseMembershipCount(size_t page) const {
+		const auto* owners = m_coarse_pages.Find(page);
+		return owners == nullptr ? 0 : owners->size();
+	}
+	[[nodiscard]] size_t TrackingMembershipCount(size_t page) const {
+		const auto* owners = m_tracking_pages.Find(page);
+		return owners == nullptr ? 0 : owners->size();
+	}
+
+private:
+	template <typename Predicate>
+	[[nodiscard]] std::vector<OwnerT> QueryImpl(uint64_t address, uint64_t size, bool strict_bytes,
+	                                           Predicate&& predicate) const {
+		typename CoarseTable::PageRange coarse_range{};
+		typename TrackingTable::PageRange tracking_range{};
+		if (!CoarseTable::TryGetPageRange(address, size, coarse_range) ||
+		    !TrackingTable::TryGetPageRange(address, size, tracking_range)) {
+			return {};
+		}
+		MembershipList candidates;
+		for (size_t page = coarse_range.first; page < coarse_range.last_exclusive; ++page) {
+			if (const auto* owners = m_coarse_pages.Find(page); owners != nullptr) {
+				AppendUnique(candidates, *owners);
+			}
+		}
+		std::vector<OwnerT> result;
+		for (const Registration* registration: candidates) {
+			if ((!strict_bytes || Overlaps(registration->ranges, address, size)) &&
+			    HasTrackingMembership(registration, tracking_range) && predicate(registration->owner)) {
+				result.push_back(registration->owner);
+			}
+		}
+		return result;
+	}
+
+	struct Registration final {
+		OwnerT                 owner;
+		std::vector<ByteRange> ranges;
+	};
+	using RegistrationIterator      = typename std::list<Registration>::iterator;
+	using ConstRegistrationIterator = typename std::list<Registration>::const_iterator;
+
+	[[nodiscard]] RegistrationIterator FindRegistration(const OwnerT& owner) {
+		return std::find_if(m_registrations.begin(), m_registrations.end(),
+		                    [&](const Registration& item) { return item.owner == owner; });
+	}
+	[[nodiscard]] ConstRegistrationIterator FindRegistration(const OwnerT& owner) const {
+		return std::find_if(m_registrations.begin(), m_registrations.end(),
+		                    [&](const Registration& item) { return item.owner == owner; });
+	}
+
+	[[nodiscard]] static std::vector<ByteRange> Normalize(const std::vector<ByteRange>& ranges) {
+		std::vector<ByteRange> sorted;
+		for (const auto& range: ranges) {
+			typename CoarseTable::PageRange ignored{};
+			if (!CoarseTable::TryGetPageRange(range.address, range.size, ignored)) {
+				return {};
+			}
+			sorted.push_back(range);
+		}
+		std::sort(sorted.begin(), sorted.end(),
+		          [](const ByteRange& lhs, const ByteRange& rhs) { return lhs.address < rhs.address; });
+		std::vector<ByteRange> merged;
+		for (const auto& range: sorted) {
+			if (merged.empty() || range.address > merged.back().address + merged.back().size) {
+				merged.push_back(range);
+			} else {
+				const uint64_t end = std::max(merged.back().address + merged.back().size, range.address + range.size);
+				merged.back().size = end - merged.back().address;
+			}
+		}
+		return merged;
+	}
+
+	template <size_t Bits>
+	[[nodiscard]] static std::vector<size_t> CollectPages(const std::vector<ByteRange>& ranges) {
+		std::vector<size_t> pages;
+		for (const auto& range: ranges) {
+			const size_t first = static_cast<size_t>(range.address >> Bits);
+			const size_t last  = static_cast<size_t>((range.address + range.size - 1) >> Bits);
+			for (size_t page = first; page <= last; ++page) {
+				pages.push_back(page);
+			}
+		}
+		std::sort(pages.begin(), pages.end());
+		pages.erase(std::unique(pages.begin(), pages.end()), pages.end());
+		return pages;
+	}
+
+	template <typename Table>
+	[[nodiscard]] static bool HasAllMemberships(const Table& table, const std::vector<size_t>& pages,
+	                                            const Registration* registration) {
+		for (const size_t page: pages) {
+			const auto* owners = table.Find(page);
+			if (owners == nullptr || std::find(owners->begin(), owners->end(), registration) == owners->end()) {
+				return false;
+			}
+		}
+		return true;
+	}
+
+	[[nodiscard]] bool HasTrackingMembership(const Registration* registration,
+	                                         const typename TrackingTable::PageRange& range) const {
+		for (size_t page = range.first; page < range.last_exclusive; ++page) {
+			const auto* owners = m_tracking_pages.Find(page);
+			if (owners != nullptr && std::find(owners->begin(), owners->end(), registration) != owners->end()) {
+				return true;
+			}
+		}
+		return false;
+	}
+
+	[[nodiscard]] static bool Overlaps(const std::vector<ByteRange>& ranges, uint64_t address, uint64_t size) {
+		const uint64_t end = address + size;
+		return std::any_of(ranges.begin(), ranges.end(), [&](const ByteRange& range) {
+			return range.address < end && address < range.address + range.size;
+		});
+	}
+	static void AppendUnique(MembershipList& destination, const MembershipList& source) {
+		for (const Registration* registration: source) {
+			if (std::find(destination.begin(), destination.end(), registration) == destination.end()) {
+				destination.push_back(registration);
+			}
+		}
+	}
+	[[nodiscard]] static std::vector<ByteRange> CoalesceTrackingPages(const std::vector<size_t>& pages) {
+		std::vector<ByteRange> result;
+		for (const size_t page: pages) {
+			const uint64_t address = static_cast<uint64_t>(page) << 12;
+			if (!result.empty() && result.back().address + result.back().size == address) {
+				result.back().size += uint64_t{1} << 12;
+			} else {
+				result.push_back({address, uint64_t{1} << 12});
+			}
+		}
+		return result;
+	}
+
+	CoarseTable               m_coarse_pages;
+	TrackingTable             m_tracking_pages;
+	std::list<Registration> m_registrations;
+};
+
+} // namespace Libs::Graphics
+
+#endif // EMULATOR_SRC_GRAPHICS_HOST_GPU_RENDERER_MULTILEVELPAGETABLE_H_

src/graphics/host_gpu/renderer/textureCache.cpp

@@ -22,6 +22,8 @@
 #include "graphics/shader/shader.h"
 #include "kernel/memory.h"
 
+#include <xxhash.h>
+
 #include <algorithm>
 #include <atomic>
 #include <cstdlib>
@@ -37,6 +39,29 @@ namespace {
 thread_local const void* g_texture_cache_lock_owner = nullptr;
 thread_local const void* g_texture_fault_owner      = nullptr;
 
+[[nodiscard]] uint64_t HashSampledImageEdges(const Image& image) {
+	constexpr uint64_t page_mask = TRACKER_PAGE_SIZE - 1;
+	const uint64_t     begin     = image.address;
+	const uint64_t     end       = image.address + image.size;
+	const uint64_t     head_end  = std::min(end, (begin + page_mask) & ~page_mask);
+	const uint64_t     tail_begin = std::max(begin, end & ~page_mask);
+	const uint64_t     head_size = head_end - begin;
+	const uint64_t     tail_size = tail_begin < head_end ? end - head_end : end - tail_begin;
+	std::array<uint8_t, TRACKER_PAGE_SIZE * 2> bytes {};
+	if (head_size + tail_size > bytes.size()) {
+		EXIT("TextureCache: sampled-image edge hash range overflow\n");
+	}
+	if ((head_size != 0 && !LibKernel::Memory::TryReadBacking(begin, bytes.data(), head_size)) ||
+	    (tail_size != 0 &&
+	     !LibKernel::Memory::TryReadBacking(tail_begin < head_end ? head_end : tail_begin,
+	                                        bytes.data() + head_size, tail_size))) {
+		EXIT("TextureCache: failed to hash sampled-image edge backing, addr=0x%016" PRIx64
+		     " size=0x%016" PRIx64 "\n",
+		     image.address, image.size);
+	}
+	return XXH3_64bits(bytes.data(), static_cast<size_t>(head_size + tail_size));
+}
+
 class FaultSafeTextureLock final {
 public:
 	FaultSafeTextureLock(const void* owner, TrackingSpinLock& mutex): m_mutex(mutex) {
@@ -87,13 +112,14 @@ struct TextureCache::CachedImage {
 	bool             gpu_modified        = false;
 	bool             buffer_modified     = false;
 	bool             stencil_initialized = false;
+	bool             registered          = false;
 
 	~CachedImage() {
-		if (ctx == nullptr || image == nullptr) {
+		if (ctx == nullptr || image == nullptr || registered) {
 			EXIT("TextureCache: cached image destroyed with invalid resources, ctx=%p image=%p "
-			     "kind=%u\n",
+			     "kind=%u registered=%d\n",
 			     static_cast<const void*>(ctx), static_cast<const void*>(image),
-			     static_cast<uint32_t>(kind));
+			     static_cast<uint32_t>(kind), registered);
 		}
 		switch (kind) {
 			case Kind::Texture:
@@ -191,6 +217,43 @@ struct TextureCache::CachedImage {
 	}
 };
 
+void TextureCache::RegisterImageLocked(CachedImage& image) {
+	if (image.registered) {
+		EXIT("TextureCache: registering an already registered image\n");
+	}
+	std::vector<ImageOwnerIndex::ByteRange> ranges;
+	ranges.reserve(image.RangeCount());
+	for (uint32_t range = 0; range < image.RangeCount(); range++) {
+		ranges.push_back({image.Address(range), image.Size(range)});
+	}
+	if (!m_image_owner_index.Register(&image, ranges)) {
+		EXIT("TextureCache: invalid or duplicate image registration\n");
+	}
+	image.registered = true;
+}
+
+void TextureCache::UnregisterImageLocked(CachedImage& image, bool release_tracking) {
+	if (!image.registered) {
+		EXIT("TextureCache: unregistering an unregistered image\n");
+	}
+	std::vector<ImageOwnerIndex::ByteRange> final_releases;
+	if (!m_image_owner_index.Unregister(&image, final_releases)) {
+		EXIT("TextureCache: image missing from owner index\n");
+	}
+	if (release_tracking) {
+		for (const auto& release: final_releases) {
+			m_memory_tracker.UntrackMemory(release.address, release.size);
+		}
+	}
+	image.registered = false;
+}
+
+std::vector<TextureCache::CachedImage*>
+TextureCache::FindImagesInRegionLocked(uint64_t vaddr, uint64_t size, bool page_overlap) {
+	return page_overlap ? m_image_owner_index.QueryCandidates(vaddr, size)
+	                    : m_image_owner_index.Query(vaddr, size);
+}
+
 struct TextureCache::ReadbackWorker {
 	enum class State : uint32_t {
 		Idle,
@@ -653,12 +716,10 @@ void TextureCache::RetireImages(const std::vector<CachedImage*>& retire,
 				     "addr=0x%016" PRIx64 " size=0x%016" PRIx64 "\n",
 				     (*it)->Address(range), (*it)->Size(range));
 			}
-			// A native image transition transfers this registration to the replacement cache
-			// record. Keeping it GPU-owned also keeps guest page protection continuous.
-			if (!native_image) {
-				m_memory_tracker.UntrackMemory((*it)->Address(range), (*it)->Size(range));
-			}
 		}
+		// A native transition keeps tracker ownership continuous until the replacement is
+		// published. Normal retirement only releases 4 KiB pages with no remaining image owner.
+		UnregisterImageLocked(**it, !native_image);
 		it = m_images.erase(it);
 		removed++;
 	}
@@ -1500,9 +1561,9 @@ VkImageView TextureCache::GetStorageTextureStorageView(GraphicContext*
 }
 
 TextureCache::CachedImage* TextureCache::FindGpuReadbackPageCandidateLocked(uint64_t vaddr,
-                                                                            uint64_t size) const {
+	                                                                         uint64_t size) {
 	CachedImage* selected = nullptr;
-	for (const auto& cached: m_images) {
+	for (auto* cached: FindImagesInRegionLocked(vaddr, size, true)) {
 		if (!cached->IsGpuReadbackPageCandidate(vaddr, size)) {
 			continue;
 		}
@@ -1510,33 +1571,36 @@ TextureCache::CachedImage* TextureCache::FindGpuReadbackPageCandidateLocked(uint
 			EXIT("TextureCache: CPU fault has multiple GPU-modified image page candidates, "
 			     "addr=0x%016" PRIx64 " size=0x%016" PRIx64 " first=%p second=%p\n",
 			     vaddr, size, static_cast<const void*>(selected),
-			     static_cast<const void*>(cached.get()));
+			     static_cast<const void*>(cached));
 		}
-		selected = cached.get();
+		selected = cached;
 	}
 	return selected;
 }
 
 void TextureCache::MarkSampledAliasesCpuDirtyLocked(uint64_t vaddr, uint64_t size) {
-	for (auto& cached: m_images) {
+	for (auto* cached: FindImagesInRegionLocked(vaddr, size, true)) {
 		if (cached->kind != CachedImage::Kind::Texture) {
 			continue;
 		}
 		if (cached->gpu_modified &&
-		    ImagePageRangesOverlap(cached->info.address, cached->info.size, vaddr, size)) {
+		    ImageRangeOverlaps(cached->info.address, cached->info.size, vaddr, size)) {
 			EXIT("TextureCache: CPU write overlaps a GPU-modified sampled texture, "
 			     "write=0x%016" PRIx64 "+0x%016" PRIx64 " image=0x%016" PRIx64 "+0x%016" PRIx64
 			     "\n",
 			     vaddr, size, cached->info.address, cached->info.size);
 		}
 		cached->info.InvalidateCpuWrite(vaddr, size);
+		if (cached->info.NeedsMaybeCpuHash()) {
+			cached->info.SetMaybeCpuHash(HashSampledImageEdges(cached->info));
+		}
 	}
 }
 
 void TextureCache::RetireSampledTargetAliases(GraphicContext* ctx, const ImageInfo& requested) {
 	std::vector<CachedImage*> retire;
 	bool                      wait_idle = false;
-	for (const auto& cached: m_images) {
+	for (auto* cached: FindImagesInRegionLocked(requested.address, requested.size, true)) {
 		if (cached->kind != CachedImage::Kind::RenderTarget) {
 			continue;
 		}
@@ -1545,7 +1609,7 @@ void TextureCache::RetireSampledTargetAliases(GraphicContext* ctx, const ImageIn
 			case RenderTargetOverlap::None: continue;
 			case RenderTargetOverlap::RetireTarget:
 				wait_idle |= cached->gpu_modified;
-				retire.push_back(cached.get());
+				retire.push_back(cached);
 				break;
 			case RenderTargetOverlap::RetireSampled:
 			case RenderTargetOverlap::PreserveStorage:
@@ -1558,13 +1622,13 @@ void TextureCache::RetireSampledTargetAliases(GraphicContext* ctx, const ImageIn
 				     cached->ctx == ctx, cached->gpu_modified, cached->buffer_modified);
 		}
 	}
-	for (const auto& cached: m_images) {
+	for (auto* cached: FindImagesInRegionLocked(requested.address, requested.size, true)) {
 		if (cached->kind != CachedImage::Kind::DepthTarget) {
 			continue;
 		}
 		const auto overlap = ClassifyImageRangeOverlap(requested.address, requested.size,
 		                                               cached->depth.address, cached->depth.size);
-		if (overlap == ImageRangeOverlap::None) {
+		if (overlap == ImageRangeOverlap::None || overlap == ImageRangeOverlap::PageOnly) {
 			continue;
 		}
 		const auto delta = cached->depth.address >= requested.address
@@ -1578,8 +1642,7 @@ void TextureCache::RetireSampledTargetAliases(GraphicContext* ctx, const ImageIn
 		const bool supported = cached->ctx == ctx && !cached->buffer_modified &&
 		                       cached->depth.stencil_address == 0 &&
 		                       cached->depth.stencil_size == 0 && cached->depth.layers == 1 &&
-		                       (overlap == ImageRangeOverlap::PageOnly ||
-		                        (contained && (exact_range || sampled_expansion)));
+		                       contained && (exact_range || sampled_expansion);
 		if (!supported) {
 			EXIT("TextureCache: unsupported sampled/depth-target alias, sampled=0x%016" PRIx64
 			     "+0x%016" PRIx64 " depth=0x%016" PRIx64 "+0x%016" PRIx64
@@ -1599,7 +1662,7 @@ void TextureCache::RetireSampledTargetAliases(GraphicContext* ctx, const ImageIn
 			     cached->depth.tile_mode, cached->depth.htile_address, cached->depth.htile_size);
 		}
 		wait_idle |= cached->gpu_modified;
-		retire.push_back(cached.get());
+		retire.push_back(cached);
 	}
 	if (retire.empty()) {
 		return;
@@ -1660,10 +1723,9 @@ void TextureCache::RetireSampledTargetAliases(GraphicContext* ctx, const ImageIn
 	RetireImages(retire);
 }
 
-void TextureCache::RetireStoragePageNeighbors(GraphicContext* ctx, const ImageInfo& requested) {
+void TextureCache::ResolveStorageImageOverlaps(GraphicContext* ctx, const ImageInfo& requested) {
 	std::vector<CachedImage*> retire;
-	bool                      wait_idle = false;
-	for (const auto& cached: m_images) {
+	for (auto* cached: FindImagesInRegionLocked(requested.address, requested.size, true)) {
 		const bool tracker_gpu =
 		    m_memory_tracker.IsRegionGpuModified(cached->Address(), cached->Size());
 		switch (ClassifyStorageImageOverlap(
@@ -1671,8 +1733,8 @@ void TextureCache::RetireStoragePageNeighbors(GraphicContext* ctx, const ImageIn
 		    cached->kind == CachedImage::Kind::Texture, cached->ctx == ctx, cached->gpu_modified,
 		    cached->buffer_modified, tracker_gpu)) {
 			case StorageImageOverlap::None: continue;
-			case StorageImageOverlap::RetireSampled: retire.push_back(cached.get()); continue;
-			case StorageImageOverlap::PageNeighbor: break;
+			case StorageImageOverlap::RetireSampled: retire.push_back(cached); continue;
+			case StorageImageOverlap::PageNeighbor: continue;
 			case StorageImageOverlap::Unsupported:
 				EXIT("TextureCache: unsupported storage-image byte alias, requested=0x%016" PRIx64
 				     "+0x%016" PRIx64 " existing=0x%016" PRIx64 "+0x%016" PRIx64
@@ -1681,25 +1743,8 @@ void TextureCache::RetireStoragePageNeighbors(GraphicContext* ctx, const ImageIn
 				     static_cast<uint32_t>(cached->kind), cached->ctx == ctx, cached->gpu_modified,
 				     tracker_gpu, cached->buffer_modified);
 		}
-		const bool tracker_cpu =
-		    m_memory_tracker.IsRegionCpuModified(cached->Address(), cached->Size());
-		if (cached->kind != CachedImage::Kind::StorageTexture || cached->ctx != ctx ||
-		    cached->buffer_modified || cached->info.IsCpuDirty() ||
-		    cached->gpu_modified != tracker_gpu || (tracker_gpu && tracker_cpu)) {
-			EXIT("TextureCache: unsupported storage-image page neighbor, requested=0x%016" PRIx64
-			     "+0x%016" PRIx64 " existing=0x%016" PRIx64 "+0x%016" PRIx64
-			     " kind=%u same_context=%d gpu=%d/%d cpu=%d buffer=%d image_cpu=%d\n",
-			     requested.address, requested.size, cached->Address(), cached->Size(),
-			     static_cast<uint32_t>(cached->kind), cached->ctx == ctx, cached->gpu_modified,
-			     tracker_gpu, tracker_cpu, cached->buffer_modified, cached->info.IsCpuDirty());
-		}
-		wait_idle |= cached->gpu_modified;
-		retire.push_back(cached.get());
 	}
 	RequireRetirementIsolation(retire, "storage neighbor", requested.address, requested.size);
-	if (wait_idle) {
-		VulkanDeviceWaitIdle(ctx);
-	}
 	for (auto* cached: retire) {
 		if (!cached->gpu_modified) {
 			continue;
@@ -1715,7 +1760,7 @@ void TextureCache::RetireStoragePageNeighbors(GraphicContext* ctx, const ImageIn
 void TextureCache::RetireStorageDepthAliasLocked(GraphicContext* ctx,
 	                                              const ImageInfo& requested) {
 	CachedImage* selected = nullptr;
-	for (const auto& entry: m_images) {
+	for (auto* entry: FindImagesInRegionLocked(requested.address, requested.size, true)) {
 		auto& cached = *entry;
 		if (cached.kind != CachedImage::Kind::DepthTarget ||
 		    !cached.OverlapsRange(requested.address, requested.size, true)) {
@@ -1765,6 +1810,9 @@ TextureCache::~TextureCache() {
 	if (ctx != nullptr) {
 		VulkanDeviceWaitIdle(ctx);
 	}
+	for (const auto& image: m_images) {
+		UnregisterImageLocked(*image, false);
+	}
 	m_images.clear();
 	for (size_t i = 0; i < m_dummy_sampled_textures.size(); i++) {
 		if (m_dummy_sampled_textures[i] != nullptr) {
@@ -1881,6 +1929,15 @@ VulkanImage* TextureCache::FindTexture(CommandBuffer* command, GraphicContext* c
 		match = cached;
 	}
 	if (match != nullptr) {
+		if (match->info.IsMaybeCpuDirty() && !match->info.IsDefinitelyCpuDirty()) {
+			const bool changed =
+			    match->info.ResolveMaybeCpuHash(HashSampledImageEdges(match->info));
+			if (!changed) {
+				m_memory_tracker.ForEachUploadRange(
+				    info.address, info.size, false, [](uint64_t, uint64_t) noexcept {},
+				    []() noexcept {});
+			}
+		}
 		const bool buffer_dirty = match->buffer_modified;
 		if (buffer_dirty) {
 			if (match->gpu_modified ||
@@ -1952,6 +2009,7 @@ VulkanImage* TextureCache::FindTexture(CommandBuffer* command, GraphicContext* c
 	auto* image = cached->image;
 	command->RetainResourceUntilFence(cached);
 	m_images.push_back(std::move(cached));
+	RegisterImageLocked(*m_images.back());
 	return image;
 }
 
@@ -2073,7 +2131,7 @@ StorageTextureVulkanImage* TextureCache::FindStorageTexture(CommandBuffer*   com
 		return static_cast<StorageTextureVulkanImage*>(match->image);
 	}
 
-	RetireStoragePageNeighbors(ctx, info);
+	ResolveStorageImageOverlaps(ctx, info);
 	m_images.reserve(m_images.size() + 1);
 	auto cached   = std::make_shared<CachedImage>();
 	cached->kind  = CachedImage::Kind::StorageTexture;
@@ -2094,6 +2152,7 @@ StorageTextureVulkanImage* TextureCache::FindStorageTexture(CommandBuffer*   com
 	auto* image = static_cast<StorageTextureVulkanImage*>(cached->image);
 	command->RetainResourceUntilFence(cached);
 	m_images.push_back(std::move(cached));
+	RegisterImageLocked(*m_images.back());
 	return image;
 }
 
@@ -2233,6 +2292,9 @@ RenderTextureVulkanImage* TextureCache::FindRenderTarget(CommandBuffer*
 				break;
 			case CachedImage::Kind::VideoOut: break;
 		}
+		if (overlap == RenderTargetOverlap::None) {
+			continue;
+		}
 		bool supported = false;
 		switch (overlap) {
 			case RenderTargetOverlap::RetireSampled:
@@ -2328,6 +2390,7 @@ RenderTextureVulkanImage* TextureCache::FindRenderTarget(CommandBuffer*
 	auto* image = static_cast<RenderTextureVulkanImage*>(cached->image);
 	command->RetainResourceUntilFence(cached);
 	m_images.push_back(std::move(cached));
+	RegisterImageLocked(*m_images.back());
 	return image;
 }
 
@@ -2649,6 +2712,7 @@ DepthStencilVulkanImage* TextureCache::FindDepthTarget(CommandBuffer* command, G
 	auto* image = static_cast<DepthStencilVulkanImage*>(cached->image);
 	command->RetainResourceUntilFence(cached);
 	m_images.push_back(std::move(cached));
+	RegisterImageLocked(*m_images.back());
 	return image;
 }
 
@@ -2715,6 +2779,7 @@ TextureCache::RegisterVideoOutSurfaces(GraphicContext*                  ctx,
 		}
 		result.push_back(static_cast<VideoOutVulkanImage*>(cached->image));
 		m_images.push_back(std::move(cached));
+		RegisterImageLocked(*m_images.back());
 	}
 	return result;
 }
@@ -2813,7 +2878,7 @@ void TextureCache::UnregisterVideoOutSurfaces(const std::vector<VideoOutVulkanIm
 			m_memory_tracker.UnmarkRegionAsGpuModified(cached->Address(), cached->Size());
 			cached->gpu_modified = false;
 		}
-		m_memory_tracker.UntrackMemory(cached->Address(), cached->Size());
+		UnregisterImageLocked(*cached, true);
 	}
 	for (auto* image: images) {
 		auto it = std::find_if(m_images.begin(), m_images.end(),
@@ -3485,8 +3550,8 @@ RenderTextureVulkanImage* TextureCache::FindRenderTargetByRange(CommandBuffer* c
 		     vaddr, size);
 	}
 	FaultSafeTextureLock         lock(this, m_lock);
-	std::shared_ptr<CachedImage> found;
-	for (auto& cached: m_images) {
+	CachedImage* found = nullptr;
+	for (auto* cached: FindImagesInRegionLocked(vaddr, size, false)) {
 		if (cached->kind != CachedImage::Kind::RenderTarget ||
 		    !ImageRangeOverlaps(vaddr, size, cached->Address(), cached->Size())) {
 			continue;
@@ -3503,14 +3568,19 @@ RenderTextureVulkanImage* TextureCache::FindRenderTargetByRange(CommandBuffer* c
 			EXIT("TextureCache: incompatible or ambiguous render-target range, addr=0x%016" PRIx64
 			     " size=0x%016" PRIx64 " cached=0x%016" PRIx64 "+0x%016" PRIx64 " previous=%p\n",
 			     vaddr, size, cached->Address(), cached->Size(),
-			     static_cast<const void*>(found.get()));
+			     static_cast<const void*>(found));
 		}
 		found = cached;
 	}
 	if (found == nullptr) {
 		return nullptr;
 	}
-	command->RetainResourceUntilFence(found);
+	const auto owner = std::find_if(m_images.begin(), m_images.end(),
+	                                [found](const auto& image) { return image.get() == found; });
+	if (owner == m_images.end()) {
+		EXIT("TextureCache: page-table render target has no cache owner\n");
+	}
+	command->RetainResourceUntilFence(*owner);
 	return static_cast<RenderTextureVulkanImage*>(found->image);
 }
 
@@ -3522,7 +3592,7 @@ bool TextureCache::HasGpuTargetPageOverlap(uint64_t vaddr, uint64_t size) {
 		     vaddr, size);
 	}
 	FaultSafeTextureLock lock(this, m_lock);
-	for (const auto& cached: m_images) {
+	for (const auto* cached: FindImagesInRegionLocked(vaddr, size, true)) {
 		if (cached->kind != CachedImage::Kind::Texture &&
 		    cached->OverlapsRange(vaddr, size, true)) {
 			return true;
@@ -3539,12 +3609,7 @@ bool TextureCache::HasPageOverlap(uint64_t vaddr, uint64_t size) {
 		     vaddr, size);
 	}
 	FaultSafeTextureLock lock(this, m_lock);
-	for (const auto& cached: m_images) {
-		if (cached->OverlapsRange(vaddr, size, true)) {
-			return true;
-		}
-	}
-	return false;
+	return !FindImagesInRegionLocked(vaddr, size, true).empty();
 }
 
 bool TextureCache::HasRangeOverlap(uint64_t vaddr, uint64_t size) {
@@ -3555,12 +3620,7 @@ bool TextureCache::HasRangeOverlap(uint64_t vaddr, uint64_t size) {
 		     vaddr, size);
 	}
 	FaultSafeTextureLock lock(this, m_lock);
-	for (const auto& cached: m_images) {
-		if (cached->OverlapsRange(vaddr, size, false)) {
-			return true;
-		}
-	}
-	return false;
+	return !FindImagesInRegionLocked(vaddr, size, false).empty();
 }
 
 bool TextureCache::HasGpuModifiedRangeOverlap(uint64_t vaddr, uint64_t size) {
@@ -3571,7 +3631,7 @@ bool TextureCache::HasGpuModifiedRangeOverlap(uint64_t vaddr, uint64_t size) {
 		     vaddr, size);
 	}
 	FaultSafeTextureLock lock(this, m_lock);
-	for (const auto& cached: m_images) {
+	for (const auto* cached: FindImagesInRegionLocked(vaddr, size, false)) {
 		if (cached->gpu_modified && cached->OverlapsRange(vaddr, size, false)) {
 			return true;
 		}
@@ -3997,14 +4057,6 @@ void TextureCache::UnmapMemory(uint64_t vaddr, uint64_t size) {
 		}
 		cached->gpu_modified = false;
 	}
-	for (auto& cached: m_images) {
-		if (!cached->OverlapsRange(vaddr, size, false)) {
-			continue;
-		}
-		for (uint32_t i = 0; i < cached->RangeCount(); i++) {
-			m_memory_tracker.UntrackMemory(cached->Address(i), cached->Size(i));
-		}
-	}
 	for (auto it = m_surface_metas.begin(); it != m_surface_metas.end();) {
 		const bool allocation_unmapped =
 		    ImageRangeOverlaps(vaddr, size, it->first, it->second.size);
@@ -4034,6 +4086,7 @@ void TextureCache::UnmapMemory(uint64_t vaddr, uint64_t size) {
 			++it;
 			continue;
 		}
+		UnregisterImageLocked(**it, false);
 		it = m_images.erase(it);
 	}
 }

src/graphics/host_gpu/renderer/textureCache.h

@@ -6,6 +6,7 @@
 #include "common/threads.h"
 #include "graphics/host_gpu/memoryTracker.h"
 #include "graphics/host_gpu/renderer/imageInfo.h"
+#include "graphics/host_gpu/renderer/multiLevelPageTable.h"
 #include "graphics/host_gpu/renderer/tiler.h"
 
 #include <array>
@@ -124,6 +125,7 @@ public:
 
 private:
 	struct CachedImage;
+	using ImageOwnerIndex = MultiRangePageOwnerIndex<CachedImage*>;
 	struct ReadbackWorker;
 	struct MetaDataInfo {
 		uint64_t size         = 0;
@@ -135,13 +137,16 @@ private:
 	[[nodiscard]] VkImageView  GetImageView(GraphicContext* ctx, VulkanImage* image,
 	                                        const ImageViewInfo& info);
 	[[nodiscard]] bool         HasMetaOverlapLocked(uint64_t vaddr, uint64_t size) const;
-	[[nodiscard]] CachedImage* FindGpuReadbackPageCandidateLocked(uint64_t vaddr,
-	                                                              uint64_t size) const;
+	[[nodiscard]] CachedImage* FindGpuReadbackPageCandidateLocked(uint64_t vaddr, uint64_t size);
 	void                       RequireNoMetaOverlapLocked(uint64_t vaddr, uint64_t size) const;
 	void                       MarkSampledAliasesCpuDirtyLocked(uint64_t vaddr, uint64_t size);
 	void RetireSampledTargetAliases(GraphicContext* ctx, const ImageInfo& requested);
-	void RetireStoragePageNeighbors(GraphicContext* ctx, const ImageInfo& requested);
+	void ResolveStorageImageOverlaps(GraphicContext* ctx, const ImageInfo& requested);
 	void RetireStorageDepthAliasLocked(GraphicContext* ctx, const ImageInfo& requested);
+	void RegisterImageLocked(CachedImage& image);
+	void UnregisterImageLocked(CachedImage& image, bool release_tracking);
+	[[nodiscard]] std::vector<CachedImage*>
+	FindImagesInRegionLocked(uint64_t vaddr, uint64_t size, bool page_overlap);
 	void RequireRetirementIsolation(const std::vector<CachedImage*>& retire, const char* operation,
 	                                uint64_t address, uint64_t size) const;
 	void RetireImages(const std::vector<CachedImage*>& retire,
@@ -165,6 +170,7 @@ private:
 	BufferCache&                              m_buffer_cache;
 	ResourceMutex&                            m_resource_mutex;
 	std::vector<std::shared_ptr<CachedImage>> m_images;
+	ImageOwnerIndex                           m_image_owner_index;
 	std::map<uint64_t, MetaDataInfo>          m_surface_metas;
 	std::unique_ptr<ReadbackWorker>           m_readback;
 	std::vector<uint8_t>                      m_buffer_transition_linear;

tests/ImagePageTableTests.cpp

@@ -0,0 +1,138 @@
+#include "graphics/host_gpu/renderer/multiLevelPageTable.h"
+
+#include <cstdint>
+#include <cstdio>
+#include <cstdlib>
+#include <stdexcept>
+#include <vector>
+
+namespace {
+
+using Owners = std::vector<uint32_t>;
+using Table  = Libs::Graphics::MultiLevelPageTable<Owners>;
+using OwnerIndex = Libs::Graphics::MultiRangePageOwnerIndex<uint32_t>;
+
+void Check(bool value, const char* text) {
+	if (!value) {
+		std::fprintf(stderr, "ImagePageTableTests: failed: %s\n", text);
+		std::abort();
+	}
+}
+
+void TestMultiOwnerAndExactErase() {
+	Table table;
+	auto& owners = table.GetOrCreate(17);
+	owners.push_back(11);
+	owners.push_back(22);
+
+	Check(table.Find(17) != nullptr && table.Find(17)->size() == 2, "both page owners are retained");
+	Check(Libs::Graphics::EraseExact(owners, 11U), "registered owner is erased");
+	Check(owners.size() == 1 && owners.front() == 22, "erasing one owner preserves its neighbor");
+	Check(!Libs::Graphics::EraseExact(owners, 33U), "missing owner is reported without mutation");
+}
+
+void TestCrossBucketRange() {
+	Table::PageRange range{};
+	constexpr uint64_t bucket_boundary = uint64_t{Table::kBucketEntries} << Table::kPageBits;
+	Check(Table::TryGetPageRange(bucket_boundary - 1, 2, range), "cross-bucket range is valid");
+	Check(range.first == Table::kBucketEntries - 1 && range.last_exclusive == Table::kBucketEntries + 1,
+	      "cross-bucket range covers both pages");
+
+	Table table;
+	table[range.first].push_back(1);
+	table[range.last_exclusive - 1].push_back(2);
+	Check(table.AllocatedBucketCount() == 2, "pages across the L1 boundary use distinct sparse buckets");
+}
+
+void TestQueriesDoNotAllocate() {
+	Table table;
+	Check(table.Find(123) == nullptr, "unallocated page query is empty");
+	Check(table.AllocatedBucketCount() == 0, "mutable query does not allocate");
+	const Table& const_table = table;
+	Check(const_table.Find(Table::kPageCount - 1) == nullptr, "const query is empty");
+	Check(table.AllocatedBucketCount() == 0, "const query does not allocate");
+	Check(table.Find(Table::kPageCount) == nullptr, "out-of-range query is empty");
+	Check(table.AllocatedBucketCount() == 0, "out-of-range query does not allocate");
+}
+
+void TestAddressSpaceBoundaries() {
+	Table::PageRange range{};
+	Check(Table::TryGetPageRange(Table::kAddressSpaceSize - 1, 1, range), "last guest byte is valid");
+	Check(range.first == Table::kPageCount - 1 && range.last_exclusive == Table::kPageCount,
+	      "last guest byte maps to the final page");
+	Check(!Table::TryGetPageRange(0, 0, range), "empty ranges are rejected");
+	Check(!Table::TryGetPageRange(Table::kAddressSpaceSize, 1, range), "first out-of-range byte is rejected");
+	Check(!Table::TryGetPageRange(Table::kAddressSpaceSize - 1, 2, range), "crossing the address-space end is rejected");
+	Check(!Table::TryGetPageRange(UINT64_MAX - 1, 4, range), "wrapping input is rejected");
+
+	Table table;
+	table.GetOrCreate(Table::kPageCount - 1).push_back(99);
+	Check(table.Find(Table::kPageCount - 1) != nullptr && table.Find(Table::kPageCount - 1)->front() == 99,
+	      "final page supports allocating and nonallocating access");
+	bool  threw = false;
+	try {
+		(void)table.GetOrCreate(Table::kPageCount);
+	} catch (const std::out_of_range&) {
+		threw = true;
+	}
+	Check(threw, "allocating access hard-fails outside the guest address space");
+}
+
+void TestMultiRangeRegistrationDeduplicatesPages() {
+	OwnerIndex index;
+	// Depth and stencil-like planes overlap tracking pages and share one 1 MiB bucket.
+	Check(index.Register(7, {{0x101000, 0x2800}, {0x102000, 0x3000}}), "multi-range owner registers");
+	Check(index.CoarseMembershipCount(1) == 1, "one owner is inserted once in a shared 1 MiB bucket");
+	Check(index.TrackingMembershipCount(0x102) == 1, "overlapping planes insert one 4 KiB membership");
+	Check(!index.Register(7, {{0x101000, 0x1000}}), "duplicate owner registration hard-fails");
+
+	const auto owners = index.Query(0x100000, 0x10000);
+	Check(owners.size() == 1 && owners.front() == 7, "multi-page query returns an owner once");
+}
+
+void TestSharedPageUnregisterLifecycle() {
+	OwnerIndex index;
+	const std::vector<OwnerIndex::ByteRange> ranges{{0x202000, 0x2000}};
+	Check(index.Register(11, ranges) && index.Register(22, ranges), "two owners register on identical pages");
+	Check(index.CoarseMembershipCount(2) == 2 && index.TrackingMembershipCount(0x202) == 2,
+	      "coarse and tracking pages retain both owners");
+
+	std::vector<OwnerIndex::ByteRange> releases;
+	Check(index.Unregister(11, releases), "first owner unregisters");
+	Check(releases.empty(), "shared tracking pages are not released with one owner remaining");
+	Check(index.Query(0x202000, 1).size() == 1 && index.Query(0x202000, 1).front() == 22,
+	      "unregistering one owner preserves the other");
+	Check(!index.Unregister(11, releases), "missing membership hard-fails without mutation");
+
+	Check(index.Unregister(22, releases), "final owner unregisters");
+	Check(releases.size() == 1 && releases.front().address == 0x202000 && releases.front().size == 0x2000,
+	      "adjacent final-owner tracking pages return one contiguous release");
+}
+
+void TestStrictByteFilteringAndPredicate() {
+	OwnerIndex index;
+	Check(index.Register(31, {{0x300100, 0x100}}), "first byte-disjoint owner registers");
+	Check(index.Register(32, {{0x300800, 0x100}}), "second byte-disjoint owner registers");
+	Check(index.TrackingMembershipCount(0x300) == 2, "byte-disjoint owners share one tracking page");
+	Check(index.Query(0x300400, 0x40).empty(), "page hit without byte overlap is filtered out");
+	const auto page_candidates = index.QueryCandidates(0x300400, 0x40);
+	Check(page_candidates.size() == 2, "fault candidate query retains byte-disjoint owners on the touched page");
+	const auto first = index.Query(0x300180, 0x10);
+	Check(first.size() == 1 && first.front() == 31, "strict byte overlap selects only the matching owner");
+	const auto predicate_filtered = index.Query(0x300000, 0x1000, [](uint32_t owner) { return owner == 32; });
+	Check(predicate_filtered.size() == 1 && predicate_filtered.front() == 32, "supplied predicate filters query owners");
+}
+
+} // namespace
+
+int main() {
+	TestMultiOwnerAndExactErase();
+	TestCrossBucketRange();
+	TestQueriesDoNotAllocate();
+	TestAddressSpaceBoundaries();
+	TestMultiRangeRegistrationDeduplicatesPages();
+	TestSharedPageUnregisterLifecycle();
+	TestStrictByteFilteringAndPredicate();
+	std::printf("ImagePageTableTests: all cases passed\n");
+	return 0;
+}

tests/ShaderRecompilerComputeTests.cpp

@@ -11490,6 +11490,49 @@ void CheckImageOverlapResolution() {
           ClassifySampledOverlap(sampled_alias, sampled, false, false) ==
               SampledOverlap::Unsupported,
           "cross-context sampled alias was admitted");
+  ImageInfo dirty_info{};
+  dirty_info.address = 0x500100;
+  dirty_info.size = 0x100;
+  Image dirty_image;
+  dirty_image = dirty_info;
+  dirty_image.InvalidateCpuWrite(0x500300, 0x10);
+  Require("ImageOverlapResolution", "sampled edge-page maybe dirty",
+          dirty_image.IsCpuDirty() && dirty_image.IsMaybeCpuDirty() &&
+              !dirty_image.IsDefinitelyCpuDirty(),
+          "byte-disjoint write sharing a tracker page was not kept maybe-dirty");
+  Require("ImageOverlapResolution", "sampled edge hash required",
+          dirty_image.NeedsMaybeCpuHash(),
+          "collapsed edge tracking did not request a baseline hash");
+  dirty_image.SetMaybeCpuHash(0x1234);
+  Require("ImageOverlapResolution", "sampled unchanged edge hash",
+          !dirty_image.ResolveMaybeCpuHash(0x1234) && !dirty_image.IsCpuDirty() &&
+              dirty_image.IsCpuTrackingComplete(),
+          "unchanged edge bytes forced an image upload");
+  dirty_image.InvalidateCpuWrite(0x500300, 0x10);
+  dirty_image.SetMaybeCpuHash(0x1234);
+  Require("ImageOverlapResolution", "sampled changed edge hash",
+          dirty_image.ResolveMaybeCpuHash(0x5678) &&
+              dirty_image.IsDefinitelyCpuDirty(),
+          "changed edge bytes were not promoted to definitely dirty");
+  dirty_image.RefreshComplete();
+  dirty_image.InvalidateCpuWrite(dirty_info.address, 1);
+  Require("ImageOverlapResolution", "sampled byte overlap dirty",
+          dirty_image.IsCpuDirty() && dirty_image.IsDefinitelyCpuDirty() &&
+              !dirty_image.IsMaybeCpuDirty(),
+          "true byte overlap was not promoted to definitely dirty");
+  ImageInfo two_page_info{};
+  two_page_info.address = 0x600100;
+  two_page_info.size = 0x1800;
+  Image two_page_image;
+  two_page_image = two_page_info;
+  two_page_image.InvalidateCpuWrite(0x600000, 1);
+  Require("ImageOverlapResolution", "sampled head tracking shrink",
+          !two_page_image.IsCpuDirty(),
+          "a single disjoint edge fault prematurely dirtied a two-page image");
+  two_page_image.InvalidateCpuWrite(0x601a00, 1);
+  Require("ImageOverlapResolution", "sampled head-tail collapse",
+          two_page_image.IsMaybeCpuDirty() && two_page_image.NeedsMaybeCpuHash(),
+          "both disjoint edge faults did not collapse logical tracking");
   constexpr uint64_t storage_subrange = 0x649b0100;
   constexpr uint64_t storage_subrange_size = 0x800;
   constexpr uint64_t sampled_backing = 0x649a3000;
@@ -11824,14 +11867,24 @@ void CheckImageOverlapResolution() {
   target.address++;
   Require("ImageOverlapResolution", "render target partial page",
           ClassifyRenderTargetOverlap(sampled, false, true, target) ==
-              RenderTargetOverlap::Unsupported,
-          "partially shared tracker page was retired for a render target");
+              RenderTargetOverlap::RetireSampled,
+          "true byte overlap within a partial tracker page was not retired");
+  target.address = sampled.address + 0x6101;
+  target.size = 0x4000;
+  Require("ImageOverlapResolution", "render target unaligned chance overlap",
+          ClassifyRenderTargetOverlap(sampled, false, true, target) ==
+              RenderTargetOverlap::RetireSampled,
+          "unaligned clean sampled chance overlap was not retired");
   target.address = page_left.address + page_left.size;
   target.size = page_left.size;
   Require("ImageOverlapResolution", "render target shared page",
           ClassifyRenderTargetOverlap(page_left, false, true, target) ==
-              RenderTargetOverlap::Unsupported,
-          "byte-disjoint allocation sharing a tracker page was silently admitted");
+              RenderTargetOverlap::None,
+          "byte-disjoint allocation sharing a tracker page was treated as an alias");
+  Require("ImageOverlapResolution", "sampled render-target shared page",
+          ClassifySampledRenderTargetOverlap(page_left, target, false, true) ==
+              RenderTargetOverlap::None,
+          "reverse byte-disjoint tracker-page relationship was treated as an alias");
   target.address = sampled.address + sampled.size;
   target.size = sampled.size;
   Require("ImageOverlapResolution", "render target adjacent",