kernel: optimize virtual range edits with an ordered map

Co-authored-by: shadowbeat070 <70843316+shadowbeat070@users.noreply.github.com>

src/kernel/memory.cpp

@@ -232,15 +232,15 @@ public:
 	         VirtualRangeType type, const char* name, bool disallow_merge = false) {
 		Common::LockGuard lock(m_mutex);
 
-		if (start == 0 || size == 0) {
+		if (start == 0 || size == 0 || size > UINT64_MAX - start) {
 			return false;
 		}
 		auto position = LowerBound(start);
 		if ((position != m_ranges.end() &&
-		     VirtualRangesOverlap(start, size, position->start, position->size)) ||
+		     VirtualRangesOverlap(start, size, position->second.start, position->second.size)) ||
 		    (position != m_ranges.begin() &&
-		     VirtualRangesOverlap(start, size, std::prev(position)->start,
+		     VirtualRangesOverlap(start, size, std::prev(position)->second.start,
-		                          std::prev(position)->size))) {
+		                          std::prev(position)->second.size))) {
 			return false;
 		}
 
@@ -253,23 +253,21 @@ public:
 		r.type           = type;
 		r.disallow_merge = disallow_merge;
 		CopyVirtualRangeName(r.name, name);
-		const auto index = static_cast<size_t>(position - m_ranges.begin());
+		auto inserted = m_ranges.emplace_hint(position, start, r);
-		m_ranges.insert(position, r);
+		UpdatePageTableEntries(inserted->second, true);
-		MergeAroundUnlocked(index);
+		MergeAroundUnlocked(inserted);
 		return true;
 	}
 
 	bool Remove(uint64_t start, uint64_t size) {
 		Common::LockGuard lock(m_mutex);
 
-		auto position = LowerBound(start);
+		auto position = m_ranges.find(start);
-		if (position != m_ranges.end() && position->start == start && position->size == size) {
+		if (position != m_ranges.end() && position->second.size == size) {
-			m_ranges.erase(position);
+			EraseUnlocked(position);
 			return true;
 		}
-		auto removed = RemoveUnlocked(start, size);
+		return RemoveUnlocked(start, size);
-		MergeUnlocked();
-		return removed;
 	}
 
 	bool HasOverlap(uint64_t start, uint64_t size) {
@@ -293,40 +291,31 @@ public:
 	bool ReleaseReserved(uint64_t start, uint64_t size) {
 		Common::LockGuard lock(m_mutex);
 
-		for (size_t index = 0; index < m_ranges.size(); index++) {
+		auto position = m_ranges.find(start);
-			auto& r = m_ranges[index];
+		if (position != m_ranges.end() && position->second.size == size &&
-			if (r.start == start && r.size == size && IsReservedRangeType(r.type)) {
+		    IsReservedRangeType(position->second.type)) {
-				m_ranges.erase(m_ranges.begin() + static_cast<std::ptrdiff_t>(index));
+			EraseUnlocked(position);
-				return true;
-			}
 		}
 		return true;
 	}
 
-	bool ReplaceSpan(uint64_t start, uint64_t size, VirtualRangeType expected_type,
+	bool ReplaceSpan(uint64_t start, uint64_t size, VirtualRangeType expected_type, uint64_t offset,
-	                 uint64_t offset, int protection, int memory_type, VirtualRangeType type,
+	                 int protection, int memory_type, VirtualRangeType type, const char* name,
-	                 const char* name, bool disallow_merge = false) {
+	                 bool disallow_merge = false) {
 		Common::LockGuard lock(m_mutex);
 
 		if (start == 0 || size == 0 || size > UINT64_MAX - start) {
 			return false;
 		}
 
-		auto current = start;
+		auto       current = start;
-		const auto end = start + size;
+		const auto end     = start + size;
-		while (current < end) {
+		for (auto candidate = FindContaining(start); current < end; ++candidate) {
-			const Range* candidate = nullptr;
+			if (candidate == m_ranges.end() || candidate->second.start > current ||
-			for (const auto& r: m_ranges) {
+			    candidate->second.type != expected_type) {
-				if (r.type == expected_type && current >= r.start &&
-				    current < End(r.start, r.size)) {
-					candidate = &r;
-					break;
-				}
-			}
-			if (candidate == nullptr) {
 				return false;
 			}
-			current = std::min(end, End(candidate->start, candidate->size));
+			current = std::min(end, End(candidate->second.start, candidate->second.size));
 		}
 
 #if defined(KYTY_VIRTUAL_MEMORY_ALLOCATION_TESTS)
@@ -344,21 +333,27 @@ public:
 		replacement.type           = type;
 		replacement.disallow_merge = disallow_merge;
 		CopyVirtualRangeName(replacement.name, name);
-		RemoveUnlocked(start, size);
+		// Validation precedes all mutations. Keep the first node for the replacement so
-		auto position = LowerBound(start);
+		// replacing an existing arena does not allocate another tree node.
-		const auto index = static_cast<size_t>(position - m_ranges.begin());
+		auto first = SplitAtUnlocked(start);
-		m_ranges.insert(position, replacement);
+		auto last  = SplitAtUnlocked(end);
-		MergeAroundUnlocked(index);
+		for (auto it = std::next(first); it != last;) {
+			it = EraseUnlocked(it);
+		}
+		UpdatePageTableEntries(first->second, false);
+		first->second = replacement;
+		UpdatePageTableEntries(first->second, true);
+		MergeAroundUnlocked(first);
 		return true;
 	}
 
 	void Rename(uint64_t start, uint64_t size, const char* name) {
 		Common::LockGuard lock(m_mutex);
 
-		auto position = LowerBound(start);
+		auto position = m_ranges.find(start);
-		if (position != m_ranges.end() && position->start == start && position->size == size) {
+		if (position != m_ranges.end() && position->second.size == size) {
-			CopyVirtualRangeName(position->name, name);
+			CopyVirtualRangeName(position->second.name, name);
-			MergeAroundUnlocked(static_cast<size_t>(position - m_ranges.begin()));
+			MergeAroundUnlocked(position);
 			return;
 		}
 		EditUnlocked(start, size, [name](Range* r) { CopyVirtualRangeName(r->name, name); });
@@ -375,13 +370,11 @@ public:
 
 		Common::LockGuard lock(m_mutex);
 
-		auto next = std::upper_bound(
+		auto next = m_ranges.upper_bound(addr);
-		    m_ranges.begin(), m_ranges.end(), addr,
-		    [](uint64_t value, const Range& range) { return value < range.start; });
 		if (next != m_ranges.begin()) {
 			auto current = std::prev(next);
-			if (addr < End(current->start, current->size)) {
+			if (addr < End(current->second.start, current->second.size)) {
-				*out = *current;
+				*out = current->second;
 				return true;
 			}
 		}
@@ -389,7 +382,7 @@ public:
 			return false;
 		}
 
-		*out = *next;
+		*out = next->second;
 		return true;
 	}
 
@@ -404,11 +397,9 @@ public:
 
 		const auto end     = start + size;
 		auto       current = start;
-		for (const auto& range: m_ranges) {
+		for (auto it = FindContaining(start); it != m_ranges.end(); ++it) {
-			const auto range_end = End(range.start, range.size);
+			const auto& range     = it->second;
-			if (range_end <= current) {
+			const auto  range_end = End(range.start, range.size);
-				continue;
-			}
 			if (range.start > current) {
 				break;
 			}
@@ -437,27 +428,19 @@ public:
 			return 0;
 		}
 
-		auto vma = std::upper_bound(
+		auto vma = FindContaining(virtual_addr);
-		    m_ranges.begin(), m_ranges.end(), virtual_addr,
+		if (vma == m_ranges.end() || !IsCommittedRangeType(vma->second.type)) {
-		    [](uint64_t value, const Range& range) { return value < range.start; });
-		if (vma == m_ranges.begin()) {
-			return 0;
-		}
-		--vma;
-
-		const auto vma_end = End(vma->start, vma->size);
-		if (virtual_addr < vma->start || virtual_addr >= vma_end ||
-		    !IsCommittedRangeType(vma->type)) {
 			return 0;
 		}
+		const auto vma_end = End(vma->second.start, vma->second.size);
 
 		uint64_t clamped_size = std::min(size, vma_end - virtual_addr);
 		uint64_t expected     = virtual_addr + clamped_size;
 		++vma;
 
-		while (vma != m_ranges.end() && vma->start == expected && IsCommittedRangeType(vma->type) &&
+		while (vma != m_ranges.end() && vma->second.start == expected &&
-		       clamped_size < size) {
+		       IsCommittedRangeType(vma->second.type) && clamped_size < size) {
-			const auto chunk = std::min(size - clamped_size, vma->size);
+			const auto chunk = std::min(size - clamped_size, vma->second.size);
 			clamped_size += chunk;
 			expected += chunk;
 			++vma;
@@ -468,27 +451,7 @@ public:
 
 	uint64_t CountPageTableEntries(bool gpu) {
 		Common::LockGuard lock(m_mutex);
-
+		return gpu ? m_gpu_page_table_entries : m_cpu_page_table_entries;
-		uint64_t used = 0;
-		for (const auto& r: m_ranges) {
-			if (!IsCommittedRangeType(r.type) || r.size == 0) {
-				continue;
-			}
-
-			const bool has_cpu_access =
-			    (r.protection & (PROT_CPU_READ | PROT_CPU_WRITE | PROT_CPU_EXEC)) != 0;
-			const bool has_gpu_access = (r.protection & (PROT_GPU_READ | PROT_GPU_WRITE)) != 0;
-			if (gpu ? !has_gpu_access : !has_cpu_access) {
-				continue;
-			}
-
-			const auto end         = End(r.start, r.size);
-			const auto first_entry = r.start / PAGE_TABLE_GRANULARITY;
-			const auto last_entry  = (end - 1) / PAGE_TABLE_GRANULARITY;
-			used += last_entry - first_entry + 1;
-		}
-
-		return used;
 	}
 
 private:
@@ -507,51 +470,112 @@ private:
 		       std::strncmp(left.name, right.name, KERNEL_MAXIMUM_NAME_LENGTH) == 0;
 	}
 
-	static void AddPiece(std::vector<Range>* ranges, const Range& source, uint64_t start,
+	// Keys are range starts; ranges are nonempty, non-overlapping and locally coalesced.
-	                     uint64_t end) {
+	// A tree avoids shifting unrelated ranges when streaming edits split or remove an arena.
-		EXIT_IF(ranges == nullptr);
+	using RangeMap = std::map<uint64_t, Range>;
+	using Iterator = RangeMap::iterator;
 
-		if (end <= start) {
+	Iterator LowerBound(uint64_t start) { return m_ranges.lower_bound(start); }
-			return;
-		}
 
-		Range piece = source;
+	Iterator FindContaining(uint64_t address) {
-		piece.start = start;
+		auto next = m_ranges.upper_bound(address);
-		piece.size  = end - start;
+		if (next == m_ranges.begin()) {
-		if (piece.type == VirtualRangeType::Direct) {
+			return m_ranges.end();
-			piece.offset += start - source.start;
 		}
-		ranges->push_back(piece);
+		auto current = std::prev(next);
+		return address < End(current->second.start, current->second.size) ? current
+		                                                                  : m_ranges.end();
 	}
 
-	std::vector<Range>::iterator LowerBound(uint64_t start) {
+	static bool CanMerge(const Range& left, const Range& right) {
-		return std::lower_bound(
+		return End(left.start, left.size) == right.start && SameMergeKey(left, right);
-		    m_ranges.begin(), m_ranges.end(), start,
-		    [](const Range& range, uint64_t value) { return range.start < value; });
 	}
 
-	void MergeAroundUnlocked(size_t index) {
+	void UpdatePageTableEntries(const Range& range, bool add) {
-		if (index >= m_ranges.size()) {
+		if (!IsCommittedRangeType(range.type)) {
 			return;
 		}
-		if (index != 0) {
+		// Preserve the statistics API's per-range accounting: splitting within a page-table
-			auto& previous = m_ranges[index - 1];
+		// entry counts it for each fragment, and merging removes that duplicate contribution.
-			auto& current  = m_ranges[index];
+		const auto entries = (End(range.start, range.size) - 1) / PAGE_TABLE_GRANULARITY -
-			if (End(previous.start, previous.size) == current.start &&
+		                     range.start / PAGE_TABLE_GRANULARITY + 1;
-			    SameMergeKey(previous, current)) {
+		if ((range.protection & (PROT_CPU_READ | PROT_CPU_WRITE | PROT_CPU_EXEC)) != 0) {
-				previous.size += current.size;
+			m_cpu_page_table_entries =
-				m_ranges.erase(m_ranges.begin() + static_cast<std::ptrdiff_t>(index));
+			    add ? m_cpu_page_table_entries + entries : m_cpu_page_table_entries - entries;
-				index--;
+		}
+		if ((range.protection & (PROT_GPU_READ | PROT_GPU_WRITE)) != 0) {
+			m_gpu_page_table_entries =
+			    add ? m_gpu_page_table_entries + entries : m_gpu_page_table_entries - entries;
+		}
+	}
+
+	Iterator EraseUnlocked(Iterator position) {
+		UpdatePageTableEntries(position->second, false);
+		return m_ranges.erase(position);
+	}
+
+	Iterator MergeNextUnlocked(Iterator left, Iterator right) {
+		UpdatePageTableEntries(left->second, false);
+		left->second.size += right->second.size;
+		auto next = EraseUnlocked(right);
+		UpdatePageTableEntries(left->second, true);
+		return next;
+	}
+
+	void MergeAroundUnlocked(Iterator current) {
+		if (current != m_ranges.begin()) {
+			auto previous = std::prev(current);
+			if (CanMerge(previous->second, current->second)) {
+				MergeNextUnlocked(previous, current);
+				current = previous;
 			}
 		}
-		while (index + 1 < m_ranges.size()) {
+		for (auto next = std::next(current); next != m_ranges.end();) {
-			auto& current = m_ranges[index];
+			if (!CanMerge(current->second, next->second)) {
-			auto& next    = m_ranges[index + 1];
-			if (End(current.start, current.size) != next.start || !SameMergeKey(current, next)) {
 				break;
 			}
-			current.size += next.size;
+			next = MergeNextUnlocked(current, next);
-			m_ranges.erase(m_ranges.begin() + static_cast<std::ptrdiff_t>(index + 1));
+		}
+	}
+
+	Iterator SplitAtUnlocked(uint64_t address) {
+		auto position = LowerBound(address);
+		if (position != m_ranges.begin()) {
+			auto&      previous     = std::prev(position)->second;
+			const auto previous_end = End(previous.start, previous.size);
+			if (previous.start < address && address < previous_end) {
+				Range right = previous;
+				right.start = address;
+				right.size  = previous_end - address;
+				if (right.type == VirtualRangeType::Direct) {
+					right.offset += address - previous.start;
+				}
+				position = m_ranges.emplace_hint(position, address, right);
+				UpdatePageTableEntries(previous, false);
+				previous.size = address - previous.start;
+				UpdatePageTableEntries(previous, true);
+				UpdatePageTableEntries(right, true);
+			}
+		}
+		return position;
+	}
+
+	void MergeSpanUnlocked(Iterator first, Iterator last) {
+		if (first == last) {
+			return;
+		}
+		// Include both neighbors. The right neighbor itself can be erased by a merge,
+		// so the sentinel must be the node after it, which this loop never touches.
+		const auto stop = last == m_ranges.end() ? last : std::next(last);
+		if (first != m_ranges.begin()) {
+			--first;
+		}
+		for (auto next = std::next(first); next != stop;) {
+			if (CanMerge(first->second, next->second)) {
+				next = MergeNextUnlocked(first, next);
+			} else {
+				first = next++;
+			}
 		}
 	}
 
@@ -560,102 +584,51 @@ private:
 		if (size == 0) {
 			return;
 		}
-
+		auto first = SplitAtUnlocked(start);
-		std::vector<Range> out;
+		auto last  = SplitAtUnlocked(End(start, size));
-		auto               edit_end = End(start, size);
+		for (auto it = first; it != last; ++it) {
-
+			UpdatePageTableEntries(it->second, false);
-		for (const auto& r: m_ranges) {
+			edit(&it->second);
-			auto r_end = End(r.start, r.size);
+			UpdatePageTableEntries(it->second, true);
-			if (!VirtualRangesOverlap(start, size, r.start, r.size)) {
-				out.push_back(r);
-				continue;
-			}
-
-			auto mid_start = std::max(start, r.start);
-			auto mid_end   = std::min(edit_end, r_end);
-
-			AddPiece(&out, r, r.start, mid_start);
-
-			Range mid = r;
-			mid.start = mid_start;
-			mid.size  = mid_end - mid_start;
-			if (mid.type == VirtualRangeType::Direct) {
-				mid.offset += mid_start - r.start;
-			}
-			edit(&mid);
-			out.push_back(mid);
-
-			AddPiece(&out, r, mid_end, r_end);
 		}
-
+		MergeSpanUnlocked(first, last);
-		m_ranges = out;
-		MergeUnlocked();
 	}
 
 	bool RemoveUnlocked(uint64_t start, uint64_t size) {
 		if (size == 0) {
 			return false;
 		}
-
+		auto first = SplitAtUnlocked(start);
-		std::vector<Range> out;
+		auto last  = SplitAtUnlocked(End(start, size));
-		bool               removed = false;
+		if (first == last) {
-		auto               rem_end = End(start, size);
+			return false;
-
-		for (const auto& r: m_ranges) {
-			auto r_end = End(r.start, r.size);
-			if (!VirtualRangesOverlap(start, size, r.start, r.size)) {
-				out.push_back(r);
-				continue;
-			}
-
-			removed = true;
-			AddPiece(&out, r, r.start, std::max(start, r.start));
-			AddPiece(&out, r, std::min(rem_end, r_end), r_end);
 		}
-
+		while (first != last) {
-		m_ranges = out;
+			first = EraseUnlocked(first);
-		return removed;
-	}
-
-	void MergeUnlocked() {
-		if (m_ranges.size() < 2) {
-			return;
 		}
-
+		// Removal leaves an address gap, so it cannot make surviving ranges mergeable.
-		std::sort(m_ranges.begin(), m_ranges.end(),
+		return true;
-		          [](const Range& left, const Range& right) { return left.start < right.start; });
-
-		std::vector<Range> merged;
-		for (const auto& r: m_ranges) {
-			if (!merged.empty()) {
-				auto& last = merged[merged.size() - 1];
-				if (End(last.start, last.size) == r.start && SameMergeKey(last, r)) {
-					last.size += r.size;
-					continue;
-				}
-			}
-			merged.push_back(r);
-		}
-		m_ranges = merged;
 	}
 
 	Range* FindOverlap(uint64_t start, uint64_t size) {
 		auto position = LowerBound(start);
 		if (position != m_ranges.end() &&
-		    VirtualRangesOverlap(start, size, position->start, position->size)) {
+		    VirtualRangesOverlap(start, size, position->second.start, position->second.size)) {
-			return &*position;
+			return &position->second;
 		}
 		if (position != m_ranges.begin()) {
 			auto previous = std::prev(position);
-			if (VirtualRangesOverlap(start, size, previous->start, previous->size)) {
+			if (VirtualRangesOverlap(start, size, previous->second.start, previous->second.size)) {
-				return &*previous;
+				return &previous->second;
 			}
 		}
 		return nullptr;
 	}
 
-	std::vector<Range> m_ranges;
+	RangeMap      m_ranges;
-	Common::Mutex      m_mutex;
+	Common::Mutex m_mutex;
+	uint64_t      m_cpu_page_table_entries = 0;
+	uint64_t      m_gpu_page_table_entries = 0;
 };
 
 #if defined(KYTY_VIRTUAL_MEMORY_ALLOCATION_TESTS)

tests/VirtualMemoryAllocationTests.cpp

@@ -926,6 +926,19 @@ void TestFlexibleNoCoalescePreservesBoundaries() {
 	            0, 1, "no_coalesce");
 	ExpectRange(test, Query(test, base + SceKernelPageSize), base + SceKernelPageSize,
 	            base + SceKernelPageSize * 2, SceKernelProtCpuRw, 1, 0, 0, 1, "no_coalesce");
+	CheckOk(test,
+	        Libs::LibKernel::Memory::KernelMprotect(reserve, SceKernelPageSize * 2,
+	                                                SceKernelProtCpuRead),
+	        "KernelMprotect(NO_COALESCE span)");
+	CheckOk(test,
+	        Libs::LibKernel::Memory::KernelSetVirtualRangeName(reserve, SceKernelPageSize * 2,
+	                                                           "renamed_no_coalesce"),
+	        "KernelSetVirtualRangeName(NO_COALESCE span)");
+	for (uint64_t i = 0; i < 2; ++i) {
+		ExpectRange(test, Query(test, base + i * SceKernelPageSize), base + i * SceKernelPageSize,
+		            base + (i + 1) * SceKernelPageSize, SceKernelProtCpuRead, 1, 0, 0, 1,
+		            "renamed_no_coalesce");
+	}
 
 	CheckOk(test, Libs::LibKernel::Memory::KernelMunmap(base, SceKernelPageSize * 2),
 	        "KernelMunmap");
@@ -1262,6 +1275,179 @@ void TestPartialFlexibleMunmapAndFindNext() {
 	std::printf("[host]    %-48s ok\n", test);
 }
 
+void TestFlexibleRangeEditsCoalesce() {
+	using namespace Libs::LibKernel::Memory;
+	const char*        test     = "FlexibleRangeEditsCoalesce";
+	constexpr uint64_t page     = SceKernelPageSize;
+	constexpr uint64_t pages    = 129;
+	constexpr uint64_t size     = pages * page;
+	const auto         baseline = AvailableFlexibleMemory(test);
+	const auto         base     = MapNamedFlexible(test, size, SceKernelProtCpuRw, "dense_ranges");
+
+	// Alternate protections fragment one allocation into many independently queryable ranges.
+	for (uint64_t i = 1; i < pages; i += 2) {
+		CheckOk(
+		    test,
+		    KernelMprotect(reinterpret_cast<void*>(base + i * page), page, SceKernelProtCpuRead),
+		    "KernelMprotect(fragment)");
+	}
+	for (uint64_t i = 0; i < pages; ++i) {
+		ExpectRange(test, Query(test, base + i * page + page - 1), base + i * page,
+		            base + (i + 1) * page, i % 2 == 0 ? SceKernelProtCpuRw : SceKernelProtCpuRead,
+		            1, 0, 0, 1, "dense_ranges");
+	}
+	CheckOk(test, KernelMprotect(reinterpret_cast<void*>(base), size, SceKernelProtCpuRw),
+	        "KernelMprotect(coalesce all fragments)");
+	ExpectRange(test, Query(test, base + size - 1), base, base + size, SceKernelProtCpuRw, 1, 0, 0,
+	            1, "dense_ranges");
+
+	// Renaming the interior must split both boundaries, then restoring it must join both sides.
+	CheckOk(test,
+	        KernelSetVirtualRangeName(reinterpret_cast<void*>(base + page), size - 2 * page,
+	                                  "renamed_interior"),
+	        "KernelSetVirtualRangeName(interior)");
+	ExpectRange(test, Query(test, base), base, base + page, SceKernelProtCpuRw, 1, 0, 0, 1,
+	            "dense_ranges");
+	ExpectRange(test, Query(test, base + page), base + page, base + size - page, SceKernelProtCpuRw,
+	            1, 0, 0, 1, "renamed_interior");
+	ExpectRange(test, Query(test, base + size - page), base + size - page, base + size,
+	            SceKernelProtCpuRw, 1, 0, 0, 1, "dense_ranges");
+	CheckOk(test,
+	        KernelSetVirtualRangeName(reinterpret_cast<void*>(base + page), size - 2 * page,
+	                                  "dense_ranges"),
+	        "KernelSetVirtualRangeName(restore interior)");
+	ExpectRange(test, Query(test, base), base, base + size, SceKernelProtCpuRw, 1, 0, 0, 1,
+	            "dense_ranges");
+
+	for (uint64_t i = 1; i < pages; i += 2) {
+		CheckOk(test,
+		        KernelSetVirtualRangeName(reinterpret_cast<void*>(base + i * page), page,
+		                                  "fragment_name"),
+		        "KernelSetVirtualRangeName(fragment)");
+	}
+	CheckOk(test, KernelClearVirtualRangeName(reinterpret_cast<void*>(base), size),
+	        "KernelClearVirtualRangeName(coalesce all fragments)");
+	ExpectRange(test, Query(test, base + size - 1), base, base + size, SceKernelProtCpuRw, 1, 0, 0,
+	            1, "");
+
+	constexpr uint64_t middle = pages / 2 * page;
+	CheckOk(test, KernelMunmap(base + middle, page), "KernelMunmap(create hole)");
+	CheckOk(test, KernelSetVirtualRangeName(reinterpret_cast<void*>(base), size, "across_hole"),
+	        "KernelSetVirtualRangeName(across hole)");
+	ExpectRange(test, Query(test, base + middle - 1), base, base + middle, SceKernelProtCpuRw, 1, 0,
+	            0, 1, "across_hole");
+	ExpectUnmapped(test, base + middle);
+	ExpectRange(test, Query(test, base + middle, SceKernelVqFindNext), base + middle + page,
+	            base + size, SceKernelProtCpuRw, 1, 0, 0, 1, "across_hole");
+	Check(test, ClampRangeSize(base + middle - page / 2, page) == page / 2,
+	      "range clamp crossed the hole after renaming");
+	CheckOk(test, KernelMunmap(base, middle), "KernelMunmap(left cleanup)");
+	CheckOk(test, KernelMunmap(base + middle + page, size - middle - page),
+	        "KernelMunmap(right cleanup)");
+	Check(test, AvailableFlexibleMemory(test) == baseline,
+	      "range edit cleanup did not restore flexible capacity");
+	std::printf("[host]    %-48s ok\n", test);
+}
+
+void TestPageTableStatsTrackRangeEdits() {
+	using namespace Libs::LibKernel::Memory;
+	const char*        test       = "PageTableStatsTrackRangeEdits";
+	constexpr uint64_t page       = SceKernelPageSize;
+	constexpr uint64_t table_span = 0x200000;
+	constexpr uint64_t size       = table_span + 3 * page;
+	constexpr int      gpu_read   = 0x10;
+	constexpr int      gpu_write  = 0x20;
+	constexpr int      both       = SceKernelProtCpuRw | gpu_read;
+	const auto         read_stats = [&] {
+		std::array<int, 4> stats {};
+		CheckOk(test, KernelGetPageTableStats(&stats[0], &stats[1], &stats[2], &stats[3]),
+		        "KernelGetPageTableStats");
+		return stats;
+	};
+	const auto baseline    = read_stats();
+	const auto expect_used = [&](int cpu, int gpu, const char* action) {
+		const auto stats = read_stats();
+		Check(test,
+		      stats[0] == baseline[0] && stats[2] == baseline[2] && baseline[1] - stats[1] == cpu &&
+		          baseline[3] - stats[3] == gpu,
+		      std::string(action) + " changed page-table statistics unexpectedly");
+	};
+	void* address = nullptr;
+	CheckOk(test, KernelReserveVirtualRange(&address, size, 0, table_span),
+	        "KernelReserveVirtualRange(flexible stats)");
+	const auto base = reinterpret_cast<uint64_t>(address);
+	Check(test, base % table_span == 0, "stats reservation is not aligned");
+	expect_used(0, 0, "reservation");
+	CheckOk(test,
+	        KernelMapNamedFlexibleMemory(&address, size, SceKernelProtCpuRw, SceKernelMapFixed,
+	                                     "table_stats"),
+	        "KernelMapNamedFlexibleMemory(stats)");
+	expect_used(2, 0, "flexible mapping across two table entries");
+
+	// The existing ABI estimate counts each fragment's touched 2 MiB entries independently.
+	void* middle = reinterpret_cast<void*>(base + page);
+	CheckOk(test, KernelSetVirtualRangeName(middle, page, "table_stats_middle"),
+	        "KernelSetVirtualRangeName(stats split)");
+	expect_used(4, 0, "name split");
+	CheckOk(test, KernelSetVirtualRangeName(middle, page, "table_stats"),
+	        "KernelSetVirtualRangeName(stats merge)");
+	expect_used(2, 0, "name coalescing");
+	CheckOk(test, KernelMprotect(middle, page, both), "KernelMprotect(stats split)");
+	expect_used(4, 1, "protection split");
+	CheckOk(test, KernelMprotect(address, size, both), "KernelMprotect(stats merge)");
+	expect_used(2, 2, "protection coalescing");
+	CheckOk(test, KernelMprotect(address, size, gpu_write), "KernelMprotect(GPU only)");
+	expect_used(0, 2, "GPU-only protection");
+	CheckOk(test, KernelMprotect(address, size, SceKernelProtCpuRw), "KernelMprotect(CPU only)");
+	expect_used(2, 0, "CPU-only protection");
+	CheckOk(test, KernelMprotect(address, size, 0), "KernelMprotect(no access)");
+	expect_used(0, 0, "no-access protection");
+	CheckOk(test, KernelMprotect(address, size, both), "KernelMprotect(restore stats access)");
+	CheckOk(test, KernelMunmap(base + page, page), "KernelMunmap(stats middle)");
+	expect_used(3, 3, "partial removal");
+	CheckOk(test, KernelMunmap(base, page), "KernelMunmap(stats left)");
+	expect_used(2, 2, "exact removal");
+	CheckOk(test, KernelMunmap(base + 2 * page, size - 2 * page), "KernelMunmap(stats right)");
+	expect_used(0, 0, "flexible cleanup");
+
+	int64_t physical = 0;
+	CheckOk(test,
+	        KernelAllocateDirectMemory(0, KernelGetDirectMemorySize(), 3 * page, page,
+	                                   SceKernelMtypeC, &physical),
+	        "KernelAllocateDirectMemory(stats)");
+	address = nullptr;
+	CheckOk(test, KernelReserveVirtualRange(&address, 3 * page, 0, table_span),
+	        "KernelReserveVirtualRange(direct stats)");
+	const auto direct_base = reinterpret_cast<uint64_t>(address);
+	CheckOk(test,
+	        KernelMapNamedDirectMemory(&address, 3 * page, both, SceKernelMapFixed, physical, page,
+	                                   "direct_table_stats"),
+	        "KernelMapNamedDirectMemory(stats)");
+	expect_used(1, 1, "direct mapping");
+	CheckOk(test, KernelMprotect(reinterpret_cast<void*>(direct_base + page), page, both),
+	        "KernelMprotect(unchanged direct stats subspan)");
+	expect_used(3, 3, "unchanged direct protection split");
+#if defined(__linux__)
+	TestFailNextVirtualRangeReplacement();
+	CheckFailed(test,
+	            KernelMapNamedDirectMemory(&address, 3 * page, SceKernelProtCpuRw,
+	                                       SceKernelMapFixed, physical, page,
+	                                       "rejected_table_stats"),
+	            "KernelMapNamedDirectMemory(stats publication failure)");
+	expect_used(3, 3, "failed direct replacement");
+#endif
+	address = reinterpret_cast<void*>(direct_base);
+	CheckOk(test,
+	        KernelMapNamedDirectMemory(&address, 3 * page, SceKernelProtCpuRw, SceKernelMapFixed,
+	                                   physical, page, "replacement_table_stats"),
+	        "KernelMapNamedDirectMemory(stats replacement)");
+	expect_used(1, 0, "direct replacement");
+	CheckOk(test, KernelReleaseDirectMemory(physical, 3 * page),
+	        "KernelReleaseDirectMemory(stats cleanup)");
+	expect_used(0, 0, "direct cleanup");
+	std::printf("[host]    %-48s ok\n", test);
+}
+
 void TestReserveMapFixedAndNoOverwrite() {
 	const char* test = "ReserveMapFixedAndNoOverwrite";
 	void*       addr = nullptr;
@@ -1505,6 +1691,88 @@ void TestDirectPartialProtectUnmapPreservesNeighbors() {
 	std::printf("[host]    %-48s ok\n", test);
 }
 
+void TestFixedArenaChurnKeepsRangesExact() {
+	using namespace Libs::LibKernel::Memory;
+	const char*        test        = "FixedArenaChurnKeepsRangesExact";
+	constexpr uint64_t page        = SceKernelPageSize;
+	constexpr uint64_t arena_size  = page * 4;
+	constexpr uint64_t arena_count = 8;
+	constexpr uint64_t size        = arena_size * arena_count;
+	int64_t            physical    = 0;
+	CheckOk(test,
+	        KernelAllocateDirectMemory(0, KernelGetDirectMemorySize(), size, page, SceKernelMtypeC,
+	                                   &physical),
+	        "KernelAllocateDirectMemory");
+	void* reservation = nullptr;
+	CheckOk(test, KernelReserveVirtualRange(&reservation, size, 0, page),
+	        "KernelReserveVirtualRange");
+	const auto base      = reinterpret_cast<uint64_t>(reservation);
+	const auto offset    = static_cast<uint64_t>(physical);
+	const auto map_arena = [&](uint64_t i) {
+		void* target = reinterpret_cast<void*>(base + i * arena_size);
+		CheckOk(test,
+		        KernelMapNamedDirectMemory(&target, arena_size, SceKernelProtCpuRw,
+		                                   SceKernelMapFixed, physical + i * arena_size, page,
+		                                   "fixed_arena"),
+		        "KernelMapNamedDirectMemory(arena)");
+		Check(test, reinterpret_cast<uint64_t>(target) == base + i * arena_size,
+		      "fixed arena mapping moved");
+	};
+	for (uint64_t i = 0; i < arena_count; ++i) {
+		map_arena(i);
+	}
+	CheckOk(test, KernelMunmap(base + 2 * arena_size, 4 * arena_size), "KernelMunmap(arenas 2-5)");
+	map_arena(3);
+	for (uint64_t i: {0, 1, 3, 6, 7}) {
+		ExpectRange(test, Query(test, base + i * arena_size), base + i * arena_size,
+		            base + (i + 1) * arena_size, SceKernelProtCpuRw, 0, 1, 0, 1, "fixed_arena",
+		            offset + i * arena_size);
+	}
+	for (uint64_t i: {2, 4, 5}) {
+		ExpectUnmapped(test, base + i * arena_size);
+	}
+	ExpectRange(test, Query(test, base + 4 * arena_size, SceKernelVqFindNext),
+	            base + 6 * arena_size, base + 7 * arena_size, SceKernelProtCpuRw, 0, 1, 0, 1,
+	            "fixed_arena", offset + 6 * arena_size);
+
+	// A span crossing two direct mappings splits each one and preserves its physical offset.
+	CheckOk(
+	    test,
+	    KernelMprotect(reinterpret_cast<void*>(base + 2 * page), 4 * page, SceKernelProtCpuRead),
+	    "KernelMprotect(across direct arenas)");
+	for (uint64_t i = 0; i < 4; ++i) {
+		ExpectRange(test, Query(test, base + i * 2 * page), base + i * 2 * page,
+		            base + (i + 1) * 2 * page,
+		            i == 1 || i == 2 ? SceKernelProtCpuRead : SceKernelProtCpuRw, 0, 1, 0, 1,
+		            "fixed_arena", offset + i * 2 * page);
+	}
+	CheckOk(test, KernelMprotect(reservation, 2 * arena_size, SceKernelProtCpuRw),
+	        "KernelMprotect(restore direct arenas)");
+	CheckOk(test, KernelSetVirtualRangeName(reservation, 2 * arena_size, "renamed_arena"),
+	        "KernelSetVirtualRangeName(direct arenas)");
+	for (uint64_t i = 0; i < 4; ++i) {
+		ExpectRange(test, Query(test, base + i * 2 * page), base + i * 2 * page,
+		            base + (i + 1) * 2 * page, SceKernelProtCpuRw, 0, 1, 0, 1, "renamed_arena",
+		            offset + i * 2 * page);
+	}
+	// Even unchanged protection preserves the new boundary when direct ranges cannot merge.
+	CheckOk(test, KernelMprotect(reinterpret_cast<void*>(base + page), page, SceKernelProtCpuRw),
+	        "KernelMprotect(unchanged direct subspan)");
+	ExpectRange(test, Query(test, base), base, base + page, SceKernelProtCpuRw, 0, 1, 0, 1,
+	            "renamed_arena", offset);
+	ExpectRange(test, Query(test, base + page), base + page, base + 2 * page, SceKernelProtCpuRw, 0,
+	            1, 0, 1, "renamed_arena", offset + page);
+	Check(test, ClampRangeSize(base + page, 2 * arena_size) == 2 * arena_size - page,
+	      "range clamp did not traverse direct fragments up to the hole");
+	CheckOk(test, KernelMunmap(base, 2 * arena_size), "KernelMunmap(arenas 0-1)");
+	CheckOk(test, KernelMunmap(base + 3 * arena_size, arena_size), "KernelMunmap(arena 3)");
+	CheckOk(test, KernelMunmap(base + 6 * arena_size, 2 * arena_size), "KernelMunmap(arenas 6-7)");
+	ExpectUnmapped(test, base);
+	ExpectUnmapped(test, base + size - 1);
+	CheckOk(test, KernelReleaseDirectMemory(physical, size), "KernelReleaseDirectMemory");
+	std::printf("[host]    %-48s ok\n", test);
+}
+
 #if defined(__linux__)
 void TestPartialUnmapPreservesHostPermissions() {
 	const char* test = "PartialUnmapPreservesHostPermissions";
@@ -4509,6 +4777,25 @@ void TestSmallFiberStacksAndMigration() {
 
 int main(int argc, char** argv) {
 	InitSubsystems();
+	if (argc == 2 && std::strcmp(argv[1], "--virtual-ranges-only") == 0) {
+		RunTest(TestFlexibleNoCoalescePreservesBoundaries);
+		RunTest(TestFlexibleRangeEditsCoalesce);
+		RunTest(TestPageTableStatsTrackRangeEdits);
+		RunTest(TestPartialFlexibleMunmapAndFindNext);
+		RunTest(TestReserveMapFixedAndNoOverwrite);
+		RunTest(TestFixedNoOverwriteRejectsReservedRange);
+		RunTest(TestFixedArenaChurnKeepsRangesExact);
+		RunTest(TestDirectMapQueryOffsetAndPartialMunmap);
+		RunTest(TestDirectPartialProtectUnmapPreservesNeighbors);
+#if defined(__linux__)
+		RunTest(TestFixedDirectReplacementPreservesAccess);
+#endif
+		RunTest(TestFixedReserveReplacesPartialDirectMapping);
+		RunTest(TestFixedReserveRangeAddRollbackKeepsPlaceholder);
+		RunTest(TestMemoryPoolMultiRangeDecommit);
+		RunTest(TestProgramMemoryAllocationAndProtection);
+		return g_failed_tests == 0 ? 0 : 1;
+	}
 	if (argc == 2 && std::strcmp(argv[1], "--backing-transfers-only") == 0) {
 		RunTest(TestConcurrentBackingReads);
 		RunTest(TestBackingReadExcludesWritesAndUnmap);
@@ -4585,12 +4872,15 @@ int main(int argc, char** argv) {
 	RunTest(TestRuntimeMemoryOwnerLifecycle);
 	RunTest(TestFlexibleMapQueryAndWholeMunmap);
 	RunTest(TestPartialFlexibleMunmapAndFindNext);
+	RunTest(TestFlexibleRangeEditsCoalesce);
+	RunTest(TestPageTableStatsTrackRangeEdits);
 	RunTest(TestReserveMapFixedAndNoOverwrite);
 	RunTest(TestFixedNoOverwriteRejectsReservedRange);
 	RunTest(TestReleasedReserveCanBeReused);
 	RunTest(TestMunmapAcrossAdjacentFlexibleMappings);
 	RunTest(TestDirectMapQueryOffsetAndPartialMunmap);
 	RunTest(TestDirectPartialProtectUnmapPreservesNeighbors);
+	RunTest(TestFixedArenaChurnKeepsRangesExact);
 #if defined(__linux__)
 	RunTest(TestPartialUnmapPreservesHostPermissions);
 #endif