kernel: coalesce fragmented Windows placeholders before reuse

src/kernel/memoryAddressSpace.inc

@@ -22,6 +22,13 @@ static bool GuestFreeRangeContains(uint64_t base, uint64_t range_size, uint64_t
 	return candidate <= range_end && size <= range_end - candidate;
 }
 
+#if KYTY_PLATFORM == KYTY_PLATFORM_WINDOWS
+static bool CoalescePlaceholders(uint64_t vaddr, uint64_t size) {
+	return size != 0 && VirtualFree(reinterpret_cast<void*>(vaddr), size,
+	                                MEM_RELEASE | MEM_COALESCE_PLACEHOLDERS) != 0;
+}
+#endif
+
 class GuestBackingStore {
 public:
 	enum class FailureReason {
@@ -264,6 +271,19 @@ public:
 			removed.push_back(entry);
 		}
 
+#if KYTY_PLATFORM == KYTY_PLATFORM_WINDOWS
+		// Each unmapped file view becomes a separate Windows placeholder. The address-space owner
+		// records this contiguous guest span as one free range, so normalize fragmented backing
+		// views into one placeholder before publishing it there.
+		if (removed.size() > 1 && !CoalescePlaceholders(vaddr, size)) {
+			for (auto it = removed.rbegin(); it != removed.rend(); ++it) {
+				EXIT_IF(!MapExistingPlaceholderFixed(it->vaddr, it->size, it->backing_offset,
+				                                     it->mode));
+			}
+			return false;
+		}
+#endif
+
 		if (placeholder_preserved != nullptr) {
 			*placeholder_preserved = true;
 		}
@@ -832,6 +852,8 @@ public:
 			EXIT_IF(VirtualFree(reinterpret_cast<void*>(region_vaddr), region_size,
 			                    MEM_RELEASE | MEM_PRESERVE_PLACEHOLDER) == 0);
 		}
+		// ReleaseCommitted also publishes the aggregate span as one logical free range.
+		EXIT_IF(regions.size() > 1 && !CoalescePlaceholders(vaddr, size));
 #else
 		for (const auto& [region_vaddr, region_size]: regions) {
 			RestorePlaceholder(region_vaddr, region_size);
@@ -1252,8 +1274,7 @@ private:
 
 		if (base != it->first || end != it->first + it->second) {
 #if KYTY_PLATFORM == KYTY_PLATFORM_WINDOWS
-			if (VirtualFree(reinterpret_cast<void*>(base), end - base,
+			if (CoalescePlaceholders(base, end - base)) {
-			                MEM_RELEASE | MEM_COALESCE_PLACEHOLDERS) != 0) {
 				m_free.erase(first, last);
 				m_free.emplace(base, end - base);
 			}

tests/VirtualMemoryAllocationTests.cpp

@@ -495,6 +495,85 @@ void TestFlexibleMemoryReuseIsZeroFilled() {
 	std::printf("[host]    %-48s ok\n", test);
 }
 
+void TestSmallerFlexibleMapReusesReleasedHole() {
+	const char*        test       = "SmallerFlexibleMapReusesReleasedHole";
+	const auto         baseline   = AvailableFlexibleMemory(test);
+	constexpr uint64_t SmallSize  = SceKernelPageSize * 12;
+	constexpr uint64_t LargeSize  = SceKernelPageSize * 16;
+	constexpr uint64_t BlockSize  = SceKernelPageSize * 4;
+	constexpr int      RuntimeMap = 0x8000;
+
+	// Leave a 0x30000 hole before an allocated backing block. The large mapping must then be
+	// assembled from two flexible-backing extents, matching the allocation that exposed the
+	// PPSA30528 regression.
+	void* seed_hole = nullptr;
+	CheckOk(test,
+	        Libs::LibKernel::Memory::KernelMapNamedFlexibleMemory(
+	            &seed_hole, SmallSize, SceKernelProtCpuRw, RuntimeMap, "backing_seed_hole"),
+	        "KernelMapNamedFlexibleMemory(seed hole)");
+	void* seed_blocker = nullptr;
+	CheckOk(test,
+	        Libs::LibKernel::Memory::KernelMapNamedFlexibleMemory(
+	            &seed_blocker, BlockSize, SceKernelProtCpuRw, RuntimeMap, "backing_seed_blocker"),
+	        "KernelMapNamedFlexibleMemory(seed blocker)");
+	CheckOk(test,
+	        Libs::LibKernel::Memory::KernelMunmap(reinterpret_cast<uint64_t>(seed_hole), SmallSize),
+	        "KernelMunmap(seed hole)");
+	void* virtual_blocker = seed_hole;
+	CheckOk(test,
+	        Libs::LibKernel::Memory::KernelReserveVirtualRange(&virtual_blocker, SmallSize, 0,
+	                                                           SceKernelPageSize),
+	        "KernelReserveVirtualRange(seed virtual blocker)");
+	Check(test, virtual_blocker == seed_hole, "seed virtual hole was not reserved in place");
+
+	void* large = nullptr;
+	CheckOk(test,
+	        Libs::LibKernel::Memory::KernelMapNamedFlexibleMemory(
+	            &large, LargeSize, SceKernelProtCpuRw, RuntimeMap, "released_large_hole"),
+	        "KernelMapNamedFlexibleMemory(large)");
+	void* blocker = nullptr;
+	CheckOk(test,
+	        Libs::LibKernel::Memory::KernelMapNamedFlexibleMemory(
+	            &blocker, SmallSize, SceKernelProtCpuRw, RuntimeMap, "adjacent_blocker"),
+	        "KernelMapNamedFlexibleMemory(blocker)");
+	const auto large_base   = reinterpret_cast<uint64_t>(large);
+	const auto blocker_base = reinterpret_cast<uint64_t>(blocker);
+	Check(test, blocker_base == large_base + LargeSize,
+	      "hint-less flexible maps were not adjacent");
+
+	CheckOk(test, Libs::LibKernel::Memory::KernelMunmap(large_base, LargeSize),
+	        "KernelMunmap(large)");
+
+	void* reused = nullptr;
+	CheckOk(test,
+	        Libs::LibKernel::Memory::KernelMapNamedFlexibleMemory(
+	            &reused, SmallSize, SceKernelProtCpuRw, RuntimeMap, "smaller_hole_reuse"),
+	        "KernelMapNamedFlexibleMemory(smaller reuse)");
+	Check(test, reinterpret_cast<uint64_t>(reused) == large_base,
+	      "smaller flexible map did not reuse the released hole");
+	*reinterpret_cast<uint64_t*>(reused) = 0x4b59545952455553ull; // "KYTYREUS"
+	Check(test, *reinterpret_cast<const uint64_t*>(reused) == 0x4b59545952455553ull,
+	      "reused flexible hole is not writable");
+
+	CheckOk(test,
+	        Libs::LibKernel::Memory::KernelMunmap(reinterpret_cast<uint64_t>(reused), SmallSize),
+	        "KernelMunmap(reused)");
+	CheckOk(test, Libs::LibKernel::Memory::KernelMunmap(blocker_base, SmallSize),
+	        "KernelMunmap(blocker)");
+	CheckOk(
+	    test,
+	    Libs::LibKernel::Memory::KernelMunmap(reinterpret_cast<uint64_t>(seed_blocker), BlockSize),
+	    "KernelMunmap(seed blocker)");
+	CheckOk(test,
+	        Libs::LibKernel::Memory::KernelMunmap(reinterpret_cast<uint64_t>(virtual_blocker),
+	                                              SmallSize),
+	        "KernelMunmap(seed virtual blocker)");
+	Check(test, AvailableFlexibleMemory(test) == baseline,
+	      "released-hole reuse leaked flexible memory capacity");
+
+	std::printf("[host]    %-48s ok\n", test);
+}
+
 void TestGuestStackUsesPrivateOwnerMemoryAndCache() {
 	const char* test     = "GuestStackUsesPrivateOwnerMemoryAndCache";
 	const auto  baseline = AvailableFlexibleMemory(test);
@@ -605,6 +684,13 @@ void TestRuntimeMemoryOwnerLifecycle() {
 	    test,
 	    Libs::LibKernel::Memory::TestPlaceholderRangeIsFree(adjacent_first, SceKernelPageSize * 2),
 	    "combined adjacent runtime free did not restore one owner placeholder");
+	const auto adjacent_reused = Libs::LibKernel::Memory::AllocateRuntimeMemory(
+	    adjacent_first, SceKernelPageSize, Common::VirtualMemory::Mode::ReadWrite,
+	    "runtime_adjacent_reuse", true);
+	Check(test, adjacent_reused == adjacent_first,
+	      "combined adjacent runtime placeholder could not be split for reuse");
+	Check(test, Libs::LibKernel::Memory::FreeGuestMemory(adjacent_reused, SceKernelPageSize),
+	      "adjacent runtime reuse cleanup failed");
 
 	std::printf("[host]    %-48s ok\n", test);
 }
@@ -2258,6 +2344,7 @@ int main() {
 	RunTest(TestFlexibleDmemCompatAndAlignmentFlags);
 	RunTest(TestFlexibleNoCoalescePreservesBoundaries);
 	RunTest(TestFlexibleMemoryReuseIsZeroFilled);
+	RunTest(TestSmallerFlexibleMapReusesReleasedHole);
 	RunTest(TestGuestStackUsesPrivateOwnerMemoryAndCache);
 	RunTest(TestMainEntryUsesGuestStackAndDisablesHostChecks);
 	RunTest(TestFragmentedBackingUnmapRollback);