From f7ae86c9829f93a85f0bc810ffcaba75fd21904d Mon Sep 17 00:00:00 2001 From: nmzik Date: Fri, 16 Oct 2026 00:58:57 +0200 Subject: [PATCH] kernel: optimize virtual range edits with an ordered map Co-authored-by: shadowbeat070 <70843316+shadowbeat070@users.noreply.github.com> --- src/kernel/memory.cpp | 365 ++++++++++++------------- tests/VirtualMemoryAllocationTests.cpp | 290 ++++++++++++++++++++ 2 files changed, 459 insertions(+), 196 deletions(-) diff --git a/src/kernel/memory.cpp b/src/kernel/memory.cpp index 246f0d126..441eecaf9 100644 --- a/src/kernel/memory.cpp +++ b/src/kernel/memory.cpp @@ -232,15 +232,15 @@ public: VirtualRangeType type, const char* name, bool disallow_merge = false) { Common::LockGuard lock(m_mutex); - if (start == 0 || size == 0) { + if (start == 0 || size == 0 || size > UINT64_MAX - start) { return false; } auto position = LowerBound(start); if ((position != m_ranges.end() && - VirtualRangesOverlap(start, size, position->start, position->size)) || + VirtualRangesOverlap(start, size, position->second.start, position->second.size)) || (position != m_ranges.begin() && - VirtualRangesOverlap(start, size, std::prev(position)->start, - std::prev(position)->size))) { + VirtualRangesOverlap(start, size, std::prev(position)->second.start, + std::prev(position)->second.size))) { return false; } @@ -253,23 +253,21 @@ public: r.type = type; r.disallow_merge = disallow_merge; CopyVirtualRangeName(r.name, name); - const auto index = static_cast(position - m_ranges.begin()); - m_ranges.insert(position, r); - MergeAroundUnlocked(index); + auto inserted = m_ranges.emplace_hint(position, start, r); + UpdatePageTableEntries(inserted->second, true); + MergeAroundUnlocked(inserted); return true; } bool Remove(uint64_t start, uint64_t size) { Common::LockGuard lock(m_mutex); - auto position = LowerBound(start); - if (position != m_ranges.end() && position->start == start && position->size == size) { - m_ranges.erase(position); + auto position = m_ranges.find(start); + if (position != m_ranges.end() && position->second.size == size) { + EraseUnlocked(position); return true; } - auto removed = RemoveUnlocked(start, size); - MergeUnlocked(); - return removed; + return RemoveUnlocked(start, size); } bool HasOverlap(uint64_t start, uint64_t size) { @@ -293,40 +291,31 @@ public: bool ReleaseReserved(uint64_t start, uint64_t size) { Common::LockGuard lock(m_mutex); - for (size_t index = 0; index < m_ranges.size(); index++) { - auto& r = m_ranges[index]; - if (r.start == start && r.size == size && IsReservedRangeType(r.type)) { - m_ranges.erase(m_ranges.begin() + static_cast(index)); - return true; - } + auto position = m_ranges.find(start); + if (position != m_ranges.end() && position->second.size == size && + IsReservedRangeType(position->second.type)) { + EraseUnlocked(position); } return true; } - bool ReplaceSpan(uint64_t start, uint64_t size, VirtualRangeType expected_type, - uint64_t offset, int protection, int memory_type, VirtualRangeType type, - const char* name, bool disallow_merge = false) { + bool ReplaceSpan(uint64_t start, uint64_t size, VirtualRangeType expected_type, uint64_t offset, + int protection, int memory_type, VirtualRangeType type, const char* name, + bool disallow_merge = false) { Common::LockGuard lock(m_mutex); if (start == 0 || size == 0 || size > UINT64_MAX - start) { return false; } - auto current = start; - const auto end = start + size; - while (current < end) { - const Range* candidate = nullptr; - for (const auto& r: m_ranges) { - if (r.type == expected_type && current >= r.start && - current < End(r.start, r.size)) { - candidate = &r; - break; - } - } - if (candidate == nullptr) { + auto current = start; + const auto end = start + size; + for (auto candidate = FindContaining(start); current < end; ++candidate) { + if (candidate == m_ranges.end() || candidate->second.start > current || + candidate->second.type != expected_type) { return false; } - current = std::min(end, End(candidate->start, candidate->size)); + current = std::min(end, End(candidate->second.start, candidate->second.size)); } #if defined(KYTY_VIRTUAL_MEMORY_ALLOCATION_TESTS) @@ -344,21 +333,27 @@ public: replacement.type = type; replacement.disallow_merge = disallow_merge; CopyVirtualRangeName(replacement.name, name); - RemoveUnlocked(start, size); - auto position = LowerBound(start); - const auto index = static_cast(position - m_ranges.begin()); - m_ranges.insert(position, replacement); - MergeAroundUnlocked(index); + // Validation precedes all mutations. Keep the first node for the replacement so + // replacing an existing arena does not allocate another tree node. + auto first = SplitAtUnlocked(start); + auto last = SplitAtUnlocked(end); + for (auto it = std::next(first); it != last;) { + it = EraseUnlocked(it); + } + UpdatePageTableEntries(first->second, false); + first->second = replacement; + UpdatePageTableEntries(first->second, true); + MergeAroundUnlocked(first); return true; } void Rename(uint64_t start, uint64_t size, const char* name) { Common::LockGuard lock(m_mutex); - auto position = LowerBound(start); - if (position != m_ranges.end() && position->start == start && position->size == size) { - CopyVirtualRangeName(position->name, name); - MergeAroundUnlocked(static_cast(position - m_ranges.begin())); + auto position = m_ranges.find(start); + if (position != m_ranges.end() && position->second.size == size) { + CopyVirtualRangeName(position->second.name, name); + MergeAroundUnlocked(position); return; } EditUnlocked(start, size, [name](Range* r) { CopyVirtualRangeName(r->name, name); }); @@ -375,13 +370,11 @@ public: Common::LockGuard lock(m_mutex); - auto next = std::upper_bound( - m_ranges.begin(), m_ranges.end(), addr, - [](uint64_t value, const Range& range) { return value < range.start; }); + auto next = m_ranges.upper_bound(addr); if (next != m_ranges.begin()) { auto current = std::prev(next); - if (addr < End(current->start, current->size)) { - *out = *current; + if (addr < End(current->second.start, current->second.size)) { + *out = current->second; return true; } } @@ -389,7 +382,7 @@ public: return false; } - *out = *next; + *out = next->second; return true; } @@ -404,11 +397,9 @@ public: const auto end = start + size; auto current = start; - for (const auto& range: m_ranges) { - const auto range_end = End(range.start, range.size); - if (range_end <= current) { - continue; - } + for (auto it = FindContaining(start); it != m_ranges.end(); ++it) { + const auto& range = it->second; + const auto range_end = End(range.start, range.size); if (range.start > current) { break; } @@ -437,27 +428,19 @@ public: return 0; } - auto vma = std::upper_bound( - m_ranges.begin(), m_ranges.end(), virtual_addr, - [](uint64_t value, const Range& range) { return value < range.start; }); - if (vma == m_ranges.begin()) { - return 0; - } - --vma; - - const auto vma_end = End(vma->start, vma->size); - if (virtual_addr < vma->start || virtual_addr >= vma_end || - !IsCommittedRangeType(vma->type)) { + auto vma = FindContaining(virtual_addr); + if (vma == m_ranges.end() || !IsCommittedRangeType(vma->second.type)) { return 0; } + const auto vma_end = End(vma->second.start, vma->second.size); uint64_t clamped_size = std::min(size, vma_end - virtual_addr); uint64_t expected = virtual_addr + clamped_size; ++vma; - while (vma != m_ranges.end() && vma->start == expected && IsCommittedRangeType(vma->type) && - clamped_size < size) { - const auto chunk = std::min(size - clamped_size, vma->size); + while (vma != m_ranges.end() && vma->second.start == expected && + IsCommittedRangeType(vma->second.type) && clamped_size < size) { + const auto chunk = std::min(size - clamped_size, vma->second.size); clamped_size += chunk; expected += chunk; ++vma; @@ -468,27 +451,7 @@ public: uint64_t CountPageTableEntries(bool gpu) { Common::LockGuard lock(m_mutex); - - uint64_t used = 0; - for (const auto& r: m_ranges) { - if (!IsCommittedRangeType(r.type) || r.size == 0) { - continue; - } - - const bool has_cpu_access = - (r.protection & (PROT_CPU_READ | PROT_CPU_WRITE | PROT_CPU_EXEC)) != 0; - const bool has_gpu_access = (r.protection & (PROT_GPU_READ | PROT_GPU_WRITE)) != 0; - if (gpu ? !has_gpu_access : !has_cpu_access) { - continue; - } - - const auto end = End(r.start, r.size); - const auto first_entry = r.start / PAGE_TABLE_GRANULARITY; - const auto last_entry = (end - 1) / PAGE_TABLE_GRANULARITY; - used += last_entry - first_entry + 1; - } - - return used; + return gpu ? m_gpu_page_table_entries : m_cpu_page_table_entries; } private: @@ -507,51 +470,112 @@ private: std::strncmp(left.name, right.name, KERNEL_MAXIMUM_NAME_LENGTH) == 0; } - static void AddPiece(std::vector* ranges, const Range& source, uint64_t start, - uint64_t end) { - EXIT_IF(ranges == nullptr); + // Keys are range starts; ranges are nonempty, non-overlapping and locally coalesced. + // A tree avoids shifting unrelated ranges when streaming edits split or remove an arena. + using RangeMap = std::map; + using Iterator = RangeMap::iterator; - if (end <= start) { - return; - } + Iterator LowerBound(uint64_t start) { return m_ranges.lower_bound(start); } - Range piece = source; - piece.start = start; - piece.size = end - start; - if (piece.type == VirtualRangeType::Direct) { - piece.offset += start - source.start; + Iterator FindContaining(uint64_t address) { + auto next = m_ranges.upper_bound(address); + if (next == m_ranges.begin()) { + return m_ranges.end(); } - ranges->push_back(piece); + auto current = std::prev(next); + return address < End(current->second.start, current->second.size) ? current + : m_ranges.end(); } - std::vector::iterator LowerBound(uint64_t start) { - return std::lower_bound( - m_ranges.begin(), m_ranges.end(), start, - [](const Range& range, uint64_t value) { return range.start < value; }); + static bool CanMerge(const Range& left, const Range& right) { + return End(left.start, left.size) == right.start && SameMergeKey(left, right); } - void MergeAroundUnlocked(size_t index) { - if (index >= m_ranges.size()) { + void UpdatePageTableEntries(const Range& range, bool add) { + if (!IsCommittedRangeType(range.type)) { return; } - if (index != 0) { - auto& previous = m_ranges[index - 1]; - auto& current = m_ranges[index]; - if (End(previous.start, previous.size) == current.start && - SameMergeKey(previous, current)) { - previous.size += current.size; - m_ranges.erase(m_ranges.begin() + static_cast(index)); - index--; + // Preserve the statistics API's per-range accounting: splitting within a page-table + // entry counts it for each fragment, and merging removes that duplicate contribution. + const auto entries = (End(range.start, range.size) - 1) / PAGE_TABLE_GRANULARITY - + range.start / PAGE_TABLE_GRANULARITY + 1; + if ((range.protection & (PROT_CPU_READ | PROT_CPU_WRITE | PROT_CPU_EXEC)) != 0) { + m_cpu_page_table_entries = + add ? m_cpu_page_table_entries + entries : m_cpu_page_table_entries - entries; + } + if ((range.protection & (PROT_GPU_READ | PROT_GPU_WRITE)) != 0) { + m_gpu_page_table_entries = + add ? m_gpu_page_table_entries + entries : m_gpu_page_table_entries - entries; + } + } + + Iterator EraseUnlocked(Iterator position) { + UpdatePageTableEntries(position->second, false); + return m_ranges.erase(position); + } + + Iterator MergeNextUnlocked(Iterator left, Iterator right) { + UpdatePageTableEntries(left->second, false); + left->second.size += right->second.size; + auto next = EraseUnlocked(right); + UpdatePageTableEntries(left->second, true); + return next; + } + + void MergeAroundUnlocked(Iterator current) { + if (current != m_ranges.begin()) { + auto previous = std::prev(current); + if (CanMerge(previous->second, current->second)) { + MergeNextUnlocked(previous, current); + current = previous; } } - while (index + 1 < m_ranges.size()) { - auto& current = m_ranges[index]; - auto& next = m_ranges[index + 1]; - if (End(current.start, current.size) != next.start || !SameMergeKey(current, next)) { + for (auto next = std::next(current); next != m_ranges.end();) { + if (!CanMerge(current->second, next->second)) { break; } - current.size += next.size; - m_ranges.erase(m_ranges.begin() + static_cast(index + 1)); + next = MergeNextUnlocked(current, next); + } + } + + Iterator SplitAtUnlocked(uint64_t address) { + auto position = LowerBound(address); + if (position != m_ranges.begin()) { + auto& previous = std::prev(position)->second; + const auto previous_end = End(previous.start, previous.size); + if (previous.start < address && address < previous_end) { + Range right = previous; + right.start = address; + right.size = previous_end - address; + if (right.type == VirtualRangeType::Direct) { + right.offset += address - previous.start; + } + position = m_ranges.emplace_hint(position, address, right); + UpdatePageTableEntries(previous, false); + previous.size = address - previous.start; + UpdatePageTableEntries(previous, true); + UpdatePageTableEntries(right, true); + } + } + return position; + } + + void MergeSpanUnlocked(Iterator first, Iterator last) { + if (first == last) { + return; + } + // Include both neighbors. The right neighbor itself can be erased by a merge, + // so the sentinel must be the node after it, which this loop never touches. + const auto stop = last == m_ranges.end() ? last : std::next(last); + if (first != m_ranges.begin()) { + --first; + } + for (auto next = std::next(first); next != stop;) { + if (CanMerge(first->second, next->second)) { + next = MergeNextUnlocked(first, next); + } else { + first = next++; + } } } @@ -560,102 +584,51 @@ private: if (size == 0) { return; } - - std::vector out; - auto edit_end = End(start, size); - - for (const auto& r: m_ranges) { - auto r_end = End(r.start, r.size); - if (!VirtualRangesOverlap(start, size, r.start, r.size)) { - out.push_back(r); - continue; - } - - auto mid_start = std::max(start, r.start); - auto mid_end = std::min(edit_end, r_end); - - AddPiece(&out, r, r.start, mid_start); - - Range mid = r; - mid.start = mid_start; - mid.size = mid_end - mid_start; - if (mid.type == VirtualRangeType::Direct) { - mid.offset += mid_start - r.start; - } - edit(&mid); - out.push_back(mid); - - AddPiece(&out, r, mid_end, r_end); + auto first = SplitAtUnlocked(start); + auto last = SplitAtUnlocked(End(start, size)); + for (auto it = first; it != last; ++it) { + UpdatePageTableEntries(it->second, false); + edit(&it->second); + UpdatePageTableEntries(it->second, true); } - - m_ranges = out; - MergeUnlocked(); + MergeSpanUnlocked(first, last); } bool RemoveUnlocked(uint64_t start, uint64_t size) { if (size == 0) { return false; } - - std::vector out; - bool removed = false; - auto rem_end = End(start, size); - - for (const auto& r: m_ranges) { - auto r_end = End(r.start, r.size); - if (!VirtualRangesOverlap(start, size, r.start, r.size)) { - out.push_back(r); - continue; - } - - removed = true; - AddPiece(&out, r, r.start, std::max(start, r.start)); - AddPiece(&out, r, std::min(rem_end, r_end), r_end); + auto first = SplitAtUnlocked(start); + auto last = SplitAtUnlocked(End(start, size)); + if (first == last) { + return false; } - - m_ranges = out; - return removed; - } - - void MergeUnlocked() { - if (m_ranges.size() < 2) { - return; + while (first != last) { + first = EraseUnlocked(first); } - - std::sort(m_ranges.begin(), m_ranges.end(), - [](const Range& left, const Range& right) { return left.start < right.start; }); - - std::vector merged; - for (const auto& r: m_ranges) { - if (!merged.empty()) { - auto& last = merged[merged.size() - 1]; - if (End(last.start, last.size) == r.start && SameMergeKey(last, r)) { - last.size += r.size; - continue; - } - } - merged.push_back(r); - } - m_ranges = merged; + // Removal leaves an address gap, so it cannot make surviving ranges mergeable. + return true; } Range* FindOverlap(uint64_t start, uint64_t size) { auto position = LowerBound(start); if (position != m_ranges.end() && - VirtualRangesOverlap(start, size, position->start, position->size)) { - return &*position; + VirtualRangesOverlap(start, size, position->second.start, position->second.size)) { + return &position->second; } if (position != m_ranges.begin()) { auto previous = std::prev(position); - if (VirtualRangesOverlap(start, size, previous->start, previous->size)) { - return &*previous; + if (VirtualRangesOverlap(start, size, previous->second.start, previous->second.size)) { + return &previous->second; } } return nullptr; } - std::vector m_ranges; - Common::Mutex m_mutex; + RangeMap m_ranges; + Common::Mutex m_mutex; + uint64_t m_cpu_page_table_entries = 0; + uint64_t m_gpu_page_table_entries = 0; }; #if defined(KYTY_VIRTUAL_MEMORY_ALLOCATION_TESTS) diff --git a/tests/VirtualMemoryAllocationTests.cpp b/tests/VirtualMemoryAllocationTests.cpp index 369b9088b..111240933 100644 --- a/tests/VirtualMemoryAllocationTests.cpp +++ b/tests/VirtualMemoryAllocationTests.cpp @@ -926,6 +926,19 @@ void TestFlexibleNoCoalescePreservesBoundaries() { 0, 1, "no_coalesce"); ExpectRange(test, Query(test, base + SceKernelPageSize), base + SceKernelPageSize, base + SceKernelPageSize * 2, SceKernelProtCpuRw, 1, 0, 0, 1, "no_coalesce"); + CheckOk(test, + Libs::LibKernel::Memory::KernelMprotect(reserve, SceKernelPageSize * 2, + SceKernelProtCpuRead), + "KernelMprotect(NO_COALESCE span)"); + CheckOk(test, + Libs::LibKernel::Memory::KernelSetVirtualRangeName(reserve, SceKernelPageSize * 2, + "renamed_no_coalesce"), + "KernelSetVirtualRangeName(NO_COALESCE span)"); + for (uint64_t i = 0; i < 2; ++i) { + ExpectRange(test, Query(test, base + i * SceKernelPageSize), base + i * SceKernelPageSize, + base + (i + 1) * SceKernelPageSize, SceKernelProtCpuRead, 1, 0, 0, 1, + "renamed_no_coalesce"); + } CheckOk(test, Libs::LibKernel::Memory::KernelMunmap(base, SceKernelPageSize * 2), "KernelMunmap"); @@ -1262,6 +1275,179 @@ void TestPartialFlexibleMunmapAndFindNext() { std::printf("[host] %-48s ok\n", test); } +void TestFlexibleRangeEditsCoalesce() { + using namespace Libs::LibKernel::Memory; + const char* test = "FlexibleRangeEditsCoalesce"; + constexpr uint64_t page = SceKernelPageSize; + constexpr uint64_t pages = 129; + constexpr uint64_t size = pages * page; + const auto baseline = AvailableFlexibleMemory(test); + const auto base = MapNamedFlexible(test, size, SceKernelProtCpuRw, "dense_ranges"); + + // Alternate protections fragment one allocation into many independently queryable ranges. + for (uint64_t i = 1; i < pages; i += 2) { + CheckOk( + test, + KernelMprotect(reinterpret_cast(base + i * page), page, SceKernelProtCpuRead), + "KernelMprotect(fragment)"); + } + for (uint64_t i = 0; i < pages; ++i) { + ExpectRange(test, Query(test, base + i * page + page - 1), base + i * page, + base + (i + 1) * page, i % 2 == 0 ? SceKernelProtCpuRw : SceKernelProtCpuRead, + 1, 0, 0, 1, "dense_ranges"); + } + CheckOk(test, KernelMprotect(reinterpret_cast(base), size, SceKernelProtCpuRw), + "KernelMprotect(coalesce all fragments)"); + ExpectRange(test, Query(test, base + size - 1), base, base + size, SceKernelProtCpuRw, 1, 0, 0, + 1, "dense_ranges"); + + // Renaming the interior must split both boundaries, then restoring it must join both sides. + CheckOk(test, + KernelSetVirtualRangeName(reinterpret_cast(base + page), size - 2 * page, + "renamed_interior"), + "KernelSetVirtualRangeName(interior)"); + ExpectRange(test, Query(test, base), base, base + page, SceKernelProtCpuRw, 1, 0, 0, 1, + "dense_ranges"); + ExpectRange(test, Query(test, base + page), base + page, base + size - page, SceKernelProtCpuRw, + 1, 0, 0, 1, "renamed_interior"); + ExpectRange(test, Query(test, base + size - page), base + size - page, base + size, + SceKernelProtCpuRw, 1, 0, 0, 1, "dense_ranges"); + CheckOk(test, + KernelSetVirtualRangeName(reinterpret_cast(base + page), size - 2 * page, + "dense_ranges"), + "KernelSetVirtualRangeName(restore interior)"); + ExpectRange(test, Query(test, base), base, base + size, SceKernelProtCpuRw, 1, 0, 0, 1, + "dense_ranges"); + + for (uint64_t i = 1; i < pages; i += 2) { + CheckOk(test, + KernelSetVirtualRangeName(reinterpret_cast(base + i * page), page, + "fragment_name"), + "KernelSetVirtualRangeName(fragment)"); + } + CheckOk(test, KernelClearVirtualRangeName(reinterpret_cast(base), size), + "KernelClearVirtualRangeName(coalesce all fragments)"); + ExpectRange(test, Query(test, base + size - 1), base, base + size, SceKernelProtCpuRw, 1, 0, 0, + 1, ""); + + constexpr uint64_t middle = pages / 2 * page; + CheckOk(test, KernelMunmap(base + middle, page), "KernelMunmap(create hole)"); + CheckOk(test, KernelSetVirtualRangeName(reinterpret_cast(base), size, "across_hole"), + "KernelSetVirtualRangeName(across hole)"); + ExpectRange(test, Query(test, base + middle - 1), base, base + middle, SceKernelProtCpuRw, 1, 0, + 0, 1, "across_hole"); + ExpectUnmapped(test, base + middle); + ExpectRange(test, Query(test, base + middle, SceKernelVqFindNext), base + middle + page, + base + size, SceKernelProtCpuRw, 1, 0, 0, 1, "across_hole"); + Check(test, ClampRangeSize(base + middle - page / 2, page) == page / 2, + "range clamp crossed the hole after renaming"); + CheckOk(test, KernelMunmap(base, middle), "KernelMunmap(left cleanup)"); + CheckOk(test, KernelMunmap(base + middle + page, size - middle - page), + "KernelMunmap(right cleanup)"); + Check(test, AvailableFlexibleMemory(test) == baseline, + "range edit cleanup did not restore flexible capacity"); + std::printf("[host] %-48s ok\n", test); +} + +void TestPageTableStatsTrackRangeEdits() { + using namespace Libs::LibKernel::Memory; + const char* test = "PageTableStatsTrackRangeEdits"; + constexpr uint64_t page = SceKernelPageSize; + constexpr uint64_t table_span = 0x200000; + constexpr uint64_t size = table_span + 3 * page; + constexpr int gpu_read = 0x10; + constexpr int gpu_write = 0x20; + constexpr int both = SceKernelProtCpuRw | gpu_read; + const auto read_stats = [&] { + std::array stats {}; + CheckOk(test, KernelGetPageTableStats(&stats[0], &stats[1], &stats[2], &stats[3]), + "KernelGetPageTableStats"); + return stats; + }; + const auto baseline = read_stats(); + const auto expect_used = [&](int cpu, int gpu, const char* action) { + const auto stats = read_stats(); + Check(test, + stats[0] == baseline[0] && stats[2] == baseline[2] && baseline[1] - stats[1] == cpu && + baseline[3] - stats[3] == gpu, + std::string(action) + " changed page-table statistics unexpectedly"); + }; + void* address = nullptr; + CheckOk(test, KernelReserveVirtualRange(&address, size, 0, table_span), + "KernelReserveVirtualRange(flexible stats)"); + const auto base = reinterpret_cast(address); + Check(test, base % table_span == 0, "stats reservation is not aligned"); + expect_used(0, 0, "reservation"); + CheckOk(test, + KernelMapNamedFlexibleMemory(&address, size, SceKernelProtCpuRw, SceKernelMapFixed, + "table_stats"), + "KernelMapNamedFlexibleMemory(stats)"); + expect_used(2, 0, "flexible mapping across two table entries"); + + // The existing ABI estimate counts each fragment's touched 2 MiB entries independently. + void* middle = reinterpret_cast(base + page); + CheckOk(test, KernelSetVirtualRangeName(middle, page, "table_stats_middle"), + "KernelSetVirtualRangeName(stats split)"); + expect_used(4, 0, "name split"); + CheckOk(test, KernelSetVirtualRangeName(middle, page, "table_stats"), + "KernelSetVirtualRangeName(stats merge)"); + expect_used(2, 0, "name coalescing"); + CheckOk(test, KernelMprotect(middle, page, both), "KernelMprotect(stats split)"); + expect_used(4, 1, "protection split"); + CheckOk(test, KernelMprotect(address, size, both), "KernelMprotect(stats merge)"); + expect_used(2, 2, "protection coalescing"); + CheckOk(test, KernelMprotect(address, size, gpu_write), "KernelMprotect(GPU only)"); + expect_used(0, 2, "GPU-only protection"); + CheckOk(test, KernelMprotect(address, size, SceKernelProtCpuRw), "KernelMprotect(CPU only)"); + expect_used(2, 0, "CPU-only protection"); + CheckOk(test, KernelMprotect(address, size, 0), "KernelMprotect(no access)"); + expect_used(0, 0, "no-access protection"); + CheckOk(test, KernelMprotect(address, size, both), "KernelMprotect(restore stats access)"); + CheckOk(test, KernelMunmap(base + page, page), "KernelMunmap(stats middle)"); + expect_used(3, 3, "partial removal"); + CheckOk(test, KernelMunmap(base, page), "KernelMunmap(stats left)"); + expect_used(2, 2, "exact removal"); + CheckOk(test, KernelMunmap(base + 2 * page, size - 2 * page), "KernelMunmap(stats right)"); + expect_used(0, 0, "flexible cleanup"); + + int64_t physical = 0; + CheckOk(test, + KernelAllocateDirectMemory(0, KernelGetDirectMemorySize(), 3 * page, page, + SceKernelMtypeC, &physical), + "KernelAllocateDirectMemory(stats)"); + address = nullptr; + CheckOk(test, KernelReserveVirtualRange(&address, 3 * page, 0, table_span), + "KernelReserveVirtualRange(direct stats)"); + const auto direct_base = reinterpret_cast(address); + CheckOk(test, + KernelMapNamedDirectMemory(&address, 3 * page, both, SceKernelMapFixed, physical, page, + "direct_table_stats"), + "KernelMapNamedDirectMemory(stats)"); + expect_used(1, 1, "direct mapping"); + CheckOk(test, KernelMprotect(reinterpret_cast(direct_base + page), page, both), + "KernelMprotect(unchanged direct stats subspan)"); + expect_used(3, 3, "unchanged direct protection split"); +#if defined(__linux__) + TestFailNextVirtualRangeReplacement(); + CheckFailed(test, + KernelMapNamedDirectMemory(&address, 3 * page, SceKernelProtCpuRw, + SceKernelMapFixed, physical, page, + "rejected_table_stats"), + "KernelMapNamedDirectMemory(stats publication failure)"); + expect_used(3, 3, "failed direct replacement"); +#endif + address = reinterpret_cast(direct_base); + CheckOk(test, + KernelMapNamedDirectMemory(&address, 3 * page, SceKernelProtCpuRw, SceKernelMapFixed, + physical, page, "replacement_table_stats"), + "KernelMapNamedDirectMemory(stats replacement)"); + expect_used(1, 0, "direct replacement"); + CheckOk(test, KernelReleaseDirectMemory(physical, 3 * page), + "KernelReleaseDirectMemory(stats cleanup)"); + expect_used(0, 0, "direct cleanup"); + std::printf("[host] %-48s ok\n", test); +} + void TestReserveMapFixedAndNoOverwrite() { const char* test = "ReserveMapFixedAndNoOverwrite"; void* addr = nullptr; @@ -1505,6 +1691,88 @@ void TestDirectPartialProtectUnmapPreservesNeighbors() { std::printf("[host] %-48s ok\n", test); } +void TestFixedArenaChurnKeepsRangesExact() { + using namespace Libs::LibKernel::Memory; + const char* test = "FixedArenaChurnKeepsRangesExact"; + constexpr uint64_t page = SceKernelPageSize; + constexpr uint64_t arena_size = page * 4; + constexpr uint64_t arena_count = 8; + constexpr uint64_t size = arena_size * arena_count; + int64_t physical = 0; + CheckOk(test, + KernelAllocateDirectMemory(0, KernelGetDirectMemorySize(), size, page, SceKernelMtypeC, + &physical), + "KernelAllocateDirectMemory"); + void* reservation = nullptr; + CheckOk(test, KernelReserveVirtualRange(&reservation, size, 0, page), + "KernelReserveVirtualRange"); + const auto base = reinterpret_cast(reservation); + const auto offset = static_cast(physical); + const auto map_arena = [&](uint64_t i) { + void* target = reinterpret_cast(base + i * arena_size); + CheckOk(test, + KernelMapNamedDirectMemory(&target, arena_size, SceKernelProtCpuRw, + SceKernelMapFixed, physical + i * arena_size, page, + "fixed_arena"), + "KernelMapNamedDirectMemory(arena)"); + Check(test, reinterpret_cast(target) == base + i * arena_size, + "fixed arena mapping moved"); + }; + for (uint64_t i = 0; i < arena_count; ++i) { + map_arena(i); + } + CheckOk(test, KernelMunmap(base + 2 * arena_size, 4 * arena_size), "KernelMunmap(arenas 2-5)"); + map_arena(3); + for (uint64_t i: {0, 1, 3, 6, 7}) { + ExpectRange(test, Query(test, base + i * arena_size), base + i * arena_size, + base + (i + 1) * arena_size, SceKernelProtCpuRw, 0, 1, 0, 1, "fixed_arena", + offset + i * arena_size); + } + for (uint64_t i: {2, 4, 5}) { + ExpectUnmapped(test, base + i * arena_size); + } + ExpectRange(test, Query(test, base + 4 * arena_size, SceKernelVqFindNext), + base + 6 * arena_size, base + 7 * arena_size, SceKernelProtCpuRw, 0, 1, 0, 1, + "fixed_arena", offset + 6 * arena_size); + + // A span crossing two direct mappings splits each one and preserves its physical offset. + CheckOk( + test, + KernelMprotect(reinterpret_cast(base + 2 * page), 4 * page, SceKernelProtCpuRead), + "KernelMprotect(across direct arenas)"); + for (uint64_t i = 0; i < 4; ++i) { + ExpectRange(test, Query(test, base + i * 2 * page), base + i * 2 * page, + base + (i + 1) * 2 * page, + i == 1 || i == 2 ? SceKernelProtCpuRead : SceKernelProtCpuRw, 0, 1, 0, 1, + "fixed_arena", offset + i * 2 * page); + } + CheckOk(test, KernelMprotect(reservation, 2 * arena_size, SceKernelProtCpuRw), + "KernelMprotect(restore direct arenas)"); + CheckOk(test, KernelSetVirtualRangeName(reservation, 2 * arena_size, "renamed_arena"), + "KernelSetVirtualRangeName(direct arenas)"); + for (uint64_t i = 0; i < 4; ++i) { + ExpectRange(test, Query(test, base + i * 2 * page), base + i * 2 * page, + base + (i + 1) * 2 * page, SceKernelProtCpuRw, 0, 1, 0, 1, "renamed_arena", + offset + i * 2 * page); + } + // Even unchanged protection preserves the new boundary when direct ranges cannot merge. + CheckOk(test, KernelMprotect(reinterpret_cast(base + page), page, SceKernelProtCpuRw), + "KernelMprotect(unchanged direct subspan)"); + ExpectRange(test, Query(test, base), base, base + page, SceKernelProtCpuRw, 0, 1, 0, 1, + "renamed_arena", offset); + ExpectRange(test, Query(test, base + page), base + page, base + 2 * page, SceKernelProtCpuRw, 0, + 1, 0, 1, "renamed_arena", offset + page); + Check(test, ClampRangeSize(base + page, 2 * arena_size) == 2 * arena_size - page, + "range clamp did not traverse direct fragments up to the hole"); + CheckOk(test, KernelMunmap(base, 2 * arena_size), "KernelMunmap(arenas 0-1)"); + CheckOk(test, KernelMunmap(base + 3 * arena_size, arena_size), "KernelMunmap(arena 3)"); + CheckOk(test, KernelMunmap(base + 6 * arena_size, 2 * arena_size), "KernelMunmap(arenas 6-7)"); + ExpectUnmapped(test, base); + ExpectUnmapped(test, base + size - 1); + CheckOk(test, KernelReleaseDirectMemory(physical, size), "KernelReleaseDirectMemory"); + std::printf("[host] %-48s ok\n", test); +} + #if defined(__linux__) void TestPartialUnmapPreservesHostPermissions() { const char* test = "PartialUnmapPreservesHostPermissions"; @@ -4509,6 +4777,25 @@ void TestSmallFiberStacksAndMigration() { int main(int argc, char** argv) { InitSubsystems(); + if (argc == 2 && std::strcmp(argv[1], "--virtual-ranges-only") == 0) { + RunTest(TestFlexibleNoCoalescePreservesBoundaries); + RunTest(TestFlexibleRangeEditsCoalesce); + RunTest(TestPageTableStatsTrackRangeEdits); + RunTest(TestPartialFlexibleMunmapAndFindNext); + RunTest(TestReserveMapFixedAndNoOverwrite); + RunTest(TestFixedNoOverwriteRejectsReservedRange); + RunTest(TestFixedArenaChurnKeepsRangesExact); + RunTest(TestDirectMapQueryOffsetAndPartialMunmap); + RunTest(TestDirectPartialProtectUnmapPreservesNeighbors); +#if defined(__linux__) + RunTest(TestFixedDirectReplacementPreservesAccess); +#endif + RunTest(TestFixedReserveReplacesPartialDirectMapping); + RunTest(TestFixedReserveRangeAddRollbackKeepsPlaceholder); + RunTest(TestMemoryPoolMultiRangeDecommit); + RunTest(TestProgramMemoryAllocationAndProtection); + return g_failed_tests == 0 ? 0 : 1; + } if (argc == 2 && std::strcmp(argv[1], "--backing-transfers-only") == 0) { RunTest(TestConcurrentBackingReads); RunTest(TestBackingReadExcludesWritesAndUnmap); @@ -4585,12 +4872,15 @@ int main(int argc, char** argv) { RunTest(TestRuntimeMemoryOwnerLifecycle); RunTest(TestFlexibleMapQueryAndWholeMunmap); RunTest(TestPartialFlexibleMunmapAndFindNext); + RunTest(TestFlexibleRangeEditsCoalesce); + RunTest(TestPageTableStatsTrackRangeEdits); RunTest(TestReserveMapFixedAndNoOverwrite); RunTest(TestFixedNoOverwriteRejectsReservedRange); RunTest(TestReleasedReserveCanBeReused); RunTest(TestMunmapAcrossAdjacentFlexibleMappings); RunTest(TestDirectMapQueryOffsetAndPartialMunmap); RunTest(TestDirectPartialProtectUnmapPreservesNeighbors); + RunTest(TestFixedArenaChurnKeepsRangesExact); #if defined(__linux__) RunTest(TestPartialUnmapPreservesHostPermissions); #endif