diff --git a/src/kernel/memory.cpp b/src/kernel/memory.cpp index 178fa79ef..86fb55af0 100644 --- a/src/kernel/memory.cpp +++ b/src/kernel/memory.cpp @@ -23,6 +23,7 @@ #include #include #include +#include #include #if KYTY_PLATFORM == KYTY_PLATFORM_WINDOWS @@ -62,8 +63,6 @@ constexpr int PROT_CPU_EXEC = 0x04; constexpr int PROT_GPU_READ = 0x10; constexpr int PROT_GPU_WRITE = 0x20; -enum class GpuAccessMode { NoAccess, Read, Write, ReadWrite }; - constexpr uint64_t PAGE_TABLE_GRANULARITY = 2ull * 1024ull * 1024ull; constexpr uint64_t DEFAULT_FLEXIBLE_MEMORY_SIZE = 448ull * 1024ull * 1024ull; // A game gets 12.5 GiB. A fixed 64 MiB cache and mapping table memory past the 64 MiB included @@ -112,9 +111,8 @@ static void UnmapGpuRange(uint64_t vaddr, uint64_t size) { GetGpuResources().UnmapMemory(vaddr, size); } -static bool DecodeMemoryProtection(int prot, VirtualMemory::Mode* mode, GpuAccessMode* gpu_mode) { +static bool DecodeMemoryProtection(int prot, VirtualMemory::Mode* mode) { EXIT_IF(mode == nullptr); - EXIT_IF(gpu_mode == nullptr); bool cpu_read = (prot & PROT_CPU_READ) != 0; bool cpu_write = (prot & PROT_CPU_WRITE) != 0; @@ -128,16 +126,6 @@ static bool DecodeMemoryProtection(int prot, VirtualMemory::Mode* mode, GpuAcces return false; } - if (gpu_read && gpu_write) { - *gpu_mode = GpuAccessMode::ReadWrite; - } else if (gpu_read) { - *gpu_mode = GpuAccessMode::Read; - } else if (gpu_write) { - *gpu_mode = GpuAccessMode::Write; - } else { - *gpu_mode = GpuAccessMode::NoAccess; - } - bool host_read = cpu_read || gpu_read; bool host_write = cpu_write || gpu_write; @@ -183,9 +171,11 @@ static bool VirtualRangesOverlap(uint64_t left_start, uint64_t left_size, uint64 } #if defined(KYTY_VIRTUAL_MEMORY_ALLOCATION_TESTS) -static uint32_t g_test_backing_store_unmaps_before_failure = UINT32_MAX; -static callback_func_t g_test_before_backing_map = nullptr; -static callback_func_t g_test_backing_read = nullptr; +static uint32_t g_test_backing_store_unmaps_before_failure = UINT32_MAX; +static callback_func_t g_test_before_backing_map = nullptr; +static callback_func_t g_test_backing_read = nullptr; +static uint64_t g_test_backing_bytes_zeroed = 0; +static std::atomic g_test_free_range_visits {0}; #endif #include "memoryAddressSpace.inc" @@ -643,31 +633,68 @@ static uint32_t g_test_physical_memory_unmaps_before_failure = UINT32_MAX; static bool g_test_fail_next_fixed_reserve_range_add = false; #endif +static void RemoveFreeRange(std::map& ranges, uint64_t start, uint64_t size) { + const auto end = start + size; + auto range = ranges.upper_bound(start); + if (range != ranges.begin()) { + --range; + } + while (range != ranges.end() && range->first < end) { + const auto range_start = range->first; + const auto range_end = range_start + range->second; + if (range_end <= start) { + ++range; + continue; + } + range = ranges.erase(range); + if (range_start < start) { + ranges.emplace(range_start, start - range_start); + } + if (end < range_end) { + ranges.emplace(end, range_end - end); + } + } +} + +static void AddFreeRange(std::map& ranges, uint64_t start, uint64_t size) { + auto end = start + size; + auto next = ranges.lower_bound(start); + if (next != ranges.begin()) { + auto previous = std::prev(next); + if (previous->first + previous->second >= start) { + start = previous->first; + end = std::max(end, previous->first + previous->second); + next = ranges.erase(previous); + } + } + while (next != ranges.end() && next->first <= end) { + end = std::max(end, next->first + next->second); + next = ranges.erase(next); + } + ranges.emplace(start, end - start); +} + class PhysicalMemory { public: enum class AllocationKind { Direct, Pooled, Automatic }; using PhysicalRanges = std::vector>; struct AllocatedBlock { - uint64_t start_addr; - uint64_t size; - uint64_t map_vaddr; - uint64_t map_size; - uint64_t host_vaddr; - uint64_t host_size; - int prot; - VirtualMemory::Mode mode; - GpuAccessMode gpu_mode; - int memory_type; - AllocationKind kind; - char name[KERNEL_MAXIMUM_NAME_LENGTH]; + uint64_t start_addr; + uint64_t size; + int memory_type; + AllocationKind kind; + }; + + struct Mapping { + uint64_t start_addr; + uint64_t size; }; PhysicalMemory() { EXIT_NOT_IMPLEMENTED(!Common::Thread::IsMainThread()); m_free.emplace(0, Size()); } - virtual ~PhysicalMemory() = default; KYTY_CLASS_NO_COPY(PhysicalMemory); @@ -681,62 +708,47 @@ public: void RestoreAutomatic(const PhysicalRanges& ranges); bool Available(uint64_t search_start, uint64_t search_end, size_t alignment, uint64_t* phys_addr_out, uint64_t* size_out); - bool Release(uint64_t start, size_t len, uint64_t* vaddr, uint64_t* size, - GpuAccessMode* gpu_mode); - bool Map(uint64_t vaddr, uint64_t phys_addr, size_t len, int prot, VirtualMemory::Mode mode, - GpuAccessMode gpu_mode, const char* name, int memory_type); - bool Unmap(uint64_t vaddr, uint64_t size, GpuAccessMode* gpu_mode, - uint64_t* host_vaddr_to_release = nullptr); - bool Find(uint64_t vaddr, uint64_t* base_addr, size_t* len, int* prot, - VirtualMemory::Mode* mode, GpuAccessMode* gpu_mode); - bool Find(uint64_t phys_addr, bool next, PhysicalMemory::AllocatedBlock* out); + bool Release(uint64_t start, size_t len); + bool Map(uint64_t vaddr, uint64_t phys_addr, size_t len); + bool Unmap(uint64_t vaddr, uint64_t size); + bool Find(uint64_t phys_addr, PhysicalMemory::AllocatedBlock* out); bool CanMapDirect(uint64_t phys_addr, size_t len); bool ReleasePoolExpansion(uint64_t phys_addr, size_t len); bool GetAllocatedSpan(uint64_t phys_addr, size_t len, bool skip_gaps, std::vector* blocks); - std::vector FindMappings(uint64_t phys_addr, size_t len); - void ProtectMapping(uint64_t vaddr, uint64_t size, int prot, VirtualMemory::Mode mode, - GpuAccessMode gpu_mode); - void SetVirtualRangeName(uint64_t vaddr, uint64_t len, const char* name); + std::vector FindMappings(uint64_t phys_addr, size_t len); [[nodiscard]] Common::Mutex& GetMutex() { return m_mutex; } [[nodiscard]] const std::map& GetPhysicalBlocks() const { return m_physical; } - [[nodiscard]] const std::vector& GetMappings() const { return m_mappings; } + [[nodiscard]] const std::map& GetMappings() const { return m_mappings; } private: - static void RemoveFreeRange(std::map& ranges, uint64_t start, - uint64_t size); - static void AddFreeRange(std::map& ranges, uint64_t start, uint64_t size); - void ReclaimAutomatic(uint64_t start, uint64_t size); + void ReclaimAutomatic(uint64_t start, uint64_t size); + std::map::iterator SplitMapping(uint64_t vaddr); std::map m_physical; std::map m_free; std::map m_automatic_free; - std::vector m_mappings; - Common::Mutex m_mutex; + // Track backing ownership while a transaction changes views before publishing VirtualRanges. + // Names, protections and memory types live only in VirtualRanges. Physical offsets may alias. + std::map m_mappings; + Common::Mutex m_mutex; }; class FlexibleMemory { public: - struct AllocatedBlock { - uint64_t map_vaddr; - uint64_t map_size; - uint64_t backing_offset; - uint64_t host_vaddr; - uint64_t host_size; - int prot; - VirtualMemory::Mode mode; - GpuAccessMode gpu_mode; - char name[KERNEL_MAXIMUM_NAME_LENGTH]; + struct Mapping { + uint64_t vaddr; + uint64_t size; + uint64_t backing_offset; }; FlexibleMemory() { EXIT_NOT_IMPLEMENTED(!Common::Thread::IsMainThread()); m_free.emplace(PhysicalMemory::Size(), Size()); } - virtual ~FlexibleMemory() = default; KYTY_CLASS_NO_COPY(FlexibleMemory); @@ -745,63 +757,45 @@ public: void Charge(uint64_t size); void Release(uint64_t size); - bool Map(uint64_t vaddr, size_t len, int prot, VirtualMemory::Mode mode, GpuAccessMode gpu_mode, - const char* name); - bool Unmap(uint64_t vaddr, uint64_t size, GpuAccessMode* gpu_mode, - uint64_t* host_vaddr_to_release = nullptr); - bool Find(uint64_t vaddr, uint64_t* base_addr, size_t* len, int* prot, - VirtualMemory::Mode* mode, GpuAccessMode* gpu_mode); - bool Snapshot(uint64_t vaddr, uint64_t size, std::vector* blocks); - bool Restore(const std::vector& blocks); - void Protect(uint64_t vaddr, uint64_t size, int prot, VirtualMemory::Mode mode, - GpuAccessMode gpu_mode); - void SetVirtualRangeName(uint64_t vaddr, uint64_t len, const char* name); + bool Map(uint64_t vaddr, size_t len, VirtualMemory::Mode mode); + bool Unmap(uint64_t vaddr, uint64_t size); + bool Snapshot(uint64_t vaddr, uint64_t size, std::vector* blocks); + bool Restore(const std::vector& blocks, VirtualMemory::Mode mode); [[nodiscard]] Common::Mutex& GetMutex() { return m_mutex; } - [[nodiscard]] const std::vector& GetBlocks() const { return m_allocated; } + [[nodiscard]] const std::map& GetMappings() const { return m_mappings; } +#if defined(KYTY_VIRTUAL_MEMORY_ALLOCATION_TESTS) + uint64_t TestBackingHighWater() const { return m_backing_high_water; } +#endif private: - void ConsumeFreeRange(std::map::iterator range, uint64_t start, - uint64_t size); - void AddFreeRange(uint64_t start, uint64_t size); - - std::vector m_allocated; + std::map m_mappings; std::map m_free; uint64_t m_allocated_total = 0; - Common::Mutex m_mutex; + // Only never-issued backing is guaranteed to retain its initial zero contents. + uint64_t m_backing_high_water = PhysicalMemory::Size(); + Common::Mutex m_mutex; }; +// All access is serialized by g_memory_operation_mutex. class PooledMemory { public: struct Mapping { - uint64_t vaddr; - uint64_t size; - uint64_t phys_addr; - GpuAccessMode gpu_mode; + uint64_t vaddr; + uint64_t size; + uint64_t phys_addr; }; void Expand(uint64_t phys_addr, uint64_t size); bool ReleaseExpansion(uint64_t phys_addr, uint64_t size); - bool Allocate(uint64_t vaddr, uint64_t size, GpuAccessMode gpu_mode, - std::vector* mappings); - bool Query(uint64_t vaddr, uint64_t size, std::vector* mappings); - bool Release(uint64_t vaddr, uint64_t size, GpuAccessMode* gpu_mode); + bool Allocate(uint64_t vaddr, uint64_t size, std::vector* mappings); + bool Release(uint64_t vaddr, uint64_t size); [[nodiscard]] uint64_t Available(); - [[nodiscard]] std::vector GetMappings(); + [[nodiscard]] const std::map& GetMappings() const { return m_mappings; } private: - struct PhysicalRange { - uint64_t start; - uint64_t size; - }; - - void AddFreeUnlocked(uint64_t start, uint64_t size); - bool QueryUnlocked(uint64_t vaddr, uint64_t size, std::vector* mappings) const; - - std::vector m_free; - std::vector m_expansions; - std::vector m_mappings; - Common::Mutex m_mutex; + std::map m_free; + std::map m_mappings; }; static std::unique_ptr g_physical_memory; @@ -1067,20 +1061,18 @@ void RegisterCallbacks(callback_func_t alloc_func, callback_func_t free_func) { g_free_callback = free_func; g_physical_memory->GetMutex().Lock(); - for (const auto& b: g_physical_memory->GetMappings()) { - if (b.map_vaddr != 0 && b.map_size != 0) { - g_alloc_callback(b.map_vaddr, b.map_size); - } + for (const auto& [vaddr, mapping]: g_physical_memory->GetMappings()) { + g_alloc_callback(vaddr, mapping.size); } g_physical_memory->GetMutex().Unlock(); g_flexible_memory->GetMutex().Lock(); - for (const auto& b: g_flexible_memory->GetBlocks()) { - g_alloc_callback(b.map_vaddr, b.map_size); + for (const auto& [_, block]: g_flexible_memory->GetMappings()) { + g_alloc_callback(block.vaddr, block.size); } g_flexible_memory->GetMutex().Unlock(); - for (const auto& mapping: g_pooled_memory->GetMappings()) { + for (const auto& [_, mapping]: g_pooled_memory->GetMappings()) { g_alloc_callback(mapping.vaddr, mapping.size); } } @@ -1115,11 +1107,6 @@ bool PhysicalMemory::Alloc(uint64_t search_start, uint64_t search_end, size_t le AllocatedBlock b {}; b.size = len; b.start_addr = free_pos; - b.gpu_mode = GpuAccessMode::NoAccess; - b.map_size = 0; - b.map_vaddr = 0; - b.prot = 0; - b.mode = VirtualMemory::Mode::NoAccess; b.memory_type = memory_type; b.kind = kind; @@ -1152,7 +1139,10 @@ bool PhysicalMemory::Available(uint64_t search_start, uint64_t search_end, size_ uint64_t best_addr = 0; uint64_t best_size = 0; - for (const auto& [range_start, range_size]: m_free) { + auto range = m_free.upper_bound(search_start); + if (range != m_free.begin()) --range; + for (; range != m_free.end(); ++range) { + const auto [range_start, range_size] = *range; if (range_start >= search_end) { break; } @@ -1176,49 +1166,6 @@ bool PhysicalMemory::Available(uint64_t search_start, uint64_t search_end, size_ return true; } -void PhysicalMemory::RemoveFreeRange(std::map& ranges, uint64_t start, - uint64_t size) { - const auto end = start + size; - auto range = ranges.upper_bound(start); - if (range != ranges.begin()) { - --range; - } - while (range != ranges.end() && range->first < end) { - const auto range_start = range->first; - const auto range_end = range_start + range->second; - if (range_end <= start) { - ++range; - continue; - } - range = ranges.erase(range); - if (range_start < start) { - ranges.emplace(range_start, start - range_start); - } - if (end < range_end) { - ranges.emplace(end, range_end - end); - } - } -} - -void PhysicalMemory::AddFreeRange(std::map& ranges, uint64_t start, - uint64_t size) { - auto end = start + size; - auto next = ranges.lower_bound(start); - if (next != ranges.begin()) { - auto previous = std::prev(next); - if (previous->first + previous->second >= start) { - start = previous->first; - end = std::max(end, previous->first + previous->second); - next = ranges.erase(previous); - } - } - while (next != ranges.end() && next->first <= end) { - end = std::max(end, next->first + next->second); - next = ranges.erase(next); - } - ranges.emplace(start, end - start); -} - bool PhysicalMemory::ReserveAutomatic(uint64_t size, PhysicalRanges* ranges) { Common::LockGuard lock(m_mutex); for (const auto& [start, available]: m_automatic_free) { @@ -1261,7 +1208,7 @@ void PhysicalMemory::ReclaimAutomatic(uint64_t start, uint64_t size) { automatic = true; } if (automatic) { - for (const auto& mapping: m_mappings) { + for (const auto& [vaddr, mapping]: m_mappings) { const auto first = std::max(start, mapping.start_addr); const auto last = std::min(end, mapping.start_addr + mapping.size); if (first < last) { @@ -1271,126 +1218,59 @@ void PhysicalMemory::ReclaimAutomatic(uint64_t start, uint64_t size) { } } -bool PhysicalMemory::Release(uint64_t start, size_t len, uint64_t* vaddr, uint64_t* size, - GpuAccessMode* gpu_mode) { - EXIT_IF(vaddr == nullptr); - EXIT_IF(size == nullptr); - EXIT_IF(gpu_mode == nullptr); - +bool PhysicalMemory::Release(uint64_t start, size_t len) { Common::LockGuard lock(m_mutex); auto next = m_physical.upper_bound(start); if (next == m_physical.begin()) { return false; } - auto it = std::prev(next); - auto& b = it->second; - if (b.kind == AllocationKind::Pooled || start < b.start_addr || - start >= b.start_addr + b.size || len > b.start_addr + b.size - start) { + auto it = std::prev(next); + const auto block = it->second; + if (len == 0 || block.kind == AllocationKind::Pooled || start < block.start_addr || + start >= block.start_addr + block.size || len > block.start_addr + block.size - start) { return false; } - if (b.kind == AllocationKind::Automatic) { + if (block.kind == AllocationKind::Automatic) { RemoveFreeRange(m_automatic_free, start, len); } - - if (start == b.start_addr && len == b.size) { - *vaddr = b.map_vaddr; - *size = b.map_size; - *gpu_mode = b.gpu_mode; - - m_physical.erase(it); - AddFreeRange(m_free, start, len); - return true; + m_physical.erase(it); + if (block.start_addr < start) { + auto left = block; + left.size = start - block.start_addr; + m_physical.emplace(left.start_addr, left); } - if (start > b.start_addr && start + len < b.start_addr + b.size) { - auto old_start = b.start_addr; - auto old_end = b.start_addr + b.size; - - *vaddr = (b.map_vaddr != 0 ? b.map_vaddr + (start - old_start) : 0); - *size = (b.map_vaddr != 0 ? len : 0); - *gpu_mode = b.gpu_mode; - - AllocatedBlock right = b; - right.start_addr = start + len; - right.size = old_end - right.start_addr; - if (right.map_vaddr != 0) { - right.map_vaddr += right.start_addr - old_start; - right.map_size = right.size; - } - - b.size = start - old_start; - if (b.map_vaddr != 0) { - b.map_size = b.size; - } - + if (start + len < block.start_addr + block.size) { + auto right = block; + right.start_addr = start + len; + right.size = block.start_addr + block.size - right.start_addr; m_physical.emplace(right.start_addr, right); - AddFreeRange(m_free, start, len); - return true; } - if (start == b.start_addr && len < b.size) { - *vaddr = b.map_vaddr; - *size = (b.map_vaddr != 0 ? len : 0); - *gpu_mode = b.gpu_mode; - - AllocatedBlock remaining = b; - m_physical.erase(it); - remaining.start_addr += len; - remaining.size -= len; - if (remaining.map_vaddr != 0) { - remaining.map_vaddr += len; - remaining.map_size -= len; - } - m_physical.emplace(remaining.start_addr, remaining); - AddFreeRange(m_free, start, len); - return true; - } - if (start > b.start_addr && start + len == b.start_addr + b.size) { - *vaddr = (b.map_vaddr != 0 ? b.map_vaddr + (start - b.start_addr) : 0); - *size = (b.map_vaddr != 0 ? len : 0); - *gpu_mode = b.gpu_mode; - - b.size = start - b.start_addr; - if (b.map_vaddr != 0) { - b.map_size = b.size; - } - AddFreeRange(m_free, start, len); - return true; - } - - return false; + AddFreeRange(m_free, start, len); + return true; } -bool PhysicalMemory::Map(uint64_t vaddr, uint64_t phys_addr, size_t len, int prot, - VirtualMemory::Mode mode, GpuAccessMode gpu_mode, const char* name, - int memory_type) { +bool PhysicalMemory::Map(uint64_t vaddr, uint64_t phys_addr, size_t len) { Common::LockGuard lock(m_mutex); - if (!CanMapDirect(phys_addr, len)) { + if (vaddr == 0 || len == 0 || UINT64_MAX - vaddr < len || !CanMapDirect(phys_addr, len)) { + return false; + } + auto next = m_mappings.lower_bound(vaddr); + if ((next != m_mappings.end() && next->first < vaddr + len) || + (next != m_mappings.begin() && + std::prev(next)->first + std::prev(next)->second.size > vaddr)) { return false; } - AllocatedBlock mapping {}; - mapping.start_addr = phys_addr; - mapping.size = len; - mapping.map_vaddr = vaddr; - mapping.map_size = len; - mapping.host_vaddr = vaddr; - mapping.host_size = len; - mapping.prot = prot; - mapping.mode = mode; - mapping.gpu_mode = gpu_mode; - mapping.memory_type = memory_type; - CopyVirtualRangeName(mapping.name, name); - m_mappings.push_back(mapping); + m_mappings.emplace_hint(next, vaddr, Mapping {phys_addr, len}); RemoveFreeRange(m_automatic_free, phys_addr, len); - return true; } bool PhysicalMemory::CanMapDirect(uint64_t phys_addr, size_t len) { - std::vector blocks; - return GetAllocatedSpan(phys_addr, len, false, &blocks); + return GetAllocatedSpan(phys_addr, len, false, nullptr); } bool PhysicalMemory::ReleasePoolExpansion(uint64_t phys_addr, size_t len) { @@ -1407,8 +1287,9 @@ bool PhysicalMemory::ReleasePoolExpansion(uint64_t phys_addr, size_t len) { bool PhysicalMemory::GetAllocatedSpan(uint64_t phys_addr, size_t len, bool skip_gaps, std::vector* blocks) { - EXIT_IF(blocks == nullptr); - blocks->clear(); + if (blocks != nullptr) { + blocks->clear(); + } if (len == 0 || UINT64_MAX - phys_addr < len) { return false; } @@ -1433,13 +1314,33 @@ bool PhysicalMemory::GetAllocatedSpan(uint64_t phys_addr, size_t len, bool skip_ part.start_addr = std::max(current, part.start_addr); current = std::min(end, block_end); part.size = current - part.start_addr; - blocks->push_back(part); + if (blocks != nullptr) { + blocks->push_back(part); + } } return skip_gaps || current == end; } -bool PhysicalMemory::Unmap(uint64_t vaddr, uint64_t size, GpuAccessMode* gpu_mode, - uint64_t* host_vaddr_to_release) { +// Return the first mapping beginning at or after vaddr, splitting a containing interval. +std::map::iterator PhysicalMemory::SplitMapping(uint64_t vaddr) { + auto next = m_mappings.lower_bound(vaddr); + if (next != m_mappings.end() && next->first == vaddr) { + return next; + } + if (next != m_mappings.begin()) { + auto previous = std::prev(next); + const auto offset = vaddr - previous->first; + if (offset < previous->second.size) { + const Mapping right {previous->second.start_addr + offset, + previous->second.size - offset}; + next = m_mappings.emplace_hint(next, vaddr, right); + previous->second.size = offset; + } + } + return next; +} + +bool PhysicalMemory::Unmap(uint64_t vaddr, uint64_t size) { #if defined(KYTY_VIRTUAL_MEMORY_ALLOCATION_TESTS) if (g_test_physical_memory_unmaps_before_failure == 0) { g_test_physical_memory_unmaps_before_failure = UINT32_MAX; @@ -1450,112 +1351,36 @@ bool PhysicalMemory::Unmap(uint64_t vaddr, uint64_t size, GpuAccessMode* gpu_mod } #endif - EXIT_IF(gpu_mode == nullptr); - - Common::LockGuard lock(m_mutex); - - if (host_vaddr_to_release != nullptr) { - *host_vaddr_to_release = 0; - } - - auto set_host_release_if_last = [this, host_vaddr_to_release](uint64_t host_vaddr, - uint64_t host_size) { - if (host_vaddr_to_release == nullptr || host_vaddr == 0 || host_size == 0) { - return; - } - const bool still_mapped = std::any_of( - m_mappings.begin(), m_mappings.end(), [host_vaddr, host_size](const auto& block) { - return block.host_vaddr == host_vaddr && block.host_size == host_size; - }); - if (!still_mapped) { - *host_vaddr_to_release = host_vaddr; - } - }; - - for (size_t index = 0; index < m_mappings.size(); ++index) { - auto& b = m_mappings[index]; - if (vaddr < b.map_vaddr || vaddr >= b.map_vaddr + b.map_size || - size > b.map_vaddr + b.map_size - vaddr) { - continue; - } - *gpu_mode = b.gpu_mode; - const auto phys_addr = b.start_addr + vaddr - b.map_vaddr; - const auto host_vaddr = b.host_vaddr; - const auto host_size = b.host_size; - if (vaddr == b.map_vaddr && size == b.map_size) { - m_mappings.erase(m_mappings.begin() + static_cast(index)); - set_host_release_if_last(host_vaddr, host_size); - } else if (vaddr > b.map_vaddr && vaddr + size < b.map_vaddr + b.map_size) { - AllocatedBlock right = b; - right.start_addr += (vaddr + size) - b.map_vaddr; - right.size = b.map_vaddr + b.map_size - (vaddr + size); - right.map_size = right.size; - right.map_vaddr = vaddr + size; - b.size = vaddr - b.map_vaddr; - b.map_size = b.size; - m_mappings.push_back(right); - } else if (vaddr == b.map_vaddr) { - b.start_addr += size; - b.size -= size; - b.map_vaddr += size; - b.map_size -= size; - } else { - b.size = vaddr - b.map_vaddr; - b.map_size = b.size; - } - ReclaimAutomatic(phys_addr, size); - return true; - } - - return false; -} - -void PhysicalMemory::ProtectMapping(uint64_t vaddr, uint64_t size, int prot, - VirtualMemory::Mode mode, GpuAccessMode gpu_mode) { Common::LockGuard lock(m_mutex); if (size == 0 || UINT64_MAX - vaddr < size) { - return; + return false; } - - const auto end = vaddr + size; - std::vector updated; - updated.reserve(m_mappings.size() + 2); - for (const auto& block: m_mappings) { - const auto block_end = block.map_vaddr + block.map_size; - if (!VirtualRangesOverlap(vaddr, size, block.map_vaddr, block.map_size)) { - updated.push_back(block); - continue; - } - const auto overlap_start = std::max(vaddr, block.map_vaddr); - const auto overlap_end = std::min(end, block_end); - if (block.map_vaddr < overlap_start) { - auto left = block; - left.size = overlap_start - block.map_vaddr; - left.map_size = left.size; - updated.push_back(left); - } - auto middle = block; - middle.start_addr = block.start_addr + overlap_start - block.map_vaddr; - middle.size = overlap_end - overlap_start; - middle.map_vaddr = overlap_start; - middle.map_size = middle.size; - middle.prot = prot; - middle.mode = mode; - middle.gpu_mode = gpu_mode; - updated.push_back(middle); - if (overlap_end < block_end) { - auto right = block; - right.start_addr = block.start_addr + overlap_end - block.map_vaddr; - right.size = block_end - overlap_end; - right.map_vaddr = overlap_end; - right.map_size = right.size; - updated.push_back(right); - } + const auto end = vaddr + size; + auto first = m_mappings.upper_bound(vaddr); + if (first != m_mappings.begin()) { + --first; } - m_mappings = std::move(updated); + // Rollback can leave several ownership intervals inside one semantic virtual range. + // Validate the complete span before changing any ownership records. + auto current = vaddr; + for (auto it = first; current < end; ++it) { + if (it == m_mappings.end() || it->first > current || + current >= it->first + it->second.size) { + return false; + } + current = std::min(end, it->first + it->second.size); + } + const auto last = SplitMapping(end); + first = SplitMapping(vaddr); + while (first != last) { + const auto removed = first->second; + first = m_mappings.erase(first); + ReclaimAutomatic(removed.start_addr, removed.size); + } + return true; } -bool PhysicalMemory::Find(uint64_t phys_addr, bool next, AllocatedBlock* out) { +bool PhysicalMemory::Find(uint64_t phys_addr, AllocatedBlock* out) { EXIT_IF(out == nullptr); Common::LockGuard lock(m_mutex); @@ -1569,410 +1394,171 @@ bool PhysicalMemory::Find(uint64_t phys_addr, bool next, AllocatedBlock* out) { } } - if (next && following != m_physical.end()) { - *out = following->second; - return true; - } - return false; } -std::vector PhysicalMemory::FindMappings(uint64_t phys_addr, - size_t len) { +std::vector PhysicalMemory::FindMappings(uint64_t phys_addr, size_t len) { Common::LockGuard lock(m_mutex); - std::vector mappings; - for (const auto& m: m_mappings) { - if (!VirtualRangesOverlap(phys_addr, len, m.start_addr, m.size)) { + std::vector mappings; + std::vector parts; + for (const auto& [vaddr, mapping]: m_mappings) { + if (!VirtualRangesOverlap(phys_addr, len, mapping.start_addr, mapping.size)) { continue; } - - const uint64_t overlap_start = std::max(phys_addr, m.start_addr); - const uint64_t overlap_end = std::min(phys_addr + len, m.start_addr + m.size); - AllocatedBlock part = m; - part.start_addr = overlap_start; - part.size = overlap_end - overlap_start; - part.map_vaddr += overlap_start - m.start_addr; - part.map_size = part.size; - mappings.push_back(part); + const auto overlap_start = std::max(phys_addr, mapping.start_addr); + const auto overlap_end = std::min(phys_addr + len, mapping.start_addr + mapping.size); + EXIT_IF(!g_virtual_ranges->QuerySpan(vaddr + overlap_start - mapping.start_addr, + overlap_end - overlap_start, &parts)); + for (const auto& part: parts) { + EXIT_IF(part.type != VirtualRangeType::Direct); + mappings.push_back(part); + } } - - std::sort(mappings.begin(), mappings.end(), - [](const auto& a, const auto& b) { return a.map_vaddr < b.map_vaddr; }); return mappings; } -bool PhysicalMemory::Find(uint64_t vaddr, uint64_t* base_addr, size_t* len, int* prot, - VirtualMemory::Mode* mode, GpuAccessMode* gpu_mode) { +bool FlexibleMemory::Map(uint64_t vaddr, size_t len, VirtualMemory::Mode mode) { Common::LockGuard lock(m_mutex); - - return std::any_of(m_mappings.begin(), m_mappings.end(), - [vaddr, base_addr, len, prot, mode, gpu_mode](auto& b) { - if (vaddr >= b.map_vaddr && vaddr < b.map_vaddr + b.map_size) { - if (base_addr != nullptr) { - *base_addr = b.map_vaddr; - } - if (len != nullptr) { - *len = b.map_size; - } - if (prot != nullptr) { - *prot = b.prot; - } - if (mode != nullptr) { - *mode = b.mode; - } - if (gpu_mode != nullptr) { - *gpu_mode = b.gpu_mode; - } - - return true; - } - return false; - }); -} - -void PhysicalMemory::SetVirtualRangeName(uint64_t vaddr, uint64_t len, const char* name) { - Common::LockGuard lock(m_mutex); - - for (auto& b: m_mappings) { - if (VirtualRangesOverlap(vaddr, len, b.map_vaddr, b.map_size)) { - CopyVirtualRangeName(b.name, name); - } - } -} - -bool FlexibleMemory::Map(uint64_t vaddr, size_t len, int prot, VirtualMemory::Mode mode, - GpuAccessMode gpu_mode, const char* name) { - Common::LockGuard lock(m_mutex); - - const auto available = (Size() >= m_allocated_total ? Size() - m_allocated_total : 0); - if (len == 0 || len > available) { - LOGF_COLOR(Log::Color::Red, - "\t flexible memory exhausted: configured = 0x%016" PRIx64 - ", allocated = 0x%016" PRIx64 ", available = 0x%016" PRIx64 - ", requested = 0x%016" PRIx64 "\n", - Size(), m_allocated_total, available, static_cast(len)); + if (len == 0 || len > Size() - std::min(Size(), m_allocated_total)) { return false; } - std::vector blocks; - auto current = vaddr; - auto remaining = static_cast(len); + std::vector blocks; + auto current = vaddr; + auto remaining = static_cast(len); for (const auto& [backing_offset, free_size]: m_free) { - if (remaining == 0) { - break; - } - const auto chunk = std::min(remaining, free_size); - AllocatedBlock block {}; - block.map_vaddr = current; - block.map_size = chunk; - block.backing_offset = backing_offset; - block.host_vaddr = vaddr; - block.host_size = len; - block.prot = prot; - block.mode = mode; - block.gpu_mode = gpu_mode; - CopyVirtualRangeName(block.name, name); - blocks.push_back(block); + if (remaining == 0) break; + const auto chunk = std::min(remaining, free_size); + blocks.push_back({current, chunk, backing_offset}); current += chunk; remaining -= chunk; } - if (remaining != 0) { - return false; - } + if (remaining != 0) return false; - std::vector mapped; + size_t mapped = 0; for (const auto& block: blocks) { - if (!g_guest_address_space->ZeroBacking(block.backing_offset, block.map_size)) { - for (auto it = mapped.rbegin(); it != mapped.rend(); ++it) { - EXIT_IF(!g_guest_address_space->UnmapBacking(it->map_vaddr, it->map_size)); + const auto reused = block.backing_offset < m_backing_high_water + ? std::min(block.size, m_backing_high_water - block.backing_offset) + : 0; + const bool cleared = + reused == 0 || g_guest_address_space->ZeroBacking(block.backing_offset, reused); + // Never roll this back: a successfully exposed view can be written before a later failure. + m_backing_high_water = std::max(m_backing_high_water, block.backing_offset + block.size); + if (!cleared || !g_guest_address_space->MapBacking(block.vaddr, block.size, + block.backing_offset, mode)) { + while (mapped != 0) { + const auto& previous = blocks[--mapped]; + EXIT_IF(!g_guest_address_space->UnmapBacking(previous.vaddr, previous.size)); } return false; } - if (!g_guest_address_space->MapBacking(block.map_vaddr, block.map_size, - block.backing_offset, block.mode)) { - for (auto it = mapped.rbegin(); it != mapped.rend(); ++it) { - EXIT_IF(!g_guest_address_space->UnmapBacking(it->map_vaddr, it->map_size)); - } - return false; - } - mapped.push_back(block); + ++mapped; } for (const auto& block: blocks) { - auto next = m_free.upper_bound(block.backing_offset); - EXIT_IF(next == m_free.begin()); - auto range = std::prev(next); - EXIT_IF(block.backing_offset < range->first || - block.map_size > range->first + range->second - block.backing_offset); - ConsumeFreeRange(range, block.backing_offset, block.map_size); - m_allocated.push_back(block); + RemoveFreeRange(m_free, block.backing_offset, block.size); + EXIT_IF(!m_mappings.emplace(block.vaddr, block).second); } - std::sort(m_allocated.begin(), m_allocated.end(), - [](const auto& left, const auto& right) { return left.map_vaddr < right.map_vaddr; }); m_allocated_total += len; - return true; } -bool FlexibleMemory::Unmap(uint64_t vaddr, uint64_t size, GpuAccessMode* gpu_mode, - uint64_t* host_vaddr_to_release) { - EXIT_IF(gpu_mode == nullptr); - +bool FlexibleMemory::Unmap(uint64_t vaddr, uint64_t size) { Common::LockGuard lock(m_mutex); + if (size == 0 || UINT64_MAX - vaddr < size) return false; - if (host_vaddr_to_release != nullptr) { - *host_vaddr_to_release = 0; - } - if (size == 0 || UINT64_MAX - vaddr < size) { - return false; + const auto end = vaddr + size; + auto first = m_mappings.upper_bound(vaddr); + if (first == m_mappings.begin()) return false; + --first; + auto current = vaddr; + for (auto it = first; it != m_mappings.end() && current < end; ++it) { + const auto& block = it->second; + if (block.vaddr > current || current >= block.vaddr + block.size) return false; + current = std::min(end, block.vaddr + block.size); } + if (current != end || !g_guest_address_space->UnmapBacking(vaddr, size)) return false; - const auto end = vaddr + size; - auto current = vaddr; - bool found = false; - for (const auto& block: m_allocated) { - if (block.map_vaddr + block.map_size <= current) { - continue; + for (auto it = first; it != m_mappings.end() && it->first < end;) { + const auto block = it->second; + const auto block_end = block.vaddr + block.size; + const auto cut_start = std::max(vaddr, block.vaddr); + const auto cut_end = std::min(end, block_end); + it = m_mappings.erase(it); + AddFreeRange(m_free, block.backing_offset + cut_start - block.vaddr, cut_end - cut_start); + if (block.vaddr < cut_start) { + m_mappings.emplace( + block.vaddr, Mapping {block.vaddr, cut_start - block.vaddr, block.backing_offset}); } - if (block.map_vaddr > current || block.map_vaddr >= end) { - break; - } - if (!found) { - *gpu_mode = block.gpu_mode; - found = true; - } - current = std::min(end, block.map_vaddr + block.map_size); - if (current == end) { - break; + if (cut_end < block_end) { + m_mappings.emplace(cut_end, Mapping {cut_end, block_end - cut_end, + block.backing_offset + cut_end - block.vaddr}); } } - if (!found || current != end || !g_guest_address_space->UnmapBacking(vaddr, size)) { - return false; - } - - std::vector remaining; - remaining.reserve(m_allocated.size() + 1); - uint64_t removed = 0; - for (const auto& block: m_allocated) { - const auto block_end = block.map_vaddr + block.map_size; - if (!VirtualRangesOverlap(vaddr, size, block.map_vaddr, block.map_size)) { - remaining.push_back(block); - continue; - } - const auto overlap_start = std::max(vaddr, block.map_vaddr); - const auto overlap_end = std::min(end, block_end); - const auto overlap_size = overlap_end - overlap_start; - AddFreeRange(block.backing_offset + overlap_start - block.map_vaddr, overlap_size); - removed += overlap_size; - - if (block.map_vaddr < overlap_start) { - auto left = block; - left.map_size = overlap_start - block.map_vaddr; - remaining.push_back(left); - } - if (overlap_end < block_end) { - auto right = block; - right.map_vaddr = overlap_end; - right.map_size = block_end - overlap_end; - right.backing_offset = block.backing_offset + overlap_end - block.map_vaddr; - remaining.push_back(right); - } - } - EXIT_IF(removed != size || removed > m_allocated_total); - m_allocated = std::move(remaining); - m_allocated_total -= removed; + EXIT_IF(size > m_allocated_total); + m_allocated_total -= size; return true; } -void FlexibleMemory::ConsumeFreeRange(std::map::iterator range, uint64_t start, - uint64_t size) { - const auto range_start = range->first; - const auto range_end = range->first + range->second; - m_free.erase(range); - if (range_start < start) { - m_free.emplace(range_start, start - range_start); - } - if (start + size < range_end) { - m_free.emplace(start + size, range_end - start - size); - } -} - -void FlexibleMemory::AddFreeRange(uint64_t start, uint64_t size) { - auto end = start + size; - auto next = m_free.lower_bound(start); - if (next != m_free.begin()) { - auto previous = std::prev(next); - if (previous->first + previous->second >= start) { - start = previous->first; - end = std::max(end, previous->first + previous->second); - next = m_free.erase(previous); - } - } - while (next != m_free.end() && next->first <= end) { - end = std::max(end, next->first + next->second); - next = m_free.erase(next); - } - m_free.emplace(start, end - start); -} - -bool FlexibleMemory::Snapshot(uint64_t vaddr, uint64_t size, std::vector* blocks) { +bool FlexibleMemory::Snapshot(uint64_t vaddr, uint64_t size, std::vector* blocks) { EXIT_IF(blocks == nullptr); Common::LockGuard lock(m_mutex); blocks->clear(); - if (size == 0 || UINT64_MAX - vaddr < size) { - return false; - } + if (size == 0 || UINT64_MAX - vaddr < size) return false; + auto it = m_mappings.upper_bound(vaddr); + if (it == m_mappings.begin()) return false; + --it; const auto end = vaddr + size; auto current = vaddr; - for (const auto& block: m_allocated) { - const auto block_end = block.map_vaddr + block.map_size; - if (block_end <= current) { - continue; - } - if (block.map_vaddr > current || block.map_vaddr >= end) { - break; - } - const auto part_end = std::min(end, block_end); - auto part = block; - part.map_vaddr = current; - part.map_size = part_end - current; - part.backing_offset += current - block.map_vaddr; - blocks->push_back(part); + for (; it != m_mappings.end() && current < end; ++it) { + const auto& block = it->second; + if (block.vaddr > current || current >= block.vaddr + block.size) break; + const auto part_end = std::min(end, block.vaddr + block.size); + blocks->push_back( + {current, part_end - current, block.backing_offset + current - block.vaddr}); current = part_end; - if (current == end) { - return true; - } } + if (current == end) return true; blocks->clear(); return false; } -bool FlexibleMemory::Restore(const std::vector& blocks) { +bool FlexibleMemory::Restore(const std::vector& blocks, VirtualMemory::Mode mode) { Common::LockGuard lock(m_mutex); - if (blocks.empty()) { - return false; - } + if (blocks.empty()) return false; for (const auto& block: blocks) { auto next = m_free.upper_bound(block.backing_offset); - if (next == m_free.begin()) { - return false; - } + if (next == m_free.begin()) return false; const auto range = std::prev(next); - if (block.backing_offset < range->first || - block.map_size > range->first + range->second - block.backing_offset) { + if (!GuestFreeRangeContains(range->first, range->second, block.backing_offset, + block.size)) { return false; } } - std::vector mapped; + size_t mapped = 0; for (const auto& block: blocks) { - if (!g_guest_address_space->MapBacking(block.map_vaddr, block.map_size, - block.backing_offset, block.mode)) { - for (auto it = mapped.rbegin(); it != mapped.rend(); ++it) { - EXIT_IF(!g_guest_address_space->UnmapBacking(it->map_vaddr, it->map_size)); + if (!g_guest_address_space->MapBacking(block.vaddr, block.size, block.backing_offset, + mode)) { + while (mapped != 0) { + const auto& previous = blocks[--mapped]; + EXIT_IF(!g_guest_address_space->UnmapBacking(previous.vaddr, previous.size)); } return false; } - mapped.push_back(block); + ++mapped; } - for (const auto& block: blocks) { - auto next = m_free.upper_bound(block.backing_offset); - EXIT_IF(next == m_free.begin()); - auto range = std::prev(next); - ConsumeFreeRange(range, block.backing_offset, block.map_size); - m_allocated.push_back(block); - m_allocated_total += block.map_size; + RemoveFreeRange(m_free, block.backing_offset, block.size); + EXIT_IF(!m_mappings.emplace(block.vaddr, block).second); + m_allocated_total += block.size; } - std::sort(m_allocated.begin(), m_allocated.end(), - [](const auto& left, const auto& right) { return left.map_vaddr < right.map_vaddr; }); return true; } -void FlexibleMemory::Protect(uint64_t vaddr, uint64_t size, int prot, VirtualMemory::Mode mode, - GpuAccessMode gpu_mode) { - Common::LockGuard lock(m_mutex); - if (size == 0 || UINT64_MAX - vaddr < size) { - return; - } - - const auto end = vaddr + size; - std::vector updated; - updated.reserve(m_allocated.size() + 2); - for (const auto& block: m_allocated) { - const auto block_end = block.map_vaddr + block.map_size; - if (!VirtualRangesOverlap(vaddr, size, block.map_vaddr, block.map_size)) { - updated.push_back(block); - continue; - } - const auto overlap_start = std::max(vaddr, block.map_vaddr); - const auto overlap_end = std::min(end, block_end); - if (block.map_vaddr < overlap_start) { - auto left = block; - left.map_size = overlap_start - block.map_vaddr; - updated.push_back(left); - } - auto middle = block; - middle.map_vaddr = overlap_start; - middle.map_size = overlap_end - overlap_start; - middle.backing_offset = block.backing_offset + overlap_start - block.map_vaddr; - middle.prot = prot; - middle.mode = mode; - middle.gpu_mode = gpu_mode; - updated.push_back(middle); - if (overlap_end < block_end) { - auto right = block; - right.map_vaddr = overlap_end; - right.map_size = block_end - overlap_end; - right.backing_offset = block.backing_offset + overlap_end - block.map_vaddr; - updated.push_back(right); - } - } - m_allocated = std::move(updated); -} - -bool FlexibleMemory::Find(uint64_t vaddr, uint64_t* base_addr, size_t* len, int* prot, - VirtualMemory::Mode* mode, GpuAccessMode* gpu_mode) { - Common::LockGuard lock(m_mutex); - - return std::any_of(m_allocated.begin(), m_allocated.end(), - [vaddr, base_addr, len, prot, mode, gpu_mode](auto& b) { - if (vaddr >= b.map_vaddr && vaddr < b.map_vaddr + b.map_size) { - if (base_addr != nullptr) { - *base_addr = b.map_vaddr; - } - if (len != nullptr) { - *len = b.map_size; - } - if (prot != nullptr) { - *prot = b.prot; - } - if (mode != nullptr) { - *mode = b.mode; - } - if (gpu_mode != nullptr) { - *gpu_mode = b.gpu_mode; - } - - return true; - } - return false; - }); -} - -void FlexibleMemory::SetVirtualRangeName(uint64_t vaddr, uint64_t len, const char* name) { - Common::LockGuard lock(m_mutex); - - for (auto& b: m_allocated) { - if (VirtualRangesOverlap(vaddr, len, b.map_vaddr, b.map_size)) { - CopyVirtualRangeName(b.name, name); - } - } -} - uint64_t FlexibleMemory::Available() { Common::LockGuard lock(m_mutex); @@ -1992,95 +1578,30 @@ void FlexibleMemory::Release(uint64_t size) { m_allocated_total -= size; } -void PooledMemory::AddFreeUnlocked(uint64_t start, uint64_t size) { - if (size == 0) { - return; - } - - m_free.push_back({start, size}); - std::sort(m_free.begin(), m_free.end(), - [](const auto& left, const auto& right) { return left.start < right.start; }); - - std::vector merged; - for (const auto& range: m_free) { - if (!merged.empty() && range.start <= merged.back().start + merged.back().size) { - const auto end = - std::max(merged.back().start + merged.back().size, range.start + range.size); - merged.back().size = end - merged.back().start; - } else { - merged.push_back(range); - } - } - m_free = std::move(merged); -} - void PooledMemory::Expand(uint64_t phys_addr, uint64_t size) { - Common::LockGuard lock(m_mutex); - m_expansions.push_back({phys_addr, size}); - AddFreeUnlocked(phys_addr, size); + AddFreeRange(m_free, phys_addr, size); } bool PooledMemory::ReleaseExpansion(uint64_t phys_addr, uint64_t size) { - Common::LockGuard lock(m_mutex); - const auto expansion = std::find_if(m_expansions.begin(), m_expansions.end(), - [phys_addr, size](const auto& range) { - return range.start == phys_addr && range.size == size; - }); - if (expansion == m_expansions.end()) { - return false; - } - if (std::any_of(m_mappings.begin(), m_mappings.end(), [phys_addr, size](const auto& mapping) { - return VirtualRangesOverlap(phys_addr, size, mapping.phys_addr, mapping.size); - })) { - return false; - } - - const auto end = phys_addr + size; - const auto free_range = - std::find_if(m_free.begin(), m_free.end(), [phys_addr, end](const auto& r) { - return phys_addr >= r.start && end <= r.start + r.size; - }); - if (free_range == m_free.end()) { - return false; - } - - const auto old = *free_range; - const auto old_end = old.start + old.size; - const auto left_size = phys_addr - old.start; - const auto right_size = old_end - end; - m_free.erase(free_range); - if (left_size != 0) { - m_free.push_back({old.start, left_size}); - } - if (right_size != 0) { - m_free.push_back({end, right_size}); - } - std::sort(m_free.begin(), m_free.end(), - [](const auto& left, const auto& right) { return left.start < right.start; }); - m_expansions.erase(expansion); + auto next = m_free.upper_bound(phys_addr); + if (next == m_free.begin()) return false; + const auto range = std::prev(next); + if (!GuestFreeRangeContains(range->first, range->second, phys_addr, size)) return false; + RemoveFreeRange(m_free, phys_addr, size); return true; } -bool PooledMemory::Allocate(uint64_t vaddr, uint64_t size, GpuAccessMode gpu_mode, - std::vector* mappings) { +bool PooledMemory::Allocate(uint64_t vaddr, uint64_t size, std::vector* mappings) { EXIT_IF(mappings == nullptr); mappings->clear(); - if (vaddr == 0 || size == 0 || UINT64_MAX - vaddr < size) { - return false; - } + if (vaddr == 0 || size == 0 || UINT64_MAX - vaddr < size) return false; - Common::LockGuard lock(m_mutex); - auto free = m_free; - auto current = vaddr; - auto remaining = size; - for (auto& range: free) { - if (remaining == 0 || range.size == 0) { - continue; - } - const auto part_size = std::min(range.size, remaining); - mappings->push_back({current, part_size, range.start, gpu_mode}); - range.start += part_size; - range.size -= part_size; + auto current = vaddr; + auto remaining = size; + for (const auto& [start, free_size]: m_free) { + if (remaining == 0) break; + const auto part_size = std::min(free_size, remaining); + mappings->push_back({current, part_size, start}); current += part_size; remaining -= part_size; } @@ -2088,122 +1609,54 @@ bool PooledMemory::Allocate(uint64_t vaddr, uint64_t size, GpuAccessMode gpu_mod mappings->clear(); return false; } - - free.erase( - std::remove_if(free.begin(), free.end(), [](const auto& range) { return range.size == 0; }), - free.end()); - m_free = std::move(free); - m_mappings.insert(m_mappings.end(), mappings->begin(), mappings->end()); - return true; -} - -bool PooledMemory::QueryUnlocked(uint64_t vaddr, uint64_t size, - std::vector* mappings) const { - EXIT_IF(mappings == nullptr); - mappings->clear(); - if (vaddr == 0 || size == 0 || UINT64_MAX - vaddr < size) { - return false; - } - - const auto end = vaddr + size; - auto current = vaddr; - while (current < end) { - const auto it = - std::find_if(m_mappings.begin(), m_mappings.end(), [current](const auto& m) { - return current >= m.vaddr && current < m.vaddr + m.size; - }); - if (it == m_mappings.end()) { - mappings->clear(); - return false; - } - const auto part_size = std::min(end, it->vaddr + it->size) - current; - mappings->push_back( - {current, part_size, it->phys_addr + (current - it->vaddr), it->gpu_mode}); - current += part_size; + for (const auto& mapping: *mappings) { + RemoveFreeRange(m_free, mapping.phys_addr, mapping.size); + EXIT_IF(!m_mappings.emplace(mapping.vaddr, mapping).second); } return true; } -bool PooledMemory::Query(uint64_t vaddr, uint64_t size, std::vector* mappings) { - Common::LockGuard lock(m_mutex); - return QueryUnlocked(vaddr, size, mappings); -} +bool PooledMemory::Release(uint64_t vaddr, uint64_t size) { + if (vaddr == 0 || size == 0 || UINT64_MAX - vaddr < size) return false; -bool PooledMemory::Release(uint64_t vaddr, uint64_t size, GpuAccessMode* gpu_mode) { - EXIT_IF(gpu_mode == nullptr); - - Common::LockGuard lock(m_mutex); - std::vector released; - if (!QueryUnlocked(vaddr, size, &released)) { - return false; + const auto end = vaddr + size; + auto first = m_mappings.upper_bound(vaddr); + if (first == m_mappings.begin()) return false; + --first; + auto current = vaddr; + for (auto it = first; it != m_mappings.end() && current < end; ++it) { + const auto& mapping = it->second; + if (mapping.vaddr > current || current >= mapping.vaddr + mapping.size) return false; + current = std::min(end, mapping.vaddr + mapping.size); } + if (current != end) return false; - *gpu_mode = released.front().gpu_mode; - const auto end = vaddr + size; - std::vector kept; - for (const auto& mapping: m_mappings) { + for (auto it = first; it != m_mappings.end() && it->first < end;) { + const auto mapping = it->second; const auto mapping_end = mapping.vaddr + mapping.size; const auto cut_start = std::max(vaddr, mapping.vaddr); const auto cut_end = std::min(end, mapping_end); - if (cut_start >= cut_end) { - kept.push_back(mapping); - continue; + it = m_mappings.erase(it); + AddFreeRange(m_free, mapping.phys_addr + cut_start - mapping.vaddr, cut_end - cut_start); + if (mapping.vaddr < vaddr) { + m_mappings.emplace(mapping.vaddr, + Mapping {mapping.vaddr, vaddr - mapping.vaddr, mapping.phys_addr}); } - if (mapping.vaddr < cut_start) { - auto left = mapping; - left.size = cut_start - mapping.vaddr; - kept.push_back(left); + if (end < mapping_end) { + m_mappings.emplace( + end, Mapping {end, mapping_end - end, mapping.phys_addr + end - mapping.vaddr}); } - if (cut_end < mapping_end) { - auto right = mapping; - right.vaddr = cut_end; - right.size = mapping_end - cut_end; - right.phys_addr = mapping.phys_addr + (cut_end - mapping.vaddr); - kept.push_back(right); - } - } - m_mappings = std::move(kept); - for (const auto& mapping: released) { - AddFreeUnlocked(mapping.phys_addr, mapping.size); } return true; } uint64_t PooledMemory::Available() { - Common::LockGuard lock(m_mutex); - uint64_t available = 0; - for (const auto& range: m_free) { - available += range.size; - } + uint64_t available = 0; + for (const auto& [_, size]: m_free) + available += size; return available; } -std::vector PooledMemory::GetMappings() { - Common::LockGuard lock(m_mutex); - return m_mappings; -} - -static bool UnmapPooledBackingTransactional(const std::vector& mappings, - VirtualMemory::Mode mode) { - std::vector removed; - for (const auto& mapping: mappings) { - if (!g_guest_address_space->UnmapBacking(mapping.vaddr, mapping.size)) { - for (auto it = removed.rbegin(); it != removed.rend(); ++it) { - auto failure_reason = GuestBackingStore::FailureReason::None; - const bool restored = g_guest_address_space->MapBacking( - it->vaddr, it->size, it->phys_addr, mode, &failure_reason); - if (!restored) { - EXIT("pooled-memory unmap rollback failed: %s\n", - GuestBackingStore::GetFailureReasonName(failure_reason)); - } - } - return false; - } - removed.push_back(mapping); - } - return true; -} - // The ABI entry points and batch operations hold g_memory_operation_mutex. static int MapFlexibleMemory(void** addr_in_out, size_t len, int prot, int flags, const char* name) { @@ -2248,9 +1701,8 @@ static int MapFlexibleMemory(void** addr_in_out, size_t len, int prot, int flags alignment_shift != 0 ? uint64_t {1} << alignment_shift : PAGE_SIZE; VirtualMemory::Mode mode = VirtualMemory::Mode::NoAccess; - GpuAccessMode gpu_mode = GpuAccessMode::NoAccess; - if (!DecodeMemoryProtection(prot, &mode, &gpu_mode)) { + if (!DecodeMemoryProtection(prot, &mode)) { EXIT("unknown prot: %d\n", prot); } @@ -2293,7 +1745,7 @@ static int MapFlexibleMemory(void** addr_in_out, size_t len, int prot, int flags return KERNEL_ERROR_ENOMEM; } - if (!g_flexible_memory->Map(out_addr, len, prot, mode, gpu_mode, name)) { + if (!g_flexible_memory->Map(out_addr, len, mode)) { LOGF_COLOR(Log::Color::Red, "\t [Fail]\n"); return KERNEL_ERROR_ENOMEM; } @@ -2307,8 +1759,7 @@ static int MapFlexibleMemory(void** addr_in_out, size_t len, int prot, int flags VirtualRangeType::Flexible, name, (map_flags & GUEST_MAP_NO_COALESCE) != 0); if (!published) { - GpuAccessMode rollback_gpu_mode = GpuAccessMode::NoAccess; - EXIT_IF(!g_flexible_memory->Unmap(out_addr, len, &rollback_gpu_mode)); + EXIT_IF(!g_flexible_memory->Unmap(out_addr, len)); return KERNEL_ERROR_EBUSY; } @@ -2317,10 +1768,8 @@ static int MapFlexibleMemory(void** addr_in_out, size_t len, int prot, int flags "\t size = %" PRIu64 "\n" "\t mode = %s\n" "\t flags = 0x%08" PRIx32 "\n" - "\t name = %s\n" - "\t gpu_mode = %s\n", - in_addr, out_addr, len, magic_enum::enum_name(mode), static_cast(flags), name, - magic_enum::enum_name(gpu_mode)); + "\t name = %s\n", + in_addr, out_addr, len, magic_enum::enum_name(mode), static_cast(flags), name); MapGpuRange(out_addr, len); @@ -2442,8 +1891,6 @@ int KYTY_SYSV_ABI KernelSetVirtualRangeName(const void* addr, uint64_t len, cons return KERNEL_ERROR_ENAMETOOLONG; } - g_physical_memory->SetVirtualRangeName(vaddr, len, name); - g_flexible_memory->SetVirtualRangeName(vaddr, len, name); g_virtual_ranges->Rename(vaddr, len, name); return OK; @@ -2460,8 +1907,6 @@ int KYTY_SYSV_ABI KernelClearVirtualRangeName(const void* addr, uint64_t len) { "\t len = 0x%016" PRIx64 "\n", vaddr, len); - g_physical_memory->SetVirtualRangeName(vaddr, len, ""); - g_flexible_memory->SetVirtualRangeName(vaddr, len, ""); g_virtual_ranges->Rename(vaddr, len, ""); return OK; @@ -2472,7 +1917,8 @@ static bool FreeGuestMemoryOwner(uint64_t vaddr, uint64_t size) { g_virtual_ranges->Remove(vaddr, size); } -static int UnmapMemoryOwner(const VirtualRanges::Range& range, uint64_t vaddr, size_t len) { +static int UnmapMemoryOwner(const VirtualRanges::Range& range, uint64_t vaddr, size_t len, + bool keep_pool_reserved = false) { if (IsReservedRangeType(range.type)) { if (!g_guest_address_space->ReleaseFree(vaddr, len)) { return KERNEL_ERROR_EACCES; @@ -2484,31 +1930,23 @@ static int UnmapMemoryOwner(const VirtualRanges::Range& range, uint64_t vaddr, s return FreeGuestMemoryOwner(vaddr, len) ? OK : KERNEL_ERROR_EACCES; } - GpuAccessMode gpu_mode = GpuAccessMode::NoAccess; - bool owner_unmapped = false; + bool owner_unmapped = false; if (range.type == VirtualRangeType::Pooled) { - std::vector mappings; - if (g_pooled_memory->Query(vaddr, len, &mappings)) { - VirtualMemory::Mode mode = VirtualMemory::Mode::NoAccess; - GpuAccessMode decoded_gpu = GpuAccessMode::NoAccess; - owner_unmapped = DecodeMemoryProtection(range.protection, &mode, &decoded_gpu) && - UnmapPooledBackingTransactional(mappings, mode); - if (owner_unmapped && !g_pooled_memory->Release(vaddr, len, &gpu_mode)) { - EXIT("failed to release unmapped pooled-memory range\n"); - } + owner_unmapped = g_guest_address_space->UnmapBacking(vaddr, len); + if (owner_unmapped && !g_pooled_memory->Release(vaddr, len)) { + EXIT("failed to release unmapped pooled-memory range\n"); } if (!owner_unmapped) { return KERNEL_ERROR_EACCES; } } else if (range.type == VirtualRangeType::Direct) { - VirtualMemory::Mode direct_mode = VirtualMemory::Mode::NoAccess; - GpuAccessMode direct_gpu = GpuAccessMode::NoAccess; - const auto direct_offset = range.offset + vaddr - range.start; - if (DecodeMemoryProtection(range.protection, &direct_mode, &direct_gpu) && + VirtualMemory::Mode direct_mode = VirtualMemory::Mode::NoAccess; + + const auto direct_offset = range.offset + vaddr - range.start; + if (DecodeMemoryProtection(range.protection, &direct_mode) && g_guest_address_space->BackingContains(vaddr, len) && g_guest_address_space->UnmapBacking(vaddr, len)) { - uint64_t ignored_host = 0; - if (g_physical_memory->Unmap(vaddr, len, &gpu_mode, &ignored_host)) { + if (g_physical_memory->Unmap(vaddr, len)) { owner_unmapped = true; } else { EXIT_IF(!g_guest_address_space->MapBacking(vaddr, len, direct_offset, direct_mode)); @@ -2517,14 +1955,17 @@ static int UnmapMemoryOwner(const VirtualRanges::Range& range, uint64_t vaddr, s } else if (range.type == VirtualRangeType::Stack) { owner_unmapped = g_guest_address_space->ReleaseCommitted(vaddr, len); } else { - uint64_t ignored_host = 0; - owner_unmapped = g_flexible_memory->Unmap(vaddr, len, &gpu_mode, &ignored_host); + owner_unmapped = g_flexible_memory->Unmap(vaddr, len); } if (!owner_unmapped) { return KERNEL_ERROR_EACCES; } g_virtual_ranges->Remove(vaddr, len); + if (keep_pool_reserved) { + EXIT_IF(!g_virtual_ranges->Add(vaddr, len, 0, 0, 0, VirtualRangeType::PoolReserved, + range.name)); + } if (g_free_callback != nullptr && IsCommittedRangeType(range.type)) { g_free_callback(vaddr, len); @@ -2539,7 +1980,12 @@ static int UnmapMemoryOwner(const VirtualRanges::Range& range, uint64_t vaddr, s static int UnmapMemoryRange(const VirtualRanges::Range& range, uint64_t vaddr, size_t len) { UnmapGpuRange(vaddr, len); const int result = UnmapMemoryOwner(range, vaddr, len); - if (result != OK && IsCommittedRangeType(range.type)) MapGpuRange(vaddr, len); + if (result != OK && IsCommittedRangeType(range.type)) { + VirtualMemory::Mode mode = VirtualMemory::Mode::NoAccess; + EXIT_IF(!DecodeMemoryProtection(range.protection, &mode)); + EXIT_IF(!g_guest_address_space->Protect(vaddr, len, mode)); + MapGpuRange(vaddr, len); + } return result; } @@ -2820,7 +2266,13 @@ int KYTY_SYSV_ABI KernelAllocateMainDirectMemory(size_t len, size_t alignment, i static int ReleaseDirectMemoryInternal(int64_t start, size_t len, bool skip_gaps) { std::lock_guard memory_operation_lock(g_memory_operation_mutex); - if (g_pooled_memory->ReleaseExpansion(static_cast(start), len)) { + PhysicalMemory::AllocatedBlock allocation {}; + if (g_physical_memory->Find(static_cast(start), &allocation) && + allocation.start_addr == static_cast(start) && allocation.size == len && + allocation.kind == PhysicalMemory::AllocationKind::Pooled) { + if (!g_pooled_memory->ReleaseExpansion(static_cast(start), len)) { + return KERNEL_ERROR_EACCES; + } if (!g_physical_memory->ReleasePoolExpansion(static_cast(start), len)) { EXIT("failed to release physical pool expansion\n"); } @@ -2835,73 +2287,63 @@ static int ReleaseDirectMemoryInternal(int64_t start, size_t len, bool skip_gaps const auto mapped_aliases = g_physical_memory->FindMappings(start, len); for (const auto& alias: mapped_aliases) { - VirtualRanges::Range range {}; - if (!g_guest_address_space->BackingContains(alias.map_vaddr, alias.map_size) || - !g_virtual_ranges->Query(alias.map_vaddr, 0, &range) || - range.type != VirtualRangeType::Direct || - alias.map_size > range.start + range.size - alias.map_vaddr) { + if (!g_guest_address_space->BackingContains(alias.start, alias.size)) { EXIT("direct-memory alias escaped guest address-space ownership\n"); } } for (const auto& alias: mapped_aliases) { - UnmapGpuRange(alias.map_vaddr, alias.map_size); + UnmapGpuRange(alias.start, alias.size); } auto restore_gpu_aliases = [&mapped_aliases]() { for (const auto& alias: mapped_aliases) { - MapGpuRange(alias.map_vaddr, alias.map_size); + MapGpuRange(alias.start, alias.size); } }; - auto restore_owner_aliases = [](const std::vector& aliases) { + auto restore_owner_aliases = [](std::span aliases) { for (auto it = aliases.rbegin(); it != aliases.rend(); ++it) { - EXIT_IF(!g_guest_address_space->MapBacking(it->map_vaddr, it->map_size, it->start_addr, - it->mode)); + VirtualMemory::Mode mode = VirtualMemory::Mode::NoAccess; + + EXIT_IF(!DecodeMemoryProtection(it->protection, &mode)); + EXIT_IF(!g_guest_address_space->MapBacking(it->start, it->size, it->offset, mode)); } }; - std::vector owner_unmapped; + size_t owner_unmapped = 0; for (const auto& alias: mapped_aliases) { - if (!g_guest_address_space->UnmapBacking(alias.map_vaddr, alias.map_size)) { - restore_owner_aliases(owner_unmapped); + if (!g_guest_address_space->UnmapBacking(alias.start, alias.size)) { + restore_owner_aliases(std::span(mapped_aliases).first(owner_unmapped)); restore_gpu_aliases(); return KERNEL_ERROR_EACCES; } - owner_unmapped.push_back(alias); + ++owner_unmapped; } - std::vector metadata_unmapped; + size_t metadata_unmapped = 0; for (const auto& alias: mapped_aliases) { - GpuAccessMode alias_gpu_mode = GpuAccessMode::NoAccess; - if (!g_physical_memory->Unmap(alias.map_vaddr, alias.map_size, &alias_gpu_mode)) { - for (const auto& removed: metadata_unmapped) { - EXIT_IF(!g_physical_memory->Map(removed.map_vaddr, removed.start_addr, - removed.map_size, removed.prot, removed.mode, - removed.gpu_mode, removed.name, - removed.memory_type)); + if (!g_physical_memory->Unmap(alias.start, alias.size)) { + for (const auto& removed: std::span(mapped_aliases).first(metadata_unmapped)) { + EXIT_IF(!g_physical_memory->Map(removed.start, removed.offset, removed.size)); } - restore_owner_aliases(owner_unmapped); + restore_owner_aliases(std::span(mapped_aliases).first(owner_unmapped)); restore_gpu_aliases(); return KERNEL_ERROR_EACCES; } - metadata_unmapped.push_back(alias); + ++metadata_unmapped; } for (const auto& alias: mapped_aliases) { - EXIT_IF(!g_virtual_ranges->Remove(alias.map_vaddr, alias.map_size)); + EXIT_IF(!g_virtual_ranges->Remove(alias.start, alias.size)); } for (const auto& block: allocated_span) { - uint64_t unused_vaddr = 0; - uint64_t unused_size = 0; - GpuAccessMode unused_gpu = GpuAccessMode::NoAccess; - EXIT_IF(!g_physical_memory->Release(block.start_addr, block.size, &unused_vaddr, - &unused_size, &unused_gpu)); + EXIT_IF(!g_physical_memory->Release(block.start_addr, block.size)); } if (g_free_callback != nullptr) { for (const auto& alias: mapped_aliases) { - g_free_callback(alias.map_vaddr, alias.map_size); + g_free_callback(alias.start, alias.size); } } @@ -2975,9 +2417,8 @@ static int MapDirectMemory(void** addr, size_t len, int prot, int flags, bool no_overwrite = ((flags & GUEST_MAP_NO_OVERWRITE) != 0); VirtualMemory::Mode mode = VirtualMemory::Mode::NoAccess; - GpuAccessMode gpu_mode = GpuAccessMode::NoAccess; - if (!DecodeMemoryProtection(prot, &mode, &gpu_mode)) { + if (!DecodeMemoryProtection(prot, &mode)) { return KERNEL_ERROR_EINVAL; } if (!g_physical_memory->CanMapDirect(static_cast(direct_memory_start), len)) { @@ -2995,17 +2436,18 @@ static int MapDirectMemory(void** addr, size_t len, int prot, int flags, return g_guest_address_space->MapBacking(target_addr, len, direct_memory_start, mode, &shared_failure); }; - std::vector replaced_mappings; + std::vector replaced_mappings; auto restore_mappings = [&](size_t count, bool restore_views) { for (size_t index = 0; index < count; ++index) { const auto& old = replaced_mappings[index]; if (restore_views) { - EXIT_IF(!g_guest_address_space->MapBacking(old.map_vaddr, old.map_size, - old.start_addr, old.mode)); + VirtualMemory::Mode old_mode = VirtualMemory::Mode::NoAccess; + + EXIT_IF(!DecodeMemoryProtection(old.protection, &old_mode)); + EXIT_IF( + !g_guest_address_space->MapBacking(old.start, old.size, old.offset, old_mode)); } - EXIT_IF(!g_physical_memory->Map(old.map_vaddr, old.start_addr, old.map_size, - old.prot, old.mode, old.gpu_mode, old.name, - old.memory_type)); + EXIT_IF(!g_physical_memory->Map(old.start, old.offset, old.size)); } MapGpuRange(in_addr, len); }; @@ -3024,31 +2466,11 @@ static int MapDirectMemory(void** addr, size_t len, int prot, int flags, std::all_of(reserved_ranges.begin(), reserved_ranges.end(), [](const auto& range) { return range.type == VirtualRangeType::Direct; })) { - uint64_t covered = 0; - { - Common::LockGuard lock(g_physical_memory->GetMutex()); - for (const auto& mapping: g_physical_memory->GetMappings()) { - const auto start = std::max(in_addr, mapping.map_vaddr); - const auto end = std::min(in_addr + len, mapping.map_vaddr + mapping.map_size); - if (start >= end) { - continue; - } - auto old = mapping; - old.start_addr += start - old.map_vaddr; - old.map_vaddr = start; - old.size = old.map_size = end - start; - covered += old.map_size; - replaced_mappings.push_back(old); - } - } - if (covered != len) { - return KERNEL_ERROR_EBUSY; - } + replaced_mappings = std::move(reserved_ranges); UnmapGpuRange(in_addr, len); for (size_t index = 0; index < replaced_mappings.size(); ++index) { const auto& old = replaced_mappings[index]; - GpuAccessMode old_gpu_mode = GpuAccessMode::NoAccess; - if (!g_physical_memory->Unmap(old.map_vaddr, old.map_size, &old_gpu_mode)) { + if (!g_physical_memory->Unmap(old.start, old.size)) { restore_mappings(index, false); return KERNEL_ERROR_EBUSY; } @@ -3123,22 +2545,20 @@ static int MapDirectMemory(void** addr, size_t len, int prot, int flags, "\t mode = %s\n" "\t flags = 0x%08" PRIx32 "\n" "\t align = 0x%016" PRIx64 "\n" - "\t gpu_mode = %s\n" "\t shared = %s\n" "\t reason = %s\n", in_addr, out_addr, static_cast(direct_memory_start), len, magic_enum::enum_name(mode), static_cast(flags), alignment, - magic_enum::enum_name(gpu_mode), shared_backing ? "yes" : "no", shared_reason); + shared_backing ? "yes" : "no", shared_reason); if (out_addr == 0) { return KERNEL_ERROR_ENOMEM; } PhysicalMemory::AllocatedBlock mapped_block {}; - g_physical_memory->Find(direct_memory_start, false, &mapped_block); + g_physical_memory->Find(direct_memory_start, &mapped_block); const int memory_type = type == -1 ? mapped_block.memory_type : type; - if (!g_physical_memory->Map(out_addr, direct_memory_start, len, prot, mode, gpu_mode, name, - memory_type)) { + if (!g_physical_memory->Map(out_addr, direct_memory_start, len)) { LOGF_COLOR(Log::Color::Red, "\t [Fail]\n"); if (replaced_mappings.empty()) { EXIT_IF(!g_guest_address_space->UnmapBacking(out_addr, len)); @@ -3159,8 +2579,7 @@ static int MapDirectMemory(void** addr, size_t len, int prot, int flags, memory_type, VirtualRangeType::Direct, name); if (!published) { - GpuAccessMode rollback_gpu_mode = GpuAccessMode::NoAccess; - EXIT_IF(!g_physical_memory->Unmap(out_addr, len, &rollback_gpu_mode)); + EXIT_IF(!g_physical_memory->Unmap(out_addr, len)); if (replaced_mappings.empty()) { EXIT_IF(!g_guest_address_space->UnmapBacking(out_addr, len)); } else { @@ -3276,8 +2695,8 @@ static bool ReplaceFixedRangeWithReserved(uint64_t start, uint64_t size) { struct ReplacedChunk { VirtualRanges::Range range {}; VirtualMemory::Mode mode = VirtualMemory::Mode::NoAccess; - GpuAccessMode gpu_mode = GpuAccessMode::NoAccess; - std::vector flexible_blocks; + + std::vector flexible_blocks; bool host_unmapped = false; bool backend_unmapped = false; }; @@ -3309,7 +2728,7 @@ static bool ReplaceFixedRangeWithReserved(uint64_t start, uint64_t size) { if (range.type == VirtualRangeType::Direct) { replaced.range.offset += current - range.start; } - DecodeMemoryProtection(replaced.range.protection, &replaced.mode, &replaced.gpu_mode); + DecodeMemoryProtection(replaced.range.protection, &replaced.mode); if (range.type == VirtualRangeType::Direct && !g_guest_address_space->BackingContains(current, chunk)) { return false; @@ -3323,7 +2742,7 @@ static bool ReplaceFixedRangeWithReserved(uint64_t start, uint64_t size) { " size=0x%016" PRIx64 "\n", replaced.range.start, replaced.range.size); } - chunks.push_back(replaced); + chunks.push_back(std::move(replaced)); current += chunk; } @@ -3341,14 +2760,12 @@ static bool ReplaceFixedRangeWithReserved(uint64_t start, uint64_t size) { chunk.range.start, chunk.range.size, chunk.range.offset, chunk.mode); } if (chunk.backend_unmapped) { - backend_restored = g_physical_memory->Map( - chunk.range.start, chunk.range.offset, chunk.range.size, - chunk.range.protection, chunk.mode, chunk.gpu_mode, chunk.range.name, - chunk.range.memory_type); + backend_restored = g_physical_memory->Map(chunk.range.start, chunk.range.offset, + chunk.range.size); } } else if (chunk.range.type == VirtualRangeType::Flexible && chunk.backend_unmapped) { host_restored = backend_restored = - g_flexible_memory->Restore(chunk.flexible_blocks); + g_flexible_memory->Restore(chunk.flexible_blocks, chunk.mode); } else if (IsPrivateCommittedRangeType(chunk.range.type) && chunk.host_unmapped) { host_restored = g_guest_address_space->Commit(chunk.range.start, chunk.range.size, chunk.mode); @@ -3377,8 +2794,7 @@ static bool ReplaceFixedRangeWithReserved(uint64_t start, uint64_t size) { g_virtual_ranges->Remove(start, size); for (auto& chunk: chunks) { - GpuAccessMode gpu_mode = GpuAccessMode::NoAccess; - bool unmapped = true; + bool unmapped = true; if (chunk.range.type == VirtualRangeType::Direct) { if (!g_guest_address_space->UnmapBacking(chunk.range.start, chunk.range.size)) { @@ -3387,15 +2803,13 @@ static bool ReplaceFixedRangeWithReserved(uint64_t start, uint64_t size) { chunk.host_unmapped = true; } if (unmapped) { - unmapped = g_physical_memory->Unmap(chunk.range.start, chunk.range.size, &gpu_mode); - chunk.gpu_mode = gpu_mode; + unmapped = g_physical_memory->Unmap(chunk.range.start, chunk.range.size); chunk.backend_unmapped = unmapped; } } else if (chunk.range.type == VirtualRangeType::Flexible) { - unmapped = g_flexible_memory->Unmap(chunk.range.start, chunk.range.size, &gpu_mode); + unmapped = g_flexible_memory->Unmap(chunk.range.start, chunk.range.size); chunk.host_unmapped = unmapped; chunk.backend_unmapped = unmapped; - chunk.gpu_mode = gpu_mode; } else if (IsPrivateCommittedRangeType(chunk.range.type)) { unmapped = g_guest_address_space->ReleaseCommitted(chunk.range.start, chunk.range.size); chunk.host_unmapped = unmapped; @@ -3405,11 +2819,6 @@ static bool ReplaceFixedRangeWithReserved(uint64_t start, uint64_t size) { } else { unmapped = false; } - if (chunk.range.type == VirtualRangeType::Direct) { - chunk.gpu_mode = gpu_mode; - chunk.backend_unmapped = chunk.backend_unmapped && unmapped; - } - if (!unmapped) { LOGF_COLOR(Log::Color::Red, "\t reserve-fixed replace: backend unmap failed at 0x%016" PRIx64 @@ -3580,6 +2989,30 @@ bool TestGuestBackingOutsideAddressSpace() { g_guest_address_space->GetBackingSize()); } +uint64_t TestGuestFreeRangeVisits() { + return g_test_free_range_visits.load(std::memory_order_relaxed); +} + +uint64_t TestDirectMappingCount() { + std::lock_guard lock(g_memory_operation_mutex); + return g_physical_memory->GetMappings().size(); +} + +uint64_t TestFlexibleMappingCount() { + std::lock_guard lock(g_memory_operation_mutex); + return g_flexible_memory->GetMappings().size(); +} + +uint64_t TestFlexibleBackingHighWater() { + std::lock_guard lock(g_memory_operation_mutex); + return g_flexible_memory->TestBackingHighWater(); +} + +uint64_t TestBackingBytesZeroed() { + std::lock_guard lock(g_memory_operation_mutex); + return g_test_backing_bytes_zeroed; +} + uint64_t TestGuestBackingSize() { return g_guest_address_space->GetBackingSize(); } @@ -3898,9 +3331,8 @@ static int ProtectMemory(const void* addr, size_t len, int prot) { } VirtualMemory::Mode mode = VirtualMemory::Mode::NoAccess; - GpuAccessMode gpu_mode = GpuAccessMode::NoAccess; - if (!DecodeMemoryProtection(prot, &mode, &gpu_mode)) { + if (!DecodeMemoryProtection(prot, &mode)) { return KERNEL_ERROR_EINVAL; } std::vector old_ranges; @@ -3920,14 +3352,6 @@ static int ProtectMemory(const void* addr, size_t len, int prot) { " prot=0x%08x\n", aligned_addr, aligned_len, prot); } - for (const auto& old_range: old_ranges) { - if (old_range.type == VirtualRangeType::Direct) { - g_physical_memory->ProtectMapping(old_range.start, old_range.size, prot, mode, - gpu_mode); - } else if (old_range.type == VirtualRangeType::Flexible) { - g_flexible_memory->Protect(old_range.start, old_range.size, prot, mode, gpu_mode); - } - } g_virtual_ranges->Protect(aligned_addr, aligned_len, prot); LOGF("\t prot: %s -> %s\n", magic_enum::enum_name(old_mode), magic_enum::enum_name(mode)); @@ -4035,14 +3459,10 @@ static bool IsAligned(uint64_t value, uint64_t alignment) { } static void MemoryPoolSubtractCommitted(uint64_t len) { - auto current = g_memory_pool_committed.load(std::memory_order_relaxed); - while (current != 0) { - const auto next = (current > len ? current - len : 0); - if (g_memory_pool_committed.compare_exchange_weak(current, next, - std::memory_order_relaxed)) { - return; - } - } + // Writers are serialized by g_memory_operation_mutex; debug stats read without that lock. + const auto current = g_memory_pool_committed.load(std::memory_order_relaxed); + EXIT_IF(len > current); + g_memory_pool_committed.store(current - len, std::memory_order_relaxed); } int KYTY_SYSV_ABI KernelMemoryPoolExpand(int64_t search_start, int64_t search_end, size_t len, @@ -4152,8 +3572,8 @@ int KYTY_SYSV_ABI KernelMemoryPoolCommit(void* addr, size_t len, int type, int p } VirtualMemory::Mode mode = VirtualMemory::Mode::NoAccess; - GpuAccessMode gpu_mode = GpuAccessMode::NoAccess; - if (!DecodeMemoryProtection(prot, &mode, &gpu_mode)) { + + if (!DecodeMemoryProtection(prot, &mode)) { return KERNEL_ERROR_EINVAL; } @@ -4167,17 +3587,16 @@ int KYTY_SYSV_ABI KernelMemoryPoolCommit(void* addr, size_t len, int type, int p } std::vector mappings; - if (!g_pooled_memory->Allocate(vaddr, len, gpu_mode, &mappings)) { + if (!g_pooled_memory->Allocate(vaddr, len, &mappings)) { return KERNEL_ERROR_ENOMEM; } - std::vector mapped; - auto rollback = [&]() { - if (!UnmapPooledBackingTransactional(mapped, mode)) { + size_t mapped_size = 0; + auto rollback = [&]() { + if (mapped_size != 0 && !g_guest_address_space->UnmapBacking(vaddr, mapped_size)) { EXIT("failed to roll back pooled-memory backing maps\n"); } - GpuAccessMode rollback_gpu_mode = GpuAccessMode::NoAccess; - if (!g_pooled_memory->Release(vaddr, len, &rollback_gpu_mode)) { + if (!g_pooled_memory->Release(vaddr, len)) { EXIT("failed to release pooled-memory rollback allocation\n"); } }; @@ -4192,7 +3611,7 @@ int KYTY_SYSV_ABI KernelMemoryPoolCommit(void* addr, size_t len, int type, int p rollback(); return KERNEL_ERROR_ENOMEM; } - mapped.push_back(mapping); + mapped_size += mapping.size; } if (!g_virtual_ranges->ReplaceSpan(vaddr, len, VirtualRangeType::PoolReserved, 0, prot, type, @@ -4212,55 +3631,6 @@ int KYTY_SYSV_ABI KernelMemoryPoolCommit(void* addr, size_t len, int type, int p return OK; } -static int DecommitMemoryPoolRange(uint64_t vaddr, size_t len) { - VirtualRanges::Range old_range {}; - if (!g_virtual_ranges->Query(vaddr, 0, &old_range)) { - return KERNEL_ERROR_EACCES; - } - const auto chunk_len = std::min(len, old_range.size - (vaddr - old_range.start)); - if (old_range.type == VirtualRangeType::PoolReserved) { - return chunk_len < len ? DecommitMemoryPoolRange(vaddr + chunk_len, len - chunk_len) : OK; - } - if (old_range.type != VirtualRangeType::Pooled) { - return KERNEL_ERROR_EACCES; - } - if (chunk_len < len) { - const int ret = DecommitMemoryPoolRange(vaddr, chunk_len); - return ret == OK ? DecommitMemoryPoolRange(vaddr + chunk_len, len - chunk_len) : ret; - } - - std::vector mappings; - if (!g_pooled_memory->Query(vaddr, len, &mappings)) { - return KERNEL_ERROR_EACCES; - } - - VirtualMemory::Mode mode = VirtualMemory::Mode::NoAccess; - GpuAccessMode decoded_gpu = GpuAccessMode::NoAccess; - if (!DecodeMemoryProtection(old_range.protection, &mode, &decoded_gpu)) { - return KERNEL_ERROR_EACCES; - } - if (!UnmapPooledBackingTransactional(mappings, mode)) { - EXIT("pooled-memory backing transaction failed after GPU unmap: addr=0x%016" PRIx64 - " size=0x%016" PRIx64 "\n", - vaddr, len); - } - - GpuAccessMode gpu_mode = GpuAccessMode::NoAccess; - if (!g_pooled_memory->Release(vaddr, len, &gpu_mode)) { - EXIT("failed to release decommitted pooled-memory range\n"); - } - - g_virtual_ranges->Remove(vaddr, len); - g_virtual_ranges->Add(vaddr, len, 0, 0, 0, VirtualRangeType::PoolReserved, old_range.name); - - if (g_free_callback != nullptr) { - g_free_callback(vaddr, len); - } - - MemoryPoolSubtractCommitted(len); - return OK; -} - int KYTY_SYSV_ABI KernelMemoryPoolDecommit(void* addr, size_t len, int flags) { PRINT_NAME(); @@ -4282,24 +3652,23 @@ int KYTY_SYSV_ABI KernelMemoryPoolDecommit(void* addr, size_t len, int flags) { if (UINT64_MAX - vaddr < len) { return KERNEL_ERROR_EINVAL; } - const auto end = vaddr + len; - auto scan = vaddr; - while (scan < end) { - VirtualRanges::Range scan_range {}; - if (!g_virtual_ranges->Query(scan, 0, &scan_range) || - (scan_range.type != VirtualRangeType::Pooled && - scan_range.type != VirtualRangeType::PoolReserved)) { - return KERNEL_ERROR_EACCES; - } - const auto next = std::min(end, scan_range.start + scan_range.size); - if (next <= scan) { - return KERNEL_ERROR_EACCES; - } - scan = next; + std::vector ranges; + if (!g_virtual_ranges->QuerySpan(vaddr, len, &ranges) || + std::any_of(ranges.begin(), ranges.end(), + [](const auto& range) { return !IsPooledRangeType(range.type); })) { + return KERNEL_ERROR_EACCES; } UnmapGpuRange(vaddr, len); - return DecommitMemoryPoolRange(vaddr, len); + for (const auto& range: ranges) { + if (range.type == VirtualRangeType::Pooled && + UnmapMemoryOwner(range, range.start, range.size, true) != OK) { + EXIT("pooled-memory backing transaction failed after GPU unmap: addr=0x%016" PRIx64 + " size=0x%016" PRIx64 "\n", + range.start, range.size); + } + } + return OK; } int KYTY_SYSV_ABI KernelMemoryPoolBatch(const KernelMemoryPoolBatchEntry* entries, int num_entries, @@ -4408,8 +3777,8 @@ DebugStats GetDebugStats() { break; } } - for (const auto& mapping: g_physical_memory->GetMappings()) { - stats.direct_mapped += mapping.map_size; + for (const auto& [vaddr, mapping]: g_physical_memory->GetMappings()) { + stats.direct_mapped += mapping.size; } } if (g_flexible_memory != nullptr) { diff --git a/src/kernel/memory.h b/src/kernel/memory.h index fed2611ee..182b2a7ad 100644 --- a/src/kernel/memory.h +++ b/src/kernel/memory.h @@ -225,6 +225,11 @@ bool TestPlaceholderRangeIsFree(uint64_t vaddr, uint64_t size); bool TestGuestAddressRangeIsOwned(uint64_t vaddr, uint64_t size); bool TestGuestBackingOutsideAddressSpace(); uint64_t TestGuestBackingSize(); +uint64_t TestFlexibleBackingHighWater(); +uint64_t TestGuestFreeRangeVisits(); +uint64_t TestDirectMappingCount(); +uint64_t TestFlexibleMappingCount(); +uint64_t TestBackingBytesZeroed(); bool TestGuestFreeRangeBounds(); #endif diff --git a/src/kernel/memoryAddressSpace.inc b/src/kernel/memoryAddressSpace.inc index 2a926e6fb..04aa47982 100644 --- a/src/kernel/memoryAddressSpace.inc +++ b/src/kernel/memoryAddressSpace.inc @@ -203,6 +203,9 @@ public: } std::lock_guard lock(m_mutex); std::memset(m_backing_base + backing_offset, 0, size); +#if defined(KYTY_VIRTUAL_MEMORY_ALLOCATION_TESTS) + g_test_backing_bytes_zeroed += size; +#endif return true; } @@ -287,6 +290,18 @@ public: return true; } + bool ProtectViews(uint64_t vaddr, uint64_t size, VirtualMemory::Mode mode) { + std::shared_lock lock(m_mutex); + bool ok = true; + ForEachSparseRange( + m_maps, vaddr, size, + [&](const auto&, uint64_t start, uint64_t bytes) { + ok = ok && VirtualMemory::Protect(start, bytes, mode); + }, + [&](uint64_t, uint64_t) { ok = false; }); + return ok; + } + bool MapFixed(uint64_t vaddr, uint64_t size, uint64_t backing_offset, VirtualMemory::Mode mode, FailureReason* failure_reason = nullptr) { if (!IsAvailable() || !RangeOk(backing_offset, size)) { @@ -324,15 +339,11 @@ public: #endif } - bool Unmap(uint64_t vaddr, uint64_t size, bool* placeholder_preserved = nullptr) { + bool Unmap(uint64_t vaddr, uint64_t size) { if (size == 0 || UINT64_MAX - vaddr < size) { return false; } - if (placeholder_preserved != nullptr) { - *placeholder_preserved = false; - } - const auto end = vaddr + size; std::vector targets; { @@ -350,42 +361,36 @@ public: } } - std::vector removed; - removed.reserve(targets.size()); + size_t removed = 0; + auto rollback = [&] { + while (removed != 0) { + const auto& previous = targets[--removed]; + EXIT_IF(!MapFixed(previous.vaddr, previous.size, previous.backing_offset, + previous.mode)); + } + }; for (const auto& entry: targets) { - bool part_placeholder_preserved = false; - if (!UnmapOne(entry.vaddr, entry.size, &part_placeholder_preserved)) { - for (auto it = removed.rbegin(); it != removed.rend(); ++it) { - EXIT_IF(!MapFixed(it->vaddr, it->size, it->backing_offset, it->mode)); - } - if (placeholder_preserved != nullptr) { - *placeholder_preserved = false; - } + if (!UnmapOne(entry.vaddr, entry.size)) { + rollback(); return false; } - EXIT_IF(!part_placeholder_preserved); - removed.push_back(entry); + ++removed; } #if KYTY_PLATFORM == KYTY_PLATFORM_WINDOWS // Each unmapped file view becomes a separate Windows placeholder. The address-space owner // records this contiguous guest span as one free range, so normalize fragmented backing // views into one placeholder before publishing it there. - if (removed.size() > 1 && !CoalescePlaceholders(vaddr, size)) { - for (auto it = removed.rbegin(); it != removed.rend(); ++it) { - EXIT_IF(!MapFixed(it->vaddr, it->size, it->backing_offset, it->mode)); - } + if (removed > 1 && !CoalescePlaceholders(vaddr, size)) { + rollback(); return false; } #endif - if (placeholder_preserved != nullptr) { - *placeholder_preserved = true; - } return true; } - bool UnmapOne(uint64_t vaddr, uint64_t size, bool* placeholder_preserved = nullptr) { + bool UnmapOne(uint64_t vaddr, uint64_t size) { #if defined(KYTY_VIRTUAL_MEMORY_ALLOCATION_TESTS) if (g_test_backing_store_unmaps_before_failure == 0) { g_test_backing_store_unmaps_before_failure = UINT32_MAX; @@ -428,9 +433,7 @@ public: m_maps.emplace(old.vaddr, old); return false; } - if (placeholder_preserved != nullptr) { - *placeholder_preserved = true; - } + return true; } @@ -451,9 +454,6 @@ public: ok; } if (ok) { - if (placeholder_preserved != nullptr) { - *placeholder_preserved = true; - } return true; } @@ -481,9 +481,6 @@ public: std::lock_guard lock(m_mutex); RemoveMappingsUnlocked(vaddr, size); - if (placeholder_preserved != nullptr) { - *placeholder_preserved = true; - } return true; #endif } @@ -634,8 +631,6 @@ private: HANDLE m_handle = nullptr; #elif KYTY_PLATFORM == KYTY_PLATFORM_LINUX - [[nodiscard]] static uint64_t GetMappableTestSize() { return 0x4000; } - static int GetProtect(VirtualMemory::Mode mode) { switch (mode) { case VirtualMemory::Mode::Read: return PROT_READ; @@ -760,7 +755,7 @@ public: if (size == 0 || UINT64_MAX - vaddr < size) { return false; } - std::lock_guard lock(m_mutex); + std::shared_lock lock(m_mutex); return std::any_of(m_owned.begin(), m_owned.end(), [vaddr, size](const auto& range) { return VirtualRangesOverlap(vaddr, size, range.first, range.second); }); @@ -864,11 +859,7 @@ public: if (!ContainsMappedUnlocked(vaddr, size, MappingType::Backing)) { return false; } - bool placeholder_preserved = false; - if (!m_backing->Unmap(vaddr, size, &placeholder_preserved)) { - return false; - } - if (!placeholder_preserved) { + if (!m_backing->Unmap(vaddr, size)) { return false; } EXIT_IF(!RemoveMappedUnlocked(vaddr, size, MappingType::Backing)); @@ -876,12 +867,13 @@ public: return true; } + // Reservations live in VirtualRanges; these free-range checks only read the host placeholders. uint64_t ReserveAligned(uint64_t search_addr, uint64_t size, uint64_t alignment) { if (size == 0) { return 0; } alignment = std::max(alignment != 0 ? alignment : PageSize(), PageSize()); - std::lock_guard lock(m_mutex); + std::shared_lock lock(m_mutex); if (search_addr == 0) { auto addr = FindFreeAlignedUnlocked(HOST_SYSTEM_MANAGED_MIN, HOST_SYSTEM_MANAGED_MAX + 1u, size, alignment); @@ -898,7 +890,7 @@ public: return 0; } alignment = std::max(alignment, PageSize()); - std::lock_guard lock(m_mutex); + std::shared_lock lock(m_mutex); return FindFreeAlignedUnlocked(search_start, search_end, size, alignment); } @@ -906,19 +898,19 @@ public: if (vaddr == 0 || size == 0 || !IsPageAligned(vaddr) || !IsPageAligned(size)) { return false; } - std::lock_guard lock(m_mutex); + std::shared_lock lock(m_mutex); return FindContaining(vaddr, size) != m_free.end(); } #if defined(KYTY_VIRTUAL_MEMORY_ALLOCATION_TESTS) bool TestContainsFree(uint64_t vaddr, uint64_t size) { - std::lock_guard lock(m_mutex); + std::shared_lock lock(m_mutex); return FindContaining(vaddr, size) != m_free.end(); } #endif bool ReleaseFree(uint64_t vaddr, uint64_t size) { - std::lock_guard lock(m_mutex); + std::shared_lock lock(m_mutex); return FindContaining(vaddr, size) != m_free.end(); } @@ -1041,10 +1033,6 @@ private: return (value & (PageSize() - 1u)) == 0; } - [[nodiscard]] bool IsGranularityAligned(uint64_t value) const { - return (value & (m_granularity - 1u)) == 0; - } - bool ProtectMappedUnlocked(uint64_t vaddr, uint64_t size, VirtualMemory::Mode mode) { const auto end = vaddr + size; auto it = m_mapped.upper_bound(vaddr); @@ -1055,8 +1043,20 @@ private: const auto& region = it->second; const auto range_start = std::max(vaddr, region.vaddr); const auto range_end = std::min(end, region.vaddr + region.size); - if (range_start < range_end && - !VirtualMemory::Protect(range_start, range_end - range_start, mode)) { + if (range_start >= range_end) { + continue; + } +#if KYTY_PLATFORM == KYTY_PLATFORM_WINDOWS + // Rollback can split a backing view without changing the owner interval. + // Windows protection calls must stay within each actual view. + if (region.type == MappingType::Backing) { + if (!m_backing->ProtectViews(range_start, range_end - range_start, mode)) { + return false; + } + continue; + } +#endif + if (!VirtualMemory::Protect(range_start, range_end - range_start, mode)) { return false; } } @@ -1312,12 +1312,21 @@ private: } uint64_t FindFreeAlignedUnlocked(uint64_t search_start, uint64_t search_end, uint64_t size, - uint64_t alignment) { + uint64_t alignment) const { if (search_start >= search_end || size > search_end - search_start) { return 0; } - for (const auto& [base, range_size]: m_free) { - const auto candidate = AlignUp(std::max(base, search_start), alignment); + // Include the predecessor because search_start can lie inside a free range. + auto it = m_free.upper_bound(search_start); + if (it != m_free.begin()) { + --it; + } + for (; it != m_free.end() && it->first < search_end; ++it) { +#if defined(KYTY_VIRTUAL_MEMORY_ALLOCATION_TESTS) + g_test_free_range_visits.fetch_add(1, std::memory_order_relaxed); +#endif + const auto& [base, range_size] = *it; + const auto candidate = AlignUp(std::max(base, search_start), alignment); if (candidate < search_end && size <= search_end - candidate && GuestFreeRangeContains(base, range_size, candidate, size)) { return candidate; @@ -1331,9 +1340,8 @@ private: if (next == m_free.begin()) { return m_free.end(); } - auto it = std::prev(next); - auto end = it->first + it->second; - return (vaddr >= it->first && vaddr + size <= end ? it : m_free.end()); + auto it = std::prev(next); + return GuestFreeRangeContains(it->first, it->second, vaddr, size) ? it : m_free.end(); } bool SplitUnlocked(std::map::iterator it, uint64_t vaddr, uint64_t size) { diff --git a/src/libs/libDbgAsan.cpp b/src/libs/libDbgAsan.cpp index 6a9c27fce..6a2e3ec9a 100644 --- a/src/libs/libDbgAsan.cpp +++ b/src/libs/libDbgAsan.cpp @@ -38,8 +38,8 @@ struct Global { }; struct Shadow { - uintptr_t start = 0; - int count = 0; + uintptr_t start = 0; + uint64_t covered_bytes = 0; }; struct AsanContext { @@ -50,6 +50,14 @@ struct AsanContext { static uint32_t g_fake_stack = 0; static AsanContext* g_ctx = nullptr; +// Allocation and release callbacks can partition the same guest range differently. +static uint64_t ShadowCoverage(uintptr_t page, uintptr_t min_addr, uintptr_t max_addr) { + const auto guest_start = (page - 0x10000000000ull) << 3u; + const auto guest_end = guest_start + 0x80000ull - 1; + return std::min(max_addr, guest_end) - std::max(min_addr, guest_start) + + 1; +} + static void AllocShadow(uintptr_t min_addr, uintptr_t max_addr) { EXIT_IF(g_ctx == nullptr); @@ -67,10 +75,11 @@ static void AllocShadow(uintptr_t min_addr, uintptr_t max_addr) { static_cast(min_shadow), static_cast(size)); for (uintptr_t page = min_shadow; page <= max_shadow; page += page_size) { - const auto it = std::find_if(g_ctx->shadows.begin(), g_ctx->shadows.end(), - [page](const Shadow& s) { return s.start == page; }); + const auto bytes = ShadowCoverage(page, min_addr, max_addr); + const auto it = std::find_if(g_ctx->shadows.begin(), g_ctx->shadows.end(), + [page](const Shadow& s) { return s.start == page; }); if (it != g_ctx->shadows.end()) { - it->count++; + it->covered_bytes += bytes; } else { if (!Common::VirtualMemory::AllocFixed(page, page_size, Common::VirtualMemory::Mode::ReadWrite)) { @@ -78,8 +87,8 @@ static void AllocShadow(uintptr_t min_addr, uintptr_t max_addr) { } Shadow s; - s.start = page; - s.count = 1; + s.start = page; + s.covered_bytes = bytes; g_ctx->shadows.push_back(s); } } @@ -105,14 +114,15 @@ static void FreeShadow(uintptr_t min_addr, uintptr_t max_addr) { const auto it = std::find_if(g_ctx->shadows.begin(), g_ctx->shadows.end(), [page](const Shadow& s) { return s.start == page; }); if (it != g_ctx->shadows.end()) { - it->count--; - - if (it->count <= 0) { + const auto bytes = ShadowCoverage(page, min_addr, max_addr); + if (bytes >= it->covered_bytes) { if (!Common::VirtualMemory::Free(page)) { EXIT("can't free shadow memory\n"); } g_ctx->shadows.erase(it); + } else { + it->covered_bytes -= bytes; } } } @@ -126,6 +136,14 @@ static void KernelFree(uintptr_t addr, size_t size) { FreeShadow(addr, addr + size - 1); } +#if defined(KYTY_VIRTUAL_MEMORY_ALLOCATION_TESTS) +void TestRegisterMemoryCallbacks() { + EXIT_IF(g_ctx != nullptr); + g_ctx = new AsanContext; + LibKernel::Memory::RegisterCallbacks(KernelAlloc, KernelFree); +} +#endif + static void KYTY_SYSV_ABI asan_init() { PRINT_NAME(); diff --git a/tests/VirtualMemoryAllocationTests.cpp b/tests/VirtualMemoryAllocationTests.cpp index 31674b3aa..1a313fd0b 100644 --- a/tests/VirtualMemoryAllocationTests.cpp +++ b/tests/VirtualMemoryAllocationTests.cpp @@ -47,12 +47,17 @@ #if defined(__linux__) #include +#include #include #include #include #include #endif +#if defined(__APPLE__) +#include +#endif + #if KYTY_PLATFORM == KYTY_PLATFORM_WINDOWS #ifndef NOMINMAX #define NOMINMAX @@ -82,6 +87,10 @@ int32_t KYTY_SYSV_ABI FiberGetSelf(FiberObject**); int32_t KYTY_SYSV_ABI FiberReturnToThread(uint64_t, uint64_t*); } // namespace Libs::Fiber +namespace Libs::DbgAddressSanitizer { +void TestRegisterMemoryCallbacks(); +} + namespace { using Libs::LibKernel::Memory::VirtualQueryInfo; @@ -357,6 +366,22 @@ size_t AvailableFlexibleMemory(const char* test) { return size; } +void ExpectHostReadOnly(const char* test, uint64_t address) { +#if defined(__linux__) + uint64_t value = 0; + iovec local {&value, sizeof(value)}; + iovec remote {reinterpret_cast(address), sizeof(value)}; + Check(test, process_vm_writev(getpid(), &local, 1, &remote, 1, 0) == -1, + "failed unmap lost read-only host protection"); +#elif KYTY_PLATFORM == KYTY_PLATFORM_WINDOWS + MEMORY_BASIC_INFORMATION info {}; + Check(test, + VirtualQuery(reinterpret_cast(address), &info, sizeof(info)) != 0 && + info.Protect == PAGE_READONLY, + "failed unmap lost read-only host protection"); +#endif +} + size_t ConfiguredFlexibleMemory(const char* test) { size_t size = 0; CheckOk(test, Libs::LibKernel::Memory::KernelConfiguredFlexibleMemorySize(&size), @@ -823,6 +848,160 @@ void TestGuestFreeRangeSearchDoesNotUnderflow() { std::printf("[host] %-48s ok\n", test); } +void TestFragmentedGuestFreeRangeHints() { + using namespace Libs::LibKernel::Memory; + const char* test = "FragmentedGuestFreeRangeHints"; + constexpr uint64_t page = SceKernelPageSize; + void* arena = nullptr; + CheckOk(test, KernelReserveVirtualRange(&arena, 32 * page, 0, 4 * page), "reserve arena"); + const auto base = reinterpret_cast(arena); + CheckOk(test, KernelMunmap(base, 32 * page), "release arena"); + for (const uint64_t offset: {0, 8, 24}) { + void* blocker = reinterpret_cast(base + offset * page); + CheckOk(test, + KernelMapNamedFlexibleMemory(&blocker, 4 * page, SceKernelProtCpuRw, + SceKernelMapFixed | SceKernelMapNoOverwrite, + "search_blocker"), + "map search blocker"); + } + const auto expect_hint = [&](uint64_t hint, uint64_t size, uint64_t alignment, + uint64_t expected) { + void* address = reinterpret_cast(base + hint * page); + CheckOk(test, KernelReserveVirtualRange(&address, size * page, 0, alignment * page), + "reserve hinted range"); + Check(test, reinterpret_cast(address) == base + expected * page, + "hinted search skipped a usable predecessor or selected an " + "occupied/aligned-out range"); + CheckOk(test, KernelMunmap(reinterpret_cast(address), size * page), + "release hinted range"); + }; + expect_hint(5, 1, 1, 5); // Inside the predecessor free interval. + expect_hint(4, 1, 1, 4); // Exactly at its start. + expect_hint(7, 1, 1, 7); // Exactly fills its tail. + expect_hint(8, 1, 1, 12); // At the interval end, inside a host mapping. + expect_hint(6, 2, 4, 12); // Alignment makes the first interval unusable. + + // Reservations remain host placeholders. Searching repeatedly from the same + // hint must skip virtual reservations while retaining the containing + // host-free predecessor. + for (uint64_t i = 12; i < 24; ++i) { + void* address = reinterpret_cast(base + 12 * page); + CheckOk(test, KernelReserveVirtualRange(&address, page, 0, page), "reserve fragmented gap"); + Check(test, reinterpret_cast(address) == base + i * page, + "search retries did not advance through the fragmented virtual " + "reservations"); + CheckOk(test, KernelSetVirtualRangeName(address, page, i % 2 == 0 ? "even" : "odd"), + "keep neighboring reservations distinct"); + } + expect_hint(12, 1, 1, 28); + CheckOk(test, KernelMunmap(base + 12 * page, 12 * page), "release gap reservations"); + for (const uint64_t offset: {0, 8, 24}) { + CheckOk(test, KernelMunmap(base + offset * page, 4 * page), "release search blocker"); + } + std::printf("[host] %-48s ok\n", test); +} + +void TestGuestFreeRangeSearchSkipsEarlierFragments() { + using namespace Libs::LibKernel::Memory; + const char* test = "GuestFreeRangeSearchSkipsEarlierFragments"; + constexpr uint64_t page = SceKernelPageSize; + constexpr uint64_t count = 256; + constexpr uint64_t size = count * 4 * page; + void* arena = nullptr; + CheckOk(test, KernelReserveVirtualRange(&arena, size, 0, page), "reserve fragmented arena"); + const auto base = reinterpret_cast(arena); + CheckOk(test, KernelMunmap(base, size), "release fragmented arena"); + for (uint64_t i = 0; i < count; ++i) { + void* address = reinterpret_cast(base + i * 4 * page); + CheckOk(test, + KernelMapNamedFlexibleMemory(&address, page, SceKernelProtCpuRw, + SceKernelMapFixed | SceKernelMapNoOverwrite, + "fragmented_search"), + "map fragmented search blocker"); + } + + const auto hint = base + size - 2 * page; + void* address = reinterpret_cast(hint); + const auto before = TestGuestFreeRangeVisits(); + CheckOk(test, KernelReserveVirtualRange(&address, page, 0, page), + "reserve near final fragment"); + const auto visited = TestGuestFreeRangeVisits() - before; + Check(test, reinterpret_cast(address) == hint, + "search skipped the final containing free interval"); + Check(test, visited > 0 && visited <= 2, + "hinted free search revisited unrelated lower-address fragments"); + CheckOk(test, KernelMunmap(hint, page), "release final reservation"); + for (uint64_t i = 0; i < count; ++i) { + CheckOk(test, KernelMunmap(base + i * 4 * page, page), "release fragmented search blocker"); + } + std::printf("[host] %-48s ok\n", test); +} + +void TestFlexibleZeroingTracksBackingLifetime() { + using namespace Libs::LibKernel::Memory; + const char* test = "FlexibleZeroingTracksBackingLifetime"; + constexpr uint64_t page = SceKernelPageSize; + const auto available = AvailableFlexibleMemory(test); + const auto backing_start = KernelGetDirectMemorySize(); + Check(test, TestFlexibleBackingHighWater() == backing_start, + "fresh-backing test must run before other flexible allocations"); + const auto expect_zero = [&](uint64_t address, uint64_t size) { + const auto* bytes = reinterpret_cast(address); + Check(test, std::all_of(bytes, bytes + size, [](uint8_t value) { return value == 0; }), + "flexible allocation exposed stale contents"); + }; + + const auto zeroed_before = TestBackingBytesZeroed(); + const auto first = MapNamedFlexible(test, 3 * page, SceKernelProtCpuRw, "fresh_backing"); + Check(test, TestBackingBytesZeroed() == zeroed_before, + "first-use flexible backing was eagerly zeroed"); + expect_zero(first, 3 * page); + std::memset(reinterpret_cast(first), 0xa5, 3 * page); + CheckOk(test, KernelMunmap(first, 3 * page), "unmap dirty backing"); + + const auto mixed = MapNamedFlexible(test, 5 * page, SceKernelProtCpuRw, "mixed_backing"); + Check(test, TestBackingBytesZeroed() - zeroed_before == 3 * page, + "mixed allocation did not zero exactly its previously exposed prefix"); + Check(test, TestFlexibleBackingHighWater() == backing_start + 5 * page, + "mixed allocation did not advance the backing high-water mark"); + expect_zero(mixed, 5 * page); + std::memset(reinterpret_cast(mixed), 0x5a, 5 * page); + + void* failed = nullptr; + CheckOk(test, KernelReserveVirtualRange(&failed, 3 * page, 0, page), "reserve failed map"); + const auto failed_address = reinterpret_cast(failed); + TestFailNextVirtualRangeReplacement(); + CheckFailed(test, + KernelMapNamedFlexibleMemory(&failed, 3 * page, SceKernelProtCpuRw, + SceKernelMapFixed, "failed_fresh_backing"), + "publish fresh flexible mapping with injected failure"); + Check(test, + TestBackingBytesZeroed() - zeroed_before == 3 * page && + TestFlexibleBackingHighWater() == backing_start + 8 * page, + "failed publication zeroed pristine backing or forgot previously " + "exposed backing"); + Check(test, AvailableFlexibleMemory(test) == available - 5 * page, + "failed publication leaked flexible capacity"); + ExpectRange(test, Query(test, failed_address), failed_address, failed_address + 3 * page, 0, 0, + 0, 0, 0); + CheckOk(test, + KernelMapNamedFlexibleMemory(&failed, 3 * page, SceKernelProtCpuRw, SceKernelMapFixed, + "retry_backing"), + "retry failed flexible mapping"); + Check(test, TestBackingBytesZeroed() - zeroed_before == 6 * page, + "retry treated previously exposed backing as pristine"); + expect_zero(failed_address, 3 * page); + Check(test, + *reinterpret_cast(mixed) == 0x5a && + *reinterpret_cast(mixed + 5 * page - 1) == 0x5a, + "zeroing reused backing changed a neighboring live allocation"); + CheckOk(test, KernelMunmap(failed_address, 3 * page), "unmap retry"); + CheckOk(test, KernelMunmap(mixed, 5 * page), "unmap mixed backing"); + Check(test, AvailableFlexibleMemory(test) == available, + "zeroing test leaked flexible capacity"); + std::printf("[host] %-48s ok\n", test); +} + void TestFlexibleMemoryCapacityIsBootFixed() { const char* test = "FlexibleMemoryCapacityIsBootFixed"; const auto configured = ConfiguredFlexibleMemory(test); @@ -1133,42 +1312,104 @@ void TestMainEntryUsesGuestStackAndDisablesHostChecks() { } void TestFragmentedBackingUnmapRollback() { - const char* test = "FragmentedBackingUnmapRollback"; - const auto baseline = AvailableFlexibleMemory(test); - const auto left = - MapNamedFlexible(test, SceKernelPageSize, SceKernelProtCpuRw, "backing_hole_left"); - const auto blocker = - MapNamedFlexible(test, SceKernelPageSize, SceKernelProtCpuRw, "backing_blocker"); - const auto right = - MapNamedFlexible(test, SceKernelPageSize, SceKernelProtCpuRw, "backing_hole_right"); - CheckOk(test, Libs::LibKernel::Memory::KernelMunmap(left, SceKernelPageSize), - "KernelMunmap(left hole)"); - CheckOk(test, Libs::LibKernel::Memory::KernelMunmap(right, SceKernelPageSize), - "KernelMunmap(right hole)"); - - const auto fragmented = - MapNamedFlexible(test, SceKernelPageSize * 2, SceKernelProtCpuRw, "fragmented_backing"); + using namespace Libs::LibKernel::Memory; + const char* test = "FragmentedBackingUnmapRollback"; + constexpr uint64_t page = SceKernelPageSize; + const auto baseline = AvailableFlexibleMemory(test); + const auto left = MapNamedFlexible(test, 2 * page, SceKernelProtCpuRw, "backing_hole_left"); + const auto blocker = MapNamedFlexible(test, page, SceKernelProtCpuRw, "backing_blocker"); + const auto right = MapNamedFlexible(test, 2 * page, SceKernelProtCpuRw, "backing_hole_right"); + std::memset(reinterpret_cast(left), 0xa5, 2 * page); + std::memset(reinterpret_cast(right), 0x5a, 2 * page); + *reinterpret_cast(blocker) = 0x424c4f434b455231ull; + CheckOk(test, KernelMunmap(left, 2 * page), "KernelMunmap(left hole)"); + CheckOk(test, KernelMunmap(right, 2 * page), "KernelMunmap(right hole)"); + void* target = nullptr; + CheckOk(test, KernelReserveVirtualRange(&target, 4 * page, 0, page), + "reserve fragmented publication target"); + const auto fragmented = reinterpret_cast(target); + const auto available = AvailableFlexibleMemory(test); + const auto mappings = TestFlexibleMappingCount(); + TestFailNextVirtualRangeReplacement(); + CheckFailed(test, + KernelMapNamedFlexibleMemory(&target, 4 * page, SceKernelProtCpuRw, + SceKernelMapFixed, "fragmented_backing"), + "publish fragmented flexible mapping with injected failure"); + ExpectRange(test, Query(test, fragmented), fragmented, fragmented + 4 * page, 0, 0, 0, 0, 0, + "anon"); + Check(test, + AvailableFlexibleMemory(test) == available && TestFlexibleMappingCount() == mappings, + "fragmented publication rollback retained capacity or ownership records"); + Check(test, *reinterpret_cast(blocker) == 0x424c4f434b455231ull, + "fragmented publication rollback changed the live backing gap"); + CheckOk(test, + KernelMapNamedFlexibleMemory(&target, 4 * page, SceKernelProtCpuRw, SceKernelMapFixed, + "fragmented_backing"), + "retry fragmented flexible publication"); + const auto* bytes = reinterpret_cast(fragmented); + Check(test, + std::all_of(bytes, bytes + 4 * page, [](uint8_t byte) { return byte == 0; }) && + *reinterpret_cast(blocker) == 0x424c4f434b455231ull, + "fragmented publication retry exposed stale bytes or changed the live gap"); auto* first_word = reinterpret_cast(fragmented); - auto* last_word = - reinterpret_cast(fragmented + SceKernelPageSize * 2 - sizeof(uint64_t)); - *first_word = 0x465241474c454654ull; // "FRAGLEFT" - *last_word = 0x4652414752474854ull; // "FRAGRGHT" - - Libs::LibKernel::Memory::TestFailGuestBackingStoreUnmapAfter(1); - CheckFailed(test, Libs::LibKernel::Memory::KernelMunmap(fragmented, SceKernelPageSize * 2), - "KernelMunmap(injected second-view failure)"); - ExpectRange(test, Query(test, fragmented), fragmented, fragmented + SceKernelPageSize * 2, - SceKernelProtCpuRw, 1, 0, 0, 1, "fragmented_backing"); + auto* last_word = reinterpret_cast(fragmented + 4 * page - sizeof(uint64_t)); + *first_word = 0x465241474c454654ull; + *last_word = 0x4652414752474854ull; + CheckOk(test, KernelMprotect(target, 4 * page, SceKernelProtCpuRead), + "protect fragmented backing before partial unmap"); + TestFailGuestBackingStoreUnmapAfter(1); + CheckFailed(test, KernelMunmap(fragmented + page, 2 * page), + "KernelMunmap(partial first view then injected second-view failure)"); + ExpectRange(test, Query(test, fragmented), fragmented, fragmented + 4 * page, + SceKernelProtCpuRead, 1, 0, 0, 1, "fragmented_backing"); Check(test, *first_word == 0x465241474c454654ull && *last_word == 0x4652414752474854ull, "transactional backing-unmap rollback lost mapped contents"); - - CheckOk(test, Libs::LibKernel::Memory::KernelMunmap(fragmented, SceKernelPageSize * 2), - "KernelMunmap(retry)"); - CheckOk(test, Libs::LibKernel::Memory::KernelMunmap(blocker, SceKernelPageSize), - "KernelMunmap(blocker)"); + ExpectHostReadOnly(test, fragmented + page); + ExpectHostReadOnly(test, fragmented + 2 * page); + CheckOk(test, KernelMprotect(target, 4 * page, SceKernelProtCpuRead), + "reprotect views after partial-first-view rollback"); + for (uint64_t i = 0; i < 4; ++i) + ExpectHostReadOnly(test, fragmented + i * page); + CheckOk(test, KernelMunmap(fragmented, 4 * page), "KernelMunmap(retry)"); + CheckOk(test, KernelMunmap(blocker, page), "KernelMunmap(blocker)"); Check(test, AvailableFlexibleMemory(test) == baseline, "fragmented backing rollback test leaked flexible capacity"); + std::printf("[host] %-48s ok\n", test); +} +void TestFlexibleReplacementRollbackPreservesMetadata() { + using namespace Libs::LibKernel::Memory; + const char* test = "FlexibleReplacementRollbackPreservesMetadata"; + constexpr uint64_t page = SceKernelPageSize; + const auto base = MapNamedFlexible(test, 3 * page, SceKernelProtCpuRw, "flex_restore"); + const auto mapping_count = TestFlexibleMappingCount(); + for (uint64_t i = 0; i < 3; ++i) { + *reinterpret_cast(base + i * page) = 0x12345678 + i; + } + CheckOk(test, KernelMprotect(reinterpret_cast(base + page), page, SceKernelProtCpuRead), + "protect middle flexible page"); + CheckOk(test, + KernelSetVirtualRangeName(reinterpret_cast(base + page), page, "flex_middle"), + "rename middle flexible page"); + Check(test, TestFlexibleMappingCount() == mapping_count, + "flexible metadata edits fragmented backing ownership records"); + const auto zeroed_before = TestBackingBytesZeroed(); + TestFailNextFixedReserveRangeRegistration(); + void* replacement = reinterpret_cast(base); + CheckFailed(test, KernelReserveVirtualRange(&replacement, 3 * page, SceKernelMapFixed, page), + "replace fragmented flexible range with injected failure"); + Check(test, TestBackingBytesZeroed() == zeroed_before, + "rollback cleared preserved flexible backing"); + for (uint64_t i = 0; i < 3; ++i) { + ExpectRange(test, Query(test, base + i * page), base + i * page, base + (i + 1) * page, + i == 1 ? SceKernelProtCpuRead : SceKernelProtCpuRw, 1, 0, 0, 1, + i == 1 ? "flex_middle" : "flex_restore"); + Check(test, *reinterpret_cast(base + i * page) == 0x12345678 + i, + "rollback lost flexible contents across protection boundaries"); + Check(test, !TestPlaceholderRangeIsFree(base + i * page, page), + "restored flexible range remained marked as a free placeholder"); + } + CheckOk(test, KernelMunmap(base, 3 * page), "release restored flexible range"); std::printf("[host] %-48s ok\n", test); } @@ -1233,6 +1474,304 @@ void TestRuntimeMemoryOwnerLifecycle() { std::printf("[host] %-48s ok\n", test); } +constexpr uint64_t AsanShadowPageSize = 0x10000; +constexpr uint64_t AsanShadowCoverage = AsanShadowPageSize * 8; + +uint64_t AsanShadowAddress(uint64_t address) { + return (address >> 3) + 0x10000000000ull; +} + +bool AsanShadowPresent(uint64_t address) { + return Common::VirtualMemory::Protect(AsanShadowAddress(address) & ~(AsanShadowPageSize - 1), + AsanShadowPageSize, + Common::VirtualMemory::Mode::ReadWrite); +} + +uint64_t ReserveAsanCoverage(const char* test, uint64_t pages = 1) { + void* address = nullptr; + CheckOk(test, + Libs::LibKernel::Memory::KernelReserveVirtualRange(&address, pages * AsanShadowCoverage, + 0, AsanShadowCoverage), + "reserve isolated shadow coverage"); + return reinterpret_cast(address); +} + +void MapAsanFlexible(const char* test, uint64_t base, uint64_t size, + const char* name = "asan_shadow") { + void* address = reinterpret_cast(base); + CheckOk(test, + Libs::LibKernel::Memory::KernelMapNamedFlexibleMemory( + &address, size, SceKernelProtCpuRw, SceKernelMapFixed, name), + "map flexible shadow fixture"); +} + +void TestAsanShadowLifetimeAcrossRangeEdits() { + using namespace Libs::LibKernel::Memory; + const char* test = "AsanShadowLifetimeAcrossRangeEdits"; + constexpr uint64_t page = SceKernelPageSize; + constexpr uint64_t coverage = AsanShadowCoverage; + int64_t physical = -1; + CheckOk(test, + KernelAllocateDirectMemory(0, KernelGetDirectMemorySize(), 2 * page, page, + SceKernelMtypeC, &physical), + "allocate direct shadow fixture"); + const std::array bases {ReserveAsanCoverage(test), ReserveAsanCoverage(test)}; + MapAsanFlexible(test, bases[0], 2 * page); + void* direct = reinterpret_cast(bases[1]); + CheckOk(test, + KernelMapDirectMemory(&direct, 2 * page, SceKernelProtCpuRw, SceKernelMapFixed, + physical, page), + "map direct shadow fixture"); + for (const auto base: bases) { + CheckOk(test, + KernelMprotect(reinterpret_cast(base + page), page, SceKernelProtCpuRead), + "split virtual protection before callback registration"); + } + Libs::DbgAddressSanitizer::TestRegisterMemoryCallbacks(); + for (const auto base: bases) { + Check(test, AsanShadowPresent(base), "registration did not allocate shadow memory"); + auto* sentinel = reinterpret_cast(AsanShadowAddress(base + page)); + *sentinel = 0x534841444f574c49ull; + CheckOk(test, KernelMunmap(base, page), "unmap first page with shared shadow"); + Check(test, AsanShadowPresent(base) && *sentinel == 0x534841444f574c49ull, + "partial unmap freed or changed a surviving page's shadow"); + CheckOk(test, KernelMunmap(base + page, page), "unmap final page with shared shadow"); + Check(test, !AsanShadowPresent(base), "final unmap retained shadow memory"); + CheckOk(test, KernelMunmap(base + 2 * page, coverage - 2 * page), + "release isolated reservation"); + } + CheckOk(test, KernelReleaseDirectMemory(physical, 2 * page), "release direct shadow fixture"); + + const auto merged = ReserveAsanCoverage(test); + MapAsanFlexible(test, merged, page); + MapAsanFlexible(test, merged + page, page); + ExpectRange(test, Query(test, merged), merged, merged + 2 * page, SceKernelProtCpuRw, 1, 0, 0, + 1, "asan_shadow"); + CheckOk(test, KernelMunmap(merged, 2 * page), "unmap merged allocations with one callback"); + Check(test, !AsanShadowPresent(merged), + "merged free retained shadow from separate allocations"); + CheckOk(test, KernelMunmap(merged + 2 * page, coverage - 2 * page), + "release merged reservation"); + + const auto crossing = ReserveAsanCoverage(test, 2); + MapAsanFlexible(test, crossing + coverage - page, 2 * page); + auto* sentinel = reinterpret_cast(AsanShadowAddress(crossing + coverage)); + *sentinel = 0x43524f5353534844ull; + CheckOk(test, KernelMunmap(crossing + coverage - page, page), + "unmap first shadow-page coverage"); + Check(test, + !AsanShadowPresent(crossing) && AsanShadowPresent(crossing + coverage) && + *sentinel == 0x43524f5353534844ull, + "cross-page unmap released the wrong shadow page"); + CheckOk(test, KernelMunmap(crossing + coverage, page), "unmap final shadow-page coverage"); + Check(test, !AsanShadowPresent(crossing + coverage), "cross-page final unmap retained shadow"); + CheckOk(test, KernelMunmap(crossing, coverage - page), "release crossing prefix reservation"); + CheckOk(test, KernelMunmap(crossing + coverage + page, coverage - page), + "release crossing suffix reservation"); + std::printf("[host] %-48s ok\n", test); +} + +void TestAsanFailedUnmapPreservesShadow() { + using namespace Libs::LibKernel::Memory; + const char* test = "AsanFailedUnmapPreservesShadow"; + constexpr uint64_t page = SceKernelPageSize; + for (const bool direct: {false, true}) { + const auto base = ReserveAsanCoverage(test); + void* address = reinterpret_cast(base); + int64_t physical = -1; + if (direct) { + CheckOk(test, + KernelAllocateDirectMemory(0, KernelGetDirectMemorySize(), 3 * page, page, + SceKernelMtypeC, &physical), + "allocate failed-unmap direct fixture"); + CheckOk(test, + KernelMapNamedDirectMemory(&address, 3 * page, SceKernelProtCpuRw, + SceKernelMapFixed, physical, page, "asan_failure"), + "map failed-unmap direct fixture"); + } else { + MapAsanFlexible(test, base, 3 * page, "asan_failure"); + } + for (uint64_t i = 0; i < 3; ++i) + *reinterpret_cast(base + i * page) = 0xabc0 + i; + auto* sentinel = reinterpret_cast(AsanShadowAddress(base + 2 * page)); + *sentinel = 0x4641494c53484457ull; + CheckOk(test, + KernelMprotect(reinterpret_cast(base + page), page, SceKernelProtCpuRead), + "split protection after callback registration"); + CheckOk(test, KernelMprotect(address, 3 * page, SceKernelProtCpuRead), + "merge protection after callback registration"); + CheckOk( + test, + KernelSetVirtualRangeName(reinterpret_cast(base + page), page, "split_shadow"), + "split name after callback registration"); + CheckOk(test, KernelSetVirtualRangeName(address, 3 * page, "asan_failure"), + "merge name after callback registration"); + const auto check_ranges = [&](uint64_t first) { + for (uint64_t i = first; i < 3; ++i) { + ExpectRange(test, Query(test, base + i * page), base + (direct ? i : first) * page, + base + (direct ? i + 1 : 3) * page, SceKernelProtCpuRead, !direct, + direct, 0, 1, "asan_failure", direct ? physical + i * page : 0); + } + }; + check_ranges(0); + if (direct) + TestFailPhysicalMemoryUnmapAfter(0); + else + TestFailGuestBackingStoreUnmapAfter(0); + CheckFailed(test, KernelMunmap(base, 3 * page), "reject unmap before any range completes"); + Check(test, AsanShadowPresent(base) && *sentinel == 0x4641494c53484457ull, + "failed unmap released or changed ASan shadow"); + for (uint64_t i = 0; i < 3; ++i) { + Check(test, *reinterpret_cast(base + i * page) == 0xabc0 + i, + "failed unmap changed guest data"); + // Windows recreates untouched direct neighbors with their original host mode. + if (!direct || i == 0 || KYTY_PLATFORM != KYTY_PLATFORM_WINDOWS) + ExpectHostReadOnly(test, base + i * page); + } + CheckOk(test, KernelMprotect(address, 3 * page, SceKernelProtCpuRead), + "restore complete fixture protection before partial-progress unmap"); + CheckOk(test, KernelMprotect(address, page, SceKernelProtCpuRw), "split first unmap chunk"); + if (direct) + TestFailPhysicalMemoryUnmapAfter(1); + else + TestFailGuestBackingStoreUnmapAfter(1); + CheckFailed(test, KernelMunmap(base, 3 * page), + "reject later unmap after first range completes"); + ExpectUnmapped(test, base); + check_ranges(1); + Check(test, + AsanShadowPresent(base) && *sentinel == 0x4641494c53484457ull && + *reinterpret_cast(base + 2 * page) == 0xabc2, + "partially completed unmap damaged the remaining shadow or data"); + ExpectHostReadOnly(test, base + page); + if (!direct || KYTY_PLATFORM != KYTY_PLATFORM_WINDOWS) + ExpectHostReadOnly(test, base + 2 * page); + CheckOk(test, KernelMunmap(base + page, page), "retry first surviving page"); + Check(test, AsanShadowPresent(base) && *sentinel == 0x4641494c53484457ull, + "partial retry released shadow still covering the last page"); + CheckOk(test, KernelMunmap(base + 2 * page, page), "retry final surviving page"); + Check(test, !AsanShadowPresent(base), + "final retry leaked shadow coverage after failed unmap"); + CheckOk(test, KernelMunmap(base + 3 * page, AsanShadowCoverage - 3 * page), + "release failure fixture reservation"); + if (direct) + CheckOk(test, KernelReleaseDirectMemory(physical, 3 * page), + "release failed-unmap physical memory"); + } + std::printf("[host] %-48s ok\n", test); +} + +void TestAsanAliasedPhysicalRelease() { + using namespace Libs::LibKernel::Memory; + const char* test = "AsanAliasedPhysicalRelease"; + constexpr uint64_t page = SceKernelPageSize; + int64_t physical = -1; + CheckOk(test, + KernelAllocateDirectMemory(0, KernelGetDirectMemorySize(), 4 * page, page, + SceKernelMtypeC, &physical), + "allocate aliased shadow fixture"); + const std::array bases {ReserveAsanCoverage(test), ReserveAsanCoverage(test)}; + for (const auto base: bases) { + void* address = reinterpret_cast(base); + CheckOk(test, + KernelMapNamedDirectMemory(&address, 4 * page, SceKernelProtCpuRw, + SceKernelMapFixed, physical, page, "asan_alias"), + "map shared physical shadow fixture"); + *reinterpret_cast(AsanShadowAddress(base + 3 * page)) = 0x414c494153534844ull; + } + for (uint64_t i = 0; i < 4; ++i) + *reinterpret_cast(bases[0] + i * page) = 0xd1a0 + i; + CheckOk( + test, + KernelMprotect(reinterpret_cast(bases[0] + page), 2 * page, SceKernelProtCpuRead), + "split alias protections after callback registration"); + CheckOk(test, + KernelSetVirtualRangeName(reinterpret_cast(bases[0] + 2 * page), page, + "alias_split"), + "split alias names after callback registration"); + TestFailPhysicalMemoryUnmapAfter(1); + CheckFailed(test, KernelCheckedReleaseDirectMemory(physical + page, 2 * page), + "release aliased middle with injected metadata failure"); + for (const auto base: bases) { + Check(test, + AsanShadowPresent(base) && *reinterpret_cast(AsanShadowAddress( + base + 3 * page)) == 0x414c494153534844ull, + "failed aliased physical release changed shadow coverage"); + for (uint64_t i = 0; i < 4; ++i) + Check(test, *reinterpret_cast(base + i * page) == 0xd1a0 + i, + "failed aliased physical release changed backing contents"); + } + CheckOk(test, KernelCheckedReleaseDirectMemory(physical + page, 2 * page), + "release aliased middle after rollback"); + for (const auto base: bases) { + ExpectUnmapped(test, base + page); + ExpectUnmapped(test, base + 2 * page); + Check(test, + AsanShadowPresent(base) && + *reinterpret_cast(AsanShadowAddress(base + 3 * page)) == + 0x414c494153534844ull && + *reinterpret_cast(base) == 0xd1a0 && + *reinterpret_cast(base + 3 * page) == 0xd1a3, + "split alias release damaged surviving shadow or data"); + } + CheckOk(test, KernelCheckedReleaseDirectMemory(physical, page), "release aliased prefix"); + for (const auto base: bases) + Check(test, AsanShadowPresent(base), + "aliased prefix release freed the final page's shadow"); + CheckOk(test, KernelCheckedReleaseDirectMemory(physical + 3 * page, page), + "release final aliased page"); + for (const auto base: bases) { + Check(test, !AsanShadowPresent(base), + "final aliased physical release leaked shadow coverage"); + CheckOk(test, KernelMunmap(base + 4 * page, AsanShadowCoverage - 4 * page), + "release alias reservation"); + } + std::printf("[host] %-48s ok\n", test); +} + +void TestAsanPartialFreeAcrossSharedShadowBoundary() { + using namespace Libs::LibKernel::Memory; + const char* test = "AsanPartialFreeAcrossSharedShadowBoundary"; + constexpr uint64_t page = SceKernelPageSize; + const auto base = ReserveAsanCoverage(test, 2); + const auto boundary = base + AsanShadowCoverage; + const auto left = boundary - 3 * page; + const auto right = boundary + 2 * page; + MapAsanFlexible(test, boundary - 2 * page, 4 * page, "asan_crossing"); + MapAsanFlexible(test, left, page, "asan_neighbor"); + MapAsanFlexible(test, right, page, "asan_neighbor"); + for (const auto neighbor: {left, right}) { + *reinterpret_cast(neighbor) = neighbor; + *reinterpret_cast(AsanShadowAddress(neighbor)) = neighbor; + } + const auto check_neighbors = [&] { + for (const auto neighbor: {left, right}) + Check(test, + AsanShadowPresent(neighbor) && + *reinterpret_cast(AsanShadowAddress(neighbor)) == neighbor && + *reinterpret_cast(neighbor) == neighbor, + "cross-boundary partial free changed a neighbor's shadow or contents"); + }; + CheckOk(test, KernelMunmap(boundary - page, 2 * page), + "free middle spanning exact shadow boundary"); + check_neighbors(); + CheckOk(test, KernelMunmap(boundary - 2 * page, page), "free crossing allocation prefix"); + CheckOk(test, KernelMunmap(boundary + page, page), "free crossing allocation suffix"); + check_neighbors(); + CheckOk(test, KernelMunmap(left, page), "free final left shadow user"); + Check(test, + !AsanShadowPresent(left) && AsanShadowPresent(right) && + *reinterpret_cast(AsanShadowAddress(right)) == right, + "left final free retained its shadow or released the shared right shadow"); + CheckOk(test, KernelMunmap(right, page), "free final right shadow user"); + Check(test, !AsanShadowPresent(right), "right final free leaked shadow after boundary split"); + CheckOk(test, KernelMunmap(base, AsanShadowCoverage - 3 * page), + "release boundary prefix reservation"); + CheckOk(test, KernelMunmap(boundary + 3 * page, AsanShadowCoverage - 3 * page), + "release boundary suffix reservation"); + std::printf("[host] %-48s ok\n", test); +} + void TestCopySparseMemoryMappings() { using namespace Libs::LibKernel::Memory; const char* test = "CopySparseMemoryMappings"; @@ -1752,6 +2291,82 @@ void TestDirectPartialProtectUnmapPreservesNeighbors() { std::printf("[host] %-48s ok\n", test); } +void TestDirectTrimmedAliasReleaseRollback() { + using namespace Libs::LibKernel::Memory; + const char* test = "DirectTrimmedAliasReleaseRollback"; + constexpr uint64_t page = SceKernelPageSize; + constexpr uint64_t size = 6 * page; + int64_t physical = -1; + CheckOk(test, + KernelAllocateDirectMemory(0, KernelGetDirectMemorySize(), size, page, SceKernelMtypeC, + &physical), + "allocate alias backing"); + void* original = nullptr; + void* alias = nullptr; + CheckOk(test, + KernelMapNamedDirectMemory(&original, size, SceKernelProtCpuRw, 0, physical, page, + "trim_original"), + "map original"); + CheckOk(test, + KernelMapNamedDirectMemory(&alias, size, SceKernelProtCpuRw, 0, physical, page, + "trim_alias"), + "map alias"); + const auto base = reinterpret_cast(original); + const auto alias_base = reinterpret_cast(alias); + const auto mapping_count = TestDirectMappingCount(); + for (uint64_t i = 0; i < 6; ++i) { + *reinterpret_cast(alias_base + i * page) = 0x414c494153000000ull + i; + } + CheckOk( + test, + KernelMprotect(reinterpret_cast(base + 2 * page), 2 * page, SceKernelProtCpuRead), + "protect original middle"); + CheckOk( + test, + KernelSetVirtualRangeName(reinterpret_cast(base + page), 4 * page, "trim_middle"), + "rename original interior"); + Check(test, TestDirectMappingCount() == mapping_count, + "direct metadata edits fragmented backing ownership records"); + CheckOk(test, KernelMunmap(base, page), "trim original prefix"); + CheckOk(test, KernelMunmap(base + 5 * page, page), "trim original suffix"); + const auto check_survivors = [&] { + ExpectUnmapped(test, base); + ExpectUnmapped(test, base + 5 * page); + ExpectRange(test, Query(test, base + page), base + page, base + 2 * page, + SceKernelProtCpuRw, 0, 1, 0, 1, "trim_middle", physical + page); + ExpectRange(test, Query(test, base + 2 * page), base + 2 * page, base + 4 * page, + SceKernelProtCpuRead, 0, 1, 0, 1, "trim_middle", physical + 2 * page); + ExpectRange(test, Query(test, base + 4 * page), base + 4 * page, base + 5 * page, + SceKernelProtCpuRw, 0, 1, 0, 1, "trim_middle", physical + 4 * page); + ExpectRange(test, Query(test, alias_base), alias_base, alias_base + size, + SceKernelProtCpuRw, 0, 1, 0, 1, "trim_alias", physical); + for (uint64_t i = 0; i < 6; ++i) { + Check(test, + *reinterpret_cast(alias_base + i * page) == + 0x414c494153000000ull + i, + "trimmed mapping or rollback damaged its alias backing"); + if (i > 0 && i < 5) { + Check(test, + *reinterpret_cast(base + i * page) == + 0x414c494153000000ull + i, + "trimmed mapping lost its physical offset after rekey or restore"); + } + } + }; + check_survivors(); + TestFailPhysicalMemoryUnmapAfter(1); + CheckFailed(test, KernelCheckedReleaseDirectMemory(physical + 3 * page, page), + "release aliases with injected second-range failure"); + check_survivors(); + // The failed physical release cut inside the read-only VMA. Its rollback restored a + // separate ownership extent there; a single semantic VMA now crosses two extents. + CheckOk(test, KernelMunmap(base + page, 4 * page), "unmap restored ownership fragments"); + CheckOk(test, KernelCheckedReleaseDirectMemory(physical, size), "release aliases"); + ExpectUnmapped(test, base + page); + ExpectUnmapped(test, alias_base); + std::printf("[host] %-48s ok\n", test); +} + void TestFixedArenaChurnKeepsRangesExact() { using namespace Libs::LibKernel::Memory; const char* test = "FixedArenaChurnKeepsRangesExact"; @@ -3398,88 +4013,150 @@ void TestProsperoSampleMemoryPoolExpandCommit() { } void TestFragmentedMemoryPoolBacking() { - const char* test = "FragmentedMemoryPoolBacking"; - - int64_t first_pool = -1; - int64_t direct_gap = -1; - int64_t second_pool = -1; - const auto direct_end = - static_cast(Libs::LibKernel::Memory::KernelGetDirectMemorySize()); + using namespace Libs::LibKernel::Memory; + const char* test = "FragmentedMemoryPoolBacking"; + constexpr uint64_t block = SceKernelMemoryPoolCommitLen; + int64_t first_pool = -1, direct_gap = -1, second_pool = -1; + const auto direct_end = static_cast(KernelGetDirectMemorySize()); CheckOk( test, - Libs::LibKernel::Memory::KernelMemoryPoolExpand(0, direct_end, SceKernelMemoryPoolCommitLen, - SceKernelMemoryPoolAlignment, &first_pool), + KernelMemoryPoolExpand(0, direct_end, 2 * block, SceKernelMemoryPoolAlignment, &first_pool), "KernelMemoryPoolExpand(first)"); CheckOk(test, - Libs::LibKernel::Memory::KernelAllocateDirectMemory( - 0, direct_end, SceKernelMemoryPoolCommitLen, SceKernelMemoryPoolAlignment, - SceKernelMtypeC, &direct_gap), + KernelAllocateDirectMemory(0, direct_end, block, SceKernelMemoryPoolAlignment, + SceKernelMtypeC, &direct_gap), "KernelAllocateDirectMemory(gap)"); CheckOk( test, - Libs::LibKernel::Memory::KernelMemoryPoolExpand(0, direct_end, SceKernelMemoryPoolCommitLen, - SceKernelMemoryPoolAlignment, &second_pool), + KernelMemoryPoolExpand(0, direct_end, block, SceKernelMemoryPoolAlignment, &second_pool), "KernelMemoryPoolExpand(second)"); Check(test, - first_pool + static_cast(SceKernelMemoryPoolCommitLen) == direct_gap && - direct_gap + static_cast(SceKernelMemoryPoolCommitLen) == second_pool, + first_pool + static_cast(2 * block) == direct_gap && + direct_gap + static_cast(block) == second_pool, "test setup did not create nonadjacent pool expansions"); - void* arena = nullptr; - CheckOk(test, - Libs::LibKernel::Memory::KernelMemoryPoolReserve(nullptr, SceKernelMemoryPoolReserveLen, - 0, 0, &arena), + CheckOk(test, KernelMemoryPoolReserve(nullptr, SceKernelMemoryPoolReserveLen, 0, 0, &arena), "KernelMemoryPoolReserve"); - const auto base = reinterpret_cast(arena); - const auto commit_len = SceKernelMemoryPoolCommitLen * 2; - CheckOk(test, - Libs::LibKernel::Memory::KernelMemoryPoolCommit(arena, commit_len, SceKernelMtypeC, - SceKernelProtCpuRw, 0), + const auto base = reinterpret_cast(arena); + constexpr auto commit_len = 3 * block; + CheckOk(test, KernelMemoryPoolCommit(arena, commit_len, SceKernelMtypeC, SceKernelProtCpuRw, 0), "KernelMemoryPoolCommit(fragmented)"); - CheckFailed(test, - Libs::LibKernel::Memory::KernelMemoryPoolCommit( - reinterpret_cast(base + commit_len), SceKernelMemoryPoolCommitLen, - SceKernelMtypeC, SceKernelProtCpuRw, 0), + KernelMemoryPoolCommit(reinterpret_cast(base + commit_len), block, + SceKernelMtypeC, SceKernelProtCpuRw, 0), "KernelMemoryPoolCommit(exhausted)"); ExpectRange(test, Query(test, base + commit_len), base + commit_len, base + SceKernelMemoryPoolReserveLen, 0, 0, 0, 1, 0); - - *reinterpret_cast(base) = 0x465241474d454e54ull; // "FRAGMENT" - *reinterpret_cast(base + SceKernelMemoryPoolCommitLen) = 0x504f4f4c50414745ull; - CheckOk(test, Libs::LibKernel::Memory::KernelMemoryPoolDecommit(arena, commit_len, 0), + for (uint64_t i = 0; i < 3; ++i) + *reinterpret_cast(base + i * block) = 0x504f4f4c0000ull + i; + CheckOk(test, KernelMemoryPoolDecommit(arena, commit_len, 0), "KernelMemoryPoolDecommit(fragmented)"); - - Libs::LibKernel::Memory::KernelMemoryPoolBlockStats stats {}; - CheckOk(test, Libs::LibKernel::Memory::KernelMemoryPoolGetBlockStats(&stats, sizeof(stats)), - "KernelMemoryPoolGetBlockStats(fragmented decommit)"); - Check(test, stats.available_flushed_blocks == 2 && stats.allocated_flushed_blocks == 0, - "fragmented decommit did not restore both pool pages"); - - CheckOk(test, - Libs::LibKernel::Memory::KernelMemoryPoolCommit(arena, commit_len, SceKernelMtypeC, - SceKernelProtCpuRw, 0), + KernelMemoryPoolBlockStats stats {}; + CheckOk(test, KernelMemoryPoolGetBlockStats(&stats, sizeof(stats)), + "query fragmented decommit budget"); + Check(test, stats.available_flushed_blocks == 3 && stats.allocated_flushed_blocks == 0, + "fragmented decommit did not restore all pool blocks"); + CheckOk(test, KernelMemoryPoolCommit(arena, commit_len, SceKernelMtypeC, SceKernelProtCpuRw, 0), "KernelMemoryPoolCommit(fragmented recommit)"); - CheckOk(test, Libs::LibKernel::Memory::KernelMunmap(base, commit_len), - "KernelMunmap(fragmented commit)"); - CheckOk(test, - Libs::LibKernel::Memory::KernelMunmap(base + commit_len, - SceKernelMemoryPoolReserveLen - commit_len), + CheckOk(test, KernelMprotect(arena, commit_len, SceKernelProtCpuRead), + "protect fragmented pool views"); + for (const bool partial: {true, false}) { + const auto address = partial ? base + block : base; + const auto size = partial ? 2 * block : commit_len; + g_gpu_range_events.clear(); + TestSetGpuRangeCallback(ObserveGpuRange); + TestFailGuestBackingStoreUnmapAfter(1); + const int result = KernelMunmap(address, size); + TestSetGpuRangeCallback(nullptr); + CheckFailed(test, result, + partial ? "KernelMunmap(partial pooled view rollback)" + : "KernelMunmap(whole pooled range after split rollback)"); + ExpectRange(test, Query(test, base), base, base + commit_len, SceKernelProtCpuRead, 0, 0, 1, + 1); + Check(test, + g_gpu_range_events == + std::vector {{false, address, size}, {true, address, size}}, + "failed pooled unmap did not restore the complete GPU range"); + if (partial) { + CheckOk(test, KernelMprotect(arena, commit_len, SceKernelProtCpuRead), + "protect whole pool range after partial-first-view rollback"); + } + for (uint64_t i = 0; i < 3; ++i) { + Check(test, + *reinterpret_cast(base + i * block) == 0x504f4f4c0000ull + i, + "pooled rollback changed a backing view's contents"); + ExpectHostReadOnly(test, base + i * block); + } + } + CheckOk(test, KernelMemoryPoolGetBlockStats(&stats, sizeof(stats)), + "query pooled rollback budget"); + Check(test, stats.available_flushed_blocks == 0 && stats.allocated_flushed_blocks == 3, + "failed pooled unmap changed the pool budget"); + CheckOk(test, KernelMunmap(base, commit_len), "KernelMunmap(fragmented commit)"); + CheckOk(test, KernelMunmap(base + commit_len, SceKernelMemoryPoolReserveLen - commit_len), "KernelMunmap(fragmented reserve remainder)"); + CheckOk(test, KernelReleaseDirectMemory(first_pool, 2 * block), "release first pool expansion"); + CheckOk(test, KernelReleaseDirectMemory(second_pool, block), "release second pool expansion"); + CheckOk(test, KernelReleaseDirectMemory(direct_gap, block), "release pool expansion gap"); + std::printf("[host] %-48s ok\n", test); +} +void TestMemoryPoolMiddleDecommitAndReuse() { + using namespace Libs::LibKernel::Memory; + const char* test = "MemoryPoolMiddleDecommitAndReuse"; + constexpr uint64_t block = SceKernelMemoryPoolCommitLen; + const auto flexible_before = AvailableFlexibleMemory(test); + const auto expect_stats = [&](int free, int allocated) { + KernelMemoryPoolBlockStats stats {}; + CheckOk(test, KernelMemoryPoolGetBlockStats(&stats, sizeof(stats)), "query pool blocks"); + Check(test, + stats.available_flushed_blocks == free && stats.allocated_flushed_blocks == allocated, + "middle decommit/recommit changed the wrong pool budget"); + }; + int64_t physical = -1; CheckOk(test, - Libs::LibKernel::Memory::KernelReleaseDirectMemory(first_pool, - SceKernelMemoryPoolCommitLen), - "KernelReleaseDirectMemory(first pool)"); - CheckOk(test, - Libs::LibKernel::Memory::KernelReleaseDirectMemory(second_pool, - SceKernelMemoryPoolCommitLen), - "KernelReleaseDirectMemory(second pool)"); - CheckOk(test, - Libs::LibKernel::Memory::KernelReleaseDirectMemory(direct_gap, - SceKernelMemoryPoolCommitLen), - "KernelReleaseDirectMemory(gap)"); - + KernelMemoryPoolExpand(0, KernelGetDirectMemorySize(), 3 * block, block, &physical), + "expand three contiguous pool blocks"); + void* arena = nullptr; + CheckOk(test, KernelMemoryPoolReserve(nullptr, SceKernelMemoryPoolReserveLen, 0, 0, &arena), + "reserve pool arena"); + const auto base = reinterpret_cast(arena); + CheckOk(test, KernelMemoryPoolCommit(arena, 3 * block, SceKernelMtypeC, SceKernelProtCpuRw, 0), + "commit one contiguous mapping"); + for (uint64_t i = 0; i < 3; ++i) { + *reinterpret_cast(base + i * block) = 0x4d4944444c450000ull + i; + } + void* middle = reinterpret_cast(base + block); + CheckOk(test, KernelMemoryPoolDecommit(middle, block, 0), "decommit middle block"); + ExpectRange(test, Query(test, base), base, base + block, SceKernelProtCpuRw, 0, 0, 1, 1); + ExpectRange(test, Query(test, base + block), base + block, base + 2 * block, 0, 0, 0, 1, 0); + ExpectRange(test, Query(test, base + 2 * block), base + 2 * block, base + 3 * block, + SceKernelProtCpuRw, 0, 0, 1, 1); + Check(test, + *reinterpret_cast(base) == 0x4d4944444c450000ull && + *reinterpret_cast(base + 2 * block) == 0x4d4944444c450002ull, + "middle decommit damaged a surviving neighbor"); + uint64_t value = 0; + Check(test, !TryReadBacking(base + block, &value, sizeof(value)), + "decommitted middle block retained its host backing view"); + expect_stats(1, 2); + CheckOk(test, KernelMemoryPoolCommit(middle, block, SceKernelMtypeC, SceKernelProtCpuRead, 0), + "recommit middle block read-only"); + for (uint64_t i = 0; i < 3; ++i) { + ExpectRange(test, Query(test, base + i * block), base + i * block, base + (i + 1) * block, + i == 1 ? SceKernelProtCpuRead : SceKernelProtCpuRw, 0, 0, 1, 1); + Check(test, + *reinterpret_cast(base + i * block) == 0x4d4944444c450000ull + i, + "middle recommit did not reuse its own physical backing and contents"); + } + expect_stats(0, 3); + Check(test, AvailableFlexibleMemory(test) == flexible_before, + "middle pool decommit/recommit consumed flexible memory"); + CheckOk(test, KernelMemoryPoolDecommit(arena, 3 * block, 0), "decommit all three blocks"); + expect_stats(3, 0); + CheckOk(test, KernelMunmap(base, SceKernelMemoryPoolReserveLen), "release pool arena"); + CheckOk(test, KernelCheckedReleaseDirectMemory(physical, 3 * block), "release pool expansion"); + expect_stats(0, 0); std::printf("[host] %-48s ok\n", test); } @@ -3563,6 +4240,84 @@ void TestMemoryPoolMultiRangeDecommit() { std::printf("[host] %-48s ok\n", test); } +struct PoolDecommitRequest { + void* address; + size_t size; + int result = -1; +}; + +#if KYTY_PLATFORM == KYTY_PLATFORM_WINDOWS +DWORD WINAPI DecommitPoolOnSmallStack(void* argument) { +#else +void* DecommitPoolOnSmallStack(void* argument) { +#endif + auto& request = *static_cast(argument); + request.result = + Libs::LibKernel::Memory::KernelMemoryPoolDecommit(request.address, request.size, 0); + return 0; +} + +void TestFragmentedPoolDecommitOnSmallStack() { + using namespace Libs::LibKernel::Memory; + const char* test = "FragmentedPoolDecommitOnSmallStack"; + constexpr uint64_t count = 1024; + constexpr uint64_t block = SceKernelMemoryPoolCommitLen; + constexpr uint64_t size = count * block; + constexpr size_t stack_size = 320 * 1024; + int64_t physical = -1; + CheckOk(test, + KernelMemoryPoolExpand(0, KernelGetDirectMemorySize(), size, + SceKernelMemoryPoolAlignment, &physical), + "expand fragmented pool"); + void* arena = nullptr; + CheckOk(test, KernelMemoryPoolReserve(nullptr, size, 0, 0, &arena), "reserve fragmented pool"); + const auto base = reinterpret_cast(arena); + for (uint64_t i = 0; i < count; ++i) { + const auto protection = i % 2 == 0 ? SceKernelProtCpuRw : SceKernelProtCpuRead; + CheckOk(test, + KernelMemoryPoolCommit(reinterpret_cast(base + i * block), block, + SceKernelMtypeC, protection, 0), + "commit alternating-protection pool block"); + const uint64_t sentinel = 0x504f4f4c00000000ull + i; + Check(test, TryWriteBacking(base + i * block, &sentinel, sizeof(sentinel)), + "could not initialize committed pool backing"); + ExpectRange(test, Query(test, base + i * block), base + i * block, base + (i + 1) * block, + protection, 0, 0, 1, 1); + } + + KernelMemoryPoolBlockStats stats {}; + PoolDecommitRequest request {arena, size}; +#if KYTY_PLATFORM == KYTY_PLATFORM_WINDOWS + HANDLE thread = CreateThread(nullptr, stack_size, DecommitPoolOnSmallStack, &request, + STACK_SIZE_PARAM_IS_A_RESERVATION, nullptr); + Check(test, thread != nullptr, "failed to create small-stack decommit thread"); + const auto waited = WaitForSingleObject(thread, INFINITE); + CloseHandle(thread); + Check(test, waited == WAIT_OBJECT_0, "failed to join small-stack decommit thread"); +#else + pthread_attr_t attributes; + Check(test, pthread_attr_init(&attributes) == 0, "failed to initialize thread attributes"); + const int stack_result = pthread_attr_setstacksize(&attributes, stack_size); + pthread_t thread {}; + const int create_result = + stack_result == 0 ? pthread_create(&thread, &attributes, DecommitPoolOnSmallStack, &request) + : stack_result; + pthread_attr_destroy(&attributes); + if (create_result != 0) { + Fail(test, "failed to create small-stack decommit thread"); + } + Check(test, pthread_join(thread, nullptr) == 0, "failed to join small-stack decommit thread"); +#endif + CheckOk(test, request.result, "decommit 1000+ ranges on 320 KiB stack"); + ExpectRange(test, Query(test, base), base, base + size, 0, 0, 0, 1, 0); + CheckOk(test, KernelMemoryPoolGetBlockStats(&stats, sizeof(stats)), "query decommitted pool"); + Check(test, stats.available_flushed_blocks == count && stats.allocated_flushed_blocks == 0, + "fragmented decommit leaked pool blocks"); + CheckOk(test, KernelMunmap(base, size), "release fragmented pool reservation"); + CheckOk(test, KernelReleaseDirectMemory(physical, size), "release fragmented pool backing"); + std::printf("[host] %-48s ok\n", test); +} + void TestMemoryPoolCommitDecommitQueryFlags() { const char* test = "MemoryPoolCommitDecommitQueryFlags"; void* addr = nullptr; @@ -4955,6 +5710,34 @@ void TestTlsStubsPreserveFlags() { int main(int argc, char** argv) { InitSubsystems(); + if (argc == 2 && std::strcmp(argv[1], "--memory-bottlenecks-only") == 0) { + RunTest(TestFlexibleZeroingTracksBackingLifetime); + RunTest(TestFlexibleMemoryReuseIsZeroFilled); + RunTest(TestFlexibleReplacementRollbackPreservesMetadata); + RunTest(TestFragmentedBackingUnmapRollback); + RunTest(TestSmallerFlexibleMapReusesReleasedHole); + RunTest(TestFragmentedGuestFreeRangeHints); + RunTest(TestGuestFreeRangeSearchSkipsEarlierFragments); + RunTest(TestGuestFreeRangeSearchDoesNotUnderflow); + RunTest(TestDirectMapQueryOffsetAndPartialMunmap); + RunTest(TestDirectPartialProtectUnmapPreservesNeighbors); + RunTest(TestDirectTrimmedAliasReleaseRollback); + RunTest(TestDirectMappingNamesTypesAndValidation); + RunTest(TestFixedReserveRollbackSkipsUntouchedChunks); + RunTest(TestFixedReserveRangeAddRollbackKeepsPlaceholder); + RunTest(TestAutomaticMemoryReuseAndAliases); + RunTest(TestAutomaticMemoryFragmentedMapRollback); + RunTest(TestFragmentedMemoryPoolBacking); + RunTest(TestMemoryPoolMiddleDecommitAndReuse); + RunTest(TestMemoryPoolMultiRangeDecommit); + RunTest(TestFragmentedPoolDecommitOnSmallStack); + // Memory callbacks remain registered for the rest of the process. + RunTest(TestAsanShadowLifetimeAcrossRangeEdits); + RunTest(TestAsanFailedUnmapPreservesShadow); + RunTest(TestAsanAliasedPhysicalRelease); + RunTest(TestAsanPartialFreeAcrossSharedShadowBoundary); + return g_failed_tests == 0 ? 0 : 1; + } if (argc == 2 && std::strcmp(argv[1], "--virtual-ranges-only") == 0) { RunTest(TestFlexibleNoCoalescePreservesBoundaries); RunTest(TestFlexibleRangeEditsCoalesce); @@ -4975,6 +5758,7 @@ int main(int argc, char** argv) { return g_failed_tests == 0 ? 0 : 1; } if (argc == 2 && std::strcmp(argv[1], "--backing-transfers-only") == 0) { + RunTest(TestFlexibleZeroingTracksBackingLifetime); RunTest(TestConcurrentBackingReads); RunTest(TestBackingReadExcludesWritesAndUnmap); RunTest(TestTrackerProtectSharesBackingReads); @@ -5019,6 +5803,7 @@ int main(int argc, char** argv) { return g_failed_tests == 0 ? 0 : 1; } + RunTest(TestFlexibleZeroingTracksBackingLifetime); #if defined(__x86_64__) || defined(_M_X64) RunTest(TestSmallFiberStacksAndMigration); #endif @@ -5040,6 +5825,8 @@ int main(int argc, char** argv) { RunTest(TestTrackerProtectSharesBackingReads); RunTest(TestGuestAddressSpaceHasNoFixedFallback); RunTest(TestGuestFreeRangeSearchDoesNotUnderflow); + RunTest(TestFragmentedGuestFreeRangeHints); + RunTest(TestGuestFreeRangeSearchSkipsEarlierFragments); RunTest(TestFlexibleMemoryCapacityIsBootFixed); RunTest(TestFlexibleMemoryUsesSharedBacking); RunTest(TestFlexibleDmemCompatAndAlignmentFlags); @@ -5049,6 +5836,7 @@ int main(int argc, char** argv) { RunTest(TestGuestStackUsesPrivateOwnerMemoryAndCache); RunTest(TestMainEntryUsesGuestStackAndDisablesHostChecks); RunTest(TestFragmentedBackingUnmapRollback); + RunTest(TestFlexibleReplacementRollbackPreservesMetadata); RunTest(TestRuntimeMemoryOwnerLifecycle); RunTest(TestFlexibleMapQueryAndWholeMunmap); RunTest(TestPartialFlexibleMunmapAndFindNext); @@ -5060,6 +5848,7 @@ int main(int argc, char** argv) { RunTest(TestMunmapAcrossAdjacentFlexibleMappings); RunTest(TestDirectMapQueryOffsetAndPartialMunmap); RunTest(TestDirectPartialProtectUnmapPreservesNeighbors); + RunTest(TestDirectTrimmedAliasReleaseRollback); RunTest(TestFixedArenaChurnKeepsRangesExact); #if defined(__linux__) RunTest(TestPartialUnmapPreservesHostPermissions); @@ -5095,7 +5884,9 @@ int main(int argc, char** argv) { RunTest(TestMemoryPoolAlignmentContracts); RunTest(TestProsperoSampleMemoryPoolExpandCommit); RunTest(TestFragmentedMemoryPoolBacking); + RunTest(TestMemoryPoolMiddleDecommitAndReuse); RunTest(TestMemoryPoolMultiRangeDecommit); + RunTest(TestFragmentedPoolDecommitOnSmallStack); RunTest(TestMemoryPoolCommitDecommitQueryFlags); RunTest(TestProgramMemoryAllocationAndProtection); RunTest(TestModuleRelocationUsesWritableHostMapping); @@ -5103,6 +5894,11 @@ int main(int argc, char** argv) { #if defined(__linux__) || KYTY_PLATFORM == KYTY_PLATFORM_WINDOWS RunTest(TestTlsStubsPreserveFlags); #endif + // Memory callbacks remain registered for the rest of the process. + RunTest(TestAsanShadowLifetimeAcrossRangeEdits); + RunTest(TestAsanFailedUnmapPreservesShadow); + RunTest(TestAsanAliasedPhysicalRelease); + RunTest(TestAsanPartialFreeAcrossSharedShadowBoundary); if (g_failed_tests != 0) { std::printf("VirtualMemoryAllocationTests: %d case(s) failed\n", g_failed_tests);