diff --git a/src/kernel/memory.cpp b/src/kernel/memory.cpp index d41d30ad7..62b0e3d1d 100644 --- a/src/kernel/memory.cpp +++ b/src/kernel/memory.cpp @@ -166,7 +166,8 @@ static bool VirtualRangesOverlap(uint64_t left_start, uint64_t left_size, uint64 } #if defined(KYTY_VIRTUAL_MEMORY_ALLOCATION_TESTS) -static uint32_t g_test_backing_store_unmaps_before_failure = UINT32_MAX; +static uint32_t g_test_backing_store_unmaps_before_failure = UINT32_MAX; +static callback_func_t g_test_before_backing_map = nullptr; #endif #include "memoryAddressSpace.inc" @@ -3024,6 +3025,22 @@ int KYTY_SYSV_ABI KernelMapDirectMemory(void** addr, size_t len, int prot, int f return g_guest_address_space->MapBacking(target_addr, len, direct_memory_start, mode, &shared_failure); }; + std::vector replaced_mappings; + auto restore_mappings = [&](size_t count, bool restore_views) { + for (size_t index = 0; index < count; ++index) { + const auto& old = replaced_mappings[index]; + if (restore_views) { + EXIT_IF(!g_guest_address_space->MapBacking(old.map_vaddr, old.map_size, + old.start_addr, old.mode)); + } + EXIT_IF(!g_physical_memory->Map(old.map_vaddr, old.start_addr, old.map_size, + old.prot, old.mode, old.gpu_mode)); + g_physical_memory->SetVirtualRangeName(old.map_vaddr, old.map_size, old.name); + g_physical_memory->SetVirtualRangeMemoryType(old.map_vaddr, old.map_size, + old.memory_type); + } + MapGpuRange(in_addr, len); + }; if (fixed) { if (in_addr == 0 || (in_addr & (PAGE_SIZE - 1u)) != 0 || (alignment != 0 && in_addr % alignment != 0)) { @@ -3034,22 +3051,68 @@ int KYTY_SYSV_ABI KernelMapDirectMemory(void** addr, size_t len, int prot, int f } std::vector reserved_ranges; +#if KYTY_PLATFORM == KYTY_PLATFORM_LINUX if (g_virtual_ranges->QuerySpan(in_addr, len, &reserved_ranges) && std::all_of(reserved_ranges.begin(), reserved_ranges.end(), [](const auto& range) { - return range.type == VirtualRangeType::Reserved; + return range.type == VirtualRangeType::Direct; })) { + uint64_t covered = 0; + { + Common::LockGuard lock(g_physical_memory->GetMutex()); + for (const auto& mapping: g_physical_memory->GetMappings()) { + const auto start = std::max(in_addr, mapping.map_vaddr); + const auto end = std::min(in_addr + len, mapping.map_vaddr + mapping.map_size); + if (start >= end) { + continue; + } + auto old = mapping; + old.start_addr += start - old.map_vaddr; + old.map_vaddr = start; + old.size = old.map_size = end - start; + covered += old.map_size; + replaced_mappings.push_back(old); + } + } + if (covered != len) { + return KERNEL_ERROR_EBUSY; + } UnmapGpuRange(in_addr, len); - reserved_target = true; - } - if (!reserved_target && ReplaceFixedRangeWithReserved(in_addr, len)) { - reserved_target = true; - } - if (!reserved_target) { - return KERNEL_ERROR_ENOMEM; - } - if (map_shared_fixed(in_addr)) { + for (size_t index = 0; index < replaced_mappings.size(); ++index) { + const auto& old = replaced_mappings[index]; + GpuAccessMode old_gpu_mode = GpuAccessMode::NoAccess; + if (!g_physical_memory->Unmap(old.map_vaddr, old.map_size, &old_gpu_mode)) { + restore_mappings(index, false); + return KERNEL_ERROR_EBUSY; + } + } + // Keep the old view accessible until mmap atomically replaces it. In particular, + // do not publish a temporary reserved/PROT_NONE range between two direct views. + if (!map_shared_fixed(in_addr)) { + restore_mappings(replaced_mappings.size(), true); + return KERNEL_ERROR_ENOMEM; + } out_addr = in_addr; shared_backing = true; + } else +#endif + { + if (g_virtual_ranges->QuerySpan(in_addr, len, &reserved_ranges) && + std::all_of(reserved_ranges.begin(), reserved_ranges.end(), [](const auto& range) { + return range.type == VirtualRangeType::Reserved; + })) { + UnmapGpuRange(in_addr, len); + reserved_target = true; + } + if (!reserved_target && ReplaceFixedRangeWithReserved(in_addr, len)) { + reserved_target = true; + } + if (!reserved_target) { + return KERNEL_ERROR_ENOMEM; + } + if (map_shared_fixed(in_addr)) { + out_addr = in_addr; + shared_backing = true; + } } } else { constexpr size_t DEFAULT_ALIGNMENT = 0x4000; @@ -3105,15 +3168,22 @@ int KYTY_SYSV_ABI KernelMapDirectMemory(void** addr, size_t len, int prot, int f if (!g_physical_memory->Map(out_addr, direct_memory_start, len, prot, mode, gpu_mode)) { LOGF_COLOR(Log::Color::Red, "\t [Fail]\n"); - EXIT_IF(!g_guest_address_space->UnmapBacking(out_addr, len)); + if (replaced_mappings.empty()) { + EXIT_IF(!g_guest_address_space->UnmapBacking(out_addr, len)); + } else { + restore_mappings(replaced_mappings.size(), true); + } return KERNEL_ERROR_EBUSY; } PhysicalMemory::AllocatedBlock mapped_block {}; g_physical_memory->Find(direct_memory_start, false, &mapped_block); - const bool published = reserved_target + const bool published = !replaced_mappings.empty() || reserved_target ? g_virtual_ranges->ReplaceSpan( - out_addr, len, VirtualRangeType::Reserved, direct_memory_start, + out_addr, len, replaced_mappings.empty() + ? VirtualRangeType::Reserved + : VirtualRangeType::Direct, + direct_memory_start, prot, mapped_block.memory_type, VirtualRangeType::Direct, "") : g_virtual_ranges->Add(out_addr, len, direct_memory_start, prot, mapped_block.memory_type, @@ -3121,7 +3191,11 @@ int KYTY_SYSV_ABI KernelMapDirectMemory(void** addr, size_t len, int prot, int f if (!published) { GpuAccessMode rollback_gpu_mode = GpuAccessMode::NoAccess; EXIT_IF(!g_physical_memory->Unmap(out_addr, len, &rollback_gpu_mode)); - EXIT_IF(!g_guest_address_space->UnmapBacking(out_addr, len)); + if (replaced_mappings.empty()) { + EXIT_IF(!g_guest_address_space->UnmapBacking(out_addr, len)); + } else { + restore_mappings(replaced_mappings.size(), true); + } return KERNEL_ERROR_EBUSY; } @@ -3504,6 +3578,10 @@ int KYTY_SYSV_ABI KernelReserveVirtualRange(void** addr, size_t len, int flags, } #if defined(KYTY_VIRTUAL_MEMORY_ALLOCATION_TESTS) +void TestBeforeNextBackingMap(callback_func_t callback) { + g_test_before_backing_map = callback; +} + void TestFailNextPhysicalMemoryUnmap() { TestFailPhysicalMemoryUnmapAfter(0); } diff --git a/src/kernel/memory.h b/src/kernel/memory.h index a41fd14cc..193a7da4c 100644 --- a/src/kernel/memory.h +++ b/src/kernel/memory.h @@ -191,6 +191,7 @@ bool ProtectGuestHostMemory(uint64_t vaddr, uint64_t size, Common::VirtualMemory bool FreeGuestMemory(uint64_t vaddr, uint64_t size); #if defined(KYTY_VIRTUAL_MEMORY_ALLOCATION_TESTS) +void TestBeforeNextBackingMap(callback_func_t callback); void TestFailNextPhysicalMemoryUnmap(); void TestFailPhysicalMemoryUnmapAfter(uint32_t successful_unmaps); void TestFailGuestBackingStoreUnmapAfter(uint32_t successful_unmaps); diff --git a/src/kernel/memoryAddressSpace.inc b/src/kernel/memoryAddressSpace.inc index d461c7080..40f4e5385 100644 --- a/src/kernel/memoryAddressSpace.inc +++ b/src/kernel/memoryAddressSpace.inc @@ -231,9 +231,8 @@ public: return true; } - bool MapExistingPlaceholderFixed(uint64_t vaddr, uint64_t size, uint64_t backing_offset, - VirtualMemory::Mode mode, - FailureReason* failure_reason = nullptr) { + bool MapFixed(uint64_t vaddr, uint64_t size, uint64_t backing_offset, + VirtualMemory::Mode mode, FailureReason* failure_reason = nullptr) { if (!IsAvailable() || !RangeOk(backing_offset, size)) { SetFailure(failure_reason, !IsAvailable() ? FailureReason::BackingUnavailable : FailureReason::PhysicalRangeOutOfBounds); @@ -257,14 +256,13 @@ public: SetFailure(failure_reason, FailureReason::None); return true; #else + std::lock_guard lock(m_mutex); void* mapped = MapView(vaddr, size, backing_offset, mode, failure_reason); if (mapped == nullptr || reinterpret_cast(mapped) != vaddr) { return false; } - { - std::lock_guard lock(m_mutex); - EXIT_IF(!m_maps.emplace(vaddr, MapEntry {vaddr, size, backing_offset, mode}).second); - } + RemoveMappingsUnlocked(vaddr, size); + EXIT_IF(!m_maps.emplace(vaddr, MapEntry {vaddr, size, backing_offset, mode}).second); SetFailure(failure_reason, FailureReason::None); return true; #endif @@ -302,8 +300,7 @@ public: bool part_placeholder_preserved = false; if (!UnmapOne(entry.vaddr, entry.size, &part_placeholder_preserved)) { for (auto it = removed.rbegin(); it != removed.rend(); ++it) { - EXIT_IF(!MapExistingPlaceholderFixed(it->vaddr, it->size, it->backing_offset, - it->mode)); + EXIT_IF(!MapFixed(it->vaddr, it->size, it->backing_offset, it->mode)); } if (placeholder_preserved != nullptr) { *placeholder_preserved = false; @@ -320,8 +317,7 @@ public: // views into one placeholder before publishing it there. if (removed.size() > 1 && !CoalescePlaceholders(vaddr, size)) { for (auto it = removed.rbegin(); it != removed.rend(); ++it) { - EXIT_IF(!MapExistingPlaceholderFixed(it->vaddr, it->size, it->backing_offset, - it->mode)); + EXIT_IF(!MapFixed(it->vaddr, it->size, it->backing_offset, it->mode)); } return false; } @@ -363,12 +359,10 @@ public: return false; } +#if KYTY_PLATFORM == KYTY_PLATFORM_WINDOWS const uint64_t left_size = vaddr - old.vaddr; const uint64_t right_addr = vaddr + size; - const uint64_t old_end = old.vaddr + old.size; - const uint64_t right_size = old_end - right_addr; - -#if KYTY_PLATFORM == KYTY_PLATFORM_WINDOWS + const uint64_t right_size = old.vaddr + old.size - right_addr; const bool exact_unmap = (vaddr == old.vaddr && size == old.size); if (exact_unmap) { const bool unmapped = @@ -393,14 +387,11 @@ public: ok = SplitPlaceholder(vaddr, size) && ok; } if (left_size != 0) { - ok = MapExistingPlaceholderFixed(old.vaddr, left_size, old.backing_offset, - old.mode) && - ok; + ok = MapFixed(old.vaddr, left_size, old.backing_offset, old.mode) && ok; } if (right_size != 0) { - ok = MapExistingPlaceholderFixed(right_addr, right_size, - old.backing_offset + (right_addr - old.vaddr), - old.mode) && + ok = MapFixed(right_addr, right_size, + old.backing_offset + (right_addr - old.vaddr), old.mode) && ok; } if (ok) { @@ -418,8 +409,7 @@ public: } EXIT_IF(VirtualFree(reinterpret_cast(old.vaddr), old.size, MEM_RELEASE | MEM_COALESCE_PLACEHOLDERS) == 0); - EXIT_IF( - !MapExistingPlaceholderFixed(old.vaddr, old.size, old.backing_offset, old.mode)); + EXIT_IF(!MapFixed(old.vaddr, old.size, old.backing_offset, old.mode)); return false; } std::lock_guard lock(m_mutex); @@ -433,16 +423,7 @@ public: } std::lock_guard lock(m_mutex); - m_maps.erase(old.vaddr); - if (left_size != 0) { - m_maps.emplace(old.vaddr, - MapEntry {old.vaddr, left_size, old.backing_offset, old.mode}); - } - if (right_size != 0) { - m_maps.emplace(right_addr, - MapEntry {right_addr, right_size, - old.backing_offset + right_addr - old.vaddr, old.mode}); - } + RemoveMappingsUnlocked(vaddr, size); if (placeholder_preserved != nullptr) { *placeholder_preserved = true; @@ -516,6 +497,33 @@ private: VirtualMemory::Mode mode = VirtualMemory::Mode::NoAccess; }; +#if KYTY_PLATFORM == KYTY_PLATFORM_LINUX + void RemoveMappingsUnlocked(uint64_t vaddr, uint64_t size) { + const auto end = vaddr + size; + auto it = m_maps.upper_bound(vaddr); + if (it != m_maps.begin()) { + --it; + } + while (it != m_maps.end() && it->first < end) { + const auto old = it->second; + const auto old_end = old.vaddr + old.size; + if (old_end <= vaddr) { + ++it; + continue; + } + it = m_maps.erase(it); + if (old.vaddr < vaddr) { + m_maps.emplace(old.vaddr, + MapEntry {old.vaddr, vaddr - old.vaddr, old.backing_offset, old.mode}); + } + if (end < old_end) { + m_maps.emplace(end, MapEntry {end, old_end - end, + old.backing_offset + end - old.vaddr, old.mode}); + } + } + } +#endif + bool FindContainingUnlocked(uint64_t vaddr, uint64_t size, MapEntry* out) const { EXIT_IF(out == nullptr); @@ -621,6 +629,12 @@ private: void* MapView(uint64_t vaddr, uint64_t size, uint64_t backing_offset, VirtualMemory::Mode mode, FailureReason* failure_reason) { +#if defined(KYTY_VIRTUAL_MEMORY_ALLOCATION_TESTS) + if (const auto callback = g_test_before_backing_map) { + g_test_before_backing_map = nullptr; + callback(vaddr, size); + } +#endif void* ptr = mmap(reinterpret_cast(vaddr), size, GetProtect(mode), MAP_SHARED | MAP_FIXED, m_fd, static_cast(backing_offset)); if (ptr == MAP_FAILED) { @@ -778,7 +792,12 @@ public: VirtualMemory::Mode mode, GuestBackingStore::FailureReason* failure_reason = nullptr) { std::lock_guard lock(m_mutex); - if (!ConsumeUnlocked(vaddr, size)) { +#if KYTY_PLATFORM == KYTY_PLATFORM_LINUX + const bool replacing = ContainsMappedUnlocked(vaddr, size, MappingType::Backing); +#else + constexpr bool replacing = false; +#endif + if (!replacing && !ConsumeUnlocked(vaddr, size)) { if (failure_reason != nullptr) { *failure_reason = GuestBackingStore::FailureReason::AddressRangeUnavailable; } @@ -787,12 +806,15 @@ public: const auto host_mode = VirtualMemory::IsExecute(mode) ? VirtualMemory::Mode::ExecuteReadWrite : VirtualMemory::Mode::ReadWrite; - if (m_backing->MapExistingPlaceholderFixed(vaddr, size, backing_offset, host_mode, - failure_reason)) { - EXIT_IF(!AddMappedUnlocked(vaddr, size, MappingType::Backing)); + if (m_backing->MapFixed(vaddr, size, backing_offset, host_mode, failure_reason)) { + if (!replacing) { + EXIT_IF(!AddMappedUnlocked(vaddr, size, MappingType::Backing)); + } return true; } - AddFreeUnlocked(vaddr, size); + if (!replacing) { + AddFreeUnlocked(vaddr, size); + } return false; } diff --git a/tests/VirtualMemoryAllocationTests.cpp b/tests/VirtualMemoryAllocationTests.cpp index e3c69fe18..14c60a9c9 100644 --- a/tests/VirtualMemoryAllocationTests.cpp +++ b/tests/VirtualMemoryAllocationTests.cpp @@ -1967,6 +1967,122 @@ void TestDirectMapUnmapReusesHostAddress() { std::printf("[host] %-48s ok\n", test); } +#if defined(__linux__) +void TestFixedDirectReplacementPreservesAccess() { + using namespace Libs::LibKernel::Memory; + using Common::VirtualMemory::Mode; + const char* test = "FixedDirectReplacementPreservesAccess"; + constexpr uint64_t page = SceKernelPageSize; + int64_t physical = 0; + CheckOk(test, KernelAllocateDirectMemory(0, KernelGetDirectMemorySize(), page * 4, page, + SceKernelMtypeC, &physical), + "KernelAllocateDirectMemory"); + void* alias = nullptr; + void* address = nullptr; + CheckOk(test, KernelMapNamedDirectMemory(&alias, page * 4, SceKernelProtCpuRw, 0, physical, + page, "replacement_alias"), + "KernelMapNamedDirectMemory(alias)"); + CheckOk(test, KernelMapNamedDirectMemory(&address, page * 3, SceKernelProtCpuRw, 0, physical, + page, "replacement_original"), + "KernelMapNamedDirectMemory(original)"); + const auto base = reinterpret_cast(address); + const auto alias_base = reinterpret_cast(alias); + for (uint64_t i = 0; i < 4; ++i) { + *reinterpret_cast(alias_base + i * page) = (i + 1) * 0x11; + } + const auto replace = [&](uint64_t offset, uint64_t size, int protection, const char* name) { + void* middle = reinterpret_cast(base + page); + const int result = KernelMapNamedDirectMemory(&middle, size, protection, SceKernelMapFixed, + physical + offset, page, name); + if (result == OK) { + Check(test, reinterpret_cast(middle) == base + page, + "fixed replacement moved the destination"); + } + return result; + }; + static unsigned observations = 0; + observations = 0; + const auto observe_old_mapping = [](uintptr_t start, size_t size) { + ++observations; + uint64_t value = 0; + ssize_t bytes = -1; + std::thread reader([&] { + iovec local {&value, sizeof(value)}; + iovec remote {reinterpret_cast(start), sizeof(value)}; + bytes = process_vm_readv(getpid(), &local, 1, &remote, 1, 0); + }); + reader.join(); + Check("FixedDirectReplacementPreservesAccess", + size == SceKernelPageSize && bytes == sizeof(value) && value == 0x22, + "fixed replacement exposed an inaccessible or changed old mapping before mmap"); + }; + Check(test, ProtectGuestHostMemory(base, page, Mode::Read) && + ProtectGuestHostMemory(base + page * 2, page, Mode::NoAccess), + "could not protect neighboring pages"); + TestBeforeNextBackingMap(observe_old_mapping); + CheckOk(test, replace(page, page, SceKernelProtCpuRead, "replacement_same"), + "KernelMapNamedDirectMemory(same backing)"); + ExpectRange(test, Query(test, base + page), base + page, base + page * 2, + SceKernelProtCpuRead, 0, 1, 0, 1, "replacement_same", physical + page); + TestBeforeNextBackingMap(observe_old_mapping); + CheckOk(test, replace(page * 3, page, SceKernelProtCpuRw, "replacement_changed"), + "KernelMapNamedDirectMemory(different backing)"); + Check(test, observations == 2, "pre-map observations did not execute"); + Check(test, *reinterpret_cast(base + page) == 0x44, + "replacement did not select the new physical offset"); + *reinterpret_cast(base + page) = 0x45; + Check(test, *reinterpret_cast(alias_base + page * 3) == 0x45 && + *reinterpret_cast(alias_base + page) == 0x22, + "replacement lost aliasing or changed the previous backing"); + uint64_t value = 0; + iovec local {&value, sizeof(value)}; + iovec left {reinterpret_cast(base), sizeof(value)}; + iovec right {reinterpret_cast(base + page * 2), sizeof(value)}; + Check(test, process_vm_readv(getpid(), &local, 1, &left, 1, 0) == sizeof(value) && + value == 0x11 && process_vm_writev(getpid(), &local, 1, &left, 1, 0) == -1 && + process_vm_readv(getpid(), &local, 1, &right, 1, 0) == -1, + "middle replacement changed a neighboring page's bytes or host protection"); + Check(test, ProtectGuestHostMemory(base, page * 3, Mode::ReadWrite), + "could not restore neighboring permissions"); + const auto check_preserved = [&] { + Check(test, *reinterpret_cast(base) == 0x11 && + *reinterpret_cast(base + page) == 0x45 && + *reinterpret_cast(base + page * 2) == 0x33, + "failed replacement changed bytes"); + for (uint64_t i = 0; i < 3; ++i) { + Check(test, !TestPlaceholderRangeIsFree(base + i * page, page), + "failed replacement released part of the owned mapping"); + } + ExpectRange(test, Query(test, base), base, base + page, SceKernelProtCpuRw, + 0, 1, 0, 1, "replacement_original", physical); + ExpectRange(test, Query(test, base + page), base + page, base + page * 2, + SceKernelProtCpuRw, 0, 1, 0, 1, "replacement_changed", physical + page * 3); + ExpectRange(test, Query(test, base + page * 2), base + page * 2, base + page * 3, + SceKernelProtCpuRw, 0, 1, 0, 1, "replacement_original", physical + page * 2); + }; + TestFailPhysicalMemoryUnmapAfter(1); + CheckFailed(test, replace(page, page * 2, SceKernelProtCpuRead, "replacement_span"), + "KernelMapNamedDirectMemory(second-view removal failure)"); + check_preserved(); + TestFailNextVirtualRangeReplacement(); + CheckFailed(test, replace(page, page * 2, SceKernelProtCpuRead, "replacement_span"), + "KernelMapNamedDirectMemory(publication failure)"); + check_preserved(); + CheckOk(test, replace(page, page * 2, SceKernelProtCpuRead, "replacement_span"), + "KernelMapNamedDirectMemory(multiple backing views)"); + Check(test, *reinterpret_cast(base) == 0x11 && + *reinterpret_cast(base + page) == 0x22 && + *reinterpret_cast(base + page * 2) == 0x33, + "multi-view replacement changed the prefix or chose incorrect backing offsets"); + ExpectRange(test, Query(test, base + page), base + page, base + page * 3, + SceKernelProtCpuRead, 0, 1, 0, 1, "replacement_span", physical + page); + CheckOk(test, KernelMunmap(base, page * 3), "KernelMunmap(replacement)"); + CheckOk(test, KernelMunmap(alias_base, page * 4), "KernelMunmap(alias)"); + CheckOk(test, KernelReleaseDirectMemory(physical, page * 4), "KernelReleaseDirectMemory"); + std::printf("[host] %-48s ok\n", test); +} +#endif + void TestFixedReserveReplacesPartialDirectMapping() { const char* test = "FixedReserveReplacesPartialDirectMapping"; constexpr uint64_t page_count = 13; @@ -4062,6 +4178,12 @@ void TestSmallFiberStacksAndMigration() { int main(int argc, char** argv) { InitSubsystems(); +#if defined(__linux__) + if (argc == 2 && std::strcmp(argv[1], "--fixed-direct-replacement-only") == 0) { + RunTest(TestFixedDirectReplacementPreservesAccess); + return g_failed_tests == 0 ? 0 : 1; + } +#endif #if defined(__x86_64__) || defined(_M_X64) if (argc == 2 && std::strcmp(argv[1], "--fiber-only") == 0) { RunTest(TestSmallFiberStacksAndMigration); @@ -4139,6 +4261,9 @@ int main(int argc, char** argv) { RunTest(TestHintlessDirectMapUsesCanonicalGuestBase); RunTest(TestDirectMemoryContentPersistsAcrossRemap); RunTest(TestDirectMapUnmapReusesHostAddress); +#if defined(__linux__) + RunTest(TestFixedDirectReplacementPreservesAccess); +#endif RunTest(TestFixedReserveReplacesPartialDirectMapping); RunTest(TestFixedReserveRollbackConsumesRestoredPlaceholder); RunTest(TestFixedReserveRollbackSkipsUntouchedChunks);