raphiaura/KytyPS5
1
0
Fork 0
mirror of https://github.com/KytyPS5/KytyPS5.git synced 2026-10-21 19:36:55 +02:00
Code Issues Packages Projects Releases Wiki Activity

memory: replace live direct mappings without exposing an inaccessible gap

Browse Source
This commit is contained in:
nmzik
2026-10-04 08:16:28 +04:00
parent 1f6be3d570
commit 46900d8042
4 changed files with 279 additions and 53 deletions
Download Patch File Download Diff File Expand all files Collapse all files
src/kernel/memory.cpp
+93 -15
View File
@@ -166,7 +166,8 @@ static bool VirtualRangesOverlap(uint64_t left_start, uint64_t left_size, uint64
}
#if defined(KYTY_VIRTUAL_MEMORY_ALLOCATION_TESTS)
static uint32_t g_test_backing_store_unmaps_before_failure = UINT32_MAX;
static uint32_t g_test_backing_store_unmaps_before_failure = UINT32_MAX;
static callback_func_t g_test_before_backing_map = nullptr;
#endif
#include "memoryAddressSpace.inc"
@@ -3024,6 +3025,22 @@ int KYTY_SYSV_ABI KernelMapDirectMemory(void** addr, size_t len, int prot, int f
return g_guest_address_space->MapBacking(target_addr, len, direct_memory_start, mode,
&shared_failure);
};
std::vector<PhysicalMemory::AllocatedBlock> replaced_mappings;
auto restore_mappings = [&](size_t count, bool restore_views) {
for (size_t index = 0; index < count; ++index) {
const auto& old = replaced_mappings[index];
if (restore_views) {
EXIT_IF(!g_guest_address_space->MapBacking(old.map_vaddr, old.map_size,
old.start_addr, old.mode));
}
EXIT_IF(!g_physical_memory->Map(old.map_vaddr, old.start_addr, old.map_size,
old.prot, old.mode, old.gpu_mode));
g_physical_memory->SetVirtualRangeName(old.map_vaddr, old.map_size, old.name);
g_physical_memory->SetVirtualRangeMemoryType(old.map_vaddr, old.map_size,
old.memory_type);
}
MapGpuRange(in_addr, len);
};
if (fixed) {
if (in_addr == 0 || (in_addr & (PAGE_SIZE - 1u)) != 0 ||
(alignment != 0 && in_addr % alignment != 0)) {
@@ -3034,22 +3051,68 @@ int KYTY_SYSV_ABI KernelMapDirectMemory(void** addr, size_t len, int prot, int f
}
std::vector<VirtualRanges::Range> reserved_ranges;
#if KYTY_PLATFORM == KYTY_PLATFORM_LINUX
if (g_virtual_ranges->QuerySpan(in_addr, len, &reserved_ranges) &&
std::all_of(reserved_ranges.begin(), reserved_ranges.end(), [](const auto& range) {
return range.type == VirtualRangeType::Reserved;
return range.type == VirtualRangeType::Direct;
})) {
uint64_t covered = 0;
{
Common::LockGuard lock(g_physical_memory->GetMutex());
for (const auto& mapping: g_physical_memory->GetMappings()) {
const auto start = std::max(in_addr, mapping.map_vaddr);
const auto end = std::min(in_addr + len, mapping.map_vaddr + mapping.map_size);
if (start >= end) {
continue;
}
auto old = mapping;
old.start_addr += start - old.map_vaddr;
old.map_vaddr = start;
old.size = old.map_size = end - start;
covered += old.map_size;
replaced_mappings.push_back(old);
}
}
if (covered != len) {
return KERNEL_ERROR_EBUSY;
}
UnmapGpuRange(in_addr, len);
reserved_target = true;
}
if (!reserved_target && ReplaceFixedRangeWithReserved(in_addr, len)) {
reserved_target = true;
}
if (!reserved_target) {
return KERNEL_ERROR_ENOMEM;
}
if (map_shared_fixed(in_addr)) {
for (size_t index = 0; index < replaced_mappings.size(); ++index) {
const auto& old = replaced_mappings[index];
GpuAccessMode old_gpu_mode = GpuAccessMode::NoAccess;
if (!g_physical_memory->Unmap(old.map_vaddr, old.map_size, &old_gpu_mode)) {
restore_mappings(index, false);
return KERNEL_ERROR_EBUSY;
}
}
// Keep the old view accessible until mmap atomically replaces it. In particular,
// do not publish a temporary reserved/PROT_NONE range between two direct views.
if (!map_shared_fixed(in_addr)) {
restore_mappings(replaced_mappings.size(), true);
return KERNEL_ERROR_ENOMEM;
}
out_addr = in_addr;
shared_backing = true;
} else
#endif
{
if (g_virtual_ranges->QuerySpan(in_addr, len, &reserved_ranges) &&
std::all_of(reserved_ranges.begin(), reserved_ranges.end(), [](const auto& range) {
return range.type == VirtualRangeType::Reserved;
})) {
UnmapGpuRange(in_addr, len);
reserved_target = true;
}
if (!reserved_target && ReplaceFixedRangeWithReserved(in_addr, len)) {
reserved_target = true;
}
if (!reserved_target) {
return KERNEL_ERROR_ENOMEM;
}
if (map_shared_fixed(in_addr)) {
out_addr = in_addr;
shared_backing = true;
}
}
} else {
constexpr size_t DEFAULT_ALIGNMENT = 0x4000;
@@ -3105,15 +3168,22 @@ int KYTY_SYSV_ABI KernelMapDirectMemory(void** addr, size_t len, int prot, int f
if (!g_physical_memory->Map(out_addr, direct_memory_start, len, prot, mode, gpu_mode)) {
LOGF_COLOR(Log::Color::Red, "\t [Fail]\n");
EXIT_IF(!g_guest_address_space->UnmapBacking(out_addr, len));
if (replaced_mappings.empty()) {
EXIT_IF(!g_guest_address_space->UnmapBacking(out_addr, len));
} else {
restore_mappings(replaced_mappings.size(), true);
}
return KERNEL_ERROR_EBUSY;
}
PhysicalMemory::AllocatedBlock mapped_block {};
g_physical_memory->Find(direct_memory_start, false, &mapped_block);
const bool published = reserved_target
const bool published = !replaced_mappings.empty() || reserved_target
? g_virtual_ranges->ReplaceSpan(
out_addr, len, VirtualRangeType::Reserved, direct_memory_start,
out_addr, len, replaced_mappings.empty()
? VirtualRangeType::Reserved
: VirtualRangeType::Direct,
direct_memory_start,
prot, mapped_block.memory_type, VirtualRangeType::Direct, "")
: g_virtual_ranges->Add(out_addr, len, direct_memory_start, prot,
mapped_block.memory_type,
@@ -3121,7 +3191,11 @@ int KYTY_SYSV_ABI KernelMapDirectMemory(void** addr, size_t len, int prot, int f
if (!published) {
GpuAccessMode rollback_gpu_mode = GpuAccessMode::NoAccess;
EXIT_IF(!g_physical_memory->Unmap(out_addr, len, &rollback_gpu_mode));
EXIT_IF(!g_guest_address_space->UnmapBacking(out_addr, len));
if (replaced_mappings.empty()) {
EXIT_IF(!g_guest_address_space->UnmapBacking(out_addr, len));
} else {
restore_mappings(replaced_mappings.size(), true);
}
return KERNEL_ERROR_EBUSY;
}
@@ -3504,6 +3578,10 @@ int KYTY_SYSV_ABI KernelReserveVirtualRange(void** addr, size_t len, int flags,
}
#if defined(KYTY_VIRTUAL_MEMORY_ALLOCATION_TESTS)
void TestBeforeNextBackingMap(callback_func_t callback) {
g_test_before_backing_map = callback;
}
void TestFailNextPhysicalMemoryUnmap() {
TestFailPhysicalMemoryUnmapAfter(0);
}
src/kernel/memory.h
+1
View File
@@ -191,6 +191,7 @@ bool ProtectGuestHostMemory(uint64_t vaddr, uint64_t size, Common::VirtualMemory
bool FreeGuestMemory(uint64_t vaddr, uint64_t size);
#if defined(KYTY_VIRTUAL_MEMORY_ALLOCATION_TESTS)
void TestBeforeNextBackingMap(callback_func_t callback);
void TestFailNextPhysicalMemoryUnmap();
void TestFailPhysicalMemoryUnmapAfter(uint32_t successful_unmaps);
void TestFailGuestBackingStoreUnmapAfter(uint32_t successful_unmaps);
src/kernel/memoryAddressSpace.inc
+60 -38
View File
@@ -231,9 +231,8 @@ public:
return true;
}
bool MapExistingPlaceholderFixed(uint64_t vaddr, uint64_t size, uint64_t backing_offset,
VirtualMemory::Mode mode,
FailureReason* failure_reason = nullptr) {
bool MapFixed(uint64_t vaddr, uint64_t size, uint64_t backing_offset,
VirtualMemory::Mode mode, FailureReason* failure_reason = nullptr) {
if (!IsAvailable() || !RangeOk(backing_offset, size)) {
SetFailure(failure_reason, !IsAvailable() ? FailureReason::BackingUnavailable
: FailureReason::PhysicalRangeOutOfBounds);
@@ -257,14 +256,13 @@ public:
SetFailure(failure_reason, FailureReason::None);
return true;
#else
std::lock_guard lock(m_mutex);
void* mapped = MapView(vaddr, size, backing_offset, mode, failure_reason);
if (mapped == nullptr || reinterpret_cast<uint64_t>(mapped) != vaddr) {
return false;
}
{
std::lock_guard lock(m_mutex);
EXIT_IF(!m_maps.emplace(vaddr, MapEntry {vaddr, size, backing_offset, mode}).second);
}
RemoveMappingsUnlocked(vaddr, size);
EXIT_IF(!m_maps.emplace(vaddr, MapEntry {vaddr, size, backing_offset, mode}).second);
SetFailure(failure_reason, FailureReason::None);
return true;
#endif
@@ -302,8 +300,7 @@ public:
bool part_placeholder_preserved = false;
if (!UnmapOne(entry.vaddr, entry.size, &part_placeholder_preserved)) {
for (auto it = removed.rbegin(); it != removed.rend(); ++it) {
EXIT_IF(!MapExistingPlaceholderFixed(it->vaddr, it->size, it->backing_offset,
it->mode));
EXIT_IF(!MapFixed(it->vaddr, it->size, it->backing_offset, it->mode));
}
if (placeholder_preserved != nullptr) {
*placeholder_preserved = false;
@@ -320,8 +317,7 @@ public:
// views into one placeholder before publishing it there.
if (removed.size() > 1 && !CoalescePlaceholders(vaddr, size)) {
for (auto it = removed.rbegin(); it != removed.rend(); ++it) {
EXIT_IF(!MapExistingPlaceholderFixed(it->vaddr, it->size, it->backing_offset,
it->mode));
EXIT_IF(!MapFixed(it->vaddr, it->size, it->backing_offset, it->mode));
}
return false;
}
@@ -363,12 +359,10 @@ public:
return false;
}
#if KYTY_PLATFORM == KYTY_PLATFORM_WINDOWS
const uint64_t left_size = vaddr - old.vaddr;
const uint64_t right_addr = vaddr + size;
const uint64_t old_end = old.vaddr + old.size;
const uint64_t right_size = old_end - right_addr;
#if KYTY_PLATFORM == KYTY_PLATFORM_WINDOWS
const uint64_t right_size = old.vaddr + old.size - right_addr;
const bool exact_unmap = (vaddr == old.vaddr && size == old.size);
if (exact_unmap) {
const bool unmapped =
@@ -393,14 +387,11 @@ public:
ok = SplitPlaceholder(vaddr, size) && ok;
}
if (left_size != 0) {
ok = MapExistingPlaceholderFixed(old.vaddr, left_size, old.backing_offset,
old.mode) &&
ok;
ok = MapFixed(old.vaddr, left_size, old.backing_offset, old.mode) && ok;
}
if (right_size != 0) {
ok = MapExistingPlaceholderFixed(right_addr, right_size,
old.backing_offset + (right_addr - old.vaddr),
old.mode) &&
ok = MapFixed(right_addr, right_size,
old.backing_offset + (right_addr - old.vaddr), old.mode) &&
ok;
}
if (ok) {
@@ -418,8 +409,7 @@ public:
}
EXIT_IF(VirtualFree(reinterpret_cast<void*>(old.vaddr), old.size,
MEM_RELEASE | MEM_COALESCE_PLACEHOLDERS) == 0);
EXIT_IF(
!MapExistingPlaceholderFixed(old.vaddr, old.size, old.backing_offset, old.mode));
EXIT_IF(!MapFixed(old.vaddr, old.size, old.backing_offset, old.mode));
return false;
}
std::lock_guard lock(m_mutex);
@@ -433,16 +423,7 @@ public:
}
std::lock_guard lock(m_mutex);
m_maps.erase(old.vaddr);
if (left_size != 0) {
m_maps.emplace(old.vaddr,
MapEntry {old.vaddr, left_size, old.backing_offset, old.mode});
}
if (right_size != 0) {
m_maps.emplace(right_addr,
MapEntry {right_addr, right_size,
old.backing_offset + right_addr - old.vaddr, old.mode});
}
RemoveMappingsUnlocked(vaddr, size);
if (placeholder_preserved != nullptr) {
*placeholder_preserved = true;
@@ -516,6 +497,33 @@ private:
VirtualMemory::Mode mode = VirtualMemory::Mode::NoAccess;
};
#if KYTY_PLATFORM == KYTY_PLATFORM_LINUX
void RemoveMappingsUnlocked(uint64_t vaddr, uint64_t size) {
const auto end = vaddr + size;
auto it = m_maps.upper_bound(vaddr);
if (it != m_maps.begin()) {
--it;
}
while (it != m_maps.end() && it->first < end) {
const auto old = it->second;
const auto old_end = old.vaddr + old.size;
if (old_end <= vaddr) {
++it;
continue;
}
it = m_maps.erase(it);
if (old.vaddr < vaddr) {
m_maps.emplace(old.vaddr,
MapEntry {old.vaddr, vaddr - old.vaddr, old.backing_offset, old.mode});
}
if (end < old_end) {
m_maps.emplace(end, MapEntry {end, old_end - end,
old.backing_offset + end - old.vaddr, old.mode});
}
}
}
#endif
bool FindContainingUnlocked(uint64_t vaddr, uint64_t size, MapEntry* out) const {
EXIT_IF(out == nullptr);
@@ -621,6 +629,12 @@ private:
void* MapView(uint64_t vaddr, uint64_t size, uint64_t backing_offset, VirtualMemory::Mode mode,
FailureReason* failure_reason) {
#if defined(KYTY_VIRTUAL_MEMORY_ALLOCATION_TESTS)
if (const auto callback = g_test_before_backing_map) {
g_test_before_backing_map = nullptr;
callback(vaddr, size);
}
#endif
void* ptr = mmap(reinterpret_cast<void*>(vaddr), size, GetProtect(mode),
MAP_SHARED | MAP_FIXED, m_fd, static_cast<off_t>(backing_offset));
if (ptr == MAP_FAILED) {
@@ -778,7 +792,12 @@ public:
VirtualMemory::Mode mode,
GuestBackingStore::FailureReason* failure_reason = nullptr) {
std::lock_guard lock(m_mutex);
if (!ConsumeUnlocked(vaddr, size)) {
#if KYTY_PLATFORM == KYTY_PLATFORM_LINUX
const bool replacing = ContainsMappedUnlocked(vaddr, size, MappingType::Backing);
#else
constexpr bool replacing = false;
#endif
if (!replacing && !ConsumeUnlocked(vaddr, size)) {
if (failure_reason != nullptr) {
*failure_reason = GuestBackingStore::FailureReason::AddressRangeUnavailable;
}
@@ -787,12 +806,15 @@ public:
const auto host_mode = VirtualMemory::IsExecute(mode)
? VirtualMemory::Mode::ExecuteReadWrite
: VirtualMemory::Mode::ReadWrite;
if (m_backing->MapExistingPlaceholderFixed(vaddr, size, backing_offset, host_mode,
failure_reason)) {
EXIT_IF(!AddMappedUnlocked(vaddr, size, MappingType::Backing));
if (m_backing->MapFixed(vaddr, size, backing_offset, host_mode, failure_reason)) {
if (!replacing) {
EXIT_IF(!AddMappedUnlocked(vaddr, size, MappingType::Backing));
}
return true;
}
AddFreeUnlocked(vaddr, size);
if (!replacing) {
AddFreeUnlocked(vaddr, size);
}
return false;
}