raphiaura/KytyPS5
1
0
Fork 0
mirror of https://github.com/KytyPS5/KytyPS5.git synced 2026-10-21 11:26:56 +02:00
Code Issues Packages Projects Releases Wiki Activity

Rework guest memory tracking/virtual address space/direct and flexible memory (#135)

Browse Source 
* Rework guest memory tracking

* add unknwon flag

* Fix macOS guest address-space reservation
This commit is contained in:
nmzik
2026-07-31 03:07:17 +02:00
committed by GitHub
parent ecb48f90bb
commit 2f5396c6a5
28 changed files with 3155 additions and 2887 deletions
Download Patch File Download Diff File Expand all files Collapse all files
src/loader/runtimeLinker.cpp
+216 -52
View File
@@ -62,14 +62,16 @@ static void FreeTlsBlock(ThreadLocalStorage::Block* block) {
if (block->free_func != nullptr) {
block->free_func(block->ptr);
} else if (block->vm_alloc) {
Common::VirtualMemory::Free(reinterpret_cast<uint64_t>(block->ptr));
EXIT_IF(!Libs::LibKernel::Memory::FreeGuestMemory(reinterpret_cast<uint64_t>(block->ptr),
block->alloc_size));
} else {
delete[] block->ptr;
}
block->ptr = nullptr;
block->free_func = nullptr;
block->vm_alloc = false;
block->ptr = nullptr;
block->free_func = nullptr;
block->vm_alloc = false;
block->alloc_size = 0;
}
static uint64_t AlignUp(uint64_t value, uint64_t alignment) {
@@ -131,17 +133,25 @@ static std::vector<StubbedImportRecord> g_stubbed_imports;
static std::atomic_uint32_t g_unresolved_stub_call_log_count {0};
static std::vector<uint64_t> g_unresolved_stub_thunk_pages;
static uint64_t g_unresolved_stub_thunk_offset = 0;
static constexpr uint64_t UNRESOLVED_STUB_PAGE_SIZE = 4096;
static KYTY_SYSV_ABI uint64_t ResolveImportStubWithId(uint64_t record_id);
static bool PatchGuestMemory64(uint64_t vaddr, uint64_t value) {
auto* ptr = reinterpret_cast<uint64_t*>(vaddr);
bool changed = (*ptr != value);
std::memcpy(ptr, &value, sizeof(value));
return changed;
}
static uint64_t AllocateUnresolvedImportThunk(uint64_t record_id) {
constexpr uint64_t page_size = 4096;
constexpr uint64_t thunk_size = 162;
if (g_unresolved_stub_thunk_pages.empty() ||
g_unresolved_stub_thunk_offset + thunk_size > page_size) {
auto page = Common::VirtualMemory::Alloc(0, page_size,
Common::VirtualMemory::Mode::ExecuteReadWrite);
g_unresolved_stub_thunk_offset + thunk_size > UNRESOLVED_STUB_PAGE_SIZE) {
auto page = Libs::LibKernel::Memory::AllocateRuntimeMemory(
0, UNRESOLVED_STUB_PAGE_SIZE, Common::VirtualMemory::Mode::ExecuteReadWrite,
"unresolved_import_thunk");
EXIT_NOT_IMPLEMENTED(page == 0);
g_unresolved_stub_thunk_pages.push_back(page);
g_unresolved_stub_thunk_offset = 0;
@@ -298,7 +308,7 @@ static KYTY_SYSV_ABI uint64_t ResolveImportStubWithId(uint64_t record_id) {
resolved.name.c_str(), resolved.vaddr);
if (record.patch_vaddr != 0) {
*reinterpret_cast<uint64_t*>(record.patch_vaddr) = resolved.vaddr;
PatchGuestMemory64(record.patch_vaddr, resolved.vaddr);
}
return resolved.vaddr;
@@ -360,7 +370,7 @@ static KYTY_SYSV_ABI void RunEntry(uint64_t addr, EntryParams* params, atexit_fu
register uintptr_t guest_rbp_reg asm("r15") = guest_rbp;
#endif
#if defined(__APPLE__) || KYTY_PLATFORM == KYTY_PLATFORM_WINDOWS
#if defined(__APPLE__)
asm volatile(
"pushq %%r12\n\t"
"pushq %%r13\n\t"
@@ -374,16 +384,44 @@ static KYTY_SYSV_ABI void RunEntry(uint64_t addr, EntryParams* params, atexit_fu
"popq %%r13\n\t"
"popq %%r12\n\t"
:
#if defined(__APPLE__)
: [func] "r"(func_reg), "D"(params),
"S"(atexit_func), [guest_rsp] "r"(guest_rsp_reg), [guest_rbp] "r"(guest_rbp_reg)
#else
: [func] "r"(func), "D"(params),
"S"(atexit_func), [guest_rsp] "r"(guest_rsp), [guest_rbp] "r"(guest_rbp)
#endif
: "cc", "memory", "rax", "rcx", "rdx", "r8", "r9", "r10", "r11", "xmm0", "xmm1", "xmm2",
"xmm3", "xmm4", "xmm5", "xmm6", "xmm7", "xmm8", "xmm9", "xmm10", "xmm11", "xmm12",
"xmm13", "xmm14", "xmm15");
#elif KYTY_PLATFORM == KYTY_PLATFORM_WINDOWS
// Windows stack probes use the TEB stack limits during the guest stack switch.
// bounds, which describe the host stack and are invalid while RSP is in guest memory.
register entry_func_t func_reg asm("rbx") = func;
register uintptr_t guest_rsp_reg asm("r8") = guest_rsp;
register uintptr_t guest_rbp_reg asm("r9") = guest_rbp;
asm volatile("pushq %%r12\n\t"
"pushq %%r13\n\t"
"pushq %%r14\n\t"
"pushq %%r15\n\t"
"movq %%gs:0x08, %%r14\n\t"
"movq %%gs:0x10, %%r15\n\t"
"xorq %%rcx, %%rcx\n\t"
"movq %%rcx, %%gs:0x08\n\t"
"movq %%rcx, %%gs:0x10\n\t"
"movq %%rsp, %%r12\n\t"
"movq %%rbp, %%r13\n\t"
"movq %[guest_rsp], %%rsp\n\t"
"movq %[guest_rbp], %%rbp\n\t"
"callq *%[func]\n\t"
"movq %%r13, %%rbp\n\t"
"movq %%r12, %%rsp\n\t"
"movq %%r14, %%gs:0x08\n\t"
"movq %%r15, %%gs:0x10\n\t"
"popq %%r15\n\t"
"popq %%r14\n\t"
"popq %%r13\n\t"
"popq %%r12\n\t"
: [guest_rsp] "+r"(guest_rsp_reg), [guest_rbp] "+r"(guest_rbp_reg)
: [func] "r"(func_reg), "D"(params), "S"(atexit_func)
: "cc", "memory", "rax", "rcx", "rdx", "r10", "r11", "xmm0", "xmm1", "xmm2",
"xmm3", "xmm4", "xmm5", "xmm6", "xmm7", "xmm8", "xmm9", "xmm10", "xmm11",
"xmm12", "xmm13", "xmm14", "xmm15");
#else
// Clobbers prevent inputs from being allocated to r12/r13.
asm volatile("movq %%rsp, %%r12\n\t"
@@ -449,6 +487,110 @@ static KYTY_SYSV_ABI void RunEntry(uint64_t addr, EntryParams* params, atexit_fu
#endif
}
#if defined(KYTY_VIRTUAL_MEMORY_ALLOCATION_TESTS)
struct MainEntryStackTestState {
bool called = false;
uintptr_t rsp = 0;
#if KYTY_PLATFORM == KYTY_PLATFORM_WINDOWS
uintptr_t teb_stack_base = UINTPTR_MAX;
uintptr_t teb_stack_limit = UINTPTR_MAX;
#endif
};
static KYTY_SYSV_ABI void TestMainEntryStackCallback(EntryParams* params,
atexit_func_t /*atexit_func*/) {
auto* state = reinterpret_cast<MainEntryStackTestState*>(const_cast<char*>(params->argv[0]));
asm volatile("movq %%rsp, %0" : "=r"(state->rsp) : : "memory");
#if KYTY_PLATFORM == KYTY_PLATFORM_WINDOWS
asm volatile("movq %%gs:0x08, %0\n\t"
"movq %%gs:0x10, %1\n\t"
: "=r"(state->teb_stack_base), "=r"(state->teb_stack_limit)
:
: "memory");
#endif
state->called = true;
}
bool TestMainEntryUsesGuestStack() {
constexpr uint64_t stack_size = 0x10000;
const auto stack_base = Libs::LibKernel::Memory::AllocateRuntimeMemory(
0, stack_size, Common::VirtualMemory::Mode::ReadWrite, "main_entry_stack_test");
if (stack_base == 0) {
return false;
}
MainEntryStackTestState state {};
EntryParams params {};
params.argv[0] = reinterpret_cast<const char*>(&state);
#if KYTY_PLATFORM == KYTY_PLATFORM_WINDOWS
uintptr_t original_teb_stack_base = 0;
uintptr_t original_teb_stack_limit = 0;
asm volatile("movq %%gs:0x08, %0\n\t"
"movq %%gs:0x10, %1\n\t"
: "=r"(original_teb_stack_base), "=r"(original_teb_stack_limit)
:
: "memory");
#endif
RunEntry(reinterpret_cast<uint64_t>(TestMainEntryStackCallback), &params, nullptr,
reinterpret_cast<void*>(stack_base + stack_size));
#if KYTY_PLATFORM == KYTY_PLATFORM_WINDOWS
uintptr_t restored_teb_stack_base = 0;
uintptr_t restored_teb_stack_limit = 0;
asm volatile("movq %%gs:0x08, %0\n\t"
"movq %%gs:0x10, %1\n\t"
: "=r"(restored_teb_stack_base), "=r"(restored_teb_stack_limit)
:
: "memory");
const bool teb_ok = state.teb_stack_base == 0 && state.teb_stack_limit == 0 &&
restored_teb_stack_base == original_teb_stack_base &&
restored_teb_stack_limit == original_teb_stack_limit;
#else
constexpr bool teb_ok = true;
#endif
const bool rsp_ok = state.rsp >= stack_base && state.rsp < stack_base + stack_size;
const bool freed = Libs::LibKernel::Memory::FreeGuestMemory(stack_base, stack_size);
return state.called && rsp_ok && teb_ok && freed;
}
bool TestModuleRelocationUsesWritableHostMapping() {
constexpr uint64_t page_size = 0x4000;
constexpr uint64_t value = 0x4b59545950415443;
const auto base = Libs::LibKernel::Memory::AllocateProgramMemory(
0, page_size, Common::VirtualMemory::Mode::ReadWrite, "host_only_patch_test");
if (base == 0) {
return false;
}
Libs::LibKernel::Memory::SetProgramMemoryProtection(base, page_size,
Common::VirtualMemory::Mode::Read);
Libs::LibKernel::Memory::VirtualQueryInfo before {};
Libs::LibKernel::Memory::VirtualQueryInfo after {};
const bool before_ok =
Libs::LibKernel::Memory::KernelVirtualQuery(reinterpret_cast<const void*>(base), 0, &before,
sizeof(before)) == 0;
const bool changed = PatchGuestMemory64(base, value);
const bool after_ok = Libs::LibKernel::Memory::KernelVirtualQuery(
reinterpret_cast<const void*>(base), 0, &after, sizeof(after)) == 0;
const bool value_ok = *reinterpret_cast<const uint64_t*>(base) == value;
#if KYTY_PLATFORM == KYTY_PLATFORM_WINDOWS
MEMORY_BASIC_INFORMATION mbi {};
const bool host_mode_ok =
VirtualQuery(reinterpret_cast<const void*>(base), &mbi, sizeof(mbi)) != 0 &&
mbi.Protect == PAGE_READWRITE;
#else
constexpr bool host_mode_ok = true;
#endif
const bool freed = Libs::LibKernel::Memory::FreeGuestMemory(base, page_size);
return before_ok && after_ok && changed && value_ok && host_mode_ok && freed &&
before.protection == after.protection;
}
#endif
static uint64_t GetAlignedSize(const Elf64_Phdr* p) {
return (p->p_align != 0 ? (p->p_memsz + (p->p_align - 1)) & ~(p->p_align - 1) : p->p_memsz);
}
@@ -1038,7 +1180,7 @@ static void RelocateRecord(uint32_t index, Elf64_Rela* r, Program* program, bool
// KYTY_PROFILER_BLOCK("patch");
if (ri.resolved) {
patched = Common::VirtualMemory::PatchReplace(ri.vaddr, ri.value);
patched = PatchGuestMemory64(ri.vaddr, ri.value);
} else {
uint64_t value = 0;
bool weak = (ri.bind == BindType::Weak || !program->fail_if_global_not_resolved);
@@ -1056,7 +1198,7 @@ static void RelocateRecord(uint32_t index, Elf64_Rela* r, Program* program, bool
}
if (value != 0) {
patched = Common::VirtualMemory::PatchReplace(ri.vaddr, value);
patched = PatchGuestMemory64(ri.vaddr, value);
} else {
auto dbg_str = fmt::format("[{:016x}] <- {:016x}, {}, {}, {}, {}", ri.vaddr, ri.value,
ri.name.c_str(), Common::EnumName(ri.type).c_str(),
@@ -1079,7 +1221,7 @@ static void RelocateRecord(uint32_t index, Elf64_Rela* r, Program* program, bool
}
if (value != 0) {
patched = Common::VirtualMemory::PatchReplace(ri.vaddr, value);
patched = PatchGuestMemory64(ri.vaddr, value);
}
}
}
@@ -1459,6 +1601,7 @@ void RuntimeLinker::Execute(const std::filesystem::path& game_patch) {
PreloadAdjacentPrograms();
RelocateAll();
if (!game_patch.empty()) {
GamePatch::Apply(game_patch, m_programs.empty() ? nullptr : m_programs.front());
}
@@ -1489,6 +1632,21 @@ void RuntimeLinker::Clear() {
DeleteProgram(p);
}
m_programs.clear();
for (const auto page: g_unresolved_stub_thunk_pages) {
EXIT_IF(!Libs::LibKernel::Memory::FreeGuestMemory(page, UNRESOLVED_STUB_PAGE_SIZE));
}
g_unresolved_stub_thunk_pages.clear();
g_unresolved_stub_thunk_offset = 0;
g_stubbed_imports.clear();
g_unresolved_stub_call_log_count.store(0);
if (g_invalid_memory != 0) {
EXIT_IF(!Libs::LibKernel::Memory::FreeGuestMemory(g_invalid_memory, 4096));
g_invalid_memory = 0;
}
g_tls_main_program = nullptr;
g_tls_cached_main_program = nullptr;
g_tls_cached_main_tcb = nullptr;
g_desired_base_addr = SYSTEM_RESERVED + CODE_BASE_OFFSET;
m_symbols.reset();
m_relocated = false;
}
@@ -1926,10 +2084,11 @@ uint8_t* RuntimeLinker::TlsGetAddr(Program* program) {
const auto tcb_offset =
program->tls.tcb_offset != 0 ? program->tls.tcb_offset : program->tls.image_size;
const auto alloc_size = AlignUp(tcb_offset, TCB_ALIGN) + TCB_SIZE;
tls.ptr = reinterpret_cast<uint8_t*>(
Common::VirtualMemory::Alloc(0, alloc_size, Common::VirtualMemory::Mode::ReadWrite));
tls.free_func = nullptr;
tls.vm_alloc = true;
tls.ptr = reinterpret_cast<uint8_t*>(Libs::LibKernel::Memory::AllocateRuntimeMemory(
0, alloc_size, Common::VirtualMemory::Mode::ReadWrite, "thread_local_storage"));
tls.free_func = nullptr;
tls.vm_alloc = true;
tls.alloc_size = alloc_size;
EXIT_IF(tls.ptr == nullptr);
@@ -2006,8 +2165,9 @@ void RuntimeLinker::LoadProgramToMemory(Program* program) {
EXIT_IF(tls_handler_size > UINT64_MAX - program->base_size_aligned);
program->mapped_size = program->base_size_aligned + tls_handler_size;
program->base_vaddr = Common::VirtualMemory::Alloc(
g_desired_base_addr, program->mapped_size, Common::VirtualMemory::Mode::ExecuteReadWrite);
program->base_vaddr = Libs::LibKernel::Memory::AllocateProgramMemory(
g_desired_base_addr, program->mapped_size, Common::VirtualMemory::Mode::ExecuteReadWrite,
Common::PathToString(program->file_name.filename()).c_str());
if (!is_shared) {
program->tls.handler_vaddr = program->base_vaddr + program->base_size_aligned;
@@ -2017,10 +2177,6 @@ void RuntimeLinker::LoadProgramToMemory(Program* program) {
EXIT_IF(program->base_vaddr == 0);
EXIT_IF(program->base_size_aligned < program->base_size);
Libs::LibKernel::Memory::RegisterProgramMemory(
program->base_vaddr, program->mapped_size, Common::VirtualMemory::Mode::ExecuteReadWrite,
Common::PathToString(program->file_name.filename()).c_str());
LOGF("base_vaddr = 0x%016" PRIx64 "\n"
"base_size = 0x%016" PRIx64 "\n"
"base_size_aligned = 0x%016" PRIx64 "\n"
@@ -2060,11 +2216,8 @@ void RuntimeLinker::LoadProgramToMemory(Program* program) {
}
if (!skip_protect) {
if (!Common::VirtualMemory::Protect(segment_addr, segment_memory_size, mode)) {
EXIT("failed to protect ELF segment %u\n", static_cast<unsigned>(i));
}
Libs::LibKernel::Memory::UpdateProgramMemoryProtection(segment_addr,
segment_memory_size, mode);
Libs::LibKernel::Memory::SetProgramMemoryProtection(segment_addr,
segment_memory_size, mode);
if (Common::VirtualMemory::IsExecute(mode)) {
Common::VirtualMemory::FlushInstructionCache(segment_addr, segment_memory_size);
@@ -2105,15 +2258,29 @@ void RuntimeLinker::LoadProgramToMemory(Program* program) {
void RuntimeLinker::DeleteProgram(Program* p) {
auto program = std::unique_ptr<Program>(p);
if (g_tls_main_program == program.get()) {
g_tls_main_program = nullptr;
}
if (g_tls_cached_main_program == program.get()) {
g_tls_cached_main_program = nullptr;
g_tls_cached_main_tcb = nullptr;
}
for (auto& record: g_stubbed_imports) {
if (record.patch_vaddr >= program->base_vaddr &&
record.patch_vaddr < program->base_vaddr + program->mapped_size) {
record.patch_vaddr = 0;
}
}
if (program->base_vaddr != 0 || program->mapped_size != 0) {
EXIT_IF(program->base_vaddr == 0 || program->mapped_size == 0);
Libs::LibKernel::Memory::UnregisterProgramMemory(program->base_vaddr, program->mapped_size);
EXIT_IF(!Common::VirtualMemory::Free(program->base_vaddr));
EXIT_IF(
!Libs::LibKernel::Memory::FreeGuestMemory(program->base_vaddr, program->mapped_size));
}
if (program->custom_call_plt_vaddr != 0 || program->custom_call_plt_num != 0) {
Common::VirtualMemory::Free(program->custom_call_plt_vaddr);
const auto size = Jit::CallPlt::GetSize(program->custom_call_plt_num);
EXIT_IF(!Libs::LibKernel::Memory::FreeGuestMemory(program->custom_call_plt_vaddr, size));
}
}
@@ -2237,13 +2404,13 @@ static void InstallRelocateHandler(Program* program) {
void** pltgot = reinterpret_cast<void**>(pltgot_vaddr);
Common::VirtualMemory::Mode old_mode {};
Common::VirtualMemory::Protect(pltgot_vaddr, pltgot_size, Common::VirtualMemory::Mode::Write,
&old_mode);
EXIT_IF(!Libs::LibKernel::Memory::ProtectGuestMemory(
pltgot_vaddr, pltgot_size, Common::VirtualMemory::Mode::Write, &old_mode));
pltgot[1] = program;
pltgot[2] = reinterpret_cast<void*>(RelocateHandler);
Common::VirtualMemory::Protect(pltgot_vaddr, pltgot_size, old_mode);
EXIT_IF(!Libs::LibKernel::Memory::ProtectGuestMemory(pltgot_vaddr, pltgot_size, old_mode));
if (Common::VirtualMemory::IsExecute(old_mode)) {
Common::VirtualMemory::FlushInstructionCache(pltgot_vaddr, pltgot_size);
@@ -2253,15 +2420,15 @@ static void InstallRelocateHandler(Program* program) {
if (program->custom_call_plt_vaddr == 0) {
program->custom_call_plt_num =
program->dynamic_info->jmprela_table_size / sizeof(Elf64_Rela);
auto size = Jit::CallPlt::GetSize(program->custom_call_plt_num);
program->custom_call_plt_vaddr =
Common::VirtualMemory::Alloc(SYSTEM_RESERVED, size, Common::VirtualMemory::Mode::Write);
auto size = Jit::CallPlt::GetSize(program->custom_call_plt_num);
program->custom_call_plt_vaddr = Libs::LibKernel::Memory::AllocateRuntimeMemory(
SYSTEM_RESERVED, size, Common::VirtualMemory::Mode::Write, "custom_call_plt");
EXIT_NOT_IMPLEMENTED(program->custom_call_plt_vaddr == 0);
auto* code = new (reinterpret_cast<void*>(program->custom_call_plt_vaddr))
Jit::CallPlt(program->custom_call_plt_num);
code->SetPltGot(pltgot_vaddr);
Common::VirtualMemory::Protect(program->custom_call_plt_vaddr, size,
Common::VirtualMemory::Mode::Execute);
EXIT_IF(!Libs::LibKernel::Memory::ProtectGuestMemory(program->custom_call_plt_vaddr, size,
Common::VirtualMemory::Mode::Execute));
Common::VirtualMemory::FlushInstructionCache(program->custom_call_plt_vaddr, size);
}
}
@@ -2272,8 +2439,8 @@ void RuntimeLinker::Relocate(Program* program) {
EXIT_IF(program == nullptr);
if (g_invalid_memory == 0) {
g_invalid_memory = Common::VirtualMemory::Alloc(INVALID_MEMORY, 4096,
Common::VirtualMemory::Mode::NoAccess);
g_invalid_memory = Libs::LibKernel::Memory::AllocateRuntimeMemory(
INVALID_MEMORY, 4096, Common::VirtualMemory::Mode::NoAccess, "invalid_memory", true);
EXIT_NOT_IMPLEMENTED(g_invalid_memory == 0);
}
@@ -2447,12 +2614,9 @@ void RuntimeLinker::SetupTlsHandler(Program* program) {
stub->SetOutputReg(reg);
}
if (!Common::VirtualMemory::Protect(program->tls.handler_vaddr, Jit::SafeCall::GetSize(),
Common::VirtualMemory::Mode::Execute)) {
EXIT("failed to protect program TLS handler\n");
}
Libs::LibKernel::Memory::UpdateProgramMemoryProtection(
program->tls.handler_vaddr, Jit::SafeCall::GetSize(), Common::VirtualMemory::Mode::Execute);
EXIT_IF(!Libs::LibKernel::Memory::ProtectGuestMemory(program->tls.handler_vaddr,
Jit::SafeCall::GetSize(),
Common::VirtualMemory::Mode::Execute));
Common::VirtualMemory::FlushInstructionCache(program->tls.handler_vaddr,
Jit::SafeCall::GetSize());
}