diff --git a/src/kernel/memory.cpp b/src/kernel/memory.cpp index 80b923f4e..4f97c6344 100644 --- a/src/kernel/memory.cpp +++ b/src/kernel/memory.cpp @@ -67,7 +67,7 @@ constexpr uint64_t PAGE_TABLE_POOL_SIZE = 4ull * 1024ull * 1024ull * 1024ull; constexpr uint64_t PAGE_TABLE_GRANULARITY = 2ull * 1024ull * 1024ull; constexpr int PAGE_TABLE_POOL_ENTRIES = static_cast(PAGE_TABLE_POOL_SIZE / PAGE_TABLE_GRANULARITY); -constexpr uint64_t DEFAULT_FLEXIBLE_MEMORY_SIZE = 1ull * 1024ull * 1024ull * 1024ull; +constexpr uint64_t DEFAULT_FLEXIBLE_MEMORY_SIZE = 448ull * 1024ull * 1024ull; static uint64_t g_flexible_memory_size = DEFAULT_FLEXIBLE_MEMORY_SIZE; static bool g_flexible_memory_size_frozen = false; @@ -738,6 +738,8 @@ public: static uint64_t Size() { return g_flexible_memory_size; } uint64_t Available(); + void Charge(uint64_t size); + void Release(uint64_t size); bool Map(uint64_t vaddr, size_t len, int prot, VirtualMemory::Mode mode, GpuAccessMode gpu_mode, const char* name); @@ -1967,6 +1969,19 @@ uint64_t FlexibleMemory::Available() { return (Size() >= m_allocated_total ? Size() - m_allocated_total : 0); } +void FlexibleMemory::Charge(uint64_t size) { + Common::LockGuard lock(m_mutex); + + m_allocated_total += size; +} + +void FlexibleMemory::Release(uint64_t size) { + Common::LockGuard lock(m_mutex); + + EXIT_IF(size > m_allocated_total); + m_allocated_total -= size; +} + void PooledMemory::AddFreeUnlocked(uint64_t start, uint64_t size) { if (size == 0) { return; @@ -3770,6 +3785,14 @@ uint64_t AllocateProgramMemory(uint64_t search_addr, uint64_t size, VirtualMemor return AllocateGuestRuntimeMemory(search_addr, size, mode, name, VirtualRangeType::Code, false); } +void ChargeFlexibleMemory(uint64_t size) { + g_flexible_memory->Charge(size); +} + +void ReleaseFlexibleMemory(uint64_t size) { + g_flexible_memory->Release(size); +} + void SetProgramMemoryProtection(uint64_t vaddr, uint64_t size, VirtualMemory::Mode mode) { std::lock_guard memory_operation_lock(g_memory_operation_mutex); diff --git a/src/kernel/memory.h b/src/kernel/memory.h index e350d755d..244565839 100644 --- a/src/kernel/memory.h +++ b/src/kernel/memory.h @@ -197,6 +197,9 @@ int KYTY_SYSV_ABI KernelMemoryPoolGetBlockStats(KernelMemoryPoolBlockStats* outp uint64_t AllocateProgramMemory(uint64_t search_addr, uint64_t size, Common::VirtualMemory::Mode mode, const char* name); +// Loaded modules' segments use flexible memory without taking its backing. +void ChargeFlexibleMemory(uint64_t size); +void ReleaseFlexibleMemory(uint64_t size); void SetProgramMemoryProtection(uint64_t vaddr, uint64_t size, Common::VirtualMemory::Mode mode); uint64_t AllocateRuntimeMemory(uint64_t search_addr, uint64_t size, Common::VirtualMemory::Mode mode, const char* name, diff --git a/src/loader/runtimeLinker.cpp b/src/loader/runtimeLinker.cpp index 8c57790b0..daeb39c22 100644 --- a/src/loader/runtimeLinker.cpp +++ b/src/loader/runtimeLinker.cpp @@ -1636,6 +1636,8 @@ void RuntimeLinker::DeleteTls(Program* program, int thread_id) { } } +constexpr uint64_t GUEST_PAGE_SIZE = 0x4000; + static uint64_t CalcBaseSize(const Elf64_Ehdr* ehdr, const Elf64_Phdr* phdr) { uint64_t base_size = 0; for (Elf64_Half i = 0; i < ehdr->e_phnum; i++) { @@ -1649,6 +1651,20 @@ static uint64_t CalcBaseSize(const Elf64_Ehdr* ehdr, const Elf64_Phdr* phdr) { return base_size; } +// A module's E, R and RW LOAD segments use flexible memory in 16 KiB pages; GNU_RELRO adds none +// since it lies inside an RW LOAD. +uint64_t CalcFlexibleMemorySize(const Elf64_Phdr* phdr, Elf64_Half phnum) { + uint64_t size = 0; + for (Elf64_Half i = 0; i < phnum; i++) { + const auto flags = phdr[i].p_flags; + if (phdr[i].p_type == PT_LOAD && + (flags == PF_X || flags == PF_R || flags == (PF_R | PF_W))) { + size += AlignUp(phdr[i].p_memsz, GUEST_PAGE_SIZE); + } + } + return size; +} + // NOLINTNEXTLINE(readability-function-cognitive-complexity) void RuntimeLinker::LoadProgramToMemory(Program* program) { KYTY_PROFILER_FUNCTION(); @@ -1668,8 +1684,7 @@ void RuntimeLinker::LoadProgramToMemory(Program* program) { EXIT_IF(phdr == nullptr || ehdr == nullptr); - program->base_size = CalcBaseSize(ehdr, phdr); - constexpr uint64_t GUEST_PAGE_SIZE = 0x4000; + program->base_size = CalcBaseSize(ehdr, phdr); EXIT_IF(program->base_size > UINT64_MAX - (GUEST_PAGE_SIZE - 1)); program->base_size_aligned = AlignUp(program->base_size, GUEST_PAGE_SIZE); @@ -1694,6 +1709,8 @@ void RuntimeLinker::LoadProgramToMemory(Program* program) { g_desired_base_addr, program->mapped_size, Common::VirtualMemory::Mode::ExecuteReadWrite, Common::PathToString(program->file_name.filename()).c_str()); EXIT_IF(program->base_vaddr == 0); + program->flexible_size = CalcFlexibleMemorySize(phdr, ehdr->e_phnum); + Libs::LibKernel::Memory::ChargeFlexibleMemory(program->flexible_size); if (patch_guest_instructions) { const auto trampoline_addr = program->base_vaddr + program->base_size_aligned; @@ -1879,6 +1896,7 @@ void RuntimeLinker::DeleteProgram(Program* p) { } EXIT_IF( !Libs::LibKernel::Memory::FreeGuestMemory(program->base_vaddr, program->mapped_size)); + Libs::LibKernel::Memory::ReleaseFlexibleMemory(program->flexible_size); } } diff --git a/src/loader/runtimeLinker.h b/src/loader/runtimeLinker.h index f79cb50f1..6735dd427 100644 --- a/src/loader/runtimeLinker.h +++ b/src/loader/runtimeLinker.h @@ -21,6 +21,7 @@ struct EntryParams { }; class Elf64; +struct Elf64_Phdr; struct Elf64_Sym; struct Elf64_Rela; class RuntimeLinker; @@ -128,6 +129,7 @@ struct Program { uint64_t base_size = 0; uint64_t base_size_aligned = 0; uint64_t mapped_size = 0; + uint64_t flexible_size = 0; uint64_t instruction_trampoline_size = 0; std::unique_ptr export_symbols; std::unique_ptr import_symbols; @@ -203,6 +205,8 @@ private: application_heap_posix_memalign_func_t m_application_heap_posix_memalign = nullptr; }; +uint64_t CalcFlexibleMemorySize(const Elf64_Phdr* phdr, uint16_t phnum); + #if defined(KYTY_VIRTUAL_MEMORY_ALLOCATION_TESTS) bool TestMainEntryUsesGuestStack(); bool TestModuleRelocationUsesWritableHostMapping(); diff --git a/tests/VirtualMemoryAllocationTests.cpp b/tests/VirtualMemoryAllocationTests.cpp index 1fa39fdad..76efd1989 100644 --- a/tests/VirtualMemoryAllocationTests.cpp +++ b/tests/VirtualMemoryAllocationTests.cpp @@ -7,6 +7,7 @@ #include "kernel/memory.h" #include "kernel/pthread.h" #include "libs/errno.h" +#include "loader/elf.h" #include "loader/guestInstructionPatcher.h" #include "loader/jit.h" #include "loader/runtimeLinker.h" @@ -88,6 +89,7 @@ using Libs::LibKernel::Memory::VirtualQueryInfo; // Prospero ABI? constexpr uint64_t SceKernelPageSize = 0x4000; constexpr uint64_t SceKernelTotalPhysicalSize = 13824ull * 1024ull * 1024ull; +constexpr uint64_t SceKernelFlexibleMemorySize = 448ull * 1024ull * 1024ull; constexpr uint64_t TestFlexibleMemorySize = 3072ull * 1024ull * 1024ull; constexpr int SceKernelProtCpuRead = 0x01; constexpr int SceKernelProtCpuRw = 0x02; @@ -188,6 +190,10 @@ void InitSubsystems() { "failed to remove temporary param.json"); Check("InitSubsystems", flexible_memory_size == TestFlexibleMemorySize, "failed to read flexible memory size from param.json"); + size_t default_flexible_memory_size = 0; + Libs::LibKernel::Memory::KernelConfiguredFlexibleMemorySize(&default_flexible_memory_size); + Check("InitSubsystems", default_flexible_memory_size == SceKernelFlexibleMemorySize, + "flexible memory does not default to 448 MiB without param.json"); Libs::LibKernel::Memory::SetFlexibleMemorySize(flexible_memory_size); subsystems.Initialize(); @@ -3650,6 +3656,32 @@ void TestModuleRelocationUsesWritableHostMapping() { std::printf("[host] %-48s ok\n", test); } +void TestModuleSegmentsUseFlexibleMemory() { + const char* test = "ModuleSegmentsUseFlexibleMemory"; + using namespace Loader; + // E, R and RW LOAD pages; GNU_RELRO over the RW page and the flagless LOAD add none. + const std::array phdrs {{ + {PT_LOAD, PF_X, 0x4000, 0x0000, 0x0000, 0x1ac, 0x1ac, 0x4000}, + {PT_LOAD, PF_R, 0x8000, 0x4000, 0x4000, 0x83, 0x83, 0x4000}, + {PT_LOAD, PF_R | PF_W, 0xc000, 0x8000, 0x8000, 0x338, 0x338, 0x4000}, + {0x6474e552, PF_R, 0xc000, 0x8000, 0x8000, 0x338, 0x4000, 0x1}, + {PT_LOAD, PF_R | PF_W, 0x10000, 0xc000, 0xc000, 0x21, 0x21, 0x4000}, + {PT_LOAD, 0, 0x10030, 0xc030, 0xc030, 0x550, 0x550, 0x4000}, + }}; + const auto size = CalcFlexibleMemorySize(phdrs.data(), phdrs.size()); + Check(test, size == 4 * SceKernelPageSize, "module segments counted wrong"); + + const auto baseline = AvailableFlexibleMemory(test); + Libs::LibKernel::Memory::ChargeFlexibleMemory(size); + Check(test, AvailableFlexibleMemory(test) == baseline - size, + "loaded module did not use flexible memory"); + Libs::LibKernel::Memory::ReleaseFlexibleMemory(size); + Check(test, AvailableFlexibleMemory(test) == baseline, + "unloaded module did not return flexible memory"); + + std::printf("[host] %-48s ok\n", test); +} + #if defined(__linux__) || KYTY_PLATFORM == KYTY_PLATFORM_WINDOWS volatile sig_atomic_t g_instruction_traps = 0; @@ -5054,6 +5086,7 @@ int main(int argc, char** argv) { RunTest(TestMemoryPoolCommitDecommitQueryFlags); RunTest(TestProgramMemoryAllocationAndProtection); RunTest(TestModuleRelocationUsesWritableHostMapping); + RunTest(TestModuleSegmentsUseFlexibleMemory); #if defined(__linux__) || KYTY_PLATFORM == KYTY_PLATFORM_WINDOWS RunTest(TestTlsStubsPreserveFlags); #endif