kernel: default flexible memory to 448 MiB and charge loaded modules' segments to it

src/kernel/memory.cpp

@@ -67,7 +67,7 @@ constexpr uint64_t PAGE_TABLE_POOL_SIZE   = 4ull * 1024ull * 1024ull * 1024ull;
 constexpr uint64_t PAGE_TABLE_GRANULARITY = 2ull * 1024ull * 1024ull;
 constexpr int      PAGE_TABLE_POOL_ENTRIES =
     static_cast<int>(PAGE_TABLE_POOL_SIZE / PAGE_TABLE_GRANULARITY);
-constexpr uint64_t DEFAULT_FLEXIBLE_MEMORY_SIZE = 1ull * 1024ull * 1024ull * 1024ull;
+constexpr uint64_t DEFAULT_FLEXIBLE_MEMORY_SIZE = 448ull * 1024ull * 1024ull;
 
 static uint64_t                      g_flexible_memory_size        = DEFAULT_FLEXIBLE_MEMORY_SIZE;
 static bool                          g_flexible_memory_size_frozen = false;
@@ -738,6 +738,8 @@ public:
 
 	static uint64_t Size() { return g_flexible_memory_size; }
 	uint64_t        Available();
+	void            Charge(uint64_t size);
+	void            Release(uint64_t size);
 
 	bool Map(uint64_t vaddr, size_t len, int prot, VirtualMemory::Mode mode, GpuAccessMode gpu_mode,
 	         const char* name);
@@ -1967,6 +1969,19 @@ uint64_t FlexibleMemory::Available() {
 	return (Size() >= m_allocated_total ? Size() - m_allocated_total : 0);
 }
 
+void FlexibleMemory::Charge(uint64_t size) {
+	Common::LockGuard lock(m_mutex);
+
+	m_allocated_total += size;
+}
+
+void FlexibleMemory::Release(uint64_t size) {
+	Common::LockGuard lock(m_mutex);
+
+	EXIT_IF(size > m_allocated_total);
+	m_allocated_total -= size;
+}
+
 void PooledMemory::AddFreeUnlocked(uint64_t start, uint64_t size) {
 	if (size == 0) {
 		return;
@@ -3770,6 +3785,14 @@ uint64_t AllocateProgramMemory(uint64_t search_addr, uint64_t size, VirtualMemor
 	return AllocateGuestRuntimeMemory(search_addr, size, mode, name, VirtualRangeType::Code, false);
 }
 
+void ChargeFlexibleMemory(uint64_t size) {
+	g_flexible_memory->Charge(size);
+}
+
+void ReleaseFlexibleMemory(uint64_t size) {
+	g_flexible_memory->Release(size);
+}
+
 void SetProgramMemoryProtection(uint64_t vaddr, uint64_t size, VirtualMemory::Mode mode) {
 	std::lock_guard<std::recursive_mutex> memory_operation_lock(g_memory_operation_mutex);
 

src/kernel/memory.h

@@ -197,6 +197,9 @@ int KYTY_SYSV_ABI KernelMemoryPoolGetBlockStats(KernelMemoryPoolBlockStats* outp
 
 uint64_t AllocateProgramMemory(uint64_t search_addr, uint64_t size,
                                Common::VirtualMemory::Mode mode, const char* name);
+// Loaded modules' segments use flexible memory without taking its backing.
+void ChargeFlexibleMemory(uint64_t size);
+void ReleaseFlexibleMemory(uint64_t size);
 void SetProgramMemoryProtection(uint64_t vaddr, uint64_t size, Common::VirtualMemory::Mode mode);
 uint64_t AllocateRuntimeMemory(uint64_t search_addr, uint64_t size,
                                Common::VirtualMemory::Mode mode, const char* name,

src/loader/runtimeLinker.cpp

@@ -1636,6 +1636,8 @@ void RuntimeLinker::DeleteTls(Program* program, int thread_id) {
 	}
 }
 
+constexpr uint64_t GUEST_PAGE_SIZE = 0x4000;
+
 static uint64_t CalcBaseSize(const Elf64_Ehdr* ehdr, const Elf64_Phdr* phdr) {
 	uint64_t base_size = 0;
 	for (Elf64_Half i = 0; i < ehdr->e_phnum; i++) {
@@ -1649,6 +1651,20 @@ static uint64_t CalcBaseSize(const Elf64_Ehdr* ehdr, const Elf64_Phdr* phdr) {
 	return base_size;
 }
 
+// A module's E, R and RW LOAD segments use flexible memory in 16 KiB pages; GNU_RELRO adds none
+// since it lies inside an RW LOAD.
+uint64_t CalcFlexibleMemorySize(const Elf64_Phdr* phdr, Elf64_Half phnum) {
+	uint64_t size = 0;
+	for (Elf64_Half i = 0; i < phnum; i++) {
+		const auto flags = phdr[i].p_flags;
+		if (phdr[i].p_type == PT_LOAD &&
+		    (flags == PF_X || flags == PF_R || flags == (PF_R | PF_W))) {
+			size += AlignUp(phdr[i].p_memsz, GUEST_PAGE_SIZE);
+		}
+	}
+	return size;
+}
+
 // NOLINTNEXTLINE(readability-function-cognitive-complexity)
 void RuntimeLinker::LoadProgramToMemory(Program* program) {
 	KYTY_PROFILER_FUNCTION();
@@ -1668,8 +1684,7 @@ void RuntimeLinker::LoadProgramToMemory(Program* program) {
 
 	EXIT_IF(phdr == nullptr || ehdr == nullptr);
 
-	program->base_size                 = CalcBaseSize(ehdr, phdr);
-	constexpr uint64_t GUEST_PAGE_SIZE = 0x4000;
+	program->base_size = CalcBaseSize(ehdr, phdr);
 	EXIT_IF(program->base_size > UINT64_MAX - (GUEST_PAGE_SIZE - 1));
 	program->base_size_aligned = AlignUp(program->base_size, GUEST_PAGE_SIZE);
 
@@ -1694,6 +1709,8 @@ void RuntimeLinker::LoadProgramToMemory(Program* program) {
 	    g_desired_base_addr, program->mapped_size, Common::VirtualMemory::Mode::ExecuteReadWrite,
 	    Common::PathToString(program->file_name.filename()).c_str());
 	EXIT_IF(program->base_vaddr == 0);
+	program->flexible_size = CalcFlexibleMemorySize(phdr, ehdr->e_phnum);
+	Libs::LibKernel::Memory::ChargeFlexibleMemory(program->flexible_size);
 
 	if (patch_guest_instructions) {
 		const auto trampoline_addr           = program->base_vaddr + program->base_size_aligned;
@@ -1879,6 +1896,7 @@ void RuntimeLinker::DeleteProgram(Program* p) {
 		}
 		EXIT_IF(
 		    !Libs::LibKernel::Memory::FreeGuestMemory(program->base_vaddr, program->mapped_size));
+		Libs::LibKernel::Memory::ReleaseFlexibleMemory(program->flexible_size);
 	}
 }
 

src/loader/runtimeLinker.h

@@ -21,6 +21,7 @@ struct EntryParams {
 };
 
 class Elf64;
+struct Elf64_Phdr;
 struct Elf64_Sym;
 struct Elf64_Rela;
 class RuntimeLinker;
@@ -128,6 +129,7 @@ struct Program {
 	uint64_t                     base_size         = 0;
 	uint64_t                     base_size_aligned = 0;
 	uint64_t                     mapped_size       = 0;
+	uint64_t                     flexible_size     = 0;
 	uint64_t instruction_trampoline_size = 0;
 	std::unique_ptr<SymbolDatabase> export_symbols;
 	std::unique_ptr<SymbolDatabase> import_symbols;
@@ -203,6 +205,8 @@ private:
 	application_heap_posix_memalign_func_t m_application_heap_posix_memalign = nullptr;
 };
 
+uint64_t CalcFlexibleMemorySize(const Elf64_Phdr* phdr, uint16_t phnum);
+
 #if defined(KYTY_VIRTUAL_MEMORY_ALLOCATION_TESTS)
 bool TestMainEntryUsesGuestStack();
 bool TestModuleRelocationUsesWritableHostMapping();

tests/VirtualMemoryAllocationTests.cpp

@@ -7,6 +7,7 @@
 #include "kernel/memory.h"
 #include "kernel/pthread.h"
 #include "libs/errno.h"
+#include "loader/elf.h"
 #include "loader/guestInstructionPatcher.h"
 #include "loader/jit.h"
 #include "loader/runtimeLinker.h"
@@ -88,6 +89,7 @@ using Libs::LibKernel::Memory::VirtualQueryInfo;
 // Prospero ABI?
 constexpr uint64_t SceKernelPageSize             = 0x4000;
 constexpr uint64_t SceKernelTotalPhysicalSize    = 13824ull * 1024ull * 1024ull;
+constexpr uint64_t SceKernelFlexibleMemorySize   = 448ull * 1024ull * 1024ull;
 constexpr uint64_t TestFlexibleMemorySize        = 3072ull * 1024ull * 1024ull;
 constexpr int      SceKernelProtCpuRead          = 0x01;
 constexpr int      SceKernelProtCpuRw            = 0x02;
@@ -188,6 +190,10 @@ void InitSubsystems() {
 	      "failed to remove temporary param.json");
 	Check("InitSubsystems", flexible_memory_size == TestFlexibleMemorySize,
 	      "failed to read flexible memory size from param.json");
+	size_t default_flexible_memory_size = 0;
+	Libs::LibKernel::Memory::KernelConfiguredFlexibleMemorySize(&default_flexible_memory_size);
+	Check("InitSubsystems", default_flexible_memory_size == SceKernelFlexibleMemorySize,
+	      "flexible memory does not default to 448 MiB without param.json");
 	Libs::LibKernel::Memory::SetFlexibleMemorySize(flexible_memory_size);
 
 	subsystems.Initialize<Libs::LibKernel::Memory::Lifecycle>();
@@ -3650,6 +3656,32 @@ void TestModuleRelocationUsesWritableHostMapping() {
 	std::printf("[host]    %-48s ok\n", test);
 }
 
+void TestModuleSegmentsUseFlexibleMemory() {
+	const char* test = "ModuleSegmentsUseFlexibleMemory";
+	using namespace Loader;
+	// E, R and RW LOAD pages; GNU_RELRO over the RW page and the flagless LOAD add none.
+	const std::array<Elf64_Phdr, 6> phdrs {{
+	    {PT_LOAD, PF_X, 0x4000, 0x0000, 0x0000, 0x1ac, 0x1ac, 0x4000},
+	    {PT_LOAD, PF_R, 0x8000, 0x4000, 0x4000, 0x83, 0x83, 0x4000},
+	    {PT_LOAD, PF_R | PF_W, 0xc000, 0x8000, 0x8000, 0x338, 0x338, 0x4000},
+	    {0x6474e552, PF_R, 0xc000, 0x8000, 0x8000, 0x338, 0x4000, 0x1},
+	    {PT_LOAD, PF_R | PF_W, 0x10000, 0xc000, 0xc000, 0x21, 0x21, 0x4000},
+	    {PT_LOAD, 0, 0x10030, 0xc030, 0xc030, 0x550, 0x550, 0x4000},
+	}};
+	const auto size = CalcFlexibleMemorySize(phdrs.data(), phdrs.size());
+	Check(test, size == 4 * SceKernelPageSize, "module segments counted wrong");
+
+	const auto baseline = AvailableFlexibleMemory(test);
+	Libs::LibKernel::Memory::ChargeFlexibleMemory(size);
+	Check(test, AvailableFlexibleMemory(test) == baseline - size,
+	      "loaded module did not use flexible memory");
+	Libs::LibKernel::Memory::ReleaseFlexibleMemory(size);
+	Check(test, AvailableFlexibleMemory(test) == baseline,
+	      "unloaded module did not return flexible memory");
+
+	std::printf("[host]    %-48s ok\n", test);
+}
+
 #if defined(__linux__) || KYTY_PLATFORM == KYTY_PLATFORM_WINDOWS
 volatile sig_atomic_t g_instruction_traps = 0;
 
@@ -5054,6 +5086,7 @@ int main(int argc, char** argv) {
 	RunTest(TestMemoryPoolCommitDecommitQueryFlags);
 	RunTest(TestProgramMemoryAllocationAndProtection);
 	RunTest(TestModuleRelocationUsesWritableHostMapping);
+	RunTest(TestModuleSegmentsUseFlexibleMemory);
 #if defined(__linux__) || KYTY_PLATFORM == KYTY_PLATFORM_WINDOWS
 	RunTest(TestTlsStubsPreserveFlags);
 #endif