raphiaura/KytyPS5
1
0
Fork 0
mirror of https://github.com/KytyPS5/KytyPS5.git synced 2026-10-22 03:46:53 +02:00
Code Issues Packages Projects Releases Wiki Activity

fix(windows): add experimental guest red-zone protection.

Browse Source 
Port the static instruction patcher from shadPS4.
This commit is contained in:
nmzik
2026-08-07 05:14:56 +02:00
parent d5c95bc3ee
commit 26781e6fea
18 changed files with 1840 additions and 107 deletions
Download Patch File Download Diff File Expand all files Collapse all files
src/common/emulatorConfig.cpp
+6
View File
@@ -101,6 +101,12 @@ bool ReadbackLinearImagesEnabled() {
return g_config->readback_linear_images;
}
#if KYTY_PLATFORM == KYTY_PLATFORM_WINDOWS
bool RedZoneProtectionEnabled() {
return g_config->red_zone_protection_enabled;
}
#endif
const Keymap& GetKeymap() {
return g_config->keymap;
}
src/common/emulatorConfig.h
+7 -1
View File
@@ -45,7 +45,10 @@ struct ConfigOptions {
bool spirv_debug_printf_enabled = false;
bool renderdoc_enabled = false;
bool readback_linear_images = false;
Keymap keymap;
#if KYTY_PLATFORM == KYTY_PLATFORM_WINDOWS
bool red_zone_protection_enabled = false;
#endif
Keymap keymap;
};
void Load(const ConfigOptions& cfg);
@@ -76,6 +79,9 @@ bool SpirvDebugPrintfEnabled();
bool RenderDocEnabled();
bool ReadbackLinearImagesEnabled();
#if KYTY_PLATFORM == KYTY_PLATFORM_WINDOWS
bool RedZoneProtectionEnabled();
#endif
const Keymap& GetKeymap();
src/graphics/guest_gpu/command_processor/pm4Handlers.cpp
+1 -1
View File
@@ -2135,7 +2135,7 @@ KYTY_CP_OP_PARSER(CpOpClearState) {
EXIT_NOT_IMPLEMENTED(cmd_id != 0xc0001200);
EXIT_NOT_IMPLEMENTED((buffer[0] & ~0xfu) != 0);
cp.Reset();
cp.ApplyContextStateOperation(ContextStateOperation::Clear);
return 1;
}
src/launcher/forms/configuration_edit_dialog.ui
+10
View File
@@ -88,6 +88,16 @@
</property>
</widget>
</item>
<item row="2" column="1">
<widget class="QCheckBox" name="checkBox_red_zone_protection">
<property name="toolTip">
<string>Helps prevent crashes in some games on Windows</string>
</property>
<property name="text">
<string>Windows SysV red zone crash protection (Experimental)</string>
</property>
</widget>
</item>
</layout>
</widget>
</item>
src/launcher/include/configuration.h
+15 -2
View File
@@ -99,7 +99,10 @@ public:
QString printf_output_file = "_kyty.txt";
ProfilerDirection profiler_direction = ProfilerDirection::None;
bool renderdoc_enabled = false;
QStringList host_input_mapping;
#if defined(_WIN32)
bool red_zone_protection_enabled = false;
#endif
QStringList host_input_mapping;
QString elf = QStringLiteral("eboot.bin");
@@ -119,7 +122,10 @@ public:
printf_output_file = other.printf_output_file;
profiler_direction = other.profiler_direction;
renderdoc_enabled = other.renderdoc_enabled;
host_input_mapping = other.host_input_mapping;
#if defined(_WIN32)
red_zone_protection_enabled = other.red_zone_protection_enabled;
#endif
host_input_mapping = other.host_input_mapping;
}
void CopyFrom(const Configuration& other) {
@@ -156,6 +162,9 @@ public:
KYTY_CFG_SET(printf_output_file);
KYTY_CFG_SET(profiler_direction);
KYTY_CFG_SET(renderdoc_enabled);
#if defined(_WIN32)
KYTY_CFG_SET(red_zone_protection_enabled);
#endif
s->setValue("host_input_mapping", host_input_mapping);
KYTY_CFG_SET(elf);
}
@@ -183,6 +192,10 @@ public:
KYTY_CFG_GET(printf_output_file);
KYTY_CFG_GET(profiler_direction);
KYTY_CFG_GET(renderdoc_enabled);
#if defined(_WIN32)
red_zone_protection_enabled =
s->value("red_zone_protection_enabled", red_zone_protection_enabled).toBool();
#endif
host_input_mapping = s->value("host_input_mapping", host_input_mapping).toStringList();
elf = s->value("elf", elf).toString();
}
src/launcher/src/configurationEditDialog.cpp
+9 -1
View File
@@ -164,6 +164,11 @@ void ConfigurationEditDialog::Init(const Configuration& info) {
m_ui->checkBox_shader_validation->setChecked(info.shader_validation_enabled);
m_ui->checkBox_vulkan_validation->setChecked(info.vulkan_validation_enabled);
m_ui->checkBox_renderdoc_capture->setChecked(info.renderdoc_enabled);
#if defined(_WIN32)
m_ui->checkBox_red_zone_protection->setChecked(info.red_zone_protection_enabled);
#else
m_ui->checkBox_red_zone_protection->setVisible(false);
#endif
ListInit(m_ui->comboBox_shader_optimization_type, info.shader_optimization_type);
ListInit(m_ui->comboBox_shader_log_direction, info.shader_log_direction);
m_ui->lineEdit_shader_log_folder->setText(info.shader_log_folder);
@@ -286,7 +291,10 @@ static void UpdateInfo(Configuration& info, Ui::ConfigurationEditDialog& ui) {
info.vulkan_validation_enabled = ui.checkBox_vulkan_validation->isChecked();
info.shader_validation_enabled = ui.checkBox_shader_validation->isChecked();
info.renderdoc_enabled = ui.checkBox_renderdoc_capture->isChecked();
info.shader_optimization_type = TextToEnum<Configuration::ShaderOptimizationType>(
#if defined(_WIN32)
info.red_zone_protection_enabled = ui.checkBox_red_zone_protection->isChecked();
#endif
info.shader_optimization_type = TextToEnum<Configuration::ShaderOptimizationType>(
ui.comboBox_shader_optimization_type->currentText());
info.shader_log_direction = TextToEnum<Configuration::ShaderLogDirection>(
ui.comboBox_shader_log_direction->currentText());
src/launcher/src/mainDialog.cpp
+5
View File
@@ -220,6 +220,11 @@ static QStringList CreateEmulatorArgs(const Configuration& info) {
args << "--printf-output-file" << info.printf_output_file;
args << "--profiler-direction" << EnumToText(info.profiler_direction);
args << "--spirv-debug-printf" << "false";
#if defined(_WIN32)
if (info.red_zone_protection_enabled) {
args << "--redzone";
}
#endif
for (const auto& binding: info.host_input_mapping) {
args << "--keymap" << binding;
}
src/libs/libKernel.cpp
+3 -60
View File
@@ -516,6 +516,7 @@ static void QueuePendingSignal(Pthread thread, int signum) {
PthreadQueuePendingSignal(thread, signum);
}
#if KYTY_PLATFORM != KYTY_PLATFORM_WINDOWS
static void WaitForSignalDispatch(Pthread thread, int signum) {
constexpr auto DISPATCH_WAIT_STEP = std::chrono::microseconds(1000);
constexpr auto DISPATCH_WAIT_MAX = std::chrono::milliseconds(2);
@@ -526,6 +527,7 @@ static void WaitForSignalDispatch(Pthread thread, int signum) {
waited += DISPATCH_WAIT_STEP;
}
}
#endif
static bool TakePendingSignal(Pthread thread, int signum) {
return PthreadTakePendingSignal(thread, signum);
@@ -1070,60 +1072,6 @@ static void SignalApcHandler(void* arg1, void* arg2, void* /*arg3*/, PCONTEXT co
}
}
static bool DispatchSignalWithSuspendedThreadContext(HANDLE target_thread, Pthread thread,
int signum, exception_handler_func_t handler) {
if (target_thread == nullptr || thread == nullptr || handler == nullptr) {
return false;
}
if (!PthreadHasPendingSignal(thread, signum)) {
return true;
}
if (SuspendThread(target_thread) == static_cast<DWORD>(-1)) {
return false;
}
bool dispatched = false;
CONTEXT host_ctx {};
host_ctx.ContextFlags = CONTEXT_FULL | CONTEXT_SEGMENTS;
if (GetThreadContext(target_thread, &host_ctx) != FALSE) {
if (TakePendingSignal(thread, signum)) {
auto ctx = CreateSignalUcontext(&host_ctx);
auto guest_context = IsGuestCodeAddress(ctx.uc_mcontext.mc_rip);
HANDLE helper_thread_handle = nullptr;
if (guest_context) {
if (DuplicateHandle(GetCurrentProcess(), target_thread, GetCurrentProcess(),
&helper_thread_handle, THREAD_SUSPEND_RESUME, FALSE,
0) == FALSE) {
ResumeThread(target_thread);
return false;
}
} else {
SanitizeNonGuestSignalUcontext(&ctx, thread);
ResumeThread(target_thread);
}
std::thread([helper_thread_handle, thread, signum, handler, ctx]() mutable {
SignalDispatchScope scope;
auto* previous_self = PthreadSwapSelfForSignal(thread);
handler(signum, &ctx);
PthreadSwapSelfForSignal(previous_self);
if (helper_thread_handle != nullptr) {
ResumeThread(helper_thread_handle);
CloseHandle(helper_thread_handle);
}
}).detach();
dispatched = true;
return true;
}
dispatched = true;
}
ResumeThread(target_thread);
return dispatched;
}
#endif
void KernelDispatchPendingSignalForCurrentThread() {
@@ -1206,8 +1154,7 @@ static int KYTY_SYSV_ABI KernelRaiseException(Pthread thread, int signum) {
}
HANDLE target_thread =
OpenThread(THREAD_SET_CONTEXT | THREAD_SUSPEND_RESUME | THREAD_GET_CONTEXT, FALSE,
static_cast<DWORD>(target_thread_id));
OpenThread(THREAD_SET_CONTEXT, FALSE, static_cast<DWORD>(target_thread_id));
if (target_thread == nullptr) {
return KERNEL_ERROR_EINVAL;
}
@@ -1231,10 +1178,6 @@ static int KYTY_SYSV_ABI KernelRaiseException(Pthread thread, int signum) {
Common::CondVar::SignalThread(PthreadGetUniqueId(thread));
PthreadWakeForSignal(thread);
WaitForSignalDispatch(thread, signum);
if (PthreadHasPendingSignal(thread, signum)) {
DispatchSignalWithSuspendedThreadContext(target_thread, thread, signum, handler);
}
CloseHandle(target_thread);
return OK;
#elif defined(__x86_64__)
src/loader/elf.h
+1
View File
@@ -53,6 +53,7 @@ constexpr Elf64_Half ET_DYNAMIC = 0xfe18; // Shared
constexpr Elf64_Word PT_LOAD = 1;
constexpr Elf64_Word PT_DYNAMIC = 2;
constexpr Elf64_Word PT_TLS = 7;
constexpr Elf64_Word PT_GNU_EH_FRAME = 0x6474e550;
constexpr Elf64_Word PT_OS_DYNLIBDATA = 0x61000000;
constexpr Elf64_Word PT_OS_PROCPARAM = 0x61000001;
constexpr Elf64_Word PT_OS_RELRO = 0x61000010;
src/loader/redZonePatcher.cpp
+1443
View File
File diff suppressed because it is too large Load Diff
src/loader/redZonePatcher.h
+39
View File
@@ -0,0 +1,39 @@
// SPDX-FileCopyrightText: Copyright 2026 shadPS4 Emulator Project
// SPDX-License-Identifier: GPL-2.0-or-later
#ifndef KYTY_LOADER_RED_ZONE_PATCHER_H_
#define KYTY_LOADER_RED_ZONE_PATCHER_H_
#include "common/common.h"
#include <span>
#include <vector>
namespace Loader {
struct RedZonePatchResult {
uint64_t function_count = 0;
uint64_t instruction_count = 0;
uint64_t red_zone_function_count = 0;
uint64_t memory_instruction_count = 0;
uint64_t short_memory_instruction_count = 0;
uint64_t patched_memory_instruction_count = 0;
uint64_t stack_dependent_memory_instruction_count = 0;
uint64_t control_flow_memory_instruction_count = 0;
uint64_t unrelocatable_memory_instruction_count = 0;
uint64_t indirect_red_zone_function_count = 0;
};
void RegisterRedZonePatchModule(void* module_ptr, uint64_t module_size, void* trampoline_area_ptr,
uint64_t trampoline_area_size);
void UnregisterRedZonePatchModule(void* module_ptr);
RedZonePatchResult PatchRedZoneMemoryInstructions(uint64_t segment_addr, uint64_t segment_size,
std::span<const uintptr_t> function_starts);
bool DecodeEhFrameFunctionStarts(uint64_t eh_frame_header_addr, uint64_t eh_frame_header_size,
std::vector<uintptr_t>* function_starts);
} // namespace Loader
#endif /* KYTY_LOADER_RED_ZONE_PATCHER_H_ */
src/loader/runtimeLinker.cpp
+73 -1
View File
@@ -19,6 +19,7 @@
#include "loader/elf.h"
#include "loader/gamePatch.h"
#include "loader/jit.h"
#include "loader/redZonePatcher.h"
#include "loader/symbolDatabase.h"
#include "loader/x64InstructionEmulator.h"
@@ -2202,17 +2203,39 @@ void RuntimeLinker::LoadProgramToMemory(Program* program) {
EXIT_IF(tls_handler_size > UINT64_MAX - program->base_size_aligned);
program->mapped_size = program->base_size_aligned + tls_handler_size;
#if KYTY_PLATFORM == KYTY_PLATFORM_WINDOWS
const bool use_red_zone_protection = Config::RedZoneProtectionEnabled();
constexpr uint64_t RED_ZONE_TRAMPOLINE_SIZE = 8u * 1024u * 1024u;
if (use_red_zone_protection) {
EXIT_IF(RED_ZONE_TRAMPOLINE_SIZE > UINT64_MAX - program->mapped_size);
program->mapped_size += RED_ZONE_TRAMPOLINE_SIZE;
}
#endif
program->base_vaddr = Libs::LibKernel::Memory::AllocateProgramMemory(
g_desired_base_addr, program->mapped_size, Common::VirtualMemory::Mode::ExecuteReadWrite,
Common::PathToString(program->file_name.filename()).c_str());
EXIT_IF(program->base_vaddr == 0);
#if KYTY_PLATFORM == KYTY_PLATFORM_WINDOWS
if (use_red_zone_protection) {
program->red_zone_trampoline_vaddr = program->base_vaddr + program->base_size_aligned;
program->red_zone_trampoline_size = RED_ZONE_TRAMPOLINE_SIZE;
RegisterRedZonePatchModule(reinterpret_cast<void*>(program->base_vaddr),
program->base_size_aligned,
reinterpret_cast<void*>(program->red_zone_trampoline_vaddr),
program->red_zone_trampoline_size);
}
#endif
if (!is_shared) {
program->tls.handler_vaddr = program->base_vaddr + program->base_size_aligned;
#if KYTY_PLATFORM == KYTY_PLATFORM_WINDOWS
program->tls.handler_vaddr += program->red_zone_trampoline_size;
#endif
}
g_desired_base_addr += CODE_BASE_INCR * (1 + program->mapped_size / CODE_BASE_INCR);
EXIT_IF(program->base_vaddr == 0);
EXIT_IF(program->base_size_aligned < program->base_size);
LOGF("base_vaddr = 0x%016" PRIx64 "\n"
"base_size = 0x%016" PRIx64 "\n"
@@ -2228,6 +2251,11 @@ void RuntimeLinker::LoadProgramToMemory(Program* program) {
}
// program->elf->SetBaseVAddr(program->base_vaddr);
#if KYTY_PLATFORM == KYTY_PLATFORM_WINDOWS
std::vector<std::pair<uint64_t, uint64_t>> executable_segments;
uint64_t eh_frame_header_addr = 0;
uint64_t eh_frame_header_size = 0;
#endif
for (Elf64_Half i = 0; i < ehdr->e_phnum; i++) {
if (phdr[i].p_memsz != 0 && (phdr[i].p_type == PT_LOAD || phdr[i].p_type == PT_OS_RELRO)) {
@@ -2250,6 +2278,11 @@ void RuntimeLinker::LoadProgramToMemory(Program* program) {
if (Common::VirtualMemory::IsExecute(mode)) {
PatchProgram(program, segment_addr, segment_memory_size);
#if KYTY_PLATFORM == KYTY_PLATFORM_WINDOWS
if (use_red_zone_protection) {
executable_segments.emplace_back(segment_addr, segment_file_size);
}
#endif
}
if (!skip_protect) {
@@ -2284,8 +2317,42 @@ void RuntimeLinker::LoadProgramToMemory(Program* program) {
program->proc_param_vaddr = phdr[i].p_vaddr + program->base_vaddr;
}
#if KYTY_PLATFORM == KYTY_PLATFORM_WINDOWS
if (use_red_zone_protection && phdr[i].p_type == PT_GNU_EH_FRAME) {
eh_frame_header_addr = phdr[i].p_vaddr + program->base_vaddr;
eh_frame_header_size = phdr[i].p_memsz;
}
#endif
}
#if KYTY_PLATFORM == KYTY_PLATFORM_WINDOWS
if (use_red_zone_protection) {
std::vector<uintptr_t> function_starts;
if (!DecodeEhFrameFunctionStarts(eh_frame_header_addr, eh_frame_header_size,
&function_starts)) {
LOGF("Windows guest red-zone patching could not decode function boundaries for %s\n",
Common::PathToString(program->file_name).c_str());
}
for (const auto& [segment_addr, segment_size]: executable_segments) {
const auto result =
PatchRedZoneMemoryInstructions(segment_addr, segment_size, function_starts);
LOGF("Windows guest red-zone patching: %s, functions=%" PRIu64 ", red_zone=%" PRIu64
", memory=%" PRIu64 ", patched=%" PRIu64 ", short=%" PRIu64 ", stack=%" PRIu64
", control=%" PRIu64 ", unrelocatable=%" PRIu64 "\n",
Common::PathToString(program->file_name.filename()).c_str(), result.function_count,
result.red_zone_function_count, result.memory_instruction_count,
result.patched_memory_instruction_count, result.short_memory_instruction_count,
result.stack_dependent_memory_instruction_count,
result.control_flow_memory_instruction_count,
result.unrelocatable_memory_instruction_count);
Common::VirtualMemory::FlushInstructionCache(segment_addr, segment_size);
}
Common::VirtualMemory::FlushInstructionCache(program->red_zone_trampoline_vaddr,
program->red_zone_trampoline_size);
}
#endif
if (!is_shared) {
SetupTlsHandler(program);
}
@@ -2311,6 +2378,11 @@ void RuntimeLinker::DeleteProgram(Program* p) {
if (program->base_vaddr != 0 || program->mapped_size != 0) {
EXIT_IF(program->base_vaddr == 0 || program->mapped_size == 0);
#if KYTY_PLATFORM == KYTY_PLATFORM_WINDOWS
if (program->red_zone_trampoline_size != 0) {
UnregisterRedZonePatchModule(reinterpret_cast<void*>(program->base_vaddr));
}
#endif
EXIT_IF(
!Libs::LibKernel::Memory::FreeGuestMemory(program->base_vaddr, program->mapped_size));
}
src/loader/runtimeLinker.h
+13 -9
View File
@@ -114,15 +114,19 @@ struct Program {
Program();
~Program();
int32_t unique_id = -1;
RuntimeLinker* rt = nullptr;
std::filesystem::path file_name;
std::unique_ptr<Elf64> elf;
std::unique_ptr<DynamicInfo> dynamic_info;
uint64_t base_vaddr = 0;
uint64_t base_size = 0;
uint64_t base_size_aligned = 0;
uint64_t mapped_size = 0;
int32_t unique_id = -1;
RuntimeLinker* rt = nullptr;
std::filesystem::path file_name;
std::unique_ptr<Elf64> elf;
std::unique_ptr<DynamicInfo> dynamic_info;
uint64_t base_vaddr = 0;
uint64_t base_size = 0;
uint64_t base_size_aligned = 0;
uint64_t mapped_size = 0;
#if KYTY_PLATFORM == KYTY_PLATFORM_WINDOWS
uint64_t red_zone_trampoline_vaddr = 0;
uint64_t red_zone_trampoline_size = 0;
#endif
std::unique_ptr<SymbolDatabase> export_symbols;
std::unique_ptr<SymbolDatabase> import_symbols;
ThreadLocalStorage tls;
src/main.cpp
+10
View File
@@ -64,6 +64,9 @@ static void PrintUsage() {
::printf(" --spirv-debug-printf <true|false> Enable SPIR-V debug printf.\n");
::printf(
" --readback-linear-images <true|false> Read back writable linear images on submit.\n");
#if KYTY_PLATFORM == KYTY_PLATFORM_WINDOWS
::printf(" --redzone Protect the guest SysV red zone.\n");
#endif
::printf(" --keymap <Pad=Key> DualSense mapping; may be repeated.\n");
::printf(" --rd Enable RenderDoc capture.\n");
}
@@ -138,6 +141,13 @@ static bool ParseArgs(int argc, char* argv[], RunOptions& options, bool& show_he
continue;
}
#if KYTY_PLATFORM == KYTY_PLATFORM_WINDOWS
if (arg == "--redzone") {
options.config.red_zone_protection_enabled = true;
continue;
}
#endif
if (!Common::StartsWith(arg, "--")) {
::printf("game input must be provided with --game\n");
return false;