From 86e5e014032f00c3912645165aada7ad690a8381 Mon Sep 17 00:00:00 2001 From: nmzik Date: Wed, 16 Sep 2026 06:30:20 +0200 Subject: [PATCH] Protect Windows guest red zones when emulating VRSQRTPS --- src/launcher/src/configurationEditDialog.cpp | 2 +- src/loader/redZonePatcher.cpp | 975 ++++++++++--------- src/loader/redZonePatcher.h | 9 +- src/loader/runtimeLinker.cpp | 49 +- src/loader/x64InstructionEmulator.cpp | 39 +- src/loader/x64InstructionEmulator.h | 3 + src/main.cpp | 2 +- tests/VirtualMemoryAllocationTests.cpp | 128 ++- 8 files changed, 700 insertions(+), 507 deletions(-) diff --git a/src/launcher/src/configurationEditDialog.cpp b/src/launcher/src/configurationEditDialog.cpp index 3e63778e0..f68dd3e38 100644 --- a/src/launcher/src/configurationEditDialog.cpp +++ b/src/launcher/src/configurationEditDialog.cpp @@ -240,7 +240,7 @@ void ConfigurationEditDialog::Init(const Configuration& info) { m_ui->checkBox_vulkan_validation->setChecked(info.vulkan_validation_enabled); m_ui->checkBox_renderdoc_capture->setChecked(info.renderdoc_enabled); m_ui->checkBox_amd_cpu->setChecked(info.amd_cpu_enabled); -#if !defined(__linux__) +#if defined(__APPLE__) m_ui->checkBox_amd_cpu->setVisible(false); #endif #if defined(_WIN32) diff --git a/src/loader/redZonePatcher.cpp b/src/loader/redZonePatcher.cpp index 921870d9e..eff93a15c 100644 --- a/src/loader/redZonePatcher.cpp +++ b/src/loader/redZonePatcher.cpp @@ -5,6 +5,8 @@ #include "common/assert.h" #include "common/logging/log.h" +#include "common/virtualMemory.h" +#include "loader/x64InstructionEmulator.h" #include #include @@ -739,10 +741,512 @@ bool GenerateProtectedIndirectCall(const DecodedCodeInstruction& decoded, generator.call(ptr[rsp - GuestRedZoneSize - sizeof(uintptr_t)]); return true; } + +void CollectRedZoneMemoryInstructions(const DecodedFunction& function, + std::map& rewrite_sites, + RedZonePatchResult& result) { + if (!function.uses_red_zone) { + return; + } + for (const auto& [address, decoded]: function.instructions) { + if (!decoded.accesses_memory || !decoded.red_zone_live.any() || + rewrite_sites.contains(address)) { + continue; + } + ++result.memory_instruction_count; + if (decoded.instruction.length < NearJumpSize) { + ++result.short_memory_instruction_count; + } + if (decoded.instruction.meta.category == ZYDIS_CATEGORY_CALL && + decoded.operands[0].type == ZYDIS_OPERAND_TYPE_MEMORY && + decoded.operands[0].size == sizeof(uintptr_t) * CHAR_BIT && + !IsStackPointerRegister(decoded.operands[0].mem.base) && + !IsStackPointerRegister(decoded.operands[0].mem.index)) { + rewrite_sites[address] = { + .protect_red_zone = true, + .protected_indirect_call = true, + }; + continue; + } + if (decoded.uses_stack_pointer && !decoded.replaces_stack_pointer) { + ++result.stack_dependent_memory_instruction_count; + continue; + } + if (decoded.instruction.meta.category == ZYDIS_CATEGORY_CALL || + IsControlFlowTerminator(decoded.instruction)) { + ++result.control_flow_memory_instruction_count; + continue; + } + rewrite_sites[address].protect_red_zone = true; + } +} + +struct ReciprocalSquareRootSite { + uintptr_t address; + u8 length; + bool requires_red_zone_protection; +}; + +void CollectReciprocalSquareRoots(const DecodedFunction& function, + std::map& rewrite_sites, + std::vector& sites) { + for (const auto& [address, decoded]: function.instructions) { + if (!X64InstructionEmulator::IsReciprocalSquareRoot(decoded.instruction, + decoded.operands.data())) { + continue; + } + const bool protect_red_zone = decoded.red_zone_live.any(); + sites.push_back({address, decoded.instruction.length, protect_red_zone}); + if (protect_red_zone) { + rewrite_sites[address].protect_red_zone = true; + } + } +} + +uint64_t ApplyReciprocalSquareRootPatches(const PatchModule& module, + std::span sites, + uint64_t trampoline_addr, uint64_t trampoline_size) { + // Validate every required relocation before introducing any traps. + for (const auto& site: sites) { + if (site.requires_red_zone_protection && + !module.patched.contains(reinterpret_cast(site.address))) { + EXIT("Cannot preserve the guest red zone at emulated VRSQRTPS 0x%016" PRIx64 "\n", + static_cast(site.address)); + } + } + + uint64_t patched = 0; + for (const auto& site: sites) { + if (!module.patched.contains(reinterpret_cast(site.address))) { + patched += X64InstructionEmulator::PatchReciprocalSquareRoots(site.address, site.length); + } + } + return patched + + X64InstructionEmulator::PatchReciprocalSquareRoots(trampoline_addr, trampoline_size); +} + +void RelocateRedZoneInstructions(PatchModule* module, const DecodedFunction& function, + const std::map& rewrite_sites, + RedZonePatchResult& result) { + struct RelocationSpan { + std::vector instructions; + uintptr_t patch_start {}; + uintptr_t continuation {}; + size_t patch_size {}; + }; + + const auto emit_span = [&](const RelocationSpan& span) -> std::optional { + const size_t trampoline_offset = module->trampoline_gen.getSize(); + if (module->trampoline_exhausted) { + return std::nullopt; + } + try { + for (const auto* decoded: span.instructions) { + const auto rewrite = rewrite_sites.find(decoded->address); + const bool protected_indirect_call = + rewrite != rewrite_sites.end() && rewrite->second.protected_indirect_call; + const bool protect_red_zone = rewrite != rewrite_sites.end() && + rewrite->second.protect_red_zone && + !protected_indirect_call; + if (protect_red_zone) { + module->trampoline_gen.lea(rsp, ptr[rsp - GuestRedZoneSize]); + } + if (protected_indirect_call) { + if (!GenerateProtectedIndirectCall(*decoded, module->trampoline_gen)) { + module->trampoline_gen.setSize(trampoline_offset); + return std::nullopt; + } + } else if (!EncodeRelocatedInstruction(*decoded, module->trampoline_gen)) { + module->trampoline_gen.setSize(trampoline_offset); + return std::nullopt; + } + if (protect_red_zone && !decoded->replaces_stack_pointer) { + module->trampoline_gen.lea(rsp, ptr[rsp + GuestRedZoneSize]); + } + } + module->trampoline_gen.jmp(reinterpret_cast(span.continuation)); + } catch (const Xbyak::Error& error) { + module->trampoline_gen.setSize(trampoline_offset); + if (HandleTrampolineError(module, error)) { + return std::nullopt; + } + throw; + } + return trampoline_offset; + }; + + const auto record_rewrites = [&](const RelocationSpan& span) { + for (const auto* decoded: span.instructions) { + module->patched.insert(reinterpret_cast(decoded->address)); + const auto rewrite = rewrite_sites.find(decoded->address); + if (rewrite == rewrite_sites.end()) { + continue; + } + if (rewrite->second.protect_red_zone && decoded->accesses_memory) { + ++result.patched_memory_instruction_count; + } + } + }; + + std::vector> patched_spans; + std::vector relay_slots; + std::vector short_relay_slots; + std::vector unresolved_sites; + uintptr_t covered_until {}; + for (auto site_it = rewrite_sites.begin(); site_it != rewrite_sites.end(); ++site_it) { + const uintptr_t site = site_it->first; + if (site < covered_until) { + continue; + } + + const auto collect_forward_span = [&]() -> std::optional { + RelocationSpan span {.patch_start = site, .continuation = site}; + while (span.patch_size < NearJumpSize) { + const auto decoded_it = function.instructions.find(span.continuation); + if (decoded_it == function.instructions.end() || + (span.continuation != site && + function.branch_targets.contains(span.continuation))) { + return std::nullopt; + } + + const auto& decoded = decoded_it->second; + span.instructions.push_back(&decoded); + span.patch_size += decoded.instruction.length; + span.continuation += decoded.instruction.length; + if (span.patch_size < NearJumpSize && + IsControlFlowTerminator(decoded.instruction)) { + return std::nullopt; + } + } + return span; + }; + + const auto collect_backward_span = [&]() -> std::optional { + const auto site_instruction = function.instructions.find(site); + ASSERT(site_instruction != function.instructions.end()); + RelocationSpan span { + .instructions = {&site_instruction->second}, + .patch_start = site, + .continuation = site + site_instruction->second.instruction.length, + .patch_size = site_instruction->second.instruction.length, + }; + + while (span.patch_size < NearJumpSize) { + const auto previous_end = function.instructions.lower_bound(span.patch_start); + if (previous_end == function.instructions.begin() || + function.branch_targets.contains(span.patch_start)) { + return std::nullopt; + } + + const auto previous = std::prev(previous_end); + const auto& decoded = previous->second; + if (previous->first + decoded.instruction.length != span.patch_start || + previous->first < covered_until || + IsControlFlowTerminator(decoded.instruction)) { + return std::nullopt; + } + + span.instructions.insert(span.instructions.begin(), &decoded); + span.patch_start = previous->first; + span.patch_size += decoded.instruction.length; + } + return span; + }; + + std::optional selected_span; + std::optional trampoline_offset; + const auto& site_instruction = function.instructions.at(site); + const bool can_relocate_neighbors = !function.has_indirect_branch; + if (can_relocate_neighbors || site_instruction.instruction.length >= NearJumpSize) { + auto forward_span = collect_forward_span(); + if (forward_span) { + trampoline_offset = emit_span(*forward_span); + if (trampoline_offset) { + selected_span = std::move(forward_span); + } + } + } + if (!selected_span && can_relocate_neighbors) { + if (auto backward_span = collect_backward_span()) { + trampoline_offset = emit_span(*backward_span); + if (trampoline_offset) { + selected_span = std::move(backward_span); + } + } + } + if (!selected_span) { + unresolved_sites.push_back(site); + continue; + } + + const auto& span = *selected_span; + const auto* trampoline = module->trampoline_gen.getCode() + *trampoline_offset; + + auto& patch_gen = module->patch_gen; + patch_gen.reset(); + patch_gen.setSize(span.patch_start - reinterpret_cast(patch_gen.getCode())); + patch_gen.jmp(trampoline, Xbyak::CodeGenerator::LabelType::T_NEAR); + patch_gen.nop(span.patch_size - NearJumpSize); + + record_rewrites(span); + patched_spans.emplace_back(span.patch_start, span.continuation); + if (span.patch_size >= NearJumpSize * 2) { + relay_slots.push_back(span.patch_start + NearJumpSize); + } + if (span.patch_size >= NearJumpSize + ShortJumpSize) { + short_relay_slots.push_back(span.patch_start + NearJumpSize); + } + covered_until = span.continuation; + } + + const auto record_unsupported = [&](uintptr_t site) { + const auto& decoded = function.instructions.at(site); + const auto& rewrite = rewrite_sites.at(site); + if (rewrite.protect_red_zone && decoded.accesses_memory) { + ++result.unrelocatable_memory_instruction_count; + } + }; + const auto overlaps_patched_span = [&patched_spans](uintptr_t start, uintptr_t end) { + return std::ranges::any_of(patched_spans, [start, end](const auto& patched) { + return start < patched.second && patched.first < end; + }); + }; + + for (const uintptr_t site: unresolved_sites) { + if (module->patched.contains(reinterpret_cast(site))) { + continue; + } + + const auto site_instruction = function.instructions.find(site); + ASSERT(site_instruction != function.instructions.end()); + if (function.has_indirect_branch || + site_instruction->second.instruction.length < ShortJumpSize) { + record_unsupported(site); + continue; + } + + const RelocationSpan site_span { + .instructions = {&site_instruction->second}, + .patch_start = site, + .continuation = site + site_instruction->second.instruction.length, + .patch_size = site_instruction->second.instruction.length, + }; + const size_t trampoline_start = module->trampoline_gen.getSize(); + const auto site_trampoline_offset = emit_span(site_span); + if (!site_trampoline_offset) { + record_unsupported(site); + continue; + } + + constexpr s64 ShortJumpMin = std::numeric_limits::min(); + constexpr s64 ShortJumpMax = std::numeric_limits::max(); + const auto relay_slot = std::ranges::find_if(relay_slots, [site](uintptr_t address) { + const s64 displacement = + static_cast(address) - static_cast(site + ShortJumpSize); + return displacement >= ShortJumpMin && displacement <= ShortJumpMax; + }); + if (relay_slot != relay_slots.end()) { + const auto* site_trampoline = + module->trampoline_gen.getCode() + *site_trampoline_offset; + + auto& patch_gen = module->patch_gen; + patch_gen.reset(); + patch_gen.setSize(*relay_slot - reinterpret_cast(patch_gen.getCode())); + patch_gen.jmp(site_trampoline, Xbyak::CodeGenerator::LabelType::T_NEAR); + + patch_gen.reset(); + patch_gen.setSize(site - reinterpret_cast(patch_gen.getCode())); + patch_gen.jmp(reinterpret_cast(*relay_slot), + Xbyak::CodeGenerator::LabelType::T_SHORT); + patch_gen.nop(site_span.patch_size - ShortJumpSize); + + record_rewrites(site_span); + patched_spans.emplace_back(site_span.patch_start, site_span.continuation); + std::erase(short_relay_slots, *relay_slot); + relay_slots.erase(relay_slot); + continue; + } + + const auto find_host = [&](uintptr_t short_jump_address) { + std::pair, std::optional> result; + const s64 minimum_host = static_cast(short_jump_address) + + static_cast(ShortJumpSize) - + static_cast(NearJumpSize) + ShortJumpMin; + const s64 maximum_host = static_cast(short_jump_address) + + static_cast(ShortJumpSize) - + static_cast(NearJumpSize) + ShortJumpMax; + + auto candidate = function.instructions.lower_bound( + static_cast(std::max(minimum_host, 0))); + for (; candidate != function.instructions.end() && + static_cast(candidate->first) <= maximum_host; + ++candidate) { + RelocationSpan span {.patch_start = candidate->first, + .continuation = candidate->first}; + while (span.patch_size < NearJumpSize * 2) { + const auto instruction = function.instructions.find(span.continuation); + if (instruction == function.instructions.end() || + (span.continuation != span.patch_start && + function.branch_targets.contains(span.continuation))) { + span.instructions.clear(); + break; + } + span.instructions.push_back(&instruction->second); + span.patch_size += instruction->second.instruction.length; + span.continuation += instruction->second.instruction.length; + if (span.patch_size < NearJumpSize * 2 && + IsControlFlowTerminator(instruction->second.instruction)) { + span.instructions.clear(); + break; + } + } + if (span.instructions.empty() || + !(span.continuation <= site || + span.patch_start >= site_span.continuation) || + overlaps_patched_span(span.patch_start, span.continuation)) { + continue; + } + + const uintptr_t relay_address = span.patch_start + NearJumpSize; + const s64 displacement = static_cast(relay_address) - + static_cast(short_jump_address + ShortJumpSize); + if (displacement < ShortJumpMin || displacement > ShortJumpMax) { + continue; + } + + const auto offset = emit_span(span); + if (!offset) { + continue; + } + result.first = std::move(span); + result.second = offset; + break; + } + return result; + }; + + auto [host_span, host_trampoline_offset] = find_host(site); + std::optional final_relay_slot; + uintptr_t final_short_jump = site; + std::map relay_parent {{site, site}}; + std::vector relay_queue {site}; + for (size_t queue_index = 0; + !host_span && !final_relay_slot && queue_index < relay_queue.size(); + ++queue_index) { + const uintptr_t current = relay_queue[queue_index]; + if (current != site) { + if (std::ranges::find(relay_slots, current) != relay_slots.end()) { + final_relay_slot = current; + final_short_jump = relay_parent.at(current); + break; + } + const auto near_slot = + std::ranges::find_if(relay_slots, [current](uintptr_t address) { + const s64 displacement = static_cast(address) - + static_cast(current + ShortJumpSize); + return address != current && displacement >= ShortJumpMin && + displacement <= ShortJumpMax; + }); + if (near_slot != relay_slots.end()) { + final_relay_slot = *near_slot; + final_short_jump = current; + break; + } + + auto bridge_host = find_host(current); + if (bridge_host.first) { + host_span = std::move(bridge_host.first); + host_trampoline_offset = bridge_host.second; + final_short_jump = current; + break; + } + } + + for (const uintptr_t slot: short_relay_slots) { + if (relay_parent.contains(slot)) { + continue; + } + const s64 displacement = + static_cast(slot) - static_cast(current + ShortJumpSize); + if (displacement < ShortJumpMin || displacement > ShortJumpMax) { + continue; + } + relay_parent.emplace(slot, current); + relay_queue.push_back(slot); + } + } + + if (!host_span && !final_relay_slot) { + module->trampoline_gen.setSize(trampoline_start); + record_unsupported(site); + continue; + } + + const auto* site_trampoline = + module->trampoline_gen.getCode() + *site_trampoline_offset; + auto& patch_gen = module->patch_gen; + uintptr_t relay_address {}; + if (final_relay_slot) { + relay_address = *final_relay_slot; + patch_gen.reset(); + patch_gen.setSize(relay_address - reinterpret_cast(patch_gen.getCode())); + patch_gen.jmp(site_trampoline, Xbyak::CodeGenerator::LabelType::T_NEAR); + std::erase(relay_slots, relay_address); + std::erase(short_relay_slots, relay_address); + } else { + ASSERT(host_span && host_trampoline_offset); + const auto* host_trampoline = + module->trampoline_gen.getCode() + *host_trampoline_offset; + relay_address = host_span->patch_start + NearJumpSize; + + patch_gen.reset(); + patch_gen.setSize(host_span->patch_start - + reinterpret_cast(patch_gen.getCode())); + patch_gen.jmp(host_trampoline, Xbyak::CodeGenerator::LabelType::T_NEAR); + patch_gen.jmp(site_trampoline, Xbyak::CodeGenerator::LabelType::T_NEAR); + patch_gen.nop(host_span->patch_size - NearJumpSize * 2); + } + + uintptr_t jump_target = relay_address; + while (final_short_jump != site) { + patch_gen.reset(); + patch_gen.setSize(final_short_jump - + reinterpret_cast(patch_gen.getCode())); + patch_gen.jmp(reinterpret_cast(jump_target), + Xbyak::CodeGenerator::LabelType::T_SHORT); + jump_target = final_short_jump; + const auto parent = relay_parent.find(final_short_jump); + ASSERT(parent != relay_parent.end()); + final_short_jump = parent->second; + std::erase(relay_slots, jump_target); + std::erase(short_relay_slots, jump_target); + } + + patch_gen.reset(); + patch_gen.setSize(site - reinterpret_cast(patch_gen.getCode())); + patch_gen.jmp(reinterpret_cast(jump_target), + Xbyak::CodeGenerator::LabelType::T_SHORT); + patch_gen.nop(site_span.patch_size - ShortJumpSize); + + if (host_span) { + record_rewrites(*host_span); + patched_spans.emplace_back(host_span->patch_start, host_span->continuation); + if (host_span->patch_size >= NearJumpSize * 3) { + relay_slots.push_back(host_span->patch_start + NearJumpSize * 2); + } + if (host_span->patch_size >= NearJumpSize * 2 + ShortJumpSize) { + short_relay_slots.push_back(host_span->patch_start + NearJumpSize * 2); + } + } + record_rewrites(site_span); + patched_spans.emplace_back(site_span.patch_start, site_span.continuation); + } +} } // namespace -RedZonePatchResult PatchRedZoneMemoryInstructions(u64 segment_addr, u64 segment_size, - std::span function_starts) { +RedZonePatchResult PatchGuestInstructions(u64 segment_addr, u64 segment_size, + std::span function_starts, + bool protect_memory, bool emulate_rsqrt) { RedZonePatchResult result {}; auto* module = GetContainingModule(reinterpret_cast(segment_addr)); if (module == nullptr || function_starts.empty()) { @@ -762,6 +1266,8 @@ RedZonePatchResult PatchRedZoneMemoryInstructions(u64 segment_addr, u64 segment_ starts.erase(unique_end, starts.end()); std::unique_lock lock {module->mutex}; + const size_t trampoline_begin = module->trampoline_gen.getSize(); + std::vector reciprocal_sqrt_sites; for (size_t function_index = 0; function_index < starts.size(); ++function_index) { const uintptr_t function_start = starts[function_index]; const uintptr_t function_end = @@ -780,461 +1286,36 @@ RedZonePatchResult PatchRedZoneMemoryInstructions(u64 segment_addr, u64 segment_ if (function.uses_red_zone) { ++result.red_zone_function_count; result.indirect_red_zone_function_count += function.has_indirect_branch; - for (const auto& [address, decoded]: function.instructions) { - if (!decoded.accesses_memory || !decoded.red_zone_live.any() || - rewrite_sites.contains(address)) { - continue; - } - ++result.memory_instruction_count; - if (decoded.instruction.length < NearJumpSize) { - ++result.short_memory_instruction_count; - } - if (decoded.instruction.meta.category == ZYDIS_CATEGORY_CALL && - decoded.operands[0].type == ZYDIS_OPERAND_TYPE_MEMORY && - decoded.operands[0].size == sizeof(uintptr_t) * CHAR_BIT && - !IsStackPointerRegister(decoded.operands[0].mem.base) && - !IsStackPointerRegister(decoded.operands[0].mem.index)) { - rewrite_sites[address] = { - .protect_red_zone = true, - .protected_indirect_call = true, - }; - continue; - } - if (decoded.uses_stack_pointer && !decoded.replaces_stack_pointer) { - ++result.stack_dependent_memory_instruction_count; - continue; - } - if (decoded.instruction.meta.category == ZYDIS_CATEGORY_CALL || - IsControlFlowTerminator(decoded.instruction)) { - ++result.control_flow_memory_instruction_count; - continue; - } - rewrite_sites[address].protect_red_zone = true; - } } - if (rewrite_sites.empty()) { - continue; + if (protect_memory) { + CollectRedZoneMemoryInstructions(function, rewrite_sites, result); } - - struct RelocationSpan { - std::vector instructions; - uintptr_t patch_start {}; - uintptr_t continuation {}; - size_t patch_size {}; - }; - - const auto emit_span = [&](const RelocationSpan& span) -> std::optional { - const size_t trampoline_offset = module->trampoline_gen.getSize(); - if (module->trampoline_exhausted) { - return std::nullopt; - } - try { - for (const auto* decoded: span.instructions) { - const auto rewrite = rewrite_sites.find(decoded->address); - const bool protected_indirect_call = - rewrite != rewrite_sites.end() && rewrite->second.protected_indirect_call; - const bool protect_red_zone = rewrite != rewrite_sites.end() && - rewrite->second.protect_red_zone && - !protected_indirect_call; - if (protect_red_zone) { - module->trampoline_gen.lea(rsp, ptr[rsp - GuestRedZoneSize]); - } - if (protected_indirect_call) { - if (!GenerateProtectedIndirectCall(*decoded, module->trampoline_gen)) { - module->trampoline_gen.setSize(trampoline_offset); - return std::nullopt; - } - } else if (!EncodeRelocatedInstruction(*decoded, module->trampoline_gen)) { - module->trampoline_gen.setSize(trampoline_offset); - return std::nullopt; - } - if (protect_red_zone && !decoded->replaces_stack_pointer) { - module->trampoline_gen.lea(rsp, ptr[rsp + GuestRedZoneSize]); - } - } - module->trampoline_gen.jmp(reinterpret_cast(span.continuation)); - } catch (const Xbyak::Error& error) { - module->trampoline_gen.setSize(trampoline_offset); - if (HandleTrampolineError(module, error)) { - return std::nullopt; - } - throw; - } - return trampoline_offset; - }; - - const auto record_rewrites = [&](const RelocationSpan& span) { - for (const auto* decoded: span.instructions) { - const auto rewrite = rewrite_sites.find(decoded->address); - if (rewrite == rewrite_sites.end()) { - continue; - } - if (rewrite->second.protect_red_zone) { - ++result.patched_memory_instruction_count; - } - module->patched.insert(reinterpret_cast(decoded->address)); - } - }; - - std::vector> patched_spans; - std::vector relay_slots; - std::vector short_relay_slots; - std::vector unresolved_sites; - uintptr_t covered_until {}; - for (auto site_it = rewrite_sites.begin(); site_it != rewrite_sites.end(); ++site_it) { - const uintptr_t site = site_it->first; - if (site < covered_until) { - continue; - } - - const auto collect_forward_span = [&]() -> std::optional { - RelocationSpan span {.patch_start = site, .continuation = site}; - while (span.patch_size < NearJumpSize) { - const auto decoded_it = function.instructions.find(span.continuation); - if (decoded_it == function.instructions.end() || - (span.continuation != site && - function.branch_targets.contains(span.continuation))) { - return std::nullopt; - } - - const auto& decoded = decoded_it->second; - span.instructions.push_back(&decoded); - span.patch_size += decoded.instruction.length; - span.continuation += decoded.instruction.length; - if (span.patch_size < NearJumpSize && - IsControlFlowTerminator(decoded.instruction)) { - return std::nullopt; - } - } - return span; - }; - - const auto collect_backward_span = [&]() -> std::optional { - const auto site_instruction = function.instructions.find(site); - ASSERT(site_instruction != function.instructions.end()); - RelocationSpan span { - .instructions = {&site_instruction->second}, - .patch_start = site, - .continuation = site + site_instruction->second.instruction.length, - .patch_size = site_instruction->second.instruction.length, - }; - - while (span.patch_size < NearJumpSize) { - const auto previous_end = function.instructions.lower_bound(span.patch_start); - if (previous_end == function.instructions.begin() || - function.branch_targets.contains(span.patch_start)) { - return std::nullopt; - } - - const auto previous = std::prev(previous_end); - const auto& decoded = previous->second; - if (previous->first + decoded.instruction.length != span.patch_start || - previous->first < covered_until || - IsControlFlowTerminator(decoded.instruction)) { - return std::nullopt; - } - - span.instructions.insert(span.instructions.begin(), &decoded); - span.patch_start = previous->first; - span.patch_size += decoded.instruction.length; - } - return span; - }; - - std::optional selected_span; - std::optional trampoline_offset; - const auto& site_instruction = function.instructions.at(site); - const bool can_relocate_neighbors = !function.has_indirect_branch; - if (can_relocate_neighbors || site_instruction.instruction.length >= NearJumpSize) { - auto forward_span = collect_forward_span(); - if (forward_span) { - trampoline_offset = emit_span(*forward_span); - if (trampoline_offset) { - selected_span = std::move(forward_span); - } - } - } - if (!selected_span && can_relocate_neighbors) { - if (auto backward_span = collect_backward_span()) { - trampoline_offset = emit_span(*backward_span); - if (trampoline_offset) { - selected_span = std::move(backward_span); - } - } - } - if (!selected_span) { - unresolved_sites.push_back(site); - continue; - } - - const auto& span = *selected_span; - const auto* trampoline = module->trampoline_gen.getCode() + *trampoline_offset; - - auto& patch_gen = module->patch_gen; - patch_gen.reset(); - patch_gen.setSize(span.patch_start - reinterpret_cast(patch_gen.getCode())); - patch_gen.jmp(trampoline, Xbyak::CodeGenerator::LabelType::T_NEAR); - patch_gen.nop(span.patch_size - NearJumpSize); - - record_rewrites(span); - patched_spans.emplace_back(span.patch_start, span.continuation); - if (span.patch_size >= NearJumpSize * 2) { - relay_slots.push_back(span.patch_start + NearJumpSize); - } - if (span.patch_size >= NearJumpSize + ShortJumpSize) { - short_relay_slots.push_back(span.patch_start + NearJumpSize); - } - covered_until = span.continuation; + if (emulate_rsqrt) { + CollectReciprocalSquareRoots(function, rewrite_sites, reciprocal_sqrt_sites); } - - const auto record_unsupported = [&](uintptr_t site) { - const auto& rewrite = rewrite_sites.at(site); - if (rewrite.protect_red_zone) { - ++result.unrelocatable_memory_instruction_count; - } - }; - const auto overlaps_patched_span = [&patched_spans](uintptr_t start, uintptr_t end) { - return std::ranges::any_of(patched_spans, [start, end](const auto& patched) { - return start < patched.second && patched.first < end; - }); - }; - - for (const uintptr_t site: unresolved_sites) { - if (module->patched.contains(reinterpret_cast(site))) { - continue; - } - - const auto site_instruction = function.instructions.find(site); - ASSERT(site_instruction != function.instructions.end()); - if (function.has_indirect_branch || - site_instruction->second.instruction.length < ShortJumpSize) { - record_unsupported(site); - continue; - } - - const RelocationSpan site_span { - .instructions = {&site_instruction->second}, - .patch_start = site, - .continuation = site + site_instruction->second.instruction.length, - .patch_size = site_instruction->second.instruction.length, - }; - const size_t trampoline_start = module->trampoline_gen.getSize(); - const auto site_trampoline_offset = emit_span(site_span); - if (!site_trampoline_offset) { - record_unsupported(site); - continue; - } - - constexpr s64 ShortJumpMin = std::numeric_limits::min(); - constexpr s64 ShortJumpMax = std::numeric_limits::max(); - const auto relay_slot = std::ranges::find_if(relay_slots, [site](uintptr_t address) { - const s64 displacement = - static_cast(address) - static_cast(site + ShortJumpSize); - return displacement >= ShortJumpMin && displacement <= ShortJumpMax; - }); - if (relay_slot != relay_slots.end()) { - const auto* site_trampoline = - module->trampoline_gen.getCode() + *site_trampoline_offset; - - auto& patch_gen = module->patch_gen; - patch_gen.reset(); - patch_gen.setSize(*relay_slot - reinterpret_cast(patch_gen.getCode())); - patch_gen.jmp(site_trampoline, Xbyak::CodeGenerator::LabelType::T_NEAR); - - patch_gen.reset(); - patch_gen.setSize(site - reinterpret_cast(patch_gen.getCode())); - patch_gen.jmp(reinterpret_cast(*relay_slot), - Xbyak::CodeGenerator::LabelType::T_SHORT); - patch_gen.nop(site_span.patch_size - ShortJumpSize); - - record_rewrites(site_span); - patched_spans.emplace_back(site_span.patch_start, site_span.continuation); - std::erase(short_relay_slots, *relay_slot); - relay_slots.erase(relay_slot); - continue; - } - - const auto find_host = [&](uintptr_t short_jump_address) { - std::pair, std::optional> result; - const s64 minimum_host = static_cast(short_jump_address) + - static_cast(ShortJumpSize) - - static_cast(NearJumpSize) + ShortJumpMin; - const s64 maximum_host = static_cast(short_jump_address) + - static_cast(ShortJumpSize) - - static_cast(NearJumpSize) + ShortJumpMax; - - auto candidate = function.instructions.lower_bound( - static_cast(std::max(minimum_host, 0))); - for (; candidate != function.instructions.end() && - static_cast(candidate->first) <= maximum_host; - ++candidate) { - RelocationSpan span {.patch_start = candidate->first, - .continuation = candidate->first}; - while (span.patch_size < NearJumpSize * 2) { - const auto instruction = function.instructions.find(span.continuation); - if (instruction == function.instructions.end() || - (span.continuation != span.patch_start && - function.branch_targets.contains(span.continuation))) { - span.instructions.clear(); - break; - } - span.instructions.push_back(&instruction->second); - span.patch_size += instruction->second.instruction.length; - span.continuation += instruction->second.instruction.length; - if (span.patch_size < NearJumpSize * 2 && - IsControlFlowTerminator(instruction->second.instruction)) { - span.instructions.clear(); - break; - } - } - if (span.instructions.empty() || - !(span.continuation <= site || - span.patch_start >= site_span.continuation) || - overlaps_patched_span(span.patch_start, span.continuation)) { - continue; - } - - const uintptr_t relay_address = span.patch_start + NearJumpSize; - const s64 displacement = static_cast(relay_address) - - static_cast(short_jump_address + ShortJumpSize); - if (displacement < ShortJumpMin || displacement > ShortJumpMax) { - continue; - } - - const auto offset = emit_span(span); - if (!offset) { - continue; - } - result.first = std::move(span); - result.second = offset; - break; - } - return result; - }; - - auto [host_span, host_trampoline_offset] = find_host(site); - std::optional final_relay_slot; - uintptr_t final_short_jump = site; - std::map relay_parent {{site, site}}; - std::vector relay_queue {site}; - for (size_t queue_index = 0; - !host_span && !final_relay_slot && queue_index < relay_queue.size(); - ++queue_index) { - const uintptr_t current = relay_queue[queue_index]; - if (current != site) { - if (std::ranges::find(relay_slots, current) != relay_slots.end()) { - final_relay_slot = current; - final_short_jump = relay_parent.at(current); - break; - } - const auto near_slot = - std::ranges::find_if(relay_slots, [current](uintptr_t address) { - const s64 displacement = static_cast(address) - - static_cast(current + ShortJumpSize); - return address != current && displacement >= ShortJumpMin && - displacement <= ShortJumpMax; - }); - if (near_slot != relay_slots.end()) { - final_relay_slot = *near_slot; - final_short_jump = current; - break; - } - - auto bridge_host = find_host(current); - if (bridge_host.first) { - host_span = std::move(bridge_host.first); - host_trampoline_offset = bridge_host.second; - final_short_jump = current; - break; - } - } - - for (const uintptr_t slot: short_relay_slots) { - if (relay_parent.contains(slot)) { - continue; - } - const s64 displacement = - static_cast(slot) - static_cast(current + ShortJumpSize); - if (displacement < ShortJumpMin || displacement > ShortJumpMax) { - continue; - } - relay_parent.emplace(slot, current); - relay_queue.push_back(slot); - } - } - - if (!host_span && !final_relay_slot) { - module->trampoline_gen.setSize(trampoline_start); - record_unsupported(site); - continue; - } - - const auto* site_trampoline = - module->trampoline_gen.getCode() + *site_trampoline_offset; - auto& patch_gen = module->patch_gen; - uintptr_t relay_address {}; - if (final_relay_slot) { - relay_address = *final_relay_slot; - patch_gen.reset(); - patch_gen.setSize(relay_address - reinterpret_cast(patch_gen.getCode())); - patch_gen.jmp(site_trampoline, Xbyak::CodeGenerator::LabelType::T_NEAR); - std::erase(relay_slots, relay_address); - std::erase(short_relay_slots, relay_address); - } else { - ASSERT(host_span && host_trampoline_offset); - const auto* host_trampoline = - module->trampoline_gen.getCode() + *host_trampoline_offset; - relay_address = host_span->patch_start + NearJumpSize; - - patch_gen.reset(); - patch_gen.setSize(host_span->patch_start - - reinterpret_cast(patch_gen.getCode())); - patch_gen.jmp(host_trampoline, Xbyak::CodeGenerator::LabelType::T_NEAR); - patch_gen.jmp(site_trampoline, Xbyak::CodeGenerator::LabelType::T_NEAR); - patch_gen.nop(host_span->patch_size - NearJumpSize * 2); - } - - uintptr_t jump_target = relay_address; - while (final_short_jump != site) { - patch_gen.reset(); - patch_gen.setSize(final_short_jump - - reinterpret_cast(patch_gen.getCode())); - patch_gen.jmp(reinterpret_cast(jump_target), - Xbyak::CodeGenerator::LabelType::T_SHORT); - jump_target = final_short_jump; - const auto parent = relay_parent.find(final_short_jump); - ASSERT(parent != relay_parent.end()); - final_short_jump = parent->second; - std::erase(relay_slots, jump_target); - std::erase(short_relay_slots, jump_target); - } - - patch_gen.reset(); - patch_gen.setSize(site - reinterpret_cast(patch_gen.getCode())); - patch_gen.jmp(reinterpret_cast(jump_target), - Xbyak::CodeGenerator::LabelType::T_SHORT); - patch_gen.nop(site_span.patch_size - ShortJumpSize); - - if (host_span) { - record_rewrites(*host_span); - patched_spans.emplace_back(host_span->patch_start, host_span->continuation); - if (host_span->patch_size >= NearJumpSize * 3) { - relay_slots.push_back(host_span->patch_start + NearJumpSize * 2); - } - if (host_span->patch_size >= NearJumpSize * 2 + ShortJumpSize) { - short_relay_slots.push_back(host_span->patch_start + NearJumpSize * 2); - } - } - record_rewrites(site_span); - patched_spans.emplace_back(site_span.patch_start, site_span.continuation); + if (!rewrite_sites.empty()) { + RelocateRedZoneInstructions(module, function, rewrite_sites, result); } } + // Preserve valid instruction encodings throughout CFG analysis and relocation. + // Only now turn reciprocal roots into traps, including the relocated copies. + const auto trampoline_addr = + reinterpret_cast(module->trampoline_gen.getCode()) + trampoline_begin; + const auto trampoline_size = module->trampoline_gen.getSize() - trampoline_begin; + if (emulate_rsqrt) { + result.reciprocal_sqrt_instruction_count = ApplyReciprocalSquareRootPatches( + *module, reciprocal_sqrt_sites, trampoline_addr, trampoline_size); + } + Common::VirtualMemory::FlushInstructionCache(segment_addr, segment_size); + if (trampoline_size != 0) { + Common::VirtualMemory::FlushInstructionCache(trampoline_addr, trampoline_size); + } return result; } #else -RedZonePatchResult PatchRedZoneMemoryInstructions(u64, u64, std::span) { +RedZonePatchResult PatchGuestInstructions(u64, u64, std::span, bool, bool) { return {}; } diff --git a/src/loader/redZonePatcher.h b/src/loader/redZonePatcher.h index a9d2a24f6..6bc7fc589 100644 --- a/src/loader/redZonePatcher.h +++ b/src/loader/redZonePatcher.h @@ -14,7 +14,7 @@ namespace Loader { struct RedZonePatchResult { uint64_t function_count = 0; uint64_t instruction_count = 0; - uint64_t red_zone_function_count = 0; + uint64_t red_zone_function_count = 0; uint64_t memory_instruction_count = 0; uint64_t short_memory_instruction_count = 0; uint64_t patched_memory_instruction_count = 0; @@ -22,14 +22,17 @@ struct RedZonePatchResult { uint64_t control_flow_memory_instruction_count = 0; uint64_t unrelocatable_memory_instruction_count = 0; uint64_t indirect_red_zone_function_count = 0; + uint64_t reciprocal_sqrt_instruction_count = 0; }; void RegisterRedZonePatchModule(void* module_ptr, uint64_t module_size, void* trampoline_area_ptr, uint64_t trampoline_area_size); void UnregisterRedZonePatchModule(void* module_ptr); -RedZonePatchResult PatchRedZoneMemoryInstructions(uint64_t segment_addr, uint64_t segment_size, - std::span function_starts); +// Analyze and protect fault sites before replacing reciprocal roots with traps. +RedZonePatchResult PatchGuestInstructions(uint64_t segment_addr, uint64_t segment_size, + std::span function_starts, + bool protect_memory, bool emulate_rsqrt); bool DecodeEhFrameFunctionStarts(uint64_t eh_frame_header_addr, uint64_t eh_frame_header_size, std::vector* function_starts); diff --git a/src/loader/runtimeLinker.cpp b/src/loader/runtimeLinker.cpp index 621e4a003..03e4d4b81 100644 --- a/src/loader/runtimeLinker.cpp +++ b/src/loader/runtimeLinker.cpp @@ -2014,9 +2014,11 @@ void RuntimeLinker::LoadProgramToMemory(Program* program) { uint64_t tls_handler_size = is_shared ? 0 : Jit::SafeCall::GetSize(); EXIT_IF(tls_handler_size > UINT64_MAX - program->base_size_aligned); program->mapped_size = program->base_size_aligned + tls_handler_size; + const bool emulate_rsqrt = Config::AmdCpuEnabled(); #if KYTY_PLATFORM == KYTY_PLATFORM_WINDOWS - const bool use_red_zone_protection = Config::RedZoneProtectionEnabled(); + const bool protect_memory_faults = Config::RedZoneProtectionEnabled(); + const bool use_red_zone_protection = protect_memory_faults || emulate_rsqrt; constexpr uint64_t RED_ZONE_TRAMPOLINE_SIZE = 8u * 1024u * 1024u; if (use_red_zone_protection) { EXIT_IF(RED_ZONE_TRAMPOLINE_SIZE > UINT64_MAX - program->mapped_size); @@ -2062,9 +2064,8 @@ void RuntimeLinker::LoadProgramToMemory(Program* program) { EXIT("Failed to install the required vectored exception handler\n"); } - // program->elf->SetBaseVAddr(program->base_vaddr); -#if KYTY_PLATFORM == KYTY_PLATFORM_WINDOWS std::vector> executable_segments; +#if KYTY_PLATFORM == KYTY_PLATFORM_WINDOWS uint64_t eh_frame_header_addr = 0; uint64_t eh_frame_header_size = 0; #endif @@ -2089,22 +2090,8 @@ void RuntimeLinker::LoadProgramToMemory(Program* program) { mode == Common::VirtualMemory::Mode::NoAccess); if (Common::VirtualMemory::IsExecute(mode)) { - if (Config::AmdCpuEnabled()) { - const auto reciprocal_sqrt_count = - X64InstructionEmulator::PatchReciprocalSquareRoots(segment_addr, - segment_file_size); - if (reciprocal_sqrt_count != 0) { - LOGF("Guest VRSQRTPS emulation: %s, instructions=%" PRIu64 "\n", - Common::PathToString(program->file_name.filename()).c_str(), - reciprocal_sqrt_count); - } - } PatchProgram(program, segment_addr, segment_memory_size); -#if KYTY_PLATFORM == KYTY_PLATFORM_WINDOWS - if (use_red_zone_protection) { - executable_segments.emplace_back(segment_addr, segment_file_size); - } -#endif + executable_segments.emplace_back(segment_addr, segment_file_size); } if (!skip_protect) { @@ -2149,16 +2136,22 @@ void RuntimeLinker::LoadProgramToMemory(Program* program) { } #if KYTY_PLATFORM == KYTY_PLATFORM_WINDOWS + std::vector function_starts; if (use_red_zone_protection) { - std::vector function_starts; if (!DecodeEhFrameFunctionStarts(eh_frame_header_addr, eh_frame_header_size, &function_starts)) { LOGF("Windows guest red-zone patching could not decode function boundaries for %s\n", Common::PathToString(program->file_name).c_str()); } - for (const auto& [segment_addr, segment_size]: executable_segments) { + } +#endif + for (const auto& [segment_addr, segment_size]: executable_segments) { + uint64_t reciprocal_sqrt_count = 0; +#if KYTY_PLATFORM == KYTY_PLATFORM_WINDOWS + if (use_red_zone_protection) { const auto result = - PatchRedZoneMemoryInstructions(segment_addr, segment_size, function_starts); + PatchGuestInstructions(segment_addr, segment_size, function_starts, + protect_memory_faults, emulate_rsqrt); LOGF("Windows guest red-zone patching: %s, functions=%" PRIu64 ", red_zone=%" PRIu64 ", memory=%" PRIu64 ", patched=%" PRIu64 ", short=%" PRIu64 ", stack=%" PRIu64 ", control=%" PRIu64 ", unrelocatable=%" PRIu64 "\n", @@ -2168,12 +2161,20 @@ void RuntimeLinker::LoadProgramToMemory(Program* program) { result.stack_dependent_memory_instruction_count, result.control_flow_memory_instruction_count, result.unrelocatable_memory_instruction_count); + reciprocal_sqrt_count = result.reciprocal_sqrt_instruction_count; + } +#else + if (emulate_rsqrt) { + reciprocal_sqrt_count = + X64InstructionEmulator::PatchReciprocalSquareRoots(segment_addr, segment_size); Common::VirtualMemory::FlushInstructionCache(segment_addr, segment_size); } - Common::VirtualMemory::FlushInstructionCache(program->red_zone_trampoline_vaddr, - program->red_zone_trampoline_size); - } #endif + if (reciprocal_sqrt_count != 0) { + LOGF("Guest VRSQRTPS emulation: %s, instructions=%" PRIu64 "\n", + Common::PathToString(program->file_name.filename()).c_str(), reciprocal_sqrt_count); + } + } if (!is_shared) { SetupTlsHandler(program); diff --git a/src/loader/x64InstructionEmulator.cpp b/src/loader/x64InstructionEmulator.cpp index 1ec9f4b49..fd762be4b 100644 --- a/src/loader/x64InstructionEmulator.cpp +++ b/src/loader/x64InstructionEmulator.cpp @@ -5,7 +5,7 @@ #include #include #include -#if KYTY_PLATFORM == KYTY_PLATFORM_LINUX && !defined(__APPLE__) +#if !defined(__APPLE__) #include #include #endif @@ -424,6 +424,21 @@ struct Context { native->R12, native->R13, native->R14, native->R15}; std::memcpy(gpr, registers, sizeof(gpr)); } + + void ClearUpperYmm(uint8_t index) const { + if ((native->ContextFlags & CONTEXT_XSTATE) != CONTEXT_XSTATE) { + return; + } + DWORD64 features = 0; + if (!GetXStateFeaturesMask(native, &features) || (features & XSTATE_MASK_AVX) == 0) { + return; // An absent AVX component restores zeroes. + } + DWORD size = 0; + auto* ymm = static_cast(LocateXStateFeature(native, XSTATE_AVX, &size)); + if (ymm != nullptr && size >= (index + 1u) * sizeof(M128A)) { + ymm[index] = {}; + } + } #else ucontext_t* native; @@ -588,8 +603,6 @@ static bool TryEmulateMonitorxMwaitx(Context& context) { return true; } -#if KYTY_PLATFORM == KYTY_PLATFORM_LINUX - static uint32_t ReciprocalSquareRoot(uint32_t bits) { const uint32_t magnitude = bits & 0x7fffffffu; const uint32_t exponent = magnitude & 0x7f800000u; @@ -658,11 +671,18 @@ static bool TryEmulateReciprocalSquareRoot(Context& context) { } #endif -#endif + +bool IsReciprocalSquareRoot(const ZydisDecodedInstruction& instruction, + const ZydisDecodedOperand* operands) { + return instruction.mnemonic == ZYDIS_MNEMONIC_VRSQRTPS && + instruction.encoding == ZYDIS_INSTRUCTION_ENCODING_VEX && + instruction.raw.vex.offset == 0 && operands[0].size == 128 && + operands[1].type == ZYDIS_OPERAND_TYPE_REGISTER; +} uint64_t PatchReciprocalSquareRoots(uint64_t address, uint64_t size) { uint64_t patched = 0; -#if KYTY_PLATFORM == KYTY_PLATFORM_LINUX && !defined(__APPLE__) +#if !defined(__APPLE__) ZydisDecoder decoder {}; if (!ZYAN_SUCCESS( ZydisDecoderInit(&decoder, ZYDIS_MACHINE_MODE_LONG_64, ZYDIS_STACK_WIDTH_64))) { @@ -677,12 +697,9 @@ uint64_t PatchReciprocalSquareRoots(uint64_t address, uint64_t size) { ++offset; continue; } - if (instruction.mnemonic == ZYDIS_MNEMONIC_VRSQRTPS && - instruction.encoding == ZYDIS_INSTRUCTION_ENCODING_VEX && - instruction.raw.vex.offset == 0 && operands[0].size == 128 && - operands[1].type == ZYDIS_OPERAND_TYPE_REGISTER) { + if (IsReciprocalSquareRoot(instruction, operands)) { // vvvv is reserved (must be 1111b). Clear one bit to route this - // otherwise intact instruction through the existing SIGILL emulator. + // otherwise intact instruction through the illegal-instruction emulator. code[instruction.raw.vex.size - 1] &= ~0x08u; ++patched; } @@ -704,10 +721,10 @@ bool TryEmulate(void* native_context) { Context context {static_cast(native_context)}; #else Context context {static_cast(native_context)}; +#endif if (TryEmulateReciprocalSquareRoot(context)) { return true; } -#endif return TryEmulateMonitorxMwaitx(context) || TryEmulateSse4a(context) || TryEmulateShaNi(context); #else diff --git a/src/loader/x64InstructionEmulator.h b/src/loader/x64InstructionEmulator.h index e826707ed..b8149c1f8 100644 --- a/src/loader/x64InstructionEmulator.h +++ b/src/loader/x64InstructionEmulator.h @@ -1,10 +1,13 @@ #ifndef KYTY_LOADER_X64_INSTRUCTION_EMULATOR_H_ #define KYTY_LOADER_X64_INSTRUCTION_EMULATOR_H_ +#include #include namespace Loader::X64InstructionEmulator { +[[nodiscard]] bool IsReciprocalSquareRoot(const ZydisDecodedInstruction& instruction, + const ZydisDecodedOperand* operands); uint64_t PatchReciprocalSquareRoots(uint64_t address, uint64_t size); [[nodiscard]] bool TryEmulate(void* native_context); diff --git a/src/main.cpp b/src/main.cpp index c5ddc5256..40222b9a1 100644 --- a/src/main.cpp +++ b/src/main.cpp @@ -56,7 +56,7 @@ static void PrintUsage() { " --gpu Vulkan physical device index. Default: auto.\n"); ::printf(" --fullscreen Run in borderless desktop fullscreen.\n"); ::printf(" --vr Enable the virtual VR headset.\n"); - ::printf(" --amd-cpu Apply AMD CPU instruction patches (Linux).\n"); + ::printf(" --amd-cpu Apply AMD CPU instruction patches.\n"); ::printf(" --vblank-frequency Virtual vblank frequency. Default: 60.\n"); ::printf(" --console-language <0-29> Console language. Default: 1 (English US).\n"); ::printf(" --vulkan-validation Enable Vulkan validation.\n"); diff --git a/tests/VirtualMemoryAllocationTests.cpp b/tests/VirtualMemoryAllocationTests.cpp index 88f91b5b2..debc0f0c9 100644 --- a/tests/VirtualMemoryAllocationTests.cpp +++ b/tests/VirtualMemoryAllocationTests.cpp @@ -24,13 +24,16 @@ #include #include -#if defined(__linux__) +#if defined(__linux__) || KYTY_PLATFORM == KYTY_PLATFORM_WINDOWS #include #include +#include +#endif + +#if defined(__linux__) #include #include #include -#include #endif #if KYTY_PLATFORM == KYTY_PLATFORM_WINDOWS @@ -217,10 +220,8 @@ void TestWindowsGuestRedZoneStaticPatcher() { reinterpret_cast(mapping + CODE_SIZE), TRAMPOLINE_SIZE); const std::array function_starts = {static_cast(mapping)}; - const auto result = Loader::PatchRedZoneMemoryInstructions( - mapping, code.size(), function_starts); - Check(test, Common::VirtualMemory::FlushInstructionCache(mapping, CODE_SIZE + TRAMPOLINE_SIZE), - "failed to flush patched test code"); + const auto result = Loader::PatchGuestInstructions( + mapping, code.size(), function_starts, true, false); const bool patched_preserved = function(static_cast(g_red_zone_fault_page)) == 1; Loader::UnregisterRedZonePatchModule(reinterpret_cast(mapping)); @@ -2546,55 +2547,93 @@ void TestModuleRelocationUsesWritableHostMapping() { std::printf("[host] %-48s ok\n", test); } -#if defined(__linux__) +#if defined(__linux__) || KYTY_PLATFORM == KYTY_PLATFORM_WINDOWS volatile sig_atomic_t g_rsqrt_traps = 0; -void ReciprocalSquareRootHandler(int, siginfo_t*, void* context) { +bool EmulateReciprocalSquareRootContext(void* context) { const auto host_mxcsr = _mm_getcsr(); _mm_setcsr(0x7fe1); // Exercise the handler under a distinct rounding mode and raised flags. - if (!Loader::X64InstructionEmulator::TryEmulate(context)) { + const bool emulated = Loader::X64InstructionEmulator::TryEmulate(context); + const bool preserved_mxcsr = _mm_getcsr() == 0x7fe1; + _mm_setcsr(host_mxcsr); + if (!emulated || !preserved_mxcsr) { + return false; + } + g_rsqrt_traps = g_rsqrt_traps + 1; + return true; +} + +#if KYTY_PLATFORM == KYTY_PLATFORM_WINDOWS +LONG CALLBACK ReciprocalSquareRootHandler(EXCEPTION_POINTERS* exception) { + if (exception->ExceptionRecord->ExceptionCode != EXCEPTION_ILLEGAL_INSTRUCTION || + !EmulateReciprocalSquareRootContext(exception->ContextRecord)) { + return EXCEPTION_CONTINUE_SEARCH; + } + return EXCEPTION_CONTINUE_EXECUTION; +} +#else +void ReciprocalSquareRootHandler(int, siginfo_t*, void* context) { + if (!EmulateReciprocalSquareRootContext(context)) { _exit(190); } - if (_mm_getcsr() != 0x7fe1) { - _exit(191); - } - _mm_setcsr(host_mxcsr); - g_rsqrt_traps = g_rsqrt_traps + 1; } +#endif void TestPackedReciprocalSquareRoot() { const char* test = "PackedReciprocalSquareRoot"; constexpr uint64_t code_size = 0x4000; +#if KYTY_PLATFORM == KYTY_PLATFORM_WINDOWS + constexpr uint64_t allocation_size = code_size * 2; +#else + constexpr uint64_t allocation_size = code_size; +#endif const auto mapping = Libs::LibKernel::Memory::AllocateProgramMemory( - 0x902000000, code_size, Common::VirtualMemory::Mode::ExecuteReadWrite, "rsqrt_test"); + 0x902000000, allocation_size, Common::VirtualMemory::Mode::ExecuteReadWrite, "rsqrt_test"); Check(test, mapping != 0, "failed to allocate instruction test code"); struct RestoreState { uint64_t mapping; uint64_t size; uint32_t mxcsr; +#if KYTY_PLATFORM == KYTY_PLATFORM_WINDOWS + void* handler = nullptr; +#else struct sigaction previous {}; bool installed = false; +#endif ~RestoreState() { _mm_setcsr(mxcsr); +#if KYTY_PLATFORM == KYTY_PLATFORM_WINDOWS + if (handler != nullptr) { + RemoveVectoredExceptionHandler(handler); + } + Loader::UnregisterRedZonePatchModule(reinterpret_cast(mapping)); +#else if (installed) { sigaction(SIGILL, &previous, nullptr); } +#endif Libs::LibKernel::Memory::FreeGuestMemory(mapping, size); } - } restore {mapping, code_size, _mm_getcsr()}; + } restore {mapping, allocation_size, _mm_getcsr()}; +#if KYTY_PLATFORM == KYTY_PLATFORM_WINDOWS + restore.handler = AddVectoredExceptionHandler(1, ReciprocalSquareRootHandler); + Check(test, restore.handler != nullptr, "failed to install instruction handler"); +#else struct sigaction action {}; action.sa_sigaction = ReciprocalSquareRootHandler; action.sa_flags = SA_SIGINFO; sigemptyset(&action.sa_mask); restore.installed = sigaction(SIGILL, &action, &restore.previous) == 0; Check(test, restore.installed, "failed to install instruction handler"); +#endif using GuestFunction = void(KYTY_SYSV_ABI*)(const uint32_t*, uint32_t*); const auto function = reinterpret_cast(mapping); const auto refine = [](float estimate) { return (estimate * 0.5f) * std::fma(-estimate, estimate, 3.0f); }; + constexpr uint64_t red_zone_sentinel = 0x1122334455667788ull; std::array input {}; - std::array output {}; + std::array output {}; input.fill(0x3f800000); for (const auto registers: {std::array {2, 1}, std::array {10, 9}, std::array {1, 1}}) { const auto destination = registers[0]; @@ -2613,16 +2652,55 @@ void TestPackedReciprocalSquareRoot() { if (destination != source) { code.vmovups(Xbyak::Ymm(destination), code.ptr[code.rdi + 32]); } + for (uint32_t offset = 8; offset <= 128; offset += 8) { + code.mov(code.rax, red_zone_sentinel ^ offset); + code.mov(code.qword[code.rsp - offset], code.rax); + } code.vrsqrtps(Xbyak::Xmm(destination), Xbyak::Xmm(source)); + for (uint32_t offset = 8; offset <= 128; offset += 8) { + code.mov(code.rax, code.qword[code.rsp - offset]); + code.mov(code.qword[code.rsi + 64 + offset - 8], code.rax); + } code.vmovups(code.ptr[code.rsi], Xbyak::Ymm(destination)); code.vmovups(code.ptr[code.rsi + 32], Xbyak::Ymm(source)); code.vzeroupper(); code.ret(); +#if defined(__linux__) const std::vector original(code.getCode(), code.getCurr()); +#endif + Check(test, Common::VirtualMemory::FlushInstructionCache(mapping, code.getSize()), + "failed to flush generated instruction test code"); function(input.data(), output.data()); const float native = std::bit_cast(output[0]); + const auto red_zone_intact = [&] { + for (uint32_t offset = 8; offset <= 128; offset += 8) { + const auto index = 16 + (offset - 8) / sizeof(uint32_t); + const auto value = static_cast(output[index]) | + (static_cast(output[index + 1]) << 32); + if (value != (red_zone_sentinel ^ offset)) { + return false; + } + } + return true; + }; + Check(test, red_zone_intact(), "native instruction corrupted the guest red zone"); Check(test, std::isfinite(native) && std::abs(native - std::bit_cast(expected)) < 0.001f, "native reciprocal root is outside its error bound"); +#if KYTY_PLATFORM == KYTY_PLATFORM_WINDOWS + Loader::RegisterRedZonePatchModule(reinterpret_cast(mapping), code_size, + reinterpret_cast(mapping + code_size), code_size); + const std::array function_starts {mapping}; + // The extended-register case also relocates ordinary memory accesses while + // red-zone data is live, exercising both enabled patchers in one function. + const bool protect_memory = source == 9; + const auto patched = Loader::PatchGuestInstructions( + mapping, code.getSize(), function_starts, protect_memory, true); + Check(test, patched.reciprocal_sqrt_instruction_count == 1 && + patched.unrelocatable_memory_instruction_count == 0 && + (protect_memory ? patched.patched_memory_instruction_count > 0 + : patched.patched_memory_instruction_count == 0), + "instruction pass lost reciprocal root or memory patch coverage"); +#else Check(test, Loader::X64InstructionEmulator::PatchReciprocalSquareRoots(mapping, code.getSize()) == 1, "instruction pass did not patch exactly one packed reciprocal root"); const auto* patched = reinterpret_cast(mapping); @@ -2631,11 +2709,18 @@ void TestPackedReciprocalSquareRoot() { changed += original[i] != patched[i]; } Check(test, changed == 1, "instruction pass changed unrelated code bytes"); - Common::VirtualMemory::FlushInstructionCache(mapping, code.getSize()); + Check(test, Common::VirtualMemory::FlushInstructionCache(mapping, code.getSize()), + "failed to flush patched instruction test code"); +#endif const auto before = g_rsqrt_traps; function(input.data(), output.data()); Check(test, g_rsqrt_traps == before + 1, "patched instruction did not execute its handler"); + Check(test, red_zone_intact(), "patched instruction corrupted the guest red zone"); Check(test, output[0] == expected, "patched instruction read or wrote the wrong register"); + if (destination != source) { + Check(test, std::equal(input.begin(), input.begin() + 4, output.begin() + 8), + "source lower XMM lanes were corrupted"); + } if (source != 9) { Check(test, refine(refine(std::bit_cast(output[0]))) == 1.0f, "identity quaternion normalization drifted below one"); @@ -2668,6 +2753,7 @@ void TestPackedReciprocalSquareRoot() { function(input.data(), output.data()); const auto result_mxcsr = _mm_getcsr(); _mm_setcsr(restore.mxcsr); + Check(test, red_zone_intact(), "special-value trap corrupted the guest red zone"); Check(test, result_mxcsr == mxcsr, "instruction changed MXCSR controls or exception flags"); Check(test, std::equal(output.begin(), output.begin() + 4, values.begin() + 4), "special values or round-independent reciprocal roots differ from ISA semantics"); @@ -2675,6 +2761,7 @@ void TestPackedReciprocalSquareRoot() { } input.fill(0x3f800000); } +#if defined(__linux__) std::array unknown {0x0f, 0x0b}; ucontext_t context {}; context.uc_mcontext.gregs[REG_RIP] = reinterpret_cast(unknown.data()); @@ -2708,6 +2795,7 @@ void TestPackedReciprocalSquareRoot() { Check(test, std::equal(sha_expected.begin(), sha_expected.end(), fpstate._xmm[8].element), "SHA emulation lost its memory operand or extended destination register"); emulate({0x0f, 0x01, 0xfa}, 3); // monitorx +#endif std::printf("[host] %-48s ok\n", test); } #endif @@ -2716,7 +2804,7 @@ void TestPackedReciprocalSquareRoot() { int main(int argc, char** argv) { InitSubsystems(); -#if defined(__linux__) +#if defined(__linux__) || KYTY_PLATFORM == KYTY_PLATFORM_WINDOWS if (argc == 2 && std::strcmp(argv[1], "--rsqrt-only") == 0) { RunTest(TestPackedReciprocalSquareRoot); return g_failed_tests == 0 ? 0 : 1; @@ -2727,7 +2815,7 @@ int main(int argc, char** argv) { return g_failed_tests == 0 ? 0 : 1; } -#if defined(__linux__) +#if defined(__linux__) || KYTY_PLATFORM == KYTY_PLATFORM_WINDOWS RunTest(TestPackedReciprocalSquareRoot); #endif RunTest(TestWindowsGuestRedZoneStaticPatcher);