mirror of
https://github.com/KytyPS5/KytyPS5.git
synced 2026-10-28 22:06:59 +01:00
Protect Windows guest red zones when emulating VRSQRTPS
This commit is contained in:
@@ -240,7 +240,7 @@ void ConfigurationEditDialog::Init(const Configuration& info) {
|
|||||||
m_ui->checkBox_vulkan_validation->setChecked(info.vulkan_validation_enabled);
|
m_ui->checkBox_vulkan_validation->setChecked(info.vulkan_validation_enabled);
|
||||||
m_ui->checkBox_renderdoc_capture->setChecked(info.renderdoc_enabled);
|
m_ui->checkBox_renderdoc_capture->setChecked(info.renderdoc_enabled);
|
||||||
m_ui->checkBox_amd_cpu->setChecked(info.amd_cpu_enabled);
|
m_ui->checkBox_amd_cpu->setChecked(info.amd_cpu_enabled);
|
||||||
#if !defined(__linux__)
|
#if defined(__APPLE__)
|
||||||
m_ui->checkBox_amd_cpu->setVisible(false);
|
m_ui->checkBox_amd_cpu->setVisible(false);
|
||||||
#endif
|
#endif
|
||||||
#if defined(_WIN32)
|
#if defined(_WIN32)
|
||||||
|
|||||||
+125
-44
@@ -5,6 +5,8 @@
|
|||||||
|
|
||||||
#include "common/assert.h"
|
#include "common/assert.h"
|
||||||
#include "common/logging/log.h"
|
#include "common/logging/log.h"
|
||||||
|
#include "common/virtualMemory.h"
|
||||||
|
#include "loader/x64InstructionEmulator.h"
|
||||||
|
|
||||||
#include <Zydis/Zydis.h>
|
#include <Zydis/Zydis.h>
|
||||||
#include <algorithm>
|
#include <algorithm>
|
||||||
@@ -739,47 +741,13 @@ bool GenerateProtectedIndirectCall(const DecodedCodeInstruction& decoded,
|
|||||||
generator.call(ptr[rsp - GuestRedZoneSize - sizeof(uintptr_t)]);
|
generator.call(ptr[rsp - GuestRedZoneSize - sizeof(uintptr_t)]);
|
||||||
return true;
|
return true;
|
||||||
}
|
}
|
||||||
} // namespace
|
|
||||||
|
|
||||||
RedZonePatchResult PatchRedZoneMemoryInstructions(u64 segment_addr, u64 segment_size,
|
void CollectRedZoneMemoryInstructions(const DecodedFunction& function,
|
||||||
std::span<const uintptr_t> function_starts) {
|
std::map<uintptr_t, InstructionRewrite>& rewrite_sites,
|
||||||
RedZonePatchResult result {};
|
RedZonePatchResult& result) {
|
||||||
auto* module = GetContainingModule(reinterpret_cast<void*>(segment_addr));
|
if (!function.uses_red_zone) {
|
||||||
if (module == nullptr || function_starts.empty()) {
|
return;
|
||||||
return result;
|
|
||||||
}
|
}
|
||||||
|
|
||||||
const uintptr_t segment_end = segment_addr + segment_size;
|
|
||||||
std::vector<uintptr_t> starts;
|
|
||||||
starts.reserve(function_starts.size());
|
|
||||||
for (const uintptr_t start: function_starts) {
|
|
||||||
if (start >= segment_addr && start < segment_end) {
|
|
||||||
starts.push_back(start);
|
|
||||||
}
|
|
||||||
}
|
|
||||||
std::ranges::sort(starts);
|
|
||||||
const auto unique_end = std::ranges::unique(starts).begin();
|
|
||||||
starts.erase(unique_end, starts.end());
|
|
||||||
|
|
||||||
std::unique_lock lock {module->mutex};
|
|
||||||
for (size_t function_index = 0; function_index < starts.size(); ++function_index) {
|
|
||||||
const uintptr_t function_start = starts[function_index];
|
|
||||||
const uintptr_t function_end =
|
|
||||||
function_index + 1 < starts.size() ? starts[function_index + 1] : segment_end;
|
|
||||||
if (function_end <= function_start) {
|
|
||||||
continue;
|
|
||||||
}
|
|
||||||
|
|
||||||
++result.function_count;
|
|
||||||
auto function = DecodeFunction(function_start, function_end, segment_addr, segment_end);
|
|
||||||
AnalyzeRedZoneLiveness(function);
|
|
||||||
result.instruction_count += function.instructions.size();
|
|
||||||
|
|
||||||
std::map<uintptr_t, InstructionRewrite> rewrite_sites;
|
|
||||||
|
|
||||||
if (function.uses_red_zone) {
|
|
||||||
++result.red_zone_function_count;
|
|
||||||
result.indirect_red_zone_function_count += function.has_indirect_branch;
|
|
||||||
for (const auto& [address, decoded]: function.instructions) {
|
for (const auto& [address, decoded]: function.instructions) {
|
||||||
if (!decoded.accesses_memory || !decoded.red_zone_live.any() ||
|
if (!decoded.accesses_memory || !decoded.red_zone_live.any() ||
|
||||||
rewrite_sites.contains(address)) {
|
rewrite_sites.contains(address)) {
|
||||||
@@ -812,10 +780,54 @@ RedZonePatchResult PatchRedZoneMemoryInstructions(u64 segment_addr, u64 segment_
|
|||||||
rewrite_sites[address].protect_red_zone = true;
|
rewrite_sites[address].protect_red_zone = true;
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
if (rewrite_sites.empty()) {
|
|
||||||
|
struct ReciprocalSquareRootSite {
|
||||||
|
uintptr_t address;
|
||||||
|
u8 length;
|
||||||
|
bool requires_red_zone_protection;
|
||||||
|
};
|
||||||
|
|
||||||
|
void CollectReciprocalSquareRoots(const DecodedFunction& function,
|
||||||
|
std::map<uintptr_t, InstructionRewrite>& rewrite_sites,
|
||||||
|
std::vector<ReciprocalSquareRootSite>& sites) {
|
||||||
|
for (const auto& [address, decoded]: function.instructions) {
|
||||||
|
if (!X64InstructionEmulator::IsReciprocalSquareRoot(decoded.instruction,
|
||||||
|
decoded.operands.data())) {
|
||||||
continue;
|
continue;
|
||||||
}
|
}
|
||||||
|
const bool protect_red_zone = decoded.red_zone_live.any();
|
||||||
|
sites.push_back({address, decoded.instruction.length, protect_red_zone});
|
||||||
|
if (protect_red_zone) {
|
||||||
|
rewrite_sites[address].protect_red_zone = true;
|
||||||
|
}
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
uint64_t ApplyReciprocalSquareRootPatches(const PatchModule& module,
|
||||||
|
std::span<const ReciprocalSquareRootSite> sites,
|
||||||
|
uint64_t trampoline_addr, uint64_t trampoline_size) {
|
||||||
|
// Validate every required relocation before introducing any traps.
|
||||||
|
for (const auto& site: sites) {
|
||||||
|
if (site.requires_red_zone_protection &&
|
||||||
|
!module.patched.contains(reinterpret_cast<u8*>(site.address))) {
|
||||||
|
EXIT("Cannot preserve the guest red zone at emulated VRSQRTPS 0x%016" PRIx64 "\n",
|
||||||
|
static_cast<u64>(site.address));
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
uint64_t patched = 0;
|
||||||
|
for (const auto& site: sites) {
|
||||||
|
if (!module.patched.contains(reinterpret_cast<u8*>(site.address))) {
|
||||||
|
patched += X64InstructionEmulator::PatchReciprocalSquareRoots(site.address, site.length);
|
||||||
|
}
|
||||||
|
}
|
||||||
|
return patched +
|
||||||
|
X64InstructionEmulator::PatchReciprocalSquareRoots(trampoline_addr, trampoline_size);
|
||||||
|
}
|
||||||
|
|
||||||
|
void RelocateRedZoneInstructions(PatchModule* module, const DecodedFunction& function,
|
||||||
|
const std::map<uintptr_t, InstructionRewrite>& rewrite_sites,
|
||||||
|
RedZonePatchResult& result) {
|
||||||
struct RelocationSpan {
|
struct RelocationSpan {
|
||||||
std::vector<const DecodedCodeInstruction*> instructions;
|
std::vector<const DecodedCodeInstruction*> instructions;
|
||||||
uintptr_t patch_start {};
|
uintptr_t patch_start {};
|
||||||
@@ -865,14 +877,14 @@ RedZonePatchResult PatchRedZoneMemoryInstructions(u64 segment_addr, u64 segment_
|
|||||||
|
|
||||||
const auto record_rewrites = [&](const RelocationSpan& span) {
|
const auto record_rewrites = [&](const RelocationSpan& span) {
|
||||||
for (const auto* decoded: span.instructions) {
|
for (const auto* decoded: span.instructions) {
|
||||||
|
module->patched.insert(reinterpret_cast<u8*>(decoded->address));
|
||||||
const auto rewrite = rewrite_sites.find(decoded->address);
|
const auto rewrite = rewrite_sites.find(decoded->address);
|
||||||
if (rewrite == rewrite_sites.end()) {
|
if (rewrite == rewrite_sites.end()) {
|
||||||
continue;
|
continue;
|
||||||
}
|
}
|
||||||
if (rewrite->second.protect_red_zone) {
|
if (rewrite->second.protect_red_zone && decoded->accesses_memory) {
|
||||||
++result.patched_memory_instruction_count;
|
++result.patched_memory_instruction_count;
|
||||||
}
|
}
|
||||||
module->patched.insert(reinterpret_cast<u8*>(decoded->address));
|
|
||||||
}
|
}
|
||||||
};
|
};
|
||||||
|
|
||||||
@@ -988,8 +1000,9 @@ RedZonePatchResult PatchRedZoneMemoryInstructions(u64 segment_addr, u64 segment_
|
|||||||
}
|
}
|
||||||
|
|
||||||
const auto record_unsupported = [&](uintptr_t site) {
|
const auto record_unsupported = [&](uintptr_t site) {
|
||||||
|
const auto& decoded = function.instructions.at(site);
|
||||||
const auto& rewrite = rewrite_sites.at(site);
|
const auto& rewrite = rewrite_sites.at(site);
|
||||||
if (rewrite.protect_red_zone) {
|
if (rewrite.protect_red_zone && decoded.accesses_memory) {
|
||||||
++result.unrelocatable_memory_instruction_count;
|
++result.unrelocatable_memory_instruction_count;
|
||||||
}
|
}
|
||||||
};
|
};
|
||||||
@@ -1229,12 +1242,80 @@ RedZonePatchResult PatchRedZoneMemoryInstructions(u64 segment_addr, u64 segment_
|
|||||||
patched_spans.emplace_back(site_span.patch_start, site_span.continuation);
|
patched_spans.emplace_back(site_span.patch_start, site_span.continuation);
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
} // namespace
|
||||||
|
|
||||||
|
RedZonePatchResult PatchGuestInstructions(u64 segment_addr, u64 segment_size,
|
||||||
|
std::span<const uintptr_t> function_starts,
|
||||||
|
bool protect_memory, bool emulate_rsqrt) {
|
||||||
|
RedZonePatchResult result {};
|
||||||
|
auto* module = GetContainingModule(reinterpret_cast<void*>(segment_addr));
|
||||||
|
if (module == nullptr || function_starts.empty()) {
|
||||||
|
return result;
|
||||||
|
}
|
||||||
|
|
||||||
|
const uintptr_t segment_end = segment_addr + segment_size;
|
||||||
|
std::vector<uintptr_t> starts;
|
||||||
|
starts.reserve(function_starts.size());
|
||||||
|
for (const uintptr_t start: function_starts) {
|
||||||
|
if (start >= segment_addr && start < segment_end) {
|
||||||
|
starts.push_back(start);
|
||||||
|
}
|
||||||
|
}
|
||||||
|
std::ranges::sort(starts);
|
||||||
|
const auto unique_end = std::ranges::unique(starts).begin();
|
||||||
|
starts.erase(unique_end, starts.end());
|
||||||
|
|
||||||
|
std::unique_lock lock {module->mutex};
|
||||||
|
const size_t trampoline_begin = module->trampoline_gen.getSize();
|
||||||
|
std::vector<ReciprocalSquareRootSite> reciprocal_sqrt_sites;
|
||||||
|
for (size_t function_index = 0; function_index < starts.size(); ++function_index) {
|
||||||
|
const uintptr_t function_start = starts[function_index];
|
||||||
|
const uintptr_t function_end =
|
||||||
|
function_index + 1 < starts.size() ? starts[function_index + 1] : segment_end;
|
||||||
|
if (function_end <= function_start) {
|
||||||
|
continue;
|
||||||
|
}
|
||||||
|
|
||||||
|
++result.function_count;
|
||||||
|
auto function = DecodeFunction(function_start, function_end, segment_addr, segment_end);
|
||||||
|
AnalyzeRedZoneLiveness(function);
|
||||||
|
result.instruction_count += function.instructions.size();
|
||||||
|
|
||||||
|
std::map<uintptr_t, InstructionRewrite> rewrite_sites;
|
||||||
|
|
||||||
|
if (function.uses_red_zone) {
|
||||||
|
++result.red_zone_function_count;
|
||||||
|
result.indirect_red_zone_function_count += function.has_indirect_branch;
|
||||||
|
}
|
||||||
|
if (protect_memory) {
|
||||||
|
CollectRedZoneMemoryInstructions(function, rewrite_sites, result);
|
||||||
|
}
|
||||||
|
if (emulate_rsqrt) {
|
||||||
|
CollectReciprocalSquareRoots(function, rewrite_sites, reciprocal_sqrt_sites);
|
||||||
|
}
|
||||||
|
if (!rewrite_sites.empty()) {
|
||||||
|
RelocateRedZoneInstructions(module, function, rewrite_sites, result);
|
||||||
|
}
|
||||||
|
}
|
||||||
|
// Preserve valid instruction encodings throughout CFG analysis and relocation.
|
||||||
|
// Only now turn reciprocal roots into traps, including the relocated copies.
|
||||||
|
const auto trampoline_addr =
|
||||||
|
reinterpret_cast<u64>(module->trampoline_gen.getCode()) + trampoline_begin;
|
||||||
|
const auto trampoline_size = module->trampoline_gen.getSize() - trampoline_begin;
|
||||||
|
if (emulate_rsqrt) {
|
||||||
|
result.reciprocal_sqrt_instruction_count = ApplyReciprocalSquareRootPatches(
|
||||||
|
*module, reciprocal_sqrt_sites, trampoline_addr, trampoline_size);
|
||||||
|
}
|
||||||
|
Common::VirtualMemory::FlushInstructionCache(segment_addr, segment_size);
|
||||||
|
if (trampoline_size != 0) {
|
||||||
|
Common::VirtualMemory::FlushInstructionCache(trampoline_addr, trampoline_size);
|
||||||
|
}
|
||||||
return result;
|
return result;
|
||||||
}
|
}
|
||||||
|
|
||||||
#else
|
#else
|
||||||
|
|
||||||
RedZonePatchResult PatchRedZoneMemoryInstructions(u64, u64, std::span<const uintptr_t>) {
|
RedZonePatchResult PatchGuestInstructions(u64, u64, std::span<const uintptr_t>, bool, bool) {
|
||||||
return {};
|
return {};
|
||||||
}
|
}
|
||||||
|
|
||||||
|
|||||||
@@ -22,14 +22,17 @@ struct RedZonePatchResult {
|
|||||||
uint64_t control_flow_memory_instruction_count = 0;
|
uint64_t control_flow_memory_instruction_count = 0;
|
||||||
uint64_t unrelocatable_memory_instruction_count = 0;
|
uint64_t unrelocatable_memory_instruction_count = 0;
|
||||||
uint64_t indirect_red_zone_function_count = 0;
|
uint64_t indirect_red_zone_function_count = 0;
|
||||||
|
uint64_t reciprocal_sqrt_instruction_count = 0;
|
||||||
};
|
};
|
||||||
|
|
||||||
void RegisterRedZonePatchModule(void* module_ptr, uint64_t module_size, void* trampoline_area_ptr,
|
void RegisterRedZonePatchModule(void* module_ptr, uint64_t module_size, void* trampoline_area_ptr,
|
||||||
uint64_t trampoline_area_size);
|
uint64_t trampoline_area_size);
|
||||||
void UnregisterRedZonePatchModule(void* module_ptr);
|
void UnregisterRedZonePatchModule(void* module_ptr);
|
||||||
|
|
||||||
RedZonePatchResult PatchRedZoneMemoryInstructions(uint64_t segment_addr, uint64_t segment_size,
|
// Analyze and protect fault sites before replacing reciprocal roots with traps.
|
||||||
std::span<const uintptr_t> function_starts);
|
RedZonePatchResult PatchGuestInstructions(uint64_t segment_addr, uint64_t segment_size,
|
||||||
|
std::span<const uintptr_t> function_starts,
|
||||||
|
bool protect_memory, bool emulate_rsqrt);
|
||||||
|
|
||||||
bool DecodeEhFrameFunctionStarts(uint64_t eh_frame_header_addr, uint64_t eh_frame_header_size,
|
bool DecodeEhFrameFunctionStarts(uint64_t eh_frame_header_addr, uint64_t eh_frame_header_size,
|
||||||
std::vector<uintptr_t>* function_starts);
|
std::vector<uintptr_t>* function_starts);
|
||||||
|
|||||||
@@ -2014,9 +2014,11 @@ void RuntimeLinker::LoadProgramToMemory(Program* program) {
|
|||||||
uint64_t tls_handler_size = is_shared ? 0 : Jit::SafeCall::GetSize();
|
uint64_t tls_handler_size = is_shared ? 0 : Jit::SafeCall::GetSize();
|
||||||
EXIT_IF(tls_handler_size > UINT64_MAX - program->base_size_aligned);
|
EXIT_IF(tls_handler_size > UINT64_MAX - program->base_size_aligned);
|
||||||
program->mapped_size = program->base_size_aligned + tls_handler_size;
|
program->mapped_size = program->base_size_aligned + tls_handler_size;
|
||||||
|
const bool emulate_rsqrt = Config::AmdCpuEnabled();
|
||||||
|
|
||||||
#if KYTY_PLATFORM == KYTY_PLATFORM_WINDOWS
|
#if KYTY_PLATFORM == KYTY_PLATFORM_WINDOWS
|
||||||
const bool use_red_zone_protection = Config::RedZoneProtectionEnabled();
|
const bool protect_memory_faults = Config::RedZoneProtectionEnabled();
|
||||||
|
const bool use_red_zone_protection = protect_memory_faults || emulate_rsqrt;
|
||||||
constexpr uint64_t RED_ZONE_TRAMPOLINE_SIZE = 8u * 1024u * 1024u;
|
constexpr uint64_t RED_ZONE_TRAMPOLINE_SIZE = 8u * 1024u * 1024u;
|
||||||
if (use_red_zone_protection) {
|
if (use_red_zone_protection) {
|
||||||
EXIT_IF(RED_ZONE_TRAMPOLINE_SIZE > UINT64_MAX - program->mapped_size);
|
EXIT_IF(RED_ZONE_TRAMPOLINE_SIZE > UINT64_MAX - program->mapped_size);
|
||||||
@@ -2062,9 +2064,8 @@ void RuntimeLinker::LoadProgramToMemory(Program* program) {
|
|||||||
EXIT("Failed to install the required vectored exception handler\n");
|
EXIT("Failed to install the required vectored exception handler\n");
|
||||||
}
|
}
|
||||||
|
|
||||||
// program->elf->SetBaseVAddr(program->base_vaddr);
|
|
||||||
#if KYTY_PLATFORM == KYTY_PLATFORM_WINDOWS
|
|
||||||
std::vector<std::pair<uint64_t, uint64_t>> executable_segments;
|
std::vector<std::pair<uint64_t, uint64_t>> executable_segments;
|
||||||
|
#if KYTY_PLATFORM == KYTY_PLATFORM_WINDOWS
|
||||||
uint64_t eh_frame_header_addr = 0;
|
uint64_t eh_frame_header_addr = 0;
|
||||||
uint64_t eh_frame_header_size = 0;
|
uint64_t eh_frame_header_size = 0;
|
||||||
#endif
|
#endif
|
||||||
@@ -2089,23 +2090,9 @@ void RuntimeLinker::LoadProgramToMemory(Program* program) {
|
|||||||
mode == Common::VirtualMemory::Mode::NoAccess);
|
mode == Common::VirtualMemory::Mode::NoAccess);
|
||||||
|
|
||||||
if (Common::VirtualMemory::IsExecute(mode)) {
|
if (Common::VirtualMemory::IsExecute(mode)) {
|
||||||
if (Config::AmdCpuEnabled()) {
|
|
||||||
const auto reciprocal_sqrt_count =
|
|
||||||
X64InstructionEmulator::PatchReciprocalSquareRoots(segment_addr,
|
|
||||||
segment_file_size);
|
|
||||||
if (reciprocal_sqrt_count != 0) {
|
|
||||||
LOGF("Guest VRSQRTPS emulation: %s, instructions=%" PRIu64 "\n",
|
|
||||||
Common::PathToString(program->file_name.filename()).c_str(),
|
|
||||||
reciprocal_sqrt_count);
|
|
||||||
}
|
|
||||||
}
|
|
||||||
PatchProgram(program, segment_addr, segment_memory_size);
|
PatchProgram(program, segment_addr, segment_memory_size);
|
||||||
#if KYTY_PLATFORM == KYTY_PLATFORM_WINDOWS
|
|
||||||
if (use_red_zone_protection) {
|
|
||||||
executable_segments.emplace_back(segment_addr, segment_file_size);
|
executable_segments.emplace_back(segment_addr, segment_file_size);
|
||||||
}
|
}
|
||||||
#endif
|
|
||||||
}
|
|
||||||
|
|
||||||
if (!skip_protect) {
|
if (!skip_protect) {
|
||||||
Libs::LibKernel::Memory::SetProgramMemoryProtection(segment_addr,
|
Libs::LibKernel::Memory::SetProgramMemoryProtection(segment_addr,
|
||||||
@@ -2149,16 +2136,22 @@ void RuntimeLinker::LoadProgramToMemory(Program* program) {
|
|||||||
}
|
}
|
||||||
|
|
||||||
#if KYTY_PLATFORM == KYTY_PLATFORM_WINDOWS
|
#if KYTY_PLATFORM == KYTY_PLATFORM_WINDOWS
|
||||||
if (use_red_zone_protection) {
|
|
||||||
std::vector<uintptr_t> function_starts;
|
std::vector<uintptr_t> function_starts;
|
||||||
|
if (use_red_zone_protection) {
|
||||||
if (!DecodeEhFrameFunctionStarts(eh_frame_header_addr, eh_frame_header_size,
|
if (!DecodeEhFrameFunctionStarts(eh_frame_header_addr, eh_frame_header_size,
|
||||||
&function_starts)) {
|
&function_starts)) {
|
||||||
LOGF("Windows guest red-zone patching could not decode function boundaries for %s\n",
|
LOGF("Windows guest red-zone patching could not decode function boundaries for %s\n",
|
||||||
Common::PathToString(program->file_name).c_str());
|
Common::PathToString(program->file_name).c_str());
|
||||||
}
|
}
|
||||||
|
}
|
||||||
|
#endif
|
||||||
for (const auto& [segment_addr, segment_size]: executable_segments) {
|
for (const auto& [segment_addr, segment_size]: executable_segments) {
|
||||||
|
uint64_t reciprocal_sqrt_count = 0;
|
||||||
|
#if KYTY_PLATFORM == KYTY_PLATFORM_WINDOWS
|
||||||
|
if (use_red_zone_protection) {
|
||||||
const auto result =
|
const auto result =
|
||||||
PatchRedZoneMemoryInstructions(segment_addr, segment_size, function_starts);
|
PatchGuestInstructions(segment_addr, segment_size, function_starts,
|
||||||
|
protect_memory_faults, emulate_rsqrt);
|
||||||
LOGF("Windows guest red-zone patching: %s, functions=%" PRIu64 ", red_zone=%" PRIu64
|
LOGF("Windows guest red-zone patching: %s, functions=%" PRIu64 ", red_zone=%" PRIu64
|
||||||
", memory=%" PRIu64 ", patched=%" PRIu64 ", short=%" PRIu64 ", stack=%" PRIu64
|
", memory=%" PRIu64 ", patched=%" PRIu64 ", short=%" PRIu64 ", stack=%" PRIu64
|
||||||
", control=%" PRIu64 ", unrelocatable=%" PRIu64 "\n",
|
", control=%" PRIu64 ", unrelocatable=%" PRIu64 "\n",
|
||||||
@@ -2168,12 +2161,20 @@ void RuntimeLinker::LoadProgramToMemory(Program* program) {
|
|||||||
result.stack_dependent_memory_instruction_count,
|
result.stack_dependent_memory_instruction_count,
|
||||||
result.control_flow_memory_instruction_count,
|
result.control_flow_memory_instruction_count,
|
||||||
result.unrelocatable_memory_instruction_count);
|
result.unrelocatable_memory_instruction_count);
|
||||||
|
reciprocal_sqrt_count = result.reciprocal_sqrt_instruction_count;
|
||||||
|
}
|
||||||
|
#else
|
||||||
|
if (emulate_rsqrt) {
|
||||||
|
reciprocal_sqrt_count =
|
||||||
|
X64InstructionEmulator::PatchReciprocalSquareRoots(segment_addr, segment_size);
|
||||||
Common::VirtualMemory::FlushInstructionCache(segment_addr, segment_size);
|
Common::VirtualMemory::FlushInstructionCache(segment_addr, segment_size);
|
||||||
}
|
}
|
||||||
Common::VirtualMemory::FlushInstructionCache(program->red_zone_trampoline_vaddr,
|
|
||||||
program->red_zone_trampoline_size);
|
|
||||||
}
|
|
||||||
#endif
|
#endif
|
||||||
|
if (reciprocal_sqrt_count != 0) {
|
||||||
|
LOGF("Guest VRSQRTPS emulation: %s, instructions=%" PRIu64 "\n",
|
||||||
|
Common::PathToString(program->file_name.filename()).c_str(), reciprocal_sqrt_count);
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
if (!is_shared) {
|
if (!is_shared) {
|
||||||
SetupTlsHandler(program);
|
SetupTlsHandler(program);
|
||||||
|
|||||||
@@ -5,7 +5,7 @@
|
|||||||
#include <Zydis/Zydis.h>
|
#include <Zydis/Zydis.h>
|
||||||
#include <bit>
|
#include <bit>
|
||||||
#include <cstring>
|
#include <cstring>
|
||||||
#if KYTY_PLATFORM == KYTY_PLATFORM_LINUX && !defined(__APPLE__)
|
#if !defined(__APPLE__)
|
||||||
#include <emmintrin.h>
|
#include <emmintrin.h>
|
||||||
#include <xmmintrin.h>
|
#include <xmmintrin.h>
|
||||||
#endif
|
#endif
|
||||||
@@ -424,6 +424,21 @@ struct Context {
|
|||||||
native->R12, native->R13, native->R14, native->R15};
|
native->R12, native->R13, native->R14, native->R15};
|
||||||
std::memcpy(gpr, registers, sizeof(gpr));
|
std::memcpy(gpr, registers, sizeof(gpr));
|
||||||
}
|
}
|
||||||
|
|
||||||
|
void ClearUpperYmm(uint8_t index) const {
|
||||||
|
if ((native->ContextFlags & CONTEXT_XSTATE) != CONTEXT_XSTATE) {
|
||||||
|
return;
|
||||||
|
}
|
||||||
|
DWORD64 features = 0;
|
||||||
|
if (!GetXStateFeaturesMask(native, &features) || (features & XSTATE_MASK_AVX) == 0) {
|
||||||
|
return; // An absent AVX component restores zeroes.
|
||||||
|
}
|
||||||
|
DWORD size = 0;
|
||||||
|
auto* ymm = static_cast<M128A*>(LocateXStateFeature(native, XSTATE_AVX, &size));
|
||||||
|
if (ymm != nullptr && size >= (index + 1u) * sizeof(M128A)) {
|
||||||
|
ymm[index] = {};
|
||||||
|
}
|
||||||
|
}
|
||||||
#else
|
#else
|
||||||
ucontext_t* native;
|
ucontext_t* native;
|
||||||
|
|
||||||
@@ -588,8 +603,6 @@ static bool TryEmulateMonitorxMwaitx(Context& context) {
|
|||||||
return true;
|
return true;
|
||||||
}
|
}
|
||||||
|
|
||||||
#if KYTY_PLATFORM == KYTY_PLATFORM_LINUX
|
|
||||||
|
|
||||||
static uint32_t ReciprocalSquareRoot(uint32_t bits) {
|
static uint32_t ReciprocalSquareRoot(uint32_t bits) {
|
||||||
const uint32_t magnitude = bits & 0x7fffffffu;
|
const uint32_t magnitude = bits & 0x7fffffffu;
|
||||||
const uint32_t exponent = magnitude & 0x7f800000u;
|
const uint32_t exponent = magnitude & 0x7f800000u;
|
||||||
@@ -658,11 +671,18 @@ static bool TryEmulateReciprocalSquareRoot(Context& context) {
|
|||||||
}
|
}
|
||||||
|
|
||||||
#endif
|
#endif
|
||||||
#endif
|
|
||||||
|
bool IsReciprocalSquareRoot(const ZydisDecodedInstruction& instruction,
|
||||||
|
const ZydisDecodedOperand* operands) {
|
||||||
|
return instruction.mnemonic == ZYDIS_MNEMONIC_VRSQRTPS &&
|
||||||
|
instruction.encoding == ZYDIS_INSTRUCTION_ENCODING_VEX &&
|
||||||
|
instruction.raw.vex.offset == 0 && operands[0].size == 128 &&
|
||||||
|
operands[1].type == ZYDIS_OPERAND_TYPE_REGISTER;
|
||||||
|
}
|
||||||
|
|
||||||
uint64_t PatchReciprocalSquareRoots(uint64_t address, uint64_t size) {
|
uint64_t PatchReciprocalSquareRoots(uint64_t address, uint64_t size) {
|
||||||
uint64_t patched = 0;
|
uint64_t patched = 0;
|
||||||
#if KYTY_PLATFORM == KYTY_PLATFORM_LINUX && !defined(__APPLE__)
|
#if !defined(__APPLE__)
|
||||||
ZydisDecoder decoder {};
|
ZydisDecoder decoder {};
|
||||||
if (!ZYAN_SUCCESS(
|
if (!ZYAN_SUCCESS(
|
||||||
ZydisDecoderInit(&decoder, ZYDIS_MACHINE_MODE_LONG_64, ZYDIS_STACK_WIDTH_64))) {
|
ZydisDecoderInit(&decoder, ZYDIS_MACHINE_MODE_LONG_64, ZYDIS_STACK_WIDTH_64))) {
|
||||||
@@ -677,12 +697,9 @@ uint64_t PatchReciprocalSquareRoots(uint64_t address, uint64_t size) {
|
|||||||
++offset;
|
++offset;
|
||||||
continue;
|
continue;
|
||||||
}
|
}
|
||||||
if (instruction.mnemonic == ZYDIS_MNEMONIC_VRSQRTPS &&
|
if (IsReciprocalSquareRoot(instruction, operands)) {
|
||||||
instruction.encoding == ZYDIS_INSTRUCTION_ENCODING_VEX &&
|
|
||||||
instruction.raw.vex.offset == 0 && operands[0].size == 128 &&
|
|
||||||
operands[1].type == ZYDIS_OPERAND_TYPE_REGISTER) {
|
|
||||||
// vvvv is reserved (must be 1111b). Clear one bit to route this
|
// vvvv is reserved (must be 1111b). Clear one bit to route this
|
||||||
// otherwise intact instruction through the existing SIGILL emulator.
|
// otherwise intact instruction through the illegal-instruction emulator.
|
||||||
code[instruction.raw.vex.size - 1] &= ~0x08u;
|
code[instruction.raw.vex.size - 1] &= ~0x08u;
|
||||||
++patched;
|
++patched;
|
||||||
}
|
}
|
||||||
@@ -704,10 +721,10 @@ bool TryEmulate(void* native_context) {
|
|||||||
Context context {static_cast<PCONTEXT>(native_context)};
|
Context context {static_cast<PCONTEXT>(native_context)};
|
||||||
#else
|
#else
|
||||||
Context context {static_cast<ucontext_t*>(native_context)};
|
Context context {static_cast<ucontext_t*>(native_context)};
|
||||||
|
#endif
|
||||||
if (TryEmulateReciprocalSquareRoot(context)) {
|
if (TryEmulateReciprocalSquareRoot(context)) {
|
||||||
return true;
|
return true;
|
||||||
}
|
}
|
||||||
#endif
|
|
||||||
return TryEmulateMonitorxMwaitx(context) || TryEmulateSse4a(context) ||
|
return TryEmulateMonitorxMwaitx(context) || TryEmulateSse4a(context) ||
|
||||||
TryEmulateShaNi(context);
|
TryEmulateShaNi(context);
|
||||||
#else
|
#else
|
||||||
|
|||||||
@@ -1,10 +1,13 @@
|
|||||||
#ifndef KYTY_LOADER_X64_INSTRUCTION_EMULATOR_H_
|
#ifndef KYTY_LOADER_X64_INSTRUCTION_EMULATOR_H_
|
||||||
#define KYTY_LOADER_X64_INSTRUCTION_EMULATOR_H_
|
#define KYTY_LOADER_X64_INSTRUCTION_EMULATOR_H_
|
||||||
|
|
||||||
|
#include <Zydis/DecoderTypes.h>
|
||||||
#include <cstdint>
|
#include <cstdint>
|
||||||
|
|
||||||
namespace Loader::X64InstructionEmulator {
|
namespace Loader::X64InstructionEmulator {
|
||||||
|
|
||||||
|
[[nodiscard]] bool IsReciprocalSquareRoot(const ZydisDecodedInstruction& instruction,
|
||||||
|
const ZydisDecodedOperand* operands);
|
||||||
uint64_t PatchReciprocalSquareRoots(uint64_t address, uint64_t size);
|
uint64_t PatchReciprocalSquareRoots(uint64_t address, uint64_t size);
|
||||||
[[nodiscard]] bool TryEmulate(void* native_context);
|
[[nodiscard]] bool TryEmulate(void* native_context);
|
||||||
|
|
||||||
|
|||||||
+1
-1
@@ -56,7 +56,7 @@ static void PrintUsage() {
|
|||||||
" --gpu <index> Vulkan physical device index. Default: auto.\n");
|
" --gpu <index> Vulkan physical device index. Default: auto.\n");
|
||||||
::printf(" --fullscreen Run in borderless desktop fullscreen.\n");
|
::printf(" --fullscreen Run in borderless desktop fullscreen.\n");
|
||||||
::printf(" --vr Enable the virtual VR headset.\n");
|
::printf(" --vr Enable the virtual VR headset.\n");
|
||||||
::printf(" --amd-cpu Apply AMD CPU instruction patches (Linux).\n");
|
::printf(" --amd-cpu Apply AMD CPU instruction patches.\n");
|
||||||
::printf(" --vblank-frequency <num> Virtual vblank frequency. Default: 60.\n");
|
::printf(" --vblank-frequency <num> Virtual vblank frequency. Default: 60.\n");
|
||||||
::printf(" --console-language <0-29> Console language. Default: 1 (English US).\n");
|
::printf(" --console-language <0-29> Console language. Default: 1 (English US).\n");
|
||||||
::printf(" --vulkan-validation <true|false> Enable Vulkan validation.\n");
|
::printf(" --vulkan-validation <true|false> Enable Vulkan validation.\n");
|
||||||
|
|||||||
@@ -24,13 +24,16 @@
|
|||||||
#include <string>
|
#include <string>
|
||||||
#include <vector>
|
#include <vector>
|
||||||
|
|
||||||
#if defined(__linux__)
|
#if defined(__linux__) || KYTY_PLATFORM == KYTY_PLATFORM_WINDOWS
|
||||||
#include <csignal>
|
#include <csignal>
|
||||||
#include <immintrin.h>
|
#include <immintrin.h>
|
||||||
|
#include <xbyak/xbyak.h>
|
||||||
|
#endif
|
||||||
|
|
||||||
|
#if defined(__linux__)
|
||||||
#include <sys/uio.h>
|
#include <sys/uio.h>
|
||||||
#include <ucontext.h>
|
#include <ucontext.h>
|
||||||
#include <unistd.h>
|
#include <unistd.h>
|
||||||
#include <xbyak/xbyak.h>
|
|
||||||
#endif
|
#endif
|
||||||
|
|
||||||
#if KYTY_PLATFORM == KYTY_PLATFORM_WINDOWS
|
#if KYTY_PLATFORM == KYTY_PLATFORM_WINDOWS
|
||||||
@@ -217,10 +220,8 @@ void TestWindowsGuestRedZoneStaticPatcher() {
|
|||||||
reinterpret_cast<void*>(mapping + CODE_SIZE),
|
reinterpret_cast<void*>(mapping + CODE_SIZE),
|
||||||
TRAMPOLINE_SIZE);
|
TRAMPOLINE_SIZE);
|
||||||
const std::array<uintptr_t, 1> function_starts = {static_cast<uintptr_t>(mapping)};
|
const std::array<uintptr_t, 1> function_starts = {static_cast<uintptr_t>(mapping)};
|
||||||
const auto result = Loader::PatchRedZoneMemoryInstructions(
|
const auto result = Loader::PatchGuestInstructions(
|
||||||
mapping, code.size(), function_starts);
|
mapping, code.size(), function_starts, true, false);
|
||||||
Check(test, Common::VirtualMemory::FlushInstructionCache(mapping, CODE_SIZE + TRAMPOLINE_SIZE),
|
|
||||||
"failed to flush patched test code");
|
|
||||||
const bool patched_preserved = function(static_cast<const uint64_t*>(g_red_zone_fault_page)) == 1;
|
const bool patched_preserved = function(static_cast<const uint64_t*>(g_red_zone_fault_page)) == 1;
|
||||||
|
|
||||||
Loader::UnregisterRedZonePatchModule(reinterpret_cast<void*>(mapping));
|
Loader::UnregisterRedZonePatchModule(reinterpret_cast<void*>(mapping));
|
||||||
@@ -2546,55 +2547,93 @@ void TestModuleRelocationUsesWritableHostMapping() {
|
|||||||
std::printf("[host] %-48s ok\n", test);
|
std::printf("[host] %-48s ok\n", test);
|
||||||
}
|
}
|
||||||
|
|
||||||
#if defined(__linux__)
|
#if defined(__linux__) || KYTY_PLATFORM == KYTY_PLATFORM_WINDOWS
|
||||||
volatile sig_atomic_t g_rsqrt_traps = 0;
|
volatile sig_atomic_t g_rsqrt_traps = 0;
|
||||||
|
|
||||||
void ReciprocalSquareRootHandler(int, siginfo_t*, void* context) {
|
bool EmulateReciprocalSquareRootContext(void* context) {
|
||||||
const auto host_mxcsr = _mm_getcsr();
|
const auto host_mxcsr = _mm_getcsr();
|
||||||
_mm_setcsr(0x7fe1); // Exercise the handler under a distinct rounding mode and raised flags.
|
_mm_setcsr(0x7fe1); // Exercise the handler under a distinct rounding mode and raised flags.
|
||||||
if (!Loader::X64InstructionEmulator::TryEmulate(context)) {
|
const bool emulated = Loader::X64InstructionEmulator::TryEmulate(context);
|
||||||
|
const bool preserved_mxcsr = _mm_getcsr() == 0x7fe1;
|
||||||
|
_mm_setcsr(host_mxcsr);
|
||||||
|
if (!emulated || !preserved_mxcsr) {
|
||||||
|
return false;
|
||||||
|
}
|
||||||
|
g_rsqrt_traps = g_rsqrt_traps + 1;
|
||||||
|
return true;
|
||||||
|
}
|
||||||
|
|
||||||
|
#if KYTY_PLATFORM == KYTY_PLATFORM_WINDOWS
|
||||||
|
LONG CALLBACK ReciprocalSquareRootHandler(EXCEPTION_POINTERS* exception) {
|
||||||
|
if (exception->ExceptionRecord->ExceptionCode != EXCEPTION_ILLEGAL_INSTRUCTION ||
|
||||||
|
!EmulateReciprocalSquareRootContext(exception->ContextRecord)) {
|
||||||
|
return EXCEPTION_CONTINUE_SEARCH;
|
||||||
|
}
|
||||||
|
return EXCEPTION_CONTINUE_EXECUTION;
|
||||||
|
}
|
||||||
|
#else
|
||||||
|
void ReciprocalSquareRootHandler(int, siginfo_t*, void* context) {
|
||||||
|
if (!EmulateReciprocalSquareRootContext(context)) {
|
||||||
_exit(190);
|
_exit(190);
|
||||||
}
|
}
|
||||||
if (_mm_getcsr() != 0x7fe1) {
|
|
||||||
_exit(191);
|
|
||||||
}
|
|
||||||
_mm_setcsr(host_mxcsr);
|
|
||||||
g_rsqrt_traps = g_rsqrt_traps + 1;
|
|
||||||
}
|
}
|
||||||
|
#endif
|
||||||
|
|
||||||
void TestPackedReciprocalSquareRoot() {
|
void TestPackedReciprocalSquareRoot() {
|
||||||
const char* test = "PackedReciprocalSquareRoot";
|
const char* test = "PackedReciprocalSquareRoot";
|
||||||
constexpr uint64_t code_size = 0x4000;
|
constexpr uint64_t code_size = 0x4000;
|
||||||
|
#if KYTY_PLATFORM == KYTY_PLATFORM_WINDOWS
|
||||||
|
constexpr uint64_t allocation_size = code_size * 2;
|
||||||
|
#else
|
||||||
|
constexpr uint64_t allocation_size = code_size;
|
||||||
|
#endif
|
||||||
const auto mapping = Libs::LibKernel::Memory::AllocateProgramMemory(
|
const auto mapping = Libs::LibKernel::Memory::AllocateProgramMemory(
|
||||||
0x902000000, code_size, Common::VirtualMemory::Mode::ExecuteReadWrite, "rsqrt_test");
|
0x902000000, allocation_size, Common::VirtualMemory::Mode::ExecuteReadWrite, "rsqrt_test");
|
||||||
Check(test, mapping != 0, "failed to allocate instruction test code");
|
Check(test, mapping != 0, "failed to allocate instruction test code");
|
||||||
struct RestoreState {
|
struct RestoreState {
|
||||||
uint64_t mapping;
|
uint64_t mapping;
|
||||||
uint64_t size;
|
uint64_t size;
|
||||||
uint32_t mxcsr;
|
uint32_t mxcsr;
|
||||||
|
#if KYTY_PLATFORM == KYTY_PLATFORM_WINDOWS
|
||||||
|
void* handler = nullptr;
|
||||||
|
#else
|
||||||
struct sigaction previous {};
|
struct sigaction previous {};
|
||||||
bool installed = false;
|
bool installed = false;
|
||||||
|
#endif
|
||||||
~RestoreState() {
|
~RestoreState() {
|
||||||
_mm_setcsr(mxcsr);
|
_mm_setcsr(mxcsr);
|
||||||
|
#if KYTY_PLATFORM == KYTY_PLATFORM_WINDOWS
|
||||||
|
if (handler != nullptr) {
|
||||||
|
RemoveVectoredExceptionHandler(handler);
|
||||||
|
}
|
||||||
|
Loader::UnregisterRedZonePatchModule(reinterpret_cast<void*>(mapping));
|
||||||
|
#else
|
||||||
if (installed) {
|
if (installed) {
|
||||||
sigaction(SIGILL, &previous, nullptr);
|
sigaction(SIGILL, &previous, nullptr);
|
||||||
}
|
}
|
||||||
|
#endif
|
||||||
Libs::LibKernel::Memory::FreeGuestMemory(mapping, size);
|
Libs::LibKernel::Memory::FreeGuestMemory(mapping, size);
|
||||||
}
|
}
|
||||||
} restore {mapping, code_size, _mm_getcsr()};
|
} restore {mapping, allocation_size, _mm_getcsr()};
|
||||||
|
#if KYTY_PLATFORM == KYTY_PLATFORM_WINDOWS
|
||||||
|
restore.handler = AddVectoredExceptionHandler(1, ReciprocalSquareRootHandler);
|
||||||
|
Check(test, restore.handler != nullptr, "failed to install instruction handler");
|
||||||
|
#else
|
||||||
struct sigaction action {};
|
struct sigaction action {};
|
||||||
action.sa_sigaction = ReciprocalSquareRootHandler;
|
action.sa_sigaction = ReciprocalSquareRootHandler;
|
||||||
action.sa_flags = SA_SIGINFO;
|
action.sa_flags = SA_SIGINFO;
|
||||||
sigemptyset(&action.sa_mask);
|
sigemptyset(&action.sa_mask);
|
||||||
restore.installed = sigaction(SIGILL, &action, &restore.previous) == 0;
|
restore.installed = sigaction(SIGILL, &action, &restore.previous) == 0;
|
||||||
Check(test, restore.installed, "failed to install instruction handler");
|
Check(test, restore.installed, "failed to install instruction handler");
|
||||||
|
#endif
|
||||||
using GuestFunction = void(KYTY_SYSV_ABI*)(const uint32_t*, uint32_t*);
|
using GuestFunction = void(KYTY_SYSV_ABI*)(const uint32_t*, uint32_t*);
|
||||||
const auto function = reinterpret_cast<GuestFunction>(mapping);
|
const auto function = reinterpret_cast<GuestFunction>(mapping);
|
||||||
const auto refine = [](float estimate) {
|
const auto refine = [](float estimate) {
|
||||||
return (estimate * 0.5f) * std::fma(-estimate, estimate, 3.0f);
|
return (estimate * 0.5f) * std::fma(-estimate, estimate, 3.0f);
|
||||||
};
|
};
|
||||||
|
constexpr uint64_t red_zone_sentinel = 0x1122334455667788ull;
|
||||||
std::array<uint32_t, 16> input {};
|
std::array<uint32_t, 16> input {};
|
||||||
std::array<uint32_t, 16> output {};
|
std::array<uint32_t, 48> output {};
|
||||||
input.fill(0x3f800000);
|
input.fill(0x3f800000);
|
||||||
for (const auto registers: {std::array {2, 1}, std::array {10, 9}, std::array {1, 1}}) {
|
for (const auto registers: {std::array {2, 1}, std::array {10, 9}, std::array {1, 1}}) {
|
||||||
const auto destination = registers[0];
|
const auto destination = registers[0];
|
||||||
@@ -2613,16 +2652,55 @@ void TestPackedReciprocalSquareRoot() {
|
|||||||
if (destination != source) {
|
if (destination != source) {
|
||||||
code.vmovups(Xbyak::Ymm(destination), code.ptr[code.rdi + 32]);
|
code.vmovups(Xbyak::Ymm(destination), code.ptr[code.rdi + 32]);
|
||||||
}
|
}
|
||||||
|
for (uint32_t offset = 8; offset <= 128; offset += 8) {
|
||||||
|
code.mov(code.rax, red_zone_sentinel ^ offset);
|
||||||
|
code.mov(code.qword[code.rsp - offset], code.rax);
|
||||||
|
}
|
||||||
code.vrsqrtps(Xbyak::Xmm(destination), Xbyak::Xmm(source));
|
code.vrsqrtps(Xbyak::Xmm(destination), Xbyak::Xmm(source));
|
||||||
|
for (uint32_t offset = 8; offset <= 128; offset += 8) {
|
||||||
|
code.mov(code.rax, code.qword[code.rsp - offset]);
|
||||||
|
code.mov(code.qword[code.rsi + 64 + offset - 8], code.rax);
|
||||||
|
}
|
||||||
code.vmovups(code.ptr[code.rsi], Xbyak::Ymm(destination));
|
code.vmovups(code.ptr[code.rsi], Xbyak::Ymm(destination));
|
||||||
code.vmovups(code.ptr[code.rsi + 32], Xbyak::Ymm(source));
|
code.vmovups(code.ptr[code.rsi + 32], Xbyak::Ymm(source));
|
||||||
code.vzeroupper();
|
code.vzeroupper();
|
||||||
code.ret();
|
code.ret();
|
||||||
|
#if defined(__linux__)
|
||||||
const std::vector<uint8_t> original(code.getCode(), code.getCurr());
|
const std::vector<uint8_t> original(code.getCode(), code.getCurr());
|
||||||
|
#endif
|
||||||
|
Check(test, Common::VirtualMemory::FlushInstructionCache(mapping, code.getSize()),
|
||||||
|
"failed to flush generated instruction test code");
|
||||||
function(input.data(), output.data());
|
function(input.data(), output.data());
|
||||||
const float native = std::bit_cast<float>(output[0]);
|
const float native = std::bit_cast<float>(output[0]);
|
||||||
|
const auto red_zone_intact = [&] {
|
||||||
|
for (uint32_t offset = 8; offset <= 128; offset += 8) {
|
||||||
|
const auto index = 16 + (offset - 8) / sizeof(uint32_t);
|
||||||
|
const auto value = static_cast<uint64_t>(output[index]) |
|
||||||
|
(static_cast<uint64_t>(output[index + 1]) << 32);
|
||||||
|
if (value != (red_zone_sentinel ^ offset)) {
|
||||||
|
return false;
|
||||||
|
}
|
||||||
|
}
|
||||||
|
return true;
|
||||||
|
};
|
||||||
|
Check(test, red_zone_intact(), "native instruction corrupted the guest red zone");
|
||||||
Check(test, std::isfinite(native) && std::abs(native - std::bit_cast<float>(expected)) < 0.001f,
|
Check(test, std::isfinite(native) && std::abs(native - std::bit_cast<float>(expected)) < 0.001f,
|
||||||
"native reciprocal root is outside its error bound");
|
"native reciprocal root is outside its error bound");
|
||||||
|
#if KYTY_PLATFORM == KYTY_PLATFORM_WINDOWS
|
||||||
|
Loader::RegisterRedZonePatchModule(reinterpret_cast<void*>(mapping), code_size,
|
||||||
|
reinterpret_cast<void*>(mapping + code_size), code_size);
|
||||||
|
const std::array<uintptr_t, 1> function_starts {mapping};
|
||||||
|
// The extended-register case also relocates ordinary memory accesses while
|
||||||
|
// red-zone data is live, exercising both enabled patchers in one function.
|
||||||
|
const bool protect_memory = source == 9;
|
||||||
|
const auto patched = Loader::PatchGuestInstructions(
|
||||||
|
mapping, code.getSize(), function_starts, protect_memory, true);
|
||||||
|
Check(test, patched.reciprocal_sqrt_instruction_count == 1 &&
|
||||||
|
patched.unrelocatable_memory_instruction_count == 0 &&
|
||||||
|
(protect_memory ? patched.patched_memory_instruction_count > 0
|
||||||
|
: patched.patched_memory_instruction_count == 0),
|
||||||
|
"instruction pass lost reciprocal root or memory patch coverage");
|
||||||
|
#else
|
||||||
Check(test, Loader::X64InstructionEmulator::PatchReciprocalSquareRoots(mapping, code.getSize()) == 1,
|
Check(test, Loader::X64InstructionEmulator::PatchReciprocalSquareRoots(mapping, code.getSize()) == 1,
|
||||||
"instruction pass did not patch exactly one packed reciprocal root");
|
"instruction pass did not patch exactly one packed reciprocal root");
|
||||||
const auto* patched = reinterpret_cast<const uint8_t*>(mapping);
|
const auto* patched = reinterpret_cast<const uint8_t*>(mapping);
|
||||||
@@ -2631,11 +2709,18 @@ void TestPackedReciprocalSquareRoot() {
|
|||||||
changed += original[i] != patched[i];
|
changed += original[i] != patched[i];
|
||||||
}
|
}
|
||||||
Check(test, changed == 1, "instruction pass changed unrelated code bytes");
|
Check(test, changed == 1, "instruction pass changed unrelated code bytes");
|
||||||
Common::VirtualMemory::FlushInstructionCache(mapping, code.getSize());
|
Check(test, Common::VirtualMemory::FlushInstructionCache(mapping, code.getSize()),
|
||||||
|
"failed to flush patched instruction test code");
|
||||||
|
#endif
|
||||||
const auto before = g_rsqrt_traps;
|
const auto before = g_rsqrt_traps;
|
||||||
function(input.data(), output.data());
|
function(input.data(), output.data());
|
||||||
Check(test, g_rsqrt_traps == before + 1, "patched instruction did not execute its handler");
|
Check(test, g_rsqrt_traps == before + 1, "patched instruction did not execute its handler");
|
||||||
|
Check(test, red_zone_intact(), "patched instruction corrupted the guest red zone");
|
||||||
Check(test, output[0] == expected, "patched instruction read or wrote the wrong register");
|
Check(test, output[0] == expected, "patched instruction read or wrote the wrong register");
|
||||||
|
if (destination != source) {
|
||||||
|
Check(test, std::equal(input.begin(), input.begin() + 4, output.begin() + 8),
|
||||||
|
"source lower XMM lanes were corrupted");
|
||||||
|
}
|
||||||
if (source != 9) {
|
if (source != 9) {
|
||||||
Check(test, refine(refine(std::bit_cast<float>(output[0]))) == 1.0f,
|
Check(test, refine(refine(std::bit_cast<float>(output[0]))) == 1.0f,
|
||||||
"identity quaternion normalization drifted below one");
|
"identity quaternion normalization drifted below one");
|
||||||
@@ -2668,6 +2753,7 @@ void TestPackedReciprocalSquareRoot() {
|
|||||||
function(input.data(), output.data());
|
function(input.data(), output.data());
|
||||||
const auto result_mxcsr = _mm_getcsr();
|
const auto result_mxcsr = _mm_getcsr();
|
||||||
_mm_setcsr(restore.mxcsr);
|
_mm_setcsr(restore.mxcsr);
|
||||||
|
Check(test, red_zone_intact(), "special-value trap corrupted the guest red zone");
|
||||||
Check(test, result_mxcsr == mxcsr, "instruction changed MXCSR controls or exception flags");
|
Check(test, result_mxcsr == mxcsr, "instruction changed MXCSR controls or exception flags");
|
||||||
Check(test, std::equal(output.begin(), output.begin() + 4, values.begin() + 4),
|
Check(test, std::equal(output.begin(), output.begin() + 4, values.begin() + 4),
|
||||||
"special values or round-independent reciprocal roots differ from ISA semantics");
|
"special values or round-independent reciprocal roots differ from ISA semantics");
|
||||||
@@ -2675,6 +2761,7 @@ void TestPackedReciprocalSquareRoot() {
|
|||||||
}
|
}
|
||||||
input.fill(0x3f800000);
|
input.fill(0x3f800000);
|
||||||
}
|
}
|
||||||
|
#if defined(__linux__)
|
||||||
std::array<uint8_t, 16> unknown {0x0f, 0x0b};
|
std::array<uint8_t, 16> unknown {0x0f, 0x0b};
|
||||||
ucontext_t context {};
|
ucontext_t context {};
|
||||||
context.uc_mcontext.gregs[REG_RIP] = reinterpret_cast<greg_t>(unknown.data());
|
context.uc_mcontext.gregs[REG_RIP] = reinterpret_cast<greg_t>(unknown.data());
|
||||||
@@ -2708,6 +2795,7 @@ void TestPackedReciprocalSquareRoot() {
|
|||||||
Check(test, std::equal(sha_expected.begin(), sha_expected.end(), fpstate._xmm[8].element),
|
Check(test, std::equal(sha_expected.begin(), sha_expected.end(), fpstate._xmm[8].element),
|
||||||
"SHA emulation lost its memory operand or extended destination register");
|
"SHA emulation lost its memory operand or extended destination register");
|
||||||
emulate({0x0f, 0x01, 0xfa}, 3); // monitorx
|
emulate({0x0f, 0x01, 0xfa}, 3); // monitorx
|
||||||
|
#endif
|
||||||
std::printf("[host] %-48s ok\n", test);
|
std::printf("[host] %-48s ok\n", test);
|
||||||
}
|
}
|
||||||
#endif
|
#endif
|
||||||
@@ -2716,7 +2804,7 @@ void TestPackedReciprocalSquareRoot() {
|
|||||||
|
|
||||||
int main(int argc, char** argv) {
|
int main(int argc, char** argv) {
|
||||||
InitSubsystems();
|
InitSubsystems();
|
||||||
#if defined(__linux__)
|
#if defined(__linux__) || KYTY_PLATFORM == KYTY_PLATFORM_WINDOWS
|
||||||
if (argc == 2 && std::strcmp(argv[1], "--rsqrt-only") == 0) {
|
if (argc == 2 && std::strcmp(argv[1], "--rsqrt-only") == 0) {
|
||||||
RunTest(TestPackedReciprocalSquareRoot);
|
RunTest(TestPackedReciprocalSquareRoot);
|
||||||
return g_failed_tests == 0 ? 0 : 1;
|
return g_failed_tests == 0 ? 0 : 1;
|
||||||
@@ -2727,7 +2815,7 @@ int main(int argc, char** argv) {
|
|||||||
return g_failed_tests == 0 ? 0 : 1;
|
return g_failed_tests == 0 ? 0 : 1;
|
||||||
}
|
}
|
||||||
|
|
||||||
#if defined(__linux__)
|
#if defined(__linux__) || KYTY_PLATFORM == KYTY_PLATFORM_WINDOWS
|
||||||
RunTest(TestPackedReciprocalSquareRoot);
|
RunTest(TestPackedReciprocalSquareRoot);
|
||||||
#endif
|
#endif
|
||||||
RunTest(TestWindowsGuestRedZoneStaticPatcher);
|
RunTest(TestWindowsGuestRedZoneStaticPatcher);
|
||||||
|
|||||||
Reference in New Issue
Block a user