raphiaura/KytyPS5
1
0
Fork 0
mirror of https://github.com/KytyPS5/KytyPS5.git synced 2026-10-19 18:36:53 +02:00
Code Issues Packages Projects Releases Wiki Activity

loader: add INSERTQ RDPID and CLWB fallbacks

Browse Source 
Co-authored-by: nomolao2-cell <nomolao2@gmail.com>
This commit is contained in:
nmzik
2026-10-03 20:13:58 +04:00
parent a4d26cf81a
commit 19a9464b7d
6 changed files with 667 additions and 107 deletions
Download Patch File Download Diff File Expand all files Collapse all files
src/loader/guestInstructionPatcher.cpp
+104 -35
View File
@@ -132,7 +132,14 @@ struct DecodedFunction {
bool requires_conservative_red_zone_tracking {};
};
enum class InstructionReplacement { None, ReciprocalSquareRoot, ExtractQ };
enum class InstructionReplacement {
None,
ReciprocalSquareRoot,
ExtractQ,
InsertQ,
ReadProcessorId,
CacheLineWriteBack
};
struct InstructionRewrite {
bool protect_red_zone {};
@@ -143,8 +150,19 @@ struct InstructionRewrite {
InstructionPatchCounts& ReplacementCounts(GuestInstructionPatchResult& result,
InstructionReplacement replacement) {
ASSERT(replacement != InstructionReplacement::None);
return replacement == InstructionReplacement::ReciprocalSquareRoot ? result.reciprocal_sqrt
: result.extrq;
switch (replacement) {
case InstructionReplacement::ReciprocalSquareRoot: return result.reciprocal_sqrt;
case InstructionReplacement::ExtractQ: return result.extrq;
case InstructionReplacement::InsertQ: return result.insertq;
case InstructionReplacement::ReadProcessorId: return result.rdpid;
default: return result.clwb;
}
}
bool UsesInstructionTrap(InstructionReplacement replacement, bool trap_replacements) {
return replacement != InstructionReplacement::None &&
(trap_replacements || replacement == InstructionReplacement::InsertQ ||
replacement == InstructionReplacement::ReadProcessorId);
}
bool IsStackPointerRegister(ZydisRegister reg) {
@@ -216,10 +234,11 @@ DecodedCodeInstruction DecodeCodeInstruction(uintptr_t address, uintptr_t end) {
if (decoded.changes_stack_pointer) {
const auto& operands = decoded.operands;
if (decoded.instruction.mnemonic == ZYDIS_MNEMONIC_MOV &&
operands[0].type == ZYDIS_OPERAND_TYPE_REGISTER &&
if (operands[0].type == ZYDIS_OPERAND_TYPE_REGISTER &&
IsStackPointerRegister(operands[0].reg.value) &&
operands[1].type == ZYDIS_OPERAND_TYPE_MEMORY) {
(decoded.instruction.mnemonic == ZYDIS_MNEMONIC_RDPID ||
(decoded.instruction.mnemonic == ZYDIS_MNEMONIC_MOV &&
operands[1].type == ZYDIS_OPERAND_TYPE_MEMORY))) {
decoded.replaces_stack_pointer = true;
} else if ((decoded.instruction.mnemonic == ZYDIS_MNEMONIC_ADD ||
decoded.instruction.mnemonic == ZYDIS_MNEMONIC_SUB) &&
@@ -653,13 +672,17 @@ void AnalyzeRedZoneLiveness(DecodedFunction& function) {
}
bool EncodeRelocatedInstruction(const DecodedCodeInstruction& decoded,
Xbyak::CodeGenerator& generator) {
Xbyak::CodeGenerator& generator, s64 stack_adjustment = 0,
ZydisMnemonic replacement = ZYDIS_MNEMONIC_INVALID) {
ZydisEncoderRequest request;
if (!ZYAN_SUCCESS(ZydisEncoderDecodedInstructionToEncoderRequest(
&decoded.instruction, decoded.operands.data(),
decoded.instruction.operand_count_visible, &request))) {
return false;
}
if (replacement != ZYDIS_MNEMONIC_INVALID) {
request.mnemonic = replacement;
}
for (u8 index = 0; index < decoded.instruction.operand_count_visible; ++index) {
const auto& operand = decoded.operands[index];
@@ -679,6 +702,9 @@ bool EncodeRelocatedInstruction(const DecodedCodeInstruction& decoded,
return false;
}
request.operands[index].mem.displacement = static_cast<ZyanI64>(absolute_address);
} else if (operand.type == ZYDIS_OPERAND_TYPE_MEMORY &&
IsStackPointerRegister(operand.mem.base)) {
request.operands[index].mem.displacement += stack_adjustment;
}
}
@@ -823,8 +849,9 @@ void GenerateReciprocalSquareRoot(const DecodedCodeInstruction& decoded,
generator.L(done);
}
bool IsSupportedExtractQ(const DecodedCodeInstruction& decoded) {
if (decoded.instruction.mnemonic != ZYDIS_MNEMONIC_EXTRQ) {
bool IsSupportedBitFieldInstruction(const DecodedCodeInstruction& decoded) {
const bool insert = decoded.instruction.mnemonic == ZYDIS_MNEMONIC_INSERTQ;
if (!insert && decoded.instruction.mnemonic != ZYDIS_MNEMONIC_EXTRQ) {
return false;
}
const auto is_xmm = [](const ZydisDecodedOperand& operand) {
@@ -832,18 +859,20 @@ bool IsSupportedExtractQ(const DecodedCodeInstruction& decoded) {
operand.reg.value >= ZYDIS_REGISTER_XMM0 &&
operand.reg.value <= ZYDIS_REGISTER_XMM15;
};
const bool immediate = decoded.instruction.operand_count_visible == 3 &&
decoded.operands[1].type == ZYDIS_OPERAND_TYPE_IMMEDIATE &&
decoded.operands[2].type == ZYDIS_OPERAND_TYPE_IMMEDIATE;
if (!is_xmm(decoded.operands[0]) ||
const size_t immediate_index = insert ? 2 : 1;
const bool immediate =
decoded.instruction.operand_count_visible == immediate_index + 2 &&
decoded.operands[immediate_index].type == ZYDIS_OPERAND_TYPE_IMMEDIATE &&
decoded.operands[immediate_index + 1].type == ZYDIS_OPERAND_TYPE_IMMEDIATE;
if (!is_xmm(decoded.operands[0]) || (insert && !is_xmm(decoded.operands[1])) ||
(!immediate &&
!(decoded.instruction.operand_count_visible == 2 && is_xmm(decoded.operands[1])))) {
return false;
}
// Match the trap handler's encoding support: 66 [REX] 0F 78/79 ModRM [imm8 imm8].
// Match the trap handler's encoding support: 66/F2 [REX] 0F 78/79 ModRM [imm8 imm8].
const auto* code = reinterpret_cast<const u8*>(decoded.address);
const size_t opcode = (code[1] & 0xf0u) == 0x40u ? 2 : 1;
return code[0] == 0x66 && code[opcode] == 0x0f &&
return code[0] == (insert ? 0xf2 : 0x66) && code[opcode] == 0x0f &&
code[opcode + 1] == (immediate ? 0x78 : 0x79) &&
decoded.instruction.length == opcode + (immediate ? 5 : 3);
}
@@ -890,8 +919,8 @@ void GenerateExtractQ(const DecodedCodeInstruction& decoded, Xbyak::CodeGenerato
void CollectAmdInstructions(const DecodedFunction& function,
std::map<uintptr_t, InstructionRewrite>& rewrite_sites,
GuestInstructionPatchResult& result) {
static const bool host_lacks_sse4a = !Xbyak::util::Cpu().has(Xbyak::util::Cpu::tSSE4a);
GuestInstructionPatchResult& result,
GuestInstructionHostFeatures host_features) {
for (const auto& [address, decoded]: function.instructions) {
const auto& instruction = decoded.instruction;
InstructionReplacement replacement {};
@@ -900,8 +929,14 @@ void CollectAmdInstructions(const DecodedFunction& function,
instruction.raw.vex.offset == 0 && decoded.operands[0].size == 128 &&
decoded.operands[1].type == ZYDIS_OPERAND_TYPE_REGISTER) {
replacement = InstructionReplacement::ReciprocalSquareRoot;
} else if (host_lacks_sse4a && IsSupportedExtractQ(decoded)) {
replacement = InstructionReplacement::ExtractQ;
} else if (!host_features.sse4a && IsSupportedBitFieldInstruction(decoded)) {
replacement = instruction.mnemonic == ZYDIS_MNEMONIC_EXTRQ
? InstructionReplacement::ExtractQ
: InstructionReplacement::InsertQ;
} else if (!host_features.rdpid && instruction.mnemonic == ZYDIS_MNEMONIC_RDPID) {
replacement = InstructionReplacement::ReadProcessorId;
} else if (!host_features.clwb && instruction.mnemonic == ZYDIS_MNEMONIC_CLWB) {
replacement = InstructionReplacement::CacheLineWriteBack;
} else {
continue;
}
@@ -916,16 +951,21 @@ void MarkInstructionTrap(u8* code, const ZydisDecodedInstruction& instruction,
// Clear a reserved VEX.vvvv bit, retaining both register operands for the handler.
code[instruction.raw.vex.size - 1] &= ~0x08u;
}
// EXTRQ already raises #UD on hosts without SSE4a; keep its original encoding.
// Other replacements already raise #UD on hosts without their CPU feature.
}
void GenerateInstructionTrap(const DecodedCodeInstruction& decoded,
InstructionReplacement replacement, Xbyak::CodeGenerator& generator) {
bool GenerateInstructionTrap(const DecodedCodeInstruction& decoded,
InstructionReplacement replacement, Xbyak::CodeGenerator& generator,
s64 stack_adjustment) {
if (replacement == InstructionReplacement::CacheLineWriteBack) {
return EncodeRelocatedInstruction(decoded, generator, stack_adjustment);
}
std::array<u8, ZYDIS_MAX_INSTRUCTION_LENGTH> code {};
std::memcpy(code.data(), reinterpret_cast<const void*>(decoded.address),
decoded.instruction.length);
MarkInstructionTrap(code.data(), decoded.instruction, replacement);
generator.db(code.data(), decoded.instruction.length);
return true;
}
bool NeedsTrapRedZoneProtection(const DecodedCodeInstruction& decoded) {
@@ -946,11 +986,11 @@ void TrapUnrelocatedInstructions(const PatchModule& module, const DecodedFunctio
}
const auto& decoded = function.instructions.at(address);
if (NeedsTrapRedZoneProtection(decoded)) {
if (rewrite.replacement == InstructionReplacement::ExtractQ) {
// Unlike VRSQRTPS, leaving EXTRQ unchanged would still trap and corrupt the red
// zone.
EXIT("AMD CPU compatibility: cannot safely trap EXTRQ at %p (guest red zone is "
if (rewrite.replacement != InstructionReplacement::ReciprocalSquareRoot) {
// Leaving an unsupported instruction unchanged still traps with a live red zone.
EXIT("AMD CPU compatibility: cannot safely trap %s at %p (guest red zone is "
"live)\n",
ZydisMnemonicGetString(decoded.instruction.mnemonic),
reinterpret_cast<void*>(address));
}
continue;
@@ -990,23 +1030,31 @@ void RelocateGuestInstructions(PatchModule* module, const DecodedFunction& funct
const auto replacement = rewrite != rewrite_sites.end()
? rewrite->second.replacement
: InstructionReplacement::None;
const bool uses_trap = UsesInstructionTrap(replacement, trap_replacements);
const bool protected_indirect_call =
rewrite != rewrite_sites.end() && rewrite->second.protected_indirect_call;
const bool protect_red_zone =
(rewrite != rewrite_sites.end() && rewrite->second.protect_red_zone &&
!protected_indirect_call) ||
(replacement != InstructionReplacement::None && trap_replacements &&
((uses_trap || replacement == InstructionReplacement::CacheLineWriteBack) &&
NeedsTrapRedZoneProtection(*decoded));
if (protect_red_zone) {
generator.lea(rsp, ptr[rsp - GuestRedZoneSize]);
}
if (replacement != InstructionReplacement::None) {
if (trap_replacements) {
GenerateInstructionTrap(*decoded, replacement, generator);
if (uses_trap) {
encoded = GenerateInstructionTrap(*decoded, replacement, generator,
protect_red_zone ? GuestRedZoneSize : 0);
} else if (replacement == InstructionReplacement::ReciprocalSquareRoot) {
GenerateReciprocalSquareRoot(*decoded, generator);
} else {
} else if (replacement == InstructionReplacement::ExtractQ) {
GenerateExtractQ(*decoded, generator);
} else {
// CLFLUSH writes back dirty data too; invalidation is a permitted stronger
// action.
encoded = EncodeRelocatedInstruction(
*decoded, generator, protect_red_zone ? GuestRedZoneSize : 0,
ZYDIS_MNEMONIC_CLFLUSH);
}
} else if (protected_indirect_call) {
encoded = GenerateProtectedIndirectCall(*decoded, generator);
@@ -1053,7 +1101,7 @@ void RelocateGuestInstructions(PatchModule* module, const DecodedFunction& funct
}
if (rewrite->second.replacement != InstructionReplacement::None) {
auto& counts = ReplacementCounts(result, rewrite->second.replacement);
if (span.trap_replacements) {
if (UsesInstructionTrap(rewrite->second.replacement, span.trap_replacements)) {
++counts.trapped;
} else {
++counts.native;
@@ -1425,9 +1473,26 @@ void RelocateGuestInstructions(PatchModule* module, const DecodedFunction& funct
}
} // namespace
GuestInstructionHostFeatures GetGuestInstructionHostFeatures() {
static const auto features = [] {
const Xbyak::util::Cpu cpu;
uint32_t leaf[4] {};
Xbyak::util::Cpu::getCpuid(0, leaf);
bool rdpid = false;
if (leaf[0] >= 7) {
Xbyak::util::Cpu::getCpuidEx(7, 0, leaf);
rdpid = (leaf[2] & (1u << 22)) != 0;
}
return GuestInstructionHostFeatures {cpu.has(Xbyak::util::Cpu::tSSE4a), rdpid,
cpu.has(Xbyak::util::Cpu::tCLWB)};
}();
return features;
}
GuestInstructionPatchResult PatchGuestInstructions(u64 segment_addr, u64 segment_size,
std::span<const uintptr_t> function_starts,
bool protect_memory, bool emulate_amd) {
bool protect_memory, bool emulate_amd,
GuestInstructionHostFeatures host_features) {
GuestInstructionPatchResult result {};
if (!protect_memory && !emulate_amd) {
return result;
@@ -1480,7 +1545,7 @@ GuestInstructionPatchResult PatchGuestInstructions(u64 segment_addr, u64 segment
CollectRedZoneMemoryInstructions(function, rewrite_sites, result);
}
if (emulate_amd) {
CollectAmdInstructions(function, rewrite_sites, result);
CollectAmdInstructions(function, rewrite_sites, result, host_features);
}
if (!rewrite_sites.empty()) {
RelocateGuestInstructions(module, function, rewrite_sites, result);
@@ -1499,8 +1564,12 @@ GuestInstructionPatchResult PatchGuestInstructions(u64 segment_addr, u64 segment
#else
GuestInstructionPatchResult PatchGuestInstructions(u64, u64, std::span<const uintptr_t>, bool,
bool) {
GuestInstructionHostFeatures GetGuestInstructionHostFeatures() {
return {};
}
GuestInstructionPatchResult PatchGuestInstructions(u64, u64, std::span<const uintptr_t>, bool, bool,
GuestInstructionHostFeatures) {
return {};
}
src/loader/guestInstructionPatcher.h
+15 -3
View File
@@ -11,6 +11,14 @@
namespace Loader {
struct GuestInstructionHostFeatures {
bool sse4a = false;
bool rdpid = false;
bool clwb = false;
};
GuestInstructionHostFeatures GetGuestInstructionHostFeatures();
struct InstructionPatchCounts {
uint64_t found = 0;
uint64_t native = 0;
@@ -39,6 +47,9 @@ struct GuestInstructionPatchResult {
uint64_t indirect_red_zone_function_count = 0;
InstructionPatchCounts reciprocal_sqrt;
InstructionPatchCounts extrq;
InstructionPatchCounts insertq;
InstructionPatchCounts rdpid;
InstructionPatchCounts clwb;
};
void RegisterGuestInstructionPatchModule(void* module_ptr, uint64_t module_size,
@@ -46,9 +57,10 @@ void RegisterGuestInstructionPatchModule(void* module_ptr, uint64_t module_size,
void UnregisterGuestInstructionPatchModule(void* module_ptr);
// Apply enabled instruction fixes using native trampolines or safe trap fallbacks.
GuestInstructionPatchResult PatchGuestInstructions(uint64_t segment_addr, uint64_t segment_size,
std::span<const uintptr_t> function_starts,
bool protect_memory, bool emulate_amd);
GuestInstructionPatchResult PatchGuestInstructions(
uint64_t segment_addr, uint64_t segment_size, std::span<const uintptr_t> function_starts,
bool protect_memory, bool emulate_amd,
GuestInstructionHostFeatures host_features = GetGuestInstructionHostFeatures());
bool DecodeEhFrameFunctionStarts(uint64_t eh_frame_header_addr, uint64_t eh_frame_header_size,
std::vector<uintptr_t>* function_starts);
src/loader/runtimeLinker.cpp
+19 -8
View File
@@ -1949,6 +1949,9 @@ void RuntimeLinker::LoadProgramToMemory(Program* program) {
protect_memory_faults, emulate_amd);
totals.reciprocal_sqrt += result.reciprocal_sqrt;
totals.extrq += result.extrq;
totals.insertq += result.insertq;
totals.rdpid += result.rdpid;
totals.clwb += result.clwb;
if (protect_memory_faults) {
LOGF("Windows guest red-zone patching: %s, functions=%" PRIu64 ", red_zone=%" PRIu64
", memory=%" PRIu64 ", patched=%" PRIu64 ", short=%" PRIu64 ", stack=%" PRIu64
@@ -1963,18 +1966,26 @@ void RuntimeLinker::LoadProgramToMemory(Program* program) {
}
}
if (emulate_amd && have_function_starts) {
const auto found = totals.reciprocal_sqrt.found + totals.extrq.found;
const auto skipped = totals.reciprocal_sqrt.Skipped() + totals.extrq.Skipped();
InstructionPatchCounts combined {};
std::string details;
for (const auto& [name, counts]: {std::pair {"VRSQRTPS", totals.reciprocal_sqrt},
{"EXTRQ", totals.extrq},
{"INSERTQ", totals.insertq},
{"RDPID", totals.rdpid},
{"CLWB", totals.clwb}}) {
combined += counts;
if (!details.empty()) details += "; ";
details += fmt::format("{}: native={}, trapped={}, skipped={}", name, counts.native,
counts.trapped, counts.Skipped());
}
const auto found = combined.found;
const auto skipped = combined.Skipped();
const char* status = found == 0 ? "no matching instructions"
: skipped == found ? "not patched"
: skipped != 0 ? "partially patched"
: "patched";
Log::WriteToConsoleAndLog(fmt::format(
"AMD CPU compatibility: {} {} (VRSQRTPS: native={}, trapped={}, skipped={}; "
"EXTRQ: native={}, trapped={}, skipped={})\n",
module_name, status, totals.reciprocal_sqrt.native, totals.reciprocal_sqrt.trapped,
totals.reciprocal_sqrt.Skipped(), totals.extrq.native, totals.extrq.trapped,
totals.extrq.Skipped()));
Log::WriteToConsoleAndLog(
fmt::format("AMD CPU compatibility: {} {} ({})\n", module_name, status, details));
}
}
src/loader/x64InstructionEmulator.cpp
+103 -23
View File
@@ -2,12 +2,12 @@
#include "common/common.h"
#include <Zydis/Zydis.h>
#include <bit>
#include <cstring>
#if !defined(__APPLE__)
#include <emmintrin.h>
#include <xmmintrin.h>
#endif
#include <cpuid.h>
#if KYTY_PLATFORM == KYTY_PLATFORM_WINDOWS
#include <windows.h> // IWYU pragma: keep
@@ -406,12 +406,12 @@ struct Context {
void Advance(size_t length) { native->Rip += length; }
[[nodiscard]] void* Xmm(uint8_t index) const { return &native->Xmm0 + index; }
void LoadGprs(uint64_t (&gpr)[16]) const {
const uint64_t registers[] = {native->Rax, native->Rcx, native->Rdx, native->Rbx,
native->Rsp, native->Rbp, native->Rsi, native->Rdi,
native->R8, native->R9, native->R10, native->R11,
native->R12, native->R13, native->R14, native->R15};
std::memcpy(gpr, registers, sizeof(gpr));
[[nodiscard]] auto& Gpr(uint8_t index) const {
DWORD64* registers[] = {&native->Rax, &native->Rcx, &native->Rdx, &native->Rbx,
&native->Rsp, &native->Rbp, &native->Rsi, &native->Rdi,
&native->R8, &native->R9, &native->R10, &native->R11,
&native->R12, &native->R13, &native->R14, &native->R15};
return *registers[index];
}
void ClearUpperYmm(uint8_t index) const {
@@ -437,6 +437,14 @@ struct Context {
void Advance(size_t length) {
native->uc_mcontext->__ss.__rip += static_cast<uint64_t>(length);
}
[[nodiscard]] auto& Gpr(uint8_t index) const {
auto* state = &native->uc_mcontext->__ss;
uint64_t* registers[] = {&state->__rax, &state->__rcx, &state->__rdx, &state->__rbx,
&state->__rsp, &state->__rbp, &state->__rsi, &state->__rdi,
&state->__r8, &state->__r9, &state->__r10, &state->__r11,
&state->__r12, &state->__r13, &state->__r14, &state->__r15};
return *registers[index];
}
// Darwin names the XMM file __fpu_xmm0..__fpu_xmm15 instead of exposing an array.
[[nodiscard]] void* Xmm(uint8_t index) const {
auto* fs = &native->uc_mcontext->__fs;
@@ -476,13 +484,11 @@ struct Context {
return native->uc_mcontext.fpregs->_xmm[index].element;
}
void LoadGprs(uint64_t (&gpr)[16]) const {
[[nodiscard]] auto& Gpr(uint8_t index) const {
constexpr int registers[] = {REG_RAX, REG_RCX, REG_RDX, REG_RBX, REG_RSP, REG_RBP,
REG_RSI, REG_RDI, REG_R8, REG_R9, REG_R10, REG_R11,
REG_R12, REG_R13, REG_R14, REG_R15};
for (size_t i = 0; i < 16; ++i) {
gpr[i] = static_cast<uint64_t>(native->uc_mcontext.gregs[registers[i]]);
}
return native->uc_mcontext.gregs[registers[index]];
}
void ClearUpperYmm(uint8_t index) const {
@@ -506,6 +512,12 @@ struct Context {
}
}
#endif
void LoadGprs(uint64_t (&gpr)[16]) const {
for (uint8_t i = 0; i < 16; ++i) {
gpr[i] = static_cast<uint64_t>(Gpr(i));
}
}
};
#if !defined(__APPLE__)
@@ -574,8 +586,8 @@ static bool TryEmulateSse4a(Context& context) {
if (rip[offset] != 0x0f) {
return false;
}
const bool register_extract = prefix == 0x66 && rip[offset + 1] == 0x79;
if (rip[offset + 1] != 0x78 && !register_extract) {
const bool register_form = rip[offset + 1] == 0x79;
if (rip[offset + 1] != 0x78 && !register_form) {
return false;
}
@@ -589,7 +601,7 @@ static bool TryEmulateSse4a(Context& context) {
// Immediate EXTRQ encodes its destination in r/m; the two-register form uses reg.
uint8_t dest_index = reg;
if (prefix == 0x66 && !register_extract) {
if (prefix == 0x66 && !register_form) {
dest_index = rm;
}
auto* dest_xmm = context.Xmm(dest_index);
@@ -598,15 +610,17 @@ static bool TryEmulateSse4a(Context& context) {
return false;
}
uint64_t dest[2] {};
uint64_t source = 0;
uint64_t source[2] {};
std::memcpy(dest, dest_xmm, sizeof(dest));
std::memcpy(&source, src_xmm, sizeof(source));
std::memcpy(source, src_xmm, sizeof(source));
uint8_t length = 0;
uint8_t index = 0;
size_t instruction_length = offset + 3;
if (register_extract) {
length = static_cast<uint8_t>(source);
index = static_cast<uint8_t>(source >> 8u);
if (register_form) {
// EXTRQ controls occupy the low quadword; INSERTQ uses the upper quadword.
const uint64_t controls = source[prefix == 0x66 ? 0 : 1];
length = static_cast<uint8_t>(controls);
index = static_cast<uint8_t>(controls >> 8u);
} else {
length = rip[offset + 3];
index = rip[offset + 4];
@@ -616,13 +630,79 @@ static bool TryEmulateSse4a(Context& context) {
dest[0] = ExtractBitField(dest[0], length, index);
dest[1] = 0;
} else {
dest[0] = InsertBitField(dest[0], source, length, index);
dest[0] = InsertBitField(dest[0], source[0], length, index);
}
std::memcpy(dest_xmm, dest, sizeof(dest));
context.Advance(instruction_length);
return true;
}
static bool ReadTscAux(uint32_t& auxiliary) {
uint32_t eax = 0;
uint32_t ebx = 0;
uint32_t ecx = 0;
uint32_t edx = 0;
if (!__get_cpuid(0x80000001u, &eax, &ebx, &ecx, &edx) || (edx & (1u << 27u)) == 0) {
return false;
}
asm volatile("rdtscp" : "=a"(eax), "=d"(edx), "=c"(auxiliary));
return true;
}
static void FlushCacheLine(uint64_t address, ZydisRegister segment) {
// Preserve FS/GS addressing rather than treating a segment-relative offset as a pointer.
if (segment == ZYDIS_REGISTER_FS) {
asm volatile("clflush %%fs:(%0)" : : "r"(address) : "memory");
} else if (segment == ZYDIS_REGISTER_GS) {
asm volatile("clflush %%gs:(%0)" : : "r"(address) : "memory");
} else {
_mm_clflush(reinterpret_cast<const void*>(address));
}
}
static bool TryEmulateCpuExtensions(Context& context) {
ZydisDecoder decoder {};
ZydisDecodedInstruction instruction {};
ZydisDecodedOperand operands[ZYDIS_MAX_OPERAND_COUNT] {};
if (!ZYAN_SUCCESS(
ZydisDecoderInit(&decoder, ZYDIS_MACHINE_MODE_LONG_64, ZYDIS_STACK_WIDTH_64)) ||
!ZYAN_SUCCESS(ZydisDecoderDecodeFull(&decoder, reinterpret_cast<const void*>(context.Rip()),
ZYDIS_MAX_INSTRUCTION_LENGTH, &instruction,
operands))) {
return false;
}
if (instruction.mnemonic == ZYDIS_MNEMONIC_RDPID &&
operands[0].type == ZYDIS_OPERAND_TYPE_REGISTER) {
const auto reg_class = ZydisRegisterGetClass(operands[0].reg.value);
const auto index = ZydisRegisterGetId(operands[0].reg.value);
uint32_t auxiliary = 0;
if ((reg_class != ZYDIS_REGCLASS_GPR32 && reg_class != ZYDIS_REGCLASS_GPR64) || index < 0 ||
index >= 16 || !ReadTscAux(auxiliary)) {
return false;
}
// RDPID and guest RDTSCP observe the same host TSC_AUX; only RDPID's destination changes.
context.Gpr(static_cast<uint8_t>(index)) = auxiliary;
} else if (instruction.mnemonic == ZYDIS_MNEMONIC_CLWB &&
operands[0].type == ZYDIS_OPERAND_TYPE_MEMORY) {
ZydisRegisterContext registers {};
for (uint8_t i = 0; i < 16; ++i) {
const auto value = static_cast<uint64_t>(context.Gpr(i));
registers.values[ZYDIS_REGISTER_RAX + i] = value;
registers.values[ZYDIS_REGISTER_EAX + i] = static_cast<uint32_t>(value);
}
uint64_t address = 0;
if (!ZYAN_SUCCESS(ZydisCalcAbsoluteAddressEx(&instruction, &operands[0], context.Rip(),
&registers, &address))) {
return false;
}
FlushCacheLine(address, operands[0].mem.segment);
} else {
return false;
}
context.Advance(instruction.length);
return true;
}
#if !defined(__APPLE__)
static bool TryEmulateMonitorxMwaitx(Context& context) {
@@ -729,9 +809,9 @@ bool TryEmulate(void* native_context) {
#endif
#if !defined(__APPLE__)
return TryEmulateReciprocalSquareRoot(context) || TryEmulateMonitorxMwaitx(context) ||
TryEmulateSse4a(context) || TryEmulateShaNi(context);
TryEmulateSse4a(context) || TryEmulateShaNi(context) || TryEmulateCpuExtensions(context);
#else
return TryEmulateSse4a(context);
return TryEmulateSse4a(context) || TryEmulateCpuExtensions(context);
#endif
}
tests/VirtualMemoryAllocationTests.cpp
+384 -38
View File
@@ -26,16 +26,25 @@
#include <filesystem>
#include <string>
#include <thread>
#include <utility>
#include <vector>
#if defined(__linux__) || KYTY_PLATFORM == KYTY_PLATFORM_WINDOWS
#include <csignal>
#include <immintrin.h>
#if defined(_MSC_VER)
#include <intrin.h>
#else
#include <x86intrin.h>
#endif
#include <Zydis/Zydis.h>
#include <xbyak/xbyak.h>
#include <xbyak/xbyak_util.h>
#endif
#if defined(__linux__)
#include <asm/prctl.h>
#include <sys/syscall.h>
#include <sys/uio.h>
#include <ucontext.h>
#include <unistd.h>
@@ -2900,6 +2909,140 @@ struct InstructionTestScope {
}
};
struct SavedInstructionContext {
#if KYTY_PLATFORM == KYTY_PLATFORM_WINDOWS
CONTEXT native {};
auto& Gpr(size_t index) {
const std::array registers {&native.Rax, &native.Rcx, &native.Rdx, &native.Rbx,
&native.Rsp, &native.Rbp, &native.Rsi, &native.Rdi,
&native.R8, &native.R9, &native.R10, &native.R11,
&native.R12, &native.R13, &native.R14, &native.R15};
return *registers[index];
}
auto& Rip() { return native.Rip; }
auto& Flags() { return native.EFlags; }
void* Xmm() { return &native.Xmm0; }
#else
ucontext_t native {};
_libc_fpstate fpstate {};
SavedInstructionContext() { native.uc_mcontext.fpregs = &fpstate; }
auto& Gpr(size_t index) {
constexpr std::array registers {REG_RAX, REG_RCX, REG_RDX, REG_RBX, REG_RSP, REG_RBP,
REG_RSI, REG_RDI, REG_R8, REG_R9, REG_R10, REG_R11,
REG_R12, REG_R13, REG_R14, REG_R15};
return native.uc_mcontext.gregs[registers[index]];
}
auto& Rip() { return native.uc_mcontext.gregs[REG_RIP]; }
auto& Flags() { return native.uc_mcontext.gregs[REG_EFL]; }
void* Xmm() { return fpstate._xmm; }
#endif
};
void TestCpuExtensionContexts() {
const char* test = "CpuExtensionContexts";
SavedInstructionContext context;
constexpr uint64_t sentinel = 0x8192a3b4c5d6e700ull;
std::array<uint8_t, 256> xmm;
for (size_t i = 0; i < xmm.size(); ++i) {
xmm[i] = static_cast<uint8_t>(i);
}
std::memcpy(context.Xmm(), xmm.data(), xmm.size());
const auto check_state = [&](const std::array<uint64_t, 16>& expected) {
for (size_t reg = 0; reg < expected.size(); ++reg) {
Check(test, static_cast<uint64_t>(context.Gpr(reg)) == expected[reg],
"CPU extension changed an unexpected GPR");
}
Check(test,
context.Flags() == 0x897 && std::memcmp(context.Xmm(), xmm.data(), xmm.size()) == 0,
"CPU extension changed RFLAGS or XMM state");
};
for (uint8_t encoding = 0; encoding < 32; ++encoding) {
const uint8_t destination = encoding & 15;
bool checked = false;
for (size_t attempt = 0; attempt < 100 && !checked; ++attempt) {
std::array<uint64_t, 16> expected;
for (size_t reg = 0; reg < expected.size(); ++reg) {
context.Gpr(reg) = expected[reg] = sentinel | reg;
}
context.Flags() = 0x897;
std::array<uint8_t, 16> instruction {
0xf3, static_cast<uint8_t>(0x40 | ((encoding >> 4) << 3) | (destination >> 3)),
0x0f, 0xc7, static_cast<uint8_t>(0xf8 | (destination & 7))};
context.Rip() = reinterpret_cast<uintptr_t>(instruction.data());
uint32_t before = 0;
uint32_t after = 0;
__rdtscp(&before);
Check(test, EmulateInstructionContext(&context.native), "RDPID fallback was rejected");
__rdtscp(&after);
Check(test, context.Rip() == reinterpret_cast<uintptr_t>(instruction.data() + 5),
"RDPID advanced RIP incorrectly");
if (before == after) {
expected[destination] = before;
check_state(expected);
checked = true;
}
}
Check(test, checked, "thread migrated repeatedly during RDPID comparison");
}
constexpr uint64_t size = 0x4000;
const auto mapping = Libs::LibKernel::Memory::AllocateProgramMemory(
0x30000000, size, Common::VirtualMemory::Mode::ReadWrite, "clwb_context_test");
Check(test, mapping != 0 && mapping + size <= UINT32_MAX,
"failed to allocate low CLWB fixture");
InstructionTestScope restore {mapping, size};
auto* instruction = reinterpret_cast<uint8_t*>(mapping);
auto* data = instruction + 0x200;
std::memset(data, 0xa5, 64);
const auto flush = [&](std::initializer_list<uint8_t> bytes, int base, uint64_t value,
int index = -1, uint64_t index_value = 0) {
std::array<uint64_t, 16> expected;
for (size_t reg = 0; reg < expected.size(); ++reg) {
context.Gpr(reg) = expected[reg] = sentinel | reg;
}
if (base >= 0) {
context.Gpr(base) = expected[base] = value;
}
if (index >= 0) {
context.Gpr(index) = expected[index] = index_value;
}
context.Flags() = 0x897;
std::fill_n(instruction, 16, 0);
std::copy(bytes.begin(), bytes.end(), instruction);
context.Rip() = mapping;
Check(test, EmulateInstructionContext(&context.native), "CLWB fallback was rejected");
Check(test, static_cast<uint64_t>(context.Rip()) == mapping + bytes.size(),
"CLWB advanced RIP incorrectly");
check_state(expected);
Check(test, std::all_of(data, data + 64, [](uint8_t byte) { return byte == 0xa5; }),
"CLWB changed guest memory");
};
const auto address = reinterpret_cast<uintptr_t>(data);
flush({0x66, 0x0f, 0xae, 0x37}, 7, address); // [rdi]
flush({0x66, 0x0f, 0xae, 0x74, 0x24, 0xf8}, 4, address + 8); // [rsp-8]
flush({0x66, 0x41, 0x0f, 0xae, 0x34, 0x24}, 12, address); // [r12]
flush({0x66, 0x43, 0x0f, 0xae, 0x74, 0xac, 0x20}, 12, address - 0x20 - 12, 13,
3); // [r12+r13*4+32]
flush({0x66, 0x41, 0x0f, 0xae, 0x35, 0xf7, 1, 0, 0}, -1, 0); // RIP+503; REX.B ignored
flush({0x67, 0x66, 0x0f, 0xae, 0x37}, 7, address | (uint64_t {1} << 32)); // [edi]
const auto byte = [&](int shift) { return static_cast<uint8_t>(address >> shift); };
flush({0x67, 0x66, 0x0f, 0xae, 0x35, 0xf7, 1, 0, 0}, -1, 0); // EIP+503
flush({0x66, 0x41, 0x0f, 0xae, 0x34, 0x25, byte(0), byte(8), byte(16), byte(24)}, -1,
0); // SIB absolute; REX.B ignored
#if defined(__linux__)
for (const auto segment:
{std::pair {ARCH_GET_FS, uint8_t {0x64}}, std::pair {ARCH_GET_GS, uint8_t {0x65}}}) {
unsigned long base = 0;
Check(test, syscall(SYS_arch_prctl, segment.first, &base) == 0,
"failed to read segment base");
flush({segment.second, 0x66, 0x0f, 0xae, 0x37}, 7, address - base);
}
#else
flush({0x65, 0x66, 0x0f, 0xae, 0x37}, 7, address - reinterpret_cast<uintptr_t>(NtCurrentTeb()));
#endif
std::printf("[host] %-48s ok\n", test);
}
void TestPackedReciprocalSquareRoot() {
const char* test = "PackedReciprocalSquareRoot";
constexpr uint64_t code_size = 0x4000;
@@ -3229,12 +3372,12 @@ void TestPackedReciprocalSquareRoot() {
std::printf("[host] %-48s ok\n", test);
}
void TestPackedBitFieldExtract() {
const char* test = "PackedBitFieldExtract";
void TestPackedBitField(bool insert) {
const char* test = insert ? "PackedBitFieldInsert" : "PackedBitFieldExtract";
constexpr uint64_t code_size = 0x80000;
const auto mapping = Libs::LibKernel::Memory::AllocateProgramMemory(
0x904000000, code_size * 2, Common::VirtualMemory::Mode::ExecuteReadWrite, "extrq_test");
Check(test, mapping != 0, "failed to allocate EXTRQ test code");
0x904000000, code_size * 2, Common::VirtualMemory::Mode::ExecuteReadWrite, "bitfield_test");
Check(test, mapping != 0, "failed to allocate SSE4a bitfield test code");
InstructionTestScope restore {mapping, code_size * 2};
restore.InstallHandler(test);
const bool host_sse4a = Xbyak::util::Cpu().has(Xbyak::util::Cpu::tSSE4a);
@@ -3243,19 +3386,23 @@ void TestPackedBitFieldExtract() {
reinterpret_cast<void*>(mapping + code_size),
capacity);
};
// Xbyak has no EXTRQ mnemonic. Emit the two canonical SSE4a encodings directly.
const auto extrq = [](Xbyak::CodeGenerator& code, int destination, int source,
uint8_t length = 0, uint8_t index = 0) {
code.db(0x66);
// Xbyak has no SSE4a bitfield mnemonics. A negative source selects the immediate form.
const auto emit_bit_field = [insert](Xbyak::CodeGenerator& code, int destination, int source,
uint8_t length = 0, uint8_t index = 0) {
const bool immediate = source < 0;
if (insert && immediate) {
source = 9;
}
code.db(insert ? 0xf2 : 0x66);
const int rex = source < 0 ? (destination >> 3) : ((destination >> 3) << 2) | (source >> 3);
if (rex != 0) {
code.db(0x40 | rex);
}
code.db(0x0f);
code.db(source < 0 ? 0x78 : 0x79);
code.db(immediate ? 0x78 : 0x79);
code.db(source < 0 ? 0xc0 | (destination & 7)
: 0xc0 | ((destination & 7) << 3) | (source & 7));
if (source < 0) {
if (immediate) {
code.db(length);
code.db(index);
}
@@ -3270,6 +3417,16 @@ void TestPackedBitFieldExtract() {
}
return result;
};
const auto insert_bits = [](uint64_t destination, uint64_t source, uint32_t length,
uint32_t index) {
const uint32_t count = (length & 63u) == 0 ? 64 : length & 63u;
const uint32_t first = index & 63u;
for (uint32_t bit = 0; bit < count && first + bit < 64; ++bit) {
const uint64_t mask = uint64_t {1} << (first + bit);
destination = (destination & ~mask) | (((source >> bit) & 1u) << (first + bit));
}
return destination;
};
struct Result {
std::array<uint64_t, 4> destination;
std::array<uint64_t, 4> source;
@@ -3299,7 +3456,8 @@ void TestPackedBitFieldExtract() {
for (const auto registers: {std::array {1, 8}, std::array {8, 9}, std::array {3, 3},
std::array {15, 0}, std::array {8, -1}}) {
const auto destination = registers[0];
const auto source = registers[1];
const auto source = insert && registers[1] < 0 ? 9 : registers[1];
const bool immediate = registers[1] < 0;
Xbyak::CodeGenerator code(code_size, reinterpret_cast<void*>(mapping));
const std::array saved {code.rbx, code.rbp, code.r12, code.r13, code.r14, code.r15};
const std::array gprs {code.rax, code.rcx, code.rdx, code.rbx, code.rbp, code.r8, code.r9,
@@ -3322,7 +3480,7 @@ void TestPackedBitFieldExtract() {
code.mov(gprs[index], gpr_sentinel | index);
}
code.mov(code.qword[code.rsi + offsetof(Result, stack_before)], code.rsp);
extrq(code, destination, source, 0xc8, 0xc4);
emit_bit_field(code, destination, registers[1], 0xc8, 0xc4);
code.mov(code.qword[code.rsi + offsetof(Result, stack_after)], code.rsp);
for (size_t index = 0; index < gprs.size(); ++index) {
code.mov(code.qword[code.rsi + offsetof(Result, gprs) + index * 8], gprs[index]);
@@ -3344,7 +3502,7 @@ void TestPackedBitFieldExtract() {
code.pop(*reg);
}
code.ret();
Check(test, Xbyak::GetError() == 0, "failed to generate EXTRQ state fixture");
Check(test, Xbyak::GetError() == 0, "failed to generate SSE4a bitfield state fixture");
const std::vector<uint8_t> original(code.getCode(), code.getCurr());
register_module(code_size);
const std::vector<uint8_t> trampoline(code.getCode() + code_size,
@@ -3352,45 +3510,51 @@ void TestPackedBitFieldExtract() {
const auto disabled =
Loader::PatchGuestInstructions(mapping, code.getSize(), function_starts, false, false);
Check(test,
disabled.extrq.found == 0 &&
disabled.extrq.found == 0 && disabled.insertq.found == 0 &&
std::equal(original.begin(), original.end(), code.getCode()) &&
std::equal(trampoline.begin(), trampoline.end(), code.getCode() + code_size),
"disabled AMD patching changed EXTRQ code or trampolines");
"disabled AMD patching changed SSE4a bitfield code or trampolines");
for (const bool fallback: {false, true}) {
if (insert && fallback) {
continue; // INSERTQ always uses the trap path.
}
std::memcpy(reinterpret_cast<void*>(mapping), original.data(), original.size());
register_module(fallback ? 32 : code_size);
const auto counts = Loader::PatchGuestInstructions(mapping, code.getSize(),
function_starts, false, true)
.extrq;
const auto patched = Loader::PatchGuestInstructions(mapping, code.getSize(),
function_starts, false, true);
const auto counts = insert ? patched.insertq : patched.extrq;
if (host_sse4a) {
Check(test,
counts.found == 0 &&
std::equal(original.begin(), original.end(), code.getCode()),
"SSE4a host received unnecessary EXTRQ patches");
"SSE4a host received unnecessary SSE4a bitfield patches");
std::printf("[host] %-48s skipped (native SSE4a host)\n", test);
return;
}
std::printf("[host] EXTRQ xmm%d,%d %s found=%" PRIu64 " native=%" PRIu64
std::printf("[host] bitfield xmm%d,%d %s found=%" PRIu64 " native=%" PRIu64
" trapped=%" PRIu64 "\n",
destination, source, fallback ? "fallback" : "replacement", counts.found,
counts.native, counts.trapped);
Check(test,
Xbyak::GetError() == 0 && counts.found == 1 && counts.Skipped() == 0 &&
counts.native == (fallback ? 0u : 1u) &&
counts.trapped == (fallback ? 1u : 0u),
"EXTRQ native/fallback patch counts differ");
counts.native == (fallback || insert ? 0u : 1u) &&
counts.trapped == (fallback || insert ? 1u : 0u),
"SSE4a bitfield native/fallback patch counts differ");
const auto traps_before = g_instruction_traps;
uint64_t calls = 0;
for (uint32_t length = 0; length < 64; ++length) {
for (uint32_t index = 0; index < 64; ++index) {
if (source < 0 && (length != 8 || index != 4)) {
if (immediate && (length != 8 || index != 4)) {
continue;
}
for (const uint64_t pattern: {UINT64_MAX, uint64_t {0x0123456789abcdef}}) {
const uint64_t controls = (pattern & ~uint64_t {0xffff}) |
((index | 0xc0u) << 8) | (length | 0xc0u);
input[0] = source == destination ? controls : pattern;
input[4] = controls;
input[0] = !insert && source == destination ? controls : pattern;
input[4] = insert ? ~pattern : controls;
if (insert) {
input[source == destination ? 1 : 5] = controls;
}
Result output {};
_mm_setcsr(0x5fa5);
function(input.data(), &output);
@@ -3398,38 +3562,47 @@ void TestPackedBitFieldExtract() {
_mm_setcsr(restore.mxcsr);
++calls;
Check(test,
output.destination[0] == extract(input[0], length, index) &&
output.destination[1] == 0 && output.destination[2] == input[2] &&
output.destination[0] ==
(insert ? insert_bits(input[0],
input[source == destination ? 0 : 4],
length, index)
: extract(input[0], length, index)) &&
output.destination[1] == (insert ? input[1] : 0) &&
output.destination[2] == input[2] &&
output.destination[3] == input[3],
"EXTRQ result or destination lanes differ");
"SSE4a bitfield result or destination lanes differ");
Check(test,
source < 0 || source == destination
? output.source == output.destination
: std::equal(output.source.begin(), output.source.end(),
input.begin() + 4),
"EXTRQ changed its separate source register");
"SSE4a bitfield changed its separate source register");
Check(test,
mxcsr == 0x5fa5 && (output.flags & 0x8d5u) == 0x895u &&
output.stack_before == output.stack_after,
"EXTRQ changed MXCSR, RFLAGS or RSP");
"SSE4a bitfield changed MXCSR, RFLAGS or RSP");
for (size_t reg = 0; reg < output.gprs.size(); ++reg) {
Check(test, output.gprs[reg] == (gpr_sentinel | reg),
"EXTRQ corrupted a guest GPR");
"SSE4a bitfield corrupted a guest GPR");
}
for (size_t slot = 0; slot < output.red_zone.size(); ++slot) {
Check(test,
output.red_zone[slot] == (red_zone_sentinel | ((slot + 1) * 8)),
"EXTRQ corrupted the guest red zone");
"SSE4a bitfield corrupted the guest red zone");
}
}
}
}
Check(test,
static_cast<uint64_t>(g_instruction_traps - traps_before) ==
(fallback ? calls : 0),
"EXTRQ execution did not match native/fallback selection");
(fallback || insert ? calls : 0),
"SSE4a bitfield execution did not match native/fallback selection");
}
}
if (insert) {
std::printf("[host] %-48s ok (all 4096 length/index pairs)\n", test);
return;
}
// Cover every immediate length/index pair without re-decoding thousands of large fixtures.
Xbyak::CodeGenerator code(code_size, reinterpret_cast<void*>(mapping));
@@ -3438,7 +3611,7 @@ void TestPackedBitFieldExtract() {
for (uint32_t index = 0; index < 64; ++index) {
starts.push_back(mapping + code.getSize());
code.vmovups(code.ymm8, code.ptr[code.rdi]);
extrq(code, 8, -1, length | 0xc0, index | 0xc0);
emit_bit_field(code, 8, -1, length | 0xc0, index | 0xc0);
code.vmovups(code.ptr[code.rsi], code.ymm8);
code.vzeroupper();
code.ret();
@@ -3480,7 +3653,7 @@ void TestPackedBitFieldExtract() {
code.vmovups(code.ymm1, code.ptr[code.rdi]);
code.vmovups(code.ymm2, code.ptr[code.rdi + 32]);
code.lea(code.rax, code.ptr[code.rip + continuation]);
extrq(code, 1, 2);
emit_bit_field(code, 1, 2);
code.jmp(code.rax);
code.L(continuation);
code.vmovups(code.ptr[code.rsi], code.ymm1);
@@ -3508,7 +3681,7 @@ void TestPackedBitFieldExtract() {
code.vmovups(code.ymm1, code.ptr[code.rdi]);
code.vmovups(code.ymm2, code.ptr[code.rdi + 32]);
code.vmovups(code.ymm3, code.ptr[code.rdi + 64]);
extrq(code, 1, 2);
emit_bit_field(code, 1, 2);
code.vrsqrtps(code.xmm4, code.xmm3);
code.vmovups(code.ptr[code.rsi], code.ymm1);
code.vmovups(code.ptr[code.rsi + 32], code.ymm4);
@@ -3538,6 +3711,170 @@ void TestPackedBitFieldExtract() {
std::printf("[host] %-48s ok (all 4096 length/index pairs)\n", test);
}
void TestPackedBitFieldExtract() {
TestPackedBitField(false);
}
void TestPackedBitFieldInsert() {
TestPackedBitField(true);
}
void TestCpuExtensionPatches() {
const char* test = "CpuExtensionPatches";
constexpr uint64_t code_size = 0x4000;
const auto mapping = Libs::LibKernel::Memory::AllocateProgramMemory(
0x906000000, code_size * 2, Common::VirtualMemory::Mode::ExecuteReadWrite,
"cpu_extension_test");
Check(test, mapping != 0, "failed to allocate CPU extension fixture");
InstructionTestScope restore {mapping, code_size * 2};
restore.InstallHandler(test);
const auto host = Loader::GetGuestInstructionHostFeatures();
const std::array<uintptr_t, 1> starts {mapping};
const auto register_module = [&](uint64_t capacity) {
std::memset(reinterpret_cast<void*>(mapping + code_size), 0, code_size);
Loader::RegisterGuestInstructionPatchModule(reinterpret_cast<void*>(mapping), code_size,
reinterpret_cast<void*>(mapping + code_size),
capacity);
};
Xbyak::CodeGenerator code(code_size, reinterpret_cast<void*>(mapping));
using GuestFunction = void(KYTY_SYSV_ABI*)(const uint64_t*, uint64_t*);
const auto function = reinterpret_cast<GuestFunction>(mapping);
for (const bool writes_rsp: {false, true}) {
code.reset();
code.push(code.r12);
code.mov(code.r12, code.rsp);
code.vmovups(code.ymm8, code.ptr[code.rdi]);
code.vmovups(code.ymm9, code.ptr[code.rdi + 32]);
code.mov(code.r10, 0x123456789abcdef0ull);
code.mov(code.qword[code.rsp - 8], code.r10);
code.clwb(code.ptr[code.rsp - 8]);
for (const uint8_t byte: {0xf2, 0x45, 0x0f, 0x79, 0xc1}) {
code.db(byte); // INSERTQ xmm8,xmm9
}
code.db(0xf3); // RDPID rax/rsp
code.db(0x48);
code.db(0x0f);
code.db(0xc7);
code.db(writes_rsp ? 0xfc : 0xf8);
if (writes_rsp) {
code.mov(code.rax, code.rsp);
code.mov(code.rsp, code.r12);
}
code.mov(code.qword[code.rsi], code.rax);
code.mov(code.rax, code.qword[code.rsp - 8]);
code.mov(code.qword[code.rsi + 8], code.rax);
code.vmovups(code.ptr[code.rsi + 16], code.ymm8);
code.vzeroupper();
code.pop(code.r12);
code.ret();
Check(test, Xbyak::GetError() == 0, "failed to generate mixed CPU extension fixture");
const std::vector<uint8_t> original(code.getCode(), code.getCurr());
register_module(code_size);
const auto disabled =
Loader::PatchGuestInstructions(mapping, code.getSize(), starts, false, false, {});
Check(test,
disabled.insertq.found == 0 && disabled.rdpid.found == 0 &&
disabled.clwb.found == 0 &&
std::equal(original.begin(), original.end(), code.getCode()) &&
std::all_of(code.getCode() + code_size, code.getCode() + code_size * 2,
[](uint8_t byte) { return byte == 0; }),
"disabled AMD option changed CPU extension code or trampolines");
const auto supported = Loader::PatchGuestInstructions(mapping, code.getSize(), starts,
false, true, {true, true, true});
Check(test,
supported.insertq.found == 0 && supported.rdpid.found == 0 &&
supported.clwb.found == 0 &&
std::equal(original.begin(), original.end(), code.getCode()),
"available host CPU extensions were unnecessarily patched");
const auto enabled =
Loader::PatchGuestInstructions(mapping, code.getSize(), starts, false, true, {});
Check(test,
enabled.clwb.found == 1 && enabled.clwb.native == 1 && enabled.clwb.trapped == 0 &&
enabled.insertq.found == 1 && enabled.insertq.native == 0 &&
enabled.insertq.trapped == 1 && enabled.rdpid.found == 1 &&
enabled.rdpid.native == 0 && enabled.rdpid.trapped == 1,
"mixed native CLWB and trapped INSERTQ/RDPID counts differ");
// Verify the actual stack-relative CLFLUSH operand, since flushing the wrong mapped
// cache line would otherwise leave the sentinel unchanged too.
ZydisDecoder decoder {};
Check(test,
ZYAN_SUCCESS(
ZydisDecoderInit(&decoder, ZYDIS_MACHINE_MODE_LONG_64, ZYDIS_STACK_WIDTH_64)),
"failed to initialize decoder");
bool found_flush = false;
for (size_t offset = 0; offset < 512 && !found_flush;) {
ZydisDecodedInstruction instruction {};
ZydisDecodedOperand operands[ZYDIS_MAX_OPERAND_COUNT] {};
Check(test,
ZYAN_SUCCESS(ZydisDecoderDecodeFull(&decoder, code.getCode() + code_size + offset,
code_size - offset, &instruction, operands)),
"failed to decode CPU extension trampoline");
if (instruction.mnemonic == ZYDIS_MNEMONIC_CLFLUSH) {
found_flush = true;
Check(test,
operands[0].mem.base == ZYDIS_REGISTER_RSP &&
#if KYTY_PLATFORM == KYTY_PLATFORM_WINDOWS
operands[0].mem.disp.value == 120,
#else
operands[0].mem.disp.value == -8,
#endif
"CLWB replacement did not preserve its guest stack address");
}
offset += instruction.length;
}
Check(test, found_flush, "CLWB native replacement did not emit CLFLUSH");
const std::array<uint64_t, 8> input {0x1111, 0xdeadbeef, 0x12345678, 0x87654321,
0xab, 0xffffc4c8, 0, 0};
std::array<uint64_t, 6> output {};
bool checked_aux = false;
for (size_t attempt = 0; attempt < 100 && !checked_aux; ++attempt) {
uint32_t before = 0;
uint32_t after = 0;
const auto traps = g_instruction_traps;
__rdtscp(&before);
function(input.data(), output.data());
__rdtscp(&after);
Check(test, g_instruction_traps == traps + !host.sse4a + !host.rdpid,
"mixed CPU extension fixture used an unexpected trap path");
Check(test,
output[1] == 0x123456789abcdef0ull && output[2] == 0x1ab1 &&
(host.sse4a || output[3] == input[1]) &&
std::equal(output.begin() + 4, output.end(), input.begin() + 2),
"mixed CPU extensions corrupted INSERTQ state or the guest red zone");
if (before == after) {
Check(test, output[0] == before,
"RDPID patch lost TSC_AUX or adjusted its RSP destination");
checked_aux = true;
}
}
Check(test, checked_aux, "thread migrated repeatedly during patched RDPID comparison");
}
code.reset();
code.clwb(code.ptr[code.rdi]);
code.ret();
Check(test, Xbyak::GetError() == 0, "failed to generate exhausted CLWB fixture");
const std::vector<uint8_t> original(code.getCode(), code.getCurr());
register_module(1);
const auto exhausted =
Loader::PatchGuestInstructions(mapping, code.getSize(), starts, false, true, {}).clwb;
Check(test,
exhausted.found == 1 && exhausted.native == 0 && exhausted.trapped == 1 &&
std::equal(original.begin(), original.end(), code.getCode()) &&
Xbyak::GetError() == 0,
"exhausted CLWB replacement did not retain its natural trap fallback");
std::array<uint64_t, 8> data;
data.fill(0x123456789abcdef0ull);
const auto traps = g_instruction_traps;
function(data.data(), nullptr);
Check(test,
g_instruction_traps == traps + !host.clwb &&
std::all_of(data.begin(), data.end(),
[](uint64_t value) { return value == 0x123456789abcdef0ull; }),
"CLWB direct fallback changed memory or used an unexpected trap path");
std::printf("[host] %-48s ok\n", test);
}
#endif
#if defined(__x86_64__) || defined(_M_X64)
@@ -3714,6 +4051,12 @@ int main(int argc, char** argv) {
}
#endif
#if defined(__linux__) || KYTY_PLATFORM == KYTY_PLATFORM_WINDOWS
if (argc == 2 && std::strcmp(argv[1], "--cpu-extensions-only") == 0) {
RunTest(TestCpuExtensionContexts);
RunTest(TestPackedBitFieldInsert);
RunTest(TestCpuExtensionPatches);
return g_failed_tests == 0 ? 0 : 1;
}
if (argc == 2 && std::strcmp(argv[1], "--extrq-only") == 0) {
RunTest(TestPackedBitFieldExtract);
return g_failed_tests == 0 ? 0 : 1;
@@ -3734,6 +4077,9 @@ int main(int argc, char** argv) {
#if defined(__linux__) || KYTY_PLATFORM == KYTY_PLATFORM_WINDOWS
RunTest(TestPackedReciprocalSquareRoot);
RunTest(TestPackedBitFieldExtract);
RunTest(TestCpuExtensionContexts);
RunTest(TestPackedBitFieldInsert);
RunTest(TestCpuExtensionPatches);
#endif
RunTest(TestWindowsGuestRedZoneStaticPatcher);
RunTest(TestProsperoArgumentAndInfoSizeContracts);
tests/macosSse4aTests.cpp
+42
View File
@@ -8,6 +8,7 @@
#include <cstdio>
#include <cstdlib>
#include <cstring>
#include <initializer_list>
#include <sys/mman.h>
#include <sys/ucontext.h>
#include <unistd.h>
@@ -61,6 +62,21 @@ constexpr uint8_t kInsertqXmm10Xmm9Len16Index8[] = {
0xc3, // ret
};
constexpr uint8_t kInsertqXmm10Xmm9[] = {
0xf3, 0x44, 0x0f, 0x6f, 0x0e, // movdqu (%rsi), %xmm9
0xf3, 0x44, 0x0f, 0x6f, 0x12, // movdqu (%rdx), %xmm10
0xf2, 0x45, 0x0f, 0x79, 0xd1, // insertq xmm10, xmm9
0xf3, 0x44, 0x0f, 0x7f, 0x17, // movdqu %xmm10, (%rdi)
0xc3,
};
constexpr uint8_t kInsertqXmm9Xmm9[] = {
0xf3, 0x44, 0x0f, 0x6f, 0x0e, // movdqu (%rsi), %xmm9
0xf2, 0x45, 0x0f, 0x79, 0xc9, // insertq xmm9, xmm9
0xf3, 0x44, 0x0f, 0x7f, 0x0f, // movdqu %xmm9, (%rdi)
0xc3,
};
constexpr uint8_t kUd2[] = {
0x0f, 0x0b, // ud2
0xc3, // ret
@@ -192,6 +208,31 @@ void TestRexHighRegisterInsertqWithIndex() {
Check(out[1] == 0xbbbbbbbbbbbbbbbbull, "insertq xmm10, xmm9, 16, 8 leaves the upper 64 bits untouched");
}
void TestInsertqRegister() {
const auto separate = MapCode<PairBlobFn>(kInsertqXmm10Xmm9);
const auto aliased = MapCode<PairBlobFn>(kInsertqXmm9Xmm9);
for (const auto fn: {separate, aliased}) {
for (const uint64_t controls: {0xc4c8ull, 0xc0c0ull, 0xffc1ull}) {
const uint64_t source[2] {0x0123456789abcdefull, 0xdeadbeef00000000ull | controls};
const uint64_t destination[2] {0x8877665544332211ull, 0xfedcba9876543210ull};
const auto* initial = fn == aliased ? source : destination;
uint64_t expected = initial[0];
const auto length = (controls & 63) == 0 ? 64 : controls & 63;
const auto index = (controls >> 8) & 63;
for (uint64_t bit = 0; bit < length && index + bit < 64; ++bit) {
const uint64_t mask = uint64_t {1} << (index + bit);
expected = (expected & ~mask) | (((source[0] >> bit) & 1) << (index + bit));
}
uint64_t out[2] {};
const int before = IllegalHits();
fn(out, source, destination);
Check(IllegalHits() == before + 1, "register INSERTQ raises one SIGILL");
Check(out[0] == expected && out[1] == initial[1],
"register INSERTQ uses upper source controls and preserves upper destination");
}
}
}
void TestUnknownInstructionRefused() {
g_refused.store(false, std::memory_order_relaxed);
g_refuse_skip.store(2, std::memory_order_relaxed);
@@ -213,6 +254,7 @@ int main() {
TestInsertqImmediate();
TestRexHighRegisterExtrq();
TestRexHighRegisterInsertqWithIndex();
TestInsertqRegister();
TestUnknownInstructionRefused();
std::printf("macosSse4aTests: all passed\n");
return 0;