diff --git a/src/loader/guestInstructionPatcher.cpp b/src/loader/guestInstructionPatcher.cpp index f70265eda..7c16030f9 100644 --- a/src/loader/guestInstructionPatcher.cpp +++ b/src/loader/guestInstructionPatcher.cpp @@ -132,7 +132,14 @@ struct DecodedFunction { bool requires_conservative_red_zone_tracking {}; }; -enum class InstructionReplacement { None, ReciprocalSquareRoot, ExtractQ }; +enum class InstructionReplacement { + None, + ReciprocalSquareRoot, + ExtractQ, + InsertQ, + ReadProcessorId, + CacheLineWriteBack +}; struct InstructionRewrite { bool protect_red_zone {}; @@ -143,8 +150,19 @@ struct InstructionRewrite { InstructionPatchCounts& ReplacementCounts(GuestInstructionPatchResult& result, InstructionReplacement replacement) { ASSERT(replacement != InstructionReplacement::None); - return replacement == InstructionReplacement::ReciprocalSquareRoot ? result.reciprocal_sqrt - : result.extrq; + switch (replacement) { + case InstructionReplacement::ReciprocalSquareRoot: return result.reciprocal_sqrt; + case InstructionReplacement::ExtractQ: return result.extrq; + case InstructionReplacement::InsertQ: return result.insertq; + case InstructionReplacement::ReadProcessorId: return result.rdpid; + default: return result.clwb; + } +} + +bool UsesInstructionTrap(InstructionReplacement replacement, bool trap_replacements) { + return replacement != InstructionReplacement::None && + (trap_replacements || replacement == InstructionReplacement::InsertQ || + replacement == InstructionReplacement::ReadProcessorId); } bool IsStackPointerRegister(ZydisRegister reg) { @@ -216,10 +234,11 @@ DecodedCodeInstruction DecodeCodeInstruction(uintptr_t address, uintptr_t end) { if (decoded.changes_stack_pointer) { const auto& operands = decoded.operands; - if (decoded.instruction.mnemonic == ZYDIS_MNEMONIC_MOV && - operands[0].type == ZYDIS_OPERAND_TYPE_REGISTER && + if (operands[0].type == ZYDIS_OPERAND_TYPE_REGISTER && IsStackPointerRegister(operands[0].reg.value) && - operands[1].type == ZYDIS_OPERAND_TYPE_MEMORY) { + (decoded.instruction.mnemonic == ZYDIS_MNEMONIC_RDPID || + (decoded.instruction.mnemonic == ZYDIS_MNEMONIC_MOV && + operands[1].type == ZYDIS_OPERAND_TYPE_MEMORY))) { decoded.replaces_stack_pointer = true; } else if ((decoded.instruction.mnemonic == ZYDIS_MNEMONIC_ADD || decoded.instruction.mnemonic == ZYDIS_MNEMONIC_SUB) && @@ -653,13 +672,17 @@ void AnalyzeRedZoneLiveness(DecodedFunction& function) { } bool EncodeRelocatedInstruction(const DecodedCodeInstruction& decoded, - Xbyak::CodeGenerator& generator) { + Xbyak::CodeGenerator& generator, s64 stack_adjustment = 0, + ZydisMnemonic replacement = ZYDIS_MNEMONIC_INVALID) { ZydisEncoderRequest request; if (!ZYAN_SUCCESS(ZydisEncoderDecodedInstructionToEncoderRequest( &decoded.instruction, decoded.operands.data(), decoded.instruction.operand_count_visible, &request))) { return false; } + if (replacement != ZYDIS_MNEMONIC_INVALID) { + request.mnemonic = replacement; + } for (u8 index = 0; index < decoded.instruction.operand_count_visible; ++index) { const auto& operand = decoded.operands[index]; @@ -679,6 +702,9 @@ bool EncodeRelocatedInstruction(const DecodedCodeInstruction& decoded, return false; } request.operands[index].mem.displacement = static_cast(absolute_address); + } else if (operand.type == ZYDIS_OPERAND_TYPE_MEMORY && + IsStackPointerRegister(operand.mem.base)) { + request.operands[index].mem.displacement += stack_adjustment; } } @@ -823,8 +849,9 @@ void GenerateReciprocalSquareRoot(const DecodedCodeInstruction& decoded, generator.L(done); } -bool IsSupportedExtractQ(const DecodedCodeInstruction& decoded) { - if (decoded.instruction.mnemonic != ZYDIS_MNEMONIC_EXTRQ) { +bool IsSupportedBitFieldInstruction(const DecodedCodeInstruction& decoded) { + const bool insert = decoded.instruction.mnemonic == ZYDIS_MNEMONIC_INSERTQ; + if (!insert && decoded.instruction.mnemonic != ZYDIS_MNEMONIC_EXTRQ) { return false; } const auto is_xmm = [](const ZydisDecodedOperand& operand) { @@ -832,18 +859,20 @@ bool IsSupportedExtractQ(const DecodedCodeInstruction& decoded) { operand.reg.value >= ZYDIS_REGISTER_XMM0 && operand.reg.value <= ZYDIS_REGISTER_XMM15; }; - const bool immediate = decoded.instruction.operand_count_visible == 3 && - decoded.operands[1].type == ZYDIS_OPERAND_TYPE_IMMEDIATE && - decoded.operands[2].type == ZYDIS_OPERAND_TYPE_IMMEDIATE; - if (!is_xmm(decoded.operands[0]) || + const size_t immediate_index = insert ? 2 : 1; + const bool immediate = + decoded.instruction.operand_count_visible == immediate_index + 2 && + decoded.operands[immediate_index].type == ZYDIS_OPERAND_TYPE_IMMEDIATE && + decoded.operands[immediate_index + 1].type == ZYDIS_OPERAND_TYPE_IMMEDIATE; + if (!is_xmm(decoded.operands[0]) || (insert && !is_xmm(decoded.operands[1])) || (!immediate && !(decoded.instruction.operand_count_visible == 2 && is_xmm(decoded.operands[1])))) { return false; } - // Match the trap handler's encoding support: 66 [REX] 0F 78/79 ModRM [imm8 imm8]. + // Match the trap handler's encoding support: 66/F2 [REX] 0F 78/79 ModRM [imm8 imm8]. const auto* code = reinterpret_cast(decoded.address); const size_t opcode = (code[1] & 0xf0u) == 0x40u ? 2 : 1; - return code[0] == 0x66 && code[opcode] == 0x0f && + return code[0] == (insert ? 0xf2 : 0x66) && code[opcode] == 0x0f && code[opcode + 1] == (immediate ? 0x78 : 0x79) && decoded.instruction.length == opcode + (immediate ? 5 : 3); } @@ -890,8 +919,8 @@ void GenerateExtractQ(const DecodedCodeInstruction& decoded, Xbyak::CodeGenerato void CollectAmdInstructions(const DecodedFunction& function, std::map& rewrite_sites, - GuestInstructionPatchResult& result) { - static const bool host_lacks_sse4a = !Xbyak::util::Cpu().has(Xbyak::util::Cpu::tSSE4a); + GuestInstructionPatchResult& result, + GuestInstructionHostFeatures host_features) { for (const auto& [address, decoded]: function.instructions) { const auto& instruction = decoded.instruction; InstructionReplacement replacement {}; @@ -900,8 +929,14 @@ void CollectAmdInstructions(const DecodedFunction& function, instruction.raw.vex.offset == 0 && decoded.operands[0].size == 128 && decoded.operands[1].type == ZYDIS_OPERAND_TYPE_REGISTER) { replacement = InstructionReplacement::ReciprocalSquareRoot; - } else if (host_lacks_sse4a && IsSupportedExtractQ(decoded)) { - replacement = InstructionReplacement::ExtractQ; + } else if (!host_features.sse4a && IsSupportedBitFieldInstruction(decoded)) { + replacement = instruction.mnemonic == ZYDIS_MNEMONIC_EXTRQ + ? InstructionReplacement::ExtractQ + : InstructionReplacement::InsertQ; + } else if (!host_features.rdpid && instruction.mnemonic == ZYDIS_MNEMONIC_RDPID) { + replacement = InstructionReplacement::ReadProcessorId; + } else if (!host_features.clwb && instruction.mnemonic == ZYDIS_MNEMONIC_CLWB) { + replacement = InstructionReplacement::CacheLineWriteBack; } else { continue; } @@ -916,16 +951,21 @@ void MarkInstructionTrap(u8* code, const ZydisDecodedInstruction& instruction, // Clear a reserved VEX.vvvv bit, retaining both register operands for the handler. code[instruction.raw.vex.size - 1] &= ~0x08u; } - // EXTRQ already raises #UD on hosts without SSE4a; keep its original encoding. + // Other replacements already raise #UD on hosts without their CPU feature. } -void GenerateInstructionTrap(const DecodedCodeInstruction& decoded, - InstructionReplacement replacement, Xbyak::CodeGenerator& generator) { +bool GenerateInstructionTrap(const DecodedCodeInstruction& decoded, + InstructionReplacement replacement, Xbyak::CodeGenerator& generator, + s64 stack_adjustment) { + if (replacement == InstructionReplacement::CacheLineWriteBack) { + return EncodeRelocatedInstruction(decoded, generator, stack_adjustment); + } std::array code {}; std::memcpy(code.data(), reinterpret_cast(decoded.address), decoded.instruction.length); MarkInstructionTrap(code.data(), decoded.instruction, replacement); generator.db(code.data(), decoded.instruction.length); + return true; } bool NeedsTrapRedZoneProtection(const DecodedCodeInstruction& decoded) { @@ -946,11 +986,11 @@ void TrapUnrelocatedInstructions(const PatchModule& module, const DecodedFunctio } const auto& decoded = function.instructions.at(address); if (NeedsTrapRedZoneProtection(decoded)) { - if (rewrite.replacement == InstructionReplacement::ExtractQ) { - // Unlike VRSQRTPS, leaving EXTRQ unchanged would still trap and corrupt the red - // zone. - EXIT("AMD CPU compatibility: cannot safely trap EXTRQ at %p (guest red zone is " + if (rewrite.replacement != InstructionReplacement::ReciprocalSquareRoot) { + // Leaving an unsupported instruction unchanged still traps with a live red zone. + EXIT("AMD CPU compatibility: cannot safely trap %s at %p (guest red zone is " "live)\n", + ZydisMnemonicGetString(decoded.instruction.mnemonic), reinterpret_cast(address)); } continue; @@ -990,23 +1030,31 @@ void RelocateGuestInstructions(PatchModule* module, const DecodedFunction& funct const auto replacement = rewrite != rewrite_sites.end() ? rewrite->second.replacement : InstructionReplacement::None; + const bool uses_trap = UsesInstructionTrap(replacement, trap_replacements); const bool protected_indirect_call = rewrite != rewrite_sites.end() && rewrite->second.protected_indirect_call; const bool protect_red_zone = (rewrite != rewrite_sites.end() && rewrite->second.protect_red_zone && !protected_indirect_call) || - (replacement != InstructionReplacement::None && trap_replacements && + ((uses_trap || replacement == InstructionReplacement::CacheLineWriteBack) && NeedsTrapRedZoneProtection(*decoded)); if (protect_red_zone) { generator.lea(rsp, ptr[rsp - GuestRedZoneSize]); } if (replacement != InstructionReplacement::None) { - if (trap_replacements) { - GenerateInstructionTrap(*decoded, replacement, generator); + if (uses_trap) { + encoded = GenerateInstructionTrap(*decoded, replacement, generator, + protect_red_zone ? GuestRedZoneSize : 0); } else if (replacement == InstructionReplacement::ReciprocalSquareRoot) { GenerateReciprocalSquareRoot(*decoded, generator); - } else { + } else if (replacement == InstructionReplacement::ExtractQ) { GenerateExtractQ(*decoded, generator); + } else { + // CLFLUSH writes back dirty data too; invalidation is a permitted stronger + // action. + encoded = EncodeRelocatedInstruction( + *decoded, generator, protect_red_zone ? GuestRedZoneSize : 0, + ZYDIS_MNEMONIC_CLFLUSH); } } else if (protected_indirect_call) { encoded = GenerateProtectedIndirectCall(*decoded, generator); @@ -1053,7 +1101,7 @@ void RelocateGuestInstructions(PatchModule* module, const DecodedFunction& funct } if (rewrite->second.replacement != InstructionReplacement::None) { auto& counts = ReplacementCounts(result, rewrite->second.replacement); - if (span.trap_replacements) { + if (UsesInstructionTrap(rewrite->second.replacement, span.trap_replacements)) { ++counts.trapped; } else { ++counts.native; @@ -1425,9 +1473,26 @@ void RelocateGuestInstructions(PatchModule* module, const DecodedFunction& funct } } // namespace +GuestInstructionHostFeatures GetGuestInstructionHostFeatures() { + static const auto features = [] { + const Xbyak::util::Cpu cpu; + uint32_t leaf[4] {}; + Xbyak::util::Cpu::getCpuid(0, leaf); + bool rdpid = false; + if (leaf[0] >= 7) { + Xbyak::util::Cpu::getCpuidEx(7, 0, leaf); + rdpid = (leaf[2] & (1u << 22)) != 0; + } + return GuestInstructionHostFeatures {cpu.has(Xbyak::util::Cpu::tSSE4a), rdpid, + cpu.has(Xbyak::util::Cpu::tCLWB)}; + }(); + return features; +} + GuestInstructionPatchResult PatchGuestInstructions(u64 segment_addr, u64 segment_size, std::span function_starts, - bool protect_memory, bool emulate_amd) { + bool protect_memory, bool emulate_amd, + GuestInstructionHostFeatures host_features) { GuestInstructionPatchResult result {}; if (!protect_memory && !emulate_amd) { return result; @@ -1480,7 +1545,7 @@ GuestInstructionPatchResult PatchGuestInstructions(u64 segment_addr, u64 segment CollectRedZoneMemoryInstructions(function, rewrite_sites, result); } if (emulate_amd) { - CollectAmdInstructions(function, rewrite_sites, result); + CollectAmdInstructions(function, rewrite_sites, result, host_features); } if (!rewrite_sites.empty()) { RelocateGuestInstructions(module, function, rewrite_sites, result); @@ -1499,8 +1564,12 @@ GuestInstructionPatchResult PatchGuestInstructions(u64 segment_addr, u64 segment #else -GuestInstructionPatchResult PatchGuestInstructions(u64, u64, std::span, bool, - bool) { +GuestInstructionHostFeatures GetGuestInstructionHostFeatures() { + return {}; +} + +GuestInstructionPatchResult PatchGuestInstructions(u64, u64, std::span, bool, bool, + GuestInstructionHostFeatures) { return {}; } diff --git a/src/loader/guestInstructionPatcher.h b/src/loader/guestInstructionPatcher.h index 68cf5b49e..1c6f78c03 100644 --- a/src/loader/guestInstructionPatcher.h +++ b/src/loader/guestInstructionPatcher.h @@ -11,6 +11,14 @@ namespace Loader { +struct GuestInstructionHostFeatures { + bool sse4a = false; + bool rdpid = false; + bool clwb = false; +}; + +GuestInstructionHostFeatures GetGuestInstructionHostFeatures(); + struct InstructionPatchCounts { uint64_t found = 0; uint64_t native = 0; @@ -39,6 +47,9 @@ struct GuestInstructionPatchResult { uint64_t indirect_red_zone_function_count = 0; InstructionPatchCounts reciprocal_sqrt; InstructionPatchCounts extrq; + InstructionPatchCounts insertq; + InstructionPatchCounts rdpid; + InstructionPatchCounts clwb; }; void RegisterGuestInstructionPatchModule(void* module_ptr, uint64_t module_size, @@ -46,9 +57,10 @@ void RegisterGuestInstructionPatchModule(void* module_ptr, uint64_t module_size, void UnregisterGuestInstructionPatchModule(void* module_ptr); // Apply enabled instruction fixes using native trampolines or safe trap fallbacks. -GuestInstructionPatchResult PatchGuestInstructions(uint64_t segment_addr, uint64_t segment_size, - std::span function_starts, - bool protect_memory, bool emulate_amd); +GuestInstructionPatchResult PatchGuestInstructions( + uint64_t segment_addr, uint64_t segment_size, std::span function_starts, + bool protect_memory, bool emulate_amd, + GuestInstructionHostFeatures host_features = GetGuestInstructionHostFeatures()); bool DecodeEhFrameFunctionStarts(uint64_t eh_frame_header_addr, uint64_t eh_frame_header_size, std::vector* function_starts); diff --git a/src/loader/runtimeLinker.cpp b/src/loader/runtimeLinker.cpp index fcaef06ce..721382a82 100644 --- a/src/loader/runtimeLinker.cpp +++ b/src/loader/runtimeLinker.cpp @@ -1949,6 +1949,9 @@ void RuntimeLinker::LoadProgramToMemory(Program* program) { protect_memory_faults, emulate_amd); totals.reciprocal_sqrt += result.reciprocal_sqrt; totals.extrq += result.extrq; + totals.insertq += result.insertq; + totals.rdpid += result.rdpid; + totals.clwb += result.clwb; if (protect_memory_faults) { LOGF("Windows guest red-zone patching: %s, functions=%" PRIu64 ", red_zone=%" PRIu64 ", memory=%" PRIu64 ", patched=%" PRIu64 ", short=%" PRIu64 ", stack=%" PRIu64 @@ -1963,18 +1966,26 @@ void RuntimeLinker::LoadProgramToMemory(Program* program) { } } if (emulate_amd && have_function_starts) { - const auto found = totals.reciprocal_sqrt.found + totals.extrq.found; - const auto skipped = totals.reciprocal_sqrt.Skipped() + totals.extrq.Skipped(); + InstructionPatchCounts combined {}; + std::string details; + for (const auto& [name, counts]: {std::pair {"VRSQRTPS", totals.reciprocal_sqrt}, + {"EXTRQ", totals.extrq}, + {"INSERTQ", totals.insertq}, + {"RDPID", totals.rdpid}, + {"CLWB", totals.clwb}}) { + combined += counts; + if (!details.empty()) details += "; "; + details += fmt::format("{}: native={}, trapped={}, skipped={}", name, counts.native, + counts.trapped, counts.Skipped()); + } + const auto found = combined.found; + const auto skipped = combined.Skipped(); const char* status = found == 0 ? "no matching instructions" : skipped == found ? "not patched" : skipped != 0 ? "partially patched" : "patched"; - Log::WriteToConsoleAndLog(fmt::format( - "AMD CPU compatibility: {} {} (VRSQRTPS: native={}, trapped={}, skipped={}; " - "EXTRQ: native={}, trapped={}, skipped={})\n", - module_name, status, totals.reciprocal_sqrt.native, totals.reciprocal_sqrt.trapped, - totals.reciprocal_sqrt.Skipped(), totals.extrq.native, totals.extrq.trapped, - totals.extrq.Skipped())); + Log::WriteToConsoleAndLog( + fmt::format("AMD CPU compatibility: {} {} ({})\n", module_name, status, details)); } } diff --git a/src/loader/x64InstructionEmulator.cpp b/src/loader/x64InstructionEmulator.cpp index ef3659133..55fa9798e 100644 --- a/src/loader/x64InstructionEmulator.cpp +++ b/src/loader/x64InstructionEmulator.cpp @@ -2,12 +2,12 @@ #include "common/common.h" +#include #include #include -#if !defined(__APPLE__) #include #include -#endif +#include #if KYTY_PLATFORM == KYTY_PLATFORM_WINDOWS #include // IWYU pragma: keep @@ -406,12 +406,12 @@ struct Context { void Advance(size_t length) { native->Rip += length; } [[nodiscard]] void* Xmm(uint8_t index) const { return &native->Xmm0 + index; } - void LoadGprs(uint64_t (&gpr)[16]) const { - const uint64_t registers[] = {native->Rax, native->Rcx, native->Rdx, native->Rbx, - native->Rsp, native->Rbp, native->Rsi, native->Rdi, - native->R8, native->R9, native->R10, native->R11, - native->R12, native->R13, native->R14, native->R15}; - std::memcpy(gpr, registers, sizeof(gpr)); + [[nodiscard]] auto& Gpr(uint8_t index) const { + DWORD64* registers[] = {&native->Rax, &native->Rcx, &native->Rdx, &native->Rbx, + &native->Rsp, &native->Rbp, &native->Rsi, &native->Rdi, + &native->R8, &native->R9, &native->R10, &native->R11, + &native->R12, &native->R13, &native->R14, &native->R15}; + return *registers[index]; } void ClearUpperYmm(uint8_t index) const { @@ -437,6 +437,14 @@ struct Context { void Advance(size_t length) { native->uc_mcontext->__ss.__rip += static_cast(length); } + [[nodiscard]] auto& Gpr(uint8_t index) const { + auto* state = &native->uc_mcontext->__ss; + uint64_t* registers[] = {&state->__rax, &state->__rcx, &state->__rdx, &state->__rbx, + &state->__rsp, &state->__rbp, &state->__rsi, &state->__rdi, + &state->__r8, &state->__r9, &state->__r10, &state->__r11, + &state->__r12, &state->__r13, &state->__r14, &state->__r15}; + return *registers[index]; + } // Darwin names the XMM file __fpu_xmm0..__fpu_xmm15 instead of exposing an array. [[nodiscard]] void* Xmm(uint8_t index) const { auto* fs = &native->uc_mcontext->__fs; @@ -476,13 +484,11 @@ struct Context { return native->uc_mcontext.fpregs->_xmm[index].element; } - void LoadGprs(uint64_t (&gpr)[16]) const { + [[nodiscard]] auto& Gpr(uint8_t index) const { constexpr int registers[] = {REG_RAX, REG_RCX, REG_RDX, REG_RBX, REG_RSP, REG_RBP, REG_RSI, REG_RDI, REG_R8, REG_R9, REG_R10, REG_R11, REG_R12, REG_R13, REG_R14, REG_R15}; - for (size_t i = 0; i < 16; ++i) { - gpr[i] = static_cast(native->uc_mcontext.gregs[registers[i]]); - } + return native->uc_mcontext.gregs[registers[index]]; } void ClearUpperYmm(uint8_t index) const { @@ -506,6 +512,12 @@ struct Context { } } #endif + + void LoadGprs(uint64_t (&gpr)[16]) const { + for (uint8_t i = 0; i < 16; ++i) { + gpr[i] = static_cast(Gpr(i)); + } + } }; #if !defined(__APPLE__) @@ -574,8 +586,8 @@ static bool TryEmulateSse4a(Context& context) { if (rip[offset] != 0x0f) { return false; } - const bool register_extract = prefix == 0x66 && rip[offset + 1] == 0x79; - if (rip[offset + 1] != 0x78 && !register_extract) { + const bool register_form = rip[offset + 1] == 0x79; + if (rip[offset + 1] != 0x78 && !register_form) { return false; } @@ -589,7 +601,7 @@ static bool TryEmulateSse4a(Context& context) { // Immediate EXTRQ encodes its destination in r/m; the two-register form uses reg. uint8_t dest_index = reg; - if (prefix == 0x66 && !register_extract) { + if (prefix == 0x66 && !register_form) { dest_index = rm; } auto* dest_xmm = context.Xmm(dest_index); @@ -598,15 +610,17 @@ static bool TryEmulateSse4a(Context& context) { return false; } uint64_t dest[2] {}; - uint64_t source = 0; + uint64_t source[2] {}; std::memcpy(dest, dest_xmm, sizeof(dest)); - std::memcpy(&source, src_xmm, sizeof(source)); + std::memcpy(source, src_xmm, sizeof(source)); uint8_t length = 0; uint8_t index = 0; size_t instruction_length = offset + 3; - if (register_extract) { - length = static_cast(source); - index = static_cast(source >> 8u); + if (register_form) { + // EXTRQ controls occupy the low quadword; INSERTQ uses the upper quadword. + const uint64_t controls = source[prefix == 0x66 ? 0 : 1]; + length = static_cast(controls); + index = static_cast(controls >> 8u); } else { length = rip[offset + 3]; index = rip[offset + 4]; @@ -616,13 +630,79 @@ static bool TryEmulateSse4a(Context& context) { dest[0] = ExtractBitField(dest[0], length, index); dest[1] = 0; } else { - dest[0] = InsertBitField(dest[0], source, length, index); + dest[0] = InsertBitField(dest[0], source[0], length, index); } std::memcpy(dest_xmm, dest, sizeof(dest)); context.Advance(instruction_length); return true; } +static bool ReadTscAux(uint32_t& auxiliary) { + uint32_t eax = 0; + uint32_t ebx = 0; + uint32_t ecx = 0; + uint32_t edx = 0; + if (!__get_cpuid(0x80000001u, &eax, &ebx, &ecx, &edx) || (edx & (1u << 27u)) == 0) { + return false; + } + asm volatile("rdtscp" : "=a"(eax), "=d"(edx), "=c"(auxiliary)); + return true; +} + +static void FlushCacheLine(uint64_t address, ZydisRegister segment) { + // Preserve FS/GS addressing rather than treating a segment-relative offset as a pointer. + if (segment == ZYDIS_REGISTER_FS) { + asm volatile("clflush %%fs:(%0)" : : "r"(address) : "memory"); + } else if (segment == ZYDIS_REGISTER_GS) { + asm volatile("clflush %%gs:(%0)" : : "r"(address) : "memory"); + } else { + _mm_clflush(reinterpret_cast(address)); + } +} + +static bool TryEmulateCpuExtensions(Context& context) { + ZydisDecoder decoder {}; + ZydisDecodedInstruction instruction {}; + ZydisDecodedOperand operands[ZYDIS_MAX_OPERAND_COUNT] {}; + if (!ZYAN_SUCCESS( + ZydisDecoderInit(&decoder, ZYDIS_MACHINE_MODE_LONG_64, ZYDIS_STACK_WIDTH_64)) || + !ZYAN_SUCCESS(ZydisDecoderDecodeFull(&decoder, reinterpret_cast(context.Rip()), + ZYDIS_MAX_INSTRUCTION_LENGTH, &instruction, + operands))) { + return false; + } + if (instruction.mnemonic == ZYDIS_MNEMONIC_RDPID && + operands[0].type == ZYDIS_OPERAND_TYPE_REGISTER) { + const auto reg_class = ZydisRegisterGetClass(operands[0].reg.value); + const auto index = ZydisRegisterGetId(operands[0].reg.value); + uint32_t auxiliary = 0; + if ((reg_class != ZYDIS_REGCLASS_GPR32 && reg_class != ZYDIS_REGCLASS_GPR64) || index < 0 || + index >= 16 || !ReadTscAux(auxiliary)) { + return false; + } + // RDPID and guest RDTSCP observe the same host TSC_AUX; only RDPID's destination changes. + context.Gpr(static_cast(index)) = auxiliary; + } else if (instruction.mnemonic == ZYDIS_MNEMONIC_CLWB && + operands[0].type == ZYDIS_OPERAND_TYPE_MEMORY) { + ZydisRegisterContext registers {}; + for (uint8_t i = 0; i < 16; ++i) { + const auto value = static_cast(context.Gpr(i)); + registers.values[ZYDIS_REGISTER_RAX + i] = value; + registers.values[ZYDIS_REGISTER_EAX + i] = static_cast(value); + } + uint64_t address = 0; + if (!ZYAN_SUCCESS(ZydisCalcAbsoluteAddressEx(&instruction, &operands[0], context.Rip(), + ®isters, &address))) { + return false; + } + FlushCacheLine(address, operands[0].mem.segment); + } else { + return false; + } + context.Advance(instruction.length); + return true; +} + #if !defined(__APPLE__) static bool TryEmulateMonitorxMwaitx(Context& context) { @@ -729,9 +809,9 @@ bool TryEmulate(void* native_context) { #endif #if !defined(__APPLE__) return TryEmulateReciprocalSquareRoot(context) || TryEmulateMonitorxMwaitx(context) || - TryEmulateSse4a(context) || TryEmulateShaNi(context); + TryEmulateSse4a(context) || TryEmulateShaNi(context) || TryEmulateCpuExtensions(context); #else - return TryEmulateSse4a(context); + return TryEmulateSse4a(context) || TryEmulateCpuExtensions(context); #endif } diff --git a/tests/VirtualMemoryAllocationTests.cpp b/tests/VirtualMemoryAllocationTests.cpp index 237d71b0b..f529d9d5b 100644 --- a/tests/VirtualMemoryAllocationTests.cpp +++ b/tests/VirtualMemoryAllocationTests.cpp @@ -26,16 +26,25 @@ #include #include #include +#include #include #if defined(__linux__) || KYTY_PLATFORM == KYTY_PLATFORM_WINDOWS #include #include +#if defined(_MSC_VER) +#include +#else +#include +#endif +#include #include #include #endif #if defined(__linux__) +#include +#include #include #include #include @@ -2900,6 +2909,140 @@ struct InstructionTestScope { } }; +struct SavedInstructionContext { +#if KYTY_PLATFORM == KYTY_PLATFORM_WINDOWS + CONTEXT native {}; + auto& Gpr(size_t index) { + const std::array registers {&native.Rax, &native.Rcx, &native.Rdx, &native.Rbx, + &native.Rsp, &native.Rbp, &native.Rsi, &native.Rdi, + &native.R8, &native.R9, &native.R10, &native.R11, + &native.R12, &native.R13, &native.R14, &native.R15}; + return *registers[index]; + } + auto& Rip() { return native.Rip; } + auto& Flags() { return native.EFlags; } + void* Xmm() { return &native.Xmm0; } +#else + ucontext_t native {}; + _libc_fpstate fpstate {}; + SavedInstructionContext() { native.uc_mcontext.fpregs = &fpstate; } + auto& Gpr(size_t index) { + constexpr std::array registers {REG_RAX, REG_RCX, REG_RDX, REG_RBX, REG_RSP, REG_RBP, + REG_RSI, REG_RDI, REG_R8, REG_R9, REG_R10, REG_R11, + REG_R12, REG_R13, REG_R14, REG_R15}; + return native.uc_mcontext.gregs[registers[index]]; + } + auto& Rip() { return native.uc_mcontext.gregs[REG_RIP]; } + auto& Flags() { return native.uc_mcontext.gregs[REG_EFL]; } + void* Xmm() { return fpstate._xmm; } +#endif +}; + +void TestCpuExtensionContexts() { + const char* test = "CpuExtensionContexts"; + SavedInstructionContext context; + constexpr uint64_t sentinel = 0x8192a3b4c5d6e700ull; + std::array xmm; + for (size_t i = 0; i < xmm.size(); ++i) { + xmm[i] = static_cast(i); + } + std::memcpy(context.Xmm(), xmm.data(), xmm.size()); + const auto check_state = [&](const std::array& expected) { + for (size_t reg = 0; reg < expected.size(); ++reg) { + Check(test, static_cast(context.Gpr(reg)) == expected[reg], + "CPU extension changed an unexpected GPR"); + } + Check(test, + context.Flags() == 0x897 && std::memcmp(context.Xmm(), xmm.data(), xmm.size()) == 0, + "CPU extension changed RFLAGS or XMM state"); + }; + for (uint8_t encoding = 0; encoding < 32; ++encoding) { + const uint8_t destination = encoding & 15; + bool checked = false; + for (size_t attempt = 0; attempt < 100 && !checked; ++attempt) { + std::array expected; + for (size_t reg = 0; reg < expected.size(); ++reg) { + context.Gpr(reg) = expected[reg] = sentinel | reg; + } + context.Flags() = 0x897; + std::array instruction { + 0xf3, static_cast(0x40 | ((encoding >> 4) << 3) | (destination >> 3)), + 0x0f, 0xc7, static_cast(0xf8 | (destination & 7))}; + context.Rip() = reinterpret_cast(instruction.data()); + uint32_t before = 0; + uint32_t after = 0; + __rdtscp(&before); + Check(test, EmulateInstructionContext(&context.native), "RDPID fallback was rejected"); + __rdtscp(&after); + Check(test, context.Rip() == reinterpret_cast(instruction.data() + 5), + "RDPID advanced RIP incorrectly"); + if (before == after) { + expected[destination] = before; + check_state(expected); + checked = true; + } + } + Check(test, checked, "thread migrated repeatedly during RDPID comparison"); + } + + constexpr uint64_t size = 0x4000; + const auto mapping = Libs::LibKernel::Memory::AllocateProgramMemory( + 0x30000000, size, Common::VirtualMemory::Mode::ReadWrite, "clwb_context_test"); + Check(test, mapping != 0 && mapping + size <= UINT32_MAX, + "failed to allocate low CLWB fixture"); + InstructionTestScope restore {mapping, size}; + auto* instruction = reinterpret_cast(mapping); + auto* data = instruction + 0x200; + std::memset(data, 0xa5, 64); + const auto flush = [&](std::initializer_list bytes, int base, uint64_t value, + int index = -1, uint64_t index_value = 0) { + std::array expected; + for (size_t reg = 0; reg < expected.size(); ++reg) { + context.Gpr(reg) = expected[reg] = sentinel | reg; + } + if (base >= 0) { + context.Gpr(base) = expected[base] = value; + } + if (index >= 0) { + context.Gpr(index) = expected[index] = index_value; + } + context.Flags() = 0x897; + std::fill_n(instruction, 16, 0); + std::copy(bytes.begin(), bytes.end(), instruction); + context.Rip() = mapping; + Check(test, EmulateInstructionContext(&context.native), "CLWB fallback was rejected"); + Check(test, static_cast(context.Rip()) == mapping + bytes.size(), + "CLWB advanced RIP incorrectly"); + check_state(expected); + Check(test, std::all_of(data, data + 64, [](uint8_t byte) { return byte == 0xa5; }), + "CLWB changed guest memory"); + }; + const auto address = reinterpret_cast(data); + flush({0x66, 0x0f, 0xae, 0x37}, 7, address); // [rdi] + flush({0x66, 0x0f, 0xae, 0x74, 0x24, 0xf8}, 4, address + 8); // [rsp-8] + flush({0x66, 0x41, 0x0f, 0xae, 0x34, 0x24}, 12, address); // [r12] + flush({0x66, 0x43, 0x0f, 0xae, 0x74, 0xac, 0x20}, 12, address - 0x20 - 12, 13, + 3); // [r12+r13*4+32] + flush({0x66, 0x41, 0x0f, 0xae, 0x35, 0xf7, 1, 0, 0}, -1, 0); // RIP+503; REX.B ignored + flush({0x67, 0x66, 0x0f, 0xae, 0x37}, 7, address | (uint64_t {1} << 32)); // [edi] + const auto byte = [&](int shift) { return static_cast(address >> shift); }; + flush({0x67, 0x66, 0x0f, 0xae, 0x35, 0xf7, 1, 0, 0}, -1, 0); // EIP+503 + flush({0x66, 0x41, 0x0f, 0xae, 0x34, 0x25, byte(0), byte(8), byte(16), byte(24)}, -1, + 0); // SIB absolute; REX.B ignored +#if defined(__linux__) + for (const auto segment: + {std::pair {ARCH_GET_FS, uint8_t {0x64}}, std::pair {ARCH_GET_GS, uint8_t {0x65}}}) { + unsigned long base = 0; + Check(test, syscall(SYS_arch_prctl, segment.first, &base) == 0, + "failed to read segment base"); + flush({segment.second, 0x66, 0x0f, 0xae, 0x37}, 7, address - base); + } +#else + flush({0x65, 0x66, 0x0f, 0xae, 0x37}, 7, address - reinterpret_cast(NtCurrentTeb())); +#endif + std::printf("[host] %-48s ok\n", test); +} + void TestPackedReciprocalSquareRoot() { const char* test = "PackedReciprocalSquareRoot"; constexpr uint64_t code_size = 0x4000; @@ -3229,12 +3372,12 @@ void TestPackedReciprocalSquareRoot() { std::printf("[host] %-48s ok\n", test); } -void TestPackedBitFieldExtract() { - const char* test = "PackedBitFieldExtract"; +void TestPackedBitField(bool insert) { + const char* test = insert ? "PackedBitFieldInsert" : "PackedBitFieldExtract"; constexpr uint64_t code_size = 0x80000; const auto mapping = Libs::LibKernel::Memory::AllocateProgramMemory( - 0x904000000, code_size * 2, Common::VirtualMemory::Mode::ExecuteReadWrite, "extrq_test"); - Check(test, mapping != 0, "failed to allocate EXTRQ test code"); + 0x904000000, code_size * 2, Common::VirtualMemory::Mode::ExecuteReadWrite, "bitfield_test"); + Check(test, mapping != 0, "failed to allocate SSE4a bitfield test code"); InstructionTestScope restore {mapping, code_size * 2}; restore.InstallHandler(test); const bool host_sse4a = Xbyak::util::Cpu().has(Xbyak::util::Cpu::tSSE4a); @@ -3243,19 +3386,23 @@ void TestPackedBitFieldExtract() { reinterpret_cast(mapping + code_size), capacity); }; - // Xbyak has no EXTRQ mnemonic. Emit the two canonical SSE4a encodings directly. - const auto extrq = [](Xbyak::CodeGenerator& code, int destination, int source, - uint8_t length = 0, uint8_t index = 0) { - code.db(0x66); + // Xbyak has no SSE4a bitfield mnemonics. A negative source selects the immediate form. + const auto emit_bit_field = [insert](Xbyak::CodeGenerator& code, int destination, int source, + uint8_t length = 0, uint8_t index = 0) { + const bool immediate = source < 0; + if (insert && immediate) { + source = 9; + } + code.db(insert ? 0xf2 : 0x66); const int rex = source < 0 ? (destination >> 3) : ((destination >> 3) << 2) | (source >> 3); if (rex != 0) { code.db(0x40 | rex); } code.db(0x0f); - code.db(source < 0 ? 0x78 : 0x79); + code.db(immediate ? 0x78 : 0x79); code.db(source < 0 ? 0xc0 | (destination & 7) : 0xc0 | ((destination & 7) << 3) | (source & 7)); - if (source < 0) { + if (immediate) { code.db(length); code.db(index); } @@ -3270,6 +3417,16 @@ void TestPackedBitFieldExtract() { } return result; }; + const auto insert_bits = [](uint64_t destination, uint64_t source, uint32_t length, + uint32_t index) { + const uint32_t count = (length & 63u) == 0 ? 64 : length & 63u; + const uint32_t first = index & 63u; + for (uint32_t bit = 0; bit < count && first + bit < 64; ++bit) { + const uint64_t mask = uint64_t {1} << (first + bit); + destination = (destination & ~mask) | (((source >> bit) & 1u) << (first + bit)); + } + return destination; + }; struct Result { std::array destination; std::array source; @@ -3299,7 +3456,8 @@ void TestPackedBitFieldExtract() { for (const auto registers: {std::array {1, 8}, std::array {8, 9}, std::array {3, 3}, std::array {15, 0}, std::array {8, -1}}) { const auto destination = registers[0]; - const auto source = registers[1]; + const auto source = insert && registers[1] < 0 ? 9 : registers[1]; + const bool immediate = registers[1] < 0; Xbyak::CodeGenerator code(code_size, reinterpret_cast(mapping)); const std::array saved {code.rbx, code.rbp, code.r12, code.r13, code.r14, code.r15}; const std::array gprs {code.rax, code.rcx, code.rdx, code.rbx, code.rbp, code.r8, code.r9, @@ -3322,7 +3480,7 @@ void TestPackedBitFieldExtract() { code.mov(gprs[index], gpr_sentinel | index); } code.mov(code.qword[code.rsi + offsetof(Result, stack_before)], code.rsp); - extrq(code, destination, source, 0xc8, 0xc4); + emit_bit_field(code, destination, registers[1], 0xc8, 0xc4); code.mov(code.qword[code.rsi + offsetof(Result, stack_after)], code.rsp); for (size_t index = 0; index < gprs.size(); ++index) { code.mov(code.qword[code.rsi + offsetof(Result, gprs) + index * 8], gprs[index]); @@ -3344,7 +3502,7 @@ void TestPackedBitFieldExtract() { code.pop(*reg); } code.ret(); - Check(test, Xbyak::GetError() == 0, "failed to generate EXTRQ state fixture"); + Check(test, Xbyak::GetError() == 0, "failed to generate SSE4a bitfield state fixture"); const std::vector original(code.getCode(), code.getCurr()); register_module(code_size); const std::vector trampoline(code.getCode() + code_size, @@ -3352,45 +3510,51 @@ void TestPackedBitFieldExtract() { const auto disabled = Loader::PatchGuestInstructions(mapping, code.getSize(), function_starts, false, false); Check(test, - disabled.extrq.found == 0 && + disabled.extrq.found == 0 && disabled.insertq.found == 0 && std::equal(original.begin(), original.end(), code.getCode()) && std::equal(trampoline.begin(), trampoline.end(), code.getCode() + code_size), - "disabled AMD patching changed EXTRQ code or trampolines"); + "disabled AMD patching changed SSE4a bitfield code or trampolines"); for (const bool fallback: {false, true}) { + if (insert && fallback) { + continue; // INSERTQ always uses the trap path. + } std::memcpy(reinterpret_cast(mapping), original.data(), original.size()); register_module(fallback ? 32 : code_size); - const auto counts = Loader::PatchGuestInstructions(mapping, code.getSize(), - function_starts, false, true) - .extrq; + const auto patched = Loader::PatchGuestInstructions(mapping, code.getSize(), + function_starts, false, true); + const auto counts = insert ? patched.insertq : patched.extrq; if (host_sse4a) { Check(test, counts.found == 0 && std::equal(original.begin(), original.end(), code.getCode()), - "SSE4a host received unnecessary EXTRQ patches"); + "SSE4a host received unnecessary SSE4a bitfield patches"); std::printf("[host] %-48s skipped (native SSE4a host)\n", test); return; } - std::printf("[host] EXTRQ xmm%d,%d %s found=%" PRIu64 " native=%" PRIu64 + std::printf("[host] bitfield xmm%d,%d %s found=%" PRIu64 " native=%" PRIu64 " trapped=%" PRIu64 "\n", destination, source, fallback ? "fallback" : "replacement", counts.found, counts.native, counts.trapped); Check(test, Xbyak::GetError() == 0 && counts.found == 1 && counts.Skipped() == 0 && - counts.native == (fallback ? 0u : 1u) && - counts.trapped == (fallback ? 1u : 0u), - "EXTRQ native/fallback patch counts differ"); + counts.native == (fallback || insert ? 0u : 1u) && + counts.trapped == (fallback || insert ? 1u : 0u), + "SSE4a bitfield native/fallback patch counts differ"); const auto traps_before = g_instruction_traps; uint64_t calls = 0; for (uint32_t length = 0; length < 64; ++length) { for (uint32_t index = 0; index < 64; ++index) { - if (source < 0 && (length != 8 || index != 4)) { + if (immediate && (length != 8 || index != 4)) { continue; } for (const uint64_t pattern: {UINT64_MAX, uint64_t {0x0123456789abcdef}}) { const uint64_t controls = (pattern & ~uint64_t {0xffff}) | ((index | 0xc0u) << 8) | (length | 0xc0u); - input[0] = source == destination ? controls : pattern; - input[4] = controls; + input[0] = !insert && source == destination ? controls : pattern; + input[4] = insert ? ~pattern : controls; + if (insert) { + input[source == destination ? 1 : 5] = controls; + } Result output {}; _mm_setcsr(0x5fa5); function(input.data(), &output); @@ -3398,38 +3562,47 @@ void TestPackedBitFieldExtract() { _mm_setcsr(restore.mxcsr); ++calls; Check(test, - output.destination[0] == extract(input[0], length, index) && - output.destination[1] == 0 && output.destination[2] == input[2] && + output.destination[0] == + (insert ? insert_bits(input[0], + input[source == destination ? 0 : 4], + length, index) + : extract(input[0], length, index)) && + output.destination[1] == (insert ? input[1] : 0) && + output.destination[2] == input[2] && output.destination[3] == input[3], - "EXTRQ result or destination lanes differ"); + "SSE4a bitfield result or destination lanes differ"); Check(test, source < 0 || source == destination ? output.source == output.destination : std::equal(output.source.begin(), output.source.end(), input.begin() + 4), - "EXTRQ changed its separate source register"); + "SSE4a bitfield changed its separate source register"); Check(test, mxcsr == 0x5fa5 && (output.flags & 0x8d5u) == 0x895u && output.stack_before == output.stack_after, - "EXTRQ changed MXCSR, RFLAGS or RSP"); + "SSE4a bitfield changed MXCSR, RFLAGS or RSP"); for (size_t reg = 0; reg < output.gprs.size(); ++reg) { Check(test, output.gprs[reg] == (gpr_sentinel | reg), - "EXTRQ corrupted a guest GPR"); + "SSE4a bitfield corrupted a guest GPR"); } for (size_t slot = 0; slot < output.red_zone.size(); ++slot) { Check(test, output.red_zone[slot] == (red_zone_sentinel | ((slot + 1) * 8)), - "EXTRQ corrupted the guest red zone"); + "SSE4a bitfield corrupted the guest red zone"); } } } } Check(test, static_cast(g_instruction_traps - traps_before) == - (fallback ? calls : 0), - "EXTRQ execution did not match native/fallback selection"); + (fallback || insert ? calls : 0), + "SSE4a bitfield execution did not match native/fallback selection"); } } + if (insert) { + std::printf("[host] %-48s ok (all 4096 length/index pairs)\n", test); + return; + } // Cover every immediate length/index pair without re-decoding thousands of large fixtures. Xbyak::CodeGenerator code(code_size, reinterpret_cast(mapping)); @@ -3438,7 +3611,7 @@ void TestPackedBitFieldExtract() { for (uint32_t index = 0; index < 64; ++index) { starts.push_back(mapping + code.getSize()); code.vmovups(code.ymm8, code.ptr[code.rdi]); - extrq(code, 8, -1, length | 0xc0, index | 0xc0); + emit_bit_field(code, 8, -1, length | 0xc0, index | 0xc0); code.vmovups(code.ptr[code.rsi], code.ymm8); code.vzeroupper(); code.ret(); @@ -3480,7 +3653,7 @@ void TestPackedBitFieldExtract() { code.vmovups(code.ymm1, code.ptr[code.rdi]); code.vmovups(code.ymm2, code.ptr[code.rdi + 32]); code.lea(code.rax, code.ptr[code.rip + continuation]); - extrq(code, 1, 2); + emit_bit_field(code, 1, 2); code.jmp(code.rax); code.L(continuation); code.vmovups(code.ptr[code.rsi], code.ymm1); @@ -3508,7 +3681,7 @@ void TestPackedBitFieldExtract() { code.vmovups(code.ymm1, code.ptr[code.rdi]); code.vmovups(code.ymm2, code.ptr[code.rdi + 32]); code.vmovups(code.ymm3, code.ptr[code.rdi + 64]); - extrq(code, 1, 2); + emit_bit_field(code, 1, 2); code.vrsqrtps(code.xmm4, code.xmm3); code.vmovups(code.ptr[code.rsi], code.ymm1); code.vmovups(code.ptr[code.rsi + 32], code.ymm4); @@ -3538,6 +3711,170 @@ void TestPackedBitFieldExtract() { std::printf("[host] %-48s ok (all 4096 length/index pairs)\n", test); } +void TestPackedBitFieldExtract() { + TestPackedBitField(false); +} + +void TestPackedBitFieldInsert() { + TestPackedBitField(true); +} + +void TestCpuExtensionPatches() { + const char* test = "CpuExtensionPatches"; + constexpr uint64_t code_size = 0x4000; + const auto mapping = Libs::LibKernel::Memory::AllocateProgramMemory( + 0x906000000, code_size * 2, Common::VirtualMemory::Mode::ExecuteReadWrite, + "cpu_extension_test"); + Check(test, mapping != 0, "failed to allocate CPU extension fixture"); + InstructionTestScope restore {mapping, code_size * 2}; + restore.InstallHandler(test); + const auto host = Loader::GetGuestInstructionHostFeatures(); + const std::array starts {mapping}; + const auto register_module = [&](uint64_t capacity) { + std::memset(reinterpret_cast(mapping + code_size), 0, code_size); + Loader::RegisterGuestInstructionPatchModule(reinterpret_cast(mapping), code_size, + reinterpret_cast(mapping + code_size), + capacity); + }; + Xbyak::CodeGenerator code(code_size, reinterpret_cast(mapping)); + using GuestFunction = void(KYTY_SYSV_ABI*)(const uint64_t*, uint64_t*); + const auto function = reinterpret_cast(mapping); + for (const bool writes_rsp: {false, true}) { + code.reset(); + code.push(code.r12); + code.mov(code.r12, code.rsp); + code.vmovups(code.ymm8, code.ptr[code.rdi]); + code.vmovups(code.ymm9, code.ptr[code.rdi + 32]); + code.mov(code.r10, 0x123456789abcdef0ull); + code.mov(code.qword[code.rsp - 8], code.r10); + code.clwb(code.ptr[code.rsp - 8]); + for (const uint8_t byte: {0xf2, 0x45, 0x0f, 0x79, 0xc1}) { + code.db(byte); // INSERTQ xmm8,xmm9 + } + code.db(0xf3); // RDPID rax/rsp + code.db(0x48); + code.db(0x0f); + code.db(0xc7); + code.db(writes_rsp ? 0xfc : 0xf8); + if (writes_rsp) { + code.mov(code.rax, code.rsp); + code.mov(code.rsp, code.r12); + } + code.mov(code.qword[code.rsi], code.rax); + code.mov(code.rax, code.qword[code.rsp - 8]); + code.mov(code.qword[code.rsi + 8], code.rax); + code.vmovups(code.ptr[code.rsi + 16], code.ymm8); + code.vzeroupper(); + code.pop(code.r12); + code.ret(); + Check(test, Xbyak::GetError() == 0, "failed to generate mixed CPU extension fixture"); + const std::vector original(code.getCode(), code.getCurr()); + register_module(code_size); + const auto disabled = + Loader::PatchGuestInstructions(mapping, code.getSize(), starts, false, false, {}); + Check(test, + disabled.insertq.found == 0 && disabled.rdpid.found == 0 && + disabled.clwb.found == 0 && + std::equal(original.begin(), original.end(), code.getCode()) && + std::all_of(code.getCode() + code_size, code.getCode() + code_size * 2, + [](uint8_t byte) { return byte == 0; }), + "disabled AMD option changed CPU extension code or trampolines"); + const auto supported = Loader::PatchGuestInstructions(mapping, code.getSize(), starts, + false, true, {true, true, true}); + Check(test, + supported.insertq.found == 0 && supported.rdpid.found == 0 && + supported.clwb.found == 0 && + std::equal(original.begin(), original.end(), code.getCode()), + "available host CPU extensions were unnecessarily patched"); + const auto enabled = + Loader::PatchGuestInstructions(mapping, code.getSize(), starts, false, true, {}); + Check(test, + enabled.clwb.found == 1 && enabled.clwb.native == 1 && enabled.clwb.trapped == 0 && + enabled.insertq.found == 1 && enabled.insertq.native == 0 && + enabled.insertq.trapped == 1 && enabled.rdpid.found == 1 && + enabled.rdpid.native == 0 && enabled.rdpid.trapped == 1, + "mixed native CLWB and trapped INSERTQ/RDPID counts differ"); + // Verify the actual stack-relative CLFLUSH operand, since flushing the wrong mapped + // cache line would otherwise leave the sentinel unchanged too. + ZydisDecoder decoder {}; + Check(test, + ZYAN_SUCCESS( + ZydisDecoderInit(&decoder, ZYDIS_MACHINE_MODE_LONG_64, ZYDIS_STACK_WIDTH_64)), + "failed to initialize decoder"); + bool found_flush = false; + for (size_t offset = 0; offset < 512 && !found_flush;) { + ZydisDecodedInstruction instruction {}; + ZydisDecodedOperand operands[ZYDIS_MAX_OPERAND_COUNT] {}; + Check(test, + ZYAN_SUCCESS(ZydisDecoderDecodeFull(&decoder, code.getCode() + code_size + offset, + code_size - offset, &instruction, operands)), + "failed to decode CPU extension trampoline"); + if (instruction.mnemonic == ZYDIS_MNEMONIC_CLFLUSH) { + found_flush = true; + Check(test, + operands[0].mem.base == ZYDIS_REGISTER_RSP && +#if KYTY_PLATFORM == KYTY_PLATFORM_WINDOWS + operands[0].mem.disp.value == 120, +#else + operands[0].mem.disp.value == -8, +#endif + "CLWB replacement did not preserve its guest stack address"); + } + offset += instruction.length; + } + Check(test, found_flush, "CLWB native replacement did not emit CLFLUSH"); + const std::array input {0x1111, 0xdeadbeef, 0x12345678, 0x87654321, + 0xab, 0xffffc4c8, 0, 0}; + std::array output {}; + bool checked_aux = false; + for (size_t attempt = 0; attempt < 100 && !checked_aux; ++attempt) { + uint32_t before = 0; + uint32_t after = 0; + const auto traps = g_instruction_traps; + __rdtscp(&before); + function(input.data(), output.data()); + __rdtscp(&after); + Check(test, g_instruction_traps == traps + !host.sse4a + !host.rdpid, + "mixed CPU extension fixture used an unexpected trap path"); + Check(test, + output[1] == 0x123456789abcdef0ull && output[2] == 0x1ab1 && + (host.sse4a || output[3] == input[1]) && + std::equal(output.begin() + 4, output.end(), input.begin() + 2), + "mixed CPU extensions corrupted INSERTQ state or the guest red zone"); + if (before == after) { + Check(test, output[0] == before, + "RDPID patch lost TSC_AUX or adjusted its RSP destination"); + checked_aux = true; + } + } + Check(test, checked_aux, "thread migrated repeatedly during patched RDPID comparison"); + } + + code.reset(); + code.clwb(code.ptr[code.rdi]); + code.ret(); + Check(test, Xbyak::GetError() == 0, "failed to generate exhausted CLWB fixture"); + const std::vector original(code.getCode(), code.getCurr()); + register_module(1); + const auto exhausted = + Loader::PatchGuestInstructions(mapping, code.getSize(), starts, false, true, {}).clwb; + Check(test, + exhausted.found == 1 && exhausted.native == 0 && exhausted.trapped == 1 && + std::equal(original.begin(), original.end(), code.getCode()) && + Xbyak::GetError() == 0, + "exhausted CLWB replacement did not retain its natural trap fallback"); + std::array data; + data.fill(0x123456789abcdef0ull); + const auto traps = g_instruction_traps; + function(data.data(), nullptr); + Check(test, + g_instruction_traps == traps + !host.clwb && + std::all_of(data.begin(), data.end(), + [](uint64_t value) { return value == 0x123456789abcdef0ull; }), + "CLWB direct fallback changed memory or used an unexpected trap path"); + std::printf("[host] %-48s ok\n", test); +} + #endif #if defined(__x86_64__) || defined(_M_X64) @@ -3714,6 +4051,12 @@ int main(int argc, char** argv) { } #endif #if defined(__linux__) || KYTY_PLATFORM == KYTY_PLATFORM_WINDOWS + if (argc == 2 && std::strcmp(argv[1], "--cpu-extensions-only") == 0) { + RunTest(TestCpuExtensionContexts); + RunTest(TestPackedBitFieldInsert); + RunTest(TestCpuExtensionPatches); + return g_failed_tests == 0 ? 0 : 1; + } if (argc == 2 && std::strcmp(argv[1], "--extrq-only") == 0) { RunTest(TestPackedBitFieldExtract); return g_failed_tests == 0 ? 0 : 1; @@ -3734,6 +4077,9 @@ int main(int argc, char** argv) { #if defined(__linux__) || KYTY_PLATFORM == KYTY_PLATFORM_WINDOWS RunTest(TestPackedReciprocalSquareRoot); RunTest(TestPackedBitFieldExtract); + RunTest(TestCpuExtensionContexts); + RunTest(TestPackedBitFieldInsert); + RunTest(TestCpuExtensionPatches); #endif RunTest(TestWindowsGuestRedZoneStaticPatcher); RunTest(TestProsperoArgumentAndInfoSizeContracts); diff --git a/tests/macosSse4aTests.cpp b/tests/macosSse4aTests.cpp index 479e82402..4e7b502f1 100644 --- a/tests/macosSse4aTests.cpp +++ b/tests/macosSse4aTests.cpp @@ -8,6 +8,7 @@ #include #include #include +#include #include #include #include @@ -61,6 +62,21 @@ constexpr uint8_t kInsertqXmm10Xmm9Len16Index8[] = { 0xc3, // ret }; +constexpr uint8_t kInsertqXmm10Xmm9[] = { + 0xf3, 0x44, 0x0f, 0x6f, 0x0e, // movdqu (%rsi), %xmm9 + 0xf3, 0x44, 0x0f, 0x6f, 0x12, // movdqu (%rdx), %xmm10 + 0xf2, 0x45, 0x0f, 0x79, 0xd1, // insertq xmm10, xmm9 + 0xf3, 0x44, 0x0f, 0x7f, 0x17, // movdqu %xmm10, (%rdi) + 0xc3, +}; + +constexpr uint8_t kInsertqXmm9Xmm9[] = { + 0xf3, 0x44, 0x0f, 0x6f, 0x0e, // movdqu (%rsi), %xmm9 + 0xf2, 0x45, 0x0f, 0x79, 0xc9, // insertq xmm9, xmm9 + 0xf3, 0x44, 0x0f, 0x7f, 0x0f, // movdqu %xmm9, (%rdi) + 0xc3, +}; + constexpr uint8_t kUd2[] = { 0x0f, 0x0b, // ud2 0xc3, // ret @@ -192,6 +208,31 @@ void TestRexHighRegisterInsertqWithIndex() { Check(out[1] == 0xbbbbbbbbbbbbbbbbull, "insertq xmm10, xmm9, 16, 8 leaves the upper 64 bits untouched"); } +void TestInsertqRegister() { + const auto separate = MapCode(kInsertqXmm10Xmm9); + const auto aliased = MapCode(kInsertqXmm9Xmm9); + for (const auto fn: {separate, aliased}) { + for (const uint64_t controls: {0xc4c8ull, 0xc0c0ull, 0xffc1ull}) { + const uint64_t source[2] {0x0123456789abcdefull, 0xdeadbeef00000000ull | controls}; + const uint64_t destination[2] {0x8877665544332211ull, 0xfedcba9876543210ull}; + const auto* initial = fn == aliased ? source : destination; + uint64_t expected = initial[0]; + const auto length = (controls & 63) == 0 ? 64 : controls & 63; + const auto index = (controls >> 8) & 63; + for (uint64_t bit = 0; bit < length && index + bit < 64; ++bit) { + const uint64_t mask = uint64_t {1} << (index + bit); + expected = (expected & ~mask) | (((source[0] >> bit) & 1) << (index + bit)); + } + uint64_t out[2] {}; + const int before = IllegalHits(); + fn(out, source, destination); + Check(IllegalHits() == before + 1, "register INSERTQ raises one SIGILL"); + Check(out[0] == expected && out[1] == initial[1], + "register INSERTQ uses upper source controls and preserves upper destination"); + } + } +} + void TestUnknownInstructionRefused() { g_refused.store(false, std::memory_order_relaxed); g_refuse_skip.store(2, std::memory_order_relaxed); @@ -213,6 +254,7 @@ int main() { TestInsertqImmediate(); TestRexHighRegisterExtrq(); TestRexHighRegisterInsertqWithIndex(); + TestInsertqRegister(); TestUnknownInstructionRefused(); std::printf("macosSse4aTests: all passed\n"); return 0;