loader: enable the AMD CPU instruction patcher on macOS (#1014)

* loader: enable AMD CPU instruction patching on macOS

The guest instruction patcher was compiled out on macOS and runtimeLinker forced emulate_amd off,
so under Rosetta every SSE4a EXTRQ/INSERTQ took the SIGILL path (about 18 us each). Build the
patcher on all hosts and let --amd-cpu enable it on macOS like on Windows and Linux. The host
feature probe already reports no SSE4a under Rosetta, so EXTRQ becomes a native trampoline and the
remaining fallbacks keep using the x64 instruction emulator through the host exception handler.

* tests: macos_sse4a covers the guest instruction patcher on macOS

Patch EXTRQ/INSERTQ blobs in an RWX area with a trampoline page behind it, with the host features of
Rosetta (no SSE4a/RDPID/CLWB), and compare against the SIGILL emulator: EXTRQ must become a native
trampoline without SIGILL, INSERTQ stays a trap handled by the emulator, and nothing is touched
without emulate_amd.

* loader: emulate the patcher's VRSQRTPS trap and MWAITX on macOS

The AMD CPU patcher marks VRSQRTPS it cannot relocate by clearing a reserved
VEX.vvvv bit, which raises #UD on every host including Rosetta. The macOS build
left TryEmulateReciprocalSquareRoot and TryEmulateMonitorxMwaitx out, so the
guest died with an unhandled SIGILL at the first such site.

Both are plain SSE2/libc, so enable them on macOS. The Darwin context also
gains ClearUpperYmm so the VEX.128 result zeroes YMM[255:128] when the thread
has the AVX context. macos_sse4a covers the trapped VRSQRTPS and MWAITX.

* tests: macos_sse4a runs a plain VEX.128 VRSQRTPS through the patcher

The existing case started from an instruction that was already marked as a trap. The
new cases feed the ordinary encoding to PatchGuestInstructions and compare the result
with the emulated reference, both through a native trampoline and, when the instruction
cannot be relocated, through the trap mark and the signal emulator.

* loader: compact macOS instruction patcher coverage

Share the emulator dispatch chain while keeping SHA emulation limited to non-Apple hosts and preserving handler order. Remove a duplicate VRSQRTPS signal test already covered by both native and trapped patcher cases, plus blank lines left by removed platform guards.

---------

Co-authored-by: nmzik <Nmzik@mail.ru>

src/loader/guestInstructionPatcher.cpp

@@ -21,10 +21,8 @@
 #include <set>
 #include <unordered_set>
 #include <vector>
-#if !defined(__APPLE__)
 #include <xbyak/xbyak.h>
 #include <xbyak/xbyak_util.h>
-#endif
 
 #ifdef min
 #undef min
@@ -33,9 +31,7 @@
 #undef max
 #endif
 
-#if !defined(__APPLE__)
 using namespace Xbyak::util;
-#endif
 
 namespace Loader {
 
@@ -52,8 +48,6 @@ using s64 = int64_t;
 
 constexpr size_t NearJumpSize = 5;
 
-#if !defined(__APPLE__)
-
 struct PatchModule {
 	std::mutex           mutex {};
 	u8*                  start = nullptr;
@@ -1573,19 +1567,6 @@ GuestInstructionPatchResult PatchGuestInstructions(u64 segment_addr, u64 segment
 	return result;
 }
 
-#else
-
-GuestInstructionHostFeatures GetGuestInstructionHostFeatures() {
-	return {};
-}
-
-GuestInstructionPatchResult PatchGuestInstructions(u64, u64, std::span<const uintptr_t>, bool, bool,
-                                                   GuestInstructionHostFeatures) {
-	return {};
-}
-
-#endif
-
 namespace {
 
 constexpr uint8_t DW_EH_PE_FORMAT_MASK      = 0x0f;
@@ -1763,7 +1744,6 @@ bool DecodeEhFrameFunctionStarts(uint64_t eh_frame_header_addr, uint64_t eh_fram
 
 void RegisterGuestInstructionPatchModule(void* module_ptr, uint64_t module_size,
                                          void* trampoline_area_ptr, uint64_t trampoline_area_size) {
-#if !defined(__APPLE__)
 	EXIT_IF(module_ptr == nullptr || module_size == 0 || trampoline_area_ptr == nullptr ||
 	        trampoline_area_size == 0);
 	const auto module_addr = reinterpret_cast<u64>(module_ptr);
@@ -1772,20 +1752,10 @@ void RegisterGuestInstructionPatchModule(void* module_ptr, uint64_t module_size,
 	                        std::forward_as_tuple(static_cast<u8*>(module_ptr), module_size,
 	                                              static_cast<u8*>(trampoline_area_ptr),
 	                                              trampoline_area_size));
-#else
-	(void)module_ptr;
-	(void)module_size;
-	(void)trampoline_area_ptr;
-	(void)trampoline_area_size;
-#endif
 }
 
 void UnregisterGuestInstructionPatchModule(void* module_ptr) {
-#if !defined(__APPLE__)
 	g_patch_modules.erase(reinterpret_cast<u64>(module_ptr));
-#else
-	(void)module_ptr;
-#endif
 }
 
 #undef ASSERT

src/loader/runtimeLinker.cpp

@@ -1670,11 +1670,7 @@ void RuntimeLinker::LoadProgramToMemory(Program* program) {
 	uint64_t tls_handler_size = is_shared ? 0 : Jit::SafeCall::GetSize();
 	EXIT_IF(tls_handler_size > UINT64_MAX - program->base_size_aligned);
 	program->mapped_size = program->base_size_aligned + tls_handler_size;
-#if !defined(__APPLE__)
 	const bool emulate_amd = Config::AmdCpuEnabled();
-#else
-	const bool emulate_amd = false;
-#endif
 #if KYTY_PLATFORM == KYTY_PLATFORM_WINDOWS
 	const bool protect_memory_faults = Config::RedZoneProtectionEnabled();
 #else

src/loader/x64InstructionEmulator.cpp

@@ -12,6 +12,7 @@
 #if KYTY_PLATFORM == KYTY_PLATFORM_WINDOWS
 #include <windows.h> // IWYU pragma: keep
 #elif defined(__APPLE__)
+#include <sched.h>
 #include <sys/ucontext.h>
 #else
 #include <sched.h>
@@ -468,6 +469,16 @@ struct Context {
 			default: return nullptr;
 		}
 	}
+	// The kernel hands AVX threads the longer context that carries YMM[255:128]; a plain SSE
+	// context has no such state to clear.
+	void ClearUpperYmm(uint8_t index) const {
+		if (index >= 16 || native->uc_mcsize < sizeof(_STRUCT_MCONTEXT_AVX64)) {
+			return;
+		}
+		auto* avx = reinterpret_cast<_STRUCT_MCONTEXT_AVX64*>(native->uc_mcontext);
+		auto* ymmh = &avx->__fs.__fpu_ymmh0;
+		ymmh[index] = {};
+	}
 #else
 	ucontext_t* native;
 
@@ -703,8 +714,6 @@ static bool TryEmulateCpuExtensions(Context& context) {
 	return true;
 }
 
-#if !defined(__APPLE__)
-
 static bool TryEmulateMonitorxMwaitx(Context& context) {
 	const auto* rip = reinterpret_cast<const uint8_t*>(context.Rip());
 	if (rip[0] != 0x0f || rip[1] != 0x01 || (rip[2] != 0xfa && rip[2] != 0xfb)) {
@@ -790,8 +799,6 @@ static bool TryEmulateReciprocalSquareRoot(Context& context) {
 	return true;
 }
 
-#endif
-
 bool TryEmulate(void* native_context) {
 	if (native_context == nullptr) {
 		return false;
@@ -807,12 +814,12 @@ bool TryEmulate(void* native_context) {
 #else
 	Context context {static_cast<ucontext_t*>(native_context)};
 #endif
-#if !defined(__APPLE__)
 	return TryEmulateReciprocalSquareRoot(context) || TryEmulateMonitorxMwaitx(context) ||
-	       TryEmulateSse4a(context) || TryEmulateShaNi(context) || TryEmulateCpuExtensions(context);
-#else
-	return TryEmulateSse4a(context) || TryEmulateCpuExtensions(context);
+	       TryEmulateSse4a(context) ||
+#if !defined(__APPLE__)
+	       TryEmulateShaNi(context) ||
 #endif
+	       TryEmulateCpuExtensions(context);
 }
 
 } // namespace Loader::X64InstructionEmulator