loader: emulate immediate SSE4a instructions on macOS (#635)

* loader: emulate immediate SSE4a instructions on macOS

* tests: simplify macOS SSE4a test integration

Keep the Darwin target, CTest registration, and aggregate dependency in
one conditional block without a separate target list. Preserve the
standalone target when BUILD_TESTING is disabled.

Lowercase the test filename and diagnostic prefix, and drop unrelated
formatting changes from the instruction emulator.

Credit Brandon Strong's related SSE4a work in
https://github.com/KytyPS5/KytyPS5/pull/467.

Co-authored-by: Brandon Strong <brandostrong@gmail.com>

---------

Co-authored-by: nmzik <Nmzik@mail.ru>
Co-authored-by: Brandon Strong <brandostrong@gmail.com>

CMakeLists.txt

@@ -601,6 +601,21 @@ if(BUILD_TESTING)
 	)
 endif()
 
+# Test Darwin signal-context restoration for SSE4a instructions.
+if(APPLE AND (CMAKE_OSX_ARCHITECTURES STREQUAL "x86_64" OR
+		(NOT CMAKE_OSX_ARCHITECTURES AND CMAKE_SYSTEM_PROCESSOR MATCHES "^(x86_64|AMD64)$")))
+	add_executable(macos_sse4a_tests EXCLUDE_FROM_ALL
+		"${KYTY_TESTS_DIR}/macosSse4aTests.cpp"
+		"${KYTY_SOURCE_DIR}/loader/x64InstructionEmulator.cpp"
+	)
+	target_link_libraries(macos_sse4a_tests common fmt::fmt Zydis::Zydis)
+	target_include_directories(macos_sse4a_tests PRIVATE ${inc_headers})
+	if(BUILD_TESTING)
+		add_test(NAME macos_sse4a COMMAND $<TARGET_FILE:macos_sse4a_tests>)
+		set_tests_properties(macos_sse4a PROPERTIES TIMEOUT 15)
+		add_dependencies(kyty_tests macos_sse4a_tests)
+	endif()
+endif()
 
 add_executable(kyty_emulator "${KYTY_SOURCE_DIR}/main.cpp" ${kyty_emulator_src})
 configure_macos_guest_address_space(kyty_emulator)

src/loader/x64InstructionEmulator.cpp

@@ -12,7 +12,9 @@
 
 #if KYTY_PLATFORM == KYTY_PLATFORM_WINDOWS
 #include <windows.h> // IWYU pragma: keep
-#elif !defined(__APPLE__)
+#elif defined(__APPLE__)
+#include <sys/ucontext.h>
+#else
 #include <sched.h>
 #include <ucontext.h>
 #endif
@@ -406,8 +408,6 @@ static bool ExecuteShaNiInsn(const ShaNiInsn& insn, const XmmWords& src2, const
 	}
 }
 
-#if !defined(__APPLE__)
-
 // Keep instruction semantics shared; only access to the saved host context differs.
 struct Context {
 #if KYTY_PLATFORM == KYTY_PLATFORM_WINDOWS
@@ -439,6 +439,38 @@ struct Context {
 			ymm[index] = {};
 		}
 	}
+#elif defined(__APPLE__)
+	ucontext_t* native;
+
+	[[nodiscard]] uint64_t Rip() const {
+		return static_cast<uint64_t>(native->uc_mcontext->__ss.__rip);
+	}
+	void Advance(size_t length) {
+		native->uc_mcontext->__ss.__rip += static_cast<uint64_t>(length);
+	}
+	// Darwin names the XMM file __fpu_xmm0..__fpu_xmm15 instead of exposing an array.
+	[[nodiscard]] void* Xmm(uint8_t index) const {
+		auto* fs = &native->uc_mcontext->__fs;
+		switch (index) {
+			case 0: return &fs->__fpu_xmm0;
+			case 1: return &fs->__fpu_xmm1;
+			case 2: return &fs->__fpu_xmm2;
+			case 3: return &fs->__fpu_xmm3;
+			case 4: return &fs->__fpu_xmm4;
+			case 5: return &fs->__fpu_xmm5;
+			case 6: return &fs->__fpu_xmm6;
+			case 7: return &fs->__fpu_xmm7;
+			case 8: return &fs->__fpu_xmm8;
+			case 9: return &fs->__fpu_xmm9;
+			case 10: return &fs->__fpu_xmm10;
+			case 11: return &fs->__fpu_xmm11;
+			case 12: return &fs->__fpu_xmm12;
+			case 13: return &fs->__fpu_xmm13;
+			case 14: return &fs->__fpu_xmm14;
+			case 15: return &fs->__fpu_xmm15;
+			default: return nullptr;
+		}
+	}
 #else
 	ucontext_t* native;
 
@@ -487,6 +519,8 @@ struct Context {
 #endif
 };
 
+#if !defined(__APPLE__)
+
 static bool TryEmulateShaNi(Context& context) {
 	const auto* rip = reinterpret_cast<const uint8_t*>(context.Rip());
 	ShaNiInsn   insn {};
@@ -534,6 +568,8 @@ static bool TryEmulateShaNi(Context& context) {
 	return true;
 }
 
+#endif
+
 static bool TryEmulateSse4a(Context& context) {
 	const auto*   rip    = reinterpret_cast<const uint8_t*>(context.Rip());
 	const uint8_t prefix = rip[0];
@@ -598,6 +634,8 @@ static bool TryEmulateSse4a(Context& context) {
 	return true;
 }
 
+#if !defined(__APPLE__)
+
 static bool TryEmulateMonitorxMwaitx(Context& context) {
 	const auto* rip = reinterpret_cast<const uint8_t*>(context.Rip());
 	if (rip[0] != 0x0f || rip[1] != 0x01 || (rip[2] != 0xfa && rip[2] != 0xfb)) {
@@ -726,23 +764,28 @@ uint64_t PatchReciprocalSquareRoots(uint64_t address, uint64_t size) {
 }
 
 bool TryEmulate(void* native_context) {
-#if !defined(__APPLE__)
 	if (native_context == nullptr) {
 		return false;
 	}
 #if KYTY_PLATFORM == KYTY_PLATFORM_WINDOWS
 	Context context {static_cast<PCONTEXT>(native_context)};
+#elif defined(__APPLE__)
+	auto* saved_context = static_cast<ucontext_t*>(native_context);
+	if (saved_context->uc_mcontext == nullptr) {
+		return false;
+	}
+	Context context {saved_context};
 #else
 	Context context {static_cast<ucontext_t*>(native_context)};
 #endif
+#if !defined(__APPLE__)
 	if (TryEmulateReciprocalSquareRoot(context)) {
 		return true;
 	}
 	return TryEmulateMonitorxMwaitx(context) || TryEmulateSse4a(context) ||
 	       TryEmulateShaNi(context);
 #else
-	(void)native_context;
+	return TryEmulateSse4a(context);
-	return false;
 #endif
 }
 

tests/macosSse4aTests.cpp

@@ -0,0 +1,219 @@
+#include "common/hostException.h"
+#include "loader/x64InstructionEmulator.h"
+
+#include <atomic>
+#include <bit>
+#include <cstddef>
+#include <cstdint>
+#include <cstdio>
+#include <cstdlib>
+#include <cstring>
+#include <sys/mman.h>
+#include <sys/ucontext.h>
+#include <unistd.h>
+
+namespace {
+
+using Common::HostException::ExceptionInfo;
+using Common::HostException::ExceptionType;
+
+// The SSE4a bytes are the point of this test, so blobs are spelled out byte by
+// byte instead of going through an encoder. Every blob is a complete routine:
+// the instruction under test, a store of the result, and ret. Encodings were
+// disassembled with otool to confirm the operand roles: EXTRQ immediate is
+// unary (ModRM.r/m is both source and destination, the reg field is unused),
+// INSERTQ immediate reads ModRM.r/m into the reg field.
+constexpr uint8_t kExtrqXmm2Len40Index0[] = {
+	0x66, 0x0f, 0x78, 0xc2, 0x28, 0x00, // extrq xmm2, 40, 0
+	0xf3, 0x0f, 0x7f, 0x17,             // movdqu %xmm2, (%rdi)
+	0xc3,                               // ret
+};
+
+constexpr uint8_t kExtrqXmm2Len64Index0[] = {
+	0x66, 0x0f, 0x78, 0xc2, 0x40, 0x00, // extrq xmm2, 64, 0
+	0xf3, 0x0f, 0x7f, 0x17,             // movdqu %xmm2, (%rdi)
+	0xc3,                               // ret
+};
+
+constexpr uint8_t kInsertqXmm2Xmm0Len32Index0[] = {
+	0xf2, 0x0f, 0x78, 0xd0, 0x20, 0x00, // insertq xmm2, xmm0, 32, 0
+	0xf3, 0x0f, 0x7f, 0x17,             // movdqu %xmm2, (%rdi)
+	0xc3,                               // ret
+};
+
+// REX.B moves the operand into the high register file. xmm10 is not an argument
+// register, so the blob loads it from memory. EXTRQ has a single operand, so
+// only the destination needs loading.
+constexpr uint8_t kExtrqXmm10Len40Index0[] = {
+	0xf3, 0x44, 0x0f, 0x6f, 0x16,             // movdqu (%rsi), %xmm10
+	0x66, 0x41, 0x0f, 0x78, 0xc2, 0x28, 0x00, // extrq xmm10, 40, 0
+	0xf3, 0x44, 0x0f, 0x7f, 0x17,             // movdqu %xmm10, (%rdi)
+	0xc3,                                     // ret
+};
+
+// REX.R keeps the destination in xmm10 while REX.B puts the source in xmm9, and
+// the inserted field sits at a nonzero index.
+constexpr uint8_t kInsertqXmm10Xmm9Len16Index8[] = {
+	0xf3, 0x44, 0x0f, 0x6f, 0x0e,             // movdqu (%rsi), %xmm9
+	0xf3, 0x44, 0x0f, 0x6f, 0x12,             // movdqu (%rdx), %xmm10
+	0xf2, 0x45, 0x0f, 0x78, 0xd1, 0x10, 0x08, // insertq xmm10, xmm9, 16, 8
+	0xf3, 0x44, 0x0f, 0x7f, 0x17,             // movdqu %xmm10, (%rdi)
+	0xc3,                                     // ret
+};
+
+constexpr uint8_t kUd2[] = {
+	0x0f, 0x0b, // ud2
+	0xc3,       // ret
+};
+
+constexpr size_t kPageSize = 4096;
+
+std::atomic<int>    g_illegal_hits {0};
+std::atomic<bool>   g_refused {false};
+std::atomic<size_t> g_refuse_skip {0};
+
+static_assert(decltype(g_illegal_hits)::is_always_lock_free);
+static_assert(decltype(g_refused)::is_always_lock_free);
+static_assert(decltype(g_refuse_skip)::is_always_lock_free);
+
+void Check(bool value, const char *text) {
+	if (!value) {
+		std::fprintf(stderr, "macosSse4aTests: failed: %s\n", text);
+		std::abort();
+	}
+}
+
+bool Handler(const ExceptionInfo &info) {
+	if (info.type != ExceptionType::IllegalInstruction) {
+		return false;
+	}
+	g_illegal_hits.fetch_add(1, std::memory_order_relaxed);
+	if (Loader::X64InstructionEmulator::TryEmulate(info.native_context)) {
+		return true;
+	}
+	// A test that expects a refusal arms g_refuse_skip so the host-illegal
+	// instruction can be stepped over. Any other refusal is outside the
+	// emulator's contract, and returning there would re-run the faulting
+	// instruction forever, so end the run instead.
+	const size_t skip = g_refuse_skip.load(std::memory_order_relaxed);
+	if (skip == 0) {
+		static const char message[] = "macosSse4aTests: failed: unexpected emulator refusal\n";
+		::write(STDERR_FILENO, message, sizeof(message) - 1);
+		::_exit(1);
+	}
+	g_refused.store(true, std::memory_order_relaxed);
+	auto *saved_context = static_cast<ucontext_t *>(info.native_context);
+	saved_context->uc_mcontext->__ss.__rip += static_cast<uint64_t>(skip);
+	return true;
+}
+
+template <typename Fn, size_t N>
+Fn MapCode(const uint8_t (&code)[N]) {
+	void *page = ::mmap(nullptr, kPageSize, PROT_READ | PROT_WRITE, MAP_PRIVATE | MAP_ANON, -1, 0);
+	Check(page != MAP_FAILED, "map test code page");
+	std::memcpy(page, code, N);
+	Check(::mprotect(page, kPageSize, PROT_READ | PROT_EXEC) == 0, "make test code page executable");
+	return reinterpret_cast<Fn>(page);
+}
+
+// Blobs receive their inputs through the SysV ABI, so the xmm argument registers
+// are reachable as ordinary function parameters.
+using BlobFn     = void (*)(uint64_t *out, double xmm0, double xmm1, double xmm2);
+using LoadBlobFn = void (*)(uint64_t *out, const uint64_t *in);
+using PairBlobFn = void (*)(uint64_t *out, const uint64_t *src, const uint64_t *dest);
+using VoidBlobFn = void (*)();
+
+int IllegalHits() {
+	return g_illegal_hits.load(std::memory_order_relaxed);
+}
+
+void TestRefusals() {
+	Check(!Loader::X64InstructionEmulator::TryEmulate(nullptr), "TryEmulate refuses a null native context");
+	ucontext_t saved_context {};
+	saved_context.uc_mcontext = nullptr;
+	Check(!Loader::X64InstructionEmulator::TryEmulate(&saved_context), "TryEmulate refuses a saved context without mcontext");
+}
+
+void TestExtrqImmediate() {
+	auto     fn      = MapCode<BlobFn>(kExtrqXmm2Len40Index0);
+	uint64_t out[2] {};
+	// The pattern is already in xmm2; the emulator keeps bits [39:0] and zeroes
+	// the upper half.
+	const double pattern = std::bit_cast<double>(0xaabbccddeeff1122ull);
+	const int    before  = IllegalHits();
+	fn(out, 0.0, 0.0, pattern);
+	Check(IllegalHits() > before, "host raises SIGILL for EXTRQ");
+	Check(out[0] == 0x000000ddeeff1122ull, "extrq xmm2, 40, 0 keeps bits [39:0] of xmm2");
+	Check(out[1] == 0, "extrq xmm2, 40, 0 zero-extends the upper 64 bits");
+}
+
+void TestExtrqFullWidth() {
+	auto     fn        = MapCode<BlobFn>(kExtrqXmm2Len64Index0);
+	uint64_t out[2] {};
+	const double pattern = std::bit_cast<double>(0x0123456789abcdefull);
+	const int    before  = IllegalHits();
+	fn(out, 0.0, 0.0, pattern);
+	Check(IllegalHits() > before, "host raises SIGILL for a full width EXTRQ");
+	Check(out[0] == 0x0123456789abcdefull, "extrq xmm2, 64, 0 keeps the whole low 64 bits");
+	Check(out[1] == 0, "extrq xmm2, 64, 0 zero-extends the upper 64 bits");
+}
+
+void TestInsertqImmediate() {
+	auto     fn        = MapCode<BlobFn>(kInsertqXmm2Xmm0Len32Index0);
+	uint64_t out[2] {};
+	const double source = std::bit_cast<double>(0x1122334455667788ull);
+	const double dest   = std::bit_cast<double>(0x0123456789abcdefull);
+	const int    before = IllegalHits();
+	fn(out, source, 0.0, dest);
+	Check(IllegalHits() > before, "host raises SIGILL for INSERTQ");
+	Check(out[0] == 0x0123456755667788ull, "insertq xmm2, xmm0, 32, 0 inserts bits [31:0] of xmm0");
+}
+
+void TestRexHighRegisterExtrq() {
+	auto           fn     = MapCode<LoadBlobFn>(kExtrqXmm10Len40Index0);
+	uint64_t       out[2] {};
+	const uint64_t in[2] {0xaabbccddeeff1122ull, 0};
+	const int      before = IllegalHits();
+	fn(out, in);
+	Check(IllegalHits() > before, "host raises SIGILL for a REX.B EXTRQ");
+	Check(out[0] == 0x000000ddeeff1122ull, "REX.B destination xmm10 is emulated through the saved context");
+	Check(out[1] == 0, "REX.B destination xmm10 zero-extends the upper 64 bits");
+}
+
+void TestRexHighRegisterInsertqWithIndex() {
+	auto           fn     = MapCode<PairBlobFn>(kInsertqXmm10Xmm9Len16Index8);
+	uint64_t       out[2] {};
+	const uint64_t src[2] {0x8877665544332211ull, 0xdeadbeefcafebabeull};
+	const uint64_t dest[2] {0xaaaaaaaaaaaaaaaaull, 0xbbbbbbbbbbbbbbbbull};
+	const int      before = IllegalHits();
+	fn(out, src, dest);
+	Check(IllegalHits() > before, "host raises SIGILL for a REX.R/REX.B INSERTQ");
+	Check(out[0] == 0xaaaaaaaaaa2211aaull, "insertq xmm10, xmm9, 16, 8 inserts bits [15:0] at index 8");
+	Check(out[1] == 0xbbbbbbbbbbbbbbbbull, "insertq xmm10, xmm9, 16, 8 leaves the upper 64 bits untouched");
+}
+
+void TestUnknownInstructionRefused() {
+	g_refused.store(false, std::memory_order_relaxed);
+	g_refuse_skip.store(2, std::memory_order_relaxed);
+	auto      fn     = MapCode<VoidBlobFn>(kUd2);
+	const int before = IllegalHits();
+	fn();
+	Check(IllegalHits() > before, "host raises SIGILL for the unknown instruction");
+	Check(g_refused.load(std::memory_order_relaxed), "TryEmulate refuses an unknown instruction");
+	g_refuse_skip.store(0, std::memory_order_relaxed);
+}
+
+} // namespace
+
+int main() {
+	Check(Common::HostException::InstallHandler(Handler), "install host exception handler");
+	TestRefusals();
+	TestExtrqImmediate();
+	TestExtrqFullWidth();
+	TestInsertqImmediate();
+	TestRexHighRegisterExtrq();
+	TestRexHighRegisterInsertqWithIndex();
+	TestUnknownInstructionRefused();
+	std::printf("macosSse4aTests: all passed\n");
+	return 0;
+}