Linux: port the emulator to a working state on Linux (#117)

Linux: port the emulator to a working state

src/loader/x64InstructionEmulator.cpp

@@ -4,20 +4,13 @@
 
 #if KYTY_PLATFORM == KYTY_PLATFORM_WINDOWS
 #include <windows.h> // IWYU pragma: keep
+#elif !defined(__APPLE__)
+#include <sched.h>
+#include <ucontext.h>
 #endif
 
 namespace Loader::X64InstructionEmulator {
 
-#if KYTY_PLATFORM == KYTY_PLATFORM_WINDOWS
-
-static M128A* GetContextXmm(PCONTEXT context, uint8_t index) {
-	if (context == nullptr || index >= 16) {
-		return nullptr;
-	}
-
-	return &context->Xmm0 + index;
-}
-
 static uint64_t ExtractBitField(uint64_t value, uint32_t length, uint32_t index) {
 	length &= 0x3fu;
 	index &= 0x3fu;
@@ -63,6 +56,16 @@ static uint64_t InsertBitField(uint64_t dst, uint64_t src, uint32_t length, uint
 	return (dst & ~shifted) | src_shifted;
 }
 
+#if KYTY_PLATFORM == KYTY_PLATFORM_WINDOWS
+
+static M128A* GetContextXmm(PCONTEXT context, uint8_t index) {
+	if (context == nullptr || index >= 16) {
+		return nullptr;
+	}
+
+	return &context->Xmm0 + index;
+}
+
 static bool TryEmulateSse4a(PCONTEXT context) {
 	if (context == nullptr) {
 		return false;
@@ -140,12 +143,125 @@ static bool TryEmulateMonitorxMwaitx(PCONTEXT context) {
 	return true;
 }
 
+#elif !defined(__APPLE__)
+
+// Linux signal contexts expose registers through ucontext_t.
+
+static uint32_t* GetContextXmm(ucontext_t* context, uint8_t index) {
+	if (context == nullptr || index >= 16) {
+		return nullptr;
+	}
+
+	auto* fpregs = context->uc_mcontext.fpregs;
+	if (fpregs == nullptr) {
+		return nullptr;
+	}
+
+	return static_cast<uint32_t*>(fpregs->_xmm[index].element);
+}
+
+static uint64_t GetXmmLow(const uint32_t* xmm) {
+	return static_cast<uint64_t>(xmm[0]) | (static_cast<uint64_t>(xmm[1]) << 32u);
+}
+
+static void SetXmmLow(uint32_t* xmm, uint64_t value) {
+	xmm[0] = static_cast<uint32_t>(value);
+	xmm[1] = static_cast<uint32_t>(value >> 32u);
+}
+
+static void SetXmmHigh(uint32_t* xmm, uint64_t value) {
+	xmm[2] = static_cast<uint32_t>(value);
+	xmm[3] = static_cast<uint32_t>(value >> 32u);
+}
+
+static bool TryEmulateSse4a(ucontext_t* context) {
+	if (context == nullptr) {
+		return false;
+	}
+
+	auto& rip_reg = context->uc_mcontext.gregs[REG_RIP];
+
+	const auto* rip = reinterpret_cast<const uint8_t*>(rip_reg);
+
+	const uint8_t prefix = rip[0];
+	if (prefix != 0x66 && prefix != 0xf2) {
+		return false;
+	}
+
+	size_t  offset = 1;
+	uint8_t rex    = 0;
+	if ((rip[offset] & 0xf0u) == 0x40u) {
+		rex = rip[offset];
+		offset++;
+	}
+
+	if (rip[offset] != 0x0f || rip[offset + 1] != 0x78) {
+		return false;
+	}
+
+	auto modrm = rip[offset + 2];
+	if ((modrm & 0xc0u) != 0xc0u) {
+		return false;
+	}
+
+	const uint8_t reg    = ((modrm >> 3u) & 0x07u) | ((rex & 0x04u) << 1u);
+	const uint8_t rm     = (modrm & 0x07u) | ((rex & 0x01u) << 3u);
+	const uint8_t length = rip[offset + 3];
+	const uint8_t index  = rip[offset + 4];
+
+	// AMD SSE4a immediate-form EXTRQ/INSERTQ.
+	if (prefix == 0x66) {
+		auto* dst = GetContextXmm(context, rm);
+		if (dst == nullptr) {
+			return false;
+		}
+
+		SetXmmLow(dst, ExtractBitField(GetXmmLow(dst), length, index));
+		SetXmmHigh(dst, 0);
+		rip_reg += static_cast<greg_t>(offset + 5);
+		return true;
+	}
+
+	auto* dst = GetContextXmm(context, reg);
+	auto* src = GetContextXmm(context, rm);
+	if (dst == nullptr || src == nullptr) {
+		return false;
+	}
+
+	SetXmmLow(dst, InsertBitField(GetXmmLow(dst), GetXmmLow(src), length, index));
+	rip_reg += static_cast<greg_t>(offset + 5);
+	return true;
+}
+
+static bool TryEmulateMonitorxMwaitx(ucontext_t* context) {
+	if (context == nullptr) {
+		return false;
+	}
+
+	auto& rip_reg = context->uc_mcontext.gregs[REG_RIP];
+
+	const auto* rip = reinterpret_cast<const uint8_t*>(rip_reg);
+	if (rip[0] != 0x0f || rip[1] != 0x01 || (rip[2] != 0xfa && rip[2] != 0xfb)) {
+		return false;
+	}
+
+	// Approximate AMD MONITORX/MWAITX as no-op/yield.
+	if (rip[2] == 0xfb) {
+		::sched_yield();
+	}
+	rip_reg += 3;
+	return true;
+}
+
 #endif
 
 bool TryEmulate(void* native_context) {
 #if KYTY_PLATFORM == KYTY_PLATFORM_WINDOWS
 	auto* context = static_cast<PCONTEXT>(native_context);
 	return TryEmulateMonitorxMwaitx(context) || TryEmulateSse4a(context);
+#elif !defined(__APPLE__)
+	auto* context = static_cast<ucontext_t*>(native_context);
+	return TryEmulateMonitorxMwaitx(context) || TryEmulateSse4a(context);
 #else
 	(void)native_context;
 	return false;