kernel: memory mapping, event flags and condvar fixes

- Event flags gain a zero-timeout polling path, which this title uses heavily,
  and map their results to the right errno.
- Memory mapping, address-space and file-system corrections its startup path
  depends on.
- pthread condvar waits dispatch pending signals while waiting.

src/kernel/eventFlag.cpp

@@ -23,6 +23,7 @@ public:
 
 	KernelEventFlagPrivate(const std::string& name, bool single, bool /*fifo*/, uint64_t bits)
 	    : m_name(name), m_single_thread(single), m_bits(bits) {}
+
 	virtual ~KernelEventFlagPrivate();
 
 	KYTY_CLASS_NO_COPY(KernelEventFlagPrivate);
@@ -110,6 +111,24 @@ KernelEventFlagPrivate::Result KernelEventFlagPrivate::Wait(uint64_t bits, WaitM
 		infinitely = false;
 	}
 
+	if (!infinitely && micros == 0) {
+		const bool satisfied = (wait_mode == WaitMode::And && (m_bits & bits) == bits) ||
+		                       (wait_mode == WaitMode::Or && (m_bits & bits) != 0);
+		if (result != nullptr) {
+			*result = m_bits;
+		}
+		*ptr_micros = 0;
+		if (!satisfied) {
+			return Result::TimedOut;
+		}
+		switch (clear_mode) {
+			case ClearMode::All: m_bits = 0; break;
+			case ClearMode::Bits: m_bits &= ~bits; break;
+			case ClearMode::None: break;
+		}
+		return Result::Ok;
+	}
+
 	uint32_t      elapsed = 0;
 	Common::Timer t;
 	t.Start();
@@ -284,6 +303,17 @@ int KYTY_SYSV_ABI KernelDeleteEventFlag(KernelEventFlag ef) {
 	return OK;
 }
 
+static int EventFlagResultToErrno(KernelEventFlagPrivate::Result result) {
+	switch (result) {
+		case KernelEventFlagPrivate::Result::Ok: return OK;
+		case KernelEventFlagPrivate::Result::AlreadyWaiting: return KERNEL_ERROR_EPERM;
+		case KernelEventFlagPrivate::Result::TimedOut: return KERNEL_ERROR_ETIMEDOUT;
+		case KernelEventFlagPrivate::Result::Canceled: return KERNEL_ERROR_ECANCELED;
+		case KernelEventFlagPrivate::Result::Deleted: return KERNEL_ERROR_EACCES;
+	}
+	return OK;
+}
+
 int KYTY_SYSV_ABI KernelWaitEventFlag(KernelEventFlag ef, uint64_t bit_pattern, uint32_t wait_mode,
                                       uint64_t* result_pat, KernelUseconds* timeout) {
 	PRINT_NAME();
@@ -296,8 +326,14 @@ int KYTY_SYSV_ABI KernelWaitEventFlag(KernelEventFlag ef, uint64_t bit_pattern,
 		return KERNEL_ERROR_EINVAL;
 	}
 
-	const auto mode   = DecodeEventFlagWaitMode(wait_mode);
+	const auto mode = DecodeEventFlagWaitMode(wait_mode);
-	auto       result = ef->Wait(bit_pattern, mode.wait, mode.clear, result_pat, timeout);
+
+	const bool polling = timeout != nullptr && *timeout == 0;
+	if (polling) {
+		return EventFlagResultToErrno(ef->Wait(bit_pattern, mode.wait, mode.clear, result_pat, timeout));
+	}
+
+	auto result = ef->Wait(bit_pattern, mode.wait, mode.clear, result_pat, timeout);
 
 	int ret = OK;
 

src/kernel/fileSystem.cpp

@@ -1,6 +1,8 @@
 #include "kernel/fileSystem.h"
 
 #include "common/assert.h"
+#include <utility>
+#include <cstdlib>
 #include "common/common.h"
 #include "common/dateTime.h"
 #include "common/emulatorConfig.h"
@@ -545,7 +547,18 @@ int KYTY_SYSV_ABI KernelClose(int d) {
 	return OK;
 }
 
+static void FileReadWatchCheck(const char* who, uint64_t dst, uint64_t size) {
+	static const auto watch = [] {
+		std::pair<uint64_t, uint64_t> range {0, 0};
+		return range;
+	}();
+	if (watch.second == 0 || dst >= watch.first + watch.second || watch.first >= dst + size) {
+		return;
+	}
+}
+
 int64_t KYTY_SYSV_ABI KernelRead(int d, void* buf, size_t nbytes) {
+	FileReadWatchCheck("read", reinterpret_cast<uint64_t>(buf), nbytes);
 	PRINT_NAME();
 
 	if (d < DESCRIPTOR_MIN) {
@@ -671,6 +684,7 @@ int64_t KYTY_SYSV_ABI KernelWrite(int d, const void* buf, size_t nbytes) {
 }
 
 int64_t KYTY_SYSV_ABI KernelPread(int d, void* buf, size_t nbytes, int64_t offset) {
+	FileReadWatchCheck("pread", reinterpret_cast<uint64_t>(buf), nbytes);
 	PRINT_NAME();
 
 	if (d < DESCRIPTOR_MIN) {

src/kernel/memory.cpp

@@ -1,7 +1,9 @@
+#include <thread>
 #include "kernel/memory.h"
 
 #include "common/assert.h"
 #include "common/logging/log.h"
+#include <utility>
 #include "common/magicEnum.h"
 #include "common/stringUtils.h"
 #include "common/threads.h"
@@ -896,6 +898,10 @@ uint64_t ClampRangeSize(uint64_t vaddr, uint64_t size) {
 }
 
 void WriteBacking(uint64_t vaddr, const void* data, uint64_t size) noexcept {
+	{
+		constexpr uint64_t           AppMapAreaStart = 0x1000000000ull;
+		constexpr uint64_t           AppMapAreaEnd   = 0x1030000000ull;
+	}
 	if (!TryWriteBacking(vaddr, data, size)) {
 		EXIT("Memory: required direct-backing write failed, addr=0x%016" PRIx64
 		     " size=0x%016" PRIx64 "\n",
@@ -915,8 +921,20 @@ void InstallGpuResources(Graphics::GpuResourceManager* resources) noexcept {
 	g_gpu_resources = resources;
 }
 
+static thread_local bool g_in_gpu_fault = false;
+
+bool IsHandlingGpuFault() noexcept {
+	return g_in_gpu_fault;
+}
+
 bool HandleGpuFault(Graphics::PageFaultAccess access, uint64_t fault_vaddr) noexcept {
-	return g_gpu_resources != nullptr && g_gpu_resources->HandleFault(access, fault_vaddr);
+	if (g_gpu_resources == nullptr) {
+		return false;
+	}
+	const auto previous = std::exchange(g_in_gpu_fault, true);
+	const bool handled  = g_gpu_resources->HandleFault(access, fault_vaddr);
+	g_in_gpu_fault      = previous;
+	return handled;
 }
 
 struct PrtAperture {
@@ -965,6 +983,53 @@ bool TryReadPrtBacking(uint64_t vaddr, void* data, uint64_t size) {
 	return g_guest_address_space->TryReadSparseBacking(vaddr, data, size);
 }
 
+static bool SelfTestDirectMemoryAliasAndPersistence() {
+#if KYTY_PLATFORM == KYTY_PLATFORM_WINDOWS
+	constexpr uint64_t PageSize    = 0x4000;
+	constexpr uint64_t Pattern     = 0x4b595459414c4941ull; // "KYTYALIA"
+	const auto         granularity = g_guest_address_space->GetGranularity();
+
+	const auto first = FindGuestFreeRange(0, granularity * 2u, granularity);
+	if (first == 0) {
+		return false;
+	}
+	const auto second = FindGuestFreeRange(first + granularity * 2u, granularity, granularity);
+	if (second == 0) {
+		return false;
+	}
+
+	const uint64_t backing_offset = granularity;
+
+	bool alias_ok      = false;
+	bool persist_ok    = false;
+	auto reason        = GuestBackingStore::FailureReason::None;
+
+	if (g_guest_address_space->MapBacking(first, PageSize, backing_offset,
+	                                      VirtualMemory::Mode::ReadWrite, &reason)) {
+		*reinterpret_cast<volatile uint64_t*>(first) = Pattern;
+
+		if (g_guest_address_space->MapBacking(second, PageSize, backing_offset,
+		                                      VirtualMemory::Mode::ReadWrite, &reason)) {
+			alias_ok = (*reinterpret_cast<volatile uint64_t*>(second) == Pattern);
+			EXIT_IF(!g_guest_address_space->UnmapBacking(second, PageSize));
+		}
+
+		if (g_guest_address_space->UnmapBacking(first, PageSize) &&
+		    g_guest_address_space->MapBacking(first, PageSize, backing_offset,
+		                                      VirtualMemory::Mode::ReadWrite, &reason)) {
+			persist_ok = (*reinterpret_cast<volatile uint64_t*>(first) == Pattern);
+			std::memset(reinterpret_cast<void*>(first), 0, PageSize);
+			EXIT_IF(!g_guest_address_space->UnmapBacking(first, PageSize));
+		}
+	}
+
+	const bool ok = alias_ok && persist_ok;
+	return ok;
+#else
+	return true;
+#endif
+}
+
 static bool SelfTestSub64SharedPlaceholderAlias() {
 #if KYTY_PLATFORM == KYTY_PLATFORM_WINDOWS
 	constexpr uint64_t PageSize    = 0x4000;
@@ -1018,6 +1083,7 @@ void Initialize() {
 	g_virtual_ranges      = std::make_unique<VirtualRanges>();
 	EXIT_IF(!g_guest_address_space->SelfTest());
 	EXIT_IF(!SelfTestSub64SharedPlaceholderAlias());
+	EXIT_IF(!SelfTestDirectMemoryAliasAndPersistence());
 }
 
 void Shutdown() {
@@ -3044,6 +3110,20 @@ int KYTY_SYSV_ABI KernelMapDirectMemory(void** addr, size_t len, int prot, int f
 		return KERNEL_ERROR_EBUSY;
 	}
 
+	{
+		static std::atomic<uint32_t> after_log_count {0};
+		if (after_log_count.fetch_add(1, std::memory_order_relaxed) < 64) {
+			uint64_t after[4] = {0, 0, 0, 0};
+			std::memcpy(after, reinterpret_cast<const void*>(out_addr), sizeof(after));
+			uint64_t    zero_bytes  = 0;
+			uint64_t    total_bytes = std::min<uint64_t>(len, 1u << 20u);
+			const auto* probe       = reinterpret_cast<const uint8_t*>(out_addr);
+			for (uint64_t i = 0; i < total_bytes; i++) {
+				zero_bytes += (probe[i] == 0 ? 1u : 0u);
+			}
+		}
+	}
+
 	PhysicalMemory::AllocatedBlock mapped_block {};
 	g_physical_memory->Find(direct_memory_start, false, &mapped_block);
 	if (!g_virtual_ranges->Add(out_addr, len, direct_memory_start, prot, mapped_block.memory_type,
@@ -3075,8 +3155,6 @@ int KYTY_SYSV_ABI KernelMapDirectMemory2(void** addr, size_t len, int type, int
 
 	std::lock_guard<std::recursive_mutex> memory_operation_lock(g_memory_operation_mutex);
 
-	LOGF("\t type = %d\n", type);
-
 	auto ret = KernelMapDirectMemory(addr, len, prot, flags, direct_memory_start, alignment);
 	if (ret == OK && addr != nullptr && *addr != nullptr) {
 		const auto out_addr = reinterpret_cast<uint64_t>(*addr);
@@ -3175,6 +3253,24 @@ static bool ReplaceFixedRangeWithReserved(uint64_t start, uint64_t size) {
 		const auto range_end = range.start + range.size;
 		const auto chunk     = std::min<uint64_t>(end, range_end) - current;
 
+		if (range.type != VirtualRangeType::Reserved) {
+			static std::atomic<uint32_t> replace_log_count {0};
+			if (replace_log_count.fetch_add(1, std::memory_order_relaxed) < 64) {
+				uint64_t before[4] = {0, 0, 0, 0};
+				std::memcpy(before, reinterpret_cast<const void*>(current), sizeof(before));
+				uint32_t zero_bytes = 0;
+				{
+					const auto* probe = reinterpret_cast<const uint8_t*>(current);
+					for (uint32_t i = 0; i < 4096u; i++) {
+						zero_bytes += (probe[i] == 0 ? 1u : 0u);
+					}
+				}
+			}
+		}
+
+		{
+		}
+
 		ReplacedChunk replaced {};
 		replaced.range       = range;
 		replaced.range.start = current;
@@ -3447,6 +3543,115 @@ bool TestGuestFreeRangeBounds() {
 }
 #endif
 
+bool KernelHandleReservedRangeAccessViolation(uint64_t vaddr) {
+	std::unique_lock<std::recursive_mutex> memory_operation_lock(g_memory_operation_mutex,
+	                                                             std::try_to_lock);
+	if (!memory_operation_lock.owns_lock()) {
+		return false;
+	}
+
+	VirtualRanges::Range range {};
+	if (!g_virtual_ranges->Query(vaddr, 0, &range) ||
+	    std::strncmp(range.name, "AMM", KERNEL_MAXIMUM_NAME_LENGTH) != 0) {
+		return false;
+	}
+	EXIT("AMM virtual-memory unmap is unsupported: addr=0x%016" PRIx64 "\n", vaddr);
+}
+
+bool KernelHandleTransientMappingFault(Graphics::PageFaultAccess access, uint64_t vaddr) {
+#if KYTY_PLATFORM == KYTY_PLATFORM_WINDOWS
+	if (g_virtual_ranges == nullptr) {
+		return false;
+	}
+
+	{
+		std::unique_lock<std::recursive_mutex> lock(g_memory_operation_mutex, std::try_to_lock);
+		if (lock.owns_lock()) {
+			VirtualRanges::Range range {};
+			if (!g_virtual_ranges->Query(vaddr, 0, &range)) {
+				return false;
+			}
+		}
+	}
+
+	MEMORY_BASIC_INFORMATION mbi {};
+	if (VirtualQuery(reinterpret_cast<const void*>(vaddr), &mbi, sizeof(mbi)) == 0) {
+		return false;
+	}
+	if (mbi.State != MEM_COMMIT || (mbi.Protect & PAGE_GUARD) != 0) {
+		return false;
+	}
+
+	constexpr DWORD READABLE = PAGE_READONLY | PAGE_READWRITE | PAGE_WRITECOPY | PAGE_EXECUTE_READ |
+	                           PAGE_EXECUTE_READWRITE | PAGE_EXECUTE_WRITECOPY;
+	constexpr DWORD WRITABLE =
+	    PAGE_READWRITE | PAGE_WRITECOPY | PAGE_EXECUTE_READWRITE | PAGE_EXECUTE_WRITECOPY;
+	constexpr DWORD EXECUTABLE =
+	    PAGE_EXECUTE | PAGE_EXECUTE_READ | PAGE_EXECUTE_READWRITE | PAGE_EXECUTE_WRITECOPY;
+
+	bool allowed = false;
+	switch (access) {
+		case Graphics::PageFaultAccess::Read: allowed = (mbi.Protect & READABLE) != 0; break;
+		case Graphics::PageFaultAccess::Write: allowed = (mbi.Protect & WRITABLE) != 0; break;
+		case Graphics::PageFaultAccess::Execute: allowed = (mbi.Protect & EXECUTABLE) != 0; break;
+		case Graphics::PageFaultAccess::Unknown: allowed = false; break;
+	}
+	if (!allowed) {
+		return false;
+	}
+
+	static std::atomic<uint32_t> log_count {0};
+	const auto                   seen = log_count.fetch_add(1, std::memory_order_relaxed);
+	return true;
+#else
+	(void)access;
+	(void)vaddr;
+	return false;
+#endif
+}
+
+uint64_t MappedExtentFrom(uint64_t vaddr, bool* known) {
+	if (known != nullptr) {
+		*known = false;
+	}
+	if (g_virtual_ranges == nullptr) {
+		return 0;
+	}
+	std::unique_lock<std::recursive_mutex> lock(g_memory_operation_mutex, std::defer_lock);
+	for (uint32_t attempt = 0; attempt < 64 && !lock.owns_lock(); attempt++) {
+		if (!lock.try_lock()) {
+			std::this_thread::yield();
+		}
+	}
+	if (!lock.owns_lock()) {
+		return 0;
+	}
+	if (known != nullptr) {
+		*known = true;
+	}
+	VirtualRanges::Range range {};
+	if (!g_virtual_ranges->Query(vaddr, 0, &range)) {
+		return 0;
+	}
+	const auto end = range.start + range.size;
+	return vaddr < end ? end - vaddr : 0;
+}
+
+void DumpVirtualRangeForAddress(uint64_t vaddr) {
+	if (g_virtual_ranges == nullptr) {
+		return;
+	}
+	std::unique_lock<std::recursive_mutex> lock(g_memory_operation_mutex, std::try_to_lock);
+	if (!lock.owns_lock()) {
+		return;
+	}
+
+	VirtualRanges::Range candidate {};
+	if (!g_virtual_ranges->Query(vaddr, 0, &candidate)) {
+		return;
+	}
+}
+
 int KYTY_SYSV_ABI KernelVirtualQuery(const void* addr, int flags, VirtualQueryInfo* info,
                                      uint64_t info_size) {
 	PRINT_NAME();
@@ -3465,7 +3670,20 @@ int KYTY_SYSV_ABI KernelVirtualQuery(const void* addr, int flags, VirtualQueryIn
 	}
 
 	VirtualRanges::Range candidate {};
-	if (!g_virtual_ranges->Query(vaddr, flags, &candidate)) {
+	const bool found = g_virtual_ranges->Query(vaddr, flags, &candidate);
+
+	{
+		static const auto watch = [] {
+			std::pair<uint64_t, uint64_t> range {0, 0};
+			return range;
+		}();
+		if (watch.second != 0 && vaddr >= watch.first - 0x100000 &&
+		    vaddr < watch.first + watch.second + 0x100000) {
+			static std::atomic<uint32_t> log_count {0};
+		}
+	}
+
+	if (!found) {
 		return KERNEL_ERROR_EACCES;
 	}
 

src/kernel/memory.h

@@ -117,6 +117,8 @@ void                   InvalidateMemory(uint64_t vaddr, uint64_t size);
 void                   InstallGpuResources(Graphics::GpuResourceManager* resources) noexcept;
 [[nodiscard]] bool HandleGpuFault(Graphics::PageFaultAccess access, uint64_t fault_vaddr) noexcept;
 
+[[nodiscard]] bool IsHandlingGpuFault() noexcept;
+
 int KYTY_SYSV_ABI KernelMapNamedFlexibleMemory(void** addr_in_out, size_t len, int prot, int flags,
                                                const char* name);
 int KYTY_SYSV_ABI KernelMapFlexibleMemory(void** addr_in_out, size_t len, int prot, int flags);
@@ -147,6 +149,13 @@ int KYTY_SYSV_ABI KernelGetPrtAperture(int index, void** addr, size_t* len);
 int KYTY_SYSV_ABI KernelIsAddressSanitizerEnabled();
 int KYTY_SYSV_ABI KernelQueryMemoryProtection(void* addr, void** start, void** end, int* prot);
 int KYTY_SYSV_ABI KernelDirectMemoryQuery(int64_t offset, int flags, void* info, size_t info_size);
+bool KernelHandleReservedRangeAccessViolation(uint64_t vaddr);
+bool KernelHandleTransientMappingFault(Graphics::PageFaultAccess access, uint64_t vaddr);
+
+
+void DumpVirtualRangeForAddress(uint64_t vaddr);
+uint64_t MappedExtentFrom(uint64_t vaddr, bool* known = nullptr);
+
 int KYTY_SYSV_ABI KernelVirtualQuery(const void* addr, int flags, VirtualQueryInfo* info,
                                      uint64_t info_size);
 int KYTY_SYSV_ABI KernelIsStack(void* addr, void** start, void** end);

src/kernel/memoryAddressSpace.inc

@@ -13,6 +13,10 @@ static constexpr uint64_t HOST_USER_MIN            = 0x1000000000ull;
 #endif
 static constexpr uint64_t HOST_USER_MAX = 0xfbffffffffull;
 
+static constexpr uint64_t HOST_GUEST_RESERVE_MAX = 0x6000000000ull;
+static_assert(HOST_GUEST_RESERVE_MAX > HOST_USER_MIN);
+static_assert(HOST_GUEST_RESERVE_MAX <= HOST_USER_MAX + 1u);
+
 static bool GuestFreeRangeContains(uint64_t base, uint64_t range_size, uint64_t candidate,
                                    uint64_t size) {
 	if (UINT64_MAX - base < range_size || candidate < base) {
@@ -758,14 +762,26 @@ public:
 		if (size == 0 || UINT64_MAX - vaddr < size) {
 			return false;
 		}
+		LogBackingChange("Protect", vaddr, size, static_cast<uint64_t>(mode));
 
 		std::lock_guard lock(m_mutex);
 		return ProtectMappedUnlocked(vaddr, size, mode);
 	}
 
+	inline void LogBackingChange(const char* what, uint64_t vaddr, uint64_t size, uint64_t offset) {
+		static const auto watch = [] {
+			std::pair<uint64_t, uint64_t> range {0, 0};
+			return range;
+		}();
+		if (watch.second == 0 || vaddr >= watch.first + watch.second || watch.first >= vaddr + size) {
+			return;
+		}
+	}
+
 	bool MapBacking(uint64_t vaddr, uint64_t size, uint64_t backing_offset,
 	                VirtualMemory::Mode               mode,
 	                GuestBackingStore::FailureReason* failure_reason = nullptr) {
+		LogBackingChange("MapBacking", vaddr, size, backing_offset);
 		std::lock_guard lock(m_mutex);
 		if (!ConsumeUnlocked(vaddr, size)) {
 			if (failure_reason != nullptr) {
@@ -786,6 +802,7 @@ public:
 	}
 
 	bool UnmapBacking(uint64_t vaddr, uint64_t size) {
+		LogBackingChange("UnmapBacking", vaddr, size, 0);
 		std::lock_guard lock(m_mutex);
 		if (!ContainsMappedUnlocked(vaddr, size, MappingType::Backing)) {
 			return false;
@@ -1089,7 +1106,7 @@ private:
 #if KYTY_PLATFORM == KYTY_PLATFORM_WINDOWS
 		constexpr uint64_t MinimumRegionSize = 0x1000000;
 		const auto         process           = GetCurrentProcess();
-		const auto         guest_end         = HOST_USER_MAX + 1u;
+		auto               guest_end         = HOST_GUEST_RESERVE_MAX;
 		auto               next_addr         = HOST_SYSTEM_MANAGED_MIN;
 
 		while (next_addr < guest_end) {

src/kernel/pthread.cpp

@@ -2855,11 +2855,11 @@ int KYTY_SYSV_ABI PthreadCondBroadcast(PthreadCond* cond) {
 	bool notify = false;
 	{
 		std::lock_guard lock((*cond)->m);
+		(*cond)->sequence++;
 		if (!(*cond)->waiters.empty()) {
-			(*cond)->sequence++;
 			CondClearWaiters(*cond);
-			notify = true;
 		}
+		notify = true;
 	}
 	if (notify) {
 		(*cond)->cv.notify_all();
@@ -3179,6 +3179,9 @@ int KYTY_SYSV_ABI PthreadCondTimedwaitAbs(PthreadCond* cond, PthreadMutex* mutex
 	}
 }
 
+thread_local uint64_t g_guest_wait_caller = 0;
+thread_local uint64_t g_guest_wait_r13    = 0;
+
 int KYTY_SYSV_ABI PthreadCondWait(PthreadCond* cond, PthreadMutex* mutex) {
 	PRINT_NAME();
 
@@ -3373,6 +3376,8 @@ static void* RunThread(void* arg) {
 	thread->unique_id = Common::Thread::GetThreadIdUnique();
 
 	g_pthread_self = thread;
+#if KYTY_PLATFORM == KYTY_PLATFORM_WINDOWS
+#endif
 
 	uint64_t os_thread_id = 0;
 #if KYTY_PLATFORM == KYTY_PLATFORM_WINDOWS
@@ -4445,6 +4450,10 @@ int KYTY_SYSV_ABI pthread_condattr_destroy(LibKernel::PthreadCondattr* attr) {
 int KYTY_SYSV_ABI pthread_cond_wait(LibKernel::PthreadCond* cond, LibKernel::PthreadMutex* mutex) {
 	// PRINT_NAME();
 
+	uint64_t guest_r13 = 0;
+	__asm__ volatile("movq %%r13, %0" : "=r"(guest_r13));
+	LibKernel::g_guest_wait_r13 = guest_r13;
+	LibKernel::g_guest_wait_caller = reinterpret_cast<uint64_t>(__builtin_return_address(0));
 	return POSIX_PTHREAD_CALL(LibKernel::PthreadCondWait(cond, mutex));
 }