diff --git a/src/kernel/eventFlag.cpp b/src/kernel/eventFlag.cpp index 594e7e148..16939edac 100644 --- a/src/kernel/eventFlag.cpp +++ b/src/kernel/eventFlag.cpp @@ -23,6 +23,7 @@ public: KernelEventFlagPrivate(const std::string& name, bool single, bool /*fifo*/, uint64_t bits) : m_name(name), m_single_thread(single), m_bits(bits) {} + virtual ~KernelEventFlagPrivate(); KYTY_CLASS_NO_COPY(KernelEventFlagPrivate); @@ -110,6 +111,24 @@ KernelEventFlagPrivate::Result KernelEventFlagPrivate::Wait(uint64_t bits, WaitM infinitely = false; } + if (!infinitely && micros == 0) { + const bool satisfied = (wait_mode == WaitMode::And && (m_bits & bits) == bits) || + (wait_mode == WaitMode::Or && (m_bits & bits) != 0); + if (result != nullptr) { + *result = m_bits; + } + *ptr_micros = 0; + if (!satisfied) { + return Result::TimedOut; + } + switch (clear_mode) { + case ClearMode::All: m_bits = 0; break; + case ClearMode::Bits: m_bits &= ~bits; break; + case ClearMode::None: break; + } + return Result::Ok; + } + uint32_t elapsed = 0; Common::Timer t; t.Start(); @@ -284,6 +303,17 @@ int KYTY_SYSV_ABI KernelDeleteEventFlag(KernelEventFlag ef) { return OK; } +static int EventFlagResultToErrno(KernelEventFlagPrivate::Result result) { + switch (result) { + case KernelEventFlagPrivate::Result::Ok: return OK; + case KernelEventFlagPrivate::Result::AlreadyWaiting: return KERNEL_ERROR_EPERM; + case KernelEventFlagPrivate::Result::TimedOut: return KERNEL_ERROR_ETIMEDOUT; + case KernelEventFlagPrivate::Result::Canceled: return KERNEL_ERROR_ECANCELED; + case KernelEventFlagPrivate::Result::Deleted: return KERNEL_ERROR_EACCES; + } + return OK; +} + int KYTY_SYSV_ABI KernelWaitEventFlag(KernelEventFlag ef, uint64_t bit_pattern, uint32_t wait_mode, uint64_t* result_pat, KernelUseconds* timeout) { PRINT_NAME(); @@ -296,8 +326,14 @@ int KYTY_SYSV_ABI KernelWaitEventFlag(KernelEventFlag ef, uint64_t bit_pattern, return KERNEL_ERROR_EINVAL; } - const auto mode = DecodeEventFlagWaitMode(wait_mode); - auto result = ef->Wait(bit_pattern, mode.wait, mode.clear, result_pat, timeout); + const auto mode = DecodeEventFlagWaitMode(wait_mode); + + const bool polling = timeout != nullptr && *timeout == 0; + if (polling) { + return EventFlagResultToErrno(ef->Wait(bit_pattern, mode.wait, mode.clear, result_pat, timeout)); + } + + auto result = ef->Wait(bit_pattern, mode.wait, mode.clear, result_pat, timeout); int ret = OK; diff --git a/src/kernel/fileSystem.cpp b/src/kernel/fileSystem.cpp index 7a5c63630..2d280f29c 100644 --- a/src/kernel/fileSystem.cpp +++ b/src/kernel/fileSystem.cpp @@ -1,6 +1,8 @@ #include "kernel/fileSystem.h" #include "common/assert.h" +#include +#include #include "common/common.h" #include "common/dateTime.h" #include "common/emulatorConfig.h" @@ -545,7 +547,18 @@ int KYTY_SYSV_ABI KernelClose(int d) { return OK; } +static void FileReadWatchCheck(const char* who, uint64_t dst, uint64_t size) { + static const auto watch = [] { + std::pair range {0, 0}; + return range; + }(); + if (watch.second == 0 || dst >= watch.first + watch.second || watch.first >= dst + size) { + return; + } +} + int64_t KYTY_SYSV_ABI KernelRead(int d, void* buf, size_t nbytes) { + FileReadWatchCheck("read", reinterpret_cast(buf), nbytes); PRINT_NAME(); if (d < DESCRIPTOR_MIN) { @@ -671,6 +684,7 @@ int64_t KYTY_SYSV_ABI KernelWrite(int d, const void* buf, size_t nbytes) { } int64_t KYTY_SYSV_ABI KernelPread(int d, void* buf, size_t nbytes, int64_t offset) { + FileReadWatchCheck("pread", reinterpret_cast(buf), nbytes); PRINT_NAME(); if (d < DESCRIPTOR_MIN) { diff --git a/src/kernel/memory.cpp b/src/kernel/memory.cpp index d73a043c7..26d84847f 100644 --- a/src/kernel/memory.cpp +++ b/src/kernel/memory.cpp @@ -1,7 +1,9 @@ +#include #include "kernel/memory.h" #include "common/assert.h" #include "common/logging/log.h" +#include #include "common/magicEnum.h" #include "common/stringUtils.h" #include "common/threads.h" @@ -896,6 +898,10 @@ uint64_t ClampRangeSize(uint64_t vaddr, uint64_t size) { } void WriteBacking(uint64_t vaddr, const void* data, uint64_t size) noexcept { + { + constexpr uint64_t AppMapAreaStart = 0x1000000000ull; + constexpr uint64_t AppMapAreaEnd = 0x1030000000ull; + } if (!TryWriteBacking(vaddr, data, size)) { EXIT("Memory: required direct-backing write failed, addr=0x%016" PRIx64 " size=0x%016" PRIx64 "\n", @@ -915,8 +921,20 @@ void InstallGpuResources(Graphics::GpuResourceManager* resources) noexcept { g_gpu_resources = resources; } +static thread_local bool g_in_gpu_fault = false; + +bool IsHandlingGpuFault() noexcept { + return g_in_gpu_fault; +} + bool HandleGpuFault(Graphics::PageFaultAccess access, uint64_t fault_vaddr) noexcept { - return g_gpu_resources != nullptr && g_gpu_resources->HandleFault(access, fault_vaddr); + if (g_gpu_resources == nullptr) { + return false; + } + const auto previous = std::exchange(g_in_gpu_fault, true); + const bool handled = g_gpu_resources->HandleFault(access, fault_vaddr); + g_in_gpu_fault = previous; + return handled; } struct PrtAperture { @@ -965,6 +983,53 @@ bool TryReadPrtBacking(uint64_t vaddr, void* data, uint64_t size) { return g_guest_address_space->TryReadSparseBacking(vaddr, data, size); } +static bool SelfTestDirectMemoryAliasAndPersistence() { +#if KYTY_PLATFORM == KYTY_PLATFORM_WINDOWS + constexpr uint64_t PageSize = 0x4000; + constexpr uint64_t Pattern = 0x4b595459414c4941ull; // "KYTYALIA" + const auto granularity = g_guest_address_space->GetGranularity(); + + const auto first = FindGuestFreeRange(0, granularity * 2u, granularity); + if (first == 0) { + return false; + } + const auto second = FindGuestFreeRange(first + granularity * 2u, granularity, granularity); + if (second == 0) { + return false; + } + + const uint64_t backing_offset = granularity; + + bool alias_ok = false; + bool persist_ok = false; + auto reason = GuestBackingStore::FailureReason::None; + + if (g_guest_address_space->MapBacking(first, PageSize, backing_offset, + VirtualMemory::Mode::ReadWrite, &reason)) { + *reinterpret_cast(first) = Pattern; + + if (g_guest_address_space->MapBacking(second, PageSize, backing_offset, + VirtualMemory::Mode::ReadWrite, &reason)) { + alias_ok = (*reinterpret_cast(second) == Pattern); + EXIT_IF(!g_guest_address_space->UnmapBacking(second, PageSize)); + } + + if (g_guest_address_space->UnmapBacking(first, PageSize) && + g_guest_address_space->MapBacking(first, PageSize, backing_offset, + VirtualMemory::Mode::ReadWrite, &reason)) { + persist_ok = (*reinterpret_cast(first) == Pattern); + std::memset(reinterpret_cast(first), 0, PageSize); + EXIT_IF(!g_guest_address_space->UnmapBacking(first, PageSize)); + } + } + + const bool ok = alias_ok && persist_ok; + return ok; +#else + return true; +#endif +} + static bool SelfTestSub64SharedPlaceholderAlias() { #if KYTY_PLATFORM == KYTY_PLATFORM_WINDOWS constexpr uint64_t PageSize = 0x4000; @@ -1018,6 +1083,7 @@ void Initialize() { g_virtual_ranges = std::make_unique(); EXIT_IF(!g_guest_address_space->SelfTest()); EXIT_IF(!SelfTestSub64SharedPlaceholderAlias()); + EXIT_IF(!SelfTestDirectMemoryAliasAndPersistence()); } void Shutdown() { @@ -3044,6 +3110,20 @@ int KYTY_SYSV_ABI KernelMapDirectMemory(void** addr, size_t len, int prot, int f return KERNEL_ERROR_EBUSY; } + { + static std::atomic after_log_count {0}; + if (after_log_count.fetch_add(1, std::memory_order_relaxed) < 64) { + uint64_t after[4] = {0, 0, 0, 0}; + std::memcpy(after, reinterpret_cast(out_addr), sizeof(after)); + uint64_t zero_bytes = 0; + uint64_t total_bytes = std::min(len, 1u << 20u); + const auto* probe = reinterpret_cast(out_addr); + for (uint64_t i = 0; i < total_bytes; i++) { + zero_bytes += (probe[i] == 0 ? 1u : 0u); + } + } + } + PhysicalMemory::AllocatedBlock mapped_block {}; g_physical_memory->Find(direct_memory_start, false, &mapped_block); if (!g_virtual_ranges->Add(out_addr, len, direct_memory_start, prot, mapped_block.memory_type, @@ -3075,8 +3155,6 @@ int KYTY_SYSV_ABI KernelMapDirectMemory2(void** addr, size_t len, int type, int std::lock_guard memory_operation_lock(g_memory_operation_mutex); - LOGF("\t type = %d\n", type); - auto ret = KernelMapDirectMemory(addr, len, prot, flags, direct_memory_start, alignment); if (ret == OK && addr != nullptr && *addr != nullptr) { const auto out_addr = reinterpret_cast(*addr); @@ -3175,6 +3253,24 @@ static bool ReplaceFixedRangeWithReserved(uint64_t start, uint64_t size) { const auto range_end = range.start + range.size; const auto chunk = std::min(end, range_end) - current; + if (range.type != VirtualRangeType::Reserved) { + static std::atomic replace_log_count {0}; + if (replace_log_count.fetch_add(1, std::memory_order_relaxed) < 64) { + uint64_t before[4] = {0, 0, 0, 0}; + std::memcpy(before, reinterpret_cast(current), sizeof(before)); + uint32_t zero_bytes = 0; + { + const auto* probe = reinterpret_cast(current); + for (uint32_t i = 0; i < 4096u; i++) { + zero_bytes += (probe[i] == 0 ? 1u : 0u); + } + } + } + } + + { + } + ReplacedChunk replaced {}; replaced.range = range; replaced.range.start = current; @@ -3447,6 +3543,115 @@ bool TestGuestFreeRangeBounds() { } #endif +bool KernelHandleReservedRangeAccessViolation(uint64_t vaddr) { + std::unique_lock memory_operation_lock(g_memory_operation_mutex, + std::try_to_lock); + if (!memory_operation_lock.owns_lock()) { + return false; + } + + VirtualRanges::Range range {}; + if (!g_virtual_ranges->Query(vaddr, 0, &range) || + std::strncmp(range.name, "AMM", KERNEL_MAXIMUM_NAME_LENGTH) != 0) { + return false; + } + EXIT("AMM virtual-memory unmap is unsupported: addr=0x%016" PRIx64 "\n", vaddr); +} + +bool KernelHandleTransientMappingFault(Graphics::PageFaultAccess access, uint64_t vaddr) { +#if KYTY_PLATFORM == KYTY_PLATFORM_WINDOWS + if (g_virtual_ranges == nullptr) { + return false; + } + + { + std::unique_lock lock(g_memory_operation_mutex, std::try_to_lock); + if (lock.owns_lock()) { + VirtualRanges::Range range {}; + if (!g_virtual_ranges->Query(vaddr, 0, &range)) { + return false; + } + } + } + + MEMORY_BASIC_INFORMATION mbi {}; + if (VirtualQuery(reinterpret_cast(vaddr), &mbi, sizeof(mbi)) == 0) { + return false; + } + if (mbi.State != MEM_COMMIT || (mbi.Protect & PAGE_GUARD) != 0) { + return false; + } + + constexpr DWORD READABLE = PAGE_READONLY | PAGE_READWRITE | PAGE_WRITECOPY | PAGE_EXECUTE_READ | + PAGE_EXECUTE_READWRITE | PAGE_EXECUTE_WRITECOPY; + constexpr DWORD WRITABLE = + PAGE_READWRITE | PAGE_WRITECOPY | PAGE_EXECUTE_READWRITE | PAGE_EXECUTE_WRITECOPY; + constexpr DWORD EXECUTABLE = + PAGE_EXECUTE | PAGE_EXECUTE_READ | PAGE_EXECUTE_READWRITE | PAGE_EXECUTE_WRITECOPY; + + bool allowed = false; + switch (access) { + case Graphics::PageFaultAccess::Read: allowed = (mbi.Protect & READABLE) != 0; break; + case Graphics::PageFaultAccess::Write: allowed = (mbi.Protect & WRITABLE) != 0; break; + case Graphics::PageFaultAccess::Execute: allowed = (mbi.Protect & EXECUTABLE) != 0; break; + case Graphics::PageFaultAccess::Unknown: allowed = false; break; + } + if (!allowed) { + return false; + } + + static std::atomic log_count {0}; + const auto seen = log_count.fetch_add(1, std::memory_order_relaxed); + return true; +#else + (void)access; + (void)vaddr; + return false; +#endif +} + +uint64_t MappedExtentFrom(uint64_t vaddr, bool* known) { + if (known != nullptr) { + *known = false; + } + if (g_virtual_ranges == nullptr) { + return 0; + } + std::unique_lock lock(g_memory_operation_mutex, std::defer_lock); + for (uint32_t attempt = 0; attempt < 64 && !lock.owns_lock(); attempt++) { + if (!lock.try_lock()) { + std::this_thread::yield(); + } + } + if (!lock.owns_lock()) { + return 0; + } + if (known != nullptr) { + *known = true; + } + VirtualRanges::Range range {}; + if (!g_virtual_ranges->Query(vaddr, 0, &range)) { + return 0; + } + const auto end = range.start + range.size; + return vaddr < end ? end - vaddr : 0; +} + +void DumpVirtualRangeForAddress(uint64_t vaddr) { + if (g_virtual_ranges == nullptr) { + return; + } + std::unique_lock lock(g_memory_operation_mutex, std::try_to_lock); + if (!lock.owns_lock()) { + return; + } + + VirtualRanges::Range candidate {}; + if (!g_virtual_ranges->Query(vaddr, 0, &candidate)) { + return; + } +} + int KYTY_SYSV_ABI KernelVirtualQuery(const void* addr, int flags, VirtualQueryInfo* info, uint64_t info_size) { PRINT_NAME(); @@ -3465,7 +3670,20 @@ int KYTY_SYSV_ABI KernelVirtualQuery(const void* addr, int flags, VirtualQueryIn } VirtualRanges::Range candidate {}; - if (!g_virtual_ranges->Query(vaddr, flags, &candidate)) { + const bool found = g_virtual_ranges->Query(vaddr, flags, &candidate); + + { + static const auto watch = [] { + std::pair range {0, 0}; + return range; + }(); + if (watch.second != 0 && vaddr >= watch.first - 0x100000 && + vaddr < watch.first + watch.second + 0x100000) { + static std::atomic log_count {0}; + } + } + + if (!found) { return KERNEL_ERROR_EACCES; } diff --git a/src/kernel/memory.h b/src/kernel/memory.h index d667e008e..69f5b21ae 100644 --- a/src/kernel/memory.h +++ b/src/kernel/memory.h @@ -117,6 +117,8 @@ void InvalidateMemory(uint64_t vaddr, uint64_t size); void InstallGpuResources(Graphics::GpuResourceManager* resources) noexcept; [[nodiscard]] bool HandleGpuFault(Graphics::PageFaultAccess access, uint64_t fault_vaddr) noexcept; +[[nodiscard]] bool IsHandlingGpuFault() noexcept; + int KYTY_SYSV_ABI KernelMapNamedFlexibleMemory(void** addr_in_out, size_t len, int prot, int flags, const char* name); int KYTY_SYSV_ABI KernelMapFlexibleMemory(void** addr_in_out, size_t len, int prot, int flags); @@ -147,6 +149,13 @@ int KYTY_SYSV_ABI KernelGetPrtAperture(int index, void** addr, size_t* len); int KYTY_SYSV_ABI KernelIsAddressSanitizerEnabled(); int KYTY_SYSV_ABI KernelQueryMemoryProtection(void* addr, void** start, void** end, int* prot); int KYTY_SYSV_ABI KernelDirectMemoryQuery(int64_t offset, int flags, void* info, size_t info_size); +bool KernelHandleReservedRangeAccessViolation(uint64_t vaddr); +bool KernelHandleTransientMappingFault(Graphics::PageFaultAccess access, uint64_t vaddr); + + +void DumpVirtualRangeForAddress(uint64_t vaddr); +uint64_t MappedExtentFrom(uint64_t vaddr, bool* known = nullptr); + int KYTY_SYSV_ABI KernelVirtualQuery(const void* addr, int flags, VirtualQueryInfo* info, uint64_t info_size); int KYTY_SYSV_ABI KernelIsStack(void* addr, void** start, void** end); diff --git a/src/kernel/memoryAddressSpace.inc b/src/kernel/memoryAddressSpace.inc index 17510688e..b3726271b 100644 --- a/src/kernel/memoryAddressSpace.inc +++ b/src/kernel/memoryAddressSpace.inc @@ -13,6 +13,10 @@ static constexpr uint64_t HOST_USER_MIN = 0x1000000000ull; #endif static constexpr uint64_t HOST_USER_MAX = 0xfbffffffffull; +static constexpr uint64_t HOST_GUEST_RESERVE_MAX = 0x6000000000ull; +static_assert(HOST_GUEST_RESERVE_MAX > HOST_USER_MIN); +static_assert(HOST_GUEST_RESERVE_MAX <= HOST_USER_MAX + 1u); + static bool GuestFreeRangeContains(uint64_t base, uint64_t range_size, uint64_t candidate, uint64_t size) { if (UINT64_MAX - base < range_size || candidate < base) { @@ -758,14 +762,26 @@ public: if (size == 0 || UINT64_MAX - vaddr < size) { return false; } + LogBackingChange("Protect", vaddr, size, static_cast(mode)); std::lock_guard lock(m_mutex); return ProtectMappedUnlocked(vaddr, size, mode); } + inline void LogBackingChange(const char* what, uint64_t vaddr, uint64_t size, uint64_t offset) { + static const auto watch = [] { + std::pair range {0, 0}; + return range; + }(); + if (watch.second == 0 || vaddr >= watch.first + watch.second || watch.first >= vaddr + size) { + return; + } + } + bool MapBacking(uint64_t vaddr, uint64_t size, uint64_t backing_offset, VirtualMemory::Mode mode, GuestBackingStore::FailureReason* failure_reason = nullptr) { + LogBackingChange("MapBacking", vaddr, size, backing_offset); std::lock_guard lock(m_mutex); if (!ConsumeUnlocked(vaddr, size)) { if (failure_reason != nullptr) { @@ -786,6 +802,7 @@ public: } bool UnmapBacking(uint64_t vaddr, uint64_t size) { + LogBackingChange("UnmapBacking", vaddr, size, 0); std::lock_guard lock(m_mutex); if (!ContainsMappedUnlocked(vaddr, size, MappingType::Backing)) { return false; @@ -1089,7 +1106,7 @@ private: #if KYTY_PLATFORM == KYTY_PLATFORM_WINDOWS constexpr uint64_t MinimumRegionSize = 0x1000000; const auto process = GetCurrentProcess(); - const auto guest_end = HOST_USER_MAX + 1u; + auto guest_end = HOST_GUEST_RESERVE_MAX; auto next_addr = HOST_SYSTEM_MANAGED_MIN; while (next_addr < guest_end) { diff --git a/src/kernel/pthread.cpp b/src/kernel/pthread.cpp index cbd668799..60654f58c 100644 --- a/src/kernel/pthread.cpp +++ b/src/kernel/pthread.cpp @@ -2855,11 +2855,11 @@ int KYTY_SYSV_ABI PthreadCondBroadcast(PthreadCond* cond) { bool notify = false; { std::lock_guard lock((*cond)->m); + (*cond)->sequence++; if (!(*cond)->waiters.empty()) { - (*cond)->sequence++; CondClearWaiters(*cond); - notify = true; } + notify = true; } if (notify) { (*cond)->cv.notify_all(); @@ -3179,6 +3179,9 @@ int KYTY_SYSV_ABI PthreadCondTimedwaitAbs(PthreadCond* cond, PthreadMutex* mutex } } +thread_local uint64_t g_guest_wait_caller = 0; +thread_local uint64_t g_guest_wait_r13 = 0; + int KYTY_SYSV_ABI PthreadCondWait(PthreadCond* cond, PthreadMutex* mutex) { PRINT_NAME(); @@ -3373,6 +3376,8 @@ static void* RunThread(void* arg) { thread->unique_id = Common::Thread::GetThreadIdUnique(); g_pthread_self = thread; +#if KYTY_PLATFORM == KYTY_PLATFORM_WINDOWS +#endif uint64_t os_thread_id = 0; #if KYTY_PLATFORM == KYTY_PLATFORM_WINDOWS @@ -4445,6 +4450,10 @@ int KYTY_SYSV_ABI pthread_condattr_destroy(LibKernel::PthreadCondattr* attr) { int KYTY_SYSV_ABI pthread_cond_wait(LibKernel::PthreadCond* cond, LibKernel::PthreadMutex* mutex) { // PRINT_NAME(); + uint64_t guest_r13 = 0; + __asm__ volatile("movq %%r13, %0" : "=r"(guest_r13)); + LibKernel::g_guest_wait_r13 = guest_r13; + LibKernel::g_guest_wait_caller = reinterpret_cast(__builtin_return_address(0)); return POSIX_PTHREAD_CALL(LibKernel::PthreadCondWait(cond, mutex)); }