mirror of
https://github.com/KytyPS5/KytyPS5.git
synced 2026-10-23 20:36:58 +02:00
Simplify fiber ownership and preserve small guest stacks
This commit is contained in:
nmzik
+29
-146
@@ -29,11 +29,9 @@
|
||||
#include <cstdio>
|
||||
#include <cstdlib>
|
||||
#include <cstring>
|
||||
#include <mutex>
|
||||
#include <random>
|
||||
#include <string>
|
||||
#include <thread>
|
||||
#include <unordered_map>
|
||||
#include <vector>
|
||||
|
||||
#if KYTY_PLATFORM == KYTY_PLATFORM_WINDOWS
|
||||
@@ -2356,70 +2354,24 @@ static bool FiberIsValid(const FiberObject* fiber) {
|
||||
fiber->magic_end == FIBER_MAGIC_END;
|
||||
}
|
||||
|
||||
static thread_local FiberObject* g_current_fiber = nullptr;
|
||||
static thread_local FiberObject* g_thread_return_fiber = nullptr;
|
||||
static thread_local FiberObject* g_starting_fiber = nullptr;
|
||||
static thread_local FiberObject* g_pending_idle_fiber = nullptr;
|
||||
static thread_local FiberCpuContext g_thread_fiber_context {};
|
||||
static std::mutex g_fiber_owner_mutex;
|
||||
static std::unordered_map<FiberObject*, uint64_t> g_fiber_owner_thread;
|
||||
static std::unordered_map<uint64_t, FiberObject*> g_fiber_current_by_thread;
|
||||
static thread_local FiberObject* g_current_fiber = nullptr;
|
||||
static thread_local FiberObject* g_thread_return_fiber = nullptr;
|
||||
static thread_local FiberObject* g_pending_idle_fiber = nullptr;
|
||||
static thread_local FiberCpuContext g_thread_fiber_context {};
|
||||
|
||||
static uint32_t FiberLoadState(const FiberObject* fiber) {
|
||||
auto& state = const_cast<uint32_t&>(fiber->state);
|
||||
return std::atomic_ref<uint32_t>(state).load(std::memory_order_acquire);
|
||||
}
|
||||
|
||||
static uint64_t FiberCurrentHostThreadId() {
|
||||
#if KYTY_PLATFORM == KYTY_PLATFORM_WINDOWS
|
||||
return static_cast<uint64_t>(::GetCurrentThreadId());
|
||||
#else
|
||||
return std::hash<std::thread::id> {}(std::this_thread::get_id());
|
||||
#endif
|
||||
}
|
||||
|
||||
static void FiberSetOwner(FiberObject* fiber) {
|
||||
std::lock_guard lock(g_fiber_owner_mutex);
|
||||
g_fiber_owner_thread[fiber] = FiberCurrentHostThreadId();
|
||||
}
|
||||
|
||||
static void FiberClearOwner(FiberObject* fiber) {
|
||||
std::lock_guard lock(g_fiber_owner_mutex);
|
||||
g_fiber_owner_thread.erase(fiber);
|
||||
}
|
||||
|
||||
static uint64_t FiberGetOwner(FiberObject* fiber) {
|
||||
std::lock_guard lock(g_fiber_owner_mutex);
|
||||
auto it = g_fiber_owner_thread.find(fiber);
|
||||
return it != g_fiber_owner_thread.end() ? it->second : 0;
|
||||
}
|
||||
|
||||
static void FiberSetCurrentFiber(FiberObject* fiber) {
|
||||
g_current_fiber = fiber;
|
||||
|
||||
std::lock_guard lock(g_fiber_owner_mutex);
|
||||
const auto thread_id = FiberCurrentHostThreadId();
|
||||
if (fiber != nullptr) {
|
||||
g_fiber_current_by_thread[thread_id] = fiber;
|
||||
} else {
|
||||
g_fiber_current_by_thread.erase(thread_id);
|
||||
}
|
||||
}
|
||||
|
||||
static void FiberStoreState(FiberObject* fiber, uint32_t state) {
|
||||
std::atomic_ref<uint32_t>(fiber->state).store(state, std::memory_order_release);
|
||||
if (state == FIBER_STATE_RUNNING) {
|
||||
FiberSetOwner(fiber);
|
||||
} else {
|
||||
FiberClearOwner(fiber);
|
||||
}
|
||||
}
|
||||
|
||||
static void FiberDeferIdle(FiberObject* fiber) {
|
||||
g_pending_idle_fiber = fiber;
|
||||
}
|
||||
|
||||
static void FiberCommitDeferredIdle() {
|
||||
// A fiber may resume on another host thread. Do not cache TLS across the switch.
|
||||
[[gnu::noinline]] static void FiberCompleteSwitch(FiberObject* current) {
|
||||
g_current_fiber = current;
|
||||
// Only publish IDLE after switching away from the departing fiber's stack.
|
||||
auto* fiber = g_pending_idle_fiber;
|
||||
g_pending_idle_fiber = nullptr;
|
||||
if (fiber != nullptr) {
|
||||
@@ -2428,60 +2380,9 @@ static void FiberCommitDeferredIdle() {
|
||||
}
|
||||
|
||||
static bool FiberCompareExchangeState(FiberObject* fiber, uint32_t expected, uint32_t desired) {
|
||||
const bool ok = std::atomic_ref<uint32_t>(fiber->state)
|
||||
.compare_exchange_strong(expected, desired, std::memory_order_acq_rel,
|
||||
std::memory_order_acquire);
|
||||
if (ok) {
|
||||
if (desired == FIBER_STATE_RUNNING) {
|
||||
FiberSetOwner(fiber);
|
||||
} else {
|
||||
FiberClearOwner(fiber);
|
||||
}
|
||||
}
|
||||
return ok;
|
||||
}
|
||||
|
||||
static bool FiberWaitAndEnterRunning(FiberObject* fiber, uint32_t* observed_state) {
|
||||
const auto start = std::chrono::steady_clock::now();
|
||||
uint32_t spin = 0;
|
||||
auto& state = fiber->state;
|
||||
auto ref = std::atomic_ref<uint32_t>(state);
|
||||
|
||||
for (;;) {
|
||||
uint32_t expected = FIBER_STATE_IDLE;
|
||||
if (ref.compare_exchange_strong(expected, FIBER_STATE_RUNNING, std::memory_order_acq_rel,
|
||||
std::memory_order_acquire)) {
|
||||
FiberSetOwner(fiber);
|
||||
return true;
|
||||
}
|
||||
if (observed_state != nullptr) {
|
||||
*observed_state = expected;
|
||||
}
|
||||
if (expected != FIBER_STATE_RUNNING) {
|
||||
return false;
|
||||
}
|
||||
if (std::chrono::steady_clock::now() - start >= std::chrono::milliseconds(5)) {
|
||||
return false;
|
||||
}
|
||||
if (spin++ < 64) {
|
||||
std::this_thread::yield();
|
||||
} else {
|
||||
std::this_thread::sleep_for(std::chrono::microseconds(100));
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
static bool FiberRepairStaleRunningOnThisThread(FiberObject* fiber, uint32_t observed_state) {
|
||||
if (observed_state != FIBER_STATE_RUNNING || fiber == g_current_fiber) {
|
||||
return false;
|
||||
}
|
||||
|
||||
const auto owner = FiberGetOwner(fiber);
|
||||
if (owner != 0) {
|
||||
return false;
|
||||
}
|
||||
|
||||
return FiberCompareExchangeState(fiber, FIBER_STATE_RUNNING, FIBER_STATE_IDLE);
|
||||
return std::atomic_ref<uint32_t>(fiber->state)
|
||||
.compare_exchange_strong(expected, desired, std::memory_order_acq_rel,
|
||||
std::memory_order_acquire);
|
||||
}
|
||||
|
||||
static void FiberSetContextValid(FiberObject* fiber, bool valid) {
|
||||
@@ -2538,35 +2439,25 @@ static void FiberRestoreContext(FiberCpuContext* ctx, uint64_t ret) {
|
||||
}
|
||||
#endif
|
||||
|
||||
[[noreturn]] static void FiberStartTrampoline();
|
||||
[[noreturn]] static KYTY_SYSV_ABI void FiberStartTrampoline(FiberObject* fiber);
|
||||
|
||||
[[noreturn]] static void FiberStartOnGuestStack(FiberObject* fiber) {
|
||||
[[noreturn]] static KYTY_SYSV_ABI void FiberStartOnGuestStack(FiberObject* fiber) {
|
||||
FiberCpuContext ctx {};
|
||||
const auto stack_top = reinterpret_cast<uintptr_t>(fiber->addr_context) +
|
||||
static_cast<uintptr_t>(fiber->size_context);
|
||||
auto rsp = (stack_top & ~static_cast<uintptr_t>(0x0f));
|
||||
#if KYTY_PLATFORM == KYTY_PLATFORM_WINDOWS
|
||||
rsp -= 4u * sizeof(uint64_t);
|
||||
#endif
|
||||
rsp -= sizeof(uint64_t);
|
||||
*reinterpret_cast<uint64_t*>(rsp) = 0;
|
||||
|
||||
ctx.rsp = rsp;
|
||||
ctx.rip = reinterpret_cast<uint64_t>(&FiberStartTrampoline);
|
||||
ctx.rdi = reinterpret_cast<uint64_t>(fiber);
|
||||
|
||||
g_starting_fiber = fiber;
|
||||
FiberRestoreContext(&ctx, 1);
|
||||
}
|
||||
|
||||
[[noreturn]] static void FiberStartTrampoline() {
|
||||
auto* fiber = g_starting_fiber;
|
||||
g_starting_fiber = nullptr;
|
||||
if (fiber == nullptr) {
|
||||
EXIT("Fiber start without current fiber\n");
|
||||
}
|
||||
|
||||
FiberCommitDeferredIdle();
|
||||
FiberSetCurrentFiber(fiber);
|
||||
[[noreturn]] static KYTY_SYSV_ABI void FiberStartTrampoline(FiberObject* fiber) {
|
||||
FiberCompleteSwitch(fiber);
|
||||
|
||||
fiber->entry(fiber->arg_on_initialize, fiber->arg_on_run);
|
||||
|
||||
@@ -2574,7 +2465,7 @@ static void FiberRestoreContext(FiberCpuContext* ctx, uint64_t ret) {
|
||||
FiberSetContextValid(fiber, false);
|
||||
fiber->arg_on_return = 0;
|
||||
g_thread_return_fiber = fiber;
|
||||
FiberSetCurrentFiber(nullptr);
|
||||
g_current_fiber = nullptr;
|
||||
|
||||
FiberRestoreContext(&g_thread_fiber_context, 1);
|
||||
}
|
||||
@@ -2704,16 +2595,19 @@ int32_t KYTY_SYSV_ABI FiberRun(FiberObject* fiber, uint64_t arg_on_run, uint64_t
|
||||
FiberStartOnGuestStack(fiber);
|
||||
}
|
||||
|
||||
FiberCommitDeferredIdle();
|
||||
FiberSetCurrentFiber(nullptr);
|
||||
auto* returned_fiber = (g_thread_return_fiber != nullptr ? g_thread_return_fiber : fiber);
|
||||
const auto return_value = returned_fiber->arg_on_return;
|
||||
const auto return_code =
|
||||
FiberLoadState(returned_fiber) == FIBER_STATE_TERMINATED ? FIBER_ERROR_STATE : OK;
|
||||
// Copy results before another thread can acquire and resume the fiber.
|
||||
g_thread_return_fiber = nullptr;
|
||||
FiberCompleteSwitch(nullptr);
|
||||
|
||||
if (arg_on_return != nullptr) {
|
||||
*arg_on_return = returned_fiber->arg_on_return;
|
||||
*arg_on_return = return_value;
|
||||
}
|
||||
|
||||
return (FiberLoadState(returned_fiber) == FIBER_STATE_TERMINATED ? FIBER_ERROR_STATE : OK);
|
||||
return return_code;
|
||||
}
|
||||
|
||||
int32_t KYTY_SYSV_ABI FiberSwitch(FiberObject* fiber, uint64_t arg_on_run,
|
||||
@@ -2727,14 +2621,7 @@ int32_t KYTY_SYSV_ABI FiberSwitch(FiberObject* fiber, uint64_t arg_on_run,
|
||||
return FIBER_ERROR_PERMISSION;
|
||||
}
|
||||
|
||||
for (;;) {
|
||||
uint32_t observed_state = 0;
|
||||
if (FiberWaitAndEnterRunning(fiber, &observed_state)) {
|
||||
break;
|
||||
}
|
||||
if (FiberRepairStaleRunningOnThisThread(fiber, observed_state)) {
|
||||
continue;
|
||||
}
|
||||
if (!FiberCompareExchangeState(fiber, FIBER_STATE_IDLE, FIBER_STATE_RUNNING)) {
|
||||
return FIBER_ERROR_STATE;
|
||||
}
|
||||
|
||||
@@ -2745,16 +2632,14 @@ int32_t KYTY_SYSV_ABI FiberSwitch(FiberObject* fiber, uint64_t arg_on_run,
|
||||
|
||||
if (FiberSaveContext(&caller->saved_context) == 0) {
|
||||
FiberSetContextValid(caller, true);
|
||||
FiberDeferIdle(caller);
|
||||
g_pending_idle_fiber = caller;
|
||||
if (fiber->context_valid) {
|
||||
FiberRestoreContext(&fiber->saved_context, 1);
|
||||
}
|
||||
FiberStartOnGuestStack(fiber);
|
||||
}
|
||||
|
||||
FiberCommitDeferredIdle();
|
||||
FiberSetCurrentFiber(caller);
|
||||
FiberStoreState(caller, FIBER_STATE_RUNNING);
|
||||
FiberCompleteSwitch(caller);
|
||||
|
||||
if (arg_on_return != nullptr) {
|
||||
*arg_on_return = caller->arg_on_run;
|
||||
@@ -2788,13 +2673,11 @@ int32_t KYTY_SYSV_ABI FiberReturnToThread(uint64_t arg_on_return, uint64_t* arg_
|
||||
if (FiberSaveContext(&fiber->saved_context) == 0) {
|
||||
FiberSetContextValid(fiber, true);
|
||||
g_thread_return_fiber = fiber;
|
||||
FiberDeferIdle(fiber);
|
||||
g_pending_idle_fiber = fiber;
|
||||
FiberRestoreContext(&g_thread_fiber_context, 1);
|
||||
}
|
||||
|
||||
FiberCommitDeferredIdle();
|
||||
FiberSetCurrentFiber(fiber);
|
||||
FiberStoreState(fiber, FIBER_STATE_RUNNING);
|
||||
FiberCompleteSwitch(fiber);
|
||||
if (arg_on_run != nullptr) {
|
||||
*arg_on_run = fiber->arg_on_run;
|
||||
}
|
||||
|
||||
Reference in New Issue
Block a user