Implement syncOnAddress ABIs (perf improvement)

Co-authored-by: DeathWhispers <56944463+Death-Whispers-U@users.noreply.github.com>

.github/workflows/build.yml

@@ -94,6 +94,7 @@ jobs:
           cmake -S . -B _Build/windows ^
             -G Ninja ^
             -DCMAKE_BUILD_TYPE=Release ^
+            -DCMAKE_INTERPROCEDURAL_OPTIMIZATION=ON ^
             -DKYTY_BUILD_ORIGIN=%KYTY_BUILD_ORIGIN% ^
             -DKYTY_BUILD_REPOSITORY=%KYTY_BUILD_REPOSITORY% ^
             -DKYTY_RELEASE_TAG=${{ needs.metadata.outputs.release_tag }} ^
@@ -172,6 +173,7 @@ jobs:
           cmake -S . -B _Build/macos \
             -G Ninja \
             -DCMAKE_BUILD_TYPE=Release \
+            -DCMAKE_INTERPROCEDURAL_OPTIMIZATION=ON \
             -DKYTY_BUILD_ORIGIN="$KYTY_BUILD_ORIGIN" \
             -DKYTY_BUILD_REPOSITORY="$KYTY_BUILD_REPOSITORY" \
             -DKYTY_RELEASE_TAG="${{ needs.metadata.outputs.release_tag }}" \
@@ -299,6 +301,7 @@ jobs:
           cmake -S . -B _Build/linux \
             -G Ninja \
             -DCMAKE_BUILD_TYPE=Release \
+            -DCMAKE_INTERPROCEDURAL_OPTIMIZATION=ON \
             -DKYTY_BUILD_ORIGIN="$KYTY_BUILD_ORIGIN" \
             -DKYTY_BUILD_REPOSITORY="$KYTY_BUILD_REPOSITORY" \
             -DKYTY_RELEASE_TAG="${{ needs.metadata.outputs.release_tag }}" \

CMakeLists.txt

@@ -4,6 +4,12 @@ if(CMAKE_VERSION VERSION_GREATER_EQUAL 4.0)
 	set(CMAKE_POLICY_VERSION_MINIMUM 3.5 CACHE STRING "Minimum policy version for bundled dependencies")
 endif()
 
+get_property(KYTY_GENERATOR_IS_MULTI_CONFIG GLOBAL PROPERTY GENERATOR_IS_MULTI_CONFIG)
+if(NOT KYTY_GENERATOR_IS_MULTI_CONFIG AND NOT CMAKE_BUILD_TYPE)
+	set(CMAKE_BUILD_TYPE Release CACHE STRING "Choose the type of build" FORCE)
+	set_property(CACHE CMAKE_BUILD_TYPE PROPERTY STRINGS Debug Release RelWithDebInfo MinSizeRel)
+endif()
+
 project(Kyty)
 
 set(KYTY_SOURCE_DIR "${CMAKE_CURRENT_SOURCE_DIR}/src")
@@ -444,6 +450,13 @@ add_executable(event_queue_lifetime_tests EXCLUDE_FROM_ALL
 target_link_libraries(event_queue_lifetime_tests common fmt::fmt)
 target_include_directories(event_queue_lifetime_tests PRIVATE ${inc_headers})
 
+add_executable(sync_on_address_tests EXCLUDE_FROM_ALL
+	"${KYTY_TESTS_DIR}/SyncOnAddressTests.cpp"
+	"${KYTY_SOURCE_DIR}/kernel/syncOnAddress.cpp"
+)
+target_link_libraries(sync_on_address_tests common)
+target_include_directories(sync_on_address_tests PRIVATE ${inc_headers})
+
 add_executable(image_page_table_tests EXCLUDE_FROM_ALL
 	"${KYTY_TESTS_DIR}/ImagePageTableTests.cpp"
 )
@@ -497,6 +510,7 @@ if(BUILD_TESTING)
 	add_test(NAME resource_tracking COMMAND $<TARGET_FILE:resource_tracking_tests>)
 	add_test(NAME resource_mutex COMMAND $<TARGET_FILE:resource_mutex_tests>)
 	add_test(NAME event_queue_lifetime COMMAND $<TARGET_FILE:event_queue_lifetime_tests>)
+	add_test(NAME sync_on_address COMMAND $<TARGET_FILE:sync_on_address_tests>)
 	add_test(NAME audio_out2_port COMMAND $<TARGET_FILE:audio_out2_port_tests>)
 	add_test(NAME shader_recompiler_compute COMMAND $<TARGET_FILE:shader_recompiler_compute_tests>)
 	add_test(NAME virtual_memory_allocation
@@ -507,6 +521,8 @@ if(BUILD_TESTING)
 		COMMAND $<TARGET_FILE:shader_recompiler_compute_tests> --stream-buffer-only)
 	add_test(NAME gpu_command_lane
 		COMMAND $<TARGET_FILE:shader_recompiler_compute_tests> --gpu-command-lane-only)
+	add_test(NAME pm4_context_state
+		COMMAND $<TARGET_FILE:shader_recompiler_compute_tests> --context-state-only)
 	add_test(NAME gpu_tiler
 		COMMAND $<TARGET_FILE:shader_recompiler_compute_tests> --gpu-tiler-only)
 
@@ -543,6 +559,7 @@ if(BUILD_TESTING)
 		resource_tracking_tests
 		resource_mutex_tests
 		event_queue_lifetime_tests
+		sync_on_address_tests
 		shader_recompiler_compute_tests
 		virtual_memory_allocation_tests
 	)

src/graphics/guest_gpu/command_processor/commandProcessor.h

@@ -15,6 +15,13 @@ bool TestWaitRegMemValue(uint64_t value, uint64_t ref, uint64_t mask, uint32_t f
 
 enum class Pm4ProcessResult { Complete, Blocked };
 
+enum class ContextStateOperation : uint32_t {
+	Clear     = 0,
+	Push      = 1,
+	Pop       = 2,
+	PushClear = 3,
+};
+
 class Pm4Execution {
 public:
 	[[nodiscard]] bool MadeProgress() const noexcept { return m_made_progress; }
@@ -49,6 +56,7 @@ public:
 	KYTY_CLASS_NO_COPY(CommandProcessor);
 
 	void Reset();
+	void ApplyContextStateOperation(ContextStateOperation operation);
 
 	void            BufferInit();
 	void            BufferFlush();
@@ -149,6 +157,8 @@ private:
 
 	RenderContext&   m_renderer;
 	HW::Context      m_ctx;
+	HW::Context      m_saved_ctx;
+	bool             m_context_state_pushed = false;
 	HW::UserConfig   m_ucfg;
 	HW::Shader       m_sh_ctx;
 	HW::UserSgprType m_user_data_marker                 = HW::UserSgprType::Unknown;

src/graphics/guest_gpu/command_processor/pm4Dispatch.cpp

@@ -254,6 +254,7 @@ constexpr auto MakeCustomOpcodeDispatchTable() {
 	g_cp_op_custom_func[Pm4::R_WAIT_MEM_64]    = CpOpWaitRegMem64;
 	g_cp_op_custom_func[Pm4::R_FLIP]           = CpOpFlip;
 	g_cp_op_custom_func[Pm4::R_RELEASE_MEM]    = CpOpReleaseMem;
+	g_cp_op_custom_func[Pm4::R_CONTEXT_STATE]  = CpOpContextState;
 
 	return g_cp_op_custom_func;
 }

src/graphics/guest_gpu/command_processor/pm4Dispatch.h

@@ -121,6 +121,7 @@ uint32_t HwUcSetGeIndexOffset(CommandProcessor&, uint32_t, uint32_t, const uint3
 uint32_t HwUcSetGdsOaRegisters(CommandProcessor&, uint32_t, uint32_t, const uint32_t*, uint32_t);
 
 uint32_t CpOpNop(CommandProcessor&, uint32_t, const uint32_t*, uint32_t, uint32_t);
+uint32_t CpOpContextState(CommandProcessor&, uint32_t, const uint32_t*, uint32_t, uint32_t);
 uint32_t CpOpSetBase(CommandProcessor&, uint32_t, const uint32_t*, uint32_t, uint32_t);
 uint32_t CpOpClearState(CommandProcessor&, uint32_t, const uint32_t*, uint32_t, uint32_t);
 uint32_t CpOpIndexBase(CommandProcessor&, uint32_t, const uint32_t*, uint32_t, uint32_t);

src/graphics/guest_gpu/command_processor/pm4Handlers.cpp

@@ -2107,6 +2107,19 @@ KYTY_CP_OP_PARSER(CpOpClearState) {
 	return 1;
 }
 
+KYTY_CP_OP_PARSER(CpOpContextState) {
+	KYTY_PROFILER_FUNCTION();
+
+	const auto packet_size_dw = KYTY_PM4_LEN(cmd_id);
+	EXIT_NOT_IMPLEMENTED(KYTY_PM4_R(cmd_id) != Pm4::R_CONTEXT_STATE);
+	EXIT_NOT_IMPLEMENTED(packet_size_dw != 3u && packet_size_dw != 5u);
+	EXIT_NOT_IMPLEMENTED(buffer[0] > static_cast<uint32_t>(ContextStateOperation::PushClear));
+
+	cp.ApplyContextStateOperation(static_cast<ContextStateOperation>(buffer[0]));
+
+	return packet_size_dw - 1u;
+}
+
 KYTY_CP_OP_PARSER(CpOpDumpConstRam) {
 	KYTY_PROFILER_FUNCTION();
 

src/graphics/guest_gpu/graphicsRun.cpp

@@ -312,6 +312,8 @@ void CommandProcessor::Reset() {
 	m_sh_ctx.Reset();
 	m_ucfg.Reset();
 	m_ctx.Reset();
+	m_saved_ctx.Reset();
+	m_context_state_pushed = false;
 	m_index_type_and_size              = 0;
 	m_index_buffer_size                = 0;
 	m_user_data_marker                 = HW::UserSgprType::Unknown;
@@ -321,6 +323,30 @@ void CommandProcessor::Reset() {
 	std::memset(m_const_ram, 0, sizeof(m_const_ram));
 }
 
+void CommandProcessor::ApplyContextStateOperation(ContextStateOperation operation) {
+	switch (operation) {
+		case ContextStateOperation::Clear: m_ctx.Reset(); break;
+		case ContextStateOperation::Push:
+			EXIT_IF(m_context_state_pushed);
+			m_saved_ctx            = m_ctx;
+			m_context_state_pushed = true;
+			break;
+		case ContextStateOperation::Pop:
+			EXIT_IF(!m_context_state_pushed);
+			m_ctx                  = m_saved_ctx;
+			m_saved_ctx            = {};
+			m_context_state_pushed = false;
+			break;
+		case ContextStateOperation::PushClear:
+			EXIT_IF(m_context_state_pushed);
+			m_saved_ctx            = m_ctx;
+			m_context_state_pushed = true;
+			m_ctx.Reset();
+			break;
+		default: EXIT("unknown context state operation: %u\n", static_cast<uint32_t>(operation));
+	}
+}
+
 void CommandProcessor::BufferInit() {
 	GetScheduler().Begin(m_ctx, m_ucfg, m_sh_ctx);
 }

src/graphics/guest_gpu/pm4.cpp

@@ -30,6 +30,7 @@ constexpr auto MakeRegisterNames() {
 	names[R_FLIP]           = "R_FLIP";
 	names[R_RELEASE_MEM]    = "R_RELEASE_MEM";
 	names[R_DMA_DATA]       = "R_DMA_DATA";
+	names[R_CONTEXT_STATE]  = "R_CONTEXT_STATE";
 	return names;
 }
 

src/graphics/guest_gpu/pm4.h

@@ -87,6 +87,7 @@ constexpr uint32_t R_WAIT_MEM_64    = 0x16;
 constexpr uint32_t R_FLIP           = 0x17;
 constexpr uint32_t R_RELEASE_MEM    = 0x18;
 constexpr uint32_t R_DMA_DATA       = 0x19;
+constexpr uint32_t R_CONTEXT_STATE  = 0x1A;
 
 constexpr uint32_t R_NUM = 0x3F + 1;
 

src/kernel/syncOnAddress.cpp

@@ -0,0 +1,287 @@
+#include "kernel/syncOnAddress.h"
+
+#include "common/threads.h"
+#include "libs/errno.h"
+
+#include <algorithm>
+#include <chrono>
+#include <climits>
+#include <cstdint>
+#include <list>
+#include <memory>
+#include <mutex>
+#include <unordered_map>
+
+#if KYTY_PLATFORM == KYTY_PLATFORM_LINUX && !defined(__APPLE__)
+#include <cerrno>
+#include <linux/futex.h>
+#include <sys/syscall.h>
+#include <unistd.h>
+#endif
+
+namespace Libs::LibKernel::SyncOnAddress {
+
+namespace {
+
+constexpr uint32_t SIGNAL_POLL_MICROS = 10000;
+
+using Clock = std::chrono::steady_clock;
+
+template <typename T>
+[[nodiscard]] bool IsValidWaitAddress(const volatile T* address) {
+	return address != nullptr && (reinterpret_cast<uintptr_t>(address) & (alignof(T) - 1u)) == 0;
+}
+
+[[nodiscard]] bool IsValidWakeAddress(const volatile void* address) {
+	return address != nullptr &&
+	       (reinterpret_cast<uintptr_t>(address) & (alignof(uint32_t) - 1u)) == 0;
+}
+
+template <typename T>
+[[nodiscard]] T ReadWord(const volatile T* address) {
+	return __atomic_load_n(address, __ATOMIC_ACQUIRE);
+}
+
+struct WaitDeadline {
+	bool              finite = false;
+	Clock::time_point end {};
+};
+
+[[nodiscard]] WaitDeadline MakeDeadline(const uint32_t* timeout_micros) {
+	if (timeout_micros == nullptr) {
+		return {};
+	}
+	return {true, Clock::now() + std::chrono::microseconds(*timeout_micros)};
+}
+
+[[nodiscard]] uint32_t GetWaitSliceMicros(const WaitDeadline& deadline, bool first_wait) {
+	if (!deadline.finite) {
+		return SIGNAL_POLL_MICROS;
+	}
+
+	const auto now = Clock::now();
+	if (now >= deadline.end) {
+		return first_wait ? 0u : UINT32_MAX;
+	}
+
+	const auto remaining =
+	    std::chrono::duration_cast<std::chrono::microseconds>(deadline.end - now).count();
+	return static_cast<uint32_t>(std::min<int64_t>(remaining, SIGNAL_POLL_MICROS));
+}
+
+void PollSignals(signal_poll_func_t signal_poll) {
+	if (signal_poll != nullptr) {
+		signal_poll();
+	}
+}
+
+#if KYTY_PLATFORM == KYTY_PLATFORM_LINUX && !defined(__APPLE__)
+
+template <typename T>
+int WaitLinux(volatile T* address, T expected, const uint32_t* timeout_micros,
+              signal_poll_func_t signal_poll) {
+	const auto deadline   = MakeDeadline(timeout_micros);
+	bool       first_wait = true;
+
+	for (;;) {
+		if (ReadWord(address) != expected) {
+			return OK;
+		}
+		const auto slice_micros = GetWaitSliceMicros(deadline, first_wait);
+		if (slice_micros == UINT32_MAX) {
+			return ReadWord(address) == expected ? KERNEL_ERROR_ETIMEDOUT : OK;
+		}
+		const timespec timeout = {
+		    .tv_sec  = static_cast<time_t>(slice_micros / 1000000u),
+		    .tv_nsec = static_cast<long>(slice_micros % 1000000u) * 1000L,
+		};
+
+		long result     = 0;
+		int  wait_error = 0;
+		result          = syscall(SYS_futex, const_cast<T*>(address), FUTEX_WAIT_PRIVATE,
+		                          static_cast<uint32_t>(expected), &timeout, nullptr, 0);
+		if (result != 0) {
+			wait_error = errno;
+		}
+		if (result == 0 || wait_error == EAGAIN) {
+			return OK;
+		}
+		if (wait_error != ETIMEDOUT && wait_error != EINTR) {
+			return KERNEL_ERROR_EINVAL;
+		}
+
+		PollSignals(signal_poll);
+		if (deadline.finite && Clock::now() >= deadline.end) {
+			return ReadWord(address) == expected ? KERNEL_ERROR_ETIMEDOUT : OK;
+		}
+		first_wait = false;
+	}
+}
+
+int WakeLinux(volatile void* address, int32_t count) {
+	const auto result = syscall(SYS_futex, const_cast<void*>(address), FUTEX_WAKE_PRIVATE, count,
+	                            nullptr, nullptr, 0);
+	return result < 0 ? KERNEL_ERROR_EINVAL : OK;
+}
+
+#endif
+
+struct PortableWaiter {
+	Common::CondVar condition;
+	bool            wake_requested = false;
+};
+
+struct PortableAddressEntry {
+	Common::Mutex              mutex;
+	std::list<PortableWaiter*> waiters;
+};
+
+struct PortableAddressRegistry {
+	std::mutex                                                           mutex;
+	std::unordered_map<uintptr_t, std::shared_ptr<PortableAddressEntry>> entries;
+};
+
+PortableAddressRegistry& GetPortableRegistry() {
+	static PortableAddressRegistry registry;
+	return registry;
+}
+
+std::shared_ptr<PortableAddressEntry> RegisterPortableWaiter(volatile void*  address,
+                                                             PortableWaiter* waiter) {
+	auto&           registry = GetPortableRegistry();
+	std::lock_guard registry_lock(registry.mutex);
+	auto&           entry = registry.entries[reinterpret_cast<uintptr_t>(address)];
+	if (!entry) {
+		entry = std::make_shared<PortableAddressEntry>();
+	}
+	entry->mutex.Lock();
+	entry->waiters.push_back(waiter);
+	return entry;
+}
+
+void UnregisterPortableWaiter(volatile void*                               address,
+                              const std::shared_ptr<PortableAddressEntry>& entry,
+                              PortableWaiter*                              waiter) {
+	entry->waiters.remove(waiter);
+	const bool empty = entry->waiters.empty();
+	entry->mutex.Unlock();
+	if (!empty) {
+		return;
+	}
+
+	auto&           registry = GetPortableRegistry();
+	std::lock_guard registry_lock(registry.mutex);
+	entry->mutex.Lock();
+	const auto it = registry.entries.find(reinterpret_cast<uintptr_t>(address));
+	if (it != registry.entries.end() && it->second == entry && entry->waiters.empty()) {
+		registry.entries.erase(it);
+	}
+	entry->mutex.Unlock();
+}
+
+template <typename T>
+int WaitPortable(volatile T* address, T expected, const uint32_t* timeout_micros,
+                 signal_poll_func_t signal_poll) {
+	PortableWaiter waiter;
+	auto           entry      = RegisterPortableWaiter(address, &waiter);
+	const auto     deadline   = MakeDeadline(timeout_micros);
+	bool           first_wait = true;
+	int            result     = OK;
+
+	while (ReadWord(address) == expected && !waiter.wake_requested) {
+		const auto slice_micros = GetWaitSliceMicros(deadline, first_wait);
+		if (slice_micros == UINT32_MAX) {
+			result = KERNEL_ERROR_ETIMEDOUT;
+			break;
+		}
+		if (slice_micros == 0) {
+			result = KERNEL_ERROR_ETIMEDOUT;
+			break;
+		}
+
+		(void)waiter.condition.WaitFor(&entry->mutex, slice_micros);
+		entry->mutex.Unlock();
+		PollSignals(signal_poll);
+		entry->mutex.Lock();
+
+		if (deadline.finite && Clock::now() >= deadline.end && ReadWord(address) == expected &&
+		    !waiter.wake_requested) {
+			result = KERNEL_ERROR_ETIMEDOUT;
+			break;
+		}
+		first_wait = false;
+	}
+
+	UnregisterPortableWaiter(address, entry, &waiter);
+	return result;
+}
+
+int WakePortable(volatile void* address, int32_t count) {
+	if (count == 0) {
+		return OK;
+	}
+	auto&            registry = GetPortableRegistry();
+	std::unique_lock registry_lock(registry.mutex);
+	const auto       it = registry.entries.find(reinterpret_cast<uintptr_t>(address));
+	if (it == registry.entries.end()) {
+		return OK;
+	}
+	auto entry = it->second;
+	entry->mutex.Lock();
+	registry_lock.unlock();
+
+	int32_t remaining = count;
+	for (auto* waiter: entry->waiters) {
+		if (!waiter->wake_requested) {
+			waiter->wake_requested = true;
+			waiter->condition.Signal();
+			if (remaining != INT_MAX && --remaining == 0) {
+				break;
+			}
+		}
+	}
+	entry->mutex.Unlock();
+	return OK;
+}
+
+template <typename T>
+int WaitImpl(volatile T* address, T expected, const uint32_t* timeout_micros,
+             signal_poll_func_t signal_poll) {
+	if (!IsValidWaitAddress(address)) {
+		return KERNEL_ERROR_EINVAL;
+	}
+
+	int result = OK;
+#if KYTY_PLATFORM == KYTY_PLATFORM_LINUX && !defined(__APPLE__)
+	result = WaitLinux(address, expected, timeout_micros, signal_poll);
+#else
+	result = WaitPortable(address, expected, timeout_micros, signal_poll);
+#endif
+	PollSignals(signal_poll);
+	return result;
+}
+
+} // namespace
+
+int Wait32(volatile uint32_t* address, uint32_t expected, const uint32_t* timeout_micros,
+           signal_poll_func_t signal_poll) {
+	return WaitImpl(address, expected, timeout_micros, signal_poll);
+}
+
+int Wait64(volatile uint64_t* address, uint64_t expected, const uint32_t* timeout_micros,
+           signal_poll_func_t signal_poll) {
+	return WaitImpl(address, expected, timeout_micros, signal_poll);
+}
+
+int Wake(volatile void* address, int32_t count) {
+	if (!IsValidWakeAddress(address) || count < 0) {
+		return KERNEL_ERROR_EINVAL;
+	}
+
+#if KYTY_PLATFORM == KYTY_PLATFORM_LINUX && !defined(__APPLE__)
+	return WakeLinux(address, count);
+#endif
+	return WakePortable(address, count);
+}
+
+} // namespace Libs::LibKernel::SyncOnAddress

src/kernel/syncOnAddress.h

@@ -0,0 +1,19 @@
+#ifndef EMULATOR_INCLUDE_EMULATOR_KERNEL_SYNC_ON_ADDRESS_H_
+#define EMULATOR_INCLUDE_EMULATOR_KERNEL_SYNC_ON_ADDRESS_H_
+
+#include "common/abi.h"
+#include "common/common.h"
+
+namespace Libs::LibKernel::SyncOnAddress {
+
+using signal_poll_func_t = void (*)();
+
+int Wait32(volatile uint32_t* address, uint32_t expected, const uint32_t* timeout_micros,
+           signal_poll_func_t signal_poll = nullptr);
+int Wait64(volatile uint64_t* address, uint64_t expected, const uint32_t* timeout_micros,
+           signal_poll_func_t signal_poll = nullptr);
+int Wake(volatile void* address, int32_t count);
+
+} // namespace Libs::LibKernel::SyncOnAddress
+
+#endif /* EMULATOR_INCLUDE_EMULATOR_KERNEL_SYNC_ON_ADDRESS_H_ */

src/libs/agc.cpp

@@ -6,6 +6,7 @@
 #include "common/logging/log.h"
 #include "common/stringUtils.h"
 #include "common/virtualMemory.h"
+#include "graphics/guest_gpu/command_processor/commandProcessor.h"
 #include "graphics/guest_gpu/gpu_defs.h"
 #include "graphics/guest_gpu/graphicsRun.h"
 #include "graphics/guest_gpu/hardwareContext.h"
@@ -1444,29 +1445,72 @@ int KYTY_SYSV_ABI GraphicsSuspendPoint() {
 	return OK;
 }
 
-uint32_t* KYTY_SYSV_ABI GraphicsUnknownQj7QZpgr9Uw(CommandBuffer* buf, uint32_t mode,
-                                                   uint32_t value) {
-	PRINT_NAME();
-	// ResetContextState?
-	LOGF("\t mode  = 0x%08" PRIx32 "\n"
-	     "\t value = 0x%08" PRIx32 "\n",
-	     mode, value);
+[[nodiscard]] static constexpr uint32_t context_state_op_size_dw(uint32_t operation) {
+	switch (static_cast<ContextStateOperation>(operation)) {
+		case ContextStateOperation::Clear: return 5;
+		case ContextStateOperation::Push:
+		case ContextStateOperation::Pop: return 27;
+		case ContextStateOperation::PushClear: return 32;
+		default: return 0;
+	}
+}
 
-	if (buf == nullptr) {
+uint32_t* KYTY_SYSV_ABI GraphicsDcbContextStateOp(CommandBuffer* buf, uint32_t operation) {
+	PRINT_NAME();
+	LOGF("\t operation = 0x%08" PRIx32 "\n", operation);
+
+	const auto size_dw = context_state_op_size_dw(operation);
+	if (buf == nullptr || size_dw == 0) {
 		return nullptr;
 	}
 
 	buf->DbgDump();
 
-	auto* cmd = buf->AllocateDW(1);
+	uint32_t* first = nullptr;
+	const auto append = [&](uint32_t segment_size_dw) {
+		auto* segment = buf->AllocateDW(segment_size_dw);
+		if (segment == nullptr) {
+			return false;
+		}
+		std::fill_n(segment, segment_size_dw, 0u);
+		if (first == nullptr) {
+			first      = segment;
+			segment[0] = KYTY_PM4(segment_size_dw, Pm4::IT_NOP, Pm4::R_CONTEXT_STATE);
+			segment[1] = operation;
+		} else {
+			segment[0] = KYTY_PM4(segment_size_dw, Pm4::IT_NOP, Pm4::R_ZERO);
+		}
+		return true;
+	};
+	const auto append_save_restore = [&] {
+		// The native helper reserves 22 dwords, then emits 5-, 8-, and 9-dword packets.
+		return buf->ReserveDW(22) && append(5) && append(8) && append(9);
+	};
 
-	if (cmd == nullptr) {
-		return nullptr;
+	// Preserve native packet sizes, allocation boundaries, Push/Pop ordering, and
+	// returned first-packet address. The native packets reference process-private state buffers,
+	// so their net Cx operation is represented by the first HLE packet and type-3 NOPs.
+	bool complete = false;
+	switch (static_cast<ContextStateOperation>(operation)) {
+		case ContextStateOperation::Clear: complete = append(5); break;
+		case ContextStateOperation::Push:
+			complete = append_save_restore() && append(3) && append(2);
+			break;
+		case ContextStateOperation::Pop:
+			complete = append(3) && append_save_restore() && append(2);
+			break;
+		case ContextStateOperation::PushClear:
+			complete = append_save_restore() && append(3) && append(2) && append(5);
+			break;
+		default: break;
 	}
 
-	cmd[0] = 0x80000000u;
+	return complete ? first : nullptr;
+}
 
-	return cmd;
+uint64_t KYTY_SYSV_ABI GraphicsDcbContextStateOpGetSize(uint32_t operation) {
+	PRINT_NAME();
+	return static_cast<uint64_t>(context_state_op_size_dw(operation)) * sizeof(uint32_t);
 }
 
 uint64_t KYTY_SYSV_ABI GraphicsGetIsTrinityMode() {

src/libs/agc.h

@@ -67,8 +67,8 @@ int KYTY_SYSV_ABI GraphicsUnknownJumpPatchSetTarget(uint32_t* cmd, const volatil
 int KYTY_SYSV_ABI GraphicsJumpPatchSetTarget(uint32_t* cmd, const volatile uint32_t* target,
                                              uint32_t size_in_dwords);
 int KYTY_SYSV_ABI GraphicsSuspendPoint();
-uint32_t* KYTY_SYSV_ABI GraphicsUnknownQj7QZpgr9Uw(CommandBuffer* buf, uint32_t mode,
-                                                   uint32_t value);
+uint32_t* KYTY_SYSV_ABI GraphicsDcbContextStateOp(CommandBuffer* buf, uint32_t operation);
+uint64_t KYTY_SYSV_ABI  GraphicsDcbContextStateOpGetSize(uint32_t operation);
 uint64_t KYTY_SYSV_ABI  GraphicsGetIsTrinityMode();
 
 uint32_t KYTY_SYSV_ABI GraphicsDriverGetDefaultOwner();

src/libs/libGraphicsDriver.cpp

@@ -178,7 +178,8 @@ LIB_DEFINE(InitGraphicsDriver_1) {
 	LIB_FUNC("bxca0BK4FNg", Gen5::GraphicsUnknownJumpPatchSetTarget);
 	LIB_FUNC("2BS4EtAaF28", Gen5::GraphicsJumpPatchSetTarget);
 	LIB_FUNC("h9z6+0hEydk", Gen5::GraphicsSuspendPoint);
-	LIB_FUNC("qj7QZpgr9Uw", Gen5::GraphicsUnknownQj7QZpgr9Uw);
+	LIB_FUNC("qj7QZpgr9Uw", Gen5::GraphicsDcbContextStateOp);
+	LIB_FUNC("H6vHS5cidSA", Gen5::GraphicsDcbContextStateOpGetSize);
 	LIB_FUNC("BfBDZGbti7A", Gen5::GraphicsGetIsTrinityMode);
 	LIB_FUNC("dbOlWdppb4o", LibGen5::Graphics5UnknownDb);
 

src/libs/libKernel.cpp

@@ -13,6 +13,7 @@
 #include "kernel/memory.h"
 #include "kernel/pthread.h"
 #include "kernel/semaphore.h"
+#include "kernel/syncOnAddress.h"
 #include "libs/errno.h"
 #include "libs/libs.h"
 #include "libs/network.h"
@@ -2134,26 +2135,37 @@ uint64_t KYTY_SYSV_ABI cfwBSQyr5Ys(uint64_t a1, uint64_t a2, uint64_t a3, uint64
 	return 0;
 }
 
-uint64_t KYTY_SYSV_ABI KernelSyncOnAddressV1(uint64_t op, uint64_t address, uint64_t value,
-                                             uint64_t size, uint64_t timeout, uint64_t flags) {
-	static std::atomic_uint32_t log_count = 0;
-	const auto                  index     = log_count.fetch_add(1, std::memory_order_relaxed);
-
-	if (index < 16) {
-		LOGF("\t libkernel_sync_on_address_v1: op=0x%016" PRIx64 ", address=0x%016" PRIx64
-		     ", value=0x%016" PRIx64 ", size=0x%016" PRIx64 ", timeout=0x%016" PRIx64
-		     ", flags=0x%016" PRIx64 "\n",
-		     op, address, value, size, timeout, flags);
+static void LogExperimentalSyncOnAddress(std::atomic_bool& logged, const char* function_name) {
+	if (!logged.exchange(true, std::memory_order_relaxed)) {
+		::printf("WARNING: %s is experimental\n", function_name);
+		LOGF("WARNING: %s is experimental\n", function_name);
 	}
+}
 
-	if (op != 0 && address == 0 && value == 0 && size == 0) {
-		// This unsupported form is used as a yield/wait by some Unity jobs.
-		// SleepMicro() uses a sub-millisecond busy wait on Windows, which can
-		// pin every worker thread when the guest polls this path.
-		Common::Thread::Sleep(timeout == 0 ? 1 : 2);
-	}
+int KYTY_SYSV_ABI KernelSyncOnAddressWait(volatile uint32_t* address, uint32_t expected,
+                                          const uint32_t* timeout_micros) {
+	return LibKernel::SyncOnAddress::Wait32(address, expected, timeout_micros,
+	                                        LibKernel::KernelDispatchPendingSignalForCurrentThread);
+}
 
-	return 0;
+int KYTY_SYSV_ABI KernelSyncOnAddressWait32(volatile uint32_t* address, uint32_t expected,
+                                            const uint32_t* timeout_micros) {
+	static std::atomic_bool logged {false};
+	LogExperimentalSyncOnAddress(logged, "sceKernelSyncOnAddressWait32");
+	return LibKernel::SyncOnAddress::Wait32(address, expected, timeout_micros,
+	                                        LibKernel::KernelDispatchPendingSignalForCurrentThread);
+}
+
+int KYTY_SYSV_ABI KernelSyncOnAddressWait64(volatile uint64_t* address, uint64_t expected,
+                                            const uint32_t* timeout_micros) {
+	static std::atomic_bool logged {false};
+	LogExperimentalSyncOnAddress(logged, "sceKernelSyncOnAddressWait64");
+	return LibKernel::SyncOnAddress::Wait64(address, expected, timeout_micros,
+	                                        LibKernel::KernelDispatchPendingSignalForCurrentThread);
+}
+
+int KYTY_SYSV_ABI KernelSyncOnAddressWake(volatile void* address, int32_t count) {
+	return LibKernel::SyncOnAddress::Wake(address, count);
 }
 
 LIB_DEFINE(InitLibKernel_1_Posix) {
@@ -3405,8 +3417,10 @@ LIB_DEFINE(InitLibKernel_1) {
 	LIB_FUNC("Xjoosiw+XPI", LibKernel::KernelUuidCreate);
 	LIB_FUNC("DLORcroUqbc", LibKernel::KernelGetOpenPsId);
 	LIB_FUNC("zE-wXIZjLoM", LibKernel::KernelDebugRaiseExceptionOnReleaseMode);
-	LIB_FUNC("Hc4CaR6JBL0", Posix::KernelSyncOnAddressV1);
-	LIB_FUNC("q2y-wDIVWZA", Posix::KernelSyncOnAddressV1);
+	LIB_FUNC("Hc4CaR6JBL0", Posix::KernelSyncOnAddressWait);
+	LIB_FUNC("B2n8aDorSH4", Posix::KernelSyncOnAddressWait32);
+	LIB_FUNC("PZQhiiLXRFs", Posix::KernelSyncOnAddressWait64);
+	LIB_FUNC("q2y-wDIVWZA", Posix::KernelSyncOnAddressWake);
 
 	AddLibkernelUnityFunc(s, "Qhv5ARAoOEc",
 	                      reinterpret_cast<uint64_t>(LibKernel::KernelRemoveExceptionHandler),

src/utils.cmake

@@ -101,8 +101,6 @@ if(KYTY_CLANG_CL)
 	string(REGEX REPLACE "/MD" "/MT" CMAKE_C_FLAGS "${CMAKE_C_FLAGS}")
 			
 	set(KYTY_CPP_FLAGS "${KYTY_CPP_FLAGS} /utf-8 /Oy- /wd4244 /wd4305 /wd4800 /wd4345")
-	
-	add_link_options("$<$<CONFIG:RELEASE>:/OPT:NOREF>")
   
 	if(KYTY_WARNINGS_ARE_ERRORS)
 		#set(KYTY_CPP_FLAGS "${KYTY_CPP_FLAGS} /WX")

tests/ShaderRecompilerComputeTests.cpp

@@ -42,6 +42,7 @@
 #include "graphics/shader/rectListShader.h"
 #include "graphics/shader/shader.h"
 #include "kernel/memory.h"
+#include "libs/agc.h"
 #include "spirv-tools/libspirv.hpp"
 
 #if __has_include("graphics/host_gpu/renderer/renderTargetBarriers.h")
@@ -17653,6 +17654,136 @@ void CheckPm4StencilInfoValueLane(RenderContext& renderer) {
 	std::printf("[host]    %-32s ok\n", "Pm4StencilInfoValueLane");
 }
 
+void CheckPm4ContextStateOperations(RenderContext& renderer) {
+	GraphicsInitJmpTables();
+	CommandProcessor processor(renderer);
+	struct AgcCommandBufferLayout {
+		using Callback = KYTY_SYSV_ABI bool (*)(Gen5::CommandBuffer*, uint32_t, void*);
+
+		uint32_t* bottom;
+		uint32_t* top;
+		uint32_t* cursor_up;
+		uint32_t* cursor_down;
+		Callback  callback;
+		void*     user_data;
+		uint32_t  reserved_dw;
+	};
+	static_assert(offsetof(AgcCommandBufferLayout, cursor_up) == 0x10);
+	static_assert(offsetof(AgcCommandBufferLayout, reserved_dw) == 0x30);
+
+	HW::RenderControl original_control {};
+	original_control.depth_clear_enable   = true;
+	original_control.stencil_clear_enable = true;
+	processor.GetCtx().SetRenderControl(original_control);
+	processor.GetCtx().SetRenderTargetMask(0x1234abcd);
+	processor.GetUcfg().SetPrimitiveType(0x35);
+	g_hw_sh_indirect_func[Pm4::SPI_SHADER_PGM_LO_PS](processor, Pm4::SPI_SHADER_PGM_LO_PS,
+	                                                 0x00123456);
+	const auto shader_address = processor.GetShCtx().GetPs().ps_regs.data_addr;
+	const auto invoke = [&](ContextStateOperation operation) {
+		std::array<uint32_t, 32> packet {};
+		std::array<uint32_t, 6>  segment_sizes {};
+		AgcCommandBufferLayout   dcb {packet.data(), packet.data() + packet.size(), packet.data(),
+		                              packet.data() + packet.size(), nullptr, nullptr, 0};
+		const auto operation_value = static_cast<uint32_t>(operation);
+		size_t     segment_count  = 0;
+		switch (operation) {
+			case ContextStateOperation::Clear: segment_sizes = {5}; segment_count = 1; break;
+			case ContextStateOperation::Push:
+				segment_sizes = {5, 8, 9, 3, 2};
+				segment_count = 5;
+				break;
+			case ContextStateOperation::Pop:
+				segment_sizes = {3, 5, 8, 9, 2};
+				segment_count = 5;
+				break;
+			case ContextStateOperation::PushClear:
+				segment_sizes = {5, 8, 9, 3, 2, 5};
+				segment_count = 6;
+				break;
+		}
+		const auto size_dw =
+		    static_cast<size_t>(Gen5::GraphicsDcbContextStateOpGetSize(operation_value) / 4);
+		auto* emitted = Gen5::GraphicsDcbContextStateOp(
+		    reinterpret_cast<Gen5::CommandBuffer*>(&dcb), operation_value);
+		bool   packet_matches = true;
+		size_t segment_offset = 0;
+		for (size_t i = 0; i < segment_count; i++) {
+			const auto segment_size = segment_sizes[i];
+			const auto selector = (i == 0 ? Pm4::R_CONTEXT_STATE : Pm4::R_ZERO);
+			packet_matches &=
+			    packet[segment_offset] == KYTY_PM4(segment_size, Pm4::IT_NOP, selector);
+			for (size_t j = 1; j < segment_size; j++) {
+				packet_matches &= packet[segment_offset + j] ==
+				                  (i == 0 && j == 1 ? operation_value : 0u);
+			}
+			segment_offset += segment_size;
+		}
+		Require("Pm4ContextState", "HLE packet",
+		        emitted == packet.data() && dcb.cursor_up == packet.data() + size_dw &&
+		            segment_offset == size_dw && packet_matches,
+		        "context-state HLE packet does not match its real allocation size");
+		Pm4Execution execution;
+		return processor.Process(execution, packet.data(), size_dw);
+	};
+
+	Require("Pm4ContextState", "push-clear",
+	        invoke(ContextStateOperation::PushClear) == Pm4ProcessResult::Complete &&
+	            !processor.GetCtx().GetRenderControl().depth_clear_enable &&
+	            !processor.GetCtx().GetRenderControl().stencil_clear_enable &&
+	            processor.GetCtx().GetRenderTargetMask() == HW::Context {}.GetRenderTargetMask() &&
+	            processor.GetUcfg().GetPrimType() == 0x35 &&
+	            processor.GetShCtx().GetPs().ps_regs.data_addr == shader_address,
+	        "push-clear did not save and clear only Cx state");
+
+	processor.GetCtx().SetRenderTargetMask(0x55aa55aa);
+	Require("Pm4ContextState", "pop",
+	        invoke(ContextStateOperation::Pop) == Pm4ProcessResult::Complete &&
+	            processor.GetCtx().GetRenderControl().depth_clear_enable &&
+	            processor.GetCtx().GetRenderControl().stencil_clear_enable &&
+	            processor.GetCtx().GetRenderTargetMask() == 0x1234abcd,
+	        "pop did not restore the complete saved Cx state");
+
+	Require("Pm4ContextState", "push",
+	        invoke(ContextStateOperation::Push) == Pm4ProcessResult::Complete &&
+	            processor.GetCtx().GetRenderTargetMask() == 0x1234abcd,
+	        "push changed live Cx state");
+	processor.GetCtx().SetRenderTargetMask(0x01020304);
+	Require("Pm4ContextState", "push-pop restore",
+	        invoke(ContextStateOperation::Pop) == Pm4ProcessResult::Complete &&
+	            processor.GetCtx().GetRenderTargetMask() == 0x1234abcd,
+	        "ordinary push/pop did not restore Cx state");
+
+	Require("Pm4ContextState", "clear",
+	        invoke(ContextStateOperation::Clear) == Pm4ProcessResult::Complete &&
+	            !processor.GetCtx().GetRenderControl().depth_clear_enable &&
+	            processor.GetCtx().GetRenderTargetMask() == HW::Context {}.GetRenderTargetMask() &&
+	            processor.GetUcfg().GetPrimType() == 0x35 &&
+	            processor.GetShCtx().GetPs().ps_regs.data_addr == shader_address,
+	        "clear reset state outside the Cx register domain");
+
+	processor.GetCtx().SetRenderTargetMask(0xabcdef01);
+	Require("Pm4ContextState", "push before processor reset",
+	        invoke(ContextStateOperation::Push) == Pm4ProcessResult::Complete,
+	        "push before reset failed");
+	processor.Reset();
+	processor.GetCtx().SetRenderTargetMask(0x76543210);
+	Require("Pm4ContextState", "processor reset discards push",
+	        invoke(ContextStateOperation::Push) == Pm4ProcessResult::Complete &&
+	            invoke(ContextStateOperation::Pop) == Pm4ProcessResult::Complete &&
+	            processor.GetCtx().GetRenderTargetMask() == 0x76543210,
+	        "processor reset retained an invalid saved Cx state");
+
+	Require("Pm4ContextState", "HLE packet size",
+	        Gen5::GraphicsDcbContextStateOpGetSize(0) == 20 &&
+	            Gen5::GraphicsDcbContextStateOpGetSize(1) == 108 &&
+	            Gen5::GraphicsDcbContextStateOpGetSize(2) == 108 &&
+	            Gen5::GraphicsDcbContextStateOpGetSize(3) == 128 &&
+	            Gen5::GraphicsDcbContextStateOpGetSize(4) == 0,
+	        "context-state HLE sizes do not match libSceAgc");
+	std::printf("[host]    %-32s ok\n", "Pm4ContextState");
+}
+
 void CheckPm4WaitResume(RenderContext& renderer) {
 	GraphicsInitJmpTables();
 	CommandProcessor processor(renderer);
@@ -17789,6 +17920,11 @@ int main(int argc, char** argv) {
 		vulkan.CheckGpuCommandLane();
 		return 0;
 	}
+	if (argc == 2 && std::strcmp(argv[1], "--context-state-only") == 0) {
+		VulkanHarness vulkan;
+		CheckPm4ContextStateOperations(vulkan.RuntimeRenderer());
+		return 0;
+	}
 	if (argc == 2 && std::strcmp(argv[1], "--image-overlap-only") == 0) {
 		CheckDepthAttachmentWrites();
 		CheckDynamicRenderingState();
@@ -17923,6 +18059,7 @@ int main(int argc, char** argv) {
 	CheckVulkan13FeatureRequirements();
 	CheckPm4AcquireMemNoOp(vulkan.RuntimeRenderer());
 	CheckPm4StencilInfoValueLane(vulkan.RuntimeRenderer());
+	CheckPm4ContextStateOperations(vulkan.RuntimeRenderer());
 	CheckPm4WaitResume(vulkan.RuntimeRenderer());
 	CheckPm4CeCompletion(vulkan.RuntimeRenderer());
 	CheckEmbeddedFetchVertexOffset();

tests/SyncOnAddressTests.cpp

@@ -0,0 +1,269 @@
+#include "kernel/syncOnAddress.h"
+
+#include "libs/errno.h"
+
+#include <atomic>
+#include <chrono>
+#include <climits>
+#include <cstdio>
+#include <cstdlib>
+#include <thread>
+#include <vector>
+
+namespace {
+
+using Libs::LibKernel::SyncOnAddress::Wait32;
+using Libs::LibKernel::SyncOnAddress::Wait64;
+using Libs::LibKernel::SyncOnAddress::Wake;
+
+std::atomic<int> g_signal_poll_count{0};
+
+void Check(bool value, const char *text) {
+  if (!value) {
+    std::fprintf(stderr, "SyncOnAddressTests: failed: %s\n", text);
+    std::abort();
+  }
+}
+
+template <typename T> void Store(T *address, T value) {
+  std::atomic_ref<T>(*address).store(value, std::memory_order_release);
+}
+
+void CountSignalPoll() {
+  g_signal_poll_count.fetch_add(1, std::memory_order_relaxed);
+}
+
+void TestInvalidAddress() {
+  uint32_t timeout = 1;
+  Check(Wait32(nullptr, 0, &timeout) == Libs::LibKernel::KERNEL_ERROR_EINVAL,
+        "wait32 rejects a null address");
+  Check(Wait64(nullptr, 0, &timeout) == Libs::LibKernel::KERNEL_ERROR_EINVAL,
+        "wait64 rejects a null address");
+  Check(Wake(nullptr, 1) == Libs::LibKernel::KERNEL_ERROR_EINVAL,
+        "wake rejects a null address");
+
+  alignas(uint64_t) uint8_t bytes[16] = {};
+  auto *misaligned = reinterpret_cast<uint32_t *>(bytes + 1);
+  Check(Wait32(misaligned, 0, &timeout) == Libs::LibKernel::KERNEL_ERROR_EINVAL,
+        "wait32 rejects a misaligned address");
+  Check(Wait64(reinterpret_cast<uint64_t *>(bytes + 4), 0, &timeout) ==
+            Libs::LibKernel::KERNEL_ERROR_EINVAL,
+        "wait64 rejects a misaligned address");
+  Check(Wake(misaligned, 1) == Libs::LibKernel::KERNEL_ERROR_EINVAL,
+        "wake rejects a misaligned address");
+  uint64_t aligned = 0;
+  Check(Wake(&aligned, -1) == Libs::LibKernel::KERNEL_ERROR_EINVAL,
+        "wake rejects a negative count");
+}
+
+void TestMismatchReturnsImmediately() {
+  uint32_t word = 7;
+  uint64_t word64 = UINT64_C(0x100000000);
+  uint32_t timeout = 500000;
+  const auto start = std::chrono::steady_clock::now();
+  Check(Wait32(&word, 6, &timeout, CountSignalPoll) == OK,
+        "mismatch succeeds without parking");
+  Check(Wait64(&word64, 0, &timeout, CountSignalPoll) == OK,
+        "wait64 compares all 64 bits");
+  Check(std::chrono::steady_clock::now() - start <
+            std::chrono::milliseconds(100),
+        "mismatched value is a fast path");
+  Check(g_signal_poll_count.load(std::memory_order_relaxed) >= 2,
+        "mismatch remains a guest signal safe-point");
+}
+
+void TestTimeout() {
+  uint32_t word = 0;
+  uint32_t timeout = 20000;
+  const auto start = std::chrono::steady_clock::now();
+  Check(Wait32(&word, 0, &timeout) == Libs::LibKernel::KERNEL_ERROR_ETIMEDOUT,
+        "matching value times out");
+  const auto elapsed = std::chrono::steady_clock::now() - start;
+  Check(elapsed >= std::chrono::milliseconds(10),
+        "timeout does not return too early");
+  Check(elapsed < std::chrono::milliseconds(500), "timeout remains bounded");
+
+  timeout = 0;
+  Check(Wait32(&word, 0, &timeout) == Libs::LibKernel::KERNEL_ERROR_ETIMEDOUT,
+        "zero timeout polls a matching value");
+  word = 1;
+  Check(Wait32(&word, 0, &timeout) == OK,
+        "zero timeout succeeds for a mismatched value");
+}
+
+void TestValueChangeAndWake() {
+  uint64_t word = 0;
+  uint32_t timeout = 1000000;
+  std::atomic<bool> ready{false};
+  int result = Libs::LibKernel::KERNEL_ERROR_ETIMEDOUT;
+
+  std::thread waiter([&] {
+    ready.store(true, std::memory_order_release);
+    result = Wait64(&word, 0, &timeout);
+  });
+  while (!ready.load(std::memory_order_acquire)) {
+    std::this_thread::yield();
+  }
+  Store(&word, UINT64_C(0x100000000));
+  Check(Wake(&word, 1) == OK, "wake-one succeeds");
+  waiter.join();
+  Check(result == OK, "a value change plus wake releases the waiter");
+}
+
+void TestWakeOneThenAll() {
+  constexpr int WAITER_COUNT = 4;
+  uint32_t word = 0;
+  uint32_t timeout = 1000000;
+  std::atomic<int> ready{0};
+  std::atomic<int> returned{0};
+  int results[WAITER_COUNT] = {};
+  std::vector<std::thread> waiters;
+
+  for (int i = 0; i < WAITER_COUNT; i++) {
+    waiters.emplace_back([&, i] {
+      ready.fetch_add(1, std::memory_order_release);
+      results[i] = Wait32(&word, 0, &timeout);
+      returned.fetch_add(1, std::memory_order_release);
+    });
+  }
+  while (ready.load(std::memory_order_acquire) != WAITER_COUNT) {
+    std::this_thread::yield();
+  }
+  std::this_thread::sleep_for(std::chrono::milliseconds(50));
+  Check(Wake(&word, 1) == OK, "wake-one succeeds with multiple waiters");
+  const auto one_deadline =
+      std::chrono::steady_clock::now() + std::chrono::milliseconds(500);
+  while (returned.load(std::memory_order_acquire) == 0 &&
+         std::chrono::steady_clock::now() < one_deadline) {
+    std::this_thread::yield();
+  }
+  Check(returned.load(std::memory_order_acquire) == 1,
+        "wake-one releases exactly one waiter");
+
+  Check(Wake(&word, 2) == OK, "wake-two succeeds");
+  const auto two_deadline =
+      std::chrono::steady_clock::now() + std::chrono::milliseconds(500);
+  while (returned.load(std::memory_order_acquire) < 3 &&
+         std::chrono::steady_clock::now() < two_deadline) {
+    std::this_thread::yield();
+  }
+  Check(returned.load(std::memory_order_acquire) == 3,
+        "wake-two releases exactly two more waiters");
+
+  Check(Wake(&word, INT_MAX) == OK, "wake-all succeeds");
+  for (auto &waiter : waiters) {
+    waiter.join();
+  }
+  Check(returned.load(std::memory_order_acquire) == WAITER_COUNT,
+        "wake-all releases the remaining waiters");
+  for (int result : results) {
+    Check(result == OK, "explicitly woken waiters return success");
+  }
+}
+
+void TestAddressesAreIsolated() {
+  uint32_t first = 0;
+  uint32_t second = 0;
+  uint32_t first_timeout = 1000000;
+  uint32_t second_timeout = 1000000;
+  std::atomic<int> ready{0};
+  std::atomic<bool> first_returned{false};
+  std::atomic<bool> second_returned{false};
+  int first_result = 0;
+  int second_result = 0;
+
+  std::thread first_waiter([&] {
+    ready.fetch_add(1, std::memory_order_release);
+    first_result = Wait32(&first, 0, &first_timeout);
+    first_returned.store(true, std::memory_order_release);
+  });
+  std::thread second_waiter([&] {
+    ready.fetch_add(1, std::memory_order_release);
+    second_result = Wait32(&second, 0, &second_timeout);
+    second_returned.store(true, std::memory_order_release);
+  });
+  while (ready.load(std::memory_order_acquire) != 2) {
+    std::this_thread::yield();
+  }
+  std::this_thread::sleep_for(std::chrono::milliseconds(50));
+  Check(Wake(&first, 1) == OK, "first address wakes");
+  std::this_thread::sleep_for(std::chrono::milliseconds(50));
+  Check(first_returned.load(std::memory_order_acquire),
+        "first address waiter returned");
+  Check(!second_returned.load(std::memory_order_acquire),
+        "waking one address does not release another address");
+  Check(Wake(&second, 1) == OK, "second address wakes");
+
+  first_waiter.join();
+  second_waiter.join();
+  Check(first_result == OK && second_result == OK,
+        "isolated waiters return success");
+}
+
+void TestCompareRegisterWakeRace() {
+  for (int i = 0; i < 100; i++) {
+    uint32_t word = 0;
+    uint32_t timeout = 500000;
+    std::atomic<bool> ready{false};
+    int result = Libs::LibKernel::KERNEL_ERROR_ETIMEDOUT;
+    std::thread waiter([&] {
+      ready.store(true, std::memory_order_release);
+      result = Wait32(&word, 0, &timeout);
+    });
+    while (!ready.load(std::memory_order_acquire)) {
+      std::this_thread::yield();
+    }
+    Store(&word, uint32_t{1});
+    (void)Wake(&word, 1);
+    waiter.join();
+    Check(result == OK, "compare/register/wake race never loses progress");
+  }
+}
+
+void TestWakeZeroIsNoOp() {
+  constexpr int WAITER_COUNT = 2;
+  uint32_t word = 0;
+  uint32_t timeout = 1000000;
+  std::atomic<int> ready{0};
+  std::atomic<int> returned{0};
+  int results[WAITER_COUNT] = {};
+  std::vector<std::thread> waiters;
+
+  for (int i = 0; i < WAITER_COUNT; i++) {
+    waiters.emplace_back([&, i] {
+      ready.fetch_add(1, std::memory_order_release);
+      results[i] = Wait32(&word, 0, &timeout);
+      returned.fetch_add(1, std::memory_order_release);
+    });
+  }
+  while (ready.load(std::memory_order_acquire) != WAITER_COUNT) {
+    std::this_thread::yield();
+  }
+  std::this_thread::sleep_for(std::chrono::milliseconds(50));
+  Check(Wake(&word, 0) == OK, "zero-count wake succeeds");
+  std::this_thread::sleep_for(std::chrono::milliseconds(20));
+  Check(returned.load(std::memory_order_acquire) == 0,
+        "zero-count wake releases no waiters");
+  Check(Wake(&word, INT_MAX) == OK, "wake-all succeeds after zero-count wake");
+  for (auto &waiter : waiters) {
+    waiter.join();
+  }
+  for (int result : results) {
+    Check(result == OK, "wake-all releases waiters after zero-count no-op");
+  }
+}
+
+} // namespace
+
+int main() {
+  TestInvalidAddress();
+  TestMismatchReturnsImmediately();
+  TestTimeout();
+  TestValueChangeAndWake();
+  TestWakeOneThenAll();
+  TestAddressesAreIsolated();
+  TestCompareRegisterWakeRace();
+  TestWakeZeroIsNoOp();
+  std::printf("SyncOnAddressTests: all passed\n");
+  return 0;
+}