Rewrite RenderDoc capture flow and host exceptions

src/common/hostException.cpp

@@ -2,7 +2,6 @@
 
 #include <atomic>
 #include <cstdio>
-#include <cstdlib>
 
 #if KYTY_PLATFORM == KYTY_PLATFORM_WINDOWS
 #include <windows.h> // IWYU pragma: keep
@@ -28,54 +27,14 @@ namespace Common::HostException {
 
 static std::atomic<Handler> g_handler {nullptr};
 static std::atomic_uint32_t g_install_state {0};
-static thread_local bool    g_in_exception_filter = false;
 
 static_assert(decltype(g_handler)::is_always_lock_free);
 static_assert(decltype(g_install_state)::is_always_lock_free);
-
-[[noreturn]] static void FailFast(const char* reason) noexcept {
-	std::fputs("HostException fail-fast: ", stderr);
-	std::fputs(reason != nullptr ? reason : "unspecified", stderr);
-	std::fputc('\n', stderr);
-	std::fflush(stderr);
-#if KYTY_PLATFORM == KYTY_PLATFORM_WINDOWS
-	TerminateProcess(GetCurrentProcess(), static_cast<UINT>(EXCEPTION_NONCONTINUABLE_EXCEPTION));
-#endif
-	std::_Exit(321);
-}
-
-class FilterScope final {
-public:
-	FilterScope() noexcept {
-		if (g_in_exception_filter) {
-			FailFast("nested exception while resolving a host fault");
-		}
-		g_in_exception_filter = true;
-	}
-
-	~FilterScope() { g_in_exception_filter = false; }
-
-	KYTY_CLASS_NO_COPY(FilterScope);
-};
-
-static Handler LoadInstalledHandler() noexcept {
-	if (g_install_state.load(std::memory_order_acquire) == 0) {
-		FailFast("host exception handler is not installed");
-	}
-
-	const auto handler = g_handler.load(std::memory_order_acquire);
-	if (handler == nullptr) {
-		FailFast("host exception callback is null");
-	}
-	return handler;
-}
 #endif
 
 #if KYTY_PLATFORM == KYTY_PLATFORM_WINDOWS
 
-static LONG WINAPI ExceptionFilter(PEXCEPTION_POINTERS exception) {
-	FilterScope filter_scope;
-
+static LONG WINAPI ExceptionFilter(PEXCEPTION_POINTERS exception) noexcept {
 	auto* exception_record = exception->ExceptionRecord;
 
 	if (exception_record->ExceptionCode == DBG_PRINTEXCEPTION_C ||
@@ -105,12 +64,6 @@ static LONG WINAPI ExceptionFilter(PEXCEPTION_POINTERS exception) {
 	} else if (exception_record->ExceptionCode == EXCEPTION_ILLEGAL_INSTRUCTION) {
 		info.type = ExceptionType::IllegalInstruction;
 	} else {
-		printf("Unhandled win exception: code=0x%08" PRIx32 ", addr=0x%016" PRIx64
-		       ", rip=0x%016" PRIx64 ", rsp=0x%016" PRIx64 ", rbp=0x%016" PRIx64 "\n",
-		       static_cast<uint32_t>(exception_record->ExceptionCode),
-		       reinterpret_cast<uint64_t>(exception_record->ExceptionAddress),
-		       exception->ContextRecord->Rip, exception->ContextRecord->Rsp,
-		       exception->ContextRecord->Rbp);
 		return EXCEPTION_CONTINUE_SEARCH;
 	}
 
@@ -131,28 +84,21 @@ static LONG WINAPI ExceptionFilter(PEXCEPTION_POINTERS exception) {
 	info.r14 = exception->ContextRecord->R14;
 	info.r15 = exception->ContextRecord->R15;
 
-	const auto handler = LoadInstalledHandler();
-
-	return handler(info) ? EXCEPTION_CONTINUE_EXECUTION : EXCEPTION_CONTINUE_SEARCH;
+	const auto handler = g_handler.load(std::memory_order_acquire);
+	if (handler != nullptr && handler(info)) {
+		return EXCEPTION_CONTINUE_EXECUTION;
+	}
+	return EXCEPTION_CONTINUE_SEARCH;
 }
 
 #elif defined(__APPLE__)
 
 static std::atomic<Handler> g_handler {nullptr};
 static std::atomic_uint32_t g_install_state {0};
-static thread_local bool    g_in_exception_filter = false;
 
 static_assert(decltype(g_handler)::is_always_lock_free);
 static_assert(decltype(g_install_state)::is_always_lock_free);
 
-[[noreturn]] static void FailFast(const char* reason) noexcept {
-	std::fputs("HostException fail-fast: ", stderr);
-	std::fputs(reason != nullptr ? reason : "unspecified", stderr);
-	std::fputc('\n', stderr);
-	std::fflush(stderr);
-	std::_Exit(321);
-}
-
 // Translate the x86-64 page-fault error code (mcontext __es.__err) into an access type.
 // bit 1 (0x2) = write, bit 4 (0x10) = instruction fetch, otherwise a read.
 static AccessViolationType DecodeAccess(uint64_t err) {
@@ -170,11 +116,6 @@ static AccessViolationType DecodeAccess(uint64_t err) {
 // re-executing the faulting instruction against the now-fixed protection. An unresolved
 // fault restores the default disposition so the retry terminates the process.
 static void SignalHandler(int sig, siginfo_t* si, void* uctx) {
-	if (g_in_exception_filter) {
-		FailFast("nested exception while resolving a host fault");
-	}
-	g_in_exception_filter = true;
-
 	auto*       uc = static_cast<ucontext_t*>(uctx);
 	const auto* mc = uc->uc_mcontext;
 	const auto& ss = mc->__ss;
@@ -210,14 +151,7 @@ static void SignalHandler(int sig, siginfo_t* si, void* uctx) {
 	info.r15 = ss.__r15;
 
 	const auto handler = g_handler.load(std::memory_order_acquire);
-	if (handler == nullptr) {
-		FailFast("host exception callback is null");
-	}
-
-	const bool resolved   = handler(info);
-	g_in_exception_filter = false;
-
-	if (resolved) {
+	if (handler != nullptr && handler(info)) {
 		return; // retry the faulting instruction against the fixed mapping
 	}
 
@@ -244,8 +178,6 @@ static void ChainToDefault(int signal_number) noexcept {
 }
 
 static void SignalHandler(int signal_number, siginfo_t* signal_info, void* native_context) {
-	FilterScope filter_scope;
-
 	auto* context = static_cast<ucontext_t*>(native_context);
 	auto* gregs   = context->uc_mcontext.gregs;
 
@@ -289,9 +221,8 @@ static void SignalHandler(int signal_number, siginfo_t* signal_info, void* nativ
 	info.r14 = static_cast<uint64_t>(gregs[REG_R14]);
 	info.r15 = static_cast<uint64_t>(gregs[REG_R15]);
 
-	const auto handler = LoadInstalledHandler();
-
-	if (handler(info)) {
+	const auto handler = g_handler.load(std::memory_order_acquire);
+	if (handler != nullptr && handler(info)) {
 		return;
 	}
 
@@ -313,7 +244,7 @@ bool InstallHandler(Handler handler) {
 	g_handler.store(handler, std::memory_order_release);
 
 #if KYTY_PLATFORM == KYTY_PLATFORM_WINDOWS
-	if (AddVectoredExceptionHandler(1, ExceptionFilter) == nullptr) {
+	if (AddVectoredExceptionHandler(0, ExceptionFilter) == nullptr) {
 		g_handler.store(nullptr, std::memory_order_release);
 		g_install_state.store(0, std::memory_order_release);
 		printf("AddVectoredExceptionHandler() failed\n");

src/graphics/guest_gpu/graphicsRun.cpp

@@ -13,6 +13,7 @@
 #include "graphics/host_gpu/renderer/render.h"
 #include "graphics/host_gpu/renderer/renderContext.h"
 #include "graphics/host_gpu/renderer/sync.h"
+#include "graphics/presentation/renderDoc.h"
 #include "graphics/presentation/videoOut.h"
 #include "graphics/presentation/window.h"
 #include "graphics/shader/shader.h"
@@ -278,6 +279,12 @@ void GpuState::Done() {
 	if (!IsGpuThread()) {
 		WaitForIdle();
 	}
+	if (RenderDocCaptureInProgress()) {
+		SendCommandSync([this] {
+			Common::LockGuard render_lock(m_renderer.GetMutex());
+			RenderDocEndCapture();
+		});
+	}
 	m_graphics_done = true;
 	m_done_num++;
 }
@@ -638,8 +645,12 @@ void GpuState::ThreadRun(void* data) {
 }
 
 bool GpuState::Process(Submission& submission) {
-	auto&      cp          = GetProcessor(submission.queue_id);
 	const bool first_slice = !submission.started;
+	if (first_slice && RenderDocCaptureRequested()) {
+		Common::LockGuard render_lock(m_renderer.GetMutex());
+		RenderDocStartCapture();
+	}
+	auto& cp = GetProcessor(submission.queue_id);
 
 	if (first_slice && submission.reset_processor) {
 		cp.Reset();

src/graphics/host_gpu/vulkanCommon.h

@@ -22,11 +22,6 @@ namespace Libs::Graphics {
 
 using VulkanMemoryBarrier = vk::MemoryBarrier;
 
-template <typename Handle>
-[[nodiscard]] void* VulkanHandleToPointer(Handle handle) {
-	return reinterpret_cast<void*>(static_cast<typename Handle::CType>(handle));
-}
-
 std::string VulkanToString(vk::Result value);
 std::string VulkanToString(vk::Format value);
 std::string VulkanToString(vk::ImageLayout value);

src/graphics/presentation/renderDoc.cpp

@@ -1,15 +1,12 @@
 #include "graphics/presentation/renderDoc.h"
 
-#include "SDL_syswm.h"
-#include "SDL_version.h"
-#include "SDL_video.h"
 #include "common/logging/log.h"
 
 #include <array>
 #include <atomic>
-#include <cstdint>
-#include <cstring>
+#include <chrono>
 #include <filesystem>
+#include <renderdoc_app.h>
 #include <string>
 
 #if KYTY_PLATFORM == KYTY_PLATFORM_WINDOWS
@@ -21,362 +18,161 @@
 #undef max
 #else
 #include <dlfcn.h>
-
-// RenderDoc uses Windows-style names in its cross-platform API.
-#define __cdecl
-using HMODULE = void*;
 #endif
 
 namespace Libs::Graphics {
 
-using RenderDocDevicePointer = void*;
-using RenderDocWindowHandle  = void*;
-
-enum RenderDocVersion {
-	eRENDERDOC_API_Version_1_4_2 = 10402,
-};
-
-enum RenderDocInputButton {
-	eRENDERDOC_Key_NonPrintable = 0x100,
-	eRENDERDOC_Key_Divide,
-	eRENDERDOC_Key_Multiply,
-	eRENDERDOC_Key_Subtract,
-	eRENDERDOC_Key_Plus,
-	eRENDERDOC_Key_F1,
-};
-
-using pRENDERDOC_SetCaptureKeys             = void(__cdecl*)(RenderDocInputButton* keys, int num);
-using pRENDERDOC_SetCaptureFilePathTemplate = void(__cdecl*)(const char* pathtemplate);
-using pRENDERDOC_GetCaptureFilePathTemplate = const char*(__cdecl*)();
-using pRENDERDOC_GetNumCaptures             = uint32_t(__cdecl*)();
-using pRENDERDOC_GetCapture = uint32_t(__cdecl*)(uint32_t idx, char* filename, uint32_t* pathlength,
-                                                 uint64_t* timestamp);
-using pRENDERDOC_UnloadCrashHandler = void(__cdecl*)();
-using pRENDERDOC_SetActiveWindow    = void(__cdecl*)(RenderDocDevicePointer device,
-                                                     RenderDocWindowHandle  wndHandle);
-using pRENDERDOC_StartFrameCapture  = void(__cdecl*)(RenderDocDevicePointer device,
-                                                     RenderDocWindowHandle  wndHandle);
-using pRENDERDOC_IsFrameCapturing   = uint32_t(__cdecl*)();
-using pRENDERDOC_EndFrameCapture    = uint32_t(__cdecl*)(RenderDocDevicePointer device,
-                                                         RenderDocWindowHandle  wndHandle);
-using pRENDERDOC_GetAPI = int(__cdecl*)(RenderDocVersion version, void** out_api_pointers);
-
-struct RenderDocApi {
-	void*                                 GetAPIVersion;
-	void*                                 SetCaptureOptionU32;
-	void*                                 SetCaptureOptionF32;
-	void*                                 GetCaptureOptionU32;
-	void*                                 GetCaptureOptionF32;
-	void*                                 SetFocusToggleKeys;
-	pRENDERDOC_SetCaptureKeys             SetCaptureKeys;
-	void*                                 GetOverlayBits;
-	void*                                 MaskOverlayBits;
-	void*                                 RemoveHooks;
-	pRENDERDOC_UnloadCrashHandler         UnloadCrashHandler;
-	pRENDERDOC_SetCaptureFilePathTemplate SetCaptureFilePathTemplate;
-	pRENDERDOC_GetCaptureFilePathTemplate GetCaptureFilePathTemplate;
-	pRENDERDOC_GetNumCaptures             GetNumCaptures;
-	pRENDERDOC_GetCapture                 GetCapture;
-	void*                                 TriggerCapture;
-	void*                                 IsTargetControlConnected;
-	void*                                 LaunchReplayUI;
-	pRENDERDOC_SetActiveWindow            SetActiveWindow;
-	pRENDERDOC_StartFrameCapture          StartFrameCapture;
-	pRENDERDOC_IsFrameCapturing           IsFrameCapturing;
-	pRENDERDOC_EndFrameCapture            EndFrameCapture;
-	void*                                 TriggerMultiFrameCapture;
-	void*                                 SetCaptureFileComments;
-	void*                                 DiscardFrameCapture;
-};
-
 enum class RenderDocState : uint32_t {
 	Idle,
 	Requested,
 	Capturing,
 };
 
-static RenderDocApi*               g_api             = nullptr;
-static HMODULE                     g_module          = nullptr;
-static RenderDocDevicePointer      g_device          = nullptr;
-static RenderDocWindowHandle       g_window          = nullptr;
+static RENDERDOC_API_1_6_0*        g_api             = nullptr;
 static std::atomic<RenderDocState> g_state           = RenderDocState::Idle;
-static std::atomic_bool            g_init_done       = false;
 static std::atomic_bool            g_unavailable_log = false;
 
-static RenderDocDevicePointer GetRenderDocDevicePointer(vk::Instance instance) {
-	if (instance == nullptr) {
-		return nullptr;
-	}
-
-	return VulkanHandleToPointer(instance);
-}
-
 #if KYTY_PLATFORM == KYTY_PLATFORM_WINDOWS
 
 static bool BindRenderDocApi(HMODULE module) {
 	auto* get_api = reinterpret_cast<pRENDERDOC_GetAPI>(GetProcAddress(module, "RENDERDOC_GetAPI"));
-	if (get_api == nullptr) {
-		return false;
-	}
-
-	void* api = nullptr;
-	if (get_api(eRENDERDOC_API_Version_1_4_2, &api) == 0 || api == nullptr) {
-		return false;
-	}
-
-	g_module = module;
-	g_api    = static_cast<RenderDocApi*>(api);
-
-	g_api->SetCaptureFilePathTemplate("_RenderDoc/kyty");
-	g_api->SetCaptureKeys(nullptr, 0);
-	g_api->UnloadCrashHandler();
-
-	char module_path[MAX_PATH] = {};
-	GetModuleFileNameA(module, module_path, sizeof(module_path));
-	LOGF("RenderDoc: bound API from %s\n", module_path);
-	return true;
-}
-
-static RenderDocWindowHandle GetRenderDocWindowHandle(SDL_Window* window) {
-	if (window == nullptr) {
-		return nullptr;
-	}
-
-	SDL_SysWMinfo info {};
-	SDL_VERSION(&info.version);
-
-	if (SDL_GetWindowWMInfo(window, &info) != SDL_TRUE || info.subsystem != SDL_SYSWM_WINDOWS) {
-		return nullptr;
-	}
-
-	return info.info.win.window;
-}
-
 #else
-
-static bool BindRenderDocApi(HMODULE module) {
+static bool BindRenderDocApi(void* module) {
 	auto* get_api = reinterpret_cast<pRENDERDOC_GetAPI>(::dlsym(module, "RENDERDOC_GetAPI"));
+#endif
 	if (get_api == nullptr) {
 		return false;
 	}
 
 	void* api = nullptr;
-	if (get_api(eRENDERDOC_API_Version_1_4_2, &api) == 0 || api == nullptr) {
+	if (get_api(eRENDERDOC_API_Version_1_6_0, &api) != 1 || api == nullptr) {
 		return false;
 	}
 
-	g_module = module;
-	g_api    = static_cast<RenderDocApi*>(api);
-
-	g_api->SetCaptureFilePathTemplate("_RenderDoc/kyty");
+	g_api = static_cast<RENDERDOC_API_1_6_0*>(api);
 	g_api->SetCaptureKeys(nullptr, 0);
 	g_api->UnloadCrashHandler();
-
-	Dl_info info {};
-	if (::dladdr(reinterpret_cast<void*>(get_api), &info) != 0 && info.dli_fname != nullptr) {
-		LOGF("RenderDoc: bound API from %s\n", info.dli_fname);
-	} else {
-		LOGF("RenderDoc: bound API\n");
-	}
+	LOGF("RenderDoc: API 1.6.0 bound\n");
 	return true;
 }
 
-static RenderDocWindowHandle GetRenderDocWindowHandle(SDL_Window* window) {
-	if (window == nullptr) {
-		return nullptr;
-	}
-
-	SDL_SysWMinfo info {};
-	SDL_VERSION(&info.version);
-
-	if (SDL_GetWindowWMInfo(window, &info) != SDL_TRUE) {
-		return nullptr;
-	}
-
-#if defined(SDL_VIDEO_DRIVER_X11)
-	if (info.subsystem == SDL_SYSWM_X11) {
-		// RenderDoc takes the raw xlib Window id in the pointer slot, not a Display*.
-		return reinterpret_cast<RenderDocWindowHandle>(
-		    static_cast<uintptr_t>(info.info.x11.window));
-	}
-#endif
-
-	// Wayland capture works without an active-window handle.
-	static std::atomic_bool logged = false;
-	if (!logged.exchange(true)) {
-		LOGF("RenderDoc: no native window handle for SDL subsystem %d (Wayland?); the in-app "
-		     "overlay is unavailable, but --rd captures still work\n",
-		     static_cast<int>(info.subsystem));
-	}
-	return nullptr;
-}
-
-#endif
-
-#if KYTY_PLATFORM == KYTY_PLATFORM_WINDOWS
-
-static bool IsAvailable() {
-	return g_api != nullptr && g_device != nullptr && g_window != nullptr;
-}
-
-#else
-
-static bool IsAvailable() {
-	return g_api != nullptr && g_device != nullptr;
-}
-
-#endif
-
 #if KYTY_PLATFORM == KYTY_PLATFORM_WINDOWS
 
 void RenderDocInit() {
-	bool expected = false;
-	if (!g_init_done.compare_exchange_strong(expected, true)) {
+	if (g_api != nullptr) {
 		return;
 	}
 
-	HKEY h_reg_key;
-	LONG result = RegOpenKeyExW(HKEY_LOCAL_MACHINE,
-	                            L"SOFTWARE\\Classes\\RenderDoc.RDCCapture.1\\DefaultIcon\\", 0,
-	                            KEY_READ, &h_reg_key);
-	if (result != ERROR_SUCCESS) {
-		return;
-	}
-	std::array<wchar_t, MAX_PATH> key_str {};
-	DWORD                         str_sz_out {key_str.size()};
-	result = RegQueryValueExW(h_reg_key, L"", 0, NULL, (LPBYTE)key_str.data(), &str_sz_out);
-	RegCloseKey(h_reg_key);
-	if (result != ERROR_SUCCESS) {
-		return;
-	}
-
-	std::filesystem::path path {key_str.cbegin(), key_str.cend()};
-	path                   = path.parent_path().append("renderdoc.dll");
-	const auto path_to_lib = path.generic_string();
-	auto*      module      = LoadLibraryA(path_to_lib.c_str());
+	auto* module = GetModuleHandleA("renderdoc.dll");
 	if (module == nullptr) {
-		return;
+		HKEY key = nullptr;
+		if (RegOpenKeyExW(HKEY_LOCAL_MACHINE,
+		                  L"SOFTWARE\\Classes\\RenderDoc.RDCCapture.1\\DefaultIcon\\", 0, KEY_READ,
+		                  &key) != ERROR_SUCCESS) {
+			return;
+		}
+
+		std::array<wchar_t, MAX_PATH> path_buffer {};
+		DWORD      path_size = static_cast<DWORD>(path_buffer.size() * sizeof(wchar_t));
+		const auto result    = RegQueryValueExW(
+		    key, L"", nullptr, nullptr, reinterpret_cast<LPBYTE>(path_buffer.data()), &path_size);
+		RegCloseKey(key);
+		if (result != ERROR_SUCCESS) {
+			return;
+		}
+
+		auto path = std::filesystem::path(path_buffer.data()).parent_path() / "renderdoc.dll";
+		module    = LoadLibraryW(path.c_str());
+		if (module == nullptr) {
+			return;
+		}
 	}
 
 	if (!BindRenderDocApi(module)) {
-		LOGF("RenderDoc: API 1.4.2 is not available; in-app capture disabled\n");
-		FreeLibrary(module);
-		return;
+		LOGF("RenderDoc: API 1.6.0 is unavailable\n");
 	}
 }
 
 #else
 
 void RenderDocInit() {
-	bool expected = false;
-	if (!g_init_done.compare_exchange_strong(expected, true)) {
+	if (g_api != nullptr) {
 		return;
 	}
 
-	// Prefer an injected RenderDoc instance.
 	auto* module = ::dlopen("librenderdoc.so", RTLD_NOW | RTLD_NOLOAD);
 	if (module == nullptr) {
 		module = ::dlopen("librenderdoc.so", RTLD_NOW);
 	}
 	if (module == nullptr) {
-		LOGF("RenderDoc: librenderdoc.so was not found; in-app capture disabled\n");
 		return;
 	}
 
 	if (!BindRenderDocApi(module)) {
-		LOGF("RenderDoc: API 1.4.2 is not available; in-app capture disabled\n");
+		LOGF("RenderDoc: API 1.6.0 is unavailable\n");
 		::dlclose(module);
-		return;
 	}
 }
 
 #endif
 
-void RenderDocSetActiveWindow(vk::Instance instance, SDL_Window* window) {
-	if (g_api == nullptr) {
-		return;
-	}
-
-	g_device = GetRenderDocDevicePointer(instance);
-	g_window = GetRenderDocWindowHandle(window);
-
-	if (g_device == nullptr || g_window == nullptr) {
-		LOGF("RenderDoc: active Vulkan window was not registered\n");
-		return;
-	}
-
-	g_api->SetActiveWindow(g_device, g_window);
-	LOGF("RenderDoc: active Vulkan window registered\n");
-}
-
 void RenderDocRequestCapture() {
-	if (!IsAvailable()) {
+	if (g_api == nullptr) {
 		if (!g_unavailable_log.exchange(true)) {
-			LOGF("RenderDoc: capture requested, but RenderDoc is not available\n");
+			LOGF("RenderDoc: capture requested, but RenderDoc is unavailable\n");
 		}
 		return;
 	}
 
 	RenderDocState expected = RenderDocState::Idle;
 	if (g_state.compare_exchange_strong(expected, RenderDocState::Requested)) {
-		LOGF("RenderDoc: capture requested; next complete presented frame will be captured\n");
-	} else {
-		LOGF("RenderDoc: capture request ignored because a capture is already pending\n");
+		LOGF("RenderDoc: capture requested\n");
 	}
 }
 
-static void LogNewestCapture() {
-	const auto count = g_api->GetNumCaptures();
-	if (count == 0) {
-		return;
-	}
-
-	char     filename[4096] = {};
-	uint32_t path_length    = sizeof(filename);
-	uint64_t timestamp      = 0;
-
-	if (g_api->GetCapture(count - 1, filename, &path_length, &timestamp) != 0) {
-		filename[sizeof(filename) - 1] = '\0';
-		LOGF("RenderDoc: wrote capture %s\n", filename);
-	}
+bool RenderDocCaptureRequested() {
+	return g_state.load(std::memory_order_acquire) == RenderDocState::Requested;
 }
 
-void RenderDocOnPresent() {
-	if (!IsAvailable()) {
+bool RenderDocCaptureInProgress() {
+	return g_state.load(std::memory_order_acquire) == RenderDocState::Capturing;
+}
+
+void RenderDocStartCapture() {
+	RenderDocState expected = RenderDocState::Requested;
+	if (g_api == nullptr || !g_state.compare_exchange_strong(expected, RenderDocState::Capturing,
+	                                                         std::memory_order_acq_rel)) {
 		return;
 	}
 
-	switch (g_state.load()) {
-		case RenderDocState::Idle: return;
-		case RenderDocState::Requested:
-			if (g_api->IsFrameCapturing() != 0) {
-				LOGF("RenderDoc: capture request ignored because RenderDoc is already capturing\n");
-				g_state.store(RenderDocState::Idle);
-				return;
-			}
+	if (g_api->IsFrameCapturing() != 0) {
+		g_state.store(RenderDocState::Idle, std::memory_order_release);
+		LOGF("RenderDoc: capture request ignored because a capture is already active\n");
+		return;
+	}
 
-			g_api->StartFrameCapture(nullptr, nullptr);
-			if (g_api->IsFrameCapturing() == 0) {
-				LOGF("RenderDoc: StartFrameCapture returned, but RenderDoc is not capturing\n");
-				g_state.store(RenderDocState::Idle);
-				return;
-			}
-			g_state.store(RenderDocState::Capturing);
-			LOGF("RenderDoc: capture started\n");
-			return;
-		case RenderDocState::Capturing: break;
+	const auto capture_id   = std::chrono::duration_cast<std::chrono::microseconds>(
+	                              std::chrono::system_clock::now().time_since_epoch())
+	                              .count();
+	const auto capture_path = "_RenderDoc/kyty_" + std::to_string(capture_id);
+	g_api->SetCaptureFilePathTemplate(capture_path.c_str());
+	g_api->StartFrameCapture(nullptr, nullptr);
+	if (g_api->IsFrameCapturing() == 0) {
+		g_state.store(RenderDocState::Idle, std::memory_order_release);
+		LOGF("RenderDoc: capture failed to start\n");
+		return;
+	}
+	LOGF("RenderDoc: capture started\n");
+}
+
+void RenderDocEndCapture() {
+	if (g_api == nullptr || !RenderDocCaptureInProgress()) {
+		return;
 	}
 
 	const auto ok = g_api->EndFrameCapture(nullptr, nullptr);
-	g_state.store(RenderDocState::Idle);
-
-	if (ok != 0) {
-		const auto count = g_api->GetNumCaptures();
-		LOGF("RenderDoc: capture finished, count=%u\n", count);
-		LogNewestCapture();
-	} else {
-		LOGF("RenderDoc: capture failed\n");
-	}
+	g_state.store(RenderDocState::Idle, std::memory_order_release);
+	LOGF(ok != 0 ? "RenderDoc: capture finished\n" : "RenderDoc: capture failed\n");
 }
 
 } // namespace Libs::Graphics

src/graphics/presentation/renderDoc.h

@@ -1,17 +1,15 @@
 #ifndef EMULATOR_INCLUDE_EMULATOR_GRAPHICS_RENDERDOC_H_
 #define EMULATOR_INCLUDE_EMULATOR_GRAPHICS_RENDERDOC_H_
 
-#include "common/common.h"
-#include "graphics/host_gpu/vulkanCommon.h"
-
-struct SDL_Window;
-
 namespace Libs::Graphics {
 
 void RenderDocInit();
-void RenderDocSetActiveWindow(vk::Instance instance, SDL_Window* window);
 void RenderDocRequestCapture();
-void RenderDocOnPresent();
+void RenderDocStartCapture();
+void RenderDocEndCapture();
+
+[[nodiscard]] bool RenderDocCaptureRequested();
+[[nodiscard]] bool RenderDocCaptureInProgress();
 
 } // namespace Libs::Graphics
 

src/graphics/presentation/window/swapchain.cpp

@@ -32,7 +32,6 @@
 #include "graphics/host_gpu/vulkanCommon.h"
 #include "graphics/presentation/imeOverlay.h"
 #include "graphics/presentation/presenter.h"
-#include "graphics/presentation/renderDoc.h"
 #include "graphics/presentation/videoOut.h"
 #include "graphics/presentation/window/windowInternal.h"
 #include "libs/controller.h"
@@ -866,7 +865,6 @@ void Presenter::Present(Frame& frame, bool reuse) {
 			continue;
 		}
 
-		RenderDocOnPresent();
 		m_impl->presented_ime_revision.store(ime_visual.revision, std::memory_order_release);
 		window.UpdateTitle();
 		m_impl->frames.Release(&frame, true);

src/graphics/presentation/window/vulkanWindow.cpp

@@ -32,7 +32,6 @@
 #include "graphics/host_gpu/vulkanCommon.h"
 #include "graphics/presentation/imeOverlay.h"
 #include "graphics/presentation/presenter.h"
-#include "graphics/presentation/renderDoc.h"
 #include "graphics/presentation/videoOut.h"
 #include "graphics/presentation/window.h"
 #include "graphics/presentation/window/windowInternal.h"
@@ -601,11 +600,11 @@ static vk::Device VulkanCreateDevice(vk::PhysicalDevice physical_device, const V
 	     robustness2_ext_enabled && robustness2.robustImageAccess2 == VK_TRUE ? "true" : "false");
 
 	vk::DeviceCreateInfo create_info {};
-	create_info.sType                = vk::StructureType::eDeviceCreateInfo;
-	create_info.pNext                = &features13;
-	create_info.flags                = {};
-	create_info.pQueueCreateInfos    = &queue_create_info;
-	create_info.queueCreateInfoCount = 1;
+	create_info.sType                   = vk::StructureType::eDeviceCreateInfo;
+	create_info.pNext                   = &features13;
+	create_info.flags                   = {};
+	create_info.pQueueCreateInfos       = &queue_create_info;
+	create_info.queueCreateInfoCount    = 1;
 	create_info.enabledExtensionCount   = static_cast<uint32_t>(device_extensions.size());
 	create_info.ppEnabledExtensionNames = device_extensions.data();
 	create_info.pEnabledFeatures        = &device_features;
@@ -986,7 +985,6 @@ void WindowContext::CreateVulkan() {
 	render_context = std::make_unique<RenderContext>(graphic_ctx);
 	LibKernel::Memory::InstallGpuResources(&render_context->GetGpuResources());
 	presenter = std::make_unique<Presenter>(*this);
-	RenderDocSetActiveWindow(graphic_ctx.instance, window);
 }
 
 void WindowContext::RefreshSurfaceCapabilities() {

src/kernel/memory.cpp

@@ -3447,17 +3447,6 @@ bool TestGuestFreeRangeBounds() {
 }
 #endif
 
-bool KernelHandleReservedRangeAccessViolation(uint64_t vaddr) {
-	std::lock_guard<std::recursive_mutex> memory_operation_lock(g_memory_operation_mutex);
-
-	VirtualRanges::Range range {};
-	if (!g_virtual_ranges->Query(vaddr, 0, &range) ||
-	    std::strncmp(range.name, "AMM", KERNEL_MAXIMUM_NAME_LENGTH) != 0) {
-		return false;
-	}
-	EXIT("AMM virtual-memory unmap is unsupported: addr=0x%016" PRIx64 "\n", vaddr);
-}
-
 int KYTY_SYSV_ABI KernelVirtualQuery(const void* addr, int flags, VirtualQueryInfo* info,
                                      uint64_t info_size) {
 	PRINT_NAME();

src/kernel/memory.h

@@ -151,7 +151,6 @@ int KYTY_SYSV_ABI KernelVirtualQuery(const void* addr, int flags, VirtualQueryIn
                                      uint64_t info_size);
 int KYTY_SYSV_ABI KernelIsStack(void* addr, void** start, void** end);
 int KYTY_SYSV_ABI KernelReserveVirtualRange(void** addr, size_t len, int flags, size_t alignment);
-bool              KernelHandleReservedRangeAccessViolation(uint64_t vaddr);
 int KYTY_SYSV_ABI KernelAvailableFlexibleMemorySize(size_t* size);
 int KYTY_SYSV_ABI KernelConfiguredFlexibleMemorySize(size_t* size);
 int KYTY_SYSV_ABI KernelMprotect(const void* addr, size_t len, int prot);

src/loader/runtimeLinker.cpp

@@ -38,7 +38,6 @@
 #endif
 #include <windows.h>
 #else
-#include <dlfcn.h>
 #if defined(__APPLE__)
 #include <mach/mach.h>
 #include <mach/mach_vm.h>
@@ -780,15 +779,6 @@ static bool IsReadableRange(uint64_t addr, uint64_t size) {
 	return true;
 }
 
-static bool IsDumpableRange(uint64_t addr, uint64_t size) {
-#if KYTY_PLATFORM == KYTY_PLATFORM_LINUX
-	return IsReadableRange(addr, size);
-#else
-	(void)size;
-	return addr != 0;
-#endif
-}
-
 static bool KytyExceptionHandler(const Common::HostException::ExceptionInfo& exception_info) {
 	const auto* info = &exception_info;
 
@@ -798,217 +788,19 @@ static bool KytyExceptionHandler(const Common::HostException::ExceptionInfo& exc
 	}
 
 	if (info->type == Common::HostException::ExceptionType::AccessViolation) {
-		using CoreAccess  = Common::HostException::AccessViolationType;
-		using GpuAccess   = Libs::Graphics::PageFaultAccess;
-		const auto access = [&]() {
-			switch (info->access_violation_type) {
-				case CoreAccess::Read: return GpuAccess::Read;
-				case CoreAccess::Write: return GpuAccess::Write;
-				case CoreAccess::Execute: return GpuAccess::Execute;
-				case CoreAccess::Unknown:
-					EXIT("unknown access type for page fault at 0x%016" PRIx64 "\n",
-					     info->access_violation_vaddr);
-			}
-			EXIT("invalid access type for page fault at 0x%016" PRIx64 "\n",
-			     info->access_violation_vaddr);
-		}();
+		using CoreAccess = Common::HostException::AccessViolationType;
+		using GpuAccess  = Libs::Graphics::PageFaultAccess;
+		GpuAccess access;
+		switch (info->access_violation_type) {
+			case CoreAccess::Read: access = GpuAccess::Read; break;
+			case CoreAccess::Write: access = GpuAccess::Write; break;
+			case CoreAccess::Execute: access = GpuAccess::Execute; break;
+			case CoreAccess::Unknown: return false;
+		}
 		if (Libs::LibKernel::Memory::HandleGpuFault(access, info->access_violation_vaddr)) {
 			return true;
 		}
-
-		if (Libs::LibKernel::Memory::KernelHandleReservedRangeAccessViolation(
-		        info->access_violation_vaddr)) {
-			return true;
-		}
 	}
-
-	LOGF("kyty_exception_handler: %016" PRIx64 "\n", info->exception_address);
-#if KYTY_PLATFORM == KYTY_PLATFORM_WINDOWS
-	HMODULE owner_module = nullptr;
-	if (GetModuleHandleExA(GET_MODULE_HANDLE_EX_FLAG_FROM_ADDRESS |
-	                           GET_MODULE_HANDLE_EX_FLAG_UNCHANGED_REFCOUNT,
-	                       reinterpret_cast<LPCSTR>(info->exception_address), &owner_module) != 0 &&
-	    owner_module != nullptr) {
-		char module_name[MAX_PATH] = {};
-		if (GetModuleFileNameA(owner_module, module_name, MAX_PATH) != 0) {
-			LOGF("exception module: %s\n", module_name);
-		}
-	}
-#else
-	Dl_info module_info {};
-	if (::dladdr(reinterpret_cast<void*>(info->exception_address), &module_info) != 0 &&
-	    module_info.dli_fname != nullptr) {
-		LOGF("exception module: %s\n", module_info.dli_fname);
-	}
-#endif
-	if (info->exception_address != 0) {
-#if KYTY_PLATFORM == KYTY_PLATFORM_WINDOWS
-		MEMORY_BASIC_INFORMATION mem_info = {};
-		auto* dump_ptr = reinterpret_cast<const uint8_t*>(info->exception_address - 32);
-		if (VirtualQuery(dump_ptr, &mem_info, sizeof(mem_info)) != 0 &&
-		    mem_info.State == MEM_COMMIT && mem_info.Protect != PAGE_NOACCESS &&
-		    (mem_info.Protect & PAGE_GUARD) == 0) {
-			const auto dump_start = reinterpret_cast<uint64_t>(dump_ptr);
-			const auto region_end =
-			    reinterpret_cast<uint64_t>(mem_info.BaseAddress) + mem_info.RegionSize;
-			const auto dump_size =
-			    (dump_start + 64 <= region_end ? 64u
-			                                   : static_cast<uint32_t>(region_end - dump_start));
-			LOGF("code-32:");
-			for (uint32_t i = 0; i < dump_size; i++) {
-				LOGF(" %02" PRIx32, static_cast<uint32_t>(dump_ptr[i]));
-			}
-			LOGF("\n");
-		} else {
-			LOGF("code-32: unavailable\n");
-		}
-#else
-		const auto fault_addr = info->exception_address;
-		const auto dump_start = (fault_addr >= 32 ? fault_addr - 32 : fault_addr);
-		if (IsReadableRange(dump_start, 64)) {
-			auto* dump_ptr = reinterpret_cast<const uint8_t*>(dump_start);
-			LOGF("code-32:");
-			for (uint32_t i = 0; i < 64; i++) {
-				LOGF(" %02" PRIx32, static_cast<uint32_t>(dump_ptr[i]));
-			}
-			LOGF("\n");
-		} else {
-			LOGF("code-32: unavailable\n");
-		}
-#endif
-	} else {
-		LOGF("code: unavailable\n");
-	}
-	LOGF("exception: type=%s, av_type=%s, av_addr=%016" PRIx64 ", native_code=%08" PRIx32 "\n",
-	     Common::EnumName(info->type).c_str(),
-	     Common::EnumName(info->access_violation_type).c_str(), info->access_violation_vaddr,
-	     info->native_code);
-	LOGF("regs: rax=%016" PRIx64 " rbx=%016" PRIx64 " rcx=%016" PRIx64 " rdx=%016" PRIx64 "\n",
-	     info->rax, info->rbx, info->rcx, info->rdx);
-	LOGF("regs: rsi=%016" PRIx64 " rdi=%016" PRIx64 " rbp=%016" PRIx64 " rsp=%016" PRIx64 "\n",
-	     info->rsi, info->rdi, info->rbp, info->rsp);
-	LOGF("regs: r8 =%016" PRIx64 " r9 =%016" PRIx64 " r10=%016" PRIx64 " r11=%016" PRIx64 "\n",
-	     info->r8, info->r9, info->r10, info->r11);
-	LOGF("regs: r12=%016" PRIx64 " r13=%016" PRIx64 " r14=%016" PRIx64 " r15=%016" PRIx64 "\n",
-	     info->r12, info->r13, info->r14, info->r15);
-
-	if (IsReadableRange(info->rsp, 16u * sizeof(uint64_t))) {
-		auto* stack = reinterpret_cast<const uint64_t*>(info->rsp);
-		LOGF("stack:");
-		for (uint64_t i = 0; i < 16; i++) {
-			LOGF(" [%02" PRIu64 "]=%016" PRIx64, i, stack[i]);
-		}
-		LOGF("\n");
-	} else {
-		LOGF("stack: unavailable\n");
-	}
-
-	auto dump_guest_code = [](const char* name, uint64_t addr) {
-		auto* p = Common::Singleton<Loader::RuntimeLinker>::Instance()->FindProgramByAddr(addr);
-		if (p == nullptr || addr < p->base_vaddr) {
-			return;
-		}
-
-#if KYTY_PLATFORM == KYTY_PLATFORM_WINDOWS
-		MEMORY_BASIC_INFORMATION mbi {};
-		auto* dump_ptr = reinterpret_cast<const uint8_t*>(addr >= 16 ? addr - 16 : addr);
-		if (VirtualQuery(dump_ptr, &mbi, sizeof(mbi)) == 0 || mbi.State != MEM_COMMIT ||
-		    (mbi.Protect & (PAGE_NOACCESS | PAGE_GUARD)) != 0) {
-			return;
-		}
-		const auto dump_start = reinterpret_cast<uint64_t>(dump_ptr);
-		const auto region_end = reinterpret_cast<uint64_t>(mbi.BaseAddress) + mbi.RegionSize;
-		const auto dump_size =
-		    (dump_start + 32 <= region_end ? 32u : static_cast<uint32_t>(region_end - dump_start));
-#else
-		auto*      dump_ptr  = reinterpret_cast<const uint8_t*>(addr >= 16 ? addr - 16 : addr);
-		const auto dump_size = 32u;
-		if (!IsReadableRange(reinterpret_cast<uint64_t>(dump_ptr), dump_size)) {
-			return;
-		}
-#endif
-
-		LOGF("%s code: addr=%016" PRIx64 ", off=%016" PRIx64 ", module=%s:", name, addr,
-		     addr - p->base_vaddr,
-		     Common::FilenameWithoutDirectory(Common::PathToGenericString(p->file_name)).c_str());
-		for (uint32_t i = 0; i < dump_size; i++) {
-			LOGF(" %02" PRIx32, static_cast<uint32_t>(dump_ptr[i]));
-		}
-		LOGF("\n");
-	};
-
-	dump_guest_code("guest rax[0]", info->rax);
-	dump_guest_code("guest rbx[0]", info->rbx);
-	dump_guest_code("guest rcx[0]", info->rcx);
-	dump_guest_code("guest rsi[0]", info->rsi);
-	if (IsDumpableRange(info->rsp, 16u * sizeof(uint64_t))) {
-		auto* stack = reinterpret_cast<const uint64_t*>(info->rsp);
-		for (uint64_t i = 0; i < 16; i++) {
-			char name[32] {};
-			std::snprintf(name, sizeof(name), "stack[%" PRIu64 "]", i);
-			dump_guest_code(name, stack[i]);
-		}
-	}
-
-	if (info->type == Common::HostException::ExceptionType::AccessViolation) {
-		if (info->rbp != 0) {
-			Common::Singleton<Loader::RuntimeLinker>::Instance()->StackTrace(info->rbp, info->rsp);
-		}
-
-		auto dump_guest_qwords = [](const char* name, uint64_t addr) {
-			if (addr == 0) {
-				LOGF("%s = 0\n", name);
-				return;
-			}
-
-#if KYTY_PLATFORM == KYTY_PLATFORM_WINDOWS
-			MEMORY_BASIC_INFORMATION mbi {};
-			if (VirtualQuery(reinterpret_cast<const void*>(addr), &mbi, sizeof(mbi)) == 0 ||
-			    mbi.State != MEM_COMMIT || (mbi.Protect & (PAGE_NOACCESS | PAGE_GUARD)) != 0) {
-				LOGF("%s = %016" PRIx64 " (unmapped)\n", name, addr);
-				return;
-			}
-#endif
-
-			if (!IsReadableRange(addr, 8u * sizeof(uint64_t))) {
-				LOGF("%s = %016" PRIx64 " (unmapped)\n", name, addr);
-				return;
-			}
-
-			auto* q = reinterpret_cast<const uint64_t*>(addr);
-			LOGF("%s = %016" PRIx64 ": %016" PRIx64 " %016" PRIx64 " %016" PRIx64 " %016" PRIx64
-			     " %016" PRIx64 " %016" PRIx64 " %016" PRIx64 " %016" PRIx64 "\n",
-			     name, addr, q[0], q[1], q[2], q[3], q[4], q[5], q[6], q[7]);
-		};
-
-		dump_guest_qwords("guest rbx", info->rbx);
-		dump_guest_qwords("guest rax", info->rax);
-		dump_guest_qwords("guest rcx", info->rcx);
-		dump_guest_qwords("guest rsi", info->rsi);
-		dump_guest_qwords("guest rdi", info->rdi);
-		dump_guest_qwords("guest r8 ", info->r8);
-		dump_guest_qwords("guest r9 ", info->r9);
-		dump_guest_qwords("guest r10", info->r10);
-		dump_guest_qwords("guest r11", info->r11);
-		dump_guest_qwords("guest r12", info->r12);
-		dump_guest_qwords("guest r13", info->r13);
-		dump_guest_qwords("guest r14", info->r14);
-		dump_guest_qwords("guest r15", info->r15);
-
-		if (info->exception_address == 0x000000090064364e &&
-		    IsDumpableRange(info->rbx, sizeof(uint64_t))) {
-			auto* local = reinterpret_cast<const uint64_t*>(info->rbx);
-			dump_guest_qwords("vorbis obj", local[0]);
-			dump_guest_qwords("vorbis len", info->rcx);
-		}
-
-		EXIT("Access violation: %s [%016" PRIx64 "] %s\n",
-		     Common::EnumName(info->access_violation_type).c_str(), info->access_violation_vaddr,
-		     (info->access_violation_vaddr == g_invalid_memory ? "(Unpatched object)" : ""));
-		return false;
-	}
-
-	EXIT("Unknown exception!!! (%08" PRIx32 ")", info->native_code);
 	return false;
 }