loader: keep the caller's flags in the TLS stubs (#1571)

A patched TLS load calls TlsRegStub and SafeCall, which moved RSP past the
red zone with sub/add rsp,0x80. ADD and SUB write the arithmetic flags, so
the sub ran before SafeCall's pushfq saved them and the add ran after its
popfq restored them: guest code that compares, loads fs:[0] and then
branches on the flags could branch the wrong way. lea moves RSP without
touching the flags. The entry lea plus a two-byte nop keeps every patched
offset in both stubs; the exit lea is one byte longer and moves only ret.

TlsStubsPreserveFlags calls the stubs with two flag patterns and checks
the status flags, the output register and RAX; it fails with the old
bytes.

Co-authored-by: Claude Opus 5.5 <noreply@anthropic.com>

src/graphics/host_gpu/memoryTracker.cpp

@@ -66,6 +66,18 @@ RegionManager* MemoryTracker::GetOrCreateRegion(uint64_t index) {
 	return ptr;
 }
 
+void MemoryTracker::QueueBdaSync(uint64_t vaddr, uint64_t size) {
+	const auto begin = Common::AlignDown(vaddr, TRACKER_PAGE_SIZE);
+	const auto end   = Common::AlignUp(vaddr + size, TRACKER_PAGE_SIZE);
+	std::lock_guard lock(m_bda_sync_mutex);
+	m_bda_sync.Add(begin, end - begin);
+}
+
+RangeSet MemoryTracker::TakeBdaSync() {
+	std::lock_guard lock(m_bda_sync_mutex);
+	return std::exchange(m_bda_sync, RangeSet {});
+}
+
 bool MemoryTracker::IsRegionCpuModified(uint64_t vaddr, uint64_t size) {
 	CheckNotInUploadCallback();
 	return Iterate<true>(vaddr, size, [](RegionManager* manager, uint64_t offset, uint64_t bytes) {
@@ -88,6 +100,7 @@ void MemoryTracker::MarkRegionAsCpuModified(uint64_t vaddr, uint64_t size) {
 		std::scoped_lock lock(manager->lock);
 		manager->ChangeState<DirtySource::Cpu, true>(manager->GetCpuAddr() + offset, bytes);
 	});
+	QueueBdaSync(vaddr, size);
 }
 
 void MemoryTracker::MarkRegionAsGpuModified(uint64_t vaddr, uint64_t size) {
@@ -128,6 +141,8 @@ void MemoryTracker::UntrackMemory(uint64_t vaddr, uint64_t size) {
 	Iterate<false>(vaddr, size, [](RegionManager* manager, uint64_t offset, uint64_t bytes) {
 		manager->ChangeState<DirtySource::Cpu, true>(manager->GetCpuAddr() + offset, bytes);
 	});
+	locks.clear();
+	QueueBdaSync(vaddr, size);
 }
 
 } // namespace Libs::Graphics

src/graphics/host_gpu/memoryTracker.h

@@ -29,6 +29,10 @@ public:
 	void               MarkRegionAsGpuModified(uint64_t vaddr, uint64_t size);
 	void               UnmarkRegionAsGpuModified(uint64_t vaddr, uint64_t size);
 	void               UntrackMemory(uint64_t vaddr, uint64_t size);
+	// Queue after changing ownership; a concurrent drain must not consume an event
+	// before the dirty pages it describes become visible.
+	void                   QueueBdaSync(uint64_t vaddr, uint64_t size);
+	[[nodiscard]] RangeSet TakeBdaSync();
 	// Removes protection from a range and flushes GPU-owned data when required.
 	template <typename Flush>
 	void InvalidateRegion(uint64_t vaddr, uint64_t size, Flush&& on_flush) noexcept {
@@ -51,6 +55,7 @@ public:
 				on_flush();
 			}
 		});
+		QueueBdaSync(vaddr, size);
 	}
 #if KYTY_BUILD == KYTY_BUILD_DEBUG
 	void ValidateGpuDirtyPages(const RangeSet& dirty, uint64_t vaddr, uint64_t size,
@@ -149,6 +154,8 @@ private:
 	std::unique_ptr<std::atomic<RegionManager*>[]> m_regions;
 	std::vector<std::unique_ptr<RegionManager>>    m_region_storage;
 	std::mutex                                     m_region_mutex;
+	std::mutex                                     m_bda_sync_mutex;
+	RangeSet                                       m_bda_sync;
 	PageManager&                                   m_page_manager;
 };
 

src/graphics/host_gpu/renderer/cache/bufferCache.cpp

@@ -126,6 +126,7 @@ void BufferCache::ChangeRegister(BufferId id) {
 		}
 		WriteDataBuffer(m_bda_pagetable_buffer, table_offset,
 		                addresses.data(), addresses.size() * sizeof(vk::DeviceAddress));
+		m_memory_tracker.QueueBdaSync(buffer.CpuAddress(), buffer.Size());
 	} else {
 		const auto found = m_buffers.find(buffer.CpuAddress());
 		EXIT_IF(found == m_buffers.end() || found->second != id);
@@ -685,6 +686,16 @@ void BufferCache::ProcessFaultBuffer() {
 	m_fault_manager.ProcessFaultBuffer();
 }
 
+void BufferCache::SynchronizeBda(const RangeSet& mapped) {
+	// Take before walking: writes queued during synchronization remain pending.
+	const auto pending = m_memory_tracker.TakeBdaSync();
+	pending.ForEach([&](uint64_t begin, uint64_t end) {
+		mapped.ForEachInRange(begin, end - begin, [this](uint64_t start, uint64_t finish) {
+			SynchronizeBuffersInRange(start, finish - start);
+		});
+	});
+}
+
 void BufferCache::SynchronizeBuffersInRange(uint64_t vaddr, uint64_t size) {
 	const auto end = vaddr + size;
 	auto       it  = m_buffers.upper_bound(vaddr);

src/graphics/host_gpu/renderer/cache/bufferCache.h

@@ -80,7 +80,10 @@ public:
 	[[nodiscard]] bool IsRegionCpuModified(uint64_t vaddr, uint64_t size);
 	[[nodiscard]] bool IsRegionGpuModified(uint64_t vaddr, uint64_t size);
 	void               ProcessFaultBuffer();
-	void               SynchronizeBuffersInRange(uint64_t vaddr, uint64_t size);
+	void               SynchronizeBda(const RangeSet& mapped);
+	void               QueueBdaSync(uint64_t vaddr, uint64_t size) {
+		m_memory_tracker.QueueBdaSync(vaddr, size);
+	}
 	void               RunGarbageCollector();
 
 private:
@@ -114,6 +117,7 @@ private:
 	void DeleteBuffer(BufferId id);
 	[[nodiscard]] bool SynchronizeBuffer(Buffer& buffer, uint64_t vaddr, uint64_t size,
 	                                     bool is_written, bool is_texel_buffer);
+	void SynchronizeBuffersInRange(uint64_t vaddr, uint64_t size);
 	[[nodiscard]] vk::Buffer UploadCopies(Buffer& buffer, std::span<vk::BufferCopy> copies,
 	                                      uint64_t total_size);
 	[[nodiscard]] bool SynchronizeBufferFromImage(Buffer& buffer, uint64_t vaddr, uint64_t size);

src/graphics/host_gpu/renderer/cache/streamBuffer.cpp

@@ -67,11 +67,9 @@ Buffer::Buffer(GraphicContext& graphics, CommandScheduler& scheduler, MemoryUsag
 	buffer_info.usage       = flags;
 
 	const bool with_bda = bool(flags & vk::BufferUsageFlagBits::eShaderDeviceAddress);
-	const VmaAllocationCreateFlags bda_flag =
-	    with_bda ? VMA_ALLOCATION_CREATE_DEDICATED_MEMORY_BIT : 0;
 	VmaAllocationCreateInfo allocation_info {};
 	allocation_info.flags =
-	    VMA_ALLOCATION_CREATE_WITHIN_BUDGET_BIT | bda_flag | AllocationFlags(usage);
+	    VMA_ALLOCATION_CREATE_WITHIN_BUDGET_BIT | AllocationFlags(usage);
 	allocation_info.usage = AllocationUsage(usage);
 	allocation_info.preferredFlags = usage == MemoryUsage::DeviceLocal
 	                                     ? VkMemoryPropertyFlags {}

src/graphics/host_gpu/renderer/pipeline/pipelineCache.cpp

@@ -18,7 +18,6 @@
 #include "graphics/shader/recompiler/ShaderRecompiler.h"
 #include "graphics/shader/shaderCompiler.h"
 #include "kernel/memory.h"
-#include "kytyGitVersion.h"
 #include "loader/systemContent.h"
 
 #include <algorithm>
@@ -81,7 +80,8 @@ std::string DriverCacheSignature(const vk::PhysicalDeviceProperties& properties)
 		uuid[i * 2]     = hex[properties.pipelineCacheUUID[i] >> 4u];
 		uuid[i * 2 + 1] = hex[properties.pipelineCacheUUID[i] & 0xfu];
 	}
-	return fmt::format("KytyPC1:{}:{:08x}:{:08x}:{:08x}:{}\n", KYTY_GIT_REVISION,
+	// Driver cache entries already distinguish shader code and pipeline state across builds.
+	return fmt::format("KytyPC2:{:08x}:{:08x}:{:08x}:{}\n",
 	                   properties.vendorID, properties.deviceID, properties.driverVersion, uuid);
 }
 
@@ -516,17 +516,6 @@ void PipelineCache::InitializeDriverCache() {
 		PipelineCacheLog("Vulkan pipeline cache: disabled (non-Release build)");
 		return;
 	}
-	const std::string_view git_hash     = KYTY_GIT_HASH;
-	const std::string_view git_revision = KYTY_GIT_REVISION;
-	if (git_hash == "unknown" || git_revision == "unknown") {
-		PipelineCacheLog("Vulkan pipeline cache: disabled (unknown git revision)");
-		return;
-	}
-	if (git_hash.ends_with("-dirty")) {
-		PipelineCacheLog("Vulkan pipeline cache: disabled (dirty build)");
-		return;
-	}
-
 	m_driver_cache_path     = std::filesystem::path("_PipelineCache") / (title_id + ".bin");
 	const auto path         = Common::PathToString(m_driver_cache_path);
 	const bool cache_exists = Common::File::IsFileExisting(m_driver_cache_path);
@@ -559,7 +548,7 @@ void PipelineCache::InitializeDriverCache() {
 			    XXH3_64bits(initial_data.data(), initial_data.size()) != payload_hash) {
 				initial_data.clear();
 				PipelineCacheLog(
-				    "Vulkan pipeline cache: invalidating {} (driver, emulator, or data mismatch)",
+				    "Vulkan pipeline cache: invalidating {} (driver or data mismatch)",
 				    path);
 			}
 		} else {

src/graphics/host_gpu/renderer/renderContext.cpp

@@ -90,6 +90,7 @@ bool RenderContext::IsMapped(uint64_t vaddr, uint64_t size) const noexcept {
 void RenderContext::MapMemory(uint64_t vaddr, uint64_t size) {
 	std::lock_guard lock(m_mapped_ranges_mutex);
 	m_mapped_ranges.Add(vaddr, size);
+	m_buffer_cache.QueueBdaSync(vaddr, size);
 }
 
 void RenderContext::UnmapMemory(uint64_t vaddr, uint64_t size) {
@@ -129,9 +130,7 @@ void RenderContext::PrepareBda() {
 		m_bda_logged = true;
 	}
 	std::shared_lock lock(m_mapped_ranges_mutex);
-	m_mapped_ranges.ForEach([this](uint64_t start, uint64_t end) {
-		m_buffer_cache.SynchronizeBuffersInRange(start, end - start);
-	});
+	m_buffer_cache.SynchronizeBda(m_mapped_ranges);
 	m_fault_process_pending = true;
 }
 

src/graphics/host_gpu/renderer/renderDraw.cpp

@@ -536,7 +536,10 @@ RenderState RenderExecutor::AcquireRenderTargets(CommandBuffer& buffer, RenderCo
 			}
 		}
 		if (feedback_aspects && !m_context.GetGraphics().attachment_feedback_loop_enabled) {
-			EXIT("depth attachment feedback loop is not supported by the host\n");
+			static std::atomic_bool logged = false;
+			if (!logged.exchange(true, std::memory_order_relaxed)) {
+				std::printf("depth attachment feedback loop is not supported by the host\n");
+			}
 		}
 		auto layout = depth_attachment_layout(depth);
 		if (sampled_aspects & ~DepthReadableAspects(layout)) {

src/graphics/presentation/window/swapchain.cpp

@@ -945,7 +945,6 @@ Presenter::Frame& Presenter::PrepareBlankFrame(uint32_t width, uint32_t height,
 	auto*             frame  = m_impl->frames.Acquire({width, height}, format);
 	frame->hdr   = false;
 	frame->blank = true;
-	Common::LockGuard render_lock(m_impl->renderer.GetMutex());
 	frame->Configure(m_impl->window.graphic_ctx, {width, height}, format);
 	vk::ClearColorValue clear {};
 	clear.float32[3] = opaque ? 1.0f : 0.0f;
@@ -953,6 +952,7 @@ Presenter::Frame& Presenter::PrepareBlankFrame(uint32_t width, uint32_t height,
 		EXIT_IF(producer->IsInvalid());
 		frame->Clear(*producer, clear);
 	} else {
+		Common::LockGuard lock(m_impl->present_mutex);
 		auto& command = m_impl->present_scheduler.BeginCommand();
 		frame->Clear(command, clear);
 		frame->present_tick = m_impl->present_scheduler.Submit();
@@ -1036,18 +1036,17 @@ void Presenter::Impl::Present() {
 			RecoverSwapchain(status);
 			continue;
 		}
-		{
-			Common::LockGuard render_lock(renderer.GetMutex());
-			auto&             command = present_scheduler.BeginCommand();
-			const bool        draw_system_overlay =
-			    overlay_visual.active && swapchain.PrepareSystemOverlay();
-			swapchain.RecordPresentCommands(command, layers[0].frame, layers[1],
-			                                draw_system_overlay);
-			const auto tick = swapchain.Submit(present_scheduler);
-			for (const auto& layer: layers) {
-				if (layer.frame != nullptr) {
-					layer.frame->present_tick = tick;
-				}
+		// present_mutex owns the prepared frames and this scheduler. Only queue submission
+		// is shared with rendering, and it has its own queue_mutex.
+		auto&      command = present_scheduler.BeginCommand();
+		const bool draw_system_overlay =
+		    overlay_visual.active && swapchain.PrepareSystemOverlay();
+		swapchain.RecordPresentCommands(command, layers[0].frame, layers[1],
+		                                draw_system_overlay);
+		const auto tick = swapchain.Submit(present_scheduler);
+		for (const auto& layer: layers) {
+			if (layer.frame != nullptr) {
+				layer.frame->present_tick = tick;
 			}
 		}
 		status = swapchain.Present();

src/graphics/shader/recompiler/ir/passes/ConstantPropagation.cpp

@@ -608,13 +608,23 @@ void FoldInstruction(Block& block, Block::iterator instruction,
 			    ReplaceBinaryIdentity(inst, Type::U32, UINT32_MAX)) return;
 			for (uint32_t arm = 0; arm < 2u; ++arm) {
 				const auto mask = Arg(inst, arm);
+				if (!IsImmediate(mask, Type::U32)) continue;
+				if (mask.U32() == 0u) {
+					Replace(inst, Value(0u));
+					return;
+				}
 				const auto* shifted = Arg(inst, arm ^ 1u).TryInstruction();
-				if (!IsImmediate(mask, Type::U32) || shifted == nullptr ||
-				    shifted->GetOpcode() != ValueOpcode::ShiftLeftLogical32 || !HasOneUse(*shifted)) continue;
+				if (shifted == nullptr || shifted->GetOpcode() != ValueOpcode::ShiftLeftLogical32) continue;
 				const auto shift = Arg(*shifted, 1);
-				if (!IsImmediate(shift, Type::U32) || shift.U32() >= 32u) continue;
+				if (!IsImmediate(shift, Type::U32)) continue;
+				const auto shifted_mask = mask.U32() >> (shift.U32() & 31u);
+				if (shifted_mask == 0u) {
+					Replace(inst, Value(0u));
+					return;
+				}
+				if (!HasOneUse(*shifted) || shift.U32() >= 32u) continue;
 				const auto masked = worklist.Prepend(block, instruction, ValueOpcode::BitwiseAnd32,
-				    {Arg(*shifted, 0), Value(mask.U32() >> shift.U32())});
+				    {Arg(*shifted, 0), Value(shifted_mask)});
 				Replace(inst, Value(&*worklist.Prepend(block, instruction, ValueOpcode::ShiftLeftLogical32,
 				    {Value(&*masked), shift})));
 				return;

src/kernel/module.h

@@ -0,0 +1,24 @@
+#ifndef EMULATOR_SRC_KERNEL_MODULE_H_
+#define EMULATOR_SRC_KERNEL_MODULE_H_
+
+#include "common/abi.h"
+
+#include <cstddef>
+#include <cstdint>
+
+namespace Libs::LibKernel {
+
+using KernelModule = int32_t;
+struct KernelLoadModuleOpt;
+struct KernelUnloadModuleOpt;
+
+KernelModule KYTY_SYSV_ABI KernelLoadStartModule(const char* module_file_name, size_t args,
+                                                 const void* argp, uint32_t flags,
+                                                 const KernelLoadModuleOpt* opt, int* res);
+int KYTY_SYSV_ABI KernelStopUnloadModule(KernelModule handle, size_t args, const void* argp,
+                                        uint32_t flags, const KernelUnloadModuleOpt* opt,
+                                        int* res);
+
+} // namespace Libs::LibKernel
+
+#endif // EMULATOR_SRC_KERNEL_MODULE_H_

src/libs/libKernel.cpp

@@ -11,6 +11,7 @@
 #include "kernel/eventQueue.h"
 #include "kernel/fileSystem.h"
 #include "kernel/memory.h"
+#include "kernel/module.h"
 #include "kernel/pthread.h"
 #include "kernel/semaphore.h"
 #include "kernel/syncOnAddress.h"
@@ -65,7 +66,6 @@ LIB_DEFINE(InitLibKernel_1_Apr);
 
 namespace LibKernel {
 
-using KernelModule                   = int32_t;
 using get_thread_atexit_count_func_t = KYTY_SYSV_ABI int (*)(KernelModule);
 using thread_atexit_report_func_t    = KYTY_SYSV_ABI void (*)(KernelModule);
 
@@ -1201,9 +1201,9 @@ static int KYTY_SYSV_ABI KernelRaiseException(Pthread thread, int signum) {
 	return OK;
 }
 
-static KYTY_SYSV_ABI KernelModule KernelLoadStartModule(const char* module_file_name, size_t args,
-                                                        const void* argp, uint32_t flags,
-                                                        const KernelLoadModuleOpt* opt, int* res) {
+KernelModule KYTY_SYSV_ABI KernelLoadStartModule(const char* module_file_name, size_t args,
+                                                 const void* argp, uint32_t flags,
+                                                 const KernelLoadModuleOpt* opt, int* res) {
 	PRINT_NAME();
 
 	// EXIT_NOT_IMPLEMENTED(!Common::Thread::IsMainThread());
@@ -1255,9 +1255,9 @@ static KYTY_SYSV_ABI KernelModule KernelLoadStartModule(const char* module_file_
 	return static_cast<KernelModule>(handle);
 }
 
-static int KYTY_SYSV_ABI KernelStopUnloadModule(KernelModule handle, size_t args, const void* argp,
-                                                uint32_t flags, const KernelUnloadModuleOpt* opt,
-                                                int* res) {
+int KYTY_SYSV_ABI KernelStopUnloadModule(KernelModule handle, size_t args, const void* argp,
+                                        uint32_t flags, const KernelUnloadModuleOpt* opt,
+                                        int* res) {
 	PRINT_NAME();
 
 	// EXIT_NOT_IMPLEMENTED(!Common::Thread::IsMainThread());

src/libs/libRazorCpu.cpp

@@ -1,9 +1,9 @@
 #include "common/abi.h"
-#include "common/common.h"
-#include "common/stringUtils.h"
 #include "libs/libs.h"
 #include "loader/symbolDatabase.h"
 
+#include <cstdint>
+
 namespace Libs {
 
 namespace LibRazorCpu {
@@ -16,8 +16,18 @@ static KYTY_SYSV_ABI uint32_t RazorCpuIsCapturing() {
 	return 0;
 }
 
+static int KYTY_SYSV_ABI RazorCpuPopMarker() {
+	return 0;
+}
+
+static int KYTY_SYSV_ABI RazorCpuPushMarker(const char*, uint32_t, uint32_t) {
+	return 0;
+}
+
 LIB_DEFINE(InitRazorCpu_1) {
 	LIB_FUNC("EboejOQvLL4", LibRazorCpu::RazorCpuIsCapturing);
+	LIB_FUNC("YpkGsMXP3ew", LibRazorCpu::RazorCpuPopMarker);
+	LIB_FUNC("zw+celG7zSI", LibRazorCpu::RazorCpuPushMarker);
 }
 
 } // namespace LibRazorCpu

src/libs/libSysmodule.cpp

@@ -3,6 +3,7 @@
 #include "common/common.h"
 #include "common/logging/log.h"
 #include "common/stringUtils.h"
+#include "kernel/module.h"
 #include "libs/libs.h"
 #include "loader/symbolDatabase.h"
 
@@ -20,14 +21,40 @@ int KYTY_SYSV_ABI KernelGetModuleInfoForUnwind(uint64_t addr, int flags, ModuleI
 
 namespace Sysmodule {
 
-constexpr int SYSMODULE_ERROR_UNLOADED = static_cast<int>(0x805a1001u);
+constexpr int      SYSMODULE_ERROR_UNLOADED = static_cast<int>(0x805a1001u);
+constexpr int      SYSMODULE_ERROR_FATAL    = static_cast<int>(0x805a10ffu);
 
-static std::mutex                             g_module_mutex;
-static std::unordered_map<uint16_t, uint32_t> g_module_refs;
+static constexpr struct NativeModule {
+	uint16_t    id;
+	const char* path;
+} g_native_modules[] = {{0x0115, "/app0/sce_module/libSceNpCppWebApi.prx"}};
 
-static void LoadModule(uint16_t id) {
+struct Module {
+	uint32_t                references = 0;
+	LibKernel::KernelModule handle     = -1;
+};
+
+static std::mutex                          g_module_mutex;
+static std::unordered_map<uint16_t, Module> g_modules;
+
+static int LoadModule(uint16_t id) {
 	std::scoped_lock lock(g_module_mutex);
-	++g_module_refs[id];
+	if (const auto module = g_modules.find(id); module != g_modules.end()) {
+		++module->second.references;
+		return 0;
+	}
+
+	LibKernel::KernelModule handle = -1;
+	for (const auto& module: g_native_modules) {
+		if (module.id != id) continue;
+		handle = LibKernel::KernelLoadStartModule(module.path, 0, nullptr, 0, nullptr, nullptr);
+		if (handle < 0) {
+			return handle;
+		}
+		break;
+	}
+	g_modules.emplace(id, Module {1, handle});
+	return 0;
 }
 
 static KYTY_SYSV_ABI int SysmoduleGetModuleInfoForUnwind(uint64_t addr, int flags,
@@ -40,8 +67,7 @@ static KYTY_SYSV_ABI int SysmoduleLoadModule(uint16_t id) {
 
 	LOGF("\t id = %d\n", static_cast<int>(id));
 
-	LoadModule(id);
-	return 0;
+	return LoadModule(id);
 }
 
 static KYTY_SYSV_ABI int SysmoduleUnloadModule(uint16_t id) {
@@ -50,12 +76,19 @@ static KYTY_SYSV_ABI int SysmoduleUnloadModule(uint16_t id) {
 	LOGF("\t id = %d\n", static_cast<int>(id));
 
 	std::scoped_lock lock(g_module_mutex);
-	const auto       module = g_module_refs.find(id);
-	if (module == g_module_refs.end()) {
+	const auto       module = g_modules.find(id);
+	if (module == g_modules.end()) {
 		return SYSMODULE_ERROR_UNLOADED;
 	}
-	if (--module->second == 0) {
-		g_module_refs.erase(module);
+	if (module->second.references == 1) {
+		if (module->second.handle >= 0 &&
+		    LibKernel::KernelStopUnloadModule(module->second.handle, 0, nullptr, 0, nullptr,
+		                                     nullptr) < 0) {
+			return SYSMODULE_ERROR_FATAL;
+		}
+		g_modules.erase(id);
+	} else {
+		--module->second.references;
 	}
 	return 0;
 }
@@ -71,10 +104,8 @@ static KYTY_SYSV_ABI int SysmoduleLoadModuleInternalWithArg(uint16_t id, int arg
 	EXIT_IF(arg3 != 0);
 	EXIT_IF(ret == nullptr);
 
-	LoadModule(id);
-	*ret = 0;
-
-	return 0;
+	*ret = LoadModule(id);
+	return *ret;
 }
 
 static KYTY_SYSV_ABI int SysmoduleIsLoaded(uint16_t id) {
@@ -83,7 +114,7 @@ static KYTY_SYSV_ABI int SysmoduleIsLoaded(uint16_t id) {
 	LOGF("\t id = %d\n", static_cast<int>(id));
 
 	std::scoped_lock lock(g_module_mutex);
-	return g_module_refs.contains(id) ? 0 : SYSMODULE_ERROR_UNLOADED;
+	return g_modules.contains(id) ? 0 : SYSMODULE_ERROR_UNLOADED;
 }
 
 } // namespace Sysmodule

src/loader/jit.h

@@ -58,16 +58,17 @@ struct TlsRegStub {
 	}
 	static uint64_t GetSize() { return 32; }
 
-	// sub rsp,0x80
+	// lea rsp,[rsp-0x80]
+	// nop
 	// push rax
 	// call safe_call
 	// mov <reg>,rax
 	// pop rax
-	// add rsp,0x80
+	// lea rsp,[rsp+0x80]
 	// ret
-	uint8_t code[32] = {0x48, 0x81, 0xEC, 0x80, 0x00, 0x00, 0x00, 0x50, 0xE8, 0x00, 0x00,
-	                    0x00, 0x00, 0x48, 0x89, 0xC0, 0x58, 0x48, 0x81, 0xC4, 0x80, 0x00,
-	                    0x00, 0x00, 0xC3, 0x90, 0x90, 0x90, 0x90, 0x90, 0x90, 0x90};
+	uint8_t code[32] = {0x48, 0x8D, 0x64, 0x24, 0x80, 0x66, 0x90, 0x50, 0xE8, 0x00, 0x00,
+	                    0x00, 0x00, 0x48, 0x89, 0xC0, 0x58, 0x48, 0x8D, 0xA4, 0x24, 0x80,
+	                    0x00, 0x00, 0x00, 0xC3, 0x90, 0x90, 0x90, 0x90, 0x90, 0x90};
 };
 
 struct SafeCall {
@@ -78,7 +79,8 @@ struct SafeCall {
 	static uint64_t GetSize() { return 0x1000; }
 
 	uint8_t code[0x90] = {
-	    /*00*/ 0x48, 0x81, 0xec, 0x80, 0x00, 0x00, 0x00, // sub    rsp,0x80
+	    /*00*/ 0x48, 0x8d, 0x64, 0x24, 0x80,             // lea    rsp,[rsp-0x80]
+	    /*05*/ 0x66, 0x90,                               // nop
 	    /*07*/ 0x9c,                                     // pushfq
 	    /*08*/ 0x51,                                     // push   rcx
 	    /*09*/ 0x52,                                     // push   rdx
@@ -108,9 +110,9 @@ struct SafeCall {
 	    /*40*/ 0x5a,             // pop    rdx
 	    /*41*/ 0x59,             // pop    rcx
 	    /*42*/ 0x9d,             // popfq
-	    /*43*/ 0x48, 0x81, 0xc4, 0x80, 0x00, 0x00,
-	    0x00,        // add    rsp,0x80
-	    /*4a*/ 0xc3, // ret
+	    /*43*/ 0x48, 0x8d, 0xa4, 0x24, 0x80, 0x00, 0x00,
+	    0x00,        // lea    rsp,[rsp+0x80]
+	    /*4b*/ 0xc3, // ret
 	};
 };
 

tests/KernelFileSystemTests.cpp

@@ -235,6 +235,9 @@ void TestSysmoduleReferences() {
   const auto unload = resolve("eR2bZFAAU0Q");
   const auto is_loaded = resolve("fMP5NHUOaMk");
   constexpr int unloaded = static_cast<int>(0x805a1001u);
+  Check(load(0x115) == Libs::LibKernel::KERNEL_ERROR_ENOENT && is_loaded(0x115) == unloaded &&
+            unload(0x115) == unloaded,
+        "missing packaged NpCppWebApi fails without recording a loaded module");
   Check(load(0x113) == OK && is_loaded(0xb4) == unloaded,
         "loading entitlement access does not mark AppContent loaded");
   Check(unload(0x113) == OK, "release independent module reference");
@@ -255,6 +258,9 @@ void TestSysmoduleReferences() {
       reinterpret_cast<int (KYTY_SYSV_ABI *)(uint16_t, int, int, int, int*)>(
           internal_symbol->vaddr);
   int result = -1;
+  Check(internal_load(0x115, 0, 0, 0, &result) == Libs::LibKernel::KERNEL_ERROR_ENOENT &&
+            result == Libs::LibKernel::KERNEL_ERROR_ENOENT && is_loaded(0x115) == unloaded,
+        "internal module loading propagates missing native module failure");
   Check(internal_load(0xb4, 0, 0, 0, &result) == OK && result == OK &&
             is_loaded(0xb4) == OK && unload(0xb4) == OK && is_loaded(0xb4) == unloaded,
         "internal and public module operations share load state");

tests/MemoryTrackerTests.cpp

@@ -213,6 +213,12 @@ struct ProtectionCall {
 std::vector<ProtectionCall> g_protection_log;
 std::mutex g_protection_log_mutex;
 
+struct ProtectionGate {
+  std::binary_semaphore entered{0};
+  std::binary_semaphore resume{0};
+};
+ProtectionGate *g_protection_gate = nullptr;
+
 void ResetProtectionLog() {
   g_protection_calls = 0;
   g_protection_log.clear();
@@ -220,6 +226,10 @@ void ResetProtectionLog() {
 
 bool ProtectAddressSpace(uint64_t vaddr, uint64_t size,
                          Common::VirtualMemory::Mode mode) {
+  if (g_protection_gate != nullptr && mode == Common::VirtualMemory::Mode::ReadWrite) {
+    g_protection_gate->entered.release();
+    g_protection_gate->resume.acquire();
+  }
   uint32_t protection = PAGE_NOACCESS;
   if (mode == Common::VirtualMemory::Mode::Read) {
     protection = PAGE_READONLY;
@@ -407,6 +417,74 @@ void TestCpuDirtyUpload() {
   Release(memory);
 }
 
+void TestBdaSyncDirtiness() {
+  constexpr auto page = Libs::Graphics::TRACKER_PAGE_SIZE;
+  TrackerHarness harness;
+  auto &tracker = harness.tracker;
+  auto *memory = Allocate(harness.page_manager, 3);
+  const auto address = reinterpret_cast<uint64_t>(memory);
+  Check(tracker.TakeBdaSync().Empty(), "new tracker queued unowned memory");
+  tracker.QueueBdaSync(address, 3 * page);
+  Check(tracker.IsRegionCpuModified(address, page), "new region was clean");
+  auto pending = tracker.TakeBdaSync();
+  Check(pending.Contains(address, 3 * page) && !pending.Intersects(address + 3 * page, page),
+        "mapped or registered range did not bound initial BDA synchronization");
+  tracker.ForEachUploadRange(address, 3 * page, false,
+                             [](uint64_t, uint64_t) noexcept {}, []() noexcept {});
+  Check(tracker.TakeBdaSync().Empty(), "clean upload queued redundant BDA work");
+
+  tracker.MarkRegionAsCpuModified(address + 17, 1);
+  pending = tracker.TakeBdaSync();
+  Check(pending.Contains(address, page) && !pending.Intersects(address + page, page) &&
+            tracker.TakeBdaSync().Empty(),
+        "byte dirtiness did not drain exactly its whole tracker page");
+  // A write after Take belongs to the next drain, even when its page was just taken.
+  tracker.MarkRegionAsCpuModified(address + 31, 1);
+  Check(tracker.TakeBdaSync().Contains(address, page), "re-dirty event was lost");
+  tracker.ForEachUploadRange(address, page, true,
+                             [](uint64_t, uint64_t) noexcept {}, []() noexcept {});
+  Check(tracker.TakeBdaSync().Empty(), "GPU ownership queued an unnecessary upload");
+  tracker.InvalidateRegion(address + 7, 1, [&] {
+    tracker.UnmarkRegionAsGpuModified(address, page);
+    tracker.MarkRegionAsCpuModified(address, page);
+  });
+  Check(tracker.TakeBdaSync().Contains(address, page),
+        "GPU-to-CPU invalidation did not queue BDA synchronization");
+  tracker.InvalidateRegion(address + page + 7, 1, [] {});
+  Check(tracker.TakeBdaSync().Contains(address + page, page),
+        "CPU invalidation did not queue BDA synchronization");
+  tracker.UntrackMemory(address, 3 * page);
+  Check(tracker.TakeBdaSync().Contains(address, 3 * page),
+        "untracked memory did not queue its newly dirty pages");
+  Release(memory);
+}
+
+void TestBdaSyncConcurrentDrain() {
+  constexpr auto page = Libs::Graphics::TRACKER_PAGE_SIZE;
+  TrackerHarness harness;
+  auto &tracker = harness.tracker;
+  auto *memory = Allocate(harness.page_manager, 1);
+  const auto address = reinterpret_cast<uint64_t>(memory);
+  tracker.ForEachUploadRange(address, page, false,
+                             [](uint64_t, uint64_t) noexcept {}, []() noexcept {});
+  (void)tracker.TakeBdaSync();
+  ProtectionGate gate;
+  g_protection_gate = &gate;
+  std::jthread writer([&] { tracker.MarkRegionAsCpuModified(address + 17, 1); });
+  gate.entered.acquire();
+  // Ownership is still changing. Publishing its event before this point would let
+  // a drain consume it before the writer has finished releasing page protection.
+  Check(tracker.TakeBdaSync().Empty(), "BDA event preceded its state transition");
+  gate.resume.release();
+  writer.join();
+  g_protection_gate = nullptr;
+  Check(tracker.TakeBdaSync().Contains(address, page) &&
+            tracker.IsRegionCpuModified(address, page),
+        "concurrent drain lost a completed CPU dirty transition");
+  tracker.UntrackMemory(address, page);
+  Release(memory);
+}
+
 void TestCleanUploadPreservesOwnership() {
   constexpr auto page_size = Libs::Graphics::TRACKER_PAGE_SIZE;
   constexpr auto region_size = Libs::Graphics::TRACKER_REGION_SIZE;
@@ -1202,6 +1280,8 @@ int main(int argc, char **argv) {
   TestQueriesDoNotRequireMappedOwnership();
   TestConcurrentRegionPublication();
   TestCpuDirtyUpload();
+  TestBdaSyncDirtiness();
+  TestBdaSyncConcurrentDrain();
   TestCleanUploadPreservesOwnership();
   TestRangeInvalidation();
   TestGpuReacquisitionAfterInvalidation();

tests/ShaderRecompilerComputeTests.cpp

@@ -5281,6 +5281,85 @@ public:
     std::printf("[gpu]     %-32s ok\n", name);
   }
 
+  void CheckBdaSyncUploads() {
+    constexpr const char *name = "BdaSyncUploads";
+    constexpr auto page = BufferCache::CACHING_PAGESIZE;
+    EnsureRuntimeContext();
+    const auto base = LibKernel::Memory::AllocateRuntimeMemory(
+        0, 3 * page, Common::VirtualMemory::Mode::ReadWrite, "bda_sync_uploads");
+    Require(name, "allocation", base != 0, "guest allocation failed");
+    auto *words = reinterpret_cast<u32 *>(base);
+    std::fill_n(words, 3 * page / sizeof(u32), 0x12345678u);
+    RenderContext context(m_runtime_context);
+    auto &scheduler = context.GetCommandScheduler();
+    auto &cache = context.GetBufferCache();
+    HW::Context registers{};
+    HW::UserConfig user_config{};
+    HW::Shader shaders{};
+    scheduler.Begin(registers, user_config, shaders);
+    context.MapMemory(base, page);
+    context.MapMemory(base + 2 * page, page);
+    const auto id = cache.FindBuffer(base, 3 * page);
+    auto &buffer = cache.GetBuffer(id);
+    auto readback = CreateHostBuffer(name, 6 * sizeof(u32),
+                                     vk::BufferUsageFlagBits::eTransferDst, {});
+    const auto snapshot = [&](uint64_t offset, uint32_t index) {
+      const vk::BufferCopy copy{offset, index * sizeof(u32), sizeof(u32)};
+      scheduler.Current().Handle().copyBuffer(buffer.Handle(), readback.buffer, 1, &copy);
+    };
+    context.PrepareBda();
+    Require(name, "mapped intersection",
+            !cache.IsRegionCpuModified(base, page) &&
+                cache.IsRegionCpuModified(base + page, page) &&
+                !cache.IsRegionCpuModified(base + 2 * page, page),
+            "BDA upload missed a mapped range or uploaded an unmapped hole");
+    snapshot(0, 0);
+    snapshot(2 * page, 1);
+
+    cache.InvalidateMemory(base + 17, 1);
+    words[4] = 0x87654321u;
+    words[TRACKER_PAGE_SIZE / sizeof(u32) - 1] = 0xabcdef01u;
+    context.PrepareBda();
+    snapshot(16, 2);
+    snapshot(TRACKER_PAGE_SIZE - sizeof(u32), 3);
+    Require(name, "whole-page upload", !cache.IsRegionCpuModified(base, TRACKER_PAGE_SIZE),
+            "one-byte invalidation did not upload and consume its whole page");
+
+    context.MapMemory(base + page, page);
+    context.PrepareBda();
+    snapshot(page, 4);
+    Require(name, "new mapping", !cache.IsRegionCpuModified(base + page, page),
+            "mapping into an existing buffer did not queue its dirty pages");
+    context.UnmapMemory(base + 2 * page, page);
+    words[2 * page / sizeof(u32)] = 0xfedcba98u;
+    context.PrepareBda();
+    Require(name, "unmapped dirty page", cache.IsRegionCpuModified(base + 2 * page, page),
+            "BDA synchronization consumed dirtiness outside mapped memory");
+    context.MapMemory(base + 2 * page, page);
+    context.PrepareBda();
+    snapshot(2 * page, 5);
+
+    vk::BufferMemoryBarrier barrier{};
+    barrier.srcAccessMask = vk::AccessFlagBits::eTransferWrite;
+    barrier.dstAccessMask = vk::AccessFlagBits::eHostRead;
+    barrier.srcQueueFamilyIndex = barrier.dstQueueFamilyIndex = VK_QUEUE_FAMILY_IGNORED;
+    barrier.buffer = readback.buffer;
+    barrier.size = readback.size;
+    scheduler.Current().Handle().pipelineBarrier(vk::PipelineStageFlagBits::eTransfer,
+        vk::PipelineStageFlagBits::eHost, {}, 0, nullptr, 1, &barrier, 0, nullptr);
+    scheduler.Finish();
+    Require(name, "GPU contents", ReadBuffer(name, readback, 6) ==
+        std::vector<u32>{0x12345678u, 0x12345678u, 0x87654321u, 0xabcdef01u,
+                        0x12345678u, 0xfedcba98u},
+        "initial, re-dirtied, newly mapped or remapped bytes were stale on the GPU");
+    context.UnmapMemory(base, 3 * page);
+    scheduler.Finish();
+    DestroyBuffer(&readback);
+    Require(name, "release", LibKernel::Memory::FreeGuestMemory(base, 3 * page),
+            "guest allocation release failed");
+    std::printf("[gpu]     %-32s ok\n", name);
+  }
+
   void CheckBdaPageTableUploads() {
     constexpr const char *name = "BdaPageTableUploads";
     constexpr uintptr_t base = 0x0000000207b00000ull;
@@ -48472,6 +48551,7 @@ int main(int argc, char **argv) {
   if (argc == 2 && std::strcmp(argv[1], "--bda-page-table-only") == 0) {
     VulkanHarness vulkan;
     vulkan.CheckBdaPageTableUploads();
+    vulkan.CheckBdaSyncUploads();
     return 0;
   }
   if (argc == 2 && std::strcmp(argv[1], "--buffer-cache-range-only") == 0) {
@@ -48702,6 +48782,7 @@ int main(int argc, char **argv) {
   CheckComputeLdsLimit(vulkan);
   vulkan.CheckNullTextureDescriptors();
   vulkan.CheckBdaPageTableUploads();
+  vulkan.CheckBdaSyncUploads();
   vulkan.CheckLargeSparseImageUpload();
   vulkan.CheckUnifiedTextureCacheFlow();
   vulkan.CheckUnifiedImageViewCache();

tests/VirtualMemoryAllocationTests.cpp

@@ -8,6 +8,7 @@
 #include "kernel/pthread.h"
 #include "libs/errno.h"
 #include "loader/guestInstructionPatcher.h"
+#include "loader/jit.h"
 #include "loader/runtimeLinker.h"
 #include "loader/systemContent.h"
 #include "loader/x64InstructionEmulator.h"
@@ -4775,6 +4776,59 @@ void TestSmallFiberStacksAndMigration() {
 
 } // namespace
 
+#if defined(__linux__) || KYTY_PLATFORM == KYTY_PLATFORM_WINDOWS
+KYTY_MS_ABI uint8_t* TlsStubTestAddress() {
+	return reinterpret_cast<uint8_t*>(static_cast<uintptr_t>(0x123456789abcull));
+}
+
+void TestTlsStubsPreserveFlags() {
+	const char*    test      = "TlsStubsPreserveFlags";
+	const uint64_t stub_size = Loader::Jit::SafeCall::GetSize();
+	constexpr uint64_t CODE_SIZE = 0x1000;
+	constexpr uint8_t  REG       = 1;
+	const auto         mapping   = Libs::LibKernel::Memory::AllocateProgramMemory(
+	    0x0000000906000000ull, stub_size + CODE_SIZE, Common::VirtualMemory::Mode::ExecuteReadWrite,
+	    "tls_stub_test");
+	Check(test, mapping != 0, "failed to allocate TLS stub test code");
+	auto* safe_call = new (reinterpret_cast<void*>(mapping)) Loader::Jit::SafeCall;
+	safe_call->SetFunc(TlsStubTestAddress);
+	const auto stub_address = mapping + Loader::Jit::TlsRegStub::GetOffset(REG);
+	auto*      stub         = new (reinterpret_cast<void*>(stub_address)) Loader::Jit::TlsRegStub;
+	stub->SetFunc(mapping);
+	stub->SetOutputReg(REG);
+
+	using Fixture  = uint64_t(KYTY_SYSV_ABI*)(uint64_t*);
+	bool preserved = true;
+	for (const auto [input, status]: {std::pair {uint8_t {0x7f}, uint64_t {0x890}},
+	                                  std::pair {uint8_t {0xff}, uint64_t {0x055}}}) {
+		Xbyak::CodeGenerator code(CODE_SIZE, reinterpret_cast<void*>(mapping + stub_size));
+		code.push(code.rbx);
+		code.mov(code.rbx, code.rdi);
+		code.mov(code.rax, 0x5555555555555555ull);
+		code.mov(code.al, input);
+		code.add(code.al, 1);
+		code.mov(code.r11, stub_address);
+		code.call(code.r11);
+		code.mov(code.qword[code.rbx], code.rcx);
+		code.mov(code.qword[code.rbx + 8], code.rax);
+		code.pushf();
+		code.pop(code.rax);
+		code.pop(code.rbx);
+		code.ret();
+		Check(test, Xbyak::GetError() == 0, "failed to generate TLS stub fixture");
+		Check(test, Common::VirtualMemory::FlushInstructionCache(mapping, stub_size + CODE_SIZE),
+		      "failed to flush TLS stub test code");
+		std::array<uint64_t, 2> registers {};
+		const auto flags = reinterpret_cast<Fixture>(mapping + stub_size)(registers.data());
+		const auto rax   = (0x5555555555555555ull & ~0xffull) | static_cast<uint8_t>(input + 1u);
+		preserved &= (flags & 0x8d5u) == status && registers[0] == 0x123456789abcull &&
+		             registers[1] == rax;
+	}
+	Libs::LibKernel::Memory::FreeGuestMemory(mapping, stub_size + CODE_SIZE);
+	Check(test, preserved, "TLS stubs changed the caller's flags or registers");
+}
+#endif
+
 int main(int argc, char** argv) {
 	InitSubsystems();
 	if (argc == 2 && std::strcmp(argv[1], "--virtual-ranges-only") == 0) {
@@ -4919,6 +4973,9 @@ int main(int argc, char** argv) {
 	RunTest(TestMemoryPoolCommitDecommitQueryFlags);
 	RunTest(TestProgramMemoryAllocationAndProtection);
 	RunTest(TestModuleRelocationUsesWritableHostMapping);
+#if defined(__linux__) || KYTY_PLATFORM == KYTY_PLATFORM_WINDOWS
+	RunTest(TestTlsStubsPreserveFlags);
+#endif
 
 	if (g_failed_tests != 0) {
 		std::printf("VirtualMemoryAllocationTests: %d case(s) failed\n", g_failed_tests);

tests/shaderCfgTests.cpp

@@ -12874,6 +12874,51 @@ void TestLateConstantPropagation() {
         "an ungrounded Phi cycle acquired a constant");
 }
 
+void TestMaskedShiftConstantPropagation() {
+  using namespace ShaderRecompiler::IR;
+  for (const uint32_t count : {0u, 1u, 8u, 31u, 32u, 33u, 40u, 63u, 64u, UINT32_MAX}) {
+    for (const uint32_t mask : {0u, 1u, 0xffu, 0x7fffffffu, 0x80000000u, UINT32_MAX}) {
+      for (const bool swapped : {false, true}) {
+        for (const bool shared : {false, true}) {
+          Block block;
+          IREmitter ir(&block);
+          const auto input = ir.GetUserData(static_cast<ScalarReg>(0));
+          const auto shifted = ir.Emit(ValueOpcode::ShiftLeftLogical32, {input, Value(count)});
+          const auto result = ir.Emit(ValueOpcode::BitwiseAnd32,
+              swapped ? std::initializer_list<Value>{Value(mask), shifted}
+                      : std::initializer_list<Value>{shifted, Value(mask)});
+          const auto other = shared ? ir.Emit(ValueOpcode::IAdd32, {shifted, Value(1u)}) : Value{};
+          ConstantPropagationPass({&block});
+          const bool zero = ((UINT32_MAX << (count & 31u)) & mask) == 0u;
+          Check(zero ? result.Resolve() == Value(0u) : !result.Resolve().IsImmediate(),
+                "masked shift retained impossible bits or discarded possible bits");
+          if (shared) {
+            Check(other.Resolve().Instruction()->Arg(0).Resolve() == shifted.Resolve(),
+                  "folding a masked shift changed its other consumer");
+          }
+          constexpr uint32_t source = 0xa5f03c69u;
+          input.Instruction()->ReplaceUsesWith(Value(source));
+          ConstantPropagationPass({&block});
+          const uint32_t expected = source << (count & 31u);
+          Check(result.Resolve() == Value(expected & mask) &&
+                    (!shared || other.Resolve() == Value(expected + 1u)),
+                "masked shift folding changed a runtime result");
+        }
+      }
+    }
+  }
+
+  Block block;
+  IREmitter ir(&block);
+  const auto input = ir.GetUserData(static_cast<ScalarReg>(0));
+  const auto count = ir.GetUserData(static_cast<ScalarReg>(1));
+  const auto shifted = ir.Emit(ValueOpcode::ShiftLeftLogical32, {input, count});
+  const auto result = ir.Emit(ValueOpcode::BitwiseAnd32, {shifted, Value(0xffu)});
+  ConstantPropagationPass({&block});
+  Check(result.Resolve().Instruction()->GetOpcode() == ValueOpcode::BitwiseAnd32,
+        "a variable shift count was treated as a constant");
+}
+
 void TestU64ShiftConstantPropagation() {
   using namespace ShaderRecompiler::IR;
 
@@ -16364,6 +16409,7 @@ int main() {
   TestValuePhiValidation();
   TestLaneMaskProjection();
   TestLateConstantPropagation();
+  TestMaskedShiftConstantPropagation();
   TestU64ShiftConstantPropagation();
   TestU64RegisterPairRoundTrip();
 #if KYTY_PLATFORM != KYTY_PLATFORM_WINDOWS