Support mountable-content DLC (addcont with real content) (#1581 )

Support mountable-content DLC (addcont) and fix PlayGo chunk count query
shader: support returning 32-bit LDS compare-exchange (DS_CMPST_RTN_B32) (#1578 )
2026-10-24 12:56:53 +02:00 · 2026-10-17 04:44:28 +02:00 · 2026-10-17 04:37:16 +02:00 · 2026-10-17 04:37:00 +02:00 · 2026-10-17 04:09:54 +02:00 · 2026-10-17 04:09:54 +02:00
40 changed files with 995 additions and 103 deletions
@@ -350,7 +350,7 @@ set(inc_headers
 	${KYTY_THIRD_PARTY_DIR}/SPIRV-Tools/include
 	${KYTY_THIRD_PARTY_DIR}/SPIRV-Headers/include
 	${KYTY_THIRD_PARTY_DIR}/fmt/include
-	${KYTY_THIRD_PARTY_DIR}/xxhash
+	${KYTY_THIRD_PARTY_DIR}/xxHash
 	${KYTY_THIRD_PARTY_DIR}/cpuinfo/include
 	${KYTY_THIRD_PARTY_DIR}/renderdoc
 	${KYTY_THIRD_PARTY_DIR}/stb
@@ -66,6 +66,18 @@ RegionManager* MemoryTracker::GetOrCreateRegion(uint64_t index) {
 	return ptr;
 }

+void MemoryTracker::QueueBdaSync(uint64_t vaddr, uint64_t size) {
+	const auto begin = Common::AlignDown(vaddr, TRACKER_PAGE_SIZE);
+	const auto end   = Common::AlignUp(vaddr + size, TRACKER_PAGE_SIZE);
+	std::lock_guard lock(m_bda_sync_mutex);
+	m_bda_sync.Add(begin, end - begin);
+}
+
+RangeSet MemoryTracker::TakeBdaSync() {
+	std::lock_guard lock(m_bda_sync_mutex);
+	return std::exchange(m_bda_sync, RangeSet {});
+}
+
 bool MemoryTracker::IsRegionCpuModified(uint64_t vaddr, uint64_t size) {
 	CheckNotInUploadCallback();
 	return Iterate<true>(vaddr, size, [](RegionManager* manager, uint64_t offset, uint64_t bytes) {
@@ -88,6 +100,7 @@ void MemoryTracker::MarkRegionAsCpuModified(uint64_t vaddr, uint64_t size) {
 		std::scoped_lock lock(manager->lock);
 		manager->ChangeState<DirtySource::Cpu, true>(manager->GetCpuAddr() + offset, bytes);
 	});
+	QueueBdaSync(vaddr, size);
 }

 void MemoryTracker::MarkRegionAsGpuModified(uint64_t vaddr, uint64_t size) {
@@ -128,6 +141,8 @@ void MemoryTracker::UntrackMemory(uint64_t vaddr, uint64_t size) {
 	Iterate<false>(vaddr, size, [](RegionManager* manager, uint64_t offset, uint64_t bytes) {
 		manager->ChangeState<DirtySource::Cpu, true>(manager->GetCpuAddr() + offset, bytes);
 	});
+	locks.clear();
+	QueueBdaSync(vaddr, size);
 }

 } // namespace Libs::Graphics
@@ -29,6 +29,10 @@ public:
 	void               MarkRegionAsGpuModified(uint64_t vaddr, uint64_t size);
 	void               UnmarkRegionAsGpuModified(uint64_t vaddr, uint64_t size);
 	void               UntrackMemory(uint64_t vaddr, uint64_t size);
+	// Queue after changing ownership; a concurrent drain must not consume an event
+	// before the dirty pages it describes become visible.
+	void                   QueueBdaSync(uint64_t vaddr, uint64_t size);
+	[[nodiscard]] RangeSet TakeBdaSync();
 	// Removes protection from a range and flushes GPU-owned data when required.
 	template <typename Flush>
 	void InvalidateRegion(uint64_t vaddr, uint64_t size, Flush&& on_flush) noexcept {
@@ -51,6 +55,7 @@ public:
 				on_flush();
 			}
 		});
+		QueueBdaSync(vaddr, size);
 	}
 #if KYTY_BUILD == KYTY_BUILD_DEBUG
 	void ValidateGpuDirtyPages(const RangeSet& dirty, uint64_t vaddr, uint64_t size,
@@ -149,6 +154,8 @@ private:
 	std::unique_ptr<std::atomic<RegionManager*>[]> m_regions;
 	std::vector<std::unique_ptr<RegionManager>>    m_region_storage;
 	std::mutex                                     m_region_mutex;
+	std::mutex                                     m_bda_sync_mutex;
+	RangeSet                                       m_bda_sync;
 	PageManager&                                   m_page_manager;
 };

@@ -126,6 +126,7 @@ void BufferCache::ChangeRegister(BufferId id) {
 		}
 		WriteDataBuffer(m_bda_pagetable_buffer, table_offset,
 		                addresses.data(), addresses.size() * sizeof(vk::DeviceAddress));
+		m_memory_tracker.QueueBdaSync(buffer.CpuAddress(), buffer.Size());
 	} else {
 		const auto found = m_buffers.find(buffer.CpuAddress());
 		EXIT_IF(found == m_buffers.end() || found->second != id);
@@ -685,6 +686,16 @@ void BufferCache::ProcessFaultBuffer() {
 	m_fault_manager.ProcessFaultBuffer();
 }

+void BufferCache::SynchronizeBda(const RangeSet& mapped) {
+	// Take before walking: writes queued during synchronization remain pending.
+	const auto pending = m_memory_tracker.TakeBdaSync();
+	pending.ForEach([&](uint64_t begin, uint64_t end) {
+		mapped.ForEachInRange(begin, end - begin, [this](uint64_t start, uint64_t finish) {
+			SynchronizeBuffersInRange(start, finish - start);
+		});
+	});
+}
+
 void BufferCache::SynchronizeBuffersInRange(uint64_t vaddr, uint64_t size) {
 	const auto end = vaddr + size;
 	auto       it  = m_buffers.upper_bound(vaddr);
@@ -80,7 +80,10 @@ public:
 	[[nodiscard]] bool IsRegionCpuModified(uint64_t vaddr, uint64_t size);
 	[[nodiscard]] bool IsRegionGpuModified(uint64_t vaddr, uint64_t size);
 	void               ProcessFaultBuffer();
-	void               SynchronizeBuffersInRange(uint64_t vaddr, uint64_t size);
+	void               SynchronizeBda(const RangeSet& mapped);
+	void               QueueBdaSync(uint64_t vaddr, uint64_t size) {
+		m_memory_tracker.QueueBdaSync(vaddr, size);
+	}
 	void               RunGarbageCollector();

 private:
@@ -114,6 +117,7 @@ private:
 	void DeleteBuffer(BufferId id);
 	[[nodiscard]] bool SynchronizeBuffer(Buffer& buffer, uint64_t vaddr, uint64_t size,
 	                                     bool is_written, bool is_texel_buffer);
+	void SynchronizeBuffersInRange(uint64_t vaddr, uint64_t size);
 	[[nodiscard]] vk::Buffer UploadCopies(Buffer& buffer, std::span<vk::BufferCopy> copies,
 	                                      uint64_t total_size);
 	[[nodiscard]] bool SynchronizeBufferFromImage(Buffer& buffer, uint64_t vaddr, uint64_t size);
@@ -67,11 +67,9 @@ Buffer::Buffer(GraphicContext& graphics, CommandScheduler& scheduler, MemoryUsag
 	buffer_info.usage       = flags;

 	const bool with_bda = bool(flags & vk::BufferUsageFlagBits::eShaderDeviceAddress);
-	const VmaAllocationCreateFlags bda_flag =
-	    with_bda ? VMA_ALLOCATION_CREATE_DEDICATED_MEMORY_BIT : 0;
 	VmaAllocationCreateInfo allocation_info {};
 	allocation_info.flags =
-	    VMA_ALLOCATION_CREATE_WITHIN_BUDGET_BIT | bda_flag | AllocationFlags(usage);
+	    VMA_ALLOCATION_CREATE_WITHIN_BUDGET_BIT | AllocationFlags(usage);
 	allocation_info.usage = AllocationUsage(usage);
 	allocation_info.preferredFlags = usage == MemoryUsage::DeviceLocal
 	                                     ? VkMemoryPropertyFlags {}
@@ -18,7 +18,6 @@
 #include "graphics/shader/recompiler/ShaderRecompiler.h"
 #include "graphics/shader/shaderCompiler.h"
 #include "kernel/memory.h"
-#include "kytyGitVersion.h"
 #include "loader/systemContent.h"

 #include <algorithm>
@@ -81,7 +80,8 @@ std::string DriverCacheSignature(const vk::PhysicalDeviceProperties& properties)
 		uuid[i * 2]     = hex[properties.pipelineCacheUUID[i] >> 4u];
 		uuid[i * 2 + 1] = hex[properties.pipelineCacheUUID[i] & 0xfu];
 	}
-	return fmt::format("KytyPC1:{}:{:08x}:{:08x}:{:08x}:{}\n", KYTY_GIT_REVISION,
+	// Driver cache entries already distinguish shader code and pipeline state across builds.
+	return fmt::format("KytyPC2:{:08x}:{:08x}:{:08x}:{}\n",
 	                   properties.vendorID, properties.deviceID, properties.driverVersion, uuid);
 }

@@ -516,17 +516,6 @@ void PipelineCache::InitializeDriverCache() {
 		PipelineCacheLog("Vulkan pipeline cache: disabled (non-Release build)");
 		return;
 	}
-	const std::string_view git_hash     = KYTY_GIT_HASH;
-	const std::string_view git_revision = KYTY_GIT_REVISION;
-	if (git_hash == "unknown" || git_revision == "unknown") {
-		PipelineCacheLog("Vulkan pipeline cache: disabled (unknown git revision)");
-		return;
-	}
-	if (git_hash.ends_with("-dirty")) {
-		PipelineCacheLog("Vulkan pipeline cache: disabled (dirty build)");
-		return;
-	}
-
 	m_driver_cache_path     = std::filesystem::path("_PipelineCache") / (title_id + ".bin");
 	const auto path         = Common::PathToString(m_driver_cache_path);
 	const bool cache_exists = Common::File::IsFileExisting(m_driver_cache_path);
@@ -559,7 +548,7 @@ void PipelineCache::InitializeDriverCache() {
 			    XXH3_64bits(initial_data.data(), initial_data.size()) != payload_hash) {
 				initial_data.clear();
 				PipelineCacheLog(
-				    "Vulkan pipeline cache: invalidating {} (driver, emulator, or data mismatch)",
+				    "Vulkan pipeline cache: invalidating {} (driver or data mismatch)",
 				    path);
 			}
 		} else {
@@ -94,9 +94,10 @@ bool ResolveComputeBufferFill(const ShaderComputeInputInfo& input, uint32_t grou
 	constexpr std::array formats {
 	    Prospero::BufferFormat::k32UInt, Prospero::BufferFormat::k32_32UInt,
 	    Prospero::BufferFormat::k32_32_32UInt, Prospero::BufferFormat::k32_32_32_32UInt};
+	// Offset-or-swizzle bounds do not describe the element range proved by the shader.
 	if (descriptor.Stride() != element_size || descriptor.Format() != formats[fill.words - 1] ||
 	    descriptor.SwizzleEnabled() || descriptor.IndexStride() != 0 || descriptor.AddTid() ||
-	    descriptor.Base48() == 0) {
+	    descriptor.OutOfBounds() == 3 || descriptor.Base48() == 0) {
 		return false;
 	}
 	if (input.threads_num[0] == 0 || input.threads_num[0] != fill.group_stride[0] ||
@@ -108,7 +109,11 @@ bool ResolveComputeBufferFill(const ShaderComputeInputInfo& input, uint32_t grou
 	                                 ? group_x
 	                                 : static_cast<uint64_t>(group_x) * input.threads_num[0];
 	const auto     size        = descriptor.GetSize();
-	if (invocations != descriptor.NumRecords() || size == 0 || size > UINT32_MAX ||
+	const auto elements = descriptor.NumRecords();
+	const bool covered = fill.bounded_elements != 0
+	                         ? fill.bounded_elements == elements && invocations >= elements
+	                         : invocations == elements;
+	if (!covered || invocations > uint64_t {UINT32_MAX} + 1 || size == 0 || size > UINT32_MAX ||
 	    (input.dispatch_thread_dimensions &&
 	     (group_x % input.threads_num[0] != 0 || input.dispatch_threads_num[0] != group_x ||
 	      input.dispatch_threads_num[1] != 1 || input.dispatch_threads_num[2] != 1))) {
@@ -252,10 +257,6 @@ void RenderExecutor::DispatchDirect(uint64_t submit_id, CommandBuffer& buffer,
 		return;
 	}

-	if (sh_ctx.GetCs().cs_regs.data_addr == 0) {
-		return;
-	}
-
 	constexpr uint32_t DISPATCH_INITIATOR_USE_THREAD_DIMENSIONS = 1u << 5u;
 	constexpr uint32_t DISPATCH_INITIATOR_BASE_BITS             = 0x41u;
 	constexpr uint32_t DISPATCH_INITIATOR_MODIFIER_BITS         = 0xa038u;
@@ -90,6 +90,7 @@ bool RenderContext::IsMapped(uint64_t vaddr, uint64_t size) const noexcept {
 void RenderContext::MapMemory(uint64_t vaddr, uint64_t size) {
 	std::lock_guard lock(m_mapped_ranges_mutex);
 	m_mapped_ranges.Add(vaddr, size);
+	m_buffer_cache.QueueBdaSync(vaddr, size);
 }

 void RenderContext::UnmapMemory(uint64_t vaddr, uint64_t size) {
@@ -129,9 +130,7 @@ void RenderContext::PrepareBda() {
 		m_bda_logged = true;
 	}
 	std::shared_lock lock(m_mapped_ranges_mutex);
-	m_mapped_ranges.ForEach([this](uint64_t start, uint64_t end) {
-		m_buffer_cache.SynchronizeBuffersInRange(start, end - start);
-	});
+	m_buffer_cache.SynchronizeBda(m_mapped_ranges);
 	m_fault_process_pending = true;
 }

@@ -536,7 +536,10 @@ RenderState RenderExecutor::AcquireRenderTargets(CommandBuffer& buffer, RenderCo
 			}
 		}
 		if (feedback_aspects && !m_context.GetGraphics().attachment_feedback_loop_enabled) {
-			EXIT("depth attachment feedback loop is not supported by the host\n");
+			static std::atomic_bool logged = false;
+			if (!logged.exchange(true, std::memory_order_relaxed)) {
+				std::printf("depth attachment feedback loop is not supported by the host\n");
+			}
 		}
 		auto layout = depth_attachment_layout(depth);
 		if (sampled_aspects & ~DepthReadableAspects(layout)) {
@@ -162,7 +162,8 @@ void RenderDocOnGuestFlip(RenderContext& renderer) {
 		return;
 	}
 	if (state == RenderDocState::Capturing) {
-		LOGF("RenderDoc: captured guest flip %u/2\n", ++g_captured_flips);
+		++g_captured_flips;
+		LOGF("RenderDoc: captured guest flip %u/2\n", g_captured_flips);
 		if (g_captured_flips < 2) {
 			return;
 		}
@@ -945,7 +945,6 @@ Presenter::Frame& Presenter::PrepareBlankFrame(uint32_t width, uint32_t height,
 	auto*             frame  = m_impl->frames.Acquire({width, height}, format);
 	frame->hdr   = false;
 	frame->blank = true;
-	Common::LockGuard render_lock(m_impl->renderer.GetMutex());
 	frame->Configure(m_impl->window.graphic_ctx, {width, height}, format);
 	vk::ClearColorValue clear {};
 	clear.float32[3] = opaque ? 1.0f : 0.0f;
@@ -953,6 +952,7 @@ Presenter::Frame& Presenter::PrepareBlankFrame(uint32_t width, uint32_t height,
 		EXIT_IF(producer->IsInvalid());
 		frame->Clear(*producer, clear);
 	} else {
+		Common::LockGuard lock(m_impl->present_mutex);
 		auto& command = m_impl->present_scheduler.BeginCommand();
 		frame->Clear(command, clear);
 		frame->present_tick = m_impl->present_scheduler.Submit();
@@ -1036,18 +1036,17 @@ void Presenter::Impl::Present() {
 			RecoverSwapchain(status);
 			continue;
 		}
-		{
-			Common::LockGuard render_lock(renderer.GetMutex());
-			auto&             command = present_scheduler.BeginCommand();
-			const bool        draw_system_overlay =
-			    overlay_visual.active && swapchain.PrepareSystemOverlay();
-			swapchain.RecordPresentCommands(command, layers[0].frame, layers[1],
-			                                draw_system_overlay);
-			const auto tick = swapchain.Submit(present_scheduler);
-			for (const auto& layer: layers) {
-				if (layer.frame != nullptr) {
-					layer.frame->present_tick = tick;
-				}
+		// present_mutex owns the prepared frames and this scheduler. Only queue submission
+		// is shared with rendering, and it has its own queue_mutex.
+		auto&      command = present_scheduler.BeginCommand();
+		const bool draw_system_overlay =
+		    overlay_visual.active && swapchain.PrepareSystemOverlay();
+		swapchain.RecordPresentCommands(command, layers[0].frame, layers[1],
+		                                draw_system_overlay);
+		const auto tick = swapchain.Submit(present_scheduler);
+		for (const auto& layer: layers) {
+			if (layer.frame != nullptr) {
+				layer.frame->present_tick = tick;
 			}
 		}
 		status = swapchain.Present();
@@ -230,8 +230,11 @@ uint32_t EmitPackFloat2x16Rtz(EmitterState& state, uint32_t arg0, uint32_t arg1)
 }

 uint32_t EmitFPSaturate32(EmitterState& state, uint32_t arg0) {
-	return EmitExt(state, TypeF32(state), GLSLstd450FClamp,
-	               {arg0, ConstantF32(state, 0), ConstantF32(state, 0x3f800000u)});
+	// The clamp modifier runs in DX10 clamp mode: NaN becomes 0. FClamp leaves NaN undefined.
+	const auto clamped = EmitExt(state, TypeF32(state), GLSLstd450FClamp,
+	                             {arg0, ConstantF32(state, 0), ConstantF32(state, 0x3f800000u)});
+	return Select(state, TypeF32(state), EmitClassifyF32(state, arg0).nan, ConstantF32(state, 0),
+	              clamped);
 }

 uint32_t EmitSMulHi(EmitterState& state, uint32_t arg0, uint32_t arg1) {
@@ -296,6 +296,7 @@ inline constexpr auto EmitBufferAtomicSMax32    = EmitAtomic32;
 inline constexpr auto EmitBufferAtomicUMax32    = EmitAtomic32;
 inline constexpr auto EmitBufferAtomicAnd32     = EmitAtomic32;
 inline constexpr auto EmitBufferAtomicAnd64     = EmitBufferAtomic64;
+inline constexpr auto EmitBufferAtomicIAdd64    = EmitBufferAtomic64;
 inline constexpr auto EmitBufferAtomicOr32      = EmitAtomic32;
 inline constexpr auto EmitBufferAtomicOr64      = EmitBufferAtomic64;
 inline constexpr auto EmitBufferAtomicXor32     = EmitAtomic32;
@@ -325,6 +326,7 @@ inline constexpr auto EmitSharedAtomicSMin32    = EmitAtomic32;
 inline constexpr auto EmitSharedAtomicUMin32    = EmitAtomic32;
 inline constexpr auto EmitSharedAtomicSMax32    = EmitAtomic32;
 inline constexpr auto EmitSharedAtomicUMax64    = EmitSharedAtomic64;
+inline constexpr auto EmitSharedAtomicCmpSwap32 = EmitAtomic32;
 inline constexpr auto EmitSharedAtomicCmpSwap64 = EmitSharedAtomic64;
 inline constexpr auto EmitSharedAtomicUMax32    = EmitAtomic32;
 inline constexpr auto EmitSharedAtomicAnd32     = EmitAtomic32;
@@ -504,6 +504,7 @@ uint32_t EncodePackedColorComponent(EmitterState& state, const Format::BufferFor

 spv::Op SpirvAtomicOpcode(IR::ValueOpcode opcode) {
 	switch (opcode) {
+		case IR::ValueOpcode::SharedAtomicCmpSwap32:
 		case IR::ValueOpcode::SharedAtomicCmpSwap64:
 		case IR::ValueOpcode::ImageAtomicCompareSwap32:
 		case IR::ValueOpcode::BufferAtomicCmpSwap32: return spv::OpAtomicCompareExchange;
@@ -516,6 +517,7 @@ spv::Op SpirvAtomicOpcode(IR::ValueOpcode opcode) {
 		case IR::ValueOpcode::ImageAtomicIAdd64:
 		case IR::ValueOpcode::BufferAtomicIAdd32:
 		case IR::ValueOpcode::SharedAtomicIAdd64:
+		case IR::ValueOpcode::BufferAtomicIAdd64:
 		case IR::ValueOpcode::SharedAtomicIAdd32: return spv::OpAtomicIAdd;
 		case IR::ValueOpcode::BufferAtomicISub32:
 		case IR::ValueOpcode::SharedAtomicISub32: return spv::OpAtomicISub;
@@ -560,8 +562,12 @@ uint32_t EmitAtomicOperation(ValueEmitContext& ctx, const IR::Inst& inst, uint32
 	const bool wide   = inst.GetType() == IR::Type::U64;
 	const auto type   = wide ? TypeU64(ctx.state) : TypeU32(ctx.state);
 	if (opcode == spv::OpAtomicCompareExchange) {
-		const auto desired    = ctx.Arg(inst, inst.NumArgs() - 3);
-		const auto comparator = ctx.Arg(inst, inst.NumArgs() - 2);
+		auto desired    = ctx.Arg(inst, inst.NumArgs() - 3);
+		auto comparator = ctx.Arg(inst, inst.NumArgs() - 2);
+		if (inst.GetOpcode() == IR::ValueOpcode::SharedAtomicCmpSwap32) {
+			// RDNA2 DS compares DATA and stores DATA2, opposite to BUFFER atomics.
+			std::swap(desired, comparator);
+		}
 		ctx.state.builder.AddFunction(
 		    spv::OpAtomicCompareExchange, TypeU32(ctx.state), old, pointer,
 		    ConstantU32(ctx.state, scope), ConstantU32(ctx.state, spv::MemorySemanticsMaskNone),
@@ -67,6 +67,7 @@ constexpr MemoryOpcodeInfo MUBUF_OPCODE_LIST[] = {
    {0x3fu, Opcode::BUFFER_ATOMIC_FMIN, 1, 32},
    {0x40u, Opcode::BUFFER_ATOMIC_FMAX, 1, 32},
    {0x50u, Opcode::BUFFER_ATOMIC_SWAP_X2, 2, 32},
+    {0x52u, Opcode::BUFFER_ATOMIC_ADD_X2, 2, 32},
    {0x59u, Opcode::BUFFER_ATOMIC_AND_X2, 2, 32},
    {0x5au, Opcode::BUFFER_ATOMIC_OR_X2, 2, 32},
    {0x80u, Opcode::BUFFER_LOAD_FORMAT_D16_X, 1, 16, false, false, true},
@@ -114,6 +115,7 @@ constexpr MemoryOpcodeInfo DS_OPCODE_LIST[] = {
    {0x27u, Opcode::DS_MIN_RTN_U32, 1, 32},      {0x28u, Opcode::DS_MAX_RTN_U32, 1, 32},
    {0x29u, Opcode::DS_AND_RTN_B32, 1, 32},      {0x2au, Opcode::DS_OR_RTN_B32, 1, 32},
    {0x2bu, Opcode::DS_XOR_RTN_B32, 1, 32},      {0x2du, Opcode::DS_WRXCHG_RTN_B32, 1, 32},
+    {0x30u, Opcode::DS_CMPST_RTN_B32, 1, 32},
    {0x35u, Opcode::DS_SWIZZLE_B32, 1, 32},      {0x36u, Opcode::DS_READ_B32, 1, 32},
    {0x37u, Opcode::DS_READ2_B32, 2, 32},        {0x38u, Opcode::DS_READ2ST64_B32, 2, 32},
    {0x39u, Opcode::DS_READ_I8, 1, 8, true},     {0x3au, Opcode::DS_READ_U8, 1, 8},
@@ -200,6 +202,7 @@ bool IsDsAtomicOpcode(Opcode opcode) {
 		case Opcode::DS_MAX_U32:
 		case Opcode::DS_MAX_RTN_U32:
 		case Opcode::DS_MAX_RTN_U64:
+		case Opcode::DS_CMPST_RTN_B32:
 		case Opcode::DS_CMPST_RTN_B64:
 		case Opcode::DS_AND_B32:
 		case Opcode::DS_AND_RTN_B32:
@@ -220,6 +223,7 @@ uint32_t DsSourceCount(Opcode opcode) {
 		case Opcode::DS_WRITE2ST64_B32:
 		case Opcode::DS_WRITE2_B64:
 		case Opcode::DS_WRITE2ST64_B64:
+		case Opcode::DS_CMPST_RTN_B32:
 		case Opcode::DS_CMPST_RTN_B64:
 		case Opcode::DS_MSKOR_B32: return 3u;
 		case Opcode::DS_MIN_F32:
@@ -454,6 +458,9 @@ void DecodeDs(uint32_t pc, std::span<const uint32_t> code, uint32_t word_index,
 	if (inst.opcode == Opcode::DS_SWIZZLE_B32 && inst.offset >= 0xe000u) {
 		SetUnsupported(inst, Family::DS, opcode, "DS swizzle FFT mode is not implemented");
 	}
+	if (inst.gds && inst.opcode == Opcode::DS_CMPST_RTN_B32) {
+		SetUnsupported(inst, Family::DS, opcode, "DS compare-store return B32 GDS variant is not implemented");
+	}
 	if (inst.gds &&
 	    (inst.opcode == Opcode::DS_SWIZZLE_B32 || inst.opcode == Opcode::DS_PERMUTE_B32 ||
 	     inst.opcode == Opcode::DS_BPERMUTE_B32 ||
@@ -670,6 +670,7 @@ std::string InstructionToString(const Instruction& inst) {
 		case Opcode::BUFFER_ATOMIC_CMPSWAP:
 		case Opcode::BUFFER_ATOMIC_SWAP_X2:
 		case Opcode::BUFFER_ATOMIC_ADD:
+		case Opcode::BUFFER_ATOMIC_ADD_X2:
 		case Opcode::BUFFER_ATOMIC_SUB:
 		case Opcode::BUFFER_ATOMIC_SMIN:
 		case Opcode::BUFFER_ATOMIC_UMIN:
@@ -717,6 +718,7 @@ std::string InstructionToString(const Instruction& inst) {
 		case Opcode::DS_MAX_U32:
 		case Opcode::DS_MAX_RTN_U32:
 		case Opcode::DS_MAX_RTN_U64:
+		case Opcode::DS_CMPST_RTN_B32:
 		case Opcode::DS_CMPST_RTN_B64:
 		case Opcode::DS_AND_B32:
 		case Opcode::DS_AND_RTN_B32:
@@ -586,6 +586,7 @@ enum class Opcode {
 	BUFFER_ATOMIC_CMPSWAP,
 	BUFFER_ATOMIC_SWAP_X2,
 	BUFFER_ATOMIC_ADD,
+	BUFFER_ATOMIC_ADD_X2,
 	BUFFER_ATOMIC_SUB,
 	BUFFER_ATOMIC_SMIN,
 	BUFFER_ATOMIC_UMIN,
@@ -631,6 +632,7 @@ enum class Opcode {
 	DS_MAX_U32,
 	DS_MAX_RTN_U32,
 	DS_MAX_RTN_U64,
+	DS_CMPST_RTN_B32,
 	DS_CMPST_RTN_B64,
 	DS_AND_B32,
 	DS_AND_RTN_B32,
@@ -1046,6 +1046,8 @@ void Translator::EmitMemory(const Decoder::Instruction& inst) {
 			return BUFFER_ATOMIC(inst, IR::ValueOpcode::BufferAtomicFMin32);
 		case Decoder::Opcode::BUFFER_ATOMIC_FMAX:
 			return BUFFER_ATOMIC(inst, IR::ValueOpcode::BufferAtomicFMax32);
+		case Decoder::Opcode::BUFFER_ATOMIC_ADD_X2:
+			return BUFFER_ATOMIC(inst, IR::ValueOpcode::BufferAtomicIAdd64);

 		case Decoder::Opcode::DS_ADD_U32:
 			return DS_ATOMIC(inst, IR::ValueOpcode::SharedAtomicIAdd32, false);
@@ -1083,6 +1085,8 @@ void Translator::EmitMemory(const Decoder::Instruction& inst) {
 			return DS_ATOMIC(inst, IR::ValueOpcode::SharedAtomicUMax32, true);
 		case Decoder::Opcode::DS_MAX_RTN_U64:
 			return DS_ATOMIC(inst, IR::ValueOpcode::SharedAtomicUMax64, true);
+		case Decoder::Opcode::DS_CMPST_RTN_B32:
+			return DS_ATOMIC(inst, IR::ValueOpcode::SharedAtomicCmpSwap32, true);
 		case Decoder::Opcode::DS_CMPST_RTN_B64:
 			return DS_ATOMIC(inst, IR::ValueOpcode::SharedAtomicCmpSwap64, true);
 		case Decoder::Opcode::DS_AND_B32:
@@ -25,6 +25,7 @@ struct UniformFill {
 	std::array<uint32_t, 3> group_stride {};
 	uint32_t                 words        = 0;
 	uint32_t                 value        = 0;
+	uint32_t                 bounded_elements = 0;

 	bool operator==(const UniformFill&) const = default;
 };
@@ -622,6 +622,8 @@ struct CompiledShaderInfo {
 struct UniformFillPlan {
 	UniformFill          fill;
 	std::array<Value, 4> values;
+	Value                pattern_size;
+	Value                element_count;
 };

 // Resource analysis retained by the shader cache. It owns immutable descriptor/SRT,
@@ -118,6 +118,7 @@ BufferAccess BufferAccessOf(ValueOpcode opcode) {
 		case ValueOpcode::BufferAtomicCmpSwap32:
 		case ValueOpcode::BufferAtomicSwap64:
 		case ValueOpcode::BufferAtomicIAdd32:
+		case ValueOpcode::BufferAtomicIAdd64:
 		case ValueOpcode::BufferAtomicISub32:
 		case ValueOpcode::BufferAtomicSMin32:
 		case ValueOpcode::BufferAtomicUMin32:
@@ -137,6 +138,7 @@ BufferAccess BufferAccessOf(ValueOpcode opcode) {
 uint32_t BufferComponentCount(ValueOpcode opcode) {
 	switch (opcode) {
 		case ValueOpcode::BufferAtomicSwap64:
+		case ValueOpcode::BufferAtomicIAdd64:
 		case ValueOpcode::BufferAtomicAnd64:
 		case ValueOpcode::BufferAtomicOr64:
 		case ValueOpcode::LoadBufferU32x2:
@@ -176,6 +178,7 @@ SharedAccess SharedAccessOf(ValueOpcode opcode) {
 		case ValueOpcode::SharedAtomicSMax32:
 		case ValueOpcode::SharedAtomicUMax32:
 		case ValueOpcode::SharedAtomicUMax64:
+		case ValueOpcode::SharedAtomicCmpSwap32:
 		case ValueOpcode::SharedAtomicCmpSwap64:
 		case ValueOpcode::SharedAtomicAnd32:
 		case ValueOpcode::SharedAtomicOr32:
@@ -268,6 +268,7 @@ VALUE_OPCODE(BufferAtomicSwap32, U32, BufferResource, U32, U32, U32, U32, U1)
 VALUE_OPCODE(BufferAtomicCmpSwap32, U32, BufferResource, U32, U32, U32, U32, U32, U1)
 VALUE_OPCODE(BufferAtomicSwap64, U64, BufferResource, U32, U32, U32, U64, U1)
 VALUE_OPCODE(BufferAtomicIAdd32, U32, BufferResource, U32, U32, U32, U32, U1)
+VALUE_OPCODE(BufferAtomicIAdd64, U64, BufferResource, U32, U32, U32, U64, U1)
 VALUE_OPCODE(BufferAtomicISub32, U32, BufferResource, U32, U32, U32, U32, U1)
 VALUE_OPCODE(BufferAtomicSMin32, U32, BufferResource, U32, U32, U32, U32, U1)
 VALUE_OPCODE(BufferAtomicUMin32, U32, BufferResource, U32, U32, U32, U32, U1)
@@ -306,6 +307,7 @@ VALUE_OPCODE(SharedAtomicUMin32, U32, U32, U32, U1)
 VALUE_OPCODE(SharedAtomicSMax32, U32, U32, U32, U1)
 VALUE_OPCODE(SharedAtomicUMax32, U32, U32, U32, U1)
 VALUE_OPCODE(SharedAtomicUMax64, U64, U32, U64, U1)
+VALUE_OPCODE(SharedAtomicCmpSwap32, U32, U32, U32, U32, U1)
 VALUE_OPCODE(SharedAtomicCmpSwap64, U64, U32, U64, U64, U1)
 VALUE_OPCODE(SharedAtomicAnd32, U32, U32, U32, U1)
 VALUE_OPCODE(SharedAtomicOr32, U32, U32, U32, U1)
@@ -608,13 +608,23 @@ void FoldInstruction(Block& block, Block::iterator instruction,
 			    ReplaceBinaryIdentity(inst, Type::U32, UINT32_MAX)) return;
 			for (uint32_t arm = 0; arm < 2u; ++arm) {
 				const auto mask = Arg(inst, arm);
+				if (!IsImmediate(mask, Type::U32)) continue;
+				if (mask.U32() == 0u) {
+					Replace(inst, Value(0u));
+					return;
+				}
 				const auto* shifted = Arg(inst, arm ^ 1u).TryInstruction();
-				if (!IsImmediate(mask, Type::U32) || shifted == nullptr ||
-				    shifted->GetOpcode() != ValueOpcode::ShiftLeftLogical32 || !HasOneUse(*shifted)) continue;
+				if (shifted == nullptr || shifted->GetOpcode() != ValueOpcode::ShiftLeftLogical32) continue;
 				const auto shift = Arg(*shifted, 1);
-				if (!IsImmediate(shift, Type::U32) || shift.U32() >= 32u) continue;
+				if (!IsImmediate(shift, Type::U32)) continue;
+				const auto shifted_mask = mask.U32() >> (shift.U32() & 31u);
+				if (shifted_mask == 0u) {
+					Replace(inst, Value(0u));
+					return;
+				}
+				if (!HasOneUse(*shifted) || shift.U32() >= 32u) continue;
 				const auto masked = worklist.Prepend(block, instruction, ValueOpcode::BitwiseAnd32,
-				    {Arg(*shifted, 0), Value(mask.U32() >> shift.U32())});
+				    {Arg(*shifted, 0), Value(shifted_mask)});
 				Replace(inst, Value(&*worklist.Prepend(block, instruction, ValueOpcode::ShiftLeftLogical32,
 				    {Value(&*masked), shift})));
 				return;
@@ -1424,10 +1424,11 @@ ResourcePlan ExtractResourcePlan(const Program& program) {
 		for (auto it = first; it != plan.value_storage.end(); ++it) it->Invalidate();
 		plan.value_storage.erase(first, plan.value_storage.end());
 	}
-	plan.uniform_fill = AnalyzeUniformFill(program);
-	for (uint32_t i = 0; i < plan.uniform_fill.fill.words; ++i) {
-		plan.uniform_fill.values[i] = Clone(plan.uniform_fill.values[i]);
-	}
+	plan.uniform_fill = program.uniform_fill.fill.kind != UniformFillKind::None
+	    ? program.uniform_fill : AnalyzeUniformFill(program);
+	for (auto& value: plan.uniform_fill.values) value = Clone(value);
+	plan.uniform_fill.pattern_size = Clone(plan.uniform_fill.pattern_size);
+	plan.uniform_fill.element_count = Clone(plan.uniform_fill.element_count);
 	plan.capture_specialization_reads |=
 	    capture_indirect_reads || capture_control_reads || !plan.source_reads.empty();
 	if (plan.capture_specialization_reads) {
@@ -1513,15 +1514,21 @@ bool MaterializeResources(const ResourcePlan& program, const SrtRuntime& runtime
 	}
 	snapshot.uniform_fill = {};
 	const auto& fill = program.uniform_fill;
-	const auto words = fill.fill.words;
+	auto words = fill.fill.words;
 	std::array<uint32_t, 4> stored {};
 	bool uniform_fill = words != 0;
+	if (uniform_fill && !fill.pattern_size.IsEmpty())
+		uniform_fill = clean.Evaluate(fill.pattern_size, words) && words >= 1 && words <= 4;
+	uint32_t elements = 0;
+	if (uniform_fill && !fill.element_count.IsEmpty())
+		uniform_fill = clean.Evaluate(fill.element_count, elements) && elements != 0;
 	for (uint32_t i = 0; i < words && uniform_fill; ++i) {
 		uniform_fill = clean.Evaluate(fill.values[i], stored[i]) && stored[i] == stored[0];
 	}
 	if (uniform_fill) {
 		snapshot.uniform_fill = fill.fill;
 		snapshot.uniform_fill.value = stored[0];
+		snapshot.uniform_fill.bounded_elements = elements;
 	}
 	const auto evaluate = [&](uint32_t source, DescriptorValue& value, bool written = false) {
 		if (source >= program.descriptor_sources.size()) {
@@ -15,6 +15,8 @@
 #include <tuple>
 #include <unordered_map>
 #include <utility>
+#define XXH_INLINE_ALL
+#include <xxhash.h>

 namespace Libs::Graphics::ShaderRecompiler::IR {
 namespace {
@@ -4040,11 +4042,71 @@ private:
 	bool                                       m_shader_writes = false;
 };

+UniformFillPlan RecognizePatternFill(Program& program, const Decoder::Program& decoded,
+                                   ShaderStageInputInfo input) {
+	struct Pattern {
+		size_t code_words;
+		uint64_t hash;
+		uint32_t wave_size;
+		bool bounded;
+	};
+	constexpr std::array patterns {
+		// Bounded 64-lane variant.
+		Pattern {.code_words = 69, .hash = 0x0704b9818e3cba06ull, .wave_size = 64, .bounded = true},
+		// Dispatch thread dimensions, 64-lane variants.
+		Pattern {.code_words = 42, .hash = 0x92ce57f21310ef02ull, .wave_size = 64, .bounded = false},
+		Pattern {.code_words = 40, .hash = 0x374161027f58bee7ull, .wave_size = 64, .bounded = false},
+		Pattern {.code_words = 40, .hash = 0x7d7172c56c2e296dull, .wave_size = 64, .bounded = false},
+		// Dispatch thread dimensions, 32-lane variant.
+		Pattern {.code_words = 38, .hash = 0xacbda1a1ccdd4198ull, .wave_size = 32, .bounded = false},
+	};
+	if (program.stage != ShaderType::Compute || input.compute == nullptr ||
+	    decoded.instructions.empty() || program.user_data_base != 0 || program.user_data_count < 10 ||
+	    program.info.buffers.size() != 1 || !program.info.images.empty()) return {};
+	const auto& cs = *input.compute;
+	const bool matching_inputs = cs.workgroup_register == 10 && cs.thread_ids_num == 1 &&
+	    cs.group_id[0] && !cs.group_id[1] && !cs.group_id[2] && cs.threads_num[0] == 64 &&
+	    cs.threads_num[1] == 1 && cs.threads_num[2] == 1 && cs.float_mode == 0xc0;
+	if (!matching_inputs) return {};
+	const auto& last = decoded.instructions.back();
+	const size_t words = last.pc / sizeof(uint32_t) + last.word_count;
+	if (words > decoded.code.size() ||
+	    std::ranges::find(patterns, words, &Pattern::code_words) == patterns.end()) return {};
+	// Hash executable instructions through S_ENDPGM, excluding binary metadata.
+	// Verified variants implement Destination[id] = value[id % pattern_size].
+	const auto hash = XXH3_64bits(decoded.code.data(), words * sizeof(uint32_t));
+	const auto pattern = std::ranges::find_if(patterns, [&](const Pattern& candidate) {
+		return candidate.code_words == words && candidate.hash == hash;
+	});
+	if (pattern == patterns.end() || program.wave_size != pattern->wave_size ||
+	    cs.dispatch_thread_dimensions != !pattern->bounded) return {};
+	std::array<Value, 10> userdata;
+	for (auto* block: program.blocks) {
+		for (auto& inst: *block) {
+			if (inst.GetOpcode() != ValueOpcode::GetUserData) continue;
+			const auto reg = RegIndex(inst.Arg(0).ScalarRegister());
+			if (reg < userdata.size()) userdata[reg] = Value(&inst);
+		}
+	}
+	for (uint32_t reg = 4; reg < 10; ++reg)
+		if ((reg != 8 || pattern->bounded) && userdata[reg].IsEmpty()) return {};
+	UniformFillPlan result;
+	result.fill.kind = UniformFillKind::Buffer;
+	result.fill.words = 1;
+	result.fill.group_stride[0] = 64;
+	std::copy_n(userdata.begin() + 4, 4, result.values.begin());
+	result.pattern_size = userdata[9];
+	// Only the bounded variant reads an explicit count; the other uses thread dimensions.
+	if (pattern->bounded) result.element_count = userdata[8];
+	return result;
+}
+
 } // namespace

 void TrackResources(Program& program, const Decoder::Program& decoded, const CFG::Graph& native_cfg,
                    ShaderStageInputInfo input) {
 	Tracker(program, decoded, native_cfg, input).Run();
+	program.uniform_fill = RecognizePatternFill(program, decoded, input);
 }

 } // namespace Libs::Graphics::ShaderRecompiler::IR
@@ -338,14 +338,20 @@ static bool HasWindowsForbiddenFilenameCharacter(const std::string& relative_pat
 std::filesystem::path MountPoints::ResolvePath(const std::string& mounted_name) {
 	Common::LockGuard lock(m_mutex);

+	// Guests join mount points and names with extra slashes (e.g. "/addcont0//file").
+	std::string normalized = Common::FixFilenameSlash(mounted_name);
+	normalized.erase(std::unique(normalized.begin(), normalized.end(),
+	                             [](char a, char b) { return a == '/' && b == '/'; }),
+	                 normalized.end());
+
 	// Match the entire guest path so a mount root works with or without a trailing slash.
-	const auto mounted_path = Common::FixDirectorySlash(mounted_name);
+	const auto mounted_path = Common::FixDirectorySlash(normalized);
 	const auto it           = std::find_if(
 	    m_mount_pairs.begin(), m_mount_pairs.end(),
 	    [&mounted_path](const MountPair& p) { return mounted_path.starts_with(p.point); });
 	if (it != m_mount_pairs.end()) {
 		const auto& p = *it;
-		auto rel_path = Common::RemoveFirst(Common::FixFilenameSlash(mounted_name), p.point.size());
+		auto rel_path = Common::RemoveFirst(normalized, p.point.size());
 		while (rel_path.starts_with('/')) {
 			rel_path = Common::RemoveFirst(rel_path, 1);
 		}
@@ -0,0 +1,24 @@
+#ifndef EMULATOR_SRC_KERNEL_MODULE_H_
+#define EMULATOR_SRC_KERNEL_MODULE_H_
+
+#include "common/abi.h"
+
+#include <cstddef>
+#include <cstdint>
+
+namespace Libs::LibKernel {
+
+using KernelModule = int32_t;
+struct KernelLoadModuleOpt;
+struct KernelUnloadModuleOpt;
+
+KernelModule KYTY_SYSV_ABI KernelLoadStartModule(const char* module_file_name, size_t args,
+                                                 const void* argp, uint32_t flags,
+                                                 const KernelLoadModuleOpt* opt, int* res);
+int KYTY_SYSV_ABI KernelStopUnloadModule(KernelModule handle, size_t args, const void* argp,
+                                        uint32_t flags, const KernelUnloadModuleOpt* opt,
+                                        int* res);
+
+} // namespace Libs::LibKernel
+
+#endif // EMULATOR_SRC_KERNEL_MODULE_H_
@@ -25,6 +25,9 @@
 #include <string>
 #include <thread>
 #include <unordered_map>
+#include <atomic>
+#include <cinttypes>
+#include <iterator>
 #include <variant>
 #include <vector>

@@ -176,6 +179,14 @@ static int GetHostPathStat(const std::filesystem::path& host_path, LibKernel::Fi
 	return OK;
 }

+// Mounts can appear after a failed lookup (e.g. additional content), so forget negative results.
+void InvalidateFailedResolves() {
+	std::scoped_lock lock(g_mutex);
+	for (auto it = g_resolved_paths.begin(); it != g_resolved_paths.end();) {
+		it = it->second.result != OK ? g_resolved_paths.erase(it) : std::next(it);
+	}
+}
+
 static int ResolveOnePath(const char* guest_path, uint32_t* id, uint64_t* size) {
 	if (guest_path == nullptr || guest_path[0] == '\0') {
 		return LibKernel::KERNEL_ERROR_EINVAL;
@@ -225,6 +236,10 @@ static int ResolveOnePath(const char* guest_path, uint32_t* id, uint64_t* size)
 				log_missing = true;
 			}
 		}
+		if (!log_missing && info.result == OK && path.starts_with("/addcont")) {
+			LOGF("\tAPR resolve: %s -> %s (%" PRIu64 " bytes)\n", guest_path,
+			     Common::PathToString(info.host_path).c_str(), info.file_size);
+		}
 		if (log_missing) {
 			LOGF("\tAPR resolve missing path: %s -> %s\n", guest_path, Common::PathToString(info.host_path).c_str());
 		}
@@ -1181,6 +1196,13 @@ static int ExecuteCommand(const CommandBufferState::Command& entry) {
 		uint64_t bytes_read = 0;
 		auto     result = ReadHostFileToGuest(host_path, command.file_offset, command.destination,
 		                                      command.size, &bytes_read);
+		static std::atomic<uint32_t> s_logged_dlc_reads {0};
+		if (host_path.filename() == "re_dlc_000.pak" && s_logged_dlc_reads.fetch_add(1) < 32) {
+			LOGF("\tAPR read dlc: id=0x%08" PRIx32 ", offset=0x%016" PRIx64 ", size=0x%016" PRIx64
+			     ", read=0x%016" PRIx64 ", result=0x%08" PRIx32 "\n",
+			     command.file_id, command.file_offset, command.size, bytes_read,
+			     static_cast<uint32_t>(result));
+		}
 		if (result != OK) {
 			LOGF("\tAPR submit read failed: id=0x%08" PRIx32 ", result=0x%08" PRIx32 ", path=%s\n",
 			     command.file_id, static_cast<uint32_t>(result), Common::PathToString(host_path).c_str());
@@ -22,13 +22,16 @@

 namespace Libs {

+namespace AprShared {
+void InvalidateFailedResolves();
+}
+
 LIB_VERSION("AppContent", 1, "AppContentUtil", 1, 1);

 namespace AppContent {

 constexpr uint32_t APP_CONTENT_APPPARAM_SKU_FLAG_FULL = 3;
 constexpr int      APP_CONTENT_ERROR_PARAMETER        = -2133262334; /* 0x80D90002 */
-constexpr int      APP_CONTENT_ERROR_BUSY             = -2133262333; /* 0x80D90003 */
 constexpr int      APP_CONTENT_ERROR_NOT_MOUNTED      = -2133262332; /* 0x80D90004 */
 constexpr int      APP_CONTENT_ERROR_NOT_FOUND        = -2133262331; /* 0x80D90005 */
 constexpr int      APP_CONTENT_ERROR_MOUNT_FULL       = -2133262330; /* 0x80D90006 */
@@ -207,7 +210,10 @@ int KYTY_SYSV_ABI AppContentAddcontMount(uint32_t                         servic

 	for (const auto& m: g_addcont_mounts) {
 		if (m.label == entry->entitlement_label) {
-			return APP_CONTENT_ERROR_BUSY;
+			const auto guest_point = m.point.substr(0, m.point.size() - 1);
+			std::strncpy(mount_point->data, guest_point.c_str(), sizeof(mount_point->data) - 1);
+			LOGF("\t already mounted %s\n", guest_point.c_str());
+			return OK;
 		}
 	}

@@ -232,7 +238,15 @@ int KYTY_SYSV_ABI AppContentAddcontMount(uint32_t                         servic
 	LOGF("\t mounted %s -> %s\n", mount.point.c_str(),
 	     Common::PathToString(entry->mount_dir).c_str());

-	std::strncpy(mount_point->data, mount.point.c_str(), sizeof(mount_point->data) - 1);
+	AprShared::InvalidateFailedResolves();
+
+	// Titles scan /download0/pdlc for downloaded content; it must exist even when empty.
+	if (const auto pdlc = LibKernel::FileSystem::GetRealFilename("/download0/pdlc"); !pdlc.empty()) {
+		std::filesystem::create_directories(pdlc, ec);
+	}
+
+	const auto guest_point = mount.point.substr(0, mount.point.size() - 1);
+	std::strncpy(mount_point->data, guest_point.c_str(), sizeof(mount_point->data) - 1);
 	g_addcont_mounts.push_back(std::move(mount));

 	return OK;
@@ -251,8 +265,8 @@ int KYTY_SYSV_ABI AppContentAddcontUnmount(const AppContentMountPoint* mount_poi
 		return APP_CONTENT_ERROR_NOT_MOUNTED;
 	}

-	const std::string point(mount_point->data,
-	                        strnlen(mount_point->data, sizeof(mount_point->data)));
+	const std::string point = Common::FixDirectorySlash(
+	    std::string_view(mount_point->data, strnlen(mount_point->data, sizeof(mount_point->data))));

 	std::lock_guard lock(g_addcont_mutex);

@@ -263,6 +277,7 @@ int KYTY_SYSV_ABI AppContentAddcontUnmount(const AppContentMountPoint* mount_poi
 	}
 	LibKernel::FileSystem::Umount(point);
 	g_addcont_mounts.erase(it);
+	AprShared::InvalidateFailedResolves();

 	return OK;
 }
@@ -11,6 +11,7 @@
 #include "kernel/eventQueue.h"
 #include "kernel/fileSystem.h"
 #include "kernel/memory.h"
+#include "kernel/module.h"
 #include "kernel/pthread.h"
 #include "kernel/semaphore.h"
 #include "kernel/syncOnAddress.h"
@@ -65,7 +66,6 @@ LIB_DEFINE(InitLibKernel_1_Apr);

 namespace LibKernel {

-using KernelModule                   = int32_t;
 using get_thread_atexit_count_func_t = KYTY_SYSV_ABI int (*)(KernelModule);
 using thread_atexit_report_func_t    = KYTY_SYSV_ABI void (*)(KernelModule);

@@ -1201,9 +1201,9 @@ static int KYTY_SYSV_ABI KernelRaiseException(Pthread thread, int signum) {
 	return OK;
 }

-static KYTY_SYSV_ABI KernelModule KernelLoadStartModule(const char* module_file_name, size_t args,
-                                                        const void* argp, uint32_t flags,
-                                                        const KernelLoadModuleOpt* opt, int* res) {
+KernelModule KYTY_SYSV_ABI KernelLoadStartModule(const char* module_file_name, size_t args,
+                                                 const void* argp, uint32_t flags,
+                                                 const KernelLoadModuleOpt* opt, int* res) {
 	PRINT_NAME();

 	// EXIT_NOT_IMPLEMENTED(!Common::Thread::IsMainThread());
@@ -1255,9 +1255,9 @@ static KYTY_SYSV_ABI KernelModule KernelLoadStartModule(const char* module_file_
 	return static_cast<KernelModule>(handle);
 }

-static int KYTY_SYSV_ABI KernelStopUnloadModule(KernelModule handle, size_t args, const void* argp,
-                                                uint32_t flags, const KernelUnloadModuleOpt* opt,
-                                                int* res) {
+int KYTY_SYSV_ABI KernelStopUnloadModule(KernelModule handle, size_t args, const void* argp,
+                                        uint32_t flags, const KernelUnloadModuleOpt* opt,
+                                        int* res) {
 	PRINT_NAME();

 	// EXIT_NOT_IMPLEMENTED(!Common::Thread::IsMainThread());
@@ -221,11 +221,17 @@ int KYTY_SYSV_ABI PlayGoGetChunkId(int handle, uint16_t* out_chunk_id_list,
 	if (out_entries == nullptr) {
 		return PLAYGO_ERROR_BAD_POINTER;
 	}
-	if (number_of_entries != 0 && out_chunk_id_list == nullptr) {
-		return PLAYGO_ERROR_BAD_POINTER;
+	if (out_chunk_id_list != nullptr && number_of_entries == 0) {
+		return PLAYGO_ERROR_BAD_SIZE;
 	}
 	ensure_chunks_loaded();

+	// A null output buffer asks for the total chunk count.
+	if (out_chunk_id_list == nullptr) {
+		*out_entries = g_chunks_num;
+		return OK;
+	}
+
 	const uint32_t entries = std::min(number_of_entries, g_chunks_num);
 	for (uint32_t i = 0; i < entries; i++) {
 		out_chunk_id_list[i] = static_cast<uint16_t>(i);
@@ -1,9 +1,9 @@
 #include "common/abi.h"
-#include "common/common.h"
-#include "common/stringUtils.h"
 #include "libs/libs.h"
 #include "loader/symbolDatabase.h"

+#include <cstdint>
+
 namespace Libs {

 namespace LibRazorCpu {
@@ -16,8 +16,18 @@ static KYTY_SYSV_ABI uint32_t RazorCpuIsCapturing() {
 	return 0;
 }

+static int KYTY_SYSV_ABI RazorCpuPopMarker() {
+	return 0;
+}
+
+static int KYTY_SYSV_ABI RazorCpuPushMarker(const char*, uint32_t, uint32_t) {
+	return 0;
+}
+
 LIB_DEFINE(InitRazorCpu_1) {
 	LIB_FUNC("EboejOQvLL4", LibRazorCpu::RazorCpuIsCapturing);
+	LIB_FUNC("YpkGsMXP3ew", LibRazorCpu::RazorCpuPopMarker);
+	LIB_FUNC("zw+celG7zSI", LibRazorCpu::RazorCpuPushMarker);
 }

 } // namespace LibRazorCpu
@@ -3,6 +3,7 @@
 #include "common/common.h"
 #include "common/logging/log.h"
 #include "common/stringUtils.h"
+#include "kernel/module.h"
 #include "libs/libs.h"
 #include "loader/symbolDatabase.h"

@@ -20,14 +21,40 @@ int KYTY_SYSV_ABI KernelGetModuleInfoForUnwind(uint64_t addr, int flags, ModuleI

 namespace Sysmodule {

-constexpr int SYSMODULE_ERROR_UNLOADED = static_cast<int>(0x805a1001u);
+constexpr int      SYSMODULE_ERROR_UNLOADED = static_cast<int>(0x805a1001u);
+constexpr int      SYSMODULE_ERROR_FATAL    = static_cast<int>(0x805a10ffu);

-static std::mutex                             g_module_mutex;
-static std::unordered_map<uint16_t, uint32_t> g_module_refs;
+static constexpr struct NativeModule {
+	uint16_t    id;
+	const char* path;
+} g_native_modules[] = {{0x0115, "/app0/sce_module/libSceNpCppWebApi.prx"}};

-static void LoadModule(uint16_t id) {
+struct Module {
+	uint32_t                references = 0;
+	LibKernel::KernelModule handle     = -1;
+};
+
+static std::mutex                          g_module_mutex;
+static std::unordered_map<uint16_t, Module> g_modules;
+
+static int LoadModule(uint16_t id) {
 	std::scoped_lock lock(g_module_mutex);
-	++g_module_refs[id];
+	if (const auto module = g_modules.find(id); module != g_modules.end()) {
+		++module->second.references;
+		return 0;
+	}
+
+	LibKernel::KernelModule handle = -1;
+	for (const auto& module: g_native_modules) {
+		if (module.id != id) continue;
+		handle = LibKernel::KernelLoadStartModule(module.path, 0, nullptr, 0, nullptr, nullptr);
+		if (handle < 0) {
+			return handle;
+		}
+		break;
+	}
+	g_modules.emplace(id, Module {1, handle});
+	return 0;
 }

 static KYTY_SYSV_ABI int SysmoduleGetModuleInfoForUnwind(uint64_t addr, int flags,
@@ -40,8 +67,7 @@ static KYTY_SYSV_ABI int SysmoduleLoadModule(uint16_t id) {

 	LOGF("\t id = %d\n", static_cast<int>(id));

-	LoadModule(id);
-	return 0;
+	return LoadModule(id);
 }

 static KYTY_SYSV_ABI int SysmoduleUnloadModule(uint16_t id) {
@@ -50,12 +76,19 @@ static KYTY_SYSV_ABI int SysmoduleUnloadModule(uint16_t id) {
 	LOGF("\t id = %d\n", static_cast<int>(id));

 	std::scoped_lock lock(g_module_mutex);
-	const auto       module = g_module_refs.find(id);
-	if (module == g_module_refs.end()) {
+	const auto       module = g_modules.find(id);
+	if (module == g_modules.end()) {
 		return SYSMODULE_ERROR_UNLOADED;
 	}
-	if (--module->second == 0) {
-		g_module_refs.erase(module);
+	if (module->second.references == 1) {
+		if (module->second.handle >= 0 &&
+		    LibKernel::KernelStopUnloadModule(module->second.handle, 0, nullptr, 0, nullptr,
+		                                     nullptr) < 0) {
+			return SYSMODULE_ERROR_FATAL;
+		}
+		g_modules.erase(id);
+	} else {
+		--module->second.references;
 	}
 	return 0;
 }
@@ -71,10 +104,8 @@ static KYTY_SYSV_ABI int SysmoduleLoadModuleInternalWithArg(uint16_t id, int arg
 	EXIT_IF(arg3 != 0);
 	EXIT_IF(ret == nullptr);

-	LoadModule(id);
-	*ret = 0;
-
-	return 0;
+	*ret = LoadModule(id);
+	return *ret;
 }

 static KYTY_SYSV_ABI int SysmoduleIsLoaded(uint16_t id) {
@@ -83,7 +114,7 @@ static KYTY_SYSV_ABI int SysmoduleIsLoaded(uint16_t id) {
 	LOGF("\t id = %d\n", static_cast<int>(id));

 	std::scoped_lock lock(g_module_mutex);
-	return g_module_refs.contains(id) ? 0 : SYSMODULE_ERROR_UNLOADED;
+	return g_modules.contains(id) ? 0 : SYSMODULE_ERROR_UNLOADED;
 }

 } // namespace Sysmodule
@@ -58,16 +58,17 @@ struct TlsRegStub {
 	}
 	static uint64_t GetSize() { return 32; }

-	// sub rsp,0x80
+	// lea rsp,[rsp-0x80]
+	// nop
 	// push rax
 	// call safe_call
 	// mov <reg>,rax
 	// pop rax
-	// add rsp,0x80
+	// lea rsp,[rsp+0x80]
 	// ret
-	uint8_t code[32] = {0x48, 0x81, 0xEC, 0x80, 0x00, 0x00, 0x00, 0x50, 0xE8, 0x00, 0x00,
-	                    0x00, 0x00, 0x48, 0x89, 0xC0, 0x58, 0x48, 0x81, 0xC4, 0x80, 0x00,
-	                    0x00, 0x00, 0xC3, 0x90, 0x90, 0x90, 0x90, 0x90, 0x90, 0x90};
+	uint8_t code[32] = {0x48, 0x8D, 0x64, 0x24, 0x80, 0x66, 0x90, 0x50, 0xE8, 0x00, 0x00,
+	                    0x00, 0x00, 0x48, 0x89, 0xC0, 0x58, 0x48, 0x8D, 0xA4, 0x24, 0x80,
+	                    0x00, 0x00, 0x00, 0xC3, 0x90, 0x90, 0x90, 0x90, 0x90, 0x90};
 };

 struct SafeCall {
@@ -78,7 +79,8 @@ struct SafeCall {
 	static uint64_t GetSize() { return 0x1000; }

 	uint8_t code[0x90] = {
-	    /*00*/ 0x48, 0x81, 0xec, 0x80, 0x00, 0x00, 0x00, // sub    rsp,0x80
+	    /*00*/ 0x48, 0x8d, 0x64, 0x24, 0x80,             // lea    rsp,[rsp-0x80]
+	    /*05*/ 0x66, 0x90,                               // nop
 	    /*07*/ 0x9c,                                     // pushfq
 	    /*08*/ 0x51,                                     // push   rcx
 	    /*09*/ 0x52,                                     // push   rdx
@@ -108,9 +110,9 @@ struct SafeCall {
 	    /*40*/ 0x5a,             // pop    rdx
 	    /*41*/ 0x59,             // pop    rcx
 	    /*42*/ 0x9d,             // popfq
-	    /*43*/ 0x48, 0x81, 0xc4, 0x80, 0x00, 0x00,
-	    0x00,        // add    rsp,0x80
-	    /*4a*/ 0xc3, // ret
+	    /*43*/ 0x48, 0x8d, 0xa4, 0x24, 0x80, 0x00, 0x00,
+	    0x00,        // lea    rsp,[rsp+0x80]
+	    /*4b*/ 0xc3, // ret
 	};
 };

@@ -235,6 +235,9 @@ void TestSysmoduleReferences() {
  const auto unload = resolve("eR2bZFAAU0Q");
  const auto is_loaded = resolve("fMP5NHUOaMk");
  constexpr int unloaded = static_cast<int>(0x805a1001u);
+  Check(load(0x115) == Libs::LibKernel::KERNEL_ERROR_ENOENT && is_loaded(0x115) == unloaded &&
+            unload(0x115) == unloaded,
+        "missing packaged NpCppWebApi fails without recording a loaded module");
  Check(load(0x113) == OK && is_loaded(0xb4) == unloaded,
        "loading entitlement access does not mark AppContent loaded");
  Check(unload(0x113) == OK, "release independent module reference");
@@ -255,6 +258,9 @@ void TestSysmoduleReferences() {
      reinterpret_cast<int (KYTY_SYSV_ABI *)(uint16_t, int, int, int, int*)>(
          internal_symbol->vaddr);
  int result = -1;
+  Check(internal_load(0x115, 0, 0, 0, &result) == Libs::LibKernel::KERNEL_ERROR_ENOENT &&
+            result == Libs::LibKernel::KERNEL_ERROR_ENOENT && is_loaded(0x115) == unloaded,
+        "internal module loading propagates missing native module failure");
  Check(internal_load(0xb4, 0, 0, 0, &result) == OK && result == OK &&
            is_loaded(0xb4) == OK && unload(0xb4) == OK && is_loaded(0xb4) == unloaded,
        "internal and public module operations share load state");
@@ -407,7 +413,9 @@ void TestAddContent(const std::filesystem::path& root) {
  Check(mount(0, nullptr, &point) == static_cast<int>(0x80d90002u) &&
            list(0, nullptr, 0, nullptr) == static_cast<int>(0x817d0002u),
        "null add-on parameters return parameter errors");
-  Check(mount(0, &info[3].label, &point) == OK, "mount owned add-on");
+  Check(mount(0, &info[3].label, &point) == OK &&
+            !std::string_view(point.data).ends_with('/'),
+        "mount owned add-on without trailing slash");
  const std::string mounted_file = std::string(point.data) + "/payload.txt";
  const int fd = FileSystem::KernelOpen(mounted_file.c_str(), 0, 0);
  std::array<char, 11> payload {};
@@ -416,8 +424,9 @@ void TestAddContent(const std::filesystem::path& root) {
            FileSystem::KernelClose(fd) == OK,
        "guest reads files from the selected title's add-on");
  MountPoint repeated {};
-  Check(mount(0, &info[3].label, &repeated) == static_cast<int>(0x80d90003u),
-        "duplicate mount returns BUSY");
+  Check(mount(0, &info[3].label, &repeated) == OK &&
+            std::strcmp(repeated.data, point.data) == 0,
+        "duplicate mount returns the same mount point");
  Check(unmount(&point) == OK && FileSystem::GetRealFilename(mounted_file).empty() &&
            unmount(&point) == static_cast<int>(0x80d90004u),
        "unmount removes mapping and repeated unmount reports NOT_MOUNTED");
@@ -213,6 +213,12 @@ struct ProtectionCall {
 std::vector<ProtectionCall> g_protection_log;
 std::mutex g_protection_log_mutex;

+struct ProtectionGate {
+  std::binary_semaphore entered{0};
+  std::binary_semaphore resume{0};
+};
+ProtectionGate *g_protection_gate = nullptr;
+
 void ResetProtectionLog() {
  g_protection_calls = 0;
  g_protection_log.clear();
@@ -220,6 +226,10 @@ void ResetProtectionLog() {

 bool ProtectAddressSpace(uint64_t vaddr, uint64_t size,
                         Common::VirtualMemory::Mode mode) {
+  if (g_protection_gate != nullptr && mode == Common::VirtualMemory::Mode::ReadWrite) {
+    g_protection_gate->entered.release();
+    g_protection_gate->resume.acquire();
+  }
  uint32_t protection = PAGE_NOACCESS;
  if (mode == Common::VirtualMemory::Mode::Read) {
    protection = PAGE_READONLY;
@@ -407,6 +417,74 @@ void TestCpuDirtyUpload() {
  Release(memory);
 }

+void TestBdaSyncDirtiness() {
+  constexpr auto page = Libs::Graphics::TRACKER_PAGE_SIZE;
+  TrackerHarness harness;
+  auto &tracker = harness.tracker;
+  auto *memory = Allocate(harness.page_manager, 3);
+  const auto address = reinterpret_cast<uint64_t>(memory);
+  Check(tracker.TakeBdaSync().Empty(), "new tracker queued unowned memory");
+  tracker.QueueBdaSync(address, 3 * page);
+  Check(tracker.IsRegionCpuModified(address, page), "new region was clean");
+  auto pending = tracker.TakeBdaSync();
+  Check(pending.Contains(address, 3 * page) && !pending.Intersects(address + 3 * page, page),
+        "mapped or registered range did not bound initial BDA synchronization");
+  tracker.ForEachUploadRange(address, 3 * page, false,
+                             [](uint64_t, uint64_t) noexcept {}, []() noexcept {});
+  Check(tracker.TakeBdaSync().Empty(), "clean upload queued redundant BDA work");
+
+  tracker.MarkRegionAsCpuModified(address + 17, 1);
+  pending = tracker.TakeBdaSync();
+  Check(pending.Contains(address, page) && !pending.Intersects(address + page, page) &&
+            tracker.TakeBdaSync().Empty(),
+        "byte dirtiness did not drain exactly its whole tracker page");
+  // A write after Take belongs to the next drain, even when its page was just taken.
+  tracker.MarkRegionAsCpuModified(address + 31, 1);
+  Check(tracker.TakeBdaSync().Contains(address, page), "re-dirty event was lost");
+  tracker.ForEachUploadRange(address, page, true,
+                             [](uint64_t, uint64_t) noexcept {}, []() noexcept {});
+  Check(tracker.TakeBdaSync().Empty(), "GPU ownership queued an unnecessary upload");
+  tracker.InvalidateRegion(address + 7, 1, [&] {
+    tracker.UnmarkRegionAsGpuModified(address, page);
+    tracker.MarkRegionAsCpuModified(address, page);
+  });
+  Check(tracker.TakeBdaSync().Contains(address, page),
+        "GPU-to-CPU invalidation did not queue BDA synchronization");
+  tracker.InvalidateRegion(address + page + 7, 1, [] {});
+  Check(tracker.TakeBdaSync().Contains(address + page, page),
+        "CPU invalidation did not queue BDA synchronization");
+  tracker.UntrackMemory(address, 3 * page);
+  Check(tracker.TakeBdaSync().Contains(address, 3 * page),
+        "untracked memory did not queue its newly dirty pages");
+  Release(memory);
+}
+
+void TestBdaSyncConcurrentDrain() {
+  constexpr auto page = Libs::Graphics::TRACKER_PAGE_SIZE;
+  TrackerHarness harness;
+  auto &tracker = harness.tracker;
+  auto *memory = Allocate(harness.page_manager, 1);
+  const auto address = reinterpret_cast<uint64_t>(memory);
+  tracker.ForEachUploadRange(address, page, false,
+                             [](uint64_t, uint64_t) noexcept {}, []() noexcept {});
+  (void)tracker.TakeBdaSync();
+  ProtectionGate gate;
+  g_protection_gate = &gate;
+  std::jthread writer([&] { tracker.MarkRegionAsCpuModified(address + 17, 1); });
+  gate.entered.acquire();
+  // Ownership is still changing. Publishing its event before this point would let
+  // a drain consume it before the writer has finished releasing page protection.
+  Check(tracker.TakeBdaSync().Empty(), "BDA event preceded its state transition");
+  gate.resume.release();
+  writer.join();
+  g_protection_gate = nullptr;
+  Check(tracker.TakeBdaSync().Contains(address, page) &&
+            tracker.IsRegionCpuModified(address, page),
+        "concurrent drain lost a completed CPU dirty transition");
+  tracker.UntrackMemory(address, page);
+  Release(memory);
+}
+
 void TestCleanUploadPreservesOwnership() {
  constexpr auto page_size = Libs::Graphics::TRACKER_PAGE_SIZE;
  constexpr auto region_size = Libs::Graphics::TRACKER_REGION_SIZE;
@@ -1202,6 +1280,8 @@ int main(int argc, char **argv) {
  TestQueriesDoNotRequireMappedOwnership();
  TestConcurrentRegionPublication();
  TestCpuDirtyUpload();
+  TestBdaSyncDirtiness();
+  TestBdaSyncConcurrentDrain();
  TestCleanUploadPreservesOwnership();
  TestRangeInvalidation();
  TestGpuReacquisitionAfterInvalidation();
@@ -5281,6 +5281,85 @@ public:
    std::printf("[gpu]     %-32s ok\n", name);
  }

+  void CheckBdaSyncUploads() {
+    constexpr const char *name = "BdaSyncUploads";
+    constexpr auto page = BufferCache::CACHING_PAGESIZE;
+    EnsureRuntimeContext();
+    const auto base = LibKernel::Memory::AllocateRuntimeMemory(
+        0, 3 * page, Common::VirtualMemory::Mode::ReadWrite, "bda_sync_uploads");
+    Require(name, "allocation", base != 0, "guest allocation failed");
+    auto *words = reinterpret_cast<u32 *>(base);
+    std::fill_n(words, 3 * page / sizeof(u32), 0x12345678u);
+    RenderContext context(m_runtime_context);
+    auto &scheduler = context.GetCommandScheduler();
+    auto &cache = context.GetBufferCache();
+    HW::Context registers{};
+    HW::UserConfig user_config{};
+    HW::Shader shaders{};
+    scheduler.Begin(registers, user_config, shaders);
+    context.MapMemory(base, page);
+    context.MapMemory(base + 2 * page, page);
+    const auto id = cache.FindBuffer(base, 3 * page);
+    auto &buffer = cache.GetBuffer(id);
+    auto readback = CreateHostBuffer(name, 6 * sizeof(u32),
+                                     vk::BufferUsageFlagBits::eTransferDst, {});
+    const auto snapshot = [&](uint64_t offset, uint32_t index) {
+      const vk::BufferCopy copy{offset, index * sizeof(u32), sizeof(u32)};
+      scheduler.Current().Handle().copyBuffer(buffer.Handle(), readback.buffer, 1, &copy);
+    };
+    context.PrepareBda();
+    Require(name, "mapped intersection",
+            !cache.IsRegionCpuModified(base, page) &&
+                cache.IsRegionCpuModified(base + page, page) &&
+                !cache.IsRegionCpuModified(base + 2 * page, page),
+            "BDA upload missed a mapped range or uploaded an unmapped hole");
+    snapshot(0, 0);
+    snapshot(2 * page, 1);
+
+    cache.InvalidateMemory(base + 17, 1);
+    words[4] = 0x87654321u;
+    words[TRACKER_PAGE_SIZE / sizeof(u32) - 1] = 0xabcdef01u;
+    context.PrepareBda();
+    snapshot(16, 2);
+    snapshot(TRACKER_PAGE_SIZE - sizeof(u32), 3);
+    Require(name, "whole-page upload", !cache.IsRegionCpuModified(base, TRACKER_PAGE_SIZE),
+            "one-byte invalidation did not upload and consume its whole page");
+
+    context.MapMemory(base + page, page);
+    context.PrepareBda();
+    snapshot(page, 4);
+    Require(name, "new mapping", !cache.IsRegionCpuModified(base + page, page),
+            "mapping into an existing buffer did not queue its dirty pages");
+    context.UnmapMemory(base + 2 * page, page);
+    words[2 * page / sizeof(u32)] = 0xfedcba98u;
+    context.PrepareBda();
+    Require(name, "unmapped dirty page", cache.IsRegionCpuModified(base + 2 * page, page),
+            "BDA synchronization consumed dirtiness outside mapped memory");
+    context.MapMemory(base + 2 * page, page);
+    context.PrepareBda();
+    snapshot(2 * page, 5);
+
+    vk::BufferMemoryBarrier barrier{};
+    barrier.srcAccessMask = vk::AccessFlagBits::eTransferWrite;
+    barrier.dstAccessMask = vk::AccessFlagBits::eHostRead;
+    barrier.srcQueueFamilyIndex = barrier.dstQueueFamilyIndex = VK_QUEUE_FAMILY_IGNORED;
+    barrier.buffer = readback.buffer;
+    barrier.size = readback.size;
+    scheduler.Current().Handle().pipelineBarrier(vk::PipelineStageFlagBits::eTransfer,
+        vk::PipelineStageFlagBits::eHost, {}, 0, nullptr, 1, &barrier, 0, nullptr);
+    scheduler.Finish();
+    Require(name, "GPU contents", ReadBuffer(name, readback, 6) ==
+        std::vector<u32>{0x12345678u, 0x12345678u, 0x87654321u, 0xabcdef01u,
+                        0x12345678u, 0xfedcba98u},
+        "initial, re-dirtied, newly mapped or remapped bytes were stale on the GPU");
+    context.UnmapMemory(base, 3 * page);
+    scheduler.Finish();
+    DestroyBuffer(&readback);
+    Require(name, "release", LibKernel::Memory::FreeGuestMemory(base, 3 * page),
+            "guest allocation release failed");
+    std::printf("[gpu]     %-32s ok\n", name);
+  }
+
  void CheckBdaPageTableUploads() {
    constexpr const char *name = "BdaPageTableUploads";
    constexpr uintptr_t base = 0x0000000207b00000ull;
@@ -20912,6 +20991,7 @@ CoverageClass ClassifyOpcode(ShaderOpcode opcode,
  case Opcode::BUFFER_ATOMIC_SMAX:
  case Opcode::BUFFER_ATOMIC_UMAX:
  case Opcode::BUFFER_ATOMIC_AND:
+  case Opcode::BUFFER_ATOMIC_ADD_X2:
  case Opcode::BUFFER_ATOMIC_AND_X2:
  case Opcode::BUFFER_ATOMIC_OR:
  case Opcode::BUFFER_ATOMIC_XOR:
@@ -20935,6 +21015,7 @@ CoverageClass ClassifyOpcode(ShaderOpcode opcode,
  case Opcode::DS_ADD_U32:
  case Opcode::DS_ADD_U64:
  case Opcode::DS_MAX_RTN_U64:
+  case Opcode::DS_CMPST_RTN_B32:
  case Opcode::DS_CMPST_RTN_B64:
  case Opcode::DS_ADD_RTN_U32:
  case Opcode::DS_SUB_U32:
@@ -24588,6 +24669,56 @@ TestCase Vop3MulLoU16CapturedAndSelectors() {
  return test;
 }

+TestCase Vop3F32ClampFlushesNaN() {
+  using O = ShaderOpcode;
+  struct ClampCase {
+    u32 opcode, input, expected;
+    u32 abs = 0;
+  };
+  constexpr u32 inf = 0x7f800000u, one = 0x3f800000u;
+  // GoW Ragnarok PS 3d52c7172f95fd37 0x2564: v_mul_f32 v18 clamp, inf, |v2|; and
+  // fma(x, inf, inf) clamp. The clamp modifier (DX10 clamp mode) turns their NaN into 0.
+  const std::array<ClampCase, 10> cases{{
+      {0x108, 0x00000000u, 0, 2}, // inf * |0| = NaN.
+      {0x108, 0x80000000u, 0, 2}, // inf * |-0| = NaN.
+      {0x108, 0x3f000000u, one, 2},
+      {0x108, 0xc0400000u, one, 2},
+      {0x14b, 0xbf800000u, 0}, // -1 * inf + inf = NaN.
+      {0x14b, 0x40000000u, one},
+      {0x103, 0x7fc00000u, 0}, // Quiet NaN input.
+      {0x103, 0xffc00001u, 0}, // Negative NaN input.
+      {0x103, 0x3e800000u, 0x3e800000u},
+      {0x103, 0x3fc00000u, one},
+  }};
+  TestCase test;
+  test.name = "Vop3F32ClampFlushesNaN";
+  for (const auto &entry : cases) test.initial.push_back(entry.input);
+  test.expected = test.initial;
+  auto &code = test.code;
+  AppendVMovLiteral(&code, 20, inf);
+  AppendVMovU32(&code, 21, 0);
+  for (u32 i = 0; i < cases.size(); ++i) {
+    const auto &entry = cases[i];
+    AppendVMovU32(&code, 30, i * 4u);
+    AppendBufferLoadDword(&code, 2, 30); // Runtime inputs prevent folding.
+    AppendVMovLiteral(&code, 18, 0xa5a5beefu);
+    if (entry.opcode == 0x108) {
+      AppendVop3(&code, 0x108, 18, Vgpr(20), Vgpr(2), 0, entry.abs, 0, true);
+    } else if (entry.opcode == 0x14b) {
+      AppendVop3(&code, 0x14b, 18, Vgpr(2), Vgpr(20), Vgpr(20), 0, 0, true);
+    } else {
+      AppendVop3(&code, 0x103, 18, Vgpr(2), Vgpr(21), 0, 0, 0, true);
+    }
+    AppendStoreVgpr(&code, 18, static_cast<u32>(cases.size()) + i);
+    test.expected.push_back(entry.expected);
+  }
+  AppendEnd(&code);
+  test.opcodes = {O::V_MOV_B32, O::BUFFER_LOAD_DWORD, O::V_MUL_F32, O::V_FMA_F32,
+                  O::V_ADD_F32, O::BUFFER_STORE_DWORD, O::S_ENDPGM};
+  test.decoded_counts = {{"V_MUL_F32", 4}, {"V_FMA_F32", 2}, {"V_ADD_F32", 4}};
+  return test;
+}
+
 TestCase Vop3MadI16CapturedSelectorsAndSaturation() {
  using O = ShaderOpcode;
  struct MadCase {
@@ -34568,6 +34699,60 @@ TestCase DsAtomicReturn64() {
  return test;
 }

+TestCase DsCompareStoreReturn32() {
+  using O = ShaderOpcode;
+  TestCase test;
+  test.name = "DsCompareStoreReturn32";
+  auto &code = test.code;
+  const auto result = [&](u32 reg, u32 value) {
+    AppendStoreVgpr(&code, reg, static_cast<u32>(test.expected.size()));
+    test.expected.push_back(value);
+  };
+  struct Case { u32 before, compare, replacement; };
+  constexpr std::array cases{Case{0x12345678u, 0x12345678u, 0x9abcdef0u},
+                             Case{0x12345678u, 0x9abcdef0u, 0x12345678u},
+                             Case{0x80000000u, 0u, 0x7fffffffu},
+                             Case{UINT32_MAX, UINT32_MAX, 0u}};
+  AppendVMovU32(&code, 7, 0);
+  for (const auto &item : cases) {
+    AppendVMovLiteral(&code, 2, item.before);
+    code.push_back(EncodeDs0(0x0d, 8));
+    code.push_back(EncodeDs1(0, 2, 7));
+    AppendVMovLiteral(&code, 5, item.compare);
+    AppendVMovLiteral(&code, 19, item.replacement);
+    AppendVMovU32(&code, 4, 8);
+    // CS04b812daaacd532f pc 0x3e8: DATA compares, DATA2 replaces, VDst aliases ADDR.
+    code.insert(code.end(), {0xd8c00000u, 0x04130504u});
+    result(4, item.before);
+    code.push_back(EncodeDs0(0x36, 8));
+    code.push_back(EncodeDs1(8, 0, 7));
+    result(8, item.before == item.compare ? item.replacement : item.before);
+  }
+  AppendVMovU32(&code, 5, 0);
+  AppendVMovU32(&code, 4, 12); // Past the three LDS DWORDs.
+  code.insert(code.end(), {0xd8c00000u, 0x04130504u});
+  result(4, 0);
+  AppendVMovLiteral(&code, 2, 0x12345678u);
+  code.push_back(EncodeDs0(0x0d, 8));
+  code.push_back(EncodeDs1(0, 2, 7));
+  AppendVMovLiteral(&code, 5, 0x12345678u);
+  AppendVMovU32(&code, 4, 8);
+  code.push_back(EncodeSMovB32(126, InlineU32(0)));
+  code.insert(code.end(), {0xd8c00000u, 0x04130504u});
+  code.push_back(EncodeSMovB32(126, InlineU32(1)));
+  result(4, 8); // An inactive lane keeps its destination and leaves memory alone.
+  code.push_back(EncodeDs0(0x36, 8));
+  code.push_back(EncodeDs1(8, 0, 7));
+  result(8, 0x12345678u);
+  AppendEnd(&code);
+  test.compute_info.lds_size_dwords = 3;
+  test.opcodes = {O::V_MOV_B32, O::S_MOV_B32, O::DS_WRITE_B32, O::DS_CMPST_RTN_B32,
+                  O::DS_READ_B32, O::BUFFER_STORE_DWORD, O::S_ENDPGM};
+  test.required_spirv = {"OpAtomicCompareExchange %uint"};
+  test.forbidden_spirv = {"OpLoopMerge"};
+  return test;
+}
+
 TestCase DsAtomic64Contention(bool add, u32 wave_size) {
  using O = ShaderOpcode;
  TestCase test;
@@ -35583,6 +35768,40 @@ TestCase BufferAtomicAndX2GlcAndExec() {
  return test;
 }

+TestCase BufferAtomicAddX2GlcAndExec() {
+  using O = ShaderOpcode;
+  TestCase test;
+  test.name = "BufferAtomicAddX2GlcAndExec";
+  auto &code = test.code;
+  code.push_back(EncodeSop1(0x04, 12, 126));
+  AppendVMovU32(&code, 20, 0);
+  const u32 values[][2] = {{0x0ff00ff0u, 0xf00ff00fu},
+                         {0xff00ff00u, 0x00ff00ffu},
+                         {0x76543210u, 0xfedcba98u}};
+  for (u32 i = 0; i < std::size(values); ++i) {
+    AppendVMovLiteral(&code, 2, values[i][0]);
+    AppendVMovLiteral(&code, 3, values[i][1]);
+    if (i == 2) {
+      code.push_back(EncodeSop1(0x04, 126, InlineU32(0)));
+    }
+    AppendBufferStoreOpcode(&code, 0x52, 2, 20, i != 1);
+    code.push_back(EncodeSop1(0x04, 126, 12));
+    AppendStoreVgpr(&code, 2, 2 + i * 2);
+    AppendStoreVgpr(&code, 3, 3 + i * 2);
+  }
+  AppendEnd(&code);
+  test.initial = {0x12345678u, 0x87654321u, 0, 0, 0, 0, 0, 0};
+  // The first add wraps at 64 bits; the second carries from the low dword.
+  // GLC returns the first old value, preserves the second operand, and EXEC = 0
+  // suppresses the third add and its writeback despite its nonzero operand.
+  test.expected = {0x21256568u, 0x78743430u, 0x12345678u, 0x87654321u,
+                   0xff00ff00u, 0x00ff00ffu, 0x76543210u, 0xfedcba98u};
+  test.opcodes = {O::V_MOV_B32, O::S_MOV_B64, O::BUFFER_ATOMIC_ADD_X2,
+                 O::BUFFER_STORE_DWORD, O::S_ENDPGM};
+  test.required_spirv = {"OpAtomicIAdd %ulong", "Int64Atomics"};
+  return test;
+}
+
 TestCase BufferAtomicCmpSwapExactRaw() {
  using O = ShaderOpcode;

@@ -41783,6 +42002,7 @@ std::vector<TestCase> MakeCases() {
  AddCase(Vop3CvtPkI16I32Captured);
  AddCase(Vop3MulLoU16CapturedAndSelectors);
  AddCase(Vop3MadI16CapturedSelectorsAndSaturation);
+  AddCase(Vop3F32ClampFlushesNaN);
  AddCase(Vop3Min3U16CapturedAndSelectors);
  AddCase(Vop3Med3I16Captured);
  AddCase(Vop2SdwaMinU32PreservesWordDestination);
@@ -42111,6 +42331,7 @@ std::vector<TestCase> MakeCases() {
  }
  AddCase(DsAtomicReturnVariants);
  AddCase(DsAtomicReturn64);
+  AddCase(DsCompareStoreReturn32);
  for (bool add : {false, true}) {
    cases.push_back(DsAtomic64Bounds(add));
    for (u32 wave_size : {32, 64}) {
@@ -42118,6 +42339,7 @@ std::vector<TestCase> MakeCases() {
    }
  }
  AddCase(BufferAtomicAndX2GlcAndExec);
+  AddCase(BufferAtomicAddX2GlcAndExec);
  AddCase(ImageAtomicSignedMinMax<false>);
  AddCase(ImageAtomicSignedMinMax<true>);
  AddCase(ImageAtomicSignedMinMax<false, true>);
@@ -42474,6 +42696,30 @@ void CheckPs5GameExampleImageClearRuntimeShape() {
          !ResolveComputeBufferFill(compute, 2, 1, 1, 0x41u, descriptor,
                                    packed_clear, size),
          "excess invocation coverage was accepted as a fill");
+  runtime_resources.uniform_fill.bounded_elements = 65;
+  runtime_resources.buffers[0].dwords[2] = 65;
+  for (u32 oob = 0; oob < 4; ++oob) {
+    runtime_resources.buffers[0].dwords[3] = 0x14204u | (oob << 28u);
+    Require("Ps5GameExampleImageClear", "bounded final workgroup and RDNA2 OOB mode",
+            ResolveComputeBufferFill(compute, 2, 1, 1, 0x41u, descriptor,
+                                     packed_clear, size) == (oob != 3),
+            "explicit coverage did not respect RDNA2 element bounds");
+  }
+  runtime_resources.buffers[0].dwords[3] = 0x14204u;
+  Require("Ps5GameExampleImageClear", "bounded insufficient dispatch",
+          !ResolveComputeBufferFill(compute, 1, 1, 1, 0x41u, descriptor,
+                                    packed_clear, size),
+          "a bound was mistaken for actual invocation coverage");
+  Require("Ps5GameExampleImageClear", "bounded index wrap",
+          !ResolveComputeBufferFill(compute, 0x04000001u, 1, 1, 0x41u, descriptor,
+                                    packed_clear, size),
+          "a wrapped invocation index was accepted as complete coverage");
+  runtime_resources.uniform_fill.bounded_elements = 64;
+  Require("Ps5GameExampleImageClear", "bounded incomplete destination",
+          !ResolveComputeBufferFill(compute, 2, 1, 1, 0x41u, descriptor,
+                                    packed_clear, size),
+          "a shader bound smaller than the destination was accepted as a fill");
+  runtime_resources = scalar;
  auto alias = scalar.buffers.front();
  runtime_resources.buffers.push_back(alias);
  Require("Ps5GameExampleImageClear", "aliased scalar input",
@@ -47610,7 +47856,8 @@ int main(int argc, char **argv) {
    RunCase(&vulkan, DsWideLdsPartialBounds());
    RunCase(&vulkan, DsAtomic64Bounds(false));
    RunCase(&vulkan, DsAtomic64Bounds(true));
-    for (auto test : {DsWideLdsPartialBounds(), DsAtomic64Bounds(false), DsAtomic64Bounds(true), DsAtomicReturn64()}) {
+    for (auto test : {DsWideLdsPartialBounds(), DsAtomic64Bounds(false), DsAtomic64Bounds(true), DsAtomicReturn64(),
+                      DsCompareStoreReturn32()}) {
      test.compute_info.lds_storage = true;
      std::erase(test.required_spirv, "WorkgroupMemoryExplicitLayoutKHR");
      test.required_spirv.push_back("StorageBuffer");
@@ -47656,6 +47903,7 @@ int main(int argc, char **argv) {
    }
    RunCase(&vulkan, DsAtomicReturn64());
    RunCase(&vulkan, BufferAtomicAndX2GlcAndExec());
+    RunCase(&vulkan, BufferAtomicAddX2GlcAndExec());
    RunCase(&vulkan, ImageAtomicSignedMinMax<false>());
    RunCase(&vulkan, ImageAtomicSignedMinMax<true>());
    return 0;
@@ -48339,6 +48587,11 @@ int main(int argc, char **argv) {
    RunCase(&vulkan, Vop3MulLoU16CapturedAndSelectors());
    return 0;
  }
+  if (argc == 2 && std::strcmp(argv[1], "--clamp-nan-only") == 0) {
+    VulkanHarness vulkan;
+    RunCase(&vulkan, Vop3F32ClampFlushesNaN());
+    return 0;
+  }
  if (argc == 2 && std::strcmp(argv[1], "--mad-i16-only") == 0) {
    VulkanHarness vulkan;
    RunCase(&vulkan, Vop3MadI16CapturedSelectorsAndSaturation());
@@ -48435,6 +48688,7 @@ int main(int argc, char **argv) {
  }
  if (argc == 2 && std::strcmp(argv[1], "--compute-meta-clear-only") == 0) {
    CheckDynamicRenderingState();
+    CheckPs5GameExampleImageClearRuntimeShape();
    VulkanHarness vulkan;
    vulkan.CheckComputeMetaClearClassification();
    vulkan.CheckNativeIndirectDispatch();
@@ -48472,6 +48726,7 @@ int main(int argc, char **argv) {
  if (argc == 2 && std::strcmp(argv[1], "--bda-page-table-only") == 0) {
    VulkanHarness vulkan;
    vulkan.CheckBdaPageTableUploads();
+    vulkan.CheckBdaSyncUploads();
    return 0;
  }
  if (argc == 2 && std::strcmp(argv[1], "--buffer-cache-range-only") == 0) {
@@ -48702,6 +48957,7 @@ int main(int argc, char **argv) {
  CheckComputeLdsLimit(vulkan);
  vulkan.CheckNullTextureDescriptors();
  vulkan.CheckBdaPageTableUploads();
+  vulkan.CheckBdaSyncUploads();
  vulkan.CheckLargeSparseImageUpload();
  vulkan.CheckUnifiedTextureCacheFlow();
  vulkan.CheckUnifiedImageViewCache();
@@ -8,6 +8,7 @@
 #include "kernel/pthread.h"
 #include "libs/errno.h"
 #include "loader/guestInstructionPatcher.h"
+#include "loader/jit.h"
 #include "loader/runtimeLinker.h"
 #include "loader/systemContent.h"
 #include "loader/x64InstructionEmulator.h"
@@ -4775,6 +4776,59 @@ void TestSmallFiberStacksAndMigration() {

 } // namespace

+#if defined(__linux__) || KYTY_PLATFORM == KYTY_PLATFORM_WINDOWS
+KYTY_MS_ABI uint8_t* TlsStubTestAddress() {
+	return reinterpret_cast<uint8_t*>(static_cast<uintptr_t>(0x123456789abcull));
+}
+
+void TestTlsStubsPreserveFlags() {
+	const char*    test      = "TlsStubsPreserveFlags";
+	const uint64_t stub_size = Loader::Jit::SafeCall::GetSize();
+	constexpr uint64_t CODE_SIZE = 0x1000;
+	constexpr uint8_t  REG       = 1;
+	const auto         mapping   = Libs::LibKernel::Memory::AllocateProgramMemory(
+	    0x0000000906000000ull, stub_size + CODE_SIZE, Common::VirtualMemory::Mode::ExecuteReadWrite,
+	    "tls_stub_test");
+	Check(test, mapping != 0, "failed to allocate TLS stub test code");
+	auto* safe_call = new (reinterpret_cast<void*>(mapping)) Loader::Jit::SafeCall;
+	safe_call->SetFunc(TlsStubTestAddress);
+	const auto stub_address = mapping + Loader::Jit::TlsRegStub::GetOffset(REG);
+	auto*      stub         = new (reinterpret_cast<void*>(stub_address)) Loader::Jit::TlsRegStub;
+	stub->SetFunc(mapping);
+	stub->SetOutputReg(REG);
+
+	using Fixture  = uint64_t(KYTY_SYSV_ABI*)(uint64_t*);
+	bool preserved = true;
+	for (const auto [input, status]: {std::pair {uint8_t {0x7f}, uint64_t {0x890}},
+	                                  std::pair {uint8_t {0xff}, uint64_t {0x055}}}) {
+		Xbyak::CodeGenerator code(CODE_SIZE, reinterpret_cast<void*>(mapping + stub_size));
+		code.push(code.rbx);
+		code.mov(code.rbx, code.rdi);
+		code.mov(code.rax, 0x5555555555555555ull);
+		code.mov(code.al, input);
+		code.add(code.al, 1);
+		code.mov(code.r11, stub_address);
+		code.call(code.r11);
+		code.mov(code.qword[code.rbx], code.rcx);
+		code.mov(code.qword[code.rbx + 8], code.rax);
+		code.pushf();
+		code.pop(code.rax);
+		code.pop(code.rbx);
+		code.ret();
+		Check(test, Xbyak::GetError() == 0, "failed to generate TLS stub fixture");
+		Check(test, Common::VirtualMemory::FlushInstructionCache(mapping, stub_size + CODE_SIZE),
+		      "failed to flush TLS stub test code");
+		std::array<uint64_t, 2> registers {};
+		const auto flags = reinterpret_cast<Fixture>(mapping + stub_size)(registers.data());
+		const auto rax   = (0x5555555555555555ull & ~0xffull) | static_cast<uint8_t>(input + 1u);
+		preserved &= (flags & 0x8d5u) == status && registers[0] == 0x123456789abcull &&
+		             registers[1] == rax;
+	}
+	Libs::LibKernel::Memory::FreeGuestMemory(mapping, stub_size + CODE_SIZE);
+	Check(test, preserved, "TLS stubs changed the caller's flags or registers");
+}
+#endif
+
 int main(int argc, char** argv) {
 	InitSubsystems();
 	if (argc == 2 && std::strcmp(argv[1], "--virtual-ranges-only") == 0) {
@@ -4919,6 +4973,9 @@ int main(int argc, char** argv) {
 	RunTest(TestMemoryPoolCommitDecommitQueryFlags);
 	RunTest(TestProgramMemoryAllocationAndProtection);
 	RunTest(TestModuleRelocationUsesWritableHostMapping);
+#if defined(__linux__) || KYTY_PLATFORM == KYTY_PLATFORM_WINDOWS
+	RunTest(TestTlsStubsPreserveFlags);
+#endif

 	if (g_failed_tests != 0) {
 		std::printf("VirtualMemoryAllocationTests: %d case(s) failed\n", g_failed_tests);
@@ -8119,6 +8119,24 @@ void TestDsReturningAtomic64Decoder() {
        "captured DS_CMPST_RTN_B64 lost its comparison/replacement pairs or aliases");
 }

+void TestDsCompareStore32Decoder() {
+  using namespace ShaderRecompiler::Decoder;
+  // CS04b812daaacd532f pc 0x3e8 (AC Valhalla, #281).
+  const uint32_t words[] = {0xd8c00000u, 0x04130504u};
+  Instruction decoded;
+  DecodeInstruction(words, 0u, decoded);
+  Check(decoded.opcode == Opcode::DS_CMPST_RTN_B32 && decoded.word_count == 2u &&
+            decoded.data_dwords == 1u && decoded.src_count == 3u && decoded.src0.reg == 4u &&
+            decoded.src1.reg == 5u && decoded.src2.reg == 19u && decoded.dst.reg == 4u &&
+            decoded.offset == 0u && !decoded.gds,
+        "captured DS_CMPST_RTN_B32 lost its comparison/replacement order or aliases");
+
+  const uint32_t gds_words[] = {words[0] | (1u << 17u), words[1]};
+  DecodeInstruction(gds_words, 0u, decoded);
+  Check(decoded.opcode == Opcode::UNSUPPORTED && decoded.word_count == 2u && decoded.gds,
+        "DS_CMPST_RTN_B32 must reject the unimplemented GDS variant");
+}
+
 void TestDsFloatMinMaxDecoder() {
  using namespace ShaderRecompiler::Decoder;
  for (bool max_value : {false, true}) {
@@ -10539,6 +10557,8 @@ void TestCapturedBufferAtomicsX2() {
    ValueOpcode ir_opcode;
  };
  const Case cases[] = {
+      {{0xe1480018u, 0x80000000u}, 0u, 234u, "BUFFER_ATOMIC_ADD_X2",
+       "BufferAtomicIAdd64", "OpAtomicIAdd", ValueOpcode::BufferAtomicIAdd64},
      {{0xe1680018u, 0x80000000u}, 0u, 241u, "BUFFER_ATOMIC_OR_X2",
       "BufferAtomicOr64", "OpAtomicOr", ValueOpcode::BufferAtomicOr64},
      {{0xe1402000u, 0x80000913u}, 9u, 229u, "BUFFER_ATOMIC_SWAP_X2",
@@ -12874,6 +12894,51 @@ void TestLateConstantPropagation() {
        "an ungrounded Phi cycle acquired a constant");
 }

+void TestMaskedShiftConstantPropagation() {
+  using namespace ShaderRecompiler::IR;
+  for (const uint32_t count : {0u, 1u, 8u, 31u, 32u, 33u, 40u, 63u, 64u, UINT32_MAX}) {
+    for (const uint32_t mask : {0u, 1u, 0xffu, 0x7fffffffu, 0x80000000u, UINT32_MAX}) {
+      for (const bool swapped : {false, true}) {
+        for (const bool shared : {false, true}) {
+          Block block;
+          IREmitter ir(&block);
+          const auto input = ir.GetUserData(static_cast<ScalarReg>(0));
+          const auto shifted = ir.Emit(ValueOpcode::ShiftLeftLogical32, {input, Value(count)});
+          const auto result = ir.Emit(ValueOpcode::BitwiseAnd32,
+              swapped ? std::initializer_list<Value>{Value(mask), shifted}
+                      : std::initializer_list<Value>{shifted, Value(mask)});
+          const auto other = shared ? ir.Emit(ValueOpcode::IAdd32, {shifted, Value(1u)}) : Value{};
+          ConstantPropagationPass({&block});
+          const bool zero = ((UINT32_MAX << (count & 31u)) & mask) == 0u;
+          Check(zero ? result.Resolve() == Value(0u) : !result.Resolve().IsImmediate(),
+                "masked shift retained impossible bits or discarded possible bits");
+          if (shared) {
+            Check(other.Resolve().Instruction()->Arg(0).Resolve() == shifted.Resolve(),
+                  "folding a masked shift changed its other consumer");
+          }
+          constexpr uint32_t source = 0xa5f03c69u;
+          input.Instruction()->ReplaceUsesWith(Value(source));
+          ConstantPropagationPass({&block});
+          const uint32_t expected = source << (count & 31u);
+          Check(result.Resolve() == Value(expected & mask) &&
+                    (!shared || other.Resolve() == Value(expected + 1u)),
+                "masked shift folding changed a runtime result");
+        }
+      }
+    }
+  }
+
+  Block block;
+  IREmitter ir(&block);
+  const auto input = ir.GetUserData(static_cast<ScalarReg>(0));
+  const auto count = ir.GetUserData(static_cast<ScalarReg>(1));
+  const auto shifted = ir.Emit(ValueOpcode::ShiftLeftLogical32, {input, count});
+  const auto result = ir.Emit(ValueOpcode::BitwiseAnd32, {shifted, Value(0xffu)});
+  ConstantPropagationPass({&block});
+  Check(result.Resolve().Instruction()->GetOpcode() == ValueOpcode::BitwiseAnd32,
+        "a variable shift count was treated as a constant");
+}
+
 void TestU64ShiftConstantPropagation() {
  using namespace ShaderRecompiler::IR;

@@ -14367,6 +14432,172 @@ void TestComputeImageFill() {
    Run(mutation);
 }

+void TestPatternFill() {
+  using namespace ShaderRecompiler::IR;
+  // Bounded DWORD pattern fill, including every instruction through S_ENDPGM.
+  const uint32_t bounded_shader[] = {
+      0xbfa00003u, 0xd7460002u, 0x04010c0au, 0x7da80408u,
+      0xbf88003fu, 0x7e000c09u, 0xbf070980u, 0x858a807eu,
+      0x7e005700u, 0x100000ffu, 0x4f800000u, 0x7e060f00u,
+      0xd5766a00u, 0x02020609u, 0x7d8a0280u, 0x4c020080u,
+      0x02000101u, 0xd56a0001u, 0x00020700u, 0x4c000303u,
+      0x4a020303u, 0x02000101u, 0xd56a0000u, 0x00020500u,
+      0xd5690001u, 0x00020009u, 0x4c060302u, 0x7d8c02f9u,
+      0x06068c02u, 0x7d860609u, 0x87ea6a0cu, 0x50000080u,
+      0xd5286a00u, 0x003200c1u, 0xd5010000u, 0x002a00c1u,
+      0xd5690000u, 0x00020009u, 0x4c000102u, 0x7d0a0080u,
+      0xbe88246au, 0xbf880015u, 0x7d0a0081u, 0xbe8a246au,
+      0xbf88000cu, 0x7d0a0082u, 0xbeea246au, 0xbf880003u,
+      0x7e000207u, 0xe0102000u, 0x80000002u, 0x8afe7e6au,
+      0xbf880003u, 0x7e000206u, 0xe0102000u, 0x80000002u,
+      0xbefe046au, 0x8afe7e0au, 0xbf880003u, 0x7e000205u,
+      0xe0102000u, 0x80000002u, 0xbefe040au, 0x8afe7e08u,
+      0xbf880003u, 0x7e000204u, 0xe0102000u, 0x80000002u,
+      0xbf810000u,
+  };
+  // Pattern fill whose bounds come from dispatch thread dimensions.
+  const uint32_t thread_dimension_shader[] = {
+      0xbfa00002u, 0x7e040c09u, 0x81ea0980u, 0xd7460000u,
+      0x04010c0au, 0x7e020205u, 0x7e045702u, 0x100404ffu,
+      0x4f7ffffeu, 0x7e040f02u, 0x7ed60502u, 0x936a6b6au,
+      0x9aea6b6au, 0x4a04046au, 0xd56a0002u, 0x00020102u,
+      0xd5690002u, 0x00020409u, 0x4c040500u, 0x7d860409u,
+      0x4e060409u, 0x02040702u, 0x7d860409u, 0x4e060409u,
+      0x02040702u, 0x7d840481u, 0x7d8404f9u, 0x06868882u,
+      0x7d8404f9u, 0x06868a83u, 0x02020204u, 0xd5010001u,
+      0x00200d01u, 0xd5010001u, 0x00280f01u, 0xe0102000u,
+      0x80000100u, 0xbf810000u,
+  };
+  // The 64-lane variant uses vector multiply-high for the remainder calculation.
+  const uint32_t thread_dimension_wave64_shader[] = {
+      0xbfa00002u, 0x7e060c09u, 0x4c0412f9u, 0x86860680u,
+      0xd7460000u, 0x04010c0au, 0x7e020205u, 0x7e065703u,
+      0x100606ffu, 0x4f7ffffeu, 0x7e060f03u, 0xd5690002u,
+      0x00020702u, 0xd56a0002u, 0x00020702u, 0x4a040503u,
+      0xd56a0002u, 0x00020102u, 0xd5690002u, 0x00020409u,
+      0x4c040500u, 0x7d860409u, 0x4c0612f9u, 0x86060602u,
+      0x02040702u, 0x7d860409u, 0x4c0612f9u, 0x86060602u,
+      0x02040702u, 0x7d040481u, 0x7d0404f9u, 0x06868882u,
+      0x7d0404f9u, 0x06868a83u, 0x02020204u, 0xd5010001u,
+      0x00200d01u, 0xd5010001u, 0x00280f01u, 0xe0102000u,
+      0x80000100u, 0xbf810000u,
+  };
+  // The 64-lane variant with scalar quotient setup.
+  const uint32_t thread_dimension_wave64_scalar_shader[] = {
+      0xbfa00002u, 0x7e040c09u, 0x81ea0980u, 0xd7460000u,
+      0x04010c0au, 0x7e020205u, 0x7e045702u, 0x100404ffu,
+      0x4f7ffffeu, 0x7e040f02u, 0x7ed60502u, 0x936a6b6au,
+      0x9aea6b6au, 0x816a6a6bu, 0xd56a0002u, 0x0002006au,
+      0xd5690002u, 0x00020409u, 0x4c040500u, 0x7d860409u,
+      0x4c0612f9u, 0x86060602u, 0x02040702u, 0x7d860409u,
+      0x4c0612f9u, 0x86060602u, 0x02040702u, 0x7d040481u,
+      0x7d0404f9u, 0x06868882u, 0x7d0404f9u, 0x06868a83u,
+      0x02020204u, 0xd5010001u, 0x00200d01u, 0xd5010001u,
+      0x00280f01u, 0xe0102000u, 0x80000100u, 0xbf810000u,
+  };
+  // Same layout, with vector arithmetic for the reciprocal correction.
+  auto thread_dimension_wave64_vector_shader = std::to_array(thread_dimension_wave64_scalar_shader);
+  thread_dimension_wave64_vector_shader[13] = 0x4a04046au;
+  thread_dimension_wave64_vector_shader[15] = 0x00020102u;
+  struct PatternCase {
+    std::span<const uint32_t> shader;
+    uint32_t wave_size;
+    bool bounded;
+    size_t literal_word;
+  };
+  const PatternCase cases[] = {
+      {bounded_shader, 64, true, 10},
+      {thread_dimension_shader, 32, false, 8},
+      {thread_dimension_wave64_shader, 64, false, 9},
+      {thread_dimension_wave64_scalar_shader, 64, false, 8},
+      {thread_dimension_wave64_vector_shader, 64, false, 8},
+  };
+  for (const auto &[shader, wave_size, bounded, literal_word] : cases) {
+    std::array<uint32_t, 10> userdata = {
+        0x00200000u, 4u << 16, 64u, 0x14204u,
+        0xff000000u, 0u, 0u, 0x08000000u, 64u, 1u,
+    };
+    ShaderComputeInputInfo compute{};
+    compute.threads_num[0] = 64;
+    compute.threads_num[1] = compute.threads_num[2] = 1;
+    compute.workgroup_register = 10;
+    compute.thread_ids_num = 1;
+    compute.group_id[0] = true;
+    compute.dispatch_thread_dimensions = !bounded;
+    compute.wave_size = wave_size;
+    auto options = MakeCompileOptions(ShaderType::Compute);
+    options.wave_size = compute.wave_size;
+    options.input_info.compute = &compute;
+    options.user_data = userdata;
+    // Recognition derives identity from code, independently of the caller's hash.
+    options.shader_hash = 0x12345678u;
+    const auto Build = [&](std::span<const uint32_t> code) {
+      auto translated = ShaderRecompiler::TranslateProgram(code, options);
+      return ExtractResourcePlan(translated.program);
+    };
+    auto plan = Build(shader);
+    ResourceSnapshot snapshot;
+    ResourceSpecialization specialization;
+    const auto Materialize = [&](bool expected) {
+      Check(MaterializeResources(plan, {.user_data = userdata}, snapshot, specialization),
+            "pattern fill did not materialize");
+      const auto &fill = snapshot.uniform_fill;
+      Check((fill.kind == UniformFillKind::Buffer) == expected,
+            "pattern fill used inactive words or retained a stale result");
+      if (expected)
+        Check(fill.value == userdata[4] && fill.words == 1 &&
+                  fill.bounded_elements == (bounded ? userdata[8] : 0u) &&
+                  fill.group_stride == std::array<uint32_t, 3>{64, 0, 0},
+              "pattern fill lost its value or coverage contract");
+    };
+    Materialize(true);
+    userdata[4] ^= 0x123456u;
+    Materialize(true);
+    userdata[4] ^= 0x123456u;
+    for (const uint32_t period : {0u, 2u, 3u, 4u, 5u}) {
+      userdata[9] = period;
+      Materialize(false);
+    }
+    for (uint32_t period = 2; period <= 4; ++period) {
+      userdata[9] = period;
+      userdata[3 + period] = userdata[4];
+      Materialize(true);
+      userdata[5] ^= 1u;
+      Materialize(false);
+      userdata[5] ^= 1u;
+    }
+    userdata[5] = userdata[6] = 0;
+    userdata[7] = 0x08000000u;
+    userdata[9] = 1;
+    Materialize(true);
+    userdata[8] = 0;
+    Materialize(!bounded);
+    userdata[8] = 63;
+    Materialize(true); // The renderer owns the descriptor-size equality check.
+    userdata[8] = 64;
+    Materialize(true);
+    std::vector<uint32_t> changed(shader.begin(), shader.end());
+    changed.push_back(0xdeadcafeu); // Non-executable metadata does not change identity.
+    Check(Build(changed).uniform_fill.fill.kind == UniformFillKind::Buffer,
+          "pattern recognition included bytes after S_ENDPGM");
+    changed[literal_word] ^= 0x00800000u;
+    Check(Build(changed).uniform_fill.fill.kind == UniformFillKind::None,
+          "pattern recognition accepted modified executable instructions");
+    for (const uint32_t invalid_input : {0u, 1u, 2u, 3u}) {
+      const auto saved_compute = compute;
+      const auto saved_wave = options.wave_size;
+      if (invalid_input == 0) compute.workgroup_register = 11;
+      if (invalid_input == 1) options.wave_size = wave_size == 64 ? 32 : 64;
+      if (invalid_input == 2) compute.dispatch_thread_dimensions = bounded;
+      if (invalid_input == 3) compute.float_mode ^= 1u;
+      Check(Build(shader).uniform_fill.fill.kind == UniformFillKind::None,
+            "pattern recognition accepted different shader launch inputs");
+      compute = saved_compute;
+      options.wave_size = saved_wave;
+    }
+  }
+}
+
 void TestGpuProducedWritableDescriptor() {
  using namespace ShaderRecompiler::IR;
  // Reduced from 994af59d50bedca8: the GPU-built BVH header supplies a relative
@@ -16267,6 +16498,7 @@ int main() {
  TestScalarAshrI64Decoder();
  TestImageAtomicWidthDecoder();
  TestDsReturningAtomic64Decoder();
+  TestDsCompareStore32Decoder();
  TestDsFloatMinMaxDecoder();
  TestNewShaderDecoderArchitecture();
  TestImageAddressOperands();
@@ -16364,6 +16596,7 @@ int main() {
  TestValuePhiValidation();
  TestLaneMaskProjection();
  TestLateConstantPropagation();
+  TestMaskedShiftConstantPropagation();
  TestU64ShiftConstantPropagation();
  TestU64RegisterPairRoundTrip();
 #if KYTY_PLATFORM != KYTY_PLATFORM_WINDOWS
@@ -16382,6 +16615,7 @@ int main() {
  TestNewShaderRecompilerEarlyZDisabledWhenPixelKillEnabled();
  TestTypedDescriptorRealWideMoveTranslation();
  TestComputeImageFill();
+  TestPatternFill();
  TestGpuProducedWritableDescriptor();
  TestUniformSelectedWritableDescriptor();
  TestUniformSelectedDescriptorLoadAddress();
Author	SHA1	Message	Date
Felipe	b95ea4bb5f	Support mountable-content DLC (addcont with real content) (#1581 ) Support mountable-content DLC (addcont) and fix PlayGo chunk count query	2026-10-17 04:44:28 +02:00
Nawid3333	e93b4b0378	shader: support returning 32-bit LDS compare-exchange (DS_CMPST_RTN_B32) (#1578 ) * shader: support DS_CMPST_RTN_B32 Assassin's Creed Valhalla (PPSA01490, #281) stops on DS opcode 0x30 in a compute shader. DS compares DATA and stores DATA2, the opposite of BUFFER_ATOMIC_CMPSWAP, so the 32-bit atomic path swaps the two for this opcode, as the 64-bit form does. Co-Authored-By: Claude Opus 5.5 <noreply@anthropic.com> * shader: reject unsupported GDS compare-store variant --------- Co-authored-by: Claude Opus 5.5 <noreply@anthropic.com> Co-authored-by: nmzik <Nmzik@mail.ru>	2026-10-17 04:37:16 +02:00
nmzik	0fd41ae206	Fix Linux CI xxHash include path (#1580 ) Fix xxHash include path capitalization for test builds	2026-10-17 04:37:00 +02:00
nmzik	9202bddad2	recognize verified pattern fills Co-authored-by: Juan Pablo Farias <juanpablo.profissional.a.f@gmail.com>	2026-10-17 04:09:54 +02:00
nmzik	32587337b5	renderdoc: advance capture flips independently of logging	2026-10-17 04:09:54 +02:00
Çağatay Güçlü	02ead0e2e1	shader: 64-bit buffer atomic add (BUFFER_ATOMIC_ADD_X2)	2026-10-17 04:09:54 +02:00
NeySlim	4f1f3b1bb3	shader: make the f32 clamp modifier turn NaN into 0	2026-10-17 04:09:53 +02:00
Nawid3333	42bb0da520	loader: keep the caller's flags in the TLS stubs (#1571 ) A patched TLS load calls TlsRegStub and SafeCall, which moved RSP past the red zone with sub/add rsp,0x80. ADD and SUB write the arithmetic flags, so the sub ran before SafeCall's pushfq saved them and the add ran after its popfq restored them: guest code that compares, loads fs:[0] and then branches on the flags could branch the wrong way. lea moves RSP without touching the flags. The entry lea plus a two-byte nop keeps every patched offset in both stubs; the exit lea is one byte longer and moves only ret. TlsStubsPreserveFlags calls the stubs with two flag patterns and checks the status flags, the output register and RAX; it fails with the old bytes. Co-authored-by: Claude Opus 5.5 <noreply@anthropic.com>	2026-10-17 00:05:08 +02:00
nmzik	5dec523143	Stub RazorCpu profiling markers in a dedicated library	2026-10-16 19:16:42 +02:00
nmzik	d01633e18e	Revert "Fix scalar buffer reads across sparse guest mappings" This reverts commit `d7974dc763`.	2026-10-16 19:13:36 +02:00
nmzik	e7ac242039	Load packaged system modules through the native module lifecycle	2026-10-16 19:13:36 +02:00
nmzik	d7974dc763	Fix scalar buffer reads across sparse guest mappings	2026-10-16 18:20:31 +02:00
shadowbeat070	b899717900	renderer: log unsupported depth feedback once	2026-10-16 17:24:45 +02:00
shadowbeat070	d58aefde49	renderer: reduce synchronization and allocation overhead Performance changes from #1495: - Sync changed BDA ranges instead of scanning all mappings. - Let VMA suballocate BDA buffers. - Remove the renderer lock from presentation. - Keep compatible pipeline caches across builds. - Fold masked shifts that always produce zero.	2026-10-16 17:18:49 +02:00