shader: keep quiet NaNs with payload bits out of the packed f16 min/max sNaN path (#1579)

V_PK_MIN_F16 and V_PK_MAX_F16 return the quieted NaN for a signaling NaN
operand and the other operand for a quiet one (per half, as V_MIN_F16 and
V_MAX_F16 do). The signaling test masked the half with 0x7c00 and checked
for a nonzero payload, so a quiet NaN with payload bits such as 0x7e01 was
taken as signaling and returned. Mask with 0x7e00 so the quiet bit must be
clear.

Two more vectors in PackedMinMaxF16NanAndSignedZeroEdges cover it.

Co-authored-by: Claude Opus 5.5 <noreply@anthropic.com>

CMakeLists.txt

@@ -350,7 +350,7 @@ set(inc_headers
 	${KYTY_THIRD_PARTY_DIR}/SPIRV-Tools/include
 	${KYTY_THIRD_PARTY_DIR}/SPIRV-Headers/include
 	${KYTY_THIRD_PARTY_DIR}/fmt/include
-	${KYTY_THIRD_PARTY_DIR}/xxhash
+	${KYTY_THIRD_PARTY_DIR}/xxHash
 	${KYTY_THIRD_PARTY_DIR}/cpuinfo/include
 	${KYTY_THIRD_PARTY_DIR}/renderdoc
 	${KYTY_THIRD_PARTY_DIR}/stb

src/graphics/host_gpu/renderer/renderCompute.cpp

@@ -94,9 +94,10 @@ bool ResolveComputeBufferFill(const ShaderComputeInputInfo& input, uint32_t grou
 	constexpr std::array formats {
 	    Prospero::BufferFormat::k32UInt, Prospero::BufferFormat::k32_32UInt,
 	    Prospero::BufferFormat::k32_32_32UInt, Prospero::BufferFormat::k32_32_32_32UInt};
+	// Offset-or-swizzle bounds do not describe the element range proved by the shader.
 	if (descriptor.Stride() != element_size || descriptor.Format() != formats[fill.words - 1] ||
 	    descriptor.SwizzleEnabled() || descriptor.IndexStride() != 0 || descriptor.AddTid() ||
-	    descriptor.Base48() == 0) {
+	    descriptor.OutOfBounds() == 3 || descriptor.Base48() == 0) {
 		return false;
 	}
 	if (input.threads_num[0] == 0 || input.threads_num[0] != fill.group_stride[0] ||
@@ -108,7 +109,11 @@ bool ResolveComputeBufferFill(const ShaderComputeInputInfo& input, uint32_t grou
 	                                 ? group_x
 	                                 : static_cast<uint64_t>(group_x) * input.threads_num[0];
 	const auto     size        = descriptor.GetSize();
-	if (invocations != descriptor.NumRecords() || size == 0 || size > UINT32_MAX ||
+	const auto elements = descriptor.NumRecords();
+	const bool covered = fill.bounded_elements != 0
+	                         ? fill.bounded_elements == elements && invocations >= elements
+	                         : invocations == elements;
+	if (!covered || invocations > uint64_t {UINT32_MAX} + 1 || size == 0 || size > UINT32_MAX ||
 	    (input.dispatch_thread_dimensions &&
 	     (group_x % input.threads_num[0] != 0 || input.dispatch_threads_num[0] != group_x ||
 	      input.dispatch_threads_num[1] != 1 || input.dispatch_threads_num[2] != 1))) {
@@ -252,10 +257,6 @@ void RenderExecutor::DispatchDirect(uint64_t submit_id, CommandBuffer& buffer,
 		return;
 	}
 
-	if (sh_ctx.GetCs().cs_regs.data_addr == 0) {
-		return;
-	}
-
 	constexpr uint32_t DISPATCH_INITIATOR_USE_THREAD_DIMENSIONS = 1u << 5u;
 	constexpr uint32_t DISPATCH_INITIATOR_BASE_BITS             = 0x41u;
 	constexpr uint32_t DISPATCH_INITIATOR_MODIFIER_BITS         = 0xa038u;

src/graphics/host_gpu/renderer/renderDraw.cpp

@@ -536,7 +536,10 @@ RenderState RenderExecutor::AcquireRenderTargets(CommandBuffer& buffer, RenderCo
 			}
 		}
 		if (feedback_aspects && !m_context.GetGraphics().attachment_feedback_loop_enabled) {
-			EXIT("depth attachment feedback loop is not supported by the host\n");
+			static std::atomic_bool logged = false;
+			if (!logged.exchange(true, std::memory_order_relaxed)) {
+				std::printf("depth attachment feedback loop is not supported by the host\n");
+			}
 		}
 		auto layout = depth_attachment_layout(depth);
 		if (sampled_aspects & ~DepthReadableAspects(layout)) {

src/graphics/presentation/renderDoc.cpp

@@ -162,7 +162,8 @@ void RenderDocOnGuestFlip(RenderContext& renderer) {
 		return;
 	}
 	if (state == RenderDocState::Capturing) {
-		LOGF("RenderDoc: captured guest flip %u/2\n", ++g_captured_flips);
+		++g_captured_flips;
+		LOGF("RenderDoc: captured guest flip %u/2\n", g_captured_flips);
 		if (g_captured_flips < 2) {
 			return;
 		}

src/graphics/shader/recompiler/backend/spirv/spirvEmitterAlu.cpp

@@ -230,8 +230,11 @@ uint32_t EmitPackFloat2x16Rtz(EmitterState& state, uint32_t arg0, uint32_t arg1)
 }
 
 uint32_t EmitFPSaturate32(EmitterState& state, uint32_t arg0) {
-	return EmitExt(state, TypeF32(state), GLSLstd450FClamp,
-	               {arg0, ConstantF32(state, 0), ConstantF32(state, 0x3f800000u)});
+	// The clamp modifier runs in DX10 clamp mode: NaN becomes 0. FClamp leaves NaN undefined.
+	const auto clamped = EmitExt(state, TypeF32(state), GLSLstd450FClamp,
+	                             {arg0, ConstantF32(state, 0), ConstantF32(state, 0x3f800000u)});
+	return Select(state, TypeF32(state), EmitClassifyF32(state, arg0).nan, ConstantF32(state, 0),
+	              clamped);
 }
 
 uint32_t EmitSMulHi(EmitterState& state, uint32_t arg0, uint32_t arg1) {

src/graphics/shader/recompiler/backend/spirv/spirvEmitterInstructions.h

@@ -296,6 +296,7 @@ inline constexpr auto EmitBufferAtomicSMax32    = EmitAtomic32;
 inline constexpr auto EmitBufferAtomicUMax32    = EmitAtomic32;
 inline constexpr auto EmitBufferAtomicAnd32     = EmitAtomic32;
 inline constexpr auto EmitBufferAtomicAnd64     = EmitBufferAtomic64;
+inline constexpr auto EmitBufferAtomicIAdd64    = EmitBufferAtomic64;
 inline constexpr auto EmitBufferAtomicOr32      = EmitAtomic32;
 inline constexpr auto EmitBufferAtomicOr64      = EmitBufferAtomic64;
 inline constexpr auto EmitBufferAtomicXor32     = EmitAtomic32;
@@ -325,6 +326,7 @@ inline constexpr auto EmitSharedAtomicSMin32    = EmitAtomic32;
 inline constexpr auto EmitSharedAtomicUMin32    = EmitAtomic32;
 inline constexpr auto EmitSharedAtomicSMax32    = EmitAtomic32;
 inline constexpr auto EmitSharedAtomicUMax64    = EmitSharedAtomic64;
+inline constexpr auto EmitSharedAtomicCmpSwap32 = EmitAtomic32;
 inline constexpr auto EmitSharedAtomicCmpSwap64 = EmitSharedAtomic64;
 inline constexpr auto EmitSharedAtomicUMax32    = EmitAtomic32;
 inline constexpr auto EmitSharedAtomicAnd32     = EmitAtomic32;

src/graphics/shader/recompiler/backend/spirv/spirvEmitterMemoryHelpers.cpp

@@ -504,6 +504,7 @@ uint32_t EncodePackedColorComponent(EmitterState& state, const Format::BufferFor
 
 spv::Op SpirvAtomicOpcode(IR::ValueOpcode opcode) {
 	switch (opcode) {
+		case IR::ValueOpcode::SharedAtomicCmpSwap32:
 		case IR::ValueOpcode::SharedAtomicCmpSwap64:
 		case IR::ValueOpcode::ImageAtomicCompareSwap32:
 		case IR::ValueOpcode::BufferAtomicCmpSwap32: return spv::OpAtomicCompareExchange;
@@ -516,6 +517,7 @@ spv::Op SpirvAtomicOpcode(IR::ValueOpcode opcode) {
 		case IR::ValueOpcode::ImageAtomicIAdd64:
 		case IR::ValueOpcode::BufferAtomicIAdd32:
 		case IR::ValueOpcode::SharedAtomicIAdd64:
+		case IR::ValueOpcode::BufferAtomicIAdd64:
 		case IR::ValueOpcode::SharedAtomicIAdd32: return spv::OpAtomicIAdd;
 		case IR::ValueOpcode::BufferAtomicISub32:
 		case IR::ValueOpcode::SharedAtomicISub32: return spv::OpAtomicISub;
@@ -560,8 +562,12 @@ uint32_t EmitAtomicOperation(ValueEmitContext& ctx, const IR::Inst& inst, uint32
 	const bool wide   = inst.GetType() == IR::Type::U64;
 	const auto type   = wide ? TypeU64(ctx.state) : TypeU32(ctx.state);
 	if (opcode == spv::OpAtomicCompareExchange) {
-		const auto desired    = ctx.Arg(inst, inst.NumArgs() - 3);
-		const auto comparator = ctx.Arg(inst, inst.NumArgs() - 2);
+		auto desired    = ctx.Arg(inst, inst.NumArgs() - 3);
+		auto comparator = ctx.Arg(inst, inst.NumArgs() - 2);
+		if (inst.GetOpcode() == IR::ValueOpcode::SharedAtomicCmpSwap32) {
+			// RDNA2 DS compares DATA and stores DATA2, opposite to BUFFER atomics.
+			std::swap(desired, comparator);
+		}
 		ctx.state.builder.AddFunction(
 		    spv::OpAtomicCompareExchange, TypeU32(ctx.state), old, pointer,
 		    ConstantU32(ctx.state, scope), ConstantU32(ctx.state, spv::MemorySemanticsMaskNone),

src/graphics/shader/recompiler/frontend/decode/MemoryOps.cpp

@@ -67,6 +67,7 @@ constexpr MemoryOpcodeInfo MUBUF_OPCODE_LIST[] = {
     {0x3fu, Opcode::BUFFER_ATOMIC_FMIN, 1, 32},
     {0x40u, Opcode::BUFFER_ATOMIC_FMAX, 1, 32},
     {0x50u, Opcode::BUFFER_ATOMIC_SWAP_X2, 2, 32},
+    {0x52u, Opcode::BUFFER_ATOMIC_ADD_X2, 2, 32},
     {0x59u, Opcode::BUFFER_ATOMIC_AND_X2, 2, 32},
     {0x5au, Opcode::BUFFER_ATOMIC_OR_X2, 2, 32},
     {0x80u, Opcode::BUFFER_LOAD_FORMAT_D16_X, 1, 16, false, false, true},
@@ -114,6 +115,7 @@ constexpr MemoryOpcodeInfo DS_OPCODE_LIST[] = {
     {0x27u, Opcode::DS_MIN_RTN_U32, 1, 32},      {0x28u, Opcode::DS_MAX_RTN_U32, 1, 32},
     {0x29u, Opcode::DS_AND_RTN_B32, 1, 32},      {0x2au, Opcode::DS_OR_RTN_B32, 1, 32},
     {0x2bu, Opcode::DS_XOR_RTN_B32, 1, 32},      {0x2du, Opcode::DS_WRXCHG_RTN_B32, 1, 32},
+    {0x30u, Opcode::DS_CMPST_RTN_B32, 1, 32},
     {0x35u, Opcode::DS_SWIZZLE_B32, 1, 32},      {0x36u, Opcode::DS_READ_B32, 1, 32},
     {0x37u, Opcode::DS_READ2_B32, 2, 32},        {0x38u, Opcode::DS_READ2ST64_B32, 2, 32},
     {0x39u, Opcode::DS_READ_I8, 1, 8, true},     {0x3au, Opcode::DS_READ_U8, 1, 8},
@@ -200,6 +202,7 @@ bool IsDsAtomicOpcode(Opcode opcode) {
 		case Opcode::DS_MAX_U32:
 		case Opcode::DS_MAX_RTN_U32:
 		case Opcode::DS_MAX_RTN_U64:
+		case Opcode::DS_CMPST_RTN_B32:
 		case Opcode::DS_CMPST_RTN_B64:
 		case Opcode::DS_AND_B32:
 		case Opcode::DS_AND_RTN_B32:
@@ -220,6 +223,7 @@ uint32_t DsSourceCount(Opcode opcode) {
 		case Opcode::DS_WRITE2ST64_B32:
 		case Opcode::DS_WRITE2_B64:
 		case Opcode::DS_WRITE2ST64_B64:
+		case Opcode::DS_CMPST_RTN_B32:
 		case Opcode::DS_CMPST_RTN_B64:
 		case Opcode::DS_MSKOR_B32: return 3u;
 		case Opcode::DS_MIN_F32:
@@ -454,6 +458,9 @@ void DecodeDs(uint32_t pc, std::span<const uint32_t> code, uint32_t word_index,
 	if (inst.opcode == Opcode::DS_SWIZZLE_B32 && inst.offset >= 0xe000u) {
 		SetUnsupported(inst, Family::DS, opcode, "DS swizzle FFT mode is not implemented");
 	}
+	if (inst.gds && inst.opcode == Opcode::DS_CMPST_RTN_B32) {
+		SetUnsupported(inst, Family::DS, opcode, "DS compare-store return B32 GDS variant is not implemented");
+	}
 	if (inst.gds &&
 	    (inst.opcode == Opcode::DS_SWIZZLE_B32 || inst.opcode == Opcode::DS_PERMUTE_B32 ||
 	     inst.opcode == Opcode::DS_BPERMUTE_B32 ||

src/graphics/shader/recompiler/frontend/decode/ShaderDecoder.cpp

@@ -670,6 +670,7 @@ std::string InstructionToString(const Instruction& inst) {
 		case Opcode::BUFFER_ATOMIC_CMPSWAP:
 		case Opcode::BUFFER_ATOMIC_SWAP_X2:
 		case Opcode::BUFFER_ATOMIC_ADD:
+		case Opcode::BUFFER_ATOMIC_ADD_X2:
 		case Opcode::BUFFER_ATOMIC_SUB:
 		case Opcode::BUFFER_ATOMIC_SMIN:
 		case Opcode::BUFFER_ATOMIC_UMIN:
@@ -717,6 +718,7 @@ std::string InstructionToString(const Instruction& inst) {
 		case Opcode::DS_MAX_U32:
 		case Opcode::DS_MAX_RTN_U32:
 		case Opcode::DS_MAX_RTN_U64:
+		case Opcode::DS_CMPST_RTN_B32:
 		case Opcode::DS_CMPST_RTN_B64:
 		case Opcode::DS_AND_B32:
 		case Opcode::DS_AND_RTN_B32:

src/graphics/shader/recompiler/frontend/decode/ShaderDecoder.h

@@ -586,6 +586,7 @@ enum class Opcode {
 	BUFFER_ATOMIC_CMPSWAP,
 	BUFFER_ATOMIC_SWAP_X2,
 	BUFFER_ATOMIC_ADD,
+	BUFFER_ATOMIC_ADD_X2,
 	BUFFER_ATOMIC_SUB,
 	BUFFER_ATOMIC_SMIN,
 	BUFFER_ATOMIC_UMIN,
@@ -631,6 +632,7 @@ enum class Opcode {
 	DS_MAX_U32,
 	DS_MAX_RTN_U32,
 	DS_MAX_RTN_U64,
+	DS_CMPST_RTN_B32,
 	DS_CMPST_RTN_B64,
 	DS_AND_B32,
 	DS_AND_RTN_B32,

src/graphics/shader/recompiler/frontend/translate/Float.cpp

@@ -27,7 +27,7 @@ void Translator::PackedFloat16(const Decoder::Instruction& inst, IR::ValueOpcode
 	if (quiet_snan) {
 		const auto quiet_snan_lane = [&](const Decoder::Operand& operand, bool high) {
 			const auto bits     = ReadU16LaneAsU32(operand, high, false);
-			const auto exponent = ir.BitwiseAnd(bits, IR::U32(IR::Value(0x7c00u)));
+			const auto exponent = ir.BitwiseAnd(bits, IR::U32(IR::Value(0x7e00u)));
 			const auto payload  = ir.BitwiseAnd(bits, IR::U32(IR::Value(0x01ffu)));
 			const auto snan     = ir.LogicalAnd(ir.IEqual(exponent, IR::U32(IR::Value(0x7c00u))),
 			                                    ir.INotEqual(payload, IR::U32(IR::Value(0u))));

src/graphics/shader/recompiler/frontend/translate/Memory.cpp

@@ -1046,6 +1046,8 @@ void Translator::EmitMemory(const Decoder::Instruction& inst) {
 			return BUFFER_ATOMIC(inst, IR::ValueOpcode::BufferAtomicFMin32);
 		case Decoder::Opcode::BUFFER_ATOMIC_FMAX:
 			return BUFFER_ATOMIC(inst, IR::ValueOpcode::BufferAtomicFMax32);
+		case Decoder::Opcode::BUFFER_ATOMIC_ADD_X2:
+			return BUFFER_ATOMIC(inst, IR::ValueOpcode::BufferAtomicIAdd64);
 
 		case Decoder::Opcode::DS_ADD_U32:
 			return DS_ATOMIC(inst, IR::ValueOpcode::SharedAtomicIAdd32, false);
@@ -1083,6 +1085,8 @@ void Translator::EmitMemory(const Decoder::Instruction& inst) {
 			return DS_ATOMIC(inst, IR::ValueOpcode::SharedAtomicUMax32, true);
 		case Decoder::Opcode::DS_MAX_RTN_U64:
 			return DS_ATOMIC(inst, IR::ValueOpcode::SharedAtomicUMax64, true);
+		case Decoder::Opcode::DS_CMPST_RTN_B32:
+			return DS_ATOMIC(inst, IR::ValueOpcode::SharedAtomicCmpSwap32, true);
 		case Decoder::Opcode::DS_CMPST_RTN_B64:
 			return DS_ATOMIC(inst, IR::ValueOpcode::SharedAtomicCmpSwap64, true);
 		case Decoder::Opcode::DS_AND_B32:

src/graphics/shader/recompiler/frontend/translate/Translate.cpp

@@ -546,9 +546,11 @@ IR::U32 Translator::Read16LaneBits(const Decoder::Operand& operand, bool high_la
 	auto bits             = ReadRawU32(operand);
 	if (operand.kind == Decoder::OperandKind::FloatInlineConstant) {
 		// Inline floats encode the operand's width before SDWA or OP_SEL selects bits.
-		const auto half      = IR::F16(ir.Emit(IR::ValueOpcode::ConvertF16F32, {ir.BitCastF32(bits)}));
-		const auto half_bits = IR::U16(ir.Emit(IR::ValueOpcode::BitCastU16F16, {half}));
-		bits                = IR::U32(ir.Emit(IR::ValueOpcode::ConvertU32U16, {half_bits}));
+		const auto value = operand.value;
+		bits             = IR::U32(IR::Value(value == 0x3e22f983u
+		                                         ? 0x3118u
+		                                         : ((value >> 16u) & 0x8000u) |
+		                                               ((((value >> 23u) & 0xffu) - 112u) << 10u)));
 	}
 	bits = ApplyBitSourceModifiers(raw_operand, bits);
 	uint32_t offset = (high_lane ? operand.op_sel_hi : operand.op_sel) ? 16u : 0u;

src/graphics/shader/recompiler/ir/ResourceSnapshot.h

@@ -25,6 +25,7 @@ struct UniformFill {
 	std::array<uint32_t, 3> group_stride {};
 	uint32_t                 words        = 0;
 	uint32_t                 value        = 0;
+	uint32_t                 bounded_elements = 0;
 
 	bool operator==(const UniformFill&) const = default;
 };

src/graphics/shader/recompiler/ir/ShaderIR.h

@@ -622,6 +622,8 @@ struct CompiledShaderInfo {
 struct UniformFillPlan {
 	UniformFill          fill;
 	std::array<Value, 4> values;
+	Value                pattern_size;
+	Value                element_count;
 };
 
 // Resource analysis retained by the shader cache. It owns immutable descriptor/SRT,

src/graphics/shader/recompiler/ir/opcodes/ValueOpcodes.cpp

@@ -118,6 +118,7 @@ BufferAccess BufferAccessOf(ValueOpcode opcode) {
 		case ValueOpcode::BufferAtomicCmpSwap32:
 		case ValueOpcode::BufferAtomicSwap64:
 		case ValueOpcode::BufferAtomicIAdd32:
+		case ValueOpcode::BufferAtomicIAdd64:
 		case ValueOpcode::BufferAtomicISub32:
 		case ValueOpcode::BufferAtomicSMin32:
 		case ValueOpcode::BufferAtomicUMin32:
@@ -137,6 +138,7 @@ BufferAccess BufferAccessOf(ValueOpcode opcode) {
 uint32_t BufferComponentCount(ValueOpcode opcode) {
 	switch (opcode) {
 		case ValueOpcode::BufferAtomicSwap64:
+		case ValueOpcode::BufferAtomicIAdd64:
 		case ValueOpcode::BufferAtomicAnd64:
 		case ValueOpcode::BufferAtomicOr64:
 		case ValueOpcode::LoadBufferU32x2:
@@ -176,6 +178,7 @@ SharedAccess SharedAccessOf(ValueOpcode opcode) {
 		case ValueOpcode::SharedAtomicSMax32:
 		case ValueOpcode::SharedAtomicUMax32:
 		case ValueOpcode::SharedAtomicUMax64:
+		case ValueOpcode::SharedAtomicCmpSwap32:
 		case ValueOpcode::SharedAtomicCmpSwap64:
 		case ValueOpcode::SharedAtomicAnd32:
 		case ValueOpcode::SharedAtomicOr32:

src/graphics/shader/recompiler/ir/opcodes/ValueOpcodes.inc

@@ -268,6 +268,7 @@ VALUE_OPCODE(BufferAtomicSwap32, U32, BufferResource, U32, U32, U32, U32, U1)
 VALUE_OPCODE(BufferAtomicCmpSwap32, U32, BufferResource, U32, U32, U32, U32, U32, U1)
 VALUE_OPCODE(BufferAtomicSwap64, U64, BufferResource, U32, U32, U32, U64, U1)
 VALUE_OPCODE(BufferAtomicIAdd32, U32, BufferResource, U32, U32, U32, U32, U1)
+VALUE_OPCODE(BufferAtomicIAdd64, U64, BufferResource, U32, U32, U32, U64, U1)
 VALUE_OPCODE(BufferAtomicISub32, U32, BufferResource, U32, U32, U32, U32, U1)
 VALUE_OPCODE(BufferAtomicSMin32, U32, BufferResource, U32, U32, U32, U32, U1)
 VALUE_OPCODE(BufferAtomicUMin32, U32, BufferResource, U32, U32, U32, U32, U1)
@@ -306,6 +307,7 @@ VALUE_OPCODE(SharedAtomicUMin32, U32, U32, U32, U1)
 VALUE_OPCODE(SharedAtomicSMax32, U32, U32, U32, U1)
 VALUE_OPCODE(SharedAtomicUMax32, U32, U32, U32, U1)
 VALUE_OPCODE(SharedAtomicUMax64, U64, U32, U64, U1)
+VALUE_OPCODE(SharedAtomicCmpSwap32, U32, U32, U32, U32, U1)
 VALUE_OPCODE(SharedAtomicCmpSwap64, U64, U32, U64, U64, U1)
 VALUE_OPCODE(SharedAtomicAnd32, U32, U32, U32, U1)
 VALUE_OPCODE(SharedAtomicOr32, U32, U32, U32, U1)

src/graphics/shader/recompiler/ir/passes/ResourceMaterialization.cpp

@@ -1424,10 +1424,11 @@ ResourcePlan ExtractResourcePlan(const Program& program) {
 		for (auto it = first; it != plan.value_storage.end(); ++it) it->Invalidate();
 		plan.value_storage.erase(first, plan.value_storage.end());
 	}
-	plan.uniform_fill = AnalyzeUniformFill(program);
-	for (uint32_t i = 0; i < plan.uniform_fill.fill.words; ++i) {
-		plan.uniform_fill.values[i] = Clone(plan.uniform_fill.values[i]);
-	}
+	plan.uniform_fill = program.uniform_fill.fill.kind != UniformFillKind::None
+	    ? program.uniform_fill : AnalyzeUniformFill(program);
+	for (auto& value: plan.uniform_fill.values) value = Clone(value);
+	plan.uniform_fill.pattern_size = Clone(plan.uniform_fill.pattern_size);
+	plan.uniform_fill.element_count = Clone(plan.uniform_fill.element_count);
 	plan.capture_specialization_reads |=
 	    capture_indirect_reads || capture_control_reads || !plan.source_reads.empty();
 	if (plan.capture_specialization_reads) {
@@ -1513,15 +1514,21 @@ bool MaterializeResources(const ResourcePlan& program, const SrtRuntime& runtime
 	}
 	snapshot.uniform_fill = {};
 	const auto& fill = program.uniform_fill;
-	const auto words = fill.fill.words;
+	auto words = fill.fill.words;
 	std::array<uint32_t, 4> stored {};
 	bool uniform_fill = words != 0;
+	if (uniform_fill && !fill.pattern_size.IsEmpty())
+		uniform_fill = clean.Evaluate(fill.pattern_size, words) && words >= 1 && words <= 4;
+	uint32_t elements = 0;
+	if (uniform_fill && !fill.element_count.IsEmpty())
+		uniform_fill = clean.Evaluate(fill.element_count, elements) && elements != 0;
 	for (uint32_t i = 0; i < words && uniform_fill; ++i) {
 		uniform_fill = clean.Evaluate(fill.values[i], stored[i]) && stored[i] == stored[0];
 	}
 	if (uniform_fill) {
 		snapshot.uniform_fill = fill.fill;
 		snapshot.uniform_fill.value = stored[0];
+		snapshot.uniform_fill.bounded_elements = elements;
 	}
 	const auto evaluate = [&](uint32_t source, DescriptorValue& value, bool written = false) {
 		if (source >= program.descriptor_sources.size()) {

src/graphics/shader/recompiler/ir/passes/ResourceTracking.cpp

@@ -15,6 +15,8 @@
 #include <tuple>
 #include <unordered_map>
 #include <utility>
+#define XXH_INLINE_ALL
+#include <xxhash.h>
 
 namespace Libs::Graphics::ShaderRecompiler::IR {
 namespace {
@@ -4040,11 +4042,71 @@ private:
 	bool                                       m_shader_writes = false;
 };
 
+UniformFillPlan RecognizePatternFill(Program& program, const Decoder::Program& decoded,
+                                   ShaderStageInputInfo input) {
+	struct Pattern {
+		size_t code_words;
+		uint64_t hash;
+		uint32_t wave_size;
+		bool bounded;
+	};
+	constexpr std::array patterns {
+		// Bounded 64-lane variant.
+		Pattern {.code_words = 69, .hash = 0x0704b9818e3cba06ull, .wave_size = 64, .bounded = true},
+		// Dispatch thread dimensions, 64-lane variants.
+		Pattern {.code_words = 42, .hash = 0x92ce57f21310ef02ull, .wave_size = 64, .bounded = false},
+		Pattern {.code_words = 40, .hash = 0x374161027f58bee7ull, .wave_size = 64, .bounded = false},
+		Pattern {.code_words = 40, .hash = 0x7d7172c56c2e296dull, .wave_size = 64, .bounded = false},
+		// Dispatch thread dimensions, 32-lane variant.
+		Pattern {.code_words = 38, .hash = 0xacbda1a1ccdd4198ull, .wave_size = 32, .bounded = false},
+	};
+	if (program.stage != ShaderType::Compute || input.compute == nullptr ||
+	    decoded.instructions.empty() || program.user_data_base != 0 || program.user_data_count < 10 ||
+	    program.info.buffers.size() != 1 || !program.info.images.empty()) return {};
+	const auto& cs = *input.compute;
+	const bool matching_inputs = cs.workgroup_register == 10 && cs.thread_ids_num == 1 &&
+	    cs.group_id[0] && !cs.group_id[1] && !cs.group_id[2] && cs.threads_num[0] == 64 &&
+	    cs.threads_num[1] == 1 && cs.threads_num[2] == 1 && cs.float_mode == 0xc0;
+	if (!matching_inputs) return {};
+	const auto& last = decoded.instructions.back();
+	const size_t words = last.pc / sizeof(uint32_t) + last.word_count;
+	if (words > decoded.code.size() ||
+	    std::ranges::find(patterns, words, &Pattern::code_words) == patterns.end()) return {};
+	// Hash executable instructions through S_ENDPGM, excluding binary metadata.
+	// Verified variants implement Destination[id] = value[id % pattern_size].
+	const auto hash = XXH3_64bits(decoded.code.data(), words * sizeof(uint32_t));
+	const auto pattern = std::ranges::find_if(patterns, [&](const Pattern& candidate) {
+		return candidate.code_words == words && candidate.hash == hash;
+	});
+	if (pattern == patterns.end() || program.wave_size != pattern->wave_size ||
+	    cs.dispatch_thread_dimensions != !pattern->bounded) return {};
+	std::array<Value, 10> userdata;
+	for (auto* block: program.blocks) {
+		for (auto& inst: *block) {
+			if (inst.GetOpcode() != ValueOpcode::GetUserData) continue;
+			const auto reg = RegIndex(inst.Arg(0).ScalarRegister());
+			if (reg < userdata.size()) userdata[reg] = Value(&inst);
+		}
+	}
+	for (uint32_t reg = 4; reg < 10; ++reg)
+		if ((reg != 8 || pattern->bounded) && userdata[reg].IsEmpty()) return {};
+	UniformFillPlan result;
+	result.fill.kind = UniformFillKind::Buffer;
+	result.fill.words = 1;
+	result.fill.group_stride[0] = 64;
+	std::copy_n(userdata.begin() + 4, 4, result.values.begin());
+	result.pattern_size = userdata[9];
+	// Only the bounded variant reads an explicit count; the other uses thread dimensions.
+	if (pattern->bounded) result.element_count = userdata[8];
+	return result;
+}
+
 } // namespace
 
 void TrackResources(Program& program, const Decoder::Program& decoded, const CFG::Graph& native_cfg,
                     ShaderStageInputInfo input) {
 	Tracker(program, decoded, native_cfg, input).Run();
+	program.uniform_fill = RecognizePatternFill(program, decoded, input);
 }
 
 } // namespace Libs::Graphics::ShaderRecompiler::IR

src/kernel/fileSystem.cpp

@@ -338,14 +338,20 @@ static bool HasWindowsForbiddenFilenameCharacter(const std::string& relative_pat
 std::filesystem::path MountPoints::ResolvePath(const std::string& mounted_name) {
 	Common::LockGuard lock(m_mutex);
 
+	// Guests join mount points and names with extra slashes (e.g. "/addcont0//file").
+	std::string normalized = Common::FixFilenameSlash(mounted_name);
+	normalized.erase(std::unique(normalized.begin(), normalized.end(),
+	                             [](char a, char b) { return a == '/' && b == '/'; }),
+	                 normalized.end());
+
 	// Match the entire guest path so a mount root works with or without a trailing slash.
-	const auto mounted_path = Common::FixDirectorySlash(mounted_name);
+	const auto mounted_path = Common::FixDirectorySlash(normalized);
 	const auto it           = std::find_if(
 	    m_mount_pairs.begin(), m_mount_pairs.end(),
 	    [&mounted_path](const MountPair& p) { return mounted_path.starts_with(p.point); });
 	if (it != m_mount_pairs.end()) {
 		const auto& p = *it;
-		auto rel_path = Common::RemoveFirst(Common::FixFilenameSlash(mounted_name), p.point.size());
+		auto rel_path = Common::RemoveFirst(normalized, p.point.size());
 		while (rel_path.starts_with('/')) {
 			rel_path = Common::RemoveFirst(rel_path, 1);
 		}

src/kernel/module.h

@@ -0,0 +1,24 @@
+#ifndef EMULATOR_SRC_KERNEL_MODULE_H_
+#define EMULATOR_SRC_KERNEL_MODULE_H_
+
+#include "common/abi.h"
+
+#include <cstddef>
+#include <cstdint>
+
+namespace Libs::LibKernel {
+
+using KernelModule = int32_t;
+struct KernelLoadModuleOpt;
+struct KernelUnloadModuleOpt;
+
+KernelModule KYTY_SYSV_ABI KernelLoadStartModule(const char* module_file_name, size_t args,
+                                                 const void* argp, uint32_t flags,
+                                                 const KernelLoadModuleOpt* opt, int* res);
+int KYTY_SYSV_ABI KernelStopUnloadModule(KernelModule handle, size_t args, const void* argp,
+                                        uint32_t flags, const KernelUnloadModuleOpt* opt,
+                                        int* res);
+
+} // namespace Libs::LibKernel
+
+#endif // EMULATOR_SRC_KERNEL_MODULE_H_

src/libs/libAmpr.cpp

@@ -25,6 +25,9 @@
 #include <string>
 #include <thread>
 #include <unordered_map>
+#include <atomic>
+#include <cinttypes>
+#include <iterator>
 #include <variant>
 #include <vector>
 
@@ -176,6 +179,14 @@ static int GetHostPathStat(const std::filesystem::path& host_path, LibKernel::Fi
 	return OK;
 }
 
+// Mounts can appear after a failed lookup (e.g. additional content), so forget negative results.
+void InvalidateFailedResolves() {
+	std::scoped_lock lock(g_mutex);
+	for (auto it = g_resolved_paths.begin(); it != g_resolved_paths.end();) {
+		it = it->second.result != OK ? g_resolved_paths.erase(it) : std::next(it);
+	}
+}
+
 static int ResolveOnePath(const char* guest_path, uint32_t* id, uint64_t* size) {
 	if (guest_path == nullptr || guest_path[0] == '\0') {
 		return LibKernel::KERNEL_ERROR_EINVAL;
@@ -225,6 +236,10 @@ static int ResolveOnePath(const char* guest_path, uint32_t* id, uint64_t* size)
 				log_missing = true;
 			}
 		}
+		if (!log_missing && info.result == OK && path.starts_with("/addcont")) {
+			LOGF("\tAPR resolve: %s -> %s (%" PRIu64 " bytes)\n", guest_path,
+			     Common::PathToString(info.host_path).c_str(), info.file_size);
+		}
 		if (log_missing) {
 			LOGF("\tAPR resolve missing path: %s -> %s\n", guest_path, Common::PathToString(info.host_path).c_str());
 		}
@@ -1181,6 +1196,13 @@ static int ExecuteCommand(const CommandBufferState::Command& entry) {
 		uint64_t bytes_read = 0;
 		auto     result = ReadHostFileToGuest(host_path, command.file_offset, command.destination,
 		                                      command.size, &bytes_read);
+		static std::atomic<uint32_t> s_logged_dlc_reads {0};
+		if (host_path.filename() == "re_dlc_000.pak" && s_logged_dlc_reads.fetch_add(1) < 32) {
+			LOGF("\tAPR read dlc: id=0x%08" PRIx32 ", offset=0x%016" PRIx64 ", size=0x%016" PRIx64
+			     ", read=0x%016" PRIx64 ", result=0x%08" PRIx32 "\n",
+			     command.file_id, command.file_offset, command.size, bytes_read,
+			     static_cast<uint32_t>(result));
+		}
 		if (result != OK) {
 			LOGF("\tAPR submit read failed: id=0x%08" PRIx32 ", result=0x%08" PRIx32 ", path=%s\n",
 			     command.file_id, static_cast<uint32_t>(result), Common::PathToString(host_path).c_str());

src/libs/libAppContent.cpp

@@ -22,13 +22,16 @@
 
 namespace Libs {
 
+namespace AprShared {
+void InvalidateFailedResolves();
+}
+
 LIB_VERSION("AppContent", 1, "AppContentUtil", 1, 1);
 
 namespace AppContent {
 
 constexpr uint32_t APP_CONTENT_APPPARAM_SKU_FLAG_FULL = 3;
 constexpr int      APP_CONTENT_ERROR_PARAMETER        = -2133262334; /* 0x80D90002 */
-constexpr int      APP_CONTENT_ERROR_BUSY             = -2133262333; /* 0x80D90003 */
 constexpr int      APP_CONTENT_ERROR_NOT_MOUNTED      = -2133262332; /* 0x80D90004 */
 constexpr int      APP_CONTENT_ERROR_NOT_FOUND        = -2133262331; /* 0x80D90005 */
 constexpr int      APP_CONTENT_ERROR_MOUNT_FULL       = -2133262330; /* 0x80D90006 */
@@ -207,7 +210,10 @@ int KYTY_SYSV_ABI AppContentAddcontMount(uint32_t                         servic
 
 	for (const auto& m: g_addcont_mounts) {
 		if (m.label == entry->entitlement_label) {
-			return APP_CONTENT_ERROR_BUSY;
+			const auto guest_point = m.point.substr(0, m.point.size() - 1);
+			std::strncpy(mount_point->data, guest_point.c_str(), sizeof(mount_point->data) - 1);
+			LOGF("\t already mounted %s\n", guest_point.c_str());
+			return OK;
 		}
 	}
 
@@ -232,7 +238,15 @@ int KYTY_SYSV_ABI AppContentAddcontMount(uint32_t                         servic
 	LOGF("\t mounted %s -> %s\n", mount.point.c_str(),
 	     Common::PathToString(entry->mount_dir).c_str());
 
-	std::strncpy(mount_point->data, mount.point.c_str(), sizeof(mount_point->data) - 1);
+	AprShared::InvalidateFailedResolves();
+
+	// Titles scan /download0/pdlc for downloaded content; it must exist even when empty.
+	if (const auto pdlc = LibKernel::FileSystem::GetRealFilename("/download0/pdlc"); !pdlc.empty()) {
+		std::filesystem::create_directories(pdlc, ec);
+	}
+
+	const auto guest_point = mount.point.substr(0, mount.point.size() - 1);
+	std::strncpy(mount_point->data, guest_point.c_str(), sizeof(mount_point->data) - 1);
 	g_addcont_mounts.push_back(std::move(mount));
 
 	return OK;
@@ -251,8 +265,8 @@ int KYTY_SYSV_ABI AppContentAddcontUnmount(const AppContentMountPoint* mount_poi
 		return APP_CONTENT_ERROR_NOT_MOUNTED;
 	}
 
-	const std::string point(mount_point->data,
-	                        strnlen(mount_point->data, sizeof(mount_point->data)));
+	const std::string point = Common::FixDirectorySlash(
+	    std::string_view(mount_point->data, strnlen(mount_point->data, sizeof(mount_point->data))));
 
 	std::lock_guard lock(g_addcont_mutex);
 
@@ -263,6 +277,7 @@ int KYTY_SYSV_ABI AppContentAddcontUnmount(const AppContentMountPoint* mount_poi
 	}
 	LibKernel::FileSystem::Umount(point);
 	g_addcont_mounts.erase(it);
+	AprShared::InvalidateFailedResolves();
 
 	return OK;
 }

src/libs/libKernel.cpp

@@ -11,6 +11,7 @@
 #include "kernel/eventQueue.h"
 #include "kernel/fileSystem.h"
 #include "kernel/memory.h"
+#include "kernel/module.h"
 #include "kernel/pthread.h"
 #include "kernel/semaphore.h"
 #include "kernel/syncOnAddress.h"
@@ -65,7 +66,6 @@ LIB_DEFINE(InitLibKernel_1_Apr);
 
 namespace LibKernel {
 
-using KernelModule                   = int32_t;
 using get_thread_atexit_count_func_t = KYTY_SYSV_ABI int (*)(KernelModule);
 using thread_atexit_report_func_t    = KYTY_SYSV_ABI void (*)(KernelModule);
 
@@ -1201,9 +1201,9 @@ static int KYTY_SYSV_ABI KernelRaiseException(Pthread thread, int signum) {
 	return OK;
 }
 
-static KYTY_SYSV_ABI KernelModule KernelLoadStartModule(const char* module_file_name, size_t args,
-                                                        const void* argp, uint32_t flags,
-                                                        const KernelLoadModuleOpt* opt, int* res) {
+KernelModule KYTY_SYSV_ABI KernelLoadStartModule(const char* module_file_name, size_t args,
+                                                 const void* argp, uint32_t flags,
+                                                 const KernelLoadModuleOpt* opt, int* res) {
 	PRINT_NAME();
 
 	// EXIT_NOT_IMPLEMENTED(!Common::Thread::IsMainThread());
@@ -1255,9 +1255,9 @@ static KYTY_SYSV_ABI KernelModule KernelLoadStartModule(const char* module_file_
 	return static_cast<KernelModule>(handle);
 }
 
-static int KYTY_SYSV_ABI KernelStopUnloadModule(KernelModule handle, size_t args, const void* argp,
-                                                uint32_t flags, const KernelUnloadModuleOpt* opt,
-                                                int* res) {
+int KYTY_SYSV_ABI KernelStopUnloadModule(KernelModule handle, size_t args, const void* argp,
+                                        uint32_t flags, const KernelUnloadModuleOpt* opt,
+                                        int* res) {
 	PRINT_NAME();
 
 	// EXIT_NOT_IMPLEMENTED(!Common::Thread::IsMainThread());

src/libs/libPlayGo.cpp

@@ -221,11 +221,17 @@ int KYTY_SYSV_ABI PlayGoGetChunkId(int handle, uint16_t* out_chunk_id_list,
 	if (out_entries == nullptr) {
 		return PLAYGO_ERROR_BAD_POINTER;
 	}
-	if (number_of_entries != 0 && out_chunk_id_list == nullptr) {
-		return PLAYGO_ERROR_BAD_POINTER;
+	if (out_chunk_id_list != nullptr && number_of_entries == 0) {
+		return PLAYGO_ERROR_BAD_SIZE;
 	}
 	ensure_chunks_loaded();
 
+	// A null output buffer asks for the total chunk count.
+	if (out_chunk_id_list == nullptr) {
+		*out_entries = g_chunks_num;
+		return OK;
+	}
+
 	const uint32_t entries = std::min(number_of_entries, g_chunks_num);
 	for (uint32_t i = 0; i < entries; i++) {
 		out_chunk_id_list[i] = static_cast<uint16_t>(i);

src/libs/libRazorCpu.cpp

@@ -1,9 +1,9 @@
 #include "common/abi.h"
-#include "common/common.h"
-#include "common/stringUtils.h"
 #include "libs/libs.h"
 #include "loader/symbolDatabase.h"
 
+#include <cstdint>
+
 namespace Libs {
 
 namespace LibRazorCpu {
@@ -16,8 +16,18 @@ static KYTY_SYSV_ABI uint32_t RazorCpuIsCapturing() {
 	return 0;
 }
 
+static int KYTY_SYSV_ABI RazorCpuPopMarker() {
+	return 0;
+}
+
+static int KYTY_SYSV_ABI RazorCpuPushMarker(const char*, uint32_t, uint32_t) {
+	return 0;
+}
+
 LIB_DEFINE(InitRazorCpu_1) {
 	LIB_FUNC("EboejOQvLL4", LibRazorCpu::RazorCpuIsCapturing);
+	LIB_FUNC("YpkGsMXP3ew", LibRazorCpu::RazorCpuPopMarker);
+	LIB_FUNC("zw+celG7zSI", LibRazorCpu::RazorCpuPushMarker);
 }
 
 } // namespace LibRazorCpu

src/libs/libSysmodule.cpp

@@ -3,6 +3,7 @@
 #include "common/common.h"
 #include "common/logging/log.h"
 #include "common/stringUtils.h"
+#include "kernel/module.h"
 #include "libs/libs.h"
 #include "loader/symbolDatabase.h"
 
@@ -20,14 +21,40 @@ int KYTY_SYSV_ABI KernelGetModuleInfoForUnwind(uint64_t addr, int flags, ModuleI
 
 namespace Sysmodule {
 
-constexpr int SYSMODULE_ERROR_UNLOADED = static_cast<int>(0x805a1001u);
+constexpr int      SYSMODULE_ERROR_UNLOADED = static_cast<int>(0x805a1001u);
+constexpr int      SYSMODULE_ERROR_FATAL    = static_cast<int>(0x805a10ffu);
 
-static std::mutex                             g_module_mutex;
-static std::unordered_map<uint16_t, uint32_t> g_module_refs;
+static constexpr struct NativeModule {
+	uint16_t    id;
+	const char* path;
+} g_native_modules[] = {{0x0115, "/app0/sce_module/libSceNpCppWebApi.prx"}};
 
-static void LoadModule(uint16_t id) {
+struct Module {
+	uint32_t                references = 0;
+	LibKernel::KernelModule handle     = -1;
+};
+
+static std::mutex                          g_module_mutex;
+static std::unordered_map<uint16_t, Module> g_modules;
+
+static int LoadModule(uint16_t id) {
 	std::scoped_lock lock(g_module_mutex);
-	++g_module_refs[id];
+	if (const auto module = g_modules.find(id); module != g_modules.end()) {
+		++module->second.references;
+		return 0;
+	}
+
+	LibKernel::KernelModule handle = -1;
+	for (const auto& module: g_native_modules) {
+		if (module.id != id) continue;
+		handle = LibKernel::KernelLoadStartModule(module.path, 0, nullptr, 0, nullptr, nullptr);
+		if (handle < 0) {
+			return handle;
+		}
+		break;
+	}
+	g_modules.emplace(id, Module {1, handle});
+	return 0;
 }
 
 static KYTY_SYSV_ABI int SysmoduleGetModuleInfoForUnwind(uint64_t addr, int flags,
@@ -40,8 +67,7 @@ static KYTY_SYSV_ABI int SysmoduleLoadModule(uint16_t id) {
 
 	LOGF("\t id = %d\n", static_cast<int>(id));
 
-	LoadModule(id);
-	return 0;
+	return LoadModule(id);
 }
 
 static KYTY_SYSV_ABI int SysmoduleUnloadModule(uint16_t id) {
@@ -50,12 +76,19 @@ static KYTY_SYSV_ABI int SysmoduleUnloadModule(uint16_t id) {
 	LOGF("\t id = %d\n", static_cast<int>(id));
 
 	std::scoped_lock lock(g_module_mutex);
-	const auto       module = g_module_refs.find(id);
-	if (module == g_module_refs.end()) {
+	const auto       module = g_modules.find(id);
+	if (module == g_modules.end()) {
 		return SYSMODULE_ERROR_UNLOADED;
 	}
-	if (--module->second == 0) {
-		g_module_refs.erase(module);
+	if (module->second.references == 1) {
+		if (module->second.handle >= 0 &&
+		    LibKernel::KernelStopUnloadModule(module->second.handle, 0, nullptr, 0, nullptr,
+		                                     nullptr) < 0) {
+			return SYSMODULE_ERROR_FATAL;
+		}
+		g_modules.erase(id);
+	} else {
+		--module->second.references;
 	}
 	return 0;
 }
@@ -71,10 +104,8 @@ static KYTY_SYSV_ABI int SysmoduleLoadModuleInternalWithArg(uint16_t id, int arg
 	EXIT_IF(arg3 != 0);
 	EXIT_IF(ret == nullptr);
 
-	LoadModule(id);
-	*ret = 0;
-
-	return 0;
+	*ret = LoadModule(id);
+	return *ret;
 }
 
 static KYTY_SYSV_ABI int SysmoduleIsLoaded(uint16_t id) {
@@ -83,7 +114,7 @@ static KYTY_SYSV_ABI int SysmoduleIsLoaded(uint16_t id) {
 	LOGF("\t id = %d\n", static_cast<int>(id));
 
 	std::scoped_lock lock(g_module_mutex);
-	return g_module_refs.contains(id) ? 0 : SYSMODULE_ERROR_UNLOADED;
+	return g_modules.contains(id) ? 0 : SYSMODULE_ERROR_UNLOADED;
 }
 
 } // namespace Sysmodule

src/loader/jit.h

@@ -58,16 +58,17 @@ struct TlsRegStub {
 	}
 	static uint64_t GetSize() { return 32; }
 
-	// sub rsp,0x80
+	// lea rsp,[rsp-0x80]
+	// nop
 	// push rax
 	// call safe_call
 	// mov <reg>,rax
 	// pop rax
-	// add rsp,0x80
+	// lea rsp,[rsp+0x80]
 	// ret
-	uint8_t code[32] = {0x48, 0x81, 0xEC, 0x80, 0x00, 0x00, 0x00, 0x50, 0xE8, 0x00, 0x00,
-	                    0x00, 0x00, 0x48, 0x89, 0xC0, 0x58, 0x48, 0x81, 0xC4, 0x80, 0x00,
-	                    0x00, 0x00, 0xC3, 0x90, 0x90, 0x90, 0x90, 0x90, 0x90, 0x90};
+	uint8_t code[32] = {0x48, 0x8D, 0x64, 0x24, 0x80, 0x66, 0x90, 0x50, 0xE8, 0x00, 0x00,
+	                    0x00, 0x00, 0x48, 0x89, 0xC0, 0x58, 0x48, 0x8D, 0xA4, 0x24, 0x80,
+	                    0x00, 0x00, 0x00, 0xC3, 0x90, 0x90, 0x90, 0x90, 0x90, 0x90};
 };
 
 struct SafeCall {
@@ -78,7 +79,8 @@ struct SafeCall {
 	static uint64_t GetSize() { return 0x1000; }
 
 	uint8_t code[0x90] = {
-	    /*00*/ 0x48, 0x81, 0xec, 0x80, 0x00, 0x00, 0x00, // sub    rsp,0x80
+	    /*00*/ 0x48, 0x8d, 0x64, 0x24, 0x80,             // lea    rsp,[rsp-0x80]
+	    /*05*/ 0x66, 0x90,                               // nop
 	    /*07*/ 0x9c,                                     // pushfq
 	    /*08*/ 0x51,                                     // push   rcx
 	    /*09*/ 0x52,                                     // push   rdx
@@ -108,9 +110,9 @@ struct SafeCall {
 	    /*40*/ 0x5a,             // pop    rdx
 	    /*41*/ 0x59,             // pop    rcx
 	    /*42*/ 0x9d,             // popfq
-	    /*43*/ 0x48, 0x81, 0xc4, 0x80, 0x00, 0x00,
-	    0x00,        // add    rsp,0x80
-	    /*4a*/ 0xc3, // ret
+	    /*43*/ 0x48, 0x8d, 0xa4, 0x24, 0x80, 0x00, 0x00,
+	    0x00,        // lea    rsp,[rsp+0x80]
+	    /*4b*/ 0xc3, // ret
 	};
 };
 

tests/KernelFileSystemTests.cpp

@@ -235,6 +235,9 @@ void TestSysmoduleReferences() {
   const auto unload = resolve("eR2bZFAAU0Q");
   const auto is_loaded = resolve("fMP5NHUOaMk");
   constexpr int unloaded = static_cast<int>(0x805a1001u);
+  Check(load(0x115) == Libs::LibKernel::KERNEL_ERROR_ENOENT && is_loaded(0x115) == unloaded &&
+            unload(0x115) == unloaded,
+        "missing packaged NpCppWebApi fails without recording a loaded module");
   Check(load(0x113) == OK && is_loaded(0xb4) == unloaded,
         "loading entitlement access does not mark AppContent loaded");
   Check(unload(0x113) == OK, "release independent module reference");
@@ -255,6 +258,9 @@ void TestSysmoduleReferences() {
       reinterpret_cast<int (KYTY_SYSV_ABI *)(uint16_t, int, int, int, int*)>(
           internal_symbol->vaddr);
   int result = -1;
+  Check(internal_load(0x115, 0, 0, 0, &result) == Libs::LibKernel::KERNEL_ERROR_ENOENT &&
+            result == Libs::LibKernel::KERNEL_ERROR_ENOENT && is_loaded(0x115) == unloaded,
+        "internal module loading propagates missing native module failure");
   Check(internal_load(0xb4, 0, 0, 0, &result) == OK && result == OK &&
             is_loaded(0xb4) == OK && unload(0xb4) == OK && is_loaded(0xb4) == unloaded,
         "internal and public module operations share load state");
@@ -407,7 +413,9 @@ void TestAddContent(const std::filesystem::path& root) {
   Check(mount(0, nullptr, &point) == static_cast<int>(0x80d90002u) &&
             list(0, nullptr, 0, nullptr) == static_cast<int>(0x817d0002u),
         "null add-on parameters return parameter errors");
-  Check(mount(0, &info[3].label, &point) == OK, "mount owned add-on");
+  Check(mount(0, &info[3].label, &point) == OK &&
+            !std::string_view(point.data).ends_with('/'),
+        "mount owned add-on without trailing slash");
   const std::string mounted_file = std::string(point.data) + "/payload.txt";
   const int fd = FileSystem::KernelOpen(mounted_file.c_str(), 0, 0);
   std::array<char, 11> payload {};
@@ -416,8 +424,9 @@ void TestAddContent(const std::filesystem::path& root) {
             FileSystem::KernelClose(fd) == OK,
         "guest reads files from the selected title's add-on");
   MountPoint repeated {};
-  Check(mount(0, &info[3].label, &repeated) == static_cast<int>(0x80d90003u),
-        "duplicate mount returns BUSY");
+  Check(mount(0, &info[3].label, &repeated) == OK &&
+            std::strcmp(repeated.data, point.data) == 0,
+        "duplicate mount returns the same mount point");
   Check(unmount(&point) == OK && FileSystem::GetRealFilename(mounted_file).empty() &&
             unmount(&point) == static_cast<int>(0x80d90004u),
         "unmount removes mapping and repeated unmount reports NOT_MOUNTED");

tests/ShaderRecompilerComputeTests.cpp

@@ -20991,6 +20991,7 @@ CoverageClass ClassifyOpcode(ShaderOpcode opcode,
   case Opcode::BUFFER_ATOMIC_SMAX:
   case Opcode::BUFFER_ATOMIC_UMAX:
   case Opcode::BUFFER_ATOMIC_AND:
+  case Opcode::BUFFER_ATOMIC_ADD_X2:
   case Opcode::BUFFER_ATOMIC_AND_X2:
   case Opcode::BUFFER_ATOMIC_OR:
   case Opcode::BUFFER_ATOMIC_XOR:
@@ -21014,6 +21015,7 @@ CoverageClass ClassifyOpcode(ShaderOpcode opcode,
   case Opcode::DS_ADD_U32:
   case Opcode::DS_ADD_U64:
   case Opcode::DS_MAX_RTN_U64:
+  case Opcode::DS_CMPST_RTN_B32:
   case Opcode::DS_CMPST_RTN_B64:
   case Opcode::DS_ADD_RTN_U32:
   case Opcode::DS_SUB_U32:
@@ -24667,6 +24669,56 @@ TestCase Vop3MulLoU16CapturedAndSelectors() {
   return test;
 }
 
+TestCase Vop3F32ClampFlushesNaN() {
+  using O = ShaderOpcode;
+  struct ClampCase {
+    u32 opcode, input, expected;
+    u32 abs = 0;
+  };
+  constexpr u32 inf = 0x7f800000u, one = 0x3f800000u;
+  // GoW Ragnarok PS 3d52c7172f95fd37 0x2564: v_mul_f32 v18 clamp, inf, |v2|; and
+  // fma(x, inf, inf) clamp. The clamp modifier (DX10 clamp mode) turns their NaN into 0.
+  const std::array<ClampCase, 10> cases{{
+      {0x108, 0x00000000u, 0, 2}, // inf * |0| = NaN.
+      {0x108, 0x80000000u, 0, 2}, // inf * |-0| = NaN.
+      {0x108, 0x3f000000u, one, 2},
+      {0x108, 0xc0400000u, one, 2},
+      {0x14b, 0xbf800000u, 0}, // -1 * inf + inf = NaN.
+      {0x14b, 0x40000000u, one},
+      {0x103, 0x7fc00000u, 0}, // Quiet NaN input.
+      {0x103, 0xffc00001u, 0}, // Negative NaN input.
+      {0x103, 0x3e800000u, 0x3e800000u},
+      {0x103, 0x3fc00000u, one},
+  }};
+  TestCase test;
+  test.name = "Vop3F32ClampFlushesNaN";
+  for (const auto &entry : cases) test.initial.push_back(entry.input);
+  test.expected = test.initial;
+  auto &code = test.code;
+  AppendVMovLiteral(&code, 20, inf);
+  AppendVMovU32(&code, 21, 0);
+  for (u32 i = 0; i < cases.size(); ++i) {
+    const auto &entry = cases[i];
+    AppendVMovU32(&code, 30, i * 4u);
+    AppendBufferLoadDword(&code, 2, 30); // Runtime inputs prevent folding.
+    AppendVMovLiteral(&code, 18, 0xa5a5beefu);
+    if (entry.opcode == 0x108) {
+      AppendVop3(&code, 0x108, 18, Vgpr(20), Vgpr(2), 0, entry.abs, 0, true);
+    } else if (entry.opcode == 0x14b) {
+      AppendVop3(&code, 0x14b, 18, Vgpr(2), Vgpr(20), Vgpr(20), 0, 0, true);
+    } else {
+      AppendVop3(&code, 0x103, 18, Vgpr(2), Vgpr(21), 0, 0, 0, true);
+    }
+    AppendStoreVgpr(&code, 18, static_cast<u32>(cases.size()) + i);
+    test.expected.push_back(entry.expected);
+  }
+  AppendEnd(&code);
+  test.opcodes = {O::V_MOV_B32, O::BUFFER_LOAD_DWORD, O::V_MUL_F32, O::V_FMA_F32,
+                  O::V_ADD_F32, O::BUFFER_STORE_DWORD, O::S_ENDPGM};
+  test.decoded_counts = {{"V_MUL_F32", 4}, {"V_FMA_F32", 2}, {"V_ADD_F32", 4}};
+  return test;
+}
+
 TestCase Vop3MadI16CapturedSelectorsAndSaturation() {
   using O = ShaderOpcode;
   struct MadCase {
@@ -25828,13 +25880,17 @@ TestCase PackedMinMaxF16NanAndSignedZeroEdges() {
   AppendVMovLiteral(&code, 3, 0x80007e00u);
   AppendVMovLiteral(&code, 4, 0x40007d01u);
   AppendVMovLiteral(&code, 5, 0x7d014000u);
+  AppendVMovLiteral(&code, 6, 0x40007e01u);
+  AppendVMovLiteral(&code, 7, 0x7e013c00u);
 
   AppendVop3p(&code, 0x11, 10, Vgpr(0), Vgpr(1), 0, 0x3);
   AppendVop3p(&code, 0x12, 11, Vgpr(2), Vgpr(3), 0, 0x3);
   AppendVop3p(&code, 0x11, 12, Vgpr(4), Vgpr(5), 0, 0x3);
   AppendVop3p(&code, 0x12, 13, Vgpr(4), Vgpr(5), 0, 0x3);
+  AppendVop3p(&code, 0x11, 14, Vgpr(6), Vgpr(7), 0, 0x3);
+  AppendVop3p(&code, 0x12, 15, Vgpr(6), Vgpr(7), 0, 0x3);
 
-  for (u32 i = 0; i < 4; i++) {
+  for (u32 i = 0; i < 6; i++) {
     AppendStoreVgpr(&code, 10 + i, i);
   }
   AppendEnd(&code);
@@ -25842,7 +25898,7 @@ TestCase PackedMinMaxF16NanAndSignedZeroEdges() {
   return {"PackedMinMaxF16NanAndSignedZeroEdges",
           code,
           {},
-          {0x80004000u, 0x00004000u, 0x7f017f01u, 0x7f017f01u},
+          {0x80004000u, 0x00004000u, 0x7f017f01u, 0x7f017f01u, 0x40003c00u, 0x40003c00u},
           {O::V_MOV_B32, O::V_PK_MIN_F16, O::V_PK_MAX_F16,
            O::BUFFER_STORE_DWORD, O::S_ENDPGM}};
 }
@@ -34647,6 +34703,60 @@ TestCase DsAtomicReturn64() {
   return test;
 }
 
+TestCase DsCompareStoreReturn32() {
+  using O = ShaderOpcode;
+  TestCase test;
+  test.name = "DsCompareStoreReturn32";
+  auto &code = test.code;
+  const auto result = [&](u32 reg, u32 value) {
+    AppendStoreVgpr(&code, reg, static_cast<u32>(test.expected.size()));
+    test.expected.push_back(value);
+  };
+  struct Case { u32 before, compare, replacement; };
+  constexpr std::array cases{Case{0x12345678u, 0x12345678u, 0x9abcdef0u},
+                             Case{0x12345678u, 0x9abcdef0u, 0x12345678u},
+                             Case{0x80000000u, 0u, 0x7fffffffu},
+                             Case{UINT32_MAX, UINT32_MAX, 0u}};
+  AppendVMovU32(&code, 7, 0);
+  for (const auto &item : cases) {
+    AppendVMovLiteral(&code, 2, item.before);
+    code.push_back(EncodeDs0(0x0d, 8));
+    code.push_back(EncodeDs1(0, 2, 7));
+    AppendVMovLiteral(&code, 5, item.compare);
+    AppendVMovLiteral(&code, 19, item.replacement);
+    AppendVMovU32(&code, 4, 8);
+    // CS04b812daaacd532f pc 0x3e8: DATA compares, DATA2 replaces, VDst aliases ADDR.
+    code.insert(code.end(), {0xd8c00000u, 0x04130504u});
+    result(4, item.before);
+    code.push_back(EncodeDs0(0x36, 8));
+    code.push_back(EncodeDs1(8, 0, 7));
+    result(8, item.before == item.compare ? item.replacement : item.before);
+  }
+  AppendVMovU32(&code, 5, 0);
+  AppendVMovU32(&code, 4, 12); // Past the three LDS DWORDs.
+  code.insert(code.end(), {0xd8c00000u, 0x04130504u});
+  result(4, 0);
+  AppendVMovLiteral(&code, 2, 0x12345678u);
+  code.push_back(EncodeDs0(0x0d, 8));
+  code.push_back(EncodeDs1(0, 2, 7));
+  AppendVMovLiteral(&code, 5, 0x12345678u);
+  AppendVMovU32(&code, 4, 8);
+  code.push_back(EncodeSMovB32(126, InlineU32(0)));
+  code.insert(code.end(), {0xd8c00000u, 0x04130504u});
+  code.push_back(EncodeSMovB32(126, InlineU32(1)));
+  result(4, 8); // An inactive lane keeps its destination and leaves memory alone.
+  code.push_back(EncodeDs0(0x36, 8));
+  code.push_back(EncodeDs1(8, 0, 7));
+  result(8, 0x12345678u);
+  AppendEnd(&code);
+  test.compute_info.lds_size_dwords = 3;
+  test.opcodes = {O::V_MOV_B32, O::S_MOV_B32, O::DS_WRITE_B32, O::DS_CMPST_RTN_B32,
+                  O::DS_READ_B32, O::BUFFER_STORE_DWORD, O::S_ENDPGM};
+  test.required_spirv = {"OpAtomicCompareExchange %uint"};
+  test.forbidden_spirv = {"OpLoopMerge"};
+  return test;
+}
+
 TestCase DsAtomic64Contention(bool add, u32 wave_size) {
   using O = ShaderOpcode;
   TestCase test;
@@ -35662,6 +35772,40 @@ TestCase BufferAtomicAndX2GlcAndExec() {
   return test;
 }
 
+TestCase BufferAtomicAddX2GlcAndExec() {
+  using O = ShaderOpcode;
+  TestCase test;
+  test.name = "BufferAtomicAddX2GlcAndExec";
+  auto &code = test.code;
+  code.push_back(EncodeSop1(0x04, 12, 126));
+  AppendVMovU32(&code, 20, 0);
+  const u32 values[][2] = {{0x0ff00ff0u, 0xf00ff00fu},
+                         {0xff00ff00u, 0x00ff00ffu},
+                         {0x76543210u, 0xfedcba98u}};
+  for (u32 i = 0; i < std::size(values); ++i) {
+    AppendVMovLiteral(&code, 2, values[i][0]);
+    AppendVMovLiteral(&code, 3, values[i][1]);
+    if (i == 2) {
+      code.push_back(EncodeSop1(0x04, 126, InlineU32(0)));
+    }
+    AppendBufferStoreOpcode(&code, 0x52, 2, 20, i != 1);
+    code.push_back(EncodeSop1(0x04, 126, 12));
+    AppendStoreVgpr(&code, 2, 2 + i * 2);
+    AppendStoreVgpr(&code, 3, 3 + i * 2);
+  }
+  AppendEnd(&code);
+  test.initial = {0x12345678u, 0x87654321u, 0, 0, 0, 0, 0, 0};
+  // The first add wraps at 64 bits; the second carries from the low dword.
+  // GLC returns the first old value, preserves the second operand, and EXEC = 0
+  // suppresses the third add and its writeback despite its nonzero operand.
+  test.expected = {0x21256568u, 0x78743430u, 0x12345678u, 0x87654321u,
+                   0xff00ff00u, 0x00ff00ffu, 0x76543210u, 0xfedcba98u};
+  test.opcodes = {O::V_MOV_B32, O::S_MOV_B64, O::BUFFER_ATOMIC_ADD_X2,
+                 O::BUFFER_STORE_DWORD, O::S_ENDPGM};
+  test.required_spirv = {"OpAtomicIAdd %ulong", "Int64Atomics"};
+  return test;
+}
+
 TestCase BufferAtomicCmpSwapExactRaw() {
   using O = ShaderOpcode;
 
@@ -41862,6 +42006,7 @@ std::vector<TestCase> MakeCases() {
   AddCase(Vop3CvtPkI16I32Captured);
   AddCase(Vop3MulLoU16CapturedAndSelectors);
   AddCase(Vop3MadI16CapturedSelectorsAndSaturation);
+  AddCase(Vop3F32ClampFlushesNaN);
   AddCase(Vop3Min3U16CapturedAndSelectors);
   AddCase(Vop3Med3I16Captured);
   AddCase(Vop2SdwaMinU32PreservesWordDestination);
@@ -42190,6 +42335,7 @@ std::vector<TestCase> MakeCases() {
   }
   AddCase(DsAtomicReturnVariants);
   AddCase(DsAtomicReturn64);
+  AddCase(DsCompareStoreReturn32);
   for (bool add : {false, true}) {
     cases.push_back(DsAtomic64Bounds(add));
     for (u32 wave_size : {32, 64}) {
@@ -42197,6 +42343,7 @@ std::vector<TestCase> MakeCases() {
     }
   }
   AddCase(BufferAtomicAndX2GlcAndExec);
+  AddCase(BufferAtomicAddX2GlcAndExec);
   AddCase(ImageAtomicSignedMinMax<false>);
   AddCase(ImageAtomicSignedMinMax<true>);
   AddCase(ImageAtomicSignedMinMax<false, true>);
@@ -42553,6 +42700,30 @@ void CheckPs5GameExampleImageClearRuntimeShape() {
           !ResolveComputeBufferFill(compute, 2, 1, 1, 0x41u, descriptor,
                                     packed_clear, size),
           "excess invocation coverage was accepted as a fill");
+  runtime_resources.uniform_fill.bounded_elements = 65;
+  runtime_resources.buffers[0].dwords[2] = 65;
+  for (u32 oob = 0; oob < 4; ++oob) {
+    runtime_resources.buffers[0].dwords[3] = 0x14204u | (oob << 28u);
+    Require("Ps5GameExampleImageClear", "bounded final workgroup and RDNA2 OOB mode",
+            ResolveComputeBufferFill(compute, 2, 1, 1, 0x41u, descriptor,
+                                     packed_clear, size) == (oob != 3),
+            "explicit coverage did not respect RDNA2 element bounds");
+  }
+  runtime_resources.buffers[0].dwords[3] = 0x14204u;
+  Require("Ps5GameExampleImageClear", "bounded insufficient dispatch",
+          !ResolveComputeBufferFill(compute, 1, 1, 1, 0x41u, descriptor,
+                                    packed_clear, size),
+          "a bound was mistaken for actual invocation coverage");
+  Require("Ps5GameExampleImageClear", "bounded index wrap",
+          !ResolveComputeBufferFill(compute, 0x04000001u, 1, 1, 0x41u, descriptor,
+                                    packed_clear, size),
+          "a wrapped invocation index was accepted as complete coverage");
+  runtime_resources.uniform_fill.bounded_elements = 64;
+  Require("Ps5GameExampleImageClear", "bounded incomplete destination",
+          !ResolveComputeBufferFill(compute, 2, 1, 1, 0x41u, descriptor,
+                                    packed_clear, size),
+          "a shader bound smaller than the destination was accepted as a fill");
+  runtime_resources = scalar;
   auto alias = scalar.buffers.front();
   runtime_resources.buffers.push_back(alias);
   Require("Ps5GameExampleImageClear", "aliased scalar input",
@@ -47689,7 +47860,8 @@ int main(int argc, char **argv) {
     RunCase(&vulkan, DsWideLdsPartialBounds());
     RunCase(&vulkan, DsAtomic64Bounds(false));
     RunCase(&vulkan, DsAtomic64Bounds(true));
-    for (auto test : {DsWideLdsPartialBounds(), DsAtomic64Bounds(false), DsAtomic64Bounds(true), DsAtomicReturn64()}) {
+    for (auto test : {DsWideLdsPartialBounds(), DsAtomic64Bounds(false), DsAtomic64Bounds(true), DsAtomicReturn64(),
+                      DsCompareStoreReturn32()}) {
       test.compute_info.lds_storage = true;
       std::erase(test.required_spirv, "WorkgroupMemoryExplicitLayoutKHR");
       test.required_spirv.push_back("StorageBuffer");
@@ -47735,6 +47907,7 @@ int main(int argc, char **argv) {
     }
     RunCase(&vulkan, DsAtomicReturn64());
     RunCase(&vulkan, BufferAtomicAndX2GlcAndExec());
+    RunCase(&vulkan, BufferAtomicAddX2GlcAndExec());
     RunCase(&vulkan, ImageAtomicSignedMinMax<false>());
     RunCase(&vulkan, ImageAtomicSignedMinMax<true>());
     return 0;
@@ -48418,6 +48591,11 @@ int main(int argc, char **argv) {
     RunCase(&vulkan, Vop3MulLoU16CapturedAndSelectors());
     return 0;
   }
+  if (argc == 2 && std::strcmp(argv[1], "--clamp-nan-only") == 0) {
+    VulkanHarness vulkan;
+    RunCase(&vulkan, Vop3F32ClampFlushesNaN());
+    return 0;
+  }
   if (argc == 2 && std::strcmp(argv[1], "--mad-i16-only") == 0) {
     VulkanHarness vulkan;
     RunCase(&vulkan, Vop3MadI16CapturedSelectorsAndSaturation());
@@ -48514,6 +48692,7 @@ int main(int argc, char **argv) {
   }
   if (argc == 2 && std::strcmp(argv[1], "--compute-meta-clear-only") == 0) {
     CheckDynamicRenderingState();
+    CheckPs5GameExampleImageClearRuntimeShape();
     VulkanHarness vulkan;
     vulkan.CheckComputeMetaClearClassification();
     vulkan.CheckNativeIndirectDispatch();

tests/VirtualMemoryAllocationTests.cpp

@@ -8,6 +8,7 @@
 #include "kernel/pthread.h"
 #include "libs/errno.h"
 #include "loader/guestInstructionPatcher.h"
+#include "loader/jit.h"
 #include "loader/runtimeLinker.h"
 #include "loader/systemContent.h"
 #include "loader/x64InstructionEmulator.h"
@@ -4775,6 +4776,59 @@ void TestSmallFiberStacksAndMigration() {
 
 } // namespace
 
+#if defined(__linux__) || KYTY_PLATFORM == KYTY_PLATFORM_WINDOWS
+KYTY_MS_ABI uint8_t* TlsStubTestAddress() {
+	return reinterpret_cast<uint8_t*>(static_cast<uintptr_t>(0x123456789abcull));
+}
+
+void TestTlsStubsPreserveFlags() {
+	const char*    test      = "TlsStubsPreserveFlags";
+	const uint64_t stub_size = Loader::Jit::SafeCall::GetSize();
+	constexpr uint64_t CODE_SIZE = 0x1000;
+	constexpr uint8_t  REG       = 1;
+	const auto         mapping   = Libs::LibKernel::Memory::AllocateProgramMemory(
+	    0x0000000906000000ull, stub_size + CODE_SIZE, Common::VirtualMemory::Mode::ExecuteReadWrite,
+	    "tls_stub_test");
+	Check(test, mapping != 0, "failed to allocate TLS stub test code");
+	auto* safe_call = new (reinterpret_cast<void*>(mapping)) Loader::Jit::SafeCall;
+	safe_call->SetFunc(TlsStubTestAddress);
+	const auto stub_address = mapping + Loader::Jit::TlsRegStub::GetOffset(REG);
+	auto*      stub         = new (reinterpret_cast<void*>(stub_address)) Loader::Jit::TlsRegStub;
+	stub->SetFunc(mapping);
+	stub->SetOutputReg(REG);
+
+	using Fixture  = uint64_t(KYTY_SYSV_ABI*)(uint64_t*);
+	bool preserved = true;
+	for (const auto [input, status]: {std::pair {uint8_t {0x7f}, uint64_t {0x890}},
+	                                  std::pair {uint8_t {0xff}, uint64_t {0x055}}}) {
+		Xbyak::CodeGenerator code(CODE_SIZE, reinterpret_cast<void*>(mapping + stub_size));
+		code.push(code.rbx);
+		code.mov(code.rbx, code.rdi);
+		code.mov(code.rax, 0x5555555555555555ull);
+		code.mov(code.al, input);
+		code.add(code.al, 1);
+		code.mov(code.r11, stub_address);
+		code.call(code.r11);
+		code.mov(code.qword[code.rbx], code.rcx);
+		code.mov(code.qword[code.rbx + 8], code.rax);
+		code.pushf();
+		code.pop(code.rax);
+		code.pop(code.rbx);
+		code.ret();
+		Check(test, Xbyak::GetError() == 0, "failed to generate TLS stub fixture");
+		Check(test, Common::VirtualMemory::FlushInstructionCache(mapping, stub_size + CODE_SIZE),
+		      "failed to flush TLS stub test code");
+		std::array<uint64_t, 2> registers {};
+		const auto flags = reinterpret_cast<Fixture>(mapping + stub_size)(registers.data());
+		const auto rax   = (0x5555555555555555ull & ~0xffull) | static_cast<uint8_t>(input + 1u);
+		preserved &= (flags & 0x8d5u) == status && registers[0] == 0x123456789abcull &&
+		             registers[1] == rax;
+	}
+	Libs::LibKernel::Memory::FreeGuestMemory(mapping, stub_size + CODE_SIZE);
+	Check(test, preserved, "TLS stubs changed the caller's flags or registers");
+}
+#endif
+
 int main(int argc, char** argv) {
 	InitSubsystems();
 	if (argc == 2 && std::strcmp(argv[1], "--virtual-ranges-only") == 0) {
@@ -4919,6 +4973,9 @@ int main(int argc, char** argv) {
 	RunTest(TestMemoryPoolCommitDecommitQueryFlags);
 	RunTest(TestProgramMemoryAllocationAndProtection);
 	RunTest(TestModuleRelocationUsesWritableHostMapping);
+#if defined(__linux__) || KYTY_PLATFORM == KYTY_PLATFORM_WINDOWS
+	RunTest(TestTlsStubsPreserveFlags);
+#endif
 
 	if (g_failed_tests != 0) {
 		std::printf("VirtualMemoryAllocationTests: %d case(s) failed\n", g_failed_tests);

tests/shaderCfgTests.cpp

@@ -8119,6 +8119,24 @@ void TestDsReturningAtomic64Decoder() {
         "captured DS_CMPST_RTN_B64 lost its comparison/replacement pairs or aliases");
 }
 
+void TestDsCompareStore32Decoder() {
+  using namespace ShaderRecompiler::Decoder;
+  // CS04b812daaacd532f pc 0x3e8 (AC Valhalla, #281).
+  const uint32_t words[] = {0xd8c00000u, 0x04130504u};
+  Instruction decoded;
+  DecodeInstruction(words, 0u, decoded);
+  Check(decoded.opcode == Opcode::DS_CMPST_RTN_B32 && decoded.word_count == 2u &&
+            decoded.data_dwords == 1u && decoded.src_count == 3u && decoded.src0.reg == 4u &&
+            decoded.src1.reg == 5u && decoded.src2.reg == 19u && decoded.dst.reg == 4u &&
+            decoded.offset == 0u && !decoded.gds,
+        "captured DS_CMPST_RTN_B32 lost its comparison/replacement order or aliases");
+
+  const uint32_t gds_words[] = {words[0] | (1u << 17u), words[1]};
+  DecodeInstruction(gds_words, 0u, decoded);
+  Check(decoded.opcode == Opcode::UNSUPPORTED && decoded.word_count == 2u && decoded.gds,
+        "DS_CMPST_RTN_B32 must reject the unimplemented GDS variant");
+}
+
 void TestDsFloatMinMaxDecoder() {
   using namespace ShaderRecompiler::Decoder;
   for (bool max_value : {false, true}) {
@@ -10539,6 +10557,8 @@ void TestCapturedBufferAtomicsX2() {
     ValueOpcode ir_opcode;
   };
   const Case cases[] = {
+      {{0xe1480018u, 0x80000000u}, 0u, 234u, "BUFFER_ATOMIC_ADD_X2",
+       "BufferAtomicIAdd64", "OpAtomicIAdd", ValueOpcode::BufferAtomicIAdd64},
       {{0xe1680018u, 0x80000000u}, 0u, 241u, "BUFFER_ATOMIC_OR_X2",
        "BufferAtomicOr64", "OpAtomicOr", ValueOpcode::BufferAtomicOr64},
       {{0xe1402000u, 0x80000913u}, 9u, 229u, "BUFFER_ATOMIC_SWAP_X2",
@@ -14412,6 +14432,172 @@ void TestComputeImageFill() {
     Run(mutation);
 }
 
+void TestPatternFill() {
+  using namespace ShaderRecompiler::IR;
+  // Bounded DWORD pattern fill, including every instruction through S_ENDPGM.
+  const uint32_t bounded_shader[] = {
+      0xbfa00003u, 0xd7460002u, 0x04010c0au, 0x7da80408u,
+      0xbf88003fu, 0x7e000c09u, 0xbf070980u, 0x858a807eu,
+      0x7e005700u, 0x100000ffu, 0x4f800000u, 0x7e060f00u,
+      0xd5766a00u, 0x02020609u, 0x7d8a0280u, 0x4c020080u,
+      0x02000101u, 0xd56a0001u, 0x00020700u, 0x4c000303u,
+      0x4a020303u, 0x02000101u, 0xd56a0000u, 0x00020500u,
+      0xd5690001u, 0x00020009u, 0x4c060302u, 0x7d8c02f9u,
+      0x06068c02u, 0x7d860609u, 0x87ea6a0cu, 0x50000080u,
+      0xd5286a00u, 0x003200c1u, 0xd5010000u, 0x002a00c1u,
+      0xd5690000u, 0x00020009u, 0x4c000102u, 0x7d0a0080u,
+      0xbe88246au, 0xbf880015u, 0x7d0a0081u, 0xbe8a246au,
+      0xbf88000cu, 0x7d0a0082u, 0xbeea246au, 0xbf880003u,
+      0x7e000207u, 0xe0102000u, 0x80000002u, 0x8afe7e6au,
+      0xbf880003u, 0x7e000206u, 0xe0102000u, 0x80000002u,
+      0xbefe046au, 0x8afe7e0au, 0xbf880003u, 0x7e000205u,
+      0xe0102000u, 0x80000002u, 0xbefe040au, 0x8afe7e08u,
+      0xbf880003u, 0x7e000204u, 0xe0102000u, 0x80000002u,
+      0xbf810000u,
+  };
+  // Pattern fill whose bounds come from dispatch thread dimensions.
+  const uint32_t thread_dimension_shader[] = {
+      0xbfa00002u, 0x7e040c09u, 0x81ea0980u, 0xd7460000u,
+      0x04010c0au, 0x7e020205u, 0x7e045702u, 0x100404ffu,
+      0x4f7ffffeu, 0x7e040f02u, 0x7ed60502u, 0x936a6b6au,
+      0x9aea6b6au, 0x4a04046au, 0xd56a0002u, 0x00020102u,
+      0xd5690002u, 0x00020409u, 0x4c040500u, 0x7d860409u,
+      0x4e060409u, 0x02040702u, 0x7d860409u, 0x4e060409u,
+      0x02040702u, 0x7d840481u, 0x7d8404f9u, 0x06868882u,
+      0x7d8404f9u, 0x06868a83u, 0x02020204u, 0xd5010001u,
+      0x00200d01u, 0xd5010001u, 0x00280f01u, 0xe0102000u,
+      0x80000100u, 0xbf810000u,
+  };
+  // The 64-lane variant uses vector multiply-high for the remainder calculation.
+  const uint32_t thread_dimension_wave64_shader[] = {
+      0xbfa00002u, 0x7e060c09u, 0x4c0412f9u, 0x86860680u,
+      0xd7460000u, 0x04010c0au, 0x7e020205u, 0x7e065703u,
+      0x100606ffu, 0x4f7ffffeu, 0x7e060f03u, 0xd5690002u,
+      0x00020702u, 0xd56a0002u, 0x00020702u, 0x4a040503u,
+      0xd56a0002u, 0x00020102u, 0xd5690002u, 0x00020409u,
+      0x4c040500u, 0x7d860409u, 0x4c0612f9u, 0x86060602u,
+      0x02040702u, 0x7d860409u, 0x4c0612f9u, 0x86060602u,
+      0x02040702u, 0x7d040481u, 0x7d0404f9u, 0x06868882u,
+      0x7d0404f9u, 0x06868a83u, 0x02020204u, 0xd5010001u,
+      0x00200d01u, 0xd5010001u, 0x00280f01u, 0xe0102000u,
+      0x80000100u, 0xbf810000u,
+  };
+  // The 64-lane variant with scalar quotient setup.
+  const uint32_t thread_dimension_wave64_scalar_shader[] = {
+      0xbfa00002u, 0x7e040c09u, 0x81ea0980u, 0xd7460000u,
+      0x04010c0au, 0x7e020205u, 0x7e045702u, 0x100404ffu,
+      0x4f7ffffeu, 0x7e040f02u, 0x7ed60502u, 0x936a6b6au,
+      0x9aea6b6au, 0x816a6a6bu, 0xd56a0002u, 0x0002006au,
+      0xd5690002u, 0x00020409u, 0x4c040500u, 0x7d860409u,
+      0x4c0612f9u, 0x86060602u, 0x02040702u, 0x7d860409u,
+      0x4c0612f9u, 0x86060602u, 0x02040702u, 0x7d040481u,
+      0x7d0404f9u, 0x06868882u, 0x7d0404f9u, 0x06868a83u,
+      0x02020204u, 0xd5010001u, 0x00200d01u, 0xd5010001u,
+      0x00280f01u, 0xe0102000u, 0x80000100u, 0xbf810000u,
+  };
+  // Same layout, with vector arithmetic for the reciprocal correction.
+  auto thread_dimension_wave64_vector_shader = std::to_array(thread_dimension_wave64_scalar_shader);
+  thread_dimension_wave64_vector_shader[13] = 0x4a04046au;
+  thread_dimension_wave64_vector_shader[15] = 0x00020102u;
+  struct PatternCase {
+    std::span<const uint32_t> shader;
+    uint32_t wave_size;
+    bool bounded;
+    size_t literal_word;
+  };
+  const PatternCase cases[] = {
+      {bounded_shader, 64, true, 10},
+      {thread_dimension_shader, 32, false, 8},
+      {thread_dimension_wave64_shader, 64, false, 9},
+      {thread_dimension_wave64_scalar_shader, 64, false, 8},
+      {thread_dimension_wave64_vector_shader, 64, false, 8},
+  };
+  for (const auto &[shader, wave_size, bounded, literal_word] : cases) {
+    std::array<uint32_t, 10> userdata = {
+        0x00200000u, 4u << 16, 64u, 0x14204u,
+        0xff000000u, 0u, 0u, 0x08000000u, 64u, 1u,
+    };
+    ShaderComputeInputInfo compute{};
+    compute.threads_num[0] = 64;
+    compute.threads_num[1] = compute.threads_num[2] = 1;
+    compute.workgroup_register = 10;
+    compute.thread_ids_num = 1;
+    compute.group_id[0] = true;
+    compute.dispatch_thread_dimensions = !bounded;
+    compute.wave_size = wave_size;
+    auto options = MakeCompileOptions(ShaderType::Compute);
+    options.wave_size = compute.wave_size;
+    options.input_info.compute = &compute;
+    options.user_data = userdata;
+    // Recognition derives identity from code, independently of the caller's hash.
+    options.shader_hash = 0x12345678u;
+    const auto Build = [&](std::span<const uint32_t> code) {
+      auto translated = ShaderRecompiler::TranslateProgram(code, options);
+      return ExtractResourcePlan(translated.program);
+    };
+    auto plan = Build(shader);
+    ResourceSnapshot snapshot;
+    ResourceSpecialization specialization;
+    const auto Materialize = [&](bool expected) {
+      Check(MaterializeResources(plan, {.user_data = userdata}, snapshot, specialization),
+            "pattern fill did not materialize");
+      const auto &fill = snapshot.uniform_fill;
+      Check((fill.kind == UniformFillKind::Buffer) == expected,
+            "pattern fill used inactive words or retained a stale result");
+      if (expected)
+        Check(fill.value == userdata[4] && fill.words == 1 &&
+                  fill.bounded_elements == (bounded ? userdata[8] : 0u) &&
+                  fill.group_stride == std::array<uint32_t, 3>{64, 0, 0},
+              "pattern fill lost its value or coverage contract");
+    };
+    Materialize(true);
+    userdata[4] ^= 0x123456u;
+    Materialize(true);
+    userdata[4] ^= 0x123456u;
+    for (const uint32_t period : {0u, 2u, 3u, 4u, 5u}) {
+      userdata[9] = period;
+      Materialize(false);
+    }
+    for (uint32_t period = 2; period <= 4; ++period) {
+      userdata[9] = period;
+      userdata[3 + period] = userdata[4];
+      Materialize(true);
+      userdata[5] ^= 1u;
+      Materialize(false);
+      userdata[5] ^= 1u;
+    }
+    userdata[5] = userdata[6] = 0;
+    userdata[7] = 0x08000000u;
+    userdata[9] = 1;
+    Materialize(true);
+    userdata[8] = 0;
+    Materialize(!bounded);
+    userdata[8] = 63;
+    Materialize(true); // The renderer owns the descriptor-size equality check.
+    userdata[8] = 64;
+    Materialize(true);
+    std::vector<uint32_t> changed(shader.begin(), shader.end());
+    changed.push_back(0xdeadcafeu); // Non-executable metadata does not change identity.
+    Check(Build(changed).uniform_fill.fill.kind == UniformFillKind::Buffer,
+          "pattern recognition included bytes after S_ENDPGM");
+    changed[literal_word] ^= 0x00800000u;
+    Check(Build(changed).uniform_fill.fill.kind == UniformFillKind::None,
+          "pattern recognition accepted modified executable instructions");
+    for (const uint32_t invalid_input : {0u, 1u, 2u, 3u}) {
+      const auto saved_compute = compute;
+      const auto saved_wave = options.wave_size;
+      if (invalid_input == 0) compute.workgroup_register = 11;
+      if (invalid_input == 1) options.wave_size = wave_size == 64 ? 32 : 64;
+      if (invalid_input == 2) compute.dispatch_thread_dimensions = bounded;
+      if (invalid_input == 3) compute.float_mode ^= 1u;
+      Check(Build(shader).uniform_fill.fill.kind == UniformFillKind::None,
+            "pattern recognition accepted different shader launch inputs");
+      compute = saved_compute;
+      options.wave_size = saved_wave;
+    }
+  }
+}
+
 void TestGpuProducedWritableDescriptor() {
   using namespace ShaderRecompiler::IR;
   // Reduced from 994af59d50bedca8: the GPU-built BVH header supplies a relative
@@ -16312,6 +16498,7 @@ int main() {
   TestScalarAshrI64Decoder();
   TestImageAtomicWidthDecoder();
   TestDsReturningAtomic64Decoder();
+  TestDsCompareStore32Decoder();
   TestDsFloatMinMaxDecoder();
   TestNewShaderDecoderArchitecture();
   TestImageAddressOperands();
@@ -16428,6 +16615,7 @@ int main() {
   TestNewShaderRecompilerEarlyZDisabledWhenPixelKillEnabled();
   TestTypedDescriptorRealWideMoveTranslation();
   TestComputeImageFill();
+  TestPatternFill();
   TestGpuProducedWritableDescriptor();
   TestUniformSelectedWritableDescriptor();
   TestUniformSelectedDescriptorLoadAddress();