graphics: quiet two misleading render diagnostics (#416 )

* graphics: quiet two misleading render diagnostics The three PM4 register-offset normalization messages logged once per occurrence with no limit. They are printed in the colour reserved for problems, and normalizing the offset is routine, so a short session buries the log in them: PAC-MAN WORLD 2 Re-PAC (PPSA18189) emitted 24969 of them in about a minute, a fifth of a megabyte and 1.9% of every line written. Rate-limit all three the way the surrounding diagnostics in these files already are, which keeps the first occurrences without drowning what follows. The EQAA check warned whenever a control register was set, including on single-sample targets where those controls do nothing. That title never multisamples -- all 128 of its render targets are samples=1 -- yet it still produced the warning, pointing at a cause that could not apply. Return early when multisampling is off, matching how the neighbouring sample-location check already reads msaa_num_samples. Log output only; no rendering behaviour changes. * graphics: silence normalized offset diagnostics --------- Co-authored-by: nmzik <Nmzik@mail.ru>
shader: preserve raw VCC bits in S_MOV_B64 (#418 )
2026-10-20 02:47:02 +02:00 · 2026-08-31 03:33:32 +04:00 · 2026-08-31 03:04:16 +04:00 · 2026-08-31 01:24:55 +04:00 · 2026-08-30 23:55:05 +04:00 · 2026-08-30 23:49:08 +04:00
25 changed files with 1038 additions and 316 deletions
@@ -2433,12 +2433,12 @@ KYTY_CP_OP_PARSER(CpOpIndirectShRegs) {
 		auto value          = indirect_buffer[1];

 		// Not sure if this is correct
-		if (raw_cmd_offset != cmd_offset) {
-			LOGF_COLOR(Log::Color::Red,
-			           "\t temporary: normalized indirect SH register offset 0x%08" PRIx32
-			           " -> 0x%08" PRIx32 "\n",
-			           raw_cmd_offset, cmd_offset);
-		}
+		// if (raw_cmd_offset != cmd_offset) {
+		// 	LOGF_COLOR(Log::Color::Red,
+		// 	           "\t temporary: normalized indirect SH register offset 0x%08" PRIx32
+		// 	           " -> 0x%08" PRIx32 "\n",
+		// 	           raw_cmd_offset, cmd_offset);
+		// }

 		// Not sure if this is correct
 		if (cmd_offset == Pm4::SH_NOP || raw_cmd_offset == 0xffffffffu) {
@@ -2494,12 +2494,12 @@ KYTY_CP_OP_PARSER(CpOpIndirectUcRegs) {
 		auto value          = indirect_buffer[1];

 		// Not sure if this is correct
-		if (raw_cmd_offset != cmd_offset) {
-			LOGF_COLOR(Log::Color::Red,
-			           "\t temporary: normalized indirect UC register offset 0x%08" PRIx32
-			           " -> 0x%08" PRIx32 "\n",
-			           raw_cmd_offset, cmd_offset);
-		}
+		// if (raw_cmd_offset != cmd_offset) {
+		// 	LOGF_COLOR(Log::Color::Red,
+		// 	           "\t temporary: normalized indirect UC register offset 0x%08" PRIx32
+		// 	           " -> 0x%08" PRIx32 "\n",
+		// 	           raw_cmd_offset, cmd_offset);
+		// }
 		if (HwUcTrySetFakeRegister(cmd_offset, value)) {
 			continue;
 		}
@@ -2855,12 +2855,12 @@ KYTY_CP_OP_PARSER(CpOpSetUconfigReg) {
 		cmd_offset = raw_cmd_offset & 0x0fffffffu;
 	}

-	if (raw_cmd_offset != cmd_offset) {
-		LOGF_COLOR(Log::Color::Red,
-		           "\t temporary: normalized UC register offset 0x%08" PRIx32 " -> 0x%08" PRIx32
-		           "\n",
-		           raw_cmd_offset, cmd_offset);
-	}
+	// if (raw_cmd_offset != cmd_offset) {
+	// 	LOGF_COLOR(Log::Color::Red,
+	// 	           "\t temporary: normalized UC register offset 0x%08" PRIx32 " -> 0x%08" PRIx32
+	// 	           "\n",
+	// 	           raw_cmd_offset, cmd_offset);
+	// }
 	if (HwUcTrySetFakeRegisterRange(cmd_offset, buffer + 1, KYTY_PM4_LEN(cmd_id) - 2u)) {
 		return KYTY_PM4_LEN(cmd_id) - 1u;
 	}
@@ -827,7 +827,12 @@ static void EqaaPrint(const char* func, const HW::EqaaControl& c) {
 	     c.static_anchor_associations ? "true" : "false");
 }

-static void EqaaCheck(const HW::EqaaControl& c) {
+static void EqaaCheck(const HW::EqaaControl& c, const HW::AaConfig& cf) {
+	// The EQAA controls only take effect while multisampling is on, so a single-sample
+	// target leaves them inert and has nothing to warn about.
+	if (cf.msaa_num_samples == 0) {
+		return;
+	}
 	if (c.max_anchor_samples != 0 || c.ps_iter_samples != 0 || c.mask_export_num_samples != 0 ||
 	    c.alpha_to_mask_num_samples != 0 || c.high_quality_intersections ||
 	    c.incoherent_eqaa_reads || c.interpolate_comp_z || c.static_anchor_associations) {
@@ -1192,7 +1197,7 @@ void hw_check(const CommandBuffer& buffer) {
 	log_phase("blend");
 	BcCheck(bc, bclr, cc);
 	log_phase("eqaa");
-	EqaaCheck(eqaa);
+	EqaaCheck(eqaa, ac);
 	log_phase("aa");
 	AaCheck(aa, ac);
 	log_phase("done");
@@ -351,6 +351,7 @@ enum class Opcode {
 	V_CMP_GE_I32,
 	V_CMP_T_I32,
 	V_CMP_CLASS_F32,
+	V_CMPX_CLASS_F32,
 	V_CMP_LT_I16,
 	V_CMP_EQ_I16,
 	V_CMP_LE_I16,
@@ -631,6 +632,7 @@ struct Operand {
 	uint32_t dpp_ctrl           = 0;
 	uint32_t dpp_row_mask       = 0xf;
 	uint32_t dpp_bank_mask      = 0xf;
+	bool     explicit_sdwa_dst  = false;
 	bool     sdwa_sext          = false;
 	bool     dpp_fetch_inactive = false;
 	bool     dpp_bound_ctrl     = false;
@@ -13,6 +13,7 @@ enum class Vop2SdwaProfile {
 	None,
 	Float32,
 	Float16,
+	PackedFloat16,
 	IntegerFullDestination,
 	IntegerPartialDestination,
 	ReverseRightShift,
@@ -66,7 +67,7 @@ constexpr Vop2OpcodeInfo VOP2_OPCODE_LIST[] = {
    {0x2bu, Opcode::V_MAC_F32},
    {0x2cu, Opcode::V_MADMK_F32},
    {0x2du, Opcode::V_MADAK_F32},
-    {0x2fu, Opcode::V_CVT_PKRTZ_F16_F32},
+    {0x2fu, Opcode::V_CVT_PKRTZ_F16_F32, Vop2SdwaProfile::PackedFloat16},
    {0x32u, Opcode::V_ADD_F16, Vop2SdwaProfile::Float16},
    {0x33u, Opcode::V_SUB_F16, Vop2SdwaProfile::Float16},
    {0x34u, Opcode::V_SUBREV_F16, Vop2SdwaProfile::Float16},
@@ -227,7 +228,8 @@ constexpr VopcOpcodeInfo VOPC_OPCODE_LIST[] = {
    {0x8eu, Opcode::V_CMP_GE_I16},        {0x91u, Opcode::V_CMPX_LT_I32},
    {0x92u, Opcode::V_CMPX_EQ_I32},       {0x93u, Opcode::V_CMPX_LE_I32},
    {0x94u, Opcode::V_CMPX_GT_I32},       {0x95u, Opcode::V_CMPX_NE_I32},
-    {0x96u, Opcode::V_CMPX_GE_I32},       {0xa9u, Opcode::V_CMP_LT_U16},
+    {0x96u, Opcode::V_CMPX_GE_I32},       {0x98u, Opcode::V_CMPX_CLASS_F32},
+    {0xa9u, Opcode::V_CMP_LT_U16},
    {0xaau, Opcode::V_CMP_EQ_U16},        {0xabu, Opcode::V_CMP_LE_U16},
    {0xacu, Opcode::V_CMP_GT_U16},        {0xadu, Opcode::V_CMP_NE_U16},
    {0xaeu, Opcode::V_CMP_GE_U16},        {0xbcu, Opcode::V_CMPX_GT_U16},
@@ -416,14 +418,6 @@ bool IsPermlaneOpcode(Opcode opcode) {
 	return opcode == Opcode::V_PERMLANE16_B32 || opcode == Opcode::V_PERMLANEX16_B32;
 }

-bool IsVop2LowHalfF16Opcode(Opcode opcode) {
-	return opcode == Opcode::V_ADD_F16 || opcode == Opcode::V_SUB_F16 ||
-	       opcode == Opcode::V_SUBREV_F16 || opcode == Opcode::V_MUL_F16 ||
-	       opcode == Opcode::V_FMAC_F16 || opcode == Opcode::V_FMAMK_F16 ||
-	       opcode == Opcode::V_FMAAK_F16 || opcode == Opcode::V_MAX_F16 ||
-	       opcode == Opcode::V_MIN_F16;
-}
-
 bool IsNativeVop3F16TernaryOpcode(Opcode opcode) {
 	return opcode == Opcode::V_MIN3_F16 || opcode == Opcode::V_MAX3_F16 ||
 	       opcode == Opcode::V_MED3_F16 || opcode == Opcode::V_FMA_F16;
@@ -450,12 +444,6 @@ bool IsNativeVop3B16BinaryOpcode(Opcode opcode) {
 	}
 }

-void ApplyDefaultVop2F16Destination(Instruction& inst) {
-	if (IsVop2LowHalfF16Opcode(inst.opcode)) {
-		inst.dst.sdwa_sel = 4;
-	}
-}
-
 bool IsVop1FloatResultOpcode(Opcode opcode);
 bool IsVopcCompareExec(Opcode opcode);

@@ -470,6 +458,8 @@ bool IsVop1FloatSourceOpcode(Opcode opcode) {
 		case Opcode::V_FREXP_EXP_I32_F32:
 		case Opcode::V_FREXP_MANT_F32:
 		case Opcode::V_CVT_F16_F32:
+		case Opcode::V_CVT_U16_F16:
+		case Opcode::V_CVT_I16_F16:
 		case Opcode::V_RCP_F32:
 		case Opcode::V_RCP_IFLAG_F32:
 		case Opcode::V_FRACT_F32:
@@ -481,7 +471,11 @@ bool IsVop1FloatSourceOpcode(Opcode opcode) {
 		case Opcode::V_LOG_F32:
 		case Opcode::V_RSQ_F32:
 		case Opcode::V_SQRT_F32:
+		case Opcode::V_RCP_F16:
 		case Opcode::V_SQRT_F16:
+		case Opcode::V_RSQ_F16:
+		case Opcode::V_LOG_F16:
+		case Opcode::V_EXP_F16:
 		case Opcode::V_FLOOR_F16:
 		case Opcode::V_CEIL_F16:
 		case Opcode::V_TRUNC_F16:
@@ -527,33 +521,34 @@ constexpr Vop1SdwaRule VOP1_SDWA_RULES[] = {
    {Opcode::V_NOT_B32, SdwaSelBytes() | SdwaSelWords() | SdwaSelFull(), 0, 0, false},
    {Opcode::V_FFBL_B32, SdwaSelBytes() | SdwaSelWords() | SdwaSelFull(), 0, 0, false},
    {Opcode::V_CVT_F32_F16, SdwaSelWords() | SdwaSelFull(), 0, 0, true},
-    {Opcode::V_CVT_F16_F32, SdwaSelFull(), SdwaSelWords(), SdwaSelFull(), false},
+    {Opcode::V_CVT_F16_F32, SdwaSelBytes() | SdwaSelWords() | SdwaSelFull(), SdwaSelWords(),
+     SdwaSelBytes() | SdwaSelWords() | SdwaSelFull(), true},
    {Opcode::V_CVT_F16_U16, SdwaSelWords() | SdwaSelFull(), SdwaSelWords(),
     SdwaSelWords() | SdwaSelFull(), false},
    {Opcode::V_CVT_U16_F16, SdwaSelWords() | SdwaSelFull(), SdwaSelWords(),
-     SdwaSelWords() | SdwaSelFull(), false},
+     SdwaSelWords() | SdwaSelFull(), true},
    {Opcode::V_CVT_F16_I16, SdwaSelWords() | SdwaSelFull(), SdwaSelWords(),
     SdwaSelWords() | SdwaSelFull(), false},
    {Opcode::V_CVT_I16_F16, SdwaSelWords() | SdwaSelFull(), SdwaSelWords(),
-     SdwaSelWords() | SdwaSelFull(), false},
+     SdwaSelWords() | SdwaSelFull(), true},
    {Opcode::V_RCP_F16, SdwaSelWords() | SdwaSelFull(), SdwaSelWords(),
     SdwaSelWords() | SdwaSelFull(), true},
    {Opcode::V_SQRT_F16, SdwaSelWords() | SdwaSelFull(), SdwaSelWords(),
-     SdwaSelWords() | SdwaSelFull(), false},
+     SdwaSelWords() | SdwaSelFull(), true},
    {Opcode::V_RSQ_F16, SdwaSelWords() | SdwaSelFull(), SdwaSelWords(),
-     SdwaSelWords() | SdwaSelFull(), false},
+     SdwaSelWords() | SdwaSelFull(), true},
    {Opcode::V_LOG_F16, SdwaSelWords() | SdwaSelFull(), SdwaSelWords(),
     SdwaSelWords() | SdwaSelFull(), true},
    {Opcode::V_EXP_F16, SdwaSelWords() | SdwaSelFull(), SdwaSelWords(),
     SdwaSelWords() | SdwaSelFull(), true},
    {Opcode::V_FLOOR_F16, SdwaSelWords() | SdwaSelFull(), SdwaSelWords(),
-     SdwaSelWords() | SdwaSelFull(), false},
+     SdwaSelWords() | SdwaSelFull(), true},
    {Opcode::V_CEIL_F16, SdwaSelWords() | SdwaSelFull(), SdwaSelWords(),
-     SdwaSelWords() | SdwaSelFull(), false},
+     SdwaSelWords() | SdwaSelFull(), true},
    {Opcode::V_TRUNC_F16, SdwaSelWords() | SdwaSelFull(), SdwaSelWords(),
-     SdwaSelWords() | SdwaSelFull(), false},
+     SdwaSelWords() | SdwaSelFull(), true},
    {Opcode::V_RNDNE_F16, SdwaSelWords() | SdwaSelFull(), SdwaSelWords(),
-     SdwaSelWords() | SdwaSelFull(), false},
+     SdwaSelWords() | SdwaSelFull(), true},
    {Opcode::V_SIN_F16, SdwaSelWords() | SdwaSelFull(), SdwaSelWords(),
     SdwaSelWords() | SdwaSelFull(), true},
    {Opcode::V_COS_F16, SdwaSelWords() | SdwaSelFull(), SdwaSelWords(),
@@ -585,7 +580,7 @@ bool IsValidFullSdwaDestinationUnused(uint32_t dst_u) {

 bool IsVop1SdwaSourceSupported(Opcode opcode, uint32_t src_sel, bool src_neg, bool src_abs) {
 	if (src_sel == 6u) {
-		return true;
+		return IsVop1FloatSourceOpcode(opcode) || (!src_neg && !src_abs);
 	}
 	const auto* rule = FindVop1SdwaRule(opcode);
 	if (rule == nullptr || !HasSdwaSelector(rule->source_selectors, src_sel)) {
@@ -596,29 +591,24 @@ bool IsVop1SdwaSourceSupported(Opcode opcode, uint32_t src_sel, bool src_neg, bo

 bool IsVop1SdwaDestinationSupported(Opcode opcode, uint32_t dst_sel, uint32_t dst_u,
                                    uint32_t src_sel) {
-	if ((opcode == Opcode::V_CVT_U16_F16 || opcode == Opcode::V_CVT_I16_F16) && dst_sel == 6u) {
-		return false;
-	}
 	if (dst_sel == 6u) {
 		return IsValidFullSdwaDestinationUnused(dst_u);
 	}
 	const auto* rule = FindVop1SdwaRule(opcode);
-	if (rule == nullptr || dst_u != 2u) {
+	if (rule == nullptr || dst_u == 3u) {
 		return false;
 	}
 	return HasSdwaSelector(rule->partial_dst_selectors, dst_sel) &&
 	       HasSdwaSelector(rule->partial_dst_source_selectors, src_sel);
 }

-bool HasUnsupportedVop1SdwaSourceModifiers(Opcode opcode, bool src_neg, bool src_abs) {
-	switch (opcode) {
-		case Opcode::V_CVT_U16_F16:
-		case Opcode::V_CVT_I16_F16:
-		case Opcode::V_SQRT_F16:
-		case Opcode::V_RSQ_F16: return src_neg || src_abs;
-		case Opcode::V_LOG_F16: return src_neg;
-		default: return false;
-	}
+bool UsesInexactClampControl(Opcode opcode) {
+	return opcode == Opcode::V_CVT_U32_F32 || opcode == Opcode::V_CVT_I32_F32 ||
+	       opcode == Opcode::V_CVT_U16_F16 || opcode == Opcode::V_CVT_I16_F16;
+}
+
+bool SupportsVop1Clamp(Opcode opcode) {
+	return IsVop1FloatResultOpcode(opcode) || UsesInexactClampControl(opcode);
 }

 bool ValidateVop1Sdwa(Instruction& inst, uint32_t opcode, uint32_t modifier) {
@@ -634,7 +624,8 @@ bool ValidateVop1Sdwa(Instruction& inst, uint32_t opcode, uint32_t modifier) {
 		SetUnsupported(inst, Family::VOP1, opcode, "VOP1 SDWA selector is invalid");
 		return false;
 	}
-	if ((clamp != 0u || omod != 0u) && !IsVop1FloatResultOpcode(inst.opcode)) {
+	if ((clamp != 0u && !SupportsVop1Clamp(inst.opcode)) ||
+	    (omod != 0u && !IsVop1FloatResultOpcode(inst.opcode))) {
 		SetUnsupported(inst, Family::VOP1, opcode, "VOP1 SDWA output modifiers are not supported");
 		return false;
 	}
@@ -643,10 +634,6 @@ bool ValidateVop1Sdwa(Instruction& inst, uint32_t opcode, uint32_t modifier) {
 		               "VOP1 SDWA destination selector is not supported");
 		return false;
 	}
-	if (HasUnsupportedVop1SdwaSourceModifiers(inst.opcode, src0_neg != 0u, src0_abs != 0u)) {
-		SetUnsupported(inst, Family::VOP1, opcode, "VOP1 SDWA source modifiers are not supported");
-		return false;
-	}
 	if (!IsVop1SdwaSourceSupported(inst.opcode, src0_sel, src0_neg != 0u, src0_abs != 0u)) {
 		SetUnsupported(inst, Family::VOP1, opcode, "VOP1 SDWA source selector is not supported");
 		return false;
@@ -684,10 +671,11 @@ bool DecodeVop1Sdwa(uint32_t pc, std::span<const uint32_t> code, uint32_t word_i
 	if (!DecodeScalarSource(src0 + (s0 == 0u ? 256u : 0u), pc, inst.src0, error)) {
 		return false;
 	}
-	inst.dst.sdwa_sel        = dst_sel;
-	inst.dst.sdwa_dst_unused = dst_u;
-	inst.dst.clamp           = clamp != 0u;
-	inst.dst.omod            = omod;
+	inst.dst.sdwa_sel          = dst_sel;
+	inst.dst.sdwa_dst_unused   = dst_u;
+	inst.dst.explicit_sdwa_dst = true;
+	inst.dst.clamp             = clamp != 0u;
+	inst.dst.omod              = omod;
 	inst.src0.sdwa_sel       = src0_sel;
 	inst.src0.sdwa_sext      = src0_sext != 0u;
 	inst.src0.negate         = src0_neg != 0u;
@@ -779,6 +767,7 @@ bool IsVop2FloatSourceOpcode(Opcode opcode) {
 		case Opcode::V_MAC_F32:
 		case Opcode::V_MADMK_F32:
 		case Opcode::V_MADAK_F32:
+		case Opcode::V_CVT_PKRTZ_F16_F32:
 		case Opcode::V_ADD_F16:
 		case Opcode::V_SUB_F16:
 		case Opcode::V_SUBREV_F16:
@@ -803,6 +792,7 @@ bool IsVop2FloatResultOpcode(Opcode opcode) {
 		case Opcode::V_MAC_F32:
 		case Opcode::V_MADMK_F32:
 		case Opcode::V_MADAK_F32:
+		case Opcode::V_CVT_PKRTZ_F16_F32:
 		case Opcode::V_ADD_F16:
 		case Opcode::V_SUB_F16:
 		case Opcode::V_SUBREV_F16:
@@ -821,6 +811,9 @@ bool IsVop1FloatResultOpcode(Opcode opcode) {
 		case Opcode::V_CVT_F32_I32:
 		case Opcode::V_CVT_F32_U32:
 		case Opcode::V_CVT_F32_F16:
+		case Opcode::V_CVT_F16_F32:
+		case Opcode::V_CVT_F16_U16:
+		case Opcode::V_CVT_F16_I16:
 		case Opcode::V_CVT_OFF_F32_I4:
 		case Opcode::V_CVT_F32_UBYTE0:
 		case Opcode::V_CVT_F32_UBYTE1:
@@ -839,9 +832,14 @@ bool IsVop1FloatResultOpcode(Opcode opcode) {
 		case Opcode::V_RSQ_F32:
 		case Opcode::V_SQRT_F32:
 		case Opcode::V_RCP_F16:
+		case Opcode::V_SQRT_F16:
 		case Opcode::V_RSQ_F16:
 		case Opcode::V_LOG_F16:
 		case Opcode::V_EXP_F16:
+		case Opcode::V_FLOOR_F16:
+		case Opcode::V_CEIL_F16:
+		case Opcode::V_TRUNC_F16:
+		case Opcode::V_RNDNE_F16:
 		case Opcode::V_SIN_F16:
 		case Opcode::V_COS_F16:
 		case Opcode::V_SIN_F32:
@@ -895,7 +893,8 @@ bool IsVopcFloatCompareOpcode(Opcode opcode) {
 		case Opcode::V_CMPX_NGT_F16:
 		case Opcode::V_CMPX_NEQ_F16:
 		case Opcode::V_CMPX_NLT_F16:
-		case Opcode::V_CMP_CLASS_F32: return true;
+		case Opcode::V_CMP_CLASS_F32:
+		case Opcode::V_CMPX_CLASS_F32: return true;
 		default: return false;
 	}
 }
@@ -955,6 +954,7 @@ constexpr Vop2SdwaRule VOP2_SDWA_RULES[] = {
    {SdwaSelFull(), SdwaSelFull(), SdwaSelFull(), false, true},
    {SdwaSelWords() | SdwaSelFull(), SdwaSelWords() | SdwaSelFull(), SdwaSelWords() | SdwaSelFull(),
     true, true},
+    {SdwaSelAll(), SdwaSelAll(), SdwaSelAll(), true, true},
    {SdwaSelFull(), SdwaSelAll(), SdwaSelAll(), false, false},
    {SdwaSelAll(), SdwaSelAll(), SdwaSelAll(), true, false},
    {SdwaSelFull(), SdwaSelAll(), SdwaSelWords() | SdwaSelFull(), false, false},
@@ -979,7 +979,7 @@ bool IsVop2SdwaDestinationSupported(const Vop2SdwaRule& rule, const Vop2SdwaFiel
 	if (fields.dst_sel == 6u) {
 		return IsValidFullSdwaDestinationUnused(fields.dst_u);
 	}
-	return fields.dst_u == 2u && rule.partial_dst;
+	return fields.dst_u != 3u && rule.partial_dst;
 }

 bool IsFullWidthVop2Sdwa(const Vop2SdwaFields& fields) {
@@ -1057,6 +1057,13 @@ bool FinalizeVop2Instruction(uint32_t pc, std::span<const uint32_t> code, uint32
 			inst.src2.kind = OperandKind::VccLo;
 			inst.src_count = 3;
 			break;
+		case Opcode::V_PK_FMAC_F16:
+			// VOP2 has no OP_SEL fields, so the high operation implicitly reads high halves.
+			inst.src0.op_sel_hi = true;
+			inst.src1.op_sel_hi = true;
+			inst.dst.op_sel_hi  = true;
+			inst.src_count      = 2;
+			break;
 		default: inst.src_count = 2; break;
 	}
 	ReadLiteralOperands(code, word_index, inst);
@@ -1078,10 +1085,11 @@ bool DecodeVop2Sdwa(uint32_t pc, std::span<const uint32_t> code, uint32_t word_i
 	    !DecodeScalarSource(vsrc1 + (fields.s1 == 0u ? 256u : 0u), pc, inst.src1, error)) {
 		return false;
 	}
-	inst.dst.sdwa_sel        = fields.dst_sel;
-	inst.dst.sdwa_dst_unused = fields.dst_u;
-	inst.dst.clamp           = fields.clamp != 0u;
-	inst.dst.omod            = fields.omod;
+	inst.dst.sdwa_sel          = fields.dst_sel;
+	inst.dst.sdwa_dst_unused   = fields.dst_u;
+	inst.dst.explicit_sdwa_dst = true;
+	inst.dst.clamp             = fields.clamp != 0u;
+	inst.dst.omod              = fields.omod;
 	inst.src0.sdwa_sel       = fields.src0_sel;
 	inst.src0.sdwa_sext      = fields.src0_sext != 0u;
 	inst.src0.negate         = fields.src0_neg != 0u;
@@ -1105,12 +1113,16 @@ bool DecodeVop2Dpp(uint32_t pc, std::span<const uint32_t> code, uint32_t word_in
 	if (!DecodeScalarSource(src0 + 256u, pc, inst.src0, error)) {
 		return false;
 	}
-	ApplyDefaultVop2F16Destination(inst);
 	ApplyDppModifier(inst.src0, modifier);
 	inst.src1.negate   = ((modifier >> 22u) & 0x1u) != 0u;
 	inst.src1.absolute = ((modifier >> 23u) & 0x1u) != 0u;
+	const bool packed_fmac = inst.opcode == Opcode::V_PK_FMAC_F16;
+	if (packed_fmac) {
+		inst.src0.negate_hi = inst.src0.negate;
+		inst.src1.negate_hi = inst.src1.negate;
+	}

-	if (!IsVop2FloatSourceOpcode(inst.opcode) &&
+	if (!IsVop2FloatSourceOpcode(inst.opcode) && !packed_fmac &&
 	    (inst.src0.negate || inst.src0.absolute || inst.src1.negate || inst.src1.absolute)) {
 		SetUnsupported(inst, Family::VOP2, opcode,
 		               "VOP2 DPP integer source modifiers are not supported");
@@ -1437,6 +1449,7 @@ bool SupportsNativeVop3ResultModifiers(Opcode opcode) {
 		case Opcode::V_MAD_F32:
 		case Opcode::V_FMA_F32:
 		case Opcode::V_FMA_F16:
+		case Opcode::V_CVT_PKRTZ_F16_F32:
 		case Opcode::V_LDEXP_F32:
 		case Opcode::V_MIN3_F32:
 		case Opcode::V_MAX3_F32:
@@ -1445,12 +1458,17 @@ bool SupportsNativeVop3ResultModifiers(Opcode opcode) {
 	}
 }

+bool SupportsNativeVop3Clamp(Opcode opcode) {
+	return SupportsNativeVop3ResultModifiers(opcode) || UsesInexactClampControl(opcode);
+}
+
 bool HasUnsupportedNativeVop3Modifiers(Opcode opcode, bool permlane, bool mad_mix,
                                       bool carry_in_out, bool scalar_dst, uint32_t abs,
                                       uint32_t op_sel, uint32_t clamp, uint32_t omod,
                                       uint32_t neg) {
 	const bool source_modifiers = SupportsNativeVop3SourceModifiers(opcode);
 	const bool result_modifiers = SupportsNativeVop3ResultModifiers(opcode);
+	const bool clamp_modifier   = SupportsNativeVop3Clamp(opcode);

 	if (permlane) {
 		return abs != 0u || (op_sel & ~0x3u) != 0u || clamp != 0u || omod != 0u || neg != 0u;
@@ -1482,7 +1500,7 @@ bool HasUnsupportedNativeVop3Modifiers(Opcode opcode, bool permlane, bool mad_mi
 	}
 	if (source_modifiers) {
 		return op_sel != 0u || (omod != 0u && !result_modifiers) ||
-		       (clamp != 0u && !result_modifiers);
+		       (clamp != 0u && !clamp_modifier);
 	}
 	if (result_modifiers) {
 		return abs != 0u || op_sel != 0u || neg != 0u;
@@ -1518,6 +1536,7 @@ bool IsVopcCompareExec(Opcode opcode) {
 		case Opcode::V_CMPX_GT_I32:
 		case Opcode::V_CMPX_NE_I32:
 		case Opcode::V_CMPX_GE_I32:
+		case Opcode::V_CMPX_CLASS_F32:
 		case Opcode::V_CMPX_LT_U32:
 		case Opcode::V_CMPX_EQ_U32:
 		case Opcode::V_CMPX_LE_U32:
@@ -1574,7 +1593,6 @@ bool DecodeVop2(uint32_t pc, std::span<const uint32_t> code, uint32_t word_index
 	if (!DecodeScalarSource(src0, pc, inst.src0, error)) {
 		return false;
 	}
-	ApplyDefaultVop2F16Destination(inst);
 	return FinalizeVop2Instruction(pc, code, word_index, inst, error);
 }

@@ -1698,6 +1716,7 @@ bool DecodeVop3(uint32_t pc, std::span<const uint32_t> code, uint32_t word_index
 	const bool scalar_modifier_limits  = UsesScalarDestination(inst.opcode);
 	const bool native_source_modifiers = SupportsNativeVop3SourceModifiers(inst.opcode);
 	const bool native_result_modifiers = SupportsNativeVop3ResultModifiers(inst.opcode);
+	const bool native_clamp_modifier   = SupportsNativeVop3Clamp(inst.opcode);
 	const bool modifiers =
 	    HasUnsupportedNativeVop3Modifiers(inst.opcode, permlane, mad_mix, vop3b_uses_sdst,
 	                                      scalar_modifier_limits, abs, op_sel, clamp, omod, neg);
@@ -1732,7 +1751,7 @@ bool DecodeVop3(uint32_t pc, std::span<const uint32_t> code, uint32_t word_index
 	if (!dst_ok || !DecodeScalarSource(src0, pc, inst.src0, error)) {
 		return false;
 	}
-	inst.dst.clamp = native_result_modifiers && clamp != 0u;
+	inst.dst.clamp = native_clamp_modifier && clamp != 0u;
 	inst.dst.omod  = native_result_modifiers ? omod : 0u;
 	if (permlane) {
 		inst.dst.op_sel    = (op_sel & 0x1u) != 0u;
@@ -12,6 +12,7 @@ void Translator::EmitCompareResult(const Decoder::Instruction& inst, IR::U1 valu
 	if (cmpx) {
 		const auto mask = BallotMask(masked);
 		ir.SetExec(masked);
+		ir.SetExecMaskTag(IR::U1(IR::Value(true)));
 		ir.SetExecLo(mask[0]);
 		ir.SetExecHi(mask[1]);
 		return;
@@ -55,11 +56,11 @@ void Translator::EmitFloatOrderedCompare(const Decoder::Instruction& inst, bool
 	EmitCompareResult(inst, ordered ? ir.LogicalNot(unordered) : unordered, false, false);
 }

-void Translator::V_CMP_CLASS_F32(const Decoder::Instruction& inst) {
+void Translator::EmitFloatClassCompare(const Decoder::Instruction& inst, bool cmpx) {
 	const auto value = ReadOperand(inst.src0, IR::Type::F32);
 	const auto mask  = ReadOperand(inst.src1, IR::Type::U32);
 	EmitCompareResult(inst, IR::U1(ir.Emit(IR::ValueOpcode::FPCmpClass32, {value, mask})), false,
-	                  false);
+	                  cmpx);
 }

 } // namespace Libs::Graphics::ShaderRecompiler::Frontend
@@ -23,18 +23,28 @@ Decoder::Operand ConditionOperand(Decoder::OperandKind kind) {

 void Translator::S_SAVEEXEC(const Decoder::Instruction& inst, IR::ValueOpcode operation,
                            bool negate_exec, bool negate_source, bool write_64) {
+	if (write_64) {
+		const auto exec_operand  = ConditionOperand(Decoder::OperandKind::ExecLo);
+		const auto old           = ReadScalarU64(exec_operand);
+		const auto bit_operation = operation == IR::ValueOpcode::LogicalAnd
+		                               ? IR::ValueOpcode::BitwiseAnd32
+		                               : IR::ValueOpcode::BitwiseOr32;
+		const auto result = BinaryScalarU64(old, ReadScalarU64(inst.src0), operation, bit_operation,
+		                                    negate_exec, negate_source, false);
+		WriteScalarU64(inst.dst, old);
+		WriteScalarU64(exec_operand, result);
+		ir.SetScc(ScalarU64NonZero(result));
+		return;
+	}
 	const auto old    = ir.GetExec();
 	const auto src    = ReadMask(inst.src0);
 	const auto lhs    = negate_exec ? ir.LogicalNot(old) : old;
 	const auto rhs    = negate_source ? ir.LogicalNot(src) : src;
 	const auto result = IR::U1(ir.Emit(operation, {lhs, rhs}));
-	if (write_64) {
-		WriteMask64(inst.dst, old);
-	} else {
-		WriteMask(inst.dst, old);
-	}
+	WriteMask(inst.dst, old);
 	const auto mask = BallotMask(result);
 	ir.SetExec(result);
+	ir.SetExecMaskTag(IR::U1(IR::Value(true)));
 	ir.SetExecLo(mask[0]);
 	ir.SetExecHi(mask[1]);
 	ir.SetScc(result);
@@ -196,30 +206,21 @@ void Translator::S_CSELECT_B32(const Decoder::Instruction& inst) {
 }

 void Translator::S_CSELECT_B64(const Decoder::Instruction& inst) {
-	const auto condition     = ir.GetScc();
-	const auto lhs           = ReadU32Pair(inst.src0);
-	const auto rhs           = ReadU32Pair(inst.src1);
-	const auto selected_mask = IR::U1(
-	    ir.Emit(IR::ValueOpcode::SelectU1, {condition, ReadMask(inst.src0), ReadMask(inst.src1)}));
-	const auto selected_mask_valid =
-	    IR::U1(ir.Emit(IR::ValueOpcode::SelectU1,
-	                   {condition, ReadMaskValid(inst.src0), ReadMaskValid(inst.src1)}));
-	if (IsExecOrVcc(inst.dst)) {
-		WriteMask64(inst.dst, selected_mask);
-		return;
-	}
-	WriteU32Pair(inst.dst,
-	             {ir.Select(condition, lhs[0], rhs[0]), ir.Select(condition, lhs[1], rhs[1])});
-	if (inst.dst.kind == Decoder::OperandKind::Sgpr) {
-		const auto dst = static_cast<IR::ScalarReg>(inst.dst.reg);
-		ir.SetThreadBitScalarReg(dst, selected_mask);
-		ir.SetScalarMaskTag(dst, selected_mask_valid);
-	}
+	const auto condition = ir.GetScc();
+	const auto lhs       = ReadScalarU64(inst.src0);
+	const auto rhs       = ReadScalarU64(inst.src1);
+	WriteScalarU64(inst.dst, SelectScalarU64(condition, lhs, rhs));
 }

 void Translator::MOV_B32(const Decoder::Instruction& inst, bool apply_float_modifiers) {
+	if (inst.src0.kind == Decoder::OperandKind::Sgpr && inst.dst.kind == Decoder::OperandKind::Sgpr &&
+	    inst.src0.reg == inst.dst.reg) {
+		return;
+	}
 	if (IsExecOrVcc(inst.src0) && IsExecOrVcc(inst.dst)) {
-		WriteMask(inst.dst, ReadMask(inst.src0));
+		if (inst.src0.kind != inst.dst.kind) {
+			WriteRawU32(inst.dst, ReadRawU32(inst.src0));
+		}
 	} else if (apply_float_modifiers && (inst.src0.negate || inst.src0.absolute)) {
 		WriteOperand(DestinationOperand(inst), ReadOperand(inst.src0, IR::Type::F32));
 	} else {
@@ -228,50 +229,17 @@ void Translator::MOV_B32(const Decoder::Instruction& inst, bool apply_float_modi
 }

 void Translator::S_MOV_B64(const Decoder::Instruction& inst) {
-	if (IsExecOrVcc(inst.dst) || IsExecOrVcc(inst.src0)) {
-		WriteMask64(inst.dst, ReadMask(inst.src0));
-		return;
-	}
-	const bool scalar_copy =
-	    inst.dst.kind == Decoder::OperandKind::Sgpr && inst.src0.kind == Decoder::OperandKind::Sgpr;
-	IR::U1 source_mask;
-	IR::U1 source_mask_valid;
-	if (scalar_copy) {
-		source_mask       = ir.GetThreadBitScalarReg(static_cast<IR::ScalarReg>(inst.src0.reg));
-		source_mask_valid = ir.GetScalarMaskTag(static_cast<IR::ScalarReg>(inst.src0.reg));
-	}
-	WriteU32Pair(inst.dst, ReadU32Pair(inst.src0));
-	if (scalar_copy) {
-		ir.SetThreadBitScalarReg(static_cast<IR::ScalarReg>(inst.dst.reg), source_mask);
-		ir.SetScalarMaskTag(static_cast<IR::ScalarReg>(inst.dst.reg), source_mask_valid);
-	}
+	WriteScalarU64(inst.dst, ReadScalarU64(inst.src0));
 }

 void Translator::S_WQM_B64(const Decoder::Instruction& inst) {
-	if (!IsExecOrVcc(inst.dst) && !IsExecOrVcc(inst.src0)) {
-		const auto mask_valid = ReadMaskValid(inst.src0);
-		const auto invocation_result =
-		    IR::U1(ir.Emit(IR::ValueOpcode::WqmMask, {ReadMask(inst.src0)}));
-		const auto result =
-		    IR::U64(ir.Emit(IR::ValueOpcode::WqmU64, {ReadOperand(inst.src0, IR::Type::U64)}));
-		WriteOperand(DestinationOperand(inst), result);
-		if (inst.dst.kind == Decoder::OperandKind::Sgpr) {
-			const auto dst = static_cast<IR::ScalarReg>(inst.dst.reg);
-			ir.SetThreadBitScalarReg(dst, invocation_result);
-			ir.SetScalarMaskTag(dst, mask_valid);
-			const auto raw_nonzero =
-			    IR::U1(ir.Emit(IR::ValueOpcode::INotEqual64, {result, IR::Value(uint64_t {0})}));
-			ir.SetScc(IR::U1(
-			    ir.Emit(IR::ValueOpcode::SelectU1, {mask_valid, invocation_result, raw_nonzero})));
-		} else {
-			ir.SetScc(
-			    IR::U1(ir.Emit(IR::ValueOpcode::INotEqual64, {result, IR::Value(uint64_t {0})})));
-		}
-		return;
-	}
-	const auto result = IR::U1(ir.Emit(IR::ValueOpcode::WqmMask, {ReadMask(inst.src0)}));
-	WriteMask64(inst.dst, result);
-	ir.SetScc(result);
+	const auto source = ReadScalarU64(inst.src0);
+	const auto raw    = ExtractU64(
+	    IR::U64(ir.Emit(IR::ValueOpcode::WqmU64, {ir.ConstructU64(source.raw[0], source.raw[1])})));
+	const auto invocation = IR::U1(ir.Emit(IR::ValueOpcode::WqmMask, {source.invocation}));
+	const auto result     = MakeScalarU64Result(raw, invocation, source.mask_valid);
+	WriteScalarU64(inst.dst, result);
+	ir.SetScc(ScalarU64NonZero(result));
 }

 bool Translator::V_MOVRELS_B32(const Decoder::Instruction& inst, std::string* error) {
@@ -141,8 +141,10 @@ void Translator::V_CVT_OFF_F32_I4(const Decoder::Instruction& inst) {
 }

 void Translator::V_CVT_PKRTZ_F16_F32(const Decoder::Instruction& inst) {
-	const auto lhs = ReadOperand(SourceAt(inst, 0), IR::Type::F32);
-	const auto rhs = ReadOperand(SourceAt(inst, 1), IR::Type::F32);
+	const auto lhs = ApplyF32ResultModifiers(
+	    inst.dst, IR::F32(ReadOperand(SourceAt(inst, 0), IR::Type::F32)));
+	const auto rhs = ApplyF32ResultModifiers(
+	    inst.dst, IR::F32(ReadOperand(SourceAt(inst, 1), IR::Type::F32)));
 	WriteOperand(DestinationOperand(inst), ir.Emit(IR::ValueOpcode::PackFloat2x16Rtz, {lhs, rhs}));
 }

@@ -77,68 +77,15 @@ bool Translator::PackedInteger16MinMax(const Decoder::Instruction& inst, IR::Val
 	return true;
 }

-IR::U1 Translator::U64MaskBinary(const Decoder::Instruction& inst, IR::ValueOpcode opcode,
-                                 bool negate_rhs, bool negate_result) {
-	const auto lhs = ReadMask(inst.src0);
-	auto       rhs = ReadMask(inst.src1);
-	if (negate_rhs) {
-		rhs = ir.LogicalNot(rhs);
-	}
-	auto result = IR::U1(ir.Emit(opcode, {lhs, rhs}));
-	return negate_result ? ir.LogicalNot(result) : result;
-}
-
 bool Translator::S_U64_MASK(const Decoder::Instruction& inst, IR::ValueOpcode logical_opcode,
                            IR::ValueOpcode bit_opcode, bool negate_rhs, bool negate_result,
                            bool unary) {
-	const auto invocation_result =
-	    unary ? ir.LogicalNot(ReadMask(inst.src0))
-	          : U64MaskBinary(inst, logical_opcode, negate_rhs, negate_result);
-	const auto is_exec_or_vcc = [](const Decoder::Operand& operand) {
-		switch (operand.kind) {
-			case Decoder::OperandKind::ExecLo:
-			case Decoder::OperandKind::ExecHi:
-			case Decoder::OperandKind::VccLo:
-			case Decoder::OperandKind::VccHi: return true;
-			default: return false;
-		}
-	};
-	if (is_exec_or_vcc(inst.dst) || is_exec_or_vcc(inst.src0) ||
-	    (inst.src_count > 1u && is_exec_or_vcc(inst.src1))) {
-		WriteMask64(inst.dst, invocation_result);
-		ir.SetScc(invocation_result);
-		return true;
-	}
-
-	const auto lhs        = ReadU32Pair(inst.src0);
-	auto       mask_valid = ReadMaskValid(inst.src0);
-	if (inst.src_count > 1u) {
-		mask_valid = ir.LogicalAnd(mask_valid, ReadMaskValid(inst.src1));
-	}
-	std::array<IR::U32, 2> result;
-	if (unary) {
-		result = {ir.BitwiseNot(lhs[0]), ir.BitwiseNot(lhs[1])};
-	} else {
-		const auto rhs = ReadU32Pair(inst.src1);
-		for (uint32_t component = 0; component < 2u; component++) {
-			auto value = ir.Emit(
-			    bit_opcode, {lhs[component], negate_rhs ? IR::Value(ir.BitwiseNot(rhs[component]))
-			                                            : IR::Value(rhs[component])});
-			result[component] = negate_result ? ir.BitwiseNot(IR::U32(value)) : IR::U32(value);
-		}
-	}
-	WriteU32Pair(inst.dst, result);
-	if (inst.dst.kind == Decoder::OperandKind::Sgpr) {
-		const auto dst = static_cast<IR::ScalarReg>(inst.dst.reg);
-		ir.SetThreadBitScalarReg(dst, invocation_result);
-		ir.SetScalarMaskTag(dst, mask_valid);
-		const auto raw_nonzero =
-		    ir.INotEqual(ir.BitwiseOr(result[0], result[1]), IR::U32(IR::Value(0u)));
-		ir.SetScc(IR::U1(
-		    ir.Emit(IR::ValueOpcode::SelectU1, {mask_valid, invocation_result, raw_nonzero})));
-	} else {
-		ir.SetScc(ir.INotEqual(ir.BitwiseOr(result[0], result[1]), IR::U32(IR::Value(0u))));
-	}
+	const auto lhs    = ReadScalarU64(inst.src0);
+	const auto result = unary ? NotScalarU64(lhs)
+	                          : BinaryScalarU64(lhs, ReadScalarU64(inst.src1), logical_opcode,
+	                                            bit_opcode, false, negate_rhs, negate_result);
+	WriteScalarU64(inst.dst, result);
+	ir.SetScc(ScalarU64NonZero(result));
 	return true;
 }

@@ -37,6 +37,7 @@ Decoder::Operand OffsetDecodedRegister(const Decoder::Operand& operand, uint32_t
 	result.dpp_ctrl           = 0;
 	result.dpp_row_mask       = 0xf;
 	result.dpp_bank_mask      = 0xf;
+	result.explicit_sdwa_dst  = false;
 	result.dpp_fetch_inactive = false;
 	result.dpp_bound_ctrl     = false;
 	result.dpp                = false;
@@ -56,11 +57,14 @@ uint32_t ResourceIndexFromOperand(const Decoder::Operand& operand) {
 	}
 }

-uint32_t RawScalarLoadBase(const Decoder::Operand& operand) {
-	if (operand.kind == Decoder::OperandKind::Sgpr) {
-		return operand.reg;
+Decoder::Operand RawScalarLoadBase(const Decoder::Operand& operand) {
+	if (operand.kind != Decoder::OperandKind::M0) {
+		return operand;
 	}
-	return operand.kind == Decoder::OperandKind::VccLo ? 106u : 0u;
+	auto result = operand;
+	// The compressed SBASE value 62 decodes as code 124, but means NULL for S_LOAD.
+	result.kind = Decoder::OperandKind::Null;
+	return result;
 }

 ResourceKind FlatSegmentResourceKind(uint32_t segment) {
@@ -320,8 +324,9 @@ Translator::AddressOperands Translator::ReadAddressOperands(const Decoder::Instr
 	return {GetAddressResource(low, high), low, high};
 }

-IR::Value Translator::GetScalarAddressResource(uint32_t base) {
-	return GetAddressResource(ReadScalarCode(base), ReadScalarCode(base + 1u));
+IR::Value Translator::GetScalarAddressResource(const Decoder::Operand& base) {
+	const auto raw = ReadU32Pair(base);
+	return GetAddressResource(raw[0], raw[1]);
 }

 IR::Value Translator::GetImageResource(const IR::MemoryInfo& memory) {
@@ -104,6 +104,7 @@ Decoder::Operand Translator::PlainOperand(const Decoder::Operand& operand) {
 	result.dpp_ctrl           = 0;
 	result.dpp_row_mask       = 0xf;
 	result.dpp_bank_mask      = 0xf;
+	result.explicit_sdwa_dst  = false;
 	result.dpp_fetch_inactive = false;
 	result.dpp_bound_ctrl     = false;
 	result.dpp                = false;
@@ -152,11 +153,8 @@ IR::U32 Translator::ReadScalarCode(uint32_t code) {
 		case 106u: return ir.GetVccLo();
 		case 107u: return ir.GetVccHi();
 		case 124u: return ir.GetM0();
-		case 126u:
-		case 127u: {
-			const auto mask = BallotMask(ir.GetExec());
-			return mask[code - 126u];
-		}
+		case 126u: return ir.GetExecLo();
+		case 127u: return ir.GetExecHi();
 		default: return IR::U32(IR::Value(0u));
 	}
 }
@@ -316,19 +314,23 @@ void Translator::WriteRawU32(const Decoder::Operand& operand, IR::U32 value) {
 		case Decoder::OperandKind::VccLo:
 			ir.SetVccLo(IR::U32(value));
 			ir.SetVcc(ThreadBit(IR::U32(value)));
+			ir.SetVccMaskValidTag(IR::U1(IR::Value(false)));
 			break;
 		case Decoder::OperandKind::VccHi:
 			ir.SetVccHi(IR::U32(value));
 			ir.SetVcc(ThreadBit(ir.GetVccLo()));
+			ir.SetVccMaskValidTag(IR::U1(IR::Value(false)));
 			break;
 		case Decoder::OperandKind::M0: ir.SetM0(IR::U32(value)); break;
 		case Decoder::OperandKind::ExecLo:
 			ir.SetExecLo(IR::U32(value));
 			ir.SetExec(ThreadBit(IR::U32(value)));
+			ir.SetExecMaskTag(IR::U1(IR::Value(false)));
 			break;
 		case Decoder::OperandKind::ExecHi:
 			ir.SetExecHi(IR::U32(value));
 			ir.SetExec(ThreadBit(ir.GetExecLo()));
+			ir.SetExecMaskTag(IR::U1(IR::Value(false)));
 			break;
 		case Decoder::OperandKind::Scc:
 			ir.SetScc(ir.INotEqual(value, IR::U32(IR::Value(0u))));
@@ -373,6 +375,7 @@ void Translator::WriteOperand(const Decoder::Operand& operand, IR::Value value)
 			case Decoder::OperandKind::ExecHi: {
 				const auto mask = BallotMask(IR::U1(value));
 				ir.SetExec(IR::U1(value));
+				ir.SetExecMaskTag(IR::U1(IR::Value(true)));
 				ir.SetExecLo(mask[0]);
 				ir.SetExecHi(mask[1]);
 				return;
@@ -381,6 +384,7 @@ void Translator::WriteOperand(const Decoder::Operand& operand, IR::Value value)
 			case Decoder::OperandKind::VccHi: {
 				const auto mask = BallotMask(IR::U1(value));
 				ir.SetVcc(IR::U1(value));
+				ir.SetVccMaskValidTag(IR::U1(IR::Value(true)));
 				ir.SetVccLo(mask[0]);
 				ir.SetVccHi(mask[1]);
 				return;
@@ -392,12 +396,13 @@ void Translator::WriteOperand(const Decoder::Operand& operand, IR::Value value)
 		}
 	}
 	if (type == IR::Type::U16) {
-		WriteRawU32(operand, IR::U32(ir.Emit(IR::ValueOpcode::ConvertU32U16, {IR::U16(value)})));
+		Write16Bits(operand,
+		            IR::U32(ir.Emit(IR::ValueOpcode::ConvertU32U16, {IR::U16(value)})));
 		return;
 	}
 	if (type == IR::Type::F16) {
 		const auto bits = IR::U16(ir.Emit(IR::ValueOpcode::BitCastU16F16, {value}));
-		WriteRawU32(operand, IR::U32(ir.Emit(IR::ValueOpcode::ConvertU32U16, {bits})));
+		Write16Bits(operand, IR::U32(ir.Emit(IR::ValueOpcode::ConvertU32U16, {bits})));
 		return;
 	}
 	if (type == IR::Type::U64) {
@@ -417,38 +422,32 @@ IR::U32 Translator::PackHalf2x16(IR::F32 low, IR::F32 high) {
 	return IR::U32(ir.Emit(IR::ValueOpcode::PackHalf2x16, {pair}));
 }

+void Translator::Write16Bits(const Decoder::Operand& operand, IR::U32 value) {
+	auto destination = operand;
+	// Native GFX10 16-bit results preserve the unselected half. Plain/DPP destinations use the
+	// low half; native VOP3 may already select either half. Explicit SDWA retains encoded DST_U.
+	if (!destination.explicit_sdwa_dst) {
+		if (destination.sdwa_sel == 6u) {
+			destination.sdwa_sel = 4u;
+		}
+		destination.sdwa_dst_unused = 2u;
+	}
+	destination.omod   = 0u;
+	destination.op_sel = false;
+	destination.clamp  = false;
+	WriteRawU32(destination, ir.BitwiseAnd(value, IR::U32(IR::Value(0xffffu))));
+}
+
 void Translator::WriteF16(const Decoder::Operand& operand, IR::F32 value) {
 	value           = ApplyF32ResultModifiers(operand, value);
 	const auto half = IR::F16(ir.Emit(IR::ValueOpcode::ConvertF16F32, {value}));
 	const auto bits = IR::U32(ir.Emit(IR::ValueOpcode::ConvertU32U16,
 	                                  {IR::U16(ir.Emit(IR::ValueOpcode::BitCastU16F16, {half}))}));
-	auto       merged = bits;
-	if (operand.sdwa_sel == 4u || operand.sdwa_sel == 5u) {
-		merged = IR::U32(ir.Emit(IR::ValueOpcode::BitFieldInsert,
-		                         {ReadRawU32(PlainOperand(operand)), bits,
-		                          IR::Value(operand.sdwa_sel == 5u ? 16u : 0u), IR::Value(16u)}));
-	}
-	auto raw     = operand;
-	raw.sdwa_sel = 6u;
-	raw.omod     = 0u;
-	raw.op_sel   = false;
-	raw.clamp    = false;
-	WriteRawU32(raw, merged);
+	Write16Bits(operand, bits);
 }

 void Translator::WriteU16(const Decoder::Operand& operand, IR::U32 value) {
-	auto merged = ir.BitwiseAnd(value, IR::U32(IR::Value(0xffffu)));
-	if (operand.sdwa_sel == 4u || operand.sdwa_sel == 5u) {
-		merged = IR::U32(ir.Emit(IR::ValueOpcode::BitFieldInsert,
-		                         {ReadRawU32(PlainOperand(operand)), merged,
-		                          IR::Value(operand.sdwa_sel == 5u ? 16u : 0u), IR::Value(16u)}));
-	}
-	auto raw     = operand;
-	raw.sdwa_sel = 6u;
-	raw.omod     = 0u;
-	raw.op_sel   = false;
-	raw.clamp    = false;
-	WriteRawU32(raw, merged);
+	Write16Bits(operand, value);
 }

 IR::U32 Translator::ReadU32(const Decoder::Operand& operand) {
@@ -587,11 +586,13 @@ void Translator::WriteU32Pair(const Decoder::Operand&       operand,
 	switch (operand.kind) {
 		case Decoder::OperandKind::ExecLo:
 			ir.SetExec(thread_bit);
+			ir.SetExecMaskTag(IR::U1(IR::Value(false)));
 			ir.SetExecLo(value[0]);
 			ir.SetExecHi(value[1]);
 			return;
 		case Decoder::OperandKind::VccLo:
 			ir.SetVcc(thread_bit);
+			ir.SetVccMaskValidTag(IR::U1(IR::Value(false)));
 			ir.SetVccLo(value[0]);
 			ir.SetVccHi(value[1]);
 			return;
@@ -602,6 +603,87 @@ void Translator::WriteU32Pair(const Decoder::Operand&       operand,
 	WriteRawU32(OffsetOperand(operand, 1), value[1]);
 }

+Translator::ScalarU64 Translator::ReadScalarU64(const Decoder::Operand& operand) {
+	return {ReadU32Pair(operand), ReadMask(operand), ReadMaskValid(operand)};
+}
+
+Translator::ScalarU64 Translator::MakeScalarU64Mask(IR::U1 value) {
+	return {BallotMask(value), value, IR::U1(IR::Value(true))};
+}
+
+Translator::ScalarU64 Translator::MakeScalarU64Result(const std::array<IR::U32, 2>& raw,
+                                                      IR::U1 invocation, IR::U1 mask_valid) {
+	return {raw,
+	        IR::U1(ir.Emit(IR::ValueOpcode::SelectU1, {mask_valid, invocation, ThreadBit(raw[0])})),
+	        mask_valid};
+}
+
+Translator::ScalarU64 Translator::SelectScalarU64(IR::U1 condition, const ScalarU64& lhs,
+                                                  const ScalarU64& rhs) {
+	return {
+	    {ir.Select(condition, lhs.raw[0], rhs.raw[0]),
+	     ir.Select(condition, lhs.raw[1], rhs.raw[1])},
+	    IR::U1(ir.Emit(IR::ValueOpcode::SelectU1, {condition, lhs.invocation, rhs.invocation})),
+	    IR::U1(ir.Emit(IR::ValueOpcode::SelectU1, {condition, lhs.mask_valid, rhs.mask_valid}))};
+}
+
+Translator::ScalarU64 Translator::NotScalarU64(const ScalarU64& value) {
+	return MakeScalarU64Result({ir.BitwiseNot(value.raw[0]), ir.BitwiseNot(value.raw[1])},
+	                           ir.LogicalNot(value.invocation), value.mask_valid);
+}
+
+Translator::ScalarU64 Translator::BinaryScalarU64(const ScalarU64& lhs, const ScalarU64& rhs,
+                                                  IR::ValueOpcode logical_opcode,
+                                                  IR::ValueOpcode bit_opcode, bool negate_lhs,
+                                                  bool negate_rhs, bool negate_result) {
+	auto lhs_mask = negate_lhs ? ir.LogicalNot(lhs.invocation) : lhs.invocation;
+	auto rhs_mask = negate_rhs ? ir.LogicalNot(rhs.invocation) : rhs.invocation;
+	auto mask     = IR::U1(ir.Emit(logical_opcode, {lhs_mask, rhs_mask}));
+	if (negate_result) {
+		mask = ir.LogicalNot(mask);
+	}
+	std::array<IR::U32, 2> raw;
+	for (uint32_t component = 0; component < raw.size(); component++) {
+		auto lhs_raw   = negate_lhs ? ir.BitwiseNot(lhs.raw[component]) : lhs.raw[component];
+		auto rhs_raw   = negate_rhs ? ir.BitwiseNot(rhs.raw[component]) : rhs.raw[component];
+		raw[component] = IR::U32(ir.Emit(bit_opcode, {lhs_raw, rhs_raw}));
+		if (negate_result) {
+			raw[component] = ir.BitwiseNot(raw[component]);
+		}
+	}
+	return MakeScalarU64Result(raw, mask, ir.LogicalAnd(lhs.mask_valid, rhs.mask_valid));
+}
+
+void Translator::WriteScalarU64(const Decoder::Operand& operand, const ScalarU64& value) {
+	WriteU32Pair(operand, value.raw);
+	switch (operand.kind) {
+		case Decoder::OperandKind::ExecLo:
+		case Decoder::OperandKind::ExecHi:
+			ir.SetExec(value.invocation);
+			ir.SetExecMaskTag(value.mask_valid);
+			break;
+		case Decoder::OperandKind::VccLo:
+		case Decoder::OperandKind::VccHi:
+			ir.SetVcc(value.invocation);
+			ir.SetVccMaskValidTag(value.mask_valid);
+			break;
+		case Decoder::OperandKind::Sgpr: {
+			const auto reg = static_cast<IR::ScalarReg>(operand.reg);
+			ir.SetThreadBitScalarReg(reg, value.invocation);
+			ir.SetScalarMaskTag(reg, value.mask_valid);
+			break;
+		}
+		default: break;
+	}
+}
+
+IR::U1 Translator::ScalarU64NonZero(const ScalarU64& value) {
+	const auto raw_nonzero =
+	    ir.INotEqual(ir.BitwiseOr(value.raw[0], value.raw[1]), IR::U32(IR::Value(0u)));
+	return IR::U1(
+	    ir.Emit(IR::ValueOpcode::SelectU1, {value.mask_valid, value.invocation, raw_nonzero}));
+}
+
 IR::U1 Translator::ThreadBit(IR::U32 low) {
 	return ir.INotEqual(low, IR::U32(IR::Value(0u)));
 }
@@ -657,9 +739,9 @@ IR::U1 Translator::ReadMaskValid(const Decoder::Operand& operand) {
 		case Decoder::OperandKind::Sgpr:
 			return ir.GetScalarMaskTag(static_cast<IR::ScalarReg>(operand.reg));
 		case Decoder::OperandKind::ExecLo:
-		case Decoder::OperandKind::ExecHi:
+		case Decoder::OperandKind::ExecHi: return ir.GetExecMaskTag();
 		case Decoder::OperandKind::VccLo:
-		case Decoder::OperandKind::VccHi:
+		case Decoder::OperandKind::VccHi: return ir.GetVccMaskValidTag();
 		case Decoder::OperandKind::VccZ:
 		case Decoder::OperandKind::ExecZ:
 		case Decoder::OperandKind::Scc: return IR::U1(IR::Value(true));
@@ -692,6 +774,7 @@ void Translator::WriteMask(const Decoder::Operand& operand, IR::U1 value) {
 		case Decoder::OperandKind::ExecHi: {
 			const auto mask = BallotMask(value);
 			ir.SetExec(value);
+			ir.SetExecMaskTag(IR::U1(IR::Value(true)));
 			ir.SetExecLo(mask[0]);
 			ir.SetExecHi(mask[1]);
 			return;
@@ -700,6 +783,7 @@ void Translator::WriteMask(const Decoder::Operand& operand, IR::U1 value) {
 		case Decoder::OperandKind::VccHi: {
 			const auto mask = BallotMask(value);
 			ir.SetVcc(value);
+			ir.SetVccMaskValidTag(IR::U1(IR::Value(true)));
 			ir.SetVccLo(mask[0]);
 			ir.SetVccHi(mask[1]);
 			return;
@@ -713,25 +797,7 @@ void Translator::WriteMask(const Decoder::Operand& operand, IR::U1 value) {
 }

 void Translator::WriteMask64(const Decoder::Operand& operand, IR::U1 value) {
-	if (operand.kind != Decoder::OperandKind::Sgpr) {
-		WriteMask(operand, value);
-		return;
-	}
-	const auto reg  = static_cast<IR::ScalarReg>(operand.reg);
-	const auto mask = BallotMask(value);
-	ir.SetThreadBitScalarReg(reg, value);
-	ir.SetScalarMaskTag(reg, IR::U1(IR::Value(true)));
-	if (IR::RegIndex(reg) > 0u) {
-		ir.SetScalarMaskTag(static_cast<IR::ScalarReg>(IR::RegIndex(reg) - 1u),
-		                    IR::U1(IR::Value(false)));
-	}
-	ir.SetScalarReg(reg, mask[0]);
-	if (IR::RegIndex(reg) + 1u < IR::NumScalarRegs) {
-		const auto high = static_cast<IR::ScalarReg>(IR::RegIndex(reg) + 1u);
-		ir.SetScalarReg(high, mask[1]);
-		ir.SetThreadBitScalarReg(high, IR::U1(IR::Value(false)));
-		ir.SetScalarMaskTag(high, IR::U1(IR::Value(false)));
-	}
+	WriteScalarU64(operand, MakeScalarU64Mask(value));
 }

 void Translator::WriteCompareResult(const Decoder::Operand& operand, IR::U1 value) {
@@ -1041,6 +1107,7 @@ bool TranslateProgram(const Decoder::Program& decoded, const CFG::Graph& cfg,
 			entry_ir.SetScalarMaskTag(reg, IR::U1(IR::Value(false)));
 		}
 		entry_ir.SetExec(IR::U1(IR::Value(true)));
+		entry_ir.SetExecMaskTag(IR::U1(IR::Value(true)));
 		entry_ir.SetExecLo(IR::U32(IR::Value(1u)));
 		entry_ir.SetExecHi(IR::U32(IR::Value(0u)));
 		if (options.stage == ShaderType::Compute) {
@@ -19,6 +19,14 @@ public:
 	bool AddBranchCondition(const CFG::BasicBlock& source, IR::BlockInfo& info, std::string* error);

 private:
+	struct ScalarU64 {
+		std::array<IR::U32, 2> raw;
+		// Raw words and Kyty's per-invocation mask model remain independent.
+		// mask_valid selects invocation for logical consumers and SCC.
+		IR::U1 invocation;
+		IR::U1 mask_valid;
+	};
+
 	Decoder::Operand SourceAt(const Decoder::Instruction& inst, uint32_t index);
 	Decoder::Operand DestinationOperand(const Decoder::Instruction& inst);
 	Decoder::Operand OffsetOperand(const Decoder::Operand& operand, uint32_t offset);
@@ -34,6 +42,7 @@ private:
 	IR::F32                ApplyF32ResultModifiers(const Decoder::Operand& operand, IR::F32 value);
 	void                   WriteOperand(const Decoder::Operand& operand, IR::Value value);
 	IR::U32                PackHalf2x16(IR::F32 low, IR::F32 high);
+	void                   Write16Bits(const Decoder::Operand& operand, IR::U32 value);
 	void                   WriteF16(const Decoder::Operand& operand, IR::F32 value);
 	void                   WriteU16(const Decoder::Operand& operand, IR::U32 value);
 	IR::U32                ReadU32(const Decoder::Operand& operand);
@@ -47,14 +56,25 @@ private:
 	IR::U32 ReadU16AsU32(const Decoder::Operand& operand, bool sign_extend);
 	IR::U32 ReadF16LaneBits(const Decoder::Operand& operand, bool high_lane);
 	std::array<IR::U32, 2> ExtractU64(IR::U64 value);
-	void    WriteU32Pair(const Decoder::Operand& operand, const std::array<IR::U32, 2>& value);
-	IR::U1  ReadCondition(const Decoder::Operand& operand);
-	IR::U32 ConditionBit(const Decoder::Operand& operand);
-	IR::U1  ReadMask(const Decoder::Operand& operand);
-	IR::U1  ReadMaskValid(const Decoder::Operand& operand);
-	void    WriteMask(const Decoder::Operand& operand, IR::U1 value);
-	void    WriteMask64(const Decoder::Operand& operand, IR::U1 value);
-	void    WriteCompareResult(const Decoder::Operand& operand, IR::U1 value);
+	void      WriteU32Pair(const Decoder::Operand& operand, const std::array<IR::U32, 2>& value);
+	ScalarU64 ReadScalarU64(const Decoder::Operand& operand);
+	ScalarU64 MakeScalarU64Mask(IR::U1 value);
+	ScalarU64 MakeScalarU64Result(const std::array<IR::U32, 2>& raw, IR::U1 invocation,
+	                              IR::U1 mask_valid);
+	ScalarU64 SelectScalarU64(IR::U1 condition, const ScalarU64& lhs, const ScalarU64& rhs);
+	ScalarU64 NotScalarU64(const ScalarU64& value);
+	ScalarU64 BinaryScalarU64(const ScalarU64& lhs, const ScalarU64& rhs,
+	                          IR::ValueOpcode logical_opcode, IR::ValueOpcode bit_opcode,
+	                          bool negate_lhs, bool negate_rhs, bool negate_result);
+	void      WriteScalarU64(const Decoder::Operand& operand, const ScalarU64& value);
+	IR::U1    ScalarU64NonZero(const ScalarU64& value);
+	IR::U1    ReadCondition(const Decoder::Operand& operand);
+	IR::U32   ConditionBit(const Decoder::Operand& operand);
+	IR::U1    ReadMask(const Decoder::Operand& operand);
+	IR::U1    ReadMaskValid(const Decoder::Operand& operand);
+	void      WriteMask(const Decoder::Operand& operand, IR::U1 value);
+	void      WriteMask64(const Decoder::Operand& operand, IR::U1 value);
+	void      WriteCompareResult(const Decoder::Operand& operand, IR::U1 value);

 	IR::MemoryFlags AddMemoryInfo(const IR::MemoryInfo& memory, uint32_t pc);
 	IR::ExportFlags AddExportInfo(const Decoder::Instruction& inst);
@@ -72,7 +92,7 @@ private:
 	IR::U32         GetResourceDword(uint32_t index, uint32_t dword);
 	IR::Value       GetBufferResource(const IR::MemoryInfo& memory);
 	IR::Value       GetAddressResource(IR::Value low, IR::Value high);
-	IR::Value       GetScalarAddressResource(uint32_t base);
+	IR::Value       GetScalarAddressResource(const Decoder::Operand& base);
 	IR::Value       GetImageResource(const IR::MemoryInfo& memory);
 	IR::Value       GetSamplerResource(const IR::MemoryInfo& memory);
 	IR::Value     MakeImageAddress(const Decoder::Instruction& inst, const Decoder::Operand& base);
@@ -127,7 +147,7 @@ private:
 	void EmitFloatCompare(const Decoder::Instruction& inst, IR::ValueOpcode opcode, bool half,
 	                      bool cmpx);
 	void EmitFloatOrderedCompare(const Decoder::Instruction& inst, bool ordered);
-	void V_CMP_CLASS_F32(const Decoder::Instruction& inst);
+	void EmitFloatClassCompare(const Decoder::Instruction& inst, bool cmpx);
 	void V_CVT_F32_UBYTE(const Decoder::Instruction& inst, uint32_t byte_index);
 	void V_CVT_F32_U32(const Decoder::Instruction& inst);
 	void V_CVT_F32_I32(const Decoder::Instruction& inst);
@@ -174,8 +194,6 @@ private:
 	bool PackedInteger16MinMax(const Decoder::Instruction& inst, IR::ValueOpcode opcode, bool sign);
 	bool S_U64_MASK(const Decoder::Instruction& inst, IR::ValueOpcode logical_opcode,
 	                IR::ValueOpcode bit_opcode, bool negate_rhs, bool negate_result, bool unary);
-	IR::U1  U64MaskBinary(const Decoder::Instruction& inst, IR::ValueOpcode opcode, bool negate_rhs,
-	                      bool negate_result);
 	bool    SimpleInteger(const Decoder::Instruction& inst, IR::ValueOpcode opcode, IR::Type type,
 	                      bool reverse, bool mask_shift_count, bool update_scc);
 	bool    ComposedIntegerBinary(const Decoder::Instruction& inst, IR::ValueOpcode opcode,
@@ -271,7 +271,8 @@ bool Translator::EmitVector(const Decoder::Instruction& inst, std::string* error
 			return true;
 		case O::V_CMP_O_F32: EmitFloatOrderedCompare(inst, true); return true;
 		case O::V_CMP_U_F32: EmitFloatOrderedCompare(inst, false); return true;
-		case O::V_CMP_CLASS_F32: V_CMP_CLASS_F32(inst); return true;
+		case O::V_CMP_CLASS_F32: EmitFloatClassCompare(inst, false); return true;
+		case O::V_CMPX_CLASS_F32: EmitFloatClassCompare(inst, true); return true;

 		case O::V_CVT_F32_UBYTE0: V_CVT_F32_UBYTE(inst, 0); return true;
 		case O::V_CVT_F32_UBYTE1: V_CVT_F32_UBYTE(inst, 1); return true;
@@ -76,6 +76,14 @@ void IREmitter::SetExec(U1 value) {
 	Emit(ValueOpcode::SetExec, {value});
 }

+U1 IREmitter::GetExecMaskTag() {
+	return U1(Emit(ValueOpcode::GetExecMaskTag));
+}
+
+void IREmitter::SetExecMaskTag(U1 value) {
+	Emit(ValueOpcode::SetExecMaskTag, {value});
+}
+
 U32 IREmitter::GetExecLo() {
 	return U32(Emit(ValueOpcode::GetExecLo));
 }
@@ -100,6 +108,14 @@ void IREmitter::SetVcc(U1 value) {
 	Emit(ValueOpcode::SetVcc, {value});
 }

+U1 IREmitter::GetVccMaskValidTag() {
+	return U1(Emit(ValueOpcode::GetVccMaskValidTag));
+}
+
+void IREmitter::SetVccMaskValidTag(U1 value) {
+	Emit(ValueOpcode::SetVccMaskValidTag, {value});
+}
+
 U32 IREmitter::GetVccLo() {
 	return U32(Emit(ValueOpcode::GetVccLo));
 }
@@ -39,12 +39,16 @@ public:
 	void SetScc(U1 value);
 	U1   GetExec();
 	void SetExec(U1 value);
+	U1   GetExecMaskTag();
+	void SetExecMaskTag(U1 value);
 	U32  GetExecLo();
 	void SetExecLo(U32 value);
 	U32  GetExecHi();
 	void SetExecHi(U32 value);
 	U1   GetVcc();
 	void SetVcc(U1 value);
+	U1   GetVccMaskValidTag();
+	void SetVccMaskValidTag(U1 value);
 	U32  GetVccLo();
 	void SetVccLo(U32 value);
 	U32  GetVccHi();
@@ -31,12 +31,16 @@ bool IsRegisterStatePseudo(ValueOpcode opcode) {
 		case ValueOpcode::SetScc:
 		case ValueOpcode::GetExec:
 		case ValueOpcode::SetExec:
+		case ValueOpcode::GetExecMaskTag:
+		case ValueOpcode::SetExecMaskTag:
 		case ValueOpcode::GetExecLo:
 		case ValueOpcode::SetExecLo:
 		case ValueOpcode::GetExecHi:
 		case ValueOpcode::SetExecHi:
 		case ValueOpcode::GetVcc:
 		case ValueOpcode::SetVcc:
+		case ValueOpcode::GetVccMaskValidTag:
+		case ValueOpcode::SetVccMaskValidTag:
 		case ValueOpcode::GetVccLo:
 		case ValueOpcode::SetVccLo:
 		case ValueOpcode::GetVccHi:
@@ -23,12 +23,16 @@ VALUE_OPCODE(GetScc, U1)
 VALUE_OPCODE(SetScc, Void, U1)
 VALUE_OPCODE(GetExec, U1)
 VALUE_OPCODE(SetExec, Void, U1)
+VALUE_OPCODE(GetExecMaskTag, U1)
+VALUE_OPCODE(SetExecMaskTag, Void, U1)
 VALUE_OPCODE(GetExecLo, U32)
 VALUE_OPCODE(SetExecLo, Void, U32)
 VALUE_OPCODE(GetExecHi, U32)
 VALUE_OPCODE(SetExecHi, Void, U32)
 VALUE_OPCODE(GetVcc, U1)
 VALUE_OPCODE(SetVcc, Void, U1)
+VALUE_OPCODE(GetVccMaskValidTag, U1)
+VALUE_OPCODE(SetVccMaskValidTag, Void, U1)
 VALUE_OPCODE(GetVccLo, U32)
 VALUE_OPCODE(SetVccLo, Void, U32)
 VALUE_OPCODE(GetVccHi, U32)
@@ -22,6 +22,9 @@ struct ScalarMaskTag {
 struct ExecTag {
 	auto operator<=>(const ExecTag&) const = default;
 };
+struct ExecMaskTag {
+	auto operator<=>(const ExecMaskTag&) const = default;
+};
 struct ExecLoTag {
 	auto operator<=>(const ExecLoTag&) const = default;
 };
@@ -31,6 +34,9 @@ struct ExecHiTag {
 struct VccTag {
 	auto operator<=>(const VccTag&) const = default;
 };
+struct VccMaskValidTag {
+	auto operator<=>(const VccMaskValidTag&) const = default;
+};
 struct VccLoTag {
 	auto operator<=>(const VccLoTag&) const = default;
 };
@@ -45,9 +51,9 @@ struct GotoVariable {
 	auto     operator<=>(const GotoVariable&) const = default;
 };

-using Variable =
-    std::variant<ScalarReg, ThreadBitScalarReg, ScalarMaskTag, VectorReg, GotoVariable, SccTag,
-                 ExecTag, ExecLoTag, ExecHiTag, VccTag, VccLoTag, VccHiTag, M0Tag>;
+using Variable = std::variant<ScalarReg, ThreadBitScalarReg, ScalarMaskTag, VectorReg, GotoVariable,
+                              SccTag, ExecTag, ExecMaskTag, ExecLoTag, ExecHiTag, VccTag,
+                              VccMaskValidTag, VccLoTag, VccHiTag, M0Tag>;
 using ValueMap = std::unordered_map<Block*, Value>;

 struct DefTable {
@@ -99,12 +105,16 @@ struct DefTable {
 	void         Set(Block* block, SccTag, Value value) { scc[block] = value; }
 	const Value& Get(Block* block, ExecTag) { return exec[block]; }
 	void         Set(Block* block, ExecTag, Value value) { exec[block] = value; }
+	const Value& Get(Block* block, ExecMaskTag) { return exec_mask_tag[block]; }
+	void         Set(Block* block, ExecMaskTag, Value value) { exec_mask_tag[block] = value; }
 	const Value& Get(Block* block, ExecLoTag) { return exec_lo[block]; }
 	void         Set(Block* block, ExecLoTag, Value value) { exec_lo[block] = value; }
 	const Value& Get(Block* block, ExecHiTag) { return exec_hi[block]; }
 	void         Set(Block* block, ExecHiTag, Value value) { exec_hi[block] = value; }
 	const Value& Get(Block* block, VccTag) { return vcc[block]; }
 	void         Set(Block* block, VccTag, Value value) { vcc[block] = value; }
+	const Value& Get(Block* block, VccMaskValidTag) { return vcc_mask_valid_tag[block]; }
+	void Set(Block* block, VccMaskValidTag, Value value) { vcc_mask_valid_tag[block] = value; }
 	const Value& Get(Block* block, VccLoTag) { return vcc_lo[block]; }
 	void         Set(Block* block, VccLoTag, Value value) { vcc_lo[block] = value; }
 	const Value& Get(Block* block, VccHiTag) { return vcc_hi[block]; }
@@ -114,9 +124,11 @@ struct DefTable {

 	ValueMap                               scc;
 	ValueMap                               exec;
+	ValueMap                               exec_mask_tag;
 	ValueMap                               exec_lo;
 	ValueMap                               exec_hi;
 	ValueMap                               vcc;
+	ValueMap                               vcc_mask_valid_tag;
 	ValueMap                               vcc_lo;
 	ValueMap                               vcc_hi;
 	ValueMap                               m0;
@@ -144,6 +156,9 @@ ValueOpcode UndefOpcode(SccTag) {
 ValueOpcode UndefOpcode(ExecTag) {
 	return ValueOpcode::UndefU1;
 }
+ValueOpcode UndefOpcode(ExecMaskTag) {
+	return ValueOpcode::UndefU1;
+}
 ValueOpcode UndefOpcode(ExecLoTag) {
 	return ValueOpcode::UndefU32;
 }
@@ -153,6 +168,9 @@ ValueOpcode UndefOpcode(ExecHiTag) {
 ValueOpcode UndefOpcode(VccTag) {
 	return ValueOpcode::UndefU1;
 }
+ValueOpcode UndefOpcode(VccMaskValidTag) {
+	return ValueOpcode::UndefU1;
+}
 ValueOpcode UndefOpcode(VccLoTag) {
 	return ValueOpcode::UndefU32;
 }
@@ -184,6 +202,9 @@ Value InitialValue(SccTag) {
 Value InitialValue(ExecTag) {
 	return Value(false);
 }
+Value InitialValue(ExecMaskTag) {
+	return Value(false);
+}
 Value InitialValue(ExecLoTag) {
 	return Value(0u);
 }
@@ -193,6 +214,9 @@ Value InitialValue(ExecHiTag) {
 Value InitialValue(VccTag) {
 	return Value(false);
 }
+Value InitialValue(VccMaskValidTag) {
+	return Value(false);
+}
 Value InitialValue(VccLoTag) {
 	return Value(0u);
 }
@@ -351,9 +375,15 @@ void VisitInstruction(Pass& pass, Block* block, Inst& inst) {
 			break;
 		case ValueOpcode::SetScc: pass.Write(SccTag {}, block, inst.Arg(0)); break;
 		case ValueOpcode::SetExec: pass.Write(ExecTag {}, block, inst.Arg(0)); break;
+		case ValueOpcode::SetExecMaskTag:
+			pass.Write(ExecMaskTag {}, block, inst.Arg(0));
+			break;
 		case ValueOpcode::SetExecLo: pass.Write(ExecLoTag {}, block, inst.Arg(0)); break;
 		case ValueOpcode::SetExecHi: pass.Write(ExecHiTag {}, block, inst.Arg(0)); break;
 		case ValueOpcode::SetVcc: pass.Write(VccTag {}, block, inst.Arg(0)); break;
+		case ValueOpcode::SetVccMaskValidTag:
+			pass.Write(VccMaskValidTag {}, block, inst.Arg(0));
+			break;
 		case ValueOpcode::SetVccLo: pass.Write(VccLoTag {}, block, inst.Arg(0)); break;
 		case ValueOpcode::SetVccHi: pass.Write(VccHiTag {}, block, inst.Arg(0)); break;
 		case ValueOpcode::SetM0: pass.Write(M0Tag {}, block, inst.Arg(0)); break;
@@ -375,9 +405,15 @@ void VisitInstruction(Pass& pass, Block* block, Inst& inst) {
 			break;
 		case ValueOpcode::GetScc: inst.ReplaceUsesWith(pass.Read(SccTag {}, block)); break;
 		case ValueOpcode::GetExec: inst.ReplaceUsesWith(pass.Read(ExecTag {}, block)); break;
+		case ValueOpcode::GetExecMaskTag:
+			inst.ReplaceUsesWith(pass.Read(ExecMaskTag {}, block));
+			break;
 		case ValueOpcode::GetExecLo: inst.ReplaceUsesWith(pass.Read(ExecLoTag {}, block)); break;
 		case ValueOpcode::GetExecHi: inst.ReplaceUsesWith(pass.Read(ExecHiTag {}, block)); break;
 		case ValueOpcode::GetVcc: inst.ReplaceUsesWith(pass.Read(VccTag {}, block)); break;
+		case ValueOpcode::GetVccMaskValidTag:
+			inst.ReplaceUsesWith(pass.Read(VccMaskValidTag {}, block));
+			break;
 		case ValueOpcode::GetVccLo: inst.ReplaceUsesWith(pass.Read(VccLoTag {}, block)); break;
 		case ValueOpcode::GetVccHi: inst.ReplaceUsesWith(pass.Read(VccHiTag {}, block)); break;
 		case ValueOpcode::GetM0: inst.ReplaceUsesWith(pass.Read(M0Tag {}, block)); break;
@@ -3804,8 +3804,8 @@ int KYTY_SYSV_ABI KernelBatchMap2(KernelBatchMapEntry* entries, int num_entries,
 		LOGF("\t [%d] start = %p, offset = 0x%016" PRIx64 ", length = 0x%016" PRIx64
 		     ", prot = 0x%02" PRIx32 ", type = 0x%02" PRIx32 ", op = %d\n",
 		     i, entry->start, entry->offset, entry->length,
-		     static_cast<uint32_t>(static_cast<unsigned char>(entry->protection)),
-		     static_cast<uint32_t>(static_cast<unsigned char>(entry->type)), entry->operation);
+		     static_cast<uint32_t>(entry->protection), static_cast<uint32_t>(entry->type),
+		     entry->operation);

 		if (entry->length == 0 || entry->operation < MAP_OP_MAP_DIRECT ||
 		    entry->operation > MAP_OP_TYPE_PROTECT) {
@@ -47,13 +47,13 @@ struct VirtualQueryInfo {
 static_assert(sizeof(VirtualQueryInfo) == 72, "VirtualQueryInfo struct size is incorrect");

 struct KernelBatchMapEntry {
-	void*    start;
-	uint64_t offset;
-	uint64_t length;
-	char     protection;
-	char     type;
-	int16_t  reserved;
-	int32_t  operation;
+	void*         start;
+	uint64_t      offset;
+	uint64_t      length;
+	unsigned char protection;
+	unsigned char type;
+	int16_t       reserved;
+	int32_t       operation;
 };

 static_assert(sizeof(KernelBatchMapEntry) == 32, "KernelBatchMapEntry struct size is incorrect");
@@ -25,6 +25,7 @@
 #include "libs/libs.h"

 #include <algorithm>
+#include <array>
 #include <atomic>
 #include <cmath>
 #include <cstddef>
@@ -142,8 +143,8 @@ struct RegisterDefaultsStorage {

 #include <fmt/format.h>

-static constexpr uint32_t GRAPHICS_REGISTER_DEFAULTS_MAX_VERSION      = 12;
-static constexpr uint32_t GRAPHICS_REGISTER_DEFAULTS_FALLBACK_VERSION = 11;
+static constexpr uint32_t GRAPHICS_REGISTER_DEFAULTS_MAX_VERSION      = 13;
+static constexpr uint32_t GRAPHICS_REGISTER_DEFAULTS_FALLBACK_VERSION = 13;
 static constexpr uint32_t GRAPHICS_INIT_NO_FEATURE_STATE              = 0;

 static std::mutex g_register_defaults_mutex;
@@ -3239,6 +3240,27 @@ uint32_t KYTY_SYSV_ABI AgcAcbCondExecGetSize() {
 	return AgcDcbCondExecGetSize();
 }

+uint32_t* KYTY_SYSV_ABI AgcAcbJump(CommandBuffer* buf, uint8_t cache_policy,
+                                   const uint32_t* target, uint32_t size_in_dwords) {
+	if (buf == nullptr) {
+		return nullptr;
+	}
+
+	auto* cmd = buf->AllocateDW(4);
+	if (cmd == nullptr) {
+		return nullptr;
+	}
+
+	auto target_address = reinterpret_cast<uint64_t>(target);
+	cmd[0]              = KYTY_PM4(4, Pm4::IT_INDIRECT_BUFFER, 0u);
+	cmd[1]              = static_cast<uint32_t>(target_address) & ~0x3u;
+	cmd[2]              = static_cast<uint32_t>(target_address >> 32u);
+	cmd[3] = 0x0f900000u | ((static_cast<uint32_t>(cache_policy) & 0x3u) << 28u) |
+	         (size_in_dwords & 0xfffffu);
+
+	return cmd;
+}
+
 uint32_t KYTY_SYSV_ABI AgcAcbJumpGetSize() {
 	return 0x10u;
 }
@@ -3250,6 +3272,12 @@ uint32_t* KYTY_SYSV_ABI AgcAcbWaitRegMem(CommandBuffer* buf, uint8_t size, uint8
 	                        poll_cycles);
 }

+uint64_t KYTY_SYSV_ABI AgcAcbWaitOnAddressGetSize(uint8_t size) {
+	PRINT_NAME();
+
+	return AgcDcbWaitOnAddressGetSize(size);
+}
+
 uint32_t* KYTY_SYSV_ABI AgcAcbDmaData(CommandBuffer* buf, uint8_t dst, uint8_t dst_cache_policy,
                                      uint64_t dst_address_or_offset, uint8_t src,
                                      uint8_t  src_cache_policy,
@@ -3321,6 +3349,12 @@ uint32_t* KYTY_SYSV_ABI AgcAcbCopyData(CommandBuffer* buf, uint8_t dst, uint8_t
 	return cmd;
 }

+uint64_t KYTY_SYSV_ABI AgcAcbCopyDataGetSize() {
+	PRINT_NAME();
+
+	return AgcDcbCopyDataGetSize();
+}
+
 uint32_t* KYTY_SYSV_ABI AgcAcbDispatchIndirect(CommandBuffer*       buf,
                                               const volatile void* indirect_args,
                                               uint32_t             modifier) {
@@ -3439,6 +3473,12 @@ uint32_t* KYTY_SYSV_ABI AgcDcbCopyData(CommandBuffer* buf, uint8_t dst, uint8_t
 	return cmd;
 }

+uint64_t KYTY_SYSV_ABI AgcDcbCopyDataGetSize() {
+	PRINT_NAME();
+
+	return 6 * sizeof(uint32_t);
+}
+
 uint32_t* KYTY_SYSV_ABI AgcDcbDmaData(CommandBuffer* buf, uint8_t engine, uint8_t dst,
                                      uint8_t dst_cache_policy, uint64_t dst_address_or_offset,
                                      uint8_t src, uint8_t src_cache_policy,
@@ -204,10 +204,13 @@ uint32_t KYTY_SYSV_ABI  AgcAcbAcquireMemGetSize();
 uint32_t* KYTY_SYSV_ABI AgcAcbCondExec(CommandBuffer* buf, const volatile uint32_t* address,
                                       uint32_t num_dwords);
 uint32_t KYTY_SYSV_ABI  AgcAcbCondExecGetSize();
+uint32_t* KYTY_SYSV_ABI AgcAcbJump(CommandBuffer* buf, uint8_t cache_policy,
+                                   const uint32_t* target, uint32_t size_in_dwords);
 uint32_t KYTY_SYSV_ABI  AgcAcbJumpGetSize();
 uint32_t* KYTY_SYSV_ABI AgcAcbWaitRegMem(CommandBuffer* buf, uint8_t size, uint8_t compare_function,
                                         uint8_t cache_policy, const volatile void* address,
                                         uint64_t reference, uint64_t mask, uint32_t poll_cycles);
+uint64_t KYTY_SYSV_ABI  AgcAcbWaitOnAddressGetSize(uint8_t size);
 uint32_t* KYTY_SYSV_ABI AgcAcbDmaData(CommandBuffer* buf, uint8_t dst, uint8_t dst_cache_policy,
                                      uint64_t dst_address_or_offset, uint8_t src,
                                      uint8_t  src_cache_policy,
@@ -218,6 +221,7 @@ uint32_t* KYTY_SYSV_ABI AgcAcbCopyData(CommandBuffer* buf, uint8_t dst, uint8_t
                                       uint64_t dst_address, uint8_t src, uint8_t src_cache_policy,
                                       uint64_t src_address_or_immediate, uint8_t item_size,
                                       uint8_t write_confirm);
+uint64_t KYTY_SYSV_ABI  AgcAcbCopyDataGetSize();
 uint32_t* KYTY_SYSV_ABI AgcAcbDispatchIndirect(CommandBuffer*       buf,
                                               const volatile void* indirect_args,
                                               uint32_t             modifier);
@@ -233,6 +237,7 @@ uint32_t* KYTY_SYSV_ABI AgcDcbCopyData(CommandBuffer* buf, uint8_t dst, uint8_t
                                       uint64_t dst_address, uint8_t src, uint8_t src_cache_policy,
                                       uint64_t src_address_or_immediate, uint8_t item_size,
                                       uint8_t write_confirm);
+uint64_t KYTY_SYSV_ABI  AgcDcbCopyDataGetSize();
 uint32_t* KYTY_SYSV_ABI AgcDcbDmaData(CommandBuffer* buf, uint8_t engine, uint8_t dst,
                                      uint8_t dst_cache_policy, uint64_t dst_address_or_offset,
                                      uint8_t src, uint8_t src_cache_policy,
@@ -1,4 +1,4 @@
-// No-feature, non-0x840f00 AgcGetRegisterDefaults2 dispatch, versions 0..12.
+// No-feature, non-0x840f00 AgcGetRegisterDefaults2 dispatch, versions 0..13.
 // Type entries keep the pointer index in bits 0..9; higher bits are AGC flags.

 // g_agc_internal_reg_defaults_v0: regs=(1, 12, 0, 4), ptrs=(1, 12, 0, 2), types=15.
@@ -185,6 +185,45 @@ static CompactRegisterDefaults g_agc_internal_reg_defaults_v11 = {
    g_agc_internal_reg_defaults_v11_types,     24,
 };

+// Version 13 takes the version-11 fallback and selects the >=13 state-0 lookup bank.
+// Only two internal table-1 values differ in that bank.
+static auto g_agc_internal_reg_defaults_v13_tbl1_regs = [] {
+	constexpr auto                    count = sizeof(g_agc_internal_reg_defaults_v11_tbl1_regs) /
+	                                          sizeof(g_agc_internal_reg_defaults_v11_tbl1_regs[0]);
+	std::array<ShaderRegister, count> regs {};
+	std::copy_n(g_agc_internal_reg_defaults_v11_tbl1_regs, count, regs.begin());
+
+	for (auto& reg: regs) {
+		if (reg.offset == 0x0081) {
+			reg.value = 0x00000001;
+		} else if (reg.offset == 0x0101) {
+			reg.value = 0x00000003;
+		}
+	}
+	return regs;
+}();
+
+static CompactRegisterDefaults g_agc_internal_reg_defaults_v13 = {
+    g_agc_internal_reg_defaults_v11_tbl0_regs,
+    8,
+    g_agc_internal_reg_defaults_v11_tbl0_ptrs,
+    8,
+    g_agc_internal_reg_defaults_v13_tbl1_regs.data(),
+    12,
+    g_agc_internal_reg_defaults_v11_tbl1_ptrs,
+    12,
+    g_agc_internal_reg_defaults_v11_tbl2_regs,
+    1,
+    g_agc_internal_reg_defaults_v11_tbl2_ptrs,
+    1,
+    g_agc_internal_reg_defaults_v11_tbl3_regs,
+    6,
+    g_agc_internal_reg_defaults_v11_tbl3_ptrs,
+    3,
+    g_agc_internal_reg_defaults_v11_types,
+    24,
+};
+
 // g_agc_internal_reg_defaults_v4: regs=(2, 15, 0, 4), ptrs=(2, 15, 0, 2), types=19.
 static ShaderRegister g_agc_internal_reg_defaults_v4_tbl0_regs[] = {
    {0x02af, 0x00040000},
@@ -1366,8 +1405,28 @@ static uint32_t g_agc_public_reg_defaults_v11_types[] = {
    0x036ac8a6, 0x000c0452, 0x00000000,
 };

+template <std::size_t Size>
+static std::array<ShaderRegister, Size>
+make_v11_plus_public_tbl0(const ShaderRegister (&source)[Size]) {
+	std::array<ShaderRegister, Size> regs {};
+	std::copy_n(source, Size, regs.begin());
+
+	for (auto& reg: regs) {
+		if (reg.offset == 0x0201) {
+			reg.value |= 0x00010000;
+		}
+	}
+	return regs;
+}
+
+// The common setup helper forces this table-0 patch for every version >= 11.
+static auto g_agc_public_reg_defaults_v11_tbl0_effective =
+    make_v11_plus_public_tbl0(g_agc_public_reg_defaults_v11_tbl0_regs);
+static auto g_agc_public_reg_defaults_v12_tbl0_effective =
+    make_v11_plus_public_tbl0(g_agc_public_reg_defaults_v10_tbl0_regs);
+
 static CompactRegisterDefaults g_agc_public_reg_defaults_v11 = {
-    g_agc_public_reg_defaults_v11_tbl0_regs,
+    g_agc_public_reg_defaults_v11_tbl0_effective.data(),
    488,
    g_agc_public_reg_defaults_v11_tbl0_ptrs,
    84,
@@ -1387,6 +1446,27 @@ static CompactRegisterDefaults g_agc_public_reg_defaults_v11 = {
    137,
 };

+static CompactRegisterDefaults g_agc_public_reg_defaults_v12 = {
+    g_agc_public_reg_defaults_v12_tbl0_effective.data(),
+    482,
+    g_agc_public_reg_defaults_v10_tbl0_ptrs,
+    79,
+    g_agc_public_reg_defaults_v10_tbl1_regs,
+    159,
+    g_agc_public_reg_defaults_v10_tbl1_ptrs,
+    29,
+    g_agc_public_reg_defaults_v10_tbl2_regs,
+    55,
+    g_agc_public_reg_defaults_v10_tbl2_ptrs,
+    20,
+    nullptr,
+    0,
+    nullptr,
+    0,
+    g_agc_public_reg_defaults_v10_types,
+    128,
+};
+
 // g_agc_public_reg_defaults_v4: regs=(488, 158, 43, 0), ptrs=(80, 28, 42, 0), types=150.
 static ShaderRegister g_agc_public_reg_defaults_v4_tbl0_regs[] = {
    {0x0202, 0x00cc0010}, {0x0109, 0x00000008}, {0x0104, 0x00000000}, {0x008f, 0x00000000},
@@ -2825,7 +2905,8 @@ static CompactRegisterDefaults* g_agc_public_reg_defaults_by_version[] = {
    &g_agc_public_reg_defaults_v9,  // version 9
    &g_agc_public_reg_defaults_v10, // version 10
    &g_agc_public_reg_defaults_v11, // version 11
-    &g_agc_public_reg_defaults_v10, // version 12
+    &g_agc_public_reg_defaults_v12, // version 12
+    &g_agc_public_reg_defaults_v11, // version 13: v11 fallback with the >=13 state-0 bank
 };

 static CompactRegisterDefaults* g_agc_internal_reg_defaults_by_version[] = {
@@ -2842,4 +2923,5 @@ static CompactRegisterDefaults* g_agc_internal_reg_defaults_by_version[] = {
    &g_agc_internal_reg_defaults_v10, // version 10
    &g_agc_internal_reg_defaults_v11, // version 11
    &g_agc_internal_reg_defaults_v10, // version 12
+    &g_agc_internal_reg_defaults_v13, // version 13
 };
@@ -73,10 +73,13 @@ LIB_DEFINE(InitAgcDriver_1) {
 	LIB_FUNC("ewobAQeMo5k", Gen5::AgcAcbAcquireMemGetSize);
 	LIB_FUNC("qyM2bxYFPAk", Gen5::AgcAcbCondExec);
 	LIB_FUNC("ozKzBP4aki4", Gen5::AgcAcbCondExecGetSize);
+	LIB_FUNC("e1DFTg+Sd8U", Gen5::AgcAcbJump);
 	LIB_FUNC("b-oySn+G2tE", Gen5::AgcAcbJumpGetSize);
 	LIB_FUNC("htn36gPnBk4", Gen5::AgcAcbWaitRegMem);
+	LIB_FUNC("idlaArvdXEs", Gen5::AgcAcbWaitOnAddressGetSize);
 	LIB_FUNC("-RnpfpxIhec", Gen5::AgcAcbDmaData);
 	LIB_FUNC("qzMN2XKGA4k", Gen5::AgcAcbCopyData);
+	LIB_FUNC("CbQh3DKMSno", Gen5::AgcAcbCopyDataGetSize);
 	LIB_FUNC("j3EtxFkSIhQ", Gen5::AgcAcbDispatchIndirect);
 	LIB_FUNC("eZ4+17OQz4Q", Gen5::AgcAcbWriteData);
 	LIB_FUNC("xAeBOa0A3kk", Gen5::AgcAcbSetMarker);
@@ -121,6 +124,7 @@ LIB_DEFINE(InitAgcDriver_1) {
 	LIB_FUNC("57labkp+rSQ", Gen5::AgcDcbAcquireMem);
 	LIB_FUNC("-vnlTPPXPrw", Gen5::AgcDcbAcquireMemGetSize);
 	LIB_FUNC("1rZSWUv1IRc", Gen5::AgcDcbCopyData);
+	LIB_FUNC("b5u0Jzm8TF8", Gen5::AgcDcbCopyDataGetSize); // SpongeBob PPSA26893
 	LIB_FUNC("WmAc2MEj6Io", Gen5::AgcDcbDmaData);
 	LIB_FUNC("xSAR0LTcRKM", Gen5::AgcDcbJump);
 	LIB_FUNC("VEGu4dixjUg", Gen5::AgcDcbJumpGetSize);
@@ -553,6 +553,12 @@ constexpr u32 EncodeVop1Sdwa(u32 src0, u32 dst_sel = 6, u32 dst_u = 0,
         ((s0 & 0x1u) << 23u);
 }

+constexpr u32 EncodeVop1Dpp(u32 src0, u32 dpp_ctrl = 0, u32 row_mask = 0xf,
+                            u32 bank_mask = 0xf) {
+  return (src0 & 0xffu) | ((dpp_ctrl & 0x1ffu) << 8u) |
+         ((bank_mask & 0xfu) << 24u) | ((row_mask & 0xfu) << 28u);
+}
+
 constexpr u32 EncodeVop2(u32 opcode, u32 dst, u32 src0, u32 src1) {
  return ((opcode & 0x3fu) << 25u) | ((dst & 0xffu) << 17u) |
         ((src1 & 0xffu) << 9u) | (src0 & 0x1ffu);
@@ -11931,6 +11937,7 @@ CoverageClass ClassifyOpcode(ShaderOpcode opcode,
  case Opcode::V_CMPX_NEQ_F32:
  case Opcode::V_CMPX_NLT_F32:
  case Opcode::V_CMP_CLASS_F32:
+  case Opcode::V_CMPX_CLASS_F32:
  case Opcode::V_CMP_LT_F16:
  case Opcode::V_CMP_EQ_F16:
  case Opcode::V_CMP_LE_F16:
@@ -12733,6 +12740,160 @@ TestCase ScalarNotB64UpdatesScc() {
           O::S_ENDPGM}};
 }

+TestCase ScalarMovB64PreservesRawSpecialData() {
+  using O = ShaderOpcode;
+
+  std::vector<u32> code;
+  AppendSMovLiteral(&code, 0, 0x09500000u);
+  AppendSMovLiteral(&code, 1, 0x81234567u);
+  code.push_back(EncodeSop1(0x04, 106, 0)); // s_mov_b64 vcc, s[0:1]
+  code.push_back(EncodeSop1(0x04, 2, 106)); // s_mov_b64 s[2:3], vcc
+  code.push_back(EncodeSop1(0x08, 4, 2));   // s_not_b64 s[4:5], s[2:3]
+  code.push_back(EncodeSop2(0x0a, 6, InlineU32(1), InlineU32(0)));
+
+  AppendStoreSgprPair(&code, 2, 0);
+  AppendStoreSgprPair(&code, 4, 2);
+  AppendStoreSgpr(&code, 6, 4);
+
+  AppendSMovLiteral(&code, 8, 0x09500001u);
+  AppendSMovLiteral(&code, 9, 0x81234567u);
+  code.push_back(EncodeSop1(0x04, 126, 8));  // s_mov_b64 exec, s[8:9]
+  code.push_back(EncodeSop1(0x04, 10, 126)); // s_mov_b64 s[10:11], exec
+  code.push_back(EncodeSop1(0x08, 12, 10));  // s_not_b64 s[12:13], s[10:11]
+  code.push_back(EncodeSop2(0x0a, 14, InlineU32(1), InlineU32(0)));
+
+  AppendStoreSgprPair(&code, 10, 5);
+  AppendStoreSgprPair(&code, 12, 7);
+  AppendStoreSgpr(&code, 14, 9);
+  AppendEnd(&code);
+
+  return {"ScalarMovB64PreservesRawSpecialData",
+          code,
+          {},
+          {0x09500000u, 0x81234567u, 0xf6afffffu, 0x7edcba98u, 1u,
+           0x09500001u, 0x81234567u, 0xf6affffeu, 0x7edcba98u, 1u},
+          {O::S_MOV_B32, O::S_MOV_B64, O::S_NOT_B64, O::S_CSELECT_B32,
+           O::V_MOV_B32, O::BUFFER_STORE_DWORD, O::S_ENDPGM}};
+}
+
+TestCase Scalar64RawSpecialOps() {
+  using O = ShaderOpcode;
+
+  std::vector<u32> code;
+  const auto capture_scc = [&](u32 dst) {
+    code.push_back(EncodeSop2(0x0a, dst, InlineU32(1), InlineU32(0)));
+  };
+
+  AppendSMovLiteral(&code, 0, 0x09500001u);
+  AppendSMovLiteral(&code, 1, 0x81234567u);
+  AppendSMovLiteral(&code, 2, 0xff00ff01u);
+  AppendSMovLiteral(&code, 3, 0x0f0f0f0fu);
+  code.push_back(EncodeSop1(0x04, 126, 0));
+  code.push_back(EncodeSop2(0x0f, 106, 126, 2));
+  code.push_back(EncodeSop1(0x04, 4, 106));
+  capture_scc(6);
+  code.push_back(EncodeSop1(0x08, 106, 106));
+  code.push_back(EncodeSop1(0x04, 8, 106));
+  capture_scc(10);
+
+  AppendSMovLiteral(&code, 12, 0x80000011u);
+  AppendSMovLiteral(&code, 13, 0x80000001u);
+  code.push_back(EncodeSop1(0x04, 106, 12));
+  code.push_back(EncodeSop1(0x0a, 106, 106));
+  code.push_back(EncodeSop1(0x04, 14, 106));
+  capture_scc(16);
+
+  code.push_back(EncodeSop1(0x0a, 26, 193u));
+  capture_scc(28);
+
+  code.push_back(EncodeSMovB32(18, InlineU32(1)));
+  code.push_back(EncodeSMovB32(19, InlineU32(0)));
+  code.push_back(EncodeSopc(0x06, InlineU32(1), InlineU32(1)));
+  code.push_back(EncodeSop2(0x0b, 126, 0, 18));
+  code.push_back(EncodeSop1(0x04, 20, 126));
+  code.push_back(EncodeSop1(0x08, 106, 126));
+  code.push_back(EncodeSop1(0x04, 22, 106));
+  capture_scc(24);
+  code.push_back(EncodeSop1(0x04, 126, 18));
+
+  AppendStoreSgprPair(&code, 4, 0);
+  AppendStoreSgpr(&code, 6, 2);
+  AppendStoreSgprPair(&code, 8, 3);
+  AppendStoreSgpr(&code, 10, 5);
+  AppendStoreSgprPair(&code, 14, 6);
+  AppendStoreSgpr(&code, 16, 8);
+  AppendStoreSgprPair(&code, 26, 9);
+  AppendStoreSgpr(&code, 28, 11);
+  AppendStoreSgprPair(&code, 20, 12);
+  AppendStoreSgprPair(&code, 22, 14);
+  AppendStoreSgpr(&code, 24, 16);
+  AppendEnd(&code);
+
+  return {"Scalar64RawSpecialOps",
+          code,
+          {},
+          {0x09000001u, 0x01030507u, 1, 0xf6fffffeu, 0xfefcfaf8u, 1,
+           0xf00000ffu, 0xf000000fu, 1, 0xffffffffu, 0xffffffffu, 1,
+           0x09500001u, 0x81234567u, 0xf6affffeu, 0x7edcba98u, 1},
+          {O::S_MOV_B32, O::S_MOV_B64, O::S_AND_B64, O::S_NOT_B64,
+           O::S_CSELECT_B32, O::S_WQM_B64, O::S_CMP_EQ_U32, O::S_CSELECT_B64,
+           O::V_MOV_B32, O::BUFFER_STORE_DWORD, O::S_ENDPGM}};
+}
+
+TestCase ScalarMovB32SpecialAliases() {
+  using O = ShaderOpcode;
+
+  std::vector<u32> code;
+  const auto capture_scc = [&](u32 dst) {
+    code.push_back(EncodeSop2(0x0a, dst, InlineU32(1), InlineU32(0)));
+  };
+
+  AppendSMovLiteral(&code, 0, 0x09500001u);
+  AppendSMovLiteral(&code, 1, 0x81234567u);
+  AppendSMovLiteral(&code, 2, 0xaaaaaaaau);
+  AppendSMovLiteral(&code, 3, 0xbbbbbbbbu);
+  code.push_back(EncodeSMovB32(4, InlineU32(1)));
+  code.push_back(EncodeSMovB32(5, InlineU32(0)));
+  code.push_back(EncodeSop1(0x04, 126, 0));
+  code.push_back(EncodeSop1(0x04, 106, 2));
+  code.push_back(EncodeSMovB32(107, 126));
+  code.push_back(EncodeSop1(0x04, 6, 106));
+  code.push_back(EncodeSop1(0x08, 8, 106));
+  capture_scc(10);
+
+  AppendSMovLiteral(&code, 12, 0x13579bdfu);
+  AppendSMovLiteral(&code, 13, 0x2468ace0u);
+  code.push_back(EncodeSop1(0x04, 106, 12));
+  code.push_back(EncodeSMovB32(126, 106));
+  code.push_back(EncodeSop1(0x04, 14, 126));
+  code.push_back(EncodeSop1(0x04, 126, 4));
+
+  code.push_back(EncodeVopc(0xc2, InlineU32(0), 0));
+  code.push_back(EncodeSMovB32(106, 106));
+  code.push_back(EncodeSop1(0x08, 16, 106));
+  capture_scc(18);
+  code.push_back(EncodeSop1(0x04, 20, 106));
+  code.push_back(EncodeSMovB32(20, 20));
+  code.push_back(EncodeSop1(0x08, 22, 20));
+  capture_scc(24);
+
+  AppendStoreSgprPair(&code, 6, 0);
+  AppendStoreSgprPair(&code, 8, 2);
+  AppendStoreSgpr(&code, 10, 4);
+  AppendStoreSgprPair(&code, 14, 5);
+  AppendStoreSgpr(&code, 18, 7);
+  AppendStoreSgpr(&code, 24, 8);
+  AppendEnd(&code);
+
+  return {"ScalarMovB32SpecialAliases",
+          code,
+          {},
+          {0xaaaaaaaau, 0x09500001u, 0x55555555u, 0xf6affffeu, 1, 0x13579bdfu,
+           0x81234567u, 0, 0},
+          {O::S_MOV_B32, O::S_MOV_B64, O::S_NOT_B64, O::S_CSELECT_B32,
+           O::V_CMP_EQ_U32, O::V_MOV_B32, O::BUFFER_STORE_DWORD, O::S_ENDPGM}};
+}
+
 TestCase ScalarQuadmaskB64() {
  using O = ShaderOpcode;

@@ -12831,8 +12992,7 @@ TestCase ScalarSaveExecOps() {
  return {"ScalarSaveExecOps",
          code,
          {},
-          // EXEC and saved EXEC values are invocation-local Booleans.
-          {1, 0, 1, 0, 1, 0, 1, 1, 0, 1, 0, 0, 0},
+          {1, 0, 1, 0, 0xffffffffu, 0xffffffffu, 1, 1, 0, 1, 0, 0, 0},
          {O::S_MOV_B32, O::S_AND_SAVEEXEC_B64, O::S_ORN2_SAVEEXEC_B64,
           O::S_ANDN1_SAVEEXEC_B64, O::S_AND_SAVEEXEC_B32,
           O::S_ANDN1_SAVEEXEC_B32, O::S_MOV_B64, O::V_MOV_B32,
@@ -12859,7 +13019,7 @@ TestCase ScalarOrn2SaveexecUsesSourceOrNotExec() {
  return {"ScalarOrn2SaveexecUsesSourceOrNotExec",
          code,
          {},
-          {1, 0, 1, 0, 1},
+          {0x0000000cu, 0x80000000u, 0xfffffff3u, 0x7fffffffu, 1},
          {O::S_MOV_B32, O::S_ORN2_SAVEEXEC_B64, O::S_MOV_B64, O::V_MOV_B32,
           O::BUFFER_STORE_DWORD, O::S_ENDPGM}};
 }
@@ -14249,6 +14409,53 @@ TestCase VectorLaneAndPackedOps() {
           O::BUFFER_STORE_DWORD, O::S_ENDPGM}};
 }

+TestCase Vop2PkFmacF16AccumulatesPackedHalvesIndependently() {
+  using O = ShaderOpcode;
+
+  std::vector<u32> code;
+  AppendVMovLiteral(&code, 0, 0x50004c00u); // low=16.0h, high=32.0h
+  AppendVMovLiteral(&code, 1, 0x42003c00u); // low=1.0h, high=3.0h
+  AppendVMovLiteral(&code, 2, 0x48004400u); // low=4.0h, high=8.0h
+
+  code.push_back(EncodeVop2(0x3c, 0, Vgpr(1), 2));
+  AppendStoreVgpr(&code, 0, 0);
+  code.push_back(EncodeVop2(0x3c, 0, Vgpr(1), 2));
+  AppendStoreVgpr(&code, 0, 1);
+
+  AppendVMovLiteral(&code, 3, 0x50004c00u);
+  code.push_back(EncodeVop2(0x3c, 3, 242, 2)); // Inline 1.0h is {1.0h, 0.0h}.
+  AppendStoreVgpr(&code, 3, 2);
+  AppendEnd(&code);
+
+  return {"Vop2PkFmacF16AccumulatesPackedHalvesIndependently",
+          code,
+          {},
+          {0x53004d00u, 0x55004e00u, 0x50004d00u},
+          {O::V_MOV_B32, O::V_PK_FMAC_F16, O::BUFFER_STORE_DWORD,
+           O::S_ENDPGM}};
+}
+
+TestCase Vop2PkFmacF16DppNegatesBothPackedHalves() {
+  using O = ShaderOpcode;
+
+  std::vector<u32> code;
+  AppendVMovLiteral(&code, 0, 0x50004c00u); // low=16.0h, high=32.0h
+  AppendVMovLiteral(&code, 1, 0x42003c00u); // low=1.0h, high=3.0h
+  AppendVMovLiteral(&code, 2, 0x48004400u); // low=4.0h, high=8.0h
+
+  code.push_back(EncodeVop2(0x3c, 0, 250, 2));
+  code.push_back(EncodeVop2Dpp(1, 0, 0xf, 0xf, 1));
+  AppendStoreVgpr(&code, 0, 0);
+  AppendEnd(&code);
+
+  return {"Vop2PkFmacF16DppNegatesBothPackedHalves",
+          code,
+          {},
+          {0x48004a00u},
+          {O::V_MOV_B32, O::V_PK_FMAC_F16, O::BUFFER_STORE_DWORD,
+           O::S_ENDPGM}};
+}
+
 TestCase Vop3pOpselHiUsesArchitecturalSourceBits() {
  using O = ShaderOpcode;

@@ -14320,6 +14527,54 @@ TestCase CvtPkrtzF16F32SubnormalRoundsTowardZero() {
           O::S_ENDPGM}};
 }

+TestCase CvtPkrtzF16F32SdwaAndOutputModifiers() {
+  using O = ShaderOpcode;
+
+  std::vector<u32> code;
+  AppendVMovLiteral(&code, 0, 0x3f000000u); // 0.5f
+  AppendVMovLiteral(&code, 1, 0x40000000u); // 2.0f
+  AppendVMovLiteral(&code, 2, 0x3f802000u); // exactly 0x3c01h
+  AppendVMovLiteral(&code, 3, 0x00000000u);
+  AppendVMovLiteral(&code, 4, 0xbf000000u); // -0.5f
+  for (u32 dst = 10; dst <= 18; dst++) {
+    AppendVMovLiteral(&code, dst, 0xa1b2c3d4u);
+  }
+
+  code.push_back(EncodeVop2(0x2f, 10, 249, 1));
+  code.push_back(EncodeVop2Sdwa(0, 6, 2, 6, 6, 0, 0, 0, 0, 0, 0, 0, 0,
+                                1));
+  code.push_back(EncodeVop2(0x2f, 11, 249, 3));
+  code.push_back(EncodeVop2Sdwa(2, 0, 0));
+  code.push_back(EncodeVop2(0x2f, 12, 249, 1));
+  code.push_back(EncodeVop2Sdwa(0, 6, 0, 3, 5));
+  code.push_back(EncodeVop2(0x2f, 13, 249, 1));
+  code.push_back(EncodeVop2Sdwa(4, 6, 0, 6, 6, 0, 0, 0, 1, 1));
+
+  code.push_back(EncodeVop2(0x2f, 14, 250, 1));
+  code.push_back(EncodeVop2Dpp(0, 0, 0xf, 0xf, 1));
+  AppendVop3(&code, 0x12f, 15, Vgpr(0), Vgpr(1), 0, 0, 0, true);
+  code.push_back(EncodeVop2(0x2f, 16, 249, 1));
+  code.push_back(EncodeVop2Sdwa(0, 4, 2));
+  code.push_back(EncodeVop2(0x2f, 17, 249, 1));
+  code.push_back(EncodeVop2Sdwa(0, 6, 2, 6, 6, 0, 0, 0, 0, 0, 0, 0, 0,
+                                0, 1));
+  AppendVop3(&code, 0x12f, 18, Vgpr(0), Vgpr(1), 0, 0, 0, false, 1);
+
+  for (u32 dst = 10; dst <= 18; dst++) {
+    AppendStoreVgpr(&code, dst, dst - 10);
+  }
+  AppendEnd(&code);
+
+  return {"CvtPkrtzF16F32SdwaAndOutputModifiers",
+          code,
+          {},
+          {0x3c003800u, 0x00000001u, 0x00000000u, 0xc0003800u,
+           0x4000b800u, 0x3c003800u, 0xa1b23800u, 0x44003c00u,
+           0x44003c00u},
+          {O::V_MOV_B32, O::V_CVT_PKRTZ_F16_F32, O::BUFFER_STORE_DWORD,
+           O::S_ENDPGM}};
+}
+
 TestCase PackedMinMaxF16NanAndSignedZeroEdges() {
  using O = ShaderOpcode;

@@ -14427,6 +14682,99 @@ TestCase VectorCvtU16F16Sdwa() {
          {O::V_MOV_B32, O::V_CVT_U16_F16, O::BUFFER_STORE_DWORD, O::S_ENDPGM}};
 }

+TestCase NativeAndSdwa16BitDestinationWrites() {
+  using O = ShaderOpcode;
+
+  std::vector<u32> code;
+  AppendVMovLiteral(&code, 0, 0x3fc00000u); // 1.5f -> 0x3e00h
+  AppendVMovLiteral(&code, 1, 0x00004200u); // low=3.0h
+  AppendVMovLiteral(&code, 2, 0x0000c000u); // low=-2.0h
+  AppendVMovLiteral(&code, 3, 0x00003c00u); // low=1.0h
+  AppendVMovLiteral(&code, 4, 0x00004000u); // low=2.0h
+
+  const u32 sentinels[] = {
+      0x3c001234u, 0xbbbb2345u, 0xcccc3456u, 0xdddd4567u, 0xeeee5678u,
+      0xffff6789u, 0xf00d789au, 0x11118888u, 0x22229999u, 0x3333aaaau,
+      0xaaaa0000u, 0xbbbb0000u, 0xcccc0000u, 0xabcd1234u, 0xbcde2345u,
+      0xcdef3456u, 0xdef01234u, 0xef012345u, 0xf0123456u, 0x01234567u,
+      0xaaaa1234u, 0xbbbb5678u, 0x24681234u, 0x13572468u, 0x89abcdefu,
+      0xfedcba98u, 0x0bad1234u,
+  };
+  // AppendStoreVgpr uses v31 as its address scratch register.
+  const auto output_vgpr = [](u32 index) { return 10u + index + (index >= 21u ? 1u : 0u); };
+  for (u32 i = 0; i < static_cast<u32>(std::size(sentinels)); i++) {
+    AppendVMovLiteral(&code, output_vgpr(i), sentinels[i]);
+  }
+
+  code.push_back(EncodeVop1(0x0a, 10, Vgpr(0)));
+  code.push_back(EncodeVop1(0x0a, 11, 250));
+  code.push_back(EncodeVop1Dpp(0));
+  AppendVop3(&code, 0x18a, 12, Vgpr(0), 0);
+  code.push_back(EncodeVop1(0x52, 13, Vgpr(1)));
+  code.push_back(EncodeVop1(0x53, 14, Vgpr(2)));
+
+  code.push_back(EncodeVop1(0x0a, 15, 249));
+  code.push_back(EncodeVop1Sdwa(0));
+  code.push_back(EncodeVop1(0x54, 16, 249));
+  code.push_back(EncodeVop1Sdwa(10, 4, 2, 5));
+  code.push_back(EncodeVop1(0x52, 17, 249));
+  code.push_back(EncodeVop1Sdwa(1, 6, 0, 4));
+  code.push_back(EncodeVop1(0x52, 18, 249));
+  code.push_back(EncodeVop1Sdwa(1, 4, 0, 4));
+  code.push_back(EncodeVop1(0x53, 19, 249));
+  code.push_back(EncodeVop1Sdwa(2, 4, 1, 4));
+
+  code.push_back(EncodeVop2(0x32, 20, 250, 4));
+  code.push_back(EncodeVop2Dpp(3));
+  AppendVop3(&code, 0x132, 21, Vgpr(3), Vgpr(4));
+  code.push_back(EncodeVop2(0x32, 22, 249, 4));
+  code.push_back(EncodeVop2Sdwa(3, 4, 0, 4, 4));
+
+  code.push_back(EncodeVop1(0x0a, 23, 249));
+  code.push_back(EncodeVop1Sdwa(0, 4, 2, 3));
+  code.push_back(EncodeVop1(0x0a, 24, 249));
+  code.push_back(EncodeVop1Sdwa(0, 4, 2, 5));
+  code.push_back(EncodeVop1(0x0a, 25, 249));
+  code.push_back(EncodeVop1Sdwa(0, 4, 2, 6, 0, 1, 1));
+
+  code.push_back(EncodeVop1(0x0a, 26, 249));
+  code.push_back(EncodeVop1Sdwa(0, 4, 2, 6, 0, 0, 0, 0, 0, 1));
+  code.push_back(EncodeVop1(0x0a, 27, 249));
+  code.push_back(EncodeVop1Sdwa(0, 4, 2, 6, 0, 0, 0, 0, 1));
+  AppendVop3(&code, 0x18a, 28, Vgpr(0), 0, 0, 0, 0, false, 1);
+  AppendVop3(&code, 0x18a, 29, Vgpr(0), 0, 0, 0, 0, true);
+  AppendVop3(&code, 0x34b, 30, Vgpr(3), Vgpr(4), Vgpr(3));
+  AppendVop3(&code, 0x34b, 32, Vgpr(3), Vgpr(4), Vgpr(3), 0, 0x8);
+  code.push_back(EncodeVop1(0x52, 33, 249));
+  code.push_back(EncodeVop1Sdwa(1, 4, 2, 4, 0, 1, 1, 0, 1));
+  AppendVop3(&code, 0x1d3, 34, Vgpr(1), 0, 0, 1, 0, true, 0, 1);
+  code.push_back(EncodeVop1(0x07, 35, 249));
+  code.push_back(EncodeVop1Sdwa(0, 6, 0, 6, 0, 1, 1, 0, 1));
+  AppendVop3(&code, 0x188, 36, Vgpr(0), 0, 0, 1, 0, true, 0, 1);
+  code.push_back(EncodeVop1(0x54, 37, 249));
+  code.push_back(EncodeVop1Sdwa(4, 4, 2, 4, 0, 1, 1));
+
+  for (u32 i = 0; i < static_cast<u32>(std::size(sentinels)); i++) {
+    AppendStoreVgpr(&code, output_vgpr(i), i);
+  }
+  AppendEnd(&code);
+
+  return {"NativeAndSdwa16BitDestinationWrites",
+          code,
+          {},
+          {0x3c003e00u, 0xbbbb3e00u, 0xcccc3e00u, 0xdddd0003u,
+           0xeeeefffeu, 0x00003e00u, 0xf00d3c00u, 0x00000003u,
+           0x00000003u, 0xfffffffeu, 0xaaaa4200u, 0xbbbb4200u,
+           0x00004200u, 0xabcd0000u, 0xbcde0000u, 0xcdefbe00u,
+           0xdef04200u, 0xef013c00u, 0xf0124200u, 0x01233c00u,
+           0xaaaa4200u, 0x42005678u, 0x24680000u, 0x1357fffdu,
+           0x00000000u, 0xffffffffu, 0x0badb800u},
+          {O::V_MOV_B32, O::V_CVT_F16_F32, O::V_CVT_U32_F32,
+           O::V_CVT_I32_F32, O::V_CVT_U16_F16, O::V_CVT_I16_F16,
+           O::V_RCP_F16, O::V_ADD_F16, O::V_FMA_F16, O::BUFFER_STORE_DWORD,
+           O::S_ENDPGM}};
+}
+
 TestCase VectorMinMaxMed3F16Ops() {
  using O = ShaderOpcode;

@@ -15785,6 +16133,38 @@ TestCase VectorCompareClassF32() {
           O::BUFFER_STORE_DWORD, O::S_ENDPGM}};
 }

+TestCase VectorVopcSdwaCmpxClassF32CapturedExecMask() {
+  using O = ShaderOpcode;
+
+  std::vector<u32> code;
+  AppendVMovLiteral(&code, 13, 0x7fc00000u); // Quiet NaN.
+  AppendVMovU32(&code, 3, 7);
+  AppendVMovU32(&code, 31, 0);
+  code.push_back(EncodeSMovB32(106, InlineU32(2))); // qNaN class mask.
+  code.insert(code.end(), {0x7d30d4f9u, 0x8606000du});
+  AppendBufferStoreDword(&code, 3, 31);
+
+  code.push_back(EncodeSMovB32(126, InlineU32(1))); // Restore lane-zero EXEC.
+  AppendVMovLiteral(&code, 13, 0x3f800000u);        // Positive normal.
+  AppendVMovU32(&code, 3, 9);
+  AppendVMovU32(&code, 31, 4);
+  code.push_back(EncodeSMovB32(106, InlineU32(2))); // qNaN class mask.
+  code.insert(code.end(), {0x7d30d4f9u, 0x8606000du});
+  AppendBufferStoreDword(&code, 3, 31);
+  AppendEnd(&code);
+
+  TestCase test{
+      "VectorVopcSdwaCmpxClassF32CapturedExecMask",
+      code,
+      {0x11111111u, 0x22222222u},
+      {7u, 0x22222222u},
+      {O::S_MOV_B32, O::V_MOV_B32, O::V_CMPX_CLASS_F32,
+       O::BUFFER_STORE_DWORD, O::S_ENDPGM}};
+  test.decoded_counts = {
+      {"V_CMPX_CLASS_F32 exec_lo, v13, vcc_lo", 2}};
+  return test;
+}
+
 TestCase VectorCompareF16Ops() {
  using O = ShaderOpcode;

@@ -16269,6 +16649,30 @@ TestCase ScalarMemoryLoadVariants() {
           O::BUFFER_STORE_DWORD, O::S_ENDPGM}};
 }

+TestCase ScalarLoadUsesRawExecAddress() {
+  using O = ShaderOpcode;
+
+  std::vector<u32> code;
+  code.push_back(EncodeSMovB32(0, InlineU32(0)));
+  code.push_back(EncodeSMovB32(1, InlineU32(0)));
+  code.push_back(EncodeSMovB32(2, InlineU32(4)));
+  code.push_back(EncodeSMovB32(3, InlineU32(0)));
+  code.push_back(EncodeSop1(0x04, 126, 2));
+  code.push_back(EncodeSmem0(0x00, 20, 63));
+  code.push_back(EncodeSmem1(0));
+  code.push_back(EncodeSMovB32(2, InlineU32(1)));
+  code.push_back(EncodeSop1(0x04, 126, 2));
+  AppendStoreSgpr(&code, 20, 0);
+  AppendEnd(&code);
+
+  return {"ScalarLoadUsesRawExecAddress",
+          code,
+          {0x11111111u, 0x22222222u},
+          {0x22222222u, 0x22222222u},
+          {O::S_MOV_B32, O::S_MOV_B64, O::S_LOAD_DWORD, O::V_MOV_B32,
+           O::BUFFER_STORE_DWORD, O::S_ENDPGM}};
+}
+
 TestCase ScalarLoadSignedImmediateOffsetAddsSoffset() {
  using O = ShaderOpcode;

@@ -20721,6 +21125,9 @@ std::vector<TestCase> MakeCases() {
  AddCase(ScalarCompareOps);
  AddCase(ScalarShiftAddAndMaskOps);
  AddCase(ScalarNotB64UpdatesScc);
+  AddCase(ScalarMovB64PreservesRawSpecialData);
+  AddCase(Scalar64RawSpecialOps);
+  AddCase(ScalarMovB32SpecialAliases);
  AddCase(ScalarQuadmaskB64);
  AddCase(ScalarFlbitI32B64Gpu);
  AddCase(ScalarSaveExecOps);
@@ -20767,12 +21174,16 @@ std::vector<TestCase> MakeCases() {
  AddCase(VectorVop3BSubCoU32UsesRdna2Opcode310);
  AddCase(VectorMadU64U32UnsignedCarryOut);
  AddCase(VectorLaneAndPackedOps);
+  AddCase(Vop2PkFmacF16AccumulatesPackedHalvesIndependently);
+  AddCase(Vop2PkFmacF16DppNegatesBothPackedHalves);
  AddCase(Vop3pOpselHiUsesArchitecturalSourceBits);
  AddCase(CvtPkU8F32PacksSelectedByte);
  AddCase(CvtPkrtzF16F32SubnormalRoundsTowardZero);
+  AddCase(CvtPkrtzF16F32SdwaAndOutputModifiers);
  AddCase(PackedMinMaxF16NanAndSignedZeroEdges);
  AddCase(VectorMinMaxF16Ops);
  AddCase(VectorCvtU16F16Sdwa);
+  AddCase(NativeAndSdwa16BitDestinationWrites);
  AddCase(VectorMinMaxMed3F16Ops);
  AddCase(VectorSpecialF16Ops);
  AddCase(VectorCosF16CapturedSdwaAndEdges);
@@ -20812,6 +21223,7 @@ std::vector<TestCase> MakeCases() {
  AddCase(VectorVopcCmpxNeU64CapturedExecMask);
  AddCase(VectorVop3CmpxNeI64CapturedExecMask);
  AddCase(VectorCompareClassF32);
+  AddCase(VectorVopcSdwaCmpxClassF32CapturedExecMask);
  AddCase(VectorCompareF16Ops);
  AddCase(Vop2SdwaCndmaskSourceModifier);
  AddCase(Vop2SdwaCndmaskCapturedByte3Source);
@@ -20829,6 +21241,7 @@ std::vector<TestCase> MakeCases() {
  AddCase(BranchVccnzUsesInvocationMask);
  AddCase(BranchVccnzUsesCarryProducedInvocationMask);
  AddCase(ScalarMemoryLoadVariants);
+  AddCase(ScalarLoadUsesRawExecAddress);
  AddCase(ScalarLoadSignedImmediateOffsetAddsSoffset);
  AddCase(ScalarLoadAlignsComponentsAndMasksAddress);
  AddCase(BufferLoadStore);
@@ -3169,16 +3169,16 @@ void TestNewShaderRecompilerStagedShaderOps() {
        "new decoder did not decode RDNA2 S_SUBB_U32");
  Check(Common::ContainsStr(result.decoded_dump, "s_bitset0_b32 s3, s1"),
        "new decoder did not decode RDNA2 S_BITSET0_B32");
-  Check(Common::ContainsStr(result.decoded_dump, "v_fmac_f16 v70.sdwa(sel=4"),
+  Check(Common::ContainsStr(result.decoded_dump, "v_fmac_f16 v70, v5, v6"),
        "new decoder did not decode RDNA2 V_FMAC_F16");
  Check(
      Common::ContainsStr(
          result.decoded_dump,
-          "v_fmamk_f16 v71.sdwa(sel=4,sext=0), v7, 0x3c003c00, v8"),
+          "v_fmamk_f16 v71, v7, 0x3c003c00, v8"),
      "new decoder did not consume V_FMAMK_F16 literal as the multiply source");
  Check(Common::ContainsStr(
            result.decoded_dump,
-            "v_fmaak_f16 v72.sdwa(sel=4,sext=0), v9, v10, 0x40004000"),
+            "v_fmaak_f16 v72, v9, v10, 0x40004000"),
        "new decoder did not consume V_FMAAK_F16 literal as the add source");
  Check(Common::ContainsStr(result.ir_dump,
                            "ScalarSubBorrowCarryU32 s2, s0, s1, scc"),
@@ -3189,15 +3189,15 @@ void TestNewShaderRecompilerStagedShaderOps() {
  Check(
      Common::ContainsStr(
          result.ir_dump,
-          "FmaF16 v70.sdwa(sel=4,sext=0), v5, v6, v70.sdwa(sel=4,sext=0)"),
+          "FmaF16 v70, v5, v6, v70"),
      "V_FMAC_F16 did not lower using the destination as the FMA accumulator");
  Check(
      Common::ContainsStr(result.ir_dump,
-                          "FmaF16 v71.sdwa(sel=4,sext=0), v7, 0x3c003c00, v8"),
+                          "FmaF16 v71, v7, 0x3c003c00, v8"),
      "V_FMAMK_F16 did not lower with the literal in source 1");
  Check(
      Common::ContainsStr(result.ir_dump,
-                          "FmaF16 v72.sdwa(sel=4,sext=0), v9, v10, 0x40004000"),
+                          "FmaF16 v72, v9, v10, 0x40004000"),
      "V_FMAAK_F16 did not lower with the literal in source 2");
  Check(SpirvContainsOpcode(result.spirv, 130),
        "SPIR-V binary does not contain OpISub for S_SUBB_U32");
@@ -4203,6 +4203,17 @@ void TestNewShaderDecoderArchitecture() {
              packed_inst.src2.op_sel_hi == (source == 2u),
          "VOP3P OPSEL_HI source bit mapping is incorrect");
  }
+
+  const uint32_t packed_fmac_dpp[] = {0x780402fau, 0xff500000u};
+  Instruction packed_fmac;
+  Check(DecodeInstruction(packed_fmac_dpp, 0u, packed_fmac, &error),
+        error.c_str());
+  Check(packed_fmac.opcode == Opcode::V_PK_FMAC_F16 &&
+            packed_fmac.src0.dpp && packed_fmac.src0.op_sel_hi &&
+            packed_fmac.src1.op_sel_hi && packed_fmac.dst.op_sel_hi &&
+            packed_fmac.src0.negate && packed_fmac.src0.negate_hi &&
+            packed_fmac.src1.negate && packed_fmac.src1.negate_hi,
+        "VOP2 DPP V_PK_FMAC_F16 lost its implicit packed modifiers");
 }

 void TestNewShaderRecompilerRejectsDppOn64BitCompares() {
@@ -4231,6 +4242,67 @@ void TestNewShaderRecompilerRejectsDppOn64BitCompares() {
  }
 }

+void TestNewShaderRecompilerCapturedVopcSdwaCmpxClass() {
+  using namespace ShaderRecompiler;
+
+  const uint32_t shader[] = {
+      0x7d30d4f9u, 0x8606000du, // v_cmpx_class_f32 exec, v13, vcc_lo (SDWA)
+      EncodeSopp(0x01),
+  };
+
+  Decoder::Instruction decoded;
+  std::string error;
+  Check(Decoder::DecodeInstruction(shader, 0u, decoded, &error),
+        error.c_str());
+  Check(decoded.family == Decoder::Family::VOPC &&
+            decoded.opcode == Decoder::Opcode::V_CMPX_CLASS_F32 &&
+            decoded.opcode_id == 0x98u && decoded.word_count == 2u &&
+            decoded.raw_count == 2u &&
+            decoded.dst.kind == Decoder::OperandKind::ExecLo &&
+            decoded.src_count == 2u &&
+            decoded.src0.kind == Decoder::OperandKind::Vgpr &&
+            decoded.src0.reg == 13u && decoded.src0.sdwa_sel == 6u &&
+            decoded.src1.kind == Decoder::OperandKind::VccLo &&
+            decoded.src1.sdwa_sel == 6u,
+        "decoder rejected captured SDWA V_CMPX_CLASS_F32 fields");
+
+  Decoder::Program program;
+  CFG::Graph graph;
+  IR::Program ir;
+  ShaderPixelInputInfo pixel{};
+  Frontend::TranslateOptions translate_options{};
+  translate_options.stage = ShaderType::Pixel;
+  translate_options.wave_size = 64u;
+  translate_options.pixel = &pixel;
+  Check(Decoder::DecodeProgram(shader, program, &error) &&
+            CFG::BuildGraph(program, graph, &error) &&
+            Frontend::TranslateProgram(program, graph, translate_options, ir,
+                                       &error),
+        error.c_str());
+  uint32_t class_compares = 0u;
+  uint32_t dynamic_exec_writes = 0u;
+  for (const auto *block : ir.blocks) {
+    for (const auto &inst : *block) {
+      class_compares +=
+          inst.GetOpcode() == IR::ValueOpcode::FPCmpClass32 ? 1u : 0u;
+      if (inst.GetOpcode() == IR::ValueOpcode::SetExec &&
+          !inst.Arg(0).Resolve().IsImmediate()) {
+        dynamic_exec_writes++;
+      }
+    }
+  }
+  Check(class_compares == 1u && dynamic_exec_writes == 1u,
+        "captured V_CMPX_CLASS_F32 did not lower to class compare plus EXEC update");
+
+  auto options = MakeCompileOptions(ShaderType::Pixel);
+  CompileResult result;
+  Check(TryRecompile(shader, options, result, &error), error.c_str());
+  Check(Common::ContainsStr(result.decoded_dump,
+                            "V_CMPX_CLASS_F32 exec_lo, v13, vcc_lo"),
+        "captured SDWA V_CMPX_CLASS_F32 was not present in the decoded dump");
+  CheckSpirvBinaryValidates(result.spirv);
+}
+
 void TestNewShaderRecompilerIrLookupMissFailsExplicitly() {
  ShaderRecompiler::Decoder::Program decoded;
  ShaderRecompiler::Decoder::Instruction missing;
@@ -9398,10 +9470,12 @@ void TestTypedEntryStateIsMinimal() {
          error.c_str());

    uint32_t set_exec = 0;
+    uint32_t set_exec_mask_tag = 0;
    uint32_t set_exec_lo = 0;
    uint32_t set_exec_hi = 0;
    uint32_t ballots = 0;
    IR::Value exec;
+    IR::Value exec_mask_tag;
    IR::Value exec_lo;
    IR::Value exec_hi;
    for (const auto *block : values.blocks) {
@@ -9414,6 +9488,10 @@ void TestTypedEntryStateIsMinimal() {
          set_exec++;
          exec = inst.Arg(0).Resolve();
          break;
+        case IR::ValueOpcode::SetExecMaskTag:
+          set_exec_mask_tag++;
+          exec_mask_tag = inst.Arg(0).Resolve();
+          break;
        case IR::ValueOpcode::SetExecLo:
          set_exec_lo++;
          exec_lo = inst.Arg(0).Resolve();
@@ -9428,6 +9506,7 @@ void TestTypedEntryStateIsMinimal() {
        case IR::ValueOpcode::SetVectorRegister:
        case IR::ValueOpcode::SetScc:
        case IR::ValueOpcode::SetVcc:
+        case IR::ValueOpcode::SetVccMaskValidTag:
        case IR::ValueOpcode::SetVccLo:
        case IR::ValueOpcode::SetVccHi:
        case IR::ValueOpcode::SetM0:
@@ -9438,8 +9517,9 @@ void TestTypedEntryStateIsMinimal() {
        }
      }
    }
-    Check(set_exec == 1u && set_exec_lo == 1u && set_exec_hi == 1u &&
-              ballots == 0u && exec.IsImmediate() && exec.U1() &&
+    Check(set_exec == 1u && set_exec_mask_tag == 1u && set_exec_lo == 1u &&
+              set_exec_hi == 1u && ballots == 0u && exec.IsImmediate() &&
+              exec.U1() && exec_mask_tag.IsImmediate() && exec_mask_tag.U1() &&
              exec_lo.IsImmediate() && exec_lo.U32() == 1u &&
              exec_hi.IsImmediate() && exec_hi.U32() == 0u,
          "typed entry is not the local {1,0} invocation mask");
@@ -10300,11 +10380,8 @@ void TestNewShaderRecompilerPerInvocationMasksWithoutMirrors() {
        error.c_str());
  CheckSpirvBinaryValidates(result.spirv);
  const auto wqm_source = DisassembleSpirvBinary(result.spirv);
-  Check(Common::ContainsStr(wqm_source, "OpCapability GroupNonUniformBallot") &&
-            Common::ContainsStr(wqm_source, "OpGroupNonUniformBallot"),
-        "per-invocation scalar WQM omitted its subgroup ballot capability");
-  Check(SpirvInstructionOpcodeCount(result.spirv, 132u) == 2u,
-        "wave64 WQM did not compact exactly two ballot words");
+  Check(!Common::ContainsStr(wqm_source, "OpGroupNonUniformBallot"),
+        "raw scalar WQM reconstructed its source from invocation provenance");

  auto wave32_options = options;
  wave32_options.wave_size = 32u;
@@ -10312,8 +10389,9 @@ void TestNewShaderRecompilerPerInvocationMasksWithoutMirrors() {
                                       &error),
        error.c_str());
  CheckSpirvBinaryValidates(result.spirv);
-  Check(SpirvInstructionOpcodeCount(result.spirv, 132u) == 1u,
-        "wave32 WQM retained the unused high ballot-word expansion");
+  Check(!Common::ContainsStr(DisassembleSpirvBinary(result.spirv),
+                             "OpGroupNonUniformBallot"),
+        "wave32 raw scalar WQM reconstructed invocation provenance");

  const uint32_t cross_lane_shader[] = {
      EncodeSop2(0x25, 126, 132, 128), // s_bfm_b64 exec, 4, 0
@@ -12416,8 +12494,8 @@ void TestNewShaderRecompilerSpirvSizeBaselines() {
                                   .conditional_branches = 1,
                                   .ballots = 1},
                                  ShaderType::Vertex);
-  Check(Common::ContainsStr(wqm_result.ir_dump, "WqmMask"),
-        "WQM size fixture no longer reaches per-invocation WqmMask IR");
+  Check(Common::ContainsStr(wqm_result.ir_dump, "WqmU64"),
+        "WQM size fixture no longer reaches raw WqmU64 IR");

  const uint32_t dispatcher[] = {
      EncodeSopp(0x05, 2),       // entry -> B, fallthrough A
@@ -12476,6 +12554,7 @@ int main() {
  // ShaderRecompilerComputeTests; keep the distinct decoder contract checks
  // here.
  TestNewShaderDecoderArchitecture();
+  TestNewShaderRecompilerCapturedVopcSdwaCmpxClass();
  TestNewShaderRecompilerRejectsDppOn64BitCompares();
  TestNewShaderRecompilerIrLookupMissFailsExplicitly();
  TestPsInputCountRegisterDecode();
Author	SHA1	Message	Date
Mehmet Cambaz	cd0f0a428a	graphics: quiet two misleading render diagnostics (#416 ) * graphics: quiet two misleading render diagnostics The three PM4 register-offset normalization messages logged once per occurrence with no limit. They are printed in the colour reserved for problems, and normalizing the offset is routine, so a short session buries the log in them: PAC-MAN WORLD 2 Re-PAC (PPSA18189) emitted 24969 of them in about a minute, a fifth of a megabyte and 1.9% of every line written. Rate-limit all three the way the surrounding diagnostics in these files already are, which keeps the first occurrences without drowning what follows. The EQAA check warned whenever a control register was set, including on single-sample targets where those controls do nothing. That title never multisamples -- all 128 of its render targets are samples=1 -- yet it still produced the warning, pointing at a cause that could not apply. Return early when multisampling is off, matching how the neighbouring sample-location check already reads msaa_num_samples. Log output only; no rendering behaviour changes. * graphics: silence normalized offset diagnostics --------- Co-authored-by: nmzik <Nmzik@mail.ru>	2026-08-31 03:33:32 +04:00
nomolao2-cell	6758d67942	shader: preserve raw VCC bits in S_MOV_B64 (#418 ) * shader: preserve raw VCC bits in S_MOV_B64 * fix(shader): centralize scalar U64 mask provenance --------- Co-authored-by: nmzik <Nmzik@mail.ru>	2026-08-31 03:04:16 +04:00
nmzik	dc5b7c19a6	Revert "fix(agc): honor top bound for single-ended buffers" This reverts commit `1469c53048`.	2026-08-31 01:24:55 +04:00
nmzik	2a54306e8b	agc: implement async command buffer jump	2026-08-30 23:55:05 +04:00
nmzik	7a6aaafe52	agc: expose async copy and wait packet sizes	2026-08-30 23:49:08 +04:00
nmzik	1469c53048	fix(agc): honor top bound for single-ended buffers	2026-08-30 23:39:18 +04:00
GodsWill Dev	9992ab18fc	shader: preserve untouched half in VOP1 B16 writes (#370 ) shader: preserve native 16-bit destinations Co-authored-by: nmzik <Nmzik@mail.ru>	2026-08-30 19:33:26 +04:00
Brandon Strong	18af08460f	shader: take the high half per lane in V_PK_FMAC_F16 (#404 ) Co-authored-by: nmzik <Nmzik@mail.ru>	2026-08-30 15:08:22 +04:00
nmzik	39ee9972b8	shader: V_CMPX_CLASS_F32	2026-08-30 14:32:07 +04:00
YuhiAida	e500333bcd	fix(agc): implement AgcDcbCopyDataGetSize (#410 ) Register b5u0Jzm8TF8 for SpongeBob PPSA26893 using the native zero-argument, uint64_t size-helper ABI. Adapted from PR #410.	2026-08-30 13:59:16 +04:00
nmzik	4498d75b9b	add agc ver 13	2026-08-30 13:29:49 +04:00
Brandon Strong	1df53b169f	kernel: stop sign-extending batch-map protection and type (#405 )	2026-08-30 01:46:43 +04:00