shader: take the high half per lane in V_PK_FMAC_F16 (#404)

Co-authored-by: nmzik <Nmzik@mail.ru>

src/graphics/shader/recompiler/frontend/decode/ShaderDecoder.h

@@ -351,6 +351,7 @@ enum class Opcode {
 	V_CMP_GE_I32,
 	V_CMP_T_I32,
 	V_CMP_CLASS_F32,
+	V_CMPX_CLASS_F32,
 	V_CMP_LT_I16,
 	V_CMP_EQ_I16,
 	V_CMP_LE_I16,

src/graphics/shader/recompiler/frontend/decode/VectorAluOps.cpp

@@ -227,7 +227,8 @@ constexpr VopcOpcodeInfo VOPC_OPCODE_LIST[] = {
     {0x8eu, Opcode::V_CMP_GE_I16},        {0x91u, Opcode::V_CMPX_LT_I32},
     {0x92u, Opcode::V_CMPX_EQ_I32},       {0x93u, Opcode::V_CMPX_LE_I32},
     {0x94u, Opcode::V_CMPX_GT_I32},       {0x95u, Opcode::V_CMPX_NE_I32},
-    {0x96u, Opcode::V_CMPX_GE_I32},       {0xa9u, Opcode::V_CMP_LT_U16},
+    {0x96u, Opcode::V_CMPX_GE_I32},       {0x98u, Opcode::V_CMPX_CLASS_F32},
+    {0xa9u, Opcode::V_CMP_LT_U16},
     {0xaau, Opcode::V_CMP_EQ_U16},        {0xabu, Opcode::V_CMP_LE_U16},
     {0xacu, Opcode::V_CMP_GT_U16},        {0xadu, Opcode::V_CMP_NE_U16},
     {0xaeu, Opcode::V_CMP_GE_U16},        {0xbcu, Opcode::V_CMPX_GT_U16},
@@ -895,7 +896,8 @@ bool IsVopcFloatCompareOpcode(Opcode opcode) {
 		case Opcode::V_CMPX_NGT_F16:
 		case Opcode::V_CMPX_NEQ_F16:
 		case Opcode::V_CMPX_NLT_F16:
-		case Opcode::V_CMP_CLASS_F32: return true;
+		case Opcode::V_CMP_CLASS_F32:
+		case Opcode::V_CMPX_CLASS_F32: return true;
 		default: return false;
 	}
 }
@@ -1057,6 +1059,13 @@ bool FinalizeVop2Instruction(uint32_t pc, std::span<const uint32_t> code, uint32
 			inst.src2.kind = OperandKind::VccLo;
 			inst.src_count = 3;
 			break;
+		case Opcode::V_PK_FMAC_F16:
+			// VOP2 has no OP_SEL fields, so the high operation implicitly reads high halves.
+			inst.src0.op_sel_hi = true;
+			inst.src1.op_sel_hi = true;
+			inst.dst.op_sel_hi  = true;
+			inst.src_count      = 2;
+			break;
 		default: inst.src_count = 2; break;
 	}
 	ReadLiteralOperands(code, word_index, inst);
@@ -1105,12 +1114,17 @@ bool DecodeVop2Dpp(uint32_t pc, std::span<const uint32_t> code, uint32_t word_in
 	if (!DecodeScalarSource(src0 + 256u, pc, inst.src0, error)) {
 		return false;
 	}
-	ApplyDefaultVop2F16Destination(inst);
 	ApplyDppModifier(inst.src0, modifier);
 	inst.src1.negate   = ((modifier >> 22u) & 0x1u) != 0u;
 	inst.src1.absolute = ((modifier >> 23u) & 0x1u) != 0u;
+	ApplyDefaultVop2F16Destination(inst);
+	const bool packed_fmac = inst.opcode == Opcode::V_PK_FMAC_F16;
+	if (packed_fmac) {
+		inst.src0.negate_hi = inst.src0.negate;
+		inst.src1.negate_hi = inst.src1.negate;
+	}
 
-	if (!IsVop2FloatSourceOpcode(inst.opcode) &&
+	if (!IsVop2FloatSourceOpcode(inst.opcode) && !packed_fmac &&
 	    (inst.src0.negate || inst.src0.absolute || inst.src1.negate || inst.src1.absolute)) {
 		SetUnsupported(inst, Family::VOP2, opcode,
 		               "VOP2 DPP integer source modifiers are not supported");
@@ -1518,6 +1532,7 @@ bool IsVopcCompareExec(Opcode opcode) {
 		case Opcode::V_CMPX_GT_I32:
 		case Opcode::V_CMPX_NE_I32:
 		case Opcode::V_CMPX_GE_I32:
+		case Opcode::V_CMPX_CLASS_F32:
 		case Opcode::V_CMPX_LT_U32:
 		case Opcode::V_CMPX_EQ_U32:
 		case Opcode::V_CMPX_LE_U32:

src/graphics/shader/recompiler/frontend/translate/Compare.cpp

@@ -55,11 +55,11 @@ void Translator::EmitFloatOrderedCompare(const Decoder::Instruction& inst, bool
 	EmitCompareResult(inst, ordered ? ir.LogicalNot(unordered) : unordered, false, false);
 }
 
-void Translator::V_CMP_CLASS_F32(const Decoder::Instruction& inst) {
+void Translator::EmitFloatClassCompare(const Decoder::Instruction& inst, bool cmpx) {
 	const auto value = ReadOperand(inst.src0, IR::Type::F32);
 	const auto mask  = ReadOperand(inst.src1, IR::Type::U32);
 	EmitCompareResult(inst, IR::U1(ir.Emit(IR::ValueOpcode::FPCmpClass32, {value, mask})), false,
-	                  false);
+	                  cmpx);
 }
 
 } // namespace Libs::Graphics::ShaderRecompiler::Frontend

src/graphics/shader/recompiler/frontend/translate/Translator.h

@@ -127,7 +127,7 @@ private:
 	void EmitFloatCompare(const Decoder::Instruction& inst, IR::ValueOpcode opcode, bool half,
 	                      bool cmpx);
 	void EmitFloatOrderedCompare(const Decoder::Instruction& inst, bool ordered);
-	void V_CMP_CLASS_F32(const Decoder::Instruction& inst);
+	void EmitFloatClassCompare(const Decoder::Instruction& inst, bool cmpx);
 	void V_CVT_F32_UBYTE(const Decoder::Instruction& inst, uint32_t byte_index);
 	void V_CVT_F32_U32(const Decoder::Instruction& inst);
 	void V_CVT_F32_I32(const Decoder::Instruction& inst);

src/graphics/shader/recompiler/frontend/translate/Vector.cpp

@@ -271,7 +271,8 @@ bool Translator::EmitVector(const Decoder::Instruction& inst, std::string* error
 			return true;
 		case O::V_CMP_O_F32: EmitFloatOrderedCompare(inst, true); return true;
 		case O::V_CMP_U_F32: EmitFloatOrderedCompare(inst, false); return true;
-		case O::V_CMP_CLASS_F32: V_CMP_CLASS_F32(inst); return true;
+		case O::V_CMP_CLASS_F32: EmitFloatClassCompare(inst, false); return true;
+		case O::V_CMPX_CLASS_F32: EmitFloatClassCompare(inst, true); return true;
 
 		case O::V_CVT_F32_UBYTE0: V_CVT_F32_UBYTE(inst, 0); return true;
 		case O::V_CVT_F32_UBYTE1: V_CVT_F32_UBYTE(inst, 1); return true;

tests/ShaderRecompilerComputeTests.cpp

@@ -11931,6 +11931,7 @@ CoverageClass ClassifyOpcode(ShaderOpcode opcode,
   case Opcode::V_CMPX_NEQ_F32:
   case Opcode::V_CMPX_NLT_F32:
   case Opcode::V_CMP_CLASS_F32:
+  case Opcode::V_CMPX_CLASS_F32:
   case Opcode::V_CMP_LT_F16:
   case Opcode::V_CMP_EQ_F16:
   case Opcode::V_CMP_LE_F16:
@@ -14249,6 +14250,53 @@ TestCase VectorLaneAndPackedOps() {
            O::BUFFER_STORE_DWORD, O::S_ENDPGM}};
 }
 
+TestCase Vop2PkFmacF16AccumulatesPackedHalvesIndependently() {
+  using O = ShaderOpcode;
+
+  std::vector<u32> code;
+  AppendVMovLiteral(&code, 0, 0x50004c00u); // low=16.0h, high=32.0h
+  AppendVMovLiteral(&code, 1, 0x42003c00u); // low=1.0h, high=3.0h
+  AppendVMovLiteral(&code, 2, 0x48004400u); // low=4.0h, high=8.0h
+
+  code.push_back(EncodeVop2(0x3c, 0, Vgpr(1), 2));
+  AppendStoreVgpr(&code, 0, 0);
+  code.push_back(EncodeVop2(0x3c, 0, Vgpr(1), 2));
+  AppendStoreVgpr(&code, 0, 1);
+
+  AppendVMovLiteral(&code, 3, 0x50004c00u);
+  code.push_back(EncodeVop2(0x3c, 3, 242, 2)); // Inline 1.0h is {1.0h, 0.0h}.
+  AppendStoreVgpr(&code, 3, 2);
+  AppendEnd(&code);
+
+  return {"Vop2PkFmacF16AccumulatesPackedHalvesIndependently",
+          code,
+          {},
+          {0x53004d00u, 0x55004e00u, 0x50004d00u},
+          {O::V_MOV_B32, O::V_PK_FMAC_F16, O::BUFFER_STORE_DWORD,
+           O::S_ENDPGM}};
+}
+
+TestCase Vop2PkFmacF16DppNegatesBothPackedHalves() {
+  using O = ShaderOpcode;
+
+  std::vector<u32> code;
+  AppendVMovLiteral(&code, 0, 0x50004c00u); // low=16.0h, high=32.0h
+  AppendVMovLiteral(&code, 1, 0x42003c00u); // low=1.0h, high=3.0h
+  AppendVMovLiteral(&code, 2, 0x48004400u); // low=4.0h, high=8.0h
+
+  code.push_back(EncodeVop2(0x3c, 0, 250, 2));
+  code.push_back(EncodeVop2Dpp(1, 0, 0xf, 0xf, 1));
+  AppendStoreVgpr(&code, 0, 0);
+  AppendEnd(&code);
+
+  return {"Vop2PkFmacF16DppNegatesBothPackedHalves",
+          code,
+          {},
+          {0x48004a00u},
+          {O::V_MOV_B32, O::V_PK_FMAC_F16, O::BUFFER_STORE_DWORD,
+           O::S_ENDPGM}};
+}
+
 TestCase Vop3pOpselHiUsesArchitecturalSourceBits() {
   using O = ShaderOpcode;
 
@@ -15785,6 +15833,38 @@ TestCase VectorCompareClassF32() {
            O::BUFFER_STORE_DWORD, O::S_ENDPGM}};
 }
 
+TestCase VectorVopcSdwaCmpxClassF32CapturedExecMask() {
+  using O = ShaderOpcode;
+
+  std::vector<u32> code;
+  AppendVMovLiteral(&code, 13, 0x7fc00000u); // Quiet NaN.
+  AppendVMovU32(&code, 3, 7);
+  AppendVMovU32(&code, 31, 0);
+  code.push_back(EncodeSMovB32(106, InlineU32(2))); // qNaN class mask.
+  code.insert(code.end(), {0x7d30d4f9u, 0x8606000du});
+  AppendBufferStoreDword(&code, 3, 31);
+
+  code.push_back(EncodeSMovB32(126, InlineU32(1))); // Restore lane-zero EXEC.
+  AppendVMovLiteral(&code, 13, 0x3f800000u);        // Positive normal.
+  AppendVMovU32(&code, 3, 9);
+  AppendVMovU32(&code, 31, 4);
+  code.push_back(EncodeSMovB32(106, InlineU32(2))); // qNaN class mask.
+  code.insert(code.end(), {0x7d30d4f9u, 0x8606000du});
+  AppendBufferStoreDword(&code, 3, 31);
+  AppendEnd(&code);
+
+  TestCase test{
+      "VectorVopcSdwaCmpxClassF32CapturedExecMask",
+      code,
+      {0x11111111u, 0x22222222u},
+      {7u, 0x22222222u},
+      {O::S_MOV_B32, O::V_MOV_B32, O::V_CMPX_CLASS_F32,
+       O::BUFFER_STORE_DWORD, O::S_ENDPGM}};
+  test.decoded_counts = {
+      {"V_CMPX_CLASS_F32 exec_lo, v13, vcc_lo", 2}};
+  return test;
+}
+
 TestCase VectorCompareF16Ops() {
   using O = ShaderOpcode;
 
@@ -20767,6 +20847,8 @@ std::vector<TestCase> MakeCases() {
   AddCase(VectorVop3BSubCoU32UsesRdna2Opcode310);
   AddCase(VectorMadU64U32UnsignedCarryOut);
   AddCase(VectorLaneAndPackedOps);
+  AddCase(Vop2PkFmacF16AccumulatesPackedHalvesIndependently);
+  AddCase(Vop2PkFmacF16DppNegatesBothPackedHalves);
   AddCase(Vop3pOpselHiUsesArchitecturalSourceBits);
   AddCase(CvtPkU8F32PacksSelectedByte);
   AddCase(CvtPkrtzF16F32SubnormalRoundsTowardZero);
@@ -20812,6 +20894,7 @@ std::vector<TestCase> MakeCases() {
   AddCase(VectorVopcCmpxNeU64CapturedExecMask);
   AddCase(VectorVop3CmpxNeI64CapturedExecMask);
   AddCase(VectorCompareClassF32);
+  AddCase(VectorVopcSdwaCmpxClassF32CapturedExecMask);
   AddCase(VectorCompareF16Ops);
   AddCase(Vop2SdwaCndmaskSourceModifier);
   AddCase(Vop2SdwaCndmaskCapturedByte3Source);

tests/shaderCfgTests.cpp

@@ -4203,6 +4203,17 @@ void TestNewShaderDecoderArchitecture() {
               packed_inst.src2.op_sel_hi == (source == 2u),
           "VOP3P OPSEL_HI source bit mapping is incorrect");
   }
+
+  const uint32_t packed_fmac_dpp[] = {0x780402fau, 0xff500000u};
+  Instruction packed_fmac;
+  Check(DecodeInstruction(packed_fmac_dpp, 0u, packed_fmac, &error),
+        error.c_str());
+  Check(packed_fmac.opcode == Opcode::V_PK_FMAC_F16 &&
+            packed_fmac.src0.dpp && packed_fmac.src0.op_sel_hi &&
+            packed_fmac.src1.op_sel_hi && packed_fmac.dst.op_sel_hi &&
+            packed_fmac.src0.negate && packed_fmac.src0.negate_hi &&
+            packed_fmac.src1.negate && packed_fmac.src1.negate_hi,
+        "VOP2 DPP V_PK_FMAC_F16 lost its implicit packed modifiers");
 }
 
 void TestNewShaderRecompilerRejectsDppOn64BitCompares() {
@@ -4231,6 +4242,67 @@ void TestNewShaderRecompilerRejectsDppOn64BitCompares() {
   }
 }
 
+void TestNewShaderRecompilerCapturedVopcSdwaCmpxClass() {
+  using namespace ShaderRecompiler;
+
+  const uint32_t shader[] = {
+      0x7d30d4f9u, 0x8606000du, // v_cmpx_class_f32 exec, v13, vcc_lo (SDWA)
+      EncodeSopp(0x01),
+  };
+
+  Decoder::Instruction decoded;
+  std::string error;
+  Check(Decoder::DecodeInstruction(shader, 0u, decoded, &error),
+        error.c_str());
+  Check(decoded.family == Decoder::Family::VOPC &&
+            decoded.opcode == Decoder::Opcode::V_CMPX_CLASS_F32 &&
+            decoded.opcode_id == 0x98u && decoded.word_count == 2u &&
+            decoded.raw_count == 2u &&
+            decoded.dst.kind == Decoder::OperandKind::ExecLo &&
+            decoded.src_count == 2u &&
+            decoded.src0.kind == Decoder::OperandKind::Vgpr &&
+            decoded.src0.reg == 13u && decoded.src0.sdwa_sel == 6u &&
+            decoded.src1.kind == Decoder::OperandKind::VccLo &&
+            decoded.src1.sdwa_sel == 6u,
+        "decoder rejected captured SDWA V_CMPX_CLASS_F32 fields");
+
+  Decoder::Program program;
+  CFG::Graph graph;
+  IR::Program ir;
+  ShaderPixelInputInfo pixel{};
+  Frontend::TranslateOptions translate_options{};
+  translate_options.stage = ShaderType::Pixel;
+  translate_options.wave_size = 64u;
+  translate_options.pixel = &pixel;
+  Check(Decoder::DecodeProgram(shader, program, &error) &&
+            CFG::BuildGraph(program, graph, &error) &&
+            Frontend::TranslateProgram(program, graph, translate_options, ir,
+                                       &error),
+        error.c_str());
+  uint32_t class_compares = 0u;
+  uint32_t dynamic_exec_writes = 0u;
+  for (const auto *block : ir.blocks) {
+    for (const auto &inst : *block) {
+      class_compares +=
+          inst.GetOpcode() == IR::ValueOpcode::FPCmpClass32 ? 1u : 0u;
+      if (inst.GetOpcode() == IR::ValueOpcode::SetExec &&
+          !inst.Arg(0).Resolve().IsImmediate()) {
+        dynamic_exec_writes++;
+      }
+    }
+  }
+  Check(class_compares == 1u && dynamic_exec_writes == 1u,
+        "captured V_CMPX_CLASS_F32 did not lower to class compare plus EXEC update");
+
+  auto options = MakeCompileOptions(ShaderType::Pixel);
+  CompileResult result;
+  Check(TryRecompile(shader, options, result, &error), error.c_str());
+  Check(Common::ContainsStr(result.decoded_dump,
+                            "V_CMPX_CLASS_F32 exec_lo, v13, vcc_lo"),
+        "captured SDWA V_CMPX_CLASS_F32 was not present in the decoded dump");
+  CheckSpirvBinaryValidates(result.spirv);
+}
+
 void TestNewShaderRecompilerIrLookupMissFailsExplicitly() {
   ShaderRecompiler::Decoder::Program decoded;
   ShaderRecompiler::Decoder::Instruction missing;
@@ -12476,6 +12548,7 @@ int main() {
   // ShaderRecompilerComputeTests; keep the distinct decoder contract checks
   // here.
   TestNewShaderDecoderArchitecture();
+  TestNewShaderRecompilerCapturedVopcSdwaCmpxClass();
   TestNewShaderRecompilerRejectsDppOn64BitCompares();
   TestNewShaderRecompilerIrLookupMissFailsExplicitly();
   TestPsInputCountRegisterDecode();