shader: implement RDNA2 subrev borrow

src/graphics/shader/recompiler/frontend/decode/ShaderDecoder.cpp

@@ -671,6 +671,7 @@ std::string OpcodeToString(Opcode opcode) {
 		case Opcode::VMadMixhiF16: return "v_mad_mixhi_f16";
 		case Opcode::VAddNcU32: return "v_add_nc_u32";
 		case Opcode::VAddcU32: return "v_addc_u32";
+		case Opcode::VSubrevCoCiU32: return "v_subrev_co_ci_u32";
 		case Opcode::VSubNcU32: return "v_sub_nc_u32";
 		case Opcode::VSubrevNcU32: return "v_subrev_nc_u32";
 		case Opcode::VAddNcU16: return "v_add_nc_u16";

src/graphics/shader/recompiler/frontend/decode/ShaderDecoder.h

@@ -269,6 +269,7 @@ enum class Opcode {
 	VMadMixhiF16,
 	VAddNcU32,
 	VAddcU32,
+	VSubrevCoCiU32,
 	VSubNcU32,
 	VSubrevNcU32,
 	VAddNcU16,

src/graphics/shader/recompiler/frontend/decode/VectorAluOps.cpp

@@ -63,6 +63,7 @@ constexpr Vop2OpcodeInfo VOP2_OPS[] = {
     {0x24u, Opcode::VMbcntHiU32B32},
     {0x25u, Opcode::VAddNcU32, Vop2SdwaProfile::IntegerFullDestination},
     {0x28u, Opcode::VAddcU32},
+    {0x2au, Opcode::VSubrevCoCiU32},
     {0x26u, Opcode::VSubNcU32, Vop2SdwaProfile::IntegerPartialDestination},
     {0x27u, Opcode::VSubrevNcU32, Vop2SdwaProfile::IntegerFullDestination},
     {0x2bu, Opcode::VMacF32},
@@ -967,6 +968,7 @@ bool FinalizeVop2Instruction(uint32_t pc, std::span<const uint32_t> code, uint32
 			inst.src_count = 3;
 			break;
 		case Opcode::VAddcU32:
+		case Opcode::VSubrevCoCiU32:
 			inst.dst2.kind = OperandKind::VccLo;
 			inst.src2.kind = OperandKind::VccLo;
 			inst.src_count = 3;
@@ -1373,9 +1375,10 @@ bool SupportsNativeVop3ResultModifiers(Opcode opcode) {
 	}
 }
 
-bool HasUnsupportedNativeVop3Modifiers(Opcode opcode, bool permlane, bool mad_mix, bool addc,
+bool HasUnsupportedNativeVop3Modifiers(Opcode opcode, bool permlane, bool mad_mix,
-                                       bool scalar_dst, uint32_t abs, uint32_t op_sel,
+                                       bool carry_in_out, bool scalar_dst, uint32_t abs,
-                                       uint32_t clamp, uint32_t omod, uint32_t neg) {
+                                       uint32_t op_sel, uint32_t clamp, uint32_t omod,
+                                       uint32_t neg) {
 	const bool source_modifiers = SupportsNativeVop3SourceModifiers(opcode);
 	const bool result_modifiers = SupportsNativeVop3ResultModifiers(opcode);
 
@@ -1391,7 +1394,7 @@ bool HasUnsupportedNativeVop3Modifiers(Opcode opcode, bool permlane, bool mad_mi
 	if (IsNativeVop3B16BinaryOpcode(opcode)) {
 		return abs != 0u || clamp != 0u || omod != 0u || neg != 0u;
 	}
-	if (addc || scalar_dst) {
+	if (carry_in_out || scalar_dst) {
 		return clamp != 0u || omod != 0u || neg != 0u;
 	}
 	switch (opcode) {
@@ -1606,19 +1609,20 @@ bool DecodeVop3(uint32_t pc, std::span<const uint32_t> code, uint32_t word_index
 	inst.opcode    = LookupVop3Opcode(opcode);
 	SetRawWords(inst, code, word_index, 2);
 
-	const bool addc                    = inst.opcode == Opcode::VAddcU32 && opcode == 0x128u;
+	const bool carry_in_out           = (inst.opcode == Opcode::VAddcU32 && opcode == 0x128u) ||
-	const bool vop3b_carry_out         = IsVop3BCarryOutOpcode(inst.opcode);
+	                                    (inst.opcode == Opcode::VSubrevCoCiU32 && opcode == 0x12au);
-	const bool vop3b_mad_u64           = IsVop3BMadU64Opcode(inst.opcode);
+	const bool vop3b_carry_out        = IsVop3BCarryOutOpcode(inst.opcode);
-	const bool vop3b_uses_sdst         = addc || vop3b_carry_out || vop3b_mad_u64;
+	const bool vop3b_mad_u64          = IsVop3BMadU64Opcode(inst.opcode);
-	const bool mad_mix                 = false;
+	const bool vop3b_uses_sdst        = carry_in_out || vop3b_carry_out || vop3b_mad_u64;
-	const bool f16_ternary             = IsNativeVop3F16TernaryOpcode(inst.opcode);
+	const bool mad_mix                = false;
-	const bool b16_binary              = IsNativeVop3B16BinaryOpcode(inst.opcode);
+	const bool f16_ternary            = IsNativeVop3F16TernaryOpcode(inst.opcode);
-	const bool pack_b32_f16            = inst.opcode == Opcode::VPackB32F16;
+	const bool b16_binary             = IsNativeVop3B16BinaryOpcode(inst.opcode);
-	const bool permlane                = IsPermlaneOpcode(inst.opcode);
+	const bool pack_b32_f16           = inst.opcode == Opcode::VPackB32F16;
-	const bool vop3_vopc               = IsVop3EncodedVopc(opcode);
+	const bool permlane               = IsPermlaneOpcode(inst.opcode);
-	const bool compare_exec            = vop3_vopc && IsVopcCompareExec(inst.opcode);
+	const bool vop3_vopc              = IsVop3EncodedVopc(opcode);
-	const bool scalar_dst              = vop3_vopc || UsesScalarDestination(inst.opcode);
+	const bool compare_exec           = vop3_vopc && IsVopcCompareExec(inst.opcode);
-	const bool scalar_modifier_limits  = UsesScalarDestination(inst.opcode);
+	const bool scalar_dst             = vop3_vopc || UsesScalarDestination(inst.opcode);
+	const bool scalar_modifier_limits = UsesScalarDestination(inst.opcode);
 	const bool native_source_modifiers = SupportsNativeVop3SourceModifiers(inst.opcode);
 	const bool native_result_modifiers = SupportsNativeVop3ResultModifiers(inst.opcode);
 	const bool modifiers =
@@ -1678,7 +1682,7 @@ bool DecodeVop3(uint32_t pc, std::span<const uint32_t> code, uint32_t word_index
 		}
 		return ReadLiteralOperands(code, word_index, inst, error);
 	}
-	if (addc) {
+	if (carry_in_out) {
 		if (!DecodeScalarDestination(sdst, pc, inst.dst2, error) ||
 		    !DecodeScalarSource(src1, pc, inst.src1, error) ||
 		    !DecodeScalarSource(src2, pc, inst.src2, error)) {

src/graphics/shader/recompiler/frontend/translate/Control.cpp

@@ -74,7 +74,8 @@ bool Translator::TranslateStateOperation(const IR::Instruction& inst) {
 			}
 			return true;
 		}
-		case IR::Opcode::ScalarSubBorrowCarryU32: {
+		case IR::Opcode::ScalarSubBorrowCarryU32:
+		case IR::Opcode::ISubBorrowCarryU32: {
 			const auto lhs       = ReadU32(inst.src[0]);
 			const auto rhs       = ReadU32(inst.src[1]);
 			const auto borrow_in = ConditionBit(inst.src[2]);
@@ -83,7 +84,12 @@ bool Translator::TranslateStateOperation(const IR::Instruction& inst) {
 			const auto borrow0   = ir.UGreaterThan(rhs, lhs);
 			const auto borrow1   = ir.UGreaterThan(borrow_in, partial);
 			WriteOperand(inst.dst, result);
-			ir.SetScc(ir.LogicalOr(borrow0, borrow1));
+			const auto borrow = ir.LogicalOr(borrow0, borrow1);
+			if (inst.op == IR::Opcode::ScalarSubBorrowCarryU32) {
+				ir.SetScc(borrow);
+			} else {
+				WriteMask(inst.dst2, borrow);
+			}
 			return true;
 		}
 		case IR::Opcode::ScalarSignedAddOverflowI32:

src/graphics/shader/recompiler/frontend/translate/Translate.cpp

@@ -585,7 +585,7 @@ IR::U32 Translator::ConditionBit(const IR::Operand& operand) {
 	if (operand.kind == IR::OperandKind::ImmediateU32) {
 		return IR::U32(IR::Value(operand.imm & 1u));
 	}
-	return ir.Select(ReadCondition(operand), IR::U32(IR::Value(1u)), IR::U32(IR::Value(0u)));
+	return ir.Select(ReadMask(operand), IR::U32(IR::Value(1u)), IR::U32(IR::Value(0u)));
 }
 
 IR::U1 Translator::ReadMask(const IR::Operand& operand) {

src/graphics/shader/recompiler/ir/ShaderIR.cpp

@@ -741,6 +741,23 @@ bool LowerVectorAddCarry(const Decoder::Instruction& decoded, BasicBlock& block,
 	return true;
 }
 
+bool LowerVectorSubrevBorrowCarry(const Decoder::Instruction& decoded, BasicBlock& block,
+                                  std::string* error) {
+	Instruction inst;
+	inst.pc        = decoded.pc;
+	inst.op        = Opcode::ISubBorrowCarryU32;
+	inst.src_count = 3;
+	if (!LowerRegisterOperand(decoded.dst, inst.dst, error) ||
+	    !LowerRegisterOperand(decoded.dst2, inst.dst2, error) ||
+	    !LowerSourceOperand(decoded.src1, inst.src[0], error) ||
+	    !LowerSourceOperand(decoded.src0, inst.src[1], error) ||
+	    !LowerSourceOperand(decoded.src2, inst.src[2], error)) {
+		return false;
+	}
+	block.instructions.push_back(inst);
+	return true;
+}
+
 bool LowerVectorCarryOut(const Decoder::Instruction& decoded, BasicBlock& block,
                          std::string* error) {
 	Instruction inst;
@@ -1124,6 +1141,8 @@ bool LowerDecodedInstruction(const Decoder::Instruction& inst, BasicBlock& block
 		case Decoder::Opcode::VMovreldB32: return LowerVectorMoveRelDestination(inst, block, error);
 		case Decoder::Opcode::VMovrelsB32: return LowerVectorMoveRelSource(inst, block, error);
 		case Decoder::Opcode::VAddcU32: return LowerVectorAddCarry(inst, block, error);
+		case Decoder::Opcode::VSubrevCoCiU32:
+			return LowerVectorSubrevBorrowCarry(inst, block, error);
 		case Decoder::Opcode::VMadU64U32: return LowerVectorMadU64U32(inst, block, error);
 		case Decoder::Opcode::VMacF32: return LowerVectorMacF32(inst, block, error);
 		case Decoder::Opcode::VPkFmacF16: return LowerVectorPkFmacF16(inst, block, error);

src/graphics/shader/recompiler/ir/opcodes/ShaderIROpcodes.inc

@@ -22,6 +22,7 @@ IR_OPCODE(Permlanex16B32, General, Lane)
 IR_OPCODE(AbsI32, General, SimpleInteger)
 IR_OPCODE(IAddU32, General, SimpleInteger)
 IR_OPCODE(IAddCarryU32, General, State)
+IR_OPCODE(ISubBorrowCarryU32, General, State)
 IR_OPCODE(ISubBorrowU32, General, State)
 IR_OPCODE(ScalarAddCarryU32, General, State)
 IR_OPCODE(ScalarSubBorrowU32, General, State)

tests/ShaderRecompilerComputeTests.cpp

@@ -11694,6 +11694,79 @@ TestCase VectorAddcWritesPerLaneCarryOut() {
   return test;
 }
 
+TestCase VectorSubrevCoCiU32ExactRawOnGpu() {
+  using O = ShaderOpcode;
+
+  std::vector<u32> code;
+  code.push_back(EncodeVop1(0x01, 24, Vgpr(0)));
+  code.push_back(EncodeVop1(0x01, 1, InlineU32(2)));
+  code.push_back(EncodeVop1(0x01, 2, InlineU32(1)));
+  code.push_back(EncodeVop2(0x1a, 4, InlineU32(2), 0));
+  code.push_back(EncodeVop2(0x25, 5, InlineU32(16), 4));
+  code.push_back(EncodeVopc(0xc1, Vgpr(0), 1));
+  code.push_back(0x54003080u); // v_subrev_co_ci_u32 v0, 0, v24, vcc
+  code.push_back(EncodeVop2(0x01, 3, InlineU32(0), 2));
+  AppendBufferStoreDword(&code, 0, 4);
+  AppendBufferStoreDword(&code, 3, 5);
+  AppendEnd(&code);
+
+  TestCase test;
+  test.name = "VectorSubrevCoCiU32ExactRawOnGpu";
+  test.code = code;
+  test.expected = {0xffffffffu, 0, 2, 3, 1, 0, 0, 0};
+  test.opcodes = {O::VMovB32,          O::VLshlrevB32,    O::VAddNcU32,
+                  O::VCmpLtU32,        O::VSubrevCoCiU32, O::VCndmaskB32,
+                  O::BufferStoreDword, O::SEndpgm};
+  test.required_spirv = {"OpISub", "OpUGreaterThan"};
+  test.compute_info.threads_num[0] = 4;
+  test.compute_info.threads_num[1] = 1;
+  test.compute_info.threads_num[2] = 1;
+  test.compute_info.group_id[0] = true;
+  test.compute_info.wave_size = 32;
+  test.compute_info.thread_ids_num = 1;
+  test.compute_info.workgroup_register = 0;
+  test.has_compute_info = true;
+  return test;
+}
+
+TestCase VectorVop3BSubrevCoCiUsesEncodedMasks() {
+  using O = ShaderOpcode;
+
+  std::vector<u32> code;
+  code.push_back(EncodeVop1(0x01, 24, Vgpr(0)));
+  code.push_back(EncodeVop1(0x01, 1, InlineU32(2)));
+  code.push_back(EncodeVop1(0x01, 2, InlineU32(1)));
+  code.push_back(EncodeVop2(0x1a, 4, InlineU32(2), 0));
+  code.push_back(EncodeVop2(0x25, 5, InlineU32(16), 4));
+  code.push_back(EncodeVopc(0xc1, Vgpr(0), 1));
+  code.push_back(EncodeSop1(0x04, 20, 106)); // s_mov_b64 s[20:21], vcc
+  AppendSMovLiteral(&code, 106, 0);
+  AppendSMovLiteral(&code, 107, 0);
+  AppendVop3B(&code, 0x12au, 10, 22, InlineU32(0), Vgpr(24), 20);
+  AppendVop3(&code, 0x101u, 3, InlineU32(0), Vgpr(2), 22);
+  AppendBufferStoreDword(&code, 10, 4);
+  AppendBufferStoreDword(&code, 3, 5);
+  AppendEnd(&code);
+
+  TestCase test;
+  test.name = "VectorVop3BSubrevCoCiUsesEncodedMasks";
+  test.code = code;
+  test.expected = {0xffffffffu, 0, 2, 3, 1, 0, 0, 0};
+  test.opcodes = {O::VMovB32,        O::VLshlrevB32, O::VAddNcU32,
+                  O::VCmpLtU32,      O::SMovB64,     O::SMovB32,
+                  O::VSubrevCoCiU32, O::VCndmaskB32, O::BufferStoreDword,
+                  O::SEndpgm};
+  test.compute_info.threads_num[0] = 4;
+  test.compute_info.threads_num[1] = 1;
+  test.compute_info.threads_num[2] = 1;
+  test.compute_info.group_id[0] = true;
+  test.compute_info.wave_size = 32;
+  test.compute_info.thread_ids_num = 1;
+  test.compute_info.workgroup_register = 0;
+  test.has_compute_info = true;
+  return test;
+}
+
 TestCase VectorVop3BCarryOutWritesSgprMask() {
   using O = ShaderOpcode;
 
@@ -16800,6 +16873,8 @@ std::vector<TestCase> MakeCases() {
   AddCase(VectorMbcntUsesThreadMask);
   AddCase(VectorAddcWritesPerLaneCarryOut);
   AddCase(VectorAddcUsesPerLaneCarryIn);
+  AddCase(VectorSubrevCoCiU32ExactRawOnGpu);
+  AddCase(VectorVop3BSubrevCoCiUsesEncodedMasks);
   AddCase(VectorVop3BCarryOutWritesSgprMask);
   AddCase(VectorVop3BCarryOutUsesEncodedSdst);
   AddCase(VectorVop3BSubCoU32UsesRdna2Opcode310);