From 4a7bc55afabc6c4d612f5bae9d47bee5eddbd281 Mon Sep 17 00:00:00 2001 From: nmzik Date: Sat, 15 Aug 2026 04:47:25 +0200 Subject: [PATCH] shader: implement RDNA2 subrev borrow --- .../frontend/decode/ShaderDecoder.cpp | 1 + .../frontend/decode/ShaderDecoder.h | 1 + .../frontend/decode/VectorAluOps.cpp | 40 +++++----- .../recompiler/frontend/translate/Control.cpp | 10 ++- .../frontend/translate/Translate.cpp | 2 +- .../shader/recompiler/ir/ShaderIR.cpp | 19 +++++ .../recompiler/ir/opcodes/ShaderIROpcodes.inc | 1 + tests/ShaderRecompilerComputeTests.cpp | 75 +++++++++++++++++++ 8 files changed, 128 insertions(+), 21 deletions(-) diff --git a/src/graphics/shader/recompiler/frontend/decode/ShaderDecoder.cpp b/src/graphics/shader/recompiler/frontend/decode/ShaderDecoder.cpp index dd961f693..e2aa67bf0 100644 --- a/src/graphics/shader/recompiler/frontend/decode/ShaderDecoder.cpp +++ b/src/graphics/shader/recompiler/frontend/decode/ShaderDecoder.cpp @@ -671,6 +671,7 @@ std::string OpcodeToString(Opcode opcode) { case Opcode::VMadMixhiF16: return "v_mad_mixhi_f16"; case Opcode::VAddNcU32: return "v_add_nc_u32"; case Opcode::VAddcU32: return "v_addc_u32"; + case Opcode::VSubrevCoCiU32: return "v_subrev_co_ci_u32"; case Opcode::VSubNcU32: return "v_sub_nc_u32"; case Opcode::VSubrevNcU32: return "v_subrev_nc_u32"; case Opcode::VAddNcU16: return "v_add_nc_u16"; diff --git a/src/graphics/shader/recompiler/frontend/decode/ShaderDecoder.h b/src/graphics/shader/recompiler/frontend/decode/ShaderDecoder.h index c12a10602..d83adb239 100644 --- a/src/graphics/shader/recompiler/frontend/decode/ShaderDecoder.h +++ b/src/graphics/shader/recompiler/frontend/decode/ShaderDecoder.h @@ -269,6 +269,7 @@ enum class Opcode { VMadMixhiF16, VAddNcU32, VAddcU32, + VSubrevCoCiU32, VSubNcU32, VSubrevNcU32, VAddNcU16, diff --git a/src/graphics/shader/recompiler/frontend/decode/VectorAluOps.cpp b/src/graphics/shader/recompiler/frontend/decode/VectorAluOps.cpp index 63625f14f..3eb8bde0d 100644 --- a/src/graphics/shader/recompiler/frontend/decode/VectorAluOps.cpp +++ b/src/graphics/shader/recompiler/frontend/decode/VectorAluOps.cpp @@ -63,6 +63,7 @@ constexpr Vop2OpcodeInfo VOP2_OPS[] = { {0x24u, Opcode::VMbcntHiU32B32}, {0x25u, Opcode::VAddNcU32, Vop2SdwaProfile::IntegerFullDestination}, {0x28u, Opcode::VAddcU32}, + {0x2au, Opcode::VSubrevCoCiU32}, {0x26u, Opcode::VSubNcU32, Vop2SdwaProfile::IntegerPartialDestination}, {0x27u, Opcode::VSubrevNcU32, Vop2SdwaProfile::IntegerFullDestination}, {0x2bu, Opcode::VMacF32}, @@ -967,6 +968,7 @@ bool FinalizeVop2Instruction(uint32_t pc, std::span code, uint32 inst.src_count = 3; break; case Opcode::VAddcU32: + case Opcode::VSubrevCoCiU32: inst.dst2.kind = OperandKind::VccLo; inst.src2.kind = OperandKind::VccLo; inst.src_count = 3; @@ -1373,9 +1375,10 @@ bool SupportsNativeVop3ResultModifiers(Opcode opcode) { } } -bool HasUnsupportedNativeVop3Modifiers(Opcode opcode, bool permlane, bool mad_mix, bool addc, - bool scalar_dst, uint32_t abs, uint32_t op_sel, - uint32_t clamp, uint32_t omod, uint32_t neg) { +bool HasUnsupportedNativeVop3Modifiers(Opcode opcode, bool permlane, bool mad_mix, + bool carry_in_out, bool scalar_dst, uint32_t abs, + uint32_t op_sel, uint32_t clamp, uint32_t omod, + uint32_t neg) { const bool source_modifiers = SupportsNativeVop3SourceModifiers(opcode); const bool result_modifiers = SupportsNativeVop3ResultModifiers(opcode); @@ -1391,7 +1394,7 @@ bool HasUnsupportedNativeVop3Modifiers(Opcode opcode, bool permlane, bool mad_mi if (IsNativeVop3B16BinaryOpcode(opcode)) { return abs != 0u || clamp != 0u || omod != 0u || neg != 0u; } - if (addc || scalar_dst) { + if (carry_in_out || scalar_dst) { return clamp != 0u || omod != 0u || neg != 0u; } switch (opcode) { @@ -1606,19 +1609,20 @@ bool DecodeVop3(uint32_t pc, std::span code, uint32_t word_index inst.opcode = LookupVop3Opcode(opcode); SetRawWords(inst, code, word_index, 2); - const bool addc = inst.opcode == Opcode::VAddcU32 && opcode == 0x128u; - const bool vop3b_carry_out = IsVop3BCarryOutOpcode(inst.opcode); - const bool vop3b_mad_u64 = IsVop3BMadU64Opcode(inst.opcode); - const bool vop3b_uses_sdst = addc || vop3b_carry_out || vop3b_mad_u64; - const bool mad_mix = false; - const bool f16_ternary = IsNativeVop3F16TernaryOpcode(inst.opcode); - const bool b16_binary = IsNativeVop3B16BinaryOpcode(inst.opcode); - const bool pack_b32_f16 = inst.opcode == Opcode::VPackB32F16; - const bool permlane = IsPermlaneOpcode(inst.opcode); - const bool vop3_vopc = IsVop3EncodedVopc(opcode); - const bool compare_exec = vop3_vopc && IsVopcCompareExec(inst.opcode); - const bool scalar_dst = vop3_vopc || UsesScalarDestination(inst.opcode); - const bool scalar_modifier_limits = UsesScalarDestination(inst.opcode); + const bool carry_in_out = (inst.opcode == Opcode::VAddcU32 && opcode == 0x128u) || + (inst.opcode == Opcode::VSubrevCoCiU32 && opcode == 0x12au); + const bool vop3b_carry_out = IsVop3BCarryOutOpcode(inst.opcode); + const bool vop3b_mad_u64 = IsVop3BMadU64Opcode(inst.opcode); + const bool vop3b_uses_sdst = carry_in_out || vop3b_carry_out || vop3b_mad_u64; + const bool mad_mix = false; + const bool f16_ternary = IsNativeVop3F16TernaryOpcode(inst.opcode); + const bool b16_binary = IsNativeVop3B16BinaryOpcode(inst.opcode); + const bool pack_b32_f16 = inst.opcode == Opcode::VPackB32F16; + const bool permlane = IsPermlaneOpcode(inst.opcode); + const bool vop3_vopc = IsVop3EncodedVopc(opcode); + const bool compare_exec = vop3_vopc && IsVopcCompareExec(inst.opcode); + const bool scalar_dst = vop3_vopc || UsesScalarDestination(inst.opcode); + const bool scalar_modifier_limits = UsesScalarDestination(inst.opcode); const bool native_source_modifiers = SupportsNativeVop3SourceModifiers(inst.opcode); const bool native_result_modifiers = SupportsNativeVop3ResultModifiers(inst.opcode); const bool modifiers = @@ -1678,7 +1682,7 @@ bool DecodeVop3(uint32_t pc, std::span code, uint32_t word_index } return ReadLiteralOperands(code, word_index, inst, error); } - if (addc) { + if (carry_in_out) { if (!DecodeScalarDestination(sdst, pc, inst.dst2, error) || !DecodeScalarSource(src1, pc, inst.src1, error) || !DecodeScalarSource(src2, pc, inst.src2, error)) { diff --git a/src/graphics/shader/recompiler/frontend/translate/Control.cpp b/src/graphics/shader/recompiler/frontend/translate/Control.cpp index b168d344d..cf7d61283 100644 --- a/src/graphics/shader/recompiler/frontend/translate/Control.cpp +++ b/src/graphics/shader/recompiler/frontend/translate/Control.cpp @@ -74,7 +74,8 @@ bool Translator::TranslateStateOperation(const IR::Instruction& inst) { } return true; } - case IR::Opcode::ScalarSubBorrowCarryU32: { + case IR::Opcode::ScalarSubBorrowCarryU32: + case IR::Opcode::ISubBorrowCarryU32: { const auto lhs = ReadU32(inst.src[0]); const auto rhs = ReadU32(inst.src[1]); const auto borrow_in = ConditionBit(inst.src[2]); @@ -83,7 +84,12 @@ bool Translator::TranslateStateOperation(const IR::Instruction& inst) { const auto borrow0 = ir.UGreaterThan(rhs, lhs); const auto borrow1 = ir.UGreaterThan(borrow_in, partial); WriteOperand(inst.dst, result); - ir.SetScc(ir.LogicalOr(borrow0, borrow1)); + const auto borrow = ir.LogicalOr(borrow0, borrow1); + if (inst.op == IR::Opcode::ScalarSubBorrowCarryU32) { + ir.SetScc(borrow); + } else { + WriteMask(inst.dst2, borrow); + } return true; } case IR::Opcode::ScalarSignedAddOverflowI32: diff --git a/src/graphics/shader/recompiler/frontend/translate/Translate.cpp b/src/graphics/shader/recompiler/frontend/translate/Translate.cpp index 82d069db5..f67c0c35f 100644 --- a/src/graphics/shader/recompiler/frontend/translate/Translate.cpp +++ b/src/graphics/shader/recompiler/frontend/translate/Translate.cpp @@ -585,7 +585,7 @@ IR::U32 Translator::ConditionBit(const IR::Operand& operand) { if (operand.kind == IR::OperandKind::ImmediateU32) { return IR::U32(IR::Value(operand.imm & 1u)); } - return ir.Select(ReadCondition(operand), IR::U32(IR::Value(1u)), IR::U32(IR::Value(0u))); + return ir.Select(ReadMask(operand), IR::U32(IR::Value(1u)), IR::U32(IR::Value(0u))); } IR::U1 Translator::ReadMask(const IR::Operand& operand) { diff --git a/src/graphics/shader/recompiler/ir/ShaderIR.cpp b/src/graphics/shader/recompiler/ir/ShaderIR.cpp index 5adf2da81..19004e9da 100644 --- a/src/graphics/shader/recompiler/ir/ShaderIR.cpp +++ b/src/graphics/shader/recompiler/ir/ShaderIR.cpp @@ -741,6 +741,23 @@ bool LowerVectorAddCarry(const Decoder::Instruction& decoded, BasicBlock& block, return true; } +bool LowerVectorSubrevBorrowCarry(const Decoder::Instruction& decoded, BasicBlock& block, + std::string* error) { + Instruction inst; + inst.pc = decoded.pc; + inst.op = Opcode::ISubBorrowCarryU32; + inst.src_count = 3; + if (!LowerRegisterOperand(decoded.dst, inst.dst, error) || + !LowerRegisterOperand(decoded.dst2, inst.dst2, error) || + !LowerSourceOperand(decoded.src1, inst.src[0], error) || + !LowerSourceOperand(decoded.src0, inst.src[1], error) || + !LowerSourceOperand(decoded.src2, inst.src[2], error)) { + return false; + } + block.instructions.push_back(inst); + return true; +} + bool LowerVectorCarryOut(const Decoder::Instruction& decoded, BasicBlock& block, std::string* error) { Instruction inst; @@ -1124,6 +1141,8 @@ bool LowerDecodedInstruction(const Decoder::Instruction& inst, BasicBlock& block case Decoder::Opcode::VMovreldB32: return LowerVectorMoveRelDestination(inst, block, error); case Decoder::Opcode::VMovrelsB32: return LowerVectorMoveRelSource(inst, block, error); case Decoder::Opcode::VAddcU32: return LowerVectorAddCarry(inst, block, error); + case Decoder::Opcode::VSubrevCoCiU32: + return LowerVectorSubrevBorrowCarry(inst, block, error); case Decoder::Opcode::VMadU64U32: return LowerVectorMadU64U32(inst, block, error); case Decoder::Opcode::VMacF32: return LowerVectorMacF32(inst, block, error); case Decoder::Opcode::VPkFmacF16: return LowerVectorPkFmacF16(inst, block, error); diff --git a/src/graphics/shader/recompiler/ir/opcodes/ShaderIROpcodes.inc b/src/graphics/shader/recompiler/ir/opcodes/ShaderIROpcodes.inc index dce998b8f..b60b6e31d 100644 --- a/src/graphics/shader/recompiler/ir/opcodes/ShaderIROpcodes.inc +++ b/src/graphics/shader/recompiler/ir/opcodes/ShaderIROpcodes.inc @@ -22,6 +22,7 @@ IR_OPCODE(Permlanex16B32, General, Lane) IR_OPCODE(AbsI32, General, SimpleInteger) IR_OPCODE(IAddU32, General, SimpleInteger) IR_OPCODE(IAddCarryU32, General, State) +IR_OPCODE(ISubBorrowCarryU32, General, State) IR_OPCODE(ISubBorrowU32, General, State) IR_OPCODE(ScalarAddCarryU32, General, State) IR_OPCODE(ScalarSubBorrowU32, General, State) diff --git a/tests/ShaderRecompilerComputeTests.cpp b/tests/ShaderRecompilerComputeTests.cpp index bae823460..aaf40631c 100644 --- a/tests/ShaderRecompilerComputeTests.cpp +++ b/tests/ShaderRecompilerComputeTests.cpp @@ -11694,6 +11694,79 @@ TestCase VectorAddcWritesPerLaneCarryOut() { return test; } +TestCase VectorSubrevCoCiU32ExactRawOnGpu() { + using O = ShaderOpcode; + + std::vector code; + code.push_back(EncodeVop1(0x01, 24, Vgpr(0))); + code.push_back(EncodeVop1(0x01, 1, InlineU32(2))); + code.push_back(EncodeVop1(0x01, 2, InlineU32(1))); + code.push_back(EncodeVop2(0x1a, 4, InlineU32(2), 0)); + code.push_back(EncodeVop2(0x25, 5, InlineU32(16), 4)); + code.push_back(EncodeVopc(0xc1, Vgpr(0), 1)); + code.push_back(0x54003080u); // v_subrev_co_ci_u32 v0, 0, v24, vcc + code.push_back(EncodeVop2(0x01, 3, InlineU32(0), 2)); + AppendBufferStoreDword(&code, 0, 4); + AppendBufferStoreDword(&code, 3, 5); + AppendEnd(&code); + + TestCase test; + test.name = "VectorSubrevCoCiU32ExactRawOnGpu"; + test.code = code; + test.expected = {0xffffffffu, 0, 2, 3, 1, 0, 0, 0}; + test.opcodes = {O::VMovB32, O::VLshlrevB32, O::VAddNcU32, + O::VCmpLtU32, O::VSubrevCoCiU32, O::VCndmaskB32, + O::BufferStoreDword, O::SEndpgm}; + test.required_spirv = {"OpISub", "OpUGreaterThan"}; + test.compute_info.threads_num[0] = 4; + test.compute_info.threads_num[1] = 1; + test.compute_info.threads_num[2] = 1; + test.compute_info.group_id[0] = true; + test.compute_info.wave_size = 32; + test.compute_info.thread_ids_num = 1; + test.compute_info.workgroup_register = 0; + test.has_compute_info = true; + return test; +} + +TestCase VectorVop3BSubrevCoCiUsesEncodedMasks() { + using O = ShaderOpcode; + + std::vector code; + code.push_back(EncodeVop1(0x01, 24, Vgpr(0))); + code.push_back(EncodeVop1(0x01, 1, InlineU32(2))); + code.push_back(EncodeVop1(0x01, 2, InlineU32(1))); + code.push_back(EncodeVop2(0x1a, 4, InlineU32(2), 0)); + code.push_back(EncodeVop2(0x25, 5, InlineU32(16), 4)); + code.push_back(EncodeVopc(0xc1, Vgpr(0), 1)); + code.push_back(EncodeSop1(0x04, 20, 106)); // s_mov_b64 s[20:21], vcc + AppendSMovLiteral(&code, 106, 0); + AppendSMovLiteral(&code, 107, 0); + AppendVop3B(&code, 0x12au, 10, 22, InlineU32(0), Vgpr(24), 20); + AppendVop3(&code, 0x101u, 3, InlineU32(0), Vgpr(2), 22); + AppendBufferStoreDword(&code, 10, 4); + AppendBufferStoreDword(&code, 3, 5); + AppendEnd(&code); + + TestCase test; + test.name = "VectorVop3BSubrevCoCiUsesEncodedMasks"; + test.code = code; + test.expected = {0xffffffffu, 0, 2, 3, 1, 0, 0, 0}; + test.opcodes = {O::VMovB32, O::VLshlrevB32, O::VAddNcU32, + O::VCmpLtU32, O::SMovB64, O::SMovB32, + O::VSubrevCoCiU32, O::VCndmaskB32, O::BufferStoreDword, + O::SEndpgm}; + test.compute_info.threads_num[0] = 4; + test.compute_info.threads_num[1] = 1; + test.compute_info.threads_num[2] = 1; + test.compute_info.group_id[0] = true; + test.compute_info.wave_size = 32; + test.compute_info.thread_ids_num = 1; + test.compute_info.workgroup_register = 0; + test.has_compute_info = true; + return test; +} + TestCase VectorVop3BCarryOutWritesSgprMask() { using O = ShaderOpcode; @@ -16800,6 +16873,8 @@ std::vector MakeCases() { AddCase(VectorMbcntUsesThreadMask); AddCase(VectorAddcWritesPerLaneCarryOut); AddCase(VectorAddcUsesPerLaneCarryIn); + AddCase(VectorSubrevCoCiU32ExactRawOnGpu); + AddCase(VectorVop3BSubrevCoCiUsesEncodedMasks); AddCase(VectorVop3BCarryOutWritesSgprMask); AddCase(VectorVop3BCarryOutUsesEncodedSdst); AddCase(VectorVop3BSubCoU32UsesRdna2Opcode310);