From c913951f7b13fe179d94dd187dfbd2097654256e Mon Sep 17 00:00:00 2001 From: nmzik Date: Wed, 9 Sep 2026 19:04:03 +0400 Subject: [PATCH] shader: implement RDNA2 subvector loop mask and branch semantics --- .../backend/spirv/spirvEmitterProgram.cpp | 2 + .../recompiler/frontend/cfg/ShaderCFG.cpp | 3 + .../recompiler/frontend/cfg/ShaderCFG.h | 1 + .../frontend/decode/ScalarAluOps.cpp | 6 + .../frontend/decode/ShaderDecoder.cpp | 9 +- .../frontend/decode/ShaderDecoder.h | 2 + .../recompiler/frontend/translate/Control.cpp | 27 +++++ .../recompiler/frontend/translate/Scalar.cpp | 2 + .../frontend/translate/Translate.cpp | 4 + .../frontend/translate/Translator.h | 2 + tests/ShaderRecompilerComputeTests.cpp | 110 ++++++++++++++++++ 11 files changed, 167 insertions(+), 1 deletion(-) diff --git a/src/graphics/shader/recompiler/backend/spirv/spirvEmitterProgram.cpp b/src/graphics/shader/recompiler/backend/spirv/spirvEmitterProgram.cpp index 3b8ff85da..c225355bb 100644 --- a/src/graphics/shader/recompiler/backend/spirv/spirvEmitterProgram.cpp +++ b/src/graphics/shader/recompiler/backend/spirv/spirvEmitterProgram.cpp @@ -96,7 +96,9 @@ void EmitReturn(ValueEmitContext& ctx) { } uint32_t BranchCondition(ValueEmitContext& ctx, const IR::BlockInfo& info) { + // Scalar-instruction conditions already test the full wave's raw register values. if (ctx.other_half == nullptr || + info.terminator.condition == CFG::BranchCondition::ScalarInstruction || info.terminator.condition == CFG::BranchCondition::GotoVariable) { return ctx.Def(info.condition); } diff --git a/src/graphics/shader/recompiler/frontend/cfg/ShaderCFG.cpp b/src/graphics/shader/recompiler/frontend/cfg/ShaderCFG.cpp index f1ef32a07..c00421d45 100644 --- a/src/graphics/shader/recompiler/frontend/cfg/ShaderCFG.cpp +++ b/src/graphics/shader/recompiler/frontend/cfg/ShaderCFG.cpp @@ -61,6 +61,8 @@ BranchCondition ConditionForOpcode(Opcode opcode) { case Opcode::S_CBRANCH_VCCNZ: return BranchCondition::VccNonZero; case Opcode::S_CBRANCH_EXECZ: return BranchCondition::ExecZero; case Opcode::S_CBRANCH_EXECNZ: return BranchCondition::ExecNonZero; + case Opcode::S_SUBVECTOR_LOOP_BEGIN: + case Opcode::S_SUBVECTOR_LOOP_END: return BranchCondition::ScalarInstruction; default: return BranchCondition::Unknown; } } @@ -2224,6 +2226,7 @@ std::string BranchConditionToString(BranchCondition condition) { case BranchCondition::VccNonZero: return "vccnz"; case BranchCondition::ExecZero: return "execz"; case BranchCondition::ExecNonZero: return "execnz"; + case BranchCondition::ScalarInstruction: return "scalar_instruction"; case BranchCondition::GotoVariable: return "goto_variable"; default: return "unknown"; } diff --git a/src/graphics/shader/recompiler/frontend/cfg/ShaderCFG.h b/src/graphics/shader/recompiler/frontend/cfg/ShaderCFG.h index 0c26a1105..fb2a66898 100644 --- a/src/graphics/shader/recompiler/frontend/cfg/ShaderCFG.h +++ b/src/graphics/shader/recompiler/frontend/cfg/ShaderCFG.h @@ -17,6 +17,7 @@ enum class BranchCondition { VccNonZero, ExecZero, ExecNonZero, + ScalarInstruction, GotoVariable, Unknown }; diff --git a/src/graphics/shader/recompiler/frontend/decode/ScalarAluOps.cpp b/src/graphics/shader/recompiler/frontend/decode/ScalarAluOps.cpp index 815787037..f86d1b583 100644 --- a/src/graphics/shader/recompiler/frontend/decode/ScalarAluOps.cpp +++ b/src/graphics/shader/recompiler/frontend/decode/ScalarAluOps.cpp @@ -83,6 +83,7 @@ constexpr OpcodeMap SOPK_OPCODE_LIST[] = { {0x0eu, Opcode::S_CMP_LE_U32}, {0x0fu, Opcode::S_ADD_I32}, {0x10u, Opcode::S_MULK_I32}, {0x13u, Opcode::S_SETREG_B32}, {0x17u, Opcode::S_WAITCNT}, {0x18u, Opcode::S_WAITCNT}, {0x19u, Opcode::S_WAITCNT}, {0x1au, Opcode::S_WAITCNT}, + {0x1bu, Opcode::S_SUBVECTOR_LOOP_BEGIN}, {0x1cu, Opcode::S_SUBVECTOR_LOOP_END}, }; constexpr OpcodeMap SOPP_OPCODE_LIST[] = { @@ -209,6 +210,11 @@ void DecodeSopk(uint32_t pc, std::span code, uint32_t word_index switch (inst.opcode) { case Opcode::S_MOVK_I32: DecodeScalarDestination(sdst, pc, inst.dst); return; + case Opcode::S_SUBVECTOR_LOOP_BEGIN: + case Opcode::S_SUBVECTOR_LOOP_END: + DecodeScalarDestination(sdst, pc, inst.dst); + inst.branch_target = pc + 4u + static_cast(imm * 4); + return; case Opcode::S_WAITCNT: { const uint32_t waitcnt = word & 0xffffu; inst.dst.kind = OperandKind::Null; diff --git a/src/graphics/shader/recompiler/frontend/decode/ShaderDecoder.cpp b/src/graphics/shader/recompiler/frontend/decode/ShaderDecoder.cpp index 565b81ec1..ab5850317 100644 --- a/src/graphics/shader/recompiler/frontend/decode/ShaderDecoder.cpp +++ b/src/graphics/shader/recompiler/frontend/decode/ShaderDecoder.cpp @@ -192,7 +192,9 @@ bool IsConditionalBranch(Opcode opcode) { case Opcode::S_CBRANCH_VCCZ: case Opcode::S_CBRANCH_VCCNZ: case Opcode::S_CBRANCH_EXECZ: - case Opcode::S_CBRANCH_EXECNZ: return true; + case Opcode::S_CBRANCH_EXECNZ: + case Opcode::S_SUBVECTOR_LOOP_BEGIN: + case Opcode::S_SUBVECTOR_LOOP_END: return true; default: return false; } } @@ -535,6 +537,11 @@ std::string InstructionToString(const Instruction& inst) { return WithUnsupportedReason(inst, fmt::format("0x{:08x}: {} 0x{:08x}", inst.pc, magic_enum::enum_name(inst.opcode), inst.branch_target)); + case Opcode::S_SUBVECTOR_LOOP_BEGIN: + case Opcode::S_SUBVECTOR_LOOP_END: + return WithUnsupportedReason(inst, fmt::format( + "0x{:08x}: {} {}, 0x{:08x}", inst.pc, magic_enum::enum_name(inst.opcode), + OperandToString(inst.dst), inst.branch_target)); case Opcode::EXP: return WithUnsupportedReason(inst, FormatExp(inst)); case Opcode::IMAGE_SAMPLE: case Opcode::IMAGE_STORE: diff --git a/src/graphics/shader/recompiler/frontend/decode/ShaderDecoder.h b/src/graphics/shader/recompiler/frontend/decode/ShaderDecoder.h index f94073e0e..c0a06473d 100644 --- a/src/graphics/shader/recompiler/frontend/decode/ShaderDecoder.h +++ b/src/graphics/shader/recompiler/frontend/decode/ShaderDecoder.h @@ -53,6 +53,8 @@ enum class Opcode { S_QUADMASK_B64, S_GETPC_B64, S_SETPC_B64, + S_SUBVECTOR_LOOP_BEGIN, + S_SUBVECTOR_LOOP_END, S_AND_SAVEEXEC_B32, S_ORN2_SAVEEXEC_B32, S_ANDN1_SAVEEXEC_B32, diff --git a/src/graphics/shader/recompiler/frontend/translate/Control.cpp b/src/graphics/shader/recompiler/frontend/translate/Control.cpp index 366e1da69..f8a73abef 100644 --- a/src/graphics/shader/recompiler/frontend/translate/Control.cpp +++ b/src/graphics/shader/recompiler/frontend/translate/Control.cpp @@ -22,6 +22,33 @@ Decoder::Operand ConditionOperand(Decoder::OperandKind kind) { } // namespace +void Translator::S_SUBVECTOR_LOOP(const Decoder::Instruction& inst, bool begin) { + const auto zero = IR::U32(IR::Value(0u)); + const auto lo = ir.GetExecLo(); + const auto hi = ir.GetExecHi(); + const auto saved = ReadU32(inst.dst); + if (begin) { + const auto low_active = ir.INotEqual(lo, zero); + instruction_branch_condition = ir.IEqual(ir.BitwiseOr(lo, hi), zero); + WriteRawU32(inst.dst, ir.Select(instruction_branch_condition, saved, + ir.Select(low_active, hi, lo))); + // Keep the ISA assignment order: SDST may itself name an EXEC half. + WriteRawU32(ConditionOperand(Decoder::OperandKind::ExecHi), + ir.Select(low_active, zero, ir.GetExecHi())); + } else { + const auto high_active = ir.INotEqual(hi, zero); + instruction_branch_condition = + ir.LogicalAnd(ir.LogicalNot(high_active), ir.INotEqual(saved, zero)); + WriteRawU32(ConditionOperand(Decoder::OperandKind::ExecHi), + ir.Select(instruction_branch_condition, saved, hi)); + WriteRawU32(inst.dst, + ir.Select(instruction_branch_condition, lo, ReadU32(inst.dst))); + WriteRawU32(ConditionOperand(Decoder::OperandKind::ExecLo), + ir.Select(high_active, saved, + ir.Select(instruction_branch_condition, zero, ir.GetExecLo()))); + } +} + void Translator::S_SAVEEXEC(const Decoder::Instruction& inst, IR::ValueOpcode operation, bool negate_exec, bool negate_source, bool write_64) { if (!write_64) { diff --git a/src/graphics/shader/recompiler/frontend/translate/Scalar.cpp b/src/graphics/shader/recompiler/frontend/translate/Scalar.cpp index 30dacff02..e30f79541 100644 --- a/src/graphics/shader/recompiler/frontend/translate/Scalar.cpp +++ b/src/graphics/shader/recompiler/frontend/translate/Scalar.cpp @@ -11,6 +11,8 @@ bool Translator::EmitScalar(const Decoder::Instruction& inst) { case O::S_WQM_B64: S_WQM_B64(inst); return true; case O::S_GETPC_B64: S_GETPC_B64(inst); return true; case O::S_SETPC_B64: return true; + case O::S_SUBVECTOR_LOOP_BEGIN: S_SUBVECTOR_LOOP(inst, true); return true; + case O::S_SUBVECTOR_LOOP_END: S_SUBVECTOR_LOOP(inst, false); return true; case O::S_CSELECT_B32: S_CSELECT_B32(inst); return true; case O::S_CSELECT_B64: ScalarSelect64(inst, inst.src1); return true; case O::S_CMOV_B64: ScalarSelect64(inst, inst.dst); return true; diff --git a/src/graphics/shader/recompiler/frontend/translate/Translate.cpp b/src/graphics/shader/recompiler/frontend/translate/Translate.cpp index 3820ca9e8..9f15e61f8 100644 --- a/src/graphics/shader/recompiler/frontend/translate/Translate.cpp +++ b/src/graphics/shader/recompiler/frontend/translate/Translate.cpp @@ -758,6 +758,10 @@ void Translator::AddBranchCondition(const CFG::BasicBlock& source, IR::BlockInfo case CFG::BranchCondition::VccNonZero: condition = ir.GetVcc(); break; case CFG::BranchCondition::ExecZero: condition = ir.LogicalNot(ir.GetExec()); break; case CFG::BranchCondition::ExecNonZero: condition = ir.GetExec(); break; + case CFG::BranchCondition::ScalarInstruction: + EXIT_IF(instruction_branch_condition.IsEmpty()); + condition = instruction_branch_condition; + break; case CFG::BranchCondition::GotoVariable: if (source.terminator.goto_variable == UINT32_MAX) { EXIT("block %u reads an invalid goto variable", source.id); diff --git a/src/graphics/shader/recompiler/frontend/translate/Translator.h b/src/graphics/shader/recompiler/frontend/translate/Translator.h index f903e2196..0372dde7b 100644 --- a/src/graphics/shader/recompiler/frontend/translate/Translator.h +++ b/src/graphics/shader/recompiler/frontend/translate/Translator.h @@ -212,6 +212,7 @@ private: bool V_CNDMASK_B32(const Decoder::Instruction& inst); bool PackB16(const Decoder::Instruction& inst, bool high0, bool high1); + void S_SUBVECTOR_LOOP(const Decoder::Instruction& inst, bool begin); void S_SAVEEXEC(const Decoder::Instruction& inst, IR::ValueOpcode operation, bool negate_exec, bool negate_source, bool write_64); void ADD_U32(const Decoder::Instruction& inst, bool vector, bool use_carry_in); @@ -252,6 +253,7 @@ private: IR::Program& program; IR::IREmitter ir; + IR::U1 instruction_branch_condition; Decoder::Opcode current_opcode = Decoder::Opcode::UNKNOWN; uint32_t current_pc = 0; uint32_t current_vector_limit = 1; diff --git a/tests/ShaderRecompilerComputeTests.cpp b/tests/ShaderRecompilerComputeTests.cpp index b1332029c..3c5bfb879 100644 --- a/tests/ShaderRecompilerComputeTests.cpp +++ b/tests/ShaderRecompilerComputeTests.cpp @@ -15389,6 +15389,112 @@ TestCase ScalarOrn2SaveexecB32(u32 wave_size, u32 threads) { return test; } +TestCase ScalarSubvectorLoops(u32 wave_size) { + using O = ShaderOpcode; + using namespace ShaderRecompiler::Decoder; + TestCase test; + test.name = wave_size == 64 ? "ScalarSubvectorLoopsWave64" + : "ScalarSubvectorLoopsWave32"; + // The captured mesh shader's signed SOPK targets are relative to PC + 4. + std::array captured{}; + captured[0x30 / 4] = 0xbda60022u; + captured[0xb8 / 4] = 0xbe26ffdeu; + Instruction begin, end; + DecodeInstruction(captured, 0x30 / 4, begin); + DecodeInstruction(captured, 0xb8 / 4, end); + Require(test.name, "captured branch decode", + begin.opcode == O::S_SUBVECTOR_LOOP_BEGIN && begin.dst.reg == 38 && + begin.dst.kind == OperandKind::Sgpr && begin.branch_target == 0xbc && + end.opcode == O::S_SUBVECTOR_LOOP_END && end.dst.reg == 38 && + end.dst.kind == OperandKind::Sgpr && end.branch_target == 0x34 && + begin.word_count == 1 && end.word_count == 1, + "subvector SDST or signed PC-relative target was decoded incorrectly"); + + constexpr u32 sentinel = 0x5a13abcdu; + constexpr u32 low = 0x80000005u, high = 0x42000002u; + struct Case { + u32 lo, hi, body_mode, saved, final_lo, final_hi, passes; + }; + constexpr Case cases[] = { + {0, 0, 0, sentinel, 0, 0, 0}, + {low, 0, 0, 0, low, 0, 1}, + {0, high, 0, 0, 0, high, 1}, + {low, high, 0, low, low, high, 2}, + {~0u, ~0u, 0, ~0u, ~0u, ~0u, 2}, + // The game resets the full EXEC mask before END, selecting its HI != 0 arm. + {low, high, 1, high, high, ~0u, 1}, + // END must restore the low mask saved after the body has changed it. + {low, high, 2, 0, 0, high, 2}, + // BEGIN EXEC_LO first copies HI to its aliased destination, then clears HI. + {low, high, 3, 0, 0, high, 2}, + }; + auto &code = test.code; + for (u32 index = 0; index < std::size(cases); ++index) { + const auto &item = cases[index]; + const u32 scc = index & 1u; + const u32 saved_reg = item.body_mode == 3 ? 126u : 38u; + AppendVMovU32(&code, 2, 0); + AppendVMovU32(&code, 3, 0); + AppendSMovLiteral(&code, 38, sentinel); + AppendSMovLiteral(&code, 32, 1); // Scalar 2^passes; MULK preserves SCC. + AppendSMovLiteral(&code, 33, 0xff); + AppendSMovLiteral(&code, 126, item.lo); + AppendSMovLiteral(&code, 127, item.hi); + code.push_back(EncodeSopc(0x06, InlineU32(0), InlineU32(scc == 0 ? 1 : 0))); + const auto begin_word = static_cast(code.size()); + code.push_back(0); + const auto body_word = static_cast(code.size()); + code.push_back(EncodeSMovB32(33, 253)); // Observe SCC immediately after BEGIN. + code.push_back(EncodeSopk(0x10, 32, 2)); + code.push_back(EncodeVop2(0x25, 2, InlineU32(1), 2)); + code.push_back(EncodeVop1(0x01, 3, 32)); + if (item.body_mode == 1) { + code.push_back(EncodeSop1(0x04, 126, 193)); // S_MOV_B64 EXEC, -1. + } else if (item.body_mode == 2) { + code.push_back(EncodeSMovB32(126, InlineU32(0))); + } + const auto end_word = static_cast(code.size()); + code.push_back(EncodeSopk(0x1c, saved_reg, body_word - end_word - 1u)); + code[begin_word] = EncodeSopk(0x1b, saved_reg, end_word - begin_word); + const u32 sources[] = {saved_reg, 126, 127, 253, 32, 33}; + for (u32 i = 0; i < std::size(sources); ++i) { + code.push_back(EncodeSMovB32(20 + i, sources[i])); + } + code.push_back(EncodeSop1(0x04, 126, 193)); // Restore all lanes for readback. + const u32 expected[] = {item.saved, item.final_lo, item.final_hi, scc, + 1u << item.passes, item.passes == 0 ? 0xff : scc}; + for (u32 i = 0; i < std::size(expected); ++i) { + AppendStoreSgprAtLaneDwordOffset(&code, 20 + i, 0, + static_cast(test.expected.size())); + test.expected.insert(test.expected.end(), wave_size, expected[i]); + } + for (u32 reg : {2u, 3u}) { + AppendStoreVgprAtLaneDwordOffset(&code, reg, 0, + static_cast(test.expected.size())); + for (u32 lane = 0; lane < wave_size; ++lane) { + const bool upper = lane >= 32; + const u32 mask = upper || item.body_mode == 3 ? item.hi : item.lo; + const bool active = ((mask >> (lane % 32)) & 1u) != 0 && + !(upper && item.body_mode == 1); + const u32 pass_value = upper && item.lo != 0 ? 4u : 2u; + test.expected.push_back(active ? (reg == 2 ? 1u : pass_value) : 0u); + } + } + } + AppendEnd(&code); + test.initial.resize(test.expected.size()); + test.opcodes = {O::S_SUBVECTOR_LOOP_BEGIN, O::S_SUBVECTOR_LOOP_END, + O::S_MOV_B32, O::S_MOV_B64, O::S_MULK_I32, O::S_CMP_EQ_U32, + O::V_MOV_B32, O::V_ADD_NC_U32, O::V_LSHLREV_B32, + O::BUFFER_STORE_DWORD, O::S_ENDPGM}; + test.compute_info.threads_num[0] = wave_size; + test.compute_info.threads_num[1] = test.compute_info.threads_num[2] = 1; + test.compute_info.thread_ids_num = 1; + test.compute_info.wave_size = wave_size; + test.has_compute_info = true; + return test; +} + TestCase ScalarGetpcWritesNextInstructionPc() { using O = ShaderOpcode; @@ -24417,6 +24523,8 @@ std::vector MakeCases() { cases.push_back(ScalarOrn2SaveexecB32(32, 32)); cases.push_back(ScalarOrn2SaveexecB32(64, 64)); cases.push_back(ScalarOrn2SaveexecB32(32, 4)); + cases.push_back(ScalarSubvectorLoops(32)); + cases.push_back(ScalarSubvectorLoops(64)); AddCase(ScalarGetpcWritesNextInstructionPc); AddCase(ScalarBitfieldPack); AddCase(ScalarBrevB32PreservesScc); @@ -28918,6 +29026,8 @@ int main(int argc, char **argv) { RunCase(&vulkan, ScalarOrn2SaveexecB32(32, 32)); RunCase(&vulkan, ScalarOrn2SaveexecB32(64, 64)); RunCase(&vulkan, ScalarOrn2SaveexecB32(32, 4)); + RunCase(&vulkan, ScalarSubvectorLoops(32)); + RunCase(&vulkan, ScalarSubvectorLoops(64)); RunCase(&vulkan, ScalarNotB64UpdatesScc()); RunCase(&vulkan, ScalarSelectB64PreservesMaskProvenance()); RunCase(&vulkan, ScalarConditionalMoveB64());