From a8beac190f13903ea7cc7a1a5e0c335fad0127aa Mon Sep 17 00:00:00 2001 From: Claxten Date: Wed, 7 Oct 2026 19:18:50 +0400 Subject: [PATCH] shader: evaluate scalar branch decisions across the complete guest wave --- .../recompiler/backend/spirv/SpirvEmitter.cpp | 4 ++ .../backend/spirv/spirvEmitterFlow.cpp | 24 ++++------ tests/ShaderRecompilerComputeTests.cpp | 44 +++++++++++++++++++ tests/shaderCfgTests.cpp | 21 +++++---- 4 files changed, 68 insertions(+), 25 deletions(-) diff --git a/src/graphics/shader/recompiler/backend/spirv/SpirvEmitter.cpp b/src/graphics/shader/recompiler/backend/spirv/SpirvEmitter.cpp index 4bdde55ce..e5600fe86 100644 --- a/src/graphics/shader/recompiler/backend/spirv/SpirvEmitter.cpp +++ b/src/graphics/shader/recompiler/backend/spirv/SpirvEmitter.cpp @@ -280,6 +280,10 @@ Emitter::SpirvRequirements Emitter::AnalyzeProgramRequirements(const IR::Program switch (inst.GetOpcode()) { case IR::ValueOpcode::StoreCompletion: requirements.subgroup_barrier = true; break; case IR::ValueOpcode::BvhIntersect: requirements.bvh = true; break; + case IR::ValueOpcode::ConditionRef: + requirements.subgroup_ballot |= + inst.Flags() != CFG::BranchCondition::ScalarInstruction; + break; case IR::ValueOpcode::Ballot: requirements.subgroup_ballot = true; break; case IR::ValueOpcode::DppMoveU32: case IR::ValueOpcode::ReadFirstLane: diff --git a/src/graphics/shader/recompiler/backend/spirv/spirvEmitterFlow.cpp b/src/graphics/shader/recompiler/backend/spirv/spirvEmitterFlow.cpp index f77b4a3d0..cd69d2d37 100644 --- a/src/graphics/shader/recompiler/backend/spirv/spirvEmitterFlow.cpp +++ b/src/graphics/shader/recompiler/backend/spirv/spirvEmitterFlow.cpp @@ -713,26 +713,18 @@ uint32_t EmitDppUpdateU32(ValueEmitContext& ctx, const IR::Inst& inst) { } uint32_t EmitConditionRef(ValueEmitContext& ctx, const IR::Inst& inst) { - if (ctx.other_half == nullptr) return ctx.Arg(inst, 0); - // A native scalar branch makes one decision for both emulated wave halves. - if (ctx.half != 0) return ctx.other_half->Def(IR::Value(&inst)); + // Scalar branches include lanes disabled by EXEC, which the branch body may restore. + if (ctx.other_half != nullptr && ctx.half != 0) return ctx.other_half->Def(IR::Value(&inst)); const auto kind = inst.Flags(); if (kind == CFG::BranchCondition::ScalarInstruction) return ctx.Arg(inst, 0); - const auto ballot = ctx.Ballot(inst.Arg(0)); - const auto low = ctx.state.builder.AllocateId(); - const auto high = ctx.state.builder.AllocateId(); - const auto combined = ctx.state.builder.AllocateId(); - const auto result = ctx.state.builder.AllocateId(); - ctx.state.builder.AddFunction(spv::OpCompositeExtract, TypeU32(ctx.state), low, ballot, 0); - ctx.state.builder.AddFunction(spv::OpCompositeExtract, TypeU32(ctx.state), high, ballot, 1); const bool zero = kind == CFG::BranchCondition::ExecZero || kind == CFG::BranchCondition::VccZero || kind == CFG::BranchCondition::SccZero; - ctx.state.builder.AddFunction(zero ? spv::OpBitwiseAnd : spv::OpBitwiseOr, - TypeU32(ctx.state), combined, low, high); - ctx.state.builder.AddFunction(zero ? spv::OpIEqual : spv::OpINotEqual, - TypeBool(ctx.state), result, combined, - ConstantU32(ctx.state, zero ? ~0u : 0u)); - return result; + const auto ballot = ctx.Ballot(inst.Arg(0)); + const auto reference = zero ? ctx.Ballot(IR::Value(true)) + : ConstantU32CompositeZero(ctx.state, 4); + const auto compare = Binary(ctx.state, zero ? spv::OpIEqual : spv::OpINotEqual, + TypeBoolVector(ctx.state, 4), ballot, reference); + return Unary(ctx.state, zero ? spv::OpAll : spv::OpAny, TypeBool(ctx.state), compare); } uint32_t EmitBallot(ValueEmitContext& ctx, IR::Value predicate) { diff --git a/tests/ShaderRecompilerComputeTests.cpp b/tests/ShaderRecompilerComputeTests.cpp index bd0a1d726..1dd501103 100644 --- a/tests/ShaderRecompilerComputeTests.cpp +++ b/tests/ShaderRecompilerComputeTests.cpp @@ -27535,6 +27535,42 @@ TestCase BranchVccnzUsesWaveMask() { return test; } +TestCase ScalarBranchRestoresInactiveLanes(u32 wave_size, u32 threads) { + using O = ShaderOpcode; + TestCase test; + if (threads < wave_size) { + test.name = wave_size == 32 ? "ScalarBranchRestoresPartialWave32" + : "ScalarBranchRestoresPartialWave64"; + } else { + test.name = wave_size == 32 ? "ScalarBranchRestoresInactiveWave32" + : "ScalarBranchRestoresInactiveWave64"; + } + auto &code = test.code; + code = {EncodeVop1(0x01, 1, InlineU32(32)), + EncodeVop2(0x25, 1, Vgpr(0), 1), + EncodeSop1(0x04, 8, 126), // Save all participating lanes. + EncodeSMovB32(20, InlineU32(0)), + EncodeSMovB32(126, InlineU32(1)), EncodeSMovB32(127, InlineU32(0)), + EncodeSopp(0x08, 3), // EXEC is nonzero for the whole wave. + EncodeSop1(0x04, 126, 8)}; + // The branch restores inactive lanes before reading the last lane's value. + AppendVop3(&code, 0x360, 20, Vgpr(1), InlineU32(threads - 1)); + code.push_back(EncodeSop1(0x04, 126, 8)); + AppendStoreSgprAtLaneDwordOffset(&code, 20, 0, 0); + AppendEnd(&code); + test.initial.assign(threads, 0); + test.expected.assign(threads, 32 + threads - 1); + test.opcodes = {O::V_MOV_B32, O::V_ADD_NC_U32, O::S_MOV_B32, O::S_MOV_B64, + O::S_CBRANCH_EXECZ, O::V_READLANE_B32, O::V_LSHLREV_B32, + O::BUFFER_STORE_DWORD, O::S_ENDPGM}; + test.required_spirv = {"OpGroupNonUniformBallot", "OpAll"}; + test.compute_info.wave_size = wave_size; + test.compute_info.threads_num[0] = threads; + test.compute_info.thread_ids_num = 1; + test.has_compute_info = true; + return test; +} + TestCase ScalarMemRealtimeCapturedPlaceholder() { using O = ShaderOpcode; namespace D = ShaderRecompiler::Decoder; @@ -35775,6 +35811,10 @@ std::vector MakeCases() { AddCase(SharedReturnKeepsSelectedValues); AddCase(SiblingSharedExitKeepsCapturedConditions); AddCase(BranchVccnzUsesWaveMask); + AddCase([] { return ScalarBranchRestoresInactiveLanes(32, 32); }); + AddCase([] { return ScalarBranchRestoresInactiveLanes(64, 64); }); + AddCase([] { return ScalarBranchRestoresInactiveLanes(32, 4); }); + AddCase([] { return ScalarBranchRestoresInactiveLanes(64, 4); }); AddCase(BranchVccnzUsesCarryProducedWaveMask); AddCase(ScalarMemRealtimeCapturedPlaceholder); AddCase(ScalarMemoryLoadVariants); @@ -41350,6 +41390,10 @@ int main(int argc, char **argv) { RunCase(&vulkan, ScratchIsPrivatePerInvocation()); RunCase(&vulkan, Vop1MoveRelDestination()); RunCase(&vulkan, BranchVccnzUsesWaveMask()); + RunCase(&vulkan, ScalarBranchRestoresInactiveLanes(32, 32)); + RunCase(&vulkan, ScalarBranchRestoresInactiveLanes(64, 64)); + RunCase(&vulkan, ScalarBranchRestoresInactiveLanes(32, 4)); + RunCase(&vulkan, ScalarBranchRestoresInactiveLanes(64, 4)); RunCase(&vulkan, BranchVccnzUsesCarryProducedWaveMask()); RunCase(&vulkan, ImageSampleAndGather()); RunCase(&vulkan, ImageGatherExplicitLod()); diff --git a/tests/shaderCfgTests.cpp b/tests/shaderCfgTests.cpp index 522ae810b..926e0cbf5 100644 --- a/tests/shaderCfgTests.cpp +++ b/tests/shaderCfgTests.cpp @@ -14731,13 +14731,14 @@ void TestNewShaderRecompilerSpirvSizeBaselines() { EncodeSopp(0x01), }; const auto structured_result = compile("structured-phi", structured_phi, - {.words = 140, - .instructions = 41, + {.words = 174, + .instructions = 49, .phis = 1, .labels = 8, .loop_merges = 1, .branches = 6, - .conditional_branches = 1}); + .conditional_branches = 1, + .ballots = 2}); const auto structured_metrics = MeasureSpirv(structured_result.spirv); Check((structured_result.ir_dump.find("Phi") != std::string::npos), "structured Phi size fixture no longer contains an IR Phi"); @@ -14748,13 +14749,14 @@ void TestNewShaderRecompilerSpirvSizeBaselines() { CheckSpirvPhiParents(structured_result.spirv); const auto structured_repeat = compile("structured-phi-repeat", structured_phi, - {.words = 140, - .instructions = 41, + {.words = 174, + .instructions = 49, .phis = 1, .labels = 8, .loop_merges = 1, .branches = 6, - .conditional_branches = 1}); + .conditional_branches = 1, + .ballots = 2}); Check(structured_repeat.spirv == structured_result.spirv, "deferred Phi patching is not deterministic"); @@ -14918,8 +14920,8 @@ void TestNewShaderRecompilerSpirvSizeBaselines() { EncodeSopp(0x01), }; const auto dispatcher_result = compile("dispatcher", dispatcher, - {.words = 242, - .instructions = 67, + {.words = 255, + .instructions = 70, .variables = 3, .function_variables = 3, .loads = 3, @@ -14930,7 +14932,8 @@ void TestNewShaderRecompilerSpirvSizeBaselines() { .selection_merges = 1, .branches = 10, .conditional_branches = 1, - .switches = 1}); + .switches = 1, + .ballots = 1}); Check(dispatcher_result.program.dispatcher_fallback && (dispatcher_result.ir_dump.find("Phi") != std::string::npos) && SpirvInstructionOpcodeCount(dispatcher_result.spirv, 245u) == 2u &&