diff --git a/src/graphics/shader/recompiler/backend/spirv/spirvEmitterFlow.cpp b/src/graphics/shader/recompiler/backend/spirv/spirvEmitterFlow.cpp index 32652e02a..c49b1d6b1 100644 --- a/src/graphics/shader/recompiler/backend/spirv/spirvEmitterFlow.cpp +++ b/src/graphics/shader/recompiler/backend/spirv/spirvEmitterFlow.cpp @@ -636,6 +636,29 @@ uint32_t EmitDppUpdateU32(ValueEmitContext& ctx, const IR::Inst& inst) { ctx.Arg(inst, 1)); } +uint32_t EmitConditionRef(ValueEmitContext& ctx, const IR::Inst& inst) { + if (ctx.other_half == nullptr) return ctx.Arg(inst, 0); + // A native scalar branch makes one decision for both emulated wave halves. + if (ctx.half != 0) return ctx.other_half->Def(IR::Value(&inst)); + const auto kind = inst.Flags(); + if (kind == CFG::BranchCondition::ScalarInstruction) return ctx.Arg(inst, 0); + const auto ballot = ctx.Ballot(inst.Arg(0)); + const auto low = ctx.state.builder.AllocateId(); + const auto high = ctx.state.builder.AllocateId(); + const auto combined = ctx.state.builder.AllocateId(); + const auto result = ctx.state.builder.AllocateId(); + ctx.state.builder.AddFunction(spv::OpCompositeExtract, TypeU32(ctx.state), low, ballot, 0); + ctx.state.builder.AddFunction(spv::OpCompositeExtract, TypeU32(ctx.state), high, ballot, 1); + const bool zero = kind == CFG::BranchCondition::ExecZero || + kind == CFG::BranchCondition::VccZero || kind == CFG::BranchCondition::SccZero; + ctx.state.builder.AddFunction(zero ? spv::OpBitwiseAnd : spv::OpBitwiseOr, + TypeU32(ctx.state), combined, low, high); + ctx.state.builder.AddFunction(zero ? spv::OpIEqual : spv::OpINotEqual, + TypeBool(ctx.state), result, combined, + ConstantU32(ctx.state, zero ? ~0u : 0u)); + return result; +} + uint32_t EmitBallot(ValueEmitContext& ctx, IR::Value predicate) { return ctx.Ballot(predicate); } diff --git a/src/graphics/shader/recompiler/backend/spirv/spirvEmitterInstructions.h b/src/graphics/shader/recompiler/backend/spirv/spirvEmitterInstructions.h index 53dde198b..1610f54f6 100644 --- a/src/graphics/shader/recompiler/backend/spirv/spirvEmitterInstructions.h +++ b/src/graphics/shader/recompiler/backend/spirv/spirvEmitterInstructions.h @@ -182,6 +182,7 @@ uint32_t EmitDppUpdateU32(ValueEmitContext& ctx, const IR::Inst& in uint32_t EmitWqmU64(EmitterState& state, uint32_t value); uint32_t EmitLaneId(EmitterState& state); uint32_t EmitBallot(ValueEmitContext& ctx, IR::Value predicate); +uint32_t EmitConditionRef(ValueEmitContext& ctx, const IR::Inst& inst); uint32_t EmitReadFirstLane(ValueEmitContext& ctx, const IR::Inst& inst); uint32_t EmitReadLane(ValueEmitContext& ctx, const IR::Inst& inst); uint32_t EmitWriteLane(ValueEmitContext& ctx, const IR::Inst& inst); diff --git a/src/graphics/shader/recompiler/backend/spirv/spirvEmitterProgram.cpp b/src/graphics/shader/recompiler/backend/spirv/spirvEmitterProgram.cpp index d8b9df738..6f3b06322 100644 --- a/src/graphics/shader/recompiler/backend/spirv/spirvEmitterProgram.cpp +++ b/src/graphics/shader/recompiler/backend/spirv/spirvEmitterProgram.cpp @@ -97,30 +97,6 @@ void EmitReturn(ValueEmitContext& ctx) { ctx.state.builder.AddFunction(spv::OpReturn); } -uint32_t BranchCondition(ValueEmitContext& ctx, const IR::BlockInfo& info) { - // Scalar-instruction conditions already test the full wave's raw register values. - if (ctx.other_half == nullptr || - info.terminator.condition == CFG::BranchCondition::ScalarInstruction || - info.terminator.condition == CFG::BranchCondition::GotoVariable) { - return ctx.Def(info.condition); - } - const auto ballot = ctx.Ballot(info.condition); - const auto low = ctx.state.builder.AllocateId(); - const auto high = ctx.state.builder.AllocateId(); - const auto result = ctx.state.builder.AllocateId(); - ctx.state.builder.AddFunction(spv::OpCompositeExtract, TypeU32(ctx.state), low, ballot, 0); - ctx.state.builder.AddFunction(spv::OpCompositeExtract, TypeU32(ctx.state), high, ballot, 1); - const auto kind = info.terminator.condition; - const bool zero = kind == CFG::BranchCondition::ExecZero || - kind == CFG::BranchCondition::VccZero || - kind == CFG::BranchCondition::SccZero; - const auto combined = - EmitBinaryU32(ctx.state, zero ? spv::OpBitwiseAnd : spv::OpBitwiseOr, low, high); - ctx.state.builder.AddFunction(zero ? spv::OpIEqual : spv::OpINotEqual, TypeBool(ctx.state), - result, combined, ConstantU32(ctx.state, zero ? ~0u : 0u)); - return result; -} - void EmitStructuredTerminator(ValueEmitContext& ctx, const IR::Block* block, const IR::BlockInfo& info) { const auto& program = ctx.state.program; @@ -160,7 +136,7 @@ void EmitStructuredTerminator(ValueEmitContext& ctx, const IR::Block* block, EmitReturn(ctx); return; } - const auto condition = BranchCondition(ctx, info); + const auto condition = ctx.Def(info.condition); emit_merge(); ctx.state.builder.AddFunction(spv::OpBranchConditional, condition, ctx.Label(true_block), ctx.Label(false_block)); @@ -193,7 +169,7 @@ uint32_t EmitDispatcherNextPc(ValueEmitContext& ctx, const DispatcherFunctionSta EmitDispatcherTarget(ctx, dispatcher, block, term.false_block); const auto selected = ctx.state.builder.AllocateId(); ctx.state.builder.AddFunction( - spv::OpSelect, TypeU32(ctx.state), selected, BranchCondition(ctx, info), + spv::OpSelect, TypeU32(ctx.state), selected, ctx.Def(info.condition), ConstantU32(ctx.state, term.true_block), ConstantU32(ctx.state, term.false_block)); return selected; } diff --git a/src/graphics/shader/recompiler/frontend/cfg/ShaderCFG.cpp b/src/graphics/shader/recompiler/frontend/cfg/ShaderCFG.cpp index 9c279baf7..f2a2e5283 100644 --- a/src/graphics/shader/recompiler/frontend/cfg/ShaderCFG.cpp +++ b/src/graphics/shader/recompiler/frontend/cfg/ShaderCFG.cpp @@ -5,10 +5,11 @@ #include #include #include +#include +#include #include #include #include -#include namespace Libs::Graphics::ShaderRecompiler::CFG { namespace { @@ -598,39 +599,6 @@ bool Contains(const std::vector& values, uint32_t value) { return std::find(values.begin(), values.end(), value) != values.end(); } -bool ReplaceValue(std::vector& values, uint32_t old_value, uint32_t new_value) { - bool changed = false; - for (auto& value: values) { - if (value == old_value) { - value = new_value; - changed = true; - } - } - if (changed) { - SortUnique(values); - } - return changed; -} - -bool RemoveValue(std::vector& values, uint32_t value) { - const auto old_size = values.size(); - values.erase(std::remove(values.begin(), values.end(), value), values.end()); - return values.size() != old_size; -} - -bool ReplaceTerminatorTarget(Terminator& terminator, uint32_t old_value, uint32_t new_value) { - bool changed = false; - if (terminator.true_block == old_value) { - terminator.true_block = new_value; - changed = true; - } - if (terminator.false_block == old_value) { - terminator.false_block = new_value; - changed = true; - } - return changed; -} - uint32_t RemapId(uint32_t id, const std::vector& id_map) { return id != UINT32_MAX && id < id_map.size() ? id_map[id] : id; } @@ -994,562 +962,6 @@ std::vector ApplyBlockOrder(Graph& graph, std::vector bloc return id_map; } -uint32_t MoveBlockBefore(Graph& graph, uint32_t block_id, uint32_t before_id) { - if (block_id == before_id || block_id >= graph.blocks.size() || - before_id >= graph.blocks.size()) { - return block_id; - } - - const auto block_pos = block_id; - const auto before_pos = before_id; - std::vector old_blocks = std::move(graph.blocks); - std::vector new_blocks; - new_blocks.reserve(old_blocks.size()); - - for (uint32_t i = 0; i < old_blocks.size(); i++) { - if (i == before_pos) { - new_blocks.push_back(std::move(old_blocks[block_pos])); - } - if (i != block_pos) { - new_blocks.push_back(std::move(old_blocks[i])); - } - } - - const auto id_map = ApplyBlockOrder(graph, std::move(new_blocks)); - - return RemapId(block_id, id_map); -} - -std::vector DominatedBlocks(const Graph& graph, uint32_t header, - uint32_t stop_block = UINT32_MAX) { - std::vector blocks; - std::vector stack = {header}; - blocks.reserve(graph.blocks.size()); - stack.reserve(graph.blocks.size()); - - while (!stack.empty()) { - const auto block_id = stack.back(); - stack.pop_back(); - if (block_id == stop_block || Contains(blocks, block_id) || - !graph.Dominates(header, block_id)) { - continue; - } - - const auto* block = graph.FindBlock(block_id); - if (block == nullptr) { - continue; - } - - AddUnique(blocks, block_id); - for (auto succ: block->successors) { - if (succ != stop_block && graph.Dominates(header, succ)) { - stack.push_back(succ); - } - } - } - - SortUnique(blocks); - return blocks; -} - -uint32_t AppendSyntheticBranchBlock(Graph& graph, uint32_t target) { - const auto* target_block = graph.FindBlock(target); - - BasicBlock block; - block.id = static_cast(graph.blocks.size()); - block.start_pc = target_block != nullptr ? target_block->start_pc : 0u; - block.end_pc = block.start_pc; - block.inst_begin = target_block != nullptr ? target_block->inst_begin : 0u; - block.inst_end = block.inst_begin; - block.successors = {target}; - block.terminator.kind = TerminatorKind::Branch; - block.terminator.condition = BranchCondition::Always; - block.terminator.true_block = target; - graph.blocks.push_back(std::move(block)); - return graph.blocks.back().id; -} - -bool IsolateSemanticLoopHeader(Graph& graph, uint32_t old_header) { - const auto* header = graph.FindBlock(old_header); - if (header == nullptr || header->inst_begin == header->inst_end) { - return false; - } - - const auto predecessors = header->predecessors; - const auto new_header = AppendSyntheticBranchBlock(graph, old_header); - for (auto pred: predecessors) { - auto* block = graph.FindBlock(pred); - if (block != nullptr) { - ReplaceValue(block->successors, old_header, new_header); - ReplaceTerminatorTarget(block->terminator, old_header, new_header); - } - } - if (graph.entry_block == old_header) { - graph.entry_block = new_header; - } - MoveBlockBefore(graph, new_header, old_header); - RebuildPredecessors(graph); - RecomputeAnalyses(graph); - return true; -} - -const NaturalLoop* FindInnermostContainingLoop(const Graph& graph, uint32_t block_id) { - const NaturalLoop* innermost = nullptr; - for (const auto& loop: graph.natural_loops) { - if (Contains(loop.body_blocks, block_id) && - (innermost == nullptr || loop.body_blocks.size() < innermost->body_blocks.size())) { - innermost = &loop; - } - } - return innermost; -} - -bool IsInsideLoopConstruct(const Graph& graph, const NaturalLoop& loop, uint32_t block_id) { - return block_id != UINT32_MAX && block_id != loop.merge && block_id != loop.continue_block && - graph.Dominates(loop.header, block_id) && !graph.Dominates(loop.merge, block_id); -} - -bool IsLoopControlGateway(const Graph& graph, const NaturalLoop& loop, uint32_t block_id) { - const auto* block = graph.FindBlock(block_id); - if (block == nullptr || block->terminator.kind != TerminatorKind::ConditionalBranch) { - return false; - } - const auto is_control_target = [&](uint32_t target) { - return target == loop.merge || target == loop.continue_block; - }; - return is_control_target(block->terminator.true_block) && - is_control_target(block->terminator.false_block); -} - -bool HasLinearPathToTerminal(const Graph& graph, uint32_t start) { - std::vector visited(graph.blocks.size(), false); - for (auto block_id = start;;) { - const auto* block = graph.FindBlock(block_id); - if (block == nullptr || block_id >= visited.size() || visited[block_id]) { - return false; - } - if (block->successors.empty()) { - return true; - } - if (block->successors.size() != 1) { - return false; - } - visited[block_id] = true; - block_id = block->successors.front(); - } -} - -bool IsEnclosingLinearExit(const Graph& graph, uint32_t header, uint32_t block_id) { - // AGC commonly lowers nested early returns through a terminal epilogue shared with an - // enclosing conditional. Such a path is an exit boundary, not part of the inner selection. - if (!HasLinearPathToTerminal(graph, block_id)) { - return false; - } - const auto* block = graph.FindBlock(block_id); - while (block != nullptr && graph.Dominates(header, block->id) && - block->successors.size() == 1u) { - block = graph.FindBlock(block->successors.front()); - } - if (block == nullptr || graph.Dominates(header, block->id) || - block->predecessors.empty()) { - return false; - } - return std::ranges::all_of(block->predecessors, [&](uint32_t predecessor) { - return graph.Dominates(header, predecessor) || graph.Dominates(predecessor, header); - }); -} - -bool CanReachBefore(const Graph& graph, uint32_t start, uint32_t target, uint32_t stop) { - std::vector pending = {start}; - std::vector visited(graph.blocks.size(), false); - while (!pending.empty()) { - const auto block_id = pending.back(); - pending.pop_back(); - if (block_id == target) { - return true; - } - if (block_id == stop || block_id >= visited.size() || visited[block_id]) { - continue; - } - visited[block_id] = true; - const auto* block = graph.FindBlock(block_id); - if (block != nullptr) { - pending.insert(pending.end(), block->successors.begin(), block->successors.end()); - } - } - return false; -} - -uint32_t FindSelectionMerge(const Graph& graph, const BasicBlock& block) { - const auto global_merge = graph.FindNearestCommonPostDominator(block.terminator.true_block, - block.terminator.false_block); - const auto* loop = FindInnermostContainingLoop(graph, block.id); - if (loop == nullptr) { - const auto* global_block = graph.FindBlock(global_merge); - const auto true_target = block.terminator.true_block; - const auto false_target = block.terminator.false_block; - if (global_block == nullptr || global_block->successors.empty()) { - const bool false_reaches_true = - CanReachBefore(graph, false_target, true_target, global_merge); - const bool true_reaches_false = - CanReachBefore(graph, true_target, false_target, global_merge); - if (false_reaches_true != true_reaches_false) { - return false_reaches_true ? true_target : false_target; - } - if (global_merge == UINT32_MAX) { - // An enclosing selection's shared return needs its own inner merge - // gateway; the other arm can return directly without joining live state. - if (IsEnclosingLinearExit(graph, block.id, true_target)) { - return true_target; - } - if (IsEnclosingLinearExit(graph, block.id, false_target)) { - return false_target; - } - // A return can leave a selection without reaching its merge. Keep the - // continuing arm as the merge. Shared continuations receive a dedicated - // gateway in SplitSharedMergeBlock before structured control is emitted. - if (HasLinearPathToTerminal(graph, true_target)) { - return false_target; - } - if (HasLinearPathToTerminal(graph, false_target)) { - return true_target; - } - } - } - return global_merge; - } - - // A selection can join at the loop-control gateway even when another arm breaks directly to - // the enclosing loop merge. Global post-dominance sees that break and incorrectly expands the - // selection through the backedge. - const auto true_target = block.terminator.true_block; - const auto false_target = block.terminator.false_block; - if (IsLoopControlGateway(graph, *loop, true_target) && graph.Dominates(block.id, true_target) && - IsInsideLoopConstruct(graph, *loop, false_target)) { - return true_target; - } - if (IsLoopControlGateway(graph, *loop, false_target) && - graph.Dominates(block.id, false_target) && - IsInsideLoopConstruct(graph, *loop, true_target)) { - return false_target; - } - return global_merge; -} - -bool IsInnermostLoopControlConditional(const Graph& graph, const BasicBlock& block) { - if (block.terminator.kind != TerminatorKind::ConditionalBranch) { - return false; - } - const auto* loop = FindInnermostContainingLoop(graph, block.id); - if (loop == nullptr || loop->merge == UINT32_MAX || loop->continue_block == UINT32_MAX) { - return false; - } - const auto true_target = block.terminator.true_block; - const auto false_target = block.terminator.false_block; - if (block.id == loop->continue_block) { - const auto is_repeat_target = [&](uint32_t target) { - return target == loop->header || target == loop->merge; - }; - return is_repeat_target(true_target) && is_repeat_target(false_target); - } - const bool true_in_body = Contains(loop->body_blocks, true_target); - const bool false_in_body = Contains(loop->body_blocks, false_target); - if (true_in_body != false_in_body) { - return true; - } - const auto is_control_target = [&](uint32_t target) { - return target == loop->merge || target == loop->continue_block; - }; - return (is_control_target(true_target) && - (is_control_target(false_target) || - IsInsideLoopConstruct(graph, *loop, false_target))) || - (is_control_target(false_target) && IsInsideLoopConstruct(graph, *loop, true_target)); -} - -bool MergeLeavesContainingLoop(const Graph& graph, uint32_t header, uint32_t merge) { - for (const auto& loop: graph.natural_loops) { - if (loop.header != header && IsInsideLoopConstruct(graph, loop, header) && - !IsInsideLoopConstruct(graph, loop, merge)) { - return true; - } - } - return false; -} - -bool CanonicalizeNaturalLoops(Graph& graph) { - const auto rewrite_budget = graph.blocks.size() * 2u + 16u; - for (size_t rewrite = 0; rewrite < rewrite_budget; rewrite++) { - bool changed = false; - for (const auto& loop: graph.natural_loops) { - std::vector latches; - for (const auto& edge: graph.back_edges) { - if (edge.to == loop.header) { - AddUnique(latches, edge.from); - } - } - if (latches.size() <= 1u) { - continue; - } - - const auto continue_block = AppendSyntheticBranchBlock(graph, loop.header); - for (auto latch: latches) { - auto* block = graph.FindBlock(latch); - if (block != nullptr) { - ReplaceValue(block->successors, loop.header, continue_block); - ReplaceTerminatorTarget(block->terminator, loop.header, continue_block); - } - } - RebuildPredecessors(graph); - RecomputeAnalyses(graph); - changed = true; - break; - } - if (changed) { - continue; - } - - for (const auto& loop: graph.natural_loops) { - const auto* header = graph.FindBlock(loop.header); - const auto is_loop_control_target = [&](uint32_t target) { - return target == loop.merge || target == loop.continue_block; - }; - if (header == nullptr || header->terminator.kind != TerminatorKind::ConditionalBranch || - is_loop_control_target(header->terminator.true_block) || - is_loop_control_target(header->terminator.false_block) || - !Contains(loop.body_blocks, header->terminator.true_block) || - !Contains(loop.body_blocks, header->terminator.false_block)) { - continue; - } - - IsolateSemanticLoopHeader(graph, loop.header); - changed = true; - break; - } - if (!changed) { - return true; - } - } - - SetFailure(graph, FailureKind::StructuredControlFlow, graph.entry_block, - "CFG loop canonicalization exceeded rewrite budget"); - return false; -} - -bool IsolateSemanticLoopHeaders(Graph& graph) { - const auto isolation_budget = graph.natural_loops.size() + 1u; - for (size_t isolation = 0; isolation < isolation_budget; isolation++) { - const auto loop = std::find_if( - graph.natural_loops.begin(), graph.natural_loops.end(), [&](const auto& value) { - const auto* header = graph.FindBlock(value.header); - return header != nullptr && header->inst_begin != header->inst_end; - }); - if (loop == graph.natural_loops.end()) { - return true; - } - - // SPIR-V requires OpLoopMerge and its branch to remain in the loop header's - // physical block. Keep that header as a dedicated control node, exactly like - // a separate Loop node in a structured AST. Guest instructions live - // in the body so later translation may introduce bounds/EXEC control flow without - // displacing OpLoopMerge into a helper-created block. - if (!IsolateSemanticLoopHeader(graph, loop->header)) { - SetFailure(graph, FailureKind::StructuredControlFlow, loop->header, - fmt::format("failed to isolate semantic loop header {}", loop->header)); - return false; - } - } - - SetFailure(graph, FailureKind::StructuredControlFlow, graph.entry_block, - "CFG semantic loop-header isolation exceeded rewrite budget"); - return false; -} - -bool SplitSharedMergeBlock(Graph& graph, uint32_t merge, - const std::vector& construct_blocks, - bool force_split = false) { - if (merge == UINT32_MAX || merge >= graph.blocks.size() || Contains(construct_blocks, merge)) { - return false; - } - - const auto* merge_block = graph.FindBlock(merge); - if (merge_block == nullptr) { - return false; - } - - std::vector construct_predecessors; - bool has_external_predecessor = false; - for (auto pred: merge_block->predecessors) { - if (Contains(construct_blocks, pred)) { - AddUnique(construct_predecessors, pred); - } else { - has_external_predecessor = true; - } - } - - if (construct_predecessors.empty() || (!force_split && !has_external_predecessor)) { - return false; - } - if (construct_predecessors.size() == 1u) { - const auto* predecessor = graph.FindBlock(construct_predecessors.front()); - if (predecessor != nullptr && predecessor->inst_begin == predecessor->inst_end && - predecessor->terminator.kind == TerminatorKind::Branch && - predecessor->terminator.true_block == merge && - predecessor->start_pc == merge_block->start_pc) { - return false; - } - } - const auto synthetic_merge = AppendSyntheticBranchBlock(graph, merge); - auto* synthetic_block = graph.FindBlock(synthetic_merge); - if (synthetic_block != nullptr) { - synthetic_block->predecessors = construct_predecessors; - SortUnique(synthetic_block->predecessors); - } - - for (auto pred: construct_predecessors) { - auto* block = graph.FindBlock(pred); - if (block == nullptr) { - continue; - } - ReplaceValue(block->successors, merge, synthetic_merge); - ReplaceTerminatorTarget(block->terminator, merge, synthetic_merge); - } - - auto* old_merge = graph.FindBlock(merge); - if (old_merge != nullptr) { - for (auto pred: construct_predecessors) { - RemoveValue(old_merge->predecessors, pred); - } - AddUnique(old_merge->predecessors, synthetic_merge); - SortUnique(old_merge->predecessors); - } - - MoveBlockBefore(graph, synthetic_merge, merge); - return true; -} - -bool SplitOneLoopMerge(Graph& graph) { - const auto& loops = graph.natural_loops; - for (const auto& loop: loops) { - const auto construct_blocks = DominatedBlocks(graph, loop.header, loop.merge); - const auto force_split = MergeLeavesContainingLoop(graph, loop.header, loop.merge); - if (SplitSharedMergeBlock(graph, loop.merge, construct_blocks, force_split)) { - return true; - } - } - return false; -} - -std::vector SelectionRegion(const Graph& graph, const BasicBlock& header, - uint32_t merge) { - std::vector region; - std::vector pending = {header.terminator.true_block, header.terminator.false_block}; - const auto* loop = FindInnermostContainingLoop(graph, header.id); - while (!pending.empty()) { - const auto block_id = pending.back(); - pending.pop_back(); - if (block_id == merge || Contains(region, block_id) || - (loop != nullptr && (block_id == loop->merge || block_id == loop->continue_block))) { - continue; - } - const auto* block = graph.FindBlock(block_id); - if (block == nullptr) { - continue; - } - // A return terminates its own block; a branch to a shared return still has to - // obey selection entry/exit rules, just like any other branch. - AddUnique(region, block_id); - pending.insert(pending.end(), block->successors.begin(), block->successors.end()); - } - SortUnique(region); - return region; -} - -bool SplitOneSelectionMerge(Graph& graph) { - std::vector loop_headers; - loop_headers.reserve(graph.natural_loops.size()); - for (const auto& loop: graph.natural_loops) { - AddUnique(loop_headers, loop.header); - } - - std::vector selection_headers; - for (const auto& block: graph.blocks) { - if (block.terminator.kind == TerminatorKind::ConditionalBranch && - !Contains(loop_headers, block.id)) { - selection_headers.push_back(block.id); - } - } - std::sort(selection_headers.begin(), selection_headers.end(), [&](uint32_t lhs, uint32_t rhs) { - const auto* lhs_block = graph.FindBlock(lhs); - const auto* rhs_block = graph.FindBlock(rhs); - const auto lhs_depth = lhs_block != nullptr ? lhs_block->dominators.size() : 0u; - const auto rhs_depth = rhs_block != nullptr ? rhs_block->dominators.size() : 0u; - return lhs_depth != rhs_depth ? lhs_depth > rhs_depth : lhs < rhs; - }); - - for (const auto block_id: selection_headers) { - const auto* block = graph.FindBlock(block_id); - if (block == nullptr) { - continue; - } - if (IsInnermostLoopControlConditional(graph, *block)) { - continue; - } - - const auto merge = FindSelectionMerge(graph, *block); - if (merge == UINT32_MAX || graph.FindBlock(merge) == nullptr) { - continue; - } - const auto region = SelectionRegion(graph, *block, merge); - const auto external = std::find_if(region.begin(), region.end(), [&](uint32_t member) { - const auto* member_block = graph.FindBlock(member); - return member_block != nullptr && - std::ranges::any_of(member_block->predecessors, [&](uint32_t predecessor) { - return predecessor != block_id && !Contains(region, predecessor); - }); - }); - if (external != region.end()) { - SetFailure( - graph, FailureKind::StructuredControlFlow, block_id, - fmt::format("selection header block {} has externally entered region block {}; " - "semantic block cloning is disabled", - block_id, *external)); - return false; - } - const auto construct_blocks = DominatedBlocks(graph, block_id, merge); - const auto force_split = MergeLeavesContainingLoop(graph, block_id, merge); - if (SplitSharedMergeBlock(graph, merge, construct_blocks, force_split)) { - return true; - } - } - return false; -} - -bool SplitSharedMergeBlocks(Graph& graph) { - const auto original_block_count = static_cast(graph.blocks.size()); - const auto split_budget = std::max(16u, original_block_count * 4u); - for (uint32_t splits = 0; splits < split_budget; splits++) { - if (!SplitOneLoopMerge(graph) && !SplitOneSelectionMerge(graph)) { - return !graph.unsupported; - } - RebuildPredecessors(graph); - RecomputeAnalyses(graph); - } - SetFailure(graph, FailureKind::StructuredControlFlow, graph.entry_block, - fmt::format("CFG shared merge splitting exceeded budget: original_blocks={} " - "current_blocks={} split_budget={}", - original_block_count, static_cast(graph.blocks.size()), - split_budget)); - return false; -} - -void ClearStructuredTerminators(Graph& graph) { - for (auto& block: graph.blocks) { - block.terminator.merge_block = UINT32_MAX; - block.terminator.continue_block = UINT32_MAX; - block.terminator.loop_header = false; - } -} - std::string VectorToString(const std::vector& values) { std::string text; for (uint32_t i = 0; i < values.size(); i++) { @@ -1627,262 +1039,493 @@ uint32_t Graph::FindNearestCommonPostDominator(uint32_t block_a, uint32_t block_ namespace { -struct GotoRouteBlocks { - uint32_t before = UINT32_MAX; - std::vector blocks; -}; +// Eliminate gotos by moving them through a structured statement tree. Every Code node +// retains exactly one guest block; only Boolean route values cross construct boundaries. +// This follows the outward/inward/lifting transformations used by shadPS4's GotoPass. +class GotoStructurizer { + using ExprOp = ConditionExpression::Op; + enum class Kind { Root, Code, Label, Goto, If, Loop, Break, Return, Set }; + struct Node; + using List = std::list; + struct Node { + Kind kind; + Node* parent = nullptr; + Node* target = nullptr; + List children; + List::iterator position; + uint32_t condition = 1; + uint32_t id = UINT32_MAX; + }; -void PlaceGotoRouteBlocks(Graph& graph, uint32_t original_block_count, - const GotoRouteBlocks& route) { - std::vector old_blocks = std::move(graph.blocks); - std::vector new_blocks; - new_blocks.reserve(old_blocks.size()); - for (uint32_t block_id = 0; block_id < original_block_count; block_id++) { - if (block_id == route.before) { - for (const auto route_block: route.blocks) { - new_blocks.push_back(std::move(old_blocks[route_block])); - } +public: + explicit GotoStructurizer(const Graph& source): m_source(source) { + m_graph.expressions = {{ExprOp::Constant, 0}, {ExprOp::Constant, 1}}; + m_graph.code_table_load_pcs = source.code_table_load_pcs; + m_root = New(Kind::Root); + std::vector labels; + for (const auto& block: source.blocks) { + auto* label = Insert(m_root, m_root->children.end(), Kind::Label); + label->id = block.id; + labels.push_back(label); } - new_blocks.push_back(std::move(old_blocks[block_id])); - } - - ApplyBlockOrder(graph, std::move(new_blocks)); -} - -uint32_t AppendGotoSelectBlock(Graph& graph, uint32_t route_variable, uint32_t true_target, - uint32_t false_target) { - const auto* target = graph.FindBlock(false_target); - BasicBlock block; - block.id = static_cast(graph.blocks.size()); - block.start_pc = target != nullptr ? target->start_pc : 0u; - block.end_pc = block.start_pc; - block.inst_begin = target != nullptr ? target->inst_begin : 0u; - block.inst_end = block.inst_begin; - block.successors = {true_target, false_target}; - block.terminator.kind = TerminatorKind::ConditionalBranch; - block.terminator.condition = BranchCondition::GotoVariable; - block.terminator.true_block = true_target; - block.terminator.false_block = false_target; - block.terminator.goto_variable = route_variable; - graph.blocks.push_back(std::move(block)); - return graph.blocks.back().id; -} - -uint32_t AppendGotoSetBlock(Graph& graph, uint32_t route_variable, bool value, uint32_t target) { - const auto block_id = AppendSyntheticBranchBlock(graph, target); - auto* block = graph.FindBlock(block_id); - block->terminator.goto_variable = route_variable; - block->terminator.goto_value = value ? 1 : 0; - return block_id; -} - -void AppendEnclosingRouteBlocks(Graph& graph, uint32_t original_block_count, - uint32_t route_variable, uint32_t shared, uint32_t other, - uint32_t route_select, uint32_t route_root, - std::vector& route_blocks) { - for (uint32_t depth = 0; depth < original_block_count; depth++) { - uint32_t predecessor = UINT32_MAX; - uint32_t leaf = UINT32_MAX; - for (uint32_t candidate = 0; candidate < original_block_count; candidate++) { - const auto* candidate_block = graph.FindBlock(candidate); - if (candidate_block == nullptr || - candidate_block->terminator.kind != TerminatorKind::ConditionalBranch || - IsInnermostLoopControlConditional(graph, *candidate_block)) { - continue; - } - const auto candidate_true = candidate_block->terminator.true_block; - const auto candidate_false = candidate_block->terminator.false_block; - if (candidate_true == route_root && - (candidate_false == shared || candidate_false == other)) { - predecessor = candidate; - leaf = candidate_false; - break; - } - if (candidate_false == route_root && - (candidate_true == shared || candidate_true == other)) { - predecessor = candidate; - leaf = candidate_true; - break; + // Give every natural loop one backedge before eliminating gotos. Otherwise + // multiple latches become nested loops that retest the same exit predicate. + struct Continue { uint32_t header; uint32_t after; size_t body_size; Node* label; }; + std::vector continues; + for (const auto& loop: source.natural_loops) { + if (std::ranges::any_of(continues, [&](const auto& value) { return value.header == loop.header; }) || + std::ranges::count(source.natural_loops, loop.header, &NaturalLoop::header) < 2) continue; + std::vector body; + for (const auto& member: source.natural_loops) { + if (member.header == loop.header) + body.insert(body.end(), member.body_blocks.begin(), member.body_blocks.end()); } + SortUnique(body); + auto* label = New(Kind::Label); + label->id = static_cast(labels.size()); + labels.push_back(label); + continues.push_back({loop.header, body.back(), body.size(), label}); } - if (predecessor == UINT32_MAX) { - break; - } - - const auto routed_leaf = - AppendGotoSetBlock(graph, route_variable, leaf == other, route_select); - auto* predecessor_block = graph.FindBlock(predecessor); - ReplaceValue(predecessor_block->successors, leaf, routed_leaf); - ReplaceTerminatorTarget(predecessor_block->terminator, leaf, routed_leaf); - route_blocks.push_back(routed_leaf); - route_root = predecessor; - } -} - -// Turn H0 -> shared/H1, H1 -> shared/other into nested selections whose empty -// forwarding blocks set one typed SSA route value. Guest semantic blocks remain unique. -// Preserve loop-control branches just as SplitOneSelectionMerge does. Moving a loop -// exit through the route selector can create an inner cycle whose merge is also the -// enclosing loop merge, so splitting that shared exit can never separate the loops. -bool RouteOneSharedArm(Graph& graph, uint32_t original_block_count, uint32_t outer_id, - uint32_t route_variable, GotoRouteBlocks& route) { - auto* outer = graph.FindBlock(outer_id); - if (outer == nullptr || outer->inst_begin == outer->inst_end || - outer->terminator.kind != TerminatorKind::ConditionalBranch || - IsInnermostLoopControlConditional(graph, *outer)) { - return false; - } - - for (const bool inner_is_true: {true, false}) { - const auto inner_id = - inner_is_true ? outer->terminator.true_block : outer->terminator.false_block; - const auto shared = - inner_is_true ? outer->terminator.false_block : outer->terminator.true_block; - const auto* inner = graph.FindBlock(inner_id); - if (inner == nullptr || inner->inst_begin == inner->inst_end || - inner->predecessors.size() != 1u || inner->predecessors.front() != outer_id || - inner->terminator.kind != TerminatorKind::ConditionalBranch || - IsInnermostLoopControlConditional(graph, *inner)) { - continue; - } - - const bool true_is_shared = inner->terminator.true_block == shared; - const bool false_is_shared = inner->terminator.false_block == shared; - if (true_is_shared != false_is_shared) { - const auto other = - true_is_shared ? inner->terminator.false_block : inner->terminator.true_block; - if (other == outer_id || other == inner_id || other == shared) { - continue; - } - const auto first_arm = std::min(shared, other); - if (first_arm >= original_block_count || outer_id >= inner_id || - inner_id >= first_arm) { - continue; - } - - const auto route_select = AppendGotoSelectBlock(graph, route_variable, other, shared); - const auto inner_merge = AppendSyntheticBranchBlock(graph, route_select); - const auto outer_shared = - AppendGotoSetBlock(graph, route_variable, false, route_select); - const auto inner_shared = AppendGotoSetBlock(graph, route_variable, false, inner_merge); - const auto inner_other = AppendGotoSetBlock(graph, route_variable, true, inner_merge); - - outer = graph.FindBlock(outer_id); - auto* mutable_inner = graph.FindBlock(inner_id); - outer->terminator.true_block = inner_is_true ? inner_id : outer_shared; - outer->terminator.false_block = inner_is_true ? outer_shared : inner_id; - outer->successors = {outer->terminator.true_block, outer->terminator.false_block}; - mutable_inner->terminator.true_block = true_is_shared ? inner_shared : inner_other; - mutable_inner->terminator.false_block = true_is_shared ? inner_other : inner_shared; - mutable_inner->successors = {mutable_inner->terminator.true_block, - mutable_inner->terminator.false_block}; - - std::vector route_blocks = {inner_shared, inner_other, inner_merge, - outer_shared}; - AppendEnclosingRouteBlocks(graph, original_block_count, route_variable, shared, other, - route_select, outer_id, route_blocks); - route_blocks.push_back(route_select); - route = {first_arm, std::move(route_blocks)}; - return true; - } - - // H0 -> arm/H1, H1 -> exit/body, and both arms reach a common continuation. - // Route exit/continuation after joining H0 and H1 so neither has a nonlocal edge. - for (const bool exit_is_true: {true, false}) { - const auto other = - exit_is_true ? inner->terminator.true_block : inner->terminator.false_block; - const auto body = - exit_is_true ? inner->terminator.false_block : inner->terminator.true_block; - if (other == outer_id || other == inner_id || other == shared || body == outer_id || - body == inner_id || body == shared || !graph.Dominates(inner_id, body)) { - continue; - } - - const auto continuation = graph.FindNearestCommonPostDominator(shared, body); - const auto* continuation_block = graph.FindBlock(continuation); - if (continuation_block == nullptr || continuation == other || - CanReachBefore(graph, other, continuation, UINT32_MAX)) { - continue; - } - std::vector outer_predecessors; - std::vector inner_predecessors; - bool external_predecessor = false; - for (const auto predecessor: continuation_block->predecessors) { - if (predecessor == outer_id || graph.Dominates(shared, predecessor)) { - outer_predecessors.push_back(predecessor); - } else if (predecessor == inner_id || graph.Dominates(body, predecessor)) { - inner_predecessors.push_back(predecessor); - } else { - external_predecessor = true; + std::ranges::sort(continues, [](const auto& a, const auto& b) { + return a.after != b.after ? a.after < b.after : a.body_size < b.body_size; + }); + m_route_expressions.resize(labels.size(), UINT32_MAX); + for (const auto& block: source.blocks) { + auto* label = labels.at(block.id); + const auto end = std::next(label->position); + auto* code = Insert(m_root, end, Kind::Code); + code->id = block.id; + const auto add_goto = [&](uint32_t condition, uint32_t target) { + auto* node = Insert(m_root, end, Kind::Goto); + node->condition = condition; + node->target = labels.at(target); + for (const auto& value: continues) { + if (value.header == target && source.Dominates(target, block.id)) + node->target = value.label; } - } - const auto first_arm = std::min(continuation, other); - if (outer_predecessors.empty() || inner_predecessors.empty() || - external_predecessor || first_arm >= original_block_count || - outer_id >= inner_id || inner_id >= first_arm) { - continue; - } - - const auto route_select = - AppendGotoSelectBlock(graph, route_variable, other, continuation); - const auto inner_merge = AppendSyntheticBranchBlock(graph, route_select); - const auto outer_continue = - AppendGotoSetBlock(graph, route_variable, false, route_select); - const auto inner_continue = - AppendGotoSetBlock(graph, route_variable, false, inner_merge); - const auto inner_other = AppendGotoSetBlock(graph, route_variable, true, inner_merge); - - auto* mutable_inner = graph.FindBlock(inner_id); - if (exit_is_true) { - mutable_inner->terminator.true_block = inner_other; + m_gotos.push_back(node); + }; + const auto& term = block.terminator; + if (term.kind == TerminatorKind::ConditionalBranch) { + // Capture at the semantic block, before movement can pass an SCC/EXEC write. + const auto condition = Expression(ExprOp::Variable, CaptureVariable(block.id)); + add_goto(condition, term.true_block); + add_goto(1, term.false_block); + } else if (term.kind == TerminatorKind::Branch) { + add_goto(1, term.true_block); + } else if (term.kind == TerminatorKind::Return) { + Insert(m_root, end, Kind::Return); } else { - mutable_inner->terminator.false_block = inner_other; + Fail("indirect control flow requires dispatcher lowering"); } - ReplaceValue(mutable_inner->successors, other, inner_other); - for (const auto predecessor: outer_predecessors) { - auto* predecessor_block = graph.FindBlock(predecessor); - ReplaceValue(predecessor_block->successors, continuation, outer_continue); - ReplaceTerminatorTarget(predecessor_block->terminator, continuation, - outer_continue); + for (const auto& value: continues) { + if (value.after != block.id) continue; + value.label->parent = m_root; + value.label->position = m_root->children.insert(end, value.label); + auto* repeat = Insert(m_root, end, Kind::Goto); + repeat->target = labels[value.header]; + m_gotos.push_back(repeat); } - for (const auto predecessor: inner_predecessors) { - auto* predecessor_block = graph.FindBlock(predecessor); - ReplaceValue(predecessor_block->successors, continuation, inner_continue); - ReplaceTerminatorTarget(predecessor_block->terminator, continuation, - inner_continue); - } - - std::vector route_blocks = {inner_continue, inner_other, inner_merge, - outer_continue}; - if (continuation == shared) { - AppendEnclosingRouteBlocks(graph, original_block_count, route_variable, shared, - other, route_select, outer_id, route_blocks); - } - route_blocks.push_back(route_select); - route = {first_arm, std::move(route_blocks)}; - return true; } } - return false; -} -bool RouteSharedSelectionArm(Graph& graph, uint32_t route_variable) { - if (graph.irreducible) { + Graph Run() { + for (auto it = m_gotos.rbegin(); it != m_gotos.rend() && m_failure == nullptr; ++it) { + Eliminate(*it); + } + if (m_failure != nullptr) return Failed(); + const auto entry = Allocate(); + m_graph.entry_block = entry; + Place(entry); + Branch(Visit(m_root, entry, UINT32_MAX), UINT32_MAX); + if (m_failure != nullptr) return Failed(); + std::vector ordered; + ordered.reserve(m_order.size()); + for (const auto id: m_order) { + ordered.push_back(std::move(m_graph.blocks[id])); + } + ApplyBlockOrder(m_graph, std::move(ordered)); + RebuildPredecessors(m_graph); + if (std::ranges::any_of(m_graph.blocks, [&](const auto& block) { + return block.id != m_graph.entry_block && block.predecessors.empty(); + })) { + Fail("structured control flow requires an unreachable merge"); + return Failed(); + } + RecomputeAnalyses(m_graph); + if (m_graph.irreducible) { + Fail("goto elimination produced irreducible control flow"); + return Failed(); + } + return std::move(m_graph); + } + +private: + void Fail(const char* reason) { m_failure = reason; } + Graph Failed() { + SetFailure(m_graph, FailureKind::StructuredControlFlow, UINT32_MAX, m_failure); + return std::move(m_graph); + } + uint32_t CaptureVariable(uint32_t block) const { + return static_cast(m_route_expressions.size()) + block; + } + Node* New(Kind kind) { + m_nodes.emplace_back(); + m_nodes.back().kind = kind; + return &m_nodes.back(); + } + Node* Insert(Node* parent, List::iterator before, Kind kind) { + auto* node = New(kind); + node->parent = parent; + node->position = parent->children.insert(before, node); + return node; + } + void Erase(Node* node) { node->parent->children.erase(node->position); } + uint32_t Expression(ExprOp op, uint32_t lhs, uint32_t rhs = UINT32_MAX) { + if (op == ExprOp::Not) { + const auto& inner = m_graph.expressions[lhs]; + if (inner.op == ExprOp::Constant) return inner.lhs ? 0 : 1; + if (inner.op == ExprOp::Not) return inner.lhs; + } + if (op == ExprOp::Or) { + if (lhs == 1 || rhs == 1) return 1; + if (lhs == 0 || lhs == rhs) return rhs; + if (rhs == 0) return lhs; + } + const auto result = static_cast(m_graph.expressions.size()); + m_graph.expressions.push_back({op, lhs, rhs}); + return result; + } + uint32_t RouteVariable(Node* label) { + auto& expression = m_route_expressions[label->id]; + if (expression == UINT32_MAX) { + expression = Expression(ExprOp::Variable, label->id); + // SSA initializes route variables to false; clear them again on each label visit. + auto* reset = Insert(label->parent, std::next(label->position), Kind::Set); + reset->id = label->id; + reset->condition = 0; + } + return expression; + } + static size_t Depth(Node* node) { + size_t depth = 0; + for (; node->parent != nullptr; node = node->parent) ++depth; + return depth; + } + static bool Related(Node* a, Node* b) { + auto a_depth = Depth(a); + auto b_depth = Depth(b); + while (a_depth > b_depth) { a = a->parent; --a_depth; } + while (b_depth > a_depth) { b = b->parent; --b_depth; } + return a->parent == b->parent; + } + static bool Ordered(Node* a, Node* b) { + for (auto it = a->position; it != a->parent->children.end(); ++it) { + if (*it == b) return true; + } return false; } - const auto block_count = static_cast(graph.blocks.size()); - // Route later/deeper candidates first so earlier shared arms remain structured. - for (uint32_t block_id = block_count; block_id-- > 0;) { - GotoRouteBlocks route; - if (RouteOneSharedArm(graph, block_count, block_id, route_variable, route)) { - PlaceGotoRouteBlocks(graph, block_count, route); - RebuildPredecessors(graph); - RecomputeAnalyses(graph); - return true; + static Node* Sibling(Node* uncle, Node* nephew) { + while (nephew->parent != uncle->parent) nephew = nephew->parent; + return nephew; + } + static void Adopt(Node* parent) { + for (auto* child: parent->children) child->parent = parent; + } + void SetBefore(Node* node, uint32_t id, uint32_t expression) { + auto* set = Insert(node->parent, node->position, Kind::Set); + set->id = id; + set->condition = expression; + } + Node* GotoAfter(Node* construct, Node* label, uint32_t condition) { + auto* node = Insert(construct->parent, std::next(construct->position), Kind::Goto); + node->target = label; + node->condition = condition; + return node; + } + Node* Outward(Node* node) { + auto* parent = node->parent; + auto* target = node->target; + const auto variable = RouteVariable(target); + SetBefore(node, target->id, node->condition); + if (parent->kind == Kind::If) { + auto* rest = Insert(parent, node->position, Kind::If); + rest->condition = Expression(ExprOp::Not, variable); + rest->children.splice(rest->children.end(), parent->children, + std::next(node->position), parent->children.end()); + Adopt(rest); + } else if (parent->kind == Kind::Loop) { + auto* stop = Insert(parent, node->position, Kind::Break); + stop->condition = variable; + } else { + Fail("goto has no enclosing selection or loop"); + return node; + } + Erase(node); + return GotoAfter(parent, target, variable); + } + Node* Inward(Node* node) { + auto* parent = node->parent; + auto* target = node->target; + auto* nested = Sibling(node, target); + const auto variable = RouteVariable(target); + SetBefore(node, target->id, node->condition); + auto* rest = Insert(parent, node->position, Kind::If); + rest->condition = Expression(ExprOp::Not, variable); + rest->children.splice(rest->children.end(), parent->children, + std::next(node->position), nested->position); + Adopt(rest); + Erase(node); + if (nested->kind == Kind::If) { + nested->condition = Expression(ExprOp::Or, variable, nested->condition); + } else if (nested->kind != Kind::Loop) { + Fail("goto target has no enclosing construct"); + return node; + } + auto* moved = Insert(nested, nested->children.begin(), Kind::Goto); + moved->target = target; + moved->condition = variable; + return moved; + } + Node* Lift(Node* node) { + auto* parent = node->parent; + auto* target = node->target; + auto* nested = Sibling(node, target); + const auto variable = RouteVariable(target); + auto* loop = Insert(parent, node->position, Kind::Loop); + loop->condition = variable; + loop->children.splice(loop->children.end(), parent->children, + nested->position, loop->position); + Adopt(loop); + if (std::ranges::any_of(loop->children, [](auto* child) { return child->kind == Kind::Break; })) { + Fail("goto lifting would capture an enclosing loop exit"); + return node; + } + auto* moved = Insert(loop, loop->children.begin(), Kind::Goto); + moved->target = target; + moved->condition = variable; + auto* set = Insert(loop, loop->children.end(), Kind::Set); + set->id = target->id; + set->condition = node->condition; + Erase(node); + return moved; + } + void Eliminate(Node* node) { + auto* target = node->target; + while (node->parent != target->parent && m_failure == nullptr) { + if (!Related(node, target) || Depth(node) > Depth(target)) { + node = Outward(node); + } else if (Ordered(Sibling(node, target), node)) { + node = Lift(node); + } else { + node = Inward(node); + } + } + if (m_failure != nullptr) return; + if (std::next(node->position) == target->position) { + Erase(node); + return; + } + auto* parent = node->parent; + if (Ordered(node, target)) { + auto* selection = Insert(parent, node->position, Kind::If); + selection->condition = Expression(ExprOp::Not, node->condition); + selection->children.splice(selection->children.end(), parent->children, + std::next(node->position), target->position); + Adopt(selection); + } else { + auto* loop = Insert(parent, node->position, Kind::Loop); + loop->condition = node->condition; + loop->children.splice(loop->children.end(), parent->children, + target->position, loop->position); + Adopt(loop); + } + Erase(node); + } + + uint32_t Allocate() { + BasicBlock block; + block.id = static_cast(m_graph.blocks.size()); + m_graph.blocks.push_back(std::move(block)); + return m_graph.blocks.back().id; + } + void Place(uint32_t block) { m_order.push_back(block); } + void Branch(uint32_t from, uint32_t to) { + if (from == UINT32_MAX) return; + auto& block = m_graph.blocks[from]; + block.terminator.kind = to == UINT32_MAX ? TerminatorKind::Return : TerminatorKind::Branch; + block.terminator.true_block = to; + block.successors = to == UINT32_MAX ? std::vector {} : std::vector {to}; + } + void Conditional(uint32_t block_id, uint32_t condition, uint32_t yes, uint32_t no, + uint32_t merge = UINT32_MAX) { + auto& block = m_graph.blocks[block_id]; + auto& term = block.terminator; + term.kind = TerminatorKind::ConditionalBranch; + term.expression = condition; + term.condition = BranchCondition::Expression; + term.true_block = yes; + term.false_block = no; + term.merge_block = merge; + block.successors = {yes, no}; + } + uint32_t Advance(uint32_t current) { + const auto next = Allocate(); + Branch(current, next); + Place(next); + return next; + } + bool RefersTo(uint32_t expression, uint32_t variable) const { + const auto& value = m_graph.expressions[expression]; + switch (value.op) { + case ExprOp::Variable: return value.lhs == variable; + case ExprOp::Not: return RefersTo(value.lhs, variable); + case ExprOp::Or: return RefersTo(value.lhs, variable) || RefersTo(value.rhs, variable); + default: return false; } } - return false; -} + bool Modifies(Node* node, uint32_t expression) const { + if (node->kind == Kind::Set && RefersTo(expression, node->id)) return true; + if (node->kind == Kind::Code && + m_source.blocks[node->id].terminator.kind == TerminatorKind::ConditionalBranch && + RefersTo(expression, CaptureVariable(node->id))) return true; + return std::ranges::any_of(node->children, [&](auto* child) { return Modifies(child, expression); }); + } + bool Complements(uint32_t first, uint32_t second) const { + const auto& a = m_graph.expressions[first]; + const auto& b = m_graph.expressions[second]; + return (a.op == ExprOp::Not && a.lhs == second) || + (b.op == ExprOp::Not && b.lhs == first); + } + uint32_t Visit(Node* parent, uint32_t current, uint32_t break_block) { + for (auto it = parent->children.begin(); it != parent->children.end(); ++it) { + auto* node = *it; + if (node->kind == Kind::Label) continue; + if (current == UINT32_MAX) { + current = Allocate(); + Place(current); + } + switch (node->kind) { + case Kind::Code: { + const auto& source = m_source.blocks[node->id]; + const auto& previous = m_graph.blocks[current]; + // Route assignments cannot observe guest writes; native predicates must stay at their original Code. + const bool captures_native = std::ranges::any_of(previous.assignments, [&](const auto& assignment) { + return m_graph.expressions[assignment.expression].op == ExprOp::Native; + }); + if (captures_native || + (previous.inst_begin != previous.inst_end && previous.inst_end != source.inst_begin)) { + current = Advance(current); + } + auto& block = m_graph.blocks[current]; + if (block.inst_begin == block.inst_end) { + block.start_pc = source.start_pc; + block.inst_begin = source.inst_begin; + } + block.end_pc = source.end_pc; + block.inst_end = source.inst_end; + if (source.terminator.kind == TerminatorKind::ConditionalBranch) { + block.assignments.push_back({CaptureVariable(source.id), + Expression(ExprOp::Native, static_cast(source.terminator.condition))}); + } + break; + } + case Kind::Set: { + m_graph.blocks[current].assignments.push_back({node->id, node->condition}); + break; + } + case Kind::If: { + if (node->condition == 0 || node->children.empty()) break; + if (node->condition == 1) { + current = Visit(node, current, break_block); + break; + } + auto next = std::next(it); + while (next != parent->children.end() && (*next)->kind == Kind::Label) ++next; + Node* other = next != parent->children.end() ? *next : nullptr; + const bool paired = other != nullptr && other->kind == Kind::If && + Complements(node->condition, other->condition) && + !Modifies(node, node->condition); + const auto body = Allocate(); + const auto merge = Allocate(); + const auto alternative = paired ? Allocate() : merge; + Conditional(current, node->condition, body, alternative, merge); + Place(body); + Branch(Visit(node, body, break_block), merge); + if (paired) { + Place(alternative); + Branch(Visit(other, alternative, break_block), merge); + it = next; + } + Place(merge); + current = merge; + break; + } + case Kind::Loop: { + const auto& previous = m_graph.blocks[current]; + const bool empty = previous.inst_begin == previous.inst_end && + previous.assignments.empty(); + const auto header = empty ? current : Advance(current); + const auto body = Allocate(); + const auto merge = Allocate(); + Branch(header, body); + Place(body); + auto repeat = Visit(node, body, merge); + // A construct merge belongs to the loop body, not its continue construct. + if (repeat == UINT32_MAX || std::ranges::any_of(m_graph.blocks, [&](const auto& block) { + return block.terminator.merge_block == repeat; + })) { + repeat = Advance(repeat); + } + auto& term = m_graph.blocks[header].terminator; + term.loop_header = true; + term.merge_block = merge; + term.continue_block = repeat; + if (node->condition <= 1) Branch(repeat, node->condition ? header : merge); + else Conditional(repeat, node->condition, header, merge); + Place(merge); + current = merge; + break; + } + case Kind::Break: { + if (break_block == UINT32_MAX) { + Fail("break outside a loop"); + return UINT32_MAX; + } + if (node->condition == 0) break; + if (node->condition == 1) { + Branch(current, break_block); + current = UINT32_MAX; + } else { + const auto next = Allocate(); + Conditional(current, node->condition, break_block, next); + Place(next); + current = next; + } + break; + } + case Kind::Return: + Branch(current, UINT32_MAX); + current = UINT32_MAX; + break; + default: + Fail("uneliminated control-flow statement"); + return UINT32_MAX; + } + if (m_failure != nullptr) return UINT32_MAX; + } + return current; + } + + const char* m_failure = nullptr; + const Graph& m_source; + Graph m_graph; + std::deque m_nodes; + Node* m_root = nullptr; + std::vector m_gotos; + std::vector m_route_expressions; + std::vector m_order; +}; } // namespace @@ -2115,125 +1758,16 @@ Graph BuildGraph(const Decoder::Program& program) { return graph; } -namespace { - -bool StructurizeImpl(Graph& graph) { - if (graph.unsupported || graph.irreducible) { - if (graph.unsupported_reason.empty()) { - graph.unsupported_reason = "unsupported CFG"; - } - return false; - } - - if (!CanonicalizeNaturalLoops(graph)) { - return false; - } - if (!SplitSharedMergeBlocks(graph)) { - return false; - } - if (!IsolateSemanticLoopHeaders(graph)) { - return false; - } - ClearStructuredTerminators(graph); - - std::map merge_headers; - auto reserve_merge_block = [&](uint32_t header, uint32_t merge) { - const auto [it, inserted] = merge_headers.emplace(merge, header); - if (!inserted && it->second != header) { - SetFailure( - graph, FailureKind::StructuredControlFlow, header, - fmt::format( - "duplicate structured merge block {} for header {} (already used by header {})", - merge, header, it->second)); - return false; - } - return true; - }; - - for (const auto& loop: graph.natural_loops) { - auto* header = graph.FindBlock(loop.header); - if (header == nullptr || loop.merge == UINT32_MAX || loop.continue_block == UINT32_MAX) { - SetFailure( - graph, FailureKind::StructuredControlFlow, loop.header, - fmt::format("loop at block {} has no structured merge/continue", loop.header)); - return false; - } - if (header->inst_begin != header->inst_end || - header->terminator.kind != TerminatorKind::Branch) { - SetFailure( - graph, FailureKind::StructuredControlFlow, loop.header, - fmt::format("loop header {} is not a dedicated empty control block", loop.header)); - return false; - } - if (!reserve_merge_block(loop.header, loop.merge)) { - return false; - } - header->terminator.loop_header = true; - header->terminator.merge_block = loop.merge; - header->terminator.continue_block = loop.continue_block; - } - - for (auto& block: graph.blocks) { - if (block.terminator.kind != TerminatorKind::ConditionalBranch || - block.terminator.loop_header) { - continue; - } - if (IsInnermostLoopControlConditional(graph, block)) { - continue; - } - - const auto merge = FindSelectionMerge(graph, block); - if (merge == UINT32_MAX) { - SetFailure(graph, FailureKind::StructuredControlFlow, block.id, - fmt::format("conditional block {} has no structured merge", block.id)); - return false; - } - - if (!reserve_merge_block(block.id, merge)) { - return false; - } - block.terminator.merge_block = merge; - } - - return true; -} - -} // namespace - bool Structurize(Graph& graph) { - Graph structured = graph; - if (StructurizeImpl(structured)) { - graph = std::move(structured); - return true; + if (graph.unsupported || graph.irreducible) return false; + auto structured = GotoStructurizer(graph).Run(); + if (structured.unsupported) { + SetFailure(graph, structured.failure_kind, structured.failure_block, + structured.unsupported_reason); + return false; } - - const auto failure_kind = structured.failure_kind; - // Structurization inserts and renumbers blocks. Recover source identity for a - // semantic block; a synthetic block has no corresponding original diagnostic ID. - const auto* failed = structured.FindBlock(structured.failure_block); - const auto original = std::ranges::find_if(graph.blocks, [&](const BasicBlock& block) { - return failed != nullptr && failed->inst_begin != failed->inst_end && - block.inst_begin == failed->inst_begin && block.inst_end == failed->inst_end && - block.start_pc == failed->start_pc && block.end_pc == failed->end_pc; - }); - const auto failure_block = original != graph.blocks.end() ? original->id : UINT32_MAX; - auto failure_reason = std::move(structured.unsupported_reason); - Graph routed = graph; - const auto route_budget = static_cast(graph.blocks.size()); - // Apply one route at a time and retry. Eagerly routing every matching diamond can - // rewrite unrelated selections that were already structurally valid. - for (uint32_t route_variable = 0; route_variable < route_budget; route_variable++) { - if (!RouteSharedSelectionArm(routed, route_variable)) { - break; - } - structured = routed; - if (StructurizeImpl(structured)) { - graph = std::move(structured); - return true; - } - } - SetFailure(graph, failure_kind, failure_block, failure_reason); - return false; + graph = std::move(structured); + return true; } std::string BranchConditionToString(BranchCondition condition) { @@ -2246,7 +1780,7 @@ std::string BranchConditionToString(BranchCondition condition) { case BranchCondition::ExecZero: return "execz"; case BranchCondition::ExecNonZero: return "execnz"; case BranchCondition::ScalarInstruction: return "scalar_instruction"; - case BranchCondition::GotoVariable: return "goto_variable"; + case BranchCondition::Expression: return "expression"; default: return "unknown"; } } @@ -2289,7 +1823,7 @@ std::string GraphToString(const Graph& graph) { text += fmt::format( " terminator={} condition={} true={} false={} merge={} continue={} loop_header={} " "indirect_sgpr={} indirect_selector={} indirect_targets=[{}] selector_values=[{}] " - "goto_variable={} goto_value={}\n", + "expression={}\n", static_cast(block.terminator.kind), BranchConditionToString(block.terminator.condition).c_str(), block.terminator.true_block, block.terminator.false_block, block.terminator.merge_block, @@ -2297,7 +1831,14 @@ std::string GraphToString(const Graph& graph) { block.terminator.indirect_pc_sgpr, block.terminator.indirect_selector_code, VectorToString(block.terminator.indirect_targets).c_str(), VectorToString(block.terminator.indirect_selector_values).c_str(), - block.terminator.goto_variable, block.terminator.goto_value); + block.terminator.expression); + for (const auto& assignment: block.assignments) { + text += fmt::format(" variable_{} = expression_{}", assignment.variable, assignment.expression); + const auto& expression = graph.expressions[assignment.expression]; + if (expression.op == ConditionExpression::Op::Native) + text += fmt::format(" condition={}", BranchConditionToString(static_cast(expression.lhs))); + text += "\n"; + } } for (const auto& edge: graph.back_edges) { diff --git a/src/graphics/shader/recompiler/frontend/cfg/ShaderCFG.h b/src/graphics/shader/recompiler/frontend/cfg/ShaderCFG.h index f85c093fc..04985f15a 100644 --- a/src/graphics/shader/recompiler/frontend/cfg/ShaderCFG.h +++ b/src/graphics/shader/recompiler/frontend/cfg/ShaderCFG.h @@ -18,10 +18,17 @@ enum class BranchCondition { ExecZero, ExecNonZero, ScalarInstruction, - GotoVariable, + Expression, Unknown }; +struct ConditionExpression { + enum class Op { Constant, Variable, Native, Not, Or }; + Op op; + uint32_t lhs; + uint32_t rhs = UINT32_MAX; +}; + enum class TerminatorKind { Branch, ConditionalBranch, IndirectBranch, Return, Unsupported }; enum class FailureKind { @@ -48,8 +55,7 @@ struct Terminator { std::vector indirect_targets; std::vector indirect_selector_values; std::vector indirect_selector_targets; - uint32_t goto_variable = UINT32_MAX; - int32_t goto_value = -1; + uint32_t expression = UINT32_MAX; bool loop_header = false; }; @@ -59,6 +65,8 @@ struct BasicBlock { uint32_t end_pc = 0; uint32_t inst_begin = 0; uint32_t inst_end = 0; + struct Assignment { uint32_t variable; uint32_t expression; }; + std::vector assignments; std::vector predecessors; std::vector successors; std::vector dominators; @@ -89,6 +97,7 @@ struct StronglyConnectedComponent { struct Graph { std::vector blocks; + std::vector expressions; std::vector back_edges; std::vector natural_loops; std::vector components; diff --git a/src/graphics/shader/recompiler/frontend/translate/Translate.cpp b/src/graphics/shader/recompiler/frontend/translate/Translate.cpp index 3edb31f67..521d0e667 100644 --- a/src/graphics/shader/recompiler/frontend/translate/Translate.cpp +++ b/src/graphics/shader/recompiler/frontend/translate/Translate.cpp @@ -746,55 +746,61 @@ void Translator::WriteCompareResult(const Decoder::Operand& operand, IR::U1 valu WriteMask(operand, ir.LogicalAnd(ir.GetExec(), value)); } -void Translator::AddBranchCondition(const CFG::BasicBlock& source, IR::BlockInfo& info) { - if (source.terminator.goto_value >= 0) { - if (source.terminator.goto_variable == UINT32_MAX) { - EXIT("block %u sets an invalid goto variable", source.id); +void Translator::AddBranchCondition(const CFG::Graph& graph, const CFG::BasicBlock& source, + IR::BlockInfo& info) { + const auto native_condition = [&](CFG::BranchCondition kind) -> IR::U1 { + IR::U1 condition; + switch (kind) { + case CFG::BranchCondition::Always: condition = IR::U1(IR::Value(true)); break; + case CFG::BranchCondition::SccZero: condition = ir.LogicalNot(ir.GetScc()); break; + case CFG::BranchCondition::SccNonZero: condition = ir.GetScc(); break; + case CFG::BranchCondition::VccZero: condition = ir.LogicalNot(ir.GetVcc()); break; + case CFG::BranchCondition::VccNonZero: condition = ir.GetVcc(); break; + case CFG::BranchCondition::ExecZero: condition = ir.LogicalNot(ir.GetExec()); break; + case CFG::BranchCondition::ExecNonZero: condition = ir.GetExec(); break; + case CFG::BranchCondition::ScalarInstruction: + EXIT_IF(instruction_branch_condition.IsEmpty()); + condition = instruction_branch_condition; + break; + default: EXIT("block %u has an invalid native branch condition", source.id); } - ir.SetGotoVariable(source.terminator.goto_variable, - IR::U1(IR::Value(source.terminator.goto_value != 0))); + return IR::U1(ir.Emit(IR::ValueOpcode::ConditionRef, {condition}, kind)); + }; + const auto expression = [&](auto&& self, uint32_t index) -> IR::U1 { + const auto& value = graph.expressions.at(index); + switch (value.op) { + case CFG::ConditionExpression::Op::Constant: return IR::U1(IR::Value(value.lhs != 0)); + case CFG::ConditionExpression::Op::Variable: return ir.GetGotoVariable(value.lhs); + case CFG::ConditionExpression::Op::Native: + return native_condition(static_cast(value.lhs)); + case CFG::ConditionExpression::Op::Not: return ir.LogicalNot(self(self, value.lhs)); + case CFG::ConditionExpression::Op::Or: + return ir.LogicalOr(self(self, value.lhs), self(self, value.rhs)); + } + EXIT("invalid CFG condition expression"); + }; + for (const auto& assignment: source.assignments) { + ir.SetGotoVariable(assignment.variable, expression(expression, assignment.expression)); } - if (source.terminator.kind == CFG::TerminatorKind::IndirectBranch) { - if (source.terminator.indirect_selector_code != UINT32_MAX) { - info.indirect_target = ReadScalarCode(source.terminator.indirect_selector_code); - } else if (source.terminator.indirect_pc_sgpr != UINT32_MAX) { - info.indirect_target = - ir.GetScalarReg(static_cast(source.terminator.indirect_pc_sgpr)); + const auto& term = source.terminator; + if (term.kind == CFG::TerminatorKind::IndirectBranch) { + if (term.indirect_selector_code != UINT32_MAX) { + info.indirect_target = ReadScalarCode(term.indirect_selector_code); + } else if (term.indirect_pc_sgpr != UINT32_MAX) { + info.indirect_target = ir.GetScalarReg(static_cast(term.indirect_pc_sgpr)); } else { EXIT("block %u has no indirect branch selector", source.id); } ir.Emit(IR::ValueOpcode::ReferenceU32, {info.indirect_target}); return; } - if (source.terminator.kind != CFG::TerminatorKind::ConditionalBranch) { - return; + if (term.kind == CFG::TerminatorKind::ConditionalBranch) { + const auto condition = term.expression != UINT32_MAX + ? expression(expression, term.expression) + : native_condition(term.condition); + info.condition = condition; + ir.Emit(IR::ValueOpcode::Reference, {condition}); } - // EXEC and VCC are invocation-local Boolean masks. Branching on that Boolean lets inactive - // invocations leave the region without reconstructing a host-subgroup mask. - IR::U1 condition; - switch (source.terminator.condition) { - case CFG::BranchCondition::Always: condition = IR::U1(IR::Value(true)); break; - case CFG::BranchCondition::SccZero: condition = ir.LogicalNot(ir.GetScc()); break; - case CFG::BranchCondition::SccNonZero: condition = ir.GetScc(); break; - case CFG::BranchCondition::VccZero: condition = ir.LogicalNot(ir.GetVcc()); break; - case CFG::BranchCondition::VccNonZero: condition = ir.GetVcc(); break; - case CFG::BranchCondition::ExecZero: condition = ir.LogicalNot(ir.GetExec()); break; - case CFG::BranchCondition::ExecNonZero: condition = ir.GetExec(); break; - case CFG::BranchCondition::ScalarInstruction: - EXIT_IF(instruction_branch_condition.IsEmpty()); - condition = instruction_branch_condition; - break; - case CFG::BranchCondition::GotoVariable: - if (source.terminator.goto_variable == UINT32_MAX) { - EXIT("block %u reads an invalid goto variable", source.id); - } - condition = ir.GetGotoVariable(source.terminator.goto_variable); - break; - case CFG::BranchCondition::Unknown: - EXIT("block %u has an unknown branch condition", source.id); - } - info.condition = condition; - ir.Emit(IR::ValueOpcode::Reference, {condition}); } namespace { @@ -1297,7 +1303,7 @@ IR::Program TranslateProgram(const Decoder::Program& decoded, const CFG::Graph& } translator.TranslateInstruction(instruction); } - translator.AddBranchCondition(cfg_block, result.block_info[typed_index]); + translator.AddBranchCondition(cfg, cfg_block, result.block_info[typed_index]); } IR::ValidateProgram(result, false); return result; diff --git a/src/graphics/shader/recompiler/frontend/translate/Translator.h b/src/graphics/shader/recompiler/frontend/translate/Translator.h index 88afad485..687b2aeec 100644 --- a/src/graphics/shader/recompiler/frontend/translate/Translator.h +++ b/src/graphics/shader/recompiler/frontend/translate/Translator.h @@ -15,7 +15,7 @@ public: void TranslateInstruction(const Decoder::Instruction& inst); void TranslateEmbeddedFetch(const Decoder::Instruction& inst, uint32_t attribute, uint32_t component_count, const ShaderBufferResource& resource); - void AddBranchCondition(const CFG::BasicBlock& source, IR::BlockInfo& info); + void AddBranchCondition(const CFG::Graph& graph, const CFG::BasicBlock& source, IR::BlockInfo& info); private: const Decoder::Operand& SourceAt(const Decoder::Instruction& inst, uint32_t index); diff --git a/src/graphics/shader/recompiler/ir/opcodes/ValueOpcodes.inc b/src/graphics/shader/recompiler/ir/opcodes/ValueOpcodes.inc index 7a903b492..74c3da30f 100644 --- a/src/graphics/shader/recompiler/ir/opcodes/ValueOpcodes.inc +++ b/src/graphics/shader/recompiler/ir/opcodes/ValueOpcodes.inc @@ -4,6 +4,7 @@ VALUE_OPCODE(Identity, Opaque, Opaque) VALUE_OPCODE(Void, Void) VALUE_OPCODE(Reference, Void, U1) VALUE_OPCODE(ReferenceU32, Void, U32) +VALUE_OPCODE(ConditionRef, U1, U1) // Translation and register-state pseudo operations. VALUE_OPCODE(GetUserData, U32, ScalarReg) diff --git a/src/graphics/shader/recompiler/ir/passes/ConstantPropagation.cpp b/src/graphics/shader/recompiler/ir/passes/ConstantPropagation.cpp index 011ab562c..f4952e470 100644 --- a/src/graphics/shader/recompiler/ir/passes/ConstantPropagation.cpp +++ b/src/graphics/shader/recompiler/ir/passes/ConstantPropagation.cpp @@ -630,6 +630,11 @@ void FoldInstruction(Block& block, Block::iterator instruction, } } return; + case ValueOpcode::ConditionRef: { + const auto value = Arg(inst, 0); + if (IsImmediate(value, Type::U1)) Replace(inst, value); + return; + } case ValueOpcode::LogicalNot: { const auto value = Arg(inst, 0); if (IsImmediate(value, Type::U1)) { diff --git a/src/graphics/shader/recompiler/ir/passes/SrtWalker.cpp b/src/graphics/shader/recompiler/ir/passes/SrtWalker.cpp index 4349d0c7a..4d02d4f83 100644 --- a/src/graphics/shader/recompiler/ir/passes/SrtWalker.cpp +++ b/src/graphics/shader/recompiler/ir/passes/SrtWalker.cpp @@ -89,6 +89,7 @@ bool IsRuntimeSelect(ValueOpcode op) { bool IsRuntimeUniformOp(ValueOpcode op) { switch (op) { + case ValueOpcode::ConditionRef: case ValueOpcode::BitCastU32F32: case ValueOpcode::BitCastF32U32: case ValueOpcode::ConvertU32F32: @@ -970,6 +971,7 @@ bool SrtWalker::EvaluateInst(const Inst& inst, uint64_t& result) { return true; } return false; + case ValueOpcode::ConditionRef: return Arg(inst, 0, result); case ValueOpcode::LogicalNot: if (Arg(inst, 0, a)) { result = a == 0u; diff --git a/tests/ShaderRecompilerComputeTests.cpp b/tests/ShaderRecompilerComputeTests.cpp index 86c8759fd..a3f901792 100644 --- a/tests/ShaderRecompilerComputeTests.cpp +++ b/tests/ShaderRecompilerComputeTests.cpp @@ -23390,6 +23390,73 @@ TestCase SharedReturnKeepsSelectedValues() { return test; } +TestCase SiblingSharedExitKeepsCapturedConditions() { + using O = ShaderOpcode; + std::vector code = { + EncodeVop1(0x01, 5, 4), + EncodeVop2(0x1a, 5, InlineU32(6), 5), + EncodeVop2(0x25, 5, Vgpr(0), 5), // group * 64 + lane + EncodeSop2(0x0e, 6, 4, InlineU32(1)), + 0, // choose left or right + EncodeSMovB32(8, InlineU32(20)), + EncodeSop2(0x0e, 6, 4, InlineU32(2)), + 0, // left: skip nested exit + EncodeSop2(0x0e, 6, 4, InlineU32(4)), + 0, // nested left: shared exit + 0, // left -> normal return + EncodeSMovB32(8, InlineU32(30)), + EncodeSop2(0x0e, 6, 4, InlineU32(8)), + 0, // first right shared exit + EncodeSMovB32(8, InlineU32(40)), + EncodeSop2(0x0e, 6, 4, InlineU32(16)), + 0, // second right shared exit + EncodeSMovB32(8, InlineU32(50)), + EncodeSopc(0x06, InlineU32(0), InlineU32(1)), // overwrite SCC + }; + AppendStoreSgprAtLaneDwordOffset(&code, 8, 5, 0); + AppendEnd(&code); + const auto shared_exit = static_cast(code.size()); + code.push_back(EncodeSop2(0x00, 8, 8, InlineU32(1))); // overwrite SCC again + AppendStoreSgprAtLaneDwordOffset(&code, 8, 5, 0); + AppendEnd(&code); + code[4] = EncodeSopp(0x04, 11u - 5u); + code[7] = EncodeSopp(0x04, 10u - 8u); + code[9] = EncodeSopp(0x04, shared_exit - 10u); + code[10] = EncodeSopp(0x02, 17u - 11u); + code[13] = EncodeSopp(0x04, shared_exit - 14u); + code[16] = EncodeSopp(0x04, shared_exit - 17u); + + TestCase test; + test.name = "SiblingSharedExitKeepsCapturedConditions"; + test.code = std::move(code); + for (u32 group = 0; group < 32; ++group) { + u32 result = 50; + if (group & 1u) { + if ((group & 2u) && !(group & 4u)) result = 21; + } else if (!(group & 8u)) { + result = 31; + } else if (!(group & 16u)) { + result = 41; + } + test.expected.insert(test.expected.end(), 64, result); + } + test.opcodes = {O::V_MOV_B32, O::V_LSHLREV_B32, O::V_ADD_NC_U32, + O::S_AND_B32, O::S_MOV_B32, O::S_CMP_EQ_U32, O::S_ADD_U32, + O::S_CBRANCH_SCC0, O::S_BRANCH, O::BUFFER_STORE_DWORD, O::S_ENDPGM}; + test.required_spirv = {"OpSelectionMerge"}; + test.forbidden_spirv = {"OpSwitch"}; + test.compute_info.wave_size = 64; + test.compute_info.threads_num[0] = 64; + test.compute_info.threads_num[1] = 1; + test.compute_info.threads_num[2] = 1; + test.compute_info.group_id[0] = true; + test.compute_info.thread_ids_num = 1; + test.compute_info.workgroup_register = 4; + test.has_compute_info = true; + test.dispatch_x = 32; + return test; +} + TestCase BranchVccnzUsesWaveMask() { using O = ShaderOpcode; @@ -29477,6 +29544,7 @@ std::vector MakeCases() { AddCase(BranchSelect); AddCase(SimpleLoop); AddCase(SharedReturnKeepsSelectedValues); + AddCase(SiblingSharedExitKeepsCapturedConditions); AddCase(BranchVccnzUsesWaveMask); AddCase(BranchVccnzUsesCarryProducedWaveMask); AddCase(ScalarMemRealtimeCapturedPlaceholder); @@ -34450,6 +34518,7 @@ int main(int argc, char **argv) { RunCase(&vulkan, ImageSampleAndGather()); RunCase(&vulkan, ImageGatherLodApproximatesLevelZero()); RunCase(&vulkan, SharedReturnKeepsSelectedValues()); + RunCase(&vulkan, SiblingSharedExitKeepsCapturedConditions()); return 0; } if (argc == 2 && std::strcmp(argv[1], "--position-w-only") == 0) { diff --git a/tests/shaderCfgTests.cpp b/tests/shaderCfgTests.cpp index 084bfbc14..74220d56a 100644 --- a/tests/shaderCfgTests.cpp +++ b/tests/shaderCfgTests.cpp @@ -1818,12 +1818,12 @@ void TestSopkCompareImmediateExtension() { TranslateProgram(shader, MakeCompileOptions(ShaderType::Vertex)); const auto branch = std::ranges::find_if( translated.program.block_info, [](const auto &block) { - return block.terminator.condition == CFG::BranchCondition::SccZero; + return block.terminator.kind == CFG::TerminatorKind::ConditionalBranch; }); Check(branch != translated.program.block_info.end(), "captured LUT parameter export branch was lost"); const auto condition = branch->condition.Resolve(); - Check(condition.IsImmediate() && condition.U1() == (counts == 6u), + Check(condition.IsImmediate() && condition.U1() == (counts != 6u), "captured LUT unsigned compare suppressed a live parameter export"); } } @@ -7882,17 +7882,13 @@ void TestNewShaderRecompilerCfgLoopExitSharedWithSelection() { const auto coverage = CfgInstructionCoverage(graph, decoded.instructions.size()); Check(block_count == 8u && graph.natural_loops.size() == 1u, "shared loop-exit fixture has the wrong native CFG"); - Check(!ShaderRecompiler::CFG::Structurize(graph) && - graph.unsupported_reason.find("duplicate structured merge block") != - std::string::npos, - "shared loop exit did not terminate at its structured merge conflict"); - Check(graph.blocks.size() == block_count && - CfgInstructionCoverage(graph, decoded.instructions.size()) == coverage, - "shared loop-exit fallback changed semantic instruction coverage"); + Check(ShaderRecompiler::CFG::Structurize(graph), graph.unsupported_reason.c_str()); + Check(CfgInstructionCoverage(graph, decoded.instructions.size()) == coverage, + "shared loop-exit routing changed semantic instruction coverage"); auto result = RecompileForTest(shader, MakeCompileOptions(ShaderType::Compute)); - Check(result.program.dispatcher_fallback && SpirvContainsOpcode(result.spirv, 251), - "shared loop exit did not emit its dispatcher fallback"); + Check(!result.program.dispatcher_fallback && !SpirvContainsOpcode(result.spirv, 251), + "shared loop exit did not retain structured control flow"); CheckSpirvBinaryValidates(result.spirv); } @@ -7922,7 +7918,7 @@ void TestNewShaderRecompilerCfgLoopEarlyBreakNoSelection() { CheckSpirvBinaryValidates(result.spirv); } -void TestNewShaderRecompilerCfgNestedLoopNonlocalExitDispatcher() { +void TestNewShaderRecompilerCfgNestedLoopNonlocalExitStructured() { const uint32_t shader[] = { EncodeSopc(0x0a, 0, 129), // outer loop: s_cmp_lt_u32 s0, 1 EncodeSopp(0x04, 9), // outer exit -> end @@ -7942,10 +7938,10 @@ void TestNewShaderRecompilerCfgNestedLoopNonlocalExitDispatcher() { options.dump_ir = true; auto result = RecompileForTest(shader, options); - Check((result.ir_dump.find("mode=dispatcher") != std::string::npos), - "nested-loop nonlocal exit did not select dispatcher fallback"); - Check(SpirvInstructionOpcodeCount(result.spirv, 251) != 0, - "nested-loop nonlocal exit dispatcher SPIR-V lacks OpSwitch"); + Check((result.ir_dump.find("mode=structured") != std::string::npos), + "nested-loop nonlocal exit did not retain structured control flow"); + Check(SpirvInstructionOpcodeCount(result.spirv, 251) == 0, + "nested-loop nonlocal exit unexpectedly selected dispatcher OpSwitch"); CheckSpirvBinaryValidates(result.spirv); } @@ -8006,27 +8002,23 @@ void TestNewShaderRecompilerCfgNestedLoopExitTailMergeSplit() { Check(graph.blocks.size() > original_block_count, "nested loop exit tails did not create a private inner merge"); - const auto *outer_header = graph.FindBlockByPc(0); - const auto *inner_header = graph.FindBlockByPc(8); - Check(outer_header != nullptr && inner_header != nullptr && - outer_header->terminator.loop_header && - inner_header->terminator.loop_header, - "nested loop exit-tail fixture did not retain both loop headers"); - Check(inner_header->terminator.merge_block != - outer_header->terminator.continue_block, - "inner loop merge still aliases the outer continue target"); - const auto *inner_merge = - graph.FindBlock(inner_header->terminator.merge_block); - Check(inner_merge != nullptr && - inner_merge->inst_begin == inner_merge->inst_end && - inner_merge->terminator.kind == - ShaderRecompiler::CFG::TerminatorKind::Branch && - inner_merge->terminator.true_block == - outer_header->terminator.continue_block, - "private inner merge does not forward to the outer continue target"); + std::vector headers; + for (const auto &block : graph.blocks) + if (block.terminator.loop_header) headers.push_back(&block); + Check(headers.size() == 2u, "nested loop exit-tail fixture lost a loop header"); + if (!graph.Dominates(headers[0]->id, headers[1]->id)) std::swap(headers[0], headers[1]); + const auto *outer_header = headers[0]; + const auto *inner_header = headers[1]; + Check(inner_header->terminator.merge_block != outer_header->terminator.continue_block && + graph.Dominates(outer_header->id, inner_header->terminator.merge_block), + "inner loop merge escaped the enclosing loop body"); + auto result = RecompileForTest(shader, MakeCompileOptions(ShaderType::Compute)); + Check(!result.program.dispatcher_fallback, + "nested loop exit tails selected the dispatcher"); + CheckSpirvBinaryValidates(result.spirv); } -void TestNewShaderRecompilerCfgMixedContinueNonmergeExitDispatcher() { +void TestNewShaderRecompilerCfgMixedContinueNonmergeExitStructured() { const uint32_t shader[] = { EncodeSopc(0x06, 7, 7), // entry branch bypasses loop -> exit X EncodeSopp(0x05, 5), // entry -> X @@ -8046,10 +8038,10 @@ void TestNewShaderRecompilerCfgMixedContinueNonmergeExitDispatcher() { options.dump_ir = true; auto result = RecompileForTest(shader, options); - Check((result.ir_dump.find("mode=dispatcher") != std::string::npos), - "mixed continue/nonmerge exit did not select dispatcher fallback"); - Check(SpirvInstructionOpcodeCount(result.spirv, 251) != 0, - "mixed continue/nonmerge exit dispatcher SPIR-V lacks OpSwitch"); + Check((result.ir_dump.find("mode=structured") != std::string::npos), + "mixed continue/nonmerge exit did not retain structured control flow"); + Check(SpirvInstructionOpcodeCount(result.spirv, 251) == 0, + "mixed continue/nonmerge exit unexpectedly selected dispatcher OpSwitch"); CheckSpirvBinaryValidates(result.spirv); } @@ -8100,7 +8092,7 @@ void TestNewShaderRecompilerCfgDirectConditionalLatchNoSelection() { CheckSpirvBinaryValidates(result.spirv); } -void TestNewShaderRecompilerCfgLoopEarlyContinuesNoSelection() { +void TestNewShaderRecompilerCfgLoopEarlyContinuesStructured() { const uint32_t shader[] = { EncodeSMovB32(0, 128), // s0 = 0 EncodeSopc(0x0a, 0, 130), // loop: s_cmp_lt_u32 s0, 2 @@ -8125,8 +8117,8 @@ void TestNewShaderRecompilerCfgLoopEarlyContinuesNoSelection() { "loop early continues should stay on structured path"); Check(SpirvInstructionOpcodeCount(result.spirv, 246) != 0, "loop early continues SPIR-V lacks OpLoopMerge"); - Check(SpirvInstructionOpcodeCount(result.spirv, 247) == 0, - "loop early continues SPIR-V unexpectedly used OpSelectionMerge"); + Check(SpirvInstructionOpcodeCount(result.spirv, 250) <= 4u, + "loop early continues introduced duplicate conditional branches"); Check(SpirvInstructionOpcodeCount(result.spirv, 251) == 0, "loop early continues unexpectedly used dispatcher OpSwitch"); CheckSpirvBinaryValidates(result.spirv); @@ -8242,29 +8234,28 @@ void TestNewShaderRecompilerCfgMultipleLoopLatches() { ShaderRecompiler::Decoder::DecodeProgram(std::span{shader}, decoded); ShaderRecompiler::CFG::Graph graph; graph = ShaderRecompiler::CFG::BuildGraph(decoded); - const auto original_block_count = graph.blocks.size(); + const auto coverage = CfgInstructionCoverage(graph, decoded.instructions.size()); Check(graph.back_edges.size() == 2u, "multiple-latch fixture lacks two native backedges"); Check(ShaderRecompiler::CFG::Structurize(graph), graph.unsupported_reason.c_str()); - Check(graph.blocks.size() == original_block_count + 2u, - "multiple native latches did not create one synthetic continue and one " - "empty header"); + Check(CfgInstructionCoverage(graph, decoded.instructions.size()) == coverage, + "shared backedge construction duplicated guest instructions"); Check(graph.back_edges.size() == 1u && graph.natural_loops.size() == 1u, "multiple native latches were not coalesced to one SPIR-V backedge"); const auto &loop = graph.natural_loops.front(); const auto *continue_block = graph.FindBlock(loop.continue_block); Check(continue_block != nullptr && - continue_block->inst_begin == continue_block->inst_end && - continue_block->predecessors.size() == 2u, - "canonical continue does not join both native latches"); + continue_block->terminator.true_block == loop.header && + graph.Dominates(loop.header, continue_block->id), + "canonical continue does not return to its natural loop header"); auto options = MakeCompileOptions(ShaderType::Compute); auto result = RecompileForTest(shader, options); Check(SpirvInstructionOpcodeCount(result.spirv, 246) == 1u, "multiple-latch SPIR-V has the wrong loop-merge count"); - Check(SpirvInstructionOpcodeCount(result.spirv, 247) == 0u, - "multiple-latch SPIR-V unexpectedly used a selection merge"); + Check(SpirvInstructionOpcodeCount(result.spirv, 250) <= 2u, + "multiple-latch routing duplicated a native conditional branch"); Check(SpirvInstructionOpcodeCount(result.spirv, 251) == 0u, "multiple-latch SPIR-V unexpectedly used dispatcher OpSwitch"); CheckSpirvBinaryValidates(result.spirv); @@ -8319,36 +8310,20 @@ void TestNewShaderRecompilerCfgNestedEarlyExitLoopForwarders() { const auto original_block_count = graph.blocks.size(); const auto original_coverage = CfgInstructionCoverage(graph, decoded.instructions.size()); - const auto original_empty_blocks = - std::ranges::count_if(graph.blocks, [](const auto &block) { - return block.inst_begin == block.inst_end; - }); Check(original_block_count == 7u && graph.natural_loops.size() == 1u, "nested early-exit fixture has the wrong native CFG"); const bool structured = ShaderRecompiler::CFG::Structurize(graph); Check(structured, graph.unsupported_reason.c_str()); - Check(graph.blocks.size() == original_block_count + 4u && - CfgInstructionCoverage(graph, decoded.instructions.size()) == - original_coverage && - std::ranges::count_if(graph.blocks, - [](const auto &block) { - return block.inst_begin == block.inst_end; - }) == original_empty_blocks + 4, + Check(CfgInstructionCoverage(graph, decoded.instructions.size()) == original_coverage, "nested early-exit structurization changed semantic coverage"); - const auto *preheader = graph.FindBlockByPc(0x00u); - const auto *outer = graph.FindBlockByPc(0x08u); - const auto *inner = graph.FindBlockByPc(0x10u); - const auto *loop = graph.FindBlockByPc(0x20u); - Check( - preheader != nullptr && outer != nullptr && inner != nullptr && - loop != nullptr && loop->terminator.loop_header && - preheader->terminator.merge_block != UINT32_MAX && - outer->terminator.merge_block != UINT32_MAX && - inner->terminator.merge_block != UINT32_MAX && - preheader->terminator.merge_block != outer->terminator.merge_block && - preheader->terminator.merge_block != inner->terminator.merge_block && - outer->terminator.merge_block != inner->terminator.merge_block, - "nested early-exit constructs do not have distinct structured merges"); + std::unordered_set merges; + for (const auto &block : graph.blocks) { + if (block.terminator.merge_block != UINT32_MAX) + Check(merges.insert(block.terminator.merge_block).second, + "nested early-exit constructs share a structured merge"); + } + Check(graph.natural_loops.size() == 1u, + "nested early-exit routing changed the natural loop count"); auto options = MakeCompileOptions(ShaderType::Compute); options.dump_ir = true; @@ -8365,7 +8340,7 @@ void TestNewShaderRecompilerCfgNestedEarlyExitLoopForwarders() { void TestNewShaderRecompilerCfgExecSccSharedArm() { const uint32_t shader[] = { - EncodeSop2(0x15, 126, 4, 126), // s_andn2_b64 exec, s4, exec + EncodeSop2(0x0f, 126, 4, 126), // s_and_b64 exec, s4, exec (runtime mask) EncodeSopp(0x08, 2), // execz -> shared arm EncodeSop2(0x15, 30, 30, 126), // s_andn2_b64 s30, s30, exec EncodeSopp(0x04, 2), // scc0 -> other arm, else shared arm @@ -8386,15 +8361,6 @@ void TestNewShaderRecompilerCfgExecSccSharedArm() { "shared-arm fixture already duplicated a semantic instruction"); Check(ShaderRecompiler::CFG::Structurize(graph), graph.unsupported_reason.c_str()); - uint32_t route_selects = 0; - uint32_t route_sets = 0; - for (const auto &block : graph.blocks) { - route_selects += block.terminator.condition == - ShaderRecompiler::CFG::BranchCondition::GotoVariable; - route_sets += block.terminator.goto_value >= 0; - } - Check(graph.blocks.size() == 5u && route_selects == 0u && route_sets == 0u, - "shared return arm introduced synthetic routing state"); Check(CfgInstructionCoverage(graph, decoded.instructions.size()) == original_coverage, "EXEC/SCC shared-arm structurization changed semantic coverage"); @@ -8437,7 +8403,7 @@ void TestSharedReturnPreservesDescriptorDominance() { graph.unsupported_reason.c_str()); const auto *overwrite = graph.FindBlockByPc(0x10u); const auto *body = graph.FindBlockByPc(0x20u); - Check(graph.blocks.size() == 5u && overwrite != nullptr && body != nullptr && + Check(overwrite != nullptr && body != nullptr && graph.Dominates(overwrite->id, body->id) && std::ranges::count_if(graph.blocks, [](const auto &block) { return block.terminator.kind == @@ -8466,6 +8432,78 @@ void TestSharedReturnPreservesDescriptorDominance() { CheckSpirvBinaryValidates(result.spirv); } +void TestCfgSiblingSharedExit() { + // f7030726b9470dd8: a nested exit on one side and two exits on the + // other side share an epilogue, separate from the normal return. + const uint32_t shader[] = { + EncodeSopc(0x06, 0, 128), EncodeSopp(0x04, 5), // right or left + EncodeSopc(0x06, 1, 128), EncodeSopp(0x04, 2), // join or nested exit + EncodeSopc(0x06, 2, 128), EncodeSopp(0x04, 7), // shared exit + EncodeSopp(0x02, 4), // left -> main + EncodeSopc(0x06, 3, 128), EncodeSopp(0x04, 4), // right -> shared exit + EncodeSopc(0x06, 4, 128), EncodeSopp(0x04, 2), // second shared exit + EncodeSMovB32(5, 129), EncodeSopp(0x01), // main and normal return + EncodeSMovB32(5, 130), EncodeSopp(0x01), // shared exit epilogue + }; + using namespace ShaderRecompiler; + Decoder::Program decoded; + Decoder::DecodeProgram(std::span{shader}, decoded); + const auto original = CFG::BuildGraph(decoded); + auto structured = original; + Check(CFG::Structurize(structured), structured.unsupported_reason.c_str()); + Check(CfgInstructionCoverage(structured, decoded.instructions.size()) == + CfgInstructionCoverage(original, decoded.instructions.size()), + "shared sibling exit duplicated or dropped guest instructions"); + const auto trace = [&](const CFG::Graph &graph, uint32_t decisions) { + std::vector executed; + std::vector variables(original.blocks.size() * 2); + const auto expression = [&](auto &&self, uint32_t index) -> bool { + const auto &value = graph.expressions.at(index); + switch (value.op) { + case CFG::ConditionExpression::Op::Constant: return value.lhs != 0; + case CFG::ConditionExpression::Op::Variable: return variables.at(value.lhs); + case CFG::ConditionExpression::Op::Native: std::abort(); + case CFG::ConditionExpression::Op::Not: return !self(self, value.lhs); + case CFG::ConditionExpression::Op::Or: + return self(self, value.lhs) || self(self, value.rhs); + } + std::abort(); + }; + auto current = graph.entry_block; + for (unsigned steps = 0; steps < 100; ++steps) { + const auto &block = graph.blocks.at(current); + for (auto inst = block.inst_begin; inst < block.inst_end; ++inst) + executed.push_back(inst); + for (const auto &assignment : block.assignments) { + variables.at(assignment.variable) = + graph.expressions[assignment.expression].op == CFG::ConditionExpression::Op::Native + ? (decisions & (1u << (assignment.variable - original.blocks.size()))) != 0 + : expression(expression, assignment.expression); + } + const auto &term = block.terminator; + if (term.kind == CFG::TerminatorKind::Return) return executed; + Check(term.kind == CFG::TerminatorKind::Branch || + term.kind == CFG::TerminatorKind::ConditionalBranch, + "shared exit trace has an unsupported terminator"); + const bool condition = term.expression != UINT32_MAX + ? expression(expression, term.expression) + : (decisions & (1u << block.id)) != 0; + current = term.kind == CFG::TerminatorKind::Branch || condition + ? term.true_block : term.false_block; + } + Check(false, "shared exit trace did not terminate"); + return executed; + }; + for (uint32_t decisions = 0; decisions < (1u << original.blocks.size()); ++decisions) + Check(trace(original, decisions) == trace(structured, decisions), + "goto elimination changed the executed shared-exit path"); + auto result = RecompileForTest(shader, MakeCompileOptions(ShaderType::Compute)); + Check(!result.program.dispatcher_fallback && + SpirvInstructionOpcodeCount(result.spirv, 251) == 0, + "shared sibling exit selected dispatcher lowering"); + CheckSpirvBinaryValidates(result.spirv); +} + void TestNewShaderRecompilerCfgNestedTailEarlyExit() { const uint32_t shader[] = { EncodeSopc(0x06, 0, 0), // preceding outer condition @@ -8499,17 +8537,6 @@ void TestNewShaderRecompilerCfgNestedTailEarlyExit() { Check(CfgInstructionCoverage(graph, decoded.instructions.size()) == original_coverage, "nested-tail routing changed semantic instruction coverage"); - Check(std::ranges::count_if( - graph.blocks, - [](const auto &block) { - return block.terminator.condition == - ShaderRecompiler::CFG::BranchCondition::GotoVariable; - }) == 1u && - std::ranges::count_if(graph.blocks, - [](const auto &block) { - return block.terminator.goto_value >= 0; - }) == 3u, - "nested-tail early exit did not use typed route state"); auto options = MakeCompileOptions(ShaderType::Pixel); options.dump_ir = true; @@ -8549,17 +8576,6 @@ void TestNewShaderRecompilerCfgSharedReturnAfterNestedSelections() { Check(CfgInstructionCoverage(graph, decoded.instructions.size()) == original_coverage, "shared-exit route ordering changed semantic instruction coverage"); - const auto route_selects = - std::ranges::count_if(graph.blocks, [](const auto &block) { - return block.terminator.condition == - ShaderRecompiler::CFG::BranchCondition::GotoVariable; - }); - const auto route_sets = - std::ranges::count_if(graph.blocks, [](const auto &block) { - return block.terminator.goto_value >= 0; - }); - Check(route_selects == 0u && route_sets == 0u, - "nested selections introduced routing state for a shared return"); auto options = MakeCompileOptions(ShaderType::Compute); options.dump_ir = true; @@ -8628,7 +8644,6 @@ void TestNewShaderRecompilerCfgLoopSharedRegion() { ShaderRecompiler::Decoder::DecodeProgram(std::span{shader}, decoded); ShaderRecompiler::CFG::Graph graph; graph = ShaderRecompiler::CFG::BuildGraph(decoded); - const auto original_block_count = graph.blocks.size(); const auto original_coverage = CfgInstructionCoverage(graph, decoded.instructions.size()); Check(graph.natural_loops.size() == 1u, @@ -8641,18 +8656,8 @@ void TestNewShaderRecompilerCfgLoopSharedRegion() { loop_header->terminator.continue_block != UINT32_MAX, "loop shared-region routing did not preserve the natural loop"); - uint32_t route_selects = 0; - uint32_t route_sets = 0; - for (const auto &block : graph.blocks) { - route_selects += block.terminator.condition == - ShaderRecompiler::CFG::BranchCondition::GotoVariable; - route_sets += block.terminator.goto_value >= 0; - } - Check(route_selects == 1u && route_sets == 3u && - graph.blocks.size() >= original_block_count + 5u && - CfgInstructionCoverage(graph, decoded.instructions.size()) == - original_coverage, - "loop shared region was not routed without semantic duplication"); + Check(CfgInstructionCoverage(graph, decoded.instructions.size()) == original_coverage, + "loop shared region changed guest instruction coverage"); auto options = MakeCompileOptions(ShaderType::Compute); options.dump_ir = true; @@ -8701,11 +8706,6 @@ void TestNewShaderRecompilerCfgSharedRegionBeforeEarlyBreakLoop() { graph.unsupported_reason.c_str()); Check(graph.natural_loops.size() == 1u && graph.back_edges.size() == 1u, "selection routing introduced a cycle around a structured loop exit"); - Check(std::ranges::count_if(graph.blocks, [](const auto &block) { - return block.terminator.condition == - ShaderRecompiler::CFG::BranchCondition::GotoVariable; - }) == 1u, - "shared-region routing rewrote unrelated loop-control branches"); Check(CfgInstructionCoverage(graph, decoded.instructions.size()) == original_coverage, "shared-region/early-break routing changed semantic coverage"); @@ -8770,17 +8770,6 @@ void TestNewShaderRecompilerCfgOverlappingEarlyExitLadder() { "early-exit ladder routing changed semantic instruction coverage"); Check(graph.blocks.size() > original_block_count, "early-exit ladder routing did not add forwarding blocks"); - const auto route_selects = - std::ranges::count_if(graph.blocks, [](const auto &block) { - return block.terminator.condition == - ShaderRecompiler::CFG::BranchCondition::GotoVariable; - }); - const auto route_sets = - std::ranges::count_if(graph.blocks, [](const auto &block) { - return block.terminator.goto_value >= 0; - }); - Check(route_selects != 0u && route_sets >= 3u, - "early-exit ladder lacks explicit typed route state"); auto options = MakeCompileOptions(ShaderType::Pixel); options.dump_ir = true; @@ -8818,17 +8807,6 @@ void TestNewShaderRecompilerCfgNestedEarlyExitSharedTerminal() { Check(CfgInstructionCoverage(graph, decoded.instructions.size()) == original_coverage, "nested early-exit structurization changed semantic coverage"); - const auto route_selects = - std::ranges::count_if(graph.blocks, [](const auto &block) { - return block.terminator.condition == - ShaderRecompiler::CFG::BranchCondition::GotoVariable; - }); - const auto route_sets = - std::ranges::count_if(graph.blocks, [](const auto &block) { - return block.terminator.goto_value >= 0; - }); - Check(route_selects == 1u && route_sets == 3u, - "nested early exit lacks complete typed route state"); auto options = MakeCompileOptions(ShaderType::Compute); options.dump_ir = true; @@ -8873,12 +8851,6 @@ void TestNewShaderRecompilerCfgEarlyReturnSharedLoopContinuation() { original_coverage && graph.natural_loops.size() == 2u, "shared-continuation gateway changed semantic blocks or the loops"); - Check(std::ranges::none_of(graph.blocks, - [](const auto &block) { - return block.terminator.goto_variable != - UINT32_MAX; - }), - "private early return introduced unnecessary routing state"); auto options = MakeCompileOptions(ShaderType::Compute); auto result = RecompileForTest(shader, options); @@ -13629,13 +13601,13 @@ int main() { TestNewShaderRecompilerCfgSharedOuterAndLoopMerge(); TestNewShaderRecompilerCfgLoopExitSharedWithSelection(); TestNewShaderRecompilerCfgLoopEarlyBreakNoSelection(); - TestNewShaderRecompilerCfgNestedLoopNonlocalExitDispatcher(); + TestNewShaderRecompilerCfgNestedLoopNonlocalExitStructured(); TestNewShaderRecompilerCfgNestedLoopLocalExitNoSelection(); TestNewShaderRecompilerCfgNestedLoopExitTailMergeSplit(); - TestNewShaderRecompilerCfgMixedContinueNonmergeExitDispatcher(); + TestNewShaderRecompilerCfgMixedContinueNonmergeExitStructured(); TestNewShaderRecompilerCfgConditionalLatchNoSelection(); TestNewShaderRecompilerCfgDirectConditionalLatchNoSelection(); - TestNewShaderRecompilerCfgLoopEarlyContinuesNoSelection(); + TestNewShaderRecompilerCfgLoopEarlyContinuesStructured(); TestNewShaderRecompilerCfgLoopGatewaySelection(); TestNewShaderRecompilerCfgConditionalLoopHeaderSelection(); TestNewShaderRecompilerCfgMultipleLoopLatches(); @@ -13643,6 +13615,7 @@ int main() { TestNewShaderRecompilerCfgNestedEarlyExitLoopForwarders(); TestNewShaderRecompilerCfgExecSccSharedArm(); TestSharedReturnPreservesDescriptorDominance(); + TestCfgSiblingSharedExit(); TestNewShaderRecompilerCfgNestedTailEarlyExit(); TestNewShaderRecompilerCfgSharedReturnAfterNestedSelections(); TestNewShaderRecompilerCfgAlternatingSharedReturns();