From ce5e57aa36fdbf052e59d2e064e1278dae667a84 Mon Sep 17 00:00:00 2001 From: nmzik Date: Mon, 26 Oct 2026 01:37:18 +0100 Subject: [PATCH] shader: fold complementary masks so wave reductions see full EXEC Co-authored-by: Senaxx Co-authored-by: Mac --- .../ir/passes/ConstantPropagation.cpp | 17 +++- .../ir/passes/ReadLaneElimination.cpp | 82 ++++++------------- tests/ScalarProvenanceTests.cpp | 59 ++++++------- 3 files changed, 66 insertions(+), 92 deletions(-) diff --git a/src/graphics/shader/recompiler/ir/passes/ConstantPropagation.cpp b/src/graphics/shader/recompiler/ir/passes/ConstantPropagation.cpp index 7e516cf7d..d0e0de093 100644 --- a/src/graphics/shader/recompiler/ir/passes/ConstantPropagation.cpp +++ b/src/graphics/shader/recompiler/ir/passes/ConstantPropagation.cpp @@ -167,6 +167,16 @@ bool ReplaceBinaryIdentity(Inst& inst, Type type, uint64_t identity) { return false; } +// Whether one operand is the other's negation. +bool IsComplementPair(const Inst& inst, ValueOpcode negation) { + for (uint32_t arg = 0; arg < 2u; ++arg) { + const auto* negate = Arg(inst, arg).TryInstruction(); + if (negate != nullptr && negate->GetOpcode() == negation && + Arg(*negate, 0) == Arg(inst, arg ^ 1u)) return true; + } + return false; +} + bool FoldSelect(Inst& inst) { const auto condition = Arg(inst, 0); const auto true_value = Arg(inst, 1); @@ -626,8 +636,9 @@ void FoldInstruction(Block& block, Block::iterator instruction, case ValueOpcode::BitwiseXor32: if (!FoldU32(inst, [opcode = inst.GetOpcode()](uint32_t a, uint32_t b) { return opcode == ValueOpcode::BitwiseOr32 ? a | b : a ^ b; - })) { - ReplaceBinaryIdentity(inst, Type::U32, 0u); + }) && !ReplaceBinaryIdentity(inst, Type::U32, 0u) && + IsComplementPair(inst, ValueOpcode::BitwiseNot32)) { + Replace(inst, Value(UINT32_MAX)); } return; case ValueOpcode::BitwiseNot32: { @@ -802,6 +813,8 @@ void FoldInstruction(Block& block, Block::iterator instruction, Replace(inst, lhs.U1() ? lhs : rhs); } else if (IsImmediate(rhs, Type::U1)) { Replace(inst, rhs.U1() ? rhs : lhs); + } else if (IsComplementPair(inst, ValueOpcode::LogicalNot)) { + Replace(inst, Value(true)); } else { const auto* left = lhs.TryInstruction(); const auto* right = rhs.TryInstruction(); diff --git a/src/graphics/shader/recompiler/ir/passes/ReadLaneElimination.cpp b/src/graphics/shader/recompiler/ir/passes/ReadLaneElimination.cpp index 7e444aacd..b3be7a196 100644 --- a/src/graphics/shader/recompiler/ir/passes/ReadLaneElimination.cpp +++ b/src/graphics/shader/recompiler/ir/passes/ReadLaneElimination.cpp @@ -104,11 +104,10 @@ Value GetRealValue(PhiMap& phi_map, Value source, uint32_t lane, uint32_t wave_s // Fragment subgroups may have no physical invocation at either endpoint. class WaveReductionMatcher { public: - WaveReductionMatcher(Block* block, ValueOpcode operation) - : block(block), operation(operation), - identity(operation == ValueOpcode::UMin32 ? UINT32_MAX : 0u) {} + WaveReductionMatcher(const Block* block, ValueOpcode operation) + : block(block), operation(operation) {} - Value Match(const Inst& reduction, uint32_t wave_size) { + Value Match(const Inst& reduction, uint32_t wave_size) const { const auto* low = Get(reduction.Arg(0), ValueOpcode::ReadLane); const auto* high = Get(reduction.Arg(1), ValueOpcode::ReadLane); if (low == nullptr || high == nullptr || low->Arg(0).Resolve() != high->Arg(0).Resolve()) { @@ -124,45 +123,32 @@ public: (lane(low->Arg(1), wave_size - 1u) && lane(high->Arg(1), half - 1u)))) { return {}; } - if (wave_size == 32u) return MatchScan(Unwrap(low->Arg(0))); - const auto* joined = Get(Unwrap(low->Arg(0)), operation); + if (wave_size == 32u) return MatchScan(low->Arg(0)); + const auto* joined = Get(low->Arg(0), operation); if (joined == nullptr) return {}; for (size_t index = 0; index < 2; index++) { - auto attempt = *this; - const auto* cross = attempt.Get(attempt.Unwrap(joined->Arg(index)), ValueOpcode::Permlane16U32); + const auto* cross = Get(joined->Arg(index), ValueOpcode::Permlane16U32); if (cross == nullptr) continue; - const auto flags = cross->Flags(); - auto prefix = attempt.Unwrap(joined->Arg(1 - index)); + const auto flags = cross->Flags(); + const auto prefix = joined->Arg(1 - index).Resolve(); if (!flags.x16 || flags.fetch_inactive || !flags.bound_control || cross->Arg(1).Resolve() != Value(UINT32_MAX) || - cross->Arg(2).Resolve() != Value(UINT32_MAX) || - !attempt.FullExec(cross->Arg(3)) || attempt.Unwrap(cross->Arg(0)) != prefix) { + cross->Arg(2).Resolve() != Value(UINT32_MAX) || !FullExec(cross->Arg(3)) || + cross->Arg(0).Resolve() != prefix) { continue; } - if (const auto seed = attempt.MatchScan(prefix); !seed.IsEmpty()) return seed; + if (const auto seed = MatchScan(prefix); !seed.IsEmpty()) return seed; } return {}; } private: - // The row scan's input, whose lanes outside the saved EXEC hold the operation's identity. - Value MatchScan(Value prefix) { + // The row scan's input. + Value MatchScan(Value prefix) const { for (const uint32_t shift: {8u, 4u, 2u, 1u}) { prefix = MatchRow(prefix, shift); } - prefix = Unwrap(prefix); - if (prefix.IsEmpty() || saved_exec.IsEmpty()) return prefix; - const auto* seed = Get(prefix, ValueOpcode::SelectU32); - if (seed != nullptr && seed->Arg(2).Resolve() == Value(identity)) { - const auto predicate = seed->Arg(0).Resolve(); - const auto* subset = Get(predicate, ValueOpcode::LogicalAnd); - if (predicate == saved_exec || - (subset != nullptr && (subset->Arg(0).Resolve() == saved_exec || - subset->Arg(1).Resolve() == saved_exec))) { - return prefix; - } - } - return {}; + return prefix; } Inst* Get(Value value, ValueOpcode opcode) const { @@ -171,28 +157,8 @@ private: ? inst : nullptr; } - bool FullExec(Value predicate) { - predicate = predicate.Resolve(); - if (predicate == Value(true)) return true; - const auto* either = Get(predicate, ValueOpcode::LogicalOr); - if (either == nullptr) return false; - for (size_t index = 0; index < 2; index++) { - const auto* negate = Get(either->Arg(index), ValueOpcode::LogicalNot); - if (negate == nullptr || negate->Arg(0).Resolve() != either->Arg(1 - index).Resolve()) continue; - const auto original = negate->Arg(0).Resolve(); - if (saved_exec.IsEmpty()) saved_exec = original; - return saved_exec == original; - } - return false; - } - - Value Unwrap(Value value) { - while (const auto* select = Get(value, ValueOpcode::SelectU32)) { - if (!FullExec(select->Arg(0))) break; - value = select->Arg(1); - } - return value.Resolve(); - } + // Constant propagation folds the x | ~x EXEC of s_orn2_saveexec to true. + static bool FullExec(Value predicate) { return predicate.Resolve() == Value(true); } static bool IsRowShift(const Inst& inst, uint32_t shift) { const auto flags = inst.Flags(); @@ -200,24 +166,24 @@ private: flags.bank_mask == 0xf && !flags.fetch_inactive && !flags.bound_control && !flags.dpp8; } - Value MatchRow(Value value, uint32_t shift) { - const auto* update = Get(Unwrap(value), ValueOpcode::DppUpdateU32); + Value MatchRow(Value value, uint32_t shift) const { + const auto* update = Get(value, ValueOpcode::DppUpdateU32); if (update == nullptr || !IsRowShift(*update, shift) || !FullExec(update->Arg(2))) return {}; const auto* combined = Get(update->Arg(0), operation); if (combined == nullptr) return {}; for (size_t index = 0; index < 2; index++) { const auto* move = Get(combined->Arg(index), ValueOpcode::DppMoveU32); if (move == nullptr || !IsRowShift(*move, shift) || !FullExec(move->Arg(1))) continue; - const auto previous = Unwrap(combined->Arg(1 - index)); - if (Unwrap(move->Arg(0)) == previous && Unwrap(update->Arg(1)) == previous) return previous; + const auto previous = combined->Arg(1 - index).Resolve(); + if (move->Arg(0).Resolve() == previous && update->Arg(1).Resolve() == previous) { + return previous; + } } return {}; } - Block* block; - ValueOpcode operation; - uint32_t identity; - Value saved_exec; + const Block* block; + ValueOpcode operation; }; } // namespace diff --git a/tests/ScalarProvenanceTests.cpp b/tests/ScalarProvenanceTests.cpp index 04b361d46..de59a8cd6 100644 --- a/tests/ScalarProvenanceTests.cpp +++ b/tests/ScalarProvenanceTests.cpp @@ -630,39 +630,39 @@ void TestReadLaneElimination() { void TestCompleteWaveReduction() { enum class Case { - Masked, Subset, Literal, LiteralMasked, LiteralMove, Wave32, Wave32Rows, - Wave32Maximum, UnrelatedSeed, UnmaskedSymbolic, WrongNeutral, RowMask, BankMask, + Masked, ScalarExec, Wave32, Wave32Rows, Wave32Maximum, RowMask, BankMask, Shift, FetchInactive, BoundControl, Dpp8, MoveExec, UpdateExec, Previous, DppSource, CrossExec, CrossRow, CrossSelectorLow, CrossSelectorHigh, CrossFetch, CrossBound, CrossSource, FinalExec, - Endpoint, EndpointSource, DifferentWitness, CrossBlock, WrongOperation + Endpoint, EndpointSource, CrossBlock, WrongOperation }; - for (const auto test : {Case::Masked, Case::Subset, Case::Literal, - Case::LiteralMasked, Case::LiteralMove, Case::Wave32, Case::Wave32Rows, - Case::Wave32Maximum, Case::UnrelatedSeed, - Case::UnmaskedSymbolic, Case::WrongNeutral, Case::RowMask, Case::BankMask, + for (const auto test : {Case::Masked, Case::ScalarExec, Case::Wave32, + Case::Wave32Rows, Case::Wave32Maximum, Case::RowMask, Case::BankMask, Case::Shift, Case::FetchInactive, Case::BoundControl, Case::Dpp8, Case::MoveExec, Case::UpdateExec, Case::Previous, Case::DppSource, Case::CrossExec, Case::CrossRow, Case::CrossSelectorLow, Case::CrossSelectorHigh, Case::CrossFetch, Case::CrossBound, Case::CrossSource, Case::FinalExec, Case::Endpoint, Case::EndpointSource, - Case::DifferentWitness, Case::CrossBlock, Case::WrongOperation}) { + Case::CrossBlock, Case::WrongOperation}) { Fixture fixture(test == Case::CrossBlock ? 2 : 1); const auto source = fixture.Emit(ValueOpcode::LaneId); const auto mask = fixture.Emit(ValueOpcode::ULessThan32, {source, Value(19u)}); - const auto other_mask = fixture.Emit(ValueOpcode::IEqual32, {source, Value(7u)}); const auto negated = fixture.Emit(ValueOpcode::LogicalNot, {mask}); - const bool literal = test == Case::Literal || test == Case::LiteralMasked; - const auto enabled = literal ? Value(true) : - fixture.Emit(ValueOpcode::LogicalOr, {negated, mask}); - auto seed_mask = test == Case::UnrelatedSeed ? other_mask : mask; - if (test == Case::Subset) - seed_mask = fixture.Emit(ValueOpcode::LogicalAnd, {other_mask, mask}); + auto enabled = fixture.Emit(ValueOpcode::LogicalOr, {negated, mask}); + if (test == Case::ScalarExec) { + // s_orn2_saveexec of a mask that is not a ballot, read back as a lane bit. + const auto exec = fixture.Emit(ValueOpcode::GetUserData, {Value(static_cast(2))}); + const auto full = fixture.Emit(ValueOpcode::BitwiseOr32, + {fixture.Emit(ValueOpcode::BitwiseNot32, {exec}), exec}); + const auto bit = fixture.Emit(ValueOpcode::BitwiseAnd32, {source, Value(31u)}); + const auto shifted = fixture.Emit(ValueOpcode::ShiftRightLogical32, {full, bit}); + enabled = fixture.Emit(ValueOpcode::INotEqual32, + {fixture.Emit(ValueOpcode::BitwiseAnd32, {shifted, Value(1u)}), Value(0u)}); + } // A UMax scan seeds lanes outside EXEC with zero. const bool maximum = test == Case::Wave32Maximum; - const uint32_t neutral = test == Case::WrongNeutral || maximum ? 0u : UINT32_MAX; - const auto seed = test == Case::Literal || test == Case::UnmaskedSymbolic ? source : - fixture.Emit(ValueOpcode::SelectU32, {seed_mask, source, Value(neutral)}); + const uint32_t neutral = maximum ? 0u : UINT32_MAX; + const auto seed = fixture.Emit(ValueOpcode::SelectU32, {mask, source, Value(neutral)}); auto prefix = fixture.Emit(ValueOpcode::SelectU32, {enabled, seed, source}); Value preserved; auto operation = maximum ? ValueOpcode::UMax32 : ValueOpcode::UMin32; @@ -678,8 +678,7 @@ void TestCompleteWaveReduction() { if (test == Case::Dpp8) flags.dpp8 = true; } const auto move = fixture.Emit(ValueOpcode::DppMoveU32, - {test == Case::DppSource ? source : prefix, - test == Case::MoveExec ? mask : test == Case::LiteralMove ? Value(true) : enabled}); + {test == Case::DppSource ? source : prefix, test == Case::MoveExec ? mask : enabled}); move.Instruction()->SetFlags(flags); const auto combined = fixture.Emit(operation, {move, prefix}); const auto update = fixture.Emit(ValueOpcode::DppUpdateU32, @@ -689,11 +688,7 @@ void TestCompleteWaveReduction() { prefix = update; if (shift == 2) preserved = fixture.Emit(ValueOpcode::ReferenceU32, {prefix}); } - auto cross_exec = test == Case::CrossExec ? mask : enabled; - if (test == Case::DifferentWitness) { - const auto other_not = fixture.Emit(ValueOpcode::LogicalNot, {other_mask}); - cross_exec = fixture.Emit(ValueOpcode::LogicalOr, {other_mask, other_not}); - } + const auto cross_exec = test == Case::CrossExec ? mask : enabled; const auto cross = fixture.Emit(ValueOpcode::Permlane16U32, {test == Case::CrossSource ? source : prefix, Value(test == Case::CrossSelectorLow ? 0xfffffff0u : UINT32_MAX), @@ -708,7 +703,7 @@ void TestCompleteWaveReduction() { {test == Case::FinalExec ? mask : enabled, combined, prefix}); const uint32_t result_block = test == Case::CrossBlock ? 1u : 0u; // Wave32 reads each half's row endpoint from the scan itself. - const bool rows = test == Case::Wave32Rows || maximum; + const bool rows = test == Case::Wave32Rows || test == Case::ScalarExec || maximum; const auto endpoints = rows ? prefix : final; uint32_t low_lane = test == Case::Endpoint ? 30u : 31u; if (rows) low_lane = 15u; @@ -727,11 +722,11 @@ void TestCompleteWaveReduction() { fixture.BlockAt().terminator.true_block = &fixture.BlockAt(1); } ValidateProgram(fixture.program, true); - const auto stats = - EliminateReadLane(fixture.program, test == Case::Wave32 || rows ? 32u : 64u); - const bool matches = test == Case::Masked || test == Case::Subset || - test == Case::Literal || test == Case::LiteralMasked || test == Case::LiteralMove || - rows; + // The pipeline folds full-EXEC predicates to true before eliminating reads. + const uint32_t wave_size = test == Case::Wave32 || rows ? 32u : 64u; + ConstantPropagationPass(fixture.program.blocks, wave_size); + const auto stats = EliminateReadLane(fixture.program, wave_size); + const bool matches = test == Case::Masked || rows; Check(stats.rewritten_reductions == (matches ? 1u : 0u) && stats.rewritten_reads == 0, ("wave reduction proof failed case " + std::to_string(static_cast(test))).c_str()); if (matches) { @@ -754,7 +749,7 @@ void TestCompleteWaveReduction() { ValidateProgram(fixture.program, true); } else { Check(use.Instruction()->Arg(0).Resolve() == result, - "rejected UMin proof changed its original result"); + "rejected wave reduction proof changed its original result"); } } }