shader: fold complementary masks so wave reductions see full EXEC

Co-authored-by: Senaxx <michelbeld@gmail.com>
Co-authored-by: Mac <me@itsmemac.com>

src/graphics/shader/recompiler/ir/passes/ConstantPropagation.cpp

@@ -167,6 +167,16 @@ bool ReplaceBinaryIdentity(Inst& inst, Type type, uint64_t identity) {
 	return false;
 }
 
+// Whether one operand is the other's negation.
+bool IsComplementPair(const Inst& inst, ValueOpcode negation) {
+	for (uint32_t arg = 0; arg < 2u; ++arg) {
+		const auto* negate = Arg(inst, arg).TryInstruction();
+		if (negate != nullptr && negate->GetOpcode() == negation &&
+		    Arg(*negate, 0) == Arg(inst, arg ^ 1u)) return true;
+	}
+	return false;
+}
+
 bool FoldSelect(Inst& inst) {
 	const auto condition   = Arg(inst, 0);
 	const auto true_value  = Arg(inst, 1);
@@ -626,8 +636,9 @@ void FoldInstruction(Block& block, Block::iterator instruction,
 		case ValueOpcode::BitwiseXor32:
 			if (!FoldU32(inst, [opcode = inst.GetOpcode()](uint32_t a, uint32_t b) {
 				    return opcode == ValueOpcode::BitwiseOr32 ? a | b : a ^ b;
-			    })) {
+			    }) && !ReplaceBinaryIdentity(inst, Type::U32, 0u) &&
-				ReplaceBinaryIdentity(inst, Type::U32, 0u);
+			    IsComplementPair(inst, ValueOpcode::BitwiseNot32)) {
+				Replace(inst, Value(UINT32_MAX));
 			}
 			return;
 		case ValueOpcode::BitwiseNot32: {
@@ -802,6 +813,8 @@ void FoldInstruction(Block& block, Block::iterator instruction,
 					Replace(inst, lhs.U1() ? lhs : rhs);
 				} else if (IsImmediate(rhs, Type::U1)) {
 					Replace(inst, rhs.U1() ? rhs : lhs);
+				} else if (IsComplementPair(inst, ValueOpcode::LogicalNot)) {
+					Replace(inst, Value(true));
 				} else {
 					const auto* left = lhs.TryInstruction();
 					const auto* right = rhs.TryInstruction();

src/graphics/shader/recompiler/ir/passes/ReadLaneElimination.cpp

@@ -104,11 +104,10 @@ Value GetRealValue(PhiMap& phi_map, Value source, uint32_t lane, uint32_t wave_s
 // Fragment subgroups may have no physical invocation at either endpoint.
 class WaveReductionMatcher {
 public:
-	WaveReductionMatcher(Block* block, ValueOpcode operation)
+	WaveReductionMatcher(const Block* block, ValueOpcode operation)
-	    : block(block), operation(operation),
+	    : block(block), operation(operation) {}
-	      identity(operation == ValueOpcode::UMin32 ? UINT32_MAX : 0u) {}
 
-	Value Match(const Inst& reduction, uint32_t wave_size) {
+	Value Match(const Inst& reduction, uint32_t wave_size) const {
 		const auto* low  = Get(reduction.Arg(0), ValueOpcode::ReadLane);
 		const auto* high = Get(reduction.Arg(1), ValueOpcode::ReadLane);
 		if (low == nullptr || high == nullptr || low->Arg(0).Resolve() != high->Arg(0).Resolve()) {
@@ -124,45 +123,32 @@ public:
 		      (lane(low->Arg(1), wave_size - 1u) && lane(high->Arg(1), half - 1u)))) {
 			return {};
 		}
-		if (wave_size == 32u) return MatchScan(Unwrap(low->Arg(0)));
+		if (wave_size == 32u) return MatchScan(low->Arg(0));
-		const auto* joined = Get(Unwrap(low->Arg(0)), operation);
+		const auto* joined = Get(low->Arg(0), operation);
 		if (joined == nullptr) return {};
 		for (size_t index = 0; index < 2; index++) {
-			auto attempt = *this;
+			const auto* cross = Get(joined->Arg(index), ValueOpcode::Permlane16U32);
-			const auto* cross = attempt.Get(attempt.Unwrap(joined->Arg(index)), ValueOpcode::Permlane16U32);
 			if (cross == nullptr) continue;
-			const auto flags = cross->Flags<PermlaneFlags>();
+			const auto flags  = cross->Flags<PermlaneFlags>();
-			auto prefix = attempt.Unwrap(joined->Arg(1 - index));
+			const auto prefix = joined->Arg(1 - index).Resolve();
 			if (!flags.x16 || flags.fetch_inactive || !flags.bound_control ||
 			    cross->Arg(1).Resolve() != Value(UINT32_MAX) ||
-			    cross->Arg(2).Resolve() != Value(UINT32_MAX) ||
+			    cross->Arg(2).Resolve() != Value(UINT32_MAX) || !FullExec(cross->Arg(3)) ||
-			    !attempt.FullExec(cross->Arg(3)) || attempt.Unwrap(cross->Arg(0)) != prefix) {
+			    cross->Arg(0).Resolve() != prefix) {
 				continue;
 			}
-			if (const auto seed = attempt.MatchScan(prefix); !seed.IsEmpty()) return seed;
+			if (const auto seed = MatchScan(prefix); !seed.IsEmpty()) return seed;
 		}
 		return {};
 	}
 
 private:
-	// The row scan's input, whose lanes outside the saved EXEC hold the operation's identity.
+	// The row scan's input.
-	Value MatchScan(Value prefix) {
+	Value MatchScan(Value prefix) const {
 		for (const uint32_t shift: {8u, 4u, 2u, 1u}) {
 			prefix = MatchRow(prefix, shift);
 		}
-		prefix = Unwrap(prefix);
+		return prefix;
-		if (prefix.IsEmpty() || saved_exec.IsEmpty()) return prefix;
-		const auto* seed = Get(prefix, ValueOpcode::SelectU32);
-		if (seed != nullptr && seed->Arg(2).Resolve() == Value(identity)) {
-			const auto  predicate = seed->Arg(0).Resolve();
-			const auto* subset    = Get(predicate, ValueOpcode::LogicalAnd);
-			if (predicate == saved_exec ||
-			    (subset != nullptr && (subset->Arg(0).Resolve() == saved_exec ||
-			                           subset->Arg(1).Resolve() == saved_exec))) {
-				return prefix;
-			}
-		}
-		return {};
 	}
 
 	Inst* Get(Value value, ValueOpcode opcode) const {
@@ -171,28 +157,8 @@ private:
 		           ? inst : nullptr;
 	}
 
-	bool FullExec(Value predicate) {
+	// Constant propagation folds the x | ~x EXEC of s_orn2_saveexec to true.
-		predicate = predicate.Resolve();
+	static bool FullExec(Value predicate) { return predicate.Resolve() == Value(true); }
-		if (predicate == Value(true)) return true;
-		const auto* either = Get(predicate, ValueOpcode::LogicalOr);
-		if (either == nullptr) return false;
-		for (size_t index = 0; index < 2; index++) {
-			const auto* negate = Get(either->Arg(index), ValueOpcode::LogicalNot);
-			if (negate == nullptr || negate->Arg(0).Resolve() != either->Arg(1 - index).Resolve()) continue;
-			const auto original = negate->Arg(0).Resolve();
-			if (saved_exec.IsEmpty()) saved_exec = original;
-			return saved_exec == original;
-		}
-		return false;
-	}
-
-	Value Unwrap(Value value) {
-		while (const auto* select = Get(value, ValueOpcode::SelectU32)) {
-			if (!FullExec(select->Arg(0))) break;
-			value = select->Arg(1);
-		}
-		return value.Resolve();
-	}
 
 	static bool IsRowShift(const Inst& inst, uint32_t shift) {
 		const auto flags = inst.Flags<DppMoveFlags>();
@@ -200,24 +166,24 @@ private:
 		       flags.bank_mask == 0xf && !flags.fetch_inactive && !flags.bound_control && !flags.dpp8;
 	}
 
-	Value MatchRow(Value value, uint32_t shift) {
+	Value MatchRow(Value value, uint32_t shift) const {
-		const auto* update = Get(Unwrap(value), ValueOpcode::DppUpdateU32);
+		const auto* update = Get(value, ValueOpcode::DppUpdateU32);
 		if (update == nullptr || !IsRowShift(*update, shift) || !FullExec(update->Arg(2))) return {};
 		const auto* combined = Get(update->Arg(0), operation);
 		if (combined == nullptr) return {};
 		for (size_t index = 0; index < 2; index++) {
 			const auto* move = Get(combined->Arg(index), ValueOpcode::DppMoveU32);
 			if (move == nullptr || !IsRowShift(*move, shift) || !FullExec(move->Arg(1))) continue;
-			const auto previous = Unwrap(combined->Arg(1 - index));
+			const auto previous = combined->Arg(1 - index).Resolve();
-			if (Unwrap(move->Arg(0)) == previous && Unwrap(update->Arg(1)) == previous) return previous;
+			if (move->Arg(0).Resolve() == previous && update->Arg(1).Resolve() == previous) {
+				return previous;
+			}
 		}
 		return {};
 	}
 
-	Block*      block;
+	const Block* block;
-	ValueOpcode operation;
+	ValueOpcode  operation;
-	uint32_t    identity;
-	Value       saved_exec;
 };
 
 } // namespace

tests/ScalarProvenanceTests.cpp

@@ -630,39 +630,39 @@ void TestReadLaneElimination() {
 
 void TestCompleteWaveReduction() {
   enum class Case {
-    Masked, Subset, Literal, LiteralMasked, LiteralMove, Wave32, Wave32Rows,
+    Masked, ScalarExec, Wave32, Wave32Rows, Wave32Maximum, RowMask, BankMask,
-    Wave32Maximum, UnrelatedSeed, UnmaskedSymbolic, WrongNeutral, RowMask, BankMask,
     Shift, FetchInactive, BoundControl, Dpp8, MoveExec, UpdateExec,
     Previous, DppSource, CrossExec, CrossRow, CrossSelectorLow,
     CrossSelectorHigh, CrossFetch, CrossBound, CrossSource, FinalExec,
-    Endpoint, EndpointSource, DifferentWitness, CrossBlock, WrongOperation
+    Endpoint, EndpointSource, CrossBlock, WrongOperation
   };
-  for (const auto test : {Case::Masked, Case::Subset, Case::Literal,
+  for (const auto test : {Case::Masked, Case::ScalarExec, Case::Wave32,
-       Case::LiteralMasked, Case::LiteralMove, Case::Wave32, Case::Wave32Rows,
+       Case::Wave32Rows, Case::Wave32Maximum, Case::RowMask, Case::BankMask,
-       Case::Wave32Maximum, Case::UnrelatedSeed,
-       Case::UnmaskedSymbolic, Case::WrongNeutral, Case::RowMask, Case::BankMask,
        Case::Shift, Case::FetchInactive, Case::BoundControl, Case::Dpp8,
        Case::MoveExec, Case::UpdateExec, Case::Previous, Case::DppSource,
        Case::CrossExec, Case::CrossRow, Case::CrossSelectorLow,
        Case::CrossSelectorHigh, Case::CrossFetch, Case::CrossBound,
        Case::CrossSource, Case::FinalExec, Case::Endpoint, Case::EndpointSource,
-       Case::DifferentWitness, Case::CrossBlock, Case::WrongOperation}) {
+       Case::CrossBlock, Case::WrongOperation}) {
     Fixture fixture(test == Case::CrossBlock ? 2 : 1);
     const auto source = fixture.Emit(ValueOpcode::LaneId);
     const auto mask = fixture.Emit(ValueOpcode::ULessThan32, {source, Value(19u)});
-    const auto other_mask = fixture.Emit(ValueOpcode::IEqual32, {source, Value(7u)});
     const auto negated = fixture.Emit(ValueOpcode::LogicalNot, {mask});
-    const bool literal = test == Case::Literal || test == Case::LiteralMasked;
+    auto enabled = fixture.Emit(ValueOpcode::LogicalOr, {negated, mask});
-    const auto enabled = literal ? Value(true) :
+    if (test == Case::ScalarExec) {
-        fixture.Emit(ValueOpcode::LogicalOr, {negated, mask});
+      // s_orn2_saveexec of a mask that is not a ballot, read back as a lane bit.
-    auto seed_mask = test == Case::UnrelatedSeed ? other_mask : mask;
+      const auto exec = fixture.Emit(ValueOpcode::GetUserData, {Value(static_cast<ScalarReg>(2))});
-    if (test == Case::Subset)
+      const auto full = fixture.Emit(ValueOpcode::BitwiseOr32,
-      seed_mask = fixture.Emit(ValueOpcode::LogicalAnd, {other_mask, mask});
+          {fixture.Emit(ValueOpcode::BitwiseNot32, {exec}), exec});
+      const auto bit = fixture.Emit(ValueOpcode::BitwiseAnd32, {source, Value(31u)});
+      const auto shifted = fixture.Emit(ValueOpcode::ShiftRightLogical32, {full, bit});
+      enabled = fixture.Emit(ValueOpcode::INotEqual32,
+          {fixture.Emit(ValueOpcode::BitwiseAnd32, {shifted, Value(1u)}), Value(0u)});
+    }
     // A UMax scan seeds lanes outside EXEC with zero.
     const bool maximum = test == Case::Wave32Maximum;
-    const uint32_t neutral = test == Case::WrongNeutral || maximum ? 0u : UINT32_MAX;
+    const uint32_t neutral = maximum ? 0u : UINT32_MAX;
-    const auto seed = test == Case::Literal || test == Case::UnmaskedSymbolic ? source :
+    const auto seed = fixture.Emit(ValueOpcode::SelectU32, {mask, source, Value(neutral)});
-        fixture.Emit(ValueOpcode::SelectU32, {seed_mask, source, Value(neutral)});
     auto prefix = fixture.Emit(ValueOpcode::SelectU32, {enabled, seed, source});
     Value preserved;
     auto operation = maximum ? ValueOpcode::UMax32 : ValueOpcode::UMin32;
@@ -678,8 +678,7 @@ void TestCompleteWaveReduction() {
         if (test == Case::Dpp8) flags.dpp8 = true;
       }
       const auto move = fixture.Emit(ValueOpcode::DppMoveU32,
-          {test == Case::DppSource ? source : prefix,
+          {test == Case::DppSource ? source : prefix, test == Case::MoveExec ? mask : enabled});
-           test == Case::MoveExec ? mask : test == Case::LiteralMove ? Value(true) : enabled});
       move.Instruction()->SetFlags(flags);
       const auto combined = fixture.Emit(operation, {move, prefix});
       const auto update = fixture.Emit(ValueOpcode::DppUpdateU32,
@@ -689,11 +688,7 @@ void TestCompleteWaveReduction() {
       prefix = update;
       if (shift == 2) preserved = fixture.Emit(ValueOpcode::ReferenceU32, {prefix});
     }
-    auto cross_exec = test == Case::CrossExec ? mask : enabled;
+    const auto cross_exec = test == Case::CrossExec ? mask : enabled;
-    if (test == Case::DifferentWitness) {
-      const auto other_not = fixture.Emit(ValueOpcode::LogicalNot, {other_mask});
-      cross_exec = fixture.Emit(ValueOpcode::LogicalOr, {other_mask, other_not});
-    }
     const auto cross = fixture.Emit(ValueOpcode::Permlane16U32,
         {test == Case::CrossSource ? source : prefix,
          Value(test == Case::CrossSelectorLow ? 0xfffffff0u : UINT32_MAX),
@@ -708,7 +703,7 @@ void TestCompleteWaveReduction() {
         {test == Case::FinalExec ? mask : enabled, combined, prefix});
     const uint32_t result_block = test == Case::CrossBlock ? 1u : 0u;
     // Wave32 reads each half's row endpoint from the scan itself.
-    const bool rows = test == Case::Wave32Rows || maximum;
+    const bool rows = test == Case::Wave32Rows || test == Case::ScalarExec || maximum;
     const auto endpoints = rows ? prefix : final;
     uint32_t low_lane = test == Case::Endpoint ? 30u : 31u;
     if (rows) low_lane = 15u;
@@ -727,11 +722,11 @@ void TestCompleteWaveReduction() {
       fixture.BlockAt().terminator.true_block = &fixture.BlockAt(1);
     }
     ValidateProgram(fixture.program, true);
-    const auto stats =
+    // The pipeline folds full-EXEC predicates to true before eliminating reads.
-        EliminateReadLane(fixture.program, test == Case::Wave32 || rows ? 32u : 64u);
+    const uint32_t wave_size = test == Case::Wave32 || rows ? 32u : 64u;
-    const bool matches = test == Case::Masked || test == Case::Subset ||
+    ConstantPropagationPass(fixture.program.blocks, wave_size);
-        test == Case::Literal || test == Case::LiteralMasked || test == Case::LiteralMove ||
+    const auto stats = EliminateReadLane(fixture.program, wave_size);
-        rows;
+    const bool matches = test == Case::Masked || rows;
     Check(stats.rewritten_reductions == (matches ? 1u : 0u) && stats.rewritten_reads == 0,
           ("wave reduction proof failed case " + std::to_string(static_cast<unsigned>(test))).c_str());
     if (matches) {
@@ -754,7 +749,7 @@ void TestCompleteWaveReduction() {
       ValidateProgram(fixture.program, true);
     } else {
       Check(use.Instruction()->Arg(0).Resolve() == result,
-            "rejected UMin proof changed its original result");
+            "rejected wave reduction proof changed its original result");
     }
   }
 }