shader: recover counted image rows through private LDS spills

src/graphics/shader/recompiler/ir/passes/ResourceTracking.cpp

@@ -356,6 +356,7 @@ public:
 		if (m_program.resource_tracking_complete) {
 			Fail(0, "resources already tracked");
 		}
+		ForwardPrivateLds();
 		FoldBoundedLoopSelectors();
 		PlanScalarReads();
 		EliminateDeadCode(m_program.blocks);
@@ -1622,10 +1623,18 @@ private:
 		    [&](Value at) -> std::optional<size_t> {
 			    const auto* inst = at.TryInstruction();
 			    if (inst == nullptr) return {};
+			    uint32_t bits = 32u;
+			    if (inst->GetOpcode() == ValueOpcode::BitFieldUExtract &&
+			        inst->Arg(1).Resolve() == Value(0u) &&
+			        ImmediateU32(inst->Arg(2), bits) && bits <= 32u) {
+				    inst = inst->Arg(0).Resolve().TryInstruction();
+				    if (inst == nullptr) return {};
+			    }
 			    uint32_t axis = 0;
 			    if (inst->GetOpcode() == ValueOpcode::GetBuiltin &&
 			        inst->Arg(0) == Value(static_cast<uint32_t>(StageInputKind::LocalInvocationId)) &&
-			        ImmediateU32(inst->Arg(1), axis) && axis < 3u) return axis + 1u;
+			        ImmediateU32(inst->Arg(1), axis) && axis < 3u &&
+			        m_workgroup->threads_num[axis] <= (uint64_t {1} << bits)) return axis + 1u;
 			    if (!allow_induction || inst->GetOpcode() != ValueOpcode::Phi ||
 			        (!induction.IsEmpty() && induction != at)) return {};
 			    induction = at;
@@ -1642,6 +1651,66 @@ private:
 		return index[0] == 0u;
 	}
 
+	void ForwardPrivateLds() {
+		if (m_workgroup == nullptr) return;
+		uint64_t bytes = 4u;
+		for (const auto size: m_workgroup->threads_num) {
+			if (size == 0u || size > 1024u) return;
+			bytes *= size;
+		}
+		if (bytes > uint64_t {m_workgroup->lds_size_dwords} * 4u) return;
+		struct Access { Inst* inst; uint32_t offset; };
+		std::vector<Access> reads, writes;
+		Value address;
+		for (auto* block: m_program.blocks) {
+			for (auto& inst: *block) {
+				const auto op = inst.GetOpcode();
+				const auto shared = SharedAccessOf(op);
+				if (shared == SharedAccess::None &&
+				    AddressOpcodeInfoOf(op).access != AddressAccess::Write) continue;
+				const auto& memory = m_program.memory_info.at(inst.Flags<MemoryFlags>().index);
+				if (memory.kind == ResourceKind::Flat || memory.kind == ResourceKind::FlatLocal) return;
+				if (memory.kind != ResourceKind::Lds) continue;
+				if ((op != ValueOpcode::LoadSharedU32 && op != ValueOpcode::WriteSharedU32) ||
+				    memory.data_bits != 32u || memory.secondary_offset != 0u ||
+				    (memory.offset & 3u) != 0u ||
+				    uint64_t {memory.offset} + bytes > uint64_t {m_workgroup->lds_size_dwords} * 4u)
+					return;
+				if (address.IsEmpty()) address = inst.Arg(0).Resolve();
+				else if (!EquivalentValue(m_program, address, inst.Arg(0))) return;
+				(op == ValueOpcode::LoadSharedU32 ? reads : writes).push_back({&inst, memory.offset});
+			}
+		}
+		Value induction;
+		auto index = LocalIndex(address, Value(true), induction, false);
+		if (!index) return;
+		for (auto& coefficient: *index) {
+			if ((coefficient & 3u) != 0u) return;
+			coefficient /= 4u;
+		}
+		if (!LinearLocalIndex(*index)) return;
+		// Each invocation owns one DWORD in each disjoint slot. A unique dominating
+		// writer therefore survives loops and barriers without cross-lane interference.
+		for (const auto& store: writes) {
+			if (InCycle(store.inst->Parent()) || std::ranges::any_of(writes, [&](const Access& other) {
+				return other.inst != store.inst && uint64_t {other.offset} < store.offset + bytes &&
+				       uint64_t {store.offset} < other.offset + bytes;
+			})) continue;
+			for (const auto& load: reads) {
+				if (load.offset != store.offset || !AvailableBefore(*store.inst, *load.inst) ||
+				    !Implies(load.inst->Arg(1), store.inst->Arg(2))) continue;
+				auto* block = load.inst->Parent();
+				const auto where = std::ranges::find_if(block->Instructions(),
+				    [&](const Inst& at) { return &at == load.inst; });
+				const auto guard = load.inst->Arg(1);
+				const auto value = ActiveValue(store.inst->Arg(1), guard);
+				auto& forwarded = *block->PrependNewInst(where, ValueOpcode::SelectU32,
+				    {guard, value, Value(0u)});
+				load.inst->ReplaceUsesWith(Value(&forwarded));
+			}
+		}
+	}
+
 	bool LocalZeroWitness(Value guard) const {
 		const auto* equal = SimplifyGuard(guard).TryInstruction();
 		if (equal == nullptr || equal->GetOpcode() != ValueOpcode::IEqual32) return false;
@@ -1692,8 +1761,8 @@ private:
 		return {};
 	}
 
-	Value UnsignedBoundOnEntry(Value key, const Block* block, bool finite_bitscan = false,
+	Value UnsignedBoundOnEntry(Value key, const Block* block, bool finite_bitscan,
-	                           Value witness = {}) const {
+	                           Value witness, const auto& accepts_bound) const {
 		const auto* definition = key.Resolve().TryInstruction();
 		// A mask-nonzero guard may precede FF1. Its proof lasts until the mask's
 		// definition, including a loop Phi that changes it for the next iteration.
@@ -1705,15 +1774,22 @@ private:
 		}, [&](const EdgePredicate& edge) {
 			if (edge.lanes != LaneQuantifier::All) return false;
 			const auto candidate = UnsignedBound(edge.condition, key, edge.positive, finite_bitscan, witness);
-			if (candidate.IsEmpty() ||
-			    !ValidateRuntimeValue(m_program, candidate, RuntimeValueType::Integer)) return false;
-			if (bound.IsEmpty()) bound = candidate;
 			// Every incoming path must prove the same bound for this key definition.
-			return EquivalentValue(m_program, candidate, bound);
+			if (candidate.IsEmpty() || (!bound.IsEmpty() &&
+			    !EquivalentValue(m_program, candidate, bound)) || !accepts_bound(candidate)) return false;
+			bound = candidate;
+			return true;
 		});
 		return guarded ? bound : Value {};
 	}
 
+	Value UnsignedBoundOnEntry(Value key, const Block* block, bool finite_bitscan = false,
+	                           Value witness = {}) const {
+		return UnsignedBoundOnEntry(key, block, finite_bitscan, witness, [&](Value bound) {
+			return ValidateRuntimeValue(m_program, bound, RuntimeValueType::Integer);
+		});
+	}
+
 	Value UnsignedImageBound(Value key, const Inst& handle) const {
 		Value bound;
 		for (const auto& use: handle.Uses()) {
@@ -1903,10 +1979,11 @@ private:
 		const auto* sum = element.TryInstruction();
 		if (sum == nullptr || sum->GetOpcode() != ValueOpcode::IAdd32) return false;
 		for (uint32_t arm = 0; arm < 2u; ++arm) {
-			const auto induction = sum->Arg(arm).Resolve();
+			const auto index = sum->Arg(arm).Resolve();
-			if (BoundedLoop(induction, read.Parent(), [&](const Inst& compare, uint32_t) {
+			// Enumerating every unsigned index below the count needs the access guard,
-				if (compare.GetOpcode() != ValueOpcode::ULessThan32) return false;
+			// independently of the loop's initial value, step or termination.
-				const auto* count = compare.Arg(1).Resolve().TryInstruction();
+			if (!UnsignedBoundOnEntry(index, read.Parent(), false, {}, [&](Value bound) {
+				const auto* count = bound.Resolve().TryInstruction();
 				if (count == nullptr || count->GetOpcode() != ValueOpcode::ReadConstBuffer) return false;
 				uint32_t memory_index = 0;
 				const auto* memory = ScalarReadMemory(*count, memory_index);
@@ -1918,14 +1995,17 @@ private:
 				if (selected_lane) {
 					const auto* row_lane = row.TryInstruction();
 					if (row_lane == nullptr || row_lane->GetOpcode() != ValueOpcode::ReadFirstLane ||
-					    !ValidateWaveUniformValue(m_program, row_lane->Arg(0))) return false;
+					    !ValidateWaveUniformValue(m_program,
-					// Different EXEC masks may select different lanes. A globally uniform row
+					        EquivalentValue(m_program, row_lane->Arg(1), lane->Arg(1))
-					// keeps their count and key addresses correlated; i is bounded on every lane.
+					            ? row : row_lane->Arg(0))) return false;
-					row = row_lane->Arg(0).Resolve();
+					// Equal EXEC masks select the same row; differing masks need a globally
+					// uniform row to keep count and key addresses correlated.
+					row = ActiveValue(row_lane->Arg(0), lane->Arg(1));
 				}
-				const auto affine = AffineIndex<3>(element, [](Value value) { return value.Resolve(); },
+				const auto affine = AffineIndex<3>(element,
+				    [&](Value value) { return selected_lane ? ActiveValue(value, lane->Arg(1)) : value.Resolve(); },
 				    [&](Value value) -> std::optional<size_t> {
-					    if (value == induction) return 1u;
+					    if (value == index) return 1u;
 					    if (EquivalentValue(m_program, value, row)) return 2u;
 					    return {};
 				    });
@@ -1936,7 +2016,7 @@ private:
 				// spacing preserves those aliases in the U32 selector address too.
 				row_stride = static_cast<uint32_t>((*affine)[2] * 4u);
 				return true;
-			}, selected_lane ? LaneQuantifier::All : LaneQuantifier::Any) != nullptr) return true;
+			}).IsEmpty()) return true;
 		}
 		return false;
 	}

tests/ResourceTrackingTests.cpp

@@ -2409,27 +2409,91 @@ void TestScalarBufferSelectorDomain() {
         "large V# bounds extended the native U32 domain or invented an OOB zero key");
 }
 
+void TestPrivateLdsForwarding() {
+  namespace CFG = Libs::Graphics::ShaderRecompiler::CFG;
+  enum class Variant { Valid, Overlap, Duplicate, CrossLane, Truncated, OutOfBounds, Mask, Bypass };
+  for (const auto variant : {Variant::Valid, Variant::Overlap, Variant::Duplicate,
+                            Variant::CrossLane, Variant::Truncated, Variant::OutOfBounds,
+                            Variant::Mask, Variant::Bypass}) {
+    Fixture fixture;
+    auto *entry = fixture.block;
+    auto *body = fixture.AddBlock();
+    auto *exit = fixture.AddBlock();
+    const auto local = [&](uint32_t axis) {
+      return fixture.Emit(ValueOpcode::GetBuiltin,
+          {Value(static_cast<uint32_t>(StageInputKind::LocalInvocationId)), Value(axis)});
+    };
+    const auto x = local(0u), y = local(1u);
+    const auto address = fixture.Emit(ValueOpcode::ShiftLeftLogical32,
+        {fixture.Emit(ValueOpcode::IAdd32,
+            {x, fixture.Emit(ValueOpcode::IMul32,
+                {Value(16u), fixture.Emit(ValueOpcode::BitFieldUExtract,
+                    {y, Value(0u), Value(variant == Variant::Truncated ? 1u : 24u)})})}), Value(2u)});
+    const auto active = fixture.Emit(ValueOpcode::INotEqual32, {x, Value(0u)});
+    entry->AddBranch(body);
+    entry->terminator = {.kind = CFG::TerminatorKind::Branch, .true_block = body};
+    if (variant == Variant::Bypass) {
+      entry->condition = active;
+      entry->AddBranch(exit);
+      entry->terminator = {.kind = CFG::TerminatorKind::ConditionalBranch,
+                          .true_block = body, .false_block = exit};
+    }
+    MemoryInfo memory;
+    memory.kind = ResourceKind::Lds;
+    memory.offset = 4096u;
+    fixture.Emit(ValueOpcode::WriteSharedU32,
+        {address, Value(0x12345678u), variant == Variant::Mask ? active : Value(true)},
+        fixture.AddMemory(memory, 0x10), variant == Variant::Bypass ? body : entry);
+    if (variant == Variant::Overlap || variant == Variant::Duplicate) {
+      auto other = memory;
+      other.offset += variant == Variant::Overlap ? 4u : 0u;
+      fixture.Emit(ValueOpcode::WriteSharedU32, {address, Value(0u), Value(true)},
+          fixture.AddMemory(other, 0x14), body);
+    }
+    body->AddBranch(exit);
+    body->terminator = {.kind = CFG::TerminatorKind::Branch, .true_block = exit};
+    const auto load = fixture.Emit(ValueOpcode::LoadSharedU32,
+        {variant == Variant::CrossLane
+             ? fixture.Emit(ValueOpcode::IAdd32, {address, Value(4u)}) : address, Value(true)},
+        fixture.AddMemory(memory, 0x20), exit);
+    const auto use = fixture.Emit(ValueOpcode::ReferenceU32, {load}, 0, exit);
+    ShaderComputeInputInfo compute;
+    compute.threads_num[0] = compute.threads_num[1] = 16u;
+    compute.threads_num[2] = 1u;
+    compute.lds_size_dwords = variant == Variant::OutOfBounds ? 1024u : 2048u;
+    fixture.PlanAndTrack({.compute = &compute});
+    const auto result = use.Instruction()->Arg(0).ResolveInstruction();
+    Check(result != nullptr &&
+              (result->GetOpcode() == ValueOpcode::SelectU32) == (variant == Variant::Valid),
+          "private LDS forwarding lost its value or crossed an alias, mask, bound or dominance hazard");
+  }
+}
+
 void TestCountedRowImageSelector() {
   namespace CFG = Libs::Graphics::ShaderRecompiler::CFG;
-  enum class Variant { Valid, Signed, WrongStep, Bypass, WrongRow, CountStride,
+  enum class Variant { Valid, Signed, StepTwo, SkippedIteration, WrongRow, CountStride,
-                       KeyStride, CountImmediate, KeyImmediate, Masked, EmptyCountMask, VaryingRow,
+                       KeyStride, CountImmediate, KeyImmediate, DifferentBackedgeBound,
-                       EmptyMask, UnprovenMask, WrongMask, AnyBound, MaskedWrongStep };
+                       Masked, EmptyCountMask, VaryingRow,
+                       EmptyMask, UnprovenMask, WrongMask, AnyBound, MaskedStepTwo, Spilled,
+                       MergedInitial, MissingBound, PreviousIteration };
   const auto make_plan = [](Variant variant = Variant::Valid, uint32_t capacity = 16u) {
     Fixture fixture;
     const bool masked = variant >= Variant::Masked;
+    const bool stable_index = variant == Variant::DifferentBackedgeBound;
     auto *entry = fixture.block;
     auto *header = fixture.AddBlock();
     auto *body = fixture.AddBlock();
     auto *latch = fixture.AddBlock();
     auto *exit = fixture.AddBlock();
     auto *preheader = masked ? fixture.AddBlock() : entry;
+    auto *prepare = variant == Variant::MergedInitial ? fixture.AddBlock() : preheader;
     const auto branch = [](Block *from, Block *to) {
       from->AddBranch(to);
       from->terminator = {.kind = CFG::TerminatorKind::Branch, .true_block = to};
     };
     branch(preheader, header);
     branch(body, latch);
-    branch(latch, header);
+    if (!stable_index) branch(latch, header);
     const auto keys = fixture.Buffer({fixture.UserData(0), fixture.UserData(1),
                                       fixture.UserData(2), fixture.UserData(3)});
     const auto counts = fixture.Buffer({fixture.UserData(4), fixture.UserData(5),
@@ -2462,15 +2526,42 @@ void TestCountedRowImageSelector() {
         entry->condition = fixture.Emit(ValueOpcode::INotEqual32,
             {fixture.UserData(13u), Value(0u)});
       }
-      entry->AddBranch(preheader);
+      entry->AddBranch(prepare);
       entry->AddBranch(exit);
       entry->terminator = {.kind = CFG::TerminatorKind::ConditionalBranch,
-                            .true_block = preheader, .false_block = exit};
+                            .true_block = prepare, .false_block = exit};
       if (variant == Variant::EmptyMask) active = Value(false);
     }
+    Value initial(0u);
+    if (variant == Variant::MergedInitial) {
+      auto *zero = fixture.AddBlock();
+      prepare->condition = fixture.Emit(ValueOpcode::INotEqual32,
+          {fixture.UserData(13u), Value(0u)}, 0, prepare);
+      prepare->AddBranch(zero);
+      prepare->AddBranch(preheader);
+      prepare->terminator = {.kind = CFG::TerminatorKind::ConditionalBranch,
+                              .true_block = zero, .false_block = preheader};
+      branch(zero, preheader);
+      auto &merged = preheader->AppendNewInst(ValueOpcode::Phi, {}, uint64_t(Type::U32));
+      merged.AddPhiOperand(prepare, fixture.UserData(14u));
+      merged.AddPhiOperand(zero, Value(0u));
+      initial = Value(&merged);
+    }
+    Value spill_address;
+    MemoryInfo spill;
+    spill.kind = ResourceKind::Lds;
+    spill.offset = 4096u;
+    if (variant == Variant::Spilled) {
+      spill_address = fixture.Emit(ValueOpcode::ShiftLeftLogical32,
+          {fixture.Emit(ValueOpcode::GetBuiltin,
+              {Value(static_cast<uint32_t>(StageInputKind::LocalInvocationId)), Value(0u)}), Value(2u)});
+      row = fixture.Emit(ValueOpcode::SelectU32, {active, row, Value(0xdeadbeefu)});
+      fixture.Emit(ValueOpcode::WriteSharedU32, {spill_address, row, active},
+                   fixture.AddMemory(spill, 0x08));
+    }
     auto count_row = row;
     if (masked) count_row = fixture.Emit(ValueOpcode::ReadFirstLane,
-        {row, Value(variant != Variant::EmptyCountMask)});
+        {row, variant == Variant::Spilled ? active : Value(variant != Variant::EmptyCountMask)});
     if (variant == Variant::WrongRow)
       count_row = fixture.Emit(ValueOpcode::IAdd32, {row, Value(1u)});
     const auto count_offset = fixture.Emit(ValueOpcode::ShiftLeftLogical32,
@@ -2478,21 +2569,46 @@ void TestCountedRowImageSelector() {
     scalar.offset = variant == Variant::CountImmediate ? 4u : 0u;
     const auto count = fixture.Emit(ValueOpcode::ReadConstBuffer,
         {counts, count_offset}, fixture.AddMemory(scalar, 0x10));
-    auto &induction = header->AppendNewInst(ValueOpcode::Phi, {}, uint64_t(Type::U32));
+    auto *induction = stable_index ? nullptr
+        : &header->AppendNewInst(ValueOpcode::Phi, {}, uint64_t(Type::U32));
+    auto index = stable_index ? fixture.UserData(13u) : Value(induction);
     const auto guard = fixture.Emit(variant == Variant::Signed ? ValueOpcode::SLessThan32
                                                               : ValueOpcode::ULessThan32,
-        {Value(&induction), count}, 0, header);
+        {index, count}, 0, header);
     header->AddBranch(body);
-    auto *outside = variant == Variant::Bypass ? latch : exit;
+    if (variant == Variant::PreviousIteration) {
-    header->AddBranch(outside);
+      header->terminator = {.kind = CFG::TerminatorKind::Branch, .true_block = body};
-    header->condition = variant == Variant::AnyBound
+      latch->AddBranch(exit);
-        ? fixture.Emit(ValueOpcode::ConditionRef, {guard}, CFG::BranchCondition::ExecNonZero, header)
+      latch->condition = guard;
-        : guard;
+      latch->terminator = {.kind = CFG::TerminatorKind::ConditionalBranch,
-    header->terminator = {.kind = CFG::TerminatorKind::ConditionalBranch,
+                            .true_block = header, .false_block = exit};
-                          .true_block = body, .false_block = outside};
+    } else {
+      auto *outside = variant == Variant::SkippedIteration ? latch : exit;
+      header->AddBranch(outside);
+      header->condition = variant == Variant::AnyBound
+          ? fixture.Emit(ValueOpcode::ConditionRef, {guard}, CFG::BranchCondition::ExecNonZero, header)
+          : variant == Variant::MissingBound
+              ? fixture.Emit(ValueOpcode::INotEqual32, {fixture.UserData(13u), Value(0u)}, 0, header)
+              : guard;
+      header->terminator = {.kind = CFG::TerminatorKind::ConditionalBranch,
+                            .true_block = body, .false_block = outside};
+    }
+    if (stable_index) {
+      const auto backedge_count = fixture.Emit(ValueOpcode::ReadConstBuffer,
+          {keys, count_offset}, fixture.AddMemory(scalar, 0x14), latch);
+      latch->condition = fixture.Emit(ValueOpcode::ULessThan32, {index, backedge_count}, 0, latch);
+      latch->AddBranch(body);
+      latch->AddBranch(exit);
+      latch->terminator = {.kind = CFG::TerminatorKind::ConditionalBranch,
+                            .true_block = body, .false_block = exit};
+    }
     fixture.block = body;
+    if (variant == Variant::Spilled) {
+      row = fixture.Emit(ValueOpcode::SelectU32,
+          {active, fixture.Emit(ValueOpcode::LoadSharedU32, {spill_address, active},
+                                 fixture.AddMemory(spill, 0x18)), Value(0xbadbeefu)});
+    }
     const auto row_start = fixture.Emit(ValueOpcode::IMul32, {row, Value(capacity)});
-    auto index = Value(&induction);
     if (variant == Variant::KeyStride)
       index = fixture.Emit(ValueOpcode::ShiftLeftLogical32, {index, Value(1u)});
     auto element = fixture.Emit(ValueOpcode::IAdd32, {row_start, index});
@@ -2520,33 +2636,50 @@ void TestCountedRowImageSelector() {
     fixture.Emit(ValueOpcode::ImageSampleRaw,
         {fixture.Image(image_words), fixture.Sampler({Value(0u), Value(0u), Value(0u), Value(0u)}),
          fixture.ImageAddress(), Value(true)}, fixture.AddMemory(sample, 0x40));
-    const auto step = fixture.Emit(ValueOpcode::IAdd32,
+    if (induction != nullptr) {
-        {Value(&induction), Value(variant == Variant::WrongStep ||
+      const auto step = fixture.Emit(ValueOpcode::IAdd32,
-                                  variant == Variant::MaskedWrongStep ? 2u : 1u)}, 0, latch);
+          {Value(induction), Value(variant == Variant::StepTwo ||
-    induction.AddPhiOperand(preheader, Value(0u));
+                                   variant == Variant::MaskedStepTwo ? 2u : 1u)}, 0, latch);
-    induction.AddPhiOperand(latch, step);
+      induction->AddPhiOperand(preheader, initial);
-    fixture.PlanAndTrack();
+      induction->AddPhiOperand(latch, step);
+    }
+    ShaderComputeInputInfo compute;
+    compute.threads_num[0] = 64u;
+    compute.threads_num[1] = compute.threads_num[2] = 1u;
+    compute.lds_size_dwords = 2048u;
+    fixture.PlanAndTrack({.compute = &compute});
     const auto &source = fixture.program.descriptor_sources[fixture.program.info.images[0].source];
     Check(source.indirect_descriptor.has_value(), "counted-row image lost its indirect source");
     const auto &selector = source.indirect_descriptor->selector;
-    const bool valid = variant == Variant::Valid || variant == Variant::Masked ||
+    const bool valid = variant == Variant::Valid || variant == Variant::Masked || variant == Variant::Spilled ||
-                       variant == Variant::EmptyCountMask;
+                       variant == Variant::EmptyCountMask || variant == Variant::MergedInitial ||
+                       variant == Variant::StepTwo || variant == Variant::MaskedStepTwo ||
+                       variant == Variant::SkippedIteration || stable_index;
     Check((selector && selector->rows.has_value()) == valid,
           "counted-row selector lost its proof or accepted mismatched row/induction bounds");
     if (valid)
       Check(selector->stride == 4u && selector->offset == 0u &&
                 selector->rows->stride == uint64_t(capacity) * 4u,
             "counted-row selector changed native row or element byte spacing");
+    if (stable_index)
+      Check(EquivalentValue(fixture.program,
+                fixture.program.descriptor_sources[selector->rows->source].dwords[0],
+                counts.Instruction()->Arg(0)),
+            "a rejected backedge bound replaced the count source proven on entry");
     return ExtractResourcePlan(fixture.program);
   };
-  auto plan = make_plan();
+  auto plan = make_plan(Variant::MergedInitial);
+  make_plan();
   make_plan(Variant::Masked);
+  make_plan(Variant::Spilled);
   make_plan(Variant::EmptyCountMask);
-  for (const auto variant : {Variant::Signed, Variant::WrongStep, Variant::Bypass,
+  make_plan(Variant::DifferentBackedgeBound);
+  for (const auto variant : {Variant::Signed, Variant::StepTwo, Variant::SkippedIteration,
                              Variant::WrongRow, Variant::CountStride, Variant::KeyStride,
                              Variant::CountImmediate, Variant::KeyImmediate, Variant::VaryingRow,
                              Variant::EmptyMask, Variant::UnprovenMask, Variant::WrongMask,
-                             Variant::AnyBound, Variant::MaskedWrongStep})
+                             Variant::AnyBound, Variant::MaskedStepTwo,
+                             Variant::MissingBound, Variant::PreviousIteration})
     make_plan(variant);
 
   constexpr uint32_t CountBase = 0x1000u, KeyBase = 0x2000u, MaterialBase = 0x4000u;
@@ -5369,6 +5502,7 @@ int main() {
     Run("image descriptor fields", TestImageDescriptorFields);
     Run("draw-uniform scalar image", TestUniformScalarBufferImage);
     Run("scalar buffer selector domain", TestScalarBufferSelectorDomain);
+    Run("private LDS forwarding", TestPrivateLdsForwarding);
     Run("counted row image selector", TestCountedRowImageSelector);
     Run("shared append selector bounds", TestSharedAppendSelectorBounds);
     Run("descending image range", TestDescendingImageRange);