shader: correlate masked first-lane material rows

This commit is contained in:
nmzik
2026-10-12 22:35:48 +02:00
parent 8ee31abd9d
commit 68d7bc94e2
5 changed files with 162 additions and 39 deletions
@@ -1481,7 +1481,8 @@ private:
}
const Inst* BoundedInduction(Value key, const Block* use, const auto& accepts_step,
const auto& accepts_bound) const {
const auto& accepts_bound,
LaneQuantifier lanes = LaneQuantifier::Any) const {
const auto* phi = key.Resolve().TryInstruction();
if (phi == nullptr || phi->GetOpcode() != ValueOpcode::Phi || phi->GetType() != Type::U32 ||
phi->NumArgs() != 2u || phi->NumPhiBlocks() != 2u)
@@ -1523,7 +1524,8 @@ private:
if (use_of_key.operand != 0u ||
!accepts_bound(*compare, initial_arm)) continue;
if (!GuardedOnEntry(use != nullptr ? use : increment_block, stops, [&](const EdgePredicate& edge) {
return edge.positive && Implies(edge.condition, Value(use_of_key.user));
return edge.positive && (lanes == LaneQuantifier::Any || edge.lanes == lanes) &&
Implies(edge.condition, Value(use_of_key.user));
})) continue;
LoopBoundProof proof(m_program, *phi, *compare);
// The image bound and the increment guard are separate obligations: a
@@ -1535,14 +1537,15 @@ private:
return {};
}
const Inst* BoundedLoop(Value key, const Block* use, const auto& accepts_bound) const {
const Inst* BoundedLoop(Value key, const Block* use, const auto& accepts_bound,
LaneQuantifier lanes = LaneQuantifier::Any) const {
return BoundedInduction(key, use,
[](uint32_t initial, uint32_t step) { return initial == 0u && step == 1u; },
[&](const Inst& compare, uint32_t initial) {
return (compare.GetOpcode() == ValueOpcode::SLessThan32 ||
compare.GetOpcode() == ValueOpcode::ULessThan32) &&
accepts_bound(compare, initial);
});
}, lanes);
}
const Inst* DescendingLoop(Value key, const Block* use, uint32_t& maximum) const {
@@ -1830,6 +1833,12 @@ private:
uint32_t offset = 0, stride = 0;
if (!MatchTableOffset(read.Arg(1), element, offset, stride) || offset != 0u || stride != 4u)
return false;
const auto* lane = element.TryInstruction();
const bool selected_lane = lane != nullptr && lane->GetOpcode() == ValueOpcode::ReadFirstLane;
if (selected_lane) {
if (!HasActiveLane(lane->Arg(1), lane->Parent())) return false;
element = ActiveValue(lane->Arg(0), lane->Arg(1));
}
const auto* sum = element.TryInstruction();
if (sum == nullptr || sum->GetOpcode() != ValueOpcode::IAdd32) return false;
for (uint32_t arm = 0; arm < 2u; ++arm) {
@@ -1845,20 +1854,28 @@ private:
if (memory == nullptr || memory->offset != 0u ||
!MatchTableOffset(count->Arg(1), row, count_offset, count_stride) ||
count_offset != 0u || count_stride != 4u) return false;
const auto affine = AffineIndex<3>(read.Arg(1), [](Value value) { return value.Resolve(); },
if (selected_lane) {
const auto* row_lane = row.TryInstruction();
if (row_lane == nullptr || row_lane->GetOpcode() != ValueOpcode::ReadFirstLane ||
!ValidateWaveUniformValue(m_program, row_lane->Arg(0))) return false;
// Different EXEC masks may select different lanes. A globally uniform row
// keeps their count and key addresses correlated; i is bounded on every lane.
row = row_lane->Arg(0).Resolve();
}
const auto affine = AffineIndex<3>(element, [](Value value) { return value.Resolve(); },
[&](Value value) -> std::optional<size_t> {
if (value == induction) return 1u;
if (EquivalentValue(m_program, value, row)) return 2u;
return {};
});
if (!affine || (*affine)[0] != 0u || (*affine)[1] != 4u ||
(*affine)[2] == 0u || ((*affine)[2] & 3u) != 0u ||
if (!affine || (*affine)[0] != 0u || (*affine)[1] != 1u ||
(*affine)[2] == 0u || (*affine)[2] > UINT32_MAX / 4u ||
!MakeRuntimeTableSource(*count, count_source)) return false;
// Count addresses identify rows modulo 2^30. A DWORD-aligned row
// spacing preserves those aliases in the U32 selector address too.
row_stride = static_cast<uint32_t>((*affine)[2]);
row_stride = static_cast<uint32_t>((*affine)[2] * 4u);
return true;
}) != nullptr) return true;
}, selected_lane ? LaneQuantifier::All : LaneQuantifier::Any) != nullptr) return true;
}
return false;
}
@@ -178,8 +178,9 @@ bool IsRuntimeUniformOp(ValueOpcode op) {
class RuntimeValidator {
public:
explicit RuntimeValidator(const ResourcePlan& program, RuntimeValueType type)
: m_program(program), m_type(type) {}
explicit RuntimeValidator(const ResourcePlan& program, RuntimeValueType type,
bool wave_uniform = false)
: m_program(program), m_type(type), m_wave_uniform(wave_uniform) {}
bool Run(Value value) { return Validate(value); }
@@ -278,6 +279,12 @@ private:
}
return finish(true);
}
if (m_wave_uniform && op == ValueOpcode::GetBuiltin) {
const auto axis = inst->NumArgs() == 2u ? inst->Arg(1).Resolve() : Value {};
return finish(m_program.stage == ShaderType::Compute && inst->NumArgs() == 2u &&
inst->Arg(0) == Value(static_cast<uint32_t>(StageInputKind::WorkgroupId)) &&
axis.IsImmediate() && axis.GetType() == Type::U32 && axis.U32() < 3u);
}
if (op == ValueOpcode::Phi) {
if (m_type != RuntimeValueType::Any && !ValidateArguments(*inst, false)) {
return finish(false);
@@ -357,6 +364,7 @@ private:
const ResourcePlan& m_program;
RuntimeValueType m_type;
bool m_wave_uniform;
Value m_active_mask;
std::unordered_set<const Inst*> m_visiting;
std::unordered_set<const Inst*> m_validated_dependencies;
@@ -1153,5 +1161,9 @@ bool ValidateRuntimeValue(const ResourcePlan& program, Value value, RuntimeValue
return RuntimeValidator(program, type).Run(value);
}
bool ValidateWaveUniformValue(const ResourcePlan& program, Value value) {
return RuntimeValidator(program, RuntimeValueType::Any, true).Run(value);
}
} // namespace Libs::Graphics::ShaderRecompiler::IR
@@ -46,6 +46,9 @@ enum class RuntimeValueType { Any, Integer, ImmutableInteger };
bool ValidateRuntimeValue(const ResourcePlan& program, Value value,
RuntimeValueType type = RuntimeValueType::Any);
// Uniform across a compute wave, including values derived from its workgroup ID.
// This does not imply that the host can evaluate the value.
bool ValidateWaveUniformValue(const ResourcePlan& program, Value value);
void CompileSrtWalker(ResourcePlan& program);
// Uses the strict reader for values that affect shader specialization.
SrtRuntime CleanRuntime(SrtRuntime runtime);