From fa68e104ea7dadef890a2888021728ae788b0c4d Mon Sep 17 00:00:00 2001 From: nmzik Date: Mon, 12 Oct 2026 17:20:42 +0200 Subject: [PATCH] shader: fold full wave64 masks before register predication --- .../ir/passes/ConstantPropagation.cpp | 16 +++++++---- tests/shaderCfgTests.cpp | 27 +++++++++++++++++-- 2 files changed, 36 insertions(+), 7 deletions(-) diff --git a/src/graphics/shader/recompiler/ir/passes/ConstantPropagation.cpp b/src/graphics/shader/recompiler/ir/passes/ConstantPropagation.cpp index 1be9b25d9..b3b95d012 100644 --- a/src/graphics/shader/recompiler/ir/passes/ConstantPropagation.cpp +++ b/src/graphics/shader/recompiler/ir/passes/ConstantPropagation.cpp @@ -706,14 +706,14 @@ void FoldInstruction(Block& block, Block::iterator instruction, } } -// In wave32, extracting this invocation's bit from a ballot recovers its predicate. -// Keep that identity through scalar EXEC operations and their loop-carried mask Phis. +// Full and empty masks have a constant predicate in either wave size. In wave32, +// ballot projection also survives scalar EXEC operations and loop-carried mask Phis. class LaneMaskProjection { public: explicit LaneMaskProjection(std::unordered_set& lowered_ancillary) : m_lowered_ancillary(lowered_ancillary) {} - void Fold(Inst& inst) { + void Fold(Inst& inst, uint32_t wave_size) { if (inst.GetOpcode() != ValueOpcode::INotEqual32 || !Immediate(Arg(inst, 1), 0u)) return; const auto* bit = Arg(inst, 0).TryInstruction(); if (bit == nullptr || bit->GetOpcode() != ValueOpcode::BitwiseAnd32 || @@ -725,9 +725,15 @@ public: !Immediate(Arg(*index, 1), 31u)) return; const auto* lane = Arg(*index, 0).TryInstruction(); if (lane == nullptr || lane->GetOpcode() != ValueOpcode::LaneId) return; + const auto mask = Arg(*shift, 0); + if (Immediate(mask, 0u) || Immediate(mask, UINT32_MAX)) { + Replace(inst, Value(mask.U32() != 0u)); + return; + } + if (wave_size != 32u) return; m_visited.clear(); m_grounded = false; - if (CanProject(Arg(*shift, 0)) && m_grounded) Replace(inst, Project(Arg(*shift, 0))); + if (CanProject(mask) && m_grounded) Replace(inst, Project(mask)); } private: @@ -809,7 +815,7 @@ void ConstantPropagationPass(const BlockList& blocks, uint32_t wave_size) { LaneMaskProjection mask_projection(lowered_ancillary); for (auto* block: blocks) { for (auto inst = block->begin(); inst != block->end(); ++inst) { - if (wave_size == 32u) mask_projection.Fold(*inst); + mask_projection.Fold(*inst, wave_size); FoldInstruction(*block, inst, lowered_ancillary); } } diff --git a/tests/shaderCfgTests.cpp b/tests/shaderCfgTests.cpp index ed5b8aace..86e97d2bd 100644 --- a/tests/shaderCfgTests.cpp +++ b/tests/shaderCfgTests.cpp @@ -12390,7 +12390,7 @@ void TestValuePhiValidation() { #endif } -void TestWave32MaskProjection() { +void TestLaneMaskProjection() { using namespace ShaderRecompiler::IR; for (const uint32_t wave_size : {32u, 64u}) { Program program; @@ -12429,6 +12429,19 @@ void TestWave32MaskProjection() { : U32(Value(3u)); return ir.INotEqual(ir.BitwiseAnd(ir.ShiftRightLogical(word, bit), U32(Value(1u))), zero); }; + const auto low_half = e.ULessThan(U32(e.Emit(ValueOpcode::LaneId)), U32(Value(32u))); + const auto mask_bit = [&](uint32_t low, uint32_t high) { + const auto word = wave_size == 64u ? e.Select(low_half, U32(Value(low)), U32(Value(high))) + : U32(Value(low)); + return thread_bit(e, word); + }; + const auto full = mask_bit(UINT32_MAX, UINT32_MAX); + const auto empty = mask_bit(0u, 0u); + const auto low_only = mask_bit(UINT32_MAX, 0u); + const auto high_only = mask_bit(0u, UINT32_MAX); + const auto partial = mask_bit(0x80000000u, 0x80000000u); + const auto full_write = e.Select(full, U32(Value(42u)), raw); + const auto empty_write = e.Select(empty, U32(Value(42u)), raw); const auto active_mask = ballot(e, active); const auto direct = thread_bit(e, active_mask); const auto arbitrary = thread_bit(e, raw); @@ -12471,7 +12484,14 @@ void TestWave32MaskProjection() { program.blocks[1]->condition = remaining_predicate; ConstantPropagationPass(program.blocks, wave_size); + Check(full.Resolve() == Value(true) && empty.Resolve() == Value(false) && + full_write.Resolve() == Value(42u) && empty_write.Resolve() == raw.Resolve(), + "constant EXEC mask retained lane extraction or register predication"); + Check(partial.Resolve().TryInstruction() != nullptr, + "partial mask was incorrectly folded to a constant predicate"); if (wave_size == 32u) { + Check(low_only.Resolve() == Value(true) && high_only.Resolve() == Value(false), + "wave32 constant mask incorrectly consumed the high half"); Check(direct.Resolve() == active.Resolve(), "wave32 ballot round trip lost its predicate"); Check(EquivalentValue(program, combined, combined_predicate), "wave32 OR mask projection changed a lane predicate"); @@ -12491,6 +12511,9 @@ void TestWave32MaskProjection() { projected->PhiBlock(1) == loop, "mask projection lost loop Phi type or predecessor order"); } else { + Check(low_only.Resolve().TryInstruction() != nullptr && + high_only.Resolve().TryInstruction() != nullptr, + "wave64 mixed halves were incorrectly folded to a constant predicate"); Check(direct.Resolve().TryInstruction()->GetOpcode() == ValueOpcode::INotEqual32 && live.Resolve().TryInstruction()->GetOpcode() == ValueOpcode::INotEqual32 && combined.Resolve().TryInstruction()->GetOpcode() == ValueOpcode::INotEqual32 && @@ -15698,7 +15721,7 @@ int main() { TestFinalSsaRejectsRegisterStatePseudos(); #endif TestValuePhiValidation(); - TestWave32MaskProjection(); + TestLaneMaskProjection(); TestU64ShiftConstantPropagation(); TestU64RegisterPairRoundTrip(); #if KYTY_PLATFORM != KYTY_PLATFORM_WINDOWS