From 4b549b9cb36f64c23cf075359a2ef32c2444f34c Mon Sep 17 00:00:00 2001 From: nmzik Date: Wed, 14 Oct 2026 15:11:12 +0200 Subject: [PATCH] shader: prove constant post-tested image selector loops Co-authored-by: fxpw --- .../recompiler/ir/passes/ResourceTracking.cpp | 92 ++++++++++++++----- tests/ResourceTrackingTests.cpp | 90 +++++++++++++++--- tests/ShaderRecompilerComputeTests.cpp | 43 ++++++--- 3 files changed, 174 insertions(+), 51 deletions(-) diff --git a/src/graphics/shader/recompiler/ir/passes/ResourceTracking.cpp b/src/graphics/shader/recompiler/ir/passes/ResourceTracking.cpp index 049ed3f9a..8f98112e5 100644 --- a/src/graphics/shader/recompiler/ir/passes/ResourceTracking.cpp +++ b/src/graphics/shader/recompiler/ir/passes/ResourceTracking.cpp @@ -1573,43 +1573,52 @@ private: return true; } - const Inst* BoundedInduction(Value key, const Block* use, const auto& accepts_step, - const auto& accepts_bound, - LaneQuantifier lanes = LaneQuantifier::Any) const { - const auto* phi = key.Resolve().TryInstruction(); + struct Induction { + const Inst* phi; + const Inst* step; + uint32_t initial_arm; + }; + + std::optional MatchInduction(Value key, const auto& accepts_step) const { + key = key.Resolve(); + const auto* phi = key.TryInstruction(); if (phi == nullptr || phi->GetOpcode() != ValueOpcode::Phi || phi->GetType() != Type::U32 || - phi->NumArgs() != 2u || phi->NumPhiBlocks() != 2u) + phi->NumArgs() != 2u || phi->NumPhiBlocks() != 2u || phi->Parent() == nullptr) return {}; - const Block* increment_block = nullptr; - uint32_t initial_arm = 0; for (uint32_t initial = 0; initial < 2u; ++initial) { const auto initial_value = phi->Arg(initial).Resolve(); const auto* step = phi->Arg(initial ^ 1u).Resolve().TryInstruction(); if (!initial_value.IsImmediate() || initial_value.GetType() != Type::U32 || step == nullptr || step->GetOpcode() != ValueOpcode::IAdd32 || step->Parent() == nullptr) continue; - // Structurization can forward the native increment through a synthetic continue block. - const auto* incoming = phi->PhiBlock(initial ^ 1u); - const auto* next = phi->Parent(); - for (size_t edges = 0; incoming != step->Parent() && edges < m_program.blocks.size(); ++edges) { - if (incoming == nullptr || incoming->terminator.kind != CFG::TerminatorKind::Branch || - incoming->terminator.true_block != next || incoming->ImmPredecessors().size() != 1u) - break; - next = incoming; - incoming = incoming->ImmPredecessors()[0]; - } - if (incoming != step->Parent()) continue; uint32_t increment = 0; if ((step->Arg(0).Resolve() == key && ImmediateU32(step->Arg(1), increment) && accepts_step(initial_value.U32(), increment)) || (step->Arg(1).Resolve() == key && - ImmediateU32(step->Arg(0), increment) && accepts_step(initial_value.U32(), increment))) { - increment_block = step->Parent(); - initial_arm = initial; - break; - } + ImmediateU32(step->Arg(0), increment) && accepts_step(initial_value.U32(), increment))) + return Induction {phi, step, initial}; } - if (increment_block == nullptr) return {}; + return {}; + } + + const Inst* BoundedInduction(Value key, const Block* use, const auto& accepts_step, + const auto& accepts_bound, + LaneQuantifier lanes = LaneQuantifier::Any) const { + const auto induction = MatchInduction(key, accepts_step); + if (!induction) return {}; + const auto [phi, step, initial_arm] = *induction; + const auto* increment_block = step->Parent(); + // Structurization can forward the native increment through a synthetic continue block. + const auto* incoming = phi->PhiBlock(initial_arm ^ 1u); + const auto* next = phi->Parent(); + for (size_t edges = 0; incoming != increment_block && edges < m_program.blocks.size(); ++edges) { + if (incoming == nullptr || incoming->terminator.kind != CFG::TerminatorKind::Branch || + incoming->terminator.true_block != next || incoming->ImmPredecessors().size() != 1u) + break; + next = incoming; + incoming = incoming->ImmPredecessors()[0]; + } + if (incoming != increment_block) return {}; const auto stops = [&](const Block* block) { return block == phi->Parent(); }; for (const auto& use_of_key: phi->Uses()) { @@ -1630,6 +1639,39 @@ private: return {}; } + uint32_t PostTestLoopMaximum(Value key) const { + const auto induction = MatchInduction(key, + [](uint32_t initial, uint32_t step) { return initial == 0u && step == 1u; }); + if (!induction) return UINT32_MAX; + const auto [phi, step, initial_arm] = *induction; + const auto* incoming = phi->PhiBlock(initial_arm ^ 1u); + if (incoming == nullptr) return UINT32_MAX; + for (const auto& use: step->Uses()) { + const auto* compare = use.user; + uint32_t bound = 0; + if (use.operand != 0u || + (compare->GetOpcode() != ValueOpcode::SLessThan32 && + compare->GetOpcode() != ValueOpcode::ULessThan32) || + !ImmediateU32(compare->Arg(1), bound) || bound == 0u || + (compare->GetOpcode() == ValueOpcode::SLessThan32 && bound > INT32_MAX) || + !AvailableBefore(*step, *compare)) continue; + const auto accepts = [&](const EdgePredicate& edge) { + return edge.positive && edge.lanes == LaneQuantifier::All && + Implies(edge.condition, Value(const_cast(compare))); + }; + const auto edge = ConditionalEdge(incoming, phi->Parent()); + if ((edge && accepts(*edge)) || GuardedOnEntry(incoming, [&](const Block* block) { + // A guard before the increment may belong to the previous iteration. + return block == phi->Parent() || block == step->Parent(); + }, accepts)) { + // Starting at zero, next < bound keeps the carried value in range; + // incrementing bound - 1 cannot overflow either accepted comparison domain. + return bound - 1u; + } + } + return UINT32_MAX; + } + const Inst* BoundedLoop(Value key, const Block* use, const auto& accepts_bound, LaneQuantifier lanes = LaneQuantifier::Any) const { return BoundedInduction(key, use, @@ -2980,6 +3022,8 @@ private: case ValueOpcode::Phi: { const auto initial = InitialCandidateMask(value); if (!initial.IsEmpty()) return SelectorMaximum(initial, bounds, depth + 1u, guard); + const auto post_test = PostTestLoopMaximum(value); + if (post_test != UINT32_MAX) return post_test; if (m_native_cfg.irreducible || !m_native_cfg.back_edges.empty() || inst->NumArgs() == 0u || (inst->Parent() != nullptr && InCycle(inst->Parent()))) return UINT32_MAX; diff --git a/tests/ResourceTrackingTests.cpp b/tests/ResourceTrackingTests.cpp index c1c272259..f2bb4fddf 100644 --- a/tests/ResourceTrackingTests.cpp +++ b/tests/ResourceTrackingTests.cpp @@ -1060,6 +1060,7 @@ void TestGuardedDirectImageTable() { void TestBoundedComputeImageLoop() { namespace CFG = Libs::Graphics::ShaderRecompiler::CFG; enum class Clamp { None, Right, Left, Dynamic }; + enum class PostTest { None, Signed, Unsigned, WrongPolarity, Previous, Bypass, Variable, AnyLane }; enum class Variant { Bounded, Plain, Nonzero, TrueEdge, WrongGuard, EntryBypass, ExitBypass, GuardBlock, WrongStep, DivergentBound, DivergentKey, Disjunction, WrongPolarity, @@ -1069,7 +1070,9 @@ void TestBoundedComputeImageLoop() { GuardedDiamond, GuardedDiamondBypass }; const auto make_plan = [](Variant variant, bool material = false, uint32_t first = 0u, - Clamp clamp = Clamp::None) { + Clamp clamp = Clamp::None, PostTest post = PostTest::None, + uint32_t post_bound = 6u) { + const bool post_test = post != PostTest::None; const bool diamond = variant >= Variant::GuardedDiamond; Fixture fixture(diamond ? ShaderType::Vertex : ShaderType::Compute); fixture.program.wave_size = 64u; @@ -1083,12 +1086,13 @@ void TestBoundedComputeImageLoop() { auto *guard = masked ? fixture.AddBlock() : header; auto *increment = masked ? fixture.AddBlock() : latch; auto *final_exit = variant == Variant::PreviousBound ? fixture.AddBlock() : exit; - const bool forwarded = variant >= Variant::Forwarded && variant <= Variant::ForwardedIncrementBypass; + const bool forwarded = post_test || + (variant >= Variant::Forwarded && variant <= Variant::ForwardedIncrementBypass); auto *forward = forwarded ? fixture.AddBlock() : increment; auto *incoming = variant == Variant::ForwardedChain ? fixture.AddBlock() : forward; - auto *detour = variant == Variant::ForwardedJoin ? fixture.AddBlock() : nullptr; + auto *detour = variant == Variant::ForwardedJoin || post == PostTest::Bypass ? fixture.AddBlock() : nullptr; entry->AddBranch(header); - if (!masked) { + if (!masked && !post_test) { header->AddBranch(exit); header->AddBranch(body); body->AddBranch(latch); @@ -1144,18 +1148,19 @@ void TestBoundedComputeImageLoop() { ValueOpcode::GetBuiltin, {Value(static_cast(diamond ? StageInputKind::VertexIndex : StageInputKind::LocalInvocationId)), Value(0u)}); - auto count = variant == Variant::DivergentBound ? local : fixture.UserData(2); - if (first != 0u && variant != Variant::DivergentBound) + auto count = post_test && post != PostTest::Variable ? Value(post_bound) : + variant == Variant::DivergentBound ? local : fixture.UserData(2); + if (!post_test && first != 0u && variant != Variant::DivergentBound) count = fixture.Emit(ValueOpcode::LoadAddressU32, {fixture.Address(fixture.UserData(0), fixture.UserData(1)), Value(0u), Value(0u), Value(true)}, fixture.AddMemory({.kind = ResourceKind::ScalarAddress}, 0x10)); - const auto in_range = fixture.Emit(material ? ValueOpcode::ULessThan32 : ValueOpcode::SLessThan32, + const auto in_range = post_test ? Value{} : fixture.Emit(material ? ValueOpcode::ULessThan32 : ValueOpcode::SLessThan32, {variant == Variant::WrongGuard ? Value(0u) : key, count}, 0, compare); const auto initial_active = fixture.Emit(ValueOpcode::INotEqual32, {local, Value(0u)}, 0, entry); - if (forwarded) { + if (forwarded && !post_test) { forward->AddBranch(incoming == forward ? header : incoming); forward->terminator = {.kind = CFG::TerminatorKind::Branch, .true_block = incoming == forward ? header : incoming}; @@ -1260,7 +1265,7 @@ void TestBoundedComputeImageLoop() { .kind = CFG::TerminatorKind::Branch, .true_block = fixture.program.blocks[1u]}; fixture.program.blocks[4]->terminator.kind = CFG::TerminatorKind::Return; - } else { + } else if (!post_test) { const auto active = initial_active; const auto allowed = fixture.Emit(variant == Variant::Disjunction ? ValueOpcode::LogicalOr : ValueOpcode::LogicalAnd, @@ -1289,6 +1294,35 @@ void TestBoundedComputeImageLoop() { const auto step = fixture.Emit(ValueOpcode::IAdd32, {key, Value(variant == Variant::WrongStep ? 2u : 1u)}, 0, increment); + if (post_test) { + const auto branch = [](Block *from, Block *to) { + from->AddBranch(to); + from->terminator = {.kind = CFG::TerminatorKind::Branch, .true_block = to}; + }; + const auto choose = [](Block *from, Block *yes, Block *no, Value condition) { + from->AddBranch(yes); + from->AddBranch(no); + from->condition = condition; + from->terminator = {.kind = CFG::TerminatorKind::ConditionalBranch, + .true_block = yes, .false_block = no}; + }; + entry->terminator = {.kind = CFG::TerminatorKind::Branch, .true_block = header}; + branch(header, body); + branch(body, latch); + auto *test_block = detour != nullptr ? detour : latch; + if (detour != nullptr) choose(latch, detour, forward, initial_active); + const auto compared = post == PostTest::Previous ? key : step; + const auto predicate = fixture.Emit(post == PostTest::Unsigned ? ValueOpcode::ULessThan32 : + ValueOpcode::SLessThan32, {compared, count}, 0, test_block); + const auto condition = fixture.Emit(ValueOpcode::ConditionRef, {predicate}, + post == PostTest::AnyLane ? CFG::BranchCondition::ExecNonZero : + CFG::BranchCondition::SccNonZero, test_block); + choose(test_block, post == PostTest::WrongPolarity ? exit : forward, + post == PostTest::WrongPolarity ? forward : exit, condition); + branch(forward, incoming == forward ? header : incoming); + if (incoming != forward) branch(incoming, header); + exit->terminator.kind = CFG::TerminatorKind::Return; + } phi.AddPhiOperand(entry, variant == Variant::DivergentKey ? local : Value(first)); phi.AddPhiOperand(forwarded ? incoming : increment, step); Check(!forwarded || step.ResolveInstruction()->Parent() != phi.PhiBlock(1), @@ -1332,7 +1366,7 @@ void TestBoundedComputeImageLoop() { fixture.Emit(ValueOpcode::ImageSampleRaw, {image, sampler, fixture.ImageAddress(), Value(true)}, fixture.AddMemory(sample, 0x29c)); - if (first != 0u) { + if (first != 0u && !post_test) { const auto output = fixture.Image({Value(0x8000u), Value(static_cast(Libs::Graphics::Prospero::BufferFormat::k32UInt) << 20u), Value(0u), Value((static_cast(Libs::Graphics::Prospero::ImageType::kColor2D) << 28u) | @@ -1348,7 +1382,7 @@ void TestBoundedComputeImageLoop() { const auto &indirect = fixture.program.descriptor_sources[source].indirect_descriptor; Check(indirect && indirect->selector.has_value() == material && indirect->table_offset == 0x6b0u && - (first == 0u ? indirect->key_count.Resolve() == count.Resolve() + (post_test || first == 0u ? indirect->key_count.Resolve() == count.Resolve() : indirect->selector_first == Value(first) && !indirect->signed_count), "bounded compute loop lost its runtime image count"); if (first != 0u) @@ -1401,10 +1435,29 @@ void TestBoundedComputeImageLoop() { CheckFatal([&] { make_plan(variant); }, "not a valid runtime value", "compute image loop accepted a guard without a uniform bound"); } + const auto post_test_plan = [&](PostTest post = PostTest::Signed, uint32_t bound = 6u, + uint32_t first = 0u, Variant variant = Variant::Forwarded) { + return make_plan(variant, false, first, Clamp::None, post, bound); + }; + post_test_plan(PostTest::Signed, 6u, 0u, Variant::ForwardedChain); + for (const auto post : {PostTest::WrongPolarity, PostTest::Previous, PostTest::Bypass, + PostTest::Variable, PostTest::AnyLane}) + CheckFatal([&] { post_test_plan(post); }, "not a valid runtime value", + "post-tested image loop accepted an unproved carried-edge bound"); + for (const auto bound : {0u, 0x80000000u, UINT32_MAX}) + CheckFatal([&] { post_test_plan(PostTest::Signed, bound); }, "not a valid runtime value", + "post-tested image loop accepted a nonpositive signed bound"); + CheckFatal([&] { post_test_plan(PostTest::Unsigned, 0u); }, "not a valid runtime value", + "post-tested image loop accepted a zero unsigned bound"); + for (const auto first : {1u, UINT32_MAX}) + CheckFatal([&] { post_test_plan(PostTest::Unsigned, 6u, first); }, "not a valid runtime value", + "post-tested image loop accepted a nonzero or wrapping initial value"); + CheckFatal([&] { post_test_plan(PostTest::Signed, 6u, 0u, Variant::WrongStep); }, + "not a valid runtime value", "post-tested image loop accepted a two-step induction"); LinearTestMemory memory; const auto table = 0x1800u + 0x6b0u; - for (uint32_t key = 0; key < 3u; ++key) { + for (uint32_t key = 0; key < 6u; ++key) { const auto word = (table - memory.base) / 4u + key * 8u; memory.words[word] = 0x100u + key; memory.words[word + 1u] = static_cast( @@ -1419,6 +1472,19 @@ void TestBoundedComputeImageLoop() { .read_specialization_memory = ReadLinearTestMemory}; ResourceSnapshot snapshot; ResourceSpecialization specialization; + for (const auto post : {PostTest::Signed, PostTest::Unsigned}) { + for (const auto count : {6u, 1u}) { + const auto post_plan = post_test_plan(post, count); + memory.descriptor_reads = 0u; + memory.fail_address = table + count * 32u; + Check(MaterializeResources(post_plan, runtime, snapshot, specialization) && + memory.descriptor_reads == count && snapshot.images.size() == count && + snapshot.images.front().dwords[0] == 0x100u && + snapshot.images.back().dwords[0] == 0x100u + count - 1u, + "post-tested image loop did not enumerate precisely zero through bound minus one"); + } + } + memory.fail_address = UINT64_MAX; for (const uint32_t count : {2u, 3u, 2u}) { user_data[2] = count; Check(MaterializeResources(plan, runtime, snapshot, specialization) && diff --git a/tests/ShaderRecompilerComputeTests.cpp b/tests/ShaderRecompilerComputeTests.cpp index 086d2196e..55036be37 100644 --- a/tests/ShaderRecompilerComputeTests.cpp +++ b/tests/ShaderRecompilerComputeTests.cpp @@ -37372,12 +37372,13 @@ void CheckCountedRowImageSelector(VulkanHarness &vulkan) { void CheckPointerImageTable(VulkanHarness &vulkan) { using O = ShaderOpcode; constexpr u32 base = 256u, table_offset = 24364u; - for (const u32 variant : {0u, 1u, 2u}) + for (const u32 variant : {0u, 1u, 2u, 3u, 4u}) for (const u32 wave : {32u, 64u}) { - const bool packed = variant == 2u; - const u32 count = variant * 2u; + const bool packed = variant == 2u, post_tested = variant >= 3u; + const u32 count = post_tested ? (variant == 3u ? 1u : 6u) : variant * 2u; TestCase test; - test.name = packed ? "PackedPointerImageTable" : "CountedPointerImageTable"; + test.name = post_tested ? "PostTestedPointerImageTable" + : packed ? "PackedPointerImageTable" : "CountedPointerImageTable"; test.dispatch_x = packed ? 4u : 1u; test.bda_mappings = {{0u, 0u}}; test.initial.resize((base + table_offset + 32u * 32u) / 4u); @@ -37407,10 +37408,15 @@ void CheckPointerImageTable(VulkanHarness &vulkan) { test.compute_info.workgroup_register = 44u; auto &code = test.code; AppendVMovLiteral(&code, 0, 0u); - AppendVMovLiteral(&code, 20, 0x3f000000u); - AppendVMovLiteral(&code, 21, 0x3f000000u); + AppendVMovLiteral(&code, 20, post_tested ? 0u : 0x3f000000u); + AppendVMovLiteral(&code, 21, post_tested ? 0u : 0x3f000000u); size_t loop = 0, exit = 0, skip = 0; - if (packed) { + if (post_tested) { + // Captured 5f3fdf61a7ca4a20: load row before incrementing and testing next < 6. + code.push_back(EncodeSMovB32(20, InlineU32(0))); + loop = code.size(); + code.push_back(EncodeSop2(0x1e, 23, 20, InlineU32(5))); + } else if (packed) { code.insert(code.end(), {EncodeSop2(0x26, 23, 44, InlineU32(60)), EncodeSop2(0x02, 23, 23, 255), 5008u, EncodeSmem0(0x00, 24), EncodeSmem1(0, 23), @@ -37432,9 +37438,13 @@ void CheckPointerImageTable(VulkanHarness &vulkan) { } code.insert(code.end(), {EncodeSop2(0x02, 23, 23, 255), table_offset, EncodeSmem0(0x03, 8), EncodeSmem1(0, 23), - EncodeMimg0(0x27, 1), EncodeMimg1(4, 20, 2, 10), + EncodeMimg0(post_tested ? 0x00 : 0x27, 1), EncodeMimg1(4, 20, 2, 10), EncodeVop2(0x03, 0, Vgpr(4), 0)}); - if (!packed) { + if (post_tested) { + code.insert(code.end(), {EncodeSop2(0x02, 20, 20, InlineU32(1)), + EncodeSopc(0x04, 20, InlineU32(count))}); + code.push_back(EncodeSopp(0x05, loop - code.size() - 1u)); + } else if (!packed) { code[skip] = EncodeSopp(0x04, code.size() - skip - 1u); code.push_back(EncodeSop2(0x02, 20, 20, InlineU32(1))); code.push_back(EncodeSopp(0x02, loop - code.size() - 1u)); @@ -37444,10 +37454,13 @@ void CheckPointerImageTable(VulkanHarness &vulkan) { code.push_back(EncodeVop1(0x01, 30, 25)); AppendBufferStoreDword(&code, 0, 30); AppendEnd(&code); - test.opcodes = {O::S_LOAD_DWORD, O::S_MUL_I32, O::S_ADD_I32, O::S_LSHL_B32, - O::S_LOAD_DWORDX8, O::IMAGE_SAMPLE, O::V_ADD_F32, O::V_MOV_B32, + test.opcodes = {O::S_ADD_I32, O::S_LSHL_B32, + O::S_LOAD_DWORDX8, post_tested ? O::IMAGE_LOAD : O::IMAGE_SAMPLE, O::V_ADD_F32, O::V_MOV_B32, O::BUFFER_STORE_DWORD, O::S_ENDPGM}; - if (packed) test.opcodes.insert(test.opcodes.end(), {O::S_LSHR_B32, O::S_AND_B32}); + if (!post_tested) test.opcodes.insert(test.opcodes.end(), {O::S_LOAD_DWORD, O::S_MUL_I32}); + if (post_tested) test.opcodes.insert(test.opcodes.end(), {O::S_MOV_B32, O::S_CMP_LT_I32, + O::S_CBRANCH_SCC1}); + else if (packed) test.opcodes.insert(test.opcodes.end(), {O::S_LSHR_B32, O::S_AND_B32}); else test.opcodes.insert(test.opcodes.end(), {O::S_MOV_B32, O::S_CMP_LT_U32, O::S_CMP_GE_I32, O::S_CBRANCH_SCC0, O::S_BRANCH}); const auto compiled = CompileCase(test, vulkan.SubgroupSize()); @@ -37462,15 +37475,15 @@ void CheckPointerImageTable(VulkanHarness &vulkan) { {std::bit_cast(static_cast(descriptor.dwords[0]))}, 1, vk::ImageLayout::eShaderReadOnlyOptimal)); auto buffer = vulkan.CreateStorageBuffer(test.name, test.initial, test.initial.size(), true); - auto sampler = vulkan.CreateSampler(test.name, 1); + auto sampler = post_tested ? vk::Sampler{} : vulkan.CreateSampler(test.name, 1); vulkan.Dispatch(test, compiled, buffer, nullptr, nullptr, nullptr, nullptr, sampler, images); - std::vector expected(test.dispatch_x, std::bit_cast(count == 0u ? 0.0f : 34.0f)); + std::vector expected(test.dispatch_x, std::bit_cast(post_tested ? float(count * (count + 1u) / 2u) : count == 0u ? 0.0f : 34.0f)); if (packed) for (u32 row = 0; row < fields.size(); ++row) expected[row] = std::bit_cast(static_cast(fields[row] + 1u)); CompareWords(test, "bounded pointer selectors and packed fields", expected, vulkan.ReadBuffer(test.name, buffer, expected.size())); vulkan.DestroyBuffer(&buffer); - vulkan.Device().destroySampler(sampler); + if (sampler) vulkan.Device().destroySampler(sampler); for (auto &image : images) vulkan.DestroyImage(&image); std::printf("[compute] %s wave=%u count=%u ok\n", test.name, wave, count); }