diff --git a/src/graphics/shader/recompiler/ir/ShaderIR.h b/src/graphics/shader/recompiler/ir/ShaderIR.h index 5e5dc8eea..243e24f39 100644 --- a/src/graphics/shader/recompiler/ir/ShaderIR.h +++ b/src/graphics/shader/recompiler/ir/ShaderIR.h @@ -28,6 +28,8 @@ class CodeGenerator; namespace Libs::Graphics::ShaderRecompiler::IR { +class SrtWalker; + // Finite descriptor domains also bound the arithmetic used to prove their indices. constexpr uint32_t MaxIndirectDescriptorProbes = 65536u; @@ -596,6 +598,9 @@ struct ResourcePlan { std::vector values; uint64_t generation = 0; + const SrtWalker* session = nullptr; + Value active_mask; + uint8_t read_policy = 0; }; ResourcePlan(); @@ -627,7 +632,6 @@ struct ResourcePlan { // GPU-thread scratch for nested clean/EXEC memos, activity and material keys. mutable std::deque evaluation_contexts; mutable uint32_t evaluation_value_count = 0; - mutable uint32_t evaluation_depth = 0; mutable std::vector active_sources; mutable std::vector visited_blocks; mutable std::vector reachable_blocks; diff --git a/src/graphics/shader/recompiler/ir/passes/SrtWalker.cpp b/src/graphics/shader/recompiler/ir/passes/SrtWalker.cpp index 09f29fcbb..4e9a878ee 100644 --- a/src/graphics/shader/recompiler/ir/passes/SrtWalker.cpp +++ b/src/graphics/shader/recompiler/ir/passes/SrtWalker.cpp @@ -499,12 +499,24 @@ void CompileSrtWalker(ResourcePlan& program) { } SrtWalker::SrtWalker(const ResourcePlan& program, const SrtRuntime& runtime, - SrtWalker* clean_evaluator, Value active_mask) + SrtWalker* clean_evaluator) : m_program(program), m_runtime(runtime), m_clean_evaluator(clean_evaluator), - m_active_mask(active_mask.Resolve()), - m_context(AcquireContext(program)) {} + m_session(this), + m_read_policy(clean_evaluator != nullptr ? 2u : 0u), m_context(AcquireContext()) {} -SrtWalker::~SrtWalker() { --m_program.evaluation_depth; } +SrtWalker::SrtWalker(SrtWalker& parent, Value active_mask, bool clean, + SrtWalker* clean_evaluator) + : m_program(parent.m_program), m_runtime(clean ? CleanRuntime(parent.m_runtime) : parent.m_runtime), + m_clean_evaluator(clean_evaluator), m_active_mask(active_mask.Resolve()), + m_reachable_blocks(parent.m_reachable_blocks), m_session(parent.m_session), + m_read_policy((parent.m_read_policy & 1u) | (clean ? 1u : 0u) | + (clean_evaluator != nullptr ? 2u : 0u)), m_context(AcquireContext()) {} + +SrtWalker::~SrtWalker() { + if (m_session != this) return; + for (auto& context: m_program.evaluation_contexts) + if (context.session == this) context.session = nullptr; +} bool SrtWalker::Evaluate(Value value, uint32_t& result) { uint64_t wide = 0; @@ -515,15 +527,29 @@ bool SrtWalker::Evaluate(Value value, uint32_t& result) { return true; } -ResourcePlan::EvaluationContext& SrtWalker::AcquireContext(const ResourcePlan& program) { - if (program.evaluation_depth == program.evaluation_contexts.size()) { - program.evaluation_contexts.emplace_back(); +ResourcePlan::EvaluationContext& SrtWalker::AcquireContext() { + ResourcePlan::EvaluationContext* unused = nullptr; + for (auto& context: m_program.evaluation_contexts) { + if (context.session == m_session && context.active_mask == m_active_mask && + context.read_policy == m_read_policy) return context; + if (context.session == nullptr && unused == nullptr) unused = &context; } - auto& context = program.evaluation_contexts[program.evaluation_depth++]; - context.generation += 2; + auto& context = unused != nullptr ? *unused : m_program.evaluation_contexts.emplace_back(); + context.session = m_session; + context.active_mask = m_active_mask; + context.read_policy = m_read_policy; + context.generation += 4; return context; } +void SrtWalker::ResetUnknowns() { + for (auto& context: m_program.evaluation_contexts) { + if (context.session != m_session) continue; + for (auto& entry: context.values) + if (entry.generation == (context.generation | 2u)) entry.generation = 0; + } +} + float SrtWalker::Float32(uint64_t bits) { return std::bit_cast(static_cast(bits)); } @@ -561,8 +587,10 @@ bool SrtWalker::EvaluateWide(Value value, uint64_t& result) { result = m_context.values[index].value; return true; } - // The low generation bit marks an instruction that is still being evaluated. - if (m_context.values[index].generation == (m_context.generation | 1u)) { + // Visiting and completed-unknown entries both stop repeated evaluation in this + // context. Only completed unknowns are retried when reachability becomes tighter. + if (m_context.values[index].generation == (m_context.generation | 1u) || + m_context.values[index].generation == (m_context.generation | 2u)) { return false; } m_context.values[index].generation = m_context.generation | 1u; @@ -571,7 +599,7 @@ bool SrtWalker::EvaluateWide(Value value, uint64_t& result) { // Recursive evaluation may grow the dense memo vector. auto& memo = m_context.values[index]; if (!evaluated) { - memo.generation = 0; + memo.generation = m_context.generation | 2u; return false; } memo.value = out; @@ -744,9 +772,8 @@ bool SrtWalker::EvaluateInst(const Inst& inst, uint64_t& result) { if (!m_active_mask.IsEmpty() && inst.Arg(1).Resolve() == m_active_mask) { return EvaluateWide(inst.Arg(0), result); } - const auto clean_runtime = CleanRuntime(m_runtime); - SrtWalker clean_active(m_program, clean_runtime, nullptr, inst.Arg(1)); - SrtWalker active(m_program, m_runtime, &clean_active, inst.Arg(1)); + SrtWalker clean_active(*this, inst.Arg(1), true); + SrtWalker active(*this, inst.Arg(1), false, &clean_active); return active.EvaluateWide(inst.Arg(0), result); } case ValueOpcode::BitCastU32F32: @@ -1070,9 +1097,8 @@ bool SrtWalker::EvaluateInst(const Inst& inst, uint64_t& result) { if (!left) { // Only lanes satisfying the left EXEC factor can make the conjunction // true. Use a separate memo so this assumption cannot escape its RHS. - SrtWalker active(m_program, m_clean_evaluator != nullptr - ? CleanRuntime(m_runtime) : m_runtime, nullptr, inst.Arg(0).Resolve()); - active.m_reachable_blocks = m_reachable_blocks; + SrtWalker active(m_clean_evaluator != nullptr ? *m_clean_evaluator : *this, + inst.Arg(0)); if (active.EvaluateWide(inst.Arg(1), b) && b == 0u) { result = 0u; return true; @@ -1161,8 +1187,12 @@ bool SrtWalker::RefreshFlatBuffer(std::vector& flat) { auto& reachable = m_program.reachable_blocks; auto& pending = m_program.pending_blocks; visited.assign(m_program.control_flow.size(), 1u); + ResetUnknowns(); m_reachable_blocks = &visited; - if (m_clean_evaluator != nullptr) m_clean_evaluator->m_reachable_blocks = &visited; + if (m_clean_evaluator != nullptr) { + m_clean_evaluator->ResetUnknowns(); + m_clean_evaluator->m_reachable_blocks = &visited; + } for (;;) { reachable.assign(m_program.control_flow.size(), 0u); pending.clear(); @@ -1188,6 +1218,8 @@ bool SrtWalker::RefreshFlatBuffer(std::vector& flat) { if (m_program.control_phi_predecessors.empty()) break; // Only unknown values need another evaluation: successful Phi facts remain // true as their possible predecessors shrink. Keep uniform-read memos stable. + ResetUnknowns(); + if (m_clean_evaluator != nullptr) m_clean_evaluator->ResetUnknowns(); } flat.assign(m_program.srt_reads.size(), 0u); active.assign(m_program.descriptor_sources.size(), 1u); diff --git a/src/graphics/shader/recompiler/ir/passes/SrtWalker.h b/src/graphics/shader/recompiler/ir/passes/SrtWalker.h index 2bbc8f987..9b9192625 100644 --- a/src/graphics/shader/recompiler/ir/passes/SrtWalker.h +++ b/src/graphics/shader/recompiler/ir/passes/SrtWalker.h @@ -99,7 +99,7 @@ std::optional> AffineIndex( class SrtWalker { public: SrtWalker(const ResourcePlan& program, const SrtRuntime& runtime, - SrtWalker* clean_evaluator = nullptr, Value active_mask = {}); + SrtWalker* clean_evaluator = nullptr); ~SrtWalker(); SrtWalker(const SrtWalker&) = delete; SrtWalker& operator=(const SrtWalker&) = delete; @@ -110,7 +110,10 @@ public: bool RefreshFlatBuffer(std::vector& flat); private: - static ResourcePlan::EvaluationContext& AcquireContext(const ResourcePlan& program); + SrtWalker(SrtWalker& parent, Value active_mask, bool clean = false, + SrtWalker* clean_evaluator = nullptr); + ResourcePlan::EvaluationContext& AcquireContext(); + void ResetUnknowns(); static float Float32(uint64_t bits); bool EvaluateWide(Value value, uint64_t& result); bool Arg(const Inst& inst, size_t index, uint64_t& result); @@ -124,6 +127,8 @@ private: SrtWalker* m_clean_evaluator = nullptr; Value m_active_mask; const std::vector* m_reachable_blocks = nullptr; + const SrtWalker* m_session; + uint8_t m_read_policy; ResourcePlan::EvaluationContext& m_context; }; diff --git a/tests/ResourceMaterializationTests.cpp b/tests/ResourceMaterializationTests.cpp index d38a339b8..bab545b90 100644 --- a/tests/ResourceMaterializationTests.cpp +++ b/tests/ResourceMaterializationTests.cpp @@ -616,11 +616,11 @@ void TestWrittenDescriptorPredicateReads(bool memory_condition) { ResourceSpecialization specialization; for (const bool clean : {false, true}) { reads = {.clean = clean}; - // A failed predicate explores both edges and the writable descriptor retries its clean read. + // A failed predicate explores both edges; its dirty read also rejects the writable descriptor. const bool expected = !memory_condition || clean; Check(MaterializeResources(plan, runtime, snapshot, specialization) == expected && reads.ordinary == (memory_condition ? 0u : 1u) && - reads.strict == (memory_condition ? (clean ? 1u : 2u) : 0u), + reads.strict == (memory_condition ? 1u : 0u), "writable descriptor changed reader policy or bypassed a dirty memory predicate"); if (!expected) continue; Check(snapshot.buffers[0].dwords[0] == 0x8000u && @@ -698,6 +698,87 @@ void TestSharedControlPredicates() { } } +void TestNestedExecPredicatesReuseReads() { + using namespace Libs::Graphics::ShaderRecompiler::IR; + using Libs::Graphics::ShaderRecompiler::CFG::TerminatorKind; + constexpr uint32_t depth = 12; + Program program; + program.stage = Libs::Graphics::ShaderType::Compute; + program.srt_plan_complete = program.resource_tracking_complete = true; + for (uint32_t i = 0; i < 3; ++i) AddValueBlock(program); + auto &entry = *program.blocks[0]; + auto &use = *program.blocks[1]; + auto &exit = *program.blocks[2]; + const auto emit = [&](ValueOpcode opcode, std::initializer_list args) { + return Value(&entry.AppendNewInst(opcode, args)); + }; + const auto lane = emit(ValueOpcode::GetBuiltin, + {Value(static_cast(StageInputKind::LocalInvocationId)), Value(0u)}); + program.memory_info.push_back({.kind = ResourceKind::ScalarAddress, .planning_only = true}); + const auto address = emit(ValueOpcode::GetAddressResource, {Value(0x1000u), Value(0u)}); + const auto count = emit(ValueOpcode::LoadAddressU32, + {address, Value(0u), Value(0u), Value(true)}); + count.Instruction()->SetFlags(MemoryFlags{.index = 0}); + auto condition = emit(ValueOpcode::LogicalNot, + {emit(ValueOpcode::ULessThanEqual32, {count, lane})}); + // A compact shared DAG must not become a tree of fresh EXEC evaluations. + for (uint32_t i = 0; i < depth; ++i) { + const auto mask = emit(ValueOpcode::INotEqual32, {lane, Value(i)}); + condition = emit(ValueOpcode::LogicalAnd, + {mask, emit(ValueOpcode::LogicalOr, {condition, condition})}); + } + entry.condition = condition; + entry.terminator.kind = TerminatorKind::ConditionalBranch; + entry.terminator.true_block = &use; + entry.terminator.false_block = &exit; + entry.AddBranch(&use); + entry.AddBranch(&exit); + use.terminator.kind = TerminatorKind::Branch; + use.terminator.true_block = &exit; + use.AddBranch(&exit); + DescriptorSource source; + source.dwords = {Value(0x8000u), Value(1u << 16u), Value(16u), Value(3u << 28u)}; + source.dword_count = 4; + program.descriptor_sources.push_back(source); + program.info.buffers.push_back({.source = 0, .read = true}); + program.memory_info.push_back({.kind = ResourceKind::Buffer, .resource = 0}); + const auto buffer = Value(&use.AppendNewInst(ValueOpcode::GetBufferResource, + {source.dwords[0], source.dwords[1], source.dwords[2], source.dwords[3]})); + use.AppendNewInst(ValueOpcode::LoadBufferU32, + {buffer, Value(0u), Value(0u), Value(0u), Value(true)}) + .SetFlags(MemoryFlags{.index = 1}); + auto plan = ExtractResourcePlan(program); + struct Reads { uint32_t value = 0; bool clean = true; uint32_t strict = 0, ordinary = 0; } reads; + const SrtRuntime runtime{ + .read_memory = +[](void *data, uint64_t, std::span) { + ++static_cast(data)->ordinary; + return false; + }, + .userdata = &reads, + .read_specialization_memory = +[](void *data, uint64_t address, std::span words) { + auto &reads = *static_cast(data); + ++reads.strict; + Check(reads.strict <= depth + 1, + "shared EXEC predicates repeated scalar reads beyond their distinct masks"); + if (!reads.clean || address != 0x1000u || words.size() != 1) return false; + words[0] = reads.value; + return true; + }}; + ResourceSnapshot snapshot; + ResourceSpecialization specialization; + for (const auto state : {Reads{64u, true}, Reads{0u, true}, Reads{0u, false}, Reads{64u, true}}) { + reads = state; + Check(MaterializeResources(plan, runtime, snapshot, specialization), + "nested EXEC predicates rejected a valid resource snapshot"); + const bool inactive = reads.clean && reads.value == 0; + Check(snapshot.buffers[0].dwords[0] == (inactive ? 0u : 0x8000u), + "nested EXEC predicates lost an unknown branch or retained stale runtime facts"); + Check(reads.strict > 0 && reads.ordinary == 0 && + snapshot.specialization_reads.size() <= depth + 1, + "shared EXEC predicates repeated scalar reads beyond their distinct masks"); + } +} + void TestFiniteImageRefreshReusesScalarReads() { using namespace Libs::Graphics::ShaderRecompiler::IR; Program program; @@ -1059,6 +1140,7 @@ int main() { TestWrittenDescriptorPredicateReads(memory_condition); TestFailedMaterializationRejectsStage(); TestSharedControlPredicates(); + TestNestedExecPredicatesReuseReads(); TestFiniteImageRefreshReusesScalarReads(); for (const bool independent_mask : {false, true}) TestEmptyTraversalSkipsUnmappedDescriptor(independent_mask);