diff --git a/src/graphics/shader/recompiler/ShaderRecompiler.cpp b/src/graphics/shader/recompiler/ShaderRecompiler.cpp index 3d54df0f5..4530ee01c 100644 --- a/src/graphics/shader/recompiler/ShaderRecompiler.cpp +++ b/src/graphics/shader/recompiler/ShaderRecompiler.cpp @@ -584,10 +584,13 @@ void PrepareCallTargets(ShaderSource& source, const CompileOptions& options) { for (auto& inst: block) { inst.SetParent(nullptr); if (inst.GetOpcode() == ValueOpcode::LoadAddressU32 || inst.GetOpcode() == ValueOpcode::ReadConstBuffer) - inst.SetFlags(SrtReadFlags {.index = inst.Flags().index}); + inst.SetFlags(MemoryFlags {.index = inst.Flags().index}); } query.value_storage.splice(query.value_storage.end(), block.Instructions()); source.call_targets = std::move(static_cast(query)); + std::vector targets; + for (const auto& site: source.calls) targets.insert(targets.end(), site.target.begin(), site.target.end()); + CompileSrtWalker(source.call_targets, targets); } } // namespace @@ -617,12 +620,18 @@ const Decoder::Program& RefreshShaderSource(ShaderSource& source, const IR::SrtR EXIT_IF(source.unsupported_call_pc != UINT32_MAX); auto& reads = source.reads; reads.clear(); - IR::SrtReadCapture capture(runtime, reads); - const auto clean = IR::CleanRuntime(capture.ObservedRuntime()); - IR::SrtWalker walker(source.call_targets, clean); + std::vector flat; + const bool walked = IR::WalkSrt(source.call_targets, runtime, flat, &reads); + const auto read_code = [&](uint64_t address, std::span words) { + if (runtime.read_specialization_memory == nullptr || + !runtime.read_specialization_memory(runtime.userdata, address, words)) return false; + reads.emplace_back(address, words.size_bytes()); + return true; + }; for (auto& site: source.calls) { uint32_t low = 0, high = 0; - if (!walker.Evaluate(site.target[0], low) || !walker.Evaluate(site.target[1], high)) + if (!walked || !IR::WalkedValue(source.call_targets, site.target[0], low) || + !IR::WalkedValue(source.call_targets, site.target[1], high)) EXIT("shader call at pc 0x%08x has an unavailable scalar target", source.decoded.instructions[site.instruction].pc); // SWAPPC reads the old aliased pair and ignores the target's low two bits. @@ -646,7 +655,7 @@ const Decoder::Program& RefreshShaderSource(ShaderSource& source, const IR::SrtR break; } linked.observed_function.resize(function.size); - if (!clean.read_memory(clean.userdata, source.calls[i].address, linked.observed_function) || + if (!read_code(source.calls[i].address, linked.observed_function) || !std::ranges::equal(expected, linked.observed_function)) { unchanged = false; break; @@ -676,8 +685,7 @@ const Decoder::Program& RefreshShaderSource(ShaderSource& source, const IR::SrtR uint32_t count = 1; Decoder::Instruction inst; for (;;) { - if (!clean.read_memory(clean.userdata, address + uint64_t {index} * sizeof(uint32_t), - std::span(words).first(count))) + if (!read_code(address + uint64_t {index} * sizeof(uint32_t), std::span(words).first(count))) EXIT("shader call at pc 0x%08x cannot read leaf code", call.pc); inst = {}; Decoder::DecodeInstruction(words, 0u, inst); diff --git a/src/graphics/shader/recompiler/ir/ShaderIR.h b/src/graphics/shader/recompiler/ir/ShaderIR.h index 353f7f10e..163c653b3 100644 --- a/src/graphics/shader/recompiler/ir/ShaderIR.h +++ b/src/graphics/shader/recompiler/ir/ShaderIR.h @@ -14,7 +14,6 @@ #include #include -#include #include #include #include @@ -28,8 +27,6 @@ class CodeGenerator; namespace Libs::Graphics::ShaderRecompiler::IR { -class SrtWalker; - // Finite descriptor domains also bound the arithmetic used to prove their indices. constexpr uint32_t MaxIndirectDescriptorProbes = 65536u; @@ -663,19 +660,6 @@ struct ResourcePlan { bool resolved = false; }; - struct EvaluationContext { - struct Entry { - uint64_t value = 0; - uint64_t generation = 0; - }; - - std::vector values; - uint64_t generation = 0; - const SrtWalker* session = nullptr; - Value active_mask; - uint8_t read_policy = 0; - }; - ResourcePlan(); ~ResourcePlan(); @@ -695,21 +679,18 @@ struct ResourcePlan { std::vector control_flow; std::unordered_map> control_phi_predecessors; std::vector srt_reads; - bool requires_specialization_memory = false; std::vector> source_reads; bool capture_specialization_reads = false; bool srt_plan_complete = false; bool resource_tracking_complete = false; ShaderInfo info; UniformFillPlan uniform_fill; + // x86 walker over the retained SRT, descriptor and condition values. std::unique_ptr srt_code; - // GPU-thread scratch for nested clean/EXEC memos, activity and material keys. - mutable std::deque evaluation_contexts; - mutable uint32_t evaluation_value_count = 0; - mutable std::vector active_sources; - mutable std::vector visited_blocks; - mutable std::vector reachable_blocks; - mutable std::vector pending_blocks; + uint32_t srt_user_data = 0; // User-data DWORDs the walker reads. + uint32_t srt_captures = 0; // Scratch offset of loaded addresses. + // GPU-thread scratch for walker results and material keys. + mutable std::vector srt_scratch; mutable std::vector material_keys; mutable std::vector selector_keys; }; diff --git a/src/graphics/shader/recompiler/ir/Value.h b/src/graphics/shader/recompiler/ir/Value.h index 72583eef8..8779ba459 100644 --- a/src/graphics/shader/recompiler/ir/Value.h +++ b/src/graphics/shader/recompiler/ir/Value.h @@ -123,13 +123,9 @@ public: [[nodiscard]] Block* PhiBlock(size_t index) const; [[nodiscard]] Block* Parent() const; [[nodiscard]] const std::vector& Uses() const; - // Runtime indices belong to the resource plan that owns this instruction. - [[nodiscard]] uint32_t EvaluationIndex(uint32_t& count) const { - if (evaluation_index == UINT32_MAX) { - evaluation_index = count++; - } - return evaluation_index; - } + // SRT walker output index, assigned by the resource plan that owns this instruction. + [[nodiscard]] uint32_t WalkOutput() const { return walk_output; } + void SetWalkOutput(uint32_t index) const { walk_output = index; } // Backend definitions belong to one emission. Wave64 emulation needs one per half. [[nodiscard]] uint32_t Definition(uint32_t half) const { return definitions[half]; } void SetDefinition(uint32_t half, uint32_t definition) const { @@ -171,7 +167,7 @@ private: ValueOpcode opcode; uint8_t num_args; bool live = false; - mutable uint32_t evaluation_index = UINT32_MAX; + mutable uint32_t walk_output = UINT32_MAX; mutable std::array definitions {}; uint64_t flags; Block* parent = nullptr; diff --git a/src/graphics/shader/recompiler/ir/passes/ResourceMaterialization.cpp b/src/graphics/shader/recompiler/ir/passes/ResourceMaterialization.cpp index 1e89cf62d..052da8251 100644 --- a/src/graphics/shader/recompiler/ir/passes/ResourceMaterialization.cpp +++ b/src/graphics/shader/recompiler/ir/passes/ResourceMaterialization.cpp @@ -186,35 +186,6 @@ const DescriptorSource* Source(const ResourcePlan& program, uint32_t source) { return &program.descriptor_sources[source]; } -void MarkCleanReads(const std::unordered_set& planned_reads, - const DescriptorSource* source, Value extra = {}) { - if (source == nullptr && extra.IsEmpty()) { - return; - } - std::vector pending; - if (source != nullptr) { - pending.assign(source->dwords.begin(), source->dwords.begin() + source->dword_count); - } - if (!extra.IsEmpty()) pending.push_back(extra); - std::vector visited; - while (!pending.empty()) { - auto* inst = pending.back().Resolve().TryInstruction(); - pending.pop_back(); - if (inst == nullptr || std::ranges::find(visited, inst) != visited.end()) { - continue; - } - visited.push_back(inst); - if (planned_reads.contains(inst)) { - auto flags = inst->Flags(); - flags.clean = 1u; - inst->SetFlags(flags); - } - for (size_t arg = 0; arg < inst->NumArgs(); arg++) { - pending.push_back(inst->Arg(arg)); - } - } -} - bool ReadTable(uint64_t base, uint64_t size, uint64_t offset, const SrtRuntime& runtime, std::span words) { const auto count = std::min(words.size(), offset < size ? (size - offset) / 4u : 0u); @@ -309,22 +280,22 @@ bool CanReadIndexedSelector(const ShaderBufferResource& material, uint32_t offse uint64_t {UINT32_MAX} + 1u; } -bool EvaluateIndirectCount(const DescriptorSource::IndirectDescriptor& indirect, - const SrtRuntime& runtime, SrtWalker& clean, uint32_t& count) { +bool EvaluateIndirectCount(const ResourcePlan& program, const DescriptorSource::IndirectDescriptor& indirect, + const SrtRuntime& runtime, uint32_t& count) { if (indirect.workgroup_axis != UINT32_MAX) { if (runtime.workgroup_counts.size() != 3u || indirect.workgroup_axis >= 3u) return false; count = std::ranges::find(runtime.workgroup_counts, 0u) != runtime.workgroup_counts.end() ? 0u : runtime.workgroup_counts[indirect.workgroup_axis]; return true; } - if (!clean.Evaluate(indirect.key_count, count)) return false; + if (!WalkedValue(program, indirect.key_count, count)) return false; if (indirect.signed_count && std::bit_cast(count) < 0) count = 0; return true; } -bool ResolveSelectorKeys(const DescriptorSource::IndirectDescriptor& indirect, +bool ResolveSelectorKeys(const ResourcePlan& program, const DescriptorSource::IndirectDescriptor& indirect, const DescriptorValue& material_value, const SrtRuntime& runtime, - SrtWalker& clean, std::vector& keys) { + std::vector& keys) { const auto* selector = &*indirect.selector; keys.clear(); const auto read_selector = [&](uint64_t base, uint64_t size, uint64_t offset, @@ -363,7 +334,7 @@ bool ResolveSelectorKeys(const DescriptorSource::IndirectDescriptor& indirect, const auto& rows = *selector->rows; DescriptorValue count_value; ShaderBufferResource counts, material; - if (!clean.EvaluateDescriptor(rows.source, count_value) || + if (!WalkedDescriptor(program, rows.source, count_value) || !DecodeBufferDescriptor(count_value, counts) || !DecodeBufferDescriptor(material_value, material) || selector->stride != 4u || selector->offset != 0u || rows.stride == 0u || (rows.stride & 3u) != 0u || @@ -392,7 +363,7 @@ bool ResolveSelectorKeys(const DescriptorSource::IndirectDescriptor& indirect, if (candidate_end - first > MaxIndirectDescriptorProbes) break; end = candidate_end; } - // Coalesce small gaps, but interpret only selected prefixes. The strict + // Coalesce small gaps, but interpret only selected prefixes. The specialization // reader captures the entire fetched span for writable-alias rejection. batch.resize(end - first); if (!ReadScalarTable(material.Base48(), material.GetSize(), first * 4u, runtime, batch)) @@ -408,8 +379,8 @@ bool ResolveSelectorKeys(const DescriptorSource::IndirectDescriptor& indirect, std::array masks {}; uint32_t first = 0, count = 0; if (!DecodeBufferDescriptor(material_value, material) || - !clean.Evaluate(indirect.selector_first, first) || - !EvaluateIndirectCount(indirect, runtime, clean, count) || count != masks.size() * 32u || + !WalkedValue(program, indirect.selector_first, first) || + !EvaluateIndirectCount(program, indirect, runtime, count) || count != masks.size() * 32u || selector->stride != sizeof(masks) || selector->offset != 0u || material.Type() != 0u || material.Stride() != sizeof(masks) || material.SwizzleEnabled() || material.AddTid() || material.OutOfBounds() != 0u || @@ -462,8 +433,8 @@ bool ResolveSelectorKeys(const DescriptorSource::IndirectDescriptor& indirect, base = material.Base48(); size = material.GetSize(); } - if (!clean.Evaluate(indirect.selector_first, first) || - !EvaluateIndirectCount(indirect, runtime, clean, count) || count > MaxIndirectDescriptorProbes) + if (!WalkedValue(program, indirect.selector_first, first) || + !EvaluateIndirectCount(program, indirect, runtime, count) || count > MaxIndirectDescriptorProbes) return false; // SMEM uses its instruction's byte stride; V# stride only supplies its bounds. const auto step = selector->stride != 0u ? selector->stride : material.Stride(); @@ -483,8 +454,8 @@ bool ResolveSelectorKeys(const DescriptorSource::IndirectDescriptor& indirect, uint32_t mask = 0; uint32_t count = 0; if (material_value.dword_count != 2u || - !clean.Evaluate(indirect.selector_mask, mask) || - !EvaluateIndirectCount(indirect, runtime, clean, count) || count == 0u || count > 32u) { + !WalkedValue(program, indirect.selector_mask, mask) || + !EvaluateIndirectCount(program, indirect, runtime, count) || count == 0u || count > 32u) { return false; } if (count < 32u) mask &= (1u << count) - 1u; @@ -541,8 +512,7 @@ template bool MaterializeIndirectDescriptor(const ResourcePlan& program, uint32_t source_index, uint32_t resource_index, uint32_t dword_count, uint32_t read_dwords, - const SrtRuntime& runtime, SrtWalker& clean, - ResourceSnapshot& snapshot, + const SrtRuntime& runtime, ResourceSnapshot& snapshot, std::vector& descriptors, std::vector& specializations, Normalize&& normalize) { @@ -605,8 +575,8 @@ bool MaterializeIndirectDescriptor(const ResourcePlan& p if (selector == nullptr) keys.clear(); DescriptorValue material_value; if ((selector != nullptr && (byte_keys || !selector_keys->resolved) && - !clean.EvaluateDescriptor(selector->source, material_value)) || - !clean.EvaluateDescriptor(indirect.table_source, table_value)) { + !WalkedDescriptor(program, selector->source, material_value)) || + !WalkedDescriptor(program, indirect.table_source, table_value)) { return false; } if (table_value.dword_count == 2u) { @@ -659,7 +629,7 @@ bool MaterializeIndirectDescriptor(const ResourcePlan& p : uint64_t {1} << 32u; if (!indirect.key_count.IsEmpty()) { uint32_t key_count = 0u; - if (!EvaluateIndirectCount(indirect, runtime, clean, key_count)) return false; + if (!EvaluateIndirectCount(program, indirect, runtime, key_count)) return false; domain = std::min(domain, uint64_t {key_count}); } if (indexed_pointer && @@ -771,8 +741,8 @@ bool MaterializeIndirectDescriptor(const ResourcePlan& p if (selector == nullptr) { uint32_t key_count = 0; uint32_t first_key = 0; - if (!EvaluateIndirectCount(indirect, runtime, clean, key_count)) return false; - if (!indirect.selector_first.IsEmpty() && !clean.Evaluate(indirect.selector_first, first_key)) + if (!EvaluateIndirectCount(program, indirect, runtime, key_count)) return false; + if (!indirect.selector_first.IsEmpty() && !WalkedValue(program, indirect.selector_first, first_key)) return false; if (indirect.table_stride == 0u || key_count > MaxIndirectDescriptorProbes || (indirect.selector_first.IsEmpty() && key_count != 0u && @@ -788,7 +758,7 @@ bool MaterializeIndirectDescriptor(const ResourcePlan& p } else { if (!indirect.selector_mask.IsEmpty() && table_value.dword_count != 2u) return false; if (!selector_keys->resolved) { - if (!ResolveSelectorKeys(indirect, material_value, runtime, clean, keys)) return false; + if (!ResolveSelectorKeys(program, indirect, material_value, runtime, keys)) return false; selector_keys->resolved = true; } } @@ -827,7 +797,7 @@ bool MaterializeIndirectDescriptor(const ResourcePlan& p std::span(candidate.dwords).first(read_dwords))) { return read_failed("table descriptor read", key, 0u, table_base, table_offset, descriptor_bytes); } - } else if (!clean.EvaluateDescriptor(sources[entry], candidate)) { + } else if (!WalkedDescriptor(program, sources[entry], candidate)) { return false; } if (!normalize(candidate)) @@ -1380,16 +1350,13 @@ ResourcePlan ExtractResourcePlan(const Program& program) { if (const auto found = cloned.find(source); found != cloned.end()) { return Value(found->second); } - if (source->GetOpcode() == ValueOpcode::LoadBufferU32) { - plan.requires_specialization_memory = true; - plan.capture_specialization_reads = true; - } + if (source->GetOpcode() == ValueOpcode::LoadBufferU32) plan.capture_specialization_reads = true; auto& target = plan.value_storage.emplace_back(source->GetOpcode(), source->Flags()); if (source->GetOpcode() == ValueOpcode::LoadAddressU32 || source->GetOpcode() == ValueOpcode::ReadConstBuffer || source->GetOpcode() == ValueOpcode::LoadBufferU32) { - target.SetFlags(SrtReadFlags {.index = source->Flags().index}); + target.SetFlags(MemoryFlags {.index = source->Flags().index}); } cloned.emplace(source, &target); if (source->GetOpcode() == ValueOpcode::Phi) { @@ -1411,7 +1378,6 @@ ResourcePlan ExtractResourcePlan(const Program& program) { target.dword_count = source.dword_count; target.indirect_descriptor = source.indirect_descriptor; if (target.indirect_descriptor.has_value()) { - plan.requires_specialization_memory = true; capture_indirect_reads = true; target.indirect_descriptor->key_count = Clone(target.indirect_descriptor->key_count); target.indirect_descriptor->selector_first = @@ -1523,57 +1489,16 @@ ResourcePlan ExtractResourcePlan(const Program& program) { plan.uniform_fill.element_count = Clone(plan.uniform_fill.element_count); plan.capture_specialization_reads |= capture_indirect_reads || capture_control_reads || !plan.source_reads.empty(); - if (plan.capture_specialization_reads) { - // Every former flat alias now targets its retained raw read. Mark those reads - // on the normalized graph, so discarded Phi/EXEC dependencies stay discarded. - std::unordered_set planned_reads; - for (const auto& read: plan.srt_reads) { - auto* inst = read.value.ResolveInstruction(); - EXIT_IF(inst == nullptr || (inst->GetOpcode() != ValueOpcode::LoadAddressU32 && - inst->GetOpcode() != ValueOpcode::ReadConstBuffer)); - planned_reads.insert(inst); - } - for (const auto& source: plan.descriptor_sources) { - if (!source.indirect_descriptor.has_value()) continue; - const auto& indirect = *source.indirect_descriptor; - if (indirect.selector) { - MarkCleanReads(planned_reads, Source(plan, indirect.selector->source)); - if (indirect.selector->rows) - MarkCleanReads(planned_reads, Source(plan, indirect.selector->rows->source)); - } - MarkCleanReads(planned_reads, nullptr, indirect.selector_mask); - MarkCleanReads(planned_reads, nullptr, indirect.selector_first); - MarkCleanReads(planned_reads, nullptr, indirect.key_count); - if (indirect.sources.empty()) { - MarkCleanReads(planned_reads, Source(plan, indirect.table_source)); - } else { - for (const auto candidate: indirect.sources) { - MarkCleanReads(planned_reads, Source(plan, candidate)); - } - } - } - // Clean writable addresses prove that shader writes cannot overlap captured reads. - for (const auto& buffer: plan.info.buffers) { - if (buffer.written) MarkCleanReads(planned_reads, Source(plan, buffer.source)); - } - for (const auto& image: plan.info.images) { - if (image.written) MarkCleanReads(planned_reads, Source(plan, image.source)); - } - } if (capture_indirect_reads || !plan.source_reads.empty()) plan.resource_tracking_complete &= !program.has_address_writes; - CompileSrtWalker(plan); + if (plan.srt_plan_complete && plan.resource_tracking_complete) CompileSrtWalker(plan); return plan; } bool MaterializeResources(const ResourcePlan& program, const SrtRuntime& runtime, ResourceSnapshot& snapshot, ResourceSpecialization& specialization) { for (auto& keys: program.selector_keys) keys.resolved = false; - if (!program.resource_tracking_complete || - ((program.requires_specialization_memory || !program.source_reads.empty()) && - runtime.read_specialization_memory == nullptr)) { - return false; - } + if (!program.resource_tracking_complete) return false; const auto uses_workgroup_count = [](const DescriptorSource& source) { return source.indirect_descriptor && source.indirect_descriptor->workgroup_axis != UINT32_MAX; }; @@ -1582,20 +1507,16 @@ bool MaterializeResources(const ResourcePlan& program, const SrtRuntime& runtime std::ranges::any_of(program.descriptor_sources, uses_workgroup_count)); auto& reads = snapshot.specialization_reads; reads = program.source_reads; - SrtReadCapture capture {runtime, reads}; - auto observed = capture_reads ? capture.ObservedRuntime() : runtime; - SrtWalker clean(program, CleanRuntime(observed)); - SrtWalker walker(program, observed, - capture_reads || program.requires_specialization_memory ? &clean : nullptr); - if (!walker.RefreshFlatBuffer(snapshot.flattened_srt)) { + if (!WalkSrt(program, runtime, snapshot.flattened_srt, capture_reads ? &reads : nullptr)) { return SpecializationFail(fmt::format("SRT refresh complete={} reads={} user_data={}", program.srt_plan_complete, program.srt_reads.size(), runtime.user_data.size())); } - const auto active = std::span(program.active_sources); + SrtReadCapture capture {runtime, reads}; + auto observed = capture_reads ? capture.ObservedRuntime() : runtime; std::array workgroup_counts {}; if (runtime.indirect_args_address != 0u) { for (uint32_t source = 0; source < program.descriptor_sources.size(); ++source) { - if ((!active.empty() && !active[source]) || + if (!WalkedActive(program, source) || !uses_workgroup_count(program.descriptor_sources[source])) continue; if ((runtime.indirect_args_address & 3u) != 0u || !ReadTable(runtime.indirect_args_address, sizeof(workgroup_counts), 0u, @@ -1610,24 +1531,24 @@ bool MaterializeResources(const ResourcePlan& program, const SrtRuntime& runtime std::array stored {}; bool uniform_fill = words != 0; if (uniform_fill && !fill.pattern_size.IsEmpty()) - uniform_fill = clean.Evaluate(fill.pattern_size, words) && words >= 1 && words <= 4; + uniform_fill = WalkedValue(program, fill.pattern_size, words) && words >= 1 && words <= 4; uint32_t elements = 0; if (uniform_fill && !fill.element_count.IsEmpty()) - uniform_fill = clean.Evaluate(fill.element_count, elements) && elements != 0; + uniform_fill = WalkedValue(program, fill.element_count, elements) && elements != 0; for (uint32_t i = 0; i < words && uniform_fill; ++i) { - uniform_fill = clean.Evaluate(fill.values[i], stored[i]) && stored[i] == stored[0]; + uniform_fill = WalkedValue(program, fill.values[i], stored[i]) && stored[i] == stored[0]; } if (uniform_fill) { snapshot.uniform_fill = fill.fill; snapshot.uniform_fill.value = stored[0]; snapshot.uniform_fill.bounded_elements = elements; } - const auto evaluate = [&](uint32_t source, DescriptorValue& value, bool written = false) { + const auto evaluate = [&](uint32_t source, DescriptorValue& value) { if (source >= program.descriptor_sources.size()) { return false; } - if (active.empty() || active[source]) { - return (capture_reads && written ? clean : walker).EvaluateDescriptor(source, value) || + if (WalkedActive(program, source)) { + return WalkedDescriptor(program, source, value) || SpecializationFail(fmt::format("descriptor evaluation source={}", source)); } value = {}; @@ -1651,9 +1572,9 @@ bool MaterializeResources(const ResourcePlan& program, const SrtRuntime& runtime if (source == nullptr) return false; if (source->indirect_descriptor.has_value()) { snapshot.buffers[i] = {.dword_count = 4u}; - if (active.empty() || active[base.source]) { + if (WalkedActive(program, base.source)) { if (!MaterializeIndirectDescriptor( - program, base.source, i, 4u, 4u, observed, clean, snapshot, + program, base.source, i, 4u, 4u, observed, snapshot, snapshot.buffers, specialization.buffers, [&](DescriptorValue& value) { return NormalizeIndirectBuffer( @@ -1661,7 +1582,7 @@ bool MaterializeResources(const ResourcePlan& program, const SrtRuntime& runtime })) return resource_failed("buffer", i, base.source, snapshot.buffers.size()); } - } else if (!evaluate(base.source, snapshot.buffers[i], base.written)) { + } else if (!evaluate(base.source, snapshot.buffers[i])) { return false; } } @@ -1711,13 +1632,13 @@ bool MaterializeResources(const ResourcePlan& program, const SrtRuntime& runtime } if (source->indirect_descriptor.has_value()) { snapshot.images[i] = {.dword_count = 8u}; - if (!active.empty() && !active[image.source]) { + if (!WalkedActive(program, image.source)) { continue; } const auto& indirect = *source->indirect_descriptor; const bool bounded_table = IsBoundedDescriptorTable(program, indirect); if (!MaterializeIndirectDescriptor( - program, image.source, i, 8u, image.r128 ? 4u : 8u, observed, clean, snapshot, snapshot.images, + program, image.source, i, 8u, image.r128 ? 4u : 8u, observed, snapshot, snapshot.images, specialization.images, [&](DescriptorValue& value) { // A broad heap also contains resources for other typed image operations. if (NullImageDescriptor(value) || !ValidImageDescriptor(value, image.r128) || @@ -1728,7 +1649,7 @@ bool MaterializeResources(const ResourcePlan& program, const SrtRuntime& runtime return resource_failed("image", i, image.source, snapshot.images.size()); } } else { - if (!evaluate(image.source, snapshot.images[i], image.written)) { + if (!evaluate(image.source, snapshot.images[i])) { return false; } if (!ValidImageDescriptor(snapshot.images[i], image.r128)) { @@ -1749,8 +1670,8 @@ bool MaterializeResources(const ResourcePlan& program, const SrtRuntime& runtime const auto* indirect = descriptor->indirect_descriptor ? &*descriptor->indirect_descriptor : nullptr; if (indirect != nullptr && indirect->sources.empty()) { snapshot.samplers[i] = {.dword_count = 4u}; - if ((active.empty() || active[source]) && !MaterializeIndirectDescriptor( - program, source, i, 4u, 4u, observed, clean, snapshot, snapshot.samplers, + if (WalkedActive(program, source) && !MaterializeIndirectDescriptor( + program, source, i, 4u, 4u, observed, snapshot, snapshot.samplers, specialization.samplers, [](DescriptorValue&) { return true; })) return resource_failed("sampler", i, source, snapshot.samplers.size()); diff --git a/src/graphics/shader/recompiler/ir/passes/SrtWalker.cpp b/src/graphics/shader/recompiler/ir/passes/SrtWalker.cpp index babb5959d..314bf2e4a 100644 --- a/src/graphics/shader/recompiler/ir/passes/SrtWalker.cpp +++ b/src/graphics/shader/recompiler/ir/passes/SrtWalker.cpp @@ -8,10 +8,9 @@ #include #include -#include #include +#include #include -#include #include namespace Libs::Graphics::ShaderRecompiler::IR { @@ -20,12 +19,11 @@ SrtRuntime SrtReadCapture::ObservedRuntime() { auto runtime = m_source; runtime.userdata = this; runtime.read_specialization_memory = m_source.read_specialization_memory != nullptr - ? ReadStrict : nullptr; - runtime.read_memory = ReadOrdinary; + ? Read : nullptr; return runtime; } -bool SrtReadCapture::ReadStrict(void* userdata, uint64_t address, std::span values) { +bool SrtReadCapture::Read(void* userdata, uint64_t address, std::span values) { auto& capture = *static_cast(userdata); if (!capture.m_source.read_specialization_memory(capture.m_source.userdata, address, values)) { return false; @@ -34,48 +32,8 @@ bool SrtReadCapture::ReadStrict(void* userdata, uint64_t address, std::span values) { - auto& capture = *static_cast(userdata); - if (capture.m_source.read_memory != nullptr) { - if (!capture.m_source.read_memory(capture.m_source.userdata, address, values)) return false; - } else { - std::memcpy(values.data(), reinterpret_cast(address), values.size_bytes()); - } - capture.m_ranges.emplace_back(address, values.size_bytes()); - return true; -} - -SrtRuntime CleanRuntime(SrtRuntime runtime) { - runtime.read_memory = runtime.read_specialization_memory != nullptr - ? runtime.read_specialization_memory - : +[](void*, uint64_t, std::span) { return false; }; - return runtime; -} - namespace { -constexpr uint64_t AddressMask = 0x0000ffffffffffffull; - -bool AddSignedAddress(uint64_t base, int64_t offset, uint64_t& result) { - if (base > AddressMask) { - return false; - } - if (offset < 0) { - const auto magnitude = uint64_t {0} - static_cast(offset); - if (magnitude > base) { - return false; - } - result = base - magnitude; - return true; - } - const auto magnitude = static_cast(offset); - if (magnitude > AddressMask - base) { - return false; - } - result = base + magnitude; - return true; -} - bool IsRawRead(const ResourcePlan& values, const Inst& inst) { const auto op = inst.GetOpcode(); if (op != ValueOpcode::LoadAddressU32 && op != ValueOpcode::ReadConstBuffer && @@ -176,6 +134,35 @@ bool IsRuntimeUniformOp(ValueOpcode op) { } } +bool ImmediateBits(Value value, uint64_t& bits) { + switch (value.GetType()) { + case Type::U1: bits = value.U1(); return true; + case Type::U8: bits = value.U8(); return true; + case Type::U16: bits = value.U16(); return true; + case Type::U32: bits = value.U32(); return true; + case Type::U64: bits = value.U64(); return true; + case Type::F32: bits = std::bit_cast(value.F32Value()); return true; + default: return false; + } +} + +// Scratch layout in bytes: per descriptor source its eight DWORDs and a known DWORD, one +// activity byte per source, then one qword per walked value (all ones when unknown). The +// walker's value slots, captured addresses, reachability and states follow. +constexpr uint32_t DescriptorStride = 9u * sizeof(uint32_t); + +uint32_t ActiveOffset(const ResourcePlan& plan) { + return static_cast(plan.descriptor_sources.size()) * DescriptorStride; +} + +uint32_t OutputOffset(const ResourcePlan& plan) { + return (ActiveOffset(plan) + static_cast(plan.descriptor_sources.size()) + 7u) & ~7u; +} + +const uint8_t* Scratch(const ResourcePlan& plan) { + return reinterpret_cast(plan.srt_scratch.data()); +} + thread_local const Xbyak::CodeGenerator* t_running_walker = nullptr; // Every guest load is mov eax, [r10] (41 8b 02), so a fault completes it as a zero read. @@ -375,865 +362,1033 @@ private: }; -} // namespace +// Emits one x86 function per plan. Each value, per EXEC-mask context, is a memoized thunk with a +// state byte (0 unevaluated, 1 unknown or in progress, 2 known, 3 a control Phi holding its first +// value) and a qword slot; a reference is `call thunk; cmp state, 2`. Operands that are unknown, +// constant or plain user data are folded at emission. Registers: rdi user data, rsi flat buffer, +// rdx scratch, r8 capture cursor. +class WalkerCompiler { +public: + WalkerCompiler(ResourcePlan& plan, Xbyak::CodeGenerator& code): m_plan(plan), m_code(code) {} -void CompileSrtWalker(ResourcePlan& program) { - if (!program.resource_tracking_complete || !program.srt_plan_complete || - program.capture_specialization_reads || !program.control_flow.empty() || - program.srt_reads.empty()) return; + void Compile(std::span values); - using Reads = std::map; - std::map roots; - std::unordered_map pointer_uses; - std::unordered_map offsets; - for (const auto& read: program.srt_reads) { - const auto* inst = read.value.Resolve().TryInstruction(); - if (inst == nullptr || inst->GetOpcode() != ValueOpcode::LoadAddressU32 || - inst->NumArgs() < 2 || !IsRawRead(program, *inst) || - read.flat_offset >= program.srt_reads.size()) return; - const auto offset = inst->Arg(1).Resolve(); - if (!offset.IsImmediate() || offset.GetType() != Type::U32) return; - const auto& memory = program.memory_info[inst->Flags().index]; - const auto relative = - (static_cast(static_cast(memory.offset)) & ~int64_t {3}) + - static_cast(offset.U32() & ~3u); - if (relative < INT32_MIN || relative > INT32_MAX || - !offsets.emplace(inst, static_cast(relative)).second) return; +private: + enum class Kind : uint8_t { Unknown, Constant, UserData, ShaderBase, Node }; + struct Operand { + Kind kind = Kind::Unknown; + uint64_t value = 0; + + bool operator==(const Operand& other) const = default; + }; + // masked: the operand depends on its context's EXEC mask, so it is not the root context's. + struct Ref { + Operand operand; + bool masked = false; + }; + struct Node { + const Inst* inst = nullptr; + uint32_t context = 0; + Operand result; + std::vector args; + uint32_t slot = 0, state = 0; + Xbyak::Label label; + bool pending = true, referenced = false, masked = false; + }; + + static constexpr auto Near = Xbyak::CodeGenerator::T_NEAR; + + bool Thunk(uint32_t index) const { + const auto& node = m_nodes[index]; + return node.referenced || node.result == Operand {Kind::Node, index}; } - uint32_t user_data_count = 0; - for (const auto& read: program.srt_reads) { - const auto* inst = read.value.ResolveInstruction(); - const auto* handle = inst->Arg(0).Resolve().TryInstruction(); - if (handle == nullptr || handle->GetOpcode() != ValueOpcode::GetAddressResource || - handle->NumArgs() != 2) return; - const auto* low = handle->Arg(0).Resolve().TryInstruction(); - const auto* high = handle->Arg(1).Resolve().TryInstruction(); - if (low == nullptr || high == nullptr) return; - Reads* uses = nullptr; - if (low->GetOpcode() == ValueOpcode::GetUserData && - high->GetOpcode() == ValueOpcode::GetUserData && low->NumArgs() == 1 && - high->NumArgs() == 1 && low->Arg(0).GetType() == Type::ScalarReg && - high->Arg(0).GetType() == Type::ScalarReg) { - const auto reg = RegIndex(low->Arg(0).ScalarRegister()); - if (reg < program.user_data_base || - RegIndex(high->Arg(0).ScalarRegister()) != reg + 1u) return; - const auto base = reg - program.user_data_base; - user_data_count = std::max(user_data_count, base + 2u); - uses = &roots[base]; + static bool MayBeUnknown(const Operand& operand) { + return operand.kind == Kind::Unknown || operand.kind == Kind::Node; + } + + uint32_t Context(Value mask) { + mask = mask.Resolve(); + const auto found = std::ranges::find(m_masks, mask); + if (found != m_masks.end()) return static_cast(found - m_masks.begin()); + m_masks.push_back(mask); + return static_cast(m_masks.size() - 1u); + } + + void AddOutput(Value value, uint32_t gate); + void Consume(std::vector>& gates); + Ref Evaluate(Value value, uint32_t context); + Operand Analyze(uint32_t index); + Operand AnalyzeRead(Node& node, const auto& arg, Operand self); + void Layout(); + void Gate(std::vector& gates, const Xbyak::Label& skip); + void Zero(uint32_t offset, uint32_t size); + void Branch(const Operand& operand, const Xbyak::Label& unknown); + void Load(const Xbyak::Reg64& reg, const Operand& operand); + void Fetch(const Operand& operand, const Xbyak::Label& unknown); + void Set(void (Xbyak::CodeGenerator::*condition)(const Xbyak::Operand&)); + void Store(const Node& node); + void EmitNode(uint32_t index); + void EmitRead(const Node& node); + void EmitStrict(const Node& node); + + ResourcePlan& m_plan; + Xbyak::CodeGenerator& m_code; + std::vector m_masks {Value {}}; + std::deque m_nodes; + std::map, uint32_t> m_memo; + std::vector m_outputs; + std::vector> m_output_gates; + Xbyak::Label m_ret; + uint32_t m_shader_base = 0, m_reach = 0, m_states = 0, m_states_end = 0; + uint32_t m_user_data = 0, m_captures = 0; +}; + +void WalkerCompiler::AddOutput(Value value, uint32_t gate) { + const auto* inst = value.Resolve().TryInstruction(); + if (inst == nullptr) return; + auto index = inst->WalkOutput(); + if (index >= m_outputs.size() || m_outputs[index].TryInstruction() != inst) { + index = static_cast(m_outputs.size()); + inst->SetWalkOutput(index); + m_outputs.push_back(value.Resolve()); + m_output_gates.emplace_back(); + } + m_output_gates[index].push_back(gate); +} + +// Mirrors MaterializeResources: each descriptor it reads back is gated by the activity of the +// resource root that needs it, and so are an indirect root's selector values. +void WalkerCompiler::Consume(std::vector>& gates) { + const auto& sources = m_plan.descriptor_sources; + const auto consume = [&](uint32_t root) { + if (root >= sources.size()) return; + const auto& indirect = sources[root].indirect_descriptor; + if (!indirect) { + gates[root].push_back(root); + return; + } + const auto add = [&](uint32_t source) { + if (source < sources.size()) gates[source].push_back(root); + }; + if (indirect->sources.empty()) { + add(indirect->table_source); + if (indirect->selector) { + add(indirect->selector->source); + if (indirect->selector->rows) add(indirect->selector->rows->source); + } + } + for (const auto source: indirect->sources) add(source); + AddOutput(indirect->key_count, root); + AddOutput(indirect->selector_first, root); + AddOutput(indirect->selector_mask, root); + }; + for (const auto& buffer: m_plan.info.buffers) consume(buffer.source); + for (const auto& image: m_plan.info.images) consume(image.source); + for (const auto& sampler: m_plan.info.samplers) { + const auto* indirect = sampler.source < sources.size() && sources[sampler.source].indirect_descriptor + ? &*sources[sampler.source].indirect_descriptor : nullptr; + if (indirect == nullptr || indirect->sources.empty()) { + consume(sampler.source); + } else if (indirect->sources[0] < sources.size()) { + gates[indirect->sources[0]].push_back(indirect->sources[0]); + } + } + const auto& fill = m_plan.uniform_fill; + if (fill.fill.words != 0) { + AddOutput(fill.pattern_size, UINT32_MAX); + AddOutput(fill.element_count, UINT32_MAX); + for (const auto value: fill.values) AddOutput(value, UINT32_MAX); + } +} + +WalkerCompiler::Ref WalkerCompiler::Evaluate(Value value, uint32_t context) { + value = value.Resolve(); + if (value.IsEmpty()) return {}; + if (value.IsImmediate()) { + uint64_t bits = 0; + return ImmediateBits(value, bits) ? Ref {{Kind::Constant, bits}} : Ref {}; + } + const auto mask = m_masks[context]; + if (context != 0 && value == mask) return {{Kind::Constant, 1u}, true}; + const auto* inst = value.TryInstruction(); + if (inst == nullptr) return {}; + const auto op = inst->GetOpcode(); + if (context != 0 && IsRuntimeSelect(op) && inst->NumArgs() == 3 && inst->Arg(0).Resolve() == mask) { + return {Evaluate(inst->Arg(1), context).operand, true}; + } + switch (op) { + case ValueOpcode::BitCastU32F32: + case ValueOpcode::BitCastF32U32: + case ValueOpcode::ConditionRef: return Evaluate(inst->Arg(0), context); + case ValueOpcode::CompositeExtractU32x2: { + const auto* source = inst->Arg(0).ResolveInstruction(); + const auto index = inst->Arg(1).Resolve(); + if (source == nullptr || source->GetOpcode() != ValueOpcode::CompositeConstructU32x2 || + !index.IsImmediate() || index.GetType() != Type::U32 || index.U32() >= 2u) return {}; + return Evaluate(source->Arg(index.U32()), context); + } + case ValueOpcode::ReadFirstLane: + // The lane read does not depend on the EXEC mask it is read under. + return {Evaluate(inst->Arg(0), Context(inst->Arg(1))).operand}; + case ValueOpcode::GetUserData: { + const auto reg = RegIndex(inst->Arg(0).ScalarRegister()); + if (reg < m_plan.user_data_base || reg - m_plan.user_data_base >= m_plan.user_data_count) return {}; + return {{Kind::UserData, reg - m_plan.user_data_base}}; + } + case ValueOpcode::GetShaderBase: return {{Kind::ShaderBase}}; + case ValueOpcode::Phi: + if (!m_plan.control_phi_predecessors.contains(inst)) { + const auto invariant = ResolveInvariantPhi(m_plan, value); + return invariant.IsEmpty() ? Ref {} : Evaluate(invariant, context); + } + break; + default: break; + } + const auto [found, inserted] = m_memo.try_emplace({inst, context}, static_cast(m_nodes.size())); + const auto index = found->second; + if (!inserted) { + auto& node = m_nodes[index]; + if (!node.pending) return {node.result, node.masked}; + // A cycle: the thunk reads as unknown while it runs. + node.referenced = true; + return {{Kind::Node, index}, context != 0}; + } + auto& node = m_nodes.emplace_back(); + node.inst = inst; + node.context = context; + node.result = Analyze(index); + node.pending = false; + if (context != 0 && !node.masked && !node.referenced) { + node.result = Evaluate(value, 0).operand; + return {node.result}; + } + return {node.result, node.masked}; +} + +WalkerCompiler::Operand WalkerCompiler::Analyze(uint32_t index) { + auto& node = m_nodes[index]; + const auto& inst = *node.inst; + const auto self = Operand {Kind::Node, index}; + const auto arg = [&](Value value) { + const auto ref = Evaluate(value, node.context); + node.masked |= ref.masked; + return ref.operand; + }; + const auto constant = [](uint64_t value) { return Operand {Kind::Constant, value}; }; + switch (inst.GetOpcode()) { + case ValueOpcode::Phi: { + bool known = false; + for (size_t i = 0; i < inst.NumArgs(); ++i) { + node.args.push_back(arg(inst.Arg(i))); + known |= node.args.back().kind != Kind::Unknown; + } + return known ? self : Operand {}; + } + case ValueOpcode::LoadAddressU32: + case ValueOpcode::ReadConstBuffer: + case ValueOpcode::LoadBufferU32: return AnalyzeRead(node, arg, self); + case ValueOpcode::SelectU1: + case ValueOpcode::SelectU32: + case ValueOpcode::SelectF32: { + const auto predicate = arg(inst.Arg(0)); + if (predicate.kind == Kind::Unknown) return {}; + if (predicate.kind == Kind::Constant) return arg(inst.Arg(predicate.value != 0u ? 1 : 2)); + node.args = {predicate, arg(inst.Arg(1)), arg(inst.Arg(2))}; + return node.args[1].kind == Kind::Unknown && node.args[2].kind == Kind::Unknown ? Operand {} : self; + } + case ValueOpcode::ULessThan32: + case ValueOpcode::ULessThanEqual32: { + // No U32 is below zero and every U32 is at least zero, including an unknown loop-carried + // or varying lane value. args[0] is that zero-tested bound. + const bool less = inst.GetOpcode() == ValueOpcode::ULessThan32; + const auto bound = arg(inst.Arg(less ? 1 : 0)); + if (bound.kind == Kind::Unknown) return {}; + if (bound.kind == Kind::Constant && static_cast(bound.value) == 0u) return constant(!less); + node.args = {bound, arg(inst.Arg(less ? 0 : 1))}; + return node.args[1].kind == Kind::Unknown && bound.kind == Kind::Constant ? Operand {} : self; + } + case ValueOpcode::LogicalOr: + case ValueOpcode::LogicalAnd: { + // A deciding operand, nonzero for Or and zero for And, fixes the result. + const bool conjunction = inst.GetOpcode() == ValueOpcode::LogicalAnd; + const auto decides = [&](const Operand& x) { + return x.kind == Kind::Constant && (x.value == 0u) == conjunction; + }; + const auto a = arg(inst.Arg(0)); + if (decides(a)) return constant(!conjunction); + const auto b = arg(inst.Arg(1)); + if (decides(b)) return constant(!conjunction); + if (a.kind == Kind::Constant && b.kind == Kind::Constant) return constant(conjunction); + if (a.kind == Kind::Constant && b.kind == Kind::Unknown) return {}; + // Only lanes satisfying the left EXEC factor can make a conjunction true, so an + // unknown left side retries the right one under that factor as its mask. + Operand refined; + if (conjunction && MayBeUnknown(a)) { + refined = Evaluate(inst.Arg(1), Context(inst.Arg(0))).operand; + if (refined == b || (refined.kind == Kind::Constant && refined.value != 0u)) refined = {}; + if (a.kind == Kind::Unknown && refined.kind == Kind::Constant) return constant(0u); + } + if (a.kind == Kind::Unknown && refined.kind == Kind::Unknown && + (b.kind == Kind::Unknown || b.kind == Kind::Constant)) return {}; + node.args = {a, b, refined}; + return self; + } + case ValueOpcode::CompositeExtractU64: { + const auto component = inst.Arg(1).Resolve(); + if (!component.IsImmediate() || component.GetType() != Type::U32 || component.U32() >= 2u) return {}; + [[fallthrough]]; + } + default: + // The other runtime-uniform ops are strict; a U32x2 pair is only read through its extracts. + if (!IsRuntimeUniformOp(inst.GetOpcode()) || inst.GetOpcode() == ValueOpcode::CompositeConstructU32x2) + return {}; + for (size_t i = 0; i < inst.NumArgs(); ++i) { + node.args.push_back(arg(inst.Arg(i))); + if (node.args.back().kind == Kind::Unknown) return {}; + } + return self; + } +} + +WalkerCompiler::Operand WalkerCompiler::AnalyzeRead(Node& node, const auto& arg, Operand self) { + const auto& inst = *node.inst; + const auto* handle = inst.Arg(0).ResolveInstruction(); + if (!IsRawRead(m_plan, inst) || handle == nullptr) return {}; + const bool vector = inst.GetOpcode() == ValueOpcode::LoadBufferU32; + if (vector) { + const auto guard = arg(inst.Arg(4)); + if (guard.kind == Kind::Unknown) return {}; + if (guard.kind == Kind::Constant && guard.value == 0u) return {Kind::Constant, 0u}; + node.args.push_back(guard); + } + const bool buffer = inst.GetOpcode() != ValueOpcode::LoadAddressU32; + const auto offset = static_cast(m_plan.memory_info[inst.Flags().index].offset); + bool failed = buffer && (handle->NumArgs() != 4u || offset < 0); + for (const auto& word: {handle->Arg(0), handle->Arg(1), inst.Arg(1)}) { + node.args.push_back(arg(word)); + failed |= node.args.back().kind == Kind::Unknown; + } + for (uint32_t word = 2; buffer && !failed && word < 4u; ++word) { + node.args.push_back(arg(handle->Arg(word))); + failed |= node.args.back().kind == Kind::Unknown; + } + if (!failed) return self; + // A disabled vector lane still reads zero. + if (!vector || node.args[0].kind == Kind::Constant) return {}; + node.args.resize(1); + return self; +} + +void WalkerCompiler::Layout() { + uint32_t offset = OutputOffset(m_plan) + static_cast(m_outputs.size()) * 8u; + m_shader_base = offset; + offset += 8u; + for (uint32_t index = 0; index < m_nodes.size(); ++index) { + if (!Thunk(index)) continue; + m_nodes[index].slot = offset; + offset += 8u; + } + m_reach = offset; + offset += static_cast(m_plan.control_flow.size()); + m_states = offset; + for (uint32_t index = 0; index < m_nodes.size(); ++index) { + if (Thunk(index)) m_nodes[index].state = offset++; + } + m_states_end = offset; + // Captured addresses come last; EmitRead counts them. + m_plan.srt_captures = (offset + 7u) & ~7u; +} + +// Falls through when any gating source is active. The list is not empty. +void WalkerCompiler::Gate(std::vector& gates, const Xbyak::Label& skip) { + auto& c = m_code; + std::ranges::sort(gates); + if (gates.back() == UINT32_MAX) return; + gates.erase(std::unique(gates.begin(), gates.end()), gates.end()); + Xbyak::Label active; + for (size_t i = 0; i < gates.size(); ++i) { + c.cmp(c.byte[c.rdx + ActiveOffset(m_plan) + gates[i]], 0); + if (i + 1u < gates.size()) { + c.jne(active, Near); } else { - const auto lo = offsets.find(low); - const auto hi = offsets.find(high); - if (lo == offsets.end() || hi == offsets.end() || - static_cast(lo->second) + 4 != hi->second || - !EquivalentValue(program, low->Arg(0), high->Arg(0))) return; - uses = &pointer_uses[low]; + c.je(skip, Near); } - if (!uses->emplace(offsets.at(inst), &read).second) return; } - std::unordered_set visited; - const auto reachable = [&](auto&& self, const Reads& reads) -> bool { - for (const auto& [offset, read]: reads) { - const auto* inst = read->value.ResolveInstruction(); - const auto child = pointer_uses.find(inst); - if (child != pointer_uses.end() && - (!visited.insert(inst).second || !self(self, child->second))) return false; - } - return true; - }; - for (const auto& [reg, reads]: roots) { - if (!reachable(reachable, reads)) return; - } - if (visited.size() != pointer_uses.size()) return; + c.L(active); +} - program.srt_code = std::make_unique(4096, Xbyak::AutoGrow); - auto& code = *program.srt_code; - Xbyak::Label failed, finished; - code.push(code.rbp); - code.mov(code.rbp, code.rsp); - code.cmp(code.r8, user_data_count); - code.jb(failed, Xbyak::CodeGenerator::T_NEAR); - code.mov(code.r9, AddressMask & ~uint64_t {3}); - // Follow shadPS4's VisitPointer: copy one table level in offset order, then - // descend through its child pointers. Seed the descriptor evaluator's memo. - const auto visit = [&](auto&& self, int32_t pointer_offset, const Reads& reads) -> void { - code.push(code.rdi); - LoadGuest(code, code.rdi + pointer_offset + 4); - code.shl(code.rax, 32); - code.mov(code.r11, code.rax); - LoadGuest(code, code.rdi + pointer_offset); - code.or_(code.rax, code.r11); - code.and_(code.rax, code.r9); - code.mov(code.rdi, code.rax); - const auto first = reads.begin()->first; - const auto last = reads.rbegin()->first; - if (first < 0) { - code.mov(code.r10, -static_cast(first)); - code.cmp(code.rdi, code.r10); - code.jb(failed, Xbyak::CodeGenerator::T_NEAR); - } - if (last > 0) { - code.mov(code.r10, AddressMask - last); - code.cmp(code.rdi, code.r10); - code.ja(failed, Xbyak::CodeGenerator::T_NEAR); - } - for (const auto& [offset, read]: reads) { - const auto index = read->value.ResolveInstruction()->EvaluationIndex(program.evaluation_value_count); - using Entry = ResourcePlan::EvaluationContext::Entry; - const auto slot = code.rdx + index * sizeof(Entry); - LoadGuest(code, code.rdi + offset); - code.mov(code.dword[code.rsi + read->flat_offset * sizeof(uint32_t)], code.eax); - code.mov(code.qword[slot + offsetof(Entry, value)], code.rax); - code.mov(code.qword[slot + offsetof(Entry, generation)], code.rcx); - } - for (const auto& [offset, read]: reads) { - const auto child = pointer_uses.find(read->value.ResolveInstruction()); - if (child == pointer_uses.end()) continue; - self(self, offset, child->second); - } - code.pop(code.rdi); - }; - for (const auto& [reg, reads]: roots) { - visit(visit, reg * sizeof(uint32_t), reads); +void WalkerCompiler::Zero(uint32_t offset, uint32_t size) { + auto& c = m_code; + if (size == 0) return; + c.mov(c.r11, c.rdi); + c.lea(c.rdi, c.ptr[c.rdx + offset]); + c.mov(c.ecx, size); + c.xor_(c.eax, c.eax); + c.rep(); + c.stosb(); + c.mov(c.rdi, c.r11); +} + +// Jumps to unknown unless the operand has a value at run time. +void WalkerCompiler::Branch(const Operand& operand, const Xbyak::Label& unknown) { + auto& c = m_code; + if (operand.kind == Kind::Unknown) { + c.jmp(unknown, Near); + } else if (operand.kind == Kind::Node) { + auto& node = m_nodes[operand.value]; + c.call(node.label); + c.cmp(c.byte[c.rdx + node.state], 2); + c.jne(unknown, Near); } - code.mov(code.eax, 1); - code.jmp(finished, Xbyak::CodeGenerator::T_NEAR); - code.L(failed); - code.xor_(code.eax, code.eax); - code.L(finished); - code.mov(code.rsp, code.rbp); - code.pop(code.rbp); - code.ret(); - code.readyRE(); +} + +void WalkerCompiler::Load(const Xbyak::Reg64& reg, const Operand& operand) { + auto& c = m_code; + switch (operand.kind) { + case Kind::Constant: c.mov(reg, operand.value); break; + case Kind::UserData: + m_user_data = std::max(m_user_data, static_cast(operand.value) + 1u); + c.mov(reg.cvt32(), c.dword[c.rdi + operand.value * sizeof(uint32_t)]); + break; + case Kind::ShaderBase: c.mov(reg, c.qword[c.rdx + m_shader_base]); break; + case Kind::Node: c.mov(reg, c.qword[c.rdx + m_nodes[operand.value].slot]); break; + case Kind::Unknown: break; + } +} + +// Loads the operand into rax, or jumps to unknown when it has no value at run time. +void WalkerCompiler::Fetch(const Operand& operand, const Xbyak::Label& unknown) { + Branch(operand, unknown); + Load(m_code.rax, operand); +} + +// Sets eax to 1 when the condition holds, else 0. +void WalkerCompiler::Set(void (Xbyak::CodeGenerator::*condition)(const Xbyak::Operand&)) { + (m_code.*condition)(m_code.al); + m_code.movzx(m_code.eax, m_code.al); +} + +void WalkerCompiler::Store(const Node& node) { + auto& c = m_code; + c.mov(c.qword[c.rdx + node.slot], c.rax); + c.mov(c.byte[c.rdx + node.state], 2); + c.ret(); +} + +void WalkerCompiler::EmitNode(uint32_t index) { + auto& c = m_code; + auto& node = m_nodes[index]; + const auto& args = node.args; + c.L(node.label); + c.cmp(c.byte[c.rdx + node.state], 0); + c.jne(m_ret, Near); + c.mov(c.byte[c.rdx + node.state], 1); + if (node.result != Operand {Kind::Node, index}) { + // A cycle referenced this value before it folded to another operand. + Fetch(node.result, m_ret); + Store(node); + return; + } + switch (node.inst->GetOpcode()) { + case ValueOpcode::Phi: { + // Only predecessors reached by this walk contribute, and they must agree. The state is 3 + // once one of them has stored the value; callers read it like 1, as unknown. + const auto& blocks = m_plan.control_phi_predecessors.at(node.inst); + const auto state = c.byte[c.rdx + node.state]; + for (size_t i = 0; i < args.size(); ++i) { + Xbyak::Label next, first; + c.cmp(c.byte[c.rdx + m_reach + blocks[i]], 0); + c.je(next, Near); + Fetch(args[i], m_ret); + c.cmp(state, 1); + c.je(first, Near); + c.cmp(c.rax, c.qword[c.rdx + node.slot]); + c.jne(m_ret, Near); + c.jmp(next, Near); + c.L(first); + c.mov(c.qword[c.rdx + node.slot], c.rax); + c.mov(state, 3); + c.L(next); + } + c.cmp(state, 3); + c.jne(m_ret, Near); + c.mov(state, 2); + c.ret(); + return; + } + case ValueOpcode::LoadAddressU32: + case ValueOpcode::ReadConstBuffer: + case ValueOpcode::LoadBufferU32: EmitRead(node); return; + case ValueOpcode::SelectU1: + case ValueOpcode::SelectU32: + case ValueOpcode::SelectF32: { + Xbyak::Label other; + Fetch(args[0], m_ret); + c.test(c.rax, c.rax); + c.jz(other, Near); + Fetch(args[1], m_ret); + Store(node); + c.L(other); + Fetch(args[2], m_ret); + Store(node); + return; + } + case ValueOpcode::ULessThan32: + case ValueOpcode::ULessThanEqual32: { + const bool less = node.inst->GetOpcode() == ValueOpcode::ULessThan32; + Xbyak::Label bound_zero; + Fetch(args[0], m_ret); + c.test(c.eax, c.eax); + c.jz(bound_zero, Near); + Branch(args[1], m_ret); + Load(c.rax, args[0]); + Load(c.rcx, args[1]); + // ULT tests other < bound, ULE bound <= other. + c.cmp(c.ecx, c.eax); + Set(less ? &Xbyak::CodeGenerator::setb : &Xbyak::CodeGenerator::setae); + Store(node); + c.L(bound_zero); + c.mov(c.eax, uint32_t {!less}); + Store(node); + return; + } + case ValueOpcode::LogicalOr: + case ValueOpcode::LogicalAnd: { + const bool conjunction = node.inst->GetOpcode() == ValueOpcode::LogicalAnd; + Xbyak::Label unknown_left, refine, decided; + const auto decide = [&] { + if (conjunction) { + c.jz(decided, Near); + } else { + c.jnz(decided, Near); + } + }; + Branch(args[0], unknown_left); + if (args[0].kind != Kind::Unknown) { + Load(c.rax, args[0]); + c.test(c.rax, c.rax); + decide(); + Fetch(args[1], m_ret); + c.test(c.rax, c.rax); + Set(&Xbyak::CodeGenerator::setne); + Store(node); + } + c.L(unknown_left); + if (MayBeUnknown(args[0])) { + // Without the left side only a deciding right side, or a conjunction's zero + // refined right side, is known. + if (args[1].kind != Kind::Unknown) { + Fetch(args[1], refine); + c.test(c.rax, c.rax); + decide(); + } + c.L(refine); + Fetch(args[2], m_ret); + c.test(c.rax, c.rax); + c.jnz(m_ret, Near); + } + c.L(decided); + c.mov(c.eax, uint32_t {!conjunction}); + Store(node); + return; + } + default: EmitStrict(node); return; + } +} + +// Raw scalar and uniform DWORD buffer loads; out-of-bounds reads are zero. +void WalkerCompiler::EmitRead(const Node& node) { + auto& c = m_code; + const auto& inst = *node.inst; + const auto op = inst.GetOpcode(); + const bool vector = op == ValueOpcode::LoadBufferU32; + auto args = std::span(node.args); + Xbyak::Label zero; + if (vector) { + Fetch(args[0], m_ret); + c.test(c.rax, c.rax); + c.jz(zero, Near); + args = args.subspan(1); + if (args.empty()) c.jmp(m_ret, Near); + } + if (!args.empty()) { + for (const auto& arg: args) Branch(arg, m_ret); + const auto immediate = + static_cast(static_cast(m_plan.memory_info[inst.Flags().index].offset)) & + ~int64_t {3}; + // rax = the 48-bit base (high << 32 | low) with its low two bits clear. + Load(c.rax, args[1]); + c.shl(c.rax, 48); + c.shr(c.rax, 16); + Load(c.rcx, args[0]); + c.and_(c.ecx, ~3u); + c.or_(c.rax, c.rcx); + Load(c.r9, args[2]); + c.and_(c.r9d, ~3u); + if (op == ValueOpcode::LoadAddressU32) { + c.lea(c.rax, c.ptr[c.rax + c.r9 + static_cast(immediate)]); + c.mov(c.rcx, c.rax); + c.shr(c.rcx, 48); + c.jnz(m_ret, Near); + LoadGuest(c, c.rax); + } else { + Load(c.r11, args[1]); + c.shr(c.r11d, 16); + if (vector) { + // Only the uniform, unswizzled structured DWORD address is evaluated on the host. + c.test(c.r11d, 1u << 15u); + c.jnz(m_ret, Near); + Load(c.rcx, args[4]); + c.test(c.ecx, (1u << 23u) | 0xf0000000u); + c.jnz(m_ret, Near); + c.test(c.r11d, 0x3fffu); + c.jz(zero, Near); + c.test(c.ecx, 0x7fu << 12u); + c.jz(zero, Near); + } + c.and_(c.r11d, 0x3fffu); + Load(c.rcx, args[3]); + if (vector) { + c.test(c.ecx, c.ecx); + c.jz(zero, Near); + } + Xbyak::Label sized; + c.test(c.r11d, c.r11d); + c.jz(sized, Near); + c.imul(c.rcx, c.r11); + c.L(sized); + c.lea(c.r11, c.ptr[c.r9 + static_cast(immediate) + 4u]); + c.cmp(c.r11, c.rcx); + c.ja(zero, Near); + LoadGuest(c, c.rax + c.r9 + static_cast(immediate)); + } + ++m_captures; + c.mov(c.qword[c.r8], c.r10); + c.add(c.r8, 8); + Store(node); + } + c.L(zero); + c.xor_(c.eax, c.eax); + Store(node); +} + +void WalkerCompiler::EmitStrict(const Node& node) { + auto& c = m_code; + const auto& inst = *node.inst; + for (const auto& arg: node.args) Branch(arg, m_ret); + const std::array regs {c.rax, c.rcx, c.r9, c.r11}; + for (size_t i = 0; i < node.args.size() && i < regs.size(); ++i) Load(regs[i], node.args[i]); + // Field ops require offset + width <= 32. Leaves (1 << width) - 1 in r11, the offset in + // cl and the former rcx (the offset or the inserted value) in r10. + const auto field = [&](const Xbyak::Reg32& offset, const Xbyak::Reg32& width) { + c.lea(c.r10, c.ptr[offset.cvt64() + width.cvt64()]); + c.cmp(c.r10, 32); + c.ja(m_ret, Near); + c.mov(c.r10, c.rcx); + c.mov(c.ecx, width); + c.mov(c.r11d, 1); + c.shl(c.r11, c.cl); + c.dec(c.r11); + c.mov(c.ecx, offset.getIdx() == Xbyak::Operand::RCX ? c.r10d : offset); + }; + switch (inst.GetOpcode()) { + case ValueOpcode::CompositeConstructU64: + c.shl(c.rcx, 32); + c.or_(c.rax, c.rcx); + break; + case ValueOpcode::CompositeExtractU64: + if (inst.Arg(1).Resolve().U32() != 0u) c.shr(c.rax, 32); + c.mov(c.eax, c.eax); + break; + case ValueOpcode::IAdd32: c.add(c.eax, c.ecx); break; + case ValueOpcode::IAdd64: c.add(c.rax, c.rcx); break; + case ValueOpcode::ISub32: c.sub(c.eax, c.ecx); break; + case ValueOpcode::ISub64: c.sub(c.rax, c.rcx); break; + case ValueOpcode::IMul32: c.imul(c.eax, c.ecx); break; + case ValueOpcode::UMulHi: + c.imul(c.rax, c.rcx); + c.shr(c.rax, 32); + break; + case ValueOpcode::UMin32: + c.cmp(c.eax, c.ecx); + c.cmova(c.eax, c.ecx); + break; + case ValueOpcode::UMax32: + c.cmp(c.eax, c.ecx); + c.cmovb(c.eax, c.ecx); + break; + case ValueOpcode::ConvertF32U32: + c.cvtsi2ss(c.xmm0, c.rax); + c.movd(c.eax, c.xmm0); + break; + case ValueOpcode::ConvertU32F32: { + // The conversion saturates and maps NaN to zero. + Xbyak::Label done; + c.movd(c.xmm0, c.eax); + c.xorps(c.xmm1, c.xmm1); + c.xor_(c.eax, c.eax); + c.ucomiss(c.xmm0, c.xmm1); + c.jbe(done, Near); + c.mov(c.ecx, 0x4f800000u); + c.movd(c.xmm1, c.ecx); + c.mov(c.eax, UINT32_MAX); + c.ucomiss(c.xmm0, c.xmm1); + c.jae(done, Near); + c.cvttss2si(c.rax, c.xmm0); + c.L(done); + break; + } + case ValueOpcode::FPMul32: + c.movd(c.xmm0, c.eax); + c.movd(c.xmm1, c.ecx); + c.mulss(c.xmm0, c.xmm1); + c.movd(c.eax, c.xmm0); + break; + case ValueOpcode::FPTrunc32: + c.movd(c.xmm0, c.eax); + c.roundss(c.xmm0, c.xmm0, 3); + c.movd(c.eax, c.xmm0); + break; + case ValueOpcode::FPRecipIFlag32: + // Normal positive powers of two have exact normal reciprocals in every FP mode. + c.test(c.eax, 0x807fffffu); + c.jnz(m_ret, Near); + c.shr(c.eax, 23); + c.jz(m_ret, Near); + c.cmp(c.eax, 254); + c.jae(m_ret, Near); + c.neg(c.eax); + c.add(c.eax, 254); + c.shl(c.eax, 23); + break; + case ValueOpcode::FPIsNan32: + c.and_(c.eax, 0x7fffffffu); + c.cmp(c.eax, 0x7f800000u); + Set(&Xbyak::CodeGenerator::seta); + break; + case ValueOpcode::FPOrdLessThanEqual32: + case ValueOpcode::FPOrdGreaterThanEqual32: + if (inst.Flags().flush_input_denorms) { + for (const auto& reg: {c.eax, c.ecx}) { + Xbyak::Label normal; + c.mov(c.r11d, reg); + c.and_(c.r11d, 0x7fffffffu); + c.cmp(c.r11d, 0x00800000u); + c.jae(normal, Near); + c.and_(reg, 0x80000000u); + c.L(normal); + } + } + c.movd(c.xmm0, c.eax); + c.movd(c.xmm1, c.ecx); + // Unordered operands set CF, so AE is false for NaN. + if (inst.GetOpcode() == ValueOpcode::FPOrdLessThanEqual32) { + c.ucomiss(c.xmm1, c.xmm0); + } else { + c.ucomiss(c.xmm0, c.xmm1); + } + Set(&Xbyak::CodeGenerator::setae); + break; + case ValueOpcode::BitwiseAnd32: c.and_(c.eax, c.ecx); break; + case ValueOpcode::BitwiseAnd64: c.and_(c.rax, c.rcx); break; + case ValueOpcode::BitwiseOr32: c.or_(c.eax, c.ecx); break; + case ValueOpcode::BitwiseXor32: c.xor_(c.eax, c.ecx); break; + case ValueOpcode::BitwiseNot32: c.not_(c.eax); break; + case ValueOpcode::ShiftLeftLogical32: c.shl(c.eax, c.cl); break; + case ValueOpcode::ShiftLeftLogical64: c.shl(c.rax, c.cl); break; + case ValueOpcode::ShiftRightLogical32: c.shr(c.eax, c.cl); break; + case ValueOpcode::ShiftRightLogical64: c.shr(c.rax, c.cl); break; + case ValueOpcode::ShiftRightArithmetic32: c.sar(c.eax, c.cl); break; + case ValueOpcode::ShiftRightArithmetic64: c.sar(c.rax, c.cl); break; + case ValueOpcode::BitFieldUExtract: + field(c.ecx, c.r9d); + c.shr(c.rax, c.cl); + c.and_(c.rax, c.r11); + break; + case ValueOpcode::BitFieldSExtract: { + Xbyak::Label done; + field(c.ecx, c.r9d); + c.shr(c.rax, c.cl); + c.and_(c.rax, c.r11); + // Width zero tests bit 63, which is clear. + c.lea(c.ecx, c.ptr[c.r9 - 1]); + c.bt(c.rax, c.rcx); + c.jnc(done, Near); + c.not_(c.r11); + c.or_(c.rax, c.r11); + c.mov(c.eax, c.eax); + c.L(done); + break; + } + case ValueOpcode::BitFieldInsert: + field(c.r9d, c.r11d); + c.shl(c.r11, c.cl); + c.shl(c.r10, c.cl); + c.and_(c.r10, c.r11); + c.not_(c.r11); + c.and_(c.rax, c.r11); + c.or_(c.rax, c.r10); + c.mov(c.eax, c.eax); + break; + case ValueOpcode::IEqual32: + c.cmp(c.eax, c.ecx); + Set(&Xbyak::CodeGenerator::sete); + break; + case ValueOpcode::INotEqual32: + c.cmp(c.eax, c.ecx); + Set(&Xbyak::CodeGenerator::setne); + break; + case ValueOpcode::IEqual64: + c.cmp(c.rax, c.rcx); + Set(&Xbyak::CodeGenerator::sete); + break; + case ValueOpcode::INotEqual64: + c.cmp(c.rax, c.rcx); + Set(&Xbyak::CodeGenerator::setne); + break; + case ValueOpcode::UGreaterThan32: + c.cmp(c.eax, c.ecx); + Set(&Xbyak::CodeGenerator::seta); + break; + case ValueOpcode::SGreaterThanEqual32: + c.cmp(c.eax, c.ecx); + Set(&Xbyak::CodeGenerator::setge); + break; + case ValueOpcode::LogicalXor: + c.test(c.rax, c.rax); + c.setne(c.al); + c.test(c.rcx, c.rcx); + c.setne(c.cl); + c.xor_(c.al, c.cl); + c.movzx(c.eax, c.al); + break; + case ValueOpcode::LogicalNot: + c.test(c.rax, c.rax); + Set(&Xbyak::CodeGenerator::sete); + break; + default: EXIT("unexpected SRT walker opcode\n"); + } + Store(node); +} + +void WalkerCompiler::Compile(std::span values) { + auto& c = m_code; + const auto& sources = m_plan.descriptor_sources; + const auto& blocks = m_plan.control_flow; + const bool resources = values.empty(); + std::vector> gates(sources.size()); + if (resources) Consume(gates); + for (const auto value: values) AddOutput(value, UINT32_MAX); + // Analyze every root first, so thunk bodies only reuse finished operands. + std::vector conditions, reads, outputs; + std::vector> descriptors(sources.size()); + if (resources) { + for (const auto& block: blocks) conditions.push_back(Evaluate(block.condition, 0).operand); + for (const auto& read: m_plan.srt_reads) reads.push_back(Evaluate(read.value, 0).operand); + for (uint32_t source = 0; source < sources.size(); ++source) { + if (gates[source].empty()) continue; + for (uint32_t word = 0; word < sources[source].dword_count; ++word) + descriptors[source].push_back(Evaluate(sources[source].dwords[word], 0).operand); + } + } + for (const auto value: m_outputs) outputs.push_back(Evaluate(value, 0).operand); + Layout(); + + Xbyak::Label pass, fail, reset; + c.mov(c.qword[c.rdx + m_shader_base], c.rcx); + Zero(m_reach, m_states_end - m_reach); + c.lea(c.r8, c.ptr[c.rdx + m_plan.srt_captures]); + if (!resources) { + for (uint32_t block = 0; block < blocks.size(); ++block) c.mov(c.byte[c.rdx + m_reach + block], 1); + } else if (!blocks.empty()) { + // Reachability grows monotonically. A newly reached predecessor of an evaluated + // control Phi restarts the walk with fresh values; a retreating edge resumes at its target. + std::vector> rechecks(blocks.size()); + for (uint32_t index = 0; index < m_nodes.size(); ++index) { + const auto& node = m_nodes[index]; + if (node.inst->GetOpcode() != ValueOpcode::Phi || node.result != Operand {Kind::Node, index}) continue; + for (const auto block: m_plan.control_phi_predecessors.at(node.inst)) rechecks[block].push_back(node.state); + } + std::vector starts(blocks.size()); + c.mov(c.byte[c.rdx + m_reach], 1); + c.L(pass); + for (uint32_t block = 0; block < blocks.size(); ++block) { + Xbyak::Label next; + const auto mark = [&](uint32_t target) { + Xbyak::Label reached; + c.cmp(c.byte[c.rdx + m_reach + target], 0); + c.jne(reached, Near); + c.mov(c.byte[c.rdx + m_reach + target], 1); + for (const auto state: rechecks[target]) { + c.cmp(c.byte[c.rdx + state], 0); + c.jne(reset, Near); + } + if (target <= block) c.jmp(starts[target], Near); + c.L(reached); + }; + const auto& successors = blocks[block].successors; + const auto condition = conditions[block]; + c.L(starts[block]); + c.cmp(c.byte[c.rdx + m_reach + block], 0); + c.je(next, Near); + if (condition.kind == Kind::Constant) { + mark(successors[condition.value != 0u ? 0 : 1]); + } else { + if (condition.kind != Kind::Unknown) { + Xbyak::Label both, other; + Fetch(condition, both); + c.test(c.rax, c.rax); + c.jz(other, Near); + mark(successors[0]); + c.jmp(next, Near); + c.L(other); + mark(successors[1]); + c.jmp(next, Near); + c.L(both); + } + for (const auto successor: successors) mark(successor); + } + c.L(next); + } + } + if (resources) { + const auto flat = [&](uint32_t slot) { + const auto offset = slot < reads.size() ? m_plan.srt_reads[slot].flat_offset : UINT32_MAX; + if (offset >= reads.size()) { + c.jmp(fail, Near); + return; + } + Fetch(reads[slot], fail); + c.mov(c.dword[c.rsi + offset * sizeof(uint32_t)], c.eax); + }; + if (blocks.empty()) { + for (uint32_t slot = 0; slot < reads.size(); ++slot) flat(slot); + } + for (uint32_t block = 0; block < blocks.size(); ++block) { + if (blocks[block].srt_reads.empty()) continue; + Xbyak::Label skip; + c.cmp(c.byte[c.rdx + m_reach + block], 0); + c.je(skip, Near); + for (const auto slot: blocks[block].srt_reads) flat(slot); + c.L(skip); + } + // A source no block lists is always active. + for (uint32_t source = 0; source < sources.size(); ++source) { + bool listed = false; + for (uint32_t block = 0; block < blocks.size(); ++block) { + if (std::ranges::find(blocks[block].sources, source) == blocks[block].sources.end()) continue; + if (listed) { + c.or_(c.al, c.byte[c.rdx + m_reach + block]); + } else { + c.mov(c.al, c.byte[c.rdx + m_reach + block]); + } + listed = true; + } + const auto active = c.byte[c.rdx + ActiveOffset(m_plan) + source]; + if (listed) { + c.mov(active, c.al); + } else { + c.mov(active, 1); + } + } + for (uint32_t source = 0; source < sources.size(); ++source) { + if (gates[source].empty()) continue; + Xbyak::Label skip; + const auto known = c.dword[c.rdx + source * DescriptorStride + 8u * sizeof(uint32_t)]; + c.mov(known, 0); + Gate(gates[source], skip); + for (uint32_t word = 0; word < descriptors[source].size(); ++word) { + Fetch(descriptors[source][word], skip); + c.mov(c.dword[c.rdx + source * DescriptorStride + word * sizeof(uint32_t)], c.eax); + } + c.mov(known, 1); + c.L(skip); + } + } + for (uint32_t index = 0; index < outputs.size(); ++index) { + Xbyak::Label skip; + const auto output = c.qword[c.rdx + OutputOffset(m_plan) + index * 8u]; + c.mov(output, -1); + Gate(m_output_gates[index], skip); + Fetch(outputs[index], skip); + c.mov(c.eax, c.eax); + c.mov(output, c.rax); + c.L(skip); + } + c.mov(c.rax, c.r8); + c.L(m_ret); + c.ret(); + c.L(fail); + c.xor_(c.eax, c.eax); + c.ret(); + if (resources && !blocks.empty()) { + c.L(reset); + Zero(m_states, m_states_end - m_states); + c.lea(c.r8, c.ptr[c.rdx + m_plan.srt_captures]); + c.jmp(pass, Near); + } + for (uint32_t index = 0; index < m_nodes.size(); ++index) { + if (Thunk(index)) EmitNode(index); + } + m_plan.srt_scratch.assign(m_plan.srt_captures / 8u + m_captures, 0u); + m_plan.srt_user_data = m_user_data; + c.readyRE(); EXIT_IF(Xbyak::GetError() != 0); } -SrtWalker::SrtWalker(const ResourcePlan& program, const SrtRuntime& runtime, - SrtWalker* clean_evaluator) - : m_program(program), m_runtime(runtime), m_clean_evaluator(clean_evaluator), - m_session(this), - m_read_policy(clean_evaluator != nullptr ? 2u : 0u), m_context(AcquireContext()) {} +} // namespace -SrtWalker::SrtWalker(SrtWalker& parent, Value active_mask, bool clean, - SrtWalker* clean_evaluator) - : m_program(parent.m_program), m_runtime(clean ? CleanRuntime(parent.m_runtime) : parent.m_runtime), - m_clean_evaluator(clean_evaluator), m_active_mask(active_mask.Resolve()), - m_reachable_blocks(parent.m_reachable_blocks), m_session(parent.m_session), - m_read_policy((parent.m_read_policy & 1u) | (clean ? 1u : 0u) | - (clean_evaluator != nullptr ? 2u : 0u)), m_context(AcquireContext()) {} - -SrtWalker::~SrtWalker() { - if (m_session != this) return; - for (auto& context: m_program.evaluation_contexts) - if (context.session == this) context.session = nullptr; +void CompileSrtWalker(ResourcePlan& plan, std::span values) { + plan.srt_code = std::make_unique(4096, Xbyak::AutoGrow); + WalkerCompiler(plan, *plan.srt_code).Compile(values); } -bool SrtWalker::Evaluate(Value value, uint32_t& result) { - uint64_t wide = 0; - if (!EvaluateWide(value, wide)) { - return false; +bool WalkSrt(const ResourcePlan& plan, const SrtRuntime& runtime, std::vector& flat, + std::vector>* reads) { + if (plan.srt_code == nullptr || runtime.user_data.size() < plan.srt_user_data) return false; + flat.assign(plan.srt_reads.size(), 0u); + using Walker = const uint64_t* KYTY_SYSV_ABI (*)(const uint32_t*, uint32_t*, uint64_t*, uint64_t); + t_running_walker = plan.srt_code.get(); + const auto* end = plan.srt_code->getCode()(runtime.user_data.data(), flat.data(), + plan.srt_scratch.data(), runtime.shader_base); + t_running_walker = nullptr; + if (end == nullptr) return false; + if (reads != nullptr) { + for (auto* read = plan.srt_scratch.data() + plan.srt_captures / 8u; read != end; ++read) + reads->emplace_back(*read, sizeof(uint32_t)); } - result = static_cast(wide); return true; } -ResourcePlan::EvaluationContext& SrtWalker::AcquireContext() { - ResourcePlan::EvaluationContext* unused = nullptr; - for (auto& context: m_program.evaluation_contexts) { - if (context.session == m_session && context.active_mask == m_active_mask && - context.read_policy == m_read_policy) return context; - if (context.session == nullptr && unused == nullptr) unused = &context; - } - auto& context = unused != nullptr ? *unused : m_program.evaluation_contexts.emplace_back(); - context.session = m_session; - context.active_mask = m_active_mask; - context.read_policy = m_read_policy; - context.generation += 4; - return context; -} - -void SrtWalker::ResetUnknowns() { - for (auto& context: m_program.evaluation_contexts) { - if (context.session != m_session) continue; - for (auto& entry: context.values) - if (entry.generation == (context.generation | 2u)) entry.generation = 0; - } -} - -float SrtWalker::Float32(uint64_t bits) { - return std::bit_cast(static_cast(bits)); -} - -bool SrtWalker::EvaluateWide(Value value, uint64_t& result) { +bool WalkedValue(const ResourcePlan& plan, Value value, uint32_t& result) { value = value.Resolve(); + uint64_t bits = 0; if (value.IsImmediate()) { - switch (value.GetType()) { - case Type::U1: result = value.U1(); return true; - case Type::U8: result = value.U8(); return true; - case Type::U16: result = value.U16(); return true; - case Type::U32: result = value.U32(); return true; - case Type::U64: result = value.U64(); return true; - case Type::F32: result = std::bit_cast(value.F32Value()); return true; - default: return false; - } - } - if (!m_active_mask.IsEmpty() && value == m_active_mask) { - result = 1u; + if (!ImmediateBits(value, bits)) return false; + result = static_cast(bits); return true; } - auto* inst = value.TryInstruction(); - if (inst == nullptr) { - return false; - } - if (!m_active_mask.IsEmpty() && IsRuntimeSelect(inst->GetOpcode()) && - inst->NumArgs() == 3 && inst->Arg(0).Resolve() == m_active_mask) { - return EvaluateWide(inst->Arg(1), result); - } - const auto index = inst->EvaluationIndex(m_program.evaluation_value_count); - if (index >= m_context.values.size()) { - m_context.values.resize(m_program.evaluation_value_count); - } - if (m_context.values[index].generation == m_context.generation) { - result = m_context.values[index].value; - return true; - } - // Visiting and completed-unknown entries both stop repeated evaluation in this - // context. Only completed unknowns are retried when reachability becomes tighter. - if (m_context.values[index].generation == (m_context.generation | 1u) || - m_context.values[index].generation == (m_context.generation | 2u)) { - return false; - } - m_context.values[index].generation = m_context.generation | 1u; - uint64_t out = 0; - const bool evaluated = EvaluateInst(*inst, out); - // Recursive evaluation may grow the dense memo vector. - auto& memo = m_context.values[index]; - if (!evaluated) { - memo.generation = m_context.generation | 2u; - return false; - } - memo.value = out; - memo.generation = m_context.generation; - result = out; - return true; + const auto* inst = value.TryInstruction(); + if (inst == nullptr) return false; + EXIT_IF(inst->WalkOutput() == UINT32_MAX); + std::memcpy(&bits, Scratch(plan) + OutputOffset(plan) + inst->WalkOutput() * 8u, sizeof(bits)); + result = static_cast(bits); + return bits <= UINT32_MAX; } -bool SrtWalker::Arg(const Inst& inst, size_t index, uint64_t& result) { - return EvaluateWide(inst.Arg(index), result); -} - -bool SrtWalker::EvaluatePhi(const Inst& inst, uint64_t& result) { - if (const auto control = m_program.control_phi_predecessors.find(&inst); - control != m_program.control_phi_predecessors.end()) { - bool found = false; - for (size_t index = 0; index < inst.NumArgs(); ++index) { - if (m_reachable_blocks != nullptr && - !m_reachable_blocks->at(control->second[index])) continue; - uint64_t incoming = 0; - if (!Arg(inst, index, incoming) || (found && incoming != result)) return false; - result = incoming; - found = true; - } - return found; - } - const auto value = ResolveInvariantPhi(m_program, Value(const_cast(&inst))); - return !value.IsEmpty() && EvaluateWide(value, result); -} - -bool SrtWalker::EvaluateExtract(const Inst& inst, uint64_t& result) { - const auto index = inst.Arg(1).Resolve(); - if (!index.IsImmediate() || index.GetType() != Type::U32) { - return false; - } - const auto component = index.U32(); - if (component >= 2u) { - return false; - } - if (inst.GetOpcode() == ValueOpcode::CompositeExtractU64) { - uint64_t packed = 0; - if (!Arg(inst, 0, packed)) { - return false; - } - result = static_cast(packed >> (component * 32u)); - return true; - } - const auto* source = inst.Arg(0).ResolveInstruction(); - if (source == nullptr) { - return false; - } - if (source->GetOpcode() == ValueOpcode::CompositeConstructU32x2) { - return EvaluateWide(source->Arg(component), result); - } - return false; -} - -bool SrtWalker::EvaluateRawRead(const Inst& inst, uint64_t& result) { - const auto flags = inst.Flags(); - if (flags.index >= m_program.memory_info.size()) { - return false; - } - const auto& mem = m_program.memory_info[flags.index]; - const bool vector = inst.GetOpcode() == ValueOpcode::LoadBufferU32; - const auto guard = vector ? inst.Arg(4).Resolve() : Value {}; - if (vector && (guard.IsImmediate() || guard != m_active_mask)) { - uint64_t enabled = 0; - if (!Arg(inst, 4, enabled)) return false; - if (enabled == 0u) { - result = 0u; - return true; - } - } - const auto* handle = inst.Arg(0).ResolveInstruction(); - if (handle == nullptr) { - return false; - } - uint64_t low = 0; - uint64_t high = 0; - uint64_t offset = 0; - if (!Arg(*handle, 0, low) || !Arg(*handle, 1, high) || !Arg(inst, 1, offset)) { - return false; - } - const auto base = ((high << 32u) | static_cast(low)) & AddressMask; - const auto immediate = static_cast(static_cast(mem.offset)); - uint64_t address = 0; - if (inst.GetOpcode() == ValueOpcode::ReadConstBuffer || vector) { - uint64_t records = 0; - uint64_t word3 = 0; - if (handle->NumArgs() != 4u || !Arg(*handle, 2, records) || !Arg(*handle, 3, word3)) { - return false; - } - if (immediate < 0) { - return false; - } - const auto byte_offset = - (static_cast(immediate) & ~uint64_t {3}) + (static_cast(offset) & ~3u); - const auto stride = (static_cast(high) >> 16u) & 0x3fffu; - if (vector) { - // Only the uniform, unswizzled structured DWORD address is evaluated on the host. - if ((high & (1u << 31u)) != 0u || (word3 & ((1u << 23u) | 0xf0000000u)) != 0u) - return false; - if (stride == 0u || records == 0u || ((word3 >> 12u) & 0x7fu) == 0u) { - result = 0u; - return true; - } - } - const auto size = stride == 0u - ? static_cast(static_cast(records)) - : static_cast(stride) * static_cast(records); - if (byte_offset > size || size - byte_offset < sizeof(uint32_t)) { - result = 0u; - return true; - } - address = (base & ~uint64_t {3}) + byte_offset; - } else { - const auto relative = (immediate & ~int64_t {3}) + - static_cast(static_cast(offset) & ~3u); - if (!AddSignedAddress(base & ~uint64_t {3}, relative, address)) { - return false; - } - } - uint32_t word = 0; - const auto reader = vector ? m_runtime.read_specialization_memory : m_runtime.read_memory; - if (reader != nullptr) { - if (!reader(m_runtime.userdata, address, {&word, 1})) { - return false; - } - } else { - if (vector) return false; - std::memcpy(&word, reinterpret_cast(address), sizeof(word)); - } - result = word; - return true; -} - -bool SrtWalker::EvaluateInst(const Inst& inst, uint64_t& result) { - uint64_t a = 0; - uint64_t b = 0; - uint64_t c = 0; - const auto binary = [&]() { return Arg(inst, 0, a) && Arg(inst, 1, b); }; - const auto ternary = [&]() { - return Arg(inst, 0, a) && Arg(inst, 1, b) && Arg(inst, 2, c); - }; - switch (inst.GetOpcode()) { - case ValueOpcode::GetUserData: { - const auto reg = RegIndex(inst.Arg(0).ScalarRegister()); - if (reg < m_program.user_data_base || - reg - m_program.user_data_base >= m_runtime.user_data.size()) { - return false; - } - result = m_runtime.user_data[reg - m_program.user_data_base]; - return true; - } - case ValueOpcode::GetShaderBase: result = m_runtime.shader_base; return true; - case ValueOpcode::Phi: return EvaluatePhi(inst, result); - case ValueOpcode::ReadFirstLane: { - if (!m_active_mask.IsEmpty() && inst.Arg(1).Resolve() == m_active_mask) { - return EvaluateWide(inst.Arg(0), result); - } - SrtWalker clean_active(*this, inst.Arg(1), true); - SrtWalker active(*this, inst.Arg(1), false, &clean_active); - return active.EvaluateWide(inst.Arg(0), result); - } - case ValueOpcode::BitCastU32F32: - case ValueOpcode::BitCastF32U32: return Arg(inst, 0, result); - case ValueOpcode::CompositeExtractU64: - case ValueOpcode::CompositeExtractU32x2: return EvaluateExtract(inst, result); - case ValueOpcode::CompositeConstructU64: - if (!binary()) { - return false; - } - result = static_cast(a) | - (static_cast(static_cast(b)) << 32u); - return true; - case ValueOpcode::LoadAddressU32: - case ValueOpcode::ReadConstBuffer: - case ValueOpcode::LoadBufferU32: - if (IsRawRead(m_program, inst)) { - if (m_clean_evaluator != nullptr && - (inst.Flags().clean != 0u || - (inst.GetOpcode() == ValueOpcode::LoadBufferU32 && - m_clean_evaluator->m_active_mask == m_active_mask))) { - return m_clean_evaluator->EvaluateWide(Value(const_cast(&inst)), result); - } - return EvaluateRawRead(inst, result); - } - break; - case ValueOpcode::IAdd32: - if (binary()) { - result = static_cast(a + b); - return true; - } - return false; - case ValueOpcode::IAdd64: - if (binary()) { - result = a + b; - return true; - } - return false; - case ValueOpcode::ISub32: - if (binary()) { - result = static_cast(a - b); - return true; - } - return false; - case ValueOpcode::ISub64: - if (binary()) { - result = a - b; - return true; - } - return false; - case ValueOpcode::IMul32: - if (binary()) { - result = static_cast(a * b); - return true; - } - return false; - case ValueOpcode::UMulHi: - if (binary()) { - result = (static_cast(static_cast(a)) * static_cast(b)) >> 32u; - return true; - } - return false; - case ValueOpcode::UMin32: - case ValueOpcode::UMax32: - if (binary()) { - result = inst.GetOpcode() == ValueOpcode::UMin32 - ? std::min(static_cast(a), static_cast(b)) - : std::max(static_cast(a), static_cast(b)); - return true; - } - return false; - case ValueOpcode::ConvertF32U32: - if (Arg(inst, 0, a)) { - result = std::bit_cast(static_cast(static_cast(a))); - return true; - } - return false; - case ValueOpcode::ConvertU32F32: - if (Arg(inst, 0, a)) { - // The conversion saturates and maps NaN to zero. - const auto value = Float32(a); - result = std::isnan(value) || value <= 0.0f ? 0u - : value >= 4294967296.0f ? UINT32_MAX - : static_cast(value); - return true; - } - return false; - case ValueOpcode::FPMul32: - if (binary()) { - result = std::bit_cast(Float32(a) * Float32(b)); - return true; - } - return false; - case ValueOpcode::FPTrunc32: - if (Arg(inst, 0, a)) { - result = std::bit_cast(std::trunc(Float32(a))); - return true; - } - return false; - case ValueOpcode::FPRecipIFlag32: - if (Arg(inst, 0, a)) { - const auto exponent = (a >> 23u) & 0xffu; - // Normal positive powers of two have exact normal reciprocals in every FP mode. - if ((a & 0x807fffffu) != 0u || exponent == 0u || exponent >= 254u) return false; - result = (254u - exponent) << 23u; - return true; - } - return false; - case ValueOpcode::FPIsNan32: - if (Arg(inst, 0, a)) { - result = std::isnan(Float32(a)); - return true; - } - return false; - case ValueOpcode::FPOrdLessThanEqual32: - case ValueOpcode::FPOrdGreaterThanEqual32: - if (binary()) { - const auto operand = [&](uint64_t bits) { - if (inst.Flags().flush_input_denorms && - (bits & 0x7fffffffu) < 0x00800000u) { - bits &= 0x80000000u; - } - return Float32(bits); - }; - result = inst.GetOpcode() == ValueOpcode::FPOrdLessThanEqual32 - ? operand(a) <= operand(b) - : operand(a) >= operand(b); - return true; - } - return false; - case ValueOpcode::BitwiseAnd32: - if (binary()) { - result = static_cast(a & b); - return true; - } - return false; - case ValueOpcode::BitwiseAnd64: - if (binary()) { - result = a & b; - return true; - } - return false; - case ValueOpcode::BitwiseOr32: - if (binary()) { - result = static_cast(a | b); - return true; - } - return false; - case ValueOpcode::BitwiseXor32: - if (binary()) { - result = static_cast(a ^ b); - return true; - } - return false; - case ValueOpcode::BitwiseNot32: - if (Arg(inst, 0, a)) { - result = ~static_cast(a); - return true; - } - return false; - case ValueOpcode::ShiftLeftLogical32: - if (binary()) { - result = static_cast(a) << (b & 31u); - return true; - } - return false; - case ValueOpcode::ShiftLeftLogical64: - if (binary()) { - result = a << (b & 63u); - return true; - } - return false; - case ValueOpcode::ShiftRightLogical32: - if (binary()) { - result = static_cast(a) >> (b & 31u); - return true; - } - return false; - case ValueOpcode::ShiftRightLogical64: - if (binary()) { - result = a >> (b & 63u); - return true; - } - return false; - case ValueOpcode::ShiftRightArithmetic32: - if (binary()) { - result = static_cast( - std::bit_cast(static_cast(a)) >> (b & 31u)); - return true; - } - return false; - case ValueOpcode::ShiftRightArithmetic64: - if (binary()) { - result = static_cast(std::bit_cast(a) >> (b & 63u)); - return true; - } - return false; - case ValueOpcode::BitFieldUExtract: - if (ternary()) { - const auto offset = static_cast(b); - const auto width = static_cast(c); - if (offset > 32u || width > 32u - offset) { - return false; - } - const auto mask = width == 32u ? UINT32_MAX - : width == 0u ? 0u - : (uint32_t {1} << width) - 1u; - result = width == 0u ? 0u : (static_cast(a) >> offset) & mask; - return true; - } - return false; - case ValueOpcode::BitFieldSExtract: - if (ternary()) { - const auto offset = static_cast(b); - const auto width = static_cast(c); - if (offset > 32u || width > 32u - offset) { - return false; - } - if (width == 0u) { - result = 0; - return true; - } - const auto mask = width == 32u ? UINT32_MAX : (uint32_t {1} << width) - 1u; - auto bits = (static_cast(a) >> offset) & mask; - if (width < 32u && (bits & (uint32_t {1} << (width - 1u))) != 0u) { - bits |= ~mask; - } - result = bits; - return true; - } - return false; - case ValueOpcode::BitFieldInsert: { - uint64_t d = 0; - if (!ternary() || !Arg(inst, 3, d)) { - return false; - } - const auto offset = static_cast(c); - const auto width = static_cast(d); - if (offset > 32u || width > 32u - offset) { - return false; - } - if (width == 0u) { - result = static_cast(a); - return true; - } - const auto mask = - width == 32u ? UINT32_MAX : ((uint32_t {1} << width) - 1u) << offset; - result = (static_cast(a) & ~mask) | - ((static_cast(b) << offset) & mask); - return true; - } - case ValueOpcode::SelectU32: - case ValueOpcode::SelectU1: - case ValueOpcode::SelectF32: { - auto& predicate = m_clean_evaluator != nullptr ? *m_clean_evaluator : *this; - if (predicate.EvaluateWide(inst.Arg(0), a)) { - return Arg(inst, a != 0u ? 1u : 2u, result); - } - return false; - } - case ValueOpcode::IEqual32: - if (binary()) { - result = static_cast(a) == static_cast(b); - return true; - } - return false; - case ValueOpcode::INotEqual32: - if (binary()) { - result = static_cast(a) != static_cast(b); - return true; - } - return false; - case ValueOpcode::IEqual64: - case ValueOpcode::INotEqual64: - if (!binary()) return false; - result = inst.GetOpcode() == ValueOpcode::IEqual64 ? a == b : a != b; - return true; - case ValueOpcode::ULessThan32: - // No U32 is below zero, including an unknown loop-carried value. - if (!Arg(inst, 1, b)) return false; - if (static_cast(b) == 0u) { - result = 0u; - return true; - } - if (Arg(inst, 0, a)) { - result = static_cast(a) < static_cast(b); - return true; - } - return false; - case ValueOpcode::ULessThanEqual32: - // A zero lower bound accepts every U32, including a varying lane value. - if (!Arg(inst, 0, a)) return false; - if (a == 0u) { - result = 1u; - return true; - } - if (Arg(inst, 1, b)) { - result = static_cast(a) <= static_cast(b); - return true; - } - return false; - case ValueOpcode::UGreaterThan32: - if (binary()) { - result = static_cast(a) > static_cast(b); - return true; - } - return false; - case ValueOpcode::SGreaterThanEqual32: - if (binary()) { - result = std::bit_cast(static_cast(a)) >= - std::bit_cast(static_cast(b)); - return true; - } - return false; - case ValueOpcode::LogicalAnd: { - const bool left = Arg(inst, 0, a); - if (left && a == 0u) { - result = 0u; - return true; - } - if (Arg(inst, 1, b) && (b == 0u || left)) { - result = b != 0u; - return true; - } - if (!left) { - // Only lanes satisfying the left EXEC factor can make the conjunction - // true. Use a separate memo so this assumption cannot escape its RHS. - SrtWalker active(m_clean_evaluator != nullptr ? *m_clean_evaluator : *this, - inst.Arg(0)); - if (active.EvaluateWide(inst.Arg(1), b) && b == 0u) { - result = 0u; - return true; - } - } - return false; - } - case ValueOpcode::LogicalOr: { - const bool left = Arg(inst, 0, a); - if (left && a != 0u) { - result = 1u; - return true; - } - if (!Arg(inst, 1, b) || (b == 0u && !left)) return false; - result = b != 0u; - return true; - } - case ValueOpcode::LogicalXor: - if (binary()) { - result = (a != 0u) != (b != 0u); - return true; - } - return false; - case ValueOpcode::ConditionRef: return Arg(inst, 0, result); - case ValueOpcode::LogicalNot: - if (Arg(inst, 0, a)) { - result = a == 0u; - return true; - } - return false; - case ValueOpcode::UndefU1: - case ValueOpcode::UndefU32: return false; - default: break; - } - return false; -} -bool SrtWalker::EvaluateDescriptor(uint32_t source, DescriptorValue& result) { - if (source >= m_program.descriptor_sources.size()) { - return false; - } - const auto& descriptor = m_program.descriptor_sources[source]; +bool WalkedDescriptor(const ResourcePlan& plan, uint32_t source, DescriptorValue& result) { + if (source >= plan.descriptor_sources.size()) return false; + std::array words {}; + std::memcpy(words.data(), Scratch(plan) + source * DescriptorStride, DescriptorStride); + if (words[8] == 0u) return false; result = {}; - result.dword_count = descriptor.dword_count; - for (uint32_t index = 0; index < descriptor.dword_count; ++index) { - if (!Evaluate(descriptor.dwords[index], result.dwords[index])) { - return false; - } - } + result.dword_count = plan.descriptor_sources[source].dword_count; + std::copy_n(words.begin(), result.dword_count, result.dwords.begin()); return true; } -bool SrtWalker::RefreshFlatBuffer(std::vector& flat) { - if (!m_program.srt_plan_complete) return false; - if (m_program.srt_code && m_runtime.read_memory == nullptr) { - flat.resize(m_program.srt_reads.size()); - m_program.active_sources.clear(); - m_context.values.resize(m_program.evaluation_value_count); - using Walker = bool KYTY_SYSV_ABI (*)(const uint32_t*, uint32_t*, - ResourcePlan::EvaluationContext::Entry*, uint64_t, size_t); - t_running_walker = m_program.srt_code.get(); - const bool walked = m_program.srt_code->getCode()(m_runtime.user_data.data(), flat.data(), - m_context.values.data(), m_context.generation, m_runtime.user_data.size()); - t_running_walker = nullptr; - return walked; - } - const auto refresh = [&](uint32_t slot) { - if (slot >= m_program.srt_reads.size()) return false; - const auto& read = m_program.srt_reads[slot]; - const auto* inst = read.value.ResolveInstruction(); - const bool clean = inst != nullptr && inst->Flags().clean != 0u; - if (clean && (m_clean_evaluator == nullptr || m_runtime.read_specialization_memory == nullptr)) - return false; - auto& evaluator = clean ? *m_clean_evaluator : *this; - return read.flat_offset < flat.size() && evaluator.Evaluate(read.value, flat[read.flat_offset]); - }; - auto& active = m_program.active_sources; - if (m_program.control_flow.empty()) { - active.clear(); - flat.resize(m_program.srt_reads.size()); - for (uint32_t slot = 0; slot < m_program.srt_reads.size(); ++slot) { - if (!refresh(slot)) return false; - } - return true; - } - auto& visited = m_program.visited_blocks; - auto& reachable = m_program.reachable_blocks; - auto& pending = m_program.pending_blocks; - visited.assign(m_program.control_flow.size(), 1u); - ResetUnknowns(); - m_reachable_blocks = &visited; - if (m_clean_evaluator != nullptr) { - m_clean_evaluator->ResetUnknowns(); - m_clean_evaluator->m_reachable_blocks = &visited; - } - for (;;) { - reachable.assign(m_program.control_flow.size(), 0u); - pending.clear(); - pending.push_back(0u); - while (!pending.empty()) { - const auto index = pending.back(); - pending.pop_back(); - if (reachable.at(index)) continue; - reachable[index] = 1u; - const auto& block = m_program.control_flow[index]; - uint32_t condition = 0; - auto& predicate = m_clean_evaluator != nullptr ? *m_clean_evaluator : *this; - if (!block.condition.IsEmpty() && - (m_runtime.read_specialization_memory != nullptr || !m_program.capture_specialization_reads) && - predicate.Evaluate(block.condition, condition)) { - pending.push_back(block.successors[condition != 0u ? 0u : 1u]); - } else { - pending.insert(pending.end(), block.successors.begin(), block.successors.end()); - } - } - if (reachable == visited) break; - visited.swap(reachable); - if (m_program.control_phi_predecessors.empty()) break; - // Only unknown values need another evaluation: successful Phi facts remain - // true as their possible predecessors shrink. Keep uniform-read memos stable. - ResetUnknowns(); - if (m_clean_evaluator != nullptr) m_clean_evaluator->ResetUnknowns(); - } - flat.assign(m_program.srt_reads.size(), 0u); - active.assign(m_program.descriptor_sources.size(), 1u); - for (const auto& block: m_program.control_flow) - for (const auto source: block.sources) active.at(source) = 0u; - for (uint32_t index = 0; index < visited.size(); ++index) { - if (!visited[index]) continue; - const auto& block = m_program.control_flow[index]; - for (const auto source: block.sources) active[source] = 1u; - for (const auto slot: block.srt_reads) - if (!refresh(slot)) return false; - } - return true; +bool WalkedActive(const ResourcePlan& plan, uint32_t source) { + return Scratch(plan)[ActiveOffset(plan) + source] != 0u; } bool ValidateRuntimeValue(const ResourcePlan& program, Value value, RuntimeValueType type) { diff --git a/src/graphics/shader/recompiler/ir/passes/SrtWalker.h b/src/graphics/shader/recompiler/ir/passes/SrtWalker.h index daa21d480..adf02dfe4 100644 --- a/src/graphics/shader/recompiler/ir/passes/SrtWalker.h +++ b/src/graphics/shader/recompiler/ir/passes/SrtWalker.h @@ -8,8 +8,6 @@ namespace Libs::Graphics::ShaderRecompiler::IR { -class Value; - using SrtMemoryReader = bool (*)(void* userdata, uint64_t address, std::span values); // Image descriptors at base + first + k * step, k < count, within size bytes that a sampled @@ -42,7 +40,6 @@ protected: struct SrtRuntime { std::span user_data; uint64_t shader_base = 0; - SrtMemoryReader read_memory = nullptr; void* userdata = nullptr; SrtMemoryReader read_specialization_memory = nullptr; std::span workgroup_counts; @@ -57,20 +54,12 @@ public: SrtRuntime ObservedRuntime(); private: - static bool ReadStrict(void* userdata, uint64_t address, std::span values); - static bool ReadOrdinary(void* userdata, uint64_t address, std::span values); + static bool Read(void* userdata, uint64_t address, std::span values); SrtRuntime m_source; std::vector>& m_ranges; }; -// Retained reads no longer need the guest instruction PC. A clean read evaluates -// its address and bounds through the strict reader as well as the final DWORD. -struct SrtReadFlags { - uint32_t index = 0; - uint32_t clean = 0; -}; - enum class RuntimeValueType { Any, Integer, ImmutableInteger }; bool ValidateRuntimeValue(const ResourcePlan& program, Value value, @@ -78,12 +67,20 @@ bool ValidateRuntimeValue(const ResourcePlan& program, Value value, // Uniform across a compute wave, including values derived from its workgroup ID. // This does not imply that the host can evaluate the value. bool ValidateWaveUniformValue(const ResourcePlan& program, Value value); -void CompileSrtWalker(ResourcePlan& program); +// Compiles the plan's x86 walker. Without values it walks the plan's resources; otherwise it +// only evaluates the values, with every block reachable. +void CompileSrtWalker(ResourcePlan& plan, std::span values = {}); +// Runs the walker. flat receives the scalar reads of reachable blocks and reads, when non-null, +// one DWORD range per guest load. False when a reachable read is unknown. +bool WalkSrt(const ResourcePlan& plan, const SrtRuntime& runtime, std::vector& flat, + std::vector>* reads); +// Results of the last walk; false when unknown. +bool WalkedValue(const ResourcePlan& plan, Value value, uint32_t& result); +bool WalkedDescriptor(const ResourcePlan& plan, uint32_t source, DescriptorValue& result); +bool WalkedActive(const ResourcePlan& plan, uint32_t source); // Length of the guest load at rip if it belongs to the walker running on this thread, else 0. // The exception handler completes such a load as a zero read. uint32_t SrtWalkerLoadLength(uint64_t rip); -// Uses the strict reader for values that affect shader specialization. -SrtRuntime CleanRuntime(SrtRuntime runtime); // Nonnegative affine coefficients: element zero is constant, remaining elements are // caller-selected leaves. Runtime range proofs must additionally exclude U32 wrap. @@ -127,43 +124,6 @@ std::optional> AffineIndex( return left; } -// One memoized evaluation session shared by the entire shader resource refresh. -class SrtWalker { -public: - SrtWalker(const ResourcePlan& program, const SrtRuntime& runtime, - SrtWalker* clean_evaluator = nullptr); - ~SrtWalker(); - SrtWalker(const SrtWalker&) = delete; - SrtWalker& operator=(const SrtWalker&) = delete; - - bool Evaluate(Value value, uint32_t& result); - bool EvaluateDescriptor(uint32_t source, DescriptorValue& result); - // Refreshes reachable scalar reads and active descriptor sources in one walk. - bool RefreshFlatBuffer(std::vector& flat); - -private: - SrtWalker(SrtWalker& parent, Value active_mask, bool clean = false, - SrtWalker* clean_evaluator = nullptr); - ResourcePlan::EvaluationContext& AcquireContext(); - void ResetUnknowns(); - static float Float32(uint64_t bits); - bool EvaluateWide(Value value, uint64_t& result); - bool Arg(const Inst& inst, size_t index, uint64_t& result); - bool EvaluatePhi(const Inst& inst, uint64_t& result); - bool EvaluateExtract(const Inst& inst, uint64_t& result); - bool EvaluateRawRead(const Inst& inst, uint64_t& result); - bool EvaluateInst(const Inst& inst, uint64_t& result); - - const ResourcePlan& m_program; - SrtRuntime m_runtime; - SrtWalker* m_clean_evaluator = nullptr; - Value m_active_mask; - const std::vector* m_reachable_blocks = nullptr; - const SrtWalker* m_session; - uint8_t m_read_policy; - ResourcePlan::EvaluationContext& m_context; -}; - } // namespace Libs::Graphics::ShaderRecompiler::IR #endif /* EMULATOR_INCLUDE_EMULATOR_GRAPHICS_SHADER_RECOMPILER_SRTWALKER_H_ */ diff --git a/tests/ResourceMaterializationTests.cpp b/tests/ResourceMaterializationTests.cpp index 672c7896e..db5d8dfee 100644 --- a/tests/ResourceMaterializationTests.cpp +++ b/tests/ResourceMaterializationTests.cpp @@ -1,5 +1,6 @@ #include "graphics/shader/recompiler/ir/ShaderIR.h" #include "graphics/shader/recompiler/ir/passes/ResourceMaterialization.h" +#include "SrtWalkerTestMemory.h" #include #include @@ -7,12 +8,6 @@ #include #include -#if defined(__linux__) -#include -#include -#include -#endif - namespace { void Check(bool value, const char *text) { @@ -38,6 +33,24 @@ AddValueBlock(Libs::Graphics::ShaderRecompiler::IR::Program &program) { return *result; } +// Lists one extra source per block and walks the plan: each source's activity is its block's reachability. +std::vector ReachedBlocks(Libs::Graphics::ShaderRecompiler::IR::ResourcePlan &plan, + const Libs::Graphics::ShaderRecompiler::IR::SrtRuntime &runtime) { + using namespace Libs::Graphics::ShaderRecompiler::IR; + const auto first = static_cast(plan.descriptor_sources.size()); + for (uint32_t block = 0; block < plan.control_flow.size(); ++block) { + plan.descriptor_sources.emplace_back(); + plan.control_flow[block].sources.push_back(first + block); + } + CompileSrtWalker(plan); + std::vector flat; + Check(WalkSrt(plan, runtime, flat, nullptr), "reachability walk failed"); + std::vector reached; + for (uint32_t block = 0; block < plan.control_flow.size(); ++block) + reached.push_back(WalkedActive(plan, first + block)); + return reached; +} + Libs::Graphics::ShaderRecompiler::IR::Program SrtProgram(uint64_t address) { using namespace Libs::Graphics::ShaderRecompiler::IR; Program program; @@ -155,7 +168,7 @@ Libs::Graphics::ShaderRecompiler::IR::Program MixedSamplerProgram() { return program; } -void TestMappedSrtUsesDirectReaderByDefault() { +void TestWalkerLoadsBypassSpecializationReader() { using namespace Libs::Graphics::ShaderRecompiler::IR; const uint32_t dword = 0x12345678; auto plan = ExtractResourcePlan(SrtProgram(reinterpret_cast(&dword))); @@ -168,10 +181,10 @@ void TestMappedSrtUsesDirectReaderByDefault() { Check(MaterializeResources(plan, runtime, snapshot, specialization), "mapped SRT stage materialization failed"); Check(specialization_reads == 0, - "ordinary SRT read used the specialization reader"); + "walker load used the specialization reader"); Check(snapshot.flattened_srt.size() == 1 && snapshot.flattened_srt[0] == dword, - "cache rematerialization did not use the direct reader by default"); + "walker did not load the mapped SRT DWORD"); } void TestNativeNestedSrtRefresh() { @@ -215,17 +228,6 @@ void TestNativeNestedSrtRefresh() { ResourceSnapshot snapshot; ResourceSpecialization specialization; const SrtRuntime runtime{.user_data = user_data}; - { - SrtWalker walker(plan, runtime); - Check(walker.RefreshFlatBuffer(snapshot.flattened_srt), - "native SRT refresh failed"); - const auto loaded = first[1]; - first[1] ^= 0xffffffffu; - uint32_t value = 0; - Check(walker.Evaluate(plan.srt_reads[0].value, value) && value == loaded, - "descriptor evaluation reloaded a value already copied by the native walker"); - first[1] = loaded; - } for (const auto *data : {&first, &second}) { table = pointer_words(data->data()); Check(MaterializeResources(plan, runtime, snapshot, specialization) && @@ -233,32 +235,20 @@ void TestNativeNestedSrtRefresh() { std::vector{(*data)[1], table[0], table[1], (*data)[2]}, "native SRT refresh lost pointer masking, alignment, or current values"); } -#if defined(__linux__) - struct sigaction action {}, previous {}; - action.sa_sigaction = +[](int, siginfo_t *, void *context) { - auto ®isters = static_cast(context)->uc_mcontext.gregs; - const auto length = SrtWalkerLoadLength(static_cast(registers[REG_RIP])); - if (length == 0) std::abort(); - registers[REG_RAX] = 0; - registers[REG_RIP] += length; - }; - action.sa_flags = SA_SIGINFO; - auto *page = mmap(nullptr, 4096, PROT_NONE, MAP_PRIVATE | MAP_ANONYMOUS, -1, 0); - Check(page != MAP_FAILED && sigaction(SIGSEGV, &action, &previous) == 0 && - MaterializeResources(plan, {.user_data = pointer_words(page)}, snapshot, specialization) && - snapshot.flattened_srt == std::vector(4, 0u), - "faulting walker loads of a table and a null pointer did not read zero"); - sigaction(SIGSEGV, &previous, nullptr); - munmap(page, 4096); -#endif + { + // Unmapped tables and the null pointer they yield read as zero. + FakeGuestMemory unmapped; + const std::array unmapped_table{0x1000u, 0u}; + Check(MaterializeResources(plan, {.user_data = unmapped_table}, snapshot, specialization) && + snapshot.flattened_srt == std::vector(4, 0u), + "faulting walker loads of a table and a null pointer did not read zero"); + } Check(!MaterializeResources(plan, {.user_data = std::span(user_data).first(1)}, snapshot, specialization), "native SRT walk accepted truncated user data"); const std::array fake_user_data{0x1003u, 0xabcd0000u}; uint32_t reads = 0; - const SrtRuntime callback_runtime{ - .user_data = fake_user_data, - .read_memory = +[](void *data, uint64_t address, std::span words) { + FakeGuestMemory fake(+[](void *data, uint64_t address, std::span words) { ++*static_cast(data); switch (address) { case 0x1000u: words[0] = 0x2003u; break; @@ -268,12 +258,11 @@ void TestNativeNestedSrtRefresh() { default: return false; } return true; - }, - .userdata = &reads}; - Check(MaterializeResources(plan, callback_runtime, snapshot, specialization) && reads == 4 && + }, &reads); + Check(MaterializeResources(plan, {.user_data = fake_user_data}, snapshot, specialization) && reads == 4 && snapshot.flattened_srt == std::vector{0x1234u, 0x2003u, 0xabcd0000u, 0x5678u}, - "native-eligible SRT bypassed its callback or repeated nested pointer reads"); + "native SRT walk repeated nested pointer reads"); leaf->SetArg(0, root); leaf->SetArg(1, Value(0u)); program.srt_reads.resize(1); @@ -290,17 +279,6 @@ void TestNativeNestedSrtRefresh() { Check(!MaterializeResources(bounded, {.user_data = base}, snapshot, specialization), "native SRT accepted address underflow or 48-bit overflow"); } - high.Instruction()->SetArg(0, Value(ScalarReg(6))); - Check(ExtractResourcePlan(program).srt_code == nullptr, - "native SRT accepted nonadjacent user-data pointer words"); - high.Instruction()->SetArg(0, Value(ScalarReg(5))); - program.memory_info[0].offset = 0; - program.srt_reads.push_back({pointer_low, 1}); - program.srt_reads.push_back({pointer_high, 2}); - leaf->SetArg(0, emit(ValueOpcode::GetAddressResource, {pointer_low, pointer_high})); - pointer_high.Instruction()->SetArg(1, Value(8u)); - Check(ExtractResourcePlan(program).srt_code == nullptr, - "native SRT accepted nonadjacent nested pointer words"); } void TestIntegerRuntimeValueFollowsSrtReads() { @@ -352,7 +330,7 @@ void TestIntegerRuntimeValueFollowsSrtReads() { "cyclic SRT read-first-lane dependency was accepted"); } -void TestSrtAliasesRetainReadPolicy() { +void TestSrtAliasesShareReads() { using namespace Libs::Graphics::ShaderRecompiler::IR; auto program = SrtProgram(0x1000u); auto &block = *program.blocks[0]; @@ -386,64 +364,27 @@ void TestSrtAliasesRetainReadPolicy() { inst.GetOpcode() != ValueOpcode::GetSrtResource, "retained plan still contains flattened-read aliases"); } - Check(plan.descriptor_sources[0].dwords[0] == plan.srt_reads[0].value && - plan.srt_reads[0].value.Instruction()->Flags().clean == 1u && - plan.srt_reads[1].value.Instruction()->Flags().clean == 0u && - plan.srt_reads[2].value.Instruction()->Flags().clean == 1u, - "alias normalization lost read identity or retained discarded EXEC provenance"); - Check(plan.srt_code == nullptr, - "captured strict reads compiled an unused x86 walker"); - struct Reads { uint32_t strict = 0; uint32_t ordinary = 0; bool dirty = false; } reads; - const SrtRuntime runtime{ - .read_memory = +[](void *data, uint64_t address, std::span words) { - ++static_cast(data)->ordinary; - if (address == 0x2000u) words[0] = 0x2222u; - else if (address == 0x3000u) words[0] = 0x8000u; - else if (address == 0x8000u) words[0] = 0xeeeeu; - else return false; - return true; - }, - .userdata = &reads, - .read_specialization_memory = +[](void *data, uint64_t address, - std::span words) { + Check(plan.descriptor_sources[0].dwords[0] == plan.srt_reads[0].value, + "alias normalization lost read identity"); + struct Reads { uint32_t count = 0; bool dirty = false; } reads; + FakeGuestMemory fake(+[](void *data, uint64_t address, std::span words) { auto &reads = *static_cast(data); - ++reads.strict; - if (reads.dirty) return false; - if (address == 0x3000u) words[0] = 0x1000u; - else if (address == 0x1000u) words[0] = 0x1111u; + ++reads.count; + if (address == 0x1000u) words[0] = 0x1111u; + else if (address == 0x2000u) words[0] = 0x2222u; + else if (address == 0x3000u && !reads.dirty) words[0] = 0x1000u; else return false; return true; - }}; + }, &reads); std::vector flat; - DescriptorValue descriptor; - { - SrtWalker clean(plan, CleanRuntime(runtime)); - SrtWalker walker(plan, runtime, &clean); - Check(walker.RefreshFlatBuffer(flat) && - flat == std::vector{0x1111u, 0x2222u, 0x1000u} && - walker.EvaluateDescriptor(0, descriptor) && descriptor.dwords[0] == 0x1111u && - reads.strict == 2 && reads.ordinary == 1, - "normalized aliases changed nested strict reads or repeated a shared read"); - } - Check(!SrtWalker(plan, runtime).RefreshFlatBuffer(flat), - "strict flat read accepted a missing clean evaluator"); - auto no_reader = runtime; - no_reader.read_specialization_memory = nullptr; - { - SrtWalker clean(plan, CleanRuntime(no_reader)); - Check(!SrtWalker(plan, no_reader, &clean).RefreshFlatBuffer(flat), - "strict flat read accepted a missing strict reader"); - } - Check(SrtWalker(plan, runtime).EvaluateDescriptor(0, descriptor) && - descriptor.dwords[0] == 0xeeeeu && reads.strict == 2 && reads.ordinary == 3, - "direct descriptor evaluation without a clean evaluator changed read domains"); - reads.dirty = true; - { - SrtWalker clean(plan, CleanRuntime(runtime)); - Check(!SrtWalker(plan, runtime, &clean).RefreshFlatBuffer(flat) && - reads.strict == 3 && reads.ordinary == 3, - "dirty strict pointer fell back to an ordinary read"); - } + Check(WalkSrt(plan, {}, flat, nullptr) && flat == std::vector{0x1111u, 0x2222u, 0x1000u} && + reads.count == 3, + "normalized aliases changed nested reads or repeated a shared read"); + // An unreadable pointer reads as zero, and so does the null table it yields. + reads = {.dirty = true}; + Check(WalkSrt(plan, {}, flat, nullptr) && flat == std::vector{0u, 0x2222u, 0u} && + reads.count == 3, + "unreadable pointer did not read as zero"); } void TestUniformVectorDescriptorRead() { @@ -477,59 +418,66 @@ void TestUniformVectorDescriptorRead() { program.descriptor_sources.push_back(source); program.info.buffers.push_back({.source = 0, .written = true}); Check(ValidateRuntimeValue(program, Value(&count)), "uniform DWORD count was rejected"); - struct Reads { uint32_t value = 72; uint32_t strict = 0; uint32_t ordinary = 0; bool clean = true; } reads; - const SrtRuntime runtime{ - .read_memory = [](void *data, uint64_t, std::span words) { - ++static_cast(data)->ordinary; - words[0] = 999; - return true; - }, - .userdata = &reads, - .read_specialization_memory = [](void *data, uint64_t address, std::span words) { + struct Reads { uint32_t value = 72; uint32_t count = 0; bool readable = true; } reads; + FakeGuestMemory fake([](void *data, uint64_t address, std::span words) { auto &reads = *static_cast(data); - ++reads.strict; - if (!reads.clean || address != 0x1000u || words.size() != 1) return false; + ++reads.count; + if (!reads.readable || address != 0x1000u || words.size() != 1) return false; words[0] = reads.value; return true; - }}; + }, &reads); + const SrtRuntime runtime; ResourceSnapshot snapshot; ResourceSpecialization specialization; for (const bool written : {false, true}) { program.info.buffers[0].written = written; auto plan = ExtractResourcePlan(program); - Check(plan.control_flow.empty() && plan.requires_specialization_memory && - plan.capture_specialization_reads, + Check(plan.control_flow.empty() && plan.capture_specialization_reads, "vector descriptor read depended on incidental control-flow capture"); reads = {}; Check(MaterializeResources(plan, runtime, snapshot, specialization) && - snapshot.buffers[0].dwords[2] == 145 && reads.strict == 1 && reads.ordinary == 0 && + snapshot.buffers[0].dwords[2] == 145 && reads.count == 1 && snapshot.specialization_reads == std::vector>{{0x1000u, 4u}}, "vector descriptor input was not read and captured exactly once"); - reads.clean = false; - reads.strict = 0; - Check(!MaterializeResources(plan, runtime, snapshot, specialization) && - reads.strict == 1 && reads.ordinary == 0, - "dirty vector descriptor input fell back to an ordinary memory read"); + // An unreadable input reads as zero, so the count selects 0 + 1. + reads = {.readable = false}; + Check(MaterializeResources(plan, runtime, snapshot, specialization) && + snapshot.buffers[0].dwords[2] == 1 && reads.count == 1, + "unreadable vector descriptor input did not read as zero"); + } + // A read guarded by its EXEC mask depends on that mask, so it cannot reuse the unmasked read. + auto &guarded = block.AppendNewInst(ValueOpcode::LoadBufferU32, + {Value(&handle), Value(0u), Value(0u), Value(0u), Value(&active)}); + guarded.SetFlags(MemoryFlags{.index = 0}); + auto &guarded_lane = block.AppendNewInst(ValueOpcode::ReadFirstLane, {Value(&guarded), Value(&active)}); + program.descriptor_sources.push_back({.dwords = {Value(&guarded_lane)}, .dword_count = 1}); + { + auto plan = ExtractResourcePlan(program); + uint32_t value = 0; + reads = {}; + Check(EvaluateSrtValue(plan, runtime, plan.descriptor_sources.back().dwords[0], value) && + value == 72u && reads.count == 1, + "EXEC-guarded vector read lost its mask"); } count.SetArg(4, Value(false)); auto &inactive = block.AppendNewInst(ValueOpcode::ReadFirstLane, {Value(&count), Value(false)}); program.descriptor_sources.push_back({.dwords = {Value(&inactive)}, .dword_count = 1}); - reads.strict = 0; + reads.count = 0; uint32_t result = 99; { - const auto plan = ExtractResourcePlan(program); - Check(SrtWalker(plan, runtime).Evaluate(plan.descriptor_sources.back().dwords[0], result) && - result == 0 && reads.strict == 0 && reads.ordinary == 0, + auto plan = ExtractResourcePlan(program); + Check(EvaluateSrtValue(plan, runtime, plan.descriptor_sources.back().dwords[0], result) && + result == 0 && reads.count == 0, "literal false EXEC read vector memory"); } count.SetArg(4, Value(true)); handle.SetArg(3, Value(0x204u)); program.descriptor_sources.back().dwords[0] = Value(&count); { - const auto plan = ExtractResourcePlan(program); - Check(SrtWalker(plan, runtime).Evaluate(plan.descriptor_sources.back().dwords[0], result) && - result == 0 && reads.strict == 0 && reads.ordinary == 0, + auto plan = ExtractResourcePlan(program); + Check(EvaluateSrtValue(plan, runtime, plan.descriptor_sources.back().dwords[0], result) && + result == 0 && reads.count == 0, "invalid vector buffer format read memory"); } handle.SetArg(3, Value(0x16204u)); @@ -547,7 +495,7 @@ void TestExactReciprocalDescriptorArithmetic() { std::bit_cast(253u << 23u)}) { auto &reciprocal = block.AppendNewInst(ValueOpcode::FPRecipIFlag32, {Value::F32(divisor)}); uint32_t result = 0; - Check(SrtWalker(program, {}).Evaluate(Value(&reciprocal), result) && + Check(EvaluateSrtValue(program, {}, Value(&reciprocal), result) && result == std::bit_cast(1.f / divisor), "power-of-two reciprocal was not exact"); } @@ -556,7 +504,7 @@ void TestExactReciprocalDescriptorArithmetic() { std::bit_cast(254u << 23u)}) { auto &reciprocal = block.AppendNewInst(ValueOpcode::FPRecipIFlag32, {Value::F32(divisor)}); uint32_t result = 0; - Check(!SrtWalker(program, {}).Evaluate(Value(&reciprocal), result), + Check(!EvaluateSrtValue(program, {}, Value(&reciprocal), result), "unsupported reciprocal rounding or exceptional input was accepted"); } } @@ -593,7 +541,7 @@ void TestWrittenDescriptorPredicateReads(bool memory_condition) { source.dword_count = 4; program.descriptor_sources.push_back(source); program.info.buffers.push_back({.source = 0, .written = true}); - // Only memory-derived host decisions need clean, disjoint descriptor reads. + // Only memory-derived host decisions capture their descriptor reads. auto &condition = block.AppendNewInst(ValueOpcode::IEqual32, {memory_condition ? Value(&read) : Value(&offset), Value(memory_condition ? 0x8000u : 4u)}); @@ -619,35 +567,25 @@ void TestWrittenDescriptorPredicateReads(bool memory_condition) { auto plan = ExtractResourcePlan(program); Check(plan.capture_specialization_reads == memory_condition && !plan.control_flow.empty(), "conditional writable descriptor used the wrong memory dependency policy"); - struct Reads { uint32_t ordinary = 0; uint32_t strict = 0; bool clean = false; } reads; + struct Reads { uint32_t count = 0; bool readable = false; } reads; const std::array user_data{4u}; - const SrtRuntime runtime{ - .user_data = user_data, - .read_memory = +[](void *data, uint64_t, std::span words) { - ++static_cast(data)->ordinary; - words[0] = 0x8000u; - return true; - }, - .userdata = &reads, - .read_specialization_memory = +[](void *data, uint64_t address, std::span words) { + FakeGuestMemory fake(+[](void *data, uint64_t address, std::span words) { auto &reads = *static_cast(data); - ++reads.strict; - if (!reads.clean || address != 0x1004u) return false; + ++reads.count; + if (!reads.readable || address != 0x1004u) return false; words[0] = 0x8000u; return true; - }}; + }, &reads); ResourceSnapshot snapshot; ResourceSpecialization specialization; - for (const bool clean : {false, true}) { - reads = {.clean = clean}; - // A failed predicate explores both edges; its dirty read also rejects the writable descriptor. - const bool expected = !memory_condition || clean; - Check(MaterializeResources(plan, runtime, snapshot, specialization) == expected && - reads.ordinary == (memory_condition ? 0u : 1u) && - reads.strict == (memory_condition ? 1u : 0u), - "writable descriptor changed reader policy or bypassed a dirty memory predicate"); - if (!expected) continue; - Check(snapshot.buffers[0].dwords[0] == 0x8000u && + for (const bool readable : {false, true}) { + reads = {.readable = readable}; + // The predicate and the descriptor share one read. An unreadable one is zero, which + // makes a memory predicate skip the store block and leave its descriptor inactive. + Check(MaterializeResources(plan, {.user_data = user_data}, snapshot, specialization) && + reads.count == 1, + "writable descriptor repeated or skipped its shared read"); + Check(snapshot.buffers[0].dwords[0] == (readable ? 0x8000u : 0u) && snapshot.specialization_reads == (memory_condition ? std::vector>{{0x1004u, 4u}} : std::vector>{}), @@ -704,16 +642,13 @@ void TestSharedControlPredicates() { "shared predicates expanded into trees or retained varying dependencies"); for (const uint32_t flag : {0u, 1u}) { const std::array user_data{flag}; - SrtWalker walker(plan, {.user_data = user_data}); uint32_t result = 99; const bool conjunction = operation == ValueOpcode::LogicalAnd; const bool known = conjunction ? flag == 0 : flag != 0; - Check(walker.Evaluate(plan.control_flow[0].condition, result) == known && + Check(EvaluateSrtValue(plan, {.user_data = user_data}, plan.control_flow[0].condition, result) == known && (!known || result == uint32_t{conjunction}), "shared predicate changed its uniform factor or cached runtime values"); - std::vector flat; - Check(walker.RefreshFlatBuffer(flat) && - plan.visited_blocks == (known + Check(ReachedBlocks(plan, {.user_data = user_data}) == (known ? (conjunction ? std::vector{1, 1, 0, 1} : std::vector{1, 0, 1, 0}) : std::vector{1, 1, 1, 1}), @@ -772,34 +707,27 @@ void TestNestedExecPredicatesReuseReads() { {buffer, Value(0u), Value(0u), Value(0u), Value(true)}) .SetFlags(MemoryFlags{.index = 1}); auto plan = ExtractResourcePlan(program); - struct Reads { uint32_t value = 0; bool clean = true; uint32_t strict = 0, ordinary = 0; } reads; - const SrtRuntime runtime{ - .read_memory = +[](void *data, uint64_t, std::span) { - ++static_cast(data)->ordinary; - return false; - }, - .userdata = &reads, - .read_specialization_memory = +[](void *data, uint64_t address, std::span words) { + struct Reads { uint32_t value = 0; bool readable = true; uint32_t count = 0; } reads; + FakeGuestMemory fake(+[](void *data, uint64_t address, std::span words) { auto &reads = *static_cast(data); - ++reads.strict; - Check(reads.strict <= depth + 1, - "shared EXEC predicates repeated scalar reads beyond their distinct masks"); - if (!reads.clean || address != 0x1000u || words.size() != 1) return false; + ++reads.count; + if (!reads.readable || address != 0x1000u || words.size() != 1) return false; words[0] = reads.value; return true; - }}; + }, &reads); ResourceSnapshot snapshot; ResourceSpecialization specialization; - for (const auto state : {Reads{64u, true}, Reads{0u, true}, Reads{0u, false}, Reads{64u, true}}) { + for (const auto state : {Reads{64u, true}, Reads{0u, true}, Reads{64u, false}, Reads{64u, true}}) { reads = state; - Check(MaterializeResources(plan, runtime, snapshot, specialization), + Check(MaterializeResources(plan, {}, snapshot, specialization), "nested EXEC predicates rejected a valid resource snapshot"); - const bool inactive = reads.clean && reads.value == 0; + // An unreadable count reads as zero. + const bool inactive = !reads.readable || reads.value == 0; Check(snapshot.buffers[0].dwords[0] == (inactive ? 0u : 0x8000u), "nested EXEC predicates lost an unknown branch or retained stale runtime facts"); - Check(reads.strict > 0 && reads.ordinary == 0 && - snapshot.specialization_reads.size() <= depth + 1, - "shared EXEC predicates repeated scalar reads beyond their distinct masks"); + Check(reads.count == 1 && snapshot.specialization_reads == + std::vector>{{0x1000u, 4u}}, + "EXEC contexts repeated a mask-independent scalar read"); } } @@ -845,16 +773,8 @@ void TestFiniteImageRefreshReusesScalarReads() { struct Reads { std::array words{0x100u, 0x200u, 0x200u}; std::array counts{}; - uint32_t ordinary = 0; } reads; - const SrtRuntime runtime{ - .read_memory = +[](void *data, uint64_t, std::span) { - ++static_cast(data)->ordinary; - return false; - }, - .userdata = &reads, - .read_specialization_memory = +[](void *data, uint64_t address, - std::span words) { + FakeGuestMemory fake(+[](void *data, uint64_t address, std::span words) { if (words.size() != 1 || address < 0x1000u || address >= 0x100cu || (address & 3u) != 0) return false; auto &reads = *static_cast(data); @@ -862,7 +782,8 @@ void TestFiniteImageRefreshReusesScalarReads() { ++reads.counts[index]; words[0] = reads.words[index]; return true; - }}; + }, &reads); + const SrtRuntime runtime; ResourceSnapshot snapshot; ResourceSpecialization specialization; const auto capacities = [&] { @@ -881,15 +802,15 @@ void TestFiniteImageRefreshReusesScalarReads() { }; Check(MaterializeResources(plan, runtime, snapshot, specialization), "finite image materialization failed"); - Check(reads.ordinary == 0 && reads.counts == std::array{1, 1, 1} && + Check(reads.counts == std::array{1, 1, 1} && snapshot.specialization_reads.size() == 3, - "finite image candidates repeated scalar reads or bypassed clean provenance"); + "finite image candidates repeated or skipped scalar reads"); check_mapping({0, 1, 1, 1}); const auto warm_capacities = capacities(); reads.words = {0x300u, 0x300u, 0x400u}; Check(MaterializeResources(plan, runtime, snapshot, specialization), "finite image refresh failed after descriptor changes"); - Check(reads.ordinary == 0 && reads.counts == std::array{2, 2, 2} && + Check(reads.counts == std::array{2, 2, 2} && snapshot.specialization_reads.size() == 3 && snapshot.images[0].dwords[0] == 0x300u && snapshot.images[1].dwords[0] == 0x400u, "finite image refresh retained old scalar values or repeated reads"); @@ -992,36 +913,35 @@ void TestEmptyTraversalSkipsUnmappedDescriptor(bool independent_mask, bool count emit(use, ValueOpcode::LoadBufferU32, {buffer, Value(0u), Value(0u), Value(0u), Value(true)}) .Instruction()->SetFlags(MemoryFlags{.index = buffer_index}); auto plan = ExtractResourcePlan(program); - struct Reads { uint32_t count = 0; bool clean = true; uint32_t descriptors = 0; } reads; - const SrtRuntime runtime{ - .read_memory = +[](void *data, uint64_t address, std::span words) { + struct Reads { uint32_t count = 0; bool readable = true; uint32_t descriptors = 0; } reads; + FakeGuestMemory fake(+[](void *data, uint64_t address, std::span words) { constexpr std::array descriptor{0x8000u, 1u << 16u, 16u, (3u << 28u) | 0x4dfacu}; + auto &reads = *static_cast(data); + if (address == 0x1000u && reads.readable) { + words[0] = reads.count; + return true; + } if (address < 0x2000u || address >= 0x2010u || address % 4u || words.size() != 1) return false; - ++static_cast(data)->descriptors; + ++reads.descriptors; words[0] = descriptor[(address - 0x2000u) / 4u]; return true; - }, - .userdata = &reads, - .read_specialization_memory = +[](void *data, uint64_t address, std::span words) { - const auto &reads = *static_cast(data); - if (!reads.clean || address != 0x1000u || words.size() != 1) return false; - words[0] = reads.count; - return true; - }}; + }, &reads); + const SrtRuntime runtime; ResourceSnapshot snapshot; ResourceSpecialization specialization; - for (const auto state : {Reads{0u, true}, Reads{4u, true}, Reads{0u, false}, Reads{0u, true}}) { + for (const auto state : {Reads{0u, true}, Reads{4u, true}, Reads{4u, false}, Reads{0u, true}}) { reads = state; if (!counted_loop) { uint32_t condition = 99; - Check(!SrtWalker(plan, runtime).Evaluate(plan.control_flow[3].condition, condition), + Check(!EvaluateSrtValue(plan, runtime, plan.control_flow[3].condition, condition), "direct merge evaluation used missing or previous runtime reachability"); } Check(MaterializeResources(plan, runtime, snapshot, specialization), "partial traversal proof rejected a valid resource snapshot"); - const bool inactive = !independent_mask && reads.clean && reads.count == 0u; + // An unreadable count reads as zero. + const bool inactive = !independent_mask && (!reads.readable || reads.count == 0u); Check(snapshot.buffers[0].dwords[0] == (inactive ? 0u : 0x8000u), "empty traversal kept its descriptor or an uncertain traversal lost it"); Check(specialization.buffers[0].zero_stride_oob == inactive, @@ -1029,7 +949,7 @@ void TestEmptyTraversalSkipsUnmappedDescriptor(bool independent_mask, bool count Check(reads.descriptors == (inactive ? 0u : 4u), "inactive traversal read its descriptor or an uncertain traversal skipped it"); if (inactive) { - Check(!snapshot.specialization_reads.empty(), "empty traversal lost its clean scalar read"); + Check(!snapshot.specialization_reads.empty(), "empty traversal lost its captured scalar read"); for (const auto &read : snapshot.specialization_reads) Check(read == std::pair{0x1000u, 4u}, "empty traversal proof captured an unrelated memory range"); @@ -1075,16 +995,89 @@ void TestCyclicControlPredicatesRemainUnknown() { branch(left, a, right, exit); branch(right, b, left, exit); auto plan = ExtractResourcePlan(program); - SrtWalker walker(plan, {}); uint32_t value = 99; - Check(!walker.Evaluate(plan.control_flow[1].condition, value) && - !walker.Evaluate(plan.control_flow[2].condition, value), + Check(!EvaluateSrtValue(plan, {}, plan.control_flow[1].condition, value) && + !EvaluateSrtValue(plan, {}, plan.control_flow[2].condition, value), "cyclic varying control acquired an unsupported uniform value"); - std::vector flat; - Check(walker.RefreshFlatBuffer(flat) && plan.visited_blocks == std::vector({1, 1, 1, 1}), + Check(ReachedBlocks(plan, {}) == std::vector({1, 1, 1, 1}), "cyclic control lost a possible branch during reachability"); } +void TestBackEdgeReachesPhiPredecessor() { + using namespace Libs::Graphics::ShaderRecompiler::IR; + using Libs::Graphics::ShaderRecompiler::CFG::TerminatorKind; + Program program; + program.stage = Libs::Graphics::ShaderType::Compute; + program.user_data_count = 3; + program.srt_plan_complete = program.resource_tracking_complete = true; + for (uint32_t i = 0; i < 4; ++i) AddValueBlock(program); + auto &entry = *program.blocks[0]; + auto &header = *program.blocks[1]; + auto &body = *program.blocks[2]; + auto &exit = *program.blocks[3]; + const auto branch = [](Block &block, Block &target, Value condition = {}, Block *other = nullptr) { + block.terminator.kind = other ? TerminatorKind::ConditionalBranch : TerminatorKind::Branch; + block.terminator.true_block = ⌖ + block.terminator.false_block = other; + block.condition = condition; + block.AddBranch(&target); + if (other) block.AddBranch(other); + }; + // header: p = Phi(entry: 0, body: 1); table[1] == p enters body, which returns to header. The + // first pass evaluates p and loads table[1] before body is reached; the walk must revisit both, + // capture the load once and reach exit, which loads table[0]. + program.memory_info.push_back({.kind = ResourceKind::ScalarAddress}); + const auto table = Value(&entry.AppendNewInst(ValueOpcode::GetAddressResource, + {Value(&entry.AppendNewInst(ValueOpcode::GetUserData, {Value(ScalarReg(0))})), + Value(&entry.AppendNewInst(ValueOpcode::GetUserData, {Value(ScalarReg(1))}))})); + const auto load = [&](Block &block, uint32_t offset) -> Inst & { + auto &read = block.AppendNewInst(ValueOpcode::LoadAddressU32, {table, Value(offset), Value(0u), Value(true)}); + read.SetFlags(MemoryFlags{.index = 0}); + return read; + }; + auto &p = header.AppendNewInst(ValueOpcode::Phi); + p.SetFlags(Type::U32); + p.AddPhiOperand(&entry, Value(0u)); + p.AddPhiOperand(&body, Value(1u)); + branch(entry, header); + branch(header, body, Value(&header.AppendNewInst(ValueOpcode::IEqual32, {Value(&load(header, 4u)), Value(&p)})), + &exit); + branch(body, header); + exit.terminator.kind = TerminatorKind::Return; + program.srt_reads.push_back({Value(&load(exit, 0u)), 0}); + exit.AppendNewInst(ValueOpcode::ReadConst, + {Value(&exit.AppendNewInst(ValueOpcode::GetSrtResource)), Value(0u)}); + auto plan = ExtractResourcePlan(program); + Check(plan.control_flow.size() == 4 && plan.control_phi_predecessors.size() == 1, + "back-edge plan lost its control flow or Phi"); + plan.capture_specialization_reads = true; + const std::array user_data{0x1000u, 0u}; + ResourceSnapshot snapshot; + ResourceSpecialization specialization; + { + // The table is unmapped: its loads read zero instead of faulting the walk. + FakeGuestMemory unmapped; + Check(MaterializeResources(plan, {.user_data = user_data}, snapshot, specialization) && + snapshot.flattened_srt == std::vector{0u}, + "faulting load in a reachable block did not read zero"); + } + for (uint32_t selected : {0u, 1u}) { + FakeGuestMemory fake(+[](void *userdata, uint64_t address, std::span words) { + words[0] = address == 0x1000u ? 0x5678u : *static_cast(userdata); + return address == 0x1000u || address == 0x1004u; + }, &selected); + Check(MaterializeResources(plan, {.user_data = user_data}, snapshot, specialization) && + snapshot.flattened_srt == std::vector{0x5678u}, + "back edge to an evaluated Phi left its exit unreachable"); + Check(snapshot.specialization_reads == + std::vector>{{0x1004u, 4u}, {0x1000u, 4u}}, + "Phi restart kept the loads of the abandoned pass"); + Check(ReachedBlocks(plan, {.user_data = user_data}) == + (selected == 0u ? std::vector{1, 1, 1, 1} : std::vector{1, 1, 0, 1}), + "Phi restart followed a rejected edge or missed a reachable one"); + } +} + void TestEmptyScalarBufferSpecialization() { using namespace Libs::Graphics::ShaderRecompiler::IR; struct Fixture { @@ -1144,8 +1137,9 @@ void TestEmptyScalarBufferSpecialization() { auto plan = ExtractResourcePlan(fixture.program); std::array descriptor{0x1000u, 8u << 16u, 0u, 3u << 28u}; uint32_t reads = 0u; - const SrtRuntime runtime{.user_data = descriptor, .read_memory = RejectSpecializationRead, - .userdata = &reads, .read_specialization_memory = RejectSpecializationRead}; + const SrtRuntime runtime{.user_data = descriptor, .userdata = &reads, + .read_specialization_memory = RejectSpecializationRead}; + FakeGuestMemory fake(RejectSpecializationRead, &reads); ResourceSnapshot snapshot; ResourceSpecialization specialization, empty, populated; for (const uint32_t records : {0u, 1u, 0x20000000u, 0u}) { @@ -1377,10 +1371,10 @@ void DbgExit(int) { std::abort(); } } // namespace Common int main() { - TestMappedSrtUsesDirectReaderByDefault(); + TestWalkerLoadsBypassSpecializationReader(); TestNativeNestedSrtRefresh(); TestIntegerRuntimeValueFollowsSrtReads(); - TestSrtAliasesRetainReadPolicy(); + TestSrtAliasesShareReads(); TestUniformVectorDescriptorRead(); TestExactReciprocalDescriptorArithmetic(); TestUnbasedFlatCacheHitMaterializes(); @@ -1394,6 +1388,7 @@ int main() { TestEmptyTraversalSkipsUnmappedDescriptor(independent_mask); TestEmptyTraversalSkipsUnmappedDescriptor(false, true); TestCyclicControlPredicatesRemainUnknown(); + TestBackEdgeReachesPhiPredecessor(); TestEmptyScalarBufferSpecialization(); TestMixedSamplerVariantsShareRuntimeDescriptor(); TestSignedStorageSpecialization(); diff --git a/tests/ResourceTrackingTests.cpp b/tests/ResourceTrackingTests.cpp index fd34df80c..9a3d96469 100644 --- a/tests/ResourceTrackingTests.cpp +++ b/tests/ResourceTrackingTests.cpp @@ -5,7 +5,7 @@ #include "graphics/shader/recompiler/ir/passes/ResourceMaterialization.h" #include "graphics/shader/recompiler/ir/passes/ResourceTracking.h" #include "graphics/shader/recompiler/ir/passes/ShaderInfoCollection.h" -#include "graphics/shader/recompiler/ir/passes/SrtWalker.h" +#include "SrtWalkerTestMemory.h" #include #include @@ -212,6 +212,12 @@ bool ReadLinearTestMemory(void *userdata, uint64_t address, std::span return true; } +// Serves walker loads from the memory a test runtime currently points at. +bool ReadRuntimeMemory(void *userdata, uint64_t address, std::span values) { + const auto &runtime = *static_cast(userdata); + return runtime.read_specialization_memory(runtime.userdata, address, values); +} + std::unique_ptr MakeIndirectImageFixture(bool malformed, uint32_t table_stride = 32, uint32_t table_offset = 0, uint32_t table_shader_offset = 0, bool nested = false, @@ -424,9 +430,8 @@ void TestInvariantIndirectImageMaterialization(bool nested) { const auto handle = std::ranges::find_if(*wrap->block, [](const Inst &inst) { return inst.GetOpcode() == ValueOpcode::GetImageResource; }); - SrtWalker walker(wrap->program, runtime); uint32_t offset = 0; - Check(walker.Evaluate(handle->Arg(0), offset) && offset == expected, + Check(EvaluateSrtValue(wrap->program, runtime, handle->Arg(0), offset) && offset == expected, "GPU descriptor offset projection lost U32 wrap or scalar immediate carry"); } auto malformed = MakeIndirectImageFixture(true); @@ -624,6 +629,7 @@ void TestBoundedImageViewEligibility() { plan.descriptor_sources.push_back(source); } plan.descriptor_sources[image_source].indirect_descriptor->sources = std::move(selected); + CompileSrtWalker(plan); Check(!MaterializeResources(plan, runtime, snapshot, specialization), "explicitly selected incompatible descriptor was silently normalized to null"); } @@ -873,9 +879,8 @@ void TestWaterfallImageTable() { std::array{UINT32_MAX, 0x5204u, UINT32_MAX}, std::array{0u, 0u, UINT32_MAX}}) { data[4] = index; data[3] = format; - SrtWalker walker(fixture->program, runtime); uint32_t offset = 0; - Check(walker.Evaluate(handle->Arg(0), offset) && offset == expected, + Check(EvaluateSrtValue(fixture->program, runtime, handle->Arg(0), offset) && offset == expected, "VMEM table projection lost record bounds, format disable or multiplication carry"); } auto disabled = make(true, false); @@ -1113,7 +1118,7 @@ void TestNestedPointerImageTable() { memory.fail_address = 0x1800u; *at(table) = 0u; Check(!MaterializeResources(plan, runtime, snapshot, specialization), - "unreadable null pointer row bypassed the strict pointee reader"); + "unreadable null pointer row bypassed the specialization reader"); *at(table) = objects; auto capped = make(4u); @@ -1381,7 +1386,7 @@ void TestIndexedPointerImageTable() { *at(table) = 0u; *at(table + 4u) = 0u; Check(!MaterializeResources(plan, runtime, snapshot, specialization), - "explicit null pointer bypassed the strict child reader"); + "explicit null pointer bypassed the specialization reader"); auto reversed = make(Case::ReverseSigned); reversed->PlanAndTrack(); for (const auto variant : {Case::MissingParent, Case::WrongParent, Case::MissingChild, @@ -1398,8 +1403,9 @@ void TestIndexedPointerImageTable() { const auto counted_plan = ExtractResourcePlan(counted->program); constexpr uint32_t count_address = 0x1800u; std::array counted_data{table, 1u | (stride << 16u), 2u, 0x5204u, count_address, 1u}; - const SrtRuntime counted_runtime{.user_data = counted_data, .read_memory = ReadLinearTestMemory, .userdata = &memory, + const SrtRuntime counted_runtime{.user_data = counted_data, .userdata = &memory, .read_specialization_memory = ReadLinearTestMemory}; + FakeGuestMemory fake(ReadLinearTestMemory, &memory); *at(table) = objects; *at(table + 4u) = 0xbeef0001u; *at(count_address) = *at(count_address + 4u) = 2u; @@ -1431,9 +1437,11 @@ void TestIndexedPointerImageTable() { Check(!MaterializeResources(counted_plan, counted_runtime, snapshot, specialization) && memory.watched_reads == 0u, "runtime parent count exceeding V# records was clamped or read candidates"); *at(count_address) = 2u; + // An unreadable parent count reads as zero: an empty domain without candidate reads. memory.fail_address = high + count_address; - Check(!MaterializeResources(counted_plan, counted_runtime, snapshot, specialization) && memory.watched_reads == 0u, - "unreadable parent count was ignored before candidate reads"); + Check(MaterializeResources(counted_plan, counted_runtime, snapshot, specialization) && + snapshot.images.size() == 1u && memory.watched_reads == 0u, + "unreadable parent count did not read as an empty domain"); memory.fail_address = UINT64_MAX; for (const auto variant : {Case::MissingCount, Case::DifferentCount, Case::UnprovenMask}) { auto unproved = make(variant); @@ -1780,9 +1788,9 @@ void TestGuardedDirectImageTable() { }; fill_table(memory, table); std::array user_data{0x1000u, 0u}; - SrtRuntime runtime{.user_data = user_data, .read_memory = ReadLinearTestMemory, - .userdata = &memory, + SrtRuntime runtime{.user_data = user_data, .userdata = &memory, .read_specialization_memory = ReadLinearTestMemory}; + FakeGuestMemory fake(ReadRuntimeMemory, &runtime); ResourceSnapshot snapshot; ResourceSpecialization specialization; auto packed = make_plan(Guard::Plain, 0u, true); @@ -2352,10 +2360,9 @@ void TestBoundedComputeImageLoop() { (static_cast(Libs::Graphics::Prospero::ImageType::kColor2D) << 28u); } std::array user_data{0x1800u, 0u, 2u}; - SrtRuntime runtime{.user_data = user_data, - .read_memory = ReadLinearTestMemory, - .userdata = &memory, + SrtRuntime runtime{.user_data = user_data, .userdata = &memory, .read_specialization_memory = ReadLinearTestMemory}; + FakeGuestMemory fake(ReadLinearTestMemory, &memory); ResourceSnapshot snapshot; ResourceSpecialization specialization; for (const auto post : {PostTest::Signed, PostTest::Unsigned}) { @@ -2668,8 +2675,8 @@ void TestDescendingImageRange() { std::array user_data {0x1000u, 0u, 0u, UINT32_MAX}; ResourceSnapshot snapshot; ResourceSpecialization specialization; - const SrtRuntime runtime {.user_data = user_data, .read_memory = ReadLinearTestMemory, - .userdata = &memory, .read_specialization_memory = ReadLinearTestMemory}; + const SrtRuntime runtime {.user_data = user_data, .userdata = &memory, + .read_specialization_memory = ReadLinearTestMemory}; for (const auto count : {0u, 2u, 0u}) { user_data[2] = count; Check(MaterializeResources(plan, runtime, snapshot, specialization) && snapshot.images.size() == 2u, @@ -3176,7 +3183,7 @@ void TestUniformizedMaterialImageKeys() { make_plan(Variant::FirstLane); make_plan(Variant::FirstLaneDirect); plan = make_plan(Variant::FirstLaneAndNot); - Check(plan.requires_specialization_memory && + Check(plan.capture_specialization_reads && plan.descriptor_sources[plan.info.images[0].source] .indirect_descriptor->selector_mask.Resolve().TryInstruction() != nullptr, "uniformized image mask did not survive extraction"); @@ -3207,10 +3214,8 @@ void TestUniformizedMaterialImageKeys() { user_data[9] = 16u << 16u; user_data[10] = 1u; memory.fail_address = base + 0xc00u + 3u * 0x90u; - const SrtRuntime runtime{ - .user_data = user_data, .read_memory = ReadLinearTestMemory, - .userdata = &memory, - .read_specialization_memory = ReadLinearTestMemory}; + const SrtRuntime runtime{.user_data = user_data, .userdata = &memory, + .read_specialization_memory = ReadLinearTestMemory}; ResourceSnapshot snapshot; ResourceSpecialization specialization; Check(MaterializeResources(plan, runtime, snapshot, specialization) && @@ -3388,7 +3393,7 @@ void TestIndexedMaterialImageKeys() { if (shared) { owner_key.ResolveInstruction()->ReplaceUsesWith(Value(12994u)); uint32_t offset = 0u; - Check(SrtWalker(f.program, {}).Evaluate(shared_image.ResolveInstruction()->Arg(0), offset) && + Check(EvaluateSrtValue(f.program, {}, shared_image.ResolveInstruction()->Arg(0), offset) && offset == 12994u * 32u, "inherited indexed selector changed GPU byte offsets into raw material keys"); } @@ -3811,10 +3816,9 @@ void TestUniformScalarBufferImage() { memory.words.begin() + (0x1010u + index * 48u) / 4u); memory.words[(0x1010u + index * 48u) / 4u] += index; } - SrtRuntime runtime{.user_data = user_data, - .read_memory = ReadLinearTestMemory, - .userdata = &memory, + SrtRuntime runtime{.user_data = user_data, .userdata = &memory, .read_specialization_memory = ReadLinearTestMemory}; + FakeGuestMemory fake(ReadLinearTestMemory, &memory); ResourceSnapshot snapshot; ResourceSpecialization specialization; const auto materializes = [&](uint32_t address) { @@ -3827,9 +3831,10 @@ void TestUniformScalarBufferImage() { memory.words[0x44u / 4u] = 0u; memory.words[0x1010u / 4u] = 0x30u; Check(materializes(0x30u), "changed scalar table memory reused the previous image descriptor"); + // An unreadable key reads as zero and selects the first record instead of the second. + memory.words[0x44u / 4u] = 1u; memory.fail_address = 0x1044u; - Check(!MaterializeResources(plan, runtime, snapshot, specialization), - "unavailable nested image table was accepted"); + Check(materializes(0x30u), "unavailable nested image key did not read as zero"); } void TestCompactScalarImageTable() { @@ -4258,8 +4263,8 @@ void TestScalarBufferSelectorDomain() { (static_cast(Libs::Graphics::Prospero::ImageType::kColor2D) << 28u); } std::array user_data {0x1000u, 0u, 180u, 0u, 0x2000u, 0u}; - const SrtRuntime runtime {.user_data = user_data, .read_memory = ReadLinearTestMemory, - .userdata = &memory, .read_specialization_memory = ReadLinearTestMemory}; + const SrtRuntime runtime {.user_data = user_data, .userdata = &memory, + .read_specialization_memory = ReadLinearTestMemory}; ResourceSnapshot snapshot; ResourceSpecialization specialization; memory.watched_address = 0x1000u + 144u; @@ -4314,8 +4319,8 @@ void TestScalarBufferSelectorDomain() { user_data[1] = 2u << 16u; user_data[2] = UINT32_MAX; Check(MaterializeResources(plan, - {.user_data = user_data, .read_memory = read_sparse, .userdata = &sparse, - .read_specialization_memory = read_sparse}, snapshot, specialization) && + {.user_data = user_data, .userdata = &sparse, .read_specialization_memory = read_sparse}, + snapshot, specialization) && sparse.offsets == std::vector{144u, 0x80000090u} && snapshot.images.size() == 2u, "large V# bounds extended the native U32 domain or invented an OOB zero key"); @@ -4644,7 +4649,8 @@ void TestCountedRowImageSelector() { at(MaterialBase + 256u) = 0x200u; at(MaterialBase + 260u) = static_cast(Libs::Graphics::Prospero::BufferFormat::k8SScaled) << 20u; at(MaterialBase + 268u) = at(MaterialBase + 196u); - std::array user_data{KeyBase, 4u << 16u, 64u, 0x14004u, + // The walker needs every user-data DWORD the plan reads, including the zero s12-s14. + std::array user_data{KeyBase, 4u << 16u, 64u, 0x14004u, CountBase, 4u << 16u, 4u, 0x14004u, MaterialBase, 396u << 16u, 4u, 0x14004u}; const SrtRuntime runtime{.user_data = user_data, .userdata = &memory, .read_specialization_memory = ReadLinearTestMemory}; @@ -5002,8 +5008,9 @@ void TestSharedAppendSelectorBounds() { (static_cast(Libs::Graphics::Prospero::ImageType::kColor2D) << 28u); } std::array user_data {0x1000u, 160u << 16u, 8u, 0u, 0x2000u, 0u, 0x1800u, 0u, 0u}; - const SrtRuntime runtime {.user_data = user_data, .read_memory = ReadLinearTestMemory, - .userdata = &memory, .read_specialization_memory = ReadLinearTestMemory}; + SrtRuntime runtime {.user_data = user_data, .userdata = &memory, + .read_specialization_memory = ReadLinearTestMemory}; + FakeGuestMemory fake(ReadRuntimeMemory, &runtime); ResourceSnapshot snapshot; ResourceSpecialization specialization; memory.fail_address = memory.watched_address = 0x1090u; @@ -5085,9 +5092,8 @@ void TestSharedAppendSelectorBounds() { user_data[2] = 65536u; user_data[4] = 0x50000000u; user_data[6] = 0x40000000u; - Check(MaterializeResources(plan, - {.user_data = user_data, .read_memory = read_boundary, .userdata = &boundary, - .read_specialization_memory = read_boundary}, snapshot, specialization) && + runtime = {.user_data = user_data, .userdata = &boundary, .read_specialization_memory = read_boundary}; + Check(MaterializeResources(plan, runtime, snapshot, specialization) && boundary.keys_read == 65536u && snapshot.images.size() == 1u && snapshot.images[0].dwords[0] == 0x101u, "shared append materialization rejected its exact nonwrapping probe limit"); @@ -5103,9 +5109,7 @@ void TestSharedAppendSelectorBounds() { for (const auto& test : overlaps) { boundary.counts = test.counts; boundary.keys_read = 0u; - Check(MaterializeResources(overlap_plan, - {.user_data = user_data, .read_memory = read_boundary, .userdata = &boundary, - .read_specialization_memory = read_boundary}, snapshot, specialization) && + Check(MaterializeResources(overlap_plan, runtime, snapshot, specialization) && boundary.keys_read == test.expected[layout] && snapshot.images.size() == 1u && snapshot.images[0].dwords[0] == (test.expected[layout] != 0u ? 0x101u : 0u), "shared append bounds lost overlap, included a touching list or retained stale keys"); @@ -5119,7 +5123,6 @@ void TestComputeBufferFill() { bool conditional = false; bool shifted = false; bool extra_store = false; - bool clean = false; bool branch = false; }; const auto Run = [](Options options) { @@ -5188,17 +5191,13 @@ void TestComputeBufferFill() { words[0] = memory.words[0]; return true; }; - Check(MaterializeResources( - plan, - {.user_data = userdata, - .read_memory = Read, - .userdata = &memory, - .read_specialization_memory = options.clean ? Read : nullptr}, - snapshot, specialization), - "fill fixture did not materialize"); + const auto materialize = [&](SrtMemoryReader reader) { + FakeGuestMemory fake(reader, &memory); + return MaterializeResources(plan, {.user_data = userdata}, snapshot, specialization); + }; + Check(materialize(Read), "fill fixture did not materialize"); const bool expected = !options.conditional && !options.shifted && - !options.extra_store && !options.branch && - (!options.scalar || options.clean); + !options.extra_store && !options.branch; Check((snapshot.uniform_fill.words != 0) == expected, "fill proof accepted an unsafe store or missed the real GTA3 clear"); if (expected) { @@ -5207,21 +5206,19 @@ void TestComputeBufferFill() { snapshot.uniform_fill.value == (options.scalar ? 0x40404040u : 0u), "fill proof lost address coverage or the actual stored scalar"); - if (options.scalar && options.clean) { - Check(MaterializeResources(plan, - {.user_data = userdata, .read_memory = Read, .userdata = &memory}, - snapshot, specialization) && snapshot.uniform_fill.words == 0, - "an unavailable clean value retained a previous uniform fill"); + if (options.scalar) { + Check(materialize(nullptr) && snapshot.uniform_fill.words == 1 && + snapshot.uniform_fill.value == 0u, + "an unreadable fill value retained a previous uniform fill"); } } }; Run({}); - Run({.scalar = true, .clean = true}); Run({.scalar = true}); - Run({.conditional = true, .clean = true}); - Run({.shifted = true, .clean = true}); - Run({.extra_store = true, .clean = true}); - Run({.clean = true, .branch = true}); + Run({.conditional = true}); + Run({.shifted = true}); + Run({.extra_store = true}); + Run({.branch = true}); } void TestDenseBufferTracking() { @@ -5332,12 +5329,12 @@ void TestRuntimeUnsignedMinDescriptor() { SrtRuntime runtime{.user_data = user_data}; DescriptorValue value; const auto source = fixture.program.info.buffers[0].source; - Check(SrtWalker(plan, runtime).EvaluateDescriptor(source, value) && + Check(EvaluateSrtDescriptor(plan, runtime, source, value) && value.dwords[3] == 0x100u, "runtime descriptor unsigned minimum did not clamp its first operand"); user_data[0] = 0x80u; Check( - SrtWalker(plan, runtime).EvaluateDescriptor(source, value) && + EvaluateSrtDescriptor(plan, runtime, source, value) && value.dwords[3] == 0x80u, "runtime descriptor unsigned minimum did not preserve its first operand"); } @@ -5488,12 +5485,12 @@ void TestSampleAdjustSamplerScratch() { std::array user_data{4u, 1u, 2u, 0x80000abcu}; SrtRuntime runtime{.user_data = user_data}; DescriptorValue descriptor; - Check(SrtWalker(plan, runtime).EvaluateDescriptor(source, descriptor) && + Check(EvaluateSrtDescriptor(plan, runtime, source, descriptor) && descriptor.dwords[3] == 0x80000abcu, "SampleAdjust canonicalization lost sampler border fields"); const auto shift_source = fixture.program.info.samplers[ shift_sampler.Instruction()->Flags()].source; - Check(SrtWalker(plan, runtime).EvaluateDescriptor(shift_source, descriptor) && + Check(EvaluateSrtDescriptor(plan, runtime, shift_source, descriptor) && std::ranges::equal(std::span(descriptor.dwords).first(4), std::array{0x36u, 0xfff000u, 0x02500000u, 0u}), "SampleAdjust shift-only scratch was not reduced to its zero border payload"); @@ -5601,11 +5598,11 @@ void TestFmaskLoadSpecialization() { "FMASK load did not lower to a value vector"); result = vector->Arg(0); uint32_t value = 0; - Check(SrtWalker(fixture.program, {.user_data = user_data}).Evaluate(result, value) && + Check(EvaluateSrtValue(fixture.program, {.user_data = user_data}, result, value) && value == 0x76543210u, "FMASK load did not return the native sample-to-fragment mapping"); user_data[8] = 1; - Check(SrtWalker(fixture.program, {.user_data = user_data}).Evaluate(result, value) && + Check(EvaluateSrtValue(fixture.program, {.user_data = user_data}, result, value) && value == 0u, "inactive FMASK load did not preserve the execution mask"); ShaderComputeInputInfo compute{}; @@ -5849,16 +5846,14 @@ void TestSrtFlatteningAndRuntimeMemoization() { std::array user_data{0x1000u, 0u}; TestMemory memory; memory.words[1] = 0xdeadbeefu; - SrtRuntime runtime{.user_data = user_data, - .read_memory = ReadTestMemory, - .userdata = &memory}; + SrtRuntime runtime{.user_data = user_data}; + FakeGuestMemory fake(ReadTestMemory, &memory); auto plan = ExtractResourcePlan(fixture.program); DescriptorValue descriptor; std::vector flat; const uint32_t request = fixture.program.info.buffers[0].source; const auto refresh = [&](const ResourcePlan& plan) { - SrtWalker walker(plan, runtime); - return walker.EvaluateDescriptor(request, descriptor) && walker.RefreshFlatBuffer(flat); + return WalkSrt(plan, runtime, flat, nullptr) && WalkedDescriptor(plan, request, descriptor); }; Check(refresh(plan), "typed runtime source evaluation failed"); Check(descriptor.dwords[0] == 0xdeadbeefu && @@ -5873,10 +5868,11 @@ void TestSrtFlatteningAndRuntimeMemoization() { memory.reads = 0; memory.fail_after = 0; - Check(!refresh(plan), "unavailable scalar memory was accepted"); + Check(refresh(plan) && descriptor.dwords[0] == 0u && flat == std::vector{0u}, + "unavailable scalar memory did not read as zero"); memory.fail_after = UINT32_MAX; Check(refresh(plan) && descriptor.dwords[0] == 0x12345678u && memory.reads == 1, - "failed runtime evaluation left a value marked as visiting"); + "a zero read left a stale value in the next walk"); auto moved = std::move(plan); memory.reads = 0; @@ -5924,18 +5920,17 @@ void TestScalarDataReadControlFlow() { {address, Value(0u), Value(0u), Value(true)}, fixture.AddMemory(scalar, 4), use); fixture.PlanAndTrack(); auto plan = ExtractResourcePlan(fixture.program); - Check(plan.srt_reads.size() == 1 && !fixture.program.info.uses_dma && - bool(plan.srt_code) == (comparison == ValueOpcode::Void), - "scalar data lost flattening or conditional reads entered the unconditional JIT"); + Check(plan.srt_reads.size() == 1 && !fixture.program.info.uses_dma && plan.srt_code, + "scalar data lost flattening or its walker"); uint32_t reads = 0; std::array user_data {}; - SrtRuntime runtime {.user_data = user_data, - .read_memory = +[](void* data, uint64_t address, std::span words) { + SrtRuntime runtime {.user_data = user_data}; + FakeGuestMemory fake(+[](void* data, uint64_t address, std::span words) { ++*static_cast(data); if (address == 0) return false; words[0] = 42; return true; - }, .userdata = &reads}; + }, &reads); ResourceSnapshot snapshot; ResourceSpecialization specialization; for (const uint64_t pointer : {uint64_t{0}, uint64_t{0x1000}, uint64_t{1} << 32u}) { @@ -5977,11 +5972,10 @@ void TestDynamicSrtReadRemainsExplicit() { std::array user_data{0x1000u, 0u, 4u}; TestMemory memory; memory.words[1] = 0xabcdef01u; - SrtRuntime runtime{.user_data = user_data, - .read_memory = ReadTestMemory, - .userdata = &memory}; + SrtRuntime runtime{.user_data = user_data}; + FakeGuestMemory fake(ReadTestMemory, &memory); DescriptorValue value; - Check(SrtWalker(plan, runtime).EvaluateDescriptor(fixture.program.info.buffers[0].source, value) && + Check(EvaluateSrtDescriptor(plan, runtime, fixture.program.info.buffers[0].source, value) && value.dwords[0] == 0xabcdef01u && memory.reads == 1, "dynamic typed scalar descriptor source was not evaluated"); @@ -6091,8 +6085,9 @@ void TestBoundedFormattedBufferStore() { memory.words[word + 3u] = static_cast(Prospero::BufferFormat::k32UInt) << 12u | 4u; } std::array user_data{static_cast(memory.base), 0u}; - SrtRuntime runtime{.user_data = user_data, .read_memory = ReadLinearTestMemory, - .userdata = &memory, .read_specialization_memory = ReadLinearTestMemory}; + SrtRuntime runtime{.user_data = user_data, .userdata = &memory, + .read_specialization_memory = ReadLinearTestMemory}; + FakeGuestMemory fake(ReadLinearTestMemory, &memory); ResourceSnapshot snapshot; ResourceSpecialization specialization; const auto materialize = [&] { return MaterializeResources(plan, runtime, snapshot, specialization); }; @@ -6316,8 +6311,9 @@ void TestBoundedBufferRecords(uint32_t d32_components) { } std::array user_data{uint32_t(memory.base) + table_offset, 8u << 16u, 4u * 192u / 8u, 0x5204u, uint32_t(memory.base), 0u}; - SrtRuntime runtime{.user_data = user_data, .read_memory = ReadLinearTestMemory, - .userdata = &memory, .read_specialization_memory = ReadLinearTestMemory}; + SrtRuntime runtime{.user_data = user_data, .userdata = &memory, + .read_specialization_memory = ReadLinearTestMemory}; + FakeGuestMemory fake(ReadLinearTestMemory, &memory); ResourceSnapshot snapshot; ResourceSpecialization specialization; const auto materialize = [&] { return MaterializeResources(plan, runtime, snapshot, specialization); }; @@ -6498,8 +6494,8 @@ void TestGuardedWaterfallBufferRecords() { std::copy(poison.begin(), poison.end(), memory.words.begin() + (table_offset + 480u) / 4u); std::array user_data{uint32_t(memory.base) + table_offset, 488u << 16u, 2u, 0x5204u, 0x9000u, 4u << 16u, 32u, 0x5204u}; - SrtRuntime runtime{.user_data = user_data, .read_memory = ReadLinearTestMemory, - .userdata = &memory, .read_specialization_memory = ReadLinearTestMemory}; + SrtRuntime runtime{.user_data = user_data, .userdata = &memory, + .read_specialization_memory = ReadLinearTestMemory}; ResourceSnapshot snapshot; ResourceSpecialization specialization; for (const auto guard : {Guard::Exact, Guard::Reversed, Guard::Greater, Guard::GreaterEqual, @@ -6617,8 +6613,8 @@ void TestGpuSelectedBufferRecords() { // SMEM aligns the base and dynamic offset independently; its V# format is ignored. std::array user_data{uint32_t(memory.base) + table_offset + 3u, 0u, 0u, 0u, uint32_t(memory.fail_address), 12u << 16u, 129600u, 0x5204u}; - SrtRuntime runtime{.user_data = user_data, .read_memory = ReadLinearTestMemory, - .userdata = &memory, .read_specialization_memory = ReadLinearTestMemory}; + SrtRuntime runtime{.user_data = user_data, .userdata = &memory, + .read_specialization_memory = ReadLinearTestMemory}; ResourceSnapshot snapshot; ResourceSpecialization specialization; const auto materialize = [&] { return MaterializeResources(plan, runtime, snapshot, specialization); }; @@ -6709,8 +6705,8 @@ void TestWorkgroupSelectedBufferRecords() { std::array user_data{uint32_t(memory.base) + table, 192u << 16u, 2u, 0x5204u, uint32_t(memory.base) + selector, 12u << 16u, MaxIndirectDescriptorProbes, 0x5204u}; memory.watched_address = memory.base; - SrtRuntime runtime{.user_data = user_data, .read_memory = ReadLinearTestMemory, - .userdata = &memory, .read_specialization_memory = ReadLinearTestMemory, + SrtRuntime runtime{.user_data = user_data, .userdata = &memory, + .read_specialization_memory = ReadLinearTestMemory, .indirect_args_address = memory.base}; ResourceSnapshot snapshot; ResourceSpecialization specialization; @@ -6938,8 +6934,9 @@ void TestGpuSelectedFormattedRecordStore() { memory.fail_address = memory.base + 0x2000u; std::array user_data{uint32_t(memory.base) + table_offset, 592u << 16u, 6u, 0x5204u, uint32_t(memory.fail_address), 4u << 16u, 64u, 0x5204u}; - SrtRuntime runtime{.user_data = user_data, .read_memory = ReadLinearTestMemory, - .userdata = &memory, .read_specialization_memory = ReadLinearTestMemory}; + SrtRuntime runtime{.user_data = user_data, .userdata = &memory, + .read_specialization_memory = ReadLinearTestMemory}; + FakeGuestMemory fake(ReadRuntimeMemory, &runtime); ResourceSnapshot snapshot; ResourceSpecialization specialization; const auto materialize = [&] { return MaterializeResources(plan, runtime, snapshot, specialization); }; @@ -7127,9 +7124,8 @@ void TestGpuSelectedFormattedRecordStore() { std::array prefix_data{uint32_t(prefix_memory.base) + table_offset - 48u, 0u, 0u, 0u, uint32_t(prefix_memory.base) + selector_offset, 8u << 16u, 12u, 0x5204u}; std::array counts{2u, 1u, 1u}; - SrtRuntime prefix_runtime{.user_data = prefix_data, .read_memory = read_prefix, - .userdata = &prefix_memory, .read_specialization_memory = read_prefix, - .workgroup_counts = counts}; + runtime = {.user_data = prefix_data, .userdata = &prefix_memory, + .read_specialization_memory = read_prefix, .workgroup_counts = counts}; prefix_memory.watched_address = prefix_memory.base + selector_offset + 4u; for (const auto prefix : {Prefix::Forward, Prefix::Reversed}) { const auto selected = make(16u, 16u, false, true, prefix); @@ -7143,7 +7139,7 @@ void TestGpuSelectedFormattedRecordStore() { const auto refresh = [&] { prefix_memory.watched_reads = 0u; prefix_memory.read_attempts = 0u; - return MaterializeResources(selected_plan, prefix_runtime, snapshot, specialization); + return MaterializeResources(selected_plan, runtime, snapshot, specialization); }; for (const uint32_t first : {0u, 2u, 4u, 0u}) { prefix_memory.words[first_offset / 4u] = first; @@ -7155,9 +7151,12 @@ void TestGpuSelectedFormattedRecordStore() { Check(std::ranges::find(snapshot.specialization_reads, read) != snapshot.specialization_reads.end(), "runtime first or selected prefix escaped specialization read capture"); } + // An unreadable runtime first reads as zero. + prefix_memory.words[first_offset / 4u] = 4u; prefix_memory.fail_address = prefix_memory.base + first_offset; - Check(!refresh() && prefix_memory.read_attempts == 1u, - "unreadable runtime first reached the selector payload"); + Check(refresh() && snapshot.buffers.size() == selected_plan.info.buffers.size() + 1u && + snapshot.buffers[wide_root].dwords[0] == 0x100000u, + "unreadable runtime first did not read as zero"); prefix_memory.fail_address = UINT64_MAX; prefix_memory.words[first_offset / 4u] = 6u; Check(!refresh(), "an actually selected invalid numeric descriptor was silently omitted"); @@ -7257,8 +7256,8 @@ void TestFiniteBufferSelection() { } memory.fail_address = memory.base + 32u + 5u * 16u; const std::array user_data{static_cast(memory.base), 0u}; - SrtRuntime runtime{.user_data = user_data, .read_memory = ReadLinearTestMemory, - .userdata = &memory, .read_specialization_memory = ReadLinearTestMemory}; + SrtRuntime runtime{.user_data = user_data, .userdata = &memory, + .read_specialization_memory = ReadLinearTestMemory}; ResourceSnapshot snapshot; ResourceSpecialization specialization; Check(MaterializeResources(plan, runtime, snapshot, specialization) && snapshot.buffers.size() == 5u, @@ -7339,8 +7338,8 @@ void TestPackedScalarBufferSelection() { // No descriptor exists after key 18; broad enumeration would read this address. memory.fail_address = memory.base + 0x170u; const std::array user_data{static_cast(memory.base), 0u}; - const SrtRuntime runtime{.user_data = user_data, .read_memory = ReadLinearTestMemory, - .userdata = &memory, .read_specialization_memory = ReadLinearTestMemory}; + const SrtRuntime runtime{.user_data = user_data, .userdata = &memory, + .read_specialization_memory = ReadLinearTestMemory}; ResourceSnapshot snapshot; ResourceSpecialization specialization; const auto materialize = [&] { return MaterializeResources(plan, runtime, snapshot, specialization); }; @@ -7497,38 +7496,27 @@ void TestConditionalSamplerPhi() { for (uint32_t word = 448 / 4; word < (480 + 16) / 4; ++word) { memory.words[word] = 0x400u + word; } - const SrtRuntime runtime{.read_memory = ReadLinearTestMemory, - .userdata = &memory, - .read_specialization_memory = - ReadLinearTestMemory}; + FakeGuestMemory fake(ReadLinearTestMemory, &memory); // Sampler selection compares a signed value against zero. for (const auto flag : {-1, 0, 1, INT32_MIN, INT32_MAX}) { memory.words[(0x1000 + 196) / 4] = std::bit_cast(flag); const uint32_t first = (flag < 0) != reverse ? 480 / 4 : 448 / 4; memory.fail_address = 0x1000 + (first == 448 / 4 ? 480u : 448u); DescriptorValue selected; - SrtWalker clean(plan, CleanRuntime(runtime)); - Check(SrtWalker(plan, runtime, &clean).EvaluateDescriptor(source, selected), + Check(EvaluateSrtDescriptor(plan, {}, source, selected), "conditional sampler did not survive detached plan lifetime"); for (uint32_t word = 0; word < 4; ++word) { Check(selected.dwords[word] == memory.words[first + word], "conditional sampler chose the wrong incoming descriptor"); } } - DescriptorValue selected; - auto no_clean_reader = runtime; - no_clean_reader.read_specialization_memory = nullptr; - { - SrtWalker clean(plan, CleanRuntime(no_clean_reader)); - Check(!SrtWalker(plan, no_clean_reader, &clean).EvaluateDescriptor(source, selected), - "conditional sampler used unchecked memory for its predicate"); - } + // An unreadable negative predicate reads as zero and selects the nonnegative edge. + memory.words[(0x1000 + 196) / 4] = UINT32_MAX; memory.fail_address = 0x2000 + 196; - { - SrtWalker clean(plan, CleanRuntime(runtime)); - Check(!SrtWalker(plan, runtime, &clean).EvaluateDescriptor(source, selected), - "conditional sampler ignored unavailable coherent predicate memory"); - } + DescriptorValue selected; + Check(EvaluateSrtDescriptor(plan, {}, source, selected) && + selected.dwords[0] == memory.words[reverse ? 480 / 4 : 448 / 4], + "conditional sampler did not read unavailable predicate memory as zero"); } } CheckFatal([&] { ConditionalSamplerPlan(diamond, false, false, true); }, @@ -7592,10 +7580,9 @@ void TestFiniteImagePhiCycle() { "finite image key did not preserve the loop-carried selection"); std::array data; for (uint32_t i = 0; i < data.size(); ++i) data[i] = 100u + i; - SrtWalker walker(plan, SrtRuntime{.user_data = data}); for (uint32_t candidate = 0; candidate < 2; ++candidate) { DescriptorValue descriptor; - Check(walker.EvaluateDescriptor(finite->sources[candidate], descriptor), + Check(EvaluateSrtDescriptor(plan, {.user_data = data}, finite->sources[candidate], descriptor), "finite image candidate is not host-readable"); for (uint32_t word = 0; word < 8; ++word) Check(descriptor.dwords[word] == data[candidate * 8 + word], @@ -7666,8 +7653,8 @@ void TestPackedDescriptorWaterfall() { memory.words[word + 1u] = 0x00fff000u; memory.words[word + 2u] = (row & 1u) ? 0x0a500000u : 0u; } - const SrtRuntime runtime{.user_data = user_data, .read_memory = ReadLinearTestMemory, - .userdata = &memory, .read_specialization_memory = ReadLinearTestMemory}; + const SrtRuntime runtime{.user_data = user_data, .userdata = &memory, + .read_specialization_memory = ReadLinearTestMemory}; ResourceSnapshot snapshot; ResourceSpecialization specialization; Check(MaterializeResources(plan, runtime, snapshot, specialization) && @@ -7795,8 +7782,9 @@ void TestFiniteImageWaterfall() { (uint32_t(Libs::Graphics::Prospero::ImageType::kColor2D) << 28u); } std::array user_data{uint32_t(memory.base) + 256u, 8u << 16u, 60u, 0x5204u}; - SrtRuntime runtime{.user_data = user_data, .read_memory = ReadLinearTestMemory, - .userdata = &memory, .read_specialization_memory = ReadLinearTestMemory}; + SrtRuntime runtime{.user_data = user_data, .userdata = &memory, + .read_specialization_memory = ReadLinearTestMemory}; + FakeGuestMemory fake(ReadLinearTestMemory, &memory); ResourceSnapshot snapshot; ResourceSpecialization specialization, populated; for (const uint32_t count : {0u, 60u, 0u, 60u}) { @@ -8048,7 +8036,7 @@ void TestLoopCycleEnteredThroughRuntimeValue() { const std::array user_data{0x4000u}; DescriptorValue descriptor; Check(fixture.program.info.buffers.size() == 1u && - SrtWalker(plan, {.user_data = user_data}).EvaluateDescriptor( + EvaluateSrtDescriptor(plan, {.user_data = user_data}, fixture.program.info.buffers[0].source, descriptor) && descriptor.dwords[0] == user_data[0], "runtime-rooted invariant loop lost its buffer source"); @@ -8079,7 +8067,7 @@ void TestInvariantLoopPhi() { std::array user_data{0x12345678u}; SrtRuntime runtime{.user_data = user_data}; DescriptorValue descriptor; - Check(SrtWalker(plan, runtime).EvaluateDescriptor(fixture.program.info.buffers[0].source, descriptor) && + Check(EvaluateSrtDescriptor(plan, runtime, fixture.program.info.buffers[0].source, descriptor) && descriptor.dwords[0] == user_data[0], "loop-invariant descriptor phi was not evaluated through typed SSA"); } @@ -8196,8 +8184,7 @@ void TestBoundedRelativeRegisterWrites() { const std::array user_data{0x1000u, 0u, 72u}; SrtRuntime runtime{.user_data = user_data}; DescriptorValue descriptor; - Check(SrtWalker(plan, runtime).EvaluateDescriptor( - fixture.program.info.buffers[0].source, descriptor) && + Check(EvaluateSrtDescriptor(plan, runtime, fixture.program.info.buffers[0].source, descriptor) && descriptor.dwords[2] == 72u, "impossible relative writes left a false descriptor dependency"); } @@ -8412,6 +8399,7 @@ void TestConditionalBufferMaterialization() { TestMemory memory; SrtRuntime runtime{.user_data = user_data, .userdata = &memory, .read_specialization_memory = ReadTestMemory}; + FakeGuestMemory fake(ReadTestMemory, &memory); ResourceSnapshot snapshot; ResourceSpecialization specialization; Check(MaterializeResources(plan, runtime, snapshot, specialization) && @@ -8422,12 +8410,11 @@ void TestConditionalBufferMaterialization() { Check(snapshot.user_data == std::vector(user_data.begin(), user_data.end()), "resource reachability changed native shader user data"); + // The walker needs every user-data DWORD it may read, even for an untaken branch. runtime.user_data = std::span(user_data).first(4); - Check(MaterializeResources(plan, runtime, snapshot, specialization), - "untaken branch evaluated its unavailable descriptor"); - memory.words[0] = 1; Check(!MaterializeResources(plan, runtime, snapshot, specialization), - "taken branch accepted an unavailable descriptor"); + "truncated user data was accepted"); + memory.words[0] = 1; runtime.user_data = user_data; const auto CheckActive = [&] { @@ -8438,9 +8425,13 @@ void TestConditionalBufferMaterialization() { "potentially executed buffer descriptor was discarded"); }; CheckActive(); - memory.words[0] = 0; + // An unreadable control word reads as zero and skips the branch. memory.fail_after = memory.reads; - CheckActive(); + Check(MaterializeResources(plan, runtime, snapshot, specialization) && + snapshot.buffers.size() == 2 && snapshot.buffers[1].dwords == std::array{}, + "unreadable scalar control word did not read as zero"); + // Walker loads do not depend on the specialization reader. + memory.fail_after = UINT32_MAX; runtime.read_specialization_memory = nullptr; CheckActive(); } @@ -8515,8 +8506,9 @@ void TestGuardedScalarDescriptorReads() { return ReadLinearTestMemory(&memory.data, address, words); }; const std::array user_data{0x1000u, 0u, 64u, 0u, 0x1040u}; - const SrtRuntime runtime{.user_data = user_data, .read_memory = Read, - .userdata = &memory, .read_specialization_memory = Read}; + const SrtRuntime runtime{.user_data = user_data, .userdata = &memory, + .read_specialization_memory = Read}; + FakeGuestMemory fake(Read, &memory); memory.data.words[0xa8 / 4] = 0x2000u; memory.data.words[0xb0 / 4] = 4u; ResourceSnapshot snapshot; @@ -8526,8 +8518,9 @@ void TestGuardedScalarDescriptorReads() { std::ranges::all_of(snapshot.flattened_srt, [](auto word) { return word == 0; }), "disabled feature speculatively dereferenced its null BVH table"); memory.data.words[7] = 1; - Check(!MaterializeResources(plan, runtime, snapshot, specialization) && memory.null_reads == 1, - "active feature accepted an unreadable BVH table"); + Check(MaterializeResources(plan, runtime, snapshot, specialization) && memory.null_reads == 1 && + std::ranges::all_of(snapshot.flattened_srt, [](auto word) { return word == 0; }), + "active feature did not read its unreadable null BVH table as zero"); } memory.data.words[0x40 / 4] = 0x1080u; memory.data.watched_address = 0x10a8u; @@ -8557,19 +8550,24 @@ void TestConservativeBufferReachability() { TestMemory memory; const SrtRuntime runtime{.user_data = user_data, .userdata = &memory, .read_specialization_memory = ReadTestMemory}; + FakeGuestMemory fake(ReadTestMemory, &memory); for (const auto use : {ConditionalBufferUse::Shared, ConditionalBufferUse::Loop, ConditionalBufferUse::Writable}) { auto plan = ConditionalBufferPlan(use); ResourceSnapshot snapshot; ResourceSpecialization specialization; - std::array expected{}; - if (use != ConditionalBufferUse::Writable) - std::copy_n(user_data.begin() + 4, expected.size(), expected.begin()); - Check(MaterializeResources(plan, runtime, snapshot, specialization) && - snapshot.buffers.size() == 2 && - std::equal(expected.begin(), expected.end(), - snapshot.buffers[1].dwords.begin()), - "shared or loop-dependent resource was pruned, or inactive scalar branch retained"); + // With a zero control word, the loop body's only other predecessor is itself. + for (const uint32_t flag : {0u, 1u}) { + memory.words[0] = flag; + std::array expected{}; + if (use == ConditionalBufferUse::Shared || flag != 0u) + std::copy_n(user_data.begin() + 4, expected.size(), expected.begin()); + Check(MaterializeResources(plan, runtime, snapshot, specialization) && + snapshot.buffers.size() == 2 && + std::equal(expected.begin(), expected.end(), + snapshot.buffers[1].dwords.begin()), + "shared or entered loop resource was pruned, or an untaken branch or loop retained"); + } } } diff --git a/tests/ScalarProvenanceTests.cpp b/tests/ScalarProvenanceTests.cpp index 3069829e6..33dd6457f 100644 --- a/tests/ScalarProvenanceTests.cpp +++ b/tests/ScalarProvenanceTests.cpp @@ -4,7 +4,7 @@ #include "graphics/shader/recompiler/ir/passes/ReadLaneElimination.h" #include "graphics/shader/recompiler/ir/passes/ResourceTracking.h" #include "graphics/shader/recompiler/ir/passes/ResourceMaterialization.h" -#include "graphics/shader/recompiler/ir/passes/SrtWalker.h" +#include "SrtWalkerTestMemory.h" #include #include @@ -95,6 +95,16 @@ bool ReadMemory(void *userdata, uint64_t address, std::span values) { return true; } +// Evaluates the DWORDs of a descriptor source that no resource consumes. +bool EvaluateSource(ResourcePlan &plan, const SrtRuntime &runtime, uint32_t source, + DescriptorValue &result) { + const auto &words = plan.descriptor_sources[source]; + result = {.dword_count = words.dword_count}; + for (uint32_t word = 0; word < words.dword_count; ++word) + if (!EvaluateSrtValue(plan, runtime, words.dwords[word], result.dwords[word])) return false; + return true; +} + Value Address(Fixture &fixture, Value low, Value high, uint32_t block = 0) { return fixture.Emit(ValueOpcode::GetAddressResource, {low, high}, 0, block); } @@ -132,9 +142,9 @@ void TestImmediateFlatteningAndGvn() { "flattened raw read was not kept as a planning-only root"); TestMemory memory_image{{{0x1020u, 0xfeedbeefu}}}; - SrtRuntime runtime{.read_memory = ReadMemory, .userdata = &memory_image}; + FakeGuestMemory fake(ReadMemory, &memory_image); std::vector flat; - Check(SrtWalker(ExtractResourcePlan(fixture.program), runtime).RefreshFlatBuffer(flat), "flattened SRT walk failed"); + Check(WalkSrt(ExtractResourcePlan(fixture.program), {}, flat, nullptr), "flattened SRT walk failed"); Check(flat == std::vector{0xfeedbeefu} && memory_image.reads == 1, "flattened SRT did not evaluate its canonical read once"); } @@ -148,9 +158,9 @@ void TestRawScalarComponentAlignment() { fixture.Plan(); TestMemory memory_image{{{0x1000u, 0x12345678u}}}; - SrtRuntime runtime{.read_memory = ReadMemory, .userdata = &memory_image}; + FakeGuestMemory fake(ReadMemory, &memory_image); std::vector flat; - Check(SrtWalker(ExtractResourcePlan(fixture.program), runtime).RefreshFlatBuffer(flat), "raw scalar SRT walk failed"); + Check(WalkSrt(ExtractResourcePlan(fixture.program), {}, flat, nullptr), "raw scalar SRT walk failed"); Check( flat == std::vector{0x12345678u} && memory_image.reads == 1, "raw scalar base, immediate, and offset were not aligned independently"); @@ -203,9 +213,9 @@ void TestNestedSrtWalk() { fixture.Plan(); TestMemory memory_image{{{0x1000u, 0x2000u}, {0x2000u, 0xabcdef01u}}}; - SrtRuntime runtime{.read_memory = ReadMemory, .userdata = &memory_image}; + FakeGuestMemory fake(ReadMemory, &memory_image); std::vector flat; - Check(SrtWalker(ExtractResourcePlan(fixture.program), runtime).RefreshFlatBuffer(flat), "nested SRT walk failed"); + Check(WalkSrt(ExtractResourcePlan(fixture.program), {}, flat, nullptr), "nested SRT walk failed"); Check(flat == std::vector({0x2000u, 0xabcdef01u}), "nested typed SRT reads were not evaluated in dependency order"); } @@ -227,8 +237,9 @@ void TestShaderBaseAndUserData() { const std::array user_data{0x20u}; SrtRuntime runtime{.user_data = user_data, .shader_base = 0x12345678abcdef00ull}; + auto plan = ExtractResourcePlan(fixture.program); DescriptorValue result; - Check(SrtWalker(ExtractResourcePlan(fixture.program), runtime).EvaluateDescriptor(0, result), + Check(EvaluateSource(plan, runtime, 0, result), "shader-relative descriptor evaluation failed"); Check(result.dword_count == 3 && result.dwords[0] == 0xabcdef00u && result.dwords[1] == 0x12345678u && result.dwords[2] == 0x24u, @@ -249,8 +260,9 @@ void TestCarryAndBitFields() { fixture.program.descriptor_sources.push_back( {.dwords = {low, high, inserted, sign}, .dword_count = 4}); + auto plan = ExtractResourcePlan(fixture.program); DescriptorValue result; - Check(SrtWalker(ExtractResourcePlan(fixture.program), {}).EvaluateDescriptor(0, result), + Check(EvaluateSource(plan, {}, 0, result), "carry and bit-field descriptor evaluation failed"); Check(result.dwords[0] == 1u && result.dwords[1] == 1u && result.dwords[2] == 0x89abcdefu && result.dwords[3] == 0xffffffffu, @@ -273,12 +285,12 @@ void TestInvariantAndDivergentPhi() { fixture.program.descriptor_sources.push_back( {.dwords = {Value(&divergent)}, .dword_count = 1}); - const auto plan = ExtractResourcePlan(fixture.program); + auto plan = ExtractResourcePlan(fixture.program); DescriptorValue result; - Check(SrtWalker(plan, {}).EvaluateDescriptor(0, result) && + Check(EvaluateSource(plan, {}, 0, result) && result.dwords[0] == 7u, "loop-invariant typed phi was rejected"); - Check(!SrtWalker(plan, {}).EvaluateDescriptor(1, result), + Check(!EvaluateSource(plan, {}, 1, result), "divergent phi was accepted"); } @@ -294,7 +306,7 @@ void TestOperandMutationLifecycle() { source->ReplaceUsesWith(Value(0xaau)); uint32_t result = 0; Check(source->UseCount() == 0 && - SrtWalker(fixture.program, {}).Evaluate(inserted, result) && result == 0x3au, + EvaluateSrtValue(fixture.program, {}, inserted, result) && result == 0x3au, "inline operand replacement changed the evaluated value"); for (const auto opcode : {ValueOpcode::GetImageResource, ValueOpcode::MakeImageAddress}) { @@ -333,7 +345,7 @@ void TestOperandMutationLifecycle() { const auto sum = fixture.Emit(ValueOpcode::IAdd32, {Value(&phi), Value(5u)}, 0, 5); phi.ReplaceUsesWith(Value(6u)); Check(source->UseCount() == 0 && phi.NumPhiBlocks() == 0 && phi.NumArgs() == 1 && - SrtWalker(fixture.program, {}).Evaluate(sum, result) && result == 11, + EvaluateSrtValue(fixture.program, {}, sum, result) && result == 11, "Phi replacement retained incoming uses or changed the consumer value"); } @@ -374,8 +386,9 @@ void TestRuntime64BitDescriptorOps() { fixture.program.descriptor_sources.push_back( {.dwords = {low, high}, .dword_count = 2}); + auto plan = ExtractResourcePlan(fixture.program); DescriptorValue result; - Check(SrtWalker(ExtractResourcePlan(fixture.program), {}).EvaluateDescriptor(0, result) && + Check(EvaluateSource(plan, {}, 0, result) && result.dwords[0] == 0xabcdu && result.dwords[1] == 0x1235u, "64-bit typed descriptor arithmetic evaluation is incorrect"); } @@ -448,13 +461,13 @@ void TestUniformFirstLaneSamplerLod() { std::array user_data{}; user_data[6] = 3u; SrtRuntime runtime{.user_data = user_data}; - const auto plan = ExtractResourcePlan(fixture.program); + auto plan = ExtractResourcePlan(fixture.program); DescriptorValue result; - Check(SrtWalker(plan, runtime).EvaluateDescriptor(0, result) && + Check(EvaluateSource(plan, runtime, 0, result) && result.dwords[0] == 0x00300300u, "uniform sampler LOD evaluated incorrectly"); user_data[6] = 20u; - Check(SrtWalker(plan, runtime).EvaluateDescriptor(0, result) && + Check(EvaluateSource(plan, runtime, 0, result) && result.dwords[0] == 0x00ffffffu, "uniform sampler LOD clamp evaluated incorrectly"); } @@ -489,13 +502,12 @@ void TestFloatComparisonDescriptorInputs() { fixture.program.descriptor_sources.push_back( {.dwords = {low, high}, .dword_count = 2}); - const auto plan = ExtractResourcePlan(fixture.program); + auto plan = ExtractResourcePlan(fixture.program); for (size_t index = 0; index < inputs.size(); index++) { const auto &input = inputs[index]; const std::array user_data{input.bits}; DescriptorValue result; - Check(SrtWalker(plan, {.user_data = user_data}) - .EvaluateDescriptor(0, result) && + Check(EvaluateSource(plan, {.user_data = user_data}, 0, result) && result.dwords[0] == (flush && index < 4 ? 1 : input.less_equal) && result.dwords[1] == (flush && index < 4 ? 1 : input.greater_equal), "descriptor comparison disagrees with native FP32 input mode"); @@ -551,7 +563,6 @@ void TestConstantBufferBounds() { uint32_t immediate; uint32_t stride; uint32_t records; - bool valid; uint32_t expected; uint32_t reads; }; @@ -565,17 +576,17 @@ void TestConstantBufferBounds() { ++probe.attempts; return ReadMemory(&probe.memory, address, values); }; - // PS5 scalar-buffer loads return zero per out-of-range DWORD; an unreadable - // in-range address remains a failed memory read. - for (const auto &test : {Case{12u, 0u, 0u, 16u, true, 0xa5a5a5a5u, 1u}, - Case{3u, 1u, 0u, 16u, true, 0x12345678u, 1u}, - Case{0xfffffffcu, 4u, 0u, 16u, true, 0u, 0u}, - Case{16u, 0u, 0u, 16u, true, 0u, 0u}, - Case{12u, 0u, 0u, 14u, true, 0u, 0u}, - Case{0u, 0u, 0u, 0u, true, 0u, 0u}, - Case{8u, 0u, 0u, 16u, false, 0u, 1u}, - Case{12u, 0u, 48u, 32768u, true, 0xa5a5a5a5u, 1u}, - Case{0x6339c0u, 16u, 48u, 32768u, true, 0u, 0u}}) { + // PS5 scalar-buffer loads return zero per out-of-range DWORD without an access; + // an unreadable in-range address is accessed and reads as zero too. + for (const auto &test : {Case{12u, 0u, 0u, 16u, 0xa5a5a5a5u, 1u}, + Case{3u, 1u, 0u, 16u, 0x12345678u, 1u}, + Case{0xfffffffcu, 4u, 0u, 16u, 0u, 0u}, + Case{16u, 0u, 0u, 16u, 0u, 0u}, + Case{12u, 0u, 0u, 14u, 0u, 0u}, + Case{0u, 0u, 0u, 0u, 0u, 0u}, + Case{8u, 0u, 0u, 16u, 0u, 1u}, + Case{12u, 0u, 48u, 32768u, 0xa5a5a5a5u, 1u}, + Case{0x6339c0u, 16u, 48u, 32768u, 0u, 0u}}) { Fixture fixture; const auto memory = fixture.AddMemory(ResourceKind::ScalarBuffer, test.immediate); const auto buffer = @@ -588,12 +599,11 @@ void TestConstantBufferBounds() { fixture.Plan(); MemoryProbe memory_image; - SrtRuntime runtime{.read_memory = read_memory, .userdata = &memory_image}; + FakeGuestMemory fake(read_memory, &memory_image); std::vector flat; - Check(SrtWalker(ExtractResourcePlan(fixture.program), runtime).RefreshFlatBuffer(flat) == test.valid, + Check(WalkSrt(ExtractResourcePlan(fixture.program), {}, flat, nullptr), "constant-buffer walk misaligned or wrapped its offset components"); - Check((!test.valid || flat == std::vector{test.expected}) && - memory_image.attempts == test.reads, + Check(flat == std::vector{test.expected} && memory_image.attempts == test.reads, "constant-buffer walk read the wrong word or accessed an out-of-bounds address"); } } @@ -831,8 +841,9 @@ void TestUndefinedRuntimeValueFails() { fixture.program.descriptor_sources.push_back( {.dwords = {undef}, .dword_count = 1}); + auto plan = ExtractResourcePlan(fixture.program); DescriptorValue result; - Check(!SrtWalker(ExtractResourcePlan(fixture.program), {}).EvaluateDescriptor(0, result), + Check(!EvaluateSource(plan, {}, 0, result), "undefined typed descriptor source was accepted"); } diff --git a/tests/ShaderRecompilerComputeTests.cpp b/tests/ShaderRecompilerComputeTests.cpp index ef41eb9b1..ea02c9606 100644 --- a/tests/ShaderRecompilerComputeTests.cpp +++ b/tests/ShaderRecompilerComputeTests.cpp @@ -60,6 +60,7 @@ #include "libs/errno.h" #include "loader/symbolDatabase.h" #include "spirv-tools/libspirv.hpp" +#include "SrtWalkerTestMemory.h" #if __has_include("graphics/host_gpu/renderer/renderTargetBarriers.h") #error "legacy render-target barrier API must remain deleted" @@ -1724,15 +1725,17 @@ CompiledShader CompileCase(const TestCase &test, u32 host_subgroup_size = 64) { const ShaderRecompiler::IR::SrtRuntime runtime{ .user_data = options.user_data, .shader_base = reinterpret_cast(test.code.data()), - .read_memory = ReadTestMemory, .userdata = const_cast *>(&test.initial), .read_specialization_memory = ReadTestMemory, .workgroup_counts = workgroup_counts, }; - Require(test.name, "resource materialization", - ShaderRecompiler::IR::MaterializeResources( - resource_plan, runtime, resources, specialization), - "translated resources could not be materialized"); + { + FakeGuestMemory fake(ReadTestMemory, runtime.userdata); + Require(test.name, "resource materialization", + ShaderRecompiler::IR::MaterializeResources( + resource_plan, runtime, resources, specialization), + "translated resources could not be materialized"); + } auto result = ShaderRecompiler::CompileProgram( std::move(translated), options, specialization); for (const auto &[text, expected] : test.decoded_counts) { @@ -11235,7 +11238,7 @@ public: }; const ShaderRecompiler::IR::SrtRuntime runtime{ .user_data = user_data, .shader_base = reinterpret_cast(caller.data()), - .read_memory = read, .read_specialization_memory = read}; + .read_specialization_memory = read}; auto source = ShaderRecompiler::PrepareShaderSource(caller, options); auto translated = ShaderRecompiler::TranslateProgram( ShaderRecompiler::RefreshShaderSource(source, runtime), options); @@ -17158,12 +17161,7 @@ public: stencil_compute.stage.resources = &stencil_snapshot; Require(name, "stencil fill materialization", ShaderRecompiler::IR::MaterializeResources(stencil_plan, - {.user_data = stencil_userdata, .userdata = &stencil_byte, - .read_specialization_memory = +[](void *data, uint64_t address, std::span words) { - if (address != reinterpret_cast(data) || words.size() != 1u) return false; - words[0] = *static_cast(data); - return true; - }}, stencil_snapshot, stencil_specialization), + {.user_data = stencil_userdata}, stencil_snapshot, stencil_specialization), "captured stencil shader could not resolve its clear byte"); ShaderRecompiler::IR::ApplyResourceSpecialization(stencil_translated.program, stencil_specialization); @@ -39523,25 +39521,25 @@ void CheckRuntimeBufferRecords(VulkanHarness &vulkan) { options.input_info.compute = &test.compute_info; auto translated = ShaderRecompiler::TranslateProgram(test.code, options); auto plan = ExtractResourcePlan(translated.program); - struct CountMemory { u32 value = 72; u32 reads = 0; bool clean = true; } memory; - SrtRuntime runtime{.user_data = test.user_data, .userdata = &memory}; + struct CountMemory { u32 value = 72; u32 reads = 0; bool readable = true; } memory; ResourceSnapshot snapshot; ResourceSpecialization specialization; - Require(test.name, "strict reader required", - !MaterializeResources(plan, runtime, snapshot, specialization), - "vector descriptor input bypassed the clean-memory reader"); - runtime.read_specialization_memory = [](void *data, uint64_t address, std::span words) { - auto &memory = *static_cast(data); - ++memory.reads; - if (!memory.clean || address != count_address || words.size() != 1) return false; - words[0] = memory.value; - return true; + const auto materialize = [&] { + FakeGuestMemory fake(+[](void *data, uint64_t address, std::span words) { + auto &memory = *static_cast(data); + ++memory.reads; + if (!memory.readable || address != count_address) return false; + words[0] = memory.value; + return true; + }, &memory); + return MaterializeResources(plan, {.user_data = test.user_data}, snapshot, specialization); }; - memory.clean = false; - Require(test.name, "dirty input rejected", - !MaterializeResources(plan, runtime, snapshot, specialization), - "GPU-dirty count was accepted for host descriptor evaluation"); - memory.clean = true; + // An unreadable count reads as zero, which leaves no records. + memory.readable = false; + Require(test.name, "unreadable input", + materialize() && memory.reads == 1 && snapshot.buffers[2].dwords[2] == 0, + "unreadable count did not read as zero"); + memory.readable = true; ResourceSpecialization initial_specialization; // Reuse the same resource plan and compiled shader as the count and coverage change. for (const u32 count : {0u, 72u, 512u, 8192u, 262144u}) { @@ -39550,7 +39548,7 @@ void CheckRuntimeBufferRecords(VulkanHarness &vulkan) { const u32 chunk = count < 512 ? 64 : count < 8192 ? 128 : count < 262144 ? 256 : 512; const u32 records = (count + chunk - 1) / chunk; Require(test.name, "cached descriptor refresh", - MaterializeResources(plan, runtime, snapshot, specialization) && memory.reads == 1 && + materialize() && memory.reads == 1 && snapshot.buffers[2].dwords[2] == records && snapshot.specialization_reads == std::vector>{{count_address, 4}}, @@ -44598,7 +44596,6 @@ void CheckPs5GameExampleImageClearRuntimeShape() { compute.thread_ids_num = 1; compute.workgroup_register = 8; u32 scalar_clear = 0x40404040u; - bool clean_scalar = false; const auto Compile = [&](const std::vector &code) { ShaderRecompiler::CompileOptions options; @@ -44616,13 +44613,6 @@ void CheckPs5GameExampleImageClearRuntimeShape() { const ShaderRecompiler::IR::SrtRuntime runtime{ .user_data = options.user_data, .shader_base = reinterpret_cast(code.data()), - .userdata = &scalar_clear, - .read_specialization_memory = clean_scalar - ? +[](void *data, uint64_t address, std::span words) { - if (address != reinterpret_cast(data) || words.size() != 1u) return false; - words[0] = *static_cast(data); - return true; - } : nullptr, }; Require("Ps5GameExampleImageClear", "resource materialization", ShaderRecompiler::IR::MaterializeResources( @@ -44780,7 +44770,7 @@ void CheckPs5GameExampleImageClearRuntimeShape() { scalar_code.push_back(EncodeMubuf0(0x04u, 0, true, false)); scalar_code.push_back(EncodeMubuf1(1, 0, 0)); AppendEnd(&scalar_code); - clean_scalar = true; + // The scalar input is a walker load, so it needs no specialization memory reader. auto loaded = Compile(scalar_code); runtime_resources = loaded; Require("Ps5GameExampleImageClear", "scalar-loaded fill", @@ -44788,12 +44778,6 @@ void CheckPs5GameExampleImageClearRuntimeShape() { packed_clear, size) && packed_clear == scalar_clear && descriptor.Base48() == 0x200000u, "GTA3's scalar input was confused with its destination"); - clean_scalar = false; - runtime_resources = Compile(scalar_code); - Require("Ps5GameExampleImageClear", "unavailable clean scalar", - !ResolveComputeBufferFill(compute, 1, 1, 1, 0x41u, descriptor, - packed_clear, size), - "a scalar without a clean reader was replaced by a fill"); std::printf("[host] %-32s ok\n", "Ps5GameExampleImageClear"); } @@ -47260,8 +47244,9 @@ void CheckImageSamplerSpecialization() { auto plan = ExtractResourcePlan(translated.program); ResourceSnapshot snapshot; ResourceSpecialization specialization; - const SrtRuntime runtime{.user_data = user_data, .read_memory = ReadTestMemory, - .userdata = &memory, .read_specialization_memory = ReadTestMemory}; + const SrtRuntime runtime{.user_data = user_data, .userdata = &memory, + .read_specialization_memory = ReadTestMemory}; + FakeGuestMemory fake(ReadTestMemory, &memory); Require(name, "bounded native material plan", plan.capture_specialization_reads && !translated.program.has_address_writes && MaterializeResources(plan, runtime, snapshot, specialization) && @@ -47351,10 +47336,8 @@ void CheckResourcePlanHandoff() { auto plan = ExtractResourcePlan(translated.program); ResourceSnapshot snapshot; ResourceSpecialization specialization; - const SrtRuntime runtime{.user_data = user_data, - .read_memory = ReadTestMemory, - .userdata = &memory, - .read_specialization_memory = ReadTestMemory}; + const SrtRuntime runtime{.user_data = user_data}; + FakeGuestMemory fake(ReadTestMemory, &memory); Require(name, "initial descriptors", MaterializeResources(plan, runtime, snapshot, specialization) && snapshot.buffers.size() == 1 && snapshot.buffers[0].dwords[0] == 0x1000u, diff --git a/tests/SrtWalkerTestMemory.h b/tests/SrtWalkerTestMemory.h new file mode 100644 index 000000000..b792237d4 --- /dev/null +++ b/tests/SrtWalkerTestMemory.h @@ -0,0 +1,113 @@ +#ifndef KYTY_TESTS_SRTWALKERTESTMEMORY_H_ +#define KYTY_TESTS_SRTWALKERTESTMEMORY_H_ + +#include "graphics/shader/recompiler/ir/passes/SrtWalker.h" + +#include + +#include +#include +#include +#include + +#if defined(__linux__) +#include +#elif defined(_WIN32) +#include +#endif + +// Serves SRT walker loads from fake guest addresses: a walker load that faults completes with +// the test reader's DWORD, or with zero like the emulator when the reader fails or is null. +class FakeGuestMemory { +public: + explicit FakeGuestMemory(Libs::Graphics::ShaderRecompiler::IR::SrtMemoryReader reader = nullptr, + void *userdata = nullptr) { + Current() = {reader, userdata}; +#if defined(__linux__) + struct sigaction action {}; + action.sa_sigaction = Handle; + action.sa_flags = SA_SIGINFO; + sigaction(SIGSEGV, &action, &m_previous); +#elif defined(_WIN32) + m_handler = AddVectoredExceptionHandler(1, Handle); +#endif + } + ~FakeGuestMemory() { +#if defined(__linux__) + sigaction(SIGSEGV, &m_previous, nullptr); +#elif defined(_WIN32) + RemoveVectoredExceptionHandler(m_handler); +#endif + Current() = {}; + } + FakeGuestMemory(const FakeGuestMemory &) = delete; + FakeGuestMemory &operator=(const FakeGuestMemory &) = delete; + +private: + using Reader = std::pair; + static Reader &Current() { + static thread_local Reader reader; + return reader; + } + static uint32_t Read(uint64_t address) { + uint32_t word = 0; + const auto [reader, userdata] = Current(); + return reader != nullptr && reader(userdata, address, {&word, 1}) ? word : 0u; + } +#if defined(__linux__) + static void Handle(int, siginfo_t *, void *context) { + auto ®isters = static_cast(context)->uc_mcontext.gregs; + const auto length = + Libs::Graphics::ShaderRecompiler::IR::SrtWalkerLoadLength(registers[REG_RIP]); + if (length == 0) std::abort(); + registers[REG_RAX] = Read(registers[REG_R10]); + registers[REG_RIP] += length; + } + struct sigaction m_previous {}; +#elif defined(_WIN32) + static LONG CALLBACK Handle(EXCEPTION_POINTERS *info) { + auto &context = *info->ContextRecord; + const auto length = Libs::Graphics::ShaderRecompiler::IR::SrtWalkerLoadLength(context.Rip); + if (info->ExceptionRecord->ExceptionCode != EXCEPTION_ACCESS_VIOLATION || length == 0) + return EXCEPTION_CONTINUE_SEARCH; + context.Rax = Read(context.R10); + context.Rip += length; + return EXCEPTION_CONTINUE_EXECUTION; + } + void *m_handler = nullptr; +#endif +}; + +// Evaluates one value through a walker compiled for it, keeping the plan's resource walker. +inline bool EvaluateSrtValue(Libs::Graphics::ShaderRecompiler::IR::ResourcePlan &plan, + const Libs::Graphics::ShaderRecompiler::IR::SrtRuntime &runtime, + Libs::Graphics::ShaderRecompiler::IR::Value value, uint32_t &result) { + using namespace Libs::Graphics::ShaderRecompiler::IR; + const auto *inst = value.Resolve().TryInstruction(); + const auto output = inst != nullptr ? inst->WalkOutput() : UINT32_MAX; + auto code = std::move(plan.srt_code); + auto scratch = std::move(plan.srt_scratch); + const auto user_data = plan.srt_user_data; + const auto captures = plan.srt_captures; + CompileSrtWalker(plan, {&value, 1}); + std::vector flat; + const bool known = WalkSrt(plan, runtime, flat, nullptr) && WalkedValue(plan, value, result); + plan.srt_code = std::move(code); + plan.srt_scratch = std::move(scratch); + plan.srt_user_data = user_data; + plan.srt_captures = captures; + if (inst != nullptr) inst->SetWalkOutput(output); + return known; +} + +// Walks the plan's resources and reads back one descriptor. +inline bool EvaluateSrtDescriptor(const Libs::Graphics::ShaderRecompiler::IR::ResourcePlan &plan, + const Libs::Graphics::ShaderRecompiler::IR::SrtRuntime &runtime, + uint32_t source, + Libs::Graphics::ShaderRecompiler::IR::DescriptorValue &result) { + using namespace Libs::Graphics::ShaderRecompiler::IR; + std::vector flat; + return WalkSrt(plan, runtime, flat, nullptr) && WalkedDescriptor(plan, source, result); +} + +#endif // KYTY_TESTS_SRTWALKERTESTMEMORY_H_ diff --git a/tests/shaderCfgTests.cpp b/tests/shaderCfgTests.cpp index 3d155c8c1..8b912d184 100644 --- a/tests/shaderCfgTests.cpp +++ b/tests/shaderCfgTests.cpp @@ -24,13 +24,13 @@ #include "graphics/shader/recompiler/ir/passes/ResourceTracking.h" #include "graphics/shader/recompiler/ir/passes/ResourceMaterialization.h" #include "graphics/shader/recompiler/ir/passes/ShaderInfoCollection.h" -#include "graphics/shader/recompiler/ir/passes/SrtWalker.h" #include "graphics/shader/recompiler/ir/passes/SsaRewrite.h" #include "graphics/shader/shader.h" #include "graphics/shader/shaderCompiler.h" #include "libs/agc.h" #include "spirv-tools/libspirv.hpp" #include "xxhash.h" +#include "SrtWalkerTestMemory.h" #include #include @@ -135,8 +135,7 @@ struct TestCompileResult { TestCompileResult RecompileForTest( std::span code, const ShaderRecompiler::CompileOptions &options, - ShaderRecompiler::IR::SrtMemoryReader read_memory = nullptr, - void *read_memory_data = nullptr, uint32_t push_data_start_dword = 0) { + uint32_t push_data_start_dword = 0) { auto translated = ShaderRecompiler::TranslateProgram(code, options); auto plan = ShaderRecompiler::IR::ExtractResourcePlan(translated.program); ShaderRecompiler::IR::ResourceSnapshot resources; @@ -144,8 +143,6 @@ TestCompileResult RecompileForTest( const ShaderRecompiler::IR::SrtRuntime runtime{ .user_data = options.user_data, .shader_base = reinterpret_cast(code.data()), - .read_memory = read_memory, - .userdata = read_memory_data, .read_specialization_memory = ReadHostTestMemory, }; Check(ShaderRecompiler::IR::MaterializeResources( @@ -1588,11 +1585,6 @@ void SetImageTestFormat(std::array *data, uint32_t srsrc, (*data)[format_dword] = static_cast(format) << 20u; } -bool ReadZeroTestMemory(void *, uint64_t, std::span values) { - std::ranges::fill(values, 0u); - return true; -} - constexpr uint32_t EncodeVop2(uint32_t opcode, uint32_t dst, uint32_t src0, uint32_t src1) { return ((opcode & 0x3fu) << 25u) | ((dst & 0xffu) << 17u) | @@ -5408,7 +5400,8 @@ void TestNewShaderRecompilerMemoryFamilyTranslation() { options.dump_ir = true; options.user_data = user_data; - auto result = RecompileForTest(shader, options, ReadZeroTestMemory); + FakeGuestMemory zero_memory; + auto result = RecompileForTest(shader, options); Check((result.decoded_dump.find("s_load_dword") != std::string::npos), "new decoder did not decode SMEM dword load"); Check((result.decoded_dump.find("s_buffer_load_dword") != std::string::npos), @@ -9223,7 +9216,7 @@ void TestSharedReturnPreservesDescriptorDominance() { user_data[29] = static_cast(address >> 32u); auto options = MakeCompileOptions(ShaderType::Pixel); options.user_data = user_data; - auto result = RecompileForTest(shader, options, ReadHostTestMemory); + auto result = RecompileForTest(shader, options); Check(!result.program.dispatcher_fallback && result.resources.buffers.size() == 1u && result.resources.buffers[0].dwords[0] == 0x2000u, @@ -9283,7 +9276,7 @@ void TestNestedSelectionPreservesDescriptorSources() { auto options = MakeCompileOptions(ShaderType::Compute); options.user_data_base = 8; options.user_data = user_data; - const auto result = RecompileForTest(shader, options, ReadHostTestMemory); + const auto result = RecompileForTest(shader, options); Check(!result.program.dispatcher_fallback && result.resources.buffers.size() == 4u, "nested descriptor selection lost its structured resource bindings"); for (size_t i = 0; i < result.program.info.buffers.size(); ++i) { @@ -9359,7 +9352,7 @@ void TestSharedExitPreservesNativeDescriptorSources() { user_data[2] = arm; auto options = MakeCompileOptions(ShaderType::Pixel); options.user_data = user_data; - const auto result = RecompileForTest(shader, options, ReadHostTestMemory); + const auto result = RecompileForTest(shader, options); Check(!result.program.dispatcher_fallback && result.resources.buffers.size() == 2u, "shared-exit descriptor selection lost its structured resource bindings"); for (size_t i = 0; i < result.program.info.buffers.size(); ++i) { @@ -9405,7 +9398,7 @@ void TestNativeScalarReadDescriptorPlanning() { user_data[8] = mode; auto options = MakeCompileOptions(ShaderType::Pixel); options.user_data = user_data; - const auto result = RecompileForTest(shader, options, ReadHostTestMemory); + const auto result = RecompileForTest(shader, options); const auto expected = mode == 2u ? table : std::array{}; Check(!result.program.dispatcher_fallback && result.resources.buffers.size() == 1u && std::ranges::equal(result.resources.flattened_srt, expected), @@ -9507,7 +9500,7 @@ void TestNativeGuardedSamplerSource(bool inline_constants) { auto options = MakeCompileOptions(ShaderType::Pixel); options.user_data_base = 8; options.user_data = user_data; - const auto result = RecompileForTest(code, options, ReadHostTestMemory); + const auto result = RecompileForTest(code, options); const auto expected = mode == 2u ? sampler : std::array{}; Check(!result.program.dispatcher_fallback && result.resources.samplers.size() == 1u && result.resources.samplers[0].dword_count == expected.size() && @@ -9615,8 +9608,7 @@ void TestBoundedMaterialBufferStores(bool scalar_key) { ShaderRecompiler::IR::ResourceSnapshot snapshot; ShaderRecompiler::IR::ResourceSpecialization specialization; const ShaderRecompiler::IR::SrtRuntime runtime{ - .user_data = user_data, .read_memory = ReadHostTestMemory, - .read_specialization_memory = ReadHostTestMemory}; + .user_data = user_data, .read_specialization_memory = ReadHostTestMemory}; const auto materialize = [&] { return ShaderRecompiler::IR::MaterializeResources(plan, runtime, snapshot, specialization); }; @@ -9701,7 +9693,7 @@ void TestNativeDescriptorProvenanceKeepsGpuSelection() { options.input_info.compute = &input_info; options.user_data = user_data; for (const auto &shader : {selected, partial, loop}) { - const auto result = RecompileForTest(shader, options, ReadHostTestMemory); + const auto result = RecompileForTest(shader, options); Check(result.program.info.uses_dma && result.program.info.buffers.size() == 1u && result.program.info.buffers[0].written, "GPU-selected scalar descriptor was materialized as a host binding"); @@ -10864,7 +10856,7 @@ void TestMemoryFedScalarLeafCall() { struct Memory { std::array words{}; uint64_t end = base + 0x10c; - bool clean = true; + bool readable = true; } memory; std::copy_n(std::array{0xe0040060u, 0x0010000fu, 8u, 0x5204u}.begin(), 4, memory.words.begin()); @@ -10875,13 +10867,14 @@ void TestMemoryFedScalarLeafCall() { memory.words[66] = EncodeSop1(0x21, 125, 14); const auto read = [](void *data, uint64_t address, std::span out) { const auto& m = *static_cast(data); - if (!m.clean || address < base || address > m.end || out.size_bytes() > m.end - address) + if (!m.readable || address < base || address > m.end || out.size_bytes() > m.end - address) return false; std::memcpy(out.data(), reinterpret_cast(m.words.data()) + address - base, out.size_bytes()); return true; }; - IR::SrtRuntime runtime{.read_memory = read, .userdata = &memory, .read_specialization_memory = read}; + FakeGuestMemory fake(read, &memory); + IR::SrtRuntime runtime{.userdata = &memory, .read_specialization_memory = read}; const auto options = MakeCompileOptions(ShaderType::Compute); const uint32_t gpu_callable[] = {EncodeVop1(0x02, 4, 256), EncodeSmem0(0x09, 14, 0), 4u << 25u, EncodeSop1(0x21, 14, 14), EncodeSopp(0x01)}; @@ -10965,14 +10958,14 @@ void TestMemoryFedScalarLeafCall() { Check(source.revision == 3 && source.reads.back().second == 8, "shorter exact-boundary leaf was not decoded after a failed old-span read"); #if KYTY_PLATFORM != KYTY_PLATFORM_WINDOWS - memory.clean = false; + memory.readable = false; ExpectFatal([&] { (void)RefreshShaderSource(source, runtime); }, "unreadable call dependencies reused stale linked code"); - memory.clean = true; - auto ordinary = runtime; - ordinary.read_specialization_memory = nullptr; - ExpectFatal([&] { (void)RefreshShaderSource(source, ordinary); }, - "call target query accepted an ordinary non-strict memory reader"); + memory.readable = true; + auto no_reader = runtime; + no_reader.read_specialization_memory = nullptr; + ExpectFatal([&] { (void)RefreshShaderSource(source, no_reader); }, + "leaf code was linked without a specialization memory reader"); std::vector escaping(std::begin(shader), std::end(shader) - 1); escaping.insert(escaping.end(), {EncodeVop1(0x01, 1, 14), EncodeMubuf0(0x1c, 0, false), EncodeMubuf1(1, 0, 0), EncodeSopp(0x01)}); @@ -10988,9 +10981,10 @@ void TestMemoryFedScalarLeafCall() { EncodeSMovB32(15, 129), EncodeSop2(0x0f, 14, 128, 128), EncodeSop1(0x21, 14, 14), EncodeSopp(0x01)}; auto pair_source = PrepareShaderSource(high_writer, options); - IR::SrtWalker pair_walker(pair_source.call_targets, runtime); + std::vector flat; uint32_t high = 1; - Check(pair_walker.Evaluate(pair_source.calls.front().target[1], high) && high == 0, + Check(IR::WalkSrt(pair_source.call_targets, runtime, flat, nullptr) && + IR::WalkedValue(pair_source.call_targets, pair_source.calls.front().target[1], high) && high == 0, "scalar pair producer used a stale preceding high-half definition"); const uint32_t joined[] = { EncodeSopp(0x04, 2), EncodeSMovB32(14, 129), EncodeSopp(0x02, 1), @@ -11136,7 +11130,7 @@ void TestMeshExportStorage() { options.back_code = split ? std::span{back} : std::span{}; const auto result = RecompileForTest( split ? std::span{front} : std::span{monolithic}, - options, nullptr, nullptr, push_data_start); + options, push_data_start); CheckSpirvBinaryValidates(result.spirv); const auto &layout = result.program.bindings; Check(layout.UsesPushData() == (push_data_start == PushData::MeshDrawDwordCount) && @@ -11284,13 +11278,11 @@ void TestMergedShaderUserDataSnapshot() { const auto plan = IR::ExtractResourcePlan(program); for (const auto *params : {&first, &second}) { const auto user_data = std::span(params->user_data).first(params->user_data_count); - const IR::SrtRuntime runtime{.user_data = user_data, - .read_memory = ReadHostTestMemory}; - IR::SrtWalker walker(plan, runtime); + const IR::SrtRuntime runtime{.user_data = user_data}; IR::DescriptorValue front_descriptor, back_descriptor; const auto &table = params == &first ? first_table : second_table; - Check(walker.EvaluateDescriptor(program.info.buffers[0].source, front_descriptor) && - walker.EvaluateDescriptor(program.info.buffers[1].source, back_descriptor) && + Check(EvaluateSrtDescriptor(plan, runtime, program.info.buffers[0].source, front_descriptor) && + EvaluateSrtDescriptor(plan, runtime, program.info.buffers[1].source, back_descriptor) && std::equal(user_data.begin() + 8, user_data.end(), front_descriptor.dwords.begin()) && std::equal(table.begin(), table.end(), back_descriptor.dwords.begin()), @@ -13824,7 +13816,7 @@ void TestAutomaticObjectPrimitiveId() { PushData::GraphicsDrawDwords(true, false) == PushData::MeshDrawDwordCount, "graphics draw data reserved an incorrect prefix"); for (const uint32_t start : {prefix, PushData::DwordCount}) { - const auto result = RecompileForTest(shader, options, nullptr, nullptr, start); + const auto result = RecompileForTest(shader, options, start); Check(result.program.bindings.UsesPushData() == (start == prefix), "object ID changed shader-data push/spill selection"); Check(std::ranges::any_of(result.program.info.outputs, [](const StageOutput& output) { @@ -13841,7 +13833,7 @@ void TestAutomaticObjectPrimitiveId() { } #if KYTY_PLATFORM != KYTY_PLATFORM_WINDOWS pixel.target_shader_mask = 0xf0000000u; - ExpectFatal([&] { (void)RecompileForTest(shader, options, nullptr, nullptr, prefix); }, + ExpectFatal([&] { (void)RecompileForTest(shader, options, prefix); }, "automatic RT7 silently overlapped an explicit MRT7 export"); #endif } @@ -14489,33 +14481,6 @@ void CheckFlattenedReadSlots(const ShaderRecompiler::IR::Program &program, } } -bool ReadSrtHostDword(void *, uint64_t address, std::span values) { - if (address == 0 || values.empty()) { - return false; - } - std::memcpy(values.data(), reinterpret_cast(address), values.size_bytes()); - return true; -} - -struct SrtHostRange { - const uint32_t *data; - size_t count; -}; - -bool ReadSrtHostRangeDword(void *userdata, uint64_t address, std::span values) { - const auto *range = static_cast(userdata); - if (range == nullptr || range->data == nullptr || range->count == 0 || values.empty()) { - return false; - } - const auto base = reinterpret_cast(range->data); - const auto size = range->count * sizeof(uint32_t); - if (address < base || values.size_bytes() > size || address - base > size - values.size_bytes()) { - return false; - } - std::memcpy(values.data(), reinterpret_cast(address), values.size_bytes()); - return true; -} - void TestTypedDescriptorRealWideMoveTranslation() { const uint32_t shader[] = { EncodeSop1(0x04, 0, 20), // s_mov_b64 s[0:1], s[20:21] @@ -14581,7 +14546,7 @@ void TestComputeImageFill() { options.input_info.compute = &compute; options.user_data = userdata; enum class Mutation { None, Offset, WrongAxis, VaryingValue, Predicate, ExtraStore, A16 }; - const auto Run = [&](Mutation mutation, bool clean = true) { + const auto Run = [&](Mutation mutation) { auto translated = ShaderRecompiler::TranslateProgram(shader, options); auto &program = translated.program; for (auto *block : program.blocks) { @@ -14618,13 +14583,11 @@ void TestComputeImageFill() { auto plan = ExtractResourcePlan(program); ResourceSnapshot snapshot; ResourceSpecialization specialization; - Check(MaterializeResources(plan, - {.user_data = userdata, .read_memory = ReadHostTestMemory, - .read_specialization_memory = clean ? ReadHostTestMemory : nullptr}, - snapshot, specialization), + // The fill value is a walker load, so it needs no specialization memory reader. + Check(MaterializeResources(plan, {.user_data = userdata}, snapshot, specialization), "captured stencil clear did not materialize"); const auto &fill = snapshot.uniform_fill; - const bool expected = mutation == Mutation::None && clean; + const bool expected = mutation == Mutation::None; Check((fill.kind == UniformFillKind::Image) == expected, "image fill proof accepted an unsafe store or missed the captured clear"); if (expected) { @@ -14634,7 +14597,6 @@ void TestComputeImageFill() { } }; Run(Mutation::None); - Run(Mutation::None, false); for (const auto mutation : {Mutation::Offset, Mutation::WrongAxis, Mutation::VaryingValue, Mutation::Predicate, Mutation::ExtraStore, Mutation::A16}) Run(mutation); @@ -14835,27 +14797,22 @@ void TestGpuProducedWritableDescriptor() { "native GPU-built writable descriptor acquired an unnecessary alias proof"); struct Reads { std::array header{0x26c0u, 0u, 151u}; - uint32_t ordinary = 0; - uint32_t strict = 0; + uint32_t count = 0; } reads; - const SrtRuntime runtime{ - .user_data = user_data, - .read_memory = +[](void *data, uint64_t address, std::span words) { - if (words.size() != 1) return false; + FakeGuestMemory fake(+[](void *data, uint64_t address, std::span words) { auto &reads = *static_cast(data); - const uint32_t index = address == 0x300315028ull ? 0u - : address == 0x30031502cull ? 1u - : address == 0x300315074ull ? 2u : 3u; - if (index == 3u) return false; - ++reads.ordinary; + uint32_t index = 0; + switch (address) { + case 0x300315028ull: index = 0u; break; + case 0x30031502cull: index = 1u; break; + case 0x300315074ull: index = 2u; break; + default: return false; + } + ++reads.count; words[0] = reads.header[index]; return true; - }, - .userdata = &reads, - .read_specialization_memory = +[](void *data, uint64_t, std::span) { - ++static_cast(data)->strict; - return false; // Current GPU bytes are available only through the ordinary reader. - }}; + }, &reads); + const SrtRuntime runtime{.user_data = user_data}; ResourceSnapshot snapshot; ResourceSpecialization specialization; const auto check = [&](uint32_t low, uint32_t high, uint32_t count, uint32_t reads_expected) { @@ -14866,8 +14823,7 @@ void TestGpuProducedWritableDescriptor() { snapshot.buffers[0].dwords[1] == high && snapshot.buffers[0].dwords[2] == count && snapshot.buffers[0].dwords[3] == 0x16204u && - snapshot.specialization_reads.empty() && reads.strict == 0 && - reads.ordinary == reads_expected, + snapshot.specialization_reads.empty() && reads.count == reads_expected, "GPU-produced descriptor lost current scalar data or repeated its reads"); }; check(0x003176c0u, 0x00800003u, 151u, 3u); @@ -14929,25 +14885,29 @@ void TestUniformSelectedWritableDescriptor() { auto plan = ExtractResourcePlan(translated.program); Check(plan.info.buffers.size() == 1 && plan.info.buffers[0].written, "uniform address selection lost its writable buffer"); - const auto read = +[](void *data, uint64_t address, std::span words) { - const bool alternate = *static_cast(data) == 0; - if (words.size() != 1) return false; + // An unavailable read would complete as zero, so the reader counts them. + struct Memory { const uint32_t *flag; uint32_t unavailable = 0; } memory{&user_data[2]}; + FakeGuestMemory fake(+[](void *data, uint64_t address, std::span words) { + auto &memory = *static_cast(data); + const bool alternate = *memory.flag == 0; + bool available = false; switch (address) { - case 0x1040: words[0] = 8u; return alternate; - case 0x1044: words[0] = 1u; return alternate; - case 0x1080: words[0] = 3u; return !alternate; - case 0x109c: words[0] = 2u; return !alternate; - case 0x1084: words[0] = 10u; return true; - default: return false; + case 0x1040: words[0] = 8u; available = alternate; break; + case 0x1044: words[0] = 1u; available = alternate; break; + case 0x1080: words[0] = 3u; available = !alternate; break; + case 0x109c: words[0] = 2u; available = !alternate; break; + case 0x1084: words[0] = 10u; available = true; break; + default: break; } - }; - const SrtRuntime runtime{.user_data = user_data, .read_memory = read, - .userdata = &user_data[2], .read_specialization_memory = read}; + memory.unavailable += !available; + return available; + }, &memory); + const SrtRuntime runtime{.user_data = user_data}; ResourceSnapshot snapshot; ResourceSpecialization specialization; for (const uint32_t flag : {1u, 0u, 1u}) { user_data[2] = flag; - Check(MaterializeResources(plan, runtime, snapshot, specialization), + Check(MaterializeResources(plan, runtime, snapshot, specialization) && memory.unavailable == 0, "uniform selection evaluated an unavailable untaken scalar source"); const auto &descriptor = snapshot.buffers[0].dwords; Check(descriptor[0] == (flag ? 0xe0u : 4u) && @@ -14977,18 +14937,16 @@ void TestUniformSelectedDescriptorLoadAddress() { Check(plan.srt_reads.size() == 4u && plan.info.buffers.size() == 1u && plan.info.buffers[0].written && !plan.capture_specialization_reads, "conditional scalar address duplicated or detached descriptor reads"); - const auto read = +[](void *data, uint64_t address, std::span words) { + FakeGuestMemory fake(+[](void *data, uint64_t address, std::span words) { const auto flag = *static_cast(data); const uint64_t base = flag ? 0x2000u : 0x1000u; const std::array descriptor{0x3000u + flag * 0x1000u, 4u << 16u, 16u, 0x16204u}; - if (words.size() != 1u || address < base || address >= base + 16u || - (address & 3u) != 0u) return false; + if (address < base || address >= base + 16u || (address & 3u) != 0u) return false; words[0] = descriptor[(address - base) / 4u]; return true; - }; - const SrtRuntime runtime{.user_data = user_data, .read_memory = read, - .userdata = &user_data[0]}; + }, &user_data[0]); + const SrtRuntime runtime{.user_data = user_data}; ResourceSnapshot snapshot; ResourceSpecialization specialization; for (const uint32_t flag : {0u, 1u, 0u}) { @@ -15058,7 +15016,7 @@ void TestBoundedScalarMaterialImageKeys() { uint32_t stride = 160; uint32_t records = 409; uint64_t unavailable = 0; - bool clean = true; + bool readable = true; bool mixed = false; bool same_key = false; uint32_t key_reads = 0; @@ -15066,7 +15024,7 @@ void TestBoundedScalarMaterialImageKeys() { } memory; const auto read = +[](void *data, uint64_t address, std::span words) { auto &m = *static_cast(data); - if (!m.clean || (m.unavailable >= address && + if (!m.readable || (m.unavailable >= address && m.unavailable - address < words.size_bytes())) return false; if (words.size() == 1 && address >= 0x10a0u && address < 0x10c0u) { const uint32_t descriptors[] = {0x2000u, m.stride << 16u, m.records, 0x5204u, @@ -15101,6 +15059,7 @@ void TestBoundedScalarMaterialImageKeys() { } return false; }; + FakeGuestMemory fake(read, &memory); const SrtRuntime runtime{.user_data = user_data, .userdata = &memory, .read_specialization_memory = read}; ResourceSnapshot snapshot; @@ -15138,9 +15097,11 @@ void TestBoundedScalarMaterialImageKeys() { Check(MaterializeResources(plan, runtime, snapshot, specialization) && memory.key_reads == 0 && memory.images == 0 && snapshot.images.size() == 2, "empty scalar material loop retained image probes"); - memory = {.clean = false}; - Check(!MaterializeResources(plan, runtime, snapshot, specialization), - "dirty scalar material proof was accepted"); + // Unreadable descriptor and count reads are zero, which leaves the loop empty. + memory = {.readable = false}; + Check(MaterializeResources(plan, runtime, snapshot, specialization) && + memory.key_reads == 0 && memory.images == 0 && snapshot.images.size() == 2, + "unreadable scalar material count did not read as zero"); for (const uint64_t unavailable : {0x2094u, 0x211cu}) { memory = {.unavailable = unavailable}; Check(!MaterializeResources(plan, runtime, snapshot, specialization), @@ -15218,12 +15179,14 @@ void TestTypedDescriptorRealCarryAndScalarLoads() { std::array carry_user_data{}; const uint64_t shader_base = 0x00000012fffffffbull; const uint64_t expected_pc = shader_base + 5u; - ShaderRecompiler::IR::SrtRuntime carry_runtime{carry_user_data, shader_base, - nullptr, nullptr}; - ShaderRecompiler::IR::DescriptorValue carry_value; - Check(ShaderRecompiler::IR::SrtWalker(ShaderRecompiler::IR::ExtractResourcePlan(carry_ir), carry_runtime).EvaluateDescriptor(carry_source_index, carry_value) && - carry_value.dwords[0] == static_cast(expected_pc) && - carry_value.dwords[1] == static_cast(expected_pc >> 32u), + const ShaderRecompiler::IR::SrtRuntime carry_runtime{carry_user_data, shader_base}; + auto carry_plan = ShaderRecompiler::IR::ExtractResourcePlan(carry_ir); + const auto &carry_words = carry_plan.descriptor_sources[carry_source_index].dwords; + uint32_t carry_low = 0, carry_high = 0; + Check(EvaluateSrtValue(carry_plan, carry_runtime, carry_words[0], carry_low) && + EvaluateSrtValue(carry_plan, carry_runtime, carry_words[1], carry_high) && + carry_low == static_cast(expected_pc) && + carry_high == static_cast(expected_pc >> 32u), "S_GETPC shader-base or add/addc carry evaluation was incorrect"); const uint32_t load_shader[] = { @@ -15285,19 +15248,21 @@ void TestTypedDescriptorRealCarryAndScalarLoads() { BuildTypedPlan(inline_sampler_shader, static_cast(std::size(inline_sampler_shader)), inline_sampler_ir); + // The walk also reads the sampled image's user-data descriptor. + auto user_data = ImageTestUserData(); ShaderRecompiler::IR::DescriptorValue sampler; - ShaderRecompiler::IR::SrtRuntime runtime; + const ShaderRecompiler::IR::SrtRuntime runtime{user_data}; Check(inline_sampler_ir.info.samplers.size() == 1 && TypedDescriptorSource(inline_sampler_ir, inline_sampler_ir.info.samplers[0].source) != nullptr && - ShaderRecompiler::IR::SrtWalker(ShaderRecompiler::IR::ExtractResourcePlan(inline_sampler_ir), runtime).EvaluateDescriptor(inline_sampler_ir.info.samplers[0].source, sampler) && + EvaluateSrtDescriptor(ShaderRecompiler::IR::ExtractResourcePlan(inline_sampler_ir), runtime, + inline_sampler_ir.info.samplers[0].source, sampler) && sampler.dwords[0] == 0 && sampler.dwords[1] == 0x00fff000u && sampler.dwords[2] == 0x09500000u && sampler.dwords[3] == 0, "real inline sampler construction was unresolved or evaluated " "incorrectly"); - auto user_data = ImageTestUserData(); auto options = MakeCompileOptions(ShaderType::Pixel); options.user_data = user_data; auto result = RecompileForTest(inline_sampler_shader, options); @@ -15325,9 +15290,8 @@ void TestSrtWalkerRealSmemTranslation() { user_data[8] = static_cast(address); user_data[9] = static_cast(address >> 32u); std::vector flat; - const ShaderRecompiler::IR::SrtRuntime runtime{user_data, 0, ReadSrtHostDword, - nullptr}; - const auto walked = ShaderRecompiler::IR::SrtWalker(ShaderRecompiler::IR::ExtractResourcePlan(ir), runtime).RefreshFlatBuffer(flat); + const ShaderRecompiler::IR::SrtRuntime runtime{user_data}; + const auto walked = ShaderRecompiler::IR::WalkSrt(ShaderRecompiler::IR::ExtractResourcePlan(ir), runtime, flat, nullptr); Check(walked, "SRT walk failed"); Check(flat.size() == table.size() && std::equal(flat.begin(), flat.end(), table.begin()), @@ -15353,10 +15317,9 @@ void TestSrtWalkerVccBaseTranslation() { const auto address = reinterpret_cast(table.data()); user_data[27] = static_cast(address); user_data[28] = static_cast(address >> 32u); - const ShaderRecompiler::IR::SrtRuntime runtime{user_data, 0, ReadSrtHostDword, - nullptr}; + const ShaderRecompiler::IR::SrtRuntime runtime{user_data}; std::vector flat; - Check(ShaderRecompiler::IR::SrtWalker(ShaderRecompiler::IR::ExtractResourcePlan(ir), runtime).RefreshFlatBuffer(flat), "SRT walk failed"); + Check(ShaderRecompiler::IR::WalkSrt(ShaderRecompiler::IR::ExtractResourcePlan(ir), runtime, flat, nullptr), "SRT walk failed"); Check(flat.size() == table.size() && std::equal(flat.begin(), flat.end(), table.begin()), "typed SSA lost an SMEM base copied through VCC"); @@ -15382,17 +15345,16 @@ void TestSrtWalkerRealSBufferTranslation() { user_data[9] = static_cast(address >> 32u); user_data[10] = sizeof(table); std::vector flat; - const ShaderRecompiler::IR::SrtRuntime runtime{user_data, 0, ReadSrtHostDword, - nullptr}; + const ShaderRecompiler::IR::SrtRuntime runtime{user_data}; const auto plan = ShaderRecompiler::IR::ExtractResourcePlan(ir); - const auto walked = ShaderRecompiler::IR::SrtWalker(plan, runtime).RefreshFlatBuffer(flat); + const auto walked = ShaderRecompiler::IR::WalkSrt(plan, runtime, flat, nullptr); Check(walked, "SRT walk failed"); Check(flat.size() == 4 && std::equal(flat.begin(), flat.end(), table.begin()), "real S_BUFFER_LOAD walk did not align offset components independently"); user_data[10] = 3 * sizeof(uint32_t); - const auto bounds_walked = ShaderRecompiler::IR::SrtWalker(plan, runtime).RefreshFlatBuffer(flat); + const auto bounds_walked = ShaderRecompiler::IR::WalkSrt(plan, runtime, flat, nullptr); Check(bounds_walked && flat == std::vector{table[0], table[1], table[2], 0u}, "real S_BUFFER_LOAD walk did not zero only the out-of-bounds DWORD"); CheckFlattenedReadSlots( @@ -15408,7 +15370,7 @@ void TestSrtWalkerRealSBufferTranslation() { static_cast(std::size(negative_shader)), negative_ir); user_data[10] = sizeof(table); - Check(!ShaderRecompiler::IR::SrtWalker(ShaderRecompiler::IR::ExtractResourcePlan(negative_ir), runtime).RefreshFlatBuffer(flat), + Check(!ShaderRecompiler::IR::WalkSrt(ShaderRecompiler::IR::ExtractResourcePlan(negative_ir), runtime, flat, nullptr), "real S_BUFFER_LOAD walk accepted a negative immediate"); } @@ -15443,11 +15405,9 @@ void TestScalarMemorySourcesCapturedBeforeWrites() { if (opcode == 0x0a) { user_data[base + 2u] = sizeof(table); } - SrtHostRange range{table.data(), table.size()}; - const ShaderRecompiler::IR::SrtRuntime runtime{ - user_data, 0, ReadSrtHostRangeDword, &range}; + const ShaderRecompiler::IR::SrtRuntime runtime{user_data}; std::vector flat; - Check(ShaderRecompiler::IR::SrtWalker(ShaderRecompiler::IR::ExtractResourcePlan(ir), runtime).RefreshFlatBuffer(flat), "SRT walk failed"); + Check(ShaderRecompiler::IR::WalkSrt(ShaderRecompiler::IR::ExtractResourcePlan(ir), runtime, flat, nullptr), "SRT walk failed"); Check(flat.size() == table.size() && std::equal(flat.begin(), flat.end(), table.begin()), "overlapping scalar-memory load did not capture its sources before " @@ -16007,7 +15967,7 @@ void TestGraphicsPushConstantPlacement() { constexpr uint32_t PushDataStart = 7; auto placed = - RecompileForTest(shader, options, nullptr, nullptr, PushDataStart); + RecompileForTest(shader, options, PushDataStart); Check(placed.program.bindings.UsesPushData() && placed.program.bindings.push_data_start_dword == PushDataStart && placed.program.bindings.descriptor_counts[static_cast(BindingKind::ShaderData)] == 0, @@ -16016,7 +15976,7 @@ void TestGraphicsPushConstantPlacement() { "SPIR-V push-data block did not use the canonical zero base"); CheckSpirvBinaryValidates(placed.spirv); - auto exact = RecompileForTest(shader, options, nullptr, nullptr, 28); + auto exact = RecompileForTest(shader, options, 28); Check(exact.program.bindings.UsesPushData() && exact.program.bindings.push_data_start_dword == 28 && exact.program.bindings.ShaderDataDwords() == 4, @@ -16026,8 +15986,8 @@ void TestGraphicsPushConstantPlacement() { Check(cursor == 32, "exact-fit shader data advanced the cursor incorrectly"); CheckSpirvBinaryValidates(exact.spirv); - auto spill = RecompileForTest(shader, options, nullptr, nullptr, 29); - auto full = RecompileForTest(shader, options, nullptr, nullptr, 32); + auto spill = RecompileForTest(shader, options, 29); + auto full = RecompileForTest(shader, options, 32); Check(!spill.program.bindings.UsesPushData() && spill.program.bindings.descriptor_counts[static_cast(BindingKind::ShaderData)] == 1 && spill.program.bindings == full.program.bindings && spill.spirv == full.spirv,