diff --git a/src/graphics/host_gpu/renderer/pipeline/descriptors.cpp b/src/graphics/host_gpu/renderer/pipeline/descriptors.cpp index 7969b29ed..68a59f579 100644 --- a/src/graphics/host_gpu/renderer/pipeline/descriptors.cpp +++ b/src/graphics/host_gpu/renderer/pipeline/descriptors.cpp @@ -753,7 +753,7 @@ PreparedBindings RenderExecutor::PrepareBindings(const ShaderStageRuntime& runti KYTY_PROFILER_FUNCTION(); EXIT_IF(!runtime); const auto& program = *runtime.program; - const auto& snapshot = runtime.resources; + const auto& snapshot = *runtime.resources; PreparedBindings prepared; prepared.runtime = &runtime; prepared.images.reserve(program.info.images.size()); @@ -782,7 +782,7 @@ void RenderExecutor::FindBuffers(PreparedBindings& prepared) { KYTY_PROFILER_FUNCTION(); EXIT_IF(prepared.runtime == nullptr || !*prepared.runtime); const auto& program = *prepared.runtime->program; - const auto& snapshot = prepared.runtime->resources; + const auto& snapshot = *prepared.runtime->resources; auto& cache = m_context.GetBufferCache(); prepared.buffer_sources.clear(); @@ -804,7 +804,7 @@ void RenderExecutor::RebindBuffers(PreparedBindings& prepared) { KYTY_PROFILER_FUNCTION(); EXIT_IF(prepared.runtime == nullptr || !*prepared.runtime); const auto& program = *prepared.runtime->program; - const auto& snapshot = prepared.runtime->resources; + const auto& snapshot = *prepared.runtime->resources; const auto& layout = program.bindings; EXIT_IF(prepared.buffer_sources.size() != program.info.buffers.size()); @@ -839,7 +839,7 @@ void RenderExecutor::RebindImages(PreparedBindings& prepared) { KYTY_PROFILER_FUNCTION(); EXIT_IF(prepared.runtime == nullptr || !*prepared.runtime); const auto& program = *prepared.runtime->program; - const auto& snapshot = prepared.runtime->resources; + const auto& snapshot = *prepared.runtime->resources; auto& images = prepared.images; EXIT_IF(images.size() != program.info.images.size()); auto& texture_cache = m_context.GetTextureCache(); diff --git a/src/graphics/host_gpu/renderer/pipeline/pipelineCache.cpp b/src/graphics/host_gpu/renderer/pipeline/pipelineCache.cpp index c1408c47d..26351a6e0 100644 --- a/src/graphics/host_gpu/renderer/pipeline/pipelineCache.cpp +++ b/src/graphics/host_gpu/renderer/pipeline/pipelineCache.cpp @@ -200,8 +200,10 @@ struct PipelineCache::ProgramCache { permutations.reserve(8); } - ShaderRecompiler::IR::ResourcePlan resource_plan; - std::vector permutations; + ShaderRecompiler::IR::ResourcePlan resource_plan; + ShaderRecompiler::IR::ResourceSnapshot resources; + ShaderRecompiler::IR::ResourceSpecialization specialization; + std::vector permutations; }; struct ProgramKeyHash { @@ -280,8 +282,6 @@ struct PipelineCache::ProgramCache { lookup_key.code_size = static_cast(params.code.size()); BuildStageStaticKey(input_info, lookup_key.static_state); auto entry = programs.find(lookup_key); - ShaderRecompiler::IR::ResourceSnapshot resources; - ShaderRecompiler::IR::ResourceSpecialization specialization; const ShaderRecompiler::IR::SrtRuntime runtime { .user_data = params.user_data, .shader_base = params.Base(), @@ -289,18 +289,19 @@ struct PipelineCache::ProgramCache { }; if (entry != programs.end()) { EXIT_IF(!ShaderRecompiler::IR::MaterializeResources( - entry->second.resource_plan, runtime, resources, specialization)); + entry->second.resource_plan, runtime, entry->second.resources, + entry->second.specialization)); if (const auto permutation = std::ranges::find_if( entry->second.permutations, [&](const Permutation& candidate) { const auto& layout = candidate.program.bindings; return layout.push_data_start_dword == ShaderRecompiler::IR::PushData::StartFor( push_data_cursor, layout.ShaderDataDwords()) && - candidate.specialization == specialization; + candidate.specialization == entry->second.specialization; }); permutation != entry->second.permutations.end()) { input_info.stage = {.program = &permutation->program, - .resources = std::move(resources)}; + .resources = &entry->second.resources}; permutation->program.bindings.AdvancePushData(push_data_cursor); return permutation->handle; } @@ -347,15 +348,16 @@ struct PipelineCache::ProgramCache { } auto translated = ShaderRecompiler::TranslateProgram(params.code, options); if (entry == programs.end()) { - auto resource_plan = ShaderRecompiler::IR::ExtractResourcePlan(translated.program); - EXIT_IF(!ShaderRecompiler::IR::MaterializeResources(resource_plan, runtime, resources, - specialization)); - entry = programs.try_emplace(lookup_key, std::move(resource_plan)).first; + entry = programs.try_emplace(lookup_key, + ShaderRecompiler::IR::ExtractResourcePlan(translated.program)).first; + EXIT_IF(!ShaderRecompiler::IR::MaterializeResources( + entry->second.resource_plan, runtime, entry->second.resources, + entry->second.specialization)); } entry->second.permutations.push_back(CompilePermutation( - params, options, std::move(translated), std::move(specialization), push_data_cursor)); + params, options, std::move(translated), entry->second.specialization, push_data_cursor)); const auto& permutation = entry->second.permutations.back(); - input_info.stage = {.program = &permutation.program, .resources = std::move(resources)}; + input_info.stage = {.program = &permutation.program, .resources = &entry->second.resources}; permutation.program.bindings.AdvancePushData(push_data_cursor); std::array(ShaderType::TessellationEvaluation) + 1> counts {}; diff --git a/src/graphics/host_gpu/renderer/pipeline/shaderResourceBarrier.cpp b/src/graphics/host_gpu/renderer/pipeline/shaderResourceBarrier.cpp index 4370aed19..f0f7d4557 100644 --- a/src/graphics/host_gpu/renderer/pipeline/shaderResourceBarrier.cpp +++ b/src/graphics/host_gpu/renderer/pipeline/shaderResourceBarrier.cpp @@ -90,7 +90,7 @@ vk::BufferMemoryBarrier MakeGdsDependency(vk::Buffer buffer) { bool HasShaderBufferWrites(const ShaderStageRuntime& runtime) { EXIT_IF(!runtime); const auto& program = *runtime.program; - const auto& resources = runtime.resources; + const auto& resources = *runtime.resources; EXIT_IF(resources.buffers.size() != program.info.buffers.size()); bool has_writes = false; for (uint32_t i = 0; i < program.info.buffers.size(); i++) { diff --git a/src/graphics/host_gpu/renderer/renderCompute.cpp b/src/graphics/host_gpu/renderer/renderCompute.cpp index 1ef9ee345..c057e3d14 100644 --- a/src/graphics/host_gpu/renderer/renderCompute.cpp +++ b/src/graphics/host_gpu/renderer/renderCompute.cpp @@ -51,7 +51,7 @@ static bool FillSourcesDisjoint(std::span ResolveDrawOffsets(uint32_t index_offset, EXIT_IF(!vs_input_info.stage); const auto& program = *vs_input_info.stage.program; - const auto& resources = vs_input_info.stage.resources; + const auto& resources = *vs_input_info.stage.resources; if (index_offset == 0 && program.info.vertex_offset_sgpr >= static_cast(program.user_data_base)) { const auto index = diff --git a/src/graphics/shader/recompiler/ir/ShaderIR.h b/src/graphics/shader/recompiler/ir/ShaderIR.h index a97dc4564..fa8535f87 100644 --- a/src/graphics/shader/recompiler/ir/ShaderIR.h +++ b/src/graphics/shader/recompiler/ir/ShaderIR.h @@ -507,8 +507,8 @@ struct UniformFillPlan { std::array values; }; -// Immutable runtime resource analysis retained by the shader cache. It owns descriptor/SRT, -// uniform condition and fill values without retaining translated blocks. +// Resource analysis retained by the shader cache. It owns immutable descriptor/SRT, +// condition and fill values without translated blocks, plus reusable evaluation scratch. struct ResourcePlan { struct EvaluationContext { struct Entry { @@ -536,7 +536,6 @@ struct ResourcePlan { std::vector memory_info; std::vector descriptor_sources; std::vector control_flow; - std::vector materialization_sources; std::vector srt_reads; std::vector clean_flat_slots; bool requires_specialization_memory = false; @@ -544,10 +543,14 @@ struct ResourcePlan { bool resource_tracking_complete = false; ShaderInfo info; UniformFillPlan uniform_fill; - // GPU-thread scratch. Each nested clean/EXEC evaluation has its own reusable memo. + // GPU-thread scratch for nested clean/EXEC memos, activity and material keys. mutable std::deque evaluation_contexts; mutable uint32_t evaluation_value_count = 0; mutable uint32_t evaluation_depth = 0; + mutable std::vector active_sources; + mutable std::vector visited_blocks; + mutable std::vector pending_blocks; + mutable std::vector material_keys; }; struct Program: ResourcePlan { diff --git a/src/graphics/shader/recompiler/ir/passes/ResourceMaterialization.cpp b/src/graphics/shader/recompiler/ir/passes/ResourceMaterialization.cpp index 18b922ba0..10867632e 100644 --- a/src/graphics/shader/recompiler/ir/passes/ResourceMaterialization.cpp +++ b/src/graphics/shader/recompiler/ir/passes/ResourceMaterialization.cpp @@ -21,18 +21,6 @@ namespace { constexpr uint64_t AddressMask = 0x0000ffffffffffffull; constexpr uint64_t MaxIndirectImageProbes = 65536u; -struct IndirectImage { - uint32_t resource = 0; - std::vector keys; - std::vector candidates; - std::vector descriptors; -}; - -struct MaterializedSnapshot { - ResourceSnapshot resources; - std::vector indirect_images; -}; - bool SpecializationFail(std::string_view message) { std::fprintf(stderr, "shader resource specialization failed: %.*s\n", static_cast(message.size()), message.data()); @@ -198,159 +186,96 @@ bool ReadScalarBufferWord(const ShaderBufferResource& descriptor, uint32_t dynam return true; } -bool MaterializeIndirectImage(const DescriptorSource::IndirectImage& indirect, - const DescriptorValue& material_value, - const DescriptorValue& heap_value, bool r128, - const SrtRuntime& runtime, IndirectImage& result) { +bool MaterializeIndirectImage(const ResourcePlan& program, + const DescriptorSource::IndirectImage& indirect, + const DescriptorValue& material_value, + const DescriptorValue& heap_value, uint32_t image_index, + const SrtRuntime& runtime, ResourceSnapshot& snapshot, + ResourceSpecialization& specialization) { ShaderBufferResource material; ShaderBufferResource heap; if (!DecodeBufferDescriptor(material_value, material) || - !DecodeBufferDescriptor(heap_value, heap)) { + !DecodeBufferDescriptor(heap_value, heap) || material.Stride() != indirect.selector_stride) { return false; } - if (material.Stride() != indirect.selector_stride) { - return false; - } - - // S_BUFFER_LOAD ignores vector-buffer swizzle/add-thread fields. The shader computes the - // record stride explicitly; enumerate every wrapped 32-bit offset that can pass bounds. - const auto period = uint64_t {1} << 32u; - const auto step = std::gcd(indirect.selector_stride, period); - const auto residue = static_cast(indirect.selector_offset) % step; - const auto size = material.GetSize(); - const auto limit = std::min(UINT32_MAX, size + 3u); + // Enumerate every wrapped scalar-buffer offset that can pass the descriptor bounds. + const auto step = std::gcd(indirect.selector_stride, uint64_t {1} << 32u); + const auto residue = static_cast(indirect.selector_offset) % step; + const auto limit = std::min(UINT32_MAX, material.GetSize() + 3u); const auto probe_count = residue <= limit ? (limit - residue) / step + 1u : 0u; if (probe_count > MaxIndirectImageProbes) { return false; } - - std::vector keys {0u}; - std::unordered_set seen {0u}; + auto& keys = program.material_keys; + keys.clear(); keys.reserve(static_cast(probe_count) + 1u); - seen.reserve(static_cast(probe_count) + 1u); + keys.push_back(0u); for (uint64_t offset = residue; offset <= limit && probe_count != 0u; offset += step) { uint32_t key = 0; if (!ReadScalarBufferWord(material, static_cast(offset), 0u, runtime, key)) { return false; } - if (seen.insert(key).second) { - keys.push_back(key); - } + keys.push_back(key); if (limit - offset < step) { break; } } + std::ranges::sort(keys); + keys.erase(std::unique(keys.begin(), keys.end()), keys.end()); - IndirectImage next; - next.keys = std::move(keys); - next.candidates.reserve(next.keys.size()); - next.descriptors.reserve( - std::min(next.keys.size(), static_cast(ShaderInfo::MaxImages))); - for (const auto key: next.keys) { + const auto children_begin = snapshot.images.size(); + const auto mapping_offset = snapshot.flattened_srt.size(); + const auto root_image = specialization.images[image_index]; + snapshot.flattened_srt.resize(mapping_offset + 1u + keys.size() * 2u); + snapshot.flattened_srt[mapping_offset] = static_cast(keys.size()); + for (uint32_t entry = 0; entry < keys.size(); ++entry) { + const auto key = keys[entry]; DescriptorValue candidate; - candidate.dword_count = 8u; + candidate.dword_count = 8u; const auto heap_offset = key << 5u; - for (uint32_t dword = 0; dword < candidate.dword_count; dword++) { + for (uint32_t dword = 0; dword < candidate.dword_count; ++dword) { if (!ReadScalarBufferWord(heap, heap_offset, dword * sizeof(uint32_t), runtime, candidate.dwords[dword])) { return false; } } - if (NullImageDescriptor(candidate) || !ValidImageDescriptor(candidate, r128)) { + if (NullImageDescriptor(candidate) || + !ValidImageDescriptor(candidate, program.info.images[image_index].r128)) { candidate.dwords.fill(0); } - const auto found = std::ranges::find(next.descriptors, candidate); - if (found == next.descriptors.end()) { - if (next.descriptors.size() >= ShaderInfo::MaxImages) { - return false; + uint32_t ordinal = 0; + if (entry == 0) { + snapshot.images[image_index] = candidate; + } else if (snapshot.images[image_index] != candidate) { + const auto found = std::find(snapshot.images.begin() + children_begin, + snapshot.images.end(), candidate); + ordinal = static_cast(found - snapshot.images.begin() - children_begin + 1u); + if (found == snapshot.images.end()) { + if (snapshot.images.size() >= ShaderInfo::MaxImages) { + return false; + } + snapshot.images.push_back(candidate); + auto child = root_image; + child.indirect_root = image_index; + specialization.images.push_back(child); } - next.descriptors.push_back(candidate); - next.candidates.push_back(static_cast(next.descriptors.size() - 1u)); - } else { - next.candidates.push_back(static_cast(found - next.descriptors.begin())); } + snapshot.flattened_srt[mapping_offset + 1u + entry * 2u] = key; + snapshot.flattened_srt[mapping_offset + 2u + entry * 2u] = ordinal; + } + if (snapshot.images.size() == children_begin) { + snapshot.flattened_srt.resize(mapping_offset); + } else { + auto& root = specialization.images[image_index]; + root.indirect_root = image_index; + root.indirect_mapping_offset = static_cast(mapping_offset); + root.indirect_search_iterations = std::bit_width(keys.size()); } - result = std::move(next); return true; } } // namespace -static bool MaterializeSnapshot(const ResourcePlan& program, const SrtRuntime& runtime, - MaterializedSnapshot& snapshot) { - if (!program.resource_tracking_complete) { - return false; - } - - if (program.requires_specialization_memory && runtime.read_specialization_memory == nullptr) { - return false; - } - std::vector values; - std::vector flattened_srt; - std::vector active_sources; - if (!EvaluateRuntimeSources(program, program.materialization_sources, runtime, values, - flattened_srt, program.clean_flat_slots, active_sources)) { - return false; - } - - auto& next = snapshot.resources; - const auto& fill = program.uniform_fill; - const auto words = fill.fill.words; - std::array stored {}; - if (words != 0 && - EvaluateUniformValues(program, std::span(fill.values).first(words), runtime, - std::span(stored).first(words)) && - std::all_of(stored.begin(), stored.begin() + words, - [&](uint32_t value) { return value == stored[0]; })) { - next.uniform_fill = fill.fill; - next.uniform_fill.value = stored[0]; - } - auto cursor = values.begin(); - next.buffers.assign(cursor, cursor + program.info.buffers.size()); - cursor += program.info.buffers.size(); - next.flattened_srt = std::move(flattened_srt); - next.images.resize(program.info.images.size()); - for (uint32_t image_index = 0; image_index < program.info.images.size(); image_index++) { - const auto& image = program.info.images[image_index]; - const auto* source = Source(program, image.source); - if (source != nullptr && source->indirect_image.has_value()) { - if (!active_sources[image.source]) { - next.images[image_index].dword_count = 8u; - continue; - } - const std::array requests {source->indirect_image->material_source, - source->indirect_image->heap_source}; - SrtRuntime clean_runtime = runtime; - clean_runtime.read_memory = runtime.read_specialization_memory; - std::vector tables; - if (!EvaluateDescriptorSources(program, requests, clean_runtime, tables)) { - return false; - } - const auto& material = tables[0]; - const auto& heap = tables[1]; - IndirectImage table; - if (!MaterializeIndirectImage(*source->indirect_image, material, heap, image.r128, - runtime, table)) { - return false; - } - next.images[image_index] = table.descriptors[table.candidates[0]]; - if (table.descriptors.size() > 1u) { - table.resource = image_index; - snapshot.indirect_images.push_back(std::move(table)); - } - } else { - auto descriptor = *cursor++; - if (!ValidImageDescriptor(descriptor, image.r128)) { - descriptor.dwords.fill(0); - } - next.images[image_index] = descriptor; - } - } - next.samplers.assign(cursor, cursor + program.info.samplers.size()); - next.user_data.assign(runtime.user_data.begin(), runtime.user_data.end()); - return true; -} - struct SamplerPlan { std::array point_sampler {}; uint32_t sampler_count = 0; @@ -390,68 +315,12 @@ private: template bool BuildSamplerPlan(const ShaderInfo& base, const Images& images, SamplerPlan& plan); -static bool BuildResourceSpecialization(const ResourcePlan& program, MaterializedSnapshot snapshot, - ResourceSnapshot& specialized_snapshot, +static bool BuildResourceSpecialization(const ResourcePlan& program, ResourceSnapshot& snapshot, ResourceSpecialization& specialization) { - auto next_snapshot = std::move(snapshot.resources); - ResourceSpecialization next_specialization; - next_specialization.buffers.reserve(program.info.buffers.size()); - size_t image_count = program.info.images.size(); - size_t mapping_words = 0; - for (const auto& table: snapshot.indirect_images) { - if (table.resource >= program.info.images.size() || table.descriptors.size() < 2u || - image_count + table.descriptors.size() - 1u > ShaderInfo::MaxImages) { - return SpecializationFail( - "indirect image candidates exceed the dense image resource limit"); - } - image_count += table.descriptors.size() - 1u; - mapping_words += 1u + table.keys.size() * 2u; - } - next_snapshot.images.reserve(image_count); - next_snapshot.flattened_srt.reserve(next_snapshot.flattened_srt.size() + mapping_words); - next_specialization.images.reserve(image_count); - for (const auto& image: program.info.images) { - next_specialization.images.push_back({ - .numeric_class = image.numeric_class, - .dimension = image.dimension, - .mip_count = image.mip_count, - .conversion_format = image.conversion_format, - .shader_swizzle = image.shader_swizzle, - .indirect_root = image.indirect_root, - .indirect_mapping_offset = image.indirect_mapping_offset, - .indirect_search_iterations = image.indirect_search_iterations, - .cube = image.cube, - }); - } - for (const auto& table: snapshot.indirect_images) { - const auto root_image = next_specialization.images[table.resource]; - for (uint32_t candidate = 1; candidate < table.descriptors.size(); candidate++) { - auto image = root_image; - image.indirect_root = table.resource; - next_specialization.images.push_back(image); - next_snapshot.images.push_back(table.descriptors[candidate]); - } - auto& root = next_specialization.images[table.resource]; - root.indirect_root = table.resource; - root.indirect_mapping_offset = static_cast(next_snapshot.flattened_srt.size()); - root.indirect_search_iterations = std::bit_width(table.keys.size()); - next_snapshot.flattened_srt.resize(next_snapshot.flattened_srt.size() + 1u + - table.keys.size() * 2u); - std::vector order(table.keys.size()); - std::iota(order.begin(), order.end(), 0u); - std::ranges::sort(order, {}, [&](uint32_t index) { return table.keys[index]; }); - next_snapshot.flattened_srt[root.indirect_mapping_offset] = - static_cast(table.keys.size()); - for (uint32_t entry = 0; entry < order.size(); entry++) { - const auto source = order[entry]; - const auto offset = root.indirect_mapping_offset + 1u + entry * 2u; - next_snapshot.flattened_srt[offset] = table.keys[source]; - next_snapshot.flattened_srt[offset + 1] = table.candidates[source]; - } - next_snapshot.images[table.resource] = table.descriptors[0]; - } + specialization.buffers.clear(); + specialization.buffers.reserve(program.info.buffers.size()); for (uint32_t i = 0; i < program.info.buffers.size(); i++) { - auto& descriptor_value = next_snapshot.buffers[i]; + auto& descriptor_value = snapshot.buffers[i]; ShaderBufferResource descriptor; if (!DecodeBufferDescriptor(descriptor_value, descriptor)) { return SpecializationFail(fmt::format("buffer descriptor {} has invalid width", i)); @@ -468,7 +337,7 @@ static bool BuildResourceSpecialization(const ResourcePlan& program, Materialize } else if (!swizzle) { packed_stride &= ~(3u << 16u); } - next_specialization.buffers.push_back({ + specialization.buffers.push_back({ .packed_stride = packed_stride, .descriptor_format = program.info.buffers[i].formatted ? descriptor.Format() @@ -477,9 +346,9 @@ static bool BuildResourceSpecialization(const ResourcePlan& program, Materialize program.info.buffers[i].formatted ? descriptor.DstSelXYZW() : DstSel(4, 5, 6, 7), }); } - for (uint32_t i = 0; i < next_specialization.images.size(); i++) { - const auto& descriptor = next_snapshot.images[i]; - auto& image = next_specialization.images[i]; + for (uint32_t i = 0; i < specialization.images.size(); i++) { + const auto& descriptor = snapshot.images[i]; + auto& image = specialization.images[i]; const auto base_index = i < program.info.images.size() ? i : image.indirect_root; if (base_index >= program.info.images.size()) { return SpecializationFail(fmt::format("image resource {} has an invalid root", i)); @@ -554,42 +423,42 @@ static bool BuildResourceSpecialization(const ResourcePlan& program, Materialize static_cast(format))); } } - for (uint32_t root_index = 0; root_index < next_specialization.images.size(); root_index++) { - auto& root = next_specialization.images[root_index]; + for (uint32_t root_index = 0; root_index < specialization.images.size(); root_index++) { + auto& root = specialization.images[root_index]; if (root.indirect_root != root_index) { continue; } - const auto key_count = root.indirect_mapping_offset < next_snapshot.flattened_srt.size() - ? next_snapshot.flattened_srt[root.indirect_mapping_offset] + const auto key_count = root.indirect_mapping_offset < snapshot.flattened_srt.size() + ? snapshot.flattened_srt[root.indirect_mapping_offset] : 0u; if (root.indirect_search_iterations == 0u || key_count < 2u || static_cast(root.indirect_mapping_offset) + 1u + static_cast(key_count) * 2u > - next_snapshot.flattened_srt.size()) { + snapshot.flattened_srt.size()) { return SpecializationFail("indirect image specialization has an invalid key mapping"); } uint32_t exemplar = ImageResource::NoIndirectImage; uint32_t resource_count = 0; - for (uint32_t resource = 0; resource < next_specialization.images.size(); resource++) { - if (next_specialization.images[resource].indirect_root != root_index) { + for (uint32_t resource = 0; resource < specialization.images.size(); resource++) { + if (specialization.images[resource].indirect_root != root_index) { continue; } resource_count++; if (exemplar == ImageResource::NoIndirectImage && - !NullImageDescriptor(next_snapshot.images[resource])) { + !NullImageDescriptor(snapshot.images[resource])) { exemplar = resource; } } if (resource_count < 2u || exemplar == ImageResource::NoIndirectImage) { return SpecializationFail("indirect image specialization has no typed candidate"); } - const auto& image_class = next_specialization.images[exemplar]; - for (uint32_t candidate = 0; candidate < next_specialization.images.size(); candidate++) { - auto& image = next_specialization.images[candidate]; + const auto& image_class = specialization.images[exemplar]; + for (uint32_t candidate = 0; candidate < specialization.images.size(); candidate++) { + auto& image = specialization.images[candidate]; if (image.indirect_root != root_index) { continue; } - if (NullImageDescriptor(next_snapshot.images[candidate])) { + if (NullImageDescriptor(snapshot.images[candidate])) { image.numeric_class = image_class.numeric_class; image.dimension = image_class.dimension; image.mip_count = image_class.mip_count; @@ -610,18 +479,16 @@ static bool BuildResourceSpecialization(const ResourcePlan& program, Materialize } } SamplerPlan sampler_plan; - if (!BuildSamplerPlan(program.info, next_specialization.images, sampler_plan)) { + if (!BuildSamplerPlan(program.info, specialization.images, sampler_plan)) { return SpecializationFail("specialized sampler layout exceeds its resource limit"); } for (uint32_t index = 0; index < program.info.samplers.size(); index++) { const auto target = sampler_plan.point_sampler[index]; if (target != UINT32_MAX && target >= program.info.samplers.size()) { - next_snapshot.samplers.push_back(next_snapshot.samplers[index]); + snapshot.samplers.push_back(snapshot.samplers[index]); } } - ImageRemap(next_specialization).Apply(next_snapshot.images); - specialization = std::move(next_specialization); - specialized_snapshot = std::move(next_snapshot); + ImageRemap(specialization).Apply(snapshot.images); return true; } @@ -935,28 +802,13 @@ ResourcePlan ExtractResourcePlan(const Program& program) { for (uint32_t i = 0; i < plan.uniform_fill.fill.words; ++i) { plan.uniform_fill.values[i] = Clone(plan.uniform_fill.values[i]); } - plan.materialization_sources.reserve(plan.info.buffers.size() + plan.info.images.size() + - plan.info.samplers.size()); - for (const auto& buffer: plan.info.buffers) { - plan.materialization_sources.push_back(buffer.source); - } - for (const auto& image: plan.info.images) { - const auto* source = Source(plan, image.source); - if (source != nullptr && source->indirect_image.has_value()) { - plan.requires_specialization_memory = true; - } else { - plan.materialization_sources.push_back(image.source); - } - } - for (const auto& sampler: plan.info.samplers) { - plan.materialization_sources.push_back(sampler.source); - } plan.clean_flat_slots.resize(plan.srt_reads.size()); for (const auto& image: plan.info.images) { const auto* source = Source(plan, image.source); if (source == nullptr || !source->indirect_image.has_value()) { continue; } + plan.requires_specialization_memory = true; MarkCleanFlatSlots(plan, Source(plan, source->indirect_image->material_source), plan.clean_flat_slots); MarkCleanFlatSlots(plan, Source(plan, source->indirect_image->heap_source), @@ -967,11 +819,98 @@ ResourcePlan ExtractResourcePlan(const Program& program) { bool MaterializeResources(const ResourcePlan& program, const SrtRuntime& runtime, ResourceSnapshot& snapshot, ResourceSpecialization& specialization) { - MaterializedSnapshot materialized; - if (!MaterializeSnapshot(program, runtime, materialized)) { + if (!program.resource_tracking_complete || + (program.requires_specialization_memory && runtime.read_specialization_memory == nullptr)) { return false; } - return BuildResourceSpecialization(program, std::move(materialized), snapshot, specialization); + SrtWalker clean(program, CleanRuntime(runtime)); + SrtWalker walker(program, runtime, program.clean_flat_slots, &clean); + const auto active = clean.FindActiveSources(); + if (!walker.RefreshFlatBuffer(snapshot.flattened_srt)) { + return false; + } + snapshot.uniform_fill = {}; + const auto& fill = program.uniform_fill; + const auto words = fill.fill.words; + std::array stored {}; + bool uniform_fill = words != 0; + for (uint32_t i = 0; i < words && uniform_fill; ++i) { + uniform_fill = clean.Evaluate(fill.values[i], stored[i]) && stored[i] == stored[0]; + } + if (uniform_fill) { + snapshot.uniform_fill = fill.fill; + snapshot.uniform_fill.value = stored[0]; + } + const auto evaluate = [&](uint32_t source, DescriptorValue& value) { + if (source >= program.descriptor_sources.size()) { + return false; + } + if (active.empty() || active[source]) { + return walker.EvaluateDescriptor(source, value); + } + value = {}; + value.dword_count = program.descriptor_sources[source].dword_count; + return true; + }; + snapshot.buffers.resize(program.info.buffers.size()); + for (uint32_t i = 0; i < program.info.buffers.size(); ++i) { + if (!evaluate(program.info.buffers[i].source, snapshot.buffers[i])) { + return false; + } + } + snapshot.images.resize(program.info.images.size()); + specialization.images.clear(); + specialization.images.reserve(program.info.images.size()); + for (const auto& image: program.info.images) { + specialization.images.push_back({ + .numeric_class = image.numeric_class, + .dimension = image.dimension, + .mip_count = image.mip_count, + .conversion_format = image.conversion_format, + .shader_swizzle = image.shader_swizzle, + .indirect_root = image.indirect_root, + .indirect_mapping_offset = image.indirect_mapping_offset, + .indirect_search_iterations = image.indirect_search_iterations, + .cube = image.cube, + }); + } + for (uint32_t i = 0; i < program.info.images.size(); ++i) { + const auto& image = program.info.images[i]; + const auto* source = Source(program, image.source); + if (source == nullptr) { + return false; + } + if (source->indirect_image.has_value()) { + snapshot.images[i] = {.dword_count = 8u}; + if (!active.empty() && !active[image.source]) { + continue; + } + const auto& indirect = *source->indirect_image; + DescriptorValue material; + DescriptorValue heap; + if (!clean.EvaluateDescriptor(indirect.material_source, material) || + !clean.EvaluateDescriptor(indirect.heap_source, heap) || + !MaterializeIndirectImage(program, indirect, material, heap, i, runtime, snapshot, + specialization)) { + return false; + } + } else { + if (!evaluate(image.source, snapshot.images[i])) { + return false; + } + if (!ValidImageDescriptor(snapshot.images[i], image.r128)) { + snapshot.images[i].dwords.fill(0); + } + } + } + snapshot.samplers.resize(program.info.samplers.size()); + for (uint32_t i = 0; i < program.info.samplers.size(); ++i) { + if (!evaluate(program.info.samplers[i].source, snapshot.samplers[i])) { + return false; + } + } + snapshot.user_data.assign(runtime.user_data.begin(), runtime.user_data.end()); + return BuildResourceSpecialization(program, snapshot, specialization); } void ApplyResourceSpecialization(Program& program, const ResourceSpecialization& specialization) { diff --git a/src/graphics/shader/recompiler/ir/passes/ResourceMaterialization.h b/src/graphics/shader/recompiler/ir/passes/ResourceMaterialization.h index f0b686ca3..812e633cb 100644 --- a/src/graphics/shader/recompiler/ir/passes/ResourceMaterialization.h +++ b/src/graphics/shader/recompiler/ir/passes/ResourceMaterialization.h @@ -39,8 +39,7 @@ struct ResourceSpecialization { // its values and is independent of the translated shader CFG. ResourcePlan ExtractResourcePlan(const Program& program); -// Resolves and specializes the immutable resource plan in one transaction. On failure both -// destinations are unchanged. +// Refreshes cached resources and specialization in place. A failed refresh must not be used. bool MaterializeResources(const ResourcePlan& program, const SrtRuntime& runtime, ResourceSnapshot& snapshot, ResourceSpecialization& specialization); diff --git a/src/graphics/shader/recompiler/ir/passes/SrtWalker.cpp b/src/graphics/shader/recompiler/ir/passes/SrtWalker.cpp index 1a54aeee7..5bf69fbd2 100644 --- a/src/graphics/shader/recompiler/ir/passes/SrtWalker.cpp +++ b/src/graphics/shader/recompiler/ir/passes/SrtWalker.cpp @@ -11,9 +11,6 @@ #include namespace Libs::Graphics::ShaderRecompiler::IR { -namespace { - -constexpr uint64_t AddressMask = 0x0000ffffffffffffull; SrtRuntime CleanRuntime(SrtRuntime runtime) { runtime.read_memory = runtime.read_specialization_memory != nullptr @@ -22,6 +19,10 @@ SrtRuntime CleanRuntime(SrtRuntime runtime) { return runtime; } +namespace { + +constexpr uint64_t AddressMask = 0x0000ffffffffffffull; + const char* StageName(ShaderType stage) { switch (stage) { case ShaderType::Vertex: return "vertex"; @@ -457,640 +458,604 @@ private: std::vector m_patches; }; -class Evaluator { -public: - Evaluator(const ResourcePlan& program, const SrtRuntime& runtime, - std::span clean_flat_slots = {}, Evaluator* clean_evaluator = nullptr, - Value active_mask = {}) - : m_program(program), m_runtime(runtime), m_clean_flat_slots(clean_flat_slots), - m_clean_evaluator(clean_evaluator), m_active_mask(active_mask.Resolve()), - m_context(AcquireContext(program)) {} +} // namespace - ~Evaluator() { --m_program.evaluation_depth; } - Evaluator(const Evaluator&) = delete; - Evaluator& operator=(const Evaluator&) = delete; +SrtWalker::SrtWalker(const ResourcePlan& program, const SrtRuntime& runtime, + std::span clean_flat_slots, SrtWalker* clean_evaluator, + Value active_mask) + : m_program(program), m_runtime(runtime), m_clean_flat_slots(clean_flat_slots), + m_clean_evaluator(clean_evaluator), m_active_mask(active_mask.Resolve()), + m_context(AcquireContext(program)) {} - bool Evaluate(Value value, uint32_t& result) { - uint64_t wide = 0; - if (!EvaluateWide(value, wide)) { - return false; - } - result = static_cast(wide); - return true; - } +SrtWalker::~SrtWalker() { --m_program.evaluation_depth; } -private: - static ResourcePlan::EvaluationContext& AcquireContext(const ResourcePlan& program) { - if (program.evaluation_depth == program.evaluation_contexts.size()) { - program.evaluation_contexts.emplace_back(); - } - auto& context = program.evaluation_contexts[program.evaluation_depth++]; - context.generation += 2; - return context; - } - - static float Float32(uint64_t bits) { - return std::bit_cast(static_cast(bits)); - } - - bool EvaluateWide(Value value, uint64_t& result) { - value = value.Resolve(); - if (value.IsImmediate()) { - switch (value.GetType()) { - case Type::U1: result = value.U1(); return true; - case Type::U8: result = value.U8(); return true; - case Type::U16: result = value.U16(); return true; - case Type::U32: result = value.U32(); return true; - case Type::U64: result = value.U64(); return true; - case Type::F32: result = std::bit_cast(value.F32Value()); return true; - default: return false; - } - } - auto* inst = value.TryInstruction(); - if (inst == nullptr) { - return false; - } - if (!m_active_mask.IsEmpty() && IsRuntimeSelect(inst->GetOpcode()) && - inst->NumArgs() == 3 && inst->Arg(0).Resolve() == m_active_mask) { - return EvaluateWide(inst->Arg(1), result); - } - const auto index = inst->EvaluationIndex(m_program.evaluation_value_count); - if (index >= m_context.values.size()) { - m_context.values.resize(m_program.evaluation_value_count); - } - if (m_context.values[index].generation == m_context.generation) { - result = m_context.values[index].value; - return true; - } - // The low generation bit marks an instruction that is still being evaluated. - if (m_context.values[index].generation == (m_context.generation | 1u)) { - return false; - } - m_context.values[index].generation = m_context.generation | 1u; - uint64_t out = 0; - const bool evaluated = EvaluateInst(*inst, out); - // Recursive evaluation may grow the dense memo vector. - auto& memo = m_context.values[index]; - if (!evaluated) { - memo.generation = 0; - return false; - } - memo.value = out; - memo.generation = m_context.generation; - result = out; - return true; - } - - bool Arg(const Inst& inst, size_t index, uint64_t& result) { - return EvaluateWide(inst.Arg(index), result); - } - - bool EvaluatePhi(const Inst& inst, uint64_t& result) { - const auto value = ResolveInvariantPhi(m_program, Value(const_cast(&inst))); - return !value.IsEmpty() && EvaluateWide(value, result); - } - - bool EvaluateExtract(const Inst& inst, uint64_t& result) { - const auto index = inst.Arg(1).Resolve(); - if (!index.IsImmediate() || index.GetType() != Type::U32) { - return false; - } - const auto component = index.U32(); - if (component >= 2u) { - return false; - } - if (inst.GetOpcode() == ValueOpcode::CompositeExtractU64) { - uint64_t packed = 0; - if (!Arg(inst, 0, packed)) { - return false; - } - result = static_cast(packed >> (component * 32u)); - return true; - } - const auto* source = inst.Arg(0).ResolveInstruction(); - if (source == nullptr) { - return false; - } - if (source->GetOpcode() == ValueOpcode::CompositeConstructU32x2) { - return EvaluateWide(source->Arg(component), result); - } - if (source->GetOpcode() == ValueOpcode::IAddCarry32) { - uint64_t lhs = 0; - uint64_t rhs = 0; - if (!Arg(*source, 0, lhs) || !Arg(*source, 1, rhs)) { - return false; - } - const auto sum = - static_cast(static_cast(lhs)) + static_cast(rhs); - result = - component == 0u ? static_cast(sum) : static_cast(sum >> 32u); - return true; - } +bool SrtWalker::Evaluate(Value value, uint32_t& result) { + uint64_t wide = 0; + if (!EvaluateWide(value, wide)) { return false; } + result = static_cast(wide); + return true; +} - bool EvaluateRawRead(const Inst& inst, uint64_t& result) { - const auto flags = inst.Flags(); - if (flags.index >= m_program.memory_info.size()) { - return false; +ResourcePlan::EvaluationContext& SrtWalker::AcquireContext(const ResourcePlan& program) { + if (program.evaluation_depth == program.evaluation_contexts.size()) { + program.evaluation_contexts.emplace_back(); + } + auto& context = program.evaluation_contexts[program.evaluation_depth++]; + context.generation += 2; + return context; +} + +float SrtWalker::Float32(uint64_t bits) { + return std::bit_cast(static_cast(bits)); +} + +bool SrtWalker::EvaluateWide(Value value, uint64_t& result) { + value = value.Resolve(); + if (value.IsImmediate()) { + switch (value.GetType()) { + case Type::U1: result = value.U1(); return true; + case Type::U8: result = value.U8(); return true; + case Type::U16: result = value.U16(); return true; + case Type::U32: result = value.U32(); return true; + case Type::U64: result = value.U64(); return true; + case Type::F32: result = std::bit_cast(value.F32Value()); return true; + default: return false; } - const auto& mem = m_program.memory_info[flags.index]; - const auto* handle = inst.Arg(0).ResolveInstruction(); - if (handle == nullptr) { - return false; - } - uint64_t low = 0; - uint64_t high = 0; - uint64_t offset = 0; - if (!Arg(*handle, 0, low) || !Arg(*handle, 1, high) || !Arg(inst, 1, offset)) { - return false; - } - const auto base = ((high << 32u) | static_cast(low)) & AddressMask; - const auto immediate = static_cast(static_cast(mem.offset)); - uint64_t address = 0; - if (inst.GetOpcode() == ValueOpcode::ReadConstBuffer) { - uint64_t records = 0; - uint64_t word3 = 0; - if (handle->NumArgs() != 4u || !Arg(*handle, 2, records) || !Arg(*handle, 3, word3)) { - return false; - } - if (immediate < 0) { - return false; - } - const auto byte_offset = - static_cast(immediate) + static_cast(offset); - const auto aligned = byte_offset & ~uint64_t {3}; - const auto stride = (static_cast(high) >> 16u) & 0x3fffu; - const auto size = stride == 0u - ? static_cast(static_cast(records)) - : static_cast(stride) * static_cast(records); - if (aligned > size || size - aligned < sizeof(uint32_t)) { - return false; - } - address = ((base & ~uint64_t {3}) + byte_offset) & ~uint64_t {3}; - } else { - const auto relative = (immediate & ~int64_t {3}) + - static_cast(static_cast(offset) & ~3u); - if (!AddSignedAddress(base & ~uint64_t {3}, relative, address)) { - return false; - } - } - uint32_t word = 0; - if (m_runtime.read_memory != nullptr) { - if (!m_runtime.read_memory(m_runtime.userdata, address, &word)) { - return false; - } - } else { - std::memcpy(&word, reinterpret_cast(address), sizeof(word)); - } - result = word; + } + auto* inst = value.TryInstruction(); + if (inst == nullptr) { + return false; + } + if (!m_active_mask.IsEmpty() && IsRuntimeSelect(inst->GetOpcode()) && + inst->NumArgs() == 3 && inst->Arg(0).Resolve() == m_active_mask) { + return EvaluateWide(inst->Arg(1), result); + } + const auto index = inst->EvaluationIndex(m_program.evaluation_value_count); + if (index >= m_context.values.size()) { + m_context.values.resize(m_program.evaluation_value_count); + } + if (m_context.values[index].generation == m_context.generation) { + result = m_context.values[index].value; return true; } + // The low generation bit marks an instruction that is still being evaluated. + if (m_context.values[index].generation == (m_context.generation | 1u)) { + return false; + } + m_context.values[index].generation = m_context.generation | 1u; + uint64_t out = 0; + const bool evaluated = EvaluateInst(*inst, out); + // Recursive evaluation may grow the dense memo vector. + auto& memo = m_context.values[index]; + if (!evaluated) { + memo.generation = 0; + return false; + } + memo.value = out; + memo.generation = m_context.generation; + result = out; + return true; +} - bool EvaluateInst(const Inst& inst, uint64_t& result) { - uint64_t a = 0; - uint64_t b = 0; - uint64_t c = 0; - const auto binary = [&]() { return Arg(inst, 0, a) && Arg(inst, 1, b); }; - const auto ternary = [&]() { - return Arg(inst, 0, a) && Arg(inst, 1, b) && Arg(inst, 2, c); - }; - switch (inst.GetOpcode()) { - case ValueOpcode::GetUserData: { - const auto reg = RegIndex(inst.Arg(0).ScalarRegister()); - if (reg < m_program.user_data_base || - reg - m_program.user_data_base >= m_runtime.user_data.size()) { - return false; - } - result = m_runtime.user_data[reg - m_program.user_data_base]; +bool SrtWalker::Arg(const Inst& inst, size_t index, uint64_t& result) { + return EvaluateWide(inst.Arg(index), result); +} + +bool SrtWalker::EvaluatePhi(const Inst& inst, uint64_t& result) { + const auto value = ResolveInvariantPhi(m_program, Value(const_cast(&inst))); + return !value.IsEmpty() && EvaluateWide(value, result); +} + +bool SrtWalker::EvaluateExtract(const Inst& inst, uint64_t& result) { + const auto index = inst.Arg(1).Resolve(); + if (!index.IsImmediate() || index.GetType() != Type::U32) { + return false; + } + const auto component = index.U32(); + if (component >= 2u) { + return false; + } + if (inst.GetOpcode() == ValueOpcode::CompositeExtractU64) { + uint64_t packed = 0; + if (!Arg(inst, 0, packed)) { + return false; + } + result = static_cast(packed >> (component * 32u)); + return true; + } + const auto* source = inst.Arg(0).ResolveInstruction(); + if (source == nullptr) { + return false; + } + if (source->GetOpcode() == ValueOpcode::CompositeConstructU32x2) { + return EvaluateWide(source->Arg(component), result); + } + if (source->GetOpcode() == ValueOpcode::IAddCarry32) { + uint64_t lhs = 0; + uint64_t rhs = 0; + if (!Arg(*source, 0, lhs) || !Arg(*source, 1, rhs)) { + return false; + } + const auto sum = + static_cast(static_cast(lhs)) + static_cast(rhs); + result = + component == 0u ? static_cast(sum) : static_cast(sum >> 32u); + return true; + } + return false; +} + +bool SrtWalker::EvaluateRawRead(const Inst& inst, uint64_t& result) { + const auto flags = inst.Flags(); + if (flags.index >= m_program.memory_info.size()) { + return false; + } + const auto& mem = m_program.memory_info[flags.index]; + const auto* handle = inst.Arg(0).ResolveInstruction(); + if (handle == nullptr) { + return false; + } + uint64_t low = 0; + uint64_t high = 0; + uint64_t offset = 0; + if (!Arg(*handle, 0, low) || !Arg(*handle, 1, high) || !Arg(inst, 1, offset)) { + return false; + } + const auto base = ((high << 32u) | static_cast(low)) & AddressMask; + const auto immediate = static_cast(static_cast(mem.offset)); + uint64_t address = 0; + if (inst.GetOpcode() == ValueOpcode::ReadConstBuffer) { + uint64_t records = 0; + uint64_t word3 = 0; + if (handle->NumArgs() != 4u || !Arg(*handle, 2, records) || !Arg(*handle, 3, word3)) { + return false; + } + if (immediate < 0) { + return false; + } + const auto byte_offset = + static_cast(immediate) + static_cast(offset); + const auto aligned = byte_offset & ~uint64_t {3}; + const auto stride = (static_cast(high) >> 16u) & 0x3fffu; + const auto size = stride == 0u + ? static_cast(static_cast(records)) + : static_cast(stride) * static_cast(records); + if (aligned > size || size - aligned < sizeof(uint32_t)) { + return false; + } + address = ((base & ~uint64_t {3}) + byte_offset) & ~uint64_t {3}; + } else { + const auto relative = (immediate & ~int64_t {3}) + + static_cast(static_cast(offset) & ~3u); + if (!AddSignedAddress(base & ~uint64_t {3}, relative, address)) { + return false; + } + } + uint32_t word = 0; + if (m_runtime.read_memory != nullptr) { + if (!m_runtime.read_memory(m_runtime.userdata, address, &word)) { + return false; + } + } else { + std::memcpy(&word, reinterpret_cast(address), sizeof(word)); + } + result = word; + return true; +} + +bool SrtWalker::EvaluateInst(const Inst& inst, uint64_t& result) { + uint64_t a = 0; + uint64_t b = 0; + uint64_t c = 0; + const auto binary = [&]() { return Arg(inst, 0, a) && Arg(inst, 1, b); }; + const auto ternary = [&]() { + return Arg(inst, 0, a) && Arg(inst, 1, b) && Arg(inst, 2, c); + }; + switch (inst.GetOpcode()) { + case ValueOpcode::GetUserData: { + const auto reg = RegIndex(inst.Arg(0).ScalarRegister()); + if (reg < m_program.user_data_base || + reg - m_program.user_data_base >= m_runtime.user_data.size()) { + return false; + } + result = m_runtime.user_data[reg - m_program.user_data_base]; + return true; + } + case ValueOpcode::GetShaderBase: result = m_runtime.shader_base; return true; + case ValueOpcode::Phi: return EvaluatePhi(inst, result); + case ValueOpcode::ReadFirstLane: { + const auto clean_runtime = CleanRuntime(m_runtime); + SrtWalker clean_active(m_program, clean_runtime, {}, nullptr, inst.Arg(1)); + SrtWalker active(m_program, m_runtime, m_clean_flat_slots, &clean_active, + inst.Arg(1)); + return active.EvaluateWide(inst.Arg(0), result); + } + case ValueOpcode::BitCastU32F32: + case ValueOpcode::BitCastF32U32: return Arg(inst, 0, result); + case ValueOpcode::CompositeExtractU64: + case ValueOpcode::CompositeExtractU32x2: return EvaluateExtract(inst, result); + case ValueOpcode::CompositeConstructU64: + if (!binary()) { + return false; + } + result = static_cast(a) | + (static_cast(static_cast(b)) << 32u); + return true; + case ValueOpcode::ReadConst: { + const auto slot = inst.Arg(1).Resolve(); + if (!slot.IsImmediate() || slot.GetType() != Type::U32 || + slot.U32() >= m_program.srt_reads.size()) { + return false; + } + if (slot.U32() < m_clean_flat_slots.size() && + m_clean_flat_slots[slot.U32()] != 0u && m_clean_evaluator != nullptr) { + return m_clean_evaluator->EvaluateWide(m_program.srt_reads[slot.U32()].value, + result); + } + return EvaluateWide(m_program.srt_reads[slot.U32()].value, result); + } + case ValueOpcode::LoadAddressU32: + case ValueOpcode::ReadConstBuffer: + if (IsRawRead(m_program, inst)) { + return EvaluateRawRead(inst, result); + } + break; + case ValueOpcode::IAdd32: + if (binary()) { + result = static_cast(a + b); return true; } - case ValueOpcode::GetShaderBase: result = m_runtime.shader_base; return true; - case ValueOpcode::Phi: return EvaluatePhi(inst, result); - case ValueOpcode::ReadFirstLane: { - const auto clean_runtime = CleanRuntime(m_runtime); - Evaluator clean_active(m_program, clean_runtime, {}, nullptr, inst.Arg(1)); - Evaluator active(m_program, m_runtime, m_clean_flat_slots, &clean_active, - inst.Arg(1)); - return active.EvaluateWide(inst.Arg(0), result); - } - case ValueOpcode::BitCastU32F32: - case ValueOpcode::BitCastF32U32: return Arg(inst, 0, result); - case ValueOpcode::CompositeExtractU64: - case ValueOpcode::CompositeExtractU32x2: return EvaluateExtract(inst, result); - case ValueOpcode::CompositeConstructU64: - if (!binary()) { - return false; - } - result = static_cast(a) | - (static_cast(static_cast(b)) << 32u); + return false; + case ValueOpcode::IAdd64: + if (binary()) { + result = a + b; return true; - case ValueOpcode::ReadConst: { - const auto slot = inst.Arg(1).Resolve(); - if (!slot.IsImmediate() || slot.GetType() != Type::U32 || - slot.U32() >= m_program.srt_reads.size()) { - return false; - } - if (slot.U32() < m_clean_flat_slots.size() && - m_clean_flat_slots[slot.U32()] != 0u && m_clean_evaluator != nullptr) { - return m_clean_evaluator->EvaluateWide(m_program.srt_reads[slot.U32()].value, - result); - } - return EvaluateWide(m_program.srt_reads[slot.U32()].value, result); } - case ValueOpcode::LoadAddressU32: - case ValueOpcode::ReadConstBuffer: - if (IsRawRead(m_program, inst)) { - return EvaluateRawRead(inst, result); - } - break; - case ValueOpcode::IAdd32: - if (binary()) { - result = static_cast(a + b); - return true; - } - return false; - case ValueOpcode::IAdd64: - if (binary()) { - result = a + b; - return true; - } - return false; - case ValueOpcode::ISub32: - if (binary()) { - result = static_cast(a - b); - return true; - } - return false; - case ValueOpcode::ISub64: - if (binary()) { - result = a - b; - return true; - } - return false; - case ValueOpcode::IMul32: - if (binary()) { - result = static_cast(a * b); - return true; - } - return false; - case ValueOpcode::IMul64: - if (binary()) { - result = a * b; - return true; - } - return false; - case ValueOpcode::UMin32: - if (binary()) { - result = std::min(static_cast(a), static_cast(b)); - return true; - } - return false; - case ValueOpcode::ConvertF32U32: - if (Arg(inst, 0, a)) { - result = std::bit_cast(static_cast(static_cast(a))); - return true; - } - return false; - case ValueOpcode::ConvertU32F32: - if (Arg(inst, 0, a)) { - const auto value = Float32(a); - if (!std::isfinite(value) || value < 0.0f || - static_cast(value) > UINT32_MAX) { - return false; - } - result = static_cast(value); - return true; - } - return false; - case ValueOpcode::FPMul32: - if (binary()) { - result = std::bit_cast(Float32(a) * Float32(b)); - return true; - } - return false; - case ValueOpcode::FPTrunc32: - if (Arg(inst, 0, a)) { - result = std::bit_cast(std::trunc(Float32(a))); - return true; - } - return false; - case ValueOpcode::FPIsNan32: - if (Arg(inst, 0, a)) { - result = std::isnan(Float32(a)); - return true; - } - return false; - case ValueOpcode::FPOrdLessThanEqual32: - if (binary()) { - result = Float32(a) <= Float32(b); - return true; - } - return false; - case ValueOpcode::FPOrdGreaterThanEqual32: - if (binary()) { - result = Float32(a) >= Float32(b); - return true; - } - return false; - case ValueOpcode::BitwiseAnd32: - if (binary()) { - result = static_cast(a & b); - return true; - } - return false; - case ValueOpcode::BitwiseAnd64: - if (binary()) { - result = a & b; - return true; - } - return false; - case ValueOpcode::BitwiseOr32: - if (binary()) { - result = static_cast(a | b); - return true; - } - return false; - case ValueOpcode::BitwiseXor32: - if (binary()) { - result = static_cast(a ^ b); - return true; - } - return false; - case ValueOpcode::BitwiseNot32: - if (Arg(inst, 0, a)) { - result = ~static_cast(a); - return true; - } - return false; - case ValueOpcode::ShiftLeftLogical32: - if (binary()) { - result = static_cast(a) << (b & 31u); - return true; - } - return false; - case ValueOpcode::ShiftLeftLogical64: - if (binary()) { - result = a << (b & 63u); - return true; - } - return false; - case ValueOpcode::ShiftRightLogical32: - if (binary()) { - result = static_cast(a) >> (b & 31u); - return true; - } - return false; - case ValueOpcode::ShiftRightLogical64: - if (binary()) { - result = a >> (b & 63u); - return true; - } - return false; - case ValueOpcode::ShiftRightArithmetic32: - if (binary()) { - result = static_cast( - std::bit_cast(static_cast(a)) >> (b & 31u)); - return true; - } - return false; - case ValueOpcode::ShiftRightArithmetic64: - if (binary()) { - result = static_cast(std::bit_cast(a) >> (b & 63u)); - return true; - } - return false; - case ValueOpcode::BitFieldUExtract: - if (ternary()) { - const auto offset = static_cast(b); - const auto width = static_cast(c); - if (offset > 32u || width > 32u - offset) { - return false; - } - const auto mask = width == 32u ? UINT32_MAX - : width == 0u ? 0u - : (uint32_t {1} << width) - 1u; - result = width == 0u ? 0u : (static_cast(a) >> offset) & mask; - return true; - } - return false; - case ValueOpcode::BitFieldSExtract: - if (ternary()) { - const auto offset = static_cast(b); - const auto width = static_cast(c); - if (offset > 32u || width > 32u - offset) { - return false; - } - if (width == 0u) { - result = 0; - return true; - } - const auto mask = width == 32u ? UINT32_MAX : (uint32_t {1} << width) - 1u; - auto bits = (static_cast(a) >> offset) & mask; - if (width < 32u && (bits & (uint32_t {1} << (width - 1u))) != 0u) { - bits |= ~mask; - } - result = bits; - return true; - } - return false; - case ValueOpcode::BitFieldInsert: { - uint64_t d = 0; - if (!ternary() || !Arg(inst, 3, d)) { + return false; + case ValueOpcode::ISub32: + if (binary()) { + result = static_cast(a - b); + return true; + } + return false; + case ValueOpcode::ISub64: + if (binary()) { + result = a - b; + return true; + } + return false; + case ValueOpcode::IMul32: + if (binary()) { + result = static_cast(a * b); + return true; + } + return false; + case ValueOpcode::IMul64: + if (binary()) { + result = a * b; + return true; + } + return false; + case ValueOpcode::UMin32: + if (binary()) { + result = std::min(static_cast(a), static_cast(b)); + return true; + } + return false; + case ValueOpcode::ConvertF32U32: + if (Arg(inst, 0, a)) { + result = std::bit_cast(static_cast(static_cast(a))); + return true; + } + return false; + case ValueOpcode::ConvertU32F32: + if (Arg(inst, 0, a)) { + const auto value = Float32(a); + if (!std::isfinite(value) || value < 0.0f || + static_cast(value) > UINT32_MAX) { return false; } - const auto offset = static_cast(c); - const auto width = static_cast(d); + result = static_cast(value); + return true; + } + return false; + case ValueOpcode::FPMul32: + if (binary()) { + result = std::bit_cast(Float32(a) * Float32(b)); + return true; + } + return false; + case ValueOpcode::FPTrunc32: + if (Arg(inst, 0, a)) { + result = std::bit_cast(std::trunc(Float32(a))); + return true; + } + return false; + case ValueOpcode::FPIsNan32: + if (Arg(inst, 0, a)) { + result = std::isnan(Float32(a)); + return true; + } + return false; + case ValueOpcode::FPOrdLessThanEqual32: + if (binary()) { + result = Float32(a) <= Float32(b); + return true; + } + return false; + case ValueOpcode::FPOrdGreaterThanEqual32: + if (binary()) { + result = Float32(a) >= Float32(b); + return true; + } + return false; + case ValueOpcode::BitwiseAnd32: + if (binary()) { + result = static_cast(a & b); + return true; + } + return false; + case ValueOpcode::BitwiseAnd64: + if (binary()) { + result = a & b; + return true; + } + return false; + case ValueOpcode::BitwiseOr32: + if (binary()) { + result = static_cast(a | b); + return true; + } + return false; + case ValueOpcode::BitwiseXor32: + if (binary()) { + result = static_cast(a ^ b); + return true; + } + return false; + case ValueOpcode::BitwiseNot32: + if (Arg(inst, 0, a)) { + result = ~static_cast(a); + return true; + } + return false; + case ValueOpcode::ShiftLeftLogical32: + if (binary()) { + result = static_cast(a) << (b & 31u); + return true; + } + return false; + case ValueOpcode::ShiftLeftLogical64: + if (binary()) { + result = a << (b & 63u); + return true; + } + return false; + case ValueOpcode::ShiftRightLogical32: + if (binary()) { + result = static_cast(a) >> (b & 31u); + return true; + } + return false; + case ValueOpcode::ShiftRightLogical64: + if (binary()) { + result = a >> (b & 63u); + return true; + } + return false; + case ValueOpcode::ShiftRightArithmetic32: + if (binary()) { + result = static_cast( + std::bit_cast(static_cast(a)) >> (b & 31u)); + return true; + } + return false; + case ValueOpcode::ShiftRightArithmetic64: + if (binary()) { + result = static_cast(std::bit_cast(a) >> (b & 63u)); + return true; + } + return false; + case ValueOpcode::BitFieldUExtract: + if (ternary()) { + const auto offset = static_cast(b); + const auto width = static_cast(c); + if (offset > 32u || width > 32u - offset) { + return false; + } + const auto mask = width == 32u ? UINT32_MAX + : width == 0u ? 0u + : (uint32_t {1} << width) - 1u; + result = width == 0u ? 0u : (static_cast(a) >> offset) & mask; + return true; + } + return false; + case ValueOpcode::BitFieldSExtract: + if (ternary()) { + const auto offset = static_cast(b); + const auto width = static_cast(c); if (offset > 32u || width > 32u - offset) { return false; } if (width == 0u) { - result = static_cast(a); + result = 0; return true; } - const auto mask = - width == 32u ? UINT32_MAX : ((uint32_t {1} << width) - 1u) << offset; - result = (static_cast(a) & ~mask) | - ((static_cast(b) << offset) & mask); + const auto mask = width == 32u ? UINT32_MAX : (uint32_t {1} << width) - 1u; + auto bits = (static_cast(a) >> offset) & mask; + if (width < 32u && (bits & (uint32_t {1} << (width - 1u))) != 0u) { + bits |= ~mask; + } + result = bits; return true; } - case ValueOpcode::SelectU32: - case ValueOpcode::SelectU1: - case ValueOpcode::SelectF32: { - auto& predicate = m_clean_evaluator != nullptr ? *m_clean_evaluator : *this; - if (predicate.EvaluateWide(inst.Arg(0), a)) { - return Arg(inst, a != 0u ? 1u : 2u, result); - } + return false; + case ValueOpcode::BitFieldInsert: { + uint64_t d = 0; + if (!ternary() || !Arg(inst, 3, d)) { return false; } - case ValueOpcode::IEqual32: - if (binary()) { - result = static_cast(a) == static_cast(b); - return true; - } + const auto offset = static_cast(c); + const auto width = static_cast(d); + if (offset > 32u || width > 32u - offset) { return false; - case ValueOpcode::INotEqual32: - if (binary()) { - result = static_cast(a) != static_cast(b); - return true; - } - return false; - case ValueOpcode::ULessThan32: - if (binary()) { - result = static_cast(a) < static_cast(b); - return true; - } - return false; - case ValueOpcode::UGreaterThan32: - if (binary()) { - result = static_cast(a) > static_cast(b); - return true; - } - return false; - case ValueOpcode::SGreaterThanEqual32: - if (binary()) { - result = std::bit_cast(static_cast(a)) >= - std::bit_cast(static_cast(b)); - return true; - } - return false; - case ValueOpcode::LogicalAnd: - if (binary()) { - result = (a != 0u) && (b != 0u); - return true; - } - return false; - case ValueOpcode::LogicalOr: - if (binary()) { - result = (a != 0u) || (b != 0u); - return true; - } - return false; - case ValueOpcode::LogicalXor: - if (binary()) { - result = (a != 0u) != (b != 0u); - return true; - } - return false; - case ValueOpcode::LogicalNot: - if (Arg(inst, 0, a)) { - result = a == 0u; - return true; - } - return false; - case ValueOpcode::UndefU1: - case ValueOpcode::UndefU8: - case ValueOpcode::UndefU16: - case ValueOpcode::UndefU32: - case ValueOpcode::UndefU64: return false; - default: break; + } + if (width == 0u) { + result = static_cast(a); + return true; + } + const auto mask = + width == 32u ? UINT32_MAX : ((uint32_t {1} << width) - 1u) << offset; + result = (static_cast(a) & ~mask) | + ((static_cast(b) << offset) & mask); + return true; } - return false; - } - - const ResourcePlan& m_program; - const SrtRuntime& m_runtime; - std::span m_clean_flat_slots; - Evaluator* m_clean_evaluator = nullptr; - Value m_active_mask; - ResourcePlan::EvaluationContext& m_context; -}; - -const DescriptorSource* Source(const ResourcePlan& program, uint32_t source) { - if (source >= program.descriptor_sources.size()) { - return nullptr; - } - return &program.descriptor_sources[source]; -} - -bool EvaluateRuntimeSourcesImpl(const ResourcePlan& program, std::span sources, - const SrtRuntime& runtime, std::vector& results, - std::vector& flat, bool evaluate_flat, - std::span clean_flat_slots, - std::vector& active_sources) { - if (!program.srt_plan_complete) { - return false; - } - if (std::ranges::any_of(clean_flat_slots, [](uint8_t clean) { return clean != 0u; }) && - runtime.read_specialization_memory == nullptr) { - return false; - } - const auto clean_runtime = CleanRuntime(runtime); - Evaluator clean_evaluator(program, clean_runtime); - Evaluator evaluator(program, runtime, clean_flat_slots, &clean_evaluator); - std::vector active; - if (evaluate_flat) { - active.assign(program.descriptor_sources.size(), 1u); - } - if (evaluate_flat && !program.control_flow.empty()) { - for (const auto& block: program.control_flow) { - for (const auto source: block.sources) { - active.at(source) = 0u; + case ValueOpcode::SelectU32: + case ValueOpcode::SelectU1: + case ValueOpcode::SelectF32: { + auto& predicate = m_clean_evaluator != nullptr ? *m_clean_evaluator : *this; + if (predicate.EvaluateWide(inst.Arg(0), a)) { + return Arg(inst, a != 0u ? 1u : 2u, result); } - } - std::vector visited(program.control_flow.size()); - std::vector pending {0}; - while (!pending.empty()) { - const auto index = pending.back(); - pending.pop_back(); - if (visited.at(index)) { - continue; - } - visited[index] = 1u; - const auto& block = program.control_flow[index]; - for (const auto source: block.sources) { - active[source] = 1u; - } - uint32_t condition = 0; - // A missing clean reader must never fall through to the evaluator's raw-memory path. - if (!block.condition.IsEmpty() && runtime.read_specialization_memory != nullptr && - clean_evaluator.Evaluate(block.condition, condition)) { - pending.push_back(block.successors[condition != 0u ? 0u : 1u]); - } else { - pending.insert(pending.end(), block.successors.begin(), block.successors.end()); - } - } - } - std::vector evaluated; - evaluated.reserve(sources.size()); - for (const auto source_index: sources) { - const auto* source = Source(program, source_index); - if (source == nullptr) { return false; } - DescriptorValue value; - value.dword_count = source->dword_count; - if (!evaluate_flat || active[source_index]) { - for (uint32_t index = 0; index < source->dword_count; index++) { - if (!evaluator.Evaluate(source->dwords[index], value.dwords[index])) { - return false; - } + case ValueOpcode::IEqual32: + if (binary()) { + result = static_cast(a) == static_cast(b); + return true; } - } - evaluated.push_back(value); - } - std::vector flattened; - if (evaluate_flat) { - flattened.resize(program.srt_reads.size()); - for (const auto& read: program.srt_reads) { - const bool clean = read.flat_offset < clean_flat_slots.size() && - clean_flat_slots[read.flat_offset] != 0u; - auto& selected = clean ? clean_evaluator : evaluator; - if (read.flat_offset >= flattened.size() || - !selected.Evaluate(read.value, flattened[read.flat_offset])) { - return false; + return false; + case ValueOpcode::INotEqual32: + if (binary()) { + result = static_cast(a) != static_cast(b); + return true; } - } + return false; + case ValueOpcode::ULessThan32: + if (binary()) { + result = static_cast(a) < static_cast(b); + return true; + } + return false; + case ValueOpcode::UGreaterThan32: + if (binary()) { + result = static_cast(a) > static_cast(b); + return true; + } + return false; + case ValueOpcode::SGreaterThanEqual32: + if (binary()) { + result = std::bit_cast(static_cast(a)) >= + std::bit_cast(static_cast(b)); + return true; + } + return false; + case ValueOpcode::LogicalAnd: + if (binary()) { + result = (a != 0u) && (b != 0u); + return true; + } + return false; + case ValueOpcode::LogicalOr: + if (binary()) { + result = (a != 0u) || (b != 0u); + return true; + } + return false; + case ValueOpcode::LogicalXor: + if (binary()) { + result = (a != 0u) != (b != 0u); + return true; + } + return false; + case ValueOpcode::LogicalNot: + if (Arg(inst, 0, a)) { + result = a == 0u; + return true; + } + return false; + case ValueOpcode::UndefU1: + case ValueOpcode::UndefU8: + case ValueOpcode::UndefU16: + case ValueOpcode::UndefU32: + case ValueOpcode::UndefU64: return false; + default: break; } - results = std::move(evaluated); - active_sources = std::move(active); - if (evaluate_flat) { - flat = std::move(flattened); + return false; +} +bool SrtWalker::EvaluateDescriptor(uint32_t source, DescriptorValue& result) { + if (source >= m_program.descriptor_sources.size()) { + return false; + } + const auto& descriptor = m_program.descriptor_sources[source]; + result = {}; + result.dword_count = descriptor.dword_count; + for (uint32_t index = 0; index < descriptor.dword_count; ++index) { + if (!Evaluate(descriptor.dwords[index], result.dwords[index])) { + return false; + } } return true; } -} // namespace +std::span SrtWalker::FindActiveSources() { + if (m_program.control_flow.empty()) { + return {}; + } + auto& active = m_program.active_sources; + active.assign(m_program.descriptor_sources.size(), 1u); + for (const auto& block: m_program.control_flow) { + for (const auto source: block.sources) { + active.at(source) = 0u; + } + } + auto& visited = m_program.visited_blocks; + auto& pending = m_program.pending_blocks; + visited.assign(m_program.control_flow.size(), 0u); + pending.clear(); + pending.push_back(0u); + while (!pending.empty()) { + const auto index = pending.back(); + pending.pop_back(); + if (visited.at(index)) { + continue; + } + visited[index] = 1u; + const auto& block = m_program.control_flow[index]; + for (const auto source: block.sources) { + active[source] = 1u; + } + uint32_t condition = 0; + if (!block.condition.IsEmpty() && m_runtime.read_specialization_memory != nullptr && + Evaluate(block.condition, condition)) { + pending.push_back(block.successors[condition != 0u ? 0u : 1u]); + } else { + pending.insert(pending.end(), block.successors.begin(), block.successors.end()); + } + } + return active; +} + +bool SrtWalker::RefreshFlatBuffer(std::vector& flat) { + if (!m_program.srt_plan_complete) { + return false; + } + flat.resize(m_program.srt_reads.size()); + for (const auto& read: m_program.srt_reads) { + const bool clean = read.flat_offset < m_clean_flat_slots.size() && + m_clean_flat_slots[read.flat_offset] != 0u; + if (clean && (m_clean_evaluator == nullptr || m_runtime.read_specialization_memory == nullptr)) { + return false; + } + auto& evaluator = clean ? *m_clean_evaluator : *this; + if (read.flat_offset >= flat.size() || !evaluator.Evaluate(read.value, flat[read.flat_offset])) { + return false; + } + } + return true; +} bool ValidateRuntimeValue(const ResourcePlan& program, Value value, RuntimeValueType type) { return RuntimeValidator(program, type).Run(value); @@ -1105,51 +1070,4 @@ void BuildSrtPlan(Program& program) { program.srt_plan_complete = true; } -bool EvaluateUniformValues(const ResourcePlan& program, std::span values, - const SrtRuntime& runtime, std::span results) { - if (values.size() != results.size()) { - return false; - } - const auto clean = CleanRuntime(runtime); - Evaluator evaluator(program, clean); - for (size_t i = 0; i < values.size(); ++i) { - if (!evaluator.Evaluate(values[i], results[i])) { - return false; - } - } - return true; -} - -bool EvaluateDescriptorSource(const ResourcePlan& program, uint32_t source, - const SrtRuntime& runtime, DescriptorValue& result) { - std::vector results; - if (!EvaluateDescriptorSources(program, std::span {&source, 1}, runtime, results)) { - return false; - } - result = results.front(); - return true; -} - -bool EvaluateDescriptorSources(const ResourcePlan& program, std::span sources, - const SrtRuntime& runtime, std::vector& results) { - std::vector ignored; - std::vector active; - return EvaluateRuntimeSourcesImpl(program, sources, runtime, results, ignored, false, {}, - active); -} - -bool EvaluateRuntimeSources(const ResourcePlan& program, std::span sources, - const SrtRuntime& runtime, std::vector& results, - std::vector& flat, std::span clean_flat_slots, - std::vector& active_sources) { - return EvaluateRuntimeSourcesImpl(program, sources, runtime, results, flat, true, - clean_flat_slots, active_sources); -} - -bool WalkSrt(const ResourcePlan& program, const SrtRuntime& runtime, std::vector& flat) { - std::vector ignored; - std::vector active; - return EvaluateRuntimeSources(program, {}, runtime, ignored, flat, {}, active); -} - } // namespace Libs::Graphics::ShaderRecompiler::IR diff --git a/src/graphics/shader/recompiler/ir/passes/SrtWalker.h b/src/graphics/shader/recompiler/ir/passes/SrtWalker.h index eb5ca6138..a1b15c282 100644 --- a/src/graphics/shader/recompiler/ir/passes/SrtWalker.h +++ b/src/graphics/shader/recompiler/ir/passes/SrtWalker.h @@ -26,26 +26,42 @@ enum class RuntimeValueType { Any, Integer }; void BuildSrtPlan(Program& program); bool ValidateRuntimeValue(const ResourcePlan& program, Value value, RuntimeValueType type = RuntimeValueType::Any); -bool EvaluateUniformValues(const ResourcePlan& program, std::span values, - const SrtRuntime& runtime, std::span results); +// Uses the strict reader for values that affect shader specialization. +SrtRuntime CleanRuntime(SrtRuntime runtime); -bool EvaluateDescriptorSource(const ResourcePlan& program, uint32_t source, - const SrtRuntime& runtime, DescriptorValue& result); +// One memoized evaluation session shared by the entire shader resource refresh. +class SrtWalker { +public: + SrtWalker(const ResourcePlan& program, const SrtRuntime& runtime, + std::span clean_flat_slots = {}, SrtWalker* clean_evaluator = nullptr, + Value active_mask = {}); + ~SrtWalker(); + SrtWalker(const SrtWalker&) = delete; + SrtWalker& operator=(const SrtWalker&) = delete; -// Evaluates one runtime snapshot transactionally. Scalar values and ReadConst results shared by -// several descriptors are memoized once across the batch. -bool EvaluateDescriptorSources(const ResourcePlan& program, std::span sources, - const SrtRuntime& runtime, std::vector& results); + bool Evaluate(Value value, uint32_t& result); + bool EvaluateDescriptor(uint32_t source, DescriptorValue& result); + // An empty span means that all sources are active. + std::span FindActiveSources(); + bool RefreshFlatBuffer(std::vector& flat); -// Evaluates potentially reachable descriptor sources and the flattened immediate SRT with one -// memoized scalar walk. Inactive descriptors are zero; on failure no destination is changed. -bool EvaluateRuntimeSources(const ResourcePlan& program, std::span sources, - const SrtRuntime& runtime, std::vector& results, - std::vector& flat, std::span clean_flat_slots, - std::vector& active_sources); +private: + static ResourcePlan::EvaluationContext& AcquireContext(const ResourcePlan& program); + static float Float32(uint64_t bits); + bool EvaluateWide(Value value, uint64_t& result); + bool Arg(const Inst& inst, size_t index, uint64_t& result); + bool EvaluatePhi(const Inst& inst, uint64_t& result); + bool EvaluateExtract(const Inst& inst, uint64_t& result); + bool EvaluateRawRead(const Inst& inst, uint64_t& result); + bool EvaluateInst(const Inst& inst, uint64_t& result); -bool WalkSrt(const ResourcePlan& program, const SrtRuntime& runtime, - std::vector& flat); + const ResourcePlan& m_program; + SrtRuntime m_runtime; + std::span m_clean_flat_slots; + SrtWalker* m_clean_evaluator = nullptr; + Value m_active_mask; + ResourcePlan::EvaluationContext& m_context; +}; } // namespace Libs::Graphics::ShaderRecompiler::IR diff --git a/src/graphics/shader/shader.h b/src/graphics/shader/shader.h index 7667629e5..21114b757 100644 --- a/src/graphics/shader/shader.h +++ b/src/graphics/shader/shader.h @@ -51,10 +51,10 @@ struct CompiledShaderInfo; struct ShaderStageRuntime { const ShaderRecompiler::IR::CompiledShaderInfo* program = nullptr; - ShaderRecompiler::IR::ResourceSnapshot resources; + const ShaderRecompiler::IR::ResourceSnapshot* resources = nullptr; [[nodiscard]] explicit operator bool() const { - return program != nullptr; + return program != nullptr && resources != nullptr; } }; diff --git a/tests/ResourceMaterializationTests.cpp b/tests/ResourceMaterializationTests.cpp index d0567254e..9069bf3f3 100644 --- a/tests/ResourceMaterializationTests.cpp +++ b/tests/ResourceMaterializationTests.cpp @@ -219,19 +219,13 @@ void TestUnbasedFlatCacheHitMaterializes() { "unbased FLAT plan produced unexpected descriptors"); } -void TestFailedMaterializationPreservesPriorStage() { +void TestFailedMaterializationRejectsStage() { using namespace Libs::Graphics::ShaderRecompiler::IR; auto plan = UserDataBufferPlan(); ResourceSnapshot snapshot; - snapshot.user_data.push_back(0xfeedbeefu); ResourceSpecialization specialization; - specialization.buffers.push_back({.packed_stride = 7}); Check(!MaterializeResources(plan, {}, snapshot, specialization), "missing runtime user data did not reject the cached stage"); - Check(snapshot.user_data == std::vector{0xfeedbeefu} && - specialization.buffers.size() == 1 && - specialization.buffers[0].packed_stride == 7, - "failed cache materialization changed its destinations"); } void TestMixedSamplerDuplicatesTheCorrectSnapshot() { @@ -268,7 +262,7 @@ int main() { TestMappedSrtUsesDirectReaderByDefault(); TestIntegerRuntimeValueFollowsSrtReads(); TestUnbasedFlatCacheHitMaterializes(); - TestFailedMaterializationPreservesPriorStage(); + TestFailedMaterializationRejectsStage(); TestMixedSamplerDuplicatesTheCorrectSnapshot(); std::puts("ResourceMaterializationTests: all cases passed"); return 0; diff --git a/tests/ResourceTrackingTests.cpp b/tests/ResourceTrackingTests.cpp index 222d88632..16bdb3882 100644 --- a/tests/ResourceTrackingTests.cpp +++ b/tests/ResourceTrackingTests.cpp @@ -30,14 +30,6 @@ void Check(bool condition, const char *message) { } } -bool SameResourceSnapshot(const ResourceSnapshot &lhs, - const ResourceSnapshot &rhs) { - return lhs.buffers == rhs.buffers && lhs.images == rhs.images && - lhs.samplers == rhs.samplers && - lhs.flattened_srt == rhs.flattened_srt && - lhs.user_data == rhs.user_data && lhs.uniform_fill == rhs.uniform_fill; -} - template void CheckFatal(F &&function, std::string_view expected, const char *message) { try { @@ -162,6 +154,8 @@ struct LinearTestMemory { uint64_t base = 0x1000; std::vector words = std::vector(0x2200 / 4); uint64_t fail_address = UINT64_MAX; + uint64_t watched_address = UINT64_MAX; + uint32_t watched_reads = 0; }; bool ReadLinearTestMemory(void *userdata, uint64_t address, uint32_t *value) { @@ -172,6 +166,7 @@ bool ReadLinearTestMemory(void *userdata, uint64_t address, uint32_t *value) { return false; } *value = memory->words[(address - memory->base) / sizeof(uint32_t)]; + if (address == memory->watched_address) ++memory->watched_reads; return true; } @@ -318,14 +313,10 @@ void TestInvariantIndirectImageMaterialization() { snapshot.images[0].dwords.begin()), "invariant indirect image table did not materialize"); - const auto prior_snapshot = snapshot; - const auto prior_specialization = specialization; memory.fail_address = 0x1004u; Check(!MaterializeResources(resource_plan, runtime, snapshot, - specialization) && - SameResourceSnapshot(snapshot, prior_snapshot) && - specialization == prior_specialization, - "rejected planning memory read mutated the snapshot"); + specialization), + "unreadable material-table selector was accepted"); memory.fail_address = UINT64_MAX; memory.words[(0x1000u - memory.base + 36u) / 4u] = 1u; @@ -433,15 +424,36 @@ void TestInvariantIndirectImageMaterialization() { .userdata = &memory, .read_specialization_memory = ReadLinearTestMemory}; - const auto memory_backed_prior_snapshot = snapshot; - const auto memory_backed_prior_specialization = specialization; Check(!MaterializeResources(memory_backed_plan, memory_backed_runtime, - snapshot, specialization) && - SameResourceSnapshot(snapshot, memory_backed_prior_snapshot) && - specialization == memory_backed_prior_specialization, - "rejected indirect table descriptor read mutated the snapshot"); + snapshot, specialization), + "unreadable indirect table descriptor was accepted"); memory.fail_address = UINT64_MAX; + memory.watched_address = 0x3100u; + memory.watched_reads = 0; + Check(MaterializeResources(memory_backed_plan, memory_backed_runtime, + snapshot, specialization) && memory.watched_reads == 1u, + "descriptor, flattened SRT and indirect-table roots repeated a clean pointer read"); + const auto* buffer_storage = snapshot.buffers.data(); + const auto* image_storage = snapshot.images.data(); + const auto* flat_storage = snapshot.flattened_srt.data(); + const auto* user_data_storage = snapshot.user_data.data(); + const auto* buffer_specialization_storage = specialization.buffers.data(); + const auto* image_specialization_storage = specialization.images.data(); + const auto old_address = snapshot.images[0].dwords[0]; + memory.words[(0x2000u - memory.base) / 4u] += 0x100u; + memory.watched_reads = 0; + Check(MaterializeResources(memory_backed_plan, memory_backed_runtime, + snapshot, specialization) && memory.watched_reads == 1u && + snapshot.images[0].dwords[0] == old_address + 0x100u, + "cache refresh reused stale table contents or repeated its clean pointer read"); + Check(snapshot.buffers.data() == buffer_storage && snapshot.images.data() == image_storage && + snapshot.flattened_srt.data() == flat_storage && + snapshot.user_data.data() == user_data_storage && + specialization.buffers.data() == buffer_specialization_storage && + specialization.images.data() == image_specialization_storage, + "a same-shape refresh discarded the runtime output storage"); + auto malformed = MakeIndirectImageFixture(true); BuildSrtPlan(malformed->program); CheckFatal([&] { TrackResources(malformed->program); }, "not a valid runtime value", @@ -528,13 +540,9 @@ void TestUniformScalarBufferImage() { memory.words[0x44u / 4u] = 0u; memory.words[0x1010u / 4u] = 0x30u; Check(materializes(0x30u), "changed scalar table memory reused the previous image descriptor"); - const auto prior_snapshot = snapshot; - const auto prior_specialization = specialization; memory.fail_address = 0x1044u; - Check(!MaterializeResources(plan, runtime, snapshot, specialization) && - SameResourceSnapshot(snapshot, prior_snapshot) && - specialization == prior_specialization, - "unavailable nested image table partially changed the resource snapshot"); + Check(!MaterializeResources(plan, runtime, snapshot, specialization), + "unavailable nested image table was accepted"); } void TestComputeBufferFill() { @@ -631,6 +639,12 @@ void TestComputeBufferFill() { snapshot.uniform_fill.value == (options.scalar ? 0x40404040u : 0u), "fill proof lost address coverage or the actual stored scalar"); + if (options.scalar && options.clean) { + Check(MaterializeResources(plan, + {.user_data = userdata, .read_memory = Read, .userdata = &memory}, + snapshot, specialization) && snapshot.uniform_fill.words == 0, + "an unavailable clean value retained a previous uniform fill"); + } } }; Run({}); @@ -749,12 +763,12 @@ void TestRuntimeUnsignedMinDescriptor() { SrtRuntime runtime{.user_data = user_data}; DescriptorValue value; const auto source = fixture.program.info.buffers[0].source; - Check(EvaluateDescriptorSource(fixture.program, source, runtime, value) && + Check(SrtWalker(fixture.program, runtime).EvaluateDescriptor(source, value) && value.dwords[3] == 0x100u, "runtime descriptor unsigned minimum did not clamp its first operand"); user_data[0] = 0x80u; Check( - EvaluateDescriptorSource(fixture.program, source, runtime, value) && + SrtWalker(fixture.program, runtime).EvaluateDescriptor(source, value) && value.dwords[3] == 0x80u, "runtime descriptor unsigned minimum did not preserve its first operand"); } @@ -865,7 +879,7 @@ void TestSampleAdjustSamplerScratch() { std::array user_data{4u, 1u, 2u, 0x80000abcu}; SrtRuntime runtime{.user_data = user_data}; DescriptorValue descriptor; - Check(EvaluateDescriptorSource(fixture.program, source, runtime, descriptor) && + Check(SrtWalker(fixture.program, runtime).EvaluateDescriptor(source, descriptor) && descriptor.dwords[3] == 0x80000abcu, "SampleAdjust canonicalization lost sampler border fields"); @@ -972,13 +986,11 @@ void TestFmaskLoadSpecialization() { "FMASK load did not lower to a value vector"); result = vector->Arg(0); uint32_t value = 0; - Check(EvaluateUniformValues(fixture.program, {&result, 1}, - {.user_data = user_data}, {&value, 1}) && + Check(SrtWalker(fixture.program, {.user_data = user_data}).Evaluate(result, value) && value == 0x76543210u, "FMASK load did not return the native sample-to-fragment mapping"); user_data[8] = 1; - Check(EvaluateUniformValues(fixture.program, {&result, 1}, - {.user_data = user_data}, {&value, 1}) && + Check(SrtWalker(fixture.program, {.user_data = user_data}).Evaluate(result, value) && value == 0u, "inactive FMASK load did not preserve the execution mask"); ShaderComputeInputInfo compute{}; @@ -1128,13 +1140,9 @@ void TestDynamicStorageMipTracking() { changed_user_data[3] = (changed_user_data[3] & ~((0xfu << 12u) | (0xfu << 16u))) | (4u << 12u) | (3u << 16u); - const auto valid_snapshot = changed_snapshot; - const auto valid_specialization = changed_specialization; Check(!MaterializeResources(resource_plan, {.user_data = changed_user_data}, - changed_snapshot, changed_specialization) && - SameResourceSnapshot(changed_snapshot, valid_snapshot) && - changed_specialization == valid_specialization, - "an inverted dynamic storage mip range was accepted or mutated output"); + changed_snapshot, changed_specialization), + "an inverted dynamic storage mip range was accepted"); } void TestSrtFlatteningAndRuntimeMemoization() { @@ -1175,53 +1183,37 @@ void TestSrtFlatteningAndRuntimeMemoization() { SrtRuntime runtime{.user_data = user_data, .read_memory = ReadTestMemory, .userdata = &memory}; - std::vector descriptors; + DescriptorValue descriptor; std::vector flat; - std::vector active_sources; const uint32_t request = fixture.program.info.buffers[0].source; - Check(EvaluateRuntimeSources(fixture.program, std::span{&request, 1}, runtime, - descriptors, flat, {}, active_sources), - "typed runtime source evaluation failed"); - Check(descriptors.size() == 1 && descriptors[0].dwords[0] == 0xdeadbeefu && + const auto refresh = [&](const ResourcePlan& plan) { + SrtWalker walker(plan, runtime); + return walker.EvaluateDescriptor(request, descriptor) && walker.RefreshFlatBuffer(flat); + }; + Check(refresh(fixture.program), "typed runtime source evaluation failed"); + Check(descriptor.dwords[0] == 0xdeadbeefu && flat == std::vector{0xdeadbeefu} && memory.reads == 1, "descriptor and flat SRT evaluation did not share one memoized read"); memory.reads = 0; memory.words[1] = 0x12345678u; - Check(EvaluateRuntimeSources(fixture.program, std::span{&request, 1}, runtime, - descriptors, flat, {}, active_sources) && - descriptors[0].dwords[0] == 0x12345678u && + Check(refresh(fixture.program) && descriptor.dwords[0] == 0x12345678u && flat == std::vector{0x12345678u} && memory.reads == 1, "repeated runtime evaluation reused stale scalar memory"); memory.reads = 0; memory.fail_after = 0; - descriptors = {{{1u}, 1u}}; - flat = {2u}; - active_sources = {3u}; - Check(!EvaluateRuntimeSources(fixture.program, std::span{&request, 1}, - runtime, descriptors, flat, {}, active_sources) && - descriptors == std::vector{{{1u}, 1u}} && - flat == std::vector{2u} && - active_sources == std::vector{3u}, - "runtime evaluation failure was not transactional"); - + Check(!refresh(fixture.program), "unavailable scalar memory was accepted"); memory.fail_after = UINT32_MAX; - Check(EvaluateRuntimeSources(fixture.program, std::span{&request, 1}, runtime, - descriptors, flat, {}, active_sources) && - descriptors[0].dwords[0] == 0x12345678u && memory.reads == 1, + Check(refresh(fixture.program) && descriptor.dwords[0] == 0x12345678u && memory.reads == 1, "failed runtime evaluation left a value marked as visiting"); auto detached = ExtractResourcePlan(fixture.program); - Check(EvaluateRuntimeSources(detached, std::span{&request, 1}, runtime, - descriptors, flat, {}, active_sources), - "detached resource plan did not evaluate"); + Check(refresh(detached), "detached resource plan did not evaluate"); auto moved = std::move(detached); memory.reads = 0; memory.words[1] = 0x87654321u; - Check(EvaluateRuntimeSources(moved, std::span{&request, 1}, runtime, - descriptors, flat, {}, active_sources) && - descriptors[0].dwords[0] == 0x87654321u && memory.reads == 1, + Check(refresh(moved) && descriptor.dwords[0] == 0x87654321u && memory.reads == 1, "moving a cached resource plan lost its evaluation state"); ShaderComputeInputInfo compute{}; @@ -1262,8 +1254,7 @@ void TestDynamicSrtReadRemainsExplicit() { .read_memory = ReadTestMemory, .userdata = &memory}; DescriptorValue value; - Check(EvaluateDescriptorSource(fixture.program, - fixture.program.info.buffers[0].source, runtime, value) && + Check(SrtWalker(fixture.program, runtime).EvaluateDescriptor(fixture.program.info.buffers[0].source, value) && value.dwords[0] == 0xabcdef01u && memory.reads == 1, "dynamic typed scalar descriptor source was not evaluated"); @@ -1438,7 +1429,8 @@ void TestConditionalSamplerPhi() { const uint32_t first = (flag < 0) != reverse ? 480 / 4 : 448 / 4; memory.fail_address = 0x1000 + (first == 448 / 4 ? 480u : 448u); DescriptorValue selected; - Check(EvaluateDescriptorSource(plan, source, runtime, selected), + SrtWalker clean(plan, CleanRuntime(runtime)); + Check(SrtWalker(plan, runtime, {}, &clean).EvaluateDescriptor(source, selected), "conditional sampler did not survive detached plan lifetime"); for (uint32_t word = 0; word < 4; ++word) { Check(selected.dwords[word] == memory.words[first + word], @@ -1448,13 +1440,17 @@ void TestConditionalSamplerPhi() { DescriptorValue selected; auto no_clean_reader = runtime; no_clean_reader.read_specialization_memory = nullptr; - Check( - !EvaluateDescriptorSource(plan, source, no_clean_reader, selected), - "conditional sampler used unchecked memory for its predicate"); + { + SrtWalker clean(plan, CleanRuntime(no_clean_reader)); + Check(!SrtWalker(plan, no_clean_reader, {}, &clean).EvaluateDescriptor(source, selected), + "conditional sampler used unchecked memory for its predicate"); + } memory.fail_address = 0x2000 + 196; - Check(!EvaluateDescriptorSource(plan, source, runtime, selected), - "conditional sampler ignored unavailable coherent predicate " - "memory"); + { + SrtWalker clean(plan, CleanRuntime(runtime)); + Check(!SrtWalker(plan, runtime, {}, &clean).EvaluateDescriptor(source, selected), + "conditional sampler ignored unavailable coherent predicate memory"); + } } } CheckFatal([&] { ConditionalSamplerPlan(diamond, false, false, true); }, @@ -1511,8 +1507,7 @@ void TestInvariantLoopPhi() { std::array user_data{0x12345678u}; SrtRuntime runtime{.user_data = user_data}; DescriptorValue descriptor; - Check(EvaluateDescriptorSource(fixture.program, - fixture.program.info.buffers[0].source, runtime, descriptor) && + Check(SrtWalker(fixture.program, runtime).EvaluateDescriptor(fixture.program.info.buffers[0].source, descriptor) && descriptor.dwords[0] == user_data[0], "loop-invariant descriptor phi was not evaluated through typed SSA"); } @@ -1738,11 +1733,9 @@ void TestConditionalBufferMaterialization() { runtime.user_data = std::span(user_data).first(4); Check(MaterializeResources(plan, runtime, snapshot, specialization), "untaken branch evaluated its unavailable descriptor"); - const auto prior = snapshot; memory.words[0] = 1; - Check(!MaterializeResources(plan, runtime, snapshot, specialization) && - SameResourceSnapshot(snapshot, prior), - "taken branch accepted an unavailable descriptor or changed the snapshot"); + Check(!MaterializeResources(plan, runtime, snapshot, specialization), + "taken branch accepted an unavailable descriptor"); runtime.user_data = user_data; const auto CheckActive = [&] { diff --git a/tests/ScalarProvenanceTests.cpp b/tests/ScalarProvenanceTests.cpp index 225d0fc69..f91af58a3 100644 --- a/tests/ScalarProvenanceTests.cpp +++ b/tests/ScalarProvenanceTests.cpp @@ -122,7 +122,7 @@ void TestImmediateFlatteningAndGvn() { TestMemory memory_image{{{0x1020u, 0xfeedbeefu}}}; SrtRuntime runtime{.read_memory = ReadMemory, .userdata = &memory_image}; std::vector flat; - Check(WalkSrt(fixture.program, runtime, flat), "flattened SRT walk failed"); + Check(SrtWalker(fixture.program, runtime).RefreshFlatBuffer(flat), "flattened SRT walk failed"); Check(flat == std::vector{0xfeedbeefu} && memory_image.reads == 1, "flattened SRT did not evaluate its canonical read once"); } @@ -139,7 +139,7 @@ void TestRawScalarComponentAlignment() { TestMemory memory_image{{{0x1000u, 0x12345678u}}}; SrtRuntime runtime{.read_memory = ReadMemory, .userdata = &memory_image}; std::vector flat; - Check(WalkSrt(fixture.program, runtime, flat), "raw scalar SRT walk failed"); + Check(SrtWalker(fixture.program, runtime).RefreshFlatBuffer(flat), "raw scalar SRT walk failed"); Check( flat == std::vector{0x12345678u} && memory_image.reads == 1, "raw scalar base, immediate, and offset were not aligned independently"); @@ -195,7 +195,7 @@ void TestNestedSrtWalk() { TestMemory memory_image{{{0x1000u, 0x2000u}, {0x2000u, 0xabcdef01u}}}; SrtRuntime runtime{.read_memory = ReadMemory, .userdata = &memory_image}; std::vector flat; - Check(WalkSrt(fixture.program, runtime, flat), "nested SRT walk failed"); + Check(SrtWalker(fixture.program, runtime).RefreshFlatBuffer(flat), "nested SRT walk failed"); Check(flat == std::vector({0x2000u, 0xabcdef01u}), "nested typed SRT reads were not evaluated in dependency order"); } @@ -218,7 +218,7 @@ void TestShaderBaseAndUserData() { SrtRuntime runtime{.user_data = user_data, .shader_base = 0x12345678abcdef00ull}; DescriptorValue result; - Check(EvaluateDescriptorSource(fixture.program, 0, runtime, result), + Check(SrtWalker(fixture.program, runtime).EvaluateDescriptor(0, result), "shader-relative descriptor evaluation failed"); Check(result.dword_count == 3 && result.dwords[0] == 0xabcdef00u && result.dwords[1] == 0x12345678u && result.dwords[2] == 0x24u, @@ -243,7 +243,7 @@ void TestCarryAndBitFields() { {.dwords = {low, high, inserted, sign}, .dword_count = 4}); DescriptorValue result; - Check(EvaluateDescriptorSource(fixture.program, 0, {}, result), + Check(SrtWalker(fixture.program, {}).EvaluateDescriptor(0, result), "carry and bit-field descriptor evaluation failed"); Check(result.dwords[0] == 1u && result.dwords[1] == 1u && result.dwords[2] == 0x89abcdefu && result.dwords[3] == 0xffffffffu, @@ -267,14 +267,11 @@ void TestInvariantAndDivergentPhi() { {.dwords = {Value(&divergent)}, .dword_count = 1}); DescriptorValue result; - Check(EvaluateDescriptorSource(fixture.program, 0, {}, result) && + Check(SrtWalker(fixture.program, {}).EvaluateDescriptor(0, result) && result.dwords[0] == 7u, "loop-invariant typed phi was rejected"); - result.dword_count = 4; - result.dwords[0] = 0xdeadbeefu; - Check(!EvaluateDescriptorSource(fixture.program, 1, {}, result) && - result.dword_count == 4 && result.dwords[0] == 0xdeadbeefu, - "divergent phi did not fail transactionally"); + Check(!SrtWalker(fixture.program, {}).EvaluateDescriptor(1, result), + "divergent phi was accepted"); } void TestControlDependentStandaloneLoadStaysTyped() { @@ -314,7 +311,7 @@ void TestRuntime64BitDescriptorOps() { {.dwords = {low, high}, .dword_count = 2}); DescriptorValue result; - Check(EvaluateDescriptorSource(fixture.program, 0, {}, result) && + Check(SrtWalker(fixture.program, {}).EvaluateDescriptor(0, result) && result.dwords[0] == 0xabcdu && result.dwords[1] == 0x1235u, "64-bit typed descriptor arithmetic evaluation is incorrect"); } @@ -389,11 +386,11 @@ void TestUniformFirstLaneSamplerLod() { user_data[6] = 3u; SrtRuntime runtime{.user_data = user_data}; DescriptorValue result; - Check(EvaluateDescriptorSource(fixture.program, 0, runtime, result) && + Check(SrtWalker(fixture.program, runtime).EvaluateDescriptor(0, result) && result.dwords[0] == 0x00300300u, "uniform sampler LOD evaluated incorrectly"); user_data[6] = 20u; - Check(EvaluateDescriptorSource(fixture.program, 0, runtime, result) && + Check(SrtWalker(fixture.program, runtime).EvaluateDescriptor(0, result) && result.dwords[0] == 0x00ffffffu, "uniform sampler LOD clamp evaluated incorrectly"); } @@ -455,7 +452,7 @@ void TestConstantBufferBounds() { TestMemory memory_image{{{0x300cu, 0xa5a5a5a5u}}}; SrtRuntime runtime{.read_memory = ReadMemory, .userdata = &memory_image}; std::vector flat; - Check(WalkSrt(fixture.program, runtime, flat) && + Check(SrtWalker(fixture.program, runtime).RefreshFlatBuffer(flat) && flat == std::vector{0xa5a5a5a5u}, "constant-buffer SRT walk failed"); @@ -470,10 +467,8 @@ void TestConstantBufferBounds() { overflow.Emit(ValueOpcode::GetBufferResource, {overflow_read, Value(0u), Value(16u), Value(0u)}); overflow.Plan(); - flat = {0x55u}; - Check(!WalkSrt(overflow.program, runtime, flat) && - flat == std::vector{0x55u}, - "out-of-bounds constant-buffer walk was not transactional"); + Check(!SrtWalker(overflow.program, runtime).RefreshFlatBuffer(flat), + "out-of-bounds constant-buffer walk was accepted"); } void TestReadLaneElimination() { @@ -560,10 +555,8 @@ void TestUndefinedRuntimeValueFails() { fixture.program.descriptor_sources.push_back( {.dwords = {undef}, .dword_count = 1}); DescriptorValue result; - result.dword_count = 3; - Check(!EvaluateDescriptorSource(fixture.program, 0, {}, result) && - result.dword_count == 3, - "undefined typed descriptor source did not fail transactionally"); + Check(!SrtWalker(fixture.program, {}).EvaluateDescriptor(0, result), + "undefined typed descriptor source was accepted"); } } // namespace diff --git a/tests/ShaderRecompilerComputeTests.cpp b/tests/ShaderRecompilerComputeTests.cpp index 3327f4e90..04e5d140d 100644 --- a/tests/ShaderRecompilerComputeTests.cpp +++ b/tests/ShaderRecompilerComputeTests.cpp @@ -1325,12 +1325,15 @@ void CheckRectListShaders() { ShaderVertexInputInfo vertex{}; ShaderRecompiler::IR::CompiledShaderInfo vertex_program{.param_export_mask = 1u}; + ShaderRecompiler::IR::ResourceSnapshot empty_snapshot; vertex.stage.program = &vertex_program; + vertex.stage.resources = &empty_snapshot; ShaderPixelInputInfo pixel{}; pixel.input_num = 2; pixel.interpolator_settings[0] = 0x400u; pixel.interpolator_settings[1] = 0; pixel.stage.program = &program; + pixel.stage.resources = &empty_snapshot; std::vector perspective_specialization; BuildStageStaticKey(pixel, perspective_specialization); pixel.ps_no_perspective = true; @@ -1967,10 +1970,12 @@ public: pixel_program.bindings.push_data_start_dword = 0; pixel_program.bindings.user_data_registers = {0, 1}; pixel_program.bindings.memory_offset_dword = 2; - ShaderStageRuntime vertex_runtime{.program = &vertex_program}; - vertex_runtime.resources.user_data = {0x11111111u, 0x22222222u}; - ShaderStageRuntime pixel_runtime{.program = &pixel_program}; - pixel_runtime.resources.user_data = {0x33333333u, 0x44444444u}; + ShaderRecompiler::IR::ResourceSnapshot vertex_snapshot; + vertex_snapshot.user_data = {0x11111111u, 0x22222222u}; + ShaderStageRuntime vertex_runtime{&vertex_program, &vertex_snapshot}; + ShaderRecompiler::IR::ResourceSnapshot pixel_snapshot; + pixel_snapshot.user_data = {0x33333333u, 0x44444444u}; + ShaderStageRuntime pixel_runtime{&pixel_program, &pixel_snapshot}; auto vertex = context.GetRenderExecutor().PrepareBindings(vertex_runtime); auto pixel = context.GetRenderExecutor().PrepareBindings(pixel_runtime); @@ -4498,7 +4503,8 @@ public: "test HTile entries were not registered in their initial state"); const auto MakeInput = [](uint64_t address, bool read, bool written, - ShaderRecompiler::IR::CompiledShaderInfo &program) { + ShaderRecompiler::IR::CompiledShaderInfo &program, + ShaderRecompiler::IR::ResourceSnapshot &snapshot) { program.stage = ShaderType::Compute; ShaderRecompiler::IR::BufferResource resource{}; resource.read = read; @@ -4519,14 +4525,17 @@ public: std::copy_n(descriptor.fields, value.dword_count, value.dwords.begin()); ShaderComputeInputInfo input{}; input.stage.program = &program; - input.stage.resources.buffers.push_back(value); + snapshot.buffers.push_back(value); + input.stage.resources = &snapshot; return input; }; auto &executor = context.GetRenderExecutor(); auto &command = scheduler.Current(); ShaderRecompiler::IR::CompiledShaderInfo read_only_program{}; + ShaderRecompiler::IR::ResourceSnapshot read_only_snapshot; const auto read_only_input = - MakeInput(read_only_meta, true, false, read_only_program); + MakeInput(read_only_meta, true, false, read_only_program, + read_only_snapshot); const bool read_only_consumed = RenderExecutorTestAccess::TryConsumeComputeMetaClear( executor, read_only_input, command); @@ -4536,8 +4545,10 @@ public: "a metadata read-only dispatch changed logical clear state"); ShaderRecompiler::IR::CompiledShaderInfo read_write_program{}; + ShaderRecompiler::IR::ResourceSnapshot read_write_snapshot; const auto read_write_input = - MakeInput(read_write_meta, true, true, read_write_program); + MakeInput(read_write_meta, true, true, read_write_program, + read_write_snapshot); const bool read_write_consumed = RenderExecutorTestAccess::TryConsumeComputeMetaClear( executor, read_write_input, command); @@ -4547,8 +4558,10 @@ public: "a metadata read/modify/write dispatch was replaced by a clear"); ShaderRecompiler::IR::CompiledShaderInfo write_only_program{}; + ShaderRecompiler::IR::ResourceSnapshot write_only_snapshot; const auto write_only_input = - MakeInput(write_only_meta, false, true, write_only_program); + MakeInput(write_only_meta, false, true, write_only_program, + write_only_snapshot); const bool write_only_consumed = RenderExecutorTestAccess::TryConsumeComputeMetaClear( executor, write_only_input, command); @@ -9891,8 +9904,9 @@ public: ShaderBufferResource buffer_descriptor{}; buffer_descriptor.UpdateAddress48(buffer_address); buffer_descriptor.fields[2] = 0x8000; - ShaderStageRuntime buffer_runtime{.program = &buffer_program}; - auto &value = buffer_runtime.resources.buffers.emplace_back(); + ShaderRecompiler::IR::ResourceSnapshot buffer_snapshot; + ShaderStageRuntime buffer_runtime{&buffer_program, &buffer_snapshot}; + auto &value = buffer_snapshot.buffers.emplace_back(); std::memcpy(value.dwords.data(), buffer_descriptor.fields, sizeof(buffer_descriptor.fields)); value.dword_count = 4; @@ -9962,7 +9976,7 @@ public: null_info.stage = null_program.stage; null_info.info = std::move(null_program.info); null_info.bindings = std::move(null_program.bindings); - ShaderStageRuntime null_runtime{&null_info, std::move(null_snapshot)}; + ShaderStageRuntime null_runtime{&null_info, &null_snapshot}; auto null_bindings = executor.PrepareBindings(null_runtime); executor.RebindImages(null_bindings); Require(name, "null descriptor count", @@ -10161,7 +10175,7 @@ public: mipped_storage_descriptor, overwide_mipped_storage_descriptor, overwide_mipped_storage_descriptor}; - ShaderStageRuntime mipped_runtime{&mipped_program, std::move(mipped_snapshot)}; + ShaderStageRuntime mipped_runtime{&mipped_program, &mipped_snapshot}; mipped_prepared.runtime = &mipped_runtime; mipped_prepared.images.push_back( std::move(plain_mipped_storage_binding)); @@ -10468,8 +10482,7 @@ public: storage_info.stage = storage_program.stage; storage_info.info = std::move(storage_program.info); storage_info.bindings = std::move(storage_program.bindings); - ShaderStageRuntime storage_runtime{&storage_info, - std::move(storage_snapshot)}; + ShaderStageRuntime storage_runtime{&storage_info, &storage_snapshot}; auto storage_discovery = executor.PrepareBindings(storage_runtime); const auto storage_id = storage_discovery.images[0].image_id; Require(name, "storage prefetch purity", @@ -10499,8 +10512,7 @@ public: sampled_info.stage = sampled_program.stage; sampled_info.info = std::move(sampled_program.info); sampled_info.bindings = std::move(sampled_program.bindings); - ShaderStageRuntime sampled_runtime{&sampled_info, - std::move(sampled_snapshot)}; + ShaderStageRuntime sampled_runtime{&sampled_info, &sampled_snapshot}; constexpr uint64_t ordered_sampled_address = base + 0x10000; const uint32_t ordered_sampled_value = 0x89abcdefu; @@ -10521,7 +10533,7 @@ public: ShaderRecompiler::IR::ResourceSnapshot ordered_snapshot{}; ordered_snapshot.images.push_back(ordered_descriptor); ShaderStageRuntime ordered_sampled_runtime{sampled_runtime.program, - std::move(ordered_snapshot)}; + &ordered_snapshot}; auto ordered_bindings = RenderExecutorTestAccess::PrepareGraphicsBindings( executor, storage_runtime, ordered_sampled_runtime, true); const auto ordered_sampled_id = @@ -10669,7 +10681,8 @@ public: split_program.info = std::move(split_ir.info); split_program.bindings = std::move(split_ir.bindings); PreparedBindings split_bindings{}; - ShaderStageRuntime split_runtime{.program = &split_program}; + ShaderRecompiler::IR::ResourceSnapshot split_snapshot; + ShaderStageRuntime split_runtime{&split_program, &split_snapshot}; split_bindings.runtime = &split_runtime; split_bindings.images.push_back( {split_id, texture_cache.FindTexture(split_id, split_storage_desc), @@ -10993,10 +11006,10 @@ public: registers.SetDepthControl(bounds_control); registers.SetRenderControl({}); registers.SetDepthClearValue(0.375f); - ShaderStageRuntime bounds_vertex{&vertex_sampled_info, {}}; - ShaderStageRuntime bounds_pixel{&sampled_info, {}}; - bounds_vertex.resources.images.push_back(sampled_depth_value); - bounds_pixel.resources.images.push_back(sampled_depth_value); + ShaderRecompiler::IR::ResourceSnapshot bounds_snapshot; + bounds_snapshot.images.push_back(sampled_depth_value); + ShaderStageRuntime bounds_vertex{&vertex_sampled_info, &bounds_snapshot}; + ShaderStageRuntime bounds_pixel{&sampled_info, &bounds_snapshot}; const auto depth_texels = depth_only.desc.info.extent.width * depth_only.desc.info.extent.height; const auto depth_bytes = depth_texels * sizeof(uint32_t); @@ -11341,10 +11354,12 @@ public: std::end(shared_depth_descriptor.fields), shared_depth_value.dwords.begin()); shared_depth_value.dword_count = 8; - ShaderStageRuntime shared_depth_vertex{&vertex_sampled_info, {}}; - ShaderStageRuntime shared_depth_pixel{&sampled_info, {}}; - shared_depth_vertex.resources.images.push_back(shared_depth_value); - shared_depth_pixel.resources.images.push_back(shared_depth_value); + ShaderRecompiler::IR::ResourceSnapshot shared_depth_snapshot; + shared_depth_snapshot.images.push_back(shared_depth_value); + ShaderStageRuntime shared_depth_vertex{&vertex_sampled_info, + &shared_depth_snapshot}; + ShaderStageRuntime shared_depth_pixel{&sampled_info, + &shared_depth_snapshot}; for (const bool stencil_write : {true, false}) { registers.SetDepthRenderTarget(phased_depth_target); registers.SetRenderControl({}); @@ -11486,8 +11501,7 @@ public: array_descriptor.dword_count = 8; ShaderRecompiler::IR::ResourceSnapshot array_snapshot{}; array_snapshot.images.push_back(array_descriptor); - ShaderStageRuntime array_runtime{&array_program, - std::move(array_snapshot)}; + ShaderStageRuntime array_runtime{&array_program, &array_snapshot}; auto array_binding = executor.PrepareBindings(array_runtime); executor.RebindImages(array_binding); @@ -11572,8 +11586,9 @@ public: descriptor.fields[1] |= 4u << 16u; descriptor.fields[2] = buffer_size / sizeof(uint32_t); descriptor.fields[3] = 0x00014004u; - ShaderStageRuntime buffer_runtime{.program = &buffer_program}; - auto &value = buffer_runtime.resources.buffers.emplace_back(); + ShaderRecompiler::IR::ResourceSnapshot buffer_snapshot; + ShaderStageRuntime buffer_runtime{&buffer_program, &buffer_snapshot}; + auto &value = buffer_snapshot.buffers.emplace_back(); std::memcpy(value.dwords.data(), descriptor.fields, sizeof(descriptor.fields)); value.dword_count = 4; auto buffer_bindings = executor.PrepareBindings(buffer_runtime); @@ -11660,7 +11675,7 @@ public: ShaderRecompiler::IR::ResourceSnapshot colliding_msaa_snapshot{}; colliding_msaa_snapshot.images.push_back(colliding_msaa_descriptor); ShaderStageRuntime colliding_msaa_runtime{ - &colliding_msaa_program, std::move(colliding_msaa_snapshot)}; + &colliding_msaa_program, &colliding_msaa_snapshot}; auto colliding_msaa_binding = executor.PrepareBindings(colliding_msaa_runtime); executor.RebindImages(colliding_msaa_binding); @@ -11719,7 +11734,7 @@ public: msaa_program.info.images.push_back(msaa_resource); ShaderRecompiler::IR::ResourceSnapshot msaa_snapshot{}; msaa_snapshot.images.push_back(msaa_descriptor); - ShaderStageRuntime msaa_runtime{&msaa_program, std::move(msaa_snapshot)}; + ShaderStageRuntime msaa_runtime{&msaa_program, &msaa_snapshot}; auto msaa_binding = executor.PrepareBindings(msaa_runtime); executor.RebindImages(msaa_binding); const auto &resolved_msaa = msaa_binding.images[0]; @@ -11758,8 +11773,7 @@ public: msaa_array_program.info.images.push_back(msaa_array_resource); ShaderRecompiler::IR::ResourceSnapshot msaa_array_snapshot{}; msaa_array_snapshot.images.push_back(msaa_array_descriptor); - ShaderStageRuntime msaa_array_runtime{&msaa_array_program, - std::move(msaa_array_snapshot)}; + ShaderStageRuntime msaa_array_runtime{&msaa_array_program, &msaa_array_snapshot}; auto msaa_array_binding = executor.PrepareBindings(msaa_array_runtime); executor.RebindImages(msaa_array_binding); const auto &resolved_msaa_array = msaa_array_binding.images[0]; @@ -12022,7 +12036,7 @@ public: descriptor.dwords.begin()); descriptor.dword_count = 8; snapshot.images.push_back(descriptor); - ShaderStageRuntime runtime{&program, std::move(snapshot)}; + ShaderStageRuntime runtime{&program, &snapshot}; auto prepared = context.GetRenderExecutor().PrepareBindings(runtime); const auto sampled_stencil_id = prepared.images[0].image_id; @@ -12103,6 +12117,8 @@ public: auto stencil_translated = ShaderRecompiler::TranslateProgram(stencil_shader, stencil_options); auto stencil_plan = ShaderRecompiler::IR::ExtractResourcePlan(stencil_translated.program); ShaderRecompiler::IR::ResourceSpecialization stencil_specialization; + ShaderRecompiler::IR::ResourceSnapshot stencil_snapshot; + stencil_compute.stage.resources = &stencil_snapshot; Require(name, "stencil fill materialization", ShaderRecompiler::IR::MaterializeResources(stencil_plan, {.user_data = stencil_userdata, .userdata = &stencil_byte, @@ -12110,7 +12126,7 @@ public: if (address != reinterpret_cast(data)) return false; *word = *static_cast(data); return true; - }}, stencil_compute.stage.resources, stencil_specialization), + }}, stencil_snapshot, stencil_specialization), "captured stencil shader could not resolve its clear byte"); ShaderRecompiler::IR::ApplyResourceSpecialization(stencil_translated.program, stencil_specialization); @@ -12129,7 +12145,7 @@ public: !RenderExecutorTestAccess::TryConsumeComputeImageClear(executor, stencil_compute, scheduler.Current(), 2, 1, 1, 0x41u), "incomplete or excessive stencil dispatch was consumed"); - auto &stencil_source = stencil_compute.stage.resources.buffers[0]; + auto &stencil_source = stencil_snapshot.buffers[0]; const auto clean_source = stencil_source; stencil_source.dwords[0] = static_cast(stencil_address); stencil_source.dwords[1] = static_cast(stencil_address >> 32) | (4u << 16); @@ -13068,7 +13084,9 @@ public: pixel_program.info = fragment.program.info; pixel_program.bindings = fragment.program.bindings; ShaderVertexInputInfo vertex{}; + ShaderRecompiler::IR::ResourceSnapshot vertex_snapshot; vertex.stage.program = &vertex_program; + vertex.stage.resources = &vertex_snapshot; vertex.resources_num = 2; vertex.buffers_num = 1; vertex.buffers[0].stride = 6 * sizeof(float); @@ -13086,7 +13104,7 @@ public: } ShaderPixelInputInfo pixel{}; pixel.stage.program = &pixel_program; - pixel.stage.resources = std::move(fragment.resources); + pixel.stage.resources = &fragment.resources; RenderColorInfo color{}; color.desc.type = BindingType::RenderTarget; @@ -27901,8 +27919,8 @@ void CheckPs5GameExampleImageClearRuntimeShape() { const auto code = MakeCode(); auto positive = Compile(code); - - compute.stage.resources = positive; + auto runtime_resources = positive; + compute.stage.resources = &runtime_resources; ShaderBufferResource descriptor{}; u32 packed_clear = 0; uint64_t size = 0; @@ -27915,19 +27933,19 @@ void CheckPs5GameExampleImageClearRuntimeShape() { auto wrong_index = code; wrong_index[0] = (wrong_index[0] & ~(0x3ffu << 16u)) | (0x347u << 16u); - compute.stage.resources = Compile(wrong_index); + runtime_resources = Compile(wrong_index); Require("Ps5GameExampleImageClear", "add-before-shift address", !ResolveComputeBufferFill(compute, 64, 1, 1, 0x61u, descriptor, packed_clear, size), "V_ADD_LSHL was mistaken for V_LSHL_ADD contiguous coverage"); user_data[7] ^= 1u; - compute.stage.resources = Compile(code); + runtime_resources = Compile(code); Require("Ps5GameExampleImageClear", "non-repeated clear", !ResolveComputeBufferFill(compute, 64, 1, 1, 0x61u, descriptor, packed_clear, size), "non-uniform uint4 data was replaced with a color clear"); - compute.stage.resources = positive; + runtime_resources = positive; compute.dispatch_threads_num[0] = 32; Require("Ps5GameExampleImageClear", "partial dispatch", !ResolveComputeBufferFill(compute, 32, 1, 1, 0x61u, descriptor, @@ -27946,7 +27964,7 @@ void CheckPs5GameExampleImageClearRuntimeShape() { scalar_code.push_back(EncodeMubuf1(1, 0, 0)); AppendEnd(&scalar_code); auto scalar = Compile(scalar_code); - compute.stage.resources = scalar; + runtime_resources = scalar; Require("Ps5GameExampleImageClear", "inline scalar fill", ResolveComputeBufferFill(compute, 1, 1, 1, 0x41u, descriptor, packed_clear, size) && @@ -27958,13 +27976,13 @@ void CheckPs5GameExampleImageClearRuntimeShape() { packed_clear, size), "excess invocation coverage was accepted as a fill"); auto alias = scalar.buffers.front(); - compute.stage.resources.buffers.push_back(alias); + runtime_resources.buffers.push_back(alias); Require("Ps5GameExampleImageClear", "aliased scalar input", !ResolveComputeBufferFill(compute, 1, 1, 1, 0x41u, descriptor, packed_clear, size), "a scalar read aliasing the destination was accepted as uniform"); - compute.stage.resources = scalar; - compute.stage.resources.buffers[0].dwords[3] = + runtime_resources = scalar; + runtime_resources.buffers[0].dwords[3] = (static_cast(Prospero::BufferFormat::k32Float) << 12u) | 0x204u; Require("Ps5GameExampleImageClear", "wrong store format", !ResolveComputeBufferFill(compute, 1, 1, 1, 0x41u, descriptor, @@ -27973,14 +27991,14 @@ void CheckPs5GameExampleImageClearRuntimeShape() { // A body change preserving every descriptor and dispatch field must change // the result. scalar_code[2] = EncodeVop1(0x01u, 1, InlineU32(7)); - compute.stage.resources = Compile(scalar_code); + runtime_resources = Compile(scalar_code); Require("Ps5GameExampleImageClear", "changed store value", ResolveComputeBufferFill(compute, 1, 1, 1, 0x41u, descriptor, packed_clear, size) && packed_clear == 7u, "clear recognition ignored the shader's actual stored value"); scalar_code[2] = EncodeVop1(0x01u, 1, Vgpr(0)); - compute.stage.resources = Compile(scalar_code); + runtime_resources = Compile(scalar_code); Require("Ps5GameExampleImageClear", "varying store value", !ResolveComputeBufferFill(compute, 1, 1, 1, 0x41u, descriptor, packed_clear, size), @@ -28002,14 +28020,14 @@ void CheckPs5GameExampleImageClearRuntimeShape() { AppendEnd(&scalar_code); clean_scalar = true; auto loaded = Compile(scalar_code); - compute.stage.resources = loaded; + runtime_resources = loaded; Require("Ps5GameExampleImageClear", "scalar-loaded fill", ResolveComputeBufferFill(compute, 1, 1, 1, 0x41u, descriptor, packed_clear, size) && packed_clear == scalar_clear && descriptor.Base48() == 0x200000u, "GTA3's scalar input was confused with its destination"); clean_scalar = false; - compute.stage.resources = Compile(scalar_code); + runtime_resources = Compile(scalar_code); Require("Ps5GameExampleImageClear", "unavailable clean scalar", !ResolveComputeBufferFill(compute, 1, 1, 1, 0x41u, descriptor, packed_clear, size), @@ -28303,7 +28321,7 @@ void CheckEmbeddedFetchVertexOffset() { program.user_data_base = result.program.user_data_base; program.info.vertex_offset_sgpr = result.program.info.vertex_offset_sgpr; vertex.stage.program = &program; - vertex.stage.resources = result.resources; + vertex.stage.resources = &result.resources; return ResolveDrawOffsets(index_offset, vertex).first; }; @@ -28315,7 +28333,7 @@ void CheckEmbeddedFetchVertexOffset() { program.info.instance_offset_sgpr = result.program.info.instance_offset_sgpr; vertex.stage.program = &program; - vertex.stage.resources = result.resources; + vertex.stage.resources = &result.resources; return ResolveDrawOffsets(0, vertex).second; }; diff --git a/tests/shaderCfgTests.cpp b/tests/shaderCfgTests.cpp index cd03b93d1..52ad6d1b8 100644 --- a/tests/shaderCfgTests.cpp +++ b/tests/shaderCfgTests.cpp @@ -158,9 +158,11 @@ void CompilePixelRuntime(const ShaderParams ¶ms, options.input_info.pixel = &input_info; auto result = RecompileForTest(params.code, options); static std::deque programs; + static std::deque resources; programs.push_back(std::move(result.program).TakeCompiledInfo()); input_info.stage.program = &programs.back(); - input_info.stage.resources = std::move(result.resources); + resources.push_back(std::move(result.resources)); + input_info.stage.resources = &resources.back(); } template @@ -1154,20 +1156,17 @@ void TestNativeShaderResourceDependencies() { .info = program.info, .bindings = program.bindings, }; - ShaderStageRuntime runtime{.program = &compiled, .resources = resources}; + ShaderStageRuntime runtime{.program = &compiled, .resources = &resources}; Check(HasShaderBufferWrites(runtime), "graphics/compute write predicate lost nonempty written buffers"); set_buffer(0, 0, 16, 3); set_buffer(2, 0x2000, 0, 0); - runtime.resources = resources; Check(!HasShaderBufferWrites(runtime), "graphics/compute write predicate included null, empty, or read-only buffers"); set_buffer(2, 0x2000, 0, 64); - runtime.resources = resources; Check(HasShaderBufferWrites(runtime), "graphics/compute write predicate lost a byte-addressed buffer"); set_buffer(2, 0x2000, 0x3FFF, UINT32_MAX); - runtime.resources = resources; Check(HasShaderBufferWrites(runtime), "graphics/compute write predicate lost a maximum-size strided buffer"); @@ -9669,10 +9668,8 @@ void TestMergedShaderUserDataSnapshot() { .read_memory = ReadHostTestMemory}; IR::DescriptorValue front_descriptor, back_descriptor; const auto &table = params == &first ? first_table : second_table; - Check(IR::EvaluateDescriptorSource(program, program.info.buffers[0].source, - runtime, front_descriptor) && - IR::EvaluateDescriptorSource(program, program.info.buffers[1].source, - runtime, back_descriptor) && + Check(IR::SrtWalker(program, runtime).EvaluateDescriptor(program.info.buffers[0].source, front_descriptor) && + IR::SrtWalker(program, runtime).EvaluateDescriptor(program.info.buffers[1].source, back_descriptor) && std::equal(params->user_data.begin() + 8, params->user_data.end(), front_descriptor.dwords.begin()) && std::equal(table.begin(), table.end(), back_descriptor.dwords.begin()), @@ -12281,8 +12278,7 @@ void TestTypedDescriptorRealCarryAndScalarLoads() { ShaderRecompiler::IR::SrtRuntime carry_runtime{carry_user_data, shader_base, nullptr, nullptr}; ShaderRecompiler::IR::DescriptorValue carry_value; - Check(ShaderRecompiler::IR::EvaluateDescriptorSource( - carry_ir, carry_source_index, carry_runtime, carry_value) && + Check(ShaderRecompiler::IR::SrtWalker(carry_ir, carry_runtime).EvaluateDescriptor(carry_source_index, carry_value) && carry_value.dwords[0] == static_cast(expected_pc) && carry_value.dwords[1] == static_cast(expected_pc >> 32u), "S_GETPC shader-base or add/addc carry evaluation was incorrect"); @@ -12355,9 +12351,7 @@ void TestTypedDescriptorRealCarryAndScalarLoads() { TypedDescriptorSource(inline_sampler_ir, inline_sampler_ir.info.samplers[0].source) != nullptr && - ShaderRecompiler::IR::EvaluateDescriptorSource( - inline_sampler_ir, inline_sampler_ir.info.samplers[0].source, - runtime, sampler) && + ShaderRecompiler::IR::SrtWalker(inline_sampler_ir, runtime).EvaluateDescriptor(inline_sampler_ir.info.samplers[0].source, sampler) && sampler.dwords[0] == 0 && sampler.dwords[1] == 0x00fff000u && sampler.dwords[2] == 0x09500000u && sampler.dwords[3] == 0, "real inline sampler construction was unresolved or evaluated " @@ -12393,7 +12387,7 @@ void TestSrtWalkerRealSmemTranslation() { std::vector flat; const ShaderRecompiler::IR::SrtRuntime runtime{user_data, 0, ReadSrtHostDword, nullptr}; - const auto walked = ShaderRecompiler::IR::WalkSrt(ir, runtime, flat); + const auto walked = ShaderRecompiler::IR::SrtWalker(ir, runtime).RefreshFlatBuffer(flat); Check(walked, "SRT walk failed"); Check(flat.size() == table.size() && std::equal(flat.begin(), flat.end(), table.begin()), @@ -12422,7 +12416,7 @@ void TestSrtWalkerVccBaseTranslation() { const ShaderRecompiler::IR::SrtRuntime runtime{user_data, 0, ReadSrtHostDword, nullptr}; std::vector flat; - Check(ShaderRecompiler::IR::WalkSrt(ir, runtime, flat), "SRT walk failed"); + Check(ShaderRecompiler::IR::SrtWalker(ir, runtime).RefreshFlatBuffer(flat), "SRT walk failed"); Check(flat.size() == table.size() && std::equal(flat.begin(), flat.end(), table.begin()), "typed SSA lost an SMEM base copied through VCC"); @@ -12450,18 +12444,15 @@ void TestSrtWalkerRealSBufferTranslation() { std::vector flat; const ShaderRecompiler::IR::SrtRuntime runtime{user_data, 0, ReadSrtHostDword, nullptr}; - const auto walked = ShaderRecompiler::IR::WalkSrt(ir, runtime, flat); + const auto walked = ShaderRecompiler::IR::SrtWalker(ir, runtime).RefreshFlatBuffer(flat); Check(walked, "SRT walk failed"); Check(flat.size() == 4 && std::equal(flat.begin(), flat.end(), table.begin() + 1), "real S_BUFFER_LOAD walk used the wrong final alignment"); user_data[10] = 4 * sizeof(uint32_t); - const auto flat_before_failure = flat; - const auto bounds_walked = ShaderRecompiler::IR::WalkSrt(ir, runtime, flat); + const auto bounds_walked = ShaderRecompiler::IR::SrtWalker(ir, runtime).RefreshFlatBuffer(flat); Check(!bounds_walked, "real S_BUFFER_LOAD walk ignored descriptor bounds"); - Check(flat == flat_before_failure, - "failed real S_BUFFER_LOAD walk changed the prior flat snapshot"); CheckFlattenedReadSlots( ir, 4, "real S_BUFFER_LOAD patch used the wrong flat offsets"); @@ -12475,7 +12466,7 @@ void TestSrtWalkerRealSBufferTranslation() { static_cast(std::size(negative_shader)), negative_ir); user_data[10] = sizeof(table); - Check(!ShaderRecompiler::IR::WalkSrt(negative_ir, runtime, flat), + Check(!ShaderRecompiler::IR::SrtWalker(negative_ir, runtime).RefreshFlatBuffer(flat), "real S_BUFFER_LOAD walk accepted a negative immediate"); } @@ -12514,7 +12505,7 @@ void TestScalarMemorySourcesCapturedBeforeWrites() { const ShaderRecompiler::IR::SrtRuntime runtime{ user_data, 0, ReadSrtHostRangeDword, &range}; std::vector flat; - Check(ShaderRecompiler::IR::WalkSrt(ir, runtime, flat), "SRT walk failed"); + Check(ShaderRecompiler::IR::SrtWalker(ir, runtime).RefreshFlatBuffer(flat), "SRT walk failed"); Check(flat.size() == table.size() && std::equal(flat.begin(), flat.end(), table.begin()), "overlapping scalar-memory load did not capture its sources before "