diff --git a/src/graphics/shader/recompiler/ShaderRecompiler.cpp b/src/graphics/shader/recompiler/ShaderRecompiler.cpp index 398501ffa..f83b79fa0 100644 --- a/src/graphics/shader/recompiler/ShaderRecompiler.cpp +++ b/src/graphics/shader/recompiler/ShaderRecompiler.cpp @@ -901,6 +901,7 @@ bool TryRecompile(std::span code, const CompileOptions& options, if (!IR::TrackResources(ir, error)) { return false; } + IR::EliminateDeadCode(ir.values->blocks); if (options.stage == ShaderType::Vertex) { ir.info.vertex_offset_sgpr = embedded_fetch.vertex_offset_sgpr; } @@ -922,6 +923,7 @@ bool TryRecompile(std::span code, const CompileOptions& options, runtime.shader_base = options.shader_base != 0 ? options.shader_base : reinterpret_cast(code.data()); runtime.read_memory = options.read_memory; + runtime.read_specialization_memory = options.read_specialization_memory; if (runtime.read_memory == nullptr && options.user_data == nullptr) { runtime.read_memory = ReadZeroMemory; } diff --git a/src/graphics/shader/recompiler/ShaderRecompiler.h b/src/graphics/shader/recompiler/ShaderRecompiler.h index 072f56148..a7b99bdaf 100644 --- a/src/graphics/shader/recompiler/ShaderRecompiler.h +++ b/src/graphics/shader/recompiler/ShaderRecompiler.h @@ -21,18 +21,19 @@ struct CompileOptions { uint64_t shader_hash = 0; uint64_t shader_base = 0; std::optional flat_memory_base; - uint32_t descriptor_set = 0; - uint32_t push_constant_offset = 0; - bool dump_ir = true; - bool early_dump = false; - const char* dump_label = nullptr; - const uint32_t* user_data = nullptr; - IR::SrtMemoryReader read_memory = nullptr; - void* read_memory_data = nullptr; - const IR::ResourceSnapshot* resource_snapshot = nullptr; - const ShaderVertexInputInfo* vertex_input_info = nullptr; - const ShaderPixelInputInfo* pixel_input_info = nullptr; - const ShaderComputeInputInfo* compute_input_info = nullptr; + uint32_t descriptor_set = 0; + uint32_t push_constant_offset = 0; + bool dump_ir = true; + bool early_dump = false; + const char* dump_label = nullptr; + const uint32_t* user_data = nullptr; + IR::SrtMemoryReader read_memory = nullptr; + IR::SrtMemoryReader read_specialization_memory = nullptr; + void* read_memory_data = nullptr; + const IR::ResourceSnapshot* resource_snapshot = nullptr; + const ShaderVertexInputInfo* vertex_input_info = nullptr; + const ShaderPixelInputInfo* pixel_input_info = nullptr; + const ShaderComputeInputInfo* compute_input_info = nullptr; }; struct CompileResult { diff --git a/src/graphics/shader/recompiler/ir/ValueProgram.h b/src/graphics/shader/recompiler/ir/ValueProgram.h index d5324dbae..2743264ca 100644 --- a/src/graphics/shader/recompiler/ir/ValueProgram.h +++ b/src/graphics/shader/recompiler/ir/ValueProgram.h @@ -4,6 +4,7 @@ #include "graphics/shader/recompiler/ir/ShaderIR.h" #include +#include #include #include @@ -19,8 +20,18 @@ struct ValueBlockInfo { }; struct DescriptorSource { - std::array dwords {}; - uint32_t dword_count = 0; + struct IndirectImage { + uint32_t material_source = 0; + uint32_t heap_source = 0; + uint32_t selector_stride = 0; + uint32_t selector_offset = 0; + + bool operator==(const IndirectImage& other) const = default; + }; + + std::array dwords {}; + uint32_t dword_count = 0; + std::optional indirect_image; bool operator==(const DescriptorSource& other) const = default; }; diff --git a/src/graphics/shader/recompiler/ir/passes/ResourceMaterialization.cpp b/src/graphics/shader/recompiler/ir/passes/ResourceMaterialization.cpp index b1dd5a988..c72d8cd5f 100644 --- a/src/graphics/shader/recompiler/ir/passes/ResourceMaterialization.cpp +++ b/src/graphics/shader/recompiler/ir/passes/ResourceMaterialization.cpp @@ -5,13 +5,18 @@ #include "graphics/shader/shaderBindings.h" #include +#include +#include #include #include +#include +#include namespace Libs::Graphics::ShaderRecompiler::IR { namespace { -constexpr uint64_t AddressMask = 0x0000ffffffffffffull; +constexpr uint64_t AddressMask = 0x0000ffffffffffffull; +constexpr uint64_t MaxIndirectImageProbes = 65536u; Decoder::ImageDimension DescriptorDimension(const DescriptorValue& descriptor, Decoder::ImageDimension requested) { @@ -89,6 +94,167 @@ bool DecodeBufferDescriptor(const DescriptorValue& descriptor, ShaderBufferResou return true; } +const DescriptorSource* Source(const Program& program, uint32_t source) { + if (program.values == nullptr || source >= program.values->descriptor_sources.size()) { + return nullptr; + } + return &program.values->descriptor_sources[source]; +} + +void MarkCleanFlatSlots(const Program& program, const DescriptorSource* source, + std::span slots) { + if (source == nullptr) { + return; + } + std::vector pending(source->dwords.begin(), + source->dwords.begin() + source->dword_count); + std::vector visited; + while (!pending.empty()) { + auto value = pending.back().Resolve(); + pending.pop_back(); + const auto* inst = value.TryInstruction(); + if (inst == nullptr || std::ranges::find(visited, inst) != visited.end()) { + continue; + } + visited.push_back(inst); + if (inst->GetOpcode() == ValueOpcode::ReadConst) { + const auto slot = inst->Arg(1).Resolve(); + if (slot.IsImmediate() && slot.GetType() == Type::U32 && slot.U32() < slots.size()) { + slots[slot.U32()] = 1u; + pending.push_back(program.values->srt_reads[slot.U32()].value); + } + continue; + } + for (size_t arg = 0; arg < inst->NumArgs(); arg++) { + pending.push_back(inst->Arg(arg)); + } + } +} + +uint64_t ScalarBufferSize(const ShaderBufferResource& descriptor) { + return descriptor.Stride() == 0u + ? descriptor.NumRecords() + : static_cast(descriptor.Stride()) * descriptor.NumRecords(); +} + +bool ReadSpecializationWord(const SrtRuntime& runtime, uint64_t address, uint32_t& word) { + return runtime.read_specialization_memory != nullptr && + runtime.read_specialization_memory(runtime.userdata, address, &word); +} + +bool ReadScalarBufferWord(const ShaderBufferResource& descriptor, uint32_t dynamic_offset, + uint32_t immediate_offset, const SrtRuntime& runtime, uint32_t& word, + std::string* error, uint32_t use_pc) { + const auto byte_offset = static_cast(dynamic_offset) + immediate_offset; + const auto aligned = byte_offset & ~uint64_t {3}; + const auto size = ScalarBufferSize(descriptor); + if (aligned > size || size - aligned < sizeof(uint32_t)) { + word = 0; + return true; + } + const auto base = descriptor.Base48() & ~uint64_t {3}; + if (aligned > AddressMask - base) { + if (error != nullptr) { + *error = fmt::format("indirect image scalar read at pc 0x{:08x} exceeds the " + "48-bit address space", + use_pc); + } + return false; + } + const auto address = base + aligned; + if (!ReadSpecializationWord(runtime, address, word)) { + if (error != nullptr) { + *error = fmt::format("indirect image scalar read at pc 0x{:08x} failed at " + "0x{:016x}", + use_pc, address); + } + return false; + } + return true; +} + +bool MaterializeIndirectImage(const DescriptorSource::IndirectImage& indirect, + const DescriptorValue& material_value, + const DescriptorValue& heap_value, const SrtRuntime& runtime, + DescriptorValue& result, std::string* error, uint32_t use_pc) { + ShaderBufferResource material; + ShaderBufferResource heap; + if (!DecodeBufferDescriptor(material_value, material) || + !DecodeBufferDescriptor(heap_value, heap)) { + if (error != nullptr) { + *error = fmt::format("indirect image tables at pc 0x{:08x} have invalid descriptors", + use_pc); + } + return false; + } + if (material.Stride() != indirect.selector_stride) { + if (error != nullptr) { + *error = + fmt::format("indirect image material table at pc 0x{:08x} changed stride", use_pc); + } + return false; + } + + // S_BUFFER_LOAD ignores vector-buffer swizzle/add-thread fields. The shader computes the + // record stride explicitly; enumerate every wrapped 32-bit offset that can pass bounds. + const auto period = uint64_t {1} << 32u; + const auto step = std::gcd(indirect.selector_stride, period); + const auto residue = static_cast(indirect.selector_offset) % step; + const auto size = ScalarBufferSize(material); + const auto limit = std::min(UINT32_MAX, size + 3u); + const auto probe_count = residue <= limit ? (limit - residue) / step + 1u : 0u; + if (probe_count > MaxIndirectImageProbes) { + if (error != nullptr) { + *error = fmt::format("indirect image material table at pc 0x{:08x} requires {} " + "bounded probes", + use_pc, probe_count); + } + return false; + } + + std::vector keys {0u}; + std::unordered_set seen {0u}; + for (uint64_t offset = residue; offset <= limit && probe_count != 0u; offset += step) { + uint32_t key = 0; + if (!ReadScalarBufferWord(material, static_cast(offset), 0u, runtime, key, error, + use_pc)) { + return false; + } + if (seen.insert(key).second) { + keys.push_back(key); + } + if (limit - offset < step) { + break; + } + } + + DescriptorValue invariant; + bool have_invariant = false; + for (const auto key: keys) { + DescriptorValue candidate; + candidate.dword_count = 8u; + const auto heap_offset = key << 5u; + for (uint32_t dword = 0; dword < candidate.dword_count; dword++) { + if (!ReadScalarBufferWord(heap, heap_offset, dword * sizeof(uint32_t), runtime, + candidate.dwords[dword], error, use_pc)) { + return false; + } + } + if (!have_invariant) { + invariant = candidate; + have_invariant = true; + } else if (candidate != invariant) { + if (error != nullptr) { + *error = + fmt::format("indirect image table at pc 0x{:08x} is not invariant", use_pc); + } + return false; + } + } + result = invariant; + return true; +} + uint64_t AddressSpecialization(const AddressResource& resource, const ResourceSnapshot::Address& snapshot) { return resource.kind == ResourceKind::Flat || resource.unbased @@ -305,13 +471,30 @@ bool MaterializeResources(const Program& program, const SrtRuntime& runtime, } std::vector requests; - requests.reserve(program.info.buffers.size() + program.info.images.size() + + std::vector clean_flat_slots(program.values->srt_reads.size()); + requests.reserve(program.info.buffers.size() + program.info.images.size() * 2u + program.info.samplers.size() + program.info.addresses.size()); for (const auto& buffer: program.info.buffers) { requests.push_back({buffer.source, buffer.first_use_pc}); } for (const auto& image: program.info.images) { - requests.push_back({image.source, image.first_use_pc}); + const auto* source = Source(program, image.source); + if (source != nullptr && source->indirect_image.has_value()) { + if (runtime.read_specialization_memory == nullptr) { + if (error != nullptr) { + *error = fmt::format("indirect image table at pc 0x{:08x} has no clean " + "memory reader", + image.first_use_pc); + } + return false; + } + MarkCleanFlatSlots(program, Source(program, source->indirect_image->material_source), + clean_flat_slots); + MarkCleanFlatSlots(program, Source(program, source->indirect_image->heap_source), + clean_flat_slots); + } else { + requests.push_back({image.source, image.first_use_pc}); + } } for (const auto& sampler: program.info.samplers) { requests.push_back({sampler.source, sampler.first_use_pc}); @@ -324,7 +507,8 @@ bool MaterializeResources(const Program& program, const SrtRuntime& runtime, std::vector values; std::vector flattened_srt; - if (!EvaluateRuntimeSources(program, requests, runtime, values, flattened_srt, error)) { + if (!EvaluateRuntimeSources(program, requests, runtime, values, flattened_srt, clean_flat_slots, + error)) { return false; } @@ -338,12 +522,34 @@ bool MaterializeResources(const Program& program, const SrtRuntime& runtime, descriptor.dwords.fill(0); } } - next.images.assign(cursor, cursor + program.info.images.size()); - cursor += program.info.images.size(); - for (auto& descriptor: next.images) { + next.images.reserve(program.info.images.size()); + for (const auto& image: program.info.images) { + const auto* source = Source(program, image.source); + DescriptorValue descriptor; + if (source != nullptr && source->indirect_image.has_value()) { + const std::array requests { + DescriptorSourceRequest {source->indirect_image->material_source, + image.first_use_pc}, + DescriptorSourceRequest {source->indirect_image->heap_source, image.first_use_pc}}; + SrtRuntime clean_runtime = runtime; + clean_runtime.read_memory = runtime.read_specialization_memory; + std::vector tables; + if (!EvaluateDescriptorSources(program, requests, clean_runtime, tables, error)) { + return false; + } + const auto& material = tables[0]; + const auto& heap = tables[1]; + if (!MaterializeIndirectImage(*source->indirect_image, material, heap, runtime, + descriptor, error, image.first_use_pc)) { + return false; + } + } else { + descriptor = *cursor++; + } if (!ValidImageDescriptor(descriptor)) { descriptor.dwords.fill(0); } + next.images.push_back(descriptor); } next.samplers.assign(cursor, cursor + program.info.samplers.size()); cursor += program.info.samplers.size(); diff --git a/src/graphics/shader/recompiler/ir/passes/ResourceTracking.cpp b/src/graphics/shader/recompiler/ir/passes/ResourceTracking.cpp index d44e973e5..250376d05 100644 --- a/src/graphics/shader/recompiler/ir/passes/ResourceTracking.cpp +++ b/src/graphics/shader/recompiler/ir/passes/ResourceTracking.cpp @@ -5,6 +5,7 @@ #include #include +#include #include #include @@ -148,6 +149,7 @@ public: if (!m_program.srt_plan_complete) { return Fail(0, error, "SRT plan is not ready"); } + PlanIndirectImages(); for (auto* block: m_values.blocks) { for (auto& inst: *block) { if (!Collect(inst, error)) { @@ -166,6 +168,20 @@ public: memory.sampler = patch.sampler; } } + for (const auto& plan: m_indirect_images) { + for (uint32_t dword = 0; dword < plan.roots.size(); dword++) { + plan.handle->SetArg(dword, plan.roots[dword]); + } + for (const auto index: plan.memory) { + m_values.memory_info[index].planning_only = true; + } + } + std::erase_if(m_values.dynamic_reads, [&](Value value) { + const auto* inst = value.Resolve().TryInstruction(); + return std::ranges::any_of(m_indirect_images, [&](const IndirectImagePlan& plan) { + return std::ranges::find(plan.reads, inst) != plan.reads.end(); + }); + }); m_values.descriptor_sources = std::move(m_sources); m_program.info = std::move(m_info); m_program.resource_tracking_complete = true; @@ -185,6 +201,14 @@ private: bool has_sampler = false; }; + struct IndirectImagePlan { + Inst* handle = nullptr; + uint32_t source = 0; + std::array roots {}; + std::array memory {}; + std::array reads {}; + }; + bool Fail(uint32_t pc, std::string* error, const std::string& reason) const { if (error != nullptr) { *error = fmt::format("shader resource tracking: hash=0x{:016x} stage={} pc=0x{:08x} {}", @@ -310,7 +334,8 @@ private: uint32_t InternSource(const DescriptorSource& descriptor) { for (uint32_t candidate = 0; candidate < m_sources.size(); candidate++) { const auto& current = m_sources[candidate]; - if (current.dword_count != descriptor.dword_count) { + if (current.dword_count != descriptor.dword_count || + current.indirect_image != descriptor.indirect_image) { continue; } bool same = true; @@ -325,6 +350,236 @@ private: return static_cast(m_sources.size() - 1); } + static bool ImmediateU32(Value value, uint32_t& result) { + value = value.Resolve(); + if (!value.IsImmediate() || value.GetType() != Type::U32) { + return false; + } + result = value.U32(); + return true; + } + + static bool UsesOnly(const Inst& value, std::span users) { + return !value.Uses().empty() && std::ranges::all_of(value.Uses(), [&](const Use& use) { + return std::ranges::find(users, use.user) != users.end(); + }); + } + + const MemoryInfo* ScalarReadMemory(const Inst& read, uint32_t& index) const { + if (read.GetOpcode() != ValueOpcode::ReadConstBuffer || read.NumArgs() != 2u) { + return nullptr; + } + index = read.Flags().index; + if (index >= m_values.memory_info.size()) { + return nullptr; + } + const auto& memory = m_values.memory_info[index]; + return memory.kind == ResourceKind::ScalarBuffer && memory.data_bits == 32u && + memory.data_dwords == 1u + ? &memory + : nullptr; + } + + bool MemoryIndexBelongsTo(uint32_t index, const Inst& owner) const { + for (const auto* block: m_values.blocks) { + for (const auto& inst: *block) { + const auto op = inst.GetOpcode(); + if ((!IsBuffer(op) && !IsAddress(op) && !IsImage(op)) || &inst == &owner) { + continue; + } + if (inst.Flags().index == index) { + return false; + } + } + } + return true; + } + + bool MakeRuntimeBufferSource(const Inst& handle, uint32_t pc, uint32_t& source, + DescriptorSource& descriptor) { + if (handle.GetOpcode() != ValueOpcode::GetBufferResource || + !MakeSource(handle, 4u, false, descriptor, pc, nullptr)) { + return false; + } + std::string reason; + uint32_t bad_dword = 0; + if (!ValidateSource(descriptor, reason, bad_dword)) { + return false; + } + source = InternSource(descriptor); + return true; + } + + bool MatchMaterialOffset(Value value, Value& selector, uint32_t& stride, + uint32_t& offset) const { + value = value.Resolve(); + offset = 0; + auto* candidate = value.TryInstruction(); + if (candidate != nullptr && candidate->GetOpcode() == ValueOpcode::IAdd32 && + candidate->NumArgs() == 2u) { + uint32_t immediate = 0; + if (ImmediateU32(candidate->Arg(0), immediate)) { + value = candidate->Arg(1).Resolve(); + } else if (ImmediateU32(candidate->Arg(1), immediate)) { + value = candidate->Arg(0).Resolve(); + } else { + return false; + } + offset = immediate; + } + const auto* multiply = value.TryInstruction(); + if (multiply == nullptr || multiply->GetOpcode() != ValueOpcode::IMul32 || + multiply->NumArgs() != 2u) { + return false; + } + if (ImmediateU32(multiply->Arg(0), stride)) { + selector = multiply->Arg(1).Resolve(); + } else if (ImmediateU32(multiply->Arg(1), stride)) { + selector = multiply->Arg(0).Resolve(); + } else { + return false; + } + const auto* selector_inst = selector.TryInstruction(); + return stride != 0u && selector_inst != nullptr && + selector_inst->GetOpcode() == ValueOpcode::ReadFirstLane; + } + + bool TryMakeIndirectImage(Inst& handle, uint32_t pc, IndirectImagePlan& plan) { + if (handle.GetOpcode() != ValueOpcode::GetImageResource || handle.NumArgs() != 8u) { + return false; + } + + std::array heap_reads {}; + Inst* heap_handle = nullptr; + Value heap_offset; + for (uint32_t dword = 0; dword < heap_reads.size(); dword++) { + heap_reads[dword] = handle.Arg(dword).Resolve().TryInstruction(); + if (heap_reads[dword] == nullptr) { + return false; + } + uint32_t memory_index = 0; + const auto* memory = ScalarReadMemory(*heap_reads[dword], memory_index); + if (memory == nullptr || memory->offset != dword * sizeof(uint32_t) || + !MemoryIndexBelongsTo(memory_index, *heap_reads[dword])) { + return false; + } + auto* current_handle = heap_reads[dword]->Arg(0).Resolve().TryInstruction(); + if (current_handle == nullptr || + (heap_handle != nullptr && current_handle != heap_handle)) { + return false; + } + heap_handle = current_handle; + if (dword == 0u) { + heap_offset = heap_reads[dword]->Arg(1).Resolve(); + } else if (!EquivalentValue(m_values, heap_offset, heap_reads[dword]->Arg(1))) { + return false; + } + plan.memory[dword + 1u] = memory_index; + plan.reads[dword + 1u] = heap_reads[dword]; + } + + const auto* shift = heap_offset.TryInstruction(); + uint32_t shift_amount = 0; + if (shift == nullptr || shift->GetOpcode() != ValueOpcode::ShiftLeftLogical32 || + shift->NumArgs() != 2u || !ImmediateU32(shift->Arg(1), shift_amount) || + shift_amount != 5u) { + return false; + } + auto* material_read = shift->Arg(0).Resolve().TryInstruction(); + if (material_read == nullptr) { + return false; + } + uint32_t material_memory_index = 0; + const auto* material_memory = ScalarReadMemory(*material_read, material_memory_index); + if (material_memory == nullptr || material_memory->offset != 0u || + !MemoryIndexBelongsTo(material_memory_index, *material_read)) { + return false; + } + auto* material_handle = material_read->Arg(0).Resolve().TryInstruction(); + if (material_handle == nullptr) { + return false; + } + + Value selector; + uint32_t selector_stride = 0; + uint32_t selector_offset = 0; + if (!MatchMaterialOffset(material_read->Arg(1), selector, selector_stride, + selector_offset)) { + return false; + } + + const std::array material_users {shift}; + std::array heap_users {}; + std::copy(heap_reads.begin(), heap_reads.end(), heap_users.begin()); + const std::array image_users {&handle}; + if (!UsesOnly(*material_read, material_users) || !UsesOnly(*shift, heap_users)) { + return false; + } + for (const auto* read: heap_reads) { + if (!UsesOnly(*read, image_users)) { + return false; + } + } + + DescriptorSource material_source; + DescriptorSource heap_source; + uint32_t material_source_index = 0; + uint32_t heap_source_index = 0; + if (!MakeRuntimeBufferSource(*material_handle, pc, material_source_index, + material_source) || + !MakeRuntimeBufferSource(*heap_handle, pc, heap_source_index, heap_source)) { + return false; + } + + DescriptorSource image_source; + image_source.dword_count = 8u; + std::copy(material_source.dwords.begin(), material_source.dwords.begin() + 4u, + image_source.dwords.begin()); + std::copy(heap_source.dwords.begin(), heap_source.dwords.begin() + 4u, + image_source.dwords.begin() + 4u); + image_source.indirect_image = DescriptorSource::IndirectImage { + material_source_index, heap_source_index, selector_stride, selector_offset}; + + plan.handle = &handle; + plan.source = InternSource(image_source); + plan.roots = image_source.dwords; + plan.memory[0] = material_memory_index; + plan.reads[0] = material_read; + return true; + } + + const IndirectImagePlan* FindIndirectImage(const Inst& handle) const { + const auto found = + std::ranges::find_if(m_indirect_images, [&](const IndirectImagePlan& plan) { + return plan.handle == &handle; + }); + return found == m_indirect_images.end() ? nullptr : &*found; + } + + bool IsIndirectPlanningMemory(uint32_t index) const { + return std::ranges::any_of(m_indirect_images, [&](const IndirectImagePlan& plan) { + return std::ranges::find(plan.memory, index) != plan.memory.end(); + }); + } + + void PlanIndirectImages() { + for (auto* block: m_values.blocks) { + for (auto& inst: *block) { + if (!IsImage(inst.GetOpcode()) || inst.NumArgs() == 0u) { + continue; + } + auto* handle = inst.Arg(0).Resolve().TryInstruction(); + if (handle == nullptr || FindIndirectImage(*handle) != nullptr) { + continue; + } + IndirectImagePlan plan; + if (TryMakeIndirectImage(*handle, inst.Flags().pc, plan)) { + m_indirect_images.push_back(std::move(plan)); + } + } + } + } + bool GetHandle(Value value, ValueOpcode expected, uint32_t width, uint32_t pc, Inst*& handle, uint32_t& source, std::string* error, bool sampler = false) { handle = value.Resolve().TryInstruction(); @@ -560,7 +815,7 @@ private: return Fail(flags.pc, error, "memory operation has no resource handle"); } const auto& memory = m_values.memory_info[flags.index]; - if (memory.planning_only) { + if (memory.planning_only || IsIndirectPlanningMemory(flags.index)) { return true; } Inst* handle = nullptr; @@ -592,8 +847,12 @@ private: AddMemoryPatch(flags.index, resource, 0, false, flags.pc, error); } - if (!GetHandle(inst.Arg(0), ValueOpcode::GetImageResource, 8, flags.pc, handle, source, - error)) { + handle = inst.Arg(0).Resolve().TryInstruction(); + const auto* indirect = handle != nullptr ? FindIndirectImage(*handle) : nullptr; + if (indirect != nullptr) { + source = indirect->source; + } else if (!GetHandle(inst.Arg(0), ValueOpcode::GetImageResource, 8, flags.pc, handle, + source, error)) { return false; } resource = AddImage(source, memory, op, flags.pc); @@ -638,7 +897,8 @@ private: } for (uint32_t image = 0; image < m_info.images.size(); image++) { const auto* image_source = Source(m_info.images[image].source); - if (image_source == nullptr || image_source->dword_count != 8) { + if (image_source == nullptr || image_source->dword_count != 8 || + image_source->indirect_image.has_value()) { continue; } bool alias = true; @@ -654,12 +914,13 @@ private: } } - Program& m_program; - ValueProgram& m_values; - ShaderInfo m_info; - std::vector m_sources; - std::vector m_handle_patches; - std::vector m_memory_patches; + Program& m_program; + ValueProgram& m_values; + ShaderInfo m_info; + std::vector m_sources; + std::vector m_handle_patches; + std::vector m_memory_patches; + std::vector m_indirect_images; }; } // namespace diff --git a/src/graphics/shader/recompiler/ir/passes/SrtWalker.cpp b/src/graphics/shader/recompiler/ir/passes/SrtWalker.cpp index 05c08301c..3f8060baa 100644 --- a/src/graphics/shader/recompiler/ir/passes/SrtWalker.cpp +++ b/src/graphics/shader/recompiler/ir/passes/SrtWalker.cpp @@ -396,8 +396,10 @@ private: class Evaluator { public: - Evaluator(const Program& program, const SrtRuntime& runtime) - : m_program(program), m_values(*program.values), m_runtime(runtime) {} + Evaluator(const Program& program, const SrtRuntime& runtime, + std::span clean_flat_slots = {}, Evaluator* clean_evaluator = nullptr) + : m_program(program), m_values(*program.values), m_runtime(runtime), + m_clean_flat_slots(clean_flat_slots), m_clean_evaluator(clean_evaluator) {} void SetUsePc(uint32_t pc) { m_use_pc = pc; } @@ -596,6 +598,12 @@ private: slot.U32() >= m_values.srt_reads.size()) { return Fail(error, "invalid flattened SRT index"); } + if (slot.U32() < m_clean_flat_slots.size() && + m_clean_flat_slots[slot.U32()] != 0u && m_clean_evaluator != nullptr) { + m_clean_evaluator->SetUsePc(m_use_pc); + return m_clean_evaluator->EvaluateWide(m_values.srt_reads[slot.U32()].value, + result, error); + } return EvaluateWide(m_values.srt_reads[slot.U32()].value, result, error); } case ValueOpcode::LoadAddressU32: @@ -842,7 +850,9 @@ private: const Program& m_program; const ValueProgram& m_values; const SrtRuntime& m_runtime; - uint32_t m_use_pc = 0; + std::span m_clean_flat_slots; + Evaluator* m_clean_evaluator = nullptr; + uint32_t m_use_pc = 0; std::unordered_map m_cache; std::vector m_visiting; }; @@ -858,14 +868,24 @@ bool EvaluateRuntimeSourcesImpl(const Program& program std::span requests, const SrtRuntime& runtime, std::vector& results, std::vector& flat, bool evaluate_flat, - std::string* error) { + std::span clean_flat_slots, std::string* error) { if (program.values == nullptr || !program.srt_plan_complete) { if (error != nullptr) { *error = Diagnostic(program, 0, "typed SRT plan is not ready"); } return false; } - Evaluator evaluator(program, runtime); + if (std::ranges::any_of(clean_flat_slots, [](uint8_t clean) { return clean != 0u; }) && + runtime.read_specialization_memory == nullptr) { + if (error != nullptr) { + *error = Diagnostic(program, 0, "clean flattened SRT read has no memory reader"); + } + return false; + } + SrtRuntime clean_runtime = runtime; + clean_runtime.read_memory = runtime.read_specialization_memory; + Evaluator clean_evaluator(program, clean_runtime); + Evaluator evaluator(program, runtime, clean_flat_slots, &clean_evaluator); std::vector evaluated; evaluated.reserve(requests.size()); for (const auto& request: requests) { @@ -891,9 +911,12 @@ bool EvaluateRuntimeSourcesImpl(const Program& program if (evaluate_flat) { flattened.resize(program.values->srt_reads.size()); for (const auto& read: program.values->srt_reads) { - evaluator.SetUsePc(read.use_pc); + const bool clean = read.flat_offset < clean_flat_slots.size() && + clean_flat_slots[read.flat_offset] != 0u; + auto& selected = clean ? clean_evaluator : evaluator; + selected.SetUsePc(read.use_pc); if (read.flat_offset >= flattened.size() || - !evaluator.Evaluate(read.value, flattened[read.flat_offset], error)) { + !selected.Evaluate(read.value, flattened[read.flat_offset], error)) { return false; } } @@ -948,20 +971,23 @@ bool EvaluateDescriptorSources(const Program& program, const SrtRuntime& runtime, std::vector& results, std::string* error) { std::vector ignored; - return EvaluateRuntimeSourcesImpl(program, requests, runtime, results, ignored, false, error); + return EvaluateRuntimeSourcesImpl(program, requests, runtime, results, ignored, false, {}, + error); } bool EvaluateRuntimeSources(const Program& program, std::span requests, const SrtRuntime& runtime, std::vector& results, - std::vector& flat, std::string* error) { - return EvaluateRuntimeSourcesImpl(program, requests, runtime, results, flat, true, error); + std::vector& flat, std::span clean_flat_slots, + std::string* error) { + return EvaluateRuntimeSourcesImpl(program, requests, runtime, results, flat, true, + clean_flat_slots, error); } bool WalkSrt(const Program& program, const SrtRuntime& runtime, std::vector& flat, std::string* error) { std::vector ignored; - return EvaluateRuntimeSources(program, {}, runtime, ignored, flat, error); + return EvaluateRuntimeSources(program, {}, runtime, ignored, flat, {}, error); } } // namespace Libs::Graphics::ShaderRecompiler::IR diff --git a/src/graphics/shader/recompiler/ir/passes/SrtWalker.h b/src/graphics/shader/recompiler/ir/passes/SrtWalker.h index c22dff069..f21223ce5 100644 --- a/src/graphics/shader/recompiler/ir/passes/SrtWalker.h +++ b/src/graphics/shader/recompiler/ir/passes/SrtWalker.h @@ -18,6 +18,7 @@ struct SrtRuntime { SrtMemoryReader read_memory = nullptr; void* userdata = nullptr; std::optional flat_memory_base; + SrtMemoryReader read_specialization_memory = nullptr; }; struct DescriptorSourceRequest { @@ -46,7 +47,8 @@ bool EvaluateDescriptorSources(const Program& program, bool EvaluateRuntimeSources(const Program& program, std::span requests, const SrtRuntime& runtime, std::vector& results, - std::vector& flat, std::string* error); + std::vector& flat, std::span clean_flat_slots, + std::string* error); bool WalkSrt(const Program& program, const SrtRuntime& runtime, std::vector& flat, std::string* error); diff --git a/src/graphics/shader/shader.cpp b/src/graphics/shader/shader.cpp index cd831c4ed..3789182fe 100644 --- a/src/graphics/shader/shader.cpp +++ b/src/graphics/shader/shader.cpp @@ -17,6 +17,7 @@ #include "graphics/shader/recompiler/frontend/decode/ShaderDecoder.h" #include "graphics/shader/recompiler/ir/ValueProgram.h" #include "graphics/shader/shaderVertexMetadata.h" +#include "kernel/memory.h" #include "libs/errno.h" #include "spirv-tools/libspirv.h" #include "spirv-tools/libspirv.hpp" @@ -51,6 +52,11 @@ namespace { constexpr uint32_t PsInputOffsetMask = 0x0000001fu; constexpr uint32_t PsInputFlatShade = 0x00000400u; +bool ReadShaderGuestMemory(void*, uint64_t address, uint32_t* value) { + return value != nullptr && + Libs::LibKernel::Memory::TryReadGpuCleanBacking(address, value, sizeof(*value)); +} + } // namespace uint32_t ShaderPixelParameterMappedLocation(const ShaderPixelInputInfo& info, uint32_t input) { @@ -958,7 +964,7 @@ static bool TryUseVertexPermutation(const ShaderProgramPermutation& permutation, if (!ShaderMaterializeStageRuntime( permutation.program, std::span(regs.gs_user_sgpr.value, regs.gs_regs.rsrc2.user_sgpr), - regs.es_regs.data_addr, info.stage, &error)) { + regs.es_regs.data_addr, info.stage, &error, ReadShaderGuestMemory)) { return LogPermutationMismatch(permutation, "VS", shader_hash, error); } ApplyVertexOutputs(info, *permutation.program); @@ -972,7 +978,7 @@ static bool TryUsePixelPermutation(const ShaderProgramPermutation& permutation, if (!ShaderMaterializeStageRuntime( permutation.program, std::span(regs.ps_user_sgpr.value, regs.ps_regs.rsrc2.user_sgpr), - regs.ps_regs.data_addr, info.stage, &error)) { + regs.ps_regs.data_addr, info.stage, &error, ReadShaderGuestMemory)) { return LogPermutationMismatch(permutation, "PS", shader_hash, error); } ApplyPixelOutputs(info, *permutation.program); @@ -986,7 +992,7 @@ static bool TryUseComputePermutation(const ShaderProgramPermutation& permutation if (!ShaderMaterializeStageRuntime( permutation.program, std::span(regs.cs_user_sgpr.value, regs.cs_regs.user_sgpr), - regs.cs_regs.data_addr, info.stage, &error)) { + regs.cs_regs.data_addr, info.stage, &error, ReadShaderGuestMemory)) { return LogPermutationMismatch(permutation, "CS", shader_hash, error); } return true; @@ -1429,19 +1435,20 @@ bool ShaderCompileSpirvVS(const HW::VertexShaderInfo& regs, const HW::ShaderRegi const auto code = ShaderGetMappedCode(shader_addr, "ShaderRecompiler VS", regs.gs_regs.chksum); ShaderRecompiler::CompileOptions options; - options.stage = ShaderType::Vertex; - options.lane_mask_mode = lane_mask_mode; - options.shader_hash = regs.gs_regs.chksum; - options.shader_base = shader_addr; - options.user_data_base = 8; - options.user_data_count = regs.gs_regs.rsrc2.user_sgpr; - options.user_data = regs.gs_user_sgpr.value; - options.descriptor_set = 0; - options.push_constant_offset = 0; - options.vertex_input_info = &input_info; - options.dump_ir = ShaderRecompilerTextDumpEnabled(); - options.early_dump = options.dump_ir; - options.dump_label = "ShaderRecompiler VS"; + options.stage = ShaderType::Vertex; + options.lane_mask_mode = lane_mask_mode; + options.shader_hash = regs.gs_regs.chksum; + options.shader_base = shader_addr; + options.user_data_base = 8; + options.user_data_count = regs.gs_regs.rsrc2.user_sgpr; + options.user_data = regs.gs_user_sgpr.value; + options.read_specialization_memory = ReadShaderGuestMemory; + options.descriptor_set = 0; + options.push_constant_offset = 0; + options.vertex_input_info = &input_info; + options.dump_ir = ShaderRecompilerTextDumpEnabled(); + options.early_dump = options.dump_ir; + options.dump_label = "ShaderRecompiler VS"; ShaderRecompiler::CompileResult result; std::string error; @@ -1483,18 +1490,19 @@ bool ShaderCompileSpirvPS(const HW::PixelShaderInfo& regs, const HW::ShaderRegis const auto code = ShaderGetMappedCode(shader_addr, "ShaderRecompiler PS", shader_hash); ShaderRecompiler::CompileOptions options; - options.stage = ShaderType::Pixel; - options.lane_mask_mode = lane_mask_mode; - options.shader_hash = shader_hash; - options.shader_base = shader_addr; - options.user_data_count = regs.ps_regs.rsrc2.user_sgpr; - options.user_data = regs.ps_user_sgpr.value; - options.descriptor_set = input_info.descriptor_set; - options.push_constant_offset = input_info.push_constant_offset; - options.pixel_input_info = &input_info; - options.dump_ir = ShaderRecompilerTextDumpEnabled(); - options.early_dump = options.dump_ir; - options.dump_label = "ShaderRecompiler PS"; + options.stage = ShaderType::Pixel; + options.lane_mask_mode = lane_mask_mode; + options.shader_hash = shader_hash; + options.shader_base = shader_addr; + options.user_data_count = regs.ps_regs.rsrc2.user_sgpr; + options.user_data = regs.ps_user_sgpr.value; + options.read_specialization_memory = ReadShaderGuestMemory; + options.descriptor_set = input_info.descriptor_set; + options.push_constant_offset = input_info.push_constant_offset; + options.pixel_input_info = &input_info; + options.dump_ir = ShaderRecompilerTextDumpEnabled(); + options.early_dump = options.dump_ir; + options.dump_label = "ShaderRecompiler PS"; ShaderRecompiler::CompileResult result; std::string error; @@ -1533,18 +1541,19 @@ bool ShaderCompileSpirvCS(const HW::ComputeShaderInfo& regs, const HW::ShaderReg const auto code = ShaderGetMappedCode(shader_addr, "ShaderRecompiler CS", shader_addr); ShaderRecompiler::CompileOptions options; - options.stage = ShaderType::Compute; - options.shader_hash = shader_addr; - options.shader_base = shader_addr; - options.user_data_count = regs.cs_regs.user_sgpr; - options.user_data = regs.cs_user_sgpr.value; - options.descriptor_set = 0; - options.push_constant_offset = 0; - options.compute_input_info = &input_info; - options.wave_size = input_info.wave_size; - options.dump_ir = ShaderRecompilerTextDumpEnabled(); - options.early_dump = options.dump_ir; - options.dump_label = "ShaderRecompiler CS"; + options.stage = ShaderType::Compute; + options.shader_hash = shader_addr; + options.shader_base = shader_addr; + options.user_data_count = regs.cs_regs.user_sgpr; + options.user_data = regs.cs_user_sgpr.value; + options.read_specialization_memory = ReadShaderGuestMemory; + options.descriptor_set = 0; + options.push_constant_offset = 0; + options.compute_input_info = &input_info; + options.wave_size = input_info.wave_size; + options.dump_ir = ShaderRecompilerTextDumpEnabled(); + options.early_dump = options.dump_ir; + options.dump_label = "ShaderRecompiler CS"; ShaderRecompiler::CompileResult result; std::string error; diff --git a/src/graphics/shader/shader.h b/src/graphics/shader/shader.h index 7bcd976df..8e2e61597 100644 --- a/src/graphics/shader/shader.h +++ b/src/graphics/shader/shader.h @@ -39,11 +39,16 @@ struct ShaderStageRuntime { } }; +using ShaderSpecializationMemoryReader = bool (*)(void* userdata, uint64_t address, + uint32_t* value); + // Resolves an immutable native shader plan against current user data. The prior stage is preserved // if any ReadConst, snapshot, or specialization check fails. -bool ShaderMaterializeStageRuntime(std::shared_ptr program, - std::span user_data, uint64_t shader_base, - ShaderStageRuntime& stage, std::string* error); +bool ShaderMaterializeStageRuntime( + std::shared_ptr program, + std::span user_data, uint64_t shader_base, ShaderStageRuntime& stage, + std::string* error, ShaderSpecializationMemoryReader read_specialization_memory = nullptr, + void* read_memory_data = nullptr); struct ShaderId { uint32_t hash0 = 0; diff --git a/src/graphics/shader/shaderStageRuntime.cpp b/src/graphics/shader/shaderStageRuntime.cpp index 29bf5fd46..dd4db4d3f 100644 --- a/src/graphics/shader/shaderStageRuntime.cpp +++ b/src/graphics/shader/shaderStageRuntime.cpp @@ -7,7 +7,9 @@ namespace Libs::Graphics { bool ShaderMaterializeStageRuntime(std::shared_ptr program, std::span user_data, uint64_t shader_base, - ShaderStageRuntime& stage, std::string* error) { + ShaderStageRuntime& stage, std::string* error, + ShaderSpecializationMemoryReader read_specialization_memory, + void* read_memory_data) { if (program == nullptr) { if (error != nullptr) { *error = "missing native shader plan"; @@ -15,8 +17,10 @@ bool ShaderMaterializeStageRuntime(std::shared_ptrTryReadBacking(vaddr, data, size); } +bool TryReadGpuCleanBacking(uint64_t vaddr, void* data, uint64_t size) { + if (g_gpu_resources != nullptr && IsGpuAddressRange(vaddr, size)) { + if (!Graphics::Gpu::IsGpuThread() || + GetGpuResources().GetBufferCache().HasGpuDirtyBytes(vaddr, size) || + GetGpuResources().GetTextureCache().QueryRegion(vaddr, size).gpu_image_bytes) { + return false; + } + } + return TryReadBacking(vaddr, data, size); +} + uint64_t ClampRangeSize(uint64_t vaddr, uint64_t size) { EXIT_IF(g_virtual_ranges == nullptr); diff --git a/src/kernel/memory.h b/src/kernel/memory.h index 21cffef19..e51f9d2d0 100644 --- a/src/kernel/memory.h +++ b/src/kernel/memory.h @@ -109,6 +109,7 @@ void RegisterCallbacks(callback_func_t alloc_func, callback_fu void SetFlexibleMemorySize(uint64_t size); bool TryWriteBacking(uint64_t vaddr, const void* data, uint64_t size); bool TryReadBacking(uint64_t vaddr, void* data, uint64_t size); +bool TryReadGpuCleanBacking(uint64_t vaddr, void* data, uint64_t size); bool TryReadPrtBacking(uint64_t vaddr, void* data, uint64_t size); [[nodiscard]] uint64_t ClampRangeSize(uint64_t vaddr, uint64_t size); void WriteBacking(uint64_t vaddr, const void* data, uint64_t size) noexcept; diff --git a/tests/ResourceTrackingTests.cpp b/tests/ResourceTrackingTests.cpp index 2d831980a..0fed5c5b5 100644 --- a/tests/ResourceTrackingTests.cpp +++ b/tests/ResourceTrackingTests.cpp @@ -1,10 +1,11 @@ #include "graphics/guest_gpu/gpu_defs.h" +#include "graphics/shader/recompiler/ir/ValueProgram.h" #include "graphics/shader/recompiler/ir/passes/BindingLayout.h" +#include "graphics/shader/recompiler/ir/passes/DeadCodeElimination.h" #include "graphics/shader/recompiler/ir/passes/ResourceMaterialization.h" #include "graphics/shader/recompiler/ir/passes/ResourceTracking.h" #include "graphics/shader/recompiler/ir/passes/ShaderInfoCollection.h" #include "graphics/shader/recompiler/ir/passes/SrtWalker.h" -#include "graphics/shader/recompiler/ir/ValueProgram.h" #include #include @@ -139,6 +140,206 @@ bool ReadTestMemory(void *userdata, uint64_t address, uint32_t *value) { return true; } +struct LinearTestMemory { + uint64_t base = 0x1000; + std::vector words = std::vector(0x2200 / 4); + uint64_t fail_address = UINT64_MAX; +}; + +bool ReadLinearTestMemory(void *userdata, uint64_t address, uint32_t *value) { + auto *memory = static_cast(userdata); + if (memory == nullptr || value == nullptr || address < memory->base || + address - memory->base >= memory->words.size() * sizeof(uint32_t) || + (address & 3u) != 0u || address == memory->fail_address) { + return false; + } + *value = memory->words[(address - memory->base) / sizeof(uint32_t)]; + return true; +} + +std::unique_ptr +MakeIndirectImageFixture(bool malformed, uint32_t material_immediate = 0, + bool memory_backed_material = false) { + auto fixture = std::make_unique(); + std::array material_words; + std::array heap_words; + for (uint32_t dword = 0; dword < 4; dword++) { + material_words[dword] = fixture->UserData(dword); + heap_words[dword] = fixture->UserData(dword + 4u); + } + if (memory_backed_material) { + const auto pointer_address = + fixture->Address(fixture->UserData(9), fixture->UserData(10), 0x10b0); + MemoryInfo pointer_word; + pointer_word.kind = ResourceKind::ScalarBuffer; + const auto pointer = + fixture->Emit(ValueOpcode::LoadAddressU32, + {pointer_address, Value(0u), Value(0u), Value(true)}, + fixture->AddMemory(pointer_word, 0x10b0)); + const auto address = fixture->Address(pointer, Value(0u), 0x10c0); + MemoryInfo descriptor_word; + descriptor_word.kind = ResourceKind::ScalarBuffer; + material_words[0] = + fixture->Emit(ValueOpcode::LoadAddressU32, + {address, Value(0u), Value(0u), Value(true)}, + fixture->AddMemory(descriptor_word, 0x10c0)); + } + const auto material = fixture->Buffer(material_words, 0x10d8); + const auto heap = fixture->Buffer(heap_words, 0x10d8); + if (memory_backed_material) { + MemoryInfo shared_buffer; + shared_buffer.kind = ResourceKind::Buffer; + const auto load = + fixture->Emit(ValueOpcode::LoadBufferU32, + {material, Value(0u), Value(0u), Value(0u), Value(true)}, + fixture->AddMemory(shared_buffer, 0x10d8)); + fixture->Emit(ValueOpcode::ReferenceU32, {load}); + } + const auto selector = fixture->Emit(ValueOpcode::ReadFirstLane, + {fixture->UserData(8), Value(true)}); + const auto record = + fixture->Emit(ValueOpcode::IMul32, {selector, Value(224u)}); + const auto member = fixture->Emit(ValueOpcode::IAdd32, {record, Value(4u)}); + fixture->Emit(ValueOpcode::ReferenceU32, {record}); + fixture->Emit(ValueOpcode::ReferenceU32, {member}); + MemoryInfo material_scalar; + material_scalar.kind = ResourceKind::ScalarBuffer; + material_scalar.offset = material_immediate; + const auto key = + fixture->Emit(ValueOpcode::ReadConstBuffer, {material, member}, + fixture->AddMemory(material_scalar, 0x10d8)); + const auto heap_offset = + fixture->Emit(ValueOpcode::ShiftLeftLogical32, {key, Value(5u)}); + std::array image_words; + MemoryInfo heap_scalar; + heap_scalar.kind = ResourceKind::ScalarBuffer; + for (uint32_t dword = 0; dword < image_words.size(); dword++) { + auto component = heap_scalar; + component.offset = dword * sizeof(uint32_t); + if (malformed && dword == image_words.size() - 1u) { + component.offset += sizeof(uint32_t); + } + image_words[dword] = + fixture->Emit(ValueOpcode::ReadConstBuffer, {heap, heap_offset}, + fixture->AddMemory(component, 0x10d8)); + } + const auto image = fixture->Image(image_words, 0x10f0); + const auto sampler = + fixture->Sampler({Value(0u), Value(0u), Value(0u), Value(0u)}, 0x10f0); + MemoryInfo sample; + sample.kind = ResourceKind::Image; + sample.image_dimension = Decoder::ImageDimension::Dim2D; + const auto sampled = fixture->Emit(ValueOpcode::ImageSampleRaw, + {image, sampler, fixture->ImageAddress()}, + fixture->AddMemory(sample, 0x10f0)); + const auto sampled_x = + fixture->Emit(ValueOpcode::CompositeExtractU32x4, {sampled, Value(0u)}); + fixture->Emit(ValueOpcode::ReferenceU32, {sampled_x}); + return fixture; +} + +void TestInvariantIndirectImageMaterialization() { + auto fixture = MakeIndirectImageFixture(false); + fixture->PlanAndTrack(); + EliminateDeadCode(fixture->program.values->blocks); + std::string validation_error; + Check( + ValidateValueProgram(*fixture->program.values, true, &validation_error), + "post-tracking dead-code elimination invalidated descriptor provenance"); + + Check(fixture->program.info.buffers.empty() && + fixture->program.info.images.size() == 1 && + fixture->program.values->dynamic_reads.empty(), + "indirect image planning reads leaked into shader resources"); + const auto source = fixture->program.info.images[0].source; + Check(source < fixture->program.values->descriptor_sources.size() && + fixture->program.values->descriptor_sources[source] + .indirect_image.has_value(), + "indirect image source was not retained for runtime proof"); + + std::array user_data{0x1000u, 224u << 16u, 2u, 0u, 0x2000u, + 16u << 16u, 4u, 0u, 7u}; + LinearTestMemory memory; + std::array image_descriptor{}; + image_descriptor[0] = 0x20u; + image_descriptor[1] = + static_cast( + Libs::Graphics::Prospero::BufferFormat::k32_32_32_32Float) + << 20u; + image_descriptor[2] = 3u | (3u << 14u); + image_descriptor[3] = + Libs::Graphics::DstSel(4, 5, 6, 7) | + (static_cast(Libs::Graphics::Prospero::ImageType::kColor2D) + << 28u); + for (uint32_t dword = 0; dword < image_descriptor.size(); dword++) { + memory.words[(0x2000u - memory.base) / 4u + dword] = + image_descriptor[dword]; + memory.words[(0x2020u - memory.base) / 4u + dword] = + image_descriptor[dword]; + } + memory.words[(0x2020u - memory.base) / 4u] ^= 1u; + + SrtRuntime runtime{.user_data = user_data, + .userdata = &memory, + .read_specialization_memory = ReadLinearTestMemory}; + ResourceSnapshot snapshot; + std::string error; + Check(MaterializeResources(fixture->program, runtime, snapshot, &error) && + snapshot.images.size() == 1 && + std::equal(image_descriptor.begin(), image_descriptor.end(), + snapshot.images[0].dwords.begin()), + "invariant indirect image table did not materialize"); + + const auto prior_image = snapshot.images[0]; + memory.fail_address = 0x1004u; + Check(!MaterializeResources(fixture->program, runtime, snapshot, &error) && + error.find("scalar read") != std::string::npos && + snapshot.images[0] == prior_image, + "rejected planning memory read mutated the snapshot"); + memory.fail_address = UINT64_MAX; + memory.words[(0x1000u - memory.base + 36u) / 4u] = 1u; + Check(!MaterializeResources(fixture->program, runtime, snapshot, &error) && + error.find("not invariant") != std::string::npos && + snapshot.images[0] == prior_image, + "divergent indirect image table was accepted or mutated the snapshot"); + + auto memory_backed = MakeIndirectImageFixture(false, 0u, true); + memory_backed->PlanAndTrack(); + EliminateDeadCode(memory_backed->program.values->blocks); + std::array memory_backed_user_data{0x1000u, 224u << 16u, 2u, 0u, + 0x2000u, 16u << 16u, 4u, 0u, + 7u, 0x3100u, 0u}; + memory.words[(0x3100u - memory.base) / 4u] = 0x3000u; + memory.words[(0x3000u - memory.base) / 4u] = 0x1000u; + memory.fail_address = 0x3100u; + SrtRuntime memory_backed_runtime{.user_data = memory_backed_user_data, + .userdata = &memory, + .read_specialization_memory = + ReadLinearTestMemory}; + Check(!MaterializeResources(memory_backed->program, memory_backed_runtime, + snapshot, &error) && + error.find("constant read failed") != std::string::npos && + snapshot.images[0] == prior_image, + "rejected indirect table descriptor read mutated the snapshot"); + memory.fail_address = UINT64_MAX; + + auto malformed = MakeIndirectImageFixture(true); + Check(BuildSrtPlan(malformed->program, &error) && + !TrackResources(malformed->program, &error) && + error.find("ReadFirstLane") != std::string::npos && + !malformed->program.resource_tracking_complete && + malformed->program.info.images.empty() && + malformed->program.values->descriptor_sources.empty(), + "malformed indirect image pattern was partially accepted"); + + auto wrapped_immediate = MakeIndirectImageFixture(false, 4u); + Check(BuildSrtPlan(wrapped_immediate->program, &error) && + !TrackResources(wrapped_immediate->program, &error) && + error.find("ReadFirstLane") != std::string::npos && + !wrapped_immediate->program.resource_tracking_complete, + "wrapped scalar immediate entered the invariant image proof"); +} + void TestDenseBufferTracking() { Fixture fixture; std::array userdata; @@ -236,8 +437,8 @@ void TestScalarAndVectorBufferAlias() { void TestRuntimeUnsignedMinDescriptor() { Fixture fixture; - const auto word3 = fixture.Emit( - ValueOpcode::UMin32, {fixture.UserData(0), Value(0x100u)}); + const auto word3 = + fixture.Emit(ValueOpcode::UMin32, {fixture.UserData(0), Value(0x100u)}); const auto descriptor = fixture.Buffer({Value(0u), Value(0u), Value(64u), word3}, 0x330); MemoryInfo memory; @@ -257,10 +458,11 @@ void TestRuntimeUnsignedMinDescriptor() { value.dwords[3] == 0x100u, "runtime descriptor unsigned minimum did not clamp its first operand"); user_data[0] = 0x80u; - Check(EvaluateDescriptorSource(fixture.program, source, 0x330, runtime, value, - &error) && - value.dwords[3] == 0x80u, - "runtime descriptor unsigned minimum did not preserve its first operand"); + Check( + EvaluateDescriptorSource(fixture.program, source, 0x330, runtime, value, + &error) && + value.dwords[3] == 0x80u, + "runtime descriptor unsigned minimum did not preserve its first operand"); } void TestImagesSamplersAndAliases() { @@ -400,8 +602,8 @@ void TestDynamicStorageMipTracking() { Check(CollectShaderInfo(fixture.program, {.compute = &compute}, &error) && AllocateBindings(fixture.program, {}, &error), "dynamic storage mip bindings were not allocated"); - const auto *storage_binding = FindBinding( - fixture.program.bindings, DescriptorBindingKind::Storage2D); + const auto *storage_binding = + FindBinding(fixture.program.bindings, DescriptorBindingKind::Storage2D); Check(storage_binding != nullptr && storage_binding->resources == std::vector({0, 1, 1, 1}), "dynamic storage mip descriptors were not expanded consecutively"); @@ -479,7 +681,7 @@ void TestSrtFlatteningAndRuntimeMemoization() { 12}; std::string error; Check(EvaluateRuntimeSources(fixture.program, std::span{&request, 1}, runtime, - descriptors, flat, &error), + descriptors, flat, {}, &error), "typed runtime source evaluation failed"); Check(descriptors.size() == 1 && descriptors[0].dwords[0] == 0xdeadbeefu && flat == std::vector{0xdeadbeefu} && memory.reads == 1, @@ -490,7 +692,7 @@ void TestSrtFlatteningAndRuntimeMemoization() { descriptors = {{{1u}, 1u}}; flat = {2u}; Check(!EvaluateRuntimeSources(fixture.program, std::span{&request, 1}, - runtime, descriptors, flat, &error) && + runtime, descriptors, flat, {}, &error) && descriptors == std::vector{{{1u}, 1u}} && flat == std::vector{2u}, "runtime evaluation failure was not transactional"); @@ -561,12 +763,11 @@ void TestPhiValidation() { static_cast(Type::U32)); phi.AddPhiOperand(left, Value(1u)); phi.AddPhiOperand(right, Value(2u)); - const auto word3 = fixture.Emit( - ValueOpcode::UMin32, {Value(&phi), Value(0x100u)}, 0, merge); - const auto handle = - fixture.Emit(ValueOpcode::GetBufferResource, - {Value(0u), Value(0u), Value(0u), word3}, - MemoryFlags{0, 20}, merge); + const auto word3 = + fixture.Emit(ValueOpcode::UMin32, {Value(&phi), Value(0x100u)}, 0, merge); + const auto handle = fixture.Emit(ValueOpcode::GetBufferResource, + {Value(0u), Value(0u), Value(0u), word3}, + MemoryFlags{0, 20}, merge); MemoryInfo memory; memory.kind = ResourceKind::Buffer; fixture.Emit(ValueOpcode::LoadBufferU32, @@ -593,13 +794,13 @@ void TestLoopCycleEnteredThroughRuntimeValue() { loop->AddBranch(loop); auto &phi = loop->AppendNewInst(ValueOpcode::Phi, {}, static_cast(Type::U32)); - const auto carried = fixture.Emit( - ValueOpcode::BitwiseAnd32, {Value(&phi), Value(0xffffffffu)}, 0, loop); + const auto carried = fixture.Emit(ValueOpcode::BitwiseAnd32, + {Value(&phi), Value(0xffffffffu)}, 0, loop); phi.AddPhiOperand(entry, initial); phi.AddPhiOperand(loop, carried); fixture.Emit(ValueOpcode::GetBufferResource, - {carried, Value(0u), Value(0u), Value(0u)}, - MemoryFlags{0, 12}, loop); + {carried, Value(0u), Value(0u), Value(0u)}, MemoryFlags{0, 12}, + loop); std::string error; Check(BuildSrtPlan(fixture.program, &error), @@ -683,10 +884,9 @@ void TestAddressMaterializationAndSpecialization() { void TestBufferSwizzleSpecialization() { Fixture fixture; - const auto handle = fixture.Buffer( - {fixture.UserData(0), fixture.UserData(1), fixture.UserData(2), - fixture.UserData(3)}, - 4); + const auto handle = fixture.Buffer({fixture.UserData(0), fixture.UserData(1), + fixture.UserData(2), fixture.UserData(3)}, + 4); MemoryInfo memory; memory.kind = ResourceKind::Buffer; memory.formatted = true; @@ -804,6 +1004,7 @@ int main() { Run("runtime unsigned min", TestRuntimeUnsignedMinDescriptor); Run("images and samplers", TestImagesSamplersAndAliases); Run("dynamic storage mips", TestDynamicStorageMipTracking); + Run("invariant indirect images", TestInvariantIndirectImageMaterialization); Run("SRT runtime", TestSrtFlatteningAndRuntimeMemoization); Run("dynamic SRT", TestDynamicSrtReadRemainsExplicit); Run("phi validation", TestPhiValidation); @@ -838,5 +1039,6 @@ void DbgExit(int) { throw std::runtime_error("typed IR assertion failed"); } #include "graphics/shader/recompiler/ir/Block.cpp" #include "graphics/shader/recompiler/ir/Type.cpp" #include "graphics/shader/recompiler/ir/Value.cpp" -#include "graphics/shader/recompiler/ir/opcodes/ValueOpcodes.cpp" #include "graphics/shader/recompiler/ir/ValueProgram.cpp" +#include "graphics/shader/recompiler/ir/opcodes/ValueOpcodes.cpp" +#include "graphics/shader/recompiler/ir/passes/DeadCodeElimination.cpp" diff --git a/tests/ShaderStageRuntimeTests.cpp b/tests/ShaderStageRuntimeTests.cpp index 633282e7c..05154f4a2 100644 --- a/tests/ShaderStageRuntimeTests.cpp +++ b/tests/ShaderStageRuntimeTests.cpp @@ -1,5 +1,5 @@ -#include "graphics/shader/recompiler/ir/passes/ResourceMaterialization.h" #include "graphics/shader/recompiler/ir/ValueProgram.h" +#include "graphics/shader/recompiler/ir/passes/ResourceMaterialization.h" #include "graphics/shader/shader.h" #include @@ -15,6 +15,11 @@ void Check(bool value, const char *text) { } } +bool RejectSpecializationRead(void *userdata, uint64_t, uint32_t *) { + ++*static_cast(userdata); + return false; +} + Libs::Graphics::ShaderRecompiler::IR::Block & AddValueBlock(Libs::Graphics::ShaderRecompiler::IR::Program &program) { using namespace Libs::Graphics::ShaderRecompiler::IR; @@ -84,8 +89,13 @@ void TestMappedSrtUsesDirectReaderByDefault() { auto cached_program = SrtProgram(reinterpret_cast(&dword)); ShaderStageRuntime stage; std::string error; - Check(ShaderMaterializeStageRuntime(cached_program, {}, 0, stage, &error), + uint32_t specialization_reads = 0; + Check(ShaderMaterializeStageRuntime(cached_program, {}, 0, stage, &error, + RejectSpecializationRead, + &specialization_reads), error.c_str()); + Check(specialization_reads == 0, + "ordinary SRT read used the specialization reader"); Check(stage.program == cached_program && stage.resources != nullptr, "cache rematerialization did not publish the mapped stage"); Check(stage.resources->flattened_srt.size() == 1 && @@ -131,5 +141,5 @@ int main() { #include "graphics/shader/recompiler/ir/Block.cpp" #include "graphics/shader/recompiler/ir/Type.cpp" #include "graphics/shader/recompiler/ir/Value.cpp" -#include "graphics/shader/recompiler/ir/opcodes/ValueOpcodes.cpp" #include "graphics/shader/recompiler/ir/ValueProgram.cpp" +#include "graphics/shader/recompiler/ir/opcodes/ValueOpcodes.cpp"