From e291589177acfe04531d011d0b65854a6c079bdc Mon Sep 17 00:00:00 2001 From: nmzik Date: Thu, 15 Oct 2026 15:06:28 +0200 Subject: [PATCH] shader: resolve GPU-selected parent pointers and bounded child image tables Co-authored-by: fxpw --- src/graphics/shader/recompiler/ir/ShaderIR.h | 10 +- .../ir/passes/ResourceMaterialization.cpp | 76 +++--- .../recompiler/ir/passes/ResourceTracking.cpp | 237 +++++++++++++----- tests/ResourceTrackingTests.cpp | 220 +++++++++++++++- tests/ShaderRecompilerComputeTests.cpp | 186 ++++++++++---- 5 files changed, 583 insertions(+), 146 deletions(-) diff --git a/src/graphics/shader/recompiler/ir/ShaderIR.h b/src/graphics/shader/recompiler/ir/ShaderIR.h index a86345036..450995080 100644 --- a/src/graphics/shader/recompiler/ir/ShaderIR.h +++ b/src/graphics/shader/recompiler/ir/ShaderIR.h @@ -549,9 +549,17 @@ struct DescriptorSource { bool operator==(const SelectorRead& other) const = default; }; + struct PointerTable { + uint32_t offset = 0; + uint32_t stride = 0; + uint32_t count = 1; + bool indexed = false; // Parent pointers use raw MUBUF record addressing. + + bool operator==(const PointerTable& other) const = default; + }; std::optional selector; std::optional table_maximum; - std::optional pointer_offset; + std::optional pointer_table; uint32_t table_source = 0; uint32_t table_offset = 0; uint32_t table_immediate = 0; diff --git a/src/graphics/shader/recompiler/ir/passes/ResourceMaterialization.cpp b/src/graphics/shader/recompiler/ir/passes/ResourceMaterialization.cpp index 0de29de82..d0ed1859f 100644 --- a/src/graphics/shader/recompiler/ir/passes/ResourceMaterialization.cpp +++ b/src/graphics/shader/recompiler/ir/passes/ResourceMaterialization.cpp @@ -588,9 +588,18 @@ bool MaterializeIndirectDescriptor(const ResourcePlan& p table.OutOfBounds() != 0u || (table_base & 3u) != 0u || (table.Stride() & 3u) != 0u || table.Stride() < indirect.table_record_bytes))) return false; - const bool affine = indirect.table_stride != 0u; - const bool image_heap = dword_count == 8u && !indirect.pointer_offset; - const uint64_t step = indirect.table_maximum ? indirect.table_stride + const auto* pointer = indirect.pointer_table ? &*indirect.pointer_table : nullptr; + const bool indexed_pointer = pointer != nullptr && pointer->indexed; + const auto child_count = pointer != nullptr ? pointer->count : 1u; + if (indexed_pointer && + (!indirect.table_maximum || table.Type() != 0u || table.SwizzleEnabled() || table.AddTid() || + table.OutOfBounds() != 0u || (table_base & 3u) != 0u || (table.Stride() & 3u) != 0u || + uint64_t {indirect.table_immediate} + 8u > table.Stride() || + *indirect.table_maximum >= table.NumRecords())) return false; + const bool affine = indirect.table_stride != 0u || indexed_pointer; + const bool image_heap = dword_count == 8u && pointer == nullptr; + const uint64_t step = indexed_pointer ? table.Stride() + : indirect.table_maximum ? indirect.table_stride : affine ? std::max(uint64_t {1} << std::countr_zero(indirect.table_stride), uint64_t {4}) : 4u; const uint64_t first = ((indirect.table_maximum ? indirect.table_offset @@ -601,7 +610,8 @@ bool MaterializeIndirectDescriptor(const ResourcePlan& p const auto count = table_size >= first + 4u ? std::min((table_size - first - 4u) / step + 1u, domain) : 0u; - if (!image_heap && count > MaxIndirectDescriptorProbes) return false; + if (child_count == 0u || (!image_heap && count * child_count > MaxIndirectDescriptorProbes)) + return false; // Affine SMEM offsets wrap in U32 before adding the aligned immediate. // Image heaps retain one bulk snapshot and their absolute byte-offset mapping. if (image_heap) { @@ -612,34 +622,40 @@ bool MaterializeIndirectDescriptor(const ResourcePlan& p descriptors[resource_index] = {.dword_count = dword_count}; snapshot.flattened_srt.resize(mapping_offset + 2u, 0u); for (uint32_t word = 0; word < count; ++word) { - DescriptorValue candidate {.dword_count = dword_count}; - const auto table_offset = affine - ? first + word * step : uint64_t {word} * 4u; - if (indirect.pointer_offset) { + const auto table_offset = affine ? first + word * step : uint64_t {word} * 4u; + uint64_t pointer_base = 0u; + if (pointer != nullptr) { std::array address {}; - if (!ReadScalarTable(table_base, table_size, table_offset, runtime, address) || - !ReadScalarTable((uint64_t {address[1]} << 32u) | address[0], UINT64_MAX, - *indirect.pointer_offset, runtime, - std::span(candidate.dwords).first(read_dwords))) return false; - } else if (!image_heap) { - if (!ReadScalarTable(table_base, table_size, table_offset, runtime, - std::span(candidate.dwords).first(read_dwords))) return false; - } else { - const auto table_word = table_offset / 4u; - std::copy_n(keys.begin() + table_word, - std::min(read_dwords, keys.size() - table_word), - candidate.dwords.begin()); + if (!ReadScalarTable(table_base, table_size, table_offset, runtime, address)) return false; + pointer_base = (uint64_t {address[1]} << 32u) | address[0]; + } + for (uint32_t child = 0; child < child_count; ++child) { + DescriptorValue candidate {.dword_count = dword_count}; + if (pointer != nullptr) { + if (!ReadScalarTable(pointer_base, UINT64_MAX, + uint64_t {child} * pointer->stride + pointer->offset, runtime, + std::span(candidate.dwords).first(read_dwords))) return false; + } else if (!image_heap) { + if (!ReadScalarTable(table_base, table_size, table_offset, runtime, + std::span(candidate.dwords).first(read_dwords))) return false; + } else { + const auto table_word = table_offset / 4u; + std::copy_n(keys.begin() + table_word, + std::min(read_dwords, keys.size() - table_word), + candidate.dwords.begin()); + } + if (!normalize(candidate)) + return normalization_failed(table_offset, word, step, + indirect.table_maximum ? "key" : "probe"); + uint32_t ordinal; + if (!intern_candidate(candidate, ordinal)) return false; + if (ordinal == 0u) continue; + const auto mapping_word = image_heap ? table_offset / 4u : word * child_count + child; + snapshot.flattened_srt.resize(mapping_offset + 2u + mapping_word, 0u); + snapshot.flattened_srt[mapping_offset + 1u + mapping_word] = ordinal; } - if (!normalize(candidate)) - return normalization_failed(table_offset, word, step, - indirect.table_maximum ? "key" : "probe"); - uint32_t ordinal; - if (!intern_candidate(candidate, ordinal)) return false; - if (ordinal == 0u) continue; - const auto mapping_word = image_heap ? table_offset / 4u : word; - snapshot.flattened_srt.resize(mapping_offset + 2u + mapping_word, 0u); - snapshot.flattened_srt[mapping_offset + 1u + mapping_word] = ordinal; } + if (descriptors.size() == children_begin) { snapshot.flattened_srt.resize(mapping_offset); } else { @@ -871,7 +887,7 @@ static bool BuildResourceSpecialization(const ResourcePlan& program, ResourceSna if (i >= program.info.images.size() && source != nullptr && source->indirect_descriptor && IsBoundedDescriptorTable(program, *source->indirect_descriptor) && - !source->indirect_descriptor->pointer_offset) { + !source->indirect_descriptor->pointer_table) { const auto& indirect = *source->indirect_descriptor; const size_t mapping = specialization.images[base_index].indirect_mapping_offset; const auto& flat = snapshot.flattened_srt; diff --git a/src/graphics/shader/recompiler/ir/passes/ResourceTracking.cpp b/src/graphics/shader/recompiler/ir/passes/ResourceTracking.cpp index aa9dda1c0..e7f79b239 100644 --- a/src/graphics/shader/recompiler/ir/passes/ResourceTracking.cpp +++ b/src/graphics/shader/recompiler/ir/passes/ResourceTracking.cpp @@ -985,7 +985,7 @@ private: shared_image_heap = descriptor.dword_count == 8u && (a.table_record_bytes != 0u || b.table_record_bytes != 0u); if (a.selector != b.selector || a.table_maximum != b.table_maximum || - a.pointer_offset != b.pointer_offset || + a.pointer_table != b.pointer_table || a.table_source != b.table_source || (!shared_image_heap && (a.table_offset != b.table_offset || a.table_immediate != b.table_immediate || @@ -2691,12 +2691,13 @@ private: memory.component_count == 4u)); } - bool MatchDescriptorTable(Inst& handle, const DescriptorSource& descriptor, - IndirectDescriptorPlan& plan, - DescriptorSource& table_source, Value& key, uint32_t& table_offset, - uint32_t& table_stride, uint32_t& table_immediate) { + bool MatchDescriptorReads(Inst& handle, const DescriptorSource& descriptor, + IndirectDescriptorPlan& plan, Value& key, uint32_t& table_offset, + uint32_t& table_stride, uint32_t& table_immediate, + bool allow_fixed = false) { Inst* table_handle = nullptr; Value table_guard; + const uint32_t alignment_mask = allow_fixed ? ~3u : UINT32_MAX; for (uint32_t dword = 0; dword < plan.read_dwords; ++dword) { auto* read = descriptor.dwords[dword].Resolve().TryInstruction(); if (read == nullptr) { @@ -2728,7 +2729,7 @@ private: (dword != 0u && !EquivalentValue(m_program, table_guard, read->Arg(4)))) return false; table_guard = read->Arg(4); } - if (memory->offset > INT32_MAX || (memory->offset & 3u) != 0u || + if (memory->offset > INT32_MAX || (!allow_fixed && (memory->offset & 3u) != 0u) || !MemoryIndexBelongsTo(memory_index, *read)) { return false; } @@ -2743,7 +2744,8 @@ private: (memory->kind == ResourceKind::ScalarAddress && !AvailableBefore(*read, handle)) || (table_handle != nullptr && !EquivalentValue(m_program, Value(table_handle), Value(current_handle))) || - (!indexed && !MatchTableOffset(read->Arg(1), current_key, offset, stride))) { + (!indexed && !(allow_fixed && ImmediateU32(read->Arg(1), offset)) && + !MatchTableOffset(read->Arg(1), current_key, offset, stride))) { return false; } if (indexed) { @@ -2759,8 +2761,9 @@ private: table_stride = stride; } else if (plan.table_indexed != indexed || table_stride != stride || !EquivalentValue(m_program, key, current_key) || - uint64_t {table_offset} + table_immediate + dword * sizeof(uint32_t) != - uint64_t {offset} + memory->offset) { + uint64_t {table_offset & alignment_mask} + (table_immediate & alignment_mask) + + dword * sizeof(uint32_t) != + uint64_t {offset & alignment_mask} + (memory->offset & alignment_mask)) { return false; } plan.split_offsets |= !indexed && offset != table_offset; @@ -2780,14 +2783,20 @@ private: plan.reads[dword] = read; } - if (!MakeRuntimeTableSource(*plan.reads[0], table_source)) { - return false; - } return true; } - bool SetExactTableMaximum(Value key, DescriptorSource::IndirectDescriptor& indirect) { - const auto maximum = SelectorMaximum(key); + bool SetExactTableMaximum(Value key, DescriptorSource::IndirectDescriptor& indirect, + const Inst* handle = nullptr) { + uint32_t maximum = 0u; + if (handle != nullptr) { + if (handle->Uses().empty()) return false; + for (const auto& use: handle->Uses()) { + if (ImageOpcodeInfoOf(use.user->GetOpcode()).access == ImageAccess::None) return false; + maximum = std::max(maximum, SelectorMaximum(key, {}, 0u, + use.user->Arg(use.user->NumArgs() - 1u))); + } + } else maximum = SelectorMaximum(key); if (maximum > UINT32_MAX / 4u || uint64_t {maximum} * indirect.table_stride + indirect.table_offset > UINT32_MAX) return false; @@ -2799,61 +2808,112 @@ private: IndirectDescriptorPlan& plan) { if (handle.GetOpcode() != ValueOpcode::GetImageResource && handle.GetOpcode() != ValueOpcode::GetSamplerResource) return false; - Inst* pointer = nullptr; - uint32_t descriptor_offset = 0; - for (uint32_t word = 0; word < plan.read_dwords; ++word) { - const auto* read = descriptor.dwords[word].Resolve().TryInstruction(); - uint32_t index, offset; - const auto* memory = read != nullptr ? ScalarReadMemory(*read, index) : nullptr; - if (memory == nullptr || memory->kind != ResourceKind::ScalarAddress || - memory->offset > INT32_MAX || !ImmediateU32(read->Arg(1), offset) || - !AvailableBefore(*read, handle)) return false; - auto* address = read->Arg(0).Resolve().TryInstruction(); - const uint64_t current_offset = uint64_t {offset & ~3u} + (memory->offset & ~3u); - if (address == nullptr || address->GetOpcode() != ValueOpcode::GetAddressResource || - address->NumArgs() != 2u || current_offset > UINT32_MAX || - (pointer != nullptr && (!EquivalentValue(m_program, Value(pointer), Value(address)) || - current_offset != uint64_t {descriptor_offset} + word * 4u))) - return false; - if (word == 0u) descriptor_offset = static_cast(current_offset); - pointer = address; - } - // An out-of-bounds pointer pair is zero. Guest mappings exclude the first - // 16 KiB page, so these pointed reads have the existing null-image semantics. - if (pointer == nullptr || uint64_t {descriptor_offset} + plan.read_dwords * 4u > 0x4000u) - return false; - const auto* low = pointer->Arg(0).Resolve().TryInstruction(); - const auto* high = pointer->Arg(1).Resolve().TryInstruction(); - uint32_t low_index, high_index; - const auto* low_memory = low != nullptr ? ScalarReadMemory(*low, low_index) : nullptr; - const auto* high_memory = high != nullptr ? ScalarReadMemory(*high, high_index) : nullptr; - if (low_memory == nullptr || high_memory == nullptr || - low_memory->kind != ResourceKind::ScalarBuffer || - high_memory->kind != ResourceKind::ScalarBuffer || - low_memory->offset > INT32_MAX || high_memory->offset > INT32_MAX || - (low_memory->offset & ~3u) + 4u != (high_memory->offset & ~3u) || - !EquivalentValue(m_program, low->Arg(0), high->Arg(0)) || - !EquivalentValue(m_program, low->Arg(1), high->Arg(1)) || - !AvailableBefore(*low, *pointer) || !AvailableBefore(*high, *pointer)) return false; - Value key; + auto reads = plan; + Value child_key; + uint32_t offset = 0, stride = 0, immediate = 0; + if (!MatchDescriptorReads(handle, descriptor, reads, child_key, offset, stride, immediate, true) || + m_program.memory_info[reads.memory[0]].kind != ResourceKind::ScalarAddress || + uint64_t {offset & ~3u} + (immediate & ~3u) > UINT32_MAX) return false; + const auto* pointer = reads.reads[0]->Arg(0).ResolveInstruction(); DescriptorSource::IndirectDescriptor indirect; - if (!MatchTableOffset(low->Arg(1), key, indirect.table_offset, indirect.table_stride) || - indirect.table_stride < 8u || (indirect.table_stride & 3u) != 0u) return false; - if (!SetExactTableMaximum(key, indirect) || - (key.TryInstruction() != nullptr && !AvailableBefore(*key.Instruction(), handle))) - return false; + DescriptorSource::IndirectDescriptor::PointerTable child { + .offset = (offset & ~3u) + (immediate & ~3u), .stride = stride}; DescriptorSource table_source; - if (!MakeRuntimeTableSource(*low, table_source) || table_source.dword_count != 4u) - return false; + Value key; + if (child_key.IsEmpty()) { + // Missing scalar-buffer pointer words are zero-filled. Only the existing + // unmapped first page permits omitted rows to map directly to a null image. + if (uint64_t {child.offset} + plan.read_dwords * 4u > 0x4000u) return false; + const auto* low = pointer->Arg(0).Resolve().TryInstruction(); + const auto* high = pointer->Arg(1).Resolve().TryInstruction(); + uint32_t low_index, high_index; + const auto* low_memory = low != nullptr ? ScalarReadMemory(*low, low_index) : nullptr; + const auto* high_memory = high != nullptr ? ScalarReadMemory(*high, high_index) : nullptr; + if (low_memory == nullptr || high_memory == nullptr || + low_memory->kind != ResourceKind::ScalarBuffer || + high_memory->kind != ResourceKind::ScalarBuffer || + low_memory->offset > INT32_MAX || high_memory->offset > INT32_MAX || + (low_memory->offset & ~3u) + 4u != (high_memory->offset & ~3u) || + !EquivalentValue(m_program, low->Arg(0), high->Arg(0)) || + !EquivalentValue(m_program, low->Arg(1), high->Arg(1)) || + !AvailableBefore(*low, *pointer) || !AvailableBefore(*high, *pointer) || + !MatchTableOffset(low->Arg(1), key, indirect.table_offset, indirect.table_stride) || + indirect.table_stride < 8u || (indirect.table_stride & 3u) != 0u || + !SetExactTableMaximum(key, indirect) || + (key.TryInstruction() != nullptr && !AvailableBefore(*key.Instruction(), handle)) || + !MakeRuntimeTableSource(*low, table_source)) return false; + indirect.table_immediate = low_memory->offset & ~3u; + } else { + if (handle.GetOpcode() != ValueOpcode::GetImageResource || + !ImmediateU32(UnsignedResourceBound(child_key, handle), child.count) || child.count == 0u || + child.count > MaxIndirectDescriptorProbes || stride < plan.read_dwords * 4u || + ((offset | stride) & 3u) != 0u || + uint64_t {child.offset} + uint64_t {child.count - 1u} * stride + + plan.read_dwords * 4u > UINT32_MAX) return false; + const auto* low = pointer->Arg(0).Resolve().TryInstruction(); + const auto* high = pointer->Arg(1).Resolve().TryInstruction(); + if (low == nullptr || high == nullptr || low->GetOpcode() != ValueOpcode::ReadFirstLane || + high->GetOpcode() != ValueOpcode::ReadFirstLane || + !EquivalentValue(m_program, low->Arg(1), high->Arg(1))) return false; + const auto active = low->Arg(1); + const auto* low_word = ActiveValue(low->Arg(0), active).TryInstruction(); + auto high_value = ActiveValue(high->Arg(0), active); + const auto* extended = high_value.TryInstruction(); + // Scalar addresses use the low 48 bits, so signed high-WORD extension + // preserves exactly the same pointer as the original pair in memory. + if (extended != nullptr && extended->GetOpcode() == ValueOpcode::BitFieldSExtract && + extended->Arg(1).Resolve() == Value(0u) && extended->Arg(2).Resolve() == Value(16u)) + high_value = ActiveValue(extended->Arg(0), active); + const auto* high_word = high_value.TryInstruction(); + if (low_word == nullptr || high_word == nullptr || + low_word->GetOpcode() != ValueOpcode::CompositeExtractU32x2 || + high_word->GetOpcode() != ValueOpcode::CompositeExtractU32x2 || + low_word->Arg(1).Resolve() != Value(0u) || high_word->Arg(1).Resolve() != Value(1u) || + low_word->Arg(0).Resolve() != high_word->Arg(0).Resolve()) return false; + const auto* read = low_word->Arg(0).Resolve().TryInstruction(); + if (read == nullptr || read->GetOpcode() != ValueOpcode::LoadBufferU32x2 || + !Implies(active, read->Arg(4))) return false; + const auto& memory = m_program.memory_info[read->Flags().index]; + if (memory.kind != ResourceKind::Buffer || memory.typed || memory.formatted || + !memory.idxen || memory.offen || memory.data_bits != 32u || memory.data_dwords != 2u || + (memory.offset & 3u) != 0u || memory.offset > UINT32_MAX - 8u || + read->Arg(2).Resolve() != Value(0u) || read->Arg(3).Resolve() != Value(0u)) return false; + for (const auto& use: handle.Uses()) { + if (use.operand != 0u || ImageOpcodeInfoOf(use.user->GetOpcode()).access == ImageAccess::None) + return false; + const auto executing = use.user->Arg(use.user->NumArgs() - 1u); + if (!Implies(executing, active)) return false; + const auto outer = EqualLocalKey(executing, read->Arg(1)); + const auto* lane = outer.TryInstruction(); + if (lane == nullptr || lane->GetOpcode() != ValueOpcode::ReadFirstLane || + !EquivalentValue(m_program, lane->Arg(0), read->Arg(1)) || + !EquivalentValue(m_program, lane->Arg(1), active) || + !AvailableBefore(*lane, handle) || + (!key.IsEmpty() && !EquivalentValue(m_program, key, outer))) return false; + key = outer; + } + if (!SetExactTableMaximum(key, indirect, &handle) || + (uint64_t {*indirect.table_maximum} + 1u) * child.count > MaxIndirectDescriptorProbes || + !MakeRuntimeTableSource(*read, table_source)) return false; + indirect.table_immediate = memory.offset; + child.indexed = true; + } + if (table_source.dword_count != 4u) return false; indirect.table_source = InternSource(table_source); - indirect.table_immediate = low_memory->offset & ~3u; indirect.table_scalar = true; - indirect.pointer_offset = descriptor_offset; + indirect.pointer_table = child; DescriptorSource source; source.dword_count = handle.NumArgs(); source.dwords.fill(Value(0u)); std::copy_n(table_source.dwords.begin(), 4u, source.dwords.begin()); source.indirect_descriptor = indirect; + if (!child_key.IsEmpty()) { + auto* block = handle.Parent(); + const auto where = std::ranges::find_if(block->Instructions(), + [&](const Inst& inst) { return &inst == &handle; }); + key = Value(&*block->PrependNewInst(where, ValueOpcode::IMul32, {key, Value(child.count)})); + key = Value(&*block->PrependNewInst(where, ValueOpcode::IAdd32, {key, child_key})); + } plan.handle = &handle; plan.source = InternSource(source); plan.key = key; @@ -2872,8 +2932,9 @@ private: uint32_t table_offset = 0; uint32_t table_stride = 0; uint32_t table_immediate = 0; - if (!MatchDescriptorTable(handle, descriptor, plan, table_source, key, table_offset, - table_stride, table_immediate)) return false; + if (!MatchDescriptorReads(handle, descriptor, plan, key, table_offset, + table_stride, table_immediate) || + !MakeRuntimeTableSource(*plan.reads[0], table_source)) return false; DescriptorSource material_source; DescriptorSource::IndirectDescriptor indirect; indirect.table_offset = table_offset; @@ -2890,6 +2951,10 @@ private: // Keep live descriptor payloads, but project descriptor-only SMEM onto // the GPU key using the same plan as other indirect image reads. plan.retain_reads |= table_source.dword_count == 4u; + } else if (sampler && table_source.dword_count == 4u && !plan.table_indexed && + SetExactTableMaximum(key, indirect, &handle)) { + indirect.table_scalar = true; + plan.retain_reads = true; } else if (table_source.dword_count == 2u) { auto guarded_count = UnsignedBoundOnEntry(key, handle.Parent()); if (guarded_count.IsEmpty()) guarded_count = UnsignedResourceBound(key, handle); @@ -3068,8 +3133,9 @@ private: uint32_t table_offset = 0; uint32_t table_stride = 0; uint32_t table_immediate = 0; - if (!MatchDescriptorTable(handle, descriptor, plan, table_source, key, table_offset, + if (!MatchDescriptorReads(handle, descriptor, plan, key, table_offset, table_stride, table_immediate) || + !MakeRuntimeTableSource(*plan.reads[0], table_source) || table_stride < 16u || (table_stride & 3u) != 0u) return false; DescriptorSource::IndirectDescriptor indirect; indirect.table_stride = table_stride; @@ -3206,6 +3272,47 @@ private: return false; } + uint32_t FiniteSelectorMaximum(Value value, Value guard) const { + value = value.Resolve(); + bool nonnegative = false; + if (const auto* root = value.TryInstruction()) { + for (const auto& use: root->Uses()) { + const auto& compare = *use.user; + const auto op = compare.GetOpcode(); + if (((op == ValueOpcode::SLessThanEqual32 && use.operand == 1u && + compare.Arg(0).Resolve() == Value(0u)) || + (op == ValueOpcode::SGreaterThanEqual32 && use.operand == 0u && + compare.Arg(1).Resolve() == Value(0u))) && + Implies(guard, Value(use.user))) nonnegative = true; + } + } + std::vector pending {value}; + std::vector visited; + uint32_t maximum = 0u; + bool found = false; + for (size_t at = 0; at < pending.size(); ++at) { + uint32_t literal; + const auto current = pending[at].Resolve(); + if (ImmediateU32(current, literal)) { + if (nonnegative && std::bit_cast(literal) < 0) continue; + maximum = std::max(maximum, literal); + found = true; + continue; + } + const auto* choice = current.TryInstruction(); + if (choice == nullptr) return UINT32_MAX; + const bool phi = choice->GetOpcode() == ValueOpcode::Phi; + if ((!phi && choice->GetOpcode() != ValueOpcode::SelectU32) || + (phi && (choice->NumArgs() == 0u || choice->Parent() == nullptr || + InCycle(choice->Parent())))) return UINT32_MAX; + if (std::ranges::find(visited, choice) != visited.end()) continue; + visited.push_back(choice); + for (size_t arm = phi ? 0u : 1u; arm < choice->NumArgs(); ++arm) + pending.push_back(choice->Arg(arm)); + } + return found ? maximum : UINT32_MAX; + } + uint32_t SelectorMaximum(Value value, SelectorBounds bounds = {}, uint32_t depth = 0, Value guard = {}) const { if (depth > 32u) return UINT32_MAX; @@ -3219,6 +3326,10 @@ private: return SelectorMaximum(inst->Arg(arg), bounds, depth + 1u, guard); }; switch (inst->GetOpcode()) { + case ValueOpcode::ReadFirstLane: + if (guard.IsEmpty() ? !HasActiveLane(inst->Arg(1), inst->Parent()) + : !Implies(guard, inst->Arg(1))) return UINT32_MAX; + return FiniteSelectorMaximum(inst->Arg(0), inst->Arg(1)); case ValueOpcode::ReadLane: // ReadLane ignores EXEC; only context-free bounds on every source lane apply. return PossibleU32Bits(inst->Arg(0)); diff --git a/tests/ResourceTrackingTests.cpp b/tests/ResourceTrackingTests.cpp index 2ca7c9562..15ce25202 100644 --- a/tests/ResourceTrackingTests.cpp +++ b/tests/ResourceTrackingTests.cpp @@ -868,7 +868,9 @@ void TestNestedPointerImageTable() { const auto& indirect = descriptor.indirect_descriptor; Check(indirect && !indirect->selector && indirect->table_maximum == 65535u && indirect->table_stride == 592u && - indirect->pointer_offset == (descriptor.dword_count == 8u ? 0x3d0u : 0x3f8u), + indirect->pointer_table && indirect->pointer_table->count == 1u && + indirect->pointer_table->stride == 0u && !indirect->pointer_table->indexed && + indirect->pointer_table->offset == (descriptor.dword_count == 8u ? 0x3d0u : 0x3f8u), "nested pointer descriptors lost their row geometry or captured the GPU selector payload"); } @@ -976,6 +978,217 @@ void TestNestedPointerImageTable() { } } +void TestIndexedPointerImageTable() { + namespace CFG = Libs::Graphics::ShaderRecompiler::CFG; + enum class Case { Valid, ReverseSigned, MissingParent, WrongParent, MissingChild, + InclusiveChild, MismatchedPair, DifferentLaneMask, ExtraUse, + UnboundedChoice, MissingOuterWitness, WrongOuterWitness, CyclicChoice, + ExcessiveDomain }; + const auto make = [](Case variant = Case::Valid) { + auto fixture = std::make_unique(); + std::array table_words; + for (uint32_t word = 0; word < 4u; ++word) table_words[word] = fixture->UserData(word); + const auto table = fixture->Buffer(table_words); + const auto lane = fixture->Emit(ValueOpcode::LaneId); + const auto choice = fixture->Emit(ValueOpcode::ULessThan32, {lane, Value(8u)}); + const auto valid = fixture->Emit(ValueOpcode::ULessThan32, {lane, Value(16u)}); + auto row = fixture->Emit(ValueOpcode::SelectU32, + {valid, fixture->Emit(ValueOpcode::SelectU32, + {choice, Value(0u), variant == Case::UnboundedChoice ? lane : Value(1u)}), + Value(UINT32_MAX)}); + auto* entry = fixture->block; + auto* left = fixture->AddBlock(); + auto* right = fixture->AddBlock(); + auto* merge = fixture->AddBlock(); + entry->AddBranch(left); + entry->AddBranch(right); + entry->condition = choice; + entry->terminator = {.kind = CFG::TerminatorKind::ConditionalBranch, + .true_block = left, .false_block = right}; + for (auto* from : {left, right}) { + from->AddBranch(merge); + from->terminator = {.kind = CFG::TerminatorKind::Branch, .true_block = merge}; + } + fixture->block = merge; + auto& phi = merge->AppendNewInst(ValueOpcode::Phi, {}, uint64_t(Type::U32)); + phi.AddPhiOperand(left, row); + phi.AddPhiOperand(right, row); + row = Value(&phi); + if (variant == Case::CyclicChoice) { + merge->AddBranch(merge); + merge->terminator = {.kind = CFG::TerminatorKind::Branch, .true_block = merge}; + phi.AddPhiOperand(merge, row); + } else merge->terminator.kind = CFG::TerminatorKind::Return; + const auto parent = variant == Case::MissingParent ? Value(true) + : variant == Case::ReverseSigned + ? fixture->Emit(ValueOpcode::SGreaterThanEqual32, {row, Value(0u)}) + : fixture->Emit(variant == Case::WrongParent ? ValueOpcode::ULessThanEqual32 + : ValueOpcode::SLessThanEqual32, + {Value(0u), row}); + MemoryInfo pointer_memory; + pointer_memory.kind = ResourceKind::Buffer; + pointer_memory.idxen = true; + pointer_memory.data_dwords = pointer_memory.component_count = 2u; + const auto load = fixture->Emit(ValueOpcode::LoadBufferU32x2, + {table, row, Value(0u), Value(0u), parent}, + fixture->AddMemory(pointer_memory, 0x3d4u)); + const auto other = variant == Case::MismatchedPair + ? fixture->Emit(ValueOpcode::LoadBufferU32x2, + {table, Value(0u), Value(0u), Value(0u), parent}, + fixture->AddMemory(pointer_memory, 0x3dcu)) : load; + const auto low = fixture->Emit(ValueOpcode::SelectU32, + {parent, fixture->Emit(ValueOpcode::CompositeExtractU32x2, {load, Value(0u)}), Value(0u)}); + const auto high = fixture->Emit(ValueOpcode::SelectU32, + {parent, fixture->Emit(ValueOpcode::CompositeExtractU32x2, {other, Value(1u)}), Value(0u)}); + const auto child = fixture->Emit(ValueOpcode::ReadFirstLane, {lane, parent}); + const auto range = variant == Case::MissingChild ? Value(true) + : fixture->Emit(variant == Case::InclusiveChild ? ValueOpcode::ULessThanEqual32 + : ValueOpcode::ULessThan32, + {child, Value(variant == Case::ExcessiveDomain ? MaxIndirectDescriptorProbes : 3u)}); + const auto active = fixture->Emit(ValueOpcode::LogicalAnd, {parent, range}); + const auto extended = fixture->Emit(ValueOpcode::SelectU32, + {active, fixture->Emit(ValueOpcode::BitFieldSExtract, {high, Value(0u), Value(16u)}), Value(0u)}); + const auto uniform_low = fixture->Emit(ValueOpcode::ReadFirstLane, {low, active}); + const auto uniform_high = fixture->Emit(ValueOpcode::ReadFirstLane, + {extended, variant == Case::DifferentLaneMask + ? fixture->Emit(ValueOpcode::LogicalNot, {choice}) : active}); + const auto uniform_row = fixture->Emit(ValueOpcode::ReadFirstLane, {row, active}); + const auto row_equal = fixture->Emit(ValueOpcode::IEqual32, + {uniform_row, variant == Case::WrongOuterWitness ? lane : row}); + const auto executing = variant == Case::MissingOuterWitness ? active + : fixture->Emit(ValueOpcode::LogicalAnd, {active, row_equal}); + const auto address = fixture->Address(uniform_low, uniform_high); + const auto offset = fixture->Emit(ValueOpcode::IMul32, {child, Value(368u)}); + std::array words; + for (uint32_t word = 0; word < 8u; ++word) { + MemoryInfo memory; + memory.kind = ResourceKind::ScalarAddress; + memory.offset = 64u + word * 4u; + words[word] = fixture->Emit(ValueOpcode::LoadAddressU32, + {address, offset, Value(0u), Value(true)}, fixture->AddMemory(memory, 0x898u)); + } + const auto image = fixture->Image(words); + const auto sampler = fixture->Sampler({Value(0u), Value(0x00fff000u), Value(0u), Value(0u)}); + MemoryInfo image_memory; + image_memory.kind = ResourceKind::Image; + image_memory.image_dimension = Decoder::ImageDimension::Dim2D; + fixture->Emit(ValueOpcode::ImageSampleRaw, + {image, sampler, fixture->ImageAddress(), executing}, fixture->AddMemory(image_memory, 0x8b4u)); + if (variant == Case::ExtraUse) + fixture->Emit(ValueOpcode::ImageSampleRaw, + {image, sampler, fixture->ImageAddress(), parent}, fixture->AddMemory(image_memory, 0x8bcu)); + return fixture; + }; + auto fixture = make(); + fixture->PlanAndTrack(); + const auto plan = ExtractResourcePlan(fixture->program); + Check(plan.info.images.size() == 1u && plan.info.buffers.size() == 1u, + "indexed nested table expanded IR candidates or discarded the GPU pointer load"); + const auto& indirect = *plan.descriptor_sources[plan.info.images[0].source].indirect_descriptor; + Check(indirect.table_maximum == 1u && indirect.pointer_table && + indirect.pointer_table->indexed && indirect.pointer_table->count == 3u && + indirect.pointer_table->stride == 368u && indirect.pointer_table->offset == 64u, + "nested table did not preserve separate parent and child geometry"); + constexpr uint64_t high = uint64_t{1} << 32u; + constexpr uint32_t table = 0x2000u, objects = 0x10000u, row_bytes = 1536u; + LinearTestMemory memory; + memory.base += high; + memory.words.resize((objects + 2u * row_bytes - 0x1000u) / 4u, 0xdeadbeefu); + const auto at = [&](uint32_t address) { return memory.words.begin() + (address - 0x1000u) / 4u; }; + const auto image_words = [](uint32_t id) { + return std::array{id, + uint32_t(Libs::Graphics::Prospero::BufferFormat::k32Float) << 20u, 0u, + Libs::Graphics::DstSel(4, 0, 0, 1) | (9u << 28u), 0u, 0u, 0u, 0u}; + }; + // Runtime stride differs from the native sampler's 136-byte table arithmetic. + constexpr uint32_t stride = 160u; + for (uint32_t row = 0; row < 2u; ++row) { + *at(table + row * stride) = objects + row * row_bytes; + *at(table + row * stride + 4u) = 0xbeef0001u; + for (uint32_t child = 0; child < 3u; ++child) { + const auto image = image_words(10u + row * 3u + child); + std::copy(image.begin(), image.end(), at(objects + row * row_bytes + child * 368u + 64u)); + } + } + std::array user_data{table, 1u | (stride << 16u), 2u, 0x5204u}; + const SrtRuntime runtime{.user_data = user_data, .userdata = &memory, + .read_specialization_memory = ReadLinearTestMemory}; + ResourceSnapshot snapshot; + ResourceSpecialization specialization; + Check(MaterializeResources(plan, runtime, snapshot, specialization) && snapshot.images.size() == 7u, + "indexed pointer/child domain lost candidates or read poisoned child padding"); + const auto captured = [&](uint64_t address, uint64_t bytes) { + return std::ranges::any_of(snapshot.specialization_reads, [&](const auto& span) { + return span.first <= address && address - span.first <= span.second && + bytes <= span.second - (address - span.first); + }); + }; + for (uint32_t row = 0; row < 2u; ++row) { + Check(captured(high + table + row * stride, 8u), + "nested pointer read was absent from the captured alias ranges"); + for (uint32_t child = 0; child < 3u; ++child) + Check(captured(high + objects + row * row_bytes + child * 368u + 64u, 32u), + "nested child read was absent from the captured alias ranges"); + } + const auto mapping = specialization.images[0].indirect_mapping_offset; + Check(snapshot.flattened_srt[mapping] == 6u, + "nested child table did not produce one dense composite-key mapping"); + for (uint32_t key = 0; key < 6u; ++key) + Check(snapshot.images[key + 1u].dwords[0] == 10u + key && + snapshot.flattened_srt[mapping + key + 1u] == key + 1u, + "nested composite key mixed parent rows or child records"); + const auto original = specialization; + user_data[2] = 100u; + Check(MaterializeResources(plan, runtime, snapshot, specialization) && snapshot.images.size() == 7u, + "nested parent enumeration included unreachable poison rows"); + user_data[2] = 2u; + *at(table) = objects + row_bytes; + *at(objects + row_bytes + 64u) = 42u; + Check(MaterializeResources(plan, runtime, snapshot, specialization) && snapshot.images.size() == 4u && + snapshot.images[1].dwords[0] == 42u && specialization != original, + "nested table refresh retained stale pointer/descriptor data or duplicated aliases"); + *at(table) = objects; + for (const uint32_t records : {0u, 1u}) { + user_data[2] = records; + const auto reads = memory.reads; + Check(!MaterializeResources(plan, runtime, snapshot, specialization) && memory.reads == reads, + "incomplete indexed parent domain became null images or performed partial candidate reads"); + } + user_data[2] = 2u; + for (const uint32_t bad_stride : {0u, 4u, 161u}) { + user_data[1] = 1u | (bad_stride << 16u); + Check(!MaterializeResources(plan, runtime, snapshot, specialization), + "indexed pointer table accepted incomplete or misaligned records"); + } + user_data[1] = 1u | (stride << 16u); + for (const auto [word, bit] : {std::pair{1u, 31u}, std::pair{3u, 23u}, + std::pair{3u, 28u}, std::pair{3u, 30u}}) { + user_data[word] |= 1u << bit; + const auto reads = memory.reads; + Check(!MaterializeResources(plan, runtime, snapshot, specialization) && memory.reads == reads, + "indexed pointer table accepted unsupported RDNA2 geometry before reading candidates"); + user_data[word] &= ~(1u << bit); + } + memory.fail_address = high + objects + 2u * 368u + 64u; + Check(!MaterializeResources(plan, runtime, snapshot, specialization), + "unreadable nested child was silently converted to a null image"); + memory.fail_address = UINT64_MAX; + *at(table) = 0u; + *at(table + 4u) = 0u; + Check(!MaterializeResources(plan, runtime, snapshot, specialization), + "explicit null pointer bypassed the strict child reader"); + auto reversed = make(Case::ReverseSigned); + reversed->PlanAndTrack(); + for (const auto variant : {Case::MissingParent, Case::WrongParent, Case::MissingChild, + Case::InclusiveChild, Case::MismatchedPair, Case::DifferentLaneMask, + Case::ExtraUse, Case::UnboundedChoice, Case::MissingOuterWitness, Case::WrongOuterWitness, + Case::CyclicChoice, Case::ExcessiveDomain}) { + auto invalid = make(variant); + CheckFatal([&] { invalid->PlanAndTrack(); }, "not a valid runtime value", + "nested table accepted an unproved domain, mismatched pointer pair or unguarded use"); + } +} + void TestGuardedDirectImageTable() { namespace CFG = Libs::Graphics::ShaderRecompiler::CFG; enum class Guard { @@ -5877,7 +6090,7 @@ void TestGpuSelectedFormattedRecordStore() { const auto& indirect = plan.descriptor_sources[plan.info.buffers[resource].source].indirect_descriptor; if (!indirect) continue; Check(root == UINT32_MAX && indirect->table_scalar && indirect->table_maximum == 65535u && - !indirect->pointer_offset && !indirect->selector && indirect->key_count.IsEmpty() && + !indirect->pointer_table && !indirect->selector && indirect->key_count.IsEmpty() && indirect->table_stride == 592u && indirect->table_immediate == 448u, "formatted record store lost its exact rows or captured the GPU selector"); root = resource; @@ -6002,7 +6215,7 @@ void TestGpuSelectedFormattedRecordStore() { if (!source.indirect_descriptor) continue; const auto& indirect = *source.indirect_descriptor; Check(wide_root == UINT32_MAX && !indirect.table_scalar && !indirect.selector && - !indirect.table_maximum && !indirect.pointer_offset && + !indirect.table_maximum && !indirect.pointer_table && indirect.key_count == Value(32u) && indirect.table_stride == 16u && indirect.table_offset == 0u && indirect.table_immediate == 48u && wide_plan.descriptor_sources[indirect.table_source].dword_count == 2u && @@ -7837,6 +8050,7 @@ int main() { Run("exact unfiltered image candidates", TestExactUnfilteredImageCandidates); Run("waterfall image table", TestWaterfallImageTable); Run("nested pointer image table", TestNestedPointerImageTable); + Run("indexed pointer image table", TestIndexedPointerImageTable); Run("guarded direct image table", TestGuardedDirectImageTable); Run("bounded compute image loop", TestBoundedComputeImageLoop); Run("uniformized material image keys", TestUniformizedMaterialImageKeys); diff --git a/tests/ShaderRecompilerComputeTests.cpp b/tests/ShaderRecompilerComputeTests.cpp index bb83b036f..2e0a936f0 100644 --- a/tests/ShaderRecompilerComputeTests.cpp +++ b/tests/ShaderRecompilerComputeTests.cpp @@ -39011,35 +39011,49 @@ void CheckPointerImageTable(VulkanHarness &vulkan) { void CheckNestedPointerImageTable(VulkanHarness &vulkan) { using O = ShaderOpcode; - constexpr u32 keys_address = 128, list_address = 160, table = 256, objects = 4096; + constexpr u32 list_address = 160, objects = 4096; constexpr std::array keys{5u, 0u, 3u, 1u, 4u, 2u}; constexpr std::array list{2u, 1u, 4u, 0u, 5u, 3u}; - for (const u32 member : {0u, 0x98u, 0x16cu}) - for (const u32 wave : {32u, 64u}) { - const bool compact = member != 0u; - const u32 stride = compact ? 0x330u : 592u; - const u32 table_bytes = (keys.size() - 1u) * stride + member + 16u; + for (const u32 member : {0u, 0x98u, 0x16cu, 64u}) + for (const auto [wave, upper_only] : {std::pair{32u, false}, std::pair{64u, false}, + std::pair{64u, true}}) { + const bool child_table = member == 64u, compact = member != 0u && !child_table; + if (upper_only && !child_table) continue; + const u32 keys_address = child_table ? 1024u : 128u, table = child_table ? 2048u : 256u; + const u32 rows = child_table ? 4u : keys.size(), children = child_table ? 3u : 1u; + const u32 stride = child_table ? 136u : compact ? 0x330u : 592u; + const u32 object_stride = child_table ? 1536u : 0x430u; + const u32 table_bytes = (rows - 1u) * stride + member + 16u; + const u32 outputs = child_table ? wave * 2u : keys.size(); TestCase test; - test.name = member == 0x16cu ? "CompactPairedImageSamplerTable" + test.name = child_table ? (upper_only ? "IndexedPointerChildTableUpper64" : "IndexedPointerChildTable") + : member == 0x16cu ? "CompactPairedImageSamplerTable" : compact ? "CompactRecordImageSamplerTable" : "NestedPointerImageSamplerTable"; - test.initial.resize((compact ? table + table_bytes : objects + keys.size() * 0x430u) / 4u, - compact ? 0xdeadbeefu : 0u); - std::fill_n(test.initial.begin(), keys.size(), 0xdeadbeefu); - std::fill_n(test.initial.begin() + keys_address / 4u, keys.size(), UINT32_MAX); + test.initial.resize((compact ? table + table_bytes : objects + rows * object_stride) / 4u, + (compact || child_table) ? 0xdeadbeefu : 0u); + std::fill_n(test.initial.begin(), outputs, 0xdeadbeefu); + std::fill_n(test.initial.begin() + keys_address / 4u, child_table ? wave * 2u : keys.size(), UINT32_MAX); if (compact) std::copy(list.begin(), list.end(), test.initial.begin() + list_address / 4u); - for (u32 row = 0; row < keys.size(); ++row) { - const u32 object = compact ? table + row * stride : objects + row * 0x430u; - if (!compact) test.initial[(table + row * stride) / 4u] = object; - auto image = MakeSampledTextureData(Prospero::BufferFormat::k32Float); - image[0] = 10u + row; - image[1] |= 1u << 30u; // Two texels: zero, then the row's value. - image[3] = DstSel(4, 0, 0, 1) | - (static_cast(Prospero::ImageType::kColor2D) << 28u); - std::copy_n(image.begin(), compact ? 4u : 8u, - test.initial.begin() + (object + (compact ? member : 0x3d0u)) / 4u); + for (u32 row = 0; row < rows; ++row) { + const u32 object = compact ? table + row * stride : objects + row * object_stride; + if (!compact) { + test.initial[(table + row * stride) / 4u] = object; + // The actual vector path sign-extends WORD0 and discards the packed upper bits. + test.initial[(table + row * stride) / 4u + 1u] = child_table ? 0xbeef0000u : 0u; + } + for (u32 child = 0; child < children; ++child) { + auto image = MakeSampledTextureData(Prospero::BufferFormat::k32Float); + image[0] = 10u + row * children + child; + image[1] |= 1u << 30u; // Two texels: zero, then the selected child's value. + image[3] = DstSel(4, 0, 0, 1) | + (static_cast(Prospero::ImageType::kColor2D) << 28u); + std::copy_n(image.begin(), compact ? 4u : 8u, + test.initial.begin() + (object + (child_table ? child * 368u + member + : compact ? member : 0x3d0u)) / 4u); + } const std::array sampler{0u, 0x00fff000u, (row & 1u) ? 0x0a500000u : 0u, 0u}; - std::copy(sampler.begin(), sampler.end(), - test.initial.begin() + (object + (compact ? 0x88u : 0x3f8u)) / 4u); + std::copy(sampler.begin(), sampler.end(), test.initial.begin() + + (child_table ? table + row * stride + 80u : object + (compact ? 0x88u : 0x3f8u)) / 4u); } if (compact) { // Member 0x16c admits this tuple's residue; only the paired selector list excludes it. @@ -39048,10 +39062,11 @@ void CheckNestedPointerImageTable(VulkanHarness &vulkan) { test.initial.begin() + (table + (member == 0x16cu ? 0x17cu : 0xbcu)) / 4u); } auto records = MakeStructuredStorageBufferData(compact ? 0u : stride, - compact ? table_bytes : keys.size()); + compact ? table_bytes : rows); records[0] = table; std::copy_n(records.begin(), 4, test.user_data.begin() + 32); - auto key_buffer = MakeStructuredStorageBufferData(0, keys.size() * 4u); + auto key_buffer = MakeStructuredStorageBufferData(child_table ? 8u : 0u, + child_table ? wave : keys.size() * 4u); key_buffer[0] = keys_address; std::copy_n(key_buffer.begin(), 4, test.user_data.begin() + 4); if (compact) { @@ -39059,22 +39074,56 @@ void CheckNestedPointerImageTable(VulkanHarness &vulkan) { indices[0] = list_address; std::copy_n(indices.begin(), 4, test.user_data.begin() + 36); } - const auto output = MakeStructuredStorageBufferData(0, keys.size() * 4u); + const auto output = MakeStructuredStorageBufferData(0, outputs * 4u); std::copy_n(output.begin(), 4, test.user_data.begin() + 48); test.bda_mappings = {{0u, 0u}}; test.has_user_data = test.has_compute_info = true; test.compute_info.wave_size = wave; - test.compute_info.threads_num[0] = test.compute_info.threads_num[1] = - test.compute_info.threads_num[2] = 1; + test.compute_info.threads_num[0] = child_table ? wave : 1u; + test.compute_info.threads_num[1] = test.compute_info.threads_num[2] = 1u; + test.compute_info.thread_ids_num = child_table ? 1u : 0u; test.compute_info.group_id[0] = true; test.compute_info.workgroup_register = 44; - test.dispatch_x = keys.size(); + test.dispatch_x = child_table ? 1u : keys.size(); auto &code = test.code; - code.push_back(EncodeSop2(0x1e, 23, 44, InlineU32(2))); - code.push_back(EncodeVop1(0x01, 30, 23)); - code.insert(code.end(), {EncodeMubuf0(0x0c), EncodeMubuf1(4, 1, 30), - EncodeVop2(0x16, 6, InlineU32(16), 4)}); - AppendVop3(&code, 0x360, 22, Vgpr(6), InlineU32(0)); + size_t loop = 0, empty = 0, skip_child = 0; + if (child_table) { + code.push_back(EncodeVop2(0x1a, 30, InlineU32(2), 0)); + if (upper_only) { + AppendSMovLiteral(&code, 126, 0u); + AppendSMovLiteral(&code, 127, UINT32_MAX); + } + code.insert(code.end(), {EncodeSop1(0x04, 60, 126), + EncodeMubuf0(0x0d, 0, true, false), EncodeMubuf1(4, 1, 0), EncodeSopp(0x0c, 0)}); + AppendVMovLiteral(&code, 6, UINT32_MAX); + for (u32 row = 0; row < rows; ++row) { + code.push_back(EncodeVopc(0xc2, InlineU32(row), 4)); + AppendVop3(&code, 0x101, 6, Vgpr(6), InlineU32(row), 106); + } + code.push_back(EncodeVopc(0x93, InlineU32(0), 6)); // Only finite nonnegative parent rows. + empty = code.size(); code.push_back(0u); + code.insert(code.end(), {EncodeMubuf0(0x0d, 0, true, false), EncodeMubuf1(10, 8, 6), + EncodeSopp(0x0c, 0)}); + AppendVop3(&code, 0x149, 11, Vgpr(11), InlineU32(0), InlineU32(16)); + loop = code.size(); + code.insert(code.end(), {EncodeVop1(0x02, 22, Vgpr(6)), + EncodeVop1(0x02, 20, Vgpr(10)), EncodeVop1(0x02, 21, Vgpr(11)), + EncodeVop1(0x02, 24, Vgpr(5)), EncodeVopc(0xc2, 22, 6), EncodeSop1(0x04, 56, 106)}); + for (const auto [sgpr, vgpr] : {std::pair{20u, 10u}, std::pair{21u, 11u}, std::pair{24u, 5u}}) + code.insert(code.end(), {EncodeVopc(0xc2, sgpr, vgpr), EncodeSop2(0x0f, 56, 56, 106)}); + code.insert(code.end(), {EncodeSop1(0x24, 58, 56), EncodeSopc(0x09, 24, InlineU32(children))}); + skip_child = code.size(); code.push_back(0u); + code.insert(code.end(), {EncodeSop2(0x26, 23, 24, 255), 368u, + EncodeSmem0(0x03, 8, 10), EncodeSmem1(member, 23), + EncodeSop2(0x26, 25, 22, 255), stride, + EncodeSmem0(0x0a, 16, 16), EncodeSmem1(80u, 25), EncodeSopp(0x0c, 0)}); + } else { + code.push_back(EncodeSop2(0x1e, 23, 44, InlineU32(2))); + code.push_back(EncodeVop1(0x01, 30, 23)); + code.insert(code.end(), {EncodeMubuf0(0x0c), EncodeMubuf1(4, 1, 30), + EncodeVop2(0x16, 6, InlineU32(16), 4)}); + AppendVop3(&code, 0x360, 22, Vgpr(6), InlineU32(0)); + } if (compact) { code.insert(code.end(), {EncodeSop2(0x1e, 23, 22, InlineU32(2)), EncodeSmem0(0x08, 22, 18), EncodeSmem1(0, 23), @@ -39084,26 +39133,53 @@ void CheckNestedPointerImageTable(VulkanHarness &vulkan) { else code.insert(code.end(), {EncodeSmem0(0x0a, 8, 16), EncodeSmem1(0x88u, 23), EncodeSmem0(0x0a, 12, 16), EncodeSmem1(member, 23)}); for (u32 reg = 16u; reg < 20u; ++reg) AppendSMovLiteral(&code, reg, 0xdeadbeefu); - } else code.insert(code.end(), {EncodeSop2(0x26, 23, 22, 255), stride, + } else if (!child_table) code.insert(code.end(), {EncodeSop2(0x26, 23, 22, 255), stride, EncodeSmem0(0x09, 20, 16), EncodeSmem1(0, 23), EncodeSmem0(0x03, 8, 10), EncodeSmem1(0x3d0, 125), EncodeSmem0(0x02, 16, 10), EncodeSmem1(0x3f8, 125)}); AppendVMovLiteral(&code, 20, std::bit_cast(0.5f)); AppendVMovLiteral(&code, 21, std::bit_cast(0.5f)); code.insert(code.end(), {EncodeMimg0(0x27, 1, 0, false, 1, compact), - EncodeMimg1(5, 20, compact ? 3u : 2u, compact ? 2u : 4u)}); - AppendBufferStoreDword(&code, 5, 30); + EncodeMimg1(12, 20, compact ? 3u : 2u, compact ? 2u : 4u)}); + AppendBufferStoreDword(&code, 12, 30); + if (child_table) { + code[skip_child] = EncodeSopp(0x05, code.size() - skip_child - 1u); + code.push_back(EncodeSop2(0x15, 126, 58, 56)); + code.push_back(EncodeSopp(0x09, static_cast(loop - code.size() - 1u))); + code[empty] = EncodeSopp(0x08, code.size() - empty - 1u); + code.push_back(EncodeSop1(0x04, 126, 60)); + AppendVMovLiteral(&code, 12, 0x76543210u); + code.insert(code.end(), {EncodeMubuf0(0x1c, wave * 4u), EncodeMubuf1(12, 12, 30)}); + } AppendEnd(&code); - test.opcodes = {O::S_LSHL_B32, O::V_MOV_B32, O::BUFFER_LOAD_DWORD, + if (child_table) { + test.decoded_counts = {{"BUFFER_LOAD_DWORDX2", 2u}, {"V_READFIRSTLANE_B32", 4u}, + {"V_BFE_I32", 1u}, {"V_CMPX_LE_I32", 1u}, {"S_CBRANCH_EXECNZ", 1u}}; + test.required_spirv = {"OpLoopMerge", "OpGroupNonUniformShuffle", "OpBitFieldSExtract"}; + } else test.opcodes = {O::S_LSHL_B32, O::V_MOV_B32, O::BUFFER_LOAD_DWORD, O::V_LSHRREV_B32, O::V_READLANE_B32, O::S_MUL_I32, O::IMAGE_SAMPLE, O::BUFFER_STORE_DWORD, O::S_ENDPGM}; if (compact) { test.opcodes.insert(test.opcodes.end(), {O::S_BUFFER_LOAD_DWORD, member == 0x98u ? O::S_BUFFER_LOAD_DWORDX8 : O::S_BUFFER_LOAD_DWORDX4, O::S_MOV_B32}); test.decoded_counts = {{"r128=1", 1}}; - } else test.opcodes.insert(test.opcodes.end(), + } else if (!child_table) test.opcodes.insert(test.opcodes.end(), {O::S_BUFFER_LOAD_DWORDX2, O::S_LOAD_DWORDX8, O::S_LOAD_DWORDX4}); const auto compiled = CompileCase(test, vulkan.SubgroupSize()); + if (child_table) { + std::vector candidates; + for (const auto &image : compiled.resources.images) + if (image.dwords[0] != 0u) candidates.push_back(image.dwords[0]); + std::ranges::sort(candidates); + std::vector expected_candidates(rows * children); + std::iota(expected_candidates.begin(), expected_candidates.end(), 10u); + Require(test.name, "complete child domain", candidates == expected_candidates, + "parent/child planning omitted a record or admitted poisoned padding"); + Require(test.name, "GPU selectors remain unread by materialization", + std::ranges::none_of(compiled.resources.specialization_reads, [&](const auto &span) { + return span.first < keys_address + wave * 8u && keys_address < span.first + span.second; + }), "host planning captured GPU-only selector bytes"); + } if (compact) { std::vector candidates; for (const auto &image : compiled.resources.images) candidates.push_back(image.dwords[0]); @@ -39143,10 +39219,27 @@ void CheckNestedPointerImageTable(VulkanHarness &vulkan) { producer.compute_info.threads_num[0] = producer.compute_info.threads_num[1] = producer.compute_info.threads_num[2] = 1; producer.compute_info.wave_size = 32; - for (u32 group = 0; group < keys.size(); ++group) { - AppendVMovLiteral(&producer.code, 1, (keys[group] << 16u) | 0xbeefu); - AppendVMovLiteral(&producer.code, 2, keys_address + group * 4u); + test.expected = test.initial; + for (u32 group = 0; group < (child_table ? wave : keys.size()); ++group) { + const u32 outer = child_table ? (group % 8u == 7u ? UINT32_MAX : group % rows) : keys[group]; + const u32 child = child_table ? (group / 8u) % 4u : 0u; + const u32 address = keys_address + group * (child_table ? 8u : 4u); + AppendVMovLiteral(&producer.code, 1, child_table ? outer : (outer << 16u) | 0xbeefu); + AppendVMovLiteral(&producer.code, 2, address); AppendBufferStoreDword(&producer.code, 1, 2); + test.expected[address / 4u] = child_table ? outer : (outer << 16u) | 0xbeefu; + if (child_table) { + AppendVMovLiteral(&producer.code, 1, child); + AppendVMovLiteral(&producer.code, 2, address + 4u); + AppendBufferStoreDword(&producer.code, 1, 2); + test.expected[address / 4u + 1u] = child; + if (upper_only && group < 32u) continue; + test.expected[wave + group] = 0x76543210u; + if (outer >= rows || child >= children) continue; + } + const u32 row = compact ? list[outer] : outer; + test.expected[group] = std::bit_cast(float(10u + row * children + child) * + ((row & 1u) ? 0.5f : 1.0f)); } AppendEnd(&producer.code); const auto producer_shader = CompileCase(producer, vulkan.SubgroupSize()); @@ -39154,13 +39247,8 @@ void CheckNestedPointerImageTable(VulkanHarness &vulkan) { vulkan.Dispatch(producer, producer_shader, buffer); vulkan.Dispatch(test, compiled, buffer, nullptr, nullptr, nullptr, nullptr, nullptr, images, samplers); - std::vector expected; - for (auto key : keys) { - if (compact) key = list[key]; - expected.push_back(std::bit_cast(float(10u + key) * ((key & 1u) ? 0.5f : 1.0f))); - } - CompareWords(test, "GPU-selected nested image and sampler", expected, - vulkan.ReadBuffer(test.name, buffer, expected.size())); + CompareWords(test, "GPU-selected parent/child, paired sampler, EXEC restoration and canaries", + test.expected, vulkan.ReadBuffer(test.name, buffer, test.expected.size())); vulkan.DestroyBuffer(&buffer); for (const auto sampler : samplers) vulkan.Device().destroySampler(sampler); for (auto &image : images) vulkan.DestroyImage(&image);