From 145578ac03bf83e3fbb6594b63b1c9e0c0f06533 Mon Sep 17 00:00:00 2001 From: nmzik Date: Mon, 12 Oct 2026 20:44:12 +0200 Subject: [PATCH] shader: materialize image keys from proven GPU bitmap subsets --- src/graphics/shader/recompiler/ir/ShaderIR.h | 4 +- .../ir/passes/ResourceMaterialization.cpp | 27 +++ .../recompiler/ir/passes/ResourceTracking.cpp | 226 +++++++++++++++++- tests/ResourceTrackingTests.cpp | 201 +++++++++++++++- 4 files changed, 443 insertions(+), 15 deletions(-) diff --git a/src/graphics/shader/recompiler/ir/ShaderIR.h b/src/graphics/shader/recompiler/ir/ShaderIR.h index 0bb399679..33d794888 100644 --- a/src/graphics/shader/recompiler/ir/ShaderIR.h +++ b/src/graphics/shader/recompiler/ir/ShaderIR.h @@ -507,6 +507,8 @@ struct DescriptorSource { uint32_t source = UINT32_MAX; uint32_t stride = 0; uint32_t offset = 0; + // Four DWORD bitmap words per record; set-bit ranks subtract from the last key. + bool bitset = false; std::vector> byte_ranges; bool operator==(const SelectorRead& other) const = default; @@ -521,7 +523,7 @@ struct DescriptorSource { bool indexed_store = false; uint32_t workgroup_axis = UINT32_MAX; Value key_count; - // First selector index, or first descriptor key when no selector is present. + // First selector index, or first descriptor key for a direct or bitmap range. Value selector_first; Value selector_mask; std::vector sources; diff --git a/src/graphics/shader/recompiler/ir/passes/ResourceMaterialization.cpp b/src/graphics/shader/recompiler/ir/passes/ResourceMaterialization.cpp index 8e68135dc..4fd5baea7 100644 --- a/src/graphics/shader/recompiler/ir/passes/ResourceMaterialization.cpp +++ b/src/graphics/shader/recompiler/ir/passes/ResourceMaterialization.cpp @@ -338,6 +338,33 @@ bool MaterializeIndirectDescriptor(const ResourcePlan& p std::iota(keys.begin(), keys.end(), first_key); if (!keys.empty() && keys.back() < keys.front()) std::rotate(keys.begin(), keys.begin() + (UINT32_MAX - first_key + 1ull), keys.end()); + } else if (selector->bitset) { + ShaderBufferResource material; + std::array masks {}; + uint32_t first = 0, count = 0; + if (!DecodeBufferDescriptor(material_value, material) || + !clean.Evaluate(indirect.selector_first, first) || + !clean.Evaluate(indirect.key_count, count) || count != masks.size() * 32u || + selector->stride != sizeof(masks) || selector->offset != 0u || + material.Type() != 0u || material.Stride() != sizeof(masks) || + material.SwizzleEnabled() || material.AddTid() || material.OutOfBounds() != 0u || + (material.Base48() & 3u) != 0u || + material.GetSize() / sizeof(uint32_t) > MaxIndirectDescriptorProbes) return false; + if (material.Format() != Prospero::BufferFormat::kInvalid) { + keys.resize(material.GetSize() / sizeof(uint32_t)); + if (!keys.empty() && !ReadScalarTable(material.Base48(), material.GetSize(), 0u, + runtime, keys)) return false; + for (size_t word = 0; word < keys.size(); ++word) masks[word % masks.size()] |= keys[word]; + keys.clear(); + for (uint32_t word = 0; word < masks.size(); ++word) { + auto mask = masks[word]; + while (mask != 0u) { + const auto rank = word * 32u + std::countr_zero(mask); + keys.push_back(first + count - 1u - rank); + mask &= mask - 1u; + } + } + } } else if (!selector->byte_ranges.empty()) { if (material_value.dword_count != 2u) return false; const auto base = (uint64_t {material_value.dwords[1]} << 32u) | material_value.dwords[0]; diff --git a/src/graphics/shader/recompiler/ir/passes/ResourceTracking.cpp b/src/graphics/shader/recompiler/ir/passes/ResourceTracking.cpp index 838ed6330..342f0c1f0 100644 --- a/src/graphics/shader/recompiler/ir/passes/ResourceTracking.cpp +++ b/src/graphics/shader/recompiler/ir/passes/ResourceTracking.cpp @@ -1202,20 +1202,24 @@ private: (Implies(inst->Arg(0), required) || Implies(inst->Arg(1), required)); } - Value EqualLocalKey(Value guard, Value key) const { + Value EqualLocalValue(Value guard, const auto& matches) const { guard = SimplifyGuard(guard); const auto* inst = guard.TryInstruction(); if (inst == nullptr) return {}; if (inst->GetOpcode() == ValueOpcode::LogicalAnd) { - const auto left = EqualLocalKey(inst->Arg(0), key); - return left.IsEmpty() ? EqualLocalKey(inst->Arg(1), key) : left; + const auto left = EqualLocalValue(inst->Arg(0), matches); + return left.IsEmpty() ? EqualLocalValue(inst->Arg(1), matches) : left; } if (inst->GetOpcode() != ValueOpcode::IEqual32 || inst->NumArgs() != 2u) return {}; - if (EquivalentValue(m_program, inst->Arg(0), key)) return inst->Arg(1).Resolve(); - if (EquivalentValue(m_program, inst->Arg(1), key)) return inst->Arg(0).Resolve(); + if (matches(inst->Arg(0))) return inst->Arg(1).Resolve(); + if (matches(inst->Arg(1))) return inst->Arg(0).Resolve(); return {}; } + Value EqualLocalKey(Value guard, Value key) const { + return EqualLocalValue(guard, [&](Value at) { return EquivalentValue(m_program, at, key); }); + } + struct AffineOffset { Value index; uint64_t stride = 0; @@ -1977,6 +1981,216 @@ private: return false; } + using SelectorBounds = std::span>; + + // Each equation is an inductive subset obligation. Phis require every incoming + // value; AND needs either operand. Grounding rejects cycles without an initial + // value, then the greatest fixed point checks every initial and carried edge. + bool BitsetMaskSubset(Value mask, Value active, Value word, const Inst& read) const { + struct State { + Value value; + Value guard; + uint32_t word = UINT32_MAX; + uint32_t component = UINT32_MAX; // U1 implication of the load guard. + bool operator==(const State&) const = default; + }; + struct Equation { + bool all = true; + std::vector terms; + }; + const auto load_guard = read.Arg(4).Resolve(); + std::vector states(2); + std::vector equations(2); + const auto add = [&](State state) { + state.value = state.value.Resolve(); + const auto found = std::find(states.begin() + 2u, states.end(), state); + if (found != states.end()) return static_cast(found - states.begin()); + states.push_back(state); + equations.emplace_back(); + return states.size() - 1u; + }; + std::array roots {add({.value = active})}; + for (uint32_t component = 0; component < 4u; ++component) + roots[component + 1u] = add({mask, active, component, component}); + for (size_t index = 2; index < states.size(); ++index) { + const auto state = states[index]; + const auto* inst = state.value.TryInstruction(); + Equation equation; + const auto term = [&](Value value, Value guard, uint32_t selected) { + return add({value, guard, selected, state.component}); + }; + if (state.component == UINT32_MAX) { + if (Implies(state.value, load_guard)) equation.terms = {1u}; + else if (inst != nullptr && (inst->GetOpcode() == ValueOpcode::LogicalAnd || + inst->GetOpcode() == ValueOpcode::Phi)) { + equation.all = inst->GetOpcode() == ValueOpcode::Phi; + for (size_t arg = 0; arg < inst->NumArgs(); ++arg) + equation.terms.push_back(add({.value = inst->Arg(arg)})); + } + } else if (state.value == Value(0u)) { + equation.terms = {1u}; + } else if (inst != nullptr) { + const auto constant = [&](const auto& self, Value value) -> std::optional { + uint32_t literal; + if (ImmediateU32(value, literal)) return literal; + if (state.word != UINT32_MAX && EquivalentValue(m_program, value, word)) return state.word; + const auto* operation = value.Resolve().TryInstruction(); + if (operation == nullptr || operation->GetOpcode() != ValueOpcode::BitwiseAnd32) return {}; + const auto left = self(self, operation->Arg(0)); + const auto right = self(self, operation->Arg(1)); + return left && right ? std::optional {*left & *right} : std::nullopt; + }; + const auto condition = [&](const auto& self, Value value) -> std::optional { + value = value.Resolve(); + if (value.IsImmediate() && value.GetType() == Type::U1) return value.U1(); + if (Implies(state.guard, value) || Implies(load_guard, value)) return true; + const auto* test = value.TryInstruction(); + if (test == nullptr) return {}; + if (test->GetOpcode() == ValueOpcode::LogicalAnd) { + const auto left = self(self, test->Arg(0)); + const auto right = self(self, test->Arg(1)); + if ((left && !*left) || (right && !*right)) return false; + if (left && right) return *left && *right; + } else if (test->GetOpcode() == ValueOpcode::IEqual32) { + const auto left = constant(constant, test->Arg(0)); + const auto right = constant(constant, test->Arg(1)); + if (left && right) return *left == *right; + } + return {}; + }; + switch (inst->GetOpcode()) { + case ValueOpcode::CompositeExtractU32x4: { + uint32_t component; + if (inst->Arg(0).Resolve().TryInstruction() == &read && + ImmediateU32(inst->Arg(1), component) && component == state.component) + equation.terms = {1u}; + break; + } + case ValueOpcode::Phi: + // The predecessor may be an earlier outer iteration. Establish + // this register's invariant without the current word or EXEC. + for (size_t arg = 0; arg < inst->NumArgs(); ++arg) + equation.terms.push_back(term(inst->Arg(arg), load_guard, UINT32_MAX)); + break; + case ValueOpcode::BitwiseAnd32: + equation.all = false; + equation.terms = {term(inst->Arg(0), state.guard, state.word), + term(inst->Arg(1), state.guard, state.word)}; + break; + case ValueOpcode::SelectU32: { + const auto selected = condition(condition, inst->Arg(0)); + if (!selected || *selected) { + auto selected_word = state.word; + const auto equality = EqualLocalValue(inst->Arg(0), [&](Value key) { + key = key.Resolve(); + const auto* masked = key.TryInstruction(); + uint32_t bound = 0; + if (masked != nullptr && masked->GetOpcode() == ValueOpcode::BitwiseAnd32 && + DescendingLoop(word, masked->Parent(), bound) && bound == 3u) { + for (uint32_t side = 0; side < 2u; ++side) { + uint32_t bits; + if (ImmediateU32(masked->Arg(side), bits) && (bits & 3u) == 3u) + key = masked->Arg(side ^ 1u).Resolve(); + } + } + return EquivalentValue(m_program, key, word); + }); + uint32_t literal; + if (ImmediateU32(equality, literal) && literal < 4u) selected_word = literal; + equation.terms.push_back(term(inst->Arg(1), inst->Arg(0), selected_word)); + } + if (!selected || !*selected) + equation.terms.push_back(term(inst->Arg(2), state.guard, state.word)); + break; + } + default: break; + } + } + if (equation.terms.empty()) equation.terms = {0u}; + equations[index] = std::move(equation); + } + std::vector valid(states.size(), false); + valid[1] = true; + bool changed; + do { + changed = false; + for (size_t index = 2; index < equations.size(); ++index) { + if (!valid[index] && std::ranges::any_of(equations[index].terms, + [&](size_t term) { return valid[term]; })) valid[index] = changed = true; + } + } while (changed); + do { + changed = false; + for (size_t index = 2; index < equations.size(); ++index) { + const auto& equation = equations[index]; + const auto proven = [&](size_t term) { return valid[term]; }; + if (valid[index] && !(equation.all ? std::ranges::all_of(equation.terms, proven) + : std::ranges::any_of(equation.terms, proven))) { + valid[index] = false; + changed = true; + } + } + } while (changed); + return std::ranges::all_of(roots, [&](size_t root) { return valid[root]; }); + } + + bool MatchBitsetSelector(Value rank, Value active, SelectorBounds bounds, + DescriptorSource::IndirectDescriptor& indirect) { + const auto selected_value = [&](Value value) { + for (;;) { + value = ActiveValue(value, active); + const auto* select = value.TryInstruction(); + if (select == nullptr || select->GetOpcode() != ValueOpcode::SelectU32 || + !SelectorPredicateFalse(select->Arg(0), bounds, 0u, active)) return value; + value = select->Arg(2); + } + }; + const auto* sum = selected_value(rank).TryInstruction(); + if (sum == nullptr || sum->GetOpcode() != ValueOpcode::IAdd32) return false; + Value word; + Value bit; + for (uint32_t arg = 0; arg < 2u; ++arg) { + const auto* shift = selected_value(sum->Arg(arg)).TryInstruction(); + if (shift == nullptr || shift->GetOpcode() != ValueOpcode::ShiftLeftLogical32 || + shift->Arg(1).Resolve() != Value(5u)) continue; + word = selected_value(shift->Arg(0)); + bit = selected_value(sum->Arg(arg ^ 1u)); + } + const auto* word_inst = word.TryInstruction(); + if (word_inst == nullptr || std::ranges::none_of(bounds, [&](const auto& bound) { + return bound.first == word_inst && bound.second == 3u; + })) return false; + // FFBH's leading-zero count is inverted again to recover the set-bit rank. + for (uint32_t inverse = 0; inverse < 2u; ++inverse) { + const auto* subtract = bit.TryInstruction(); + if (subtract == nullptr || subtract->GetOpcode() != ValueOpcode::ISub32 || + subtract->Arg(0).Resolve() != Value(31u)) return false; + bit = selected_value(subtract->Arg(1)); + } + const auto* scan = bit.TryInstruction(); + if (scan == nullptr || scan->GetOpcode() != ValueOpcode::FindUMsb32 || + !ImpliesNonzero(active, scan->Arg(0))) return false; + for (const auto* block: m_program.blocks) { + for (const auto& read: *block) { + if (read.GetOpcode() != ValueOpcode::LoadBufferU32x4 || InCycle(block)) continue; + const auto index = read.Flags().index; + if (index >= m_program.memory_info.size()) continue; + const auto& memory = m_program.memory_info[index]; + if (memory.kind != ResourceKind::Buffer || memory.typed || memory.formatted || + !memory.idxen || memory.offen || memory.offset != 0u || memory.data_bits != 32u || + memory.data_dwords != 4u || read.Arg(2).Resolve() != Value(0u) || + read.Arg(3).Resolve() != Value(0u) || + !BitsetMaskSubset(scan->Arg(0), active, word, read)) continue; + DescriptorSource source; + if (!MakeRuntimeTableSource(read, source)) continue; + indirect.selector.emplace(DescriptorSource::IndirectDescriptor::SelectorRead { + .source = InternSource(source), .stride = 16u, .offset = 0u, .bitset = true}); + return true; + } + } + return false; + } + bool MatchReversedImageRange(Value key, DescriptorSource::IndirectDescriptor& indirect) { const auto* subtract = key.Resolve().TryInstruction(); if (subtract == nullptr || subtract->GetOpcode() != ValueOpcode::ISub32 || @@ -2000,6 +2214,7 @@ private: first.SetArg(1, Value(maximum)); indirect.selector_first = Value(&first); indirect.key_count = Value(maximum + 1u); + if (maximum == 127u) MatchBitsetSelector(lane->Arg(0), lane->Arg(1), bounds, indirect); return true; } @@ -2300,7 +2515,6 @@ private: return finish(); } - using SelectorBounds = std::span>; bool SelectorPredicateFalse(Value value, SelectorBounds bounds, uint32_t depth = 0, Value guard = {}) const { diff --git a/tests/ResourceTrackingTests.cpp b/tests/ResourceTrackingTests.cpp index 9d56f5d33..c9c4c7fc6 100644 --- a/tests/ResourceTrackingTests.cpp +++ b/tests/ResourceTrackingTests.cpp @@ -1096,9 +1096,14 @@ void TestBoundedComputeImageLoop() { void TestDescendingImageRange() { namespace CFG = Libs::Graphics::ShaderRecompiler::CFG; - enum class Variant { Valid, WrongStep, IncrementBypass, MissingNonzero, NegativeInitial }; + enum class Variant { + Valid, WrongStep, IncrementBypass, MissingNonzero, NegativeInitial, + Bitmap, BitmapAddsBit, BitmapWrongWord, BitmapUnknownInitial, + BitmapByteOffset, BitmapScalarOffset, BitmapOuterClobber, BitmapWrongLoadMask + }; const auto make_plan = [](Variant variant) { Fixture fixture; + const bool bitmap = variant >= Variant::Bitmap; auto *entry = fixture.block; auto *header = fixture.AddBlock(); auto *guard = fixture.AddBlock(); @@ -1117,21 +1122,69 @@ void TestDescendingImageRange() { .true_block = yes, .false_block = no}; }; branch(entry, header); - branch(body, latch); + branch(body, bitmap ? guard : latch); branch(latch, header); auto &induction = header->AppendNewInst(ValueOpcode::Phi); induction.SetFlags(Type::U32); + std::array outer_words {}, remaining_words {}; + std::array loaded_words {}; + if (bitmap) + for (auto& word : outer_words) + word = &header->AppendNewInst(ValueOpcode::Phi, {}, uint64_t(Type::U32)); const auto bound = fixture.Emit(ValueOpcode::SGreaterThanEqual32, {Value(&induction), Value(0u)}, 0, header); choose(header, guard, variant == Variant::IncrementBypass ? latch : exit, bound); - MemoryInfo lds; - lds.kind = ResourceKind::Lds; - const auto mask = fixture.Emit(ValueOpcode::LoadSharedU32, - {Value(0u), Value(true)}, fixture.AddMemory(lds, 0x10), guard); + Value mask; + Value load_active(true); + if (bitmap) { + fixture.block = entry; + const auto lane = fixture.Emit(ValueOpcode::LaneId); + load_active = fixture.Emit(ValueOpcode::ULessThan32, {lane, Value(32u)}); + const auto buffer = fixture.Buffer({fixture.UserData(4), fixture.UserData(5), + fixture.UserData(6), fixture.UserData(7)}); + MemoryInfo read; + read.kind = ResourceKind::Buffer; + read.idxen = true; + read.offen = variant == Variant::BitmapByteOffset; + read.data_dwords = 4u; + const auto packed = fixture.Emit(ValueOpcode::LoadBufferU32x4, + {buffer, lane, + Value(variant == Variant::BitmapByteOffset ? 4u : 0u), + Value(variant == Variant::BitmapScalarOffset ? 4u : 0u), load_active}, + fixture.AddMemory(read, 0x10)); + for (uint32_t word = 0; word < 4u; ++word) { + auto component = fixture.Emit(ValueOpcode::CompositeExtractU32x4, + {packed, Value(variant == Variant::BitmapWrongWord ? (word + 1u) % 4u : word)}, 0, entry); + component = fixture.Emit(ValueOpcode::SelectU32, + {load_active, component, fixture.Emit(ValueOpcode::UndefU32)}); + loaded_words[word] = component; + if (variant == Variant::BitmapUnknownInitial && word == 0u) + component = fixture.Emit(ValueOpcode::SelectU32, + {fixture.Emit(ValueOpcode::IEqual32, {fixture.Emit(ValueOpcode::LaneId), Value(0u)}), + component, fixture.UserData(3)}); + outer_words[word]->AddPhiOperand(entry, component); + remaining_words[word] = &guard->AppendNewInst(ValueOpcode::Phi, {}, uint64_t(Type::U32)); + remaining_words[word]->AddPhiOperand(header, Value(outer_words[word])); + } + auto selected = fixture.Emit(ValueOpcode::UndefU32, {}, 0, guard); + for (uint32_t word = 0; word < 4u; ++word) { + selected = fixture.Emit(ValueOpcode::SelectU32, + {fixture.Emit(ValueOpcode::IEqual32, {Value(&induction), Value(word)}, 0, guard), + Value(remaining_words[word]), selected}, 0, guard); + } + mask = selected; + } else { + MemoryInfo lds; + lds.kind = ResourceKind::Lds; + mask = fixture.Emit(ValueOpcode::LoadSharedU32, + {Value(0u), Value(true)}, fixture.AddMemory(lds, 0x10), guard); + } auto active = bound; if (variant != Variant::MissingNonzero) active = fixture.Emit(ValueOpcode::LogicalAnd, {bound, fixture.Emit(ValueOpcode::ULessThan32, {Value(0u), mask}, 0, guard)}, 0, guard); + if (bitmap && variant != Variant::BitmapWrongLoadMask) + active = fixture.Emit(ValueOpcode::LogicalAnd, {load_active, active}, 0, guard); choose(guard, body, latch, active); fixture.block = body; const auto stale = fixture.UserData(3); @@ -1171,6 +1224,26 @@ void TestDescendingImageRange() { fixture.Emit(ValueOpcode::ImageSampleRaw, {fixture.Image(words), fixture.Sampler({Value(0u), Value(0u), Value(0u), Value(0u)}), fixture.ImageAddress(), active}, fixture.AddMemory(sample, 0x30)); + if (bitmap) { + auto next = fixture.Emit(ValueOpcode::BitwiseAnd32, + {mask, fixture.Emit(ValueOpcode::BitwiseNot32, + {fixture.Emit(ValueOpcode::ShiftLeftLogical32, {Value(1u), bit})})}); + if (variant == Variant::BitmapAddsBit) + next = fixture.Emit(ValueOpcode::BitwiseOr32, {next, Value(1u << 21u)}); + for (uint32_t word = 0; word < 4u; ++word) { + const auto selected_word = fixture.Emit(ValueOpcode::LogicalAnd, + {active, fixture.Emit(ValueOpcode::IEqual32, {Value(&induction), Value(word)})}); + remaining_words[word]->AddPhiOperand(body, fixture.Emit(ValueOpcode::SelectU32, + {selected_word, next, Value(remaining_words[word])})); + auto carried = Value(remaining_words[word]); + if (variant == Variant::BitmapOuterClobber) + carried = fixture.Emit(ValueOpcode::SelectU32, + {fixture.Emit(ValueOpcode::IEqual32, + {Value(&induction), Value(word + 1u)}, 0, latch), + loaded_words[(word + 1u) % 4u], carried}, 0, latch); + outer_words[word]->AddPhiOperand(latch, carried); + } + } const auto step = fixture.Emit(ValueOpcode::IAdd32, {Value(&induction), Value(variant == Variant::WrongStep ? UINT32_MAX - 1u : UINT32_MAX)}, 0, latch); induction.AddPhiOperand(entry, Value(variant == Variant::NegativeInitial ? UINT32_MAX : 3u)); @@ -1180,9 +1253,11 @@ void TestDescendingImageRange() { "descending range retained impossible relative-register stores"); const auto &indirect = fixture.program.descriptor_sources[ fixture.program.info.images[0].source].indirect_descriptor; - Check(indirect && !indirect->selector && indirect->key_count == Value(128u) && + Check(indirect && indirect->selector.has_value() == (variant == Variant::Bitmap) && + (!indirect->selector || indirect->selector->bitset) && + indirect->key_count == Value(128u) && !indirect->selector_first.IsEmpty(), - "descending bitscan range lost its runtime first key or 128-key bound"); + "descending bitscan range lost its bound or accepted an unproved bitmap subset"); return ExtractResourcePlan(fixture.program); }; auto plan = make_plan(Variant::Valid); @@ -1218,6 +1293,116 @@ void TestDescendingImageRange() { previous = key; } } + + auto bitmap_plan = make_plan(Variant::Bitmap); + for (const auto variant : {Variant::BitmapAddsBit, Variant::BitmapWrongWord, + Variant::BitmapUnknownInitial, Variant::BitmapByteOffset, + Variant::BitmapScalarOffset, Variant::BitmapOuterClobber, + Variant::BitmapWrongLoadMask}) + make_plan(variant); + // Every descriptor is distinct, so retained keys are visible in the snapshot as well as + // its lookup table. An unrelated table slot must never be read, even when it is malformed. + for (uint32_t record = 1u; record <= 130u; ++record) + memory.words[record * 8u] = 0x100u + record; + constexpr uint32_t bitmap_address = 0x4000u; + const auto bitmap_word = (bitmap_address - memory.base) / 4u; + memory.words.resize(bitmap_word + MaxIndirectDescriptorProbes + 4u, 0u); + std::array bitmap_data {0x1000u, 0u, 0u, UINT32_MAX, + bitmap_address, 16u << 16u, 3u, 0x5204u}; + const SrtRuntime bitmap_runtime {.user_data = bitmap_data, .userdata = &memory, + .read_specialization_memory = ReadLinearTestMemory}; + memory.watched_address = bitmap_address; + const auto check_bitmap = [&](std::initializer_list ranks, uint32_t count) { + bitmap_data[2] = count; + memory.watched_reads = memory.descriptor_reads = 0u; + std::vector keys; + for (const auto rank : ranks) keys.push_back(count - rank); + std::ranges::sort(keys); + Check(MaterializeResources(bitmap_plan, bitmap_runtime, snapshot, specialization) && + memory.watched_reads == uint32_t(bitmap_data[6] != 0u) && + (bitmap_data[6] == 0u || memory.watched_dwords == bitmap_data[6] * 4u) && + memory.descriptor_reads == keys.size() && + snapshot.images.size() == std::max(keys.size(), 1u), + "bitmap selector did not read one bounded snapshot or retained unreachable descriptors"); + Check(bitmap_data[6] == 0u || std::ranges::find(snapshot.specialization_reads, + std::pair{uint64_t{bitmap_address}, uint64_t{bitmap_data[6]} * 16u}) != + snapshot.specialization_reads.end(), + "bitmap snapshot was omitted from same-dispatch write-alias protection"); + if (keys.empty()) { + Check(std::ranges::all_of(snapshot.images[0].dwords, [](uint32_t value) { return value == 0u; }) && + specialization.images[0].indirect_root == ImageResource::NoIndirectImage, + "empty bitmap retained a previously materialized image"); + return; + } + if (keys.size() == 1u) { + Check(snapshot.images[0].dwords[0] == 0x100u + uint32_t(0x1000u + keys[0] * 32u) / 32u && + specialization.images[0].indirect_root == ImageResource::NoIndirectImage, + "a single bitmap rank became empty or required an indirect image mapping"); + return; + } + const auto offset = specialization.images[0].indirect_mapping_offset; + Check(snapshot.flattened_srt[offset] == keys.size(), "bitmap mapping lost a selected key"); + for (uint32_t index = 0; index < keys.size(); ++index) { + const auto descriptor_record = uint32_t(0x1000u + keys[index] * 32u) / 32u; + Check(snapshot.flattened_srt[offset + 1u + index * 2u] == keys[index] && + snapshot.images[index].dwords[0] == 0x100u + descriptor_record, + "bitmap word association or U32-wrapped key selected a different descriptor"); + } + }; + memory.fail_address = 0x2000u - 21u * 32u; + check_bitmap({}, 0u); + memory.words[bitmap_word] = 1u; + check_bitmap({0u}, 0u); + memory.words[bitmap_word + 1u] = 1u << 3u; + memory.words[bitmap_word + 4u] = 1u; // Duplicate rank in another record. + memory.words[bitmap_word + 6u] = 1u << 31u; + memory.words[bitmap_word + 11u] = 1u << 1u; + check_bitmap({0u, 35u, 95u, 97u}, 0u); + check_bitmap({0u, 35u, 95u, 97u}, 2u); + std::fill_n(memory.words.begin() + bitmap_word, 12u, 0u); + for (uint32_t word = 0; word < 4u; ++word) { + memory.words[bitmap_word + word] = 1u << 31u; + memory.words[bitmap_word + 4u + word] = 1u; + } + check_bitmap({0u, 31u, 32u, 63u, 64u, 95u, 96u, 127u}, 0u); + std::fill_n(memory.words.begin() + bitmap_word, 12u, 0u); + memory.words[bitmap_word + 3u] = 1u << 31u; + memory.words[bitmap_word + 8u] = 1u << 2u; + check_bitmap({2u, 127u}, 0u); + std::fill_n(memory.words.begin() + bitmap_word, 12u, 0u); + check_bitmap({}, 0u); + memory.fail_address = bitmap_address + 16u; + Check(!MaterializeResources(bitmap_plan, bitmap_runtime, snapshot, specialization), + "bitmap selector used a stale or partial snapshot after a failed coherent read"); + memory.fail_address = UINT64_MAX; + + const auto valid_data = bitmap_data; + for (const auto [word, value] : std::array, 7>{{ + {4u, bitmap_address + 1u}, {5u, 8u << 16u}, {5u, (16u << 16u) | (1u << 31u)}, + {7u, 0x5204u | (1u << 23u)}, {7u, 0x5204u | (1u << 28u)}, + {7u, 0x5204u | (1u << 30u)}, + {6u, MaxIndirectDescriptorProbes / 4u + 1u}}}) { + bitmap_data[word] = value; + Check(!MaterializeResources(bitmap_plan, bitmap_runtime, snapshot, specialization), + "bitmap selector accepted an unproved buffer layout or exceeded its read bound"); + bitmap_data = valid_data; + } + bitmap_data[7] &= ~(0x7fu << 12u); // FORMAT=INVALID makes even an unformatted load unbound. + memory.words[bitmap_word] = 1u; + memory.watched_reads = memory.descriptor_reads = 0u; + Check(MaterializeResources(bitmap_plan, bitmap_runtime, snapshot, specialization) && + memory.watched_reads == 0u && memory.descriptor_reads == 0u && + snapshot.images.size() == 1u && + std::ranges::all_of(snapshot.images[0].dwords, [](uint32_t value) { return value == 0u; }), + "unbound bitmap V# read backing bytes or manufactured an image key"); + bitmap_data = valid_data; + memory.words[bitmap_word] = 0u; + bitmap_data[6] = 0u; + check_bitmap({}, 0u); + bitmap_data[6] = MaxIndirectDescriptorProbes / 4u; + memory.words[bitmap_word + MaxIndirectDescriptorProbes - 1u] = 1u << 31u; + memory.words[bitmap_word] = 1u; + check_bitmap({0u, 127u}, 0u); } void TestMixedIndirectImageSamplers() {