diff --git a/src/graphics/shader/recompiler/ir/passes/ResourceMaterialization.cpp b/src/graphics/shader/recompiler/ir/passes/ResourceMaterialization.cpp index 83bf32872..a5f622a6b 100644 --- a/src/graphics/shader/recompiler/ir/passes/ResourceMaterialization.cpp +++ b/src/graphics/shader/recompiler/ir/passes/ResourceMaterialization.cpp @@ -377,19 +377,21 @@ bool ResolveSelectorKeys(const ResourcePlan& program, const DescriptorSource::In } } else if (selector->bitset) { ShaderBufferResource material; - std::array masks {}; + std::array storage {}; + const auto record_words = std::min(selector->stride / 4u, storage.size()); + const auto masks = std::span(storage).first(record_words); uint32_t first = 0, count = 0; if (!DecodeBufferDescriptor(material_value, material) || !WalkedValue(program, indirect.selector_first, first) || !EvaluateIndirectCount(program, indirect, runtime, count) || count != masks.size() * 32u || - selector->stride != sizeof(masks) || selector->offset != 0u || - material.Type() != 0u || material.Stride() != sizeof(masks) || + selector->stride != masks.size_bytes() || selector->offset != 0u || + material.Type() != 0u || material.Stride() != selector->stride || material.SwizzleEnabled() || material.AddTid() || material.OutOfBounds() != 0u || (material.Base48() & 3u) != 0u) return false; if (material.Format() != Prospero::BufferFormat::kInvalid) { const auto words = material.GetSize() / sizeof(uint32_t); - // Bitmap storage can grow with resolution while its candidate domain stays - // at 128 keys. Bound scratch memory, preserving each coherent read's capture. + // Bitmap storage can grow with resolution while its candidate domain stays at 32 keys + // per record word. Bound scratch memory, preserving each coherent read's capture. for (uint64_t first_word = 0; first_word < words; first_word += keys.size()) { keys.resize(std::min(words - first_word, MaxIndirectDescriptorProbes)); if (!ReadScalarTable(material.Base48(), material.GetSize(), first_word * 4u, diff --git a/src/graphics/shader/recompiler/ir/passes/ResourceTracking.cpp b/src/graphics/shader/recompiler/ir/passes/ResourceTracking.cpp index 527dd20e9..3d144f302 100644 --- a/src/graphics/shader/recompiler/ir/passes/ResourceTracking.cpp +++ b/src/graphics/shader/recompiler/ir/passes/ResourceTracking.cpp @@ -2567,7 +2567,8 @@ private: // Each equation is an inductive subset obligation. Phis require every incoming // value; AND needs either operand. Grounding rejects cycles without an initial // value, then the greatest fixed point checks every initial and carried edge. - bool BitsetMaskSubset(Value mask, Value active, Value word, const Inst& read) const { + bool BitsetMaskSubset(Value mask, Value active, Value word, + std::span reads) const { struct State { Value value; Value guard; @@ -2579,7 +2580,8 @@ private: bool all = true; std::vector terms; }; - const auto load_guard = read.Arg(4).Resolve(); + const auto load_guard = reads[0]->Arg(4).Resolve(); + const auto words = static_cast(reads.size()) * 4u; std::vector states(2); std::vector equations(2); const auto add = [&](State state) { @@ -2590,9 +2592,9 @@ private: equations.emplace_back(); return states.size() - 1u; }; - std::array roots {add({.value = active})}; - for (uint32_t component = 0; component < 4u; ++component) - roots[component + 1u] = add({mask, active, component, component}); + std::vector roots {add({.value = active})}; + for (uint32_t component = 0; component < words; ++component) + roots.push_back(add({mask, active, component, component})); for (size_t index = 2; index < states.size(); ++index) { const auto state = states[index]; const auto* inst = state.value.TryInstruction(); @@ -2642,8 +2644,8 @@ private: switch (inst->GetOpcode()) { case ValueOpcode::CompositeExtractU32x4: { uint32_t component; - if (inst->Arg(0).Resolve().TryInstruction() == &read && - ImmediateU32(inst->Arg(1), component) && component == state.component) + if (inst->Arg(0).Resolve().TryInstruction() == reads[state.component / 4u] && + ImmediateU32(inst->Arg(1), component) && component == state.component % 4u) equation.terms = {1u}; break; } @@ -2667,17 +2669,17 @@ private: const auto* masked = key.TryInstruction(); uint32_t bound = 0; if (masked != nullptr && masked->GetOpcode() == ValueOpcode::BitwiseAnd32 && - DescendingLoop(word, masked->Parent(), bound) && bound == 3u) { + DescendingLoop(word, masked->Parent(), bound) && bound == words - 1u) { for (uint32_t side = 0; side < 2u; ++side) { uint32_t bits; - if (ImmediateU32(masked->Arg(side), bits) && (bits & 3u) == 3u) + if (ImmediateU32(masked->Arg(side), bits) && (bits & bound) == bound) key = masked->Arg(side ^ 1u).Resolve(); } } return EquivalentValue(m_program, key, word); }); uint32_t literal; - if (ImmediateU32(equality, literal) && literal < 4u) selected_word = literal; + if (ImmediateU32(equality, literal) && literal < words) selected_word = literal; equation.terms.push_back(term(inst->Arg(1), inst->Arg(0), selected_word)); } if (!selected || !*selected) @@ -2715,7 +2717,7 @@ private: return std::ranges::all_of(roots, [&](size_t root) { return valid[root]; }); } - bool MatchBitsetSelector(Value rank, Value active, SelectorBounds bounds, + bool MatchBitsetSelector(Value rank, Value active, SelectorBounds bounds, uint32_t words, DescriptorSource::IndirectDescriptor& indirect) { const auto selected_value = [&](Value value) { for (;;) { @@ -2739,7 +2741,7 @@ private: } const auto* word_inst = word.TryInstruction(); if (word_inst == nullptr || std::ranges::none_of(bounds, [&](const auto& bound) { - return bound.first == word_inst && bound.second == 3u; + return bound.first == word_inst && bound.second == words - 1u; })) return false; // FFBH's leading-zero count is inverted again to recover the set-bit rank. for (uint32_t inverse = 0; inverse < 2u; ++inverse) { @@ -2751,21 +2753,34 @@ private: const auto* scan = bit.TryInstruction(); if (scan == nullptr || scan->GetOpcode() != ValueOpcode::FindUMsb32 || !ImpliesNonzero(active, scan->Arg(0))) return false; + const auto bitmap_load = [&](const Inst& read, uint32_t offset) { + const auto index = read.Flags().index; + if (read.GetOpcode() != ValueOpcode::LoadBufferU32x4 || InCycle(read.Parent()) || + index >= m_program.memory_info.size()) return false; + const auto& memory = m_program.memory_info[index]; + return memory.kind == ResourceKind::Buffer && !memory.typed && !memory.formatted && + memory.idxen && !memory.offen && memory.offset == offset && + memory.data_bits == 32u && memory.data_dwords == 4u && + read.Arg(2).Resolve() == Value(0u) && read.Arg(3).Resolve() == Value(0u); + }; for (const auto* block: m_program.blocks) { for (const auto& read: *block) { - if (read.GetOpcode() != ValueOpcode::LoadBufferU32x4 || InCycle(block)) continue; - const auto index = read.Flags().index; - if (index >= m_program.memory_info.size()) continue; - const auto& memory = m_program.memory_info[index]; - if (memory.kind != ResourceKind::Buffer || memory.typed || memory.formatted || - !memory.idxen || memory.offen || memory.offset != 0u || memory.data_bits != 32u || - memory.data_dwords != 4u || read.Arg(2).Resolve() != Value(0u) || - read.Arg(3).Resolve() != Value(0u) || - !BitsetMaskSubset(scan->Arg(0), active, word, read)) continue; + if (!bitmap_load(read, 0u)) continue; + // An eight-word record continues in a second four-word load of the same buffer and EXEC. + std::array reads {&read}; + if (words == 8u) + for (const auto* other: m_program.blocks) + for (const auto& load: *other) + if (reads[1] == nullptr && bitmap_load(load, 16u) && + EquivalentValue(m_program, load.Arg(0), read.Arg(0)) && + EquivalentValue(m_program, load.Arg(4), read.Arg(4))) reads[1] = &load; + const auto parts = std::span(reads).first(words / 4u); DescriptorSource source; - if (!MakeRuntimeTableSource(read, source)) continue; + if (std::ranges::find(parts, nullptr) != parts.end() || + !BitsetMaskSubset(scan->Arg(0), active, word, parts) || + !MakeRuntimeTableSource(read, source)) continue; indirect.selector.emplace(DescriptorSource::IndirectDescriptor::SelectorRead { - .source = InternSource(source), .stride = 16u, .offset = 0u, .bitset = true}); + .source = InternSource(source), .stride = words * 4u, .offset = 0u, .bitset = true}); return true; } } @@ -2795,7 +2810,8 @@ private: first.SetArg(1, Value(maximum)); indirect.selector_first = Value(&first); indirect.key_count = Value(maximum + 1u); - if (maximum == 127u) MatchBitsetSelector(lane->Arg(0), lane->Arg(1), bounds, indirect); + if (maximum == 127u || maximum == 255u) + MatchBitsetSelector(lane->Arg(0), lane->Arg(1), bounds, (maximum + 1u) / 32u, indirect); return true; } diff --git a/tests/ResourceTrackingTests.cpp b/tests/ResourceTrackingTests.cpp index ad5d7563b..5eac1eadd 100644 --- a/tests/ResourceTrackingTests.cpp +++ b/tests/ResourceTrackingTests.cpp @@ -2505,11 +2505,15 @@ void TestDescendingImageRange() { enum class Variant { Valid, WrongStep, IncrementBypass, MissingNonzero, NegativeInitial, Bitmap, BitmapAddsBit, BitmapWrongWord, BitmapUnknownInitial, - BitmapByteOffset, BitmapScalarOffset, BitmapOuterClobber, BitmapWrongLoadMask + BitmapByteOffset, BitmapScalarOffset, BitmapOuterClobber, BitmapWrongLoadMask, + Bitmap8, Bitmap8MisplacedLoad, Bitmap8OtherBuffer, Bitmap8OuterClobber, Bitmap8WrongUpperWord, + Bitmap8NarrowMask, Bitmap8OtherGuard }; const auto make_plan = [](Variant variant) { Fixture fixture; const bool bitmap = variant >= Variant::Bitmap; + // Eight-word records are two four-word loads of one buffer record. + const uint32_t bitmap_words = variant >= Variant::Bitmap8 ? 8u : 4u; auto *entry = fixture.block; auto *header = fixture.AddBlock(); auto *guard = fixture.AddBlock(); @@ -2532,11 +2536,11 @@ void TestDescendingImageRange() { branch(latch, header); auto &induction = header->AppendNewInst(ValueOpcode::Phi); induction.SetFlags(Type::U32); - std::array outer_words {}, remaining_words {}; - std::array loaded_words {}; + std::array outer_words {}, remaining_words {}; + std::array loaded_words {}; if (bitmap) - for (auto& word : outer_words) - word = &header->AppendNewInst(ValueOpcode::Phi, {}, uint64_t(Type::U32)); + for (uint32_t word = 0; word < bitmap_words; ++word) + outer_words[word] = &header->AppendNewInst(ValueOpcode::Phi, {}, uint64_t(Type::U32)); const auto bound = fixture.Emit(ValueOpcode::SGreaterThanEqual32, {Value(&induction), Value(0u)}, 0, header); choose(header, guard, variant == Variant::IncrementBypass ? latch : exit, bound); @@ -2553,14 +2557,27 @@ void TestDescendingImageRange() { read.idxen = true; read.offen = variant == Variant::BitmapByteOffset; read.data_dwords = 4u; - const auto packed = fixture.Emit(ValueOpcode::LoadBufferU32x4, - {buffer, lane, - Value(variant == Variant::BitmapByteOffset ? 4u : 0u), - Value(variant == Variant::BitmapScalarOffset ? 4u : 0u), load_active}, - fixture.AddMemory(read, 0x10)); - for (uint32_t word = 0; word < 4u; ++word) { + std::array packed; + for (uint32_t part = 0; part < bitmap_words / 4u; ++part) { + read.offset = part * (variant == Variant::Bitmap8MisplacedLoad ? 32u : 16u); + const auto source = variant == Variant::Bitmap8OtherBuffer && part != 0u + ? fixture.Buffer({fixture.UserData(8), fixture.UserData(9), fixture.UserData(10), + fixture.UserData(11)}) : buffer; + const auto guard = variant == Variant::Bitmap8OtherGuard && part != 0u + ? fixture.Emit(ValueOpcode::LogicalAnd, + {load_active, fixture.Emit(ValueOpcode::ULessThan32, {lane, Value(16u)})}) + : load_active; + packed[part] = fixture.Emit(ValueOpcode::LoadBufferU32x4, + {source, lane, + Value(variant == Variant::BitmapByteOffset ? 4u : 0u), + Value(variant == Variant::BitmapScalarOffset ? 4u : 0u), guard}, + fixture.AddMemory(read, 0x10 + part * 8u)); + } + for (uint32_t word = 0; word < bitmap_words; ++word) { + const bool wrong_word = variant == Variant::BitmapWrongWord || + (variant == Variant::Bitmap8WrongUpperWord && word >= 4u); auto component = fixture.Emit(ValueOpcode::CompositeExtractU32x4, - {packed, Value(variant == Variant::BitmapWrongWord ? (word + 1u) % 4u : word)}, 0, entry); + {packed[word / 4u], Value((wrong_word ? word + 1u : word) % 4u)}, 0, entry); component = fixture.Emit(ValueOpcode::SelectU32, {load_active, component, fixture.Emit(ValueOpcode::UndefU32)}); loaded_words[word] = component; @@ -2573,7 +2590,7 @@ void TestDescendingImageRange() { remaining_words[word]->AddPhiOperand(header, Value(outer_words[word])); } auto selected = fixture.Emit(ValueOpcode::UndefU32, {}, 0, guard); - for (uint32_t word = 0; word < 4u; ++word) { + for (uint32_t word = 0; word < bitmap_words; ++word) { selected = fixture.Emit(ValueOpcode::SelectU32, {fixture.Emit(ValueOpcode::IEqual32, {Value(&induction), Value(word)}, 0, guard), Value(remaining_words[word]), selected}, 0, guard); @@ -2636,32 +2653,39 @@ void TestDescendingImageRange() { {fixture.Emit(ValueOpcode::ShiftLeftLogical32, {Value(1u), bit})})}); if (variant == Variant::BitmapAddsBit) next = fixture.Emit(ValueOpcode::BitwiseOr32, {next, Value(1u << 21u)}); - for (uint32_t word = 0; word < 4u; ++word) { + for (uint32_t word = 0; word < bitmap_words; ++word) { + // Relative register writes index with M0 & 0xff; a narrower mask aliases upper words. + const auto index = bitmap_words == 4u ? Value(&induction) + : fixture.Emit(ValueOpcode::BitwiseAnd32, + {Value(&induction), Value(variant == Variant::Bitmap8NarrowMask ? 3u : 255u)}); const auto selected_word = fixture.Emit(ValueOpcode::LogicalAnd, - {active, fixture.Emit(ValueOpcode::IEqual32, {Value(&induction), Value(word)})}); + {active, fixture.Emit(ValueOpcode::IEqual32, {index, Value(word)})}); remaining_words[word]->AddPhiOperand(body, fixture.Emit(ValueOpcode::SelectU32, {selected_word, next, Value(remaining_words[word])})); auto carried = Value(remaining_words[word]); - if (variant == Variant::BitmapOuterClobber) + if (variant == Variant::BitmapOuterClobber || variant == Variant::Bitmap8OuterClobber) carried = fixture.Emit(ValueOpcode::SelectU32, {fixture.Emit(ValueOpcode::IEqual32, {Value(&induction), Value(word + 1u)}, 0, latch), - loaded_words[(word + 1u) % 4u], carried}, 0, latch); + loaded_words[(word + 1u) % bitmap_words], carried}, 0, latch); outer_words[word]->AddPhiOperand(latch, carried); } } const auto step = fixture.Emit(ValueOpcode::IAdd32, {Value(&induction), Value(variant == Variant::WrongStep ? UINT32_MAX - 1u : UINT32_MAX)}, 0, latch); - induction.AddPhiOperand(entry, Value(variant == Variant::NegativeInitial ? UINT32_MAX : 3u)); + induction.AddPhiOperand(entry, + Value(variant == Variant::NegativeInitial ? UINT32_MAX : bitmap_words - 1u)); induction.AddPhiOperand(latch, step); fixture.PlanAndTrack(); Check(ordinal.Instruction()->Arg(1).Resolve() == bit.Resolve(), "descending range retained impossible relative-register stores"); const auto &indirect = fixture.program.descriptor_sources[ fixture.program.info.images[0].source].indirect_descriptor; - Check(indirect && indirect->selector.has_value() == (variant == Variant::Bitmap) && - (!indirect->selector || indirect->selector->bitset) && - indirect->key_count == Value(128u) && + const bool selects = variant == Variant::Bitmap || variant == Variant::Bitmap8; + Check(indirect && indirect->selector.has_value() == selects && + (!indirect->selector || (indirect->selector->bitset && + indirect->selector->stride == bitmap_words * 4u)) && + indirect->key_count == Value(bitmap_words * 32u) && !indirect->selector_first.IsEmpty(), "descending bitscan range lost its bound or accepted an unproved bitmap subset"); return ExtractResourcePlan(fixture.program); @@ -2701,10 +2725,14 @@ void TestDescendingImageRange() { } auto bitmap_plan = make_plan(Variant::Bitmap); + auto wide_plan = make_plan(Variant::Bitmap8); for (const auto variant : {Variant::BitmapAddsBit, Variant::BitmapWrongWord, Variant::BitmapUnknownInitial, Variant::BitmapByteOffset, Variant::BitmapScalarOffset, Variant::BitmapOuterClobber, - Variant::BitmapWrongLoadMask}) + Variant::BitmapWrongLoadMask, Variant::Bitmap8MisplacedLoad, + Variant::Bitmap8OtherBuffer, Variant::Bitmap8OuterClobber, + Variant::Bitmap8WrongUpperWord, Variant::Bitmap8NarrowMask, + Variant::Bitmap8OtherGuard}) make_plan(variant); // Every descriptor is distinct, so retained keys are visible in the snapshot as well as // its lookup table. An unrelated table slot must never be read, even when it is malformed. @@ -2719,23 +2747,25 @@ void TestDescendingImageRange() { const SrtRuntime bitmap_runtime {.user_data = bitmap_data, .userdata = &memory, .read_specialization_memory = ReadLinearTestMemory}; memory.watched_address = bitmap_address; + auto *checked_plan = &bitmap_plan; const auto check_bitmap = [&](std::initializer_list ranks, uint32_t count) { bitmap_data[2] = count; memory.watched_reads = memory.descriptor_reads = 0u; std::vector keys; for (const auto rank : ranks) keys.push_back(count - rank); std::ranges::sort(keys); - Check(MaterializeResources(bitmap_plan, bitmap_runtime, snapshot, specialization) && + const auto dwords = bitmap_data[6] * (bitmap_data[5] >> 18u); + Check(MaterializeResources(*checked_plan, bitmap_runtime, snapshot, specialization) && memory.watched_reads == uint32_t(bitmap_data[6] != 0u) && (bitmap_data[6] == 0u || memory.watched_dwords == - std::min(bitmap_data[6] * 4u, MaxIndirectDescriptorProbes)) && + std::min(dwords, MaxIndirectDescriptorProbes)) && memory.descriptor_reads == keys.size() && snapshot.images.size() == std::max(keys.size(), 1u), "bitmap selector lost bounded reads or retained unreachable descriptors"); - for (uint32_t first = 0; first < bitmap_data[6] * 4u; first += MaxIndirectDescriptorProbes) + for (uint32_t first = 0; first < dwords; first += MaxIndirectDescriptorProbes) Check(std::ranges::find(snapshot.specialization_reads, std::pair{uint64_t{bitmap_address} + first * 4u, - uint64_t{std::min(bitmap_data[6] * 4u - first, MaxIndirectDescriptorProbes)} * 4u}) != + uint64_t{std::min(dwords - first, MaxIndirectDescriptorProbes)} * 4u}) != snapshot.specialization_reads.end(), "bitmap chunk was omitted from same-dispatch write-alias protection"); if (keys.empty()) { @@ -2787,8 +2817,8 @@ void TestDescendingImageRange() { memory.fail_address = UINT64_MAX; const auto valid_data = bitmap_data; - for (const auto [word, value] : std::array, 6>{{ - {4u, bitmap_address + 1u}, {5u, 8u << 16u}, {5u, (16u << 16u) | (1u << 31u)}, + for (const auto [word, value] : std::array, 7>{{ + {4u, bitmap_address + 1u}, {5u, 8u << 16u}, {5u, 32u << 16u}, {5u, (16u << 16u) | (1u << 31u)}, {7u, 0x5204u | (1u << 23u)}, {7u, 0x5204u | (1u << 28u)}, {7u, 0x5204u | (1u << 30u)}}}) { bitmap_data[word] = value; @@ -2828,6 +2858,23 @@ void TestDescendingImageRange() { return read.first >= memory.fail_address; }), "a complete bitmap domain read redundant chunks or lost candidate keys"); + + // Eight-word records keep their upper words' ranks, and a key below the table is never read. + checked_plan = &wide_plan; + bitmap_data = valid_data; + bitmap_data[5] = 32u << 16u; + bitmap_data[6] = 2u; + std::fill_n(memory.words.begin() + bitmap_word, 16u, 0u); + memory.words[bitmap_word] = 1u | (1u << 2u); + memory.words[bitmap_word + 12u] = 1u << 1u; + memory.fail_address = 0x2000u - 32u; + check_bitmap({0u, 2u, 129u}, 2u); + memory.fail_address = UINT64_MAX; + for (const auto stride : {16u, 64u}) { + bitmap_data[5] = stride << 16u; + Check(!MaterializeResources(wide_plan, bitmap_runtime, snapshot, specialization), + "an eight-word bitmap selector accepted a different record stride"); + } } void TestUniformizedMaterialImageKeys() {