diff --git a/src/graphics/shader/recompiler/ir/passes/ResourceTracking.cpp b/src/graphics/shader/recompiler/ir/passes/ResourceTracking.cpp index c27b77610..d7809e591 100644 --- a/src/graphics/shader/recompiler/ir/passes/ResourceTracking.cpp +++ b/src/graphics/shader/recompiler/ir/passes/ResourceTracking.cpp @@ -6,6 +6,7 @@ #include "graphics/shader/recompiler/ir/passes/SrtWalker.h" #include +#include #include #include #include @@ -1293,14 +1294,15 @@ private: EquivalentValue(m_program, inst->Arg(1), index)); } - bool MaskOnlyLosesBits(Value value, Value mask) const { + bool MaskOnlyLosesBits(Value value, Value mask, bool nonzero) const { const auto* update = value.Resolve().TryInstruction(); if (update == nullptr || update->NumArgs() != 2u) return false; if (update->GetOpcode() == ValueOpcode::BitwiseAnd32) { return EquivalentValue(m_program, update->Arg(0), mask) || EquivalentValue(m_program, update->Arg(1), mask); } - if (update->GetOpcode() != ValueOpcode::BitwiseXor32) return false; + // XOR of the first-bit mask adds bit 31 when FindILsb receives zero. + if (!nonzero || update->GetOpcode() != ValueOpcode::BitwiseXor32) return false; Value bit; if (EquivalentValue(m_program, update->Arg(0), mask)) bit = update->Arg(1); else if (EquivalentValue(m_program, update->Arg(1), mask)) bit = update->Arg(0); @@ -1325,16 +1327,15 @@ private: first->NumArgs() == 1u && EquivalentValue(m_program, first->Arg(0), mask); } - Value InitialCandidateMask(Value value, const Block* update_block) const { + Value InitialCandidateMask(Value value, const Block* update_block = nullptr, + bool nonzero = false) const { const auto* phi = value.Resolve().TryInstruction(); if (phi == nullptr || phi->GetOpcode() != ValueOpcode::Phi || phi->NumArgs() != 2u || phi->GetType() != Type::U32) return {}; for (uint32_t back = 0; back < 2u; ++back) { - if (phi->PhiBlock(back) != update_block || - !MaskOnlyLosesBits(phi->Arg(back), value)) continue; - const auto initial = phi->Arg(back ^ 1u).Resolve(); - return ValidateRuntimeValue(m_program, initial, RuntimeValueType::Integer) - ? initial : Value {}; + if ((update_block != nullptr && phi->PhiBlock(back) != update_block) || + !MaskOnlyLosesBits(phi->Arg(back), value, nonzero)) continue; + return phi->Arg(back ^ 1u).Resolve(); } return {}; } @@ -1362,8 +1363,9 @@ private: if (first == nullptr || first->GetOpcode() != ValueOpcode::FindILsb32 || first->NumArgs() != 1u || !ImpliesNonzero(bit_guard, first->Arg(0))) continue; - const auto initial_mask = InitialCandidateMask(first->Arg(0), phi->PhiBlock(bit_arm)); - if (initial_mask.IsEmpty()) continue; + const auto initial_mask = InitialCandidateMask(first->Arg(0), phi->PhiBlock(bit_arm), true); + if (initial_mask.IsEmpty() || + !ValidateRuntimeValue(m_program, initial_mask, RuntimeValueType::Integer)) continue; for (uint32_t sentinel_arm = 0; sentinel_arm < 3u; ++sentinel_arm) { if (sentinel_arm == bit_arm) continue; const auto sentinel_guard = PositiveLaneWitness(phi->PhiBlock(sentinel_arm)); @@ -1650,32 +1652,48 @@ private: return false; } - Value UnsignedBound(Value guard, Value value, bool positive = true) const { + Value UnsignedBound(Value guard, Value value, bool positive = true, + bool finite_bitscan = false) const { const auto* test = SimplifyGuard(guard).TryInstruction(); if (test == nullptr) return {}; const auto op = test->GetOpcode(); if (op == ValueOpcode::LogicalNot) - return UnsignedBound(test->Arg(0), value, !positive); + return UnsignedBound(test->Arg(0), value, !positive, finite_bitscan); if ((op == ValueOpcode::LogicalAnd && positive) || (op == ValueOpcode::LogicalOr && !positive)) { - const auto left = UnsignedBound(test->Arg(0), value, positive); - return left.IsEmpty() ? UnsignedBound(test->Arg(1), value, positive) : left; + const auto left = UnsignedBound(test->Arg(0), value, positive, finite_bitscan); + return left.IsEmpty() ? UnsignedBound(test->Arg(1), value, positive, finite_bitscan) : left; } if ((positive ? op == ValueOpcode::ULessThan32 : op == ValueOpcode::UGreaterThanEqual32) && EquivalentValue(m_program, test->Arg(0), value)) return test->Arg(1); if (!positive && op == ValueOpcode::ULessThanEqual32 && EquivalentValue(m_program, test->Arg(1), value)) return test->Arg(0); + const auto* scan = value.Resolve().TryInstruction(); + if (finite_bitscan && scan != nullptr && scan->GetOpcode() == ValueOpcode::FindILsb32 && + (positive ? op == ValueOpcode::INotEqual32 : op == ValueOpcode::IEqual32)) { + for (uint32_t arg = 0; arg < 2u; ++arg) { + uint32_t excluded; + if (!ImmediateU32(test->Arg(arg), excluded)) continue; + if ((excluded == UINT32_MAX && EquivalentValue(m_program, test->Arg(arg ^ 1u), value)) || + (excluded == 0u && EquivalentValue(m_program, test->Arg(arg ^ 1u), scan->Arg(0)))) + return Value(static_cast(std::bit_width(SelectorMaximum(scan->Arg(0))))); + } + } return {}; } - Value UnsignedBoundOnEntry(Value key, const Block* block) const { + Value UnsignedBoundOnEntry(Value key, const Block* block, bool finite_bitscan = false) const { const auto* definition = key.Resolve().TryInstruction(); + // A mask-nonzero guard may precede FF1. Its proof lasts until the mask's + // definition, including a loop Phi that changes it for the next iteration. + if (definition != nullptr && definition->GetOpcode() == ValueOpcode::FindILsb32) + definition = definition->Arg(0).Resolve().TryInstruction(); Value bound; const bool guarded = GuardedOnEntry(block, [&](const Block* current) { return definition != nullptr && current == definition->Parent(); }, [&](const EdgePredicate& edge) { if (edge.lanes != LaneQuantifier::All) return false; - const auto candidate = UnsignedBound(edge.condition, key, edge.positive); + const auto candidate = UnsignedBound(edge.condition, key, edge.positive, finite_bitscan); if (candidate.IsEmpty() || !ValidateRuntimeValue(m_program, candidate, RuntimeValueType::Integer)) return false; if (bound.IsEmpty()) bound = candidate; @@ -2438,11 +2456,8 @@ private: } else if (table_source.dword_count == 2u) { auto guarded_count = UnsignedBoundOnEntry(key, handle.Parent()); if (guarded_count.IsEmpty()) guarded_count = UnsignedImageBound(key, handle); - const auto* selector = key.Resolve().TryInstruction(); - const bool bitscan = guarded_count.IsEmpty() && selector != nullptr && - selector->GetOpcode() == ValueOpcode::FindILsb32 && - selector->NumArgs() == 1u && !m_shader_writes && - NonzeroOnEntry(selector->Arg(0), handle.Parent()); + // A runtime count can describe fewer rows than the mask's static bit range. + if (guarded_count.IsEmpty()) guarded_count = UnsignedBoundOnEntry(key, handle.Parent(), true); if (!guarded_count.IsEmpty()) { if (table_stride < 32u || (table_stride & 3u) != 0u) return false; // Materialization captures the count and descriptor reads; renderer alias @@ -2450,8 +2465,6 @@ private: indirect.key_count = guarded_count; } else if (table_stride != 32u) { return false; - } else if (bitscan) { - indirect.key_count = Value(32u); } else if (!m_shader_writes) { if (const auto* bound = BoundedLoop(key, handle.Parent())) { indirect.key_count = bound->Arg(1); @@ -2463,8 +2476,7 @@ private: !MatchReversedImageRange(key, indirect)) { return false; } - if ((table_offset & 3u) != 0u || - (bitscan && table_offset > UINT32_MAX - (32u * 32u - 1u))) return false; + if ((table_offset & 3u) != 0u) return false; } else { // A bounded V# supplies the complete image table. Leave every GPU selector // and descriptor read in the shader; the host only translates table bytes. @@ -2665,6 +2677,16 @@ private: return SelectorMaximum(inst->Arg(arg), bounds, depth + 1u, guard); }; switch (inst->GetOpcode()) { + case ValueOpcode::Phi: { + const auto initial = InitialCandidateMask(value); + return initial.IsEmpty() ? UINT32_MAX + : SelectorMaximum(initial, bounds, depth + 1u, guard); + } + case ValueOpcode::BitFieldUExtract: { + uint32_t width; + if (ImmediateU32(inst->Arg(2), width) && width < 32u) return (1u << width) - 1u; + break; + } case ValueOpcode::FindILsb32: case ValueOpcode::FindUMsb32: return (!guard.IsEmpty() && ImpliesNonzero(guard, inst->Arg(0))) || diff --git a/tests/ResourceTrackingTests.cpp b/tests/ResourceTrackingTests.cpp index 80a862def..0f52186d9 100644 --- a/tests/ResourceTrackingTests.cpp +++ b/tests/ResourceTrackingTests.cpp @@ -565,26 +565,28 @@ void TestGuardedDirectImageTable() { Nonzero, SccNonZero, Plain, Zero, Unrelated, Bypass, ExecZero, VccZero, Bounded, BoundMissing, BoundUnrelated, BoundBypass, BoundAnyLane, BoundDiamond, BoundBranchSpecific, BoundMismatched, - BoundExec, BoundExecUnguarded, BoundExecMismatched, BoundExecAnd, BoundExecUnprovenActive + BoundExec, BoundExecUnguarded, BoundExecMismatched, BoundExecAnd, BoundExecUnprovenActive, + Finite, FiniteZero, FiniteMissing, FiniteWrong, FiniteSigned, FiniteAdd, FiniteXor }; const auto make_plan = [](Guard guard) { - const bool bounded = guard >= Guard::Bounded; - const bool exec_bound = guard >= Guard::BoundExec; + const bool finite = guard >= Guard::Finite; + const bool bounded = guard >= Guard::Bounded && !finite; + const bool exec_bound = bounded && guard >= Guard::BoundExec; const bool bypass = guard == Guard::Bypass || guard == Guard::BoundBypass; - const uint32_t table_offset = exec_bound ? 672u : bounded ? 3696u : 344u; + const uint32_t table_offset = finite ? 3344u : exec_bound ? 672u : bounded ? 3696u : 344u; const uint32_t table_stride = exec_bound ? 784u : bounded ? 272u : 32u; - Fixture fixture(bounded ? ShaderType::Compute : ShaderType::Pixel); + Fixture fixture(bounded || finite ? ShaderType::Compute : ShaderType::Pixel); fixture.program.wave_size = 64u; auto *entry = fixture.block; auto *middle = fixture.AddBlock(); auto *before_sample = fixture.AddBlock(); auto *sample = fixture.AddBlock(); auto *exit = fixture.AddBlock(); - const auto varying = bounded + const auto varying = bounded || finite ? fixture.Emit(ValueOpcode::GetBuiltin, {Value(static_cast(StageInputKind::LocalInvocationId)), Value(0u)}) : fixture.Emit(ValueOpcode::GetAttribute, {Value(0u), Value(0u)}); - const auto mask = fixture.Emit(ValueOpcode::ReadFirstLane, {varying, Value(true)}); + auto mask = fixture.Emit(ValueOpcode::ReadFirstLane, {varying, Value(true)}); const auto srt = fixture.Address(fixture.UserData(0), fixture.UserData(1)); Value key; Value limit; @@ -596,13 +598,44 @@ void TestGuardedDirectImageTable() { limit = fixture.Emit(ValueOpcode::LoadAddressU32, {srt, Value(0u), Value(0u), Value(true)}, fixture.AddMemory(memory, 0x10)); } + Block *latch = nullptr; + if (finite) { + latch = fixture.AddBlock(); + MemoryInfo memory; + memory.kind = ResourceKind::ScalarBuffer; + const auto input = fixture.Buffer({Value(0x900000u), Value(0u), Value(4096u), Value(0u)}); + const auto loaded = fixture.Emit(ValueOpcode::ReadConstBuffer, {input, mask}, + fixture.AddMemory(memory, 0x2ecu)); + const auto initial = fixture.Emit(ValueOpcode::BitFieldUExtract, + {loaded, Value(0u), Value(guard == Guard::FiniteZero ? 0u : 12u)}); + auto &phi = middle->AppendNewInst(ValueOpcode::Phi, {}, static_cast(Type::U32)); + mask = Value(&phi); + key = fixture.Emit(ValueOpcode::FindILsb32, {mask}, 0, middle); + const auto bit = fixture.Emit(ValueOpcode::ShiftLeftLogical32, + {Value(1u), fixture.Emit(ValueOpcode::BitwiseAnd32, {key, Value(31u)}, 0, latch)}, 0, latch); + const auto next = guard == Guard::FiniteAdd + ? fixture.Emit(ValueOpcode::BitwiseOr32, {mask, Value(1u << 31u)}, 0, latch) + : guard == Guard::FiniteXor + ? fixture.Emit(ValueOpcode::BitwiseXor32, {mask, bit}, 0, latch) + : fixture.Emit(ValueOpcode::BitwiseAnd32, + {mask, fixture.Emit(ValueOpcode::BitwiseNot32, {bit}, 0, latch)}, 0, latch); + phi.AddPhiOperand(entry, initial); + phi.AddPhiOperand(latch, next); + } const auto kind = guard == Guard::ExecZero || guard == Guard::BoundAnyLane || exec_bound ? CFG::BranchCondition::ExecZero : guard == Guard::VccZero ? CFG::BranchCondition::VccZero : CFG::BranchCondition::SccZero; Value condition; Value active(true); - if (bounded) { + if (finite) { + condition = guard == Guard::FiniteMissing ? Value(false) + : guard == Guard::FiniteSigned + ? fixture.Emit(ValueOpcode::LogicalNot, + {fixture.Emit(ValueOpcode::SLessThan32, {key, Value(12u)}, 0, middle)}, 0, middle) + : fixture.Emit(ValueOpcode::IEqual32, + {key, Value(guard == Guard::FiniteWrong ? 31u : UINT32_MAX)}, 0, middle); + } else if (bounded) { condition = guard == Guard::BoundMissing ? Value(false) : fixture.Emit( ValueOpcode::UGreaterThanEqual32, {guard == Guard::BoundUnrelated ? fixture.UserData(2) : key, limit}); @@ -630,7 +663,7 @@ void TestGuardedDirectImageTable() { condition = fixture.Emit(ValueOpcode::LogicalNot, {condition}); } if (guard != Guard::Plain && guard != Guard::SccNonZero) { - condition = fixture.Emit(ValueOpcode::ConditionRef, {condition}, kind); + condition = fixture.Emit(ValueOpcode::ConditionRef, {condition}, kind, finite ? middle : entry); } fixture.program.blocks[0]->condition = condition; fixture.program.blocks[0]->terminator = { @@ -645,6 +678,18 @@ void TestGuardedDirectImageTable() { fixture.program.blocks[4]->terminator = { .kind = bypass ? CFG::TerminatorKind::Branch : CFG::TerminatorKind::Return, .true_block = fixture.program.blocks[3u]}; + if (finite) { + entry->condition = {}; + entry->terminator = {.kind = CFG::TerminatorKind::Branch, .true_block = middle}; + middle->condition = condition; + middle->terminator = {.kind = CFG::TerminatorKind::ConditionalBranch, + .true_block = guard == Guard::FiniteXor ? latch : exit, .false_block = before_sample}; + before_sample->condition = fixture.Emit(ValueOpcode::IEqual32, {varying, Value(0u)}, 0, before_sample); + before_sample->terminator = {.kind = CFG::TerminatorKind::ConditionalBranch, + .true_block = sample, .false_block = latch}; + sample->terminator = {.kind = CFG::TerminatorKind::Branch, .true_block = latch}; + latch->terminator = {.kind = CFG::TerminatorKind::Branch, .true_block = middle}; + } if (guard == Guard::BoundDiamond || exec_bound) { middle->condition = fixture.Emit(ValueOpcode::IEqual32, {mask, Value(1u)}); @@ -673,7 +718,7 @@ void TestGuardedDirectImageTable() { } std::array pointer; for (uint32_t word = 0; word < pointer.size(); ++word) { - if (bounded) { + if (bounded || finite) { pointer[word] = srt.Instruction()->Arg(word); continue; } @@ -685,7 +730,7 @@ void TestGuardedDirectImageTable() { fixture.AddMemory(memory, 0x20)); } fixture.block = sample; - if (!bounded) key = fixture.Emit(ValueOpcode::FindILsb32, {mask}); + if (!bounded && !finite) key = fixture.Emit(ValueOpcode::FindILsb32, {mask}); const auto offset = fixture.Emit(ValueOpcode::IAdd32, {bounded ? fixture.Emit(ValueOpcode::IMul32, {key, Value(table_stride)}) : fixture.Emit(ValueOpcode::ShiftLeftLogical32, {key, Value(5u)}), @@ -719,7 +764,7 @@ void TestGuardedDirectImageTable() { fixture.Emit(ValueOpcode::ImageSampleRaw, {image, sampler, fixture.ImageAddress(), other_active}, fixture.AddMemory(memory, 0x12c)); } - if (bounded) { + if (bounded || finite) { const auto output = fixture.Image({Value(0x8000u), Value(static_cast(Libs::Graphics::Prospero::BufferFormat::k32UInt) << 20u), Value(0u), Value((static_cast(Libs::Graphics::Prospero::ImageType::kColor2D) << 28u) | @@ -751,7 +796,8 @@ void TestGuardedDirectImageTable() { Check(indirect && !indirect->selector.has_value() && indirect->table_offset == table_offset && indirect->table_stride == table_stride && (bounded ? !indirect->key_count.IsEmpty() - : indirect->key_count.Resolve() == Value(32u)) && + : indirect->key_count.Resolve() == Value(guard == Guard::FiniteZero ? 0u : + finite && guard != Guard::FiniteAdd && guard != Guard::FiniteXor ? 12u : 32u)) && fixture.program.descriptor_sources[indirect->table_source].dword_count == 2u, "guarded direct image table lost its pointer or proven selector range"); return ExtractResourcePlan(fixture.program); @@ -765,7 +811,8 @@ void TestGuardedDirectImageTable() { Guard::BoundUnrelated, Guard::BoundBypass, Guard::BoundAnyLane, Guard::BoundBranchSpecific, Guard::BoundMismatched, Guard::BoundExecUnguarded, Guard::BoundExecMismatched, - Guard::BoundExecAnd, Guard::BoundExecUnprovenActive}) { + Guard::BoundExecAnd, Guard::BoundExecUnprovenActive, + Guard::FiniteMissing, Guard::FiniteWrong, Guard::FiniteSigned}) { CheckFatal([&] { make_plan(guard); }, "not a valid runtime value", "direct table accepted a selector without a dominating range guard"); } @@ -892,6 +939,38 @@ void TestGuardedDirectImageTable() { records.descriptor_reads == 0u, "unrepresentable table span was read before validating its count"); } + for (const auto guard : {Guard::Finite, Guard::FiniteZero, Guard::FiniteAdd, Guard::FiniteXor}) { + const auto finite = make_plan(guard); + Check(finite.capture_specialization_reads && finite.info.images[1].written, + "finite bit table with image writes omitted specialization read capture"); + LinearTestMemory records; + const uint32_t count = guard == Guard::FiniteZero ? 0u : guard == Guard::Finite ? 12u : 32u; + const uint64_t first = records.base + 3344u; + fill_table(records, first); + // The GPU may select no bits or a sparse subset. Null rows remain valid candidates. + std::fill_n(records.words.begin() + (first - records.base) / 4u + 5u * 8u, 8u, 0u); + records.fail_address = first + count * 32u; + user_data = {static_cast(records.base), 0u}; + runtime.userdata = &records; + Check(MaterializeResources(finite, runtime, snapshot, specialization) && + records.descriptor_reads == count && snapshot.images.size() == std::max(count, 1u) + 1u, + "finite bit table probed beyond its mask bound or lost a null row"); + if (count == 0u) { + Check(std::ranges::all_of(snapshot.images[0].dwords, [](uint32_t word) { return word == 0u; }), + "empty bit loop retained a reachable image"); + continue; + } + Check(snapshot.flattened_srt[specialization.images[0].indirect_mapping_offset] == count, + "finite bit table lost its candidate count"); + for (uint32_t key = 0; key < count; ++key) { + Check(std::ranges::any_of(snapshot.specialization_reads, [&](const auto &read) { + return read.first == first + key * 32u && read.second == 32u; + }), "finite bit table omitted a descriptor from writable-alias validation"); + } + records.fail_address = first + (count - 1u) * 32u + 28u; + Check(!MaterializeResources(finite, runtime, snapshot, specialization), + "finite bit table accepted an unreadable last candidate"); + } } void TestBoundedComputeImageLoop() { diff --git a/tests/ShaderRecompilerComputeTests.cpp b/tests/ShaderRecompilerComputeTests.cpp index ecced8559..a0597b3a6 100644 --- a/tests/ShaderRecompilerComputeTests.cpp +++ b/tests/ShaderRecompilerComputeTests.cpp @@ -36492,18 +36492,22 @@ void CheckCountedRowImageSelector(VulkanHarness &vulkan) { void CheckGuardedRecordImageTable(VulkanHarness &vulkan) { using O = ShaderOpcode; constexpr u32 table = 256u; - for (const bool masked : {false, true}) + for (const u32 variant : {0u, 1u, 2u}) for (const u32 wave : {32u, 64u}) for (const u32 count : {0u, 2u, 3u}) { - const u32 stride = masked ? 784u : 272u, offset = masked ? 672u : 3696u; + const bool masked = variant == 1u, bitset = variant == 2u; + const u32 stride = bitset ? 32u : masked ? 784u : 272u; + const u32 offset = bitset ? 3344u : masked ? 672u : 3696u; + const u32 rows = bitset ? 12u : count, members = bitset ? 1u : 2u; const u32 lanes = masked ? 2u : 1u; TestCase test; - test.name = masked ? (wave == 32u ? "ExecGuardedRecordTable32" : "ExecGuardedRecordTable64") + test.name = bitset ? (wave == 32u ? "BitsetRecordTable32" : "BitsetRecordTable64") + : masked ? (wave == 32u ? "ExecGuardedRecordTable32" : "ExecGuardedRecordTable64") : (wave == 32u ? "GuardedRecordImageTable32" : "GuardedRecordImageTable64"); - test.initial.resize((table + offset + stride * count) / 4u, 0xdeadbeefu); - test.initial[table / 4u + (masked ? 0u : 1u)] = count; - for (u32 row = 0; row < count; ++row) - for (const u32 member : {0u, 1u}) { + test.initial.resize((table + offset + stride * rows) / 4u, 0xdeadbeefu); + test.initial[table / 4u + (masked ? 0u : 1u)] = bitset ? (1u << count) - 1u : count; + for (u32 row = 0; row < rows; ++row) + for (u32 member = 0; member < members; ++member) { auto image = MakeSampledTextureData(Prospero::BufferFormat::k32Float); image[0] = (row + 1u) * (member == 0u ? 1u : 10u); image[3] = DstSel(4, 0, 0, 1) | @@ -36530,6 +36534,9 @@ void CheckGuardedRecordImageTable(VulkanHarness &vulkan) { auto &code = test.code; code.insert(code.end(), {EncodeSop2(0x20, 20, InlineU32(5), 44), EncodeSmem0(0x00, 21), EncodeSmem1(masked ? 0u : 4u, 125)}); + if (bitset) code.insert(code.end(), { + EncodeSop2(0x0e, 20, 20, 21), + EncodeSop2(0x27, 20, 20, 255), 12u << 16u}); code.push_back(EncodeSop1(0x04, 30, 126)); // Preserve EXEC for the final write. code.push_back(EncodeVop1(0x01, 13, Vgpr(0))); // Odd lane is already done. AppendVMovLiteral(&code, 10, 0xdeadbeefu); @@ -36540,33 +36547,41 @@ void CheckGuardedRecordImageTable(VulkanHarness &vulkan) { code.push_back(EncodeSop1(0x13, 22, 20)); // Find the next set bit in the GPU worklist. code.push_back(EncodeSopc(0x07, 22, 193)); const auto empty_exit = code.size(); code.push_back(0u); - if (masked) code.insert(code.end(), {EncodeSop2(0x0b, 28, 126, InlineU32(0)), - EncodeSop1(0x04, 126, 28)}); - code.push_back(EncodeSopc(0x09, 22, 21)); - if (masked) code.insert(code.end(), { - EncodeVop2(0x1c, 10, 253, 13), // SCC encoded as 0/1, OR per-lane done. - EncodeVopc(0xc5, InlineU32(0), 10), - EncodeSop2(0x15, 126, 28, 106)}); - const auto range_exit = code.size(); code.push_back(0u); + size_t range_exit = 0; + if (!bitset) { + if (masked) code.insert(code.end(), {EncodeSop2(0x0b, 28, 126, InlineU32(0)), + EncodeSop1(0x04, 126, 28)}); + code.push_back(EncodeSopc(0x09, 22, 21)); + if (masked) code.insert(code.end(), { + EncodeVop2(0x1c, 10, 253, 13), // SCC encoded as 0/1, OR per-lane done. + EncodeVopc(0xc5, InlineU32(0), 10), + EncodeSop2(0x15, 126, 28, 106)}); + range_exit = code.size(); code.push_back(0u); + } code.insert(code.end(), {EncodeSop2(0x26, 23, 22, 255), stride, - EncodeSop2(0x02, 23, 23, 255), offset, - EncodeSopc(0x0d, 22, InlineU32(0))}); - const auto skip_first = code.size(); code.push_back(0u); - code.insert(code.end(), { - EncodeSmem0(0x03, 8), EncodeSmem1(0, 23), + EncodeSop2(0x02, 23, 23, 255), offset}); + size_t skip_first = 0; + if (!bitset) { + code.push_back(EncodeSopc(0x0d, 22, InlineU32(0))); + skip_first = code.size(); code.push_back(0u); + } + code.insert(code.end(), {EncodeSmem0(0x03, 8), EncodeSmem1(0, 23), EncodeMimg0(0x27, 1), EncodeMimg1(4, 20, 2, 10), EncodeVop2(0x03, 0, Vgpr(4), 0)}); - code[skip_first] = EncodeSopp(0x05, code.size() - skip_first - 1u); - // The second descriptor is reached through both sides of the optional first sample. - code.insert(code.end(), {EncodeSop2(0x02, 23, 23, InlineU32(32)), - EncodeSmem0(0x03, 8), EncodeSmem1(0, 23), - EncodeMimg0(0x27, 1), EncodeMimg1(4, 20, 2, 10), - EncodeVop2(0x03, 0, Vgpr(4), 0), - EncodeSop2(0x1e, 24, InlineU32(1), 22), + if (!bitset) { + code[skip_first] = EncodeSopp(0x05, code.size() - skip_first - 1u); + // Both sides of the optional first sample reach the second descriptor. + code.insert(code.end(), {EncodeSop2(0x02, 23, 23, InlineU32(32)), + EncodeSmem0(0x03, 8), EncodeSmem1(0, 23), + EncodeMimg0(0x27, 1), EncodeMimg1(4, 20, 2, 10), + EncodeVop2(0x03, 0, Vgpr(4), 0)}); + } + code.insert(code.end(), {EncodeSop2(0x1e, 24, InlineU32(1), 22), EncodeSop2(0x14, 20, 20, 24)}); code.push_back(EncodeSopp(0x02, loop - code.size() - 1u)); code[empty_exit] = EncodeSopp(0x04, code.size() - empty_exit - 1u); - code[range_exit] = EncodeSopp(masked ? 0x08 : 0x05, code.size() - range_exit - 1u); + if (!bitset) code[range_exit] = + EncodeSopp(masked ? 0x08 : 0x05, code.size() - range_exit - 1u); code.push_back(EncodeSop1(0x04, 126, 30)); code.push_back(EncodeSop2(0x26, 25, 44, InlineU32(lanes))); code.push_back(EncodeVop2(0x25, 30, 25, 13)); @@ -36574,10 +36589,13 @@ void CheckGuardedRecordImageTable(VulkanHarness &vulkan) { code.insert(code.end(), {EncodeMimg0(0x08, 1), EncodeMimg1(0, 30, 8, 0)}); AppendEnd(&code); test.opcodes = {O::S_LSHR_B32, O::S_LOAD_DWORD, O::V_MOV_B32, O::S_FF1_I32_B32, - O::S_CMP_LG_U32, O::S_CMP_GE_U32, O::S_CBRANCH_SCC0, O::S_CBRANCH_SCC1, - O::S_MUL_I32, O::S_ADD_I32, O::S_BITCMP1_B32, O::S_LOAD_DWORDX8, O::IMAGE_SAMPLE, + O::S_CMP_LG_U32, O::S_CBRANCH_SCC0, + O::S_MUL_I32, O::S_ADD_I32, O::S_LOAD_DWORDX8, O::IMAGE_SAMPLE, O::V_ADD_F32, O::S_LSHL_B32, O::S_ANDN2_B32, O::S_BRANCH, O::IMAGE_STORE, O::S_ENDPGM}; + if (bitset) test.opcodes.insert(test.opcodes.end(), {O::S_AND_B32, O::S_BFE_U32}); + else test.opcodes.insert(test.opcodes.end(), {O::S_CMP_GE_U32, O::S_CBRANCH_SCC1, + O::S_BITCMP1_B32}); if (masked) test.opcodes.insert(test.opcodes.end(), {O::S_CSELECT_B64, O::V_OR_B32, O::V_CMP_NE_U32, O::S_ANDN2_B64, O::S_CBRANCH_EXECZ}); test.opcodes.insert(test.opcodes.end(), {O::S_MOV_B64, O::V_ADD_NC_U32}); @@ -36587,7 +36605,7 @@ void CheckGuardedRecordImageTable(VulkanHarness &vulkan) { return image.resource_class == ShaderRecompiler::IR::ImageResourceClass::Sampled; }); Require(test.name, "bounded descriptor rows", - count == 0u ? sampled_count <= 2u : sampled_count == count * 2u, + rows == 0u ? sampled_count <= members : sampled_count == rows * members, "the runtime count lost a row or probed record padding"); std::vector images(compiled.program.info.images.size()); for (u32 resource = 0; resource < images.size(); ++resource) { @@ -36606,9 +36624,10 @@ void CheckGuardedRecordImageTable(VulkanHarness &vulkan) { vulkan.Dispatch(test, compiled, buffer, nullptr, nullptr, &storage, nullptr, sampler, images); std::vector expected(4u * lanes, 0u); if (count != 0u) { - expected[0] = std::bit_cast(count == 3 ? 44.0f : 11.0f); - expected[lanes] = std::bit_cast(20.0f); - expected[2u * lanes] = std::bit_cast(11.0f); + expected[0] = std::bit_cast(bitset ? (count == 3 ? 4.0f : 1.0f) + : (count == 3 ? 44.0f : 11.0f)); + expected[lanes] = std::bit_cast(bitset ? 2.0f : 20.0f); + expected[2u * lanes] = std::bit_cast(bitset ? 1.0f : 11.0f); } CompareWords(test, "GPU worklist, guarded count and image writes", expected, vulkan.ReadImage(test.name, &storage));