raphiaura/KytyPS5
1
0
Fork 0
mirror of https://github.com/KytyPS5/KytyPS5.git synced 2026-10-25 04:16:55 +01:00
Code Issues Packages Projects Releases Wiki Activity

shader: bound descriptor worklists by their finite bit masks

Browse Source
This commit is contained in:
nmzik
2026-10-13 00:37:14 +02:00
parent 9bcf80bf67
commit d70cbd4184
3 changed files with 193 additions and 73 deletions
Download Patch File Download Diff File Expand all files Collapse all files
src/graphics/shader/recompiler/ir/passes/ResourceTracking.cpp
+47 -25
View File
@@ -6,6 +6,7 @@
#include "graphics/shader/recompiler/ir/passes/SrtWalker.h"
#include <algorithm>
#include <bit>
#include <fmt/format.h>
#include <map>
#include <numeric>
@@ -1293,14 +1294,15 @@ private:
EquivalentValue(m_program, inst->Arg(1), index));
}
bool MaskOnlyLosesBits(Value value, Value mask) const {
bool MaskOnlyLosesBits(Value value, Value mask, bool nonzero) const {
const auto* update = value.Resolve().TryInstruction();
if (update == nullptr || update->NumArgs() != 2u) return false;
if (update->GetOpcode() == ValueOpcode::BitwiseAnd32) {
return EquivalentValue(m_program, update->Arg(0), mask) ||
EquivalentValue(m_program, update->Arg(1), mask);
}
if (update->GetOpcode() != ValueOpcode::BitwiseXor32) return false;
// XOR of the first-bit mask adds bit 31 when FindILsb receives zero.
if (!nonzero || update->GetOpcode() != ValueOpcode::BitwiseXor32) return false;
Value bit;
if (EquivalentValue(m_program, update->Arg(0), mask)) bit = update->Arg(1);
else if (EquivalentValue(m_program, update->Arg(1), mask)) bit = update->Arg(0);
@@ -1325,16 +1327,15 @@ private:
first->NumArgs() == 1u && EquivalentValue(m_program, first->Arg(0), mask);
}
Value InitialCandidateMask(Value value, const Block* update_block) const {
Value InitialCandidateMask(Value value, const Block* update_block = nullptr,
bool nonzero = false) const {
const auto* phi = value.Resolve().TryInstruction();
if (phi == nullptr || phi->GetOpcode() != ValueOpcode::Phi ||
phi->NumArgs() != 2u || phi->GetType() != Type::U32) return {};
for (uint32_t back = 0; back < 2u; ++back) {
if (phi->PhiBlock(back) != update_block ||
!MaskOnlyLosesBits(phi->Arg(back), value)) continue;
const auto initial = phi->Arg(back ^ 1u).Resolve();
return ValidateRuntimeValue(m_program, initial, RuntimeValueType::Integer)
? initial : Value {};
if ((update_block != nullptr && phi->PhiBlock(back) != update_block) ||
!MaskOnlyLosesBits(phi->Arg(back), value, nonzero)) continue;
return phi->Arg(back ^ 1u).Resolve();
}
return {};
}
@@ -1362,8 +1363,9 @@ private:
if (first == nullptr || first->GetOpcode() != ValueOpcode::FindILsb32 ||
first->NumArgs() != 1u || !ImpliesNonzero(bit_guard, first->Arg(0)))
continue;
const auto initial_mask = InitialCandidateMask(first->Arg(0), phi->PhiBlock(bit_arm));
if (initial_mask.IsEmpty()) continue;
const auto initial_mask = InitialCandidateMask(first->Arg(0), phi->PhiBlock(bit_arm), true);
if (initial_mask.IsEmpty() ||
!ValidateRuntimeValue(m_program, initial_mask, RuntimeValueType::Integer)) continue;
for (uint32_t sentinel_arm = 0; sentinel_arm < 3u; ++sentinel_arm) {
if (sentinel_arm == bit_arm) continue;
const auto sentinel_guard = PositiveLaneWitness(phi->PhiBlock(sentinel_arm));
@@ -1650,32 +1652,48 @@ private:
return false;
}
Value UnsignedBound(Value guard, Value value, bool positive = true) const {
Value UnsignedBound(Value guard, Value value, bool positive = true,
bool finite_bitscan = false) const {
const auto* test = SimplifyGuard(guard).TryInstruction();
if (test == nullptr) return {};
const auto op = test->GetOpcode();
if (op == ValueOpcode::LogicalNot)
return UnsignedBound(test->Arg(0), value, !positive);
return UnsignedBound(test->Arg(0), value, !positive, finite_bitscan);
if ((op == ValueOpcode::LogicalAnd && positive) ||
(op == ValueOpcode::LogicalOr && !positive)) {
const auto left = UnsignedBound(test->Arg(0), value, positive);
return left.IsEmpty() ? UnsignedBound(test->Arg(1), value, positive) : left;
const auto left = UnsignedBound(test->Arg(0), value, positive, finite_bitscan);
return left.IsEmpty() ? UnsignedBound(test->Arg(1), value, positive, finite_bitscan) : left;
}
if ((positive ? op == ValueOpcode::ULessThan32 : op == ValueOpcode::UGreaterThanEqual32) &&
EquivalentValue(m_program, test->Arg(0), value)) return test->Arg(1);
if (!positive && op == ValueOpcode::ULessThanEqual32 &&
EquivalentValue(m_program, test->Arg(1), value)) return test->Arg(0);
const auto* scan = value.Resolve().TryInstruction();
if (finite_bitscan && scan != nullptr && scan->GetOpcode() == ValueOpcode::FindILsb32 &&
(positive ? op == ValueOpcode::INotEqual32 : op == ValueOpcode::IEqual32)) {
for (uint32_t arg = 0; arg < 2u; ++arg) {
uint32_t excluded;
if (!ImmediateU32(test->Arg(arg), excluded)) continue;
if ((excluded == UINT32_MAX && EquivalentValue(m_program, test->Arg(arg ^ 1u), value)) ||
(excluded == 0u && EquivalentValue(m_program, test->Arg(arg ^ 1u), scan->Arg(0))))
return Value(static_cast<uint32_t>(std::bit_width(SelectorMaximum(scan->Arg(0)))));
}
}
return {};
}
Value UnsignedBoundOnEntry(Value key, const Block* block) const {
Value UnsignedBoundOnEntry(Value key, const Block* block, bool finite_bitscan = false) const {
const auto* definition = key.Resolve().TryInstruction();
// A mask-nonzero guard may precede FF1. Its proof lasts until the mask's
// definition, including a loop Phi that changes it for the next iteration.
if (definition != nullptr && definition->GetOpcode() == ValueOpcode::FindILsb32)
definition = definition->Arg(0).Resolve().TryInstruction();
Value bound;
const bool guarded = GuardedOnEntry(block, [&](const Block* current) {
return definition != nullptr && current == definition->Parent();
}, [&](const EdgePredicate& edge) {
if (edge.lanes != LaneQuantifier::All) return false;
const auto candidate = UnsignedBound(edge.condition, key, edge.positive);
const auto candidate = UnsignedBound(edge.condition, key, edge.positive, finite_bitscan);
if (candidate.IsEmpty() ||
!ValidateRuntimeValue(m_program, candidate, RuntimeValueType::Integer)) return false;
if (bound.IsEmpty()) bound = candidate;
@@ -2438,11 +2456,8 @@ private:
} else if (table_source.dword_count == 2u) {
auto guarded_count = UnsignedBoundOnEntry(key, handle.Parent());
if (guarded_count.IsEmpty()) guarded_count = UnsignedImageBound(key, handle);
const auto* selector = key.Resolve().TryInstruction();
const bool bitscan = guarded_count.IsEmpty() && selector != nullptr &&
selector->GetOpcode() == ValueOpcode::FindILsb32 &&
selector->NumArgs() == 1u && !m_shader_writes &&
NonzeroOnEntry(selector->Arg(0), handle.Parent());
// A runtime count can describe fewer rows than the mask's static bit range.
if (guarded_count.IsEmpty()) guarded_count = UnsignedBoundOnEntry(key, handle.Parent(), true);
if (!guarded_count.IsEmpty()) {
if (table_stride < 32u || (table_stride & 3u) != 0u) return false;
// Materialization captures the count and descriptor reads; renderer alias
@@ -2450,8 +2465,6 @@ private:
indirect.key_count = guarded_count;
} else if (table_stride != 32u) {
return false;
} else if (bitscan) {
indirect.key_count = Value(32u);
} else if (!m_shader_writes) {
if (const auto* bound = BoundedLoop(key, handle.Parent())) {
indirect.key_count = bound->Arg(1);
@@ -2463,8 +2476,7 @@ private:
!MatchReversedImageRange(key, indirect)) {
return false;
}
if ((table_offset & 3u) != 0u ||
(bitscan && table_offset > UINT32_MAX - (32u * 32u - 1u))) return false;
if ((table_offset & 3u) != 0u) return false;
} else {
// A bounded V# supplies the complete image table. Leave every GPU selector
// and descriptor read in the shader; the host only translates table bytes.
@@ -2665,6 +2677,16 @@ private:
return SelectorMaximum(inst->Arg(arg), bounds, depth + 1u, guard);
};
switch (inst->GetOpcode()) {
case ValueOpcode::Phi: {
const auto initial = InitialCandidateMask(value);
return initial.IsEmpty() ? UINT32_MAX
: SelectorMaximum(initial, bounds, depth + 1u, guard);
}
case ValueOpcode::BitFieldUExtract: {
uint32_t width;
if (ImmediateU32(inst->Arg(2), width) && width < 32u) return (1u << width) - 1u;
break;
}
case ValueOpcode::FindILsb32:
case ValueOpcode::FindUMsb32:
return (!guard.IsEmpty() && ImpliesNonzero(guard, inst->Arg(0))) ||
tests/ResourceTrackingTests.cpp
+93 -14
View File
@@ -565,26 +565,28 @@ void TestGuardedDirectImageTable() {
Nonzero, SccNonZero, Plain, Zero, Unrelated, Bypass, ExecZero, VccZero,
Bounded, BoundMissing, BoundUnrelated, BoundBypass, BoundAnyLane,
BoundDiamond, BoundBranchSpecific, BoundMismatched,
BoundExec, BoundExecUnguarded, BoundExecMismatched, BoundExecAnd, BoundExecUnprovenActive
BoundExec, BoundExecUnguarded, BoundExecMismatched, BoundExecAnd, BoundExecUnprovenActive,
Finite, FiniteZero, FiniteMissing, FiniteWrong, FiniteSigned, FiniteAdd, FiniteXor
};
const auto make_plan = [](Guard guard) {
const bool bounded = guard >= Guard::Bounded;
const bool exec_bound = guard >= Guard::BoundExec;
const bool finite = guard >= Guard::Finite;
const bool bounded = guard >= Guard::Bounded && !finite;
const bool exec_bound = bounded && guard >= Guard::BoundExec;
const bool bypass = guard == Guard::Bypass || guard == Guard::BoundBypass;
const uint32_t table_offset = exec_bound ? 672u : bounded ? 3696u : 344u;
const uint32_t table_offset = finite ? 3344u : exec_bound ? 672u : bounded ? 3696u : 344u;
const uint32_t table_stride = exec_bound ? 784u : bounded ? 272u : 32u;
Fixture fixture(bounded ? ShaderType::Compute : ShaderType::Pixel);
Fixture fixture(bounded || finite ? ShaderType::Compute : ShaderType::Pixel);
fixture.program.wave_size = 64u;
auto *entry = fixture.block;
auto *middle = fixture.AddBlock();
auto *before_sample = fixture.AddBlock();
auto *sample = fixture.AddBlock();
auto *exit = fixture.AddBlock();
const auto varying = bounded
const auto varying = bounded || finite
? fixture.Emit(ValueOpcode::GetBuiltin,
{Value(static_cast<uint32_t>(StageInputKind::LocalInvocationId)), Value(0u)})
: fixture.Emit(ValueOpcode::GetAttribute, {Value(0u), Value(0u)});
const auto mask = fixture.Emit(ValueOpcode::ReadFirstLane, {varying, Value(true)});
auto mask = fixture.Emit(ValueOpcode::ReadFirstLane, {varying, Value(true)});
const auto srt = fixture.Address(fixture.UserData(0), fixture.UserData(1));
Value key;
Value limit;
@@ -596,13 +598,44 @@ void TestGuardedDirectImageTable() {
limit = fixture.Emit(ValueOpcode::LoadAddressU32,
{srt, Value(0u), Value(0u), Value(true)}, fixture.AddMemory(memory, 0x10));
}
Block *latch = nullptr;
if (finite) {
latch = fixture.AddBlock();
MemoryInfo memory;
memory.kind = ResourceKind::ScalarBuffer;
const auto input = fixture.Buffer({Value(0x900000u), Value(0u), Value(4096u), Value(0u)});
const auto loaded = fixture.Emit(ValueOpcode::ReadConstBuffer, {input, mask},
fixture.AddMemory(memory, 0x2ecu));
const auto initial = fixture.Emit(ValueOpcode::BitFieldUExtract,
{loaded, Value(0u), Value(guard == Guard::FiniteZero ? 0u : 12u)});
auto &phi = middle->AppendNewInst(ValueOpcode::Phi, {}, static_cast<uint64_t>(Type::U32));
mask = Value(&phi);
key = fixture.Emit(ValueOpcode::FindILsb32, {mask}, 0, middle);
const auto bit = fixture.Emit(ValueOpcode::ShiftLeftLogical32,
{Value(1u), fixture.Emit(ValueOpcode::BitwiseAnd32, {key, Value(31u)}, 0, latch)}, 0, latch);
const auto next = guard == Guard::FiniteAdd
? fixture.Emit(ValueOpcode::BitwiseOr32, {mask, Value(1u << 31u)}, 0, latch)
: guard == Guard::FiniteXor
? fixture.Emit(ValueOpcode::BitwiseXor32, {mask, bit}, 0, latch)
: fixture.Emit(ValueOpcode::BitwiseAnd32,
{mask, fixture.Emit(ValueOpcode::BitwiseNot32, {bit}, 0, latch)}, 0, latch);
phi.AddPhiOperand(entry, initial);
phi.AddPhiOperand(latch, next);
}
const auto kind = guard == Guard::ExecZero || guard == Guard::BoundAnyLane || exec_bound
? CFG::BranchCondition::ExecZero
: guard == Guard::VccZero ? CFG::BranchCondition::VccZero
: CFG::BranchCondition::SccZero;
Value condition;
Value active(true);
if (bounded) {
if (finite) {
condition = guard == Guard::FiniteMissing ? Value(false)
: guard == Guard::FiniteSigned
? fixture.Emit(ValueOpcode::LogicalNot,
{fixture.Emit(ValueOpcode::SLessThan32, {key, Value(12u)}, 0, middle)}, 0, middle)
: fixture.Emit(ValueOpcode::IEqual32,
{key, Value(guard == Guard::FiniteWrong ? 31u : UINT32_MAX)}, 0, middle);
} else if (bounded) {
condition = guard == Guard::BoundMissing ? Value(false) : fixture.Emit(
ValueOpcode::UGreaterThanEqual32,
{guard == Guard::BoundUnrelated ? fixture.UserData(2) : key, limit});
@@ -630,7 +663,7 @@ void TestGuardedDirectImageTable() {
condition = fixture.Emit(ValueOpcode::LogicalNot, {condition});
}
if (guard != Guard::Plain && guard != Guard::SccNonZero) {
condition = fixture.Emit(ValueOpcode::ConditionRef, {condition}, kind);
condition = fixture.Emit(ValueOpcode::ConditionRef, {condition}, kind, finite ? middle : entry);
}
fixture.program.blocks[0]->condition = condition;
fixture.program.blocks[0]->terminator = {
@@ -645,6 +678,18 @@ void TestGuardedDirectImageTable() {
fixture.program.blocks[4]->terminator = {
.kind = bypass ? CFG::TerminatorKind::Branch : CFG::TerminatorKind::Return,
.true_block = fixture.program.blocks[3u]};
if (finite) {
entry->condition = {};
entry->terminator = {.kind = CFG::TerminatorKind::Branch, .true_block = middle};
middle->condition = condition;
middle->terminator = {.kind = CFG::TerminatorKind::ConditionalBranch,
.true_block = guard == Guard::FiniteXor ? latch : exit, .false_block = before_sample};
before_sample->condition = fixture.Emit(ValueOpcode::IEqual32, {varying, Value(0u)}, 0, before_sample);
before_sample->terminator = {.kind = CFG::TerminatorKind::ConditionalBranch,
.true_block = sample, .false_block = latch};
sample->terminator = {.kind = CFG::TerminatorKind::Branch, .true_block = latch};
latch->terminator = {.kind = CFG::TerminatorKind::Branch, .true_block = middle};
}
if (guard == Guard::BoundDiamond || exec_bound) {
middle->condition = fixture.Emit(ValueOpcode::IEqual32,
{mask, Value(1u)});
@@ -673,7 +718,7 @@ void TestGuardedDirectImageTable() {
}
std::array<Value, 2> pointer;
for (uint32_t word = 0; word < pointer.size(); ++word) {
if (bounded) {
if (bounded || finite) {
pointer[word] = srt.Instruction()->Arg(word);
continue;
}
@@ -685,7 +730,7 @@ void TestGuardedDirectImageTable() {
fixture.AddMemory(memory, 0x20));
}
fixture.block = sample;
if (!bounded) key = fixture.Emit(ValueOpcode::FindILsb32, {mask});
if (!bounded && !finite) key = fixture.Emit(ValueOpcode::FindILsb32, {mask});
const auto offset = fixture.Emit(ValueOpcode::IAdd32,
{bounded ? fixture.Emit(ValueOpcode::IMul32, {key, Value(table_stride)})
: fixture.Emit(ValueOpcode::ShiftLeftLogical32, {key, Value(5u)}),
@@ -719,7 +764,7 @@ void TestGuardedDirectImageTable() {
fixture.Emit(ValueOpcode::ImageSampleRaw,
{image, sampler, fixture.ImageAddress(), other_active}, fixture.AddMemory(memory, 0x12c));
}
if (bounded) {
if (bounded || finite) {
const auto output = fixture.Image({Value(0x8000u),
Value(static_cast<uint32_t>(Libs::Graphics::Prospero::BufferFormat::k32UInt) << 20u),
Value(0u), Value((static_cast<uint32_t>(Libs::Graphics::Prospero::ImageType::kColor2D) << 28u) |
@@ -751,7 +796,8 @@ void TestGuardedDirectImageTable() {
Check(indirect && !indirect->selector.has_value() &&
indirect->table_offset == table_offset && indirect->table_stride == table_stride &&
(bounded ? !indirect->key_count.IsEmpty()
: indirect->key_count.Resolve() == Value(32u)) &&
: indirect->key_count.Resolve() == Value(guard == Guard::FiniteZero ? 0u :
finite && guard != Guard::FiniteAdd && guard != Guard::FiniteXor ? 12u : 32u)) &&
fixture.program.descriptor_sources[indirect->table_source].dword_count == 2u,
"guarded direct image table lost its pointer or proven selector range");
return ExtractResourcePlan(fixture.program);
@@ -765,7 +811,8 @@ void TestGuardedDirectImageTable() {
Guard::BoundUnrelated, Guard::BoundBypass, Guard::BoundAnyLane,
Guard::BoundBranchSpecific, Guard::BoundMismatched,
Guard::BoundExecUnguarded, Guard::BoundExecMismatched,
Guard::BoundExecAnd, Guard::BoundExecUnprovenActive}) {
Guard::BoundExecAnd, Guard::BoundExecUnprovenActive,
Guard::FiniteMissing, Guard::FiniteWrong, Guard::FiniteSigned}) {
CheckFatal([&] { make_plan(guard); }, "not a valid runtime value",
"direct table accepted a selector without a dominating range guard");
}
@@ -892,6 +939,38 @@ void TestGuardedDirectImageTable() {
records.descriptor_reads == 0u,
"unrepresentable table span was read before validating its count");
}
for (const auto guard : {Guard::Finite, Guard::FiniteZero, Guard::FiniteAdd, Guard::FiniteXor}) {
const auto finite = make_plan(guard);
Check(finite.capture_specialization_reads && finite.info.images[1].written,
"finite bit table with image writes omitted specialization read capture");
LinearTestMemory records;
const uint32_t count = guard == Guard::FiniteZero ? 0u : guard == Guard::Finite ? 12u : 32u;
const uint64_t first = records.base + 3344u;
fill_table(records, first);
// The GPU may select no bits or a sparse subset. Null rows remain valid candidates.
std::fill_n(records.words.begin() + (first - records.base) / 4u + 5u * 8u, 8u, 0u);
records.fail_address = first + count * 32u;
user_data = {static_cast<uint32_t>(records.base), 0u};
runtime.userdata = &records;
Check(MaterializeResources(finite, runtime, snapshot, specialization) &&
records.descriptor_reads == count && snapshot.images.size() == std::max(count, 1u) + 1u,
"finite bit table probed beyond its mask bound or lost a null row");
if (count == 0u) {
Check(std::ranges::all_of(snapshot.images[0].dwords, [](uint32_t word) { return word == 0u; }),
"empty bit loop retained a reachable image");
continue;
}
Check(snapshot.flattened_srt[specialization.images[0].indirect_mapping_offset] == count,
"finite bit table lost its candidate count");
for (uint32_t key = 0; key < count; ++key) {
Check(std::ranges::any_of(snapshot.specialization_reads, [&](const auto &read) {
return read.first == first + key * 32u && read.second == 32u;
}), "finite bit table omitted a descriptor from writable-alias validation");
}
records.fail_address = first + (count - 1u) * 32u + 28u;
Check(!MaterializeResources(finite, runtime, snapshot, specialization),
"finite bit table accepted an unreadable last candidate");
}
}
void TestBoundedComputeImageLoop() {
tests/ShaderRecompilerComputeTests.cpp
+53 -34
View File
@@ -36492,18 +36492,22 @@ void CheckCountedRowImageSelector(VulkanHarness &vulkan) {
void CheckGuardedRecordImageTable(VulkanHarness &vulkan) {
using O = ShaderOpcode;
constexpr u32 table = 256u;
for (const bool masked : {false, true})
for (const u32 variant : {0u, 1u, 2u})
for (const u32 wave : {32u, 64u})
for (const u32 count : {0u, 2u, 3u}) {
const u32 stride = masked ? 784u : 272u, offset = masked ? 672u : 3696u;
const bool masked = variant == 1u, bitset = variant == 2u;
const u32 stride = bitset ? 32u : masked ? 784u : 272u;
const u32 offset = bitset ? 3344u : masked ? 672u : 3696u;
const u32 rows = bitset ? 12u : count, members = bitset ? 1u : 2u;
const u32 lanes = masked ? 2u : 1u;
TestCase test;
test.name = masked ? (wave == 32u ? "ExecGuardedRecordTable32" : "ExecGuardedRecordTable64")
test.name = bitset ? (wave == 32u ? "BitsetRecordTable32" : "BitsetRecordTable64")
: masked ? (wave == 32u ? "ExecGuardedRecordTable32" : "ExecGuardedRecordTable64")
: (wave == 32u ? "GuardedRecordImageTable32" : "GuardedRecordImageTable64");
test.initial.resize((table + offset + stride * count) / 4u, 0xdeadbeefu);
test.initial[table / 4u + (masked ? 0u : 1u)] = count;
for (u32 row = 0; row < count; ++row)
for (const u32 member : {0u, 1u}) {
test.initial.resize((table + offset + stride * rows) / 4u, 0xdeadbeefu);
test.initial[table / 4u + (masked ? 0u : 1u)] = bitset ? (1u << count) - 1u : count;
for (u32 row = 0; row < rows; ++row)
for (u32 member = 0; member < members; ++member) {
auto image = MakeSampledTextureData(Prospero::BufferFormat::k32Float);
image[0] = (row + 1u) * (member == 0u ? 1u : 10u);
image[3] = DstSel(4, 0, 0, 1) |
@@ -36530,6 +36534,9 @@ void CheckGuardedRecordImageTable(VulkanHarness &vulkan) {
auto &code = test.code;
code.insert(code.end(), {EncodeSop2(0x20, 20, InlineU32(5), 44),
EncodeSmem0(0x00, 21), EncodeSmem1(masked ? 0u : 4u, 125)});
if (bitset) code.insert(code.end(), {
EncodeSop2(0x0e, 20, 20, 21),
EncodeSop2(0x27, 20, 20, 255), 12u << 16u});
code.push_back(EncodeSop1(0x04, 30, 126)); // Preserve EXEC for the final write.
code.push_back(EncodeVop1(0x01, 13, Vgpr(0))); // Odd lane is already done.
AppendVMovLiteral(&code, 10, 0xdeadbeefu);
@@ -36540,33 +36547,41 @@ void CheckGuardedRecordImageTable(VulkanHarness &vulkan) {
code.push_back(EncodeSop1(0x13, 22, 20)); // Find the next set bit in the GPU worklist.
code.push_back(EncodeSopc(0x07, 22, 193));
const auto empty_exit = code.size(); code.push_back(0u);
if (masked) code.insert(code.end(), {EncodeSop2(0x0b, 28, 126, InlineU32(0)),
EncodeSop1(0x04, 126, 28)});
code.push_back(EncodeSopc(0x09, 22, 21));
if (masked) code.insert(code.end(), {
EncodeVop2(0x1c, 10, 253, 13), // SCC encoded as 0/1, OR per-lane done.
EncodeVopc(0xc5, InlineU32(0), 10),
EncodeSop2(0x15, 126, 28, 106)});
const auto range_exit = code.size(); code.push_back(0u);
size_t range_exit = 0;
if (!bitset) {
if (masked) code.insert(code.end(), {EncodeSop2(0x0b, 28, 126, InlineU32(0)),
EncodeSop1(0x04, 126, 28)});
code.push_back(EncodeSopc(0x09, 22, 21));
if (masked) code.insert(code.end(), {
EncodeVop2(0x1c, 10, 253, 13), // SCC encoded as 0/1, OR per-lane done.
EncodeVopc(0xc5, InlineU32(0), 10),
EncodeSop2(0x15, 126, 28, 106)});
range_exit = code.size(); code.push_back(0u);
}
code.insert(code.end(), {EncodeSop2(0x26, 23, 22, 255), stride,
EncodeSop2(0x02, 23, 23, 255), offset,
EncodeSopc(0x0d, 22, InlineU32(0))});
const auto skip_first = code.size(); code.push_back(0u);
code.insert(code.end(), {
EncodeSmem0(0x03, 8), EncodeSmem1(0, 23),
EncodeSop2(0x02, 23, 23, 255), offset});
size_t skip_first = 0;
if (!bitset) {
code.push_back(EncodeSopc(0x0d, 22, InlineU32(0)));
skip_first = code.size(); code.push_back(0u);
}
code.insert(code.end(), {EncodeSmem0(0x03, 8), EncodeSmem1(0, 23),
EncodeMimg0(0x27, 1), EncodeMimg1(4, 20, 2, 10),
EncodeVop2(0x03, 0, Vgpr(4), 0)});
code[skip_first] = EncodeSopp(0x05, code.size() - skip_first - 1u);
// The second descriptor is reached through both sides of the optional first sample.
code.insert(code.end(), {EncodeSop2(0x02, 23, 23, InlineU32(32)),
EncodeSmem0(0x03, 8), EncodeSmem1(0, 23),
EncodeMimg0(0x27, 1), EncodeMimg1(4, 20, 2, 10),
EncodeVop2(0x03, 0, Vgpr(4), 0),
EncodeSop2(0x1e, 24, InlineU32(1), 22),
if (!bitset) {
code[skip_first] = EncodeSopp(0x05, code.size() - skip_first - 1u);
// Both sides of the optional first sample reach the second descriptor.
code.insert(code.end(), {EncodeSop2(0x02, 23, 23, InlineU32(32)),
EncodeSmem0(0x03, 8), EncodeSmem1(0, 23),
EncodeMimg0(0x27, 1), EncodeMimg1(4, 20, 2, 10),
EncodeVop2(0x03, 0, Vgpr(4), 0)});
}
code.insert(code.end(), {EncodeSop2(0x1e, 24, InlineU32(1), 22),
EncodeSop2(0x14, 20, 20, 24)});
code.push_back(EncodeSopp(0x02, loop - code.size() - 1u));
code[empty_exit] = EncodeSopp(0x04, code.size() - empty_exit - 1u);
code[range_exit] = EncodeSopp(masked ? 0x08 : 0x05, code.size() - range_exit - 1u);
if (!bitset) code[range_exit] =
EncodeSopp(masked ? 0x08 : 0x05, code.size() - range_exit - 1u);
code.push_back(EncodeSop1(0x04, 126, 30));
code.push_back(EncodeSop2(0x26, 25, 44, InlineU32(lanes)));
code.push_back(EncodeVop2(0x25, 30, 25, 13));
@@ -36574,10 +36589,13 @@ void CheckGuardedRecordImageTable(VulkanHarness &vulkan) {
code.insert(code.end(), {EncodeMimg0(0x08, 1), EncodeMimg1(0, 30, 8, 0)});
AppendEnd(&code);
test.opcodes = {O::S_LSHR_B32, O::S_LOAD_DWORD, O::V_MOV_B32, O::S_FF1_I32_B32,
O::S_CMP_LG_U32, O::S_CMP_GE_U32, O::S_CBRANCH_SCC0, O::S_CBRANCH_SCC1,
O::S_MUL_I32, O::S_ADD_I32, O::S_BITCMP1_B32, O::S_LOAD_DWORDX8, O::IMAGE_SAMPLE,
O::S_CMP_LG_U32, O::S_CBRANCH_SCC0,
O::S_MUL_I32, O::S_ADD_I32, O::S_LOAD_DWORDX8, O::IMAGE_SAMPLE,
O::V_ADD_F32, O::S_LSHL_B32, O::S_ANDN2_B32, O::S_BRANCH,
O::IMAGE_STORE, O::S_ENDPGM};
if (bitset) test.opcodes.insert(test.opcodes.end(), {O::S_AND_B32, O::S_BFE_U32});
else test.opcodes.insert(test.opcodes.end(), {O::S_CMP_GE_U32, O::S_CBRANCH_SCC1,
O::S_BITCMP1_B32});
if (masked) test.opcodes.insert(test.opcodes.end(), {O::S_CSELECT_B64,
O::V_OR_B32, O::V_CMP_NE_U32, O::S_ANDN2_B64, O::S_CBRANCH_EXECZ});
test.opcodes.insert(test.opcodes.end(), {O::S_MOV_B64, O::V_ADD_NC_U32});
@@ -36587,7 +36605,7 @@ void CheckGuardedRecordImageTable(VulkanHarness &vulkan) {
return image.resource_class == ShaderRecompiler::IR::ImageResourceClass::Sampled;
});
Require(test.name, "bounded descriptor rows",
count == 0u ? sampled_count <= 2u : sampled_count == count * 2u,
rows == 0u ? sampled_count <= members : sampled_count == rows * members,
"the runtime count lost a row or probed record padding");
std::vector<VulkanHarness::Image> images(compiled.program.info.images.size());
for (u32 resource = 0; resource < images.size(); ++resource) {
@@ -36606,9 +36624,10 @@ void CheckGuardedRecordImageTable(VulkanHarness &vulkan) {
vulkan.Dispatch(test, compiled, buffer, nullptr, nullptr, &storage, nullptr, sampler, images);
std::vector<u32> expected(4u * lanes, 0u);
if (count != 0u) {
expected[0] = std::bit_cast<u32>(count == 3 ? 44.0f : 11.0f);
expected[lanes] = std::bit_cast<u32>(20.0f);
expected[2u * lanes] = std::bit_cast<u32>(11.0f);
expected[0] = std::bit_cast<u32>(bitset ? (count == 3 ? 4.0f : 1.0f)
: (count == 3 ? 44.0f : 11.0f));
expected[lanes] = std::bit_cast<u32>(bitset ? 2.0f : 20.0f);
expected[2u * lanes] = std::bit_cast<u32>(bitset ? 1.0f : 11.0f);
}
CompareWords(test, "GPU worklist, guarded count and image writes", expected,
vulkan.ReadImage(test.name, &storage));