mirror of
https://github.com/KytyPS5/KytyPS5.git
synced 2026-10-21 19:36:55 +02:00
shader: materialize image keys from proven GPU bitmap subsets
This commit is contained in:
nmzik
@@ -507,6 +507,8 @@ struct DescriptorSource {
|
||||
uint32_t source = UINT32_MAX;
|
||||
uint32_t stride = 0;
|
||||
uint32_t offset = 0;
|
||||
// Four DWORD bitmap words per record; set-bit ranks subtract from the last key.
|
||||
bool bitset = false;
|
||||
std::vector<std::pair<uint32_t, uint32_t>> byte_ranges;
|
||||
|
||||
bool operator==(const SelectorRead& other) const = default;
|
||||
@@ -521,7 +523,7 @@ struct DescriptorSource {
|
||||
bool indexed_store = false;
|
||||
uint32_t workgroup_axis = UINT32_MAX;
|
||||
Value key_count;
|
||||
// First selector index, or first descriptor key when no selector is present.
|
||||
// First selector index, or first descriptor key for a direct or bitmap range.
|
||||
Value selector_first;
|
||||
Value selector_mask;
|
||||
std::vector<uint32_t> sources;
|
||||
|
||||
@@ -338,6 +338,33 @@ bool MaterializeIndirectDescriptor(const ResourcePlan& p
|
||||
std::iota(keys.begin(), keys.end(), first_key);
|
||||
if (!keys.empty() && keys.back() < keys.front())
|
||||
std::rotate(keys.begin(), keys.begin() + (UINT32_MAX - first_key + 1ull), keys.end());
|
||||
} else if (selector->bitset) {
|
||||
ShaderBufferResource material;
|
||||
std::array<uint32_t, 4> masks {};
|
||||
uint32_t first = 0, count = 0;
|
||||
if (!DecodeBufferDescriptor(material_value, material) ||
|
||||
!clean.Evaluate(indirect.selector_first, first) ||
|
||||
!clean.Evaluate(indirect.key_count, count) || count != masks.size() * 32u ||
|
||||
selector->stride != sizeof(masks) || selector->offset != 0u ||
|
||||
material.Type() != 0u || material.Stride() != sizeof(masks) ||
|
||||
material.SwizzleEnabled() || material.AddTid() || material.OutOfBounds() != 0u ||
|
||||
(material.Base48() & 3u) != 0u ||
|
||||
material.GetSize() / sizeof(uint32_t) > MaxIndirectDescriptorProbes) return false;
|
||||
if (material.Format() != Prospero::BufferFormat::kInvalid) {
|
||||
keys.resize(material.GetSize() / sizeof(uint32_t));
|
||||
if (!keys.empty() && !ReadScalarTable(material.Base48(), material.GetSize(), 0u,
|
||||
runtime, keys)) return false;
|
||||
for (size_t word = 0; word < keys.size(); ++word) masks[word % masks.size()] |= keys[word];
|
||||
keys.clear();
|
||||
for (uint32_t word = 0; word < masks.size(); ++word) {
|
||||
auto mask = masks[word];
|
||||
while (mask != 0u) {
|
||||
const auto rank = word * 32u + std::countr_zero(mask);
|
||||
keys.push_back(first + count - 1u - rank);
|
||||
mask &= mask - 1u;
|
||||
}
|
||||
}
|
||||
}
|
||||
} else if (!selector->byte_ranges.empty()) {
|
||||
if (material_value.dword_count != 2u) return false;
|
||||
const auto base = (uint64_t {material_value.dwords[1]} << 32u) | material_value.dwords[0];
|
||||
|
||||
@@ -1202,20 +1202,24 @@ private:
|
||||
(Implies(inst->Arg(0), required) || Implies(inst->Arg(1), required));
|
||||
}
|
||||
|
||||
Value EqualLocalKey(Value guard, Value key) const {
|
||||
Value EqualLocalValue(Value guard, const auto& matches) const {
|
||||
guard = SimplifyGuard(guard);
|
||||
const auto* inst = guard.TryInstruction();
|
||||
if (inst == nullptr) return {};
|
||||
if (inst->GetOpcode() == ValueOpcode::LogicalAnd) {
|
||||
const auto left = EqualLocalKey(inst->Arg(0), key);
|
||||
return left.IsEmpty() ? EqualLocalKey(inst->Arg(1), key) : left;
|
||||
const auto left = EqualLocalValue(inst->Arg(0), matches);
|
||||
return left.IsEmpty() ? EqualLocalValue(inst->Arg(1), matches) : left;
|
||||
}
|
||||
if (inst->GetOpcode() != ValueOpcode::IEqual32 || inst->NumArgs() != 2u) return {};
|
||||
if (EquivalentValue(m_program, inst->Arg(0), key)) return inst->Arg(1).Resolve();
|
||||
if (EquivalentValue(m_program, inst->Arg(1), key)) return inst->Arg(0).Resolve();
|
||||
if (matches(inst->Arg(0))) return inst->Arg(1).Resolve();
|
||||
if (matches(inst->Arg(1))) return inst->Arg(0).Resolve();
|
||||
return {};
|
||||
}
|
||||
|
||||
Value EqualLocalKey(Value guard, Value key) const {
|
||||
return EqualLocalValue(guard, [&](Value at) { return EquivalentValue(m_program, at, key); });
|
||||
}
|
||||
|
||||
struct AffineOffset {
|
||||
Value index;
|
||||
uint64_t stride = 0;
|
||||
@@ -1977,6 +1981,216 @@ private:
|
||||
return false;
|
||||
}
|
||||
|
||||
using SelectorBounds = std::span<const std::pair<const Inst*, uint32_t>>;
|
||||
|
||||
// Each equation is an inductive subset obligation. Phis require every incoming
|
||||
// value; AND needs either operand. Grounding rejects cycles without an initial
|
||||
// value, then the greatest fixed point checks every initial and carried edge.
|
||||
bool BitsetMaskSubset(Value mask, Value active, Value word, const Inst& read) const {
|
||||
struct State {
|
||||
Value value;
|
||||
Value guard;
|
||||
uint32_t word = UINT32_MAX;
|
||||
uint32_t component = UINT32_MAX; // U1 implication of the load guard.
|
||||
bool operator==(const State&) const = default;
|
||||
};
|
||||
struct Equation {
|
||||
bool all = true;
|
||||
std::vector<size_t> terms;
|
||||
};
|
||||
const auto load_guard = read.Arg(4).Resolve();
|
||||
std::vector<State> states(2);
|
||||
std::vector<Equation> equations(2);
|
||||
const auto add = [&](State state) {
|
||||
state.value = state.value.Resolve();
|
||||
const auto found = std::find(states.begin() + 2u, states.end(), state);
|
||||
if (found != states.end()) return static_cast<size_t>(found - states.begin());
|
||||
states.push_back(state);
|
||||
equations.emplace_back();
|
||||
return states.size() - 1u;
|
||||
};
|
||||
std::array<size_t, 5> roots {add({.value = active})};
|
||||
for (uint32_t component = 0; component < 4u; ++component)
|
||||
roots[component + 1u] = add({mask, active, component, component});
|
||||
for (size_t index = 2; index < states.size(); ++index) {
|
||||
const auto state = states[index];
|
||||
const auto* inst = state.value.TryInstruction();
|
||||
Equation equation;
|
||||
const auto term = [&](Value value, Value guard, uint32_t selected) {
|
||||
return add({value, guard, selected, state.component});
|
||||
};
|
||||
if (state.component == UINT32_MAX) {
|
||||
if (Implies(state.value, load_guard)) equation.terms = {1u};
|
||||
else if (inst != nullptr && (inst->GetOpcode() == ValueOpcode::LogicalAnd ||
|
||||
inst->GetOpcode() == ValueOpcode::Phi)) {
|
||||
equation.all = inst->GetOpcode() == ValueOpcode::Phi;
|
||||
for (size_t arg = 0; arg < inst->NumArgs(); ++arg)
|
||||
equation.terms.push_back(add({.value = inst->Arg(arg)}));
|
||||
}
|
||||
} else if (state.value == Value(0u)) {
|
||||
equation.terms = {1u};
|
||||
} else if (inst != nullptr) {
|
||||
const auto constant = [&](const auto& self, Value value) -> std::optional<uint32_t> {
|
||||
uint32_t literal;
|
||||
if (ImmediateU32(value, literal)) return literal;
|
||||
if (state.word != UINT32_MAX && EquivalentValue(m_program, value, word)) return state.word;
|
||||
const auto* operation = value.Resolve().TryInstruction();
|
||||
if (operation == nullptr || operation->GetOpcode() != ValueOpcode::BitwiseAnd32) return {};
|
||||
const auto left = self(self, operation->Arg(0));
|
||||
const auto right = self(self, operation->Arg(1));
|
||||
return left && right ? std::optional {*left & *right} : std::nullopt;
|
||||
};
|
||||
const auto condition = [&](const auto& self, Value value) -> std::optional<bool> {
|
||||
value = value.Resolve();
|
||||
if (value.IsImmediate() && value.GetType() == Type::U1) return value.U1();
|
||||
if (Implies(state.guard, value) || Implies(load_guard, value)) return true;
|
||||
const auto* test = value.TryInstruction();
|
||||
if (test == nullptr) return {};
|
||||
if (test->GetOpcode() == ValueOpcode::LogicalAnd) {
|
||||
const auto left = self(self, test->Arg(0));
|
||||
const auto right = self(self, test->Arg(1));
|
||||
if ((left && !*left) || (right && !*right)) return false;
|
||||
if (left && right) return *left && *right;
|
||||
} else if (test->GetOpcode() == ValueOpcode::IEqual32) {
|
||||
const auto left = constant(constant, test->Arg(0));
|
||||
const auto right = constant(constant, test->Arg(1));
|
||||
if (left && right) return *left == *right;
|
||||
}
|
||||
return {};
|
||||
};
|
||||
switch (inst->GetOpcode()) {
|
||||
case ValueOpcode::CompositeExtractU32x4: {
|
||||
uint32_t component;
|
||||
if (inst->Arg(0).Resolve().TryInstruction() == &read &&
|
||||
ImmediateU32(inst->Arg(1), component) && component == state.component)
|
||||
equation.terms = {1u};
|
||||
break;
|
||||
}
|
||||
case ValueOpcode::Phi:
|
||||
// The predecessor may be an earlier outer iteration. Establish
|
||||
// this register's invariant without the current word or EXEC.
|
||||
for (size_t arg = 0; arg < inst->NumArgs(); ++arg)
|
||||
equation.terms.push_back(term(inst->Arg(arg), load_guard, UINT32_MAX));
|
||||
break;
|
||||
case ValueOpcode::BitwiseAnd32:
|
||||
equation.all = false;
|
||||
equation.terms = {term(inst->Arg(0), state.guard, state.word),
|
||||
term(inst->Arg(1), state.guard, state.word)};
|
||||
break;
|
||||
case ValueOpcode::SelectU32: {
|
||||
const auto selected = condition(condition, inst->Arg(0));
|
||||
if (!selected || *selected) {
|
||||
auto selected_word = state.word;
|
||||
const auto equality = EqualLocalValue(inst->Arg(0), [&](Value key) {
|
||||
key = key.Resolve();
|
||||
const auto* masked = key.TryInstruction();
|
||||
uint32_t bound = 0;
|
||||
if (masked != nullptr && masked->GetOpcode() == ValueOpcode::BitwiseAnd32 &&
|
||||
DescendingLoop(word, masked->Parent(), bound) && bound == 3u) {
|
||||
for (uint32_t side = 0; side < 2u; ++side) {
|
||||
uint32_t bits;
|
||||
if (ImmediateU32(masked->Arg(side), bits) && (bits & 3u) == 3u)
|
||||
key = masked->Arg(side ^ 1u).Resolve();
|
||||
}
|
||||
}
|
||||
return EquivalentValue(m_program, key, word);
|
||||
});
|
||||
uint32_t literal;
|
||||
if (ImmediateU32(equality, literal) && literal < 4u) selected_word = literal;
|
||||
equation.terms.push_back(term(inst->Arg(1), inst->Arg(0), selected_word));
|
||||
}
|
||||
if (!selected || !*selected)
|
||||
equation.terms.push_back(term(inst->Arg(2), state.guard, state.word));
|
||||
break;
|
||||
}
|
||||
default: break;
|
||||
}
|
||||
}
|
||||
if (equation.terms.empty()) equation.terms = {0u};
|
||||
equations[index] = std::move(equation);
|
||||
}
|
||||
std::vector<bool> valid(states.size(), false);
|
||||
valid[1] = true;
|
||||
bool changed;
|
||||
do {
|
||||
changed = false;
|
||||
for (size_t index = 2; index < equations.size(); ++index) {
|
||||
if (!valid[index] && std::ranges::any_of(equations[index].terms,
|
||||
[&](size_t term) { return valid[term]; })) valid[index] = changed = true;
|
||||
}
|
||||
} while (changed);
|
||||
do {
|
||||
changed = false;
|
||||
for (size_t index = 2; index < equations.size(); ++index) {
|
||||
const auto& equation = equations[index];
|
||||
const auto proven = [&](size_t term) { return valid[term]; };
|
||||
if (valid[index] && !(equation.all ? std::ranges::all_of(equation.terms, proven)
|
||||
: std::ranges::any_of(equation.terms, proven))) {
|
||||
valid[index] = false;
|
||||
changed = true;
|
||||
}
|
||||
}
|
||||
} while (changed);
|
||||
return std::ranges::all_of(roots, [&](size_t root) { return valid[root]; });
|
||||
}
|
||||
|
||||
bool MatchBitsetSelector(Value rank, Value active, SelectorBounds bounds,
|
||||
DescriptorSource::IndirectDescriptor& indirect) {
|
||||
const auto selected_value = [&](Value value) {
|
||||
for (;;) {
|
||||
value = ActiveValue(value, active);
|
||||
const auto* select = value.TryInstruction();
|
||||
if (select == nullptr || select->GetOpcode() != ValueOpcode::SelectU32 ||
|
||||
!SelectorPredicateFalse(select->Arg(0), bounds, 0u, active)) return value;
|
||||
value = select->Arg(2);
|
||||
}
|
||||
};
|
||||
const auto* sum = selected_value(rank).TryInstruction();
|
||||
if (sum == nullptr || sum->GetOpcode() != ValueOpcode::IAdd32) return false;
|
||||
Value word;
|
||||
Value bit;
|
||||
for (uint32_t arg = 0; arg < 2u; ++arg) {
|
||||
const auto* shift = selected_value(sum->Arg(arg)).TryInstruction();
|
||||
if (shift == nullptr || shift->GetOpcode() != ValueOpcode::ShiftLeftLogical32 ||
|
||||
shift->Arg(1).Resolve() != Value(5u)) continue;
|
||||
word = selected_value(shift->Arg(0));
|
||||
bit = selected_value(sum->Arg(arg ^ 1u));
|
||||
}
|
||||
const auto* word_inst = word.TryInstruction();
|
||||
if (word_inst == nullptr || std::ranges::none_of(bounds, [&](const auto& bound) {
|
||||
return bound.first == word_inst && bound.second == 3u;
|
||||
})) return false;
|
||||
// FFBH's leading-zero count is inverted again to recover the set-bit rank.
|
||||
for (uint32_t inverse = 0; inverse < 2u; ++inverse) {
|
||||
const auto* subtract = bit.TryInstruction();
|
||||
if (subtract == nullptr || subtract->GetOpcode() != ValueOpcode::ISub32 ||
|
||||
subtract->Arg(0).Resolve() != Value(31u)) return false;
|
||||
bit = selected_value(subtract->Arg(1));
|
||||
}
|
||||
const auto* scan = bit.TryInstruction();
|
||||
if (scan == nullptr || scan->GetOpcode() != ValueOpcode::FindUMsb32 ||
|
||||
!ImpliesNonzero(active, scan->Arg(0))) return false;
|
||||
for (const auto* block: m_program.blocks) {
|
||||
for (const auto& read: *block) {
|
||||
if (read.GetOpcode() != ValueOpcode::LoadBufferU32x4 || InCycle(block)) continue;
|
||||
const auto index = read.Flags<MemoryFlags>().index;
|
||||
if (index >= m_program.memory_info.size()) continue;
|
||||
const auto& memory = m_program.memory_info[index];
|
||||
if (memory.kind != ResourceKind::Buffer || memory.typed || memory.formatted ||
|
||||
!memory.idxen || memory.offen || memory.offset != 0u || memory.data_bits != 32u ||
|
||||
memory.data_dwords != 4u || read.Arg(2).Resolve() != Value(0u) ||
|
||||
read.Arg(3).Resolve() != Value(0u) ||
|
||||
!BitsetMaskSubset(scan->Arg(0), active, word, read)) continue;
|
||||
DescriptorSource source;
|
||||
if (!MakeRuntimeTableSource(read, source)) continue;
|
||||
indirect.selector.emplace(DescriptorSource::IndirectDescriptor::SelectorRead {
|
||||
.source = InternSource(source), .stride = 16u, .offset = 0u, .bitset = true});
|
||||
return true;
|
||||
}
|
||||
}
|
||||
return false;
|
||||
}
|
||||
|
||||
bool MatchReversedImageRange(Value key, DescriptorSource::IndirectDescriptor& indirect) {
|
||||
const auto* subtract = key.Resolve().TryInstruction();
|
||||
if (subtract == nullptr || subtract->GetOpcode() != ValueOpcode::ISub32 ||
|
||||
@@ -2000,6 +2214,7 @@ private:
|
||||
first.SetArg(1, Value(maximum));
|
||||
indirect.selector_first = Value(&first);
|
||||
indirect.key_count = Value(maximum + 1u);
|
||||
if (maximum == 127u) MatchBitsetSelector(lane->Arg(0), lane->Arg(1), bounds, indirect);
|
||||
return true;
|
||||
}
|
||||
|
||||
@@ -2300,7 +2515,6 @@ private:
|
||||
return finish();
|
||||
}
|
||||
|
||||
using SelectorBounds = std::span<const std::pair<const Inst*, uint32_t>>;
|
||||
|
||||
bool SelectorPredicateFalse(Value value, SelectorBounds bounds, uint32_t depth = 0,
|
||||
Value guard = {}) const {
|
||||
|
||||
@@ -1096,9 +1096,14 @@ void TestBoundedComputeImageLoop() {
|
||||
|
||||
void TestDescendingImageRange() {
|
||||
namespace CFG = Libs::Graphics::ShaderRecompiler::CFG;
|
||||
enum class Variant { Valid, WrongStep, IncrementBypass, MissingNonzero, NegativeInitial };
|
||||
enum class Variant {
|
||||
Valid, WrongStep, IncrementBypass, MissingNonzero, NegativeInitial,
|
||||
Bitmap, BitmapAddsBit, BitmapWrongWord, BitmapUnknownInitial,
|
||||
BitmapByteOffset, BitmapScalarOffset, BitmapOuterClobber, BitmapWrongLoadMask
|
||||
};
|
||||
const auto make_plan = [](Variant variant) {
|
||||
Fixture fixture;
|
||||
const bool bitmap = variant >= Variant::Bitmap;
|
||||
auto *entry = fixture.block;
|
||||
auto *header = fixture.AddBlock();
|
||||
auto *guard = fixture.AddBlock();
|
||||
@@ -1117,21 +1122,69 @@ void TestDescendingImageRange() {
|
||||
.true_block = yes, .false_block = no};
|
||||
};
|
||||
branch(entry, header);
|
||||
branch(body, latch);
|
||||
branch(body, bitmap ? guard : latch);
|
||||
branch(latch, header);
|
||||
auto &induction = header->AppendNewInst(ValueOpcode::Phi);
|
||||
induction.SetFlags(Type::U32);
|
||||
std::array<Inst *, 4> outer_words {}, remaining_words {};
|
||||
std::array<Value, 4> loaded_words {};
|
||||
if (bitmap)
|
||||
for (auto& word : outer_words)
|
||||
word = &header->AppendNewInst(ValueOpcode::Phi, {}, uint64_t(Type::U32));
|
||||
const auto bound = fixture.Emit(ValueOpcode::SGreaterThanEqual32,
|
||||
{Value(&induction), Value(0u)}, 0, header);
|
||||
choose(header, guard, variant == Variant::IncrementBypass ? latch : exit, bound);
|
||||
MemoryInfo lds;
|
||||
lds.kind = ResourceKind::Lds;
|
||||
const auto mask = fixture.Emit(ValueOpcode::LoadSharedU32,
|
||||
{Value(0u), Value(true)}, fixture.AddMemory(lds, 0x10), guard);
|
||||
Value mask;
|
||||
Value load_active(true);
|
||||
if (bitmap) {
|
||||
fixture.block = entry;
|
||||
const auto lane = fixture.Emit(ValueOpcode::LaneId);
|
||||
load_active = fixture.Emit(ValueOpcode::ULessThan32, {lane, Value(32u)});
|
||||
const auto buffer = fixture.Buffer({fixture.UserData(4), fixture.UserData(5),
|
||||
fixture.UserData(6), fixture.UserData(7)});
|
||||
MemoryInfo read;
|
||||
read.kind = ResourceKind::Buffer;
|
||||
read.idxen = true;
|
||||
read.offen = variant == Variant::BitmapByteOffset;
|
||||
read.data_dwords = 4u;
|
||||
const auto packed = fixture.Emit(ValueOpcode::LoadBufferU32x4,
|
||||
{buffer, lane,
|
||||
Value(variant == Variant::BitmapByteOffset ? 4u : 0u),
|
||||
Value(variant == Variant::BitmapScalarOffset ? 4u : 0u), load_active},
|
||||
fixture.AddMemory(read, 0x10));
|
||||
for (uint32_t word = 0; word < 4u; ++word) {
|
||||
auto component = fixture.Emit(ValueOpcode::CompositeExtractU32x4,
|
||||
{packed, Value(variant == Variant::BitmapWrongWord ? (word + 1u) % 4u : word)}, 0, entry);
|
||||
component = fixture.Emit(ValueOpcode::SelectU32,
|
||||
{load_active, component, fixture.Emit(ValueOpcode::UndefU32)});
|
||||
loaded_words[word] = component;
|
||||
if (variant == Variant::BitmapUnknownInitial && word == 0u)
|
||||
component = fixture.Emit(ValueOpcode::SelectU32,
|
||||
{fixture.Emit(ValueOpcode::IEqual32, {fixture.Emit(ValueOpcode::LaneId), Value(0u)}),
|
||||
component, fixture.UserData(3)});
|
||||
outer_words[word]->AddPhiOperand(entry, component);
|
||||
remaining_words[word] = &guard->AppendNewInst(ValueOpcode::Phi, {}, uint64_t(Type::U32));
|
||||
remaining_words[word]->AddPhiOperand(header, Value(outer_words[word]));
|
||||
}
|
||||
auto selected = fixture.Emit(ValueOpcode::UndefU32, {}, 0, guard);
|
||||
for (uint32_t word = 0; word < 4u; ++word) {
|
||||
selected = fixture.Emit(ValueOpcode::SelectU32,
|
||||
{fixture.Emit(ValueOpcode::IEqual32, {Value(&induction), Value(word)}, 0, guard),
|
||||
Value(remaining_words[word]), selected}, 0, guard);
|
||||
}
|
||||
mask = selected;
|
||||
} else {
|
||||
MemoryInfo lds;
|
||||
lds.kind = ResourceKind::Lds;
|
||||
mask = fixture.Emit(ValueOpcode::LoadSharedU32,
|
||||
{Value(0u), Value(true)}, fixture.AddMemory(lds, 0x10), guard);
|
||||
}
|
||||
auto active = bound;
|
||||
if (variant != Variant::MissingNonzero)
|
||||
active = fixture.Emit(ValueOpcode::LogicalAnd,
|
||||
{bound, fixture.Emit(ValueOpcode::ULessThan32, {Value(0u), mask}, 0, guard)}, 0, guard);
|
||||
if (bitmap && variant != Variant::BitmapWrongLoadMask)
|
||||
active = fixture.Emit(ValueOpcode::LogicalAnd, {load_active, active}, 0, guard);
|
||||
choose(guard, body, latch, active);
|
||||
fixture.block = body;
|
||||
const auto stale = fixture.UserData(3);
|
||||
@@ -1171,6 +1224,26 @@ void TestDescendingImageRange() {
|
||||
fixture.Emit(ValueOpcode::ImageSampleRaw,
|
||||
{fixture.Image(words), fixture.Sampler({Value(0u), Value(0u), Value(0u), Value(0u)}),
|
||||
fixture.ImageAddress(), active}, fixture.AddMemory(sample, 0x30));
|
||||
if (bitmap) {
|
||||
auto next = fixture.Emit(ValueOpcode::BitwiseAnd32,
|
||||
{mask, fixture.Emit(ValueOpcode::BitwiseNot32,
|
||||
{fixture.Emit(ValueOpcode::ShiftLeftLogical32, {Value(1u), bit})})});
|
||||
if (variant == Variant::BitmapAddsBit)
|
||||
next = fixture.Emit(ValueOpcode::BitwiseOr32, {next, Value(1u << 21u)});
|
||||
for (uint32_t word = 0; word < 4u; ++word) {
|
||||
const auto selected_word = fixture.Emit(ValueOpcode::LogicalAnd,
|
||||
{active, fixture.Emit(ValueOpcode::IEqual32, {Value(&induction), Value(word)})});
|
||||
remaining_words[word]->AddPhiOperand(body, fixture.Emit(ValueOpcode::SelectU32,
|
||||
{selected_word, next, Value(remaining_words[word])}));
|
||||
auto carried = Value(remaining_words[word]);
|
||||
if (variant == Variant::BitmapOuterClobber)
|
||||
carried = fixture.Emit(ValueOpcode::SelectU32,
|
||||
{fixture.Emit(ValueOpcode::IEqual32,
|
||||
{Value(&induction), Value(word + 1u)}, 0, latch),
|
||||
loaded_words[(word + 1u) % 4u], carried}, 0, latch);
|
||||
outer_words[word]->AddPhiOperand(latch, carried);
|
||||
}
|
||||
}
|
||||
const auto step = fixture.Emit(ValueOpcode::IAdd32,
|
||||
{Value(&induction), Value(variant == Variant::WrongStep ? UINT32_MAX - 1u : UINT32_MAX)}, 0, latch);
|
||||
induction.AddPhiOperand(entry, Value(variant == Variant::NegativeInitial ? UINT32_MAX : 3u));
|
||||
@@ -1180,9 +1253,11 @@ void TestDescendingImageRange() {
|
||||
"descending range retained impossible relative-register stores");
|
||||
const auto &indirect = fixture.program.descriptor_sources[
|
||||
fixture.program.info.images[0].source].indirect_descriptor;
|
||||
Check(indirect && !indirect->selector && indirect->key_count == Value(128u) &&
|
||||
Check(indirect && indirect->selector.has_value() == (variant == Variant::Bitmap) &&
|
||||
(!indirect->selector || indirect->selector->bitset) &&
|
||||
indirect->key_count == Value(128u) &&
|
||||
!indirect->selector_first.IsEmpty(),
|
||||
"descending bitscan range lost its runtime first key or 128-key bound");
|
||||
"descending bitscan range lost its bound or accepted an unproved bitmap subset");
|
||||
return ExtractResourcePlan(fixture.program);
|
||||
};
|
||||
auto plan = make_plan(Variant::Valid);
|
||||
@@ -1218,6 +1293,116 @@ void TestDescendingImageRange() {
|
||||
previous = key;
|
||||
}
|
||||
}
|
||||
|
||||
auto bitmap_plan = make_plan(Variant::Bitmap);
|
||||
for (const auto variant : {Variant::BitmapAddsBit, Variant::BitmapWrongWord,
|
||||
Variant::BitmapUnknownInitial, Variant::BitmapByteOffset,
|
||||
Variant::BitmapScalarOffset, Variant::BitmapOuterClobber,
|
||||
Variant::BitmapWrongLoadMask})
|
||||
make_plan(variant);
|
||||
// Every descriptor is distinct, so retained keys are visible in the snapshot as well as
|
||||
// its lookup table. An unrelated table slot must never be read, even when it is malformed.
|
||||
for (uint32_t record = 1u; record <= 130u; ++record)
|
||||
memory.words[record * 8u] = 0x100u + record;
|
||||
constexpr uint32_t bitmap_address = 0x4000u;
|
||||
const auto bitmap_word = (bitmap_address - memory.base) / 4u;
|
||||
memory.words.resize(bitmap_word + MaxIndirectDescriptorProbes + 4u, 0u);
|
||||
std::array<uint32_t, 8> bitmap_data {0x1000u, 0u, 0u, UINT32_MAX,
|
||||
bitmap_address, 16u << 16u, 3u, 0x5204u};
|
||||
const SrtRuntime bitmap_runtime {.user_data = bitmap_data, .userdata = &memory,
|
||||
.read_specialization_memory = ReadLinearTestMemory};
|
||||
memory.watched_address = bitmap_address;
|
||||
const auto check_bitmap = [&](std::initializer_list<uint32_t> ranks, uint32_t count) {
|
||||
bitmap_data[2] = count;
|
||||
memory.watched_reads = memory.descriptor_reads = 0u;
|
||||
std::vector<uint32_t> keys;
|
||||
for (const auto rank : ranks) keys.push_back(count - rank);
|
||||
std::ranges::sort(keys);
|
||||
Check(MaterializeResources(bitmap_plan, bitmap_runtime, snapshot, specialization) &&
|
||||
memory.watched_reads == uint32_t(bitmap_data[6] != 0u) &&
|
||||
(bitmap_data[6] == 0u || memory.watched_dwords == bitmap_data[6] * 4u) &&
|
||||
memory.descriptor_reads == keys.size() &&
|
||||
snapshot.images.size() == std::max<size_t>(keys.size(), 1u),
|
||||
"bitmap selector did not read one bounded snapshot or retained unreachable descriptors");
|
||||
Check(bitmap_data[6] == 0u || std::ranges::find(snapshot.specialization_reads,
|
||||
std::pair{uint64_t{bitmap_address}, uint64_t{bitmap_data[6]} * 16u}) !=
|
||||
snapshot.specialization_reads.end(),
|
||||
"bitmap snapshot was omitted from same-dispatch write-alias protection");
|
||||
if (keys.empty()) {
|
||||
Check(std::ranges::all_of(snapshot.images[0].dwords, [](uint32_t value) { return value == 0u; }) &&
|
||||
specialization.images[0].indirect_root == ImageResource::NoIndirectImage,
|
||||
"empty bitmap retained a previously materialized image");
|
||||
return;
|
||||
}
|
||||
if (keys.size() == 1u) {
|
||||
Check(snapshot.images[0].dwords[0] == 0x100u + uint32_t(0x1000u + keys[0] * 32u) / 32u &&
|
||||
specialization.images[0].indirect_root == ImageResource::NoIndirectImage,
|
||||
"a single bitmap rank became empty or required an indirect image mapping");
|
||||
return;
|
||||
}
|
||||
const auto offset = specialization.images[0].indirect_mapping_offset;
|
||||
Check(snapshot.flattened_srt[offset] == keys.size(), "bitmap mapping lost a selected key");
|
||||
for (uint32_t index = 0; index < keys.size(); ++index) {
|
||||
const auto descriptor_record = uint32_t(0x1000u + keys[index] * 32u) / 32u;
|
||||
Check(snapshot.flattened_srt[offset + 1u + index * 2u] == keys[index] &&
|
||||
snapshot.images[index].dwords[0] == 0x100u + descriptor_record,
|
||||
"bitmap word association or U32-wrapped key selected a different descriptor");
|
||||
}
|
||||
};
|
||||
memory.fail_address = 0x2000u - 21u * 32u;
|
||||
check_bitmap({}, 0u);
|
||||
memory.words[bitmap_word] = 1u;
|
||||
check_bitmap({0u}, 0u);
|
||||
memory.words[bitmap_word + 1u] = 1u << 3u;
|
||||
memory.words[bitmap_word + 4u] = 1u; // Duplicate rank in another record.
|
||||
memory.words[bitmap_word + 6u] = 1u << 31u;
|
||||
memory.words[bitmap_word + 11u] = 1u << 1u;
|
||||
check_bitmap({0u, 35u, 95u, 97u}, 0u);
|
||||
check_bitmap({0u, 35u, 95u, 97u}, 2u);
|
||||
std::fill_n(memory.words.begin() + bitmap_word, 12u, 0u);
|
||||
for (uint32_t word = 0; word < 4u; ++word) {
|
||||
memory.words[bitmap_word + word] = 1u << 31u;
|
||||
memory.words[bitmap_word + 4u + word] = 1u;
|
||||
}
|
||||
check_bitmap({0u, 31u, 32u, 63u, 64u, 95u, 96u, 127u}, 0u);
|
||||
std::fill_n(memory.words.begin() + bitmap_word, 12u, 0u);
|
||||
memory.words[bitmap_word + 3u] = 1u << 31u;
|
||||
memory.words[bitmap_word + 8u] = 1u << 2u;
|
||||
check_bitmap({2u, 127u}, 0u);
|
||||
std::fill_n(memory.words.begin() + bitmap_word, 12u, 0u);
|
||||
check_bitmap({}, 0u);
|
||||
memory.fail_address = bitmap_address + 16u;
|
||||
Check(!MaterializeResources(bitmap_plan, bitmap_runtime, snapshot, specialization),
|
||||
"bitmap selector used a stale or partial snapshot after a failed coherent read");
|
||||
memory.fail_address = UINT64_MAX;
|
||||
|
||||
const auto valid_data = bitmap_data;
|
||||
for (const auto [word, value] : std::array<std::pair<uint32_t, uint32_t>, 7>{{
|
||||
{4u, bitmap_address + 1u}, {5u, 8u << 16u}, {5u, (16u << 16u) | (1u << 31u)},
|
||||
{7u, 0x5204u | (1u << 23u)}, {7u, 0x5204u | (1u << 28u)},
|
||||
{7u, 0x5204u | (1u << 30u)},
|
||||
{6u, MaxIndirectDescriptorProbes / 4u + 1u}}}) {
|
||||
bitmap_data[word] = value;
|
||||
Check(!MaterializeResources(bitmap_plan, bitmap_runtime, snapshot, specialization),
|
||||
"bitmap selector accepted an unproved buffer layout or exceeded its read bound");
|
||||
bitmap_data = valid_data;
|
||||
}
|
||||
bitmap_data[7] &= ~(0x7fu << 12u); // FORMAT=INVALID makes even an unformatted load unbound.
|
||||
memory.words[bitmap_word] = 1u;
|
||||
memory.watched_reads = memory.descriptor_reads = 0u;
|
||||
Check(MaterializeResources(bitmap_plan, bitmap_runtime, snapshot, specialization) &&
|
||||
memory.watched_reads == 0u && memory.descriptor_reads == 0u &&
|
||||
snapshot.images.size() == 1u &&
|
||||
std::ranges::all_of(snapshot.images[0].dwords, [](uint32_t value) { return value == 0u; }),
|
||||
"unbound bitmap V# read backing bytes or manufactured an image key");
|
||||
bitmap_data = valid_data;
|
||||
memory.words[bitmap_word] = 0u;
|
||||
bitmap_data[6] = 0u;
|
||||
check_bitmap({}, 0u);
|
||||
bitmap_data[6] = MaxIndirectDescriptorProbes / 4u;
|
||||
memory.words[bitmap_word + MaxIndirectDescriptorProbes - 1u] = 1u << 31u;
|
||||
memory.words[bitmap_word] = 1u;
|
||||
check_bitmap({0u, 127u}, 0u);
|
||||
}
|
||||
|
||||
void TestMixedIndirectImageSamplers() {
|
||||
|
||||
Reference in New Issue
Block a user