mirror of
https://github.com/KytyPS5/KytyPS5.git
synced 2026-10-20 10:56:58 +02:00
Lower finite GPU-selected image descriptors through shared indirect bindings
This commit is contained in:
@@ -617,7 +617,6 @@ TranslateResult TranslateProgram(std::span<const uint32_t> code, const CompileOp
|
|||||||
}
|
}
|
||||||
}
|
}
|
||||||
IR::TrackResources(ir, decoded, native_cfg);
|
IR::TrackResources(ir, decoded, native_cfg);
|
||||||
IR::EliminateDeadCode(ir.blocks);
|
|
||||||
TranslateResult result;
|
TranslateResult result;
|
||||||
result.program = std::move(ir);
|
result.program = std::move(ir);
|
||||||
if (options.dump_ir) {
|
if (options.dump_ir) {
|
||||||
|
|||||||
@@ -476,6 +476,7 @@ struct DescriptorSource {
|
|||||||
uint32_t table_offset = 0;
|
uint32_t table_offset = 0;
|
||||||
Value key_count;
|
Value key_count;
|
||||||
Value selector_mask;
|
Value selector_mask;
|
||||||
|
std::vector<uint32_t> sources;
|
||||||
|
|
||||||
bool operator==(const IndirectImage& other) const = default;
|
bool operator==(const IndirectImage& other) const = default;
|
||||||
};
|
};
|
||||||
|
|||||||
@@ -227,100 +227,114 @@ bool ReadScalarTable(uint64_t base, uint64_t size, uint64_t dynamic_offset,
|
|||||||
|
|
||||||
bool MaterializeIndirectImage(const ResourcePlan& program,
|
bool MaterializeIndirectImage(const ResourcePlan& program,
|
||||||
const DescriptorSource::IndirectImage& indirect,
|
const DescriptorSource::IndirectImage& indirect,
|
||||||
const DescriptorValue& material_value,
|
uint32_t image_index,
|
||||||
const DescriptorValue& table_value, uint32_t image_index,
|
|
||||||
const SrtRuntime& runtime, SrtWalker& clean,
|
const SrtRuntime& runtime, SrtWalker& clean,
|
||||||
ResourceSnapshot& snapshot,
|
ResourceSnapshot& snapshot,
|
||||||
ResourceSpecialization& specialization) {
|
ResourceSpecialization& specialization) {
|
||||||
|
const auto& sources = indirect.sources;
|
||||||
|
auto& keys = program.material_keys;
|
||||||
uint64_t table_base = 0;
|
uint64_t table_base = 0;
|
||||||
uint64_t table_size = UINT64_MAX; // Scalar addresses have no buffer descriptor bounds.
|
uint64_t table_size = UINT64_MAX; // Scalar addresses have no buffer descriptor bounds.
|
||||||
ShaderBufferResource table;
|
if (sources.empty()) {
|
||||||
if (table_value.dword_count == 2u) {
|
keys.clear();
|
||||||
table_base = (static_cast<uint64_t>(table_value.dwords[1]) << 32u) | table_value.dwords[0];
|
DescriptorValue material_value;
|
||||||
} else if (DecodeBufferDescriptor(table_value, table)) {
|
DescriptorValue table_value;
|
||||||
table_base = table.Base48();
|
if ((indirect.material_source != UINT32_MAX &&
|
||||||
table_size = table.GetSize();
|
!clean.EvaluateDescriptor(indirect.material_source, material_value)) ||
|
||||||
} else {
|
!clean.EvaluateDescriptor(indirect.table_source, table_value)) {
|
||||||
return false;
|
|
||||||
}
|
|
||||||
auto& keys = program.material_keys;
|
|
||||||
keys.clear();
|
|
||||||
if (indirect.material_source == UINT32_MAX) {
|
|
||||||
uint32_t key_count = 0;
|
|
||||||
const bool evaluated = clean.Evaluate(indirect.key_count, key_count);
|
|
||||||
if (std::bit_cast<int32_t>(key_count) <= 0) key_count = 0;
|
|
||||||
if (table_value.dword_count != 2u || !evaluated ||
|
|
||||||
key_count > MaxIndirectImageProbes ||
|
|
||||||
uint64_t {indirect.table_offset} + uint64_t {key_count} * 32u > UINT32_MAX + 1ull) {
|
|
||||||
return false;
|
return false;
|
||||||
}
|
}
|
||||||
keys.resize(key_count);
|
ShaderBufferResource table;
|
||||||
std::iota(keys.begin(), keys.end(), 0u);
|
if (table_value.dword_count == 2u) {
|
||||||
} else if (!indirect.selector_mask.IsEmpty()) {
|
table_base = (static_cast<uint64_t>(table_value.dwords[1]) << 32u) | table_value.dwords[0];
|
||||||
uint32_t mask = 0;
|
} else if (DecodeBufferDescriptor(table_value, table)) {
|
||||||
uint32_t count = 0;
|
table_base = table.Base48();
|
||||||
if (material_value.dword_count != 2u || table_value.dword_count != 2u ||
|
table_size = table.GetSize();
|
||||||
!clean.Evaluate(indirect.selector_mask, mask) ||
|
} else {
|
||||||
!clean.Evaluate(indirect.key_count, count) || count == 0u || count > 32u) {
|
|
||||||
return false;
|
return false;
|
||||||
}
|
}
|
||||||
if (count < 32u) mask &= (1u << count) - 1u;
|
if (indirect.material_source == UINT32_MAX) {
|
||||||
const auto material_base =
|
uint32_t key_count = 0;
|
||||||
(static_cast<uint64_t>(material_value.dwords[1]) << 32u) | material_value.dwords[0];
|
const bool evaluated = clean.Evaluate(indirect.key_count, key_count);
|
||||||
keys.reserve(std::popcount(mask));
|
if (std::bit_cast<int32_t>(key_count) <= 0) key_count = 0;
|
||||||
while (mask != 0u) {
|
if (table_value.dword_count != 2u || !evaluated ||
|
||||||
const auto index = std::countr_zero(mask);
|
key_count > MaxIndirectImageProbes ||
|
||||||
const auto offset = static_cast<uint64_t>(indirect.selector_offset) +
|
uint64_t {indirect.table_offset} + uint64_t {key_count} * 32u > UINT32_MAX + 1ull) {
|
||||||
static_cast<uint64_t>(index) * indirect.selector_stride;
|
|
||||||
if (offset > UINT32_MAX) return false;
|
|
||||||
uint32_t key = 0;
|
|
||||||
if (!ReadScalarTable(material_base, UINT64_MAX, static_cast<uint32_t>(offset),
|
|
||||||
runtime, {&key, 1})) return false;
|
|
||||||
keys.push_back(key);
|
|
||||||
mask &= mask - 1u;
|
|
||||||
}
|
|
||||||
std::ranges::sort(keys);
|
|
||||||
keys.erase(std::unique(keys.begin(), keys.end()), keys.end());
|
|
||||||
} else {
|
|
||||||
ShaderBufferResource material;
|
|
||||||
if (!DecodeBufferDescriptor(material_value, material) || table_value.dword_count != 4u ||
|
|
||||||
material.Stride() != indirect.selector_stride) {
|
|
||||||
return false;
|
|
||||||
}
|
|
||||||
// The first aligned offset includes the immediate added after shader U32 arithmetic.
|
|
||||||
const auto step = std::max<uint64_t>(4u,
|
|
||||||
std::gcd<uint64_t>(indirect.selector_stride, uint64_t {1} << 32u));
|
|
||||||
const uint64_t first = indirect.selector_offset;
|
|
||||||
const auto size = material.GetSize();
|
|
||||||
const auto limit = std::min(first + (uint64_t {1} << 32u) - step, size >= 4u ? size - 4u : 0u);
|
|
||||||
const auto probe_count = size >= 4u && first <= limit ? (limit - first) / step + 1u : 0u;
|
|
||||||
if (probe_count > MaxIndirectImageProbes) {
|
|
||||||
return false;
|
|
||||||
}
|
|
||||||
keys.reserve(static_cast<size_t>(probe_count) + 1u);
|
|
||||||
keys.push_back(0u);
|
|
||||||
for (uint64_t probe = 0, offset = first; probe < probe_count; ++probe, offset += step) {
|
|
||||||
uint32_t key = 0;
|
|
||||||
if (!ReadScalarTable(material.Base48(), size, offset, runtime, {&key, 1})) {
|
|
||||||
return false;
|
return false;
|
||||||
}
|
}
|
||||||
keys.push_back(key);
|
keys.resize(key_count);
|
||||||
|
std::iota(keys.begin(), keys.end(), 0u);
|
||||||
|
} else if (!indirect.selector_mask.IsEmpty()) {
|
||||||
|
uint32_t mask = 0;
|
||||||
|
uint32_t count = 0;
|
||||||
|
if (material_value.dword_count != 2u || table_value.dword_count != 2u ||
|
||||||
|
!clean.Evaluate(indirect.selector_mask, mask) ||
|
||||||
|
!clean.Evaluate(indirect.key_count, count) || count == 0u || count > 32u) {
|
||||||
|
return false;
|
||||||
|
}
|
||||||
|
if (count < 32u) mask &= (1u << count) - 1u;
|
||||||
|
const auto material_base =
|
||||||
|
(static_cast<uint64_t>(material_value.dwords[1]) << 32u) | material_value.dwords[0];
|
||||||
|
keys.reserve(std::popcount(mask));
|
||||||
|
while (mask != 0u) {
|
||||||
|
const auto index = std::countr_zero(mask);
|
||||||
|
const auto offset = static_cast<uint64_t>(indirect.selector_offset) +
|
||||||
|
static_cast<uint64_t>(index) * indirect.selector_stride;
|
||||||
|
if (offset > UINT32_MAX) return false;
|
||||||
|
uint32_t key = 0;
|
||||||
|
if (!ReadScalarTable(material_base, UINT64_MAX, static_cast<uint32_t>(offset),
|
||||||
|
runtime, {&key, 1})) return false;
|
||||||
|
keys.push_back(key);
|
||||||
|
mask &= mask - 1u;
|
||||||
|
}
|
||||||
|
std::ranges::sort(keys);
|
||||||
|
keys.erase(std::unique(keys.begin(), keys.end()), keys.end());
|
||||||
|
} else {
|
||||||
|
ShaderBufferResource material;
|
||||||
|
if (!DecodeBufferDescriptor(material_value, material) || table_value.dword_count != 4u ||
|
||||||
|
material.Stride() != indirect.selector_stride) {
|
||||||
|
return false;
|
||||||
|
}
|
||||||
|
// The first aligned offset includes the immediate added after shader U32 arithmetic.
|
||||||
|
const auto step = std::max<uint64_t>(4u,
|
||||||
|
std::gcd<uint64_t>(indirect.selector_stride, uint64_t {1} << 32u));
|
||||||
|
const uint64_t first = indirect.selector_offset;
|
||||||
|
const auto size = material.GetSize();
|
||||||
|
const auto limit = std::min(first + (uint64_t {1} << 32u) - step, size >= 4u ? size - 4u : 0u);
|
||||||
|
const auto probe_count = size >= 4u && first <= limit ? (limit - first) / step + 1u : 0u;
|
||||||
|
if (probe_count > MaxIndirectImageProbes) {
|
||||||
|
return false;
|
||||||
|
}
|
||||||
|
keys.reserve(static_cast<size_t>(probe_count) + 1u);
|
||||||
|
keys.push_back(0u);
|
||||||
|
for (uint64_t probe = 0, offset = first; probe < probe_count; ++probe, offset += step) {
|
||||||
|
uint32_t key = 0;
|
||||||
|
if (!ReadScalarTable(material.Base48(), size, offset, runtime, {&key, 1})) {
|
||||||
|
return false;
|
||||||
|
}
|
||||||
|
keys.push_back(key);
|
||||||
|
}
|
||||||
|
std::ranges::sort(keys);
|
||||||
|
keys.erase(std::unique(keys.begin(), keys.end()), keys.end());
|
||||||
}
|
}
|
||||||
std::ranges::sort(keys);
|
|
||||||
keys.erase(std::unique(keys.begin(), keys.end()), keys.end());
|
|
||||||
}
|
}
|
||||||
|
|
||||||
const auto children_begin = snapshot.images.size();
|
const auto children_begin = snapshot.images.size();
|
||||||
const auto mapping_offset = snapshot.flattened_srt.size();
|
const auto mapping_offset = snapshot.flattened_srt.size();
|
||||||
const auto root_image = specialization.images[image_index];
|
const auto root_image = specialization.images[image_index];
|
||||||
snapshot.flattened_srt.resize(mapping_offset + 1u + keys.size() * 2u);
|
const auto key_count = sources.empty() ? keys.size() : sources.size();
|
||||||
snapshot.flattened_srt[mapping_offset] = static_cast<uint32_t>(keys.size());
|
snapshot.flattened_srt.resize(mapping_offset + 1u + key_count * 2u);
|
||||||
for (uint32_t entry = 0; entry < keys.size(); ++entry) {
|
snapshot.flattened_srt[mapping_offset] = static_cast<uint32_t>(key_count);
|
||||||
const auto key = keys[entry];
|
for (uint32_t entry = 0; entry < key_count; ++entry) {
|
||||||
|
const auto key = sources.empty() ? keys[entry] : entry;
|
||||||
DescriptorValue candidate;
|
DescriptorValue candidate;
|
||||||
candidate.dword_count = 8u;
|
candidate.dword_count = 8u;
|
||||||
const auto table_offset = (key << 5u) + indirect.table_offset;
|
if (sources.empty()) {
|
||||||
if (!ReadScalarTable(table_base, table_size, table_offset, runtime, candidate.dwords)) {
|
const auto table_offset = (key << 5u) + indirect.table_offset;
|
||||||
|
if (!ReadScalarTable(table_base, table_size, table_offset, runtime, candidate.dwords)) {
|
||||||
|
return false;
|
||||||
|
}
|
||||||
|
} else if (!clean.EvaluateDescriptor(sources[entry], candidate)) {
|
||||||
return false;
|
return false;
|
||||||
}
|
}
|
||||||
if (NullImageDescriptor(candidate) ||
|
if (NullImageDescriptor(candidate) ||
|
||||||
@@ -353,7 +367,7 @@ bool MaterializeIndirectImage(const ResourcePlan& program,
|
|||||||
auto& root = specialization.images[image_index];
|
auto& root = specialization.images[image_index];
|
||||||
root.indirect_root = image_index;
|
root.indirect_root = image_index;
|
||||||
root.indirect_mapping_offset = static_cast<uint32_t>(mapping_offset);
|
root.indirect_mapping_offset = static_cast<uint32_t>(mapping_offset);
|
||||||
root.indirect_search_iterations = std::bit_width(keys.size());
|
root.indirect_search_iterations = std::bit_width(key_count);
|
||||||
}
|
}
|
||||||
return true;
|
return true;
|
||||||
}
|
}
|
||||||
@@ -936,20 +950,27 @@ ResourcePlan ExtractResourcePlan(const Program& program) {
|
|||||||
plan.uniform_fill.values[i] = Clone(plan.uniform_fill.values[i]);
|
plan.uniform_fill.values[i] = Clone(plan.uniform_fill.values[i]);
|
||||||
}
|
}
|
||||||
plan.clean_flat_slots.resize(plan.srt_reads.size());
|
plan.clean_flat_slots.resize(plan.srt_reads.size());
|
||||||
bool masked_image = false;
|
bool capture_image_reads = false;
|
||||||
for (const auto& image: plan.info.images) {
|
for (const auto& image: plan.info.images) {
|
||||||
const auto* source = Source(plan, image.source);
|
const auto* source = Source(plan, image.source);
|
||||||
if (source == nullptr || !source->indirect_image.has_value()) {
|
if (source == nullptr || !source->indirect_image.has_value()) {
|
||||||
continue;
|
continue;
|
||||||
}
|
}
|
||||||
plan.requires_specialization_memory = true;
|
plan.requires_specialization_memory = true;
|
||||||
masked_image |= !source->indirect_image->selector_mask.IsEmpty();
|
capture_image_reads |= !source->indirect_image->selector_mask.IsEmpty() ||
|
||||||
|
!source->indirect_image->sources.empty();
|
||||||
MarkCleanFlatSlots(plan, Source(plan, source->indirect_image->material_source),
|
MarkCleanFlatSlots(plan, Source(plan, source->indirect_image->material_source),
|
||||||
plan.clean_flat_slots, source->indirect_image->selector_mask);
|
plan.clean_flat_slots, source->indirect_image->selector_mask);
|
||||||
MarkCleanFlatSlots(plan, Source(plan, source->indirect_image->table_source),
|
if (source->indirect_image->sources.empty()) {
|
||||||
plan.clean_flat_slots);
|
MarkCleanFlatSlots(plan, Source(plan, source->indirect_image->table_source),
|
||||||
|
plan.clean_flat_slots);
|
||||||
|
} else {
|
||||||
|
for (const auto candidate: source->indirect_image->sources) {
|
||||||
|
MarkCleanFlatSlots(plan, Source(plan, candidate), plan.clean_flat_slots);
|
||||||
|
}
|
||||||
|
}
|
||||||
}
|
}
|
||||||
plan.capture_specialization_reads = masked_image || !plan.control_flow.empty();
|
plan.capture_specialization_reads = capture_image_reads || !plan.control_flow.empty();
|
||||||
// Writable descriptor addresses must come from clean backing for the alias proof.
|
// Writable descriptor addresses must come from clean backing for the alias proof.
|
||||||
for (const auto& buffer: plan.info.buffers) {
|
for (const auto& buffer: plan.info.buffers) {
|
||||||
if (buffer.written) MarkCleanFlatSlots(plan, Source(plan, buffer.source), plan.clean_flat_slots);
|
if (buffer.written) MarkCleanFlatSlots(plan, Source(plan, buffer.source), plan.clean_flat_slots);
|
||||||
@@ -957,7 +978,7 @@ ResourcePlan ExtractResourcePlan(const Program& program) {
|
|||||||
for (const auto& image: plan.info.images) {
|
for (const auto& image: plan.info.images) {
|
||||||
if (image.written) MarkCleanFlatSlots(plan, Source(plan, image.source), plan.clean_flat_slots);
|
if (image.written) MarkCleanFlatSlots(plan, Source(plan, image.source), plan.clean_flat_slots);
|
||||||
}
|
}
|
||||||
if (masked_image) plan.resource_tracking_complete &= !program.has_address_writes;
|
if (capture_image_reads) plan.resource_tracking_complete &= !program.has_address_writes;
|
||||||
return plan;
|
return plan;
|
||||||
}
|
}
|
||||||
|
|
||||||
@@ -1039,13 +1060,7 @@ bool MaterializeResources(const ResourcePlan& program, const SrtRuntime& runtime
|
|||||||
continue;
|
continue;
|
||||||
}
|
}
|
||||||
const auto& indirect = *source->indirect_image;
|
const auto& indirect = *source->indirect_image;
|
||||||
DescriptorValue material;
|
if (!MaterializeIndirectImage(program, indirect, i, observed, clean, snapshot, specialization)) {
|
||||||
DescriptorValue table;
|
|
||||||
if ((indirect.material_source != UINT32_MAX &&
|
|
||||||
!clean.EvaluateDescriptor(indirect.material_source, material)) ||
|
|
||||||
!clean.EvaluateDescriptor(indirect.table_source, table) ||
|
|
||||||
!MaterializeIndirectImage(program, indirect, material, table, i, observed, clean, snapshot,
|
|
||||||
specialization)) {
|
|
||||||
return false;
|
return false;
|
||||||
}
|
}
|
||||||
} else {
|
} else {
|
||||||
|
|||||||
@@ -321,8 +321,9 @@ public:
|
|||||||
for (uint32_t dword = 5u; dword < plan.roots.size(); dword++) {
|
for (uint32_t dword = 5u; dword < plan.roots.size(); dword++) {
|
||||||
plan.handle->SetArg(dword, plan.key);
|
plan.handle->SetArg(dword, plan.key);
|
||||||
}
|
}
|
||||||
for (const auto index: plan.memory) {
|
if (plan.reads[0] != nullptr) {
|
||||||
m_program.memory_info[index].planning_only = true;
|
for (const auto index: plan.memory)
|
||||||
|
m_program.memory_info[index].planning_only = true;
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
m_program.descriptor_sources = std::move(m_sources);
|
m_program.descriptor_sources = std::move(m_sources);
|
||||||
@@ -762,7 +763,7 @@ private:
|
|||||||
const auto& b = *descriptor.indirect_image;
|
const auto& b = *descriptor.indirect_image;
|
||||||
if (a.material_source != b.material_source || a.table_source != b.table_source ||
|
if (a.material_source != b.material_source || a.table_source != b.table_source ||
|
||||||
a.selector_stride != b.selector_stride || a.selector_offset != b.selector_offset ||
|
a.selector_stride != b.selector_stride || a.selector_offset != b.selector_offset ||
|
||||||
a.table_offset != b.table_offset ||
|
a.table_offset != b.table_offset || a.sources != b.sources ||
|
||||||
!EquivalentValue(m_program, a.key_count, b.key_count) ||
|
!EquivalentValue(m_program, a.key_count, b.key_count) ||
|
||||||
a.selector_mask.IsEmpty() != b.selector_mask.IsEmpty() ||
|
a.selector_mask.IsEmpty() != b.selector_mask.IsEmpty() ||
|
||||||
(!a.selector_mask.IsEmpty() &&
|
(!a.selector_mask.IsEmpty() &&
|
||||||
@@ -1474,6 +1475,152 @@ private:
|
|||||||
return true;
|
return true;
|
||||||
}
|
}
|
||||||
|
|
||||||
|
uint32_t SelectorMaximum(Value value) const {
|
||||||
|
value = value.Resolve();
|
||||||
|
if (value.IsImmediate() && value.GetType() == Type::U32) return value.U32();
|
||||||
|
const auto* inst = value.TryInstruction();
|
||||||
|
if (inst == nullptr) return UINT32_MAX;
|
||||||
|
if (inst->GetOpcode() == ValueOpcode::FindILsb32)
|
||||||
|
return NonzeroOnEntry(inst->Arg(0), inst->Parent()) ? 31u : UINT32_MAX;
|
||||||
|
if (inst->GetOpcode() == ValueOpcode::UMin32)
|
||||||
|
return std::min(SelectorMaximum(inst->Arg(0)), SelectorMaximum(inst->Arg(1)));
|
||||||
|
uint32_t shift;
|
||||||
|
if (inst->GetOpcode() == ValueOpcode::ShiftLeftLogical32 &&
|
||||||
|
ImmediateU32(inst->Arg(1), shift) && shift < 32u) {
|
||||||
|
const auto maximum = uint64_t {SelectorMaximum(inst->Arg(0))} << shift;
|
||||||
|
if (maximum <= UINT32_MAX) return static_cast<uint32_t>(maximum);
|
||||||
|
}
|
||||||
|
return UINT32_MAX;
|
||||||
|
}
|
||||||
|
|
||||||
|
bool ImpossibleSwitchEdge(const Block* from, const Block* to) const {
|
||||||
|
const auto from_it = std::ranges::find(m_program.blocks, from);
|
||||||
|
const auto to_it = std::ranges::find(m_program.blocks, to);
|
||||||
|
if (from_it == m_program.blocks.end() || to_it == m_program.blocks.end() ||
|
||||||
|
m_program.block_info.size() != m_program.blocks.size()) return false;
|
||||||
|
const auto& info = m_program.block_info[from_it - m_program.blocks.begin()];
|
||||||
|
const auto& term = info.terminator;
|
||||||
|
if (term.kind != CFG::TerminatorKind::IndirectBranch ||
|
||||||
|
term.indirect_selector_code == UINT32_MAX ||
|
||||||
|
term.indirect_selector_values.empty() ||
|
||||||
|
term.indirect_selector_values.size() != term.indirect_selector_targets.size()) return false;
|
||||||
|
const auto target = m_program.block_info[to_it - m_program.blocks.begin()].id;
|
||||||
|
const auto maximum = SelectorMaximum(info.indirect_target);
|
||||||
|
bool found = false;
|
||||||
|
for (size_t i = 0; i < term.indirect_selector_values.size(); ++i) {
|
||||||
|
if (term.indirect_selector_targets[i] != target) continue;
|
||||||
|
if (term.indirect_selector_values[i] <= maximum) return false;
|
||||||
|
found = true;
|
||||||
|
}
|
||||||
|
return found;
|
||||||
|
}
|
||||||
|
|
||||||
|
bool TryMakeFiniteImage(Inst& handle, IndirectImagePlan& plan) {
|
||||||
|
if (handle.GetOpcode() != ValueOpcode::GetImageResource || handle.NumArgs() != 8u)
|
||||||
|
return false;
|
||||||
|
if (std::ranges::any_of(handle.Uses(), [](const Use& use) {
|
||||||
|
const auto op = use.user->GetOpcode();
|
||||||
|
return op != ValueOpcode::ImageSampleRaw && op != ValueOpcode::ImageGatherRaw;
|
||||||
|
})) return false;
|
||||||
|
bool has_phi = false;
|
||||||
|
for (size_t word = 0; word < handle.NumArgs(); ++word) has_phi |= handle.Arg(word).Resolve().IsPhi();
|
||||||
|
if (!has_phi) return false;
|
||||||
|
// Normalize the eight synchronized descriptor words to one GPU ordinal.
|
||||||
|
// Build the complete graph before changing IR so an unsupported leaf is transactional.
|
||||||
|
struct Choice {
|
||||||
|
DescriptorSource descriptor;
|
||||||
|
const Inst* branch = nullptr;
|
||||||
|
std::vector<uint32_t> children;
|
||||||
|
Value key;
|
||||||
|
};
|
||||||
|
std::vector<Choice> choices;
|
||||||
|
const auto visit = [&](const auto& self, DescriptorSource descriptor) -> uint32_t {
|
||||||
|
for (auto& word: descriptor.dwords) word = word.Resolve();
|
||||||
|
for (uint32_t i = 0; i < choices.size(); ++i)
|
||||||
|
if (choices[i].descriptor.dwords == descriptor.dwords) return i;
|
||||||
|
const auto index = static_cast<uint32_t>(choices.size());
|
||||||
|
choices.push_back({descriptor});
|
||||||
|
uint32_t bad = 0;
|
||||||
|
if (ValidateSource(descriptor, bad)) return index;
|
||||||
|
const auto* branch = descriptor.dwords[bad].TryInstruction();
|
||||||
|
if (branch == nullptr || branch->Parent() == nullptr ||
|
||||||
|
branch->GetOpcode() != ValueOpcode::Phi ||
|
||||||
|
branch->NumPhiBlocks() != branch->NumArgs()) return UINT32_MAX;
|
||||||
|
choices[index].branch = branch;
|
||||||
|
const auto count = branch->NumArgs();
|
||||||
|
for (uint32_t arm = 0; arm < count; ++arm) {
|
||||||
|
// A nonzero bit scan cannot choose its zero-input switch sentinel.
|
||||||
|
// Preserve the Phi edge; its ordinal is never observed at runtime.
|
||||||
|
if (ImpossibleSwitchEdge(branch->PhiBlock(arm), branch->Parent())) {
|
||||||
|
choices[index].children.push_back(UINT32_MAX);
|
||||||
|
continue;
|
||||||
|
}
|
||||||
|
auto child = descriptor;
|
||||||
|
for (uint32_t word = 0; word < child.dword_count; ++word) {
|
||||||
|
const auto* selected = descriptor.dwords[word].TryInstruction();
|
||||||
|
if (selected == nullptr || selected->GetOpcode() != branch->GetOpcode() ||
|
||||||
|
selected->Parent() != branch->Parent()) {
|
||||||
|
if (!ValidateRuntimeValue(m_program, descriptor.dwords[word])) return UINT32_MAX;
|
||||||
|
continue;
|
||||||
|
}
|
||||||
|
if (selected->NumArgs() != count || selected->NumPhiBlocks() != count)
|
||||||
|
return UINT32_MAX;
|
||||||
|
uint32_t incoming = 0;
|
||||||
|
while (incoming < count && selected->PhiBlock(incoming) != branch->PhiBlock(arm))
|
||||||
|
++incoming;
|
||||||
|
if (incoming == count) return UINT32_MAX;
|
||||||
|
child.dwords[word] = selected->Arg(incoming);
|
||||||
|
}
|
||||||
|
const auto next = self(self, child);
|
||||||
|
if (next == UINT32_MAX) return UINT32_MAX;
|
||||||
|
choices[index].children.push_back(next);
|
||||||
|
}
|
||||||
|
return index;
|
||||||
|
};
|
||||||
|
DescriptorSource root;
|
||||||
|
root.dword_count = 8u;
|
||||||
|
for (uint32_t word = 0; word < root.dword_count; ++word) root.dwords[word] = handle.Arg(word);
|
||||||
|
if (visit(visit, root) == UINT32_MAX || choices.front().branch == nullptr) return false;
|
||||||
|
DescriptorSource image_source;
|
||||||
|
image_source.dword_count = 8u;
|
||||||
|
image_source.dwords.fill(Value(0u));
|
||||||
|
image_source.indirect_image.emplace();
|
||||||
|
auto& sources = image_source.indirect_image->sources;
|
||||||
|
for (auto& choice: choices) {
|
||||||
|
if (choice.branch != nullptr) continue;
|
||||||
|
const auto source = InternSource(choice.descriptor);
|
||||||
|
auto found = std::ranges::find(sources, source);
|
||||||
|
choice.key = Value(static_cast<uint32_t>(found - sources.begin()));
|
||||||
|
if (found == sources.end()) sources.push_back(source);
|
||||||
|
}
|
||||||
|
if (sources.empty()) return false;
|
||||||
|
for (auto& choice: choices) {
|
||||||
|
if (choice.branch == nullptr) continue;
|
||||||
|
auto* block = choice.branch->Parent();
|
||||||
|
const auto where = std::ranges::find_if(block->Instructions(), [&](const Inst& inst) {
|
||||||
|
return &inst == choice.branch;
|
||||||
|
});
|
||||||
|
auto& key = *block->PrependNewInst(where, choice.branch->GetOpcode());
|
||||||
|
key.SetFlags(Type::U32);
|
||||||
|
choice.key = Value(&key);
|
||||||
|
}
|
||||||
|
for (const auto& choice: choices) {
|
||||||
|
if (choice.branch == nullptr) continue;
|
||||||
|
auto* key = choice.key.Instruction();
|
||||||
|
for (uint32_t arm = 0; arm < choice.children.size(); ++arm) {
|
||||||
|
const auto child = choice.children[arm];
|
||||||
|
const auto value = child == UINT32_MAX ? Value(0u) : choices[child].key;
|
||||||
|
key->AddPhiOperand(choice.branch->PhiBlock(arm), value);
|
||||||
|
}
|
||||||
|
}
|
||||||
|
plan.handle = &handle;
|
||||||
|
plan.source = InternSource(image_source);
|
||||||
|
plan.key = choices.front().key;
|
||||||
|
plan.roots = image_source.dwords;
|
||||||
|
plan.reads.fill(nullptr);
|
||||||
|
return true;
|
||||||
|
}
|
||||||
|
|
||||||
const IndirectImagePlan* FindIndirectImage(const Inst& handle) const {
|
const IndirectImagePlan* FindIndirectImage(const Inst& handle) const {
|
||||||
const auto found =
|
const auto found =
|
||||||
std::find_if(m_indirect_images.begin(), m_indirect_images.end(),
|
std::find_if(m_indirect_images.begin(), m_indirect_images.end(),
|
||||||
@@ -1486,7 +1633,8 @@ private:
|
|||||||
bool IsIndirectPlanningMemory(uint32_t index) const {
|
bool IsIndirectPlanningMemory(uint32_t index) const {
|
||||||
return std::any_of(m_indirect_images.begin(), m_indirect_images.end(),
|
return std::any_of(m_indirect_images.begin(), m_indirect_images.end(),
|
||||||
[&](const IndirectImagePlan& plan) {
|
[&](const IndirectImagePlan& plan) {
|
||||||
return std::ranges::find(plan.memory, index) != plan.memory.end();
|
return plan.reads[0] != nullptr &&
|
||||||
|
std::ranges::find(plan.memory, index) != plan.memory.end();
|
||||||
});
|
});
|
||||||
}
|
}
|
||||||
|
|
||||||
@@ -1502,7 +1650,7 @@ private:
|
|||||||
continue;
|
continue;
|
||||||
}
|
}
|
||||||
IndirectImagePlan plan;
|
IndirectImagePlan plan;
|
||||||
if (TryMakeIndirectImage(*handle, plan)) {
|
if (TryMakeIndirectImage(*handle, plan) || TryMakeFiniteImage(*handle, plan)) {
|
||||||
m_indirect_images.push_back(std::move(plan));
|
m_indirect_images.push_back(std::move(plan));
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|||||||
@@ -278,6 +278,101 @@ void TestFailedMaterializationRejectsStage() {
|
|||||||
"missing runtime user data did not reject the cached stage");
|
"missing runtime user data did not reject the cached stage");
|
||||||
}
|
}
|
||||||
|
|
||||||
|
void TestFiniteImageRefreshReusesScalarReads() {
|
||||||
|
using namespace Libs::Graphics::ShaderRecompiler::IR;
|
||||||
|
Program program;
|
||||||
|
program.stage = Libs::Graphics::ShaderType::Compute;
|
||||||
|
program.srt_plan_complete = true;
|
||||||
|
program.resource_tracking_complete = true;
|
||||||
|
auto &block = AddValueBlock(program);
|
||||||
|
program.memory_info.push_back({.kind = ResourceKind::ScalarAddress});
|
||||||
|
auto &handle = block.AppendNewInst(ValueOpcode::GetAddressResource,
|
||||||
|
{Value(0x1000u), Value(0u)});
|
||||||
|
auto &srt = block.AppendNewInst(ValueOpcode::GetSrtResource);
|
||||||
|
for (uint32_t index = 0; index < 3; ++index) {
|
||||||
|
auto &read = block.AppendNewInst(ValueOpcode::LoadAddressU32,
|
||||||
|
{Value(&handle), Value(index * 4u), Value(0u), Value(true)});
|
||||||
|
read.SetFlags(MemoryFlags{.index = 0});
|
||||||
|
program.srt_reads.push_back({Value(&read), index});
|
||||||
|
auto &flat = block.AppendNewInst(ValueOpcode::ReadConst,
|
||||||
|
{Value(&srt), Value(index)});
|
||||||
|
DescriptorSource source;
|
||||||
|
source.dword_count = 8;
|
||||||
|
source.dwords.fill(Value(0u));
|
||||||
|
source.dwords[0] = Value(&flat);
|
||||||
|
source.dwords[1] = Value(static_cast<uint32_t>(
|
||||||
|
Libs::Graphics::Prospero::BufferFormat::k32_32_32_32Float) << 20u);
|
||||||
|
source.dwords[3] = Value(Libs::Graphics::DstSel(4, 5, 6, 7) |
|
||||||
|
(static_cast<uint32_t>(Libs::Graphics::Prospero::ImageType::kColor2D) << 28u));
|
||||||
|
program.descriptor_sources.push_back(source);
|
||||||
|
}
|
||||||
|
DescriptorSource root;
|
||||||
|
root.dword_count = 8;
|
||||||
|
root.dwords.fill(Value(0u));
|
||||||
|
root.indirect_image.emplace().sources = {0, 1, 2, 1};
|
||||||
|
program.descriptor_sources.push_back(root);
|
||||||
|
program.info.images.push_back({
|
||||||
|
.source = 3,
|
||||||
|
.resource_class = ImageResourceClass::Sampled,
|
||||||
|
.numeric_class = Libs::Graphics::Prospero::TextureNumericClass::Float,
|
||||||
|
.dimension = Libs::Graphics::ShaderRecompiler::Decoder::ImageDimension::Dim2D});
|
||||||
|
auto plan = ExtractResourcePlan(program);
|
||||||
|
struct Reads {
|
||||||
|
std::array<uint32_t, 3> words{0x100u, 0x200u, 0x200u};
|
||||||
|
std::array<uint32_t, 3> counts{};
|
||||||
|
uint32_t ordinary = 0;
|
||||||
|
} reads;
|
||||||
|
const SrtRuntime runtime{
|
||||||
|
.read_memory = +[](void *data, uint64_t, std::span<uint32_t>) {
|
||||||
|
++static_cast<Reads *>(data)->ordinary;
|
||||||
|
return false;
|
||||||
|
},
|
||||||
|
.userdata = &reads,
|
||||||
|
.read_specialization_memory = +[](void *data, uint64_t address,
|
||||||
|
std::span<uint32_t> words) {
|
||||||
|
if (words.size() != 1 || address < 0x1000u || address >= 0x100cu ||
|
||||||
|
(address & 3u) != 0) return false;
|
||||||
|
auto &reads = *static_cast<Reads *>(data);
|
||||||
|
const auto index = (address - 0x1000u) / 4u;
|
||||||
|
++reads.counts[index];
|
||||||
|
words[0] = reads.words[index];
|
||||||
|
return true;
|
||||||
|
}};
|
||||||
|
ResourceSnapshot snapshot;
|
||||||
|
ResourceSpecialization specialization;
|
||||||
|
const auto capacities = [&] {
|
||||||
|
return std::array{snapshot.images.capacity(), snapshot.flattened_srt.capacity(),
|
||||||
|
snapshot.specialization_reads.capacity(), specialization.images.capacity()};
|
||||||
|
};
|
||||||
|
const auto check_mapping = [&](std::array<uint32_t, 4> ordinals) {
|
||||||
|
const auto offset = specialization.images[0].indirect_mapping_offset;
|
||||||
|
Check(snapshot.images.size() == 2 && snapshot.flattened_srt[offset] == 4,
|
||||||
|
"finite image candidates were not deduplicated");
|
||||||
|
for (uint32_t key = 0; key < ordinals.size(); ++key) {
|
||||||
|
Check(snapshot.flattened_srt[offset + 1u + key * 2u] == key &&
|
||||||
|
snapshot.flattened_srt[offset + 2u + key * 2u] == ordinals[key],
|
||||||
|
"finite image selector mapping is stale or incorrect");
|
||||||
|
}
|
||||||
|
};
|
||||||
|
Check(MaterializeResources(plan, runtime, snapshot, specialization),
|
||||||
|
"finite image materialization failed");
|
||||||
|
Check(reads.ordinary == 0 && reads.counts == std::array<uint32_t, 3>{1, 1, 1} &&
|
||||||
|
snapshot.specialization_reads.size() == 3,
|
||||||
|
"finite image candidates repeated scalar reads or bypassed clean provenance");
|
||||||
|
check_mapping({0, 1, 1, 1});
|
||||||
|
const auto warm_capacities = capacities();
|
||||||
|
reads.words = {0x300u, 0x300u, 0x400u};
|
||||||
|
Check(MaterializeResources(plan, runtime, snapshot, specialization),
|
||||||
|
"finite image refresh failed after descriptor changes");
|
||||||
|
Check(reads.ordinary == 0 && reads.counts == std::array<uint32_t, 3>{2, 2, 2} &&
|
||||||
|
snapshot.specialization_reads.size() == 3 &&
|
||||||
|
snapshot.images[0].dwords[0] == 0x300u && snapshot.images[1].dwords[0] == 0x400u,
|
||||||
|
"finite image refresh retained old scalar values or repeated reads");
|
||||||
|
check_mapping({0, 0, 1, 0});
|
||||||
|
Check(capacities() == warm_capacities,
|
||||||
|
"finite image refresh grew reusable resource storage after warmup");
|
||||||
|
}
|
||||||
|
|
||||||
void TestMixedSamplerDuplicatesTheCorrectSnapshot() {
|
void TestMixedSamplerDuplicatesTheCorrectSnapshot() {
|
||||||
using namespace Libs::Graphics::ShaderRecompiler::IR;
|
using namespace Libs::Graphics::ShaderRecompiler::IR;
|
||||||
auto plan = MixedSamplerPlan();
|
auto plan = MixedSamplerPlan();
|
||||||
@@ -314,6 +409,7 @@ int main() {
|
|||||||
TestUnbasedFlatCacheHitMaterializes();
|
TestUnbasedFlatCacheHitMaterializes();
|
||||||
TestWrittenDescriptorUsesStrictReaderOnce();
|
TestWrittenDescriptorUsesStrictReaderOnce();
|
||||||
TestFailedMaterializationRejectsStage();
|
TestFailedMaterializationRejectsStage();
|
||||||
|
TestFiniteImageRefreshReusesScalarReads();
|
||||||
TestMixedSamplerDuplicatesTheCorrectSnapshot();
|
TestMixedSamplerDuplicatesTheCorrectSnapshot();
|
||||||
std::puts("ResourceMaterializationTests: all cases passed");
|
std::puts("ResourceMaterializationTests: all cases passed");
|
||||||
return 0;
|
return 0;
|
||||||
|
|||||||
@@ -2488,6 +2488,132 @@ void TestConditionalSamplerPhi() {
|
|||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
|
void TestFiniteImagePhiCycle() {
|
||||||
|
Fixture fixture;
|
||||||
|
auto *entry = fixture.block;
|
||||||
|
auto *header = fixture.AddBlock();
|
||||||
|
auto *reload = fixture.AddBlock();
|
||||||
|
auto *merge = fixture.AddBlock();
|
||||||
|
entry->AddBranch(header);
|
||||||
|
header->AddBranch(reload);
|
||||||
|
header->AddBranch(merge);
|
||||||
|
reload->AddBranch(merge);
|
||||||
|
merge->AddBranch(header);
|
||||||
|
std::array<Value, 8> words;
|
||||||
|
for (uint32_t word = 0; word < words.size(); ++word) {
|
||||||
|
auto &loop = header->AppendNewInst(ValueOpcode::Phi, {}, uint64_t(Type::U32));
|
||||||
|
auto &choice = merge->AppendNewInst(ValueOpcode::Phi, {}, uint64_t(Type::U32));
|
||||||
|
loop.AddPhiOperand(entry, fixture.UserData(word));
|
||||||
|
loop.AddPhiOperand(merge, Value(&choice));
|
||||||
|
// Incoming order is independent for each descriptor DWORD.
|
||||||
|
if (word & 1u) {
|
||||||
|
choice.AddPhiOperand(reload, fixture.UserData(word + 8));
|
||||||
|
choice.AddPhiOperand(header, Value(&loop));
|
||||||
|
} else {
|
||||||
|
choice.AddPhiOperand(header, Value(&loop));
|
||||||
|
choice.AddPhiOperand(reload, fixture.UserData(word + 8));
|
||||||
|
}
|
||||||
|
words[word] = Value(&choice);
|
||||||
|
}
|
||||||
|
fixture.block = merge;
|
||||||
|
const auto image = fixture.Image(words);
|
||||||
|
const auto sampler = fixture.Sampler({Value(0u), Value(0u), Value(0u), Value(0u)});
|
||||||
|
MemoryInfo memory;
|
||||||
|
memory.kind = ResourceKind::Image;
|
||||||
|
memory.image_dimension = Decoder::ImageDimension::Dim2D;
|
||||||
|
fixture.Emit(ValueOpcode::ImageSampleRaw, {image, sampler, fixture.ImageAddress()},
|
||||||
|
fixture.AddMemory(memory, 4));
|
||||||
|
fixture.PlanAndTrack();
|
||||||
|
const auto source = fixture.program.info.images[0].source;
|
||||||
|
const auto &finite = fixture.program.descriptor_sources[source].indirect_image;
|
||||||
|
Check(finite && finite->sources.size() == 2,
|
||||||
|
"cyclic image selection did not retain two complete descriptor candidates");
|
||||||
|
const auto *key = image.Instruction()->Arg(0).Instruction();
|
||||||
|
Check(key->GetOpcode() == ValueOpcode::Phi && key->Parent() == merge,
|
||||||
|
"finite image key lost its original merge block");
|
||||||
|
const auto *loop = key->Arg(0).Instruction();
|
||||||
|
Check(loop->GetOpcode() == ValueOpcode::Phi && loop->Parent() == header &&
|
||||||
|
loop->Arg(1).Resolve() == Value(const_cast<Inst *>(key)) &&
|
||||||
|
loop->Arg(0).U32() == 0 && key->Arg(1).U32() == 1,
|
||||||
|
"finite image key did not preserve the loop-carried selection");
|
||||||
|
std::array<uint32_t, 16> data;
|
||||||
|
for (uint32_t i = 0; i < data.size(); ++i) data[i] = 100u + i;
|
||||||
|
SrtWalker walker(fixture.program, SrtRuntime{.user_data = data});
|
||||||
|
for (uint32_t candidate = 0; candidate < 2; ++candidate) {
|
||||||
|
DescriptorValue descriptor;
|
||||||
|
Check(walker.EvaluateDescriptor(finite->sources[candidate], descriptor),
|
||||||
|
"finite image candidate is not host-readable");
|
||||||
|
for (uint32_t word = 0; word < 8; ++word)
|
||||||
|
Check(descriptor.dwords[word] == data[candidate * 8 + word],
|
||||||
|
"finite image selection mixed descriptor DWORDs across predecessors");
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
void TestFiniteImageBitScanSentinel() {
|
||||||
|
namespace CFG = Libs::Graphics::ShaderRecompiler::CFG;
|
||||||
|
for (const bool nonzero : {false, true}) {
|
||||||
|
Fixture fixture;
|
||||||
|
auto *entry = fixture.block;
|
||||||
|
auto *dispatch = fixture.AddBlock();
|
||||||
|
auto *load = fixture.AddBlock();
|
||||||
|
auto *merge = fixture.AddBlock();
|
||||||
|
auto *exit = fixture.AddBlock();
|
||||||
|
entry->AddBranch(dispatch);
|
||||||
|
entry->AddBranch(exit);
|
||||||
|
dispatch->AddBranch(load);
|
||||||
|
dispatch->AddBranch(merge);
|
||||||
|
load->AddBranch(merge);
|
||||||
|
const auto mask = fixture.Emit(ValueOpcode::GetBuiltin,
|
||||||
|
{Value(uint32_t(StageInputKind::LocalInvocationIndex)), Value(0u)});
|
||||||
|
const auto condition = fixture.Emit(nonzero ? ValueOpcode::INotEqual32 : ValueOpcode::IEqual32,
|
||||||
|
{mask, Value(0u)});
|
||||||
|
auto &guard = fixture.program.block_info[0];
|
||||||
|
guard.condition = condition;
|
||||||
|
guard.terminator.kind = CFG::TerminatorKind::ConditionalBranch;
|
||||||
|
guard.terminator.true_block = 1;
|
||||||
|
guard.terminator.false_block = 4;
|
||||||
|
fixture.block = dispatch;
|
||||||
|
const auto first = fixture.Emit(ValueOpcode::FindILsb32, {mask});
|
||||||
|
const auto minimum = fixture.Emit(ValueOpcode::UMin32, {first, Value(32u)});
|
||||||
|
const auto selector = fixture.Emit(ValueOpcode::ShiftLeftLogical32, {minimum, Value(2u)});
|
||||||
|
fixture.Emit(ValueOpcode::ReferenceU32, {selector});
|
||||||
|
auto &table = fixture.program.block_info[1];
|
||||||
|
table.indirect_target = selector;
|
||||||
|
table.terminator.kind = CFG::TerminatorKind::IndirectBranch;
|
||||||
|
table.terminator.indirect_selector_code = 0;
|
||||||
|
table.terminator.indirect_selector_values = {0u, 128u};
|
||||||
|
table.terminator.indirect_selector_targets = {2u, 3u};
|
||||||
|
std::array<Value, 8> words;
|
||||||
|
for (uint32_t word = 0; word < words.size(); ++word) {
|
||||||
|
auto &phi = merge->AppendNewInst(ValueOpcode::Phi, {}, uint64_t(Type::U32));
|
||||||
|
phi.AddPhiOperand(dispatch, mask);
|
||||||
|
phi.AddPhiOperand(load, Value(100u + word));
|
||||||
|
words[word] = Value(&phi);
|
||||||
|
}
|
||||||
|
fixture.block = merge;
|
||||||
|
const auto image = fixture.Image(words);
|
||||||
|
const auto sampler = fixture.Sampler({Value(0u), Value(0u), Value(0u), Value(0u)});
|
||||||
|
MemoryInfo memory;
|
||||||
|
memory.kind = ResourceKind::Image;
|
||||||
|
memory.image_dimension = Decoder::ImageDimension::Dim2D;
|
||||||
|
fixture.Emit(ValueOpcode::ImageSampleRaw, {image, sampler, fixture.ImageAddress()},
|
||||||
|
fixture.AddMemory(memory, 4));
|
||||||
|
if (!nonzero) {
|
||||||
|
CheckFatal([&] { fixture.PlanAndTrack(); }, "not a valid runtime value",
|
||||||
|
"zero bit scan silently accepted a GPU-valued descriptor sentinel");
|
||||||
|
Check(fixture.program.descriptor_sources.empty() && image.Instruction()->Arg(0) == words[0],
|
||||||
|
"failed finite descriptor analysis partially mutated the resource plan");
|
||||||
|
continue;
|
||||||
|
}
|
||||||
|
fixture.PlanAndTrack();
|
||||||
|
const auto source = fixture.program.info.images[0].source;
|
||||||
|
const auto &finite = fixture.program.descriptor_sources[source].indirect_image;
|
||||||
|
Check(finite && finite->sources.size() == 1 &&
|
||||||
|
image.Instruction()->Arg(0).Instruction()->NumPhiBlocks() == 2,
|
||||||
|
"nonzero bit scan retained its impossible sentinel or removed a Phi edge");
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
void TestLoopCycleEnteredThroughRuntimeValue() {
|
void TestLoopCycleEnteredThroughRuntimeValue() {
|
||||||
Fixture fixture;
|
Fixture fixture;
|
||||||
auto *entry = fixture.block;
|
auto *entry = fixture.block;
|
||||||
@@ -3292,6 +3418,8 @@ int main() {
|
|||||||
Run("dynamic SRT", TestDynamicSrtReadRemainsExplicit);
|
Run("dynamic SRT", TestDynamicSrtReadRemainsExplicit);
|
||||||
Run("phi validation", TestPhiValidation);
|
Run("phi validation", TestPhiValidation);
|
||||||
Run("conditional sampler phi", TestConditionalSamplerPhi);
|
Run("conditional sampler phi", TestConditionalSamplerPhi);
|
||||||
|
Run("finite image phi cycle", TestFiniteImagePhiCycle);
|
||||||
|
Run("finite image bit scan sentinel", TestFiniteImageBitScanSentinel);
|
||||||
Run("runtime-rooted loop", TestLoopCycleEnteredThroughRuntimeValue);
|
Run("runtime-rooted loop", TestLoopCycleEnteredThroughRuntimeValue);
|
||||||
Run("invariant loop phi", TestInvariantLoopPhi);
|
Run("invariant loop phi", TestInvariantLoopPhi);
|
||||||
Run("DMA address materialization", TestDmaAddressMaterialization);
|
Run("DMA address materialization", TestDmaAddressMaterialization);
|
||||||
|
|||||||
Reference in New Issue
Block a user