raphiaura/KytyPS5
1
0
Fork 0
mirror of https://github.com/KytyPS5/KytyPS5.git synced 2026-10-19 18:36:53 +02:00
Code Issues Packages Projects Releases Wiki Activity

shader: resolve shared selector keys once per materialization

Browse Source 
Co-authored-by: fxpw <fxpw@outlook.com>
This commit is contained in:
nmzik
2026-10-15 03:54:41 +02:00
parent 5916a184cc
commit 9c04abacc5
3 changed files with 379 additions and 230 deletions
Download Patch File Download Diff File Expand all files Collapse all files
src/graphics/shader/recompiler/ir/ShaderIR.h
+8
View File
@@ -614,6 +614,12 @@ struct UniformFillPlan {
// Resource analysis retained by the shader cache. It owns immutable descriptor/SRT,
// condition and fill values without translated blocks, plus reusable evaluation scratch.
struct ResourcePlan {
struct SelectorKeys {
uint32_t source = UINT32_MAX;
std::vector<uint32_t> values;
bool resolved = false;
};
struct EvaluationContext {
struct Entry {
uint64_t value = 0;
@@ -642,6 +648,7 @@ struct ResourcePlan {
std::list<Inst> value_storage;
std::vector<MemoryInfo> memory_info;
std::vector<DescriptorSource> descriptor_sources;
std::vector<uint32_t> selector_groups;
std::vector<ResourceBlock> control_flow;
std::unordered_map<const Inst*, std::vector<uint32_t>> control_phi_predecessors;
std::vector<SrtRead> srt_reads;
@@ -661,6 +668,7 @@ struct ResourcePlan {
mutable std::vector<uint8_t> reachable_blocks;
mutable std::vector<uint32_t> pending_blocks;
mutable std::vector<uint32_t> material_keys;
mutable std::vector<SelectorKeys> selector_keys;
};
struct Program: ResourcePlan {
src/graphics/shader/recompiler/ir/passes/ResourceMaterialization.cpp
+249 -209
View File
@@ -296,46 +296,24 @@ bool IsBoundedDescriptorTable(const ResourcePlan& program,
table != nullptr && table->dword_count == 4u;
}
template <typename Specialization, typename Normalize>
bool MaterializeIndirectDescriptor(const ResourcePlan& program,
const DescriptorSource::IndirectDescriptor& indirect,
uint32_t resource_index, uint32_t dword_count, uint32_t read_dwords,
const SrtRuntime& runtime, SrtWalker& clean,
ResourceSnapshot& snapshot,
std::vector<DescriptorValue>& descriptors,
std::vector<Specialization>& specializations,
uint32_t maximum_resources, Normalize&& normalize) {
const auto descriptor_bytes = read_dwords * sizeof(uint32_t);
const auto& sources = indirect.sources;
const auto* selector = indirect.selector ? &*indirect.selector : nullptr;
auto& keys = program.material_keys;
const auto children_begin = descriptors.size();
const auto root_resource = specializations[resource_index];
const auto evaluate_count = [&](uint32_t& count) {
if (indirect.workgroup_axis != UINT32_MAX) {
if (runtime.workgroup_counts.size() != 3u || indirect.workgroup_axis >= 3u) return false;
count = std::ranges::find(runtime.workgroup_counts, 0u) != runtime.workgroup_counts.end()
? 0u : runtime.workgroup_counts[indirect.workgroup_axis];
return true;
}
if (!clean.Evaluate(indirect.key_count, count)) return false;
if (indirect.signed_count && std::bit_cast<int32_t>(count) < 0) count = 0;
bool EvaluateIndirectCount(const DescriptorSource::IndirectDescriptor& indirect,
const SrtRuntime& runtime, SrtWalker& clean, uint32_t& count) {
if (indirect.workgroup_axis != UINT32_MAX) {
if (runtime.workgroup_counts.size() != 3u || indirect.workgroup_axis >= 3u) return false;
count = std::ranges::find(runtime.workgroup_counts, 0u) != runtime.workgroup_counts.end()
? 0u : runtime.workgroup_counts[indirect.workgroup_axis];
return true;
};
const auto intern_candidate = [&](const DescriptorValue& candidate, uint32_t& ordinal) {
ordinal = 0u;
if (candidate == descriptors[resource_index]) return true;
const auto found = std::find(descriptors.begin() + children_begin, descriptors.end(), candidate);
ordinal = static_cast<uint32_t>(found - descriptors.begin() - children_begin + 1u);
if (found == descriptors.end()) {
if (descriptors.size() >= maximum_resources) return false;
descriptors.push_back(candidate);
auto child = root_resource;
child.indirect_root = resource_index;
specializations.push_back(child);
}
return true;
};
}
if (!clean.Evaluate(indirect.key_count, count)) return false;
if (indirect.signed_count && std::bit_cast<int32_t>(count) < 0) count = 0;
return true;
}
bool ResolveSelectorKeys(const DescriptorSource::IndirectDescriptor& indirect,
const DescriptorValue& material_value, const SrtRuntime& runtime,
SrtWalker& clean, std::vector<uint32_t>& keys) {
const auto* selector = &*indirect.selector;
keys.clear();
const auto read_selector = [&](uint64_t base, uint64_t size, uint64_t offset,
std::span<uint32_t> words) {
return selector->global ? ReadTable(base, size, offset, runtime, words)
@@ -368,6 +346,207 @@ bool MaterializeIndirectDescriptor(const ResourcePlan& p
keys.resize(count);
return true;
};
if (selector->rows) {
const auto& rows = *selector->rows;
DescriptorValue count_value;
ShaderBufferResource counts, material;
if (!clean.EvaluateDescriptor(rows.source, count_value) ||
!DecodeBufferDescriptor(count_value, counts) ||
!DecodeBufferDescriptor(material_value, material) || selector->stride != 4u ||
selector->offset != 0u || rows.stride == 0u || (rows.stride & 3u) != 0u ||
counts.GetSize() / 4u > MaxIndirectDescriptorProbes) return false;
std::vector<uint32_t> lengths(counts.GetSize() / 4u);
if (!ReadScalarTable(counts.Base48(), counts.GetSize(), 0u, runtime, lengths)) return false;
uint32_t total = 0;
for (size_t row = 0; row < lengths.size(); ++row) {
const auto length = lengths[row];
if (length > MaxIndirectDescriptorProbes - total ||
(length != 0u && uint64_t {row} * rows.stride + uint64_t {length} * 4u >
UINT32_MAX + 1ull)) return false;
total += length;
}
keys.reserve(total);
std::vector<uint32_t> batch;
for (size_t row = 0; row < lengths.size();) {
if (lengths[row] == 0u) { ++row; continue; }
const auto first = uint64_t {row} * rows.stride / 4u;
auto end = first + lengths[row];
size_t next = row + 1u;
for (; next < lengths.size(); ++next) {
if (lengths[next] == 0u) continue;
const auto candidate_end =
std::max(end, uint64_t {next} * rows.stride / 4u + lengths[next]);
if (candidate_end - first > MaxIndirectDescriptorProbes) break;
end = candidate_end;
}
// Coalesce small gaps, but interpret only selected prefixes. The strict
// reader captures the entire fetched span for writable-alias rejection.
batch.resize(end - first);
if (!ReadScalarTable(material.Base48(), material.GetSize(), first * 4u, runtime, batch))
return false;
for (; row < next; ++row) {
if (lengths[row] == 0u) continue;
const auto begin = batch.begin() + (uint64_t {row} * rows.stride / 4u - first);
keys.insert(keys.end(), begin, begin + lengths[row]);
}
}
} else if (selector->bitset) {
ShaderBufferResource material;
std::array<uint32_t, 4> masks {};
uint32_t first = 0, count = 0;
if (!DecodeBufferDescriptor(material_value, material) ||
!clean.Evaluate(indirect.selector_first, first) ||
!EvaluateIndirectCount(indirect, runtime, clean, count) || count != masks.size() * 32u ||
selector->stride != sizeof(masks) || selector->offset != 0u ||
material.Type() != 0u || material.Stride() != sizeof(masks) ||
material.SwizzleEnabled() || material.AddTid() || material.OutOfBounds() != 0u ||
(material.Base48() & 3u) != 0u) return false;
if (material.Format() != Prospero::BufferFormat::kInvalid) {
const auto words = material.GetSize() / sizeof(uint32_t);
// Bitmap storage can grow with resolution while its candidate domain stays
// at 128 keys. Bound scratch memory, preserving each coherent read's capture.
for (uint64_t first_word = 0; first_word < words; first_word += keys.size()) {
keys.resize(std::min<uint64_t>(words - first_word, MaxIndirectDescriptorProbes));
if (!ReadScalarTable(material.Base48(), material.GetSize(), first_word * 4u,
runtime, keys)) return false;
for (size_t word = 0; word < keys.size(); ++word) masks[word % masks.size()] |= keys[word];
if (std::ranges::all_of(masks, [](uint32_t mask) { return mask == UINT32_MAX; })) break;
}
keys.clear();
for (uint32_t word = 0; word < masks.size(); ++word) {
auto mask = masks[word];
while (mask != 0u) {
const auto rank = word * 32u + std::countr_zero(mask);
keys.push_back(first + count - 1u - rank);
mask &= mask - 1u;
}
}
}
} else if (!selector->byte_ranges.empty()) {
if (material_value.dword_count != 2u) return false;
const auto base = (uint64_t {material_value.dwords[1]} << 32u) | material_value.dwords[0];
std::array<bool, 256> selected {};
for (const auto [offset, size]: selector->byte_ranges) {
if (size > MaxIndirectDescriptorProbes || ((offset | size) & 3u) != 0u) return false;
keys.resize(size / 4u);
if (!ReadScalarTable(base, UINT64_MAX, offset, runtime, keys)) return false;
for (const auto word: keys)
for (uint32_t shift = 0; shift < 32u; shift += 8u)
selected[(word >> shift) & 255u] = true;
}
keys.clear();
for (uint32_t key = 0; key < selected.size(); ++key)
if (selected[key]) keys.push_back(key);
} else if (!indirect.selector_first.IsEmpty()) {
ShaderBufferResource material;
uint64_t base = 0, size = UINT64_MAX;
uint32_t first = 0, count = 0;
if (material_value.dword_count == 2u) {
if (selector->stride == 0u) return false;
base = (uint64_t {material_value.dwords[1]} << 32u) | material_value.dwords[0];
} else {
if (!DecodeBufferDescriptor(material_value, material)) return false;
base = material.Base48();
size = material.GetSize();
}
if (!clean.Evaluate(indirect.selector_first, first) ||
!EvaluateIndirectCount(indirect, runtime, clean, count) || count > MaxIndirectDescriptorProbes)
return false;
// SMEM uses its instruction's byte stride; V# stride only supplies its bounds.
const auto step = selector->stride != 0u ? selector->stride : material.Stride();
if (selector->stride == 0u &&
(material.Type() != 0u || material.SwizzleEnabled() || material.AddTid() ||
material.OutOfBounds() != 0u || uint64_t {selector->offset} + 4u > step ||
uint64_t {first} + count > material.NumRecords())) return false;
const auto last = count != 0u ? uint64_t {first} + count - 1u : 0u;
if (last > UINT32_MAX) return false;
const auto end = count != 0u ? last * step + selector->offset + 4u : 0u;
if (end > uint64_t {UINT32_MAX} + 1u ||
(indirect.workgroup_axis == UINT32_MAX && end > size))
return false;
if (!read_keys(base, size, uint64_t {first} * step + selector->offset, step, count))
return false;
} else if (!indirect.selector_mask.IsEmpty()) {
uint32_t mask = 0;
uint32_t count = 0;
if (material_value.dword_count != 2u ||
!clean.Evaluate(indirect.selector_mask, mask) ||
!EvaluateIndirectCount(indirect, runtime, clean, count) || count == 0u || count > 32u) {
return false;
}
if (count < 32u) mask &= (1u << count) - 1u;
const auto material_base =
(static_cast<uint64_t>(material_value.dwords[1]) << 32u) | material_value.dwords[0];
keys.reserve(std::popcount(mask));
while (mask != 0u) {
const auto index = std::countr_zero(mask);
const auto offset = static_cast<uint64_t>(selector->offset) +
static_cast<uint64_t>(index) * selector->stride;
if (offset > UINT32_MAX) return false;
uint32_t key = 0;
if (!read_selector(material_base, UINT64_MAX, static_cast<uint32_t>(offset),
{&key, 1})) return false;
keys.push_back(key);
mask &= mask - 1u;
}
} else {
ShaderBufferResource material;
if (!DecodeBufferDescriptor(material_value, material) || selector->stride < 4u ||
!std::has_single_bit(selector->stride) || (selector->offset & 3u) != 0u)
return false;
// Empty selector bounds denote the complete U32 congruence class. SMEM
// adds its aligned immediate after the dynamic offset, without U32 wrap.
const auto domain_count = (uint64_t {1} << 32u) / selector->stride;
const auto available = material.GetSize() > selector->offset
? material.GetSize() - selector->offset : 0u;
const auto count = std::min(domain_count,
available < 4u ? 0u : (available - 4u) / selector->stride + 1u);
if (count > MaxIndirectDescriptorProbes ||
!read_keys(material.Base48(), material.GetSize(), selector->offset, selector->stride, count)) return false;
if (count < domain_count) keys.push_back(0u); // An out-of-bounds scalar read returns zero.
}
if (selector->byte_ranges.empty()) {
if (selector->bit_offset != 0u || selector->bit_mask != UINT32_MAX || selector->maximum != UINT32_MAX)
for (auto& key: keys)
key = std::min((key >> selector->bit_offset) & selector->bit_mask, selector->maximum);
std::ranges::sort(keys);
keys.erase(std::unique(keys.begin(), keys.end()), keys.end());
}
return true;
}
template <typename Specialization, typename Normalize>
bool MaterializeIndirectDescriptor(const ResourcePlan& program,
uint32_t source_index,
uint32_t resource_index, uint32_t dword_count, uint32_t read_dwords,
const SrtRuntime& runtime, SrtWalker& clean,
ResourceSnapshot& snapshot,
std::vector<DescriptorValue>& descriptors,
std::vector<Specialization>& specializations,
uint32_t maximum_resources, Normalize&& normalize) {
const auto& indirect = *program.descriptor_sources[source_index].indirect_descriptor;
const auto descriptor_bytes = read_dwords * sizeof(uint32_t);
const auto& sources = indirect.sources;
const auto* selector = indirect.selector ? &*indirect.selector : nullptr;
auto* selector_keys = selector != nullptr
? &program.selector_keys[program.selector_groups[source_index]] : nullptr;
auto& keys = selector_keys != nullptr ? selector_keys->values : program.material_keys;
const auto children_begin = descriptors.size();
const auto root_resource = specializations[resource_index];
const auto intern_candidate = [&](const DescriptorValue& candidate, uint32_t& ordinal) {
ordinal = 0u;
if (candidate == descriptors[resource_index]) return true;
const auto found = std::find(descriptors.begin() + children_begin, descriptors.end(), candidate);
ordinal = static_cast<uint32_t>(found - descriptors.begin() - children_begin + 1u);
if (found == descriptors.end()) {
if (descriptors.size() >= maximum_resources) return false;
descriptors.push_back(candidate);
auto child = root_resource;
child.indirect_root = resource_index;
specializations.push_back(child);
}
return true;
};
uint64_t table_base = 0;
uint64_t table_size = UINT64_MAX; // Scalar addresses have no buffer descriptor bounds.
const auto normalization_failed = [&](uint64_t offset, uint32_t key, uint64_t step,
@@ -381,10 +560,10 @@ bool MaterializeIndirectDescriptor(const ResourcePlan& p
indirect.table_offset, indirect.table_immediate));
};
if (sources.empty()) {
keys.clear();
if (selector == nullptr) keys.clear();
DescriptorValue material_value;
DescriptorValue table_value;
if ((selector != nullptr &&
if ((selector != nullptr && !selector_keys->resolved &&
!clean.EvaluateDescriptor(selector->source, material_value)) ||
!clean.EvaluateDescriptor(indirect.table_source, table_value)) {
return false;
@@ -476,7 +655,7 @@ bool MaterializeIndirectDescriptor(const ResourcePlan& p
if (selector == nullptr) {
uint32_t key_count = 0;
uint32_t first_key = 0;
if (!evaluate_count(key_count)) return false;
if (!EvaluateIndirectCount(indirect, runtime, clean, key_count)) return false;
if (!indirect.selector_first.IsEmpty() && !clean.Evaluate(indirect.selector_first, first_key))
return false;
if (indirect.table_stride == 0u || key_count > MaxIndirectDescriptorProbes ||
@@ -490,171 +669,12 @@ bool MaterializeIndirectDescriptor(const ResourcePlan& p
std::iota(keys.begin(), keys.end(), first_key);
if (!keys.empty() && keys.back() < keys.front())
std::rotate(keys.begin(), keys.begin() + (UINT32_MAX - first_key + 1ull), keys.end());
} else if (selector->rows) {
const auto& rows = *selector->rows;
DescriptorValue count_value;
ShaderBufferResource counts, material;
if (!clean.EvaluateDescriptor(rows.source, count_value) ||
!DecodeBufferDescriptor(count_value, counts) ||
!DecodeBufferDescriptor(material_value, material) || selector->stride != 4u ||
selector->offset != 0u || rows.stride == 0u || (rows.stride & 3u) != 0u ||
counts.GetSize() / 4u > MaxIndirectDescriptorProbes) return false;
std::vector<uint32_t> lengths(counts.GetSize() / 4u);
if (!ReadScalarTable(counts.Base48(), counts.GetSize(), 0u, runtime, lengths)) return false;
uint32_t total = 0;
for (size_t row = 0; row < lengths.size(); ++row) {
const auto length = lengths[row];
if (length > MaxIndirectDescriptorProbes - total ||
(length != 0u && uint64_t {row} * rows.stride + uint64_t {length} * 4u >
UINT32_MAX + 1ull)) return false;
total += length;
}
keys.reserve(total);
std::vector<uint32_t> batch;
for (size_t row = 0; row < lengths.size();) {
if (lengths[row] == 0u) { ++row; continue; }
const auto first = uint64_t {row} * rows.stride / 4u;
auto end = first + lengths[row];
size_t next = row + 1u;
for (; next < lengths.size(); ++next) {
if (lengths[next] == 0u) continue;
const auto candidate_end =
std::max(end, uint64_t {next} * rows.stride / 4u + lengths[next]);
if (candidate_end - first > MaxIndirectDescriptorProbes) break;
end = candidate_end;
}
// Coalesce small gaps, but interpret only selected prefixes. The strict
// reader captures the entire fetched span for writable-alias rejection.
batch.resize(end - first);
if (!ReadScalarTable(material.Base48(), material.GetSize(), first * 4u, runtime, batch))
return false;
for (; row < next; ++row) {
if (lengths[row] == 0u) continue;
const auto begin = batch.begin() + (uint64_t {row} * rows.stride / 4u - first);
keys.insert(keys.end(), begin, begin + lengths[row]);
}
}
} else if (selector->bitset) {
ShaderBufferResource material;
std::array<uint32_t, 4> masks {};
uint32_t first = 0, count = 0;
if (!DecodeBufferDescriptor(material_value, material) ||
!clean.Evaluate(indirect.selector_first, first) ||
!evaluate_count(count) || count != masks.size() * 32u ||
selector->stride != sizeof(masks) || selector->offset != 0u ||
material.Type() != 0u || material.Stride() != sizeof(masks) ||
material.SwizzleEnabled() || material.AddTid() || material.OutOfBounds() != 0u ||
(material.Base48() & 3u) != 0u) return false;
if (material.Format() != Prospero::BufferFormat::kInvalid) {
const auto words = material.GetSize() / sizeof(uint32_t);
// Bitmap storage can grow with resolution while its candidate domain stays
// at 128 keys. Bound scratch memory, preserving each coherent read's capture.
for (uint64_t first_word = 0; first_word < words; first_word += keys.size()) {
keys.resize(std::min<uint64_t>(words - first_word, MaxIndirectDescriptorProbes));
if (!ReadScalarTable(material.Base48(), material.GetSize(), first_word * 4u,
runtime, keys)) return false;
for (size_t word = 0; word < keys.size(); ++word) masks[word % masks.size()] |= keys[word];
if (std::ranges::all_of(masks, [](uint32_t mask) { return mask == UINT32_MAX; })) break;
}
keys.clear();
for (uint32_t word = 0; word < masks.size(); ++word) {
auto mask = masks[word];
while (mask != 0u) {
const auto rank = word * 32u + std::countr_zero(mask);
keys.push_back(first + count - 1u - rank);
mask &= mask - 1u;
}
}
}
} else if (!selector->byte_ranges.empty()) {
if (material_value.dword_count != 2u) return false;
const auto base = (uint64_t {material_value.dwords[1]} << 32u) | material_value.dwords[0];
std::array<bool, 256> selected {};
for (const auto [offset, size]: selector->byte_ranges) {
if (size > MaxIndirectDescriptorProbes || ((offset | size) & 3u) != 0u) return false;
keys.resize(size / 4u);
if (!ReadScalarTable(base, UINT64_MAX, offset, runtime, keys)) return false;
for (const auto word: keys)
for (uint32_t shift = 0; shift < 32u; shift += 8u)
selected[(word >> shift) & 255u] = true;
}
keys.clear();
for (uint32_t key = 0; key < selected.size(); ++key)
if (selected[key]) keys.push_back(key);
} else if (!indirect.selector_first.IsEmpty()) {
ShaderBufferResource material;
uint64_t base = 0, size = UINT64_MAX;
uint32_t first = 0, count = 0;
if (material_value.dword_count == 2u) {
if (selector->stride == 0u) return false;
base = (uint64_t {material_value.dwords[1]} << 32u) | material_value.dwords[0];
} else {
if (!DecodeBufferDescriptor(material_value, material)) return false;
base = material.Base48();
size = material.GetSize();
}
if (!clean.Evaluate(indirect.selector_first, first) ||
!evaluate_count(count) || count > MaxIndirectDescriptorProbes)
return false;
// SMEM uses its instruction's byte stride; V# stride only supplies its bounds.
const auto step = selector->stride != 0u ? selector->stride : material.Stride();
if (selector->stride == 0u &&
(material.Type() != 0u || material.SwizzleEnabled() || material.AddTid() ||
material.OutOfBounds() != 0u || uint64_t {selector->offset} + 4u > step ||
uint64_t {first} + count > material.NumRecords())) return false;
const auto last = count != 0u ? uint64_t {first} + count - 1u : 0u;
if (last > UINT32_MAX) return false;
const auto end = count != 0u ? last * step + selector->offset + 4u : 0u;
if (end > uint64_t {UINT32_MAX} + 1u ||
(indirect.workgroup_axis == UINT32_MAX && end > size))
return false;
if (!read_keys(base, size, uint64_t {first} * step + selector->offset, step, count))
return false;
} else if (!indirect.selector_mask.IsEmpty()) {
uint32_t mask = 0;
uint32_t count = 0;
if (material_value.dword_count != 2u || table_value.dword_count != 2u ||
!clean.Evaluate(indirect.selector_mask, mask) ||
!evaluate_count(count) || count == 0u || count > 32u) {
return false;
}
if (count < 32u) mask &= (1u << count) - 1u;
const auto material_base =
(static_cast<uint64_t>(material_value.dwords[1]) << 32u) | material_value.dwords[0];
keys.reserve(std::popcount(mask));
while (mask != 0u) {
const auto index = std::countr_zero(mask);
const auto offset = static_cast<uint64_t>(selector->offset) +
static_cast<uint64_t>(index) * selector->stride;
if (offset > UINT32_MAX) return false;
uint32_t key = 0;
if (!read_selector(material_base, UINT64_MAX, static_cast<uint32_t>(offset),
{&key, 1})) return false;
keys.push_back(key);
mask &= mask - 1u;
}
} else {
ShaderBufferResource material;
if (!DecodeBufferDescriptor(material_value, material) || selector->stride < 4u ||
!std::has_single_bit(selector->stride) || (selector->offset & 3u) != 0u)
return false;
// Empty selector bounds denote the complete U32 congruence class. SMEM
// adds its aligned immediate after the dynamic offset, without U32 wrap.
const auto domain_count = (uint64_t {1} << 32u) / selector->stride;
const auto available = material.GetSize() > selector->offset
? material.GetSize() - selector->offset : 0u;
const auto count = std::min(domain_count,
available < 4u ? 0u : (available - 4u) / selector->stride + 1u);
if (count > MaxIndirectDescriptorProbes ||
!read_keys(material.Base48(), material.GetSize(), selector->offset, selector->stride, count)) return false;
if (count < domain_count) keys.push_back(0u); // An out-of-bounds scalar read returns zero.
}
if (selector != nullptr && selector->byte_ranges.empty()) {
if (selector->bit_offset != 0u || selector->bit_mask != UINT32_MAX || selector->maximum != UINT32_MAX)
for (auto& key: keys)
key = std::min((key >> selector->bit_offset) & selector->bit_mask, selector->maximum);
std::ranges::sort(keys);
keys.erase(std::unique(keys.begin(), keys.end()), keys.end());
if (!indirect.selector_mask.IsEmpty() && table_value.dword_count != 2u) return false;
if (!selector_keys->resolved) {
if (!ResolveSelectorKeys(indirect, material_value, runtime, clean, keys)) return false;
selector_keys->resolved = true;
}
}
}
@@ -1270,6 +1290,25 @@ ResourcePlan ExtractResourcePlan(const Program& program) {
target.dwords[dword] = Clone(source.dwords[dword]);
}
}
// Destination tables share keys when their complete selector domains agree.
// Resolve those domains lazily at dispatch time, without repeated guest reads.
plan.selector_groups.resize(plan.descriptor_sources.size(), UINT32_MAX);
for (uint32_t source = 0; source < plan.descriptor_sources.size(); ++source) {
const auto& descriptor = program.descriptor_sources[source].indirect_descriptor;
if (!descriptor || !descriptor->selector) continue;
const auto& a = *descriptor;
uint32_t group = 0;
for (; group < plan.selector_keys.size(); ++group) {
const auto& b = *program.descriptor_sources[plan.selector_keys[group].source].indirect_descriptor;
if (a.selector == b.selector && a.signed_count == b.signed_count &&
a.workgroup_axis == b.workgroup_axis &&
EquivalentValue(program, a.key_count, b.key_count) &&
EquivalentValue(program, a.selector_first, b.selector_first) &&
EquivalentValue(program, a.selector_mask, b.selector_mask)) break;
}
if (group == plan.selector_keys.size()) plan.selector_keys.push_back({.source = source});
plan.selector_groups[source] = group;
}
plan.srt_reads.reserve(program.srt_reads.size());
for (const auto& read: program.srt_reads) {
plan.srt_reads.push_back({Clone(read.value), read.flat_offset});
@@ -1398,6 +1437,7 @@ ResourcePlan ExtractResourcePlan(const Program& program) {
bool MaterializeResources(const ResourcePlan& program, const SrtRuntime& runtime,
ResourceSnapshot& snapshot, ResourceSpecialization& specialization) {
for (auto& keys: program.selector_keys) keys.resolved = false;
if (!program.resource_tracking_complete ||
((program.requires_specialization_memory || !program.source_reads.empty()) &&
runtime.read_specialization_memory == nullptr)) {
@@ -1469,7 +1509,7 @@ bool MaterializeResources(const ResourcePlan& program, const SrtRuntime& runtime
snapshot.buffers[i] = {.dword_count = 4u};
if (active.empty() || active[base.source]) {
if (!MaterializeIndirectDescriptor(
program, *source->indirect_descriptor, i, 4u, 4u, observed, clean, snapshot,
program, base.source, i, 4u, 4u, observed, clean, snapshot,
snapshot.buffers, specialization.buffers, UINT32_MAX,
[&](DescriptorValue& value) {
return NormalizeIndirectBuffer(
@@ -1533,7 +1573,7 @@ bool MaterializeResources(const ResourcePlan& program, const SrtRuntime& runtime
const auto& indirect = *source->indirect_descriptor;
const bool bounded_table = IsBoundedDescriptorTable(program, indirect);
if (!MaterializeIndirectDescriptor(
program, indirect, i, 8u, image.r128 ? 4u : 8u, observed, clean, snapshot, snapshot.images,
program, image.source, i, 8u, image.r128 ? 4u : 8u, observed, clean, snapshot, snapshot.images,
specialization.images, UINT32_MAX, [&](DescriptorValue& value) {
// A broad heap also contains resources for other typed image operations.
if (NullImageDescriptor(value) || !ValidImageDescriptor(value, image.r128) ||
@@ -1566,7 +1606,7 @@ bool MaterializeResources(const ResourcePlan& program, const SrtRuntime& runtime
if (indirect != nullptr && indirect->sources.empty()) {
snapshot.samplers[i] = {.dword_count = 4u};
if ((active.empty() || active[source]) && !MaterializeIndirectDescriptor(
program, *indirect, i, 4u, 4u, observed, clean, snapshot, snapshot.samplers,
program, source, i, 4u, 4u, observed, clean, snapshot, snapshot.samplers,
specialization.samplers, ShaderInfo::MaxSamplers,
[](DescriptorValue&) { return true; })) return false;
} else {
tests/ResourceTrackingTests.cpp
+122 -21
View File
@@ -2994,7 +2994,9 @@ void TestUniformScalarBufferImage() {
void TestCompactScalarImageTable() {
using F = Libs::Graphics::Prospero::BufferFormat;
constexpr uint32_t stride = 0x330u, image_offset = 0x98u, sampler_offset = 0x88u;
const auto make_plan = [&](bool compact, uint32_t invalid = 0u) {
enum class Domain { Unbounded, SharedPrefix, SeparatePrefix, SeparateMask };
const auto make_plan = [&](bool compact, uint32_t invalid = 0u,
Domain domain = Domain::Unbounded) {
Fixture fixture;
const auto list = invalid == 1u
? fixture.Address(fixture.UserData(0), fixture.UserData(1))
@@ -3002,10 +3004,12 @@ void TestCompactScalarImageTable() {
fixture.UserData(2), fixture.UserData(3)});
const auto table = fixture.Buffer({fixture.UserData(4), fixture.UserData(5),
fixture.UserData(6), fixture.UserData(7)});
// Uniform on the GPU, but no CPU-evaluable row or workgroup-prefix proof.
const auto group = fixture.Emit(ValueOpcode::GetBuiltin,
{Value(static_cast<uint32_t>(StageInputKind::WorkgroupId)), Value(0u)});
const auto index = fixture.Emit(ValueOpcode::BitwiseXor32, {group, Value(1u)});
// The original case has no CPU-evaluable row or workgroup-prefix proof.
const auto index = domain == Domain::Unbounded
? fixture.Emit(ValueOpcode::BitwiseXor32, {group, Value(1u)})
: fixture.Emit(ValueOpcode::IAdd32, {group, fixture.UserData(8u)});
const auto offset = fixture.Emit(invalid == 2u ? ValueOpcode::UDiv32
: ValueOpcode::ShiftLeftLogical32,
{index, Value(2u)});
@@ -3016,23 +3020,27 @@ void TestCompactScalarImageTable() {
fixture.AddMemory(scalar, 0x570u))
: fixture.Emit(ValueOpcode::ReadConstBuffer, {list, offset},
fixture.AddMemory(scalar, 0x570u));
const auto record = fixture.Emit(ValueOpcode::IMul32, {key, Value(stride)});
scalar.kind = ResourceKind::ScalarBuffer;
std::array<Value, 8> image_words;
std::array<Value, 4> sampler_words;
for (uint32_t word = 0; word < 8u; ++word) {
scalar.offset = image_offset + word * 4u;
image_words[word] = compact && word >= 4u ? Value(0u)
: fixture.Emit(ValueOpcode::ReadConstBuffer, {table, record},
fixture.AddMemory(scalar, 0x590u));
if (word < 4u) {
scalar.offset = sampler_offset + word * 4u;
sampler_words[word] = fixture.Emit(ValueOpcode::ReadConstBuffer, {table, record},
fixture.AddMemory(scalar, 0x590u));
const auto make_image = [&](Value selected, uint32_t member) {
const auto record = fixture.Emit(ValueOpcode::IMul32, {selected, Value(stride)});
std::array<Value, 8> words;
for (uint32_t word = 0; word < 8u; ++word) {
scalar.offset = member + word * 4u;
words[word] = compact && word >= 4u ? Value(0u)
: fixture.Emit(ValueOpcode::ReadConstBuffer, {table, record},
fixture.AddMemory(scalar, 0x590u));
}
if (invalid == 3u) words[7] = Value(1u);
return fixture.Image(words, 0x59cu);
};
const auto image = make_image(key, image_offset);
const auto record = fixture.Emit(ValueOpcode::IMul32, {key, Value(stride)});
std::array<Value, 4> sampler_words;
for (uint32_t word = 0; word < 4u; ++word) {
scalar.offset = sampler_offset + word * 4u;
sampler_words[word] = fixture.Emit(ValueOpcode::ReadConstBuffer, {table, record},
fixture.AddMemory(scalar, 0x590u));
}
if (invalid == 3u) image_words[7] = Value(1u);
const auto image = fixture.Image(image_words, 0x59cu);
MemoryInfo sample;
sample.kind = ResourceKind::Image;
sample.image_dimension = Decoder::ImageDimension::Dim2D;
@@ -3040,6 +3048,19 @@ void TestCompactScalarImageTable() {
const auto sampler = fixture.Sampler(sampler_words, 0x59cu);
fixture.Emit(ValueOpcode::ImageSampleRaw,
{image, sampler, fixture.ImageAddress(), Value(true)}, fixture.AddMemory(sample, 0x59cu));
if (domain != Domain::Unbounded) {
// Distinct SSA expressions with equivalent runtime first values share a domain.
const auto second_offset = fixture.Emit(ValueOpcode::ShiftLeftLogical32,
{fixture.Emit(ValueOpcode::IAdd32, {group, fixture.UserData(8u)}), Value(2u)});
scalar.offset = domain == Domain::SeparatePrefix ? 4u : 0u;
auto second_key = fixture.Emit(ValueOpcode::ReadConstBuffer, {list, second_offset},
fixture.AddMemory(scalar, 0x580u));
if (domain == Domain::SeparateMask)
second_key = fixture.Emit(ValueOpcode::BitFieldUExtract, {second_key, Value(0u), Value(1u)});
fixture.Emit(ValueOpcode::ImageSampleRaw,
{make_image(second_key, 0x16cu), sampler, fixture.ImageAddress(), Value(true)},
fixture.AddMemory(sample, 0x79cu));
}
if (invalid == 4u) {
sample.image_r128 = false;
fixture.Emit(ValueOpcode::ImageSampleRaw,
@@ -3049,7 +3070,9 @@ void TestCompactScalarImageTable() {
const auto &source = fixture.program.descriptor_sources[fixture.program.info.samplers[0].source];
Check(source.indirect_descriptor && source.indirect_descriptor->selector &&
source.indirect_descriptor->selector->stride == 4u &&
source.indirect_descriptor->selector_first.IsEmpty() &&
(domain == Domain::Unbounded ? source.indirect_descriptor->selector_first.IsEmpty()
: !source.indirect_descriptor->selector_first.IsImmediate() &&
source.indirect_descriptor->workgroup_axis == 0u) &&
source.indirect_descriptor->key_count.IsEmpty() &&
source.indirect_descriptor->table_stride == stride &&
source.indirect_descriptor->table_immediate == sampler_offset,
@@ -3123,6 +3146,84 @@ void TestCompactScalarImageTable() {
Check(!MaterializeResources(plan, runtime, snapshot, specialization) && memory.selector_attempts == 0u,
"oversized index list was read before the existing probe cap");
}
for (const auto domain : {Domain::SharedPrefix, Domain::SeparatePrefix, Domain::SeparateMask}) {
auto plan = make_plan(true, 0u, domain);
constexpr uint32_t bytes = 2u * stride + 0x16cu + 16u;
std::array<uint32_t, 9> data{0x1000u, 0u, 8u, 0u, 0x2000u, 0u, bytes, 0u, 0u};
std::array<uint32_t, 3> counts{3u, 1u, 1u};
struct Memory : LinearTestMemory {
std::vector<std::pair<uint64_t, uint64_t>> selector_reads;
} memory;
memory.words.resize((0x1000u + bytes) / 4u, 0u);
memory.words[0] = 2u;
memory.words[1] = 1u;
for (uint32_t row = 0; row < 3u; ++row) {
memory.words[(0x1000u + row * stride + sampler_offset) / 4u] = row;
for (const auto [member, base] : {std::pair{image_offset, 0x100u},
std::pair{0x16cu, 0x200u}}) {
auto at = memory.words.begin() + (0x1000u + row * stride + member) / 4u;
at[0] = base + row;
at[1] = static_cast<uint32_t>(F::k32_32_32_32Float) << 20u;
at[3] = Libs::Graphics::DstSel(4, 5, 6, 7) |
(static_cast<uint32_t>(Libs::Graphics::Prospero::ImageType::kColor2D) << 28u);
}
}
const SrtRuntime runtime{.user_data = data, .userdata = &memory,
.read_specialization_memory = [](void *userdata, uint64_t address,
std::span<uint32_t> words) {
auto &memory = *static_cast<Memory *>(userdata);
if (address < 0x2000u) memory.selector_reads.emplace_back(address, words.size_bytes());
return ReadLinearTestMemory(&memory, address, words);
}, .workgroup_counts = counts};
ResourceSnapshot snapshot;
ResourceSpecialization specialization;
const auto materialize = [&](std::vector<uint32_t> expected, std::vector<uint32_t> expected_samplers) {
memory.selector_reads.clear();
Check(MaterializeResources(plan, runtime, snapshot, specialization),
"shared selector domain failed to materialize");
std::vector<uint32_t> images;
for (const auto &image : snapshot.images) images.push_back(image.dwords[0]);
std::ranges::sort(images);
std::vector<uint32_t> samplers;
for (const auto &sampler : snapshot.samplers) samplers.push_back(sampler.dwords[0]);
std::ranges::sort(samplers);
Check(images == expected && samplers == expected_samplers,
"shared selector domain mixed descriptor fields or retained stale keys");
std::vector<std::pair<uint64_t, uint64_t>> reads;
const auto expect_read = [&](uint32_t offset) {
offset += data[8] * 4u;
if (counts[0] != 0u && offset < data[2])
reads.emplace_back(0x1000u + offset, std::min(counts[0] * 4u, data[2] - offset));
};
expect_read(0u);
if (domain != Domain::SharedPrefix) expect_read(domain == Domain::SeparatePrefix ? 4u : 0u);
Check(memory.selector_reads == reads,
"selector domain was reread per resource or distinct domains were merged");
};
materialize(domain == Domain::SharedPrefix
? std::vector<uint32_t>{0x100u, 0x101u, 0x102u, 0x200u, 0x201u, 0x202u}
: std::vector<uint32_t>{0x100u, 0x101u, 0x102u, 0x200u, 0x201u}, {0u, 1u, 2u});
memory.words[0] = 1u;
materialize({0x100u, 0x101u, 0x200u, 0x201u}, {0u, 1u});
counts[0] = 1u;
materialize({0x101u, 0x201u}, {1u});
data[8] = 1u;
materialize({0x101u, domain == Domain::SeparatePrefix ? 0x200u : 0x201u}, {1u});
data[8] = 0u;
counts[0] = 0u;
materialize({0u, 0u}, {0u});
counts[0] = 3u;
data[2] = 0u;
materialize({0x100u, 0x200u}, {0u});
data[2] = 8u;
memory.fail_address = 0x1004u;
memory.selector_reads.clear();
Check(!MaterializeResources(plan, runtime, snapshot, specialization) &&
memory.selector_reads == std::vector<std::pair<uint64_t, uint64_t>>{{0x1000u, 8u}},
"shared selector reused a previous domain after a failed refresh");
memory.fail_address = UINT64_MAX;
materialize({0x100u, 0x101u, 0x200u, 0x201u}, {0u, 1u});
}
for (uint32_t invalid : {1u, 2u, 3u, 4u})
CheckFatal([&] { make_plan(true, invalid); }, "not a valid runtime value",
"compact table accepted an unproved selector or inconsistent descriptor width");
@@ -5393,7 +5494,7 @@ void TestWorkgroupSelectedBufferRecords() {
const bool empty = std::ranges::find(dimensions, 0u) != dimensions.end();
Check(materialize() && memory.watched_reads == 1u && memory.watched_dwords == 3u &&
snapshot.buffers.size() == plan.info.buffers.size() + (empty ? 0u : (dimensions[0] - 1u) * 2u) &&
memory.reads == (empty ? 1u : 3u + dimensions[0] * 2u),
memory.reads == (empty ? 1u : 2u + dimensions[0] * 2u),
"workgroup prefix reread arguments, visited an inactive key or retained stale candidates");
Check(std::ranges::find(snapshot.specialization_reads, std::pair{memory.base, uint64_t{12u}}) !=
snapshot.specialization_reads.end(),
@@ -5433,7 +5534,7 @@ void TestWorkgroupSelectedBufferRecords() {
std::fill_n(memory.words.begin() + selector / 4u, 8u, 0xdeadbeefu);
for (uint32_t index = 0; index < 4u; ++index)
memory.words[selector / 4u + ((index + 1u) * stride) / 4u] = index & 1u;
Check(materialize() && memory.reads == 7u &&
Check(materialize() && memory.reads == 6u &&
snapshot.buffers.size() == plan.info.buffers.size() + 2u,
"batched vector selector changed independent scalar DWORD alignment");
}
@@ -5446,7 +5547,7 @@ void TestWorkgroupSelectedBufferRecords() {
user_data[6] = MaxIndirectDescriptorProbes;
std::fill(memory.words.begin() + selector / 4u, memory.words.end(), 0u);
memory.words[0] = MaxIndirectDescriptorProbes;
Check(materialize() && memory.reads == 9u,
Check(materialize() && memory.reads == 6u,
"strided selector prefix was not read in bounded contiguous batches");
memory.words[0] = MaxIndirectDescriptorProbes + 1u;
Check(!materialize() && memory.reads == 1u,