diff --git a/src/graphics/shader/recompiler/ir/ShaderIR.h b/src/graphics/shader/recompiler/ir/ShaderIR.h index 9f2e3b321..ee22d9a53 100644 --- a/src/graphics/shader/recompiler/ir/ShaderIR.h +++ b/src/graphics/shader/recompiler/ir/ShaderIR.h @@ -614,6 +614,12 @@ struct UniformFillPlan { // Resource analysis retained by the shader cache. It owns immutable descriptor/SRT, // condition and fill values without translated blocks, plus reusable evaluation scratch. struct ResourcePlan { + struct SelectorKeys { + uint32_t source = UINT32_MAX; + std::vector values; + bool resolved = false; + }; + struct EvaluationContext { struct Entry { uint64_t value = 0; @@ -642,6 +648,7 @@ struct ResourcePlan { std::list value_storage; std::vector memory_info; std::vector descriptor_sources; + std::vector selector_groups; std::vector control_flow; std::unordered_map> control_phi_predecessors; std::vector srt_reads; @@ -661,6 +668,7 @@ struct ResourcePlan { mutable std::vector reachable_blocks; mutable std::vector pending_blocks; mutable std::vector material_keys; + mutable std::vector selector_keys; }; struct Program: ResourcePlan { diff --git a/src/graphics/shader/recompiler/ir/passes/ResourceMaterialization.cpp b/src/graphics/shader/recompiler/ir/passes/ResourceMaterialization.cpp index b6f85aed1..a6a8cb443 100644 --- a/src/graphics/shader/recompiler/ir/passes/ResourceMaterialization.cpp +++ b/src/graphics/shader/recompiler/ir/passes/ResourceMaterialization.cpp @@ -296,46 +296,24 @@ bool IsBoundedDescriptorTable(const ResourcePlan& program, table != nullptr && table->dword_count == 4u; } -template -bool MaterializeIndirectDescriptor(const ResourcePlan& program, - const DescriptorSource::IndirectDescriptor& indirect, - uint32_t resource_index, uint32_t dword_count, uint32_t read_dwords, - const SrtRuntime& runtime, SrtWalker& clean, - ResourceSnapshot& snapshot, - std::vector& descriptors, - std::vector& specializations, - uint32_t maximum_resources, Normalize&& normalize) { - const auto descriptor_bytes = read_dwords * sizeof(uint32_t); - const auto& sources = indirect.sources; - const auto* selector = indirect.selector ? &*indirect.selector : nullptr; - auto& keys = program.material_keys; - const auto children_begin = descriptors.size(); - const auto root_resource = specializations[resource_index]; - const auto evaluate_count = [&](uint32_t& count) { - if (indirect.workgroup_axis != UINT32_MAX) { - if (runtime.workgroup_counts.size() != 3u || indirect.workgroup_axis >= 3u) return false; - count = std::ranges::find(runtime.workgroup_counts, 0u) != runtime.workgroup_counts.end() - ? 0u : runtime.workgroup_counts[indirect.workgroup_axis]; - return true; - } - if (!clean.Evaluate(indirect.key_count, count)) return false; - if (indirect.signed_count && std::bit_cast(count) < 0) count = 0; +bool EvaluateIndirectCount(const DescriptorSource::IndirectDescriptor& indirect, + const SrtRuntime& runtime, SrtWalker& clean, uint32_t& count) { + if (indirect.workgroup_axis != UINT32_MAX) { + if (runtime.workgroup_counts.size() != 3u || indirect.workgroup_axis >= 3u) return false; + count = std::ranges::find(runtime.workgroup_counts, 0u) != runtime.workgroup_counts.end() + ? 0u : runtime.workgroup_counts[indirect.workgroup_axis]; return true; - }; - const auto intern_candidate = [&](const DescriptorValue& candidate, uint32_t& ordinal) { - ordinal = 0u; - if (candidate == descriptors[resource_index]) return true; - const auto found = std::find(descriptors.begin() + children_begin, descriptors.end(), candidate); - ordinal = static_cast(found - descriptors.begin() - children_begin + 1u); - if (found == descriptors.end()) { - if (descriptors.size() >= maximum_resources) return false; - descriptors.push_back(candidate); - auto child = root_resource; - child.indirect_root = resource_index; - specializations.push_back(child); - } - return true; - }; + } + if (!clean.Evaluate(indirect.key_count, count)) return false; + if (indirect.signed_count && std::bit_cast(count) < 0) count = 0; + return true; +} + +bool ResolveSelectorKeys(const DescriptorSource::IndirectDescriptor& indirect, + const DescriptorValue& material_value, const SrtRuntime& runtime, + SrtWalker& clean, std::vector& keys) { + const auto* selector = &*indirect.selector; + keys.clear(); const auto read_selector = [&](uint64_t base, uint64_t size, uint64_t offset, std::span words) { return selector->global ? ReadTable(base, size, offset, runtime, words) @@ -368,6 +346,207 @@ bool MaterializeIndirectDescriptor(const ResourcePlan& p keys.resize(count); return true; }; + if (selector->rows) { + const auto& rows = *selector->rows; + DescriptorValue count_value; + ShaderBufferResource counts, material; + if (!clean.EvaluateDescriptor(rows.source, count_value) || + !DecodeBufferDescriptor(count_value, counts) || + !DecodeBufferDescriptor(material_value, material) || selector->stride != 4u || + selector->offset != 0u || rows.stride == 0u || (rows.stride & 3u) != 0u || + counts.GetSize() / 4u > MaxIndirectDescriptorProbes) return false; + std::vector lengths(counts.GetSize() / 4u); + if (!ReadScalarTable(counts.Base48(), counts.GetSize(), 0u, runtime, lengths)) return false; + uint32_t total = 0; + for (size_t row = 0; row < lengths.size(); ++row) { + const auto length = lengths[row]; + if (length > MaxIndirectDescriptorProbes - total || + (length != 0u && uint64_t {row} * rows.stride + uint64_t {length} * 4u > + UINT32_MAX + 1ull)) return false; + total += length; + } + keys.reserve(total); + std::vector batch; + for (size_t row = 0; row < lengths.size();) { + if (lengths[row] == 0u) { ++row; continue; } + const auto first = uint64_t {row} * rows.stride / 4u; + auto end = first + lengths[row]; + size_t next = row + 1u; + for (; next < lengths.size(); ++next) { + if (lengths[next] == 0u) continue; + const auto candidate_end = + std::max(end, uint64_t {next} * rows.stride / 4u + lengths[next]); + if (candidate_end - first > MaxIndirectDescriptorProbes) break; + end = candidate_end; + } + // Coalesce small gaps, but interpret only selected prefixes. The strict + // reader captures the entire fetched span for writable-alias rejection. + batch.resize(end - first); + if (!ReadScalarTable(material.Base48(), material.GetSize(), first * 4u, runtime, batch)) + return false; + for (; row < next; ++row) { + if (lengths[row] == 0u) continue; + const auto begin = batch.begin() + (uint64_t {row} * rows.stride / 4u - first); + keys.insert(keys.end(), begin, begin + lengths[row]); + } + } + } else if (selector->bitset) { + ShaderBufferResource material; + std::array masks {}; + uint32_t first = 0, count = 0; + if (!DecodeBufferDescriptor(material_value, material) || + !clean.Evaluate(indirect.selector_first, first) || + !EvaluateIndirectCount(indirect, runtime, clean, count) || count != masks.size() * 32u || + selector->stride != sizeof(masks) || selector->offset != 0u || + material.Type() != 0u || material.Stride() != sizeof(masks) || + material.SwizzleEnabled() || material.AddTid() || material.OutOfBounds() != 0u || + (material.Base48() & 3u) != 0u) return false; + if (material.Format() != Prospero::BufferFormat::kInvalid) { + const auto words = material.GetSize() / sizeof(uint32_t); + // Bitmap storage can grow with resolution while its candidate domain stays + // at 128 keys. Bound scratch memory, preserving each coherent read's capture. + for (uint64_t first_word = 0; first_word < words; first_word += keys.size()) { + keys.resize(std::min(words - first_word, MaxIndirectDescriptorProbes)); + if (!ReadScalarTable(material.Base48(), material.GetSize(), first_word * 4u, + runtime, keys)) return false; + for (size_t word = 0; word < keys.size(); ++word) masks[word % masks.size()] |= keys[word]; + if (std::ranges::all_of(masks, [](uint32_t mask) { return mask == UINT32_MAX; })) break; + } + keys.clear(); + for (uint32_t word = 0; word < masks.size(); ++word) { + auto mask = masks[word]; + while (mask != 0u) { + const auto rank = word * 32u + std::countr_zero(mask); + keys.push_back(first + count - 1u - rank); + mask &= mask - 1u; + } + } + } + } else if (!selector->byte_ranges.empty()) { + if (material_value.dword_count != 2u) return false; + const auto base = (uint64_t {material_value.dwords[1]} << 32u) | material_value.dwords[0]; + std::array selected {}; + for (const auto [offset, size]: selector->byte_ranges) { + if (size > MaxIndirectDescriptorProbes || ((offset | size) & 3u) != 0u) return false; + keys.resize(size / 4u); + if (!ReadScalarTable(base, UINT64_MAX, offset, runtime, keys)) return false; + for (const auto word: keys) + for (uint32_t shift = 0; shift < 32u; shift += 8u) + selected[(word >> shift) & 255u] = true; + } + keys.clear(); + for (uint32_t key = 0; key < selected.size(); ++key) + if (selected[key]) keys.push_back(key); + } else if (!indirect.selector_first.IsEmpty()) { + ShaderBufferResource material; + uint64_t base = 0, size = UINT64_MAX; + uint32_t first = 0, count = 0; + if (material_value.dword_count == 2u) { + if (selector->stride == 0u) return false; + base = (uint64_t {material_value.dwords[1]} << 32u) | material_value.dwords[0]; + } else { + if (!DecodeBufferDescriptor(material_value, material)) return false; + base = material.Base48(); + size = material.GetSize(); + } + if (!clean.Evaluate(indirect.selector_first, first) || + !EvaluateIndirectCount(indirect, runtime, clean, count) || count > MaxIndirectDescriptorProbes) + return false; + // SMEM uses its instruction's byte stride; V# stride only supplies its bounds. + const auto step = selector->stride != 0u ? selector->stride : material.Stride(); + if (selector->stride == 0u && + (material.Type() != 0u || material.SwizzleEnabled() || material.AddTid() || + material.OutOfBounds() != 0u || uint64_t {selector->offset} + 4u > step || + uint64_t {first} + count > material.NumRecords())) return false; + const auto last = count != 0u ? uint64_t {first} + count - 1u : 0u; + if (last > UINT32_MAX) return false; + const auto end = count != 0u ? last * step + selector->offset + 4u : 0u; + if (end > uint64_t {UINT32_MAX} + 1u || + (indirect.workgroup_axis == UINT32_MAX && end > size)) + return false; + if (!read_keys(base, size, uint64_t {first} * step + selector->offset, step, count)) + return false; + } else if (!indirect.selector_mask.IsEmpty()) { + uint32_t mask = 0; + uint32_t count = 0; + if (material_value.dword_count != 2u || + !clean.Evaluate(indirect.selector_mask, mask) || + !EvaluateIndirectCount(indirect, runtime, clean, count) || count == 0u || count > 32u) { + return false; + } + if (count < 32u) mask &= (1u << count) - 1u; + const auto material_base = + (static_cast(material_value.dwords[1]) << 32u) | material_value.dwords[0]; + keys.reserve(std::popcount(mask)); + while (mask != 0u) { + const auto index = std::countr_zero(mask); + const auto offset = static_cast(selector->offset) + + static_cast(index) * selector->stride; + if (offset > UINT32_MAX) return false; + uint32_t key = 0; + if (!read_selector(material_base, UINT64_MAX, static_cast(offset), + {&key, 1})) return false; + keys.push_back(key); + mask &= mask - 1u; + } + } else { + ShaderBufferResource material; + if (!DecodeBufferDescriptor(material_value, material) || selector->stride < 4u || + !std::has_single_bit(selector->stride) || (selector->offset & 3u) != 0u) + return false; + // Empty selector bounds denote the complete U32 congruence class. SMEM + // adds its aligned immediate after the dynamic offset, without U32 wrap. + const auto domain_count = (uint64_t {1} << 32u) / selector->stride; + const auto available = material.GetSize() > selector->offset + ? material.GetSize() - selector->offset : 0u; + const auto count = std::min(domain_count, + available < 4u ? 0u : (available - 4u) / selector->stride + 1u); + if (count > MaxIndirectDescriptorProbes || + !read_keys(material.Base48(), material.GetSize(), selector->offset, selector->stride, count)) return false; + if (count < domain_count) keys.push_back(0u); // An out-of-bounds scalar read returns zero. + } + if (selector->byte_ranges.empty()) { + if (selector->bit_offset != 0u || selector->bit_mask != UINT32_MAX || selector->maximum != UINT32_MAX) + for (auto& key: keys) + key = std::min((key >> selector->bit_offset) & selector->bit_mask, selector->maximum); + std::ranges::sort(keys); + keys.erase(std::unique(keys.begin(), keys.end()), keys.end()); + } + return true; +} + +template +bool MaterializeIndirectDescriptor(const ResourcePlan& program, + uint32_t source_index, + uint32_t resource_index, uint32_t dword_count, uint32_t read_dwords, + const SrtRuntime& runtime, SrtWalker& clean, + ResourceSnapshot& snapshot, + std::vector& descriptors, + std::vector& specializations, + uint32_t maximum_resources, Normalize&& normalize) { + const auto& indirect = *program.descriptor_sources[source_index].indirect_descriptor; + const auto descriptor_bytes = read_dwords * sizeof(uint32_t); + const auto& sources = indirect.sources; + const auto* selector = indirect.selector ? &*indirect.selector : nullptr; + auto* selector_keys = selector != nullptr + ? &program.selector_keys[program.selector_groups[source_index]] : nullptr; + auto& keys = selector_keys != nullptr ? selector_keys->values : program.material_keys; + const auto children_begin = descriptors.size(); + const auto root_resource = specializations[resource_index]; + const auto intern_candidate = [&](const DescriptorValue& candidate, uint32_t& ordinal) { + ordinal = 0u; + if (candidate == descriptors[resource_index]) return true; + const auto found = std::find(descriptors.begin() + children_begin, descriptors.end(), candidate); + ordinal = static_cast(found - descriptors.begin() - children_begin + 1u); + if (found == descriptors.end()) { + if (descriptors.size() >= maximum_resources) return false; + descriptors.push_back(candidate); + auto child = root_resource; + child.indirect_root = resource_index; + specializations.push_back(child); + } + return true; + }; uint64_t table_base = 0; uint64_t table_size = UINT64_MAX; // Scalar addresses have no buffer descriptor bounds. const auto normalization_failed = [&](uint64_t offset, uint32_t key, uint64_t step, @@ -381,10 +560,10 @@ bool MaterializeIndirectDescriptor(const ResourcePlan& p indirect.table_offset, indirect.table_immediate)); }; if (sources.empty()) { - keys.clear(); + if (selector == nullptr) keys.clear(); DescriptorValue material_value; DescriptorValue table_value; - if ((selector != nullptr && + if ((selector != nullptr && !selector_keys->resolved && !clean.EvaluateDescriptor(selector->source, material_value)) || !clean.EvaluateDescriptor(indirect.table_source, table_value)) { return false; @@ -476,7 +655,7 @@ bool MaterializeIndirectDescriptor(const ResourcePlan& p if (selector == nullptr) { uint32_t key_count = 0; uint32_t first_key = 0; - if (!evaluate_count(key_count)) return false; + if (!EvaluateIndirectCount(indirect, runtime, clean, key_count)) return false; if (!indirect.selector_first.IsEmpty() && !clean.Evaluate(indirect.selector_first, first_key)) return false; if (indirect.table_stride == 0u || key_count > MaxIndirectDescriptorProbes || @@ -490,171 +669,12 @@ bool MaterializeIndirectDescriptor(const ResourcePlan& p std::iota(keys.begin(), keys.end(), first_key); if (!keys.empty() && keys.back() < keys.front()) std::rotate(keys.begin(), keys.begin() + (UINT32_MAX - first_key + 1ull), keys.end()); - } else if (selector->rows) { - const auto& rows = *selector->rows; - DescriptorValue count_value; - ShaderBufferResource counts, material; - if (!clean.EvaluateDescriptor(rows.source, count_value) || - !DecodeBufferDescriptor(count_value, counts) || - !DecodeBufferDescriptor(material_value, material) || selector->stride != 4u || - selector->offset != 0u || rows.stride == 0u || (rows.stride & 3u) != 0u || - counts.GetSize() / 4u > MaxIndirectDescriptorProbes) return false; - std::vector lengths(counts.GetSize() / 4u); - if (!ReadScalarTable(counts.Base48(), counts.GetSize(), 0u, runtime, lengths)) return false; - uint32_t total = 0; - for (size_t row = 0; row < lengths.size(); ++row) { - const auto length = lengths[row]; - if (length > MaxIndirectDescriptorProbes - total || - (length != 0u && uint64_t {row} * rows.stride + uint64_t {length} * 4u > - UINT32_MAX + 1ull)) return false; - total += length; - } - keys.reserve(total); - std::vector batch; - for (size_t row = 0; row < lengths.size();) { - if (lengths[row] == 0u) { ++row; continue; } - const auto first = uint64_t {row} * rows.stride / 4u; - auto end = first + lengths[row]; - size_t next = row + 1u; - for (; next < lengths.size(); ++next) { - if (lengths[next] == 0u) continue; - const auto candidate_end = - std::max(end, uint64_t {next} * rows.stride / 4u + lengths[next]); - if (candidate_end - first > MaxIndirectDescriptorProbes) break; - end = candidate_end; - } - // Coalesce small gaps, but interpret only selected prefixes. The strict - // reader captures the entire fetched span for writable-alias rejection. - batch.resize(end - first); - if (!ReadScalarTable(material.Base48(), material.GetSize(), first * 4u, runtime, batch)) - return false; - for (; row < next; ++row) { - if (lengths[row] == 0u) continue; - const auto begin = batch.begin() + (uint64_t {row} * rows.stride / 4u - first); - keys.insert(keys.end(), begin, begin + lengths[row]); - } - } - } else if (selector->bitset) { - ShaderBufferResource material; - std::array masks {}; - uint32_t first = 0, count = 0; - if (!DecodeBufferDescriptor(material_value, material) || - !clean.Evaluate(indirect.selector_first, first) || - !evaluate_count(count) || count != masks.size() * 32u || - selector->stride != sizeof(masks) || selector->offset != 0u || - material.Type() != 0u || material.Stride() != sizeof(masks) || - material.SwizzleEnabled() || material.AddTid() || material.OutOfBounds() != 0u || - (material.Base48() & 3u) != 0u) return false; - if (material.Format() != Prospero::BufferFormat::kInvalid) { - const auto words = material.GetSize() / sizeof(uint32_t); - // Bitmap storage can grow with resolution while its candidate domain stays - // at 128 keys. Bound scratch memory, preserving each coherent read's capture. - for (uint64_t first_word = 0; first_word < words; first_word += keys.size()) { - keys.resize(std::min(words - first_word, MaxIndirectDescriptorProbes)); - if (!ReadScalarTable(material.Base48(), material.GetSize(), first_word * 4u, - runtime, keys)) return false; - for (size_t word = 0; word < keys.size(); ++word) masks[word % masks.size()] |= keys[word]; - if (std::ranges::all_of(masks, [](uint32_t mask) { return mask == UINT32_MAX; })) break; - } - keys.clear(); - for (uint32_t word = 0; word < masks.size(); ++word) { - auto mask = masks[word]; - while (mask != 0u) { - const auto rank = word * 32u + std::countr_zero(mask); - keys.push_back(first + count - 1u - rank); - mask &= mask - 1u; - } - } - } - } else if (!selector->byte_ranges.empty()) { - if (material_value.dword_count != 2u) return false; - const auto base = (uint64_t {material_value.dwords[1]} << 32u) | material_value.dwords[0]; - std::array selected {}; - for (const auto [offset, size]: selector->byte_ranges) { - if (size > MaxIndirectDescriptorProbes || ((offset | size) & 3u) != 0u) return false; - keys.resize(size / 4u); - if (!ReadScalarTable(base, UINT64_MAX, offset, runtime, keys)) return false; - for (const auto word: keys) - for (uint32_t shift = 0; shift < 32u; shift += 8u) - selected[(word >> shift) & 255u] = true; - } - keys.clear(); - for (uint32_t key = 0; key < selected.size(); ++key) - if (selected[key]) keys.push_back(key); - } else if (!indirect.selector_first.IsEmpty()) { - ShaderBufferResource material; - uint64_t base = 0, size = UINT64_MAX; - uint32_t first = 0, count = 0; - if (material_value.dword_count == 2u) { - if (selector->stride == 0u) return false; - base = (uint64_t {material_value.dwords[1]} << 32u) | material_value.dwords[0]; - } else { - if (!DecodeBufferDescriptor(material_value, material)) return false; - base = material.Base48(); - size = material.GetSize(); - } - if (!clean.Evaluate(indirect.selector_first, first) || - !evaluate_count(count) || count > MaxIndirectDescriptorProbes) - return false; - // SMEM uses its instruction's byte stride; V# stride only supplies its bounds. - const auto step = selector->stride != 0u ? selector->stride : material.Stride(); - if (selector->stride == 0u && - (material.Type() != 0u || material.SwizzleEnabled() || material.AddTid() || - material.OutOfBounds() != 0u || uint64_t {selector->offset} + 4u > step || - uint64_t {first} + count > material.NumRecords())) return false; - const auto last = count != 0u ? uint64_t {first} + count - 1u : 0u; - if (last > UINT32_MAX) return false; - const auto end = count != 0u ? last * step + selector->offset + 4u : 0u; - if (end > uint64_t {UINT32_MAX} + 1u || - (indirect.workgroup_axis == UINT32_MAX && end > size)) - return false; - if (!read_keys(base, size, uint64_t {first} * step + selector->offset, step, count)) - return false; - } else if (!indirect.selector_mask.IsEmpty()) { - uint32_t mask = 0; - uint32_t count = 0; - if (material_value.dword_count != 2u || table_value.dword_count != 2u || - !clean.Evaluate(indirect.selector_mask, mask) || - !evaluate_count(count) || count == 0u || count > 32u) { - return false; - } - if (count < 32u) mask &= (1u << count) - 1u; - const auto material_base = - (static_cast(material_value.dwords[1]) << 32u) | material_value.dwords[0]; - keys.reserve(std::popcount(mask)); - while (mask != 0u) { - const auto index = std::countr_zero(mask); - const auto offset = static_cast(selector->offset) + - static_cast(index) * selector->stride; - if (offset > UINT32_MAX) return false; - uint32_t key = 0; - if (!read_selector(material_base, UINT64_MAX, static_cast(offset), - {&key, 1})) return false; - keys.push_back(key); - mask &= mask - 1u; - } } else { - ShaderBufferResource material; - if (!DecodeBufferDescriptor(material_value, material) || selector->stride < 4u || - !std::has_single_bit(selector->stride) || (selector->offset & 3u) != 0u) - return false; - // Empty selector bounds denote the complete U32 congruence class. SMEM - // adds its aligned immediate after the dynamic offset, without U32 wrap. - const auto domain_count = (uint64_t {1} << 32u) / selector->stride; - const auto available = material.GetSize() > selector->offset - ? material.GetSize() - selector->offset : 0u; - const auto count = std::min(domain_count, - available < 4u ? 0u : (available - 4u) / selector->stride + 1u); - if (count > MaxIndirectDescriptorProbes || - !read_keys(material.Base48(), material.GetSize(), selector->offset, selector->stride, count)) return false; - if (count < domain_count) keys.push_back(0u); // An out-of-bounds scalar read returns zero. - } - if (selector != nullptr && selector->byte_ranges.empty()) { - if (selector->bit_offset != 0u || selector->bit_mask != UINT32_MAX || selector->maximum != UINT32_MAX) - for (auto& key: keys) - key = std::min((key >> selector->bit_offset) & selector->bit_mask, selector->maximum); - std::ranges::sort(keys); - keys.erase(std::unique(keys.begin(), keys.end()), keys.end()); + if (!indirect.selector_mask.IsEmpty() && table_value.dword_count != 2u) return false; + if (!selector_keys->resolved) { + if (!ResolveSelectorKeys(indirect, material_value, runtime, clean, keys)) return false; + selector_keys->resolved = true; + } } } @@ -1270,6 +1290,25 @@ ResourcePlan ExtractResourcePlan(const Program& program) { target.dwords[dword] = Clone(source.dwords[dword]); } } + // Destination tables share keys when their complete selector domains agree. + // Resolve those domains lazily at dispatch time, without repeated guest reads. + plan.selector_groups.resize(plan.descriptor_sources.size(), UINT32_MAX); + for (uint32_t source = 0; source < plan.descriptor_sources.size(); ++source) { + const auto& descriptor = program.descriptor_sources[source].indirect_descriptor; + if (!descriptor || !descriptor->selector) continue; + const auto& a = *descriptor; + uint32_t group = 0; + for (; group < plan.selector_keys.size(); ++group) { + const auto& b = *program.descriptor_sources[plan.selector_keys[group].source].indirect_descriptor; + if (a.selector == b.selector && a.signed_count == b.signed_count && + a.workgroup_axis == b.workgroup_axis && + EquivalentValue(program, a.key_count, b.key_count) && + EquivalentValue(program, a.selector_first, b.selector_first) && + EquivalentValue(program, a.selector_mask, b.selector_mask)) break; + } + if (group == plan.selector_keys.size()) plan.selector_keys.push_back({.source = source}); + plan.selector_groups[source] = group; + } plan.srt_reads.reserve(program.srt_reads.size()); for (const auto& read: program.srt_reads) { plan.srt_reads.push_back({Clone(read.value), read.flat_offset}); @@ -1398,6 +1437,7 @@ ResourcePlan ExtractResourcePlan(const Program& program) { bool MaterializeResources(const ResourcePlan& program, const SrtRuntime& runtime, ResourceSnapshot& snapshot, ResourceSpecialization& specialization) { + for (auto& keys: program.selector_keys) keys.resolved = false; if (!program.resource_tracking_complete || ((program.requires_specialization_memory || !program.source_reads.empty()) && runtime.read_specialization_memory == nullptr)) { @@ -1469,7 +1509,7 @@ bool MaterializeResources(const ResourcePlan& program, const SrtRuntime& runtime snapshot.buffers[i] = {.dword_count = 4u}; if (active.empty() || active[base.source]) { if (!MaterializeIndirectDescriptor( - program, *source->indirect_descriptor, i, 4u, 4u, observed, clean, snapshot, + program, base.source, i, 4u, 4u, observed, clean, snapshot, snapshot.buffers, specialization.buffers, UINT32_MAX, [&](DescriptorValue& value) { return NormalizeIndirectBuffer( @@ -1533,7 +1573,7 @@ bool MaterializeResources(const ResourcePlan& program, const SrtRuntime& runtime const auto& indirect = *source->indirect_descriptor; const bool bounded_table = IsBoundedDescriptorTable(program, indirect); if (!MaterializeIndirectDescriptor( - program, indirect, i, 8u, image.r128 ? 4u : 8u, observed, clean, snapshot, snapshot.images, + program, image.source, i, 8u, image.r128 ? 4u : 8u, observed, clean, snapshot, snapshot.images, specialization.images, UINT32_MAX, [&](DescriptorValue& value) { // A broad heap also contains resources for other typed image operations. if (NullImageDescriptor(value) || !ValidImageDescriptor(value, image.r128) || @@ -1566,7 +1606,7 @@ bool MaterializeResources(const ResourcePlan& program, const SrtRuntime& runtime if (indirect != nullptr && indirect->sources.empty()) { snapshot.samplers[i] = {.dword_count = 4u}; if ((active.empty() || active[source]) && !MaterializeIndirectDescriptor( - program, *indirect, i, 4u, 4u, observed, clean, snapshot, snapshot.samplers, + program, source, i, 4u, 4u, observed, clean, snapshot, snapshot.samplers, specialization.samplers, ShaderInfo::MaxSamplers, [](DescriptorValue&) { return true; })) return false; } else { diff --git a/tests/ResourceTrackingTests.cpp b/tests/ResourceTrackingTests.cpp index f04860ad4..465fe81e4 100644 --- a/tests/ResourceTrackingTests.cpp +++ b/tests/ResourceTrackingTests.cpp @@ -2994,7 +2994,9 @@ void TestUniformScalarBufferImage() { void TestCompactScalarImageTable() { using F = Libs::Graphics::Prospero::BufferFormat; constexpr uint32_t stride = 0x330u, image_offset = 0x98u, sampler_offset = 0x88u; - const auto make_plan = [&](bool compact, uint32_t invalid = 0u) { + enum class Domain { Unbounded, SharedPrefix, SeparatePrefix, SeparateMask }; + const auto make_plan = [&](bool compact, uint32_t invalid = 0u, + Domain domain = Domain::Unbounded) { Fixture fixture; const auto list = invalid == 1u ? fixture.Address(fixture.UserData(0), fixture.UserData(1)) @@ -3002,10 +3004,12 @@ void TestCompactScalarImageTable() { fixture.UserData(2), fixture.UserData(3)}); const auto table = fixture.Buffer({fixture.UserData(4), fixture.UserData(5), fixture.UserData(6), fixture.UserData(7)}); - // Uniform on the GPU, but no CPU-evaluable row or workgroup-prefix proof. const auto group = fixture.Emit(ValueOpcode::GetBuiltin, {Value(static_cast(StageInputKind::WorkgroupId)), Value(0u)}); - const auto index = fixture.Emit(ValueOpcode::BitwiseXor32, {group, Value(1u)}); + // The original case has no CPU-evaluable row or workgroup-prefix proof. + const auto index = domain == Domain::Unbounded + ? fixture.Emit(ValueOpcode::BitwiseXor32, {group, Value(1u)}) + : fixture.Emit(ValueOpcode::IAdd32, {group, fixture.UserData(8u)}); const auto offset = fixture.Emit(invalid == 2u ? ValueOpcode::UDiv32 : ValueOpcode::ShiftLeftLogical32, {index, Value(2u)}); @@ -3016,23 +3020,27 @@ void TestCompactScalarImageTable() { fixture.AddMemory(scalar, 0x570u)) : fixture.Emit(ValueOpcode::ReadConstBuffer, {list, offset}, fixture.AddMemory(scalar, 0x570u)); - const auto record = fixture.Emit(ValueOpcode::IMul32, {key, Value(stride)}); scalar.kind = ResourceKind::ScalarBuffer; - std::array image_words; - std::array sampler_words; - for (uint32_t word = 0; word < 8u; ++word) { - scalar.offset = image_offset + word * 4u; - image_words[word] = compact && word >= 4u ? Value(0u) - : fixture.Emit(ValueOpcode::ReadConstBuffer, {table, record}, - fixture.AddMemory(scalar, 0x590u)); - if (word < 4u) { - scalar.offset = sampler_offset + word * 4u; - sampler_words[word] = fixture.Emit(ValueOpcode::ReadConstBuffer, {table, record}, - fixture.AddMemory(scalar, 0x590u)); + const auto make_image = [&](Value selected, uint32_t member) { + const auto record = fixture.Emit(ValueOpcode::IMul32, {selected, Value(stride)}); + std::array words; + for (uint32_t word = 0; word < 8u; ++word) { + scalar.offset = member + word * 4u; + words[word] = compact && word >= 4u ? Value(0u) + : fixture.Emit(ValueOpcode::ReadConstBuffer, {table, record}, + fixture.AddMemory(scalar, 0x590u)); } + if (invalid == 3u) words[7] = Value(1u); + return fixture.Image(words, 0x59cu); + }; + const auto image = make_image(key, image_offset); + const auto record = fixture.Emit(ValueOpcode::IMul32, {key, Value(stride)}); + std::array sampler_words; + for (uint32_t word = 0; word < 4u; ++word) { + scalar.offset = sampler_offset + word * 4u; + sampler_words[word] = fixture.Emit(ValueOpcode::ReadConstBuffer, {table, record}, + fixture.AddMemory(scalar, 0x590u)); } - if (invalid == 3u) image_words[7] = Value(1u); - const auto image = fixture.Image(image_words, 0x59cu); MemoryInfo sample; sample.kind = ResourceKind::Image; sample.image_dimension = Decoder::ImageDimension::Dim2D; @@ -3040,6 +3048,19 @@ void TestCompactScalarImageTable() { const auto sampler = fixture.Sampler(sampler_words, 0x59cu); fixture.Emit(ValueOpcode::ImageSampleRaw, {image, sampler, fixture.ImageAddress(), Value(true)}, fixture.AddMemory(sample, 0x59cu)); + if (domain != Domain::Unbounded) { + // Distinct SSA expressions with equivalent runtime first values share a domain. + const auto second_offset = fixture.Emit(ValueOpcode::ShiftLeftLogical32, + {fixture.Emit(ValueOpcode::IAdd32, {group, fixture.UserData(8u)}), Value(2u)}); + scalar.offset = domain == Domain::SeparatePrefix ? 4u : 0u; + auto second_key = fixture.Emit(ValueOpcode::ReadConstBuffer, {list, second_offset}, + fixture.AddMemory(scalar, 0x580u)); + if (domain == Domain::SeparateMask) + second_key = fixture.Emit(ValueOpcode::BitFieldUExtract, {second_key, Value(0u), Value(1u)}); + fixture.Emit(ValueOpcode::ImageSampleRaw, + {make_image(second_key, 0x16cu), sampler, fixture.ImageAddress(), Value(true)}, + fixture.AddMemory(sample, 0x79cu)); + } if (invalid == 4u) { sample.image_r128 = false; fixture.Emit(ValueOpcode::ImageSampleRaw, @@ -3049,7 +3070,9 @@ void TestCompactScalarImageTable() { const auto &source = fixture.program.descriptor_sources[fixture.program.info.samplers[0].source]; Check(source.indirect_descriptor && source.indirect_descriptor->selector && source.indirect_descriptor->selector->stride == 4u && - source.indirect_descriptor->selector_first.IsEmpty() && + (domain == Domain::Unbounded ? source.indirect_descriptor->selector_first.IsEmpty() + : !source.indirect_descriptor->selector_first.IsImmediate() && + source.indirect_descriptor->workgroup_axis == 0u) && source.indirect_descriptor->key_count.IsEmpty() && source.indirect_descriptor->table_stride == stride && source.indirect_descriptor->table_immediate == sampler_offset, @@ -3123,6 +3146,84 @@ void TestCompactScalarImageTable() { Check(!MaterializeResources(plan, runtime, snapshot, specialization) && memory.selector_attempts == 0u, "oversized index list was read before the existing probe cap"); } + for (const auto domain : {Domain::SharedPrefix, Domain::SeparatePrefix, Domain::SeparateMask}) { + auto plan = make_plan(true, 0u, domain); + constexpr uint32_t bytes = 2u * stride + 0x16cu + 16u; + std::array data{0x1000u, 0u, 8u, 0u, 0x2000u, 0u, bytes, 0u, 0u}; + std::array counts{3u, 1u, 1u}; + struct Memory : LinearTestMemory { + std::vector> selector_reads; + } memory; + memory.words.resize((0x1000u + bytes) / 4u, 0u); + memory.words[0] = 2u; + memory.words[1] = 1u; + for (uint32_t row = 0; row < 3u; ++row) { + memory.words[(0x1000u + row * stride + sampler_offset) / 4u] = row; + for (const auto [member, base] : {std::pair{image_offset, 0x100u}, + std::pair{0x16cu, 0x200u}}) { + auto at = memory.words.begin() + (0x1000u + row * stride + member) / 4u; + at[0] = base + row; + at[1] = static_cast(F::k32_32_32_32Float) << 20u; + at[3] = Libs::Graphics::DstSel(4, 5, 6, 7) | + (static_cast(Libs::Graphics::Prospero::ImageType::kColor2D) << 28u); + } + } + const SrtRuntime runtime{.user_data = data, .userdata = &memory, + .read_specialization_memory = [](void *userdata, uint64_t address, + std::span words) { + auto &memory = *static_cast(userdata); + if (address < 0x2000u) memory.selector_reads.emplace_back(address, words.size_bytes()); + return ReadLinearTestMemory(&memory, address, words); + }, .workgroup_counts = counts}; + ResourceSnapshot snapshot; + ResourceSpecialization specialization; + const auto materialize = [&](std::vector expected, std::vector expected_samplers) { + memory.selector_reads.clear(); + Check(MaterializeResources(plan, runtime, snapshot, specialization), + "shared selector domain failed to materialize"); + std::vector images; + for (const auto &image : snapshot.images) images.push_back(image.dwords[0]); + std::ranges::sort(images); + std::vector samplers; + for (const auto &sampler : snapshot.samplers) samplers.push_back(sampler.dwords[0]); + std::ranges::sort(samplers); + Check(images == expected && samplers == expected_samplers, + "shared selector domain mixed descriptor fields or retained stale keys"); + std::vector> reads; + const auto expect_read = [&](uint32_t offset) { + offset += data[8] * 4u; + if (counts[0] != 0u && offset < data[2]) + reads.emplace_back(0x1000u + offset, std::min(counts[0] * 4u, data[2] - offset)); + }; + expect_read(0u); + if (domain != Domain::SharedPrefix) expect_read(domain == Domain::SeparatePrefix ? 4u : 0u); + Check(memory.selector_reads == reads, + "selector domain was reread per resource or distinct domains were merged"); + }; + materialize(domain == Domain::SharedPrefix + ? std::vector{0x100u, 0x101u, 0x102u, 0x200u, 0x201u, 0x202u} + : std::vector{0x100u, 0x101u, 0x102u, 0x200u, 0x201u}, {0u, 1u, 2u}); + memory.words[0] = 1u; + materialize({0x100u, 0x101u, 0x200u, 0x201u}, {0u, 1u}); + counts[0] = 1u; + materialize({0x101u, 0x201u}, {1u}); + data[8] = 1u; + materialize({0x101u, domain == Domain::SeparatePrefix ? 0x200u : 0x201u}, {1u}); + data[8] = 0u; + counts[0] = 0u; + materialize({0u, 0u}, {0u}); + counts[0] = 3u; + data[2] = 0u; + materialize({0x100u, 0x200u}, {0u}); + data[2] = 8u; + memory.fail_address = 0x1004u; + memory.selector_reads.clear(); + Check(!MaterializeResources(plan, runtime, snapshot, specialization) && + memory.selector_reads == std::vector>{{0x1000u, 8u}}, + "shared selector reused a previous domain after a failed refresh"); + memory.fail_address = UINT64_MAX; + materialize({0x100u, 0x101u, 0x200u, 0x201u}, {0u, 1u}); + } for (uint32_t invalid : {1u, 2u, 3u, 4u}) CheckFatal([&] { make_plan(true, invalid); }, "not a valid runtime value", "compact table accepted an unproved selector or inconsistent descriptor width"); @@ -5393,7 +5494,7 @@ void TestWorkgroupSelectedBufferRecords() { const bool empty = std::ranges::find(dimensions, 0u) != dimensions.end(); Check(materialize() && memory.watched_reads == 1u && memory.watched_dwords == 3u && snapshot.buffers.size() == plan.info.buffers.size() + (empty ? 0u : (dimensions[0] - 1u) * 2u) && - memory.reads == (empty ? 1u : 3u + dimensions[0] * 2u), + memory.reads == (empty ? 1u : 2u + dimensions[0] * 2u), "workgroup prefix reread arguments, visited an inactive key or retained stale candidates"); Check(std::ranges::find(snapshot.specialization_reads, std::pair{memory.base, uint64_t{12u}}) != snapshot.specialization_reads.end(), @@ -5433,7 +5534,7 @@ void TestWorkgroupSelectedBufferRecords() { std::fill_n(memory.words.begin() + selector / 4u, 8u, 0xdeadbeefu); for (uint32_t index = 0; index < 4u; ++index) memory.words[selector / 4u + ((index + 1u) * stride) / 4u] = index & 1u; - Check(materialize() && memory.reads == 7u && + Check(materialize() && memory.reads == 6u && snapshot.buffers.size() == plan.info.buffers.size() + 2u, "batched vector selector changed independent scalar DWORD alignment"); } @@ -5446,7 +5547,7 @@ void TestWorkgroupSelectedBufferRecords() { user_data[6] = MaxIndirectDescriptorProbes; std::fill(memory.words.begin() + selector / 4u, memory.words.end(), 0u); memory.words[0] = MaxIndirectDescriptorProbes; - Check(materialize() && memory.reads == 9u, + Check(materialize() && memory.reads == 6u, "strided selector prefix was not read in bounded contiguous batches"); memory.words[0] = MaxIndirectDescriptorProbes + 1u; Check(!materialize() && memory.reads == 1u,