recognize verified pattern fills

Co-authored-by: Juan Pablo Farias <juanpablo.profissional.a.f@gmail.com>
This commit is contained in:
nmzik
2026-10-17 01:42:34 +02:00
parent 32587337b5
commit 9202bddad2
7 changed files with 276 additions and 11 deletions
@@ -94,9 +94,10 @@ bool ResolveComputeBufferFill(const ShaderComputeInputInfo& input, uint32_t grou
constexpr std::array formats {
Prospero::BufferFormat::k32UInt, Prospero::BufferFormat::k32_32UInt,
Prospero::BufferFormat::k32_32_32UInt, Prospero::BufferFormat::k32_32_32_32UInt};
// Offset-or-swizzle bounds do not describe the element range proved by the shader.
if (descriptor.Stride() != element_size || descriptor.Format() != formats[fill.words - 1] ||
descriptor.SwizzleEnabled() || descriptor.IndexStride() != 0 || descriptor.AddTid() ||
descriptor.Base48() == 0) {
descriptor.OutOfBounds() == 3 || descriptor.Base48() == 0) {
return false;
}
if (input.threads_num[0] == 0 || input.threads_num[0] != fill.group_stride[0] ||
@@ -108,7 +109,11 @@ bool ResolveComputeBufferFill(const ShaderComputeInputInfo& input, uint32_t grou
? group_x
: static_cast<uint64_t>(group_x) * input.threads_num[0];
const auto size = descriptor.GetSize();
if (invocations != descriptor.NumRecords() || size == 0 || size > UINT32_MAX ||
const auto elements = descriptor.NumRecords();
const bool covered = fill.bounded_elements != 0
? fill.bounded_elements == elements && invocations >= elements
: invocations == elements;
if (!covered || invocations > uint64_t {UINT32_MAX} + 1 || size == 0 || size > UINT32_MAX ||
(input.dispatch_thread_dimensions &&
(group_x % input.threads_num[0] != 0 || input.dispatch_threads_num[0] != group_x ||
input.dispatch_threads_num[1] != 1 || input.dispatch_threads_num[2] != 1))) {
@@ -252,10 +257,6 @@ void RenderExecutor::DispatchDirect(uint64_t submit_id, CommandBuffer& buffer,
return;
}
if (sh_ctx.GetCs().cs_regs.data_addr == 0) {
return;
}
constexpr uint32_t DISPATCH_INITIATOR_USE_THREAD_DIMENSIONS = 1u << 5u;
constexpr uint32_t DISPATCH_INITIATOR_BASE_BITS = 0x41u;
constexpr uint32_t DISPATCH_INITIATOR_MODIFIER_BITS = 0xa038u;
@@ -25,6 +25,7 @@ struct UniformFill {
std::array<uint32_t, 3> group_stride {};
uint32_t words = 0;
uint32_t value = 0;
uint32_t bounded_elements = 0;
bool operator==(const UniformFill&) const = default;
};
@@ -622,6 +622,8 @@ struct CompiledShaderInfo {
struct UniformFillPlan {
UniformFill fill;
std::array<Value, 4> values;
Value pattern_size;
Value element_count;
};
// Resource analysis retained by the shader cache. It owns immutable descriptor/SRT,
@@ -1424,10 +1424,11 @@ ResourcePlan ExtractResourcePlan(const Program& program) {
for (auto it = first; it != plan.value_storage.end(); ++it) it->Invalidate();
plan.value_storage.erase(first, plan.value_storage.end());
}
plan.uniform_fill = AnalyzeUniformFill(program);
for (uint32_t i = 0; i < plan.uniform_fill.fill.words; ++i) {
plan.uniform_fill.values[i] = Clone(plan.uniform_fill.values[i]);
}
plan.uniform_fill = program.uniform_fill.fill.kind != UniformFillKind::None
? program.uniform_fill : AnalyzeUniformFill(program);
for (auto& value: plan.uniform_fill.values) value = Clone(value);
plan.uniform_fill.pattern_size = Clone(plan.uniform_fill.pattern_size);
plan.uniform_fill.element_count = Clone(plan.uniform_fill.element_count);
plan.capture_specialization_reads |=
capture_indirect_reads || capture_control_reads || !plan.source_reads.empty();
if (plan.capture_specialization_reads) {
@@ -1513,15 +1514,21 @@ bool MaterializeResources(const ResourcePlan& program, const SrtRuntime& runtime
}
snapshot.uniform_fill = {};
const auto& fill = program.uniform_fill;
const auto words = fill.fill.words;
auto words = fill.fill.words;
std::array<uint32_t, 4> stored {};
bool uniform_fill = words != 0;
if (uniform_fill && !fill.pattern_size.IsEmpty())
uniform_fill = clean.Evaluate(fill.pattern_size, words) && words >= 1 && words <= 4;
uint32_t elements = 0;
if (uniform_fill && !fill.element_count.IsEmpty())
uniform_fill = clean.Evaluate(fill.element_count, elements) && elements != 0;
for (uint32_t i = 0; i < words && uniform_fill; ++i) {
uniform_fill = clean.Evaluate(fill.values[i], stored[i]) && stored[i] == stored[0];
}
if (uniform_fill) {
snapshot.uniform_fill = fill.fill;
snapshot.uniform_fill.value = stored[0];
snapshot.uniform_fill.bounded_elements = elements;
}
const auto evaluate = [&](uint32_t source, DescriptorValue& value, bool written = false) {
if (source >= program.descriptor_sources.size()) {
@@ -15,6 +15,8 @@
#include <tuple>
#include <unordered_map>
#include <utility>
#define XXH_INLINE_ALL
#include <xxhash.h>
namespace Libs::Graphics::ShaderRecompiler::IR {
namespace {
@@ -4040,11 +4042,71 @@ private:
bool m_shader_writes = false;
};
UniformFillPlan RecognizePatternFill(Program& program, const Decoder::Program& decoded,
ShaderStageInputInfo input) {
struct Pattern {
size_t code_words;
uint64_t hash;
uint32_t wave_size;
bool bounded;
};
constexpr std::array patterns {
// Bounded 64-lane variant.
Pattern {.code_words = 69, .hash = 0x0704b9818e3cba06ull, .wave_size = 64, .bounded = true},
// Dispatch thread dimensions, 64-lane variants.
Pattern {.code_words = 42, .hash = 0x92ce57f21310ef02ull, .wave_size = 64, .bounded = false},
Pattern {.code_words = 40, .hash = 0x374161027f58bee7ull, .wave_size = 64, .bounded = false},
Pattern {.code_words = 40, .hash = 0x7d7172c56c2e296dull, .wave_size = 64, .bounded = false},
// Dispatch thread dimensions, 32-lane variant.
Pattern {.code_words = 38, .hash = 0xacbda1a1ccdd4198ull, .wave_size = 32, .bounded = false},
};
if (program.stage != ShaderType::Compute || input.compute == nullptr ||
decoded.instructions.empty() || program.user_data_base != 0 || program.user_data_count < 10 ||
program.info.buffers.size() != 1 || !program.info.images.empty()) return {};
const auto& cs = *input.compute;
const bool matching_inputs = cs.workgroup_register == 10 && cs.thread_ids_num == 1 &&
cs.group_id[0] && !cs.group_id[1] && !cs.group_id[2] && cs.threads_num[0] == 64 &&
cs.threads_num[1] == 1 && cs.threads_num[2] == 1 && cs.float_mode == 0xc0;
if (!matching_inputs) return {};
const auto& last = decoded.instructions.back();
const size_t words = last.pc / sizeof(uint32_t) + last.word_count;
if (words > decoded.code.size() ||
std::ranges::find(patterns, words, &Pattern::code_words) == patterns.end()) return {};
// Hash executable instructions through S_ENDPGM, excluding binary metadata.
// Verified variants implement Destination[id] = value[id % pattern_size].
const auto hash = XXH3_64bits(decoded.code.data(), words * sizeof(uint32_t));
const auto pattern = std::ranges::find_if(patterns, [&](const Pattern& candidate) {
return candidate.code_words == words && candidate.hash == hash;
});
if (pattern == patterns.end() || program.wave_size != pattern->wave_size ||
cs.dispatch_thread_dimensions != !pattern->bounded) return {};
std::array<Value, 10> userdata;
for (auto* block: program.blocks) {
for (auto& inst: *block) {
if (inst.GetOpcode() != ValueOpcode::GetUserData) continue;
const auto reg = RegIndex(inst.Arg(0).ScalarRegister());
if (reg < userdata.size()) userdata[reg] = Value(&inst);
}
}
for (uint32_t reg = 4; reg < 10; ++reg)
if ((reg != 8 || pattern->bounded) && userdata[reg].IsEmpty()) return {};
UniformFillPlan result;
result.fill.kind = UniformFillKind::Buffer;
result.fill.words = 1;
result.fill.group_stride[0] = 64;
std::copy_n(userdata.begin() + 4, 4, result.values.begin());
result.pattern_size = userdata[9];
// Only the bounded variant reads an explicit count; the other uses thread dimensions.
if (pattern->bounded) result.element_count = userdata[8];
return result;
}
} // namespace
void TrackResources(Program& program, const Decoder::Program& decoded, const CFG::Graph& native_cfg,
ShaderStageInputInfo input) {
Tracker(program, decoded, native_cfg, input).Run();
program.uniform_fill = RecognizePatternFill(program, decoded, input);
}
} // namespace Libs::Graphics::ShaderRecompiler::IR
+25
View File
@@ -42640,6 +42640,30 @@ void CheckPs5GameExampleImageClearRuntimeShape() {
!ResolveComputeBufferFill(compute, 2, 1, 1, 0x41u, descriptor,
packed_clear, size),
"excess invocation coverage was accepted as a fill");
runtime_resources.uniform_fill.bounded_elements = 65;
runtime_resources.buffers[0].dwords[2] = 65;
for (u32 oob = 0; oob < 4; ++oob) {
runtime_resources.buffers[0].dwords[3] = 0x14204u | (oob << 28u);
Require("Ps5GameExampleImageClear", "bounded final workgroup and RDNA2 OOB mode",
ResolveComputeBufferFill(compute, 2, 1, 1, 0x41u, descriptor,
packed_clear, size) == (oob != 3),
"explicit coverage did not respect RDNA2 element bounds");
}
runtime_resources.buffers[0].dwords[3] = 0x14204u;
Require("Ps5GameExampleImageClear", "bounded insufficient dispatch",
!ResolveComputeBufferFill(compute, 1, 1, 1, 0x41u, descriptor,
packed_clear, size),
"a bound was mistaken for actual invocation coverage");
Require("Ps5GameExampleImageClear", "bounded index wrap",
!ResolveComputeBufferFill(compute, 0x04000001u, 1, 1, 0x41u, descriptor,
packed_clear, size),
"a wrapped invocation index was accepted as complete coverage");
runtime_resources.uniform_fill.bounded_elements = 64;
Require("Ps5GameExampleImageClear", "bounded incomplete destination",
!ResolveComputeBufferFill(compute, 2, 1, 1, 0x41u, descriptor,
packed_clear, size),
"a shader bound smaller than the destination was accepted as a fill");
runtime_resources = scalar;
auto alias = scalar.buffers.front();
runtime_resources.buffers.push_back(alias);
Require("Ps5GameExampleImageClear", "aliased scalar input",
@@ -48607,6 +48631,7 @@ int main(int argc, char **argv) {
}
if (argc == 2 && std::strcmp(argv[1], "--compute-meta-clear-only") == 0) {
CheckDynamicRenderingState();
CheckPs5GameExampleImageClearRuntimeShape();
VulkanHarness vulkan;
vulkan.CheckComputeMetaClearClassification();
vulkan.CheckNativeIndirectDispatch();
+167
View File
@@ -14414,6 +14414,172 @@ void TestComputeImageFill() {
Run(mutation);
}
void TestPatternFill() {
using namespace ShaderRecompiler::IR;
// Bounded DWORD pattern fill, including every instruction through S_ENDPGM.
const uint32_t bounded_shader[] = {
0xbfa00003u, 0xd7460002u, 0x04010c0au, 0x7da80408u,
0xbf88003fu, 0x7e000c09u, 0xbf070980u, 0x858a807eu,
0x7e005700u, 0x100000ffu, 0x4f800000u, 0x7e060f00u,
0xd5766a00u, 0x02020609u, 0x7d8a0280u, 0x4c020080u,
0x02000101u, 0xd56a0001u, 0x00020700u, 0x4c000303u,
0x4a020303u, 0x02000101u, 0xd56a0000u, 0x00020500u,
0xd5690001u, 0x00020009u, 0x4c060302u, 0x7d8c02f9u,
0x06068c02u, 0x7d860609u, 0x87ea6a0cu, 0x50000080u,
0xd5286a00u, 0x003200c1u, 0xd5010000u, 0x002a00c1u,
0xd5690000u, 0x00020009u, 0x4c000102u, 0x7d0a0080u,
0xbe88246au, 0xbf880015u, 0x7d0a0081u, 0xbe8a246au,
0xbf88000cu, 0x7d0a0082u, 0xbeea246au, 0xbf880003u,
0x7e000207u, 0xe0102000u, 0x80000002u, 0x8afe7e6au,
0xbf880003u, 0x7e000206u, 0xe0102000u, 0x80000002u,
0xbefe046au, 0x8afe7e0au, 0xbf880003u, 0x7e000205u,
0xe0102000u, 0x80000002u, 0xbefe040au, 0x8afe7e08u,
0xbf880003u, 0x7e000204u, 0xe0102000u, 0x80000002u,
0xbf810000u,
};
// Pattern fill whose bounds come from dispatch thread dimensions.
const uint32_t thread_dimension_shader[] = {
0xbfa00002u, 0x7e040c09u, 0x81ea0980u, 0xd7460000u,
0x04010c0au, 0x7e020205u, 0x7e045702u, 0x100404ffu,
0x4f7ffffeu, 0x7e040f02u, 0x7ed60502u, 0x936a6b6au,
0x9aea6b6au, 0x4a04046au, 0xd56a0002u, 0x00020102u,
0xd5690002u, 0x00020409u, 0x4c040500u, 0x7d860409u,
0x4e060409u, 0x02040702u, 0x7d860409u, 0x4e060409u,
0x02040702u, 0x7d840481u, 0x7d8404f9u, 0x06868882u,
0x7d8404f9u, 0x06868a83u, 0x02020204u, 0xd5010001u,
0x00200d01u, 0xd5010001u, 0x00280f01u, 0xe0102000u,
0x80000100u, 0xbf810000u,
};
// The 64-lane variant uses vector multiply-high for the remainder calculation.
const uint32_t thread_dimension_wave64_shader[] = {
0xbfa00002u, 0x7e060c09u, 0x4c0412f9u, 0x86860680u,
0xd7460000u, 0x04010c0au, 0x7e020205u, 0x7e065703u,
0x100606ffu, 0x4f7ffffeu, 0x7e060f03u, 0xd5690002u,
0x00020702u, 0xd56a0002u, 0x00020702u, 0x4a040503u,
0xd56a0002u, 0x00020102u, 0xd5690002u, 0x00020409u,
0x4c040500u, 0x7d860409u, 0x4c0612f9u, 0x86060602u,
0x02040702u, 0x7d860409u, 0x4c0612f9u, 0x86060602u,
0x02040702u, 0x7d040481u, 0x7d0404f9u, 0x06868882u,
0x7d0404f9u, 0x06868a83u, 0x02020204u, 0xd5010001u,
0x00200d01u, 0xd5010001u, 0x00280f01u, 0xe0102000u,
0x80000100u, 0xbf810000u,
};
// The 64-lane variant with scalar quotient setup.
const uint32_t thread_dimension_wave64_scalar_shader[] = {
0xbfa00002u, 0x7e040c09u, 0x81ea0980u, 0xd7460000u,
0x04010c0au, 0x7e020205u, 0x7e045702u, 0x100404ffu,
0x4f7ffffeu, 0x7e040f02u, 0x7ed60502u, 0x936a6b6au,
0x9aea6b6au, 0x816a6a6bu, 0xd56a0002u, 0x0002006au,
0xd5690002u, 0x00020409u, 0x4c040500u, 0x7d860409u,
0x4c0612f9u, 0x86060602u, 0x02040702u, 0x7d860409u,
0x4c0612f9u, 0x86060602u, 0x02040702u, 0x7d040481u,
0x7d0404f9u, 0x06868882u, 0x7d0404f9u, 0x06868a83u,
0x02020204u, 0xd5010001u, 0x00200d01u, 0xd5010001u,
0x00280f01u, 0xe0102000u, 0x80000100u, 0xbf810000u,
};
// Same layout, with vector arithmetic for the reciprocal correction.
auto thread_dimension_wave64_vector_shader = std::to_array(thread_dimension_wave64_scalar_shader);
thread_dimension_wave64_vector_shader[13] = 0x4a04046au;
thread_dimension_wave64_vector_shader[15] = 0x00020102u;
struct PatternCase {
std::span<const uint32_t> shader;
uint32_t wave_size;
bool bounded;
size_t literal_word;
};
const PatternCase cases[] = {
{bounded_shader, 64, true, 10},
{thread_dimension_shader, 32, false, 8},
{thread_dimension_wave64_shader, 64, false, 9},
{thread_dimension_wave64_scalar_shader, 64, false, 8},
{thread_dimension_wave64_vector_shader, 64, false, 8},
};
for (const auto &[shader, wave_size, bounded, literal_word] : cases) {
std::array<uint32_t, 10> userdata = {
0x00200000u, 4u << 16, 64u, 0x14204u,
0xff000000u, 0u, 0u, 0x08000000u, 64u, 1u,
};
ShaderComputeInputInfo compute{};
compute.threads_num[0] = 64;
compute.threads_num[1] = compute.threads_num[2] = 1;
compute.workgroup_register = 10;
compute.thread_ids_num = 1;
compute.group_id[0] = true;
compute.dispatch_thread_dimensions = !bounded;
compute.wave_size = wave_size;
auto options = MakeCompileOptions(ShaderType::Compute);
options.wave_size = compute.wave_size;
options.input_info.compute = &compute;
options.user_data = userdata;
// Recognition derives identity from code, independently of the caller's hash.
options.shader_hash = 0x12345678u;
const auto Build = [&](std::span<const uint32_t> code) {
auto translated = ShaderRecompiler::TranslateProgram(code, options);
return ExtractResourcePlan(translated.program);
};
auto plan = Build(shader);
ResourceSnapshot snapshot;
ResourceSpecialization specialization;
const auto Materialize = [&](bool expected) {
Check(MaterializeResources(plan, {.user_data = userdata}, snapshot, specialization),
"pattern fill did not materialize");
const auto &fill = snapshot.uniform_fill;
Check((fill.kind == UniformFillKind::Buffer) == expected,
"pattern fill used inactive words or retained a stale result");
if (expected)
Check(fill.value == userdata[4] && fill.words == 1 &&
fill.bounded_elements == (bounded ? userdata[8] : 0u) &&
fill.group_stride == std::array<uint32_t, 3>{64, 0, 0},
"pattern fill lost its value or coverage contract");
};
Materialize(true);
userdata[4] ^= 0x123456u;
Materialize(true);
userdata[4] ^= 0x123456u;
for (const uint32_t period : {0u, 2u, 3u, 4u, 5u}) {
userdata[9] = period;
Materialize(false);
}
for (uint32_t period = 2; period <= 4; ++period) {
userdata[9] = period;
userdata[3 + period] = userdata[4];
Materialize(true);
userdata[5] ^= 1u;
Materialize(false);
userdata[5] ^= 1u;
}
userdata[5] = userdata[6] = 0;
userdata[7] = 0x08000000u;
userdata[9] = 1;
Materialize(true);
userdata[8] = 0;
Materialize(!bounded);
userdata[8] = 63;
Materialize(true); // The renderer owns the descriptor-size equality check.
userdata[8] = 64;
Materialize(true);
std::vector<uint32_t> changed(shader.begin(), shader.end());
changed.push_back(0xdeadcafeu); // Non-executable metadata does not change identity.
Check(Build(changed).uniform_fill.fill.kind == UniformFillKind::Buffer,
"pattern recognition included bytes after S_ENDPGM");
changed[literal_word] ^= 0x00800000u;
Check(Build(changed).uniform_fill.fill.kind == UniformFillKind::None,
"pattern recognition accepted modified executable instructions");
for (const uint32_t invalid_input : {0u, 1u, 2u, 3u}) {
const auto saved_compute = compute;
const auto saved_wave = options.wave_size;
if (invalid_input == 0) compute.workgroup_register = 11;
if (invalid_input == 1) options.wave_size = wave_size == 64 ? 32 : 64;
if (invalid_input == 2) compute.dispatch_thread_dimensions = bounded;
if (invalid_input == 3) compute.float_mode ^= 1u;
Check(Build(shader).uniform_fill.fill.kind == UniformFillKind::None,
"pattern recognition accepted different shader launch inputs");
compute = saved_compute;
options.wave_size = saved_wave;
}
}
}
void TestGpuProducedWritableDescriptor() {
using namespace ShaderRecompiler::IR;
// Reduced from 994af59d50bedca8: the GPU-built BVH header supplies a relative
@@ -16430,6 +16596,7 @@ int main() {
TestNewShaderRecompilerEarlyZDisabledWhenPixelKillEnabled();
TestTypedDescriptorRealWideMoveTranslation();
TestComputeImageFill();
TestPatternFill();
TestGpuProducedWritableDescriptor();
TestUniformSelectedWritableDescriptor();
TestUniformSelectedDescriptorLoadAddress();