From 9202bddad2c6fa66e7f0e864228aa63110c2e991 Mon Sep 17 00:00:00 2001 From: nmzik Date: Sat, 17 Oct 2026 01:42:34 +0200 Subject: [PATCH] recognize verified pattern fills Co-authored-by: Juan Pablo Farias --- .../host_gpu/renderer/renderCompute.cpp | 13 +- .../shader/recompiler/ir/ResourceSnapshot.h | 1 + src/graphics/shader/recompiler/ir/ShaderIR.h | 2 + .../ir/passes/ResourceMaterialization.cpp | 17 +- .../recompiler/ir/passes/ResourceTracking.cpp | 62 +++++++ tests/ShaderRecompilerComputeTests.cpp | 25 +++ tests/shaderCfgTests.cpp | 167 ++++++++++++++++++ 7 files changed, 276 insertions(+), 11 deletions(-) diff --git a/src/graphics/host_gpu/renderer/renderCompute.cpp b/src/graphics/host_gpu/renderer/renderCompute.cpp index d812ce24e..1b2f17bd4 100644 --- a/src/graphics/host_gpu/renderer/renderCompute.cpp +++ b/src/graphics/host_gpu/renderer/renderCompute.cpp @@ -94,9 +94,10 @@ bool ResolveComputeBufferFill(const ShaderComputeInputInfo& input, uint32_t grou constexpr std::array formats { Prospero::BufferFormat::k32UInt, Prospero::BufferFormat::k32_32UInt, Prospero::BufferFormat::k32_32_32UInt, Prospero::BufferFormat::k32_32_32_32UInt}; + // Offset-or-swizzle bounds do not describe the element range proved by the shader. if (descriptor.Stride() != element_size || descriptor.Format() != formats[fill.words - 1] || descriptor.SwizzleEnabled() || descriptor.IndexStride() != 0 || descriptor.AddTid() || - descriptor.Base48() == 0) { + descriptor.OutOfBounds() == 3 || descriptor.Base48() == 0) { return false; } if (input.threads_num[0] == 0 || input.threads_num[0] != fill.group_stride[0] || @@ -108,7 +109,11 @@ bool ResolveComputeBufferFill(const ShaderComputeInputInfo& input, uint32_t grou ? group_x : static_cast(group_x) * input.threads_num[0]; const auto size = descriptor.GetSize(); - if (invocations != descriptor.NumRecords() || size == 0 || size > UINT32_MAX || + const auto elements = descriptor.NumRecords(); + const bool covered = fill.bounded_elements != 0 + ? fill.bounded_elements == elements && invocations >= elements + : invocations == elements; + if (!covered || invocations > uint64_t {UINT32_MAX} + 1 || size == 0 || size > UINT32_MAX || (input.dispatch_thread_dimensions && (group_x % input.threads_num[0] != 0 || input.dispatch_threads_num[0] != group_x || input.dispatch_threads_num[1] != 1 || input.dispatch_threads_num[2] != 1))) { @@ -252,10 +257,6 @@ void RenderExecutor::DispatchDirect(uint64_t submit_id, CommandBuffer& buffer, return; } - if (sh_ctx.GetCs().cs_regs.data_addr == 0) { - return; - } - constexpr uint32_t DISPATCH_INITIATOR_USE_THREAD_DIMENSIONS = 1u << 5u; constexpr uint32_t DISPATCH_INITIATOR_BASE_BITS = 0x41u; constexpr uint32_t DISPATCH_INITIATOR_MODIFIER_BITS = 0xa038u; diff --git a/src/graphics/shader/recompiler/ir/ResourceSnapshot.h b/src/graphics/shader/recompiler/ir/ResourceSnapshot.h index 3e7586a7f..ac0e0a1c8 100644 --- a/src/graphics/shader/recompiler/ir/ResourceSnapshot.h +++ b/src/graphics/shader/recompiler/ir/ResourceSnapshot.h @@ -25,6 +25,7 @@ struct UniformFill { std::array group_stride {}; uint32_t words = 0; uint32_t value = 0; + uint32_t bounded_elements = 0; bool operator==(const UniformFill&) const = default; }; diff --git a/src/graphics/shader/recompiler/ir/ShaderIR.h b/src/graphics/shader/recompiler/ir/ShaderIR.h index e8283c841..c739482e0 100644 --- a/src/graphics/shader/recompiler/ir/ShaderIR.h +++ b/src/graphics/shader/recompiler/ir/ShaderIR.h @@ -622,6 +622,8 @@ struct CompiledShaderInfo { struct UniformFillPlan { UniformFill fill; std::array values; + Value pattern_size; + Value element_count; }; // Resource analysis retained by the shader cache. It owns immutable descriptor/SRT, diff --git a/src/graphics/shader/recompiler/ir/passes/ResourceMaterialization.cpp b/src/graphics/shader/recompiler/ir/passes/ResourceMaterialization.cpp index 81bd5fe1c..637f68efb 100644 --- a/src/graphics/shader/recompiler/ir/passes/ResourceMaterialization.cpp +++ b/src/graphics/shader/recompiler/ir/passes/ResourceMaterialization.cpp @@ -1424,10 +1424,11 @@ ResourcePlan ExtractResourcePlan(const Program& program) { for (auto it = first; it != plan.value_storage.end(); ++it) it->Invalidate(); plan.value_storage.erase(first, plan.value_storage.end()); } - plan.uniform_fill = AnalyzeUniformFill(program); - for (uint32_t i = 0; i < plan.uniform_fill.fill.words; ++i) { - plan.uniform_fill.values[i] = Clone(plan.uniform_fill.values[i]); - } + plan.uniform_fill = program.uniform_fill.fill.kind != UniformFillKind::None + ? program.uniform_fill : AnalyzeUniformFill(program); + for (auto& value: plan.uniform_fill.values) value = Clone(value); + plan.uniform_fill.pattern_size = Clone(plan.uniform_fill.pattern_size); + plan.uniform_fill.element_count = Clone(plan.uniform_fill.element_count); plan.capture_specialization_reads |= capture_indirect_reads || capture_control_reads || !plan.source_reads.empty(); if (plan.capture_specialization_reads) { @@ -1513,15 +1514,21 @@ bool MaterializeResources(const ResourcePlan& program, const SrtRuntime& runtime } snapshot.uniform_fill = {}; const auto& fill = program.uniform_fill; - const auto words = fill.fill.words; + auto words = fill.fill.words; std::array stored {}; bool uniform_fill = words != 0; + if (uniform_fill && !fill.pattern_size.IsEmpty()) + uniform_fill = clean.Evaluate(fill.pattern_size, words) && words >= 1 && words <= 4; + uint32_t elements = 0; + if (uniform_fill && !fill.element_count.IsEmpty()) + uniform_fill = clean.Evaluate(fill.element_count, elements) && elements != 0; for (uint32_t i = 0; i < words && uniform_fill; ++i) { uniform_fill = clean.Evaluate(fill.values[i], stored[i]) && stored[i] == stored[0]; } if (uniform_fill) { snapshot.uniform_fill = fill.fill; snapshot.uniform_fill.value = stored[0]; + snapshot.uniform_fill.bounded_elements = elements; } const auto evaluate = [&](uint32_t source, DescriptorValue& value, bool written = false) { if (source >= program.descriptor_sources.size()) { diff --git a/src/graphics/shader/recompiler/ir/passes/ResourceTracking.cpp b/src/graphics/shader/recompiler/ir/passes/ResourceTracking.cpp index c1c8286df..b21361b22 100644 --- a/src/graphics/shader/recompiler/ir/passes/ResourceTracking.cpp +++ b/src/graphics/shader/recompiler/ir/passes/ResourceTracking.cpp @@ -15,6 +15,8 @@ #include #include #include +#define XXH_INLINE_ALL +#include namespace Libs::Graphics::ShaderRecompiler::IR { namespace { @@ -4040,11 +4042,71 @@ private: bool m_shader_writes = false; }; +UniformFillPlan RecognizePatternFill(Program& program, const Decoder::Program& decoded, + ShaderStageInputInfo input) { + struct Pattern { + size_t code_words; + uint64_t hash; + uint32_t wave_size; + bool bounded; + }; + constexpr std::array patterns { + // Bounded 64-lane variant. + Pattern {.code_words = 69, .hash = 0x0704b9818e3cba06ull, .wave_size = 64, .bounded = true}, + // Dispatch thread dimensions, 64-lane variants. + Pattern {.code_words = 42, .hash = 0x92ce57f21310ef02ull, .wave_size = 64, .bounded = false}, + Pattern {.code_words = 40, .hash = 0x374161027f58bee7ull, .wave_size = 64, .bounded = false}, + Pattern {.code_words = 40, .hash = 0x7d7172c56c2e296dull, .wave_size = 64, .bounded = false}, + // Dispatch thread dimensions, 32-lane variant. + Pattern {.code_words = 38, .hash = 0xacbda1a1ccdd4198ull, .wave_size = 32, .bounded = false}, + }; + if (program.stage != ShaderType::Compute || input.compute == nullptr || + decoded.instructions.empty() || program.user_data_base != 0 || program.user_data_count < 10 || + program.info.buffers.size() != 1 || !program.info.images.empty()) return {}; + const auto& cs = *input.compute; + const bool matching_inputs = cs.workgroup_register == 10 && cs.thread_ids_num == 1 && + cs.group_id[0] && !cs.group_id[1] && !cs.group_id[2] && cs.threads_num[0] == 64 && + cs.threads_num[1] == 1 && cs.threads_num[2] == 1 && cs.float_mode == 0xc0; + if (!matching_inputs) return {}; + const auto& last = decoded.instructions.back(); + const size_t words = last.pc / sizeof(uint32_t) + last.word_count; + if (words > decoded.code.size() || + std::ranges::find(patterns, words, &Pattern::code_words) == patterns.end()) return {}; + // Hash executable instructions through S_ENDPGM, excluding binary metadata. + // Verified variants implement Destination[id] = value[id % pattern_size]. + const auto hash = XXH3_64bits(decoded.code.data(), words * sizeof(uint32_t)); + const auto pattern = std::ranges::find_if(patterns, [&](const Pattern& candidate) { + return candidate.code_words == words && candidate.hash == hash; + }); + if (pattern == patterns.end() || program.wave_size != pattern->wave_size || + cs.dispatch_thread_dimensions != !pattern->bounded) return {}; + std::array userdata; + for (auto* block: program.blocks) { + for (auto& inst: *block) { + if (inst.GetOpcode() != ValueOpcode::GetUserData) continue; + const auto reg = RegIndex(inst.Arg(0).ScalarRegister()); + if (reg < userdata.size()) userdata[reg] = Value(&inst); + } + } + for (uint32_t reg = 4; reg < 10; ++reg) + if ((reg != 8 || pattern->bounded) && userdata[reg].IsEmpty()) return {}; + UniformFillPlan result; + result.fill.kind = UniformFillKind::Buffer; + result.fill.words = 1; + result.fill.group_stride[0] = 64; + std::copy_n(userdata.begin() + 4, 4, result.values.begin()); + result.pattern_size = userdata[9]; + // Only the bounded variant reads an explicit count; the other uses thread dimensions. + if (pattern->bounded) result.element_count = userdata[8]; + return result; +} + } // namespace void TrackResources(Program& program, const Decoder::Program& decoded, const CFG::Graph& native_cfg, ShaderStageInputInfo input) { Tracker(program, decoded, native_cfg, input).Run(); + program.uniform_fill = RecognizePatternFill(program, decoded, input); } } // namespace Libs::Graphics::ShaderRecompiler::IR diff --git a/tests/ShaderRecompilerComputeTests.cpp b/tests/ShaderRecompilerComputeTests.cpp index 720a09f16..af1306247 100644 --- a/tests/ShaderRecompilerComputeTests.cpp +++ b/tests/ShaderRecompilerComputeTests.cpp @@ -42640,6 +42640,30 @@ void CheckPs5GameExampleImageClearRuntimeShape() { !ResolveComputeBufferFill(compute, 2, 1, 1, 0x41u, descriptor, packed_clear, size), "excess invocation coverage was accepted as a fill"); + runtime_resources.uniform_fill.bounded_elements = 65; + runtime_resources.buffers[0].dwords[2] = 65; + for (u32 oob = 0; oob < 4; ++oob) { + runtime_resources.buffers[0].dwords[3] = 0x14204u | (oob << 28u); + Require("Ps5GameExampleImageClear", "bounded final workgroup and RDNA2 OOB mode", + ResolveComputeBufferFill(compute, 2, 1, 1, 0x41u, descriptor, + packed_clear, size) == (oob != 3), + "explicit coverage did not respect RDNA2 element bounds"); + } + runtime_resources.buffers[0].dwords[3] = 0x14204u; + Require("Ps5GameExampleImageClear", "bounded insufficient dispatch", + !ResolveComputeBufferFill(compute, 1, 1, 1, 0x41u, descriptor, + packed_clear, size), + "a bound was mistaken for actual invocation coverage"); + Require("Ps5GameExampleImageClear", "bounded index wrap", + !ResolveComputeBufferFill(compute, 0x04000001u, 1, 1, 0x41u, descriptor, + packed_clear, size), + "a wrapped invocation index was accepted as complete coverage"); + runtime_resources.uniform_fill.bounded_elements = 64; + Require("Ps5GameExampleImageClear", "bounded incomplete destination", + !ResolveComputeBufferFill(compute, 2, 1, 1, 0x41u, descriptor, + packed_clear, size), + "a shader bound smaller than the destination was accepted as a fill"); + runtime_resources = scalar; auto alias = scalar.buffers.front(); runtime_resources.buffers.push_back(alias); Require("Ps5GameExampleImageClear", "aliased scalar input", @@ -48607,6 +48631,7 @@ int main(int argc, char **argv) { } if (argc == 2 && std::strcmp(argv[1], "--compute-meta-clear-only") == 0) { CheckDynamicRenderingState(); + CheckPs5GameExampleImageClearRuntimeShape(); VulkanHarness vulkan; vulkan.CheckComputeMetaClearClassification(); vulkan.CheckNativeIndirectDispatch(); diff --git a/tests/shaderCfgTests.cpp b/tests/shaderCfgTests.cpp index dc1761f8f..b6d660b9d 100644 --- a/tests/shaderCfgTests.cpp +++ b/tests/shaderCfgTests.cpp @@ -14414,6 +14414,172 @@ void TestComputeImageFill() { Run(mutation); } +void TestPatternFill() { + using namespace ShaderRecompiler::IR; + // Bounded DWORD pattern fill, including every instruction through S_ENDPGM. + const uint32_t bounded_shader[] = { + 0xbfa00003u, 0xd7460002u, 0x04010c0au, 0x7da80408u, + 0xbf88003fu, 0x7e000c09u, 0xbf070980u, 0x858a807eu, + 0x7e005700u, 0x100000ffu, 0x4f800000u, 0x7e060f00u, + 0xd5766a00u, 0x02020609u, 0x7d8a0280u, 0x4c020080u, + 0x02000101u, 0xd56a0001u, 0x00020700u, 0x4c000303u, + 0x4a020303u, 0x02000101u, 0xd56a0000u, 0x00020500u, + 0xd5690001u, 0x00020009u, 0x4c060302u, 0x7d8c02f9u, + 0x06068c02u, 0x7d860609u, 0x87ea6a0cu, 0x50000080u, + 0xd5286a00u, 0x003200c1u, 0xd5010000u, 0x002a00c1u, + 0xd5690000u, 0x00020009u, 0x4c000102u, 0x7d0a0080u, + 0xbe88246au, 0xbf880015u, 0x7d0a0081u, 0xbe8a246au, + 0xbf88000cu, 0x7d0a0082u, 0xbeea246au, 0xbf880003u, + 0x7e000207u, 0xe0102000u, 0x80000002u, 0x8afe7e6au, + 0xbf880003u, 0x7e000206u, 0xe0102000u, 0x80000002u, + 0xbefe046au, 0x8afe7e0au, 0xbf880003u, 0x7e000205u, + 0xe0102000u, 0x80000002u, 0xbefe040au, 0x8afe7e08u, + 0xbf880003u, 0x7e000204u, 0xe0102000u, 0x80000002u, + 0xbf810000u, + }; + // Pattern fill whose bounds come from dispatch thread dimensions. + const uint32_t thread_dimension_shader[] = { + 0xbfa00002u, 0x7e040c09u, 0x81ea0980u, 0xd7460000u, + 0x04010c0au, 0x7e020205u, 0x7e045702u, 0x100404ffu, + 0x4f7ffffeu, 0x7e040f02u, 0x7ed60502u, 0x936a6b6au, + 0x9aea6b6au, 0x4a04046au, 0xd56a0002u, 0x00020102u, + 0xd5690002u, 0x00020409u, 0x4c040500u, 0x7d860409u, + 0x4e060409u, 0x02040702u, 0x7d860409u, 0x4e060409u, + 0x02040702u, 0x7d840481u, 0x7d8404f9u, 0x06868882u, + 0x7d8404f9u, 0x06868a83u, 0x02020204u, 0xd5010001u, + 0x00200d01u, 0xd5010001u, 0x00280f01u, 0xe0102000u, + 0x80000100u, 0xbf810000u, + }; + // The 64-lane variant uses vector multiply-high for the remainder calculation. + const uint32_t thread_dimension_wave64_shader[] = { + 0xbfa00002u, 0x7e060c09u, 0x4c0412f9u, 0x86860680u, + 0xd7460000u, 0x04010c0au, 0x7e020205u, 0x7e065703u, + 0x100606ffu, 0x4f7ffffeu, 0x7e060f03u, 0xd5690002u, + 0x00020702u, 0xd56a0002u, 0x00020702u, 0x4a040503u, + 0xd56a0002u, 0x00020102u, 0xd5690002u, 0x00020409u, + 0x4c040500u, 0x7d860409u, 0x4c0612f9u, 0x86060602u, + 0x02040702u, 0x7d860409u, 0x4c0612f9u, 0x86060602u, + 0x02040702u, 0x7d040481u, 0x7d0404f9u, 0x06868882u, + 0x7d0404f9u, 0x06868a83u, 0x02020204u, 0xd5010001u, + 0x00200d01u, 0xd5010001u, 0x00280f01u, 0xe0102000u, + 0x80000100u, 0xbf810000u, + }; + // The 64-lane variant with scalar quotient setup. + const uint32_t thread_dimension_wave64_scalar_shader[] = { + 0xbfa00002u, 0x7e040c09u, 0x81ea0980u, 0xd7460000u, + 0x04010c0au, 0x7e020205u, 0x7e045702u, 0x100404ffu, + 0x4f7ffffeu, 0x7e040f02u, 0x7ed60502u, 0x936a6b6au, + 0x9aea6b6au, 0x816a6a6bu, 0xd56a0002u, 0x0002006au, + 0xd5690002u, 0x00020409u, 0x4c040500u, 0x7d860409u, + 0x4c0612f9u, 0x86060602u, 0x02040702u, 0x7d860409u, + 0x4c0612f9u, 0x86060602u, 0x02040702u, 0x7d040481u, + 0x7d0404f9u, 0x06868882u, 0x7d0404f9u, 0x06868a83u, + 0x02020204u, 0xd5010001u, 0x00200d01u, 0xd5010001u, + 0x00280f01u, 0xe0102000u, 0x80000100u, 0xbf810000u, + }; + // Same layout, with vector arithmetic for the reciprocal correction. + auto thread_dimension_wave64_vector_shader = std::to_array(thread_dimension_wave64_scalar_shader); + thread_dimension_wave64_vector_shader[13] = 0x4a04046au; + thread_dimension_wave64_vector_shader[15] = 0x00020102u; + struct PatternCase { + std::span shader; + uint32_t wave_size; + bool bounded; + size_t literal_word; + }; + const PatternCase cases[] = { + {bounded_shader, 64, true, 10}, + {thread_dimension_shader, 32, false, 8}, + {thread_dimension_wave64_shader, 64, false, 9}, + {thread_dimension_wave64_scalar_shader, 64, false, 8}, + {thread_dimension_wave64_vector_shader, 64, false, 8}, + }; + for (const auto &[shader, wave_size, bounded, literal_word] : cases) { + std::array userdata = { + 0x00200000u, 4u << 16, 64u, 0x14204u, + 0xff000000u, 0u, 0u, 0x08000000u, 64u, 1u, + }; + ShaderComputeInputInfo compute{}; + compute.threads_num[0] = 64; + compute.threads_num[1] = compute.threads_num[2] = 1; + compute.workgroup_register = 10; + compute.thread_ids_num = 1; + compute.group_id[0] = true; + compute.dispatch_thread_dimensions = !bounded; + compute.wave_size = wave_size; + auto options = MakeCompileOptions(ShaderType::Compute); + options.wave_size = compute.wave_size; + options.input_info.compute = &compute; + options.user_data = userdata; + // Recognition derives identity from code, independently of the caller's hash. + options.shader_hash = 0x12345678u; + const auto Build = [&](std::span code) { + auto translated = ShaderRecompiler::TranslateProgram(code, options); + return ExtractResourcePlan(translated.program); + }; + auto plan = Build(shader); + ResourceSnapshot snapshot; + ResourceSpecialization specialization; + const auto Materialize = [&](bool expected) { + Check(MaterializeResources(plan, {.user_data = userdata}, snapshot, specialization), + "pattern fill did not materialize"); + const auto &fill = snapshot.uniform_fill; + Check((fill.kind == UniformFillKind::Buffer) == expected, + "pattern fill used inactive words or retained a stale result"); + if (expected) + Check(fill.value == userdata[4] && fill.words == 1 && + fill.bounded_elements == (bounded ? userdata[8] : 0u) && + fill.group_stride == std::array{64, 0, 0}, + "pattern fill lost its value or coverage contract"); + }; + Materialize(true); + userdata[4] ^= 0x123456u; + Materialize(true); + userdata[4] ^= 0x123456u; + for (const uint32_t period : {0u, 2u, 3u, 4u, 5u}) { + userdata[9] = period; + Materialize(false); + } + for (uint32_t period = 2; period <= 4; ++period) { + userdata[9] = period; + userdata[3 + period] = userdata[4]; + Materialize(true); + userdata[5] ^= 1u; + Materialize(false); + userdata[5] ^= 1u; + } + userdata[5] = userdata[6] = 0; + userdata[7] = 0x08000000u; + userdata[9] = 1; + Materialize(true); + userdata[8] = 0; + Materialize(!bounded); + userdata[8] = 63; + Materialize(true); // The renderer owns the descriptor-size equality check. + userdata[8] = 64; + Materialize(true); + std::vector changed(shader.begin(), shader.end()); + changed.push_back(0xdeadcafeu); // Non-executable metadata does not change identity. + Check(Build(changed).uniform_fill.fill.kind == UniformFillKind::Buffer, + "pattern recognition included bytes after S_ENDPGM"); + changed[literal_word] ^= 0x00800000u; + Check(Build(changed).uniform_fill.fill.kind == UniformFillKind::None, + "pattern recognition accepted modified executable instructions"); + for (const uint32_t invalid_input : {0u, 1u, 2u, 3u}) { + const auto saved_compute = compute; + const auto saved_wave = options.wave_size; + if (invalid_input == 0) compute.workgroup_register = 11; + if (invalid_input == 1) options.wave_size = wave_size == 64 ? 32 : 64; + if (invalid_input == 2) compute.dispatch_thread_dimensions = bounded; + if (invalid_input == 3) compute.float_mode ^= 1u; + Check(Build(shader).uniform_fill.fill.kind == UniformFillKind::None, + "pattern recognition accepted different shader launch inputs"); + compute = saved_compute; + options.wave_size = saved_wave; + } + } +} + void TestGpuProducedWritableDescriptor() { using namespace ShaderRecompiler::IR; // Reduced from 994af59d50bedca8: the GPU-built BVH header supplies a relative @@ -16430,6 +16596,7 @@ int main() { TestNewShaderRecompilerEarlyZDisabledWhenPixelKillEnabled(); TestTypedDescriptorRealWideMoveTranslation(); TestComputeImageFill(); + TestPatternFill(); TestGpuProducedWritableDescriptor(); TestUniformSelectedWritableDescriptor(); TestUniformSelectedDescriptorLoadAddress();