From 7c9da0e56fd330e9bc374813113cc0be5fc23289 Mon Sep 17 00:00:00 2001 From: nmzik Date: Wed, 14 Oct 2026 17:29:49 +0200 Subject: [PATCH] shader: share finite texture selection through masked first-lane reads Co-authored-by: fxpw --- .../recompiler/ir/passes/ResourceTracking.cpp | 69 +++++--- tests/ResourceTrackingTests.cpp | 154 ++++++++++++++++++ tests/ShaderRecompilerComputeTests.cpp | 92 +++++++++++ 3 files changed, 292 insertions(+), 23 deletions(-) diff --git a/src/graphics/shader/recompiler/ir/passes/ResourceTracking.cpp b/src/graphics/shader/recompiler/ir/passes/ResourceTracking.cpp index 51e349b72..a6e2b7f2d 100644 --- a/src/graphics/shader/recompiler/ir/passes/ResourceTracking.cpp +++ b/src/graphics/shader/recompiler/ir/passes/ResourceTracking.cpp @@ -3196,9 +3196,6 @@ private: return sampler && (m_program.memory_info[use.user->Flags().index] .image_sample_flags & Decoder::ImageSampleFlagAdjust) != 0u; })) return false; - bool has_phi = false; - for (size_t word = 0; word < handle.NumArgs(); ++word) has_phi |= handle.Arg(word).Resolve().IsPhi(); - if (!has_phi) return false; // Normalize synchronized descriptor words to one GPU ordinal. // Build the complete graph before changing IR so an unsupported leaf is transactional. struct Choice { @@ -3225,32 +3222,51 @@ private: return index; } const auto* branch = descriptor.dwords[bad].TryInstruction(); - if (branch == nullptr || branch->Parent() == nullptr || - branch->GetOpcode() != ValueOpcode::Phi || - branch->NumPhiBlocks() != branch->NumArgs()) return UINT32_MAX; - choices[index].branch = branch; - const auto count = branch->NumArgs(); + if (branch == nullptr || branch->Parent() == nullptr) return UINT32_MAX; + const auto op = branch->GetOpcode(); + const bool phi = op == ValueOpcode::Phi; + const bool lane = op == ValueOpcode::ReadFirstLane; + if ((!phi && !lane && op != ValueOpcode::SelectU32) || + (phi && branch->NumPhiBlocks() != branch->NumArgs())) return UINT32_MAX; + if (lane && !std::ranges::all_of(handle.Uses(), [&](const Use& use) { + return Implies(use.user->Arg(use.user->NumArgs() - 1u), branch->Arg(1)); + })) return UINT32_MAX; + const auto count = phi ? branch->NumArgs() : lane ? 1u : 2u; + std::array selected_words {}; + const auto* anchor = branch; + for (uint32_t word = 0; word < descriptor.dword_count; ++word) { + const auto* selected = descriptor.dwords[word].TryInstruction(); + if (selected == nullptr || selected->GetOpcode() != op || + selected->Parent() != branch->Parent()) { + if (!ValidateRuntimeValue(m_program, descriptor.dwords[word])) return UINT32_MAX; + continue; + } + if (phi ? selected->NumArgs() != count || selected->NumPhiBlocks() != count + : !EquivalentValue(m_program, selected->Arg(lane ? 1u : 0u), + branch->Arg(lane ? 1u : 0u))) return UINT32_MAX; + selected_words[word] = selected; + if (!phi && AvailableBefore(*anchor, *selected)) anchor = selected; + } + // Later words may depend on choices computed after the first descriptor word. + choices[index].branch = anchor; for (uint32_t arm = 0; arm < count; ++arm) { // A nonzero bit scan cannot choose its zero-input switch sentinel. // Preserve the Phi edge; its ordinal is never observed at runtime. - if (ImpossibleSwitchEdge(branch->PhiBlock(arm), branch->Parent())) { + if (phi && ImpossibleSwitchEdge(anchor->PhiBlock(arm), anchor->Parent())) { choices[index].children.push_back(UINT32_MAX); continue; } auto child = descriptor; for (uint32_t word = 0; word < child.dword_count; ++word) { - const auto* selected = descriptor.dwords[word].TryInstruction(); - if (selected == nullptr || selected->GetOpcode() != branch->GetOpcode() || - selected->Parent() != branch->Parent()) { - if (!ValidateRuntimeValue(m_program, descriptor.dwords[word])) return UINT32_MAX; - continue; + const auto* selected = selected_words[word]; + if (selected == nullptr) continue; + uint32_t incoming = lane ? 0u : arm + 1u; + if (phi) { + incoming = 0u; + while (incoming < count && selected->PhiBlock(incoming) != anchor->PhiBlock(arm)) + ++incoming; + if (incoming == count) return UINT32_MAX; } - if (selected->NumArgs() != count || selected->NumPhiBlocks() != count) - return UINT32_MAX; - uint32_t incoming = 0; - while (incoming < count && selected->PhiBlock(incoming) != branch->PhiBlock(arm)) - ++incoming; - if (incoming == count) return UINT32_MAX; child.dwords[word] = selected->Arg(incoming); } const auto next = self(self, child); @@ -3282,17 +3298,24 @@ private: const auto where = std::ranges::find_if(block->Instructions(), [&](const Inst& inst) { return &inst == choice.branch; }); - auto& key = *block->PrependNewInst(where, choice.branch->GetOpcode()); - key.SetFlags(Type::U32); + const auto op = choice.branch->GetOpcode(); + auto& key = *(op == ValueOpcode::Phi ? block->PrependNewInst(where, op) + : op == ValueOpcode::ReadFirstLane + ? block->PrependNewInst(where, op, {Value(0u), choice.branch->Arg(1)}) + : block->PrependNewInst(where, op, {choice.branch->Arg(0), Value(0u), Value(0u)})); + if (op == ValueOpcode::Phi) key.SetFlags(Type::U32); choice.key = Value(&key); } for (const auto& choice: choices) { if (choice.branch == nullptr) continue; auto* key = choice.key.Instruction(); + const bool phi = choice.branch->GetOpcode() == ValueOpcode::Phi; + const bool lane = choice.branch->GetOpcode() == ValueOpcode::ReadFirstLane; for (uint32_t arm = 0; arm < choice.children.size(); ++arm) { const auto child = choice.children[arm]; const auto value = child == UINT32_MAX ? Value(0u) : choices[child].key; - key->AddPhiOperand(choice.branch->PhiBlock(arm), value); + if (phi) key->AddPhiOperand(choice.branch->PhiBlock(arm), value); + else key->SetArg(lane ? 0u : arm + 1u, value); } } plan.handle = &handle; diff --git a/tests/ResourceTrackingTests.cpp b/tests/ResourceTrackingTests.cpp index eb4fa75b7..3a9ef2023 100644 --- a/tests/ResourceTrackingTests.cpp +++ b/tests/ResourceTrackingTests.cpp @@ -5531,6 +5531,159 @@ void TestFiniteImagePhiCycle() { } } +void TestFiniteImageWaterfall() { + namespace CFG = Libs::Graphics::ShaderRecompiler::CFG; + enum class Variant { Valid, Staggered, Cycle, DifferentMask, MixedWord, DifferentSelect, WiderExec, BadLeaf }; + const auto make = [](Variant variant) { + auto fixture = std::make_unique(); + auto& f = *fixture; + const auto lane = f.Emit(ValueOpcode::LaneId); + const auto active = f.Emit(ValueOpcode::ULessThan32, {lane, Value(16u)}); + const auto choose = f.Emit(ValueOpcode::IEqual32, + {f.Emit(ValueOpcode::BitwiseAnd32, {lane, Value(1u)}), Value(0u)}); + const auto other = f.Emit(ValueOpcode::LogicalNot, {choose}); + const auto other_mask = f.Emit(ValueOpcode::LogicalNot, {active}); + std::array table_words; + for (uint32_t word = 0; word < 4u; ++word) table_words[word] = f.UserData(word); + const auto table = f.Buffer(table_words); + std::array a, b, words; + for (uint32_t word = 0; word < 8u; ++word) + for (const bool second : {false, true}) + (second ? b : a)[word] = f.Emit(ValueOpcode::ReadConstBuffer, {table, Value(0u)}, + f.AddMemory({.kind = ResourceKind::ScalarBuffer, + .offset = (second ? 440u : 0u) + word * 4u}, 0x100u + word * 4u)); + if (variant == Variant::BadLeaf) a[3] = lane; + if (variant == Variant::Staggered) { + f.block->terminator.kind = CFG::TerminatorKind::Return; + for (uint32_t word = 0; word < 8u; ++word) { + const auto yes = word == 1u ? f.Emit(ValueOpcode::SelectU32, {other, a[word], b[word]}) : a[word]; + words[word] = f.Emit(ValueOpcode::SelectU32, {choose, yes, b[word]}); + } + } else { + auto* entry = f.block; + auto* left = f.AddBlock(); + auto* right = f.AddBlock(); + auto* merge = f.AddBlock(); + entry->AddBranch(left); + (variant == Variant::Cycle ? merge : entry)->AddBranch(right); + left->AddBranch(merge); + right->AddBranch(merge); + entry->condition = variant == Variant::Cycle ? Value{} : choose; + entry->terminator = variant == Variant::Cycle + ? Block::Terminator{.kind = CFG::TerminatorKind::Branch, .true_block = left} + : Block::Terminator{.kind = CFG::TerminatorKind::ConditionalBranch, + .true_block = left, .false_block = right}; + left->terminator = right->terminator = {.kind = CFG::TerminatorKind::Branch, .true_block = merge}; + merge->terminator = {.kind = variant == Variant::Cycle ? CFG::TerminatorKind::Branch + : CFG::TerminatorKind::Return, + .true_block = variant == Variant::Cycle ? right : nullptr}; + for (uint32_t word = 0; word < 8u; ++word) { + const auto condition = variant == Variant::DifferentSelect && word == 7u ? other : choose; + const auto yes = f.Emit(ValueOpcode::SelectU32, {condition, a[word], Value(0u)}, 0, left); + auto& phi = merge->AppendNewInst(ValueOpcode::Phi, {}, uint64_t(Type::U32)); + const auto no = f.Emit(ValueOpcode::SelectU32, + {choose, b[word], variant == Variant::Cycle ? Value(&phi) : Value(0u)}, 0, right); + if (word & 1u) { + phi.AddPhiOperand(right, no); + phi.AddPhiOperand(left, yes); + } else { + phi.AddPhiOperand(left, yes); + phi.AddPhiOperand(right, no); + } + words[word] = Value(&phi); + } + f.block = merge; + } + for (uint32_t word = 0; word < 8u; ++word) { + if (variant == Variant::MixedWord && word == 7u) continue; + words[word] = f.Emit(ValueOpcode::ReadFirstLane, + {words[word], variant == Variant::DifferentMask && word == 7u ? other_mask : active}); + } + const auto image = f.Image(words); + const auto sampler = f.Sampler({Value(0u), Value(0u), Value(0u), Value(0u)}); + f.Emit(ValueOpcode::ImageSampleRaw, + {image, sampler, f.ImageAddress(), variant == Variant::WiderExec ? Value(true) : active}, + f.AddMemory({.kind = ResourceKind::Image, + .image_dimension = Decoder::ImageDimension::Dim2D}, 0x66cu)); + return fixture; + }; + for (const auto variant : {Variant::Valid, Variant::Staggered, Variant::Cycle}) { + auto fixture = make(variant); + fixture->PlanAndTrack(); + const auto plan = ExtractResourcePlan(fixture->program); + EliminateDeadCode(fixture->program.blocks); + ValidateProgram(fixture->program, true); + const auto& finite = plan.descriptor_sources[plan.info.images[0].source].indirect_descriptor; + Check(finite && finite->sources.size() == 3u, + "waterfall image selection mixed descriptor words or lost a candidate"); + uint32_t lanes = 0u, selects = 0u; + for (const auto* block : fixture->program.blocks) { + std::vector defined; + for (const auto& inst : *block) { + if (inst.GetOpcode() == ValueOpcode::ReadFirstLane || inst.GetOpcode() == ValueOpcode::SelectU32) { + lanes += inst.GetOpcode() == ValueOpcode::ReadFirstLane; + selects += inst.GetOpcode() == ValueOpcode::SelectU32; + for (size_t arg = 0; arg < inst.NumArgs(); ++arg) { + const auto* input = inst.Arg(arg).Resolve().TryInstruction(); + Check(input == nullptr || input->Parent() != block || + std::ranges::find(defined, input) != defined.end(), + "projected descriptor choice uses an SSA value defined later in its block"); + } + } + defined.push_back(&inst); + } + } + Check(lanes == 1u && selects == 2u, + "waterfall descriptor selection did not share one ordinal graph across all eight words"); + if (variant != Variant::Valid) continue; + LinearTestMemory memory; + using F = Libs::Graphics::Prospero::BufferFormat; + for (uint32_t row = 0; row < 2u; ++row) { + const auto word = (256u + row * 440u) / 4u; + memory.words[word] = 0x1000u + row * 0x1000u; + memory.words[word + 1u] = uint32_t(F::k32Float) << 20u; + memory.words[word + 3u] = Libs::Graphics::DstSel(4, 0, 0, 0) | + (uint32_t(Libs::Graphics::Prospero::ImageType::kColor2D) << 28u); + } + std::array user_data{uint32_t(memory.base) + 256u, 8u << 16u, 60u, 0x5204u}; + SrtRuntime runtime{.user_data = user_data, .read_memory = ReadLinearTestMemory, + .userdata = &memory, .read_specialization_memory = ReadLinearTestMemory}; + ResourceSnapshot snapshot; + ResourceSpecialization specialization, populated; + for (const uint32_t count : {0u, 60u, 0u, 60u}) { + user_data[2] = count; + memory.reads = 0u; + Check(MaterializeResources(plan, runtime, snapshot, specialization), + "finite waterfall resource plan could not refresh its scalar descriptor leaves"); + Check(snapshot.images.size() == (count == 0u ? 1u : 3u) && + (count == 0u ? memory.reads == 0u : memory.reads != 0u), + "finite waterfall leaves froze empty table bounds or retained stale image candidates"); + if (count != 0u) populated = specialization; + } + memory.words[256u / 4u] += 0x100u; + Check(MaterializeResources(plan, runtime, snapshot, specialization) && specialization == populated && + std::ranges::any_of(snapshot.images, [](const auto& image) { return image.dwords[0] == 0x1100u; }), + "finite image payload refresh changed its format specialization or reused an old leaf"); + } + for (const auto variant : {Variant::DifferentMask, Variant::MixedWord, Variant::DifferentSelect, + Variant::WiderExec, Variant::BadLeaf}) { + auto fixture = make(variant); + const auto choices = [&] { + uint32_t count = 0u; + for (const auto* block : fixture->program.blocks) + for (const auto& inst : *block) + count += inst.GetOpcode() == ValueOpcode::ReadFirstLane || inst.GetOpcode() == ValueOpcode::SelectU32; + return count; + }; + const auto before = choices(); + CheckFatal([&] { fixture->PlanAndTrack(); }, "not a valid runtime value", + "finite waterfall accepted mismatched lane masks, choices or an unsupported leaf"); + Check(choices() <= before && !fixture->program.resource_tracking_complete && + fixture->program.descriptor_sources.empty(), + "rejected finite descriptor graph left projected instructions or committed sources"); + } +} + void TestFiniteInlineSampler() { namespace CFG = Libs::Graphics::ShaderRecompiler::CFG; Fixture fixture; @@ -6627,6 +6780,7 @@ int main() { Run("packed scalar buffer selection", TestPackedScalarBufferSelection); Run("conditional sampler phi", TestConditionalSamplerPhi); Run("finite image phi cycle", TestFiniteImagePhiCycle); + Run("finite image waterfall", TestFiniteImageWaterfall); Run("finite inline sampler", TestFiniteInlineSampler); Run("finite image bit scan sentinel", TestFiniteImageBitScanSentinel); Run("runtime-rooted loop", TestLoopCycleEnteredThroughRuntimeValue); diff --git a/tests/ShaderRecompilerComputeTests.cpp b/tests/ShaderRecompilerComputeTests.cpp index 5ab7c7db3..80d5aaa25 100644 --- a/tests/ShaderRecompilerComputeTests.cpp +++ b/tests/ShaderRecompilerComputeTests.cpp @@ -37272,6 +37272,96 @@ void CheckIndirectBufferStore(VulkanHarness &vulkan) { std::printf("[compute] %-32s ok\n", test.name); } +void CheckFiniteFirstLaneImageChoice(VulkanHarness &vulkan) { + using O = ShaderOpcode; + constexpr u32 table = 512u, stride = 440u, poison = 0xdeadbeefu; + for (const u32 wave : {32u, 64u}) + for (const u32 count : {0u, 4u}) { + TestCase test; + test.name = "FiniteFirstLaneImageChoice"; + test.initial.assign((table + 4u * stride) / 4u, poison); + for (u32 row = 0; row < 4u; ++row) { + auto image = MakeSampledTextureData(Prospero::BufferFormat::k32Float); + image[0] = 10u + row; + image[3] = DstSel(4, 0, 0, 1) | + (static_cast(Prospero::ImageType::kColor2D) << 28u); + std::copy_n(image.begin(), 8, test.initial.begin() + (table + row * stride) / 4u); + } + auto records = MakeStructuredStorageBufferData(stride, count); + records[0] = table; + std::copy_n(records.begin(), 4, test.user_data.begin() + 56); + const auto output = MakeStructuredStorageBufferData(0, wave * 4u); + std::copy_n(output.begin(), 4, test.user_data.begin() + 48); + test.has_user_data = test.has_compute_info = true; + test.compute_info.wave_size = wave; + test.compute_info.threads_num[0] = wave; + test.compute_info.threads_num[1] = test.compute_info.threads_num[2] = 1; + test.compute_info.thread_ids_num = 1; + auto &code = test.code; + code.push_back(EncodeSop1(0x04, 40, 126)); + code.push_back(EncodeVop2(0x1b, 1, InlineU32(3), 0)); + for (u32 word = 0; word < 8u; ++word) + code.push_back(EncodeVop1(0x01, 20u + word, InlineU32(0))); + // Masked vector descriptor writes become synchronized Select/Phi chains. + for (u32 row = 0; row < 4u; ++row) { + code.push_back(EncodeVopc(0xd2, InlineU32(row), 1)); + const auto skip = code.size(); + code.push_back(0u); + code.insert(code.end(), {EncodeSmem0(0x0b, 8, 28), EncodeSmem1(row * stride, 125)}); + for (u32 word = 0; word < 8u; ++word) + code.push_back(EncodeVop1(0x01, 20u + word, 8u + word)); + code[skip] = EncodeSopp(0x08, code.size() - skip - 1u); + code.push_back(EncodeSop1(0x04, 126, 40)); + } + AppendVMovLiteral(&code, 5, poison); + AppendVMovLiteral(&code, 10, 0x3f000000u); + AppendVMovLiteral(&code, 11, 0x3f000000u); + code.push_back(EncodeVop2(0x1b, 2, InlineU32(31), 0)); + code.push_back(EncodeVopc(0xd5, InlineU32(0), 2)); + const auto loop = code.size(); + // Same native waterfall as 7ceb0f3417f926f9, including narrowing EXEC. + for (u32 word = 0; word < 8u; ++word) + code.push_back(EncodeVop1(0x02, 32u + word, Vgpr(20u + word))); + code.push_back(EncodeSop1(0x04, 44, 126)); + for (u32 word = 0; word < 8u; ++word) + code.push_back(EncodeVopc(0xd2, 32u + word, 20u + word)); + code.insert(code.end(), {EncodeMimg0(0x27, 1), EncodeMimg1(5, 10, 8, 13), + EncodeSop2(0x15, 126, 44, 126)}); + code.push_back(EncodeSopp(0x09, loop - code.size() - 1u)); + code.push_back(EncodeSop1(0x04, 126, 40)); + code.push_back(EncodeVop2(0x1a, 30, InlineU32(2), 0)); + AppendBufferStoreDword(&code, 5, 30); + AppendEnd(&code); + test.opcodes = {O::S_MOV_B64, O::V_AND_B32, O::V_MOV_B32, + O::V_CMPX_EQ_U32, O::V_CMPX_NE_U32, O::S_CBRANCH_EXECZ, + O::S_BUFFER_LOAD_DWORDX8, O::V_READFIRSTLANE_B32, O::IMAGE_SAMPLE, + O::S_ANDN2_B64, O::S_CBRANCH_EXECNZ, O::V_LSHLREV_B32, + O::BUFFER_STORE_DWORD, O::S_ENDPGM}; + const auto compiled = CompileCase(test, vulkan.SubgroupSize()); + Require(test.name, "finite correlated image choices", compiled.resources.images.size() <= 5u, + "descriptor words formed an uncorrelated product"); + std::vector images; + for (const auto &descriptor : compiled.resources.images) + images.push_back(vulkan.CreateImage2D(test.name, 1, 1, vk::Format::eR32Sfloat, + vk::ImageUsageFlagBits::eSampled, + {std::bit_cast(float(descriptor.dwords[0]))}, 1, + vk::ImageLayout::eShaderReadOnlyOptimal)); + const auto sampler = vulkan.CreateSampler(test.name, 1); + auto buffer = vulkan.CreateStorageBuffer(test.name, test.initial, test.initial.size()); + vulkan.Dispatch(test, compiled, buffer, nullptr, nullptr, nullptr, nullptr, sampler, images); + test.expected = test.initial; + for (u32 lane = 0; lane < wave; ++lane) + if ((lane & 31u) != 0u) + test.expected[lane] = count == 0u ? 0u : std::bit_cast(float(10u + (lane & 3u))); + CompareWords(test, "first active lane, descriptor correlation and inactive canaries", test.expected, + vulkan.ReadBuffer(test.name, buffer, test.expected.size())); + vulkan.DestroyBuffer(&buffer); + vulkan.Device().destroySampler(sampler); + for (auto &image : images) vulkan.DestroyImage(&image); + std::printf("[compute] %s wave=%u count=%u ok\n", test.name, wave, count); + } +} + void CheckFiniteInlineSamplerPhi(VulkanHarness &vulkan) { TestCase test; test.name = "FiniteInlineSamplerPhi"; @@ -46593,6 +46683,7 @@ int main(int argc, char **argv) { CheckNestedPointerImageTable(vulkan); CheckGuardedRecordImageTable(vulkan); CheckIndirectImageKeySwitch(vulkan); + CheckFiniteFirstLaneImageChoice(vulkan); CheckFiniteInlineSamplerPhi(vulkan); RunCase(&vulkan, ImageCubeGradientsPreserveDerivatives()); return 0; @@ -46769,6 +46860,7 @@ int main(int argc, char **argv) { CheckIndirectImageKeySwitch(vulkan); CheckDynamicSampleOffsets(vulkan, 32u); CheckDynamicSampleOffsets(vulkan, 64u); + CheckFiniteFirstLaneImageChoice(vulkan); CheckFiniteInlineSamplerPhi(vulkan); CheckWave64WholeWaveResults(); CheckPs5GameExampleImageClearRuntimeShape();