From 3e25eb09987b07e2c08101dfcb55c2f64c1999f0 Mon Sep 17 00:00:00 2001 From: nmzik Date: Sun, 23 Aug 2026 00:46:37 +0200 Subject: [PATCH] Fix wave64 LDS synchronization on subgroup32 hosts --- .../host_gpu/renderer/pipeline/shaders.cpp | 3 +- .../host_gpu/renderer/renderCompute.cpp | 4 +- src/graphics/host_gpu/vulkanInstance.h | 3 +- .../presentation/window/vulkanWindow.cpp | 20 ++- .../shader/recompiler/ShaderRecompiler.cpp | 9 ++ .../ir/passes/SharedMemoryBarrier.cpp | 153 ++++++++++++++++++ .../ir/passes/SharedMemoryBarrier.h | 15 ++ src/graphics/shader/shader.cpp | 7 +- src/graphics/shader/shader.h | 4 +- tests/shaderCfgTests.cpp | 145 ++++++++++++++++- 10 files changed, 348 insertions(+), 15 deletions(-) create mode 100644 src/graphics/shader/recompiler/ir/passes/SharedMemoryBarrier.cpp create mode 100644 src/graphics/shader/recompiler/ir/passes/SharedMemoryBarrier.h diff --git a/src/graphics/host_gpu/renderer/pipeline/shaders.cpp b/src/graphics/host_gpu/renderer/pipeline/shaders.cpp index 5f559db05..23b0953af 100644 --- a/src/graphics/host_gpu/renderer/pipeline/shaders.cpp +++ b/src/graphics/host_gpu/renderer/pipeline/shaders.cpp @@ -1019,8 +1019,7 @@ void CreatePipelineInternal(GraphicContext& graphics, PipelineCache::ComputePipe comp_shader_stage_info.pSpecializationInfo = nullptr; EXIT_IF(!input_info.stage); const auto wave_size = input_info.stage.program->wave_size; - if (graphics.subgroup_size_control_enabled && - (graphics.required_subgroup_size_stages & vk::ShaderStageFlagBits::eCompute) && + if (graphics.compute_subgroup_size_control_enabled && wave_size >= graphics.min_subgroup_size && wave_size <= graphics.max_subgroup_size) { comp_subgroup_size.sType = vk::StructureType::ePipelineShaderStageRequiredSubgroupSizeCreateInfo; diff --git a/src/graphics/host_gpu/renderer/renderCompute.cpp b/src/graphics/host_gpu/renderer/renderCompute.cpp index c9486bdc3..f780249f3 100644 --- a/src/graphics/host_gpu/renderer/renderCompute.cpp +++ b/src/graphics/host_gpu/renderer/renderCompute.cpp @@ -210,7 +210,9 @@ void RenderExecutor::DispatchDirect(uint64_t submit_id, RenderCommandBuffer& buf ShaderComputeInputInfo input_info {}; std::span cs_shader; - if (!ShaderCompileInfoCS(cs_regs, sh_regs, input_info, cs_shader)) { + if (!ShaderCompileInfoCS(cs_regs, sh_regs, + !m_context.GetGraphics().compute_wave64_supported, input_info, + cs_shader)) { EXIT("ShaderCompileInfoCS failed for dispatch with CS shader 0x%016" PRIx64 "\n", cs_regs.cs_regs.data_addr); } diff --git a/src/graphics/host_gpu/vulkanInstance.h b/src/graphics/host_gpu/vulkanInstance.h index 94c16a6f9..e2e7c66bf 100644 --- a/src/graphics/host_gpu/vulkanInstance.h +++ b/src/graphics/host_gpu/vulkanInstance.h @@ -25,7 +25,8 @@ struct VulkanInstance { VmaAllocator allocator = nullptr; bool memory_budget_ext_enabled = false; bool rt_extensions_enabled = false; - bool subgroup_size_control_enabled = false; + bool compute_subgroup_size_control_enabled = false; + bool compute_wave64_supported = false; bool sample_rate_shading_enabled = false; uint32_t subgroup_size = 0; uint32_t min_subgroup_size = 0; diff --git a/src/graphics/presentation/window/vulkanWindow.cpp b/src/graphics/presentation/window/vulkanWindow.cpp index 2b65278bf..b4ff8f37e 100644 --- a/src/graphics/presentation/window/vulkanWindow.cpp +++ b/src/graphics/presentation/window/vulkanWindow.cpp @@ -493,14 +493,19 @@ static void VulkanInitSubgroupSizeControl(vk::PhysicalDevice physical_device, graphics.min_subgroup_size = subgroup_size_control.minSubgroupSize; graphics.max_subgroup_size = subgroup_size_control.maxSubgroupSize; graphics.required_subgroup_size_stages = subgroup_size_control.requiredSubgroupSizeStages; - graphics.subgroup_size_control_enabled = features13.subgroupSizeControl == VK_TRUE && - subgroup_size_control.minSubgroupSize <= 64 && - subgroup_size_control.maxSubgroupSize >= 64; + graphics.compute_subgroup_size_control_enabled = + features13.subgroupSizeControl == VK_TRUE && + (graphics.required_subgroup_size_stages & vk::ShaderStageFlagBits::eCompute) && + subgroup_size_control.minSubgroupSize <= 64 && + subgroup_size_control.maxSubgroupSize >= 64; + graphics.compute_wave64_supported = + graphics.subgroup_size == 64u || graphics.compute_subgroup_size_control_enabled; - LOGF("Vulkan subgroup: default=%u min=%u max=%u stages=0x%08x size_control=%s\n", + LOGF("Vulkan subgroup: default=%u min=%u max=%u stages=0x%08x size_control=%s wave64=%s\n", graphics.subgroup_size, graphics.min_subgroup_size, graphics.max_subgroup_size, static_cast(graphics.required_subgroup_size_stages), - graphics.subgroup_size_control_enabled ? "true" : "false"); + graphics.compute_subgroup_size_control_enabled ? "true" : "false", + graphics.compute_wave64_supported ? "true" : "false"); } static vk::Device VulkanCreateDevice(vk::PhysicalDevice physical_device, const VulkanExtensions& r, @@ -645,8 +650,9 @@ static vk::Device VulkanCreateDevice(vk::PhysicalDevice physical_device, const V robustness2.nullDescriptor = supported_robustness2.nullDescriptor; } - const bool subgroup_size_control_enabled = graphics.subgroup_size_control_enabled && - supported_features13.subgroupSizeControl == VK_TRUE; + const bool subgroup_size_control_enabled = + graphics.compute_subgroup_size_control_enabled && + supported_features13.subgroupSizeControl == VK_TRUE; auto features13 = required_features13; #if defined(__APPLE__) diff --git a/src/graphics/shader/recompiler/ShaderRecompiler.cpp b/src/graphics/shader/recompiler/ShaderRecompiler.cpp index 27b2fbbef..9c49ae8ff 100644 --- a/src/graphics/shader/recompiler/ShaderRecompiler.cpp +++ b/src/graphics/shader/recompiler/ShaderRecompiler.cpp @@ -15,6 +15,7 @@ #include "graphics/shader/recompiler/ir/passes/ResourceMaterialization.h" #include "graphics/shader/recompiler/ir/passes/ResourceTracking.h" #include "graphics/shader/recompiler/ir/passes/ShaderInfoCollection.h" +#include "graphics/shader/recompiler/ir/passes/SharedMemoryBarrier.h" #include "graphics/shader/recompiler/ir/passes/SrtWalker.h" #include "graphics/shader/recompiler/ir/passes/SsaRewrite.h" @@ -741,6 +742,14 @@ bool TryRecompile(std::span code, const CompileOptions& options, IR::RemoveIdentities(ir.values->blocks); IR::EliminateDeadCode(ir.values->blocks); } + if (options.stage == ShaderType::Compute) { + const auto lds_barriers = + IR::InsertSharedMemoryBarriers(*ir.values, ir.wave_size, *compute); + if (lds_barriers.inserted_barriers != 0) { + LOGF("%s wave64 LDS synchronization: barriers=%" PRIu32 "\n", GetDumpLabel(options), + lds_barriers.inserted_barriers); + } + } std::string srt_error; if (!IR::BuildSrtPlan(ir, &srt_error)) { LOGF("%s SRT planning failed: %s\n", GetDumpLabel(options), srt_error.c_str()); diff --git a/src/graphics/shader/recompiler/ir/passes/SharedMemoryBarrier.cpp b/src/graphics/shader/recompiler/ir/passes/SharedMemoryBarrier.cpp new file mode 100644 index 000000000..a211f55c3 --- /dev/null +++ b/src/graphics/shader/recompiler/ir/passes/SharedMemoryBarrier.cpp @@ -0,0 +1,153 @@ +#include "graphics/shader/recompiler/ir/passes/SharedMemoryBarrier.h" + +#include "graphics/shader/recompiler/ir/ValueProgram.h" + +#include +#include +#include +#include + +namespace Libs::Graphics::ShaderRecompiler::IR { +namespace { + +enum class LdsPhase { None, Read, Write }; + +LdsPhase GetLdsPhase(const ValueProgram& program, const Inst& inst) { + const auto access = SharedAccessOf(inst.GetOpcode()); + if (access != SharedAccess::Read && access != SharedAccess::Write) { + return LdsPhase::None; + } + const auto index = inst.Flags().index; + if (index >= program.memory_info.size() || + program.memory_info[index].kind != ResourceKind::Lds) { + return LdsPhase::None; + } + return access == SharedAccess::Read ? LdsPhase::Read : LdsPhase::Write; +} + +bool DependsOnInvocation(Value value) { + std::queue queue; + std::unordered_set visited; + queue.push(value); + while (!queue.empty()) { + value = queue.front().Resolve(); + queue.pop(); + auto* inst = value.TryInstruction(); + if (inst == nullptr || !visited.insert(inst).second) { + continue; + } + if (inst->GetOpcode() == ValueOpcode::LaneId) { + return true; + } + if (inst->GetOpcode() == ValueOpcode::GetBuiltin) { + const auto kind = static_cast(inst->Arg(0).U32()); + if (kind == StageInputKind::LocalInvocationId || + kind == StageInputKind::LocalInvocationIndex || + kind == StageInputKind::GlobalInvocationId) { + return true; + } + } + for (size_t index = 0; index < inst->NumArgs(); index++) { + queue.push(inst->Arg(index)); + } + } + return false; +} + +uint32_t InsertBarrier(Block& block, Block::iterator position) { + block.PrependNewInst(position, ValueOpcode::Barrier); + return 1; +} + +uint32_t InsertBlockBarriers(ValueProgram& program, Block& block) { + uint32_t inserted = 0; + LdsPhase phase = LdsPhase::None; + for (auto inst = block.begin(); inst != block.end(); ++inst) { + if (inst->GetOpcode() == ValueOpcode::Barrier) { + phase = LdsPhase::None; + continue; + } + const auto next = GetLdsPhase(program, *inst); + if (next == LdsPhase::None) { + continue; + } + if (phase != LdsPhase::None && phase != next) { + inserted += InsertBarrier(block, inst); + } + phase = next; + } + if (phase != LdsPhase::None) { + block.AppendNewInst(ValueOpcode::Barrier); + inserted++; + } + return inserted; +} + +uint32_t InsertMergeBarrier(Block& block) { + auto position = std::ranges::find_if( + block, [](const Inst& inst) { return inst.GetOpcode() != ValueOpcode::Phi; }); + if (position != block.end() && position->GetOpcode() == ValueOpcode::Barrier) { + return 0; + } + return InsertBarrier(block, position); +} + +} // namespace + +SharedMemoryBarrierStats InsertSharedMemoryBarriers(ValueProgram& program, uint32_t wave_size, + const ShaderComputeInputInfo& compute_info) { + SharedMemoryBarrierStats stats; + const auto threadgroup_size = compute_info.threads_num[0] * compute_info.threads_num[1] * + compute_info.threads_num[2]; + if (wave_size != 64u || !compute_info.needs_lds_barriers || + compute_info.lds_size_dwords == 0u || threadgroup_size != 64u) { + return stats; + } + if (program.blocks.size() != program.block_info.size()) { + return stats; + } + for (const auto& info: program.block_info) { + if (info.terminator.kind == CFG::TerminatorKind::ConditionalBranch && + info.terminator.merge_block == UINT32_MAX && DependsOnInvocation(info.condition)) { + return stats; + } + } + + std::unordered_map blocks; + for (size_t index = 0; index < program.blocks.size(); index++) { + blocks.emplace(program.block_info[index].id, program.blocks[index]); + } + + uint32_t divergence_depth = 0; + std::unordered_map divergence_ends; + std::unordered_set synchronized_merges; + for (size_t index = 0; index < program.blocks.size(); index++) { + const auto id = program.block_info[index].id; + if (const auto end = divergence_ends.find(id); end != divergence_ends.end()) { + EXIT_IF(end->second > divergence_depth); + divergence_depth -= end->second; + } + + if (divergence_depth == 0) { + stats.inserted_barriers += InsertBlockBarriers(program, *program.blocks[index]); + } + + const auto& info = program.block_info[index]; + const auto& term = info.terminator; + if (term.kind != CFG::TerminatorKind::ConditionalBranch || term.merge_block == UINT32_MAX || + !DependsOnInvocation(info.condition)) { + continue; + } + if (divergence_depth == 0 && synchronized_merges.insert(term.merge_block).second) { + const auto merge = blocks.find(term.merge_block); + if (merge != blocks.end()) { + stats.inserted_barriers += InsertMergeBarrier(*merge->second); + } + } + divergence_depth++; + divergence_ends[term.merge_block]++; + } + return stats; +} + +} // namespace Libs::Graphics::ShaderRecompiler::IR diff --git a/src/graphics/shader/recompiler/ir/passes/SharedMemoryBarrier.h b/src/graphics/shader/recompiler/ir/passes/SharedMemoryBarrier.h new file mode 100644 index 000000000..a42faf456 --- /dev/null +++ b/src/graphics/shader/recompiler/ir/passes/SharedMemoryBarrier.h @@ -0,0 +1,15 @@ +#pragma once + +#include "graphics/shader/recompiler/ir/ShaderIR.h" + +namespace Libs::Graphics::ShaderRecompiler::IR { + +struct SharedMemoryBarrierStats { + uint32_t inserted_barriers = 0; +}; + +[[nodiscard]] SharedMemoryBarrierStats +InsertSharedMemoryBarriers(ValueProgram& program, uint32_t wave_size, + const ShaderComputeInputInfo& compute_info); + +} // namespace Libs::Graphics::ShaderRecompiler::IR diff --git a/src/graphics/shader/shader.cpp b/src/graphics/shader/shader.cpp index 92c50d136..1a04c2183 100644 --- a/src/graphics/shader/shader.cpp +++ b/src/graphics/shader/shader.cpp @@ -1212,10 +1212,12 @@ bool ShaderCompileInfoPS(const HW::PixelShaderInfo& regs, const HW::ShaderRegist } bool ShaderCompileInfoCS(const HW::ComputeShaderInfo& regs, const HW::ShaderRegisters& sh, - ShaderComputeInputInfo& info, std::span& spirv) { + bool needs_lds_barriers, ShaderComputeInputInfo& info, + std::span& spirv) { spirv = {}; ShaderGetStaticInputInfoCS(regs, sh, info); + info.needs_lds_barriers = needs_lds_barriers; const auto shader_hash = regs.cs_regs.data_addr; const auto program_id = ShaderGetIdCS(regs, info, false); const auto key = MakeShaderStageProgramKey(ShaderType::Compute, shader_hash, program_id); @@ -1344,9 +1346,11 @@ void ShaderDbgDumpInputInfo(const ShaderComputeInputInfo& info) { "\t thread_ids_num = %d\n" "\t wave_size = %u\n" "\t lds_size_dwords = %u\n" + "\t needs_lds_barriers = %s\n" "\t threads_num = {%u, %u, %u}\n" "\t tg_size_en = %s\n", info.workgroup_register, info.thread_ids_num, info.wave_size, info.lds_size_dwords, + info.needs_lds_barriers ? "true" : "false", info.threads_num[0], info.threads_num[1], info.threads_num[2], info.tg_size_en ? "true" : "false"); LOGF("\t threadgroup_id = {%s, %s, %s}\n", info.group_id[0] ? "true" : "false", @@ -1742,6 +1746,7 @@ ShaderId ShaderGetIdCS(const HW::ComputeShaderInfo& regs, const ShaderComputeInp ret.ids.push_back(input_info.thread_ids_num); ret.ids.push_back(input_info.lds_size_dwords); ret.ids.push_back(input_info.scratch_size_dwords); + ret.ids.push_back(static_cast(input_info.needs_lds_barriers)); for (int i = 0; i < 3; i++) { ret.ids.push_back(input_info.threads_num[i]); diff --git a/src/graphics/shader/shader.h b/src/graphics/shader/shader.h index d715c9d13..26af3f98f 100644 --- a/src/graphics/shader/shader.h +++ b/src/graphics/shader/shader.h @@ -107,6 +107,7 @@ struct ShaderComputeInputInfo { uint32_t scratch_size_dwords = 0; bool group_id[3] = {false, false, false}; bool dispatch_thread_dimensions = false; + bool needs_lds_barriers = false; uint32_t wave_size = 64; int thread_ids_num = 0; int workgroup_register = 0; @@ -268,7 +269,8 @@ bool ShaderCompileInfoPS(const HW::PixelShaderInfo& regs, const HW::ShaderRegist std::span target_export_mapping, ShaderPixelInputInfo& input_info, std::span& spirv); bool ShaderCompileInfoCS(const HW::ComputeShaderInfo& regs, const HW::ShaderRegisters& sh, - ShaderComputeInputInfo& input_info, std::span& spirv); + bool needs_lds_barriers, ShaderComputeInputInfo& input_info, + std::span& spirv); bool ShaderCompileSpirvVS(const HW::VertexShaderInfo& regs, const HW::ShaderRegisters& sh, ShaderVertexInputInfo& input_info, std::vector& spirv); bool ShaderCompileSpirvPS(const HW::PixelShaderInfo& regs, const HW::ShaderRegisters& sh, diff --git a/tests/shaderCfgTests.cpp b/tests/shaderCfgTests.cpp index f19b4bff4..c31c36c13 100644 --- a/tests/shaderCfgTests.cpp +++ b/tests/shaderCfgTests.cpp @@ -22,6 +22,7 @@ #include "graphics/shader/recompiler/ir/passes/ReadLaneElimination.h" #include "graphics/shader/recompiler/ir/passes/ResourceTracking.h" #include "graphics/shader/recompiler/ir/passes/ShaderInfoCollection.h" +#include "graphics/shader/recompiler/ir/passes/SharedMemoryBarrier.h" #include "graphics/shader/recompiler/ir/passes/SrtWalker.h" #include "graphics/shader/recompiler/ir/passes/SsaRewrite.h" #include "graphics/shader/shader.h" @@ -11409,7 +11410,7 @@ void TestComputeLdsAllocationIdentity() { regs.cs_regs.lds_size = encoded_lds_size; ShaderComputeInputInfo input_info{}; std::span spirv; - Check(ShaderCompileInfoCS(regs, sh, input_info, spirv), + Check(ShaderCompileInfoCS(regs, sh, false, input_info, spirv), "compute LDS allocation shader did not compile"); Check(input_info.lds_size_dwords == expected_dwords, "COMPUTE_PGM_RSRC2 LDS allocation units were not decoded"); @@ -11437,6 +11438,12 @@ void TestComputeLdsAllocationIdentity() { ShaderGetIdCS(regs, lds_896, true), "compute pipeline identity omitted the LDS allocation"); + auto split_wave = lds_896; + split_wave.needs_lds_barriers = true; + Check(ShaderGetIdCS(regs, lds_896, true) != + ShaderGetIdCS(regs, split_wave, true), + "compute shader identity omitted split-wave LDS synchronization"); + auto tg_size_disabled = lds_896; auto tg_size_enabled = lds_896; tg_size_enabled.tg_size_en = true; @@ -11467,7 +11474,7 @@ void TestComputeLdsAllocationIdentity() { regs.cs_regs.lds_size = decode_lds_field(lds_896_rsrc2); ShaderComputeInputInfo scratch_info{}; std::span scratch_spirv; - Check(ShaderCompileInfoCS(regs, sh, scratch_info, scratch_spirv), + Check(ShaderCompileInfoCS(regs, sh, false, scratch_info, scratch_spirv), "compute scratch metadata shader did not compile"); Check(scratch_info.scratch_size_dwords == 7, "AGC per-thread scratch size was not propagated"); @@ -11498,6 +11505,138 @@ void TestComputeLdsAllocationIdentity() { CheckSpirvBinaryValidates(append_result.spirv); } +void TestWave64LdsSynchronization() { + const uint32_t shader[] = { + EncodeDs0(0x0d), // ds_write_b32 v0, v1 + EncodeDs1(0, 1, 0), + EncodeSopp(0x01), + }; + + const auto compile = [&](bool needs_lds_barriers, uint32_t wave_size, + uint32_t threads) { + ShaderComputeInputInfo input_info{}; + input_info.lds_size_dwords = 128; + input_info.needs_lds_barriers = needs_lds_barriers; + input_info.threads_num[0] = threads; + input_info.threads_num[1] = 1; + input_info.threads_num[2] = 1; + + auto options = MakeCompileOptions(ShaderType::Compute); + options.wave_size = wave_size; + options.input_info.compute = &input_info; + ShaderRecompiler::CompileResult result; + std::string error; + Check(ShaderRecompiler::TryRecompile(shader, options, result, &error), + error.c_str()); + CheckSpirvBinaryValidates(result.spirv); + return result; + }; + + const auto native_wave64 = compile(false, 64, 64); + Check(!SpirvContainsOpcode(native_wave64.spirv, 224), + "native wave64 shader gained a synthetic workgroup barrier"); + + const auto split_wave64 = compile(true, 64, 64); + Check(SpirvContainsOpcode(split_wave64.spirv, 224), + "split wave64 LDS shader omitted workgroup synchronization"); + + const auto split_wave32 = compile(true, 32, 64); + Check(!SpirvContainsOpcode(split_wave32.spirv, 224), + "wave32 shader gained wave64 synchronization"); + + const auto larger_workgroup = compile(true, 64, 128); + Check(!SpirvContainsOpcode(larger_workgroup.spirv, 224), + "multi-wave workgroup gained unsupported synthetic synchronization"); +} + +void TestSharedMemoryBarrierSafety() { + using namespace ShaderRecompiler::IR; + + const auto make_program = [](size_t block_count) { + ValueProgram program; + program.memory_info.push_back({.kind = ResourceKind::Lds}); + for (size_t index = 0; index < block_count; index++) { + program.block_storage.push_back(std::make_unique()); + program.blocks.push_back(program.block_storage.back().get()); + program.block_info.emplace_back(); + program.block_info.back().id = static_cast(index); + } + return program; + }; + const auto append_write = [](Block& block) { + auto& inst = block.AppendNewInst( + ValueOpcode::WriteSharedU32, + {Value(0u), Value(1u), Value(true)}); + inst.SetFlags(MemoryFlags{.index = 0}); + }; + const auto append_read = [](Block& block) { + auto& inst = block.AppendNewInst(ValueOpcode::LoadSharedU32, + {Value(0u), Value(true)}); + inst.SetFlags(MemoryFlags{.index = 0}); + }; + const auto count_barriers = [](const Block& block) { + return std::ranges::count_if(block, [](const Inst& inst) { + return inst.GetOpcode() == ValueOpcode::Barrier; + }); + }; + const auto divergent_condition = [](ValueProgram& program, size_t block) { + IREmitter ir(program.blocks[block]); + const auto local_id = U32(ir.Emit( + ValueOpcode::GetBuiltin, + {Value(static_cast(StageInputKind::LocalInvocationId)), + Value(0u)})); + return ir.INotEqual(local_id, U32(Value(0u))); + }; + + ShaderComputeInputInfo compute_info{}; + compute_info.needs_lds_barriers = true; + compute_info.lds_size_dwords = 128; + compute_info.threads_num[0] = 64; + compute_info.threads_num[1] = 1; + compute_info.threads_num[2] = 1; + + auto phases = make_program(1); + append_write(*phases.blocks[0]); + append_read(*phases.blocks[0]); + const auto phase_stats = InsertSharedMemoryBarriers(phases, 64u, compute_info); + std::vector phase_opcodes; + for (const auto& inst : *phases.blocks[0]) { + phase_opcodes.push_back(inst.GetOpcode()); + } + Check(phase_stats.inserted_barriers == 2 && phase_opcodes.size() == 4 && + phase_opcodes[0] == ValueOpcode::WriteSharedU32 && + phase_opcodes[1] == ValueOpcode::Barrier && + phase_opcodes[2] == ValueOpcode::LoadSharedU32 && + phase_opcodes[3] == ValueOpcode::Barrier, + "LDS write/read phases were not separated by a barrier"); + + auto selection = make_program(3); + selection.block_info[0].terminator.kind = + ShaderRecompiler::CFG::TerminatorKind::ConditionalBranch; + selection.block_info[0].terminator.merge_block = 2; + selection.block_info[0].condition = divergent_condition(selection, 0); + append_write(*selection.blocks[1]); + append_read(*selection.blocks[2]); + const auto selection_stats = + InsertSharedMemoryBarriers(selection, 64u, compute_info); + Check(selection_stats.inserted_barriers == 2 && + count_barriers(*selection.blocks[1]) == 0 && + selection.blocks[2]->begin()->GetOpcode() == + ValueOpcode::Barrier, + "workgroup barrier was placed inside divergent selection flow"); + + auto loop = make_program(1); + loop.block_info[0].terminator.kind = + ShaderRecompiler::CFG::TerminatorKind::ConditionalBranch; + loop.block_info[0].condition = divergent_condition(loop, 0); + append_write(*loop.blocks[0]); + append_read(*loop.blocks[0]); + const auto loop_stats = InsertSharedMemoryBarriers(loop, 64u, compute_info); + Check(loop_stats.inserted_barriers == 0 && + count_barriers(*loop.blocks[0]) == 0, + "workgroup barrier was inserted into divergent loop control"); +} + void TestPixelProgramCacheBindingIdentity() { const uint32_t shader_01[] = {0xbf810000u}; const uint32_t shader_10[] = {0xbf810000u}; @@ -12096,6 +12235,8 @@ int main() { TestGraphicsCreateInterpolantMapping(); TestNewShaderRecompilerPixelPipelineEntry(); TestComputeLdsAllocationIdentity(); + TestWave64LdsSynchronization(); + TestSharedMemoryBarrierSafety(); TestPixelProgramCacheBindingIdentity(); TestGraphicsPushConstantPlacement(); TestNewShaderRecompilerUnsupportedMemoryDecode();