From dc9280d75db785b23ed308daf2393937596fd714 Mon Sep 17 00:00:00 2001 From: nmzik Date: Thu, 24 Sep 2026 19:02:32 +0400 Subject: [PATCH] shader: add GLC/DLC buffer cache policy support Co-authored-by: Mac --- .../recompiler/backend/spirv/SpirvEmitter.cpp | 1 + .../backend/spirv/spirvEmitterInternal.h | 14 ++-- .../backend/spirv/spirvEmitterMemory.cpp | 19 ++--- .../spirv/spirvEmitterMemoryHelpers.cpp | 4 +- .../backend/spirv/spirvEmitterModule.cpp | 10 +++ .../recompiler/frontend/decode/MemoryOps.cpp | 2 + .../frontend/decode/ShaderDecoder.cpp | 4 +- .../frontend/decode/ShaderDecoder.h | 1 + .../recompiler/frontend/translate/Memory.cpp | 5 ++ src/graphics/shader/recompiler/ir/ShaderIR.h | 1 + tests/ShaderRecompilerComputeTests.cpp | 81 +++++++++++++++++++ 11 files changed, 124 insertions(+), 18 deletions(-) diff --git a/src/graphics/shader/recompiler/backend/spirv/SpirvEmitter.cpp b/src/graphics/shader/recompiler/backend/spirv/SpirvEmitter.cpp index 2593a9c78..56e5c809d 100644 --- a/src/graphics/shader/recompiler/backend/spirv/SpirvEmitter.cpp +++ b/src/graphics/shader/recompiler/backend/spirv/SpirvEmitter.cpp @@ -227,6 +227,7 @@ Emitter::SpirvRequirements Emitter::AnalyzeProgramRequirements(const IR::Program requirements.subgroup_local_invocation_id = true; } if (memory.kind == IR::ResourceKind::Buffer) { + requirements.coherent_buffers |= memory.coherent; if (memory.resource >= program.info.buffers.size()) { Fail(program, "buffer operation has invalid resource metadata"); } diff --git a/src/graphics/shader/recompiler/backend/spirv/spirvEmitterInternal.h b/src/graphics/shader/recompiler/backend/spirv/spirvEmitterInternal.h index 3d467722d..f1000a0f3 100644 --- a/src/graphics/shader/recompiler/backend/spirv/spirvEmitterInternal.h +++ b/src/graphics/shader/recompiler/backend/spirv/spirvEmitterInternal.h @@ -67,6 +67,7 @@ struct SpirvRequirements { bool function_scratch = false; bool pixel_valid_mask = false; bool buffer_int64_atomics = false; + bool coherent_buffers = false; }; SpirvRequirements AnalyzeProgramRequirements(const IR::Program& program); @@ -371,12 +372,13 @@ void EmitMemoryOffsets(EmitterState& state); uint32_t LdsDwordCount(const EmitterState& state); struct MemoryResourceAccess { - IR::ResourceKind kind = IR::ResourceKind::None; - uint32_t object_pointer = 0; - uint32_t length = 0; - uint32_t index_offset = 0; - uint32_t byte_offset = 0; - bool add_index_offset = false; + IR::ResourceKind kind = IR::ResourceKind::None; + uint32_t object_pointer = 0; + uint32_t length = 0; + uint32_t index_offset = 0; + uint32_t byte_offset = 0; + bool add_index_offset = false; + spv::MemoryAccessMask memory_access = spv::MemoryAccessMaskNone; }; MemoryResourceAccess PrepareMemoryResourceAccess(EmitterState& state, const IR::MemoryInfo& mem); diff --git a/src/graphics/shader/recompiler/backend/spirv/spirvEmitterMemory.cpp b/src/graphics/shader/recompiler/backend/spirv/spirvEmitterMemory.cpp index e6ad801c1..fec3e3be3 100644 --- a/src/graphics/shader/recompiler/backend/spirv/spirvEmitterMemory.cpp +++ b/src/graphics/shader/recompiler/backend/spirv/spirvEmitterMemory.cpp @@ -313,7 +313,8 @@ uint32_t LoadWordInBounds(ValueEmitContext& ctx, const MemoryResourceAccess& res uint32_t index) { const auto value = ctx.state.builder.AllocateId(); const auto pointer = EmitMemoryElementPointer(ctx.state, resource, index); - ctx.state.builder.AddFunction(spv::OpLoad, TypeU32(ctx.state), value, pointer); + ctx.state.builder.AddFunction(spv::OpLoad, TypeU32(ctx.state), value, pointer, + resource.memory_access); return value; } @@ -498,21 +499,21 @@ void StoreSubword(ValueEmitContext& ctx, const IR::Inst& inst, IR::MemoryInfo me }); } +void StoreWordInBounds(ValueEmitContext& ctx, const MemoryResourceAccess& resource, uint32_t index, + uint32_t data) { + ctx.state.builder.AddFunction(spv::OpStore, + EmitMemoryElementPointer(ctx.state, resource, index), data, + resource.memory_access); +} + void StoreWordPrepared(ValueEmitContext& ctx, const IR::Inst& inst, const IR::MemoryInfo& mem, const MemoryResourceAccess& resource, uint32_t data) { const auto index = EmitMemoryElementIndex(ctx.state, resource, DwordIndex(ctx, inst, mem)); EmitIfCondition(ctx.state, EmitMemoryElementInBounds(ctx.state, resource, index), [&]() { - ctx.state.builder.AddFunction(spv::OpStore, - EmitMemoryElementPointer(ctx.state, resource, index), data); + StoreWordInBounds(ctx, resource, index, data); }); } -void StoreWordInBounds(ValueEmitContext& ctx, const MemoryResourceAccess& resource, uint32_t index, - uint32_t data) { - ctx.state.builder.AddFunction(spv::OpStore, - EmitMemoryElementPointer(ctx.state, resource, index), data); -} - void StoreWord(ValueEmitContext& ctx, const IR::Inst& inst, IR::MemoryInfo mem) { EmitIfCondition(ctx.state, ctx.Arg(inst, inst.NumArgs() - 1), [&]() { const auto resource = PrepareMemoryResourceAccess(ctx.state, mem); diff --git a/src/graphics/shader/recompiler/backend/spirv/spirvEmitterMemoryHelpers.cpp b/src/graphics/shader/recompiler/backend/spirv/spirvEmitterMemoryHelpers.cpp index b7c8b3951..2aee17430 100644 --- a/src/graphics/shader/recompiler/backend/spirv/spirvEmitterMemoryHelpers.cpp +++ b/src/graphics/shader/recompiler/backend/spirv/spirvEmitterMemoryHelpers.cpp @@ -86,7 +86,9 @@ MemoryResourceAccess PrepareStorageBufferResourceAccess(EmitterState& state, } const auto array_index = ResourceForDescriptor(state, IR::DescriptorBindingKind::Buffers, mem.resource); - MemoryResourceAccess access {.kind = mem.kind}; + MemoryResourceAccess access { + .kind = mem.kind, + .memory_access = mem.coherent ? spv::MemoryAccessVolatileMask : spv::MemoryAccessMaskNone}; access.object_pointer = state.builder.AllocateId(); state.builder.AddFunction(spv::OpAccessChain, pointer_type, access.object_pointer, variable, ConstantU32(state, array_index)); diff --git a/src/graphics/shader/recompiler/backend/spirv/spirvEmitterModule.cpp b/src/graphics/shader/recompiler/backend/spirv/spirvEmitterModule.cpp index 25d0650f3..eadb9e9bb 100644 --- a/src/graphics/shader/recompiler/backend/spirv/spirvEmitterModule.cpp +++ b/src/graphics/shader/recompiler/backend/spirv/spirvEmitterModule.cpp @@ -201,6 +201,16 @@ void DefineDescriptors(EmitterState& state) { state.builder.AddAnnotation(spv::OpDecorate, state.storage_buffer_u64_variable, spv::DecorationAliased); } + if (state.requirements.coherent_buffers) { + // RDNA2 stores publish to L2 even without GLC; every alias of the buffer + // must participate in visibility for cache-bypassing polling loads. + state.builder.AddAnnotation(spv::OpDecorate, state.storage_buffer_variable, + spv::DecorationCoherent); + if (state.storage_buffer_u64_variable != 0) { + state.builder.AddAnnotation(spv::OpDecorate, state.storage_buffer_u64_variable, + spv::DecorationCoherent); + } + } break; case IR::DescriptorBindingKind::BdaPagetable: state.bda_pagetable_variable = Define(StorageBufferU64Type(state), "bda_pagetable"); diff --git a/src/graphics/shader/recompiler/frontend/decode/MemoryOps.cpp b/src/graphics/shader/recompiler/frontend/decode/MemoryOps.cpp index 68a7d4a22..90102824d 100644 --- a/src/graphics/shader/recompiler/frontend/decode/MemoryOps.cpp +++ b/src/graphics/shader/recompiler/frontend/decode/MemoryOps.cpp @@ -273,6 +273,7 @@ void DecodeMubuf(uint32_t pc, std::span code, uint32_t word_inde inst.idxen = ((word0 >> 13u) & 1u) != 0; inst.offen = ((word0 >> 12u) & 1u) != 0; inst.glc = ((word0 >> 14u) & 1u) != 0; + inst.dlc = ((word0 >> 15u) & 1u) != 0; inst.slc = ((word1 >> 22u) & 1u) != 0; inst.family = Family::MUBUF; inst.opcode_id = opcode; @@ -308,6 +309,7 @@ void DecodeMtbuf(uint32_t pc, std::span code, uint32_t word_inde inst.idxen = ((word0 >> 13u) & 1u) != 0; inst.offen = ((word0 >> 12u) & 1u) != 0; inst.glc = ((word0 >> 14u) & 1u) != 0; + inst.dlc = ((word0 >> 15u) & 1u) != 0; inst.slc = ((word1 >> 22u) & 1u) != 0; inst.family = Family::MTBUF; inst.opcode_id = opcode; diff --git a/src/graphics/shader/recompiler/frontend/decode/ShaderDecoder.cpp b/src/graphics/shader/recompiler/frontend/decode/ShaderDecoder.cpp index 40220e48f..7ad29e93d 100644 --- a/src/graphics/shader/recompiler/frontend/decode/ShaderDecoder.cpp +++ b/src/graphics/shader/recompiler/frontend/decode/ShaderDecoder.cpp @@ -73,11 +73,11 @@ std::string FormatMemory(const Instruction& inst) { text += OperandToString(*sources[i]); } text += fmt::format(" ; offset={} offset2={} dwords={} bits={} dfmt={} nfmt={} signed={} " - "typed={} formatted={} segment={} glc={} slc={} idxen={} offen={}", + "typed={} formatted={} segment={} glc={} dlc={} slc={} idxen={} offen={}", inst.offset, inst.secondary_offset, inst.data_dwords, inst.data_bits, inst.data_format, inst.number_format, inst.data_signed ? 1u : 0u, inst.typed ? 1u : 0u, inst.formatted ? 1u : 0u, inst.memory_segment, - inst.glc ? 1u : 0u, inst.slc ? 1u : 0u, inst.idxen ? 1u : 0u, + inst.glc ? 1u : 0u, inst.dlc ? 1u : 0u, inst.slc ? 1u : 0u, inst.idxen ? 1u : 0u, inst.offen ? 1u : 0u); return text; } diff --git a/src/graphics/shader/recompiler/frontend/decode/ShaderDecoder.h b/src/graphics/shader/recompiler/frontend/decode/ShaderDecoder.h index 1a67a5d67..f7ddaa911 100644 --- a/src/graphics/shader/recompiler/frontend/decode/ShaderDecoder.h +++ b/src/graphics/shader/recompiler/frontend/decode/ShaderDecoder.h @@ -712,6 +712,7 @@ struct Instruction { bool formatted = false; bool gds = false; bool glc = false; + bool dlc = false; bool slc = false; bool idxen = false; bool offen = false; diff --git a/src/graphics/shader/recompiler/frontend/translate/Memory.cpp b/src/graphics/shader/recompiler/frontend/translate/Memory.cpp index bb6941571..d7ba2fa4e 100644 --- a/src/graphics/shader/recompiler/frontend/translate/Memory.cpp +++ b/src/graphics/shader/recompiler/frontend/translate/Memory.cpp @@ -125,6 +125,11 @@ IR::MemoryInfo MemoryInfoFromDecoded(const Decoder::Instruction& decoded) { memory.image_r128 = decoded.image_r128; memory.idxen = decoded.idxen; memory.offen = decoded.offen; + // Vector loads use GLC/DLC to bypass L0/GL1; atomics use GLC only to return data. + const bool buffer_atomic = decoded.opcode >= Decoder::Opcode::BUFFER_ATOMIC_SWAP && + decoded.opcode <= Decoder::Opcode::BUFFER_ATOMIC_FMAX; + memory.coherent = memory.kind == ResourceKind::Buffer && !buffer_atomic && + (decoded.glc || decoded.dlc); memory.resource = ResourceIndexFromOperand(decoded.src1); memory.sampler = ResourceIndexFromOperand(decoded.src2); if (memory.kind == ResourceKind::ScalarBuffer) { diff --git a/src/graphics/shader/recompiler/ir/ShaderIR.h b/src/graphics/shader/recompiler/ir/ShaderIR.h index d52fa99c2..2359479f4 100644 --- a/src/graphics/shader/recompiler/ir/ShaderIR.h +++ b/src/graphics/shader/recompiler/ir/ShaderIR.h @@ -67,6 +67,7 @@ struct MemoryInfo { bool image_r128 = false; bool idxen = false; bool offen = false; + bool coherent = false; bool planning_only = false; [[nodiscard]] bool SupportsIndirectBufferLoad(ValueOpcode opcode) const { diff --git a/tests/ShaderRecompilerComputeTests.cpp b/tests/ShaderRecompilerComputeTests.cpp index 6022979a1..90f399ea5 100644 --- a/tests/ShaderRecompilerComputeTests.cpp +++ b/tests/ShaderRecompilerComputeTests.cpp @@ -25991,6 +25991,76 @@ TestCase Wave64AppendConsumeHighHalf() { return test; } +TestCase BufferWorkgroupPublication(u32 wave_size, bool dlc_only = false) { + using O = ShaderOpcode; + constexpr u32 groups = 31; + TestCase test; + test.name = dlc_only ? "BufferWorkgroupPublicationDlc" + : wave_size == 64 ? "BufferWorkgroupPublicationWave64" + : "BufferWorkgroupPublicationWave32"; + auto &code = test.code; + // Allocate work in execution order so every polled predecessor has started. + // Only one lane per workgroup owns a ticket and publishes its prefix total. + code.push_back(EncodeSop1(0x04, 126, InlineU32(1))); + AppendVMovU32(&code, 1, 1); + AppendVMovU32(&code, 20, 0); + AppendBufferStoreOpcode(&code, 0x32, 1, 20, true); + code.push_back(EncodeVop1(0x02, 20, Vgpr(1))); + code.push_back(EncodeVop2(0x1a, 20, InlineU32(2), 1)); + AppendVMovU32(&code, 2, 0); + code.push_back(EncodeSMovB32(21, InlineU32(0))); + AppendSMovLiteral(&code, 22, 1000000); + code.push_back(EncodeSopc(0x06, 20, InlineU32(0))); + const auto first_ticket = code.size(); + code.push_back(0); + + const auto poll = code.size(); + code.push_back(EncodeMubuf0(0x0c, 0, false, true, !dlc_only) | (1u << 15u)); + code.push_back(EncodeMubuf1(2, 12, 20)); + code.push_back(EncodeSopp(0x0c, 0)); // S_WAITCNT vmcnt(0). + code.push_back(EncodeVop1(0x02, 23, Vgpr(2))); + code.push_back(EncodeSopc(0x07, 23, InlineU32(0))); + const auto ready = code.size(); + code.push_back(0); + code.push_back(EncodeSop2(0x00, 21, 21, InlineU32(1))); + code.push_back(EncodeSopc(0x0a, 21, 22)); + code.push_back(EncodeSopp(0x05, static_cast(poll - code.size() - 1))); + // A broken poll reports an incorrect result instead of hanging the test GPU. + AppendVMovLiteral(&code, 2, 0x80000000u); + const auto publish = code.size(); + code[first_ticket] = EncodeSopp(0x05, publish - first_ticket - 1); + code[ready] = EncodeSopp(0x05, publish - ready - 1); + code.push_back(EncodeVop2(0x25, 2, Vgpr(1), 2)); + code.push_back(EncodeVop2(0x25, 2, InlineU32(1), 2)); + code.push_back(EncodeVop2(0x25, 20, InlineU32(4), 20)); + // RDNA2 stores publish to L2 even without GLC/DLC on the producer. + AppendBufferStoreDword(&code, 2, 20); + AppendEnd(&code); + + test.initial.assign(groups + 1, 0); + test.expected = {groups}; + for (u32 ticket = 0; ticket < groups; ++ticket) { + test.expected.push_back((ticket + 1) * (ticket + 2) / 2); + } + test.opcodes = {O::S_MOV_B64, O::S_MOV_B32, O::V_MOV_B32, + O::BUFFER_ATOMIC_ADD, O::V_READFIRSTLANE_B32, + O::V_LSHLREV_B32, O::S_CMP_EQ_U32, O::S_CMP_LG_U32, + O::S_CMP_LT_U32, O::S_CBRANCH_SCC1, O::S_ADD_U32, + O::S_WAITCNT, O::BUFFER_LOAD_DWORD, O::V_ADD_NC_U32, + O::BUFFER_STORE_DWORD, O::S_ENDPGM}; + test.required_spirv = {"Coherent", "Volatile", "OpAtomicIAdd", "OpLoopMerge"}; + test.compute_info.threads_num[0] = wave_size; + test.compute_info.threads_num[1] = 1; + test.compute_info.threads_num[2] = 1; + test.compute_info.thread_ids_num = 1; + test.compute_info.wave_size = wave_size; + test.has_compute_info = true; + test.dispatch_x = groups; + // DLC alone does not bypass RDNA2's L0, so only check its translated policy. + test.compile_only = dlc_only; + return test; +} + TestCase BufferAtomicVariants() { using O = ShaderOpcode; @@ -28351,6 +28421,9 @@ std::vector MakeCases() { auto AddCase = [&cases](TestCase (*factory)()) { cases.push_back(factory()); }; + cases.push_back(BufferWorkgroupPublication(32)); + cases.push_back(BufferWorkgroupPublication(64)); + cases.push_back(BufferWorkgroupPublication(32, true)); AddCase(IntegerAddSubMul); AddCase(BitwiseOps); @@ -33479,6 +33552,14 @@ int main(int argc, char **argv) { RunCase(&vulkan, BufferOffsetsUsePackedLaneAndStorageFallback()); return 0; } + if (argc == 2 && std::strcmp(argv[1], "--buffer-publication-only") == 0) { + VulkanHarness vulkan; + RunCase(&vulkan, BufferWorkgroupPublication(32)); + RunCase(&vulkan, BufferWorkgroupPublication(64)); + RunCase(&vulkan, BufferWorkgroupPublication(32, true)); + RunCase(&vulkan, BufferAtomicVariants()); + return 0; + } if (argc == 2 && std::strcmp(argv[1], "--buffer-cmpswap-only") == 0) { VulkanHarness vulkan; RunCase(&vulkan, BufferAtomicCmpSwapExactRaw());