From b3c8f91a8462244672cd017ff50781ce972d7c48 Mon Sep 17 00:00:00 2001 From: nmzik Date: Thu, 3 Sep 2026 06:01:27 +0200 Subject: [PATCH] graphics: implement RDNA2 buffer atomic swap x2 --- .../recompiler/backend/spirv/SpirvEmitter.cpp | 3 +- .../backend/spirv/spirvEmitterMemory.cpp | 16 +-- .../recompiler/frontend/decode/MemoryOps.cpp | 1 + .../frontend/decode/ShaderDecoder.cpp | 1 + .../frontend/decode/ShaderDecoder.h | 1 + .../recompiler/frontend/translate/Memory.cpp | 2 + .../recompiler/ir/opcodes/ValueOpcodes.cpp | 2 + .../recompiler/ir/opcodes/ValueOpcodes.inc | 1 + tests/shaderCfgTests.cpp | 117 ++++++++++-------- 9 files changed, 83 insertions(+), 61 deletions(-) diff --git a/src/graphics/shader/recompiler/backend/spirv/SpirvEmitter.cpp b/src/graphics/shader/recompiler/backend/spirv/SpirvEmitter.cpp index e3f29ae0f..d049e3637 100644 --- a/src/graphics/shader/recompiler/backend/spirv/SpirvEmitter.cpp +++ b/src/graphics/shader/recompiler/backend/spirv/SpirvEmitter.cpp @@ -189,7 +189,8 @@ void AnalyzeProgramRequirements(IR::Program& program) { const auto MarkBallot = [&] { requirements.subgroup_ballot = true; }; for (const auto* block: program.blocks) { for (const auto& inst: *block) { - if (inst.GetOpcode() == IR::ValueOpcode::BufferAtomicOr64) { + if (IR::BufferAccessOf(inst.GetOpcode()) == IR::BufferAccess::Atomic && + inst.GetType() == IR::Type::U64) { requirements.buffer_int64_atomics = true; } const auto address_access = IR::AddressOpcodeInfoOf(inst.GetOpcode()).access; diff --git a/src/graphics/shader/recompiler/backend/spirv/spirvEmitterMemory.cpp b/src/graphics/shader/recompiler/backend/spirv/spirvEmitterMemory.cpp index 9c6a77cea..a152bb14a 100644 --- a/src/graphics/shader/recompiler/backend/spirv/spirvEmitterMemory.cpp +++ b/src/graphics/shader/recompiler/backend/spirv/spirvEmitterMemory.cpp @@ -617,6 +617,7 @@ void FormattedStore(ValueEmitContext& ctx, const IR::Inst& inst, const IR::Memor uint32_t AtomicOpcode(IR::ValueOpcode opcode) { switch (opcode) { case IR::ValueOpcode::BufferAtomicSwap32: + case IR::ValueOpcode::BufferAtomicSwap64: case IR::ValueOpcode::SharedAtomicSwap32: return OpAtomicExchange; case IR::ValueOpcode::BufferAtomicIAdd32: case IR::ValueOpcode::SharedAtomicIAdd32: return OpAtomicIAdd; @@ -633,6 +634,7 @@ uint32_t AtomicOpcode(IR::ValueOpcode opcode) { case IR::ValueOpcode::BufferAtomicAnd32: case IR::ValueOpcode::SharedAtomicAnd32: return OpAtomicAnd; case IR::ValueOpcode::BufferAtomicOr32: + case IR::ValueOpcode::BufferAtomicOr64: case IR::ValueOpcode::SharedAtomicOr32: return OpAtomicOr; case IR::ValueOpcode::BufferAtomicXor32: case IR::ValueOpcode::SharedAtomicXor32: return OpAtomicXor; @@ -672,8 +674,8 @@ uint32_t EmitAtomic(ValueEmitContext& ctx, const IR::Inst& inst, const IR::Memor return return_value ? result : ConstantU32(ctx.state, 0); } -uint32_t EmitBufferAtomicOr64(ValueEmitContext& ctx, const IR::Inst& inst, - const IR::MemoryInfo& mem) { +uint32_t EmitBufferAtomic64(ValueEmitContext& ctx, const IR::Inst& inst, + const IR::MemoryInfo& mem) { auto& state = ctx.state; return EmitValueOrDefaultIfCondition( state, ctx.Arg(inst, inst.NumArgs() - 1), TypeU64(state), ConstantU64(state, 0), [&]() { @@ -690,7 +692,7 @@ uint32_t EmitBufferAtomicOr64(ValueEmitContext& ctx, const IR::Inst& inst, ctx.Arg(inst, inst.NumArgs() - 2)); const auto old = state.builder.AllocateId(); state.builder.AddFunction( - {OpAtomicOr, TypeScalarU64(state), old, + {AtomicOpcode(inst.GetOpcode()), TypeScalarU64(state), old, EmitStorageBufferElementPointer( state, resource, index, TypeStorageBufferU64ElementPointer(state)), ConstantU32(state, ScopeDevice), @@ -1209,12 +1211,12 @@ bool EmitValueMemory(ValueEmitContext& ctx, const IR::Inst& inst) { return true; } const auto atomic_opcode = AtomicOpcode(op); - if (atomic_opcode != 0) { - ctx.Define(inst, EmitAtomic(ctx, inst, ctx.Memory(inst), true)); + if (atomic_opcode != 0 && inst.GetType() == IR::Type::U64) { + ctx.Define(inst, EmitBufferAtomic64(ctx, inst, ctx.Memory(inst))); return true; } - if (op == IR::ValueOpcode::BufferAtomicOr64) { - ctx.Define(inst, EmitBufferAtomicOr64(ctx, inst, ctx.Memory(inst))); + if (atomic_opcode != 0) { + ctx.Define(inst, EmitAtomic(ctx, inst, ctx.Memory(inst), true)); return true; } if (op == IR::ValueOpcode::BufferAtomicFMin32 || op == IR::ValueOpcode::BufferAtomicFMax32) { diff --git a/src/graphics/shader/recompiler/frontend/decode/MemoryOps.cpp b/src/graphics/shader/recompiler/frontend/decode/MemoryOps.cpp index 667bdb7aa..a1a1fabf0 100644 --- a/src/graphics/shader/recompiler/frontend/decode/MemoryOps.cpp +++ b/src/graphics/shader/recompiler/frontend/decode/MemoryOps.cpp @@ -58,6 +58,7 @@ constexpr MemoryOpcodeInfo MUBUF_OPCODE_LIST[] = { {0x3bu, Opcode::BUFFER_ATOMIC_XOR, 1, 32}, {0x3fu, Opcode::BUFFER_ATOMIC_FMIN, 1, 32}, {0x40u, Opcode::BUFFER_ATOMIC_FMAX, 1, 32}, + {0x50u, Opcode::BUFFER_ATOMIC_SWAP_X2, 2, 32}, {0x5au, Opcode::BUFFER_ATOMIC_OR_X2, 2, 32}, }; diff --git a/src/graphics/shader/recompiler/frontend/decode/ShaderDecoder.cpp b/src/graphics/shader/recompiler/frontend/decode/ShaderDecoder.cpp index 53caff218..0f80123be 100644 --- a/src/graphics/shader/recompiler/frontend/decode/ShaderDecoder.cpp +++ b/src/graphics/shader/recompiler/frontend/decode/ShaderDecoder.cpp @@ -592,6 +592,7 @@ std::string InstructionToString(const Instruction& inst) { case Opcode::TBUFFER_STORE_FORMAT_XYZ: case Opcode::TBUFFER_STORE_FORMAT_XYZW: case Opcode::BUFFER_ATOMIC_SWAP: + case Opcode::BUFFER_ATOMIC_SWAP_X2: case Opcode::BUFFER_ATOMIC_ADD: case Opcode::BUFFER_ATOMIC_SUB: case Opcode::BUFFER_ATOMIC_SMIN: diff --git a/src/graphics/shader/recompiler/frontend/decode/ShaderDecoder.h b/src/graphics/shader/recompiler/frontend/decode/ShaderDecoder.h index 71c697eca..d93a1edbd 100644 --- a/src/graphics/shader/recompiler/frontend/decode/ShaderDecoder.h +++ b/src/graphics/shader/recompiler/frontend/decode/ShaderDecoder.h @@ -455,6 +455,7 @@ enum class Opcode { TBUFFER_STORE_FORMAT_XYZ, TBUFFER_STORE_FORMAT_XYZW, BUFFER_ATOMIC_SWAP, + BUFFER_ATOMIC_SWAP_X2, BUFFER_ATOMIC_ADD, BUFFER_ATOMIC_SUB, BUFFER_ATOMIC_SMIN, diff --git a/src/graphics/shader/recompiler/frontend/translate/Memory.cpp b/src/graphics/shader/recompiler/frontend/translate/Memory.cpp index 8bddcd424..9dae13c7f 100644 --- a/src/graphics/shader/recompiler/frontend/translate/Memory.cpp +++ b/src/graphics/shader/recompiler/frontend/translate/Memory.cpp @@ -922,6 +922,8 @@ bool Translator::EmitMemory(const Decoder::Instruction& inst) { case Decoder::Opcode::BUFFER_ATOMIC_SWAP: return BUFFER_ATOMIC(inst, IR::ValueOpcode::BufferAtomicSwap32); + case Decoder::Opcode::BUFFER_ATOMIC_SWAP_X2: + return BUFFER_ATOMIC(inst, IR::ValueOpcode::BufferAtomicSwap64); case Decoder::Opcode::BUFFER_ATOMIC_ADD: return BUFFER_ATOMIC(inst, IR::ValueOpcode::BufferAtomicIAdd32); case Decoder::Opcode::BUFFER_ATOMIC_SUB: diff --git a/src/graphics/shader/recompiler/ir/opcodes/ValueOpcodes.cpp b/src/graphics/shader/recompiler/ir/opcodes/ValueOpcodes.cpp index 9f1597a47..c70a13ae2 100644 --- a/src/graphics/shader/recompiler/ir/opcodes/ValueOpcodes.cpp +++ b/src/graphics/shader/recompiler/ir/opcodes/ValueOpcodes.cpp @@ -111,6 +111,7 @@ BufferAccess BufferAccessOf(ValueOpcode opcode) { case ValueOpcode::StoreBufferU32x3: case ValueOpcode::StoreBufferU32x4: return BufferAccess::Write; case ValueOpcode::BufferAtomicSwap32: + case ValueOpcode::BufferAtomicSwap64: case ValueOpcode::BufferAtomicIAdd32: case ValueOpcode::BufferAtomicISub32: case ValueOpcode::BufferAtomicSMin32: @@ -129,6 +130,7 @@ BufferAccess BufferAccessOf(ValueOpcode opcode) { uint32_t BufferComponentCount(ValueOpcode opcode) { switch (opcode) { + case ValueOpcode::BufferAtomicSwap64: case ValueOpcode::BufferAtomicOr64: case ValueOpcode::LoadBufferU32x2: case ValueOpcode::StoreBufferU32x2: return 2u; diff --git a/src/graphics/shader/recompiler/ir/opcodes/ValueOpcodes.inc b/src/graphics/shader/recompiler/ir/opcodes/ValueOpcodes.inc index 256c6d4de..efb21177b 100644 --- a/src/graphics/shader/recompiler/ir/opcodes/ValueOpcodes.inc +++ b/src/graphics/shader/recompiler/ir/opcodes/ValueOpcodes.inc @@ -222,6 +222,7 @@ VALUE_OPCODE(StoreBufferU32x2, Void, BufferResource, U32, U32, U32, U32x2, U1) VALUE_OPCODE(StoreBufferU32x3, Void, BufferResource, U32, U32, U32, U32x3, U1) VALUE_OPCODE(StoreBufferU32x4, Void, BufferResource, U32, U32, U32, U32x4, U1) VALUE_OPCODE(BufferAtomicSwap32, U32, BufferResource, U32, U32, U32, U32, U1) +VALUE_OPCODE(BufferAtomicSwap64, U64, BufferResource, U32, U32, U32, U64, U1) VALUE_OPCODE(BufferAtomicIAdd32, U32, BufferResource, U32, U32, U32, U32, U1) VALUE_OPCODE(BufferAtomicISub32, U32, BufferResource, U32, U32, U32, U32, U1) VALUE_OPCODE(BufferAtomicSMin32, U32, BufferResource, U32, U32, U32, U32, U1) diff --git a/tests/shaderCfgTests.cpp b/tests/shaderCfgTests.cpp index 378969b98..499c2dc69 100644 --- a/tests/shaderCfgTests.cpp +++ b/tests/shaderCfgTests.cpp @@ -8593,66 +8593,77 @@ void TestNewShaderRecompilerBufferAtomicsGuardedByBounds() { "buffer atomic memory barrier was emitted before atomic"); } -void TestCapturedBufferAtomicOrX2() { - const uint32_t shader[] = { - 0xe1680018u, 0x80000000u, // buffer_atomic_or_x2 v[0:1], s[0:3], 0 offset:24 - EncodeSopp(0x01), +void TestCapturedBufferAtomicsX2() { + using ShaderRecompiler::IR::ValueOpcode; + struct Case { + std::array words; + uint32_t data_vgpr; + uint32_t spirv_opcode; + const char *decoded_name; + const char *ir_name; + const char *spirv_name; + ValueOpcode ir_opcode; + }; + const Case cases[] = { + {{0xe1680018u, 0x80000000u}, 0u, 241u, "BUFFER_ATOMIC_OR_X2", + "BufferAtomicOr64", "OpAtomicOr", ValueOpcode::BufferAtomicOr64}, + {{0xe1402000u, 0x80000913u}, 9u, 229u, "BUFFER_ATOMIC_SWAP_X2", + "BufferAtomicSwap64", "OpAtomicExchange", ValueOpcode::BufferAtomicSwap64}, }; - auto options = MakeCompileOptions(ShaderType::Compute); options.dump_ir = true; + for (const auto &test : cases) { + const std::array shader = {test.words[0], test.words[1], EncodeSopp(0x01)}; + auto result = RecompileForTest(shader, options); + Check(Common::ContainsStr(result.decoded_dump, test.decoded_name), + "captured 64-bit MUBUF atomic decoded incorrectly"); + Check(Common::ContainsStr(result.ir_dump, test.ir_name), + "64-bit MUBUF atomic did not lower to its native IR opcode"); + Check(!Common::ContainsStr(result.ir_dump, "SetVectorRegister"), + "GLC=0 64-bit MUBUF atomic unexpectedly returned the old value"); + CheckSpirvBinaryValidates(result.spirv); - auto result = RecompileForTest(shader, options); - Check(Common::ContainsStr(result.decoded_dump, "BUFFER_ATOMIC_OR_X2"), - "captured MUBUF instruction did not decode as buffer_atomic_or_x2"); - Check(Common::ContainsStr(result.ir_dump, "BufferAtomicOr64"), - "buffer_atomic_or_x2 did not lower to a 64-bit buffer atomic"); - Check(!Common::ContainsStr(result.ir_dump, "SetVectorRegister"), - "GLC=0 buffer_atomic_or_x2 unexpectedly wrote an old value to v0:v1"); - CheckSpirvBinaryValidates(result.spirv); + const auto source = DisassembleSpirvBinary(result.spirv); + Check(Common::ContainsStr(source, "OpCapability Int64Atomics"), + "64-bit buffer atomic SPIR-V lacks Int64Atomics capability"); + Check(Common::ContainsStr(source, "ArrayStride 8"), + "64-bit buffer atomic storage view does not use eight-byte elements"); + Check(CountSourceOccurrences(source, "Aliased") == 2u, + "both storage-buffer views must declare that they alias"); + const auto array_length = + Common::FindIndex(source, std::string("OpArrayLength"), 0); + const auto bounds_branch = Common::FindIndex( + source, std::string("OpBranchConditional"), array_length); + const auto atomic = + Common::FindIndex(source, std::string(test.spirv_name), 0); + const auto memory_barrier = + Common::FindIndex(source, std::string("OpMemoryBarrier"), atomic); + Check(array_length != Common::FIND_INVALID_INDEX && + bounds_branch != Common::FIND_INVALID_INDEX && + atomic != Common::FIND_INVALID_INDEX && bounds_branch < atomic, + "64-bit buffer atomic was not guarded by storage-buffer bounds"); + Check(SpirvInstructionOpcodeCount(result.spirv, test.spirv_opcode) == 1u, + "64-bit buffer atomic did not lower to one native SPIR-V atomic"); + Check(memory_barrier != Common::FIND_INVALID_INDEX && atomic < memory_barrier, + "64-bit buffer atomic lacks a following device-memory barrier"); - const auto source = DisassembleSpirvBinary(result.spirv); - Check(Common::ContainsStr(source, "OpCapability Int64Atomics"), - "64-bit buffer atomic SPIR-V lacks Int64Atomics capability"); - Check(Common::ContainsStr(source, "ArrayStride 8"), - "64-bit buffer atomic storage view does not use eight-byte elements"); - Check(CountSourceOccurrences(source, "Aliased") == 2u, - "both storage-buffer views must declare that they alias"); - const auto array_length = - Common::FindIndex(source, std::string("OpArrayLength"), 0); - const auto bounds_branch = Common::FindIndex( - source, std::string("OpBranchConditional"), array_length); - const auto atomic = Common::FindIndex(source, std::string("OpAtomicOr"), 0); - const auto memory_barrier = - Common::FindIndex(source, std::string("OpMemoryBarrier"), atomic); - Check(array_length != Common::FIND_INVALID_INDEX && - bounds_branch != Common::FIND_INVALID_INDEX && - atomic != Common::FIND_INVALID_INDEX && bounds_branch < atomic, - "64-bit buffer atomic was not guarded by storage-buffer bounds"); - Check(SpirvInstructionOpcodeCount(result.spirv, 241u) == 1u, - "buffer_atomic_or_x2 must lower to exactly one OpAtomicOr"); - Check(memory_barrier != Common::FIND_INVALID_INDEX && atomic < memory_barrier, - "64-bit buffer atomic lacks a following device-memory barrier"); - - const std::array glc_shader = { - shader[0] | (1u << 14u), shader[1], - EncodeMubuf0(0x1d, 32, false), - EncodeMubuf1(0, 0, 0), // preserve returned v0:v1 through a dwordx2 store - shader[2], - }; - auto glc_result = RecompileForTest(glc_shader, options); - const ShaderRecompiler::IR::Inst *glc_atomic = nullptr; - for (const auto *block : glc_result.program.blocks) { - for (const auto &inst : *block) { - if (inst.GetOpcode() == - ShaderRecompiler::IR::ValueOpcode::BufferAtomicOr64) { - glc_atomic = &inst; + const std::array glc_shader = { + test.words[0] | (1u << 14u), test.words[1], + EncodeMubuf0(0x1d, 32, false), + EncodeMubuf1(test.data_vgpr, 0, 0), + EncodeSopp(0x01), + }; + auto glc_result = RecompileForTest(glc_shader, options); + const ShaderRecompiler::IR::Inst *glc_atomic = nullptr; + for (const auto *block : glc_result.program.blocks) { + for (const auto &inst : *block) { + if (inst.GetOpcode() == test.ir_opcode) glc_atomic = &inst; } } + Check(glc_atomic != nullptr && glc_atomic->UseCount() == 2u, + "GLC=1 64-bit MUBUF atomic did not return both dwords"); + CheckSpirvBinaryValidates(glc_result.spirv); } - Check(glc_atomic != nullptr && glc_atomic->UseCount() == 2u, - "GLC=1 buffer_atomic_or_x2 did not feed both returned dwords"); - CheckSpirvBinaryValidates(glc_result.spirv); } void TestNewShaderRecompilerBranchConditionForms() { @@ -11997,7 +12008,7 @@ int main() { TestComputeDispatchWaveSize(); TestNewShaderRecompilerBufferLoadsGuardedByExec(); TestNewShaderRecompilerBufferAtomicsGuardedByBounds(); - TestCapturedBufferAtomicOrX2(); + TestCapturedBufferAtomicsX2(); TestNewShaderRecompilerPixelImageSampleLodSelection(); TestNewShaderRecompilerBranchConditionForms(); TestNewShaderRecompilerSetpcBranch();