diff --git a/src/graphics/shader/recompiler/backend/spirv/spirvEmitterInstructions.h b/src/graphics/shader/recompiler/backend/spirv/spirvEmitterInstructions.h index ee7f753b8..377f32733 100644 --- a/src/graphics/shader/recompiler/backend/spirv/spirvEmitterInstructions.h +++ b/src/graphics/shader/recompiler/backend/spirv/spirvEmitterInstructions.h @@ -296,6 +296,7 @@ inline constexpr auto EmitBufferAtomicSMax32 = EmitAtomic32; inline constexpr auto EmitBufferAtomicUMax32 = EmitAtomic32; inline constexpr auto EmitBufferAtomicAnd32 = EmitAtomic32; inline constexpr auto EmitBufferAtomicAnd64 = EmitBufferAtomic64; +inline constexpr auto EmitBufferAtomicIAdd64 = EmitBufferAtomic64; inline constexpr auto EmitBufferAtomicOr32 = EmitAtomic32; inline constexpr auto EmitBufferAtomicOr64 = EmitBufferAtomic64; inline constexpr auto EmitBufferAtomicXor32 = EmitAtomic32; diff --git a/src/graphics/shader/recompiler/backend/spirv/spirvEmitterMemoryHelpers.cpp b/src/graphics/shader/recompiler/backend/spirv/spirvEmitterMemoryHelpers.cpp index 4b3a9d152..6a65c4472 100644 --- a/src/graphics/shader/recompiler/backend/spirv/spirvEmitterMemoryHelpers.cpp +++ b/src/graphics/shader/recompiler/backend/spirv/spirvEmitterMemoryHelpers.cpp @@ -516,6 +516,7 @@ spv::Op SpirvAtomicOpcode(IR::ValueOpcode opcode) { case IR::ValueOpcode::ImageAtomicIAdd64: case IR::ValueOpcode::BufferAtomicIAdd32: case IR::ValueOpcode::SharedAtomicIAdd64: + case IR::ValueOpcode::BufferAtomicIAdd64: case IR::ValueOpcode::SharedAtomicIAdd32: return spv::OpAtomicIAdd; case IR::ValueOpcode::BufferAtomicISub32: case IR::ValueOpcode::SharedAtomicISub32: return spv::OpAtomicISub; diff --git a/src/graphics/shader/recompiler/frontend/decode/MemoryOps.cpp b/src/graphics/shader/recompiler/frontend/decode/MemoryOps.cpp index cd99c2eb4..d37fed8cc 100644 --- a/src/graphics/shader/recompiler/frontend/decode/MemoryOps.cpp +++ b/src/graphics/shader/recompiler/frontend/decode/MemoryOps.cpp @@ -67,6 +67,7 @@ constexpr MemoryOpcodeInfo MUBUF_OPCODE_LIST[] = { {0x3fu, Opcode::BUFFER_ATOMIC_FMIN, 1, 32}, {0x40u, Opcode::BUFFER_ATOMIC_FMAX, 1, 32}, {0x50u, Opcode::BUFFER_ATOMIC_SWAP_X2, 2, 32}, + {0x52u, Opcode::BUFFER_ATOMIC_ADD_X2, 2, 32}, {0x59u, Opcode::BUFFER_ATOMIC_AND_X2, 2, 32}, {0x5au, Opcode::BUFFER_ATOMIC_OR_X2, 2, 32}, {0x80u, Opcode::BUFFER_LOAD_FORMAT_D16_X, 1, 16, false, false, true}, diff --git a/src/graphics/shader/recompiler/frontend/decode/ShaderDecoder.cpp b/src/graphics/shader/recompiler/frontend/decode/ShaderDecoder.cpp index ef953ba1a..877fe2ccd 100644 --- a/src/graphics/shader/recompiler/frontend/decode/ShaderDecoder.cpp +++ b/src/graphics/shader/recompiler/frontend/decode/ShaderDecoder.cpp @@ -670,6 +670,7 @@ std::string InstructionToString(const Instruction& inst) { case Opcode::BUFFER_ATOMIC_CMPSWAP: case Opcode::BUFFER_ATOMIC_SWAP_X2: case Opcode::BUFFER_ATOMIC_ADD: + case Opcode::BUFFER_ATOMIC_ADD_X2: case Opcode::BUFFER_ATOMIC_SUB: case Opcode::BUFFER_ATOMIC_SMIN: case Opcode::BUFFER_ATOMIC_UMIN: diff --git a/src/graphics/shader/recompiler/frontend/decode/ShaderDecoder.h b/src/graphics/shader/recompiler/frontend/decode/ShaderDecoder.h index 3cf4085ab..4f5e3c785 100644 --- a/src/graphics/shader/recompiler/frontend/decode/ShaderDecoder.h +++ b/src/graphics/shader/recompiler/frontend/decode/ShaderDecoder.h @@ -586,6 +586,7 @@ enum class Opcode { BUFFER_ATOMIC_CMPSWAP, BUFFER_ATOMIC_SWAP_X2, BUFFER_ATOMIC_ADD, + BUFFER_ATOMIC_ADD_X2, BUFFER_ATOMIC_SUB, BUFFER_ATOMIC_SMIN, BUFFER_ATOMIC_UMIN, diff --git a/src/graphics/shader/recompiler/frontend/translate/Memory.cpp b/src/graphics/shader/recompiler/frontend/translate/Memory.cpp index 8e08ce735..0b6d3ee27 100644 --- a/src/graphics/shader/recompiler/frontend/translate/Memory.cpp +++ b/src/graphics/shader/recompiler/frontend/translate/Memory.cpp @@ -1046,6 +1046,8 @@ void Translator::EmitMemory(const Decoder::Instruction& inst) { return BUFFER_ATOMIC(inst, IR::ValueOpcode::BufferAtomicFMin32); case Decoder::Opcode::BUFFER_ATOMIC_FMAX: return BUFFER_ATOMIC(inst, IR::ValueOpcode::BufferAtomicFMax32); + case Decoder::Opcode::BUFFER_ATOMIC_ADD_X2: + return BUFFER_ATOMIC(inst, IR::ValueOpcode::BufferAtomicIAdd64); case Decoder::Opcode::DS_ADD_U32: return DS_ATOMIC(inst, IR::ValueOpcode::SharedAtomicIAdd32, false); diff --git a/src/graphics/shader/recompiler/ir/opcodes/ValueOpcodes.cpp b/src/graphics/shader/recompiler/ir/opcodes/ValueOpcodes.cpp index a6e7560d2..320716ebc 100644 --- a/src/graphics/shader/recompiler/ir/opcodes/ValueOpcodes.cpp +++ b/src/graphics/shader/recompiler/ir/opcodes/ValueOpcodes.cpp @@ -118,6 +118,7 @@ BufferAccess BufferAccessOf(ValueOpcode opcode) { case ValueOpcode::BufferAtomicCmpSwap32: case ValueOpcode::BufferAtomicSwap64: case ValueOpcode::BufferAtomicIAdd32: + case ValueOpcode::BufferAtomicIAdd64: case ValueOpcode::BufferAtomicISub32: case ValueOpcode::BufferAtomicSMin32: case ValueOpcode::BufferAtomicUMin32: @@ -137,6 +138,7 @@ BufferAccess BufferAccessOf(ValueOpcode opcode) { uint32_t BufferComponentCount(ValueOpcode opcode) { switch (opcode) { case ValueOpcode::BufferAtomicSwap64: + case ValueOpcode::BufferAtomicIAdd64: case ValueOpcode::BufferAtomicAnd64: case ValueOpcode::BufferAtomicOr64: case ValueOpcode::LoadBufferU32x2: diff --git a/src/graphics/shader/recompiler/ir/opcodes/ValueOpcodes.inc b/src/graphics/shader/recompiler/ir/opcodes/ValueOpcodes.inc index 4d6772f5b..84aaebfe0 100644 --- a/src/graphics/shader/recompiler/ir/opcodes/ValueOpcodes.inc +++ b/src/graphics/shader/recompiler/ir/opcodes/ValueOpcodes.inc @@ -268,6 +268,7 @@ VALUE_OPCODE(BufferAtomicSwap32, U32, BufferResource, U32, U32, U32, U32, U1) VALUE_OPCODE(BufferAtomicCmpSwap32, U32, BufferResource, U32, U32, U32, U32, U32, U1) VALUE_OPCODE(BufferAtomicSwap64, U64, BufferResource, U32, U32, U32, U64, U1) VALUE_OPCODE(BufferAtomicIAdd32, U32, BufferResource, U32, U32, U32, U32, U1) +VALUE_OPCODE(BufferAtomicIAdd64, U64, BufferResource, U32, U32, U32, U64, U1) VALUE_OPCODE(BufferAtomicISub32, U32, BufferResource, U32, U32, U32, U32, U1) VALUE_OPCODE(BufferAtomicSMin32, U32, BufferResource, U32, U32, U32, U32, U1) VALUE_OPCODE(BufferAtomicUMin32, U32, BufferResource, U32, U32, U32, U32, U1) diff --git a/tests/ShaderRecompilerComputeTests.cpp b/tests/ShaderRecompilerComputeTests.cpp index db3a5ad01..720a09f16 100644 --- a/tests/ShaderRecompilerComputeTests.cpp +++ b/tests/ShaderRecompilerComputeTests.cpp @@ -20991,6 +20991,7 @@ CoverageClass ClassifyOpcode(ShaderOpcode opcode, case Opcode::BUFFER_ATOMIC_SMAX: case Opcode::BUFFER_ATOMIC_UMAX: case Opcode::BUFFER_ATOMIC_AND: + case Opcode::BUFFER_ATOMIC_ADD_X2: case Opcode::BUFFER_ATOMIC_AND_X2: case Opcode::BUFFER_ATOMIC_OR: case Opcode::BUFFER_ATOMIC_XOR: @@ -35712,6 +35713,40 @@ TestCase BufferAtomicAndX2GlcAndExec() { return test; } +TestCase BufferAtomicAddX2GlcAndExec() { + using O = ShaderOpcode; + TestCase test; + test.name = "BufferAtomicAddX2GlcAndExec"; + auto &code = test.code; + code.push_back(EncodeSop1(0x04, 12, 126)); + AppendVMovU32(&code, 20, 0); + const u32 values[][2] = {{0x0ff00ff0u, 0xf00ff00fu}, + {0xff00ff00u, 0x00ff00ffu}, + {0x76543210u, 0xfedcba98u}}; + for (u32 i = 0; i < std::size(values); ++i) { + AppendVMovLiteral(&code, 2, values[i][0]); + AppendVMovLiteral(&code, 3, values[i][1]); + if (i == 2) { + code.push_back(EncodeSop1(0x04, 126, InlineU32(0))); + } + AppendBufferStoreOpcode(&code, 0x52, 2, 20, i != 1); + code.push_back(EncodeSop1(0x04, 126, 12)); + AppendStoreVgpr(&code, 2, 2 + i * 2); + AppendStoreVgpr(&code, 3, 3 + i * 2); + } + AppendEnd(&code); + test.initial = {0x12345678u, 0x87654321u, 0, 0, 0, 0, 0, 0}; + // The first add wraps at 64 bits; the second carries from the low dword. + // GLC returns the first old value, preserves the second operand, and EXEC = 0 + // suppresses the third add and its writeback despite its nonzero operand. + test.expected = {0x21256568u, 0x78743430u, 0x12345678u, 0x87654321u, + 0xff00ff00u, 0x00ff00ffu, 0x76543210u, 0xfedcba98u}; + test.opcodes = {O::V_MOV_B32, O::S_MOV_B64, O::BUFFER_ATOMIC_ADD_X2, + O::BUFFER_STORE_DWORD, O::S_ENDPGM}; + test.required_spirv = {"OpAtomicIAdd %ulong", "Int64Atomics"}; + return test; +} + TestCase BufferAtomicCmpSwapExactRaw() { using O = ShaderOpcode; @@ -42248,6 +42283,7 @@ std::vector MakeCases() { } } AddCase(BufferAtomicAndX2GlcAndExec); + AddCase(BufferAtomicAddX2GlcAndExec); AddCase(ImageAtomicSignedMinMax); AddCase(ImageAtomicSignedMinMax); AddCase(ImageAtomicSignedMinMax); @@ -47786,6 +47822,7 @@ int main(int argc, char **argv) { } RunCase(&vulkan, DsAtomicReturn64()); RunCase(&vulkan, BufferAtomicAndX2GlcAndExec()); + RunCase(&vulkan, BufferAtomicAddX2GlcAndExec()); RunCase(&vulkan, ImageAtomicSignedMinMax()); RunCase(&vulkan, ImageAtomicSignedMinMax()); return 0; diff --git a/tests/shaderCfgTests.cpp b/tests/shaderCfgTests.cpp index b8e0b75fc..dc1761f8f 100644 --- a/tests/shaderCfgTests.cpp +++ b/tests/shaderCfgTests.cpp @@ -10539,6 +10539,8 @@ void TestCapturedBufferAtomicsX2() { ValueOpcode ir_opcode; }; const Case cases[] = { + {{0xe1480018u, 0x80000000u}, 0u, 234u, "BUFFER_ATOMIC_ADD_X2", + "BufferAtomicIAdd64", "OpAtomicIAdd", ValueOpcode::BufferAtomicIAdd64}, {{0xe1680018u, 0x80000000u}, 0u, 241u, "BUFFER_ATOMIC_OR_X2", "BufferAtomicOr64", "OpAtomicOr", ValueOpcode::BufferAtomicOr64}, {{0xe1402000u, 0x80000913u}, 9u, 229u, "BUFFER_ATOMIC_SWAP_X2",