From 98557a83b64f1f6e64a8620dea194114d8fe969d Mon Sep 17 00:00:00 2001 From: nmzik Date: Sun, 4 Oct 2026 17:18:35 +0400 Subject: [PATCH] shader: implement RDNA2 DS masked OR through shared atomic updates Co-authored-by: shadowbeat070 <70843316+shadowbeat070@users.noreply.github.com> --- .../backend/spirv/spirvEmitterInstructions.h | 1 + .../backend/spirv/spirvEmitterMemory.cpp | 17 +++- .../recompiler/frontend/decode/MemoryOps.cpp | 5 +- .../frontend/decode/ShaderDecoder.cpp | 1 + .../frontend/decode/ShaderDecoder.h | 1 + .../recompiler/frontend/translate/Memory.cpp | 35 +++---- .../frontend/translate/Translator.h | 1 - .../recompiler/ir/opcodes/ValueOpcodes.cpp | 3 +- .../recompiler/ir/opcodes/ValueOpcodes.inc | 1 + tests/ShaderRecompilerComputeTests.cpp | 97 +++++++++++++++++++ tests/shaderCfgTests.cpp | 10 ++ 11 files changed, 144 insertions(+), 28 deletions(-) diff --git a/src/graphics/shader/recompiler/backend/spirv/spirvEmitterInstructions.h b/src/graphics/shader/recompiler/backend/spirv/spirvEmitterInstructions.h index cb4f5ef09..7457930e3 100644 --- a/src/graphics/shader/recompiler/backend/spirv/spirvEmitterInstructions.h +++ b/src/graphics/shader/recompiler/backend/spirv/spirvEmitterInstructions.h @@ -232,6 +232,7 @@ void EmitStoreMemory(ValueEmitContext& ctx, const IR::Inst& ins uint32_t EmitAtomic32(ValueEmitContext& ctx, const IR::Inst& inst); uint32_t EmitBufferAtomic64(ValueEmitContext& ctx, const IR::Inst& inst); void EmitSharedAtomic64(ValueEmitContext& ctx, const IR::Inst& inst); +void EmitSharedAtomicMaskedOr32(ValueEmitContext& ctx, const IR::Inst& inst); uint32_t EmitBufferFloatAtomic(ValueEmitContext& ctx, const IR::Inst& inst); void EmitSharedFloatAtomic(ValueEmitContext& ctx, const IR::Inst& inst); uint32_t EmitSharedIncDec(ValueEmitContext& ctx, const IR::Inst& inst); diff --git a/src/graphics/shader/recompiler/backend/spirv/spirvEmitterMemory.cpp b/src/graphics/shader/recompiler/backend/spirv/spirvEmitterMemory.cpp index 0a596afe3..e8afe4458 100644 --- a/src/graphics/shader/recompiler/backend/spirv/spirvEmitterMemory.cpp +++ b/src/graphics/shader/recompiler/backend/spirv/spirvEmitterMemory.cpp @@ -259,8 +259,14 @@ uint32_t ByteAddress(ValueEmitContext& ctx, const IR::Inst& inst, const IR::Memo } uint32_t DwordIndex(ValueEmitContext& ctx, const IR::Inst& inst, const IR::MemoryInfo& mem) { + auto address = ByteAddress(ctx, inst, mem); + if (mem.kind == IR::ResourceKind::Lds || mem.kind == IR::ResourceKind::Gds) { + // RDNA2 DS region addresses retain bits [15:2] after adding the byte offset. + address = Binary(ctx.state, spv::OpBitwiseAnd, TypeU32(ctx.state), address, + ConstantU32(ctx.state, 0xffffu)); + } return Binary(ctx.state, spv::OpShiftRightLogical, TypeU32(ctx.state), - ByteAddress(ctx, inst, mem), ConstantU32(ctx.state, 2)); + address, ConstantU32(ctx.state, 2)); } uint32_t LoadWordInBounds(ValueEmitContext& ctx, const MemoryResourceAccess& resource, @@ -1415,6 +1421,15 @@ uint32_t EmitSharedIncDec(ValueEmitContext& ctx, const IR::Inst& inst) { return EmitAtomicUpdate(ctx, inst, ctx.Memory(inst), replacement); } +void EmitSharedAtomicMaskedOr32(ValueEmitContext& ctx, const IR::Inst& inst) { + const auto keep = Unary(ctx.state, spv::OpNot, TypeU32(ctx.state), ctx.Arg(inst, 1)); + EmitAtomicUpdate(ctx, inst, ctx.Memory(inst), + [keep](EmitterState& state, uint32_t old, uint32_t value) { + return Binary(state, spv::OpBitwiseOr, TypeU32(state), + Binary(state, spv::OpBitwiseAnd, TypeU32(state), old, keep), value); + }); +} + uint32_t EmitSwizzleU32(ValueEmitContext& ctx, const IR::Inst& inst) { auto& state = ctx.state; state.builder.AddFunction(spv::OpStore, ctx.scratch_u32_variable, ctx.Arg(inst, 0)); diff --git a/src/graphics/shader/recompiler/frontend/decode/MemoryOps.cpp b/src/graphics/shader/recompiler/frontend/decode/MemoryOps.cpp index 8f5efb53e..d19f0e6fb 100644 --- a/src/graphics/shader/recompiler/frontend/decode/MemoryOps.cpp +++ b/src/graphics/shader/recompiler/frontend/decode/MemoryOps.cpp @@ -92,7 +92,8 @@ constexpr MemoryOpcodeInfo DS_OPCODE_LIST[] = { {0x05u, Opcode::DS_MIN_I32, 1, 32}, {0x06u, Opcode::DS_MAX_I32, 1, 32}, {0x07u, Opcode::DS_MIN_U32, 1, 32}, {0x08u, Opcode::DS_MAX_U32, 1, 32}, {0x09u, Opcode::DS_AND_B32, 1, 32}, {0x0au, Opcode::DS_OR_B32, 1, 32}, - {0x0bu, Opcode::DS_XOR_B32, 1, 32}, {0x0du, Opcode::DS_WRITE_B32, 1, 32}, + {0x0bu, Opcode::DS_XOR_B32, 1, 32}, {0x0cu, Opcode::DS_MSKOR_B32, 1, 32}, + {0x0du, Opcode::DS_WRITE_B32, 1, 32}, {0x0eu, Opcode::DS_WRITE2_B32, 2, 32}, {0x0fu, Opcode::DS_WRITE2ST64_B32, 2, 32}, {0x12u, Opcode::DS_MIN_F32, 1, 32}, {0x13u, Opcode::DS_MAX_F32, 1, 32}, {0x1eu, Opcode::DS_WRITE_B8, 1, 8}, {0x1fu, Opcode::DS_WRITE_B16, 1, 16}, @@ -192,6 +193,7 @@ bool IsDsAtomicOpcode(Opcode opcode) { case Opcode::DS_OR_RTN_B32: case Opcode::DS_XOR_B32: case Opcode::DS_XOR_RTN_B32: + case Opcode::DS_MSKOR_B32: case Opcode::DS_WRXCHG_RTN_B32: return true; default: return false; } @@ -203,6 +205,7 @@ uint32_t DsSourceCount(Opcode opcode) { case Opcode::DS_WRITE2ST64_B32: case Opcode::DS_WRITE2_B64: case Opcode::DS_WRITE2ST64_B64: + case Opcode::DS_MSKOR_B32: case Opcode::DS_MIN_F32: case Opcode::DS_MAX_F32: return 3u; case Opcode::DS_PERMUTE_B32: diff --git a/src/graphics/shader/recompiler/frontend/decode/ShaderDecoder.cpp b/src/graphics/shader/recompiler/frontend/decode/ShaderDecoder.cpp index 9825e8e70..c33637879 100644 --- a/src/graphics/shader/recompiler/frontend/decode/ShaderDecoder.cpp +++ b/src/graphics/shader/recompiler/frontend/decode/ShaderDecoder.cpp @@ -697,6 +697,7 @@ std::string InstructionToString(const Instruction& inst) { case Opcode::DS_OR_RTN_B32: case Opcode::DS_XOR_B32: case Opcode::DS_XOR_RTN_B32: + case Opcode::DS_MSKOR_B32: case Opcode::DS_WRXCHG_RTN_B32: case Opcode::DS_MIN_F32: case Opcode::DS_MAX_F32: diff --git a/src/graphics/shader/recompiler/frontend/decode/ShaderDecoder.h b/src/graphics/shader/recompiler/frontend/decode/ShaderDecoder.h index 93432b6d3..fd990a2b2 100644 --- a/src/graphics/shader/recompiler/frontend/decode/ShaderDecoder.h +++ b/src/graphics/shader/recompiler/frontend/decode/ShaderDecoder.h @@ -558,6 +558,7 @@ enum class Opcode { DS_OR_RTN_B32, DS_XOR_B32, DS_XOR_RTN_B32, + DS_MSKOR_B32, DS_WRXCHG_RTN_B32, DS_MIN_F32, DS_MAX_F32, diff --git a/src/graphics/shader/recompiler/frontend/translate/Memory.cpp b/src/graphics/shader/recompiler/frontend/translate/Memory.cpp index 5872deb62..9e1c45b18 100644 --- a/src/graphics/shader/recompiler/frontend/translate/Memory.cpp +++ b/src/graphics/shader/recompiler/frontend/translate/Memory.cpp @@ -239,9 +239,6 @@ Decoder::Operand MemorySourceAt(const Decoder::Instruction& decoded, uint32_t in case Decoder::Opcode::DS_READ_ADDTID_B32: return MakeM0Operand(); case Decoder::Opcode::DS_WRITE_ADDTID_B32: return index == 0u ? decoded.src1 : MakeM0Operand(); - case Decoder::Opcode::DS_MIN_F32: - case Decoder::Opcode::DS_MAX_F32: - return index == 0u ? decoded.src1 : index == 1u ? decoded.src0 : decoded.src2; case Decoder::Opcode::DS_WRITE_B8: case Decoder::Opcode::DS_WRITE_B16: case Decoder::Opcode::DS_WRITE_B8_D16_HI: @@ -255,12 +252,7 @@ Decoder::Operand MemorySourceAt(const Decoder::Instruction& decoded, uint32_t in case Decoder::Opcode::DS_WRITE2_B64: case Decoder::Opcode::DS_WRITE2ST64_B64: return index == 0u ? decoded.src1 : index == 1u ? decoded.src0 : decoded.src2; - default: - if (decoded.opcode >= Decoder::Opcode::DS_ADD_U32 && - decoded.opcode <= Decoder::Opcode::DS_WRXCHG_RTN_B32) { - return index == 0u ? decoded.src1 : decoded.src0; - } - return decoded.src0; + default: return decoded.src0; } } return DecodedSourceAt(decoded, index); @@ -580,12 +572,13 @@ void Translator::IMAGE_ATOMIC(const Decoder::Instruction& inst, IR::ValueOpcode void Translator::DS_ATOMIC(const Decoder::Instruction& inst, IR::ValueOpcode opcode, bool returns_value) { const auto memory = MemoryInfoFromDecoded(inst); - const auto address = ReadU32(MemorySourceAt(inst, 1)); - const auto data_src = MemorySourceAt(inst, 0); + const auto address = ReadU32(inst.src0); const IR::Value value = - memory.data_dwords == 2u ? IR::Value(ReadU64(data_src)) : IR::Value(ReadU32(data_src)); - const auto result = ir.Emit(opcode, {address, value, ir.GetExec()}, - AddMemoryInfo(memory, inst.pc)); + memory.data_dwords == 2u ? IR::Value(ReadU64(inst.src1)) : IR::Value(ReadU32(inst.src1)); + const auto flags = AddMemoryInfo(memory, inst.pc); + const auto result = inst.src_count == 3u + ? ir.Emit(opcode, {address, value, ReadU32(inst.src2), ir.GetExec()}, flags) + : ir.Emit(opcode, {address, value, ir.GetExec()}, flags); if (returns_value) { WriteOperand(inst.dst, result); } @@ -857,14 +850,6 @@ void Translator::DS_WRITE2(const Decoder::Instruction& inst) { } } -void Translator::DS_MINMAX_F32(const Decoder::Instruction& inst, IR::ValueOpcode opcode) { - const auto memory = MemoryInfoFromDecoded(inst); - ir.Emit(opcode, - {ReadU32(MemorySourceAt(inst, 1)), ReadU32(MemorySourceAt(inst, 0)), - ReadU32(MemorySourceAt(inst, 2)), ir.GetExec()}, - AddMemoryInfo(memory, inst.pc)); -} - void Translator::DS_APPEND_CONSUME(const Decoder::Instruction& inst, IR::ValueOpcode opcode) { const auto memory = MemoryInfoFromDecoded(inst); WriteOperand(inst.dst, ir.Emit(opcode, @@ -1048,6 +1033,8 @@ void Translator::EmitMemory(const Decoder::Instruction& inst) { return DS_ATOMIC(inst, IR::ValueOpcode::SharedAtomicXor32, false); case Decoder::Opcode::DS_XOR_RTN_B32: return DS_ATOMIC(inst, IR::ValueOpcode::SharedAtomicXor32, true); + case Decoder::Opcode::DS_MSKOR_B32: + return DS_ATOMIC(inst, IR::ValueOpcode::SharedAtomicMaskedOr32, false); case Decoder::Opcode::DS_WRXCHG_RTN_B32: return DS_ATOMIC(inst, IR::ValueOpcode::SharedAtomicSwap32, true); @@ -1110,9 +1097,9 @@ void Translator::EmitMemory(const Decoder::Instruction& inst) { case Decoder::Opcode::IMAGE_GATHER4H: return IMAGE_GATHER(inst); case Decoder::Opcode::DS_MIN_F32: - return DS_MINMAX_F32(inst, IR::ValueOpcode::SharedAtomicFMin32); + return DS_ATOMIC(inst, IR::ValueOpcode::SharedAtomicFMin32, false); case Decoder::Opcode::DS_MAX_F32: - return DS_MINMAX_F32(inst, IR::ValueOpcode::SharedAtomicFMax32); + return DS_ATOMIC(inst, IR::ValueOpcode::SharedAtomicFMax32, false); case Decoder::Opcode::DS_SWIZZLE_B32: return DS_SWIZZLE_B32(inst); case Decoder::Opcode::DS_PERMUTE_B32: return DS_PERMUTE(inst, false); case Decoder::Opcode::DS_BPERMUTE_B32: return DS_PERMUTE(inst, true); diff --git a/src/graphics/shader/recompiler/frontend/translate/Translator.h b/src/graphics/shader/recompiler/frontend/translate/Translator.h index 8ce2407a1..eb00ad830 100644 --- a/src/graphics/shader/recompiler/frontend/translate/Translator.h +++ b/src/graphics/shader/recompiler/frontend/translate/Translator.h @@ -105,7 +105,6 @@ private: void DS_READ2(const Decoder::Instruction& inst); void DS_WRITE(const Decoder::Instruction& inst); void DS_WRITE2(const Decoder::Instruction& inst); - void DS_MINMAX_F32(const Decoder::Instruction& inst, IR::ValueOpcode opcode); void DS_APPEND_CONSUME(const Decoder::Instruction& inst, IR::ValueOpcode opcode); void DS_ADDTID(const Decoder::Instruction& inst, bool write); void DS_SWIZZLE_B32(const Decoder::Instruction& inst); diff --git a/src/graphics/shader/recompiler/ir/opcodes/ValueOpcodes.cpp b/src/graphics/shader/recompiler/ir/opcodes/ValueOpcodes.cpp index 4e8a43a4b..656d905b9 100644 --- a/src/graphics/shader/recompiler/ir/opcodes/ValueOpcodes.cpp +++ b/src/graphics/shader/recompiler/ir/opcodes/ValueOpcodes.cpp @@ -178,7 +178,8 @@ SharedAccess SharedAccessOf(ValueOpcode opcode) { case ValueOpcode::SharedAtomicAnd32: case ValueOpcode::SharedAtomicOr32: case ValueOpcode::SharedAtomicOr64: - case ValueOpcode::SharedAtomicXor32: return SharedAccess::Atomic; + case ValueOpcode::SharedAtomicXor32: + case ValueOpcode::SharedAtomicMaskedOr32: return SharedAccess::Atomic; case ValueOpcode::DataAppend: return SharedAccess::Append; case ValueOpcode::DataConsume: return SharedAccess::Consume; default: return SharedAccess::None; diff --git a/src/graphics/shader/recompiler/ir/opcodes/ValueOpcodes.inc b/src/graphics/shader/recompiler/ir/opcodes/ValueOpcodes.inc index 500a5baa0..cc4c38323 100644 --- a/src/graphics/shader/recompiler/ir/opcodes/ValueOpcodes.inc +++ b/src/graphics/shader/recompiler/ir/opcodes/ValueOpcodes.inc @@ -290,6 +290,7 @@ VALUE_OPCODE(SharedAtomicAnd32, U32, U32, U32, U1) VALUE_OPCODE(SharedAtomicOr32, U32, U32, U32, U1) VALUE_OPCODE(SharedAtomicOr64, Void, U32, U64, U1) VALUE_OPCODE(SharedAtomicXor32, U32, U32, U32, U1) +VALUE_OPCODE(SharedAtomicMaskedOr32, Void, U32, U32, U32, U1) VALUE_OPCODE(DataAppend, U32, U32, U1) VALUE_OPCODE(DataConsume, U32, U32, U1) VALUE_OPCODE(SwizzleU32, U32, U32, U32, U1) diff --git a/tests/ShaderRecompilerComputeTests.cpp b/tests/ShaderRecompilerComputeTests.cpp index 49f8f6ab6..1db65189e 100644 --- a/tests/ShaderRecompilerComputeTests.cpp +++ b/tests/ShaderRecompilerComputeTests.cpp @@ -18116,6 +18116,7 @@ CoverageClass ClassifyOpcode(ShaderOpcode opcode, case Opcode::DS_OR_RTN_B32: case Opcode::DS_XOR_B32: case Opcode::DS_XOR_RTN_B32: + case Opcode::DS_MSKOR_B32: case Opcode::DS_WRXCHG_RTN_B32: case Opcode::DS_SWIZZLE_B32: case Opcode::DS_PERMUTE_B32: @@ -29013,6 +29014,96 @@ TestCase DsAtomicNoReturnVariants() { O::S_ENDPGM}}; } +TestCase DsMaskedOrBounds(bool gds) { + using O = ShaderOpcode; + TestCase test; + test.name = gds ? "DsMaskedOrGdsBounds" : "DsMaskedOrLdsBounds"; + auto &code = test.code; + constexpr u32 initial[] = {0xf0f00ff0u, 0x12345678u, 0x87654321u}; + constexpr u32 masks[] = {0x00ff00ffu, 0u, 0xffffffffu}; + constexpr u32 values[] = {0x33000300u, 0x80000001u, 0xabcdef01u}; + AppendVMovU32(&code, 0, 0x1234); + for (u32 i = 0; i < std::size(initial); ++i) { + AppendVMovU32(&code, 2, i * 4u); + AppendVMovLiteral(&code, 8, initial[i]); + code.push_back(EncodeDs0(0x0d, 0x144, gds)); + code.push_back(EncodeDs1(0, 8, 2)); + // Captured mask v8 and data v10 are distinct; LDS wraps and aligns the address. + AppendVMovU32(&code, 2, (gds ? 0u : 0x10000u) + i * 4u + 3u); + AppendVMovLiteral(&code, 8, masks[i]); + AppendVMovLiteral(&code, 10, values[i]); + code.insert(code.end(), {0xd8300144u | (gds ? 1u << 17u : 0u), 0x000a0802u}); + } + AppendVMovLiteral(&code, 8, 0xffffffffu); + AppendVMovU32(&code, 10, 0); + AppendVMovU32(&code, 2, 12); // First DWORD beyond the allocation. + code.insert(code.end(), {EncodeDs0(0x0c, 0x144, gds), 0x000a0802u}); + AppendVMovU32(&code, 2, 0); + code.push_back(EncodeSMovB32(126, InlineU32(0))); + code.insert(code.end(), {EncodeDs0(0x0c, 0x144, gds), 0x000a0802u}); + code.push_back(EncodeSMovB32(126, InlineU32(1))); + for (u32 i = 0; i < std::size(initial); ++i) { + code.push_back(EncodeDs0(0x36, 0x144u + i * 4u, gds)); + code.push_back(EncodeDs1(11, 0, 2)); + AppendStoreVgpr(&code, 11, i); + test.expected.push_back((initial[i] & ~masks[i]) | values[i]); + } + AppendStoreVgpr(&code, 0, 3); // The encoded VDst must stay untouched. + test.expected.push_back(0x1234); + AppendEnd(&code); + test.compute_info.lds_size_dwords = 0x144 / 4 + 3; + if (gds) { + test.gds_initial.resize(0x144 / 4 + 3); + test.expected_gds = test.gds_initial; + std::copy_n(test.expected.begin(), 3, test.expected_gds.begin() + 0x144 / 4); + } + test.opcodes = {O::V_MOV_B32, O::S_MOV_B32, O::DS_WRITE_B32, O::DS_MSKOR_B32, + O::DS_READ_B32, O::BUFFER_STORE_DWORD, O::S_ENDPGM}; + test.required_spirv = {"OpAtomicCompareExchange"}; + return test; +} + +TestCase DsMaskedOrContention(u32 wave_size) { + using O = ShaderOpcode; + TestCase test; + test.name = wave_size == 32 ? "DsMaskedOrContentionWave32" : "DsMaskedOrContentionWave64"; + auto &code = test.code; + AppendVMovU32(&code, 2, 0); + AppendVMovLiteral(&code, 8, 0xffffffffu); + code.push_back(EncodeSop1(0x04, 10, 126)); + code.push_back(EncodeVopc(0xc2, InlineU32(0), 0)); + code.push_back(EncodeSop1(0x04, 126, 106)); + code.push_back(EncodeDs0(0x0d)); + code.push_back(EncodeDs1(0, 8, 2)); + code.push_back(EncodeSop1(0x04, 126, 10)); + code.push_back(EncodeSopp(0x0a, 0)); + AppendVMovU32(&code, 8, 1); + code.push_back(EncodeVop2(0x1a, 8, Vgpr(0), 8)); + AppendVMovU32(&code, 9, 0); + AppendSMovLiteral(&code, 126, 0x55555555u); + AppendSMovLiteral(&code, 127, 0xaaaaaaaau); + code.push_back(EncodeDs0(0x0c)); + code.push_back(EncodeDs1Ex(0, 9, 8, 2)); + code.push_back(EncodeSop1(0x04, 126, 10)); + code.push_back(EncodeSopp(0x0a, 0)); + code.push_back(EncodeDs0(0x36)); + code.push_back(EncodeDs1(11, 0, 2)); + AppendStoreVgprAtLaneDwordOffset(&code, 11, 0, 0); + AppendEnd(&code); + test.expected.assign(128, wave_size == 32 ? 0xaaaaaaaau : 0u); + test.compute_info.threads_num[0] = 128; + test.compute_info.threads_num[1] = test.compute_info.threads_num[2] = 1; + test.compute_info.thread_ids_num = 1; + test.compute_info.wave_size = wave_size; + test.compute_info.lds_size_dwords = 1; + test.has_compute_info = true; + test.opcodes = {O::V_MOV_B32, O::S_MOV_B32, O::S_MOV_B64, O::V_CMP_EQ_U32, + O::V_LSHLREV_B32, O::S_BARRIER, O::DS_WRITE_B32, O::DS_MSKOR_B32, + O::DS_READ_B32, O::BUFFER_STORE_DWORD, O::S_ENDPGM}; + test.required_spirv = {"OpAtomicCompareExchange"}; + return test; +} + TestCase DsAtomicReturnVariants() { using O = ShaderOpcode; @@ -33544,6 +33635,12 @@ std::vector MakeCases() { AddCase(DsWideLdsPartialBounds); AddCase(DsWideGdsPartialBounds); AddCase(DsAtomicNoReturnVariants); + for (bool gds : {false, true}) { + cases.push_back(DsMaskedOrBounds(gds)); + } + for (u32 wave_size : {32, 64}) { + cases.push_back(DsMaskedOrContention(wave_size)); + } AddCase(DsAtomicReturnVariants); for (bool add : {false, true}) { cases.push_back(DsAtomic64Bounds(add)); diff --git a/tests/shaderCfgTests.cpp b/tests/shaderCfgTests.cpp index 292544167..e0429935e 100644 --- a/tests/shaderCfgTests.cpp +++ b/tests/shaderCfgTests.cpp @@ -4614,6 +4614,16 @@ void TestNewShaderDecoderArchitecture() { Check(ds.opcode == Opcode::DS_READ_B32 && ds.gds, "DS decoder lost the GFX10 opcode or GDS fields"); + const uint32_t masked_or_code[] = {0xd8300144u, 0x000a0802u}; + Instruction masked_or; + ShaderRecompiler::Decoder::DecodeInstruction(masked_or_code, 0u, masked_or); + Check(masked_or.opcode == Opcode::DS_MSKOR_B32 && !masked_or.gds && + masked_or.word_count == 2u && masked_or.src_count == 3u && + masked_or.offset == 0x144u && masked_or.src0.reg == 2u && + masked_or.src1.reg == 8u && masked_or.src2.reg == 10u && + masked_or.data_dwords == 1u && masked_or.data_bits == 32u, + "DS decoder misdecoded captured masked OR operands"); + for (bool decrement : {false, true}) { const uint32_t words[] = {decrement ? 0xd8920004u : 0xd88e0004u, 0x03000302u}; Instruction atomic;