From e04007a7c67f1057e076552f3e6797235901296f Mon Sep 17 00:00:00 2001 From: nmzik Date: Wed, 9 Sep 2026 12:56:20 +0400 Subject: [PATCH] shader: implement DS_INC_RTN_U32 and DS_DEC_RTN_U32 --- .../backend/spirv/spirvEmitterMemory.cpp | 95 +++++--- .../recompiler/frontend/decode/MemoryOps.cpp | 3 + .../frontend/decode/ShaderDecoder.cpp | 2 + .../frontend/decode/ShaderDecoder.h | 2 + .../recompiler/frontend/translate/Memory.cpp | 4 + .../recompiler/ir/opcodes/ValueOpcodes.cpp | 2 + .../recompiler/ir/opcodes/ValueOpcodes.inc | 2 + tests/ShaderRecompilerComputeTests.cpp | 228 ++++++++++++++++++ tests/shaderCfgTests.cpp | 13 + 9 files changed, 316 insertions(+), 35 deletions(-) diff --git a/src/graphics/shader/recompiler/backend/spirv/spirvEmitterMemory.cpp b/src/graphics/shader/recompiler/backend/spirv/spirvEmitterMemory.cpp index 08d67b09a..374f39d54 100644 --- a/src/graphics/shader/recompiler/backend/spirv/spirvEmitterMemory.cpp +++ b/src/graphics/shader/recompiler/backend/spirv/spirvEmitterMemory.cpp @@ -589,31 +589,58 @@ uint32_t EmitAtomicOperation(ValueEmitContext& ctx, const IR::Inst& inst, uint32 return old; } +template +uint32_t EmitAtomicAccess(ValueEmitContext& ctx, const IR::Inst& inst, + const IR::MemoryInfo& mem, Fn&& operation) { + return EmitValueOrZeroIfCondition(ctx.state, ctx.Arg(inst, inst.NumArgs() - 1), [&]() { + const auto access = PrepareMemoryElement(ctx, mem, DwordIndex(ctx, inst, mem)); + return EmitValueOrZeroIfCondition( + ctx.state, EmitMemoryElementInBounds(ctx.state, access.resource, access.index), [&]() { + return operation(EmitMemoryElementPointer(ctx.state, access.resource, access.index)); + }); + }); +} + uint32_t EmitAtomic(ValueEmitContext& ctx, const IR::Inst& inst, const IR::MemoryInfo& mem) { - const auto result = - EmitValueOrZeroIfCondition(ctx.state, ctx.Arg(inst, inst.NumArgs() - 1), [&]() { - const auto access = PrepareMemoryElement(ctx, mem, DwordIndex(ctx, inst, mem)); - return EmitValueOrZeroIfCondition( - ctx.state, EmitMemoryElementInBounds(ctx.state, access.resource, access.index), - [&]() { - const auto scope = - mem.kind == IR::ResourceKind::Lds ? ScopeWorkgroup : ScopeDevice; - const auto pointer = - EmitMemoryElementPointer(ctx.state, access.resource, access.index); - const auto old = EmitAtomicOperation(ctx, inst, pointer, scope); - if (mem.kind == IR::ResourceKind::Lds) { - const auto semantics = - MemorySemanticsAcquireRelease | MemorySemanticsWorkgroupMemory; - ctx.state.builder.AddFunction({OpMemoryBarrier, - ConstantU32(ctx.state, ScopeWorkgroup), - ConstantU32(ctx.state, semantics)}); - } else { - EmitDeviceAtomicMemoryBarrier(ctx.state); - } - return old; - }); - }); - return result; + return EmitAtomicAccess(ctx, inst, mem, [&](uint32_t pointer) { + const auto scope = mem.kind == IR::ResourceKind::Lds ? ScopeWorkgroup : ScopeDevice; + const auto old = EmitAtomicOperation(ctx, inst, pointer, scope); + if (mem.kind == IR::ResourceKind::Lds) { + const auto semantics = MemorySemanticsAcquireRelease | MemorySemanticsWorkgroupMemory; + ctx.state.builder.AddFunction({OpMemoryBarrier, ConstantU32(ctx.state, scope), + ConstantU32(ctx.state, semantics)}); + } else { + EmitDeviceAtomicMemoryBarrier(ctx.state); + } + return old; + }); +} + +template +uint32_t EmitAtomicUpdate(ValueEmitContext& ctx, const IR::Inst& inst, + const IR::MemoryInfo& mem, Fn&& replacement) { + const auto value = ctx.Arg(inst, inst.NumArgs() - 2); + return EmitAtomicAccess(ctx, inst, mem, [&](uint32_t pointer) { + return AtomicUpdate(ctx.state, pointer, mem.kind, [&](uint32_t old) { + return replacement(ctx.state, old, value); + }); + }); +} + +uint32_t AtomicIncrement(EmitterState& state, uint32_t old, uint32_t limit) { + // old >= limit ? 0 : old + 1 (unsigned). + const auto wrap = Binary(state, OpUGreaterThanEqual, TypeBool(state), old, limit); + const auto next = Binary(state, OpIAdd, TypeU32(state), old, ConstantU32(state, 1)); + return Select(state, TypeU32(state), wrap, ConstantU32(state, 0), next); +} + +uint32_t AtomicDecrement(EmitterState& state, uint32_t old, uint32_t limit) { + // old == 0 || old > limit ? limit : old - 1 (unsigned). + const auto zero = Binary(state, OpIEqual, TypeBool(state), old, ConstantU32(state, 0)); + const auto above = Binary(state, OpUGreaterThan, TypeBool(state), old, limit); + const auto wrap = Binary(state, OpLogicalOr, TypeBool(state), zero, above); + const auto next = Binary(state, OpISub, TypeU32(state), old, ConstantU32(state, 1)); + return Select(state, TypeU32(state), wrap, limit, next); } uint32_t EmitBufferAtomic64(ValueEmitContext& ctx, const IR::Inst& inst, @@ -647,17 +674,9 @@ uint32_t EmitBufferAtomic64(ValueEmitContext& ctx, const IR::Inst& inst, uint32_t FloatAtomic(ValueEmitContext& ctx, const IR::Inst& inst, const IR::MemoryInfo& mem, bool max_value) { - return EmitValueOrZeroIfCondition(ctx.state, ctx.Arg(inst, inst.NumArgs() - 1), [&]() { - const auto access = PrepareMemoryElement(ctx, mem, DwordIndex(ctx, inst, mem)); - return EmitValueOrZeroIfCondition( - ctx.state, EmitMemoryElementInBounds(ctx.state, access.resource, access.index), [&]() { - const auto pointer = - EmitMemoryElementPointer(ctx.state, access.resource, access.index); - const auto source = ctx.Arg(inst, inst.NumArgs() - 2); - return AtomicUpdate(ctx.state, pointer, mem.kind, [&](uint32_t old) { - return EmitFloatAtomicReplacement(ctx.state, old, source, max_value); - }); - }); + return EmitAtomicUpdate(ctx, inst, mem, [max_value](EmitterState& state, uint32_t old, + uint32_t value) { + return EmitFloatAtomicReplacement(state, old, value, max_value); }); } @@ -1138,6 +1157,12 @@ bool EmitValueMemory(ValueEmitContext& ctx, const IR::Inst& inst) { op == IR::ValueOpcode::BufferAtomicFMax32)); return true; } + if (op == IR::ValueOpcode::SharedAtomicInc32 || op == IR::ValueOpcode::SharedAtomicDec32) { + const auto replacement = op == IR::ValueOpcode::SharedAtomicInc32 + ? AtomicIncrement : AtomicDecrement; + ctx.Define(inst, EmitAtomicUpdate(ctx, inst, ctx.Memory(inst), replacement)); + return true; + } if (op == IR::ValueOpcode::SharedAtomicFMin32 || op == IR::ValueOpcode::SharedAtomicFMax32) { SharedFloatAtomic(ctx, inst, ctx.Memory(inst), op == IR::ValueOpcode::SharedAtomicFMax32); return true; diff --git a/src/graphics/shader/recompiler/frontend/decode/MemoryOps.cpp b/src/graphics/shader/recompiler/frontend/decode/MemoryOps.cpp index 3b3881be3..75974ecbb 100644 --- a/src/graphics/shader/recompiler/frontend/decode/MemoryOps.cpp +++ b/src/graphics/shader/recompiler/frontend/decode/MemoryOps.cpp @@ -94,6 +94,7 @@ constexpr MemoryOpcodeInfo DS_OPCODE_LIST[] = { {0x12u, Opcode::DS_MIN_F32, 1, 32}, {0x13u, Opcode::DS_MAX_F32, 1, 32}, {0x1eu, Opcode::DS_WRITE_B8, 1, 8}, {0x1fu, Opcode::DS_WRITE_B16, 1, 16}, {0x20u, Opcode::DS_ADD_RTN_U32, 1, 32}, {0x21u, Opcode::DS_SUB_RTN_U32, 1, 32}, + {0x23u, Opcode::DS_INC_RTN_U32, 1, 32}, {0x24u, Opcode::DS_DEC_RTN_U32, 1, 32}, {0x25u, Opcode::DS_MIN_RTN_I32, 1, 32}, {0x26u, Opcode::DS_MAX_RTN_I32, 1, 32}, {0x27u, Opcode::DS_MIN_RTN_U32, 1, 32}, {0x28u, Opcode::DS_MAX_RTN_U32, 1, 32}, {0x29u, Opcode::DS_AND_RTN_B32, 1, 32}, {0x2au, Opcode::DS_OR_RTN_B32, 1, 32}, @@ -165,6 +166,8 @@ bool IsDsAtomicOpcode(Opcode opcode) { case Opcode::DS_ADD_RTN_U32: case Opcode::DS_SUB_U32: case Opcode::DS_SUB_RTN_U32: + case Opcode::DS_INC_RTN_U32: + case Opcode::DS_DEC_RTN_U32: case Opcode::DS_MIN_I32: case Opcode::DS_MIN_RTN_I32: case Opcode::DS_MAX_I32: diff --git a/src/graphics/shader/recompiler/frontend/decode/ShaderDecoder.cpp b/src/graphics/shader/recompiler/frontend/decode/ShaderDecoder.cpp index b4a4b424f..436f79949 100644 --- a/src/graphics/shader/recompiler/frontend/decode/ShaderDecoder.cpp +++ b/src/graphics/shader/recompiler/frontend/decode/ShaderDecoder.cpp @@ -626,6 +626,8 @@ std::string InstructionToString(const Instruction& inst) { case Opcode::DS_ADD_RTN_U32: case Opcode::DS_SUB_U32: case Opcode::DS_SUB_RTN_U32: + case Opcode::DS_INC_RTN_U32: + case Opcode::DS_DEC_RTN_U32: case Opcode::DS_MIN_I32: case Opcode::DS_MIN_RTN_I32: case Opcode::DS_MAX_I32: diff --git a/src/graphics/shader/recompiler/frontend/decode/ShaderDecoder.h b/src/graphics/shader/recompiler/frontend/decode/ShaderDecoder.h index 3e5ae8ef2..60e9632d1 100644 --- a/src/graphics/shader/recompiler/frontend/decode/ShaderDecoder.h +++ b/src/graphics/shader/recompiler/frontend/decode/ShaderDecoder.h @@ -488,6 +488,8 @@ enum class Opcode { DS_ADD_RTN_U32, DS_SUB_U32, DS_SUB_RTN_U32, + DS_INC_RTN_U32, + DS_DEC_RTN_U32, DS_MIN_I32, DS_MIN_RTN_I32, DS_MAX_I32, diff --git a/src/graphics/shader/recompiler/frontend/translate/Memory.cpp b/src/graphics/shader/recompiler/frontend/translate/Memory.cpp index b03318308..fa79a40b8 100644 --- a/src/graphics/shader/recompiler/frontend/translate/Memory.cpp +++ b/src/graphics/shader/recompiler/frontend/translate/Memory.cpp @@ -968,6 +968,10 @@ bool Translator::EmitMemory(const Decoder::Instruction& inst) { return DS_ATOMIC(inst, IR::ValueOpcode::SharedAtomicISub32, false); case Decoder::Opcode::DS_SUB_RTN_U32: return DS_ATOMIC(inst, IR::ValueOpcode::SharedAtomicISub32, true); + case Decoder::Opcode::DS_INC_RTN_U32: + return DS_ATOMIC(inst, IR::ValueOpcode::SharedAtomicInc32, true); + case Decoder::Opcode::DS_DEC_RTN_U32: + return DS_ATOMIC(inst, IR::ValueOpcode::SharedAtomicDec32, true); case Decoder::Opcode::DS_MIN_I32: return DS_ATOMIC(inst, IR::ValueOpcode::SharedAtomicSMin32, false); case Decoder::Opcode::DS_MIN_RTN_I32: diff --git a/src/graphics/shader/recompiler/ir/opcodes/ValueOpcodes.cpp b/src/graphics/shader/recompiler/ir/opcodes/ValueOpcodes.cpp index 4ffe10555..18afc7874 100644 --- a/src/graphics/shader/recompiler/ir/opcodes/ValueOpcodes.cpp +++ b/src/graphics/shader/recompiler/ir/opcodes/ValueOpcodes.cpp @@ -163,6 +163,8 @@ SharedAccess SharedAccessOf(ValueOpcode opcode) { case ValueOpcode::SharedAtomicSwap32: case ValueOpcode::SharedAtomicIAdd32: case ValueOpcode::SharedAtomicISub32: + case ValueOpcode::SharedAtomicInc32: + case ValueOpcode::SharedAtomicDec32: case ValueOpcode::SharedAtomicSMin32: case ValueOpcode::SharedAtomicUMin32: case ValueOpcode::SharedAtomicSMax32: diff --git a/src/graphics/shader/recompiler/ir/opcodes/ValueOpcodes.inc b/src/graphics/shader/recompiler/ir/opcodes/ValueOpcodes.inc index e15e30d9a..a393de243 100644 --- a/src/graphics/shader/recompiler/ir/opcodes/ValueOpcodes.inc +++ b/src/graphics/shader/recompiler/ir/opcodes/ValueOpcodes.inc @@ -255,6 +255,8 @@ VALUE_OPCODE(SharedAtomicFMax32, Void, U32, U32, U32, U1) VALUE_OPCODE(SharedAtomicSwap32, U32, U32, U32, U1) VALUE_OPCODE(SharedAtomicIAdd32, U32, U32, U32, U1) VALUE_OPCODE(SharedAtomicISub32, U32, U32, U32, U1) +VALUE_OPCODE(SharedAtomicInc32, U32, U32, U32, U1) +VALUE_OPCODE(SharedAtomicDec32, U32, U32, U32, U1) VALUE_OPCODE(SharedAtomicSMin32, U32, U32, U32, U1) VALUE_OPCODE(SharedAtomicUMin32, U32, U32, U32, U1) VALUE_OPCODE(SharedAtomicSMax32, U32, U32, U32, U1) diff --git a/tests/ShaderRecompilerComputeTests.cpp b/tests/ShaderRecompilerComputeTests.cpp index 37460f38a..b6e8258e3 100644 --- a/tests/ShaderRecompilerComputeTests.cpp +++ b/tests/ShaderRecompilerComputeTests.cpp @@ -86,6 +86,7 @@ #include #include #include +#include #include #if KYTY_PLATFORM != KYTY_PLATFORM_WINDOWS @@ -14015,6 +14016,8 @@ CoverageClass ClassifyOpcode(ShaderOpcode opcode, case Opcode::DS_ADD_RTN_U32: case Opcode::DS_SUB_U32: case Opcode::DS_SUB_RTN_U32: + case Opcode::DS_INC_RTN_U32: + case Opcode::DS_DEC_RTN_U32: case Opcode::DS_MIN_I32: case Opcode::DS_MIN_RTN_I32: case Opcode::DS_MAX_I32: @@ -21221,6 +21224,213 @@ TestCase DsAtomicReturnVariants() { O::BUFFER_STORE_DWORD, O::S_ENDPGM}}; } +TestCase DsBoundedAtomicBoundaries(bool decrement, bool gds) { + using O = ShaderOpcode; + constexpr u32 guard = 0x13579bdfu; + struct Boundary { + u32 before, limit, incremented, decremented; + }; + constexpr Boundary boundaries[] = { + {4, 5, 5, 3}, + {5, 5, 0, 4}, + {6, 5, 0, 5}, + {0, 5, 1, 5}, + {0, 0, 0, 0}, + {1, 0, 0, 0}, + {0xffffffffu, 0xffffffffu, 0, 0xfffffffeu}, + {0xfffffffeu, 0xffffffffu, 0xffffffffu, 0xfffffffdu}, + {0, 0xffffffffu, 1, 0xffffffffu}, + {0x80000000u, 0x7fffffffu, 0, 0x7fffffffu}, + {0x7fffffffu, 0x80000000u, 0x80000000u, 0x7ffffffeu}, + {0xffffffffu, 0x80000000u, 0, 0x80000000u}}; + constexpr u32 count = static_cast(std::size(boundaries)); + const u32 opcode = decrement ? 0x24u : 0x23u; + TestCase test; + test.name = + decrement + ? (gds ? "DsDecReturnGdsBoundaries" : "DsDecReturnLdsBoundaries") + : (gds ? "DsIncReturnGdsBoundaries" : "DsIncReturnLdsBoundaries"); + auto &code = test.code; + AppendVMovU32(&code, 2, 0); + std::vector memory(68, guard); + for (u32 i = 0; i < count; ++i) + memory[i + 1] = boundaries[i].before; + if (!gds) { + for (u32 i = 0; i < memory.size(); ++i) { + AppendVMovLiteral(&code, 3, memory[i]); + code.push_back(EncodeDs0(0x0d, i * 4)); + code.push_back(EncodeDs1(0, 3, 2)); + } + } + auto StoreResult = [&](u32 vgpr, u32 expected) { + AppendStoreVgpr(&code, vgpr, static_cast(test.expected.size())); + test.expected.push_back(expected); + }; + for (u32 i = 0; i < count; ++i) { + AppendVMovU32(&code, 2, i * 4); + AppendVMovLiteral(&code, 3, boundaries[i].limit); + // Preserve the captured DATA0/VDST alias and exact increment words in GDS. + code.push_back(decrement ? EncodeDs0(opcode, 4, gds) + : (gds ? 0xd88e0004u : 0xd88c0004u)); + code.push_back(0x03000302u); + StoreResult(3, boundaries[i].before); + code.push_back(EncodeDs0(0x36, 4, gds)); + code.push_back(EncodeDs1(4, 0, 2)); + StoreResult(4, decrement ? boundaries[i].decremented + : boundaries[i].incremented); + } + + // EXEC=0 preserves memory and the aliased destination register. + AppendVMovU32(&code, 2, 0); + AppendVMovLiteral(&code, 3, 0x12345678u); + code.push_back(EncodeSop1(0x04, 10, 126)); + code.push_back(EncodeSop1(0x04, 126, InlineU32(0))); + code.push_back(EncodeDs0(opcode, 4, gds)); + code.push_back(0x03000302u); + code.push_back(EncodeSop1(0x04, 126, 10)); + StoreResult(3, 0x12345678u); + + // Use both offset bytes and alias VDST with ADDR instead of DATA0. + AppendVMovU32(&code, 2, 4); + AppendVMovLiteral(&code, 3, 0xffffffffu); + code.push_back(EncodeDs0(opcode, 0x104, gds)); + code.push_back(EncodeDs1(2, 3, 2)); + StoreResult(2, guard); + + // Overwriting an unused return must not let DCE discard the atomic update. + AppendVMovU32(&code, 2, 0); + code.push_back(EncodeDs0(opcode, 0x108, gds)); + code.push_back(EncodeDs1(4, 3, 2)); + AppendVMovU32(&code, 4, 0); + + // An active access just outside the resource returns zero. + AppendVMovU32(&code, 2, static_cast(memory.size() * 4 - 4)); + code.push_back(EncodeDs0(opcode, 4, gds)); + code.push_back(0x03000302u); + StoreResult(3, 0); + + // Read back all words, including guards around both address ranges. + AppendVMovU32(&code, 2, 0); + for (u32 i = 0; i < memory.size(); ++i) { + u32 expected = memory[i]; + if (i >= 1 && i <= count) { + expected = decrement ? boundaries[i - 1].decremented + : boundaries[i - 1].incremented; + } else if (i == 66) { + expected = decrement ? guard - 2 : guard + 2; + } + code.push_back(EncodeDs0(0x36, i * 4, gds)); + code.push_back(EncodeDs1(4, 0, 2)); + StoreResult(4, expected); + if (gds) + test.expected_gds.push_back(expected); + } + AppendEnd(&code); + test.initial.resize(test.expected.size()); + test.compute_info.lds_size_dwords = static_cast(memory.size()); + test.opcodes = { + O::S_MOV_B64, O::V_MOV_B32, + O::DS_READ_B32, decrement ? O::DS_DEC_RTN_U32 : O::DS_INC_RTN_U32, + O::BUFFER_STORE_DWORD, O::S_ENDPGM}; + test.required_spirv = {"OpAtomicLoad", "OpAtomicCompareExchange", + "OpLoopMerge"}; + if (gds) + test.gds_initial = memory; + else + test.opcodes.push_back(O::DS_WRITE_B32); + return test; +} + +TestCase DsBoundedAtomicContention(bool decrement, bool gds, u32 wave_size) { + using O = ShaderOpcode; + constexpr u32 guard = 0x13579bdfu; + constexpr u32 limit = 15; + TestCase test; + if (wave_size == 32) { + test.name = decrement ? (gds ? "DsDecReturnGdsContentionWave32" + : "DsDecReturnLdsContentionWave32") + : (gds ? "DsIncReturnGdsContentionWave32" + : "DsIncReturnLdsContentionWave32"); + } else { + test.name = decrement ? (gds ? "DsDecReturnGdsContentionWave64" + : "DsDecReturnLdsContentionWave64") + : (gds ? "DsIncReturnGdsContentionWave64" + : "DsIncReturnLdsContentionWave64"); + } + auto &code = test.code; + AppendVMovU32(&code, 2, 0); + AppendVMovU32(&code, 3, limit); + AppendVMovU32(&code, 5, 1); + AppendVMovLiteral(&code, 6, guard); + if (!gds) { + // One lane initializes LDS before every wave in the workgroup uses it. + code.push_back(EncodeSop1(0x04, 10, 126)); + code.push_back(EncodeVopc(0xc2, InlineU32(0), 0)); + code.push_back(EncodeSop1(0x04, 126, 106)); + code.push_back(EncodeDs0(0x0d, 4)); + code.push_back(EncodeDs1(0, 2, 2)); + code.push_back(EncodeSop1(0x04, 126, 10)); + code.push_back(EncodeSopp(0x0a, 0)); + } + // Distinct masks in the wave64 halves also check inactive return + // preservation. + code.push_back(EncodeSop1(0x04, 10, 126)); + AppendSMovLiteral(&code, 126, 0x55555555u); + AppendSMovLiteral(&code, 127, 0xaaaaaaaau); + code.push_back(EncodeDs0(decrement ? 0x24 : 0x23, 4, gds)); + code.push_back(EncodeDs1(6, 3, 2)); + // Returned old values form a histogram; no lane ordering is assumed. + code.push_back(EncodeVop2(0x1a, 7, InlineU32(2), 6)); + AppendBufferStoreOpcode(&code, 0x32, 5, 7); + code.push_back(EncodeSop1(0x04, 126, 10)); + AppendVMovLiteral(&code, 7, guard); + code.push_back(EncodeVopc(0xc2, Vgpr(7), 6)); + code.push_back(EncodeVop2(0x01, 8, InlineU32(0), 5)); + AppendStoreVgprAtLaneDwordOffset(&code, 8, 0, 16); + code.push_back(EncodeSopp(0x0a, 0)); + // S_BARRIER orders LDS; use atomic add-zero to read GDS consistently. + code.push_back(EncodeDs0(gds ? 0x20 : 0x36, 4, gds)); + code.push_back(EncodeDs1(9, 2, 2)); + AppendStoreVgprAtLaneDwordOffset(&code, 9, 0, 144); + AppendEnd(&code); + test.expected.assign(16, 4); // 64 active lanes, four complete wraps. + for (u32 lane = 0; lane < 128; ++lane) { + const uint64_t mask = 0xaaaaaaaa55555555ull; + test.expected.push_back( + (mask & (uint64_t{1} << (lane % wave_size))) == 0 ? 1 : 0); + } + test.expected.resize(272, 0); + test.initial.resize(test.expected.size()); + test.compute_info.threads_num[0] = 128; + test.compute_info.threads_num[1] = 1; + test.compute_info.threads_num[2] = 1; + test.compute_info.thread_ids_num = 1; + test.compute_info.wave_size = wave_size; + test.compute_info.lds_size_dwords = 3; + test.has_compute_info = true; + test.opcodes = {O::S_MOV_B32, + O::S_MOV_B64, + O::S_BARRIER, + O::V_MOV_B32, + O::V_LSHLREV_B32, + O::V_ADD_NC_U32, + O::V_CMP_EQ_U32, + O::V_CNDMASK_B32, + decrement ? O::DS_DEC_RTN_U32 : O::DS_INC_RTN_U32, + gds ? O::DS_ADD_RTN_U32 : O::DS_READ_B32, + O::BUFFER_ATOMIC_ADD, + O::BUFFER_STORE_DWORD, + O::S_ENDPGM}; + test.required_spirv = {"OpAtomicLoad", "OpAtomicCompareExchange", + "OpLoopMerge"}; + if (gds) { + test.gds_initial = {guard, 0, guard}; + test.expected_gds = test.gds_initial; + } else + test.opcodes.push_back(O::DS_WRITE_B32); + return test; +} + TestCase DsMiscVariants() { using O = ShaderOpcode; @@ -23989,6 +24199,14 @@ std::vector MakeCases() { AddCase(DsWideGdsPartialBounds); AddCase(DsAtomicNoReturnVariants); AddCase(DsAtomicReturnVariants); + for (bool decrement : {false, true}) { + for (bool gds : {false, true}) { + cases.push_back(DsBoundedAtomicBoundaries(decrement, gds)); + for (u32 wave_size : {32u, 64u}) { + cases.push_back(DsBoundedAtomicContention(decrement, gds, wave_size)); + } + } + } AddCase(DsMiscVariants); AddCase(DsFloatMinMaxUsesSeparateCompareOperand); AddCase(DsSwizzleInvalidSourceLaneZero); @@ -28293,6 +28511,16 @@ int main(int argc, char **argv) { RunCase(&vulkan, BufferAtomicCmpSwapExactRaw()); return 0; } + if (argc == 2 && std::strcmp(argv[1], "--ds-atomics-only") == 0) { + VulkanHarness vulkan; + for (const auto &test : MakeCases()) { + if (std::string_view(test.name).starts_with("Ds") || + std::string_view(test.name).starts_with("BufferAtomic")) { + RunCase(&vulkan, test); + } + } + return 0; + } if (argc == 2 && std::strcmp(argv[1], "--mapped-range-only") == 0) { VulkanHarness vulkan; vulkan.CheckGpuMappedRangeLifecycle(); diff --git a/tests/shaderCfgTests.cpp b/tests/shaderCfgTests.cpp index 2a0927043..efba9b081 100644 --- a/tests/shaderCfgTests.cpp +++ b/tests/shaderCfgTests.cpp @@ -4224,6 +4224,19 @@ void TestNewShaderDecoderArchitecture() { Check(ds.opcode == Opcode::DS_READ_B32 && ds.gds, "DS decoder lost the GFX10 opcode or GDS fields"); + for (bool decrement : {false, true}) { + const uint32_t words[] = {decrement ? 0xd8920004u : 0xd88e0004u, 0x03000302u}; + Instruction atomic; + ShaderRecompiler::Decoder::DecodeInstruction(words, 0u, atomic); + Check(atomic.family == Family::DS && + atomic.opcode == (decrement ? Opcode::DS_DEC_RTN_U32 : Opcode::DS_INC_RTN_U32) && + atomic.word_count == 2u && atomic.src_count == 2u && + atomic.data_dwords == 1u && atomic.data_bits == 32u && + atomic.gds && atomic.offset == 4u && atomic.dst.reg == 3u && + atomic.src0.reg == 2u && atomic.src1.reg == 3u, + "DS decoder misdecoded bounded atomic return fields"); + } + const uint32_t boot_ds[] = {0xd8d4c480u, 0x45000045u}; Instruction boot; ShaderRecompiler::Decoder::DecodeInstruction(boot_ds, 0u, boot);