From 7492cb95777b19fbacdf91f999fcc37b0a8b81c5 Mon Sep 17 00:00:00 2001 From: NeySlim <4509984+NeySlim@users.noreply.github.com> Date: Tue, 6 Oct 2026 23:45:37 +0200 Subject: [PATCH] shader: emulate DS_ORDERED_COUNT as a GDS atomic (#1117) * feat(shader): emulate DS_ORDERED_COUNT as a GDS atomic * fix(shader): issue one ordered-count atomic per wave * shader: correct ordered-count addressing and wave results Use the M0 base and pixel packer offset for the GDS counter. Broadcast the previous value to every lane and preserve the counter when EXEC is empty. Add GPU regressions for add/exchange with full, partial, and empty EXEC in wave32 and wave64. The targeted DS and buffer atomic suite passes 63 cases. --------- Co-authored-by: nmzik --- .../recompiler/frontend/decode/MemoryOps.cpp | 10 +++ .../frontend/decode/ShaderDecoder.h | 1 + .../recompiler/frontend/translate/Memory.cpp | 45 ++++++++++++ .../frontend/translate/Translator.h | 1 + tests/ShaderRecompilerComputeTests.cpp | 72 +++++++++++++++++++ 5 files changed, 129 insertions(+) diff --git a/src/graphics/shader/recompiler/frontend/decode/MemoryOps.cpp b/src/graphics/shader/recompiler/frontend/decode/MemoryOps.cpp index f963091ef..4c51be1cd 100644 --- a/src/graphics/shader/recompiler/frontend/decode/MemoryOps.cpp +++ b/src/graphics/shader/recompiler/frontend/decode/MemoryOps.cpp @@ -109,6 +109,7 @@ constexpr MemoryOpcodeInfo DS_OPCODE_LIST[] = { {0x39u, Opcode::DS_READ_I8, 1, 8, true}, {0x3au, Opcode::DS_READ_U8, 1, 8}, {0x3bu, Opcode::DS_READ_I16, 1, 16, true}, {0x3cu, Opcode::DS_READ_U16, 1, 16}, {0x3du, Opcode::DS_CONSUME, 1, 32}, {0x3eu, Opcode::DS_APPEND, 1, 32}, + {0x3fu, Opcode::DS_ORDERED_COUNT, 1, 32}, {0x40u, Opcode::DS_ADD_U64, 2, 32}, {0x4au, Opcode::DS_OR_B64, 2, 32}, {0x4du, Opcode::DS_WRITE_B64, 2, 32}, {0x4eu, Opcode::DS_WRITE2_B64, 4, 32}, {0x4fu, Opcode::DS_WRITE2ST64_B64, 4, 32}, {0x76u, Opcode::DS_READ_B64, 2, 32}, @@ -214,6 +215,7 @@ uint32_t DsSourceCount(Opcode opcode) { case Opcode::DS_READ_ADDTID_B32: case Opcode::DS_CONSUME: case Opcode::DS_APPEND: return 0u; + case Opcode::DS_ORDERED_COUNT: return 1u; default: return IsDsWriteOpcode(opcode) || IsDsAtomicOpcode(opcode) ? 2u : 1u; } } @@ -412,6 +414,14 @@ void DecodeDs(uint32_t pc, std::span code, uint32_t word_index, if (inst.opcode == Opcode::UNSUPPORTED) { SetUnsupported(inst, Family::DS, opcode, "DS opcode is not implemented"); } + if (inst.opcode == Opcode::DS_ORDERED_COUNT) { + // Keep OFFSET1's wave type as well as its add/swap selector for GDS addressing. + inst.offset = offset0 & 0xfcu; + inst.secondary_offset = offset1; + if (!inst.gds || ((offset1 >> 4u) & 3u) > 1u || ((offset1 >> 6u) & 3u) != 0u) { + SetUnsupported(inst, Family::DS, opcode, "DS ordered count variant is not implemented"); + } + } if (inst.opcode == Opcode::DS_SWIZZLE_B32 && inst.offset >= 0xe000u) { SetUnsupported(inst, Family::DS, opcode, "DS swizzle FFT mode is not implemented"); } diff --git a/src/graphics/shader/recompiler/frontend/decode/ShaderDecoder.h b/src/graphics/shader/recompiler/frontend/decode/ShaderDecoder.h index c8aa69209..d3ad3b669 100644 --- a/src/graphics/shader/recompiler/frontend/decode/ShaderDecoder.h +++ b/src/graphics/shader/recompiler/frontend/decode/ShaderDecoder.h @@ -580,6 +580,7 @@ enum class Opcode { DS_BPERMUTE_B32, DS_CONSUME, DS_APPEND, + DS_ORDERED_COUNT, DS_READ_I8, DS_READ_U8, DS_READ_I16, diff --git a/src/graphics/shader/recompiler/frontend/translate/Memory.cpp b/src/graphics/shader/recompiler/frontend/translate/Memory.cpp index 9aaf81342..d5aa60526 100644 --- a/src/graphics/shader/recompiler/frontend/translate/Memory.cpp +++ b/src/graphics/shader/recompiler/frontend/translate/Memory.cpp @@ -883,6 +883,50 @@ void Translator::DS_SWIZZLE_B32(const Decoder::Instruction& inst) { ReadU32(MemorySourceAt(inst, 1)), ir.GetExec()})); } +void Translator::DS_ORDERED_COUNT(const Decoder::Instruction& inst) { + // Approximate ordered counting with one GDS atomic per wave. Wave-launch ordering and + // release/done synchronization are not emulated. + const auto memory = MemoryInfoFromDecoded(inst); + const auto m0 = ir.GetM0(); + auto address = ir.BitwiseAnd(ir.ShiftRightLogical(m0, IR::U32(IR::Value(16u))), + IR::U32(IR::Value(0xfffcu))); + if (((inst.secondary_offset >> 2u) & 3u) == 1u) { + const auto packer = ir.BitwiseAnd(m0, IR::U32(IR::Value(0xfu))); + address = ir.IAdd(address, ir.ShiftLeftLogical(packer, IR::U32(IR::Value(2u)))); + } + const auto exec = ir.GetExec(); + const auto ballot = ir.Emit(IR::ValueOpcode::Ballot, {exec}); + const auto low = ir.CompositeExtract(ballot, 0u); + const auto high = ir.CompositeExtract(ballot, 1u); + const auto active = ir.INotEqual(ir.BitwiseOr(low, high), IR::U32(IR::Value(0u))); + const auto first = ir.Select( + ir.INotEqual(low, IR::U32(IR::Value(0u))), + IR::U32(ir.Emit(IR::ValueOpcode::FindILsb32, {low})), + ir.IAdd(IR::U32(IR::Value(32u)), IR::U32(ir.Emit(IR::ValueOpcode::FindILsb32, {high})))); + // An empty EXEC still returns the counter. Elect lane zero and perform an add of zero. + const auto selected = ir.Select(active, first, IR::U32(IR::Value(0u))); + const auto lane = IR::U32(ir.Emit(IR::ValueOpcode::LaneId)); + const auto is_first = ir.IEqual(lane, selected); + const auto source = IR::U32(ir.Emit(IR::ValueOpcode::ReadFirstLane, + {ReadU32(inst.src0), is_first})); + const auto value = ir.Select(active, source, IR::U32(IR::Value(0u))); + const auto flags = AddMemoryInfo(memory, inst.pc); + IR::U32 result; + if (((inst.secondary_offset >> 4u) & 3u) == 1u) { + const auto swapped = IR::U32(ir.Emit(IR::ValueOpcode::SharedAtomicSwap32, + {address, value, ir.LogicalAnd(is_first, active)}, flags)); + const auto unchanged = IR::U32(ir.Emit(IR::ValueOpcode::SharedAtomicIAdd32, + {address, IR::Value(0u), ir.LogicalAnd(is_first, ir.LogicalNot(active))}, flags)); + result = ir.Select(active, swapped, unchanged); + } else { + result = IR::U32(ir.Emit(IR::ValueOpcode::SharedAtomicIAdd32, + {address, value, is_first}, flags)); + } + // DS_ORDERED_COUNT writes every destination lane, regardless of EXEC. + ir.SetVectorReg(static_cast(inst.dst.reg), + IR::U32(ir.Emit(IR::ValueOpcode::ReadFirstLane, {result, is_first}))); +} + void Translator::DS_PERMUTE(const Decoder::Instruction& inst, bool backward) { const auto address = ir.IAdd(ReadU32(inst.src0), IR::U32(IR::Value(inst.offset))); WriteOperand(inst.dst, ir.Emit(backward ? IR::ValueOpcode::BpermuteU32 @@ -1114,6 +1158,7 @@ void Translator::EmitMemory(const Decoder::Instruction& inst) { case Decoder::Opcode::DS_SWIZZLE_B32: return DS_SWIZZLE_B32(inst); case Decoder::Opcode::DS_PERMUTE_B32: return DS_PERMUTE(inst, false); case Decoder::Opcode::DS_BPERMUTE_B32: return DS_PERMUTE(inst, true); + case Decoder::Opcode::DS_ORDERED_COUNT: return DS_ORDERED_COUNT(inst); case Decoder::Opcode::DS_CONSUME: return DS_APPEND_CONSUME(inst, IR::ValueOpcode::DataConsume); case Decoder::Opcode::DS_APPEND: diff --git a/src/graphics/shader/recompiler/frontend/translate/Translator.h b/src/graphics/shader/recompiler/frontend/translate/Translator.h index 5efaa4785..2da58b213 100644 --- a/src/graphics/shader/recompiler/frontend/translate/Translator.h +++ b/src/graphics/shader/recompiler/frontend/translate/Translator.h @@ -109,6 +109,7 @@ private: void DS_ADDTID(const Decoder::Instruction& inst, bool write); void DS_SWIZZLE_B32(const Decoder::Instruction& inst); void DS_PERMUTE(const Decoder::Instruction& inst, bool backward); + void DS_ORDERED_COUNT(const Decoder::Instruction& inst); IR::F32 SelectF32(IR::U1 condition, IR::F32 true_value, IR::F32 false_value); IR::U32 ConvertF32ToU32Saturated(IR::F32 value, float upper_bound, float safe_upper, diff --git a/tests/ShaderRecompilerComputeTests.cpp b/tests/ShaderRecompilerComputeTests.cpp index 34e56f130..a65c62ce2 100644 --- a/tests/ShaderRecompilerComputeTests.cpp +++ b/tests/ShaderRecompilerComputeTests.cpp @@ -32243,6 +32243,66 @@ TestCase DsAppendUsesEncodedGdsSelector() { return test; } +TestCase DsOrderedCountAddressAndExec(u32 wave_size, bool pixel_counter) { + using O = ShaderOpcode; + TestCase test; + test.name = pixel_counter + ? (wave_size == 64 ? "DsOrderedCountPixelAddressWave64" + : "DsOrderedCountPixelAddressWave32") + : (wave_size == 64 ? "DsOrderedCountAddressAndExecWave64" + : "DsOrderedCountAddressAndExecWave32"); + auto &code = test.code; + // Both the base and OFFSET0 are aligned independently; M0's low bits are + // a wave index, or a pixel packer ID, rather than a byte address or size. + AppendSMovLiteral(&code, 124, 0x01030003u); + code.push_back(EncodeSop1(0x04, 10, 126)); // Save the full EXEC mask. + code.push_back(EncodeVop2(0x25, 2, InlineU32(1), 0)); // ADDR value = lane + 1. + AppendVMovU32(&code, 9, 99); // DATA0 must not supply the count. + const u32 selected_lane = wave_size == 64 ? 47u : 7u; + const std::array masks = { + wave_size == 64 ? UINT64_MAX : uint64_t{UINT32_MAX}, + uint64_t{1} << selected_lane, 0}; + u32 counter = 1000; + for (bool exchange : {false, true}) { + for (size_t mode = 0; mode < masks.size(); ++mode) { + AppendVMovU32(&code, 3, 0xdeadbeefu); + AppendSMovLiteral(&code, 126, static_cast(masks[mode])); + AppendSMovLiteral(&code, 127, static_cast(masks[mode] >> 32u)); + const u32 release_done = exchange && mode == 2 ? 3u : 0u; + const u32 offset1 = release_done | (pixel_counter ? 4u : 0u) | + (exchange ? 0x10u : 0u); + code.push_back(EncodeDs0(0x3f, (offset1 << 8u) | 7u, true)); + code.push_back(EncodeDs1(3, 9, 2)); + code.push_back(EncodeSopp(0x0c, 0)); // Wait for the returned GDS value. + code.push_back(EncodeSop1(0x04, 126, 10)); // Restore EXEC before reading all lanes. + AppendStoreVgprAtLaneDwordOffset(&code, 3, 0, + static_cast(test.expected.size())); + test.expected.insert(test.expected.end(), wave_size, counter); + if (mode != 2) { + const u32 value = mode == 0 ? 1u : selected_lane + 1u; + counter = exchange ? value : counter + value; + } + } + } + AppendEnd(&code); + test.initial.assign(test.expected.size(), 0xdeadbeefu); + const u32 counter_address = pixel_counter ? 0x110u : 0x104u; + test.gds_initial.assign(0x120u / sizeof(u32), 100); + test.gds_initial[counter_address / sizeof(u32)] = 1000; + test.expected_gds = test.gds_initial; + test.expected_gds[counter_address / sizeof(u32)] = counter; + test.compute_info.threads_num[0] = wave_size; + test.compute_info.threads_num[1] = 1; + test.compute_info.threads_num[2] = 1; + test.compute_info.thread_ids_num = 1; + test.compute_info.wave_size = wave_size; + test.has_compute_info = true; + test.opcodes = {O::S_MOV_B32, O::S_MOV_B64, O::V_ADD_NC_U32, O::V_MOV_B32, + O::DS_ORDERED_COUNT, O::S_WAITCNT, O::V_LSHLREV_B32, + O::BUFFER_STORE_DWORD, O::S_ENDPGM}; + return test; +} + TestCase DsAppendConsumeGdsRegionBounds() { using O = ShaderOpcode; struct Access { @@ -35538,6 +35598,10 @@ std::vector MakeCases() { cases.push_back(DsAppendAllocatesAcrossWaves(32)); cases.push_back(DsAppendAllocatesAcrossWaves(64)); AddCase(DsAppendUsesEncodedGdsSelector); + for (u32 wave_size : {32, 64}) { + cases.push_back(DsOrderedCountAddressAndExec(wave_size, false)); + cases.push_back(DsOrderedCountAddressAndExec(wave_size, true)); + } AddCase(DsAppendConsumeGdsRegionBounds); AddCase(DsGdsSubdwordAndAtomicWrites); AddCase(DsReadWrite2Variants); @@ -41059,6 +41123,14 @@ int main(int argc, char **argv) { } return 0; } + if (argc == 2 && std::strcmp(argv[1], "--ds-ordered-count-only") == 0) { + VulkanHarness vulkan; + for (u32 wave_size : {32, 64}) { + RunCase(&vulkan, DsOrderedCountAddressAndExec(wave_size, false)); + RunCase(&vulkan, DsOrderedCountAddressAndExec(wave_size, true)); + } + return 0; + } if (argc == 2 && std::strcmp(argv[1], "--ds-atomics-only") == 0) { VulkanHarness vulkan; for (const auto &test : MakeCases()) {