diff --git a/src/graphics/shader/recompiler/frontend/decode/MemoryOps.cpp b/src/graphics/shader/recompiler/frontend/decode/MemoryOps.cpp index f963091ef..4c51be1cd 100644 --- a/src/graphics/shader/recompiler/frontend/decode/MemoryOps.cpp +++ b/src/graphics/shader/recompiler/frontend/decode/MemoryOps.cpp @@ -109,6 +109,7 @@ constexpr MemoryOpcodeInfo DS_OPCODE_LIST[] = { {0x39u, Opcode::DS_READ_I8, 1, 8, true}, {0x3au, Opcode::DS_READ_U8, 1, 8}, {0x3bu, Opcode::DS_READ_I16, 1, 16, true}, {0x3cu, Opcode::DS_READ_U16, 1, 16}, {0x3du, Opcode::DS_CONSUME, 1, 32}, {0x3eu, Opcode::DS_APPEND, 1, 32}, + {0x3fu, Opcode::DS_ORDERED_COUNT, 1, 32}, {0x40u, Opcode::DS_ADD_U64, 2, 32}, {0x4au, Opcode::DS_OR_B64, 2, 32}, {0x4du, Opcode::DS_WRITE_B64, 2, 32}, {0x4eu, Opcode::DS_WRITE2_B64, 4, 32}, {0x4fu, Opcode::DS_WRITE2ST64_B64, 4, 32}, {0x76u, Opcode::DS_READ_B64, 2, 32}, @@ -214,6 +215,7 @@ uint32_t DsSourceCount(Opcode opcode) { case Opcode::DS_READ_ADDTID_B32: case Opcode::DS_CONSUME: case Opcode::DS_APPEND: return 0u; + case Opcode::DS_ORDERED_COUNT: return 1u; default: return IsDsWriteOpcode(opcode) || IsDsAtomicOpcode(opcode) ? 2u : 1u; } } @@ -412,6 +414,14 @@ void DecodeDs(uint32_t pc, std::span code, uint32_t word_index, if (inst.opcode == Opcode::UNSUPPORTED) { SetUnsupported(inst, Family::DS, opcode, "DS opcode is not implemented"); } + if (inst.opcode == Opcode::DS_ORDERED_COUNT) { + // Keep OFFSET1's wave type as well as its add/swap selector for GDS addressing. + inst.offset = offset0 & 0xfcu; + inst.secondary_offset = offset1; + if (!inst.gds || ((offset1 >> 4u) & 3u) > 1u || ((offset1 >> 6u) & 3u) != 0u) { + SetUnsupported(inst, Family::DS, opcode, "DS ordered count variant is not implemented"); + } + } if (inst.opcode == Opcode::DS_SWIZZLE_B32 && inst.offset >= 0xe000u) { SetUnsupported(inst, Family::DS, opcode, "DS swizzle FFT mode is not implemented"); } diff --git a/src/graphics/shader/recompiler/frontend/decode/ShaderDecoder.h b/src/graphics/shader/recompiler/frontend/decode/ShaderDecoder.h index c8aa69209..d3ad3b669 100644 --- a/src/graphics/shader/recompiler/frontend/decode/ShaderDecoder.h +++ b/src/graphics/shader/recompiler/frontend/decode/ShaderDecoder.h @@ -580,6 +580,7 @@ enum class Opcode { DS_BPERMUTE_B32, DS_CONSUME, DS_APPEND, + DS_ORDERED_COUNT, DS_READ_I8, DS_READ_U8, DS_READ_I16, diff --git a/src/graphics/shader/recompiler/frontend/translate/Memory.cpp b/src/graphics/shader/recompiler/frontend/translate/Memory.cpp index 9aaf81342..d5aa60526 100644 --- a/src/graphics/shader/recompiler/frontend/translate/Memory.cpp +++ b/src/graphics/shader/recompiler/frontend/translate/Memory.cpp @@ -883,6 +883,50 @@ void Translator::DS_SWIZZLE_B32(const Decoder::Instruction& inst) { ReadU32(MemorySourceAt(inst, 1)), ir.GetExec()})); } +void Translator::DS_ORDERED_COUNT(const Decoder::Instruction& inst) { + // Approximate ordered counting with one GDS atomic per wave. Wave-launch ordering and + // release/done synchronization are not emulated. + const auto memory = MemoryInfoFromDecoded(inst); + const auto m0 = ir.GetM0(); + auto address = ir.BitwiseAnd(ir.ShiftRightLogical(m0, IR::U32(IR::Value(16u))), + IR::U32(IR::Value(0xfffcu))); + if (((inst.secondary_offset >> 2u) & 3u) == 1u) { + const auto packer = ir.BitwiseAnd(m0, IR::U32(IR::Value(0xfu))); + address = ir.IAdd(address, ir.ShiftLeftLogical(packer, IR::U32(IR::Value(2u)))); + } + const auto exec = ir.GetExec(); + const auto ballot = ir.Emit(IR::ValueOpcode::Ballot, {exec}); + const auto low = ir.CompositeExtract(ballot, 0u); + const auto high = ir.CompositeExtract(ballot, 1u); + const auto active = ir.INotEqual(ir.BitwiseOr(low, high), IR::U32(IR::Value(0u))); + const auto first = ir.Select( + ir.INotEqual(low, IR::U32(IR::Value(0u))), + IR::U32(ir.Emit(IR::ValueOpcode::FindILsb32, {low})), + ir.IAdd(IR::U32(IR::Value(32u)), IR::U32(ir.Emit(IR::ValueOpcode::FindILsb32, {high})))); + // An empty EXEC still returns the counter. Elect lane zero and perform an add of zero. + const auto selected = ir.Select(active, first, IR::U32(IR::Value(0u))); + const auto lane = IR::U32(ir.Emit(IR::ValueOpcode::LaneId)); + const auto is_first = ir.IEqual(lane, selected); + const auto source = IR::U32(ir.Emit(IR::ValueOpcode::ReadFirstLane, + {ReadU32(inst.src0), is_first})); + const auto value = ir.Select(active, source, IR::U32(IR::Value(0u))); + const auto flags = AddMemoryInfo(memory, inst.pc); + IR::U32 result; + if (((inst.secondary_offset >> 4u) & 3u) == 1u) { + const auto swapped = IR::U32(ir.Emit(IR::ValueOpcode::SharedAtomicSwap32, + {address, value, ir.LogicalAnd(is_first, active)}, flags)); + const auto unchanged = IR::U32(ir.Emit(IR::ValueOpcode::SharedAtomicIAdd32, + {address, IR::Value(0u), ir.LogicalAnd(is_first, ir.LogicalNot(active))}, flags)); + result = ir.Select(active, swapped, unchanged); + } else { + result = IR::U32(ir.Emit(IR::ValueOpcode::SharedAtomicIAdd32, + {address, value, is_first}, flags)); + } + // DS_ORDERED_COUNT writes every destination lane, regardless of EXEC. + ir.SetVectorReg(static_cast(inst.dst.reg), + IR::U32(ir.Emit(IR::ValueOpcode::ReadFirstLane, {result, is_first}))); +} + void Translator::DS_PERMUTE(const Decoder::Instruction& inst, bool backward) { const auto address = ir.IAdd(ReadU32(inst.src0), IR::U32(IR::Value(inst.offset))); WriteOperand(inst.dst, ir.Emit(backward ? IR::ValueOpcode::BpermuteU32 @@ -1114,6 +1158,7 @@ void Translator::EmitMemory(const Decoder::Instruction& inst) { case Decoder::Opcode::DS_SWIZZLE_B32: return DS_SWIZZLE_B32(inst); case Decoder::Opcode::DS_PERMUTE_B32: return DS_PERMUTE(inst, false); case Decoder::Opcode::DS_BPERMUTE_B32: return DS_PERMUTE(inst, true); + case Decoder::Opcode::DS_ORDERED_COUNT: return DS_ORDERED_COUNT(inst); case Decoder::Opcode::DS_CONSUME: return DS_APPEND_CONSUME(inst, IR::ValueOpcode::DataConsume); case Decoder::Opcode::DS_APPEND: diff --git a/src/graphics/shader/recompiler/frontend/translate/Translator.h b/src/graphics/shader/recompiler/frontend/translate/Translator.h index 5efaa4785..2da58b213 100644 --- a/src/graphics/shader/recompiler/frontend/translate/Translator.h +++ b/src/graphics/shader/recompiler/frontend/translate/Translator.h @@ -109,6 +109,7 @@ private: void DS_ADDTID(const Decoder::Instruction& inst, bool write); void DS_SWIZZLE_B32(const Decoder::Instruction& inst); void DS_PERMUTE(const Decoder::Instruction& inst, bool backward); + void DS_ORDERED_COUNT(const Decoder::Instruction& inst); IR::F32 SelectF32(IR::U1 condition, IR::F32 true_value, IR::F32 false_value); IR::U32 ConvertF32ToU32Saturated(IR::F32 value, float upper_bound, float safe_upper, diff --git a/tests/ShaderRecompilerComputeTests.cpp b/tests/ShaderRecompilerComputeTests.cpp index 34e56f130..a65c62ce2 100644 --- a/tests/ShaderRecompilerComputeTests.cpp +++ b/tests/ShaderRecompilerComputeTests.cpp @@ -32243,6 +32243,66 @@ TestCase DsAppendUsesEncodedGdsSelector() { return test; } +TestCase DsOrderedCountAddressAndExec(u32 wave_size, bool pixel_counter) { + using O = ShaderOpcode; + TestCase test; + test.name = pixel_counter + ? (wave_size == 64 ? "DsOrderedCountPixelAddressWave64" + : "DsOrderedCountPixelAddressWave32") + : (wave_size == 64 ? "DsOrderedCountAddressAndExecWave64" + : "DsOrderedCountAddressAndExecWave32"); + auto &code = test.code; + // Both the base and OFFSET0 are aligned independently; M0's low bits are + // a wave index, or a pixel packer ID, rather than a byte address or size. + AppendSMovLiteral(&code, 124, 0x01030003u); + code.push_back(EncodeSop1(0x04, 10, 126)); // Save the full EXEC mask. + code.push_back(EncodeVop2(0x25, 2, InlineU32(1), 0)); // ADDR value = lane + 1. + AppendVMovU32(&code, 9, 99); // DATA0 must not supply the count. + const u32 selected_lane = wave_size == 64 ? 47u : 7u; + const std::array masks = { + wave_size == 64 ? UINT64_MAX : uint64_t{UINT32_MAX}, + uint64_t{1} << selected_lane, 0}; + u32 counter = 1000; + for (bool exchange : {false, true}) { + for (size_t mode = 0; mode < masks.size(); ++mode) { + AppendVMovU32(&code, 3, 0xdeadbeefu); + AppendSMovLiteral(&code, 126, static_cast(masks[mode])); + AppendSMovLiteral(&code, 127, static_cast(masks[mode] >> 32u)); + const u32 release_done = exchange && mode == 2 ? 3u : 0u; + const u32 offset1 = release_done | (pixel_counter ? 4u : 0u) | + (exchange ? 0x10u : 0u); + code.push_back(EncodeDs0(0x3f, (offset1 << 8u) | 7u, true)); + code.push_back(EncodeDs1(3, 9, 2)); + code.push_back(EncodeSopp(0x0c, 0)); // Wait for the returned GDS value. + code.push_back(EncodeSop1(0x04, 126, 10)); // Restore EXEC before reading all lanes. + AppendStoreVgprAtLaneDwordOffset(&code, 3, 0, + static_cast(test.expected.size())); + test.expected.insert(test.expected.end(), wave_size, counter); + if (mode != 2) { + const u32 value = mode == 0 ? 1u : selected_lane + 1u; + counter = exchange ? value : counter + value; + } + } + } + AppendEnd(&code); + test.initial.assign(test.expected.size(), 0xdeadbeefu); + const u32 counter_address = pixel_counter ? 0x110u : 0x104u; + test.gds_initial.assign(0x120u / sizeof(u32), 100); + test.gds_initial[counter_address / sizeof(u32)] = 1000; + test.expected_gds = test.gds_initial; + test.expected_gds[counter_address / sizeof(u32)] = counter; + test.compute_info.threads_num[0] = wave_size; + test.compute_info.threads_num[1] = 1; + test.compute_info.threads_num[2] = 1; + test.compute_info.thread_ids_num = 1; + test.compute_info.wave_size = wave_size; + test.has_compute_info = true; + test.opcodes = {O::S_MOV_B32, O::S_MOV_B64, O::V_ADD_NC_U32, O::V_MOV_B32, + O::DS_ORDERED_COUNT, O::S_WAITCNT, O::V_LSHLREV_B32, + O::BUFFER_STORE_DWORD, O::S_ENDPGM}; + return test; +} + TestCase DsAppendConsumeGdsRegionBounds() { using O = ShaderOpcode; struct Access { @@ -35538,6 +35598,10 @@ std::vector MakeCases() { cases.push_back(DsAppendAllocatesAcrossWaves(32)); cases.push_back(DsAppendAllocatesAcrossWaves(64)); AddCase(DsAppendUsesEncodedGdsSelector); + for (u32 wave_size : {32, 64}) { + cases.push_back(DsOrderedCountAddressAndExec(wave_size, false)); + cases.push_back(DsOrderedCountAddressAndExec(wave_size, true)); + } AddCase(DsAppendConsumeGdsRegionBounds); AddCase(DsGdsSubdwordAndAtomicWrites); AddCase(DsReadWrite2Variants); @@ -41059,6 +41123,14 @@ int main(int argc, char **argv) { } return 0; } + if (argc == 2 && std::strcmp(argv[1], "--ds-ordered-count-only") == 0) { + VulkanHarness vulkan; + for (u32 wave_size : {32, 64}) { + RunCase(&vulkan, DsOrderedCountAddressAndExec(wave_size, false)); + RunCase(&vulkan, DsOrderedCountAddressAndExec(wave_size, true)); + } + return 0; + } if (argc == 2 && std::strcmp(argv[1], "--ds-atomics-only") == 0) { VulkanHarness vulkan; for (const auto &test : MakeCases()) {