From c17fa5b936dc0596c7529d6f32a86375caaf3a0d Mon Sep 17 00:00:00 2001 From: nmzik Date: Sun, 4 Oct 2026 09:26:44 +0400 Subject: [PATCH] shader: implement RDNA2 DS forward lane permute Co-authored-by: shadowbeat070 <70843316+shadowbeat070@users.noreply.github.com> --- .../recompiler/backend/spirv/SpirvEmitter.cpp | 1 + .../backend/spirv/spirvEmitterInstructions.h | 1 + .../backend/spirv/spirvEmitterMemory.cpp | 46 ++++++++++ .../recompiler/frontend/decode/MemoryOps.cpp | 6 +- .../frontend/decode/ShaderDecoder.cpp | 1 + .../frontend/decode/ShaderDecoder.h | 1 + .../recompiler/frontend/translate/Memory.cpp | 8 +- .../frontend/translate/Translator.h | 2 +- .../recompiler/ir/opcodes/ValueOpcodes.inc | 1 + tests/ShaderRecompilerComputeTests.cpp | 86 +++++++++++++++++++ 10 files changed, 147 insertions(+), 6 deletions(-) diff --git a/src/graphics/shader/recompiler/backend/spirv/SpirvEmitter.cpp b/src/graphics/shader/recompiler/backend/spirv/SpirvEmitter.cpp index d9aa57bf5..08b23c78b 100644 --- a/src/graphics/shader/recompiler/backend/spirv/SpirvEmitter.cpp +++ b/src/graphics/shader/recompiler/backend/spirv/SpirvEmitter.cpp @@ -294,6 +294,7 @@ Emitter::SpirvRequirements Emitter::AnalyzeProgramRequirements(const IR::Program break; } case IR::ValueOpcode::SwizzleU32: + case IR::ValueOpcode::PermuteU32: case IR::ValueOpcode::BpermuteU32: { requirements.subgroup_ballot = true; requirements.subgroup_shuffle = true; diff --git a/src/graphics/shader/recompiler/backend/spirv/spirvEmitterInstructions.h b/src/graphics/shader/recompiler/backend/spirv/spirvEmitterInstructions.h index bbbd2846a..cb4f5ef09 100644 --- a/src/graphics/shader/recompiler/backend/spirv/spirvEmitterInstructions.h +++ b/src/graphics/shader/recompiler/backend/spirv/spirvEmitterInstructions.h @@ -236,6 +236,7 @@ uint32_t EmitBufferFloatAtomic(ValueEmitContext& ctx, const IR::Ins void EmitSharedFloatAtomic(ValueEmitContext& ctx, const IR::Inst& inst); uint32_t EmitSharedIncDec(ValueEmitContext& ctx, const IR::Inst& inst); uint32_t EmitAppendConsume(ValueEmitContext& ctx, const IR::Inst& inst); +uint32_t EmitPermuteU32(ValueEmitContext& ctx, const IR::Inst& inst); uint32_t EmitBpermuteU32(ValueEmitContext& ctx, const IR::Inst& inst); uint32_t EmitReadConst(ValueEmitContext& ctx, const IR::Inst& inst); void EmitReadConstBuffer(ValueEmitContext& ctx, const IR::Inst& inst); diff --git a/src/graphics/shader/recompiler/backend/spirv/spirvEmitterMemory.cpp b/src/graphics/shader/recompiler/backend/spirv/spirvEmitterMemory.cpp index 1b9b223c7..66f58e480 100644 --- a/src/graphics/shader/recompiler/backend/spirv/spirvEmitterMemory.cpp +++ b/src/graphics/shader/recompiler/backend/spirv/spirvEmitterMemory.cpp @@ -1421,6 +1421,52 @@ uint32_t EmitSwizzleU32(ValueEmitContext& ctx, const IR::Inst& inst) { return EmitDsMaskedLaneRead(state, source, target, ctx.Arg(inst, 2)); } +uint32_t EmitPermuteU32(ValueEmitContext& ctx, const IR::Inst& inst) { + auto& state = ctx.state; + auto lane = EmitSubgroupLocalInvocationId(state); + if (state.lane_count == 2) { + lane = Binary(state, spv::OpBitwiseAnd, TypeU32(state), lane, ConstantU32(state, 31)); + } + const auto address = ctx.Arg(inst, 1); + const auto word = Binary(state, spv::OpShiftRightLogical, TypeU32(state), lane, + ConstantU32(state, 5)); + const auto ballot_word = [&](uint32_t predicate) { + const auto ballot = state.builder.AllocateId(); + const auto result = state.builder.AllocateId(); + state.builder.AddFunction(spv::OpGroupNonUniformBallot, TypeU32Vector(state, 4), ballot, + ConstantU32(state, spv::ScopeSubgroup), predicate); + state.builder.AddFunction(spv::OpVectorExtractDynamic, TypeU32(state), result, ballot, word); + return result; + }; + // RDNA2 permutes independently within each 32-lane half. Intersect the source + // address bit ballots to find this destination's enabled writers without LDS. + auto writers = ballot_word(ctx.Arg(inst, 2)); + for (uint32_t bit = 0; bit < 5; ++bit) { + const auto address_bit = Binary(state, spv::OpBitwiseAnd, TypeU32(state), address, + ConstantU32(state, 1u << (bit + 2))); + const auto mask = ballot_word(Binary(state, spv::OpINotEqual, TypeBool(state), + address_bit, ConstantU32(state, 0))); + const auto lane_bit = Binary(state, spv::OpBitwiseAnd, TypeU32(state), lane, + ConstantU32(state, 1u << bit)); + const auto selected = Select( + state, TypeU32(state), + Binary(state, spv::OpINotEqual, TypeBool(state), lane_bit, ConstantU32(state, 0)), + mask, Unary(state, spv::OpNot, TypeU32(state), mask)); + writers = Binary(state, spv::OpBitwiseAnd, TypeU32(state), writers, selected); + } + const auto active = Binary(state, spv::OpINotEqual, TypeBool(state), writers, + ConstantU32(state, 0)); + const auto base = Binary(state, spv::OpBitwiseAnd, TypeU32(state), lane, + ConstantU32(state, ~31u)); + const auto source = Select(state, TypeU32(state), active, + Binary(state, spv::OpBitwiseOr, TypeU32(state), base, + EmitFindUMsb32(state, writers)), lane); + const auto result = state.builder.AllocateId(); + state.builder.AddFunction(spv::OpGroupNonUniformShuffle, TypeU32(state), result, + ConstantU32(state, spv::ScopeSubgroup), ctx.Arg(inst, 0), source); + return Select(state, TypeU32(state), active, result, ConstantU32(state, 0)); +} + uint32_t EmitBpermuteU32(ValueEmitContext& ctx, const IR::Inst& inst) { auto& state = ctx.state; const auto source = ctx.Arg(inst, 0); diff --git a/src/graphics/shader/recompiler/frontend/decode/MemoryOps.cpp b/src/graphics/shader/recompiler/frontend/decode/MemoryOps.cpp index af5a58b42..8f5efb53e 100644 --- a/src/graphics/shader/recompiler/frontend/decode/MemoryOps.cpp +++ b/src/graphics/shader/recompiler/frontend/decode/MemoryOps.cpp @@ -116,7 +116,7 @@ constexpr MemoryOpcodeInfo DS_OPCODE_LIST[] = { {0xa6u, Opcode::DS_READ_U16_D16, 1, 16}, {0xa7u, Opcode::DS_READ_U16_D16_HI, 1, 16}, {0xb0u, Opcode::DS_WRITE_ADDTID_B32, 1, 32}, {0xb1u, Opcode::DS_READ_ADDTID_B32, 1, 32}, - {0xb3u, Opcode::DS_BPERMUTE_B32, 1, 32}, + {0xb2u, Opcode::DS_PERMUTE_B32, 1, 32}, {0xb3u, Opcode::DS_BPERMUTE_B32, 1, 32}, {0xdeu, Opcode::DS_WRITE_B96, 3, 32}, {0xdfu, Opcode::DS_WRITE_B128, 4, 32}, {0xfeu, Opcode::DS_READ_B96, 3, 32}, {0xffu, Opcode::DS_READ_B128, 4, 32}, }; @@ -205,6 +205,7 @@ uint32_t DsSourceCount(Opcode opcode) { case Opcode::DS_WRITE2ST64_B64: case Opcode::DS_MIN_F32: case Opcode::DS_MAX_F32: return 3u; + case Opcode::DS_PERMUTE_B32: case Opcode::DS_BPERMUTE_B32: return 2u; case Opcode::DS_READ_ADDTID_B32: case Opcode::DS_CONSUME: @@ -407,7 +408,8 @@ void DecodeDs(uint32_t pc, std::span code, uint32_t word_index, SetUnsupported(inst, Family::DS, opcode, "DS swizzle FFT mode is not implemented"); } if (inst.gds && - (inst.opcode == Opcode::DS_SWIZZLE_B32 || inst.opcode == Opcode::DS_BPERMUTE_B32 || + (inst.opcode == Opcode::DS_SWIZZLE_B32 || inst.opcode == Opcode::DS_PERMUTE_B32 || + inst.opcode == Opcode::DS_BPERMUTE_B32 || inst.opcode == Opcode::DS_WRITE_ADDTID_B32 || inst.opcode == Opcode::DS_READ_ADDTID_B32)) { SetUnsupported(inst, Family::DS, opcode, "DS lane operation is available only for LDS"); diff --git a/src/graphics/shader/recompiler/frontend/decode/ShaderDecoder.cpp b/src/graphics/shader/recompiler/frontend/decode/ShaderDecoder.cpp index 6a115a9d6..9825e8e70 100644 --- a/src/graphics/shader/recompiler/frontend/decode/ShaderDecoder.cpp +++ b/src/graphics/shader/recompiler/frontend/decode/ShaderDecoder.cpp @@ -701,6 +701,7 @@ std::string InstructionToString(const Instruction& inst) { case Opcode::DS_MIN_F32: case Opcode::DS_MAX_F32: case Opcode::DS_SWIZZLE_B32: + case Opcode::DS_PERMUTE_B32: case Opcode::DS_BPERMUTE_B32: case Opcode::DS_READ_I8: case Opcode::DS_READ_U8: diff --git a/src/graphics/shader/recompiler/frontend/decode/ShaderDecoder.h b/src/graphics/shader/recompiler/frontend/decode/ShaderDecoder.h index c213c9219..93432b6d3 100644 --- a/src/graphics/shader/recompiler/frontend/decode/ShaderDecoder.h +++ b/src/graphics/shader/recompiler/frontend/decode/ShaderDecoder.h @@ -562,6 +562,7 @@ enum class Opcode { DS_MIN_F32, DS_MAX_F32, DS_SWIZZLE_B32, + DS_PERMUTE_B32, DS_BPERMUTE_B32, DS_CONSUME, DS_APPEND, diff --git a/src/graphics/shader/recompiler/frontend/translate/Memory.cpp b/src/graphics/shader/recompiler/frontend/translate/Memory.cpp index 7c173041d..bc96c5298 100644 --- a/src/graphics/shader/recompiler/frontend/translate/Memory.cpp +++ b/src/graphics/shader/recompiler/frontend/translate/Memory.cpp @@ -894,9 +894,10 @@ void Translator::DS_SWIZZLE_B32(const Decoder::Instruction& inst) { ReadU32(MemorySourceAt(inst, 1)), ir.GetExec()})); } -void Translator::DS_BPERMUTE_B32(const Decoder::Instruction& inst) { +void Translator::DS_PERMUTE(const Decoder::Instruction& inst, bool backward) { const auto address = ir.IAdd(ReadU32(inst.src0), IR::U32(IR::Value(inst.offset))); - WriteOperand(inst.dst, ir.Emit(IR::ValueOpcode::BpermuteU32, + WriteOperand(inst.dst, ir.Emit(backward ? IR::ValueOpcode::BpermuteU32 + : IR::ValueOpcode::PermuteU32, {ReadU32(inst.src1), address, ir.GetExec()})); } @@ -1112,7 +1113,8 @@ void Translator::EmitMemory(const Decoder::Instruction& inst) { case Decoder::Opcode::DS_MAX_F32: return DS_MINMAX_F32(inst, IR::ValueOpcode::SharedAtomicFMax32); case Decoder::Opcode::DS_SWIZZLE_B32: return DS_SWIZZLE_B32(inst); - case Decoder::Opcode::DS_BPERMUTE_B32: return DS_BPERMUTE_B32(inst); + case Decoder::Opcode::DS_PERMUTE_B32: return DS_PERMUTE(inst, false); + case Decoder::Opcode::DS_BPERMUTE_B32: return DS_PERMUTE(inst, true); case Decoder::Opcode::DS_CONSUME: return DS_APPEND_CONSUME(inst, IR::ValueOpcode::DataConsume); case Decoder::Opcode::DS_APPEND: diff --git a/src/graphics/shader/recompiler/frontend/translate/Translator.h b/src/graphics/shader/recompiler/frontend/translate/Translator.h index e98219634..8ce2407a1 100644 --- a/src/graphics/shader/recompiler/frontend/translate/Translator.h +++ b/src/graphics/shader/recompiler/frontend/translate/Translator.h @@ -109,7 +109,7 @@ private: void DS_APPEND_CONSUME(const Decoder::Instruction& inst, IR::ValueOpcode opcode); void DS_ADDTID(const Decoder::Instruction& inst, bool write); void DS_SWIZZLE_B32(const Decoder::Instruction& inst); - void DS_BPERMUTE_B32(const Decoder::Instruction& inst); + void DS_PERMUTE(const Decoder::Instruction& inst, bool backward); IR::F32 SelectF32(IR::U1 condition, IR::F32 true_value, IR::F32 false_value); IR::U32 ConvertF32ToU32Saturated(IR::F32 value, float upper_bound, float safe_upper, diff --git a/src/graphics/shader/recompiler/ir/opcodes/ValueOpcodes.inc b/src/graphics/shader/recompiler/ir/opcodes/ValueOpcodes.inc index 7a840876c..500a5baa0 100644 --- a/src/graphics/shader/recompiler/ir/opcodes/ValueOpcodes.inc +++ b/src/graphics/shader/recompiler/ir/opcodes/ValueOpcodes.inc @@ -211,6 +211,7 @@ VALUE_OPCODE(ReadFirstLane, U32, U32, U1) VALUE_OPCODE(ReadLane, U32, U32, U32) VALUE_OPCODE(WriteLane, U32, U32, U32, U32) VALUE_OPCODE(Permlane16U32, U32, U32, U32, U32, U1) +VALUE_OPCODE(PermuteU32, U32, U32, U32, U1) VALUE_OPCODE(BpermuteU32, U32, U32, U32, U1) // Typed shader resources and memory operations. Descriptor state is represented by SSA values; diff --git a/tests/ShaderRecompilerComputeTests.cpp b/tests/ShaderRecompilerComputeTests.cpp index 9e6bfcad4..82c9e9445 100644 --- a/tests/ShaderRecompilerComputeTests.cpp +++ b/tests/ShaderRecompilerComputeTests.cpp @@ -17980,6 +17980,7 @@ CoverageClass ClassifyOpcode(ShaderOpcode opcode, case Opcode::DS_XOR_RTN_B32: case Opcode::DS_WRXCHG_RTN_B32: case Opcode::DS_SWIZZLE_B32: + case Opcode::DS_PERMUTE_B32: case Opcode::DS_BPERMUTE_B32: case Opcode::DS_READ_I8: case Opcode::DS_READ_U8: @@ -29219,6 +29220,88 @@ TestCase DsSwizzleInvalidSourceLaneZero() { return test; } +TestCase DsPermuteCapturedExecOffsetAndWrap() { + using O = ShaderOpcode; + + std::vector code; + code.push_back(EncodeVop2(0x1a, 30, InlineU32(2), 0)); + AppendBufferLoadDword(&code, 10, 30); + AppendVMovLiteral(&code, 1, 0xdeadbeefu); + AppendVMovU32(&code, 9, 100); + code.push_back(EncodeVop2(0x25, 9, Vgpr(0), 9)); + code.push_back(EncodeSop1(0x04, 4, 126)); + code.push_back(EncodeVop2(0x1b, 18, InlineU32(1), 0)); + code.push_back(EncodeVopc(0xc2, InlineU32(1), 18)); + code.push_back(EncodeSop1(0x04, 126, 106)); + code.push_back(0xdac80000u); + code.push_back(0x0100090au); // Captured SH2 instruction: ds_permute_b32 v1, v10, v9 + code.push_back(EncodeSop1(0x04, 126, 4)); + AppendStoreVgprAtLaneDwordOffset(&code, 1, 0, 4); + + code.push_back(0xdac80000u); + code.push_back(0x0100090au); + AppendStoreVgprAtLaneDwordOffset(&code, 1, 0, 8); + AppendVMovU32(&code, 10, 127); + code.push_back(EncodeDs0(0xb2, 1)); + code.push_back(EncodeDs1(1, 9, 10)); + AppendStoreVgprAtLaneDwordOffset(&code, 1, 0, 12); + AppendEnd(&code); + + TestCase test; + test.name = "DsPermuteCapturedExecOffsetAndWrap"; + test.code = code; + test.initial = {0, 0, 12, 4, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0}; + test.expected = {0, 0, 12, 4, 0xdeadbeefu, 103, 0xdeadbeefu, 0, + 101, 103, 0, 102, 103, 0, 0, 0}; + test.opcodes = {O::V_LSHLREV_B32, O::BUFFER_LOAD_DWORD, O::V_MOV_B32, + O::V_ADD_NC_U32, O::V_AND_B32, O::V_CMP_EQ_U32, O::S_MOV_B64, + O::DS_PERMUTE_B32, O::BUFFER_STORE_DWORD, O::S_ENDPGM}; + test.required_spirv = {"OpGroupNonUniformShuffle", "OpGroupNonUniformBallot"}; + test.forbidden_spirv = {" Workgroup", "OpControlBarrier", "OpMemoryBarrier"}; + test.decoded_counts = {{"DS_PERMUTE_B32", 3}}; + test.ir_counts = {{"PermuteU32", 3}}; + test.compute_info.threads_num[0] = 4; + test.compute_info.threads_num[1] = 1; + test.compute_info.threads_num[2] = 1; + test.compute_info.wave_size = 32; + test.compute_info.thread_ids_num = 1; + test.has_compute_info = true; + return test; +} + +TestCase DsPermuteWave64UsesIndependentHalves() { + using O = ShaderOpcode; + + std::vector code; + AppendVMovU32(&code, 9, 100); + code.push_back(EncodeVop2(0x25, 9, Vgpr(0), 9)); + code.push_back(EncodeVop2(0x1d, 10, InlineU32(31), 0)); + code.push_back(EncodeVop2(0x1a, 10, InlineU32(2), 10)); + code.push_back(0xdac80000u); + code.push_back(0x0100090au); + AppendStoreVgprAtLaneDwordOffset(&code, 1, 0, 0); + AppendEnd(&code); + + TestCase test; + test.name = "DsPermuteWave64UsesIndependentHalves"; + test.code = code; + test.initial = std::vector(64, 0); + for (u32 lane = 0; lane < 64; ++lane) { + test.expected.push_back(100 + (lane ^ 31)); + } + test.opcodes = {O::V_MOV_B32, O::V_ADD_NC_U32, O::V_XOR_B32, O::V_LSHLREV_B32, + O::DS_PERMUTE_B32, O::BUFFER_STORE_DWORD, O::S_ENDPGM}; + test.required_spirv = {"OpGroupNonUniformShuffle", "OpGroupNonUniformBallot"}; + test.forbidden_spirv = {" Workgroup", "OpControlBarrier", "OpMemoryBarrier"}; + test.compute_info.threads_num[0] = 64; + test.compute_info.threads_num[1] = 1; + test.compute_info.threads_num[2] = 1; + test.compute_info.wave_size = 64; + test.compute_info.thread_ids_num = 1; + test.has_compute_info = true; + return test; +} + TestCase DsBpermuteCapturedExecOffsetAndWrap() { using O = ShaderOpcode; @@ -33082,6 +33165,8 @@ std::vector MakeCases() { AddCase(DsMiscVariants); AddCase(DsFloatMinMaxUsesSeparateCompareOperand); AddCase(DsSwizzleInvalidSourceLaneZero); + AddCase(DsPermuteCapturedExecOffsetAndWrap); + AddCase(DsPermuteWave64UsesIndependentHalves); AddCase(DsBpermuteCapturedExecOffsetAndWrap); AddCase(DsBpermuteWave64UsesIndependentHalves); AddCase(Wave64CrossHalfLaneAndLds); @@ -38314,6 +38399,7 @@ int main(int argc, char **argv) { CheckWave64WholeWaveResults(); VulkanHarness vulkan; RunCase(&vulkan, Wave32VccMasksPreserveOtherHalf()); + RunCase(&vulkan, DsPermuteWave64UsesIndependentHalves()); RunCase(&vulkan, DsBpermuteWave64UsesIndependentHalves()); RunCase(&vulkan, Wave64CrossHalfLaneAndLds()); RunCase(&vulkan, Wave64RawMasksAndScalarBranch());