From bfc8cd0e0b4989f8209a8e348b1b4b5ec109585a Mon Sep 17 00:00:00 2001 From: nmzik Date: Fri, 18 Sep 2026 00:11:16 +0400 Subject: [PATCH] shader: DPP8 moves Co-authored-by: foufouadi Co-authored-by: fxpw --- .../backend/spirv/spirvEmitterFlow.cpp | 4 +- .../backend/spirv/spirvEmitterHelpers.cpp | 28 +++++--- .../backend/spirv/spirvEmitterInternal.h | 5 +- .../frontend/decode/ShaderDecoder.cpp | 5 +- .../frontend/decode/ShaderDecoder.h | 1 + .../frontend/decode/VectorAluOps.cpp | 16 +++-- .../recompiler/frontend/translate/Memory.cpp | 1 + .../frontend/translate/Translate.cpp | 34 +++++----- .../recompiler/ir/opcodes/ValueOpcodes.h | 7 +- tests/ShaderRecompilerComputeTests.cpp | 68 +++++++++++++++++++ 10 files changed, 127 insertions(+), 42 deletions(-) diff --git a/src/graphics/shader/recompiler/backend/spirv/spirvEmitterFlow.cpp b/src/graphics/shader/recompiler/backend/spirv/spirvEmitterFlow.cpp index 986f211e1..56a493c85 100644 --- a/src/graphics/shader/recompiler/backend/spirv/spirvEmitterFlow.cpp +++ b/src/graphics/shader/recompiler/backend/spirv/spirvEmitterFlow.cpp @@ -138,7 +138,7 @@ uint32_t EmitDppWriteCondition(ValueEmitContext& ctx, const IR::DppMoveFlags& fl state.builder.AddFunction(spv::OpLogicalAnd, TypeBool(state), masks_ok, bank_ok, row_ok); uint32_t writable = masks_ok; if (!flags.bound_control) { - const auto target = EmitDppTargetLane(state, flags.control); + const auto target = EmitDppTargetLane(state, flags); const auto bounded = state.builder.AllocateId(); state.builder.AddFunction(spv::OpLogicalAnd, TypeBool(state), bounded, writable, target.valid); @@ -611,7 +611,7 @@ uint32_t EmitUndefU1(EmitterState& state, const IR::Inst& inst) { uint32_t EmitDppMoveU32(ValueEmitContext& ctx, const IR::Inst& inst) { auto& state = ctx.state; const auto flags = inst.Flags(); - const auto target = EmitDppTargetLane(state, flags.control); + const auto target = EmitDppTargetLane(state, flags); const auto shuffled = ctx.Shuffle(inst, 0, target.lane); if (flags.fetch_inactive) { return shuffled; diff --git a/src/graphics/shader/recompiler/backend/spirv/spirvEmitterHelpers.cpp b/src/graphics/shader/recompiler/backend/spirv/spirvEmitterHelpers.cpp index 6640a5974..29db097a7 100644 --- a/src/graphics/shader/recompiler/backend/spirv/spirvEmitterHelpers.cpp +++ b/src/graphics/shader/recompiler/backend/spirv/spirvEmitterHelpers.cpp @@ -2,24 +2,26 @@ namespace Libs::Graphics::ShaderRecompiler::Spirv::Emitter { -DppTargetLane EmitDppQuadPermTargetLane(EmitterState& state, uint32_t subid, uint32_t control) { - const auto quad_base = state.builder.AllocateId(); +DppTargetLane EmitDppPermTargetLane(EmitterState& state, uint32_t subid, uint32_t control, + uint32_t lane_bits) { + const auto lane_mask = (1u << lane_bits) - 1u; + const auto group_base = state.builder.AllocateId(); const auto lane = state.builder.AllocateId(); const auto shift = state.builder.AllocateId(); const auto selected0 = state.builder.AllocateId(); const auto selected = state.builder.AllocateId(); const auto target = state.builder.AllocateId(); - state.builder.AddFunction(spv::OpBitwiseAnd, TypeU32(state), quad_base, subid, - ConstantU32(state, 0xfffffffcu)); + state.builder.AddFunction(spv::OpBitwiseAnd, TypeU32(state), group_base, subid, + ConstantU32(state, ~lane_mask)); state.builder.AddFunction(spv::OpBitwiseAnd, TypeU32(state), lane, subid, - ConstantU32(state, 3)); - state.builder.AddFunction(spv::OpShiftLeftLogical, TypeU32(state), shift, lane, - ConstantU32(state, 1)); + ConstantU32(state, lane_mask)); + state.builder.AddFunction(spv::OpIMul, TypeU32(state), shift, lane, + ConstantU32(state, lane_bits)); state.builder.AddFunction(spv::OpShiftRightLogical, TypeU32(state), selected0, ConstantU32(state, control), shift); state.builder.AddFunction(spv::OpBitwiseAnd, TypeU32(state), selected, selected0, - ConstantU32(state, 3)); - state.builder.AddFunction(spv::OpBitwiseOr, TypeU32(state), target, quad_base, selected); + ConstantU32(state, lane_mask)); + state.builder.AddFunction(spv::OpBitwiseOr, TypeU32(state), target, group_base, selected); return {target, ConstantBool(state, true)}; } @@ -89,10 +91,14 @@ DppTargetLane EmitDppMirrorTargetLane(EmitterState& state, uint32_t subid, bool return {target, ConstantBool(state, true)}; } -DppTargetLane EmitDppTargetLane(EmitterState& state, uint32_t control) { +DppTargetLane EmitDppTargetLane(EmitterState& state, const IR::DppMoveFlags& flags) { const auto subid = EmitSubgroupLocalInvocationId(state); + const auto control = flags.control; + if (flags.dpp8) { + return EmitDppPermTargetLane(state, subid, control, 3u); + } if (control <= 0xffu) { - return EmitDppQuadPermTargetLane(state, subid, control); + return EmitDppPermTargetLane(state, subid, control, 2u); } if (control >= 0x101u && control <= 0x10fu) { return EmitDppRowShiftTargetLane(state, subid, control & 0xfu, true); diff --git a/src/graphics/shader/recompiler/backend/spirv/spirvEmitterInternal.h b/src/graphics/shader/recompiler/backend/spirv/spirvEmitterInternal.h index 8af6afcf1..546f483df 100644 --- a/src/graphics/shader/recompiler/backend/spirv/spirvEmitterInternal.h +++ b/src/graphics/shader/recompiler/backend/spirv/spirvEmitterInternal.h @@ -327,7 +327,8 @@ void EmitMeshAllocate(ValueEmitContext& ctx, const IR::Inst& inst); uint32_t MeshOutputPointer(EmitterState& state, IR::StageOutputKind kind, uint32_t index = 0); uint32_t MeshPrimitivePointer(EmitterState& state); -DppTargetLane EmitDppQuadPermTargetLane(EmitterState& state, uint32_t subid, uint32_t control); +DppTargetLane EmitDppPermTargetLane(EmitterState& state, uint32_t subid, uint32_t control, + uint32_t lane_bits); DppTargetLane EmitDppRowShiftTargetLane(EmitterState& state, uint32_t subid, uint32_t amount, bool left); @@ -336,7 +337,7 @@ DppTargetLane EmitDppRowRotateRightTargetLane(EmitterState& state, uint32_t subi DppTargetLane EmitDppMirrorTargetLane(EmitterState& state, uint32_t subid, bool half_row); -DppTargetLane EmitDppTargetLane(EmitterState& state, uint32_t control); +DppTargetLane EmitDppTargetLane(EmitterState& state, const IR::DppMoveFlags& flags); uint32_t EmitSubgroupLocalInvocationId(EmitterState& state); diff --git a/src/graphics/shader/recompiler/frontend/decode/ShaderDecoder.cpp b/src/graphics/shader/recompiler/frontend/decode/ShaderDecoder.cpp index 4879f45ab..3d242d65f 100644 --- a/src/graphics/shader/recompiler/frontend/decode/ShaderDecoder.cpp +++ b/src/graphics/shader/recompiler/frontend/decode/ShaderDecoder.cpp @@ -472,8 +472,9 @@ std::string OperandToString(const Operand& operand) { text += ".clamp"; } if (operand.dpp) { - text += fmt::format(".dpp(ctrl=0x{:x},fi={},bc={})", operand.dpp_ctrl, - operand.dpp_fetch_inactive ? 1u : 0u, operand.dpp_bound_ctrl ? 1u : 0u); + text += fmt::format(".{}(ctrl=0x{:x},fi={},bc={})", operand.dpp8 ? "dpp8" : "dpp", + operand.dpp_ctrl, operand.dpp_fetch_inactive ? 1u : 0u, + operand.dpp_bound_ctrl ? 1u : 0u); } return text; } diff --git a/src/graphics/shader/recompiler/frontend/decode/ShaderDecoder.h b/src/graphics/shader/recompiler/frontend/decode/ShaderDecoder.h index aa646fe00..762eacf11 100644 --- a/src/graphics/shader/recompiler/frontend/decode/ShaderDecoder.h +++ b/src/graphics/shader/recompiler/frontend/decode/ShaderDecoder.h @@ -667,6 +667,7 @@ struct Operand { bool absolute = false; bool clamp = false; bool dpp = false; + bool dpp8 = false; }; struct Instruction { diff --git a/src/graphics/shader/recompiler/frontend/decode/VectorAluOps.cpp b/src/graphics/shader/recompiler/frontend/decode/VectorAluOps.cpp index cbeb0fe8b..de16ce1a4 100644 --- a/src/graphics/shader/recompiler/frontend/decode/VectorAluOps.cpp +++ b/src/graphics/shader/recompiler/frontend/decode/VectorAluOps.cpp @@ -682,10 +682,15 @@ void DecodeVop1Sdwa(uint32_t pc, std::span code, uint32_t word_i ReadLiteralOperands(code, word_index, inst); } -void ApplyDppModifier(Operand& operand, uint32_t modifier) { +void ApplyDppModifier(Operand& operand, uint32_t modifier, uint32_t encoding) { + operand.dpp = true; + if (encoding == 233u) { + operand.dpp8 = true; + operand.dpp_ctrl = modifier >> 8u; + return; + } operand.negate = ((modifier >> 20u) & 0x1u) != 0u; operand.absolute = ((modifier >> 21u) & 0x1u) != 0u; - operand.dpp = true; operand.dpp_ctrl = (modifier >> 8u) & 0x1ffu; operand.dpp_fetch_inactive = ((modifier >> 18u) & 0x1u) != 0u; operand.dpp_bound_ctrl = ((modifier >> 19u) & 0x1u) != 0u; @@ -706,7 +711,7 @@ void DecodeVop1Dpp(uint32_t pc, std::span code, uint32_t word_in DecodeVectorGpr(vdst, inst.dst); } DecodeScalarSource(src0 + 256u, pc, inst.src0); - ApplyDppModifier(inst.src0, modifier); + ApplyDppModifier(inst.src0, modifier, code[word_index] & 0x1ffu); inst.src_count = 1; if (!IsVop1FloatSourceOpcode(inst.opcode) && (inst.src0.negate || inst.src0.absolute)) { @@ -1044,7 +1049,7 @@ void DecodeVop2Dpp(uint32_t pc, std::span code, uint32_t word_in DecodeVectorGpr(vdst, inst.dst); DecodeVectorGpr(vsrc1, inst.src1); DecodeScalarSource(src0 + 256u, pc, inst.src0); - ApplyDppModifier(inst.src0, modifier); + ApplyDppModifier(inst.src0, modifier, code[word_index] & 0x1ffu); inst.src1.negate = ((modifier >> 22u) & 0x1u) != 0u; inst.src1.absolute = ((modifier >> 23u) & 0x1u) != 0u; const bool packed_fmac = inst.opcode == Opcode::V_PK_FMAC_F16; @@ -1153,7 +1158,7 @@ void DecodeVopcDpp(uint32_t pc, std::span code, uint32_t word_in DecodeVectorGpr(vsrc1, inst.src1); DecodeScalarSource(src0 + 256u, pc, inst.src0); inst.dst.kind = IsVopcCompareExec(inst.opcode) ? OperandKind::ExecLo : OperandKind::VccLo; - ApplyDppModifier(inst.src0, modifier); + ApplyDppModifier(inst.src0, modifier, code[word_index] & 0x1ffu); inst.src1.negate = ((modifier >> 22u) & 0x1u) != 0u; inst.src1.absolute = ((modifier >> 23u) & 0x1u) != 0u; inst.src_count = 2; @@ -1508,6 +1513,7 @@ void DecodeVop1(uint32_t pc, std::span code, uint32_t word_index return; } switch (src0) { + case 233u: DecodeVop1Dpp(pc, code, word_index, opcode, vdst, inst); return; case 249u: DecodeVop1Sdwa(pc, code, word_index, opcode, vdst, inst); return; case 250u: DecodeVop1Dpp(pc, code, word_index, opcode, vdst, inst); return; default: break; diff --git a/src/graphics/shader/recompiler/frontend/translate/Memory.cpp b/src/graphics/shader/recompiler/frontend/translate/Memory.cpp index de40e4281..df033162a 100644 --- a/src/graphics/shader/recompiler/frontend/translate/Memory.cpp +++ b/src/graphics/shader/recompiler/frontend/translate/Memory.cpp @@ -34,6 +34,7 @@ Decoder::Operand OffsetDecodedRegister(const Decoder::Operand& operand, uint32_t result.negate_hi = false; result.absolute = false; result.dpp_ctrl = 0; + result.dpp8 = false; result.dpp_row_mask = 0xf; result.dpp_bank_mask = 0xf; result.explicit_sdwa_dst = false; diff --git a/src/graphics/shader/recompiler/frontend/translate/Translate.cpp b/src/graphics/shader/recompiler/frontend/translate/Translate.cpp index 81c401be5..892278c23 100644 --- a/src/graphics/shader/recompiler/frontend/translate/Translate.cpp +++ b/src/graphics/shader/recompiler/frontend/translate/Translate.cpp @@ -9,6 +9,17 @@ namespace Libs::Graphics::ShaderRecompiler::Frontend { +static IR::DppMoveFlags DppFlags(const Decoder::Operand& operand) { + return { + .control = operand.dpp_ctrl, + .row_mask = operand.dpp_row_mask, + .bank_mask = operand.dpp_bank_mask, + .fetch_inactive = operand.dpp_fetch_inactive, + .bound_control = operand.dpp_bound_ctrl, + .dpp8 = operand.dpp8, + }; +} + const Decoder::Operand& Translator::SourceAt(const Decoder::Instruction& inst, uint32_t index) { switch (index) { case 0: return inst.src0; @@ -30,6 +41,7 @@ Decoder::Operand Translator::DestinationOperand(const Decoder::Instruction& inst continue; } destination.dpp = true; + destination.dpp8 = source.dpp8; destination.dpp_ctrl = source.dpp_ctrl; destination.dpp_row_mask = source.dpp_row_mask; destination.dpp_bank_mask = source.dpp_bank_mask; @@ -106,6 +118,7 @@ Decoder::Operand Translator::PlainOperand(const Decoder::Operand& operand) { result.dpp_fetch_inactive = false; result.dpp_bound_ctrl = false; result.dpp = false; + result.dpp8 = false; return result; } @@ -167,14 +180,8 @@ IR::U32 Translator::ReadScalarCode(uint32_t code) { IR::U32 Translator::ApplyBitSourceModifiers(const Decoder::Operand& operand, IR::U32 value) { if (operand.dpp) { - const IR::DppMoveFlags flags { - .control = static_cast(operand.dpp_ctrl), - .row_mask = static_cast(operand.dpp_row_mask), - .bank_mask = static_cast(operand.dpp_bank_mask), - .fetch_inactive = operand.dpp_fetch_inactive, - .bound_control = operand.dpp_bound_ctrl, - }; - value = IR::U32(ir.Emit(IR::ValueOpcode::DppMoveU32, {value, ir.GetExec()}, flags)); + value = + IR::U32(ir.Emit(IR::ValueOpcode::DppMoveU32, {value, ir.GetExec()}, DppFlags(operand))); } if (operand.sdwa_sel != 6u) { uint32_t offset = 0; @@ -301,15 +308,8 @@ void Translator::WriteRawU32(const Decoder::Operand& operand, IR::U32 value) { const auto reg = static_cast(operand.reg); const auto old = ir.GetVectorReg(reg); if (operand.dpp) { - const IR::DppMoveFlags flags { - .control = static_cast(operand.dpp_ctrl), - .row_mask = static_cast(operand.dpp_row_mask), - .bank_mask = static_cast(operand.dpp_bank_mask), - .fetch_inactive = operand.dpp_fetch_inactive, - .bound_control = operand.dpp_bound_ctrl, - }; - value = IR::U32( - ir.Emit(IR::ValueOpcode::DppUpdateU32, {value, old, ir.GetExec()}, flags)); + value = IR::U32(ir.Emit(IR::ValueOpcode::DppUpdateU32, {value, old, ir.GetExec()}, + DppFlags(operand))); } else { value = ir.Select(ir.GetExec(), value, old); } diff --git a/src/graphics/shader/recompiler/ir/opcodes/ValueOpcodes.h b/src/graphics/shader/recompiler/ir/opcodes/ValueOpcodes.h index ae3c1dbc7..c5da6bef9 100644 --- a/src/graphics/shader/recompiler/ir/opcodes/ValueOpcodes.h +++ b/src/graphics/shader/recompiler/ir/opcodes/ValueOpcodes.h @@ -34,11 +34,12 @@ struct ImageOpcodeInfo { }; struct DppMoveFlags { - uint16_t control = 0; - uint8_t row_mask = 0xf; - uint8_t bank_mask = 0xf; + uint32_t control : 24 = 0; + uint32_t row_mask : 4 = 0xf; + uint32_t bank_mask : 4 = 0xf; bool fetch_inactive = false; bool bound_control = false; + bool dpp8 = false; }; static_assert(sizeof(DppMoveFlags) <= sizeof(uint64_t)); static_assert(std::is_trivially_copyable_v); diff --git a/tests/ShaderRecompilerComputeTests.cpp b/tests/ShaderRecompilerComputeTests.cpp index e988c6bdb..82770b912 100644 --- a/tests/ShaderRecompilerComputeTests.cpp +++ b/tests/ShaderRecompilerComputeTests.cpp @@ -19697,6 +19697,62 @@ TestCase VectorPermlane16FetchInactiveFi() { return test; } +TestCase VectorDpp8Captured(bool masked_exec) { + using O = ShaderOpcode; + constexpr u32 sentinel = 0xaaaaaaaau; + const u32 masks[] = {masked_exec ? 0x0f0f0f0fu : 0xffffffffu, + masked_exec ? 0xf0f0f0f0u : 0xffffffffu}; + std::vector code; + AppendVMovU32(&code, 5, 100); + code.push_back(EncodeVop2(0x25, 5, Vgpr(0), 5)); + AppendVMovLiteral(&code, 2, sentinel); + AppendVMovLiteral(&code, 3, sentinel); + AppendSMovLiteral(&code, 126, masks[0]); + AppendSMovLiteral(&code, 127, masks[1]); + // Captured DPP8 selectors broadcast each group's upper/lower four lanes. + code.push_back(EncodeVop1(0x01, 2, 233)); + code.push_back(0xfacfac05u); + code.push_back(EncodeVop1(0x01, 3, 233)); + code.push_back(0x68868805u); + AppendSMovLiteral(&code, 126, 0xffffffffu); + AppendSMovLiteral(&code, 127, 0xffffffffu); + code.push_back(EncodeVop2(0x1a, 6, InlineU32(2), 0)); + AppendBufferStoreDword(&code, 2, 6); + AppendVMovU32(&code, 7, 64u * sizeof(u32)); + code.push_back(EncodeVop2(0x25, 6, Vgpr(7), 6)); + AppendBufferStoreDword(&code, 3, 6); + AppendEnd(&code); + + TestCase test; + test.name = masked_exec ? "VectorDpp8CapturedMaskedExec" + : "VectorDpp8Captured"; + test.code = std::move(code); + test.expected.resize(128, sentinel); + for (u32 lane = 0; lane < 64; ++lane) { + const auto mask = masks[lane / 32u]; + if ((mask & (1u << (lane % 32u))) == 0) { + continue; + } + const u32 sources[] = {(lane & ~7u) | 4u | (lane & 3u), + (lane & ~7u) | (lane & 3u)}; + for (u32 permutation = 0; permutation < 2; ++permutation) { + const auto source = sources[permutation]; + test.expected[permutation * 64u + lane] = + (mask & (1u << (source % 32u))) != 0 ? 100u + source : 0u; + } + } + test.opcodes = {O::V_MOV_B32, O::V_ADD_NC_U32, O::S_MOV_B32, + O::V_LSHLREV_B32, O::BUFFER_STORE_DWORD, O::S_ENDPGM}; + test.decoded_counts = {{".dpp8(", 2}}; + test.compute_info.wave_size = 64; + test.compute_info.threads_num[0] = 64; + test.compute_info.threads_num[1] = 1; + test.compute_info.threads_num[2] = 1; + test.compute_info.thread_ids_num = 1; + test.has_compute_info = true; + return test; +} + TestCase VectorDppQuadPermuteReverse() { using O = ShaderOpcode; @@ -26871,6 +26927,8 @@ std::vector MakeCases() { AddCase(VectorPermlanex16); AddCase(VectorPermlane16FetchInactiveZero); AddCase(VectorPermlane16FetchInactiveFi); + cases.push_back(VectorDpp8Captured(false)); + cases.push_back(VectorDpp8Captured(true)); AddCase(VectorDppQuadPermuteReverse); AddCase(VectorDppRowXmask); AddCase(VectorDppBankMaskPreservesDestination); @@ -31570,6 +31628,16 @@ int main(int argc, char **argv) { std::setvbuf(stdout, nullptr, _IONBF, 0); EnsureConfigInitialized(); CheckLeastRecentlyUsedCacheOrdering(); + if (argc == 2 && std::strcmp(argv[1], "--dpp-only") == 0) { + VulkanHarness vulkan; + RunCase(&vulkan, VectorDpp8Captured(false)); + RunCase(&vulkan, VectorDpp8Captured(true)); + RunCase(&vulkan, VectorDppQuadPermuteReverse()); + RunCase(&vulkan, VectorDppRowXmask()); + RunCase(&vulkan, VectorDppBankMaskPreservesDestination()); + RunCase(&vulkan, VectorDppBoundsControlZeroPreservesDestination()); + return 0; + } if (argc == 2 && std::strcmp(argv[1], "--buffer-snorm-store-only") == 0) { VulkanHarness vulkan; RunCase(&vulkan, BufferStoreFormatXyzwSnorm16CapturedSkinningVectors());