shader: DPP8 moves

Co-authored-by: foufouadi <attar.foued@gmail.com>
Co-authored-by: fxpw <fxpw@outlook.com>

src/graphics/shader/recompiler/backend/spirv/spirvEmitterFlow.cpp

@@ -138,7 +138,7 @@ uint32_t EmitDppWriteCondition(ValueEmitContext& ctx, const IR::DppMoveFlags& fl
 	state.builder.AddFunction(spv::OpLogicalAnd, TypeBool(state), masks_ok, bank_ok, row_ok);
 	uint32_t writable = masks_ok;
 	if (!flags.bound_control) {
-		const auto target  = EmitDppTargetLane(state, flags.control);
+		const auto target  = EmitDppTargetLane(state, flags);
 		const auto bounded = state.builder.AllocateId();
 		state.builder.AddFunction(spv::OpLogicalAnd, TypeBool(state), bounded, writable,
 		                          target.valid);
@@ -611,7 +611,7 @@ uint32_t EmitUndefU1(EmitterState& state, const IR::Inst& inst) {
 uint32_t EmitDppMoveU32(ValueEmitContext& ctx, const IR::Inst& inst) {
 	auto&      state    = ctx.state;
 	const auto flags    = inst.Flags<IR::DppMoveFlags>();
-	const auto target   = EmitDppTargetLane(state, flags.control);
+	const auto target   = EmitDppTargetLane(state, flags);
 	const auto shuffled = ctx.Shuffle(inst, 0, target.lane);
 	if (flags.fetch_inactive) {
 		return shuffled;

src/graphics/shader/recompiler/backend/spirv/spirvEmitterHelpers.cpp

@@ -2,24 +2,26 @@
 
 namespace Libs::Graphics::ShaderRecompiler::Spirv::Emitter {
 
-DppTargetLane EmitDppQuadPermTargetLane(EmitterState& state, uint32_t subid, uint32_t control) {
+DppTargetLane EmitDppPermTargetLane(EmitterState& state, uint32_t subid, uint32_t control,
-	const auto quad_base = state.builder.AllocateId();
+                                    uint32_t lane_bits) {
+	const auto lane_mask  = (1u << lane_bits) - 1u;
+	const auto group_base = state.builder.AllocateId();
 	const auto lane      = state.builder.AllocateId();
 	const auto shift     = state.builder.AllocateId();
 	const auto selected0 = state.builder.AllocateId();
 	const auto selected  = state.builder.AllocateId();
 	const auto target    = state.builder.AllocateId();
-	state.builder.AddFunction(spv::OpBitwiseAnd, TypeU32(state), quad_base, subid,
+	state.builder.AddFunction(spv::OpBitwiseAnd, TypeU32(state), group_base, subid,
-	                          ConstantU32(state, 0xfffffffcu));
+	                          ConstantU32(state, ~lane_mask));
 	state.builder.AddFunction(spv::OpBitwiseAnd, TypeU32(state), lane, subid,
-	                          ConstantU32(state, 3));
+	                          ConstantU32(state, lane_mask));
-	state.builder.AddFunction(spv::OpShiftLeftLogical, TypeU32(state), shift, lane,
+	state.builder.AddFunction(spv::OpIMul, TypeU32(state), shift, lane,
-	                          ConstantU32(state, 1));
+	                          ConstantU32(state, lane_bits));
 	state.builder.AddFunction(spv::OpShiftRightLogical, TypeU32(state), selected0,
 	                          ConstantU32(state, control), shift);
 	state.builder.AddFunction(spv::OpBitwiseAnd, TypeU32(state), selected, selected0,
-	                          ConstantU32(state, 3));
+	                          ConstantU32(state, lane_mask));
-	state.builder.AddFunction(spv::OpBitwiseOr, TypeU32(state), target, quad_base, selected);
+	state.builder.AddFunction(spv::OpBitwiseOr, TypeU32(state), target, group_base, selected);
 	return {target, ConstantBool(state, true)};
 }
 
@@ -89,10 +91,14 @@ DppTargetLane EmitDppMirrorTargetLane(EmitterState& state, uint32_t subid, bool
 	return {target, ConstantBool(state, true)};
 }
 
-DppTargetLane EmitDppTargetLane(EmitterState& state, uint32_t control) {
+DppTargetLane EmitDppTargetLane(EmitterState& state, const IR::DppMoveFlags& flags) {
 	const auto subid = EmitSubgroupLocalInvocationId(state);
+	const auto control = flags.control;
+	if (flags.dpp8) {
+		return EmitDppPermTargetLane(state, subid, control, 3u);
+	}
 	if (control <= 0xffu) {
-		return EmitDppQuadPermTargetLane(state, subid, control);
+		return EmitDppPermTargetLane(state, subid, control, 2u);
 	}
 	if (control >= 0x101u && control <= 0x10fu) {
 		return EmitDppRowShiftTargetLane(state, subid, control & 0xfu, true);

src/graphics/shader/recompiler/backend/spirv/spirvEmitterInternal.h

@@ -327,7 +327,8 @@ void     EmitMeshAllocate(ValueEmitContext& ctx, const IR::Inst& inst);
 uint32_t MeshOutputPointer(EmitterState& state, IR::StageOutputKind kind, uint32_t index = 0);
 uint32_t MeshPrimitivePointer(EmitterState& state);
 
-DppTargetLane EmitDppQuadPermTargetLane(EmitterState& state, uint32_t subid, uint32_t control);
+DppTargetLane EmitDppPermTargetLane(EmitterState& state, uint32_t subid, uint32_t control,
+                                    uint32_t lane_bits);
 
 DppTargetLane EmitDppRowShiftTargetLane(EmitterState& state, uint32_t subid, uint32_t amount,
                                         bool left);
@@ -336,7 +337,7 @@ DppTargetLane EmitDppRowRotateRightTargetLane(EmitterState& state, uint32_t subi
 
 DppTargetLane EmitDppMirrorTargetLane(EmitterState& state, uint32_t subid, bool half_row);
 
-DppTargetLane EmitDppTargetLane(EmitterState& state, uint32_t control);
+DppTargetLane EmitDppTargetLane(EmitterState& state, const IR::DppMoveFlags& flags);
 
 uint32_t EmitSubgroupLocalInvocationId(EmitterState& state);
 

src/graphics/shader/recompiler/frontend/decode/ShaderDecoder.cpp

@@ -472,8 +472,9 @@ std::string OperandToString(const Operand& operand) {
 		text += ".clamp";
 	}
 	if (operand.dpp) {
-		text += fmt::format(".dpp(ctrl=0x{:x},fi={},bc={})", operand.dpp_ctrl,
+		text += fmt::format(".{}(ctrl=0x{:x},fi={},bc={})", operand.dpp8 ? "dpp8" : "dpp",
-		                    operand.dpp_fetch_inactive ? 1u : 0u, operand.dpp_bound_ctrl ? 1u : 0u);
+		                    operand.dpp_ctrl, operand.dpp_fetch_inactive ? 1u : 0u,
+		                    operand.dpp_bound_ctrl ? 1u : 0u);
 	}
 	return text;
 }

src/graphics/shader/recompiler/frontend/decode/ShaderDecoder.h

@@ -667,6 +667,7 @@ struct Operand {
 	bool     absolute           = false;
 	bool     clamp              = false;
 	bool     dpp                = false;
+	bool     dpp8               = false;
 };
 
 struct Instruction {

src/graphics/shader/recompiler/frontend/decode/VectorAluOps.cpp

@@ -682,10 +682,15 @@ void DecodeVop1Sdwa(uint32_t pc, std::span<const uint32_t> code, uint32_t word_i
 	ReadLiteralOperands(code, word_index, inst);
 }
 
-void ApplyDppModifier(Operand& operand, uint32_t modifier) {
+void ApplyDppModifier(Operand& operand, uint32_t modifier, uint32_t encoding) {
+	operand.dpp = true;
+	if (encoding == 233u) {
+		operand.dpp8     = true;
+		operand.dpp_ctrl = modifier >> 8u;
+		return;
+	}
 	operand.negate             = ((modifier >> 20u) & 0x1u) != 0u;
 	operand.absolute           = ((modifier >> 21u) & 0x1u) != 0u;
-	operand.dpp                = true;
 	operand.dpp_ctrl           = (modifier >> 8u) & 0x1ffu;
 	operand.dpp_fetch_inactive = ((modifier >> 18u) & 0x1u) != 0u;
 	operand.dpp_bound_ctrl     = ((modifier >> 19u) & 0x1u) != 0u;
@@ -706,7 +711,7 @@ void DecodeVop1Dpp(uint32_t pc, std::span<const uint32_t> code, uint32_t word_in
 		DecodeVectorGpr(vdst, inst.dst);
 	}
 	DecodeScalarSource(src0 + 256u, pc, inst.src0);
-	ApplyDppModifier(inst.src0, modifier);
+	ApplyDppModifier(inst.src0, modifier, code[word_index] & 0x1ffu);
 	inst.src_count = 1;
 
 	if (!IsVop1FloatSourceOpcode(inst.opcode) && (inst.src0.negate || inst.src0.absolute)) {
@@ -1044,7 +1049,7 @@ void DecodeVop2Dpp(uint32_t pc, std::span<const uint32_t> code, uint32_t word_in
 	DecodeVectorGpr(vdst, inst.dst);
 	DecodeVectorGpr(vsrc1, inst.src1);
 	DecodeScalarSource(src0 + 256u, pc, inst.src0);
-	ApplyDppModifier(inst.src0, modifier);
+	ApplyDppModifier(inst.src0, modifier, code[word_index] & 0x1ffu);
 	inst.src1.negate       = ((modifier >> 22u) & 0x1u) != 0u;
 	inst.src1.absolute     = ((modifier >> 23u) & 0x1u) != 0u;
 	const bool packed_fmac = inst.opcode == Opcode::V_PK_FMAC_F16;
@@ -1153,7 +1158,7 @@ void DecodeVopcDpp(uint32_t pc, std::span<const uint32_t> code, uint32_t word_in
 	DecodeVectorGpr(vsrc1, inst.src1);
 	DecodeScalarSource(src0 + 256u, pc, inst.src0);
 	inst.dst.kind = IsVopcCompareExec(inst.opcode) ? OperandKind::ExecLo : OperandKind::VccLo;
-	ApplyDppModifier(inst.src0, modifier);
+	ApplyDppModifier(inst.src0, modifier, code[word_index] & 0x1ffu);
 	inst.src1.negate   = ((modifier >> 22u) & 0x1u) != 0u;
 	inst.src1.absolute = ((modifier >> 23u) & 0x1u) != 0u;
 	inst.src_count     = 2;
@@ -1508,6 +1513,7 @@ void DecodeVop1(uint32_t pc, std::span<const uint32_t> code, uint32_t word_index
 		return;
 	}
 	switch (src0) {
+		case 233u: DecodeVop1Dpp(pc, code, word_index, opcode, vdst, inst); return;
 		case 249u: DecodeVop1Sdwa(pc, code, word_index, opcode, vdst, inst); return;
 		case 250u: DecodeVop1Dpp(pc, code, word_index, opcode, vdst, inst); return;
 		default: break;

src/graphics/shader/recompiler/frontend/translate/Memory.cpp

@@ -34,6 +34,7 @@ Decoder::Operand OffsetDecodedRegister(const Decoder::Operand& operand, uint32_t
 	result.negate_hi          = false;
 	result.absolute           = false;
 	result.dpp_ctrl           = 0;
+	result.dpp8               = false;
 	result.dpp_row_mask       = 0xf;
 	result.dpp_bank_mask      = 0xf;
 	result.explicit_sdwa_dst  = false;

src/graphics/shader/recompiler/frontend/translate/Translate.cpp

@@ -9,6 +9,17 @@
 
 namespace Libs::Graphics::ShaderRecompiler::Frontend {
 
+static IR::DppMoveFlags DppFlags(const Decoder::Operand& operand) {
+	return {
+	    .control        = operand.dpp_ctrl,
+	    .row_mask       = operand.dpp_row_mask,
+	    .bank_mask      = operand.dpp_bank_mask,
+	    .fetch_inactive = operand.dpp_fetch_inactive,
+	    .bound_control  = operand.dpp_bound_ctrl,
+	    .dpp8           = operand.dpp8,
+	};
+}
+
 const Decoder::Operand& Translator::SourceAt(const Decoder::Instruction& inst, uint32_t index) {
 	switch (index) {
 		case 0: return inst.src0;
@@ -30,6 +41,7 @@ Decoder::Operand Translator::DestinationOperand(const Decoder::Instruction& inst
 			continue;
 		}
 		destination.dpp                = true;
+		destination.dpp8               = source.dpp8;
 		destination.dpp_ctrl           = source.dpp_ctrl;
 		destination.dpp_row_mask       = source.dpp_row_mask;
 		destination.dpp_bank_mask      = source.dpp_bank_mask;
@@ -106,6 +118,7 @@ Decoder::Operand Translator::PlainOperand(const Decoder::Operand& operand) {
 	result.dpp_fetch_inactive = false;
 	result.dpp_bound_ctrl     = false;
 	result.dpp                = false;
+	result.dpp8               = false;
 	return result;
 }
 
@@ -167,14 +180,8 @@ IR::U32 Translator::ReadScalarCode(uint32_t code) {
 
 IR::U32 Translator::ApplyBitSourceModifiers(const Decoder::Operand& operand, IR::U32 value) {
 	if (operand.dpp) {
-		const IR::DppMoveFlags flags {
+		value =
-		    .control        = static_cast<uint16_t>(operand.dpp_ctrl),
+		    IR::U32(ir.Emit(IR::ValueOpcode::DppMoveU32, {value, ir.GetExec()}, DppFlags(operand)));
-		    .row_mask       = static_cast<uint8_t>(operand.dpp_row_mask),
-		    .bank_mask      = static_cast<uint8_t>(operand.dpp_bank_mask),
-		    .fetch_inactive = operand.dpp_fetch_inactive,
-		    .bound_control  = operand.dpp_bound_ctrl,
-		};
-		value = IR::U32(ir.Emit(IR::ValueOpcode::DppMoveU32, {value, ir.GetExec()}, flags));
 	}
 	if (operand.sdwa_sel != 6u) {
 		uint32_t offset = 0;
@@ -301,15 +308,8 @@ void Translator::WriteRawU32(const Decoder::Operand& operand, IR::U32 value) {
 			const auto reg = static_cast<IR::VectorReg>(operand.reg);
 			const auto old = ir.GetVectorReg(reg);
 			if (operand.dpp) {
-				const IR::DppMoveFlags flags {
+				value = IR::U32(ir.Emit(IR::ValueOpcode::DppUpdateU32, {value, old, ir.GetExec()},
-				    .control        = static_cast<uint16_t>(operand.dpp_ctrl),
+				                        DppFlags(operand)));
-				    .row_mask       = static_cast<uint8_t>(operand.dpp_row_mask),
-				    .bank_mask      = static_cast<uint8_t>(operand.dpp_bank_mask),
-				    .fetch_inactive = operand.dpp_fetch_inactive,
-				    .bound_control  = operand.dpp_bound_ctrl,
-				};
-				value = IR::U32(
-				    ir.Emit(IR::ValueOpcode::DppUpdateU32, {value, old, ir.GetExec()}, flags));
 			} else {
 				value = ir.Select(ir.GetExec(), value, old);
 			}

src/graphics/shader/recompiler/ir/opcodes/ValueOpcodes.h

@@ -34,11 +34,12 @@ struct ImageOpcodeInfo {
 };
 
 struct DppMoveFlags {
-	uint16_t control        = 0;
+	uint32_t control   : 24 = 0;
-	uint8_t  row_mask       = 0xf;
+	uint32_t row_mask  : 4  = 0xf;
-	uint8_t  bank_mask      = 0xf;
+	uint32_t bank_mask : 4  = 0xf;
 	bool     fetch_inactive = false;
 	bool     bound_control  = false;
+	bool     dpp8           = false;
 };
 static_assert(sizeof(DppMoveFlags) <= sizeof(uint64_t));
 static_assert(std::is_trivially_copyable_v<DppMoveFlags>);

tests/ShaderRecompilerComputeTests.cpp

@@ -19697,6 +19697,62 @@ TestCase VectorPermlane16FetchInactiveFi() {
   return test;
 }
 
+TestCase VectorDpp8Captured(bool masked_exec) {
+  using O = ShaderOpcode;
+  constexpr u32 sentinel = 0xaaaaaaaau;
+  const u32 masks[] = {masked_exec ? 0x0f0f0f0fu : 0xffffffffu,
+                      masked_exec ? 0xf0f0f0f0u : 0xffffffffu};
+  std::vector<u32> code;
+  AppendVMovU32(&code, 5, 100);
+  code.push_back(EncodeVop2(0x25, 5, Vgpr(0), 5));
+  AppendVMovLiteral(&code, 2, sentinel);
+  AppendVMovLiteral(&code, 3, sentinel);
+  AppendSMovLiteral(&code, 126, masks[0]);
+  AppendSMovLiteral(&code, 127, masks[1]);
+  // Captured DPP8 selectors broadcast each group's upper/lower four lanes.
+  code.push_back(EncodeVop1(0x01, 2, 233));
+  code.push_back(0xfacfac05u);
+  code.push_back(EncodeVop1(0x01, 3, 233));
+  code.push_back(0x68868805u);
+  AppendSMovLiteral(&code, 126, 0xffffffffu);
+  AppendSMovLiteral(&code, 127, 0xffffffffu);
+  code.push_back(EncodeVop2(0x1a, 6, InlineU32(2), 0));
+  AppendBufferStoreDword(&code, 2, 6);
+  AppendVMovU32(&code, 7, 64u * sizeof(u32));
+  code.push_back(EncodeVop2(0x25, 6, Vgpr(7), 6));
+  AppendBufferStoreDword(&code, 3, 6);
+  AppendEnd(&code);
+
+  TestCase test;
+  test.name = masked_exec ? "VectorDpp8CapturedMaskedExec"
+                          : "VectorDpp8Captured";
+  test.code = std::move(code);
+  test.expected.resize(128, sentinel);
+  for (u32 lane = 0; lane < 64; ++lane) {
+    const auto mask = masks[lane / 32u];
+    if ((mask & (1u << (lane % 32u))) == 0) {
+      continue;
+    }
+    const u32 sources[] = {(lane & ~7u) | 4u | (lane & 3u),
+                           (lane & ~7u) | (lane & 3u)};
+    for (u32 permutation = 0; permutation < 2; ++permutation) {
+      const auto source = sources[permutation];
+      test.expected[permutation * 64u + lane] =
+          (mask & (1u << (source % 32u))) != 0 ? 100u + source : 0u;
+    }
+  }
+  test.opcodes = {O::V_MOV_B32, O::V_ADD_NC_U32, O::S_MOV_B32,
+                  O::V_LSHLREV_B32, O::BUFFER_STORE_DWORD, O::S_ENDPGM};
+  test.decoded_counts = {{".dpp8(", 2}};
+  test.compute_info.wave_size = 64;
+  test.compute_info.threads_num[0] = 64;
+  test.compute_info.threads_num[1] = 1;
+  test.compute_info.threads_num[2] = 1;
+  test.compute_info.thread_ids_num = 1;
+  test.has_compute_info = true;
+  return test;
+}
+
 TestCase VectorDppQuadPermuteReverse() {
   using O = ShaderOpcode;
 
@@ -26871,6 +26927,8 @@ std::vector<TestCase> MakeCases() {
   AddCase(VectorPermlanex16);
   AddCase(VectorPermlane16FetchInactiveZero);
   AddCase(VectorPermlane16FetchInactiveFi);
+  cases.push_back(VectorDpp8Captured(false));
+  cases.push_back(VectorDpp8Captured(true));
   AddCase(VectorDppQuadPermuteReverse);
   AddCase(VectorDppRowXmask);
   AddCase(VectorDppBankMaskPreservesDestination);
@@ -31570,6 +31628,16 @@ int main(int argc, char **argv) {
   std::setvbuf(stdout, nullptr, _IONBF, 0);
   EnsureConfigInitialized();
   CheckLeastRecentlyUsedCacheOrdering();
+  if (argc == 2 && std::strcmp(argv[1], "--dpp-only") == 0) {
+    VulkanHarness vulkan;
+    RunCase(&vulkan, VectorDpp8Captured(false));
+    RunCase(&vulkan, VectorDpp8Captured(true));
+    RunCase(&vulkan, VectorDppQuadPermuteReverse());
+    RunCase(&vulkan, VectorDppRowXmask());
+    RunCase(&vulkan, VectorDppBankMaskPreservesDestination());
+    RunCase(&vulkan, VectorDppBoundsControlZeroPreservesDestination());
+    return 0;
+  }
   if (argc == 2 && std::strcmp(argv[1], "--buffer-snorm-store-only") == 0) {
     VulkanHarness vulkan;
     RunCase(&vulkan, BufferStoreFormatXyzwSnorm16CapturedSkinningVectors());