shader: add scalar zero-bit count and search operations

src/graphics/shader/recompiler/frontend/decode/ScalarAluOps.cpp

@@ -47,8 +47,12 @@ constexpr OpcodeMap SOP1_OPCODE_LIST[] = {
     {0x0au, Opcode::S_WQM_B64},
     {0x0bu, Opcode::S_BREV_B32},
     {0x0cu, Opcode::S_BREV_B64},
+    {0x0du, Opcode::S_BCNT0_I32_B32},
+    {0x0eu, Opcode::S_BCNT0_I32_B64},
     {0x0fu, Opcode::S_BCNT1_I32_B32},
     {0x10u, Opcode::S_BCNT1_I32_B64},
+    {0x11u, Opcode::S_FF0_I32_B32},
+    {0x12u, Opcode::S_FF0_I32_B64},
     {0x13u, Opcode::S_FF1_I32_B32},
     {0x14u, Opcode::S_FF1_I32_B64},
     {0x15u, Opcode::S_FLBIT_I32_B32},

src/graphics/shader/recompiler/frontend/decode/ShaderDecoder.cpp

@@ -510,6 +510,10 @@ std::string InstructionToString(const Instruction& inst) {
 		case Opcode::S_SEXT_I32_I16:
 		case Opcode::S_BREV_B32:
 		case Opcode::S_BREV_B64:
+		case Opcode::S_BCNT0_I32_B32:
+		case Opcode::S_BCNT0_I32_B64:
+		case Opcode::S_FF0_I32_B32:
+		case Opcode::S_FF0_I32_B64:
 		case Opcode::S_BCNT1_I32_B32:
 		case Opcode::S_FLBIT_I32_B32:
 		case Opcode::S_FF1_I32_B32:

src/graphics/shader/recompiler/frontend/decode/ShaderDecoder.h

@@ -50,8 +50,12 @@ enum class Opcode {
 	S_ABSDIFF_I32,
 	S_BREV_B32,
 	S_BREV_B64,
+	S_BCNT0_I32_B32,
+	S_BCNT0_I32_B64,
 	S_BCNT1_I32_B32,
 	S_BCNT1_I32_B64,
+	S_FF0_I32_B32,
+	S_FF0_I32_B64,
 	S_FF1_I32_B32,
 	S_FF1_I32_B64,
 	S_FLBIT_I32_B32,

src/graphics/shader/recompiler/frontend/translate/Integer.cpp

@@ -221,8 +221,12 @@ void Translator::V_XOR3_B32(const Decoder::Instruction& inst) {
 	WriteOperand(DestinationOperand(inst), result);
 }
 
-void Translator::S_FF1_I32_B64(const Decoder::Instruction& inst) {
+void Translator::S_FF_I32_B64(const Decoder::Instruction& inst, bool find_zero) {
-	const auto source        = ExtractU64(ReadU64(inst.src0));
+	auto source = ReadU32Pair(inst.src0);
+	if (find_zero) {
+		source[0] = ir.BitwiseNot(source[0]);
+		source[1] = ir.BitwiseNot(source[1]);
+	}
 	const auto low_lsb       = IR::U32(ir.Emit(IR::ValueOpcode::FindILsb32, {source[0]}));
 	const auto high_lsb      = IR::U32(ir.Emit(IR::ValueOpcode::FindILsb32, {source[1]}));
 	const auto high_position = ir.IAdd(high_lsb, IR::U32(IR::Value(32u)));

src/graphics/shader/recompiler/frontend/translate/Scalar.cpp

@@ -178,12 +178,29 @@ void Translator::EmitScalar(const Decoder::Instruction& inst) {
 			                        IR::U32(ir.Emit(IR::ValueOpcode::BitReverse32, {value[0]}))});
 			return;
 		}
+		case O::S_BCNT0_I32_B32:
+		case O::S_FF0_I32_B32: {
+			const auto value = ir.BitwiseNot(ReadU32(inst.src0));
+			const bool count = inst.opcode == O::S_BCNT0_I32_B32;
+			const auto result = IR::U32(ir.Emit(count ? IR::ValueOpcode::BitCount32
+			                                       : IR::ValueOpcode::FindILsb32, {value}));
+			WriteOperand(inst.dst, result);
+			if (count) {
+				ir.SetScc(ir.INotEqual(result, IR::U32(IR::Value(0u))));
+			}
+			return;
+		}
 		case O::S_BCNT1_I32_B32:
 			return SimpleInteger(inst, IR::ValueOpcode::BitCount32, IR::Type::U32, false, false,
 			                     true);
+		case O::S_BCNT0_I32_B64:
 		case O::S_BCNT1_I32_B64: {
 			// Vulkan bit counts operate on 32-bit words; avoid packing only to split again.
-			const auto value  = ReadU32Pair(inst.src0);
+			auto value = ReadU32Pair(inst.src0);
+			if (inst.opcode == O::S_BCNT0_I32_B64) {
+				value[0] = ir.BitwiseNot(value[0]);
+				value[1] = ir.BitwiseNot(value[1]);
+			}
 			const auto low    = IR::U32(ir.Emit(IR::ValueOpcode::BitCount32, {value[0]}));
 			const auto high   = IR::U32(ir.Emit(IR::ValueOpcode::BitCount32, {value[1]}));
 			const auto result = ir.IAdd(low, high);
@@ -221,7 +238,8 @@ void Translator::EmitScalar(const Decoder::Instruction& inst) {
 			return ComposedIntegerBinary(inst, IR::ValueOpcode::BitwiseOr32, false, true, true);
 		case O::S_XNOR_B32:
 			return ComposedIntegerBinary(inst, IR::ValueOpcode::BitwiseXor32, false, true, true);
-		case O::S_FF1_I32_B64: return S_FF1_I32_B64(inst);
+		case O::S_FF0_I32_B64: return S_FF_I32_B64(inst, true);
+		case O::S_FF1_I32_B64: return S_FF_I32_B64(inst, false);
 		case O::S_FLBIT_I32_B32: return V_FFBH_32(inst, false);
 		case O::S_FLBIT_I32_B64: return S_FLBIT_I32_B64(inst);
 

src/graphics/shader/recompiler/frontend/translate/Translator.h

@@ -187,7 +187,7 @@ private:
 	void V_AND_OR_B32(const Decoder::Instruction& inst);
 	void V_OR3_B32(const Decoder::Instruction& inst);
 	void V_XOR3_B32(const Decoder::Instruction& inst);
-	void S_FF1_I32_B64(const Decoder::Instruction& inst);
+	void S_FF_I32_B64(const Decoder::Instruction& inst, bool find_zero);
 	void V_FFBH_32(const Decoder::Instruction& inst, bool sign);
 	void S_FLBIT_I32_B64(const Decoder::Instruction& inst);
 	void Integer24(const Decoder::Instruction& inst, bool sign, bool addend);

tests/ShaderRecompilerComputeTests.cpp

@@ -20860,6 +20860,66 @@ TestCase ScalarBrevB32PreservesScc() {
            O::V_MOV_B32, O::BUFFER_STORE_DWORD, O::S_ENDPGM}};
 }
 
+TestCase ScalarZeroBitOps() {
+  using O = ShaderOpcode;
+  constexpr uint64_t inputs[] = {0, UINT64_MAX, 0xaaaaaaaa55555555ull,
+      0xffffffff7fffffffull, 0xfffffffeffffffffull, 0x7fffffffffffffffull,
+      0x00000000ffffffffull, 0xffffffff00000000ull};
+  TestCase test;
+  test.name = "ScalarZeroBitOps";
+  for (const auto input : inputs)
+    test.initial.insert(test.initial.end(), {u32(input), u32(input >> 32)});
+  test.expected = test.initial;
+  auto &code = test.code;
+  const auto check = [&](u32 opcode, u32 source, uint64_t input, bool scc) {
+    const u32 width = opcode == 0x0e || opcode == 0x12 ? 64 : 32;
+    u32 count = 0, first = UINT32_MAX;
+    for (u32 bit = 0; bit < width; ++bit) {
+      if (((input >> bit) & 1u) == 0) {
+        ++count;
+        if (first == UINT32_MAX) first = bit;
+      }
+    }
+    const bool counting = opcode == 0x0d || opcode == 0x0e;
+    code.push_back(EncodeSopc(0x06, InlineU32(1), InlineU32(scc)));
+    code.push_back(EncodeSop1(opcode, 20, source));
+    if (source == 255) code.push_back(u32(input));
+    code.push_back(EncodeSMovB32(22, 253)); // Capture SCC before readback.
+    AppendStoreSgpr(&code, 20, u32(test.expected.size()));
+    AppendStoreSgpr(&code, 21, u32(test.expected.size() + 1));
+    AppendStoreSgpr(&code, 22, u32(test.expected.size() + 2));
+    test.expected.insert(test.expected.end(),
+        {counting ? count : first, u32(input >> 32), u32(counting ? count != 0 : scc)});
+  };
+  for (u32 index = 0; index < std::size(inputs); ++index) {
+    for (u32 word = 0; word < 2; ++word) {
+      AppendVMovU32(&code, 30, (index * 2 + word) * sizeof(u32));
+      AppendBufferLoadDword(&code, 1, 30);
+      code.push_back(EncodeVop1(0x02, 8 + word, Vgpr(1)));
+    }
+    for (const u32 opcode : {0x0du, 0x0eu, 0x11u, 0x12u}) {
+      for (const bool scc : {false, true}) {
+        code.push_back(EncodeSop1(0x04, 20, 8)); // Destination aliases the source.
+        check(opcode, 20, inputs[index], scc);
+      }
+    }
+  }
+  for (const u32 opcode : {0x0du, 0x0eu, 0x11u, 0x12u}) {
+    for (const u32 source : {128u, 193u, 255u}) {
+      const uint64_t input = source == 128 ? 0 : source == 193 ? UINT64_MAX : 0xffffffffull;
+      AppendSMovLiteral(&code, 21, u32(input >> 32));
+      check(opcode, source, input, true); // Only inline -1 sign extends to B64.
+    }
+  }
+  AppendEnd(&code);
+  test.initial.resize(test.expected.size());
+  test.opcodes = {O::S_BCNT0_I32_B32, O::S_BCNT0_I32_B64,
+      O::S_FF0_I32_B32, O::S_FF0_I32_B64, O::S_CMP_EQ_U32, O::S_MOV_B32,
+      O::S_MOV_B64, O::V_MOV_B32, O::V_READFIRSTLANE_B32,
+      O::BUFFER_LOAD_DWORD, O::BUFFER_STORE_DWORD, O::S_ENDPGM};
+  return test;
+}
+
 TestCase ScalarSextMasks(u32 bits, u32 wave_size) {
   using O = ShaderOpcode;
   TestCase test;
@@ -37061,6 +37121,7 @@ std::vector<TestCase> MakeCases() {
   cases.push_back(ScalarSubvectorLoops(64));
   AddCase(ScalarGetpcWritesNextInstructionPc);
   AddCase(ScalarBitfieldPack);
+  AddCase(ScalarZeroBitOps);
   AddCase(ScalarBitcmpB64DynamicOperands);
   AddCase(ScalarBitcmpB64IntegerConstants);
   AddCase(ScalarBrevB32PreservesScc);
@@ -42892,6 +42953,12 @@ int main(int argc, char **argv) {
     RunCase(&vulkan, Vop3pInt16Ops());
     return 0;
   }
+  if (argc == 2 && std::strcmp(argv[1], "--scalar-zero-bit-only") == 0) {
+    VulkanHarness vulkan;
+    RunCase(&vulkan, ScalarZeroBitOps());
+    RunCase(&vulkan, ScalarBitfieldPack());
+    return 0;
+  }
   if (argc == 2 && std::strcmp(argv[1], "--buffer-format-store-only") == 0) {
     VulkanHarness vulkan;
     RunCase(&vulkan, BufferStoreFormatXyzwPackedUnormSkinningVectors());