shader: DS_READ_U16_D16, V_LSHRREV_B64,

src/graphics/shader/recompiler/frontend/decode/MemoryOps.cpp

@@ -103,6 +103,7 @@ constexpr MemoryOpcodeInfo DS_OPCODE_LIST[] = {
     {0x4du, Opcode::DS_WRITE_B64, 2, 32},        {0x4eu, Opcode::DS_WRITE2_B64, 4, 32},
     {0x4fu, Opcode::DS_WRITE2ST64_B64, 4, 32},   {0x76u, Opcode::DS_READ_B64, 2, 32},
     {0x77u, Opcode::DS_READ2_B64, 4, 32},        {0x78u, Opcode::DS_READ2ST64_B64, 4, 32},
+    {0xa6u, Opcode::DS_READ_U16_D16, 1, 16},
     {0xb0u, Opcode::DS_WRITE_ADDTID_B32, 1, 32}, {0xb1u, Opcode::DS_READ_ADDTID_B32, 1, 32},
     {0xdeu, Opcode::DS_WRITE_B96, 3, 32},        {0xdfu, Opcode::DS_WRITE_B128, 4, 32},
     {0xfeu, Opcode::DS_READ_B96, 3, 32},         {0xffu, Opcode::DS_READ_B128, 4, 32},
@@ -420,6 +421,11 @@ void DecodeDs(uint32_t pc, std::span<const uint32_t> code, uint32_t word_index,
 	}
 
 	DecodeVectorGpr(vdst, inst.dst);
+	if (inst.opcode == Opcode::DS_READ_U16_D16) {
+		// Native D16 reads update only the selected destination half. Reuse the partial
+		// destination representation so the typed lowering preserves the upper word.
+		inst.dst.sdwa_sel = 4u;
+	}
 	DecodeVectorGpr(addr, inst.src0);
 	DecodeVectorGpr(data0, inst.src1);
 	DecodeVectorGpr(data1, inst.src2);

src/graphics/shader/recompiler/frontend/decode/ShaderDecoder.h

@@ -299,6 +299,7 @@ enum class Opcode {
 	V_LSHRREV_B32,
 	V_ASHR_I32,
 	V_ASHRREV_I32,
+	V_LSHRREV_B64,
 	V_LSHLREV_B16,
 	V_LSHRREV_B16,
 	V_ASHRREV_I16,
@@ -492,6 +493,7 @@ enum class Opcode {
 	DS_READ_U8,
 	DS_READ_I16,
 	DS_READ_U16,
+	DS_READ_U16_D16,
 	DS_READ2_B32,
 	DS_READ2ST64_B32,
 	DS_READ_B32,

src/graphics/shader/recompiler/frontend/decode/VectorAluOps.cpp

@@ -281,6 +281,7 @@ constexpr OpcodeMap VOP3_OPCODE_LIST[] = {
     {0x16au, Opcode::V_MUL_HI_U32},
     {0x16bu, Opcode::V_MUL_LO_I32},
     {0x16cu, Opcode::V_MUL_HI_I32},
+    {0x300u, Opcode::V_LSHRREV_B64},
     {0x303u, Opcode::V_ADD_NC_U16},
     {0x304u, Opcode::V_SUB_NC_U16},
     {0x307u, Opcode::V_LSHRREV_B16},
@@ -1247,6 +1248,7 @@ uint32_t NativeVop3SourceCount(Opcode opcode) {
 		case Opcode::V_MIN_I16:
 		case Opcode::V_ADD_NC_I16:
 		case Opcode::V_SUB_NC_I16:
+		case Opcode::V_LSHRREV_B64:
 		case Opcode::V_LSHLREV_B16:
 		case Opcode::V_LSHRREV_B16:
 		case Opcode::V_ASHRREV_I16:

src/graphics/shader/recompiler/ir/ShaderIRMemory.cpp

@@ -624,6 +624,7 @@ bool LowerMemoryInstruction(const Decoder::Instruction& decoded, BasicBlock& blo
 		case Decoder::Opcode::DS_READ_U8:
 		case Decoder::Opcode::DS_READ_I16:
 		case Decoder::Opcode::DS_READ_U16:
+		case Decoder::Opcode::DS_READ_U16_D16:
 		case Decoder::Opcode::DS_READ_B32:
 		case Decoder::Opcode::DS_READ_B64:
 		case Decoder::Opcode::DS_READ_B96:
@@ -763,6 +764,7 @@ bool IsMemoryOpcode(Decoder::Opcode opcode) {
 		case Decoder::Opcode::DS_READ_U8:
 		case Decoder::Opcode::DS_READ_I16:
 		case Decoder::Opcode::DS_READ_U16:
+		case Decoder::Opcode::DS_READ_U16_D16:
 		case Decoder::Opcode::DS_READ2_B32:
 		case Decoder::Opcode::DS_READ2ST64_B32:
 		case Decoder::Opcode::DS_READ_B32:

src/graphics/shader/recompiler/ir/opcodes/ShaderIROpcodes.cpp

@@ -250,6 +250,7 @@ constexpr LowerMap LOWER_OPS[] = {
     {Decoder::Opcode::V_LSHLREV_B16, Opcode::ShiftLeftLogicalU16},
     {Decoder::Opcode::V_LSHR_B32, Opcode::ShiftRightLogicalU32},
     {Decoder::Opcode::V_LSHRREV_B32, Opcode::ShiftRightLogicalU32},
+    {Decoder::Opcode::V_LSHRREV_B64, Opcode::ShiftRightLogicalU64},
     {Decoder::Opcode::V_LSHRREV_B16, Opcode::ShiftRightLogicalU16},
     {Decoder::Opcode::V_ASHR_I32, Opcode::ShiftRightArithmeticI32},
     {Decoder::Opcode::V_ASHRREV_I32, Opcode::ShiftRightArithmeticI32},
@@ -395,6 +396,7 @@ constexpr LowerMap LOWER_OPS[] = {
     {Decoder::Opcode::DS_READ_U8, Opcode::DsReadUbyte},
     {Decoder::Opcode::DS_READ_I16, Opcode::DsReadSshort},
     {Decoder::Opcode::DS_READ_U16, Opcode::DsReadUshort},
+    {Decoder::Opcode::DS_READ_U16_D16, Opcode::DsReadUshort},
     {Decoder::Opcode::DS_READ2_B32, Opcode::DsRead2B32},
     {Decoder::Opcode::DS_READ2ST64_B32, Opcode::DsRead2B32},
     {Decoder::Opcode::DS_READ_B32, Opcode::DsReadB32},
@@ -466,6 +468,7 @@ bool IsReversedBinary(Decoder::Opcode opcode) {
 		case Decoder::Opcode::V_LSHLREV_B32:
 		case Decoder::Opcode::V_LSHRREV_B32:
 		case Decoder::Opcode::V_ASHRREV_I32:
+		case Decoder::Opcode::V_LSHRREV_B64:
 		case Decoder::Opcode::V_LSHLREV_B16:
 		case Decoder::Opcode::V_LSHRREV_B16:
 		case Decoder::Opcode::V_ASHRREV_I16:

tests/ShaderRecompilerComputeTests.cpp

@@ -11393,6 +11393,7 @@ CoverageClass ClassifyOpcode(ShaderOpcode opcode,
   case Opcode::DS_READ_U8:
   case Opcode::DS_READ_I16:
   case Opcode::DS_READ_U16:
+  case Opcode::DS_READ_U16_D16:
   case Opcode::DS_READ2_B32:
   case Opcode::DS_READ_B32:
   case Opcode::DS_READ_B64:
@@ -12931,6 +12932,30 @@ TestCase VectorShiftCountsMaskLowBits() {
            O::BUFFER_STORE_DWORD, O::S_ENDPGM}};
 }
 
+TestCase VectorVop3LshrrevB64Captured() {
+  using O = ShaderOpcode;
+
+  std::vector<u32> code;
+  AppendSMovLiteral(&code, 106, 0x89abcdefu);
+  AppendSMovLiteral(&code, 107, 0x01234567u);
+  AppendVMovU32(&code, 12, 68);
+  code.push_back(0xd7000001u);
+  code.push_back(0x0000d50cu); // v_lshrrev_b64 v[1:2], v12, vcc
+  AppendStoreVgpr(&code, 1, 0);
+  AppendStoreVgpr(&code, 2, 1);
+  AppendEnd(&code);
+
+  TestCase test;
+  test.name = "VectorVop3LshrrevB64Captured";
+  test.code = std::move(code);
+  test.expected = {0x789abcdeu, 0x00123456u};
+  test.opcodes = {O::S_MOV_B32, O::V_MOV_B32, O::V_LSHRREV_B64,
+                  O::BUFFER_STORE_DWORD, O::S_ENDPGM};
+  test.required_spirv = {"OpShiftRightLogical", "OpSelect"};
+  test.forbidden_spirv = {"OpTypeInt 64"};
+  return test;
+}
+
 TestCase VectorVop3IntegerOps() {
   using O = ShaderOpcode;
 
@@ -14735,16 +14760,30 @@ TestCase VectorVopcCmpxNeU64CapturedExecMask() {
   AppendVMovU32(&code, 31, 4);
   code.push_back(0x7dea0e80u); // v_cmpx_ne_u64 exec, 0, v[7:8]
   AppendBufferStoreDword(&code, 8, 31);
+
+  AppendVMovU32(&code, 2, 0);
+  AppendVMovU32(&code, 3, 0);
+  AppendVMovU32(&code, 31, 8);
+  code.push_back(0x7dea0480u); // v_cmpx_ne_u64 exec, 0, v[2:3]
+  AppendBufferStoreDword(&code, 3, 31);
+
+  code.push_back(EncodeSMovB32(126, InlineU32(1)));
+  AppendVMovU32(&code, 3, 1);
+  AppendVMovU32(&code, 31, 12);
+  code.push_back(0x7dea0480u); // v_cmpx_ne_u64 exec, 0, v[2:3]
+  AppendBufferStoreDword(&code, 3, 31);
   AppendEnd(&code);
 
   TestCase test{"VectorVopcCmpxNeU64CapturedExecMask",
                 code,
-                {0x11223344u, 0x55667788u},
+                {0x11223344u, 0x55667788u, 0xaabbccddu, 0xeeff0011u},
-                {0x11223344u, 1u},
+                {0x11223344u, 1u, 0xaabbccddu, 1u},
                 {O::V_MOV_B32, O::V_CMPX_NE_U64, O::BUFFER_STORE_DWORD,
                  O::S_MOV_B32, O::S_ENDPGM}};
-  test.decoded_counts = {{"V_CMPX_NE_U64 exec_lo, 0, v7", 2}};
+  test.decoded_counts = {{"V_CMPX_NE_U64 exec_lo, 0, v7", 2},
-  test.native_ir_counts = {{"CompareNeU64 exec_lo, 0x00000000, v7", 2}};
+                         {"V_CMPX_NE_U64 exec_lo, 0, v2", 2}};
+  test.native_ir_counts = {{"CompareNeU64 exec_lo, 0x00000000, v7", 2},
+                           {"CompareNeU64 exec_lo, 0x00000000, v2", 2}};
   return test;
 }
 
@@ -17197,6 +17236,38 @@ TestCase DsReadWriteVariants() {
            O::BUFFER_STORE_DWORD, O::S_ENDPGM}};
 }
 
+TestCase DsReadU16D16CapturedPreservesHighHalf() {
+  using O = ShaderOpcode;
+
+  std::vector<u32> code;
+  AppendVMovU32(&code, 1, 0);
+  AppendVMovLiteral(&code, 2, 0x89abcdefu);
+  code.push_back(EncodeDs0(0x0d));
+  code.push_back(EncodeDs1(0, 2, 1)); // ds_write_b32 v2, v1
+
+  AppendVMovLiteral(&code, 3, 0x12345678u);
+  code.push_back(EncodeDs0(0xa6));
+  code.push_back(EncodeDs1(3, 0, 1)); // ds_read_u16_d16 v3, v1
+
+  AppendVMovU32(&code, 7, 0);
+  code.push_back(0xda980000u);
+  code.push_back(0x07000007u); // captured ds_read_u16_d16 v7, v7
+
+  AppendStoreVgpr(&code, 3, 0);
+  AppendStoreVgpr(&code, 7, 1);
+  AppendEnd(&code);
+
+  TestCase test{"DsReadU16D16CapturedPreservesHighHalf",
+                code,
+                std::vector<u32>(2, 0),
+                {0x1234cdefu, 0x0000cdefu},
+                {O::V_MOV_B32, O::DS_WRITE_B32, O::DS_READ_U16_D16,
+                 O::BUFFER_STORE_DWORD, O::S_ENDPGM}};
+  test.decoded_counts = {{"DS_READ_U16_D16", 2}};
+  test.native_ir_counts = {{"DsReadUshort", 2}};
+  return test;
+}
+
 TestCase DsReadWrite2Variants() {
   using O = ShaderOpcode;
 
@@ -19569,6 +19640,7 @@ std::vector<TestCase> MakeCases() {
   AddCase(Vop3Med3I16Captured);
   AddCase(Vop2SdwaMinU32PreservesWordDestination);
   AddCase(VectorShiftCountsMaskLowBits);
+  AddCase(VectorVop3LshrrevB64Captured);
   AddCase(VectorVop3IntegerOps);
   AddCase(VectorBfeI32SignExtendsField);
   AddCase(VectorAlignByteUsesFiveBitByteOffset);
@@ -19713,6 +19785,7 @@ std::vector<TestCase> MakeCases() {
   AddCase(ScratchIsPrivatePerInvocation);
   AddCase(FlatStoreVariants);
   AddCase(DsReadWriteVariants);
+  AddCase(DsReadU16D16CapturedPreservesHighHalf);
   AddCase(DsAppendConsumeUsesEncodedLdsSelector);
   AddCase(DsAppendUsesEncodedGdsSelector);
   AddCase(DsGdsSubdwordAndAtomicWrites);

tests/shaderCfgTests.cpp

@@ -6560,6 +6560,7 @@ void TestNewShaderRecompilerDsSubDwordLowering() {
       EncodeDs0(0x3a, 4), EncodeDs1(43, 0, 1), // ds_read_u8 v43, v1
       EncodeDs0(0x3b, 6), EncodeDs1(44, 0, 1), // ds_read_i16 v44, v1
       EncodeDs0(0x3c, 8), EncodeDs1(45, 0, 1), // ds_read_u16 v45, v1
+      0xda980000u, 0x07000007u, // ds_read_u16_d16 v7, v7
       0xbf810000u,
   };
 
@@ -6582,6 +6583,8 @@ void TestNewShaderRecompilerDsSubDwordLowering() {
         "new decoder did not decode DS signed short read");
   Check(Common::ContainsStr(result.decoded_dump, "ds_read_u16"),
         "new decoder did not decode DS unsigned short read");
+  Check(Common::ContainsStr(result.decoded_dump, "DS_READ_U16_D16 v7.sdwa(sel=4"),
+        "new decoder did not decode captured DS masked short read");
   Check(Common::ContainsStr(result.decoded_dump, "dwords=1 bits=8"),
         "DS byte decode did not preserve byte-width metadata");
   Check(Common::ContainsStr(result.decoded_dump, "dwords=1 bits=16"),
@@ -6598,6 +6601,8 @@ void TestNewShaderRecompilerDsSubDwordLowering() {
         "DS signed short read did not lower to explicit IR");
   Check(Common::ContainsStr(result.ir_dump, "DsReadUshort v45"),
         "DS unsigned short read did not lower to explicit IR");
+  Check(Common::ContainsStr(result.ir_dump, "DsReadUshort v7.sdwa(sel=4"),
+        "DS masked short read did not preserve its partial destination");
   Check(SpirvContainsOpcode(result.spirv, 61),
         "SPIR-V binary does not contain OpLoad");
   Check(SpirvContainsOpcode(result.spirv, 62),
@@ -8368,6 +8373,45 @@ void TestNewShaderRecompilerU64PairLowering() {
   CompileResult result;
   std::string error;
 
+  const uint32_t lshrrev_b64_shader[] = {
+      0xd7000001u, 0x0000d50cu, // v_lshrrev_b64 v[1:2], v12, vcc
+      EncodeSopp(0x01),
+  };
+  Decoder::Instruction decoded_lshrrev_b64;
+  Check(Decoder::DecodeInstruction(lshrrev_b64_shader, 0u,
+                                   decoded_lshrrev_b64, &error),
+        error.c_str());
+  Check(decoded_lshrrev_b64.opcode == Decoder::Opcode::V_LSHRREV_B64 &&
+            decoded_lshrrev_b64.dst.kind == Decoder::OperandKind::Vgpr &&
+            decoded_lshrrev_b64.dst.reg == 1u &&
+            decoded_lshrrev_b64.src_count == 2u &&
+            decoded_lshrrev_b64.src0.kind == Decoder::OperandKind::Vgpr &&
+            decoded_lshrrev_b64.src0.reg == 12u &&
+            decoded_lshrrev_b64.src1.kind == Decoder::OperandKind::VccLo,
+        "decoder rejected captured VOP3 V_LSHRREV_B64 fields");
+  Check(TryRecompile(lshrrev_b64_shader, options, result, &error),
+        error.c_str());
+  Check(Common::ContainsStr(result.decoded_dump,
+                            "V_LSHRREV_B64 v1, v12, vcc_lo"),
+        "captured VOP3 V_LSHRREV_B64 was not present in the decoded dump");
+  Decoder::Program decoded_lshrrev_b64_program;
+  CFG::Graph graph_lshrrev_b64;
+  IR::Program native_lshrrev_b64;
+  Check(Decoder::DecodeProgram(lshrrev_b64_shader,
+                               decoded_lshrrev_b64_program, &error),
+        error.c_str());
+  Check(CFG::BuildGraph(decoded_lshrrev_b64_program, graph_lshrrev_b64,
+                        &error) &&
+            CFG::Structurize(graph_lshrrev_b64, &error),
+        error.c_str());
+  Check(IR::LowerProgram(decoded_lshrrev_b64_program, graph_lshrrev_b64,
+                         ShaderType::Compute, 64u, native_lshrrev_b64, &error),
+        error.c_str());
+  Check(Common::ContainsStr(IR::ProgramToString(native_lshrrev_b64),
+                            "ShiftRightLogicalU64 v1, vcc_lo, v12"),
+        "V_LSHRREV_B64 did not reverse its encoded operands for pair-U64 IR");
+  CheckSpirvBinaryValidates(result.spirv);
+
   const uint32_t cmpx_i64_shader[] = {
       0xd4b5007eu, 0x00020e80u, // v_cmpx_ne_i64 exec, 0, v[7:8]
       EncodeSopp(0x01),
@@ -8432,6 +8476,26 @@ void TestNewShaderRecompilerU64PairLowering() {
         "V_CMPX_NE_U64 did not lower to masked 64-bit inequality");
   CheckSpirvBinaryValidates(result.spirv);
 
+  const uint32_t cmpx_v2_shader[] = {
+      0x7dea0480u, // v_cmpx_ne_u64 exec, 0, v[2:3]
+      EncodeSopp(0x01),
+  };
+  Decoder::Instruction decoded_cmpx_v2;
+  Check(Decoder::DecodeInstruction(cmpx_v2_shader, 0u, decoded_cmpx_v2,
+                                   &error),
+        error.c_str());
+  Check(decoded_cmpx_v2.opcode == Decoder::Opcode::V_CMPX_NE_U64 &&
+            decoded_cmpx_v2.dst.kind == Decoder::OperandKind::ExecLo &&
+            decoded_cmpx_v2.src0.value == 0u &&
+            decoded_cmpx_v2.src1.kind == Decoder::OperandKind::Vgpr &&
+            decoded_cmpx_v2.src1.reg == 2u,
+        "decoder rejected reported VOPC V_CMPX_NE_U64 fields");
+  Check(TryRecompile(cmpx_v2_shader, options, result, &error), error.c_str());
+  Check(Common::ContainsStr(result.decoded_dump,
+                            "V_CMPX_NE_U64 exec_lo, 0, v2"),
+        "reported VOPC V_CMPX_NE_U64 was not present in the decoded dump");
+  CheckSpirvBinaryValidates(result.spirv);
+
   const uint32_t empty_shader[] = {EncodeSopp(0x01)};
   Check(TryRecompile(empty_shader, options, result, &error), error.c_str());