graphics: implement RDNA2 D16 image packing

src/graphics/shader/recompiler/backend/spirv/spirvEmitterImage.cpp

@@ -14,6 +14,13 @@ uint32_t DmaskComponentIndex(uint32_t dmask, uint32_t component) {
 	return index;
 }
 
+uint32_t DmaskComponent(uint32_t dmask, uint32_t index) {
+	for (uint32_t component = 0; component < 4u; component++) {
+		if (((dmask >> component) & 1u) != 0u && index-- == 0u) return component;
+	}
+	return 0u;
+}
+
 uint32_t ImageGatherComponent(uint32_t dmask) {
 	switch (dmask) {
 		case 0x2u: return 1;
@@ -191,7 +198,49 @@ uint32_t FloatBits(ValueEmitContext& ctx, uint32_t value) {
 	return Unary(ctx.state, OpBitcast, TypeU32(ctx.state), value);
 }
 
-uint32_t ResultVector(ValueEmitContext& ctx, uint32_t value, bool integer, bool dref) {
+uint32_t ResultVector(ValueEmitContext& ctx, uint32_t value, bool integer, bool dref,
+                      const IR::MemoryInfo& mem, bool gather = false) {
+	if (mem.data_bits == 16u) {
+		uint32_t packed[4] = {ConstantU32(ctx.state, 0), ConstantU32(ctx.state, 0),
+		                      ConstantU32(ctx.state, 0), ConstantU32(ctx.state, 0)};
+		const auto scalar = [&](uint32_t index) {
+			if (dref) return value;
+			const auto component = gather ? index : DmaskComponent(mem.dmask, index);
+			const auto result    = ctx.state.builder.AllocateId();
+			ctx.state.builder.AddFunction(
+			    {OpCompositeExtract, integer ? TypeU32(ctx.state) : TypeF32(ctx.state), result,
+			     value, component});
+			return result;
+		};
+		for (uint32_t word = 0; word < mem.data_dwords; word++) {
+			const auto low_index  = word * 2u;
+			const auto high_index = low_index + 1u;
+			const auto low        = scalar(low_index);
+			const auto high = high_index < mem.component_count
+			                      ? scalar(high_index)
+			                      : (integer ? ConstantU32(ctx.state, 0) : ZeroF32(ctx.state));
+			if (integer) {
+				const auto mask = ConstantU32(ctx.state, 0xffffu);
+				packed[word] = Binary(
+				    ctx.state, OpBitwiseOr, TypeU32(ctx.state),
+				    Binary(ctx.state, OpBitwiseAnd, TypeU32(ctx.state), low, mask),
+				    Binary(ctx.state, OpShiftLeftLogical, TypeU32(ctx.state),
+				           Binary(ctx.state, OpBitwiseAnd, TypeU32(ctx.state), high, mask),
+				           ConstantU32(ctx.state, 16u)));
+			} else {
+				const auto pair = ctx.state.builder.AllocateId();
+				ctx.state.builder.AddFunction(
+				    {OpCompositeConstruct, TypeF32Vector(ctx.state, 2), pair, low, high});
+				packed[word] = ctx.state.builder.AllocateId();
+				ctx.state.builder.AddFunction({OpExtInst, TypeU32(ctx.state), packed[word],
+				                               GlslStd450(ctx.state), GlslPackHalf2x16, pair});
+			}
+		}
+		const auto result = ctx.state.builder.AllocateId();
+		ctx.state.builder.AddFunction({OpCompositeConstruct, TypeU32Vector(ctx.state, 4), result,
+		                               packed[0], packed[1], packed[2], packed[3]});
+		return result;
+	}
 	uint32_t component[4] {};
 	for (uint32_t index = 0; index < 4u; index++) {
 		if (dref) {
@@ -319,9 +368,21 @@ uint32_t StoreTexel(ValueEmitContext& ctx, const IR::MemoryInfo& mem, uint32_t d
 			const auto packed_index = DmaskComponentIndex(dmask, source);
 			raw                     = ctx.state.builder.AllocateId();
 			ctx.state.builder.AddFunction(
-			    {OpCompositeExtract, TypeU32(ctx.state), raw, data, packed_index});
+			    {OpCompositeExtract, TypeU32(ctx.state), raw, data,
+			     mem.data_bits == 16u ? packed_index / 2u : packed_index});
+			if (mem.data_bits == 16u) {
+				if ((packed_index & 1u) != 0u) {
+					raw = Binary(ctx.state, OpShiftRightLogical, TypeU32(ctx.state), raw,
+					             ConstantU32(ctx.state, 16u));
 				}
-		values[component] = integer ? raw : Unary(ctx.state, OpBitcast, TypeF32(ctx.state), raw);
+				raw = Binary(ctx.state, OpBitwiseAnd, TypeU32(ctx.state), raw,
+				             ConstantU32(ctx.state, 0xffffu));
+			}
+		}
+		values[component] = integer ? raw
+		                            : mem.data_bits == 16u ? EmitF16BitsToF32(ctx.state, raw)
+		                                                   : Unary(ctx.state, OpBitcast,
+		                                                           TypeF32(ctx.state), raw);
 	}
 	const auto texel = ctx.state.builder.AllocateId();
 	ctx.state.builder.AddFunction(
@@ -407,7 +468,7 @@ bool EmitValueImage(ValueEmitContext& ctx, const IR::Inst& inst) {
 				             integer ? TypeU32Vector(state, 4) : TypeF32Vector(state, 4), color,
 				             image, coord, ImageOperandsLodMask, LodU32(ctx, mem, *address, view)});
 			        }
-			        return ResultVector(ctx, color, integer, false);
+			        return ResultVector(ctx, color, integer, false, mem);
 		        }));
 		return true;
 	}
@@ -459,7 +520,7 @@ bool EmitValueImage(ValueEmitContext& ctx, const IR::Inst& inst) {
 				words.push_back(PackedOffset(ctx, mem, *address, layout, view));
 			}
 			state.builder.AddFunction(words);
-			ctx.Define(inst, ResultVector(ctx, sample, integer && !dref, false));
+			ctx.Define(inst, ResultVector(ctx, sample, integer && !dref, false, mem, true));
 			return true;
 		}
 		const bool explicit_lod = HasFlag(mem, Decoder::ImageSampleFlagDerivative) ||
@@ -509,7 +570,7 @@ bool EmitValueImage(ValueEmitContext& ctx, const IR::Inst& inst) {
 		};
 		const auto& image = state.program.info.images[mem.resource];
 		if (image.indirect_root != mem.resource) {
-			ctx.Define(inst, ResultVector(ctx, EmitSample(mem.resource), integer, dref));
+			ctx.Define(inst, ResultVector(ctx, EmitSample(mem.resource), integer, dref, mem));
 			return true;
 		}
 		const auto* handle = image_arg.ResolveInstruction();
@@ -600,7 +661,7 @@ bool EmitValueImage(ValueEmitContext& ctx, const IR::Inst& inst) {
 		}
 		EmitLabel(state, merge_label);
 		state.builder.AddFunction(phi_words);
-		ctx.Define(inst, ResultVector(ctx, phi_words[2], integer, dref));
+		ctx.Define(inst, ResultVector(ctx, phi_words[2], integer, dref, mem));
 		return true;
 	}
 	const auto atomic_opcode = ImageAtomicOpcode(op);

src/graphics/shader/recompiler/frontend/decode/ImageOps.cpp

@@ -310,6 +310,7 @@ void DecodeMimg(uint32_t pc, std::span<const uint32_t> code, uint32_t word_index
 	const uint32_t srsrc  = (word1 >> 16u) & 0x1fu;
 	const uint32_t ssamp  = (word1 >> 21u) & 0x1fu;
 	const bool     a16    = ((word1 >> 30u) & 0x1u) != 0u;
+	const bool     d16    = ((word1 >> 31u) & 0x1u) != 0u;
 	const auto*    sample = LookupSample(opcode);
 	const auto*    gather = LookupGather(opcode);
 	const auto*    atomic = LookupAtomic(opcode);
@@ -321,7 +322,9 @@ void DecodeMimg(uint32_t pc, std::span<const uint32_t> code, uint32_t word_index
 	inst.opcode_id          = opcode;
 	inst.opcode             = DecodeMimgOpcode(opcode, sample, gather, atomic);
 	inst.dmask              = (word0 >> 8u) & 0xfu;
-	inst.data_dwords        = gather != nullptr ? 4u : CountDmaskComponents(inst.dmask);
+	inst.data_components    = gather != nullptr ? 4u : CountDmaskComponents(inst.dmask);
+	inst.data_bits          = d16 ? 16u : 32u;
+	inst.data_dwords        = d16 ? (inst.data_components + 1u) / 2u : inst.data_components;
 	inst.glc                = ((word0 >> 13u) & 1u) != 0;
 	inst.slc                = ((word0 >> 25u) & 1u) != 0;
 	inst.image_sample_flags = DecodeMimgSampleFlags(sample, gather);
@@ -344,6 +347,11 @@ void DecodeMimg(uint32_t pc, std::span<const uint32_t> code, uint32_t word_index
 		SetUnsupported(inst, Family::MIMG, opcode,
 		               "MIMG image gather requires exactly one dmask bit");
 	}
+	const bool supports_d16 = sample != nullptr || gather != nullptr || opcode == 0x00u ||
+	                          opcode == 0x01u || opcode == 0x08u || opcode == 0x09u;
+	if (d16 && !supports_d16) {
+		SetUnsupported(inst, Family::MIMG, opcode, "MIMG opcode does not support D16 data");
+	}
 
 	DecodeVectorGpr(vdata, inst.dst);
 	DecodeVectorGpr(vaddr, inst.src0);

src/graphics/shader/recompiler/frontend/decode/ShaderDecoder.cpp

@@ -154,6 +154,9 @@ std::string FormatMimg(const Instruction& inst) {
 	                OperandToString(inst.dst).c_str(), OperandToString(inst.src0).c_str(),
 	                OperandToString(inst.src1).c_str(), OperandToString(inst.src2).c_str(),
 	                inst.dmask, ImageDimensionToString(inst.image_dimension));
+	if (inst.data_bits == 16u) {
+		text += " d16=1";
+	}
 	switch (inst.opcode) {
 		case Opcode::IMAGE_SAMPLE:
 		case Opcode::IMAGE_GATHER4_LZ:

src/graphics/shader/recompiler/frontend/decode/ShaderDecoder.h

@@ -648,6 +648,7 @@ struct Instruction {
 	uint32_t       offset                                       = 0;
 	uint32_t       secondary_offset                             = 0;
 	uint32_t       dmask                                        = 0;
+	uint32_t       data_components                              = 0;
 	uint32_t       data_dwords                                  = 1;
 	uint32_t       data_bits                                    = 32;
 	uint32_t       data_format                                  = 0;

src/graphics/shader/recompiler/frontend/translate/Memory.cpp

@@ -94,9 +94,17 @@ IR::Value Translator::ConstructU32x4(const IR::Operand& base, uint32_t count) {
 	               {components[0], components[1], components[2], components[3]});
 }
 
-void Translator::WriteImageComponents(const IR::Operand& dst, IR::Value value, uint32_t dmask,
+void Translator::WriteImageComponents(const IR::Operand& dst, IR::Value value,
-                                      uint32_t component_limit) {
+                                      const IR::MemoryInfo& memory, uint32_t component_limit) {
-	const auto mask      = dmask != 0u ? dmask : 1u;
+	if (memory.data_bits == 16u) {
+		for (uint32_t index = 0; index < memory.data_dwords; index++) {
+			WriteOperand(OffsetOperand(dst, index),
+			             ir.Emit(IR::ValueOpcode::CompositeExtractU32x4,
+			                     {value, IR::Value(index)}));
+		}
+		return;
+	}
+	const auto mask      = memory.dmask != 0u ? memory.dmask : 1u;
 	uint32_t   dst_index = 0;
 	for (uint32_t component = 0; component < component_limit; component++) {
 		if (((mask >> component) & 1u) == 0u) {
@@ -435,20 +443,20 @@ bool Translator::TranslateImageMemory(const IR::Instruction& inst) {
 		case IR::Opcode::ImageGetResinfo: {
 			const auto result =
 			    ir.Emit(IR::ValueOpcode::ImageQueryDimensions, {resource, address}, flags);
-			WriteImageComponents(inst.dst, result, inst.memory.dmask, 4u);
+			WriteImageComponents(inst.dst, result, inst.memory, 4u);
 			return true;
 		}
 		case IR::Opcode::ImageGetLod: {
 			const auto sampler = GetSamplerResource(inst.memory);
 			const auto result =
 			    ir.Emit(IR::ValueOpcode::ImageQueryLod, {resource, sampler, address}, flags);
-			WriteImageComponents(inst.dst, result, inst.memory.dmask, 2u);
+			WriteImageComponents(inst.dst, result, inst.memory, 2u);
 			return true;
 		}
 		case IR::Opcode::ImageLoad: {
 			const auto result =
 			    ir.Emit(IR::ValueOpcode::ImageRead, {resource, address, ir.GetExec()}, flags);
-			WriteImageComponents(inst.dst, result, inst.memory.dmask, 4u);
+			WriteImageComponents(inst.dst, result, inst.memory, 4u);
 			return true;
 		}
 		case IR::Opcode::ImageStore: {
@@ -465,7 +473,7 @@ bool Translator::TranslateImageMemory(const IR::Instruction& inst) {
 			const auto result  = ir.Emit(opcode, {resource, sampler, address}, flags);
 			const bool dref =
 			    (inst.memory.image_sample_flags & Decoder::ImageSampleFlagCompare) != 0u;
-			if (inst.op == IR::Opcode::ImageSample && dref) {
+			if (inst.op == IR::Opcode::ImageSample && dref && inst.memory.data_bits != 16u) {
 				const auto component =
 				    ir.Emit(IR::ValueOpcode::CompositeExtractU32x4, {result, IR::Value(0u)});
 				for (uint32_t index = 0; index < inst.memory.data_dwords; index++) {
@@ -478,7 +486,7 @@ bool Translator::TranslateImageMemory(const IR::Instruction& inst) {
 					                     {result, IR::Value(index)}));
 				}
 			} else {
-				WriteImageComponents(inst.dst, result, inst.memory.dmask, 4u);
+				WriteImageComponents(inst.dst, result, inst.memory, 4u);
 			}
 			return true;
 		}

src/graphics/shader/recompiler/frontend/translate/Translator.h

@@ -75,8 +75,8 @@ private:
 	IR::Value       GetSamplerResource(const IR::MemoryInfo& memory);
 	IR::Value       MakeImageAddress(const IR::Instruction& inst, const IR::Operand& base);
 	IR::Value       ConstructU32x4(const IR::Operand& base, uint32_t count);
-	void            WriteImageComponents(const IR::Operand& dst, IR::Value value, uint32_t dmask,
+	void            WriteImageComponents(const IR::Operand& dst, IR::Value value,
-	                                     uint32_t component_limit);
+	                                     const IR::MemoryInfo& memory, uint32_t component_limit);
 	IR::ValueOpcode ImageAtomicOpcode(IR::Opcode opcode);
 	BufferAddress   ReadBufferAddress(const IR::Instruction& inst, uint32_t source_offset);
 	IR::U32         WidenSubdword(IR::Value value, uint32_t bits, bool sign);

src/graphics/shader/recompiler/ir/ShaderIR.cpp

@@ -211,7 +211,8 @@ MemoryInfo MemoryInfoFromDecoded(const Decoder::Instruction& decoded, ResourceKi
 	mem.dmask                    = decoded.dmask;
 	mem.data_dwords              = decoded.data_dwords;
 	mem.data_bits                = decoded.data_bits;
-	mem.component_count          = decoded.data_dwords;
+	mem.component_count          = decoded.data_components != 0u ? decoded.data_components
+	                                                            : decoded.data_dwords;
 	mem.data_format              = decoded.data_format;
 	mem.number_format            = decoded.number_format;
 	mem.image_sample_flags       = decoded.image_sample_flags;

tests/ShaderRecompilerComputeTests.cpp

@@ -578,9 +578,10 @@ constexpr u32 EncodeMimg0(u32 opcode, u32 dmask, u32 nsa_dwords = 0,
 }
 
 constexpr u32 EncodeMimg1(u32 vdata, u32 vaddr, u32 srsrc = 0, u32 ssamp = 0,
-                          bool a16 = false) {
+                          bool a16 = false, bool d16 = false) {
   return ((ssamp & 0x1fu) << 21u) | ((srsrc & 0x1fu) << 16u) |
-         ((vdata & 0xffu) << 8u) | (vaddr & 0xffu) | (a16 ? (1u << 30u) : 0u);
+         ((vdata & 0xffu) << 8u) | (vaddr & 0xffu) |
+         (a16 ? (1u << 30u) : 0u) | (d16 ? (1u << 31u) : 0u);
 }
 
 constexpr u32 EncodeVintrp(u32 opcode, u32 dst, u32 attr, u32 chan, u32 src) {
@@ -17886,6 +17887,39 @@ TestCase ImageSampleAndGather() {
   return test;
 }
 
+TestCase ImageD16GatherPacksHalfPairs() {
+  using O = ShaderOpcode;
+
+  std::vector<u32> code;
+  AppendVMovLiteral(&code, 20, 0x3f000000u);
+  AppendVMovLiteral(&code, 21, 0x3f000000u);
+  AppendVMovU32(&code, 22, 0);
+  code.push_back(EncodeMimg0(0x47, 0x1));
+  code.push_back(EncodeMimg1(0, 20, 0, 0, false, true));
+  AppendStoreVgpr(&code, 0, 0);
+  AppendStoreVgpr(&code, 1, 1);
+  AppendEnd(&code);
+
+  auto image = MakeRgbaImage(4, 4);
+  for (u32 y = 0; y < 4u; y++) {
+    for (u32 x = 0; x < 4u; x++) {
+      SetRgbaPixel(&image, 4, x, y, 0x3f000000u, 0, 0, 0);
+    }
+  }
+
+  TestCase test;
+  test.name = "ImageD16GatherPacksHalfPairs";
+  test.code = code;
+  test.expected = {0x38003800u, 0x38003800u};
+  test.opcodes = {O::V_MOV_B32, O::IMAGE_GATHER4_LZ, O::BUFFER_STORE_DWORD,
+                  O::S_ENDPGM};
+  test.sampled_image_rgba = image;
+  test.decoded_counts = {{"d16=1", 1}};
+  test.native_ir_counts = {{"dmask=0x1 data_dwords=2 data_bits=16 component=0/4", 1}};
+  test.required_spirv = {"OpImageGather", "PackHalf2x16"};
+  return test;
+}
+
 TestCase ImageSampleA16SamplerCoordsOnGpu() {
   using O = ShaderOpcode;
 
@@ -18078,6 +18112,35 @@ TestCase ImageStoreVariants() {
   return test;
 }
 
+TestCase ImageD16StoreUnpacksHalfPairs() {
+  using O = ShaderOpcode;
+
+  std::vector<u32> code;
+  AppendVMovU32(&code, 20, 1);
+  AppendVMovU32(&code, 21, 2);
+  AppendVMovU32(&code, 22, 0);
+  AppendVMovLiteral(&code, 0, 0x38003400u);
+  AppendVMovLiteral(&code, 1, 0x3c003a00u);
+  code.push_back(EncodeMimg0(0x08, 0xf));
+  code.push_back(EncodeMimg1(0, 20, 0, 0, false, true));
+  AppendEnd(&code);
+
+  auto expected_image = MakeRgbaImage(4, 4);
+  SetRgbaPixel(&expected_image, 4, 1, 2, 0x3e800000u, 0x3f000000u,
+               0x3f400000u, 0x3f800000u);
+
+  TestCase test;
+  test.name = "ImageD16StoreUnpacksHalfPairs";
+  test.code = code;
+  test.opcodes = {O::V_MOV_B32, O::IMAGE_STORE, O::S_ENDPGM};
+  test.storage_image_rgba = MakeRgbaImage(4, 4);
+  test.expected_storage_image_rgba = expected_image;
+  test.decoded_counts = {{"d16=1", 1}};
+  test.native_ir_counts = {{"dmask=0xf data_dwords=2 data_bits=16 component=0/4", 1}};
+  test.required_spirv = {"UnpackHalf2x16", "OpImageWrite"};
+  return test;
+}
+
 void CheckIndirectImageKeySwitch() {
   constexpr const char *name = "IndirectImageKeySwitch";
   constexpr uint32_t mapping_capacity = 1793u;
@@ -19012,12 +19075,14 @@ std::vector<TestCase> MakeCases() {
   AddCase(ImageGetResinfoDmaskWidthHeight);
   AddCase(ImageGetResinfoDmaskMipLevels);
   AddCase(ImageSampleAndGather);
+  AddCase(ImageD16GatherPacksHalfPairs);
   AddCase(ImageSampleA16SamplerCoordsOnGpu);
   AddCase(ImageSampleOpcodeAliasUsesNormalCoords);
   AddCase(ImageSampleA16OffsetKeepsTexelOffset32BitOnGpu);
   AddCase(ImageSampleA16CompareBiasRdna2AddressOrder);
   AddCase(ImageGatherCompareOpcodes);
   AddCase(ImageStoreVariants);
+  AddCase(ImageD16StoreUnpacksHalfPairs);
   AddCase(ImageStoreMipSelectsPpsa01340Descriptor);
   AddCase(ImageStoreRgbOneUsesInverseSwizzle);
   AddCase(ImageStoreDuplicateSelectorUsesInverseSwizzle);