shader: canonicalize SampleAdjust sampler scratch bits

src/graphics/shader/recompiler/frontend/decode/ImageOps.cpp

@@ -153,28 +153,38 @@ constexpr MimgSampleInfo MIMG_SAMPLE_OPCODE_LIST[] = {
     SampleInfo(0x6fu, "image_sample_c_cd_cl_o",
                ImageSampleFlagCompare | ImageSampleFlagDerivative | ImageSampleFlagCd |
                    ImageSampleFlagLodClamp | ImageSampleFlagOffset),
-    SampleInfo(0xa0u, "image_sample_a", 0),
+    SampleInfo(0xa0u, "image_sample_a", ImageSampleFlagAdjust),
-    SampleInfo(0xa1u, "image_sample_cl_a", ImageSampleFlagLodClamp),
+    SampleInfo(0xa1u, "image_sample_cl_a", ImageSampleFlagLodClamp | ImageSampleFlagAdjust),
-    SampleInfo(0xa5u, "image_sample_b_a", ImageSampleFlagBias),
+    SampleInfo(0xa5u, "image_sample_b_a", ImageSampleFlagBias | ImageSampleFlagAdjust),
-    SampleInfo(0xa6u, "image_sample_b_cl_a", ImageSampleFlagBias | ImageSampleFlagLodClamp),
+    SampleInfo(0xa6u, "image_sample_b_cl_a",
-    SampleInfo(0xa8u, "image_sample_c_a", ImageSampleFlagCompare),
+               ImageSampleFlagBias | ImageSampleFlagLodClamp | ImageSampleFlagAdjust),
-    SampleInfo(0xa9u, "image_sample_c_cl_a", ImageSampleFlagCompare | ImageSampleFlagLodClamp),
+    SampleInfo(0xa8u, "image_sample_c_a", ImageSampleFlagCompare | ImageSampleFlagAdjust),
-    SampleInfo(0xadu, "image_sample_c_b_a", ImageSampleFlagCompare | ImageSampleFlagBias),
+    SampleInfo(0xa9u, "image_sample_c_cl_a",
+               ImageSampleFlagCompare | ImageSampleFlagLodClamp | ImageSampleFlagAdjust),
+    SampleInfo(0xadu, "image_sample_c_b_a",
+               ImageSampleFlagCompare | ImageSampleFlagBias | ImageSampleFlagAdjust),
     SampleInfo(0xaeu, "image_sample_c_b_cl_a",
-               ImageSampleFlagCompare | ImageSampleFlagBias | ImageSampleFlagLodClamp),
+               ImageSampleFlagCompare | ImageSampleFlagBias | ImageSampleFlagLodClamp |
-    SampleInfo(0xb0u, "image_sample_o_a", ImageSampleFlagOffset),
+                   ImageSampleFlagAdjust),
-    SampleInfo(0xb1u, "image_sample_cl_o_a", ImageSampleFlagLodClamp | ImageSampleFlagOffset),
+    SampleInfo(0xb0u, "image_sample_o_a", ImageSampleFlagOffset | ImageSampleFlagAdjust),
-    SampleInfo(0xb5u, "image_sample_b_o_a", ImageSampleFlagBias | ImageSampleFlagOffset),
+    SampleInfo(0xb1u, "image_sample_cl_o_a",
+               ImageSampleFlagLodClamp | ImageSampleFlagOffset | ImageSampleFlagAdjust),
+    SampleInfo(0xb5u, "image_sample_b_o_a",
+               ImageSampleFlagBias | ImageSampleFlagOffset | ImageSampleFlagAdjust),
     SampleInfo(0xb6u, "image_sample_b_cl_o_a",
-               ImageSampleFlagBias | ImageSampleFlagLodClamp | ImageSampleFlagOffset),
+               ImageSampleFlagBias | ImageSampleFlagLodClamp | ImageSampleFlagOffset |
-    SampleInfo(0xb8u, "image_sample_c_o_a", ImageSampleFlagCompare | ImageSampleFlagOffset),
+                   ImageSampleFlagAdjust),
+    SampleInfo(0xb8u, "image_sample_c_o_a",
+               ImageSampleFlagCompare | ImageSampleFlagOffset | ImageSampleFlagAdjust),
     SampleInfo(0xb9u, "image_sample_c_cl_o_a",
-               ImageSampleFlagCompare | ImageSampleFlagLodClamp | ImageSampleFlagOffset),
+               ImageSampleFlagCompare | ImageSampleFlagLodClamp | ImageSampleFlagOffset |
+                   ImageSampleFlagAdjust),
     SampleInfo(0xbdu, "image_sample_c_b_o_a",
-               ImageSampleFlagCompare | ImageSampleFlagBias | ImageSampleFlagOffset),
+               ImageSampleFlagCompare | ImageSampleFlagBias | ImageSampleFlagOffset |
+                   ImageSampleFlagAdjust),
     SampleInfo(0xbeu, "image_sample_c_b_cl_o_a",
                ImageSampleFlagCompare | ImageSampleFlagBias | ImageSampleFlagLodClamp |
-                   ImageSampleFlagOffset),
+                   ImageSampleFlagOffset | ImageSampleFlagAdjust),
 };
 
 constexpr MimgGatherInfo MIMG_GATHER_OPCODE_LIST[] = {

src/graphics/shader/recompiler/frontend/decode/ShaderDecoder.cpp

@@ -140,6 +140,7 @@ std::string ImageSampleFlagsToString(uint32_t flags) {
 	AppendFlag(&text, &first, flags, ImageSampleFlagA16, "a16");
 	AppendFlag(&text, &first, flags, ImageSampleFlagCd, "cd");
 	AppendFlag(&text, &first, flags, ImageSampleFlagGatherHorizontal, "gather_horizontal");
+	AppendFlag(&text, &first, flags, ImageSampleFlagAdjust, "adjust");
 	return text;
 }
 

src/graphics/shader/recompiler/frontend/decode/ShaderDecoder.h

@@ -586,6 +586,7 @@ enum ImageSampleFlag : uint32_t {
 	ImageSampleFlagA16              = 1u << 7u,
 	ImageSampleFlagCd               = 1u << 8u,
 	ImageSampleFlagGatherHorizontal = 1u << 9u,
+	ImageSampleFlagAdjust           = 1u << 10u,
 };
 
 enum class ImageDimension : uint32_t {

src/graphics/shader/recompiler/ir/passes/ResourceTracking.cpp

@@ -14,6 +14,55 @@ namespace Libs::Graphics::ShaderRecompiler::IR {
 namespace {
 
 constexpr uint32_t SamplerBorderClampMask = (1u << 2u) | (1u << 5u) | (1u << 8u);
+constexpr uint32_t SamplerDword3ReservedMask = 0x3ffff000u;
+
+uint32_t PossibleU32Bits(Value value) {
+	value = value.Resolve();
+	if (value.IsImmediate()) {
+		return value.GetType() == Type::U32 ? value.U32() : UINT32_MAX;
+	}
+	const auto* inst = value.TryInstruction();
+	if (inst == nullptr) {
+		return UINT32_MAX;
+	}
+	switch (inst->GetOpcode()) {
+		case ValueOpcode::BitwiseAnd32:
+			return PossibleU32Bits(inst->Arg(0)) & PossibleU32Bits(inst->Arg(1));
+		case ValueOpcode::BitwiseOr32:
+			return PossibleU32Bits(inst->Arg(0)) | PossibleU32Bits(inst->Arg(1));
+		case ValueOpcode::ShiftLeftLogical32: {
+			const auto shift = inst->Arg(1).Resolve();
+			return shift.IsImmediate() && shift.GetType() == Type::U32
+			           ? PossibleU32Bits(inst->Arg(0)) << (shift.U32() & 31u)
+			           : UINT32_MAX;
+		}
+		default: return UINT32_MAX;
+	}
+}
+
+Value CanonicalizeSampleAdjustDword3(Value value) {
+	for (;;) {
+		value            = value.Resolve();
+		const auto* inst = value.TryInstruction();
+		if (inst == nullptr || inst->GetOpcode() != ValueOpcode::BitwiseOr32) {
+			return value;
+		}
+		const auto left           = inst->Arg(0).Resolve();
+		const auto right          = inst->Arg(1).Resolve();
+		const bool left_reserved  = (PossibleU32Bits(left) & ~SamplerDword3ReservedMask) == 0;
+		const bool right_reserved = (PossibleU32Bits(right) & ~SamplerDword3ReservedMask) == 0;
+		if (left_reserved && right_reserved) {
+			return Value(0u);
+		}
+		if (left_reserved) {
+			value = right;
+		} else if (right_reserved) {
+			value = left;
+		} else {
+			return value;
+		}
+	}
+}
 
 const char* StageName(ShaderType stage) {
 	switch (stage) {
@@ -211,8 +260,8 @@ private:
 		return true;
 	}
 
-	bool MakeSource(const Inst& handle, uint32_t width, bool sampler, DescriptorSource& descriptor,
+	bool MakeSource(const Inst& handle, uint32_t width, bool sampler, bool sample_adjust,
-	                uint32_t pc, std::string* error) const {
+	                DescriptorSource& descriptor, uint32_t pc, std::string* error) const {
 		if (handle.NumArgs() != width) {
 			return Fail(pc, error,
 			            fmt::format("{} has {} descriptor dwords, expected {}",
@@ -222,6 +271,9 @@ private:
 		for (uint32_t i = 0; i < width; i++) {
 			descriptor.dwords[i] = handle.Arg(i).Resolve();
 		}
+		if (sample_adjust) {
+			descriptor.dwords[3] = CanonicalizeSampleAdjustDword3(descriptor.dwords[3]);
+		}
 		const auto dword0 = descriptor.dwords[0].Resolve();
 		if (sampler && dword0.IsImmediate() && dword0.GetType() == Type::U32 &&
 		    (dword0.U32() & SamplerBorderClampMask) == 0) {
@@ -316,7 +368,7 @@ private:
 	bool MakeRuntimeBufferSource(const Inst& handle, uint32_t pc, uint32_t& source,
 	                             DescriptorSource& descriptor) {
 		if (handle.GetOpcode() != ValueOpcode::GetBufferResource ||
-		    !MakeSource(handle, 4u, false, descriptor, pc, nullptr)) {
+		    !MakeSource(handle, 4u, false, false, descriptor, pc, nullptr)) {
 			return false;
 		}
 		std::string reason;
@@ -499,14 +551,15 @@ private:
 	}
 
 	bool GetHandle(Value value, ValueOpcode expected, uint32_t width, uint32_t pc, Inst*& handle,
-	               uint32_t& source, std::string* error, bool sampler = false) {
+	               uint32_t& source, std::string* error, bool sampler = false,
+	               bool sample_adjust = false) {
 		handle = value.Resolve().TryInstruction();
 		if (handle == nullptr || handle->GetOpcode() != expected) {
 			return Fail(pc, error,
 			            fmt::format("memory operation requires {}", ValueOpcodeName(expected)));
 		}
 		DescriptorSource descriptor;
-		if (!MakeSource(*handle, width, sampler, descriptor, pc, error)) {
+		if (!MakeSource(*handle, width, sampler, sample_adjust, descriptor, pc, error)) {
 			return false;
 		}
 		std::string reason;
@@ -539,7 +592,7 @@ private:
 				source  = UINT32_MAX;
 				return true;
 			}
-		} else if (!MakeSource(*handle, 2, false, descriptor, pc, error)) {
+		} else if (!MakeSource(*handle, 2, false, false, descriptor, pc, error)) {
 			return false;
 		}
 		std::string reason;
@@ -813,8 +866,10 @@ private:
 			}
 			Inst*    sampler_handle = nullptr;
 			uint32_t sampler_source = 0;
+			const bool sample_adjust =
+			    (memory.image_sample_flags & Decoder::ImageSampleFlagAdjust) != 0;
 			if (!GetHandle(inst.Arg(1), ValueOpcode::GetSamplerResource, 4, flags.pc,
-			               sampler_handle, sampler_source, error, true)) {
+			               sampler_handle, sampler_source, error, true, sample_adjust)) {
 				return false;
 			}
 			sampler = AddSampler(sampler_source, flags.pc);

tests/ResourceTrackingTests.cpp

@@ -633,6 +633,93 @@ void TestImagesSamplersAndAliases() {
         "buffer/image descriptor alias was not linked");
 }
 
+void TestSampleAdjustSamplerScratch() {
+  Fixture fixture(ShaderType::Pixel);
+  const auto active = fixture.Emit(ValueOpcode::WqmMask, {Value(true)});
+  const auto lane = fixture.Emit(ValueOpcode::SelectU32,
+                                 {active, Value(1u), Value(0u)});
+  const auto low = fixture.Emit(ValueOpcode::BitwiseAnd32,
+                                {lane, Value(0xffu)});
+  const auto high = fixture.Emit(ValueOpcode::BitwiseAnd32,
+                                 {lane, Value(0xffu)});
+  const auto quads = fixture.Emit(
+      ValueOpcode::BitwiseOr32,
+      {low, fixture.Emit(ValueOpcode::ShiftLeftLogical32,
+                         {high, Value(8u)})});
+  const auto scratch = fixture.Emit(ValueOpcode::ShiftLeftLogical32,
+                                    {quads, Value(12u)});
+  const auto word3 = fixture.Emit(ValueOpcode::BitwiseOr32,
+                                  {fixture.UserData(3), scratch});
+  const auto image = fixture.Image(
+      {Value(0u), Value(0u), Value(0u), Value(0u), Value(0u), Value(0u),
+       Value(0u), Value(0u)},
+      0x1ec);
+  const auto sampler = fixture.Sampler(
+      {fixture.UserData(0), fixture.UserData(1), fixture.UserData(2), word3},
+      0x1ec);
+  MemoryInfo memory;
+  memory.kind = ResourceKind::Image;
+  memory.image_dimension = Decoder::ImageDimension::Dim2D;
+  memory.image_sample_flags = Decoder::ImageSampleFlagAdjust;
+  fixture.Emit(ValueOpcode::ImageSampleRaw,
+               {image, sampler, fixture.ImageAddress()},
+               fixture.AddMemory(memory, 0x1ec));
+  fixture.PlanAndTrack();
+
+  const auto source = fixture.program.info.samplers[0].source;
+  const auto stored = fixture.program.values->descriptor_sources[source]
+                          .dwords[3]
+                          .Resolve()
+                          .TryInstruction();
+  Check(stored != nullptr && stored->GetOpcode() == ValueOpcode::GetUserData,
+        "SampleAdjust reserved scratch remained in sampler identity");
+  std::array<uint32_t, 4> user_data{4u, 1u, 2u, 0x80000abcu};
+  SrtRuntime runtime{.user_data = user_data};
+  DescriptorValue descriptor;
+  std::string error;
+  Check(EvaluateDescriptorSource(fixture.program, source, 0x1ec, runtime,
+                                 descriptor, &error) &&
+            descriptor.dwords[3] == 0x80000abcu,
+        "SampleAdjust canonicalization lost sampler border fields");
+
+  const auto CheckRejected = [](uint32_t flags, uint32_t shift,
+                                const char *message) {
+    Fixture rejected(ShaderType::Pixel);
+    const auto condition =
+        rejected.Emit(ValueOpcode::WqmMask, {Value(true)});
+    const auto bit = rejected.Emit(ValueOpcode::SelectU32,
+                                   {condition, Value(1u), Value(0u)});
+    const auto dynamic = rejected.Emit(ValueOpcode::ShiftLeftLogical32,
+                                       {bit, Value(shift)});
+    const auto dynamic_word3 = rejected.Emit(
+        ValueOpcode::BitwiseOr32, {rejected.UserData(3), dynamic});
+    const auto rejected_image = rejected.Image(
+        {Value(0u), Value(0u), Value(0u), Value(0u), Value(0u), Value(0u),
+         Value(0u), Value(0u)},
+        0x200);
+    const auto rejected_sampler = rejected.Sampler(
+        {rejected.UserData(0), rejected.UserData(1), rejected.UserData(2),
+         dynamic_word3},
+        0x200);
+    MemoryInfo rejected_memory;
+    rejected_memory.kind = ResourceKind::Image;
+    rejected_memory.image_dimension = Decoder::ImageDimension::Dim2D;
+    rejected_memory.image_sample_flags = flags;
+    rejected.Emit(ValueOpcode::ImageSampleRaw,
+                  {rejected_image, rejected_sampler, rejected.ImageAddress()},
+                  rejected.AddMemory(rejected_memory, 0x200));
+    std::string rejected_error;
+    Check(BuildSrtPlan(rejected.program, &rejected_error) &&
+              !TrackResources(rejected.program, &rejected_error) &&
+              rejected_error.find("WqmMask") != std::string::npos,
+          message);
+  };
+  CheckRejected(0u, 12u,
+                "ordinary sampling accepted SampleAdjust reserved scratch");
+  CheckRejected(Decoder::ImageSampleFlagAdjust, 30u,
+                "SampleAdjust canonicalization discarded border-mode bits");
+}
+
 void TestDynamicStorageMipTracking() {
   Fixture fixture;
   std::array<Value, 8> image_words;
@@ -1234,6 +1321,7 @@ int main() {
     Run("scalar/vector alias", TestScalarAndVectorBufferAlias);
     Run("runtime unsigned min", TestRuntimeUnsignedMinDescriptor);
     Run("images and samplers", TestImagesSamplersAndAliases);
+    Run("SampleAdjust sampler scratch", TestSampleAdjustSamplerScratch);
     Run("dynamic storage mips", TestDynamicStorageMipTracking);
     Run("invariant indirect images", TestInvariantIndirectImageMaterialization);
     Run("SRT runtime", TestSrtFlatteningAndRuntimeMemoization);

tests/shaderCfgTests.cpp

@@ -4550,9 +4550,11 @@ void TestNewShaderRecompilerImageSampleOpcodeAliases() {
         "MIMG opcode 0xa8 should decode as image_sample_c_a alias");
   Check(Common::ContainsStr(result.decoded_dump, "image_sample_c_b_a"),
         "MIMG opcode 0xad should decode as image_sample_c_b_a alias");
-  Check(Common::ContainsStr(result.decoded_dump, "sample_flags=none"),
+  Check(Common::ContainsStr(result.decoded_dump, "sample_flags=adjust"),
-        "opcode 0xa0 alias should use normal 32-bit sample coordinates");
+        "opcode 0xa0 alias should expose SampleAdjust with normal 32-bit "
-  Check(Common::ContainsStr(result.decoded_dump, "sample_flags=bias|compare"),
+        "coordinates");
+  Check(Common::ContainsStr(result.decoded_dump,
+                            "sample_flags=bias|compare|adjust"),
         "compare+bias opcode alias did not expose expected sample flags");
   Check(!Common::ContainsStr(result.decoded_dump, "a16"),
         "A16 must come from MIMG bit 62, not from the opcode alias");