shader: add missing 64-bit integer image atomics

Co-authored-by: shadowbeat070 <70843316+shadowbeat070@users.noreply.github.com>

src/graphics/shader/recompiler/backend/spirv/spirvEmitterInstructions.h

@@ -321,15 +321,21 @@ inline constexpr auto EmitImageSampleRaw       = EmitImage;
 inline constexpr auto EmitImageGatherRaw       = EmitImage;
 inline constexpr auto EmitImageAtomicCompareSwap32 = EmitImage;
 inline constexpr auto EmitImageAtomicSwap32    = EmitImage;
+inline constexpr auto EmitImageAtomicSwap64    = EmitImage;
 inline constexpr auto EmitImageAtomicIAdd32    = EmitImage;
+inline constexpr auto EmitImageAtomicIAdd64    = EmitImage;
 inline constexpr auto EmitImageAtomicSMin32    = EmitImage;
 inline constexpr auto EmitImageAtomicUMin32    = EmitImage;
+inline constexpr auto EmitImageAtomicUMin64    = EmitImage;
 inline constexpr auto EmitImageAtomicSMax32    = EmitImage;
 inline constexpr auto EmitImageAtomicUMax32    = EmitImage;
 inline constexpr auto EmitImageAtomicUMax64    = EmitImage;
 inline constexpr auto EmitImageAtomicAnd32     = EmitImage;
+inline constexpr auto EmitImageAtomicAnd64     = EmitImage;
 inline constexpr auto EmitImageAtomicOr32      = EmitImage;
+inline constexpr auto EmitImageAtomicOr64      = EmitImage;
 inline constexpr auto EmitImageAtomicXor32     = EmitImage;
+inline constexpr auto EmitImageAtomicXor64     = EmitImage;
 inline constexpr auto EmitImageAtomicFMin32    = EmitImage;
 inline constexpr auto EmitImageAtomicFMax32    = EmitImage;
 void                  EmitUnreachable(ValueEmitContext& ctx, const IR::Inst& inst);

src/graphics/shader/recompiler/backend/spirv/spirvEmitterMemoryHelpers.cpp

@@ -361,10 +361,12 @@ spv::Op SpirvAtomicOpcode(IR::ValueOpcode opcode) {
 		case IR::ValueOpcode::ImageAtomicCompareSwap32:
 		case IR::ValueOpcode::BufferAtomicCmpSwap32: return spv::OpAtomicCompareExchange;
 		case IR::ValueOpcode::ImageAtomicSwap32:
+		case IR::ValueOpcode::ImageAtomicSwap64:
 		case IR::ValueOpcode::BufferAtomicSwap32:
 		case IR::ValueOpcode::BufferAtomicSwap64:
 		case IR::ValueOpcode::SharedAtomicSwap32: return spv::OpAtomicExchange;
 		case IR::ValueOpcode::ImageAtomicIAdd32:
+		case IR::ValueOpcode::ImageAtomicIAdd64:
 		case IR::ValueOpcode::BufferAtomicIAdd32:
 		case IR::ValueOpcode::SharedAtomicIAdd64:
 		case IR::ValueOpcode::SharedAtomicIAdd32: return spv::OpAtomicIAdd;
@@ -374,6 +376,7 @@ spv::Op SpirvAtomicOpcode(IR::ValueOpcode opcode) {
 		case IR::ValueOpcode::BufferAtomicSMin32:
 		case IR::ValueOpcode::SharedAtomicSMin32: return spv::OpAtomicSMin;
 		case IR::ValueOpcode::ImageAtomicUMin32:
+		case IR::ValueOpcode::ImageAtomicUMin64:
 		case IR::ValueOpcode::BufferAtomicUMin32:
 		case IR::ValueOpcode::SharedAtomicUMin32: return spv::OpAtomicUMin;
 		case IR::ValueOpcode::ImageAtomicSMax32:
@@ -384,15 +387,18 @@ spv::Op SpirvAtomicOpcode(IR::ValueOpcode opcode) {
 		case IR::ValueOpcode::BufferAtomicUMax32:
 		case IR::ValueOpcode::SharedAtomicUMax32: return spv::OpAtomicUMax;
 		case IR::ValueOpcode::ImageAtomicAnd32:
+		case IR::ValueOpcode::ImageAtomicAnd64:
 		case IR::ValueOpcode::BufferAtomicAnd32:
 		case IR::ValueOpcode::BufferAtomicAnd64:
 		case IR::ValueOpcode::SharedAtomicAnd32: return spv::OpAtomicAnd;
 		case IR::ValueOpcode::ImageAtomicOr32:
+		case IR::ValueOpcode::ImageAtomicOr64:
 		case IR::ValueOpcode::BufferAtomicOr32:
 		case IR::ValueOpcode::BufferAtomicOr64:
 		case IR::ValueOpcode::SharedAtomicOr64:
 		case IR::ValueOpcode::SharedAtomicOr32: return spv::OpAtomicOr;
 		case IR::ValueOpcode::ImageAtomicXor32:
+		case IR::ValueOpcode::ImageAtomicXor64:
 		case IR::ValueOpcode::BufferAtomicXor32:
 		case IR::ValueOpcode::SharedAtomicXor32: return spv::OpAtomicXor;
 		default: return spv::OpNop;

src/graphics/shader/recompiler/frontend/decode/ImageOps.cpp

@@ -20,6 +20,12 @@ struct MimgGatherInfo {
 	uint32_t flags    = 0;
 };
 
+struct MimgAtomicInfo {
+	uint32_t encoding    = 0;
+	Opcode   decoded     = Opcode::UNSUPPORTED;
+	bool     supports_64 = false;
+};
+
 constexpr ImageDimension DecodeImageDimension(uint32_t dim) {
 	switch (dim) {
 		case 0u: return ImageDimension::Dim1D;
@@ -190,17 +196,17 @@ constexpr MimgGatherInfo MIMG_GATHER_OPCODE_LIST[] = {
     {0x61u, Opcode::IMAGE_GATHER4H, ImageSampleFlagGatherHorizontal},
 };
 
-constexpr Detail::OpcodeMap MIMG_ATOMIC_OPCODE_LIST[] = {
+constexpr MimgAtomicInfo MIMG_ATOMIC_OPCODE_LIST[] = {
-    {0x0fu, Opcode::IMAGE_ATOMIC_SWAP},
+    {0x0fu, Opcode::IMAGE_ATOMIC_SWAP, true},
     {0x10u, Opcode::IMAGE_ATOMIC_CMPSWAP},
-    {0x11u, Opcode::IMAGE_ATOMIC_ADD},
+    {0x11u, Opcode::IMAGE_ATOMIC_ADD, true},
     {0x14u, Opcode::IMAGE_ATOMIC_SMIN},
-    {0x15u, Opcode::IMAGE_ATOMIC_UMIN},
+    {0x15u, Opcode::IMAGE_ATOMIC_UMIN, true},
     {0x16u, Opcode::IMAGE_ATOMIC_SMAX},
-    {0x17u, Opcode::IMAGE_ATOMIC_UMAX},
+    {0x17u, Opcode::IMAGE_ATOMIC_UMAX, true},
-    {0x18u, Opcode::IMAGE_ATOMIC_AND},
+    {0x18u, Opcode::IMAGE_ATOMIC_AND, true},
-    {0x19u, Opcode::IMAGE_ATOMIC_OR},
+    {0x19u, Opcode::IMAGE_ATOMIC_OR, true},
-    {0x1au, Opcode::IMAGE_ATOMIC_XOR},
+    {0x1au, Opcode::IMAGE_ATOMIC_XOR, true},
     {0x1eu, Opcode::IMAGE_ATOMIC_FMIN},
     {0x1fu, Opcode::IMAGE_ATOMIC_FMAX},
 };
@@ -210,7 +216,7 @@ constexpr auto MIMG_GATHER_OPS = Detail::MakeOpcodeTable<0x100>(MIMG_GATHER_OPCO
 constexpr auto MIMG_ATOMIC_OPS = Detail::MakeOpcodeTable<0x100>(MIMG_ATOMIC_OPCODE_LIST);
 
 Opcode DecodeMimgOpcode(uint32_t opcode, const MimgSampleInfo* sample, const MimgGatherInfo* gather,
-                        const Detail::OpcodeMap* atomic) {
+                        const MimgAtomicInfo* atomic) {
 	if (sample != nullptr) {
 		return Opcode::IMAGE_SAMPLE;
 	}
@@ -246,7 +252,7 @@ uint32_t DecodeMimgSampleFlags(const MimgSampleInfo* sample, const MimgGatherInf
 
 uint32_t DecodeMimgAddressComponents(uint32_t opcode, ImageDimension dimension,
                                      const MimgSampleInfo* sample, const MimgGatherInfo* gather,
-                                     const Detail::OpcodeMap* atomic) {
+                                     const MimgAtomicInfo* atomic) {
 	if (sample != nullptr) {
 		return ImageSampleAddressComponents(sample->flags, dimension);
 	}
@@ -370,7 +376,7 @@ void DecodeMimg(uint32_t pc, std::span<const uint32_t> code, uint32_t word_index
 			SetUnsupported(inst, Family::MIMG, opcode, "MIMG image atomic has invalid DMASK");
 		} else if (inst.dmask == mask64) {
 			inst.data_bits = 64u;
-			if (inst.opcode != Opcode::IMAGE_ATOMIC_UMAX) {
+			if (!atomic->supports_64) {
 				SetUnsupported(inst, Family::MIMG, opcode,
 				               "MIMG 64-bit image atomic opcode is not implemented");
 			}

src/graphics/shader/recompiler/frontend/translate/Memory.cpp

@@ -546,7 +546,10 @@ void Translator::BUFFER_ATOMIC(const Decoder::Instruction& inst, IR::ValueOpcode
 	}
 }
 
-void Translator::IMAGE_ATOMIC(const Decoder::Instruction& inst, IR::ValueOpcode opcode) {
+void Translator::IMAGE_ATOMIC(const Decoder::Instruction& inst, IR::ValueOpcode opcode32,
+                               IR::ValueOpcode opcode64) {
+	const auto opcode = inst.data_bits == 64u ? opcode64 : opcode32;
+	EXIT_IF(opcode == IR::ValueOpcode::Count);
 	const auto memory   = MemoryInfoFromDecoded(inst);
 	const auto resource = GetImageResource(memory);
 	const auto address  = MakeImageAddress(inst, MemorySourceAt(inst, 1));
@@ -1041,24 +1044,30 @@ void Translator::EmitMemory(const Decoder::Instruction& inst) {
 		case Decoder::Opcode::IMAGE_ATOMIC_CMPSWAP:
 			return IMAGE_ATOMIC(inst, IR::ValueOpcode::ImageAtomicCompareSwap32);
 		case Decoder::Opcode::IMAGE_ATOMIC_SWAP:
-			return IMAGE_ATOMIC(inst, IR::ValueOpcode::ImageAtomicSwap32);
+			return IMAGE_ATOMIC(inst, IR::ValueOpcode::ImageAtomicSwap32,
+			                    IR::ValueOpcode::ImageAtomicSwap64);
 		case Decoder::Opcode::IMAGE_ATOMIC_ADD:
-			return IMAGE_ATOMIC(inst, IR::ValueOpcode::ImageAtomicIAdd32);
+			return IMAGE_ATOMIC(inst, IR::ValueOpcode::ImageAtomicIAdd32,
+			                    IR::ValueOpcode::ImageAtomicIAdd64);
 		case Decoder::Opcode::IMAGE_ATOMIC_SMIN:
 			return IMAGE_ATOMIC(inst, IR::ValueOpcode::ImageAtomicSMin32);
 		case Decoder::Opcode::IMAGE_ATOMIC_UMIN:
-			return IMAGE_ATOMIC(inst, IR::ValueOpcode::ImageAtomicUMin32);
+			return IMAGE_ATOMIC(inst, IR::ValueOpcode::ImageAtomicUMin32,
+			                    IR::ValueOpcode::ImageAtomicUMin64);
 		case Decoder::Opcode::IMAGE_ATOMIC_SMAX:
 			return IMAGE_ATOMIC(inst, IR::ValueOpcode::ImageAtomicSMax32);
 		case Decoder::Opcode::IMAGE_ATOMIC_UMAX:
-			return IMAGE_ATOMIC(inst, inst.data_bits == 64u ? IR::ValueOpcode::ImageAtomicUMax64
+			return IMAGE_ATOMIC(inst, IR::ValueOpcode::ImageAtomicUMax32,
-			                                                : IR::ValueOpcode::ImageAtomicUMax32);
+			                    IR::ValueOpcode::ImageAtomicUMax64);
 		case Decoder::Opcode::IMAGE_ATOMIC_AND:
-			return IMAGE_ATOMIC(inst, IR::ValueOpcode::ImageAtomicAnd32);
+			return IMAGE_ATOMIC(inst, IR::ValueOpcode::ImageAtomicAnd32,
+			                    IR::ValueOpcode::ImageAtomicAnd64);
 		case Decoder::Opcode::IMAGE_ATOMIC_OR:
-			return IMAGE_ATOMIC(inst, IR::ValueOpcode::ImageAtomicOr32);
+			return IMAGE_ATOMIC(inst, IR::ValueOpcode::ImageAtomicOr32,
+			                    IR::ValueOpcode::ImageAtomicOr64);
 		case Decoder::Opcode::IMAGE_ATOMIC_XOR:
-			return IMAGE_ATOMIC(inst, IR::ValueOpcode::ImageAtomicXor32);
+			return IMAGE_ATOMIC(inst, IR::ValueOpcode::ImageAtomicXor32,
+			                    IR::ValueOpcode::ImageAtomicXor64);
 		case Decoder::Opcode::IMAGE_ATOMIC_FMIN:
 			return IMAGE_ATOMIC(inst, IR::ValueOpcode::ImageAtomicFMin32);
 		case Decoder::Opcode::IMAGE_ATOMIC_FMAX:

src/graphics/shader/recompiler/frontend/translate/Translator.h

@@ -86,7 +86,8 @@ private:
 	void          BUFFER_LOAD(const Decoder::Instruction& inst);
 	void          BUFFER_STORE(const Decoder::Instruction& inst);
 	void          BUFFER_ATOMIC(const Decoder::Instruction& inst, IR::ValueOpcode opcode);
-	void          IMAGE_ATOMIC(const Decoder::Instruction& inst, IR::ValueOpcode opcode);
+	void          IMAGE_ATOMIC(const Decoder::Instruction& inst, IR::ValueOpcode opcode32,
+	                           IR::ValueOpcode opcode64 = IR::ValueOpcode::Count);
 	void DS_ATOMIC(const Decoder::Instruction& inst, IR::ValueOpcode opcode, bool returns_value);
 	void FLAT_LOAD(const Decoder::Instruction& inst);
 	void FLAT_STORE(const Decoder::Instruction& inst);

src/graphics/shader/recompiler/ir/opcodes/ValueOpcodes.cpp

@@ -224,15 +224,21 @@ ImageOpcodeInfo ImageOpcodeInfoOf(ValueOpcode opcode) {
 			return {ImageAccess::Write, ImageResourceClass::Storage, false};
 		case ValueOpcode::ImageAtomicCompareSwap32:
 		case ValueOpcode::ImageAtomicSwap32:
+		case ValueOpcode::ImageAtomicSwap64:
 		case ValueOpcode::ImageAtomicIAdd32:
+		case ValueOpcode::ImageAtomicIAdd64:
 		case ValueOpcode::ImageAtomicSMin32:
 		case ValueOpcode::ImageAtomicUMin32:
+		case ValueOpcode::ImageAtomicUMin64:
 		case ValueOpcode::ImageAtomicSMax32:
 		case ValueOpcode::ImageAtomicUMax32:
 		case ValueOpcode::ImageAtomicUMax64:
 		case ValueOpcode::ImageAtomicAnd32:
+		case ValueOpcode::ImageAtomicAnd64:
 		case ValueOpcode::ImageAtomicOr32:
+		case ValueOpcode::ImageAtomicOr64:
 		case ValueOpcode::ImageAtomicXor32:
+		case ValueOpcode::ImageAtomicXor64:
 		case ValueOpcode::ImageAtomicFMin32:
 		case ValueOpcode::ImageAtomicFMax32:
 			return {ImageAccess::Atomic, ImageResourceClass::Storage, false};

src/graphics/shader/recompiler/ir/opcodes/ValueOpcodes.inc

@@ -310,15 +310,21 @@ VALUE_OPCODE(ImageSampleRaw, U32x4, ImageResource, SamplerResource, ImageAddress
 VALUE_OPCODE(ImageGatherRaw, U32x4, ImageResource, SamplerResource, ImageAddress, U32, U32, U32, U32)
 VALUE_OPCODE(ImageAtomicCompareSwap32, U32, ImageResource, ImageAddress, U32, U32, U1)
 VALUE_OPCODE(ImageAtomicSwap32, U32, ImageResource, ImageAddress, U32, U1)
+VALUE_OPCODE(ImageAtomicSwap64, U64, ImageResource, ImageAddress, U64, U1)
 VALUE_OPCODE(ImageAtomicIAdd32, U32, ImageResource, ImageAddress, U32, U1)
+VALUE_OPCODE(ImageAtomicIAdd64, U64, ImageResource, ImageAddress, U64, U1)
 VALUE_OPCODE(ImageAtomicSMin32, U32, ImageResource, ImageAddress, U32, U1)
 VALUE_OPCODE(ImageAtomicUMin32, U32, ImageResource, ImageAddress, U32, U1)
+VALUE_OPCODE(ImageAtomicUMin64, U64, ImageResource, ImageAddress, U64, U1)
 VALUE_OPCODE(ImageAtomicSMax32, U32, ImageResource, ImageAddress, U32, U1)
 VALUE_OPCODE(ImageAtomicUMax32, U32, ImageResource, ImageAddress, U32, U1)
 VALUE_OPCODE(ImageAtomicUMax64, U64, ImageResource, ImageAddress, U64, U1)
 VALUE_OPCODE(ImageAtomicAnd32, U32, ImageResource, ImageAddress, U32, U1)
+VALUE_OPCODE(ImageAtomicAnd64, U64, ImageResource, ImageAddress, U64, U1)
 VALUE_OPCODE(ImageAtomicOr32, U32, ImageResource, ImageAddress, U32, U1)
+VALUE_OPCODE(ImageAtomicOr64, U64, ImageResource, ImageAddress, U64, U1)
 VALUE_OPCODE(ImageAtomicXor32, U32, ImageResource, ImageAddress, U32, U1)
+VALUE_OPCODE(ImageAtomicXor64, U64, ImageResource, ImageAddress, U64, U1)
 VALUE_OPCODE(ImageAtomicFMin32, U32, ImageResource, ImageAddress, U32, U1)
 VALUE_OPCODE(ImageAtomicFMax32, U32, ImageResource, ImageAddress, U32, U1)
 VALUE_OPCODE(GetAttribute, U32, U32, U32)

tests/ShaderRecompilerComputeTests.cpp

@@ -33646,20 +33646,68 @@ TestCase ImageAtomicVariants() {
   return test;
 }
 
-template <bool wide> TestCase ImageAtomicUMaxGlcAndExec() {
+// Input and old texel cover high-DWORD ordering, carry/wrap, the unsigned sign
-  // Input and old texel: high DWORD ordering, unsigned sign boundary, and
+// boundary, and equal-high-DWORD ordering through the RG32_UINT -> R64_UINT view.
-  // equal-high-DWORD ordering must all survive the RG32_UINT -> R64_UINT view.
+constexpr uint64_t ImageAtomicInputs[][2] = {
-  constexpr uint64_t values[][2] = {
+    {0x0000000100000001ull, 0x00000000ffffffffull},
-      {0x0000000100000001ull, 0x00000000ffffffffull},
+    {0x00000000ffffffffull, 0x0000000100000001ull},
-      {0x00000000ffffffffull, 0x0000000100000001ull},
+    {0x8000000000000000ull, 0x7fffffffffffffffull},
-      {0x8000000000000000ull, 0x7fffffffffffffffull},
+    {0x12345678ffffffffull, 0x1234567800000001ull},
-      {0x12345678ffffffffull, 0x1234567800000001ull},
+    {0xffffffffffffffffull, 0x8000000000000000ull},
-      {0xffffffffffffffffull, 0x8000000000000000ull},
+};
-  };
+
-  constexpr u32 texel_dwords = wide ? 2 : 1;
+struct ImageAtomicIntegerCase {
+  u32 encoding;
+  ShaderOpcode opcode;
+  const char *name32;
+  const char *name64;
+  const char *spirv;
+  uint64_t expected64[std::size(ImageAtomicInputs)];
+  u32 expected32[std::size(ImageAtomicInputs)];
+};
+
+constexpr ImageAtomicIntegerCase ImageAtomicIntegerCases[] = {
+    {0x0fu, ShaderOpcode::IMAGE_ATOMIC_SWAP,
+     "ImageAtomicSwap32GlcAndExec", "ImageAtomicSwap64GlcAndExec", "OpAtomicExchange",
+     {0x0000000100000001ull, 0x00000000ffffffffull, 0x8000000000000000ull,
+      0x12345678ffffffffull, 0xffffffffffffffffull},
+     {0x00000001u, 0xffffffffu, 0x00000000u, 0xffffffffu, 0xffffffffu}},
+    {0x11u, ShaderOpcode::IMAGE_ATOMIC_ADD,
+     "ImageAtomicAdd32GlcAndExec", "ImageAtomicAdd64GlcAndExec", "OpAtomicIAdd",
+     {0x0000000200000000ull, 0x0000000200000000ull, 0xffffffffffffffffull,
+      0x2468acf100000000ull, 0x7fffffffffffffffull},
+     {0x00000000u, 0x00000000u, 0xffffffffu, 0x00000000u, 0xffffffffu}},
+    {0x15u, ShaderOpcode::IMAGE_ATOMIC_UMIN,
+     "ImageAtomicUMin32GlcAndExec", "ImageAtomicUMin64GlcAndExec", "OpAtomicUMin",
+     {0x00000000ffffffffull, 0x00000000ffffffffull, 0x7fffffffffffffffull,
+      0x1234567800000001ull, 0x8000000000000000ull},
+     {0x00000001u, 0x00000001u, 0x00000000u, 0x00000001u, 0x00000000u}},
+    {0x17u, ShaderOpcode::IMAGE_ATOMIC_UMAX,
+     "ImageAtomicUMax32GlcAndExec", "ImageAtomicUMax64GlcAndExec", "OpAtomicUMax",
+     {0x0000000100000001ull, 0x0000000100000001ull, 0x8000000000000000ull,
+      0x12345678ffffffffull, 0xffffffffffffffffull},
+     {0xffffffffu, 0xffffffffu, 0xffffffffu, 0xffffffffu, 0xffffffffu}},
+    {0x18u, ShaderOpcode::IMAGE_ATOMIC_AND,
+     "ImageAtomicAnd32GlcAndExec", "ImageAtomicAnd64GlcAndExec", "OpAtomicAnd",
+     {0x0000000000000001ull, 0x0000000000000001ull, 0x0000000000000000ull,
+      0x1234567800000001ull, 0x8000000000000000ull},
+     {0x00000001u, 0x00000001u, 0x00000000u, 0x00000001u, 0x00000000u}},
+    {0x19u, ShaderOpcode::IMAGE_ATOMIC_OR,
+     "ImageAtomicOr32GlcAndExec", "ImageAtomicOr64GlcAndExec", "OpAtomicOr",
+     {0x00000001ffffffffull, 0x00000001ffffffffull, 0xffffffffffffffffull,
+      0x12345678ffffffffull, 0xffffffffffffffffull},
+     {0xffffffffu, 0xffffffffu, 0xffffffffu, 0xffffffffu, 0xffffffffu}},
+    {0x1au, ShaderOpcode::IMAGE_ATOMIC_XOR,
+     "ImageAtomicXor32GlcAndExec", "ImageAtomicXor64GlcAndExec", "OpAtomicXor",
+     {0x00000001fffffffeull, 0x00000001fffffffeull, 0xffffffffffffffffull,
+      0x00000000fffffffeull, 0x7fffffffffffffffull},
+     {0xfffffffeu, 0xfffffffeu, 0xffffffffu, 0xfffffffeu, 0xffffffffu}},
+};
+
+TestCase ImageAtomicIntegerGlcAndExec(const ImageAtomicIntegerCase &atomic, bool wide) {
+  const u32 texel_dwords = wide ? 2 : 1;
   TestCase test;
-  test.name = wide ? "ImageAtomicUMax64CapturedGlcAndExec"
+  test.name = wide ? atomic.name64 : atomic.name32;
-                   : "ImageAtomicUMax32GlcAndExec";
   test.user_data = MakeStorageTextureData(wide ? Prospero::BufferFormat::k32_32UInt
                                               : Prospero::BufferFormat::k32UInt);
   test.has_user_data = true;
@@ -33667,25 +33715,21 @@ template <bool wide> TestCase ImageAtomicUMaxGlcAndExec() {
   test.expected_storage_image_r32ui = test.storage_image_r32ui;
   auto &code = test.code;
   code.push_back(EncodeSop1(0x04, 12, 126)); // Preserve EXEC.
-  for (u32 i = 0; i < std::size(values); ++i) {
+  for (u32 i = 0; i < std::size(ImageAtomicInputs); ++i) {
-    const auto input = values[i][0];
+    const auto input = ImageAtomicInputs[i][0];
-    const auto old = values[i][1];
+    const auto old = ImageAtomicInputs[i][1];
     const bool glc = i != 0 && i != 4;
     AppendVMovU32(&code, 0, i & 3u);
     AppendVMovU32(&code, 1, i >> 2u);
     AppendVMovLiteral(&code, 2, static_cast<u32>(input));
     AppendVMovLiteral(&code, 3, static_cast<u32>(input >> 32u));
-    // Captured PS5 instruction: f05c0308 00000200, DMASK=3, VDATA=v[2:3].
+    code.push_back(EncodeMimg0(atomic.encoding, wide ? 3u : 1u, 0, glc));
-    code.insert(code.end(), {0xf05c0108u | (wide ? 0x200u : 0u) |
+    code.push_back(EncodeMimg1(2, 0));
-                                (glc ? 0x2000u : 0u),
-                            0x00000200u});
     AppendStoreVgpr(&code, 2, i * 2);
     AppendStoreVgpr(&code, 3, i * 2 + 1);
     test.expected.push_back(static_cast<u32>(glc ? old : input));
     test.expected.push_back(static_cast<u32>((wide && glc ? old : input) >> 32u));
-    const uint64_t result = wide ? std::max(input, old)
+    const uint64_t result = wide ? atomic.expected64[i] : atomic.expected32[i];
-                                 : std::max(static_cast<u32>(input),
-                                            static_cast<u32>(old));
     for (u32 word = 0; word < texel_dwords; ++word) {
       test.storage_image_r32ui[i * texel_dwords + word] =
           static_cast<u32>(old >> (word * 32u));
@@ -33693,23 +33737,30 @@ template <bool wide> TestCase ImageAtomicUMaxGlcAndExec() {
           static_cast<u32>(result >> (word * 32u));
     }
   }
+  // A leaked update must change the inactive texel for every operation.
+  const bool clear_bits = atomic.opcode == ShaderOpcode::IMAGE_ATOMIC_AND ||
+                          atomic.opcode == ShaderOpcode::IMAGE_ATOMIC_UMIN;
+  const u32 inactive = clear_bits ? 0xffffffffu : 0u;
+  for (u32 word = 0; word < texel_dwords; ++word) {
+    test.storage_image_r32ui[5u * texel_dwords + word] = inactive;
+    test.expected_storage_image_r32ui[5u * texel_dwords + word] = inactive;
+  }
   AppendVMovU32(&code, 0, 1);
-  AppendVMovU32(&code, 1, 1); // Untouched zero texel, so an EXEC leak is observable.
+  AppendVMovU32(&code, 1, 1);
   AppendVMovLiteral(&code, 2, 0xabcdef01u);
   AppendVMovLiteral(&code, 3, 0x87654321u);
   code.push_back(EncodeSop1(0x04, 126, InlineU32(0)));
-  code.insert(code.end(), {0xf05c2108u | (wide ? 0x200u : 0u), 0x00000200u});
+  code.push_back(EncodeMimg0(atomic.encoding, wide ? 3u : 1u, 0, true));
+  code.push_back(EncodeMimg1(2, 0));
   code.push_back(EncodeSop1(0x04, 126, 12));
-  AppendStoreVgpr(&code, 2, std::size(values) * 2);
+  AppendStoreVgpr(&code, 2, std::size(ImageAtomicInputs) * 2);
-  AppendStoreVgpr(&code, 3, std::size(values) * 2 + 1);
+  AppendStoreVgpr(&code, 3, std::size(ImageAtomicInputs) * 2 + 1);
   test.expected.insert(test.expected.end(), {0xabcdef01u, 0x87654321u});
   AppendEnd(&code);
   test.opcodes = {ShaderOpcode::V_MOV_B32, ShaderOpcode::S_MOV_B64,
-                 ShaderOpcode::IMAGE_ATOMIC_UMAX, ShaderOpcode::BUFFER_STORE_DWORD,
+                 atomic.opcode, ShaderOpcode::BUFFER_STORE_DWORD, ShaderOpcode::S_ENDPGM};
-                 ShaderOpcode::S_ENDPGM};
+  test.required_spirv = {atomic.spirv, "OpImageTexelPointer", wide ? "R64ui" : "R32ui"};
-  test.required_spirv = {"OpAtomicUMax", "OpImageTexelPointer",
+  if (wide) {
-                         wide ? "R64ui" : "R32ui"};
-  if constexpr (wide) {
     test.required_spirv.push_back("Int64ImageEXT");
     test.required_spirv.push_back("SPV_EXT_shader_image_int64");
   }
@@ -34802,8 +34853,10 @@ std::vector<TestCase> MakeCases() {
   AddCase(ImageAtomicSwapReturnsPreviousTexel);
   AddCase(ImageStoreAndAtomicUseSeparateBindings);
   AddCase(ImageAtomicVariants);
-  AddCase(ImageAtomicUMaxGlcAndExec<false>);
+  for (const auto &atomic : ImageAtomicIntegerCases) {
-  AddCase(ImageAtomicUMaxGlcAndExec<true>);
+    cases.push_back(ImageAtomicIntegerGlcAndExec(atomic, false));
+    cases.push_back(ImageAtomicIntegerGlcAndExec(atomic, true));
+  }
   AddCase(ImageAtomicFloatGlcAndExec<true>);
   AddCase(ImageAtomicFloatGlcAndExec<false>);
   AddCase(ImageAtomicFloatSpecialValues<true>);
@@ -40121,8 +40174,10 @@ int main(int argc, char **argv) {
   }
   if (argc == 2 && std::strcmp(argv[1], "--image-atomic64-only") == 0) {
     VulkanHarness vulkan;
-    RunCase(&vulkan, ImageAtomicUMaxGlcAndExec<false>());
+    for (const auto &atomic : ImageAtomicIntegerCases) {
-    RunCase(&vulkan, ImageAtomicUMaxGlcAndExec<true>());
+      RunCase(&vulkan, ImageAtomicIntegerGlcAndExec(atomic, false));
+      RunCase(&vulkan, ImageAtomicIntegerGlcAndExec(atomic, true));
+    }
     return 0;
   }
   if (argc == 2 && std::strcmp(argv[1], "--ds-atomics-only") == 0) {

tests/shaderCfgTests.cpp

@@ -4413,25 +4413,32 @@ void TestImageAtomicWidthDecoder() {
             decoded.dst.reg == 2u && decoded.dmask == 3u && !decoded.glc,
         "captured image atomic lost its 64-bit data width");
 
-  for (const auto opcode : {0x10u, 0x17u}) {
+  for (const auto opcode : {0x0fu, 0x10u, 0x11u, 0x14u, 0x15u, 0x16u,
+                            0x17u, 0x18u, 0x19u, 0x1au, 0x1eu, 0x1fu}) {
+    const bool compare_swap = opcode == 0x10u;
+    const bool supports_64 = opcode != 0x10u && opcode != 0x14u &&
+                             opcode != 0x16u && opcode != 0x1eu && opcode != 0x1fu;
     for (uint32_t mask = 0; mask < 16u; ++mask) {
-      const uint32_t words[] = {EncodeMimg0(opcode, mask), captured[1]};
+      for (const bool glc : {false, true}) {
-      DecodeInstruction(words, 0, decoded);
+        const uint32_t words[] = {EncodeMimg0(opcode, mask, glc), captured[1]};
-      const bool compare_swap = opcode == 0x10u;
+        DecodeInstruction(words, 0, decoded);
-      const bool supported = mask == (compare_swap ? 3u : 1u) ||
+        const bool supported = mask == (compare_swap ? 3u : 1u) ||
-                             (!compare_swap && mask == 3u);
+                               (supports_64 && mask == 3u);
-      Check((decoded.opcode != Opcode::UNSUPPORTED) == supported,
+        Check((decoded.opcode != Opcode::UNSUPPORTED) == supported,
-            "image atomic accepted an invalid or unsupported width mask");
+              "image atomic accepted an invalid or unsupported width mask");
-      if (supported) {
+        if (supported) {
-        Check(decoded.data_bits == (!compare_swap && mask == 3u ? 64u : 32u),
+          const auto bits = !compare_swap && mask == 3u ? 64u : 32u;
-              "image atomic DMASK selected the wrong data width");
+          Check(decoded.data_bits == bits && decoded.glc == glc &&
+                    decoded.data_dwords == (compare_swap ? 2u : bits / 32u),
+                "image atomic DMASK selected the wrong data width or return mode");
+        }
+        const uint32_t d16_words[] = {words[0], words[1] | (1u << 31u)};
+        DecodeInstruction(d16_words, 0, decoded);
+        Check(decoded.opcode == Opcode::UNSUPPORTED,
+              "image atomic incorrectly accepted D16 data");
       }
     }
   }
-  const uint32_t unsupported[] = {EncodeMimg0(0x11u, 3u), captured[1]};
-  DecodeInstruction(unsupported, 0, decoded);
-  Check(decoded.opcode == Opcode::UNSUPPORTED && decoded.data_bits == 64u,
-        "unsupported 64-bit image atomic silently decoded as 32-bit");
 }
 
 void TestNewShaderDecoderArchitecture() {