shader: snapshot wide SMEM sources before writes

src/graphics/shader/recompiler/frontend/translate/Dispatch.cpp

@@ -11,14 +11,16 @@ bool Translator::TranslateU64MaskOperation(const IR::Instruction& inst) {
 	           : TranslateComposedInteger(inst);
 }
 
-bool Translator::TranslateInstruction(const IR::Instruction&     inst,
+bool Translator::TranslateInstruction(const IR::Instruction&          inst,
-                                      const BufferAddressValues* address_snapshot) {
+                                      const BufferAddressValues*      address_snapshot,
+                                      const ScalarMemorySourceValues* scalar_source_snapshot) {
 	switch (IR::GetOpcodeInfo(inst.op).lowering_class) {
 		case IR::LoweringClass::Control: return TranslateControlOperation(inst);
 		case IR::LoweringClass::Move: return TranslateMove(inst);
 		case IR::LoweringClass::Lane: return TranslateLaneOperation(inst);
 		case IR::LoweringClass::State: return TranslateStateOperation(inst);
-		case IR::LoweringClass::Memory: return TranslateMemoryOperation(inst, address_snapshot);
+		case IR::LoweringClass::Memory:
+			return TranslateMemoryOperation(inst, address_snapshot, scalar_source_snapshot);
 		case IR::LoweringClass::Attribute: return TranslateAttributeOperation(inst);
 		case IR::LoweringClass::IntegerCompare: return TranslateIntegerCompare(inst);
 		case IR::LoweringClass::Integer16Compare: return TranslateInteger16Compare(inst);

src/graphics/shader/recompiler/frontend/translate/Memory.cpp

@@ -207,7 +207,19 @@ bool Translator::IsBufferLoadOperation(IR::Opcode opcode) {
 	       opcode == IR::Opcode::BufferLoadDword;
 }
 
-bool Translator::TranslateScalarMemory(const IR::Instruction& inst) {
+bool Translator::IsScalarMemoryLoadOperation(IR::Opcode opcode) {
+	return opcode == IR::Opcode::SLoadDword || opcode == IR::Opcode::SBufferLoadDword;
+}
+
+ScalarMemorySourceValues Translator::ReadScalarMemorySource(const IR::Instruction& inst) {
+	const auto resource = inst.op == IR::Opcode::SLoadDword
+	                          ? GetScalarAddressResource(inst.memory.resource)
+	                          : GetBufferResource(inst.memory);
+	return {resource, ReadU32(inst.src[0])};
+}
+
+bool Translator::TranslateScalarMemory(const IR::Instruction&          inst,
+                                       const ScalarMemorySourceValues* source_snapshot) {
 	switch (inst.op) {
 		case IR::Opcode::LoadSrtDword: {
 			const auto resource = ir.Emit(IR::ValueOpcode::GetSrtResource);
@@ -216,16 +228,18 @@ bool Translator::TranslateScalarMemory(const IR::Instruction& inst) {
 			return true;
 		}
 		case IR::Opcode::SBufferLoadDword: {
-			const auto resource = GetBufferResource(inst.memory);
+			const auto source =
+			    source_snapshot != nullptr ? *source_snapshot : ReadScalarMemorySource(inst);
 			WriteOperand(inst.dst, ir.Emit(IR::ValueOpcode::ReadConstBuffer,
-			                               {resource, ReadU32(inst.src[0])}, AddMemoryInfo(inst)));
+			                               {source.resource, source.offset}, AddMemoryInfo(inst)));
 			return true;
 		}
 		case IR::Opcode::SLoadDword: {
-			const auto resource = GetScalarAddressResource(inst.memory.resource);
+			const auto source =
+			    source_snapshot != nullptr ? *source_snapshot : ReadScalarMemorySource(inst);
 			WriteOperand(inst.dst,
 			             ir.Emit(IR::ValueOpcode::LoadAddressU32,
-			                     {resource, ReadU32(inst.src[0]), IR::Value(0u), IR::Value(true)},
+			                     {source.resource, source.offset, IR::Value(0u), IR::Value(true)},
 			                     AddMemoryInfo(inst)));
 			return true;
 		}
@@ -579,12 +593,14 @@ bool Translator::TranslateSharedMemory(const IR::Instruction& inst) {
 	}
 }
 
-bool Translator::TranslateMemoryOperation(const IR::Instruction&     inst,
+bool Translator::TranslateMemoryOperation(const IR::Instruction&          inst,
-                                          const BufferAddressValues* address_snapshot) {
+                                          const BufferAddressValues*      address_snapshot,
+                                          const ScalarMemorySourceValues* scalar_source_snapshot) {
 	switch (inst.op) {
 		case IR::Opcode::LoadSrtDword:
 		case IR::Opcode::SLoadDword:
-		case IR::Opcode::SBufferLoadDword: return TranslateScalarMemory(inst);
+		case IR::Opcode::SBufferLoadDword:
+			return TranslateScalarMemory(inst, scalar_source_snapshot);
 
 		case IR::Opcode::BufferLoadUbyte:
 		case IR::Opcode::BufferLoadSbyte:

src/graphics/shader/recompiler/frontend/translate/Translate.cpp

@@ -671,9 +671,12 @@ void Translator::WriteCompareResult(const IR::Operand& operand, IR::U1 value) {
 }
 
 bool Translator::TranslateBlock(const IR::BasicBlock& source, std::string* error) {
-	BufferAddressValues buffer_address_snapshot {};
+	BufferAddressValues      buffer_address_snapshot {};
-	uint32_t            buffer_address_pc           = 0;
+	ScalarMemorySourceValues scalar_source_snapshot {};
-	bool                has_buffer_address_snapshot = false;
+	uint32_t                 buffer_address_pc           = 0;
+	uint32_t                 scalar_source_pc            = 0;
+	bool                     has_buffer_address_snapshot = false;
+	bool                     has_scalar_source_snapshot  = false;
 	for (const auto& source_inst: source.instructions) {
 		current_opcode = source_inst.op;
 		current_pc     = source_inst.pc;
@@ -688,11 +691,23 @@ bool Translator::TranslateBlock(const IR::BasicBlock& source, std::string* error
 		} else if (!grouped_buffer_load) {
 			has_buffer_address_snapshot = false;
 		}
+		const bool grouped_scalar_load =
+		    IsScalarMemoryLoadOperation(source_inst.op) && source_inst.memory.component_count > 1u;
+		if (grouped_scalar_load &&
+		    (!has_scalar_source_snapshot || scalar_source_pc != source_inst.pc ||
+		     source_inst.memory.component_index == 0u)) {
+			scalar_source_snapshot     = ReadScalarMemorySource(source_inst);
+			scalar_source_pc           = source_inst.pc;
+			has_scalar_source_snapshot = true;
+		} else if (!grouped_scalar_load) {
+			has_scalar_source_snapshot = false;
+		}
 		if (static_cast<size_t>(source_inst.op) >= static_cast<size_t>(IR::Opcode::Count)) {
 			return Fail(error, "value IR input opcode is out of range");
 		}
 		if (TranslateInstruction(source_inst,
-		                         grouped_buffer_load ? &buffer_address_snapshot : nullptr)) {
+		                         grouped_buffer_load ? &buffer_address_snapshot : nullptr,
+		                         grouped_scalar_load ? &scalar_source_snapshot : nullptr)) {
 			continue;
 		}
 		return Fail(error, fmt::format("opcode {} at 0x{:08x} has no typed Value IR lowering",

src/graphics/shader/recompiler/frontend/translate/Translator.h

@@ -13,6 +13,11 @@ struct BufferAddressValues {
 	IR::U32 soffset;
 };
 
+struct ScalarMemorySourceValues {
+	IR::Value resource;
+	IR::U32   offset;
+};
+
 bool Fail(std::string* error, const std::string& message);
 
 class Translator {
@@ -80,21 +85,24 @@ private:
 	void            WriteImageComponents(const IR::Operand& dst, IR::Value value, uint32_t dmask,
 	                                     uint32_t component_limit);
 	IR::ValueOpcode ImageAtomicOpcode(IR::Opcode opcode);
-	BufferAddressValues ReadBufferAddress(const IR::Instruction& inst, uint32_t source_offset);
+	BufferAddressValues      ReadBufferAddress(const IR::Instruction& inst, uint32_t source_offset);
-	IR::U32             WidenSubdword(IR::Value value, uint32_t bits, bool sign);
+	IR::U32                  WidenSubdword(IR::Value value, uint32_t bits, bool sign);
-	IR::Value           NarrowSubdword(IR::U32 value, uint32_t bits);
+	IR::Value                NarrowSubdword(IR::U32 value, uint32_t bits);
-	IR::ValueOpcode     BufferAtomicOpcode(IR::Opcode opcode);
+	IR::ValueOpcode          BufferAtomicOpcode(IR::Opcode opcode);
-	IR::ValueOpcode     SharedAtomicOpcode(IR::Opcode opcode, bool gds);
+	IR::ValueOpcode          SharedAtomicOpcode(IR::Opcode opcode, bool gds);
-	bool                IsBufferLoadOperation(IR::Opcode opcode);
+	bool                     IsBufferLoadOperation(IR::Opcode opcode);
-	bool                TranslateScalarMemory(const IR::Instruction& inst);
+	bool                     IsScalarMemoryLoadOperation(IR::Opcode opcode);
-	bool                TranslateBufferLoad(const IR::Instruction&     inst,
+	ScalarMemorySourceValues ReadScalarMemorySource(const IR::Instruction& inst);
-	                                        const BufferAddressValues* address_snapshot);
+	bool                     TranslateScalarMemory(const IR::Instruction&          inst,
-	bool                TranslateBufferStore(const IR::Instruction& inst);
+	                                               const ScalarMemorySourceValues* source_snapshot);
-	bool                TranslateAtomicMemory(const IR::Instruction& inst);
+	bool                     TranslateBufferLoad(const IR::Instruction&     inst,
-	bool                TranslateFlatLoad(const IR::Instruction& inst);
+	                                             const BufferAddressValues* address_snapshot);
-	bool                TranslateFlatStore(const IR::Instruction& inst);
+	bool                     TranslateBufferStore(const IR::Instruction& inst);
-	bool                TranslateImageMemory(const IR::Instruction& inst);
+	bool                     TranslateAtomicMemory(const IR::Instruction& inst);
-	bool                TranslateSharedMemory(const IR::Instruction& inst);
+	bool                     TranslateFlatLoad(const IR::Instruction& inst);
+	bool                     TranslateFlatStore(const IR::Instruction& inst);
+	bool                     TranslateImageMemory(const IR::Instruction& inst);
+	bool                     TranslateSharedMemory(const IR::Instruction& inst);
 
 	IR::F32 SelectF32(IR::U1 condition, IR::F32 true_value, IR::F32 false_value);
 	IR::U32 ConvertF32ToU32Saturated(IR::F32 value, float upper_bound, float safe_upper,
@@ -104,15 +112,17 @@ private:
 	                                 uint32_t upper_result);
 	IR::U32 PackU16Lanes(IR::U32 low, IR::U32 high);
 
-	bool TranslateInstruction(const IR::Instruction&     inst,
+	bool TranslateInstruction(const IR::Instruction&          inst,
-	                          const BufferAddressValues* address_snapshot);
+	                          const BufferAddressValues*      address_snapshot,
+	                          const ScalarMemorySourceValues* scalar_source_snapshot);
 	bool TranslateStateOperation(const IR::Instruction& inst);
 	bool TranslateControlOperation(const IR::Instruction& inst);
 	bool TranslateMove(const IR::Instruction& inst);
 	bool TranslateLaneOperation(const IR::Instruction& inst);
 	bool TranslateAttributeOperation(const IR::Instruction& inst);
-	bool TranslateMemoryOperation(const IR::Instruction&     inst,
+	bool TranslateMemoryOperation(const IR::Instruction&          inst,
-	                              const BufferAddressValues* address_snapshot = nullptr);
+	                              const BufferAddressValues*      address_snapshot,
+	                              const ScalarMemorySourceValues* scalar_source_snapshot);
 	bool TranslateIntegerCompare(const IR::Instruction& inst);
 	bool TranslateInteger16Compare(const IR::Instruction& inst);
 	bool TranslateFloatCompare(const IR::Instruction& inst);

tests/shaderCfgTests.cpp

@@ -8052,6 +8052,26 @@ bool ReadSrtHostDword(void *, uint64_t address, uint32_t *value) {
   return true;
 }
 
+struct SrtHostRange {
+  const uint32_t *data;
+  size_t count;
+};
+
+bool ReadSrtHostRangeDword(void *userdata, uint64_t address, uint32_t *value) {
+  const auto *range = static_cast<const SrtHostRange *>(userdata);
+  if (range == nullptr || range->data == nullptr || range->count == 0 ||
+      value == nullptr) {
+    return false;
+  }
+  const auto base = reinterpret_cast<uint64_t>(range->data);
+  const auto size = range->count * sizeof(uint32_t);
+  if (address < base || address - base > size - sizeof(uint32_t)) {
+    return false;
+  }
+  std::memcpy(value, reinterpret_cast<const void *>(address), sizeof(*value));
+  return true;
+}
+
 void TestTypedDescriptorRealWideMoveLowering() {
   const uint32_t shader[] = {
       EncodeSop1(0x04, 0, 20), // s_mov_b64 s[0:1], s[20:21]
@@ -8362,29 +8382,55 @@ void TestSrtWalkerRealSBufferLowering() {
 }
 
 void TestScalarMemoryLoadsSnapshotOverlappingOperands() {
-  const auto CheckOverlap = [](uint32_t opcode) {
+  const auto CheckOverlap = [](uint32_t opcode, bool overlap_offset) {
-    const uint32_t shader[] = {
+    const uint32_t base_field = overlap_offset ? 4u : 0u;
-        EncodeSmem0(opcode, 0, 0),
+    const uint32_t soffset = overlap_offset ? 0u : 125u;
-        125u << 25u, // load s[0:3] through overlapping s[0:*], null SOFFSET
+    std::vector<uint32_t> shader;
-        EncodeMubuf0(0x1c),
+    if (overlap_offset) {
-        EncodeMubuf1(0, 0, 1),
+      shader.push_back(
-        EncodeSopp(0x01),
+          EncodeSMovB32(0, 128)); // s0 = 0 before it is overwritten
-    };
+    }
+    shader.insert(shader.end(),
+                  {EncodeSmem0(opcode, 0, base_field),
+                   soffset << 25u, // load s[0:3] with overlapping source
+                   EncodeMubuf0(0x1c), EncodeMubuf1(0, 0, 1),
+                   EncodeSopp(0x01)});
     std::string error;
     ShaderRecompiler::IR::Program ir;
-    Check(LowerTypedPlanning(shader, static_cast<uint32_t>(std::size(shader)),
+    Check(LowerTypedPlanning(shader.data(),
-                             ir, &error),
+                             static_cast<uint32_t>(shader.size()), ir, &error),
           error.c_str());
     Check(ir.values->srt_reads.size() == 4 && ir.values->dynamic_reads.empty(),
           opcode == 0x02 ? "overlapping S_LOAD operands were evaluated after a "
                            "component write"
                          : "overlapping S_BUFFER_LOAD operands were evaluated "
                            "after a component write");
+    const std::array<uint32_t, 4> table = {0x11111111u, 0x22222222u,
+                                           0x33333333u, 0x44444444u};
+    std::array<uint32_t, 12> user_data = {};
+    const auto address = reinterpret_cast<uint64_t>(table.data());
+    const uint32_t base = overlap_offset ? 8u : 0u;
+    user_data[base] = static_cast<uint32_t>(address);
+    user_data[base + 1u] = static_cast<uint32_t>(address >> 32u);
+    if (opcode == 0x0a) {
+      user_data[base + 2u] = sizeof(table);
+    }
+    SrtHostRange range{table.data(), table.size()};
+    const ShaderRecompiler::IR::SrtRuntime runtime{
+        user_data, 0, ReadSrtHostRangeDword, &range};
+    std::vector<uint32_t> flat;
+    Check(ShaderRecompiler::IR::WalkSrt(ir, runtime, flat, &error),
+          error.c_str());
+    Check(flat.size() == table.size() &&
+              std::equal(flat.begin(), flat.end(), table.begin()),
+          "overlapping scalar-memory load did not snapshot its sources");
     CheckFlattenedReadSlots(
         ir, 4, "overlapping scalar-memory patch used the wrong flat offsets");
   };
-  CheckOverlap(0x02); // s_load_dwordx4
+  CheckOverlap(0x02, false); // s_load_dwordx4 overlapping SBASE
-  CheckOverlap(0x0a); // s_buffer_load_dwordx4
+  CheckOverlap(0x0a, false); // s_buffer_load_dwordx4 overlapping SBASE
+  CheckOverlap(0x02, true);  // s_load_dwordx4 overlapping SOFFSET
+  CheckOverlap(0x0a, true);  // s_buffer_load_dwordx4 overlapping SOFFSET
 }
 
 void TestScalarMemoryLoadCrossesIntoVcc() {