shader: link multiple compute helpers without synthetic return loops

src/graphics/host_gpu/renderer/pipeline/pipelineCache.cpp

@@ -422,7 +422,7 @@ struct PipelineCache::ProgramCache {
 		auto* current = entry == programs.end() ? &source : entry->second.call_source.get();
 		ShaderRecompiler::TranslateResult translated;
 		if (current != nullptr) {
-			if (current->call) {
+			if (!current->calls.empty()) {
 				if (entry == programs.end()) ShaderRecompiler::RefreshShaderSource(*current, runtime);
 				translated = ShaderRecompiler::TranslateProgram(current->linked->decoded, options);
 				translated.program.source_reads = current->reads;
@@ -433,7 +433,7 @@ struct PipelineCache::ProgramCache {
 			translated = ShaderRecompiler::TranslateProgram(params.code, options);
 		}
 		if (entry == programs.end()) {
-			auto retained = source.call ? std::make_unique<ShaderRecompiler::ShaderSource>(std::move(source)) : nullptr;
+			auto retained = !source.calls.empty() ? std::make_unique<ShaderRecompiler::ShaderSource>(std::move(source)) : nullptr;
 			entry = programs.try_emplace(lookup_key,
 			    ShaderRecompiler::IR::ExtractResourcePlan(translated.program), std::move(retained)).first;
 			EXIT_IF(!ShaderRecompiler::IR::MaterializeResources(

src/graphics/shader/recompiler/ShaderRecompiler.cpp

@@ -481,18 +481,14 @@ Decoder::Program DecodeFusedProgram(std::span<const uint32_t> front, std::span<c
 	return result;
 }
 
-void PrepareCallTarget(ShaderSource& source, const CompileOptions& options) {
+void PrepareCallTargets(ShaderSource& source, const CompileOptions& options) {
 	using namespace IR;
+	EXIT_NOT_IMPLEMENTED(options.stage != ShaderType::Compute);
 	for (uint32_t i = 0; i < source.decoded.instructions.size(); ++i) {
 		if (source.decoded.instructions[i].opcode != Decoder::Opcode::S_SWAPPC_B64) continue;
-		EXIT_NOT_IMPLEMENTED(source.call.has_value() || options.stage != ShaderType::Compute);
-		source.call = ShaderSource::Call {.instruction = i};
+		source.calls.push_back({.instruction = i});
 	}
 	EXIT_IF(options.input_info.compute == nullptr);
-	const auto& call = source.decoded.instructions[source.call->instruction];
-	EXIT_NOT_IMPLEMENTED(call.src0.kind != Decoder::OperandKind::Sgpr ||
-	                     call.dst.kind != Decoder::OperandKind::Sgpr ||
-	                     call.src0.reg != call.dst.reg || call.dst.reg >= 105u);
 	const auto graph = CFG::BuildGraph(source.decoded);
 	Program query;
 	query.stage = options.stage;
@@ -524,7 +520,11 @@ void PrepareCallTarget(ShaderSource& source, const CompileOptions& options) {
 		return value;
 	};
 	Register = [&](uint32_t before, uint32_t code) -> Value {
-		const auto index = CFG::FindScalarDefinition(source.decoded, graph, before, code);
+		std::vector<uint32_t> crossed_calls;
+		const auto index = CFG::FindScalarDefinition(source.decoded, graph, before, code, crossed_calls);
+		for (const auto crossed: crossed_calls) {
+			std::ranges::find(source.calls, crossed, &ShaderSource::Call::instruction)->preserved_registers.set(code);
+		}
 		if (index == UINT32_MAX) {
 			EXIT_IF(code < query.user_data_base || code - query.user_data_base >= query.user_data_count);
 			return IREmitter(&block).GetUserData(static_cast<ScalarReg>(code));
@@ -557,18 +557,25 @@ void PrepareCallTarget(ShaderSource& source, const CompileOptions& options) {
 		EXIT_IF(!producer.ready || !producer.values.contains(code));
 		return producer.values.at(code).Resolve();
 	};
-	for (uint32_t i = 0; i < 2u; ++i) {
-		const auto value = Register(source.call->instruction, call.src0.reg + i);
-		EXIT_IF(!ValidateRuntimeValue(query, value, RuntimeValueType::Integer));
-		auto& root = query.value_storage.emplace_back(ValueOpcode::ReferenceU32);
-		root.SetArg(0, value);
-		source.call->target[i] = Value(&root);
+	for (auto& site: source.calls) {
+		const auto& call = source.decoded.instructions[site.instruction];
+		EXIT_NOT_IMPLEMENTED(call.src0.kind != Decoder::OperandKind::Sgpr ||
+		                     call.dst.kind != Decoder::OperandKind::Sgpr ||
+		                     call.src0.reg != call.dst.reg || call.dst.reg >= 105u);
+		for (uint32_t i = 0; i < 2u; ++i) {
+			const auto value = Register(site.instruction, call.src0.reg + i);
+			EXIT_IF(!ValidateRuntimeValue(query, value, RuntimeValueType::Integer));
+			auto& root = query.value_storage.emplace_back(ValueOpcode::ReferenceU32);
+			root.SetArg(0, value);
+			site.target[i] = Value(&root);
+		}
 	}
 	RewriteToSsa(query.blocks);
 	ConstantPropagationPass(query.blocks, query.wave_size);
 	RemoveIdentities(query);
 	EliminateDeadCode(query.blocks);
-	for (auto& value: source.call->target) value = value.Instruction()->Arg(0).Resolve();
+	for (auto& site: source.calls)
+		for (auto& value: site.target) value = value.Instruction()->Arg(0).Resolve();
 	query.value_storage.clear();
 	for (auto& inst: block) {
 		inst.SetParent(nullptr);
@@ -595,7 +602,7 @@ ShaderSource PrepareShaderSource(std::span<const uint32_t> code, const CompileOp
 		Decoder::DecodeProgram(code, source.decoded);
 	}
 	if (source.decoded.has_swap_pc) {
-		PrepareCallTarget(source, options);
+		PrepareCallTargets(source, options);
 		source.code.assign(code.begin(), code.end());
 		source.decoded.code = source.code;
 	}
@@ -608,83 +615,116 @@ const Decoder::Program& RefreshShaderSource(ShaderSource& source, const IR::SrtR
 	IR::SrtReadCapture capture(runtime, reads);
 	const auto clean = IR::CleanRuntime(capture.ObservedRuntime());
 	IR::SrtWalker walker(source.call_targets, clean);
-	uint32_t low = 0, high = 0;
-	const auto& call = source.decoded.instructions[source.call->instruction];
-	if (!walker.Evaluate(source.call->target[0], low) || !walker.Evaluate(source.call->target[1], high))
-		EXIT("shader call at pc 0x%08x has an unavailable scalar target", call.pc);
-	// SWAPPC reads the old aliased pair and ignores the target's low two bits.
-	const auto address = ((uint64_t {high} << 32u) | low) & ~uint64_t {3};
-	EXIT_IF(address == 0u);
+	for (auto& site: source.calls) {
+		uint32_t low = 0, high = 0;
+		if (!walker.Evaluate(site.target[0], low) || !walker.Evaluate(site.target[1], high))
+			EXIT("shader call at pc 0x%08x has an unavailable scalar target",
+			     source.decoded.instructions[site.instruction].pc);
+		// SWAPPC reads the old aliased pair and ignores the target's low two bits.
+		const auto address = ((uint64_t {high} << 32u) | low) & ~uint64_t {3};
+		EXIT_IF(address == 0u);
+		site.address = address;
+	}
 	const auto query_reads = reads.size();
 	if (source.linked) {
 		auto& linked = *source.linked;
-		const auto function = std::span(linked.code).subspan(source.code.size());
-		if (clean.read_memory(clean.userdata, address, linked.observed_function) &&
-		    std::ranges::equal(function, linked.observed_function)) {
-			return linked.decoded;
+		bool unchanged = true;
+		for (size_t i = 0; i < source.calls.size(); ++i) {
+			const auto& function = linked.functions[i];
+			const auto expected = std::span(linked.code).subspan(function.offset, function.size);
+			const auto preceding = std::span(source.calls).first(i);
+			const auto previous = std::ranges::find(preceding, source.calls[i].address, &ShaderSource::Call::address);
+			if (previous != preceding.end()) {
+				const auto& observed = linked.functions[previous - preceding.begin()];
+				if (std::ranges::equal(expected, std::span(linked.code).subspan(observed.offset, observed.size))) continue;
+				unchanged = false;
+				break;
+			}
+			linked.observed_function.resize(function.size);
+			if (!clean.read_memory(clean.userdata, source.calls[i].address, linked.observed_function) ||
+			    !std::ranges::equal(expected, linked.observed_function)) {
+				unchanged = false;
+				break;
+			}
 		}
+		if (unchanged) return linked.decoded;
 		reads.resize(query_reads);
 	}
-	ShaderSource::Linked linked {.code = source.code,
-	                             .decoded = source.decoded};
+	ShaderSource::Linked linked {.code = source.code, .decoded = source.decoded};
 	const auto& last = source.decoded.instructions.back();
 	EXIT_NOT_IMPLEMENTED(last.opcode != Decoder::Opcode::S_ENDPGM);
-	// Keep the caller's data/footer bytes at their native offsets. The leaf's virtual CFG
-	// PCs follow decoded instructions; its physical code is independently read and owned.
-	const auto function_pc = last.pc + last.word_count * sizeof(uint32_t);
-	for (uint32_t index = 0;;) {
-		// The observed queue helper is a scalar leaf. Scalar encodings need at most one literal.
-		EXIT_IF(index >= 64u || address > UINT64_MAX - (uint64_t {index} + 2u) * sizeof(uint32_t));
-		std::array<uint32_t, 2> words {};
-		if (!clean.read_memory(clean.userdata, address + uint64_t {index} * sizeof(uint32_t),
-		                       std::span(words).first(1)))
-			EXIT("shader call at pc 0x%08x cannot read leaf code", call.pc);
-		const auto family = Decoder::GetInstructionFamily(words[0]);
-		EXIT_NOT_IMPLEMENTED(family != Decoder::Family::SOP1 && family != Decoder::Family::SOP2 &&
-		                     family != Decoder::Family::SOPC && family != Decoder::Family::SOPK &&
-		                     family != Decoder::Family::SOPP);
-		Decoder::Instruction inst;
-		Decoder::DecodeInstruction(words, 0u, inst);
-		if (inst.word_count == 2u) {
-			if (!clean.read_memory(clean.userdata, address + uint64_t {index + 1u} * sizeof(uint32_t),
-			                       std::span(words).last(1)))
-				EXIT("shader call at pc 0x%08x cannot read leaf literal", call.pc);
-			inst = {};
-			Decoder::DecodeInstruction(words, 0u, inst);
+	// Keep the caller's data/footer at native offsets. Expand leaf CFGs per call site
+	// before SSA, so return edges neither require a dispatcher nor merge resource roots.
+	uint32_t function_pc = last.pc + last.word_count * sizeof(uint32_t);
+	std::map<uint64_t, std::vector<Decoder::Instruction>> functions;
+	for (size_t i = 0; i < source.calls.size(); ++i) {
+		const auto& site = source.calls[i];
+		const auto& call = source.decoded.instructions[site.instruction];
+		const auto address = source.calls[i].address;
+		auto [entry, inserted] = functions.try_emplace(address);
+		auto& function = entry->second;
+		if (inserted) {
+			for (uint32_t index = 0;;) {
+				std::array<uint32_t, Decoder::MaxInstructionRawWords> words {};
+				EXIT_IF(index > UINT32_MAX / sizeof(uint32_t) - words.size() ||
+				        address > UINT64_MAX - (uint64_t {index} + words.size()) * sizeof(uint32_t));
+				uint32_t count = 1;
+				Decoder::Instruction inst;
+				for (;;) {
+					if (!clean.read_memory(clean.userdata, address + uint64_t {index} * sizeof(uint32_t),
+					                       std::span(words).first(count)))
+						EXIT("shader call at pc 0x%08x cannot read leaf code", call.pc);
+					inst = {};
+					Decoder::DecodeInstruction(words, 0u, inst);
+					if (inst.word_count <= count) break;
+					count = inst.word_count;
+				}
+				inst.pc = index * sizeof(uint32_t);
+				if (Decoder::IsDirectBranch(inst.opcode)) inst.branch_target += inst.pc;
+				index += inst.word_count;
+				EXIT_NOT_IMPLEMENTED(inst.opcode == Decoder::Opcode::S_SWAPPC_B64 ||
+				                     inst.opcode == Decoder::Opcode::S_GETPC_B64 ||
+				                     inst.opcode == Decoder::Opcode::S_ENDPGM);
+				function.push_back(inst);
+				if (inst.opcode == Decoder::Opcode::S_SETPC_B64) break;
+			}
 		}
-		inst.pc = index * sizeof(uint32_t);
-		if (Decoder::IsDirectBranch(inst.opcode)) inst.branch_target += inst.pc;
-		linked.code.insert(linked.code.end(), inst.raw, inst.raw + inst.word_count);
-		index += inst.word_count;
-		EXIT_NOT_IMPLEMENTED(inst.opcode == Decoder::Opcode::S_SWAPPC_B64 ||
-		                     inst.opcode == Decoder::Opcode::S_GETPC_B64 ||
-		                     inst.opcode == Decoder::Opcode::S_ENDPGM);
-		EXIT_IF(CFG::MayWriteScalarRegister(inst, call.dst.reg) || CFG::MayWriteScalarRegister(inst, call.dst.reg + 1u));
-		const bool returns = inst.opcode == Decoder::Opcode::S_SETPC_B64;
-		if (returns) {
-			EXIT_NOT_IMPLEMENTED(inst.src0.kind != Decoder::OperandKind::Sgpr || inst.src0.reg != call.dst.reg);
-			inst.opcode = Decoder::Opcode::S_BRANCH;
-			inst.branch_target = call.pc + call.word_count * sizeof(uint32_t);
-		} else if (Decoder::IsDirectBranch(inst.opcode)) {
-			EXIT_IF(inst.branch_target > UINT32_MAX - function_pc);
-			inst.branch_target += function_pc;
+		const uint32_t function_size = function.back().pc + function.back().word_count * sizeof(uint32_t);
+		EXIT_IF(function_pc > UINT32_MAX - function_size);
+		const auto function_offset = static_cast<uint32_t>(linked.code.size());
+		for (auto inst: function) {
+			EXIT_IF(CFG::MayWriteScalarRegister(inst, call.dst.reg) ||
+			        CFG::MayWriteScalarRegister(inst, call.dst.reg + 1u));
+			for (uint32_t code = 0; code < site.preserved_registers.size(); ++code) {
+				if (site.preserved_registers[code] && CFG::MayWriteScalarRegister(inst, code))
+					EXIT("shader call at pc 0x%08x clobbers scalar target dependency s%u", call.pc, code);
+			}
+			linked.code.insert(linked.code.end(), inst.raw, inst.raw + inst.word_count);
+			if (inst.opcode == Decoder::Opcode::S_SETPC_B64) {
+				EXIT_NOT_IMPLEMENTED(inst.src0.kind != Decoder::OperandKind::Sgpr || inst.src0.reg != call.dst.reg);
+				inst.opcode = Decoder::Opcode::S_BRANCH;
+				inst.branch_target = call.pc + call.word_count * sizeof(uint32_t);
+			} else if (Decoder::IsDirectBranch(inst.opcode)) {
+				EXIT_IF(inst.branch_target >= function_size);
+				inst.branch_target += function_pc;
+			}
+			inst.pc += function_pc;
+			linked.decoded.instructions.push_back(std::move(inst));
 		}
-		inst.pc += function_pc;
-		linked.decoded.instructions.push_back(std::move(inst));
-		if (returns) break;
-	}
-	const auto function_end = linked.decoded.instructions.back().pc +
-	                          linked.decoded.instructions.back().word_count * sizeof(uint32_t);
-	for (size_t i = source.decoded.instructions.size(); i + 1u < linked.decoded.instructions.size(); ++i) {
-		const auto& inst = linked.decoded.instructions[i];
-		if (Decoder::IsDirectBranch(inst.opcode))
-			EXIT_IF(inst.branch_target < function_pc || inst.branch_target >= function_end);
+		linked.functions.push_back({function_offset, function_size / 4u});
+		linked.decoded.instructions[site.instruction].branch_target = function_pc;
+		function_pc += function_size;
 	}
 	reads.resize(query_reads);
-	reads.emplace_back(address, (linked.code.size() - source.code.size()) * sizeof(uint32_t));
-	linked.decoded.instructions[source.call->instruction].branch_target = function_pc;
+	for (size_t i = 0; i < source.calls.size(); ++i) {
+		const auto preceding = std::span(source.calls).first(i);
+		if (std::ranges::find(preceding, source.calls[i].address, &ShaderSource::Call::address) == preceding.end())
+			reads.emplace_back(source.calls[i].address, linked.functions[i].size * sizeof(uint32_t));
+	}
 	linked.decoded.code = linked.code;
-	linked.observed_function.resize(linked.code.size() - source.code.size());
+	linked.observed_function.resize(std::ranges::max_element(linked.functions, {},
+	    &ShaderSource::Linked::Function::size)->size);
+
 	source.linked = std::move(linked);
 	++source.revision;
 	return source.linked->decoded;

src/graphics/shader/recompiler/ShaderRecompiler.h

@@ -8,6 +8,7 @@
 #include "graphics/shader/shader.h"
 
 #include <array>
+#include <bitset>
 #include <optional>
 #include <span>
 #include <vector>
@@ -47,16 +48,20 @@ struct ShaderSource {
 	struct Call {
 		uint32_t instruction;
 		std::array<IR::Value, 2> target;
+		std::bitset<128> preserved_registers;
+		uint64_t address = 0;
 	};
 	struct Linked {
+		struct Function { uint32_t offset; uint32_t size; };
 		std::vector<uint32_t> code;
 		Decoder::Program decoded;
+		std::vector<Function> functions;
 		std::vector<uint32_t> observed_function;
 	};
 	std::vector<uint32_t> code;
 	Decoder::Program decoded;
 	IR::ResourcePlan call_targets;
-	std::optional<Call> call;
+	std::vector<Call> calls;
 	std::optional<Linked> linked;
 	uint64_t revision = 0;
 	std::vector<std::pair<uint64_t, uint64_t>> reads;

src/graphics/shader/recompiler/frontend/cfg/ShaderCFG.cpp

@@ -8,6 +8,8 @@
 #include <deque>
 #include <list>
 #include <map>
+#include <optional>
+#include <set>
 #include <span>
 
 namespace Libs::Graphics::ShaderRecompiler::CFG {
@@ -1650,6 +1652,32 @@ Graph BuildGraph(const Decoder::Program& program) {
 	}
 	SortUnique(graph.code_table_load_pcs);
 
+	if (!indirect_setpc && std::ranges::any_of(program.instructions, [](const auto& inst) {
+		    return inst.opcode == Opcode::S_SWAPPC_B64 && inst.branch_target != UINT32_MAX;
+		})) {
+		// Linked leaf bodies follow the caller in native PC order. Visit them in
+		// execution order so returning to the caller does not manufacture goto loops.
+		std::vector<bool> visited(graph.blocks.size(), false);
+		std::vector<std::pair<uint32_t, bool>> pending {{graph.entry_block, false}};
+		std::vector<BasicBlock> ordered;
+		ordered.reserve(graph.blocks.size());
+		while (!pending.empty()) {
+			const auto [id, finished] = pending.back();
+			pending.pop_back();
+			if (finished) {
+				ordered.push_back(std::move(graph.blocks[id]));
+				continue;
+			}
+			if (visited[id]) continue;
+			visited[id] = true;
+			pending.emplace_back(id, true);
+			for (const auto successor: graph.blocks[id].successors)
+				pending.emplace_back(successor, false);
+		}
+		std::ranges::reverse(ordered);
+		ApplyBlockOrder(graph, std::move(ordered));
+	}
+
 	RecomputeAnalyses(graph);
 
 	if (indirect_setpc) {
@@ -1681,33 +1709,56 @@ Graph Structurize(const Graph& graph) {
 }
 
 bool MayWriteScalarRegister(const Decoder::Instruction& inst, uint32_t code) {
-	uint32_t destination = UINT32_MAX;
-	// Scalar ALU pair widths are not all decoded. The typed producer must confirm this word.
-	return InstructionWritesScalarCode(inst, code) ||
-	       (ScalarOperandCode(inst.dst, destination) && code == destination + 1u);
+	// Vector comparisons and carry results can write a scalar pair in wave64.
+	if (inst.family == Decoder::Family::VOP1 || inst.family == Decoder::Family::VOP2 ||
+	    inst.family == Decoder::Family::VOPC || inst.family == Decoder::Family::VOP3 ||
+	    inst.family == Decoder::Family::VOP3P) {
+		uint32_t destination = 0;
+		for (const auto* operand: {&inst.dst, &inst.dst2})
+			if (ScalarOperandCode(*operand, destination) && code >= destination && code <= destination + 1u)
+				return true;
+	}
+	return InstructionWritesScalarCode(inst, code);
 }
 
 uint32_t FindScalarDefinition(const Decoder::Program& program, const Graph& graph,
-                              uint32_t before, uint32_t code) {
+                              uint32_t before, uint32_t code, std::vector<uint32_t>& crossed_calls) {
 	EXIT_IF(before >= program.instructions.size());
 	const auto found = std::ranges::find_if(graph.blocks, [before](const auto& block) {
 		return block.inst_begin <= before && before < block.inst_end;
 	});
 	EXIT_IF(found == graph.blocks.end());
-	const auto* block = &*found;
-	for (size_t depth = 0; depth < graph.blocks.size(); ++depth) {
-		for (uint32_t i = before; i > block->inst_begin;) {
-			const auto& inst = program.instructions[--i];
-			EXIT_IF(inst.opcode == Opcode::S_SWAPPC_B64);
-			if (MayWriteScalarRegister(inst, code)) return i;
-		}
-		if (block->id == graph.entry_block) return UINT32_MAX;
-		EXIT_IF(block->predecessors.size() != 1u);
-		block = graph.FindBlock(block->predecessors[0]);
+	std::vector<std::pair<uint32_t, uint32_t>> pending {{found->id, before}};
+	std::set<std::pair<uint32_t, uint32_t>> visited;
+	std::optional<uint32_t> definition;
+	while (!pending.empty()) {
+		const auto point = pending.back();
+		pending.pop_back();
+		if (!visited.insert(point).second) continue;
+		const auto* block = graph.FindBlock(point.first);
 		EXIT_IF(block == nullptr);
-		before = block->inst_end;
+		uint32_t index = UINT32_MAX;
+		for (uint32_t i = point.second; i > block->inst_begin;) {
+			const auto& inst = program.instructions[--i];
+			if (inst.opcode == Opcode::S_SWAPPC_B64) {
+				EXIT_NOT_IMPLEMENTED(MayWriteScalarRegister(inst, code));
+				crossed_calls.push_back(i);
+				continue;
+			}
+			if (MayWriteScalarRegister(inst, code)) { index = i; break; }
+		}
+		if (index != UINT32_MAX || block->id == graph.entry_block) {
+			EXIT_IF(definition && *definition != index);
+			definition = index;
+			continue;
+		}
+		EXIT_IF(block->predecessors.empty());
+		for (const auto predecessor: block->predecessors) {
+			pending.emplace_back(predecessor, graph.blocks[predecessor].inst_end);
+		}
 	}
-	EXIT("scalar shader call source has cyclic reaching definitions");
+	EXIT_IF(!definition);
+	return *definition;
 }
 
 std::string BranchConditionToString(BranchCondition condition) {

src/graphics/shader/recompiler/frontend/cfg/ShaderCFG.h

@@ -115,7 +115,7 @@ struct Graph {
 Graph       BuildGraph(const Decoder::Program& program);
 bool MayWriteScalarRegister(const Decoder::Instruction& inst, uint32_t code);
 uint32_t FindScalarDefinition(const Decoder::Program& program, const Graph& graph,
-                              uint32_t before, uint32_t code);
+                              uint32_t before, uint32_t code, std::vector<uint32_t>& crossed_calls);
 // Returns structured control flow or failure diagnostics without changing the native graph.
 // On failure, failure_block is an original block ID or UINT32_MAX.
 Graph       Structurize(const Graph& graph);

src/graphics/shader/recompiler/frontend/decode/ScalarAluOps.cpp

@@ -164,9 +164,23 @@ void DecodeSop1(uint32_t pc, std::span<const uint32_t> code, uint32_t word_index
 	switch (inst.opcode) {
 		case Opcode::S_SWAPPC_B64:
 			inst.branch_target = UINT32_MAX;
-			inst.data_dwords = 2;
+			[[fallthrough]];
+		case Opcode::S_MOV_B64:
+		case Opcode::S_CMOV_B64:
+		case Opcode::S_NOT_B64:
+		case Opcode::S_WQM_B64:
+		case Opcode::S_BREV_B64:
+		case Opcode::S_BITSET0_B64:
+		case Opcode::S_BITSET1_B64:
+		case Opcode::S_AND_SAVEEXEC_B64:
+		case Opcode::S_ORN2_SAVEEXEC_B64:
+		case Opcode::S_QUADMASK_B64:
+		case Opcode::S_ANDN1_SAVEEXEC_B64:
+		case Opcode::S_BITREPLICATE_B64_B32:
+			inst.data_dwords = 2u;
 			break;
 		case Opcode::S_GETPC_B64:
+			inst.data_dwords = 2u;
 			inst.src_count = 0;
 			DecodeScalarDestination(sdst, pc, inst.dst);
 			return;
@@ -206,6 +220,8 @@ void DecodeSop2(uint32_t pc, std::span<const uint32_t> code, uint32_t word_index
 
 	DecodeScalarDestination(sdst, pc, inst.dst);
 	DecodeBinarySources(pc, code, word_index, inst, ssrc0, ssrc1);
+	if (opcode == 0x0bu || (opcode >= 0x0fu && opcode <= 0x25u && (opcode & 1u)) || opcode == 0x29u)
+		inst.data_dwords = 2u;
 }
 
 void DecodeSopk(uint32_t pc, std::span<const uint32_t> code, uint32_t word_index,