refactor(shader): preserve native wide shared operations

src/graphics/shader/recompiler/backend/spirv/SpirvEmitter.cpp

@@ -95,12 +95,25 @@ bool ValidateNativeProgram(const IR::Program& program, std::string* error) {
 	if (!program.info.samplers.empty()) {
 		Expect(Kind::Samplers, Dense(program.info.samplers.size()));
 	}
-	const auto uses_gds = program.values != nullptr &&
+	bool uses_gds = false;
-	                      std::ranges::any_of(program.values->blocks, [](const IR::Block* block) {
+	if (program.values != nullptr) {
-		                      return std::ranges::any_of(*block, [](const IR::Inst& inst) {
+		for (const auto* block: program.values->blocks) {
-			                      return inst.GetOpcode() == IR::ValueOpcode::GetGdsResource;
+			for (const auto& inst: *block) {
-		                      });
+				if (IR::SharedAccessOf(inst.GetOpcode()) == IR::SharedAccess::None) {
-	                      });
+					continue;
+				}
+				const auto index = inst.Flags<IR::MemoryFlags>().index;
+				if (index >= program.values->memory_info.size()) {
+					return Fail(error, "shared operation has invalid memory metadata");
+				}
+				const auto kind = program.values->memory_info[index].kind;
+				if (kind != IR::ResourceKind::Lds && kind != IR::ResourceKind::Gds) {
+					return Fail(error, "shared operation has invalid resource kind");
+				}
+				uses_gds |= kind == IR::ResourceKind::Gds;
+			}
+		}
+	}
 	if (uses_gds) {
 		Expect(Kind::Gds);
 	}
@@ -249,6 +262,26 @@ bool AnalyzeProgramRequirements(IR::Program& program, std::string* error) {
 					}
 				}
 			}
+			const auto shared_access = IR::SharedAccessOf(inst.GetOpcode());
+			if (shared_access != IR::SharedAccess::None) {
+				const auto index = inst.Flags<IR::MemoryFlags>().index;
+				if (index >= program.values->memory_info.size()) {
+					return Fail(error, "shared operation has invalid memory metadata");
+				}
+				const auto kind = program.values->memory_info[index].kind;
+				if (kind != IR::ResourceKind::Lds && kind != IR::ResourceKind::Gds) {
+					return Fail(error, "shared operation has invalid resource kind");
+				}
+				if (program.stage != ShaderType::Compute && kind == IR::ResourceKind::Lds) {
+					requirements.function_lds = true;
+				}
+				if (shared_access == IR::SharedAccess::Append ||
+				    shared_access == IR::SharedAccess::Consume) {
+					MarkExactSubgroup();
+					requirements.subgroup_shuffle             = true;
+					requirements.subgroup_local_invocation_id = true;
+				}
+			}
 			switch (inst.GetOpcode()) {
 				case IR::ValueOpcode::Ballot: MarkExactSubgroup(); break;
 				case IR::ValueOpcode::DppMoveU32:
@@ -268,9 +301,7 @@ bool AnalyzeProgramRequirements(IR::Program& program, std::string* error) {
 					requirements.subgroup_local_invocation_id = true;
 					break;
 				}
-				case IR::ValueOpcode::Permlane16U32:
+				case IR::ValueOpcode::Permlane16U32: {
-				case IR::ValueOpcode::GdsDataAppend:
-				case IR::ValueOpcode::GdsDataConsume: {
 					MarkExactSubgroup();
 					requirements.subgroup_shuffle             = true;
 					requirements.subgroup_local_invocation_id = true;
@@ -282,41 +313,6 @@ bool AnalyzeProgramRequirements(IR::Program& program, std::string* error) {
 					requirements.subgroup_local_invocation_id = true;
 					break;
 				}
-				case IR::ValueOpcode::DataAppend:
-				case IR::ValueOpcode::DataConsume: {
-					MarkExactSubgroup();
-					requirements.subgroup_shuffle             = true;
-					requirements.subgroup_local_invocation_id = true;
-					[[fallthrough]];
-				}
-				case IR::ValueOpcode::LoadSharedU8:
-				case IR::ValueOpcode::LoadSharedU16:
-				case IR::ValueOpcode::LoadSharedU32:
-				case IR::ValueOpcode::WriteSharedU8:
-				case IR::ValueOpcode::WriteSharedU16:
-				case IR::ValueOpcode::WriteSharedU32:
-				case IR::ValueOpcode::SharedAtomicFMin32:
-				case IR::ValueOpcode::SharedAtomicFMax32:
-				case IR::ValueOpcode::SharedAtomicSwap32:
-				case IR::ValueOpcode::SharedAtomicIAdd32:
-				case IR::ValueOpcode::SharedAtomicISub32:
-				case IR::ValueOpcode::SharedAtomicSMin32:
-				case IR::ValueOpcode::SharedAtomicUMin32:
-				case IR::ValueOpcode::SharedAtomicSMax32:
-				case IR::ValueOpcode::SharedAtomicUMax32:
-				case IR::ValueOpcode::SharedAtomicAnd32:
-				case IR::ValueOpcode::SharedAtomicOr32:
-				case IR::ValueOpcode::SharedAtomicXor32: {
-					const auto index = inst.Flags<IR::MemoryFlags>().index;
-					if (index >= program.values->memory_info.size()) {
-						return Fail(error, "shared operation has invalid memory metadata");
-					}
-					if (program.stage != ShaderType::Compute &&
-					    program.values->memory_info[index].kind == IR::ResourceKind::Lds) {
-						requirements.function_lds = true;
-					}
-					break;
-				}
 				case IR::ValueOpcode::LaneId:
 					requirements.subgroup_local_invocation_id = true;
 					break;

src/graphics/shader/recompiler/backend/spirv/spirvEmitterFlow.cpp

@@ -25,16 +25,14 @@ uint32_t EmitWqmWordU32(EmitterState& state, uint32_t value) {
 	const auto merged_two  = state.builder.AllocateId();
 	const auto quad_bits   = state.builder.AllocateId();
 	const auto result      = state.builder.AllocateId();
-	state.builder.AddFunction({OpShiftRightLogical, TypeU32(state), shifted_one, value,
-	                           ConstantU32(state, 1)});
 	state.builder.AddFunction(
-	    {OpBitwiseOr, TypeU32(state), merged_one, value, shifted_one});
+	    {OpShiftRightLogical, TypeU32(state), shifted_one, value, ConstantU32(state, 1)});
-	state.builder.AddFunction({OpShiftRightLogical, TypeU32(state), shifted_two, merged_one,
+	state.builder.AddFunction({OpBitwiseOr, TypeU32(state), merged_one, value, shifted_one});
-	                           ConstantU32(state, 2)});
 	state.builder.AddFunction(
-	    {OpBitwiseOr, TypeU32(state), merged_two, merged_one, shifted_two});
+	    {OpShiftRightLogical, TypeU32(state), shifted_two, merged_one, ConstantU32(state, 2)});
-	state.builder.AddFunction({OpBitwiseAnd, TypeU32(state), quad_bits, merged_two,
+	state.builder.AddFunction({OpBitwiseOr, TypeU32(state), merged_two, merged_one, shifted_two});
-	                           ConstantU32(state, 0x11111111u)});
+	state.builder.AddFunction(
+	    {OpBitwiseAnd, TypeU32(state), quad_bits, merged_two, ConstantU32(state, 0x11111111u)});
 	state.builder.AddFunction(
 	    {OpIMul, TypeU32(state), result, quad_bits, ConstantU32(state, 0x0fu)});
 	return result;
@@ -531,8 +529,6 @@ bool EmitValueFlow(ValueEmitContext& ctx, const IR::Inst& inst) {
 		case IR::ValueOpcode::GetAddressResource:
 		case IR::ValueOpcode::GetImageResource:
 		case IR::ValueOpcode::GetSamplerResource:
-		case IR::ValueOpcode::GetLdsResource:
-		case IR::ValueOpcode::GetGdsResource:
 		case IR::ValueOpcode::MakeImageAddress: return true;
 		default: return false;
 	}

src/graphics/shader/recompiler/backend/spirv/spirvEmitterInternal.h

@@ -314,6 +314,7 @@ struct EmitterState {
 	std::array<uint32_t, IR::ShaderInfo::MaxBuffers> storage_buffer_offsets {};
 	uint32_t                                         address_memory_variable = 0;
 	uint32_t                                         gds_variable            = 0;
+	uint32_t                                         gds_length              = 0;
 	uint32_t                                         push_constant_variable  = 0;
 	uint32_t                                         vsharp_storage_variable = 0;
 	uint32_t                                         flattened_srt_variable  = 0;

src/graphics/shader/recompiler/backend/spirv/spirvEmitterMemory.cpp

@@ -141,7 +141,11 @@ uint32_t ByteAddress(ValueEmitContext& ctx, const IR::Inst& inst, const IR::Memo
 		                         ctx.Arg(inst, 3));
 	}
 	if (mem.kind == IR::ResourceKind::Lds || mem.kind == IR::ResourceKind::Gds) {
-		return ctx.Arg(inst, 1);
+		if (mem.offset == 0u) {
+			return ctx.Arg(inst, 0);
+		}
+		return Binary(ctx.state, OpIAdd, TypeU32(ctx.state), ctx.Arg(inst, 0),
+		              ConstantU32(ctx.state, mem.offset));
 	}
 	return AddressByteAddress(ctx, inst, mem, ctx.Arg(inst, 1), ctx.Arg(inst, 2));
 }
@@ -493,35 +497,25 @@ void FormattedStore(ValueEmitContext& ctx, const IR::Inst& inst, const IR::Memor
 uint32_t AtomicOpcode(IR::ValueOpcode opcode) {
 	switch (opcode) {
 		case IR::ValueOpcode::BufferAtomicSwap32:
-		case IR::ValueOpcode::SharedAtomicSwap32:
+		case IR::ValueOpcode::SharedAtomicSwap32: return OpAtomicExchange;
-		case IR::ValueOpcode::GdsAtomicSwap32: return OpAtomicExchange;
 		case IR::ValueOpcode::BufferAtomicIAdd32:
-		case IR::ValueOpcode::SharedAtomicIAdd32:
+		case IR::ValueOpcode::SharedAtomicIAdd32: return OpAtomicIAdd;
-		case IR::ValueOpcode::GdsAtomicIAdd32: return OpAtomicIAdd;
 		case IR::ValueOpcode::BufferAtomicISub32:
-		case IR::ValueOpcode::SharedAtomicISub32:
+		case IR::ValueOpcode::SharedAtomicISub32: return OpAtomicISub;
-		case IR::ValueOpcode::GdsAtomicISub32: return OpAtomicISub;
 		case IR::ValueOpcode::BufferAtomicSMin32:
-		case IR::ValueOpcode::SharedAtomicSMin32:
+		case IR::ValueOpcode::SharedAtomicSMin32: return OpAtomicSMin;
-		case IR::ValueOpcode::GdsAtomicSMin32: return OpAtomicSMin;
 		case IR::ValueOpcode::BufferAtomicUMin32:
-		case IR::ValueOpcode::SharedAtomicUMin32:
+		case IR::ValueOpcode::SharedAtomicUMin32: return OpAtomicUMin;
-		case IR::ValueOpcode::GdsAtomicUMin32: return OpAtomicUMin;
 		case IR::ValueOpcode::BufferAtomicSMax32:
-		case IR::ValueOpcode::SharedAtomicSMax32:
+		case IR::ValueOpcode::SharedAtomicSMax32: return OpAtomicSMax;
-		case IR::ValueOpcode::GdsAtomicSMax32: return OpAtomicSMax;
 		case IR::ValueOpcode::BufferAtomicUMax32:
-		case IR::ValueOpcode::SharedAtomicUMax32:
+		case IR::ValueOpcode::SharedAtomicUMax32: return OpAtomicUMax;
-		case IR::ValueOpcode::GdsAtomicUMax32: return OpAtomicUMax;
 		case IR::ValueOpcode::BufferAtomicAnd32:
-		case IR::ValueOpcode::SharedAtomicAnd32:
+		case IR::ValueOpcode::SharedAtomicAnd32: return OpAtomicAnd;
-		case IR::ValueOpcode::GdsAtomicAnd32: return OpAtomicAnd;
 		case IR::ValueOpcode::BufferAtomicOr32:
-		case IR::ValueOpcode::SharedAtomicOr32:
+		case IR::ValueOpcode::SharedAtomicOr32: return OpAtomicOr;
-		case IR::ValueOpcode::GdsAtomicOr32: return OpAtomicOr;
 		case IR::ValueOpcode::BufferAtomicXor32:
-		case IR::ValueOpcode::SharedAtomicXor32:
+		case IR::ValueOpcode::SharedAtomicXor32: return OpAtomicXor;
-		case IR::ValueOpcode::GdsAtomicXor32: return OpAtomicXor;
 		default: return 0;
 	}
 }
@@ -614,7 +608,7 @@ uint32_t SharedFloatAtomic(ValueEmitContext& ctx, const IR::Inst& inst, const IR
 		EmitIfCondition(
 		    ctx.state, EmitMemoryElementInBounds(ctx.state, access.resource, access.index), [&]() {
 			    ctx.state.builder.AddFunction(
-			        {OpStore, ctx.scratch_u32_variable, ctx.Arg(inst, 2)});
+			        {OpStore, ctx.scratch_u32_variable, ctx.Arg(inst, 1)});
 			    const auto data = ctx.state.builder.AllocateId();
 			    ctx.state.builder.AddFunction(
 			        {OpLoad, TypeU32(ctx.state), data, ctx.scratch_u32_variable});
@@ -623,7 +617,7 @@ uint32_t SharedFloatAtomic(ValueEmitContext& ctx, const IR::Inst& inst, const IR
 			        mem.kind, [&](uint32_t old) {
 				        const auto old_f = Unary(ctx.state, OpBitcast, TypeF32(ctx.state), old);
 				        const auto compare_f =
-				            Unary(ctx.state, OpBitcast, TypeF32(ctx.state), ctx.Arg(inst, 3));
+				            Unary(ctx.state, OpBitcast, TypeF32(ctx.state), ctx.Arg(inst, 2));
 				        const auto data_f = Unary(ctx.state, OpBitcast, TypeF32(ctx.state), data);
 				        const auto compare =
 				            Binary(ctx.state, max_value ? OpFOrdGreaterThan : OpFOrdLessThan,
@@ -637,9 +631,9 @@ uint32_t SharedFloatAtomic(ValueEmitContext& ctx, const IR::Inst& inst, const IR
 	return 0;
 }
 
-uint32_t AppendConsume(ValueEmitContext& ctx, const IR::Inst& inst, bool gds, bool append) {
+uint32_t AppendConsume(ValueEmitContext& ctx, const IR::Inst& inst, bool append) {
 	auto&      state = ctx.state;
-	const auto m0    = ctx.Arg(inst, 1);
+	const auto m0    = ctx.Arg(inst, 0);
 	const auto base =
 	    Binary(state, OpShiftRightLogical, TypeU32(state), m0, ConstantU32(state, 16));
 	const auto size = Binary(state, OpBitwiseAnd, TypeU32(state), m0, ConstantU32(state, 0xffffu));
@@ -647,11 +641,10 @@ uint32_t AppendConsume(ValueEmitContext& ctx, const IR::Inst& inst, bool gds, bo
 	    Binary(state, OpIAdd, TypeU32(state), base, ConstantU32(state, ctx.Memory(inst).offset));
 	const auto raw_index =
 	    Binary(state, OpShiftRightLogical, TypeU32(state), address, ConstantU32(state, 2));
-	auto mem          = ctx.Memory(inst);
+	const auto mem    = ctx.Memory(inst);
-	mem.kind          = gds ? IR::ResourceKind::Gds : IR::ResourceKind::Lds;
 	const auto access = PrepareMemoryResourceAccess(state, mem);
 	const auto index  = EmitMemoryElementIndex(state, access, raw_index);
-	const auto exec   = ctx.Arg(inst, 2);
+	const auto exec   = ctx.Arg(inst, 1);
 	const auto ballot = state.builder.AllocateId();
 	state.builder.AddFunction({OpGroupNonUniformBallot, TypeU32Vector(state, 4), ballot,
 	                           ConstantU32(state, ScopeSubgroup), exec});
@@ -659,8 +652,8 @@ uint32_t AppendConsume(ValueEmitContext& ctx, const IR::Inst& inst, bool gds, bo
 	const auto high = state.builder.AllocateId();
 	state.builder.AddFunction({OpCompositeExtract, TypeU32(state), low, ballot, 0});
 	state.builder.AddFunction({OpCompositeExtract, TypeU32(state), high, ballot, 1});
-	const auto count_low  = state.per_invocation_masks ? low : ctx.Arg(inst, 3);
+	const auto count_low  = state.per_invocation_masks ? low : ctx.Arg(inst, 2);
-	const auto count_high = state.per_invocation_masks ? high : ctx.Arg(inst, 4);
+	const auto count_high = state.per_invocation_masks ? high : ctx.Arg(inst, 3);
 	const auto count =
 	    Binary(state, OpIAdd, TypeU32(state), Unary(state, OpBitCount, TypeU32(state), count_low),
 	           Unary(state, OpBitCount, TypeU32(state), count_high));
@@ -671,16 +664,18 @@ uint32_t AppendConsume(ValueEmitContext& ctx, const IR::Inst& inst, bool gds, bo
 	    Binary(state, OpIEqual, TypeBool(state), EmitSubgroupLocalInvocationId(state), first);
 	const auto storage_bounds = EmitMemoryElementInBounds(state, access, index);
 	const auto m0_bounds =
-	    gds ? Binary(state, OpINotEqual, TypeBool(state), size, ConstantU32(state, 0))
+	    mem.kind == IR::ResourceKind::Gds
+	        ? Binary(state, OpINotEqual, TypeBool(state), size, ConstantU32(state, 0))
 	        : Binary(state, OpULessThan, TypeBool(state),
 	                 ConstantU32(state, ctx.Memory(inst).offset + 3u), size);
 	const auto condition = AndCondition(
 	    state, is_first, AndCondition(state, exec, AndCondition(state, storage_bounds, m0_bounds)));
 	const auto atomic = EmitValueOrZeroIfCondition(state, condition, [&]() {
 		const auto value = state.builder.AllocateId();
-		state.builder.AddFunction({append ? OpAtomicIAdd : OpAtomicISub, TypeU32(state), value,
+		state.builder.AddFunction(
+		    {append ? OpAtomicIAdd : OpAtomicISub, TypeU32(state), value,
 		     EmitMemoryElementPointer(state, access, index),
-		                           ConstantU32(state, gds ? ScopeDevice : ScopeWorkgroup),
+		     ConstantU32(state, mem.kind == IR::ResourceKind::Gds ? ScopeDevice : ScopeWorkgroup),
 		     ConstantU32(state, MemorySemanticsNone), count});
 		return value;
 	});
@@ -718,8 +713,8 @@ PreparedFormattedMemory PrepareFormattedMemory(ValueEmitContext& ctx, const IR::
 			}
 		}
 	} else {
-		for (uint32_t component = 0;
+		for (uint32_t component = 0; component < std::min(components, plan.info.component_count);
-		     component < std::min(components, plan.info.component_count); component++) {
+		     component++) {
 			required_components[component] = true;
 		}
 	}
@@ -862,6 +857,54 @@ void StoreWideBuffer(ValueEmitContext& ctx, const IR::Inst& inst, uint32_t compo
 	});
 }
 
+uint32_t LoadWideShared(ValueEmitContext& ctx, const IR::Inst& inst, uint32_t components) {
+	auto& state = ctx.state;
+	return EmitValueOrDefaultIfCondition(
+	    state, ctx.Arg(inst, inst.NumArgs() - 1), TypeU32Composite(state, components),
+	    ConstantU32CompositeZero(state, components), [&]() {
+		    const auto              mem      = ctx.Memory(inst);
+		    const auto              resource = PrepareMemoryResourceAccess(state, mem);
+		    const auto              base     = ByteAddress(ctx, inst, mem);
+		    std::array<uint32_t, 4> values {};
+		    for (uint32_t component = 0; component < components; component++) {
+			    const auto address = component == 0u
+			                             ? base
+			                             : Binary(state, OpIAdd, TypeU32(state), base,
+			                                      ConstantU32(state, component * 4u));
+			    const auto raw_index = Binary(state, OpShiftRightLogical, TypeU32(state), address,
+			                                  ConstantU32(state, 2));
+			    const auto access    = PrepareMemoryElement(ctx, mem, resource, raw_index);
+			    values[component]    = EmitValueOrZeroIfCondition(
+			        state, EmitMemoryElementInBounds(state, access.resource, access.index),
+			        [&]() { return LoadWordInBounds(ctx, access.resource, access.index); });
+		    }
+		    return ConstructU32Composite(state, components, values);
+	    });
+}
+
+void StoreWideShared(ValueEmitContext& ctx, const IR::Inst& inst, uint32_t components) {
+	auto& state = ctx.state;
+	EmitIfCondition(state, ctx.Arg(inst, inst.NumArgs() - 1), [&]() {
+		const auto mem      = ctx.Memory(inst);
+		const auto resource = PrepareMemoryResourceAccess(state, mem);
+		const auto base     = ByteAddress(ctx, inst, mem);
+		for (uint32_t component = 0; component < components; component++) {
+			const auto address = component == 0u
+			                         ? base
+			                         : Binary(state, OpIAdd, TypeU32(state), base,
+			                                  ConstantU32(state, component * 4u));
+			const auto raw_index =
+			    Binary(state, OpShiftRightLogical, TypeU32(state), address, ConstantU32(state, 2));
+			const auto access = PrepareMemoryElement(ctx, mem, resource, raw_index);
+			EmitIfCondition(state, EmitMemoryElementInBounds(state, access.resource, access.index),
+			                [&]() {
+				                StoreWordInBounds(ctx, access.resource, access.index,
+				                                  ctx.Arg(inst, component + 1u));
+			                });
+		}
+	});
+}
+
 } // namespace
 
 bool EmitValueMemory(ValueEmitContext& ctx, const IR::Inst& inst) {
@@ -876,6 +919,15 @@ bool EmitValueMemory(ValueEmitContext& ctx, const IR::Inst& inst) {
 		}
 		return true;
 	}
+	const auto shared_components = IR::SharedComponentCount(op);
+	if (shared_components > 1u) {
+		if (IR::SharedAccessOf(op) == IR::SharedAccess::Read) {
+			ctx.Define(inst, LoadWideShared(ctx, inst, shared_components));
+		} else {
+			StoreWideShared(ctx, inst, shared_components);
+		}
+		return true;
+	}
 	if ((op == IR::ValueOpcode::LoadAddressU32 || op == IR::ValueOpcode::ReadConstBuffer) &&
 	    ctx.Memory(inst).planning_only) {
 		return true;
@@ -917,25 +969,17 @@ bool EmitValueMemory(ValueEmitContext& ctx, const IR::Inst& inst) {
 	const bool load_buffer  = op == IR::ValueOpcode::LoadBufferU8 ||
 	                          op == IR::ValueOpcode::LoadBufferU16 ||
 	                          op == IR::ValueOpcode::LoadBufferU32;
-	const bool load_shared =
+	const bool load_shared = IR::SharedAccessOf(op) == IR::SharedAccess::Read;
-	    op == IR::ValueOpcode::LoadSharedU8 || op == IR::ValueOpcode::LoadSharedU16 ||
-	    op == IR::ValueOpcode::LoadSharedU32 || op == IR::ValueOpcode::LoadGdsU8 ||
-	    op == IR::ValueOpcode::LoadGdsU16 || op == IR::ValueOpcode::LoadGdsU32;
 	if (load_address || load_buffer || load_shared) {
-		auto mem = ctx.Memory(inst);
+		const auto mem   = ctx.Memory(inst);
-		if (load_shared)
-			mem.kind = op == IR::ValueOpcode::LoadGdsU8 || op == IR::ValueOpcode::LoadGdsU16 ||
-			                   op == IR::ValueOpcode::LoadGdsU32
-			               ? IR::ResourceKind::Gds
-			               : IR::ResourceKind::Lds;
 		uint32_t   value = 0;
 		if (op == IR::ValueOpcode::LoadBufferU32 && mem.formatted)
 			value = FormattedLoad(ctx, inst, mem);
 		else if (op == IR::ValueOpcode::LoadAddressU8 || op == IR::ValueOpcode::LoadBufferU8 ||
-		         op == IR::ValueOpcode::LoadSharedU8 || op == IR::ValueOpcode::LoadGdsU8)
+		         op == IR::ValueOpcode::LoadSharedU8)
 			value = LoadSubword(ctx, inst, mem, 8, false);
 		else if (op == IR::ValueOpcode::LoadAddressU16 || op == IR::ValueOpcode::LoadBufferU16 ||
-		         op == IR::ValueOpcode::LoadSharedU16 || op == IR::ValueOpcode::LoadGdsU16)
+		         op == IR::ValueOpcode::LoadSharedU16)
 			value = LoadSubword(ctx, inst, mem, 16, false);
 		else
 			value = LoadWord(ctx, inst, mem);
@@ -948,24 +992,16 @@ bool EmitValueMemory(ValueEmitContext& ctx, const IR::Inst& inst) {
 	const bool store_buffer  = op == IR::ValueOpcode::StoreBufferU8 ||
 	                           op == IR::ValueOpcode::StoreBufferU16 ||
 	                           op == IR::ValueOpcode::StoreBufferU32;
-	const bool store_shared =
+	const bool store_shared = IR::SharedAccessOf(op) == IR::SharedAccess::Write;
-	    op == IR::ValueOpcode::WriteSharedU8 || op == IR::ValueOpcode::WriteSharedU16 ||
-	    op == IR::ValueOpcode::WriteSharedU32 || op == IR::ValueOpcode::WriteGdsU8 ||
-	    op == IR::ValueOpcode::WriteGdsU16 || op == IR::ValueOpcode::WriteGdsU32;
 	if (store_address || store_buffer || store_shared) {
-		auto mem = ctx.Memory(inst);
+		const auto mem = ctx.Memory(inst);
-		if (store_shared)
-			mem.kind = op == IR::ValueOpcode::WriteGdsU8 || op == IR::ValueOpcode::WriteGdsU16 ||
-			                   op == IR::ValueOpcode::WriteGdsU32
-			               ? IR::ResourceKind::Gds
-			               : IR::ResourceKind::Lds;
 		if (op == IR::ValueOpcode::StoreBufferU32 && mem.formatted)
 			FormattedStore(ctx, inst, mem);
 		else if (op == IR::ValueOpcode::StoreAddressU8 || op == IR::ValueOpcode::StoreBufferU8 ||
-		         op == IR::ValueOpcode::WriteSharedU8 || op == IR::ValueOpcode::WriteGdsU8)
+		         op == IR::ValueOpcode::WriteSharedU8)
 			StoreSubword(ctx, inst, mem, 8);
 		else if (op == IR::ValueOpcode::StoreAddressU16 || op == IR::ValueOpcode::StoreBufferU16 ||
-		         op == IR::ValueOpcode::WriteSharedU16 || op == IR::ValueOpcode::WriteGdsU16)
+		         op == IR::ValueOpcode::WriteSharedU16)
 			StoreSubword(ctx, inst, mem, 16);
 		else
 			StoreWord(ctx, inst, mem);
@@ -973,12 +1009,7 @@ bool EmitValueMemory(ValueEmitContext& ctx, const IR::Inst& inst) {
 	}
 	const auto atomic_opcode = AtomicOpcode(op);
 	if (atomic_opcode != 0) {
-		auto mem = ctx.Memory(inst);
+		ctx.Define(inst, EmitAtomic(ctx, inst, ctx.Memory(inst), true));
-		if (op >= IR::ValueOpcode::SharedAtomicSwap32 && op <= IR::ValueOpcode::SharedAtomicXor32)
-			mem.kind = IR::ResourceKind::Lds;
-		else if (op >= IR::ValueOpcode::GdsAtomicSwap32 && op <= IR::ValueOpcode::GdsAtomicXor32)
-			mem.kind = IR::ResourceKind::Gds;
-		ctx.Define(inst, EmitAtomic(ctx, inst, mem, true));
 		return true;
 	}
 	if (op == IR::ValueOpcode::BufferAtomicFMin32 || op == IR::ValueOpcode::BufferAtomicFMax32) {
@@ -986,24 +1017,12 @@ bool EmitValueMemory(ValueEmitContext& ctx, const IR::Inst& inst) {
 		                             op == IR::ValueOpcode::BufferAtomicFMax32));
 		return true;
 	}
-	if (op == IR::ValueOpcode::SharedAtomicFMin32 || op == IR::ValueOpcode::SharedAtomicFMax32 ||
+	if (op == IR::ValueOpcode::SharedAtomicFMin32 || op == IR::ValueOpcode::SharedAtomicFMax32) {
-	    op == IR::ValueOpcode::GdsAtomicFMin32 || op == IR::ValueOpcode::GdsAtomicFMax32) {
+		SharedFloatAtomic(ctx, inst, ctx.Memory(inst), op == IR::ValueOpcode::SharedAtomicFMax32);
-		auto mem = ctx.Memory(inst);
-		mem.kind = op == IR::ValueOpcode::GdsAtomicFMin32 || op == IR::ValueOpcode::GdsAtomicFMax32
-		               ? IR::ResourceKind::Gds
-		               : IR::ResourceKind::Lds;
-		SharedFloatAtomic(ctx, inst, mem,
-		                  op == IR::ValueOpcode::SharedAtomicFMax32 ||
-		                      op == IR::ValueOpcode::GdsAtomicFMax32);
 		return true;
 	}
-	if (op == IR::ValueOpcode::DataAppend || op == IR::ValueOpcode::DataConsume ||
+	if (op == IR::ValueOpcode::DataAppend || op == IR::ValueOpcode::DataConsume) {
-	    op == IR::ValueOpcode::GdsDataAppend || op == IR::ValueOpcode::GdsDataConsume) {
+		ctx.Define(inst, AppendConsume(ctx, inst, op == IR::ValueOpcode::DataAppend));
-		ctx.Define(inst, AppendConsume(ctx, inst,
-		                               op == IR::ValueOpcode::GdsDataAppend ||
-		                                   op == IR::ValueOpcode::GdsDataConsume,
-		                               op == IR::ValueOpcode::DataAppend ||
-		                                   op == IR::ValueOpcode::GdsDataAppend));
 		return true;
 	}
 	if (op == IR::ValueOpcode::SwizzleU32) {

src/graphics/shader/recompiler/backend/spirv/spirvEmitterMemoryHelpers.cpp

@@ -93,7 +93,11 @@ MemoryResourceAccess PrepareMemoryResourceAccess(EmitterState& state, const IR::
 				                             "GDS binding was not emitted");
 			}
 			access.object_pointer = state.gds_variable;
-			break;
+			if (state.gds_length == 0) {
+				EXIT("GDS length was not prepared at function entry\n");
+			}
+			access.length = state.gds_length;
+			return access;
 		case IR::ResourceKind::ScalarAddress:
 		case IR::ResourceKind::Flat:
 		case IR::ResourceKind::Global:

src/graphics/shader/recompiler/backend/spirv/spirvEmitterProgram.cpp

@@ -592,9 +592,7 @@ bool EmitValueProgram(EmitterState& state, const IR::ValueProgram& program, std:
 		if (std::ranges::any_of(*block, [](const IR::Inst& inst) {
 			    return inst.GetOpcode() == IR::ValueOpcode::SwizzleU32 ||
 			           inst.GetOpcode() == IR::ValueOpcode::SharedAtomicFMin32 ||
-			           inst.GetOpcode() == IR::ValueOpcode::SharedAtomicFMax32 ||
+			           inst.GetOpcode() == IR::ValueOpcode::SharedAtomicFMax32;
-			           inst.GetOpcode() == IR::ValueOpcode::GdsAtomicFMin32 ||
-			           inst.GetOpcode() == IR::ValueOpcode::GdsAtomicFMax32;
 		    })) {
 			ctx.scratch_u32_variable = state.builder.AllocateId();
 			break;
@@ -631,6 +629,11 @@ bool EmitValueProgram(EmitterState& state, const IR::ValueProgram& program, std:
 		                           TypePointer(state, StorageClassFunction, TypeU32(state)),
 		                           ctx.scratch_u32_variable, StorageClassFunction});
 	}
+	if (state.gds_variable != 0) {
+		state.gds_length = state.builder.AllocateId();
+		state.builder.AddFunction(
+		    {OpArrayLength, TypeU32(state), state.gds_length, state.gds_variable, 0});
+	}
 	if (state.pixel_valid_mask_variable != 0) {
 		state.builder.AddFunction(
 		    {OpStore, state.pixel_valid_mask_variable, ConstantU32(state, 1)});

src/graphics/shader/recompiler/frontend/decode/MemoryOps.cpp

@@ -96,7 +96,7 @@ constexpr MemoryOpcodeInfo DS_OPCODE_LIST[] = {
     {0x29u, Opcode::DS_AND_RTN_B32, 1, 32},      {0x2au, Opcode::DS_OR_RTN_B32, 1, 32},
     {0x2bu, Opcode::DS_XOR_RTN_B32, 1, 32},      {0x2du, Opcode::DS_WRXCHG_RTN_B32, 1, 32},
     {0x35u, Opcode::DS_SWIZZLE_B32, 1, 32},      {0x36u, Opcode::DS_READ_B32, 1, 32},
-    {0x37u, Opcode::DS_READ2_B32, 2, 32},        {0x38u, Opcode::DS_READ2_B32, 2, 32},
+    {0x37u, Opcode::DS_READ2_B32, 2, 32},        {0x38u, Opcode::DS_READ2ST64_B32, 2, 32},
     {0x39u, Opcode::DS_READ_I8, 1, 8, true},     {0x3au, Opcode::DS_READ_U8, 1, 8},
     {0x3bu, Opcode::DS_READ_I16, 1, 16, true},   {0x3cu, Opcode::DS_READ_U16, 1, 16},
     {0x3du, Opcode::DS_CONSUME, 1, 32},          {0x3eu, Opcode::DS_APPEND, 1, 32},
@@ -403,10 +403,11 @@ void DecodeDs(uint32_t pc, std::span<const uint32_t> code, uint32_t word_index,
 		SetUnsupported(inst, Family::DS, opcode,
 		               "DS read addtid data operands are not implemented");
 	}
-	if (inst.opcode == Opcode::DS_WRITE2_B32 || opcode == 0x37u) {
+	if (inst.opcode == Opcode::DS_WRITE2_B32 || inst.opcode == Opcode::DS_READ2_B32) {
 		inst.offset           = offset0 * 4u;
 		inst.secondary_offset = offset1 * 4u;
-	} else if (inst.opcode == Opcode::DS_WRITE2ST64_B32 || opcode == 0x38u) {
+	} else if (inst.opcode == Opcode::DS_WRITE2ST64_B32 ||
+	           inst.opcode == Opcode::DS_READ2ST64_B32) {
 		inst.offset           = offset0 * 256u;
 		inst.secondary_offset = offset1 * 256u;
 	} else if (inst.opcode == Opcode::DS_WRITE2_B64 || inst.opcode == Opcode::DS_READ2_B64) {

src/graphics/shader/recompiler/frontend/decode/ShaderDecoder.cpp

@@ -658,6 +658,7 @@ std::string InstructionToString(const Instruction& inst) {
 		case Opcode::DS_READ_I16:
 		case Opcode::DS_READ_U16:
 		case Opcode::DS_READ2_B32:
+		case Opcode::DS_READ2ST64_B32:
 		case Opcode::DS_READ_B32:
 		case Opcode::DS_READ_B64:
 		case Opcode::DS_READ2_B64:

src/graphics/shader/recompiler/frontend/decode/ShaderDecoder.h

@@ -490,6 +490,7 @@ enum class Opcode {
 	DS_READ_I16,
 	DS_READ_U16,
 	DS_READ2_B32,
+	DS_READ2ST64_B32,
 	DS_READ_B32,
 	DS_READ_B64,
 	DS_READ2_B64,

src/graphics/shader/recompiler/frontend/translate/Memory.cpp

@@ -65,11 +65,6 @@ IR::Value Translator::GetSamplerResource(const IR::MemoryInfo& memory) {
 	                GetResourceDword(memory.sampler, 2), GetResourceDword(memory.sampler, 3)});
 }
 
-IR::Value Translator::GetSharedResource(IR::ResourceKind kind) {
-	return ir.Emit(kind == IR::ResourceKind::Gds ? IR::ValueOpcode::GetGdsResource
-	                                             : IR::ValueOpcode::GetLdsResource);
-}
-
 IR::Value Translator::MakeImageAddress(const IR::Instruction& inst, const IR::Operand& base) {
 	std::array<IR::Value, 13> components {};
 	components[0] = ReadRawU32(PlainOperand(base));
@@ -170,22 +165,7 @@ IR::ValueOpcode Translator::BufferAtomicOpcode(IR::Opcode opcode) {
 	}
 }
 
-IR::ValueOpcode Translator::SharedAtomicOpcode(IR::Opcode opcode, bool gds) {
+IR::ValueOpcode Translator::SharedAtomicOpcode(IR::Opcode opcode) {
-	if (gds) {
-		switch (opcode) {
-			case IR::Opcode::AtomicSwapU32: return IR::ValueOpcode::GdsAtomicSwap32;
-			case IR::Opcode::AtomicAddU32: return IR::ValueOpcode::GdsAtomicIAdd32;
-			case IR::Opcode::AtomicSubU32: return IR::ValueOpcode::GdsAtomicISub32;
-			case IR::Opcode::AtomicSMinI32: return IR::ValueOpcode::GdsAtomicSMin32;
-			case IR::Opcode::AtomicUMinU32: return IR::ValueOpcode::GdsAtomicUMin32;
-			case IR::Opcode::AtomicSMaxI32: return IR::ValueOpcode::GdsAtomicSMax32;
-			case IR::Opcode::AtomicUMaxU32: return IR::ValueOpcode::GdsAtomicUMax32;
-			case IR::Opcode::AtomicAndU32: return IR::ValueOpcode::GdsAtomicAnd32;
-			case IR::Opcode::AtomicOrU32: return IR::ValueOpcode::GdsAtomicOr32;
-			case IR::Opcode::AtomicXorU32: return IR::ValueOpcode::GdsAtomicXor32;
-			default: EXIT("invalid GDS atomic opcode");
-		}
-	}
 	switch (opcode) {
 		case IR::Opcode::AtomicSwapU32: return IR::ValueOpcode::SharedAtomicSwap32;
 		case IR::Opcode::AtomicAddU32: return IR::ValueOpcode::SharedAtomicIAdd32;
@@ -197,7 +177,7 @@ IR::ValueOpcode Translator::SharedAtomicOpcode(IR::Opcode opcode, bool gds) {
 		case IR::Opcode::AtomicAndU32: return IR::ValueOpcode::SharedAtomicAnd32;
 		case IR::Opcode::AtomicOrU32: return IR::ValueOpcode::SharedAtomicOr32;
 		case IR::Opcode::AtomicXorU32: return IR::ValueOpcode::SharedAtomicXor32;
-		default: EXIT("invalid LDS atomic opcode");
+		default: EXIT("invalid shared atomic opcode");
 	}
 }
 
@@ -363,13 +343,9 @@ bool Translator::TranslateAtomicMemory(const IR::Instruction& inst) {
 		}
 		case IR::ResourceKind::Lds:
 		case IR::ResourceKind::Gds: {
-			const bool gds      = inst.memory.kind == IR::ResourceKind::Gds;
+			const auto address = ReadU32(inst.src[1]);
-			const auto resource = GetSharedResource(inst.memory.kind);
+			result = ir.Emit(SharedAtomicOpcode(inst.op),
-			const auto address =
+			                 {address, ReadU32(inst.src[0]), ir.GetExec()}, AddMemoryInfo(inst));
-			    ir.IAdd(ReadU32(inst.src[1]), IR::U32(IR::Value(inst.memory.offset)));
-			result = ir.Emit(SharedAtomicOpcode(inst.op, gds),
-			                 {resource, address, ReadU32(inst.src[0]), ir.GetExec()},
-			                 AddMemoryInfo(inst));
 			break;
 		}
 		default: return false;
@@ -518,10 +494,51 @@ bool Translator::TranslateSharedMemory(const IR::Instruction& inst) {
 	if (!shared && inst.op != IR::Opcode::DsSwizzleB32) {
 		return false;
 	}
-	const bool gds            = inst.memory.kind == IR::ResourceKind::Gds;
+	const auto shared_address = [&](uint32_t source) { return ReadU32(inst.src[source]); };
-	const auto resource       = shared ? GetSharedResource(inst.memory.kind) : IR::Value {};
+	const auto load_u32 = [&](uint32_t width, IR::U32 address, const IR::Instruction& source) {
-	const auto shared_address = [&](uint32_t source) {
+		IR::ValueOpcode opcode;
-		return ir.IAdd(ReadU32(inst.src[source]), IR::U32(IR::Value(inst.memory.offset)));
+		switch (width) {
+			case 1u: opcode = IR::ValueOpcode::LoadSharedU32; break;
+			case 2u: opcode = IR::ValueOpcode::LoadSharedU32x2; break;
+			case 3u: opcode = IR::ValueOpcode::LoadSharedU32x3; break;
+			case 4u: opcode = IR::ValueOpcode::LoadSharedU32x4; break;
+			default: EXIT("invalid shared load width");
+		}
+		return ir.Emit(opcode, {address, ir.GetExec()}, AddMemoryInfo(source));
+	};
+	const auto extract_u32 = [&](IR::Value value, uint32_t width, uint32_t index) {
+		if (width == 1u) {
+			return value;
+		}
+		const auto opcode = width == 2u   ? IR::ValueOpcode::CompositeExtractU32x2
+		                    : width == 3u ? IR::ValueOpcode::CompositeExtractU32x3
+		                                  : IR::ValueOpcode::CompositeExtractU32x4;
+		return ir.Emit(opcode, {value, IR::Value(index)});
+	};
+	const auto write_u32 = [&](uint32_t width, IR::U32 address,
+	                           const std::array<IR::Value, 4>& values,
+	                           const IR::Instruction&          source) {
+		switch (width) {
+			case 1u:
+				ir.Emit(IR::ValueOpcode::WriteSharedU32, {address, values[0], ir.GetExec()},
+				        AddMemoryInfo(source));
+				break;
+			case 2u:
+				ir.Emit(IR::ValueOpcode::WriteSharedU32x2,
+				        {address, values[0], values[1], ir.GetExec()}, AddMemoryInfo(source));
+				break;
+			case 3u:
+				ir.Emit(IR::ValueOpcode::WriteSharedU32x3,
+				        {address, values[0], values[1], values[2], ir.GetExec()},
+				        AddMemoryInfo(source));
+				break;
+			case 4u:
+				ir.Emit(IR::ValueOpcode::WriteSharedU32x4,
+				        {address, values[0], values[1], values[2], values[3], ir.GetExec()},
+				        AddMemoryInfo(source));
+				break;
+			default: EXIT("invalid shared store width");
+		}
 	};
 	switch (inst.op) {
 		case IR::Opcode::DsReadUbyte:
@@ -533,65 +550,109 @@ bool Translator::TranslateSharedMemory(const IR::Instruction& inst) {
 			uint32_t        bits;
 			bool            sign;
 			if (inst.op == IR::Opcode::DsReadUbyte || inst.op == IR::Opcode::DsReadSbyte) {
-				opcode = gds ? IR::ValueOpcode::LoadGdsU8 : IR::ValueOpcode::LoadSharedU8;
+				opcode = IR::ValueOpcode::LoadSharedU8;
 				bits   = 8u;
 				sign   = inst.op == IR::Opcode::DsReadSbyte;
 			} else if (inst.op == IR::Opcode::DsReadUshort || inst.op == IR::Opcode::DsReadSshort) {
-				opcode = gds ? IR::ValueOpcode::LoadGdsU16 : IR::ValueOpcode::LoadSharedU16;
+				opcode = IR::ValueOpcode::LoadSharedU16;
 				bits   = 16u;
 				sign   = inst.op == IR::Opcode::DsReadSshort;
 			} else {
-				opcode = gds ? IR::ValueOpcode::LoadGdsU32 : IR::ValueOpcode::LoadSharedU32;
+				const auto width  = inst.memory.data_dwords;
-				bits   = 32u;
+				const auto loaded = load_u32(width, shared_address(0), inst);
-				sign   = false;
+				for (uint32_t index = 0; index < width; index++) {
+					WriteOperand(OffsetOperand(inst.dst, index), extract_u32(loaded, width, index));
+				}
+				return true;
 			}
 			const auto loaded =
-			    ir.Emit(opcode, {resource, shared_address(0), ir.GetExec()}, AddMemoryInfo(inst));
+			    ir.Emit(opcode, {shared_address(0), ir.GetExec()}, AddMemoryInfo(inst));
 			WriteOperand(inst.dst, bits == 32u ? loaded : WidenSubdword(loaded, bits, sign));
 			return true;
 		}
+		case IR::Opcode::DsRead2B32: {
+			const auto width             = inst.memory.data_dwords / 2u;
+			const auto address           = shared_address(0);
+			auto       first             = inst;
+			first.memory.data_dwords     = width;
+			first.memory.component_count = width;
+			const auto first_value       = load_u32(width, address, first);
+			IR::Value  second_value      = first_value;
+			if (inst.memory.secondary_offset != inst.memory.offset) {
+				auto second          = first;
+				second.memory.offset = inst.memory.secondary_offset;
+				second_value         = load_u32(width, address, second);
+			}
+			for (uint32_t index = 0; index < width; index++) {
+				WriteOperand(OffsetOperand(inst.dst, index),
+				             extract_u32(first_value, width, index));
+				WriteOperand(OffsetOperand(inst.dst, width + index),
+				             extract_u32(second_value, width, index));
+			}
+			return true;
+		}
 		case IR::Opcode::DsWriteByte:
 		case IR::Opcode::DsWriteShort:
 		case IR::Opcode::DsWriteB32: {
-			const auto      data    = ReadU32(inst.src[0]);
+			const auto               width = inst.memory.data_dwords;
+			std::array<IR::Value, 4> values {};
+			for (uint32_t index = 0; index < width; index++) {
+				values[index] = ReadU32(OffsetOperand(inst.src[0], index));
+			}
+			const auto      data    = IR::U32(values[0]);
 			const auto      address = shared_address(1);
 			IR::ValueOpcode opcode;
 			IR::Value       value;
 			if (inst.op == IR::Opcode::DsWriteByte) {
-				opcode = gds ? IR::ValueOpcode::WriteGdsU8 : IR::ValueOpcode::WriteSharedU8;
+				opcode = IR::ValueOpcode::WriteSharedU8;
 				value  = NarrowSubdword(data, 8u);
 			} else if (inst.op == IR::Opcode::DsWriteShort) {
-				opcode = gds ? IR::ValueOpcode::WriteGdsU16 : IR::ValueOpcode::WriteSharedU16;
+				opcode = IR::ValueOpcode::WriteSharedU16;
 				value  = NarrowSubdword(data, 16u);
 			} else {
-				opcode = gds ? IR::ValueOpcode::WriteGdsU32 : IR::ValueOpcode::WriteSharedU32;
+				write_u32(width, address, values, inst);
-				value  = data;
+				return true;
+			}
+			ir.Emit(opcode, {address, value, ir.GetExec()}, AddMemoryInfo(inst));
+			return true;
+		}
+		case IR::Opcode::DsWrite2B32: {
+			const auto               width   = inst.memory.data_dwords / 2u;
+			const auto               address = shared_address(1);
+			std::array<IR::Value, 4> first_values {};
+			std::array<IR::Value, 4> second_values {};
+			for (uint32_t index = 0; index < width; index++) {
+				first_values[index]  = ReadU32(OffsetOperand(inst.src[0], index));
+				second_values[index] = ReadU32(OffsetOperand(inst.src[2], index));
+			}
+			auto first                   = inst;
+			first.memory.data_dwords     = width;
+			first.memory.component_count = width;
+			write_u32(width, address, first_values, first);
+			if (inst.memory.secondary_offset != inst.memory.offset) {
+				auto second          = first;
+				second.memory.offset = inst.memory.secondary_offset;
+				write_u32(width, address, second_values, second);
 			}
-			ir.Emit(opcode, {resource, address, value, ir.GetExec()}, AddMemoryInfo(inst));
 			return true;
 		}
 		case IR::Opcode::DsMinF32:
 		case IR::Opcode::DsMaxF32: {
-			const auto opcode =
+			const auto opcode = inst.op == IR::Opcode::DsMinF32
-			    gds ? (inst.op == IR::Opcode::DsMinF32 ? IR::ValueOpcode::GdsAtomicFMin32
+			                        ? IR::ValueOpcode::SharedAtomicFMin32
-			                                           : IR::ValueOpcode::GdsAtomicFMax32)
+			                        : IR::ValueOpcode::SharedAtomicFMax32;
-			        : (inst.op == IR::Opcode::DsMinF32 ? IR::ValueOpcode::SharedAtomicFMin32
-			                                           : IR::ValueOpcode::SharedAtomicFMax32);
 			ir.Emit(opcode,
-			        {resource, shared_address(1), ReadU32(inst.src[0]), ReadU32(inst.src[2]),
+			        {shared_address(1), ReadU32(inst.src[0]), ReadU32(inst.src[2]), ir.GetExec()},
-			         ir.GetExec()},
 			        AddMemoryInfo(inst));
 			return true;
 		}
 		case IR::Opcode::DsAppend:
 		case IR::Opcode::DsConsume: {
 			const bool append = inst.op == IR::Opcode::DsAppend;
-			const auto opcode =
+			const auto opcode = append ? IR::ValueOpcode::DataAppend : IR::ValueOpcode::DataConsume;
-			    gds ? (append ? IR::ValueOpcode::GdsDataAppend : IR::ValueOpcode::GdsDataConsume)
-			        : (append ? IR::ValueOpcode::DataAppend : IR::ValueOpcode::DataConsume);
 			WriteOperand(inst.dst, ir.Emit(opcode,
-			                               {resource, ReadU32(inst.src[0]), ir.GetExec(),
+			                               {ReadU32(inst.src[0]), ir.GetExec(), ir.GetExecLo(),
-			                                ir.GetExecLo(), ir.GetExecHi()},
+			                                ir.GetExecHi()},
 			                               AddMemoryInfo(inst)));
 			return true;
 		}
@@ -601,16 +662,13 @@ bool Translator::TranslateSharedMemory(const IR::Instruction& inst) {
 			const auto base =
 			    ir.BitwiseAnd(ReadU32(inst.src[m0_index]), IR::U32(IR::Value(0xffffu)));
 			const auto lane    = IR::U32(ir.Emit(IR::ValueOpcode::LaneId));
-			const auto address = ir.IAdd(ir.IAdd(base, IR::U32(IR::Value(inst.memory.offset))),
+			const auto address = ir.IAdd(base, ir.ShiftLeftLogical(lane, IR::U32(IR::Value(2u))));
-			                             ir.ShiftLeftLogical(lane, IR::U32(IR::Value(2u))));
 			if (inst.op == IR::Opcode::DsWriteAddtidB32) {
 				ir.Emit(IR::ValueOpcode::WriteSharedU32,
-				        {resource, address, ReadU32(inst.src[0]), ir.GetExec()},
+				        {address, ReadU32(inst.src[0]), ir.GetExec()}, AddMemoryInfo(inst));
-				        AddMemoryInfo(inst));
 			} else {
-				WriteOperand(inst.dst,
+				WriteOperand(inst.dst, ir.Emit(IR::ValueOpcode::LoadSharedU32,
-				             ir.Emit(IR::ValueOpcode::LoadSharedU32,
+				                               {address, ir.GetExec()}, AddMemoryInfo(inst)));
-				                     {resource, address, ir.GetExec()}, AddMemoryInfo(inst)));
 			}
 			return true;
 		}
@@ -676,9 +734,11 @@ bool Translator::TranslateMemoryOperation(const IR::Instruction&          inst,
 		case IR::Opcode::DsReadUshort:
 		case IR::Opcode::DsReadSshort:
 		case IR::Opcode::DsReadB32:
+		case IR::Opcode::DsRead2B32:
 		case IR::Opcode::DsWriteByte:
 		case IR::Opcode::DsWriteShort:
 		case IR::Opcode::DsWriteB32:
+		case IR::Opcode::DsWrite2B32:
 		case IR::Opcode::DsMinF32:
 		case IR::Opcode::DsMaxF32:
 		case IR::Opcode::DsSwizzleB32:

src/graphics/shader/recompiler/frontend/translate/Translate.cpp

@@ -768,6 +768,14 @@ bool TranslateProgram(const IR::Program& source, IR::ValueProgram& result,
 				include_vector(inst.dst, std::max(inst.memory.data_dwords, 1u));
 			} else if (inst.op == IR::Opcode::BufferStoreDword) {
 				include_vector(inst.src[0], std::max(inst.memory.data_dwords, 1u));
+			} else if (inst.op == IR::Opcode::DsReadB32 || inst.op == IR::Opcode::DsRead2B32) {
+				include_vector(inst.dst, std::max(inst.memory.data_dwords, 1u));
+			} else if (inst.op == IR::Opcode::DsWriteB32) {
+				include_vector(inst.src[0], std::max(inst.memory.data_dwords, 1u));
+			} else if (inst.op == IR::Opcode::DsWrite2B32) {
+				const auto width = std::max(inst.memory.data_dwords / 2u, 1u);
+				include_vector(inst.src[0], width);
+				include_vector(inst.src[2], width);
 			} else if (inst.op == IR::Opcode::LoadInputF32) {
 				include_vector(inst.dst, std::max(inst.input_info.component_count, 1u));
 			}

src/graphics/shader/recompiler/frontend/translate/Translator.h

@@ -75,7 +75,6 @@ private:
 	IR::Value       GetScalarAddressResource(uint32_t base);
 	IR::Value       GetImageResource(const IR::MemoryInfo& memory);
 	IR::Value       GetSamplerResource(const IR::MemoryInfo& memory);
-	IR::Value       GetSharedResource(IR::ResourceKind kind);
 	IR::Value       MakeImageAddress(const IR::Instruction& inst, const IR::Operand& base);
 	IR::Value       ConstructU32x4(const IR::Operand& base, uint32_t count);
 	void            WriteImageComponents(const IR::Operand& dst, IR::Value value, uint32_t dmask,
@@ -85,7 +84,7 @@ private:
 	IR::U32         WidenSubdword(IR::Value value, uint32_t bits, bool sign);
 	IR::Value       NarrowSubdword(IR::U32 value, uint32_t bits);
 	IR::ValueOpcode BufferAtomicOpcode(IR::Opcode opcode);
-	IR::ValueOpcode SharedAtomicOpcode(IR::Opcode opcode, bool gds);
+	IR::ValueOpcode SharedAtomicOpcode(IR::Opcode opcode);
 	bool            IsScalarMemoryLoadOperation(IR::Opcode opcode);
 	ScalarMemorySourceValues ReadScalarMemorySource(const IR::Instruction& inst);
 	bool                     TranslateScalarMemory(const IR::Instruction&          inst,

src/graphics/shader/recompiler/ir/ShaderIRMemory.cpp

@@ -169,41 +169,32 @@ ResourceKind DsMemoryKind(const Decoder::Instruction& decoded) {
 
 bool LowerDsRead(const Decoder::Instruction& decoded, BasicBlock& block, std::string* error) {
 	const auto  ir_op = DsReadOpcode(decoded);
-	const auto count = decoded.data_bits == 32u ? decoded.data_dwords : 1u;
-	for (uint32_t i = 0; i < count; i++) {
 	Instruction inst;
 	inst.pc        = decoded.pc;
 	inst.op        = ir_op;
 	inst.src_count = 1;
-		inst.memory    = OffsetMemoryInfo(decoded, DsMemoryKind(decoded), i);
+	inst.memory    = MemoryInfoFromDecoded(decoded, DsMemoryKind(decoded));
-		if (!LowerRegisterOperand(OffsetDecodedRegister(decoded.dst, i), inst.dst, error) ||
+	if (!LowerRegisterOperand(decoded.dst, inst.dst, error) ||
 	    !LowerSourceOperand(decoded.src0, inst.src[0], error)) {
 		return false;
 	}
 	block.instructions.push_back(inst);
-	}
 	return true;
 }
 
 bool LowerDsRead2(const Decoder::Instruction& decoded, BasicBlock& block, uint32_t dwords_per_read,
                   std::string* error) {
-	const uint32_t offsets[] = {decoded.offset, decoded.secondary_offset};
-	for (uint32_t read = 0; read < 2u; read++) {
-		for (uint32_t dword = 0; dword < dwords_per_read; dword++) {
-			const auto  index = read * dwords_per_read + dword;
 	Instruction inst;
 	inst.pc                 = decoded.pc;
-			inst.op        = Opcode::DsReadB32;
+	inst.op                 = Opcode::DsRead2B32;
 	inst.src_count          = 1;
-			inst.memory =
+	inst.memory             = MemoryInfoFromDecoded(decoded, DsMemoryKind(decoded));
-			    ByteOffsetMemoryInfo(decoded, DsMemoryKind(decoded), offsets[read] + dword * 4u);
+	inst.memory.data_dwords = dwords_per_read * 2u;
-			if (!LowerRegisterOperand(OffsetDecodedRegister(decoded.dst, index), inst.dst, error) ||
+	if (!LowerRegisterOperand(decoded.dst, inst.dst, error) ||
 	    !LowerSourceOperand(decoded.src0, inst.src[0], error)) {
 		return false;
 	}
 	block.instructions.push_back(inst);
-		}
-	}
 	return true;
 }
 
@@ -409,44 +400,35 @@ bool LowerFlatStore(const Decoder::Instruction& decoded, BasicBlock& block, std:
 
 bool LowerDsWrite(const Decoder::Instruction& decoded, BasicBlock& block, std::string* error) {
 	const auto  ir_op = DsWriteOpcode(decoded.data_bits);
-	const auto count = decoded.data_bits == 32u ? decoded.data_dwords : 1u;
-	for (uint32_t i = 0; i < count; i++) {
 	Instruction inst;
 	inst.pc        = decoded.pc;
 	inst.op        = ir_op;
 	inst.src_count = 2;
-		inst.memory    = OffsetMemoryInfo(decoded, DsMemoryKind(decoded), i);
+	inst.memory    = MemoryInfoFromDecoded(decoded, DsMemoryKind(decoded));
 	inst.dst.kind  = OperandKind::Null;
-		if (!LowerSourceOperand(OffsetDecodedRegister(decoded.src1, i), inst.src[0], error) ||
+	if (!LowerSourceOperand(decoded.src1, inst.src[0], error) ||
 	    !LowerSourceOperand(decoded.src0, inst.src[1], error)) {
 		return false;
 	}
 	block.instructions.push_back(inst);
-	}
 	return true;
 }
 
 bool LowerDsWrite2(const Decoder::Instruction& decoded, BasicBlock& block,
                    uint32_t dwords_per_write, std::string* error) {
-	const Decoder::Operand* data[]    = {&decoded.src1, &decoded.src2};
-	const uint32_t          offsets[] = {decoded.offset, decoded.secondary_offset};
-	for (uint32_t write = 0; write < 2u; write++) {
-		for (uint32_t dword = 0; dword < dwords_per_write; dword++) {
 	Instruction inst;
 	inst.pc                 = decoded.pc;
-			inst.op        = Opcode::DsWriteB32;
+	inst.op                 = Opcode::DsWrite2B32;
-			inst.src_count = 2;
+	inst.src_count          = 3;
-			inst.memory =
+	inst.memory             = MemoryInfoFromDecoded(decoded, DsMemoryKind(decoded));
-			    ByteOffsetMemoryInfo(decoded, DsMemoryKind(decoded), offsets[write] + dword * 4u);
+	inst.memory.data_dwords = dwords_per_write * 2u;
 	inst.dst.kind           = OperandKind::Null;
-			if (!LowerSourceOperand(OffsetDecodedRegister(*data[write], dword), inst.src[0],
+	if (!LowerSourceOperand(decoded.src1, inst.src[0], error) ||
-			                        error) ||
+	    !LowerSourceOperand(decoded.src0, inst.src[1], error) ||
-			    !LowerSourceOperand(decoded.src0, inst.src[1], error)) {
+	    !LowerSourceOperand(decoded.src2, inst.src[2], error)) {
 		return false;
 	}
 	block.instructions.push_back(inst);
-		}
-	}
 	return true;
 }
 
@@ -618,7 +600,8 @@ bool LowerMemoryInstruction(const Decoder::Instruction& decoded, BasicBlock& blo
 			return LowerDsWriteAddtidB32(decoded, block, error);
 		case Decoder::Opcode::DS_READ_ADDTID_B32:
 			return LowerDsReadAddtidB32(decoded, block, error);
-		case Decoder::Opcode::DS_READ2_B32: return LowerDsRead2(decoded, block, 1, error);
+		case Decoder::Opcode::DS_READ2_B32:
+		case Decoder::Opcode::DS_READ2ST64_B32: return LowerDsRead2(decoded, block, 1, error);
 		case Decoder::Opcode::DS_READ2_B64:
 		case Decoder::Opcode::DS_READ2ST64_B64: return LowerDsRead2(decoded, block, 2, error);
 		case Decoder::Opcode::DS_READ_I8:
@@ -765,6 +748,7 @@ bool IsMemoryOpcode(Decoder::Opcode opcode) {
 		case Decoder::Opcode::DS_READ_I16:
 		case Decoder::Opcode::DS_READ_U16:
 		case Decoder::Opcode::DS_READ2_B32:
+		case Decoder::Opcode::DS_READ2ST64_B32:
 		case Decoder::Opcode::DS_READ_B32:
 		case Decoder::Opcode::DS_READ_B64:
 		case Decoder::Opcode::DS_READ2_B64:

src/graphics/shader/recompiler/ir/Type.cpp

@@ -31,8 +31,6 @@ std::string TypeName(Type type) {
 	    "AddressResource",
 	    "ImageResource",
 	    "SamplerResource",
-	    "LdsResource",
-	    "GdsResource",
 	    "ImageAddress",
 	};
 	const auto bits = static_cast<uint32_t>(type);

src/graphics/shader/recompiler/ir/Type.h

@@ -28,9 +28,7 @@ enum class Type : uint32_t {
 	AddressResource = 1u << 18u,
 	ImageResource   = 1u << 19u,
 	SamplerResource = 1u << 20u,
-	LdsResource     = 1u << 21u,
+	ImageAddress    = 1u << 21u,
-	GdsResource     = 1u << 22u,
-	ImageAddress    = 1u << 23u,
 };
 
 constexpr Type operator|(Type lhs, Type rhs) {

src/graphics/shader/recompiler/ir/ValueProgram.cpp

@@ -338,6 +338,34 @@ bool ValidateValueProgram(const ValueProgram& program, bool require_ssa, std::st
 					                               ValueOpcodeName(inst.GetOpcode())));
 				}
 			}
+			const auto shared_components = SharedComponentCount(inst.GetOpcode());
+			if (shared_components != 0u) {
+				const auto memory_index = inst.Flags<MemoryFlags>().index;
+				if (memory_index >= program.memory_info.size()) {
+					return Fail(error, fmt::format("{} has an invalid memory-info index",
+					                               ValueOpcodeName(inst.GetOpcode())));
+				}
+				const auto& memory = program.memory_info[memory_index];
+				if ((memory.kind != ResourceKind::Lds && memory.kind != ResourceKind::Gds) ||
+				    memory.resource != 0u || memory.sampler != 0u || memory.component_count == 0u ||
+				    memory.component_index >= memory.component_count) {
+					return Fail(error, fmt::format("{} has invalid shared-memory metadata",
+					                               ValueOpcodeName(inst.GetOpcode())));
+				}
+				uint32_t expected_bits = 32u;
+				if (inst.GetOpcode() == ValueOpcode::LoadSharedU8 ||
+				    inst.GetOpcode() == ValueOpcode::WriteSharedU8) {
+					expected_bits = 8u;
+				} else if (inst.GetOpcode() == ValueOpcode::LoadSharedU16 ||
+				           inst.GetOpcode() == ValueOpcode::WriteSharedU16) {
+					expected_bits = 16u;
+				}
+				if (memory.data_bits != expected_bits || memory.data_dwords != shared_components ||
+				    (shared_components > 1u && memory.component_index != 0u)) {
+					return Fail(error, fmt::format("{} has inconsistent shared-memory width",
+					                               ValueOpcodeName(inst.GetOpcode())));
+				}
+			}
 			uint32_t composite_components = 0u;
 			switch (inst.GetOpcode()) {
 				case ValueOpcode::CompositeExtractU32x2: composite_components = 2u; break;
@@ -358,7 +386,8 @@ bool ValidateValueProgram(const ValueProgram& program, bool require_ssa, std::st
 					                               ValueOpcodeName(inst.GetOpcode())));
 				}
 				if (fixed_signature && arg.GetType() != ArgTypeOf(inst.GetOpcode(), arg_index)) {
-					return Fail(error, fmt::format("{} argument {} has type {}, expected {}",
+					return Fail(error,
+					            fmt::format("{} argument {} has type {}, expected {}",
 					                        ValueOpcodeName(inst.GetOpcode()), arg_index,
 					                        TypeName(arg.GetType()),
 					                        TypeName(ArgTypeOf(inst.GetOpcode(), arg_index))));

src/graphics/shader/recompiler/ir/opcodes/ShaderIROpcodes.cpp

@@ -392,19 +392,20 @@ constexpr LowerMap LOWER_OPS[] = {
     {Decoder::Opcode::DS_READ_U8, Opcode::DsReadUbyte},
     {Decoder::Opcode::DS_READ_I16, Opcode::DsReadSshort},
     {Decoder::Opcode::DS_READ_U16, Opcode::DsReadUshort},
-    {Decoder::Opcode::DS_READ2_B32, Opcode::DsReadB32},
+    {Decoder::Opcode::DS_READ2_B32, Opcode::DsRead2B32},
+    {Decoder::Opcode::DS_READ2ST64_B32, Opcode::DsRead2B32},
     {Decoder::Opcode::DS_READ_B32, Opcode::DsReadB32},
     {Decoder::Opcode::DS_READ_B64, Opcode::DsReadB32},
-    {Decoder::Opcode::DS_READ2_B64, Opcode::DsReadB32},
+    {Decoder::Opcode::DS_READ2_B64, Opcode::DsRead2B32},
-    {Decoder::Opcode::DS_READ2ST64_B64, Opcode::DsReadB32},
+    {Decoder::Opcode::DS_READ2ST64_B64, Opcode::DsRead2B32},
     {Decoder::Opcode::DS_READ_B96, Opcode::DsReadB32},
     {Decoder::Opcode::DS_READ_B128, Opcode::DsReadB32},
     {Decoder::Opcode::DS_WRITE_B8, Opcode::DsWriteByte},
     {Decoder::Opcode::DS_WRITE_B16, Opcode::DsWriteShort},
-    {Decoder::Opcode::DS_WRITE2_B32, Opcode::DsWriteB32},
+    {Decoder::Opcode::DS_WRITE2_B32, Opcode::DsWrite2B32},
-    {Decoder::Opcode::DS_WRITE2ST64_B32, Opcode::DsWriteB32},
+    {Decoder::Opcode::DS_WRITE2ST64_B32, Opcode::DsWrite2B32},
-    {Decoder::Opcode::DS_WRITE2_B64, Opcode::DsWriteB32},
+    {Decoder::Opcode::DS_WRITE2_B64, Opcode::DsWrite2B32},
-    {Decoder::Opcode::DS_WRITE2ST64_B64, Opcode::DsWriteB32},
+    {Decoder::Opcode::DS_WRITE2ST64_B64, Opcode::DsWrite2B32},
     {Decoder::Opcode::DS_WRITE_B32, Opcode::DsWriteB32},
     {Decoder::Opcode::DS_WRITE_B64, Opcode::DsWriteB32},
     {Decoder::Opcode::DS_WRITE_B96, Opcode::DsWriteB32},

src/graphics/shader/recompiler/ir/opcodes/ShaderIROpcodes.inc

@@ -331,9 +331,11 @@ IR_OPCODE(DsReadSbyte, General, Memory)
 IR_OPCODE(DsReadUshort, General, Memory)
 IR_OPCODE(DsReadSshort, General, Memory)
 IR_OPCODE(DsReadB32, General, Memory)
+IR_OPCODE(DsRead2B32, General, Memory)
 IR_OPCODE(DsWriteByte, General, Memory)
 IR_OPCODE(DsWriteShort, General, Memory)
 IR_OPCODE(DsWriteB32, General, Memory)
+IR_OPCODE(DsWrite2B32, General, Memory)
 IR_OPCODE(DsMinF32, General, Memory)
 IR_OPCODE(DsMaxF32, General, Memory)
 IR_OPCODE(DsSwizzleB32, General, Memory)

src/graphics/shader/recompiler/ir/opcodes/ValueOpcodes.cpp

@@ -42,8 +42,6 @@ constexpr Type BufferResource  = Type::BufferResource;
 constexpr Type AddressResource = Type::AddressResource;
 constexpr Type ImageResource   = Type::ImageResource;
 constexpr Type SamplerResource = Type::SamplerResource;
-constexpr Type LdsResource     = Type::LdsResource;
-constexpr Type GdsResource     = Type::GdsResource;
 constexpr Type ImageAddress    = Type::ImageAddress;
 
 constexpr std::array<OpcodeMeta, static_cast<size_t>(ValueOpcode::Count)> MetaTable = {{
@@ -76,52 +74,23 @@ bool HasSideEffects(ValueOpcode opcode) {
 	if (buffer_access == BufferAccess::Write || buffer_access == BufferAccess::Atomic) {
 		return true;
 	}
+	const auto shared_access = SharedAccessOf(opcode);
+	if (shared_access == SharedAccess::Write || shared_access == SharedAccess::Atomic ||
+	    shared_access == SharedAccess::Append || shared_access == SharedAccess::Consume) {
+		return true;
+	}
 	switch (opcode) {
 		case ValueOpcode::Reference:
 		case ValueOpcode::ReferenceU32:
 		case ValueOpcode::StoreAddressU8:
 		case ValueOpcode::StoreAddressU16:
 		case ValueOpcode::StoreAddressU32:
-		case ValueOpcode::WriteSharedU8:
-		case ValueOpcode::WriteSharedU16:
-		case ValueOpcode::WriteSharedU32:
-		case ValueOpcode::WriteGdsU8:
-		case ValueOpcode::WriteGdsU16:
-		case ValueOpcode::WriteGdsU32:
-		case ValueOpcode::SharedAtomicFMin32:
-		case ValueOpcode::SharedAtomicFMax32:
-		case ValueOpcode::GdsAtomicFMin32:
-		case ValueOpcode::GdsAtomicFMax32:
-		case ValueOpcode::SharedAtomicSwap32:
-		case ValueOpcode::SharedAtomicIAdd32:
-		case ValueOpcode::SharedAtomicISub32:
-		case ValueOpcode::SharedAtomicSMin32:
-		case ValueOpcode::SharedAtomicUMin32:
-		case ValueOpcode::SharedAtomicSMax32:
-		case ValueOpcode::SharedAtomicUMax32:
-		case ValueOpcode::SharedAtomicAnd32:
-		case ValueOpcode::SharedAtomicOr32:
-		case ValueOpcode::SharedAtomicXor32:
-		case ValueOpcode::GdsAtomicSwap32:
-		case ValueOpcode::GdsAtomicIAdd32:
-		case ValueOpcode::GdsAtomicISub32:
-		case ValueOpcode::GdsAtomicSMin32:
-		case ValueOpcode::GdsAtomicUMin32:
-		case ValueOpcode::GdsAtomicSMax32:
-		case ValueOpcode::GdsAtomicUMax32:
-		case ValueOpcode::GdsAtomicAnd32:
-		case ValueOpcode::GdsAtomicOr32:
-		case ValueOpcode::GdsAtomicXor32:
 		case ValueOpcode::ImageAtomicIAdd32:
 		case ValueOpcode::ImageAtomicUMin32:
 		case ValueOpcode::ImageAtomicUMax32:
 		case ValueOpcode::ImageAtomicAnd32:
 		case ValueOpcode::ImageAtomicOr32:
 		case ValueOpcode::ImageAtomicXor32:
-		case ValueOpcode::DataAppend:
-		case ValueOpcode::DataConsume:
-		case ValueOpcode::GdsDataAppend:
-		case ValueOpcode::GdsDataConsume:
 		case ValueOpcode::ImageWrite:
 		case ValueOpcode::SetAttribute:
 		case ValueOpcode::Barrier: return true;
@@ -172,4 +141,48 @@ uint32_t BufferComponentCount(ValueOpcode opcode) {
 	}
 }
 
+SharedAccess SharedAccessOf(ValueOpcode opcode) {
+	switch (opcode) {
+		case ValueOpcode::LoadSharedU8:
+		case ValueOpcode::LoadSharedU16:
+		case ValueOpcode::LoadSharedU32:
+		case ValueOpcode::LoadSharedU32x2:
+		case ValueOpcode::LoadSharedU32x3:
+		case ValueOpcode::LoadSharedU32x4: return SharedAccess::Read;
+		case ValueOpcode::WriteSharedU8:
+		case ValueOpcode::WriteSharedU16:
+		case ValueOpcode::WriteSharedU32:
+		case ValueOpcode::WriteSharedU32x2:
+		case ValueOpcode::WriteSharedU32x3:
+		case ValueOpcode::WriteSharedU32x4: return SharedAccess::Write;
+		case ValueOpcode::SharedAtomicFMin32:
+		case ValueOpcode::SharedAtomicFMax32:
+		case ValueOpcode::SharedAtomicSwap32:
+		case ValueOpcode::SharedAtomicIAdd32:
+		case ValueOpcode::SharedAtomicISub32:
+		case ValueOpcode::SharedAtomicSMin32:
+		case ValueOpcode::SharedAtomicUMin32:
+		case ValueOpcode::SharedAtomicSMax32:
+		case ValueOpcode::SharedAtomicUMax32:
+		case ValueOpcode::SharedAtomicAnd32:
+		case ValueOpcode::SharedAtomicOr32:
+		case ValueOpcode::SharedAtomicXor32: return SharedAccess::Atomic;
+		case ValueOpcode::DataAppend: return SharedAccess::Append;
+		case ValueOpcode::DataConsume: return SharedAccess::Consume;
+		default: return SharedAccess::None;
+	}
+}
+
+uint32_t SharedComponentCount(ValueOpcode opcode) {
+	switch (opcode) {
+		case ValueOpcode::LoadSharedU32x2:
+		case ValueOpcode::WriteSharedU32x2: return 2u;
+		case ValueOpcode::LoadSharedU32x3:
+		case ValueOpcode::WriteSharedU32x3: return 3u;
+		case ValueOpcode::LoadSharedU32x4:
+		case ValueOpcode::WriteSharedU32x4: return 4u;
+		default: return SharedAccessOf(opcode) == SharedAccess::None ? 0u : 1u;
+	}
+}
+
 } // namespace Libs::Graphics::ShaderRecompiler::IR

src/graphics/shader/recompiler/ir/opcodes/ValueOpcodes.h

@@ -17,6 +17,7 @@ enum class ValueOpcode {
 };
 
 enum class BufferAccess { None, Read, Write, Atomic };
+enum class SharedAccess { None, Read, Write, Atomic, Append, Consume };
 
 struct DppMoveFlags {
 	uint16_t control        = 0;
@@ -56,6 +57,8 @@ static_assert(std::is_trivially_copyable_v<ExportFlags>);
 [[nodiscard]] bool             HasSideEffects(ValueOpcode opcode);
 [[nodiscard]] BufferAccess     BufferAccessOf(ValueOpcode opcode);
 [[nodiscard]] uint32_t         BufferComponentCount(ValueOpcode opcode);
+[[nodiscard]] SharedAccess     SharedAccessOf(ValueOpcode opcode);
+[[nodiscard]] uint32_t         SharedComponentCount(ValueOpcode opcode);
 [[nodiscard]] std::string_view ValueOpcodeName(ValueOpcode opcode);
 
 } // namespace Libs::Graphics::ShaderRecompiler::IR

src/graphics/shader/recompiler/ir/opcodes/ValueOpcodes.inc

@@ -202,8 +202,6 @@ VALUE_OPCODE(GetBufferResource, BufferResource, U32, U32, U32, U32)
 VALUE_OPCODE(GetAddressResource, AddressResource, U32, U32)
 VALUE_OPCODE(GetImageResource, ImageResource, U32, U32, U32, U32, U32, U32, U32, U32)
 VALUE_OPCODE(GetSamplerResource, SamplerResource, U32, U32, U32, U32)
-VALUE_OPCODE(GetLdsResource, LdsResource)
-VALUE_OPCODE(GetGdsResource, GdsResource)
 VALUE_OPCODE(MakeImageAddress, ImageAddress, U32, U32, U32, U32, U32, U32, U32, U32, U32, U32, U32,
              U32, U32)
 
@@ -240,46 +238,32 @@ VALUE_OPCODE(BufferAtomicXor32, U32, BufferResource, U32, U32, U32, U32, U1)
 VALUE_OPCODE(BufferAtomicFMin32, U32, BufferResource, U32, U32, U32, U32, U1)
 VALUE_OPCODE(BufferAtomicFMax32, U32, BufferResource, U32, U32, U32, U32, U1)
 
-VALUE_OPCODE(LoadSharedU8, U8, LdsResource, U32, U1)
+VALUE_OPCODE(LoadSharedU8, U8, U32, U1)
-VALUE_OPCODE(LoadSharedU16, U16, LdsResource, U32, U1)
+VALUE_OPCODE(LoadSharedU16, U16, U32, U1)
-VALUE_OPCODE(LoadSharedU32, U32, LdsResource, U32, U1)
+VALUE_OPCODE(LoadSharedU32, U32, U32, U1)
-VALUE_OPCODE(WriteSharedU8, Void, LdsResource, U32, U8, U1)
+VALUE_OPCODE(LoadSharedU32x2, U32x2, U32, U1)
-VALUE_OPCODE(WriteSharedU16, Void, LdsResource, U32, U16, U1)
+VALUE_OPCODE(LoadSharedU32x3, U32x3, U32, U1)
-VALUE_OPCODE(WriteSharedU32, Void, LdsResource, U32, U32, U1)
+VALUE_OPCODE(LoadSharedU32x4, U32x4, U32, U1)
-VALUE_OPCODE(LoadGdsU8, U8, GdsResource, U32, U1)
+VALUE_OPCODE(WriteSharedU8, Void, U32, U8, U1)
-VALUE_OPCODE(LoadGdsU16, U16, GdsResource, U32, U1)
+VALUE_OPCODE(WriteSharedU16, Void, U32, U16, U1)
-VALUE_OPCODE(LoadGdsU32, U32, GdsResource, U32, U1)
+VALUE_OPCODE(WriteSharedU32, Void, U32, U32, U1)
-VALUE_OPCODE(WriteGdsU8, Void, GdsResource, U32, U8, U1)
+VALUE_OPCODE(WriteSharedU32x2, Void, U32, U32, U32, U1)
-VALUE_OPCODE(WriteGdsU16, Void, GdsResource, U32, U16, U1)
+VALUE_OPCODE(WriteSharedU32x3, Void, U32, U32, U32, U32, U1)
-VALUE_OPCODE(WriteGdsU32, Void, GdsResource, U32, U32, U1)
+VALUE_OPCODE(WriteSharedU32x4, Void, U32, U32, U32, U32, U32, U1)
-VALUE_OPCODE(SharedAtomicFMin32, Void, LdsResource, U32, U32, U32, U1)
+VALUE_OPCODE(SharedAtomicFMin32, Void, U32, U32, U32, U1)
-VALUE_OPCODE(SharedAtomicFMax32, Void, LdsResource, U32, U32, U32, U1)
+VALUE_OPCODE(SharedAtomicFMax32, Void, U32, U32, U32, U1)
-VALUE_OPCODE(GdsAtomicFMin32, Void, GdsResource, U32, U32, U32, U1)
+VALUE_OPCODE(SharedAtomicSwap32, U32, U32, U32, U1)
-VALUE_OPCODE(GdsAtomicFMax32, Void, GdsResource, U32, U32, U32, U1)
+VALUE_OPCODE(SharedAtomicIAdd32, U32, U32, U32, U1)
-VALUE_OPCODE(SharedAtomicSwap32, U32, LdsResource, U32, U32, U1)
+VALUE_OPCODE(SharedAtomicISub32, U32, U32, U32, U1)
-VALUE_OPCODE(SharedAtomicIAdd32, U32, LdsResource, U32, U32, U1)
+VALUE_OPCODE(SharedAtomicSMin32, U32, U32, U32, U1)
-VALUE_OPCODE(SharedAtomicISub32, U32, LdsResource, U32, U32, U1)
+VALUE_OPCODE(SharedAtomicUMin32, U32, U32, U32, U1)
-VALUE_OPCODE(SharedAtomicSMin32, U32, LdsResource, U32, U32, U1)
+VALUE_OPCODE(SharedAtomicSMax32, U32, U32, U32, U1)
-VALUE_OPCODE(SharedAtomicUMin32, U32, LdsResource, U32, U32, U1)
+VALUE_OPCODE(SharedAtomicUMax32, U32, U32, U32, U1)
-VALUE_OPCODE(SharedAtomicSMax32, U32, LdsResource, U32, U32, U1)
+VALUE_OPCODE(SharedAtomicAnd32, U32, U32, U32, U1)
-VALUE_OPCODE(SharedAtomicUMax32, U32, LdsResource, U32, U32, U1)
+VALUE_OPCODE(SharedAtomicOr32, U32, U32, U32, U1)
-VALUE_OPCODE(SharedAtomicAnd32, U32, LdsResource, U32, U32, U1)
+VALUE_OPCODE(SharedAtomicXor32, U32, U32, U32, U1)
-VALUE_OPCODE(SharedAtomicOr32, U32, LdsResource, U32, U32, U1)
+VALUE_OPCODE(DataAppend, U32, U32, U1, U32, U32)
-VALUE_OPCODE(SharedAtomicXor32, U32, LdsResource, U32, U32, U1)
+VALUE_OPCODE(DataConsume, U32, U32, U1, U32, U32)
-VALUE_OPCODE(GdsAtomicSwap32, U32, GdsResource, U32, U32, U1)
-VALUE_OPCODE(GdsAtomicIAdd32, U32, GdsResource, U32, U32, U1)
-VALUE_OPCODE(GdsAtomicISub32, U32, GdsResource, U32, U32, U1)
-VALUE_OPCODE(GdsAtomicSMin32, U32, GdsResource, U32, U32, U1)
-VALUE_OPCODE(GdsAtomicUMin32, U32, GdsResource, U32, U32, U1)
-VALUE_OPCODE(GdsAtomicSMax32, U32, GdsResource, U32, U32, U1)
-VALUE_OPCODE(GdsAtomicUMax32, U32, GdsResource, U32, U32, U1)
-VALUE_OPCODE(GdsAtomicAnd32, U32, GdsResource, U32, U32, U1)
-VALUE_OPCODE(GdsAtomicOr32, U32, GdsResource, U32, U32, U1)
-VALUE_OPCODE(GdsAtomicXor32, U32, GdsResource, U32, U32, U1)
-VALUE_OPCODE(DataAppend, U32, LdsResource, U32, U1, U32, U32)
-VALUE_OPCODE(DataConsume, U32, LdsResource, U32, U1, U32, U32)
-VALUE_OPCODE(GdsDataAppend, U32, GdsResource, U32, U1, U32, U32)
-VALUE_OPCODE(GdsDataConsume, U32, GdsResource, U32, U1, U32, U32)
 VALUE_OPCODE(SwizzleU32, U32, U32, U32, U1)
 
 VALUE_OPCODE(ImageQueryDimensions, U32x4, ImageResource, ImageAddress)

src/graphics/shader/recompiler/ir/passes/BindingLayout.cpp

@@ -115,21 +115,26 @@ void AddBinding(BindingLayout& layout, DescriptorBindingKind kind,
 	    {kind, static_cast<uint32_t>(layout.descriptors.size()), std::move(resources)});
 }
 
-bool UsesGds(const Program& program) {
+bool UsesGds(const Program& program, bool& uses_gds) {
+	uses_gds = false;
 	for (const auto* block: program.values->blocks) {
 		for (const auto& inst: *block) {
-			if (inst.GetOpcode() == ValueOpcode::GetGdsResource && inst.HasUses()) {
+			if (SharedAccessOf(inst.GetOpcode()) == SharedAccess::None) {
-				return true;
+				continue;
-			}
-			for (size_t index = 0; index < inst.NumArgs(); index++) {
-				if (inst.Arg(index).GetType() == Type::GdsResource) {
-					return true;
-				}
-			}
-		}
 			}
+			const auto index = inst.Flags<MemoryFlags>().index;
+			if (index >= program.values->memory_info.size()) {
 				return false;
 			}
+			const auto kind = program.values->memory_info[index].kind;
+			if (kind != ResourceKind::Lds && kind != ResourceKind::Gds) {
+				return false;
+			}
+			uses_gds |= kind == ResourceKind::Gds;
+		}
+	}
+	return true;
+}
 
 } // namespace
 
@@ -220,7 +225,14 @@ bool AllocateBindings(Program& program, const BindingLayoutOptions& options, std
 		}
 		AddBinding(next, DescriptorBindingKind::Samplers, std::move(resources));
 	}
-	if (UsesGds(program)) {
+	bool uses_gds = false;
+	if (!UsesGds(program, uses_gds)) {
+		if (error != nullptr) {
+			*error = "typed shader contains invalid shared-memory metadata";
+		}
+		return false;
+	}
+	if (uses_gds) {
 		AddBinding(next, DescriptorBindingKind::Gds);
 	}
 	if (!program.info.addresses.empty()) {

tests/ResourceTrackingTests.cpp

@@ -256,8 +256,8 @@ void TestInvariantIndirectImageMaterialization() {
             fixture->program.values->descriptor_sources[source]
                 .indirect_image.has_value(),
         "indirect image source was not retained for runtime proof");
-  const auto image_handle = std::ranges::find_if(
+  const auto image_handle =
-      *fixture->block, [](const Inst &inst) {
+      std::ranges::find_if(*fixture->block, [](const Inst &inst) {
         return inst.GetOpcode() == ValueOpcode::GetImageResource;
       });
   Check(image_handle != fixture->block->end() &&
@@ -326,7 +326,8 @@ void TestInvariantIndirectImageMaterialization() {
   memory.words[(0x2020u - memory.base) / 4u + 3u] =
       image_descriptor[3] ^ (1u << 28u);
   ResourceSnapshot null_snapshot;
-  Check(MaterializeResources(fixture->program, runtime, null_snapshot, &error) &&
+  Check(
+      MaterializeResources(fixture->program, runtime, null_snapshot, &error) &&
           null_snapshot.indirect_images.empty() &&
           std::ranges::all_of(null_snapshot.images[0].dwords,
                               [](uint32_t dword) { return dword == 0u; }),
@@ -380,10 +381,11 @@ void TestInvariantIndirectImageMaterialization() {
   ResourceSnapshot capacity_snapshot;
   for (const uint32_t records : {1u, 3u}) {
     user_data[2] = records;
-    Check(MaterializeResources(fixture->program, runtime, capacity_snapshot,
+    Check(
+        MaterializeResources(fixture->program, runtime, capacity_snapshot,
                              &error) &&
-              ValidateResourceSpecialization(fixture->program,
+            ValidateResourceSpecialization(fixture->program, capacity_snapshot,
-                                             capacity_snapshot, &error),
+                                           &error),
         "runtime indirect key mapping rejected a fitting material-table size");
   }
   user_data[2] = 2u;
@@ -396,7 +398,8 @@ void TestInvariantIndirectImageMaterialization() {
         image_descriptor[dword];
   }
   memory.words[(0x1000u - memory.base + 68u) / 4u] = 2u;
-  Check(!MaterializeResources(fixture->program, runtime, rebound_snapshot,
+  Check(
+      !MaterializeResources(fixture->program, runtime, rebound_snapshot,
                             &error) &&
           error.find("candidate topology") != std::string::npos &&
           same_snapshot(rebound_snapshot, collapsed_snapshot),
@@ -985,14 +988,14 @@ void TestExecMaskedFlatAddressProvenance() {
   Fixture fixture;
   const auto low_root = fixture.UserData(0);
   const auto high_root = fixture.UserData(1);
-  const auto active = fixture.Emit(
+  const auto active =
-      ValueOpcode::INotEqual32, {fixture.UserData(2), Value(0u)});
+      fixture.Emit(ValueOpcode::INotEqual32, {fixture.UserData(2), Value(0u)});
   const auto inactive_low = fixture.Emit(ValueOpcode::UndefU32);
   const auto inactive_high = fixture.Emit(ValueOpcode::UndefU32);
-  const auto low = fixture.Emit(ValueOpcode::SelectU32,
+  const auto low =
-                                {active, low_root, inactive_low});
+      fixture.Emit(ValueOpcode::SelectU32, {active, low_root, inactive_low});
-  const auto high = fixture.Emit(ValueOpcode::SelectU32,
+  const auto high =
-                                 {active, high_root, inactive_high});
+      fixture.Emit(ValueOpcode::SelectU32, {active, high_root, inactive_high});
   const auto address = fixture.Address(low, high, 0xa4);
   MemoryInfo flat;
   flat.kind = ResourceKind::Flat;
@@ -1015,19 +1018,18 @@ void TestExecMaskedFlatAddressProvenance() {
         "exec-masked FLAT address materialized the wrong user-data root");
 
   Fixture mismatch;
-  const auto mismatch_active = mismatch.Emit(
+  const auto mismatch_active = mismatch.Emit(ValueOpcode::INotEqual32,
-      ValueOpcode::INotEqual32, {mismatch.UserData(2), Value(0u)});
+                                             {mismatch.UserData(2), Value(0u)});
-  const auto other_active = mismatch.Emit(ValueOpcode::LogicalNot,
+  const auto other_active =
-                                          {mismatch_active});
+      mismatch.Emit(ValueOpcode::LogicalNot, {mismatch_active});
   const auto mismatch_low = mismatch.Emit(
-      ValueOpcode::SelectU32,
+      ValueOpcode::SelectU32, {mismatch_active, mismatch.UserData(0),
-      {mismatch_active, mismatch.UserData(0),
                                mismatch.Emit(ValueOpcode::UndefU32)});
   const auto mismatch_high = mismatch.Emit(
-      ValueOpcode::SelectU32,
+      ValueOpcode::SelectU32, {mismatch_active, mismatch.UserData(1),
-      {mismatch_active, mismatch.UserData(1),
                                mismatch.Emit(ValueOpcode::UndefU32)});
-  const auto mismatch_address = mismatch.Address(mismatch_low, mismatch_high, 0xa4);
+  const auto mismatch_address =
+      mismatch.Address(mismatch_low, mismatch_high, 0xa4);
   mismatch.Emit(ValueOpcode::LoadAddressU8,
                 {mismatch_address, mismatch_low, mismatch_high, other_active},
                 mismatch.AddMemory(flat, 0xa4));
@@ -1086,9 +1088,10 @@ void TestShaderInfoAndBindingLayout() {
       {Value(static_cast<uint32_t>(StageInputKind::GlobalInvocationId)),
        Value(2u)});
   fixture.Emit(ValueOpcode::BitwiseXor32, {Value(1u), Value(2u)});
-  const auto gds = fixture.Emit(ValueOpcode::GetGdsResource);
+  MemoryInfo gds;
-  fixture.Emit(ValueOpcode::WriteGdsU32,
+  gds.kind = ResourceKind::Gds;
-               {gds, Value(0u), Value(1u), Value(true)});
+  fixture.Emit(ValueOpcode::WriteSharedU32, {Value(0u), Value(1u), Value(true)},
+               fixture.AddMemory(gds, 8));
   fixture.PlanAndTrack();
 
   ShaderComputeInputInfo compute{};

tests/shaderCfgTests.cpp

@@ -976,7 +976,7 @@ void TestSpirvRequirementsAnalysis() {
   block->AppendNewInst(ValueOpcode::ImageGatherRaw,
                        {Value(0u), Value(0u), Value(0u)});
   auto &shared = block->AppendNewInst(ValueOpcode::LoadSharedU32,
-                                      {Value(0u), Value(0u), Value(true)});
+                                      {Value(0u), Value(true)});
   shared.SetFlags(MemoryFlags{.index = 0});
   auto &export_value =
       block->AppendNewInst(ValueOpcode::SetAttribute, {Value(0u), Value(true)});
@@ -1095,11 +1095,9 @@ void TestNativeSubgroupPolicy() {
     } else if (opcode == ValueOpcode::Ballot) {
       block->AppendNewInst(opcode, {Value(true)});
     } else if (opcode == ValueOpcode::DataAppend) {
-      auto &resource = block->AppendNewInst(ValueOpcode::GetLdsResource);
       program.values->memory_info.push_back({.kind = ResourceKind::Lds});
-      auto &append =
+      auto &append = block->AppendNewInst(
-          block->AppendNewInst(opcode, {Value(&resource), Value(0u),
+          opcode, {Value(0u), Value(true), Value(1u), Value(0u)});
-                                        Value(true), Value(1u), Value(0u)});
       append.SetFlags(MemoryFlags{.index = 0});
     }
     std::string error;
@@ -1748,6 +1746,7 @@ void TestNewShaderRecompilerScalarVectorAlu() {
   Check(SpirvContainsOpcode(result.spirv, 199),
         "SPIR-V binary does not contain OpBitwiseAnd");
   CheckSpirvBinaryValidates(result.spirv);
+
 }
 
 void TestNewShaderRecompilerVop3LaneReadDestinationEncoding() {
@@ -4102,8 +4101,6 @@ void TestNewShaderRecompilerMemoryFamilyLowering() {
         "MIMG sample did not lower to IR");
   Check(SpirvContainsOpcode(result.spirv, 65),
         "SPIR-V binary does not contain OpAccessChain");
-  Check(SpirvContainsOpcode(result.spirv, 61),
-        "SPIR-V binary does not contain OpLoad");
   Check(SpirvContainsOpcode(result.spirv, 62),
         "SPIR-V binary does not contain OpStore");
   Check(SpirvContainsOpcode(result.spirv, 95),
@@ -5329,8 +5326,6 @@ void TestNewShaderRecompilerVintrpLowering() {
   Check(ProgramInputCount(result.program,
                           ShaderRecompiler::IR::StageInputKind::Parameter) == 2,
         "VINTRP pixel shader did not reflect parameter inputs");
-  Check(SpirvContainsOpcode(result.spirv, 61),
-        "SPIR-V binary does not contain OpLoad");
   Check(SpirvContainsOpcode(result.spirv, 62),
         "SPIR-V binary does not contain OpStore");
   Check(SpirvContainsOpcode(result.spirv, 81),
@@ -5608,8 +5603,6 @@ void TestNewShaderRecompilerWideMemoryLowering() {
         "global segment load did not lower with global metadata");
   Check(SpirvContainsOpcode(result.spirv, 65),
         "SPIR-V binary does not contain OpAccessChain");
-  Check(SpirvContainsOpcode(result.spirv, 61),
-        "SPIR-V binary does not contain OpLoad");
   Check(SpirvContainsOpcode(result.spirv, 62),
         "SPIR-V binary does not contain OpStore");
   Check(SpirvContainsOpcode(result.spirv, 194),
@@ -5807,8 +5800,6 @@ void TestNewShaderRecompilerMubufFormatLowering() {
         "MUBUF formatted stores were not preserved as native-width operations");
   Check(SpirvContainsOpcode(result.spirv, 65),
         "SPIR-V binary does not contain OpAccessChain");
-  Check(SpirvContainsOpcode(result.spirv, 61),
-        "SPIR-V binary does not contain OpLoad");
   Check(SpirvContainsOpcode(result.spirv, 62),
         "SPIR-V binary does not contain OpStore");
   CheckSpirvBinaryValidates(result.spirv);
@@ -6283,18 +6274,60 @@ void TestNewShaderRecompilerDsReadWrite2Lowering() {
       0xbf810000u,
   };
 
+  ShaderRecompiler::Decoder::Program decoded;
+  ShaderRecompiler::CFG::Graph graph;
+  ShaderRecompiler::IR::Program lowered;
+  std::string error;
+  Check(ShaderRecompiler::Decoder::DecodeProgram(std::span{shader}, decoded,
+                                                 &error) &&
+            ShaderRecompiler::CFG::BuildGraph(decoded, graph, &error) &&
+            ShaderRecompiler::IR::LowerProgram(
+                decoded, graph, ShaderType::Compute, 64, lowered, &error),
+        error.c_str());
+  uint32_t pair_reads = 0;
+  uint32_t pair_writes = 0;
+  for (const auto &block : lowered.blocks) {
+    for (const auto &inst : block.instructions) {
+      pair_reads += inst.op == ShaderRecompiler::IR::Opcode::DsRead2B32;
+      pair_writes += inst.op == ShaderRecompiler::IR::Opcode::DsWrite2B32;
+    }
+  }
+  Check(pair_reads == 3u && pair_writes == 1u,
+        "DS read/write2 was split before typed SSA");
+  ShaderRecompiler::IR::ValueProgram typed;
+  ShaderComputeInputInfo compute{};
+  Check(ShaderRecompiler::Frontend::TranslateProgram(lowered, typed, nullptr,
+                                                     nullptr, &compute, &error),
+        error.c_str());
+  uint32_t scalar_loads = 0;
+  uint32_t vector_loads = 0;
+  uint32_t scalar_stores = 0;
+  for (const auto *block : typed.blocks) {
+    for (const auto &inst : *block) {
+      scalar_loads +=
+          inst.GetOpcode() == ShaderRecompiler::IR::ValueOpcode::LoadSharedU32;
+      vector_loads += inst.GetOpcode() ==
+                      ShaderRecompiler::IR::ValueOpcode::LoadSharedU32x2;
+      scalar_stores +=
+          inst.GetOpcode() == ShaderRecompiler::IR::ValueOpcode::WriteSharedU32;
+    }
+  }
+  Check(scalar_loads == 4u && vector_loads == 2u && scalar_stores == 2u,
+        "DS read/write2 did not use the native scalar/x2 transaction shape");
+
   auto options = MakeCompileOptions(ShaderType::Compute);
   options.dump_ir = true;
 
   ShaderRecompiler::CompileResult result;
-  std::string error;
   Check(ShaderRecompiler::TryRecompile(shader, options, result, &error),
         error.c_str());
-  Check(Common::ContainsStr(result.decoded_dump, "ds_write2_b32"),
+  Check(Common::ContainsStr(result.decoded_dump, "DS_WRITE2_B32"),
         "new decoder did not decode old-backed DS write2");
-  Check(Common::ContainsStr(result.decoded_dump, "ds_read2_b32"),
+  Check(Common::ContainsStr(result.decoded_dump, "DS_READ2_B32"),
         "new decoder did not decode old-backed DS read2");
-  Check(Common::ContainsStr(result.decoded_dump, "ds_read2_b64"),
+  Check(Common::ContainsStr(result.decoded_dump, "DS_READ2ST64_B32"),
+        "new decoder did not preserve the DS read2st64 opcode identity");
+  Check(Common::ContainsStr(result.decoded_dump, "DS_READ2_B64"),
         "new decoder did not decode old-backed DS read2 b64");
   Check(Common::ContainsStr(result.decoded_dump, "offset=4"),
         "DS write2 decode did not scale offset0 to bytes");
@@ -6316,24 +6349,6 @@ void TestNewShaderRecompilerDsReadWrite2Lowering() {
         "DS read/write2 decode did not preserve two-dword metadata");
   Check(Common::ContainsStr(result.decoded_dump, "dwords=4 bits=32"),
         "DS read2 b64 decode did not preserve four-dword metadata");
-  Check(Common::ContainsStr(result.ir_dump, "DsWriteB32 null, v60"),
-        "DS write2 did not lower first dword through shared LDS store IR");
-  Check(Common::ContainsStr(result.ir_dump, "DsWriteB32 null, v61"),
-        "DS write2 did not lower second dword through shared LDS store IR");
-  Check(Common::ContainsStr(result.ir_dump, "DsReadB32 v70"),
-        "DS read2 did not lower first dword through shared LDS load IR");
-  Check(Common::ContainsStr(result.ir_dump, "DsReadB32 v71"),
-        "DS read2 did not lower second dword through shared LDS load IR");
-  Check(Common::ContainsStr(result.ir_dump, "DsReadB32 v72"),
-        "DS read2 st64 did not lower first dword through shared LDS load IR");
-  Check(Common::ContainsStr(result.ir_dump, "DsReadB32 v73"),
-        "DS read2 st64 did not lower second dword through shared LDS load IR");
-  Check(Common::ContainsStr(result.ir_dump, "DsReadB32 v80"),
-        "DS read2 b64 did not lower first dword through shared LDS load IR");
-  Check(Common::ContainsStr(result.ir_dump, "DsReadB32 v83"),
-        "DS read2 b64 did not lower fourth dword through shared LDS load IR");
-  Check(SpirvContainsOpcode(result.spirv, 61),
-        "SPIR-V binary does not contain OpLoad");
   Check(SpirvContainsOpcode(result.spirv, 62),
         "SPIR-V binary does not contain OpStore");
   Check(SpirvContainsOpcode(result.spirv, 65),
@@ -6420,49 +6435,80 @@ void TestNewShaderRecompilerDsWideAndAtomicLowering() {
       0xbf810000u,
   };
 
+  ShaderRecompiler::Decoder::Program decoded;
+  ShaderRecompiler::CFG::Graph graph;
+  ShaderRecompiler::IR::Program lowered;
+  std::string error;
+  Check(ShaderRecompiler::Decoder::DecodeProgram(std::span{shader}, decoded,
+                                                 &error) &&
+            ShaderRecompiler::CFG::BuildGraph(decoded, graph, &error) &&
+            ShaderRecompiler::IR::LowerProgram(
+                decoded, graph, ShaderType::Compute, 64, lowered, &error),
+        error.c_str());
+  uint32_t native_reads = 0;
+  uint32_t native_writes = 0;
+  for (const auto &block : lowered.blocks) {
+    for (const auto &inst : block.instructions) {
+      native_reads += inst.op == ShaderRecompiler::IR::Opcode::DsReadB32;
+      native_writes += inst.op == ShaderRecompiler::IR::Opcode::DsWriteB32;
+    }
+  }
+  Check(native_reads == 3u && native_writes == 3u,
+        "wide DS transfers were split before typed SSA");
+  ShaderRecompiler::IR::ValueProgram typed;
+  ShaderComputeInputInfo compute{};
+  Check(ShaderRecompiler::Frontend::TranslateProgram(lowered, typed, nullptr,
+                                                     nullptr, &compute, &error),
+        error.c_str());
+  std::array<uint32_t, 3> load_widths{};
+  std::array<uint32_t, 3> store_widths{};
+  for (const auto *block : typed.blocks) {
+    for (const auto &inst : *block) {
+      using ShaderRecompiler::IR::ValueOpcode;
+      if (inst.GetOpcode() == ValueOpcode::LoadSharedU32x2)
+        load_widths[0]++;
+      if (inst.GetOpcode() == ValueOpcode::LoadSharedU32x3)
+        load_widths[1]++;
+      if (inst.GetOpcode() == ValueOpcode::LoadSharedU32x4)
+        load_widths[2]++;
+      if (inst.GetOpcode() == ValueOpcode::WriteSharedU32x2)
+        store_widths[0]++;
+      if (inst.GetOpcode() == ValueOpcode::WriteSharedU32x3)
+        store_widths[1]++;
+      if (inst.GetOpcode() == ValueOpcode::WriteSharedU32x4)
+        store_widths[2]++;
+    }
+  }
+  Check(load_widths == std::array<uint32_t, 3>{1u, 1u, 1u} &&
+            store_widths == std::array<uint32_t, 3>{1u, 1u, 1u},
+        "wide DS transfers retained scalar sibling operations");
+
   auto options = MakeCompileOptions(ShaderType::Compute);
   options.dump_ir = true;
 
   ShaderRecompiler::CompileResult result;
-  std::string error;
   Check(ShaderRecompiler::TryRecompile(shader, options, result, &error),
         error.c_str());
-  Check(Common::ContainsStr(result.decoded_dump, "ds_write_b64"),
+  Check(Common::ContainsStr(result.decoded_dump, "DS_WRITE_B64"),
         "new decoder did not decode DS b64 write");
-  Check(Common::ContainsStr(result.decoded_dump, "ds_write_b96"),
+  Check(Common::ContainsStr(result.decoded_dump, "DS_WRITE_B96"),
         "new decoder did not decode DS b96 write");
-  Check(Common::ContainsStr(result.decoded_dump, "ds_write_b128"),
+  Check(Common::ContainsStr(result.decoded_dump, "DS_WRITE_B128"),
         "new decoder did not decode DS b128 write");
-  Check(Common::ContainsStr(result.decoded_dump, "ds_read_b64"),
+  Check(Common::ContainsStr(result.decoded_dump, "DS_READ_B64"),
         "new decoder did not decode DS b64 read");
-  Check(Common::ContainsStr(result.decoded_dump, "ds_read_b96"),
+  Check(Common::ContainsStr(result.decoded_dump, "DS_READ_B96"),
         "new decoder did not decode DS b96 read");
-  Check(Common::ContainsStr(result.decoded_dump, "ds_read_b128"),
+  Check(Common::ContainsStr(result.decoded_dump, "DS_READ_B128"),
         "new decoder did not decode DS b128 read");
-  Check(Common::ContainsStr(result.decoded_dump, "ds_min_u32"),
+  Check(Common::ContainsStr(result.decoded_dump, "DS_MIN_U32"),
         "new decoder did not decode DS min atomic");
-  Check(Common::ContainsStr(result.decoded_dump, "ds_max_u32"),
+  Check(Common::ContainsStr(result.decoded_dump, "DS_MAX_U32"),
         "new decoder did not decode DS max atomic");
-  Check(Common::ContainsStr(result.decoded_dump, "ds_or_b32"),
+  Check(Common::ContainsStr(result.decoded_dump, "DS_OR_B32"),
         "new decoder did not decode DS or atomic");
   Check(Common::ContainsStr(result.decoded_dump, "dwords=4 bits=32"),
         "new decoder did not expose DS wide width metadata");
-  Check(Common::ContainsStr(result.ir_dump, "DsWriteB32 null, v10"),
-        "DS b64 write did not expand to first dword store");
-  Check(Common::ContainsStr(result.ir_dump, "DsWriteB32 null, v19"),
-        "DS b128 write did not expand to last dword store");
-  Check(Common::ContainsStr(result.ir_dump, "DsReadB32 v20"),
-        "DS b64 read did not expand to first dword load");
-  Check(Common::ContainsStr(result.ir_dump, "DsReadB32 v31"),
-        "DS b128 read did not expand to last dword load");
-  Check(Common::ContainsStr(result.ir_dump, "AtomicUMinU32 null, v32"),
-        "DS min atomic did not lower to shared atomic IR");
-  Check(Common::ContainsStr(result.ir_dump, "AtomicUMaxU32 null, v33"),
-        "DS max atomic did not lower to shared atomic IR");
-  Check(Common::ContainsStr(result.ir_dump, "AtomicOrU32 null, v34"),
-        "DS or atomic did not lower to shared atomic IR");
-  Check(SpirvContainsOpcode(result.spirv, 61),
-        "SPIR-V binary does not contain OpLoad");
   Check(SpirvContainsOpcode(result.spirv, 62),
         "SPIR-V binary does not contain OpStore");
   Check(SpirvContainsOpcode(result.spirv, 237),
@@ -6570,6 +6616,7 @@ void TestNewShaderRecompilerDsAddtidLowering() {
   Check(SpirvContainsOpcode(result.spirv, 199),
         "SPIR-V binary does not contain OpBitwiseAnd");
   CheckSpirvBinaryValidates(result.spirv);
+
 }
 
 void TestNewShaderRecompilerDsFloatMinMaxLowering() {
@@ -7838,17 +7885,16 @@ void TestNewShaderRecompilerDispatcherSpillsU32x3() {
 
   auto program = result.program;
   IR::IREmitter definition(program.values->blocks[0]);
-  const auto vector = definition.Emit(IR::ValueOpcode::CompositeConstructU32x3,
+  const auto vector =
-                                      {IR::Value(1u), IR::Value(2u),
+      definition.Emit(IR::ValueOpcode::CompositeConstructU32x3,
-                                       IR::Value(3u)});
+                      {IR::Value(1u), IR::Value(2u), IR::Value(3u)});
   IR::IREmitter use(program.values->blocks[1]);
-  use.Emit(IR::ValueOpcode::CompositeExtractU32x3,
+  use.Emit(IR::ValueOpcode::CompositeExtractU32x3, {vector, IR::Value(2u)});
-           {vector, IR::Value(2u)});
   Check(Spirv::AnalyzeProgramRequirements(program, &error), error.c_str());
 
   std::vector<uint32_t> spirv;
-  Check(Spirv::EmitProgram(program, result.resources, options.input_info,
+  Check(Spirv::EmitProgram(program, result.resources, options.input_info, spirv,
-                           spirv, &error),
+                           &error),
         error.c_str());
   CheckSpirvBinaryValidates(spirv);
   const auto before = MeasureSpirv(result.spirv);
@@ -8533,7 +8579,8 @@ void TestWqmConstantPropagation() {
           "WqmU64 constant propagation crossed a quad or dword boundary");
   }
 
-  const auto check_invalid_signature = [](ValueOpcode opcode, Value valid, Value invalid) {
+  const auto check_invalid_signature = [](ValueOpcode opcode, Value valid,
+                                          Value invalid) {
     ValueProgram malformed;
     malformed.block_storage.push_back(std::make_unique<Block>());
     malformed.blocks.push_back(malformed.block_storage.back().get());
@@ -8547,7 +8594,8 @@ void TestWqmConstantPropagation() {
           "value IR accepted a WQM operand that violates opcode metadata");
   };
   check_invalid_signature(ValueOpcode::WqmU64, Value(uint64_t{1}), Value(true));
-  check_invalid_signature(ValueOpcode::WqmMask, Value(true), Value(uint64_t{1}));
+  check_invalid_signature(ValueOpcode::WqmMask, Value(true),
+                          Value(uint64_t{1}));
 }
 
 void TestNativeWideValueValidation() {
@@ -8568,8 +8616,8 @@ void TestNativeWideValueValidation() {
     IREmitter ir(program.blocks.front());
     const auto resource = ir.Emit(ValueOpcode::GetBufferResource,
                                   {Value(0u), Value(0u), Value(0u), Value(0u)});
-    ir.Emit(opcode,
+    ir.Emit(opcode, {resource, Value(0u), Value(0u), Value(0u), Value(true)},
-            {resource, Value(0u), Value(0u), Value(0u), Value(true)}, flags);
+            flags);
   };
   const auto check_rejected = [](const ValueProgram &program,
                                  const char *expected) {
@@ -8585,8 +8633,7 @@ void TestNativeWideValueValidation() {
     memory.kind = ResourceKind::Buffer;
     memory.data_dwords = 3;
     program.memory_info.push_back(memory);
-    append_load(program, ValueOpcode::LoadBufferU32x3,
+    append_load(program, ValueOpcode::LoadBufferU32x3, MemoryFlags{.index = 1});
-                MemoryFlags{.index = 1});
     check_rejected(program, "invalid memory-info index");
   }
   {
@@ -8624,6 +8671,43 @@ void TestNativeWideValueValidation() {
     ir.Emit(ValueOpcode::CompositeExtractU32x3, {vector, Value(3u)});
     check_rejected(program, "invalid component index");
   }
+  const auto append_shared = [](ValueProgram &program, ValueOpcode opcode,
+                                MemoryFlags flags) {
+    IREmitter ir(program.blocks.front());
+    ir.Emit(opcode, {Value(0u), Value(true)}, flags);
+  };
+  {
+    auto program = make_program();
+    append_shared(program, ValueOpcode::LoadSharedU32, MemoryFlags{.index = 1});
+    check_rejected(program, "invalid memory-info index");
+  }
+  {
+    auto program = make_program();
+    MemoryInfo memory;
+    memory.kind = ResourceKind::Buffer;
+    program.memory_info.push_back(memory);
+    append_shared(program, ValueOpcode::LoadSharedU32, {});
+    check_rejected(program, "invalid shared-memory metadata");
+  }
+  {
+    auto program = make_program();
+    MemoryInfo memory;
+    memory.kind = ResourceKind::Gds;
+    memory.resource = 1;
+    program.memory_info.push_back(memory);
+    append_shared(program, ValueOpcode::LoadSharedU32, {});
+    check_rejected(program, "invalid shared-memory metadata");
+  }
+  {
+    auto program = make_program();
+    MemoryInfo memory;
+    memory.kind = ResourceKind::Lds;
+    memory.data_dwords = 2;
+    memory.component_count = 2;
+    program.memory_info.push_back(memory);
+    append_shared(program, ValueOpcode::LoadSharedU32x3, {});
+    check_rejected(program, "inconsistent shared-memory width");
+  }
 }
 
 void TestNewShaderRecompilerZeroInitialRegisterState() {
@@ -8814,7 +8898,8 @@ void TestNewShaderRecompilerPerInvocationMasksWithoutMirrors() {
         "native scalar WQM retained frontend expansion or subgroup machinery");
   Check(SpirvInstructionOpcodeCount(result.spirv, 132u) == 2u &&
             SpirvInstructionOpcodeCount(result.spirv, 194u) == 4u,
-        "native scalar WQM emitter was folded away or did not use the compact path");
+        "native scalar WQM emitter was folded away or did not use the compact "
+        "path");
 
   const uint32_t native_special_wqm_shader[] = {
       EncodeSop1(0x04, 126, 0),   // s_mov_b64 exec, s[0:1]
@@ -10406,6 +10491,19 @@ void TestNewShaderRecompilerSpirvSizeBaselines() {
             empty_metrics.type_pointers == 0u,
         "empty shader emitted unused core types or pointers");
 
+  const uint32_t dead_gds[] = {
+      EncodeDs0(0x36, 0) | (1u << 17u),
+      EncodeDs1(0, 0, 1),
+      EncodeSopp(0x01),
+  };
+  const auto dead_gds_result =
+      compile("dead-gds", dead_gds,
+              {.words = 64, .instructions = 20, .labels = 4, .branches = 3});
+  Check(ShaderRecompiler::IR::FindBinding(
+            dead_gds_result.program.bindings,
+            ShaderRecompiler::IR::DescriptorBindingKind::Gds) == nullptr,
+        "dead GDS load retained a descriptor binding");
+
   const uint32_t structured_phi[] = {
       EncodeSMovB32(0, 128),       // s0 = 0
       EncodeSopc(0x0a, 0, 129),    // loop: s_cmp_lt_u32 s0, 1
@@ -10494,6 +10592,83 @@ void TestNewShaderRecompilerSpirvSizeBaselines() {
             scalar_stores == 0u,
         "wide buffer fixture retained scalar sibling memory operations");
 
+  const uint32_t wide_lds[] = {
+      EncodeDs0(0xff, 0),  EncodeDs1(0, 0, 4), // ds_read_b128 v[0:3], v4
+      EncodeDs0(0xdf, 16), EncodeDs1(0, 0, 4), // ds_write_b128 v[0:3], v4
+      EncodeSopp(0x01),
+  };
+  const auto wide_lds_result = compile("wide-lds", wide_lds,
+                                       {.words = 548,
+                                        .instructions = 156,
+                                        .variables = 1,
+                                        .workgroup_variables = 1,
+                                        .loads = 4,
+                                        .stores = 4,
+                                        .phis = 5,
+                                        .labels = 34,
+                                        .selection_merges = 10,
+                                        .branches = 23,
+                                        .conditional_branches = 10});
+  const auto wide_lds_metrics = MeasureSpirv(wide_lds_result.spirv);
+  Check(wide_lds_metrics.workgroup_variables == 1u &&
+            wide_lds_metrics.runtime_arrays == 0u &&
+            wide_lds_metrics.array_lengths == 0u,
+        "native wide LDS fixture emitted the wrong storage topology");
+
+  const uint32_t wide_gds[] = {
+      EncodeDs0(0xff, 0) | (1u << 17u),
+      EncodeDs1(0, 0, 4),
+      EncodeDs0(0xdf, 16) | (1u << 17u),
+      EncodeDs1(0, 0, 4),
+      EncodeSopp(0x01),
+  };
+  const auto wide_gds_result = compile("wide-gds", wide_gds,
+                                       {.words = 577,
+                                        .instructions = 162,
+                                        .runtime_arrays = 1,
+                                        .variables = 1,
+                                        .loads = 4,
+                                        .stores = 4,
+                                        .array_lengths = 1,
+                                        .phis = 5,
+                                        .labels = 34,
+                                        .selection_merges = 10,
+                                        .branches = 23,
+                                        .conditional_branches = 10});
+  const auto wide_gds_metrics = MeasureSpirv(wide_gds_result.spirv);
+  Check(wide_gds_metrics.runtime_arrays == 1u &&
+            wide_gds_metrics.workgroup_variables == 0u &&
+            wide_gds_metrics.array_lengths == 1u,
+        "native wide GDS fixture did not share one entry-dominating length");
+  Check(ShaderRecompiler::IR::FindBinding(
+            wide_gds_result.program.bindings,
+            ShaderRecompiler::IR::DescriptorBindingKind::Gds) != nullptr,
+        "live native wide GDS access did not allocate its descriptor binding");
+  const auto count_shared = [](const ShaderRecompiler::CompileResult &result,
+                               ShaderRecompiler::IR::ValueOpcode opcode) {
+    uint32_t count = 0;
+    for (const auto *block : result.program.values->blocks) {
+      count += std::ranges::count(*block, opcode,
+                                  &ShaderRecompiler::IR::Inst::GetOpcode);
+    }
+    return count;
+  };
+  for (const auto *result : {&wide_lds_result, &wide_gds_result}) {
+    Check(count_shared(*result,
+                       ShaderRecompiler::IR::ValueOpcode::LoadSharedU32x4) ==
+                  1u &&
+              count_shared(
+                  *result,
+                  ShaderRecompiler::IR::ValueOpcode::WriteSharedU32x4) == 1u &&
+              count_shared(*result,
+                           ShaderRecompiler::IR::ValueOpcode::LoadSharedU32) ==
+                  0u &&
+              count_shared(*result,
+                           ShaderRecompiler::IR::ValueOpcode::WriteSharedU32) ==
+                  0u,
+          "native wide shared fixture retained scalar sibling operations");
+  }
+
   const uint32_t wqm[] = {
       EncodeVopc(0xc1, 5 + 256, 8), // v_cmp_lt_u32 vcc, v5, v8
       EncodeSop1(0x0a, 2, 106),     // s_wqm_b64 s[2:3], vcc
@@ -10563,6 +10738,8 @@ int main() {
   TestNewShaderRecompilerNativeWideBufferIr();
   TestNewShaderRecompilerMubufFormatLowering();
   TestNewShaderRecompilerTypedBufferLowering();
+  TestNewShaderRecompilerDsReadWrite2Lowering();
+  TestNewShaderRecompilerDsWideAndAtomicLowering();
   TestNewShaderRecompilerCapturedVop1SdwaByteConvert();
   TestNewShaderRecompilerScalarMemoryBindingDomains();
   // Opcode semantics and optimized direct SPIR-V are exercised by