refactor(shader): compact paired 64-bit lowering

src/graphics/shader/recompiler/backend/spirv/spirvEmitterAlu.cpp

@@ -39,23 +39,22 @@ Pair ExtractPair(EmitterState& state, uint32_t value) {
 
 uint32_t MakePair(EmitterState& state, uint32_t low, uint32_t high) {
 	const auto result = state.builder.AllocateId();
-	state.builder.AddFunction({OpCompositeConstruct, TypeU32Pair(state), result, low, high});
+	state.builder.AddFunction({OpCompositeConstruct, TypeU64(state), result, low, high});
 	return result;
 }
 
-uint32_t Compare64(EmitterState& state, uint32_t lhs_value, uint32_t rhs_value,
+uint32_t CompareEqual64(EmitterState& state, uint32_t lhs_value, uint32_t rhs_value,
-                   uint32_t high_compare, uint32_t low_compare, bool equal_compare = false,
+                        bool not_equal) {
-                   bool not_equal = false) {
+	const auto compare = NewBinary(state, not_equal ? OpINotEqual : OpIEqual,
+	                               TypeBoolVector(state, 2), lhs_value, rhs_value);
+	return NewUnary(state, not_equal ? OpAny : OpAll, TypeBool(state), compare);
+}
+
+uint32_t CompareOrdered64(EmitterState& state, uint32_t lhs_value, uint32_t rhs_value,
+                          uint32_t high_compare, uint32_t low_compare) {
 	const auto lhs         = ExtractPair(state, lhs_value);
 	const auto rhs         = ExtractPair(state, rhs_value);
 	const auto high_equal  = NewBinary(state, OpIEqual, TypeBool(state), lhs.high, rhs.high);
-	const auto low_equal  = NewBinary(state, OpIEqual, TypeBool(state), lhs.low, rhs.low);
-	if (equal_compare) {
-		return NewBinary(
-		    state, not_equal ? OpLogicalOr : OpLogicalAnd, TypeBool(state),
-		    not_equal ? NewUnary(state, OpLogicalNot, TypeBool(state), high_equal) : high_equal,
-		    not_equal ? NewUnary(state, OpLogicalNot, TypeBool(state), low_equal) : low_equal);
-	}
 	const auto high_result = NewBinary(state, high_compare, TypeBool(state), lhs.high, rhs.high);
 	const auto low_result  = NewBinary(state, low_compare, TypeBool(state), lhs.low, rhs.low);
 	const auto low_path = NewBinary(state, OpLogicalAnd, TypeBool(state), high_equal, low_result);
@@ -65,10 +64,15 @@ uint32_t Compare64(EmitterState& state, uint32_t lhs_value, uint32_t rhs_value,
 uint32_t EmitAdd64(EmitterState& state, uint32_t lhs_value, uint32_t rhs_value) {
 	const auto lhs      = ExtractPair(state, lhs_value);
 	const auto rhs      = ExtractPair(state, rhs_value);
-	const auto carry = EmitAddCarryValues(state, lhs.low, rhs.low, ConstantU32(state, 0));
+	const auto low_pair = state.builder.AllocateId();
+	const auto low      = state.builder.AllocateId();
+	const auto carry    = state.builder.AllocateId();
+	state.builder.AddFunction({OpIAddCarry, TypeU32Pair(state), low_pair, lhs.low, rhs.low});
+	state.builder.AddFunction({OpCompositeExtract, TypeU32(state), low, low_pair, 0});
+	state.builder.AddFunction({OpCompositeExtract, TypeU32(state), carry, low_pair, 1});
 	const auto high0 = NewBinary(state, OpIAdd, TypeU32(state), lhs.high, rhs.high);
-	const auto high  = NewBinary(state, OpIAdd, TypeU32(state), high0, carry.carry);
+	const auto high  = NewBinary(state, OpIAdd, TypeU32(state), high0, carry);
-	return MakePair(state, carry.sum, high);
+	return MakePair(state, low, high);
 }
 
 uint32_t EmitSub64(EmitterState& state, uint32_t lhs_value, uint32_t rhs_value) {
@@ -112,14 +116,6 @@ uint32_t EmitMul64(EmitterState& state, uint32_t lhs_value, uint32_t rhs_value)
 	                          NewBinary(state, OpIAdd, TypeU32(state), high0, high1), high2));
 }
 
-uint32_t EmitBitwise64(EmitterState& state, uint32_t opcode, uint32_t lhs_value,
-                       uint32_t rhs_value) {
-	const auto lhs = ExtractPair(state, lhs_value);
-	const auto rhs = ExtractPair(state, rhs_value);
-	return MakePair(state, NewBinary(state, opcode, TypeU32(state), lhs.low, rhs.low),
-	                NewBinary(state, opcode, TypeU32(state), lhs.high, rhs.high));
-}
-
 uint32_t EmitShift64(EmitterState& state, uint32_t opcode, uint32_t value, uint32_t shift) {
 	const auto pair = ExtractPair(state, value);
 	uint32_t   low  = 0;
@@ -129,38 +125,80 @@ uint32_t EmitShift64(EmitterState& state, uint32_t opcode, uint32_t value, uint3
 	} else if (opcode == OpShiftRightLogical) {
 		EmitShiftRightLogicalU64Values(state, pair.low, pair.high, shift, low, high);
 	} else {
-		const auto logical      = EmitShift64(state, OpShiftRightLogical, value, shift);
+		const auto amount =
-		const auto logical_pair = ExtractPair(state, logical);
+		    NewBinary(state, OpBitwiseAnd, TypeU32(state), shift, ConstantU32(state, 63));
-		const auto sign =
+		const auto word_shift =
-		    NewBinary(state, OpSLessThan, TypeBool(state), pair.high, ConstantU32(state, 0));
+		    NewBinary(state, OpBitwiseAnd, TypeU32(state), amount, ConstantU32(state, 31));
-		const auto inverse =
+		const auto at_least_32 =
-		    NewBinary(state, OpISub, TypeU32(state), ConstantU32(state, 64), shift);
+		    NewBinary(state, OpUGreaterThanEqual, TypeBool(state), amount, ConstantU32(state, 32));
-		const auto fill = NewBinary(state, OpShiftLeftLogical, TypeU32(state),
+		const auto nonzero =
-		                            ConstantU32(state, 0xffffffffu), inverse);
+		    NewBinary(state, OpINotEqual, TypeBool(state), amount, ConstantU32(state, 0));
-		const auto high_shift =
+		const auto high_shifted =
-		    NewBinary(state, OpISub, TypeU32(state), shift, ConstantU32(state, 32));
+		    NewBinary(state, OpShiftRightArithmetic, TypeU32(state), pair.high, word_shift);
-		const auto high_fill = NewSelect(
+		const auto low_shifted =
-		    state, TypeU32(state),
+		    NewBinary(state, OpShiftRightLogical, TypeU32(state), pair.low, word_shift);
-		    NewBinary(state, OpUGreaterThanEqual, TypeBool(state), shift, ConstantU32(state, 32)),
+		const auto carry_count =
-		    ConstantU32(state, 0xffffffffu), fill);
+		    NewBinary(state, OpBitwiseAnd, TypeU32(state),
-		const auto fill_low =
+		              NewBinary(state, OpISub, TypeU32(state), ConstantU32(state, 32), word_shift),
-		    NewSelect(state, TypeU32(state), sign,
+		              ConstantU32(state, 31));
-		              NewSelect(state, TypeU32(state),
+		const auto carry =
-		                        NewBinary(state, OpUGreaterThanEqual, TypeBool(state), shift,
+		    NewSelect(state, TypeU32(state), nonzero,
-		                                  ConstantU32(state, 32)),
+		              NewBinary(state, OpShiftLeftLogical, TypeU32(state), pair.high, carry_count),
-		                        NewBinary(state, OpShiftLeftLogical, TypeU32(state),
-		                                  ConstantU32(state, 0xffffffffu), high_shift),
-		                        ConstantU32(state, 0)),
 		              ConstantU32(state, 0));
-		const auto fill_high =
+		const auto low_below_32 = NewBinary(state, OpBitwiseOr, TypeU32(state), low_shifted, carry);
-		    NewSelect(state, TypeU32(state), sign, high_fill, ConstantU32(state, 0));
+		const auto negative =
-		return MakePair(
+		    NewBinary(state, OpSLessThan, TypeBool(state), pair.high, ConstantU32(state, 0));
-		    state, NewBinary(state, OpBitwiseOr, TypeU32(state), logical_pair.low, fill_low),
+		const auto sign_fill = NewSelect(state, TypeU32(state), negative,
-		    NewBinary(state, OpBitwiseOr, TypeU32(state), logical_pair.high, fill_high));
+		                                 ConstantU32(state, 0xffffffffu), ConstantU32(state, 0));
+		return MakePair(state,
+		                NewSelect(state, TypeU32(state), at_least_32, high_shifted, low_below_32),
+		                NewSelect(state, TypeU32(state), at_least_32, sign_fill, high_shifted));
 	}
 	return MakePair(state, low, high);
 }
 
+uint32_t EmitConstantShift64(EmitterState& state, uint32_t opcode, uint32_t value, uint32_t shift) {
+	shift &= 63u;
+	if (shift == 0u) {
+		return value;
+	}
+	const auto pair = ExtractPair(state, value);
+	if (opcode == OpShiftLeftLogical) {
+		if (shift < 32u) {
+			const auto low  = NewBinary(state, OpShiftLeftLogical, TypeU32(state), pair.low,
+			                            ConstantU32(state, shift));
+			const auto high = NewBinary(state, OpBitwiseOr, TypeU32(state),
+			                            NewBinary(state, OpShiftLeftLogical, TypeU32(state),
+			                                      pair.high, ConstantU32(state, shift)),
+			                            NewBinary(state, OpShiftRightLogical, TypeU32(state),
+			                                      pair.low, ConstantU32(state, 32u - shift)));
+			return MakePair(state, low, high);
+		}
+		return MakePair(state, ConstantU32(state, 0),
+		                shift == 32u ? pair.low
+		                             : NewBinary(state, OpShiftLeftLogical, TypeU32(state),
+		                                         pair.low, ConstantU32(state, shift - 32u)));
+	}
+	if (shift < 32u) {
+		const auto low = NewBinary(state, OpBitwiseOr, TypeU32(state),
+		                           NewBinary(state, OpShiftRightLogical, TypeU32(state), pair.low,
+		                                     ConstantU32(state, shift)),
+		                           NewBinary(state, OpShiftLeftLogical, TypeU32(state), pair.high,
+		                                     ConstantU32(state, 32u - shift)));
+		const auto high =
+		    NewBinary(state, opcode, TypeU32(state), pair.high, ConstantU32(state, shift));
+		return MakePair(state, low, high);
+	}
+	const auto high = opcode == OpShiftRightArithmetic
+	                      ? NewBinary(state, OpShiftRightArithmetic, TypeU32(state), pair.high,
+	                                  ConstantU32(state, 31u))
+	                      : ConstantU32(state, 0);
+	const auto low  = shift == 32u ? pair.high
+	                               : NewBinary(state, opcode, TypeU32(state), pair.high,
+	                                           ConstantU32(state, shift - 32u));
+	return MakePair(state, low, high);
+}
+
 uint32_t EmitFindMsb64(EmitterState& state, uint32_t value) {
 	const auto pair   = ExtractPair(state, value);
 	const auto high_i = state.builder.AllocateId();
@@ -285,8 +323,6 @@ bool EmitValueAlu(ValueEmitContext& ctx, const IR::Inst& inst) {
 		case IR::ValueOpcode::BitCastF16U16:
 		case IR::ValueOpcode::ConvertU32U16:
 		case IR::ValueOpcode::ConvertU32U8:
-		case IR::ValueOpcode::PackUint2x32:
-		case IR::ValueOpcode::UnpackUint2x32: ctx.Define(inst, ctx.Arg(inst, 0)); return true;
 		case IR::ValueOpcode::BitCastU32F32: return unary(OpBitcast, IR::Type::U32);
 		case IR::ValueOpcode::BitCastF32U32: return unary(OpBitcast, IR::Type::F32);
 		case IR::ValueOpcode::ConvertU16U32:
@@ -319,6 +355,10 @@ bool EmitValueAlu(ValueEmitContext& ctx, const IR::Inst& inst) {
 			return true;
 		}
 		case IR::ValueOpcode::ConvertF32U32: return unary(OpConvertUToF, IR::Type::F32);
+		case IR::ValueOpcode::CompositeConstructU64:
+			ctx.Emit(inst, OpCompositeConstruct, IR::Type::U64,
+			         {ctx.Arg(inst, 0), ctx.Arg(inst, 1)});
+			return true;
 		case IR::ValueOpcode::CompositeConstructU32x2:
 			ctx.Emit(inst, OpCompositeConstruct, IR::Type::U32x2,
 			         {ctx.Arg(inst, 0), ctx.Arg(inst, 1)});
@@ -335,6 +375,7 @@ bool EmitValueAlu(ValueEmitContext& ctx, const IR::Inst& inst) {
 			ctx.Emit(inst, OpCompositeConstruct, IR::Type::U32x4,
 			         {ctx.Arg(inst, 0), ctx.Arg(inst, 1), ctx.Arg(inst, 2), ctx.Arg(inst, 3)});
 			return true;
+		case IR::ValueOpcode::CompositeExtractU64:
 		case IR::ValueOpcode::CompositeExtractU32x2:
 		case IR::ValueOpcode::CompositeExtractU32x3:
 		case IR::ValueOpcode::CompositeExtractU32x4:
@@ -429,15 +470,25 @@ bool EmitValueAlu(ValueEmitContext& ctx, const IR::Inst& inst) {
 		case IR::ValueOpcode::ShiftRightArithmetic32:
 			return binary(OpShiftRightArithmetic, IR::Type::U32);
 		case IR::ValueOpcode::ShiftLeftLogical64:
-			ctx.Define(inst,
+			ctx.Define(inst, inst.Arg(1).Resolve().IsImmediate()
-			           EmitShift64(state, OpShiftLeftLogical, ctx.Arg(inst, 0), ctx.Arg(inst, 1)));
+			                     ? EmitConstantShift64(state, OpShiftLeftLogical, ctx.Arg(inst, 0),
+			                                           inst.Arg(1).Resolve().U32())
+			                     : EmitShift64(state, OpShiftLeftLogical, ctx.Arg(inst, 0),
+			                                   ctx.Arg(inst, 1)));
 			return true;
 		case IR::ValueOpcode::ShiftRightLogical64:
-			ctx.Define(inst,
+			ctx.Define(inst, inst.Arg(1).Resolve().IsImmediate()
-			           EmitShift64(state, OpShiftRightLogical, ctx.Arg(inst, 0), ctx.Arg(inst, 1)));
+			                     ? EmitConstantShift64(state, OpShiftRightLogical, ctx.Arg(inst, 0),
+			                                           inst.Arg(1).Resolve().U32())
+			                     : EmitShift64(state, OpShiftRightLogical, ctx.Arg(inst, 0),
+			                                   ctx.Arg(inst, 1)));
 			return true;
 		case IR::ValueOpcode::ShiftRightArithmetic64:
-			ctx.Define(inst, EmitShift64(state, OpShiftRightArithmetic, ctx.Arg(inst, 0),
+			ctx.Define(inst,
+			           inst.Arg(1).Resolve().IsImmediate()
+			               ? EmitConstantShift64(state, OpShiftRightArithmetic, ctx.Arg(inst, 0),
+			                                     inst.Arg(1).Resolve().U32())
+			               : EmitShift64(state, OpShiftRightArithmetic, ctx.Arg(inst, 0),
 			                             ctx.Arg(inst, 1)));
 			return true;
 		case IR::ValueOpcode::BitwiseAnd32: return binary(OpBitwiseAnd, IR::Type::U32);
@@ -445,19 +496,15 @@ bool EmitValueAlu(ValueEmitContext& ctx, const IR::Inst& inst) {
 		case IR::ValueOpcode::BitwiseXor32: return binary(OpBitwiseXor, IR::Type::U32);
 		case IR::ValueOpcode::BitwiseNot32: return unary(OpNot, IR::Type::U32);
 		case IR::ValueOpcode::BitwiseAnd64:
-			ctx.Define(inst,
+			ctx.Define(inst, NewBinary(state, OpBitwiseAnd, TypeU64(state), ctx.Arg(inst, 0),
-			           EmitBitwise64(state, OpBitwiseAnd, ctx.Arg(inst, 0), ctx.Arg(inst, 1)));
+			                           ctx.Arg(inst, 1)));
-			return true;
-		case IR::ValueOpcode::BitwiseOr64:
-			ctx.Define(inst, EmitBitwise64(state, OpBitwiseOr, ctx.Arg(inst, 0), ctx.Arg(inst, 1)));
 			return true;
 		case IR::ValueOpcode::BitReverse32: return unary(OpBitReverse, IR::Type::U32);
 		case IR::ValueOpcode::BitCount32: return unary(OpBitCount, IR::Type::U32);
 		case IR::ValueOpcode::BitCount64: {
-			const auto pair = ExtractPair(state, ctx.Arg(inst, 0));
+			const auto pair =
-			ctx.Define(inst, NewBinary(state, OpIAdd, TypeU32(state),
+			    ExtractPair(state, NewUnary(state, OpBitCount, TypeU64(state), ctx.Arg(inst, 0)));
-			                           NewUnary(state, OpBitCount, TypeU32(state), pair.low),
+			ctx.Define(inst, NewBinary(state, OpIAdd, TypeU32(state), pair.low, pair.high));
-			                           NewUnary(state, OpBitCount, TypeU32(state), pair.high)));
 			return true;
 		}
 		case IR::ValueOpcode::FindILsb32: {
@@ -527,53 +574,43 @@ bool EmitValueAlu(ValueEmitContext& ctx, const IR::Inst& inst) {
 		case IR::ValueOpcode::SGreaterThanEqual32: return binary(OpSGreaterThanEqual, IR::Type::U1);
 		case IR::ValueOpcode::UGreaterThanEqual32: return binary(OpUGreaterThanEqual, IR::Type::U1);
 		case IR::ValueOpcode::IEqual64:
-			ctx.Define(inst,
+			ctx.Define(inst, CompareEqual64(state, ctx.Arg(inst, 0), ctx.Arg(inst, 1), false));
-			           Compare64(state, ctx.Arg(inst, 0), ctx.Arg(inst, 1), 0, 0, true, false));
 			return true;
 		case IR::ValueOpcode::INotEqual64:
-			ctx.Define(inst,
+			ctx.Define(inst, CompareEqual64(state, ctx.Arg(inst, 0), ctx.Arg(inst, 1), true));
-			           Compare64(state, ctx.Arg(inst, 0), ctx.Arg(inst, 1), 0, 0, true, true));
 			return true;
 		case IR::ValueOpcode::ULessThan64:
-			ctx.Define(inst, Compare64(state, ctx.Arg(inst, 0), ctx.Arg(inst, 1), OpULessThan,
+			ctx.Define(inst, CompareOrdered64(state, ctx.Arg(inst, 0), ctx.Arg(inst, 1),
-			                           OpULessThan));
+			                                  OpULessThan, OpULessThan));
 			return true;
 		case IR::ValueOpcode::SLessThan64:
-			ctx.Define(inst, Compare64(state, ctx.Arg(inst, 0), ctx.Arg(inst, 1), OpSLessThan,
+			ctx.Define(inst, CompareOrdered64(state, ctx.Arg(inst, 0), ctx.Arg(inst, 1),
-			                           OpULessThan));
+			                                  OpSLessThan, OpULessThan));
 			return true;
 		case IR::ValueOpcode::UGreaterThan64:
-			ctx.Define(inst, Compare64(state, ctx.Arg(inst, 0), ctx.Arg(inst, 1), OpUGreaterThan,
+			ctx.Define(inst, CompareOrdered64(state, ctx.Arg(inst, 0), ctx.Arg(inst, 1),
-			                           OpUGreaterThan));
+			                                  OpUGreaterThan, OpUGreaterThan));
 			return true;
 		case IR::ValueOpcode::SGreaterThan64:
-			ctx.Define(inst, Compare64(state, ctx.Arg(inst, 0), ctx.Arg(inst, 1), OpSGreaterThan,
+			ctx.Define(inst, CompareOrdered64(state, ctx.Arg(inst, 0), ctx.Arg(inst, 1),
-			                           OpUGreaterThan));
+			                                  OpSGreaterThan, OpUGreaterThan));
 			return true;
-		case IR::ValueOpcode::ULessThanEqual64: {
+		case IR::ValueOpcode::ULessThanEqual64:
-			const auto greater = Compare64(state, ctx.Arg(inst, 0), ctx.Arg(inst, 1),
+			ctx.Define(inst, CompareOrdered64(state, ctx.Arg(inst, 0), ctx.Arg(inst, 1),
-			                               OpUGreaterThan, OpUGreaterThan);
+			                                  OpULessThan, OpULessThanEqual));
-			ctx.Define(inst, NewUnary(state, OpLogicalNot, TypeBool(state), greater));
 			return true;
-		}
+		case IR::ValueOpcode::SLessThanEqual64:
-		case IR::ValueOpcode::SLessThanEqual64: {
+			ctx.Define(inst, CompareOrdered64(state, ctx.Arg(inst, 0), ctx.Arg(inst, 1),
-			const auto greater = Compare64(state, ctx.Arg(inst, 0), ctx.Arg(inst, 1),
+			                                  OpSLessThan, OpULessThanEqual));
-			                               OpSGreaterThan, OpUGreaterThan);
-			ctx.Define(inst, NewUnary(state, OpLogicalNot, TypeBool(state), greater));
 			return true;
-		}
+		case IR::ValueOpcode::UGreaterThanEqual64:
-		case IR::ValueOpcode::UGreaterThanEqual64: {
+			ctx.Define(inst, CompareOrdered64(state, ctx.Arg(inst, 0), ctx.Arg(inst, 1),
-			const auto less =
+			                                  OpUGreaterThan, OpUGreaterThanEqual));
-			    Compare64(state, ctx.Arg(inst, 0), ctx.Arg(inst, 1), OpULessThan, OpULessThan);
-			ctx.Define(inst, NewUnary(state, OpLogicalNot, TypeBool(state), less));
 			return true;
-		}
+		case IR::ValueOpcode::SGreaterThanEqual64:
-		case IR::ValueOpcode::SGreaterThanEqual64: {
+			ctx.Define(inst, CompareOrdered64(state, ctx.Arg(inst, 0), ctx.Arg(inst, 1),
-			const auto less =
+			                                  OpSGreaterThan, OpUGreaterThanEqual));
-			    Compare64(state, ctx.Arg(inst, 0), ctx.Arg(inst, 1), OpSLessThan, OpULessThan);
-			ctx.Define(inst, NewUnary(state, OpLogicalNot, TypeBool(state), less));
 			return true;
-		}
 		case IR::ValueOpcode::LogicalOr: return binary(OpLogicalOr, IR::Type::U1);
 		case IR::ValueOpcode::LogicalAnd: return binary(OpLogicalAnd, IR::Type::U1);
 		case IR::ValueOpcode::LogicalXor: return binary(OpLogicalNotEqual, IR::Type::U1);

src/graphics/shader/recompiler/backend/spirv/spirvEmitterAluHelpers.cpp

@@ -83,25 +83,6 @@ uint32_t EmitAndConstant(EmitterState& state, uint32_t value, uint32_t mask) {
 	return ret;
 }
 
-AddCarryResult EmitAddCarryValues(EmitterState& state, uint32_t lhs, uint32_t rhs,
-                                  uint32_t carry_in) {
-	const auto pair0  = state.builder.AllocateId();
-	const auto sum0   = state.builder.AllocateId();
-	const auto carry0 = state.builder.AllocateId();
-	const auto pair1  = state.builder.AllocateId();
-	const auto sum1   = state.builder.AllocateId();
-	const auto carry1 = state.builder.AllocateId();
-	const auto carry  = state.builder.AllocateId();
-	state.builder.AddFunction({OpIAddCarry, TypeU32Pair(state), pair0, lhs, rhs});
-	state.builder.AddFunction({OpCompositeExtract, TypeU32(state), sum0, pair0, 0});
-	state.builder.AddFunction({OpCompositeExtract, TypeU32(state), carry0, pair0, 1});
-	state.builder.AddFunction({OpIAddCarry, TypeU32Pair(state), pair1, sum0, carry_in});
-	state.builder.AddFunction({OpCompositeExtract, TypeU32(state), sum1, pair1, 0});
-	state.builder.AddFunction({OpCompositeExtract, TypeU32(state), carry1, pair1, 1});
-	state.builder.AddFunction({OpBitwiseOr, TypeU32(state), carry, carry0, carry1});
-	return {sum1, carry};
-}
-
 uint32_t EmitShiftRightConstant(EmitterState& state, uint32_t value, uint32_t shift) {
 	const auto ret = state.builder.AllocateId();
 	state.builder.AddFunction(

src/graphics/shader/recompiler/backend/spirv/spirvEmitterFlow.cpp

@@ -39,13 +39,22 @@ uint32_t EmitWqmWordU32(EmitterState& state, uint32_t value) {
 }
 
 uint32_t EmitWqmU64(EmitterState& state, uint32_t value) {
-	const auto low  = state.builder.AllocateId();
+	const auto shifted_one = state.builder.AllocateId();
-	const auto high = state.builder.AllocateId();
+	const auto merged_one  = state.builder.AllocateId();
-	state.builder.AddFunction({OpCompositeExtract, TypeU32(state), low, value, 0});
+	const auto shifted_two = state.builder.AllocateId();
-	state.builder.AddFunction({OpCompositeExtract, TypeU32(state), high, value, 1});
+	const auto merged_two  = state.builder.AllocateId();
+	const auto quad_bits   = state.builder.AllocateId();
 	const auto result      = state.builder.AllocateId();
-	state.builder.AddFunction({OpCompositeConstruct, TypeU32Pair(state), result,
+	state.builder.AddFunction({OpShiftRightLogical, TypeU64(state), shifted_one, value,
-	                           EmitWqmWordU32(state, low), EmitWqmWordU32(state, high)});
+	                           ConstantU64(state, 0x0000000100000001ull)});
+	state.builder.AddFunction({OpBitwiseOr, TypeU64(state), merged_one, value, shifted_one});
+	state.builder.AddFunction({OpShiftRightLogical, TypeU64(state), shifted_two, merged_one,
+	                           ConstantU64(state, 0x0000000200000002ull)});
+	state.builder.AddFunction({OpBitwiseOr, TypeU64(state), merged_two, merged_one, shifted_two});
+	state.builder.AddFunction({OpBitwiseAnd, TypeU64(state), quad_bits, merged_two,
+	                           ConstantU64(state, 0x1111111111111111ull)});
+	state.builder.AddFunction(
+	    {OpIMul, TypeU64(state), result, quad_bits, ConstantU64(state, 0x0000000f0000000full)});
 	return result;
 }
 

src/graphics/shader/recompiler/backend/spirv/spirvEmitterInternal.h

@@ -182,6 +182,8 @@ enum : uint32_t {
 	OpIAddCarry                    = 149,
 	OpUMulExtended                 = 151,
 	OpSMulExtended                 = 152,
+	OpAny                          = 154,
+	OpAll                          = 155,
 	OpLogicalNotEqual              = 165,
 	OpLogicalOr                    = 166,
 	OpLogicalAnd                   = 167,
@@ -337,7 +339,9 @@ struct EmitterState {
 
 uint32_t TypeVoid(EmitterState& state);
 uint32_t TypeBool(EmitterState& state);
+uint32_t TypeBoolVector(EmitterState& state, uint32_t components);
 uint32_t TypeU32(EmitterState& state);
+uint32_t TypeU64(EmitterState& state);
 uint32_t TypeU32Pair(EmitterState& state);
 uint32_t TypeI32(EmitterState& state);
 uint32_t TypeI32Pair(EmitterState& state);
@@ -505,11 +509,6 @@ constexpr uint32_t ImageSpirvMultisampled(ImageViewKind view) {
 	return view == ImageViewKind::Dim2DMsaa || view == ImageViewKind::Dim2DMsaaArray ? 1u : 0u;
 }
 
-struct AddCarryResult {
-	uint32_t sum   = 0;
-	uint32_t carry = 0;
-};
-
 struct F32Class {
 	uint32_t bits       = 0;
 	uint32_t nan        = 0;
@@ -730,8 +729,6 @@ uint32_t EmitDsSwizzleTargetLane(EmitterState& state, uint32_t subid, uint32_t c
 uint32_t EmitSelectValueU32(EmitterState& state, uint32_t cond, uint32_t true_value,
                             uint32_t false_value);
 
-uint32_t EmitAndConstant(EmitterState& state, uint32_t value, uint32_t mask);
-
 void EmitShiftLeftLogicalU64Values(EmitterState& state, uint32_t low, uint32_t high, uint32_t shift,
                                    uint32_t& out_low, uint32_t& out_high);
 
@@ -740,9 +737,6 @@ void EmitShiftRightLogicalU64Values(EmitterState& state, uint32_t low, uint32_t
 
 uint32_t EmitAndConstant(EmitterState& state, uint32_t value, uint32_t mask);
 
-AddCarryResult EmitAddCarryValues(EmitterState& state, uint32_t lhs, uint32_t rhs,
-                                  uint32_t carry_in);
-
 uint32_t EmitShiftRightConstant(EmitterState& state, uint32_t value, uint32_t shift);
 
 uint32_t EmitOrU32(EmitterState& state, uint32_t lhs, uint32_t rhs);

src/graphics/shader/recompiler/backend/spirv/spirvEmitterModule.cpp

@@ -16,10 +16,18 @@ uint32_t TypeBool(EmitterState& state) {
 	return state.builder.Type(OpTypeBool);
 }
 
+uint32_t TypeBoolVector(EmitterState& state, uint32_t components) {
+	return state.builder.Type(OpTypeVector, {TypeBool(state), components});
+}
+
 uint32_t TypeU32(EmitterState& state) {
 	return state.builder.Type(OpTypeInt, {32, 0});
 }
 
+uint32_t TypeU64(EmitterState& state) {
+	return TypeU32Vector(state, 2);
+}
+
 uint32_t TypeU32Pair(EmitterState& state) {
 	const auto element = TypeU32(state);
 	return state.builder.Type(OpTypeStruct, {element, element});
@@ -241,7 +249,7 @@ uint32_t ConstantBool(EmitterState& state, bool value) {
 }
 
 uint32_t ConstantU64(EmitterState& state, uint64_t value) {
-	return state.builder.Constant(OpConstantComposite, TypeU32Pair(state),
+	return state.builder.Constant(OpConstantComposite, TypeU64(state),
 	                              {ConstantU32(state, static_cast<uint32_t>(value)),
 	                               ConstantU32(state, static_cast<uint32_t>(value >> 32u))});
 }

src/graphics/shader/recompiler/backend/spirv/spirvEmitterProgram.cpp

@@ -44,7 +44,7 @@ uint32_t SpillPointerType(EmitterState& state, IR::Type type) {
 		case IR::Type::U32:
 		case IR::Type::F16: return TypePointer(state, StorageClassFunction, TypeU32(state));
 		case IR::Type::F32: return TypePointer(state, StorageClassFunction, TypeF32(state));
-		case IR::Type::U64:
+		case IR::Type::U64: return TypePointer(state, StorageClassFunction, TypeU64(state));
 		case IR::Type::U32x2: return TypePointer(state, StorageClassFunction, TypeU32Pair(state));
 		case IR::Type::U32x3:
 			return TypePointer(state, StorageClassFunction, TypeU32Vector(state, 3));
@@ -404,7 +404,7 @@ uint32_t ValueEmitContext::TypeId(IR::Type type) const {
 		case IR::Type::U16:
 		case IR::Type::U32:
 		case IR::Type::F16: return TypeU32(state);
-		case IR::Type::U64:
+		case IR::Type::U64: return TypeU64(state);
 		case IR::Type::U32x2: return TypeU32Pair(state);
 		case IR::Type::F32: return TypeF32(state);
 		case IR::Type::U32x3: return TypeU32Vector(state, 3);

src/graphics/shader/recompiler/frontend/translate/Control.cpp

@@ -27,7 +27,7 @@ bool Translator::TranslateStateOperation(const IR::Instruction& inst) {
 			const auto result_high =
 			    inst.op == IR::Opcode::SaveexecB64 ? apply(old_high, src_high) : old_high;
 			if (inst.op == IR::Opcode::SaveexecB64) {
-				WriteOperand(inst.dst, ir.PackUint2x32(old_low, old_high));
+				WriteU32Pair(inst.dst, {old_low, old_high});
 			} else {
 				WriteOperand(inst.dst, old_low);
 			}
@@ -157,12 +157,11 @@ bool Translator::TranslateMove(const IR::Instruction& inst) {
 			WriteOperand(inst.dst, ReadOperand(inst.src[0], IR::Type::F32));
 			return true;
 		case IR::Opcode::MoveU64: {
-			const bool scalar_pair = inst.dst.kind == IR::OperandKind::Register &&
+			WriteU32Pair(inst.dst, ReadU32Pair(inst.src[0]));
+			if (inst.dst.kind == IR::OperandKind::Register &&
 			    inst.src[0].kind == IR::OperandKind::Register &&
 			    inst.dst.reg.file == IR::RegisterFile::Scalar &&
-			                         inst.src[0].reg.file == IR::RegisterFile::Scalar;
+			    inst.src[0].reg.file == IR::RegisterFile::Scalar) {
-			WriteOperand(inst.dst, ReadOperand(inst.src[0], IR::Type::U64));
-			if (scalar_pair) {
 				ir.SetThreadBitScalarReg(
 				    static_cast<IR::ScalarReg>(inst.dst.reg.index),
 				    ir.GetThreadBitScalarReg(static_cast<IR::ScalarReg>(inst.src[0].reg.index)));
@@ -171,8 +170,8 @@ bool Translator::TranslateMove(const IR::Instruction& inst) {
 		}
 		case IR::Opcode::WqmB64: {
 			if (!current_per_invocation_masks) {
-				const auto result = IR::U64(ir.Emit(
+				const auto result = IR::U64(
-				    IR::ValueOpcode::WqmU64, {ReadOperand(inst.src[0], IR::Type::U64)}));
+				    ir.Emit(IR::ValueOpcode::WqmU64, {ReadOperand(inst.src[0], IR::Type::U64)}));
 				WriteOperand(inst.dst, result);
 				ir.SetScc(IR::U1(
 				    ir.Emit(IR::ValueOpcode::INotEqual64, {result, IR::Value(uint64_t {0})})));

src/graphics/shader/recompiler/frontend/translate/Dispatch.cpp

@@ -6,7 +6,8 @@ bool Translator::TranslateU64MaskOperation(const IR::Instruction& inst) {
 	if (current_per_invocation_masks) {
 		return TranslatePerInvocationU64Mask(inst);
 	}
-	return inst.op == IR::Opcode::BitwiseAndU64 || inst.op == IR::Opcode::BitwiseOrU64
+	return inst.op == IR::Opcode::BitwiseAndU64 || inst.op == IR::Opcode::BitwiseOrU64 ||
+	               inst.op == IR::Opcode::BitwiseXorU64 || inst.op == IR::Opcode::BitwiseNotU64
 	           ? TranslateSimpleInteger(inst)
 	           : TranslateComposedInteger(inst);
 }

src/graphics/shader/recompiler/frontend/translate/Integer.cpp

@@ -157,6 +157,31 @@ bool Translator::TranslatePerInvocationU64Mask(const IR::Instruction& inst) {
 }
 
 bool Translator::TranslateSimpleInteger(const IR::Instruction& inst) {
+	if (inst.op == IR::Opcode::BitwiseAndU64 || inst.op == IR::Opcode::BitwiseOrU64 ||
+	    inst.op == IR::Opcode::BitwiseXorU64 || inst.op == IR::Opcode::BitwiseNotU64) {
+		const auto             lhs = ReadU32Pair(inst.src[0]);
+		std::array<IR::U32, 2> result;
+		if (inst.op == IR::Opcode::BitwiseNotU64) {
+			result = {ir.BitwiseNot(lhs[0]), ir.BitwiseNot(lhs[1])};
+		} else {
+			const auto rhs = ReadU32Pair(inst.src[1]);
+			for (uint32_t component = 0; component < 2; component++) {
+				switch (inst.op) {
+					case IR::Opcode::BitwiseAndU64:
+						result[component] = ir.BitwiseAnd(lhs[component], rhs[component]);
+						break;
+					case IR::Opcode::BitwiseOrU64:
+						result[component] = ir.BitwiseOr(lhs[component], rhs[component]);
+						break;
+					default:
+						result[component] = ir.BitwiseXor(lhs[component], rhs[component]);
+						break;
+				}
+			}
+		}
+		WriteU32Pair(inst.dst, result);
+		return true;
+	}
 	IR::ValueOpcode opcode {};
 	IR::Type        type = IR::Type::U32;
 	switch (inst.op) {
@@ -187,14 +212,6 @@ bool Translator::TranslateSimpleInteger(const IR::Instruction& inst) {
 			type   = IR::Type::U64;
 			break;
 		case IR::Opcode::FindLsbU32: opcode = IR::ValueOpcode::FindILsb32; break;
-		case IR::Opcode::BitwiseAndU64:
-			opcode = IR::ValueOpcode::BitwiseAnd64;
-			type   = IR::Type::U64;
-			break;
-		case IR::Opcode::BitwiseOrU64:
-			opcode = IR::ValueOpcode::BitwiseOr64;
-			type   = IR::Type::U64;
-			break;
 		case IR::Opcode::ShiftLeftLogicalU32: opcode = IR::ValueOpcode::ShiftLeftLogical32; break;
 		case IR::Opcode::ShiftRightLogicalU32: opcode = IR::ValueOpcode::ShiftRightLogical32; break;
 		case IR::Opcode::ShiftRightArithmeticI32:
@@ -219,10 +236,6 @@ bool Translator::TranslateSimpleInteger(const IR::Instruction& inst) {
 		                    opcode == IR::ValueOpcode::ShiftRightLogical32 ||
 		                    opcode == IR::ValueOpcode::ShiftRightArithmetic32)) {
 			args[index] = ir.BitwiseAnd(IR::U32(args[index]), IR::U32(IR::Value(31u)));
-		} else if (index == 1u && (opcode == IR::ValueOpcode::ShiftLeftLogical64 ||
-		                           opcode == IR::ValueOpcode::ShiftRightLogical64 ||
-		                           opcode == IR::ValueOpcode::ShiftRightArithmetic64)) {
-			args[index] = ir.BitwiseAnd(IR::U32(args[index]), IR::U32(IR::Value(63u)));
 		}
 	}
 	IR::Value result;
@@ -233,32 +246,46 @@ bool Translator::TranslateSimpleInteger(const IR::Instruction& inst) {
 		default: EXIT("invalid simple integer source count: %u", inst.src_count);
 	}
 	WriteOperand(inst.dst, result);
-	if (type == IR::Type::U64 && inst.dst.kind == IR::OperandKind::Register &&
-	    inst.dst.reg.file == IR::RegisterFile::Scalar) {
-		IR::U1 mask;
-		if (inst.op == IR::Opcode::BitwiseAndU64) {
-			mask = ir.LogicalAnd(ReadMask(inst.src[0]), ReadMask(inst.src[1]));
-		} else if (inst.op == IR::Opcode::BitwiseOrU64) {
-			mask = ir.LogicalOr(ReadMask(inst.src[0]), ReadMask(inst.src[1]));
-		}
-		if (!mask.IsEmpty()) {
-			ir.SetThreadBitScalarReg(static_cast<IR::ScalarReg>(inst.dst.reg.index), mask);
-		}
-	}
 	return true;
 }
 
 bool Translator::TranslateComposedInteger(const IR::Instruction& inst) {
+	if (inst.op == IR::Opcode::BitwiseAndNotU64 || inst.op == IR::Opcode::BitwiseOrNotU64 ||
+	    inst.op == IR::Opcode::BitwiseNandU64 || inst.op == IR::Opcode::BitwiseNorU64 ||
+	    inst.op == IR::Opcode::BitwiseXnorU64) {
+		const auto             lhs = ReadU32Pair(inst.src[0]);
+		const auto             rhs = ReadU32Pair(inst.src[1]);
+		std::array<IR::U32, 2> result;
+		for (uint32_t component = 0; component < 2; component++) {
+			switch (inst.op) {
+				case IR::Opcode::BitwiseAndNotU64:
+					result[component] =
+					    ir.BitwiseAnd(lhs[component], ir.BitwiseNot(rhs[component]));
+					break;
+				case IR::Opcode::BitwiseOrNotU64:
+					result[component] = ir.BitwiseOr(lhs[component], ir.BitwiseNot(rhs[component]));
+					break;
+				case IR::Opcode::BitwiseNandU64:
+					result[component] =
+					    ir.BitwiseNot(ir.BitwiseAnd(lhs[component], rhs[component]));
+					break;
+				case IR::Opcode::BitwiseNorU64:
+					result[component] = ir.BitwiseNot(ir.BitwiseOr(lhs[component], rhs[component]));
+					break;
+				default:
+					result[component] =
+					    ir.BitwiseNot(ir.BitwiseXor(lhs[component], rhs[component]));
+					break;
+			}
+		}
+		WriteU32Pair(inst.dst, result);
+		return true;
+	}
 	const auto binary_u32 = [&](auto operation) {
 		const auto lhs = ReadU32(inst.src[0]);
 		const auto rhs = ReadU32(inst.src[1]);
 		return operation(lhs, rhs);
 	};
-	const auto binary_u64 = [&](auto operation) {
-		const auto lhs = UnpackU64(ReadU64(inst.src[0]));
-		const auto rhs = UnpackU64(ReadU64(inst.src[1]));
-		return ir.PackUint2x32(operation(lhs[0], rhs[0]), operation(lhs[1], rhs[1]));
-	};
 
 	IR::Value result;
 	switch (inst.op) {
@@ -282,34 +309,6 @@ bool Translator::TranslateComposedInteger(const IR::Instruction& inst) {
 			result = binary_u32(
 			    [&](IR::U32 lhs, IR::U32 rhs) { return ir.BitwiseNot(ir.BitwiseXor(lhs, rhs)); });
 			break;
-		case IR::Opcode::BitwiseXorU64:
-			result = binary_u64([&](IR::U32 lhs, IR::U32 rhs) { return ir.BitwiseXor(lhs, rhs); });
-			break;
-		case IR::Opcode::BitwiseAndNotU64:
-			result = binary_u64(
-			    [&](IR::U32 lhs, IR::U32 rhs) { return ir.BitwiseAnd(lhs, ir.BitwiseNot(rhs)); });
-			break;
-		case IR::Opcode::BitwiseOrNotU64:
-			result = binary_u64(
-			    [&](IR::U32 lhs, IR::U32 rhs) { return ir.BitwiseOr(lhs, ir.BitwiseNot(rhs)); });
-			break;
-		case IR::Opcode::BitwiseNandU64:
-			result = binary_u64(
-			    [&](IR::U32 lhs, IR::U32 rhs) { return ir.BitwiseNot(ir.BitwiseAnd(lhs, rhs)); });
-			break;
-		case IR::Opcode::BitwiseNorU64:
-			result = binary_u64(
-			    [&](IR::U32 lhs, IR::U32 rhs) { return ir.BitwiseNot(ir.BitwiseOr(lhs, rhs)); });
-			break;
-		case IR::Opcode::BitwiseXnorU64:
-			result = binary_u64(
-			    [&](IR::U32 lhs, IR::U32 rhs) { return ir.BitwiseNot(ir.BitwiseXor(lhs, rhs)); });
-			break;
-		case IR::Opcode::BitwiseNotU64: {
-			const auto source = UnpackU64(ReadU64(inst.src[0]));
-			result            = ir.PackUint2x32(ir.BitwiseNot(source[0]), ir.BitwiseNot(source[1]));
-			break;
-		}
 		case IR::Opcode::BitwiseAndOrU32: {
 			const auto lhs = ReadU32(inst.src[0]);
 			const auto rhs = ReadU32(inst.src[1]);
@@ -326,7 +325,7 @@ bool Translator::TranslateComposedInteger(const IR::Instruction& inst) {
 			                       ReadU32(inst.src[2]));
 			break;
 		case IR::Opcode::FindLsbU64: {
-			const auto source        = UnpackU64(ReadU64(inst.src[0]));
+			const auto source        = ExtractU64(ReadU64(inst.src[0]));
 			const auto low_lsb       = IR::U32(ir.Emit(IR::ValueOpcode::FindILsb32, {source[0]}));
 			const auto high_lsb      = IR::U32(ir.Emit(IR::ValueOpcode::FindILsb32, {source[1]}));
 			const auto high_position = ir.IAdd(high_lsb, IR::U32(IR::Value(32u)));
@@ -349,44 +348,13 @@ bool Translator::TranslateComposedInteger(const IR::Instruction& inst) {
 			const auto position = ir.ISub(IR::U32(IR::Value(63u)), msb);
 			const auto nonzero  = IR::U1(
 			    ir.Emit(IR::ValueOpcode::INotEqual64,
-			            {source, ir.PackUint2x32(IR::U32(IR::Value(0u)), IR::U32(IR::Value(0u)))}));
+			            {source, ir.ConstructU64(IR::U32(IR::Value(0u)), IR::U32(IR::Value(0u)))}));
 			result = ir.Select(nonzero, position, IR::U32(IR::Value(0xffffffffu)));
 			break;
 		}
 		default: return false;
 	}
 	WriteOperand(inst.dst, result);
-	if (result.GetType() == IR::Type::U64 && inst.dst.kind == IR::OperandKind::Register &&
-	    inst.dst.reg.file == IR::RegisterFile::Scalar) {
-		const auto lhs = ReadMask(inst.src[0]);
-		IR::U1     mask;
-		switch (inst.op) {
-			case IR::Opcode::BitwiseXorU64:
-				mask = IR::U1(ir.Emit(IR::ValueOpcode::LogicalXor, {lhs, ReadMask(inst.src[1])}));
-				break;
-			case IR::Opcode::BitwiseAndNotU64:
-				mask = ir.LogicalAnd(lhs, ir.LogicalNot(ReadMask(inst.src[1])));
-				break;
-			case IR::Opcode::BitwiseOrNotU64:
-				mask = ir.LogicalOr(lhs, ir.LogicalNot(ReadMask(inst.src[1])));
-				break;
-			case IR::Opcode::BitwiseNandU64:
-				mask = ir.LogicalNot(ir.LogicalAnd(lhs, ReadMask(inst.src[1])));
-				break;
-			case IR::Opcode::BitwiseNorU64:
-				mask = ir.LogicalNot(ir.LogicalOr(lhs, ReadMask(inst.src[1])));
-				break;
-			case IR::Opcode::BitwiseXnorU64:
-				mask = ir.LogicalNot(
-				    IR::U1(ir.Emit(IR::ValueOpcode::LogicalXor, {lhs, ReadMask(inst.src[1])})));
-				break;
-			case IR::Opcode::BitwiseNotU64: mask = ir.LogicalNot(lhs); break;
-			default: break;
-		}
-		if (!mask.IsEmpty()) {
-			ir.SetThreadBitScalarReg(static_cast<IR::ScalarReg>(inst.dst.reg.index), mask);
-		}
-	}
 	return true;
 }
 
@@ -402,18 +370,12 @@ bool Translator::TranslateExtendedInteger(const IR::Instruction& inst) {
 		return IR::U32(
 		    ir.Emit(IR::ValueOpcode::BitFieldInsert, {imm(0), imm(0xffffffffu), imm(0), count}));
 	};
-	const auto select_u64 = [&](IR::U1 condition, IR::U64 true_value, IR::U64 false_value) {
-		const auto lhs = UnpackU64(true_value);
-		const auto rhs = UnpackU64(false_value);
-		return ir.PackUint2x32(ir.Select(condition, lhs[0], rhs[0]),
-		                       ir.Select(condition, lhs[1], rhs[1]));
-	};
 	const auto right_mask64 = [&](IR::U32 count) {
 		const auto below32    = IR::U1(ir.Emit(IR::ValueOpcode::ULessThan32, {count, imm(32)}));
 		const auto above32    = IR::U1(ir.Emit(IR::ValueOpcode::UGreaterThan32, {count, imm(32)}));
 		const auto low_count  = ir.Select(below32, count, imm(32));
 		const auto high_count = ir.Select(above32, ir.ISub(count, imm(32)), imm(0));
-		return ir.PackUint2x32(right_mask32(low_count), right_mask32(high_count));
+		return ir.ConstructU64(right_mask32(low_count), right_mask32(high_count));
 	};
 
 	IR::Value result;
@@ -436,7 +398,7 @@ bool Translator::TranslateExtendedInteger(const IR::Instruction& inst) {
 		case IR::Opcode::UMadU64U32: {
 			const auto lhs       = ReadU32(inst.src[0]);
 			const auto rhs       = ReadU32(inst.src[1]);
-			const auto add       = UnpackU64(ReadU64(inst.src[2]));
+			const auto add       = ExtractU64(ReadU64(inst.src[2]));
 			const auto mul_low   = ir.IMul(lhs, rhs);
 			const auto mul_high  = IR::U32(ir.Emit(IR::ValueOpcode::UMulHi, {lhs, rhs}));
 			const auto low       = ir.IAdd(mul_low, add[0]);
@@ -445,7 +407,7 @@ bool Translator::TranslateExtendedInteger(const IR::Instruction& inst) {
 			const auto carry0    = ir.ULessThan(high0, mul_high);
 			const auto high      = ir.IAdd(high0, ir.Select(carry_low, imm(1), imm(0)));
 			const auto carry1    = ir.ULessThan(high, high0);
-			WriteOperand(inst.dst, ir.PackUint2x32(low, high));
+			WriteOperand(inst.dst, ir.ConstructU64(low, high));
 			if (inst.dst2.kind != IR::OperandKind::Null) {
 				WriteMask(inst.dst2, ir.LogicalOr(carry0, carry1));
 			}
@@ -503,7 +465,7 @@ bool Translator::TranslateExtendedInteger(const IR::Instruction& inst) {
 				return ir.BitwiseOr(bits, ir.ShiftLeftLogical(bits, imm(1)));
 			};
 			const auto source = ReadU32(inst.src[0]);
-			result            = ir.PackUint2x32(replicate(mask(source, 0xffffu)),
+			result            = ir.ConstructU64(replicate(mask(source, 0xffffu)),
 			                                    replicate(ir.ShiftRightLogical(source, imm(16))));
 			break;
 		}
@@ -590,12 +552,12 @@ bool Translator::TranslateExtendedInteger(const IR::Instruction& inst) {
 			const auto lo           = ReadU32(inst.src[1]);
 			const auto byte_offset  = mask(ReadU32(inst.src[2]), 31u);
 			const auto bit_offset   = ir.ShiftLeftLogical(byte_offset, imm(3));
-			const auto concatenated = ir.PackUint2x32(lo, hi);
+			const auto concatenated = ir.ConstructU64(lo, hi);
 			const auto shifted      = IR::U64(ir.Emit(IR::ValueOpcode::ShiftRightLogical64,
 			                                          {concatenated, mask(bit_offset, 63u)}));
 			const auto in_range =
 			    IR::U1(ir.Emit(IR::ValueOpcode::ULessThan32, {byte_offset, imm(8)}));
-			result = ir.Select(in_range, UnpackU64(shifted)[0], imm(0));
+			result = ir.Select(in_range, ExtractU64(shifted)[0], imm(0));
 			break;
 		}
 		case IR::Opcode::ShiftLeftAddU32:
@@ -633,10 +595,14 @@ bool Translator::TranslateExtendedInteger(const IR::Instruction& inst) {
 			                                ReadOperand(inst.src[2], IR::Type::F32)});
 			break;
 		}
-		case IR::Opcode::SelectU64:
+		case IR::Opcode::SelectU64: {
-			result =
+			const auto condition = ReadCondition(inst.src[0]);
-			    select_u64(ReadCondition(inst.src[0]), ReadU64(inst.src[1]), ReadU64(inst.src[2]));
+			const auto lhs       = ReadU32Pair(inst.src[1]);
-			break;
+			const auto rhs       = ReadU32Pair(inst.src[2]);
+			WriteU32Pair(inst.dst, {ir.Select(condition, lhs[0], rhs[0]),
+			                        ir.Select(condition, lhs[1], rhs[1])});
+			return true;
+		}
 		case IR::Opcode::PackLowLowU16:
 		case IR::Opcode::PackLowHighU16:
 		case IR::Opcode::PackHighHighU16:

src/graphics/shader/recompiler/frontend/translate/Translate.cpp

@@ -87,10 +87,7 @@ IR::U32 Translator::ReadPcRelativeU32(const IR::Operand& operand) {
 	const auto address =
 	    ir.Emit(IR::ValueOpcode::IAdd64, {ir.Emit(IR::ValueOpcode::GetShaderBase),
 	                                      IR::Value(static_cast<uint64_t>(operand.imm))});
-	const auto pair = ir.Emit(IR::ValueOpcode::UnpackUint2x32, {address});
+	return ir.CompositeExtract(address, operand.kind == IR::OperandKind::PcRelativeU32 ? 0u : 1u);
-	return IR::U32(
-	    ir.Emit(IR::ValueOpcode::CompositeExtractU32x2,
-	            {pair, IR::Value(operand.kind == IR::OperandKind::PcRelativeU32 ? 0u : 1u)}));
 }
 
 std::array<IR::U32, 2> Translator::BallotMask(IR::U1 value) {
@@ -205,22 +202,8 @@ IR::Value Translator::ReadOperand(const IR::Operand& operand, IR::Type type) {
 		return ir.INotEqual(ReadRawU32(operand), IR::U32(IR::Value(0u)));
 	}
 	if (type == IR::Type::U64) {
-		if (operand.kind == IR::OperandKind::Register &&
+		const auto pair = ReadU32Pair(operand);
-		    operand.reg.file == IR::RegisterFile::Exec) {
+		return ir.ConstructU64(pair[0], pair[1]);
-			return ir.PackUint2x32(ir.GetExecLo(), ir.GetExecHi());
-		}
-		if (operand.kind == IR::OperandKind::Register &&
-		    operand.reg.file == IR::RegisterFile::Vcc) {
-			return ir.PackUint2x32(ir.GetVccLo(), ir.GetVccHi());
-		}
-		const auto low = ApplyBitSourceModifiers(operand, ReadRawU32(operand));
-		IR::U32    high(IR::Value(0u));
-		if (operand.kind == IR::OperandKind::Register) {
-			high = ReadRawU32(OffsetOperand(operand, 1));
-		} else if (operand.kind == IR::OperandKind::ImmediateU32 && operand.sext_64) {
-			high = IR::U32(IR::Value(0xffffffffu));
-		}
-		return ir.PackUint2x32(low, high);
 	}
 	auto bits = ApplyBitSourceModifiers(operand, ReadRawU32(operand));
 	if (TypesOverlap(type, IR::Type::F32) && !TypesOverlap(type, IR::Type::U32)) {
@@ -396,31 +379,7 @@ void Translator::WriteOperand(const IR::Operand& operand, IR::Value value) {
 		return;
 	}
 	if (type == IR::Type::U64) {
-		const auto unpacked = ir.Emit(IR::ValueOpcode::UnpackUint2x32, {value});
+		WriteU32Pair(operand, {ir.CompositeExtract(value, 0), ir.CompositeExtract(value, 1)});
-		const auto low      = ir.CompositeExtract(unpacked, 0);
-		const auto high     = ir.CompositeExtract(unpacked, 1);
-		if (operand.kind == IR::OperandKind::Register) {
-			const auto thread_bit = ThreadBit(low, high);
-			switch (operand.reg.file) {
-				case IR::RegisterFile::Exec:
-					ir.SetExec(thread_bit);
-					ir.SetExecLo(low);
-					ir.SetExecHi(high);
-					return;
-				case IR::RegisterFile::Vcc:
-					ir.SetVcc(thread_bit);
-					ir.SetVccLo(low);
-					ir.SetVccHi(high);
-					return;
-				case IR::RegisterFile::Scalar:
-					ir.SetThreadBitScalarReg(static_cast<IR::ScalarReg>(operand.reg.index),
-					                         thread_bit);
-					break;
-				default: break;
-			}
-		}
-		WriteRawU32(operand, low);
-		WriteRawU32(OffsetOperand(operand, 1), high);
 		return;
 	}
 	if (type == IR::Type::F32) {
@@ -474,6 +433,25 @@ IR::U32 Translator::ReadU32(const IR::Operand& operand) {
 	return IR::U32(ReadOperand(operand, IR::Type::U32));
 }
 
+std::array<IR::U32, 2> Translator::ReadU32Pair(const IR::Operand& operand) {
+	if (operand.kind == IR::OperandKind::Register) {
+		if (operand.reg.file == IR::RegisterFile::Exec) {
+			return {ir.GetExecLo(), ir.GetExecHi()};
+		}
+		if (operand.reg.file == IR::RegisterFile::Vcc) {
+			return {ir.GetVccLo(), ir.GetVccHi()};
+		}
+	}
+	const auto low = ApplyBitSourceModifiers(operand, ReadRawU32(operand));
+	IR::U32    high(IR::Value(0u));
+	if (operand.kind == IR::OperandKind::Register) {
+		high = ReadRawU32(OffsetOperand(operand, 1));
+	} else if (operand.kind == IR::OperandKind::ImmediateU32 && operand.sext_64) {
+		high = IR::U32(IR::Value(0xffffffffu));
+	}
+	return {low, high};
+}
+
 IR::U64 Translator::ReadU64(const IR::Operand& operand) {
 	return IR::U64(ReadOperand(operand, IR::Type::U64));
 }
@@ -560,9 +538,35 @@ IR::U32 Translator::ReadF16LaneBits(const IR::Operand& operand, bool high_lane)
 	return value;
 }
 
-std::array<IR::U32, 2> Translator::UnpackU64(IR::U64 value) {
+std::array<IR::U32, 2> Translator::ExtractU64(IR::U64 value) {
-	const auto unpacked = ir.Emit(IR::ValueOpcode::UnpackUint2x32, {value});
+	return {ir.CompositeExtract(value, 0), ir.CompositeExtract(value, 1)};
-	return {ir.CompositeExtract(unpacked, 0), ir.CompositeExtract(unpacked, 1)};
+}
+
+void Translator::WriteU32Pair(const IR::Operand& operand, const std::array<IR::U32, 2>& value) {
+	if (operand.kind == IR::OperandKind::Null) {
+		return;
+	}
+	if (operand.kind == IR::OperandKind::Register) {
+		const auto thread_bit = ThreadBit(value[0], value[1]);
+		switch (operand.reg.file) {
+			case IR::RegisterFile::Exec:
+				ir.SetExec(thread_bit);
+				ir.SetExecLo(value[0]);
+				ir.SetExecHi(value[1]);
+				return;
+			case IR::RegisterFile::Vcc:
+				ir.SetVcc(thread_bit);
+				ir.SetVccLo(value[0]);
+				ir.SetVccHi(value[1]);
+				return;
+			case IR::RegisterFile::Scalar:
+				ir.SetThreadBitScalarReg(static_cast<IR::ScalarReg>(operand.reg.index), thread_bit);
+				break;
+			default: break;
+		}
+	}
+	WriteRawU32(operand, value[0]);
+	WriteRawU32(OffsetOperand(operand, 1), value[1]);
 }
 
 IR::U1 Translator::ThreadBit(IR::U32 low, IR::U32 high) {

src/graphics/shader/recompiler/frontend/translate/Translator.h

@@ -36,6 +36,7 @@ private:
 	void                   WriteF16(const IR::Operand& operand, IR::F32 value);
 	void                   WriteU16(const IR::Operand& operand, IR::U32 value);
 	IR::U32                ReadU32(const IR::Operand& operand);
+	std::array<IR::U32, 2> ReadU32Pair(const IR::Operand& operand);
 	IR::U64                ReadU64(const IR::Operand& operand);
 	IR::F32                ReadF16LaneAsF32(const IR::Operand& operand, bool high_lane);
 	IR::F32                ReadF16AsF32(const IR::Operand& operand);
@@ -44,7 +45,8 @@ private:
 	IR::U32 ReadU16LaneAsU32(const IR::Operand& operand, bool high_lane, bool sign_extend);
 	IR::U32 ReadU16AsU32(const IR::Operand& operand, bool sign_extend);
 	IR::U32 ReadF16LaneBits(const IR::Operand& operand, bool high_lane);
-	std::array<IR::U32, 2> UnpackU64(IR::U64 value);
+	std::array<IR::U32, 2> ExtractU64(IR::U64 value);
+	void    WriteU32Pair(const IR::Operand& operand, const std::array<IR::U32, 2>& value);
 	IR::U1  ReadCondition(const IR::Operand& operand);
 	IR::U32 ConditionBit(const IR::Operand& operand);
 	IR::U1  ReadMask(const IR::Operand& operand);

src/graphics/shader/recompiler/ir/IREmitter.cpp

@@ -134,14 +134,14 @@ U32 IREmitter::BitCastU32(F16 value) {
 	return U32(Emit(ValueOpcode::ConvertU32U16, {half}));
 }
 
-U64 IREmitter::PackUint2x32(U32 low, U32 high) {
+U64 IREmitter::ConstructU64(U32 low, U32 high) {
-	const auto pair = Emit(ValueOpcode::CompositeConstructU32x2, {low, high});
+	return U64(Emit(ValueOpcode::CompositeConstructU64, {low, high}));
-	return U64(Emit(ValueOpcode::PackUint2x32, {pair}));
 }
 
 U32 IREmitter::CompositeExtract(Value composite, uint32_t index) {
 	ValueOpcode opcode;
 	switch (composite.GetType()) {
+		case Type::U64: opcode = ValueOpcode::CompositeExtractU64; break;
 		case Type::U32x2: opcode = ValueOpcode::CompositeExtractU32x2; break;
 		case Type::U32x3: opcode = ValueOpcode::CompositeExtractU32x3; break;
 		case Type::U32x4: opcode = ValueOpcode::CompositeExtractU32x4; break;

src/graphics/shader/recompiler/ir/IREmitter.h

@@ -54,7 +54,7 @@ public:
 	U32 BitCastU32(F32 value);
 	F16 BitCastF16(U32 value);
 	U32 BitCastU32(F16 value);
-	U64 PackUint2x32(U32 low, U32 high);
+	U64 ConstructU64(U32 low, U32 high);
 	U32 CompositeExtract(Value composite, uint32_t index);
 
 	U32 IAdd(U32 lhs, U32 rhs);

src/graphics/shader/recompiler/ir/ValueProgram.cpp

@@ -328,12 +328,13 @@ bool ValidateValueProgram(const ValueProgram& program, bool require_ssa, std::st
 				}
 				const bool scalar_memory = scalar_buffer || scalar_address;
 				const bool valid_group_width =
-				    scalar_memory ? memory.component_count == 1u || memory.component_count == 2u ||
+				    scalar_memory
+				        ? memory.component_count == 1u || memory.component_count == 2u ||
 				              memory.component_count == 4u || memory.component_count == 8u ||
 				              memory.component_count == 16u
 				        : memory.component_count >= 1u && memory.component_count <= 4u;
-				if (memory.data_bits != 32u || memory.data_dwords != 1u ||
+				if (memory.data_bits != 32u || memory.data_dwords != 1u || !valid_group_width ||
-				    !valid_group_width || memory.component_index >= memory.component_count) {
+				    memory.component_index >= memory.component_count) {
 					return Fail(error, fmt::format("{} has inconsistent scalar-memory metadata",
 					                               ValueOpcodeName(inst.GetOpcode())));
 				}
@@ -395,6 +396,7 @@ bool ValidateValueProgram(const ValueProgram& program, bool require_ssa, std::st
 			}
 			uint32_t composite_components = 0u;
 			switch (inst.GetOpcode()) {
+				case ValueOpcode::CompositeExtractU64: composite_components = 2u; break;
 				case ValueOpcode::CompositeExtractU32x2: composite_components = 2u; break;
 				case ValueOpcode::CompositeExtractU32x3: composite_components = 3u; break;
 				case ValueOpcode::CompositeExtractU32x4: composite_components = 4u; break;

src/graphics/shader/recompiler/ir/opcodes/ValueOpcodes.inc

@@ -56,15 +56,15 @@ VALUE_OPCODE(ConvertS32F32, U32, F32)
 VALUE_OPCODE(ConvertU32F32, U32, F32)
 VALUE_OPCODE(ConvertF32S32, F32, U32)
 VALUE_OPCODE(ConvertF32U32, F32, U32)
+VALUE_OPCODE(CompositeConstructU64, U64, U32, U32)
 VALUE_OPCODE(CompositeConstructU32x2, U32x2, U32, U32)
 VALUE_OPCODE(CompositeConstructU32x3, U32x3, U32, U32, U32)
 VALUE_OPCODE(CompositeConstructF32x2, F32x2, F32, F32)
 VALUE_OPCODE(CompositeConstructU32x4, U32x4, U32, U32, U32, U32)
+VALUE_OPCODE(CompositeExtractU64, U32, U64, U32)
 VALUE_OPCODE(CompositeExtractU32x2, U32, U32x2, U32)
 VALUE_OPCODE(CompositeExtractU32x3, U32, U32x3, U32)
 VALUE_OPCODE(CompositeExtractU32x4, U32, U32x4, U32)
-VALUE_OPCODE(PackUint2x32, U64, U32x2)
-VALUE_OPCODE(UnpackUint2x32, U32x2, U64)
 VALUE_OPCODE(PackHalf2x16, U32, F32x2)
 VALUE_OPCODE(PackSnorm2x16, U32, F32x2)
 VALUE_OPCODE(PackUnorm2x16, U32, F32x2)
@@ -105,7 +105,6 @@ VALUE_OPCODE(ShiftRightArithmetic64, U64, U64, U32)
 VALUE_OPCODE(BitwiseAnd32, U32, U32, U32)
 VALUE_OPCODE(BitwiseAnd64, U64, U64, U64)
 VALUE_OPCODE(BitwiseOr32, U32, U32, U32)
-VALUE_OPCODE(BitwiseOr64, U64, U64, U64)
 VALUE_OPCODE(BitwiseXor32, U32, U32, U32)
 VALUE_OPCODE(BitwiseNot32, U32, U32)
 VALUE_OPCODE(BitReverse32, U32, U32)

src/graphics/shader/recompiler/ir/passes/ConstantPropagation.cpp

@@ -170,6 +170,10 @@ bool FoldCompositeExtract(Inst& inst, ValueOpcode construct, size_t components)
 	}
 	const auto  component = index.U32();
 	const auto* producer  = composite.TryInstruction();
+	if (IsImmediate(composite, Type::U64)) {
+		Replace(inst, Value(static_cast<uint32_t>(composite.U64() >> (component * 32u))));
+		return true;
+	}
 	if (producer != nullptr && producer->GetOpcode() == construct) {
 		Replace(inst, producer->Arg(component));
 		return true;
@@ -185,14 +189,6 @@ bool FoldCompositeExtract(Inst& inst, ValueOpcode construct, size_t components)
 			return true;
 		}
 	}
-	if (component < 2u && producer != nullptr &&
-	    producer->GetOpcode() == ValueOpcode::UnpackUint2x32) {
-		const auto packed = producer->Arg(0).Resolve();
-		if (IsImmediate(packed, Type::U64)) {
-			Replace(inst, Value(static_cast<uint32_t>(packed.U64() >> (component * 32u))));
-			return true;
-		}
-	}
 	return false;
 }
 
@@ -284,6 +280,9 @@ void FoldInstruction(Inst& inst) {
 			}
 			return;
 		}
+		case ValueOpcode::CompositeExtractU64:
+			FoldCompositeExtract(inst, ValueOpcode::CompositeConstructU64, 2);
+			return;
 		case ValueOpcode::CompositeExtractU32x2:
 			FoldCompositeExtract(inst, ValueOpcode::CompositeConstructU32x2, 2);
 			return;
@@ -293,28 +292,13 @@ void FoldInstruction(Inst& inst) {
 		case ValueOpcode::CompositeExtractU32x4:
 			FoldCompositeExtract(inst, ValueOpcode::CompositeConstructU32x4, 4);
 			return;
-		case ValueOpcode::PackUint2x32: {
+		case ValueOpcode::CompositeConstructU64: {
-			const auto  value    = Arg(inst, 0);
+			const auto low  = Arg(inst, 0);
-			const auto* producer = value.TryInstruction();
+			const auto high = Arg(inst, 1);
-			if (producer != nullptr && producer->GetOpcode() == ValueOpcode::UnpackUint2x32) {
-				Replace(inst, producer->Arg(0));
-			} else if (producer != nullptr &&
-			           producer->GetOpcode() == ValueOpcode::CompositeConstructU32x2) {
-				const auto low  = producer->Arg(0).Resolve();
-				const auto high = producer->Arg(1).Resolve();
 			if (IsImmediate(low, Type::U32) && IsImmediate(high, Type::U32)) {
 				Replace(inst, Value(static_cast<uint64_t>(low.U32()) |
 				                    (static_cast<uint64_t>(high.U32()) << 32u)));
 			}
-			}
-			return;
-		}
-		case ValueOpcode::UnpackUint2x32: {
-			const auto  value    = Arg(inst, 0);
-			const auto* producer = value.TryInstruction();
-			if (producer != nullptr && producer->GetOpcode() == ValueOpcode::PackUint2x32) {
-				Replace(inst, producer->Arg(0));
-			}
 			return;
 		}
 		case ValueOpcode::IAdd32:
@@ -367,8 +351,8 @@ void FoldInstruction(Inst& inst) {
 				};
 				const auto low  = expand(static_cast<uint32_t>(value.U64()));
 				const auto high = expand(static_cast<uint32_t>(value.U64() >> 32u));
-				Replace(inst, Value(static_cast<uint64_t>(low) |
+				Replace(inst,
-				                    (static_cast<uint64_t>(high) << 32u)));
+				        Value(static_cast<uint64_t>(low) | (static_cast<uint64_t>(high) << 32u)));
 			}
 			return;
 		}
@@ -461,11 +445,6 @@ void FoldInstruction(Inst& inst) {
 				ReplaceBinaryIdentity(inst, Type::U32, 0u);
 			}
 			return;
-		case ValueOpcode::BitwiseOr64:
-			if (!FoldU64(inst, [](uint64_t a, uint64_t b) { return a | b; })) {
-				ReplaceBinaryIdentity(inst, Type::U64, 0u);
-			}
-			return;
 		case ValueOpcode::BitwiseNot32: {
 			const auto value = Arg(inst, 0);
 			if (IsImmediate(value, Type::U32)) {

src/graphics/shader/recompiler/ir/passes/SrtWalker.cpp

@@ -76,10 +76,10 @@ bool IsDescriptorHandle(ValueOpcode opcode) {
 
 bool IsRuntimeIntegerOp(ValueOpcode op) {
 	switch (op) {
+		case ValueOpcode::CompositeConstructU64:
+		case ValueOpcode::CompositeExtractU64:
 		case ValueOpcode::CompositeConstructU32x2:
 		case ValueOpcode::CompositeExtractU32x2:
-		case ValueOpcode::PackUint2x32:
-		case ValueOpcode::UnpackUint2x32:
 		case ValueOpcode::BitFieldInsert:
 		case ValueOpcode::BitFieldUExtract:
 		case ValueOpcode::BitFieldSExtract:
@@ -100,7 +100,6 @@ bool IsRuntimeIntegerOp(ValueOpcode op) {
 		case ValueOpcode::BitwiseAnd32:
 		case ValueOpcode::BitwiseAnd64:
 		case ValueOpcode::BitwiseOr32:
-		case ValueOpcode::BitwiseOr64:
 		case ValueOpcode::BitwiseXor32:
 		case ValueOpcode::BitwiseNot32:
 		case ValueOpcode::SelectU1:
@@ -211,10 +210,10 @@ private:
 				reason = fmt::format("contains a non-scalar {}", ValueOpcodeName(op));
 				return finish(false);
 			}
-		} else if (op == ValueOpcode::PackUint2x32) {
+		} else if (op == ValueOpcode::CompositeExtractU64) {
-			const auto* pair = inst->NumArgs() == 1 ? inst->Arg(0).ResolveInstruction() : nullptr;
+			const auto index = inst->NumArgs() == 2 ? inst->Arg(1).Resolve() : Value {};
-			if (pair == nullptr || pair->GetOpcode() != ValueOpcode::CompositeConstructU32x2) {
+			if (!index.IsImmediate() || index.GetType() != Type::U32 || index.U32() >= 2u) {
-				reason = "contains an unsupported packed runtime value";
+				reason = "contains an invalid U64 component index";
 				return finish(false);
 			}
 		} else if (op == ValueOpcode::CompositeExtractU32x2) {
@@ -223,7 +222,6 @@ private:
 			if (source == nullptr || !index.IsImmediate() || index.GetType() != Type::U32 ||
 			    index.U32() >= 2u ||
 			    (source->GetOpcode() != ValueOpcode::CompositeConstructU32x2 &&
-			     source->GetOpcode() != ValueOpcode::UnpackUint2x32 &&
 			     source->GetOpcode() != ValueOpcode::IAddCarry32)) {
 				reason = "contains an unsupported composite runtime source";
 				return finish(false);
@@ -485,21 +483,24 @@ private:
 			return Fail(error, "dynamic composite extract in runtime expression");
 		}
 		const auto component = index.U32();
-		const auto* source    = inst.Arg(0).ResolveInstruction();
+		if (component >= 2u) {
-		if (source == nullptr || component >= 2u) {
 			return Fail(error, "unsupported composite runtime source");
 		}
-		if (source->GetOpcode() == ValueOpcode::CompositeConstructU32x2) {
+		if (inst.GetOpcode() == ValueOpcode::CompositeExtractU64) {
-			return EvaluateWide(source->Arg(component), result, error);
-		}
-		if (source->GetOpcode() == ValueOpcode::UnpackUint2x32) {
 			uint64_t packed = 0;
-			if (!Arg(*source, 0, packed, error)) {
+			if (!Arg(inst, 0, packed, error)) {
 				return false;
 			}
 			result = static_cast<uint32_t>(packed >> (component * 32u));
 			return true;
 		}
+		const auto* source = inst.Arg(0).ResolveInstruction();
+		if (source == nullptr) {
+			return Fail(error, "unsupported composite runtime source");
+		}
+		if (source->GetOpcode() == ValueOpcode::CompositeConstructU32x2) {
+			return EvaluateWide(source->Arg(component), result, error);
+		}
 		if (source->GetOpcode() == ValueOpcode::IAddCarry32) {
 			uint64_t lhs = 0;
 			uint64_t rhs = 0;
@@ -596,17 +597,15 @@ private:
 			}
 			case ValueOpcode::GetShaderBase: result = m_runtime.shader_base; return true;
 			case ValueOpcode::Phi: return EvaluatePhi(inst, result, error);
+			case ValueOpcode::CompositeExtractU64:
 			case ValueOpcode::CompositeExtractU32x2: return EvaluateExtract(inst, result, error);
-			case ValueOpcode::PackUint2x32: {
+			case ValueOpcode::CompositeConstructU64:
-				const auto* pair = inst.Arg(0).ResolveInstruction();
+				if (!binary()) {
-				if (pair == nullptr || pair->GetOpcode() != ValueOpcode::CompositeConstructU32x2 ||
+					return false;
-				    !Arg(*pair, 0, a, error) || !Arg(*pair, 1, b, error)) {
-					return Fail(error, "unsupported packed runtime value");
 				}
 				result = static_cast<uint32_t>(a) |
 				         (static_cast<uint64_t>(static_cast<uint32_t>(b)) << 32u);
 				return true;
-			}
 			case ValueOpcode::ReadConst: {
 				const auto slot = inst.Arg(1).Resolve();
 				if (!slot.IsImmediate() || slot.GetType() != Type::U32 ||
@@ -687,12 +686,6 @@ private:
 					return true;
 				}
 				return false;
-			case ValueOpcode::BitwiseOr64:
-				if (binary()) {
-					result = a | b;
-					return true;
-				}
-				return false;
 			case ValueOpcode::BitwiseXor32:
 				if (binary()) {
 					result = static_cast<uint32_t>(a ^ b);

tests/ScalarProvenanceTests.cpp

@@ -1,8 +1,8 @@
+#include "graphics/shader/recompiler/ir/ValueProgram.h"
 #include "graphics/shader/recompiler/ir/passes/ConstantPropagation.h"
 #include "graphics/shader/recompiler/ir/passes/DeadCodeElimination.h"
 #include "graphics/shader/recompiler/ir/passes/ReadLaneElimination.h"
 #include "graphics/shader/recompiler/ir/passes/SrtWalker.h"
-#include "graphics/shader/recompiler/ir/ValueProgram.h"
 
 #include <cstdlib>
 #include <iostream>
@@ -133,16 +133,15 @@ void TestRawScalarComponentAlignment() {
   std::vector<uint32_t> flat;
   std::string error;
   Check(WalkSrt(fixture.program, runtime, flat, &error), error.c_str());
-  Check(flat == std::vector<uint32_t>{0x12345678u} &&
+  Check(
-            memory_image.reads == 1,
+      flat == std::vector<uint32_t>{0x12345678u} && memory_image.reads == 1,
       "raw scalar base, immediate, and offset were not aligned independently");
 }
 
 void TestScalarMemoryDomainMismatchFails() {
   Fixture raw;
   const auto raw_memory = raw.AddMemory(ResourceKind::ScalarBuffer);
-  RawRead(raw, Address(raw, Value(0x1000u), Value(0u)), Value(0u),
+  RawRead(raw, Address(raw, Value(0x1000u), Value(0u)), Value(0u), raw_memory);
-          raw_memory);
   std::string error;
   Check(!BuildSrtPlan(raw.program, &error) &&
             error.find("incompatible scalar memory metadata") !=
@@ -151,11 +150,11 @@ void TestScalarMemoryDomainMismatchFails() {
 
   Fixture buffer;
   const auto buffer_memory = buffer.AddMemory(ResourceKind::ScalarAddress);
-  const auto resource = buffer.Emit(ValueOpcode::GetBufferResource,
+  const auto resource =
-                                    {Value(0x1000u), Value(0u), Value(16u),
+      buffer.Emit(ValueOpcode::GetBufferResource,
-                                     Value(0u)});
+                  {Value(0x1000u), Value(0u), Value(16u), Value(0u)});
-  buffer.EmitMemory(ValueOpcode::ReadConstBuffer,
+  buffer.EmitMemory(ValueOpcode::ReadConstBuffer, {resource, Value(0u)},
-                    {resource, Value(0u)}, buffer_memory);
+                    buffer_memory);
   error.clear();
   Check(!BuildSrtPlan(buffer.program, &error) &&
             error.find("incompatible scalar memory metadata") !=
@@ -202,11 +201,10 @@ void TestNestedSrtWalk() {
 void TestShaderBaseAndUserData() {
   Fixture fixture;
   const auto base = fixture.Emit(ValueOpcode::GetShaderBase);
-  const auto pair = fixture.Emit(ValueOpcode::UnpackUint2x32, {base});
   const auto low =
-      fixture.Emit(ValueOpcode::CompositeExtractU32x2, {pair, Value(0u)});
+      fixture.Emit(ValueOpcode::CompositeExtractU64, {base, Value(0u)});
   const auto high =
-      fixture.Emit(ValueOpcode::CompositeExtractU32x2, {pair, Value(1u)});
+      fixture.Emit(ValueOpcode::CompositeExtractU64, {base, Value(1u)});
   const auto user = fixture.Emit(ValueOpcode::GetUserData,
                                  {Value(static_cast<ScalarReg>(2))});
   const auto sum = fixture.Emit(ValueOpcode::IAdd32, {user, Value(4u)});
@@ -310,13 +308,12 @@ void TestRuntime64BitDescriptorOps() {
   const auto masked =
       fixture.Emit(ValueOpcode::BitwiseAnd64,
                    {shifted, Value(uint64_t{0x0000ffff00000000ull})});
-  const auto combined = fixture.Emit(ValueOpcode::BitwiseOr64,
+  const auto combined = fixture.Emit(
-                                     {masked, Value(uint64_t{0xabcdu})});
+      ValueOpcode::IAdd64, {masked, Value(uint64_t{0x000000010000abcdull})});
-  const auto pair = fixture.Emit(ValueOpcode::UnpackUint2x32, {combined});
   const auto low =
-      fixture.Emit(ValueOpcode::CompositeExtractU32x2, {pair, Value(0u)});
+      fixture.Emit(ValueOpcode::CompositeExtractU64, {combined, Value(0u)});
   const auto high =
-      fixture.Emit(ValueOpcode::CompositeExtractU32x2, {pair, Value(1u)});
+      fixture.Emit(ValueOpcode::CompositeExtractU64, {combined, Value(1u)});
   fixture.Plan();
   fixture.program.values->descriptor_sources.push_back(
       {.dwords = {low, high}, .dword_count = 2});
@@ -324,8 +321,8 @@ void TestRuntime64BitDescriptorOps() {
   DescriptorValue result;
   std::string error;
   Check(EvaluateDescriptorSource(fixture.program, 0, 0, {}, result, &error) &&
-            result.dwords[0] == 0xabcdu && result.dwords[1] == 0x1234u,
+            result.dwords[0] == 0xabcdu && result.dwords[1] == 0x1235u,
-        "64-bit typed descriptor shift/mask evaluation is incorrect");
+        "64-bit typed descriptor arithmetic evaluation is incorrect");
 }
 
 void TestConstantBufferBounds() {
@@ -458,9 +455,9 @@ int main() {
 // Keep this focused standalone target self-contained by amalgamating its small
 // typed-IR implementation set.
 #include "../src/graphics/shader/recompiler/ir/Block.cpp"
-#include "../src/graphics/shader/recompiler/ir/passes/ConstantPropagation.cpp"
-#include "../src/graphics/shader/recompiler/ir/passes/DeadCodeElimination.cpp"
 #include "../src/graphics/shader/recompiler/ir/Type.cpp"
 #include "../src/graphics/shader/recompiler/ir/Value.cpp"
-#include "../src/graphics/shader/recompiler/ir/opcodes/ValueOpcodes.cpp"
 #include "../src/graphics/shader/recompiler/ir/ValueProgram.cpp"
+#include "../src/graphics/shader/recompiler/ir/opcodes/ValueOpcodes.cpp"
+#include "../src/graphics/shader/recompiler/ir/passes/ConstantPropagation.cpp"
+#include "../src/graphics/shader/recompiler/ir/passes/DeadCodeElimination.cpp"

tests/shaderCfgTests.cpp

@@ -31,6 +31,7 @@
 
 #include <algorithm>
 #include <array>
+#include <bit>
 #include <cstdint>
 #include <cstdio>
 #include <cstdlib>
@@ -1746,7 +1747,6 @@ void TestNewShaderRecompilerScalarVectorAlu() {
   Check(SpirvContainsOpcode(result.spirv, 199),
         "SPIR-V binary does not contain OpBitwiseAnd");
   CheckSpirvBinaryValidates(result.spirv);
-
 }
 
 void TestNewShaderRecompilerVop3LaneReadDestinationEncoding() {
@@ -3634,6 +3634,62 @@ void TestNewShaderRecompilerScalarB64Alu() {
         "extract");
   Check(SpirvContainsOpcode(result.spirv, 205),
         "SPIR-V binary does not contain OpBitCount for scalar B64 bit count");
+  uint32_t componentwise_u64_values = 0;
+  for (const auto *block : result.program.values->blocks) {
+    for (const auto &value : *block) {
+      switch (value.GetOpcode()) {
+      case ShaderRecompiler::IR::ValueOpcode::BitwiseAnd64:
+        componentwise_u64_values++;
+        break;
+      default:
+        break;
+      }
+    }
+  }
+  Check(
+      componentwise_u64_values == 0u,
+      "architectural B64 lane operations retained vector pack/unpack traffic");
+  CheckSpirvBinaryValidates(result.spirv);
+}
+
+void TestNewShaderRecompilerScalarB64LaneLowering() {
+  const uint32_t shader[] = {
+      EncodeSop1(0x04, 2, 0),      // s_mov_b64 s[2:3], s[0:1]
+      EncodeSop1(0x08, 4, 2),      // s_not_b64 s[4:5], s[2:3]
+      EncodeSop2(0x0f, 6, 2, 4),   // s_and_b64 s[6:7], s[2:3], s[4:5]
+      EncodeSop2(0x11, 8, 6, 2),   // s_or_b64 s[8:9], s[6:7], s[2:3]
+      EncodeSop2(0x13, 10, 8, 4),  // s_xor_b64 s[10:11], s[8:9], s[4:5]
+      EncodeSopc(0x06, 0, 1),      // s_cmp_eq_u32 s0, s1
+      EncodeSop2(0x0b, 12, 10, 2), // s_cselect_b64 s[12:13], s[10:11], s[2:3]
+      EncodeSop2(0x15, 14, 12, 4), // s_andn2_b64 s[14:15], s[12:13], s[4:5]
+      EncodeVop1(0x01, 0, 14),     // v_mov_b32 v0, s14
+      EncodeExp0(0x0c, 0x1),       EncodeExp1(0, 0, 0, 0), EncodeSopp(0x01),
+  };
+  const uint32_t user_data[] = {0x01234567u, 0x89abcdefu};
+  auto options = MakeCompileOptions(ShaderType::Vertex);
+  options.user_data = user_data;
+  options.user_data_count = static_cast<uint32_t>(std::size(user_data));
+  options.dump_ir = true;
+
+  ShaderRecompiler::CompileResult result;
+  std::string error;
+  Check(ShaderRecompiler::TryRecompile(shader, options, result, &error),
+        error.c_str());
+  uint32_t componentwise_u64_values = 0;
+  for (const auto *block : result.program.values->blocks) {
+    for (const auto &value : *block) {
+      switch (value.GetOpcode()) {
+      case ShaderRecompiler::IR::ValueOpcode::BitwiseAnd64:
+        componentwise_u64_values++;
+        break;
+      default:
+        break;
+      }
+    }
+  }
+  Check(
+      componentwise_u64_values == 0u,
+      "architectural B64 lane operations retained vector pack/unpack traffic");
   CheckSpirvBinaryValidates(result.spirv);
 }
 
@@ -5517,9 +5573,8 @@ void TestNewShaderRecompilerNativeWideScalarMemoryIr() {
   std::string error;
   Check(ShaderRecompiler::Decoder::DecodeProgram(shader, decoded, &error) &&
             ShaderRecompiler::CFG::BuildGraph(decoded, graph, &error) &&
-            ShaderRecompiler::IR::LowerProgram(decoded, graph,
+            ShaderRecompiler::IR::LowerProgram(
-                                                ShaderType::Compute, 64, ir,
+                decoded, graph, ShaderType::Compute, 64, ir, &error),
-                                                &error),
         error.c_str());
   std::vector<uint32_t> address_widths;
   std::vector<uint32_t> buffer_widths;
@@ -5528,8 +5583,7 @@ void TestNewShaderRecompilerNativeWideScalarMemoryIr() {
     for (const auto &inst : block.instructions) {
       if (inst.op == ShaderRecompiler::IR::Opcode::SLoadDword) {
         address_widths.push_back(inst.memory.data_dwords);
-      } else if (inst.op ==
+      } else if (inst.op == ShaderRecompiler::IR::Opcode::SBufferLoadDword) {
-                 ShaderRecompiler::IR::Opcode::SBufferLoadDword) {
         buffer_widths.push_back(inst.memory.data_dwords);
         if (inst.memory.data_dwords == 2u) {
           buffer_x2_targets_vcc =
@@ -5541,14 +5595,15 @@ void TestNewShaderRecompilerNativeWideScalarMemoryIr() {
   }
   const std::vector<uint32_t> expected{1u, 2u, 4u, 8u, 16u};
   Check(address_widths == expected && buffer_widths == expected,
-        "SMEM x1/x2/x4/x8/x16 did not remain one width-bearing ShaderIR operation each");
+        "SMEM x1/x2/x4/x8/x16 did not remain one width-bearing ShaderIR "
+        "operation each");
   Check(buffer_x2_targets_vcc,
         "s_buffer_load_dwordx2 did not preserve its VCC destination span");
 
   const auto reject = [&](uint32_t opcode, uint32_t dst, uint32_t sbase,
                           const char *reason) {
-    const uint32_t invalid[] = {EncodeSmem0(opcode, dst, sbase),
+    const uint32_t invalid[] = {EncodeSmem0(opcode, dst, sbase), 125u << 25u,
-                                125u << 25u, EncodeSopp(0x01)};
+                                EncodeSopp(0x01)};
     ShaderRecompiler::Decoder::Program invalid_decoded;
     ShaderRecompiler::CFG::Graph invalid_graph;
     ShaderRecompiler::IR::Program invalid_ir;
@@ -6572,7 +6627,6 @@ void TestNewShaderRecompilerDsAddtidLowering() {
   Check(SpirvContainsOpcode(result.spirv, 199),
         "SPIR-V binary does not contain OpBitwiseAnd");
   CheckSpirvBinaryValidates(result.spirv);
-
 }
 
 void TestNewShaderRecompilerDsFloatMinMaxLowering() {
@@ -6693,6 +6747,53 @@ void TestNewShaderRecompilerCfgConsecutiveNativePhis() {
   CheckSpirvBinaryValidates(result.spirv);
 }
 
+void TestNewShaderRecompilerStructuredU64Phi() {
+  using namespace ShaderRecompiler;
+
+  const uint32_t shader[] = {
+      EncodeSopc(0x06, 0, 1), EncodeSopp(0x05, 2),   EncodeSMovB32(2, 129),
+      EncodeSopp(0x02, 1),    EncodeSMovB32(2, 130), EncodeVop1(0x01, 0, 2),
+      EncodeSopp(0x01),
+  };
+  auto options = MakeCompileOptions(ShaderType::Compute);
+  CompileResult result;
+  std::string error;
+  Check(TryRecompile(shader, options, result, &error), error.c_str());
+  Check(!result.program.dispatcher_fallback,
+        "U64 Phi fixture did not select structured mode");
+
+  auto program = result.program;
+  IR::Block *join = nullptr;
+  for (auto *block : program.values->blocks) {
+    if (block->ImmPredecessors().size() == 2u) {
+      join = block;
+      break;
+    }
+  }
+  Check(join != nullptr, "U64 Phi fixture has no two-parent join block");
+  auto &phi = *join->PrependNewInst(join->begin(), IR::ValueOpcode::Phi);
+  phi.SetFlags(IR::Type::U64);
+  phi.AddPhiOperand(join->ImmPredecessors()[0],
+                    IR::Value(uint64_t{0x1111111122222222ull}));
+  phi.AddPhiOperand(join->ImmPredecessors()[1],
+                    IR::Value(uint64_t{0x3333333344444444ull}));
+  IR::IREmitter use(join);
+  use.Emit(IR::ValueOpcode::CompositeExtractU64,
+           {IR::Value(&phi), IR::Value(1u)});
+  Check(Spirv::AnalyzeProgramRequirements(program, &error), error.c_str());
+
+  std::vector<uint32_t> spirv;
+  Check(Spirv::EmitProgram(program, result.resources, options.input_info, spirv,
+                           &error),
+        error.c_str());
+  CheckSpirvBinaryValidates(spirv);
+  const auto before = MeasureSpirv(result.spirv);
+  const auto after = MeasureSpirv(spirv);
+  Check(after.phis == before.phis + 1u &&
+            after.function_variables == before.function_variables,
+        "structured U64 Phi was not emitted as a native vector OpPhi");
+}
+
 void TestNewShaderRecompilerCfgTerminalExitMergePS() {
   const uint32_t shader[] = {
       EncodeSopp(0x04, 2),   // s_cbranch_scc0 end
@@ -7861,6 +7962,91 @@ void TestNewShaderRecompilerDispatcherSpillsU32x3() {
         "dispatcher did not use one canonical U32x3 spill slot");
 }
 
+void TestNewShaderRecompilerU64PairLowering() {
+  using namespace ShaderRecompiler;
+
+  const uint32_t empty_shader[] = {EncodeSopp(0x01)};
+  auto options = MakeCompileOptions(ShaderType::Compute);
+  CompileResult result;
+  std::string error;
+  Check(TryRecompile(empty_shader, options, result, &error), error.c_str());
+
+  auto program = result.program;
+  IR::IREmitter ir(program.values->blocks.front());
+  const auto lane = ir.Emit(IR::ValueOpcode::LaneId);
+  const auto high = ir.Emit(IR::ValueOpcode::IAdd32, {lane, IR::Value(1u)});
+  const auto base =
+      ir.Emit(IR::ValueOpcode::CompositeConstructU64, {lane, high});
+  const auto masked =
+      ir.Emit(IR::ValueOpcode::BitwiseAnd64,
+              {base, IR::Value(uint64_t{0xff00ff00ff00ff00ull})});
+  const auto sum = ir.Emit(IR::ValueOpcode::IAdd64, {masked, base});
+  const auto product = ir.Emit(IR::ValueOpcode::IMul64, {sum, base});
+  const auto shifted =
+      ir.Emit(IR::ValueOpcode::ShiftRightArithmetic64, {product, lane});
+  ir.Emit(IR::ValueOpcode::IEqual64, {shifted, base});
+  ir.Emit(IR::ValueOpcode::INotEqual64, {shifted, base});
+  ir.Emit(IR::ValueOpcode::ULessThan64, {shifted, base});
+  ir.Emit(IR::ValueOpcode::SLessThan64, {shifted, base});
+  ir.Emit(IR::ValueOpcode::BitCount64, {shifted});
+  ir.Emit(IR::ValueOpcode::WqmU64, {shifted});
+  for (const uint32_t count : {1u, 31u, 32u, 33u, 63u}) {
+    ir.Emit(IR::ValueOpcode::ShiftLeftLogical64, {base, IR::Value(count)});
+    ir.Emit(IR::ValueOpcode::ShiftRightLogical64, {base, IR::Value(count)});
+    ir.Emit(IR::ValueOpcode::ShiftRightArithmetic64, {base, IR::Value(count)});
+  }
+  Check(Spirv::AnalyzeProgramRequirements(program, &error), error.c_str());
+
+  std::vector<uint32_t> spirv;
+  Check(Spirv::EmitProgram(program, result.resources, options.input_info, spirv,
+                           &error),
+        error.c_str());
+  CheckSpirvBinaryValidates(spirv);
+  const auto source = DisassembleSpirvBinary(spirv);
+  Check(!Common::ContainsStr(source, "OpCapability Int64") &&
+            !Common::ContainsStr(source, "OpTypeInt 64"),
+        "portable pair-U64 lowering introduced native shader Int64");
+  Check(SpirvInstructionOpcodeCount(spirv, 149u) == 1u,
+        "pair-U64 addition did not use exactly one carry instruction");
+  Check(SpirvInstructionOpcodeCount(spirv, 154u) == 1u &&
+            SpirvInstructionOpcodeCount(spirv, 155u) == 1u,
+        "pair-U64 equality did not reduce its vector comparison with Any/All");
+  const auto direct_metrics = MeasureSpirv(spirv);
+  Check(
+      direct_metrics.words <= 940u && direct_metrics.instructions <= 199u &&
+          direct_metrics.type_vectors == 2u && direct_metrics.phis == 0u &&
+          direct_metrics.function_variables == 0u,
+      "portable pair-U64 lowering exceeded its declaration/code-size ratchet");
+
+  const uint32_t dispatcher_shader[] = {
+      EncodeSopp(0x05, 2),       EncodeSopp(0x02, 0), EncodeSopp(0x05, 0xfffeu),
+      EncodeSopp(0x02, 0xfffeu), EncodeSopp(0x01),
+  };
+  Check(TryRecompile(dispatcher_shader, options, result, &error),
+        error.c_str());
+  Check(result.program.dispatcher_fallback &&
+            result.program.values->blocks.size() >= 2u,
+        "U64 spill fixture did not select dispatcher mode");
+  program = result.program;
+  IR::IREmitter definition(program.values->blocks[0]);
+  const auto vector = definition.Emit(IR::ValueOpcode::CompositeConstructU64,
+                                      {IR::Value(1u), IR::Value(2u)});
+  IR::IREmitter use(program.values->blocks[1]);
+  use.Emit(IR::ValueOpcode::CompositeExtractU64, {vector, IR::Value(1u)});
+  Check(Spirv::AnalyzeProgramRequirements(program, &error), error.c_str());
+  spirv.clear();
+  Check(Spirv::EmitProgram(program, result.resources, options.input_info, spirv,
+                           &error),
+        error.c_str());
+  CheckSpirvBinaryValidates(spirv);
+  const auto before = MeasureSpirv(result.spirv);
+  const auto after = MeasureSpirv(spirv);
+  Check(after.function_variables == before.function_variables + 1u &&
+            after.loads == before.loads + 1u &&
+            after.stores == before.stores + 1u,
+        "dispatcher did not use one canonical U64 vector spill slot");
+}
+
 void TestComputeShaderInputWaveSize() {
   const auto decode_wave_size = [](uint32_t rsrc1) {
     const bool wave32 = (((rsrc1 >> Pm4::COMPUTE_PGM_RSRC1_W32_EN_SHIFT) &
@@ -8552,6 +8738,40 @@ void TestWqmConstantPropagation() {
   check_invalid_signature(ValueOpcode::WqmU64, Value(uint64_t{1}), Value(true));
   check_invalid_signature(ValueOpcode::WqmMask, Value(true),
                           Value(uint64_t{1}));
+
+  ValueProgram shifts;
+  shifts.block_storage.push_back(std::make_unique<Block>());
+  shifts.blocks.push_back(shifts.block_storage.back().get());
+  shifts.block_info.emplace_back();
+  IREmitter shift_ir(shifts.blocks.front());
+  struct ShiftCase {
+    Value value;
+    uint64_t expected;
+  };
+  std::vector<ShiftCase> shift_cases;
+  for (const uint64_t source :
+       {uint64_t{0x0123456789abcdefull}, uint64_t{0xf123456789abcdefull}}) {
+    for (const uint32_t count : {0u, 1u, 31u, 32u, 33u, 63u, 64u, 65u}) {
+      const uint32_t amount = count & 63u;
+      shift_cases.push_back({shift_ir.Emit(ValueOpcode::ShiftLeftLogical64,
+                                           {Value(source), Value(count)}),
+                             source << amount});
+      shift_cases.push_back({shift_ir.Emit(ValueOpcode::ShiftRightLogical64,
+                                           {Value(source), Value(count)}),
+                             source >> amount});
+      shift_cases.push_back(
+          {shift_ir.Emit(ValueOpcode::ShiftRightArithmetic64,
+                         {Value(source), Value(count)}),
+           static_cast<uint64_t>(std::bit_cast<int64_t>(source) >> amount)});
+    }
+  }
+  ConstantPropagationPass(shifts.blocks);
+  for (const auto &test : shift_cases) {
+    const auto value = test.value.Resolve();
+    Check(value.IsImmediate() && value.GetType() == Type::U64 &&
+              value.U64() == test.expected,
+          "pair-U64 shift propagation violated the masked RDNA2 count");
+  }
 }
 
 void TestNativeWideValueValidation() {
@@ -8575,20 +8795,18 @@ void TestNativeWideValueValidation() {
     ir.Emit(opcode, {resource, Value(0u), Value(0u), Value(0u), Value(true)},
             flags);
   };
-  const auto append_scalar_read = [](ValueProgram &program,
+  const auto append_scalar_read = [](ValueProgram &program, ValueOpcode opcode,
-                                     ValueOpcode opcode,
                                      MemoryFlags flags) {
     IREmitter ir(program.blocks.front());
     if (opcode == ValueOpcode::ReadConstBuffer) {
-      const auto resource = ir.Emit(
+      const auto resource =
-          ValueOpcode::GetBufferResource,
+          ir.Emit(ValueOpcode::GetBufferResource,
                   {Value(0u), Value(0u), Value(0u), Value(0u)});
       ir.Emit(opcode, {resource, Value(0u)}, flags);
     } else {
       const auto resource =
           ir.Emit(ValueOpcode::GetAddressResource, {Value(0u), Value(0u)});
-      ir.Emit(opcode,
+      ir.Emit(opcode, {resource, Value(0u), Value(0u), Value(true)}, flags);
-              {resource, Value(0u), Value(0u), Value(true)}, flags);
     }
   };
   const auto check_rejected = [](const ValueProgram &program,
@@ -8695,6 +8913,24 @@ void TestNativeWideValueValidation() {
     ir.Emit(ValueOpcode::CompositeExtractU32x3, {vector, Value(3u)});
     check_rejected(program, "invalid component index");
   }
+  {
+    auto program = make_program();
+    IREmitter ir(program.blocks.front());
+    const auto vector =
+        ir.Emit(ValueOpcode::CompositeConstructU64, {Value(1u), Value(2u)});
+    ir.Emit(ValueOpcode::CompositeExtractU64, {vector, Value(2u)});
+    check_rejected(program, "invalid component index");
+  }
+  {
+    auto program = make_program();
+    IREmitter ir(program.blocks.front());
+    const auto vector =
+        ir.Emit(ValueOpcode::CompositeConstructU64, {Value(1u), Value(2u)});
+    const auto dynamic_index =
+        ir.Emit(ValueOpcode::IAdd32, {Value(0u), Value(1u)});
+    ir.Emit(ValueOpcode::CompositeExtractU64, {vector, dynamic_index});
+    check_rejected(program, "invalid component index");
+  }
   const auto append_shared = [](ValueProgram &program, ValueOpcode opcode,
                                 MemoryFlags flags) {
     IREmitter ir(program.blocks.front());
@@ -8920,8 +9156,8 @@ void TestNewShaderRecompilerPerInvocationMasksWithoutMirrors() {
             !Common::ContainsStr(DisassembleSpirvBinary(result.spirv),
                                  "OpGroupNonUniformBallot"),
         "native scalar WQM retained frontend expansion or subgroup machinery");
-  Check(SpirvInstructionOpcodeCount(result.spirv, 132u) == 2u &&
+  Check(SpirvInstructionOpcodeCount(result.spirv, 132u) == 1u &&
-            SpirvInstructionOpcodeCount(result.spirv, 194u) == 4u,
+            SpirvInstructionOpcodeCount(result.spirv, 194u) == 2u,
         "native scalar WQM emitter was folded away or did not use the compact "
         "path");
 
@@ -9956,7 +10192,8 @@ void TestScalarMemorySourcesCapturedBeforeWrites() {
           error.c_str());
     Check(flat.size() == table.size() &&
               std::equal(flat.begin(), flat.end(), table.begin()),
-          "overlapping scalar-memory load did not capture its sources before writes");
+          "overlapping scalar-memory load did not capture its sources before "
+          "writes");
     CheckFlattenedReadSlots(
         ir, 4, "overlapping scalar-memory patch used the wrong flat offsets");
   };
@@ -10011,8 +10248,8 @@ void TestScalarMemoryUnusedTailDce() {
   };
   ShaderRecompiler::IR::Program ir;
   std::string error;
-  Check(LowerTypedPlanning(shader, static_cast<uint32_t>(std::size(shader)),
+  Check(LowerTypedPlanning(shader, static_cast<uint32_t>(std::size(shader)), ir,
-                           ir, &error),
+                           &error),
         error.c_str());
   uint32_t reads = 0;
   for (const auto *block : ir.values->blocks) {
@@ -10029,7 +10266,8 @@ void TestScalarMemoryUnusedTailDce() {
             "live x16 SMEM component lost its native-width metadata");
     }
   }
-  Check(reads == 1u,
+  Check(
+      reads == 1u,
       "grouped x16 SMEM prevented dead component reads from being eliminated");
 }
 
@@ -10794,6 +11032,7 @@ int main() {
   TestNewShaderRecompilerSMovB32();
   TestNewShaderRecompilerNativeWideScalarMemoryIr();
   TestNewShaderRecompilerNativeWideBufferIr();
+  TestNewShaderRecompilerScalarB64LaneLowering();
   TestNewShaderRecompilerMubufFormatLowering();
   TestNewShaderRecompilerTypedBufferLowering();
   TestNewShaderRecompilerDsReadWrite2Lowering();
@@ -10813,6 +11052,7 @@ int main() {
   TestNewShaderRecompilerCfgStraightLine();
   TestNewShaderRecompilerCfgIfElse();
   TestNewShaderRecompilerCfgConsecutiveNativePhis();
+  TestNewShaderRecompilerStructuredU64Phi();
   TestNewShaderRecompilerCfgTerminalExitMergePS();
   TestNewShaderRecompilerCfgPostEndTargetMergePS();
   TestNewShaderRecompilerCfgLoopBreakContinue();
@@ -10841,6 +11081,7 @@ int main() {
   TestNewShaderRecompilerCfgExternallyEnteredSelectionDispatcher();
   TestNewShaderRecompilerCfgIrreducibleDispatcher();
   TestNewShaderRecompilerDispatcherSpillsU32x3();
+  TestNewShaderRecompilerU64PairLowering();
   TestComputeShaderInputWaveSize();
   TestNewShaderRecompilerBufferLoadsGuardedByExec();
   TestNewShaderRecompilerBufferAtomicsGuardedByBounds();