From 6e9f162bd4f1a7b33d65b4f653a7d9ec66e2f62b Mon Sep 17 00:00:00 2001 From: nmzik Date: Wed, 19 Aug 2026 10:22:31 +0200 Subject: [PATCH] refactor(shader): compact paired 64-bit lowering --- .../backend/spirv/spirvEmitterAlu.cpp | 241 +++++++------ .../backend/spirv/spirvEmitterAluHelpers.cpp | 19 -- .../backend/spirv/spirvEmitterFlow.cpp | 23 +- .../backend/spirv/spirvEmitterInternal.h | 14 +- .../backend/spirv/spirvEmitterModule.cpp | 10 +- .../backend/spirv/spirvEmitterProgram.cpp | 4 +- .../recompiler/frontend/translate/Control.cpp | 17 +- .../frontend/translate/Dispatch.cpp | 3 +- .../recompiler/frontend/translate/Integer.cpp | 178 ++++------ .../frontend/translate/Translate.cpp | 100 +++--- .../frontend/translate/Translator.h | 30 +- .../shader/recompiler/ir/IREmitter.cpp | 6 +- src/graphics/shader/recompiler/ir/IREmitter.h | 2 +- .../shader/recompiler/ir/ValueProgram.cpp | 20 +- .../recompiler/ir/opcodes/ValueOpcodes.inc | 5 +- .../ir/passes/ConstantPropagation.cpp | 51 +-- .../shader/recompiler/ir/passes/SrtWalker.cpp | 49 ++- tests/ScalarProvenanceTests.cpp | 45 ++- tests/shaderCfgTests.cpp | 317 +++++++++++++++--- 19 files changed, 673 insertions(+), 461 deletions(-) diff --git a/src/graphics/shader/recompiler/backend/spirv/spirvEmitterAlu.cpp b/src/graphics/shader/recompiler/backend/spirv/spirvEmitterAlu.cpp index bdf1d23c1..2ad9cdb14 100644 --- a/src/graphics/shader/recompiler/backend/spirv/spirvEmitterAlu.cpp +++ b/src/graphics/shader/recompiler/backend/spirv/spirvEmitterAlu.cpp @@ -39,23 +39,22 @@ Pair ExtractPair(EmitterState& state, uint32_t value) { uint32_t MakePair(EmitterState& state, uint32_t low, uint32_t high) { const auto result = state.builder.AllocateId(); - state.builder.AddFunction({OpCompositeConstruct, TypeU32Pair(state), result, low, high}); + state.builder.AddFunction({OpCompositeConstruct, TypeU64(state), result, low, high}); return result; } -uint32_t Compare64(EmitterState& state, uint32_t lhs_value, uint32_t rhs_value, - uint32_t high_compare, uint32_t low_compare, bool equal_compare = false, - bool not_equal = false) { - const auto lhs = ExtractPair(state, lhs_value); - const auto rhs = ExtractPair(state, rhs_value); - const auto high_equal = NewBinary(state, OpIEqual, TypeBool(state), lhs.high, rhs.high); - const auto low_equal = NewBinary(state, OpIEqual, TypeBool(state), lhs.low, rhs.low); - if (equal_compare) { - return NewBinary( - state, not_equal ? OpLogicalOr : OpLogicalAnd, TypeBool(state), - not_equal ? NewUnary(state, OpLogicalNot, TypeBool(state), high_equal) : high_equal, - not_equal ? NewUnary(state, OpLogicalNot, TypeBool(state), low_equal) : low_equal); - } +uint32_t CompareEqual64(EmitterState& state, uint32_t lhs_value, uint32_t rhs_value, + bool not_equal) { + const auto compare = NewBinary(state, not_equal ? OpINotEqual : OpIEqual, + TypeBoolVector(state, 2), lhs_value, rhs_value); + return NewUnary(state, not_equal ? OpAny : OpAll, TypeBool(state), compare); +} + +uint32_t CompareOrdered64(EmitterState& state, uint32_t lhs_value, uint32_t rhs_value, + uint32_t high_compare, uint32_t low_compare) { + const auto lhs = ExtractPair(state, lhs_value); + const auto rhs = ExtractPair(state, rhs_value); + const auto high_equal = NewBinary(state, OpIEqual, TypeBool(state), lhs.high, rhs.high); const auto high_result = NewBinary(state, high_compare, TypeBool(state), lhs.high, rhs.high); const auto low_result = NewBinary(state, low_compare, TypeBool(state), lhs.low, rhs.low); const auto low_path = NewBinary(state, OpLogicalAnd, TypeBool(state), high_equal, low_result); @@ -63,12 +62,17 @@ uint32_t Compare64(EmitterState& state, uint32_t lhs_value, uint32_t rhs_value, } uint32_t EmitAdd64(EmitterState& state, uint32_t lhs_value, uint32_t rhs_value) { - const auto lhs = ExtractPair(state, lhs_value); - const auto rhs = ExtractPair(state, rhs_value); - const auto carry = EmitAddCarryValues(state, lhs.low, rhs.low, ConstantU32(state, 0)); + const auto lhs = ExtractPair(state, lhs_value); + const auto rhs = ExtractPair(state, rhs_value); + const auto low_pair = state.builder.AllocateId(); + const auto low = state.builder.AllocateId(); + const auto carry = state.builder.AllocateId(); + state.builder.AddFunction({OpIAddCarry, TypeU32Pair(state), low_pair, lhs.low, rhs.low}); + state.builder.AddFunction({OpCompositeExtract, TypeU32(state), low, low_pair, 0}); + state.builder.AddFunction({OpCompositeExtract, TypeU32(state), carry, low_pair, 1}); const auto high0 = NewBinary(state, OpIAdd, TypeU32(state), lhs.high, rhs.high); - const auto high = NewBinary(state, OpIAdd, TypeU32(state), high0, carry.carry); - return MakePair(state, carry.sum, high); + const auto high = NewBinary(state, OpIAdd, TypeU32(state), high0, carry); + return MakePair(state, low, high); } uint32_t EmitSub64(EmitterState& state, uint32_t lhs_value, uint32_t rhs_value) { @@ -112,14 +116,6 @@ uint32_t EmitMul64(EmitterState& state, uint32_t lhs_value, uint32_t rhs_value) NewBinary(state, OpIAdd, TypeU32(state), high0, high1), high2)); } -uint32_t EmitBitwise64(EmitterState& state, uint32_t opcode, uint32_t lhs_value, - uint32_t rhs_value) { - const auto lhs = ExtractPair(state, lhs_value); - const auto rhs = ExtractPair(state, rhs_value); - return MakePair(state, NewBinary(state, opcode, TypeU32(state), lhs.low, rhs.low), - NewBinary(state, opcode, TypeU32(state), lhs.high, rhs.high)); -} - uint32_t EmitShift64(EmitterState& state, uint32_t opcode, uint32_t value, uint32_t shift) { const auto pair = ExtractPair(state, value); uint32_t low = 0; @@ -129,38 +125,80 @@ uint32_t EmitShift64(EmitterState& state, uint32_t opcode, uint32_t value, uint3 } else if (opcode == OpShiftRightLogical) { EmitShiftRightLogicalU64Values(state, pair.low, pair.high, shift, low, high); } else { - const auto logical = EmitShift64(state, OpShiftRightLogical, value, shift); - const auto logical_pair = ExtractPair(state, logical); - const auto sign = - NewBinary(state, OpSLessThan, TypeBool(state), pair.high, ConstantU32(state, 0)); - const auto inverse = - NewBinary(state, OpISub, TypeU32(state), ConstantU32(state, 64), shift); - const auto fill = NewBinary(state, OpShiftLeftLogical, TypeU32(state), - ConstantU32(state, 0xffffffffu), inverse); - const auto high_shift = - NewBinary(state, OpISub, TypeU32(state), shift, ConstantU32(state, 32)); - const auto high_fill = NewSelect( - state, TypeU32(state), - NewBinary(state, OpUGreaterThanEqual, TypeBool(state), shift, ConstantU32(state, 32)), - ConstantU32(state, 0xffffffffu), fill); - const auto fill_low = - NewSelect(state, TypeU32(state), sign, - NewSelect(state, TypeU32(state), - NewBinary(state, OpUGreaterThanEqual, TypeBool(state), shift, - ConstantU32(state, 32)), - NewBinary(state, OpShiftLeftLogical, TypeU32(state), - ConstantU32(state, 0xffffffffu), high_shift), - ConstantU32(state, 0)), + const auto amount = + NewBinary(state, OpBitwiseAnd, TypeU32(state), shift, ConstantU32(state, 63)); + const auto word_shift = + NewBinary(state, OpBitwiseAnd, TypeU32(state), amount, ConstantU32(state, 31)); + const auto at_least_32 = + NewBinary(state, OpUGreaterThanEqual, TypeBool(state), amount, ConstantU32(state, 32)); + const auto nonzero = + NewBinary(state, OpINotEqual, TypeBool(state), amount, ConstantU32(state, 0)); + const auto high_shifted = + NewBinary(state, OpShiftRightArithmetic, TypeU32(state), pair.high, word_shift); + const auto low_shifted = + NewBinary(state, OpShiftRightLogical, TypeU32(state), pair.low, word_shift); + const auto carry_count = + NewBinary(state, OpBitwiseAnd, TypeU32(state), + NewBinary(state, OpISub, TypeU32(state), ConstantU32(state, 32), word_shift), + ConstantU32(state, 31)); + const auto carry = + NewSelect(state, TypeU32(state), nonzero, + NewBinary(state, OpShiftLeftLogical, TypeU32(state), pair.high, carry_count), ConstantU32(state, 0)); - const auto fill_high = - NewSelect(state, TypeU32(state), sign, high_fill, ConstantU32(state, 0)); - return MakePair( - state, NewBinary(state, OpBitwiseOr, TypeU32(state), logical_pair.low, fill_low), - NewBinary(state, OpBitwiseOr, TypeU32(state), logical_pair.high, fill_high)); + const auto low_below_32 = NewBinary(state, OpBitwiseOr, TypeU32(state), low_shifted, carry); + const auto negative = + NewBinary(state, OpSLessThan, TypeBool(state), pair.high, ConstantU32(state, 0)); + const auto sign_fill = NewSelect(state, TypeU32(state), negative, + ConstantU32(state, 0xffffffffu), ConstantU32(state, 0)); + return MakePair(state, + NewSelect(state, TypeU32(state), at_least_32, high_shifted, low_below_32), + NewSelect(state, TypeU32(state), at_least_32, sign_fill, high_shifted)); } return MakePair(state, low, high); } +uint32_t EmitConstantShift64(EmitterState& state, uint32_t opcode, uint32_t value, uint32_t shift) { + shift &= 63u; + if (shift == 0u) { + return value; + } + const auto pair = ExtractPair(state, value); + if (opcode == OpShiftLeftLogical) { + if (shift < 32u) { + const auto low = NewBinary(state, OpShiftLeftLogical, TypeU32(state), pair.low, + ConstantU32(state, shift)); + const auto high = NewBinary(state, OpBitwiseOr, TypeU32(state), + NewBinary(state, OpShiftLeftLogical, TypeU32(state), + pair.high, ConstantU32(state, shift)), + NewBinary(state, OpShiftRightLogical, TypeU32(state), + pair.low, ConstantU32(state, 32u - shift))); + return MakePair(state, low, high); + } + return MakePair(state, ConstantU32(state, 0), + shift == 32u ? pair.low + : NewBinary(state, OpShiftLeftLogical, TypeU32(state), + pair.low, ConstantU32(state, shift - 32u))); + } + if (shift < 32u) { + const auto low = NewBinary(state, OpBitwiseOr, TypeU32(state), + NewBinary(state, OpShiftRightLogical, TypeU32(state), pair.low, + ConstantU32(state, shift)), + NewBinary(state, OpShiftLeftLogical, TypeU32(state), pair.high, + ConstantU32(state, 32u - shift))); + const auto high = + NewBinary(state, opcode, TypeU32(state), pair.high, ConstantU32(state, shift)); + return MakePair(state, low, high); + } + const auto high = opcode == OpShiftRightArithmetic + ? NewBinary(state, OpShiftRightArithmetic, TypeU32(state), pair.high, + ConstantU32(state, 31u)) + : ConstantU32(state, 0); + const auto low = shift == 32u ? pair.high + : NewBinary(state, opcode, TypeU32(state), pair.high, + ConstantU32(state, shift - 32u)); + return MakePair(state, low, high); +} + uint32_t EmitFindMsb64(EmitterState& state, uint32_t value) { const auto pair = ExtractPair(state, value); const auto high_i = state.builder.AllocateId(); @@ -285,8 +323,6 @@ bool EmitValueAlu(ValueEmitContext& ctx, const IR::Inst& inst) { case IR::ValueOpcode::BitCastF16U16: case IR::ValueOpcode::ConvertU32U16: case IR::ValueOpcode::ConvertU32U8: - case IR::ValueOpcode::PackUint2x32: - case IR::ValueOpcode::UnpackUint2x32: ctx.Define(inst, ctx.Arg(inst, 0)); return true; case IR::ValueOpcode::BitCastU32F32: return unary(OpBitcast, IR::Type::U32); case IR::ValueOpcode::BitCastF32U32: return unary(OpBitcast, IR::Type::F32); case IR::ValueOpcode::ConvertU16U32: @@ -319,6 +355,10 @@ bool EmitValueAlu(ValueEmitContext& ctx, const IR::Inst& inst) { return true; } case IR::ValueOpcode::ConvertF32U32: return unary(OpConvertUToF, IR::Type::F32); + case IR::ValueOpcode::CompositeConstructU64: + ctx.Emit(inst, OpCompositeConstruct, IR::Type::U64, + {ctx.Arg(inst, 0), ctx.Arg(inst, 1)}); + return true; case IR::ValueOpcode::CompositeConstructU32x2: ctx.Emit(inst, OpCompositeConstruct, IR::Type::U32x2, {ctx.Arg(inst, 0), ctx.Arg(inst, 1)}); @@ -335,6 +375,7 @@ bool EmitValueAlu(ValueEmitContext& ctx, const IR::Inst& inst) { ctx.Emit(inst, OpCompositeConstruct, IR::Type::U32x4, {ctx.Arg(inst, 0), ctx.Arg(inst, 1), ctx.Arg(inst, 2), ctx.Arg(inst, 3)}); return true; + case IR::ValueOpcode::CompositeExtractU64: case IR::ValueOpcode::CompositeExtractU32x2: case IR::ValueOpcode::CompositeExtractU32x3: case IR::ValueOpcode::CompositeExtractU32x4: @@ -429,15 +470,25 @@ bool EmitValueAlu(ValueEmitContext& ctx, const IR::Inst& inst) { case IR::ValueOpcode::ShiftRightArithmetic32: return binary(OpShiftRightArithmetic, IR::Type::U32); case IR::ValueOpcode::ShiftLeftLogical64: - ctx.Define(inst, - EmitShift64(state, OpShiftLeftLogical, ctx.Arg(inst, 0), ctx.Arg(inst, 1))); + ctx.Define(inst, inst.Arg(1).Resolve().IsImmediate() + ? EmitConstantShift64(state, OpShiftLeftLogical, ctx.Arg(inst, 0), + inst.Arg(1).Resolve().U32()) + : EmitShift64(state, OpShiftLeftLogical, ctx.Arg(inst, 0), + ctx.Arg(inst, 1))); return true; case IR::ValueOpcode::ShiftRightLogical64: - ctx.Define(inst, - EmitShift64(state, OpShiftRightLogical, ctx.Arg(inst, 0), ctx.Arg(inst, 1))); + ctx.Define(inst, inst.Arg(1).Resolve().IsImmediate() + ? EmitConstantShift64(state, OpShiftRightLogical, ctx.Arg(inst, 0), + inst.Arg(1).Resolve().U32()) + : EmitShift64(state, OpShiftRightLogical, ctx.Arg(inst, 0), + ctx.Arg(inst, 1))); return true; case IR::ValueOpcode::ShiftRightArithmetic64: - ctx.Define(inst, EmitShift64(state, OpShiftRightArithmetic, ctx.Arg(inst, 0), + ctx.Define(inst, + inst.Arg(1).Resolve().IsImmediate() + ? EmitConstantShift64(state, OpShiftRightArithmetic, ctx.Arg(inst, 0), + inst.Arg(1).Resolve().U32()) + : EmitShift64(state, OpShiftRightArithmetic, ctx.Arg(inst, 0), ctx.Arg(inst, 1))); return true; case IR::ValueOpcode::BitwiseAnd32: return binary(OpBitwiseAnd, IR::Type::U32); @@ -445,19 +496,15 @@ bool EmitValueAlu(ValueEmitContext& ctx, const IR::Inst& inst) { case IR::ValueOpcode::BitwiseXor32: return binary(OpBitwiseXor, IR::Type::U32); case IR::ValueOpcode::BitwiseNot32: return unary(OpNot, IR::Type::U32); case IR::ValueOpcode::BitwiseAnd64: - ctx.Define(inst, - EmitBitwise64(state, OpBitwiseAnd, ctx.Arg(inst, 0), ctx.Arg(inst, 1))); - return true; - case IR::ValueOpcode::BitwiseOr64: - ctx.Define(inst, EmitBitwise64(state, OpBitwiseOr, ctx.Arg(inst, 0), ctx.Arg(inst, 1))); + ctx.Define(inst, NewBinary(state, OpBitwiseAnd, TypeU64(state), ctx.Arg(inst, 0), + ctx.Arg(inst, 1))); return true; case IR::ValueOpcode::BitReverse32: return unary(OpBitReverse, IR::Type::U32); case IR::ValueOpcode::BitCount32: return unary(OpBitCount, IR::Type::U32); case IR::ValueOpcode::BitCount64: { - const auto pair = ExtractPair(state, ctx.Arg(inst, 0)); - ctx.Define(inst, NewBinary(state, OpIAdd, TypeU32(state), - NewUnary(state, OpBitCount, TypeU32(state), pair.low), - NewUnary(state, OpBitCount, TypeU32(state), pair.high))); + const auto pair = + ExtractPair(state, NewUnary(state, OpBitCount, TypeU64(state), ctx.Arg(inst, 0))); + ctx.Define(inst, NewBinary(state, OpIAdd, TypeU32(state), pair.low, pair.high)); return true; } case IR::ValueOpcode::FindILsb32: { @@ -527,53 +574,43 @@ bool EmitValueAlu(ValueEmitContext& ctx, const IR::Inst& inst) { case IR::ValueOpcode::SGreaterThanEqual32: return binary(OpSGreaterThanEqual, IR::Type::U1); case IR::ValueOpcode::UGreaterThanEqual32: return binary(OpUGreaterThanEqual, IR::Type::U1); case IR::ValueOpcode::IEqual64: - ctx.Define(inst, - Compare64(state, ctx.Arg(inst, 0), ctx.Arg(inst, 1), 0, 0, true, false)); + ctx.Define(inst, CompareEqual64(state, ctx.Arg(inst, 0), ctx.Arg(inst, 1), false)); return true; case IR::ValueOpcode::INotEqual64: - ctx.Define(inst, - Compare64(state, ctx.Arg(inst, 0), ctx.Arg(inst, 1), 0, 0, true, true)); + ctx.Define(inst, CompareEqual64(state, ctx.Arg(inst, 0), ctx.Arg(inst, 1), true)); return true; case IR::ValueOpcode::ULessThan64: - ctx.Define(inst, Compare64(state, ctx.Arg(inst, 0), ctx.Arg(inst, 1), OpULessThan, - OpULessThan)); + ctx.Define(inst, CompareOrdered64(state, ctx.Arg(inst, 0), ctx.Arg(inst, 1), + OpULessThan, OpULessThan)); return true; case IR::ValueOpcode::SLessThan64: - ctx.Define(inst, Compare64(state, ctx.Arg(inst, 0), ctx.Arg(inst, 1), OpSLessThan, - OpULessThan)); + ctx.Define(inst, CompareOrdered64(state, ctx.Arg(inst, 0), ctx.Arg(inst, 1), + OpSLessThan, OpULessThan)); return true; case IR::ValueOpcode::UGreaterThan64: - ctx.Define(inst, Compare64(state, ctx.Arg(inst, 0), ctx.Arg(inst, 1), OpUGreaterThan, - OpUGreaterThan)); + ctx.Define(inst, CompareOrdered64(state, ctx.Arg(inst, 0), ctx.Arg(inst, 1), + OpUGreaterThan, OpUGreaterThan)); return true; case IR::ValueOpcode::SGreaterThan64: - ctx.Define(inst, Compare64(state, ctx.Arg(inst, 0), ctx.Arg(inst, 1), OpSGreaterThan, - OpUGreaterThan)); + ctx.Define(inst, CompareOrdered64(state, ctx.Arg(inst, 0), ctx.Arg(inst, 1), + OpSGreaterThan, OpUGreaterThan)); return true; - case IR::ValueOpcode::ULessThanEqual64: { - const auto greater = Compare64(state, ctx.Arg(inst, 0), ctx.Arg(inst, 1), - OpUGreaterThan, OpUGreaterThan); - ctx.Define(inst, NewUnary(state, OpLogicalNot, TypeBool(state), greater)); + case IR::ValueOpcode::ULessThanEqual64: + ctx.Define(inst, CompareOrdered64(state, ctx.Arg(inst, 0), ctx.Arg(inst, 1), + OpULessThan, OpULessThanEqual)); return true; - } - case IR::ValueOpcode::SLessThanEqual64: { - const auto greater = Compare64(state, ctx.Arg(inst, 0), ctx.Arg(inst, 1), - OpSGreaterThan, OpUGreaterThan); - ctx.Define(inst, NewUnary(state, OpLogicalNot, TypeBool(state), greater)); + case IR::ValueOpcode::SLessThanEqual64: + ctx.Define(inst, CompareOrdered64(state, ctx.Arg(inst, 0), ctx.Arg(inst, 1), + OpSLessThan, OpULessThanEqual)); return true; - } - case IR::ValueOpcode::UGreaterThanEqual64: { - const auto less = - Compare64(state, ctx.Arg(inst, 0), ctx.Arg(inst, 1), OpULessThan, OpULessThan); - ctx.Define(inst, NewUnary(state, OpLogicalNot, TypeBool(state), less)); + case IR::ValueOpcode::UGreaterThanEqual64: + ctx.Define(inst, CompareOrdered64(state, ctx.Arg(inst, 0), ctx.Arg(inst, 1), + OpUGreaterThan, OpUGreaterThanEqual)); return true; - } - case IR::ValueOpcode::SGreaterThanEqual64: { - const auto less = - Compare64(state, ctx.Arg(inst, 0), ctx.Arg(inst, 1), OpSLessThan, OpULessThan); - ctx.Define(inst, NewUnary(state, OpLogicalNot, TypeBool(state), less)); + case IR::ValueOpcode::SGreaterThanEqual64: + ctx.Define(inst, CompareOrdered64(state, ctx.Arg(inst, 0), ctx.Arg(inst, 1), + OpSGreaterThan, OpUGreaterThanEqual)); return true; - } case IR::ValueOpcode::LogicalOr: return binary(OpLogicalOr, IR::Type::U1); case IR::ValueOpcode::LogicalAnd: return binary(OpLogicalAnd, IR::Type::U1); case IR::ValueOpcode::LogicalXor: return binary(OpLogicalNotEqual, IR::Type::U1); diff --git a/src/graphics/shader/recompiler/backend/spirv/spirvEmitterAluHelpers.cpp b/src/graphics/shader/recompiler/backend/spirv/spirvEmitterAluHelpers.cpp index 920b42328..69997470f 100644 --- a/src/graphics/shader/recompiler/backend/spirv/spirvEmitterAluHelpers.cpp +++ b/src/graphics/shader/recompiler/backend/spirv/spirvEmitterAluHelpers.cpp @@ -83,25 +83,6 @@ uint32_t EmitAndConstant(EmitterState& state, uint32_t value, uint32_t mask) { return ret; } -AddCarryResult EmitAddCarryValues(EmitterState& state, uint32_t lhs, uint32_t rhs, - uint32_t carry_in) { - const auto pair0 = state.builder.AllocateId(); - const auto sum0 = state.builder.AllocateId(); - const auto carry0 = state.builder.AllocateId(); - const auto pair1 = state.builder.AllocateId(); - const auto sum1 = state.builder.AllocateId(); - const auto carry1 = state.builder.AllocateId(); - const auto carry = state.builder.AllocateId(); - state.builder.AddFunction({OpIAddCarry, TypeU32Pair(state), pair0, lhs, rhs}); - state.builder.AddFunction({OpCompositeExtract, TypeU32(state), sum0, pair0, 0}); - state.builder.AddFunction({OpCompositeExtract, TypeU32(state), carry0, pair0, 1}); - state.builder.AddFunction({OpIAddCarry, TypeU32Pair(state), pair1, sum0, carry_in}); - state.builder.AddFunction({OpCompositeExtract, TypeU32(state), sum1, pair1, 0}); - state.builder.AddFunction({OpCompositeExtract, TypeU32(state), carry1, pair1, 1}); - state.builder.AddFunction({OpBitwiseOr, TypeU32(state), carry, carry0, carry1}); - return {sum1, carry}; -} - uint32_t EmitShiftRightConstant(EmitterState& state, uint32_t value, uint32_t shift) { const auto ret = state.builder.AllocateId(); state.builder.AddFunction( diff --git a/src/graphics/shader/recompiler/backend/spirv/spirvEmitterFlow.cpp b/src/graphics/shader/recompiler/backend/spirv/spirvEmitterFlow.cpp index d68fae67c..49ccdb79d 100644 --- a/src/graphics/shader/recompiler/backend/spirv/spirvEmitterFlow.cpp +++ b/src/graphics/shader/recompiler/backend/spirv/spirvEmitterFlow.cpp @@ -39,13 +39,22 @@ uint32_t EmitWqmWordU32(EmitterState& state, uint32_t value) { } uint32_t EmitWqmU64(EmitterState& state, uint32_t value) { - const auto low = state.builder.AllocateId(); - const auto high = state.builder.AllocateId(); - state.builder.AddFunction({OpCompositeExtract, TypeU32(state), low, value, 0}); - state.builder.AddFunction({OpCompositeExtract, TypeU32(state), high, value, 1}); - const auto result = state.builder.AllocateId(); - state.builder.AddFunction({OpCompositeConstruct, TypeU32Pair(state), result, - EmitWqmWordU32(state, low), EmitWqmWordU32(state, high)}); + const auto shifted_one = state.builder.AllocateId(); + const auto merged_one = state.builder.AllocateId(); + const auto shifted_two = state.builder.AllocateId(); + const auto merged_two = state.builder.AllocateId(); + const auto quad_bits = state.builder.AllocateId(); + const auto result = state.builder.AllocateId(); + state.builder.AddFunction({OpShiftRightLogical, TypeU64(state), shifted_one, value, + ConstantU64(state, 0x0000000100000001ull)}); + state.builder.AddFunction({OpBitwiseOr, TypeU64(state), merged_one, value, shifted_one}); + state.builder.AddFunction({OpShiftRightLogical, TypeU64(state), shifted_two, merged_one, + ConstantU64(state, 0x0000000200000002ull)}); + state.builder.AddFunction({OpBitwiseOr, TypeU64(state), merged_two, merged_one, shifted_two}); + state.builder.AddFunction({OpBitwiseAnd, TypeU64(state), quad_bits, merged_two, + ConstantU64(state, 0x1111111111111111ull)}); + state.builder.AddFunction( + {OpIMul, TypeU64(state), result, quad_bits, ConstantU64(state, 0x0000000f0000000full)}); return result; } diff --git a/src/graphics/shader/recompiler/backend/spirv/spirvEmitterInternal.h b/src/graphics/shader/recompiler/backend/spirv/spirvEmitterInternal.h index a04347aec..da07a17b6 100644 --- a/src/graphics/shader/recompiler/backend/spirv/spirvEmitterInternal.h +++ b/src/graphics/shader/recompiler/backend/spirv/spirvEmitterInternal.h @@ -182,6 +182,8 @@ enum : uint32_t { OpIAddCarry = 149, OpUMulExtended = 151, OpSMulExtended = 152, + OpAny = 154, + OpAll = 155, OpLogicalNotEqual = 165, OpLogicalOr = 166, OpLogicalAnd = 167, @@ -337,7 +339,9 @@ struct EmitterState { uint32_t TypeVoid(EmitterState& state); uint32_t TypeBool(EmitterState& state); +uint32_t TypeBoolVector(EmitterState& state, uint32_t components); uint32_t TypeU32(EmitterState& state); +uint32_t TypeU64(EmitterState& state); uint32_t TypeU32Pair(EmitterState& state); uint32_t TypeI32(EmitterState& state); uint32_t TypeI32Pair(EmitterState& state); @@ -505,11 +509,6 @@ constexpr uint32_t ImageSpirvMultisampled(ImageViewKind view) { return view == ImageViewKind::Dim2DMsaa || view == ImageViewKind::Dim2DMsaaArray ? 1u : 0u; } -struct AddCarryResult { - uint32_t sum = 0; - uint32_t carry = 0; -}; - struct F32Class { uint32_t bits = 0; uint32_t nan = 0; @@ -730,8 +729,6 @@ uint32_t EmitDsSwizzleTargetLane(EmitterState& state, uint32_t subid, uint32_t c uint32_t EmitSelectValueU32(EmitterState& state, uint32_t cond, uint32_t true_value, uint32_t false_value); -uint32_t EmitAndConstant(EmitterState& state, uint32_t value, uint32_t mask); - void EmitShiftLeftLogicalU64Values(EmitterState& state, uint32_t low, uint32_t high, uint32_t shift, uint32_t& out_low, uint32_t& out_high); @@ -740,9 +737,6 @@ void EmitShiftRightLogicalU64Values(EmitterState& state, uint32_t low, uint32_t uint32_t EmitAndConstant(EmitterState& state, uint32_t value, uint32_t mask); -AddCarryResult EmitAddCarryValues(EmitterState& state, uint32_t lhs, uint32_t rhs, - uint32_t carry_in); - uint32_t EmitShiftRightConstant(EmitterState& state, uint32_t value, uint32_t shift); uint32_t EmitOrU32(EmitterState& state, uint32_t lhs, uint32_t rhs); diff --git a/src/graphics/shader/recompiler/backend/spirv/spirvEmitterModule.cpp b/src/graphics/shader/recompiler/backend/spirv/spirvEmitterModule.cpp index 3f5b0dc89..88c91ba33 100644 --- a/src/graphics/shader/recompiler/backend/spirv/spirvEmitterModule.cpp +++ b/src/graphics/shader/recompiler/backend/spirv/spirvEmitterModule.cpp @@ -16,10 +16,18 @@ uint32_t TypeBool(EmitterState& state) { return state.builder.Type(OpTypeBool); } +uint32_t TypeBoolVector(EmitterState& state, uint32_t components) { + return state.builder.Type(OpTypeVector, {TypeBool(state), components}); +} + uint32_t TypeU32(EmitterState& state) { return state.builder.Type(OpTypeInt, {32, 0}); } +uint32_t TypeU64(EmitterState& state) { + return TypeU32Vector(state, 2); +} + uint32_t TypeU32Pair(EmitterState& state) { const auto element = TypeU32(state); return state.builder.Type(OpTypeStruct, {element, element}); @@ -241,7 +249,7 @@ uint32_t ConstantBool(EmitterState& state, bool value) { } uint32_t ConstantU64(EmitterState& state, uint64_t value) { - return state.builder.Constant(OpConstantComposite, TypeU32Pair(state), + return state.builder.Constant(OpConstantComposite, TypeU64(state), {ConstantU32(state, static_cast(value)), ConstantU32(state, static_cast(value >> 32u))}); } diff --git a/src/graphics/shader/recompiler/backend/spirv/spirvEmitterProgram.cpp b/src/graphics/shader/recompiler/backend/spirv/spirvEmitterProgram.cpp index 0b5f38b25..0c72be994 100644 --- a/src/graphics/shader/recompiler/backend/spirv/spirvEmitterProgram.cpp +++ b/src/graphics/shader/recompiler/backend/spirv/spirvEmitterProgram.cpp @@ -44,7 +44,7 @@ uint32_t SpillPointerType(EmitterState& state, IR::Type type) { case IR::Type::U32: case IR::Type::F16: return TypePointer(state, StorageClassFunction, TypeU32(state)); case IR::Type::F32: return TypePointer(state, StorageClassFunction, TypeF32(state)); - case IR::Type::U64: + case IR::Type::U64: return TypePointer(state, StorageClassFunction, TypeU64(state)); case IR::Type::U32x2: return TypePointer(state, StorageClassFunction, TypeU32Pair(state)); case IR::Type::U32x3: return TypePointer(state, StorageClassFunction, TypeU32Vector(state, 3)); @@ -404,7 +404,7 @@ uint32_t ValueEmitContext::TypeId(IR::Type type) const { case IR::Type::U16: case IR::Type::U32: case IR::Type::F16: return TypeU32(state); - case IR::Type::U64: + case IR::Type::U64: return TypeU64(state); case IR::Type::U32x2: return TypeU32Pair(state); case IR::Type::F32: return TypeF32(state); case IR::Type::U32x3: return TypeU32Vector(state, 3); diff --git a/src/graphics/shader/recompiler/frontend/translate/Control.cpp b/src/graphics/shader/recompiler/frontend/translate/Control.cpp index 152a05c13..11c04533b 100644 --- a/src/graphics/shader/recompiler/frontend/translate/Control.cpp +++ b/src/graphics/shader/recompiler/frontend/translate/Control.cpp @@ -27,7 +27,7 @@ bool Translator::TranslateStateOperation(const IR::Instruction& inst) { const auto result_high = inst.op == IR::Opcode::SaveexecB64 ? apply(old_high, src_high) : old_high; if (inst.op == IR::Opcode::SaveexecB64) { - WriteOperand(inst.dst, ir.PackUint2x32(old_low, old_high)); + WriteU32Pair(inst.dst, {old_low, old_high}); } else { WriteOperand(inst.dst, old_low); } @@ -157,12 +157,11 @@ bool Translator::TranslateMove(const IR::Instruction& inst) { WriteOperand(inst.dst, ReadOperand(inst.src[0], IR::Type::F32)); return true; case IR::Opcode::MoveU64: { - const bool scalar_pair = inst.dst.kind == IR::OperandKind::Register && - inst.src[0].kind == IR::OperandKind::Register && - inst.dst.reg.file == IR::RegisterFile::Scalar && - inst.src[0].reg.file == IR::RegisterFile::Scalar; - WriteOperand(inst.dst, ReadOperand(inst.src[0], IR::Type::U64)); - if (scalar_pair) { + WriteU32Pair(inst.dst, ReadU32Pair(inst.src[0])); + if (inst.dst.kind == IR::OperandKind::Register && + inst.src[0].kind == IR::OperandKind::Register && + inst.dst.reg.file == IR::RegisterFile::Scalar && + inst.src[0].reg.file == IR::RegisterFile::Scalar) { ir.SetThreadBitScalarReg( static_cast(inst.dst.reg.index), ir.GetThreadBitScalarReg(static_cast(inst.src[0].reg.index))); @@ -171,8 +170,8 @@ bool Translator::TranslateMove(const IR::Instruction& inst) { } case IR::Opcode::WqmB64: { if (!current_per_invocation_masks) { - const auto result = IR::U64(ir.Emit( - IR::ValueOpcode::WqmU64, {ReadOperand(inst.src[0], IR::Type::U64)})); + const auto result = IR::U64( + ir.Emit(IR::ValueOpcode::WqmU64, {ReadOperand(inst.src[0], IR::Type::U64)})); WriteOperand(inst.dst, result); ir.SetScc(IR::U1( ir.Emit(IR::ValueOpcode::INotEqual64, {result, IR::Value(uint64_t {0})}))); diff --git a/src/graphics/shader/recompiler/frontend/translate/Dispatch.cpp b/src/graphics/shader/recompiler/frontend/translate/Dispatch.cpp index 31acabd86..6caf7544a 100644 --- a/src/graphics/shader/recompiler/frontend/translate/Dispatch.cpp +++ b/src/graphics/shader/recompiler/frontend/translate/Dispatch.cpp @@ -6,7 +6,8 @@ bool Translator::TranslateU64MaskOperation(const IR::Instruction& inst) { if (current_per_invocation_masks) { return TranslatePerInvocationU64Mask(inst); } - return inst.op == IR::Opcode::BitwiseAndU64 || inst.op == IR::Opcode::BitwiseOrU64 + return inst.op == IR::Opcode::BitwiseAndU64 || inst.op == IR::Opcode::BitwiseOrU64 || + inst.op == IR::Opcode::BitwiseXorU64 || inst.op == IR::Opcode::BitwiseNotU64 ? TranslateSimpleInteger(inst) : TranslateComposedInteger(inst); } diff --git a/src/graphics/shader/recompiler/frontend/translate/Integer.cpp b/src/graphics/shader/recompiler/frontend/translate/Integer.cpp index 8f84792d0..4298db922 100644 --- a/src/graphics/shader/recompiler/frontend/translate/Integer.cpp +++ b/src/graphics/shader/recompiler/frontend/translate/Integer.cpp @@ -157,6 +157,31 @@ bool Translator::TranslatePerInvocationU64Mask(const IR::Instruction& inst) { } bool Translator::TranslateSimpleInteger(const IR::Instruction& inst) { + if (inst.op == IR::Opcode::BitwiseAndU64 || inst.op == IR::Opcode::BitwiseOrU64 || + inst.op == IR::Opcode::BitwiseXorU64 || inst.op == IR::Opcode::BitwiseNotU64) { + const auto lhs = ReadU32Pair(inst.src[0]); + std::array result; + if (inst.op == IR::Opcode::BitwiseNotU64) { + result = {ir.BitwiseNot(lhs[0]), ir.BitwiseNot(lhs[1])}; + } else { + const auto rhs = ReadU32Pair(inst.src[1]); + for (uint32_t component = 0; component < 2; component++) { + switch (inst.op) { + case IR::Opcode::BitwiseAndU64: + result[component] = ir.BitwiseAnd(lhs[component], rhs[component]); + break; + case IR::Opcode::BitwiseOrU64: + result[component] = ir.BitwiseOr(lhs[component], rhs[component]); + break; + default: + result[component] = ir.BitwiseXor(lhs[component], rhs[component]); + break; + } + } + } + WriteU32Pair(inst.dst, result); + return true; + } IR::ValueOpcode opcode {}; IR::Type type = IR::Type::U32; switch (inst.op) { @@ -187,14 +212,6 @@ bool Translator::TranslateSimpleInteger(const IR::Instruction& inst) { type = IR::Type::U64; break; case IR::Opcode::FindLsbU32: opcode = IR::ValueOpcode::FindILsb32; break; - case IR::Opcode::BitwiseAndU64: - opcode = IR::ValueOpcode::BitwiseAnd64; - type = IR::Type::U64; - break; - case IR::Opcode::BitwiseOrU64: - opcode = IR::ValueOpcode::BitwiseOr64; - type = IR::Type::U64; - break; case IR::Opcode::ShiftLeftLogicalU32: opcode = IR::ValueOpcode::ShiftLeftLogical32; break; case IR::Opcode::ShiftRightLogicalU32: opcode = IR::ValueOpcode::ShiftRightLogical32; break; case IR::Opcode::ShiftRightArithmeticI32: @@ -219,10 +236,6 @@ bool Translator::TranslateSimpleInteger(const IR::Instruction& inst) { opcode == IR::ValueOpcode::ShiftRightLogical32 || opcode == IR::ValueOpcode::ShiftRightArithmetic32)) { args[index] = ir.BitwiseAnd(IR::U32(args[index]), IR::U32(IR::Value(31u))); - } else if (index == 1u && (opcode == IR::ValueOpcode::ShiftLeftLogical64 || - opcode == IR::ValueOpcode::ShiftRightLogical64 || - opcode == IR::ValueOpcode::ShiftRightArithmetic64)) { - args[index] = ir.BitwiseAnd(IR::U32(args[index]), IR::U32(IR::Value(63u))); } } IR::Value result; @@ -233,32 +246,46 @@ bool Translator::TranslateSimpleInteger(const IR::Instruction& inst) { default: EXIT("invalid simple integer source count: %u", inst.src_count); } WriteOperand(inst.dst, result); - if (type == IR::Type::U64 && inst.dst.kind == IR::OperandKind::Register && - inst.dst.reg.file == IR::RegisterFile::Scalar) { - IR::U1 mask; - if (inst.op == IR::Opcode::BitwiseAndU64) { - mask = ir.LogicalAnd(ReadMask(inst.src[0]), ReadMask(inst.src[1])); - } else if (inst.op == IR::Opcode::BitwiseOrU64) { - mask = ir.LogicalOr(ReadMask(inst.src[0]), ReadMask(inst.src[1])); - } - if (!mask.IsEmpty()) { - ir.SetThreadBitScalarReg(static_cast(inst.dst.reg.index), mask); - } - } return true; } bool Translator::TranslateComposedInteger(const IR::Instruction& inst) { + if (inst.op == IR::Opcode::BitwiseAndNotU64 || inst.op == IR::Opcode::BitwiseOrNotU64 || + inst.op == IR::Opcode::BitwiseNandU64 || inst.op == IR::Opcode::BitwiseNorU64 || + inst.op == IR::Opcode::BitwiseXnorU64) { + const auto lhs = ReadU32Pair(inst.src[0]); + const auto rhs = ReadU32Pair(inst.src[1]); + std::array result; + for (uint32_t component = 0; component < 2; component++) { + switch (inst.op) { + case IR::Opcode::BitwiseAndNotU64: + result[component] = + ir.BitwiseAnd(lhs[component], ir.BitwiseNot(rhs[component])); + break; + case IR::Opcode::BitwiseOrNotU64: + result[component] = ir.BitwiseOr(lhs[component], ir.BitwiseNot(rhs[component])); + break; + case IR::Opcode::BitwiseNandU64: + result[component] = + ir.BitwiseNot(ir.BitwiseAnd(lhs[component], rhs[component])); + break; + case IR::Opcode::BitwiseNorU64: + result[component] = ir.BitwiseNot(ir.BitwiseOr(lhs[component], rhs[component])); + break; + default: + result[component] = + ir.BitwiseNot(ir.BitwiseXor(lhs[component], rhs[component])); + break; + } + } + WriteU32Pair(inst.dst, result); + return true; + } const auto binary_u32 = [&](auto operation) { const auto lhs = ReadU32(inst.src[0]); const auto rhs = ReadU32(inst.src[1]); return operation(lhs, rhs); }; - const auto binary_u64 = [&](auto operation) { - const auto lhs = UnpackU64(ReadU64(inst.src[0])); - const auto rhs = UnpackU64(ReadU64(inst.src[1])); - return ir.PackUint2x32(operation(lhs[0], rhs[0]), operation(lhs[1], rhs[1])); - }; IR::Value result; switch (inst.op) { @@ -282,34 +309,6 @@ bool Translator::TranslateComposedInteger(const IR::Instruction& inst) { result = binary_u32( [&](IR::U32 lhs, IR::U32 rhs) { return ir.BitwiseNot(ir.BitwiseXor(lhs, rhs)); }); break; - case IR::Opcode::BitwiseXorU64: - result = binary_u64([&](IR::U32 lhs, IR::U32 rhs) { return ir.BitwiseXor(lhs, rhs); }); - break; - case IR::Opcode::BitwiseAndNotU64: - result = binary_u64( - [&](IR::U32 lhs, IR::U32 rhs) { return ir.BitwiseAnd(lhs, ir.BitwiseNot(rhs)); }); - break; - case IR::Opcode::BitwiseOrNotU64: - result = binary_u64( - [&](IR::U32 lhs, IR::U32 rhs) { return ir.BitwiseOr(lhs, ir.BitwiseNot(rhs)); }); - break; - case IR::Opcode::BitwiseNandU64: - result = binary_u64( - [&](IR::U32 lhs, IR::U32 rhs) { return ir.BitwiseNot(ir.BitwiseAnd(lhs, rhs)); }); - break; - case IR::Opcode::BitwiseNorU64: - result = binary_u64( - [&](IR::U32 lhs, IR::U32 rhs) { return ir.BitwiseNot(ir.BitwiseOr(lhs, rhs)); }); - break; - case IR::Opcode::BitwiseXnorU64: - result = binary_u64( - [&](IR::U32 lhs, IR::U32 rhs) { return ir.BitwiseNot(ir.BitwiseXor(lhs, rhs)); }); - break; - case IR::Opcode::BitwiseNotU64: { - const auto source = UnpackU64(ReadU64(inst.src[0])); - result = ir.PackUint2x32(ir.BitwiseNot(source[0]), ir.BitwiseNot(source[1])); - break; - } case IR::Opcode::BitwiseAndOrU32: { const auto lhs = ReadU32(inst.src[0]); const auto rhs = ReadU32(inst.src[1]); @@ -326,7 +325,7 @@ bool Translator::TranslateComposedInteger(const IR::Instruction& inst) { ReadU32(inst.src[2])); break; case IR::Opcode::FindLsbU64: { - const auto source = UnpackU64(ReadU64(inst.src[0])); + const auto source = ExtractU64(ReadU64(inst.src[0])); const auto low_lsb = IR::U32(ir.Emit(IR::ValueOpcode::FindILsb32, {source[0]})); const auto high_lsb = IR::U32(ir.Emit(IR::ValueOpcode::FindILsb32, {source[1]})); const auto high_position = ir.IAdd(high_lsb, IR::U32(IR::Value(32u))); @@ -349,44 +348,13 @@ bool Translator::TranslateComposedInteger(const IR::Instruction& inst) { const auto position = ir.ISub(IR::U32(IR::Value(63u)), msb); const auto nonzero = IR::U1( ir.Emit(IR::ValueOpcode::INotEqual64, - {source, ir.PackUint2x32(IR::U32(IR::Value(0u)), IR::U32(IR::Value(0u)))})); + {source, ir.ConstructU64(IR::U32(IR::Value(0u)), IR::U32(IR::Value(0u)))})); result = ir.Select(nonzero, position, IR::U32(IR::Value(0xffffffffu))); break; } default: return false; } WriteOperand(inst.dst, result); - if (result.GetType() == IR::Type::U64 && inst.dst.kind == IR::OperandKind::Register && - inst.dst.reg.file == IR::RegisterFile::Scalar) { - const auto lhs = ReadMask(inst.src[0]); - IR::U1 mask; - switch (inst.op) { - case IR::Opcode::BitwiseXorU64: - mask = IR::U1(ir.Emit(IR::ValueOpcode::LogicalXor, {lhs, ReadMask(inst.src[1])})); - break; - case IR::Opcode::BitwiseAndNotU64: - mask = ir.LogicalAnd(lhs, ir.LogicalNot(ReadMask(inst.src[1]))); - break; - case IR::Opcode::BitwiseOrNotU64: - mask = ir.LogicalOr(lhs, ir.LogicalNot(ReadMask(inst.src[1]))); - break; - case IR::Opcode::BitwiseNandU64: - mask = ir.LogicalNot(ir.LogicalAnd(lhs, ReadMask(inst.src[1]))); - break; - case IR::Opcode::BitwiseNorU64: - mask = ir.LogicalNot(ir.LogicalOr(lhs, ReadMask(inst.src[1]))); - break; - case IR::Opcode::BitwiseXnorU64: - mask = ir.LogicalNot( - IR::U1(ir.Emit(IR::ValueOpcode::LogicalXor, {lhs, ReadMask(inst.src[1])}))); - break; - case IR::Opcode::BitwiseNotU64: mask = ir.LogicalNot(lhs); break; - default: break; - } - if (!mask.IsEmpty()) { - ir.SetThreadBitScalarReg(static_cast(inst.dst.reg.index), mask); - } - } return true; } @@ -402,18 +370,12 @@ bool Translator::TranslateExtendedInteger(const IR::Instruction& inst) { return IR::U32( ir.Emit(IR::ValueOpcode::BitFieldInsert, {imm(0), imm(0xffffffffu), imm(0), count})); }; - const auto select_u64 = [&](IR::U1 condition, IR::U64 true_value, IR::U64 false_value) { - const auto lhs = UnpackU64(true_value); - const auto rhs = UnpackU64(false_value); - return ir.PackUint2x32(ir.Select(condition, lhs[0], rhs[0]), - ir.Select(condition, lhs[1], rhs[1])); - }; const auto right_mask64 = [&](IR::U32 count) { const auto below32 = IR::U1(ir.Emit(IR::ValueOpcode::ULessThan32, {count, imm(32)})); const auto above32 = IR::U1(ir.Emit(IR::ValueOpcode::UGreaterThan32, {count, imm(32)})); const auto low_count = ir.Select(below32, count, imm(32)); const auto high_count = ir.Select(above32, ir.ISub(count, imm(32)), imm(0)); - return ir.PackUint2x32(right_mask32(low_count), right_mask32(high_count)); + return ir.ConstructU64(right_mask32(low_count), right_mask32(high_count)); }; IR::Value result; @@ -436,7 +398,7 @@ bool Translator::TranslateExtendedInteger(const IR::Instruction& inst) { case IR::Opcode::UMadU64U32: { const auto lhs = ReadU32(inst.src[0]); const auto rhs = ReadU32(inst.src[1]); - const auto add = UnpackU64(ReadU64(inst.src[2])); + const auto add = ExtractU64(ReadU64(inst.src[2])); const auto mul_low = ir.IMul(lhs, rhs); const auto mul_high = IR::U32(ir.Emit(IR::ValueOpcode::UMulHi, {lhs, rhs})); const auto low = ir.IAdd(mul_low, add[0]); @@ -445,7 +407,7 @@ bool Translator::TranslateExtendedInteger(const IR::Instruction& inst) { const auto carry0 = ir.ULessThan(high0, mul_high); const auto high = ir.IAdd(high0, ir.Select(carry_low, imm(1), imm(0))); const auto carry1 = ir.ULessThan(high, high0); - WriteOperand(inst.dst, ir.PackUint2x32(low, high)); + WriteOperand(inst.dst, ir.ConstructU64(low, high)); if (inst.dst2.kind != IR::OperandKind::Null) { WriteMask(inst.dst2, ir.LogicalOr(carry0, carry1)); } @@ -503,7 +465,7 @@ bool Translator::TranslateExtendedInteger(const IR::Instruction& inst) { return ir.BitwiseOr(bits, ir.ShiftLeftLogical(bits, imm(1))); }; const auto source = ReadU32(inst.src[0]); - result = ir.PackUint2x32(replicate(mask(source, 0xffffu)), + result = ir.ConstructU64(replicate(mask(source, 0xffffu)), replicate(ir.ShiftRightLogical(source, imm(16)))); break; } @@ -590,12 +552,12 @@ bool Translator::TranslateExtendedInteger(const IR::Instruction& inst) { const auto lo = ReadU32(inst.src[1]); const auto byte_offset = mask(ReadU32(inst.src[2]), 31u); const auto bit_offset = ir.ShiftLeftLogical(byte_offset, imm(3)); - const auto concatenated = ir.PackUint2x32(lo, hi); + const auto concatenated = ir.ConstructU64(lo, hi); const auto shifted = IR::U64(ir.Emit(IR::ValueOpcode::ShiftRightLogical64, {concatenated, mask(bit_offset, 63u)})); const auto in_range = IR::U1(ir.Emit(IR::ValueOpcode::ULessThan32, {byte_offset, imm(8)})); - result = ir.Select(in_range, UnpackU64(shifted)[0], imm(0)); + result = ir.Select(in_range, ExtractU64(shifted)[0], imm(0)); break; } case IR::Opcode::ShiftLeftAddU32: @@ -633,10 +595,14 @@ bool Translator::TranslateExtendedInteger(const IR::Instruction& inst) { ReadOperand(inst.src[2], IR::Type::F32)}); break; } - case IR::Opcode::SelectU64: - result = - select_u64(ReadCondition(inst.src[0]), ReadU64(inst.src[1]), ReadU64(inst.src[2])); - break; + case IR::Opcode::SelectU64: { + const auto condition = ReadCondition(inst.src[0]); + const auto lhs = ReadU32Pair(inst.src[1]); + const auto rhs = ReadU32Pair(inst.src[2]); + WriteU32Pair(inst.dst, {ir.Select(condition, lhs[0], rhs[0]), + ir.Select(condition, lhs[1], rhs[1])}); + return true; + } case IR::Opcode::PackLowLowU16: case IR::Opcode::PackLowHighU16: case IR::Opcode::PackHighHighU16: diff --git a/src/graphics/shader/recompiler/frontend/translate/Translate.cpp b/src/graphics/shader/recompiler/frontend/translate/Translate.cpp index 783be909b..3804db4e1 100644 --- a/src/graphics/shader/recompiler/frontend/translate/Translate.cpp +++ b/src/graphics/shader/recompiler/frontend/translate/Translate.cpp @@ -87,10 +87,7 @@ IR::U32 Translator::ReadPcRelativeU32(const IR::Operand& operand) { const auto address = ir.Emit(IR::ValueOpcode::IAdd64, {ir.Emit(IR::ValueOpcode::GetShaderBase), IR::Value(static_cast(operand.imm))}); - const auto pair = ir.Emit(IR::ValueOpcode::UnpackUint2x32, {address}); - return IR::U32( - ir.Emit(IR::ValueOpcode::CompositeExtractU32x2, - {pair, IR::Value(operand.kind == IR::OperandKind::PcRelativeU32 ? 0u : 1u)})); + return ir.CompositeExtract(address, operand.kind == IR::OperandKind::PcRelativeU32 ? 0u : 1u); } std::array Translator::BallotMask(IR::U1 value) { @@ -205,22 +202,8 @@ IR::Value Translator::ReadOperand(const IR::Operand& operand, IR::Type type) { return ir.INotEqual(ReadRawU32(operand), IR::U32(IR::Value(0u))); } if (type == IR::Type::U64) { - if (operand.kind == IR::OperandKind::Register && - operand.reg.file == IR::RegisterFile::Exec) { - return ir.PackUint2x32(ir.GetExecLo(), ir.GetExecHi()); - } - if (operand.kind == IR::OperandKind::Register && - operand.reg.file == IR::RegisterFile::Vcc) { - return ir.PackUint2x32(ir.GetVccLo(), ir.GetVccHi()); - } - const auto low = ApplyBitSourceModifiers(operand, ReadRawU32(operand)); - IR::U32 high(IR::Value(0u)); - if (operand.kind == IR::OperandKind::Register) { - high = ReadRawU32(OffsetOperand(operand, 1)); - } else if (operand.kind == IR::OperandKind::ImmediateU32 && operand.sext_64) { - high = IR::U32(IR::Value(0xffffffffu)); - } - return ir.PackUint2x32(low, high); + const auto pair = ReadU32Pair(operand); + return ir.ConstructU64(pair[0], pair[1]); } auto bits = ApplyBitSourceModifiers(operand, ReadRawU32(operand)); if (TypesOverlap(type, IR::Type::F32) && !TypesOverlap(type, IR::Type::U32)) { @@ -396,31 +379,7 @@ void Translator::WriteOperand(const IR::Operand& operand, IR::Value value) { return; } if (type == IR::Type::U64) { - const auto unpacked = ir.Emit(IR::ValueOpcode::UnpackUint2x32, {value}); - const auto low = ir.CompositeExtract(unpacked, 0); - const auto high = ir.CompositeExtract(unpacked, 1); - if (operand.kind == IR::OperandKind::Register) { - const auto thread_bit = ThreadBit(low, high); - switch (operand.reg.file) { - case IR::RegisterFile::Exec: - ir.SetExec(thread_bit); - ir.SetExecLo(low); - ir.SetExecHi(high); - return; - case IR::RegisterFile::Vcc: - ir.SetVcc(thread_bit); - ir.SetVccLo(low); - ir.SetVccHi(high); - return; - case IR::RegisterFile::Scalar: - ir.SetThreadBitScalarReg(static_cast(operand.reg.index), - thread_bit); - break; - default: break; - } - } - WriteRawU32(operand, low); - WriteRawU32(OffsetOperand(operand, 1), high); + WriteU32Pair(operand, {ir.CompositeExtract(value, 0), ir.CompositeExtract(value, 1)}); return; } if (type == IR::Type::F32) { @@ -474,6 +433,25 @@ IR::U32 Translator::ReadU32(const IR::Operand& operand) { return IR::U32(ReadOperand(operand, IR::Type::U32)); } +std::array Translator::ReadU32Pair(const IR::Operand& operand) { + if (operand.kind == IR::OperandKind::Register) { + if (operand.reg.file == IR::RegisterFile::Exec) { + return {ir.GetExecLo(), ir.GetExecHi()}; + } + if (operand.reg.file == IR::RegisterFile::Vcc) { + return {ir.GetVccLo(), ir.GetVccHi()}; + } + } + const auto low = ApplyBitSourceModifiers(operand, ReadRawU32(operand)); + IR::U32 high(IR::Value(0u)); + if (operand.kind == IR::OperandKind::Register) { + high = ReadRawU32(OffsetOperand(operand, 1)); + } else if (operand.kind == IR::OperandKind::ImmediateU32 && operand.sext_64) { + high = IR::U32(IR::Value(0xffffffffu)); + } + return {low, high}; +} + IR::U64 Translator::ReadU64(const IR::Operand& operand) { return IR::U64(ReadOperand(operand, IR::Type::U64)); } @@ -560,9 +538,35 @@ IR::U32 Translator::ReadF16LaneBits(const IR::Operand& operand, bool high_lane) return value; } -std::array Translator::UnpackU64(IR::U64 value) { - const auto unpacked = ir.Emit(IR::ValueOpcode::UnpackUint2x32, {value}); - return {ir.CompositeExtract(unpacked, 0), ir.CompositeExtract(unpacked, 1)}; +std::array Translator::ExtractU64(IR::U64 value) { + return {ir.CompositeExtract(value, 0), ir.CompositeExtract(value, 1)}; +} + +void Translator::WriteU32Pair(const IR::Operand& operand, const std::array& value) { + if (operand.kind == IR::OperandKind::Null) { + return; + } + if (operand.kind == IR::OperandKind::Register) { + const auto thread_bit = ThreadBit(value[0], value[1]); + switch (operand.reg.file) { + case IR::RegisterFile::Exec: + ir.SetExec(thread_bit); + ir.SetExecLo(value[0]); + ir.SetExecHi(value[1]); + return; + case IR::RegisterFile::Vcc: + ir.SetVcc(thread_bit); + ir.SetVccLo(value[0]); + ir.SetVccHi(value[1]); + return; + case IR::RegisterFile::Scalar: + ir.SetThreadBitScalarReg(static_cast(operand.reg.index), thread_bit); + break; + default: break; + } + } + WriteRawU32(operand, value[0]); + WriteRawU32(OffsetOperand(operand, 1), value[1]); } IR::U1 Translator::ThreadBit(IR::U32 low, IR::U32 high) { diff --git a/src/graphics/shader/recompiler/frontend/translate/Translator.h b/src/graphics/shader/recompiler/frontend/translate/Translator.h index ba62cb608..3374d93be 100644 --- a/src/graphics/shader/recompiler/frontend/translate/Translator.h +++ b/src/graphics/shader/recompiler/frontend/translate/Translator.h @@ -36,6 +36,7 @@ private: void WriteF16(const IR::Operand& operand, IR::F32 value); void WriteU16(const IR::Operand& operand, IR::U32 value); IR::U32 ReadU32(const IR::Operand& operand); + std::array ReadU32Pair(const IR::Operand& operand); IR::U64 ReadU64(const IR::Operand& operand); IR::F32 ReadF16LaneAsF32(const IR::Operand& operand, bool high_lane); IR::F32 ReadF16AsF32(const IR::Operand& operand); @@ -44,13 +45,14 @@ private: IR::U32 ReadU16LaneAsU32(const IR::Operand& operand, bool high_lane, bool sign_extend); IR::U32 ReadU16AsU32(const IR::Operand& operand, bool sign_extend); IR::U32 ReadF16LaneBits(const IR::Operand& operand, bool high_lane); - std::array UnpackU64(IR::U64 value); - IR::U1 ReadCondition(const IR::Operand& operand); - IR::U32 ConditionBit(const IR::Operand& operand); - IR::U1 ReadMask(const IR::Operand& operand); - void WriteMask(const IR::Operand& operand, IR::U1 value); - void WriteMask64(const IR::Operand& operand, IR::U1 value); - void WriteCompareResult(const IR::Operand& operand, IR::U1 value); + std::array ExtractU64(IR::U64 value); + void WriteU32Pair(const IR::Operand& operand, const std::array& value); + IR::U1 ReadCondition(const IR::Operand& operand); + IR::U32 ConditionBit(const IR::Operand& operand); + IR::U1 ReadMask(const IR::Operand& operand); + void WriteMask(const IR::Operand& operand, IR::U1 value); + void WriteMask64(const IR::Operand& operand, IR::U1 value); + void WriteCompareResult(const IR::Operand& operand, IR::U1 value); IR::MemoryFlags AddMemoryInfo(const IR::Instruction& inst); IR::ExportFlags AddExportInfo(const IR::Instruction& inst); @@ -82,13 +84,13 @@ private: IR::ValueOpcode BufferAtomicOpcode(IR::Opcode opcode); IR::ValueOpcode SharedAtomicOpcode(IR::Opcode opcode); bool TranslateScalarMemory(const IR::Instruction& inst); - bool TranslateBufferLoad(const IR::Instruction& inst); - bool TranslateBufferStore(const IR::Instruction& inst); - bool TranslateAtomicMemory(const IR::Instruction& inst); - bool TranslateFlatLoad(const IR::Instruction& inst); - bool TranslateFlatStore(const IR::Instruction& inst); - bool TranslateImageMemory(const IR::Instruction& inst); - bool TranslateSharedMemory(const IR::Instruction& inst); + bool TranslateBufferLoad(const IR::Instruction& inst); + bool TranslateBufferStore(const IR::Instruction& inst); + bool TranslateAtomicMemory(const IR::Instruction& inst); + bool TranslateFlatLoad(const IR::Instruction& inst); + bool TranslateFlatStore(const IR::Instruction& inst); + bool TranslateImageMemory(const IR::Instruction& inst); + bool TranslateSharedMemory(const IR::Instruction& inst); IR::F32 SelectF32(IR::U1 condition, IR::F32 true_value, IR::F32 false_value); IR::U32 ConvertF32ToU32Saturated(IR::F32 value, float upper_bound, float safe_upper, diff --git a/src/graphics/shader/recompiler/ir/IREmitter.cpp b/src/graphics/shader/recompiler/ir/IREmitter.cpp index 73bc277b2..8c714f915 100644 --- a/src/graphics/shader/recompiler/ir/IREmitter.cpp +++ b/src/graphics/shader/recompiler/ir/IREmitter.cpp @@ -134,14 +134,14 @@ U32 IREmitter::BitCastU32(F16 value) { return U32(Emit(ValueOpcode::ConvertU32U16, {half})); } -U64 IREmitter::PackUint2x32(U32 low, U32 high) { - const auto pair = Emit(ValueOpcode::CompositeConstructU32x2, {low, high}); - return U64(Emit(ValueOpcode::PackUint2x32, {pair})); +U64 IREmitter::ConstructU64(U32 low, U32 high) { + return U64(Emit(ValueOpcode::CompositeConstructU64, {low, high})); } U32 IREmitter::CompositeExtract(Value composite, uint32_t index) { ValueOpcode opcode; switch (composite.GetType()) { + case Type::U64: opcode = ValueOpcode::CompositeExtractU64; break; case Type::U32x2: opcode = ValueOpcode::CompositeExtractU32x2; break; case Type::U32x3: opcode = ValueOpcode::CompositeExtractU32x3; break; case Type::U32x4: opcode = ValueOpcode::CompositeExtractU32x4; break; diff --git a/src/graphics/shader/recompiler/ir/IREmitter.h b/src/graphics/shader/recompiler/ir/IREmitter.h index 808c376bd..beef3cab8 100644 --- a/src/graphics/shader/recompiler/ir/IREmitter.h +++ b/src/graphics/shader/recompiler/ir/IREmitter.h @@ -54,7 +54,7 @@ public: U32 BitCastU32(F32 value); F16 BitCastF16(U32 value); U32 BitCastU32(F16 value); - U64 PackUint2x32(U32 low, U32 high); + U64 ConstructU64(U32 low, U32 high); U32 CompositeExtract(Value composite, uint32_t index); U32 IAdd(U32 lhs, U32 rhs); diff --git a/src/graphics/shader/recompiler/ir/ValueProgram.cpp b/src/graphics/shader/recompiler/ir/ValueProgram.cpp index d30b7e719..0509b4b85 100644 --- a/src/graphics/shader/recompiler/ir/ValueProgram.cpp +++ b/src/graphics/shader/recompiler/ir/ValueProgram.cpp @@ -317,9 +317,9 @@ bool ValidateValueProgram(const ValueProgram& program, bool require_ssa, std::st return Fail(error, fmt::format("{} has an invalid memory-info index", ValueOpcodeName(inst.GetOpcode()))); } - const auto& memory = program.memory_info[memory_index]; - const bool scalar_buffer = inst.GetOpcode() == ValueOpcode::ReadConstBuffer; - const bool scalar_address = memory.kind == ResourceKind::ScalarAddress; + const auto& memory = program.memory_info[memory_index]; + const bool scalar_buffer = inst.GetOpcode() == ValueOpcode::ReadConstBuffer; + const bool scalar_address = memory.kind == ResourceKind::ScalarAddress; if ((scalar_buffer && memory.kind != ResourceKind::ScalarBuffer) || (!scalar_buffer && !scalar_address && memory.kind != ResourceKind::Flat && memory.kind != ResourceKind::Global && memory.kind != ResourceKind::Scratch)) { @@ -328,12 +328,13 @@ bool ValidateValueProgram(const ValueProgram& program, bool require_ssa, std::st } const bool scalar_memory = scalar_buffer || scalar_address; const bool valid_group_width = - scalar_memory ? memory.component_count == 1u || memory.component_count == 2u || - memory.component_count == 4u || memory.component_count == 8u || - memory.component_count == 16u - : memory.component_count >= 1u && memory.component_count <= 4u; - if (memory.data_bits != 32u || memory.data_dwords != 1u || - !valid_group_width || memory.component_index >= memory.component_count) { + scalar_memory + ? memory.component_count == 1u || memory.component_count == 2u || + memory.component_count == 4u || memory.component_count == 8u || + memory.component_count == 16u + : memory.component_count >= 1u && memory.component_count <= 4u; + if (memory.data_bits != 32u || memory.data_dwords != 1u || !valid_group_width || + memory.component_index >= memory.component_count) { return Fail(error, fmt::format("{} has inconsistent scalar-memory metadata", ValueOpcodeName(inst.GetOpcode()))); } @@ -395,6 +396,7 @@ bool ValidateValueProgram(const ValueProgram& program, bool require_ssa, std::st } uint32_t composite_components = 0u; switch (inst.GetOpcode()) { + case ValueOpcode::CompositeExtractU64: composite_components = 2u; break; case ValueOpcode::CompositeExtractU32x2: composite_components = 2u; break; case ValueOpcode::CompositeExtractU32x3: composite_components = 3u; break; case ValueOpcode::CompositeExtractU32x4: composite_components = 4u; break; diff --git a/src/graphics/shader/recompiler/ir/opcodes/ValueOpcodes.inc b/src/graphics/shader/recompiler/ir/opcodes/ValueOpcodes.inc index a9269951f..9471a02bb 100644 --- a/src/graphics/shader/recompiler/ir/opcodes/ValueOpcodes.inc +++ b/src/graphics/shader/recompiler/ir/opcodes/ValueOpcodes.inc @@ -56,15 +56,15 @@ VALUE_OPCODE(ConvertS32F32, U32, F32) VALUE_OPCODE(ConvertU32F32, U32, F32) VALUE_OPCODE(ConvertF32S32, F32, U32) VALUE_OPCODE(ConvertF32U32, F32, U32) +VALUE_OPCODE(CompositeConstructU64, U64, U32, U32) VALUE_OPCODE(CompositeConstructU32x2, U32x2, U32, U32) VALUE_OPCODE(CompositeConstructU32x3, U32x3, U32, U32, U32) VALUE_OPCODE(CompositeConstructF32x2, F32x2, F32, F32) VALUE_OPCODE(CompositeConstructU32x4, U32x4, U32, U32, U32, U32) +VALUE_OPCODE(CompositeExtractU64, U32, U64, U32) VALUE_OPCODE(CompositeExtractU32x2, U32, U32x2, U32) VALUE_OPCODE(CompositeExtractU32x3, U32, U32x3, U32) VALUE_OPCODE(CompositeExtractU32x4, U32, U32x4, U32) -VALUE_OPCODE(PackUint2x32, U64, U32x2) -VALUE_OPCODE(UnpackUint2x32, U32x2, U64) VALUE_OPCODE(PackHalf2x16, U32, F32x2) VALUE_OPCODE(PackSnorm2x16, U32, F32x2) VALUE_OPCODE(PackUnorm2x16, U32, F32x2) @@ -105,7 +105,6 @@ VALUE_OPCODE(ShiftRightArithmetic64, U64, U64, U32) VALUE_OPCODE(BitwiseAnd32, U32, U32, U32) VALUE_OPCODE(BitwiseAnd64, U64, U64, U64) VALUE_OPCODE(BitwiseOr32, U32, U32, U32) -VALUE_OPCODE(BitwiseOr64, U64, U64, U64) VALUE_OPCODE(BitwiseXor32, U32, U32, U32) VALUE_OPCODE(BitwiseNot32, U32, U32) VALUE_OPCODE(BitReverse32, U32, U32) diff --git a/src/graphics/shader/recompiler/ir/passes/ConstantPropagation.cpp b/src/graphics/shader/recompiler/ir/passes/ConstantPropagation.cpp index ad75079f5..42cde88aa 100644 --- a/src/graphics/shader/recompiler/ir/passes/ConstantPropagation.cpp +++ b/src/graphics/shader/recompiler/ir/passes/ConstantPropagation.cpp @@ -170,6 +170,10 @@ bool FoldCompositeExtract(Inst& inst, ValueOpcode construct, size_t components) } const auto component = index.U32(); const auto* producer = composite.TryInstruction(); + if (IsImmediate(composite, Type::U64)) { + Replace(inst, Value(static_cast(composite.U64() >> (component * 32u)))); + return true; + } if (producer != nullptr && producer->GetOpcode() == construct) { Replace(inst, producer->Arg(component)); return true; @@ -185,14 +189,6 @@ bool FoldCompositeExtract(Inst& inst, ValueOpcode construct, size_t components) return true; } } - if (component < 2u && producer != nullptr && - producer->GetOpcode() == ValueOpcode::UnpackUint2x32) { - const auto packed = producer->Arg(0).Resolve(); - if (IsImmediate(packed, Type::U64)) { - Replace(inst, Value(static_cast(packed.U64() >> (component * 32u)))); - return true; - } - } return false; } @@ -284,6 +280,9 @@ void FoldInstruction(Inst& inst) { } return; } + case ValueOpcode::CompositeExtractU64: + FoldCompositeExtract(inst, ValueOpcode::CompositeConstructU64, 2); + return; case ValueOpcode::CompositeExtractU32x2: FoldCompositeExtract(inst, ValueOpcode::CompositeConstructU32x2, 2); return; @@ -293,27 +292,12 @@ void FoldInstruction(Inst& inst) { case ValueOpcode::CompositeExtractU32x4: FoldCompositeExtract(inst, ValueOpcode::CompositeConstructU32x4, 4); return; - case ValueOpcode::PackUint2x32: { - const auto value = Arg(inst, 0); - const auto* producer = value.TryInstruction(); - if (producer != nullptr && producer->GetOpcode() == ValueOpcode::UnpackUint2x32) { - Replace(inst, producer->Arg(0)); - } else if (producer != nullptr && - producer->GetOpcode() == ValueOpcode::CompositeConstructU32x2) { - const auto low = producer->Arg(0).Resolve(); - const auto high = producer->Arg(1).Resolve(); - if (IsImmediate(low, Type::U32) && IsImmediate(high, Type::U32)) { - Replace(inst, Value(static_cast(low.U32()) | - (static_cast(high.U32()) << 32u))); - } - } - return; - } - case ValueOpcode::UnpackUint2x32: { - const auto value = Arg(inst, 0); - const auto* producer = value.TryInstruction(); - if (producer != nullptr && producer->GetOpcode() == ValueOpcode::PackUint2x32) { - Replace(inst, producer->Arg(0)); + case ValueOpcode::CompositeConstructU64: { + const auto low = Arg(inst, 0); + const auto high = Arg(inst, 1); + if (IsImmediate(low, Type::U32) && IsImmediate(high, Type::U32)) { + Replace(inst, Value(static_cast(low.U32()) | + (static_cast(high.U32()) << 32u))); } return; } @@ -367,8 +351,8 @@ void FoldInstruction(Inst& inst) { }; const auto low = expand(static_cast(value.U64())); const auto high = expand(static_cast(value.U64() >> 32u)); - Replace(inst, Value(static_cast(low) | - (static_cast(high) << 32u))); + Replace(inst, + Value(static_cast(low) | (static_cast(high) << 32u))); } return; } @@ -461,11 +445,6 @@ void FoldInstruction(Inst& inst) { ReplaceBinaryIdentity(inst, Type::U32, 0u); } return; - case ValueOpcode::BitwiseOr64: - if (!FoldU64(inst, [](uint64_t a, uint64_t b) { return a | b; })) { - ReplaceBinaryIdentity(inst, Type::U64, 0u); - } - return; case ValueOpcode::BitwiseNot32: { const auto value = Arg(inst, 0); if (IsImmediate(value, Type::U32)) { diff --git a/src/graphics/shader/recompiler/ir/passes/SrtWalker.cpp b/src/graphics/shader/recompiler/ir/passes/SrtWalker.cpp index 8aeedee5d..34ec7e5a8 100644 --- a/src/graphics/shader/recompiler/ir/passes/SrtWalker.cpp +++ b/src/graphics/shader/recompiler/ir/passes/SrtWalker.cpp @@ -76,10 +76,10 @@ bool IsDescriptorHandle(ValueOpcode opcode) { bool IsRuntimeIntegerOp(ValueOpcode op) { switch (op) { + case ValueOpcode::CompositeConstructU64: + case ValueOpcode::CompositeExtractU64: case ValueOpcode::CompositeConstructU32x2: case ValueOpcode::CompositeExtractU32x2: - case ValueOpcode::PackUint2x32: - case ValueOpcode::UnpackUint2x32: case ValueOpcode::BitFieldInsert: case ValueOpcode::BitFieldUExtract: case ValueOpcode::BitFieldSExtract: @@ -100,7 +100,6 @@ bool IsRuntimeIntegerOp(ValueOpcode op) { case ValueOpcode::BitwiseAnd32: case ValueOpcode::BitwiseAnd64: case ValueOpcode::BitwiseOr32: - case ValueOpcode::BitwiseOr64: case ValueOpcode::BitwiseXor32: case ValueOpcode::BitwiseNot32: case ValueOpcode::SelectU1: @@ -211,10 +210,10 @@ private: reason = fmt::format("contains a non-scalar {}", ValueOpcodeName(op)); return finish(false); } - } else if (op == ValueOpcode::PackUint2x32) { - const auto* pair = inst->NumArgs() == 1 ? inst->Arg(0).ResolveInstruction() : nullptr; - if (pair == nullptr || pair->GetOpcode() != ValueOpcode::CompositeConstructU32x2) { - reason = "contains an unsupported packed runtime value"; + } else if (op == ValueOpcode::CompositeExtractU64) { + const auto index = inst->NumArgs() == 2 ? inst->Arg(1).Resolve() : Value {}; + if (!index.IsImmediate() || index.GetType() != Type::U32 || index.U32() >= 2u) { + reason = "contains an invalid U64 component index"; return finish(false); } } else if (op == ValueOpcode::CompositeExtractU32x2) { @@ -223,7 +222,6 @@ private: if (source == nullptr || !index.IsImmediate() || index.GetType() != Type::U32 || index.U32() >= 2u || (source->GetOpcode() != ValueOpcode::CompositeConstructU32x2 && - source->GetOpcode() != ValueOpcode::UnpackUint2x32 && source->GetOpcode() != ValueOpcode::IAddCarry32)) { reason = "contains an unsupported composite runtime source"; return finish(false); @@ -484,22 +482,25 @@ private: if (!index.IsImmediate() || index.GetType() != Type::U32) { return Fail(error, "dynamic composite extract in runtime expression"); } - const auto component = index.U32(); - const auto* source = inst.Arg(0).ResolveInstruction(); - if (source == nullptr || component >= 2u) { + const auto component = index.U32(); + if (component >= 2u) { return Fail(error, "unsupported composite runtime source"); } - if (source->GetOpcode() == ValueOpcode::CompositeConstructU32x2) { - return EvaluateWide(source->Arg(component), result, error); - } - if (source->GetOpcode() == ValueOpcode::UnpackUint2x32) { + if (inst.GetOpcode() == ValueOpcode::CompositeExtractU64) { uint64_t packed = 0; - if (!Arg(*source, 0, packed, error)) { + if (!Arg(inst, 0, packed, error)) { return false; } result = static_cast(packed >> (component * 32u)); return true; } + const auto* source = inst.Arg(0).ResolveInstruction(); + if (source == nullptr) { + return Fail(error, "unsupported composite runtime source"); + } + if (source->GetOpcode() == ValueOpcode::CompositeConstructU32x2) { + return EvaluateWide(source->Arg(component), result, error); + } if (source->GetOpcode() == ValueOpcode::IAddCarry32) { uint64_t lhs = 0; uint64_t rhs = 0; @@ -596,17 +597,15 @@ private: } case ValueOpcode::GetShaderBase: result = m_runtime.shader_base; return true; case ValueOpcode::Phi: return EvaluatePhi(inst, result, error); + case ValueOpcode::CompositeExtractU64: case ValueOpcode::CompositeExtractU32x2: return EvaluateExtract(inst, result, error); - case ValueOpcode::PackUint2x32: { - const auto* pair = inst.Arg(0).ResolveInstruction(); - if (pair == nullptr || pair->GetOpcode() != ValueOpcode::CompositeConstructU32x2 || - !Arg(*pair, 0, a, error) || !Arg(*pair, 1, b, error)) { - return Fail(error, "unsupported packed runtime value"); + case ValueOpcode::CompositeConstructU64: + if (!binary()) { + return false; } result = static_cast(a) | (static_cast(static_cast(b)) << 32u); return true; - } case ValueOpcode::ReadConst: { const auto slot = inst.Arg(1).Resolve(); if (!slot.IsImmediate() || slot.GetType() != Type::U32 || @@ -687,12 +686,6 @@ private: return true; } return false; - case ValueOpcode::BitwiseOr64: - if (binary()) { - result = a | b; - return true; - } - return false; case ValueOpcode::BitwiseXor32: if (binary()) { result = static_cast(a ^ b); diff --git a/tests/ScalarProvenanceTests.cpp b/tests/ScalarProvenanceTests.cpp index a9f30c5ee..141210f6c 100644 --- a/tests/ScalarProvenanceTests.cpp +++ b/tests/ScalarProvenanceTests.cpp @@ -1,8 +1,8 @@ +#include "graphics/shader/recompiler/ir/ValueProgram.h" #include "graphics/shader/recompiler/ir/passes/ConstantPropagation.h" #include "graphics/shader/recompiler/ir/passes/DeadCodeElimination.h" #include "graphics/shader/recompiler/ir/passes/ReadLaneElimination.h" #include "graphics/shader/recompiler/ir/passes/SrtWalker.h" -#include "graphics/shader/recompiler/ir/ValueProgram.h" #include #include @@ -133,16 +133,15 @@ void TestRawScalarComponentAlignment() { std::vector flat; std::string error; Check(WalkSrt(fixture.program, runtime, flat, &error), error.c_str()); - Check(flat == std::vector{0x12345678u} && - memory_image.reads == 1, - "raw scalar base, immediate, and offset were not aligned independently"); + Check( + flat == std::vector{0x12345678u} && memory_image.reads == 1, + "raw scalar base, immediate, and offset were not aligned independently"); } void TestScalarMemoryDomainMismatchFails() { Fixture raw; const auto raw_memory = raw.AddMemory(ResourceKind::ScalarBuffer); - RawRead(raw, Address(raw, Value(0x1000u), Value(0u)), Value(0u), - raw_memory); + RawRead(raw, Address(raw, Value(0x1000u), Value(0u)), Value(0u), raw_memory); std::string error; Check(!BuildSrtPlan(raw.program, &error) && error.find("incompatible scalar memory metadata") != @@ -151,11 +150,11 @@ void TestScalarMemoryDomainMismatchFails() { Fixture buffer; const auto buffer_memory = buffer.AddMemory(ResourceKind::ScalarAddress); - const auto resource = buffer.Emit(ValueOpcode::GetBufferResource, - {Value(0x1000u), Value(0u), Value(16u), - Value(0u)}); - buffer.EmitMemory(ValueOpcode::ReadConstBuffer, - {resource, Value(0u)}, buffer_memory); + const auto resource = + buffer.Emit(ValueOpcode::GetBufferResource, + {Value(0x1000u), Value(0u), Value(16u), Value(0u)}); + buffer.EmitMemory(ValueOpcode::ReadConstBuffer, {resource, Value(0u)}, + buffer_memory); error.clear(); Check(!BuildSrtPlan(buffer.program, &error) && error.find("incompatible scalar memory metadata") != @@ -202,11 +201,10 @@ void TestNestedSrtWalk() { void TestShaderBaseAndUserData() { Fixture fixture; const auto base = fixture.Emit(ValueOpcode::GetShaderBase); - const auto pair = fixture.Emit(ValueOpcode::UnpackUint2x32, {base}); const auto low = - fixture.Emit(ValueOpcode::CompositeExtractU32x2, {pair, Value(0u)}); + fixture.Emit(ValueOpcode::CompositeExtractU64, {base, Value(0u)}); const auto high = - fixture.Emit(ValueOpcode::CompositeExtractU32x2, {pair, Value(1u)}); + fixture.Emit(ValueOpcode::CompositeExtractU64, {base, Value(1u)}); const auto user = fixture.Emit(ValueOpcode::GetUserData, {Value(static_cast(2))}); const auto sum = fixture.Emit(ValueOpcode::IAdd32, {user, Value(4u)}); @@ -310,13 +308,12 @@ void TestRuntime64BitDescriptorOps() { const auto masked = fixture.Emit(ValueOpcode::BitwiseAnd64, {shifted, Value(uint64_t{0x0000ffff00000000ull})}); - const auto combined = fixture.Emit(ValueOpcode::BitwiseOr64, - {masked, Value(uint64_t{0xabcdu})}); - const auto pair = fixture.Emit(ValueOpcode::UnpackUint2x32, {combined}); + const auto combined = fixture.Emit( + ValueOpcode::IAdd64, {masked, Value(uint64_t{0x000000010000abcdull})}); const auto low = - fixture.Emit(ValueOpcode::CompositeExtractU32x2, {pair, Value(0u)}); + fixture.Emit(ValueOpcode::CompositeExtractU64, {combined, Value(0u)}); const auto high = - fixture.Emit(ValueOpcode::CompositeExtractU32x2, {pair, Value(1u)}); + fixture.Emit(ValueOpcode::CompositeExtractU64, {combined, Value(1u)}); fixture.Plan(); fixture.program.values->descriptor_sources.push_back( {.dwords = {low, high}, .dword_count = 2}); @@ -324,8 +321,8 @@ void TestRuntime64BitDescriptorOps() { DescriptorValue result; std::string error; Check(EvaluateDescriptorSource(fixture.program, 0, 0, {}, result, &error) && - result.dwords[0] == 0xabcdu && result.dwords[1] == 0x1234u, - "64-bit typed descriptor shift/mask evaluation is incorrect"); + result.dwords[0] == 0xabcdu && result.dwords[1] == 0x1235u, + "64-bit typed descriptor arithmetic evaluation is incorrect"); } void TestConstantBufferBounds() { @@ -458,9 +455,9 @@ int main() { // Keep this focused standalone target self-contained by amalgamating its small // typed-IR implementation set. #include "../src/graphics/shader/recompiler/ir/Block.cpp" -#include "../src/graphics/shader/recompiler/ir/passes/ConstantPropagation.cpp" -#include "../src/graphics/shader/recompiler/ir/passes/DeadCodeElimination.cpp" #include "../src/graphics/shader/recompiler/ir/Type.cpp" #include "../src/graphics/shader/recompiler/ir/Value.cpp" -#include "../src/graphics/shader/recompiler/ir/opcodes/ValueOpcodes.cpp" #include "../src/graphics/shader/recompiler/ir/ValueProgram.cpp" +#include "../src/graphics/shader/recompiler/ir/opcodes/ValueOpcodes.cpp" +#include "../src/graphics/shader/recompiler/ir/passes/ConstantPropagation.cpp" +#include "../src/graphics/shader/recompiler/ir/passes/DeadCodeElimination.cpp" diff --git a/tests/shaderCfgTests.cpp b/tests/shaderCfgTests.cpp index eb4102df4..4c8f29fd9 100644 --- a/tests/shaderCfgTests.cpp +++ b/tests/shaderCfgTests.cpp @@ -31,6 +31,7 @@ #include #include +#include #include #include #include @@ -1746,7 +1747,6 @@ void TestNewShaderRecompilerScalarVectorAlu() { Check(SpirvContainsOpcode(result.spirv, 199), "SPIR-V binary does not contain OpBitwiseAnd"); CheckSpirvBinaryValidates(result.spirv); - } void TestNewShaderRecompilerVop3LaneReadDestinationEncoding() { @@ -3634,6 +3634,62 @@ void TestNewShaderRecompilerScalarB64Alu() { "extract"); Check(SpirvContainsOpcode(result.spirv, 205), "SPIR-V binary does not contain OpBitCount for scalar B64 bit count"); + uint32_t componentwise_u64_values = 0; + for (const auto *block : result.program.values->blocks) { + for (const auto &value : *block) { + switch (value.GetOpcode()) { + case ShaderRecompiler::IR::ValueOpcode::BitwiseAnd64: + componentwise_u64_values++; + break; + default: + break; + } + } + } + Check( + componentwise_u64_values == 0u, + "architectural B64 lane operations retained vector pack/unpack traffic"); + CheckSpirvBinaryValidates(result.spirv); +} + +void TestNewShaderRecompilerScalarB64LaneLowering() { + const uint32_t shader[] = { + EncodeSop1(0x04, 2, 0), // s_mov_b64 s[2:3], s[0:1] + EncodeSop1(0x08, 4, 2), // s_not_b64 s[4:5], s[2:3] + EncodeSop2(0x0f, 6, 2, 4), // s_and_b64 s[6:7], s[2:3], s[4:5] + EncodeSop2(0x11, 8, 6, 2), // s_or_b64 s[8:9], s[6:7], s[2:3] + EncodeSop2(0x13, 10, 8, 4), // s_xor_b64 s[10:11], s[8:9], s[4:5] + EncodeSopc(0x06, 0, 1), // s_cmp_eq_u32 s0, s1 + EncodeSop2(0x0b, 12, 10, 2), // s_cselect_b64 s[12:13], s[10:11], s[2:3] + EncodeSop2(0x15, 14, 12, 4), // s_andn2_b64 s[14:15], s[12:13], s[4:5] + EncodeVop1(0x01, 0, 14), // v_mov_b32 v0, s14 + EncodeExp0(0x0c, 0x1), EncodeExp1(0, 0, 0, 0), EncodeSopp(0x01), + }; + const uint32_t user_data[] = {0x01234567u, 0x89abcdefu}; + auto options = MakeCompileOptions(ShaderType::Vertex); + options.user_data = user_data; + options.user_data_count = static_cast(std::size(user_data)); + options.dump_ir = true; + + ShaderRecompiler::CompileResult result; + std::string error; + Check(ShaderRecompiler::TryRecompile(shader, options, result, &error), + error.c_str()); + uint32_t componentwise_u64_values = 0; + for (const auto *block : result.program.values->blocks) { + for (const auto &value : *block) { + switch (value.GetOpcode()) { + case ShaderRecompiler::IR::ValueOpcode::BitwiseAnd64: + componentwise_u64_values++; + break; + default: + break; + } + } + } + Check( + componentwise_u64_values == 0u, + "architectural B64 lane operations retained vector pack/unpack traffic"); CheckSpirvBinaryValidates(result.spirv); } @@ -5499,16 +5555,16 @@ void TestGraphicsCreateInterpolantMapping() { void TestNewShaderRecompilerNativeWideScalarMemoryIr() { const uint32_t shader[] = { - EncodeSmem0(0x00, 0, 4), 125u << 25u, - EncodeSmem0(0x01, 2, 4), 125u << 25u, - EncodeSmem0(0x02, 4, 4), 125u << 25u, - EncodeSmem0(0x03, 8, 4), 125u << 25u, - EncodeSmem0(0x04, 92, 4), 125u << 25u, - EncodeSmem0(0x08, 32, 4), 125u << 25u, + EncodeSmem0(0x00, 0, 4), 125u << 25u, + EncodeSmem0(0x01, 2, 4), 125u << 25u, + EncodeSmem0(0x02, 4, 4), 125u << 25u, + EncodeSmem0(0x03, 8, 4), 125u << 25u, + EncodeSmem0(0x04, 92, 4), 125u << 25u, + EncodeSmem0(0x08, 32, 4), 125u << 25u, EncodeSmem0(0x09, 106, 4), 125u << 25u, - EncodeSmem0(0x0a, 36, 4), 125u << 25u, - EncodeSmem0(0x0b, 40, 4), 125u << 25u, - EncodeSmem0(0x0c, 48, 4), 125u << 25u, + EncodeSmem0(0x0a, 36, 4), 125u << 25u, + EncodeSmem0(0x0b, 40, 4), 125u << 25u, + EncodeSmem0(0x0c, 48, 4), 125u << 25u, EncodeSopp(0x01), }; ShaderRecompiler::Decoder::Program decoded; @@ -5517,9 +5573,8 @@ void TestNewShaderRecompilerNativeWideScalarMemoryIr() { std::string error; Check(ShaderRecompiler::Decoder::DecodeProgram(shader, decoded, &error) && ShaderRecompiler::CFG::BuildGraph(decoded, graph, &error) && - ShaderRecompiler::IR::LowerProgram(decoded, graph, - ShaderType::Compute, 64, ir, - &error), + ShaderRecompiler::IR::LowerProgram( + decoded, graph, ShaderType::Compute, 64, ir, &error), error.c_str()); std::vector address_widths; std::vector buffer_widths; @@ -5528,8 +5583,7 @@ void TestNewShaderRecompilerNativeWideScalarMemoryIr() { for (const auto &inst : block.instructions) { if (inst.op == ShaderRecompiler::IR::Opcode::SLoadDword) { address_widths.push_back(inst.memory.data_dwords); - } else if (inst.op == - ShaderRecompiler::IR::Opcode::SBufferLoadDword) { + } else if (inst.op == ShaderRecompiler::IR::Opcode::SBufferLoadDword) { buffer_widths.push_back(inst.memory.data_dwords); if (inst.memory.data_dwords == 2u) { buffer_x2_targets_vcc = @@ -5541,22 +5595,23 @@ void TestNewShaderRecompilerNativeWideScalarMemoryIr() { } const std::vector expected{1u, 2u, 4u, 8u, 16u}; Check(address_widths == expected && buffer_widths == expected, - "SMEM x1/x2/x4/x8/x16 did not remain one width-bearing ShaderIR operation each"); + "SMEM x1/x2/x4/x8/x16 did not remain one width-bearing ShaderIR " + "operation each"); Check(buffer_x2_targets_vcc, "s_buffer_load_dwordx2 did not preserve its VCC destination span"); const auto reject = [&](uint32_t opcode, uint32_t dst, uint32_t sbase, const char *reason) { - const uint32_t invalid[] = {EncodeSmem0(opcode, dst, sbase), - 125u << 25u, EncodeSopp(0x01)}; + const uint32_t invalid[] = {EncodeSmem0(opcode, dst, sbase), 125u << 25u, + EncodeSopp(0x01)}; ShaderRecompiler::Decoder::Program invalid_decoded; ShaderRecompiler::CFG::Graph invalid_graph; ShaderRecompiler::IR::Program invalid_ir; std::string invalid_error; Check(ShaderRecompiler::Decoder::DecodeProgram(invalid, invalid_decoded, - &invalid_error) && + &invalid_error) && ShaderRecompiler::CFG::BuildGraph(invalid_decoded, invalid_graph, - &invalid_error) && + &invalid_error) && !ShaderRecompiler::IR::LowerProgram( invalid_decoded, invalid_graph, ShaderType::Compute, 64, invalid_ir, &invalid_error) && @@ -6572,7 +6627,6 @@ void TestNewShaderRecompilerDsAddtidLowering() { Check(SpirvContainsOpcode(result.spirv, 199), "SPIR-V binary does not contain OpBitwiseAnd"); CheckSpirvBinaryValidates(result.spirv); - } void TestNewShaderRecompilerDsFloatMinMaxLowering() { @@ -6693,6 +6747,53 @@ void TestNewShaderRecompilerCfgConsecutiveNativePhis() { CheckSpirvBinaryValidates(result.spirv); } +void TestNewShaderRecompilerStructuredU64Phi() { + using namespace ShaderRecompiler; + + const uint32_t shader[] = { + EncodeSopc(0x06, 0, 1), EncodeSopp(0x05, 2), EncodeSMovB32(2, 129), + EncodeSopp(0x02, 1), EncodeSMovB32(2, 130), EncodeVop1(0x01, 0, 2), + EncodeSopp(0x01), + }; + auto options = MakeCompileOptions(ShaderType::Compute); + CompileResult result; + std::string error; + Check(TryRecompile(shader, options, result, &error), error.c_str()); + Check(!result.program.dispatcher_fallback, + "U64 Phi fixture did not select structured mode"); + + auto program = result.program; + IR::Block *join = nullptr; + for (auto *block : program.values->blocks) { + if (block->ImmPredecessors().size() == 2u) { + join = block; + break; + } + } + Check(join != nullptr, "U64 Phi fixture has no two-parent join block"); + auto &phi = *join->PrependNewInst(join->begin(), IR::ValueOpcode::Phi); + phi.SetFlags(IR::Type::U64); + phi.AddPhiOperand(join->ImmPredecessors()[0], + IR::Value(uint64_t{0x1111111122222222ull})); + phi.AddPhiOperand(join->ImmPredecessors()[1], + IR::Value(uint64_t{0x3333333344444444ull})); + IR::IREmitter use(join); + use.Emit(IR::ValueOpcode::CompositeExtractU64, + {IR::Value(&phi), IR::Value(1u)}); + Check(Spirv::AnalyzeProgramRequirements(program, &error), error.c_str()); + + std::vector spirv; + Check(Spirv::EmitProgram(program, result.resources, options.input_info, spirv, + &error), + error.c_str()); + CheckSpirvBinaryValidates(spirv); + const auto before = MeasureSpirv(result.spirv); + const auto after = MeasureSpirv(spirv); + Check(after.phis == before.phis + 1u && + after.function_variables == before.function_variables, + "structured U64 Phi was not emitted as a native vector OpPhi"); +} + void TestNewShaderRecompilerCfgTerminalExitMergePS() { const uint32_t shader[] = { EncodeSopp(0x04, 2), // s_cbranch_scc0 end @@ -7861,6 +7962,91 @@ void TestNewShaderRecompilerDispatcherSpillsU32x3() { "dispatcher did not use one canonical U32x3 spill slot"); } +void TestNewShaderRecompilerU64PairLowering() { + using namespace ShaderRecompiler; + + const uint32_t empty_shader[] = {EncodeSopp(0x01)}; + auto options = MakeCompileOptions(ShaderType::Compute); + CompileResult result; + std::string error; + Check(TryRecompile(empty_shader, options, result, &error), error.c_str()); + + auto program = result.program; + IR::IREmitter ir(program.values->blocks.front()); + const auto lane = ir.Emit(IR::ValueOpcode::LaneId); + const auto high = ir.Emit(IR::ValueOpcode::IAdd32, {lane, IR::Value(1u)}); + const auto base = + ir.Emit(IR::ValueOpcode::CompositeConstructU64, {lane, high}); + const auto masked = + ir.Emit(IR::ValueOpcode::BitwiseAnd64, + {base, IR::Value(uint64_t{0xff00ff00ff00ff00ull})}); + const auto sum = ir.Emit(IR::ValueOpcode::IAdd64, {masked, base}); + const auto product = ir.Emit(IR::ValueOpcode::IMul64, {sum, base}); + const auto shifted = + ir.Emit(IR::ValueOpcode::ShiftRightArithmetic64, {product, lane}); + ir.Emit(IR::ValueOpcode::IEqual64, {shifted, base}); + ir.Emit(IR::ValueOpcode::INotEqual64, {shifted, base}); + ir.Emit(IR::ValueOpcode::ULessThan64, {shifted, base}); + ir.Emit(IR::ValueOpcode::SLessThan64, {shifted, base}); + ir.Emit(IR::ValueOpcode::BitCount64, {shifted}); + ir.Emit(IR::ValueOpcode::WqmU64, {shifted}); + for (const uint32_t count : {1u, 31u, 32u, 33u, 63u}) { + ir.Emit(IR::ValueOpcode::ShiftLeftLogical64, {base, IR::Value(count)}); + ir.Emit(IR::ValueOpcode::ShiftRightLogical64, {base, IR::Value(count)}); + ir.Emit(IR::ValueOpcode::ShiftRightArithmetic64, {base, IR::Value(count)}); + } + Check(Spirv::AnalyzeProgramRequirements(program, &error), error.c_str()); + + std::vector spirv; + Check(Spirv::EmitProgram(program, result.resources, options.input_info, spirv, + &error), + error.c_str()); + CheckSpirvBinaryValidates(spirv); + const auto source = DisassembleSpirvBinary(spirv); + Check(!Common::ContainsStr(source, "OpCapability Int64") && + !Common::ContainsStr(source, "OpTypeInt 64"), + "portable pair-U64 lowering introduced native shader Int64"); + Check(SpirvInstructionOpcodeCount(spirv, 149u) == 1u, + "pair-U64 addition did not use exactly one carry instruction"); + Check(SpirvInstructionOpcodeCount(spirv, 154u) == 1u && + SpirvInstructionOpcodeCount(spirv, 155u) == 1u, + "pair-U64 equality did not reduce its vector comparison with Any/All"); + const auto direct_metrics = MeasureSpirv(spirv); + Check( + direct_metrics.words <= 940u && direct_metrics.instructions <= 199u && + direct_metrics.type_vectors == 2u && direct_metrics.phis == 0u && + direct_metrics.function_variables == 0u, + "portable pair-U64 lowering exceeded its declaration/code-size ratchet"); + + const uint32_t dispatcher_shader[] = { + EncodeSopp(0x05, 2), EncodeSopp(0x02, 0), EncodeSopp(0x05, 0xfffeu), + EncodeSopp(0x02, 0xfffeu), EncodeSopp(0x01), + }; + Check(TryRecompile(dispatcher_shader, options, result, &error), + error.c_str()); + Check(result.program.dispatcher_fallback && + result.program.values->blocks.size() >= 2u, + "U64 spill fixture did not select dispatcher mode"); + program = result.program; + IR::IREmitter definition(program.values->blocks[0]); + const auto vector = definition.Emit(IR::ValueOpcode::CompositeConstructU64, + {IR::Value(1u), IR::Value(2u)}); + IR::IREmitter use(program.values->blocks[1]); + use.Emit(IR::ValueOpcode::CompositeExtractU64, {vector, IR::Value(1u)}); + Check(Spirv::AnalyzeProgramRequirements(program, &error), error.c_str()); + spirv.clear(); + Check(Spirv::EmitProgram(program, result.resources, options.input_info, spirv, + &error), + error.c_str()); + CheckSpirvBinaryValidates(spirv); + const auto before = MeasureSpirv(result.spirv); + const auto after = MeasureSpirv(spirv); + Check(after.function_variables == before.function_variables + 1u && + after.loads == before.loads + 1u && + after.stores == before.stores + 1u, + "dispatcher did not use one canonical U64 vector spill slot"); +} + void TestComputeShaderInputWaveSize() { const auto decode_wave_size = [](uint32_t rsrc1) { const bool wave32 = (((rsrc1 >> Pm4::COMPUTE_PGM_RSRC1_W32_EN_SHIFT) & @@ -8552,6 +8738,40 @@ void TestWqmConstantPropagation() { check_invalid_signature(ValueOpcode::WqmU64, Value(uint64_t{1}), Value(true)); check_invalid_signature(ValueOpcode::WqmMask, Value(true), Value(uint64_t{1})); + + ValueProgram shifts; + shifts.block_storage.push_back(std::make_unique()); + shifts.blocks.push_back(shifts.block_storage.back().get()); + shifts.block_info.emplace_back(); + IREmitter shift_ir(shifts.blocks.front()); + struct ShiftCase { + Value value; + uint64_t expected; + }; + std::vector shift_cases; + for (const uint64_t source : + {uint64_t{0x0123456789abcdefull}, uint64_t{0xf123456789abcdefull}}) { + for (const uint32_t count : {0u, 1u, 31u, 32u, 33u, 63u, 64u, 65u}) { + const uint32_t amount = count & 63u; + shift_cases.push_back({shift_ir.Emit(ValueOpcode::ShiftLeftLogical64, + {Value(source), Value(count)}), + source << amount}); + shift_cases.push_back({shift_ir.Emit(ValueOpcode::ShiftRightLogical64, + {Value(source), Value(count)}), + source >> amount}); + shift_cases.push_back( + {shift_ir.Emit(ValueOpcode::ShiftRightArithmetic64, + {Value(source), Value(count)}), + static_cast(std::bit_cast(source) >> amount)}); + } + } + ConstantPropagationPass(shifts.blocks); + for (const auto &test : shift_cases) { + const auto value = test.value.Resolve(); + Check(value.IsImmediate() && value.GetType() == Type::U64 && + value.U64() == test.expected, + "pair-U64 shift propagation violated the masked RDNA2 count"); + } } void TestNativeWideValueValidation() { @@ -8575,20 +8795,18 @@ void TestNativeWideValueValidation() { ir.Emit(opcode, {resource, Value(0u), Value(0u), Value(0u), Value(true)}, flags); }; - const auto append_scalar_read = [](ValueProgram &program, - ValueOpcode opcode, + const auto append_scalar_read = [](ValueProgram &program, ValueOpcode opcode, MemoryFlags flags) { IREmitter ir(program.blocks.front()); if (opcode == ValueOpcode::ReadConstBuffer) { - const auto resource = ir.Emit( - ValueOpcode::GetBufferResource, - {Value(0u), Value(0u), Value(0u), Value(0u)}); + const auto resource = + ir.Emit(ValueOpcode::GetBufferResource, + {Value(0u), Value(0u), Value(0u), Value(0u)}); ir.Emit(opcode, {resource, Value(0u)}, flags); } else { const auto resource = ir.Emit(ValueOpcode::GetAddressResource, {Value(0u), Value(0u)}); - ir.Emit(opcode, - {resource, Value(0u), Value(0u), Value(true)}, flags); + ir.Emit(opcode, {resource, Value(0u), Value(0u), Value(true)}, flags); } }; const auto check_rejected = [](const ValueProgram &program, @@ -8695,6 +8913,24 @@ void TestNativeWideValueValidation() { ir.Emit(ValueOpcode::CompositeExtractU32x3, {vector, Value(3u)}); check_rejected(program, "invalid component index"); } + { + auto program = make_program(); + IREmitter ir(program.blocks.front()); + const auto vector = + ir.Emit(ValueOpcode::CompositeConstructU64, {Value(1u), Value(2u)}); + ir.Emit(ValueOpcode::CompositeExtractU64, {vector, Value(2u)}); + check_rejected(program, "invalid component index"); + } + { + auto program = make_program(); + IREmitter ir(program.blocks.front()); + const auto vector = + ir.Emit(ValueOpcode::CompositeConstructU64, {Value(1u), Value(2u)}); + const auto dynamic_index = + ir.Emit(ValueOpcode::IAdd32, {Value(0u), Value(1u)}); + ir.Emit(ValueOpcode::CompositeExtractU64, {vector, dynamic_index}); + check_rejected(program, "invalid component index"); + } const auto append_shared = [](ValueProgram &program, ValueOpcode opcode, MemoryFlags flags) { IREmitter ir(program.blocks.front()); @@ -8920,8 +9156,8 @@ void TestNewShaderRecompilerPerInvocationMasksWithoutMirrors() { !Common::ContainsStr(DisassembleSpirvBinary(result.spirv), "OpGroupNonUniformBallot"), "native scalar WQM retained frontend expansion or subgroup machinery"); - Check(SpirvInstructionOpcodeCount(result.spirv, 132u) == 2u && - SpirvInstructionOpcodeCount(result.spirv, 194u) == 4u, + Check(SpirvInstructionOpcodeCount(result.spirv, 132u) == 1u && + SpirvInstructionOpcodeCount(result.spirv, 194u) == 2u, "native scalar WQM emitter was folded away or did not use the compact " "path"); @@ -9956,7 +10192,8 @@ void TestScalarMemorySourcesCapturedBeforeWrites() { error.c_str()); Check(flat.size() == table.size() && std::equal(flat.begin(), flat.end(), table.begin()), - "overlapping scalar-memory load did not capture its sources before writes"); + "overlapping scalar-memory load did not capture its sources before " + "writes"); CheckFlattenedReadSlots( ir, 4, "overlapping scalar-memory patch used the wrong flat offsets"); }; @@ -10003,16 +10240,16 @@ void TestScalarMemoryLoadCrossesIntoVcc() { void TestScalarMemoryUnusedTailDce() { const uint32_t shader[] = { EncodeSmem0(0x0c, 16, 0), - 4u << 25u, // s_buffer_load_dwordx16 s[16:31], s[0:3], s4 - EncodeVop1(0x01, 0, 16), // v_mov_b32 v0, s16 + 4u << 25u, // s_buffer_load_dwordx16 s[16:31], s[0:3], s4 + EncodeVop1(0x01, 0, 16), // v_mov_b32 v0, s16 EncodeMubuf0(0x1c), - EncodeMubuf1(0, 2, 1), // keep only the first loaded component live + EncodeMubuf1(0, 2, 1), // keep only the first loaded component live EncodeSopp(0x01), }; ShaderRecompiler::IR::Program ir; std::string error; - Check(LowerTypedPlanning(shader, static_cast(std::size(shader)), - ir, &error), + Check(LowerTypedPlanning(shader, static_cast(std::size(shader)), ir, + &error), error.c_str()); uint32_t reads = 0; for (const auto *block : ir.values->blocks) { @@ -10029,8 +10266,9 @@ void TestScalarMemoryUnusedTailDce() { "live x16 SMEM component lost its native-width metadata"); } } - Check(reads == 1u, - "grouped x16 SMEM prevented dead component reads from being eliminated"); + Check( + reads == 1u, + "grouped x16 SMEM prevented dead component reads from being eliminated"); } void TestResourceTrackingRealDensePatching() { @@ -10794,6 +11032,7 @@ int main() { TestNewShaderRecompilerSMovB32(); TestNewShaderRecompilerNativeWideScalarMemoryIr(); TestNewShaderRecompilerNativeWideBufferIr(); + TestNewShaderRecompilerScalarB64LaneLowering(); TestNewShaderRecompilerMubufFormatLowering(); TestNewShaderRecompilerTypedBufferLowering(); TestNewShaderRecompilerDsReadWrite2Lowering(); @@ -10813,6 +11052,7 @@ int main() { TestNewShaderRecompilerCfgStraightLine(); TestNewShaderRecompilerCfgIfElse(); TestNewShaderRecompilerCfgConsecutiveNativePhis(); + TestNewShaderRecompilerStructuredU64Phi(); TestNewShaderRecompilerCfgTerminalExitMergePS(); TestNewShaderRecompilerCfgPostEndTargetMergePS(); TestNewShaderRecompilerCfgLoopBreakContinue(); @@ -10841,6 +11081,7 @@ int main() { TestNewShaderRecompilerCfgExternallyEnteredSelectionDispatcher(); TestNewShaderRecompilerCfgIrreducibleDispatcher(); TestNewShaderRecompilerDispatcherSpillsU32x3(); + TestNewShaderRecompilerU64PairLowering(); TestComputeShaderInputWaveSize(); TestNewShaderRecompilerBufferLoadsGuardedByExec(); TestNewShaderRecompilerBufferAtomicsGuardedByBounds();