From 5b826c6e4ad2c68c8841fa037e11f63f00d61807 Mon Sep 17 00:00:00 2001 From: nmzik Date: Mon, 5 Oct 2026 20:41:03 +0400 Subject: [PATCH] shader: emit native 64-bit operations where compatible --- .../backend/spirv/spirvEmitterAlu.cpp | 244 +++--------------- .../backend/spirv/spirvEmitterAnalysis.cpp | 4 +- .../backend/spirv/spirvEmitterBvh.cpp | 41 +-- .../backend/spirv/spirvEmitterFlow.cpp | 6 +- .../backend/spirv/spirvEmitterImage.cpp | 6 +- .../backend/spirv/spirvEmitterInstructions.h | 50 ++-- .../backend/spirv/spirvEmitterInternal.h | 11 +- .../backend/spirv/spirvEmitterMemory.cpp | 156 +++++------ .../spirv/spirvEmitterMemoryHelpers.cpp | 9 +- .../backend/spirv/spirvEmitterModule.cpp | 21 +- .../recompiler/frontend/translate/Control.cpp | 1 + .../recompiler/frontend/translate/Integer.cpp | 34 +-- .../recompiler/frontend/translate/Scalar.cpp | 13 +- .../ir/passes/ConstantPropagation.cpp | 11 + tests/ShaderRecompilerComputeTests.cpp | 222 ++++++++++++++-- tests/shaderCfgTests.cpp | 107 ++++++-- 16 files changed, 500 insertions(+), 436 deletions(-) diff --git a/src/graphics/shader/recompiler/backend/spirv/spirvEmitterAlu.cpp b/src/graphics/shader/recompiler/backend/spirv/spirvEmitterAlu.cpp index 2d710558f..e51dd942f 100644 --- a/src/graphics/shader/recompiler/backend/spirv/spirvEmitterAlu.cpp +++ b/src/graphics/shader/recompiler/backend/spirv/spirvEmitterAlu.cpp @@ -17,21 +17,8 @@ Pair ExtractPair(EmitterState& state, uint32_t value) { return result; } -uint32_t MakePair(EmitterState& state, uint32_t low, uint32_t high) { - const auto result = state.builder.AllocateId(); - state.builder.AddFunction(spv::OpCompositeConstruct, TypeU64(state), result, low, high); - return result; -} - -uint32_t CompareEqual64(EmitterState& state, uint32_t lhs_value, uint32_t rhs_value, - bool not_equal) { - const auto compare = Binary(state, not_equal ? spv::OpINotEqual : spv::OpIEqual, - TypeBoolVector(state, 2), lhs_value, rhs_value); - return Unary(state, not_equal ? spv::OpAny : spv::OpAll, TypeBool(state), compare); -} - uint32_t EmitMinMaxF64(EmitterState& state, uint32_t lhs, uint32_t rhs, bool max_value) { - const auto bits_type = TypeU64(state); + const auto bits_type = TypeU32Vector(state, 2); const auto lhs_bits = Unary(state, spv::OpBitcast, bits_type, lhs); const auto rhs_bits = Unary(state, spv::OpBitcast, bits_type, rhs); const auto ordered = Binary(state, max_value ? spv::OpFOrdGreaterThan : spv::OpFOrdLessThan, @@ -52,17 +39,6 @@ uint32_t EmitMinMaxF64(EmitterState& state, uint32_t lhs, uint32_t rhs, bool max rhs, result); } -uint32_t CompareOrdered64(EmitterState& state, uint32_t lhs_value, uint32_t rhs_value, - spv::Op high_compare, spv::Op low_compare) { - const auto lhs = ExtractPair(state, lhs_value); - const auto rhs = ExtractPair(state, rhs_value); - const auto high_equal = Binary(state, spv::OpIEqual, TypeBool(state), lhs.high, rhs.high); - const auto high_result = Binary(state, high_compare, TypeBool(state), lhs.high, rhs.high); - const auto low_result = Binary(state, low_compare, TypeBool(state), lhs.low, rhs.low); - const auto low_path = Binary(state, spv::OpLogicalAnd, TypeBool(state), high_equal, low_result); - return Binary(state, spv::OpLogicalOr, TypeBool(state), high_result, low_path); -} - uint32_t EmitMulHigh(EmitterState& state, uint32_t lhs, uint32_t rhs, bool signed_value) { const auto operand_type = signed_value ? TypeI32(state) : TypeU32(state); const auto pair_type = signed_value ? TypeI32Pair(state) : TypeU32Pair(state); @@ -80,86 +56,6 @@ uint32_t EmitMulHigh(EmitterState& state, uint32_t lhs, uint32_t rhs, bool signe return signed_value ? Unary(state, spv::OpBitcast, TypeU32(state), high) : high; } -uint32_t EmitShift64(EmitterState& state, spv::Op opcode, uint32_t value, uint32_t shift) { - const auto pair = ExtractPair(state, value); - const auto amount = EmitAndConstant(state, shift, 63u); - const auto word_shift = EmitAndConstant(state, amount, 31u); - const auto at_least_32 = - Binary(state, spv::OpUGreaterThanEqual, TypeBool(state), amount, ConstantU32(state, 32)); - const auto nonzero = - Binary(state, spv::OpINotEqual, TypeBool(state), amount, ConstantU32(state, 0)); - const auto carry_count = EmitAndConstant( - state, Binary(state, spv::OpISub, TypeU32(state), ConstantU32(state, 32), word_shift), 31u); - if (opcode == spv::OpShiftLeftLogical) { - const auto low = Binary(state, opcode, TypeU32(state), pair.low, word_shift); - const auto carry = - Select(state, TypeU32(state), nonzero, - Binary(state, spv::OpShiftRightLogical, TypeU32(state), pair.low, carry_count), - ConstantU32(state, 0)); - const auto high = - Binary(state, spv::OpBitwiseOr, TypeU32(state), - Binary(state, opcode, TypeU32(state), pair.high, word_shift), carry); - return MakePair(state, - Select(state, TypeU32(state), at_least_32, ConstantU32(state, 0), low), - Select(state, TypeU32(state), at_least_32, low, high)); - } - const auto high = Binary(state, opcode, TypeU32(state), pair.high, word_shift); - const auto carry = - Select(state, TypeU32(state), nonzero, - Binary(state, spv::OpShiftLeftLogical, TypeU32(state), pair.high, carry_count), - ConstantU32(state, 0)); - const auto low = Binary( - state, spv::OpBitwiseOr, TypeU32(state), - Binary(state, spv::OpShiftRightLogical, TypeU32(state), pair.low, word_shift), carry); - const auto fill = opcode == spv::OpShiftRightArithmetic - ? Binary(state, opcode, TypeU32(state), pair.high, ConstantU32(state, 31)) - : ConstantU32(state, 0); - return MakePair(state, Select(state, TypeU32(state), at_least_32, high, low), - Select(state, TypeU32(state), at_least_32, fill, high)); -} - -uint32_t EmitConstantShift64(EmitterState& state, spv::Op opcode, uint32_t value, uint32_t shift) { - shift &= 63u; - if (shift == 0u) { - return value; - } - const auto pair = ExtractPair(state, value); - if (opcode == spv::OpShiftLeftLogical) { - if (shift < 32u) { - const auto low = Binary(state, spv::OpShiftLeftLogical, TypeU32(state), pair.low, - ConstantU32(state, shift)); - const auto high = Binary(state, spv::OpBitwiseOr, TypeU32(state), - Binary(state, spv::OpShiftLeftLogical, TypeU32(state), - pair.high, ConstantU32(state, shift)), - Binary(state, spv::OpShiftRightLogical, TypeU32(state), - pair.low, ConstantU32(state, 32u - shift))); - return MakePair(state, low, high); - } - return MakePair(state, ConstantU32(state, 0), - shift == 32u ? pair.low - : Binary(state, spv::OpShiftLeftLogical, TypeU32(state), - pair.low, ConstantU32(state, shift - 32u))); - } - if (shift < 32u) { - const auto low = Binary(state, spv::OpBitwiseOr, TypeU32(state), - Binary(state, spv::OpShiftRightLogical, TypeU32(state), pair.low, - ConstantU32(state, shift)), - Binary(state, spv::OpShiftLeftLogical, TypeU32(state), pair.high, - ConstantU32(state, 32u - shift))); - const auto high = - Binary(state, opcode, TypeU32(state), pair.high, ConstantU32(state, shift)); - return MakePair(state, low, high); - } - const auto high = opcode == spv::OpShiftRightArithmetic - ? Binary(state, spv::OpShiftRightArithmetic, TypeU32(state), pair.high, - ConstantU32(state, 31u)) - : ConstantU32(state, 0); - const auto low = shift == 32u ? pair.high - : Binary(state, opcode, TypeU32(state), pair.high, - ConstantU32(state, shift - 32u)); - return MakePair(state, low, high); -} - uint32_t EmitMinMax3(EmitterState& state, uint32_t a, uint32_t b, uint32_t c, bool signed_value, bool max_value) { const auto ab = signed_value ? EmitMinMaxI32Value(state, a, b, max_value) @@ -259,57 +155,15 @@ uint32_t EmitFPMedTri32(EmitterState& state, uint32_t a, uint32_t b, uint32_t c) } uint32_t EmitFindUMsb64(EmitterState& state, uint32_t value) { - const auto pair = ExtractPair(state, value); - const auto high_i = state.builder.AllocateId(); - const auto low_i = state.builder.AllocateId(); - state.builder.AddFunction(spv::OpExtInst, TypeI32(state), high_i, GlslStd450(state), - GLSLstd450FindUMsb, pair.high); - state.builder.AddFunction(spv::OpExtInst, TypeI32(state), low_i, GlslStd450(state), - GLSLstd450FindUMsb, pair.low); - const auto high = Unary(state, spv::OpBitcast, TypeU32(state), high_i); - const auto low = Unary(state, spv::OpBitcast, TypeU32(state), low_i); - const auto high_nonzero = - Binary(state, spv::OpINotEqual, TypeBool(state), pair.high, ConstantU32(state, 0)); - return Select(state, TypeU32(state), high_nonzero, - Binary(state, spv::OpIAdd, TypeU32(state), high, ConstantU32(state, 32)), low); -} - -uint32_t EmitIMul64(EmitterState& state, uint32_t lhs_value, uint32_t rhs_value) { - const auto lhs = ExtractPair(state, lhs_value); - const auto rhs = ExtractPair(state, rhs_value); - const auto low = Binary(state, spv::OpIMul, TypeU32(state), lhs.low, rhs.low); - const auto high0 = EmitMulHigh(state, lhs.low, rhs.low, false); - const auto high1 = Binary(state, spv::OpIMul, TypeU32(state), lhs.low, rhs.high); - const auto high2 = Binary(state, spv::OpIMul, TypeU32(state), lhs.high, rhs.low); - return MakePair(state, low, - Binary(state, spv::OpIAdd, TypeU32(state), - Binary(state, spv::OpIAdd, TypeU32(state), high0, high1), high2)); -} - -uint32_t EmitISub64(EmitterState& state, uint32_t lhs_value, uint32_t rhs_value) { - const auto lhs = ExtractPair(state, lhs_value); - const auto rhs = ExtractPair(state, rhs_value); - const auto low = Binary(state, spv::OpISub, TypeU32(state), lhs.low, rhs.low); - const auto borrow = Binary(state, spv::OpULessThan, TypeBool(state), lhs.low, rhs.low); - const auto borrow_u32 = - Select(state, TypeU32(state), borrow, ConstantU32(state, 1), ConstantU32(state, 0)); - const auto high0 = Binary(state, spv::OpISub, TypeU32(state), lhs.high, rhs.high); - const auto high = Binary(state, spv::OpISub, TypeU32(state), high0, borrow_u32); - return MakePair(state, low, high); -} - -uint32_t EmitIAdd64(EmitterState& state, uint32_t lhs_value, uint32_t rhs_value) { - const auto lhs = ExtractPair(state, lhs_value); - const auto rhs = ExtractPair(state, rhs_value); - const auto low_pair = state.builder.AllocateId(); - const auto low = state.builder.AllocateId(); - const auto carry = state.builder.AllocateId(); - state.builder.AddFunction(spv::OpIAddCarry, TypeU32Pair(state), low_pair, lhs.low, rhs.low); - state.builder.AddFunction(spv::OpCompositeExtract, TypeU32(state), low, low_pair, 0); - state.builder.AddFunction(spv::OpCompositeExtract, TypeU32(state), carry, low_pair, 1); - const auto high0 = Binary(state, spv::OpIAdd, TypeU32(state), lhs.high, rhs.high); - const auto high = Binary(state, spv::OpIAdd, TypeU32(state), high0, carry); - return MakePair(state, low, high); + // GLSL FindUMsb is restricted to 32-bit components. + const auto unpacked = Unary(state, spv::OpBitcast, TypeU32Vector(state, 2), value); + const auto pair = + ExtractPair(state, EmitExt(state, TypeU32Vector(state, 2), GLSLstd450FindUMsb, {unpacked})); + const auto high_found = + Binary(state, spv::OpINotEqual, TypeBool(state), pair.high, ConstantU32(state, UINT32_MAX)); + return Select(state, TypeU32(state), high_found, + Binary(state, spv::OpIAdd, TypeU32(state), pair.high, ConstantU32(state, 32)), + pair.low); } uint32_t EmitConvertU16U32(EmitterState& state, uint32_t arg0) { @@ -338,7 +192,7 @@ uint32_t EmitConvertU32F32(EmitterState& state, uint32_t arg0) { uint32_t EmitConvertF32F64(EmitterState& state, uint32_t arg0) { const auto converted = Unary(state, spv::OpFConvert, TypeF32(state), arg0); const auto source = EmitNative( - state, Unary(state, spv::OpBitcast, TypeU64(state), arg0), 1u); + state, Unary(state, spv::OpBitcast, TypeU32Vector(state, 2), arg0), 1u); const auto exponent = EmitAndConstant(state, source, 0x7ff00000u); const auto overflow = Binary(state, spv::OpLogicalAnd, TypeBool(state), @@ -356,7 +210,21 @@ uint32_t EmitConvertF64F32(EmitterState& state, uint32_t arg0) { EmitFlushF32DenormToSignedZero(state, arg0)); } +uint32_t EmitCompositeConstructU64(ValueEmitContext& ctx, uint32_t arg0, IR::Value arg1) { + arg1 = arg1.Resolve(); + return arg1.IsImmediate() && arg1.U32() == 0u + ? Unary(ctx.state, spv::OpUConvert, TypeU64(ctx.state), arg0) + : PackU64(ctx.state, arg0, ctx.Def(arg1)); +} + uint32_t EmitCompositeExtractU64(EmitterState& state, uint32_t arg0, IR::Value arg1) { + const auto value = arg1.U32() == 0u ? arg0 + : Binary(state, spv::OpShiftRightLogical, TypeU64(state), + arg0, ConstantU32(state, 32)); + return Unary(state, spv::OpUConvert, TypeU32(state), value); +} + +uint32_t EmitCompositeExtractU32x2(EmitterState& state, uint32_t arg0, IR::Value arg1) { return EmitNative(state, arg0, arg1.U32()); } @@ -388,33 +256,11 @@ uint32_t EmitIAbs32(EmitterState& state, uint32_t arg0) { return Select(state, TypeU32(state), negative, neg, value); } -uint32_t EmitShiftLeftLogical64(ValueEmitContext& ctx, uint32_t arg0, IR::Value arg1) { - auto& state = ctx.state; - return arg1.Resolve().IsImmediate() - ? EmitConstantShift64(state, spv::OpShiftLeftLogical, arg0, arg1.Resolve().U32()) - : EmitShift64(state, spv::OpShiftLeftLogical, arg0, ctx.Def(arg1)); -} - -uint32_t EmitShiftRightLogical64(ValueEmitContext& ctx, uint32_t arg0, IR::Value arg1) { - auto& state = ctx.state; - return arg1.Resolve().IsImmediate() - ? EmitConstantShift64(state, spv::OpShiftRightLogical, arg0, arg1.Resolve().U32()) - : EmitShift64(state, spv::OpShiftRightLogical, arg0, ctx.Def(arg1)); -} - -uint32_t EmitShiftRightArithmetic64(ValueEmitContext& ctx, uint32_t arg0, IR::Value arg1) { - auto& state = ctx.state; - return arg1.Resolve().IsImmediate() - ? EmitConstantShift64(state, spv::OpShiftRightArithmetic, arg0, arg1.Resolve().U32()) - : EmitShift64(state, spv::OpShiftRightArithmetic, arg0, ctx.Def(arg1)); -} - -uint32_t EmitBitwiseAnd64(EmitterState& state, uint32_t arg0, uint32_t arg1) { - return Binary(state, spv::OpBitwiseAnd, TypeU64(state), arg0, arg1); -} - uint32_t EmitBitCount64(EmitterState& state, uint32_t arg0) { - const auto pair = ExtractPair(state, Unary(state, spv::OpBitCount, TypeU64(state), arg0)); + // Vulkan bit-count operands remain 32-bit, including when counting a U64. + const auto unpacked = Unary(state, spv::OpBitcast, TypeU32Vector(state, 2), arg0); + const auto pair = + ExtractPair(state, Unary(state, spv::OpBitCount, TypeU32Vector(state, 2), unpacked)); return Binary(state, spv::OpIAdd, TypeU32(state), pair.low, pair.high); } @@ -468,38 +314,6 @@ uint32_t EmitUMedTri32(EmitterState& state, uint32_t arg0, uint32_t arg1, uint32 return EmitMed3(state, arg0, arg1, arg2, false); } -uint32_t EmitIEqual64(EmitterState& state, uint32_t arg0, uint32_t arg1) { - return CompareEqual64(state, arg0, arg1, false); -} - -uint32_t EmitINotEqual64(EmitterState& state, uint32_t arg0, uint32_t arg1) { - return CompareEqual64(state, arg0, arg1, true); -} - -uint32_t EmitULessThan64(EmitterState& state, uint32_t arg0, uint32_t arg1) { - return CompareOrdered64(state, arg0, arg1, spv::OpULessThan, spv::OpULessThan); -} - -uint32_t EmitSLessThan64(EmitterState& state, uint32_t arg0, uint32_t arg1) { - return CompareOrdered64(state, arg0, arg1, spv::OpSLessThan, spv::OpULessThan); -} - -uint32_t EmitUGreaterThan64(EmitterState& state, uint32_t arg0, uint32_t arg1) { - return CompareOrdered64(state, arg0, arg1, spv::OpUGreaterThan, spv::OpUGreaterThan); -} - -uint32_t EmitSLessThanEqual64(EmitterState& state, uint32_t arg0, uint32_t arg1) { - return CompareOrdered64(state, arg0, arg1, spv::OpSLessThan, spv::OpULessThanEqual); -} - -uint32_t EmitULessThanEqual64(EmitterState& state, uint32_t arg0, uint32_t arg1) { - return CompareOrdered64(state, arg0, arg1, spv::OpULessThan, spv::OpULessThanEqual); -} - -uint32_t EmitUGreaterThanEqual64(EmitterState& state, uint32_t arg0, uint32_t arg1) { - return CompareOrdered64(state, arg0, arg1, spv::OpUGreaterThan, spv::OpUGreaterThanEqual); -} - uint32_t EmitFPIsNan32(EmitterState& state, uint32_t arg0) { return EmitNative(state, arg0, arg0); } diff --git a/src/graphics/shader/recompiler/backend/spirv/spirvEmitterAnalysis.cpp b/src/graphics/shader/recompiler/backend/spirv/spirvEmitterAnalysis.cpp index 604b1ea86..15af1ee60 100644 --- a/src/graphics/shader/recompiler/backend/spirv/spirvEmitterAnalysis.cpp +++ b/src/graphics/shader/recompiler/backend/spirv/spirvEmitterAnalysis.cpp @@ -136,8 +136,8 @@ uint32_t ImageType(EmitterState& state, const IR::ImageResource& image) { EXIT("invalid image resource class"); } const auto& info = ImageDimensionInfoFor(image.dimension); - const auto scalar_type = image.atomic64 ? TypeScalarU64(state) - : ImageScalarType(state, image.numeric_class); + const auto scalar_type = + image.atomic64 ? TypeU64(state) : ImageScalarType(state, image.numeric_class); return state.builder.Type(spv::OpTypeImage, scalar_type, info.spirv_dimension, image.depth_compare ? 1u : 0u, info.arrayed, info.multisampled, sampled, format); diff --git a/src/graphics/shader/recompiler/backend/spirv/spirvEmitterBvh.cpp b/src/graphics/shader/recompiler/backend/spirv/spirvEmitterBvh.cpp index fb590f9ec..aab43a23a 100644 --- a/src/graphics/shader/recompiler/backend/spirv/spirvEmitterBvh.cpp +++ b/src/graphics/shader/recompiler/backend/spirv/spirvEmitterBvh.cpp @@ -19,8 +19,8 @@ uint32_t Vector(EmitterState& s, uint32_t type, std::span compon uint32_t NodeWord(EmitterState& s, uint32_t block, uint32_t word) { const auto offset = Binary(s, spv::OpShiftLeftLogical, TypeU32(s), word, ConstantU32(s, 2)); - const auto address = Binary(s, spv::OpIAdd, TypeScalarU64(s), block, - Unary(s, spv::OpUConvert, TypeScalarU64(s), offset)); + const auto address = + Binary(s, spv::OpIAdd, TypeU64(s), block, Unary(s, spv::OpUConvert, TypeU64(s), offset)); const auto pointer = Unary(s, spv::OpConvertUToPtr, TypePhysicalU32Pointer(s), address); const auto result = s.builder.AllocateId(); s.builder.AddFunction(spv::OpLoad, TypeU32(s), result, pointer, spv::MemoryAccessAlignedMask, 4u); @@ -224,7 +224,7 @@ uint32_t Boxes(EmitterState& s, uint32_t first, uint32_t second, uint32_t half, void DefineBvhIntersect(EmitterState& s) { if (!s.requirements.bvh) return; - const auto u = TypeU32(s), f = TypeF32(s), b = TypeBool(s), wide = TypeScalarU64(s); + const auto u = TypeU32(s), f = TypeF32(s), b = TypeBool(s), wide = TypeU64(s); const auto vec4 = TypeU32Vector(s, 4), vec3 = TypeF32Vector(s, 3); const auto signature = s.builder.Type(spv::OpTypeFunction, vec4, vec4, wide, f, vec3, vec3, vec3); s.bvh_intersect_function = s.builder.AllocateId(); @@ -260,22 +260,24 @@ void DefineBvhIntersect(EmitterState& s) { const auto bary = Binary(s, spv::OpINotEqual, b, and_bits(words[3], 0x01000000), ConstantU32(s, 0)); const auto sort = Binary(s, spv::OpINotEqual, b, and_bits(words[1], 0x80000000), ConstantU32(s, 0)); const auto grow = and_bits(shr(words[1], 23), 0xff); - const auto base = Binary(s, spv::OpShiftLeftLogical, wide, - DeviceAddressFromWords(s, words[0], and_bits(words[1], 0xff)), ConstantDeviceAddress(s, 8)); - const auto index = Binary(s, spv::OpShiftRightLogical, wide, node, ConstantDeviceAddress(s, 3)); + const auto base = Binary(s, spv::OpShiftLeftLogical, wide, + PackU64(s, words[0], and_bits(words[1], 0xff)), ConstantU32(s, 8)); + const auto index = Binary(s, spv::OpShiftRightLogical, wide, node, ConstantU32(s, 3)); const auto last_index = Binary(s, spv::OpIAdd, wide, index, - Select(s, wide, full, ConstantDeviceAddress(s, 1), ConstantDeviceAddress(s, 0))); + Select(s, wide, full, ConstantU64(s, 1), ConstantU64(s, 0))); const auto address = Binary(s, spv::OpIAdd, wide, base, - Binary(s, spv::OpShiftLeftLogical, wide, index, ConstantDeviceAddress(s, 6))); - const auto last_address = Binary(s, spv::OpIAdd, wide, base, - Binary(s, spv::OpShiftLeftLogical, wide, last_index, ConstantDeviceAddress(s, 6))); + Binary(s, spv::OpShiftLeftLogical, wide, index, ConstantU32(s, 6))); + const auto last_address = + Binary(s, spv::OpIAdd, wide, base, + Binary(s, spv::OpShiftLeftLogical, wide, last_index, ConstantU32(s, 6))); auto valid = Binary(s, spv::OpIEqual, b, shr(words[3], 28), ConstantU32(s, 8)); const auto require = [&](uint32_t condition) { valid = Binary(s, spv::OpLogicalAnd, b, valid, condition); }; require(Binary(s, spv::OpULessThanEqual, b, kind, ConstantU32(s, 5))); - require(Binary(s, spv::OpULessThanEqual, b, last_index, DeviceAddressFromWords(s, words[2], and_bits(words[3], 0x3ff)))); - require(Binary(s, spv::OpULessThan, b, last_address, ConstantDeviceAddress(s, uint64_t{1} << 48))); + require(Binary(s, spv::OpULessThanEqual, b, last_index, + PackU64(s, words[2], and_bits(words[3], 0x3ff)))); + require(Binary(s, spv::OpULessThan, b, last_address, ConstantU64(s, uint64_t {1} << 48))); require(Unary(s, spv::OpLogicalNot, b, Unary(s, spv::OpIsNan, b, extent))); for (uint32_t i = 0; i < 3; ++i) { for (const auto value: {origin[i], direction[i], inverse[i]}) @@ -292,13 +294,13 @@ void DefineBvhIntersect(EmitterState& s) { const auto invalid = Vector(s, vec4, invalid_words); const auto result = EmitValueOrDefaultIfCondition(s, valid, vec4, invalid, [&] { const auto first = GetBdaPointer(s, address); - const auto second = EmitValueOrDefaultIfCondition(s, full, wide, ConstantDeviceAddress(s, 0), [&] { - return GetBdaPointer(s, Binary(s, spv::OpIAdd, wide, address, ConstantDeviceAddress(s, 64))); + const auto second = EmitValueOrDefaultIfCondition(s, full, wide, ConstantU64(s, 0), [&] { + return GetBdaPointer(s, Binary(s, spv::OpIAdd, wide, address, ConstantU64(s, 64))); }); - const auto present = Binary(s, spv::OpLogicalAnd, b, - Binary(s, spv::OpINotEqual, b, first, ConstantDeviceAddress(s, 0)), + const auto present = Binary( + s, spv::OpLogicalAnd, b, Binary(s, spv::OpINotEqual, b, first, ConstantU64(s, 0)), Binary(s, spv::OpLogicalOr, b, Unary(s, spv::OpLogicalNot, b, full), - Binary(s, spv::OpINotEqual, b, second, ConstantDeviceAddress(s, 0)))); + Binary(s, spv::OpINotEqual, b, second, ConstantU64(s, 0)))); return EmitValueOrDefaultIfCondition(s, present, vec4, invalid, [&] { const auto tri_label = s.builder.AllocateId(), box_label = s.builder.AllocateId(); const auto merge = s.builder.AllocateId(); @@ -336,9 +338,8 @@ uint32_t EmitBvhIntersect(ValueEmitContext& ctx, const IR::Inst& inst) { return Vector(s, TypeF32Vector(s, 3), values); }; const auto descriptor = ctx.Arg(inst, 0); - const auto node = node_words == 2u - ? DeviceAddressFromWords(s, ctx.Arg(*ray, 0), ctx.Arg(*ray, 1)) - : Unary(s, spv::OpUConvert, TypeScalarU64(s), ctx.Arg(*ray, 0)); + const auto node = node_words == 2u ? PackU64(s, ctx.Arg(*ray, 0), ctx.Arg(*ray, 1)) + : Unary(s, spv::OpUConvert, TypeU64(s), ctx.Arg(*ray, 0)); const auto extent = Unary(s, spv::OpBitcast, TypeF32(s), ctx.Arg(*ray, node_words)); const auto origin = vector(node_words + 1u), direction = vector(node_words + 4u), inverse = vector(node_words + 7u); diff --git a/src/graphics/shader/recompiler/backend/spirv/spirvEmitterFlow.cpp b/src/graphics/shader/recompiler/backend/spirv/spirvEmitterFlow.cpp index 8940e2521..5194717a3 100644 --- a/src/graphics/shader/recompiler/backend/spirv/spirvEmitterFlow.cpp +++ b/src/graphics/shader/recompiler/backend/spirv/spirvEmitterFlow.cpp @@ -430,16 +430,16 @@ uint32_t EmitWqmU64(EmitterState& state, uint32_t value) { const auto quad_bits = state.builder.AllocateId(); const auto result = state.builder.AllocateId(); state.builder.AddFunction(spv::OpShiftRightLogical, TypeU64(state), shifted_one, value, - ConstantU64(state, 0x0000000100000001ull)); + ConstantU32(state, 1)); state.builder.AddFunction(spv::OpBitwiseOr, TypeU64(state), merged_one, value, shifted_one); state.builder.AddFunction(spv::OpShiftRightLogical, TypeU64(state), shifted_two, merged_one, - ConstantU64(state, 0x0000000200000002ull)); + ConstantU32(state, 2)); state.builder.AddFunction(spv::OpBitwiseOr, TypeU64(state), merged_two, merged_one, shifted_two); state.builder.AddFunction(spv::OpBitwiseAnd, TypeU64(state), quad_bits, merged_two, ConstantU64(state, 0x1111111111111111ull)); state.builder.AddFunction(spv::OpIMul, TypeU64(state), result, quad_bits, - ConstantU64(state, 0x0000000f0000000full)); + ConstantU64(state, 15)); return result; } diff --git a/src/graphics/shader/recompiler/backend/spirv/spirvEmitterImage.cpp b/src/graphics/shader/recompiler/backend/spirv/spirvEmitterImage.cpp index eb3231ac4..330441140 100644 --- a/src/graphics/shader/recompiler/backend/spirv/spirvEmitterImage.cpp +++ b/src/graphics/shader/recompiler/backend/spirv/spirvEmitterImage.cpp @@ -930,9 +930,9 @@ void EmitImage(ValueEmitContext& ctx, const IR::Inst& inst) { ctx.Define(inst, EmitValueOrDefaultIfCondition( state, ctx.Arg(inst, inst.NumArgs() - 1), result_type, zero, [&]() { const auto pointer = state.builder.AllocateId(); - const auto pointer_type = state.builder.Type( - spv::OpTypePointer, spv::StorageClassImage, - image.atomic64 ? TypeScalarU64(state) : TypeU32(state)); + const auto pointer_type = + state.builder.Type(spv::OpTypePointer, spv::StorageClassImage, + image.atomic64 ? TypeU64(state) : TypeU32(state)); state.builder.AddFunction(spv::OpImageTexelPointer, pointer_type, pointer, ImageDescriptorPointer(state, mem.resource), CoordU32(ctx, mem, *address, dimension), diff --git a/src/graphics/shader/recompiler/backend/spirv/spirvEmitterInstructions.h b/src/graphics/shader/recompiler/backend/spirv/spirvEmitterInstructions.h index e3d3a8154..96b67ac07 100644 --- a/src/graphics/shader/recompiler/backend/spirv/spirvEmitterInstructions.h +++ b/src/graphics/shader/recompiler/backend/spirv/spirvEmitterInstructions.h @@ -34,16 +34,16 @@ uint32_t EmitConvertF32F64(EmitterState& state, uint32_t arg0); uint32_t EmitConvertF64F32(EmitterState& state, uint32_t arg0); EMIT_NATIVE(ConvertF32U32, OpConvertUToF, F32, uint32_t) EMIT_NATIVE(ConvertF64U32, OpConvertUToF, F64, uint32_t) -EMIT_NATIVE(CompositeConstructU64, OpCompositeConstruct, U64, uint32_t, uint32_t) +uint32_t EmitCompositeConstructU64(ValueEmitContext& ctx, uint32_t arg0, IR::Value arg1); EMIT_NATIVE(CompositeConstructU32x2, OpCompositeConstruct, U32x2, uint32_t, uint32_t) EMIT_NATIVE(CompositeConstructU32x3, OpCompositeConstruct, U32x3, uint32_t, uint32_t, uint32_t) EMIT_NATIVE(CompositeConstructF32x2, OpCompositeConstruct, F32x2, uint32_t, uint32_t) EMIT_NATIVE(CompositeConstructU32x4, OpCompositeConstruct, U32x4, uint32_t, uint32_t, uint32_t, uint32_t) uint32_t EmitCompositeExtractU64(EmitterState& state, uint32_t arg0, IR::Value arg1); -inline constexpr auto EmitCompositeExtractU32x2 = EmitCompositeExtractU64; -inline constexpr auto EmitCompositeExtractU32x3 = EmitCompositeExtractU64; -inline constexpr auto EmitCompositeExtractU32x4 = EmitCompositeExtractU64; +uint32_t EmitCompositeExtractU32x2(EmitterState& state, uint32_t arg0, IR::Value arg1); +inline constexpr auto EmitCompositeExtractU32x3 = EmitCompositeExtractU32x2; +inline constexpr auto EmitCompositeExtractU32x4 = EmitCompositeExtractU32x2; inline constexpr auto EmitPackHalf2x16 = EmitGlsl; inline constexpr auto EmitPackSnorm2x16 = EmitGlsl; @@ -63,9 +63,9 @@ EMIT_NATIVE(IAdd32, OpIAdd, U32, uint32_t, uint32_t) EMIT_NATIVE(ISub32, OpISub, U32, uint32_t, uint32_t) EMIT_NATIVE(IMul32, OpIMul, U32, uint32_t, uint32_t) EMIT_NATIVE(UDiv32, OpUDiv, U32, uint32_t, uint32_t) -uint32_t EmitIAdd64(EmitterState& state, uint32_t arg0, uint32_t arg1); -uint32_t EmitISub64(EmitterState& state, uint32_t arg0, uint32_t arg1); -uint32_t EmitIMul64(EmitterState& state, uint32_t arg0, uint32_t arg1); +EMIT_NATIVE(IAdd64, OpIAdd, U64, uint32_t, uint32_t) +EMIT_NATIVE(ISub64, OpISub, U64, uint32_t, uint32_t) +EMIT_NATIVE(IMul64, OpIMul, U64, uint32_t, uint32_t) EMIT_NATIVE(IAddCarry32, OpIAddCarry, U32x2, uint32_t, uint32_t) uint32_t EmitSMulHi(EmitterState& state, uint32_t arg0, uint32_t arg1); uint32_t EmitUMulHi(EmitterState& state, uint32_t arg0, uint32_t arg1); @@ -73,14 +73,26 @@ uint32_t EmitIAbs32(EmitterState& state, uint32_t arg0); EMIT_NATIVE(ShiftLeftLogical32, OpShiftLeftLogical, U32, uint32_t, uint32_t) EMIT_NATIVE(ShiftRightLogical32, OpShiftRightLogical, U32, uint32_t, uint32_t) EMIT_NATIVE(ShiftRightArithmetic32, OpShiftRightArithmetic, U32, uint32_t, uint32_t) -uint32_t EmitShiftLeftLogical64(ValueEmitContext& ctx, uint32_t arg0, IR::Value arg1); -uint32_t EmitShiftRightLogical64(ValueEmitContext& ctx, uint32_t arg0, IR::Value arg1); -uint32_t EmitShiftRightArithmetic64(ValueEmitContext& ctx, uint32_t arg0, IR::Value arg1); +template +uint32_t EmitShift64(ValueEmitContext& ctx, uint32_t value, IR::Value shift) { + shift = shift.Resolve(); + if (shift.IsImmediate()) { + const auto amount = shift.U32() & 63u; + return amount == 0u ? value + : Binary(ctx.state, opcode, TypeU64(ctx.state), value, + ConstantU32(ctx.state, amount)); + } + return Binary(ctx.state, opcode, TypeU64(ctx.state), value, + EmitAndConstant(ctx.state, ctx.Def(shift), 63u)); +} +inline constexpr auto EmitShiftLeftLogical64 = EmitShift64; +inline constexpr auto EmitShiftRightLogical64 = EmitShift64; +inline constexpr auto EmitShiftRightArithmetic64 = EmitShift64; EMIT_NATIVE(BitwiseAnd32, OpBitwiseAnd, U32, uint32_t, uint32_t) EMIT_NATIVE(BitwiseOr32, OpBitwiseOr, U32, uint32_t, uint32_t) EMIT_NATIVE(BitwiseXor32, OpBitwiseXor, U32, uint32_t, uint32_t) EMIT_NATIVE(BitwiseNot32, OpNot, U32, uint32_t) -uint32_t EmitBitwiseAnd64(EmitterState& state, uint32_t arg0, uint32_t arg1); +EMIT_NATIVE(BitwiseAnd64, OpBitwiseAnd, U64, uint32_t, uint32_t) EMIT_NATIVE(BitReverse32, OpBitReverse, U32, uint32_t) EMIT_NATIVE(BitCount32, OpBitCount, U32, uint32_t) uint32_t EmitBitCount64(EmitterState& state, uint32_t arg0); @@ -107,14 +119,14 @@ EMIT_NATIVE(UGreaterThan32, OpUGreaterThan, U1, uint32_t, uint32_t) EMIT_NATIVE(INotEqual32, OpINotEqual, U1, uint32_t, uint32_t) EMIT_NATIVE(SGreaterThanEqual32, OpSGreaterThanEqual, U1, uint32_t, uint32_t) EMIT_NATIVE(UGreaterThanEqual32, OpUGreaterThanEqual, U1, uint32_t, uint32_t) -uint32_t EmitIEqual64(EmitterState& state, uint32_t arg0, uint32_t arg1); -uint32_t EmitINotEqual64(EmitterState& state, uint32_t arg0, uint32_t arg1); -uint32_t EmitULessThan64(EmitterState& state, uint32_t arg0, uint32_t arg1); -uint32_t EmitSLessThan64(EmitterState& state, uint32_t arg0, uint32_t arg1); -uint32_t EmitSLessThanEqual64(EmitterState& state, uint32_t arg0, uint32_t arg1); -uint32_t EmitULessThanEqual64(EmitterState& state, uint32_t arg0, uint32_t arg1); -uint32_t EmitUGreaterThanEqual64(EmitterState& state, uint32_t arg0, uint32_t arg1); -uint32_t EmitUGreaterThan64(EmitterState& state, uint32_t arg0, uint32_t arg1); +EMIT_NATIVE(IEqual64, OpIEqual, U1, uint32_t, uint32_t) +EMIT_NATIVE(INotEqual64, OpINotEqual, U1, uint32_t, uint32_t) +EMIT_NATIVE(ULessThan64, OpULessThan, U1, uint32_t, uint32_t) +EMIT_NATIVE(SLessThan64, OpSLessThan, U1, uint32_t, uint32_t) +EMIT_NATIVE(SLessThanEqual64, OpSLessThanEqual, U1, uint32_t, uint32_t) +EMIT_NATIVE(ULessThanEqual64, OpULessThanEqual, U1, uint32_t, uint32_t) +EMIT_NATIVE(UGreaterThanEqual64, OpUGreaterThanEqual, U1, uint32_t, uint32_t) +EMIT_NATIVE(UGreaterThan64, OpUGreaterThan, U1, uint32_t, uint32_t) EMIT_NATIVE(LogicalOr, OpLogicalOr, U1, uint32_t, uint32_t) EMIT_NATIVE(LogicalAnd, OpLogicalAnd, U1, uint32_t, uint32_t) EMIT_NATIVE(LogicalXor, OpLogicalNotEqual, U1, uint32_t, uint32_t) diff --git a/src/graphics/shader/recompiler/backend/spirv/spirvEmitterInternal.h b/src/graphics/shader/recompiler/backend/spirv/spirvEmitterInternal.h index b3c4057ec..a9c7b15bd 100644 --- a/src/graphics/shader/recompiler/backend/spirv/spirvEmitterInternal.h +++ b/src/graphics/shader/recompiler/backend/spirv/spirvEmitterInternal.h @@ -98,7 +98,7 @@ struct EmitterState { uint32_t void_type = 0; uint32_t bool_type = 0; uint32_t u32_type = 0; - uint32_t native_u64_type = 0; + uint32_t u64_type = 0; uint32_t i32_type = 0; uint32_t f32_type = 0; uint32_t f64_type = 0; @@ -168,7 +168,6 @@ uint32_t TypeBool(EmitterState& state); uint32_t TypeBoolVector(EmitterState& state, uint32_t components); uint32_t TypeU32(EmitterState& state); uint32_t TypeU64(EmitterState& state); -uint32_t TypeScalarU64(EmitterState& state); uint32_t TypeU32Pair(EmitterState& state); uint32_t TypeI32(EmitterState& state); uint32_t TypeI32Pair(EmitterState& state); @@ -221,6 +220,12 @@ inline uint32_t Unary(EmitterState& state, spv::Op opcode, uint32_t type, uint32 return result; } +inline uint32_t PackU64(EmitterState& state, uint32_t low, uint32_t high) { + const auto pair = state.builder.AllocateId(); + state.builder.AddFunction(spv::OpCompositeConstruct, TypeU32Vector(state, 2), pair, low, high); + return Unary(state, spv::OpBitcast, TypeU64(state), pair); +} + inline uint32_t Binary(EmitterState& state, spv::Op opcode, uint32_t type, uint32_t lhs, uint32_t rhs) { const auto result = state.builder.AllocateId(); @@ -531,8 +536,6 @@ void EmitProgram(EmitterState& state); void DefineGetBdaPointer(EmitterState& state); void DefineBvhIntersect(EmitterState& state); uint32_t GetBdaPointer(EmitterState& state, uint32_t address); -uint32_t ConstantDeviceAddress(EmitterState& state, uint64_t value); -uint32_t DeviceAddressFromWords(EmitterState& state, uint32_t low, uint32_t high); // These templates accept local lambdas from several emitter translation units. template diff --git a/src/graphics/shader/recompiler/backend/spirv/spirvEmitterMemory.cpp b/src/graphics/shader/recompiler/backend/spirv/spirvEmitterMemory.cpp index a863cd737..8e0a3e5ec 100644 --- a/src/graphics/shader/recompiler/backend/spirv/spirvEmitterMemory.cpp +++ b/src/graphics/shader/recompiler/backend/spirv/spirvEmitterMemory.cpp @@ -133,22 +133,22 @@ uint32_t GuestAddress(ValueEmitContext& ctx, const IR::Inst& inst, const IR::Mem } uint32_t address = 0; if (mem.address_is_full) { - address = DeviceAddressFromWords(state, low, ctx.Arg(inst, 2)); + address = PackU64(state, low, ctx.Arg(inst, 2)); } else { const auto* handle = inst.Arg(0).Resolve().TryInstruction(); if (handle == nullptr || handle->GetOpcode() != IR::ValueOpcode::GetAddressResource || handle->NumArgs() != 2) { ctx.Fail(inst, "has no address base pair"); - return ConstantDeviceAddress(state, 0); + return ConstantU64(state, 0); } auto base_low = ctx.Arg(*handle, 0); if (mem.kind == IR::ResourceKind::ScalarAddress) { base_low = Binary(state, spv::OpBitwiseAnd, TypeU32(state), base_low, ConstantU32(state, ~3u)); } - const auto base = DeviceAddressFromWords(state, base_low, ctx.Arg(*handle, 1)); - address = Binary(state, spv::OpIAdd, TypeScalarU64(state), base, - Unary(state, spv::OpUConvert, TypeScalarU64(state), low)); + const auto base = PackU64(state, base_low, ctx.Arg(*handle, 1)); + address = Binary(state, spv::OpIAdd, TypeU64(state), base, + Unary(state, spv::OpUConvert, TypeU64(state), low)); } auto immediate = static_cast(mem.offset); if (mem.kind == IR::ResourceKind::ScalarAddress) { @@ -156,9 +156,8 @@ uint32_t GuestAddress(ValueEmitContext& ctx, const IR::Inst& inst, const IR::Mem } return immediate == 0 ? address - : Binary(state, spv::OpIAdd, TypeScalarU64(state), address, - ConstantDeviceAddress( - state, static_cast(static_cast(immediate)))); + : Binary(state, spv::OpIAdd, TypeU64(state), address, + ConstantU64(state, static_cast(static_cast(immediate)))); } uint32_t FaultElementPointer(EmitterState& state, uint32_t index) { @@ -185,7 +184,7 @@ uint32_t LoadBdaDword(ValueEmitContext& ctx, uint32_t address) { auto& state = ctx.state; const auto bda = GetBdaPointer(state, address); const auto present = - Binary(state, spv::OpINotEqual, TypeBool(state), bda, ConstantDeviceAddress(state, 0)); + Binary(state, spv::OpINotEqual, TypeBool(state), bda, ConstantU64(state, 0)); return EmitValueOrZeroIfCondition(state, present, [&]() { const auto pointer = state.builder.AllocateId(); state.builder.AddFunction(spv::OpConvertUToPtr, TypePhysicalU32Pointer(state), pointer, @@ -201,8 +200,8 @@ uint32_t LoadBdaDword(ValueEmitContext& ctx, uint32_t address) { uint32_t LoadBda(ValueEmitContext& ctx, uint32_t address, uint32_t active, uint32_t bits) { auto& state = ctx.state; return EmitValueOrZeroIfCondition(state, active, [&]() { - const auto aligned = Binary(state, spv::OpBitwiseAnd, TypeScalarU64(state), address, - ConstantDeviceAddress(state, ~uint64_t {3})); + const auto aligned = Binary(state, spv::OpBitwiseAnd, TypeU64(state), address, + ConstantU64(state, ~uint64_t {3})); const auto first = LoadBdaDword(ctx, aligned); const auto byte = Binary(state, spv::OpBitwiseAnd, TypeU32(state), @@ -212,8 +211,8 @@ uint32_t LoadBda(ValueEmitContext& ctx, uint32_t address, uint32_t active, uint3 : Binary(state, bits == 16u ? spv::OpUGreaterThan : spv::OpINotEqual, TypeBool(state), byte, ConstantU32(state, bits == 16u ? 2u : 0u)); const auto second = EmitValueOrZeroIfCondition(state, crosses, [&]() { - return LoadBdaDword(ctx, Binary(state, spv::OpIAdd, TypeScalarU64(state), aligned, - ConstantDeviceAddress(state, sizeof(uint32_t)))); + return LoadBdaDword(ctx, Binary(state, spv::OpIAdd, TypeU64(state), aligned, + ConstantU64(state, sizeof(uint32_t)))); }); const auto shift = Binary(state, spv::OpShiftLeftLogical, TypeU32(state), byte, ConstantU32(state, 3)); @@ -805,26 +804,25 @@ uint32_t LoadIndirectScalarBuffer(ValueEmitContext& ctx, const IR::Inst& inst) { const auto word1 = ctx.Arg(handle, 1); const auto stride = EmitBitFieldUExtract(state, word1, ConstantU32(state, 16), ConstantU32(state, 14)); - const auto size = Binary( - state, spv::OpIMul, TypeScalarU64(state), - Unary(state, spv::OpUConvert, TypeScalarU64(state), - EmitUMax32(state, stride, ConstantU32(state, 1))), - Unary(state, spv::OpUConvert, TypeScalarU64(state), ctx.Arg(handle, 2))); - const auto offset = Binary( - state, spv::OpIAdd, TypeScalarU64(state), - Unary(state, spv::OpUConvert, TypeScalarU64(state), - Binary(state, spv::OpBitwiseAnd, TypeU32(state), ctx.Arg(inst, 1), - ConstantU32(state, ~3u))), - ConstantDeviceAddress(state, ctx.Memory(inst).offset & ~3u)); - const auto end = Binary(state, spv::OpIAdd, TypeScalarU64(state), offset, - ConstantDeviceAddress(state, sizeof(uint32_t))); + const auto size = Binary(state, spv::OpIMul, TypeU64(state), + Unary(state, spv::OpUConvert, TypeU64(state), + EmitUMax32(state, stride, ConstantU32(state, 1))), + Unary(state, spv::OpUConvert, TypeU64(state), ctx.Arg(handle, 2))); + const auto offset = Binary(state, spv::OpIAdd, TypeU64(state), + Unary(state, spv::OpUConvert, TypeU64(state), + Binary(state, spv::OpBitwiseAnd, TypeU32(state), + ctx.Arg(inst, 1), ConstantU32(state, ~3u))), + ConstantU64(state, ctx.Memory(inst).offset & ~3u)); + const auto end = + Binary(state, spv::OpIAdd, TypeU64(state), offset, ConstantU64(state, sizeof(uint32_t))); return EmitValueOrZeroIfCondition( state, Binary(state, spv::OpULessThanEqual, TypeBool(state), end, size), [&]() { - const auto base = DeviceAddressFromWords( - state, Binary(state, spv::OpBitwiseAnd, TypeU32(state), ctx.Arg(handle, 0), - ConstantU32(state, ~3u)), - Binary(state, spv::OpBitwiseAnd, TypeU32(state), word1, ConstantU32(state, 0xffffu))); - return LoadBdaDword(ctx, Binary(state, spv::OpIAdd, TypeScalarU64(state), base, offset)); + const auto base = PackU64(state, + Binary(state, spv::OpBitwiseAnd, TypeU32(state), + ctx.Arg(handle, 0), ConstantU32(state, ~3u)), + Binary(state, spv::OpBitwiseAnd, TypeU32(state), word1, + ConstantU32(state, 0xffffu))); + return LoadBdaDword(ctx, Binary(state, spv::OpIAdd, TypeU64(state), base, offset)); }); } @@ -865,21 +863,21 @@ IndirectBufferAccess PrepareIndirectBuffer(ValueEmitContext& ctx, const IR::Inst const auto address = CalculateBufferAddress(state, index, ctx.Arg(inst, 2), soffset, ctx.Memory(inst).offset, stride, swizzle, field(word3, 21, 2)); - const auto base = DeviceAddressFromWords(state, ctx.Arg(handle, 0), field(word1, 0, 16)); + const auto base = PackU64(state, ctx.Arg(handle, 0), field(word1, 0, 16)); const auto raw_records = Binary(state, spv::OpISub, TypeU32(state), records, soffset); return { - .address = Binary(state, spv::OpIAdd, TypeScalarU64(state), base, - Unary(state, spv::OpUConvert, TypeScalarU64(state), address.byte)), - .offset = address.offset, - .stride = stride, - .swizzle = swizzle, - .format = field(word3, 12, 7), - .selector = ctx.Memory(inst).formatted ? field(word3, 0, 3) : 0u, - .mode = field(word3, 28, 2), - .records = records, - .index_in_bounds = Binary(state, spv::OpULessThan, TypeBool(state), index, records), + .address = Binary(state, spv::OpIAdd, TypeU64(state), base, + Unary(state, spv::OpUConvert, TypeU64(state), address.byte)), + .offset = address.offset, + .stride = stride, + .swizzle = swizzle, + .format = field(word3, 12, 7), + .selector = ctx.Memory(inst).formatted ? field(word3, 0, 3) : 0u, + .mode = field(word3, 28, 2), + .records = records, + .index_in_bounds = Binary(state, spv::OpULessThan, TypeBool(state), index, records), .scalar_in_bounds = Binary(state, spv::OpULessThanEqual, TypeBool(state), soffset, records), - .raw_records = raw_records, + .raw_records = raw_records, .raw_index_in_bounds = Binary(state, spv::OpULessThan, TypeBool(state), index, raw_records), }; } @@ -925,13 +923,13 @@ uint32_t LoadIndirectBuffer(ValueEmitContext& ctx, const IR::Inst& inst, uint32_ const auto buffer = PrepareIndirectBuffer(ctx, inst); const auto valid_format = Binary(state, spv::OpINotEqual, TypeBool(state), buffer.format, ConstantU32(state, 0)); - const auto base = Binary(state, spv::OpBitwiseAnd, TypeScalarU64(state), buffer.address, - ConstantDeviceAddress(state, ~uint64_t {3})); + const auto base = Binary(state, spv::OpBitwiseAnd, TypeU64(state), buffer.address, + ConstantU64(state, ~uint64_t {3})); std::array values {}; for (uint32_t component = 0; component < components; ++component) { - const auto address = component == 0u ? base : Binary( - state, spv::OpIAdd, TypeScalarU64(state), base, - ConstantDeviceAddress(state, component * 4u)); + const auto address = component == 0u ? base + : Binary(state, spv::OpIAdd, TypeU64(state), base, + ConstantU64(state, component * 4u)); values[component] = EmitValueOrZeroIfCondition( state, AndCondition(state, valid_format, IndirectBufferInBounds(state, buffer, component * 4u, 4u, false)), @@ -959,13 +957,14 @@ uint32_t LoadIndirectFormattedX(ValueEmitContext& ctx, const IR::Inst& inst) { TypeU32(state), constant, [&]() { const auto in_bounds = IndirectBufferInBounds(state, buffer, 0u, info.byte_size, true); const auto base = Binary( - state, spv::OpBitwiseAnd, TypeScalarU64(state), buffer.address, - ConstantDeviceAddress(state, ~(uint64_t(std::min(4u, info.byte_size)) - 1u))); + state, spv::OpBitwiseAnd, TypeU64(state), buffer.address, + ConstantU64(state, ~(uint64_t(std::min(4u, info.byte_size)) - 1u))); const auto load = [&](uint32_t component, uint32_t bits) { const auto offset = Format::GetFormatComponentByteOffset(info, component); - const auto address = offset == 0u ? base : Binary( - state, spv::OpIAdd, TypeScalarU64(state), base, - ConstantDeviceAddress(state, offset)); + const auto address = offset == 0u + ? base + : Binary(state, spv::OpIAdd, TypeU64(state), + base, ConstantU64(state, offset)); return LoadBda(ctx, address, in_bounds, bits); }; const auto emit_component = [&](uint32_t component) { @@ -1085,23 +1084,9 @@ void StoreWideShared(ValueEmitContext& ctx, const IR::Inst& inst, uint32_t compo } // namespace -uint32_t ConstantDeviceAddress(EmitterState& state, uint64_t value) { - return state.builder.Constant(spv::OpConstant, TypeScalarU64(state), - static_cast(value), - static_cast(value >> 32u)); -} - -uint32_t DeviceAddressFromWords(EmitterState& state, uint32_t low, uint32_t high) { - const auto low64 = Unary(state, spv::OpUConvert, TypeScalarU64(state), low); - const auto high64 = Binary(state, spv::OpShiftLeftLogical, TypeScalarU64(state), - Unary(state, spv::OpUConvert, TypeScalarU64(state), high), - ConstantDeviceAddress(state, 32)); - return Binary(state, spv::OpBitwiseOr, TypeScalarU64(state), low64, high64); -} - uint32_t GetBdaPointer(EmitterState& state, uint32_t address) { const auto result = state.builder.AllocateId(); - state.builder.AddFunction(spv::OpFunctionCall, TypeScalarU64(state), result, + state.builder.AddFunction(spv::OpFunctionCall, TypeU64(state), result, state.bda_pointer_function, address); return result; } @@ -1110,7 +1095,7 @@ void DefineGetBdaPointer(EmitterState& state) { if (!state.program.info.uses_dma) { return; } - const auto type = TypeScalarU64(state); + const auto type = TypeU64(state); const auto function_type = state.builder.Type(spv::OpTypeFunction, type, type); state.bda_pointer_function = state.builder.AllocateId(); const auto address = state.builder.AllocateId(); @@ -1122,14 +1107,14 @@ void DefineGetBdaPointer(EmitterState& state) { EmitLabel(state, entry_label); const auto extended = Binary(state, spv::OpUGreaterThanEqual, TypeBool(state), address, - ConstantDeviceAddress(state, LibKernel::Memory::kExtendedMemoryBase)); - const auto packed = Select(state, type, extended, - Binary(state, spv::OpISub, type, address, - ConstantDeviceAddress(state, - LibKernel::Memory::kExtendedMemoryBase - LOWER_ADDRESS_SIZE)), - address); + ConstantU64(state, LibKernel::Memory::kExtendedMemoryBase)); + const auto packed = Select( + state, type, extended, + Binary(state, spv::OpISub, type, address, + ConstantU64(state, LibKernel::Memory::kExtendedMemoryBase - LOWER_ADDRESS_SIZE)), + address); const auto page64 = Binary(state, spv::OpShiftRightLogical, type, packed, - ConstantDeviceAddress(state, BufferCache::CACHING_PAGEBITS)); + ConstantU32(state, BufferCache::CACHING_PAGEBITS)); const auto page = Unary(state, spv::OpUConvert, TypeU32(state), page64); const auto entry_pointer = state.builder.AllocateId(); state.builder.AddFunction(spv::OpAccessChain, TypeStorageBufferElementPointer(state, 64), @@ -1137,8 +1122,7 @@ void DefineGetBdaPointer(EmitterState& state) { page); const auto base = state.builder.AllocateId(); state.builder.AddFunction(spv::OpLoad, type, base, entry_pointer); - const auto missing = - Binary(state, spv::OpIEqual, TypeBool(state), base, ConstantDeviceAddress(state, 0)); + const auto missing = Binary(state, spv::OpIEqual, TypeBool(state), base, ConstantU64(state, 0)); const auto fault_label = state.builder.AllocateId(); const auto available_label = state.builder.AllocateId(); const auto merge_label = state.builder.AllocateId(); @@ -1151,14 +1135,14 @@ void DefineGetBdaPointer(EmitterState& state) { EmitLabel(state, available_label); const auto offset = Binary(state, spv::OpBitwiseAnd, type, address, - ConstantDeviceAddress(state, BufferCache::CACHING_PAGESIZE - 1)); + ConstantU64(state, BufferCache::CACHING_PAGESIZE - 1)); const auto available = Binary(state, spv::OpIAdd, type, base, offset); state.builder.AddFunction(spv::OpBranch, merge_label); EmitLabel(state, merge_label); const auto result = state.builder.AllocateId(); - state.builder.AddFunction(spv::OpPhi, type, result, ConstantDeviceAddress(state, 0), - fault_label, available, available_label); + state.builder.AddFunction(spv::OpPhi, type, result, ConstantU64(state, 0), fault_label, + available, available_label); state.builder.AddFunction(spv::OpReturnValue, result); state.builder.AddFunction(spv::OpFunctionEnd); } @@ -1216,12 +1200,12 @@ void EmitSharedAtomic64(ValueEmitContext& ctx, const IR::Inst& inst) { semantics = spv::MemorySemanticsUniformMemoryMask; } const auto pointer = state.builder.AllocateId(); - state.builder.AddFunction(spv::OpAccessChain, - TypePointer(state, state.lds_storage_class, TypeScalarU64(state)), - pointer, state.lds_u64_variable, ConstantU32(state, 0), native_index); - const auto value = Unary(state, spv::OpBitcast, TypeScalarU64(state), ctx.Arg(inst, 1)); state.builder.AddFunction( - SpirvAtomicOpcode(inst.GetOpcode()), TypeScalarU64(state), state.builder.AllocateId(), + spv::OpAccessChain, TypePointer(state, state.lds_storage_class, TypeU64(state)), + pointer, state.lds_u64_variable, ConstantU32(state, 0), native_index); + const auto value = ctx.Arg(inst, 1); + state.builder.AddFunction( + SpirvAtomicOpcode(inst.GetOpcode()), TypeU64(state), state.builder.AllocateId(), pointer, ConstantU32(state, spv::ScopeWorkgroup), ConstantU32(state, spv::MemorySemanticsAcquireReleaseMask | semantics), value); }); diff --git a/src/graphics/shader/recompiler/backend/spirv/spirvEmitterMemoryHelpers.cpp b/src/graphics/shader/recompiler/backend/spirv/spirvEmitterMemoryHelpers.cpp index daf448ec9..ad3ea70fb 100644 --- a/src/graphics/shader/recompiler/backend/spirv/spirvEmitterMemoryHelpers.cpp +++ b/src/graphics/shader/recompiler/backend/spirv/spirvEmitterMemoryHelpers.cpp @@ -138,7 +138,7 @@ void EnsureLdsStorage(EmitterState& state) { }; if (state.requirements.shared_int64_atomics) { state.lds_variable = define(TypeU32(state), 4u); - state.lds_u64_variable = define(TypeScalarU64(state), 8u); + state.lds_u64_variable = define(TypeU64(state), 8u); state.builder.AddName(state.lds_u64_variable, "lds_qwords"); } else { state.lds_variable = state.builder.DefineGlobalVariable( @@ -410,7 +410,7 @@ uint32_t EmitAtomicOperation(ValueEmitContext& ctx, const IR::Inst& inst, uint32 const auto opcode = SpirvAtomicOpcode(inst.GetOpcode()); const auto old = ctx.state.builder.AllocateId(); const bool wide = inst.GetType() == IR::Type::U64; - const auto type = wide ? TypeScalarU64(ctx.state) : TypeU32(ctx.state); + const auto type = wide ? TypeU64(ctx.state) : TypeU32(ctx.state); if (opcode == spv::OpAtomicCompareExchange) { const auto desired = ctx.Arg(inst, inst.NumArgs() - 3); const auto comparator = ctx.Arg(inst, inst.NumArgs() - 2); @@ -419,13 +419,12 @@ uint32_t EmitAtomicOperation(ValueEmitContext& ctx, const IR::Inst& inst, uint32 ConstantU32(ctx.state, scope), ConstantU32(ctx.state, spv::MemorySemanticsMaskNone), ConstantU32(ctx.state, spv::MemorySemanticsMaskNone), desired, comparator); } else { - auto value = ctx.Arg(inst, inst.NumArgs() - 2); - if (wide) value = Unary(ctx.state, spv::OpBitcast, type, value); + const auto value = ctx.Arg(inst, inst.NumArgs() - 2); ctx.state.builder.AddFunction(opcode, type, old, pointer, ConstantU32(ctx.state, scope), ConstantU32(ctx.state, spv::MemorySemanticsMaskNone), value); } - return wide ? Unary(ctx.state, spv::OpBitcast, TypeU64(ctx.state), old) : old; + return old; } void EmitAtomicMemoryBarrier(EmitterState& state, IR::ResourceKind kind) { diff --git a/src/graphics/shader/recompiler/backend/spirv/spirvEmitterModule.cpp b/src/graphics/shader/recompiler/backend/spirv/spirvEmitterModule.cpp index ea5e82db8..05fb01277 100644 --- a/src/graphics/shader/recompiler/backend/spirv/spirvEmitterModule.cpp +++ b/src/graphics/shader/recompiler/backend/spirv/spirvEmitterModule.cpp @@ -35,14 +35,11 @@ uint32_t TypeU32(EmitterState& state) { } uint32_t TypeU64(EmitterState& state) { - return TypeU32Vector(state, 2); -} - -uint32_t TypeScalarU64(EmitterState& state) { - if (state.native_u64_type == 0) { - state.native_u64_type = state.builder.Type(spv::OpTypeInt, 64, 0); + if (state.u64_type == 0) { + state.builder.RequireCapability(spv::CapabilityInt64); + state.u64_type = state.builder.Type(spv::OpTypeInt, 64, 0); } - return state.native_u64_type; + return state.u64_type; } uint32_t TypeU32Pair(EmitterState& state) { @@ -124,8 +121,9 @@ uint32_t TypeFunction(EmitterState& state) { } uint32_t TypeStorageBufferElement(EmitterState& state, uint32_t bits) { - return bits == 32u ? TypeU32(state) : bits == 64u ? TypeScalarU64(state) - : state.builder.Type(spv::OpTypeInt, bits, 0); + return bits == 32u ? TypeU32(state) + : bits == 64u ? TypeU64(state) + : state.builder.Type(spv::OpTypeInt, bits, 0); } uint32_t StorageRuntimeArrayType(EmitterState& state, uint32_t bits) { @@ -331,9 +329,8 @@ uint32_t ConstantBool(EmitterState& state, bool value) { } uint32_t ConstantU64(EmitterState& state, uint64_t value) { - return state.builder.Constant(spv::OpConstantComposite, TypeU64(state), - ConstantU32(state, static_cast(value)), - ConstantU32(state, static_cast(value >> 32u))); + return state.builder.Constant(spv::OpConstant, TypeU64(state), static_cast(value), + static_cast(value >> 32u)); } uint32_t ConstantU32CompositeZero(EmitterState& state, uint32_t components) { diff --git a/src/graphics/shader/recompiler/frontend/translate/Control.cpp b/src/graphics/shader/recompiler/frontend/translate/Control.cpp index 2940dd567..e805ae6ff 100644 --- a/src/graphics/shader/recompiler/frontend/translate/Control.cpp +++ b/src/graphics/shader/recompiler/frontend/translate/Control.cpp @@ -248,6 +248,7 @@ void Translator::S_CSELECT_B32(const Decoder::Instruction& inst) { void Translator::ScalarSelect64(const Decoder::Instruction& inst, const Decoder::Operand& false_source) { + // Preserve per-word expressions for descriptor tracking and mask provenance. const auto condition = ir.GetScc(); const auto lhs = ReadU32Pair(inst.src0); const auto rhs = ReadU32Pair(false_source); diff --git a/src/graphics/shader/recompiler/frontend/translate/Integer.cpp b/src/graphics/shader/recompiler/frontend/translate/Integer.cpp index 3d58f2b4c..2a54988d9 100644 --- a/src/graphics/shader/recompiler/frontend/translate/Integer.cpp +++ b/src/graphics/shader/recompiler/frontend/translate/Integer.cpp @@ -114,6 +114,7 @@ void Translator::S_U64_MASK(const Decoder::Instruction& inst, IR::ValueOpcode lo return; } + // Descriptor tracking reasons about each scalar word independently. const auto lhs = ReadU32Pair(inst.src0); auto mask_valid = ReadMaskValid(inst.src0); if (inst.src_count > 1u) { @@ -269,22 +270,18 @@ void Translator::Integer24(const Decoder::Instruction& inst, bool sign, bool add } void Translator::V_MAD_U64_U32(const Decoder::Instruction& inst) { - const auto lhs = ReadU32(inst.src0); - const auto rhs = ReadU32(inst.src1); - const auto add = ExtractU64(ReadU64(inst.src2)); - const auto mul_low = ir.IMul(lhs, rhs); - const auto mul_high = IR::U32(ir.Emit(IR::ValueOpcode::UMulHi, {lhs, rhs})); - const auto low = ir.IAdd(mul_low, add[0]); - const auto carry_low = ir.ULessThan(low, mul_low); - const auto high0 = ir.IAdd(mul_high, add[1]); - const auto carry0 = ir.ULessThan(high0, mul_high); - const auto high = - ir.IAdd(high0, ir.Select(carry_low, IR::U32(IR::Value(1u)), IR::U32(IR::Value(0u)))); - const auto carry1 = ir.ULessThan(high, high0); - WriteOperand(DestinationOperand(inst), ir.ConstructU64(low, high)); + const auto lhs = ReadU32(inst.src0); + const auto rhs = ReadU32(inst.src1); + // Keep the 32x32 product in words: native widening IMul64 fails NVIDIA compilation + // in large vertex shaders (Astro's Playroom). The 64-bit addition stays native. + const auto product = ir.ConstructU64( + ir.IMul(lhs, rhs), IR::U32(ir.Emit(IR::ValueOpcode::UMulHi, {lhs, rhs}))); + const auto result = ir.Emit(IR::ValueOpcode::IAdd64, {product, ReadU64(inst.src2)}); + WriteOperand(DestinationOperand(inst), result); if (inst.dst2.kind != Decoder::OperandKind::Null && inst.dst2.kind != Decoder::OperandKind::Unknown) { - WriteMask(inst.dst2, ir.LogicalOr(carry0, carry1)); + // The 32x32 product fits in 64 bits, so only the addition can carry. + WriteMask(inst.dst2, IR::U1(ir.Emit(IR::ValueOpcode::ULessThan64, {result, product}))); } } @@ -407,9 +404,12 @@ IR::U64 Translator::RightMask64(IR::U32 count) { } void Translator::S_BFM_B64(const Decoder::Instruction& inst) { - const auto count = ir.BitwiseAnd(ReadU32(inst.src0), IR::U32(IR::Value(63u))); - const auto offset = ir.BitwiseAnd(ReadU32(inst.src1), IR::U32(IR::Value(63u))); - const auto result = ir.Emit(IR::ValueOpcode::ShiftLeftLogical64, {RightMask64(count), offset}); + const auto count = ReadU32(inst.src0); + const auto offset = ReadU32(inst.src1); + const auto one = IR::Value(uint64_t {1}); + const auto limit = ir.Emit(IR::ValueOpcode::ShiftLeftLogical64, {one, count}); + const auto mask = ir.Emit(IR::ValueOpcode::ISub64, {limit, one}); + const auto result = ir.Emit(IR::ValueOpcode::ShiftLeftLogical64, {mask, offset}); WriteOperand(DestinationOperand(inst), result); } diff --git a/src/graphics/shader/recompiler/frontend/translate/Scalar.cpp b/src/graphics/shader/recompiler/frontend/translate/Scalar.cpp index f477ad9f7..4b7cec33b 100644 --- a/src/graphics/shader/recompiler/frontend/translate/Scalar.cpp +++ b/src/graphics/shader/recompiler/frontend/translate/Scalar.cpp @@ -165,9 +165,16 @@ void Translator::EmitScalar(const Decoder::Instruction& inst) { case O::S_BCNT1_I32_B32: return SimpleInteger(inst, IR::ValueOpcode::BitCount32, IR::Type::U32, false, false, true); - case O::S_BCNT1_I32_B64: - return SimpleInteger(inst, IR::ValueOpcode::BitCount64, IR::Type::U64, false, false, - true); + case O::S_BCNT1_I32_B64: { + // Vulkan bit counts operate on 32-bit words; avoid packing only to split again. + const auto value = ReadU32Pair(inst.src0); + const auto low = IR::U32(ir.Emit(IR::ValueOpcode::BitCount32, {value[0]})); + const auto high = IR::U32(ir.Emit(IR::ValueOpcode::BitCount32, {value[1]})); + const auto result = ir.IAdd(low, high); + WriteOperand(inst.dst, result); + ir.SetScc(ir.INotEqual(result, IR::U32(IR::Value(0u)))); + return; + } case O::S_FF1_I32_B32: return SimpleInteger(inst, IR::ValueOpcode::FindILsb32, IR::Type::U32, false, false, false); diff --git a/src/graphics/shader/recompiler/ir/passes/ConstantPropagation.cpp b/src/graphics/shader/recompiler/ir/passes/ConstantPropagation.cpp index 30840daad..4a6e5ec5c 100644 --- a/src/graphics/shader/recompiler/ir/passes/ConstantPropagation.cpp +++ b/src/graphics/shader/recompiler/ir/passes/ConstantPropagation.cpp @@ -347,6 +347,17 @@ void FoldInstruction(Block& block, Block::iterator instruction, if (IsImmediate(low, Type::U32) && IsImmediate(high, Type::U32)) { Replace(inst, Value(static_cast(low.U32()) | (static_cast(high.U32()) << 32u))); + } else { + const auto* low_extract = low.TryInstruction(); + const auto* high_extract = high.TryInstruction(); + if (low_extract != nullptr && high_extract != nullptr && + low_extract->GetOpcode() == ValueOpcode::CompositeExtractU64 && + high_extract->GetOpcode() == ValueOpcode::CompositeExtractU64 && + Arg(*low_extract, 1) == Value(0u) && Arg(*high_extract, 1) == Value(1u) && + Arg(*low_extract, 0) == Arg(*high_extract, 0)) { + // Guest register pairs need not unpack and repack between wide operations. + Replace(inst, Arg(*low_extract, 0)); + } } return; } diff --git a/tests/ShaderRecompilerComputeTests.cpp b/tests/ShaderRecompilerComputeTests.cpp index 89817d65b..32332873f 100644 --- a/tests/ShaderRecompilerComputeTests.cpp +++ b/tests/ShaderRecompilerComputeTests.cpp @@ -4613,6 +4613,24 @@ public: sizeof(partial_unmap_survivor_value), partial_unmap_survivor_value, false); resources.UnmapMemory(base + 0x8000, 0x4000); + Require( + name, "partial-invalidation keeps disjoint bytes dirty", + resources.IsMapped(base + partial_unmap_survivor_offset, + sizeof(partial_unmap_survivor_value)) && + !cache.IsRegionCpuModified( + base + partial_unmap_survivor_offset, + sizeof(partial_unmap_survivor_value)) && + cache.IsRegionGpuModified(base + partial_unmap_survivor_offset, + sizeof(partial_unmap_survivor_value)) && + cache.HasGpuDirtyBytes(base + partial_unmap_survivor_offset, + sizeof(partial_unmap_survivor_value)) && + !cache.HasGpuDirtyBytes(base + partial_unmap_offset, + sizeof(partial_unmap_value)), + "partial invalidation read back an unrelated mapped page"); + Require(name, "partial-invalidation survivor read fault", + resources.HandleFault(PageFaultAccess::Read, + base + partial_unmap_survivor_offset), + "CPU read did not publish the retained GPU-dirty page"); uint32_t partial_unmap_survivor_backing = 0; std::memcpy(&partial_unmap_survivor_backing, memory + partial_unmap_survivor_offset, @@ -4630,7 +4648,8 @@ public: !cache.HasGpuDirtyBytes(base + partial_unmap_survivor_offset, sizeof(partial_unmap_survivor_value)) && partial_unmap_survivor_backing == partial_unmap_survivor_value, - "widened partial invalidation mishandled ownership or backing"); + "partial invalidation or subsequent CPU read mishandled ownership or " + "backing"); resources.MapMemory(base + 0x8000, 0x4000); auto survivor = cache.ObtainBuffer(base + partial_unmap_survivor_offset, @@ -20312,6 +20331,114 @@ TestCase Scalar64BitOps() { O::V_MOV_B32, O::BUFFER_STORE_DWORD, O::S_ENDPGM}}; } +TestCase ScalarDynamic64BitOps() { + using O = ShaderOpcode; + struct Input { + uint64_t lhs; + uint64_t rhs; + u32 count; + u32 offset; + }; + constexpr Input inputs[] = { + {0, UINT64_MAX, 0, 0}, + {UINT64_MAX, 0, 63, 0}, + {0x8000000000000000ull, 1, 64, 0}, + {0x0123456789abcdefull, 0xfedcba9876543210ull, 127, 0}, + {UINT64_MAX, 0xffffffffull, 64, 1}, + {0x0123456789abcdefull, 0x0123456789abcdefull, 63, 32}, + {UINT64_MAX, 0x100000000ull, 127, 63}, + {0, 0, 0, 127}, + {0x8000000000000001ull, UINT64_MAX, UINT32_MAX, 64}, + }; + TestCase test; + test.name = "ScalarDynamic64BitOps"; + for (const auto &input : inputs) { + test.initial.insert(test.initial.end(), + {u32(input.lhs), u32(input.lhs >> 32), u32(input.rhs), + u32(input.rhs >> 32), input.count, input.offset, + ((input.count & 127u) << 16) | input.offset}); + } + test.expected = test.initial; + auto &code = test.code; + const auto store = [&](uint64_t value, bool scc) { + const auto output = static_cast(test.expected.size()); + code.push_back(EncodeSMovB32(18, 253)); + AppendStoreSgprPair(&code, 16, output); + AppendStoreSgpr(&code, 18, output + 2); + test.expected.insert(test.expected.end(), + {u32(value), u32(value >> 32), u32(scc)}); + }; + for (u32 i = 0; i < std::size(inputs); ++i) { + const auto [lhs, rhs, count, offset] = inputs[i]; + for (u32 word = 0; word < 7; ++word) { + AppendVMovU32(&code, 30, (i * 7 + word) * sizeof(u32)); + AppendBufferLoadDword(&code, 1, 30); + code.push_back(EncodeVop1(0x02, 8 + word, Vgpr(1))); + } + for (const auto [opcode, value] : + std::array, 8>{{{0x0f, lhs & rhs}, + {0x11, lhs | rhs}, + {0x13, lhs ^ rhs}, + {0x15, lhs & ~rhs}, + {0x17, lhs | ~rhs}, + {0x19, ~(lhs & rhs)}, + {0x1b, ~(lhs | rhs)}, + {0x1d, ~(lhs ^ rhs)}}}) { + code.push_back(EncodeSopc(0x06, InlineU32(0), InlineU32(value != 0))); + code.push_back(EncodeSop2(opcode, 16, 8, 10)); + store(value, value != 0); + } + for (const bool select_lhs : {false, true}) { + code.push_back(EncodeSopc(0x06, InlineU32(0), InlineU32(!select_lhs))); + code.push_back(EncodeSop2(0x0b, 16, 8, 10)); + store(select_lhs ? lhs : rhs, select_lhs); + } + // BFM preserves SCC; BFE sets it from the entire extracted result. + code.push_back(EncodeSop2(0x25, 16, 12, 13)); + store(((uint64_t{1} << (count & 63u)) - 1u) << (offset & 63u), true); + code.push_back(EncodeSop2(0x29, 16, 8, 14)); + const auto width = std::min(count & 127u, 64u - (offset & 63u)); + const auto mask = width == 64 ? UINT64_MAX : (uint64_t{1} << width) - 1u; + const auto extracted = (lhs >> (offset & 63u)) & mask; + store(extracted, extracted != 0); + for (const bool set : {false, true}) { + code.push_back(EncodeSop1(0x04, 16, 8)); + code.push_back(EncodeSop1(set ? 0x1e : 0x1c, 16, 12)); + const auto bit = uint64_t{1} << (count & 63u); + store(set ? lhs | bit : lhs & ~bit, extracted != 0); + } + code.push_back(EncodeSop1(0x2d, 16, 8)); + uint64_t quads = 0; + for (u32 quad = 0; quad < 16; ++quad) { + quads |= uint64_t((lhs >> (quad * 4) & 15u) != 0) << quad; + } + store(quads, quads != 0); + code.push_back(EncodeSop1(0x3b, 16, 8)); + uint64_t replicated = 0; + for (u32 bit = 0; bit < 32; ++bit) { + replicated |= ((lhs >> bit) & 1u) * (uint64_t{3} << (2 * bit)); + } + store(replicated, quads != 0); + code.push_back(EncodeSMovB32(17, InlineU32(0))); + code.push_back(EncodeSop1(0x10, 16, 8)); + store(std::popcount(lhs), lhs != 0); + } + AppendEnd(&code); + test.initial.resize(test.expected.size()); + test.opcodes = { + O::V_MOV_B32, O::BUFFER_LOAD_DWORD, O::V_READFIRSTLANE_B32, + O::S_MOV_B32, O::S_CMP_EQ_U32, O::S_AND_B64, + O::S_OR_B64, O::S_XOR_B64, O::S_ANDN2_B64, + O::S_ORN2_B64, O::S_NAND_B64, O::S_NOR_B64, + O::S_XNOR_B64, O::S_CSELECT_B64, O::S_BFM_B64, + O::S_BFE_U64, O::S_MOV_B64, O::S_BITSET0_B64, + O::S_BITSET1_B64, O::S_QUADMASK_B64, O::S_BITREPLICATE_B64_B32, + O::S_BCNT1_I32_B64, O::BUFFER_STORE_DWORD, O::S_ENDPGM}; + test.required_spirv = {"OpTypeInt 64 0", "OpBitwiseAnd", "OpBitwiseOr", + "OpBitwiseXor"}; + return test; +} + uint64_t ReferenceAshrI64(uint64_t bits, u32 count) { const auto shift = count & 63u; // Complementing negative values expresses sign fill using unsigned shifts. @@ -20385,8 +20512,7 @@ TestCase ScalarAshrI64Edges(bool dynamic) { O::BUFFER_STORE_DWORD, O::S_ENDPGM}; test.decoded_counts = { {"S_ASHR_I64", std::size(sources) * std::size(counts)}}; - test.required_spirv = {"OpShiftRightArithmetic"}; - test.forbidden_spirv = {"OpTypeInt 64"}; + test.required_spirv = {"OpShiftRightArithmetic", "OpTypeInt 64 0"}; return test; } @@ -20487,7 +20613,7 @@ TestCase ScalarAshrI64OperandsAndAliases() { O::S_ASHR_I64, O::V_MOV_B32, O::BUFFER_STORE_DWORD, O::S_ENDPGM}; test.decoded_counts = {{"S_ASHR_I64", instructions}}; - test.forbidden_spirv = {"OpTypeInt 64"}; + test.required_spirv.push_back("OpTypeInt 64 0"); return test; } @@ -20549,7 +20675,6 @@ TestCase ScalarAshrI64Masks(u32 wave_size) { O::S_ENDPGM}; test.decoded_counts = {{"S_ASHR_I64 vcc_lo, vcc_lo", std::size(cases)}, {"S_ASHR_I64 exec_lo, exec_lo", std::size(cases)}}; - test.forbidden_spirv = {"OpTypeInt 64"}; test.compute_info.threads_num[0] = wave_size; test.compute_info.threads_num[1] = test.compute_info.threads_num[2] = 1; test.compute_info.thread_ids_num = 1; @@ -21871,8 +21996,7 @@ TestCase VectorVop3LshrrevB64Captured() { test.expected = {0x789abcdeu, 0x00123456u}; test.opcodes = {O::S_MOV_B32, O::V_MOV_B32, O::V_LSHRREV_B64, O::BUFFER_STORE_DWORD, O::S_ENDPGM}; - test.required_spirv = {"OpShiftRightLogical", "OpSelect"}; - test.forbidden_spirv = {"OpTypeInt 64"}; + test.required_spirv = {"OpShiftRightLogical"}; return test; } @@ -21899,8 +22023,7 @@ TestCase VectorVop3LshlrevB64Captured() { test.expected = {33u, 0u, 2u, 1u, 0u}; test.opcodes = {O::BUFFER_LOAD_DWORD, O::V_LSHLREV_B64, O::V_ADD_NC_U32, O::BUFFER_STORE_DWORD, O::S_ENDPGM}; - test.required_spirv = {"OpShiftLeftLogical", "OpSelect"}; - test.forbidden_spirv = {"OpTypeInt 64"}; + test.required_spirv = {"OpShiftLeftLogical", "OpTypeInt 64 0"}; return test; } @@ -21908,7 +22031,7 @@ TestCase VectorDynamicU64ShiftEdges() { using O = ShaderOpcode; TestCase test; test.name = "VectorDynamicU64ShiftEdges"; - test.initial = {0u, 1u, 31u, 32u, 33u, 63u, 64u, 65u}; + test.initial = {0u, 1u, 31u, 32u, 33u, 63u, 64u, 65u, 127u, UINT32_MAX}; test.expected = test.initial; auto& code = test.code; for (const uint64_t source : {uint64_t{0}, uint64_t{1}, @@ -21937,8 +22060,8 @@ TestCase VectorDynamicU64ShiftEdges() { test.initial.resize(test.expected.size()); test.opcodes = {O::V_MOV_B32, O::BUFFER_LOAD_DWORD, O::V_LSHLREV_B64, O::V_LSHRREV_B64, O::BUFFER_STORE_DWORD, O::S_ENDPGM}; - test.required_spirv = {"OpShiftLeftLogical", "OpShiftRightLogical", "OpSelect"}; - test.forbidden_spirv = {"OpTypeInt 64"}; + test.required_spirv = {"OpShiftLeftLogical", "OpShiftRightLogical", + "OpTypeInt 64 0"}; return test; } @@ -22471,23 +22594,53 @@ TestCase VectorVop3BSubCoU32UsesRdna2Opcode310() { TestCase VectorMadU64U32UnsignedCarryOut() { using O = ShaderOpcode; - - std::vector code; - AppendVMovLiteral(&code, 1, 0xffffffffu); - AppendVMovU32(&code, 2, 2); - AppendVMovU32(&code, 4, 2); - AppendVMovLiteral(&code, 5, 0xffffffffu); - AppendVop3B(&code, 0x176u, 10, 20, Vgpr(1), Vgpr(2), Vgpr(4)); - AppendStoreVgpr(&code, 10, 0); - AppendStoreVgpr(&code, 11, 1); - AppendStoreSgprPair(&code, 20, 2); + struct Input { + u32 lhs; + u32 rhs; + uint64_t addend; + }; + constexpr Input inputs[] = { + {0, UINT32_MAX, UINT64_MAX}, + {UINT32_MAX, 2, 0xffffffff00000002ull}, + {UINT32_MAX, UINT32_MAX, UINT64_MAX}, + {0x80000000u, 0x80000000u, 0x4000000000000000ull}, + {3, 5, 7}, + }; + TestCase test; + test.name = "VectorMadU64U32UnsignedCarryOut"; + for (const auto &input : inputs) { + test.initial.insert( + test.initial.end(), + {input.lhs, input.rhs, u32(input.addend), u32(input.addend >> 32)}); + } + test.expected = test.initial; + auto &code = test.code; + for (u32 i = 0; i < std::size(inputs); ++i) { + for (u32 word = 0; word < 4; ++word) { + AppendVMovU32(&code, 30, (i * 4 + word) * sizeof(u32)); + AppendBufferLoadDword(&code, word < 2 ? word + 1 : word + 2, 30); + } + AppendVop3B(&code, 0x176u, 10, 20, Vgpr(1), Vgpr(2), Vgpr(4)); + const auto output = static_cast(test.expected.size()); + AppendStoreVgpr(&code, 10, output); + AppendStoreVgpr(&code, 11, output + 1); + AppendStoreSgprPair(&code, 20, output + 2); + const auto product = uint64_t(inputs[i].lhs) * inputs[i].rhs; + const auto sum = product + inputs[i].addend; + test.expected.insert(test.expected.end(), + {u32(sum), u32(sum >> 32), u32(sum < product), 0}); + } AppendEnd(&code); - - return {"VectorMadU64U32UnsignedCarryOut", - code, - {}, - {0x00000000u, 0x00000001u, 0x00000001u, 0x00000000u}, - {O::V_MOV_B32, O::V_MAD_U64_U32, O::BUFFER_STORE_DWORD, O::S_ENDPGM}}; + test.initial.resize(test.expected.size()); + test.opcodes = {O::V_MOV_B32, O::BUFFER_LOAD_DWORD, O::V_MAD_U64_U32, + O::BUFFER_STORE_DWORD, O::S_ENDPGM}; + // NVIDIA rejected the native widening multiply in a large Astro's Playroom + // vertex shader. Keep its 32x32 product portable and its 64-bit addition + // native. + test.required_spirv = {"OpTypeInt 64 0", "OpUMulExtended", "OpIMul %uint", + "OpIAdd %ulong", "OpULessThan"}; + test.forbidden_spirv = {"OpIMul %ulong", "OpIMul %long"}; + return test; } TestCase VectorLaneAndPackedOps() { @@ -34689,6 +34842,7 @@ std::vector MakeCases() { AddCase(ScalarBfeI32CapturedRawSignExtends); AddCase(BitfieldExtractWidthPastEndEdges); AddCase(Scalar64BitOps); + AddCase(ScalarDynamic64BitOps); cases.push_back(ScalarAshrI64Edges(false)); cases.push_back(ScalarAshrI64Edges(true)); AddCase(ScalarAshrI64OperandsAndAliases); @@ -40494,6 +40648,18 @@ int main(int argc, char **argv) { vulkan.CheckGpuSuspendPoint(); return 0; } + if (argc == 2 && std::strcmp(argv[1], "--native64-only") == 0) { + VulkanHarness vulkan; + RunCase(&vulkan, ScalarDynamic64BitOps()); + RunCase(&vulkan, VectorDynamicU64ShiftEdges()); + RunCase(&vulkan, ScalarAshrI64Edges(false)); + RunCase(&vulkan, ScalarAshrI64Edges(true)); + RunCase(&vulkan, VectorCompareInteger64Edges()); + RunCase(&vulkan, VectorMadU64U32UnsignedCarryOut()); + RunCase(&vulkan, ScalarWqmB64SelectsSccDomain()); + RunCase(&vulkan, ScalarWqmB64PreservesPartialMasks()); + return 0; + } if (argc == 2 && std::strcmp(argv[1], "--alignbyte-only") == 0) { VulkanHarness vulkan; RunCase(&vulkan, VectorAlignByteUsesTwoBitByteOffset()); diff --git a/tests/shaderCfgTests.cpp b/tests/shaderCfgTests.cpp index be5640305..0eb793cc3 100644 --- a/tests/shaderCfgTests.cpp +++ b/tests/shaderCfgTests.cpp @@ -7942,7 +7942,7 @@ void TestNewShaderRecompilerStructuredU64Phi() { const auto after = MeasureSpirv(spirv); Check(after.phis == before.phis + 1u && after.function_variables == before.function_variables, - "structured U64 Phi was not emitted as a native vector OpPhi"); + "structured U64 Phi was not emitted as a native scalar OpPhi"); } void TestNewShaderRecompilerCfgTerminalExitMergePS() { @@ -9866,7 +9866,7 @@ void TestNewShaderRecompilerPlanningOnlyLoads() { "planning-only loads or references emitted GPU operands/instructions"); } -void TestNewShaderRecompilerU64PairTranslation() { +void TestNewShaderRecompilerNativeU64Translation() { using namespace ShaderRecompiler; auto options = MakeCompileOptions(ShaderType::Compute); @@ -9996,20 +9996,23 @@ void TestNewShaderRecompilerU64PairTranslation() { auto spirv = ShaderRecompiler::Spirv::EmitProgram(program, options.input_info); CheckSpirvBinaryValidates(spirv); const auto source = DisassembleSpirvBinary(spirv); - Check((source.find("OpCapability Int64") == std::string::npos) && - (source.find("OpTypeInt 64") == std::string::npos), - "portable pair-U64 translation introduced native shader Int64"); - Check(SpirvInstructionOpcodeCount(spirv, 149u) == 1u, - "pair-U64 addition did not use exactly one carry instruction"); - Check(SpirvInstructionOpcodeCount(spirv, 154u) == 1u && - SpirvInstructionOpcodeCount(spirv, 155u) == 1u, - "pair-U64 equality did not reduce its vector comparison with Any/All"); + Check(source.find("OpCapability Int64") != std::string::npos && + source.find("OpTypeInt 64 0") != std::string::npos && + SpirvSourceHasInstructionUsing(source, "OpIAdd", "%ulong") && + SpirvSourceHasInstructionUsing(source, "OpIMul", "%ulong") && + SpirvSourceHasInstructionUsing(source, "OpBitwiseAnd", "%ulong"), + "U64 arithmetic did not emit native 64-bit scalar instructions"); + Check(SpirvInstructionOpcodeCount(spirv, spv::OpIAddCarry) == 0u && + SpirvInstructionOpcodeCount(spirv, spv::OpUMulExtended) == 0u && + SpirvInstructionOpcodeCount(spirv, spv::OpAny) == 0u && + SpirvInstructionOpcodeCount(spirv, spv::OpAll) == 0u, + "native U64 arithmetic retained split-word operations"); const auto direct_metrics = MeasureSpirv(spirv); - Check( - direct_metrics.words <= 940u && direct_metrics.instructions <= 199u && - direct_metrics.type_vectors == 2u && direct_metrics.phis == 0u && - direct_metrics.function_variables == 0u, - "portable pair-U64 translation exceeded its declaration/code-size ratchet"); + // The same fixture required 864 words and 182 instructions with split U64s. + Check(direct_metrics.words <= 285u && direct_metrics.instructions <= 67u && + direct_metrics.phis == 0u && + direct_metrics.function_variables == 0u, + "native U64 translation exceeded its declaration/code-size ratchet"); const uint32_t dispatcher_shader[] = { EncodeSopp(0x05, 2), EncodeSopp(0x02, 0), EncodeSopp(0x05, 0xfffeu), @@ -10034,7 +10037,39 @@ void TestNewShaderRecompilerU64PairTranslation() { Check(after.function_variables == before.function_variables + 1u && after.loads == before.loads + 1u && after.stores == before.stores + 1u, - "dispatcher did not use one canonical U64 vector spill slot"); + "dispatcher did not use one canonical U64 scalar spill slot"); +} + + +void TestVertexMadU64UsesPortableProduct() { + using StageInputKind = ShaderRecompiler::IR::StageInputKind; + for (const bool dynamic_addend : {false, true}) { + const uint32_t shader[] = { + EncodeVop1(0x01, 6, + 8 + 256), // Dynamic addend is [VertexIndex, InstanceIndex]. + EncodeVop3Word0Sdst(0x176, 0, 20), + EncodeVop3Word1(5 + 256, 8 + 256, dynamic_addend ? 5 + 256 : 128), + EncodeExp0(0x0c, 0x3), + EncodeExp1(0, 1, 0, 0), // POS0.xy keeps both product words live. + EncodeSopp(0x01), + }; + const auto result = + RecompileForTest(shader, MakeCompileOptions(ShaderType::Vertex)); + Check(ProgramHasInput(result.program, StageInputKind::VertexIndex) && + ProgramHasInput(result.program, StageInputKind::InstanceIndex), + "vertex MAD fixture lost its runtime multiplication operands"); + CheckSpirvBinaryValidates(result.spirv); + const auto source = DisassembleSpirvBinary(result.spirv); + // Large vertex shaders on NVIDIA rejected native widening multiplication. + Check(SpirvInstructionOpcodeCount(result.spirv, spv::OpUMulExtended) == + 1u && + SpirvSourceHasInstructionUsing(source, "OpIMul", "%uint") && + !SpirvSourceHasInstructionUsing(source, "OpIMul", "%ulong"), + "vertex MAD lost its portable 32x32 product"); + Check(!dynamic_addend || + SpirvSourceHasInstructionUsing(source, "OpIAdd", "%ulong"), + "vertex MAD split its live 64-bit addition into word operations"); + } } void TestComputeDispatchWaveSize() { @@ -12131,7 +12166,8 @@ void TestU64ShiftConstantPropagation() { std::vector shift_cases; for (const uint64_t source : {uint64_t{0x0123456789abcdefull}, uint64_t{0xf123456789abcdefull}}) { - for (const uint32_t count : {0u, 1u, 31u, 32u, 33u, 63u, 64u, 65u}) { + for (const uint32_t count : + {0u, 1u, 31u, 32u, 33u, 63u, 64u, 65u, 127u, UINT32_MAX}) { const uint32_t amount = count & 63u; shift_cases.push_back({shift_ir.Emit(ValueOpcode::ShiftLeftLogical64, {Value(source), Value(count)}), @@ -12150,10 +12186,41 @@ void TestU64ShiftConstantPropagation() { const auto value = test.value.Resolve(); Check(value.IsImmediate() && value.GetType() == Type::U64 && value.U64() == test.expected, - "pair-U64 shift propagation violated the masked RDNA2 count"); + "U64 shift propagation violated the masked RDNA2 count"); } } +void TestU64RegisterPairRoundTrip() { + using namespace ShaderRecompiler::IR; + Program program; + program.block_storage.push_back(std::make_unique()); + program.blocks.push_back(program.block_storage.back().get()); + IREmitter ir(program.blocks.front()); + const auto lane = ir.Emit(ValueOpcode::LaneId); + const auto pair = ir.Emit(ValueOpcode::CompositeConstructU64, {lane, lane}); + const auto first = ir.Emit(ValueOpcode::IAdd64, {pair, Value(uint64_t{1})}); + const auto second = ir.Emit(ValueOpcode::IAdd64, {pair, Value(uint64_t{2})}); + const auto low = + ir.Emit(ValueOpcode::CompositeExtractU64, {first, Value(0u)}); + const auto high = + ir.Emit(ValueOpcode::CompositeExtractU64, {first, Value(1u)}); + const auto other_high = + ir.Emit(ValueOpcode::CompositeExtractU64, {second, Value(1u)}); + const auto restored = + ir.Emit(ValueOpcode::CompositeConstructU64, {low, high}); + const auto swapped = ir.Emit(ValueOpcode::CompositeConstructU64, {high, low}); + const auto mixed = + ir.Emit(ValueOpcode::CompositeConstructU64, {low, other_high}); + ConstantPropagationPass(program.blocks); + Check(restored.Resolve() == first.Resolve(), + "unchanged SGPR pair did not retain its native U64 value"); + Check(swapped.Resolve().Instruction()->GetOpcode() == + ValueOpcode::CompositeConstructU64 && + mixed.Resolve().Instruction()->GetOpcode() == + ValueOpcode::CompositeConstructU64, + "modified SGPR pair incorrectly reused an unrelated native U64 value"); +} + #if KYTY_PLATFORM != KYTY_PLATFORM_WINDOWS void TestNativeWideValueValidation() { using namespace ShaderRecompiler::IR; @@ -14931,7 +14998,8 @@ int main() { TestNewShaderRecompilerCfgIrreducibleDispatcher(); TestNewShaderRecompilerDispatcherSpillsU32x3(); TestNewShaderRecompilerPlanningOnlyLoads(); - TestNewShaderRecompilerU64PairTranslation(); + TestNewShaderRecompilerNativeU64Translation(); + TestVertexMadU64UsesPortableProduct(); TestComputeDispatchWaveSize(); TestNewShaderRecompilerBufferLoadsGuardedByExec(); TestNewShaderRecompilerBufferAtomicsGuardedByBounds(); @@ -14956,6 +15024,7 @@ int main() { TestValuePhiValidation(); TestWave32MaskProjection(); TestU64ShiftConstantPropagation(); + TestU64RegisterPairRoundTrip(); #if KYTY_PLATFORM != KYTY_PLATFORM_WINDOWS TestNativeWideValueValidation(); #endif