raphiaura/KytyPS5
1
0
Fork 0
mirror of https://github.com/KytyPS5/KytyPS5.git synced 2026-10-25 04:16:55 +01:00
Code Issues Packages Projects Releases Wiki Activity

shader: emit native 64-bit operations where compatible

Browse Source
This commit is contained in:
nmzik
2026-10-05 20:41:03 +04:00
parent 77d7722173
commit 5b826c6e4a
16 changed files with 500 additions and 436 deletions
Download Patch File Download Diff File Expand all files Collapse all files
src/graphics/shader/recompiler/backend/spirv/spirvEmitterAlu.cpp
+29 -215
View File
@@ -17,21 +17,8 @@ Pair ExtractPair(EmitterState& state, uint32_t value) {
return result;
}
uint32_t MakePair(EmitterState& state, uint32_t low, uint32_t high) {
const auto result = state.builder.AllocateId();
state.builder.AddFunction(spv::OpCompositeConstruct, TypeU64(state), result, low, high);
return result;
}
uint32_t CompareEqual64(EmitterState& state, uint32_t lhs_value, uint32_t rhs_value,
bool not_equal) {
const auto compare = Binary(state, not_equal ? spv::OpINotEqual : spv::OpIEqual,
TypeBoolVector(state, 2), lhs_value, rhs_value);
return Unary(state, not_equal ? spv::OpAny : spv::OpAll, TypeBool(state), compare);
}
uint32_t EmitMinMaxF64(EmitterState& state, uint32_t lhs, uint32_t rhs, bool max_value) {
const auto bits_type = TypeU64(state);
const auto bits_type = TypeU32Vector(state, 2);
const auto lhs_bits = Unary(state, spv::OpBitcast, bits_type, lhs);
const auto rhs_bits = Unary(state, spv::OpBitcast, bits_type, rhs);
const auto ordered = Binary(state, max_value ? spv::OpFOrdGreaterThan : spv::OpFOrdLessThan,
@@ -52,17 +39,6 @@ uint32_t EmitMinMaxF64(EmitterState& state, uint32_t lhs, uint32_t rhs, bool max
rhs, result);
}
uint32_t CompareOrdered64(EmitterState& state, uint32_t lhs_value, uint32_t rhs_value,
spv::Op high_compare, spv::Op low_compare) {
const auto lhs = ExtractPair(state, lhs_value);
const auto rhs = ExtractPair(state, rhs_value);
const auto high_equal = Binary(state, spv::OpIEqual, TypeBool(state), lhs.high, rhs.high);
const auto high_result = Binary(state, high_compare, TypeBool(state), lhs.high, rhs.high);
const auto low_result = Binary(state, low_compare, TypeBool(state), lhs.low, rhs.low);
const auto low_path = Binary(state, spv::OpLogicalAnd, TypeBool(state), high_equal, low_result);
return Binary(state, spv::OpLogicalOr, TypeBool(state), high_result, low_path);
}
uint32_t EmitMulHigh(EmitterState& state, uint32_t lhs, uint32_t rhs, bool signed_value) {
const auto operand_type = signed_value ? TypeI32(state) : TypeU32(state);
const auto pair_type = signed_value ? TypeI32Pair(state) : TypeU32Pair(state);
@@ -80,86 +56,6 @@ uint32_t EmitMulHigh(EmitterState& state, uint32_t lhs, uint32_t rhs, bool signe
return signed_value ? Unary(state, spv::OpBitcast, TypeU32(state), high) : high;
}
uint32_t EmitShift64(EmitterState& state, spv::Op opcode, uint32_t value, uint32_t shift) {
const auto pair = ExtractPair(state, value);
const auto amount = EmitAndConstant(state, shift, 63u);
const auto word_shift = EmitAndConstant(state, amount, 31u);
const auto at_least_32 =
Binary(state, spv::OpUGreaterThanEqual, TypeBool(state), amount, ConstantU32(state, 32));
const auto nonzero =
Binary(state, spv::OpINotEqual, TypeBool(state), amount, ConstantU32(state, 0));
const auto carry_count = EmitAndConstant(
state, Binary(state, spv::OpISub, TypeU32(state), ConstantU32(state, 32), word_shift), 31u);
if (opcode == spv::OpShiftLeftLogical) {
const auto low = Binary(state, opcode, TypeU32(state), pair.low, word_shift);
const auto carry =
Select(state, TypeU32(state), nonzero,
Binary(state, spv::OpShiftRightLogical, TypeU32(state), pair.low, carry_count),
ConstantU32(state, 0));
const auto high =
Binary(state, spv::OpBitwiseOr, TypeU32(state),
Binary(state, opcode, TypeU32(state), pair.high, word_shift), carry);
return MakePair(state,
Select(state, TypeU32(state), at_least_32, ConstantU32(state, 0), low),
Select(state, TypeU32(state), at_least_32, low, high));
}
const auto high = Binary(state, opcode, TypeU32(state), pair.high, word_shift);
const auto carry =
Select(state, TypeU32(state), nonzero,
Binary(state, spv::OpShiftLeftLogical, TypeU32(state), pair.high, carry_count),
ConstantU32(state, 0));
const auto low = Binary(
state, spv::OpBitwiseOr, TypeU32(state),
Binary(state, spv::OpShiftRightLogical, TypeU32(state), pair.low, word_shift), carry);
const auto fill = opcode == spv::OpShiftRightArithmetic
? Binary(state, opcode, TypeU32(state), pair.high, ConstantU32(state, 31))
: ConstantU32(state, 0);
return MakePair(state, Select(state, TypeU32(state), at_least_32, high, low),
Select(state, TypeU32(state), at_least_32, fill, high));
}
uint32_t EmitConstantShift64(EmitterState& state, spv::Op opcode, uint32_t value, uint32_t shift) {
shift &= 63u;
if (shift == 0u) {
return value;
}
const auto pair = ExtractPair(state, value);
if (opcode == spv::OpShiftLeftLogical) {
if (shift < 32u) {
const auto low = Binary(state, spv::OpShiftLeftLogical, TypeU32(state), pair.low,
ConstantU32(state, shift));
const auto high = Binary(state, spv::OpBitwiseOr, TypeU32(state),
Binary(state, spv::OpShiftLeftLogical, TypeU32(state),
pair.high, ConstantU32(state, shift)),
Binary(state, spv::OpShiftRightLogical, TypeU32(state),
pair.low, ConstantU32(state, 32u - shift)));
return MakePair(state, low, high);
}
return MakePair(state, ConstantU32(state, 0),
shift == 32u ? pair.low
: Binary(state, spv::OpShiftLeftLogical, TypeU32(state),
pair.low, ConstantU32(state, shift - 32u)));
}
if (shift < 32u) {
const auto low = Binary(state, spv::OpBitwiseOr, TypeU32(state),
Binary(state, spv::OpShiftRightLogical, TypeU32(state), pair.low,
ConstantU32(state, shift)),
Binary(state, spv::OpShiftLeftLogical, TypeU32(state), pair.high,
ConstantU32(state, 32u - shift)));
const auto high =
Binary(state, opcode, TypeU32(state), pair.high, ConstantU32(state, shift));
return MakePair(state, low, high);
}
const auto high = opcode == spv::OpShiftRightArithmetic
? Binary(state, spv::OpShiftRightArithmetic, TypeU32(state), pair.high,
ConstantU32(state, 31u))
: ConstantU32(state, 0);
const auto low = shift == 32u ? pair.high
: Binary(state, opcode, TypeU32(state), pair.high,
ConstantU32(state, shift - 32u));
return MakePair(state, low, high);
}
uint32_t EmitMinMax3(EmitterState& state, uint32_t a, uint32_t b, uint32_t c, bool signed_value,
bool max_value) {
const auto ab = signed_value ? EmitMinMaxI32Value(state, a, b, max_value)
@@ -259,57 +155,15 @@ uint32_t EmitFPMedTri32(EmitterState& state, uint32_t a, uint32_t b, uint32_t c)
}
uint32_t EmitFindUMsb64(EmitterState& state, uint32_t value) {
const auto pair = ExtractPair(state, value);
const auto high_i = state.builder.AllocateId();
const auto low_i = state.builder.AllocateId();
state.builder.AddFunction(spv::OpExtInst, TypeI32(state), high_i, GlslStd450(state),
GLSLstd450FindUMsb, pair.high);
state.builder.AddFunction(spv::OpExtInst, TypeI32(state), low_i, GlslStd450(state),
GLSLstd450FindUMsb, pair.low);
const auto high = Unary(state, spv::OpBitcast, TypeU32(state), high_i);
const auto low = Unary(state, spv::OpBitcast, TypeU32(state), low_i);
const auto high_nonzero =
Binary(state, spv::OpINotEqual, TypeBool(state), pair.high, ConstantU32(state, 0));
return Select(state, TypeU32(state), high_nonzero,
Binary(state, spv::OpIAdd, TypeU32(state), high, ConstantU32(state, 32)), low);
}
uint32_t EmitIMul64(EmitterState& state, uint32_t lhs_value, uint32_t rhs_value) {
const auto lhs = ExtractPair(state, lhs_value);
const auto rhs = ExtractPair(state, rhs_value);
const auto low = Binary(state, spv::OpIMul, TypeU32(state), lhs.low, rhs.low);
const auto high0 = EmitMulHigh(state, lhs.low, rhs.low, false);
const auto high1 = Binary(state, spv::OpIMul, TypeU32(state), lhs.low, rhs.high);
const auto high2 = Binary(state, spv::OpIMul, TypeU32(state), lhs.high, rhs.low);
return MakePair(state, low,
Binary(state, spv::OpIAdd, TypeU32(state),
Binary(state, spv::OpIAdd, TypeU32(state), high0, high1), high2));
}
uint32_t EmitISub64(EmitterState& state, uint32_t lhs_value, uint32_t rhs_value) {
const auto lhs = ExtractPair(state, lhs_value);
const auto rhs = ExtractPair(state, rhs_value);
const auto low = Binary(state, spv::OpISub, TypeU32(state), lhs.low, rhs.low);
const auto borrow = Binary(state, spv::OpULessThan, TypeBool(state), lhs.low, rhs.low);
const auto borrow_u32 =
Select(state, TypeU32(state), borrow, ConstantU32(state, 1), ConstantU32(state, 0));
const auto high0 = Binary(state, spv::OpISub, TypeU32(state), lhs.high, rhs.high);
const auto high = Binary(state, spv::OpISub, TypeU32(state), high0, borrow_u32);
return MakePair(state, low, high);
}
uint32_t EmitIAdd64(EmitterState& state, uint32_t lhs_value, uint32_t rhs_value) {
const auto lhs = ExtractPair(state, lhs_value);
const auto rhs = ExtractPair(state, rhs_value);
const auto low_pair = state.builder.AllocateId();
const auto low = state.builder.AllocateId();
const auto carry = state.builder.AllocateId();
state.builder.AddFunction(spv::OpIAddCarry, TypeU32Pair(state), low_pair, lhs.low, rhs.low);
state.builder.AddFunction(spv::OpCompositeExtract, TypeU32(state), low, low_pair, 0);
state.builder.AddFunction(spv::OpCompositeExtract, TypeU32(state), carry, low_pair, 1);
const auto high0 = Binary(state, spv::OpIAdd, TypeU32(state), lhs.high, rhs.high);
const auto high = Binary(state, spv::OpIAdd, TypeU32(state), high0, carry);
return MakePair(state, low, high);
// GLSL FindUMsb is restricted to 32-bit components.
const auto unpacked = Unary(state, spv::OpBitcast, TypeU32Vector(state, 2), value);
const auto pair =
ExtractPair(state, EmitExt(state, TypeU32Vector(state, 2), GLSLstd450FindUMsb, {unpacked}));
const auto high_found =
Binary(state, spv::OpINotEqual, TypeBool(state), pair.high, ConstantU32(state, UINT32_MAX));
return Select(state, TypeU32(state), high_found,
Binary(state, spv::OpIAdd, TypeU32(state), pair.high, ConstantU32(state, 32)),
pair.low);
}
uint32_t EmitConvertU16U32(EmitterState& state, uint32_t arg0) {
@@ -338,7 +192,7 @@ uint32_t EmitConvertU32F32(EmitterState& state, uint32_t arg0) {
uint32_t EmitConvertF32F64(EmitterState& state, uint32_t arg0) {
const auto converted = Unary(state, spv::OpFConvert, TypeF32(state), arg0);
const auto source = EmitNative<spv::OpCompositeExtract, IR::Type::U32>(
state, Unary(state, spv::OpBitcast, TypeU64(state), arg0), 1u);
state, Unary(state, spv::OpBitcast, TypeU32Vector(state, 2), arg0), 1u);
const auto exponent = EmitAndConstant(state, source, 0x7ff00000u);
const auto overflow =
Binary(state, spv::OpLogicalAnd, TypeBool(state),
@@ -356,7 +210,21 @@ uint32_t EmitConvertF64F32(EmitterState& state, uint32_t arg0) {
EmitFlushF32DenormToSignedZero(state, arg0));
}
uint32_t EmitCompositeConstructU64(ValueEmitContext& ctx, uint32_t arg0, IR::Value arg1) {
arg1 = arg1.Resolve();
return arg1.IsImmediate() && arg1.U32() == 0u
? Unary(ctx.state, spv::OpUConvert, TypeU64(ctx.state), arg0)
: PackU64(ctx.state, arg0, ctx.Def(arg1));
}
uint32_t EmitCompositeExtractU64(EmitterState& state, uint32_t arg0, IR::Value arg1) {
const auto value = arg1.U32() == 0u ? arg0
: Binary(state, spv::OpShiftRightLogical, TypeU64(state),
arg0, ConstantU32(state, 32));
return Unary(state, spv::OpUConvert, TypeU32(state), value);
}
uint32_t EmitCompositeExtractU32x2(EmitterState& state, uint32_t arg0, IR::Value arg1) {
return EmitNative<spv::OpCompositeExtract, IR::Type::U32>(state, arg0, arg1.U32());
}
@@ -388,33 +256,11 @@ uint32_t EmitIAbs32(EmitterState& state, uint32_t arg0) {
return Select(state, TypeU32(state), negative, neg, value);
}
uint32_t EmitShiftLeftLogical64(ValueEmitContext& ctx, uint32_t arg0, IR::Value arg1) {
auto& state = ctx.state;
return arg1.Resolve().IsImmediate()
? EmitConstantShift64(state, spv::OpShiftLeftLogical, arg0, arg1.Resolve().U32())
: EmitShift64(state, spv::OpShiftLeftLogical, arg0, ctx.Def(arg1));
}
uint32_t EmitShiftRightLogical64(ValueEmitContext& ctx, uint32_t arg0, IR::Value arg1) {
auto& state = ctx.state;
return arg1.Resolve().IsImmediate()
? EmitConstantShift64(state, spv::OpShiftRightLogical, arg0, arg1.Resolve().U32())
: EmitShift64(state, spv::OpShiftRightLogical, arg0, ctx.Def(arg1));
}
uint32_t EmitShiftRightArithmetic64(ValueEmitContext& ctx, uint32_t arg0, IR::Value arg1) {
auto& state = ctx.state;
return arg1.Resolve().IsImmediate()
? EmitConstantShift64(state, spv::OpShiftRightArithmetic, arg0, arg1.Resolve().U32())
: EmitShift64(state, spv::OpShiftRightArithmetic, arg0, ctx.Def(arg1));
}
uint32_t EmitBitwiseAnd64(EmitterState& state, uint32_t arg0, uint32_t arg1) {
return Binary(state, spv::OpBitwiseAnd, TypeU64(state), arg0, arg1);
}
uint32_t EmitBitCount64(EmitterState& state, uint32_t arg0) {
const auto pair = ExtractPair(state, Unary(state, spv::OpBitCount, TypeU64(state), arg0));
// Vulkan bit-count operands remain 32-bit, including when counting a U64.
const auto unpacked = Unary(state, spv::OpBitcast, TypeU32Vector(state, 2), arg0);
const auto pair =
ExtractPair(state, Unary(state, spv::OpBitCount, TypeU32Vector(state, 2), unpacked));
return Binary(state, spv::OpIAdd, TypeU32(state), pair.low, pair.high);
}
@@ -468,38 +314,6 @@ uint32_t EmitUMedTri32(EmitterState& state, uint32_t arg0, uint32_t arg1, uint32
return EmitMed3(state, arg0, arg1, arg2, false);
}
uint32_t EmitIEqual64(EmitterState& state, uint32_t arg0, uint32_t arg1) {
return CompareEqual64(state, arg0, arg1, false);
}
uint32_t EmitINotEqual64(EmitterState& state, uint32_t arg0, uint32_t arg1) {
return CompareEqual64(state, arg0, arg1, true);
}
uint32_t EmitULessThan64(EmitterState& state, uint32_t arg0, uint32_t arg1) {
return CompareOrdered64(state, arg0, arg1, spv::OpULessThan, spv::OpULessThan);
}
uint32_t EmitSLessThan64(EmitterState& state, uint32_t arg0, uint32_t arg1) {
return CompareOrdered64(state, arg0, arg1, spv::OpSLessThan, spv::OpULessThan);
}
uint32_t EmitUGreaterThan64(EmitterState& state, uint32_t arg0, uint32_t arg1) {
return CompareOrdered64(state, arg0, arg1, spv::OpUGreaterThan, spv::OpUGreaterThan);
}
uint32_t EmitSLessThanEqual64(EmitterState& state, uint32_t arg0, uint32_t arg1) {
return CompareOrdered64(state, arg0, arg1, spv::OpSLessThan, spv::OpULessThanEqual);
}
uint32_t EmitULessThanEqual64(EmitterState& state, uint32_t arg0, uint32_t arg1) {
return CompareOrdered64(state, arg0, arg1, spv::OpULessThan, spv::OpULessThanEqual);
}
uint32_t EmitUGreaterThanEqual64(EmitterState& state, uint32_t arg0, uint32_t arg1) {
return CompareOrdered64(state, arg0, arg1, spv::OpUGreaterThan, spv::OpUGreaterThanEqual);
}
uint32_t EmitFPIsNan32(EmitterState& state, uint32_t arg0) {
return EmitNative<spv::OpFUnordNotEqual, IR::Type::U1>(state, arg0, arg0);
}
src/graphics/shader/recompiler/backend/spirv/spirvEmitterAnalysis.cpp
+2 -2
View File
@@ -136,8 +136,8 @@ uint32_t ImageType(EmitterState& state, const IR::ImageResource& image) {
EXIT("invalid image resource class");
}
const auto& info = ImageDimensionInfoFor(image.dimension);
const auto scalar_type = image.atomic64 ? TypeScalarU64(state)
: ImageScalarType(state, image.numeric_class);
const auto scalar_type =
image.atomic64 ? TypeU64(state) : ImageScalarType(state, image.numeric_class);
return state.builder.Type(spv::OpTypeImage, scalar_type,
info.spirv_dimension, image.depth_compare ? 1u : 0u, info.arrayed,
info.multisampled, sampled, format);
src/graphics/shader/recompiler/backend/spirv/spirvEmitterBvh.cpp
+21 -20
View File
@@ -19,8 +19,8 @@ uint32_t Vector(EmitterState& s, uint32_t type, std::span<const uint32_t> compon
uint32_t NodeWord(EmitterState& s, uint32_t block, uint32_t word) {
const auto offset = Binary(s, spv::OpShiftLeftLogical, TypeU32(s), word, ConstantU32(s, 2));
const auto address = Binary(s, spv::OpIAdd, TypeScalarU64(s), block,
Unary(s, spv::OpUConvert, TypeScalarU64(s), offset));
const auto address =
Binary(s, spv::OpIAdd, TypeU64(s), block, Unary(s, spv::OpUConvert, TypeU64(s), offset));
const auto pointer = Unary(s, spv::OpConvertUToPtr, TypePhysicalU32Pointer(s), address);
const auto result = s.builder.AllocateId();
s.builder.AddFunction(spv::OpLoad, TypeU32(s), result, pointer, spv::MemoryAccessAlignedMask, 4u);
@@ -224,7 +224,7 @@ uint32_t Boxes(EmitterState& s, uint32_t first, uint32_t second, uint32_t half,
void DefineBvhIntersect(EmitterState& s) {
if (!s.requirements.bvh) return;
const auto u = TypeU32(s), f = TypeF32(s), b = TypeBool(s), wide = TypeScalarU64(s);
const auto u = TypeU32(s), f = TypeF32(s), b = TypeBool(s), wide = TypeU64(s);
const auto vec4 = TypeU32Vector(s, 4), vec3 = TypeF32Vector(s, 3);
const auto signature = s.builder.Type(spv::OpTypeFunction, vec4, vec4, wide, f, vec3, vec3, vec3);
s.bvh_intersect_function = s.builder.AllocateId();
@@ -260,22 +260,24 @@ void DefineBvhIntersect(EmitterState& s) {
const auto bary = Binary(s, spv::OpINotEqual, b, and_bits(words[3], 0x01000000), ConstantU32(s, 0));
const auto sort = Binary(s, spv::OpINotEqual, b, and_bits(words[1], 0x80000000), ConstantU32(s, 0));
const auto grow = and_bits(shr(words[1], 23), 0xff);
const auto base = Binary(s, spv::OpShiftLeftLogical, wide,
DeviceAddressFromWords(s, words[0], and_bits(words[1], 0xff)), ConstantDeviceAddress(s, 8));
const auto index = Binary(s, spv::OpShiftRightLogical, wide, node, ConstantDeviceAddress(s, 3));
const auto base = Binary(s, spv::OpShiftLeftLogical, wide,
PackU64(s, words[0], and_bits(words[1], 0xff)), ConstantU32(s, 8));
const auto index = Binary(s, spv::OpShiftRightLogical, wide, node, ConstantU32(s, 3));
const auto last_index = Binary(s, spv::OpIAdd, wide, index,
Select(s, wide, full, ConstantDeviceAddress(s, 1), ConstantDeviceAddress(s, 0)));
Select(s, wide, full, ConstantU64(s, 1), ConstantU64(s, 0)));
const auto address = Binary(s, spv::OpIAdd, wide, base,
Binary(s, spv::OpShiftLeftLogical, wide, index, ConstantDeviceAddress(s, 6)));
const auto last_address = Binary(s, spv::OpIAdd, wide, base,
Binary(s, spv::OpShiftLeftLogical, wide, last_index, ConstantDeviceAddress(s, 6)));
Binary(s, spv::OpShiftLeftLogical, wide, index, ConstantU32(s, 6)));
const auto last_address =
Binary(s, spv::OpIAdd, wide, base,
Binary(s, spv::OpShiftLeftLogical, wide, last_index, ConstantU32(s, 6)));
auto valid = Binary(s, spv::OpIEqual, b, shr(words[3], 28), ConstantU32(s, 8));
const auto require = [&](uint32_t condition) {
valid = Binary(s, spv::OpLogicalAnd, b, valid, condition);
};
require(Binary(s, spv::OpULessThanEqual, b, kind, ConstantU32(s, 5)));
require(Binary(s, spv::OpULessThanEqual, b, last_index, DeviceAddressFromWords(s, words[2], and_bits(words[3], 0x3ff))));
require(Binary(s, spv::OpULessThan, b, last_address, ConstantDeviceAddress(s, uint64_t{1} << 48)));
require(Binary(s, spv::OpULessThanEqual, b, last_index,
PackU64(s, words[2], and_bits(words[3], 0x3ff))));
require(Binary(s, spv::OpULessThan, b, last_address, ConstantU64(s, uint64_t {1} << 48)));
require(Unary(s, spv::OpLogicalNot, b, Unary(s, spv::OpIsNan, b, extent)));
for (uint32_t i = 0; i < 3; ++i) {
for (const auto value: {origin[i], direction[i], inverse[i]})
@@ -292,13 +294,13 @@ void DefineBvhIntersect(EmitterState& s) {
const auto invalid = Vector(s, vec4, invalid_words);
const auto result = EmitValueOrDefaultIfCondition(s, valid, vec4, invalid, [&] {
const auto first = GetBdaPointer(s, address);
const auto second = EmitValueOrDefaultIfCondition(s, full, wide, ConstantDeviceAddress(s, 0), [&] {
return GetBdaPointer(s, Binary(s, spv::OpIAdd, wide, address, ConstantDeviceAddress(s, 64)));
const auto second = EmitValueOrDefaultIfCondition(s, full, wide, ConstantU64(s, 0), [&] {
return GetBdaPointer(s, Binary(s, spv::OpIAdd, wide, address, ConstantU64(s, 64)));
});
const auto present = Binary(s, spv::OpLogicalAnd, b,
Binary(s, spv::OpINotEqual, b, first, ConstantDeviceAddress(s, 0)),
const auto present = Binary(
s, spv::OpLogicalAnd, b, Binary(s, spv::OpINotEqual, b, first, ConstantU64(s, 0)),
Binary(s, spv::OpLogicalOr, b, Unary(s, spv::OpLogicalNot, b, full),
Binary(s, spv::OpINotEqual, b, second, ConstantDeviceAddress(s, 0))));
Binary(s, spv::OpINotEqual, b, second, ConstantU64(s, 0))));
return EmitValueOrDefaultIfCondition(s, present, vec4, invalid, [&] {
const auto tri_label = s.builder.AllocateId(), box_label = s.builder.AllocateId();
const auto merge = s.builder.AllocateId();
@@ -336,9 +338,8 @@ uint32_t EmitBvhIntersect(ValueEmitContext& ctx, const IR::Inst& inst) {
return Vector(s, TypeF32Vector(s, 3), values);
};
const auto descriptor = ctx.Arg(inst, 0);
const auto node = node_words == 2u
? DeviceAddressFromWords(s, ctx.Arg(*ray, 0), ctx.Arg(*ray, 1))
: Unary(s, spv::OpUConvert, TypeScalarU64(s), ctx.Arg(*ray, 0));
const auto node = node_words == 2u ? PackU64(s, ctx.Arg(*ray, 0), ctx.Arg(*ray, 1))
: Unary(s, spv::OpUConvert, TypeU64(s), ctx.Arg(*ray, 0));
const auto extent = Unary(s, spv::OpBitcast, TypeF32(s), ctx.Arg(*ray, node_words));
const auto origin = vector(node_words + 1u), direction = vector(node_words + 4u),
inverse = vector(node_words + 7u);
src/graphics/shader/recompiler/backend/spirv/spirvEmitterFlow.cpp
+3 -3
View File
@@ -430,16 +430,16 @@ uint32_t EmitWqmU64(EmitterState& state, uint32_t value) {
const auto quad_bits = state.builder.AllocateId();
const auto result = state.builder.AllocateId();
state.builder.AddFunction(spv::OpShiftRightLogical, TypeU64(state), shifted_one, value,
ConstantU64(state, 0x0000000100000001ull));
ConstantU32(state, 1));
state.builder.AddFunction(spv::OpBitwiseOr, TypeU64(state), merged_one, value, shifted_one);
state.builder.AddFunction(spv::OpShiftRightLogical, TypeU64(state), shifted_two, merged_one,
ConstantU64(state, 0x0000000200000002ull));
ConstantU32(state, 2));
state.builder.AddFunction(spv::OpBitwiseOr, TypeU64(state), merged_two, merged_one,
shifted_two);
state.builder.AddFunction(spv::OpBitwiseAnd, TypeU64(state), quad_bits, merged_two,
ConstantU64(state, 0x1111111111111111ull));
state.builder.AddFunction(spv::OpIMul, TypeU64(state), result, quad_bits,
ConstantU64(state, 0x0000000f0000000full));
ConstantU64(state, 15));
return result;
}
src/graphics/shader/recompiler/backend/spirv/spirvEmitterImage.cpp
+3 -3
View File
@@ -930,9 +930,9 @@ void EmitImage(ValueEmitContext& ctx, const IR::Inst& inst) {
ctx.Define(inst, EmitValueOrDefaultIfCondition(
state, ctx.Arg(inst, inst.NumArgs() - 1), result_type, zero, [&]() {
const auto pointer = state.builder.AllocateId();
const auto pointer_type = state.builder.Type(
spv::OpTypePointer, spv::StorageClassImage,
image.atomic64 ? TypeScalarU64(state) : TypeU32(state));
const auto pointer_type =
state.builder.Type(spv::OpTypePointer, spv::StorageClassImage,
image.atomic64 ? TypeU64(state) : TypeU32(state));
state.builder.AddFunction(spv::OpImageTexelPointer, pointer_type, pointer,
ImageDescriptorPointer(state, mem.resource),
CoordU32(ctx, mem, *address, dimension),
src/graphics/shader/recompiler/backend/spirv/spirvEmitterInstructions.h
+31 -19
View File
@@ -34,16 +34,16 @@ uint32_t EmitConvertF32F64(EmitterState& state, uint32_t arg0);
uint32_t EmitConvertF64F32(EmitterState& state, uint32_t arg0);
EMIT_NATIVE(ConvertF32U32, OpConvertUToF, F32, uint32_t)
EMIT_NATIVE(ConvertF64U32, OpConvertUToF, F64, uint32_t)
EMIT_NATIVE(CompositeConstructU64, OpCompositeConstruct, U64, uint32_t, uint32_t)
uint32_t EmitCompositeConstructU64(ValueEmitContext& ctx, uint32_t arg0, IR::Value arg1);
EMIT_NATIVE(CompositeConstructU32x2, OpCompositeConstruct, U32x2, uint32_t, uint32_t)
EMIT_NATIVE(CompositeConstructU32x3, OpCompositeConstruct, U32x3, uint32_t, uint32_t, uint32_t)
EMIT_NATIVE(CompositeConstructF32x2, OpCompositeConstruct, F32x2, uint32_t, uint32_t)
EMIT_NATIVE(CompositeConstructU32x4, OpCompositeConstruct, U32x4, uint32_t, uint32_t, uint32_t,
uint32_t)
uint32_t EmitCompositeExtractU64(EmitterState& state, uint32_t arg0, IR::Value arg1);
inline constexpr auto EmitCompositeExtractU32x2 = EmitCompositeExtractU64;
inline constexpr auto EmitCompositeExtractU32x3 = EmitCompositeExtractU64;
inline constexpr auto EmitCompositeExtractU32x4 = EmitCompositeExtractU64;
uint32_t EmitCompositeExtractU32x2(EmitterState& state, uint32_t arg0, IR::Value arg1);
inline constexpr auto EmitCompositeExtractU32x3 = EmitCompositeExtractU32x2;
inline constexpr auto EmitCompositeExtractU32x4 = EmitCompositeExtractU32x2;
inline constexpr auto EmitPackHalf2x16 = EmitGlsl<GLSLstd450PackHalf2x16, IR::Type::U32, uint32_t>;
inline constexpr auto EmitPackSnorm2x16 =
EmitGlsl<GLSLstd450PackSnorm2x16, IR::Type::U32, uint32_t>;
@@ -63,9 +63,9 @@ EMIT_NATIVE(IAdd32, OpIAdd, U32, uint32_t, uint32_t)
EMIT_NATIVE(ISub32, OpISub, U32, uint32_t, uint32_t)
EMIT_NATIVE(IMul32, OpIMul, U32, uint32_t, uint32_t)
EMIT_NATIVE(UDiv32, OpUDiv, U32, uint32_t, uint32_t)
uint32_t EmitIAdd64(EmitterState& state, uint32_t arg0, uint32_t arg1);
uint32_t EmitISub64(EmitterState& state, uint32_t arg0, uint32_t arg1);
uint32_t EmitIMul64(EmitterState& state, uint32_t arg0, uint32_t arg1);
EMIT_NATIVE(IAdd64, OpIAdd, U64, uint32_t, uint32_t)
EMIT_NATIVE(ISub64, OpISub, U64, uint32_t, uint32_t)
EMIT_NATIVE(IMul64, OpIMul, U64, uint32_t, uint32_t)
EMIT_NATIVE(IAddCarry32, OpIAddCarry, U32x2, uint32_t, uint32_t)
uint32_t EmitSMulHi(EmitterState& state, uint32_t arg0, uint32_t arg1);
uint32_t EmitUMulHi(EmitterState& state, uint32_t arg0, uint32_t arg1);
@@ -73,14 +73,26 @@ uint32_t EmitIAbs32(EmitterState& state, uint32_t arg0);
EMIT_NATIVE(ShiftLeftLogical32, OpShiftLeftLogical, U32, uint32_t, uint32_t)
EMIT_NATIVE(ShiftRightLogical32, OpShiftRightLogical, U32, uint32_t, uint32_t)
EMIT_NATIVE(ShiftRightArithmetic32, OpShiftRightArithmetic, U32, uint32_t, uint32_t)
uint32_t EmitShiftLeftLogical64(ValueEmitContext& ctx, uint32_t arg0, IR::Value arg1);
uint32_t EmitShiftRightLogical64(ValueEmitContext& ctx, uint32_t arg0, IR::Value arg1);
uint32_t EmitShiftRightArithmetic64(ValueEmitContext& ctx, uint32_t arg0, IR::Value arg1);
template <spv::Op opcode>
uint32_t EmitShift64(ValueEmitContext& ctx, uint32_t value, IR::Value shift) {
shift = shift.Resolve();
if (shift.IsImmediate()) {
const auto amount = shift.U32() & 63u;
return amount == 0u ? value
: Binary(ctx.state, opcode, TypeU64(ctx.state), value,
ConstantU32(ctx.state, amount));
}
return Binary(ctx.state, opcode, TypeU64(ctx.state), value,
EmitAndConstant(ctx.state, ctx.Def(shift), 63u));
}
inline constexpr auto EmitShiftLeftLogical64 = EmitShift64<spv::OpShiftLeftLogical>;
inline constexpr auto EmitShiftRightLogical64 = EmitShift64<spv::OpShiftRightLogical>;
inline constexpr auto EmitShiftRightArithmetic64 = EmitShift64<spv::OpShiftRightArithmetic>;
EMIT_NATIVE(BitwiseAnd32, OpBitwiseAnd, U32, uint32_t, uint32_t)
EMIT_NATIVE(BitwiseOr32, OpBitwiseOr, U32, uint32_t, uint32_t)
EMIT_NATIVE(BitwiseXor32, OpBitwiseXor, U32, uint32_t, uint32_t)
EMIT_NATIVE(BitwiseNot32, OpNot, U32, uint32_t)
uint32_t EmitBitwiseAnd64(EmitterState& state, uint32_t arg0, uint32_t arg1);
EMIT_NATIVE(BitwiseAnd64, OpBitwiseAnd, U64, uint32_t, uint32_t)
EMIT_NATIVE(BitReverse32, OpBitReverse, U32, uint32_t)
EMIT_NATIVE(BitCount32, OpBitCount, U32, uint32_t)
uint32_t EmitBitCount64(EmitterState& state, uint32_t arg0);
@@ -107,14 +119,14 @@ EMIT_NATIVE(UGreaterThan32, OpUGreaterThan, U1, uint32_t, uint32_t)
EMIT_NATIVE(INotEqual32, OpINotEqual, U1, uint32_t, uint32_t)
EMIT_NATIVE(SGreaterThanEqual32, OpSGreaterThanEqual, U1, uint32_t, uint32_t)
EMIT_NATIVE(UGreaterThanEqual32, OpUGreaterThanEqual, U1, uint32_t, uint32_t)
uint32_t EmitIEqual64(EmitterState& state, uint32_t arg0, uint32_t arg1);
uint32_t EmitINotEqual64(EmitterState& state, uint32_t arg0, uint32_t arg1);
uint32_t EmitULessThan64(EmitterState& state, uint32_t arg0, uint32_t arg1);
uint32_t EmitSLessThan64(EmitterState& state, uint32_t arg0, uint32_t arg1);
uint32_t EmitSLessThanEqual64(EmitterState& state, uint32_t arg0, uint32_t arg1);
uint32_t EmitULessThanEqual64(EmitterState& state, uint32_t arg0, uint32_t arg1);
uint32_t EmitUGreaterThanEqual64(EmitterState& state, uint32_t arg0, uint32_t arg1);
uint32_t EmitUGreaterThan64(EmitterState& state, uint32_t arg0, uint32_t arg1);
EMIT_NATIVE(IEqual64, OpIEqual, U1, uint32_t, uint32_t)
EMIT_NATIVE(INotEqual64, OpINotEqual, U1, uint32_t, uint32_t)
EMIT_NATIVE(ULessThan64, OpULessThan, U1, uint32_t, uint32_t)
EMIT_NATIVE(SLessThan64, OpSLessThan, U1, uint32_t, uint32_t)
EMIT_NATIVE(SLessThanEqual64, OpSLessThanEqual, U1, uint32_t, uint32_t)
EMIT_NATIVE(ULessThanEqual64, OpULessThanEqual, U1, uint32_t, uint32_t)
EMIT_NATIVE(UGreaterThanEqual64, OpUGreaterThanEqual, U1, uint32_t, uint32_t)
EMIT_NATIVE(UGreaterThan64, OpUGreaterThan, U1, uint32_t, uint32_t)
EMIT_NATIVE(LogicalOr, OpLogicalOr, U1, uint32_t, uint32_t)
EMIT_NATIVE(LogicalAnd, OpLogicalAnd, U1, uint32_t, uint32_t)
EMIT_NATIVE(LogicalXor, OpLogicalNotEqual, U1, uint32_t, uint32_t)
src/graphics/shader/recompiler/backend/spirv/spirvEmitterInternal.h
+7 -4
View File
@@ -98,7 +98,7 @@ struct EmitterState {
uint32_t void_type = 0;
uint32_t bool_type = 0;
uint32_t u32_type = 0;
uint32_t native_u64_type = 0;
uint32_t u64_type = 0;
uint32_t i32_type = 0;
uint32_t f32_type = 0;
uint32_t f64_type = 0;
@@ -168,7 +168,6 @@ uint32_t TypeBool(EmitterState& state);
uint32_t TypeBoolVector(EmitterState& state, uint32_t components);
uint32_t TypeU32(EmitterState& state);
uint32_t TypeU64(EmitterState& state);
uint32_t TypeScalarU64(EmitterState& state);
uint32_t TypeU32Pair(EmitterState& state);
uint32_t TypeI32(EmitterState& state);
uint32_t TypeI32Pair(EmitterState& state);
@@ -221,6 +220,12 @@ inline uint32_t Unary(EmitterState& state, spv::Op opcode, uint32_t type, uint32
return result;
}
inline uint32_t PackU64(EmitterState& state, uint32_t low, uint32_t high) {
const auto pair = state.builder.AllocateId();
state.builder.AddFunction(spv::OpCompositeConstruct, TypeU32Vector(state, 2), pair, low, high);
return Unary(state, spv::OpBitcast, TypeU64(state), pair);
}
inline uint32_t Binary(EmitterState& state, spv::Op opcode, uint32_t type, uint32_t lhs,
uint32_t rhs) {
const auto result = state.builder.AllocateId();
@@ -531,8 +536,6 @@ void EmitProgram(EmitterState& state);
void DefineGetBdaPointer(EmitterState& state);
void DefineBvhIntersect(EmitterState& state);
uint32_t GetBdaPointer(EmitterState& state, uint32_t address);
uint32_t ConstantDeviceAddress(EmitterState& state, uint64_t value);
uint32_t DeviceAddressFromWords(EmitterState& state, uint32_t low, uint32_t high);
// These templates accept local lambdas from several emitter translation units.
template <typename Fn>
src/graphics/shader/recompiler/backend/spirv/spirvEmitterMemory.cpp
+70 -86
View File
@@ -133,22 +133,22 @@ uint32_t GuestAddress(ValueEmitContext& ctx, const IR::Inst& inst, const IR::Mem
}
uint32_t address = 0;
if (mem.address_is_full) {
address = DeviceAddressFromWords(state, low, ctx.Arg(inst, 2));
address = PackU64(state, low, ctx.Arg(inst, 2));
} else {
const auto* handle = inst.Arg(0).Resolve().TryInstruction();
if (handle == nullptr || handle->GetOpcode() != IR::ValueOpcode::GetAddressResource ||
handle->NumArgs() != 2) {
ctx.Fail(inst, "has no address base pair");
return ConstantDeviceAddress(state, 0);
return ConstantU64(state, 0);
}
auto base_low = ctx.Arg(*handle, 0);
if (mem.kind == IR::ResourceKind::ScalarAddress) {
base_low = Binary(state, spv::OpBitwiseAnd, TypeU32(state), base_low,
ConstantU32(state, ~3u));
}
const auto base = DeviceAddressFromWords(state, base_low, ctx.Arg(*handle, 1));
address = Binary(state, spv::OpIAdd, TypeScalarU64(state), base,
Unary(state, spv::OpUConvert, TypeScalarU64(state), low));
const auto base = PackU64(state, base_low, ctx.Arg(*handle, 1));
address = Binary(state, spv::OpIAdd, TypeU64(state), base,
Unary(state, spv::OpUConvert, TypeU64(state), low));
}
auto immediate = static_cast<int32_t>(mem.offset);
if (mem.kind == IR::ResourceKind::ScalarAddress) {
@@ -156,9 +156,8 @@ uint32_t GuestAddress(ValueEmitContext& ctx, const IR::Inst& inst, const IR::Mem
}
return immediate == 0
? address
: Binary(state, spv::OpIAdd, TypeScalarU64(state), address,
ConstantDeviceAddress(
state, static_cast<uint64_t>(static_cast<int64_t>(immediate))));
: Binary(state, spv::OpIAdd, TypeU64(state), address,
ConstantU64(state, static_cast<uint64_t>(static_cast<int64_t>(immediate))));
}
uint32_t FaultElementPointer(EmitterState& state, uint32_t index) {
@@ -185,7 +184,7 @@ uint32_t LoadBdaDword(ValueEmitContext& ctx, uint32_t address) {
auto& state = ctx.state;
const auto bda = GetBdaPointer(state, address);
const auto present =
Binary(state, spv::OpINotEqual, TypeBool(state), bda, ConstantDeviceAddress(state, 0));
Binary(state, spv::OpINotEqual, TypeBool(state), bda, ConstantU64(state, 0));
return EmitValueOrZeroIfCondition(state, present, [&]() {
const auto pointer = state.builder.AllocateId();
state.builder.AddFunction(spv::OpConvertUToPtr, TypePhysicalU32Pointer(state), pointer,
@@ -201,8 +200,8 @@ uint32_t LoadBdaDword(ValueEmitContext& ctx, uint32_t address) {
uint32_t LoadBda(ValueEmitContext& ctx, uint32_t address, uint32_t active, uint32_t bits) {
auto& state = ctx.state;
return EmitValueOrZeroIfCondition(state, active, [&]() {
const auto aligned = Binary(state, spv::OpBitwiseAnd, TypeScalarU64(state), address,
ConstantDeviceAddress(state, ~uint64_t {3}));
const auto aligned = Binary(state, spv::OpBitwiseAnd, TypeU64(state), address,
ConstantU64(state, ~uint64_t {3}));
const auto first = LoadBdaDword(ctx, aligned);
const auto byte =
Binary(state, spv::OpBitwiseAnd, TypeU32(state),
@@ -212,8 +211,8 @@ uint32_t LoadBda(ValueEmitContext& ctx, uint32_t address, uint32_t active, uint3
: Binary(state, bits == 16u ? spv::OpUGreaterThan : spv::OpINotEqual,
TypeBool(state), byte, ConstantU32(state, bits == 16u ? 2u : 0u));
const auto second = EmitValueOrZeroIfCondition(state, crosses, [&]() {
return LoadBdaDword(ctx, Binary(state, spv::OpIAdd, TypeScalarU64(state), aligned,
ConstantDeviceAddress(state, sizeof(uint32_t))));
return LoadBdaDword(ctx, Binary(state, spv::OpIAdd, TypeU64(state), aligned,
ConstantU64(state, sizeof(uint32_t))));
});
const auto shift =
Binary(state, spv::OpShiftLeftLogical, TypeU32(state), byte, ConstantU32(state, 3));
@@ -805,26 +804,25 @@ uint32_t LoadIndirectScalarBuffer(ValueEmitContext& ctx, const IR::Inst& inst) {
const auto word1 = ctx.Arg(handle, 1);
const auto stride = EmitBitFieldUExtract(state, word1, ConstantU32(state, 16),
ConstantU32(state, 14));
const auto size = Binary(
state, spv::OpIMul, TypeScalarU64(state),
Unary(state, spv::OpUConvert, TypeScalarU64(state),
EmitUMax32(state, stride, ConstantU32(state, 1))),
Unary(state, spv::OpUConvert, TypeScalarU64(state), ctx.Arg(handle, 2)));
const auto offset = Binary(
state, spv::OpIAdd, TypeScalarU64(state),
Unary(state, spv::OpUConvert, TypeScalarU64(state),
Binary(state, spv::OpBitwiseAnd, TypeU32(state), ctx.Arg(inst, 1),
ConstantU32(state, ~3u))),
ConstantDeviceAddress(state, ctx.Memory(inst).offset & ~3u));
const auto end = Binary(state, spv::OpIAdd, TypeScalarU64(state), offset,
ConstantDeviceAddress(state, sizeof(uint32_t)));
const auto size = Binary(state, spv::OpIMul, TypeU64(state),
Unary(state, spv::OpUConvert, TypeU64(state),
EmitUMax32(state, stride, ConstantU32(state, 1))),
Unary(state, spv::OpUConvert, TypeU64(state), ctx.Arg(handle, 2)));
const auto offset = Binary(state, spv::OpIAdd, TypeU64(state),
Unary(state, spv::OpUConvert, TypeU64(state),
Binary(state, spv::OpBitwiseAnd, TypeU32(state),
ctx.Arg(inst, 1), ConstantU32(state, ~3u))),
ConstantU64(state, ctx.Memory(inst).offset & ~3u));
const auto end =
Binary(state, spv::OpIAdd, TypeU64(state), offset, ConstantU64(state, sizeof(uint32_t)));
return EmitValueOrZeroIfCondition(
state, Binary(state, spv::OpULessThanEqual, TypeBool(state), end, size), [&]() {
const auto base = DeviceAddressFromWords(
state, Binary(state, spv::OpBitwiseAnd, TypeU32(state), ctx.Arg(handle, 0),
ConstantU32(state, ~3u)),
Binary(state, spv::OpBitwiseAnd, TypeU32(state), word1, ConstantU32(state, 0xffffu)));
return LoadBdaDword(ctx, Binary(state, spv::OpIAdd, TypeScalarU64(state), base, offset));
const auto base = PackU64(state,
Binary(state, spv::OpBitwiseAnd, TypeU32(state),
ctx.Arg(handle, 0), ConstantU32(state, ~3u)),
Binary(state, spv::OpBitwiseAnd, TypeU32(state), word1,
ConstantU32(state, 0xffffu)));
return LoadBdaDword(ctx, Binary(state, spv::OpIAdd, TypeU64(state), base, offset));
});
}
@@ -865,21 +863,21 @@ IndirectBufferAccess PrepareIndirectBuffer(ValueEmitContext& ctx, const IR::Inst
const auto address = CalculateBufferAddress(state, index, ctx.Arg(inst, 2), soffset,
ctx.Memory(inst).offset, stride, swizzle,
field(word3, 21, 2));
const auto base = DeviceAddressFromWords(state, ctx.Arg(handle, 0), field(word1, 0, 16));
const auto base = PackU64(state, ctx.Arg(handle, 0), field(word1, 0, 16));
const auto raw_records = Binary(state, spv::OpISub, TypeU32(state), records, soffset);
return {
.address = Binary(state, spv::OpIAdd, TypeScalarU64(state), base,
Unary(state, spv::OpUConvert, TypeScalarU64(state), address.byte)),
.offset = address.offset,
.stride = stride,
.swizzle = swizzle,
.format = field(word3, 12, 7),
.selector = ctx.Memory(inst).formatted ? field(word3, 0, 3) : 0u,
.mode = field(word3, 28, 2),
.records = records,
.index_in_bounds = Binary(state, spv::OpULessThan, TypeBool(state), index, records),
.address = Binary(state, spv::OpIAdd, TypeU64(state), base,
Unary(state, spv::OpUConvert, TypeU64(state), address.byte)),
.offset = address.offset,
.stride = stride,
.swizzle = swizzle,
.format = field(word3, 12, 7),
.selector = ctx.Memory(inst).formatted ? field(word3, 0, 3) : 0u,
.mode = field(word3, 28, 2),
.records = records,
.index_in_bounds = Binary(state, spv::OpULessThan, TypeBool(state), index, records),
.scalar_in_bounds = Binary(state, spv::OpULessThanEqual, TypeBool(state), soffset, records),
.raw_records = raw_records,
.raw_records = raw_records,
.raw_index_in_bounds = Binary(state, spv::OpULessThan, TypeBool(state), index, raw_records),
};
}
@@ -925,13 +923,13 @@ uint32_t LoadIndirectBuffer(ValueEmitContext& ctx, const IR::Inst& inst, uint32_
const auto buffer = PrepareIndirectBuffer(ctx, inst);
const auto valid_format = Binary(state, spv::OpINotEqual, TypeBool(state), buffer.format,
ConstantU32(state, 0));
const auto base = Binary(state, spv::OpBitwiseAnd, TypeScalarU64(state), buffer.address,
ConstantDeviceAddress(state, ~uint64_t {3}));
const auto base = Binary(state, spv::OpBitwiseAnd, TypeU64(state), buffer.address,
ConstantU64(state, ~uint64_t {3}));
std::array<uint32_t, 4> values {};
for (uint32_t component = 0; component < components; ++component) {
const auto address = component == 0u ? base : Binary(
state, spv::OpIAdd, TypeScalarU64(state), base,
ConstantDeviceAddress(state, component * 4u));
const auto address = component == 0u ? base
: Binary(state, spv::OpIAdd, TypeU64(state), base,
ConstantU64(state, component * 4u));
values[component] = EmitValueOrZeroIfCondition(
state, AndCondition(state, valid_format,
IndirectBufferInBounds(state, buffer, component * 4u, 4u, false)),
@@ -959,13 +957,14 @@ uint32_t LoadIndirectFormattedX(ValueEmitContext& ctx, const IR::Inst& inst) {
TypeU32(state), constant, [&]() {
const auto in_bounds = IndirectBufferInBounds(state, buffer, 0u, info.byte_size, true);
const auto base = Binary(
state, spv::OpBitwiseAnd, TypeScalarU64(state), buffer.address,
ConstantDeviceAddress(state, ~(uint64_t(std::min(4u, info.byte_size)) - 1u)));
state, spv::OpBitwiseAnd, TypeU64(state), buffer.address,
ConstantU64(state, ~(uint64_t(std::min(4u, info.byte_size)) - 1u)));
const auto load = [&](uint32_t component, uint32_t bits) {
const auto offset = Format::GetFormatComponentByteOffset(info, component);
const auto address = offset == 0u ? base : Binary(
state, spv::OpIAdd, TypeScalarU64(state), base,
ConstantDeviceAddress(state, offset));
const auto address = offset == 0u
? base
: Binary(state, spv::OpIAdd, TypeU64(state),
base, ConstantU64(state, offset));
return LoadBda(ctx, address, in_bounds, bits);
};
const auto emit_component = [&](uint32_t component) {
@@ -1085,23 +1084,9 @@ void StoreWideShared(ValueEmitContext& ctx, const IR::Inst& inst, uint32_t compo
} // namespace
uint32_t ConstantDeviceAddress(EmitterState& state, uint64_t value) {
return state.builder.Constant(spv::OpConstant, TypeScalarU64(state),
static_cast<uint32_t>(value),
static_cast<uint32_t>(value >> 32u));
}
uint32_t DeviceAddressFromWords(EmitterState& state, uint32_t low, uint32_t high) {
const auto low64 = Unary(state, spv::OpUConvert, TypeScalarU64(state), low);
const auto high64 = Binary(state, spv::OpShiftLeftLogical, TypeScalarU64(state),
Unary(state, spv::OpUConvert, TypeScalarU64(state), high),
ConstantDeviceAddress(state, 32));
return Binary(state, spv::OpBitwiseOr, TypeScalarU64(state), low64, high64);
}
uint32_t GetBdaPointer(EmitterState& state, uint32_t address) {
const auto result = state.builder.AllocateId();
state.builder.AddFunction(spv::OpFunctionCall, TypeScalarU64(state), result,
state.builder.AddFunction(spv::OpFunctionCall, TypeU64(state), result,
state.bda_pointer_function, address);
return result;
}
@@ -1110,7 +1095,7 @@ void DefineGetBdaPointer(EmitterState& state) {
if (!state.program.info.uses_dma) {
return;
}
const auto type = TypeScalarU64(state);
const auto type = TypeU64(state);
const auto function_type = state.builder.Type(spv::OpTypeFunction, type, type);
state.bda_pointer_function = state.builder.AllocateId();
const auto address = state.builder.AllocateId();
@@ -1122,14 +1107,14 @@ void DefineGetBdaPointer(EmitterState& state) {
EmitLabel(state, entry_label);
const auto extended = Binary(state, spv::OpUGreaterThanEqual, TypeBool(state), address,
ConstantDeviceAddress(state, LibKernel::Memory::kExtendedMemoryBase));
const auto packed = Select(state, type, extended,
Binary(state, spv::OpISub, type, address,
ConstantDeviceAddress(state,
LibKernel::Memory::kExtendedMemoryBase - LOWER_ADDRESS_SIZE)),
address);
ConstantU64(state, LibKernel::Memory::kExtendedMemoryBase));
const auto packed = Select(
state, type, extended,
Binary(state, spv::OpISub, type, address,
ConstantU64(state, LibKernel::Memory::kExtendedMemoryBase - LOWER_ADDRESS_SIZE)),
address);
const auto page64 = Binary(state, spv::OpShiftRightLogical, type, packed,
ConstantDeviceAddress(state, BufferCache::CACHING_PAGEBITS));
ConstantU32(state, BufferCache::CACHING_PAGEBITS));
const auto page = Unary(state, spv::OpUConvert, TypeU32(state), page64);
const auto entry_pointer = state.builder.AllocateId();
state.builder.AddFunction(spv::OpAccessChain, TypeStorageBufferElementPointer(state, 64),
@@ -1137,8 +1122,7 @@ void DefineGetBdaPointer(EmitterState& state) {
page);
const auto base = state.builder.AllocateId();
state.builder.AddFunction(spv::OpLoad, type, base, entry_pointer);
const auto missing =
Binary(state, spv::OpIEqual, TypeBool(state), base, ConstantDeviceAddress(state, 0));
const auto missing = Binary(state, spv::OpIEqual, TypeBool(state), base, ConstantU64(state, 0));
const auto fault_label = state.builder.AllocateId();
const auto available_label = state.builder.AllocateId();
const auto merge_label = state.builder.AllocateId();
@@ -1151,14 +1135,14 @@ void DefineGetBdaPointer(EmitterState& state) {
EmitLabel(state, available_label);
const auto offset = Binary(state, spv::OpBitwiseAnd, type, address,
ConstantDeviceAddress(state, BufferCache::CACHING_PAGESIZE - 1));
ConstantU64(state, BufferCache::CACHING_PAGESIZE - 1));
const auto available = Binary(state, spv::OpIAdd, type, base, offset);
state.builder.AddFunction(spv::OpBranch, merge_label);
EmitLabel(state, merge_label);
const auto result = state.builder.AllocateId();
state.builder.AddFunction(spv::OpPhi, type, result, ConstantDeviceAddress(state, 0),
fault_label, available, available_label);
state.builder.AddFunction(spv::OpPhi, type, result, ConstantU64(state, 0), fault_label,
available, available_label);
state.builder.AddFunction(spv::OpReturnValue, result);
state.builder.AddFunction(spv::OpFunctionEnd);
}
@@ -1216,12 +1200,12 @@ void EmitSharedAtomic64(ValueEmitContext& ctx, const IR::Inst& inst) {
semantics = spv::MemorySemanticsUniformMemoryMask;
}
const auto pointer = state.builder.AllocateId();
state.builder.AddFunction(spv::OpAccessChain,
TypePointer(state, state.lds_storage_class, TypeScalarU64(state)),
pointer, state.lds_u64_variable, ConstantU32(state, 0), native_index);
const auto value = Unary(state, spv::OpBitcast, TypeScalarU64(state), ctx.Arg(inst, 1));
state.builder.AddFunction(
SpirvAtomicOpcode(inst.GetOpcode()), TypeScalarU64(state), state.builder.AllocateId(),
spv::OpAccessChain, TypePointer(state, state.lds_storage_class, TypeU64(state)),
pointer, state.lds_u64_variable, ConstantU32(state, 0), native_index);
const auto value = ctx.Arg(inst, 1);
state.builder.AddFunction(
SpirvAtomicOpcode(inst.GetOpcode()), TypeU64(state), state.builder.AllocateId(),
pointer, ConstantU32(state, spv::ScopeWorkgroup),
ConstantU32(state, spv::MemorySemanticsAcquireReleaseMask | semantics), value);
});
src/graphics/shader/recompiler/backend/spirv/spirvEmitterMemoryHelpers.cpp
+4 -5
View File
@@ -138,7 +138,7 @@ void EnsureLdsStorage(EmitterState& state) {
};
if (state.requirements.shared_int64_atomics) {
state.lds_variable = define(TypeU32(state), 4u);
state.lds_u64_variable = define(TypeScalarU64(state), 8u);
state.lds_u64_variable = define(TypeU64(state), 8u);
state.builder.AddName(state.lds_u64_variable, "lds_qwords");
} else {
state.lds_variable = state.builder.DefineGlobalVariable(
@@ -410,7 +410,7 @@ uint32_t EmitAtomicOperation(ValueEmitContext& ctx, const IR::Inst& inst, uint32
const auto opcode = SpirvAtomicOpcode(inst.GetOpcode());
const auto old = ctx.state.builder.AllocateId();
const bool wide = inst.GetType() == IR::Type::U64;
const auto type = wide ? TypeScalarU64(ctx.state) : TypeU32(ctx.state);
const auto type = wide ? TypeU64(ctx.state) : TypeU32(ctx.state);
if (opcode == spv::OpAtomicCompareExchange) {
const auto desired = ctx.Arg(inst, inst.NumArgs() - 3);
const auto comparator = ctx.Arg(inst, inst.NumArgs() - 2);
@@ -419,13 +419,12 @@ uint32_t EmitAtomicOperation(ValueEmitContext& ctx, const IR::Inst& inst, uint32
ConstantU32(ctx.state, scope), ConstantU32(ctx.state, spv::MemorySemanticsMaskNone),
ConstantU32(ctx.state, spv::MemorySemanticsMaskNone), desired, comparator);
} else {
auto value = ctx.Arg(inst, inst.NumArgs() - 2);
if (wide) value = Unary(ctx.state, spv::OpBitcast, type, value);
const auto value = ctx.Arg(inst, inst.NumArgs() - 2);
ctx.state.builder.AddFunction(opcode, type, old, pointer,
ConstantU32(ctx.state, scope),
ConstantU32(ctx.state, spv::MemorySemanticsMaskNone), value);
}
return wide ? Unary(ctx.state, spv::OpBitcast, TypeU64(ctx.state), old) : old;
return old;
}
void EmitAtomicMemoryBarrier(EmitterState& state, IR::ResourceKind kind) {
src/graphics/shader/recompiler/backend/spirv/spirvEmitterModule.cpp
+9 -12
View File
@@ -35,14 +35,11 @@ uint32_t TypeU32(EmitterState& state) {
}
uint32_t TypeU64(EmitterState& state) {
return TypeU32Vector(state, 2);
}
uint32_t TypeScalarU64(EmitterState& state) {
if (state.native_u64_type == 0) {
state.native_u64_type = state.builder.Type(spv::OpTypeInt, 64, 0);
if (state.u64_type == 0) {
state.builder.RequireCapability(spv::CapabilityInt64);
state.u64_type = state.builder.Type(spv::OpTypeInt, 64, 0);
}
return state.native_u64_type;
return state.u64_type;
}
uint32_t TypeU32Pair(EmitterState& state) {
@@ -124,8 +121,9 @@ uint32_t TypeFunction(EmitterState& state) {
}
uint32_t TypeStorageBufferElement(EmitterState& state, uint32_t bits) {
return bits == 32u ? TypeU32(state) : bits == 64u ? TypeScalarU64(state)
: state.builder.Type(spv::OpTypeInt, bits, 0);
return bits == 32u ? TypeU32(state)
: bits == 64u ? TypeU64(state)
: state.builder.Type(spv::OpTypeInt, bits, 0);
}
uint32_t StorageRuntimeArrayType(EmitterState& state, uint32_t bits) {
@@ -331,9 +329,8 @@ uint32_t ConstantBool(EmitterState& state, bool value) {
}
uint32_t ConstantU64(EmitterState& state, uint64_t value) {
return state.builder.Constant(spv::OpConstantComposite, TypeU64(state),
ConstantU32(state, static_cast<uint32_t>(value)),
ConstantU32(state, static_cast<uint32_t>(value >> 32u)));
return state.builder.Constant(spv::OpConstant, TypeU64(state), static_cast<uint32_t>(value),
static_cast<uint32_t>(value >> 32u));
}
uint32_t ConstantU32CompositeZero(EmitterState& state, uint32_t components) {
src/graphics/shader/recompiler/frontend/translate/Control.cpp
+1
View File
@@ -248,6 +248,7 @@ void Translator::S_CSELECT_B32(const Decoder::Instruction& inst) {
void Translator::ScalarSelect64(const Decoder::Instruction& inst,
const Decoder::Operand& false_source) {
// Preserve per-word expressions for descriptor tracking and mask provenance.
const auto condition = ir.GetScc();
const auto lhs = ReadU32Pair(inst.src0);
const auto rhs = ReadU32Pair(false_source);
src/graphics/shader/recompiler/frontend/translate/Integer.cpp
+17 -17
View File
@@ -114,6 +114,7 @@ void Translator::S_U64_MASK(const Decoder::Instruction& inst, IR::ValueOpcode lo
return;
}
// Descriptor tracking reasons about each scalar word independently.
const auto lhs = ReadU32Pair(inst.src0);
auto mask_valid = ReadMaskValid(inst.src0);
if (inst.src_count > 1u) {
@@ -269,22 +270,18 @@ void Translator::Integer24(const Decoder::Instruction& inst, bool sign, bool add
}
void Translator::V_MAD_U64_U32(const Decoder::Instruction& inst) {
const auto lhs = ReadU32(inst.src0);
const auto rhs = ReadU32(inst.src1);
const auto add = ExtractU64(ReadU64(inst.src2));
const auto mul_low = ir.IMul(lhs, rhs);
const auto mul_high = IR::U32(ir.Emit(IR::ValueOpcode::UMulHi, {lhs, rhs}));
const auto low = ir.IAdd(mul_low, add[0]);
const auto carry_low = ir.ULessThan(low, mul_low);
const auto high0 = ir.IAdd(mul_high, add[1]);
const auto carry0 = ir.ULessThan(high0, mul_high);
const auto high =
ir.IAdd(high0, ir.Select(carry_low, IR::U32(IR::Value(1u)), IR::U32(IR::Value(0u))));
const auto carry1 = ir.ULessThan(high, high0);
WriteOperand(DestinationOperand(inst), ir.ConstructU64(low, high));
const auto lhs = ReadU32(inst.src0);
const auto rhs = ReadU32(inst.src1);
// Keep the 32x32 product in words: native widening IMul64 fails NVIDIA compilation
// in large vertex shaders (Astro's Playroom). The 64-bit addition stays native.
const auto product = ir.ConstructU64(
ir.IMul(lhs, rhs), IR::U32(ir.Emit(IR::ValueOpcode::UMulHi, {lhs, rhs})));
const auto result = ir.Emit(IR::ValueOpcode::IAdd64, {product, ReadU64(inst.src2)});
WriteOperand(DestinationOperand(inst), result);
if (inst.dst2.kind != Decoder::OperandKind::Null &&
inst.dst2.kind != Decoder::OperandKind::Unknown) {
WriteMask(inst.dst2, ir.LogicalOr(carry0, carry1));
// The 32x32 product fits in 64 bits, so only the addition can carry.
WriteMask(inst.dst2, IR::U1(ir.Emit(IR::ValueOpcode::ULessThan64, {result, product})));
}
}
@@ -407,9 +404,12 @@ IR::U64 Translator::RightMask64(IR::U32 count) {
}
void Translator::S_BFM_B64(const Decoder::Instruction& inst) {
const auto count = ir.BitwiseAnd(ReadU32(inst.src0), IR::U32(IR::Value(63u)));
const auto offset = ir.BitwiseAnd(ReadU32(inst.src1), IR::U32(IR::Value(63u)));
const auto result = ir.Emit(IR::ValueOpcode::ShiftLeftLogical64, {RightMask64(count), offset});
const auto count = ReadU32(inst.src0);
const auto offset = ReadU32(inst.src1);
const auto one = IR::Value(uint64_t {1});
const auto limit = ir.Emit(IR::ValueOpcode::ShiftLeftLogical64, {one, count});
const auto mask = ir.Emit(IR::ValueOpcode::ISub64, {limit, one});
const auto result = ir.Emit(IR::ValueOpcode::ShiftLeftLogical64, {mask, offset});
WriteOperand(DestinationOperand(inst), result);
}
src/graphics/shader/recompiler/frontend/translate/Scalar.cpp
+10 -3
View File
@@ -165,9 +165,16 @@ void Translator::EmitScalar(const Decoder::Instruction& inst) {
case O::S_BCNT1_I32_B32:
return SimpleInteger(inst, IR::ValueOpcode::BitCount32, IR::Type::U32, false, false,
true);
case O::S_BCNT1_I32_B64:
return SimpleInteger(inst, IR::ValueOpcode::BitCount64, IR::Type::U64, false, false,
true);
case O::S_BCNT1_I32_B64: {
// Vulkan bit counts operate on 32-bit words; avoid packing only to split again.
const auto value = ReadU32Pair(inst.src0);
const auto low = IR::U32(ir.Emit(IR::ValueOpcode::BitCount32, {value[0]}));
const auto high = IR::U32(ir.Emit(IR::ValueOpcode::BitCount32, {value[1]}));
const auto result = ir.IAdd(low, high);
WriteOperand(inst.dst, result);
ir.SetScc(ir.INotEqual(result, IR::U32(IR::Value(0u))));
return;
}
case O::S_FF1_I32_B32:
return SimpleInteger(inst, IR::ValueOpcode::FindILsb32, IR::Type::U32, false, false,
false);
src/graphics/shader/recompiler/ir/passes/ConstantPropagation.cpp
+11
View File
@@ -347,6 +347,17 @@ void FoldInstruction(Block& block, Block::iterator instruction,
if (IsImmediate(low, Type::U32) && IsImmediate(high, Type::U32)) {
Replace(inst, Value(static_cast<uint64_t>(low.U32()) |
(static_cast<uint64_t>(high.U32()) << 32u)));
} else {
const auto* low_extract = low.TryInstruction();
const auto* high_extract = high.TryInstruction();
if (low_extract != nullptr && high_extract != nullptr &&
low_extract->GetOpcode() == ValueOpcode::CompositeExtractU64 &&
high_extract->GetOpcode() == ValueOpcode::CompositeExtractU64 &&
Arg(*low_extract, 1) == Value(0u) && Arg(*high_extract, 1) == Value(1u) &&
Arg(*low_extract, 0) == Arg(*high_extract, 0)) {
// Guest register pairs need not unpack and repack between wide operations.
Replace(inst, Arg(*low_extract, 0));
}
}
return;
}
tests/ShaderRecompilerComputeTests.cpp
+194 -28
View File
@@ -4613,6 +4613,24 @@ public:
sizeof(partial_unmap_survivor_value),
partial_unmap_survivor_value, false);
resources.UnmapMemory(base + 0x8000, 0x4000);
Require(
name, "partial-invalidation keeps disjoint bytes dirty",
resources.IsMapped(base + partial_unmap_survivor_offset,
sizeof(partial_unmap_survivor_value)) &&
!cache.IsRegionCpuModified(
base + partial_unmap_survivor_offset,
sizeof(partial_unmap_survivor_value)) &&
cache.IsRegionGpuModified(base + partial_unmap_survivor_offset,
sizeof(partial_unmap_survivor_value)) &&
cache.HasGpuDirtyBytes(base + partial_unmap_survivor_offset,
sizeof(partial_unmap_survivor_value)) &&
!cache.HasGpuDirtyBytes(base + partial_unmap_offset,
sizeof(partial_unmap_value)),
"partial invalidation read back an unrelated mapped page");
Require(name, "partial-invalidation survivor read fault",
resources.HandleFault(PageFaultAccess::Read,
base + partial_unmap_survivor_offset),
"CPU read did not publish the retained GPU-dirty page");
uint32_t partial_unmap_survivor_backing = 0;
std::memcpy(&partial_unmap_survivor_backing,
memory + partial_unmap_survivor_offset,
@@ -4630,7 +4648,8 @@ public:
!cache.HasGpuDirtyBytes(base + partial_unmap_survivor_offset,
sizeof(partial_unmap_survivor_value)) &&
partial_unmap_survivor_backing == partial_unmap_survivor_value,
"widened partial invalidation mishandled ownership or backing");
"partial invalidation or subsequent CPU read mishandled ownership or "
"backing");
resources.MapMemory(base + 0x8000, 0x4000);
auto survivor =
cache.ObtainBuffer(base + partial_unmap_survivor_offset,
@@ -20312,6 +20331,114 @@ TestCase Scalar64BitOps() {
O::V_MOV_B32, O::BUFFER_STORE_DWORD, O::S_ENDPGM}};
}
TestCase ScalarDynamic64BitOps() {
using O = ShaderOpcode;
struct Input {
uint64_t lhs;
uint64_t rhs;
u32 count;
u32 offset;
};
constexpr Input inputs[] = {
{0, UINT64_MAX, 0, 0},
{UINT64_MAX, 0, 63, 0},
{0x8000000000000000ull, 1, 64, 0},
{0x0123456789abcdefull, 0xfedcba9876543210ull, 127, 0},
{UINT64_MAX, 0xffffffffull, 64, 1},
{0x0123456789abcdefull, 0x0123456789abcdefull, 63, 32},
{UINT64_MAX, 0x100000000ull, 127, 63},
{0, 0, 0, 127},
{0x8000000000000001ull, UINT64_MAX, UINT32_MAX, 64},
};
TestCase test;
test.name = "ScalarDynamic64BitOps";
for (const auto &input : inputs) {
test.initial.insert(test.initial.end(),
{u32(input.lhs), u32(input.lhs >> 32), u32(input.rhs),
u32(input.rhs >> 32), input.count, input.offset,
((input.count & 127u) << 16) | input.offset});
}
test.expected = test.initial;
auto &code = test.code;
const auto store = [&](uint64_t value, bool scc) {
const auto output = static_cast<u32>(test.expected.size());
code.push_back(EncodeSMovB32(18, 253));
AppendStoreSgprPair(&code, 16, output);
AppendStoreSgpr(&code, 18, output + 2);
test.expected.insert(test.expected.end(),
{u32(value), u32(value >> 32), u32(scc)});
};
for (u32 i = 0; i < std::size(inputs); ++i) {
const auto [lhs, rhs, count, offset] = inputs[i];
for (u32 word = 0; word < 7; ++word) {
AppendVMovU32(&code, 30, (i * 7 + word) * sizeof(u32));
AppendBufferLoadDword(&code, 1, 30);
code.push_back(EncodeVop1(0x02, 8 + word, Vgpr(1)));
}
for (const auto [opcode, value] :
std::array<std::pair<u32, uint64_t>, 8>{{{0x0f, lhs & rhs},
{0x11, lhs | rhs},
{0x13, lhs ^ rhs},
{0x15, lhs & ~rhs},
{0x17, lhs | ~rhs},
{0x19, ~(lhs & rhs)},
{0x1b, ~(lhs | rhs)},
{0x1d, ~(lhs ^ rhs)}}}) {
code.push_back(EncodeSopc(0x06, InlineU32(0), InlineU32(value != 0)));
code.push_back(EncodeSop2(opcode, 16, 8, 10));
store(value, value != 0);
}
for (const bool select_lhs : {false, true}) {
code.push_back(EncodeSopc(0x06, InlineU32(0), InlineU32(!select_lhs)));
code.push_back(EncodeSop2(0x0b, 16, 8, 10));
store(select_lhs ? lhs : rhs, select_lhs);
}
// BFM preserves SCC; BFE sets it from the entire extracted result.
code.push_back(EncodeSop2(0x25, 16, 12, 13));
store(((uint64_t{1} << (count & 63u)) - 1u) << (offset & 63u), true);
code.push_back(EncodeSop2(0x29, 16, 8, 14));
const auto width = std::min(count & 127u, 64u - (offset & 63u));
const auto mask = width == 64 ? UINT64_MAX : (uint64_t{1} << width) - 1u;
const auto extracted = (lhs >> (offset & 63u)) & mask;
store(extracted, extracted != 0);
for (const bool set : {false, true}) {
code.push_back(EncodeSop1(0x04, 16, 8));
code.push_back(EncodeSop1(set ? 0x1e : 0x1c, 16, 12));
const auto bit = uint64_t{1} << (count & 63u);
store(set ? lhs | bit : lhs & ~bit, extracted != 0);
}
code.push_back(EncodeSop1(0x2d, 16, 8));
uint64_t quads = 0;
for (u32 quad = 0; quad < 16; ++quad) {
quads |= uint64_t((lhs >> (quad * 4) & 15u) != 0) << quad;
}
store(quads, quads != 0);
code.push_back(EncodeSop1(0x3b, 16, 8));
uint64_t replicated = 0;
for (u32 bit = 0; bit < 32; ++bit) {
replicated |= ((lhs >> bit) & 1u) * (uint64_t{3} << (2 * bit));
}
store(replicated, quads != 0);
code.push_back(EncodeSMovB32(17, InlineU32(0)));
code.push_back(EncodeSop1(0x10, 16, 8));
store(std::popcount(lhs), lhs != 0);
}
AppendEnd(&code);
test.initial.resize(test.expected.size());
test.opcodes = {
O::V_MOV_B32, O::BUFFER_LOAD_DWORD, O::V_READFIRSTLANE_B32,
O::S_MOV_B32, O::S_CMP_EQ_U32, O::S_AND_B64,
O::S_OR_B64, O::S_XOR_B64, O::S_ANDN2_B64,
O::S_ORN2_B64, O::S_NAND_B64, O::S_NOR_B64,
O::S_XNOR_B64, O::S_CSELECT_B64, O::S_BFM_B64,
O::S_BFE_U64, O::S_MOV_B64, O::S_BITSET0_B64,
O::S_BITSET1_B64, O::S_QUADMASK_B64, O::S_BITREPLICATE_B64_B32,
O::S_BCNT1_I32_B64, O::BUFFER_STORE_DWORD, O::S_ENDPGM};
test.required_spirv = {"OpTypeInt 64 0", "OpBitwiseAnd", "OpBitwiseOr",
"OpBitwiseXor"};
return test;
}
uint64_t ReferenceAshrI64(uint64_t bits, u32 count) {
const auto shift = count & 63u;
// Complementing negative values expresses sign fill using unsigned shifts.
@@ -20385,8 +20512,7 @@ TestCase ScalarAshrI64Edges(bool dynamic) {
O::BUFFER_STORE_DWORD, O::S_ENDPGM};
test.decoded_counts = {
{"S_ASHR_I64", std::size(sources) * std::size(counts)}};
test.required_spirv = {"OpShiftRightArithmetic"};
test.forbidden_spirv = {"OpTypeInt 64"};
test.required_spirv = {"OpShiftRightArithmetic", "OpTypeInt 64 0"};
return test;
}
@@ -20487,7 +20613,7 @@ TestCase ScalarAshrI64OperandsAndAliases() {
O::S_ASHR_I64, O::V_MOV_B32,
O::BUFFER_STORE_DWORD, O::S_ENDPGM};
test.decoded_counts = {{"S_ASHR_I64", instructions}};
test.forbidden_spirv = {"OpTypeInt 64"};
test.required_spirv.push_back("OpTypeInt 64 0");
return test;
}
@@ -20549,7 +20675,6 @@ TestCase ScalarAshrI64Masks(u32 wave_size) {
O::S_ENDPGM};
test.decoded_counts = {{"S_ASHR_I64 vcc_lo, vcc_lo", std::size(cases)},
{"S_ASHR_I64 exec_lo, exec_lo", std::size(cases)}};
test.forbidden_spirv = {"OpTypeInt 64"};
test.compute_info.threads_num[0] = wave_size;
test.compute_info.threads_num[1] = test.compute_info.threads_num[2] = 1;
test.compute_info.thread_ids_num = 1;
@@ -21871,8 +21996,7 @@ TestCase VectorVop3LshrrevB64Captured() {
test.expected = {0x789abcdeu, 0x00123456u};
test.opcodes = {O::S_MOV_B32, O::V_MOV_B32, O::V_LSHRREV_B64,
O::BUFFER_STORE_DWORD, O::S_ENDPGM};
test.required_spirv = {"OpShiftRightLogical", "OpSelect"};
test.forbidden_spirv = {"OpTypeInt 64"};
test.required_spirv = {"OpShiftRightLogical"};
return test;
}
@@ -21899,8 +22023,7 @@ TestCase VectorVop3LshlrevB64Captured() {
test.expected = {33u, 0u, 2u, 1u, 0u};
test.opcodes = {O::BUFFER_LOAD_DWORD, O::V_LSHLREV_B64, O::V_ADD_NC_U32,
O::BUFFER_STORE_DWORD, O::S_ENDPGM};
test.required_spirv = {"OpShiftLeftLogical", "OpSelect"};
test.forbidden_spirv = {"OpTypeInt 64"};
test.required_spirv = {"OpShiftLeftLogical", "OpTypeInt 64 0"};
return test;
}
@@ -21908,7 +22031,7 @@ TestCase VectorDynamicU64ShiftEdges() {
using O = ShaderOpcode;
TestCase test;
test.name = "VectorDynamicU64ShiftEdges";
test.initial = {0u, 1u, 31u, 32u, 33u, 63u, 64u, 65u};
test.initial = {0u, 1u, 31u, 32u, 33u, 63u, 64u, 65u, 127u, UINT32_MAX};
test.expected = test.initial;
auto& code = test.code;
for (const uint64_t source : {uint64_t{0}, uint64_t{1},
@@ -21937,8 +22060,8 @@ TestCase VectorDynamicU64ShiftEdges() {
test.initial.resize(test.expected.size());
test.opcodes = {O::V_MOV_B32, O::BUFFER_LOAD_DWORD, O::V_LSHLREV_B64,
O::V_LSHRREV_B64, O::BUFFER_STORE_DWORD, O::S_ENDPGM};
test.required_spirv = {"OpShiftLeftLogical", "OpShiftRightLogical", "OpSelect"};
test.forbidden_spirv = {"OpTypeInt 64"};
test.required_spirv = {"OpShiftLeftLogical", "OpShiftRightLogical",
"OpTypeInt 64 0"};
return test;
}
@@ -22471,23 +22594,53 @@ TestCase VectorVop3BSubCoU32UsesRdna2Opcode310() {
TestCase VectorMadU64U32UnsignedCarryOut() {
using O = ShaderOpcode;
std::vector<u32> code;
AppendVMovLiteral(&code, 1, 0xffffffffu);
AppendVMovU32(&code, 2, 2);
AppendVMovU32(&code, 4, 2);
AppendVMovLiteral(&code, 5, 0xffffffffu);
AppendVop3B(&code, 0x176u, 10, 20, Vgpr(1), Vgpr(2), Vgpr(4));
AppendStoreVgpr(&code, 10, 0);
AppendStoreVgpr(&code, 11, 1);
AppendStoreSgprPair(&code, 20, 2);
struct Input {
u32 lhs;
u32 rhs;
uint64_t addend;
};
constexpr Input inputs[] = {
{0, UINT32_MAX, UINT64_MAX},
{UINT32_MAX, 2, 0xffffffff00000002ull},
{UINT32_MAX, UINT32_MAX, UINT64_MAX},
{0x80000000u, 0x80000000u, 0x4000000000000000ull},
{3, 5, 7},
};
TestCase test;
test.name = "VectorMadU64U32UnsignedCarryOut";
for (const auto &input : inputs) {
test.initial.insert(
test.initial.end(),
{input.lhs, input.rhs, u32(input.addend), u32(input.addend >> 32)});
}
test.expected = test.initial;
auto &code = test.code;
for (u32 i = 0; i < std::size(inputs); ++i) {
for (u32 word = 0; word < 4; ++word) {
AppendVMovU32(&code, 30, (i * 4 + word) * sizeof(u32));
AppendBufferLoadDword(&code, word < 2 ? word + 1 : word + 2, 30);
}
AppendVop3B(&code, 0x176u, 10, 20, Vgpr(1), Vgpr(2), Vgpr(4));
const auto output = static_cast<u32>(test.expected.size());
AppendStoreVgpr(&code, 10, output);
AppendStoreVgpr(&code, 11, output + 1);
AppendStoreSgprPair(&code, 20, output + 2);
const auto product = uint64_t(inputs[i].lhs) * inputs[i].rhs;
const auto sum = product + inputs[i].addend;
test.expected.insert(test.expected.end(),
{u32(sum), u32(sum >> 32), u32(sum < product), 0});
}
AppendEnd(&code);
return {"VectorMadU64U32UnsignedCarryOut",
code,
{},
{0x00000000u, 0x00000001u, 0x00000001u, 0x00000000u},
{O::V_MOV_B32, O::V_MAD_U64_U32, O::BUFFER_STORE_DWORD, O::S_ENDPGM}};
test.initial.resize(test.expected.size());
test.opcodes = {O::V_MOV_B32, O::BUFFER_LOAD_DWORD, O::V_MAD_U64_U32,
O::BUFFER_STORE_DWORD, O::S_ENDPGM};
// NVIDIA rejected the native widening multiply in a large Astro's Playroom
// vertex shader. Keep its 32x32 product portable and its 64-bit addition
// native.
test.required_spirv = {"OpTypeInt 64 0", "OpUMulExtended", "OpIMul %uint",
"OpIAdd %ulong", "OpULessThan"};
test.forbidden_spirv = {"OpIMul %ulong", "OpIMul %long"};
return test;
}
TestCase VectorLaneAndPackedOps() {
@@ -34689,6 +34842,7 @@ std::vector<TestCase> MakeCases() {
AddCase(ScalarBfeI32CapturedRawSignExtends);
AddCase(BitfieldExtractWidthPastEndEdges);
AddCase(Scalar64BitOps);
AddCase(ScalarDynamic64BitOps);
cases.push_back(ScalarAshrI64Edges(false));
cases.push_back(ScalarAshrI64Edges(true));
AddCase(ScalarAshrI64OperandsAndAliases);
@@ -40494,6 +40648,18 @@ int main(int argc, char **argv) {
vulkan.CheckGpuSuspendPoint();
return 0;
}
if (argc == 2 && std::strcmp(argv[1], "--native64-only") == 0) {
VulkanHarness vulkan;
RunCase(&vulkan, ScalarDynamic64BitOps());
RunCase(&vulkan, VectorDynamicU64ShiftEdges());
RunCase(&vulkan, ScalarAshrI64Edges(false));
RunCase(&vulkan, ScalarAshrI64Edges(true));
RunCase(&vulkan, VectorCompareInteger64Edges());
RunCase(&vulkan, VectorMadU64U32UnsignedCarryOut());
RunCase(&vulkan, ScalarWqmB64SelectsSccDomain());
RunCase(&vulkan, ScalarWqmB64PreservesPartialMasks());
return 0;
}
if (argc == 2 && std::strcmp(argv[1], "--alignbyte-only") == 0) {
VulkanHarness vulkan;
RunCase(&vulkan, VectorAlignByteUsesTwoBitByteOffset());
tests/shaderCfgTests.cpp
+88 -19
View File
@@ -7942,7 +7942,7 @@ void TestNewShaderRecompilerStructuredU64Phi() {
const auto after = MeasureSpirv(spirv);
Check(after.phis == before.phis + 1u &&
after.function_variables == before.function_variables,
"structured U64 Phi was not emitted as a native vector OpPhi");
"structured U64 Phi was not emitted as a native scalar OpPhi");
}
void TestNewShaderRecompilerCfgTerminalExitMergePS() {
@@ -9866,7 +9866,7 @@ void TestNewShaderRecompilerPlanningOnlyLoads() {
"planning-only loads or references emitted GPU operands/instructions");
}
void TestNewShaderRecompilerU64PairTranslation() {
void TestNewShaderRecompilerNativeU64Translation() {
using namespace ShaderRecompiler;
auto options = MakeCompileOptions(ShaderType::Compute);
@@ -9996,20 +9996,23 @@ void TestNewShaderRecompilerU64PairTranslation() {
auto spirv = ShaderRecompiler::Spirv::EmitProgram(program, options.input_info);
CheckSpirvBinaryValidates(spirv);
const auto source = DisassembleSpirvBinary(spirv);
Check((source.find("OpCapability Int64") == std::string::npos) &&
(source.find("OpTypeInt 64") == std::string::npos),
"portable pair-U64 translation introduced native shader Int64");
Check(SpirvInstructionOpcodeCount(spirv, 149u) == 1u,
"pair-U64 addition did not use exactly one carry instruction");
Check(SpirvInstructionOpcodeCount(spirv, 154u) == 1u &&
SpirvInstructionOpcodeCount(spirv, 155u) == 1u,
"pair-U64 equality did not reduce its vector comparison with Any/All");
Check(source.find("OpCapability Int64") != std::string::npos &&
source.find("OpTypeInt 64 0") != std::string::npos &&
SpirvSourceHasInstructionUsing(source, "OpIAdd", "%ulong") &&
SpirvSourceHasInstructionUsing(source, "OpIMul", "%ulong") &&
SpirvSourceHasInstructionUsing(source, "OpBitwiseAnd", "%ulong"),
"U64 arithmetic did not emit native 64-bit scalar instructions");
Check(SpirvInstructionOpcodeCount(spirv, spv::OpIAddCarry) == 0u &&
SpirvInstructionOpcodeCount(spirv, spv::OpUMulExtended) == 0u &&
SpirvInstructionOpcodeCount(spirv, spv::OpAny) == 0u &&
SpirvInstructionOpcodeCount(spirv, spv::OpAll) == 0u,
"native U64 arithmetic retained split-word operations");
const auto direct_metrics = MeasureSpirv(spirv);
Check(
direct_metrics.words <= 940u && direct_metrics.instructions <= 199u &&
direct_metrics.type_vectors == 2u && direct_metrics.phis == 0u &&
direct_metrics.function_variables == 0u,
"portable pair-U64 translation exceeded its declaration/code-size ratchet");
// The same fixture required 864 words and 182 instructions with split U64s.
Check(direct_metrics.words <= 285u && direct_metrics.instructions <= 67u &&
direct_metrics.phis == 0u &&
direct_metrics.function_variables == 0u,
"native U64 translation exceeded its declaration/code-size ratchet");
const uint32_t dispatcher_shader[] = {
EncodeSopp(0x05, 2), EncodeSopp(0x02, 0), EncodeSopp(0x05, 0xfffeu),
@@ -10034,7 +10037,39 @@ void TestNewShaderRecompilerU64PairTranslation() {
Check(after.function_variables == before.function_variables + 1u &&
after.loads == before.loads + 1u &&
after.stores == before.stores + 1u,
"dispatcher did not use one canonical U64 vector spill slot");
"dispatcher did not use one canonical U64 scalar spill slot");
}
void TestVertexMadU64UsesPortableProduct() {
using StageInputKind = ShaderRecompiler::IR::StageInputKind;
for (const bool dynamic_addend : {false, true}) {
const uint32_t shader[] = {
EncodeVop1(0x01, 6,
8 + 256), // Dynamic addend is [VertexIndex, InstanceIndex].
EncodeVop3Word0Sdst(0x176, 0, 20),
EncodeVop3Word1(5 + 256, 8 + 256, dynamic_addend ? 5 + 256 : 128),
EncodeExp0(0x0c, 0x3),
EncodeExp1(0, 1, 0, 0), // POS0.xy keeps both product words live.
EncodeSopp(0x01),
};
const auto result =
RecompileForTest(shader, MakeCompileOptions(ShaderType::Vertex));
Check(ProgramHasInput(result.program, StageInputKind::VertexIndex) &&
ProgramHasInput(result.program, StageInputKind::InstanceIndex),
"vertex MAD fixture lost its runtime multiplication operands");
CheckSpirvBinaryValidates(result.spirv);
const auto source = DisassembleSpirvBinary(result.spirv);
// Large vertex shaders on NVIDIA rejected native widening multiplication.
Check(SpirvInstructionOpcodeCount(result.spirv, spv::OpUMulExtended) ==
1u &&
SpirvSourceHasInstructionUsing(source, "OpIMul", "%uint") &&
!SpirvSourceHasInstructionUsing(source, "OpIMul", "%ulong"),
"vertex MAD lost its portable 32x32 product");
Check(!dynamic_addend ||
SpirvSourceHasInstructionUsing(source, "OpIAdd", "%ulong"),
"vertex MAD split its live 64-bit addition into word operations");
}
}
void TestComputeDispatchWaveSize() {
@@ -12131,7 +12166,8 @@ void TestU64ShiftConstantPropagation() {
std::vector<ShiftCase> shift_cases;
for (const uint64_t source :
{uint64_t{0x0123456789abcdefull}, uint64_t{0xf123456789abcdefull}}) {
for (const uint32_t count : {0u, 1u, 31u, 32u, 33u, 63u, 64u, 65u}) {
for (const uint32_t count :
{0u, 1u, 31u, 32u, 33u, 63u, 64u, 65u, 127u, UINT32_MAX}) {
const uint32_t amount = count & 63u;
shift_cases.push_back({shift_ir.Emit(ValueOpcode::ShiftLeftLogical64,
{Value(source), Value(count)}),
@@ -12150,10 +12186,41 @@ void TestU64ShiftConstantPropagation() {
const auto value = test.value.Resolve();
Check(value.IsImmediate() && value.GetType() == Type::U64 &&
value.U64() == test.expected,
"pair-U64 shift propagation violated the masked RDNA2 count");
"U64 shift propagation violated the masked RDNA2 count");
}
}
void TestU64RegisterPairRoundTrip() {
using namespace ShaderRecompiler::IR;
Program program;
program.block_storage.push_back(std::make_unique<Block>());
program.blocks.push_back(program.block_storage.back().get());
IREmitter ir(program.blocks.front());
const auto lane = ir.Emit(ValueOpcode::LaneId);
const auto pair = ir.Emit(ValueOpcode::CompositeConstructU64, {lane, lane});
const auto first = ir.Emit(ValueOpcode::IAdd64, {pair, Value(uint64_t{1})});
const auto second = ir.Emit(ValueOpcode::IAdd64, {pair, Value(uint64_t{2})});
const auto low =
ir.Emit(ValueOpcode::CompositeExtractU64, {first, Value(0u)});
const auto high =
ir.Emit(ValueOpcode::CompositeExtractU64, {first, Value(1u)});
const auto other_high =
ir.Emit(ValueOpcode::CompositeExtractU64, {second, Value(1u)});
const auto restored =
ir.Emit(ValueOpcode::CompositeConstructU64, {low, high});
const auto swapped = ir.Emit(ValueOpcode::CompositeConstructU64, {high, low});
const auto mixed =
ir.Emit(ValueOpcode::CompositeConstructU64, {low, other_high});
ConstantPropagationPass(program.blocks);
Check(restored.Resolve() == first.Resolve(),
"unchanged SGPR pair did not retain its native U64 value");
Check(swapped.Resolve().Instruction()->GetOpcode() ==
ValueOpcode::CompositeConstructU64 &&
mixed.Resolve().Instruction()->GetOpcode() ==
ValueOpcode::CompositeConstructU64,
"modified SGPR pair incorrectly reused an unrelated native U64 value");
}
#if KYTY_PLATFORM != KYTY_PLATFORM_WINDOWS
void TestNativeWideValueValidation() {
using namespace ShaderRecompiler::IR;
@@ -14931,7 +14998,8 @@ int main() {
TestNewShaderRecompilerCfgIrreducibleDispatcher();
TestNewShaderRecompilerDispatcherSpillsU32x3();
TestNewShaderRecompilerPlanningOnlyLoads();
TestNewShaderRecompilerU64PairTranslation();
TestNewShaderRecompilerNativeU64Translation();
TestVertexMadU64UsesPortableProduct();
TestComputeDispatchWaveSize();
TestNewShaderRecompilerBufferLoadsGuardedByExec();
TestNewShaderRecompilerBufferAtomicsGuardedByBounds();
@@ -14956,6 +15024,7 @@ int main() {
TestValuePhiValidation();
TestWave32MaskProjection();
TestU64ShiftConstantPropagation();
TestU64RegisterPairRoundTrip();
#if KYTY_PLATFORM != KYTY_PLATFORM_WINDOWS
TestNativeWideValueValidation();
#endif