raphiaura/KytyPS5
1
0
Fork 0
mirror of https://github.com/KytyPS5/KytyPS5.git synced 2026-10-20 19:07:01 +02:00
Code Issues Packages Projects Releases Wiki Activity

shader: emit native 64-bit operations where compatible

Browse Source
This commit is contained in:
nmzik
2026-10-05 20:41:03 +04:00
parent 77d7722173
commit 5b826c6e4a
16 changed files with 500 additions and 436 deletions
Download Patch File Download Diff File Expand all files Collapse all files
src/graphics/shader/recompiler/backend/spirv/spirvEmitterAlu.cpp
+29 -215
View File
@@ -17,21 +17,8 @@ Pair ExtractPair(EmitterState& state, uint32_t value) {
return result;
}
uint32_t MakePair(EmitterState& state, uint32_t low, uint32_t high) {
const auto result = state.builder.AllocateId();
state.builder.AddFunction(spv::OpCompositeConstruct, TypeU64(state), result, low, high);
return result;
}
uint32_t CompareEqual64(EmitterState& state, uint32_t lhs_value, uint32_t rhs_value,
bool not_equal) {
const auto compare = Binary(state, not_equal ? spv::OpINotEqual : spv::OpIEqual,
TypeBoolVector(state, 2), lhs_value, rhs_value);
return Unary(state, not_equal ? spv::OpAny : spv::OpAll, TypeBool(state), compare);
}
uint32_t EmitMinMaxF64(EmitterState& state, uint32_t lhs, uint32_t rhs, bool max_value) {
const auto bits_type = TypeU64(state);
const auto bits_type = TypeU32Vector(state, 2);
const auto lhs_bits = Unary(state, spv::OpBitcast, bits_type, lhs);
const auto rhs_bits = Unary(state, spv::OpBitcast, bits_type, rhs);
const auto ordered = Binary(state, max_value ? spv::OpFOrdGreaterThan : spv::OpFOrdLessThan,
@@ -52,17 +39,6 @@ uint32_t EmitMinMaxF64(EmitterState& state, uint32_t lhs, uint32_t rhs, bool max
rhs, result);
}
uint32_t CompareOrdered64(EmitterState& state, uint32_t lhs_value, uint32_t rhs_value,
spv::Op high_compare, spv::Op low_compare) {
const auto lhs = ExtractPair(state, lhs_value);
const auto rhs = ExtractPair(state, rhs_value);
const auto high_equal = Binary(state, spv::OpIEqual, TypeBool(state), lhs.high, rhs.high);
const auto high_result = Binary(state, high_compare, TypeBool(state), lhs.high, rhs.high);
const auto low_result = Binary(state, low_compare, TypeBool(state), lhs.low, rhs.low);
const auto low_path = Binary(state, spv::OpLogicalAnd, TypeBool(state), high_equal, low_result);
return Binary(state, spv::OpLogicalOr, TypeBool(state), high_result, low_path);
}
uint32_t EmitMulHigh(EmitterState& state, uint32_t lhs, uint32_t rhs, bool signed_value) {
const auto operand_type = signed_value ? TypeI32(state) : TypeU32(state);
const auto pair_type = signed_value ? TypeI32Pair(state) : TypeU32Pair(state);
@@ -80,86 +56,6 @@ uint32_t EmitMulHigh(EmitterState& state, uint32_t lhs, uint32_t rhs, bool signe
return signed_value ? Unary(state, spv::OpBitcast, TypeU32(state), high) : high;
}
uint32_t EmitShift64(EmitterState& state, spv::Op opcode, uint32_t value, uint32_t shift) {
const auto pair = ExtractPair(state, value);
const auto amount = EmitAndConstant(state, shift, 63u);
const auto word_shift = EmitAndConstant(state, amount, 31u);
const auto at_least_32 =
Binary(state, spv::OpUGreaterThanEqual, TypeBool(state), amount, ConstantU32(state, 32));
const auto nonzero =
Binary(state, spv::OpINotEqual, TypeBool(state), amount, ConstantU32(state, 0));
const auto carry_count = EmitAndConstant(
state, Binary(state, spv::OpISub, TypeU32(state), ConstantU32(state, 32), word_shift), 31u);
if (opcode == spv::OpShiftLeftLogical) {
const auto low = Binary(state, opcode, TypeU32(state), pair.low, word_shift);
const auto carry =
Select(state, TypeU32(state), nonzero,
Binary(state, spv::OpShiftRightLogical, TypeU32(state), pair.low, carry_count),
ConstantU32(state, 0));
const auto high =
Binary(state, spv::OpBitwiseOr, TypeU32(state),
Binary(state, opcode, TypeU32(state), pair.high, word_shift), carry);
return MakePair(state,
Select(state, TypeU32(state), at_least_32, ConstantU32(state, 0), low),
Select(state, TypeU32(state), at_least_32, low, high));
}
const auto high = Binary(state, opcode, TypeU32(state), pair.high, word_shift);
const auto carry =
Select(state, TypeU32(state), nonzero,
Binary(state, spv::OpShiftLeftLogical, TypeU32(state), pair.high, carry_count),
ConstantU32(state, 0));
const auto low = Binary(
state, spv::OpBitwiseOr, TypeU32(state),
Binary(state, spv::OpShiftRightLogical, TypeU32(state), pair.low, word_shift), carry);
const auto fill = opcode == spv::OpShiftRightArithmetic
? Binary(state, opcode, TypeU32(state), pair.high, ConstantU32(state, 31))
: ConstantU32(state, 0);
return MakePair(state, Select(state, TypeU32(state), at_least_32, high, low),
Select(state, TypeU32(state), at_least_32, fill, high));
}
uint32_t EmitConstantShift64(EmitterState& state, spv::Op opcode, uint32_t value, uint32_t shift) {
shift &= 63u;
if (shift == 0u) {
return value;
}
const auto pair = ExtractPair(state, value);
if (opcode == spv::OpShiftLeftLogical) {
if (shift < 32u) {
const auto low = Binary(state, spv::OpShiftLeftLogical, TypeU32(state), pair.low,
ConstantU32(state, shift));
const auto high = Binary(state, spv::OpBitwiseOr, TypeU32(state),
Binary(state, spv::OpShiftLeftLogical, TypeU32(state),
pair.high, ConstantU32(state, shift)),
Binary(state, spv::OpShiftRightLogical, TypeU32(state),
pair.low, ConstantU32(state, 32u - shift)));
return MakePair(state, low, high);
}
return MakePair(state, ConstantU32(state, 0),
shift == 32u ? pair.low
: Binary(state, spv::OpShiftLeftLogical, TypeU32(state),
pair.low, ConstantU32(state, shift - 32u)));
}
if (shift < 32u) {
const auto low = Binary(state, spv::OpBitwiseOr, TypeU32(state),
Binary(state, spv::OpShiftRightLogical, TypeU32(state), pair.low,
ConstantU32(state, shift)),
Binary(state, spv::OpShiftLeftLogical, TypeU32(state), pair.high,
ConstantU32(state, 32u - shift)));
const auto high =
Binary(state, opcode, TypeU32(state), pair.high, ConstantU32(state, shift));
return MakePair(state, low, high);
}
const auto high = opcode == spv::OpShiftRightArithmetic
? Binary(state, spv::OpShiftRightArithmetic, TypeU32(state), pair.high,
ConstantU32(state, 31u))
: ConstantU32(state, 0);
const auto low = shift == 32u ? pair.high
: Binary(state, opcode, TypeU32(state), pair.high,
ConstantU32(state, shift - 32u));
return MakePair(state, low, high);
}
uint32_t EmitMinMax3(EmitterState& state, uint32_t a, uint32_t b, uint32_t c, bool signed_value,
bool max_value) {
const auto ab = signed_value ? EmitMinMaxI32Value(state, a, b, max_value)
@@ -259,57 +155,15 @@ uint32_t EmitFPMedTri32(EmitterState& state, uint32_t a, uint32_t b, uint32_t c)
}
uint32_t EmitFindUMsb64(EmitterState& state, uint32_t value) {
const auto pair = ExtractPair(state, value);
const auto high_i = state.builder.AllocateId();
const auto low_i = state.builder.AllocateId();
state.builder.AddFunction(spv::OpExtInst, TypeI32(state), high_i, GlslStd450(state),
GLSLstd450FindUMsb, pair.high);
state.builder.AddFunction(spv::OpExtInst, TypeI32(state), low_i, GlslStd450(state),
GLSLstd450FindUMsb, pair.low);
const auto high = Unary(state, spv::OpBitcast, TypeU32(state), high_i);
const auto low = Unary(state, spv::OpBitcast, TypeU32(state), low_i);
const auto high_nonzero =
Binary(state, spv::OpINotEqual, TypeBool(state), pair.high, ConstantU32(state, 0));
return Select(state, TypeU32(state), high_nonzero,
Binary(state, spv::OpIAdd, TypeU32(state), high, ConstantU32(state, 32)), low);
}
uint32_t EmitIMul64(EmitterState& state, uint32_t lhs_value, uint32_t rhs_value) {
const auto lhs = ExtractPair(state, lhs_value);
const auto rhs = ExtractPair(state, rhs_value);
const auto low = Binary(state, spv::OpIMul, TypeU32(state), lhs.low, rhs.low);
const auto high0 = EmitMulHigh(state, lhs.low, rhs.low, false);
const auto high1 = Binary(state, spv::OpIMul, TypeU32(state), lhs.low, rhs.high);
const auto high2 = Binary(state, spv::OpIMul, TypeU32(state), lhs.high, rhs.low);
return MakePair(state, low,
Binary(state, spv::OpIAdd, TypeU32(state),
Binary(state, spv::OpIAdd, TypeU32(state), high0, high1), high2));
}
uint32_t EmitISub64(EmitterState& state, uint32_t lhs_value, uint32_t rhs_value) {
const auto lhs = ExtractPair(state, lhs_value);
const auto rhs = ExtractPair(state, rhs_value);
const auto low = Binary(state, spv::OpISub, TypeU32(state), lhs.low, rhs.low);
const auto borrow = Binary(state, spv::OpULessThan, TypeBool(state), lhs.low, rhs.low);
const auto borrow_u32 =
Select(state, TypeU32(state), borrow, ConstantU32(state, 1), ConstantU32(state, 0));
const auto high0 = Binary(state, spv::OpISub, TypeU32(state), lhs.high, rhs.high);
const auto high = Binary(state, spv::OpISub, TypeU32(state), high0, borrow_u32);
return MakePair(state, low, high);
}
uint32_t EmitIAdd64(EmitterState& state, uint32_t lhs_value, uint32_t rhs_value) {
const auto lhs = ExtractPair(state, lhs_value);
const auto rhs = ExtractPair(state, rhs_value);
const auto low_pair = state.builder.AllocateId();
const auto low = state.builder.AllocateId();
const auto carry = state.builder.AllocateId();
state.builder.AddFunction(spv::OpIAddCarry, TypeU32Pair(state), low_pair, lhs.low, rhs.low);
state.builder.AddFunction(spv::OpCompositeExtract, TypeU32(state), low, low_pair, 0);
state.builder.AddFunction(spv::OpCompositeExtract, TypeU32(state), carry, low_pair, 1);
const auto high0 = Binary(state, spv::OpIAdd, TypeU32(state), lhs.high, rhs.high);
const auto high = Binary(state, spv::OpIAdd, TypeU32(state), high0, carry);
return MakePair(state, low, high);
// GLSL FindUMsb is restricted to 32-bit components.
const auto unpacked = Unary(state, spv::OpBitcast, TypeU32Vector(state, 2), value);
const auto pair =
ExtractPair(state, EmitExt(state, TypeU32Vector(state, 2), GLSLstd450FindUMsb, {unpacked}));
const auto high_found =
Binary(state, spv::OpINotEqual, TypeBool(state), pair.high, ConstantU32(state, UINT32_MAX));
return Select(state, TypeU32(state), high_found,
Binary(state, spv::OpIAdd, TypeU32(state), pair.high, ConstantU32(state, 32)),
pair.low);
}
uint32_t EmitConvertU16U32(EmitterState& state, uint32_t arg0) {
@@ -338,7 +192,7 @@ uint32_t EmitConvertU32F32(EmitterState& state, uint32_t arg0) {
uint32_t EmitConvertF32F64(EmitterState& state, uint32_t arg0) {
const auto converted = Unary(state, spv::OpFConvert, TypeF32(state), arg0);
const auto source = EmitNative<spv::OpCompositeExtract, IR::Type::U32>(
state, Unary(state, spv::OpBitcast, TypeU64(state), arg0), 1u);
state, Unary(state, spv::OpBitcast, TypeU32Vector(state, 2), arg0), 1u);
const auto exponent = EmitAndConstant(state, source, 0x7ff00000u);
const auto overflow =
Binary(state, spv::OpLogicalAnd, TypeBool(state),
@@ -356,7 +210,21 @@ uint32_t EmitConvertF64F32(EmitterState& state, uint32_t arg0) {
EmitFlushF32DenormToSignedZero(state, arg0));
}
uint32_t EmitCompositeConstructU64(ValueEmitContext& ctx, uint32_t arg0, IR::Value arg1) {
arg1 = arg1.Resolve();
return arg1.IsImmediate() && arg1.U32() == 0u
? Unary(ctx.state, spv::OpUConvert, TypeU64(ctx.state), arg0)
: PackU64(ctx.state, arg0, ctx.Def(arg1));
}
uint32_t EmitCompositeExtractU64(EmitterState& state, uint32_t arg0, IR::Value arg1) {
const auto value = arg1.U32() == 0u ? arg0
: Binary(state, spv::OpShiftRightLogical, TypeU64(state),
arg0, ConstantU32(state, 32));
return Unary(state, spv::OpUConvert, TypeU32(state), value);
}
uint32_t EmitCompositeExtractU32x2(EmitterState& state, uint32_t arg0, IR::Value arg1) {
return EmitNative<spv::OpCompositeExtract, IR::Type::U32>(state, arg0, arg1.U32());
}
@@ -388,33 +256,11 @@ uint32_t EmitIAbs32(EmitterState& state, uint32_t arg0) {
return Select(state, TypeU32(state), negative, neg, value);
}
uint32_t EmitShiftLeftLogical64(ValueEmitContext& ctx, uint32_t arg0, IR::Value arg1) {
auto& state = ctx.state;
return arg1.Resolve().IsImmediate()
? EmitConstantShift64(state, spv::OpShiftLeftLogical, arg0, arg1.Resolve().U32())
: EmitShift64(state, spv::OpShiftLeftLogical, arg0, ctx.Def(arg1));
}
uint32_t EmitShiftRightLogical64(ValueEmitContext& ctx, uint32_t arg0, IR::Value arg1) {
auto& state = ctx.state;
return arg1.Resolve().IsImmediate()
? EmitConstantShift64(state, spv::OpShiftRightLogical, arg0, arg1.Resolve().U32())
: EmitShift64(state, spv::OpShiftRightLogical, arg0, ctx.Def(arg1));
}
uint32_t EmitShiftRightArithmetic64(ValueEmitContext& ctx, uint32_t arg0, IR::Value arg1) {
auto& state = ctx.state;
return arg1.Resolve().IsImmediate()
? EmitConstantShift64(state, spv::OpShiftRightArithmetic, arg0, arg1.Resolve().U32())
: EmitShift64(state, spv::OpShiftRightArithmetic, arg0, ctx.Def(arg1));
}
uint32_t EmitBitwiseAnd64(EmitterState& state, uint32_t arg0, uint32_t arg1) {
return Binary(state, spv::OpBitwiseAnd, TypeU64(state), arg0, arg1);
}
uint32_t EmitBitCount64(EmitterState& state, uint32_t arg0) {
const auto pair = ExtractPair(state, Unary(state, spv::OpBitCount, TypeU64(state), arg0));
// Vulkan bit-count operands remain 32-bit, including when counting a U64.
const auto unpacked = Unary(state, spv::OpBitcast, TypeU32Vector(state, 2), arg0);
const auto pair =
ExtractPair(state, Unary(state, spv::OpBitCount, TypeU32Vector(state, 2), unpacked));
return Binary(state, spv::OpIAdd, TypeU32(state), pair.low, pair.high);
}
@@ -468,38 +314,6 @@ uint32_t EmitUMedTri32(EmitterState& state, uint32_t arg0, uint32_t arg1, uint32
return EmitMed3(state, arg0, arg1, arg2, false);
}
uint32_t EmitIEqual64(EmitterState& state, uint32_t arg0, uint32_t arg1) {
return CompareEqual64(state, arg0, arg1, false);
}
uint32_t EmitINotEqual64(EmitterState& state, uint32_t arg0, uint32_t arg1) {
return CompareEqual64(state, arg0, arg1, true);
}
uint32_t EmitULessThan64(EmitterState& state, uint32_t arg0, uint32_t arg1) {
return CompareOrdered64(state, arg0, arg1, spv::OpULessThan, spv::OpULessThan);
}
uint32_t EmitSLessThan64(EmitterState& state, uint32_t arg0, uint32_t arg1) {
return CompareOrdered64(state, arg0, arg1, spv::OpSLessThan, spv::OpULessThan);
}
uint32_t EmitUGreaterThan64(EmitterState& state, uint32_t arg0, uint32_t arg1) {
return CompareOrdered64(state, arg0, arg1, spv::OpUGreaterThan, spv::OpUGreaterThan);
}
uint32_t EmitSLessThanEqual64(EmitterState& state, uint32_t arg0, uint32_t arg1) {
return CompareOrdered64(state, arg0, arg1, spv::OpSLessThan, spv::OpULessThanEqual);
}
uint32_t EmitULessThanEqual64(EmitterState& state, uint32_t arg0, uint32_t arg1) {
return CompareOrdered64(state, arg0, arg1, spv::OpULessThan, spv::OpULessThanEqual);
}
uint32_t EmitUGreaterThanEqual64(EmitterState& state, uint32_t arg0, uint32_t arg1) {
return CompareOrdered64(state, arg0, arg1, spv::OpUGreaterThan, spv::OpUGreaterThanEqual);
}
uint32_t EmitFPIsNan32(EmitterState& state, uint32_t arg0) {
return EmitNative<spv::OpFUnordNotEqual, IR::Type::U1>(state, arg0, arg0);
}
src/graphics/shader/recompiler/backend/spirv/spirvEmitterAnalysis.cpp
+2 -2
View File
@@ -136,8 +136,8 @@ uint32_t ImageType(EmitterState& state, const IR::ImageResource& image) {
EXIT("invalid image resource class");
}
const auto& info = ImageDimensionInfoFor(image.dimension);
const auto scalar_type = image.atomic64 ? TypeScalarU64(state)
: ImageScalarType(state, image.numeric_class);
const auto scalar_type =
image.atomic64 ? TypeU64(state) : ImageScalarType(state, image.numeric_class);
return state.builder.Type(spv::OpTypeImage, scalar_type,
info.spirv_dimension, image.depth_compare ? 1u : 0u, info.arrayed,
info.multisampled, sampled, format);
src/graphics/shader/recompiler/backend/spirv/spirvEmitterBvh.cpp
+21 -20
View File
@@ -19,8 +19,8 @@ uint32_t Vector(EmitterState& s, uint32_t type, std::span<const uint32_t> compon
uint32_t NodeWord(EmitterState& s, uint32_t block, uint32_t word) {
const auto offset = Binary(s, spv::OpShiftLeftLogical, TypeU32(s), word, ConstantU32(s, 2));
const auto address = Binary(s, spv::OpIAdd, TypeScalarU64(s), block,
Unary(s, spv::OpUConvert, TypeScalarU64(s), offset));
const auto address =
Binary(s, spv::OpIAdd, TypeU64(s), block, Unary(s, spv::OpUConvert, TypeU64(s), offset));
const auto pointer = Unary(s, spv::OpConvertUToPtr, TypePhysicalU32Pointer(s), address);
const auto result = s.builder.AllocateId();
s.builder.AddFunction(spv::OpLoad, TypeU32(s), result, pointer, spv::MemoryAccessAlignedMask, 4u);
@@ -224,7 +224,7 @@ uint32_t Boxes(EmitterState& s, uint32_t first, uint32_t second, uint32_t half,
void DefineBvhIntersect(EmitterState& s) {
if (!s.requirements.bvh) return;
const auto u = TypeU32(s), f = TypeF32(s), b = TypeBool(s), wide = TypeScalarU64(s);
const auto u = TypeU32(s), f = TypeF32(s), b = TypeBool(s), wide = TypeU64(s);
const auto vec4 = TypeU32Vector(s, 4), vec3 = TypeF32Vector(s, 3);
const auto signature = s.builder.Type(spv::OpTypeFunction, vec4, vec4, wide, f, vec3, vec3, vec3);
s.bvh_intersect_function = s.builder.AllocateId();
@@ -260,22 +260,24 @@ void DefineBvhIntersect(EmitterState& s) {
const auto bary = Binary(s, spv::OpINotEqual, b, and_bits(words[3], 0x01000000), ConstantU32(s, 0));
const auto sort = Binary(s, spv::OpINotEqual, b, and_bits(words[1], 0x80000000), ConstantU32(s, 0));
const auto grow = and_bits(shr(words[1], 23), 0xff);
const auto base = Binary(s, spv::OpShiftLeftLogical, wide,
DeviceAddressFromWords(s, words[0], and_bits(words[1], 0xff)), ConstantDeviceAddress(s, 8));
const auto index = Binary(s, spv::OpShiftRightLogical, wide, node, ConstantDeviceAddress(s, 3));
const auto base = Binary(s, spv::OpShiftLeftLogical, wide,
PackU64(s, words[0], and_bits(words[1], 0xff)), ConstantU32(s, 8));
const auto index = Binary(s, spv::OpShiftRightLogical, wide, node, ConstantU32(s, 3));
const auto last_index = Binary(s, spv::OpIAdd, wide, index,
Select(s, wide, full, ConstantDeviceAddress(s, 1), ConstantDeviceAddress(s, 0)));
Select(s, wide, full, ConstantU64(s, 1), ConstantU64(s, 0)));
const auto address = Binary(s, spv::OpIAdd, wide, base,
Binary(s, spv::OpShiftLeftLogical, wide, index, ConstantDeviceAddress(s, 6)));
const auto last_address = Binary(s, spv::OpIAdd, wide, base,
Binary(s, spv::OpShiftLeftLogical, wide, last_index, ConstantDeviceAddress(s, 6)));
Binary(s, spv::OpShiftLeftLogical, wide, index, ConstantU32(s, 6)));
const auto last_address =
Binary(s, spv::OpIAdd, wide, base,
Binary(s, spv::OpShiftLeftLogical, wide, last_index, ConstantU32(s, 6)));
auto valid = Binary(s, spv::OpIEqual, b, shr(words[3], 28), ConstantU32(s, 8));
const auto require = [&](uint32_t condition) {
valid = Binary(s, spv::OpLogicalAnd, b, valid, condition);
};
require(Binary(s, spv::OpULessThanEqual, b, kind, ConstantU32(s, 5)));
require(Binary(s, spv::OpULessThanEqual, b, last_index, DeviceAddressFromWords(s, words[2], and_bits(words[3], 0x3ff))));
require(Binary(s, spv::OpULessThan, b, last_address, ConstantDeviceAddress(s, uint64_t{1} << 48)));
require(Binary(s, spv::OpULessThanEqual, b, last_index,
PackU64(s, words[2], and_bits(words[3], 0x3ff))));
require(Binary(s, spv::OpULessThan, b, last_address, ConstantU64(s, uint64_t {1} << 48)));
require(Unary(s, spv::OpLogicalNot, b, Unary(s, spv::OpIsNan, b, extent)));
for (uint32_t i = 0; i < 3; ++i) {
for (const auto value: {origin[i], direction[i], inverse[i]})
@@ -292,13 +294,13 @@ void DefineBvhIntersect(EmitterState& s) {
const auto invalid = Vector(s, vec4, invalid_words);
const auto result = EmitValueOrDefaultIfCondition(s, valid, vec4, invalid, [&] {
const auto first = GetBdaPointer(s, address);
const auto second = EmitValueOrDefaultIfCondition(s, full, wide, ConstantDeviceAddress(s, 0), [&] {
return GetBdaPointer(s, Binary(s, spv::OpIAdd, wide, address, ConstantDeviceAddress(s, 64)));
const auto second = EmitValueOrDefaultIfCondition(s, full, wide, ConstantU64(s, 0), [&] {
return GetBdaPointer(s, Binary(s, spv::OpIAdd, wide, address, ConstantU64(s, 64)));
});
const auto present = Binary(s, spv::OpLogicalAnd, b,
Binary(s, spv::OpINotEqual, b, first, ConstantDeviceAddress(s, 0)),
const auto present = Binary(
s, spv::OpLogicalAnd, b, Binary(s, spv::OpINotEqual, b, first, ConstantU64(s, 0)),
Binary(s, spv::OpLogicalOr, b, Unary(s, spv::OpLogicalNot, b, full),
Binary(s, spv::OpINotEqual, b, second, ConstantDeviceAddress(s, 0))));
Binary(s, spv::OpINotEqual, b, second, ConstantU64(s, 0))));
return EmitValueOrDefaultIfCondition(s, present, vec4, invalid, [&] {
const auto tri_label = s.builder.AllocateId(), box_label = s.builder.AllocateId();
const auto merge = s.builder.AllocateId();
@@ -336,9 +338,8 @@ uint32_t EmitBvhIntersect(ValueEmitContext& ctx, const IR::Inst& inst) {
return Vector(s, TypeF32Vector(s, 3), values);
};
const auto descriptor = ctx.Arg(inst, 0);
const auto node = node_words == 2u
? DeviceAddressFromWords(s, ctx.Arg(*ray, 0), ctx.Arg(*ray, 1))
: Unary(s, spv::OpUConvert, TypeScalarU64(s), ctx.Arg(*ray, 0));
const auto node = node_words == 2u ? PackU64(s, ctx.Arg(*ray, 0), ctx.Arg(*ray, 1))
: Unary(s, spv::OpUConvert, TypeU64(s), ctx.Arg(*ray, 0));
const auto extent = Unary(s, spv::OpBitcast, TypeF32(s), ctx.Arg(*ray, node_words));
const auto origin = vector(node_words + 1u), direction = vector(node_words + 4u),
inverse = vector(node_words + 7u);
src/graphics/shader/recompiler/backend/spirv/spirvEmitterFlow.cpp
+3 -3
View File
@@ -430,16 +430,16 @@ uint32_t EmitWqmU64(EmitterState& state, uint32_t value) {
const auto quad_bits = state.builder.AllocateId();
const auto result = state.builder.AllocateId();
state.builder.AddFunction(spv::OpShiftRightLogical, TypeU64(state), shifted_one, value,
ConstantU64(state, 0x0000000100000001ull));
ConstantU32(state, 1));
state.builder.AddFunction(spv::OpBitwiseOr, TypeU64(state), merged_one, value, shifted_one);
state.builder.AddFunction(spv::OpShiftRightLogical, TypeU64(state), shifted_two, merged_one,
ConstantU64(state, 0x0000000200000002ull));
ConstantU32(state, 2));
state.builder.AddFunction(spv::OpBitwiseOr, TypeU64(state), merged_two, merged_one,
shifted_two);
state.builder.AddFunction(spv::OpBitwiseAnd, TypeU64(state), quad_bits, merged_two,
ConstantU64(state, 0x1111111111111111ull));
state.builder.AddFunction(spv::OpIMul, TypeU64(state), result, quad_bits,
ConstantU64(state, 0x0000000f0000000full));
ConstantU64(state, 15));
return result;
}
src/graphics/shader/recompiler/backend/spirv/spirvEmitterImage.cpp
+3 -3
View File
@@ -930,9 +930,9 @@ void EmitImage(ValueEmitContext& ctx, const IR::Inst& inst) {
ctx.Define(inst, EmitValueOrDefaultIfCondition(
state, ctx.Arg(inst, inst.NumArgs() - 1), result_type, zero, [&]() {
const auto pointer = state.builder.AllocateId();
const auto pointer_type = state.builder.Type(
spv::OpTypePointer, spv::StorageClassImage,
image.atomic64 ? TypeScalarU64(state) : TypeU32(state));
const auto pointer_type =
state.builder.Type(spv::OpTypePointer, spv::StorageClassImage,
image.atomic64 ? TypeU64(state) : TypeU32(state));
state.builder.AddFunction(spv::OpImageTexelPointer, pointer_type, pointer,
ImageDescriptorPointer(state, mem.resource),
CoordU32(ctx, mem, *address, dimension),
src/graphics/shader/recompiler/backend/spirv/spirvEmitterInstructions.h
+31 -19
View File
@@ -34,16 +34,16 @@ uint32_t EmitConvertF32F64(EmitterState& state, uint32_t arg0);
uint32_t EmitConvertF64F32(EmitterState& state, uint32_t arg0);
EMIT_NATIVE(ConvertF32U32, OpConvertUToF, F32, uint32_t)
EMIT_NATIVE(ConvertF64U32, OpConvertUToF, F64, uint32_t)
EMIT_NATIVE(CompositeConstructU64, OpCompositeConstruct, U64, uint32_t, uint32_t)
uint32_t EmitCompositeConstructU64(ValueEmitContext& ctx, uint32_t arg0, IR::Value arg1);
EMIT_NATIVE(CompositeConstructU32x2, OpCompositeConstruct, U32x2, uint32_t, uint32_t)
EMIT_NATIVE(CompositeConstructU32x3, OpCompositeConstruct, U32x3, uint32_t, uint32_t, uint32_t)
EMIT_NATIVE(CompositeConstructF32x2, OpCompositeConstruct, F32x2, uint32_t, uint32_t)
EMIT_NATIVE(CompositeConstructU32x4, OpCompositeConstruct, U32x4, uint32_t, uint32_t, uint32_t,
uint32_t)
uint32_t EmitCompositeExtractU64(EmitterState& state, uint32_t arg0, IR::Value arg1);
inline constexpr auto EmitCompositeExtractU32x2 = EmitCompositeExtractU64;
inline constexpr auto EmitCompositeExtractU32x3 = EmitCompositeExtractU64;
inline constexpr auto EmitCompositeExtractU32x4 = EmitCompositeExtractU64;
uint32_t EmitCompositeExtractU32x2(EmitterState& state, uint32_t arg0, IR::Value arg1);
inline constexpr auto EmitCompositeExtractU32x3 = EmitCompositeExtractU32x2;
inline constexpr auto EmitCompositeExtractU32x4 = EmitCompositeExtractU32x2;
inline constexpr auto EmitPackHalf2x16 = EmitGlsl<GLSLstd450PackHalf2x16, IR::Type::U32, uint32_t>;
inline constexpr auto EmitPackSnorm2x16 =
EmitGlsl<GLSLstd450PackSnorm2x16, IR::Type::U32, uint32_t>;
@@ -63,9 +63,9 @@ EMIT_NATIVE(IAdd32, OpIAdd, U32, uint32_t, uint32_t)
EMIT_NATIVE(ISub32, OpISub, U32, uint32_t, uint32_t)
EMIT_NATIVE(IMul32, OpIMul, U32, uint32_t, uint32_t)
EMIT_NATIVE(UDiv32, OpUDiv, U32, uint32_t, uint32_t)
uint32_t EmitIAdd64(EmitterState& state, uint32_t arg0, uint32_t arg1);
uint32_t EmitISub64(EmitterState& state, uint32_t arg0, uint32_t arg1);
uint32_t EmitIMul64(EmitterState& state, uint32_t arg0, uint32_t arg1);
EMIT_NATIVE(IAdd64, OpIAdd, U64, uint32_t, uint32_t)
EMIT_NATIVE(ISub64, OpISub, U64, uint32_t, uint32_t)
EMIT_NATIVE(IMul64, OpIMul, U64, uint32_t, uint32_t)
EMIT_NATIVE(IAddCarry32, OpIAddCarry, U32x2, uint32_t, uint32_t)
uint32_t EmitSMulHi(EmitterState& state, uint32_t arg0, uint32_t arg1);
uint32_t EmitUMulHi(EmitterState& state, uint32_t arg0, uint32_t arg1);
@@ -73,14 +73,26 @@ uint32_t EmitIAbs32(EmitterState& state, uint32_t arg0);
EMIT_NATIVE(ShiftLeftLogical32, OpShiftLeftLogical, U32, uint32_t, uint32_t)
EMIT_NATIVE(ShiftRightLogical32, OpShiftRightLogical, U32, uint32_t, uint32_t)
EMIT_NATIVE(ShiftRightArithmetic32, OpShiftRightArithmetic, U32, uint32_t, uint32_t)
uint32_t EmitShiftLeftLogical64(ValueEmitContext& ctx, uint32_t arg0, IR::Value arg1);
uint32_t EmitShiftRightLogical64(ValueEmitContext& ctx, uint32_t arg0, IR::Value arg1);
uint32_t EmitShiftRightArithmetic64(ValueEmitContext& ctx, uint32_t arg0, IR::Value arg1);
template <spv::Op opcode>
uint32_t EmitShift64(ValueEmitContext& ctx, uint32_t value, IR::Value shift) {
shift = shift.Resolve();
if (shift.IsImmediate()) {
const auto amount = shift.U32() & 63u;
return amount == 0u ? value
: Binary(ctx.state, opcode, TypeU64(ctx.state), value,
ConstantU32(ctx.state, amount));
}
return Binary(ctx.state, opcode, TypeU64(ctx.state), value,
EmitAndConstant(ctx.state, ctx.Def(shift), 63u));
}
inline constexpr auto EmitShiftLeftLogical64 = EmitShift64<spv::OpShiftLeftLogical>;
inline constexpr auto EmitShiftRightLogical64 = EmitShift64<spv::OpShiftRightLogical>;
inline constexpr auto EmitShiftRightArithmetic64 = EmitShift64<spv::OpShiftRightArithmetic>;
EMIT_NATIVE(BitwiseAnd32, OpBitwiseAnd, U32, uint32_t, uint32_t)
EMIT_NATIVE(BitwiseOr32, OpBitwiseOr, U32, uint32_t, uint32_t)
EMIT_NATIVE(BitwiseXor32, OpBitwiseXor, U32, uint32_t, uint32_t)
EMIT_NATIVE(BitwiseNot32, OpNot, U32, uint32_t)
uint32_t EmitBitwiseAnd64(EmitterState& state, uint32_t arg0, uint32_t arg1);
EMIT_NATIVE(BitwiseAnd64, OpBitwiseAnd, U64, uint32_t, uint32_t)
EMIT_NATIVE(BitReverse32, OpBitReverse, U32, uint32_t)
EMIT_NATIVE(BitCount32, OpBitCount, U32, uint32_t)
uint32_t EmitBitCount64(EmitterState& state, uint32_t arg0);
@@ -107,14 +119,14 @@ EMIT_NATIVE(UGreaterThan32, OpUGreaterThan, U1, uint32_t, uint32_t)
EMIT_NATIVE(INotEqual32, OpINotEqual, U1, uint32_t, uint32_t)
EMIT_NATIVE(SGreaterThanEqual32, OpSGreaterThanEqual, U1, uint32_t, uint32_t)
EMIT_NATIVE(UGreaterThanEqual32, OpUGreaterThanEqual, U1, uint32_t, uint32_t)
uint32_t EmitIEqual64(EmitterState& state, uint32_t arg0, uint32_t arg1);
uint32_t EmitINotEqual64(EmitterState& state, uint32_t arg0, uint32_t arg1);
uint32_t EmitULessThan64(EmitterState& state, uint32_t arg0, uint32_t arg1);
uint32_t EmitSLessThan64(EmitterState& state, uint32_t arg0, uint32_t arg1);
uint32_t EmitSLessThanEqual64(EmitterState& state, uint32_t arg0, uint32_t arg1);
uint32_t EmitULessThanEqual64(EmitterState& state, uint32_t arg0, uint32_t arg1);
uint32_t EmitUGreaterThanEqual64(EmitterState& state, uint32_t arg0, uint32_t arg1);
uint32_t EmitUGreaterThan64(EmitterState& state, uint32_t arg0, uint32_t arg1);
EMIT_NATIVE(IEqual64, OpIEqual, U1, uint32_t, uint32_t)
EMIT_NATIVE(INotEqual64, OpINotEqual, U1, uint32_t, uint32_t)
EMIT_NATIVE(ULessThan64, OpULessThan, U1, uint32_t, uint32_t)
EMIT_NATIVE(SLessThan64, OpSLessThan, U1, uint32_t, uint32_t)
EMIT_NATIVE(SLessThanEqual64, OpSLessThanEqual, U1, uint32_t, uint32_t)
EMIT_NATIVE(ULessThanEqual64, OpULessThanEqual, U1, uint32_t, uint32_t)
EMIT_NATIVE(UGreaterThanEqual64, OpUGreaterThanEqual, U1, uint32_t, uint32_t)
EMIT_NATIVE(UGreaterThan64, OpUGreaterThan, U1, uint32_t, uint32_t)
EMIT_NATIVE(LogicalOr, OpLogicalOr, U1, uint32_t, uint32_t)
EMIT_NATIVE(LogicalAnd, OpLogicalAnd, U1, uint32_t, uint32_t)
EMIT_NATIVE(LogicalXor, OpLogicalNotEqual, U1, uint32_t, uint32_t)
src/graphics/shader/recompiler/backend/spirv/spirvEmitterInternal.h
+7 -4
View File
@@ -98,7 +98,7 @@ struct EmitterState {
uint32_t void_type = 0;
uint32_t bool_type = 0;
uint32_t u32_type = 0;
uint32_t native_u64_type = 0;
uint32_t u64_type = 0;
uint32_t i32_type = 0;
uint32_t f32_type = 0;
uint32_t f64_type = 0;
@@ -168,7 +168,6 @@ uint32_t TypeBool(EmitterState& state);
uint32_t TypeBoolVector(EmitterState& state, uint32_t components);
uint32_t TypeU32(EmitterState& state);
uint32_t TypeU64(EmitterState& state);
uint32_t TypeScalarU64(EmitterState& state);
uint32_t TypeU32Pair(EmitterState& state);
uint32_t TypeI32(EmitterState& state);
uint32_t TypeI32Pair(EmitterState& state);
@@ -221,6 +220,12 @@ inline uint32_t Unary(EmitterState& state, spv::Op opcode, uint32_t type, uint32
return result;
}
inline uint32_t PackU64(EmitterState& state, uint32_t low, uint32_t high) {
const auto pair = state.builder.AllocateId();
state.builder.AddFunction(spv::OpCompositeConstruct, TypeU32Vector(state, 2), pair, low, high);
return Unary(state, spv::OpBitcast, TypeU64(state), pair);
}
inline uint32_t Binary(EmitterState& state, spv::Op opcode, uint32_t type, uint32_t lhs,
uint32_t rhs) {
const auto result = state.builder.AllocateId();
@@ -531,8 +536,6 @@ void EmitProgram(EmitterState& state);
void DefineGetBdaPointer(EmitterState& state);
void DefineBvhIntersect(EmitterState& state);
uint32_t GetBdaPointer(EmitterState& state, uint32_t address);
uint32_t ConstantDeviceAddress(EmitterState& state, uint64_t value);
uint32_t DeviceAddressFromWords(EmitterState& state, uint32_t low, uint32_t high);
// These templates accept local lambdas from several emitter translation units.
template <typename Fn>
src/graphics/shader/recompiler/backend/spirv/spirvEmitterMemory.cpp
+70 -86
View File
@@ -133,22 +133,22 @@ uint32_t GuestAddress(ValueEmitContext& ctx, const IR::Inst& inst, const IR::Mem
}
uint32_t address = 0;
if (mem.address_is_full) {
address = DeviceAddressFromWords(state, low, ctx.Arg(inst, 2));
address = PackU64(state, low, ctx.Arg(inst, 2));
} else {
const auto* handle = inst.Arg(0).Resolve().TryInstruction();
if (handle == nullptr || handle->GetOpcode() != IR::ValueOpcode::GetAddressResource ||
handle->NumArgs() != 2) {
ctx.Fail(inst, "has no address base pair");
return ConstantDeviceAddress(state, 0);
return ConstantU64(state, 0);
}
auto base_low = ctx.Arg(*handle, 0);
if (mem.kind == IR::ResourceKind::ScalarAddress) {
base_low = Binary(state, spv::OpBitwiseAnd, TypeU32(state), base_low,
ConstantU32(state, ~3u));
}
const auto base = DeviceAddressFromWords(state, base_low, ctx.Arg(*handle, 1));
address = Binary(state, spv::OpIAdd, TypeScalarU64(state), base,
Unary(state, spv::OpUConvert, TypeScalarU64(state), low));
const auto base = PackU64(state, base_low, ctx.Arg(*handle, 1));
address = Binary(state, spv::OpIAdd, TypeU64(state), base,
Unary(state, spv::OpUConvert, TypeU64(state), low));
}
auto immediate = static_cast<int32_t>(mem.offset);
if (mem.kind == IR::ResourceKind::ScalarAddress) {
@@ -156,9 +156,8 @@ uint32_t GuestAddress(ValueEmitContext& ctx, const IR::Inst& inst, const IR::Mem
}
return immediate == 0
? address
: Binary(state, spv::OpIAdd, TypeScalarU64(state), address,
ConstantDeviceAddress(
state, static_cast<uint64_t>(static_cast<int64_t>(immediate))));
: Binary(state, spv::OpIAdd, TypeU64(state), address,
ConstantU64(state, static_cast<uint64_t>(static_cast<int64_t>(immediate))));
}
uint32_t FaultElementPointer(EmitterState& state, uint32_t index) {
@@ -185,7 +184,7 @@ uint32_t LoadBdaDword(ValueEmitContext& ctx, uint32_t address) {
auto& state = ctx.state;
const auto bda = GetBdaPointer(state, address);
const auto present =
Binary(state, spv::OpINotEqual, TypeBool(state), bda, ConstantDeviceAddress(state, 0));
Binary(state, spv::OpINotEqual, TypeBool(state), bda, ConstantU64(state, 0));
return EmitValueOrZeroIfCondition(state, present, [&]() {
const auto pointer = state.builder.AllocateId();
state.builder.AddFunction(spv::OpConvertUToPtr, TypePhysicalU32Pointer(state), pointer,
@@ -201,8 +200,8 @@ uint32_t LoadBdaDword(ValueEmitContext& ctx, uint32_t address) {
uint32_t LoadBda(ValueEmitContext& ctx, uint32_t address, uint32_t active, uint32_t bits) {
auto& state = ctx.state;
return EmitValueOrZeroIfCondition(state, active, [&]() {
const auto aligned = Binary(state, spv::OpBitwiseAnd, TypeScalarU64(state), address,
ConstantDeviceAddress(state, ~uint64_t {3}));
const auto aligned = Binary(state, spv::OpBitwiseAnd, TypeU64(state), address,
ConstantU64(state, ~uint64_t {3}));
const auto first = LoadBdaDword(ctx, aligned);
const auto byte =
Binary(state, spv::OpBitwiseAnd, TypeU32(state),
@@ -212,8 +211,8 @@ uint32_t LoadBda(ValueEmitContext& ctx, uint32_t address, uint32_t active, uint3
: Binary(state, bits == 16u ? spv::OpUGreaterThan : spv::OpINotEqual,
TypeBool(state), byte, ConstantU32(state, bits == 16u ? 2u : 0u));
const auto second = EmitValueOrZeroIfCondition(state, crosses, [&]() {
return LoadBdaDword(ctx, Binary(state, spv::OpIAdd, TypeScalarU64(state), aligned,
ConstantDeviceAddress(state, sizeof(uint32_t))));
return LoadBdaDword(ctx, Binary(state, spv::OpIAdd, TypeU64(state), aligned,
ConstantU64(state, sizeof(uint32_t))));
});
const auto shift =
Binary(state, spv::OpShiftLeftLogical, TypeU32(state), byte, ConstantU32(state, 3));
@@ -805,26 +804,25 @@ uint32_t LoadIndirectScalarBuffer(ValueEmitContext& ctx, const IR::Inst& inst) {
const auto word1 = ctx.Arg(handle, 1);
const auto stride = EmitBitFieldUExtract(state, word1, ConstantU32(state, 16),
ConstantU32(state, 14));
const auto size = Binary(
state, spv::OpIMul, TypeScalarU64(state),
Unary(state, spv::OpUConvert, TypeScalarU64(state),
EmitUMax32(state, stride, ConstantU32(state, 1))),
Unary(state, spv::OpUConvert, TypeScalarU64(state), ctx.Arg(handle, 2)));
const auto offset = Binary(
state, spv::OpIAdd, TypeScalarU64(state),
Unary(state, spv::OpUConvert, TypeScalarU64(state),
Binary(state, spv::OpBitwiseAnd, TypeU32(state), ctx.Arg(inst, 1),
ConstantU32(state, ~3u))),
ConstantDeviceAddress(state, ctx.Memory(inst).offset & ~3u));
const auto end = Binary(state, spv::OpIAdd, TypeScalarU64(state), offset,
ConstantDeviceAddress(state, sizeof(uint32_t)));
const auto size = Binary(state, spv::OpIMul, TypeU64(state),
Unary(state, spv::OpUConvert, TypeU64(state),
EmitUMax32(state, stride, ConstantU32(state, 1))),
Unary(state, spv::OpUConvert, TypeU64(state), ctx.Arg(handle, 2)));
const auto offset = Binary(state, spv::OpIAdd, TypeU64(state),
Unary(state, spv::OpUConvert, TypeU64(state),
Binary(state, spv::OpBitwiseAnd, TypeU32(state),
ctx.Arg(inst, 1), ConstantU32(state, ~3u))),
ConstantU64(state, ctx.Memory(inst).offset & ~3u));
const auto end =
Binary(state, spv::OpIAdd, TypeU64(state), offset, ConstantU64(state, sizeof(uint32_t)));
return EmitValueOrZeroIfCondition(
state, Binary(state, spv::OpULessThanEqual, TypeBool(state), end, size), [&]() {
const auto base = DeviceAddressFromWords(
state, Binary(state, spv::OpBitwiseAnd, TypeU32(state), ctx.Arg(handle, 0),
ConstantU32(state, ~3u)),
Binary(state, spv::OpBitwiseAnd, TypeU32(state), word1, ConstantU32(state, 0xffffu)));
return LoadBdaDword(ctx, Binary(state, spv::OpIAdd, TypeScalarU64(state), base, offset));
const auto base = PackU64(state,
Binary(state, spv::OpBitwiseAnd, TypeU32(state),
ctx.Arg(handle, 0), ConstantU32(state, ~3u)),
Binary(state, spv::OpBitwiseAnd, TypeU32(state), word1,
ConstantU32(state, 0xffffu)));
return LoadBdaDword(ctx, Binary(state, spv::OpIAdd, TypeU64(state), base, offset));
});
}
@@ -865,21 +863,21 @@ IndirectBufferAccess PrepareIndirectBuffer(ValueEmitContext& ctx, const IR::Inst
const auto address = CalculateBufferAddress(state, index, ctx.Arg(inst, 2), soffset,
ctx.Memory(inst).offset, stride, swizzle,
field(word3, 21, 2));
const auto base = DeviceAddressFromWords(state, ctx.Arg(handle, 0), field(word1, 0, 16));
const auto base = PackU64(state, ctx.Arg(handle, 0), field(word1, 0, 16));
const auto raw_records = Binary(state, spv::OpISub, TypeU32(state), records, soffset);
return {
.address = Binary(state, spv::OpIAdd, TypeScalarU64(state), base,
Unary(state, spv::OpUConvert, TypeScalarU64(state), address.byte)),
.offset = address.offset,
.stride = stride,
.swizzle = swizzle,
.format = field(word3, 12, 7),
.selector = ctx.Memory(inst).formatted ? field(word3, 0, 3) : 0u,
.mode = field(word3, 28, 2),
.records = records,
.index_in_bounds = Binary(state, spv::OpULessThan, TypeBool(state), index, records),
.address = Binary(state, spv::OpIAdd, TypeU64(state), base,
Unary(state, spv::OpUConvert, TypeU64(state), address.byte)),
.offset = address.offset,
.stride = stride,
.swizzle = swizzle,
.format = field(word3, 12, 7),
.selector = ctx.Memory(inst).formatted ? field(word3, 0, 3) : 0u,
.mode = field(word3, 28, 2),
.records = records,
.index_in_bounds = Binary(state, spv::OpULessThan, TypeBool(state), index, records),
.scalar_in_bounds = Binary(state, spv::OpULessThanEqual, TypeBool(state), soffset, records),
.raw_records = raw_records,
.raw_records = raw_records,
.raw_index_in_bounds = Binary(state, spv::OpULessThan, TypeBool(state), index, raw_records),
};
}
@@ -925,13 +923,13 @@ uint32_t LoadIndirectBuffer(ValueEmitContext& ctx, const IR::Inst& inst, uint32_
const auto buffer = PrepareIndirectBuffer(ctx, inst);
const auto valid_format = Binary(state, spv::OpINotEqual, TypeBool(state), buffer.format,
ConstantU32(state, 0));
const auto base = Binary(state, spv::OpBitwiseAnd, TypeScalarU64(state), buffer.address,
ConstantDeviceAddress(state, ~uint64_t {3}));
const auto base = Binary(state, spv::OpBitwiseAnd, TypeU64(state), buffer.address,
ConstantU64(state, ~uint64_t {3}));
std::array<uint32_t, 4> values {};
for (uint32_t component = 0; component < components; ++component) {
const auto address = component == 0u ? base : Binary(
state, spv::OpIAdd, TypeScalarU64(state), base,
ConstantDeviceAddress(state, component * 4u));
const auto address = component == 0u ? base
: Binary(state, spv::OpIAdd, TypeU64(state), base,
ConstantU64(state, component * 4u));
values[component] = EmitValueOrZeroIfCondition(
state, AndCondition(state, valid_format,
IndirectBufferInBounds(state, buffer, component * 4u, 4u, false)),
@@ -959,13 +957,14 @@ uint32_t LoadIndirectFormattedX(ValueEmitContext& ctx, const IR::Inst& inst) {
TypeU32(state), constant, [&]() {
const auto in_bounds = IndirectBufferInBounds(state, buffer, 0u, info.byte_size, true);
const auto base = Binary(
state, spv::OpBitwiseAnd, TypeScalarU64(state), buffer.address,
ConstantDeviceAddress(state, ~(uint64_t(std::min(4u, info.byte_size)) - 1u)));
state, spv::OpBitwiseAnd, TypeU64(state), buffer.address,
ConstantU64(state, ~(uint64_t(std::min(4u, info.byte_size)) - 1u)));
const auto load = [&](uint32_t component, uint32_t bits) {
const auto offset = Format::GetFormatComponentByteOffset(info, component);
const auto address = offset == 0u ? base : Binary(
state, spv::OpIAdd, TypeScalarU64(state), base,
ConstantDeviceAddress(state, offset));
const auto address = offset == 0u
? base
: Binary(state, spv::OpIAdd, TypeU64(state),
base, ConstantU64(state, offset));
return LoadBda(ctx, address, in_bounds, bits);
};
const auto emit_component = [&](uint32_t component) {
@@ -1085,23 +1084,9 @@ void StoreWideShared(ValueEmitContext& ctx, const IR::Inst& inst, uint32_t compo
} // namespace
uint32_t ConstantDeviceAddress(EmitterState& state, uint64_t value) {
return state.builder.Constant(spv::OpConstant, TypeScalarU64(state),
static_cast<uint32_t>(value),
static_cast<uint32_t>(value >> 32u));
}
uint32_t DeviceAddressFromWords(EmitterState& state, uint32_t low, uint32_t high) {
const auto low64 = Unary(state, spv::OpUConvert, TypeScalarU64(state), low);
const auto high64 = Binary(state, spv::OpShiftLeftLogical, TypeScalarU64(state),
Unary(state, spv::OpUConvert, TypeScalarU64(state), high),
ConstantDeviceAddress(state, 32));
return Binary(state, spv::OpBitwiseOr, TypeScalarU64(state), low64, high64);
}
uint32_t GetBdaPointer(EmitterState& state, uint32_t address) {
const auto result = state.builder.AllocateId();
state.builder.AddFunction(spv::OpFunctionCall, TypeScalarU64(state), result,
state.builder.AddFunction(spv::OpFunctionCall, TypeU64(state), result,
state.bda_pointer_function, address);
return result;
}
@@ -1110,7 +1095,7 @@ void DefineGetBdaPointer(EmitterState& state) {
if (!state.program.info.uses_dma) {
return;
}
const auto type = TypeScalarU64(state);
const auto type = TypeU64(state);
const auto function_type = state.builder.Type(spv::OpTypeFunction, type, type);
state.bda_pointer_function = state.builder.AllocateId();
const auto address = state.builder.AllocateId();
@@ -1122,14 +1107,14 @@ void DefineGetBdaPointer(EmitterState& state) {
EmitLabel(state, entry_label);
const auto extended = Binary(state, spv::OpUGreaterThanEqual, TypeBool(state), address,
ConstantDeviceAddress(state, LibKernel::Memory::kExtendedMemoryBase));
const auto packed = Select(state, type, extended,
Binary(state, spv::OpISub, type, address,
ConstantDeviceAddress(state,
LibKernel::Memory::kExtendedMemoryBase - LOWER_ADDRESS_SIZE)),
address);
ConstantU64(state, LibKernel::Memory::kExtendedMemoryBase));
const auto packed = Select(
state, type, extended,
Binary(state, spv::OpISub, type, address,
ConstantU64(state, LibKernel::Memory::kExtendedMemoryBase - LOWER_ADDRESS_SIZE)),
address);
const auto page64 = Binary(state, spv::OpShiftRightLogical, type, packed,
ConstantDeviceAddress(state, BufferCache::CACHING_PAGEBITS));
ConstantU32(state, BufferCache::CACHING_PAGEBITS));
const auto page = Unary(state, spv::OpUConvert, TypeU32(state), page64);
const auto entry_pointer = state.builder.AllocateId();
state.builder.AddFunction(spv::OpAccessChain, TypeStorageBufferElementPointer(state, 64),
@@ -1137,8 +1122,7 @@ void DefineGetBdaPointer(EmitterState& state) {
page);
const auto base = state.builder.AllocateId();
state.builder.AddFunction(spv::OpLoad, type, base, entry_pointer);
const auto missing =
Binary(state, spv::OpIEqual, TypeBool(state), base, ConstantDeviceAddress(state, 0));
const auto missing = Binary(state, spv::OpIEqual, TypeBool(state), base, ConstantU64(state, 0));
const auto fault_label = state.builder.AllocateId();
const auto available_label = state.builder.AllocateId();
const auto merge_label = state.builder.AllocateId();
@@ -1151,14 +1135,14 @@ void DefineGetBdaPointer(EmitterState& state) {
EmitLabel(state, available_label);
const auto offset = Binary(state, spv::OpBitwiseAnd, type, address,
ConstantDeviceAddress(state, BufferCache::CACHING_PAGESIZE - 1));
ConstantU64(state, BufferCache::CACHING_PAGESIZE - 1));
const auto available = Binary(state, spv::OpIAdd, type, base, offset);
state.builder.AddFunction(spv::OpBranch, merge_label);
EmitLabel(state, merge_label);
const auto result = state.builder.AllocateId();
state.builder.AddFunction(spv::OpPhi, type, result, ConstantDeviceAddress(state, 0),
fault_label, available, available_label);
state.builder.AddFunction(spv::OpPhi, type, result, ConstantU64(state, 0), fault_label,
available, available_label);
state.builder.AddFunction(spv::OpReturnValue, result);
state.builder.AddFunction(spv::OpFunctionEnd);
}
@@ -1216,12 +1200,12 @@ void EmitSharedAtomic64(ValueEmitContext& ctx, const IR::Inst& inst) {
semantics = spv::MemorySemanticsUniformMemoryMask;
}
const auto pointer = state.builder.AllocateId();
state.builder.AddFunction(spv::OpAccessChain,
TypePointer(state, state.lds_storage_class, TypeScalarU64(state)),
pointer, state.lds_u64_variable, ConstantU32(state, 0), native_index);
const auto value = Unary(state, spv::OpBitcast, TypeScalarU64(state), ctx.Arg(inst, 1));
state.builder.AddFunction(
SpirvAtomicOpcode(inst.GetOpcode()), TypeScalarU64(state), state.builder.AllocateId(),
spv::OpAccessChain, TypePointer(state, state.lds_storage_class, TypeU64(state)),
pointer, state.lds_u64_variable, ConstantU32(state, 0), native_index);
const auto value = ctx.Arg(inst, 1);
state.builder.AddFunction(
SpirvAtomicOpcode(inst.GetOpcode()), TypeU64(state), state.builder.AllocateId(),
pointer, ConstantU32(state, spv::ScopeWorkgroup),
ConstantU32(state, spv::MemorySemanticsAcquireReleaseMask | semantics), value);
});
src/graphics/shader/recompiler/backend/spirv/spirvEmitterMemoryHelpers.cpp
+4 -5
View File
@@ -138,7 +138,7 @@ void EnsureLdsStorage(EmitterState& state) {
};
if (state.requirements.shared_int64_atomics) {
state.lds_variable = define(TypeU32(state), 4u);
state.lds_u64_variable = define(TypeScalarU64(state), 8u);
state.lds_u64_variable = define(TypeU64(state), 8u);
state.builder.AddName(state.lds_u64_variable, "lds_qwords");
} else {
state.lds_variable = state.builder.DefineGlobalVariable(
@@ -410,7 +410,7 @@ uint32_t EmitAtomicOperation(ValueEmitContext& ctx, const IR::Inst& inst, uint32
const auto opcode = SpirvAtomicOpcode(inst.GetOpcode());
const auto old = ctx.state.builder.AllocateId();
const bool wide = inst.GetType() == IR::Type::U64;
const auto type = wide ? TypeScalarU64(ctx.state) : TypeU32(ctx.state);
const auto type = wide ? TypeU64(ctx.state) : TypeU32(ctx.state);
if (opcode == spv::OpAtomicCompareExchange) {
const auto desired = ctx.Arg(inst, inst.NumArgs() - 3);
const auto comparator = ctx.Arg(inst, inst.NumArgs() - 2);
@@ -419,13 +419,12 @@ uint32_t EmitAtomicOperation(ValueEmitContext& ctx, const IR::Inst& inst, uint32
ConstantU32(ctx.state, scope), ConstantU32(ctx.state, spv::MemorySemanticsMaskNone),
ConstantU32(ctx.state, spv::MemorySemanticsMaskNone), desired, comparator);
} else {
auto value = ctx.Arg(inst, inst.NumArgs() - 2);
if (wide) value = Unary(ctx.state, spv::OpBitcast, type, value);
const auto value = ctx.Arg(inst, inst.NumArgs() - 2);
ctx.state.builder.AddFunction(opcode, type, old, pointer,
ConstantU32(ctx.state, scope),
ConstantU32(ctx.state, spv::MemorySemanticsMaskNone), value);
}
return wide ? Unary(ctx.state, spv::OpBitcast, TypeU64(ctx.state), old) : old;
return old;
}
void EmitAtomicMemoryBarrier(EmitterState& state, IR::ResourceKind kind) {
src/graphics/shader/recompiler/backend/spirv/spirvEmitterModule.cpp
+9 -12
View File
@@ -35,14 +35,11 @@ uint32_t TypeU32(EmitterState& state) {
}
uint32_t TypeU64(EmitterState& state) {
return TypeU32Vector(state, 2);
}
uint32_t TypeScalarU64(EmitterState& state) {
if (state.native_u64_type == 0) {
state.native_u64_type = state.builder.Type(spv::OpTypeInt, 64, 0);
if (state.u64_type == 0) {
state.builder.RequireCapability(spv::CapabilityInt64);
state.u64_type = state.builder.Type(spv::OpTypeInt, 64, 0);
}
return state.native_u64_type;
return state.u64_type;
}
uint32_t TypeU32Pair(EmitterState& state) {
@@ -124,8 +121,9 @@ uint32_t TypeFunction(EmitterState& state) {
}
uint32_t TypeStorageBufferElement(EmitterState& state, uint32_t bits) {
return bits == 32u ? TypeU32(state) : bits == 64u ? TypeScalarU64(state)
: state.builder.Type(spv::OpTypeInt, bits, 0);
return bits == 32u ? TypeU32(state)
: bits == 64u ? TypeU64(state)
: state.builder.Type(spv::OpTypeInt, bits, 0);
}
uint32_t StorageRuntimeArrayType(EmitterState& state, uint32_t bits) {
@@ -331,9 +329,8 @@ uint32_t ConstantBool(EmitterState& state, bool value) {
}
uint32_t ConstantU64(EmitterState& state, uint64_t value) {
return state.builder.Constant(spv::OpConstantComposite, TypeU64(state),
ConstantU32(state, static_cast<uint32_t>(value)),
ConstantU32(state, static_cast<uint32_t>(value >> 32u)));
return state.builder.Constant(spv::OpConstant, TypeU64(state), static_cast<uint32_t>(value),
static_cast<uint32_t>(value >> 32u));
}
uint32_t ConstantU32CompositeZero(EmitterState& state, uint32_t components) {
src/graphics/shader/recompiler/frontend/translate/Control.cpp
+1
View File
@@ -248,6 +248,7 @@ void Translator::S_CSELECT_B32(const Decoder::Instruction& inst) {
void Translator::ScalarSelect64(const Decoder::Instruction& inst,
const Decoder::Operand& false_source) {
// Preserve per-word expressions for descriptor tracking and mask provenance.
const auto condition = ir.GetScc();
const auto lhs = ReadU32Pair(inst.src0);
const auto rhs = ReadU32Pair(false_source);
src/graphics/shader/recompiler/frontend/translate/Integer.cpp
+17 -17
View File
@@ -114,6 +114,7 @@ void Translator::S_U64_MASK(const Decoder::Instruction& inst, IR::ValueOpcode lo
return;
}
// Descriptor tracking reasons about each scalar word independently.
const auto lhs = ReadU32Pair(inst.src0);
auto mask_valid = ReadMaskValid(inst.src0);
if (inst.src_count > 1u) {
@@ -269,22 +270,18 @@ void Translator::Integer24(const Decoder::Instruction& inst, bool sign, bool add
}
void Translator::V_MAD_U64_U32(const Decoder::Instruction& inst) {
const auto lhs = ReadU32(inst.src0);
const auto rhs = ReadU32(inst.src1);
const auto add = ExtractU64(ReadU64(inst.src2));
const auto mul_low = ir.IMul(lhs, rhs);
const auto mul_high = IR::U32(ir.Emit(IR::ValueOpcode::UMulHi, {lhs, rhs}));
const auto low = ir.IAdd(mul_low, add[0]);
const auto carry_low = ir.ULessThan(low, mul_low);
const auto high0 = ir.IAdd(mul_high, add[1]);
const auto carry0 = ir.ULessThan(high0, mul_high);
const auto high =
ir.IAdd(high0, ir.Select(carry_low, IR::U32(IR::Value(1u)), IR::U32(IR::Value(0u))));
const auto carry1 = ir.ULessThan(high, high0);
WriteOperand(DestinationOperand(inst), ir.ConstructU64(low, high));
const auto lhs = ReadU32(inst.src0);
const auto rhs = ReadU32(inst.src1);
// Keep the 32x32 product in words: native widening IMul64 fails NVIDIA compilation
// in large vertex shaders (Astro's Playroom). The 64-bit addition stays native.
const auto product = ir.ConstructU64(
ir.IMul(lhs, rhs), IR::U32(ir.Emit(IR::ValueOpcode::UMulHi, {lhs, rhs})));
const auto result = ir.Emit(IR::ValueOpcode::IAdd64, {product, ReadU64(inst.src2)});
WriteOperand(DestinationOperand(inst), result);
if (inst.dst2.kind != Decoder::OperandKind::Null &&
inst.dst2.kind != Decoder::OperandKind::Unknown) {
WriteMask(inst.dst2, ir.LogicalOr(carry0, carry1));
// The 32x32 product fits in 64 bits, so only the addition can carry.
WriteMask(inst.dst2, IR::U1(ir.Emit(IR::ValueOpcode::ULessThan64, {result, product})));
}
}
@@ -407,9 +404,12 @@ IR::U64 Translator::RightMask64(IR::U32 count) {
}
void Translator::S_BFM_B64(const Decoder::Instruction& inst) {
const auto count = ir.BitwiseAnd(ReadU32(inst.src0), IR::U32(IR::Value(63u)));
const auto offset = ir.BitwiseAnd(ReadU32(inst.src1), IR::U32(IR::Value(63u)));
const auto result = ir.Emit(IR::ValueOpcode::ShiftLeftLogical64, {RightMask64(count), offset});
const auto count = ReadU32(inst.src0);
const auto offset = ReadU32(inst.src1);
const auto one = IR::Value(uint64_t {1});
const auto limit = ir.Emit(IR::ValueOpcode::ShiftLeftLogical64, {one, count});
const auto mask = ir.Emit(IR::ValueOpcode::ISub64, {limit, one});
const auto result = ir.Emit(IR::ValueOpcode::ShiftLeftLogical64, {mask, offset});
WriteOperand(DestinationOperand(inst), result);
}
src/graphics/shader/recompiler/frontend/translate/Scalar.cpp
+10 -3
View File
@@ -165,9 +165,16 @@ void Translator::EmitScalar(const Decoder::Instruction& inst) {
case O::S_BCNT1_I32_B32:
return SimpleInteger(inst, IR::ValueOpcode::BitCount32, IR::Type::U32, false, false,
true);
case O::S_BCNT1_I32_B64:
return SimpleInteger(inst, IR::ValueOpcode::BitCount64, IR::Type::U64, false, false,
true);
case O::S_BCNT1_I32_B64: {
// Vulkan bit counts operate on 32-bit words; avoid packing only to split again.
const auto value = ReadU32Pair(inst.src0);
const auto low = IR::U32(ir.Emit(IR::ValueOpcode::BitCount32, {value[0]}));
const auto high = IR::U32(ir.Emit(IR::ValueOpcode::BitCount32, {value[1]}));
const auto result = ir.IAdd(low, high);
WriteOperand(inst.dst, result);
ir.SetScc(ir.INotEqual(result, IR::U32(IR::Value(0u))));
return;
}
case O::S_FF1_I32_B32:
return SimpleInteger(inst, IR::ValueOpcode::FindILsb32, IR::Type::U32, false, false,
false);
src/graphics/shader/recompiler/ir/passes/ConstantPropagation.cpp
+11
View File
@@ -347,6 +347,17 @@ void FoldInstruction(Block& block, Block::iterator instruction,
if (IsImmediate(low, Type::U32) && IsImmediate(high, Type::U32)) {
Replace(inst, Value(static_cast<uint64_t>(low.U32()) |
(static_cast<uint64_t>(high.U32()) << 32u)));
} else {
const auto* low_extract = low.TryInstruction();
const auto* high_extract = high.TryInstruction();
if (low_extract != nullptr && high_extract != nullptr &&
low_extract->GetOpcode() == ValueOpcode::CompositeExtractU64 &&
high_extract->GetOpcode() == ValueOpcode::CompositeExtractU64 &&
Arg(*low_extract, 1) == Value(0u) && Arg(*high_extract, 1) == Value(1u) &&
Arg(*low_extract, 0) == Arg(*high_extract, 0)) {
// Guest register pairs need not unpack and repack between wide operations.
Replace(inst, Arg(*low_extract, 0));
}
}
return;
}