mirror of
https://github.com/KytyPS5/KytyPS5.git
synced 2026-10-20 02:47:02 +02:00
shader: preserve raw VCC bits in S_MOV_B64 (#418)
* shader: preserve raw VCC bits in S_MOV_B64 * fix(shader): centralize scalar U64 mask provenance --------- Co-authored-by: nmzik <Nmzik@mail.ru>
This commit is contained in:
nomolao2-cell
@@ -12,6 +12,7 @@ void Translator::EmitCompareResult(const Decoder::Instruction& inst, IR::U1 valu
|
||||
if (cmpx) {
|
||||
const auto mask = BallotMask(masked);
|
||||
ir.SetExec(masked);
|
||||
ir.SetExecMaskTag(IR::U1(IR::Value(true)));
|
||||
ir.SetExecLo(mask[0]);
|
||||
ir.SetExecHi(mask[1]);
|
||||
return;
|
||||
|
||||
@@ -23,18 +23,28 @@ Decoder::Operand ConditionOperand(Decoder::OperandKind kind) {
|
||||
|
||||
void Translator::S_SAVEEXEC(const Decoder::Instruction& inst, IR::ValueOpcode operation,
|
||||
bool negate_exec, bool negate_source, bool write_64) {
|
||||
if (write_64) {
|
||||
const auto exec_operand = ConditionOperand(Decoder::OperandKind::ExecLo);
|
||||
const auto old = ReadScalarU64(exec_operand);
|
||||
const auto bit_operation = operation == IR::ValueOpcode::LogicalAnd
|
||||
? IR::ValueOpcode::BitwiseAnd32
|
||||
: IR::ValueOpcode::BitwiseOr32;
|
||||
const auto result = BinaryScalarU64(old, ReadScalarU64(inst.src0), operation, bit_operation,
|
||||
negate_exec, negate_source, false);
|
||||
WriteScalarU64(inst.dst, old);
|
||||
WriteScalarU64(exec_operand, result);
|
||||
ir.SetScc(ScalarU64NonZero(result));
|
||||
return;
|
||||
}
|
||||
const auto old = ir.GetExec();
|
||||
const auto src = ReadMask(inst.src0);
|
||||
const auto lhs = negate_exec ? ir.LogicalNot(old) : old;
|
||||
const auto rhs = negate_source ? ir.LogicalNot(src) : src;
|
||||
const auto result = IR::U1(ir.Emit(operation, {lhs, rhs}));
|
||||
if (write_64) {
|
||||
WriteMask64(inst.dst, old);
|
||||
} else {
|
||||
WriteMask(inst.dst, old);
|
||||
}
|
||||
WriteMask(inst.dst, old);
|
||||
const auto mask = BallotMask(result);
|
||||
ir.SetExec(result);
|
||||
ir.SetExecMaskTag(IR::U1(IR::Value(true)));
|
||||
ir.SetExecLo(mask[0]);
|
||||
ir.SetExecHi(mask[1]);
|
||||
ir.SetScc(result);
|
||||
@@ -196,30 +206,21 @@ void Translator::S_CSELECT_B32(const Decoder::Instruction& inst) {
|
||||
}
|
||||
|
||||
void Translator::S_CSELECT_B64(const Decoder::Instruction& inst) {
|
||||
const auto condition = ir.GetScc();
|
||||
const auto lhs = ReadU32Pair(inst.src0);
|
||||
const auto rhs = ReadU32Pair(inst.src1);
|
||||
const auto selected_mask = IR::U1(
|
||||
ir.Emit(IR::ValueOpcode::SelectU1, {condition, ReadMask(inst.src0), ReadMask(inst.src1)}));
|
||||
const auto selected_mask_valid =
|
||||
IR::U1(ir.Emit(IR::ValueOpcode::SelectU1,
|
||||
{condition, ReadMaskValid(inst.src0), ReadMaskValid(inst.src1)}));
|
||||
if (IsExecOrVcc(inst.dst)) {
|
||||
WriteMask64(inst.dst, selected_mask);
|
||||
return;
|
||||
}
|
||||
WriteU32Pair(inst.dst,
|
||||
{ir.Select(condition, lhs[0], rhs[0]), ir.Select(condition, lhs[1], rhs[1])});
|
||||
if (inst.dst.kind == Decoder::OperandKind::Sgpr) {
|
||||
const auto dst = static_cast<IR::ScalarReg>(inst.dst.reg);
|
||||
ir.SetThreadBitScalarReg(dst, selected_mask);
|
||||
ir.SetScalarMaskTag(dst, selected_mask_valid);
|
||||
}
|
||||
const auto condition = ir.GetScc();
|
||||
const auto lhs = ReadScalarU64(inst.src0);
|
||||
const auto rhs = ReadScalarU64(inst.src1);
|
||||
WriteScalarU64(inst.dst, SelectScalarU64(condition, lhs, rhs));
|
||||
}
|
||||
|
||||
void Translator::MOV_B32(const Decoder::Instruction& inst, bool apply_float_modifiers) {
|
||||
if (inst.src0.kind == Decoder::OperandKind::Sgpr && inst.dst.kind == Decoder::OperandKind::Sgpr &&
|
||||
inst.src0.reg == inst.dst.reg) {
|
||||
return;
|
||||
}
|
||||
if (IsExecOrVcc(inst.src0) && IsExecOrVcc(inst.dst)) {
|
||||
WriteMask(inst.dst, ReadMask(inst.src0));
|
||||
if (inst.src0.kind != inst.dst.kind) {
|
||||
WriteRawU32(inst.dst, ReadRawU32(inst.src0));
|
||||
}
|
||||
} else if (apply_float_modifiers && (inst.src0.negate || inst.src0.absolute)) {
|
||||
WriteOperand(DestinationOperand(inst), ReadOperand(inst.src0, IR::Type::F32));
|
||||
} else {
|
||||
@@ -228,50 +229,17 @@ void Translator::MOV_B32(const Decoder::Instruction& inst, bool apply_float_modi
|
||||
}
|
||||
|
||||
void Translator::S_MOV_B64(const Decoder::Instruction& inst) {
|
||||
if (IsExecOrVcc(inst.dst) || IsExecOrVcc(inst.src0)) {
|
||||
WriteMask64(inst.dst, ReadMask(inst.src0));
|
||||
return;
|
||||
}
|
||||
const bool scalar_copy =
|
||||
inst.dst.kind == Decoder::OperandKind::Sgpr && inst.src0.kind == Decoder::OperandKind::Sgpr;
|
||||
IR::U1 source_mask;
|
||||
IR::U1 source_mask_valid;
|
||||
if (scalar_copy) {
|
||||
source_mask = ir.GetThreadBitScalarReg(static_cast<IR::ScalarReg>(inst.src0.reg));
|
||||
source_mask_valid = ir.GetScalarMaskTag(static_cast<IR::ScalarReg>(inst.src0.reg));
|
||||
}
|
||||
WriteU32Pair(inst.dst, ReadU32Pair(inst.src0));
|
||||
if (scalar_copy) {
|
||||
ir.SetThreadBitScalarReg(static_cast<IR::ScalarReg>(inst.dst.reg), source_mask);
|
||||
ir.SetScalarMaskTag(static_cast<IR::ScalarReg>(inst.dst.reg), source_mask_valid);
|
||||
}
|
||||
WriteScalarU64(inst.dst, ReadScalarU64(inst.src0));
|
||||
}
|
||||
|
||||
void Translator::S_WQM_B64(const Decoder::Instruction& inst) {
|
||||
if (!IsExecOrVcc(inst.dst) && !IsExecOrVcc(inst.src0)) {
|
||||
const auto mask_valid = ReadMaskValid(inst.src0);
|
||||
const auto invocation_result =
|
||||
IR::U1(ir.Emit(IR::ValueOpcode::WqmMask, {ReadMask(inst.src0)}));
|
||||
const auto result =
|
||||
IR::U64(ir.Emit(IR::ValueOpcode::WqmU64, {ReadOperand(inst.src0, IR::Type::U64)}));
|
||||
WriteOperand(DestinationOperand(inst), result);
|
||||
if (inst.dst.kind == Decoder::OperandKind::Sgpr) {
|
||||
const auto dst = static_cast<IR::ScalarReg>(inst.dst.reg);
|
||||
ir.SetThreadBitScalarReg(dst, invocation_result);
|
||||
ir.SetScalarMaskTag(dst, mask_valid);
|
||||
const auto raw_nonzero =
|
||||
IR::U1(ir.Emit(IR::ValueOpcode::INotEqual64, {result, IR::Value(uint64_t {0})}));
|
||||
ir.SetScc(IR::U1(
|
||||
ir.Emit(IR::ValueOpcode::SelectU1, {mask_valid, invocation_result, raw_nonzero})));
|
||||
} else {
|
||||
ir.SetScc(
|
||||
IR::U1(ir.Emit(IR::ValueOpcode::INotEqual64, {result, IR::Value(uint64_t {0})})));
|
||||
}
|
||||
return;
|
||||
}
|
||||
const auto result = IR::U1(ir.Emit(IR::ValueOpcode::WqmMask, {ReadMask(inst.src0)}));
|
||||
WriteMask64(inst.dst, result);
|
||||
ir.SetScc(result);
|
||||
const auto source = ReadScalarU64(inst.src0);
|
||||
const auto raw = ExtractU64(
|
||||
IR::U64(ir.Emit(IR::ValueOpcode::WqmU64, {ir.ConstructU64(source.raw[0], source.raw[1])})));
|
||||
const auto invocation = IR::U1(ir.Emit(IR::ValueOpcode::WqmMask, {source.invocation}));
|
||||
const auto result = MakeScalarU64Result(raw, invocation, source.mask_valid);
|
||||
WriteScalarU64(inst.dst, result);
|
||||
ir.SetScc(ScalarU64NonZero(result));
|
||||
}
|
||||
|
||||
bool Translator::V_MOVRELS_B32(const Decoder::Instruction& inst, std::string* error) {
|
||||
|
||||
@@ -77,68 +77,15 @@ bool Translator::PackedInteger16MinMax(const Decoder::Instruction& inst, IR::Val
|
||||
return true;
|
||||
}
|
||||
|
||||
IR::U1 Translator::U64MaskBinary(const Decoder::Instruction& inst, IR::ValueOpcode opcode,
|
||||
bool negate_rhs, bool negate_result) {
|
||||
const auto lhs = ReadMask(inst.src0);
|
||||
auto rhs = ReadMask(inst.src1);
|
||||
if (negate_rhs) {
|
||||
rhs = ir.LogicalNot(rhs);
|
||||
}
|
||||
auto result = IR::U1(ir.Emit(opcode, {lhs, rhs}));
|
||||
return negate_result ? ir.LogicalNot(result) : result;
|
||||
}
|
||||
|
||||
bool Translator::S_U64_MASK(const Decoder::Instruction& inst, IR::ValueOpcode logical_opcode,
|
||||
IR::ValueOpcode bit_opcode, bool negate_rhs, bool negate_result,
|
||||
bool unary) {
|
||||
const auto invocation_result =
|
||||
unary ? ir.LogicalNot(ReadMask(inst.src0))
|
||||
: U64MaskBinary(inst, logical_opcode, negate_rhs, negate_result);
|
||||
const auto is_exec_or_vcc = [](const Decoder::Operand& operand) {
|
||||
switch (operand.kind) {
|
||||
case Decoder::OperandKind::ExecLo:
|
||||
case Decoder::OperandKind::ExecHi:
|
||||
case Decoder::OperandKind::VccLo:
|
||||
case Decoder::OperandKind::VccHi: return true;
|
||||
default: return false;
|
||||
}
|
||||
};
|
||||
if (is_exec_or_vcc(inst.dst) || is_exec_or_vcc(inst.src0) ||
|
||||
(inst.src_count > 1u && is_exec_or_vcc(inst.src1))) {
|
||||
WriteMask64(inst.dst, invocation_result);
|
||||
ir.SetScc(invocation_result);
|
||||
return true;
|
||||
}
|
||||
|
||||
const auto lhs = ReadU32Pair(inst.src0);
|
||||
auto mask_valid = ReadMaskValid(inst.src0);
|
||||
if (inst.src_count > 1u) {
|
||||
mask_valid = ir.LogicalAnd(mask_valid, ReadMaskValid(inst.src1));
|
||||
}
|
||||
std::array<IR::U32, 2> result;
|
||||
if (unary) {
|
||||
result = {ir.BitwiseNot(lhs[0]), ir.BitwiseNot(lhs[1])};
|
||||
} else {
|
||||
const auto rhs = ReadU32Pair(inst.src1);
|
||||
for (uint32_t component = 0; component < 2u; component++) {
|
||||
auto value = ir.Emit(
|
||||
bit_opcode, {lhs[component], negate_rhs ? IR::Value(ir.BitwiseNot(rhs[component]))
|
||||
: IR::Value(rhs[component])});
|
||||
result[component] = negate_result ? ir.BitwiseNot(IR::U32(value)) : IR::U32(value);
|
||||
}
|
||||
}
|
||||
WriteU32Pair(inst.dst, result);
|
||||
if (inst.dst.kind == Decoder::OperandKind::Sgpr) {
|
||||
const auto dst = static_cast<IR::ScalarReg>(inst.dst.reg);
|
||||
ir.SetThreadBitScalarReg(dst, invocation_result);
|
||||
ir.SetScalarMaskTag(dst, mask_valid);
|
||||
const auto raw_nonzero =
|
||||
ir.INotEqual(ir.BitwiseOr(result[0], result[1]), IR::U32(IR::Value(0u)));
|
||||
ir.SetScc(IR::U1(
|
||||
ir.Emit(IR::ValueOpcode::SelectU1, {mask_valid, invocation_result, raw_nonzero})));
|
||||
} else {
|
||||
ir.SetScc(ir.INotEqual(ir.BitwiseOr(result[0], result[1]), IR::U32(IR::Value(0u))));
|
||||
}
|
||||
const auto lhs = ReadScalarU64(inst.src0);
|
||||
const auto result = unary ? NotScalarU64(lhs)
|
||||
: BinaryScalarU64(lhs, ReadScalarU64(inst.src1), logical_opcode,
|
||||
bit_opcode, false, negate_rhs, negate_result);
|
||||
WriteScalarU64(inst.dst, result);
|
||||
ir.SetScc(ScalarU64NonZero(result));
|
||||
return true;
|
||||
}
|
||||
|
||||
|
||||
@@ -57,11 +57,14 @@ uint32_t ResourceIndexFromOperand(const Decoder::Operand& operand) {
|
||||
}
|
||||
}
|
||||
|
||||
uint32_t RawScalarLoadBase(const Decoder::Operand& operand) {
|
||||
if (operand.kind == Decoder::OperandKind::Sgpr) {
|
||||
return operand.reg;
|
||||
Decoder::Operand RawScalarLoadBase(const Decoder::Operand& operand) {
|
||||
if (operand.kind != Decoder::OperandKind::M0) {
|
||||
return operand;
|
||||
}
|
||||
return operand.kind == Decoder::OperandKind::VccLo ? 106u : 0u;
|
||||
auto result = operand;
|
||||
// The compressed SBASE value 62 decodes as code 124, but means NULL for S_LOAD.
|
||||
result.kind = Decoder::OperandKind::Null;
|
||||
return result;
|
||||
}
|
||||
|
||||
ResourceKind FlatSegmentResourceKind(uint32_t segment) {
|
||||
@@ -321,8 +324,9 @@ Translator::AddressOperands Translator::ReadAddressOperands(const Decoder::Instr
|
||||
return {GetAddressResource(low, high), low, high};
|
||||
}
|
||||
|
||||
IR::Value Translator::GetScalarAddressResource(uint32_t base) {
|
||||
return GetAddressResource(ReadScalarCode(base), ReadScalarCode(base + 1u));
|
||||
IR::Value Translator::GetScalarAddressResource(const Decoder::Operand& base) {
|
||||
const auto raw = ReadU32Pair(base);
|
||||
return GetAddressResource(raw[0], raw[1]);
|
||||
}
|
||||
|
||||
IR::Value Translator::GetImageResource(const IR::MemoryInfo& memory) {
|
||||
|
||||
@@ -153,11 +153,8 @@ IR::U32 Translator::ReadScalarCode(uint32_t code) {
|
||||
case 106u: return ir.GetVccLo();
|
||||
case 107u: return ir.GetVccHi();
|
||||
case 124u: return ir.GetM0();
|
||||
case 126u:
|
||||
case 127u: {
|
||||
const auto mask = BallotMask(ir.GetExec());
|
||||
return mask[code - 126u];
|
||||
}
|
||||
case 126u: return ir.GetExecLo();
|
||||
case 127u: return ir.GetExecHi();
|
||||
default: return IR::U32(IR::Value(0u));
|
||||
}
|
||||
}
|
||||
@@ -317,19 +314,23 @@ void Translator::WriteRawU32(const Decoder::Operand& operand, IR::U32 value) {
|
||||
case Decoder::OperandKind::VccLo:
|
||||
ir.SetVccLo(IR::U32(value));
|
||||
ir.SetVcc(ThreadBit(IR::U32(value)));
|
||||
ir.SetVccMaskValidTag(IR::U1(IR::Value(false)));
|
||||
break;
|
||||
case Decoder::OperandKind::VccHi:
|
||||
ir.SetVccHi(IR::U32(value));
|
||||
ir.SetVcc(ThreadBit(ir.GetVccLo()));
|
||||
ir.SetVccMaskValidTag(IR::U1(IR::Value(false)));
|
||||
break;
|
||||
case Decoder::OperandKind::M0: ir.SetM0(IR::U32(value)); break;
|
||||
case Decoder::OperandKind::ExecLo:
|
||||
ir.SetExecLo(IR::U32(value));
|
||||
ir.SetExec(ThreadBit(IR::U32(value)));
|
||||
ir.SetExecMaskTag(IR::U1(IR::Value(false)));
|
||||
break;
|
||||
case Decoder::OperandKind::ExecHi:
|
||||
ir.SetExecHi(IR::U32(value));
|
||||
ir.SetExec(ThreadBit(ir.GetExecLo()));
|
||||
ir.SetExecMaskTag(IR::U1(IR::Value(false)));
|
||||
break;
|
||||
case Decoder::OperandKind::Scc:
|
||||
ir.SetScc(ir.INotEqual(value, IR::U32(IR::Value(0u))));
|
||||
@@ -374,6 +375,7 @@ void Translator::WriteOperand(const Decoder::Operand& operand, IR::Value value)
|
||||
case Decoder::OperandKind::ExecHi: {
|
||||
const auto mask = BallotMask(IR::U1(value));
|
||||
ir.SetExec(IR::U1(value));
|
||||
ir.SetExecMaskTag(IR::U1(IR::Value(true)));
|
||||
ir.SetExecLo(mask[0]);
|
||||
ir.SetExecHi(mask[1]);
|
||||
return;
|
||||
@@ -382,6 +384,7 @@ void Translator::WriteOperand(const Decoder::Operand& operand, IR::Value value)
|
||||
case Decoder::OperandKind::VccHi: {
|
||||
const auto mask = BallotMask(IR::U1(value));
|
||||
ir.SetVcc(IR::U1(value));
|
||||
ir.SetVccMaskValidTag(IR::U1(IR::Value(true)));
|
||||
ir.SetVccLo(mask[0]);
|
||||
ir.SetVccHi(mask[1]);
|
||||
return;
|
||||
@@ -583,11 +586,13 @@ void Translator::WriteU32Pair(const Decoder::Operand& operand,
|
||||
switch (operand.kind) {
|
||||
case Decoder::OperandKind::ExecLo:
|
||||
ir.SetExec(thread_bit);
|
||||
ir.SetExecMaskTag(IR::U1(IR::Value(false)));
|
||||
ir.SetExecLo(value[0]);
|
||||
ir.SetExecHi(value[1]);
|
||||
return;
|
||||
case Decoder::OperandKind::VccLo:
|
||||
ir.SetVcc(thread_bit);
|
||||
ir.SetVccMaskValidTag(IR::U1(IR::Value(false)));
|
||||
ir.SetVccLo(value[0]);
|
||||
ir.SetVccHi(value[1]);
|
||||
return;
|
||||
@@ -598,6 +603,87 @@ void Translator::WriteU32Pair(const Decoder::Operand& operand,
|
||||
WriteRawU32(OffsetOperand(operand, 1), value[1]);
|
||||
}
|
||||
|
||||
Translator::ScalarU64 Translator::ReadScalarU64(const Decoder::Operand& operand) {
|
||||
return {ReadU32Pair(operand), ReadMask(operand), ReadMaskValid(operand)};
|
||||
}
|
||||
|
||||
Translator::ScalarU64 Translator::MakeScalarU64Mask(IR::U1 value) {
|
||||
return {BallotMask(value), value, IR::U1(IR::Value(true))};
|
||||
}
|
||||
|
||||
Translator::ScalarU64 Translator::MakeScalarU64Result(const std::array<IR::U32, 2>& raw,
|
||||
IR::U1 invocation, IR::U1 mask_valid) {
|
||||
return {raw,
|
||||
IR::U1(ir.Emit(IR::ValueOpcode::SelectU1, {mask_valid, invocation, ThreadBit(raw[0])})),
|
||||
mask_valid};
|
||||
}
|
||||
|
||||
Translator::ScalarU64 Translator::SelectScalarU64(IR::U1 condition, const ScalarU64& lhs,
|
||||
const ScalarU64& rhs) {
|
||||
return {
|
||||
{ir.Select(condition, lhs.raw[0], rhs.raw[0]),
|
||||
ir.Select(condition, lhs.raw[1], rhs.raw[1])},
|
||||
IR::U1(ir.Emit(IR::ValueOpcode::SelectU1, {condition, lhs.invocation, rhs.invocation})),
|
||||
IR::U1(ir.Emit(IR::ValueOpcode::SelectU1, {condition, lhs.mask_valid, rhs.mask_valid}))};
|
||||
}
|
||||
|
||||
Translator::ScalarU64 Translator::NotScalarU64(const ScalarU64& value) {
|
||||
return MakeScalarU64Result({ir.BitwiseNot(value.raw[0]), ir.BitwiseNot(value.raw[1])},
|
||||
ir.LogicalNot(value.invocation), value.mask_valid);
|
||||
}
|
||||
|
||||
Translator::ScalarU64 Translator::BinaryScalarU64(const ScalarU64& lhs, const ScalarU64& rhs,
|
||||
IR::ValueOpcode logical_opcode,
|
||||
IR::ValueOpcode bit_opcode, bool negate_lhs,
|
||||
bool negate_rhs, bool negate_result) {
|
||||
auto lhs_mask = negate_lhs ? ir.LogicalNot(lhs.invocation) : lhs.invocation;
|
||||
auto rhs_mask = negate_rhs ? ir.LogicalNot(rhs.invocation) : rhs.invocation;
|
||||
auto mask = IR::U1(ir.Emit(logical_opcode, {lhs_mask, rhs_mask}));
|
||||
if (negate_result) {
|
||||
mask = ir.LogicalNot(mask);
|
||||
}
|
||||
std::array<IR::U32, 2> raw;
|
||||
for (uint32_t component = 0; component < raw.size(); component++) {
|
||||
auto lhs_raw = negate_lhs ? ir.BitwiseNot(lhs.raw[component]) : lhs.raw[component];
|
||||
auto rhs_raw = negate_rhs ? ir.BitwiseNot(rhs.raw[component]) : rhs.raw[component];
|
||||
raw[component] = IR::U32(ir.Emit(bit_opcode, {lhs_raw, rhs_raw}));
|
||||
if (negate_result) {
|
||||
raw[component] = ir.BitwiseNot(raw[component]);
|
||||
}
|
||||
}
|
||||
return MakeScalarU64Result(raw, mask, ir.LogicalAnd(lhs.mask_valid, rhs.mask_valid));
|
||||
}
|
||||
|
||||
void Translator::WriteScalarU64(const Decoder::Operand& operand, const ScalarU64& value) {
|
||||
WriteU32Pair(operand, value.raw);
|
||||
switch (operand.kind) {
|
||||
case Decoder::OperandKind::ExecLo:
|
||||
case Decoder::OperandKind::ExecHi:
|
||||
ir.SetExec(value.invocation);
|
||||
ir.SetExecMaskTag(value.mask_valid);
|
||||
break;
|
||||
case Decoder::OperandKind::VccLo:
|
||||
case Decoder::OperandKind::VccHi:
|
||||
ir.SetVcc(value.invocation);
|
||||
ir.SetVccMaskValidTag(value.mask_valid);
|
||||
break;
|
||||
case Decoder::OperandKind::Sgpr: {
|
||||
const auto reg = static_cast<IR::ScalarReg>(operand.reg);
|
||||
ir.SetThreadBitScalarReg(reg, value.invocation);
|
||||
ir.SetScalarMaskTag(reg, value.mask_valid);
|
||||
break;
|
||||
}
|
||||
default: break;
|
||||
}
|
||||
}
|
||||
|
||||
IR::U1 Translator::ScalarU64NonZero(const ScalarU64& value) {
|
||||
const auto raw_nonzero =
|
||||
ir.INotEqual(ir.BitwiseOr(value.raw[0], value.raw[1]), IR::U32(IR::Value(0u)));
|
||||
return IR::U1(
|
||||
ir.Emit(IR::ValueOpcode::SelectU1, {value.mask_valid, value.invocation, raw_nonzero}));
|
||||
}
|
||||
|
||||
IR::U1 Translator::ThreadBit(IR::U32 low) {
|
||||
return ir.INotEqual(low, IR::U32(IR::Value(0u)));
|
||||
}
|
||||
@@ -653,9 +739,9 @@ IR::U1 Translator::ReadMaskValid(const Decoder::Operand& operand) {
|
||||
case Decoder::OperandKind::Sgpr:
|
||||
return ir.GetScalarMaskTag(static_cast<IR::ScalarReg>(operand.reg));
|
||||
case Decoder::OperandKind::ExecLo:
|
||||
case Decoder::OperandKind::ExecHi:
|
||||
case Decoder::OperandKind::ExecHi: return ir.GetExecMaskTag();
|
||||
case Decoder::OperandKind::VccLo:
|
||||
case Decoder::OperandKind::VccHi:
|
||||
case Decoder::OperandKind::VccHi: return ir.GetVccMaskValidTag();
|
||||
case Decoder::OperandKind::VccZ:
|
||||
case Decoder::OperandKind::ExecZ:
|
||||
case Decoder::OperandKind::Scc: return IR::U1(IR::Value(true));
|
||||
@@ -688,6 +774,7 @@ void Translator::WriteMask(const Decoder::Operand& operand, IR::U1 value) {
|
||||
case Decoder::OperandKind::ExecHi: {
|
||||
const auto mask = BallotMask(value);
|
||||
ir.SetExec(value);
|
||||
ir.SetExecMaskTag(IR::U1(IR::Value(true)));
|
||||
ir.SetExecLo(mask[0]);
|
||||
ir.SetExecHi(mask[1]);
|
||||
return;
|
||||
@@ -696,6 +783,7 @@ void Translator::WriteMask(const Decoder::Operand& operand, IR::U1 value) {
|
||||
case Decoder::OperandKind::VccHi: {
|
||||
const auto mask = BallotMask(value);
|
||||
ir.SetVcc(value);
|
||||
ir.SetVccMaskValidTag(IR::U1(IR::Value(true)));
|
||||
ir.SetVccLo(mask[0]);
|
||||
ir.SetVccHi(mask[1]);
|
||||
return;
|
||||
@@ -709,25 +797,7 @@ void Translator::WriteMask(const Decoder::Operand& operand, IR::U1 value) {
|
||||
}
|
||||
|
||||
void Translator::WriteMask64(const Decoder::Operand& operand, IR::U1 value) {
|
||||
if (operand.kind != Decoder::OperandKind::Sgpr) {
|
||||
WriteMask(operand, value);
|
||||
return;
|
||||
}
|
||||
const auto reg = static_cast<IR::ScalarReg>(operand.reg);
|
||||
const auto mask = BallotMask(value);
|
||||
ir.SetThreadBitScalarReg(reg, value);
|
||||
ir.SetScalarMaskTag(reg, IR::U1(IR::Value(true)));
|
||||
if (IR::RegIndex(reg) > 0u) {
|
||||
ir.SetScalarMaskTag(static_cast<IR::ScalarReg>(IR::RegIndex(reg) - 1u),
|
||||
IR::U1(IR::Value(false)));
|
||||
}
|
||||
ir.SetScalarReg(reg, mask[0]);
|
||||
if (IR::RegIndex(reg) + 1u < IR::NumScalarRegs) {
|
||||
const auto high = static_cast<IR::ScalarReg>(IR::RegIndex(reg) + 1u);
|
||||
ir.SetScalarReg(high, mask[1]);
|
||||
ir.SetThreadBitScalarReg(high, IR::U1(IR::Value(false)));
|
||||
ir.SetScalarMaskTag(high, IR::U1(IR::Value(false)));
|
||||
}
|
||||
WriteScalarU64(operand, MakeScalarU64Mask(value));
|
||||
}
|
||||
|
||||
void Translator::WriteCompareResult(const Decoder::Operand& operand, IR::U1 value) {
|
||||
@@ -1037,6 +1107,7 @@ bool TranslateProgram(const Decoder::Program& decoded, const CFG::Graph& cfg,
|
||||
entry_ir.SetScalarMaskTag(reg, IR::U1(IR::Value(false)));
|
||||
}
|
||||
entry_ir.SetExec(IR::U1(IR::Value(true)));
|
||||
entry_ir.SetExecMaskTag(IR::U1(IR::Value(true)));
|
||||
entry_ir.SetExecLo(IR::U32(IR::Value(1u)));
|
||||
entry_ir.SetExecHi(IR::U32(IR::Value(0u)));
|
||||
if (options.stage == ShaderType::Compute) {
|
||||
|
||||
@@ -19,6 +19,14 @@ public:
|
||||
bool AddBranchCondition(const CFG::BasicBlock& source, IR::BlockInfo& info, std::string* error);
|
||||
|
||||
private:
|
||||
struct ScalarU64 {
|
||||
std::array<IR::U32, 2> raw;
|
||||
// Raw words and Kyty's per-invocation mask model remain independent.
|
||||
// mask_valid selects invocation for logical consumers and SCC.
|
||||
IR::U1 invocation;
|
||||
IR::U1 mask_valid;
|
||||
};
|
||||
|
||||
Decoder::Operand SourceAt(const Decoder::Instruction& inst, uint32_t index);
|
||||
Decoder::Operand DestinationOperand(const Decoder::Instruction& inst);
|
||||
Decoder::Operand OffsetOperand(const Decoder::Operand& operand, uint32_t offset);
|
||||
@@ -48,14 +56,25 @@ private:
|
||||
IR::U32 ReadU16AsU32(const Decoder::Operand& operand, bool sign_extend);
|
||||
IR::U32 ReadF16LaneBits(const Decoder::Operand& operand, bool high_lane);
|
||||
std::array<IR::U32, 2> ExtractU64(IR::U64 value);
|
||||
void WriteU32Pair(const Decoder::Operand& operand, const std::array<IR::U32, 2>& value);
|
||||
IR::U1 ReadCondition(const Decoder::Operand& operand);
|
||||
IR::U32 ConditionBit(const Decoder::Operand& operand);
|
||||
IR::U1 ReadMask(const Decoder::Operand& operand);
|
||||
IR::U1 ReadMaskValid(const Decoder::Operand& operand);
|
||||
void WriteMask(const Decoder::Operand& operand, IR::U1 value);
|
||||
void WriteMask64(const Decoder::Operand& operand, IR::U1 value);
|
||||
void WriteCompareResult(const Decoder::Operand& operand, IR::U1 value);
|
||||
void WriteU32Pair(const Decoder::Operand& operand, const std::array<IR::U32, 2>& value);
|
||||
ScalarU64 ReadScalarU64(const Decoder::Operand& operand);
|
||||
ScalarU64 MakeScalarU64Mask(IR::U1 value);
|
||||
ScalarU64 MakeScalarU64Result(const std::array<IR::U32, 2>& raw, IR::U1 invocation,
|
||||
IR::U1 mask_valid);
|
||||
ScalarU64 SelectScalarU64(IR::U1 condition, const ScalarU64& lhs, const ScalarU64& rhs);
|
||||
ScalarU64 NotScalarU64(const ScalarU64& value);
|
||||
ScalarU64 BinaryScalarU64(const ScalarU64& lhs, const ScalarU64& rhs,
|
||||
IR::ValueOpcode logical_opcode, IR::ValueOpcode bit_opcode,
|
||||
bool negate_lhs, bool negate_rhs, bool negate_result);
|
||||
void WriteScalarU64(const Decoder::Operand& operand, const ScalarU64& value);
|
||||
IR::U1 ScalarU64NonZero(const ScalarU64& value);
|
||||
IR::U1 ReadCondition(const Decoder::Operand& operand);
|
||||
IR::U32 ConditionBit(const Decoder::Operand& operand);
|
||||
IR::U1 ReadMask(const Decoder::Operand& operand);
|
||||
IR::U1 ReadMaskValid(const Decoder::Operand& operand);
|
||||
void WriteMask(const Decoder::Operand& operand, IR::U1 value);
|
||||
void WriteMask64(const Decoder::Operand& operand, IR::U1 value);
|
||||
void WriteCompareResult(const Decoder::Operand& operand, IR::U1 value);
|
||||
|
||||
IR::MemoryFlags AddMemoryInfo(const IR::MemoryInfo& memory, uint32_t pc);
|
||||
IR::ExportFlags AddExportInfo(const Decoder::Instruction& inst);
|
||||
@@ -73,7 +92,7 @@ private:
|
||||
IR::U32 GetResourceDword(uint32_t index, uint32_t dword);
|
||||
IR::Value GetBufferResource(const IR::MemoryInfo& memory);
|
||||
IR::Value GetAddressResource(IR::Value low, IR::Value high);
|
||||
IR::Value GetScalarAddressResource(uint32_t base);
|
||||
IR::Value GetScalarAddressResource(const Decoder::Operand& base);
|
||||
IR::Value GetImageResource(const IR::MemoryInfo& memory);
|
||||
IR::Value GetSamplerResource(const IR::MemoryInfo& memory);
|
||||
IR::Value MakeImageAddress(const Decoder::Instruction& inst, const Decoder::Operand& base);
|
||||
@@ -175,8 +194,6 @@ private:
|
||||
bool PackedInteger16MinMax(const Decoder::Instruction& inst, IR::ValueOpcode opcode, bool sign);
|
||||
bool S_U64_MASK(const Decoder::Instruction& inst, IR::ValueOpcode logical_opcode,
|
||||
IR::ValueOpcode bit_opcode, bool negate_rhs, bool negate_result, bool unary);
|
||||
IR::U1 U64MaskBinary(const Decoder::Instruction& inst, IR::ValueOpcode opcode, bool negate_rhs,
|
||||
bool negate_result);
|
||||
bool SimpleInteger(const Decoder::Instruction& inst, IR::ValueOpcode opcode, IR::Type type,
|
||||
bool reverse, bool mask_shift_count, bool update_scc);
|
||||
bool ComposedIntegerBinary(const Decoder::Instruction& inst, IR::ValueOpcode opcode,
|
||||
|
||||
@@ -76,6 +76,14 @@ void IREmitter::SetExec(U1 value) {
|
||||
Emit(ValueOpcode::SetExec, {value});
|
||||
}
|
||||
|
||||
U1 IREmitter::GetExecMaskTag() {
|
||||
return U1(Emit(ValueOpcode::GetExecMaskTag));
|
||||
}
|
||||
|
||||
void IREmitter::SetExecMaskTag(U1 value) {
|
||||
Emit(ValueOpcode::SetExecMaskTag, {value});
|
||||
}
|
||||
|
||||
U32 IREmitter::GetExecLo() {
|
||||
return U32(Emit(ValueOpcode::GetExecLo));
|
||||
}
|
||||
@@ -100,6 +108,14 @@ void IREmitter::SetVcc(U1 value) {
|
||||
Emit(ValueOpcode::SetVcc, {value});
|
||||
}
|
||||
|
||||
U1 IREmitter::GetVccMaskValidTag() {
|
||||
return U1(Emit(ValueOpcode::GetVccMaskValidTag));
|
||||
}
|
||||
|
||||
void IREmitter::SetVccMaskValidTag(U1 value) {
|
||||
Emit(ValueOpcode::SetVccMaskValidTag, {value});
|
||||
}
|
||||
|
||||
U32 IREmitter::GetVccLo() {
|
||||
return U32(Emit(ValueOpcode::GetVccLo));
|
||||
}
|
||||
|
||||
@@ -39,12 +39,16 @@ public:
|
||||
void SetScc(U1 value);
|
||||
U1 GetExec();
|
||||
void SetExec(U1 value);
|
||||
U1 GetExecMaskTag();
|
||||
void SetExecMaskTag(U1 value);
|
||||
U32 GetExecLo();
|
||||
void SetExecLo(U32 value);
|
||||
U32 GetExecHi();
|
||||
void SetExecHi(U32 value);
|
||||
U1 GetVcc();
|
||||
void SetVcc(U1 value);
|
||||
U1 GetVccMaskValidTag();
|
||||
void SetVccMaskValidTag(U1 value);
|
||||
U32 GetVccLo();
|
||||
void SetVccLo(U32 value);
|
||||
U32 GetVccHi();
|
||||
|
||||
@@ -31,12 +31,16 @@ bool IsRegisterStatePseudo(ValueOpcode opcode) {
|
||||
case ValueOpcode::SetScc:
|
||||
case ValueOpcode::GetExec:
|
||||
case ValueOpcode::SetExec:
|
||||
case ValueOpcode::GetExecMaskTag:
|
||||
case ValueOpcode::SetExecMaskTag:
|
||||
case ValueOpcode::GetExecLo:
|
||||
case ValueOpcode::SetExecLo:
|
||||
case ValueOpcode::GetExecHi:
|
||||
case ValueOpcode::SetExecHi:
|
||||
case ValueOpcode::GetVcc:
|
||||
case ValueOpcode::SetVcc:
|
||||
case ValueOpcode::GetVccMaskValidTag:
|
||||
case ValueOpcode::SetVccMaskValidTag:
|
||||
case ValueOpcode::GetVccLo:
|
||||
case ValueOpcode::SetVccLo:
|
||||
case ValueOpcode::GetVccHi:
|
||||
|
||||
@@ -23,12 +23,16 @@ VALUE_OPCODE(GetScc, U1)
|
||||
VALUE_OPCODE(SetScc, Void, U1)
|
||||
VALUE_OPCODE(GetExec, U1)
|
||||
VALUE_OPCODE(SetExec, Void, U1)
|
||||
VALUE_OPCODE(GetExecMaskTag, U1)
|
||||
VALUE_OPCODE(SetExecMaskTag, Void, U1)
|
||||
VALUE_OPCODE(GetExecLo, U32)
|
||||
VALUE_OPCODE(SetExecLo, Void, U32)
|
||||
VALUE_OPCODE(GetExecHi, U32)
|
||||
VALUE_OPCODE(SetExecHi, Void, U32)
|
||||
VALUE_OPCODE(GetVcc, U1)
|
||||
VALUE_OPCODE(SetVcc, Void, U1)
|
||||
VALUE_OPCODE(GetVccMaskValidTag, U1)
|
||||
VALUE_OPCODE(SetVccMaskValidTag, Void, U1)
|
||||
VALUE_OPCODE(GetVccLo, U32)
|
||||
VALUE_OPCODE(SetVccLo, Void, U32)
|
||||
VALUE_OPCODE(GetVccHi, U32)
|
||||
|
||||
@@ -22,6 +22,9 @@ struct ScalarMaskTag {
|
||||
struct ExecTag {
|
||||
auto operator<=>(const ExecTag&) const = default;
|
||||
};
|
||||
struct ExecMaskTag {
|
||||
auto operator<=>(const ExecMaskTag&) const = default;
|
||||
};
|
||||
struct ExecLoTag {
|
||||
auto operator<=>(const ExecLoTag&) const = default;
|
||||
};
|
||||
@@ -31,6 +34,9 @@ struct ExecHiTag {
|
||||
struct VccTag {
|
||||
auto operator<=>(const VccTag&) const = default;
|
||||
};
|
||||
struct VccMaskValidTag {
|
||||
auto operator<=>(const VccMaskValidTag&) const = default;
|
||||
};
|
||||
struct VccLoTag {
|
||||
auto operator<=>(const VccLoTag&) const = default;
|
||||
};
|
||||
@@ -45,9 +51,9 @@ struct GotoVariable {
|
||||
auto operator<=>(const GotoVariable&) const = default;
|
||||
};
|
||||
|
||||
using Variable =
|
||||
std::variant<ScalarReg, ThreadBitScalarReg, ScalarMaskTag, VectorReg, GotoVariable, SccTag,
|
||||
ExecTag, ExecLoTag, ExecHiTag, VccTag, VccLoTag, VccHiTag, M0Tag>;
|
||||
using Variable = std::variant<ScalarReg, ThreadBitScalarReg, ScalarMaskTag, VectorReg, GotoVariable,
|
||||
SccTag, ExecTag, ExecMaskTag, ExecLoTag, ExecHiTag, VccTag,
|
||||
VccMaskValidTag, VccLoTag, VccHiTag, M0Tag>;
|
||||
using ValueMap = std::unordered_map<Block*, Value>;
|
||||
|
||||
struct DefTable {
|
||||
@@ -99,12 +105,16 @@ struct DefTable {
|
||||
void Set(Block* block, SccTag, Value value) { scc[block] = value; }
|
||||
const Value& Get(Block* block, ExecTag) { return exec[block]; }
|
||||
void Set(Block* block, ExecTag, Value value) { exec[block] = value; }
|
||||
const Value& Get(Block* block, ExecMaskTag) { return exec_mask_tag[block]; }
|
||||
void Set(Block* block, ExecMaskTag, Value value) { exec_mask_tag[block] = value; }
|
||||
const Value& Get(Block* block, ExecLoTag) { return exec_lo[block]; }
|
||||
void Set(Block* block, ExecLoTag, Value value) { exec_lo[block] = value; }
|
||||
const Value& Get(Block* block, ExecHiTag) { return exec_hi[block]; }
|
||||
void Set(Block* block, ExecHiTag, Value value) { exec_hi[block] = value; }
|
||||
const Value& Get(Block* block, VccTag) { return vcc[block]; }
|
||||
void Set(Block* block, VccTag, Value value) { vcc[block] = value; }
|
||||
const Value& Get(Block* block, VccMaskValidTag) { return vcc_mask_valid_tag[block]; }
|
||||
void Set(Block* block, VccMaskValidTag, Value value) { vcc_mask_valid_tag[block] = value; }
|
||||
const Value& Get(Block* block, VccLoTag) { return vcc_lo[block]; }
|
||||
void Set(Block* block, VccLoTag, Value value) { vcc_lo[block] = value; }
|
||||
const Value& Get(Block* block, VccHiTag) { return vcc_hi[block]; }
|
||||
@@ -114,9 +124,11 @@ struct DefTable {
|
||||
|
||||
ValueMap scc;
|
||||
ValueMap exec;
|
||||
ValueMap exec_mask_tag;
|
||||
ValueMap exec_lo;
|
||||
ValueMap exec_hi;
|
||||
ValueMap vcc;
|
||||
ValueMap vcc_mask_valid_tag;
|
||||
ValueMap vcc_lo;
|
||||
ValueMap vcc_hi;
|
||||
ValueMap m0;
|
||||
@@ -144,6 +156,9 @@ ValueOpcode UndefOpcode(SccTag) {
|
||||
ValueOpcode UndefOpcode(ExecTag) {
|
||||
return ValueOpcode::UndefU1;
|
||||
}
|
||||
ValueOpcode UndefOpcode(ExecMaskTag) {
|
||||
return ValueOpcode::UndefU1;
|
||||
}
|
||||
ValueOpcode UndefOpcode(ExecLoTag) {
|
||||
return ValueOpcode::UndefU32;
|
||||
}
|
||||
@@ -153,6 +168,9 @@ ValueOpcode UndefOpcode(ExecHiTag) {
|
||||
ValueOpcode UndefOpcode(VccTag) {
|
||||
return ValueOpcode::UndefU1;
|
||||
}
|
||||
ValueOpcode UndefOpcode(VccMaskValidTag) {
|
||||
return ValueOpcode::UndefU1;
|
||||
}
|
||||
ValueOpcode UndefOpcode(VccLoTag) {
|
||||
return ValueOpcode::UndefU32;
|
||||
}
|
||||
@@ -184,6 +202,9 @@ Value InitialValue(SccTag) {
|
||||
Value InitialValue(ExecTag) {
|
||||
return Value(false);
|
||||
}
|
||||
Value InitialValue(ExecMaskTag) {
|
||||
return Value(false);
|
||||
}
|
||||
Value InitialValue(ExecLoTag) {
|
||||
return Value(0u);
|
||||
}
|
||||
@@ -193,6 +214,9 @@ Value InitialValue(ExecHiTag) {
|
||||
Value InitialValue(VccTag) {
|
||||
return Value(false);
|
||||
}
|
||||
Value InitialValue(VccMaskValidTag) {
|
||||
return Value(false);
|
||||
}
|
||||
Value InitialValue(VccLoTag) {
|
||||
return Value(0u);
|
||||
}
|
||||
@@ -351,9 +375,15 @@ void VisitInstruction(Pass& pass, Block* block, Inst& inst) {
|
||||
break;
|
||||
case ValueOpcode::SetScc: pass.Write(SccTag {}, block, inst.Arg(0)); break;
|
||||
case ValueOpcode::SetExec: pass.Write(ExecTag {}, block, inst.Arg(0)); break;
|
||||
case ValueOpcode::SetExecMaskTag:
|
||||
pass.Write(ExecMaskTag {}, block, inst.Arg(0));
|
||||
break;
|
||||
case ValueOpcode::SetExecLo: pass.Write(ExecLoTag {}, block, inst.Arg(0)); break;
|
||||
case ValueOpcode::SetExecHi: pass.Write(ExecHiTag {}, block, inst.Arg(0)); break;
|
||||
case ValueOpcode::SetVcc: pass.Write(VccTag {}, block, inst.Arg(0)); break;
|
||||
case ValueOpcode::SetVccMaskValidTag:
|
||||
pass.Write(VccMaskValidTag {}, block, inst.Arg(0));
|
||||
break;
|
||||
case ValueOpcode::SetVccLo: pass.Write(VccLoTag {}, block, inst.Arg(0)); break;
|
||||
case ValueOpcode::SetVccHi: pass.Write(VccHiTag {}, block, inst.Arg(0)); break;
|
||||
case ValueOpcode::SetM0: pass.Write(M0Tag {}, block, inst.Arg(0)); break;
|
||||
@@ -375,9 +405,15 @@ void VisitInstruction(Pass& pass, Block* block, Inst& inst) {
|
||||
break;
|
||||
case ValueOpcode::GetScc: inst.ReplaceUsesWith(pass.Read(SccTag {}, block)); break;
|
||||
case ValueOpcode::GetExec: inst.ReplaceUsesWith(pass.Read(ExecTag {}, block)); break;
|
||||
case ValueOpcode::GetExecMaskTag:
|
||||
inst.ReplaceUsesWith(pass.Read(ExecMaskTag {}, block));
|
||||
break;
|
||||
case ValueOpcode::GetExecLo: inst.ReplaceUsesWith(pass.Read(ExecLoTag {}, block)); break;
|
||||
case ValueOpcode::GetExecHi: inst.ReplaceUsesWith(pass.Read(ExecHiTag {}, block)); break;
|
||||
case ValueOpcode::GetVcc: inst.ReplaceUsesWith(pass.Read(VccTag {}, block)); break;
|
||||
case ValueOpcode::GetVccMaskValidTag:
|
||||
inst.ReplaceUsesWith(pass.Read(VccMaskValidTag {}, block));
|
||||
break;
|
||||
case ValueOpcode::GetVccLo: inst.ReplaceUsesWith(pass.Read(VccLoTag {}, block)); break;
|
||||
case ValueOpcode::GetVccHi: inst.ReplaceUsesWith(pass.Read(VccHiTag {}, block)); break;
|
||||
case ValueOpcode::GetM0: inst.ReplaceUsesWith(pass.Read(M0Tag {}, block)); break;
|
||||
|
||||
Reference in New Issue
Block a user