graphics: use per-invocation shader masks

This commit is contained in:
nmzik
2026-08-21 08:02:21 +02:00
parent 79443aa104
commit 49ea1680f8
30 changed files with 897 additions and 1140 deletions
@@ -7,37 +7,26 @@ bool Translator::TranslateStateOperation(const IR::Instruction& inst) {
switch (inst.op) {
case IR::Opcode::SaveexecB32:
case IR::Opcode::SaveexecB64: {
const auto old_low = ir.GetExecLo();
const auto old_high = ir.GetExecHi();
const auto src_low = ReadRawU32(inst.src[0]);
const auto src_high = inst.op == IR::Opcode::SaveexecB64
? ReadRawU32(OffsetOperand(inst.src[0], 1))
: IR::U32(IR::Value(0u));
const auto apply = [&](IR::U32 old_value, IR::U32 src_value) {
switch (inst.saveexec_mode) {
case IR::SaveexecMode::And: return ir.BitwiseAnd(old_value, src_value);
case IR::SaveexecMode::Andn1:
return ir.BitwiseAnd(old_value, ir.BitwiseNot(src_value));
case IR::SaveexecMode::Orn2:
return ir.BitwiseOr(ir.BitwiseNot(old_value), src_value);
}
return old_value;
};
const auto result_low = apply(old_low, src_low);
const auto result_high =
inst.op == IR::Opcode::SaveexecB64 ? apply(old_high, src_high) : old_high;
if (inst.op == IR::Opcode::SaveexecB64) {
WriteU32Pair(inst.dst, {old_low, old_high});
} else {
WriteOperand(inst.dst, old_low);
const auto old = ir.GetExec();
const auto src = ReadMask(inst.src[0]);
IR::U1 result;
switch (inst.saveexec_mode) {
case IR::SaveexecMode::And: result = ir.LogicalAnd(old, src); break;
case IR::SaveexecMode::Andn1:
result = ir.LogicalAnd(old, ir.LogicalNot(src));
break;
case IR::SaveexecMode::Orn2: result = ir.LogicalOr(ir.LogicalNot(old), src); break;
}
ir.SetExecLo(result_low);
ir.SetExecHi(result_high);
ir.SetExec(ThreadBit(result_low, result_high));
const auto nonzero = inst.op == IR::Opcode::SaveexecB64
? ir.BitwiseOr(result_low, result_high)
: result_low;
ir.SetScc(ir.INotEqual(nonzero, zero));
if (inst.op == IR::Opcode::SaveexecB64) {
WriteMask64(inst.dst, old);
} else {
WriteMask(inst.dst, old);
}
const auto mask = BallotMask(result);
ir.SetExec(result);
ir.SetExecLo(mask[0]);
ir.SetExecHi(mask[1]);
ir.SetScc(result);
return true;
}
case IR::Opcode::ScalarAddCarryU32:
@@ -157,24 +146,61 @@ bool Translator::TranslateMove(const IR::Instruction& inst) {
WriteOperand(inst.dst, ReadOperand(inst.src[0], IR::Type::F32));
return true;
case IR::Opcode::MoveU64: {
const auto is_exec_or_vcc = [](const IR::Operand& operand) {
return operand.kind == IR::OperandKind::Register &&
(operand.reg.file == IR::RegisterFile::Exec ||
operand.reg.file == IR::RegisterFile::Vcc);
};
if (is_exec_or_vcc(inst.dst) || is_exec_or_vcc(inst.src[0])) {
WriteMask64(inst.dst, ReadMask(inst.src[0]));
return true;
}
const bool scalar_copy = inst.dst.kind == IR::OperandKind::Register &&
inst.src[0].kind == IR::OperandKind::Register &&
inst.dst.reg.file == IR::RegisterFile::Scalar &&
inst.src[0].reg.file == IR::RegisterFile::Scalar;
IR::U1 source_mask;
IR::U1 source_mask_valid;
if (scalar_copy) {
const auto src = static_cast<IR::ScalarReg>(inst.src[0].reg.index);
source_mask = ir.GetThreadBitScalarReg(src);
source_mask_valid = ir.GetScalarMaskTag(src);
}
WriteU32Pair(inst.dst, ReadU32Pair(inst.src[0]));
if (inst.dst.kind == IR::OperandKind::Register &&
inst.src[0].kind == IR::OperandKind::Register &&
inst.dst.reg.file == IR::RegisterFile::Scalar &&
inst.src[0].reg.file == IR::RegisterFile::Scalar) {
ir.SetThreadBitScalarReg(
static_cast<IR::ScalarReg>(inst.dst.reg.index),
ir.GetThreadBitScalarReg(static_cast<IR::ScalarReg>(inst.src[0].reg.index)));
if (scalar_copy) {
ir.SetThreadBitScalarReg(static_cast<IR::ScalarReg>(inst.dst.reg.index),
source_mask);
ir.SetScalarMaskTag(static_cast<IR::ScalarReg>(inst.dst.reg.index),
source_mask_valid);
}
return true;
}
case IR::Opcode::WqmB64: {
if (!current_per_invocation_masks) {
const auto is_exec_or_vcc = [](const IR::Operand& operand) {
return operand.kind == IR::OperandKind::Register &&
(operand.reg.file == IR::RegisterFile::Exec ||
operand.reg.file == IR::RegisterFile::Vcc);
};
if (!is_exec_or_vcc(inst.dst) && !is_exec_or_vcc(inst.src[0])) {
const auto mask_valid = ReadMaskValid(inst.src[0]);
const auto invocation_result =
IR::U1(ir.Emit(IR::ValueOpcode::WqmMask, {ReadMask(inst.src[0])}));
const auto result = IR::U64(
ir.Emit(IR::ValueOpcode::WqmU64, {ReadOperand(inst.src[0], IR::Type::U64)}));
WriteOperand(inst.dst, result);
ir.SetScc(IR::U1(
ir.Emit(IR::ValueOpcode::INotEqual64, {result, IR::Value(uint64_t {0})})));
if (inst.dst.kind == IR::OperandKind::Register &&
inst.dst.reg.file == IR::RegisterFile::Scalar) {
const auto dst = static_cast<IR::ScalarReg>(inst.dst.reg.index);
ir.SetThreadBitScalarReg(dst, invocation_result);
ir.SetScalarMaskTag(dst, mask_valid);
const auto raw_nonzero = IR::U1(
ir.Emit(IR::ValueOpcode::INotEqual64, {result, IR::Value(uint64_t {0})}));
ir.SetScc(IR::U1(ir.Emit(
IR::ValueOpcode::SelectU1, {mask_valid, invocation_result, raw_nonzero})));
} else {
ir.SetScc(IR::U1(
ir.Emit(IR::ValueOpcode::INotEqual64, {result, IR::Value(uint64_t {0})})));
}
return true;
}
const auto result = IR::U1(ir.Emit(IR::ValueOpcode::WqmMask, {ReadMask(inst.src[0])}));
@@ -3,13 +3,76 @@
namespace Libs::Graphics::ShaderRecompiler::Frontend::Detail {
bool Translator::TranslateU64MaskOperation(const IR::Instruction& inst) {
if (current_per_invocation_masks) {
return TranslatePerInvocationU64Mask(inst);
const auto invocation_result = EvaluateU64Mask(inst);
const auto is_exec_or_vcc = [](const IR::Operand& operand) {
return operand.kind == IR::OperandKind::Register &&
(operand.reg.file == IR::RegisterFile::Exec ||
operand.reg.file == IR::RegisterFile::Vcc);
};
if (is_exec_or_vcc(inst.dst) || is_exec_or_vcc(inst.src[0]) ||
(inst.src_count > 1u && is_exec_or_vcc(inst.src[1]))) {
WriteMask64(inst.dst, invocation_result);
ir.SetScc(invocation_result);
return true;
}
return inst.op == IR::Opcode::BitwiseAndU64 || inst.op == IR::Opcode::BitwiseOrU64 ||
inst.op == IR::Opcode::BitwiseXorU64 || inst.op == IR::Opcode::BitwiseNotU64
? TranslateSimpleInteger(inst)
: TranslateComposedInteger(inst);
const auto lhs = ReadU32Pair(inst.src[0]);
auto mask_valid = ReadMaskValid(inst.src[0]);
if (inst.src_count > 1u) {
mask_valid = ir.LogicalAnd(mask_valid, ReadMaskValid(inst.src[1]));
}
std::array<IR::U32, 2> result;
if (inst.op == IR::Opcode::BitwiseNotU64) {
result = {ir.BitwiseNot(lhs[0]), ir.BitwiseNot(lhs[1])};
} else {
const auto rhs = ReadU32Pair(inst.src[1]);
for (uint32_t component = 0; component < 2u; component++) {
switch (inst.op) {
case IR::Opcode::BitwiseAndU64:
result[component] = ir.BitwiseAnd(lhs[component], rhs[component]);
break;
case IR::Opcode::BitwiseAndNotU64:
result[component] =
ir.BitwiseAnd(lhs[component], ir.BitwiseNot(rhs[component]));
break;
case IR::Opcode::BitwiseOrU64:
result[component] = ir.BitwiseOr(lhs[component], rhs[component]);
break;
case IR::Opcode::BitwiseOrNotU64:
result[component] = ir.BitwiseOr(lhs[component], ir.BitwiseNot(rhs[component]));
break;
case IR::Opcode::BitwiseXorU64:
result[component] = ir.BitwiseXor(lhs[component], rhs[component]);
break;
case IR::Opcode::BitwiseNandU64:
result[component] =
ir.BitwiseNot(ir.BitwiseAnd(lhs[component], rhs[component]));
break;
case IR::Opcode::BitwiseNorU64:
result[component] = ir.BitwiseNot(ir.BitwiseOr(lhs[component], rhs[component]));
break;
case IR::Opcode::BitwiseXnorU64:
result[component] =
ir.BitwiseNot(ir.BitwiseXor(lhs[component], rhs[component]));
break;
default: return false;
}
}
}
WriteU32Pair(inst.dst, result);
if (inst.dst.kind == IR::OperandKind::Register &&
inst.dst.reg.file == IR::RegisterFile::Scalar) {
const auto dst = static_cast<IR::ScalarReg>(inst.dst.reg.index);
ir.SetThreadBitScalarReg(dst, invocation_result);
ir.SetScalarMaskTag(dst, mask_valid);
const auto raw_nonzero =
ir.INotEqual(ir.BitwiseOr(result[0], result[1]), IR::U32(IR::Value(0u)));
ir.SetScc(IR::U1(
ir.Emit(IR::ValueOpcode::SelectU1, {mask_valid, invocation_result, raw_nonzero})));
} else {
ir.SetScc(ir.INotEqual(ir.BitwiseOr(result[0], result[1]), IR::U32(IR::Value(0u))));
}
return true;
}
bool Translator::TranslateInstruction(const IR::Instruction& inst) {
@@ -126,62 +126,26 @@ bool Translator::TranslatePackedInteger16(const IR::Instruction& inst) {
}
}
bool Translator::TranslatePerInvocationU64Mask(const IR::Instruction& inst) {
if (!current_per_invocation_masks) {
return false;
}
IR::U1 Translator::EvaluateU64Mask(const IR::Instruction& inst) {
const auto lhs = [&] { return ReadMask(inst.src[0]); };
const auto rhs = [&] { return ReadMask(inst.src[1]); };
IR::U1 result;
switch (inst.op) {
case IR::Opcode::BitwiseAndU64: result = ir.LogicalAnd(lhs(), rhs()); break;
case IR::Opcode::BitwiseOrU64: result = ir.LogicalOr(lhs(), rhs()); break;
case IR::Opcode::BitwiseAndU64: return ir.LogicalAnd(lhs(), rhs());
case IR::Opcode::BitwiseOrU64: return ir.LogicalOr(lhs(), rhs());
case IR::Opcode::BitwiseXorU64:
result = IR::U1(ir.Emit(IR::ValueOpcode::LogicalXor, {lhs(), rhs()}));
break;
case IR::Opcode::BitwiseAndNotU64:
result = ir.LogicalAnd(lhs(), ir.LogicalNot(rhs()));
break;
case IR::Opcode::BitwiseOrNotU64: result = ir.LogicalOr(lhs(), ir.LogicalNot(rhs())); break;
case IR::Opcode::BitwiseNandU64: result = ir.LogicalNot(ir.LogicalAnd(lhs(), rhs())); break;
case IR::Opcode::BitwiseNorU64: result = ir.LogicalNot(ir.LogicalOr(lhs(), rhs())); break;
return IR::U1(ir.Emit(IR::ValueOpcode::LogicalXor, {lhs(), rhs()}));
case IR::Opcode::BitwiseAndNotU64: return ir.LogicalAnd(lhs(), ir.LogicalNot(rhs()));
case IR::Opcode::BitwiseOrNotU64: return ir.LogicalOr(lhs(), ir.LogicalNot(rhs()));
case IR::Opcode::BitwiseNandU64: return ir.LogicalNot(ir.LogicalAnd(lhs(), rhs()));
case IR::Opcode::BitwiseNorU64: return ir.LogicalNot(ir.LogicalOr(lhs(), rhs()));
case IR::Opcode::BitwiseXnorU64:
result = ir.LogicalNot(IR::U1(ir.Emit(IR::ValueOpcode::LogicalXor, {lhs(), rhs()})));
break;
case IR::Opcode::BitwiseNotU64: result = ir.LogicalNot(lhs()); break;
default: return false;
return ir.LogicalNot(IR::U1(ir.Emit(IR::ValueOpcode::LogicalXor, {lhs(), rhs()})));
case IR::Opcode::BitwiseNotU64: return ir.LogicalNot(lhs());
default: EXIT("invalid 64-bit mask opcode");
}
WriteMask64(inst.dst, result);
return true;
}
bool Translator::TranslateSimpleInteger(const IR::Instruction& inst) {
if (inst.op == IR::Opcode::BitwiseAndU64 || inst.op == IR::Opcode::BitwiseOrU64 ||
inst.op == IR::Opcode::BitwiseXorU64 || inst.op == IR::Opcode::BitwiseNotU64) {
const auto lhs = ReadU32Pair(inst.src[0]);
std::array<IR::U32, 2> result;
if (inst.op == IR::Opcode::BitwiseNotU64) {
result = {ir.BitwiseNot(lhs[0]), ir.BitwiseNot(lhs[1])};
} else {
const auto rhs = ReadU32Pair(inst.src[1]);
for (uint32_t component = 0; component < 2; component++) {
switch (inst.op) {
case IR::Opcode::BitwiseAndU64:
result[component] = ir.BitwiseAnd(lhs[component], rhs[component]);
break;
case IR::Opcode::BitwiseOrU64:
result[component] = ir.BitwiseOr(lhs[component], rhs[component]);
break;
default:
result[component] = ir.BitwiseXor(lhs[component], rhs[component]);
break;
}
}
}
WriteU32Pair(inst.dst, result);
return true;
}
IR::ValueOpcode opcode {};
IR::Type type = IR::Type::U32;
switch (inst.op) {
@@ -250,37 +214,6 @@ bool Translator::TranslateSimpleInteger(const IR::Instruction& inst) {
}
bool Translator::TranslateComposedInteger(const IR::Instruction& inst) {
if (inst.op == IR::Opcode::BitwiseAndNotU64 || inst.op == IR::Opcode::BitwiseOrNotU64 ||
inst.op == IR::Opcode::BitwiseNandU64 || inst.op == IR::Opcode::BitwiseNorU64 ||
inst.op == IR::Opcode::BitwiseXnorU64) {
const auto lhs = ReadU32Pair(inst.src[0]);
const auto rhs = ReadU32Pair(inst.src[1]);
std::array<IR::U32, 2> result;
for (uint32_t component = 0; component < 2; component++) {
switch (inst.op) {
case IR::Opcode::BitwiseAndNotU64:
result[component] =
ir.BitwiseAnd(lhs[component], ir.BitwiseNot(rhs[component]));
break;
case IR::Opcode::BitwiseOrNotU64:
result[component] = ir.BitwiseOr(lhs[component], ir.BitwiseNot(rhs[component]));
break;
case IR::Opcode::BitwiseNandU64:
result[component] =
ir.BitwiseNot(ir.BitwiseAnd(lhs[component], rhs[component]));
break;
case IR::Opcode::BitwiseNorU64:
result[component] = ir.BitwiseNot(ir.BitwiseOr(lhs[component], rhs[component]));
break;
default:
result[component] =
ir.BitwiseNot(ir.BitwiseXor(lhs[component], rhs[component]));
break;
}
}
WriteU32Pair(inst.dst, result);
return true;
}
const auto binary_u32 = [&](auto operation) {
const auto lhs = ReadU32(inst.src[0]);
const auto rhs = ReadU32(inst.src[1]);
@@ -612,10 +545,28 @@ bool Translator::TranslateExtendedInteger(const IR::Instruction& inst) {
}
case IR::Opcode::SelectU64: {
const auto condition = ReadCondition(inst.src[0]);
const auto lhs = ReadU32Pair(inst.src[1]);
const auto rhs = ReadU32Pair(inst.src[2]);
const auto selected_mask =
IR::U1(ir.Emit(IR::ValueOpcode::SelectU1,
{condition, ReadMask(inst.src[1]), ReadMask(inst.src[2])}));
const auto selected_mask_valid =
IR::U1(ir.Emit(IR::ValueOpcode::SelectU1, {condition, ReadMaskValid(inst.src[1]),
ReadMaskValid(inst.src[2])}));
if (inst.dst.kind == IR::OperandKind::Register &&
(inst.dst.reg.file == IR::RegisterFile::Exec ||
inst.dst.reg.file == IR::RegisterFile::Vcc)) {
WriteMask64(inst.dst, selected_mask);
return true;
}
const auto lhs = ReadU32Pair(inst.src[1]);
const auto rhs = ReadU32Pair(inst.src[2]);
WriteU32Pair(inst.dst, {ir.Select(condition, lhs[0], rhs[0]),
ir.Select(condition, lhs[1], rhs[1])});
if (inst.dst.kind == IR::OperandKind::Register &&
inst.dst.reg.file == IR::RegisterFile::Scalar) {
const auto dst = static_cast<IR::ScalarReg>(inst.dst.reg.index);
ir.SetThreadBitScalarReg(dst, selected_mask);
ir.SetScalarMaskTag(dst, selected_mask_valid);
}
return true;
}
case IR::Opcode::PackLowLowU16:
@@ -91,19 +91,8 @@ IR::U32 Translator::ReadPcRelativeU32(const IR::Operand& operand) {
}
std::array<IR::U32, 2> Translator::BallotMask(IR::U1 value) {
if (current_per_invocation_masks) {
return {ir.Select(value, IR::U32(IR::Value(1u)), IR::U32(IR::Value(0u))),
IR::U32(IR::Value(0u))};
}
const auto ballot = ir.Emit(IR::ValueOpcode::Ballot, {value});
const auto low =
IR::U32(ir.Emit(IR::ValueOpcode::CompositeExtractU32x4, {ballot, IR::Value(0u)}));
// RDNA2 wave32 masks have one architectural dword; the upper half is unused.
const auto high =
current_wave_size == 32u
? IR::U32(IR::Value(0u))
: IR::U32(ir.Emit(IR::ValueOpcode::CompositeExtractU32x4, {ballot, IR::Value(1u)}));
return {low, high};
return {ir.Select(value, IR::U32(IR::Value(1u)), IR::U32(IR::Value(0u))),
IR::U32(IR::Value(0u))};
}
IR::U32 Translator::ReadRawU32(const IR::Operand& operand) {
@@ -271,9 +260,17 @@ void Translator::WriteRawU32(const IR::Operand& operand, IR::U32 value) {
}
}
switch (operand.reg.file) {
case IR::RegisterFile::Scalar:
ir.SetScalarReg(static_cast<IR::ScalarReg>(operand.reg.index), value);
case IR::RegisterFile::Scalar: {
const auto reg = static_cast<IR::ScalarReg>(operand.reg.index);
ir.SetScalarReg(reg, value);
ir.SetThreadBitScalarReg(reg, ThreadBit(value));
ir.SetScalarMaskTag(reg, IR::U1(IR::Value(false)));
if (IR::RegIndex(reg) > 0u) {
ir.SetScalarMaskTag(static_cast<IR::ScalarReg>(IR::RegIndex(reg) - 1u),
IR::U1(IR::Value(false)));
}
break;
}
case IR::RegisterFile::Vector: {
const auto reg = static_cast<IR::VectorReg>(operand.reg.index);
const auto old = ir.GetVectorReg(reg);
@@ -296,20 +293,20 @@ void Translator::WriteRawU32(const IR::Operand& operand, IR::U32 value) {
case IR::RegisterFile::Vcc:
if (operand.reg.index == 0) {
ir.SetVccLo(IR::U32(value));
ir.SetVcc(ThreadBit(IR::U32(value), ir.GetVccHi()));
ir.SetVcc(ThreadBit(IR::U32(value)));
} else {
ir.SetVccHi(IR::U32(value));
ir.SetVcc(ThreadBit(ir.GetVccLo(), IR::U32(value)));
ir.SetVcc(ThreadBit(ir.GetVccLo()));
}
break;
case IR::RegisterFile::M0: ir.SetM0(IR::U32(value)); break;
case IR::RegisterFile::Exec:
if (operand.reg.index == 0) {
ir.SetExecLo(IR::U32(value));
ir.SetExec(ThreadBit(IR::U32(value), ir.GetExecHi()));
ir.SetExec(ThreadBit(IR::U32(value)));
} else {
ir.SetExecHi(IR::U32(value));
ir.SetExec(ThreadBit(ir.GetExecLo(), IR::U32(value)));
ir.SetExec(ThreadBit(ir.GetExecLo()));
}
break;
case IR::RegisterFile::Scc: ir.SetScc(ir.INotEqual(value, IR::U32(IR::Value(0u)))); break;
@@ -459,10 +456,10 @@ IR::U64 Translator::ReadU64(const IR::Operand& operand) {
IR::F32 Translator::ReadF16LaneAsF32(const IR::Operand& operand, bool high_lane, bool packed) {
if (operand.float_inline) {
const bool use_zero = packed && (high_lane ? operand.op_sel_hi : operand.op_sel);
auto value = use_zero ? IR::F32(IR::Value::F32(0.0f))
: ir.BitCastF32(IR::U32(IR::Value(operand.imm)));
const auto half = IR::F16(ir.Emit(IR::ValueOpcode::ConvertF16F32, {value}));
value = IR::F32(ir.Emit(IR::ValueOpcode::ConvertF32F16, {half}));
auto value = use_zero ? IR::F32(IR::Value::F32(0.0f))
: ir.BitCastF32(IR::U32(IR::Value(operand.imm)));
const auto half = IR::F16(ir.Emit(IR::ValueOpcode::ConvertF16F32, {value}));
value = IR::F32(ir.Emit(IR::ValueOpcode::ConvertF32F16, {half}));
if (operand.absolute) {
value = IR::F32(ir.Emit(IR::ValueOpcode::FPAbs32, {value}));
}
@@ -561,7 +558,7 @@ void Translator::WriteU32Pair(const IR::Operand& operand, const std::array<IR::U
return;
}
if (operand.kind == IR::OperandKind::Register) {
const auto thread_bit = ThreadBit(value[0], value[1]);
const auto thread_bit = ThreadBit(value[0]);
switch (operand.reg.file) {
case IR::RegisterFile::Exec:
ir.SetExec(thread_bit);
@@ -573,9 +570,7 @@ void Translator::WriteU32Pair(const IR::Operand& operand, const std::array<IR::U
ir.SetVccLo(value[0]);
ir.SetVccHi(value[1]);
return;
case IR::RegisterFile::Scalar:
ir.SetThreadBitScalarReg(static_cast<IR::ScalarReg>(operand.reg.index), thread_bit);
break;
case IR::RegisterFile::Scalar: break;
default: break;
}
}
@@ -583,21 +578,8 @@ void Translator::WriteU32Pair(const IR::Operand& operand, const std::array<IR::U
WriteRawU32(OffsetOperand(operand, 1), value[1]);
}
IR::U1 Translator::ThreadBit(IR::U32 low, IR::U32 high) {
if (current_per_invocation_masks) {
return ir.INotEqual(low, IR::U32(IR::Value(0u)));
}
const auto lane = IR::U32(ir.Emit(IR::ValueOpcode::LaneId));
auto word = low;
if (current_wave_size == 64u) {
const auto high_lane =
IR::U1(ir.Emit(IR::ValueOpcode::UGreaterThanEqual32, {lane, IR::Value(32u)}));
word = ir.Select(high_lane, high, low);
}
const auto bit =
ir.BitwiseAnd(ir.ShiftRightLogical(word, ir.BitwiseAnd(lane, IR::U32(IR::Value(31u)))),
IR::U32(IR::Value(1u)));
return ir.INotEqual(bit, IR::U32(IR::Value(0u)));
IR::U1 Translator::ThreadBit(IR::U32 low) {
return ir.INotEqual(low, IR::U32(IR::Value(0u)));
}
IR::U1 Translator::ReadCondition(const IR::Operand& operand) {
@@ -619,11 +601,7 @@ IR::U1 Translator::ReadMask(const IR::Operand& operand) {
switch (operand.reg.file) {
case IR::RegisterFile::Scalar: {
const auto reg = static_cast<IR::ScalarReg>(operand.reg.index);
const auto high =
current_wave_size == 64u && IR::RegIndex(reg) + 1u < IR::NumScalarRegs
? ir.GetScalarReg(static_cast<IR::ScalarReg>(IR::RegIndex(reg) + 1u))
: IR::U32(IR::Value(0u));
return ThreadBit(ir.GetScalarReg(reg), high);
return ir.GetThreadBitScalarReg(reg);
}
case IR::RegisterFile::Exec: return ir.GetExec();
case IR::RegisterFile::Vcc: return ir.GetVcc();
@@ -632,6 +610,25 @@ IR::U1 Translator::ReadMask(const IR::Operand& operand) {
}
}
IR::U1 Translator::ReadMaskValid(const IR::Operand& operand) {
if (operand.kind == IR::OperandKind::ImmediateU32) {
const bool zero = operand.imm == 0u && !operand.sext_64;
const bool ones = operand.imm == 0xffffffffu && operand.sext_64;
return IR::U1(IR::Value(zero || ones));
}
if (operand.kind != IR::OperandKind::Register) {
return IR::U1(IR::Value(false));
}
switch (operand.reg.file) {
case IR::RegisterFile::Scalar:
return ir.GetScalarMaskTag(static_cast<IR::ScalarReg>(operand.reg.index));
case IR::RegisterFile::Exec:
case IR::RegisterFile::Vcc:
case IR::RegisterFile::Scc: return IR::U1(IR::Value(true));
default: return IR::U1(IR::Value(false));
}
}
void Translator::WriteMask(const IR::Operand& operand, IR::U1 value) {
EXIT_IF(operand.kind != IR::OperandKind::Register);
switch (operand.reg.file) {
@@ -639,10 +636,18 @@ void Translator::WriteMask(const IR::Operand& operand, IR::U1 value) {
const auto reg = static_cast<IR::ScalarReg>(operand.reg.index);
const auto mask = BallotMask(value);
ir.SetThreadBitScalarReg(reg, value);
ir.SetScalarMaskTag(reg, IR::U1(IR::Value(true)));
if (IR::RegIndex(reg) > 0u) {
ir.SetScalarMaskTag(static_cast<IR::ScalarReg>(IR::RegIndex(reg) - 1u),
IR::U1(IR::Value(false)));
}
ir.SetScalarReg(reg, mask[0]);
// A wave32 VALU mask destination must not overwrite the neighboring SGPR.
if (current_wave_size == 64u && IR::RegIndex(reg) + 1u < IR::NumScalarRegs) {
ir.SetScalarReg(static_cast<IR::ScalarReg>(IR::RegIndex(reg) + 1u), mask[1]);
const auto high = static_cast<IR::ScalarReg>(IR::RegIndex(reg) + 1u);
ir.SetScalarReg(high, mask[1]);
ir.SetThreadBitScalarReg(high, IR::U1(IR::Value(false)));
ir.SetScalarMaskTag(high, IR::U1(IR::Value(false)));
}
return;
}
@@ -676,9 +681,17 @@ void Translator::WriteMask64(const IR::Operand& operand, IR::U1 value) {
const auto reg = static_cast<IR::ScalarReg>(operand.reg.index);
const auto mask = BallotMask(value);
ir.SetThreadBitScalarReg(reg, value);
ir.SetScalarMaskTag(reg, IR::U1(IR::Value(true)));
if (IR::RegIndex(reg) > 0u) {
ir.SetScalarMaskTag(static_cast<IR::ScalarReg>(IR::RegIndex(reg) - 1u),
IR::U1(IR::Value(false)));
}
ir.SetScalarReg(reg, mask[0]);
if (IR::RegIndex(reg) + 1u < IR::NumScalarRegs) {
ir.SetScalarReg(static_cast<IR::ScalarReg>(IR::RegIndex(reg) + 1u), mask[1]);
const auto high = static_cast<IR::ScalarReg>(IR::RegIndex(reg) + 1u);
ir.SetScalarReg(high, mask[1]);
ir.SetThreadBitScalarReg(high, IR::U1(IR::Value(false)));
ir.SetScalarMaskTag(high, IR::U1(IR::Value(false)));
}
}
@@ -733,23 +746,17 @@ bool Translator::AddBranchCondition(const IR::BasicBlock& source, IR::ValueBlock
if (source.terminator.kind != CFG::TerminatorKind::ConditionalBranch) {
return true;
}
IR::U1 condition;
const auto mask_non_zero = [&](IR::U1 value) {
const auto mask = BallotMask(value);
return ir.INotEqual(ir.BitwiseOr(mask[0], mask[1]), IR::U32(IR::Value(0u)));
};
// EXEC and VCC are invocation-local Boolean masks. Branching on that Boolean lets inactive
// invocations leave the region without reconstructing a host-subgroup mask.
IR::U1 condition;
switch (source.terminator.condition) {
case CFG::BranchCondition::Always: condition = IR::U1(IR::Value(true)); break;
case CFG::BranchCondition::SccZero: condition = ir.LogicalNot(ir.GetScc()); break;
case CFG::BranchCondition::SccNonZero: condition = ir.GetScc(); break;
case CFG::BranchCondition::VccZero:
condition = ir.LogicalNot(mask_non_zero(ir.GetVcc()));
break;
case CFG::BranchCondition::VccNonZero: condition = mask_non_zero(ir.GetVcc()); break;
case CFG::BranchCondition::ExecZero:
condition = ir.LogicalNot(mask_non_zero(ir.GetExec()));
break;
case CFG::BranchCondition::ExecNonZero: condition = mask_non_zero(ir.GetExec()); break;
case CFG::BranchCondition::VccZero: condition = ir.LogicalNot(ir.GetVcc()); break;
case CFG::BranchCondition::VccNonZero: condition = ir.GetVcc(); break;
case CFG::BranchCondition::ExecZero: condition = ir.LogicalNot(ir.GetExec()); break;
case CFG::BranchCondition::ExecNonZero: condition = ir.GetExec(); break;
case CFG::BranchCondition::GotoVariable:
if (source.terminator.goto_variable == UINT32_MAX) {
return Fail(error,
@@ -771,11 +778,10 @@ bool TranslateProgram(const IR::Program& source, IR::ValueProgram& result,
const ShaderVertexInputInfo* vertex_input_info,
const ShaderPixelInputInfo* pixel_input_info,
const ShaderComputeInputInfo* compute_input_info, std::string* error) {
result = {};
const uint32_t wave_size = source.wave_size;
const bool per_invocation_masks = source.lane_mask_mode == ShaderLaneMaskMode::PerInvocation;
uint32_t vector_limit = 1u;
const auto include_vector = [&](const IR::Operand& operand, uint32_t count = 1u) {
result = {};
const uint32_t wave_size = source.wave_size;
uint32_t vector_limit = 1u;
const auto include_vector = [&](const IR::Operand& operand, uint32_t count = 1u) {
if (operand.kind == IR::OperandKind::Register &&
operand.reg.file == IR::RegisterFile::Vector) {
vector_limit =
@@ -852,19 +858,14 @@ bool TranslateProgram(const IR::Program& source, IR::ValueProgram& result,
if (IR::RegIndex(reg) >= IR::NumScalarRegs) {
break;
}
entry_ir.SetScalarReg(reg, entry_ir.GetUserData(reg));
const auto value = entry_ir.GetUserData(reg);
entry_ir.SetScalarReg(reg, value);
entry_ir.SetThreadBitScalarReg(reg, entry_ir.INotEqual(value, IR::U32(IR::Value(0u))));
entry_ir.SetScalarMaskTag(reg, IR::U1(IR::Value(false)));
}
entry_ir.SetExec(IR::U1(IR::Value(true)));
if (source.stage == ShaderType::Compute && !per_invocation_masks) {
const auto active = entry_ir.Emit(IR::ValueOpcode::Ballot, {IR::Value(true)});
entry_ir.SetExecLo(entry_ir.CompositeExtract(active, 0));
entry_ir.SetExecHi(source.wave_size > 32u ? entry_ir.CompositeExtract(active, 1)
: IR::U32(IR::Value(0u)));
} else {
entry_ir.SetExecLo(IR::U32(IR::Value(per_invocation_masks ? 1u : 0xffffffffu)));
entry_ir.SetExecHi(IR::U32(
IR::Value(!per_invocation_masks && source.wave_size > 32u ? 0xffffffffu : 0u)));
}
entry_ir.SetExecLo(IR::U32(IR::Value(1u)));
entry_ir.SetExecHi(IR::U32(IR::Value(0u)));
if (source.stage == ShaderType::Compute) {
const auto* cs = compute_input_info;
const auto thread_ids =
@@ -930,8 +931,7 @@ bool TranslateProgram(const IR::Program& source, IR::ValueProgram& result,
}
}
for (size_t index = 0; index < source.blocks.size(); index++) {
Detail::Translator translator(result, result.blocks[index + 1u], vector_limit, wave_size,
per_invocation_masks);
Detail::Translator translator(result, result.blocks[index + 1u], vector_limit, wave_size);
if (!translator.TranslateBlock(source.blocks[index], error) ||
!translator.AddBranchCondition(source.blocks[index], result.block_info[index + 1u],
error)) {
@@ -10,9 +10,9 @@ namespace Libs::Graphics::ShaderRecompiler::Frontend::Detail {
class Translator {
public:
Translator(IR::ValueProgram& program, IR::Block* block, uint32_t vector_limit,
uint32_t wave_size, bool per_invocation_masks)
uint32_t wave_size)
: value_program(program), ir(block), current_vector_limit(vector_limit),
current_wave_size(wave_size), current_per_invocation_masks(per_invocation_masks) {}
current_wave_size(wave_size) {}
bool TranslateBlock(const IR::BasicBlock& source, std::string* error);
bool AddBranchCondition(const IR::BasicBlock& source, IR::ValueBlockInfo& info,
@@ -28,7 +28,7 @@ private:
IR::U32 ReadScalarCode(uint32_t code);
IR::U32 ApplyBitSourceModifiers(const IR::Operand& operand, IR::U32 value);
IR::Value ReadOperand(const IR::Operand& operand, IR::Type type);
IR::U1 ThreadBit(IR::U32 low, IR::U32 high);
IR::U1 ThreadBit(IR::U32 low);
void WriteRawU32(const IR::Operand& operand, IR::U32 value);
IR::F32 ApplyF32ResultModifiers(const IR::Operand& operand, IR::F32 value);
void WriteOperand(const IR::Operand& operand, IR::Value value);
@@ -38,11 +38,10 @@ private:
IR::U32 ReadU32(const IR::Operand& operand);
std::array<IR::U32, 2> ReadU32Pair(const IR::Operand& operand);
IR::U64 ReadU64(const IR::Operand& operand);
IR::F32 ReadF16LaneAsF32(const IR::Operand& operand, bool high_lane,
bool packed = false);
IR::F32 ReadF16AsF32(const IR::Operand& operand);
IR::F32 ReadMixF32(const IR::Operand& operand);
IR::U32 ReadU16LaneRaw(const IR::Operand& operand, bool high_lane);
IR::F32 ReadF16LaneAsF32(const IR::Operand& operand, bool high_lane, bool packed = false);
IR::F32 ReadF16AsF32(const IR::Operand& operand);
IR::F32 ReadMixF32(const IR::Operand& operand);
IR::U32 ReadU16LaneRaw(const IR::Operand& operand, bool high_lane);
IR::U32 ReadU16LaneAsU32(const IR::Operand& operand, bool high_lane, bool sign_extend);
IR::U32 ReadU16AsU32(const IR::Operand& operand, bool sign_extend);
IR::U32 ReadF16LaneBits(const IR::Operand& operand, bool high_lane);
@@ -51,6 +50,7 @@ private:
IR::U1 ReadCondition(const IR::Operand& operand);
IR::U32 ConditionBit(const IR::Operand& operand);
IR::U1 ReadMask(const IR::Operand& operand);
IR::U1 ReadMaskValid(const IR::Operand& operand);
void WriteMask(const IR::Operand& operand, IR::U1 value);
void WriteMask64(const IR::Operand& operand, IR::U1 value);
void WriteCompareResult(const IR::Operand& operand, IR::U1 value);
@@ -76,8 +76,8 @@ private:
IR::Value GetSamplerResource(const IR::MemoryInfo& memory);
IR::Value MakeImageAddress(const IR::Instruction& inst, const IR::Operand& base);
IR::Value ConstructU32x4(const IR::Operand& base, uint32_t count);
void WriteImageComponents(const IR::Operand& dst, IR::Value value,
const IR::MemoryInfo& memory, uint32_t component_limit);
void WriteImageComponents(const IR::Operand& dst, IR::Value value, const IR::MemoryInfo& memory,
uint32_t component_limit);
IR::ValueOpcode ImageAtomicOpcode(IR::Opcode opcode);
BufferAddress ReadBufferAddress(const IR::Instruction& inst, uint32_t source_offset);
IR::U32 WidenSubdword(IR::Value value, uint32_t bits, bool sign);
@@ -101,35 +101,34 @@ private:
uint32_t upper_result);
IR::U32 PackU16Lanes(IR::U32 low, IR::U32 high);
bool TranslateInstruction(const IR::Instruction& inst);
bool TranslateStateOperation(const IR::Instruction& inst);
bool TranslateControlOperation(const IR::Instruction& inst);
bool TranslateMove(const IR::Instruction& inst);
bool TranslateLaneOperation(const IR::Instruction& inst);
bool TranslateAttributeOperation(const IR::Instruction& inst);
bool TranslateMemoryOperation(const IR::Instruction& inst);
bool TranslateIntegerCompare(const IR::Instruction& inst);
bool TranslateInteger16Compare(const IR::Instruction& inst);
bool TranslateFloatCompare(const IR::Instruction& inst);
bool TranslateConversion(const IR::Instruction& inst);
bool TranslateInteger16Operation(const IR::Instruction& inst);
bool TranslatePackedInteger16(const IR::Instruction& inst);
bool TranslatePackedFloat16(const IR::Instruction& inst);
bool TranslateFloat16Operation(const IR::Instruction& inst);
bool TranslateFloatOperation(const IR::Instruction& inst);
bool TranslatePerInvocationU64Mask(const IR::Instruction& inst);
bool TranslateU64MaskOperation(const IR::Instruction& inst);
bool TranslateSimpleInteger(const IR::Instruction& inst);
bool TranslateComposedInteger(const IR::Instruction& inst);
bool TranslateExtendedInteger(const IR::Instruction& inst);
bool TranslateInstruction(const IR::Instruction& inst);
bool TranslateStateOperation(const IR::Instruction& inst);
bool TranslateControlOperation(const IR::Instruction& inst);
bool TranslateMove(const IR::Instruction& inst);
bool TranslateLaneOperation(const IR::Instruction& inst);
bool TranslateAttributeOperation(const IR::Instruction& inst);
bool TranslateMemoryOperation(const IR::Instruction& inst);
bool TranslateIntegerCompare(const IR::Instruction& inst);
bool TranslateInteger16Compare(const IR::Instruction& inst);
bool TranslateFloatCompare(const IR::Instruction& inst);
bool TranslateConversion(const IR::Instruction& inst);
bool TranslateInteger16Operation(const IR::Instruction& inst);
bool TranslatePackedInteger16(const IR::Instruction& inst);
bool TranslatePackedFloat16(const IR::Instruction& inst);
bool TranslateFloat16Operation(const IR::Instruction& inst);
bool TranslateFloatOperation(const IR::Instruction& inst);
IR::U1 EvaluateU64Mask(const IR::Instruction& inst);
bool TranslateU64MaskOperation(const IR::Instruction& inst);
bool TranslateSimpleInteger(const IR::Instruction& inst);
bool TranslateComposedInteger(const IR::Instruction& inst);
bool TranslateExtendedInteger(const IR::Instruction& inst);
IR::ValueProgram& value_program;
IR::IREmitter ir;
IR::Opcode current_opcode = IR::Opcode::ControlNop;
uint32_t current_pc = 0;
uint32_t current_vector_limit = 1;
uint32_t current_wave_size = 64;
bool current_per_invocation_masks = false;
IR::Opcode current_opcode = IR::Opcode::ControlNop;
uint32_t current_pc = 0;
uint32_t current_vector_limit = 1;
uint32_t current_wave_size = 64;
};
} // namespace Libs::Graphics::ShaderRecompiler::Frontend::Detail