graphics: use per-invocation shader masks

This commit is contained in:
nmzik
2026-08-21 08:02:21 +02:00
parent 79443aa104
commit 49ea1680f8
30 changed files with 897 additions and 1140 deletions
@@ -49,6 +49,7 @@ public:
std::array<Value, NumScalarRegs> ssa_sreg_values {};
std::array<Value, NumScalarRegs> ssa_thread_bit_sreg_values {};
std::array<Value, NumScalarRegs> ssa_sreg_mask_tags {};
std::array<Value, NumVectorRegs> ssa_vreg_values {};
private:
@@ -36,6 +36,14 @@ void IREmitter::SetThreadBitScalarReg(ScalarReg reg, U1 value) {
Emit(ValueOpcode::SetThreadBitScalarRegister, {Value(reg), value});
}
U1 IREmitter::GetScalarMaskTag(ScalarReg reg) {
return U1(Emit(ValueOpcode::GetScalarMaskTag, {Value(reg)}));
}
void IREmitter::SetScalarMaskTag(ScalarReg reg, U1 value) {
Emit(ValueOpcode::SetScalarMaskTag, {Value(reg), value});
}
U32 IREmitter::GetVectorReg(VectorReg reg) {
return U32(Emit(ValueOpcode::GetVectorRegister, {Value(reg)}));
}
@@ -29,6 +29,8 @@ public:
void SetScalarReg(ScalarReg reg, U32 value);
U1 GetThreadBitScalarReg(ScalarReg reg);
void SetThreadBitScalarReg(ScalarReg reg, U1 value);
U1 GetScalarMaskTag(ScalarReg reg);
void SetScalarMaskTag(ScalarReg reg, U1 value);
U32 GetVectorReg(VectorReg reg);
void SetVectorReg(VectorReg reg, U32 value);
U1 GetGotoVariable(uint32_t id);
+28 -11
View File
@@ -44,6 +44,23 @@ bool AppendScalarResultSccNonZero(const Decoder::Instruction& decoded, BasicBloc
if (!ScalarResultWritesSccNonZero(decoded.opcode)) {
return true;
}
switch (decoded.opcode) {
case Decoder::Opcode::S_AND_B64:
case Decoder::Opcode::S_ANDN2_B64:
case Decoder::Opcode::S_NOT_B64:
case Decoder::Opcode::S_OR_B64:
case Decoder::Opcode::S_ORN2_B64:
case Decoder::Opcode::S_XOR_B64:
case Decoder::Opcode::S_NAND_B64:
case Decoder::Opcode::S_NOR_B64:
case Decoder::Opcode::S_XNOR_B64:
case Decoder::Opcode::S_WQM_B64:
// These instructions maintain an invocation-local mask shadow and update SCC
// while lowering the operation. A raw dword comparison would disagree after
// complementing a mask temporary.
return true;
default: break;
}
if (block.instructions.empty()) {
SetError(error, "internal IR error: missing scalar result for SCC update");
return false;
@@ -206,14 +223,14 @@ uint32_t ResourceIndexFromOperand(const Decoder::Operand& operand) {
MemoryInfo MemoryInfoFromDecoded(const Decoder::Instruction& decoded, ResourceKind kind) {
MemoryInfo mem;
mem.kind = kind;
mem.offset = decoded.offset;
mem.secondary_offset = decoded.secondary_offset;
mem.dmask = decoded.dmask;
mem.data_dwords = decoded.data_dwords;
mem.data_bits = decoded.data_bits;
mem.component_count = decoded.data_components != 0u ? decoded.data_components
: decoded.data_dwords;
mem.kind = kind;
mem.offset = decoded.offset;
mem.secondary_offset = decoded.secondary_offset;
mem.dmask = decoded.dmask;
mem.data_dwords = decoded.data_dwords;
mem.data_bits = decoded.data_bits;
mem.component_count =
decoded.data_components != 0u ? decoded.data_components : decoded.data_dwords;
mem.data_format = decoded.data_format;
mem.number_format = decoded.number_format;
mem.image_sample_flags = decoded.image_sample_flags;
@@ -224,9 +241,9 @@ MemoryInfo MemoryInfoFromDecoded(const Decoder::Instruction& decoded, ResourceKi
mem.image_nsa_addr[i] = decoded.image_nsa_addr[i];
}
mem.memory_segment = decoded.memory_segment;
mem.address_is_full = kind == ResourceKind::Flat ||
(kind == ResourceKind::Global &&
decoded.src1.kind == Decoder::OperandKind::Vgpr);
mem.address_is_full =
kind == ResourceKind::Flat ||
(kind == ResourceKind::Global && decoded.src1.kind == Decoder::OperandKind::Vgpr);
mem.data_signed = decoded.data_signed;
mem.typed = decoded.typed;
mem.formatted = decoded.formatted;
@@ -456,7 +456,6 @@ struct ShaderInfo {
};
struct SpirvRequirements {
bool requires_exact_subgroup = false;
bool subgroup_ballot = false;
bool subgroup_shuffle = false;
bool subgroup_local_invocation_id = false;
@@ -469,7 +468,6 @@ struct SpirvRequirements {
struct Program {
ShaderType stage = ShaderType::Unknown;
ShaderLaneMaskMode lane_mask_mode = ShaderLaneMaskMode::NativeWave;
uint64_t shader_hash = 0;
uint32_t wave_size = 64;
uint32_t user_data_base = 0;
@@ -192,10 +192,8 @@ std::string ExportTargetKindToString(ExportTargetKind kind) {
std::string ProgramToString(const Program& program) {
std::string text;
text += fmt::format("mode={} masks={} fallback={} reason={}\n",
text += fmt::format("mode={} fallback={} reason={}\n",
program.dispatcher_fallback ? "dispatcher" : "structured",
program.lane_mask_mode == ShaderLaneMaskMode::NativeWave ? "native-wave"
: "per-invocation",
CFG::FailureKindToString(program.cfg_failure_kind).c_str(),
program.fallback_reason.c_str());
for (const auto& block: program.blocks) {
@@ -17,6 +17,8 @@ bool IsRegisterStatePseudo(ValueOpcode opcode) {
switch (opcode) {
case ValueOpcode::GetThreadBitScalarRegister:
case ValueOpcode::SetThreadBitScalarRegister:
case ValueOpcode::GetScalarMaskTag:
case ValueOpcode::SetScalarMaskTag:
case ValueOpcode::GetScalarRegister:
case ValueOpcode::SetScalarRegister:
case ValueOpcode::GetVectorRegister:
@@ -11,6 +11,8 @@ VALUE_OPCODE(GetShaderBase, U64)
VALUE_OPCODE(GetBuiltin, U32, U32, U32)
VALUE_OPCODE(GetThreadBitScalarRegister, U1, ScalarReg)
VALUE_OPCODE(SetThreadBitScalarRegister, Void, ScalarReg, U1)
VALUE_OPCODE(GetScalarMaskTag, U1, ScalarReg)
VALUE_OPCODE(SetScalarMaskTag, Void, ScalarReg, U1)
VALUE_OPCODE(GetScalarRegister, U32, ScalarReg)
VALUE_OPCODE(SetScalarRegister, Void, ScalarReg, U32)
VALUE_OPCODE(GetVectorRegister, U32, VectorReg)
@@ -15,6 +15,10 @@ struct ThreadBitScalarReg {
ScalarReg reg {};
auto operator<=>(const ThreadBitScalarReg&) const = default;
};
struct ScalarMaskTag {
ScalarReg reg {};
auto operator<=>(const ScalarMaskTag&) const = default;
};
struct ExecTag {
auto operator<=>(const ExecTag&) const = default;
};
@@ -41,8 +45,9 @@ struct GotoVariable {
auto operator<=>(const GotoVariable&) const = default;
};
using Variable = std::variant<ScalarReg, ThreadBitScalarReg, VectorReg, GotoVariable, SccTag,
ExecTag, ExecLoTag, ExecHiTag, VccTag, VccLoTag, VccHiTag, M0Tag>;
using Variable =
std::variant<ScalarReg, ThreadBitScalarReg, ScalarMaskTag, VectorReg, GotoVariable, SccTag,
ExecTag, ExecLoTag, ExecHiTag, VccTag, VccLoTag, VccHiTag, M0Tag>;
using ValueMap = std::unordered_map<Block*, Value>;
struct DefTable {
@@ -66,6 +71,14 @@ struct DefTable {
EXIT_IF(RegIndex(value.reg) >= NumScalarRegs);
block->ssa_thread_bit_sreg_values[RegIndex(value.reg)] = definition;
}
const Value& Get(Block* block, ScalarMaskTag value) {
EXIT_IF(RegIndex(value.reg) >= NumScalarRegs);
return block->ssa_sreg_mask_tags[RegIndex(value.reg)];
}
void Set(Block* block, ScalarMaskTag value, Value definition) {
EXIT_IF(RegIndex(value.reg) >= NumScalarRegs);
block->ssa_sreg_mask_tags[RegIndex(value.reg)] = definition;
}
const Value& Get(Block* block, VectorReg reg) {
EXIT_IF(RegIndex(reg) >= NumVectorRegs);
@@ -116,6 +129,9 @@ ValueOpcode UndefOpcode(ScalarReg) {
ValueOpcode UndefOpcode(ThreadBitScalarReg) {
return ValueOpcode::UndefU1;
}
ValueOpcode UndefOpcode(ScalarMaskTag) {
return ValueOpcode::UndefU1;
}
ValueOpcode UndefOpcode(VectorReg) {
return ValueOpcode::UndefU32;
}
@@ -153,6 +169,9 @@ Value InitialValue(ScalarReg) {
Value InitialValue(ThreadBitScalarReg) {
return Value(false);
}
Value InitialValue(ScalarMaskTag) {
return Value(false);
}
Value InitialValue(VectorReg) {
return Value(0u);
}
@@ -321,6 +340,9 @@ void VisitInstruction(Pass& pass, Block* block, Inst& inst) {
case ValueOpcode::SetThreadBitScalarRegister:
pass.Write(ThreadBitScalarReg {inst.Arg(0).ScalarRegister()}, block, inst.Arg(1));
break;
case ValueOpcode::SetScalarMaskTag:
pass.Write(ScalarMaskTag {inst.Arg(0).ScalarRegister()}, block, inst.Arg(1));
break;
case ValueOpcode::SetVectorRegister:
pass.Write(inst.Arg(0).VectorRegister(), block, inst.Arg(1));
break;
@@ -342,6 +364,9 @@ void VisitInstruction(Pass& pass, Block* block, Inst& inst) {
inst.ReplaceUsesWith(
pass.Read(ThreadBitScalarReg {inst.Arg(0).ScalarRegister()}, block));
break;
case ValueOpcode::GetScalarMaskTag:
inst.ReplaceUsesWith(pass.Read(ScalarMaskTag {inst.Arg(0).ScalarRegister()}, block));
break;
case ValueOpcode::GetVectorRegister:
inst.ReplaceUsesWith(pass.Read(inst.Arg(0).VectorRegister(), block));
break;