shader: count one SGPR for lane reads and wave32 lane masks in call targets

Co-authored-by: Ali Almohaya <AliAlmohaya@gmail.com>
Co-authored-by: Senaxx <michelbeld@gmail.com>
Co-authored-by: Mac <me@itsmemac.com>
This commit is contained in:
nmzik
2026-10-25 04:31:34 +01:00
parent 153dadd0b3
commit 7524e8f79e
5 changed files with 33 additions and 12 deletions
@@ -467,6 +467,7 @@ Decoder::Program DecodeFusedProgram(std::span<const uint32_t> front, std::span<c
void PrepareCallTargets(ShaderSource& source, const CompileOptions& options) {
using namespace IR;
EXIT_NOT_IMPLEMENTED(options.stage != ShaderType::Compute);
source.wave_size = options.wave_size;
for (uint32_t i = 0; i < source.decoded.instructions.size(); ++i) {
if (source.decoded.instructions[i].opcode != Decoder::Opcode::S_SWAPPC_B64) continue;
source.calls.push_back({.instruction = i});
@@ -503,7 +504,8 @@ void PrepareCallTargets(ShaderSource& source, const CompileOptions& options) {
};
Register = [&](uint32_t before, uint32_t code) -> Value {
std::vector<uint32_t> crossed_calls;
const auto index = CFG::FindScalarDefinition(source.decoded, graph, before, code, crossed_calls);
const auto index = CFG::FindScalarDefinition(source.decoded, graph, before, code,
source.wave_size, crossed_calls);
for (const auto crossed: crossed_calls) {
std::ranges::find(source.calls, crossed, &ShaderSource::Call::instruction)->preserved_registers.set(code);
}
@@ -702,10 +704,11 @@ const Decoder::Program& RefreshShaderSource(ShaderSource& source, const IR::SrtR
EXIT_IF(function_pc > UINT32_MAX - function_size);
const auto function_offset = static_cast<uint32_t>(linked.code.size());
for (auto inst: function) {
EXIT_IF(CFG::MayWriteScalarRegister(inst, call.dst.reg) ||
CFG::MayWriteScalarRegister(inst, call.dst.reg + 1u));
EXIT_IF(CFG::MayWriteScalarRegister(inst, call.dst.reg, source.wave_size) ||
CFG::MayWriteScalarRegister(inst, call.dst.reg + 1u, source.wave_size));
for (uint32_t code = 0; code < site.preserved_registers.size(); ++code) {
if (site.preserved_registers[code] && CFG::MayWriteScalarRegister(inst, code))
if (site.preserved_registers[code] &&
CFG::MayWriteScalarRegister(inst, code, source.wave_size))
EXIT("shader call at pc 0x%08x clobbers scalar target dependency s%u", call.pc, code);
}
linked.code.insert(linked.code.end(), inst.raw, inst.raw + inst.word_count);
@@ -62,6 +62,7 @@ struct ShaderSource {
IR::ResourcePlan call_targets;
std::vector<Call> calls;
uint32_t unsupported_call_pc = UINT32_MAX;
uint32_t wave_size = 64;
std::optional<Linked> linked;
uint64_t revision = 0;
std::vector<std::pair<uint64_t, uint64_t>> reads;
@@ -1710,21 +1710,25 @@ Graph Structurize(const Graph& graph) {
return GotoStructurizer(graph).Run();
}
bool MayWriteScalarRegister(const Decoder::Instruction& inst, uint32_t code) {
// Vector comparisons and carry results can write a scalar pair in wave64.
bool MayWriteScalarRegister(const Decoder::Instruction& inst, uint32_t code, uint32_t wave_size) {
// A vector lane mask (comparison or carry) spans wave_size / 32 SGPRs; a lane read writes one.
if (inst.family == Decoder::Family::VOP1 || inst.family == Decoder::Family::VOP2 ||
inst.family == Decoder::Family::VOPC || inst.family == Decoder::Family::VOP3 ||
inst.family == Decoder::Family::VOP3P) {
const bool lane_read =
inst.opcode == Opcode::V_READLANE_B32 || inst.opcode == Opcode::V_READFIRSTLANE_B32;
const uint32_t size = lane_read ? 1u : wave_size / 32u;
uint32_t destination = 0;
for (const auto* operand: {&inst.dst, &inst.dst2})
if (ScalarOperandCode(*operand, destination) && code >= destination && code <= destination + 1u)
if (ScalarOperandCode(*operand, destination) && code >= destination && code < destination + size)
return true;
}
return InstructionWritesScalarCode(inst, code);
}
uint32_t FindScalarDefinition(const Decoder::Program& program, const Graph& graph,
uint32_t before, uint32_t code, std::vector<uint32_t>& crossed_calls) {
uint32_t before, uint32_t code, uint32_t wave_size,
std::vector<uint32_t>& crossed_calls) {
EXIT_IF(before >= program.instructions.size());
const auto found = std::ranges::find_if(graph.blocks, [before](const auto& block) {
return block.inst_begin <= before && before < block.inst_end;
@@ -1743,11 +1747,11 @@ uint32_t FindScalarDefinition(const Decoder::Program& program, const Graph& grap
for (uint32_t i = point.second; i > block->inst_begin;) {
const auto& inst = program.instructions[--i];
if (inst.opcode == Opcode::S_SWAPPC_B64) {
EXIT_NOT_IMPLEMENTED(MayWriteScalarRegister(inst, code));
EXIT_NOT_IMPLEMENTED(MayWriteScalarRegister(inst, code, wave_size));
crossed_calls.push_back(i);
continue;
}
if (MayWriteScalarRegister(inst, code)) { index = i; break; }
if (MayWriteScalarRegister(inst, code, wave_size)) { index = i; break; }
}
if (index != UINT32_MAX || block->id == graph.entry_block) {
EXIT_IF(definition && *definition != index);
@@ -113,9 +113,10 @@ struct Graph {
};
Graph BuildGraph(const Decoder::Program& program);
bool MayWriteScalarRegister(const Decoder::Instruction& inst, uint32_t code);
bool MayWriteScalarRegister(const Decoder::Instruction& inst, uint32_t code, uint32_t wave_size);
uint32_t FindScalarDefinition(const Decoder::Program& program, const Graph& graph,
uint32_t before, uint32_t code, std::vector<uint32_t>& crossed_calls);
uint32_t before, uint32_t code, uint32_t wave_size,
std::vector<uint32_t>& crossed_calls);
// Returns structured control flow or failure diagnostics without changing the native graph.
// On failure, failure_block is an original block ID or UINT32_MAX.
Graph Structurize(const Graph& graph);
+12
View File
@@ -11024,6 +11024,18 @@ void TestMemoryFedScalarLeafCall() {
ExpectFatal([&] { (void)PrepareShaderSource(multiple, options); },
"a native return pair was accepted as a later call target");
#endif
// A lane read and a wave32 lane mask write one SGPR, so neither s13 writer defines s14.
for (const auto& [wave_size, writer] : {std::pair{64u, 0x360u}, std::pair{32u, 0xc2u}}) {
const uint32_t neighbor[] = {EncodeVop3Word0(writer, 13), EncodeVop3Word1(256, 128, 0),
EncodeSop1(0x21, 14, 14), EncodeSopp(0x01)};
auto neighbor_options = options;
neighbor_options.wave_size = wave_size;
const auto neighbor_source = PrepareShaderSource(neighbor, neighbor_options);
const auto* target = neighbor_source.calls.front().target[0].TryInstruction();
Check(target != nullptr && target->GetOpcode() == IR::ValueOpcode::GetUserData &&
IR::RegIndex(target->Arg(0).ScalarRegister()) == 14u,
"a single-SGPR vector write was taken as the next SGPR's call-target definition");
}
}
void TestNewShaderRecompilerSetpcBranch() {