raphiaura/KytyPS5
1
0
Fork 0
mirror of https://github.com/KytyPS5/KytyPS5.git synced 2026-10-19 18:36:53 +02:00
Code Issues Packages Projects Releases Wiki Activity

shader: link multiple compute helpers without synthetic return loops

Browse Source
This commit is contained in:
nmzik
2026-10-18 23:19:53 +02:00
parent b2d3ce6dfc
commit ab8566591f
8 changed files with 363 additions and 109 deletions
Download Patch File Download Diff File Expand all files Collapse all files
src/graphics/host_gpu/renderer/pipeline/pipelineCache.cpp
+2 -2
View File
@@ -422,7 +422,7 @@ struct PipelineCache::ProgramCache {
auto* current = entry == programs.end() ? &source : entry->second.call_source.get();
ShaderRecompiler::TranslateResult translated;
if (current != nullptr) {
if (current->call) {
if (!current->calls.empty()) {
if (entry == programs.end()) ShaderRecompiler::RefreshShaderSource(*current, runtime);
translated = ShaderRecompiler::TranslateProgram(current->linked->decoded, options);
translated.program.source_reads = current->reads;
@@ -433,7 +433,7 @@ struct PipelineCache::ProgramCache {
translated = ShaderRecompiler::TranslateProgram(params.code, options);
}
if (entry == programs.end()) {
auto retained = source.call ? std::make_unique<ShaderRecompiler::ShaderSource>(std::move(source)) : nullptr;
auto retained = !source.calls.empty() ? std::make_unique<ShaderRecompiler::ShaderSource>(std::move(source)) : nullptr;
entry = programs.try_emplace(lookup_key,
ShaderRecompiler::IR::ExtractResourcePlan(translated.program), std::move(retained)).first;
EXIT_IF(!ShaderRecompiler::IR::MaterializeResources(
src/graphics/shader/recompiler/ShaderRecompiler.cpp
+120 -80
View File
@@ -481,18 +481,14 @@ Decoder::Program DecodeFusedProgram(std::span<const uint32_t> front, std::span<c
return result;
}
void PrepareCallTarget(ShaderSource& source, const CompileOptions& options) {
void PrepareCallTargets(ShaderSource& source, const CompileOptions& options) {
using namespace IR;
EXIT_NOT_IMPLEMENTED(options.stage != ShaderType::Compute);
for (uint32_t i = 0; i < source.decoded.instructions.size(); ++i) {
if (source.decoded.instructions[i].opcode != Decoder::Opcode::S_SWAPPC_B64) continue;
EXIT_NOT_IMPLEMENTED(source.call.has_value() || options.stage != ShaderType::Compute);
source.call = ShaderSource::Call {.instruction = i};
source.calls.push_back({.instruction = i});
}
EXIT_IF(options.input_info.compute == nullptr);
const auto& call = source.decoded.instructions[source.call->instruction];
EXIT_NOT_IMPLEMENTED(call.src0.kind != Decoder::OperandKind::Sgpr ||
call.dst.kind != Decoder::OperandKind::Sgpr ||
call.src0.reg != call.dst.reg || call.dst.reg >= 105u);
const auto graph = CFG::BuildGraph(source.decoded);
Program query;
query.stage = options.stage;
@@ -524,7 +520,11 @@ void PrepareCallTarget(ShaderSource& source, const CompileOptions& options) {
return value;
};
Register = [&](uint32_t before, uint32_t code) -> Value {
const auto index = CFG::FindScalarDefinition(source.decoded, graph, before, code);
std::vector<uint32_t> crossed_calls;
const auto index = CFG::FindScalarDefinition(source.decoded, graph, before, code, crossed_calls);
for (const auto crossed: crossed_calls) {
std::ranges::find(source.calls, crossed, &ShaderSource::Call::instruction)->preserved_registers.set(code);
}
if (index == UINT32_MAX) {
EXIT_IF(code < query.user_data_base || code - query.user_data_base >= query.user_data_count);
return IREmitter(&block).GetUserData(static_cast<ScalarReg>(code));
@@ -557,18 +557,25 @@ void PrepareCallTarget(ShaderSource& source, const CompileOptions& options) {
EXIT_IF(!producer.ready || !producer.values.contains(code));
return producer.values.at(code).Resolve();
};
for (uint32_t i = 0; i < 2u; ++i) {
const auto value = Register(source.call->instruction, call.src0.reg + i);
EXIT_IF(!ValidateRuntimeValue(query, value, RuntimeValueType::Integer));
auto& root = query.value_storage.emplace_back(ValueOpcode::ReferenceU32);
root.SetArg(0, value);
source.call->target[i] = Value(&root);
for (auto& site: source.calls) {
const auto& call = source.decoded.instructions[site.instruction];
EXIT_NOT_IMPLEMENTED(call.src0.kind != Decoder::OperandKind::Sgpr ||
call.dst.kind != Decoder::OperandKind::Sgpr ||
call.src0.reg != call.dst.reg || call.dst.reg >= 105u);
for (uint32_t i = 0; i < 2u; ++i) {
const auto value = Register(site.instruction, call.src0.reg + i);
EXIT_IF(!ValidateRuntimeValue(query, value, RuntimeValueType::Integer));
auto& root = query.value_storage.emplace_back(ValueOpcode::ReferenceU32);
root.SetArg(0, value);
site.target[i] = Value(&root);
}
}
RewriteToSsa(query.blocks);
ConstantPropagationPass(query.blocks, query.wave_size);
RemoveIdentities(query);
EliminateDeadCode(query.blocks);
for (auto& value: source.call->target) value = value.Instruction()->Arg(0).Resolve();
for (auto& site: source.calls)
for (auto& value: site.target) value = value.Instruction()->Arg(0).Resolve();
query.value_storage.clear();
for (auto& inst: block) {
inst.SetParent(nullptr);
@@ -595,7 +602,7 @@ ShaderSource PrepareShaderSource(std::span<const uint32_t> code, const CompileOp
Decoder::DecodeProgram(code, source.decoded);
}
if (source.decoded.has_swap_pc) {
PrepareCallTarget(source, options);
PrepareCallTargets(source, options);
source.code.assign(code.begin(), code.end());
source.decoded.code = source.code;
}
@@ -608,83 +615,116 @@ const Decoder::Program& RefreshShaderSource(ShaderSource& source, const IR::SrtR
IR::SrtReadCapture capture(runtime, reads);
const auto clean = IR::CleanRuntime(capture.ObservedRuntime());
IR::SrtWalker walker(source.call_targets, clean);
uint32_t low = 0, high = 0;
const auto& call = source.decoded.instructions[source.call->instruction];
if (!walker.Evaluate(source.call->target[0], low) || !walker.Evaluate(source.call->target[1], high))
EXIT("shader call at pc 0x%08x has an unavailable scalar target", call.pc);
// SWAPPC reads the old aliased pair and ignores the target's low two bits.
const auto address = ((uint64_t {high} << 32u) | low) & ~uint64_t {3};
EXIT_IF(address == 0u);
for (auto& site: source.calls) {
uint32_t low = 0, high = 0;
if (!walker.Evaluate(site.target[0], low) || !walker.Evaluate(site.target[1], high))
EXIT("shader call at pc 0x%08x has an unavailable scalar target",
source.decoded.instructions[site.instruction].pc);
// SWAPPC reads the old aliased pair and ignores the target's low two bits.
const auto address = ((uint64_t {high} << 32u) | low) & ~uint64_t {3};
EXIT_IF(address == 0u);
site.address = address;
}
const auto query_reads = reads.size();
if (source.linked) {
auto& linked = *source.linked;
const auto function = std::span(linked.code).subspan(source.code.size());
if (clean.read_memory(clean.userdata, address, linked.observed_function) &&
std::ranges::equal(function, linked.observed_function)) {
return linked.decoded;
bool unchanged = true;
for (size_t i = 0; i < source.calls.size(); ++i) {
const auto& function = linked.functions[i];
const auto expected = std::span(linked.code).subspan(function.offset, function.size);
const auto preceding = std::span(source.calls).first(i);
const auto previous = std::ranges::find(preceding, source.calls[i].address, &ShaderSource::Call::address);
if (previous != preceding.end()) {
const auto& observed = linked.functions[previous - preceding.begin()];
if (std::ranges::equal(expected, std::span(linked.code).subspan(observed.offset, observed.size))) continue;
unchanged = false;
break;
}
linked.observed_function.resize(function.size);
if (!clean.read_memory(clean.userdata, source.calls[i].address, linked.observed_function) ||
!std::ranges::equal(expected, linked.observed_function)) {
unchanged = false;
break;
}
}
if (unchanged) return linked.decoded;
reads.resize(query_reads);
}
ShaderSource::Linked linked {.code = source.code,
.decoded = source.decoded};
ShaderSource::Linked linked {.code = source.code, .decoded = source.decoded};
const auto& last = source.decoded.instructions.back();
EXIT_NOT_IMPLEMENTED(last.opcode != Decoder::Opcode::S_ENDPGM);
// Keep the caller's data/footer bytes at their native offsets. The leaf's virtual CFG
// PCs follow decoded instructions; its physical code is independently read and owned.
const auto function_pc = last.pc + last.word_count * sizeof(uint32_t);
for (uint32_t index = 0;;) {
// The observed queue helper is a scalar leaf. Scalar encodings need at most one literal.
EXIT_IF(index >= 64u || address > UINT64_MAX - (uint64_t {index} + 2u) * sizeof(uint32_t));
std::array<uint32_t, 2> words {};
if (!clean.read_memory(clean.userdata, address + uint64_t {index} * sizeof(uint32_t),
std::span(words).first(1)))
EXIT("shader call at pc 0x%08x cannot read leaf code", call.pc);
const auto family = Decoder::GetInstructionFamily(words[0]);
EXIT_NOT_IMPLEMENTED(family != Decoder::Family::SOP1 && family != Decoder::Family::SOP2 &&
family != Decoder::Family::SOPC && family != Decoder::Family::SOPK &&
family != Decoder::Family::SOPP);
Decoder::Instruction inst;
Decoder::DecodeInstruction(words, 0u, inst);
if (inst.word_count == 2u) {
if (!clean.read_memory(clean.userdata, address + uint64_t {index + 1u} * sizeof(uint32_t),
std::span(words).last(1)))
EXIT("shader call at pc 0x%08x cannot read leaf literal", call.pc);
inst = {};
Decoder::DecodeInstruction(words, 0u, inst);
// Keep the caller's data/footer at native offsets. Expand leaf CFGs per call site
// before SSA, so return edges neither require a dispatcher nor merge resource roots.
uint32_t function_pc = last.pc + last.word_count * sizeof(uint32_t);
std::map<uint64_t, std::vector<Decoder::Instruction>> functions;
for (size_t i = 0; i < source.calls.size(); ++i) {
const auto& site = source.calls[i];
const auto& call = source.decoded.instructions[site.instruction];
const auto address = source.calls[i].address;
auto [entry, inserted] = functions.try_emplace(address);
auto& function = entry->second;
if (inserted) {
for (uint32_t index = 0;;) {
std::array<uint32_t, Decoder::MaxInstructionRawWords> words {};
EXIT_IF(index > UINT32_MAX / sizeof(uint32_t) - words.size() ||
address > UINT64_MAX - (uint64_t {index} + words.size()) * sizeof(uint32_t));
uint32_t count = 1;
Decoder::Instruction inst;
for (;;) {
if (!clean.read_memory(clean.userdata, address + uint64_t {index} * sizeof(uint32_t),
std::span(words).first(count)))
EXIT("shader call at pc 0x%08x cannot read leaf code", call.pc);
inst = {};
Decoder::DecodeInstruction(words, 0u, inst);
if (inst.word_count <= count) break;
count = inst.word_count;
}
inst.pc = index * sizeof(uint32_t);
if (Decoder::IsDirectBranch(inst.opcode)) inst.branch_target += inst.pc;
index += inst.word_count;
EXIT_NOT_IMPLEMENTED(inst.opcode == Decoder::Opcode::S_SWAPPC_B64 ||
inst.opcode == Decoder::Opcode::S_GETPC_B64 ||
inst.opcode == Decoder::Opcode::S_ENDPGM);
function.push_back(inst);
if (inst.opcode == Decoder::Opcode::S_SETPC_B64) break;
}
}
inst.pc = index * sizeof(uint32_t);
if (Decoder::IsDirectBranch(inst.opcode)) inst.branch_target += inst.pc;
linked.code.insert(linked.code.end(), inst.raw, inst.raw + inst.word_count);
index += inst.word_count;
EXIT_NOT_IMPLEMENTED(inst.opcode == Decoder::Opcode::S_SWAPPC_B64 ||
inst.opcode == Decoder::Opcode::S_GETPC_B64 ||
inst.opcode == Decoder::Opcode::S_ENDPGM);
EXIT_IF(CFG::MayWriteScalarRegister(inst, call.dst.reg) || CFG::MayWriteScalarRegister(inst, call.dst.reg + 1u));
const bool returns = inst.opcode == Decoder::Opcode::S_SETPC_B64;
if (returns) {
EXIT_NOT_IMPLEMENTED(inst.src0.kind != Decoder::OperandKind::Sgpr || inst.src0.reg != call.dst.reg);
inst.opcode = Decoder::Opcode::S_BRANCH;
inst.branch_target = call.pc + call.word_count * sizeof(uint32_t);
} else if (Decoder::IsDirectBranch(inst.opcode)) {
EXIT_IF(inst.branch_target > UINT32_MAX - function_pc);
inst.branch_target += function_pc;
const uint32_t function_size = function.back().pc + function.back().word_count * sizeof(uint32_t);
EXIT_IF(function_pc > UINT32_MAX - function_size);
const auto function_offset = static_cast<uint32_t>(linked.code.size());
for (auto inst: function) {
EXIT_IF(CFG::MayWriteScalarRegister(inst, call.dst.reg) ||
CFG::MayWriteScalarRegister(inst, call.dst.reg + 1u));
for (uint32_t code = 0; code < site.preserved_registers.size(); ++code) {
if (site.preserved_registers[code] && CFG::MayWriteScalarRegister(inst, code))
EXIT("shader call at pc 0x%08x clobbers scalar target dependency s%u", call.pc, code);
}
linked.code.insert(linked.code.end(), inst.raw, inst.raw + inst.word_count);
if (inst.opcode == Decoder::Opcode::S_SETPC_B64) {
EXIT_NOT_IMPLEMENTED(inst.src0.kind != Decoder::OperandKind::Sgpr || inst.src0.reg != call.dst.reg);
inst.opcode = Decoder::Opcode::S_BRANCH;
inst.branch_target = call.pc + call.word_count * sizeof(uint32_t);
} else if (Decoder::IsDirectBranch(inst.opcode)) {
EXIT_IF(inst.branch_target >= function_size);
inst.branch_target += function_pc;
}
inst.pc += function_pc;
linked.decoded.instructions.push_back(std::move(inst));
}
inst.pc += function_pc;
linked.decoded.instructions.push_back(std::move(inst));
if (returns) break;
}
const auto function_end = linked.decoded.instructions.back().pc +
linked.decoded.instructions.back().word_count * sizeof(uint32_t);
for (size_t i = source.decoded.instructions.size(); i + 1u < linked.decoded.instructions.size(); ++i) {
const auto& inst = linked.decoded.instructions[i];
if (Decoder::IsDirectBranch(inst.opcode))
EXIT_IF(inst.branch_target < function_pc || inst.branch_target >= function_end);
linked.functions.push_back({function_offset, function_size / 4u});
linked.decoded.instructions[site.instruction].branch_target = function_pc;
function_pc += function_size;
}
reads.resize(query_reads);
reads.emplace_back(address, (linked.code.size() - source.code.size()) * sizeof(uint32_t));
linked.decoded.instructions[source.call->instruction].branch_target = function_pc;
for (size_t i = 0; i < source.calls.size(); ++i) {
const auto preceding = std::span(source.calls).first(i);
if (std::ranges::find(preceding, source.calls[i].address, &ShaderSource::Call::address) == preceding.end())
reads.emplace_back(source.calls[i].address, linked.functions[i].size * sizeof(uint32_t));
}
linked.decoded.code = linked.code;
linked.observed_function.resize(linked.code.size() - source.code.size());
linked.observed_function.resize(std::ranges::max_element(linked.functions, {},
&ShaderSource::Linked::Function::size)->size);
source.linked = std::move(linked);
++source.revision;
return source.linked->decoded;
src/graphics/shader/recompiler/ShaderRecompiler.h
+6 -1
View File
@@ -8,6 +8,7 @@
#include "graphics/shader/shader.h"
#include <array>
#include <bitset>
#include <optional>
#include <span>
#include <vector>
@@ -47,16 +48,20 @@ struct ShaderSource {
struct Call {
uint32_t instruction;
std::array<IR::Value, 2> target;
std::bitset<128> preserved_registers;
uint64_t address = 0;
};
struct Linked {
struct Function { uint32_t offset; uint32_t size; };
std::vector<uint32_t> code;
Decoder::Program decoded;
std::vector<Function> functions;
std::vector<uint32_t> observed_function;
};
std::vector<uint32_t> code;
Decoder::Program decoded;
IR::ResourcePlan call_targets;
std::optional<Call> call;
std::vector<Call> calls;
std::optional<Linked> linked;
uint64_t revision = 0;
std::vector<std::pair<uint64_t, uint64_t>> reads;
src/graphics/shader/recompiler/frontend/cfg/ShaderCFG.cpp
+68 -17
View File
@@ -8,6 +8,8 @@
#include <deque>
#include <list>
#include <map>
#include <optional>
#include <set>
#include <span>
namespace Libs::Graphics::ShaderRecompiler::CFG {
@@ -1650,6 +1652,32 @@ Graph BuildGraph(const Decoder::Program& program) {
}
SortUnique(graph.code_table_load_pcs);
if (!indirect_setpc && std::ranges::any_of(program.instructions, [](const auto& inst) {
return inst.opcode == Opcode::S_SWAPPC_B64 && inst.branch_target != UINT32_MAX;
})) {
// Linked leaf bodies follow the caller in native PC order. Visit them in
// execution order so returning to the caller does not manufacture goto loops.
std::vector<bool> visited(graph.blocks.size(), false);
std::vector<std::pair<uint32_t, bool>> pending {{graph.entry_block, false}};
std::vector<BasicBlock> ordered;
ordered.reserve(graph.blocks.size());
while (!pending.empty()) {
const auto [id, finished] = pending.back();
pending.pop_back();
if (finished) {
ordered.push_back(std::move(graph.blocks[id]));
continue;
}
if (visited[id]) continue;
visited[id] = true;
pending.emplace_back(id, true);
for (const auto successor: graph.blocks[id].successors)
pending.emplace_back(successor, false);
}
std::ranges::reverse(ordered);
ApplyBlockOrder(graph, std::move(ordered));
}
RecomputeAnalyses(graph);
if (indirect_setpc) {
@@ -1681,33 +1709,56 @@ Graph Structurize(const Graph& graph) {
}
bool MayWriteScalarRegister(const Decoder::Instruction& inst, uint32_t code) {
uint32_t destination = UINT32_MAX;
// Scalar ALU pair widths are not all decoded. The typed producer must confirm this word.
return InstructionWritesScalarCode(inst, code) ||
(ScalarOperandCode(inst.dst, destination) && code == destination + 1u);
// Vector comparisons and carry results can write a scalar pair in wave64.
if (inst.family == Decoder::Family::VOP1 || inst.family == Decoder::Family::VOP2 ||
inst.family == Decoder::Family::VOPC || inst.family == Decoder::Family::VOP3 ||
inst.family == Decoder::Family::VOP3P) {
uint32_t destination = 0;
for (const auto* operand: {&inst.dst, &inst.dst2})
if (ScalarOperandCode(*operand, destination) && code >= destination && code <= destination + 1u)
return true;
}
return InstructionWritesScalarCode(inst, code);
}
uint32_t FindScalarDefinition(const Decoder::Program& program, const Graph& graph,
uint32_t before, uint32_t code) {
uint32_t before, uint32_t code, std::vector<uint32_t>& crossed_calls) {
EXIT_IF(before >= program.instructions.size());
const auto found = std::ranges::find_if(graph.blocks, [before](const auto& block) {
return block.inst_begin <= before && before < block.inst_end;
});
EXIT_IF(found == graph.blocks.end());
const auto* block = &*found;
for (size_t depth = 0; depth < graph.blocks.size(); ++depth) {
for (uint32_t i = before; i > block->inst_begin;) {
const auto& inst = program.instructions[--i];
EXIT_IF(inst.opcode == Opcode::S_SWAPPC_B64);
if (MayWriteScalarRegister(inst, code)) return i;
}
if (block->id == graph.entry_block) return UINT32_MAX;
EXIT_IF(block->predecessors.size() != 1u);
block = graph.FindBlock(block->predecessors[0]);
std::vector<std::pair<uint32_t, uint32_t>> pending {{found->id, before}};
std::set<std::pair<uint32_t, uint32_t>> visited;
std::optional<uint32_t> definition;
while (!pending.empty()) {
const auto point = pending.back();
pending.pop_back();
if (!visited.insert(point).second) continue;
const auto* block = graph.FindBlock(point.first);
EXIT_IF(block == nullptr);
before = block->inst_end;
uint32_t index = UINT32_MAX;
for (uint32_t i = point.second; i > block->inst_begin;) {
const auto& inst = program.instructions[--i];
if (inst.opcode == Opcode::S_SWAPPC_B64) {
EXIT_NOT_IMPLEMENTED(MayWriteScalarRegister(inst, code));
crossed_calls.push_back(i);
continue;
}
if (MayWriteScalarRegister(inst, code)) { index = i; break; }
}
if (index != UINT32_MAX || block->id == graph.entry_block) {
EXIT_IF(definition && *definition != index);
definition = index;
continue;
}
EXIT_IF(block->predecessors.empty());
for (const auto predecessor: block->predecessors) {
pending.emplace_back(predecessor, graph.blocks[predecessor].inst_end);
}
}
EXIT("scalar shader call source has cyclic reaching definitions");
EXIT_IF(!definition);
return *definition;
}
std::string BranchConditionToString(BranchCondition condition) {
src/graphics/shader/recompiler/frontend/cfg/ShaderCFG.h
+1 -1
View File
@@ -115,7 +115,7 @@ struct Graph {
Graph BuildGraph(const Decoder::Program& program);
bool MayWriteScalarRegister(const Decoder::Instruction& inst, uint32_t code);
uint32_t FindScalarDefinition(const Decoder::Program& program, const Graph& graph,
uint32_t before, uint32_t code);
uint32_t before, uint32_t code, std::vector<uint32_t>& crossed_calls);
// Returns structured control flow or failure diagnostics without changing the native graph.
// On failure, failure_block is an original block ID or UINT32_MAX.
Graph Structurize(const Graph& graph);
src/graphics/shader/recompiler/frontend/decode/ScalarAluOps.cpp
+17 -1
View File
@@ -164,9 +164,23 @@ void DecodeSop1(uint32_t pc, std::span<const uint32_t> code, uint32_t word_index
switch (inst.opcode) {
case Opcode::S_SWAPPC_B64:
inst.branch_target = UINT32_MAX;
inst.data_dwords = 2;
[[fallthrough]];
case Opcode::S_MOV_B64:
case Opcode::S_CMOV_B64:
case Opcode::S_NOT_B64:
case Opcode::S_WQM_B64:
case Opcode::S_BREV_B64:
case Opcode::S_BITSET0_B64:
case Opcode::S_BITSET1_B64:
case Opcode::S_AND_SAVEEXEC_B64:
case Opcode::S_ORN2_SAVEEXEC_B64:
case Opcode::S_QUADMASK_B64:
case Opcode::S_ANDN1_SAVEEXEC_B64:
case Opcode::S_BITREPLICATE_B64_B32:
inst.data_dwords = 2u;
break;
case Opcode::S_GETPC_B64:
inst.data_dwords = 2u;
inst.src_count = 0;
DecodeScalarDestination(sdst, pc, inst.dst);
return;
@@ -206,6 +220,8 @@ void DecodeSop2(uint32_t pc, std::span<const uint32_t> code, uint32_t word_index
DecodeScalarDestination(sdst, pc, inst.dst);
DecodeBinarySources(pc, code, word_index, inst, ssrc0, ssrc1);
if (opcode == 0x0bu || (opcode >= 0x0fu && opcode <= 0x25u && (opcode & 1u)) || opcode == 0x29u)
inst.data_dwords = 2u;
}
void DecodeSopk(uint32_t pc, std::span<const uint32_t> code, uint32_t word_index,
tests/ShaderRecompilerComputeTests.cpp
+142
View File
@@ -10734,6 +10734,143 @@ public:
LibKernel::Memory::TryReadBacking(output, &actual, sizeof(actual)) && actual == test.value,
"caller did not observe the leaf's scalar result after return");
}
// Each helper reads through its context, masks EXEC, then restores all
// caller lanes. The MAD literal occupies a third instruction DWORD.
std::array<std::vector<u32>, 2> vector_leaves;
for (u32 i = 0; i < vector_leaves.size(); ++i) {
auto& helper = vector_leaves[i];
helper = {EncodeSop1(0x04, 18, 126), EncodeVopc(0xd4, 6, 0), 0,
EncodeSmem0(0x02, 20, 8), EncodeSmem1(0, 125), EncodeSopp(0x0c, 0),
EncodeMubuf0(0x0c), EncodeMubuf1(2, 5, 1), EncodeSopp(0x0c, 0),
EncodeVop2(0x03, 2, 7, 2)};
AppendVop3(&helper, 0x141, 6, 255, Vgpr(2), Vgpr(6));
helper.push_back(std::bit_cast<u32>(i == 0 ? 1.5f : 2.5f));
helper[2] = EncodeSopp(0x08, static_cast<u32>(helper.size()) - 3u);
helper.push_back(EncodeSop1(0x04, 126, 18));
helper.push_back(EncodeSop1(0x21, 125, 14));
const auto address = i == 0 ? first : second;
Require(name, "replace vector helper", context.InvalidateMemory(address, helper.size() * 4),
"vector helper escaped mapped memory");
LibKernel::Memory::WriteBacking(address, helper.data(), helper.size() * 4);
}
out.fields[3] |= 3u << 28u;
ShaderBufferResource data = out;
data.UpdateAddress48(base + 0x5000);
data.fields[2] = 64 * sizeof(float);
Require(name, "refresh helper context", context.InvalidateMemory(table, sizeof(data.fields)),
"helper context escaped mapped memory");
LibKernel::Memory::WriteBacking(table, data.fields, sizeof(data.fields));
std::array<float, 64> values;
for (u32 lane = 0; lane < values.size(); ++lane) values[lane] = float(lane + 1);
Require(name, "refresh vector input", context.InvalidateMemory(base + 0x5000, sizeof(values)),
"vector input escaped mapped memory");
LibKernel::Memory::WriteBacking(base + 0x5000, values.data(), sizeof(values));
for (const u32 wave_size : {32u, 64u}) {
for (const u32 call_count : {1u, 13u}) {
std::vector<u32> caller{EncodeVop2(0x1a, 1, InlineU32(2), 0)};
AppendVMovLiteral(&caller, 6, std::bit_cast<u32>(10.0f));
for (u32 call = 0; call < call_count; ++call) {
AppendSMovLiteral(&caller, 7, std::bit_cast<u32>(float(call + 1)));
caller.insert(caller.end(), {
EncodeSMovB32(14, call < 8 ? 8 : 12), EncodeSMovB32(15, call < 8 ? 9 : 13),
EncodeSMovB32(16, 4), EncodeSMovB32(17, 5), EncodeSop1(0x21, 14, 14)});
}
caller.insert(caller.end(), {EncodeMubuf0(0x1c), EncodeMubuf1(6, 0, 1)});
AppendEnd(&caller);
ShaderMapUserData(reinterpret_cast<uint64_t>(caller.data()),
{.type = Prospero::ShaderBinaryType::kCs,
.code_size_bytes = static_cast<u32>(caller.size() * sizeof(u32))});
shaders.SetCsShader({.data_addr = reinterpret_cast<uint64_t>(caller.data()),
.num_thread_x = wave_size, .num_thread_y = 1, .num_thread_z = 1,
.wave_size = static_cast<uint8_t>(wave_size), .user_sgpr = 14});
out.fields[2] = wave_size * sizeof(float);
std::array<u32, 16> user_data{};
std::copy_n(out.fields, 4, user_data.begin());
user_data[4] = static_cast<u32>(table);
user_data[5] = static_cast<u32>(table >> 32u);
user_data[8] = static_cast<u32>(first | 3u);
user_data[9] = static_cast<u32>(first >> 32u);
user_data[12] = static_cast<u32>(second | 3u);
user_data[13] = static_cast<u32>(second >> 32u);
uint64_t original_vector_id = 0;
for (u32 phase = 0; phase < (call_count == 1 ? 2u : 4u); ++phase) {
const u32 active_lanes = phase == 0 ? 0 : 16;
if (phase >= 2) {
std::swap(user_data[8], user_data[12]);
std::swap(user_data[9], user_data[13]);
}
user_data[6] = active_lanes;
for (u32 i = 0; i < user_data.size(); ++i)
shaders.SetCsUserSgpr(i, user_data[i], HW::UserSgprType::Unknown);
Require(name, "reset caller output", context.InvalidateMemory(output, wave_size * 4),
"caller output escaped mapped memory");
std::array<float, 64> actual{};
LibKernel::Memory::WriteBacking(output, actual.data(), wave_size * 4);
ShaderComputeInputInfo input{};
const auto program = context.GetPipelineCache().GetComputeProgram(
shaders.GetCs(), processor.GetCtx().GetShaderRegisters(), input);
if (phase == 1) original_vector_id = program.id;
if (phase >= 2)
Require(name, "ordered helper identity", (program.id == original_vector_id) == (phase == 3),
"swapped helper targets reused stale code or restored targets recompiled");
const auto& reads = input.stage.resources->specialization_reads;
for (u32 i = 0; i < (call_count == 1 ? 1u : 2u); ++i)
Require(name, "unique helper code dependency", std::ranges::count(reads,
std::pair<uint64_t, uint64_t>{i == 0 ? first : second,
vector_leaves[i].size() * sizeof(u32)}) == 1,
"repeated calls duplicated or omitted a helper code dependency");
if (phase == 1) {
ShaderRecompiler::CompileOptions options;
options.wave_size = wave_size;
options.dump_ir = false;
options.maintenance8_enabled = true;
input.host_subgroup_size = SubgroupSize();
options.input_info.compute = &input;
options.user_data = user_data;
const auto read = +[](void*, uint64_t address, std::span<u32> words) {
return LibKernel::Memory::TryReadBufferBacking(address, words.data(), words.size_bytes());
};
const ShaderRecompiler::IR::SrtRuntime runtime{
.user_data = user_data, .shader_base = reinterpret_cast<uint64_t>(caller.data()),
.read_memory = read, .read_specialization_memory = read};
auto source = ShaderRecompiler::PrepareShaderSource(caller, options);
auto translated = ShaderRecompiler::TranslateProgram(
ShaderRecompiler::RefreshShaderSource(source, runtime), options);
auto plan = ShaderRecompiler::IR::ExtractResourcePlan(translated.program);
ShaderRecompiler::IR::ResourceSnapshot snapshot;
ShaderRecompiler::IR::ResourceSpecialization specialization;
Require(name, "vector helper resources", ShaderRecompiler::IR::MaterializeResources(
plan, runtime, snapshot, specialization), "helper V# could not be materialized");
auto compiled = ShaderRecompiler::CompileProgram(std::move(translated), options, specialization);
ValidateSpirv(name, compiled.spirv);
CheckSpirvEmission({.name = name, .forbidden_spirv = {"OpLoopMerge"}}, compiled.spirv);
const u32 word_budget = wave_size == 32 ? 640 + 224 * (call_count - 1) :
1024 + 416 * (call_count - 1);
Require(name, "linear call emission", compiled.spirv.size() <= word_budget,
"acyclic calls exceeded their per-call SPIR-V budget");
std::printf("[gpu] %-32s wave%u calls%u: %zu SPIR-V words\n",
name, wave_size, call_count, compiled.spirv.size());
}
processor.DispatchDirect(1, 1, 1, 0x41u |
(wave_size == 32u ? 1u << Pm4::COMPUTE_DISPATCH_INITIATOR_CS_W32_EN_SHIFT : 0u));
cache.ReadMemory(output, wave_size * 4);
Require(name, "read vector helper output", LibKernel::Memory::TryReadBacking(
output, actual.data(), wave_size * 4), "caller output was not readable");
for (u32 lane = 0; lane < wave_size; ++lane) {
float expected = 10.0f;
if (lane < active_lanes)
for (u32 call = 0; call < call_count; ++call)
expected += ((call < 8) != (phase == 2) ? 1.5f : 2.5f) *
(values[lane] + float(call + 1));
Require(name, "repeated vector calls and EXEC return", actual[lane] == expected,
"wave" + std::to_string(wave_size) + " calls" + std::to_string(call_count) +
" phase" + std::to_string(phase) + " lane" + std::to_string(lane) +
" actual=" + std::to_string(actual[lane]) + " expected=" + std::to_string(expected));
}
}
}
}
context.UnmapMemory(base, size);
context.GetCommandScheduler().Finish();
});
@@ -49010,6 +49147,11 @@ int main(int argc, char **argv) {
vulkan.CheckGpuTilerCpuParity();
return 0;
}
if (argc == 2 && std::strcmp(argv[1], "--scalar-leaf-call-only") == 0) {
VulkanHarness vulkan;
vulkan.CheckScalarLeafCall();
return 0;
}
if (argc == 2 && std::strcmp(argv[1], "--gpu-command-lane-only") == 0) {
VulkanHarness vulkan;
vulkan.CheckGpuCommandLane();
tests/shaderCfgTests.cpp
+7 -7
View File
@@ -10805,8 +10805,8 @@ void TestMemoryFedScalarLeafCall() {
IR::SrtRuntime runtime{.read_memory = read, .userdata = &memory, .read_specialization_memory = read};
const auto options = MakeCompileOptions(ShaderType::Compute);
auto source = PrepareShaderSource(shader, options);
Check(source.call && source.decoded.has_swap_pc &&
source.decoded.instructions[source.call->instruction].branch_target == UINT32_MAX,
Check(source.calls.size() == 1 && source.decoded.has_swap_pc &&
source.decoded.instructions[source.calls.front().instruction].branch_target == UINT32_MAX,
"memory-fed SWAPPC did not retain an unresolved target");
#if KYTY_PLATFORM != KYTY_PLATFORM_WINDOWS
ExpectFatal([&] { (void)TranslateProgram(source.decoded, options); },
@@ -10814,7 +10814,7 @@ void TestMemoryFedScalarLeafCall() {
#endif
const auto& linked = RefreshShaderSource(source, runtime);
Check(linked.has_swap_pc && source.revision == 1 && linked.instructions.back().branch_target == 0x40 &&
linked.instructions[source.call->instruction].branch_target == 0x44 &&
linked.instructions[source.calls.front().instruction].branch_target == 0x44 &&
source.reads.back() == std::pair<uint64_t, uint64_t>{base + 0x100, 12},
"aliased call target, exact leaf range, or continuation was lost");
const auto* storage = linked.instructions.data();
@@ -10899,14 +10899,14 @@ void TestMemoryFedScalarLeafCall() {
memory.words[96] = EncodeSMovB32(14, 129);
ExpectFatal([&] { (void)RefreshShaderSource(source, runtime); },
"leaf clobbering its return pair was linked");
// AND_B64 writes the high half despite the decoder's generic scalar width.
// AND_B64 must replace the preceding high-half definition.
const uint32_t high_writer[] = {
EncodeSMovB32(15, 129), EncodeSop2(0x0f, 14, 128, 128),
EncodeSop1(0x21, 14, 14), EncodeSopp(0x01)};
auto pair_source = PrepareShaderSource(high_writer, options);
IR::SrtWalker pair_walker(pair_source.call_targets, runtime);
uint32_t high = 1;
Check(pair_walker.Evaluate(pair_source.call->target[1], high) && high == 0,
Check(pair_walker.Evaluate(pair_source.calls.front().target[1], high) && high == 0,
"scalar pair producer used a stale preceding high-half definition");
const uint32_t joined[] = {
EncodeSopp(0x04, 2), EncodeSMovB32(14, 129), EncodeSopp(0x02, 1),
@@ -10915,7 +10915,7 @@ void TestMemoryFedScalarLeafCall() {
"ambiguous reaching definitions were accepted for a call target");
const uint32_t multiple[] = {EncodeSop1(0x21, 14, 14), EncodeSop1(0x21, 14, 14), EncodeSopp(0x01)};
ExpectFatal([&] { (void)PrepareShaderSource(multiple, options); },
"call metadata bypassed the multiple-call rejection");
"a native return pair was accepted as a later call target");
#endif
}
@@ -10962,7 +10962,7 @@ void TestFusedShaderHandoffPreservesRegisters() {
for (const auto handoff: {EncodeSop1(0x20, 0, 6), 0xbefd2106u}) {
front[2] = handoff; // SETPC or captured SWAPPC with NULL destination
auto source = PrepareShaderSource(front, options);
Check(!source.decoded.has_swap_pc && !source.call,
Check(!source.decoded.has_swap_pc && source.calls.empty(),
"fused SETPC handoff entered scalar call preparation");
auto translated = TranslateProgram(source.decoded, options);
uint32_t allocations = 0;