diff --git a/src/graphics/host_gpu/renderer/pipeline/pipelineCache.cpp b/src/graphics/host_gpu/renderer/pipeline/pipelineCache.cpp index f93bef4ab..97618059a 100644 --- a/src/graphics/host_gpu/renderer/pipeline/pipelineCache.cpp +++ b/src/graphics/host_gpu/renderer/pipeline/pipelineCache.cpp @@ -422,7 +422,7 @@ struct PipelineCache::ProgramCache { auto* current = entry == programs.end() ? &source : entry->second.call_source.get(); ShaderRecompiler::TranslateResult translated; if (current != nullptr) { - if (current->call) { + if (!current->calls.empty()) { if (entry == programs.end()) ShaderRecompiler::RefreshShaderSource(*current, runtime); translated = ShaderRecompiler::TranslateProgram(current->linked->decoded, options); translated.program.source_reads = current->reads; @@ -433,7 +433,7 @@ struct PipelineCache::ProgramCache { translated = ShaderRecompiler::TranslateProgram(params.code, options); } if (entry == programs.end()) { - auto retained = source.call ? std::make_unique(std::move(source)) : nullptr; + auto retained = !source.calls.empty() ? std::make_unique(std::move(source)) : nullptr; entry = programs.try_emplace(lookup_key, ShaderRecompiler::IR::ExtractResourcePlan(translated.program), std::move(retained)).first; EXIT_IF(!ShaderRecompiler::IR::MaterializeResources( diff --git a/src/graphics/shader/recompiler/ShaderRecompiler.cpp b/src/graphics/shader/recompiler/ShaderRecompiler.cpp index 757ba9fcc..632bc9025 100644 --- a/src/graphics/shader/recompiler/ShaderRecompiler.cpp +++ b/src/graphics/shader/recompiler/ShaderRecompiler.cpp @@ -481,18 +481,14 @@ Decoder::Program DecodeFusedProgram(std::span front, std::spaninstruction]; - EXIT_NOT_IMPLEMENTED(call.src0.kind != Decoder::OperandKind::Sgpr || - call.dst.kind != Decoder::OperandKind::Sgpr || - call.src0.reg != call.dst.reg || call.dst.reg >= 105u); const auto graph = CFG::BuildGraph(source.decoded); Program query; query.stage = options.stage; @@ -524,7 +520,11 @@ void PrepareCallTarget(ShaderSource& source, const CompileOptions& options) { return value; }; Register = [&](uint32_t before, uint32_t code) -> Value { - const auto index = CFG::FindScalarDefinition(source.decoded, graph, before, code); + std::vector crossed_calls; + const auto index = CFG::FindScalarDefinition(source.decoded, graph, before, code, crossed_calls); + for (const auto crossed: crossed_calls) { + std::ranges::find(source.calls, crossed, &ShaderSource::Call::instruction)->preserved_registers.set(code); + } if (index == UINT32_MAX) { EXIT_IF(code < query.user_data_base || code - query.user_data_base >= query.user_data_count); return IREmitter(&block).GetUserData(static_cast(code)); @@ -557,18 +557,25 @@ void PrepareCallTarget(ShaderSource& source, const CompileOptions& options) { EXIT_IF(!producer.ready || !producer.values.contains(code)); return producer.values.at(code).Resolve(); }; - for (uint32_t i = 0; i < 2u; ++i) { - const auto value = Register(source.call->instruction, call.src0.reg + i); - EXIT_IF(!ValidateRuntimeValue(query, value, RuntimeValueType::Integer)); - auto& root = query.value_storage.emplace_back(ValueOpcode::ReferenceU32); - root.SetArg(0, value); - source.call->target[i] = Value(&root); + for (auto& site: source.calls) { + const auto& call = source.decoded.instructions[site.instruction]; + EXIT_NOT_IMPLEMENTED(call.src0.kind != Decoder::OperandKind::Sgpr || + call.dst.kind != Decoder::OperandKind::Sgpr || + call.src0.reg != call.dst.reg || call.dst.reg >= 105u); + for (uint32_t i = 0; i < 2u; ++i) { + const auto value = Register(site.instruction, call.src0.reg + i); + EXIT_IF(!ValidateRuntimeValue(query, value, RuntimeValueType::Integer)); + auto& root = query.value_storage.emplace_back(ValueOpcode::ReferenceU32); + root.SetArg(0, value); + site.target[i] = Value(&root); + } } RewriteToSsa(query.blocks); ConstantPropagationPass(query.blocks, query.wave_size); RemoveIdentities(query); EliminateDeadCode(query.blocks); - for (auto& value: source.call->target) value = value.Instruction()->Arg(0).Resolve(); + for (auto& site: source.calls) + for (auto& value: site.target) value = value.Instruction()->Arg(0).Resolve(); query.value_storage.clear(); for (auto& inst: block) { inst.SetParent(nullptr); @@ -595,7 +602,7 @@ ShaderSource PrepareShaderSource(std::span code, const CompileOp Decoder::DecodeProgram(code, source.decoded); } if (source.decoded.has_swap_pc) { - PrepareCallTarget(source, options); + PrepareCallTargets(source, options); source.code.assign(code.begin(), code.end()); source.decoded.code = source.code; } @@ -608,83 +615,116 @@ const Decoder::Program& RefreshShaderSource(ShaderSource& source, const IR::SrtR IR::SrtReadCapture capture(runtime, reads); const auto clean = IR::CleanRuntime(capture.ObservedRuntime()); IR::SrtWalker walker(source.call_targets, clean); - uint32_t low = 0, high = 0; - const auto& call = source.decoded.instructions[source.call->instruction]; - if (!walker.Evaluate(source.call->target[0], low) || !walker.Evaluate(source.call->target[1], high)) - EXIT("shader call at pc 0x%08x has an unavailable scalar target", call.pc); - // SWAPPC reads the old aliased pair and ignores the target's low two bits. - const auto address = ((uint64_t {high} << 32u) | low) & ~uint64_t {3}; - EXIT_IF(address == 0u); + for (auto& site: source.calls) { + uint32_t low = 0, high = 0; + if (!walker.Evaluate(site.target[0], low) || !walker.Evaluate(site.target[1], high)) + EXIT("shader call at pc 0x%08x has an unavailable scalar target", + source.decoded.instructions[site.instruction].pc); + // SWAPPC reads the old aliased pair and ignores the target's low two bits. + const auto address = ((uint64_t {high} << 32u) | low) & ~uint64_t {3}; + EXIT_IF(address == 0u); + site.address = address; + } const auto query_reads = reads.size(); if (source.linked) { auto& linked = *source.linked; - const auto function = std::span(linked.code).subspan(source.code.size()); - if (clean.read_memory(clean.userdata, address, linked.observed_function) && - std::ranges::equal(function, linked.observed_function)) { - return linked.decoded; + bool unchanged = true; + for (size_t i = 0; i < source.calls.size(); ++i) { + const auto& function = linked.functions[i]; + const auto expected = std::span(linked.code).subspan(function.offset, function.size); + const auto preceding = std::span(source.calls).first(i); + const auto previous = std::ranges::find(preceding, source.calls[i].address, &ShaderSource::Call::address); + if (previous != preceding.end()) { + const auto& observed = linked.functions[previous - preceding.begin()]; + if (std::ranges::equal(expected, std::span(linked.code).subspan(observed.offset, observed.size))) continue; + unchanged = false; + break; + } + linked.observed_function.resize(function.size); + if (!clean.read_memory(clean.userdata, source.calls[i].address, linked.observed_function) || + !std::ranges::equal(expected, linked.observed_function)) { + unchanged = false; + break; + } } + if (unchanged) return linked.decoded; reads.resize(query_reads); } - ShaderSource::Linked linked {.code = source.code, - .decoded = source.decoded}; + ShaderSource::Linked linked {.code = source.code, .decoded = source.decoded}; const auto& last = source.decoded.instructions.back(); EXIT_NOT_IMPLEMENTED(last.opcode != Decoder::Opcode::S_ENDPGM); - // Keep the caller's data/footer bytes at their native offsets. The leaf's virtual CFG - // PCs follow decoded instructions; its physical code is independently read and owned. - const auto function_pc = last.pc + last.word_count * sizeof(uint32_t); - for (uint32_t index = 0;;) { - // The observed queue helper is a scalar leaf. Scalar encodings need at most one literal. - EXIT_IF(index >= 64u || address > UINT64_MAX - (uint64_t {index} + 2u) * sizeof(uint32_t)); - std::array words {}; - if (!clean.read_memory(clean.userdata, address + uint64_t {index} * sizeof(uint32_t), - std::span(words).first(1))) - EXIT("shader call at pc 0x%08x cannot read leaf code", call.pc); - const auto family = Decoder::GetInstructionFamily(words[0]); - EXIT_NOT_IMPLEMENTED(family != Decoder::Family::SOP1 && family != Decoder::Family::SOP2 && - family != Decoder::Family::SOPC && family != Decoder::Family::SOPK && - family != Decoder::Family::SOPP); - Decoder::Instruction inst; - Decoder::DecodeInstruction(words, 0u, inst); - if (inst.word_count == 2u) { - if (!clean.read_memory(clean.userdata, address + uint64_t {index + 1u} * sizeof(uint32_t), - std::span(words).last(1))) - EXIT("shader call at pc 0x%08x cannot read leaf literal", call.pc); - inst = {}; - Decoder::DecodeInstruction(words, 0u, inst); + // Keep the caller's data/footer at native offsets. Expand leaf CFGs per call site + // before SSA, so return edges neither require a dispatcher nor merge resource roots. + uint32_t function_pc = last.pc + last.word_count * sizeof(uint32_t); + std::map> functions; + for (size_t i = 0; i < source.calls.size(); ++i) { + const auto& site = source.calls[i]; + const auto& call = source.decoded.instructions[site.instruction]; + const auto address = source.calls[i].address; + auto [entry, inserted] = functions.try_emplace(address); + auto& function = entry->second; + if (inserted) { + for (uint32_t index = 0;;) { + std::array words {}; + EXIT_IF(index > UINT32_MAX / sizeof(uint32_t) - words.size() || + address > UINT64_MAX - (uint64_t {index} + words.size()) * sizeof(uint32_t)); + uint32_t count = 1; + Decoder::Instruction inst; + for (;;) { + if (!clean.read_memory(clean.userdata, address + uint64_t {index} * sizeof(uint32_t), + std::span(words).first(count))) + EXIT("shader call at pc 0x%08x cannot read leaf code", call.pc); + inst = {}; + Decoder::DecodeInstruction(words, 0u, inst); + if (inst.word_count <= count) break; + count = inst.word_count; + } + inst.pc = index * sizeof(uint32_t); + if (Decoder::IsDirectBranch(inst.opcode)) inst.branch_target += inst.pc; + index += inst.word_count; + EXIT_NOT_IMPLEMENTED(inst.opcode == Decoder::Opcode::S_SWAPPC_B64 || + inst.opcode == Decoder::Opcode::S_GETPC_B64 || + inst.opcode == Decoder::Opcode::S_ENDPGM); + function.push_back(inst); + if (inst.opcode == Decoder::Opcode::S_SETPC_B64) break; + } } - inst.pc = index * sizeof(uint32_t); - if (Decoder::IsDirectBranch(inst.opcode)) inst.branch_target += inst.pc; - linked.code.insert(linked.code.end(), inst.raw, inst.raw + inst.word_count); - index += inst.word_count; - EXIT_NOT_IMPLEMENTED(inst.opcode == Decoder::Opcode::S_SWAPPC_B64 || - inst.opcode == Decoder::Opcode::S_GETPC_B64 || - inst.opcode == Decoder::Opcode::S_ENDPGM); - EXIT_IF(CFG::MayWriteScalarRegister(inst, call.dst.reg) || CFG::MayWriteScalarRegister(inst, call.dst.reg + 1u)); - const bool returns = inst.opcode == Decoder::Opcode::S_SETPC_B64; - if (returns) { - EXIT_NOT_IMPLEMENTED(inst.src0.kind != Decoder::OperandKind::Sgpr || inst.src0.reg != call.dst.reg); - inst.opcode = Decoder::Opcode::S_BRANCH; - inst.branch_target = call.pc + call.word_count * sizeof(uint32_t); - } else if (Decoder::IsDirectBranch(inst.opcode)) { - EXIT_IF(inst.branch_target > UINT32_MAX - function_pc); - inst.branch_target += function_pc; + const uint32_t function_size = function.back().pc + function.back().word_count * sizeof(uint32_t); + EXIT_IF(function_pc > UINT32_MAX - function_size); + const auto function_offset = static_cast(linked.code.size()); + for (auto inst: function) { + EXIT_IF(CFG::MayWriteScalarRegister(inst, call.dst.reg) || + CFG::MayWriteScalarRegister(inst, call.dst.reg + 1u)); + for (uint32_t code = 0; code < site.preserved_registers.size(); ++code) { + if (site.preserved_registers[code] && CFG::MayWriteScalarRegister(inst, code)) + EXIT("shader call at pc 0x%08x clobbers scalar target dependency s%u", call.pc, code); + } + linked.code.insert(linked.code.end(), inst.raw, inst.raw + inst.word_count); + if (inst.opcode == Decoder::Opcode::S_SETPC_B64) { + EXIT_NOT_IMPLEMENTED(inst.src0.kind != Decoder::OperandKind::Sgpr || inst.src0.reg != call.dst.reg); + inst.opcode = Decoder::Opcode::S_BRANCH; + inst.branch_target = call.pc + call.word_count * sizeof(uint32_t); + } else if (Decoder::IsDirectBranch(inst.opcode)) { + EXIT_IF(inst.branch_target >= function_size); + inst.branch_target += function_pc; + } + inst.pc += function_pc; + linked.decoded.instructions.push_back(std::move(inst)); } - inst.pc += function_pc; - linked.decoded.instructions.push_back(std::move(inst)); - if (returns) break; - } - const auto function_end = linked.decoded.instructions.back().pc + - linked.decoded.instructions.back().word_count * sizeof(uint32_t); - for (size_t i = source.decoded.instructions.size(); i + 1u < linked.decoded.instructions.size(); ++i) { - const auto& inst = linked.decoded.instructions[i]; - if (Decoder::IsDirectBranch(inst.opcode)) - EXIT_IF(inst.branch_target < function_pc || inst.branch_target >= function_end); + linked.functions.push_back({function_offset, function_size / 4u}); + linked.decoded.instructions[site.instruction].branch_target = function_pc; + function_pc += function_size; } reads.resize(query_reads); - reads.emplace_back(address, (linked.code.size() - source.code.size()) * sizeof(uint32_t)); - linked.decoded.instructions[source.call->instruction].branch_target = function_pc; + for (size_t i = 0; i < source.calls.size(); ++i) { + const auto preceding = std::span(source.calls).first(i); + if (std::ranges::find(preceding, source.calls[i].address, &ShaderSource::Call::address) == preceding.end()) + reads.emplace_back(source.calls[i].address, linked.functions[i].size * sizeof(uint32_t)); + } linked.decoded.code = linked.code; - linked.observed_function.resize(linked.code.size() - source.code.size()); + linked.observed_function.resize(std::ranges::max_element(linked.functions, {}, + &ShaderSource::Linked::Function::size)->size); + source.linked = std::move(linked); ++source.revision; return source.linked->decoded; diff --git a/src/graphics/shader/recompiler/ShaderRecompiler.h b/src/graphics/shader/recompiler/ShaderRecompiler.h index 2c006153c..e656d14e2 100644 --- a/src/graphics/shader/recompiler/ShaderRecompiler.h +++ b/src/graphics/shader/recompiler/ShaderRecompiler.h @@ -8,6 +8,7 @@ #include "graphics/shader/shader.h" #include +#include #include #include #include @@ -47,16 +48,20 @@ struct ShaderSource { struct Call { uint32_t instruction; std::array target; + std::bitset<128> preserved_registers; + uint64_t address = 0; }; struct Linked { + struct Function { uint32_t offset; uint32_t size; }; std::vector code; Decoder::Program decoded; + std::vector functions; std::vector observed_function; }; std::vector code; Decoder::Program decoded; IR::ResourcePlan call_targets; - std::optional call; + std::vector calls; std::optional linked; uint64_t revision = 0; std::vector> reads; diff --git a/src/graphics/shader/recompiler/frontend/cfg/ShaderCFG.cpp b/src/graphics/shader/recompiler/frontend/cfg/ShaderCFG.cpp index 7eb9cafb4..0ed740360 100644 --- a/src/graphics/shader/recompiler/frontend/cfg/ShaderCFG.cpp +++ b/src/graphics/shader/recompiler/frontend/cfg/ShaderCFG.cpp @@ -8,6 +8,8 @@ #include #include #include +#include +#include #include namespace Libs::Graphics::ShaderRecompiler::CFG { @@ -1650,6 +1652,32 @@ Graph BuildGraph(const Decoder::Program& program) { } SortUnique(graph.code_table_load_pcs); + if (!indirect_setpc && std::ranges::any_of(program.instructions, [](const auto& inst) { + return inst.opcode == Opcode::S_SWAPPC_B64 && inst.branch_target != UINT32_MAX; + })) { + // Linked leaf bodies follow the caller in native PC order. Visit them in + // execution order so returning to the caller does not manufacture goto loops. + std::vector visited(graph.blocks.size(), false); + std::vector> pending {{graph.entry_block, false}}; + std::vector ordered; + ordered.reserve(graph.blocks.size()); + while (!pending.empty()) { + const auto [id, finished] = pending.back(); + pending.pop_back(); + if (finished) { + ordered.push_back(std::move(graph.blocks[id])); + continue; + } + if (visited[id]) continue; + visited[id] = true; + pending.emplace_back(id, true); + for (const auto successor: graph.blocks[id].successors) + pending.emplace_back(successor, false); + } + std::ranges::reverse(ordered); + ApplyBlockOrder(graph, std::move(ordered)); + } + RecomputeAnalyses(graph); if (indirect_setpc) { @@ -1681,33 +1709,56 @@ Graph Structurize(const Graph& graph) { } bool MayWriteScalarRegister(const Decoder::Instruction& inst, uint32_t code) { - uint32_t destination = UINT32_MAX; - // Scalar ALU pair widths are not all decoded. The typed producer must confirm this word. - return InstructionWritesScalarCode(inst, code) || - (ScalarOperandCode(inst.dst, destination) && code == destination + 1u); + // Vector comparisons and carry results can write a scalar pair in wave64. + if (inst.family == Decoder::Family::VOP1 || inst.family == Decoder::Family::VOP2 || + inst.family == Decoder::Family::VOPC || inst.family == Decoder::Family::VOP3 || + inst.family == Decoder::Family::VOP3P) { + uint32_t destination = 0; + for (const auto* operand: {&inst.dst, &inst.dst2}) + if (ScalarOperandCode(*operand, destination) && code >= destination && code <= destination + 1u) + return true; + } + return InstructionWritesScalarCode(inst, code); } uint32_t FindScalarDefinition(const Decoder::Program& program, const Graph& graph, - uint32_t before, uint32_t code) { + uint32_t before, uint32_t code, std::vector& crossed_calls) { EXIT_IF(before >= program.instructions.size()); const auto found = std::ranges::find_if(graph.blocks, [before](const auto& block) { return block.inst_begin <= before && before < block.inst_end; }); EXIT_IF(found == graph.blocks.end()); - const auto* block = &*found; - for (size_t depth = 0; depth < graph.blocks.size(); ++depth) { - for (uint32_t i = before; i > block->inst_begin;) { - const auto& inst = program.instructions[--i]; - EXIT_IF(inst.opcode == Opcode::S_SWAPPC_B64); - if (MayWriteScalarRegister(inst, code)) return i; - } - if (block->id == graph.entry_block) return UINT32_MAX; - EXIT_IF(block->predecessors.size() != 1u); - block = graph.FindBlock(block->predecessors[0]); + std::vector> pending {{found->id, before}}; + std::set> visited; + std::optional definition; + while (!pending.empty()) { + const auto point = pending.back(); + pending.pop_back(); + if (!visited.insert(point).second) continue; + const auto* block = graph.FindBlock(point.first); EXIT_IF(block == nullptr); - before = block->inst_end; + uint32_t index = UINT32_MAX; + for (uint32_t i = point.second; i > block->inst_begin;) { + const auto& inst = program.instructions[--i]; + if (inst.opcode == Opcode::S_SWAPPC_B64) { + EXIT_NOT_IMPLEMENTED(MayWriteScalarRegister(inst, code)); + crossed_calls.push_back(i); + continue; + } + if (MayWriteScalarRegister(inst, code)) { index = i; break; } + } + if (index != UINT32_MAX || block->id == graph.entry_block) { + EXIT_IF(definition && *definition != index); + definition = index; + continue; + } + EXIT_IF(block->predecessors.empty()); + for (const auto predecessor: block->predecessors) { + pending.emplace_back(predecessor, graph.blocks[predecessor].inst_end); + } } - EXIT("scalar shader call source has cyclic reaching definitions"); + EXIT_IF(!definition); + return *definition; } std::string BranchConditionToString(BranchCondition condition) { diff --git a/src/graphics/shader/recompiler/frontend/cfg/ShaderCFG.h b/src/graphics/shader/recompiler/frontend/cfg/ShaderCFG.h index 8caad316a..3a63ef7e2 100644 --- a/src/graphics/shader/recompiler/frontend/cfg/ShaderCFG.h +++ b/src/graphics/shader/recompiler/frontend/cfg/ShaderCFG.h @@ -115,7 +115,7 @@ struct Graph { Graph BuildGraph(const Decoder::Program& program); bool MayWriteScalarRegister(const Decoder::Instruction& inst, uint32_t code); uint32_t FindScalarDefinition(const Decoder::Program& program, const Graph& graph, - uint32_t before, uint32_t code); + uint32_t before, uint32_t code, std::vector& crossed_calls); // Returns structured control flow or failure diagnostics without changing the native graph. // On failure, failure_block is an original block ID or UINT32_MAX. Graph Structurize(const Graph& graph); diff --git a/src/graphics/shader/recompiler/frontend/decode/ScalarAluOps.cpp b/src/graphics/shader/recompiler/frontend/decode/ScalarAluOps.cpp index 67351efea..ffdbae4f6 100644 --- a/src/graphics/shader/recompiler/frontend/decode/ScalarAluOps.cpp +++ b/src/graphics/shader/recompiler/frontend/decode/ScalarAluOps.cpp @@ -164,9 +164,23 @@ void DecodeSop1(uint32_t pc, std::span code, uint32_t word_index switch (inst.opcode) { case Opcode::S_SWAPPC_B64: inst.branch_target = UINT32_MAX; - inst.data_dwords = 2; + [[fallthrough]]; + case Opcode::S_MOV_B64: + case Opcode::S_CMOV_B64: + case Opcode::S_NOT_B64: + case Opcode::S_WQM_B64: + case Opcode::S_BREV_B64: + case Opcode::S_BITSET0_B64: + case Opcode::S_BITSET1_B64: + case Opcode::S_AND_SAVEEXEC_B64: + case Opcode::S_ORN2_SAVEEXEC_B64: + case Opcode::S_QUADMASK_B64: + case Opcode::S_ANDN1_SAVEEXEC_B64: + case Opcode::S_BITREPLICATE_B64_B32: + inst.data_dwords = 2u; break; case Opcode::S_GETPC_B64: + inst.data_dwords = 2u; inst.src_count = 0; DecodeScalarDestination(sdst, pc, inst.dst); return; @@ -206,6 +220,8 @@ void DecodeSop2(uint32_t pc, std::span code, uint32_t word_index DecodeScalarDestination(sdst, pc, inst.dst); DecodeBinarySources(pc, code, word_index, inst, ssrc0, ssrc1); + if (opcode == 0x0bu || (opcode >= 0x0fu && opcode <= 0x25u && (opcode & 1u)) || opcode == 0x29u) + inst.data_dwords = 2u; } void DecodeSopk(uint32_t pc, std::span code, uint32_t word_index, diff --git a/tests/ShaderRecompilerComputeTests.cpp b/tests/ShaderRecompilerComputeTests.cpp index a151fa9d8..96e8163b5 100644 --- a/tests/ShaderRecompilerComputeTests.cpp +++ b/tests/ShaderRecompilerComputeTests.cpp @@ -10734,6 +10734,143 @@ public: LibKernel::Memory::TryReadBacking(output, &actual, sizeof(actual)) && actual == test.value, "caller did not observe the leaf's scalar result after return"); } + + // Each helper reads through its context, masks EXEC, then restores all + // caller lanes. The MAD literal occupies a third instruction DWORD. + std::array, 2> vector_leaves; + for (u32 i = 0; i < vector_leaves.size(); ++i) { + auto& helper = vector_leaves[i]; + helper = {EncodeSop1(0x04, 18, 126), EncodeVopc(0xd4, 6, 0), 0, + EncodeSmem0(0x02, 20, 8), EncodeSmem1(0, 125), EncodeSopp(0x0c, 0), + EncodeMubuf0(0x0c), EncodeMubuf1(2, 5, 1), EncodeSopp(0x0c, 0), + EncodeVop2(0x03, 2, 7, 2)}; + AppendVop3(&helper, 0x141, 6, 255, Vgpr(2), Vgpr(6)); + helper.push_back(std::bit_cast(i == 0 ? 1.5f : 2.5f)); + helper[2] = EncodeSopp(0x08, static_cast(helper.size()) - 3u); + helper.push_back(EncodeSop1(0x04, 126, 18)); + helper.push_back(EncodeSop1(0x21, 125, 14)); + const auto address = i == 0 ? first : second; + Require(name, "replace vector helper", context.InvalidateMemory(address, helper.size() * 4), + "vector helper escaped mapped memory"); + LibKernel::Memory::WriteBacking(address, helper.data(), helper.size() * 4); + } + out.fields[3] |= 3u << 28u; + ShaderBufferResource data = out; + data.UpdateAddress48(base + 0x5000); + data.fields[2] = 64 * sizeof(float); + Require(name, "refresh helper context", context.InvalidateMemory(table, sizeof(data.fields)), + "helper context escaped mapped memory"); + LibKernel::Memory::WriteBacking(table, data.fields, sizeof(data.fields)); + std::array values; + for (u32 lane = 0; lane < values.size(); ++lane) values[lane] = float(lane + 1); + Require(name, "refresh vector input", context.InvalidateMemory(base + 0x5000, sizeof(values)), + "vector input escaped mapped memory"); + LibKernel::Memory::WriteBacking(base + 0x5000, values.data(), sizeof(values)); + for (const u32 wave_size : {32u, 64u}) { + for (const u32 call_count : {1u, 13u}) { + std::vector caller{EncodeVop2(0x1a, 1, InlineU32(2), 0)}; + AppendVMovLiteral(&caller, 6, std::bit_cast(10.0f)); + for (u32 call = 0; call < call_count; ++call) { + AppendSMovLiteral(&caller, 7, std::bit_cast(float(call + 1))); + caller.insert(caller.end(), { + EncodeSMovB32(14, call < 8 ? 8 : 12), EncodeSMovB32(15, call < 8 ? 9 : 13), + EncodeSMovB32(16, 4), EncodeSMovB32(17, 5), EncodeSop1(0x21, 14, 14)}); + } + caller.insert(caller.end(), {EncodeMubuf0(0x1c), EncodeMubuf1(6, 0, 1)}); + AppendEnd(&caller); + ShaderMapUserData(reinterpret_cast(caller.data()), + {.type = Prospero::ShaderBinaryType::kCs, + .code_size_bytes = static_cast(caller.size() * sizeof(u32))}); + shaders.SetCsShader({.data_addr = reinterpret_cast(caller.data()), + .num_thread_x = wave_size, .num_thread_y = 1, .num_thread_z = 1, + .wave_size = static_cast(wave_size), .user_sgpr = 14}); + out.fields[2] = wave_size * sizeof(float); + std::array user_data{}; + std::copy_n(out.fields, 4, user_data.begin()); + user_data[4] = static_cast(table); + user_data[5] = static_cast(table >> 32u); + user_data[8] = static_cast(first | 3u); + user_data[9] = static_cast(first >> 32u); + user_data[12] = static_cast(second | 3u); + user_data[13] = static_cast(second >> 32u); + uint64_t original_vector_id = 0; + for (u32 phase = 0; phase < (call_count == 1 ? 2u : 4u); ++phase) { + const u32 active_lanes = phase == 0 ? 0 : 16; + if (phase >= 2) { + std::swap(user_data[8], user_data[12]); + std::swap(user_data[9], user_data[13]); + } + user_data[6] = active_lanes; + for (u32 i = 0; i < user_data.size(); ++i) + shaders.SetCsUserSgpr(i, user_data[i], HW::UserSgprType::Unknown); + Require(name, "reset caller output", context.InvalidateMemory(output, wave_size * 4), + "caller output escaped mapped memory"); + std::array actual{}; + LibKernel::Memory::WriteBacking(output, actual.data(), wave_size * 4); + ShaderComputeInputInfo input{}; + const auto program = context.GetPipelineCache().GetComputeProgram( + shaders.GetCs(), processor.GetCtx().GetShaderRegisters(), input); + if (phase == 1) original_vector_id = program.id; + if (phase >= 2) + Require(name, "ordered helper identity", (program.id == original_vector_id) == (phase == 3), + "swapped helper targets reused stale code or restored targets recompiled"); + const auto& reads = input.stage.resources->specialization_reads; + for (u32 i = 0; i < (call_count == 1 ? 1u : 2u); ++i) + Require(name, "unique helper code dependency", std::ranges::count(reads, + std::pair{i == 0 ? first : second, + vector_leaves[i].size() * sizeof(u32)}) == 1, + "repeated calls duplicated or omitted a helper code dependency"); + if (phase == 1) { + ShaderRecompiler::CompileOptions options; + options.wave_size = wave_size; + options.dump_ir = false; + options.maintenance8_enabled = true; + input.host_subgroup_size = SubgroupSize(); + options.input_info.compute = &input; + options.user_data = user_data; + const auto read = +[](void*, uint64_t address, std::span words) { + return LibKernel::Memory::TryReadBufferBacking(address, words.data(), words.size_bytes()); + }; + const ShaderRecompiler::IR::SrtRuntime runtime{ + .user_data = user_data, .shader_base = reinterpret_cast(caller.data()), + .read_memory = read, .read_specialization_memory = read}; + auto source = ShaderRecompiler::PrepareShaderSource(caller, options); + auto translated = ShaderRecompiler::TranslateProgram( + ShaderRecompiler::RefreshShaderSource(source, runtime), options); + auto plan = ShaderRecompiler::IR::ExtractResourcePlan(translated.program); + ShaderRecompiler::IR::ResourceSnapshot snapshot; + ShaderRecompiler::IR::ResourceSpecialization specialization; + Require(name, "vector helper resources", ShaderRecompiler::IR::MaterializeResources( + plan, runtime, snapshot, specialization), "helper V# could not be materialized"); + auto compiled = ShaderRecompiler::CompileProgram(std::move(translated), options, specialization); + ValidateSpirv(name, compiled.spirv); + CheckSpirvEmission({.name = name, .forbidden_spirv = {"OpLoopMerge"}}, compiled.spirv); + const u32 word_budget = wave_size == 32 ? 640 + 224 * (call_count - 1) : + 1024 + 416 * (call_count - 1); + Require(name, "linear call emission", compiled.spirv.size() <= word_budget, + "acyclic calls exceeded their per-call SPIR-V budget"); + std::printf("[gpu] %-32s wave%u calls%u: %zu SPIR-V words\n", + name, wave_size, call_count, compiled.spirv.size()); + } + processor.DispatchDirect(1, 1, 1, 0x41u | + (wave_size == 32u ? 1u << Pm4::COMPUTE_DISPATCH_INITIATOR_CS_W32_EN_SHIFT : 0u)); + cache.ReadMemory(output, wave_size * 4); + Require(name, "read vector helper output", LibKernel::Memory::TryReadBacking( + output, actual.data(), wave_size * 4), "caller output was not readable"); + for (u32 lane = 0; lane < wave_size; ++lane) { + float expected = 10.0f; + if (lane < active_lanes) + for (u32 call = 0; call < call_count; ++call) + expected += ((call < 8) != (phase == 2) ? 1.5f : 2.5f) * + (values[lane] + float(call + 1)); + Require(name, "repeated vector calls and EXEC return", actual[lane] == expected, + "wave" + std::to_string(wave_size) + " calls" + std::to_string(call_count) + + " phase" + std::to_string(phase) + " lane" + std::to_string(lane) + + " actual=" + std::to_string(actual[lane]) + " expected=" + std::to_string(expected)); + } + } + } + } context.UnmapMemory(base, size); context.GetCommandScheduler().Finish(); }); @@ -49010,6 +49147,11 @@ int main(int argc, char **argv) { vulkan.CheckGpuTilerCpuParity(); return 0; } + if (argc == 2 && std::strcmp(argv[1], "--scalar-leaf-call-only") == 0) { + VulkanHarness vulkan; + vulkan.CheckScalarLeafCall(); + return 0; + } if (argc == 2 && std::strcmp(argv[1], "--gpu-command-lane-only") == 0) { VulkanHarness vulkan; vulkan.CheckGpuCommandLane(); diff --git a/tests/shaderCfgTests.cpp b/tests/shaderCfgTests.cpp index 30f770ef5..b487e4d8f 100644 --- a/tests/shaderCfgTests.cpp +++ b/tests/shaderCfgTests.cpp @@ -10805,8 +10805,8 @@ void TestMemoryFedScalarLeafCall() { IR::SrtRuntime runtime{.read_memory = read, .userdata = &memory, .read_specialization_memory = read}; const auto options = MakeCompileOptions(ShaderType::Compute); auto source = PrepareShaderSource(shader, options); - Check(source.call && source.decoded.has_swap_pc && - source.decoded.instructions[source.call->instruction].branch_target == UINT32_MAX, + Check(source.calls.size() == 1 && source.decoded.has_swap_pc && + source.decoded.instructions[source.calls.front().instruction].branch_target == UINT32_MAX, "memory-fed SWAPPC did not retain an unresolved target"); #if KYTY_PLATFORM != KYTY_PLATFORM_WINDOWS ExpectFatal([&] { (void)TranslateProgram(source.decoded, options); }, @@ -10814,7 +10814,7 @@ void TestMemoryFedScalarLeafCall() { #endif const auto& linked = RefreshShaderSource(source, runtime); Check(linked.has_swap_pc && source.revision == 1 && linked.instructions.back().branch_target == 0x40 && - linked.instructions[source.call->instruction].branch_target == 0x44 && + linked.instructions[source.calls.front().instruction].branch_target == 0x44 && source.reads.back() == std::pair{base + 0x100, 12}, "aliased call target, exact leaf range, or continuation was lost"); const auto* storage = linked.instructions.data(); @@ -10899,14 +10899,14 @@ void TestMemoryFedScalarLeafCall() { memory.words[96] = EncodeSMovB32(14, 129); ExpectFatal([&] { (void)RefreshShaderSource(source, runtime); }, "leaf clobbering its return pair was linked"); - // AND_B64 writes the high half despite the decoder's generic scalar width. + // AND_B64 must replace the preceding high-half definition. const uint32_t high_writer[] = { EncodeSMovB32(15, 129), EncodeSop2(0x0f, 14, 128, 128), EncodeSop1(0x21, 14, 14), EncodeSopp(0x01)}; auto pair_source = PrepareShaderSource(high_writer, options); IR::SrtWalker pair_walker(pair_source.call_targets, runtime); uint32_t high = 1; - Check(pair_walker.Evaluate(pair_source.call->target[1], high) && high == 0, + Check(pair_walker.Evaluate(pair_source.calls.front().target[1], high) && high == 0, "scalar pair producer used a stale preceding high-half definition"); const uint32_t joined[] = { EncodeSopp(0x04, 2), EncodeSMovB32(14, 129), EncodeSopp(0x02, 1), @@ -10915,7 +10915,7 @@ void TestMemoryFedScalarLeafCall() { "ambiguous reaching definitions were accepted for a call target"); const uint32_t multiple[] = {EncodeSop1(0x21, 14, 14), EncodeSop1(0x21, 14, 14), EncodeSopp(0x01)}; ExpectFatal([&] { (void)PrepareShaderSource(multiple, options); }, - "call metadata bypassed the multiple-call rejection"); + "a native return pair was accepted as a later call target"); #endif } @@ -10962,7 +10962,7 @@ void TestFusedShaderHandoffPreservesRegisters() { for (const auto handoff: {EncodeSop1(0x20, 0, 6), 0xbefd2106u}) { front[2] = handoff; // SETPC or captured SWAPPC with NULL destination auto source = PrepareShaderSource(front, options); - Check(!source.decoded.has_swap_pc && !source.call, + Check(!source.decoded.has_swap_pc && source.calls.empty(), "fused SETPC handoff entered scalar call preparation"); auto translated = TranslateProgram(source.decoded, options); uint32_t allocations = 0;