diff --git a/src/graphics/host_gpu/renderer/pipeline/pipelineCache.cpp b/src/graphics/host_gpu/renderer/pipeline/pipelineCache.cpp index b1989e08a..d56b629b9 100644 --- a/src/graphics/host_gpu/renderer/pipeline/pipelineCache.cpp +++ b/src/graphics/host_gpu/renderer/pipeline/pipelineCache.cpp @@ -230,6 +230,39 @@ struct PipelineCache::ProgramCache { permutations.reserve(8); } + std::span FunctionCode() const { + return call_source + ? std::span(call_source->linked->code).subspan(call_source->code.size()) + : std::span{}; + } + + bool RefreshCallSource(const ShaderRecompiler::IR::SrtRuntime& runtime) { + const auto revision = call_source->revision; + ShaderRecompiler::RefreshShaderSource(*call_source, runtime); + resource_plan.source_reads = call_source->reads; + return revision != call_source->revision; + } + + template + Permutation* FindPermutation(const ShaderRecompiler::IR::SrtRuntime& runtime, + uint32_t push_data_cursor) { + EXIT_IF(!ShaderRecompiler::IR::MaterializeResources( + resource_plan, runtime, resources, specialization)); + for (auto& candidate: permutations) { + const auto& layout = candidate.program.bindings; + if (layout.push_data_start_dword != ShaderRecompiler::IR::PushData::StartFor( + push_data_cursor, layout.ShaderDataDwords()) || + candidate.specialization != specialization) { + continue; + } + if constexpr (HasCalls) { + if (!std::ranges::equal(candidate.function_code, FunctionCode())) continue; + } + return &candidate; + } + return nullptr; + } + std::unique_ptr call_source; ShaderRecompiler::IR::ResourcePlan resource_plan; ShaderRecompiler::IR::ResourceSnapshot resources; @@ -313,31 +346,16 @@ struct PipelineCache::ProgramCache { runtime.workgroup_counts = input_info.workgroup_counts; } bool source_changed = false; - const auto function_code = [](const std::unique_ptr& source) { - return source ? std::span(source->linked->code).subspan(source->code.size()) - : std::span{}; - }; - if (entry != programs.end() && entry->second.call_source) { + if (entry != programs.end()) { auto& cached = entry->second; - const auto revision = cached.call_source->revision; - ShaderRecompiler::RefreshShaderSource(*cached.call_source, runtime); - source_changed = revision != cached.call_source->revision; - cached.resource_plan.source_reads = cached.call_source->reads; - } - const auto matches = [&](const Permutation& candidate) { - const auto& layout = candidate.program.bindings; - return layout.push_data_start_dword == ShaderRecompiler::IR::PushData::StartFor( - push_data_cursor, layout.ShaderDataDwords()) && - candidate.specialization == entry->second.specialization && - (!entry->second.call_source || std::ranges::equal( - candidate.function_code, function_code(entry->second.call_source))); - }; - if (entry != programs.end() && !source_changed) { - EXIT_IF(!ShaderRecompiler::IR::MaterializeResources( - entry->second.resource_plan, runtime, entry->second.resources, - entry->second.specialization)); - if (const auto permutation = std::ranges::find_if(entry->second.permutations, matches); - permutation != entry->second.permutations.end()) { + Permutation* permutation = nullptr; + if (cached.call_source) { + source_changed = cached.RefreshCallSource(runtime); + if (!source_changed) permutation = cached.FindPermutation(runtime, push_data_cursor); + } else { + permutation = cached.FindPermutation(runtime, push_data_cursor); + } + if (permutation != nullptr) { input_info.stage = {.program = &permutation->program, .resources = &entry->second.resources}; permutation->program.bindings.AdvancePushData(push_data_cursor); return permutation->handle; @@ -390,11 +408,13 @@ struct PipelineCache::ProgramCache { auto* current = entry == programs.end() ? &source : entry->second.call_source.get(); ShaderRecompiler::TranslateResult translated; if (current != nullptr) { - const auto& decoded = entry == programs.end() - ? ShaderRecompiler::RefreshShaderSource(*current, runtime) - : current->linked->decoded; - translated = ShaderRecompiler::TranslateProgram(decoded, options); - translated.program.source_reads = current->reads; + if (current->call) { + if (entry == programs.end()) ShaderRecompiler::RefreshShaderSource(*current, runtime); + translated = ShaderRecompiler::TranslateProgram(current->linked->decoded, options); + translated.program.source_reads = current->reads; + } else { + translated = ShaderRecompiler::TranslateProgram(current->decoded, options); + } } else { translated = ShaderRecompiler::TranslateProgram(params.code, options); } @@ -407,11 +427,7 @@ struct PipelineCache::ProgramCache { entry->second.specialization)); } else if (source_changed) { entry->second.resource_plan = ShaderRecompiler::IR::ExtractResourcePlan(translated.program); - EXIT_IF(!ShaderRecompiler::IR::MaterializeResources( - entry->second.resource_plan, runtime, entry->second.resources, - entry->second.specialization)); - if (const auto permutation = std::ranges::find_if(entry->second.permutations, matches); - permutation != entry->second.permutations.end()) { + if (const auto* permutation = entry->second.FindPermutation(runtime, push_data_cursor)) { input_info.stage = {.program = &permutation->program, .resources = &entry->second.resources}; permutation->program.bindings.AdvancePushData(push_data_cursor); return permutation->handle; @@ -419,7 +435,7 @@ struct PipelineCache::ProgramCache { } entry->second.permutations.push_back(CompilePermutation( stage_name, options, std::move(translated), entry->second.specialization, push_data_cursor, - function_code(entry->second.call_source))); + entry->second.FunctionCode())); const auto& permutation = entry->second.permutations.back(); input_info.stage = {.program = &permutation.program, .resources = &entry->second.resources}; permutation.program.bindings.AdvancePushData(push_data_cursor); diff --git a/src/graphics/shader/recompiler/ShaderRecompiler.cpp b/src/graphics/shader/recompiler/ShaderRecompiler.cpp index a11fa40fd..2bbda045e 100644 --- a/src/graphics/shader/recompiler/ShaderRecompiler.cpp +++ b/src/graphics/shader/recompiler/ShaderRecompiler.cpp @@ -468,6 +468,7 @@ Decoder::Program DecodeFusedProgram(std::span front, std::spaninstruction]; EXIT_NOT_IMPLEMENTED(call.src0.kind != Decoder::OperandKind::Sgpr || @@ -594,8 +594,8 @@ ShaderSource PrepareShaderSource(std::span code, const CompileOp } else { Decoder::DecodeProgram(code, source.decoded); } - PrepareCallTarget(source, options); - if (source.call) { + if (source.decoded.has_swap_pc) { + PrepareCallTarget(source, options); source.code.assign(code.begin(), code.end()); source.decoded.code = source.code; } @@ -605,7 +605,6 @@ ShaderSource PrepareShaderSource(std::span code, const CompileOp const Decoder::Program& RefreshShaderSource(ShaderSource& source, const IR::SrtRuntime& runtime) { auto& reads = source.reads; reads.clear(); - if (!source.call) return source.decoded; IR::SrtReadCapture capture(runtime, reads); const auto clean = IR::CleanRuntime(capture.ObservedRuntime()); IR::SrtWalker walker(source.call_targets, clean); @@ -698,12 +697,11 @@ TranslateResult TranslateProgram(std::span code, const CompileOp TranslateResult TranslateProgram(const Decoder::Program& decoded, const CompileOptions& options) { EXIT_IF(decoded.instructions.empty()); - bool has_call = false; - for (const auto& inst: decoded.instructions) { - if (inst.opcode != Decoder::Opcode::S_SWAPPC_B64) continue; - has_call = true; - if (inst.branch_target == UINT32_MAX) - EXIT("shader call at pc 0x%08x must be linked before translation", inst.pc); + if (decoded.has_swap_pc) { + for (const auto& inst: decoded.instructions) { + if (inst.opcode == Decoder::Opcode::S_SWAPPC_B64 && inst.branch_target == UINT32_MAX) + EXIT("shader call at pc 0x%08x must be linked before translation", inst.pc); + } } if (options.stage != ShaderType::Compute && options.stage != ShaderType::Vertex && options.stage != ShaderType::Pixel && options.stage != ShaderType::Mesh && @@ -804,7 +802,7 @@ TranslateResult TranslateProgram(const Decoder::Program& decoded, const CompileO IR::RemoveIdentities(ir.blocks); IR::EliminateDeadCode(ir.blocks); } - if (has_call) { + if (decoded.has_swap_pc) { // Leaf returns are resolved CFG edges; native PC values must not escape into GPU data. for (const auto* block: ir.blocks) for (const auto& inst: *block) diff --git a/src/graphics/shader/recompiler/frontend/decode/ShaderDecoder.cpp b/src/graphics/shader/recompiler/frontend/decode/ShaderDecoder.cpp index d2731ce10..5cd026797 100644 --- a/src/graphics/shader/recompiler/frontend/decode/ShaderDecoder.cpp +++ b/src/graphics/shader/recompiler/frontend/decode/ShaderDecoder.cpp @@ -389,6 +389,7 @@ Program DecodeFrontProgram(std::span front) { while (front_words < front.size()) { auto& inst = result.instructions.emplace_back(); DecodeInstruction(front, front_words, inst); + result.has_swap_pc |= inst.opcode == Opcode::S_SWAPPC_B64; front_words += inst.word_count; if (inst.opcode == Opcode::S_SETPC_B64) { EXIT_NOT_IMPLEMENTED(inst.src0.kind != OperandKind::Sgpr || inst.src0.reg != 6u); @@ -406,6 +407,7 @@ void DecodeProgram(std::span code, Program& program) { program.instructions.clear(); program.instructions.reserve(code.size()); program.code = code; + program.has_swap_pc = false; std::vector branch_targets; for (uint32_t word_index = 0; word_index < code.size();) { @@ -413,6 +415,7 @@ void DecodeProgram(std::span code, Program& program) { DecodeInstruction(code, word_index, program.instructions.back()); const auto& inst = program.instructions.back(); + program.has_swap_pc |= inst.opcode == Opcode::S_SWAPPC_B64; word_index += inst.word_count; if (IsDirectBranch(inst.opcode)) { diff --git a/src/graphics/shader/recompiler/frontend/decode/ShaderDecoder.h b/src/graphics/shader/recompiler/frontend/decode/ShaderDecoder.h index 0df6237d4..1d7d1c656 100644 --- a/src/graphics/shader/recompiler/frontend/decode/ShaderDecoder.h +++ b/src/graphics/shader/recompiler/frontend/decode/ShaderDecoder.h @@ -818,6 +818,7 @@ struct Instruction { struct Program { std::span code; std::vector instructions; + bool has_swap_pc = false; }; // Code spans are trusted to contain complete instructions, valid branch targets, and 32-bit PCs. diff --git a/tests/shaderCfgTests.cpp b/tests/shaderCfgTests.cpp index 283b17e4a..49c240365 100644 --- a/tests/shaderCfgTests.cpp +++ b/tests/shaderCfgTests.cpp @@ -10373,6 +10373,17 @@ void TestMemoryFedScalarLeafCall() { 0xf4080135u, 0xfa000000u, 0xbf8cc07fu, 0xf4240382u, 0xfa000020u, 0xbf8cc07fu, 0xbf130e80u, 0xbf840004u, 0xf4240402u, 0xfa000028u, 0xbf8cc07fu, 0xbe8e210eu, 0xbf810000u}; + Decoder::Program decoded; + Decoder::DecodeProgram(shader, decoded); + Check(decoded.has_swap_pc, "decoded scalar call was not recorded"); + const uint32_t literal[] = {EncodeSMovB32(0, 255), 0xbe8e210eu, EncodeSopp(0x01)}; + Decoder::DecodeProgram(literal, decoded); + Check(!decoded.has_swap_pc && decoded.instructions.size() == 2, + "reused decoder retained a call or treated a literal as SWAPPC"); + const uint32_t null_call[] = {EncodeSop1(0x21, 125, 14), EncodeSopp(0x01)}; + Decoder::DecodeProgram(null_call, decoded); + Check(!decoded.has_swap_pc && decoded.instructions.front().opcode == Decoder::Opcode::S_SETPC_B64, + "NULL-destination SWAPPC was recorded as a call"); struct Memory { std::array words{}; uint64_t end = base + 0x10c; @@ -10396,14 +10407,15 @@ void TestMemoryFedScalarLeafCall() { IR::SrtRuntime runtime{.read_memory = read, .userdata = &memory, .read_specialization_memory = read}; const auto options = MakeCompileOptions(ShaderType::Compute); auto source = PrepareShaderSource(shader, options); - Check(source.call && source.decoded.instructions[source.call->instruction].branch_target == UINT32_MAX, + Check(source.call && source.decoded.has_swap_pc && + source.decoded.instructions[source.call->instruction].branch_target == UINT32_MAX, "memory-fed SWAPPC did not retain an unresolved target"); #if KYTY_PLATFORM != KYTY_PLATFORM_WINDOWS ExpectFatal([&] { (void)TranslateProgram(source.decoded, options); }, "unresolved SWAPPC silently fell through"); #endif const auto& linked = RefreshShaderSource(source, runtime); - Check(source.revision == 1 && linked.instructions.back().branch_target == 0x40 && + Check(linked.has_swap_pc && source.revision == 1 && linked.instructions.back().branch_target == 0x40 && linked.instructions[source.call->instruction].branch_target == 0x44 && source.reads.back() == std::pair{base + 0x100, 12}, "aliased call target, exact leaf range, or continuation was lost"); @@ -10503,6 +10515,9 @@ void TestMemoryFedScalarLeafCall() { EncodeSMovB32(14, 130), EncodeSop1(0x21, 14, 14), EncodeSopp(0x01)}; ExpectFatal([&] { (void)PrepareShaderSource(joined, options); }, "ambiguous reaching definitions were accepted for a call target"); + const uint32_t multiple[] = {EncodeSop1(0x21, 14, 14), EncodeSop1(0x21, 14, 14), EncodeSopp(0x01)}; + ExpectFatal([&] { (void)PrepareShaderSource(multiple, options); }, + "call metadata bypassed the multiple-call rejection"); #endif } @@ -10548,7 +10563,10 @@ void TestFusedShaderHandoffPreservesRegisters() { options.back_code = back; for (const auto handoff: {EncodeSop1(0x20, 0, 6), 0xbefd2106u}) { front[2] = handoff; // SETPC or captured SWAPPC with NULL destination - auto translated = TranslateProgram(front, options); + auto source = PrepareShaderSource(front, options); + Check(!source.decoded.has_swap_pc && !source.call, + "fused SETPC handoff entered scalar call preparation"); + auto translated = TranslateProgram(source.decoded, options); uint32_t allocations = 0; for (const auto* block: translated.program.blocks) { for (const auto& inst: *block) { @@ -10563,6 +10581,17 @@ void TestFusedShaderHandoffPreservesRegisters() { } Check(allocations == 1u, "fused shader omitted the back shader allocation"); } + const uint32_t call_front[] = {EncodeSop1(0x21, 14, 14), EncodeSop1(0x20, 0, 6)}; + Check(Decoder::DecodeFrontProgram(call_front).has_swap_pc, + "front decoder omitted its scalar call"); +#if KYTY_PLATFORM != KYTY_PLATFORM_WINDOWS + ExpectFatal([&] { (void)PrepareShaderSource(call_front, options); }, + "fused front call bypassed the non-compute rejection"); + const uint32_t call_back[] = {EncodeSop1(0x21, 14, 14), EncodeSopp(0x01)}; + options.back_code = call_back; + ExpectFatal([&] { (void)PrepareShaderSource(front, options); }, + "fused back call metadata was not propagated"); +#endif } void TestMeshExportStorage() {