From 4b651f2ba7977be75d992df93d7f17557df09fa1 Mon Sep 17 00:00:00 2001 From: nmzik Date: Sat, 24 Oct 2026 22:02:41 +0200 Subject: [PATCH] pipeline: fold the program cache into PipelineCache with stable module ids --- .../renderer/pipeline/pipelineCache.cpp | 536 ++++++++---------- .../renderer/pipeline/pipelineCache.h | 38 +- .../shader/recompiler/ShaderRecompiler.cpp | 94 ++- .../shader/recompiler/ShaderRecompiler.h | 3 +- tests/ShaderRecompilerComputeTests.cpp | 55 ++ 5 files changed, 368 insertions(+), 358 deletions(-) diff --git a/src/graphics/host_gpu/renderer/pipeline/pipelineCache.cpp b/src/graphics/host_gpu/renderer/pipeline/pipelineCache.cpp index bff05ce79..0faa31600 100644 --- a/src/graphics/host_gpu/renderer/pipeline/pipelineCache.cpp +++ b/src/graphics/host_gpu/renderer/pipeline/pipelineCache.cpp @@ -25,13 +25,12 @@ #include #include #include -#include #include #include #include +#include #include #include -#include #include #include #include @@ -114,14 +113,11 @@ bool ReadShaderGuestMemory(void*, uint64_t address, std::span values) Libs::LibKernel::Memory::TryReadBufferBacking(address, values.data(), values.size_bytes()); } -void DumpShaderSpirv(const char* stage_name, uint64_t shader_hash, - const std::vector& spirv) { +void DumpShader(std::span words, const std::string& name) { if (!Config::GraphicsDebugDumpEnabled()) { return; } - static std::atomic_int id = 0; - const auto path = Config::GetShaderLogFolder() / fmt::format("{:04d}_new_shader_{}_{:016x}.spv", - id++, stage_name, shader_hash); + const auto path = Config::GetShaderLogFolder() / name; Common::File::CreateDirectories(path.parent_path()); Common::File file(path); if (file.IsInvalid()) { @@ -129,32 +125,13 @@ void DumpShaderSpirv(const char* stage_name, uint64_t shader_hash, LOGF_COLOR(Log::Color::BrightRed, "Can't create file: %s\n", path_text.c_str()); return; } - file.Write(spirv.data(), spirv.size() * sizeof(uint32_t)); + file.Write(words.data(), words.size_bytes()); } -void DumpShaderOriginal(const char* stage_name, uint64_t shader_hash, - std::span code) { - if (!Config::GraphicsDebugDumpEnabled()) { - return; - } - EXIT_IF(code.empty()); - static std::atomic_int id = 0; - const auto path = Config::GetShaderLogFolder() / "original" / - fmt::format("{:04d}_new_shader_{}_{:016x}.bin", id++, stage_name, shader_hash); - Common::File::CreateDirectories(path.parent_path()); - Common::File file(path); - if (file.IsInvalid()) { - const auto path_text = Common::PathToString(path); - LOGF_COLOR(Log::Color::BrightRed, "Can't create file: %s\n", path_text.c_str()); - return; - } - file.Write(code.data(), code.size_bytes()); -} - -bool ValidateShaderSpirv(const char* label, uint64_t shader_hash, +void ValidateShaderSpirv(const char* stage, uint64_t shader_hash, const std::vector& spirv, bool maintenance8_enabled) { if (!Config::ShaderValidationEnabled()) { - return true; + return; } spvtools::SpirvTools tools(SPV_ENV_VULKAN_1_3); std::string messages; @@ -167,7 +144,7 @@ bool ValidateShaderSpirv(const char* label, uint64_t shader_hash, spvtools::ValidatorOptions validator_options; validator_options.SetAllowOffsetTextureOperand(maintenance8_enabled); if (tools.Validate(spirv.data(), spirv.size(), validator_options)) { - return true; + return; } spvtools::SpirvTools disassembler(SPV_ENV_VULKAN_1_2); std::string text; @@ -177,10 +154,48 @@ bool ValidateShaderSpirv(const char* label, uint64_t shader_hash, static_cast(SPV_BINARY_TO_TEXT_OPTION_COMMENT) | static_cast(SPV_BINARY_TO_TEXT_OPTION_INDENT) | static_cast(SPV_BINARY_TO_TEXT_OPTION_COLOR)); - LOGF_COLOR(Log::Color::BrightRed, "%s SPIR-V validation failed hash=0x%016" PRIx64 ":\n%s", - label, shader_hash, messages.c_str()); LOGF("%s\n", text.c_str()); - return false; + EXIT("ShaderRecompiler %s SPIR-V validation failed hash=0x%016" PRIx64 ":\n%s", stage, + shader_hash, messages.c_str()); +} + +ShaderRecompiler::Spirv::Profile SpirvProfile(const GraphicContext& graphics) { + // AMD GPUs (vendor 0x1002) run NMin/NMax as v_min/v_max, which give the guest's NaN and + // signed-zero results; GLSL.std.450 alone does not order -0 below +0. + return {.maintenance8 = graphics.maintenance8_enabled, + .native_float_min_max = graphics.physical_device_properties.vendorID == 0x1002u, + .device_clock_period_ps = graphics.device_clock_period_ps}; +} + +static_assert(sizeof(ShaderRecompiler::IR::ResourceSpecialization::Buffer) == 28 && + sizeof(ShaderRecompiler::IR::ResourceSpecialization::Image) == 36 && + sizeof(ShaderRecompiler::IR::ResourceSpecialization::Sampler) == 12, + "PackSpecialization must list every field"); + +// Appends every field as one word, each list preceded by its size. +void PackSpecialization(const ShaderRecompiler::IR::ResourceSpecialization& specialization, + std::vector& words) { + const auto word = [](auto value) { return static_cast(value); }; + words.push_back(word(specialization.buffers.size())); + for (const auto& buffer: specialization.buffers) { + words.insert(words.end(), {buffer.packed_stride, word(buffer.descriptor_format), + buffer.descriptor_swizzle, word(buffer.zero_stride_oob), + word(buffer.scalar_empty), buffer.indirect_root, + buffer.indirect_mapping_offset, buffer.indirect_search_iterations}); + } + words.push_back(word(specialization.images.size())); + for (const auto& image: specialization.images) { + words.insert(words.end(), {word(image.numeric_class), word(image.dimension), image.mip_count, + word(image.conversion_format), image.shader_swizzle, + image.indirect_root, image.indirect_mapping_offset, + image.indirect_search_iterations, word(image.cube), + word(image.fmask), word(image.heap)}); + } + words.push_back(word(specialization.samplers.size())); + for (const auto& sampler: specialization.samplers) { + words.insert(words.end(), {sampler.indirect_root, sampler.indirect_mapping_offset, + sampler.indirect_search_iterations}); + } } } // namespace @@ -211,288 +226,203 @@ std::size_t PipelineCache::GraphicsPipelineKeyHash::operator()(const GraphicsPip return hash; } -struct PipelineCache::ProgramCache { - struct ProgramKey { - ShaderType stage = ShaderType::Unknown; - uint64_t hash = 0; - uint32_t user_data_count = 0; - uint32_t code_size = 0; - std::vector static_state; +struct PipelineCache::Module { + std::vector function_code; + ShaderRecompiler::IR::ResourceSpecialization specialization; + ShaderRecompiler::IR::CompiledShaderInfo info; + ShaderProgram handle; +}; - bool operator==(const ProgramKey&) const = default; - }; - - struct Permutation { - std::vector function_code; - ShaderRecompiler::IR::ResourceSpecialization specialization; - ShaderRecompiler::IR::CompiledShaderInfo program; - ShaderProgram handle; - }; - - struct SourceEntry { - SourceEntry(ShaderRecompiler::IR::ResourcePlan plan, - std::unique_ptr call_source) - : call_source(std::move(call_source)), resource_plan(std::move(plan)) { - permutations.reserve(8); - } - - std::span FunctionCode() const { - return call_source - ? std::span(call_source->linked->code).subspan(call_source->code.size()) - : std::span{}; - } - - bool RefreshCallSource(const ShaderRecompiler::IR::SrtRuntime& runtime) { - const auto revision = call_source->revision; - ShaderRecompiler::RefreshShaderSource(*call_source, runtime); - resource_plan.source_reads = call_source->reads; - return revision != call_source->revision; - } - - template - Permutation* FindPermutation(const ShaderRecompiler::IR::SrtRuntime& runtime, - uint32_t push_data_cursor) { - EXIT_IF(!ShaderRecompiler::IR::MaterializeResources( - resource_plan, runtime, resources, specialization)); - for (auto& candidate: permutations) { - const auto& layout = candidate.program.bindings; - if (layout.push_data_start_dword != ShaderRecompiler::IR::PushData::StartFor( - push_data_cursor, layout.ShaderDataDwords()) || - candidate.specialization != specialization) { - continue; - } - if constexpr (HasCalls) { - if (!std::ranges::equal(candidate.function_code, FunctionCode())) continue; - } - return &candidate; - } - return nullptr; - } - - std::unique_ptr call_source; - ShaderRecompiler::IR::ResourcePlan resource_plan; - ShaderRecompiler::IR::ResourceSnapshot resources; - ShaderRecompiler::IR::ResourceSpecialization specialization; - std::vector permutations; - bool skip_dispatch = false; - }; - - struct ProgramKeyHash { - std::size_t operator()(const ProgramKey& key) const { - std::size_t hash = static_cast(key.stage); - PipelineKeyHash::Mix(hash, static_cast(key.hash)); - if constexpr (sizeof(std::size_t) < sizeof(uint64_t)) { - PipelineKeyHash::Mix(hash, static_cast(key.hash >> 32u)); - } - PipelineKeyHash::Mix(hash, key.user_data_count); - PipelineKeyHash::Mix(hash, key.code_size); - PipelineKeyHash::Mix(hash, key.static_state.size()); - // Bucket same-shape static variants by source. ProgramKey equality performs the one - // exact state comparison needed on a stable hit without hashing the full state first. - return hash; - } - }; - - static constexpr std::size_t MaxStaticKeyWords = 32 + ShaderVertexInputInfo::RES_MAX * 6; - - Permutation CompilePermutation(const char* stage_name, - const ShaderRecompiler::CompileOptions& options, - ShaderRecompiler::TranslateResult translated, - ShaderRecompiler::IR::ResourceSpecialization specialization, - uint32_t push_data_start_dword, - std::span function_code) { - Common::DebugCounters::ScopedTimer timer(Common::DebugCounters::Counter::ShadersCompiled, - Common::DebugCounters::Counter::ShaderCompileNs, - Common::DebugCounters::Peak::ShaderCompileNs); - auto result = ShaderRecompiler::CompileProgram(std::move(translated), options, - specialization, push_data_start_dword); - if (!ValidateShaderSpirv(options.dump_label, options.shader_hash, result.spirv, - options.profile->maintenance8)) { - DumpShaderSpirv(stage_name, options.shader_hash, result.spirv); - EXIT("%s failed hash=0x%016" PRIx64 ": SPIR-V validation failed\n", options.dump_label, - options.shader_hash); - } - DumpShaderSpirv(stage_name, options.shader_hash, result.spirv); - - const auto module = CompileSPV(result.spirv, graphics.device); - EXIT_IF(module == nullptr); - if (options.dump_ir) { - LOGF("%s SPIR-V words=%" PRIu64 " wave_size=%u\n", options.dump_label, - static_cast(result.spirv.size()), options.wave_size); - } - return { - .function_code = {function_code.begin(), function_code.end()}, - .specialization = std::move(specialization), - .program = std::move(result.program).TakeCompiledInfo(), - .handle = {.id = ++next_shader_id, .module = module}, - }; +struct PipelineCache::Program { + std::span FunctionCode() const { + return call_source + ? std::span(call_source->linked->code).subspan(call_source->code.size()) + : std::span{}; } - template - ShaderProgram Get(const ShaderParams& params, InputInfo& input_info, - uint32_t& push_data_cursor) { - const auto user_data = std::span(params.user_data).first(params.user_data_count); - ShaderRecompiler::IR::SrtRuntime runtime { - .user_data = user_data, - .shader_base = params.Base(), - .read_specialization_memory = ReadShaderGuestMemory, - .image_heap = image_heap, - }; - ShaderRecompiler::CompileOptions options {.wave_size = input_info.wave_size}; - if constexpr (std::is_same_v) { - options.stage = input_info.logical_stage; - options.input_info.vertex = &input_info; - options.user_data_base = 8; - if (options.stage == ShaderType::Mesh || - options.stage == ShaderType::TessellationControl) { - options.user_data_base = 0; - options.wave_size = - options.stage == ShaderType::Mesh ? input_info.mesh.wave_size : 64u; - } - } else if constexpr (std::is_same_v) { - options.stage = ShaderType::Pixel; - options.input_info.pixel = &input_info; - } else { - static_assert(std::is_same_v); - options.stage = ShaderType::Compute; - options.input_info.compute = &input_info; - runtime.workgroup_counts = input_info.workgroup_counts; - runtime.indirect_args_address = input_info.indirect_args_address; - } - lookup_key.stage = options.stage; - lookup_key.hash = params.hash; - lookup_key.user_data_count = params.user_data_count; - lookup_key.code_size = static_cast(params.code.size()); - BuildStageStaticKey(input_info, lookup_key.static_state); - auto entry = programs.find(lookup_key); - const auto bind = [&](const Permutation& permutation) { - input_info.stage = {.program = &permutation.program, - .resources = &entry->second.resources, - .handle = &permutation.handle}; - permutation.program.bindings.AdvancePushData(push_data_cursor); - return permutation.handle; - }; - bool source_changed = false; - if (entry != programs.end()) { - auto& cached = entry->second; - if (cached.skip_dispatch) return {}; - Permutation* permutation = nullptr; - if (cached.call_source) { - source_changed = cached.RefreshCallSource(runtime); - if (!source_changed) permutation = cached.FindPermutation(runtime, push_data_cursor); - } else { - permutation = cached.FindPermutation(runtime, push_data_cursor); - } - if (permutation != nullptr) return bind(*permutation); - } + bool RefreshCallSource(const ShaderRecompiler::IR::SrtRuntime& runtime) { + const auto revision = call_source->revision; + ShaderRecompiler::RefreshShaderSource(*call_source, runtime); + resource_plan.source_reads = call_source->reads; + return revision != call_source->revision; + } - const char* label = nullptr; - const char* stage_name = nullptr; - switch (options.stage) { - case ShaderType::Vertex: label = "ShaderRecompiler VS"; stage_name = "vs"; break; - case ShaderType::Mesh: label = "ShaderRecompiler MS"; stage_name = "ms"; break; - case ShaderType::Local: label = "ShaderRecompiler LS"; stage_name = "ls"; break; - case ShaderType::TessellationControl: label = "ShaderRecompiler HS"; stage_name = "hs"; break; - case ShaderType::TessellationEvaluation: label = "ShaderRecompiler DS"; stage_name = "ds"; break; - case ShaderType::Pixel: label = "ShaderRecompiler PS"; stage_name = "ps"; break; - case ShaderType::Compute: label = "ShaderRecompiler CS"; stage_name = "cs"; break; - default: EXIT("invalid pipeline shader stage\n"); + template + Module* FindModule(const ShaderRecompiler::IR::SrtRuntime& runtime, uint32_t push_data_cursor) { + EXIT_IF(!ShaderRecompiler::IR::MaterializeResources( + resource_plan, runtime, resources, specialization)); + for (auto& candidate: modules) { + const auto& layout = candidate.info.bindings; + if (layout.push_data_start_dword != ShaderRecompiler::IR::PushData::StartFor( + push_data_cursor, layout.ShaderDataDwords()) || + candidate.specialization != specialization) { + continue; + } + if constexpr (HasCalls) { + if (!std::ranges::equal(candidate.function_code, FunctionCode())) continue; + } + return &candidate; } - options.profile = &profile; - options.shader_hash = params.hash; - options.user_data = user_data; - options.back_code = params.back_code; - options.dump_ir = Config::GetShaderLogDirection() != Config::LogDirection::Silent; - options.early_dump = options.dump_ir; - options.emit_debug_info = Config::GraphicsDebugDumpEnabled(); - options.dump_label = label; - DumpShaderOriginal(stage_name, options.shader_hash, params.code); - ShaderRecompiler::ShaderSource source; - if (entry == programs.end()) source = ShaderRecompiler::PrepareShaderSource(params.code, options); + return nullptr; + } + + std::unique_ptr call_source; + ShaderRecompiler::IR::ResourcePlan resource_plan; + ShaderRecompiler::IR::ResourceSnapshot resources; + ShaderRecompiler::IR::ResourceSpecialization specialization; + std::vector modules; + bool skip_dispatch = false; +}; + +PipelineCache::Module& PipelineCache::CompileModule(Program& program, const ProgramKey& key, + ShaderRecompiler::CompileOptions options, + ShaderRecompiler::TranslateResult translated, + uint32_t push_data_cursor) { + Common::DebugCounters::ScopedTimer timer(Common::DebugCounters::Counter::ShadersCompiled, + Common::DebugCounters::Counter::ShaderCompileNs, + Common::DebugCounters::Peak::ShaderCompileNs); + const auto profile = SpirvProfile(m_graphics); + options.profile = &profile; + auto result = ShaderRecompiler::CompileProgram(std::move(translated), options, + program.specialization, push_data_cursor); + auto info = std::move(result.program).TakeCompiledInfo(); + + // The same inputs give the same module id across runs. + const auto function_code = program.FunctionCode(); + std::vector words {static_cast(key.stage), static_cast(key.hash), + static_cast(key.hash >> 32u), key.user_data_count, + static_cast(key.static_state.size())}; + words.insert(words.end(), key.static_state.begin(), key.static_state.end()); + words.push_back(info.bindings.push_data_start_dword); + words.push_back(static_cast(function_code.size())); + words.insert(words.end(), function_code.begin(), function_code.end()); + PackSpecialization(program.specialization, words); + const auto id = XXH3_64bits(words.data(), words.size() * sizeof(uint32_t)); + + const auto* stage = ShaderRecompiler::StageName(key.stage); + DumpShader(result.spirv, fmt::format("{}_{:016x}_{:016x}.spv", stage, key.hash, id)); + ValidateShaderSpirv(stage, key.hash, result.spirv, profile.maintenance8); + const auto module = CompileSPV(result.spirv, m_graphics.device); + EXIT_IF(module == nullptr); + return program.modules.emplace_back(Module { + .function_code = {function_code.begin(), function_code.end()}, + .specialization = program.specialization, + .info = std::move(info), + .handle = {.id = id, .module = module}, + }); +} + +template +ShaderProgram PipelineCache::GetProgram(const ShaderParams& params, InputInfo& input_info, + uint32_t& push_data_cursor) { + const auto user_data = std::span(params.user_data).first(params.user_data_count); + ShaderRecompiler::IR::SrtRuntime runtime { + .user_data = user_data, + .shader_base = params.Base(), + .read_specialization_memory = ReadShaderGuestMemory, + .image_heap = m_image_heap, + }; + ShaderRecompiler::CompileOptions options {.wave_size = input_info.wave_size}; + if constexpr (std::is_same_v) { + options.stage = input_info.logical_stage; + options.input_info.vertex = &input_info; + options.user_data_base = 8; + if (options.stage == ShaderType::Mesh || + options.stage == ShaderType::TessellationControl) { + options.user_data_base = 0; + options.wave_size = + options.stage == ShaderType::Mesh ? input_info.mesh.wave_size : 64u; + } + } else if constexpr (std::is_same_v) { + options.stage = ShaderType::Pixel; + options.input_info.pixel = &input_info; + } else { + static_assert(std::is_same_v); + options.stage = ShaderType::Compute; + options.input_info.compute = &input_info; + runtime.workgroup_counts = input_info.workgroup_counts; + runtime.indirect_args_address = input_info.indirect_args_address; + } + m_lookup_key.stage = options.stage; + m_lookup_key.hash = params.hash; + m_lookup_key.user_data_count = params.user_data_count; + BuildStageStaticKey(input_info, m_lookup_key.static_state); + auto entry = m_program_cache.find(m_lookup_key); + const auto bind = [&](const Module& module) { + input_info.stage = {.program = &module.info, + .resources = &entry->second->resources, + .handle = &module.handle}; + module.info.bindings.AdvancePushData(push_data_cursor); + return module.handle; + }; + bool source_changed = false; + if (entry != m_program_cache.end()) { + auto& cached = *entry->second; + if (cached.skip_dispatch) return {}; + Module* module = nullptr; + if (cached.call_source) { + source_changed = cached.RefreshCallSource(runtime); + if (!source_changed) module = cached.FindModule(runtime, push_data_cursor); + } else { + module = cached.FindModule(runtime, push_data_cursor); + } + if (module != nullptr) return bind(*module); + } + + options.shader_hash = params.hash; + options.user_data = user_data; + options.back_code = params.back_code; + options.dump_ir = Config::GetShaderLogDirection() != Config::LogDirection::Silent; + options.emit_debug_info = Config::GraphicsDebugDumpEnabled(); + ShaderRecompiler::ShaderSource source; + if (entry == m_program_cache.end()) { + DumpShader(params.code, fmt::format("original/{}_{:016x}.bin", + ShaderRecompiler::StageName(options.stage), params.hash)); + source = ShaderRecompiler::PrepareShaderSource(params.code, options); if (source.unsupported_call_pc != UINT32_MAX) { PipelineCacheLog("!!! TEMPORARY COMPUTE DISPATCH SKIP !!! shader=0x{:016x} pc=0x{:08x} " "reason=unsupported GPU-selected callable target; rendering is incomplete", params.hash, source.unsupported_call_pc); - programs.try_emplace(lookup_key, ShaderRecompiler::IR::ResourcePlan{}, nullptr) - .first->second.skip_dispatch = true; + m_program_cache.emplace(m_lookup_key, std::make_unique()) + .first->second->skip_dispatch = true; return {}; } - auto* current = entry == programs.end() ? &source : entry->second.call_source.get(); - ShaderRecompiler::TranslateResult translated; - if (current != nullptr) { - if (!current->calls.empty()) { - if (entry == programs.end()) ShaderRecompiler::RefreshShaderSource(*current, runtime); - translated = ShaderRecompiler::TranslateProgram(current->linked->decoded, options); - translated.program.source_reads = current->reads; - } else { - translated = ShaderRecompiler::TranslateProgram(current->decoded, options); - } + } + auto* current = entry == m_program_cache.end() ? &source : entry->second->call_source.get(); + ShaderRecompiler::TranslateResult translated; + if (current != nullptr) { + if (!current->calls.empty()) { + if (entry == m_program_cache.end()) ShaderRecompiler::RefreshShaderSource(*current, runtime); + translated = ShaderRecompiler::TranslateProgram(current->linked->decoded, options); + translated.program.source_reads = current->reads; } else { - translated = ShaderRecompiler::TranslateProgram(params.code, options); + translated = ShaderRecompiler::TranslateProgram(current->decoded, options); } - if (entry == programs.end()) { - auto retained = !source.calls.empty() ? std::make_unique(std::move(source)) : nullptr; - entry = programs.try_emplace(lookup_key, - ShaderRecompiler::IR::ExtractResourcePlan(translated.program), std::move(retained)).first; - EXIT_IF(!ShaderRecompiler::IR::MaterializeResources( - entry->second.resource_plan, runtime, entry->second.resources, - entry->second.specialization)); - } else if (source_changed) { - entry->second.resource_plan = ShaderRecompiler::IR::ExtractResourcePlan(translated.program); - if (const auto* permutation = entry->second.FindPermutation(runtime, push_data_cursor)) { - return bind(*permutation); - } - } - entry->second.permutations.push_back(CompilePermutation( - stage_name, options, std::move(translated), entry->second.specialization, push_data_cursor, - entry->second.FunctionCode())); - - std::array(ShaderType::TessellationEvaluation) + 1> counts {}; - for (const auto& [key, source]: programs) { - counts[static_cast(key.stage)] += source.permutations.size(); - } - // Guest geometry shaders are compiled through the host mesh stage. - std::printf("Shaders: VS %zu | PS %zu | CS %zu | GS %zu | LS %zu | HS %zu | TES %zu\n", - counts[static_cast(ShaderType::Vertex)], - counts[static_cast(ShaderType::Pixel)], - counts[static_cast(ShaderType::Compute)], - counts[static_cast(ShaderType::Mesh)], - counts[static_cast(ShaderType::Local)], - counts[static_cast(ShaderType::TessellationControl)], - counts[static_cast(ShaderType::TessellationEvaluation)]); - return bind(entry->second.permutations.back()); + } else { + translated = ShaderRecompiler::TranslateProgram(params.code, options); } - - ProgramCache(GraphicContext& graphics, ShaderRecompiler::IR::ImageHeapSource* image_heap) - : graphics(graphics), image_heap(image_heap), - // AMD GPUs (vendor 0x1002) run NMin/NMax as v_min/v_max, which give the guest's NaN and - // signed-zero results; GLSL.std.450 alone does not order -0 below +0. - profile {.maintenance8 = graphics.maintenance8_enabled, - .native_float_min_max = - graphics.physical_device_properties.vendorID == 0x1002u, - .device_clock_period_ps = graphics.device_clock_period_ps} { - lookup_key.static_state.reserve(MaxStaticKeyWords); - } - ~ProgramCache() { - for (const auto& [key, entry]: programs) { - (void)key; - for (const auto& permutation: entry.permutations) { - graphics.device.destroyShaderModule(permutation.handle.module, nullptr); - } + if (entry == m_program_cache.end()) { + auto retained = !source.calls.empty() + ? std::make_unique(std::move(source)) + : nullptr; + entry = m_program_cache.emplace(m_lookup_key, std::make_unique( + std::move(retained), ShaderRecompiler::IR::ExtractResourcePlan(translated.program))).first; + EXIT_IF(!ShaderRecompiler::IR::MaterializeResources( + entry->second->resource_plan, runtime, entry->second->resources, + entry->second->specialization)); + } else if (source_changed) { + entry->second->resource_plan = ShaderRecompiler::IR::ExtractResourcePlan(translated.program); + if (const auto* module = entry->second->FindModule(runtime, push_data_cursor)) { + return bind(*module); } } - - std::unordered_map programs; - ProgramKey lookup_key; - GraphicContext& graphics; - ShaderRecompiler::IR::ImageHeapSource* image_heap; - const ShaderRecompiler::Spirv::Profile profile; - uint64_t next_shader_id = 0; -}; + return bind(CompileModule(*entry->second, entry->first, options, std::move(translated), + push_data_cursor)); +} PipelineCache::PipelineCache(GraphicContext& graphics, ShaderRecompiler::IR::ImageHeapSource* image_heap) - : m_graphics(graphics), m_program_cache(std::make_unique(graphics, image_heap)), + : m_graphics(graphics), m_image_heap(image_heap), m_next_driver_cache_save(std::chrono::steady_clock::now() + DriverCacheSaveInterval) { EXIT_NOT_IMPLEMENTED(!Common::Thread::IsMainThread()); InitializeDriverCache(); @@ -501,8 +431,7 @@ PipelineCache::PipelineCache(GraphicContext& graphics, PipelineCache::~PipelineCache() { Save(); auto destroy = [this](const auto& pipelines) { - for (const auto& [key, pipeline]: pipelines) { - (void)key; + for (const auto& pipeline: pipelines | std::views::values) { m_graphics.device.destroyPipeline(pipeline.pipeline, nullptr); m_graphics.device.destroyPipelineLayout(pipeline.pipeline_layout, nullptr); m_graphics.device.destroyDescriptorSetLayout(pipeline.descriptor_set_layout, nullptr); @@ -510,6 +439,11 @@ PipelineCache::~PipelineCache() { }; destroy(m_graphics_pipelines); destroy(m_compute_pipelines); + for (const auto& program: m_program_cache | std::views::values) { + for (const auto& module: program->modules) { + m_graphics.device.destroyShaderModule(module.handle.module, nullptr); + } + } if (m_driver_cache != nullptr) { m_graphics.device.destroyPipelineCache(m_driver_cache, nullptr); } @@ -832,7 +766,7 @@ void PipelineCache::GetGraphicsPrograms(const CommandBuffer& command, DrawRender uint32_t push_data_cursor = ShaderRecompiler::IR::PushData::GraphicsDrawDwords( mesh_active, object_primitive_id, last_stage.user_clip_planes); if (pixel_active) { - m_program_cache->Get(pixel_params, pixel_info, push_data_cursor); + GetProgram(pixel_params, pixel_info, push_data_cursor); for (const auto& output: pixel_info.stage.program->info.outputs) { if (output.kind == ShaderRecompiler::IR::StageOutputKind::Mrt || output.kind == ShaderRecompiler::IR::StageOutputKind::ObjectPrimitiveId) { @@ -842,7 +776,7 @@ void PipelineCache::GetGraphicsPrograms(const CommandBuffer& command, DrawRender state.mrt_mask &= color_output_mask; } for (uint32_t i = 0; i < (tess_active ? 3u : 1u); i++) { - m_program_cache->Get(vertex_params[i], vertex_info[i], push_data_cursor); + GetProgram(vertex_params[i], vertex_info[i], push_data_cursor); } } @@ -853,7 +787,7 @@ ShaderProgram PipelineCache::GetComputeProgram(const HW::ComputeShaderInfo& regs input_info.lds_storage = input_info.lds_size_dwords * 4u > m_graphics.GetPhysicalDeviceProperties().limits.maxComputeSharedMemorySize; uint32_t push_data_cursor = 0; - return m_program_cache->Get(params, input_info, push_data_cursor); + return GetProgram(params, input_info, push_data_cursor); } bool PipelineStaticParameters::operator==(const PipelineStaticParameters& other) const noexcept { diff --git a/src/graphics/host_gpu/renderer/pipeline/pipelineCache.h b/src/graphics/host_gpu/renderer/pipeline/pipelineCache.h index da9353394..0b92566a3 100644 --- a/src/graphics/host_gpu/renderer/pipeline/pipelineCache.h +++ b/src/graphics/host_gpu/renderer/pipeline/pipelineCache.h @@ -17,12 +17,19 @@ #include #include #include +#include namespace Libs::Graphics { struct GraphicContext; +struct ShaderParams; class CommandBuffer; +namespace ShaderRecompiler { +struct CompileOptions; +struct TranslateResult; +} // namespace ShaderRecompiler + namespace ShaderRecompiler::IR { class ImageHeapSource; } // namespace ShaderRecompiler::IR @@ -95,7 +102,7 @@ struct ShaderProgram { uint64_t id = 0; vk::ShaderModule module = nullptr; - explicit operator bool() const { return id != 0 && module != nullptr; } + explicit operator bool() const { return module != nullptr; } }; // A draw's topology, shader stages (from PipelineCache::GetGraphicsPrograms) and targets. @@ -142,7 +149,22 @@ public: Pipeline& GetComputePipeline(const ShaderComputeInputInfo& input_info); private: - struct ProgramCache; + struct Module; + struct Program; + + struct ProgramKey { + ShaderType stage = ShaderType::Unknown; + uint64_t hash = 0; + uint32_t user_data_count = 0; + std::vector static_state; + + bool operator==(const ProgramKey&) const = default; + }; + + // Variants of one source share a bucket; ProgramKey equality compares the state. + struct ProgramKeyHash { + std::size_t operator()(const ProgramKey& key) const { return key.hash; } + }; struct GraphicsPipelineKey { PipelineRenderingState rendering; @@ -169,8 +191,10 @@ private: std::size_t operator()(const GraphicsPipelineKey& key) const; }; - GraphicContext& m_graphics; - std::unique_ptr m_program_cache; + GraphicContext& m_graphics; + ShaderRecompiler::IR::ImageHeapSource* m_image_heap; + std::unordered_map, ProgramKeyHash> m_program_cache; + ProgramKey m_lookup_key; vk::PipelineCache m_driver_cache = nullptr; std::filesystem::path m_driver_cache_path; std::chrono::steady_clock::time_point m_next_driver_cache_save; @@ -178,6 +202,12 @@ private: std::unordered_map m_graphics_pipelines; std::unordered_map m_compute_pipelines; + template + ShaderProgram GetProgram(const ShaderParams& params, InputInfo& input_info, + uint32_t& push_data_cursor); + Module& CompileModule(Program& program, const ProgramKey& key, + ShaderRecompiler::CompileOptions options, + ShaderRecompiler::TranslateResult translated, uint32_t push_data_cursor); void InitializeDriverCache(); void CheckpointDriverCache(); }; diff --git a/src/graphics/shader/recompiler/ShaderRecompiler.cpp b/src/graphics/shader/recompiler/ShaderRecompiler.cpp index 4530ee01c..dc9a973fd 100644 --- a/src/graphics/shader/recompiler/ShaderRecompiler.cpp +++ b/src/graphics/shader/recompiler/ShaderRecompiler.cpp @@ -30,10 +30,6 @@ namespace Libs::Graphics::ShaderRecompiler { namespace { -const char* GetDumpLabel(const CompileOptions& options) { - return options.dump_label != nullptr ? options.dump_label : "ShaderRecompiler"; -} - std::string MakeIrDump(std::string_view cfg, const IR::Program& ir) { std::string dump = "CFG:\n"; dump += cfg; @@ -44,30 +40,17 @@ std::string MakeIrDump(std::string_view cfg, const IR::Program& ir) { return dump; } -const char* StageName(ShaderType stage) { - switch (stage) { - case ShaderType::Compute: return "CS"; - case ShaderType::Vertex: return "VS"; - case ShaderType::Local: return "LS"; - case ShaderType::TessellationControl: return "HS"; - case ShaderType::TessellationEvaluation: return "TES"; - case ShaderType::Mesh: return "MS"; - case ShaderType::Pixel: return "PS"; - default: return "unknown"; - } -} - void LogDispatcherFallback(const CompileOptions& options, const CFG::Graph& cfg, const char* phase) { const auto* block = cfg.FindBlock(cfg.failure_block); const auto start = block != nullptr ? block->start_pc : UINT32_MAX; const auto end = block != nullptr ? block->end_pc : UINT32_MAX; const auto predecessors = block != nullptr ? block->predecessors.size() : 0u; const auto successors = block != nullptr ? block->successors.size() : 0u; - LOGF("%s CFG dispatcher fallback: stage=%s hash=0x%016" PRIx64 + LOGF("ShaderRecompiler %s CFG dispatcher fallback: hash=0x%016" PRIx64 " phase=%s failure=%s block=%" PRIu32 " pc=0x%08" PRIx32 "..0x%08" PRIx32 " preds=%" PRIu64 " succs=%" PRIu64 " blocks=%" PRIu64 " loops=%" PRIu64 " back_edges=%" PRIu64 " reason=%s\n", - GetDumpLabel(options), StageName(options.stage), options.shader_hash, phase, + StageName(options.stage), options.shader_hash, phase, CFG::FailureKindToString(cfg.failure_kind).c_str(), cfg.failure_block, start, end, static_cast(predecessors), static_cast(successors), static_cast(cfg.blocks.size()), static_cast(cfg.natural_loops.size()), @@ -595,6 +578,19 @@ void PrepareCallTargets(ShaderSource& source, const CompileOptions& options) { } // namespace +const char* StageName(ShaderType stage) { + switch (stage) { + case ShaderType::Compute: return "CS"; + case ShaderType::Vertex: return "VS"; + case ShaderType::Local: return "LS"; + case ShaderType::TessellationControl: return "HS"; + case ShaderType::TessellationEvaluation: return "TES"; + case ShaderType::Mesh: return "MS"; + case ShaderType::Pixel: return "PS"; + default: return "unknown"; + } +} + ShaderSource PrepareShaderSource(std::span code, const CompileOptions& options) { EXIT_IF(code.empty()); ShaderSource source; @@ -764,6 +760,7 @@ TranslateResult TranslateProgram(const Decoder::Program& decoded, const CompileO static_cast(options.stage)); } + const auto* stage = StageName(options.stage); const auto compile_begin = std::chrono::steady_clock::now(); const auto phase_ms = [&compile_begin]() { return static_cast(std::chrono::duration_cast( @@ -773,27 +770,25 @@ TranslateResult TranslateProgram(const Decoder::Program& decoded, const CompileO std::string decoded_dump; if (options.dump_ir) { decoded_dump = Decoder::ProgramToString(decoded); - if (options.early_dump) { - LOGF("%s decoded RDNA2 (early):\n%s", GetDumpLabel(options), decoded_dump.c_str()); - } + LOGF("ShaderRecompiler %s decoded RDNA2:\n%s", stage, decoded_dump.c_str()); } - LOGF("%s phase begin: stage=%s hash=0x%016" PRIx64 " CFG BuildGraph\n", GetDumpLabel(options), - StageName(options.stage), options.shader_hash); + LOGF("ShaderRecompiler %s phase begin: hash=0x%016" PRIx64 " CFG BuildGraph\n", stage, + options.shader_hash); auto native_cfg = CFG::BuildGraph(decoded); CFG::Graph structured_cfg; auto* selected_cfg = &native_cfg; - LOGF("%s phase end: stage=%s hash=0x%016" PRIx64 " CFG BuildGraph blocks=%" PRIu64 + LOGF("ShaderRecompiler %s phase end: hash=0x%016" PRIx64 " CFG BuildGraph blocks=%" PRIu64 " loops=%" PRIu64 " back_edges=%" PRIu64 " elapsed_ms=%" PRIu64 "\n", - GetDumpLabel(options), StageName(options.stage), options.shader_hash, + stage, options.shader_hash, static_cast(native_cfg.blocks.size()), static_cast(native_cfg.natural_loops.size()), static_cast(native_cfg.back_edges.size()), phase_ms()); if (native_cfg.irreducible) { LogDispatcherFallback(options, native_cfg, "build"); } else { - LOGF("%s phase begin: stage=%s hash=0x%016" PRIx64 " CFG Structurize\n", - GetDumpLabel(options), StageName(options.stage), options.shader_hash); + LOGF("ShaderRecompiler %s phase begin: hash=0x%016" PRIx64 " CFG Structurize\n", stage, + options.shader_hash); structured_cfg = CFG::Structurize(native_cfg); if (structured_cfg.unsupported) { native_cfg.unsupported = true; @@ -803,12 +798,12 @@ TranslateResult TranslateProgram(const Decoder::Program& decoded, const CompileO LogDispatcherFallback(options, native_cfg, "structurize"); } else { selected_cfg = &structured_cfg; - LOGF("%s structured CFG success: blocks=%" PRIu64 "\n", GetDumpLabel(options), + LOGF("ShaderRecompiler %s structured CFG success: blocks=%" PRIu64 "\n", stage, static_cast(selected_cfg->blocks.size())); } - LOGF("%s phase end: stage=%s hash=0x%016" PRIx64 " CFG Structurize blocks=%" PRIu64 + LOGF("ShaderRecompiler %s phase end: hash=0x%016" PRIx64 " CFG Structurize blocks=%" PRIu64 " loops=%" PRIu64 " elapsed_ms=%" PRIu64 "\n", - GetDumpLabel(options), StageName(options.stage), options.shader_hash, + stage, options.shader_hash, static_cast(selected_cfg->blocks.size()), static_cast(selected_cfg->natural_loops.size()), phase_ms()); } @@ -821,7 +816,7 @@ TranslateResult TranslateProgram(const Decoder::Program& decoded, const CompileO decoded, *options.input_info.vertex, options.user_data_base, static_cast(options.user_data.size()), options.wave_size); if (!embedded_fetch.loads.empty()) { - LOGF("%s embedded vertex fetch plan: detected=%" PRIu64 "\n", GetDumpLabel(options), + LOGF("ShaderRecompiler %s embedded vertex fetch plan: detected=%" PRIu64 "\n", stage, static_cast(embedded_fetch.loads.size())); } } @@ -834,12 +829,12 @@ TranslateResult TranslateProgram(const Decoder::Program& decoded, const CompileO .input_info = options.input_info, .embedded_fetch = embedded_fetch.loads.empty() ? nullptr : &embedded_fetch, }; - LOGF("%s phase begin: stage=%s hash=0x%016" PRIx64 " IR TranslateProgram\n", - GetDumpLabel(options), StageName(options.stage), options.shader_hash); + LOGF("ShaderRecompiler %s phase begin: hash=0x%016" PRIx64 " IR TranslateProgram\n", stage, + options.shader_hash); auto ir = Frontend::TranslateProgram(decoded, cfg, translate_options); - LOGF("%s phase end: stage=%s hash=0x%016" PRIx64 " IR TranslateProgram blocks=%" PRIu64 + LOGF("ShaderRecompiler %s phase end: hash=0x%016" PRIx64 " IR TranslateProgram blocks=%" PRIu64 " elapsed_ms=%" PRIu64 "\n", - GetDumpLabel(options), StageName(options.stage), options.shader_hash, + stage, options.shader_hash, static_cast(ir.blocks.size()), phase_ms()); IR::RewriteToSsa(ir.blocks); const auto simplify = [&ir] { @@ -850,8 +845,9 @@ TranslateResult TranslateProgram(const Decoder::Program& decoded, const CompileO simplify(); const auto read_lane_stats = IR::EliminateReadLane(ir, ir.wave_size); if (read_lane_stats.rewritten_reads != 0 || read_lane_stats.rewritten_reductions != 0) { - LOGF("%s read-lane elimination: reads=%" PRIu32 " reductions=%" PRIu32 "\n", - GetDumpLabel(options), read_lane_stats.rewritten_reads, read_lane_stats.rewritten_reductions); + LOGF("ShaderRecompiler %s read-lane elimination: reads=%" PRIu32 + " reductions=%" PRIu32 "\n", + stage, read_lane_stats.rewritten_reads, read_lane_stats.rewritten_reductions); } // Cleanup also exposes profitable folds by removing dead predicate consumers. simplify(); @@ -865,10 +861,8 @@ TranslateResult TranslateProgram(const Decoder::Program& decoded, const CompileO std::string cfg_dump; if (options.dump_ir) { cfg_dump = CFG::GraphToString(cfg); - if (options.early_dump) { - LOGF("%s native IR before resource tracking:\n%s", GetDumpLabel(options), - MakeIrDump(cfg_dump, ir).c_str()); - } + LOGF("ShaderRecompiler %s native IR before resource tracking:\n%s", stage, + MakeIrDump(cfg_dump, ir).c_str()); } IR::TrackResources(ir, decoded, native_cfg, options.input_info); TranslateResult result; @@ -885,6 +879,7 @@ CompileResult CompileProgram(TranslateResult translated, const CompileOptions& o uint32_t push_data_start_dword) { const auto emit_begin = std::chrono::steady_clock::now(); auto& ir = translated.program; + const auto* stage = StageName(ir.stage); IR::ApplyResourceSpecialization(ir, specialization); // The resource plan owns host descriptor evaluation now. Keep only dependencies consumed // by GPU memory operations; bound descriptor dwords must not retain shader instructions. @@ -931,19 +926,16 @@ CompileResult CompileProgram(TranslateResult translated, const CompileOptions& o std::string ir_dump; if (options.dump_ir) { ir_dump = MakeIrDump(translated.cfg_dump, ir); - if (options.early_dump) { - LOGF("%s native IR (early):\n%s", GetDumpLabel(options), ir_dump.c_str()); - } + LOGF("ShaderRecompiler %s native IR:\n%s", stage, ir_dump.c_str()); } - LOGF("%s phase begin: stage=%s hash=0x%016" PRIx64 " SPIR-V EmitProgram\n", - GetDumpLabel(options), StageName(ir.stage), ir.shader_hash); + LOGF("ShaderRecompiler %s phase begin: hash=0x%016" PRIx64 " SPIR-V EmitProgram\n", stage, + ir.shader_hash); auto spirv = Spirv::EmitProgram(ir, options.input_info, push_data_start_dword, *options.profile, options.emit_debug_info); - LOGF("%s phase end: stage=%s hash=0x%016" PRIx64 " SPIR-V EmitProgram words=%" PRIu64 - " elapsed_ms=%" PRIu64 "\n", - GetDumpLabel(options), StageName(ir.stage), ir.shader_hash, - static_cast(spirv.size()), + LOGF("ShaderRecompiler %s phase end: hash=0x%016" PRIx64 " SPIR-V EmitProgram words=%" PRIu64 + " wave_size=%u elapsed_ms=%" PRIu64 "\n", + stage, ir.shader_hash, static_cast(spirv.size()), ir.wave_size, static_cast(std::chrono::duration_cast( std::chrono::steady_clock::now() - emit_begin) .count())); diff --git a/src/graphics/shader/recompiler/ShaderRecompiler.h b/src/graphics/shader/recompiler/ShaderRecompiler.h index a0dad18bf..1dd5a409a 100644 --- a/src/graphics/shader/recompiler/ShaderRecompiler.h +++ b/src/graphics/shader/recompiler/ShaderRecompiler.h @@ -22,10 +22,8 @@ struct CompileOptions { uint32_t user_data_base = 0; uint64_t shader_hash = 0; bool dump_ir = true; - bool early_dump = false; bool emit_debug_info = false; const Spirv::Profile* profile = &Spirv::DefaultProfile; - const char* dump_label = nullptr; std::span user_data; std::span back_code; ShaderStageInputInfo input_info; @@ -69,6 +67,7 @@ struct ShaderSource { std::vector> reads; }; +[[nodiscard]] const char* StageName(ShaderType stage); [[nodiscard]] ShaderSource PrepareShaderSource(std::span code, const CompileOptions& options); const Decoder::Program& RefreshShaderSource(ShaderSource& source, const IR::SrtRuntime& runtime); diff --git a/tests/ShaderRecompilerComputeTests.cpp b/tests/ShaderRecompilerComputeTests.cpp index 57613f983..1373a1387 100644 --- a/tests/ShaderRecompilerComputeTests.cpp +++ b/tests/ShaderRecompilerComputeTests.cpp @@ -44555,6 +44555,55 @@ void CheckComputeLdsLimit(VulkanHarness &vulkan) { std::printf("[gpu] %-32s ok\n", name); } +// Module ids come from the module inputs, not from the order of compilation. +void CheckStableProgramIds(VulkanHarness &vulkan) { + constexpr const char *name = "StableProgramIds"; + const auto make_code = [](u32 value) { + std::vector code; + AppendVMovU32(&code, 1, value); + code.insert(code.end(), {EncodeMubuf0(0x1c), EncodeMubuf1(1, 0, 0)}); + AppendEnd(&code); + return code; + }; + const std::array codes{make_code(7), make_code(9)}; + for (const auto &code : codes) + ShaderMapUserData(reinterpret_cast(code.data()), + {.type = Prospero::ShaderBinaryType::kCs, + .code_size_bytes = static_cast(code.size() * sizeof(u32))}); + // The first two share code and differ only in the buffer stride specialization. + struct Case { u32 code, stride; }; + constexpr std::array cases{{{0, 4}, {0, 8}, {1, 4}}}; + const auto compile = [&](PipelineCache &cache, const Case &test) { + HW::ComputeShaderInfo regs{}; + regs.cs_regs.data_addr = reinterpret_cast(codes[test.code].data()); + regs.cs_regs.num_thread_x = 1; + regs.cs_regs.num_thread_y = 1; + regs.cs_regs.num_thread_z = 1; + regs.cs_regs.wave_size = 32; + regs.cs_regs.user_sgpr = 4; + const ShaderBufferResource buffer{{0x10000u, test.stride << 16u, 64, + DstSel(4, 5, 6, 7) | (static_cast(Prospero::BufferFormat::k32UInt) << 12u)}}; + std::copy_n(buffer.fields, 4, regs.cs_user_sgpr.value); + ShaderComputeInputInfo input{}; + return cache.GetComputeProgram(regs, input).id; + }; + std::array forward{}, reverse{}; + { + PipelineCache cache(vulkan.RuntimeContext(), nullptr); + for (size_t i = 0; i < cases.size(); i++) forward[i] = compile(cache, cases[i]); + } + { + PipelineCache cache(vulkan.RuntimeContext(), nullptr); + for (size_t i = cases.size(); i-- > 0;) reverse[i] = compile(cache, cases[i]); + } + Require(name, "order independence", forward == reverse, + "a program's module id depended on the order it was compiled in"); + const std::set ids(forward.begin(), forward.end()); + Require(name, "distinct modules", ids.size() == forward.size(), + "different code or specialization shared a module id"); + std::printf("[gpu] %-32s ok\n", name); +} + void CheckPs5GameExampleImageClearRuntimeShape() { const auto MakeCode = [] { std::vector code; @@ -50176,6 +50225,11 @@ int main(int argc, char **argv) { CheckComputeThreadDimensions(vulkan); return 0; } + if (argc == 2 && std::strcmp(argv[1], "--program-id-only") == 0) { + VulkanHarness vulkan; + CheckStableProgramIds(vulkan); + return 0; + } if (argc == 2 && std::strcmp(argv[1], "--lds-limit-only") == 0) { VulkanHarness vulkan; CheckComputeLdsLimit(vulkan); @@ -51335,6 +51389,7 @@ int main(int argc, char **argv) { vulkan.CheckGpuTilerCpuParity(); vulkan.CheckNativeIndirectDispatch(); CheckComputeLdsLimit(vulkan); + CheckStableProgramIds(vulkan); vulkan.CheckOrderedDispatch(); vulkan.CheckZeroStrideBufferBinding(); vulkan.CheckNullTextureDescriptors();