pipeline: persist shaders and pipelines per game and prewarm them at boot

src/graphics/host_gpu/renderer/pipeline/pipelineCache.cpp

@@ -17,6 +17,7 @@
 #include "graphics/host_gpu/renderer/renderContext.h"
 #include "graphics/shader/recompiler/BufferFormat.h"
 #include "graphics/shader/recompiler/ShaderRecompiler.h"
+#include "graphics/shader/recompiler/Tessellation.h"
 #include "graphics/shader/recompiler/ir/Serialize.h"
 #include "graphics/shader/shaderCompiler.h"
 #include "kernel/memory.h"
@@ -162,14 +163,6 @@ void ValidateShaderSpirv(const char* stage, uint64_t shader_hash,
 	     shader_hash, messages.c_str());
 }
 
-ShaderRecompiler::Spirv::Profile SpirvProfile(const GraphicContext& graphics) {
-	// AMD GPUs (vendor 0x1002) run NMin/NMax as v_min/v_max, which give the guest's NaN and
-	// signed-zero results; GLSL.std.450 alone does not order -0 below +0.
-	return {.maintenance8           = graphics.maintenance8_enabled,
-	        .native_float_min_max   = graphics.physical_device_properties.vendorID == 0x1002u,
-	        .device_clock_period_ps = graphics.device_clock_period_ps};
-}
-
 // Formats with a narrower host variant drop the components the shader does not read.
 vk::Format NarrowInputFormat(vk::Format format, uint32_t used_components) {
 	switch (format) {
@@ -241,6 +234,26 @@ std::vector<vk::Format> VertexInputFormats(const ShaderVertexInputInfo&
 
 } // namespace
 
+ShaderRecompiler::Spirv::Profile SpirvProfile(const GraphicContext& graphics) {
+	// AMD GPUs (vendor 0x1002) run NMin/NMax as v_min/v_max, which give the guest's NaN and
+	// signed-zero results; GLSL.std.450 alone does not order -0 below +0.
+	return {.maintenance8           = graphics.maintenance8_enabled,
+	        .native_float_min_max   = graphics.physical_device_properties.vendorID == 0x1002u,
+	        .device_clock_period_ps = graphics.device_clock_period_ps};
+}
+
+std::filesystem::path ShaderStorePath() {
+	if (KYTY_BUILD != KYTY_BUILD_RELEASE || !Config::ShaderCacheEnabled() ||
+	    Config::GraphicsDebugDumpEnabled()) {
+		return {};
+	}
+	const auto title_id = PipelineCacheTitleId();
+	if (title_id.empty()) {
+		return {};
+	}
+	return std::filesystem::path("_PipelineCache") / (title_id + ".shaders");
+}
+
 bool GraphicsPipelineKey::operator==(const GraphicsPipelineKey& other) const {
 	return std::memcmp(this, &other, sizeof(*this)) == 0;
 }
@@ -331,6 +344,10 @@ ShaderModule& PipelineCache::CompileModule(Program& program, const ProgramKey& k
 	           options.input_info.pixel->parameter_mode == ShaderPixelParameterMode::Rectangle) {
 		module.rect_parameters = RectListParameters(*options.input_info.pixel, module.info);
 	}
+	// Programs with GPU-selected calls depend on their callees' current code.
+	if (program.call_source == nullptr) {
+		StoreModule(key, program, module, result.spirv);
+	}
 	return module;
 }
 
@@ -442,11 +459,13 @@ const ShaderModule* PipelineCache::GetProgram(const ShaderParams& params, InputI
 }
 
 PipelineCache::PipelineCache(GraphicContext&                         graphics,
-                             ShaderRecompiler::IR::ImageHeapSource* image_heap)
+                             ShaderRecompiler::IR::ImageHeapSource* image_heap,
+                             const std::filesystem::path&           shader_store)
     : m_graphics(graphics), m_image_heap(image_heap),
       m_next_driver_cache_save(std::chrono::steady_clock::now() + DriverCacheSaveInterval) {
 	EXIT_NOT_IMPLEMENTED(!Common::Thread::IsMainThread());
 	InitializeDriverCache();
+	WarmUp(shader_store);
 }
 
 PipelineCache::~PipelineCache() {
@@ -650,6 +669,20 @@ void PipelineCache::GetGraphicsPrograms(const CommandBuffer& command, DrawRender
 	std::array<ShaderParams, 3> vertex_params;
 	if (tess_active) {
 		vertex_params = PrepareTessellationPrograms(vertex_regs, context, vertex_info);
+		// Reflection decodes both programs; the key also covers the VGT fields in tess.
+		const uint64_t layout_key[] = {vertex_params[0].hash, vertex_params[1].hash,
+		                               uint64_t {sh.m_vgtLsHsConfig & 0xfff00u} << 32u |
+		                                   sh.m_vgtTfParam};
+		const auto [layout, inserted] = m_tess_layouts.try_emplace(
+		    XXH3_64bits(layout_key, sizeof(layout_key)), vertex_info[0].tess);
+		if (inserted) {
+			ShaderRecompiler::AnalyzeTessellationPrograms(vertex_params[0].code,
+			                                              vertex_params[1].code, layout->second);
+			StoreLayout(layout->first, layout->second);
+		}
+		for (auto& stage: vertex_info) {
+			stage.tess = layout->second;
+		}
 	} else {
 		vertex_params[0] = PrepareProgram(vertex_regs, context, user_config, vertex_info[0]);
 	}
@@ -1017,6 +1050,7 @@ PipelineCache::Pipeline& PipelineCache::GetGraphicsPipeline(const CommandBuffer&
 		                                         Common::DebugCounters::Peak::PipelineCreateNs);
 		CreatePipelineInternal(m_graphics, pipeline, key, stages, m_driver_cache);
 	}
+	StorePipeline(key);
 	CheckpointDriverCache();
 
 	return pipeline;

src/graphics/host_gpu/renderer/pipeline/pipelineCache.h

@@ -4,6 +4,7 @@
 #include "common/abi.h"
 #include "common/assert.h"
 #include "common/common.h"
+#include "common/file.h"
 #include "graphics/host_gpu/renderer/colorRenderTarget.h"
 #include "graphics/host_gpu/renderer/depthRenderTarget.h"
 #include "graphics/host_gpu/renderer/renderTarget.h"
@@ -37,6 +38,10 @@ namespace ShaderRecompiler::IR {
 class ImageHeapSource;
 } // namespace ShaderRecompiler::IR
 
+namespace ShaderRecompiler::Spirv {
+struct Profile;
+} // namespace ShaderRecompiler::Spirv
+
 #pragma pack(push, 1)
 
 struct PipelineStaticParameters {
@@ -139,7 +144,9 @@ struct DrawRenderState {
 // The owning renderer serializes access, including saves while the GPU is running.
 class PipelineCache {
 public:
-	PipelineCache(GraphicContext& graphics, ShaderRecompiler::IR::ImageHeapSource* image_heap);
+	// Without a shader store path nothing is kept across runs.
+	PipelineCache(GraphicContext& graphics, ShaderRecompiler::IR::ImageHeapSource* image_heap,
+	              const std::filesystem::path& shader_store = {});
 	~PipelineCache();
 	KYTY_CLASS_NO_COPY(PipelineCache);
 	void Save();
@@ -197,6 +204,8 @@ private:
 		std::size_t operator()(const GraphicsPipelineKey& key) const;
 	};
 
+	using ModuleMap = std::unordered_map<uint64_t, const ShaderModule*>;
+
 	GraphicContext&                        m_graphics;
 	ShaderRecompiler::IR::ImageHeapSource* m_image_heap;
 	std::unordered_map<ProgramKey, Program, ProgramKeyHash> m_program_cache;
@@ -209,6 +218,9 @@ private:
 	bool                          m_driver_cache_dirty = false;
 	std::unordered_map<GraphicsPipelineKey, Pipeline, GraphicsPipelineKeyHash> m_graphics_pipelines;
 	std::unordered_map<uint64_t, Pipeline>                                     m_compute_pipelines;
+	std::unordered_map<uint64_t, ShaderTessellationInputInfo>                  m_tess_layouts;
+	// Appends records of everything first created; invalid without a shader store.
+	Common::File m_store;
 
 	template <typename InputInfo>
 	const ShaderModule* GetProgram(const ShaderParams& params, InputInfo& input_info,
@@ -219,8 +231,27 @@ private:
 	                            uint32_t                          push_data_cursor);
 	void InitializeDriverCache();
 	void CheckpointDriverCache();
+
+	// pipelineSerialization.cpp
+	void WarmUp(const std::filesystem::path& path);
+	bool LoadRecord(std::span<const uint8_t>& records, std::vector<GraphicsPipelineKey>& pipelines);
+	void StoreModule(const ProgramKey& key, const Program& program, const ShaderModule& module,
+	                 const std::vector<uint32_t>& spirv);
+	void StorePipeline(const GraphicsPipelineKey& key);
+	void StoreLayout(uint64_t key, const ShaderTessellationInputInfo& layout);
+	// Every module by id, with id 0 naming an absent stage.
+	[[nodiscard]] ModuleMap MapModules() const;
+	// False when a stage id of the key is not in `modules`.
+	bool CreateGraphicsPipeline(Pipeline& pipeline, const GraphicsPipelineKey& key,
+	                            const ModuleMap& modules);
 };
 
+// The shader store of the running title; empty unless --shader-cache is on in a Release build
+// without debug dumps, which change the SPIR-V.
+std::filesystem::path ShaderStorePath();
+// Host capabilities that change the emitted SPIR-V.
+ShaderRecompiler::Spirv::Profile SpirvProfile(const GraphicContext& graphics);
+
 // Stages are indexed like GraphicsPipelineKey::stage_ids.
 void CreatePipelineInternal(GraphicContext& graphics, PipelineCache::Pipeline& pipeline,
                             const GraphicsPipelineKey&                 key,

src/graphics/host_gpu/renderer/pipeline/pipelineSerialization.cpp

@@ -0,0 +1,298 @@
+#include "graphics/host_gpu/renderer/pipeline/pipelineCache.h"
+
+#include "common/logging/log.h"
+#include "common/stringUtils.h"
+#include "graphics/host_gpu/vulkanCommon.h"
+#include "graphics/shader/recompiler/ShaderRecompiler.h"
+#include "graphics/shader/recompiler/ir/Serialize.h"
+
+#include <algorithm>
+#include <chrono>
+#include <cstring>
+#include <fmt/format.h>
+#include <ranges>
+#include <xxhash.h>
+#include <zstd.h>
+
+#if KYTY_PLATFORM == KYTY_PLATFORM_WINDOWS
+#define WIN32_LEAN_AND_MEAN
+#include <windows.h>
+#elif defined(__APPLE__)
+#include <mach-o/dyld.h>
+#endif
+
+// The shader store: a header, then zstd-compressed records of everything a run created first.
+// Records are only appended, so a store cut short by a crash keeps every complete record.
+namespace Libs::Graphics {
+
+namespace {
+
+using ShaderRecompiler::IR::IoAll;
+using ShaderRecompiler::IR::Reader;
+using ShaderRecompiler::IR::Writer;
+
+enum class Record : uint8_t { Module, GraphicsPipeline, TessellationLayout };
+
+// Precedes each record's zstd frame.
+struct Frame {
+	uint32_t compressed_size = 0;
+	uint32_t raw_size        = 0;
+	uint64_t checksum        = 0; // XXH3 of the compressed bytes.
+};
+
+// Changes with every build, so records never meet code that encodes them differently. Zero when
+// the executable cannot be found.
+uint64_t BuildStamp() {
+#if KYTY_PLATFORM == KYTY_PLATFORM_WINDOWS
+	std::array<wchar_t, 32768> name {};
+	GetModuleFileNameW(nullptr, name.data(), static_cast<DWORD>(name.size()));
+	const std::filesystem::path path = name.data();
+#elif defined(__APPLE__)
+	std::array<char, 4096> name {};
+	auto size = static_cast<uint32_t>(name.size());
+	_NSGetExecutablePath(name.data(), &size);
+	const std::filesystem::path path = name.data();
+#else
+	const std::filesystem::path path = "/proc/self/exe";
+#endif
+	std::error_code error;
+	const auto      time = std::filesystem::last_write_time(path, error).time_since_epoch().count();
+	return error ? 0 : static_cast<uint64_t>(time);
+}
+
+// What every record depends on besides its own fields; empty when the build is unknown.
+std::vector<uint8_t> StoreHeader(const GraphicContext& graphics) {
+	const auto stamp = BuildStamp();
+	if (stamp == 0) {
+		return {};
+	}
+	Writer header;
+	IoAll(header, stamp, SpirvProfile(graphics),
+	      graphics.GetPhysicalDeviceProperties().limits.maxComputeSharedMemorySize);
+	return std::move(header.bytes);
+}
+
+void Append(Common::File& store, const Writer& record) {
+	std::vector<uint8_t> bytes(sizeof(Frame) + ZSTD_compressBound(record.bytes.size()));
+	const auto size = ZSTD_compress(bytes.data() + sizeof(Frame), bytes.size() - sizeof(Frame),
+	                                record.bytes.data(), record.bytes.size(), 3);
+	EXIT_IF(ZSTD_isError(size));
+	const Frame frame {.compressed_size = static_cast<uint32_t>(size),
+	                   .raw_size        = static_cast<uint32_t>(record.bytes.size()),
+	                   .checksum        = XXH3_64bits(bytes.data() + sizeof(Frame), size)};
+	std::memcpy(bytes.data(), &frame, sizeof(frame));
+	// A failed write leaves a partial record; the next run drops it and everything after it.
+	store.Write(bytes.data(), static_cast<uint32_t>(sizeof(Frame) + size));
+}
+
+// Record fields, shared by writing and reading.
+template <class S, class Key> void IoKey(S& io, Key& key) {
+	auto& [stage, hash, user_data_count, static_state] = key;
+	IoAll(io, stage, hash, user_data_count, static_state);
+}
+
+template <class S, class Module> void IoModule(S& io, Module& stage) {
+	// The handle comes from the SPIR-V; stored programs have no function code.
+	auto& [id, handle, info, specialization, function_code, vertex_formats, rect_parameters] = stage;
+	IoAll(io, id, info, specialization, vertex_formats, rect_parameters);
+}
+
+} // namespace
+
+void PipelineCache::WarmUp(const std::filesystem::path& path) {
+	if (path.empty()) {
+		return;
+	}
+	const auto header = StoreHeader(m_graphics);
+	if (header.empty()) {
+		return;
+	}
+	const auto           start = std::chrono::steady_clock::now();
+	std::vector<uint8_t> bytes;
+	if (Common::File file;
+	    Common::File::IsFileExisting(path) && file.Open(path, Common::File::Mode::Read)) {
+		bytes.resize(file.Size());
+		uint32_t read = 0;
+		file.Read(bytes.data(), static_cast<uint32_t>(bytes.size()), &read);
+		bytes.resize(read);
+	}
+	std::vector<GraphicsPipelineKey> pipelines;
+	auto                             records = std::span<const uint8_t>(bytes);
+	const bool reusable =
+	    records.size() >= header.size() && std::ranges::equal(records.first(header.size()), header);
+	if (reusable) {
+		records = records.subspan(header.size());
+		while (LoadRecord(records, pipelines)) {
+		}
+	}
+	// A damaged record ends the store; the records before it stay.
+	const auto kept = reusable ? bytes.size() - records.size() : 0;
+	Common::File::CreateDirectories(path.parent_path());
+	if (kept == 0) {
+		if (m_store.Create(path)) {
+			m_store.Write(header.data(), static_cast<uint32_t>(header.size()));
+		}
+	} else if (m_store.Open(path, Common::File::Mode::ReadWrite)) {
+		m_store.Truncate(kept);
+		m_store.Seek(kept);
+	}
+
+	const auto modules = MapModules();
+	for (const auto& key: pipelines) {
+		Pipeline pipeline;
+		if (CreateGraphicsPipeline(pipeline, key, modules)) {
+			m_graphics_pipelines.emplace(key, pipeline);
+		}
+	}
+	// A compute pipeline depends only on its module.
+	for (const auto* module: modules | std::views::values) {
+		if (module != nullptr && module->info.stage == ShaderType::Compute) {
+			CreatePipelineInternal(m_graphics, m_compute_pipelines[module->id], *module,
+			                       m_driver_cache);
+		}
+	}
+	if (!m_graphics_pipelines.empty() || !m_compute_pipelines.empty()) {
+		CheckpointDriverCache();
+	}
+	Log::WriteToConsoleAndLog(fmt::format(
+	    "Shader cache: loaded {} programs, {} modules, {} pipelines from {} ({:.1f} MB) in {} ms\n",
+	    m_program_cache.size(), modules.size() - 1,
+	    m_graphics_pipelines.size() + m_compute_pipelines.size(), Common::PathToString(path),
+	    static_cast<double>(bytes.size()) / 1'000'000.0,
+	    std::chrono::duration_cast<std::chrono::milliseconds>(std::chrono::steady_clock::now() -
+	                                                          start)
+	        .count()));
+}
+
+bool PipelineCache::LoadRecord(std::span<const uint8_t>&        records,
+                               std::vector<GraphicsPipelineKey>& pipelines) {
+	Frame frame;
+	if (records.size() < sizeof(frame)) {
+		return false;
+	}
+	std::memcpy(&frame, records.data(), sizeof(frame));
+	if (records.size() - sizeof(frame) < frame.compressed_size) {
+		return false;
+	}
+	const auto compressed = records.subspan(sizeof(frame), frame.compressed_size);
+	if (XXH3_64bits(compressed.data(), compressed.size()) != frame.checksum ||
+	    ZSTD_getFrameContentSize(compressed.data(), compressed.size()) != frame.raw_size) {
+		return false;
+	}
+	std::vector<uint8_t> raw(frame.raw_size);
+	if (ZSTD_decompress(raw.data(), raw.size(), compressed.data(), compressed.size()) != raw.size()) {
+		return false;
+	}
+	Reader     reader {raw};
+	auto       type     = Record::Module;
+	const auto complete = [&] { return reader.ok && reader.bytes.empty(); };
+	IoAll(reader, type);
+	switch (type) {
+		case Record::Module: {
+			ProgramKey            key;
+			std::vector<uint8_t>  plan;
+			ShaderModule          module;
+			std::vector<uint32_t> spirv;
+			IoKey(reader, key);
+			IoAll(reader, plan);
+			IoModule(reader, module);
+			IoAll(reader, spirv);
+			if (!complete()) {
+				return false;
+			}
+			const auto [program, inserted] = m_program_cache.try_emplace(std::move(key));
+			if (inserted &&
+			    !ShaderRecompiler::IR::LoadResourcePlan(plan, program->second.resource_plan)) {
+				m_program_cache.erase(program);
+				return false;
+			}
+			module.module = CompileSPV(spirv, m_graphics.device);
+			if (module.module == nullptr) {
+				return false;
+			}
+			program->second.modules.push_back(std::move(module));
+			break;
+		}
+		case Record::GraphicsPipeline: {
+			GraphicsPipelineKey key;
+			reader.Raw(&key, sizeof(key));
+			if (!complete()) {
+				return false;
+			}
+			pipelines.push_back(key);
+			break;
+		}
+		case Record::TessellationLayout: {
+			uint64_t                    key = 0;
+			ShaderTessellationInputInfo layout;
+			IoAll(reader, key, layout);
+			if (!complete()) {
+				return false;
+			}
+			m_tess_layouts.try_emplace(key, layout);
+			break;
+		}
+		default: return false;
+	}
+	records = records.subspan(sizeof(frame) + compressed.size());
+	return true;
+}
+
+void PipelineCache::StoreModule(const ProgramKey& key, const Program& program,
+                                const ShaderModule& module, const std::vector<uint32_t>& spirv) {
+	if (m_store.IsInvalid()) {
+		return;
+	}
+	Writer record;
+	IoAll(record, Record::Module);
+	IoKey(record, key);
+	IoAll(record, ShaderRecompiler::IR::SaveResourcePlan(program.resource_plan));
+	IoModule(record, module);
+	IoAll(record, spirv);
+	Append(m_store, record);
+}
+
+void PipelineCache::StorePipeline(const GraphicsPipelineKey& key) {
+	if (m_store.IsInvalid()) {
+		return;
+	}
+	Writer record;
+	IoAll(record, Record::GraphicsPipeline);
+	record.Raw(&key, sizeof(key));
+	Append(m_store, record);
+}
+
+void PipelineCache::StoreLayout(uint64_t key, const ShaderTessellationInputInfo& layout) {
+	if (m_store.IsInvalid()) {
+		return;
+	}
+	Writer record;
+	IoAll(record, Record::TessellationLayout, key, layout);
+	Append(m_store, record);
+}
+
+PipelineCache::ModuleMap PipelineCache::MapModules() const {
+	ModuleMap modules {{0, nullptr}};
+	for (const auto& program: m_program_cache | std::views::values) {
+		for (const auto& module: program.modules) {
+			modules.emplace(module.id, &module);
+		}
+	}
+	return modules;
+}
+
+bool PipelineCache::CreateGraphicsPipeline(Pipeline& pipeline, const GraphicsPipelineKey& key,
+                                           const ModuleMap& modules) {
+	std::array<const ShaderModule*, 4> stages {};
+	for (size_t i = 0; i < stages.size(); i++) {
+		const auto found = modules.find(key.stage_ids[i]);
+		if (found == modules.end()) {
+			return false;
+		}
+		stages[i] = found->second;
+	}
+	CreatePipelineInternal(m_graphics, pipeline, key, stages, m_driver_cache);
+	return true;
+}
+
+} // namespace Libs::Graphics

src/graphics/host_gpu/renderer/renderContext.cpp

@@ -10,10 +10,10 @@
 
 namespace Libs::Graphics {
 
-RenderContext::RenderContext(GraphicContext& graphics)
+RenderContext::RenderContext(GraphicContext& graphics, const std::filesystem::path& shader_store)
     : m_graphics(graphics), m_render_executor(*this), m_command_scheduler(*this, graphics),
       m_descriptor_heap(graphics, m_command_scheduler.GetMasterSemaphore()),
-      m_pipeline_cache(graphics, &m_image_heap), m_sampler_cache(graphics),
+      m_pipeline_cache(graphics, &m_image_heap, shader_store), m_sampler_cache(graphics),
       m_buffer_cache(graphics, m_command_scheduler, m_page_manager, m_texture_cache),
       m_texture_cache(graphics, m_command_scheduler, m_page_manager, m_buffer_cache),
       m_image_heap(*this, m_page_manager), m_occlusion_queries(graphics, m_command_scheduler) {

src/graphics/host_gpu/renderer/renderContext.h

@@ -31,7 +31,8 @@ class GuestGpu;
 
 class RenderContext {
 public:
-	explicit RenderContext(GraphicContext& graphics);
+	explicit RenderContext(GraphicContext&              graphics,
+	                       const std::filesystem::path& shader_store = ShaderStorePath());
 	~RenderContext();
 	KYTY_CLASS_NO_COPY(RenderContext);