pipeline: persist shaders and pipelines per game and prewarm them at boot

CMakeLists.txt

@@ -337,7 +337,7 @@ if (KYTY_CLANG_CL)
 	set_source_files_properties(${kyty_emulator_src} PROPERTIES COMPILE_FLAGS "-Wno-pragma-pack -Wno-deprecated-declarations -D_TIMESPEC_DEFINED")
 endif()
 
-set(kyty_emulator_link_libraries common Vulkan::Headers spirv-tools-opt spirv-tools SDL3::SDL3-static imgui xxhash FFmpeg::ffmpeg fmt::fmt nlohmann_json::nlohmann_json LibAtrac9 kyty_git_version Zydis::Zydis xbyak::xbyak opus OpenXR::openxr_loader)
+set(kyty_emulator_link_libraries common Vulkan::Headers spirv-tools-opt spirv-tools SDL3::SDL3-static imgui xxhash FFmpeg::ffmpeg fmt::fmt nlohmann_json::nlohmann_json LibAtrac9 kyty_git_version Zydis::Zydis xbyak::xbyak opus OpenXR::openxr_loader libzstd_static)
 
 # Linux system libraries.
 if(LINUX)

src/common/emulatorConfig.cpp

@@ -133,6 +133,10 @@ bool ShaderValidationEnabled() {
 	return g_config->shader_validation_enabled;
 }
 
+bool ShaderCacheEnabled() {
+	return g_config->shader_cache_enabled;
+}
+
 ShaderOptimizationType GetShaderOptimizationType() {
 	return g_config->shader_optimization_type;
 }

src/common/emulatorConfig.h

@@ -72,6 +72,7 @@ struct ConfigOptions {
 	uint32_t               console_language            = DEFAULT_CONSOLE_LANGUAGE;
 	bool                   vulkan_validation_enabled   = false;
 	bool                   shader_validation_enabled   = false;
+	bool                   shader_cache_enabled        = false;
 	ShaderOptimizationType shader_optimization_type    = ShaderOptimizationType::None;
 	LogDirection           shader_log_direction        = LogDirection::Silent;
 	std::filesystem::path  shader_log_folder           = "_Shaders";
@@ -132,6 +133,7 @@ uint32_t GetConsoleLanguage();
 bool     VulkanValidationEnabled();
 
 bool                   ShaderValidationEnabled();
+bool                   ShaderCacheEnabled();
 ShaderOptimizationType GetShaderOptimizationType();
 LogDirection           GetShaderLogDirection();
 std::filesystem::path  GetShaderLogFolder();

src/graphics/host_gpu/renderer/pipeline/pipelineCache.cpp

@@ -17,6 +17,7 @@
 #include "graphics/host_gpu/renderer/renderContext.h"
 #include "graphics/shader/recompiler/BufferFormat.h"
 #include "graphics/shader/recompiler/ShaderRecompiler.h"
+#include "graphics/shader/recompiler/Tessellation.h"
 #include "graphics/shader/recompiler/ir/Serialize.h"
 #include "graphics/shader/shaderCompiler.h"
 #include "kernel/memory.h"
@@ -162,14 +163,6 @@ void ValidateShaderSpirv(const char* stage, uint64_t shader_hash,
 	     shader_hash, messages.c_str());
 }
 
-ShaderRecompiler::Spirv::Profile SpirvProfile(const GraphicContext& graphics) {
-	// AMD GPUs (vendor 0x1002) run NMin/NMax as v_min/v_max, which give the guest's NaN and
-	// signed-zero results; GLSL.std.450 alone does not order -0 below +0.
-	return {.maintenance8           = graphics.maintenance8_enabled,
-	        .native_float_min_max   = graphics.physical_device_properties.vendorID == 0x1002u,
-	        .device_clock_period_ps = graphics.device_clock_period_ps};
-}
-
 // Formats with a narrower host variant drop the components the shader does not read.
 vk::Format NarrowInputFormat(vk::Format format, uint32_t used_components) {
 	switch (format) {
@@ -241,6 +234,26 @@ std::vector<vk::Format> VertexInputFormats(const ShaderVertexInputInfo&
 
 } // namespace
 
+ShaderRecompiler::Spirv::Profile SpirvProfile(const GraphicContext& graphics) {
+	// AMD GPUs (vendor 0x1002) run NMin/NMax as v_min/v_max, which give the guest's NaN and
+	// signed-zero results; GLSL.std.450 alone does not order -0 below +0.
+	return {.maintenance8           = graphics.maintenance8_enabled,
+	        .native_float_min_max   = graphics.physical_device_properties.vendorID == 0x1002u,
+	        .device_clock_period_ps = graphics.device_clock_period_ps};
+}
+
+std::filesystem::path ShaderStorePath() {
+	if (KYTY_BUILD != KYTY_BUILD_RELEASE || !Config::ShaderCacheEnabled() ||
+	    Config::GraphicsDebugDumpEnabled()) {
+		return {};
+	}
+	const auto title_id = PipelineCacheTitleId();
+	if (title_id.empty()) {
+		return {};
+	}
+	return std::filesystem::path("_PipelineCache") / (title_id + ".shaders");
+}
+
 bool GraphicsPipelineKey::operator==(const GraphicsPipelineKey& other) const {
 	return std::memcmp(this, &other, sizeof(*this)) == 0;
 }
@@ -331,6 +344,10 @@ ShaderModule& PipelineCache::CompileModule(Program& program, const ProgramKey& k
 	           options.input_info.pixel->parameter_mode == ShaderPixelParameterMode::Rectangle) {
 		module.rect_parameters = RectListParameters(*options.input_info.pixel, module.info);
 	}
+	// Programs with GPU-selected calls depend on their callees' current code.
+	if (program.call_source == nullptr) {
+		StoreModule(key, program, module, result.spirv);
+	}
 	return module;
 }
 
@@ -442,11 +459,13 @@ const ShaderModule* PipelineCache::GetProgram(const ShaderParams& params, InputI
 }
 
 PipelineCache::PipelineCache(GraphicContext&                         graphics,
-                             ShaderRecompiler::IR::ImageHeapSource* image_heap)
+                             ShaderRecompiler::IR::ImageHeapSource* image_heap,
+                             const std::filesystem::path&           shader_store)
     : m_graphics(graphics), m_image_heap(image_heap),
       m_next_driver_cache_save(std::chrono::steady_clock::now() + DriverCacheSaveInterval) {
 	EXIT_NOT_IMPLEMENTED(!Common::Thread::IsMainThread());
 	InitializeDriverCache();
+	WarmUp(shader_store);
 }
 
 PipelineCache::~PipelineCache() {
@@ -650,6 +669,20 @@ void PipelineCache::GetGraphicsPrograms(const CommandBuffer& command, DrawRender
 	std::array<ShaderParams, 3> vertex_params;
 	if (tess_active) {
 		vertex_params = PrepareTessellationPrograms(vertex_regs, context, vertex_info);
+		// Reflection decodes both programs; the key also covers the VGT fields in tess.
+		const uint64_t layout_key[] = {vertex_params[0].hash, vertex_params[1].hash,
+		                               uint64_t {sh.m_vgtLsHsConfig & 0xfff00u} << 32u |
+		                                   sh.m_vgtTfParam};
+		const auto [layout, inserted] = m_tess_layouts.try_emplace(
+		    XXH3_64bits(layout_key, sizeof(layout_key)), vertex_info[0].tess);
+		if (inserted) {
+			ShaderRecompiler::AnalyzeTessellationPrograms(vertex_params[0].code,
+			                                              vertex_params[1].code, layout->second);
+			StoreLayout(layout->first, layout->second);
+		}
+		for (auto& stage: vertex_info) {
+			stage.tess = layout->second;
+		}
 	} else {
 		vertex_params[0] = PrepareProgram(vertex_regs, context, user_config, vertex_info[0]);
 	}
@@ -1017,6 +1050,7 @@ PipelineCache::Pipeline& PipelineCache::GetGraphicsPipeline(const CommandBuffer&
 		                                         Common::DebugCounters::Peak::PipelineCreateNs);
 		CreatePipelineInternal(m_graphics, pipeline, key, stages, m_driver_cache);
 	}
+	StorePipeline(key);
 	CheckpointDriverCache();
 
 	return pipeline;

src/graphics/host_gpu/renderer/pipeline/pipelineCache.h

@@ -4,6 +4,7 @@
 #include "common/abi.h"
 #include "common/assert.h"
 #include "common/common.h"
+#include "common/file.h"
 #include "graphics/host_gpu/renderer/colorRenderTarget.h"
 #include "graphics/host_gpu/renderer/depthRenderTarget.h"
 #include "graphics/host_gpu/renderer/renderTarget.h"
@@ -37,6 +38,10 @@ namespace ShaderRecompiler::IR {
 class ImageHeapSource;
 } // namespace ShaderRecompiler::IR
 
+namespace ShaderRecompiler::Spirv {
+struct Profile;
+} // namespace ShaderRecompiler::Spirv
+
 #pragma pack(push, 1)
 
 struct PipelineStaticParameters {
@@ -139,7 +144,9 @@ struct DrawRenderState {
 // The owning renderer serializes access, including saves while the GPU is running.
 class PipelineCache {
 public:
-	PipelineCache(GraphicContext& graphics, ShaderRecompiler::IR::ImageHeapSource* image_heap);
+	// Without a shader store path nothing is kept across runs.
+	PipelineCache(GraphicContext& graphics, ShaderRecompiler::IR::ImageHeapSource* image_heap,
+	              const std::filesystem::path& shader_store = {});
 	~PipelineCache();
 	KYTY_CLASS_NO_COPY(PipelineCache);
 	void Save();
@@ -197,6 +204,8 @@ private:
 		std::size_t operator()(const GraphicsPipelineKey& key) const;
 	};
 
+	using ModuleMap = std::unordered_map<uint64_t, const ShaderModule*>;
+
 	GraphicContext&                        m_graphics;
 	ShaderRecompiler::IR::ImageHeapSource* m_image_heap;
 	std::unordered_map<ProgramKey, Program, ProgramKeyHash> m_program_cache;
@@ -209,6 +218,9 @@ private:
 	bool                          m_driver_cache_dirty = false;
 	std::unordered_map<GraphicsPipelineKey, Pipeline, GraphicsPipelineKeyHash> m_graphics_pipelines;
 	std::unordered_map<uint64_t, Pipeline>                                     m_compute_pipelines;
+	std::unordered_map<uint64_t, ShaderTessellationInputInfo>                  m_tess_layouts;
+	// Appends records of everything first created; invalid without a shader store.
+	Common::File m_store;
 
 	template <typename InputInfo>
 	const ShaderModule* GetProgram(const ShaderParams& params, InputInfo& input_info,
@@ -219,8 +231,27 @@ private:
 	                            uint32_t                          push_data_cursor);
 	void InitializeDriverCache();
 	void CheckpointDriverCache();
+
+	// pipelineSerialization.cpp
+	void WarmUp(const std::filesystem::path& path);
+	bool LoadRecord(std::span<const uint8_t>& records, std::vector<GraphicsPipelineKey>& pipelines);
+	void StoreModule(const ProgramKey& key, const Program& program, const ShaderModule& module,
+	                 const std::vector<uint32_t>& spirv);
+	void StorePipeline(const GraphicsPipelineKey& key);
+	void StoreLayout(uint64_t key, const ShaderTessellationInputInfo& layout);
+	// Every module by id, with id 0 naming an absent stage.
+	[[nodiscard]] ModuleMap MapModules() const;
+	// False when a stage id of the key is not in `modules`.
+	bool CreateGraphicsPipeline(Pipeline& pipeline, const GraphicsPipelineKey& key,
+	                            const ModuleMap& modules);
 };
 
+// The shader store of the running title; empty unless --shader-cache is on in a Release build
+// without debug dumps, which change the SPIR-V.
+std::filesystem::path ShaderStorePath();
+// Host capabilities that change the emitted SPIR-V.
+ShaderRecompiler::Spirv::Profile SpirvProfile(const GraphicContext& graphics);
+
 // Stages are indexed like GraphicsPipelineKey::stage_ids.
 void CreatePipelineInternal(GraphicContext& graphics, PipelineCache::Pipeline& pipeline,
                             const GraphicsPipelineKey&                 key,

src/graphics/host_gpu/renderer/pipeline/pipelineSerialization.cpp

@@ -0,0 +1,298 @@
+#include "graphics/host_gpu/renderer/pipeline/pipelineCache.h"
+
+#include "common/logging/log.h"
+#include "common/stringUtils.h"
+#include "graphics/host_gpu/vulkanCommon.h"
+#include "graphics/shader/recompiler/ShaderRecompiler.h"
+#include "graphics/shader/recompiler/ir/Serialize.h"
+
+#include <algorithm>
+#include <chrono>
+#include <cstring>
+#include <fmt/format.h>
+#include <ranges>
+#include <xxhash.h>
+#include <zstd.h>
+
+#if KYTY_PLATFORM == KYTY_PLATFORM_WINDOWS
+#define WIN32_LEAN_AND_MEAN
+#include <windows.h>
+#elif defined(__APPLE__)
+#include <mach-o/dyld.h>
+#endif
+
+// The shader store: a header, then zstd-compressed records of everything a run created first.
+// Records are only appended, so a store cut short by a crash keeps every complete record.
+namespace Libs::Graphics {
+
+namespace {
+
+using ShaderRecompiler::IR::IoAll;
+using ShaderRecompiler::IR::Reader;
+using ShaderRecompiler::IR::Writer;
+
+enum class Record : uint8_t { Module, GraphicsPipeline, TessellationLayout };
+
+// Precedes each record's zstd frame.
+struct Frame {
+	uint32_t compressed_size = 0;
+	uint32_t raw_size        = 0;
+	uint64_t checksum        = 0; // XXH3 of the compressed bytes.
+};
+
+// Changes with every build, so records never meet code that encodes them differently. Zero when
+// the executable cannot be found.
+uint64_t BuildStamp() {
+#if KYTY_PLATFORM == KYTY_PLATFORM_WINDOWS
+	std::array<wchar_t, 32768> name {};
+	GetModuleFileNameW(nullptr, name.data(), static_cast<DWORD>(name.size()));
+	const std::filesystem::path path = name.data();
+#elif defined(__APPLE__)
+	std::array<char, 4096> name {};
+	auto size = static_cast<uint32_t>(name.size());
+	_NSGetExecutablePath(name.data(), &size);
+	const std::filesystem::path path = name.data();
+#else
+	const std::filesystem::path path = "/proc/self/exe";
+#endif
+	std::error_code error;
+	const auto      time = std::filesystem::last_write_time(path, error).time_since_epoch().count();
+	return error ? 0 : static_cast<uint64_t>(time);
+}
+
+// What every record depends on besides its own fields; empty when the build is unknown.
+std::vector<uint8_t> StoreHeader(const GraphicContext& graphics) {
+	const auto stamp = BuildStamp();
+	if (stamp == 0) {
+		return {};
+	}
+	Writer header;
+	IoAll(header, stamp, SpirvProfile(graphics),
+	      graphics.GetPhysicalDeviceProperties().limits.maxComputeSharedMemorySize);
+	return std::move(header.bytes);
+}
+
+void Append(Common::File& store, const Writer& record) {
+	std::vector<uint8_t> bytes(sizeof(Frame) + ZSTD_compressBound(record.bytes.size()));
+	const auto size = ZSTD_compress(bytes.data() + sizeof(Frame), bytes.size() - sizeof(Frame),
+	                                record.bytes.data(), record.bytes.size(), 3);
+	EXIT_IF(ZSTD_isError(size));
+	const Frame frame {.compressed_size = static_cast<uint32_t>(size),
+	                   .raw_size        = static_cast<uint32_t>(record.bytes.size()),
+	                   .checksum        = XXH3_64bits(bytes.data() + sizeof(Frame), size)};
+	std::memcpy(bytes.data(), &frame, sizeof(frame));
+	// A failed write leaves a partial record; the next run drops it and everything after it.
+	store.Write(bytes.data(), static_cast<uint32_t>(sizeof(Frame) + size));
+}
+
+// Record fields, shared by writing and reading.
+template <class S, class Key> void IoKey(S& io, Key& key) {
+	auto& [stage, hash, user_data_count, static_state] = key;
+	IoAll(io, stage, hash, user_data_count, static_state);
+}
+
+template <class S, class Module> void IoModule(S& io, Module& stage) {
+	// The handle comes from the SPIR-V; stored programs have no function code.
+	auto& [id, handle, info, specialization, function_code, vertex_formats, rect_parameters] = stage;
+	IoAll(io, id, info, specialization, vertex_formats, rect_parameters);
+}
+
+} // namespace
+
+void PipelineCache::WarmUp(const std::filesystem::path& path) {
+	if (path.empty()) {
+		return;
+	}
+	const auto header = StoreHeader(m_graphics);
+	if (header.empty()) {
+		return;
+	}
+	const auto           start = std::chrono::steady_clock::now();
+	std::vector<uint8_t> bytes;
+	if (Common::File file;
+	    Common::File::IsFileExisting(path) && file.Open(path, Common::File::Mode::Read)) {
+		bytes.resize(file.Size());
+		uint32_t read = 0;
+		file.Read(bytes.data(), static_cast<uint32_t>(bytes.size()), &read);
+		bytes.resize(read);
+	}
+	std::vector<GraphicsPipelineKey> pipelines;
+	auto                             records = std::span<const uint8_t>(bytes);
+	const bool reusable =
+	    records.size() >= header.size() && std::ranges::equal(records.first(header.size()), header);
+	if (reusable) {
+		records = records.subspan(header.size());
+		while (LoadRecord(records, pipelines)) {
+		}
+	}
+	// A damaged record ends the store; the records before it stay.
+	const auto kept = reusable ? bytes.size() - records.size() : 0;
+	Common::File::CreateDirectories(path.parent_path());
+	if (kept == 0) {
+		if (m_store.Create(path)) {
+			m_store.Write(header.data(), static_cast<uint32_t>(header.size()));
+		}
+	} else if (m_store.Open(path, Common::File::Mode::ReadWrite)) {
+		m_store.Truncate(kept);
+		m_store.Seek(kept);
+	}
+
+	const auto modules = MapModules();
+	for (const auto& key: pipelines) {
+		Pipeline pipeline;
+		if (CreateGraphicsPipeline(pipeline, key, modules)) {
+			m_graphics_pipelines.emplace(key, pipeline);
+		}
+	}
+	// A compute pipeline depends only on its module.
+	for (const auto* module: modules | std::views::values) {
+		if (module != nullptr && module->info.stage == ShaderType::Compute) {
+			CreatePipelineInternal(m_graphics, m_compute_pipelines[module->id], *module,
+			                       m_driver_cache);
+		}
+	}
+	if (!m_graphics_pipelines.empty() || !m_compute_pipelines.empty()) {
+		CheckpointDriverCache();
+	}
+	Log::WriteToConsoleAndLog(fmt::format(
+	    "Shader cache: loaded {} programs, {} modules, {} pipelines from {} ({:.1f} MB) in {} ms\n",
+	    m_program_cache.size(), modules.size() - 1,
+	    m_graphics_pipelines.size() + m_compute_pipelines.size(), Common::PathToString(path),
+	    static_cast<double>(bytes.size()) / 1'000'000.0,
+	    std::chrono::duration_cast<std::chrono::milliseconds>(std::chrono::steady_clock::now() -
+	                                                          start)
+	        .count()));
+}
+
+bool PipelineCache::LoadRecord(std::span<const uint8_t>&        records,
+                               std::vector<GraphicsPipelineKey>& pipelines) {
+	Frame frame;
+	if (records.size() < sizeof(frame)) {
+		return false;
+	}
+	std::memcpy(&frame, records.data(), sizeof(frame));
+	if (records.size() - sizeof(frame) < frame.compressed_size) {
+		return false;
+	}
+	const auto compressed = records.subspan(sizeof(frame), frame.compressed_size);
+	if (XXH3_64bits(compressed.data(), compressed.size()) != frame.checksum ||
+	    ZSTD_getFrameContentSize(compressed.data(), compressed.size()) != frame.raw_size) {
+		return false;
+	}
+	std::vector<uint8_t> raw(frame.raw_size);
+	if (ZSTD_decompress(raw.data(), raw.size(), compressed.data(), compressed.size()) != raw.size()) {
+		return false;
+	}
+	Reader     reader {raw};
+	auto       type     = Record::Module;
+	const auto complete = [&] { return reader.ok && reader.bytes.empty(); };
+	IoAll(reader, type);
+	switch (type) {
+		case Record::Module: {
+			ProgramKey            key;
+			std::vector<uint8_t>  plan;
+			ShaderModule          module;
+			std::vector<uint32_t> spirv;
+			IoKey(reader, key);
+			IoAll(reader, plan);
+			IoModule(reader, module);
+			IoAll(reader, spirv);
+			if (!complete()) {
+				return false;
+			}
+			const auto [program, inserted] = m_program_cache.try_emplace(std::move(key));
+			if (inserted &&
+			    !ShaderRecompiler::IR::LoadResourcePlan(plan, program->second.resource_plan)) {
+				m_program_cache.erase(program);
+				return false;
+			}
+			module.module = CompileSPV(spirv, m_graphics.device);
+			if (module.module == nullptr) {
+				return false;
+			}
+			program->second.modules.push_back(std::move(module));
+			break;
+		}
+		case Record::GraphicsPipeline: {
+			GraphicsPipelineKey key;
+			reader.Raw(&key, sizeof(key));
+			if (!complete()) {
+				return false;
+			}
+			pipelines.push_back(key);
+			break;
+		}
+		case Record::TessellationLayout: {
+			uint64_t                    key = 0;
+			ShaderTessellationInputInfo layout;
+			IoAll(reader, key, layout);
+			if (!complete()) {
+				return false;
+			}
+			m_tess_layouts.try_emplace(key, layout);
+			break;
+		}
+		default: return false;
+	}
+	records = records.subspan(sizeof(frame) + compressed.size());
+	return true;
+}
+
+void PipelineCache::StoreModule(const ProgramKey& key, const Program& program,
+                                const ShaderModule& module, const std::vector<uint32_t>& spirv) {
+	if (m_store.IsInvalid()) {
+		return;
+	}
+	Writer record;
+	IoAll(record, Record::Module);
+	IoKey(record, key);
+	IoAll(record, ShaderRecompiler::IR::SaveResourcePlan(program.resource_plan));
+	IoModule(record, module);
+	IoAll(record, spirv);
+	Append(m_store, record);
+}
+
+void PipelineCache::StorePipeline(const GraphicsPipelineKey& key) {
+	if (m_store.IsInvalid()) {
+		return;
+	}
+	Writer record;
+	IoAll(record, Record::GraphicsPipeline);
+	record.Raw(&key, sizeof(key));
+	Append(m_store, record);
+}
+
+void PipelineCache::StoreLayout(uint64_t key, const ShaderTessellationInputInfo& layout) {
+	if (m_store.IsInvalid()) {
+		return;
+	}
+	Writer record;
+	IoAll(record, Record::TessellationLayout, key, layout);
+	Append(m_store, record);
+}
+
+PipelineCache::ModuleMap PipelineCache::MapModules() const {
+	ModuleMap modules {{0, nullptr}};
+	for (const auto& program: m_program_cache | std::views::values) {
+		for (const auto& module: program.modules) {
+			modules.emplace(module.id, &module);
+		}
+	}
+	return modules;
+}
+
+bool PipelineCache::CreateGraphicsPipeline(Pipeline& pipeline, const GraphicsPipelineKey& key,
+                                           const ModuleMap& modules) {
+	std::array<const ShaderModule*, 4> stages {};
+	for (size_t i = 0; i < stages.size(); i++) {
+		const auto found = modules.find(key.stage_ids[i]);
+		if (found == modules.end()) {
+			return false;
+		}
+		stages[i] = found->second;
+	}
+	CreatePipelineInternal(m_graphics, pipeline, key, stages, m_driver_cache);
+	return true;
+}
+
+} // namespace Libs::Graphics

src/graphics/host_gpu/renderer/renderContext.cpp

@@ -10,10 +10,10 @@
 
 namespace Libs::Graphics {
 
-RenderContext::RenderContext(GraphicContext& graphics)
+RenderContext::RenderContext(GraphicContext& graphics, const std::filesystem::path& shader_store)
     : m_graphics(graphics), m_render_executor(*this), m_command_scheduler(*this, graphics),
       m_descriptor_heap(graphics, m_command_scheduler.GetMasterSemaphore()),
-      m_pipeline_cache(graphics, &m_image_heap), m_sampler_cache(graphics),
+      m_pipeline_cache(graphics, &m_image_heap, shader_store), m_sampler_cache(graphics),
       m_buffer_cache(graphics, m_command_scheduler, m_page_manager, m_texture_cache),
       m_texture_cache(graphics, m_command_scheduler, m_page_manager, m_buffer_cache),
       m_image_heap(*this, m_page_manager), m_occlusion_queries(graphics, m_command_scheduler) {

src/graphics/host_gpu/renderer/renderContext.h

@@ -31,7 +31,8 @@ class GuestGpu;
 
 class RenderContext {
 public:
-	explicit RenderContext(GraphicContext& graphics);
+	explicit RenderContext(GraphicContext&              graphics,
+	                       const std::filesystem::path& shader_store = ShaderStorePath());
 	~RenderContext();
 	KYTY_CLASS_NO_COPY(RenderContext);
 

src/graphics/shader/recompiler/ir/Serialize.h

@@ -2,8 +2,10 @@
 #define EMULATOR_INCLUDE_EMULATOR_GRAPHICS_SHADER_RECOMPILER_SERIALIZE_H_
 
 #include "common/assert.h"
+#include "graphics/shader/recompiler/backend/spirv/SpirvEmitter.h"
 #include "graphics/shader/recompiler/ir/ShaderIR.h"
 #include "graphics/shader/recompiler/ir/passes/ResourceMaterialization.h"
+#include "graphics/shader/rectListShader.h"
 
 #include <algorithm>
 #include <bit>
@@ -167,6 +169,11 @@ KYTY_IO_FIELDS(ResourceSpecialization::Buffer, a, b, c, d, e, f, g, h)
 KYTY_IO_FIELDS(ResourceSpecialization::Image, a, b, c, d, e, f, g, h, i, j, k)
 KYTY_IO_FIELDS(ResourceSpecialization::Sampler, a, b, c)
 KYTY_IO_FIELDS(ResourceSpecialization, a, b, c)
+KYTY_IO_FIELDS(Bindings, a, b, c, d, e)
+KYTY_IO_FIELDS(CompiledShaderInfo, a, b, c, d, e, f, g, h, i, j)
+KYTY_IO_FIELDS(Spirv::Profile, a, b, c)
+KYTY_IO_FIELDS(ShaderTessellationInputInfo, a, b, c, d, e, f, g, h, i)
+KYTY_IO_FIELDS(RectListParameter, a, b, c)
 #undef KYTY_IO_FIELDS
 
 } // namespace Libs::Graphics::ShaderRecompiler::IR

src/graphics/shader/shader.cpp

@@ -11,7 +11,6 @@
 #include "graphics/guest_gpu/hardwareContext.h"
 #include "graphics/host_gpu/renderer/renderContext.h"
 #include "graphics/shader/recompiler/ShaderRecompiler.h"
-#include "graphics/shader/recompiler/Tessellation.h"
 #include "graphics/shader/recompiler/frontend/decode/ShaderDecoder.h"
 #include "graphics/shader/shaderCompiler.h"
 #include "graphics/shader/shaderVertexMetadata.h"
@@ -53,8 +52,6 @@ struct ShaderMapEntry {
 
 static std::unique_ptr<std::unordered_map<uint64_t, ShaderMapEntry>> g_shader_map;
 static std::mutex                                                      g_shader_map_mutex;
-static std::unordered_map<uint64_t, ShaderTessellationInputInfo>       g_tess_layouts;
-static std::mutex                                                      g_tess_layouts_mutex;
 
 void ShaderInit() {
 	EXIT_IF(g_shader_map != nullptr);
@@ -662,18 +659,6 @@ PrepareTessellationPrograms(const HW::VertexShaderInfo& regs, const HW::Context&
 	};
 	EXIT_IF(tess.input_control_points == 0 || tess.input_control_points > 32 ||
 	        tess.output_control_points == 0 || tess.output_control_points > 32);
-	// Reflection decodes both programs; the key also covers the VGT fields cached in tess.
-	const uint64_t key[] = {local_hash, control_hash,
-	                        uint64_t {sh.m_vgtLsHsConfig & 0xfff00u} << 32u | sh.m_vgtTfParam};
-	{
-		std::scoped_lock lock(g_tess_layouts_mutex);
-		auto [entry, inserted] = g_tess_layouts.try_emplace(XXH3_64bits(key, sizeof(key)), tess);
-		if (inserted) {
-			ShaderRecompiler::AnalyzeTessellationPrograms(params[0].code, params[1].code,
-			                                              entry->second);
-		}
-		tess = entry->second;
-	}
 	for (auto& stage: input_info) {
 		stage.tess = tess;
 	}

src/graphics/shader/shaderCompiler.h

@@ -32,6 +32,7 @@ void BuildStageStaticKey(const ShaderComputeInputInfo& input_info, std::vector<u
 
 ShaderParams PrepareProgram(const HW::VertexShaderInfo& regs, const HW::Context& context,
                             const HW::UserConfig& user_config, ShaderVertexInputInfo& input_info);
+// Each stage's tess holds the VGT fields; the caller adds the layout reflected from the programs.
 std::array<ShaderParams, 3>
 PrepareTessellationPrograms(const HW::VertexShaderInfo& regs, const HW::Context& context,
                             std::array<ShaderVertexInputInfo, 3>& input_info);

src/main.cpp

@@ -80,6 +80,7 @@ static void PrintUsage() {
 	::printf("  --gpu-assisted-validation            Bounds-check shader accesses on the GPU.\n"
 	         "                                       Implies --vulkan-validation; very slow.\n");
 	::printf("  --shader-validation                  Enable shader validation.\n");
+	::printf("  --shader-cache                       Keep compiled shaders and pipelines between boots.\n");
 	::printf("  --tessellation                      Draw tessellation patches; skipped by default.\n");
 	::printf("  --occlusion-queries                  Count occlusion query samples on the GPU.\n");
 	::printf("  --shader-optimization-type <value>   None, Size, or Performance.\n");
@@ -205,6 +206,7 @@ static constexpr Flag FLAGS[] = {
     {"--vulkan-validation", &ConfigOptions::vulkan_validation_enabled},
     {"--gpu-assisted-validation", &ConfigOptions::gpu_assisted_validation_enabled},
     {"--shader-validation", &ConfigOptions::shader_validation_enabled},
+    {"--shader-cache", &ConfigOptions::shader_cache_enabled},
     {"--tessellation", &ConfigOptions::tessellation_enabled},
     {"--occlusion-queries", &ConfigOptions::occlusion_queries_enabled},
     {"--command-buffer-dump", &ConfigOptions::command_buffer_dump_enabled},

tests/ShaderRecompilerComputeTests.cpp

@@ -588,19 +588,13 @@ struct PipelineCacheTestAccess {
   // Replaces each graphics pipeline whose stage ids name compiled modules with one created
   // from only its key and those modules. The GPU must be idle. Returns how many it replaced.
   static uint32_t RecreateGraphicsPipelines(PipelineCache &cache) {
-    std::unordered_map<uint64_t, const ShaderModule *> modules{{0, nullptr}}; // Id 0: absent.
-    for (const auto &program : cache.m_program_cache | std::views::values)
-      for (const auto &module : program.modules) modules.emplace(module.id, &module);
+    const auto modules = cache.MapModules();
     auto &device = cache.m_graphics.device;
     uint32_t count = 0;
     for (auto &[key, pipeline] : cache.m_graphics_pipelines) {
-      // Test-owned modules are not in the cache.
-      if (!std::ranges::all_of(key.stage_ids, [&](uint64_t id) { return modules.contains(id); }))
-        continue;
-      std::array<const ShaderModule *, 4> stages{};
-      for (size_t i = 0; i < stages.size(); i++) stages[i] = modules.at(key.stage_ids[i]);
       PipelineCache::Pipeline recreated;
-      CreatePipelineInternal(cache.m_graphics, recreated, key, stages, nullptr);
+      // Test-owned modules are not in the cache.
+      if (!cache.CreateGraphicsPipeline(recreated, key, modules)) continue;
       device.destroyPipeline(pipeline.pipeline, nullptr);
       device.destroyPipelineLayout(pipeline.pipeline_layout, nullptr);
       device.destroyDescriptorSetLayout(pipeline.descriptor_set_layout, nullptr);
@@ -611,6 +605,17 @@ struct PipelineCacheTestAccess {
   }
 
   static auto Compiled(const PipelineCache &cache) { return cache.m_compiled; }
+
+  static void StoreLayout(PipelineCache &cache, uint64_t key,
+                          const ShaderTessellationInputInfo &layout) {
+    cache.StoreLayout(key, layout);
+  }
+
+  static const auto &TessLayouts(const PipelineCache &cache) { return cache.m_tess_layouts; }
+
+  static std::array<size_t, 2> Pipelines(const PipelineCache &cache) {
+    return {cache.m_graphics_pipelines.size(), cache.m_compute_pipelines.size()};
+  }
 };
 
 namespace {
@@ -44700,6 +44705,163 @@ void CheckStableProgramIds(VulkanHarness &vulkan) {
   std::printf("[gpu]     %-32s ok\n", name);
 }
 
+// With a shader store a restart compiles nothing and creates every pipeline before the first
+// lookup. A damaged record and everything after it are dropped; a stale store starts over.
+void CheckShaderStore(VulkanHarness &vulkan) {
+  constexpr const char *name = "ShaderStore";
+  constexpr uintptr_t base = 0x0000000205000000ull;
+  constexpr uint64_t allocation_size = 0x10000;
+  constexpr u32 value = 0x2a;
+  std::vector<u32> compute;
+  AppendVMovU32(&compute, 1, value);
+  compute.insert(compute.end(), {EncodeMubuf0(0x1c), EncodeMubuf1(1, 0, 0)});
+  AppendEnd(&compute);
+  std::vector<u32> vertex;
+  AppendVMovU32(&vertex, 0, 0);
+  AppendVMovLiteral(&vertex, 1, 0x3f800000u);
+  vertex.insert(vertex.end(), {EncodeExp0(0x0c, 0xf), EncodeExp1(0, 0, 0, 1)});
+  AppendEnd(&vertex);
+  std::vector<u32> pixel;
+  AppendVMovLiteral(&pixel, 0, 0x3f800000u);
+  pixel.insert(pixel.end(), {EncodeExp0(0x00, 0xf), EncodeExp1(0, 0, 0, 0)});
+  AppendEnd(&pixel);
+  static ShaderUserData vertex_user_data{};
+  const auto map = [](const std::vector<u32> &code, Prospero::ShaderBinaryType type,
+                      ShaderUserData *user_data = nullptr) {
+    ShaderMapUserData(reinterpret_cast<uint64_t>(code.data()),
+                      {.type = type, .user_data = user_data,
+                       .code_size_bytes = static_cast<u32>(code.size() * sizeof(u32))});
+  };
+  map(compute, Prospero::ShaderBinaryType::kCs);
+  map(vertex, Prospero::ShaderBinaryType::kGs, &vertex_user_data);
+  map(pixel, Prospero::ShaderBinaryType::kPs);
+  int64_t direct_offset = -1;
+  void *mapped = reinterpret_cast<void *>(base);
+  Require(name, "output memory",
+          LibKernel::Memory::KernelAllocateDirectMemory(
+              0, LibKernel::Memory::KernelGetDirectMemorySize(), allocation_size, 0x10000, 0,
+              &direct_offset) == 0 &&
+              LibKernel::Memory::KernelMapDirectMemory(&mapped, allocation_size, 0x3, 0x10,
+                                                       direct_offset, 0x10000) == 0,
+          "output allocation or mapping failed");
+  const auto store = std::filesystem::temp_directory_path() /
+                     fmt::format("kyty_shader_store_{}",
+                                 std::chrono::steady_clock::now().time_since_epoch().count()) /
+                     "TEST00000.shaders";
+
+  struct Session {
+    std::array<size_t, 2> prewarmed{}, pipelines{};
+    std::array<uint32_t, static_cast<size_t>(ShaderType::TessellationEvaluation) + 1> compiled{};
+    u32 output = 0;
+    bool operator==(const Session &) const = default;
+  };
+  // Dispatches the compute program and creates the graphics pipeline.
+  const auto run = [&] {
+    std::memset(mapped, 0, allocation_size);
+    RenderContext context(vulkan.RuntimeContext(), store);
+    auto &pipelines = context.GetPipelineCache();
+    Session session{.prewarmed = PipelineCacheTestAccess::Pipelines(pipelines)};
+    context.InitializeGpu(nullptr);
+    LibKernel::Memory::InstallGpuResources(&context);
+    context.GetGpu().SendCommandSync([&] {
+      CommandProcessor processor(context, 0);
+      processor.BufferInit();
+      auto &registers = processor.GetCtx();
+      auto &shaders = processor.GetShCtx();
+      registers.SetRenderTargetMask(0xf);
+      registers.SetShaderMask(0xf);
+      registers.SetTargetOutputMode(0, 4);
+      registers.SetViewportTransformControl(0x300);
+      processor.GetUcfg().SetPrimitiveType(Prospero::PrimitiveType::kTriList);
+      shaders.SetEsShaderBase(reinterpret_cast<uint64_t>(vertex.data()));
+      shaders.SetPsShaderBase(reinterpret_cast<uint64_t>(pixel.data()));
+      shaders.SetCsShader({.data_addr = reinterpret_cast<uint64_t>(compute.data()),
+                           .num_thread_x = 1, .num_thread_y = 1, .num_thread_z = 1,
+                           .wave_size = 64, .user_sgpr = 4});
+      const ShaderBufferResource output{{static_cast<u32>(base), static_cast<u32>(base >> 32u),
+          4, DstSel(4, 5, 6, 7) | (static_cast<u32>(Prospero::BufferFormat::k32UInt) << 12u)}};
+      for (u32 i = 0; i < 4; i++) shaders.SetCsUserSgpr(i, output.fields[i]);
+      context.MapMemory(base, allocation_size);
+      processor.DispatchDirect(1, 1, 1, 0x41u);
+      auto &scheduler = context.GetCommandScheduler();
+      DrawRenderState state{.topology = vk::PrimitiveTopology::eTriangleList};
+      pipelines.GetGraphicsPrograms(scheduler.Current(), state);
+      (void)pipelines.GetGraphicsPipeline(scheduler.Current(), state);
+      context.GetBufferCache().ReadMemory(base, sizeof(u32));
+      (void)LibKernel::Memory::TryReadBacking(base, &session.output, sizeof(u32));
+      context.UnmapMemory(base, allocation_size);
+      scheduler.Finish();
+    });
+    context.ShutdownGpu();
+    LibKernel::Memory::InstallGpuResources(nullptr);
+    session.compiled = PipelineCacheTestAccess::Compiled(pipelines);
+    session.pipelines = PipelineCacheTestAccess::Pipelines(pipelines);
+    return session;
+  };
+  const auto flip = [&](uint64_t offset) {
+    Common::File file(store, Common::File::Mode::ReadWrite);
+    uint8_t byte = 0;
+    file.Seek(offset);
+    file.Read(&byte, 1);
+    byte ^= 0xffu;
+    file.Seek(offset);
+    file.Write(&byte, 1);
+  };
+
+  const auto cold = run();
+  const auto size = std::filesystem::file_size(store);
+  Require(name, "cold start",
+          cold.output == value && cold.prewarmed == std::array<size_t, 2>{} &&
+              cold.pipelines == std::array<size_t, 2>{1, 1} &&
+              cold.compiled[static_cast<size_t>(ShaderType::Compute)] == 1,
+          "the first run did not create its store, dispatch or pipelines");
+  auto warm = cold;
+  warm.prewarmed = cold.pipelines;
+  warm.compiled = {};
+  Require(name, "warm restart", run() == warm && std::filesystem::file_size(store) == size,
+          "a restart compiled a module, created a pipeline late or rewrote the store");
+  std::filesystem::resize_file(store, size + 3);
+  Require(name, "torn tail", run() == warm && std::filesystem::file_size(store) == size,
+          "a torn last record was kept");
+  // The graphics pipeline's record is the last one.
+  auto lost_pipeline = warm;
+  lost_pipeline.prewarmed[0] = 0;
+  std::filesystem::resize_file(store, size - 3);
+  Require(name, "truncated record",
+          run() == lost_pipeline && std::filesystem::file_size(store) == size,
+          "a cut record was not dropped and appended again, or took earlier records along");
+  // Flips the top checksum byte, just before the last zstd frame's magic 0xfd2fb528.
+  std::vector<uint8_t> bytes(size);
+  Common::File(store, Common::File::Mode::Read).Read(bytes.data(), static_cast<u32>(size));
+  const std::array<uint8_t, 4> zstd_magic{0x28, 0xb5, 0x2f, 0xfd};
+  flip(std::ranges::find_end(bytes, zstd_magic).begin() - bytes.begin() - 1);
+  Require(name, "damaged record",
+          run() == lost_pipeline && std::filesystem::file_size(store) == size,
+          "a damaged record was loaded or not replaced");
+  flip(0);
+  Require(name, "stale store", run() == cold && std::filesystem::file_size(store) == size,
+          "a store with another header was reused instead of started over");
+  // Reflecting a layout decodes LS and HS, so a restart takes it from the store.
+  const ShaderTessellationInputInfo layout{1, 2, 3, 4, 5, 6, 7, 8, 9};
+  {
+    RenderContext context(vulkan.RuntimeContext(), store);
+    PipelineCacheTestAccess::StoreLayout(context.GetPipelineCache(), 1, layout);
+  }
+  {
+    RenderContext context(vulkan.RuntimeContext(), store);
+    const auto &layouts = PipelineCacheTestAccess::TessLayouts(context.GetPipelineCache());
+    Require(name, "tessellation layout",
+            layouts.contains(1) && std::memcmp(&layouts.at(1), &layout, sizeof(layout)) == 0,
+            "a restart did not load the stored tessellation layout");
+  }
+  std::filesystem::remove_all(store.parent_path());
+  Require(name, "release",
+          LibKernel::Memory::KernelMunmap(base, allocation_size) == 0 &&
+              LibKernel::Memory::KernelReleaseDirectMemory(direct_offset, allocation_size) == 0,
+          "output memory release failed");
+  std::printf("[gpu]     %-32s ok\n", name);
+}
+
 void CheckPs5GameExampleImageClearRuntimeShape() {
   const auto MakeCode = [] {
     std::vector<u32> code;
@@ -50326,6 +50488,11 @@ int main(int argc, char **argv) {
     CheckStableProgramIds(vulkan);
     return 0;
   }
+  if (argc == 2 && std::strcmp(argv[1], "--shader-cache-only") == 0) {
+    VulkanHarness vulkan;
+    CheckShaderStore(vulkan);
+    return 0;
+  }
   if (argc == 2 && std::strcmp(argv[1], "--lds-limit-only") == 0) {
     VulkanHarness vulkan;
     CheckComputeLdsLimit(vulkan);
@@ -51488,6 +51655,7 @@ int main(int argc, char **argv) {
   vulkan.CheckNativeIndirectDispatch();
   CheckComputeLdsLimit(vulkan);
   CheckStableProgramIds(vulkan);
+  CheckShaderStore(vulkan);
   vulkan.CheckOrderedDispatch();
   vulkan.CheckZeroStrideBufferBinding();
   vulkan.CheckNullTextureDescriptors();