diff --git a/CMakeLists.txt b/CMakeLists.txt index 5642d1da8..56e03a638 100644 --- a/CMakeLists.txt +++ b/CMakeLists.txt @@ -337,7 +337,7 @@ if (KYTY_CLANG_CL) set_source_files_properties(${kyty_emulator_src} PROPERTIES COMPILE_FLAGS "-Wno-pragma-pack -Wno-deprecated-declarations -D_TIMESPEC_DEFINED") endif() -set(kyty_emulator_link_libraries common Vulkan::Headers spirv-tools-opt spirv-tools SDL3::SDL3-static imgui xxhash FFmpeg::ffmpeg fmt::fmt nlohmann_json::nlohmann_json LibAtrac9 kyty_git_version Zydis::Zydis xbyak::xbyak opus OpenXR::openxr_loader) +set(kyty_emulator_link_libraries common Vulkan::Headers spirv-tools-opt spirv-tools SDL3::SDL3-static imgui xxhash FFmpeg::ffmpeg fmt::fmt nlohmann_json::nlohmann_json LibAtrac9 kyty_git_version Zydis::Zydis xbyak::xbyak opus OpenXR::openxr_loader libzstd_static) # Linux system libraries. if(LINUX) diff --git a/src/common/emulatorConfig.cpp b/src/common/emulatorConfig.cpp index 1bb24787f..8c16a4bf2 100644 --- a/src/common/emulatorConfig.cpp +++ b/src/common/emulatorConfig.cpp @@ -133,6 +133,10 @@ bool ShaderValidationEnabled() { return g_config->shader_validation_enabled; } +bool ShaderCacheEnabled() { + return g_config->shader_cache_enabled; +} + ShaderOptimizationType GetShaderOptimizationType() { return g_config->shader_optimization_type; } diff --git a/src/common/emulatorConfig.h b/src/common/emulatorConfig.h index 2671d6bff..56c026d84 100644 --- a/src/common/emulatorConfig.h +++ b/src/common/emulatorConfig.h @@ -72,6 +72,7 @@ struct ConfigOptions { uint32_t console_language = DEFAULT_CONSOLE_LANGUAGE; bool vulkan_validation_enabled = false; bool shader_validation_enabled = false; + bool shader_cache_enabled = false; ShaderOptimizationType shader_optimization_type = ShaderOptimizationType::None; LogDirection shader_log_direction = LogDirection::Silent; std::filesystem::path shader_log_folder = "_Shaders"; @@ -132,6 +133,7 @@ uint32_t GetConsoleLanguage(); bool VulkanValidationEnabled(); bool ShaderValidationEnabled(); +bool ShaderCacheEnabled(); ShaderOptimizationType GetShaderOptimizationType(); LogDirection GetShaderLogDirection(); std::filesystem::path GetShaderLogFolder(); diff --git a/src/graphics/host_gpu/renderer/pipeline/pipelineCache.cpp b/src/graphics/host_gpu/renderer/pipeline/pipelineCache.cpp index ed0abca18..1f41d75ae 100644 --- a/src/graphics/host_gpu/renderer/pipeline/pipelineCache.cpp +++ b/src/graphics/host_gpu/renderer/pipeline/pipelineCache.cpp @@ -17,6 +17,7 @@ #include "graphics/host_gpu/renderer/renderContext.h" #include "graphics/shader/recompiler/BufferFormat.h" #include "graphics/shader/recompiler/ShaderRecompiler.h" +#include "graphics/shader/recompiler/Tessellation.h" #include "graphics/shader/recompiler/ir/Serialize.h" #include "graphics/shader/shaderCompiler.h" #include "kernel/memory.h" @@ -162,14 +163,6 @@ void ValidateShaderSpirv(const char* stage, uint64_t shader_hash, shader_hash, messages.c_str()); } -ShaderRecompiler::Spirv::Profile SpirvProfile(const GraphicContext& graphics) { - // AMD GPUs (vendor 0x1002) run NMin/NMax as v_min/v_max, which give the guest's NaN and - // signed-zero results; GLSL.std.450 alone does not order -0 below +0. - return {.maintenance8 = graphics.maintenance8_enabled, - .native_float_min_max = graphics.physical_device_properties.vendorID == 0x1002u, - .device_clock_period_ps = graphics.device_clock_period_ps}; -} - // Formats with a narrower host variant drop the components the shader does not read. vk::Format NarrowInputFormat(vk::Format format, uint32_t used_components) { switch (format) { @@ -241,6 +234,26 @@ std::vector VertexInputFormats(const ShaderVertexInputInfo& } // namespace +ShaderRecompiler::Spirv::Profile SpirvProfile(const GraphicContext& graphics) { + // AMD GPUs (vendor 0x1002) run NMin/NMax as v_min/v_max, which give the guest's NaN and + // signed-zero results; GLSL.std.450 alone does not order -0 below +0. + return {.maintenance8 = graphics.maintenance8_enabled, + .native_float_min_max = graphics.physical_device_properties.vendorID == 0x1002u, + .device_clock_period_ps = graphics.device_clock_period_ps}; +} + +std::filesystem::path ShaderStorePath() { + if (KYTY_BUILD != KYTY_BUILD_RELEASE || !Config::ShaderCacheEnabled() || + Config::GraphicsDebugDumpEnabled()) { + return {}; + } + const auto title_id = PipelineCacheTitleId(); + if (title_id.empty()) { + return {}; + } + return std::filesystem::path("_PipelineCache") / (title_id + ".shaders"); +} + bool GraphicsPipelineKey::operator==(const GraphicsPipelineKey& other) const { return std::memcmp(this, &other, sizeof(*this)) == 0; } @@ -331,6 +344,10 @@ ShaderModule& PipelineCache::CompileModule(Program& program, const ProgramKey& k options.input_info.pixel->parameter_mode == ShaderPixelParameterMode::Rectangle) { module.rect_parameters = RectListParameters(*options.input_info.pixel, module.info); } + // Programs with GPU-selected calls depend on their callees' current code. + if (program.call_source == nullptr) { + StoreModule(key, program, module, result.spirv); + } return module; } @@ -442,11 +459,13 @@ const ShaderModule* PipelineCache::GetProgram(const ShaderParams& params, InputI } PipelineCache::PipelineCache(GraphicContext& graphics, - ShaderRecompiler::IR::ImageHeapSource* image_heap) + ShaderRecompiler::IR::ImageHeapSource* image_heap, + const std::filesystem::path& shader_store) : m_graphics(graphics), m_image_heap(image_heap), m_next_driver_cache_save(std::chrono::steady_clock::now() + DriverCacheSaveInterval) { EXIT_NOT_IMPLEMENTED(!Common::Thread::IsMainThread()); InitializeDriverCache(); + WarmUp(shader_store); } PipelineCache::~PipelineCache() { @@ -650,6 +669,20 @@ void PipelineCache::GetGraphicsPrograms(const CommandBuffer& command, DrawRender std::array vertex_params; if (tess_active) { vertex_params = PrepareTessellationPrograms(vertex_regs, context, vertex_info); + // Reflection decodes both programs; the key also covers the VGT fields in tess. + const uint64_t layout_key[] = {vertex_params[0].hash, vertex_params[1].hash, + uint64_t {sh.m_vgtLsHsConfig & 0xfff00u} << 32u | + sh.m_vgtTfParam}; + const auto [layout, inserted] = m_tess_layouts.try_emplace( + XXH3_64bits(layout_key, sizeof(layout_key)), vertex_info[0].tess); + if (inserted) { + ShaderRecompiler::AnalyzeTessellationPrograms(vertex_params[0].code, + vertex_params[1].code, layout->second); + StoreLayout(layout->first, layout->second); + } + for (auto& stage: vertex_info) { + stage.tess = layout->second; + } } else { vertex_params[0] = PrepareProgram(vertex_regs, context, user_config, vertex_info[0]); } @@ -1017,6 +1050,7 @@ PipelineCache::Pipeline& PipelineCache::GetGraphicsPipeline(const CommandBuffer& Common::DebugCounters::Peak::PipelineCreateNs); CreatePipelineInternal(m_graphics, pipeline, key, stages, m_driver_cache); } + StorePipeline(key); CheckpointDriverCache(); return pipeline; diff --git a/src/graphics/host_gpu/renderer/pipeline/pipelineCache.h b/src/graphics/host_gpu/renderer/pipeline/pipelineCache.h index 7ac17c48d..49030e173 100644 --- a/src/graphics/host_gpu/renderer/pipeline/pipelineCache.h +++ b/src/graphics/host_gpu/renderer/pipeline/pipelineCache.h @@ -4,6 +4,7 @@ #include "common/abi.h" #include "common/assert.h" #include "common/common.h" +#include "common/file.h" #include "graphics/host_gpu/renderer/colorRenderTarget.h" #include "graphics/host_gpu/renderer/depthRenderTarget.h" #include "graphics/host_gpu/renderer/renderTarget.h" @@ -37,6 +38,10 @@ namespace ShaderRecompiler::IR { class ImageHeapSource; } // namespace ShaderRecompiler::IR +namespace ShaderRecompiler::Spirv { +struct Profile; +} // namespace ShaderRecompiler::Spirv + #pragma pack(push, 1) struct PipelineStaticParameters { @@ -139,7 +144,9 @@ struct DrawRenderState { // The owning renderer serializes access, including saves while the GPU is running. class PipelineCache { public: - PipelineCache(GraphicContext& graphics, ShaderRecompiler::IR::ImageHeapSource* image_heap); + // Without a shader store path nothing is kept across runs. + PipelineCache(GraphicContext& graphics, ShaderRecompiler::IR::ImageHeapSource* image_heap, + const std::filesystem::path& shader_store = {}); ~PipelineCache(); KYTY_CLASS_NO_COPY(PipelineCache); void Save(); @@ -197,6 +204,8 @@ private: std::size_t operator()(const GraphicsPipelineKey& key) const; }; + using ModuleMap = std::unordered_map; + GraphicContext& m_graphics; ShaderRecompiler::IR::ImageHeapSource* m_image_heap; std::unordered_map m_program_cache; @@ -209,6 +218,9 @@ private: bool m_driver_cache_dirty = false; std::unordered_map m_graphics_pipelines; std::unordered_map m_compute_pipelines; + std::unordered_map m_tess_layouts; + // Appends records of everything first created; invalid without a shader store. + Common::File m_store; template const ShaderModule* GetProgram(const ShaderParams& params, InputInfo& input_info, @@ -219,8 +231,27 @@ private: uint32_t push_data_cursor); void InitializeDriverCache(); void CheckpointDriverCache(); + + // pipelineSerialization.cpp + void WarmUp(const std::filesystem::path& path); + bool LoadRecord(std::span& records, std::vector& pipelines); + void StoreModule(const ProgramKey& key, const Program& program, const ShaderModule& module, + const std::vector& spirv); + void StorePipeline(const GraphicsPipelineKey& key); + void StoreLayout(uint64_t key, const ShaderTessellationInputInfo& layout); + // Every module by id, with id 0 naming an absent stage. + [[nodiscard]] ModuleMap MapModules() const; + // False when a stage id of the key is not in `modules`. + bool CreateGraphicsPipeline(Pipeline& pipeline, const GraphicsPipelineKey& key, + const ModuleMap& modules); }; +// The shader store of the running title; empty unless --shader-cache is on in a Release build +// without debug dumps, which change the SPIR-V. +std::filesystem::path ShaderStorePath(); +// Host capabilities that change the emitted SPIR-V. +ShaderRecompiler::Spirv::Profile SpirvProfile(const GraphicContext& graphics); + // Stages are indexed like GraphicsPipelineKey::stage_ids. void CreatePipelineInternal(GraphicContext& graphics, PipelineCache::Pipeline& pipeline, const GraphicsPipelineKey& key, diff --git a/src/graphics/host_gpu/renderer/pipeline/pipelineSerialization.cpp b/src/graphics/host_gpu/renderer/pipeline/pipelineSerialization.cpp new file mode 100644 index 000000000..ec5b94a92 --- /dev/null +++ b/src/graphics/host_gpu/renderer/pipeline/pipelineSerialization.cpp @@ -0,0 +1,298 @@ +#include "graphics/host_gpu/renderer/pipeline/pipelineCache.h" + +#include "common/logging/log.h" +#include "common/stringUtils.h" +#include "graphics/host_gpu/vulkanCommon.h" +#include "graphics/shader/recompiler/ShaderRecompiler.h" +#include "graphics/shader/recompiler/ir/Serialize.h" + +#include +#include +#include +#include +#include +#include +#include + +#if KYTY_PLATFORM == KYTY_PLATFORM_WINDOWS +#define WIN32_LEAN_AND_MEAN +#include +#elif defined(__APPLE__) +#include +#endif + +// The shader store: a header, then zstd-compressed records of everything a run created first. +// Records are only appended, so a store cut short by a crash keeps every complete record. +namespace Libs::Graphics { + +namespace { + +using ShaderRecompiler::IR::IoAll; +using ShaderRecompiler::IR::Reader; +using ShaderRecompiler::IR::Writer; + +enum class Record : uint8_t { Module, GraphicsPipeline, TessellationLayout }; + +// Precedes each record's zstd frame. +struct Frame { + uint32_t compressed_size = 0; + uint32_t raw_size = 0; + uint64_t checksum = 0; // XXH3 of the compressed bytes. +}; + +// Changes with every build, so records never meet code that encodes them differently. Zero when +// the executable cannot be found. +uint64_t BuildStamp() { +#if KYTY_PLATFORM == KYTY_PLATFORM_WINDOWS + std::array name {}; + GetModuleFileNameW(nullptr, name.data(), static_cast(name.size())); + const std::filesystem::path path = name.data(); +#elif defined(__APPLE__) + std::array name {}; + auto size = static_cast(name.size()); + _NSGetExecutablePath(name.data(), &size); + const std::filesystem::path path = name.data(); +#else + const std::filesystem::path path = "/proc/self/exe"; +#endif + std::error_code error; + const auto time = std::filesystem::last_write_time(path, error).time_since_epoch().count(); + return error ? 0 : static_cast(time); +} + +// What every record depends on besides its own fields; empty when the build is unknown. +std::vector StoreHeader(const GraphicContext& graphics) { + const auto stamp = BuildStamp(); + if (stamp == 0) { + return {}; + } + Writer header; + IoAll(header, stamp, SpirvProfile(graphics), + graphics.GetPhysicalDeviceProperties().limits.maxComputeSharedMemorySize); + return std::move(header.bytes); +} + +void Append(Common::File& store, const Writer& record) { + std::vector bytes(sizeof(Frame) + ZSTD_compressBound(record.bytes.size())); + const auto size = ZSTD_compress(bytes.data() + sizeof(Frame), bytes.size() - sizeof(Frame), + record.bytes.data(), record.bytes.size(), 3); + EXIT_IF(ZSTD_isError(size)); + const Frame frame {.compressed_size = static_cast(size), + .raw_size = static_cast(record.bytes.size()), + .checksum = XXH3_64bits(bytes.data() + sizeof(Frame), size)}; + std::memcpy(bytes.data(), &frame, sizeof(frame)); + // A failed write leaves a partial record; the next run drops it and everything after it. + store.Write(bytes.data(), static_cast(sizeof(Frame) + size)); +} + +// Record fields, shared by writing and reading. +template void IoKey(S& io, Key& key) { + auto& [stage, hash, user_data_count, static_state] = key; + IoAll(io, stage, hash, user_data_count, static_state); +} + +template void IoModule(S& io, Module& stage) { + // The handle comes from the SPIR-V; stored programs have no function code. + auto& [id, handle, info, specialization, function_code, vertex_formats, rect_parameters] = stage; + IoAll(io, id, info, specialization, vertex_formats, rect_parameters); +} + +} // namespace + +void PipelineCache::WarmUp(const std::filesystem::path& path) { + if (path.empty()) { + return; + } + const auto header = StoreHeader(m_graphics); + if (header.empty()) { + return; + } + const auto start = std::chrono::steady_clock::now(); + std::vector bytes; + if (Common::File file; + Common::File::IsFileExisting(path) && file.Open(path, Common::File::Mode::Read)) { + bytes.resize(file.Size()); + uint32_t read = 0; + file.Read(bytes.data(), static_cast(bytes.size()), &read); + bytes.resize(read); + } + std::vector pipelines; + auto records = std::span(bytes); + const bool reusable = + records.size() >= header.size() && std::ranges::equal(records.first(header.size()), header); + if (reusable) { + records = records.subspan(header.size()); + while (LoadRecord(records, pipelines)) { + } + } + // A damaged record ends the store; the records before it stay. + const auto kept = reusable ? bytes.size() - records.size() : 0; + Common::File::CreateDirectories(path.parent_path()); + if (kept == 0) { + if (m_store.Create(path)) { + m_store.Write(header.data(), static_cast(header.size())); + } + } else if (m_store.Open(path, Common::File::Mode::ReadWrite)) { + m_store.Truncate(kept); + m_store.Seek(kept); + } + + const auto modules = MapModules(); + for (const auto& key: pipelines) { + Pipeline pipeline; + if (CreateGraphicsPipeline(pipeline, key, modules)) { + m_graphics_pipelines.emplace(key, pipeline); + } + } + // A compute pipeline depends only on its module. + for (const auto* module: modules | std::views::values) { + if (module != nullptr && module->info.stage == ShaderType::Compute) { + CreatePipelineInternal(m_graphics, m_compute_pipelines[module->id], *module, + m_driver_cache); + } + } + if (!m_graphics_pipelines.empty() || !m_compute_pipelines.empty()) { + CheckpointDriverCache(); + } + Log::WriteToConsoleAndLog(fmt::format( + "Shader cache: loaded {} programs, {} modules, {} pipelines from {} ({:.1f} MB) in {} ms\n", + m_program_cache.size(), modules.size() - 1, + m_graphics_pipelines.size() + m_compute_pipelines.size(), Common::PathToString(path), + static_cast(bytes.size()) / 1'000'000.0, + std::chrono::duration_cast(std::chrono::steady_clock::now() - + start) + .count())); +} + +bool PipelineCache::LoadRecord(std::span& records, + std::vector& pipelines) { + Frame frame; + if (records.size() < sizeof(frame)) { + return false; + } + std::memcpy(&frame, records.data(), sizeof(frame)); + if (records.size() - sizeof(frame) < frame.compressed_size) { + return false; + } + const auto compressed = records.subspan(sizeof(frame), frame.compressed_size); + if (XXH3_64bits(compressed.data(), compressed.size()) != frame.checksum || + ZSTD_getFrameContentSize(compressed.data(), compressed.size()) != frame.raw_size) { + return false; + } + std::vector raw(frame.raw_size); + if (ZSTD_decompress(raw.data(), raw.size(), compressed.data(), compressed.size()) != raw.size()) { + return false; + } + Reader reader {raw}; + auto type = Record::Module; + const auto complete = [&] { return reader.ok && reader.bytes.empty(); }; + IoAll(reader, type); + switch (type) { + case Record::Module: { + ProgramKey key; + std::vector plan; + ShaderModule module; + std::vector spirv; + IoKey(reader, key); + IoAll(reader, plan); + IoModule(reader, module); + IoAll(reader, spirv); + if (!complete()) { + return false; + } + const auto [program, inserted] = m_program_cache.try_emplace(std::move(key)); + if (inserted && + !ShaderRecompiler::IR::LoadResourcePlan(plan, program->second.resource_plan)) { + m_program_cache.erase(program); + return false; + } + module.module = CompileSPV(spirv, m_graphics.device); + if (module.module == nullptr) { + return false; + } + program->second.modules.push_back(std::move(module)); + break; + } + case Record::GraphicsPipeline: { + GraphicsPipelineKey key; + reader.Raw(&key, sizeof(key)); + if (!complete()) { + return false; + } + pipelines.push_back(key); + break; + } + case Record::TessellationLayout: { + uint64_t key = 0; + ShaderTessellationInputInfo layout; + IoAll(reader, key, layout); + if (!complete()) { + return false; + } + m_tess_layouts.try_emplace(key, layout); + break; + } + default: return false; + } + records = records.subspan(sizeof(frame) + compressed.size()); + return true; +} + +void PipelineCache::StoreModule(const ProgramKey& key, const Program& program, + const ShaderModule& module, const std::vector& spirv) { + if (m_store.IsInvalid()) { + return; + } + Writer record; + IoAll(record, Record::Module); + IoKey(record, key); + IoAll(record, ShaderRecompiler::IR::SaveResourcePlan(program.resource_plan)); + IoModule(record, module); + IoAll(record, spirv); + Append(m_store, record); +} + +void PipelineCache::StorePipeline(const GraphicsPipelineKey& key) { + if (m_store.IsInvalid()) { + return; + } + Writer record; + IoAll(record, Record::GraphicsPipeline); + record.Raw(&key, sizeof(key)); + Append(m_store, record); +} + +void PipelineCache::StoreLayout(uint64_t key, const ShaderTessellationInputInfo& layout) { + if (m_store.IsInvalid()) { + return; + } + Writer record; + IoAll(record, Record::TessellationLayout, key, layout); + Append(m_store, record); +} + +PipelineCache::ModuleMap PipelineCache::MapModules() const { + ModuleMap modules {{0, nullptr}}; + for (const auto& program: m_program_cache | std::views::values) { + for (const auto& module: program.modules) { + modules.emplace(module.id, &module); + } + } + return modules; +} + +bool PipelineCache::CreateGraphicsPipeline(Pipeline& pipeline, const GraphicsPipelineKey& key, + const ModuleMap& modules) { + std::array stages {}; + for (size_t i = 0; i < stages.size(); i++) { + const auto found = modules.find(key.stage_ids[i]); + if (found == modules.end()) { + return false; + } + stages[i] = found->second; + } + CreatePipelineInternal(m_graphics, pipeline, key, stages, m_driver_cache); + return true; +} + +} // namespace Libs::Graphics diff --git a/src/graphics/host_gpu/renderer/renderContext.cpp b/src/graphics/host_gpu/renderer/renderContext.cpp index d320ab46a..83db56606 100644 --- a/src/graphics/host_gpu/renderer/renderContext.cpp +++ b/src/graphics/host_gpu/renderer/renderContext.cpp @@ -10,10 +10,10 @@ namespace Libs::Graphics { -RenderContext::RenderContext(GraphicContext& graphics) +RenderContext::RenderContext(GraphicContext& graphics, const std::filesystem::path& shader_store) : m_graphics(graphics), m_render_executor(*this), m_command_scheduler(*this, graphics), m_descriptor_heap(graphics, m_command_scheduler.GetMasterSemaphore()), - m_pipeline_cache(graphics, &m_image_heap), m_sampler_cache(graphics), + m_pipeline_cache(graphics, &m_image_heap, shader_store), m_sampler_cache(graphics), m_buffer_cache(graphics, m_command_scheduler, m_page_manager, m_texture_cache), m_texture_cache(graphics, m_command_scheduler, m_page_manager, m_buffer_cache), m_image_heap(*this, m_page_manager), m_occlusion_queries(graphics, m_command_scheduler) { diff --git a/src/graphics/host_gpu/renderer/renderContext.h b/src/graphics/host_gpu/renderer/renderContext.h index aeff01afd..748902fb3 100644 --- a/src/graphics/host_gpu/renderer/renderContext.h +++ b/src/graphics/host_gpu/renderer/renderContext.h @@ -31,7 +31,8 @@ class GuestGpu; class RenderContext { public: - explicit RenderContext(GraphicContext& graphics); + explicit RenderContext(GraphicContext& graphics, + const std::filesystem::path& shader_store = ShaderStorePath()); ~RenderContext(); KYTY_CLASS_NO_COPY(RenderContext); diff --git a/src/graphics/shader/recompiler/ir/Serialize.h b/src/graphics/shader/recompiler/ir/Serialize.h index 1dfaf13b9..11156bc1b 100644 --- a/src/graphics/shader/recompiler/ir/Serialize.h +++ b/src/graphics/shader/recompiler/ir/Serialize.h @@ -2,8 +2,10 @@ #define EMULATOR_INCLUDE_EMULATOR_GRAPHICS_SHADER_RECOMPILER_SERIALIZE_H_ #include "common/assert.h" +#include "graphics/shader/recompiler/backend/spirv/SpirvEmitter.h" #include "graphics/shader/recompiler/ir/ShaderIR.h" #include "graphics/shader/recompiler/ir/passes/ResourceMaterialization.h" +#include "graphics/shader/rectListShader.h" #include #include @@ -167,6 +169,11 @@ KYTY_IO_FIELDS(ResourceSpecialization::Buffer, a, b, c, d, e, f, g, h) KYTY_IO_FIELDS(ResourceSpecialization::Image, a, b, c, d, e, f, g, h, i, j, k) KYTY_IO_FIELDS(ResourceSpecialization::Sampler, a, b, c) KYTY_IO_FIELDS(ResourceSpecialization, a, b, c) +KYTY_IO_FIELDS(Bindings, a, b, c, d, e) +KYTY_IO_FIELDS(CompiledShaderInfo, a, b, c, d, e, f, g, h, i, j) +KYTY_IO_FIELDS(Spirv::Profile, a, b, c) +KYTY_IO_FIELDS(ShaderTessellationInputInfo, a, b, c, d, e, f, g, h, i) +KYTY_IO_FIELDS(RectListParameter, a, b, c) #undef KYTY_IO_FIELDS } // namespace Libs::Graphics::ShaderRecompiler::IR diff --git a/src/graphics/shader/shader.cpp b/src/graphics/shader/shader.cpp index 1f4e49f32..4752c1df1 100644 --- a/src/graphics/shader/shader.cpp +++ b/src/graphics/shader/shader.cpp @@ -11,7 +11,6 @@ #include "graphics/guest_gpu/hardwareContext.h" #include "graphics/host_gpu/renderer/renderContext.h" #include "graphics/shader/recompiler/ShaderRecompiler.h" -#include "graphics/shader/recompiler/Tessellation.h" #include "graphics/shader/recompiler/frontend/decode/ShaderDecoder.h" #include "graphics/shader/shaderCompiler.h" #include "graphics/shader/shaderVertexMetadata.h" @@ -53,8 +52,6 @@ struct ShaderMapEntry { static std::unique_ptr> g_shader_map; static std::mutex g_shader_map_mutex; -static std::unordered_map g_tess_layouts; -static std::mutex g_tess_layouts_mutex; void ShaderInit() { EXIT_IF(g_shader_map != nullptr); @@ -662,18 +659,6 @@ PrepareTessellationPrograms(const HW::VertexShaderInfo& regs, const HW::Context& }; EXIT_IF(tess.input_control_points == 0 || tess.input_control_points > 32 || tess.output_control_points == 0 || tess.output_control_points > 32); - // Reflection decodes both programs; the key also covers the VGT fields cached in tess. - const uint64_t key[] = {local_hash, control_hash, - uint64_t {sh.m_vgtLsHsConfig & 0xfff00u} << 32u | sh.m_vgtTfParam}; - { - std::scoped_lock lock(g_tess_layouts_mutex); - auto [entry, inserted] = g_tess_layouts.try_emplace(XXH3_64bits(key, sizeof(key)), tess); - if (inserted) { - ShaderRecompiler::AnalyzeTessellationPrograms(params[0].code, params[1].code, - entry->second); - } - tess = entry->second; - } for (auto& stage: input_info) { stage.tess = tess; } diff --git a/src/graphics/shader/shaderCompiler.h b/src/graphics/shader/shaderCompiler.h index a348b6e93..dceed4fc9 100644 --- a/src/graphics/shader/shaderCompiler.h +++ b/src/graphics/shader/shaderCompiler.h @@ -32,6 +32,7 @@ void BuildStageStaticKey(const ShaderComputeInputInfo& input_info, std::vector PrepareTessellationPrograms(const HW::VertexShaderInfo& regs, const HW::Context& context, std::array& input_info); diff --git a/src/main.cpp b/src/main.cpp index 7a3795799..13c759301 100644 --- a/src/main.cpp +++ b/src/main.cpp @@ -80,6 +80,7 @@ static void PrintUsage() { ::printf(" --gpu-assisted-validation Bounds-check shader accesses on the GPU.\n" " Implies --vulkan-validation; very slow.\n"); ::printf(" --shader-validation Enable shader validation.\n"); + ::printf(" --shader-cache Keep compiled shaders and pipelines between boots.\n"); ::printf(" --tessellation Draw tessellation patches; skipped by default.\n"); ::printf(" --occlusion-queries Count occlusion query samples on the GPU.\n"); ::printf(" --shader-optimization-type None, Size, or Performance.\n"); @@ -205,6 +206,7 @@ static constexpr Flag FLAGS[] = { {"--vulkan-validation", &ConfigOptions::vulkan_validation_enabled}, {"--gpu-assisted-validation", &ConfigOptions::gpu_assisted_validation_enabled}, {"--shader-validation", &ConfigOptions::shader_validation_enabled}, + {"--shader-cache", &ConfigOptions::shader_cache_enabled}, {"--tessellation", &ConfigOptions::tessellation_enabled}, {"--occlusion-queries", &ConfigOptions::occlusion_queries_enabled}, {"--command-buffer-dump", &ConfigOptions::command_buffer_dump_enabled}, diff --git a/tests/ShaderRecompilerComputeTests.cpp b/tests/ShaderRecompilerComputeTests.cpp index 6f01ddfb5..1064d0b21 100644 --- a/tests/ShaderRecompilerComputeTests.cpp +++ b/tests/ShaderRecompilerComputeTests.cpp @@ -588,19 +588,13 @@ struct PipelineCacheTestAccess { // Replaces each graphics pipeline whose stage ids name compiled modules with one created // from only its key and those modules. The GPU must be idle. Returns how many it replaced. static uint32_t RecreateGraphicsPipelines(PipelineCache &cache) { - std::unordered_map modules{{0, nullptr}}; // Id 0: absent. - for (const auto &program : cache.m_program_cache | std::views::values) - for (const auto &module : program.modules) modules.emplace(module.id, &module); + const auto modules = cache.MapModules(); auto &device = cache.m_graphics.device; uint32_t count = 0; for (auto &[key, pipeline] : cache.m_graphics_pipelines) { - // Test-owned modules are not in the cache. - if (!std::ranges::all_of(key.stage_ids, [&](uint64_t id) { return modules.contains(id); })) - continue; - std::array stages{}; - for (size_t i = 0; i < stages.size(); i++) stages[i] = modules.at(key.stage_ids[i]); PipelineCache::Pipeline recreated; - CreatePipelineInternal(cache.m_graphics, recreated, key, stages, nullptr); + // Test-owned modules are not in the cache. + if (!cache.CreateGraphicsPipeline(recreated, key, modules)) continue; device.destroyPipeline(pipeline.pipeline, nullptr); device.destroyPipelineLayout(pipeline.pipeline_layout, nullptr); device.destroyDescriptorSetLayout(pipeline.descriptor_set_layout, nullptr); @@ -611,6 +605,17 @@ struct PipelineCacheTestAccess { } static auto Compiled(const PipelineCache &cache) { return cache.m_compiled; } + + static void StoreLayout(PipelineCache &cache, uint64_t key, + const ShaderTessellationInputInfo &layout) { + cache.StoreLayout(key, layout); + } + + static const auto &TessLayouts(const PipelineCache &cache) { return cache.m_tess_layouts; } + + static std::array Pipelines(const PipelineCache &cache) { + return {cache.m_graphics_pipelines.size(), cache.m_compute_pipelines.size()}; + } }; namespace { @@ -44700,6 +44705,163 @@ void CheckStableProgramIds(VulkanHarness &vulkan) { std::printf("[gpu] %-32s ok\n", name); } +// With a shader store a restart compiles nothing and creates every pipeline before the first +// lookup. A damaged record and everything after it are dropped; a stale store starts over. +void CheckShaderStore(VulkanHarness &vulkan) { + constexpr const char *name = "ShaderStore"; + constexpr uintptr_t base = 0x0000000205000000ull; + constexpr uint64_t allocation_size = 0x10000; + constexpr u32 value = 0x2a; + std::vector compute; + AppendVMovU32(&compute, 1, value); + compute.insert(compute.end(), {EncodeMubuf0(0x1c), EncodeMubuf1(1, 0, 0)}); + AppendEnd(&compute); + std::vector vertex; + AppendVMovU32(&vertex, 0, 0); + AppendVMovLiteral(&vertex, 1, 0x3f800000u); + vertex.insert(vertex.end(), {EncodeExp0(0x0c, 0xf), EncodeExp1(0, 0, 0, 1)}); + AppendEnd(&vertex); + std::vector pixel; + AppendVMovLiteral(&pixel, 0, 0x3f800000u); + pixel.insert(pixel.end(), {EncodeExp0(0x00, 0xf), EncodeExp1(0, 0, 0, 0)}); + AppendEnd(&pixel); + static ShaderUserData vertex_user_data{}; + const auto map = [](const std::vector &code, Prospero::ShaderBinaryType type, + ShaderUserData *user_data = nullptr) { + ShaderMapUserData(reinterpret_cast(code.data()), + {.type = type, .user_data = user_data, + .code_size_bytes = static_cast(code.size() * sizeof(u32))}); + }; + map(compute, Prospero::ShaderBinaryType::kCs); + map(vertex, Prospero::ShaderBinaryType::kGs, &vertex_user_data); + map(pixel, Prospero::ShaderBinaryType::kPs); + int64_t direct_offset = -1; + void *mapped = reinterpret_cast(base); + Require(name, "output memory", + LibKernel::Memory::KernelAllocateDirectMemory( + 0, LibKernel::Memory::KernelGetDirectMemorySize(), allocation_size, 0x10000, 0, + &direct_offset) == 0 && + LibKernel::Memory::KernelMapDirectMemory(&mapped, allocation_size, 0x3, 0x10, + direct_offset, 0x10000) == 0, + "output allocation or mapping failed"); + const auto store = std::filesystem::temp_directory_path() / + fmt::format("kyty_shader_store_{}", + std::chrono::steady_clock::now().time_since_epoch().count()) / + "TEST00000.shaders"; + + struct Session { + std::array prewarmed{}, pipelines{}; + std::array(ShaderType::TessellationEvaluation) + 1> compiled{}; + u32 output = 0; + bool operator==(const Session &) const = default; + }; + // Dispatches the compute program and creates the graphics pipeline. + const auto run = [&] { + std::memset(mapped, 0, allocation_size); + RenderContext context(vulkan.RuntimeContext(), store); + auto &pipelines = context.GetPipelineCache(); + Session session{.prewarmed = PipelineCacheTestAccess::Pipelines(pipelines)}; + context.InitializeGpu(nullptr); + LibKernel::Memory::InstallGpuResources(&context); + context.GetGpu().SendCommandSync([&] { + CommandProcessor processor(context, 0); + processor.BufferInit(); + auto ®isters = processor.GetCtx(); + auto &shaders = processor.GetShCtx(); + registers.SetRenderTargetMask(0xf); + registers.SetShaderMask(0xf); + registers.SetTargetOutputMode(0, 4); + registers.SetViewportTransformControl(0x300); + processor.GetUcfg().SetPrimitiveType(Prospero::PrimitiveType::kTriList); + shaders.SetEsShaderBase(reinterpret_cast(vertex.data())); + shaders.SetPsShaderBase(reinterpret_cast(pixel.data())); + shaders.SetCsShader({.data_addr = reinterpret_cast(compute.data()), + .num_thread_x = 1, .num_thread_y = 1, .num_thread_z = 1, + .wave_size = 64, .user_sgpr = 4}); + const ShaderBufferResource output{{static_cast(base), static_cast(base >> 32u), + 4, DstSel(4, 5, 6, 7) | (static_cast(Prospero::BufferFormat::k32UInt) << 12u)}}; + for (u32 i = 0; i < 4; i++) shaders.SetCsUserSgpr(i, output.fields[i]); + context.MapMemory(base, allocation_size); + processor.DispatchDirect(1, 1, 1, 0x41u); + auto &scheduler = context.GetCommandScheduler(); + DrawRenderState state{.topology = vk::PrimitiveTopology::eTriangleList}; + pipelines.GetGraphicsPrograms(scheduler.Current(), state); + (void)pipelines.GetGraphicsPipeline(scheduler.Current(), state); + context.GetBufferCache().ReadMemory(base, sizeof(u32)); + (void)LibKernel::Memory::TryReadBacking(base, &session.output, sizeof(u32)); + context.UnmapMemory(base, allocation_size); + scheduler.Finish(); + }); + context.ShutdownGpu(); + LibKernel::Memory::InstallGpuResources(nullptr); + session.compiled = PipelineCacheTestAccess::Compiled(pipelines); + session.pipelines = PipelineCacheTestAccess::Pipelines(pipelines); + return session; + }; + const auto flip = [&](uint64_t offset) { + Common::File file(store, Common::File::Mode::ReadWrite); + uint8_t byte = 0; + file.Seek(offset); + file.Read(&byte, 1); + byte ^= 0xffu; + file.Seek(offset); + file.Write(&byte, 1); + }; + + const auto cold = run(); + const auto size = std::filesystem::file_size(store); + Require(name, "cold start", + cold.output == value && cold.prewarmed == std::array{} && + cold.pipelines == std::array{1, 1} && + cold.compiled[static_cast(ShaderType::Compute)] == 1, + "the first run did not create its store, dispatch or pipelines"); + auto warm = cold; + warm.prewarmed = cold.pipelines; + warm.compiled = {}; + Require(name, "warm restart", run() == warm && std::filesystem::file_size(store) == size, + "a restart compiled a module, created a pipeline late or rewrote the store"); + std::filesystem::resize_file(store, size + 3); + Require(name, "torn tail", run() == warm && std::filesystem::file_size(store) == size, + "a torn last record was kept"); + // The graphics pipeline's record is the last one. + auto lost_pipeline = warm; + lost_pipeline.prewarmed[0] = 0; + std::filesystem::resize_file(store, size - 3); + Require(name, "truncated record", + run() == lost_pipeline && std::filesystem::file_size(store) == size, + "a cut record was not dropped and appended again, or took earlier records along"); + // Flips the top checksum byte, just before the last zstd frame's magic 0xfd2fb528. + std::vector bytes(size); + Common::File(store, Common::File::Mode::Read).Read(bytes.data(), static_cast(size)); + const std::array zstd_magic{0x28, 0xb5, 0x2f, 0xfd}; + flip(std::ranges::find_end(bytes, zstd_magic).begin() - bytes.begin() - 1); + Require(name, "damaged record", + run() == lost_pipeline && std::filesystem::file_size(store) == size, + "a damaged record was loaded or not replaced"); + flip(0); + Require(name, "stale store", run() == cold && std::filesystem::file_size(store) == size, + "a store with another header was reused instead of started over"); + // Reflecting a layout decodes LS and HS, so a restart takes it from the store. + const ShaderTessellationInputInfo layout{1, 2, 3, 4, 5, 6, 7, 8, 9}; + { + RenderContext context(vulkan.RuntimeContext(), store); + PipelineCacheTestAccess::StoreLayout(context.GetPipelineCache(), 1, layout); + } + { + RenderContext context(vulkan.RuntimeContext(), store); + const auto &layouts = PipelineCacheTestAccess::TessLayouts(context.GetPipelineCache()); + Require(name, "tessellation layout", + layouts.contains(1) && std::memcmp(&layouts.at(1), &layout, sizeof(layout)) == 0, + "a restart did not load the stored tessellation layout"); + } + std::filesystem::remove_all(store.parent_path()); + Require(name, "release", + LibKernel::Memory::KernelMunmap(base, allocation_size) == 0 && + LibKernel::Memory::KernelReleaseDirectMemory(direct_offset, allocation_size) == 0, + "output memory release failed"); + std::printf("[gpu] %-32s ok\n", name); +} + void CheckPs5GameExampleImageClearRuntimeShape() { const auto MakeCode = [] { std::vector code; @@ -50326,6 +50488,11 @@ int main(int argc, char **argv) { CheckStableProgramIds(vulkan); return 0; } + if (argc == 2 && std::strcmp(argv[1], "--shader-cache-only") == 0) { + VulkanHarness vulkan; + CheckShaderStore(vulkan); + return 0; + } if (argc == 2 && std::strcmp(argv[1], "--lds-limit-only") == 0) { VulkanHarness vulkan; CheckComputeLdsLimit(vulkan); @@ -51488,6 +51655,7 @@ int main(int argc, char **argv) { vulkan.CheckNativeIndirectDispatch(); CheckComputeLdsLimit(vulkan); CheckStableProgramIds(vulkan); + CheckShaderStore(vulkan); vulkan.CheckOrderedDispatch(); vulkan.CheckZeroStrideBufferBinding(); vulkan.CheckNullTextureDescriptors();