From bfc377627a3c54b280b560eb518bb31374b6c0ea Mon Sep 17 00:00:00 2001 From: nmzik Date: Sat, 24 Oct 2026 22:44:00 +0200 Subject: [PATCH] pipeline: create pipelines from GraphicsPipelineKey and shader modules --- src/graphics/host_gpu/renderer/debug.cpp | 7 - .../renderer/pipeline/descriptors.cpp | 20 +- .../renderer/pipeline/pipelineCache.cpp | 325 +++++++++-------- .../renderer/pipeline/pipelineCache.h | 130 +++---- .../pipeline/shaderResourceBarrier.cpp | 3 +- .../host_gpu/renderer/pipeline/shaders.cpp | 273 +++------------ .../host_gpu/renderer/renderCompute.cpp | 12 +- src/graphics/host_gpu/renderer/renderDraw.cpp | 12 +- src/graphics/shader/rectListShader.cpp | 128 ++++--- src/graphics/shader/rectListShader.h | 21 +- src/graphics/shader/shader.h | 13 +- tests/ShaderRecompilerComputeTests.cpp | 328 ++++++++++-------- tests/shaderCfgTests.cpp | 17 +- 13 files changed, 594 insertions(+), 695 deletions(-) diff --git a/src/graphics/host_gpu/renderer/debug.cpp b/src/graphics/host_gpu/renderer/debug.cpp index bda659b55..4b1d7198e 100644 --- a/src/graphics/host_gpu/renderer/debug.cpp +++ b/src/graphics/host_gpu/renderer/debug.cpp @@ -595,13 +595,6 @@ void LogDrawPhase(const char* draw_name, const char* phase) { } } -void LogPipelineTrace(const char* phase, uint64_t vertex_program_id, uint64_t pixel_program_id) { - if (graphics_debug_dump_enabled()) { - LOGF("PipelineTrace: %s VS=%" PRIu64 " PS=%" PRIu64 "\n", phase, vertex_program_id, - pixel_program_id); - } -} - static void VpPrint(const char* func, const HW::ScreenViewport& vp, const HW::ScanModeControl& smc) { LOGF("%s\n", func); diff --git a/src/graphics/host_gpu/renderer/pipeline/descriptors.cpp b/src/graphics/host_gpu/renderer/pipeline/descriptors.cpp index ca99e3be0..0e63244da 100644 --- a/src/graphics/host_gpu/renderer/pipeline/descriptors.cpp +++ b/src/graphics/host_gpu/renderer/pipeline/descriptors.cpp @@ -681,7 +681,7 @@ void RenderExecutor::PrepareBindings(const ShaderStageRuntime& runtime, PreparedBindings& prepared) { KYTY_PROFILER_FUNCTION(); EXIT_IF(!runtime); - const auto& program = *runtime.program; + const auto& program = runtime.module->info; const auto& snapshot = *runtime.resources; prepared.runtime = &runtime; prepared.gds = {nullptr, 0, VK_WHOLE_SIZE}; @@ -723,7 +723,7 @@ void RenderExecutor::FindBuffers(std::span stages) { for (auto* stage: stages) { auto& prepared = *stage; EXIT_IF(prepared.runtime == nullptr || !*prepared.runtime); - const auto& program = *prepared.runtime->program; + const auto& program = prepared.runtime->module->info; const auto& snapshot = *prepared.runtime->resources; const auto& layout = program.bindings; prepared.buffer_sources.resize(layout.descriptor_counts[static_cast(BindingKind::Buffers)]); @@ -780,7 +780,7 @@ void RenderExecutor::FindBuffers(std::span stages) { void RenderExecutor::RebindBuffers(PreparedBindings& prepared) { KYTY_PROFILER_FUNCTION(); EXIT_IF(prepared.runtime == nullptr || !*prepared.runtime); - const auto& program = *prepared.runtime->program; + const auto& program = prepared.runtime->module->info; const auto& snapshot = *prepared.runtime->resources; const auto& layout = program.bindings; EXIT_IF(prepared.buffer_sources.size() != layout.descriptor_counts[static_cast(BindingKind::Buffers)]); @@ -813,7 +813,7 @@ void RenderExecutor::RebindBuffers(PreparedBindings& prepared) { void RenderExecutor::RebindImages(PreparedBindings& prepared) { KYTY_PROFILER_FUNCTION(); EXIT_IF(prepared.runtime == nullptr || !*prepared.runtime); - const auto& program = *prepared.runtime->program; + const auto& program = prepared.runtime->module->info; const auto& snapshot = *prepared.runtime->resources; auto& images = prepared.images; EXIT_IF(images.size() != program.info.images.size()); @@ -859,8 +859,8 @@ void RenderExecutor::PrepareGraphicsBindings(std::span bool uses_image_heap = false; FindBuffers(stages); for (auto* stage: stages) { - uses_dma |= stage->runtime->program->info.uses_dma; - uses_image_heap |= stage->runtime->program->info.uses_image_heap; + uses_dma |= stage->runtime->module->info.info.uses_dma; + uses_image_heap |= stage->runtime->module->info.info.uses_image_heap; } if (uses_dma) { m_context.PrepareBda(); @@ -909,7 +909,7 @@ void RenderExecutor::CommitBindings(CommandBuffer& buffer, : vk::ShaderStageFlags {}; for (const auto* prepared: prepared_bindings) { EXIT_IF(prepared == nullptr || prepared->runtime == nullptr || !*prepared->runtime); - const auto& program = *prepared->runtime->program; + const auto& program = prepared->runtime->module->info; write_count += std::popcount(program.bindings.descriptor_mask); for (auto mask = program.bindings.descriptor_mask; mask != 0; mask &= mask - 1) { const auto index = std::countr_zero(mask); @@ -930,7 +930,7 @@ void RenderExecutor::CommitBindings(CommandBuffer& buffer, const auto& reads = reader->runtime->resources->specialization_reads; if (reads.empty()) continue; for (const auto* writer: prepared_bindings) { - if (writer->runtime->program->has_address_writes) { + if (writer->runtime->module->info.has_address_writes) { EXIT("scalar resource reads cannot be proven disjoint from shader address writes\n"); } } @@ -947,7 +947,7 @@ void RenderExecutor::CommitBindings(CommandBuffer& buffer, } } for (const auto* writer: prepared_bindings) { - const auto& program = *writer->runtime->program; + const auto& program = writer->runtime->module->info; for (const auto& resource: program.info.buffers) { if (!resource.written || resource.descriptor_index == UINT32_MAX) continue; const auto& written = writer->buffer_sources[resource.descriptor_index]; @@ -973,7 +973,7 @@ void RenderExecutor::CommitBindings(CommandBuffer& buffer, std::array address_buffers; for (auto* prepared: prepared_bindings) { - const auto& program = *prepared->runtime->program; + const auto& program = prepared->runtime->module->info; auto& descriptors = *prepared; const auto shader_stage = NativeShaderStage(program.stage); const auto shader_stages = ShaderPipelineStages(shader_stage); diff --git a/src/graphics/host_gpu/renderer/pipeline/pipelineCache.cpp b/src/graphics/host_gpu/renderer/pipeline/pipelineCache.cpp index 0faa31600..c5cafce05 100644 --- a/src/graphics/host_gpu/renderer/pipeline/pipelineCache.cpp +++ b/src/graphics/host_gpu/renderer/pipeline/pipelineCache.cpp @@ -15,6 +15,7 @@ #include "graphics/host_gpu/renderer/pipeline/blendMapping.h" #include "graphics/host_gpu/renderer/render.h" #include "graphics/host_gpu/renderer/renderContext.h" +#include "graphics/shader/recompiler/BufferFormat.h" #include "graphics/shader/recompiler/ShaderRecompiler.h" #include "graphics/shader/shaderCompiler.h" #include "kernel/memory.h" @@ -198,86 +199,122 @@ void PackSpecialization(const ShaderRecompiler::IR::ResourceSpecialization& spec } } -} // namespace - -std::size_t PipelineCache::GraphicsPipelineKeyHash::operator()(const GraphicsPipelineKey& key) const { - std::size_t hash = 0; - PipelineKeyHash::Mix(hash, key.rendering.color_count); - for (uint32_t i = 0; i < key.rendering.color_count; i++) { - PipelineKeyHash::Mix(hash, static_cast(key.rendering.color_formats[i])); +// Formats with a narrower host variant drop the components the shader does not read. +vk::Format NarrowInputFormat(vk::Format format, uint32_t used_components) { + switch (format) { + case vk::Format::eR32G32B32A32Sfloat: + switch (used_components) { + case 1: return vk::Format::eR32Sfloat; + case 2: return vk::Format::eR32G32Sfloat; + case 3: return vk::Format::eR32G32B32Sfloat; + default: return format; + } + case vk::Format::eR32G32B32Sfloat: + switch (used_components) { + case 1: return vk::Format::eR32Sfloat; + case 2: return vk::Format::eR32G32Sfloat; + default: return format; + } + case vk::Format::eR16G16B16A16Sfloat: + switch (used_components) { + case 1: return vk::Format::eR16Sfloat; + case 2: return vk::Format::eR16G16Sfloat; + default: return format; + } + case vk::Format::eR8G8B8A8Unorm: + switch (used_components) { + case 1: return vk::Format::eR8Unorm; + case 2: return vk::Format::eR8G8Unorm; + default: return format; + } + case vk::Format::eR8G8B8A8Snorm: + return used_components == 2 ? vk::Format::eR8G8Snorm : format; + case vk::Format::eR8G8B8A8Uint: + switch (used_components) { + case 1: return vk::Format::eR8Uint; + case 2: return vk::Format::eR8G8Uint; + default: return format; + } + default: return format; } - PipelineKeyHash::Mix(hash, static_cast(key.rendering.depth_format)); - PipelineKeyHash::Mix(hash, static_cast(key.rendering.stencil_format)); - for (const auto id: key.vertex_shader_ids) { - PipelineKeyHash::Mix(hash, id); - } - PipelineKeyHash::Mix(hash, key.ps_shader_id); - PipelineKeyHash::Mix(hash, key.vertex_input.binding_count); - for (uint32_t i = 0; i < key.vertex_input.binding_count; i++) { - PipelineKeyHash::Mix(hash, key.vertex_input.bindings[i].stride); - PipelineKeyHash::Mix(hash, key.vertex_input.bindings[i].instance); - } - PipelineKeyHash::Mix(hash, key.vertex_input.attribute_count); - for (uint32_t i = 0; i < key.vertex_input.attribute_count; i++) { - PipelineKeyHash::Mix(hash, key.vertex_input.attributes[i].offset); - PipelineKeyHash::Mix(hash, key.vertex_input.attributes[i].binding); - } - PipelineKeyHash::Mix(hash, XXH3_64bits(&key.static_params, sizeof(key.static_params))); - return hash; } -struct PipelineCache::Module { - std::vector function_code; - ShaderRecompiler::IR::ResourceSpecialization specialization; - ShaderRecompiler::IR::CompiledShaderInfo info; - ShaderProgram handle; -}; - -struct PipelineCache::Program { - std::span FunctionCode() const { - return call_source - ? std::span(call_source->linked->code).subspan(call_source->code.size()) - : std::span{}; +vk::Format InputFormat(const ShaderBufferResource& resource, uint32_t used_components) { + switch (resource.RawFormat()) { + // Temporary: invalid PS5 buffer format 113 (VertexAttribFormat::k16_16SInt). + case 113: return NarrowInputFormat(vk::Format::eR32G32B32A32Sfloat, used_components); + case 121: return vk::Format::eR16G16Sfloat; + default: break; } - - bool RefreshCallSource(const ShaderRecompiler::IR::SrtRuntime& runtime) { - const auto revision = call_source->revision; - ShaderRecompiler::RefreshShaderSource(*call_source, runtime); - resource_plan.source_reads = call_source->reads; - return revision != call_source->revision; + const auto format = VulkanFormat(resource.Format()); + if (format == vk::Format::eUndefined || + ShaderRecompiler::Format::GetFormatInfo(resource.Format()).component_count == 0) { + EXIT("unknown vertex format: fmt = %u\n", resource.RawFormat()); } + return NarrowInputFormat(format, used_components); +} - template - Module* FindModule(const ShaderRecompiler::IR::SrtRuntime& runtime, uint32_t push_data_cursor) { - EXIT_IF(!ShaderRecompiler::IR::MaterializeResources( - resource_plan, runtime, resources, specialization)); - for (auto& candidate: modules) { - const auto& layout = candidate.info.bindings; - if (layout.push_data_start_dword != ShaderRecompiler::IR::PushData::StartFor( - push_data_cursor, layout.ShaderDataDwords()) || - candidate.specialization != specialization) { - continue; - } - if constexpr (HasCalls) { - if (!std::ranges::equal(candidate.function_code, FunctionCode())) continue; - } - return &candidate; +// The host format of each vertex input attribute, narrowed to the components the stage reads. +std::vector VertexInputFormats(const ShaderVertexInputInfo& input_info, + const ShaderRecompiler::IR::CompiledShaderInfo& program) { + std::vector formats(input_info.resources_num); + for (int index = 0; index < input_info.resources_num; index++) { + const auto registers_num = input_info.resources_dst[index].registers_num; + EXIT_IF(registers_num < 1 || registers_num > 4); + const auto fetched = program.info.vertex_fetch_components[index]; + formats[index] = InputFormat(input_info.resources[index], + fetched > 0 ? fetched : static_cast(registers_num)); + } + return formats; +} + +} // namespace + +bool GraphicsPipelineKey::operator==(const GraphicsPipelineKey& other) const { + return std::memcmp(this, &other, sizeof(*this)) == 0; +} + +std::size_t PipelineCache::GraphicsPipelineKeyHash::operator()(const GraphicsPipelineKey& key) const { + return XXH3_64bits(&key, sizeof(key)); +} + +std::span PipelineCache::Program::FunctionCode() const { + return call_source + ? std::span(call_source->linked->code).subspan(call_source->code.size()) + : std::span{}; +} + +bool PipelineCache::Program::RefreshCallSource(const ShaderRecompiler::IR::SrtRuntime& runtime) { + const auto revision = call_source->revision; + ShaderRecompiler::RefreshShaderSource(*call_source, runtime); + resource_plan.source_reads = call_source->reads; + return revision != call_source->revision; +} + +template +ShaderModule* PipelineCache::Program::FindModule(const ShaderRecompiler::IR::SrtRuntime& runtime, + uint32_t push_data_cursor) { + EXIT_IF(!ShaderRecompiler::IR::MaterializeResources(resource_plan, runtime, resources, + specialization)); + for (auto& candidate: modules) { + const auto& layout = candidate.info.bindings; + if (layout.push_data_start_dword != ShaderRecompiler::IR::PushData::StartFor( + push_data_cursor, layout.ShaderDataDwords()) || + candidate.specialization != specialization) { + continue; } - return nullptr; + if constexpr (HasCalls) { + if (!std::ranges::equal(candidate.function_code, FunctionCode())) continue; + } + return &candidate; } + return nullptr; +} - std::unique_ptr call_source; - ShaderRecompiler::IR::ResourcePlan resource_plan; - ShaderRecompiler::IR::ResourceSnapshot resources; - ShaderRecompiler::IR::ResourceSpecialization specialization; - std::vector modules; - bool skip_dispatch = false; -}; - -PipelineCache::Module& PipelineCache::CompileModule(Program& program, const ProgramKey& key, - ShaderRecompiler::CompileOptions options, - ShaderRecompiler::TranslateResult translated, - uint32_t push_data_cursor) { +ShaderModule& PipelineCache::CompileModule(Program& program, const ProgramKey& key, + ShaderRecompiler::CompileOptions options, + ShaderRecompiler::TranslateResult translated, + uint32_t push_data_cursor) { Common::DebugCounters::ScopedTimer timer(Common::DebugCounters::Counter::ShadersCompiled, Common::DebugCounters::Counter::ShaderCompileNs, Common::DebugCounters::Peak::ShaderCompileNs); @@ -302,19 +339,27 @@ PipelineCache::Module& PipelineCache::CompileModule(Program& program, const Prog const auto* stage = ShaderRecompiler::StageName(key.stage); DumpShader(result.spirv, fmt::format("{}_{:016x}_{:016x}.spv", stage, key.hash, id)); ValidateShaderSpirv(stage, key.hash, result.spirv, profile.maintenance8); - const auto module = CompileSPV(result.spirv, m_graphics.device); - EXIT_IF(module == nullptr); - return program.modules.emplace_back(Module { - .function_code = {function_code.begin(), function_code.end()}, - .specialization = program.specialization, + auto& module = program.modules.emplace_back(ShaderModule { + .id = id, + .module = CompileSPV(result.spirv, m_graphics.device), .info = std::move(info), - .handle = {.id = id, .module = module}, + .specialization = program.specialization, + .function_code = {function_code.begin(), function_code.end()}, }); + EXIT_IF(module.module == nullptr); + // The inputs of these come from the stage's static key. + if (key.stage == ShaderType::Vertex || key.stage == ShaderType::Local) { + module.vertex_formats = VertexInputFormats(*options.input_info.vertex, module.info); + } else if (key.stage == ShaderType::Pixel && + options.input_info.pixel->parameter_mode == ShaderPixelParameterMode::Rectangle) { + module.rect_parameters = RectListParameters(*options.input_info.pixel, module.info); + } + return module; } template -ShaderProgram PipelineCache::GetProgram(const ShaderParams& params, InputInfo& input_info, - uint32_t& push_data_cursor) { +const ShaderModule* PipelineCache::GetProgram(const ShaderParams& params, InputInfo& input_info, + uint32_t& push_data_cursor) { const auto user_data = std::span(params.user_data).first(params.user_data_count); ShaderRecompiler::IR::SrtRuntime runtime { .user_data = user_data, @@ -348,18 +393,16 @@ ShaderProgram PipelineCache::GetProgram(const ShaderParams& params, InputInfo& i m_lookup_key.user_data_count = params.user_data_count; BuildStageStaticKey(input_info, m_lookup_key.static_state); auto entry = m_program_cache.find(m_lookup_key); - const auto bind = [&](const Module& module) { - input_info.stage = {.program = &module.info, - .resources = &entry->second->resources, - .handle = &module.handle}; + const auto bind = [&](const ShaderModule& module) { + input_info.stage = {.module = &module, .resources = &entry->second.resources}; module.info.bindings.AdvancePushData(push_data_cursor); - return module.handle; + return &module; }; bool source_changed = false; if (entry != m_program_cache.end()) { - auto& cached = *entry->second; - if (cached.skip_dispatch) return {}; - Module* module = nullptr; + auto& cached = entry->second; + if (cached.skip_dispatch) return nullptr; + ShaderModule* module = nullptr; if (cached.call_source) { source_changed = cached.RefreshCallSource(runtime); if (!source_changed) module = cached.FindModule(runtime, push_data_cursor); @@ -383,12 +426,11 @@ ShaderProgram PipelineCache::GetProgram(const ShaderParams& params, InputInfo& i PipelineCacheLog("!!! TEMPORARY COMPUTE DISPATCH SKIP !!! shader=0x{:016x} pc=0x{:08x} " "reason=unsupported GPU-selected callable target; rendering is incomplete", params.hash, source.unsupported_call_pc); - m_program_cache.emplace(m_lookup_key, std::make_unique()) - .first->second->skip_dispatch = true; - return {}; + m_program_cache.emplace(m_lookup_key, Program {.skip_dispatch = true}); + return nullptr; } } - auto* current = entry == m_program_cache.end() ? &source : entry->second->call_source.get(); + auto* current = entry == m_program_cache.end() ? &source : entry->second.call_source.get(); ShaderRecompiler::TranslateResult translated; if (current != nullptr) { if (!current->calls.empty()) { @@ -405,18 +447,20 @@ ShaderProgram PipelineCache::GetProgram(const ShaderParams& params, InputInfo& i auto retained = !source.calls.empty() ? std::make_unique(std::move(source)) : nullptr; - entry = m_program_cache.emplace(m_lookup_key, std::make_unique( - std::move(retained), ShaderRecompiler::IR::ExtractResourcePlan(translated.program))).first; + entry = m_program_cache.emplace(m_lookup_key, Program { + .call_source = std::move(retained), + .resource_plan = ShaderRecompiler::IR::ExtractResourcePlan(translated.program), + }).first; + auto& program = entry->second; EXIT_IF(!ShaderRecompiler::IR::MaterializeResources( - entry->second->resource_plan, runtime, entry->second->resources, - entry->second->specialization)); + program.resource_plan, runtime, program.resources, program.specialization)); } else if (source_changed) { - entry->second->resource_plan = ShaderRecompiler::IR::ExtractResourcePlan(translated.program); - if (const auto* module = entry->second->FindModule(runtime, push_data_cursor)) { + entry->second.resource_plan = ShaderRecompiler::IR::ExtractResourcePlan(translated.program); + if (const auto* module = entry->second.FindModule(runtime, push_data_cursor)) { return bind(*module); } } - return bind(CompileModule(*entry->second, entry->first, options, std::move(translated), + return bind(CompileModule(entry->second, entry->first, options, std::move(translated), push_data_cursor)); } @@ -440,8 +484,8 @@ PipelineCache::~PipelineCache() { destroy(m_graphics_pipelines); destroy(m_compute_pipelines); for (const auto& program: m_program_cache | std::views::values) { - for (const auto& module: program->modules) { - m_graphics.device.destroyShaderModule(module.handle.module, nullptr); + for (const auto& module: program.modules) { + m_graphics.device.destroyShaderModule(module.module, nullptr); } } if (m_driver_cache != nullptr) { @@ -766,8 +810,8 @@ void PipelineCache::GetGraphicsPrograms(const CommandBuffer& command, DrawRender uint32_t push_data_cursor = ShaderRecompiler::IR::PushData::GraphicsDrawDwords( mesh_active, object_primitive_id, last_stage.user_clip_planes); if (pixel_active) { - GetProgram(pixel_params, pixel_info, push_data_cursor); - for (const auto& output: pixel_info.stage.program->info.outputs) { + const auto& pixel_module = *GetProgram(pixel_params, pixel_info, push_data_cursor); + for (const auto& output: pixel_module.info.info.outputs) { if (output.kind == ShaderRecompiler::IR::StageOutputKind::Mrt || output.kind == ShaderRecompiler::IR::StageOutputKind::ObjectPrimitiveId) { state.mrt_mask |= 1u << output.location; @@ -780,8 +824,8 @@ void PipelineCache::GetGraphicsPrograms(const CommandBuffer& command, DrawRender } } -ShaderProgram PipelineCache::GetComputeProgram(const HW::ComputeShaderInfo& regs, - ShaderComputeInputInfo& input_info) { +const ShaderModule* PipelineCache::GetComputeProgram(const HW::ComputeShaderInfo& regs, + ShaderComputeInputInfo& input_info) { input_info.host_subgroup_size = m_graphics.SupportsComputeWave64() ? 64u : 32u; const auto params = PrepareProgram(regs, input_info); input_info.lds_storage = input_info.lds_size_dwords * 4u > @@ -790,38 +834,38 @@ ShaderProgram PipelineCache::GetComputeProgram(const HW::ComputeShaderInfo& regs return GetProgram(params, input_info, push_data_cursor); } -bool PipelineStaticParameters::operator==(const PipelineStaticParameters& other) const noexcept { - return std::memcmp(this, &other, sizeof(*this)) == 0; -} - PipelineCache::Pipeline& PipelineCache::GetGraphicsPipeline(const CommandBuffer& command, const DrawRenderState& state) { - const auto* colors = state.color_info; - const auto& depth = state.depth_info; - const auto vertex_info = state.VertexStages(); - const auto& ps_input_info = state.ps_input_info; - const auto& vs_input_info = vertex_info.front(); - const auto* vertex_program = vs_input_info.stage.handle; - const auto* pixel_program = state.ps_input_info.stage.handle; + const auto* colors = state.color_info; + const auto& depth = state.depth_info; + const auto vertex_info = state.VertexStages(); + const auto& ps_input_info = state.ps_input_info; + const auto& vs_input_info = vertex_info.front(); KYTY_PROFILER_BLOCK("PipelineCache::CreatePipeline(Gfx)", profiler::colors::DeepOrangeA200); - EXIT_IF(vertex_program == nullptr); - const bool ps_active = state.ps_active; - EXIT_IF(ps_active && pixel_program == nullptr); + const bool ps_active = state.ps_active; + const bool tessellation = vertex_info.size() == 3; + std::array stages {}; + for (uint32_t i = 0; i < vertex_info.size(); i++) { + stages[i] = vertex_info[i].stage.module; + } + if (ps_active) { + stages[3] = ps_input_info.stage.module; + } + EXIT_IF(stages[0] == nullptr || (ps_active && stages[3] == nullptr)); + const bool mesh = stages[0]->info.stage == ShaderType::Mesh; const auto color_count = state.color_count; auto& ctx = command.GetRegisters(); const HW::ModeControl& mc = ctx.GetModeControl(); - const auto vs_id = vertex_program->id; - const auto ps_id = ps_active ? pixel_program->id : 0; - GraphicsPipelineKey key {}; - for (uint32_t i = 0; i < vertex_info.size(); i++) { - key.vertex_shader_ids[i] = vertex_info[i].stage.handle->id; + for (uint32_t i = 0; i < stages.size(); i++) { + if (stages[i] != nullptr) { + key.stage_ids[i] = stages[i]->id; + } } - key.ps_shader_id = ps_id; auto& static_params = key.static_params; auto& rendering = key.rendering; rendering.color_count = 0; @@ -943,8 +987,15 @@ PipelineCache::Pipeline& PipelineCache::GetGraphicsPipeline(const CommandBuffer& static_params.provoking_vtx_last = mc.provoking_vtx_last; static_params.polygon_mode = ResolvePolygonMode(mc, static_params.cull_front, static_params.cull_back); + if (tessellation) { + static_params.patch_control_points = + static_cast(vs_input_info.tess.input_control_points); + } else if (!mesh && static_params.topology == vk::PrimitiveTopology::ePatchList) { + // Rectangle lists expand through generated tessellation stages. + static_params.patch_control_points = 3; + } - if (vs_input_info.stage.program->stage != ShaderType::Mesh) { + if (!mesh) { EXIT_IF(vs_input_info.buffers_num < 0 || vs_input_info.buffers_num > ShaderVertexInputInfo::RES_MAX || vs_input_info.resources_num < 0 || @@ -979,22 +1030,16 @@ PipelineCache::Pipeline& PipelineCache::GetGraphicsPipeline(const CommandBuffer& ShaderDbgDumpInputInfo(ps_input_info); } LOGF("PipelineTrace: shader modules VS=%" PRIu64 " module=%p PS=%" PRIu64 " module=%p\n", - vs_id, static_cast(vertex_program->module), ps_id, - ps_active ? static_cast(pixel_program->module) : nullptr); + key.stage_ids[0], static_cast(stages[0]->module), key.stage_ids[3], + ps_active ? static_cast(stages[3]->module) : nullptr); } - LogPipelineTrace("CreatePipelineInternal begin", vs_id, ps_id); { Common::DebugCounters::ScopedTimer timer(Common::DebugCounters::Counter::PipelinesCreated, Common::DebugCounters::Counter::PipelineCreateNs, Common::DebugCounters::Peak::PipelineCreateNs); - CreatePipelineInternal(m_graphics, pipeline, rendering, key.vertex_input, state, - static_params, m_driver_cache); + CreatePipelineInternal(m_graphics, pipeline, key, stages, m_driver_cache); } - LogPipelineTrace("CreatePipelineInternal done", vs_id, ps_id); - - EXIT_NOT_IMPLEMENTED(pipeline.pipeline == nullptr); - EXIT_NOT_IMPLEMENTED(pipeline.pipeline_layout == nullptr); CheckpointDriverCache(); return pipeline; @@ -1003,10 +1048,10 @@ PipelineCache::Pipeline& PipelineCache::GetGraphicsPipeline(const CommandBuffer& PipelineCache::Pipeline& PipelineCache::GetComputePipeline(const ShaderComputeInputInfo& input_info) { KYTY_PROFILER_BLOCK("PipelineCache::CreatePipeline(Compute)", profiler::colors::RedA100); - EXIT_IF(input_info.stage.handle == nullptr); - const auto& compute_program = *input_info.stage.handle; + EXIT_IF(input_info.stage.module == nullptr); + const auto& module = *input_info.stage.module; - auto [iter, inserted] = m_compute_pipelines.try_emplace(compute_program.id); + auto [iter, inserted] = m_compute_pipelines.try_emplace(module.id); auto& pipeline = iter->second; if (!inserted) { return pipeline; @@ -1020,12 +1065,8 @@ PipelineCache::Pipeline& PipelineCache::GetComputePipeline(const ShaderComputeIn Common::DebugCounters::ScopedTimer timer(Common::DebugCounters::Counter::PipelinesCreated, Common::DebugCounters::Counter::PipelineCreateNs, Common::DebugCounters::Peak::PipelineCreateNs); - CreatePipelineInternal(m_graphics, pipeline, input_info, compute_program.module, - m_driver_cache); + CreatePipelineInternal(m_graphics, pipeline, module, m_driver_cache); } - - EXIT_NOT_IMPLEMENTED(pipeline.pipeline == nullptr); - EXIT_NOT_IMPLEMENTED(pipeline.pipeline_layout == nullptr); CheckpointDriverCache(); return pipeline; diff --git a/src/graphics/host_gpu/renderer/pipeline/pipelineCache.h b/src/graphics/host_gpu/renderer/pipeline/pipelineCache.h index 0b92566a3..4615390e5 100644 --- a/src/graphics/host_gpu/renderer/pipeline/pipelineCache.h +++ b/src/graphics/host_gpu/renderer/pipeline/pipelineCache.h @@ -8,6 +8,8 @@ #include "graphics/host_gpu/renderer/depthRenderTarget.h" #include "graphics/host_gpu/renderer/renderTarget.h" #include "graphics/host_gpu/vulkanCommon.h" +#include "graphics/shader/recompiler/ir/passes/ResourceMaterialization.h" +#include "graphics/shader/rectListShader.h" #include "graphics/shader/shader.h" #include @@ -27,6 +29,7 @@ class CommandBuffer; namespace ShaderRecompiler { struct CompileOptions; +struct ShaderSource; struct TranslateResult; } // namespace ShaderRecompiler @@ -40,6 +43,8 @@ struct PipelineStaticParameters { bool negative_one_to_one = false; bool depth_clip_enable = true; vk::PrimitiveTopology topology = vk::PrimitiveTopology::ePointList; + // 0 without tessellation, 3 for a rectangle list, else the input control points. + uint8_t patch_control_points = 0; bool primitive_restart_enable = false; uint32_t samples = 1; bool sample_shading_enable = false; @@ -58,51 +63,59 @@ struct PipelineStaticParameters { uint8_t alpha_destblend[RENDER_COLOR_ATTACHMENTS_MAX] = {}; bool separate_alpha_blend[RENDER_COLOR_ATTACHMENTS_MAX] = {}; bool blend_enable[RENDER_COLOR_ATTACHMENTS_MAX] = {}; - - bool operator==(const PipelineStaticParameters& other) const noexcept; }; -#pragma pack(pop) - -static_assert(std::is_trivially_copyable_v); -static_assert(std::is_standard_layout_v); -static_assert(alignof(PipelineStaticParameters) == 1); -static_assert(sizeof(PipelineStaticParameters) == 117); - struct PipelineRenderingState { std::array color_formats {}; vk::Format depth_format = vk::Format::eUndefined; vk::Format stencil_format = vk::Format::eUndefined; uint32_t color_count = 0; - - bool operator==(const PipelineRenderingState&) const = default; }; struct PipelineVertexInputState { struct Binding { - uint32_t stride = 0; - bool instance = false; - bool operator==(const Binding&) const = default; + uint32_t stride = 0; + uint8_t instance = 0; }; struct Attribute { - uint32_t offset = 0; - uint8_t binding = 0; - bool operator==(const Attribute&) const = default; + uint32_t offset = 0; + uint8_t binding = 0; }; std::array bindings {}; std::array attributes {}; uint8_t binding_count = 0; uint8_t attribute_count = 0; - - bool operator==(const PipelineVertexInputState&) const = default; }; -struct ShaderProgram { - uint64_t id = 0; - vk::ShaderModule module = nullptr; +// Everything a graphics pipeline is made of besides the stage modules its ids name. Aligned so +// that rendering.color_formats can be passed to Vulkan. +struct alignas(4) GraphicsPipelineKey { + // VS, LS or mesh; HS; TES; PS. 0 marks an absent stage. + std::array stage_ids {}; + PipelineRenderingState rendering; + PipelineVertexInputState vertex_input; + PipelineStaticParameters static_params; - explicit operator bool() const { return module != nullptr; } + bool operator==(const GraphicsPipelineKey& other) const; +}; + +#pragma pack(pop) + +// Keys are compared and hashed as bytes. +static_assert(std::has_unique_object_representations_v); + +// A compiled stage with what pipeline creation needs from it besides the key. +struct ShaderModule { + uint64_t id = 0; + vk::ShaderModule module = nullptr; + ShaderRecompiler::IR::CompiledShaderInfo info; + ShaderRecompiler::IR::ResourceSpecialization specialization; + std::vector function_code; + // VS or LS: the host format of each vertex input attribute. + std::vector vertex_formats; + // Rectangle-list PS: the parameters it reads. + std::vector rect_parameters; }; // A draw's topology, shader stages (from PipelineCache::GetGraphicsPrograms) and targets. @@ -141,16 +154,30 @@ public: // Reads the shaders and registers bound in `command` into the state's shader stages. void GetGraphicsPrograms(const CommandBuffer& command, DrawRenderState& state); - // An empty program skips the dispatch. - ShaderProgram GetComputeProgram(const HW::ComputeShaderInfo& regs, - ShaderComputeInputInfo& input_info); + // Null skips the dispatch. + const ShaderModule* GetComputeProgram(const HW::ComputeShaderInfo& regs, + ShaderComputeInputInfo& input_info); Pipeline& GetGraphicsPipeline(const CommandBuffer& command, const DrawRenderState& state); Pipeline& GetComputePipeline(const ShaderComputeInputInfo& input_info); private: - struct Module; - struct Program; + friend struct PipelineCacheTestAccess; + + struct Program { + std::unique_ptr call_source; + ShaderRecompiler::IR::ResourcePlan resource_plan; + ShaderRecompiler::IR::ResourceSnapshot resources; + ShaderRecompiler::IR::ResourceSpecialization specialization; + std::vector modules; + bool skip_dispatch = false; + + [[nodiscard]] std::span FunctionCode() const; + bool RefreshCallSource(const ShaderRecompiler::IR::SrtRuntime& runtime); + template + ShaderModule* FindModule(const ShaderRecompiler::IR::SrtRuntime& runtime, + uint32_t push_data_cursor); + }; struct ProgramKey { ShaderType stage = ShaderType::Unknown; @@ -166,34 +193,13 @@ private: std::size_t operator()(const ProgramKey& key) const { return key.hash; } }; - struct GraphicsPipelineKey { - PipelineRenderingState rendering; - std::array vertex_shader_ids {}; - uint64_t ps_shader_id = 0; - PipelineVertexInputState vertex_input; - PipelineStaticParameters static_params; - - bool operator==(const GraphicsPipelineKey& other) const { - return rendering == other.rendering && vertex_shader_ids == other.vertex_shader_ids && - ps_shader_id == other.ps_shader_id && vertex_input == other.vertex_input && - static_params == other.static_params; - } - }; - - struct PipelineKeyHash { - static void Mix(std::size_t& hash, std::size_t value) { - hash ^= value + static_cast(0x9e3779b97f4a7c15ull) + (hash << 6u) + - (hash >> 2u); - } - }; - struct GraphicsPipelineKeyHash { std::size_t operator()(const GraphicsPipelineKey& key) const; }; GraphicContext& m_graphics; ShaderRecompiler::IR::ImageHeapSource* m_image_heap; - std::unordered_map, ProgramKeyHash> m_program_cache; + std::unordered_map m_program_cache; ProgramKey m_lookup_key; vk::PipelineCache m_driver_cache = nullptr; std::filesystem::path m_driver_cache_path; @@ -203,25 +209,23 @@ private: std::unordered_map m_compute_pipelines; template - ShaderProgram GetProgram(const ShaderParams& params, InputInfo& input_info, - uint32_t& push_data_cursor); - Module& CompileModule(Program& program, const ProgramKey& key, - ShaderRecompiler::CompileOptions options, - ShaderRecompiler::TranslateResult translated, uint32_t push_data_cursor); + const ShaderModule* GetProgram(const ShaderParams& params, InputInfo& input_info, + uint32_t& push_data_cursor); + ShaderModule& CompileModule(Program& program, const ProgramKey& key, + ShaderRecompiler::CompileOptions options, + ShaderRecompiler::TranslateResult translated, + uint32_t push_data_cursor); void InitializeDriverCache(); void CheckpointDriverCache(); }; -void LogPipelineTrace(const char* phase, uint64_t vertex_program_id, uint64_t pixel_program_id); +// Stages are indexed like GraphicsPipelineKey::stage_ids. void CreatePipelineInternal(GraphicContext& graphics, PipelineCache::Pipeline& pipeline, - const PipelineRenderingState& rendering, - const PipelineVertexInputState& vertex_input, - const DrawRenderState& state, - const PipelineStaticParameters& static_params, - vk::PipelineCache driver_cache); + const GraphicsPipelineKey& key, + const std::array& stages, + vk::PipelineCache driver_cache); void CreatePipelineInternal(GraphicContext& graphics, PipelineCache::Pipeline& pipeline, - const ShaderComputeInputInfo& input_info, - vk::ShaderModule compute_module, vk::PipelineCache driver_cache); + const ShaderModule& module, vk::PipelineCache driver_cache); } // namespace Libs::Graphics diff --git a/src/graphics/host_gpu/renderer/pipeline/shaderResourceBarrier.cpp b/src/graphics/host_gpu/renderer/pipeline/shaderResourceBarrier.cpp index 18d030b0f..46a492b7d 100644 --- a/src/graphics/host_gpu/renderer/pipeline/shaderResourceBarrier.cpp +++ b/src/graphics/host_gpu/renderer/pipeline/shaderResourceBarrier.cpp @@ -1,6 +1,7 @@ #include "graphics/host_gpu/renderer/pipeline/shaderResourceBarrier.h" #include "common/assert.h" +#include "graphics/host_gpu/renderer/pipeline/pipelineCache.h" #include "graphics/shader/shader.h" #include "graphics/shader/shaderBindings.h" @@ -90,7 +91,7 @@ vk::BufferMemoryBarrier MakeGdsDependency(vk::Buffer buffer) { bool HasShaderBufferWrites(const ShaderStageRuntime& runtime) { EXIT_IF(!runtime); - const auto& program = *runtime.program; + const auto& program = runtime.module->info; const auto& resources = *runtime.resources; EXIT_IF(resources.buffers.size() != program.info.buffers.size()); // Address writes reach memory outside the bound buffer descriptors. diff --git a/src/graphics/host_gpu/renderer/pipeline/shaders.cpp b/src/graphics/host_gpu/renderer/pipeline/shaders.cpp index 083770921..61fe8d959 100644 --- a/src/graphics/host_gpu/renderer/pipeline/shaders.cpp +++ b/src/graphics/host_gpu/renderer/pipeline/shaders.cpp @@ -13,7 +13,6 @@ #include "graphics/host_gpu/renderer/renderContext.h" #include "graphics/host_gpu/renderer/renderTarget.h" #include "graphics/host_gpu/vulkanCommon.h" -#include "graphics/shader/recompiler/BufferFormat.h" #include "graphics/shader/recompiler/ir/ShaderIR.h" #include "graphics/shader/rectListShader.h" #include "graphics/shader/shader.h" @@ -21,124 +20,11 @@ #include #include #include -#include #include #include namespace Libs::Graphics { -// IDK: maybe we can remove it? -constexpr uint8_t kTemporaryVertexAttribFormat113 = - static_cast(Prospero::VertexAttribFormat::k16_16SInt); -constexpr uint32_t kTemporaryPs5BufferFormat121 = 121u; - -static bool NarrowInputFormat(vk::Format& format, uint32_t& size, uint32_t used_components) { - if (used_components == 0 || used_components >= size) { - return false; - } - - switch (format) { - case vk::Format::eR32G32B32A32Sfloat: - switch (used_components) { - case 1: format = vk::Format::eR32Sfloat; break; - case 2: format = vk::Format::eR32G32Sfloat; break; - case 3: format = vk::Format::eR32G32B32Sfloat; break; - default: return false; - } - size = used_components; - return true; - case vk::Format::eR32G32B32Sfloat: - switch (used_components) { - case 1: format = vk::Format::eR32Sfloat; break; - case 2: format = vk::Format::eR32G32Sfloat; break; - default: return false; - } - size = used_components; - return true; - case vk::Format::eR16G16B16A16Sfloat: - switch (used_components) { - case 1: format = vk::Format::eR16Sfloat; break; - case 2: format = vk::Format::eR16G16Sfloat; break; - default: return false; - } - size = used_components; - return true; - case vk::Format::eR8G8B8A8Unorm: - switch (used_components) { - case 1: format = vk::Format::eR8Unorm; break; - case 2: format = vk::Format::eR8G8Unorm; break; - default: return false; - } - size = used_components; - return true; - case vk::Format::eR8G8B8A8Snorm: - if (used_components != 2) { - return false; - } - format = vk::Format::eR8G8Snorm; - size = 2; - return true; - case vk::Format::eR8G8B8A8Uint: - switch (used_components) { - case 1: format = vk::Format::eR8Uint; break; - case 2: format = vk::Format::eR8G8Uint; break; - default: return false; - } - size = used_components; - return true; - default: break; - } - - return false; -} - -static void GetInputFormat(const ShaderBufferResource& res, vk::Format& format, uint32_t& size, - uint32_t used_components) { - const auto fmt = res.Format(); - const auto raw_format = res.RawFormat(); - if (raw_format == kTemporaryVertexAttribFormat113) { - static bool logged_113 = false; - if (!logged_113) { - LOGF("InputFormat: temporary: accepting invalid PS5 buffer format 113 as " - "vk::Format::eR32G32B32A32Sfloat\n"); - logged_113 = true; - } - format = vk::Format::eR32G32B32A32Sfloat; - size = 4; - if (NarrowInputFormat(format, size, used_components)) { - LOGF("InputFormat: narrowing fmt=%u to %s for used_components=%u\n", raw_format, - vk::to_string(format).c_str(), used_components); - } - return; - } - if (raw_format == kTemporaryPs5BufferFormat121) { - static bool logged_121 = false; - if (!logged_121) { - LOGF("InputFormat: accepting PS5 buffer format 121 as vk::Format::eR16G16Sfloat\n"); - logged_121 = true; - } - format = vk::Format::eR16G16Sfloat; - size = 2; - return; - } - - format = VulkanFormat(fmt); - size = ShaderRecompiler::Format::GetFormatInfo(fmt).component_count; - if (format == vk::Format::eUndefined || size == 0) { - EXIT("unknown vertex format: fmt = %u\n", raw_format); - } - - if (NarrowInputFormat(format, size, used_components)) { - static std::atomic log_count = 0; - auto log_id = log_count.fetch_add(1); - if (log_id < 32) { - LOGF("VertexInput: narrowed vertex format to %" PRIu32 - " component(s) for shader fetch\n", - used_components); - } - } -} - static vk::BlendFactor GetBlendFactor(uint32_t factor) { switch (static_cast(factor)) { case Prospero::BlendFactor::kZero: return vk::BlendFactor::eZero; @@ -242,30 +128,27 @@ static void CreateDescriptorLayout(GraphicContext& graphics, PipelineCache::Pipe // NOLINTNEXTLINE(readability-function-cognitive-complexity) void CreatePipelineInternal(GraphicContext& graphics, PipelineCache::Pipeline& pipeline, - const PipelineRenderingState& rendering, - const PipelineVertexInputState& vertex_input, - const DrawRenderState& state, - const PipelineStaticParameters& static_params, - vk::PipelineCache driver_cache) { - const auto vertex_info = state.VertexStages(); - const auto& vs_input_info = vertex_info.front(); - const auto* ps_input_info = state.ps_active ? &state.ps_input_info : nullptr; - const auto& vertex_program = *vs_input_info.stage.handle; - const auto* pixel_program = state.ps_input_info.stage.handle; - const bool tessellation = vertex_info.size() == 3; - const bool ps_active = state.ps_active; - const bool with_depth = rendering.depth_format != vk::Format::eUndefined || - rendering.stencil_format != vk::Format::eUndefined; - const bool mesh = vs_input_info.stage.program->stage == ShaderType::Mesh; + const GraphicsPipelineKey& key, + const std::array& stages, + vk::PipelineCache driver_cache) { + const auto& rendering = key.rendering; + const auto& vertex_input = key.vertex_input; + const auto& static_params = key.static_params; + const auto& vertex = *stages[0]; + const auto* pixel = stages[3]; + const bool with_depth = rendering.depth_format != vk::Format::eUndefined || + rendering.stencil_format != vk::Format::eUndefined; + const bool mesh = vertex.info.stage == ShaderType::Mesh; EXIT_NOT_IMPLEMENTED(mesh && !graphics.mesh_shader_enabled); - const bool rect_list = - !mesh && !tessellation && static_params.topology == vk::PrimitiveTopology::ePatchList; + const bool rect_list = static_params.patch_control_points != 0 && stages[1] == nullptr; vk::ShaderModule tess_control_shader_module = nullptr; vk::ShaderModule tess_eval_shader_module = nullptr; if (rect_list) { - const auto shaders = BuildRectListShaders(vs_input_info, ps_input_info); + std::span parameters; + if (pixel != nullptr) parameters = pixel->rect_parameters; + const auto shaders = BuildRectListShaders(parameters, vertex.info.param_export_mask); tess_control_shader_module = CompileSPV(shaders.control, graphics.device); if (graphics_debug_dump_enabled()) { LOGF("PipelineTrace: vkCreateShaderModule RectList TCS done module=%p\n", @@ -284,11 +167,12 @@ void CreatePipelineInternal(GraphicContext& graphics, PipelineCache::Pipeline& p vk::PipelineShaderStageCreateInfo shader_stages[4] {}; uint32_t shader_stage_count = 0; - for (uint32_t i = 0; i < vertex_info.size(); i++) { - shader_stages[shader_stage_count++] = {.stage = - NativeShaderStage(vertex_info[i].logical_stage), - .module = vertex_info[i].stage.handle->module, - .pName = "main"}; + for (const auto* stage: stages) { + if (stage != nullptr) { + shader_stages[shader_stage_count++] = {.stage = NativeShaderStage(stage->info.stage), + .module = stage->module, + .pName = "main"}; + } } if (rect_list) { shader_stages[shader_stage_count++] = {.stage = @@ -300,11 +184,6 @@ void CreatePipelineInternal(GraphicContext& graphics, PipelineCache::Pipeline& p .module = tess_eval_shader_module, .pName = "main"}; } - if (ps_active) { - shader_stages[shader_stage_count++] = {.stage = vk::ShaderStageFlagBits::eFragment, - .module = pixel_program->module, - .pName = "main"}; - } vk::VertexInputAttributeDescription input_attr[ShaderVertexInputInfo::RES_MAX] {}; vk::VertexInputBindingDescription input_desc[ShaderVertexInputInfo::RES_MAX] {}; @@ -320,46 +199,7 @@ void CreatePipelineInternal(GraphicContext& graphics, PipelineCache::Pipeline& p input_attr[index].binding = vertex_input.attributes[index].binding; input_attr[index].location = index; input_attr[index].offset = vertex_input.attributes[index].offset; - - uint32_t attr_size = 4; - const auto registers_num = vs_input_info.resources_dst[index].registers_num; - const auto compiled_components = - vs_input_info.stage.program->info.vertex_fetch_components[index]; - const auto used_components = - compiled_components > 0 ? static_cast(compiled_components) : registers_num; - GetInputFormat(vs_input_info.resources[index], input_attr[index].format, attr_size, - static_cast(used_components)); - - if (graphics_debug_dump_enabled()) { - static std::atomic_uint log_count = 0; - const auto log_id = log_count.fetch_add(1, std::memory_order_relaxed); - if (log_id < 128) { - LOGF("VertexInputState[%u]: attr=%u binding=%u offset=%u stride=%u fmt=%d " - "src_fmt=%u dst=v%u regs=%u" - " fetched_components=%u attr_size=%u swizzle=%u,%u,%u,%u\n", - log_id, index, input_attr[index].binding, input_attr[index].offset, - input_desc[input_attr[index].binding].stride, - static_cast(input_attr[index].format), - static_cast(vs_input_info.resources[index].Format()), - static_cast(vs_input_info.resources_dst[index].register_start), - static_cast(registers_num), static_cast(used_components), - attr_size, static_cast(vs_input_info.resources[index].DstSelX()), - static_cast(vs_input_info.resources[index].DstSelY()), - static_cast(vs_input_info.resources[index].DstSelZ()), - static_cast(vs_input_info.resources[index].DstSelW())); - } - } - - if (vs_input_info.resources[index].OutOfBounds() != 0) { - static bool logged = false; - if (!logged) { - LOGF("VertexInput: temporary: accepting PS5 out-of-bounds behavior %" PRIu8 "\n", - vs_input_info.resources[index].OutOfBounds()); - logged = true; - } - } - - EXIT_IF(registers_num < 1 || registers_num > 4); + input_attr[index].format = vertex.vertex_formats[index]; } vk::PipelineVertexInputStateCreateInfo vertex_input_info {}; @@ -467,16 +307,13 @@ void CreatePipelineInternal(GraphicContext& graphics, PipelineCache::Pipeline& p std::vector descriptor_bindings; vk::ShaderStageFlags graphics_stages = vk::ShaderStageFlagBits::eFragment; - for (const auto& stage: vertex_info) { - const auto native_stage = NativeShaderStage(stage.logical_stage); - AddLayoutBindings(descriptor_bindings, *stage.stage.program, native_stage); - pipeline.uses_image_heap |= stage.stage.program->info.uses_image_heap; - graphics_stages |= native_stage; - } - if (ps_active) { - AddLayoutBindings(descriptor_bindings, *ps_input_info->stage.program, - vk::ShaderStageFlagBits::eFragment); - pipeline.uses_image_heap |= ps_input_info->stage.program->info.uses_image_heap; + for (const auto* stage: stages) { + if (stage != nullptr) { + const auto native_stage = NativeShaderStage(stage->info.stage); + AddLayoutBindings(descriptor_bindings, stage->info, native_stage); + pipeline.uses_image_heap |= stage->info.info.uses_image_heap; + graphics_stages |= native_stage; + } } CreateDescriptorLayout(graphics, pipeline, descriptor_bindings); const vk::PushConstantRange push_constants {graphics_stages, 0, @@ -494,7 +331,7 @@ void CreatePipelineInternal(GraphicContext& graphics, PipelineCache::Pipeline& p if (graphics_debug_dump_enabled()) { LOGF("PipelineTrace: vkCreatePipelineLayout begin VS=%" PRIu64 " PS=%" PRIu64 " set_layouts=1 push_constants=%" PRIu32 "\n", - vertex_program.id, ps_active ? pixel_program->id : 0, 1u); + key.stage_ids[0], key.stage_ids[3], 1u); } auto result = graphics.device.createPipelineLayout(&pipeline_layout_info, nullptr, &pipeline.pipeline_layout); @@ -504,8 +341,6 @@ void CreatePipelineInternal(GraphicContext& graphics, PipelineCache::Pipeline& p } EXIT_NOT_IMPLEMENTED(result != vk::Result::eSuccess); - EXIT_NOT_IMPLEMENTED(pipeline.pipeline_layout == nullptr); - vk::PipelineDepthStencilStateCreateInfo depth_stencil_info {}; std::vector dynamic_states { @@ -551,9 +386,9 @@ void CreatePipelineInternal(GraphicContext& graphics, PipelineCache::Pipeline& p pipeline_info.pVertexInputState = mesh ? nullptr : &vertex_input_info; pipeline_info.pInputAssemblyState = mesh ? nullptr : &input_assembly; vk::PipelineTessellationStateCreateInfo tessellation_state {}; - tessellation_state.patchControlPoints = - tessellation ? vs_input_info.tess.input_control_points : 3u; - pipeline_info.pTessellationState = (rect_list || tessellation) ? &tessellation_state : nullptr; + tessellation_state.patchControlPoints = static_params.patch_control_points; + pipeline_info.pTessellationState = + static_params.patch_control_points != 0 ? &tessellation_state : nullptr; pipeline_info.pViewportState = &viewport_state; pipeline_info.pRasterizationState = &rasterizer; pipeline_info.pMultisampleState = &multisampling; @@ -569,7 +404,7 @@ void CreatePipelineInternal(GraphicContext& graphics, PipelineCache::Pipeline& p LOGF("PipelineTrace: vkCreateGraphicsPipelines begin VS=%" PRIu64 " PS=%" PRIu64 " topology=%" PRIu32 " color_mask=0x%08" PRIx32 " depth=%s blend=%s dyn_states=%" PRIu32 "\n", - vertex_program.id, ps_active ? pixel_program->id : 0, + key.stage_ids[0], key.stage_ids[3], static_cast(static_params.topology), static_params.color_mask[0], (with_depth ? "true" : "false"), (static_params.blend_enable[0] ? "true" : "false"), dynamic_state.dynamicStateCount); @@ -582,29 +417,18 @@ void CreatePipelineInternal(GraphicContext& graphics, PipelineCache::Pipeline& p } EXIT_NOT_IMPLEMENTED(result != vk::Result::eSuccess); - EXIT_NOT_IMPLEMENTED(pipeline.pipeline == nullptr); - - if (tess_control_shader_module != nullptr) { - graphics.device.destroyShaderModule(tess_control_shader_module, nullptr); - } - if (tess_eval_shader_module != nullptr) { - graphics.device.destroyShaderModule(tess_eval_shader_module, nullptr); - } + graphics.device.destroyShaderModule(tess_control_shader_module, nullptr); + graphics.device.destroyShaderModule(tess_eval_shader_module, nullptr); } -// NOLINTNEXTLINE(readability-function-cognitive-complexity) void CreatePipelineInternal(GraphicContext& graphics, PipelineCache::Pipeline& pipeline, - const ShaderComputeInputInfo& input_info, - vk::ShaderModule compute_module, vk::PipelineCache driver_cache) { - EXIT_IF(compute_module == nullptr); - + const ShaderModule& module, vk::PipelineCache driver_cache) { vk::PipelineShaderStageCreateInfo comp_shader_stage_info {}; vk::PipelineShaderStageRequiredSubgroupSizeCreateInfo comp_subgroup_size {}; comp_shader_stage_info.stage = vk::ShaderStageFlagBits::eCompute; - comp_shader_stage_info.module = compute_module; + comp_shader_stage_info.module = module.module; comp_shader_stage_info.pName = "main"; - EXIT_IF(!input_info.stage); - const auto wave_size = input_info.stage.program->wave_size; + const auto wave_size = module.info.wave_size; if (graphics.compute_subgroup_size_control_enabled && wave_size >= graphics.min_subgroup_size && wave_size <= graphics.max_subgroup_size) { comp_subgroup_size.requiredSubgroupSize = wave_size; @@ -612,9 +436,8 @@ void CreatePipelineInternal(GraphicContext& graphics, PipelineCache::Pipeline& p } std::vector descriptor_bindings; - AddLayoutBindings(descriptor_bindings, *input_info.stage.program, - vk::ShaderStageFlagBits::eCompute); - pipeline.uses_image_heap = input_info.stage.program->info.uses_image_heap; + AddLayoutBindings(descriptor_bindings, module.info, vk::ShaderStageFlagBits::eCompute); + pipeline.uses_image_heap = module.info.info.uses_image_heap; CreateDescriptorLayout(graphics, pipeline, descriptor_bindings); const vk::PushConstantRange push_constants {vk::ShaderStageFlagBits::eCompute, 0, ShaderRecompiler::IR::NativePushConstantSize}; @@ -628,18 +451,12 @@ void CreatePipelineInternal(GraphicContext& graphics, PipelineCache::Pipeline& p EXIT_IF(pipeline.pipeline_layout != nullptr); - LOGF("PipelineTrace: vkCreatePipelineLayout CS begin set_layouts=1 push_constants=%u\n", - 1u); auto result = graphics.device.createPipelineLayout(&pipeline_layout_info, nullptr, &pipeline.pipeline_layout); - LOGF("PipelineTrace: vkCreatePipelineLayout CS done result=%s layout=%p\n", - vk::to_string(result).c_str(), static_cast(pipeline.pipeline_layout)); EXIT_NOT_IMPLEMENTED(result != vk::Result::eSuccess); - EXIT_NOT_IMPLEMENTED(pipeline.pipeline_layout == nullptr); - vk::ComputePipelineCreateInfo info {}; - if (input_info.stage.program->info.uses_gds) { + if (module.info.info.uses_gds) { info.flags = vk::PipelineCreateFlagBits::eDispatchBase; } info.stage = comp_shader_stage_info; @@ -648,15 +465,9 @@ void CreatePipelineInternal(GraphicContext& graphics, PipelineCache::Pipeline& p EXIT_IF(pipeline.pipeline != nullptr); - LOGF("PipelineTrace: vkCreateComputePipelines begin layout=%p\n", - static_cast(pipeline.pipeline_layout)); result = graphics.device.createComputePipelines(driver_cache, 1, &info, nullptr, &pipeline.pipeline); - LOGF("PipelineTrace: vkCreateComputePipelines done result=%s pipeline=%p\n", - vk::to_string(result).c_str(), static_cast(pipeline.pipeline)); EXIT_NOT_IMPLEMENTED(result != vk::Result::eSuccess); - - EXIT_NOT_IMPLEMENTED(pipeline.pipeline == nullptr); } } // namespace Libs::Graphics diff --git a/src/graphics/host_gpu/renderer/renderCompute.cpp b/src/graphics/host_gpu/renderer/renderCompute.cpp index c9a518bb1..d531fd7d9 100644 --- a/src/graphics/host_gpu/renderer/renderCompute.cpp +++ b/src/graphics/host_gpu/renderer/renderCompute.cpp @@ -53,7 +53,7 @@ static bool FillSourcesDisjoint(std::spaninfo; const auto& resources = *input.stage.resources; if (resources.buffers.size() != program.info.buffers.size()) { EXIT("compute runtime buffer count does not match shader metadata\n"); @@ -135,7 +135,7 @@ bool ResolveComputeBufferFill(const ShaderComputeInputInfo& input, uint32_t grou bool RenderExecutor::TryConsumeComputeFill(const ShaderComputeInputInfo& input, CommandBuffer& command, uint32_t group_x, uint32_t group_y, uint32_t group_z, uint32_t mode) { - const auto& program = *input.stage.program; + const auto& program = input.stage.module->info; const auto& resources = *input.stage.resources; const auto& fill = resources.uniform_fill; auto& cache = command.GetContext().GetTextureCache(); @@ -208,7 +208,7 @@ bool RenderExecutor::TryConsumeComputeFill(const ShaderComputeInputInfo& input, if (logged_clears.fetch_add(1, std::memory_order_relaxed) < 32) { LOGF("GraphicsRenderDispatchDirect: compute %s shader=0x%016" PRIx64 " addr=0x%016" PRIx64 " size=0x%016" PRIx64 " value=0x%08" PRIx32 "\n", - image_clear ? "image clear" : "buffer fill", input.stage.program->shader_hash, + image_clear ? "image clear" : "buffer fill", input.stage.module->info.shader_hash, descriptor.Base48(), size, packed_clear); } return true; @@ -216,7 +216,7 @@ bool RenderExecutor::TryConsumeComputeFill(const ShaderComputeInputInfo& input, static void BindSharedMemory(RenderContext& context, ShaderComputeInputInfo& input, PreparedBindings& bindings, uint64_t indirect_args = 0) { - if (input.stage.program->bindings.descriptor_counts[ + if (input.stage.module->info.bindings.descriptor_counts[ static_cast(ShaderRecompiler::IR::DescriptorBindingKind::SharedMemory)] == 0) { return; } @@ -317,7 +317,7 @@ void RenderExecutor::DispatchDirect(uint64_t submit_id, CommandBuffer& buffer, input_info.dispatch_threads_num[2] = thread_group_z; } - const auto& program = *input_info.stage.program; + const auto& program = input_info.stage.module->info; const auto& resources = *input_info.stage.resources; if (resources.specialization_reads.empty() && (TryConsumeComputeMetaClear(input_info, buffer) || @@ -504,7 +504,7 @@ void RenderExecutor::DispatchIndirect(uint64_t submit_id, CommandBuffer& buffer, PrepareBindings(input_info.stage, bindings); PreparedBindings* descriptor_stage = &bindings; FindBuffers(std::span {&descriptor_stage, 1u}); - const auto& program = *input_info.stage.program; + const auto& program = input_info.stage.module->info; if (program.info.uses_dma) { m_context.PrepareBda(); } diff --git a/src/graphics/host_gpu/renderer/renderDraw.cpp b/src/graphics/host_gpu/renderer/renderDraw.cpp index f43a63bd4..86f37055e 100644 --- a/src/graphics/host_gpu/renderer/renderDraw.cpp +++ b/src/graphics/host_gpu/renderer/renderDraw.cpp @@ -57,7 +57,7 @@ std::pair ResolveDrawOffsets(uint32_t index_offset, } EXIT_IF(!vs_input_info.stage); - const auto& program = *vs_input_info.stage.program; + const auto& program = vs_input_info.stage.module->info; const auto& resources = *vs_input_info.stage.resources; if (index_offset == 0 && program.info.vertex_offset_sgpr >= static_cast(program.user_data_base)) { @@ -197,7 +197,7 @@ static void LogDrawTargetState(const char* draw_name, const RenderColorInfo& col const auto& dc = ctx.GetDepthControl(); const auto& vp = ctx.GetScreenViewport(); const auto& vp0 = vp.viewports[0]; - const auto& ps_resources = ps_input_info.stage.program->info; + const auto& ps_resources = ps_input_info.stage.module->info.info; const auto sampled_images = std::count_if( ps_resources.images.begin(), ps_resources.images.end(), [](const auto& image) { return image.resource_class == ShaderRecompiler::IR::ImageResourceClass::Sampled; @@ -320,7 +320,7 @@ static void UpdateGraphicsDynamicState(const CommandBuffer& buffer, DynamicState const auto& ctx = buffer.GetRegisters(); const auto& vp = ctx.GetScreenViewport(); const vk::Extent2D framebuffer_extent {rendering.width, rendering.height}; - const auto& outputs = vs_input_info.stage.program->info.outputs; + const auto& outputs = vs_input_info.stage.module->info.info.outputs; const bool indexed_viewports = std::any_of(outputs.begin(), outputs.end(), [](const auto& output) { return output.kind == ShaderRecompiler::IR::StageOutputKind::ViewportIndex; @@ -1054,7 +1054,7 @@ void RenderExecutor::ExecutePreparedDraw(uint64_t submit_id, CommandBuffer& buff return; } const auto vertex_stages = state.VertexStages(); - const bool mesh_active = state.vertex_info[0].stage.program->stage == ShaderType::Mesh; + const bool mesh_active = state.vertex_info[0].stage.module->info.stage == ShaderType::Mesh; uint32_t mesh_groups = 0; if (mesh_active) { const auto& mesh = state.vertex_info[0].mesh; @@ -1241,7 +1241,7 @@ static std::optional ResolveDrawEmit(const HW::UserConfig& uint32_t first_instance, const DrawIndirectArgs& indirect) { if (indirect.address != 0) { - if (vs.stage.program->stage == ShaderType::Mesh || + if (vs.stage.module->info.stage == ShaderType::Mesh || ucfg.GetPrimType() == Prospero::PrimitiveType::kQuadListLegacy) { return std::nullopt; } @@ -1423,7 +1423,7 @@ bool RenderExecutor::DrawAuto(uint64_t submit_id, CommandBuffer& buffer, const D const bool rect_list = Prospero::IsRectList(ucfg.GetPrimType()); if (rect_list && state.vertex_info[0].buffers_num == 0 && - state.vertex_info[0].stage.program->param_export_mask == 0 && + state.vertex_info[0].stage.module->info.param_export_mask == 0 && state.ps_input_info.input_num != 0) { if (graphics_debug_dump_enabled()) { LOGF("DrawIndexAuto: skipping rect-list draw with no VS param exports and PS inputs: " diff --git a/src/graphics/shader/rectListShader.cpp b/src/graphics/shader/rectListShader.cpp index cf7f54959..32956742d 100644 --- a/src/graphics/shader/rectListShader.cpp +++ b/src/graphics/shader/rectListShader.cpp @@ -18,45 +18,9 @@ using ShaderRecompiler::Spirv::Builder; constexpr uint32_t SpirvVersion15 = 0x00010500u; -struct Parameter { - uint32_t input_location = 0; - uint32_t output_location = 0; - bool flat = false; -}; - -std::vector GetParameters(const ShaderVertexInputInfo& vertex_info, - const ShaderPixelInputInfo* pixel_info) { - if (pixel_info == nullptr) { - return {}; - } - EXIT_IF(pixel_info->input_num > ShaderVertexInputInfo::RES_MAX); - EXIT_IF(pixel_info->stage.program == nullptr); - - std::vector active_inputs; - for (const auto& input: pixel_info->stage.program->info.inputs) { - if (input.kind == ShaderRecompiler::IR::StageInputKind::Parameter) { - active_inputs.push_back(input.location); - } - } - - std::vector parameters; - std::array output_locations {}; - for (const auto input: active_inputs) { - const auto input_location = ShaderPixelParameterMappedLocation(*pixel_info, input); - const auto output_location = ShaderPixelParameterLocation(*pixel_info, active_inputs, input); - if ((vertex_info.stage.program->param_export_mask & (1u << input_location)) != 0 && - !output_locations[output_location]) { - parameters.push_back({input_location, output_location, - ShaderPixelParameterIsFlat(*pixel_info, input)}); - output_locations[output_location] = true; - } - } - return parameters; -} - class RectListEmitter { public: - RectListEmitter(const std::vector& parameters_, spv::ExecutionModel model) + RectListEmitter(const std::vector& parameters_, spv::ExecutionModel model) : parameters(parameters_) { builder.AddMemoryModel(spv::AddressingModelLogical, spv::MemoryModelGLSL450); @@ -315,42 +279,68 @@ private: Result(spv::OpFAdd, vec4_float_type, p1, p2)); } - Builder builder {SpirvVersion15}; - const std::vector& parameters; - std::vector interfaces; - std::vector inputs; - std::vector outputs; - uint32_t main = 0; - uint32_t void_type = 0; - uint32_t bool_type = 0; - uint32_t uint_type = 0; - uint32_t int_type = 0; - uint32_t float_type = 0; - uint32_t vec2_bool_type = 0; - uint32_t vec2_float_type = 0; - uint32_t vec3_float_type = 0; - uint32_t vec4_float_type = 0; - uint32_t function_type = 0; - uint32_t per_vertex_type = 0; - uint32_t ptr_input_float = 0; - uint32_t ptr_input_vec4_float = 0; - uint32_t ptr_output_float = 0; - uint32_t ptr_output_vec4_float = 0; - uint32_t gl_in = 0; - uint32_t gl_out = 0; - uint32_t tess_inner = 0; - uint32_t tess_outer = 0; - uint32_t tess_coord = 0; - uint32_t invocation_id = 0; + Builder builder {SpirvVersion15}; + const std::vector& parameters; + std::vector interfaces; + std::vector inputs; + std::vector outputs; + uint32_t main = 0; + uint32_t void_type = 0; + uint32_t bool_type = 0; + uint32_t uint_type = 0; + uint32_t int_type = 0; + uint32_t float_type = 0; + uint32_t vec2_bool_type = 0; + uint32_t vec2_float_type = 0; + uint32_t vec3_float_type = 0; + uint32_t vec4_float_type = 0; + uint32_t function_type = 0; + uint32_t per_vertex_type = 0; + uint32_t ptr_input_float = 0; + uint32_t ptr_input_vec4_float = 0; + uint32_t ptr_output_float = 0; + uint32_t ptr_output_vec4_float = 0; + uint32_t gl_in = 0; + uint32_t gl_out = 0; + uint32_t tess_inner = 0; + uint32_t tess_outer = 0; + uint32_t tess_coord = 0; + uint32_t invocation_id = 0; }; } // namespace -RectListShaders BuildRectListShaders(const ShaderVertexInputInfo& vertex_info, - const ShaderPixelInputInfo* pixel_info) { - const auto parameters = GetParameters(vertex_info, pixel_info); - RectListEmitter control(parameters, spv::ExecutionModelTessellationControl); - RectListEmitter evaluation(parameters, spv::ExecutionModelTessellationEvaluation); +std::vector RectListParameters( + const ShaderPixelInputInfo& pixel_info, const ShaderRecompiler::IR::CompiledShaderInfo& program) { + EXIT_IF(pixel_info.input_num > ShaderVertexInputInfo::RES_MAX); + std::vector active_inputs; + for (const auto& input: program.info.inputs) { + if (input.kind == ShaderRecompiler::IR::StageInputKind::Parameter) { + active_inputs.push_back(input.location); + } + } + std::vector parameters; + for (const auto input: active_inputs) { + parameters.push_back({ShaderPixelParameterMappedLocation(pixel_info, input), + ShaderPixelParameterLocation(pixel_info, active_inputs, input), + ShaderPixelParameterIsFlat(pixel_info, input)}); + } + return parameters; +} + +RectListShaders BuildRectListShaders(std::span parameters, + uint32_t param_export_mask) { + std::vector exported; + std::array output_locations {}; + for (const auto& parameter: parameters) { + if ((param_export_mask & (1u << parameter.input_location)) != 0 && + !output_locations[parameter.output_location]) { + exported.push_back(parameter); + output_locations[parameter.output_location] = true; + } + } + RectListEmitter control(exported, spv::ExecutionModelTessellationControl); + RectListEmitter evaluation(exported, spv::ExecutionModelTessellationEvaluation); return {control.EmitControl(), evaluation.EmitEvaluation()}; } diff --git a/src/graphics/shader/rectListShader.h b/src/graphics/shader/rectListShader.h index ea19a46be..824de43d3 100644 --- a/src/graphics/shader/rectListShader.h +++ b/src/graphics/shader/rectListShader.h @@ -2,20 +2,35 @@ #define EMULATOR_SRC_GRAPHICS_SHADER_RECTLISTSHADER_H_ #include +#include #include namespace Libs::Graphics { struct ShaderPixelInputInfo; -struct ShaderVertexInputInfo; + +namespace ShaderRecompiler::IR { +struct CompiledShaderInfo; +} // namespace ShaderRecompiler::IR + +// A pixel parameter the generated stages forward from vertex export input_location. +struct RectListParameter { + uint32_t input_location = 0; + uint32_t output_location = 0; + bool flat = false; +}; struct RectListShaders { std::vector control; std::vector evaluation; }; -RectListShaders BuildRectListShaders(const ShaderVertexInputInfo& vertex_info, - const ShaderPixelInputInfo* pixel_info); +// Every parameter a rectangle-list pixel program reads. +std::vector RectListParameters( + const ShaderPixelInputInfo& pixel_info, const ShaderRecompiler::IR::CompiledShaderInfo& program); +// Forwards the parameters the vertex stage exports, one per output location. +RectListShaders BuildRectListShaders(std::span parameters, + uint32_t param_export_mask); } // namespace Libs::Graphics diff --git a/src/graphics/shader/shader.h b/src/graphics/shader/shader.h index 8b50133c1..8ec841a85 100644 --- a/src/graphics/shader/shader.h +++ b/src/graphics/shader/shader.h @@ -34,19 +34,14 @@ enum class ShaderType { TessellationEvaluation }; -namespace ShaderRecompiler::IR { -struct CompiledShaderInfo; -} // namespace ShaderRecompiler::IR - -struct ShaderProgram; +struct ShaderModule; struct ShaderStageRuntime { - const ShaderRecompiler::IR::CompiledShaderInfo* program = nullptr; - const ShaderRecompiler::IR::ResourceSnapshot* resources = nullptr; - const ShaderProgram* handle = nullptr; + const ShaderModule* module = nullptr; + const ShaderRecompiler::IR::ResourceSnapshot* resources = nullptr; [[nodiscard]] explicit operator bool() const { - return program != nullptr && resources != nullptr; + return module != nullptr && resources != nullptr; } }; diff --git a/tests/ShaderRecompilerComputeTests.cpp b/tests/ShaderRecompilerComputeTests.cpp index 1373a1387..10ac156a2 100644 --- a/tests/ShaderRecompilerComputeTests.cpp +++ b/tests/ShaderRecompilerComputeTests.cpp @@ -92,6 +92,7 @@ #include #include #include +#include #include #include #include @@ -447,7 +448,7 @@ struct RenderExecutorTestAccess { std::vector layout_bindings; bool compute = false; for (const auto *prepared : stages) { - const auto &program = *prepared->runtime->program; + const auto &program = prepared->runtime->module->info; compute |= program.stage == ShaderType::Compute; const auto shader_stage = program.stage == ShaderType::Vertex ? vk::ShaderStageFlagBits::eVertex @@ -498,7 +499,7 @@ struct RenderExecutorTestAccess { PreparedBindings &bindings) { std::array stages{&bindings}; auto pipeline = CreateDescriptorPipeline(executor, stages); - const auto bind_point = bindings.runtime->program->stage == ShaderType::Compute + const auto bind_point = bindings.runtime->module->info.stage == ShaderType::Compute ? vk::PipelineBindPoint::eCompute : vk::PipelineBindPoint::eGraphics; executor.CommitBindings(buffer, bind_point, pipeline, stages); @@ -583,6 +584,33 @@ struct RenderExecutorTestAccess { } }; +struct PipelineCacheTestAccess { + // Replaces each graphics pipeline whose stage ids name compiled modules with one created + // from only its key and those modules. The GPU must be idle. Returns how many it replaced. + static uint32_t RecreateGraphicsPipelines(PipelineCache &cache) { + std::unordered_map modules{{0, nullptr}}; // Id 0: absent. + for (const auto &program : cache.m_program_cache | std::views::values) + for (const auto &module : program.modules) modules.emplace(module.id, &module); + auto &device = cache.m_graphics.device; + uint32_t count = 0; + for (auto &[key, pipeline] : cache.m_graphics_pipelines) { + // Test-owned modules are not in the cache. + if (!std::ranges::all_of(key.stage_ids, [&](uint64_t id) { return modules.contains(id); })) + continue; + std::array stages{}; + for (size_t i = 0; i < stages.size(); i++) stages[i] = modules.at(key.stage_ids[i]); + PipelineCache::Pipeline recreated; + CreatePipelineInternal(cache.m_graphics, recreated, key, stages, nullptr); + device.destroyPipeline(pipeline.pipeline, nullptr); + device.destroyPipelineLayout(pipeline.pipeline_layout, nullptr); + device.destroyDescriptorSetLayout(pipeline.descriptor_set_layout, nullptr); + pipeline = recreated; + count++; + } + return count; + } +}; + namespace { using u32 = uint32_t; @@ -1509,18 +1537,12 @@ void CheckRectListShaders() { program.info.inputs.push_back( {ShaderRecompiler::IR::StageInputKind::Parameter, 1, 4, "in_param_1"}); - ShaderVertexInputInfo vertex{}; - ShaderRecompiler::IR::CompiledShaderInfo vertex_program{.param_export_mask = 1u}; - ShaderRecompiler::IR::ResourceSnapshot empty_snapshot; - vertex.stage.program = &vertex_program; - vertex.stage.resources = &empty_snapshot; + constexpr uint32_t param_export_mask = 1u; ShaderPixelInputInfo pixel{}; pixel.parameter_mode = ShaderPixelParameterMode::Rectangle; pixel.input_num = 2; pixel.interpolator_settings[0] = 0x400u; pixel.interpolator_settings[1] = 0; - pixel.stage.program = &program; - pixel.stage.resources = &empty_snapshot; std::vector perspective_specialization; BuildStageStaticKey(pixel, perspective_specialization); pixel.ps_no_perspective = true; @@ -1539,7 +1561,8 @@ void CheckRectListShaders() { "duplicate pixel mappings must receive distinct effective output " "locations"); - const auto shaders = BuildRectListShaders(vertex, &pixel); + const auto shaders = + BuildRectListShaders(RectListParameters(pixel, program), param_export_mask); Require(name, "SPIR-V version", shaders.control.size() > 1 && shaders.control[1] == 0x00010500u && shaders.evaluation.size() > 1 && @@ -1588,7 +1611,8 @@ void CheckRectListShaders() { for (const auto flat : {0u, 0x400u}) { pixel.interpolator_settings[0] = flat; pixel.interpolator_settings[1] = flat; - const auto shared = BuildRectListShaders(vertex, &pixel); + const auto shared = + BuildRectListShaders(RectListParameters(pixel, program), param_export_mask); ValidateSpirv(name, shared.control); ValidateSpirv(name, shared.evaluation); Require(name, "same-mode alias disassembly", @@ -1604,7 +1628,7 @@ void CheckRectListShaders() { "the shared location in each rectangle-list stage"); } - const auto position_only = BuildRectListShaders(vertex, nullptr); + const auto position_only = BuildRectListShaders({}, param_export_mask); ValidateSpirv(name, position_only.control); ValidateSpirv(name, position_only.evaluation); } @@ -2210,22 +2234,24 @@ public: HW::Shader shaders{}; scheduler.Begin(registers, user_config, shaders); - ShaderRecompiler::IR::CompiledShaderInfo vertex_program{}; + ShaderModule vertex_module; + auto &vertex_program = vertex_module.info; vertex_program.stage = ShaderType::Vertex; vertex_program.bindings.push_data_start_dword = 2; vertex_program.info.user_data_registers = {0, 1}; vertex_program.bindings.memory_offset_dword = 2; - ShaderRecompiler::IR::CompiledShaderInfo pixel_program{}; + ShaderModule pixel_module; + auto &pixel_program = pixel_module.info; pixel_program.stage = ShaderType::Pixel; pixel_program.bindings.push_data_start_dword = 0; pixel_program.info.user_data_registers = {0, 1}; pixel_program.bindings.memory_offset_dword = 2; ShaderRecompiler::IR::ResourceSnapshot vertex_snapshot; vertex_snapshot.user_data = {0x11111111u, 0x22222222u}; - ShaderStageRuntime vertex_runtime{&vertex_program, &vertex_snapshot}; + ShaderStageRuntime vertex_runtime{&vertex_module, &vertex_snapshot}; ShaderRecompiler::IR::ResourceSnapshot pixel_snapshot; pixel_snapshot.user_data = {0x33333333u, 0x44444444u}; - ShaderStageRuntime pixel_runtime{&pixel_program, &pixel_snapshot}; + ShaderStageRuntime pixel_runtime{&pixel_module, &pixel_snapshot}; PreparedBindings vertex; PreparedBindings pixel; auto &executor = context.GetRenderExecutor(); @@ -5520,8 +5546,9 @@ public: "test HTile entries were not registered in their initial state"); const auto MakeInput = [](uint64_t address, bool read, bool written, - ShaderRecompiler::IR::CompiledShaderInfo &program, + ShaderModule &module, ShaderRecompiler::IR::ResourceSnapshot &snapshot) { + auto &program = module.info; program.stage = ShaderType::Compute; ShaderRecompiler::IR::BufferResource resource{}; resource.read = read; @@ -5541,14 +5568,14 @@ public: value.dword_count = 4; std::copy_n(descriptor.fields, value.dword_count, value.dwords.begin()); ShaderComputeInputInfo input{}; - input.stage.program = &program; + input.stage.module = &module; snapshot.buffers.push_back(value); input.stage.resources = &snapshot; return input; }; auto &executor = context.GetRenderExecutor(); auto &command = scheduler.Current(); - ShaderRecompiler::IR::CompiledShaderInfo read_only_program{}; + ShaderModule read_only_program; ShaderRecompiler::IR::ResourceSnapshot read_only_snapshot; const auto read_only_input = MakeInput(read_only_meta, true, false, read_only_program, @@ -5561,7 +5588,7 @@ public: !texture_cache.IsMetaCleared(read_only_meta, 0), "a metadata read-only dispatch changed logical clear state"); - ShaderRecompiler::IR::CompiledShaderInfo read_write_program{}; + ShaderModule read_write_program; ShaderRecompiler::IR::ResourceSnapshot read_write_snapshot; const auto read_write_input = MakeInput(read_write_meta, true, true, read_write_program, @@ -5574,7 +5601,7 @@ public: !texture_cache.IsMetaCleared(read_write_meta, 0), "a metadata read/modify/write dispatch was replaced by a clear"); - ShaderRecompiler::IR::CompiledShaderInfo write_only_program{}; + ShaderModule write_only_program; ShaderRecompiler::IR::ResourceSnapshot write_only_snapshot; const auto write_only_input = MakeInput(write_only_meta, false, true, write_only_program, @@ -11099,9 +11126,9 @@ public: ShaderComputeInputInfo input{}; const auto program = context.GetPipelineCache().GetComputeProgram( shaders.GetCs(), input); - if (original_id == 0) original_id = program.id; + if (original_id == 0) original_id = program->id; Require(name, "exact function identity", - (program.id == original_id) == (test.value == 41), + (program->id == original_id) == (test.value == 41), "helper bytes were stale or identical relocated/restored code recompiled"); Require(name, "current code dependency", std::ranges::find(input.stage.resources->specialization_reads, @@ -11191,9 +11218,9 @@ public: ShaderComputeInputInfo input{}; const auto program = context.GetPipelineCache().GetComputeProgram( shaders.GetCs(), input); - if (phase == 1) original_vector_id = program.id; + if (phase == 1) original_vector_id = program->id; if (phase >= 2) - Require(name, "ordered helper identity", (program.id == original_vector_id) == (phase == 3), + Require(name, "ordered helper identity", (program->id == original_vector_id) == (phase == 3), "swapped helper targets reused stale code or restored targets recompiled"); const auto& reads = input.stage.resources->specialization_reads; for (u32 i = 0; i < (call_count == 1 ? 1u : 2u); ++i) @@ -11689,7 +11716,7 @@ public: Prospero::BufferFormat::k32UInt, Prospero::ImageType::kColor2D, {8, 1, 1}, 1, 4, 1); const auto image = textures.FindImage(image_desc); - ShaderProgram previous_program; + uint64_t previous_id = 0; for (const auto count : {3u, 7u}) { cache.FillBuffer(count_output, 16u * sizeof(u32), sentinel, false); shaders.SetCsShader({.data_addr = reinterpret_cast(writer.data()), @@ -11734,11 +11761,11 @@ public: ShaderComputeInputInfo input{}; const auto program = context.GetPipelineCache().GetComputeProgram( shaders.GetCs(), input); - if (previous_program.id != 0) { - Require(name, "count permutation reuse", program.id == previous_program.id, + if (previous_id != 0) { + Require(name, "count permutation reuse", program->id == previous_id, "a changing GPU count compiled a new shader permutation"); } - previous_program = program; + previous_id = program->id; Require(name, "clean count repeat", CpOpDispatchIndirect(processor, 0xc0021600u, packet.data(), 0, 0) == 3 && scheduler.CurrentTick() == clean_tick, @@ -11818,29 +11845,29 @@ public: shaders.GetCs(), input); Require(name, "workgroup table refresh and coalescing", input.workgroup_counts[2] == test.groups && - input.stage.program->info.images.size() == 1 && - input.stage.program->info.images[0].indirect_resources.empty() && + input.stage.module->info.info.images.size() == 1 && + input.stage.module->info.info.images[0].indirect_resources.empty() && input.stage.resources->images.size() == 1 && std::equal(std::begin(sharp.fields), std::end(sharp.fields), input.stage.resources->images[0].dwords.begin()) && - !input.stage.program->info.buffers.empty(), + !input.stage.module->info.info.buffers.empty(), "workgroup bounds, refreshed sharp, or live descriptor dimension reads were lost"); Require(name, "workgroup table program reuse", - table_program_id == 0 || program.id == table_program_id, + table_program_id == 0 || program->id == table_program_id, "changing the workgroup count or identical sharp values recompiled the shader"); - table_program_id = program.id; + table_program_id = program->id; const auto *snapshot = input.stage.resources; const auto *image_storage = snapshot->images.data(); const auto flattened_capacity = snapshot->flattened_srt.capacity(); const auto repeated = context.GetPipelineCache().GetComputeProgram( shaders.GetCs(), input); Require(name, "workgroup table cache-hit storage reuse", - repeated.id == program.id && input.stage.resources == snapshot && + repeated->id == program->id && input.stage.resources == snapshot && snapshot->images.data() == image_storage && snapshot->flattened_srt.capacity() == flattened_capacity, "a repeated descriptor table hit replaced its snapshot or storage"); auto binding = RenderExecutorTestAccess::ResolveTexture( - context.GetRenderExecutor(), input.stage.program->info.images[0], + context.GetRenderExecutor(), input.stage.module->info.info.images[0], input.stage.resources->images[0]); const auto target = textures.FindImage(binding.desc); TextureCacheTestAccess::ClearImage(textures, scheduler.Current(), target, @@ -12073,11 +12100,11 @@ public: input.workgroup_counts[1] = input.workgroup_counts[2] = 1u; (void)context.GetPipelineCache().GetComputeProgram( shaders.GetCs(), input); - const auto &info = input.stage.program->info; + const auto &info = input.stage.module->info.info; const auto root = std::ranges::find_if(info.buffers, [](const auto &buffer) { return buffer.indirect_resources.size() == material_rows; }); - const auto &bindings = input.stage.program->bindings; + const auto &bindings = input.stage.module->info.bindings; const auto native_count = bindings.descriptor_counts[ static_cast(ShaderRecompiler::IR::DescriptorBindingKind::Buffers)]; Require(name, "native material descriptor array", @@ -14331,14 +14358,15 @@ public: }; const auto make_buffer_program = [](ShaderType stage, ShaderRecompiler::IR::BufferResource resource) { - ShaderRecompiler::IR::CompiledShaderInfo program{}; + ShaderModule module; + auto &program = module.info; program.stage = stage; resource.descriptor_index = 0; program.info.buffers.push_back(resource); program.bindings.descriptor_counts[0] = 1; program.bindings.descriptor_mask = 1; program.bindings.push_data_start_dword = 0; - return program; + return module; }; { @@ -14441,7 +14469,7 @@ public: bindings.buffers[0].buffer == owner.Handle() && bindings.buffers[0].offset + 2 == owner.Offset(buffer_address) && bindings.buffers[0].range == 14 && - bindings.shader_data[buffer_program.bindings.memory_offset_dword] == 2, + bindings.shader_data[buffer_program.info.bindings.memory_offset_dword] == 2, "native binding lost the byte adjustment or rounded the final halfword range"); } @@ -14449,7 +14477,8 @@ public: constexpr uint64_t address = base + allocation_size - 0x4000 + 12; constexpr uint64_t mapped_tail = base + allocation_size - address; auto writer = make_buffer_program(ShaderType::Vertex, {.written = true}); - ShaderRecompiler::IR::CompiledShaderInfo reader{}; + ShaderModule reader_module; + auto &reader = reader_module.info; reader.stage = ShaderType::Pixel; ShaderRecompiler::IR::ResourceSnapshot writer_snapshot, reader_snapshot; ShaderBufferResource descriptor{}; @@ -14459,7 +14488,7 @@ public: std::copy_n(descriptor.fields, 4, value.dwords.begin()); value.dword_count = 4; ShaderStageRuntime writer_runtime{&writer, &writer_snapshot}; - ShaderStageRuntime reader_runtime{&reader, &reader_snapshot}; + ShaderStageRuntime reader_runtime{&reader_module, &reader_snapshot}; PreparedBindings writer_bindings, reader_bindings; const auto check_range = [&](uint64_t expected, const char *description) { executor.PrepareBindings(writer_runtime, writer_bindings); @@ -14490,8 +14519,8 @@ public: value.dwords[2] = 0x300; check_range(0x300, "finite write preserves its declared range"); value.dwords[2] = UINT32_MAX; - writer.info.buffers[0].written = false; - writer.info.buffers[0].read = true; + writer.info.info.buffers[0].written = false; + writer.info.info.buffers[0].read = true; check_range(mapped_tail, "read-only sentinel preserves mapped tail"); auto &graphics = context.GetGraphics(); auto &storage_limit = graphics.physical_device_properties.limits.maxStorageBufferRange; @@ -14538,7 +14567,8 @@ public: texture_cache.MarkGpuWritten(depth_id); expected = 0xffff; } - ShaderRecompiler::IR::CompiledShaderInfo program{}; + ShaderModule module; + auto &program = module.info; program.stage = ShaderType::Compute; ShaderRecompiler::IR::ResourceSnapshot snapshot; // PPSA24156 binds one-texel R8/R16 allocations as 3D, then 2D. @@ -14560,7 +14590,7 @@ public: DstSel(4, 4, 4, 1) | (static_cast(type) << 28u), 0, 0x00700000u, 0, 0}; } - ShaderStageRuntime runtime{&program, &snapshot}; + ShaderStageRuntime runtime{&module, &snapshot}; PreparedBindings bindings; executor.PrepareBindings(runtime, bindings); Require(name, "overlapping dimension discovery", @@ -14683,11 +14713,12 @@ public: null_program.info.images.back().mip_mode = ShaderRecompiler::IR::ImageMipMode::Dynamic; null_program.info.images.back().mip_count = 3; set_image_bindings(null_program); - ShaderRecompiler::IR::CompiledShaderInfo null_info{}; + ShaderModule null_module; + auto &null_info = null_module.info; null_info.stage = null_program.stage; null_info.info = std::move(null_program.info); null_info.bindings = std::move(null_program.bindings); - ShaderStageRuntime null_runtime{&null_info, &null_snapshot}; + ShaderStageRuntime null_runtime{&null_module, &null_snapshot}; PreparedBindings null_bindings; executor.PrepareBindings(null_runtime, null_bindings); executor.RebindImages(null_bindings); @@ -14923,7 +14954,8 @@ public: tail4 == std::vector{tail_backing_value}, "expanding a shared mip tail lost GPU writes or the added mip's backing bytes"); PreparedBindings mipped_prepared{}; - ShaderRecompiler::IR::CompiledShaderInfo mipped_program{}; + ShaderModule mipped_module; + auto &mipped_program = mipped_module.info; mipped_program.info.images.push_back(storage_resource); mipped_program.info.images.push_back(mipped_storage_resource); mipped_program.info.images.push_back(storage_resource); @@ -14933,7 +14965,7 @@ public: mipped_storage_descriptor, overwide_mipped_storage_descriptor, overwide_mipped_storage_descriptor}; - ShaderStageRuntime mipped_runtime{&mipped_program, &mipped_snapshot}; + ShaderStageRuntime mipped_runtime{&mipped_module, &mipped_snapshot}; mipped_prepared.runtime = &mipped_runtime; mipped_prepared.images.push_back( std::move(plain_mipped_storage_binding)); @@ -15114,7 +15146,8 @@ public: views[2] != views[3] && views[1] == views.back(), "different texture clamps aliased or identical clamped views " "were not reused"); - ShaderRecompiler::IR::CompiledShaderInfo gather_info{}; + ShaderModule gather_module; + auto &gather_info = gather_module.info; gather_info.stage = ShaderType::Compute; gather_info.info = lod_program.program.info; gather_info.info.images[0].mip_mode = @@ -15122,7 +15155,7 @@ public: gather_info.info.images[0].mip_count = 4; auto gather_snapshot = lod_program.resources; gather_snapshot.images[0] = lod_descriptor; - ShaderStageRuntime gather_runtime{&gather_info, &gather_snapshot}; + ShaderStageRuntime gather_runtime{&gather_module, &gather_snapshot}; PreparedBindings gather_bindings; executor.PrepareBindings(gather_runtime, gather_bindings); executor.RebindImages(gather_bindings); @@ -15277,11 +15310,12 @@ public: ShaderRecompiler::IR::ResourceSnapshot storage_snapshot{}; storage_snapshot.images.push_back(storage_descriptor); - ShaderRecompiler::IR::CompiledShaderInfo storage_info{}; + ShaderModule storage_module; + auto &storage_info = storage_module.info; storage_info.stage = storage_program.stage; storage_info.info = std::move(storage_program.info); storage_info.bindings = std::move(storage_program.bindings); - ShaderStageRuntime storage_runtime{&storage_info, &storage_snapshot}; + ShaderStageRuntime storage_runtime{&storage_module, &storage_snapshot}; PreparedBindings storage_discovery; executor.PrepareBindings(storage_runtime, storage_discovery); const auto storage_id = storage_discovery.images[0].image_id; @@ -15308,11 +15342,12 @@ public: set_image_bindings(sampled_program); ShaderRecompiler::IR::ResourceSnapshot sampled_snapshot{}; sampled_snapshot.images.push_back(storage_descriptor); - ShaderRecompiler::IR::CompiledShaderInfo sampled_info{}; + ShaderModule sampled_module; + auto &sampled_info = sampled_module.info; sampled_info.stage = sampled_program.stage; sampled_info.info = std::move(sampled_program.info); sampled_info.bindings = std::move(sampled_program.bindings); - ShaderStageRuntime sampled_runtime{&sampled_info, &sampled_snapshot}; + ShaderStageRuntime sampled_runtime{&sampled_module, &sampled_snapshot}; constexpr uint64_t ordered_sampled_address = base + 0x10000; const uint32_t ordered_sampled_value = 0x89abcdefu; @@ -15332,7 +15367,7 @@ public: ordered_descriptor.dword_count = 8; ShaderRecompiler::IR::ResourceSnapshot ordered_snapshot{}; ordered_snapshot.images.push_back(ordered_descriptor); - ShaderStageRuntime ordered_sampled_runtime{sampled_runtime.program, + ShaderStageRuntime ordered_sampled_runtime{sampled_runtime.module, &ordered_snapshot}; auto ordered_bindings = RenderExecutorTestAccess::PrepareGraphicsBindings( executor, storage_runtime, ordered_sampled_runtime, true); @@ -15387,23 +15422,25 @@ public: vertex_sampled_program.resource_tracking_complete = true; vertex_sampled_program.info.images.push_back(sampled_resource); set_image_bindings(vertex_sampled_program); - ShaderRecompiler::IR::CompiledShaderInfo vertex_sampled_info{}; + ShaderModule vertex_sampled_module; + auto &vertex_sampled_info = vertex_sampled_module.info; vertex_sampled_info.stage = vertex_sampled_program.stage; vertex_sampled_info.info = std::move(vertex_sampled_program.info); vertex_sampled_info.bindings = std::move(vertex_sampled_program.bindings); ShaderStageRuntime vertex_sampled_runtime{ - &vertex_sampled_info, sampled_runtime.resources}; + &vertex_sampled_module, sampled_runtime.resources}; ShaderRecompiler::IR::Program pixel_storage_program{}; pixel_storage_program.stage = ShaderType::Pixel; pixel_storage_program.resource_tracking_complete = true; pixel_storage_program.info.images.push_back(storage_resource); set_image_bindings(pixel_storage_program); - ShaderRecompiler::IR::CompiledShaderInfo pixel_storage_info{}; + ShaderModule pixel_storage_module; + auto &pixel_storage_info = pixel_storage_module.info; pixel_storage_info.stage = pixel_storage_program.stage; pixel_storage_info.info = std::move(pixel_storage_program.info); pixel_storage_info.bindings = std::move(pixel_storage_program.bindings); - ShaderStageRuntime pixel_storage_runtime{&pixel_storage_info, + ShaderStageRuntime pixel_storage_runtime{&pixel_storage_module, storage_runtime.resources}; auto writable_alias_bindings = RenderExecutorTestAccess::PrepareGraphicsBindings( @@ -15476,13 +15513,14 @@ public: split_ir.resource_tracking_complete = true; split_ir.info.images = {storage_resource, sampled_resource}; set_image_bindings(split_ir); - ShaderRecompiler::IR::CompiledShaderInfo split_program{}; + ShaderModule split_module; + auto &split_program = split_module.info; split_program.stage = split_ir.stage; split_program.info = std::move(split_ir.info); split_program.bindings = std::move(split_ir.bindings); PreparedBindings split_bindings{}; ShaderRecompiler::IR::ResourceSnapshot split_snapshot; - ShaderStageRuntime split_runtime{&split_program, &split_snapshot}; + ShaderStageRuntime split_runtime{&split_module, &split_snapshot}; split_bindings.runtime = &split_runtime; split_bindings.images.push_back( {split_id, texture_cache.FindTexture(split_id, split_storage_desc), @@ -15569,12 +15607,13 @@ public: depth_views_ir.resource_tracking_complete = true; depth_views_ir.info.images = {sampled_resource, sampled_resource}; set_image_bindings(depth_views_ir); - ShaderRecompiler::IR::CompiledShaderInfo depth_views_program{}; + ShaderModule depth_views_module; + auto &depth_views_program = depth_views_module.info; depth_views_program.stage = depth_views_ir.stage; depth_views_program.info = std::move(depth_views_ir.info); depth_views_program.bindings = std::move(depth_views_ir.bindings); ShaderRecompiler::IR::ResourceSnapshot depth_views_snapshot; - ShaderStageRuntime depth_views_runtime{&depth_views_program, &depth_views_snapshot}; + ShaderStageRuntime depth_views_runtime{&depth_views_module, &depth_views_snapshot}; PreparedBindings depth_views_binding{}; depth_views_binding.runtime = &depth_views_runtime; depth_views_binding.images.push_back( @@ -15909,8 +15948,8 @@ public: registers.SetDepthClearValue(0.375f); ShaderRecompiler::IR::ResourceSnapshot bounds_snapshot; bounds_snapshot.images.push_back(sampled_depth_value); - ShaderStageRuntime bounds_vertex{&vertex_sampled_info, &bounds_snapshot}; - ShaderStageRuntime bounds_pixel{&sampled_info, &bounds_snapshot}; + ShaderStageRuntime bounds_vertex{&vertex_sampled_module, &bounds_snapshot}; + ShaderStageRuntime bounds_pixel{&sampled_module, &bounds_snapshot}; const auto depth_texels = depth_only.desc.info.extent.width * depth_only.desc.info.extent.height; const auto depth_bytes = depth_texels * sizeof(uint32_t); @@ -16326,9 +16365,9 @@ public: shared_depth_value.dword_count = 8; ShaderRecompiler::IR::ResourceSnapshot shared_depth_snapshot; shared_depth_snapshot.images.push_back(shared_depth_value); - ShaderStageRuntime shared_depth_vertex{&vertex_sampled_info, + ShaderStageRuntime shared_depth_vertex{&vertex_sampled_module, &shared_depth_snapshot}; - ShaderStageRuntime shared_depth_pixel{&sampled_info, + ShaderStageRuntime shared_depth_pixel{&sampled_module, &shared_depth_snapshot}; for (const bool stencil_write : {true, false}) { registers.SetDepthRenderTarget(phased_depth_target); @@ -16500,7 +16539,8 @@ public: (static_cast(Prospero::ImageType::kColor2DArray) << 28u); array_texture.fields[4] = 1; - ShaderRecompiler::IR::CompiledShaderInfo array_program{}; + ShaderModule array_module; + auto &array_program = array_module.info; array_program.stage = ShaderType::Compute; ShaderRecompiler::IR::ImageResource array_resource{}; array_resource.resource_class = @@ -16517,7 +16557,7 @@ public: array_descriptor.dword_count = 8; ShaderRecompiler::IR::ResourceSnapshot array_snapshot{}; array_snapshot.images.push_back(array_descriptor); - ShaderStageRuntime array_runtime{&array_program, &array_snapshot}; + ShaderStageRuntime array_runtime{&array_module, &array_snapshot}; PreparedBindings array_binding; executor.PrepareBindings(array_runtime, array_binding); @@ -16684,7 +16724,8 @@ public: std::end(colliding_msaa_texture.fields), colliding_msaa_descriptor.dwords.begin()); colliding_msaa_descriptor.dword_count = 8; - ShaderRecompiler::IR::CompiledShaderInfo colliding_msaa_program{}; + ShaderModule colliding_msaa_module; + auto &colliding_msaa_program = colliding_msaa_module.info; colliding_msaa_program.stage = ShaderType::Compute; auto colliding_msaa_resource = array_resource; colliding_msaa_resource.dimension = @@ -16693,7 +16734,7 @@ public: ShaderRecompiler::IR::ResourceSnapshot colliding_msaa_snapshot{}; colliding_msaa_snapshot.images.push_back(colliding_msaa_descriptor); ShaderStageRuntime colliding_msaa_runtime{ - &colliding_msaa_program, &colliding_msaa_snapshot}; + &colliding_msaa_module, &colliding_msaa_snapshot}; PreparedBindings colliding_msaa_binding; executor.PrepareBindings(colliding_msaa_runtime, colliding_msaa_binding); executor.RebindImages(colliding_msaa_binding); @@ -16744,7 +16785,8 @@ public: std::copy(std::begin(msaa_texture.fields), std::end(msaa_texture.fields), msaa_descriptor.dwords.begin()); msaa_descriptor.dword_count = 8; - ShaderRecompiler::IR::CompiledShaderInfo msaa_program{}; + ShaderModule msaa_module; + auto &msaa_program = msaa_module.info; msaa_program.stage = ShaderType::Compute; auto msaa_resource = array_resource; msaa_resource.dimension = @@ -16752,7 +16794,7 @@ public: msaa_program.info.images.push_back(msaa_resource); ShaderRecompiler::IR::ResourceSnapshot msaa_snapshot{}; msaa_snapshot.images.push_back(msaa_descriptor); - ShaderStageRuntime msaa_runtime{&msaa_program, &msaa_snapshot}; + ShaderStageRuntime msaa_runtime{&msaa_module, &msaa_snapshot}; PreparedBindings msaa_binding; executor.PrepareBindings(msaa_runtime, msaa_binding); executor.RebindImages(msaa_binding); @@ -16784,7 +16826,8 @@ public: std::end(msaa_array_texture.fields), msaa_array_descriptor.dwords.begin()); msaa_array_descriptor.dword_count = 8; - ShaderRecompiler::IR::CompiledShaderInfo msaa_array_program{}; + ShaderModule msaa_array_module; + auto &msaa_array_program = msaa_array_module.info; msaa_array_program.stage = ShaderType::Compute; auto msaa_array_resource = array_resource; msaa_array_resource.dimension = @@ -16792,7 +16835,7 @@ public: msaa_array_program.info.images.push_back(msaa_array_resource); ShaderRecompiler::IR::ResourceSnapshot msaa_array_snapshot{}; msaa_array_snapshot.images.push_back(msaa_array_descriptor); - ShaderStageRuntime msaa_array_runtime{&msaa_array_program, &msaa_array_snapshot}; + ShaderStageRuntime msaa_array_runtime{&msaa_array_module, &msaa_array_snapshot}; PreparedBindings msaa_array_binding; executor.PrepareBindings(msaa_array_runtime, msaa_array_binding); executor.RebindImages(msaa_array_binding); @@ -17033,7 +17076,8 @@ public: DstSel(4, 5, 6, 7) | (static_cast(linear) << 20u) | (static_cast(Prospero::ImageType::kColor2D) << 28u); - ShaderRecompiler::IR::CompiledShaderInfo program{}; + ShaderModule module; + auto &program = module.info; program.stage = ShaderType::Compute; ShaderRecompiler::IR::ImageResource stencil_resource{}; stencil_resource.resource_class = @@ -17050,7 +17094,7 @@ public: descriptor.dwords.begin()); descriptor.dword_count = 8; snapshot.images.push_back(descriptor); - ShaderStageRuntime runtime{&program, &snapshot}; + ShaderStageRuntime runtime{&module, &snapshot}; PreparedBindings prepared; context.GetRenderExecutor().PrepareBindings(runtime, prepared); @@ -17139,10 +17183,10 @@ public: "captured stencil shader could not resolve its clear byte"); ShaderRecompiler::IR::ApplyResourceSpecialization(stencil_translated.program, stencil_specialization); - ShaderRecompiler::IR::CompiledShaderInfo stencil_program{}; - stencil_program.stage = ShaderType::Compute; - stencil_program.info = std::move(stencil_translated.program.info); - stencil_compute.stage.program = &stencil_program; + ShaderModule stencil_program; + stencil_program.info.stage = ShaderType::Compute; + stencil_program.info.info = std::move(stencil_translated.program.info); + stencil_compute.stage.module = &stencil_program; vk::ClearValue stencil_initial{}; stencil_initial.depthStencil = vk::ClearDepthStencilValue{0.625f, 0x91}; TextureCacheTestAccess::ClearImage(texture_cache, scheduler.Current(), depth_id, @@ -18294,19 +18338,20 @@ public: AppendEnd(&test.fragment_code); auto fragment = CompileFragmentCase(test); const auto vertex_spirv = TestSpv::MakePassthroughVertexSpirv(false); - ShaderProgram vertex_shader{1, CreateShaderModule(name, vertex_spirv)}; - ShaderProgram pixel_shader{2, CreateShaderModule(name, fragment.spirv)}; - ShaderRecompiler::IR::CompiledShaderInfo vertex_program{}; - vertex_program.stage = ShaderType::Vertex; - vertex_program.info.vertex_fetch_components[0] = 2; - vertex_program.info.vertex_fetch_components[1] = packed_vertex_color ? 3 : 4; - ShaderRecompiler::IR::CompiledShaderInfo pixel_program{}; - pixel_program.stage = ShaderType::Pixel; - pixel_program.info = fragment.program.info; - pixel_program.bindings = fragment.program.bindings; + ShaderModule vertex_shader{.id = 1, .module = CreateShaderModule(name, vertex_spirv)}; + vertex_shader.info.stage = ShaderType::Vertex; + vertex_shader.info.info.vertex_fetch_components[0] = 2; + vertex_shader.info.info.vertex_fetch_components[1] = packed_vertex_color ? 3 : 4; + // No narrower host format covers the components the stage reads from either attribute. + vertex_shader.vertex_formats = {VulkanFormat(Prospero::BufferFormat::k32_32Float), + VulkanFormat(color_format)}; + ShaderModule pixel_shader{.id = 2, .module = CreateShaderModule(name, fragment.spirv)}; + pixel_shader.info.stage = ShaderType::Pixel; + pixel_shader.info.info = fragment.program.info; + pixel_shader.info.bindings = fragment.program.bindings; ShaderVertexInputInfo vertex{}; ShaderRecompiler::IR::ResourceSnapshot vertex_snapshot; - vertex.stage.program = &vertex_program; + vertex.stage.module = &vertex_shader; vertex.stage.resources = &vertex_snapshot; vertex.resources_num = 2; vertex.buffers_num = 1; @@ -18317,10 +18362,10 @@ public: vertex.resources[i].fields[3] = DstSel(4, 5, 6, 7) | (static_cast(format) << 12u); vertex.resources_dst[i].registers_num = - i == 0 ? 2 : vertex_program.info.vertex_fetch_components[1]; + i == 0 ? 2 : vertex_shader.info.info.vertex_fetch_components[1]; } ShaderPixelInputInfo pixel{}; - pixel.stage.program = &pixel_program; + pixel.stage.module = &pixel_shader; pixel.stage.resources = &fragment.resources; RenderColorInfo color{}; @@ -18368,8 +18413,6 @@ public: DrawRenderState state{.depth_info = depth, .color_info = {color}, .color_count = 1, .topology = topology, .ps_active = true, .vertex_info = {vertex}, .ps_input_info = pixel}; - state.vertex_info[0].stage.handle = &vertex_shader; - state.ps_input_info.stage.handle = &pixel_shader; return context.GetPipelineCache().GetGraphicsPipeline(scheduler.Current(), state); }; auto &filled = pipeline(true, 2, 2); @@ -18723,8 +18766,6 @@ public: .code_size_bytes = static_cast(native_pixel.size() * sizeof(u32))}); shaders.SetEsShaderBase(vertex_address); shaders.SetPsShaderBase(pixel_address); - const auto owned_vertex_shader = vertex_shader; - const auto owned_pixel_shader = pixel_shader; depth.stencil_test_enable = false; registers.SetViewportTransformControl(0x300); user_config.SetPrimitiveType(Prospero::PrimitiveType::kTriList); @@ -18738,8 +18779,6 @@ public: const auto native = refresh(); vertex = native.vertex_info[0]; pixel = native.ps_input_info; - vertex_shader = *vertex.stage.handle; - pixel_shader = *pixel.stage.handle; }; const auto silent = refresh(); registers.SetDepthShaderControl({.shader_kill_enable = true}); @@ -18763,7 +18802,7 @@ public: depth_writing.ps_active && depth_writing.mrt_mask == 0 && writing.ps_active && writing.mrt_mask == 1u && partial.mrt_mask == 1u && writing.VertexStages().size() == 1 && - writing.vertex_info[0].stage.handle != nullptr, + writing.vertex_info[0].stage.module != nullptr, "pixel activity or its MRT mask ignored the color outputs or depth side effects"); std::array wave_ids{}; std::array, 2> wave_keys; @@ -18774,17 +18813,17 @@ public: registers.SetPsInControl(wave.control); native_programs(); Require(name, "pixel wave metadata propagation", - pixel.input_num == 8 && pixel.stage.program->wave_size == wave.width, + pixel.input_num == 8 && pixel.stage.module->info.wave_size == wave.width, "SPI_PS_IN_CONTROL width did not reach the compiled pixel program"); const auto index = static_cast(wave.width == 32); std::vector key; BuildStageStaticKey(pixel, key); if (wave_ids[index] != 0) { Require(name, "pixel wave cache reuse", - pixel_shader.id == wave_ids[index] && key == wave_keys[index], + pixel.stage.module->id == wave_ids[index] && key == wave_keys[index], "restoring wave64 did not reuse its original program"); } - wave_ids[index] = pixel_shader.id; + wave_ids[index] = pixel.stage.module->id; wave_keys[index] = std::move(key); draw(pipeline(true, 2, 2)); const auto wave_pixels = read_color(); @@ -18832,8 +18871,8 @@ public: (wave == 32u ? 0x00400000u : 0u)); native_programs(); Require(name, "native NGG waterfall wave", - vertex.stage.program->stage == ShaderType::Vertex && - vertex.stage.program->wave_size == wave, + vertex.stage.module->info.stage == ShaderType::Vertex && + vertex.stage.module->info.wave_size == wave, "NGG waterfall bypassed the native vertex path or lost its guest wave width"); for (const u32 count : {3u, 6u, 33u}) { draw(pipeline(true, 2, 2), count); @@ -18907,10 +18946,10 @@ public: auto &selected = pipeline(true, 2, 2); if (test.mode == ShaderAlphaBlendSource::SourceAlpha) { Require(name, "equal blend equation cache reuse", - broadcast_program == 0 || (pixel_shader.id == broadcast_program && + broadcast_program == 0 || (pixel.stage.module->id == broadcast_program && selected.pipeline == broadcast_pipeline), "equivalent separate alpha state created another shader or pipeline"); - broadcast_program = pixel_shader.id; + broadcast_program = pixel.stage.module->id; broadcast_pipeline = selected.pipeline; } constexpr std::array destination{0.3f, 0.3f, 0.1f, 0.2f}; @@ -19333,6 +19372,13 @@ public: check("auto embedded fetch", fetch_vs, false, 4); check("indexed native fetch", vertex_address, true, 2); check("auto native fetch", vertex_address, false, 6); + // Pipelines created from only their key and stage modules draw the same. + scheduler.Finish(); + Require("PipelineFromKey", "vertex input pipelines", + PipelineCacheTestAccess::RecreateGraphicsPipelines(context.GetPipelineCache()) > 0, + "no cached graphics pipeline named only compiled modules"); + check("recreated indexed embedded fetch", fetch_vs, true, 0); + check("recreated auto native fetch", vertex_address, false, 6); } // A fourth VS invocation reads beyond the descriptor instead of reconstructing the corner. @@ -19382,10 +19428,7 @@ public: registers.SetPsInputEna(2); registers.SetPsInputAddr(2); registers.SetPsInputSettings(0, 0); - for (const auto primitive : {Prospero::PrimitiveType::kRectList, - Prospero::PrimitiveType::kRectListLegacy}) { - const char *rect_name = primitive == Prospero::PrimitiveType::kRectList - ? "RectListThreeRecords" : "LegacyRectListThreeRecords"; + const auto check_rect = [&](Prospero::PrimitiveType primitive, const char *rect_name) { user_config.SetPrimitiveType(primitive); registers.SetModeControl({}); TextureCacheTestAccess::ClearImage(cache, scheduler.Current(), sparse_color.image_id, @@ -19409,7 +19452,14 @@ public: } } } - } + }; + check_rect(Prospero::PrimitiveType::kRectList, "RectListThreeRecords"); + check_rect(Prospero::PrimitiveType::kRectListLegacy, "LegacyRectListThreeRecords"); + scheduler.Finish(); + Require("PipelineFromKey", "rectangle-list pipelines", + PipelineCacheTestAccess::RecreateGraphicsPipelines(context.GetPipelineCache()) > 0, + "no cached graphics pipeline named only compiled modules"); + check_rect(Prospero::PrimitiveType::kRectList, "PipelineFromKeyRectList"); // Captured format30 uses uncompressed mode9 at MRT2. Keep a blended // floating-point MRT live beside it, including sparse export routing. constexpr uint64_t packed_address = depth_address + 0x28000; @@ -19516,9 +19566,10 @@ public: "packed conversion or rounding contaminated another MRT or followed a physical slot"); if (test.slot == 2) { auto &id = test.truncate ? truncate_program : half_up_program; - Require(name, "packed rounding program reuse", id == 0 || id == pixel_shader.id, + Require(name, "packed rounding program reuse", + id == 0 || id == pixel.stage.module->id, "restoring half-up rounding failed to reuse its program"); - id = pixel_shader.id; + id = pixel.stage.module->id; } RenderColorInfo packed_target{}, float_target{}; RenderExecutorTestAccess::ResolveRenderColorTarget( @@ -19576,9 +19627,6 @@ public: DestroyBuffer(&packed_output); m_device.destroySampler(packed_sampler); DestroyBuffer(&stencil_readback); - - vertex_shader = owned_vertex_shader; - pixel_shader = owned_pixel_shader; } resources.UnmapMemory(depth_address, allocation_size); scheduler.Finish(); @@ -44530,7 +44578,7 @@ void CheckComputeLdsLimit(VulkanHarness &vulkan) { regs.cs_regs.num_thread_z = 1; regs.cs_regs.wave_size = 32; PipelineCache cache(graphics, nullptr); - ShaderProgram previous; + uint64_t previous_id = 0; uint16_t previous_units = 0; for (const auto units : {static_cast(limit_units - 1u), limit_units, uint16_t{112}, uint16_t{128}, uint16_t{128}}) { @@ -44544,12 +44592,12 @@ void CheckComputeLdsLimit(VulkanHarness &vulkan) { Require(name, "pipeline creation", cache.GetComputePipeline(input).pipeline != nullptr, "full-size LDS shader did not create a Vulkan compute pipeline"); - if (previous.id != 0) { + if (previous_id != 0) { Require(name, "guest-size cache identity", - (program.id == previous.id) == (units == previous_units), + (program->id == previous_id) == (units == previous_units), "different LDS sizes shared a shader, or repeating a size recompiled it"); } - previous = program; + previous_id = program->id; previous_units = units; } std::printf("[gpu] %-32s ok\n", name); @@ -44585,7 +44633,7 @@ void CheckStableProgramIds(VulkanHarness &vulkan) { DstSel(4, 5, 6, 7) | (static_cast(Prospero::BufferFormat::k32UInt) << 12u)}}; std::copy_n(buffer.fields, 4, regs.cs_user_sgpr.value); ShaderComputeInputInfo input{}; - return cache.GetComputeProgram(regs, input).id; + return cache.GetComputeProgram(regs, input)->id; }; std::array forward{}, reverse{}; { @@ -45283,10 +45331,10 @@ void CheckEmbeddedFetchVertexOffset() { u32 index_offset) { ShaderVertexInputInfo vertex; vertex.fetch_embedded = true; - ShaderRecompiler::IR::CompiledShaderInfo program{}; - program.user_data_base = result.program.user_data_base; - program.info.vertex_offset_sgpr = result.program.info.vertex_offset_sgpr; - vertex.stage.program = &program; + ShaderModule module; + module.info.user_data_base = result.program.user_data_base; + module.info.info.vertex_offset_sgpr = result.program.info.vertex_offset_sgpr; + vertex.stage.module = &module; vertex.stage.resources = &result.resources; return ResolveDrawOffsets(index_offset, vertex).first; }; @@ -45294,11 +45342,11 @@ void CheckEmbeddedFetchVertexOffset() { const auto ResolveInstance = [](const CompiledShader &result) { ShaderVertexInputInfo vertex; vertex.fetch_embedded = true; - ShaderRecompiler::IR::CompiledShaderInfo program{}; - program.user_data_base = result.program.user_data_base; - program.info.instance_offset_sgpr = + ShaderModule module; + module.info.user_data_base = result.program.user_data_base; + module.info.info.instance_offset_sgpr = result.program.info.instance_offset_sgpr; - vertex.stage.program = &program; + vertex.stage.module = &module; vertex.stage.resources = &result.resources; return ResolveDrawOffsets(0, vertex).second; }; @@ -47924,17 +47972,17 @@ void CheckDynamicRenderingState() { Require("DynamicRenderingState", "layout identity", first != changed, "attachment layout did not participate in rendering identity"); - PipelineRenderingState rgba{}; - rgba.color_count = 1; - rgba.color_formats[0] = vk::Format::eR8G8B8A8Unorm; + GraphicsPipelineKey rgba{}; + rgba.rendering.color_count = 1; + rgba.rendering.color_formats[0] = vk::Format::eR8G8B8A8Unorm; auto uint_color = rgba; - uint_color.color_formats[0] = vk::Format::eR8G8B8A8Uint; + uint_color.rendering.color_formats[0] = vk::Format::eR8G8B8A8Uint; Require("DynamicRenderingState", "pipeline color format identity", rgba != uint_color, "distinct dynamic-rendering formats can alias one pipeline key"); auto depth = rgba; - depth.depth_format = vk::Format::eD32SfloatS8Uint; - depth.stencil_format = vk::Format::eD32SfloatS8Uint; + depth.rendering.depth_format = vk::Format::eD32SfloatS8Uint; + depth.rendering.stencil_format = vk::Format::eD32SfloatS8Uint; Require("DynamicRenderingState", "pipeline depth/stencil identity", rgba != depth, "depth/stencil formats did not participate in pipeline identity"); diff --git a/tests/shaderCfgTests.cpp b/tests/shaderCfgTests.cpp index 641e48ba6..510d94c23 100644 --- a/tests/shaderCfgTests.cpp +++ b/tests/shaderCfgTests.cpp @@ -8,6 +8,7 @@ #include "graphics/host_gpu/renderer/image/imageView.h" #include "graphics/host_gpu/renderer/image/textureCommon.h" #include "graphics/host_gpu/renderer/pipeline/blendMapping.h" +#include "graphics/host_gpu/renderer/pipeline/pipelineCache.h" #include "graphics/host_gpu/renderer/pipeline/shaderResourceBarrier.h" #include "graphics/shader/recompiler/ShaderRecompiler.h" #include "graphics/shader/recompiler/backend/spirv/SpirvEmitter.h" @@ -163,10 +164,10 @@ void CompilePixelRuntime(const ShaderParams ¶ms, options.input_info.pixel = &input_info; auto result = RecompileForTest(params.code, options); - static std::deque programs; + static std::deque modules; static std::deque resources; - programs.push_back(std::move(result.program).TakeCompiledInfo()); - input_info.stage.program = &programs.back(); + modules.push_back({.info = std::move(result.program).TakeCompiledInfo()}); + input_info.stage.module = &modules.back(); resources.push_back(std::move(result.resources)); input_info.stage.resources = &resources.back(); } @@ -1157,7 +1158,7 @@ void TestNativeShaderResourceDependencies() { set_buffer(0, 0x1000, 16, 3); set_buffer(1, 0x1800, 4, 7); set_buffer(2, 0x2000, 0, 64); - const ShaderRecompiler::IR::CompiledShaderInfo compiled{ + const ShaderModule compiled{.info = { .stage = program.stage, .shader_hash = program.shader_hash, .wave_size = program.wave_size, @@ -1166,8 +1167,8 @@ void TestNativeShaderResourceDependencies() { .scratch_dwords = program.scratch_dwords, .info = program.info, .bindings = program.bindings, - }; - ShaderStageRuntime runtime{.program = &compiled, .resources = &resources}; + }}; + ShaderStageRuntime runtime{.module = &compiled, .resources = &resources}; Check(HasShaderBufferWrites(runtime), "graphics/compute write predicate lost nonempty written buffers"); set_buffer(0, 0, 16, 3); @@ -15925,8 +15926,8 @@ void TestPixelProgramCacheBindingIdentity() { first_params.hash == XXH3_64bits(shader.data(), shader.size_bytes()) && first_params.hash == second_params.hash && first_key == MakeStageStaticKey(second_info) && - first_info.stage.program != nullptr && - second_info.stage.program != nullptr, + first_info.stage.module != nullptr && + second_info.stage.module != nullptr, "pixel program matching did not preserve an identical source/static identity"); return std::pair {first_params.hash, first_key}; };