diff --git a/src/graphics/host_gpu/renderer/debug.cpp b/src/graphics/host_gpu/renderer/debug.cpp index 0f53eb1d2..6bc1e4e62 100644 --- a/src/graphics/host_gpu/renderer/debug.cpp +++ b/src/graphics/host_gpu/renderer/debug.cpp @@ -888,12 +888,10 @@ void LogDrawPhase(const char* draw_name, const char* phase) { } } -void LogPipelineTrace(const char* phase, uint32_t vs_hash0, uint32_t vs_crc32, uint32_t ps_hash0, - uint32_t ps_crc32) { +void LogPipelineTrace(const char* phase, uint64_t vertex_program_id, uint64_t pixel_program_id) { if (graphics_debug_dump_enabled()) { - LOGF("PipelineTrace: %s VS=0x%08" PRIx32 "/0x%08" PRIx32 " PS=0x%08" PRIx32 "/0x%08" PRIx32 - "\n", - phase, vs_hash0, vs_crc32, ps_hash0, ps_crc32); + LOGF("PipelineTrace: %s VS=%" PRIu64 " PS=%" PRIu64 "\n", phase, vertex_program_id, + pixel_program_id); } } diff --git a/src/graphics/host_gpu/renderer/pipeline/pipelineCache.cpp b/src/graphics/host_gpu/renderer/pipeline/pipelineCache.cpp index c4b1f56ce..f77fb981c 100644 --- a/src/graphics/host_gpu/renderer/pipeline/pipelineCache.cpp +++ b/src/graphics/host_gpu/renderer/pipeline/pipelineCache.cpp @@ -10,11 +10,14 @@ #include "graphics/host_gpu/renderer/image/imageView.h" #include "graphics/host_gpu/renderer/render.h" #include "graphics/host_gpu/renderer/renderContext.h" +#include "graphics/shader/shaderCompiler.h" #include +#include #include #include #include +#include namespace Libs::Graphics { @@ -36,6 +39,88 @@ void NormalizeStaticParamsForDynamicState(PipelineStaticParameters& static_param } // namespace +struct PipelineCache::ProgramCache { + struct SourceKey { + ShaderType stage = ShaderType::Unknown; + uint64_t hash = 0; + + bool operator==(const SourceKey&) const = default; + }; + + struct Permutation { + std::vector static_key; + std::shared_ptr program; + ShaderProgram handle; + }; + + struct SourceKeyHash { + std::size_t operator()(const SourceKey& key) const { + std::size_t hash = static_cast(key.stage); + PipelineKeyHash::Mix(hash, static_cast(key.hash)); + if constexpr (sizeof(std::size_t) < sizeof(uint64_t)) { + PipelineKeyHash::Mix(hash, static_cast(key.hash >> 32u)); + } + return hash; + } + }; + + static uint64_t MixId(uint64_t hash, uint64_t value) { + return hash ^ (value + 0x9e3779b97f4a7c15ull + (hash << 6u) + (hash >> 2u)); + } + + static constexpr std::size_t MaxStaticKeyWords = + 5 + ShaderVertexInputInfo::RES_MAX * 17; + + template + ShaderProgram Get(const ShaderParams& params, InputInfo& input_info) { + constexpr ShaderType stage = [] { + if constexpr (std::is_same_v) { + return ShaderType::Vertex; + } else if constexpr (std::is_same_v) { + return ShaderType::Pixel; + } else { + static_assert(std::is_same_v); + return ShaderType::Compute; + } + }(); + + BuildStageStaticKey(input_info, key_scratch); + auto& permutations = programs[{stage, params.hash}]; + for (const auto& permutation: permutations) { + if (permutation.static_key == key_scratch && + MaterializeProgram(permutation.program, params, input_info)) { + return permutation.handle; + } + } + + const auto module = CompileProgram(device, params, input_info); + EXIT_IF(module == nullptr || !input_info.stage); + uint64_t id = MixId(MixId(params.hash, static_cast(stage)), permutations.size()); + if (id == 0) { + id = 1; + } + const ShaderProgram handle {.id = id, .module = module}; + permutations.push_back({key_scratch, input_info.stage.program, handle}); + + std::printf("Num compiled %u shaders\n", ++num_compiled); + return handle; + } + + explicit ProgramCache(vk::Device device): device(device) { + key_scratch.reserve(MaxStaticKeyWords); + } + + std::unordered_map, SourceKeyHash> programs; + std::vector key_scratch; + vk::Device device; + uint32_t num_compiled = 0; +}; + +PipelineCache::PipelineCache(GraphicContext& graphics) + : m_graphics(graphics), m_program_cache(std::make_unique(graphics.device)) { + EXIT_NOT_IMPLEMENTED(!Common::Thread::IsMainThread()); +} + PipelineCache::~PipelineCache() { auto destroy = [this](const auto& pipelines) { for (const auto& [key, pipeline]: pipelines) { @@ -47,32 +132,62 @@ PipelineCache::~PipelineCache() { }; destroy(m_graphics_pipelines); destroy(m_compute_pipelines); + for (const auto& [key, permutations]: m_program_cache->programs) { + (void)key; + for (const auto& permutation: permutations) { + m_graphics.device.destroyShaderModule(permutation.handle.module, nullptr); + } + } +} + +ShaderProgram PipelineCache::GetVertexProgram(const HW::VertexShaderInfo& regs, + const HW::ShaderRegisters& sh, + ShaderVertexInputInfo& input_info) { + const auto params = PrepareProgram(regs, sh, input_info); + Common::LockGuard lock(m_mutex); + return m_program_cache->Get(params, input_info); +} + +ShaderProgram PipelineCache::GetPixelProgram( + const HW::PixelShaderInfo& regs, const HW::ShaderRegisters& sh, + const ShaderVertexInputInfo& vertex_info, + std::span target_export_mapping, + ShaderPixelInputInfo& input_info) { + const auto params = PrepareProgram(regs, sh, vertex_info, target_export_mapping, input_info); + Common::LockGuard lock(m_mutex); + return m_program_cache->Get(params, input_info); +} + +ShaderProgram PipelineCache::GetComputeProgram(const HW::ComputeShaderInfo& regs, + const HW::ShaderRegisters& sh, + ShaderComputeInputInfo& input_info) { + input_info.needs_lds_barriers = !m_graphics.compute_wave64_supported; + const auto params = PrepareProgram(regs, sh, input_info); + Common::LockGuard lock(m_mutex); + return m_program_cache->Get(params, input_info); } bool PipelineStaticParameters::operator==(const PipelineStaticParameters& other) const noexcept { return std::memcmp(this, &other, sizeof(*this)) == 0; } -PipelineCache::GraphicsPipeline& -PipelineCache::CreateGraphicsPipeline(RenderColorInfo* colors, uint32_t color_count, - RenderDepthInfo& depth, ShaderVertexInputInfo& vs_input_info, - CommandBuffer& command, ShaderPixelInputInfo* ps_input_info, - vk::PrimitiveTopology topology, bool primitive_restart_enable, - bool ps_active, std::span vs_spirv, - std::span ps_spirv) { +PipelineCache::GraphicsPipeline& PipelineCache::CreateGraphicsPipeline( + std::span colors, const RenderDepthInfo& depth, + const ShaderVertexInputInfo& vs_input_info, CommandBuffer& command, + const ShaderPixelInputInfo* ps_input_info, vk::PrimitiveTopology topology, + bool primitive_restart_enable, const ShaderProgram& vertex_program, + const ShaderProgram& pixel_program) { KYTY_PROFILER_BLOCK("PipelineCache::CreatePipeline(Gfx)", profiler::colors::DeepOrangeA200); - EXIT_IF(colors == nullptr); - EXIT_IF(color_count > RENDER_COLOR_ATTACHMENTS_MAX); - EXIT_IF(vs_spirv.empty()); - EXIT_IF(ps_active && ps_spirv.empty()); + EXIT_IF(colors.size() > RENDER_COLOR_ATTACHMENTS_MAX); + EXIT_IF(!vertex_program); + const bool ps_active = ps_input_info != nullptr; + EXIT_IF(ps_active && !pixel_program); + const auto color_count = static_cast(colors.size()); Common::LockGuard lock(m_mutex); - auto& ctx = command.GetRegisters(); - auto& sh_ctx = command.GetShaders(); + auto& ctx = command.GetRegisters(); - const auto& vertex_info = sh_ctx.GetVs(); - const auto& ps_regs = sh_ctx.GetPs(); const HW::BlendColor& bclr = ctx.GetBlendColor(); uint32_t color_mask[RENDER_COLOR_ATTACHMENTS_MAX] = {}; for (uint32_t i = 0; i < color_count; i++) { @@ -83,11 +198,8 @@ PipelineCache::CreateGraphicsPipeline(RenderColorInfo* colors, uint32_t color_co } const HW::ModeControl& mc = ctx.GetModeControl(); - auto vs_id = ShaderGetIdVS(vertex_info, vs_input_info, true); - ShaderId ps_id {}; - if (ps_active) { - ps_id = ShaderGetIdPS(ps_regs, *ps_input_info, true); - } + const auto vs_id = vertex_program.id; + const auto ps_id = ps_active ? pixel_program.id : 0; PipelineStaticParameters static_params {}; GraphicsPipeline p {}; @@ -203,20 +315,17 @@ PipelineCache::CreateGraphicsPipeline(RenderColorInfo* colors, uint32_t color_co if (ps_active) { ShaderDbgDumpInputInfo(*ps_input_info); } - LOGF("PipelineTrace: shader binaries VS=0x%08" PRIx32 "/0x%08" PRIx32 " words=%" PRIu64 - " PS=0x%08" PRIx32 "/0x%08" PRIx32 " words=%" PRIu64 "\n", - vs_id.hash0, vs_id.crc32, static_cast(vs_spirv.size()), ps_id.hash0, - ps_id.crc32, static_cast(ps_spirv.size())); + LOGF("PipelineTrace: shader modules VS=%" PRIu64 " module=%p PS=%" PRIu64 + " module=%p\n", + vs_id, static_cast(vertex_program.module), ps_id, + static_cast(pixel_program.module)); } auto cached = std::make_unique(p); - LogPipelineTrace("CreatePipelineInternal begin", vs_id.hash0, vs_id.crc32, ps_id.hash0, - ps_id.crc32); - CreatePipelineInternal(m_graphics, *cached, rendering, vs_input_info, vs_spirv, ps_input_info, - ps_spirv, static_params, vs_id.hash0, vs_id.crc32, ps_id.hash0, - ps_id.crc32, ps_active); - LogPipelineTrace("CreatePipelineInternal done", vs_id.hash0, vs_id.crc32, ps_id.hash0, - ps_id.crc32); + LogPipelineTrace("CreatePipelineInternal begin", vs_id, ps_id); + CreatePipelineInternal(m_graphics, *cached, rendering, vs_input_info, vertex_program.module, + ps_input_info, pixel_program.module, static_params); + LogPipelineTrace("CreatePipelineInternal done", vs_id, ps_id); EXIT_NOT_IMPLEMENTED(cached->pipeline == nullptr); EXIT_NOT_IMPLEMENTED(cached->pipeline_layout == nullptr); @@ -228,19 +337,16 @@ PipelineCache::CreateGraphicsPipeline(RenderColorInfo* colors, uint32_t color_co } PipelineCache::ComputePipeline& -PipelineCache::CreateComputePipeline(ShaderComputeInputInfo& input_info, - const HW::ComputeShaderInfo& cs_regs, - std::span cs_spirv) { +PipelineCache::CreateComputePipeline(ShaderComputeInputInfo& input_info, + const ShaderProgram& compute_program) { KYTY_PROFILER_BLOCK("PipelineCache::CreatePipeline(Compute)", profiler::colors::RedA100); - EXIT_IF(cs_spirv.empty()); + EXIT_IF(!compute_program); Common::LockGuard lock(m_mutex); - auto cs_id = ShaderGetIdCS(cs_regs, input_info, true); - ComputePipeline p {}; - p.cs_shader_id = cs_id; + p.cs_shader_id = compute_program.id; ComputePipelineKey key {}; key.cs_shader_id = p.cs_shader_id; @@ -254,7 +360,7 @@ PipelineCache::CreateComputePipeline(ShaderComputeInputInfo& input_info, } auto cached = std::make_unique(p); - CreatePipelineInternal(m_graphics, *cached, input_info, cs_spirv); + CreatePipelineInternal(m_graphics, *cached, input_info, compute_program.module); EXIT_NOT_IMPLEMENTED(cached->pipeline == nullptr); EXIT_NOT_IMPLEMENTED(cached->pipeline_layout == nullptr); diff --git a/src/graphics/host_gpu/renderer/pipeline/pipelineCache.h b/src/graphics/host_gpu/renderer/pipeline/pipelineCache.h index 6ac271973..10dd53d46 100644 --- a/src/graphics/host_gpu/renderer/pipeline/pipelineCache.h +++ b/src/graphics/host_gpu/renderer/pipeline/pipelineCache.h @@ -95,11 +95,18 @@ struct PipelineRenderingState { bool operator==(const PipelineRenderingState&) const = default; }; +struct ShaderProgram { + uint64_t id = 0; + vk::ShaderModule module = nullptr; + + explicit operator bool() const { + return id != 0 && module != nullptr; + } +}; + class PipelineCache { public: - explicit PipelineCache(GraphicContext& graphics): m_graphics(graphics) { - EXIT_NOT_IMPLEMENTED(!Common::Thread::IsMainThread()); - } + explicit PipelineCache(GraphicContext& graphics); ~PipelineCache(); KYTY_CLASS_NO_COPY(PipelineCache); @@ -111,29 +118,42 @@ public: }; struct GraphicsPipeline: Pipeline { - ShaderId vs_shader_id; - ShaderId ps_shader_id; + uint64_t vs_shader_id = 0; + uint64_t ps_shader_id = 0; }; struct ComputePipeline: Pipeline { - ShaderId cs_shader_id; + uint64_t cs_shader_id = 0; }; - GraphicsPipeline& - CreateGraphicsPipeline(RenderColorInfo* colors, uint32_t color_count, RenderDepthInfo& depth, - ShaderVertexInputInfo& vs_input_info, CommandBuffer& command, - ShaderPixelInputInfo* ps_input_info, vk::PrimitiveTopology topology, - bool primitive_restart_enable, bool ps_active, - std::span vs_spirv, std::span ps_spirv); - ComputePipeline& CreateComputePipeline(ShaderComputeInputInfo& input_info, - const HW::ComputeShaderInfo& cs_regs, - std::span cs_spirv); + ShaderProgram GetVertexProgram(const HW::VertexShaderInfo& regs, + const HW::ShaderRegisters& sh, + ShaderVertexInputInfo& input_info); + ShaderProgram + GetPixelProgram(const HW::PixelShaderInfo& regs, const HW::ShaderRegisters& sh, + const ShaderVertexInputInfo& vertex_info, + std::span target_export_mapping, + ShaderPixelInputInfo& input_info); + ShaderProgram GetComputeProgram(const HW::ComputeShaderInfo& regs, + const HW::ShaderRegisters& sh, + ShaderComputeInputInfo& input_info); + + GraphicsPipeline& CreateGraphicsPipeline( + std::span colors, const RenderDepthInfo& depth, + const ShaderVertexInputInfo& vs_input_info, CommandBuffer& command, + const ShaderPixelInputInfo* ps_input_info, vk::PrimitiveTopology topology, + bool primitive_restart_enable, const ShaderProgram& vertex_program, + const ShaderProgram& pixel_program); + ComputePipeline& CreateComputePipeline(ShaderComputeInputInfo& input_info, + const ShaderProgram& compute_program); private: + struct ProgramCache; + struct GraphicsPipelineKey { PipelineRenderingState rendering; - ShaderId vs_shader_id; - ShaderId ps_shader_id; + uint64_t vs_shader_id = 0; + uint64_t ps_shader_id = 0; PipelineStaticParameters static_params; bool operator==(const GraphicsPipelineKey& other) const { @@ -143,7 +163,7 @@ private: }; struct ComputePipelineKey { - ShaderId cs_shader_id; + uint64_t cs_shader_id = 0; bool operator==(const ComputePipelineKey& other) const { return cs_shader_id == other.cs_shader_id; @@ -156,15 +176,6 @@ private: (hash >> 2u); } - static void MixShaderId(std::size_t& hash, const ShaderId& id) { - Mix(hash, id.hash0); - Mix(hash, id.crc32); - Mix(hash, id.ids.size()); - for (auto value: id.ids) { - Mix(hash, value); - } - } - static void MixStaticParams(std::size_t& hash, const PipelineStaticParameters& params) { const auto* bytes = reinterpret_cast(¶ms); for (std::size_t i = 0; i < sizeof(params); i++) { @@ -186,8 +197,8 @@ private: std::size_t operator()(const GraphicsPipelineKey& key) const { std::size_t hash = 0; PipelineKeyHash::MixRendering(hash, key.rendering); - PipelineKeyHash::MixShaderId(hash, key.vs_shader_id); - PipelineKeyHash::MixShaderId(hash, key.ps_shader_id); + PipelineKeyHash::Mix(hash, key.vs_shader_id); + PipelineKeyHash::Mix(hash, key.ps_shader_id); PipelineKeyHash::MixStaticParams(hash, key.static_params); return hash; } @@ -196,12 +207,13 @@ private: struct ComputePipelineKeyHash { std::size_t operator()(const ComputePipelineKey& key) const { std::size_t hash = 0; - PipelineKeyHash::MixShaderId(hash, key.cs_shader_id); + PipelineKeyHash::Mix(hash, key.cs_shader_id); return hash; } }; - GraphicContext& m_graphics; + GraphicContext& m_graphics; + std::unique_ptr m_program_cache; std::unordered_map, GraphicsPipelineKeyHash> m_graphics_pipelines; @@ -210,17 +222,15 @@ private: Common::Mutex m_mutex; }; -void LogPipelineTrace(const char* phase, uint32_t vs_hash0, uint32_t vs_crc32, uint32_t ps_hash0, - uint32_t ps_crc32); +void LogPipelineTrace(const char* phase, uint64_t vertex_program_id, uint64_t pixel_program_id); void CreatePipelineInternal( GraphicContext& graphics, PipelineCache::GraphicsPipeline& pipeline, const PipelineRenderingState& rendering, const ShaderVertexInputInfo& vs_input_info, - std::span vs_shader, const ShaderPixelInputInfo* ps_input_info, - std::span ps_shader, const PipelineStaticParameters& static_params, - uint32_t vs_hash0, uint32_t vs_crc32, uint32_t ps_hash0, uint32_t ps_crc32, bool ps_active); + vk::ShaderModule vertex_module, const ShaderPixelInputInfo* ps_input_info, + vk::ShaderModule pixel_module, const PipelineStaticParameters& static_params); void CreatePipelineInternal(GraphicContext& graphics, PipelineCache::ComputePipeline& pipeline, const ShaderComputeInputInfo& input_info, - std::span cs_shader); + vk::ShaderModule compute_module); } // namespace Libs::Graphics diff --git a/src/graphics/host_gpu/renderer/pipeline/shaders.cpp b/src/graphics/host_gpu/renderer/pipeline/shaders.cpp index 939f67243..637aebf28 100644 --- a/src/graphics/host_gpu/renderer/pipeline/shaders.cpp +++ b/src/graphics/host_gpu/renderer/pipeline/shaders.cpp @@ -422,43 +422,19 @@ static void CreateDescriptorLayout(GraphicContext& graphics, PipelineCache::Pipe void CreatePipelineInternal( GraphicContext& graphics, PipelineCache::GraphicsPipeline& pipeline, const PipelineRenderingState& rendering, const ShaderVertexInputInfo& vs_input_info, - std::span vs_shader, const ShaderPixelInputInfo* ps_input_info, - std::span ps_shader, const PipelineStaticParameters& static_params, - uint32_t vs_hash0, uint32_t vs_crc32, uint32_t ps_hash0, uint32_t ps_crc32, bool ps_active) { - EXIT_IF(ps_active && ps_input_info == nullptr); + vk::ShaderModule vertex_module, const ShaderPixelInputInfo* ps_input_info, + vk::ShaderModule pixel_module, const PipelineStaticParameters& static_params) { + const bool ps_active = ps_input_info != nullptr; + EXIT_IF(vertex_module == nullptr || (ps_active && pixel_module == nullptr)); const bool rect_list = static_params.topology == vk::PrimitiveTopology::ePatchList; - vk::ShaderModule vert_shader_module = nullptr; vk::ShaderModule tess_control_shader_module = nullptr; vk::ShaderModule tess_eval_shader_module = nullptr; - vk::ShaderModule frag_shader_module = nullptr; vk::ShaderModuleCreateInfo create_info {}; - create_info.sType = vk::StructureType::eShaderModuleCreateInfo; - create_info.pNext = nullptr; - create_info.flags = {}; - - create_info.codeSize = vs_shader.size() * 4; - create_info.pCode = vs_shader.data(); - auto result = graphics.device.createShaderModule(&create_info, nullptr, &vert_shader_module); - if (graphics_debug_dump_enabled()) { - LOGF("PipelineTrace: vkCreateShaderModule VS done result=%s module=%p\n", - VulkanToString(result).c_str(), static_cast(vert_shader_module)); - } - EXIT_NOT_IMPLEMENTED(result != vk::Result::eSuccess); - - if (ps_active) { - create_info.codeSize = ps_shader.size() * 4; - create_info.pCode = ps_shader.data(); - result = graphics.device.createShaderModule(&create_info, nullptr, &frag_shader_module); - if (graphics_debug_dump_enabled()) { - LOGF("PipelineTrace: vkCreateShaderModule PS done result=%s module=%p\n", - VulkanToString(result).c_str(), static_cast(frag_shader_module)); - } - EXIT_NOT_IMPLEMENTED(result != vk::Result::eSuccess); - } + vk::Result result {}; if (rect_list) { const auto shaders = BuildRectListShaders(vs_input_info, ps_active ? ps_input_info : nullptr); @@ -483,10 +459,8 @@ void CreatePipelineInternal( EXIT_NOT_IMPLEMENTED(result != vk::Result::eSuccess); } - EXIT_NOT_IMPLEMENTED(vert_shader_module == nullptr); EXIT_NOT_IMPLEMENTED( rect_list && (tess_control_shader_module == nullptr || tess_eval_shader_module == nullptr)); - EXIT_NOT_IMPLEMENTED(ps_active && frag_shader_module == nullptr); vk::PipelineShaderStageCreateInfo vert_shader_stage_info {}; @@ -494,7 +468,7 @@ void CreatePipelineInternal( vert_shader_stage_info.pNext = nullptr; vert_shader_stage_info.flags = {}; vert_shader_stage_info.stage = vk::ShaderStageFlagBits::eVertex; - vert_shader_stage_info.module = vert_shader_module; + vert_shader_stage_info.module = vertex_module; vert_shader_stage_info.pName = "main"; vert_shader_stage_info.pSpecializationInfo = nullptr; @@ -503,7 +477,7 @@ void CreatePipelineInternal( frag_shader_stage_info.pNext = nullptr; frag_shader_stage_info.flags = {}; frag_shader_stage_info.stage = vk::ShaderStageFlagBits::eFragment; - frag_shader_stage_info.module = frag_shader_module; + frag_shader_stage_info.module = pixel_module; frag_shader_stage_info.pName = "main"; frag_shader_stage_info.pSpecializationInfo = nullptr; @@ -854,9 +828,9 @@ void CreatePipelineInternal( EXIT_IF(pipeline.pipeline_layout != nullptr); if (graphics_debug_dump_enabled()) { - LOGF("PipelineTrace: vkCreatePipelineLayout begin VS=0x%08" PRIx32 "/0x%08" PRIx32 - " PS=0x%08" PRIx32 "/0x%08" PRIx32 " set_layouts=1 push_constants=%" PRIu32 "\n", - vs_hash0, vs_crc32, ps_hash0, ps_crc32, 1u); + LOGF("PipelineTrace: vkCreatePipelineLayout begin VS=%" PRIu64 " PS=%" PRIu64 + " set_layouts=1 push_constants=%" PRIu32 "\n", + pipeline.vs_shader_id, pipeline.ps_shader_id, 1u); } result = graphics.device.createPipelineLayout(&pipeline_layout_info, nullptr, &pipeline.pipeline_layout); @@ -955,11 +929,12 @@ void CreatePipelineInternal( EXIT_IF(pipeline.pipeline != nullptr); if (graphics_debug_dump_enabled()) { - LOGF("PipelineTrace: vkCreateGraphicsPipelines begin VS=0x%08" PRIx32 "/0x%08" PRIx32 - " PS=0x%08" PRIx32 "/0x%08" PRIx32 " topology=%" PRIu32 " color_mask=0x%08" PRIx32 + LOGF("PipelineTrace: vkCreateGraphicsPipelines begin VS=%" PRIu64 " PS=%" PRIu64 + " topology=%" PRIu32 " color_mask=0x%08" PRIx32 " depth=%s blend=%s dyn_states=%" PRIu32 " viewport=%f,%f %fx%f" " scissor=%d,%d %ux%u\n", - vs_hash0, vs_crc32, ps_hash0, ps_crc32, static_cast(static_params.topology), + pipeline.vs_shader_id, pipeline.ps_shader_id, + static_cast(static_params.topology), static_params.color_mask[0], (static_params.with_depth ? "true" : "false"), (static_params.blend_enable[0] ? "true" : "false"), dynamic_states_count, viewport.x, viewport.y, viewport.width, viewport.height, scissor.offset.x, scissor.offset.y, @@ -975,39 +950,19 @@ void CreatePipelineInternal( EXIT_NOT_IMPLEMENTED(pipeline.pipeline == nullptr); - if (frag_shader_module != nullptr) { - graphics.device.destroyShaderModule(frag_shader_module, nullptr); - } if (tess_control_shader_module != nullptr) { graphics.device.destroyShaderModule(tess_control_shader_module, nullptr); } if (tess_eval_shader_module != nullptr) { graphics.device.destroyShaderModule(tess_eval_shader_module, nullptr); } - graphics.device.destroyShaderModule(vert_shader_module, nullptr); } // NOLINTNEXTLINE(readability-function-cognitive-complexity) void CreatePipelineInternal(GraphicContext& graphics, PipelineCache::ComputePipeline& pipeline, const ShaderComputeInputInfo& input_info, - std::span cs_shader) { - vk::ShaderModule comp_shader_module = nullptr; - - vk::ShaderModuleCreateInfo create_info {}; - - create_info.sType = vk::StructureType::eShaderModuleCreateInfo; - create_info.pNext = nullptr; - create_info.flags = {}; - create_info.codeSize = cs_shader.size() * 4; - create_info.pCode = cs_shader.data(); - LOGF("PipelineTrace: vkCreateShaderModule CS begin words=%" PRIu64 "\n", - static_cast(cs_shader.size())); - auto result = graphics.device.createShaderModule(&create_info, nullptr, &comp_shader_module); - LOGF("PipelineTrace: vkCreateShaderModule CS done result=%s module=%p\n", - VulkanToString(result).c_str(), static_cast(comp_shader_module)); - EXIT_NOT_IMPLEMENTED(result != vk::Result::eSuccess); - - EXIT_NOT_IMPLEMENTED(comp_shader_module == nullptr); + vk::ShaderModule compute_module) { + EXIT_IF(compute_module == nullptr); vk::PipelineShaderStageCreateInfo comp_shader_stage_info {}; vk::PipelineShaderStageRequiredSubgroupSizeCreateInfo comp_subgroup_size {}; @@ -1015,7 +970,7 @@ void CreatePipelineInternal(GraphicContext& graphics, PipelineCache::ComputePipe comp_shader_stage_info.pNext = nullptr; comp_shader_stage_info.flags = {}; comp_shader_stage_info.stage = vk::ShaderStageFlagBits::eCompute; - comp_shader_stage_info.module = comp_shader_module; + comp_shader_stage_info.module = compute_module; comp_shader_stage_info.pName = "main"; comp_shader_stage_info.pSpecializationInfo = nullptr; EXIT_IF(!input_info.stage); @@ -1048,8 +1003,8 @@ void CreatePipelineInternal(GraphicContext& graphics, PipelineCache::ComputePipe LOGF("PipelineTrace: vkCreatePipelineLayout CS begin set_layouts=1 push_constants=%u\n", 1u); - result = graphics.device.createPipelineLayout(&pipeline_layout_info, nullptr, - &pipeline.pipeline_layout); + auto result = graphics.device.createPipelineLayout(&pipeline_layout_info, nullptr, + &pipeline.pipeline_layout); LOGF("PipelineTrace: vkCreatePipelineLayout CS done result=%s layout=%p\n", VulkanToString(result).c_str(), static_cast(pipeline.pipeline_layout)); EXIT_NOT_IMPLEMENTED(result != vk::Result::eSuccess); @@ -1075,8 +1030,6 @@ void CreatePipelineInternal(GraphicContext& graphics, PipelineCache::ComputePipe EXIT_NOT_IMPLEMENTED(result != vk::Result::eSuccess); EXIT_NOT_IMPLEMENTED(pipeline.pipeline == nullptr); - - graphics.device.destroyShaderModule(comp_shader_module, nullptr); } } // namespace Libs::Graphics diff --git a/src/graphics/host_gpu/renderer/renderCompute.cpp b/src/graphics/host_gpu/renderer/renderCompute.cpp index 34040242d..f53514299 100644 --- a/src/graphics/host_gpu/renderer/renderCompute.cpp +++ b/src/graphics/host_gpu/renderer/renderCompute.cpp @@ -207,17 +207,12 @@ void RenderExecutor::DispatchDirect(uint64_t submit_id, CommandBuffer& buffer, const auto& cs_regs = sh_ctx.GetCs(); const auto& sh_regs = ctx.GetShaderRegisters(); - ShaderComputeInputInfo input_info {}; - std::span cs_shader; - if (!ShaderCompileInfoCS(cs_regs, sh_regs, !m_context.GetGraphics().compute_wave64_supported, - input_info, cs_shader)) { - EXIT("ShaderCompileInfoCS failed for dispatch with CS shader 0x%016" PRIx64 "\n", - cs_regs.cs_regs.data_addr); - } - + ShaderComputeInputInfo input_info {}; const bool use_thread_dimensions = (mode & DISPATCH_INITIATOR_USE_THREAD_DIMENSIONS) != 0; + input_info.dispatch_thread_dimensions = use_thread_dimensions; + const auto compute_program = + m_context.GetPipelineCache().GetComputeProgram(cs_regs, sh_regs, input_info); if (use_thread_dimensions) { - input_info.dispatch_thread_dimensions = true; input_info.dispatch_threads_num[0] = thread_group_x; input_info.dispatch_threads_num[1] = thread_group_y; input_info.dispatch_threads_num[2] = thread_group_z; @@ -332,7 +327,7 @@ void RenderExecutor::DispatchDirect(uint64_t submit_id, CommandBuffer& buffer, buffer.EndRendering(); auto& pipeline = - m_context.GetPipelineCache().CreateComputePipeline(input_info, sh_ctx.GetCs(), cs_shader); + m_context.GetPipelineCache().CreateComputePipeline(input_info, compute_program); auto bindings = PrepareBindings(input_info.stage); FindBuffers(bindings); if (program.info.uses_dma) { diff --git a/src/graphics/host_gpu/renderer/renderDraw.cpp b/src/graphics/host_gpu/renderer/renderDraw.cpp index 243d0231d..f9c151bbf 100644 --- a/src/graphics/host_gpu/renderer/renderDraw.cpp +++ b/src/graphics/host_gpu/renderer/renderDraw.cpp @@ -483,15 +483,15 @@ static bool ShouldSkipGeShader(const CommandBuffer& buffer) { } struct DrawRenderState { - RenderDepthInfo depth_info; - RenderColorInfo color_info[RENDER_COLOR_ATTACHMENTS_MAX] = {}; - uint32_t color_count = 0; - bool ps_active = true; - RenderState rendering; - ShaderVertexInputInfo vs_input_info; - ShaderPixelInputInfo ps_input_info; - std::span vs_shader; - std::span ps_shader; + RenderDepthInfo depth_info; + RenderColorInfo color_info[RENDER_COLOR_ATTACHMENTS_MAX] = {}; + uint32_t color_count = 0; + bool ps_active = true; + RenderState rendering; + ShaderVertexInputInfo vs_input_info; + ShaderPixelInputInfo ps_input_info; + ShaderProgram vertex_program; + ShaderProgram pixel_program; }; struct DrawCallInfo { @@ -1003,32 +1003,30 @@ static void RefreshShaders(CommandBuffer& buffer, const DrawCallInfo& draw, bool const auto& pixel_shader_info = sh_ctx.GetPs(); const auto& shader_regs = ctx.GetShaderRegisters(); - state.vs_shader = {}; - state.ps_shader = {}; - state.ps_input_info = {}; + state.vertex_program = {}; + state.pixel_program = {}; + state.ps_input_info = {}; std::array target_export_mapping {}; for (uint32_t i = 0; i < state.color_count; i++) { target_export_mapping[state.color_info[i].target_slot] = state.color_info[i].export_mapping; } if (log_phases) { - LogDrawPhase(draw.name, "ShaderCompileInfoVS"); - } - if (!ShaderCompileInfoVS(vertex_shader_info, shader_regs, state.vs_input_info, - state.vs_shader)) { - EXIT("ShaderCompileInfoVS failed for draw %s\n", draw.name); + LogDrawPhase(draw.name, "GetVertexProgram"); } + auto& pipeline_cache = buffer.GetContext().GetPipelineCache(); + state.vertex_program = + pipeline_cache.GetVertexProgram(vertex_shader_info, shader_regs, state.vs_input_info); if (!state.ps_active) { return; } if (log_phases) { - LogDrawPhase(draw.name, "ShaderCompileInfoPS"); - } - if (!ShaderCompileInfoPS(pixel_shader_info, shader_regs, state.vs_input_info, - target_export_mapping, state.ps_input_info, state.ps_shader)) { - EXIT("ShaderCompileInfoPS failed for draw %s\n", draw.name); + LogDrawPhase(draw.name, "GetPixelProgram"); } + state.pixel_program = + pipeline_cache.GetPixelProgram(pixel_shader_info, shader_regs, state.vs_input_info, + target_export_mapping, state.ps_input_info); } static PreparedVertexBuffers PrepareVertexBuffers(uint64_t submit_id, CommandBuffer& buffer, @@ -1180,9 +1178,9 @@ void RenderExecutor::ExecutePreparedDraw(uint64_t submit_id, CommandBuffer& buff LogDrawPhase(draw.name, "CreatePipeline"); } auto& pipeline = m_context.GetPipelineCache().CreateGraphicsPipeline( - state.color_info, state.color_count, state.depth_info, state.vs_input_info, buffer, - &state.ps_input_info, topology, primitive_restart_enable, state.ps_active, state.vs_shader, - state.ps_shader); + std::span {state.color_info, state.color_count}, state.depth_info, state.vs_input_info, buffer, + state.ps_active ? &state.ps_input_info : nullptr, topology, primitive_restart_enable, + state.vertex_program, state.pixel_program); // Resource preparation above may synchronously finish and restart the scheduler. From this // point onward, every operation targets the current command buffer and cannot touch guest diff --git a/src/graphics/shader/shader.cpp b/src/graphics/shader/shader.cpp index b7b1df848..b12a81f71 100644 --- a/src/graphics/shader/shader.cpp +++ b/src/graphics/shader/shader.cpp @@ -4,7 +4,6 @@ #include "common/common.h" #include "common/emulatorConfig.h" #include "common/file.h" -#include "common/hash.h" #include "common/logging/log.h" #include "common/magicEnum.h" #include "common/profiler.h" @@ -16,6 +15,7 @@ #include "graphics/shader/recompiler/ShaderRecompiler.h" #include "graphics/shader/recompiler/frontend/decode/ShaderDecoder.h" #include "graphics/shader/recompiler/ir/ShaderIR.h" +#include "graphics/shader/shaderCompiler.h" #include "graphics/shader/shaderVertexMetadata.h" #include "kernel/memory.h" #include "libs/errno.h" @@ -35,6 +35,7 @@ #include #include #include +#include #if KYTY_PLATFORM == KYTY_PLATFORM_WINDOWS #ifndef NOMINMAX @@ -79,55 +80,6 @@ struct ShaderBinaryInfo { static std::unique_ptr> g_shader_map; static std::mutex g_shader_map_mutex; -struct ShaderStageProgramKey { - ShaderType stage = ShaderType::Unknown; - uint64_t shader_hash = 0; - ShaderId program_id; - Config::ShaderOptimizationType optimization_type = Config::ShaderOptimizationType::None; - - bool operator==(const ShaderStageProgramKey& other) const { - return stage == other.stage && shader_hash == other.shader_hash && - program_id == other.program_id && optimization_type == other.optimization_type; - } -}; - -struct ShaderStageProgramKeyHash { - size_t operator()(const ShaderStageProgramKey& key) const { - uint32_t hash = 0; - auto mix = [&hash](uint32_t value) { - hash ^= Common::hash32(value) + 0x9e3779b9u + (hash << 6u) + (hash >> 2u); - }; - - mix(static_cast(key.stage)); - mix(Common::hash64(key.shader_hash)); - mix(key.program_id.hash0); - mix(key.program_id.crc32); - mix(static_cast(key.program_id.ids.size())); - for (auto value: key.program_id.ids) { - mix(value); - } - mix(static_cast(key.optimization_type)); - return hash; - } -}; - -struct ShaderProgramPermutation { - std::vector spirv; - std::shared_ptr program; -}; - -static std::unordered_map>, - ShaderStageProgramKeyHash> - g_shader_program_cache; -static std::mutex g_shader_program_cache_mutex; - -static constexpr uint32_t ShaderMaxPermutationsPerProgram = 64; - -static std::span MakeShaderSpirvView(const std::vector& spirv) { - return {spirv.data(), spirv.size()}; -} - void ShaderInit() { EXIT_IF(g_shader_map != nullptr); @@ -142,17 +94,16 @@ void ShaderMapUserData(uint64_t addr, const ShaderMappedData& data) { (*g_shader_map)[addr] = data; } -static bool ShaderGetMappedData(uint64_t addr, ShaderMappedData& data) { +static ShaderMappedData ShaderGetMappedData(uint64_t addr, const char* label) { EXIT_IF(g_shader_map == nullptr); std::scoped_lock lock(g_shader_map_mutex); if (auto iter = g_shader_map->find(addr); iter != g_shader_map->end()) { - data = iter->second; - return true; + return iter->second; } - return false; + EXIT("%s shader=0x%016" PRIx64 " is missing from ShaderMap\n", label, addr); } static bool SpirvDisassemble(const uint32_t* src_binary, size_t src_binary_size, @@ -222,23 +173,22 @@ static const ShaderBinaryInfo* GetBinaryInfo(const uint32_t* code) { return nullptr; } -static std::span ShaderGetMappedCode(uint64_t shader_addr, const char* label, - uint64_t shader_hash) { - ShaderMappedData data; - if (!ShaderGetMappedData(shader_addr, data)) { - EXIT("%s hash=0x%016" PRIx64 " shader=0x%016" PRIx64 " is missing from ShaderMap\n", label, - shader_hash, shader_addr); - } +static ShaderParams GetShaderParams(uint64_t shader_addr, const char* label, uint64_t declared_hash, + std::span user_data, + const ShaderMappedData& data) { if (data.code_size_bytes == 0 || data.code_size_bytes % sizeof(uint32_t) != 0) { EXIT("%s hash=0x%016" PRIx64 " shader=0x%016" PRIx64 " has invalid AGC shader_size=0x%08" PRIx32 "\n", - label, shader_hash, shader_addr, data.code_size_bytes); + label, declared_hash, shader_addr, data.code_size_bytes); } const auto code_words = data.code_size_bytes / sizeof(uint32_t); - LOGF("%s hash=0x%016" PRIx64 " shader=0x%016" PRIx64 " using AGC shader_size=0x%08" PRIx32 - " (%" PRIu32 " dwords)\n", - label, shader_hash, shader_addr, data.code_size_bytes, code_words); - return {reinterpret_cast(shader_addr), code_words}; + const auto code = std::span {reinterpret_cast(shader_addr), code_words}; + return { + .code = code, + .user_data = user_data, + .hash = declared_hash != 0 ? declared_hash + : XXH3_64bits(code.data(), code.size_bytes()), + }; } #if 0 @@ -441,18 +391,6 @@ static void cs_check(const HW::CsStageRegisters& cs, const HW::ShaderRegisters& } #endif -static void GetNextGenFallbackShaderId(uint64_t addr, uint32_t& hash0, uint32_t& crc32) { - auto x = addr; - x ^= x >> 33u; - x *= 0xff51afd7ed558ccdull; - x ^= x >> 33u; - x *= 0xc4ceb9fe1a85ec53ull; - x ^= x >> 33u; - - hash0 = static_cast((x >> 32u) & 0xffffffffu); - crc32 = static_cast(x & 0xffffffffu); -} - static void ShaderDetectBuffers(ShaderVertexInputInfo& info) { KYTY_PROFILER_FUNCTION(); @@ -741,7 +679,9 @@ static uint32_t ShaderCalcPsSystemInputBase(const HW::ShaderRegisters& regs) { } static bool ShaderGetStaticInputInfoVS(const HW::VertexShaderInfo& regs, - const HW::ShaderRegisters& sh, ShaderVertexInputInfo& info) { + const HW::ShaderRegisters& sh, + const ShaderMappedData& data, + ShaderVertexInputInfo& info) { KYTY_PROFILER_FUNCTION(); info = {}; @@ -754,12 +694,6 @@ static bool ShaderGetStaticInputInfoVS(const HW::VertexShaderInfo& regs, uint64_t shader_addr = regs.es_regs.data_addr; const HW::UserSgprInfo& user_sgpr = regs.gs_user_sgpr; auto user_sgpr_num = regs.gs_regs.rsrc2.user_sgpr; - ShaderMappedData data; - if (!ShaderGetMappedData(shader_addr, data)) { - LOGF("ShaderGetInputInfoVS(): shader=0x%016" PRIx64 " is missing from ShaderMap\n", - shader_addr); - return false; - } info.scratch_size_dwords = data.scratch_size_dwords; if (data.user_data == nullptr) { @@ -805,15 +739,10 @@ static void ShaderGetStaticInputInfoPS( const HW::PixelShaderInfo& regs, const HW::ShaderRegisters& sh, const ShaderVertexInputInfo& vs_info, std::span target_export_mapping, - ShaderPixelInputInfo& ps_info) { + const ShaderMappedData& data, ShaderPixelInputInfo& ps_info) { KYTY_PROFILER_FUNCTION(); ps_info = {}; - ShaderMappedData data; - if (!ShaderGetMappedData(regs.ps_regs.data_addr, data)) { - EXIT("ShaderGetInputInfoPS(): shader=0x%016" PRIx64 " is missing from ShaderMap\n", - regs.ps_regs.data_addr); - } ps_info.scratch_size_dwords = data.scratch_size_dwords; ps_info.push_constant_offset = vs_info.stage.program != nullptr @@ -824,6 +753,7 @@ static void ShaderGetStaticInputInfoPS( // SPI_PS_IN_CONTROL.NUM_INTERP occupies bits 5:0. Keep the remaining control // flags in the hardware state and extract only the input count here. ps_info.input_num = sh.ps_in_control & 0x3fu; + EXIT_NOT_IMPLEMENTED(ps_info.input_num > std::size(ps_info.interpolator_settings)); ps_info.ps_system_input_base = ShaderCalcPsSystemInputBase(sh); const uint32_t active_inputs = sh.ps_input_ena & sh.ps_input_addr; if ((active_inputs & 0x00000002u) != 0) { @@ -866,15 +796,14 @@ static void ShaderGetStaticInputInfoPS( } static void ShaderGetStaticInputInfoCS(const HW::ComputeShaderInfo& regs, - const HW::ShaderRegisters& /*sh*/, - ShaderComputeInputInfo& info) { + const HW::ShaderRegisters& /*sh*/, + const ShaderMappedData& data, + ShaderComputeInputInfo& info) { + const bool dispatch_thread_dimensions = info.dispatch_thread_dimensions; + const bool needs_lds_barriers = info.needs_lds_barriers; info = {}; - ShaderMappedData data; - if (!ShaderGetMappedData(regs.cs_regs.data_addr, data)) { - EXIT("ShaderGetInputInfoCS(): shader=0x%016" PRIx64 " is missing from ShaderMap\n", - regs.cs_regs.data_addr); - } - + info.dispatch_thread_dimensions = dispatch_thread_dimensions; + info.needs_lds_barriers = needs_lds_barriers; info.threads_num[0] = regs.cs_regs.num_thread_x; info.threads_num[1] = regs.cs_regs.num_thread_y; info.threads_num[2] = regs.cs_regs.num_thread_z; @@ -890,16 +819,6 @@ static void ShaderGetStaticInputInfoCS(const HW::ComputeShaderInfo& regs, info.workgroup_register = regs.cs_regs.user_sgpr; } -static ShaderStageProgramKey MakeShaderStageProgramKey(ShaderType stage, uint64_t shader_hash, - const ShaderId& program_id) { - ShaderStageProgramKey key {}; - key.stage = stage; - key.shader_hash = shader_hash; - key.program_id = program_id; - key.optimization_type = Config::GetShaderOptimizationType(); - return key; -} - static void ApplyVertexOutputs(ShaderVertexInputInfo& info, const ShaderRecompiler::IR::Program& program) { info.export_count = 0; @@ -922,285 +841,173 @@ static void ApplyPixelOutputs(ShaderPixelInputInfo& info, } } -static std::string ShaderDescribeSpecialization(const ShaderRecompiler::IR::Program& program); - -static bool LogPermutationMismatch(const ShaderProgramPermutation& permutation, const char* stage, - uint64_t shader_hash, const std::string& error) { +static bool LogPermutationMismatch(const ShaderRecompiler::IR::Program& program, + const char* stage, const std::string& error) { static std::atomic log_count {0}; if (Config::GraphicsDebugDumpEnabled() && log_count.fetch_add(1, std::memory_order_relaxed) < 64) { - LOGF("ShaderProgramCache native runtime mismatch %s shader=0x%016" PRIx64 - ": %s\n specialization: %s\n", - stage, shader_hash, error.c_str(), - ShaderDescribeSpecialization(*permutation.program).c_str()); + LOGF("ShaderProgramCache native runtime mismatch %s shader=0x%016" PRIx64 ": %s\n", + stage, program.shader_hash, error.c_str()); } return false; } -static bool TryUseVertexPermutation(const ShaderProgramPermutation& permutation, - const HW::VertexShaderInfo& regs, ShaderVertexInputInfo& info, - uint64_t shader_hash) { +bool MaterializeProgram(const std::shared_ptr& program, + const ShaderParams& params, ShaderVertexInputInfo& info) { std::string error; - if (!ShaderMaterializeStageRuntime( - permutation.program, - std::span(regs.gs_user_sgpr.value, regs.gs_regs.rsrc2.user_sgpr), - regs.es_regs.data_addr, info.stage, &error, ReadShaderGuestMemory)) { - return LogPermutationMismatch(permutation, "VS", shader_hash, error); + if (!ShaderMaterializeStageRuntime(program, params.user_data, params.Base(), info.stage, &error, + ReadShaderGuestMemory)) { + return LogPermutationMismatch(*program, "VS", error); } - ApplyVertexOutputs(info, *permutation.program); + ApplyVertexOutputs(info, *program); return true; } -static bool TryUsePixelPermutation(const ShaderProgramPermutation& permutation, - const HW::PixelShaderInfo& regs, ShaderPixelInputInfo& info, - uint64_t shader_hash) { +bool MaterializeProgram(const std::shared_ptr& program, + const ShaderParams& params, ShaderPixelInputInfo& info) { std::string error; - if (!ShaderMaterializeStageRuntime( - permutation.program, - std::span(regs.ps_user_sgpr.value, regs.ps_regs.rsrc2.user_sgpr), - regs.ps_regs.data_addr, info.stage, &error, ReadShaderGuestMemory)) { - return LogPermutationMismatch(permutation, "PS", shader_hash, error); + if (!ShaderMaterializeStageRuntime(program, params.user_data, params.Base(), info.stage, &error, + ReadShaderGuestMemory)) { + return LogPermutationMismatch(*program, "PS", error); } - ApplyPixelOutputs(info, *permutation.program); + ApplyPixelOutputs(info, *program); return true; } -static bool TryUseComputePermutation(const ShaderProgramPermutation& permutation, - const HW::ComputeShaderInfo& regs, - ShaderComputeInputInfo& info, uint64_t shader_hash) { +bool MaterializeProgram(const std::shared_ptr& program, + const ShaderParams& params, ShaderComputeInputInfo& info) { std::string error; - if (!ShaderMaterializeStageRuntime( - permutation.program, - std::span(regs.cs_user_sgpr.value, regs.cs_regs.user_sgpr), - regs.cs_regs.data_addr, info.stage, &error, ReadShaderGuestMemory)) { - return LogPermutationMismatch(permutation, "CS", shader_hash, error); + if (!ShaderMaterializeStageRuntime(program, params.user_data, params.Base(), info.stage, &error, + ReadShaderGuestMemory)) { + return LogPermutationMismatch(*program, "CS", error); } return true; } -static void LogShaderProgramCacheHit(const char* stage, uint64_t shader_hash, uint64_t words) { - if (!Config::GraphicsDebugDumpEnabled()) { - return; +void BuildStageStaticKey(const ShaderVertexInputInfo& info, std::vector& key) { + EXIT_IF(info.resources_num < 0 || info.resources_num > ShaderVertexInputInfo::RES_MAX || + info.buffers_num < 0 || info.buffers_num > ShaderVertexInputInfo::RES_MAX); + key.clear(); + key.push_back(static_cast(info.fetch_embedded)); + key.push_back(info.resources_num); + key.push_back(info.scratch_size_dwords); + key.push_back(info.pa_cl_vs_out_cntl); + + for (int i = 0; i < info.resources_num; i++) { + const auto& resource = info.resources[i]; + const auto& destination = info.resources_dst[i]; + key.push_back(destination.register_start); + key.push_back(destination.registers_num); + key.push_back(destination.fetch_index); + key.push_back(resource.Stride()); + key.push_back(static_cast(resource.SwizzleEnabled())); + key.push_back(resource.DstSelX()); + key.push_back(resource.DstSelY()); + key.push_back(resource.DstSelZ()); + key.push_back(resource.DstSelW()); + key.push_back(resource.RawFormat()); + key.push_back(resource.OutOfBounds()); + key.push_back(static_cast(resource.AddTid())); } - static std::atomic log_count {0}; - if (log_count.fetch_add(1, std::memory_order_relaxed) >= 512) { - return; - } - - LOGF("ShaderProgramCache: reused %s shader=0x%016" PRIx64 " words=%" PRIu64 "\n", stage, - shader_hash, words); -} - -static std::string ShaderDescribeSpecialization(const ShaderRecompiler::IR::Program& program) { - std::string ret = fmt::format( - "push={} groups={} user={} buffers={} images={} samplers={} dma={} srt={}", - program.bindings.push_constant_size, program.bindings.descriptors.size(), - program.bindings.user_data_registers.size(), program.info.buffers.size(), - program.info.images.size(), program.info.samplers.size(), program.info.uses_dma, - program.srt_reads.size()); - for (uint32_t i = 0; i < program.info.buffers.size(); i++) { - const auto& buffer = program.info.buffers[i]; - ret += - fmt::format(" b{}[stride={} format={} swizzle=0x{:03x}]", i, buffer.packed_stride, - static_cast(buffer.descriptor_format), buffer.descriptor_swizzle); - } - for (uint32_t i = 0; i < program.info.images.size(); i++) { - const auto& image = program.info.images[i]; - ret += fmt::format( - " i{}[kind={} dim={} mip={}/{} format={} swizzle=0x{:03x} indirect={}/{}+{}]", i, - static_cast(image.kind), static_cast(image.dimension), - static_cast(image.mip_mode), image.mip_count, - static_cast(image.conversion_format), image.shader_swizzle, - image.indirect_root, image.indirect_mapping_offset, image.indirect_mapping_capacity); - } - return ret; -} - -static void ShaderAppendNativeSpecialization(std::vector& ids, - const ShaderRecompiler::IR::Program& program) { - EXIT_IF(!program.binding_layout_complete); - ids.push_back(program.bindings.push_constant_size); - ids.push_back(program.bindings.memory_offset_dword); - ids.push_back(program.bindings.memory_offset_count); - ids.push_back(static_cast(program.bindings.user_data_registers.size())); - ids.insert(ids.end(), program.bindings.user_data_registers.begin(), - program.bindings.user_data_registers.end()); - ids.push_back(static_cast(program.bindings.descriptors.size())); - for (const auto& binding: program.bindings.descriptors) { - ids.push_back(static_cast(binding.kind)); - ids.push_back(static_cast(binding.resources.size())); - ids.insert(ids.end(), binding.resources.begin(), binding.resources.end()); - } - ids.push_back(static_cast(program.info.buffers.size())); - for (const auto& buffer: program.info.buffers) { - ids.push_back(buffer.packed_stride); - ids.push_back(static_cast(buffer.descriptor_format)); - ids.push_back(buffer.descriptor_swizzle); - } - ids.push_back(static_cast(program.info.images.size())); - for (const auto& image: program.info.images) { - ids.push_back(static_cast(image.kind)); - ids.push_back(static_cast(image.dimension)); - ids.push_back(static_cast(image.cube)); - ids.push_back(static_cast(image.mip_mode)); - ids.push_back(image.mip_count); - ids.push_back(static_cast(image.conversion_format)); - ids.push_back(image.shader_swizzle); - ids.push_back(image.indirect_root); - ids.push_back(image.indirect_mapping_offset); - ids.push_back(image.indirect_mapping_capacity); - ids.push_back(static_cast(image.indirect_resources.size())); - ids.insert(ids.end(), image.indirect_resources.begin(), image.indirect_resources.end()); - } -} - -static std::span AddShaderProgramPermutation(const char* stage, - uint64_t shader_hash, - const ShaderStageProgramKey& key, - ShaderProgramPermutation permutation) { - static std::atomic compiled {0}; - const auto compiled_count = compiled.fetch_add(1, std::memory_order_relaxed) + 1; - std::printf("Num compiled %d shaders\n", compiled_count); - - std::scoped_lock lock(g_shader_program_cache_mutex); - auto& permutations = g_shader_program_cache[key]; - - if (permutations.size() >= ShaderMaxPermutationsPerProgram) { - LOGF("ShaderProgramCache overflow: %s\n", - ShaderDescribeSpecialization(*permutation.program).c_str()); - for (uint64_t i = 0; i < static_cast(permutations.size()); i++) { - LOGF("ShaderProgramCache overflow existing[%" PRIu64 "]: %s\n", i, - ShaderDescribeSpecialization(*permutations[static_cast(i)]->program) - .c_str()); - } - EXIT("ShaderProgramCache: more than %u permutations for same %s shader: " - "shader=0x%016" PRIx64 " existing=%" PRIu64 " incoming_words=%" PRIu64 - " program_hash0=0x%08" PRIx32 " program_crc32=0x%08" PRIx32 " program_ids=%" PRIu64 - "\n", - ShaderMaxPermutationsPerProgram, stage, shader_hash, - static_cast(permutations.size()), - static_cast(permutation.program->bindings.descriptors.size()), - key.program_id.hash0, key.program_id.crc32, - static_cast(key.program_id.ids.size())); - } - - auto cached = std::make_unique(std::move(permutation)); - auto spirv = MakeShaderSpirvView(cached->spirv); - permutations.push_back(std::move(cached)); - return spirv; -} - -bool ShaderCompileInfoVS(const HW::VertexShaderInfo& regs, const HW::ShaderRegisters& sh, - ShaderVertexInputInfo& info, std::span& spirv) { - spirv = {}; - - if (!ShaderGetStaticInputInfoVS(regs, sh, info)) { - return false; - } - const auto shader_hash = regs.gs_regs.chksum; - const auto program_id = ShaderGetIdVS(regs, info, false); - const auto key = MakeShaderStageProgramKey(ShaderType::Vertex, shader_hash, program_id); - - { - std::scoped_lock lock(g_shader_program_cache_mutex); - if (auto iter = g_shader_program_cache.find(key); iter != g_shader_program_cache.end()) { - for (const auto& permutation: iter->second) { - if (TryUseVertexPermutation(*permutation, regs, info, shader_hash)) { - spirv = MakeShaderSpirvView(permutation->spirv); - LogShaderProgramCacheHit("VS", shader_hash, - static_cast(spirv.size())); - return true; - } - } + key.push_back(info.buffers_num); + int attributes_num = 0; + for (int i = 0; i < info.buffers_num; i++) { + const auto& buffer = info.buffers[i]; + EXIT_IF(buffer.attr_num < 0 || buffer.attr_num > ShaderVertexInputBuffer::ATTR_MAX); + attributes_num += buffer.attr_num; + EXIT_IF(attributes_num > ShaderVertexInputInfo::RES_MAX); + key.push_back(buffer.attr_num); + key.push_back(buffer.stride); + key.push_back(buffer.fetch_index); + for (int j = 0; j < buffer.attr_num; j++) { + key.push_back(buffer.attr_indices[j]); + key.push_back(buffer.attr_offsets[j]); } } - - std::vector compiled_spirv; - if (!ShaderCompileSpirvVS(regs, sh, info, compiled_spirv)) { - return false; - } - - ShaderProgramPermutation permutation {}; - permutation.spirv = std::move(compiled_spirv); - permutation.program = info.stage.program; - spirv = AddShaderProgramPermutation("VS", shader_hash, key, std::move(permutation)); - return true; } -bool ShaderCompileInfoPS(const HW::PixelShaderInfo& regs, const HW::ShaderRegisters& sh, - const ShaderVertexInputInfo& vs_info, - std::span target_export_mapping, - ShaderPixelInputInfo& ps_info, std::span& spirv) { - spirv = {}; - - ShaderGetStaticInputInfoPS(regs, sh, vs_info, target_export_mapping, ps_info); - const auto shader_hash = - regs.ps_regs.chksum != 0 ? regs.ps_regs.chksum : regs.ps_regs.data_addr; - const auto program_id = ShaderGetIdPS(regs, ps_info, false); - const auto key = MakeShaderStageProgramKey(ShaderType::Pixel, shader_hash, program_id); - - { - std::scoped_lock lock(g_shader_program_cache_mutex); - if (auto iter = g_shader_program_cache.find(key); iter != g_shader_program_cache.end()) { - for (const auto& permutation: iter->second) { - if (TryUsePixelPermutation(*permutation, regs, ps_info, shader_hash)) { - spirv = MakeShaderSpirvView(permutation->spirv); - LogShaderProgramCacheHit("PS", shader_hash, - static_cast(spirv.size())); - return true; - } - } +void BuildStageStaticKey(const ShaderPixelInputInfo& info, std::vector& key) { + EXIT_IF(info.input_num > std::size(info.interpolator_settings)); + key.clear(); + key.push_back(info.push_constant_offset); + key.push_back(info.scratch_size_dwords); + key.push_back(info.input_num); + key.push_back(info.ps_system_input_base); + key.push_back(info.custom_interpolation_mask); + key.push_back(info.ps_perspective_center_vgpr); + key.push_back(static_cast(info.ps_pos_x)); + key.push_back(static_cast(info.ps_pos_y)); + key.push_back(static_cast(info.ps_pos_z)); + key.push_back(static_cast(info.ps_pos_w)); + key.push_back(static_cast(info.ps_front_face)); + key.push_back(static_cast(info.ps_no_perspective)); + key.push_back(static_cast(info.ps_pixel_kill_enable)); + key.push_back(static_cast(info.ps_depth_export_enable)); + key.push_back(static_cast(info.ps_sample_mask_export_enable)); + key.push_back(static_cast(info.ps_early_z)); + key.insert(key.end(), std::begin(info.target_output_mode), std::end(info.target_output_mode)); + for (uint32_t base = 0; base < info.target_export_mapping.size(); base += 4u) { + uint32_t packed = 0; + for (uint32_t i = 0; i < 4u; i++) { + packed |= static_cast(info.target_export_mapping[base + i].packed) + << (i * 8u); } + key.push_back(packed); } - - std::vector compiled_spirv; - if (!ShaderCompileSpirvPS(regs, sh, ps_info, compiled_spirv)) { - return false; - } - - ShaderProgramPermutation permutation {}; - permutation.spirv = std::move(compiled_spirv); - permutation.program = ps_info.stage.program; - spirv = AddShaderProgramPermutation("PS", shader_hash, key, std::move(permutation)); - return true; + key.insert(key.end(), std::begin(info.interpolator_settings), + std::begin(info.interpolator_settings) + info.input_num); } -bool ShaderCompileInfoCS(const HW::ComputeShaderInfo& regs, const HW::ShaderRegisters& sh, - bool needs_lds_barriers, ShaderComputeInputInfo& info, - std::span& spirv) { - spirv = {}; - - ShaderGetStaticInputInfoCS(regs, sh, info); - info.needs_lds_barriers = needs_lds_barriers; - const auto shader_hash = regs.cs_regs.data_addr; - const auto program_id = ShaderGetIdCS(regs, info, false); - const auto key = MakeShaderStageProgramKey(ShaderType::Compute, shader_hash, program_id); - - { - std::scoped_lock lock(g_shader_program_cache_mutex); - if (auto iter = g_shader_program_cache.find(key); iter != g_shader_program_cache.end()) { - for (const auto& permutation: iter->second) { - if (TryUseComputePermutation(*permutation, regs, info, shader_hash)) { - spirv = MakeShaderSpirvView(permutation->spirv); - LogShaderProgramCacheHit("CS", shader_hash, - static_cast(spirv.size())); - return true; - } - } - } +void BuildStageStaticKey(const ShaderComputeInputInfo& info, std::vector& key) { + key.clear(); + key.push_back(info.workgroup_register); + key.push_back(info.wave_size); + key.push_back(info.thread_ids_num); + key.push_back(info.lds_size_dwords); + key.push_back(info.scratch_size_dwords); + key.push_back(static_cast(info.needs_lds_barriers)); + key.push_back(static_cast(info.dispatch_thread_dimensions)); + for (int i = 0; i < 3; i++) { + key.push_back(info.threads_num[i]); + key.push_back(static_cast(info.group_id[i])); } + key.push_back(static_cast(info.tg_size_en)); +} - std::vector compiled_spirv; - if (!ShaderCompileSpirvCS(regs, sh, info, compiled_spirv)) { - return false; +ShaderParams PrepareProgram(const HW::VertexShaderInfo& regs, const HW::ShaderRegisters& sh, + ShaderVertexInputInfo& info) { + const auto data = ShaderGetMappedData(regs.es_regs.data_addr, "ShaderGetInputInfoVS():"); + if (!ShaderGetStaticInputInfoVS(regs, sh, data, info)) { + EXIT("failed to prepare vertex shader program\n"); } + return GetShaderParams( + regs.es_regs.data_addr, "ShaderRecompiler VS", regs.gs_regs.chksum, + std::span(regs.gs_user_sgpr.value, regs.gs_regs.rsrc2.user_sgpr), data); +} - ShaderProgramPermutation permutation {}; - permutation.spirv = std::move(compiled_spirv); - permutation.program = info.stage.program; - spirv = AddShaderProgramPermutation("CS", shader_hash, key, std::move(permutation)); - return true; +ShaderParams PrepareProgram( + const HW::PixelShaderInfo& regs, const HW::ShaderRegisters& sh, + const ShaderVertexInputInfo& vs_info, + std::span target_export_mapping, + ShaderPixelInputInfo& ps_info) { + const auto data = ShaderGetMappedData(regs.ps_regs.data_addr, "ShaderGetInputInfoPS():"); + ShaderGetStaticInputInfoPS(regs, sh, vs_info, target_export_mapping, data, ps_info); + return GetShaderParams( + regs.ps_regs.data_addr, "ShaderRecompiler PS", regs.ps_regs.chksum, + std::span(regs.ps_user_sgpr.value, regs.ps_regs.rsrc2.user_sgpr), data); +} + +ShaderParams PrepareProgram(const HW::ComputeShaderInfo& regs, const HW::ShaderRegisters& sh, + ShaderComputeInputInfo& info) { + const auto data = ShaderGetMappedData(regs.cs_regs.data_addr, "ShaderGetInputInfoCS():"); + ShaderGetStaticInputInfoCS(regs, sh, data, info); + const auto* header = GetBinaryInfo(reinterpret_cast(regs.cs_regs.data_addr)); + const auto declared_hash = + header != nullptr ? (static_cast(header->hash1) << 32u) | header->hash0 : 0; + return GetShaderParams( + regs.cs_regs.data_addr, "ShaderRecompiler CS", declared_hash, + std::span(regs.cs_user_sgpr.value, regs.cs_regs.user_sgpr), data); } // NOLINTNEXTLINE(readability-function-cognitive-complexity) @@ -1407,314 +1214,114 @@ static void DumpShaderRecompilerOriginal(const char* type, uint64_t shader_hash, } } -bool ShaderCompileSpirvVS(const HW::VertexShaderInfo& regs, const HW::ShaderRegisters& sh, - ShaderVertexInputInfo& input_info, std::vector& spirv) { +static const char* ShaderStageName(ShaderType stage) { + switch (stage) { + case ShaderType::Vertex: return "vs"; + case ShaderType::Pixel: return "ps"; + case ShaderType::Compute: return "cs"; + default: EXIT("invalid shader stage\n"); + } +} + +static const char* ShaderStageLabel(ShaderType stage) { + switch (stage) { + case ShaderType::Vertex: return "ShaderRecompiler VS"; + case ShaderType::Pixel: return "ShaderRecompiler PS"; + case ShaderType::Compute: return "ShaderRecompiler CS"; + default: EXIT("invalid shader stage\n"); + } +} + +static ShaderRecompiler::CompileOptions MakeCompileOptions(const ShaderParams& params, + ShaderType stage) { + ShaderRecompiler::CompileOptions options; + options.stage = stage; + options.shader_hash = params.hash; + options.shader_base = params.Base(); + options.user_data_count = static_cast(params.user_data.size()); + options.user_data = params.user_data.data(); + options.read_specialization_memory = ReadShaderGuestMemory; + options.dump_ir = ShaderRecompilerTextDumpEnabled(); + options.early_dump = options.dump_ir; + options.dump_label = ShaderStageLabel(stage); + return options; +} + +static vk::ShaderModule CompileModule(vk::Device device, const ShaderParams& params, + ShaderRecompiler::CompileOptions options, + ShaderStageRuntime& stage) { + const auto* stage_name = ShaderStageName(options.stage); + const auto* label = ShaderStageLabel(options.stage); + ShaderRecompiler::CompileResult result; + std::string error; + if (!ShaderRecompiler::TryRecompile(params.code, options, result, &error)) { + ExitShaderRecompilerFailure(label, options.shader_hash, error.c_str()); + } + + DumpShaderRecompilerOriginal(stage_name, options.shader_hash, params.code, result.decoded_dump); + if (!SpirvValidateBinary(label, options.shader_hash, result.spirv)) { + DumpShaderRecompilerSpirv(stage_name, options.shader_hash, result.spirv); + ExitShaderRecompilerFailure(label, options.shader_hash, "SPIR-V validation failed"); + } + DumpShaderRecompilerSpirv(stage_name, options.shader_hash, result.spirv); + + stage.program = + std::make_shared(std::move(result.program)); + stage.resources = + std::make_shared(std::move(result.resources)); + + vk::ShaderModuleCreateInfo create_info {}; + create_info.sType = vk::StructureType::eShaderModuleCreateInfo; + create_info.codeSize = result.spirv.size() * sizeof(uint32_t); + create_info.pCode = result.spirv.data(); + vk::ShaderModule module = nullptr; + RequireVulkanSuccess(device.createShaderModule(&create_info, nullptr, &module), + "create recompiled shader module"); + EXIT_IF(module == nullptr); + + if (options.dump_ir) { + if (!options.early_dump) { + LOGF("%s decoded RDNA2:\n%s", label, result.decoded_dump.c_str()); + LOGF("%s IR:\n%s", label, result.ir_dump.c_str()); + } + LOGF("%s SPIR-V words=%" PRIu64 " wave_size=%u\n", label, + static_cast(result.spirv.size()), options.wave_size); + } + return module; +} + +vk::ShaderModule CompileProgram(vk::Device device, const ShaderParams& params, + ShaderVertexInputInfo& input_info) { KYTY_PROFILER_FUNCTION(profiler::colors::Amber300); - - EXIT_NOT_IMPLEMENTED(regs.es_regs.data_addr == 0 || regs.gs_regs.chksum == 0); - input_info.pa_cl_vs_out_cntl = sh.m_paClVsOutCntl; - - const uint64_t shader_addr = regs.es_regs.data_addr; - const auto code = ShaderGetMappedCode(shader_addr, "ShaderRecompiler VS", regs.gs_regs.chksum); - - ShaderRecompiler::CompileOptions options; - options.stage = ShaderType::Vertex; - options.shader_hash = regs.gs_regs.chksum; - options.shader_base = shader_addr; - options.user_data_base = 8; - options.user_data_count = regs.gs_regs.rsrc2.user_sgpr; - options.scratch_dwords = input_info.scratch_size_dwords; - options.user_data = regs.gs_user_sgpr.value; - options.read_specialization_memory = ReadShaderGuestMemory; - options.input_info.vertex = &input_info; - options.dump_ir = ShaderRecompilerTextDumpEnabled(); - options.early_dump = options.dump_ir; - options.dump_label = "ShaderRecompiler VS"; - - ShaderRecompiler::CompileResult result; - std::string error; - if (!ShaderRecompiler::TryRecompile(code, options, result, &error)) { - ExitShaderRecompilerFailure("ShaderRecompiler VS", options.shader_hash, error.c_str()); - } - DumpShaderRecompilerOriginal("vs", options.shader_hash, code, result.decoded_dump); - if (!SpirvValidateBinary("ShaderRecompiler VS", options.shader_hash, result.spirv)) { - DumpShaderRecompilerSpirv("vs", options.shader_hash, result.spirv); - ExitShaderRecompilerFailure("ShaderRecompiler VS", options.shader_hash, - "SPIR-V validation failed"); - } - - input_info.stage.program = - std::make_shared(std::move(result.program)); - input_info.stage.resources = - std::make_shared(std::move(result.resources)); + auto options = MakeCompileOptions(params, ShaderType::Vertex); + options.user_data_base = 8; + options.scratch_dwords = input_info.scratch_size_dwords; + options.input_info.vertex = &input_info; + const auto module = CompileModule(device, params, options, input_info.stage); ApplyVertexOutputs(input_info, *input_info.stage.program); - spirv = std::move(result.spirv); - DumpShaderRecompilerSpirv("vs", options.shader_hash, spirv); - - if (options.dump_ir) { - if (!options.early_dump) { - LOGF("ShaderRecompiler VS decoded RDNA2:\n%s", result.decoded_dump.c_str()); - LOGF("ShaderRecompiler VS IR:\n%s", result.ir_dump.c_str()); - } - LOGF("ShaderRecompiler VS SPIR-V words=%" PRIu64 "\n", static_cast(spirv.size())); - } - return true; + return module; } -bool ShaderCompileSpirvPS(const HW::PixelShaderInfo& regs, const HW::ShaderRegisters& sh, - ShaderPixelInputInfo& input_info, std::vector& spirv) { +vk::ShaderModule CompileProgram(vk::Device device, const ShaderParams& params, + ShaderPixelInputInfo& input_info) { KYTY_PROFILER_FUNCTION(profiler::colors::Blue300); - - const uint64_t shader_addr = regs.ps_regs.data_addr; - const uint64_t shader_hash = regs.ps_regs.chksum != 0 ? regs.ps_regs.chksum : shader_addr; - const auto code = ShaderGetMappedCode(shader_addr, "ShaderRecompiler PS", shader_hash); - - ShaderRecompiler::CompileOptions options; - options.stage = ShaderType::Pixel; - options.shader_hash = shader_hash; - options.shader_base = shader_addr; - options.user_data_count = regs.ps_regs.rsrc2.user_sgpr; - options.scratch_dwords = input_info.scratch_size_dwords; - options.push_constant_offset = input_info.push_constant_offset; - options.user_data = regs.ps_user_sgpr.value; - options.read_specialization_memory = ReadShaderGuestMemory; - options.input_info.pixel = &input_info; - options.dump_ir = ShaderRecompilerTextDumpEnabled(); - options.early_dump = options.dump_ir; - options.dump_label = "ShaderRecompiler PS"; - - ShaderRecompiler::CompileResult result; - std::string error; - if (!ShaderRecompiler::TryRecompile(code, options, result, &error)) { - ExitShaderRecompilerFailure("ShaderRecompiler PS", options.shader_hash, error.c_str()); - } - DumpShaderRecompilerOriginal("ps", options.shader_hash, code, result.decoded_dump); - if (!SpirvValidateBinary("ShaderRecompiler PS", options.shader_hash, result.spirv)) { - DumpShaderRecompilerSpirv("ps", options.shader_hash, result.spirv); - ExitShaderRecompilerFailure("ShaderRecompiler PS", options.shader_hash, - "SPIR-V validation failed"); - } - input_info.stage.program = - std::make_shared(std::move(result.program)); - input_info.stage.resources = - std::make_shared(std::move(result.resources)); + auto options = MakeCompileOptions(params, ShaderType::Pixel); + options.scratch_dwords = input_info.scratch_size_dwords; + options.push_constant_offset = input_info.push_constant_offset; + options.input_info.pixel = &input_info; + const auto module = CompileModule(device, params, options, input_info.stage); ApplyPixelOutputs(input_info, *input_info.stage.program); - spirv = std::move(result.spirv); - DumpShaderRecompilerSpirv("ps", options.shader_hash, spirv); - - if (options.dump_ir) { - if (!options.early_dump) { - LOGF("ShaderRecompiler PS decoded RDNA2:\n%s", result.decoded_dump.c_str()); - LOGF("ShaderRecompiler PS IR:\n%s", result.ir_dump.c_str()); - } - LOGF("ShaderRecompiler PS SPIR-V words=%" PRIu64 "\n", static_cast(spirv.size())); - } - return true; + return module; } -bool ShaderCompileSpirvCS(const HW::ComputeShaderInfo& regs, const HW::ShaderRegisters& sh, - ShaderComputeInputInfo& input_info, std::vector& spirv) { +vk::ShaderModule CompileProgram(vk::Device device, const ShaderParams& params, + ShaderComputeInputInfo& input_info) { KYTY_PROFILER_FUNCTION(profiler::colors::CyanA700); - - const uint64_t shader_addr = regs.cs_regs.data_addr; - const auto code = ShaderGetMappedCode(shader_addr, "ShaderRecompiler CS", shader_addr); - - ShaderRecompiler::CompileOptions options; - options.stage = ShaderType::Compute; - options.shader_hash = shader_addr; - options.shader_base = shader_addr; - options.user_data_count = regs.cs_regs.user_sgpr; - options.scratch_dwords = input_info.scratch_size_dwords; - options.user_data = regs.cs_user_sgpr.value; - options.read_specialization_memory = ReadShaderGuestMemory; - options.input_info.compute = &input_info; - options.wave_size = input_info.wave_size; - options.dump_ir = ShaderRecompilerTextDumpEnabled(); - options.early_dump = options.dump_ir; - options.dump_label = "ShaderRecompiler CS"; - - ShaderRecompiler::CompileResult result; - std::string error; - if (!ShaderRecompiler::TryRecompile(code, options, result, &error)) { - ExitShaderRecompilerFailure("ShaderRecompiler CS", options.shader_hash, error.c_str()); - } - DumpShaderRecompilerOriginal("cs", options.shader_hash, code, result.decoded_dump); - if (!SpirvValidateBinary("ShaderRecompiler CS", options.shader_hash, result.spirv)) { - DumpShaderRecompilerSpirv("cs", options.shader_hash, result.spirv); - ExitShaderRecompilerFailure("ShaderRecompiler CS", options.shader_hash, - "SPIR-V validation failed"); - } - input_info.stage.program = - std::make_shared(std::move(result.program)); - input_info.stage.resources = - std::make_shared(std::move(result.resources)); - spirv = std::move(result.spirv); - DumpShaderRecompilerSpirv("cs", options.shader_hash, spirv); - - if (options.dump_ir) { - if (!options.early_dump) { - LOGF("ShaderRecompiler CS decoded RDNA2:\n%s", result.decoded_dump.c_str()); - LOGF("ShaderRecompiler CS IR:\n%s", result.ir_dump.c_str()); - } - LOGF("ShaderRecompiler CS SPIR-V words=%" PRIu64 " wave_size=%u\n", - static_cast(spirv.size()), options.wave_size); - } - return true; -} - -ShaderId ShaderGetIdVS(const HW::VertexShaderInfo& regs, const ShaderVertexInputInfo& input_info, - bool include_bind_specialization) { - KYTY_PROFILER_FUNCTION(); - - ShaderId ret; - - ret.ids.reserve(64); - - EXIT_NOT_IMPLEMENTED(regs.es_regs.data_addr == 0 || regs.gs_regs.chksum == 0); - - ret.hash0 = (regs.gs_regs.chksum >> 32u) & 0xffffffffu; - ret.crc32 = regs.gs_regs.chksum & 0xffffffffu; - - ret.ids.push_back(static_cast(input_info.fetch_external)); - ret.ids.push_back(static_cast(input_info.fetch_embedded)); - ret.ids.push_back(input_info.resources_num); - ret.ids.push_back(input_info.export_count); - ret.ids.push_back(input_info.scratch_size_dwords); - ret.ids.push_back(input_info.pa_cl_vs_out_cntl); - - for (int i = 0; i < input_info.resources_num; i++) { - const auto& r = input_info.resources[i]; - const auto& rd = input_info.resources_dst[i]; - - ret.ids.push_back(rd.register_start); - ret.ids.push_back(rd.registers_num); - ret.ids.push_back(rd.fetch_index); - ret.ids.push_back(r.Stride()); - ret.ids.push_back(static_cast(r.SwizzleEnabled())); - ret.ids.push_back(r.DstSelX()); - ret.ids.push_back(r.DstSelY()); - ret.ids.push_back(r.DstSelZ()); - ret.ids.push_back(r.DstSelW()); - ret.ids.push_back(r.RawFormat()); - ret.ids.push_back(r.OutOfBounds()); - ret.ids.push_back(static_cast(r.AddTid())); - } - - ret.ids.push_back(input_info.buffers_num); - - for (int i = 0; i < input_info.buffers_num; i++) { - const auto& r = input_info.buffers[i]; - ret.ids.push_back(r.attr_num); - ret.ids.push_back(r.stride); - ret.ids.push_back(r.fetch_index); - for (int j = 0; j < r.attr_num; j++) { - ret.ids.push_back(r.attr_indices[j]); - ret.ids.push_back(r.attr_offsets[j]); - } - } - - if (include_bind_specialization) { - EXIT_IF(!input_info.stage); - ShaderAppendNativeSpecialization(ret.ids, *input_info.stage.program); - } - - return ret; -} - -ShaderId ShaderGetIdPS(const HW::PixelShaderInfo& regs, const ShaderPixelInputInfo& input_info, - bool include_bind_specialization) { - KYTY_PROFILER_FUNCTION(); - - ShaderId ret; - - ret.ids.reserve(64); - - ret.hash0 = (regs.ps_regs.chksum >> 32u) & 0xffffffffu; - ret.crc32 = regs.ps_regs.chksum & 0xffffffffu; - - ret.ids.push_back(input_info.push_constant_offset); - ret.ids.push_back(input_info.scratch_size_dwords); - ret.ids.push_back(input_info.input_num); - ret.ids.push_back(input_info.ps_system_input_base); - ret.ids.push_back(input_info.custom_interpolation_mask); - ret.ids.push_back(input_info.ps_perspective_center_vgpr); - ret.ids.push_back(static_cast(input_info.ps_pos_x)); - ret.ids.push_back(static_cast(input_info.ps_pos_y)); - ret.ids.push_back(static_cast(input_info.ps_pos_z)); - ret.ids.push_back(static_cast(input_info.ps_pos_w)); - ret.ids.push_back(static_cast(input_info.ps_front_face)); - ret.ids.push_back(static_cast(input_info.ps_no_perspective)); - ret.ids.push_back(static_cast(input_info.ps_pixel_kill_enable)); - ret.ids.push_back(static_cast(input_info.ps_depth_export_enable)); - ret.ids.push_back(static_cast(input_info.ps_sample_mask_export_enable)); - ret.ids.push_back(static_cast(input_info.ps_early_z)); - ret.ids.push_back(static_cast(input_info.ps_execute_on_noop)); - - for (auto mode: input_info.target_output_mode) { - ret.ids.push_back(mode); - } - for (uint32_t base = 0; base < input_info.target_export_mapping.size(); base += 4u) { - uint32_t packed = 0; - for (uint32_t i = 0; i < 4u; i++) { - packed |= static_cast(input_info.target_export_mapping[base + i].packed) - << (i * 8u); - } - ret.ids.push_back(packed); - } - ret.ids.push_back(input_info.mrt_output_mask); - - for (uint32_t i = 0; i < input_info.input_num; i++) { - ret.ids.push_back(input_info.interpolator_settings[i]); - } - - if (include_bind_specialization) { - EXIT_IF(!input_info.stage); - ShaderAppendNativeSpecialization(ret.ids, *input_info.stage.program); - } - - return ret; -} - -ShaderId ShaderGetIdCS(const HW::ComputeShaderInfo& regs, const ShaderComputeInputInfo& input_info, - bool include_bind_specialization) { - const auto* src = reinterpret_cast(regs.cs_regs.data_addr); - - EXIT_NOT_IMPLEMENTED(src == nullptr); - - const auto* header = GetBinaryInfo(src); - - ShaderId ret; - ret.ids.reserve(64); - - if (header != nullptr) { - ret.hash0 = header->hash0; - ret.crc32 = header->crc32; - ret.ids.push_back(header->length); - } else { - GetNextGenFallbackShaderId(regs.cs_regs.data_addr, ret.hash0, ret.crc32); - ret.ids.push_back(static_cast(regs.cs_regs.data_addr & 0xffffffffu)); - ret.ids.push_back(static_cast((regs.cs_regs.data_addr >> 32u) & 0xffffffffu)); - } - - ret.ids.push_back(input_info.workgroup_register); - ret.ids.push_back(input_info.wave_size); - ret.ids.push_back(input_info.thread_ids_num); - ret.ids.push_back(input_info.lds_size_dwords); - ret.ids.push_back(input_info.scratch_size_dwords); - ret.ids.push_back(static_cast(input_info.needs_lds_barriers)); - - for (int i = 0; i < 3; i++) { - ret.ids.push_back(input_info.threads_num[i]); - ret.ids.push_back(static_cast(input_info.group_id[i])); - } - ret.ids.push_back(static_cast(input_info.tg_size_en)); - - if (include_bind_specialization) { - EXIT_IF(!input_info.stage); - ShaderAppendNativeSpecialization(ret.ids, *input_info.stage.program); - } - - return ret; + auto options = MakeCompileOptions(params, ShaderType::Compute); + options.scratch_dwords = input_info.scratch_size_dwords; + options.wave_size = input_info.wave_size; + options.input_info.compute = &input_info; + return CompileModule(device, params, options, input_info.stage); } bool ShaderAddressValid(uint64_t addr) { diff --git a/src/graphics/shader/shader.h b/src/graphics/shader/shader.h index 14f91b152..06c5593ee 100644 --- a/src/graphics/shader/shader.h +++ b/src/graphics/shader/shader.h @@ -60,17 +60,6 @@ bool ShaderMaterializeStageRuntime( std::string* error, ShaderSpecializationMemoryReader read_specialization_memory = nullptr, void* read_memory_data = nullptr); -struct ShaderId { - uint32_t hash0 = 0; - uint32_t crc32 = 0; - std::vector ids; - - bool operator==(const ShaderId& other) const { - return hash0 == other.hash0 && crc32 == other.crc32 && ids == other.ids; - } - bool operator!=(const ShaderId& other) const { return !(*this == other); } -}; - constexpr uint32_t DstSel(uint32_t x, uint32_t y = 0, uint32_t z = 0, uint32_t w = 0) { return x | (y << 3u) | (z << 6u) | (w << 9u); } @@ -254,28 +243,6 @@ void ShaderMapUserData(uint64_t addr, const ShaderMappedData& data); void ShaderDbgDumpInputInfo(const ShaderVertexInputInfo& info); void ShaderDbgDumpInputInfo(const ShaderPixelInputInfo& info); void ShaderDbgDumpInputInfo(const ShaderComputeInputInfo& info); -ShaderId ShaderGetIdVS(const HW::VertexShaderInfo& regs, const ShaderVertexInputInfo& input_info, - bool include_bind_specialization); -ShaderId ShaderGetIdPS(const HW::PixelShaderInfo& regs, const ShaderPixelInputInfo& input_info, - bool include_bind_specialization); -ShaderId ShaderGetIdCS(const HW::ComputeShaderInfo& regs, const ShaderComputeInputInfo& input_info, - bool include_bind_specialization); -// Returned SPIR-V spans are read-only views backed by the shader program cache. -bool ShaderCompileInfoVS(const HW::VertexShaderInfo& regs, const HW::ShaderRegisters& sh, - ShaderVertexInputInfo& input_info, std::span& spirv); -bool ShaderCompileInfoPS(const HW::PixelShaderInfo& regs, const HW::ShaderRegisters& sh, - const ShaderVertexInputInfo& vs_info, - std::span target_export_mapping, - ShaderPixelInputInfo& input_info, std::span& spirv); -bool ShaderCompileInfoCS(const HW::ComputeShaderInfo& regs, const HW::ShaderRegisters& sh, - bool needs_lds_barriers, ShaderComputeInputInfo& input_info, - std::span& spirv); -bool ShaderCompileSpirvVS(const HW::VertexShaderInfo& regs, const HW::ShaderRegisters& sh, - ShaderVertexInputInfo& input_info, std::vector& spirv); -bool ShaderCompileSpirvPS(const HW::PixelShaderInfo& regs, const HW::ShaderRegisters& sh, - ShaderPixelInputInfo& input_info, std::vector& spirv); -bool ShaderCompileSpirvCS(const HW::ComputeShaderInfo& regs, const HW::ShaderRegisters& sh, - ShaderComputeInputInfo& input_info, std::vector& spirv); bool ShaderAddressValid(uint64_t addr); } // namespace Libs::Graphics diff --git a/src/graphics/shader/shaderCompiler.h b/src/graphics/shader/shaderCompiler.h new file mode 100644 index 000000000..559912d53 --- /dev/null +++ b/src/graphics/shader/shaderCompiler.h @@ -0,0 +1,53 @@ +#ifndef EMULATOR_INCLUDE_EMULATOR_GRAPHICS_SHADER_SHADERCOMPILER_H_ +#define EMULATOR_INCLUDE_EMULATOR_GRAPHICS_SHADER_SHADERCOMPILER_H_ + +#include "graphics/host_gpu/vulkanCommon.h" +#include "graphics/shader/shader.h" + +#include +#include +#include + +namespace Libs::Graphics { + +struct ShaderParams { + std::span code; + std::span user_data; + uint64_t hash = 0; + + [[nodiscard]] uint64_t Base() const { + return reinterpret_cast(code.data()); + } +}; + +void BuildStageStaticKey(const ShaderVertexInputInfo& input_info, std::vector& key); +void BuildStageStaticKey(const ShaderPixelInputInfo& input_info, std::vector& key); +void BuildStageStaticKey(const ShaderComputeInputInfo& input_info, std::vector& key); + +ShaderParams PrepareProgram(const HW::VertexShaderInfo& regs, const HW::ShaderRegisters& sh, + ShaderVertexInputInfo& input_info); +ShaderParams PrepareProgram( + const HW::PixelShaderInfo& regs, const HW::ShaderRegisters& sh, + const ShaderVertexInputInfo& vertex_info, + std::span target_export_mapping, + ShaderPixelInputInfo& input_info); +ShaderParams PrepareProgram(const HW::ComputeShaderInfo& regs, const HW::ShaderRegisters& sh, + ShaderComputeInputInfo& input_info); + +bool MaterializeProgram(const std::shared_ptr& program, + const ShaderParams& params, ShaderVertexInputInfo& input_info); +bool MaterializeProgram(const std::shared_ptr& program, + const ShaderParams& params, ShaderPixelInputInfo& input_info); +bool MaterializeProgram(const std::shared_ptr& program, + const ShaderParams& params, ShaderComputeInputInfo& input_info); + +vk::ShaderModule CompileProgram(vk::Device device, const ShaderParams& params, + ShaderVertexInputInfo& input_info); +vk::ShaderModule CompileProgram(vk::Device device, const ShaderParams& params, + ShaderPixelInputInfo& input_info); +vk::ShaderModule CompileProgram(vk::Device device, const ShaderParams& params, + ShaderComputeInputInfo& input_info); + +} // namespace Libs::Graphics + +#endif /* EMULATOR_INCLUDE_EMULATOR_GRAPHICS_SHADER_SHADERCOMPILER_H_ */ diff --git a/tests/ShaderRecompilerComputeTests.cpp b/tests/ShaderRecompilerComputeTests.cpp index 8cb1b4891..72e6d564f 100644 --- a/tests/ShaderRecompilerComputeTests.cpp +++ b/tests/ShaderRecompilerComputeTests.cpp @@ -42,6 +42,7 @@ #include "graphics/shader/recompiler/ir/passes/BindingLayout.h" #include "graphics/shader/rectListShader.h" #include "graphics/shader/shader.h" +#include "graphics/shader/shaderCompiler.h" #include "kernel/eventQueue.h" #include "kernel/memory.h" #include "libs/agc.h" @@ -1057,12 +1058,14 @@ void CheckRectListShaders() { pixel.interpolator_settings[0] = 0x400u; pixel.interpolator_settings[1] = 0; pixel.stage.program = program; - HW::PixelShaderInfo ps_regs{}; - const auto perspective_id = ShaderGetIdPS(ps_regs, pixel, false); + std::vector perspective_specialization; + BuildStageStaticKey(pixel, perspective_specialization); pixel.ps_no_perspective = true; - const auto no_perspective_id = ShaderGetIdPS(ps_regs, pixel, false); + std::vector no_perspective_specialization; + BuildStageStaticKey(pixel, no_perspective_specialization); pixel.ps_no_perspective = false; - Require(name, "pipeline identity", perspective_id != no_perspective_id, + Require(name, "pipeline identity", + perspective_specialization != no_perspective_specialization, "pixel interpolation mode must participate in the shader and " "pipeline key"); @@ -22508,83 +22511,12 @@ void CheckStorageTextureDepthTileUploadLayout() { "1x1 R8_UINT depth tile lost its 64 KiB source footprint"); std::printf("[host] %-32s ok\n", "StorageTextureDepthTileUpload"); } -void CheckImageSpecializationId() { - std::array code{}; - HW::ComputeShaderInfo regs{}; - regs.cs_regs.data_addr = reinterpret_cast(code.data()); - - ShaderRecompiler::IR::ImageResource image; - image.kind = ShaderRecompiler::IR::ResourceKind::StorageImage; - image.dimension = ShaderRecompiler::Decoder::ImageDimension::Dim2D; - const auto make_program = [](ShaderRecompiler::IR::ImageResource resource) { - ShaderRecompiler::IR::Program program; - program.binding_layout_complete = true; - program.info.images.push_back(std::move(resource)); - return program; - }; - auto rgb1_image = image; - rgb1_image.shader_swizzle = DstSel(4, 5, 6, 1); - auto converted_image = image; - converted_image.conversion_format = - Prospero::BufferFormat::k11_11_10UInt; - auto mip1_image = image; - mip1_image.mip_mode = ShaderRecompiler::IR::ImageMipMode::DynamicStorage; - auto mip3_image = mip1_image; - mip3_image.mip_count = 3; - auto array_image = image; - array_image.dimension = - ShaderRecompiler::Decoder::ImageDimension::Dim2DArray; - auto cube_image = array_image; - cube_image.cube = true; - - const auto resources = - std::make_shared(); - ShaderComputeInputInfo identity_info; - identity_info.stage.program = std::make_shared( - make_program(image)); - identity_info.stage.resources = resources; - auto rgb1_info = identity_info; - rgb1_info.stage.program = std::make_shared( - make_program(rgb1_image)); - auto converted_info = identity_info; - converted_info.stage.program = - std::make_shared( - make_program(converted_image)); - auto mip1_info = identity_info; - mip1_info.stage.program = std::make_shared( - make_program(mip1_image)); - auto mip3_info = identity_info; - mip3_info.stage.program = std::make_shared( - make_program(mip3_image)); - auto array_info = identity_info; - array_info.stage.program = std::make_shared( - make_program(array_image)); - auto cube_info = identity_info; - cube_info.stage.program = std::make_shared( - make_program(cube_image)); - - const auto identity_id = ShaderGetIdCS(regs, identity_info, true); - const auto rgb1_id = ShaderGetIdCS(regs, rgb1_info, true); - const auto converted_id = ShaderGetIdCS(regs, converted_info, true); - const auto mip1_id = ShaderGetIdCS(regs, mip1_info, true); - const auto mip3_id = ShaderGetIdCS(regs, mip3_info, true); - const auto array_id = ShaderGetIdCS(regs, array_info, true); - const auto cube_id = ShaderGetIdCS(regs, cube_info, true); - Require("ImageSpecializationId", "pipeline cache key", - identity_id != rgb1_id && - identity_id.ids.size() == rgb1_id.ids.size() && - identity_id != converted_id && - identity_id.ids.size() == converted_id.ids.size() && - mip1_id != mip3_id && mip1_id.ids.size() == mip3_id.ids.size() && - array_id != cube_id && array_id.ids.size() == cube_id.ids.size(), - "image conversion, swizzle, mip-count, or cube variants share a " - "pipeline ID"); - +void CheckImageSamplerSpecialization() { ShaderSamplerResource filtered_sampler{}; filtered_sampler.fields[2] = (1u << 20u) | (3u << 22u) | (2u << 24u) | (2u << 26u); filtered_sampler.SetPointFiltering(); - Require("ImageSpecializationId", "forced point sampler", + Require("ImageSamplerSpecialization", "forced point sampler", filtered_sampler.XyMagFilter() == 0u && filtered_sampler.XyMinFilter() == 0u && filtered_sampler.ZFilter() == 1u && @@ -22594,7 +22526,7 @@ void CheckImageSpecializationId() { ShaderSamplerResource base_mip_sampler{}; base_mip_sampler.fields[2] = (3u << 20u) | (1u << 22u) | (2u << 24u); base_mip_sampler.SetPointFiltering(); - Require("ImageSpecializationId", "base-mip point sampler", + Require("ImageSamplerSpecialization", "base-mip point sampler", base_mip_sampler.XyMagFilter() == 0u && base_mip_sampler.XyMinFilter() == 0u && base_mip_sampler.ZFilter() == 1u && @@ -22632,12 +22564,12 @@ void CheckImageSpecializationId() { sampler_descriptor.dword_count = 4; mixed_sampler_snapshot.samplers.push_back(sampler_descriptor); std::string specialization_error; - Require("ImageSpecializationId", "mixed sampler specialization", + Require("ImageSamplerSpecialization", "mixed sampler specialization", ShaderRecompiler::IR::SpecializeResources(mixed_sampler_program, mixed_sampler_snapshot, &specialization_error), specialization_error); - Require("ImageSpecializationId", "mixed sampler variant", + Require("ImageSamplerSpecialization", "mixed sampler variant", mixed_sampler_program.info.samplers.size() == 2u && !mixed_sampler_program.info.samplers[0].force_point_filtering && mixed_sampler_program.info.samplers[1].force_point_filtering && @@ -22658,7 +22590,7 @@ void CheckImageSpecializationId() { ShaderRecompiler::IR::ResourceSnapshot packed_atomic_snapshot; packed_atomic_snapshot.images.push_back(packed_image_descriptor); std::string atomic_error; - Require("ImageSpecializationId", "packed atomic rejection", + Require("ImageSamplerSpecialization", "packed atomic rejection", !ShaderRecompiler::IR::SpecializeResources( packed_atomic_program, packed_atomic_snapshot, &atomic_error) && atomic_error.find("unsupported format 34") != std::string::npos, @@ -24389,7 +24321,7 @@ int main(int argc, char **argv) { return 0; } if (argc == 2 && std::strcmp(argv[1], "--indirect-image-only") == 0) { - CheckImageSpecializationId(); + CheckImageSamplerSpecialization(); CheckIndirectImageKeySwitch(); return 0; } @@ -24431,7 +24363,7 @@ int main(int argc, char **argv) { CheckBasicStorageTextureDescriptor(); CheckStorageTextureLinearUploadLayout(); CheckStorageTextureDepthTileUploadLayout(); - CheckImageSpecializationId(); + CheckImageSamplerSpecialization(); CheckStandard64RenderTargetTileRoundTrip(); CheckStorageTextureVolumeUploadLayout(); CheckStorageTextureVolumeMipRegions(); diff --git a/tests/shaderCfgTests.cpp b/tests/shaderCfgTests.cpp index 19fdde892..070188c24 100644 --- a/tests/shaderCfgTests.cpp +++ b/tests/shaderCfgTests.cpp @@ -25,6 +25,7 @@ #include "graphics/shader/recompiler/ir/passes/SrtWalker.h" #include "graphics/shader/recompiler/ir/passes/SsaRewrite.h" #include "graphics/shader/shader.h" +#include "graphics/shader/shaderCompiler.h" #include "libs/agc.h" #include "spirv-tools/libspirv.hpp" @@ -76,6 +77,46 @@ ShaderRecompiler::CompileOptions MakeCompileOptions(ShaderType stage) { return options; } +bool ReadHostTestMemory(void *, uint64_t address, uint32_t *value) { + if (address == 0 || value == nullptr) { + return false; + } + std::memcpy(value, reinterpret_cast(address), sizeof(*value)); + return true; +} + +bool CompilePixelRuntime(const ShaderParams ¶ms, + ShaderPixelInputInfo &input_info, + std::string *error) { + auto options = MakeCompileOptions(ShaderType::Pixel); + options.shader_hash = params.hash; + options.shader_base = params.Base(); + options.user_data = params.user_data.data(); + options.user_data_count = static_cast(params.user_data.size()); + options.scratch_dwords = input_info.scratch_size_dwords; + options.push_constant_offset = input_info.push_constant_offset; + options.read_specialization_memory = ReadHostTestMemory; + options.input_info.pixel = &input_info; + ShaderRecompiler::CompileResult result; + if (!ShaderRecompiler::TryRecompile(params.code, options, result, error)) { + return false; + } + input_info.stage.program = + std::make_shared( + std::move(result.program)); + input_info.stage.resources = + std::make_shared( + std::move(result.resources)); + return true; +} + +template +std::vector MakeStageStaticKey(const InputInfo &input_info) { + std::vector key; + BuildStageStaticKey(input_info, key); + return key; +} + std::vector CfgInstructionCoverage(const ShaderRecompiler::CFG::Graph &graph, size_t instruction_count) { @@ -9107,7 +9148,7 @@ void TestNewShaderRecompilerAuxPositionExports() { ShaderVertexInputInfo key0{}; ShaderVertexInputInfo key1{}; key1.pa_cl_vs_out_cntl = 1; - Check(ShaderGetIdVS(regs, key0, false) != ShaderGetIdVS(regs, key1, false), + Check(MakeStageStaticKey(key0) != MakeStageStaticKey(key1), "PA_CL_VS_OUT_CNTL is absent from the vertex shader cache key"); } @@ -10513,8 +10554,7 @@ void TestRenderTargetReverseExportMapping() { reversed_info.target_export_mapping[0] = gr32.export_mapping; HW::PixelShaderInfo regs{}; - Check(ShaderGetIdPS(regs, identity_info, false) != - ShaderGetIdPS(regs, reversed_info, false), + Check(MakeStageStaticKey(identity_info) != MakeStageStaticKey(reversed_info), "pixel shader cache identity omitted the render-target export mapping"); regs.ps_regs.data_addr = reinterpret_cast(shader); @@ -10527,18 +10567,13 @@ void TestRenderTargetReverseExportMapping() { mappings[0] = gr32.export_mapping; ShaderPixelInputInfo compiled_info{}; ShaderVertexInputInfo vertex_info{}; - std::span compiled_spirv; - Check( - ShaderCompileInfoPS(regs, sh, vertex_info, mappings, compiled_info, - compiled_spirv) && - compiled_info.target_export_mapping[0].IsIdentity(), - "inactive reverse MRT mapping was not normalized out of the shader " + PrepareProgram(regs, sh, vertex_info, mappings, compiled_info); + Check(compiled_info.target_export_mapping[0].IsIdentity(), + "inactive reverse MRT mapping was not normalized out of the shader " "cache key"); sh.target_output_mode[0] = 4; - Check( - ShaderCompileInfoPS(regs, sh, vertex_info, mappings, compiled_info, - compiled_spirv) && - compiled_info.target_export_mapping[0] == gr32.export_mapping, + PrepareProgram(regs, sh, vertex_info, mappings, compiled_info); + Check(compiled_info.target_export_mapping[0] == gr32.export_mapping, "active reverse MRT mapping was lost before shader specialization"); } @@ -11517,15 +11552,12 @@ void TestNewShaderRecompilerPixelPipelineEntry() { ShaderMapUserData(regs.ps_regs.data_addr, mapped); HW::ShaderRegisters sh{}; + const std::array mappings{}; + ShaderVertexInputInfo vertex_info{}; ShaderPixelInputInfo input_info{}; - std::vector spirv; - Check(ShaderCompileSpirvPS(regs, sh, input_info, spirv), - "new pixel shader recompiler wrapper did not produce SPIR-V"); - Check(!spirv.empty(), - "new pixel shader recompiler wrapper returned empty SPIR-V"); - Check(spirv.front() == 0x07230203u, - "new pixel shader recompiler wrapper did not emit SPIR-V binary"); - CheckSpirvBinaryValidates(spirv); + const auto params = PrepareProgram(regs, sh, vertex_info, mappings, input_info); + Check(params.hash == regs.ps_regs.chksum && params.code.data() == shader, + "pixel shader program parameters lost source identity"); const uint32_t vcc_load_shader[] = { EncodeSMovB32(106, 27), EncodeSMovB32(107, 28), EncodeSmem0(0x02, 44, 53), @@ -11551,10 +11583,10 @@ void TestNewShaderRecompilerPixelPipelineEntry() { ShaderMapUserData(vcc_regs.ps_regs.data_addr, vcc_mapped); ShaderPixelInputInfo vcc_input{}; - std::vector vcc_spirv; - Check(ShaderCompileSpirvPS(vcc_regs, sh, vcc_input, vcc_spirv), - "VCC raw-load pointer was lost"); - CheckSpirvBinaryValidates(vcc_spirv); + const auto vcc_params = + PrepareProgram(vcc_regs, sh, vertex_info, mappings, vcc_input); + std::string error; + Check(CompilePixelRuntime(vcc_params, vcc_input, &error), error.c_str()); } void TestComputeLdsAllocationIdentity() { @@ -11577,19 +11609,28 @@ void TestComputeLdsAllocationIdentity() { uint32_t expected_dwords) { regs.cs_regs.lds_size = encoded_lds_size; ShaderComputeInputInfo input_info{}; - std::span spirv; - Check(ShaderCompileInfoCS(regs, sh, false, input_info, spirv), - "compute LDS allocation shader did not compile"); + const auto params = PrepareProgram(regs, sh, input_info); Check(input_info.lds_size_dwords == expected_dwords, "COMPUTE_PGM_RSRC2 LDS allocation units were not decoded"); - const std::vector binary(spirv.begin(), spirv.end()); - Check(MeasureSpirv(binary).workgroup_variables == 1u, + auto options = MakeCompileOptions(ShaderType::Compute); + options.shader_hash = params.hash; + options.shader_base = params.Base(); + options.user_data = params.user_data.data(); + options.user_data_count = static_cast(params.user_data.size()); + options.input_info.compute = &input_info; + options.wave_size = input_info.wave_size; + options.scratch_dwords = input_info.scratch_size_dwords; + ShaderRecompiler::CompileResult result; + std::string error; + Check(ShaderRecompiler::TryRecompile(shader, options, result, &error), + error.c_str()); + Check(MeasureSpirv(result.spirv).workgroup_variables == 1u, "LDS shader did not declare exactly one Workgroup variable"); - Check(SpirvArrayLengthCount(binary, expected_dwords) == 1u, + Check(SpirvArrayLengthCount(result.spirv, expected_dwords) == 1u, "SPIR-V workgroup array did not use the declared LDS allocation"); - Check(SpirvUnsignedLessThanBoundCount(binary, expected_dwords) == 1u, + Check(SpirvUnsignedLessThanBoundCount(result.spirv, expected_dwords) == 1u, "LDS bounds check did not use the declared allocation"); - CheckSpirvBinaryValidates(binary); + CheckSpirvBinaryValidates(result.spirv); return input_info; }; @@ -11602,21 +11643,19 @@ void TestComputeLdsAllocationIdentity() { }; const auto lds_1152 = compile(decode_lds_field(lds_1152_rsrc2), 1152u); const auto lds_896 = compile(decode_lds_field(lds_896_rsrc2), 896u); - Check(ShaderGetIdCS(regs, lds_1152, true) != - ShaderGetIdCS(regs, lds_896, true), + Check(MakeStageStaticKey(lds_1152) != MakeStageStaticKey(lds_896), "compute pipeline identity omitted the LDS allocation"); auto split_wave = lds_896; split_wave.needs_lds_barriers = true; - Check(ShaderGetIdCS(regs, lds_896, true) != - ShaderGetIdCS(regs, split_wave, true), + Check(MakeStageStaticKey(lds_896) != MakeStageStaticKey(split_wave), "compute shader identity omitted split-wave LDS synchronization"); auto tg_size_disabled = lds_896; auto tg_size_enabled = lds_896; tg_size_enabled.tg_size_en = true; - Check(ShaderGetIdCS(regs, tg_size_disabled, false) != - ShaderGetIdCS(regs, tg_size_enabled, false), + Check(MakeStageStaticKey(tg_size_disabled) != + MakeStageStaticKey(tg_size_enabled), "compute shader identity omitted TG_SIZE semantics"); auto dispatch_disabled = lds_896; @@ -11625,32 +11664,42 @@ void TestComputeLdsAllocationIdentity() { dispatch_disabled.dispatch_threads_num[2] = 1; auto dispatch_enabled = dispatch_disabled; dispatch_enabled.dispatch_thread_dimensions = true; - Check(ShaderGetIdCS(regs, dispatch_disabled, false) == - ShaderGetIdCS(regs, dispatch_enabled, false), - "runtime dispatch enable created a compute shader variant"); + Check(MakeStageStaticKey(dispatch_disabled) != + MakeStageStaticKey(dispatch_enabled), + "dispatch-dimension mode did not select a compute shader variant"); auto second_extent = dispatch_enabled; second_extent.dispatch_threads_num[0] = 1920; second_extent.dispatch_threads_num[1] = 1080; second_extent.dispatch_threads_num[2] = 4; - Check(ShaderGetIdCS(regs, dispatch_enabled, false) == - ShaderGetIdCS(regs, second_extent, false), + Check(MakeStageStaticKey(dispatch_enabled) == + MakeStageStaticKey(second_extent), "runtime dispatch extent created a compute shader variant"); mapped.scratch_size_dwords = 7; ShaderMapUserData(regs.cs_regs.data_addr, mapped); regs.cs_regs.lds_size = decode_lds_field(lds_896_rsrc2); ShaderComputeInputInfo scratch_info{}; - std::span scratch_spirv; - Check(ShaderCompileInfoCS(regs, sh, false, scratch_info, scratch_spirv), - "compute scratch metadata shader did not compile"); + const auto scratch_params = PrepareProgram(regs, sh, scratch_info); Check(scratch_info.scratch_size_dwords == 7, "AGC per-thread scratch size was not propagated"); - Check(scratch_info.stage.program != nullptr && - scratch_info.stage.program->scratch_dwords == 7, + auto scratch_options = MakeCompileOptions(ShaderType::Compute); + scratch_options.shader_hash = scratch_params.hash; + scratch_options.shader_base = scratch_params.Base(); + scratch_options.user_data = scratch_params.user_data.data(); + scratch_options.user_data_count = + static_cast(scratch_params.user_data.size()); + scratch_options.input_info.compute = &scratch_info; + scratch_options.wave_size = scratch_info.wave_size; + scratch_options.scratch_dwords = scratch_info.scratch_size_dwords; + ShaderRecompiler::CompileResult scratch_result; + std::string scratch_error; + Check(ShaderRecompiler::TryRecompile(scratch_params.code, scratch_options, + scratch_result, &scratch_error), + scratch_error.c_str()); + Check(scratch_result.program.scratch_dwords == 7, "AGC per-thread scratch size did not reach the compiler program"); - Check(ShaderGetIdCS(regs, scratch_info, true) != - ShaderGetIdCS(regs, lds_896, true), + Check(MakeStageStaticKey(scratch_info) != MakeStageStaticKey(lds_896), "compute pipeline identity omitted the scratch allocation"); const uint32_t append_shader[] = { @@ -11810,20 +11859,18 @@ void TestPixelProgramCacheBindingIdentity() { const uint32_t shader_10[] = {0xbf810000u}; HW::ShaderRegisters sh{}; - HW::PixelShaderInfo identity_regs{}; ShaderPixelInputInfo no_depth_export{}; auto depth_export = no_depth_export; depth_export.ps_depth_export_enable = true; - Check(ShaderGetIdPS(identity_regs, no_depth_export, false) != - ShaderGetIdPS(identity_regs, depth_export, false), + Check(MakeStageStaticKey(no_depth_export) != MakeStageStaticKey(depth_export), "pixel shader identity omitted depth-export semantics"); auto shifted_push = no_depth_export; shifted_push.push_constant_offset = 36; - Check(ShaderGetIdPS(identity_regs, no_depth_export, false) != - ShaderGetIdPS(identity_regs, shifted_push, false), + Check(MakeStageStaticKey(no_depth_export) != MakeStageStaticKey(shifted_push), "pixel shader identity omitted graphics push-bank placement"); - auto check_cache_identity = [&](const uint32_t *shader, uint64_t checksum) { + auto check_program_identity = [&](const uint32_t *shader, + uint64_t checksum) { HW::PixelShaderInfo regs{}; regs.ps_regs.data_addr = reinterpret_cast(shader); regs.ps_regs.chksum = checksum; @@ -11834,26 +11881,29 @@ void TestPixelProgramCacheBindingIdentity() { const std::array identity_mappings{}; ShaderVertexInputInfo vertex_info{}; ShaderPixelInputInfo first_info{}; - std::span first_spirv; - Check(ShaderCompileInfoPS(regs, sh, vertex_info, identity_mappings, first_info, - first_spirv), - "pixel program-cache fixture failed to compile"); + const auto first_params = + PrepareProgram(regs, sh, vertex_info, identity_mappings, first_info); + const auto first_key = MakeStageStaticKey(first_info); + std::string error; + Check(CompilePixelRuntime(first_params, first_info, &error), error.c_str()); ShaderPixelInputInfo second_info{}; - std::span second_spirv; - Check(ShaderCompileInfoPS(regs, sh, vertex_info, identity_mappings, second_info, - second_spirv), - "pixel program-cache lookup failed"); + const auto second_params = + PrepareProgram(regs, sh, vertex_info, identity_mappings, second_info); Check( - first_info.stage.program != nullptr && - second_info.stage.program == first_info.stage.program && - second_spirv.data() == first_spirv.data() && - second_spirv.size() == first_spirv.size(), - "pixel program cache did not reuse an identical graphics placement"); + first_params.hash == second_params.hash && + first_key == MakeStageStaticKey(second_info) && + first_info.stage.program != nullptr && + MaterializeProgram(first_info.stage.program, second_params, second_info) && + second_info.stage.program == first_info.stage.program, + "pixel program matching did not reuse an identical graphics placement"); + return std::pair {first_params.hash, first_key}; }; - check_cache_identity(shader_01, 0x91a27e6300000001ull); - check_cache_identity(shader_10, 0x91a27e6300000002ull); + const auto request_01 = check_program_identity(shader_01, 0); + const auto request_10 = check_program_identity(shader_10, 0); + Check(request_01 == request_10, + "relocated identical pixel programs did not share their source identity"); const uint32_t push_shader[] = { EncodeVop1(0x01, 0, 0), EncodeVop1(0x01, 1, 1), @@ -11883,24 +11933,36 @@ void TestPixelProgramCacheBindingIdentity() { ShaderPixelInputInfo at_zero{}; ShaderPixelInputInfo at_36{}; ShaderPixelInputInfo at_zero_again{}; - std::span spirv_at_zero; - std::span spirv_at_36; - std::span spirv_at_zero_again; - Check(ShaderCompileInfoPS(push_regs, push_sh, vertex_at_zero, mappings, - at_zero, spirv_at_zero) && - ShaderCompileInfoPS(push_regs, push_sh, vertex_at_36, mappings, - at_36, spirv_at_36) && - ShaderCompileInfoPS(push_regs, push_sh, vertex_at_zero, mappings, - at_zero_again, spirv_at_zero_again), - "pixel program-cache placement fixture failed to compile"); + const auto params_at_zero = + PrepareProgram(push_regs, push_sh, vertex_at_zero, mappings, at_zero); + const auto key_at_zero = MakeStageStaticKey(at_zero); + std::string error; + Check(CompilePixelRuntime(params_at_zero, at_zero, &error), error.c_str()); + const auto params_at_36 = + PrepareProgram(push_regs, push_sh, vertex_at_36, mappings, at_36); + const auto key_at_36 = MakeStageStaticKey(at_36); + Check(CompilePixelRuntime(params_at_36, at_36, &error), error.c_str()); + const auto params_at_zero_again = + PrepareProgram(push_regs, push_sh, vertex_at_zero, mappings, + at_zero_again); + auto shifted_options = MakeCompileOptions(ShaderType::Pixel); + shifted_options.input_info.pixel = &at_36; + shifted_options.push_constant_offset = at_36.push_constant_offset; + ShaderRecompiler::CompileResult shifted_result; + std::string shifted_error; + Check(ShaderRecompiler::TryRecompile(push_shader, shifted_options, + shifted_result, &shifted_error), + shifted_error.c_str()); Check(at_zero.push_constant_offset == 0 && at_36.push_constant_offset == 36 && + key_at_zero != key_at_36 && + MakeStageStaticKey(at_zero_again) == key_at_zero && at_zero.stage.program != at_36.stage.program && - spirv_at_zero.data() != spirv_at_36.data() && + MaterializeProgram(at_zero.stage.program, params_at_zero_again, + at_zero_again) && at_zero_again.stage.program == at_zero.stage.program && - spirv_at_zero_again.data() == spirv_at_zero.data() && - SpirvHasMemberDecorationValue(spirv_at_36, 35u, 36u), - "pixel program cache did not partition and reuse graphics push placement"); + SpirvHasMemberDecorationValue(shifted_result.spirv, 35u, 36u), + "pixel program matching did not partition and reuse graphics push placement"); } void TestGraphicsPushConstantPlacement() {