From 3213934d423f78dd5d97ee71f5882d1bebfa34e9 Mon Sep 17 00:00:00 2001 From: nmzik Date: Sat, 5 Sep 2026 08:32:23 +0400 Subject: [PATCH] shader: run merged geometry stages as mesh shaders Co-authored-by: Brandon Strong Co-authored-by: fxpw Co-authored-by: shadowbeat070 <70843316+shadowbeat070@users.noreply.github.com> --- src/graphics/host_gpu/graphicContext.h | 2 + .../renderer/pipeline/descriptors.cpp | 16 +- .../renderer/pipeline/pipelineCache.cpp | 151 +++++----- .../renderer/pipeline/pipelineCache.h | 13 +- .../pipeline/shaderResourceBarrier.cpp | 15 +- .../host_gpu/renderer/pipeline/shaders.cpp | 20 +- src/graphics/host_gpu/renderer/renderDraw.cpp | 127 ++++---- .../presentation/window/vulkanWindow.cpp | 26 +- .../shader/recompiler/ShaderRecompiler.cpp | 64 +++- .../shader/recompiler/ShaderRecompiler.h | 1 + .../recompiler/backend/spirv/SpirvBuilder.cpp | 13 + .../recompiler/backend/spirv/SpirvEmitter.cpp | 13 +- .../backend/spirv/spirvEmitterAnalysis.cpp | 1 + .../backend/spirv/spirvEmitterFlow.cpp | 160 ++++------ .../backend/spirv/spirvEmitterHelpers.cpp | 7 +- .../backend/spirv/spirvEmitterInternal.h | 26 +- .../backend/spirv/spirvEmitterMemory.cpp | 28 +- .../spirv/spirvEmitterMemoryHelpers.cpp | 14 +- .../backend/spirv/spirvEmitterMesh.cpp | 186 ++++++++++++ .../backend/spirv/spirvEmitterModule.cpp | 65 +++- .../backend/spirv/spirvEmitterProgram.cpp | 225 +++++++++++--- .../recompiler/frontend/translate/Control.cpp | 63 ++-- .../recompiler/frontend/translate/Integer.cpp | 11 +- .../recompiler/frontend/translate/Scalar.cpp | 2 +- .../frontend/translate/Translate.cpp | 168 +++++++---- .../frontend/translate/Translator.h | 8 +- src/graphics/shader/recompiler/ir/ShaderIR.h | 1 + .../recompiler/ir/opcodes/ValueOpcodes.cpp | 1 + .../recompiler/ir/opcodes/ValueOpcodes.inc | 3 +- .../ir/passes/ShaderInfoCollection.cpp | 2 + .../ir/passes/SharedMemoryBarrier.cpp | 153 ---------- .../ir/passes/SharedMemoryBarrier.h | 15 - src/graphics/shader/shader.cpp | 100 +++++-- src/graphics/shader/shader.h | 36 ++- src/graphics/shader/shaderCompiler.h | 10 +- tests/ResourceTrackingTests.cpp | 6 +- tests/ScalarProvenanceTests.cpp | 3 +- tests/ShaderRecompilerComputeTests.cpp | 278 ++++++++++++++++-- tests/shaderCfgTests.cpp | 228 +++++--------- 39 files changed, 1403 insertions(+), 858 deletions(-) create mode 100644 src/graphics/shader/recompiler/backend/spirv/spirvEmitterMesh.cpp delete mode 100644 src/graphics/shader/recompiler/ir/passes/SharedMemoryBarrier.cpp delete mode 100644 src/graphics/shader/recompiler/ir/passes/SharedMemoryBarrier.h diff --git a/src/graphics/host_gpu/graphicContext.h b/src/graphics/host_gpu/graphicContext.h index 9a47e78b9..5f271f48b 100644 --- a/src/graphics/host_gpu/graphicContext.h +++ b/src/graphics/host_gpu/graphicContext.h @@ -33,6 +33,8 @@ struct GraphicContext { bool compute_subgroup_size_control_enabled = false; bool compute_wave64_supported = false; bool sample_rate_shading_enabled = false; + bool mesh_shader_enabled = false; + vk::PhysicalDeviceMeshShaderPropertiesEXT mesh_shader_properties = {}; uint32_t subgroup_size = 0; uint32_t min_subgroup_size = 0; uint32_t max_subgroup_size = 0; diff --git a/src/graphics/host_gpu/renderer/pipeline/descriptors.cpp b/src/graphics/host_gpu/renderer/pipeline/descriptors.cpp index ba81551e0..5d81a41ca 100644 --- a/src/graphics/host_gpu/renderer/pipeline/descriptors.cpp +++ b/src/graphics/host_gpu/renderer/pipeline/descriptors.cpp @@ -87,6 +87,7 @@ vk::DescriptorImageInfo MakeImageInfo(const TextureBinding& texture, uint32_t el static const char* ShaderStageResourceName(ShaderType stage) { switch (stage) { case ShaderType::Vertex: return "Vertex"; + case ShaderType::Mesh: return "Mesh"; case ShaderType::Pixel: return "Pixel"; case ShaderType::Compute: return "Compute"; default: return "Unknown"; @@ -96,6 +97,7 @@ static const char* ShaderStageResourceName(ShaderType stage) { static vk::ShaderStageFlags NativeShaderStage(ShaderType stage) { switch (stage) { case ShaderType::Vertex: return vk::ShaderStageFlagBits::eVertex; + case ShaderType::Mesh: return vk::ShaderStageFlagBits::eMeshEXT; case ShaderType::Pixel: return vk::ShaderStageFlagBits::eFragment; case ShaderType::Compute: return vk::ShaderStageFlagBits::eCompute; default: EXIT("unknown native shader stage\n"); @@ -1034,8 +1036,12 @@ void RenderExecutor::CommitBindings(CommandBuffer& buffer, size_t write_count = 0; ShaderRecompiler::IR::PushData push_data; bool has_push_data = false; - constexpr auto GraphicsStages = - vk::ShaderStageFlagBits::eVertex | vk::ShaderStageFlagBits::eFragment; + constexpr auto GraphicsStages = vk::ShaderStageFlagBits::eVertex | + vk::ShaderStageFlagBits::eMeshEXT | + vk::ShaderStageFlagBits::eFragment; + vk::ShaderStageFlags push_stages = pipeline_bind_point == vk::PipelineBindPoint::eGraphics + ? vk::ShaderStageFlagBits::eFragment + : vk::ShaderStageFlags {}; for (const auto* prepared: prepared_bindings) { EXIT_IF(prepared == nullptr || prepared->program == nullptr || prepared->snapshot == nullptr); @@ -1044,6 +1050,7 @@ void RenderExecutor::CommitBindings(CommandBuffer& buffer, descriptor_count += NativeDescriptorCount(binding); } const auto shader_stage = NativeShaderStage(prepared->program->stage); + push_stages |= shader_stage; EXIT_IF((pipeline_bind_point == vk::PipelineBindPoint::eGraphics && (shader_stage & GraphicsStages) == vk::ShaderStageFlags {}) || (pipeline_bind_point == vk::PipelineBindPoint::eCompute && @@ -1190,10 +1197,7 @@ void RenderExecutor::CommitBindings(CommandBuffer& buffer, } if (has_push_data) { - const auto stages = pipeline_bind_point == vk::PipelineBindPoint::eGraphics - ? vk::ShaderStageFlags {GraphicsStages} - : vk::ShaderStageFlags {vk::ShaderStageFlagBits::eCompute}; - vk_buffer.pushConstants(pipeline.pipeline_layout, stages, 0, sizeof(push_data), + vk_buffer.pushConstants(pipeline.pipeline_layout, push_stages, 0, sizeof(push_data), push_data.dwords.data()); } diff --git a/src/graphics/host_gpu/renderer/pipeline/pipelineCache.cpp b/src/graphics/host_gpu/renderer/pipeline/pipelineCache.cpp index 223862f9b..c9fedbfdc 100644 --- a/src/graphics/host_gpu/renderer/pipeline/pipelineCache.cpp +++ b/src/graphics/host_gpu/renderer/pipeline/pipelineCache.cpp @@ -204,22 +204,19 @@ struct PipelineCache::ProgramCache { static constexpr std::size_t MaxStaticKeyWords = 13 + ShaderVertexInputInfo::RES_MAX * 13; - template Permutation CompilePermutation(const ShaderParams& params, const ShaderRecompiler::CompileOptions& options, ShaderRecompiler::TranslateResult translated, ShaderRecompiler::IR::ResourceSpecialization specialization, uint32_t push_data_start_dword) { - constexpr const char* stage_name = [] { - if constexpr (Stage == ShaderType::Vertex) { - return "vs"; - } else if constexpr (Stage == ShaderType::Pixel) { - return "ps"; - } else { - static_assert(Stage == ShaderType::Compute); - return "cs"; - } - }(); + const char* stage_name = nullptr; + switch (options.stage) { + case ShaderType::Vertex: stage_name = "vs"; break; + case ShaderType::Mesh: stage_name = "ms"; break; + case ShaderType::Pixel: stage_name = "ps"; break; + case ShaderType::Compute: stage_name = "cs"; break; + default: EXIT("invalid pipeline shader stage\n"); + } auto result = ShaderRecompiler::CompileProgram(std::move(translated), options, specialization, push_data_start_dword); DumpShaderOriginal(stage_name, options.shader_hash, params.code, result.decoded_dump); @@ -256,16 +253,15 @@ struct PipelineCache::ProgramCache { template ShaderProgram Get(const ShaderParams& params, InputInfo& input_info, uint32_t& push_data_cursor) { - constexpr ShaderType stage = [] { - if constexpr (std::is_same_v) { - return ShaderType::Vertex; - } else if constexpr (std::is_same_v) { - return ShaderType::Pixel; - } else { - static_assert(std::is_same_v); - return ShaderType::Compute; - } - }(); + ShaderType stage; + if constexpr (std::is_same_v) { + stage = input_info.mesh.threads_num[0] != 0 ? ShaderType::Mesh : ShaderType::Vertex; + } else if constexpr (std::is_same_v) { + stage = ShaderType::Pixel; + } else { + static_assert(std::is_same_v); + stage = ShaderType::Compute; + } lookup_key.stage = stage; lookup_key.hash = params.hash; @@ -307,31 +303,32 @@ struct PipelineCache::ProgramCache { } else { stage_input.compute = &input_info; } - constexpr const char* label = [] { - if constexpr (stage == ShaderType::Vertex) { - return "ShaderRecompiler VS"; - } else if constexpr (stage == ShaderType::Pixel) { - return "ShaderRecompiler PS"; - } else { - return "ShaderRecompiler CS"; - } - }(); + const char* label = nullptr; + switch (stage) { + case ShaderType::Vertex: label = "ShaderRecompiler VS"; break; + case ShaderType::Mesh: label = "ShaderRecompiler MS"; break; + case ShaderType::Pixel: label = "ShaderRecompiler PS"; break; + case ShaderType::Compute: label = "ShaderRecompiler CS"; break; + default: EXIT("invalid pipeline shader stage\n"); + } ShaderRecompiler::CompileOptions options; options.stage = stage; options.shader_hash = params.hash; options.user_data = params.user_data; + options.back_code = params.back_code; options.dump_ir = Config::GetShaderLogDirection() != Config::ShaderLogDirection::Silent; options.early_dump = options.dump_ir; options.dump_label = label; options.input_info = stage_input; - if constexpr (stage == ShaderType::Vertex) { + options.scratch_dwords = input_info.scratch_size_dwords; + if constexpr (std::is_same_v) { options.user_data_base = 8; - options.scratch_dwords = input_info.scratch_size_dwords; - } else if constexpr (stage == ShaderType::Pixel) { - options.scratch_dwords = input_info.scratch_size_dwords; - } else { - options.scratch_dwords = input_info.scratch_size_dwords; - options.wave_size = input_info.wave_size; + if (stage == ShaderType::Mesh) { + options.wave_size = input_info.mesh.wave_size; + options.scratch_dwords = input_info.mesh.scratch_size_dwords; + } + } else if constexpr (std::is_same_v) { + options.wave_size = input_info.wave_size; } auto translated = ShaderRecompiler::TranslateProgram(params.code, options); if (entry == programs.end()) { @@ -340,7 +337,7 @@ struct PipelineCache::ProgramCache { specialization)); entry = programs.try_emplace(lookup_key, std::move(resource_plan)).first; } - entry->second.permutations.push_back(CompilePermutation( + entry->second.permutations.push_back(CompilePermutation( params, options, std::move(translated), std::move(specialization), push_data_cursor)); const auto& permutation = entry->second.permutations.back(); input_info.stage = {.program = &permutation.program, .resources = std::move(resources)}; @@ -540,10 +537,29 @@ void PipelineCache::Save() { PipelineCache::GraphicsPrograms PipelineCache::GetGraphicsPrograms( const HW::VertexShaderInfo& vertex_regs, const HW::PixelShaderInfo& pixel_regs, - const HW::ShaderRegisters& sh, const HW::Context& context, - std::span target_export_mapping, - bool pixel_active, ShaderVertexInputInfo& vertex_info, ShaderPixelInputInfo& pixel_info) { - const auto vertex_params = PrepareProgram(vertex_regs, sh, vertex_info); + const HW::ShaderRegisters& sh, const HW::Context& context, const HW::UserConfig& user_config, + std::span target_export_mapping, bool pixel_active, + ShaderVertexInputInfo& vertex_info, ShaderPixelInputInfo& pixel_info) { + const auto vertex_params = PrepareProgram(vertex_regs, context, user_config, vertex_info); + const bool mesh_active = vertex_info.mesh.threads_num[0] != 0; + if (mesh_active) { + EXIT_NOT_IMPLEMENTED(!m_graphics.mesh_shader_enabled); + auto& mesh = vertex_info.mesh; + mesh.host_subgroup_size = m_graphics.subgroup_size; + const auto& limits = m_graphics.mesh_shader_properties; + const auto logical_threads = + mesh.threads_num[0] * mesh.threads_num[1] * mesh.threads_num[2]; + const auto host_threads = ((logical_threads + mesh.wave_size - 1u) / mesh.wave_size) * + std::min(mesh.host_subgroup_size, mesh.wave_size); + if (host_threads > limits.maxMeshWorkGroupInvocations || + host_threads > limits.maxMeshWorkGroupSize[0] || + mesh.max_vertices > limits.maxMeshOutputVertices || + mesh.max_primitives > limits.maxMeshOutputPrimitives || + mesh.lds_size_dwords * sizeof(uint32_t) > limits.maxMeshSharedMemorySize) { + EXIT("mesh shader exceeds host limits: threads=%u vertices=%u primitives=%u LDS=%u\n", + host_threads, mesh.max_vertices, mesh.max_primitives, mesh.lds_size_dwords); + } + } ShaderParams pixel_params; if (pixel_active) { pixel_params = PrepareProgram(pixel_regs, sh, target_export_mapping, pixel_info); @@ -563,7 +579,8 @@ PipelineCache::GraphicsPrograms PipelineCache::GetGraphicsPrograms( clip.enabled = true; } Common::LockGuard lock(m_mutex); - uint32_t push_data_cursor = 0; + uint32_t push_data_cursor = + mesh_active ? ShaderRecompiler::IR::PushData::MeshDrawDwordCount : 0; GraphicsPrograms result; if (pixel_active) { result.pixel = m_program_cache->Get(pixel_params, pixel_info, push_data_cursor); @@ -575,7 +592,7 @@ PipelineCache::GraphicsPrograms PipelineCache::GetGraphicsPrograms( ShaderProgram PipelineCache::GetComputeProgram(const HW::ComputeShaderInfo& regs, const HW::ShaderRegisters& sh, ShaderComputeInputInfo& input_info) { - input_info.needs_lds_barriers = !m_graphics.compute_wave64_supported; + input_info.host_subgroup_size = m_graphics.compute_wave64_supported ? 64u : 32u; const auto params = PrepareProgram(regs, sh, input_info); Common::LockGuard lock(m_mutex); uint32_t push_data_cursor = 0; @@ -709,30 +726,32 @@ PipelineCache::GraphicsPipeline& PipelineCache::CreateGraphicsPipeline( key.vs_shader_id = p.vs_shader_id; key.ps_shader_id = p.ps_shader_id; key.static_params = static_params; - EXIT_IF(vs_input_info.buffers_num < 0 || - vs_input_info.buffers_num > ShaderVertexInputInfo::RES_MAX || - vs_input_info.resources_num < 0 || - vs_input_info.resources_num > ShaderVertexInputInfo::RES_MAX); - key.vertex_input.binding_count = static_cast(vs_input_info.buffers_num); - key.vertex_input.attribute_count = static_cast(vs_input_info.resources_num); - uint32_t attributes_num = 0; - for (int binding = 0; binding < vs_input_info.buffers_num; binding++) { - const auto& buffer = vs_input_info.buffers[binding]; - EXIT_IF(buffer.attr_num < 0 || buffer.attr_num > ShaderVertexInputBuffer::ATTR_MAX); - attributes_num += static_cast(buffer.attr_num); - EXIT_IF(attributes_num > static_cast(vs_input_info.resources_num)); - key.vertex_input.bindings[binding] = {.stride = buffer.stride, - .instance = buffer.fetch_index != 0}; - for (int attribute = 0; attribute < buffer.attr_num; attribute++) { - const auto index = buffer.attr_indices[attribute]; - EXIT_IF(index < 0 || index >= vs_input_info.resources_num); - key.vertex_input.attributes[index] = { - .offset = buffer.attr_offsets[attribute], - .binding = static_cast(binding), - }; + if (vs_input_info.stage.program->stage != ShaderType::Mesh) { + EXIT_IF(vs_input_info.buffers_num < 0 || + vs_input_info.buffers_num > ShaderVertexInputInfo::RES_MAX || + vs_input_info.resources_num < 0 || + vs_input_info.resources_num > ShaderVertexInputInfo::RES_MAX); + key.vertex_input.binding_count = static_cast(vs_input_info.buffers_num); + key.vertex_input.attribute_count = static_cast(vs_input_info.resources_num); + uint32_t attributes_num = 0; + for (int binding = 0; binding < vs_input_info.buffers_num; binding++) { + const auto& buffer = vs_input_info.buffers[binding]; + EXIT_IF(buffer.attr_num < 0 || buffer.attr_num > ShaderVertexInputBuffer::ATTR_MAX); + attributes_num += static_cast(buffer.attr_num); + EXIT_IF(attributes_num > static_cast(vs_input_info.resources_num)); + key.vertex_input.bindings[binding] = {.stride = buffer.stride, + .instance = buffer.fetch_index != 0}; + for (int attribute = 0; attribute < buffer.attr_num; attribute++) { + const auto index = buffer.attr_indices[attribute]; + EXIT_IF(index < 0 || index >= vs_input_info.resources_num); + key.vertex_input.attributes[index] = { + .offset = buffer.attr_offsets[attribute], + .binding = static_cast(binding), + }; + } } + EXIT_IF(attributes_num != static_cast(vs_input_info.resources_num)); } - EXIT_IF(attributes_num != static_cast(vs_input_info.resources_num)); if (auto iter = m_graphics_pipelines.find(key); iter != m_graphics_pipelines.end()) { return *iter->second; diff --git a/src/graphics/host_gpu/renderer/pipeline/pipelineCache.h b/src/graphics/host_gpu/renderer/pipeline/pipelineCache.h index 9530cea5c..beec46b46 100644 --- a/src/graphics/host_gpu/renderer/pipeline/pipelineCache.h +++ b/src/graphics/host_gpu/renderer/pipeline/pipelineCache.h @@ -26,6 +26,7 @@ class CommandBuffer; namespace HW { class Context; class Shader; +class UserConfig; struct ComputeShaderInfo; } // namespace HW @@ -134,11 +135,13 @@ public: ShaderProgram pixel; }; - GraphicsPrograms GetGraphicsPrograms( - const HW::VertexShaderInfo& vertex_regs, const HW::PixelShaderInfo& pixel_regs, - const HW::ShaderRegisters& sh, const HW::Context& context, - std::span target_export_mapping, - bool pixel_active, ShaderVertexInputInfo& vertex_info, ShaderPixelInputInfo& pixel_info); + GraphicsPrograms + GetGraphicsPrograms(const HW::VertexShaderInfo& vertex_regs, + const HW::PixelShaderInfo& pixel_regs, const HW::ShaderRegisters& sh, + const HW::Context& context, const HW::UserConfig& user_config, + std::span target_export_mapping, + bool pixel_active, ShaderVertexInputInfo& vertex_info, + ShaderPixelInputInfo& pixel_info); ShaderProgram GetComputeProgram(const HW::ComputeShaderInfo& regs, const HW::ShaderRegisters& sh, ShaderComputeInputInfo& input_info); diff --git a/src/graphics/host_gpu/renderer/pipeline/shaderResourceBarrier.cpp b/src/graphics/host_gpu/renderer/pipeline/shaderResourceBarrier.cpp index 5c3a237b0..7f52dcdea 100644 --- a/src/graphics/host_gpu/renderer/pipeline/shaderResourceBarrier.cpp +++ b/src/graphics/host_gpu/renderer/pipeline/shaderResourceBarrier.cpp @@ -13,6 +13,9 @@ vk::PipelineStageFlags ShaderPipelineStages(vk::ShaderStageFlags stages) { if (stages & vk::ShaderStageFlagBits::eVertex) { result |= vk::PipelineStageFlagBits::eVertexShader; } + if (stages & vk::ShaderStageFlagBits::eMeshEXT) { + result |= vk::PipelineStageFlagBits::eMeshShaderEXT; + } if (stages & vk::ShaderStageFlagBits::eFragment) { result |= vk::PipelineStageFlagBits::eFragmentShader; } @@ -118,13 +121,11 @@ void ShaderWriteHazardBarrier(vk::CommandBuffer vk_buffer, void ShaderWriteBarrier(vk::CommandBuffer vk_buffer, vk::PipelineStageFlags source_stages) { EXIT_IF(vk_buffer == nullptr || !source_stages); const auto barrier = MakeShaderWriteDependency(); - vk_buffer.pipelineBarrier( - source_stages, - vk::PipelineStageFlagBits::eComputeShader | vk::PipelineStageFlagBits::eVertexInput | - vk::PipelineStageFlagBits::eVertexShader | vk::PipelineStageFlagBits::eFragmentShader | - vk::PipelineStageFlagBits::eTransfer | - vk::PipelineStageFlagBits::eColorAttachmentOutput, - vk::DependencyFlags {}, 1, &barrier, 0, nullptr, 0, nullptr); + vk_buffer.pipelineBarrier(source_stages, + vk::PipelineStageFlagBits::eComputeShader | + vk::PipelineStageFlagBits::eAllGraphics | + vk::PipelineStageFlagBits::eTransfer, + vk::DependencyFlags {}, 1, &barrier, 0, nullptr, 0, nullptr); } } // namespace Libs::Graphics diff --git a/src/graphics/host_gpu/renderer/pipeline/shaders.cpp b/src/graphics/host_gpu/renderer/pipeline/shaders.cpp index ce9623fe3..afb199ba9 100644 --- a/src/graphics/host_gpu/renderer/pipeline/shaders.cpp +++ b/src/graphics/host_gpu/renderer/pipeline/shaders.cpp @@ -427,8 +427,13 @@ void CreatePipelineInternal( const PipelineStaticParameters& static_params, vk::PipelineCache driver_cache) { const bool ps_active = ps_input_info != nullptr; EXIT_IF(vertex_module == nullptr || (ps_active && pixel_module == nullptr)); + EXIT_IF(!vs_input_info.stage); + const bool mesh = vs_input_info.stage.program->stage == ShaderType::Mesh; + EXIT_NOT_IMPLEMENTED(mesh && !graphics.mesh_shader_enabled); + const auto vertex_stage = + mesh ? vk::ShaderStageFlagBits::eMeshEXT : vk::ShaderStageFlagBits::eVertex; - const bool rect_list = static_params.topology == vk::PrimitiveTopology::ePatchList; + const bool rect_list = !mesh && static_params.topology == vk::PrimitiveTopology::ePatchList; vk::ShaderModule tess_control_shader_module = nullptr; vk::ShaderModule tess_eval_shader_module = nullptr; @@ -468,7 +473,7 @@ void CreatePipelineInternal( vert_shader_stage_info.sType = vk::StructureType::ePipelineShaderStageCreateInfo; vert_shader_stage_info.pNext = nullptr; vert_shader_stage_info.flags = {}; - vert_shader_stage_info.stage = vk::ShaderStageFlagBits::eVertex; + vert_shader_stage_info.stage = vertex_stage; vert_shader_stage_info.module = vertex_module; vert_shader_stage_info.pName = "main"; vert_shader_stage_info.pSpecializationInfo = nullptr; @@ -780,18 +785,15 @@ void CreatePipelineInternal( color_blending.attachmentCount = static_params.color_count; color_blending.pAttachments = color_blend_attachment; - EXIT_IF(!vs_input_info.stage); std::vector descriptor_bindings; - AddLayoutBindings(descriptor_bindings, *vs_input_info.stage.program, - vk::ShaderStageFlagBits::eVertex); + AddLayoutBindings(descriptor_bindings, *vs_input_info.stage.program, vertex_stage); if (ps_active) { EXIT_IF(!ps_input_info->stage); AddLayoutBindings(descriptor_bindings, *ps_input_info->stage.program, vk::ShaderStageFlagBits::eFragment); } CreateDescriptorLayout(graphics, pipeline, descriptor_bindings); - constexpr auto GraphicsStages = - vk::ShaderStageFlagBits::eVertex | vk::ShaderStageFlagBits::eFragment; + const auto GraphicsStages = vertex_stage | vk::ShaderStageFlagBits::eFragment; const vk::PushConstantRange push_constants {GraphicsStages, 0, ShaderRecompiler::IR::NativePushConstantSize}; @@ -888,8 +890,8 @@ void CreatePipelineInternal( pipeline_info.flags = {}; pipeline_info.stageCount = shader_stage_count; pipeline_info.pStages = shader_stages; - pipeline_info.pVertexInputState = &vertex_input_info; - pipeline_info.pInputAssemblyState = &input_assembly; + pipeline_info.pVertexInputState = mesh ? nullptr : &vertex_input_info; + pipeline_info.pInputAssemblyState = mesh ? nullptr : &input_assembly; vk::PipelineTessellationStateCreateInfo tessellation_state {}; tessellation_state.sType = vk::StructureType::ePipelineTessellationStateCreateInfo; tessellation_state.patchControlPoints = 3; diff --git a/src/graphics/host_gpu/renderer/renderDraw.cpp b/src/graphics/host_gpu/renderer/renderDraw.cpp index 957479337..0362be779 100644 --- a/src/graphics/host_gpu/renderer/renderDraw.cpp +++ b/src/graphics/host_gpu/renderer/renderDraw.cpp @@ -86,7 +86,6 @@ uint32_t ResolveInstanceOffset(const ShaderVertexInputInfo& vs_input_info) { static std::atomic g_draw_state_log_count = 0; static std::atomic g_draw_input_log_count = 0; static std::atomic g_mrt_state_log_count = 0; -static std::atomic g_shader_stage_log_count = 0; static std::atomic g_framebuffer_skip_log_count = 0; @@ -444,66 +443,6 @@ static bool PixelShaderHasDepthOrCoverageSideEffects(const HW::ShaderRegisters& db.shader_execute_on_noop; } -static bool ShouldSkipGeShader(const CommandBuffer& buffer) { - return false; - const auto& ctx = buffer.GetRegisters(); - const auto& ucfg = buffer.GetUserConfig(); - const auto& sh_ctx = buffer.GetShaders(); - const auto& sh_regs = ctx.GetShaderRegisters(); - const auto& ge_cntl = ucfg.GetGeControl(); - const auto& vertex_info = sh_ctx.GetVs(); - const auto stages = ctx.GetShaderStages(); - - const auto is_known_gs_out_prim_type = [](uint32_t value) { - switch (static_cast(value)) { - case Prospero::GsOutputPrimitiveType::kPoints: - case Prospero::GsOutputPrimitiveType::kLines: - case Prospero::GsOutputPrimitiveType::kTriangles: - case Prospero::GsOutputPrimitiveType::k2dRectangle: - case Prospero::GsOutputPrimitiveType::kRectList: return true; - } - - return false; - }; - - const bool ps5_ngg_vertex_path = stages == 0x02002000 && vertex_info.es_regs.data_addr != 0 && - sh_regs.m_vgtGsMaxVertOut == 0x00000000 && - is_known_gs_out_prim_type(sh_regs.m_vgtGsOutPrimType); - - const bool unsupported_stage_mask = (stages != 0 && stages != 0x02002000); - const bool unsupported_gs_stage = (vertex_info.es_regs.data_addr != 0 && - vertex_info.gs_regs.data_addr != 0 && !ps5_ngg_vertex_path); - // GE_CNTL group sizes control guest scheduling and do not constrain the host vertex path. - const bool ge_shader_regs = - (sh_regs.m_geNggSubgrpCntl != 0x00000000 && sh_regs.m_geNggSubgrpCntl != 0x00000001) || - sh_regs.m_vgtGsMaxVertOut != 0x00000000 || - !is_known_gs_out_prim_type(sh_regs.m_vgtGsOutPrimType) || - sh_regs.m_geMaxOutputPerSubgroup > 0x00000040; - - if (unsupported_stage_mask || unsupported_gs_stage || ge_shader_regs) { - static std::once_flag warning_once; - std::call_once(warning_once, [] { - std::printf("Warning: game uses unsupported graphics pipelines; some draw calls were " - "skipped.\n"); - }); - - const auto log_id = g_shader_stage_log_count.fetch_add(1); - if (log_id < 32) { - LOGF("Skipping unsupported GE shader draw: stages=0x%08" PRIx32 - " prim_group=0x%04" PRIx16 " vert_group=0x%04" PRIx16 " ngg=0x%08" PRIx32 - " max_out=0x%08" PRIx32 " gs_max_vert=0x%08" PRIx32 " gs_out_prim=0x%08" PRIx32 - " es=0x%016" PRIx64 " gs=0x%016" PRIx64 "\n", - stages, ge_cntl.primitive_group_size, ge_cntl.vertex_group_size, - sh_regs.m_geNggSubgrpCntl, sh_regs.m_geMaxOutputPerSubgroup, - sh_regs.m_vgtGsMaxVertOut, sh_regs.m_vgtGsOutPrimType, - vertex_info.es_regs.data_addr, vertex_info.gs_regs.data_addr); - } - return true; - } - - return false; -} - struct DrawRenderState { RenderDepthInfo depth_info; RenderColorInfo color_info[RENDER_COLOR_ATTACHMENTS_MAX] = {}; @@ -1036,8 +975,8 @@ static void RefreshShaders(CommandBuffer& buffer, const DrawCallInfo& draw, bool LogDrawPhase(draw.name, "GetGraphicsPrograms"); } state.programs = pipeline_cache.GetGraphicsPrograms( - vertex_shader_info, pixel_shader_info, shader_regs, ctx, target_export_mapping, - state.ps_active, state.vs_input_info, state.ps_input_info); + vertex_shader_info, pixel_shader_info, shader_regs, ctx, buffer.GetUserConfig(), + target_export_mapping, state.ps_active, state.vs_input_info, state.ps_input_info); } static PreparedVertexBuffers PrepareVertexBuffers(uint64_t submit_id, CommandBuffer& buffer, @@ -1176,12 +1115,38 @@ void RenderExecutor::ExecutePreparedDraw(uint64_t submit_id, CommandBuffer& buff bool set_bind_debug, bool set_auto_debug) { EXIT_IF(draw.name == nullptr); auto& ucfg = buffer.GetUserConfig(); + const bool mesh_active = state.vs_input_info.stage.program->stage == ShaderType::Mesh; + uint32_t mesh_groups = 0; + if (mesh_active) { + const auto& mesh = state.vs_input_info.mesh; + if (emit.indexed || ucfg.GetPrimType() != Prospero::PrimitiveType::kTriStrip || + primitive_restart_enable || mesh.primitives_per_group == 0) { + EXIT("unsupported mesh draw: primitive=%u indexed=%u restart=%u\n", + static_cast(ucfg.GetPrimType()), emit.indexed, primitive_restart_enable); + } + if (draw.index_count < 3 || draw.instance_count == 0) { + return; + } + mesh_groups = (draw.index_count - 3u) / mesh.primitives_per_group + 1u; + const auto& limits = m_context.GetGraphics().mesh_shader_properties; + if (mesh_groups > limits.maxMeshWorkGroupCount[0] || + draw.instance_count > limits.maxMeshWorkGroupCount[1] || + static_cast(mesh_groups) * draw.instance_count > + limits.maxMeshWorkGroupTotalCount) { + EXIT("mesh draw exceeds host workgroup limits: %ux%u\n", mesh_groups, + draw.instance_count); + } + } LogDrawPhase(draw.name, "PrepareBindings"); auto bindings = PrepareGraphicsBindings(state.vs_input_info.stage, state.ps_input_info.stage, state.ps_active); - auto vertex_bindings = PrepareVertexBuffers(submit_id, buffer, draw, state.vs_input_info); - auto index_binding = PrepareIndexBuffer(buffer, index_source); + PreparedVertexBuffers vertex_bindings; + PreparedIndexBuffer index_binding; + if (!mesh_active) { + vertex_bindings = PrepareVertexBuffers(submit_id, buffer, draw, state.vs_input_info); + index_binding = PrepareIndexBuffer(buffer, index_source); + } state.rendering = AcquireRenderTargets(buffer, state.color_info, state.color_count, state.depth_info); @@ -1203,7 +1168,9 @@ void RenderExecutor::ExecutePreparedDraw(uint64_t submit_id, CommandBuffer& buff if (set_auto_debug) { SetDrawDebugPhase(buffer, submit_id, draw, 0x200u); } - CommitVertexBuffers(vk_buffer, vertex_bindings); + if (!mesh_active) { + CommitVertexBuffers(vk_buffer, vertex_bindings); + } if (bindings.pixel.has_value()) { if (set_auto_debug) { SetDrawDebugPhase(buffer, submit_id, draw, 0x300u); @@ -1216,7 +1183,16 @@ void RenderExecutor::ExecutePreparedDraw(uint64_t submit_id, CommandBuffer& buff } CommitBindings(buffer, vk::PipelineBindPoint::eGraphics, pipeline, std::span {descriptor_stages.data(), descriptor_stage_count}); - CommitIndexBuffer(vk_buffer, index_binding); + if (mesh_active) { + const uint32_t draw_data[] {draw.index_count, emit.first_vertex, emit.first_instance}; + static_assert(std::size(draw_data) == ShaderRecompiler::IR::PushData::MeshDrawDwordCount); + vk_buffer.pushConstants(pipeline.pipeline_layout, + vk::ShaderStageFlagBits::eMeshEXT | + vk::ShaderStageFlagBits::eFragment, + 0, sizeof(draw_data), draw_data); + } else { + CommitIndexBuffer(vk_buffer, index_binding); + } SetGraphicsDynamicParams(buffer, vk_buffer, state.color_info, state.color_count, state.depth_info); @@ -1230,14 +1206,19 @@ void RenderExecutor::ExecutePreparedDraw(uint64_t submit_id, CommandBuffer& buff if (set_auto_debug) { SetDrawDebugPhase(buffer, submit_id, draw, 0x500u); } - EmitDrawPrimitives(ucfg, vk_buffer, state.vs_input_info, draw, emit); + if (mesh_active) { + vk_buffer.drawMeshTasksEXT(mesh_groups, draw.instance_count, 1); + } else { + EmitDrawPrimitives(ucfg, vk_buffer, state.vs_input_info, draw, emit); + } if (set_auto_debug) { SetDrawDebugPhase(buffer, submit_id, draw, 0x600u); } vk::PipelineStageFlags shader_write_stages = {}; if (HasShaderBufferWrites(state.vs_input_info.stage)) { - shader_write_stages |= vk::PipelineStageFlagBits::eVertexShader; + shader_write_stages |= mesh_active ? vk::PipelineStageFlagBits::eMeshShaderEXT + : vk::PipelineStageFlagBits::eVertexShader; } if (state.ps_active && HasShaderBufferWrites(state.ps_input_info.stage)) { shader_write_stages |= vk::PipelineStageFlagBits::eFragmentShader; @@ -1280,10 +1261,6 @@ void RenderExecutor::DrawIndex(uint64_t submit_id, CommandBuffer& buffer, return; } - if (ShouldSkipGeShader(buffer)) { - return; - } - if (graphics_debug_dump_enabled()) { sh_print("GraphicsRenderDrawIndex():Shader:", sh_ctx); uc_print("GraphicsRenderDrawIndex():UserConfig:", ucfg); @@ -1412,10 +1389,6 @@ void RenderExecutor::DrawAuto(uint64_t submit_id, CommandBuffer& buffer, const D return; } - if (ShouldSkipGeShader(buffer)) { - return; - } - if (graphics_debug_dump_enabled()) { sh_print("GraphicsRenderDrawIndexAuto():Shader:", sh_ctx); uc_print("GraphicsRenderDrawIndexAuto():UserConfig:", ucfg); diff --git a/src/graphics/presentation/window/vulkanWindow.cpp b/src/graphics/presentation/window/vulkanWindow.cpp index 88bde4280..77ebbd597 100644 --- a/src/graphics/presentation/window/vulkanWindow.cpp +++ b/src/graphics/presentation/window/vulkanWindow.cpp @@ -604,10 +604,25 @@ static vk::Device VulkanCreateDevice(vk::PhysicalDevice physical_device, const V #endif } + const bool mesh_extension = HasExtension(device_extensions, VK_EXT_MESH_SHADER_EXTENSION_NAME); + vk::PhysicalDeviceMeshShaderFeaturesEXT supported_mesh {}; + supported_mesh.pNext = &supported_features13; vk::PhysicalDeviceFeatures2 supported_features2 {}; supported_features2.sType = vk::StructureType::ePhysicalDeviceFeatures2; - supported_features2.pNext = &supported_features13; + supported_features2.pNext = mesh_extension ? static_cast(&supported_mesh) + : static_cast(&supported_features13); physical_device.getFeatures2(&supported_features2); + graphics.mesh_shader_enabled = mesh_extension && supported_mesh.meshShader; + if (graphics.mesh_shader_enabled) { + vk::PhysicalDeviceProperties2 properties {}; + properties.pNext = &graphics.mesh_shader_properties; + physical_device.getProperties2(&properties); + LOGF("Vulkan MeshEXT: invocations=%u vertices=%u primitives=%u shared=%u\n", + graphics.mesh_shader_properties.maxMeshWorkGroupInvocations, + graphics.mesh_shader_properties.maxMeshOutputVertices, + graphics.mesh_shader_properties.maxMeshOutputPrimitives, + graphics.mesh_shader_properties.maxMeshSharedMemorySize); + } const auto required_features12 = WindowContext::RequiredVulkan12Features(); const auto required_features13 = WindowContext::RequiredVulkan13Features(); EXIT_NOT_IMPLEMENTED(required_features12.samplerMirrorClampToEdge == VK_TRUE && @@ -692,8 +707,12 @@ static vk::Device VulkanCreateDevice(vk::PhysicalDevice physical_device, const V robustness2_ext_enabled && robustness2.robustImageAccess2 == VK_TRUE ? "true" : "false"); vk::DeviceCreateInfo create_info {}; + vk::PhysicalDeviceMeshShaderFeaturesEXT mesh_features {}; + mesh_features.pNext = &features13; + mesh_features.meshShader = graphics.mesh_shader_enabled; create_info.sType = vk::StructureType::eDeviceCreateInfo; - create_info.pNext = &features13; + create_info.pNext = + mesh_extension ? static_cast(&mesh_features) : static_cast(&features13); create_info.flags = {}; create_info.pQueueCreateInfos = &queue_create_info; create_info.queueCreateInfoCount = 1; @@ -1076,6 +1095,9 @@ void WindowContext::CreateVulkan() { if (HasExtension(available_extensions, VK_EXT_ROBUSTNESS_2_EXTENSION_NAME)) { device_extensions.push_back(VK_EXT_ROBUSTNESS_2_EXTENSION_NAME); } + if (HasExtension(available_extensions, VK_EXT_MESH_SHADER_EXTENSION_NAME)) { + device_extensions.push_back(VK_EXT_MESH_SHADER_EXTENSION_NAME); + } } VulkanInitSubgroupSizeControl(graphic_ctx.physical_device, graphic_ctx); diff --git a/src/graphics/shader/recompiler/ShaderRecompiler.cpp b/src/graphics/shader/recompiler/ShaderRecompiler.cpp index 7f5d2c962..64b52e58c 100644 --- a/src/graphics/shader/recompiler/ShaderRecompiler.cpp +++ b/src/graphics/shader/recompiler/ShaderRecompiler.cpp @@ -14,7 +14,6 @@ #include "graphics/shader/recompiler/ir/passes/ResourceMaterialization.h" #include "graphics/shader/recompiler/ir/passes/ResourceTracking.h" #include "graphics/shader/recompiler/ir/passes/ShaderInfoCollection.h" -#include "graphics/shader/recompiler/ir/passes/SharedMemoryBarrier.h" #include "graphics/shader/recompiler/ir/passes/SrtWalker.h" #include "graphics/shader/recompiler/ir/passes/SsaRewrite.h" @@ -48,6 +47,7 @@ const char* StageName(ShaderType stage) { switch (stage) { case ShaderType::Compute: return "CS"; case ShaderType::Vertex: return "VS"; + case ShaderType::Mesh: return "MS"; case ShaderType::Pixel: return "PS"; default: return "unknown"; } @@ -477,6 +477,45 @@ EmbeddedFetchData DetectEmbeddedVertexFetch(const Decoder::Program& decoded return data; } +Decoder::Program DecodeFusedProgram(std::span front, std::span back, + std::vector& joined_code) { + EXIT_IF(back.empty()); + Decoder::Program result; + uint32_t front_words = 0; + while (front_words < front.size()) { + auto& inst = result.instructions.emplace_back(); + Decoder::DecodeInstruction(front, front_words, inst); + front_words += inst.word_count; + if (inst.opcode == Decoder::Opcode::S_SETPC_B64) { + EXIT_NOT_IMPLEMENTED(inst.src0.kind != Decoder::OperandKind::Sgpr || + inst.src0.reg != 6u); + break; + } + EXIT_NOT_IMPLEMENTED(inst.opcode == Decoder::Opcode::S_ENDPGM); + } + EXIT_IF(result.instructions.empty() || + result.instructions.back().opcode != Decoder::Opcode::S_SETPC_B64); + joined_code.assign(front.begin(), front.begin() + front_words); + joined_code.insert(joined_code.end(), back.begin(), back.end()); + // The merged-stage ABI passes the back shader in s[6:7]. Give that handoff an + // ordinary CFG edge, retaining both bodies in one register and LDS lifetime. + joined_code[front_words - 1u] = 0xbf820000u; // s_branch to the following instruction + result.instructions.back() = {}; + Decoder::DecodeInstruction(joined_code, front_words - 1u, result.instructions.back()); + Decoder::Program back_program; + Decoder::DecodeProgram(back, back_program); + const auto back_pc = front_words * sizeof(uint32_t); + for (auto& inst: back_program.instructions) { + // A back-stage PC-relative data reference requires its guest code address. + EXIT_NOT_IMPLEMENTED(inst.opcode == Decoder::Opcode::S_GETPC_B64); + inst.pc += back_pc; + inst.branch_target += back_pc; + result.instructions.push_back(std::move(inst)); + } + result.code = joined_code; + return result; +} + } // namespace TranslateResult TranslateProgram(std::span code, const CompileOptions& options) { @@ -484,7 +523,7 @@ TranslateResult TranslateProgram(std::span code, const CompileOp EXIT("shader recompiler input is empty\n"); } if (options.stage != ShaderType::Compute && options.stage != ShaderType::Vertex && - options.stage != ShaderType::Pixel) { + options.stage != ShaderType::Pixel && options.stage != ShaderType::Mesh) { EXIT("shader recompiler received unsupported stage %u\n", static_cast(options.stage)); } @@ -501,7 +540,12 @@ TranslateResult TranslateProgram(std::span code, const CompileOp static_cast(code.size())); Decoder::Program decoded; - Decoder::DecodeProgram(code, decoded); + std::vector joined_code; + if (options.stage == ShaderType::Mesh) { + decoded = DecodeFusedProgram(code, options.back_code, joined_code); + } else { + Decoder::DecodeProgram(code, decoded); + } LOGF("%s phase end: stage=%s hash=0x%016" PRIx64 " decode instructions=%" PRIu64 " elapsed_ms=%" PRIu64 "\n", GetDumpLabel(options), StageName(options.stage), options.shader_hash, @@ -560,8 +604,7 @@ TranslateResult TranslateProgram(std::span code, const CompileOp const ShaderComputeInputInfo* compute = nullptr; switch (options.stage) { case ShaderType::Vertex: - vertex = options.input_info.vertex; - break; + case ShaderType::Mesh: vertex = options.input_info.vertex; break; case ShaderType::Pixel: pixel = options.input_info.pixel; break; @@ -617,14 +660,6 @@ TranslateResult TranslateProgram(std::span code, const CompileOp IR::RemoveIdentities(ir.blocks); IR::EliminateDeadCode(ir.blocks); } - if (options.stage == ShaderType::Compute) { - const auto lds_barriers = - IR::InsertSharedMemoryBarriers(ir, ir.wave_size, *compute); - if (lds_barriers.inserted_barriers != 0) { - LOGF("%s wave64 LDS synchronization: barriers=%" PRIu32 "\n", GetDumpLabel(options), - lds_barriers.inserted_barriers); - } - } IR::BuildSrtPlan(ir); IR::EliminateDeadCode(ir.blocks); IR::TrackResources(ir); @@ -654,7 +689,8 @@ CompileResult CompileProgram(TranslateResult translated, const CompileOptions& o const ShaderPixelInputInfo* pixel = nullptr; const ShaderComputeInputInfo* compute = nullptr; switch (options.stage) { - case ShaderType::Vertex: vertex = options.input_info.vertex; break; + case ShaderType::Vertex: + case ShaderType::Mesh: vertex = options.input_info.vertex; break; case ShaderType::Pixel: pixel = options.input_info.pixel; break; case ShaderType::Compute: compute = options.input_info.compute; break; default: EXIT("invalid shader stage\n"); diff --git a/src/graphics/shader/recompiler/ShaderRecompiler.h b/src/graphics/shader/recompiler/ShaderRecompiler.h index 94ff50776..2f7d8afac 100644 --- a/src/graphics/shader/recompiler/ShaderRecompiler.h +++ b/src/graphics/shader/recompiler/ShaderRecompiler.h @@ -21,6 +21,7 @@ struct CompileOptions { bool early_dump = false; const char* dump_label = nullptr; std::span user_data; + std::span back_code; ShaderStageInputInfo input_info; }; diff --git a/src/graphics/shader/recompiler/backend/spirv/SpirvBuilder.cpp b/src/graphics/shader/recompiler/backend/spirv/SpirvBuilder.cpp index c9f1c2b97..d58acc0b4 100644 --- a/src/graphics/shader/recompiler/backend/spirv/SpirvBuilder.cpp +++ b/src/graphics/shader/recompiler/backend/spirv/SpirvBuilder.cpp @@ -2,6 +2,7 @@ #include "common/debug.h" +#include #include namespace Libs::Graphics::ShaderRecompiler::Spirv { @@ -190,6 +191,18 @@ void Builder::AddEntryPoint(uint32_t execution_model, uint32_t entry_point, cons std::vector operands = {execution_model, entry_point}; AppendString(operands, name); operands.insert(operands.end(), interfaces.begin(), interfaces.end()); + if (m_version >= 0x00010400u) { + for (size_t offset = 0; offset < m_declarations.size();) { + const auto count = m_declarations[offset] >> 16u; + if ((m_declarations[offset] & 0xffffu) == 59u) { + const auto id = m_declarations[offset + 2u]; + if (std::find(interfaces.begin(), interfaces.end(), id) == interfaces.end()) { + operands.push_back(id); + } + } + offset += count; + } + } AppendInstruction(m_entry_points, 15u, operands); } diff --git a/src/graphics/shader/recompiler/backend/spirv/SpirvEmitter.cpp b/src/graphics/shader/recompiler/backend/spirv/SpirvEmitter.cpp index d049e3637..3d49fab33 100644 --- a/src/graphics/shader/recompiler/backend/spirv/SpirvEmitter.cpp +++ b/src/graphics/shader/recompiler/backend/spirv/SpirvEmitter.cpp @@ -236,7 +236,8 @@ void AnalyzeProgramRequirements(IR::Program& program) { if (kind != IR::ResourceKind::Lds && kind != IR::ResourceKind::Gds) { Fail(program, "shared operation has invalid resource kind"); } - if (program.stage != ShaderType::Compute && kind == IR::ResourceKind::Lds) { + if (program.stage != ShaderType::Compute && program.stage != ShaderType::Mesh && + kind == IR::ResourceKind::Lds) { requirements.function_lds = true; } if (shared_access == IR::SharedAccess::Append || @@ -259,7 +260,6 @@ void AnalyzeProgramRequirements(IR::Program& program) { break; } case IR::ValueOpcode::DppUpdateU32: - case IR::ValueOpcode::WqmMask: case IR::ValueOpcode::WriteLane: { MarkBallot(); requirements.subgroup_local_invocation_id = true; @@ -308,7 +308,7 @@ std::vector EmitProgram(const IR::Program& program, using namespace Emitter; if (program.stage != ShaderType::Compute && program.stage != ShaderType::Vertex && - program.stage != ShaderType::Pixel) { + program.stage != ShaderType::Pixel && program.stage != ShaderType::Mesh) { Fail(program, "binary SPIR-V emitter supports compute, vertex, and pixel shaders"); } if (!program.srt_plan_complete || !program.resource_tracking_complete || @@ -321,6 +321,11 @@ std::vector EmitProgram(const IR::Program& program, EmitterState state(program, input_info); state.stage = program.stage; state.wave_size = program.wave_size; + const auto* workgroup = ShaderWorkgroupInput(program.stage, input_info); + state.lane_count = + workgroup != nullptr && program.wave_size == 64u && workgroup->host_subgroup_size == 32u + ? 2u + : 1u; state.inputs.reserve(program.info.inputs.size()); state.outputs.reserve(program.info.outputs.size()); state.interface_variables.reserve(program.info.inputs.size() + program.info.outputs.size()); @@ -329,6 +334,8 @@ std::vector EmitProgram(const IR::Program& program, AllocateOutputVariables(state); DefineModule(state); EmitProgram(state, program); + state.builder.AddEntryPoint(ExecutionModelForStage(state.stage), state.main_func, "main", + state.interface_variables); auto binary = state.builder.Build(); if (binary.empty()) { diff --git a/src/graphics/shader/recompiler/backend/spirv/spirvEmitterAnalysis.cpp b/src/graphics/shader/recompiler/backend/spirv/spirvEmitterAnalysis.cpp index d0555440a..60ada8d76 100644 --- a/src/graphics/shader/recompiler/backend/spirv/spirvEmitterAnalysis.cpp +++ b/src/graphics/shader/recompiler/backend/spirv/spirvEmitterAnalysis.cpp @@ -278,6 +278,7 @@ void EmitStorageImageWrite(EmitterState& state, uint32_t resource, uint32_t mip_ uint32_t ExecutionModelForStage(ShaderType stage) { switch (stage) { case ShaderType::Vertex: return ExecutionModelVertex; + case ShaderType::Mesh: return 5365u; // MeshEXT case ShaderType::Pixel: return ExecutionModelFragment; default: return ExecutionModelGLCompute; } diff --git a/src/graphics/shader/recompiler/backend/spirv/spirvEmitterFlow.cpp b/src/graphics/shader/recompiler/backend/spirv/spirvEmitterFlow.cpp index 2f54ea6ad..bde60fa8b 100644 --- a/src/graphics/shader/recompiler/backend/spirv/spirvEmitterFlow.cpp +++ b/src/graphics/shader/recompiler/backend/spirv/spirvEmitterFlow.cpp @@ -16,26 +16,6 @@ bool UserDataDwordIndex(const EmitterState& state, IR::ScalarReg reg, uint32_t& return true; } -uint32_t EmitWqmWordU32(EmitterState& state, uint32_t value) { - const auto shifted_one = state.builder.AllocateId(); - const auto merged_one = state.builder.AllocateId(); - const auto shifted_two = state.builder.AllocateId(); - const auto merged_two = state.builder.AllocateId(); - const auto quad_bits = state.builder.AllocateId(); - const auto result = state.builder.AllocateId(); - state.builder.AddFunction( - {OpShiftRightLogical, TypeU32(state), shifted_one, value, ConstantU32(state, 1)}); - state.builder.AddFunction({OpBitwiseOr, TypeU32(state), merged_one, value, shifted_one}); - state.builder.AddFunction( - {OpShiftRightLogical, TypeU32(state), shifted_two, merged_one, ConstantU32(state, 2)}); - state.builder.AddFunction({OpBitwiseOr, TypeU32(state), merged_two, merged_one, shifted_two}); - state.builder.AddFunction( - {OpBitwiseAnd, TypeU32(state), quad_bits, merged_two, ConstantU32(state, 0x11111111u)}); - state.builder.AddFunction( - {OpIMul, TypeU32(state), result, quad_bits, ConstantU32(state, 0x0fu)}); - return result; -} - uint32_t EmitWqmU64(EmitterState& state, uint32_t value) { const auto shifted_one = state.builder.AllocateId(); const auto merged_one = state.builder.AllocateId(); @@ -61,6 +41,24 @@ uint32_t EmitBuiltinU32(ValueEmitContext& ctx, IR::StageInputKind kind, uint32_t if (kind == IR::StageInputKind::LocalInvocationIndex) { return EmitLocalInvocationIndex(state); } + if (state.lane_count == 2 && (kind == IR::StageInputKind::LocalInvocationId || + kind == IR::StageInputKind::GlobalInvocationId)) { + const auto* cs = ShaderWorkgroupInput(state.stage, state.input_info); + uint32_t divisor = 1; + for (uint32_t axis = 0; axis < component; axis++) { + divisor *= std::max(cs->threads_num[axis], 1u); + } + const auto size = std::max(cs->threads_num[component], 1u); + const auto divided = EmitBinaryU32(state, OpUDiv, EmitLocalInvocationIndex(state), + ConstantU32(state, divisor)); + const auto local = EmitBinaryU32(state, OpUMod, divided, ConstantU32(state, size)); + if (kind == IR::StageInputKind::LocalInvocationId) { + return local; + } + const auto group = EmitInputComponentU32(state, IR::StageInputKind::WorkgroupId, component); + return EmitAddU32(state, local, + EmitBinaryU32(state, OpIMul, group, ConstantU32(state, size))); + } const auto variable = InputVariableForKind(state, kind); if (variable == 0) { return ConstantU32(state, 0); @@ -106,40 +104,6 @@ uint32_t EmitBuiltinU32(ValueEmitContext& ctx, IR::StageInputKind kind, uint32_t return EmitInputComponentU32(state, kind, component); } -uint32_t EmitWqm(ValueEmitContext& ctx, uint32_t active) { - auto& state = ctx.state; - const auto ballot = state.builder.AllocateId(); - state.builder.AddFunction({OpGroupNonUniformBallot, TypeU32Vector(state, 4), ballot, - ConstantU32(state, ScopeSubgroup), active}); - const auto low = state.builder.AllocateId(); - state.builder.AddFunction({OpCompositeExtract, TypeU32(state), low, ballot, 0}); - const auto wqm_low = EmitWqmWordU32(state, low); - const auto lane = EmitSubgroupLocalInvocationId(state); - uint32_t mask = wqm_low; - uint32_t bit_lane = lane; - if (state.wave_size == 64u) { - const auto high = state.builder.AllocateId(); - state.builder.AddFunction({OpCompositeExtract, TypeU32(state), high, ballot, 1}); - const auto wqm_high = EmitWqmWordU32(state, high); - const auto upper = state.builder.AllocateId(); - mask = state.builder.AllocateId(); - bit_lane = state.builder.AllocateId(); - state.builder.AddFunction( - {OpUGreaterThanEqual, TypeBool(state), upper, lane, ConstantU32(state, 32)}); - state.builder.AddFunction({OpSelect, TypeU32(state), mask, upper, wqm_high, wqm_low}); - state.builder.AddFunction( - {OpBitwiseAnd, TypeU32(state), bit_lane, lane, ConstantU32(state, 31)}); - } - const auto bit = state.builder.AllocateId(); - const auto hit = state.builder.AllocateId(); - const auto result = state.builder.AllocateId(); - state.builder.AddFunction( - {OpShiftLeftLogical, TypeU32(state), bit, ConstantU32(state, 1), bit_lane}); - state.builder.AddFunction({OpBitwiseAnd, TypeU32(state), hit, mask, bit}); - state.builder.AddFunction({OpINotEqual, TypeBool(state), result, hit, ConstantU32(state, 0)}); - return result; -} - uint32_t EmitDppWriteCondition(ValueEmitContext& ctx, const IR::DppMoveFlags& flags, uint32_t exec) { auto& state = ctx.state; @@ -378,7 +342,10 @@ void EmitAuxPositionExport(ValueEmitContext& ctx, uint32_t data, const IR::Expor const auto layer = state.builder.AllocateId(); state.builder.AddFunction({OpBitwiseAnd, TypeU32(state), layer, raw, ConstantU32(state, 0x7ffu)}); - state.builder.AddFunction({OpStore, state.layer_variable, layer}); + const auto pointer = state.stage == ShaderType::Mesh + ? MeshOutputPointer(state, IR::StageOutputKind::Layer) + : state.layer_variable; + state.builder.AddFunction({OpStore, pointer, layer}); continue; } if (!output.point_size && output.clip_distance == UINT32_MAX && @@ -454,12 +421,18 @@ void EmitExport(ValueEmitContext& ctx, const IR::Inst& inst) { {OpSelect, TypeU32(state), value, exec, ConstantU32(state, 1), ConstantU32(state, 0)}); state.builder.AddFunction({OpStore, state.pixel_valid_mask_variable, value}); } - if (exp.kind == IR::ExportTargetKind::Null || exp.kind == IR::ExportTargetKind::Primitive || - exp.en == 0u) { + if (exp.kind == IR::ExportTargetKind::Null || exp.en == 0u) { return; } EmitIfCondition(state, exec, [&]() { const auto data = ctx.Arg(inst, 0); + if (exp.kind == IR::ExportTargetKind::Primitive) { + if (state.stage == ShaderType::Mesh) { + state.builder.AddFunction( + {OpStore, MeshPrimitivePointer(state), ExportRawComponent(ctx, data, 0)}); + } + return; + } if (exp.kind == IR::ExportTargetKind::Position && exp.index != 0) { EmitAuxPositionExport(ctx, data, exp); return; @@ -483,8 +456,9 @@ void EmitExport(ValueEmitContext& ctx, const IR::Inst& inst) { } return; } - const auto variable = OutputVariableForExport(state, exp); - if (variable == 0) { + const auto variable = + state.stage == ShaderType::Mesh ? 0u : OutputVariableForExport(state, exp); + if (state.stage != ShaderType::Mesh && variable == 0) { return; } const bool uint_output = MrtOutputMode(state, exp) == 7u; @@ -501,10 +475,16 @@ void EmitExport(ValueEmitContext& ctx, const IR::Inst& inst) { value = mapped; } } - if (exp.kind == IR::ExportTargetKind::Position) { - if (state.stage == ShaderType::Vertex && state.input_info.vertex->clip_space.enabled) { - value = ConvertPositionToClipSpace(state, value); - } + if (exp.kind == IR::ExportTargetKind::Position && + state.input_info.vertex->clip_space.enabled) { + value = ConvertPositionToClipSpace(state, value); + } + if (state.stage == ShaderType::Mesh) { + const auto kind = exp.kind == IR::ExportTargetKind::Position + ? IR::StageOutputKind::Position + : IR::StageOutputKind::Parameter; + state.builder.AddFunction({OpStore, MeshOutputPointer(state, kind, exp.index), value}); + } else if (exp.kind == IR::ExportTargetKind::Position) { const auto pointer = state.builder.AllocateId(); state.builder.AddFunction( {OpAccessChain, TypePointer(state, StorageClassOutput, TypeF32Vector(state, 4)), @@ -537,6 +517,19 @@ bool EmitValueFlow(ValueEmitContext& ctx, const IR::Inst& inst) { ConstantU32(state, semantics)}); return true; } + case IR::ValueOpcode::MeshAllocate: EmitMeshAllocate(ctx, inst); return true; + case IR::ValueOpcode::MeshDrawParameter: { + const auto index = inst.Arg(0).U32(); + if (state.stage != ShaderType::Mesh || index >= IR::PushData::MeshDrawDwordCount) { + ctx.Fail(inst, "invalid mesh draw parameter"); + } + const auto pointer = state.builder.AllocateId(); + state.builder.AddFunction({OpAccessChain, TypePushConstantElementPointer(state), + pointer, state.push_constant_variable, ConstantU32(state, 0), + ConstantU32(state, index)}); + state.builder.AddFunction({OpLoad, TypeU32(state), ctx.Result(inst), pointer}); + return true; + } case IR::ValueOpcode::GetUserData: { const auto reg = inst.Arg(0).ScalarRegister(); uint32_t dword = 0; @@ -561,17 +554,12 @@ bool EmitValueFlow(ValueEmitContext& ctx, const IR::Inst& inst) { case IR::ValueOpcode::DppMoveU32: { const auto flags = inst.Flags(); const auto target = EmitDppTargetLane(state, flags.control); - const auto shuffled = state.builder.AllocateId(); - state.builder.AddFunction({OpGroupNonUniformShuffle, TypeU32(state), shuffled, - ConstantU32(state, ScopeSubgroup), ctx.Arg(inst, 0), - target.lane}); + const auto shuffled = ctx.Shuffle(inst, 0, target.lane); if (flags.fetch_inactive) { ctx.Define(inst, shuffled); return true; } - const auto ballot = state.builder.AllocateId(); - state.builder.AddFunction({OpGroupNonUniformBallot, TypeU32Vector(state, 4), ballot, - ConstantU32(state, ScopeSubgroup), ctx.Arg(inst, 1)}); + const auto ballot = ctx.Ballot(inst.Arg(1)); const auto source_active = EmitBallotLaneActiveBool(state, ballot, target.lane); const auto can_fetch = state.builder.AllocateId(); state.builder.AddFunction( @@ -588,30 +576,18 @@ bool EmitValueFlow(ValueEmitContext& ctx, const IR::Inst& inst) { case IR::ValueOpcode::WqmU64: ctx.Define(inst, EmitWqmU64(ctx.state, ctx.Arg(inst, 0))); return true; - case IR::ValueOpcode::WqmMask: - ctx.Define(inst, EmitWqm(ctx, ctx.Arg(inst, 0))); - return true; case IR::ValueOpcode::LaneId: ctx.Define(inst, EmitSubgroupLocalInvocationId(state)); return true; - case IR::ValueOpcode::Ballot: - ctx.Emit(inst, OpGroupNonUniformBallot, IR::Type::U32x4, - {ConstantU32(state, ScopeSubgroup), ctx.Arg(inst, 0)}); - return true; + case IR::ValueOpcode::Ballot: ctx.Define(inst, ctx.Ballot(inst.Arg(0))); return true; case IR::ValueOpcode::ReadFirstLane: { - const auto ballot = state.builder.AllocateId(); - const auto lane = state.builder.AllocateId(); - state.builder.AddFunction({OpGroupNonUniformBallot, TypeU32Vector(state, 4), ballot, - ConstantU32(state, ScopeSubgroup), ctx.Arg(inst, 1)}); - state.builder.AddFunction({OpGroupNonUniformBallotFindLSB, TypeU32(state), lane, - ConstantU32(state, ScopeSubgroup), ballot}); - ctx.Emit(inst, OpGroupNonUniformShuffle, IR::Type::U32, - {ConstantU32(state, ScopeSubgroup), ctx.Arg(inst, 0), lane}); + const auto ballot = ctx.Ballot(inst.Arg(1)); + const auto lane = ctx.FirstLane(ballot); + ctx.Define(inst, ctx.Shuffle(inst, 0, lane)); return true; } case IR::ValueOpcode::ReadLane: - ctx.Emit(inst, OpGroupNonUniformShuffle, IR::Type::U32, - {ConstantU32(state, ScopeSubgroup), ctx.Arg(inst, 0), ctx.Arg(inst, 1)}); + ctx.Define(inst, ctx.Shuffle(inst, 0, ctx.Arg(inst, 1))); return true; case IR::ValueOpcode::WriteLane: { const auto hit = state.builder.AllocateId(); @@ -656,16 +632,10 @@ bool EmitValueFlow(ValueEmitContext& ctx, const IR::Inst& inst) { state.builder.AddFunction( {OpBitwiseAnd, TypeU32(state), index, shifted, ConstantU32(state, 15)}); state.builder.AddFunction({OpBitwiseOr, TypeU32(state), target, row_value, index}); - const auto shuffled = state.builder.AllocateId(); - state.builder.AddFunction({OpGroupNonUniformShuffle, TypeU32(state), shuffled, - ConstantU32(state, ScopeSubgroup), ctx.Arg(inst, 0), - target}); + const auto shuffled = ctx.Shuffle(inst, 0, target); uint32_t result = shuffled; if (!flags.fetch_inactive) { - const auto source_exec = state.builder.AllocateId(); - state.builder.AddFunction({OpGroupNonUniformShuffle, TypeBool(state), source_exec, - ConstantU32(state, ScopeSubgroup), ctx.Arg(inst, 3), - target}); + const auto source_exec = ctx.Shuffle(inst, 3, target); result = state.builder.AllocateId(); state.builder.AddFunction({OpSelect, TypeU32(state), result, source_exec, shuffled, ConstantU32(state, 0)}); diff --git a/src/graphics/shader/recompiler/backend/spirv/spirvEmitterHelpers.cpp b/src/graphics/shader/recompiler/backend/spirv/spirvEmitterHelpers.cpp index 7bd7c3d58..e763caef9 100644 --- a/src/graphics/shader/recompiler/backend/spirv/spirvEmitterHelpers.cpp +++ b/src/graphics/shader/recompiler/backend/spirv/spirvEmitterHelpers.cpp @@ -126,7 +126,7 @@ uint32_t EmitSubgroupLocalInvocationId(EmitterState& state) { const auto value = state.builder.AllocateId(); state.builder.AddFunction( {OpLoad, TypeU32(state), value, state.subgroup_local_invocation_id_variable}); - return value; + return state.lane_half == 0 ? value : EmitAddU32(state, value, ConstantU32(state, 32)); } uint32_t InputVariableForKind(const EmitterState& state, IR::StageInputKind kind) { @@ -167,6 +167,11 @@ uint32_t EmitLocalInvocationIndex(EmitterState& state) { } const auto value = state.builder.AllocateId(); state.builder.AddFunction({OpLoad, TypeU32(state), value, variable}); + if (state.lane_count == 2) { + const auto wave_base = EmitBinaryU32(state, OpBitwiseAnd, value, ConstantU32(state, ~31u)); + return EmitAddU32(state, EmitAddU32(state, value, wave_base), + ConstantU32(state, state.lane_half * 32)); + } return value; } diff --git a/src/graphics/shader/recompiler/backend/spirv/spirvEmitterInternal.h b/src/graphics/shader/recompiler/backend/spirv/spirvEmitterInternal.h index fcebd1d49..a86bf5491 100644 --- a/src/graphics/shader/recompiler/backend/spirv/spirvEmitterInternal.h +++ b/src/graphics/shader/recompiler/backend/spirv/spirvEmitterInternal.h @@ -198,6 +198,7 @@ enum : uint32_t { OpIMul = 132, OpFMul = 133, OpUDiv = 134, + OpUMod = 137, OpFDiv = 136, OpIAddCarry = 149, OpUMulExtended = 151, @@ -318,6 +319,7 @@ struct OutputBinding { uint32_t location = 0; uint32_t variable_id = 0; std::string debug_name; + uint32_t mesh_data_variable = 0; }; using ImageDimension = Decoder::ImageDimension; @@ -345,8 +347,8 @@ const ImageDimensionInfo& ImageDimensionInfoFor(ImageDimension dimension); struct EmitterState { EmitterState(const IR::Program& program_, ShaderStageInputInfo input_info_) - : program(program_), input_info(input_info_), - requirements(*program_.spirv_requirements) {} + : builder(program_.stage == ShaderType::Mesh ? 0x00010400u : 0x00010300u), + program(program_), input_info(input_info_), requirements(*program_.spirv_requirements) {} Builder builder; const IR::Program& program; @@ -354,6 +356,8 @@ struct EmitterState { const IR::SpirvRequirements& requirements; ShaderType stage = ShaderType::Unknown; uint32_t wave_size = 64; + uint32_t lane_count = 1; + uint32_t lane_half = 0; uint32_t storage_buffer_variable = 0; uint32_t storage_buffer_u64_variable = 0; std::array memory_byte_offsets {}; @@ -366,10 +370,15 @@ struct EmitterState { uint32_t shader_data_storage_variable = 0; uint32_t flattened_srt_variable = 0; uint32_t lds_variable = 0; - uint32_t scratch_variable = 0; + std::array scratch_variable {}; std::array image_variables {}; uint32_t sampler_variable = 0; uint32_t main_func = 0; + uint32_t mesh_guest_func = 0; + uint32_t mesh_allocation = 0; + uint32_t mesh_primitive_data = 0; + uint32_t mesh_primitives = 0; + uint32_t mesh_cull = 0; uint32_t entry_label = 0; uint32_t current_label = 0; uint32_t pixel_valid_mask_variable = 0; @@ -427,6 +436,10 @@ struct ValueEmitContext { uint32_t Def(IR::Value value); uint32_t Arg(const IR::Inst& inst, size_t index); + uint32_t HalfArg(const IR::Inst& inst, size_t index, uint32_t half); + uint32_t Ballot(IR::Value predicate); + uint32_t FirstLane(uint32_t ballot); + uint32_t Shuffle(const IR::Inst& inst, size_t index, uint32_t lane); uint32_t Result(const IR::Inst& inst); uint32_t TypeId(IR::Type type) const; uint32_t Emit(const IR::Inst& inst, uint32_t opcode, IR::Type type, @@ -448,6 +461,8 @@ struct ValueEmitContext { std::unordered_map> dispatcher_block_loads; const IR::Block* current_block = nullptr; uint32_t scratch_u32_variable = 0; + ValueEmitContext* other_half = nullptr; + uint32_t half = 0; }; enum class VertexInputScalarKind { Float, Sint, Uint }; @@ -575,6 +590,11 @@ void DecorateDescriptor(EmitterState& state, uint32_t variable, const char* name void AddDescriptorAnnotationsAndNames(EmitterState& state); void DefineModule(EmitterState& state); +void DefineMeshOutputs(EmitterState& state); +void EmitMeshEntryPoint(EmitterState& state); +void EmitMeshAllocate(ValueEmitContext& ctx, const IR::Inst& inst); +uint32_t MeshOutputPointer(EmitterState& state, IR::StageOutputKind kind, uint32_t index = 0); +uint32_t MeshPrimitivePointer(EmitterState& state); uint32_t EmitTrueBool(EmitterState& state); diff --git a/src/graphics/shader/recompiler/backend/spirv/spirvEmitterMemory.cpp b/src/graphics/shader/recompiler/backend/spirv/spirvEmitterMemory.cpp index e93651296..fa12ab19f 100644 --- a/src/graphics/shader/recompiler/backend/spirv/spirvEmitterMemory.cpp +++ b/src/graphics/shader/recompiler/backend/spirv/spirvEmitterMemory.cpp @@ -32,6 +32,9 @@ uint32_t AndCondition(EmitterState& state, uint32_t lhs, uint32_t rhs) { uint32_t EmitDsMaskedLaneRead(EmitterState& state, uint32_t source, uint32_t target, uint32_t exec) { + if (state.lane_count == 2) { + target = Binary(state, OpBitwiseAnd, TypeU32(state), target, ConstantU32(state, 31)); + } const auto shuffled = state.builder.AllocateId(); state.builder.AddFunction({OpGroupNonUniformShuffle, TypeU32(state), shuffled, ConstantU32(state, ScopeSubgroup), source, target}); @@ -731,6 +734,9 @@ uint32_t SharedFloatAtomic(ValueEmitContext& ctx, const IR::Inst& inst, const IR uint32_t AppendConsume(ValueEmitContext& ctx, const IR::Inst& inst, bool append) { auto& state = ctx.state; + if (ctx.half == 1) { + return ctx.other_half->Def(IR::Value(const_cast(&inst))); + } const auto m0 = ctx.Arg(inst, 0); const auto base = Binary(state, OpShiftRightLogical, TypeU32(state), m0, ConstantU32(state, 16)); @@ -743,9 +749,7 @@ uint32_t AppendConsume(ValueEmitContext& ctx, const IR::Inst& inst, bool append) const auto access = PrepareMemoryResourceAccess(state, mem); const auto index = EmitMemoryElementIndex(state, access, raw_index); const auto exec = ctx.Arg(inst, 1); - const auto ballot = state.builder.AllocateId(); - state.builder.AddFunction({OpGroupNonUniformBallot, TypeU32Vector(state, 4), ballot, - ConstantU32(state, ScopeSubgroup), exec}); + const auto ballot = ctx.Ballot(inst.Arg(1)); const auto low = state.builder.AllocateId(); const auto high = state.builder.AllocateId(); state.builder.AddFunction({OpCompositeExtract, TypeU32(state), low, ballot, 0}); @@ -753,11 +757,12 @@ uint32_t AppendConsume(ValueEmitContext& ctx, const IR::Inst& inst, bool append) const auto count = Binary(state, OpIAdd, TypeU32(state), Unary(state, OpBitCount, TypeU32(state), low), Unary(state, OpBitCount, TypeU32(state), high)); - const auto first = state.builder.AllocateId(); - state.builder.AddFunction({OpGroupNonUniformBallotFindLSB, TypeU32(state), first, - ConstantU32(state, ScopeSubgroup), ballot}); + const auto first = ctx.FirstLane(ballot); + const auto source_lane = state.lane_count == 2 ? Binary(state, OpBitwiseAnd, TypeU32(state), + first, ConstantU32(state, 31)) + : first; const auto is_first = - Binary(state, OpIEqual, TypeBool(state), EmitSubgroupLocalInvocationId(state), first); + Binary(state, OpIEqual, TypeBool(state), EmitSubgroupLocalInvocationId(state), source_lane); const auto storage_bounds = EmitMemoryElementInBounds(state, access, index); const auto m0_bounds = mem.kind == IR::ResourceKind::Gds @@ -765,7 +770,12 @@ uint32_t AppendConsume(ValueEmitContext& ctx, const IR::Inst& inst, bool append) : Binary(state, OpULessThan, TypeBool(state), ConstantU32(state, ctx.Memory(inst).offset + 3u), size); const auto condition = AndCondition( - state, is_first, AndCondition(state, exec, AndCondition(state, storage_bounds, m0_bounds))); + state, is_first, + AndCondition(state, + state.lane_count == 2 + ? Binary(state, OpINotEqual, TypeBool(state), count, ConstantU32(state, 0)) + : exec, + AndCondition(state, storage_bounds, m0_bounds))); const auto atomic = EmitValueOrZeroIfCondition(state, condition, [&]() { const auto value = state.builder.AllocateId(); state.builder.AddFunction( @@ -777,7 +787,7 @@ uint32_t AppendConsume(ValueEmitContext& ctx, const IR::Inst& inst, bool append) }); const auto result = state.builder.AllocateId(); state.builder.AddFunction({OpGroupNonUniformShuffle, TypeU32(state), result, - ConstantU32(state, ScopeSubgroup), atomic, first}); + ConstantU32(state, ScopeSubgroup), atomic, source_lane}); return result; } diff --git a/src/graphics/shader/recompiler/backend/spirv/spirvEmitterMemoryHelpers.cpp b/src/graphics/shader/recompiler/backend/spirv/spirvEmitterMemoryHelpers.cpp index 1e2e46e8c..1a6a1d22e 100644 --- a/src/graphics/shader/recompiler/backend/spirv/spirvEmitterMemoryHelpers.cpp +++ b/src/graphics/shader/recompiler/backend/spirv/spirvEmitterMemoryHelpers.cpp @@ -65,14 +65,15 @@ void EmitMemoryOffsets(EmitterState& state) { } uint32_t LdsDwordCount(const EmitterState& state) { - return state.stage == ShaderType::Compute ? state.input_info.compute->lds_size_dwords : 8192u; + const auto* workgroup = ShaderWorkgroupInput(state.stage, state.input_info); + return workgroup != nullptr ? workgroup->lds_size_dwords : 8192u; } static void EnsureLdsStorage(EmitterState& state) { if (state.lds_variable != 0) { return; } - if (state.stage != ShaderType::Compute) { + if (ShaderWorkgroupInput(state.stage, state.input_info) == nullptr) { EXIT("function LDS was not prepared before SPIR-V function emission\n"); } state.lds_variable = state.builder.DefineGlobalVariable( @@ -122,10 +123,10 @@ MemoryResourceAccess PrepareMemoryResourceAccess(EmitterState& state, const IR:: access.length = state.gds_length; return access; case IR::ResourceKind::Scratch: - if (state.scratch_variable == 0) { + if (state.scratch_variable[state.lane_half] == 0) { EXIT("scratch storage was not prepared before SPIR-V function emission\n"); } - access.object_pointer = state.scratch_variable; + access.object_pointer = state.scratch_variable[state.lane_half]; access.length = ConstantU32(state, state.program.scratch_dwords); return access; case IR::ResourceKind::ScalarAddress: @@ -167,8 +168,9 @@ uint32_t EmitMemoryElementPointer(EmitterState& state, const MemoryResourceAcces if (access.kind == IR::ResourceKind::Lds || access.kind == IR::ResourceKind::Scratch) { const auto pointer = state.builder.AllocateId(); const auto storage_class = access.kind == IR::ResourceKind::Scratch ? StorageClassFunction - : state.stage == ShaderType::Compute ? StorageClassWorkgroup - : StorageClassFunction; + : ShaderWorkgroupInput(state.stage, state.input_info) != nullptr + ? StorageClassWorkgroup + : StorageClassFunction; state.builder.AddFunction({OpAccessChain, TypeU32ElementPointer(state, storage_class), pointer, access.object_pointer, index}); return pointer; diff --git a/src/graphics/shader/recompiler/backend/spirv/spirvEmitterMesh.cpp b/src/graphics/shader/recompiler/backend/spirv/spirvEmitterMesh.cpp new file mode 100644 index 000000000..d8885bde3 --- /dev/null +++ b/src/graphics/shader/recompiler/backend/spirv/spirvEmitterMesh.cpp @@ -0,0 +1,186 @@ +#include "graphics/shader/recompiler/backend/spirv/spirvEmitterInternal.h" + +namespace Libs::Graphics::ShaderRecompiler::Spirv::Emitter { +namespace { + +uint32_t MeshArray(EmitterState& state, uint32_t storage, uint32_t type, uint32_t count) { + const auto array = state.builder.Type(OpTypeArray, {type, ConstantU32(state, count)}); + return state.builder.DefineGlobalVariable(TypePointer(state, storage, array), storage); +} + +uint32_t MeshElement(EmitterState& state, uint32_t variable, uint32_t storage, uint32_t type, + uint32_t index) { + const auto pointer = state.builder.AllocateId(); + state.builder.AddFunction( + {OpAccessChain, TypePointer(state, storage, type), pointer, variable, index}); + return pointer; +} + +uint32_t MeshLoad(EmitterState& state, uint32_t variable, uint32_t type, uint32_t index) { + const auto pointer = MeshElement(state, variable, StorageClassWorkgroup, type, index); + const auto value = state.builder.AllocateId(); + state.builder.AddFunction({OpLoad, type, value, pointer}); + return value; +} + +uint32_t MeshOutputType(EmitterState& state, IR::StageOutputKind kind) { + return kind == IR::StageOutputKind::Layer ? TypeU32(state) : TypeF32Vector(state, 4); +} + +} // namespace + +void DefineMeshOutputs(EmitterState& state) { + const auto& mesh = state.input_info.vertex->mesh; + for (auto& output: state.outputs) { + if (output.kind != IR::StageOutputKind::Position && + output.kind != IR::StageOutputKind::Parameter && + output.kind != IR::StageOutputKind::Layer) { + EXIT("unsupported mesh output kind=%u\n", static_cast(output.kind)); + } + const auto type = MeshOutputType(state, output.kind); + output.variable_id = MeshArray( + state, StorageClassOutput, type, + output.kind == IR::StageOutputKind::Layer ? mesh.max_primitives : mesh.max_vertices); + output.mesh_data_variable = + MeshArray(state, StorageClassWorkgroup, type, mesh.max_vertices); + state.interface_variables.push_back(output.variable_id); + state.builder.AddName(output.variable_id, output.debug_name.c_str()); + if (output.kind == IR::StageOutputKind::Parameter) { + state.builder.AddAnnotation( + {OpDecorate, output.variable_id, DecorationLocation, output.location}); + } else { + state.builder.AddAnnotation( + {OpDecorate, output.variable_id, DecorationBuiltIn, + output.kind == IR::StageOutputKind::Layer ? BuiltInLayer : BuiltInPosition}); + } + if (output.kind == IR::StageOutputKind::Layer) { + state.builder.AddAnnotation({OpDecorate, output.variable_id, 5271u}); // PerPrimitiveEXT + } + } + state.mesh_allocation = MeshArray(state, StorageClassWorkgroup, TypeU32(state), 2); + state.mesh_primitive_data = + MeshArray(state, StorageClassWorkgroup, TypeU32(state), mesh.max_primitives); + state.mesh_primitives = + MeshArray(state, StorageClassOutput, TypeU32Vector(state, 3), mesh.max_primitives); + state.mesh_cull = MeshArray(state, StorageClassOutput, TypeBool(state), mesh.max_primitives); + state.interface_variables.push_back(state.mesh_primitives); + state.interface_variables.push_back(state.mesh_cull); + state.builder.AddAnnotation({OpDecorate, state.mesh_primitives, DecorationBuiltIn, + 5296u}); // PrimitiveTriangleIndicesEXT + state.builder.AddAnnotation( + {OpDecorate, state.mesh_cull, DecorationBuiltIn, 5299u}); // CullPrimitiveEXT + state.builder.AddAnnotation({OpDecorate, state.mesh_cull, 5271u}); +} + +uint32_t MeshOutputPointer(EmitterState& state, IR::StageOutputKind kind, uint32_t index) { + const auto output = std::ranges::find_if(state.outputs, [=](const OutputBinding& binding) { + return binding.kind == kind && binding.index == index; + }); + if (output == state.outputs.end()) { + EXIT("mesh export has no output binding: kind=%u index=%u\n", static_cast(kind), + index); + } + return MeshElement(state, output->mesh_data_variable, StorageClassWorkgroup, + MeshOutputType(state, kind), EmitLocalInvocationIndex(state)); +} + +uint32_t MeshPrimitivePointer(EmitterState& state) { + return MeshElement(state, state.mesh_primitive_data, StorageClassWorkgroup, TypeU32(state), + EmitLocalInvocationIndex(state)); +} + +void EmitMeshAllocate(ValueEmitContext& ctx, const IR::Inst& inst) { + auto& state = ctx.state; + const auto first = state.builder.AllocateId(); + state.builder.AddFunction( + {OpIEqual, TypeBool(state), first, EmitLocalInvocationIndex(state), ConstantU32(state, 0)}); + EmitIfCondition(state, first, [&] { + const auto allocation = ctx.Arg(inst, 0); + for (uint32_t field = 0; field < 2; field++) { + const auto value = state.builder.AllocateId(); + state.builder.AddFunction({OpBitFieldUExtract, TypeU32(state), value, allocation, + ConstantU32(state, field * 12u), + ConstantU32(state, field == 0 ? 10u : 11u)}); + const auto pointer = MeshElement(state, state.mesh_allocation, StorageClassWorkgroup, + TypeU32(state), ConstantU32(state, field)); + state.builder.AddFunction({OpStore, pointer, value}); + } + }); +} + +void EmitMeshEntryPoint(EmitterState& state) { + state.builder.AddFunction( + {OpFunction, TypeVoid(state), state.main_func, FunctionControlNone, TypeFunction(state)}); + EmitLabel(state, state.builder.AllocateId()); + state.builder.AddFunction( + {OpFunctionCall, TypeVoid(state), state.builder.AllocateId(), state.mesh_guest_func}); + // All guest waves finish before the uniform Vulkan allocation and output stores. + state.builder.AddFunction( + {OpControlBarrier, ConstantU32(state, ScopeWorkgroup), ConstantU32(state, ScopeWorkgroup), + ConstantU32(state, MemorySemanticsAcquireRelease | MemorySemanticsWorkgroupMemory)}); + const auto vertices = + MeshLoad(state, state.mesh_allocation, TypeU32(state), ConstantU32(state, 0)); + const auto primitives = + MeshLoad(state, state.mesh_allocation, TypeU32(state), ConstantU32(state, 1)); + state.builder.AddFunction({5295u, vertices, primitives}); // OpSetMeshOutputsEXT + for (uint32_t half = 0; half < state.lane_count; half++) { + state.lane_half = half; + const auto index = EmitLocalInvocationIndex(state); + const auto is_vertex = state.builder.AllocateId(); + state.builder.AddFunction({OpULessThan, TypeBool(state), is_vertex, index, vertices}); + EmitIfCondition(state, is_vertex, [&] { + for (const auto& output: state.outputs) { + if (output.kind == IR::StageOutputKind::Layer) { + continue; + } + const auto type = MeshOutputType(state, output.kind); + const auto value = MeshLoad(state, output.mesh_data_variable, type, index); + const auto pointer = + MeshElement(state, output.variable_id, StorageClassOutput, type, index); + state.builder.AddFunction({OpStore, pointer, value}); + } + }); + const auto is_primitive = state.builder.AllocateId(); + state.builder.AddFunction({OpULessThan, TypeBool(state), is_primitive, index, primitives}); + EmitIfCondition(state, is_primitive, [&] { + const auto packed = MeshLoad(state, state.mesh_primitive_data, TypeU32(state), index); + uint32_t vertex[3] {}; + for (uint32_t component = 0; component < 3; component++) { + vertex[component] = state.builder.AllocateId(); + state.builder.AddFunction({OpBitFieldUExtract, TypeU32(state), vertex[component], + packed, ConstantU32(state, component * 10u), + ConstantU32(state, 10)}); + } + const auto triangle = state.builder.AllocateId(); + state.builder.AddFunction({OpCompositeConstruct, TypeU32Vector(state, 3), triangle, + vertex[0], vertex[1], vertex[2]}); + const auto triangle_pointer = MeshElement( + state, state.mesh_primitives, StorageClassOutput, TypeU32Vector(state, 3), index); + state.builder.AddFunction({OpStore, triangle_pointer, triangle}); + const auto null_bit = + EmitBinaryU32(state, OpBitwiseAnd, packed, ConstantU32(state, 0x80000000u)); + const auto culled = state.builder.AllocateId(); + state.builder.AddFunction( + {OpINotEqual, TypeBool(state), culled, null_bit, ConstantU32(state, 0)}); + state.builder.AddFunction( + {OpStore, + MeshElement(state, state.mesh_cull, StorageClassOutput, TypeBool(state), index), + culled}); + for (const auto& output: state.outputs) { + if (output.kind != IR::StageOutputKind::Layer) { + continue; + } + const auto layer = MeshLoad(state, output.mesh_data_variable, TypeU32(state), + vertex[state.input_info.vertex->mesh.provoking_vertex]); + const auto pointer = MeshElement(state, output.variable_id, StorageClassOutput, + TypeU32(state), index); + state.builder.AddFunction({OpStore, pointer, layer}); + } + }); + } + state.lane_half = 0; + state.builder.AddFunction({OpReturn}); + state.builder.AddFunction({OpFunctionEnd}); +} + +} // namespace Libs::Graphics::ShaderRecompiler::Spirv::Emitter diff --git a/src/graphics/shader/recompiler/backend/spirv/spirvEmitterModule.cpp b/src/graphics/shader/recompiler/backend/spirv/spirvEmitterModule.cpp index 2fc66ee14..0ee81495b 100644 --- a/src/graphics/shader/recompiler/backend/spirv/spirvEmitterModule.cpp +++ b/src/graphics/shader/recompiler/backend/spirv/spirvEmitterModule.cpp @@ -202,7 +202,7 @@ void DefineDescriptorVariables(EmitterState& state) { state.fault_buffer_variable = state.builder.DefineGlobalVariable( TypeStorageBufferPointer(state), StorageClassStorageBuffer); } - if (state.program.bindings.UsesPushData()) { + if (state.program.bindings.UsesPushData() || state.stage == ShaderType::Mesh) { const auto pointer_type = TypePointer(state, StorageClassPushConstant, PushConstantBlockType(state)); state.push_constant_variable = @@ -430,6 +430,22 @@ static bool MrtUsesUintOutput(const EmitterState& state, uint32_t index) { } void AllocateInputVariables(EmitterState& state) { + if (state.lane_count == 2) { + const auto add_builtin = [&](IR::StageInputKind kind, uint32_t components, + const char* name) { + if (std::ranges::none_of(state.inputs, [kind](const InputBinding& input) { + return input.kind == kind; + })) { + state.inputs.push_back({kind, 0, components, 0, name}); + } + }; + add_builtin(IR::StageInputKind::LocalInvocationIndex, 1, "gl_LocalInvocationIndex"); + if (std::ranges::any_of(state.inputs, [](const InputBinding& input) { + return input.kind == IR::StageInputKind::GlobalInvocationId; + })) { + add_builtin(IR::StageInputKind::WorkgroupId, 3, "gl_WorkGroupID"); + } + } for (auto& binding: state.inputs) { binding.variable_id = state.builder.AllocateId(); state.interface_variables.push_back(binding.variable_id); @@ -454,6 +470,10 @@ static uint32_t AllocateSharedOutputVariable(EmitterState& state, uint32_t& vari } void AllocateOutputVariables(EmitterState& state) { + if (state.stage == ShaderType::Mesh) { + DefineMeshOutputs(state); + return; + } for (auto& binding: state.outputs) { switch (binding.kind) { case IR::StageOutputKind::Position: @@ -548,6 +568,9 @@ void AddInputAnnotationsAndNames(EmitterState& state) { } void AddOutputAnnotationsAndNames(EmitterState& state) { + if (state.stage == ShaderType::Mesh) { + return; + } if (state.per_vertex_variable != 0) { state.builder.AddName(PerVertexType(state), "gl_PerVertex"); state.builder.AddName(state.per_vertex_variable, "outPerVertex"); @@ -653,9 +676,21 @@ void DefineModule(EmitterState& state) { state.lds_variable = state.builder.AllocateId(); } if (state.requirements.function_scratch) { - state.scratch_variable = state.builder.AllocateId(); + for (uint32_t half = 0; half < state.lane_count; half++) { + state.scratch_variable[half] = state.builder.AllocateId(); + } } state.main_func = state.builder.AllocateId(); + if (state.stage == ShaderType::Mesh) { + state.mesh_guest_func = state.builder.AllocateId(); + state.builder.RequireCapability(5283u); // MeshShadingEXT + state.builder.RequireExtension("SPV_EXT_mesh_shader"); + state.builder.AddExecutionMode({state.main_func, 5298u}); // OutputTrianglesEXT + state.builder.AddExecutionMode( + {state.main_func, 26u, state.input_info.vertex->mesh.max_vertices}); + state.builder.AddExecutionMode( + {state.main_func, 5270u, state.input_info.vertex->mesh.max_primitives}); + } state.entry_label = state.builder.AllocateId(); state.builder.RequireCapability(CapabilityShader); @@ -682,11 +717,11 @@ void DefineModule(EmitterState& state) { if (state.requirements.image_gather_extended) { state.builder.RequireCapability(CapabilityImageGatherExtended); } - if (state.requirements.subgroup_ballot || state.requirements.subgroup_shuffle || - state.requirements.subgroup_local_invocation_id) { + if (state.lane_count == 2 || state.requirements.subgroup_ballot || + state.requirements.subgroup_shuffle || state.requirements.subgroup_local_invocation_id) { state.builder.RequireCapability(CapabilityGroupNonUniform); } - if (state.requirements.subgroup_ballot) { + if (state.lane_count == 2 || state.requirements.subgroup_ballot) { state.builder.RequireCapability(CapabilityGroupNonUniformBallot); } if (state.requirements.subgroup_shuffle) { @@ -711,19 +746,24 @@ void DefineModule(EmitterState& state) { {state.program.info.uses_dma ? AddressingModelPhysicalStorageBuffer64 : AddressingModelLogical, MemoryModelGLSL450}); - state.builder.AddEntryPoint(ExecutionModelForStage(state.stage), state.main_func, "main", - state.interface_variables); // GCN/RDNA arithmetic preserves 32-bit signed zero, infinity, and NaN. Declaring that // contract prevents host compilers from treating synthesized IEEE values as finite. state.builder.AddExecutionMode({state.main_func, ExecutionModeSignedZeroInfNanPreserve, 32u}); - if (state.stage == ShaderType::Compute) { + if (const auto* cs = ShaderWorkgroupInput(state.stage, state.input_info)) { uint32_t local_x = state.requirements.compute_derivatives ? 2u : 1u; uint32_t local_y = state.requirements.compute_derivatives ? 2u : 1u; uint32_t local_z = 1u; - const auto* cs = state.input_info.compute; local_x = cs->threads_num[0] != 0u ? cs->threads_num[0] : local_x; local_y = cs->threads_num[1] != 0u ? cs->threads_num[1] : local_y; local_z = cs->threads_num[2] != 0u ? cs->threads_num[2] : local_z; + if (state.lane_count == 2) { + local_x = ((local_x * local_y * local_z + 63u) / 64u) * 32u; + local_y = local_z = 1u; + if (state.requirements.compute_derivatives) { + local_y = local_x / 2u; + local_x = 2u; + } + } state.builder.AddExecutionMode( {state.main_func, ExecutionModeLocalSize, local_x, local_y, local_z}); } @@ -746,7 +786,9 @@ void DefineModule(EmitterState& state) { state.builder.AddName(state.lds_variable, "lds_dwords"); } if (state.requirements.function_scratch) { - state.builder.AddName(state.scratch_variable, "scratch_dwords"); + for (uint32_t half = 0; half < state.lane_count; half++) { + state.builder.AddName(state.scratch_variable[half], "scratch_dwords"); + } } AddInputAnnotationsAndNames(state); AddOutputAnnotationsAndNames(state); @@ -797,6 +839,9 @@ void DefineModule(EmitterState& state) { } state.builder.DefineGlobalVariable(input.variable_id, ptr_type, StorageClassInput); } + if (state.stage == ShaderType::Mesh) { + return; + } if (state.per_vertex_variable != 0) { state.builder.DefineGlobalVariable( state.per_vertex_variable, TypePointer(state, StorageClassOutput, PerVertexType(state)), diff --git a/src/graphics/shader/recompiler/backend/spirv/spirvEmitterProgram.cpp b/src/graphics/shader/recompiler/backend/spirv/spirvEmitterProgram.cpp index 48e0fb66a..44de10c50 100644 --- a/src/graphics/shader/recompiler/backend/spirv/spirvEmitterProgram.cpp +++ b/src/graphics/shader/recompiler/backend/spirv/spirvEmitterProgram.cpp @@ -45,6 +45,7 @@ uint32_t SpillPointerType(ValueEmitContext& ctx, IR::Type type) { struct DeferredPhiPatch { DeferredPhi phi; const IR::Inst* instruction = nullptr; + uint32_t half = 0; }; struct StructuredFunctionState { @@ -53,7 +54,7 @@ struct StructuredFunctionState { }; struct DispatcherFunctionState { - std::unordered_map spills; + std::array, 2> spills; uint32_t header_label = 0; uint32_t select_label = 0; uint32_t after_switch_label = 0; @@ -73,7 +74,7 @@ void StoreDispatcherPhiEdge(ValueEmitContext& ctx, const DispatcherFunctionState for (size_t index = 0; index < phi.NumArgs(); index++) { if (phi.PhiBlock(index) == from) { ctx.state.builder.AddFunction( - {OpStore, dispatcher.spills.at(&phi), ctx.Def(phi.Arg(index))}); + {OpStore, dispatcher.spills[ctx.half].at(&phi), ctx.Def(phi.Arg(index))}); break; } } @@ -94,6 +95,27 @@ void EmitReturn(ValueEmitContext& ctx) { ctx.state.builder.AddFunction({OpReturn}); } +uint32_t BranchCondition(ValueEmitContext& ctx, const IR::BlockInfo& info) { + if (ctx.other_half == nullptr || + info.terminator.condition == CFG::BranchCondition::GotoVariable) { + return ctx.Def(info.condition); + } + const auto ballot = ctx.Ballot(info.condition); + const auto low = ctx.state.builder.AllocateId(); + const auto high = ctx.state.builder.AllocateId(); + const auto result = ctx.state.builder.AllocateId(); + ctx.state.builder.AddFunction({OpCompositeExtract, TypeU32(ctx.state), low, ballot, 0}); + ctx.state.builder.AddFunction({OpCompositeExtract, TypeU32(ctx.state), high, ballot, 1}); + const auto kind = info.terminator.condition; + const bool zero = kind == CFG::BranchCondition::ExecZero || + kind == CFG::BranchCondition::VccZero || + kind == CFG::BranchCondition::SccZero; + const auto combined = EmitBinaryU32(ctx.state, zero ? OpBitwiseAnd : OpBitwiseOr, low, high); + ctx.state.builder.AddFunction({zero ? OpIEqual : OpINotEqual, TypeBool(ctx.state), result, + combined, ConstantU32(ctx.state, zero ? ~0u : 0u)}); + return result; +} + void EmitStructuredTerminator(ValueEmitContext& ctx, const IR::Block* block, const IR::BlockInfo& info) { const auto& term = info.terminator; @@ -132,7 +154,7 @@ void EmitStructuredTerminator(ValueEmitContext& ctx, const IR::Block* block, EmitReturn(ctx); return; } - const auto condition = ctx.Def(info.condition); + const auto condition = BranchCondition(ctx, info); emit_merge(); ctx.state.builder.AddFunction( {OpBranchConditional, condition, ctx.Label(true_block), ctx.Label(false_block)}); @@ -147,6 +169,9 @@ void EmitDispatcherTarget(ValueEmitContext& ctx, const DispatcherFunctionState& const auto* block = TargetBlock(ctx.program, target); if (block != nullptr) { StoreDispatcherPhiEdge(ctx, dispatcher, from, block); + if (ctx.other_half != nullptr) { + StoreDispatcherPhiEdge(*ctx.other_half, dispatcher, from, block); + } } } @@ -162,7 +187,7 @@ uint32_t EmitDispatcherNextPc(ValueEmitContext& ctx, const DispatcherFunctionSta EmitDispatcherTarget(ctx, dispatcher, block, term.false_block); const auto selected = ctx.state.builder.AllocateId(); ctx.state.builder.AddFunction({OpSelect, TypeU32(ctx.state), selected, - ctx.Def(info.condition), + BranchCondition(ctx, info), ConstantU32(ctx.state, term.true_block), ConstantU32(ctx.state, term.false_block)}); return selected; @@ -219,7 +244,8 @@ void EmitStructuredInstruction(ValueEmitContext& ctx, StructuredFunctionState& s } } structured.deferred_phis.push_back( - {ctx.state.builder.AddDeferredPhi(type, ctx.Result(inst), inst.NumArgs()), &inst}); + {ctx.state.builder.AddDeferredPhi(type, ctx.Result(inst), inst.NumArgs()), &inst, + ctx.half}); return; } EmitDirectInstruction(ctx, inst); @@ -233,11 +259,12 @@ void EmitDispatcherInstruction(ValueEmitContext& ctx, const DispatcherFunctionSt ctx.Fail(inst, "cannot be loaded by the dispatcher"); } ctx.state.builder.AddFunction( - {OpLoad, type, ctx.Result(inst), dispatcher.spills.at(&inst)}); + {OpLoad, type, ctx.Result(inst), dispatcher.spills[ctx.half].at(&inst)}); return; } EmitDirectInstruction(ctx, inst); - if (const auto found = dispatcher.spills.find(&inst); found != dispatcher.spills.end()) { + if (const auto found = dispatcher.spills[ctx.half].find(&inst); + found != dispatcher.spills[ctx.half].end()) { ctx.state.builder.AddFunction( {OpStore, found->second, ctx.Def(IR::Value(const_cast(&inst)))}); } @@ -246,6 +273,9 @@ void EmitDispatcherInstruction(ValueEmitContext& ctx, const DispatcherFunctionSt template void EmitBlock(ValueEmitContext& ctx, const IR::Block* block, EmitInstruction&& emit_instruction) { ctx.current_block = block; + if (ctx.other_half != nullptr) { + ctx.other_half->current_block = block; + } EmitLabel(ctx.state, ctx.Label(block)); bool emitted_non_phi = false; for (const auto& inst: *block) { @@ -256,12 +286,22 @@ void EmitBlock(ValueEmitContext& ctx, const IR::Block* block, EmitInstruction&& } else { emitted_non_phi = true; } - emit_instruction(inst); + for (uint32_t half = 0; half < ctx.state.lane_count; half++) { + auto& lane = half == 0 ? ctx : *ctx.other_half; + lane.current_block = block; + ctx.state.lane_half = half; + if (half == 0 || (inst.GetOpcode() != IR::ValueOpcode::Barrier && + inst.GetOpcode() != IR::ValueOpcode::MeshAllocate)) { + emit_instruction(lane, inst); + } + } + ctx.state.lane_half = 0; } } void PatchStructuredPhis(ValueEmitContext& ctx, StructuredFunctionState& structured) { for (const auto& deferred: structured.deferred_phis) { + auto& lane = deferred.half == 0 ? ctx : *ctx.other_half; for (size_t index = 0; index < deferred.instruction->NumArgs(); index++) { const auto* predecessor = deferred.instruction->PhiBlock(index); const auto found = structured.block_exit_labels.find(predecessor); @@ -269,7 +309,7 @@ void PatchStructuredPhis(ValueEmitContext& ctx, StructuredFunctionState& structu ctx.Fail(*deferred.instruction, "has a predecessor that was not emitted"); } ctx.state.builder.PatchDeferredPhi( - deferred.phi, index, ctx.Def(deferred.instruction->Arg(index)), found->second); + deferred.phi, index, lane.Def(deferred.instruction->Arg(index)), found->second); } } } @@ -279,8 +319,8 @@ void EmitStructuredFunction(ValueEmitContext& ctx) { ctx.state.builder.AddFunction({OpBranch, ctx.Label(ctx.program.blocks.front())}); for (size_t index = 0; index < ctx.program.blocks.size(); index++) { const auto* block = ctx.program.blocks[index]; - EmitBlock(ctx, block, [&](const IR::Inst& inst) { - EmitStructuredInstruction(ctx, structured, inst); + EmitBlock(ctx, block, [&](ValueEmitContext& lane, const IR::Inst& inst) { + EmitStructuredInstruction(lane, structured, inst); }); structured.block_exit_labels.emplace(block, ctx.state.current_label); EmitStructuredTerminator(ctx, block, ctx.program.block_info[index]); @@ -292,8 +332,8 @@ void EmitDispatcherFunction(ValueEmitContext& ctx, const DispatcherFunctionState auto& state = ctx.state; const auto* entry = ctx.program.blocks.front(); state.builder.AddFunction({OpBranch, ctx.Label(entry)}); - EmitBlock(ctx, entry, [&](const IR::Inst& inst) { - EmitDispatcherInstruction(ctx, dispatcher, inst); + EmitBlock(ctx, entry, [&](ValueEmitContext& lane, const IR::Inst& inst) { + EmitDispatcherInstruction(lane, dispatcher, inst); }); const auto initial_pc = EmitDispatcherNextPc(ctx, dispatcher, entry, ctx.program.block_info.front()); @@ -326,9 +366,10 @@ void EmitDispatcherFunction(ValueEmitContext& ctx, const DispatcherFunctionState ConstantU32(state, UINT32_MAX), dispatcher.select_label}; for (size_t index = 1; index < ctx.program.blocks.size(); index++) { - EmitBlock(ctx, ctx.program.blocks[index], [&](const IR::Inst& inst) { - EmitDispatcherInstruction(ctx, dispatcher, inst); - }); + EmitBlock(ctx, ctx.program.blocks[index], + [&](ValueEmitContext& lane, const IR::Inst& inst) { + EmitDispatcherInstruction(lane, dispatcher, inst); + }); const auto selected = EmitDispatcherNextPc(ctx, dispatcher, ctx.program.blocks[index], ctx.program.block_info[index]); next_pc_words.push_back(selected); @@ -408,6 +449,83 @@ uint32_t ValueEmitContext::Arg(const IR::Inst& inst, size_t index) { return Def(inst.Arg(index)); } +uint32_t ValueEmitContext::HalfArg(const IR::Inst& inst, size_t index, uint32_t lane_half) { + return lane_half == half ? Arg(inst, index) : other_half->Arg(inst, index); +} + +uint32_t ValueEmitContext::Ballot(IR::Value predicate) { + const auto ballot_type = TypeU32Vector(state, 4); + const auto scope = ConstantU32(state, ScopeSubgroup); + const auto low = state.builder.AllocateId(); + state.builder.AddFunction( + {OpGroupNonUniformBallot, ballot_type, low, scope, + other_half == nullptr || half == 0 ? Def(predicate) : other_half->Def(predicate)}); + if (other_half == nullptr) { + return low; + } + const auto high = state.builder.AllocateId(); + const auto low_word = state.builder.AllocateId(); + const auto high_word = state.builder.AllocateId(); + const auto ballot = state.builder.AllocateId(); + state.builder.AddFunction({OpGroupNonUniformBallot, ballot_type, high, scope, + half == 1 ? Def(predicate) : other_half->Def(predicate)}); + state.builder.AddFunction({OpCompositeExtract, TypeU32(state), low_word, low, 0}); + state.builder.AddFunction({OpCompositeExtract, TypeU32(state), high_word, high, 0}); + state.builder.AddFunction({OpCompositeConstruct, ballot_type, ballot, low_word, high_word, + ConstantU32(state, 0), ConstantU32(state, 0)}); + return ballot; +} + +uint32_t ValueEmitContext::FirstLane(uint32_t ballot) { + if (other_half == nullptr) { + const auto result = state.builder.AllocateId(); + state.builder.AddFunction({OpGroupNonUniformBallotFindLSB, TypeU32(state), result, + ConstantU32(state, ScopeSubgroup), ballot}); + return result; + } + const auto low = state.builder.AllocateId(); + const auto high = state.builder.AllocateId(); + const auto low_first = state.builder.AllocateId(); + const auto high_first = state.builder.AllocateId(); + const auto low_active = state.builder.AllocateId(); + const auto result = state.builder.AllocateId(); + state.builder.AddFunction({OpCompositeExtract, TypeU32(state), low, ballot, 0}); + state.builder.AddFunction({OpCompositeExtract, TypeU32(state), high, ballot, 1}); + state.builder.AddFunction( + {OpExtInst, TypeU32(state), low_first, GlslStd450(state), GlslFindILsb, low}); + state.builder.AddFunction( + {OpExtInst, TypeU32(state), high_first, GlslStd450(state), GlslFindILsb, high}); + state.builder.AddFunction( + {OpINotEqual, TypeBool(state), low_active, low, ConstantU32(state, 0)}); + state.builder.AddFunction({OpSelect, TypeU32(state), result, low_active, low_first, + EmitAddU32(state, high_first, ConstantU32(state, 32))}); + return result; +} + +uint32_t ValueEmitContext::Shuffle(const IR::Inst& inst, size_t index, uint32_t lane) { + const auto type = TypeId(inst.Arg(index).GetType()); + const auto scope = ConstantU32(state, ScopeSubgroup); + const auto low = state.builder.AllocateId(); + if (other_half == nullptr) { + state.builder.AddFunction( + {OpGroupNonUniformShuffle, type, low, scope, Arg(inst, index), lane}); + return low; + } + const auto physical_lane = EmitBinaryU32(state, OpBitwiseAnd, lane, ConstantU32(state, 31)); + const auto high = state.builder.AllocateId(); + const auto in_high = state.builder.AllocateId(); + const auto value = state.builder.AllocateId(); + state.builder.AddFunction( + {OpGroupNonUniformShuffle, type, low, scope, HalfArg(inst, index, 0), physical_lane}); + state.builder.AddFunction( + {OpGroupNonUniformShuffle, type, high, scope, HalfArg(inst, index, 1), physical_lane}); + state.builder.AddFunction({OpINotEqual, TypeBool(state), in_high, + EmitBinaryU32(state, OpBitwiseAnd, lane, ConstantU32(state, 32)), + ConstantU32(state, 0)}); + state.builder.AddFunction({OpSelect, type, value, in_high, high, low}); + return value; +} + uint32_t ValueEmitContext::Result(const IR::Inst& inst) { if (const auto found = definitions.find(&inst); found != definitions.end()) { return found->second; @@ -478,14 +596,22 @@ uint32_t ValueEmitContext::Label(const IR::Block* block) const { } void EmitProgram(EmitterState& state, const IR::Program& program) { - ValueEmitContext ctx(state, program); + ValueEmitContext ctx(state, program); + ValueEmitContext high(state, program); + if (state.lane_count == 2) { + ctx.other_half = &high; + high.other_half = &ctx; + high.half = 1; + } std::optional dispatcher; if (state.stage == ShaderType::Pixel && state.requirements.pixel_valid_mask) { state.pixel_valid_mask_variable = state.builder.AllocateId(); state.builder.AddName(state.pixel_valid_mask_variable, "pixel_valid_mask_active"); } for (const auto* block: program.blocks) { - ctx.labels.emplace(block, state.builder.AllocateId()); + const auto label = state.builder.AllocateId(); + ctx.labels.emplace(block, label); + high.labels.emplace(block, label); } if (state.program.dispatcher_fallback) { auto& dispatch = dispatcher.emplace(); @@ -498,7 +624,7 @@ void EmitProgram(EmitterState& state, const IR::Program& program) { ctx.Fail(inst, "cannot be stored by the dispatcher"); break; } - dispatch.spills.emplace(&inst, state.builder.AllocateId()); + dispatch.spills[0].emplace(&inst, state.builder.AllocateId()); } } const auto mark_cross_block = [&](IR::Value value, const IR::Block* consumer) { @@ -512,8 +638,8 @@ void EmitProgram(EmitterState& state, const IR::Program& program) { ctx.Fail(*definition, "cannot be stored by the dispatcher"); return; } - if (!dispatch.spills.contains(definition)) { - dispatch.spills.emplace(definition, state.builder.AllocateId()); + if (!dispatch.spills[0].contains(definition)) { + dispatch.spills[0].emplace(definition, state.builder.AllocateId()); } }; for (const auto* block: program.blocks) { @@ -534,7 +660,13 @@ void EmitProgram(EmitterState& state, const IR::Program& program) { dispatch.after_switch_label = state.builder.AllocateId(); dispatch.continue_label = state.builder.AllocateId(); dispatch.merge_label = state.builder.AllocateId(); - ctx.dispatcher_spills = &dispatch.spills; + ctx.dispatcher_spills = &dispatch.spills[0]; + if (state.lane_count == 2) { + for (const auto& [inst, id]: dispatch.spills[0]) { + dispatch.spills[1].emplace(inst, state.builder.AllocateId()); + } + high.dispatcher_spills = &dispatch.spills[1]; + } } DefineGetBdaPointer(state); for (const auto* block: program.blocks) { @@ -544,11 +676,15 @@ void EmitProgram(EmitterState& state, const IR::Program& program) { inst.GetOpcode() == IR::ValueOpcode::SharedAtomicFMax32; })) { ctx.scratch_u32_variable = state.builder.AllocateId(); + if (state.lane_count == 2) { + high.scratch_u32_variable = state.builder.AllocateId(); + } break; } } - state.builder.AddFunction( - {OpFunction, TypeVoid(state), state.main_func, FunctionControlNone, TypeFunction(state)}); + state.builder.AddFunction({OpFunction, TypeVoid(state), + state.mesh_guest_func != 0 ? state.mesh_guest_func : state.main_func, + FunctionControlNone, TypeFunction(state)}); EmitLabel(state, state.entry_label); if (state.requirements.function_lds) { state.builder.AddFunction( @@ -556,31 +692,37 @@ void EmitProgram(EmitterState& state, const IR::Program& program) { state.lds_variable, StorageClassFunction}); } if (state.requirements.function_scratch) { - state.builder.AddFunction( - {OpVariable, - TypeU32ArrayPointer(state, StorageClassFunction, state.program.scratch_dwords), - state.scratch_variable, StorageClassFunction}); + for (uint32_t half = 0; half < state.lane_count; half++) { + state.builder.AddFunction( + {OpVariable, + TypeU32ArrayPointer(state, StorageClassFunction, state.program.scratch_dwords), + state.scratch_variable[half], StorageClassFunction}); + } } if (state.pixel_valid_mask_variable != 0) { state.builder.AddFunction({OpVariable, TypePointer(state, StorageClassFunction, TypeU32(state)), state.pixel_valid_mask_variable, StorageClassFunction}); } - if (state.program.dispatcher_fallback) { - for (const auto* block: program.blocks) { - for (const auto& inst: *block) { - if (const auto found = dispatcher->spills.find(&inst); - found != dispatcher->spills.end()) { - state.builder.AddFunction({OpVariable, SpillPointerType(ctx, inst.GetType()), - found->second, StorageClassFunction}); + for (uint32_t half = 0; half < state.lane_count; half++) { + auto& lane = half == 0 ? ctx : high; + if (state.program.dispatcher_fallback) { + for (const auto* block: program.blocks) { + for (const auto& inst: *block) { + if (const auto found = dispatcher->spills[half].find(&inst); + found != dispatcher->spills[half].end()) { + state.builder.AddFunction({OpVariable, + SpillPointerType(lane, inst.GetType()), + found->second, StorageClassFunction}); + } } } } - } - if (ctx.scratch_u32_variable != 0) { - state.builder.AddFunction({OpVariable, - TypePointer(state, StorageClassFunction, TypeU32(state)), - ctx.scratch_u32_variable, StorageClassFunction}); + if (lane.scratch_u32_variable != 0) { + state.builder.AddFunction({OpVariable, + TypePointer(state, StorageClassFunction, TypeU32(state)), + lane.scratch_u32_variable, StorageClassFunction}); + } } if (state.gds_variable != 0) { state.gds_length = state.builder.AllocateId(); @@ -600,6 +742,9 @@ void EmitProgram(EmitterState& state, const IR::Program& program) { EmitStructuredFunction(ctx); } state.builder.AddFunction({OpFunctionEnd}); + if (state.stage == ShaderType::Mesh) { + EmitMeshEntryPoint(state); + } } } // namespace Libs::Graphics::ShaderRecompiler::Spirv::Emitter diff --git a/src/graphics/shader/recompiler/frontend/translate/Control.cpp b/src/graphics/shader/recompiler/frontend/translate/Control.cpp index 8be7e2a93..1cbd6ee1d 100644 --- a/src/graphics/shader/recompiler/frontend/translate/Control.cpp +++ b/src/graphics/shader/recompiler/frontend/translate/Control.cpp @@ -28,17 +28,23 @@ void Translator::S_SAVEEXEC(const Decoder::Instruction& inst, IR::ValueOpcode op const auto src = ReadMask(inst.src0); const auto lhs = negate_exec ? ir.LogicalNot(old) : old; const auto rhs = negate_source ? ir.LogicalNot(src) : src; - const auto result = IR::U1(ir.Emit(operation, {lhs, rhs})); + auto result = IR::U1(ir.Emit(operation, {lhs, rhs})); if (write_64) { - WriteMask64(inst.dst, old); + WriteMask(inst.dst, old, true); } else { - WriteMask(inst.dst, old); + WriteRawU32(inst.dst, ir.GetExecLo()); + if (current_wave_size == 64u) { + const auto low_half = + ir.ULessThan(IR::U32(ir.Emit(IR::ValueOpcode::LaneId)), IR::U32(IR::Value(32u))); + result = IR::U1(ir.Emit(IR::ValueOpcode::SelectU1, {low_half, result, old})); + } } const auto mask = BallotMask(result); ir.SetExec(result); ir.SetExecLo(mask[0]); ir.SetExecHi(mask[1]); - ir.SetScc(result); + ir.SetScc( + ir.INotEqual(write_64 ? ir.BitwiseOr(mask[0], mask[1]) : mask[0], IR::U32(IR::Value(0u)))); } void Translator::ADD_U32(const Decoder::Instruction& inst, bool vector, bool use_carry_in) { @@ -149,8 +155,13 @@ void Translator::S_BARRIER() { ir.Emit(IR::ValueOpcode::Barrier); } -void Translator::S_SENDMSG() { - ir.Emit(IR::ValueOpcode::Sendmsg); +void Translator::S_SENDMSG(const Decoder::Instruction& inst) { + if (program.stage == ShaderType::Mesh) { + EXIT_NOT_IMPLEMENTED(inst.src0.value != 9u); // MSG_GS_ALLOC_REQ + ir.Emit(IR::ValueOpcode::MeshAllocate, {ir.GetM0()}); + } else { + ir.Emit(IR::ValueOpcode::Sendmsg); + } } void Translator::S_TTRACEDATA() { @@ -207,7 +218,7 @@ void Translator::S_CSELECT_B64(const Decoder::Instruction& inst) { IR::U1(ir.Emit(IR::ValueOpcode::SelectU1, {condition, ReadMaskValid(inst.src0), ReadMaskValid(inst.src1)})); if (IsExecOrVcc(inst.dst)) { - WriteMask64(inst.dst, selected_mask); + WriteMask(inst.dst, selected_mask, true); return; } WriteU32Pair(inst.dst, @@ -220,9 +231,7 @@ void Translator::S_CSELECT_B64(const Decoder::Instruction& inst) { } void Translator::MOV_B32(const Decoder::Instruction& inst, bool apply_float_modifiers) { - if (IsExecOrVcc(inst.src0) && IsExecOrVcc(inst.dst)) { - WriteMask(inst.dst, ReadMask(inst.src0)); - } else if (apply_float_modifiers && (inst.src0.negate || inst.src0.absolute)) { + if (apply_float_modifiers && (inst.src0.negate || inst.src0.absolute)) { WriteOperand(DestinationOperand(inst), ReadOperand(inst.src0, IR::Type::F32)); } else { WriteOperand(DestinationOperand(inst), ReadOperand(inst.src0, IR::Type::U32)); @@ -231,7 +240,7 @@ void Translator::MOV_B32(const Decoder::Instruction& inst, bool apply_float_modi void Translator::S_MOV_B64(const Decoder::Instruction& inst) { if (IsExecOrVcc(inst.dst) || IsExecOrVcc(inst.src0)) { - WriteMask64(inst.dst, ReadMask(inst.src0)); + WriteMask(inst.dst, ReadMask(inst.src0), true); return; } const bool scalar_copy = @@ -250,30 +259,16 @@ void Translator::S_MOV_B64(const Decoder::Instruction& inst) { } void Translator::S_WQM_B64(const Decoder::Instruction& inst) { - if (!IsExecOrVcc(inst.dst) && !IsExecOrVcc(inst.src0)) { - const auto mask_valid = ReadMaskValid(inst.src0); - const auto invocation_result = - IR::U1(ir.Emit(IR::ValueOpcode::WqmMask, {ReadMask(inst.src0)})); - const auto result = - IR::U64(ir.Emit(IR::ValueOpcode::WqmU64, {ReadOperand(inst.src0, IR::Type::U64)})); - WriteOperand(DestinationOperand(inst), result); - if (inst.dst.kind == Decoder::OperandKind::Sgpr) { - const auto dst = static_cast(inst.dst.reg); - ir.SetThreadBitScalarReg(dst, invocation_result); - ir.SetScalarMaskTag(dst, mask_valid); - const auto raw_nonzero = - IR::U1(ir.Emit(IR::ValueOpcode::INotEqual64, {result, IR::Value(uint64_t {0})})); - ir.SetScc(IR::U1( - ir.Emit(IR::ValueOpcode::SelectU1, {mask_valid, invocation_result, raw_nonzero}))); - } else { - ir.SetScc( - IR::U1(ir.Emit(IR::ValueOpcode::INotEqual64, {result, IR::Value(uint64_t {0})}))); - } - return; + const auto mask_valid = ReadMaskValid(inst.src0); + const auto result = + IR::U64(ir.Emit(IR::ValueOpcode::WqmU64, {ReadOperand(inst.src0, IR::Type::U64)})); + WriteOperand(DestinationOperand(inst), result); + if (inst.dst.kind == Decoder::OperandKind::Sgpr) { + const auto dst = static_cast(inst.dst.reg); + ir.SetThreadBitScalarReg(dst, ThreadBit(ExtractU64(result))); + ir.SetScalarMaskTag(dst, mask_valid); } - const auto result = IR::U1(ir.Emit(IR::ValueOpcode::WqmMask, {ReadMask(inst.src0)})); - WriteMask64(inst.dst, result); - ir.SetScc(result); + ir.SetScc(IR::U1(ir.Emit(IR::ValueOpcode::INotEqual64, {result, IR::Value(uint64_t {0})}))); } void Translator::V_MOVRELS_B32(const Decoder::Instruction& inst) { diff --git a/src/graphics/shader/recompiler/frontend/translate/Integer.cpp b/src/graphics/shader/recompiler/frontend/translate/Integer.cpp index 1f5714f94..f03f7a169 100644 --- a/src/graphics/shader/recompiler/frontend/translate/Integer.cpp +++ b/src/graphics/shader/recompiler/frontend/translate/Integer.cpp @@ -105,8 +105,8 @@ bool Translator::S_U64_MASK(const Decoder::Instruction& inst, IR::ValueOpcode lo }; if (is_exec_or_vcc(inst.dst) || is_exec_or_vcc(inst.src0) || (inst.src_count > 1u && is_exec_or_vcc(inst.src1))) { - WriteMask64(inst.dst, invocation_result); - ir.SetScc(invocation_result); + const auto mask = WriteMask(inst.dst, invocation_result, true); + ir.SetScc(ir.INotEqual(ir.BitwiseOr(mask[0], mask[1]), IR::U32(IR::Value(0u)))); return true; } @@ -132,13 +132,8 @@ bool Translator::S_U64_MASK(const Decoder::Instruction& inst, IR::ValueOpcode lo const auto dst = static_cast(inst.dst.reg); ir.SetThreadBitScalarReg(dst, invocation_result); ir.SetScalarMaskTag(dst, mask_valid); - const auto raw_nonzero = - ir.INotEqual(ir.BitwiseOr(result[0], result[1]), IR::U32(IR::Value(0u))); - ir.SetScc(IR::U1( - ir.Emit(IR::ValueOpcode::SelectU1, {mask_valid, invocation_result, raw_nonzero}))); - } else { - ir.SetScc(ir.INotEqual(ir.BitwiseOr(result[0], result[1]), IR::U32(IR::Value(0u)))); } + ir.SetScc(ir.INotEqual(ir.BitwiseOr(result[0], result[1]), IR::U32(IR::Value(0u)))); return true; } diff --git a/src/graphics/shader/recompiler/frontend/translate/Scalar.cpp b/src/graphics/shader/recompiler/frontend/translate/Scalar.cpp index 8a9265479..66b49ffa1 100644 --- a/src/graphics/shader/recompiler/frontend/translate/Scalar.cpp +++ b/src/graphics/shader/recompiler/frontend/translate/Scalar.cpp @@ -208,7 +208,7 @@ bool Translator::EmitScalar(const Decoder::Instruction& inst) { case O::S_TRAP: EmitControlNop(); return true; case O::S_WAITCNT_DEPCTR: EmitWaitcnt(); return true; case O::S_BARRIER: S_BARRIER(); return true; - case O::S_SENDMSG: S_SENDMSG(); return true; + case O::S_SENDMSG: S_SENDMSG(inst); return true; case O::S_TTRACEDATA: S_TTRACEDATA(); return true; case O::S_INST_PREFETCH: S_INST_PREFETCH(); return true; case O::S_BRANCH: diff --git a/src/graphics/shader/recompiler/frontend/translate/Translate.cpp b/src/graphics/shader/recompiler/frontend/translate/Translate.cpp index f42ebb41d..db11eae0b 100644 --- a/src/graphics/shader/recompiler/frontend/translate/Translate.cpp +++ b/src/graphics/shader/recompiler/frontend/translate/Translate.cpp @@ -110,8 +110,9 @@ Decoder::Operand Translator::PlainOperand(const Decoder::Operand& operand) { } std::array Translator::BallotMask(IR::U1 value) { - return {ir.Select(value, IR::U32(IR::Value(1u)), IR::U32(IR::Value(0u))), - IR::U32(IR::Value(0u))}; + const auto mask = ir.Emit(IR::ValueOpcode::Ballot, {value}); + return {ir.CompositeExtract(mask, 0), + current_wave_size == 64u ? ir.CompositeExtract(mask, 1) : IR::U32(IR::Value(0u))}; } IR::U32 Translator::ReadRawU32(const Decoder::Operand& operand) { @@ -285,7 +286,6 @@ void Translator::WriteRawU32(const Decoder::Operand& operand, IR::U32 value) { case Decoder::OperandKind::Sgpr: { const auto reg = static_cast(operand.reg); ir.SetScalarReg(reg, value); - ir.SetThreadBitScalarReg(reg, ThreadBit(value)); ir.SetScalarMaskTag(reg, IR::U1(IR::Value(false))); if (IR::RegIndex(reg) > 0u) { ir.SetScalarMaskTag(static_cast(IR::RegIndex(reg) - 1u), @@ -314,20 +314,20 @@ void Translator::WriteRawU32(const Decoder::Operand& operand, IR::U32 value) { } case Decoder::OperandKind::VccLo: ir.SetVccLo(IR::U32(value)); - ir.SetVcc(ThreadBit(IR::U32(value))); + ir.SetVcc(ThreadBit({value, ir.GetVccHi()})); break; case Decoder::OperandKind::VccHi: ir.SetVccHi(IR::U32(value)); - ir.SetVcc(ThreadBit(ir.GetVccLo())); + ir.SetVcc(ThreadBit({ir.GetVccLo(), value})); break; case Decoder::OperandKind::M0: ir.SetM0(IR::U32(value)); break; case Decoder::OperandKind::ExecLo: ir.SetExecLo(IR::U32(value)); - ir.SetExec(ThreadBit(IR::U32(value))); + ir.SetExec(ThreadBit({value, ir.GetExecHi()})); break; case Decoder::OperandKind::ExecHi: ir.SetExecHi(IR::U32(value)); - ir.SetExec(ThreadBit(ir.GetExecLo())); + ir.SetExec(ThreadBit({ir.GetExecLo(), value})); break; case Decoder::OperandKind::Scc: ir.SetScc(ir.INotEqual(value, IR::U32(IR::Value(0u)))); @@ -576,15 +576,14 @@ void Translator::WriteU32Pair(const Decoder::Operand& operand, if (operand.kind == Decoder::OperandKind::Null) { return; } - const auto thread_bit = ThreadBit(value[0]); switch (operand.kind) { case Decoder::OperandKind::ExecLo: - ir.SetExec(thread_bit); + ir.SetExec(ThreadBit(value)); ir.SetExecLo(value[0]); ir.SetExecHi(value[1]); return; case Decoder::OperandKind::VccLo: - ir.SetVcc(thread_bit); + ir.SetVcc(ThreadBit(value)); ir.SetVccLo(value[0]); ir.SetVccHi(value[1]); return; @@ -595,8 +594,14 @@ void Translator::WriteU32Pair(const Decoder::Operand& operand, WriteRawU32(OffsetOperand(operand, 1), value[1]); } -IR::U1 Translator::ThreadBit(IR::U32 low) { - return ir.INotEqual(low, IR::U32(IR::Value(0u))); +IR::U1 Translator::ThreadBit(const std::array& mask) { + const auto lane = IR::U32(ir.Emit(IR::ValueOpcode::LaneId)); + const auto word = current_wave_size == 64u + ? ir.Select(ir.ULessThan(lane, IR::U32(IR::Value(32u))), mask[0], mask[1]) + : mask[0]; + const auto bit = ir.BitwiseAnd(lane, IR::U32(IR::Value(31u))); + return ir.INotEqual(ir.BitwiseAnd(ir.ShiftRightLogical(word, bit), IR::U32(IR::Value(1u))), + IR::U32(IR::Value(0u))); } IR::U1 Translator::ReadCondition(const Decoder::Operand& operand) { @@ -604,11 +609,6 @@ IR::U1 Translator::ReadCondition(const Decoder::Operand& operand) { } IR::U32 Translator::ConditionBit(const Decoder::Operand& operand) { - if (operand.kind == Decoder::OperandKind::LiteralConstant || - operand.kind == Decoder::OperandKind::IntegerInlineConstant || - operand.kind == Decoder::OperandKind::FloatInlineConstant) { - return IR::U32(IR::Value(operand.value & 1u)); - } return ir.Select(ReadMask(operand), IR::U32(IR::Value(1u)), IR::U32(IR::Value(0u))); } @@ -616,12 +616,17 @@ IR::U1 Translator::ReadMask(const Decoder::Operand& operand) { if (operand.kind == Decoder::OperandKind::LiteralConstant || operand.kind == Decoder::OperandKind::IntegerInlineConstant || operand.kind == Decoder::OperandKind::FloatInlineConstant) { - return IR::U1(IR::Value(operand.value != 0u)); + return ThreadBit(ReadU32Pair(operand)); } switch (operand.kind) { case Decoder::OperandKind::Sgpr: { const auto reg = static_cast(operand.reg); - return ir.GetThreadBitScalarReg(reg); + const auto mask = current_wave_size == 64u + ? ReadU32Pair(operand) + : std::array {ReadRawU32(operand), IR::U32(IR::Value(0u))}; + return IR::U1(ir.Emit( + IR::ValueOpcode::SelectU1, + {ir.GetScalarMaskTag(reg), ir.GetThreadBitScalarReg(reg), ThreadBit(mask)})); } case Decoder::OperandKind::ExecLo: case Decoder::OperandKind::ExecHi: return ir.GetExec(); @@ -660,11 +665,12 @@ IR::U1 Translator::ReadMaskValid(const Decoder::Operand& operand) { } } -void Translator::WriteMask(const Decoder::Operand& operand, IR::U1 value) { +std::array Translator::WriteMask(const Decoder::Operand& operand, IR::U1 value, + bool write_64) { + const auto mask = BallotMask(value); switch (operand.kind) { case Decoder::OperandKind::Sgpr: { const auto reg = static_cast(operand.reg); - const auto mask = BallotMask(value); ir.SetThreadBitScalarReg(reg, value); ir.SetScalarMaskTag(reg, IR::U1(IR::Value(true))); if (IR::RegIndex(reg) > 0u) { @@ -673,57 +679,34 @@ void Translator::WriteMask(const Decoder::Operand& operand, IR::U1 value) { } ir.SetScalarReg(reg, mask[0]); // A wave32 VALU mask destination must not overwrite the neighboring SGPR. - if (current_wave_size == 64u && IR::RegIndex(reg) + 1u < IR::NumScalarRegs) { + if ((write_64 || current_wave_size == 64u) && + IR::RegIndex(reg) + 1u < IR::NumScalarRegs) { const auto high = static_cast(IR::RegIndex(reg) + 1u); ir.SetScalarReg(high, mask[1]); ir.SetThreadBitScalarReg(high, IR::U1(IR::Value(false))); ir.SetScalarMaskTag(high, IR::U1(IR::Value(false))); } - return; + return mask; } case Decoder::OperandKind::ExecLo: case Decoder::OperandKind::ExecHi: { - const auto mask = BallotMask(value); ir.SetExec(value); ir.SetExecLo(mask[0]); ir.SetExecHi(mask[1]); - return; + return mask; } case Decoder::OperandKind::VccLo: case Decoder::OperandKind::VccHi: { - const auto mask = BallotMask(value); ir.SetVcc(value); ir.SetVccLo(mask[0]); ir.SetVccHi(mask[1]); - return; + return mask; } - case Decoder::OperandKind::Scc: ir.SetScc(value); return; + case Decoder::OperandKind::Scc: ir.SetScc(value); return mask; default: - WriteRawU32(operand, ir.Select(value, IR::U32(IR::Value(1u)), IR::U32(IR::Value(0u)))); - WriteRawU32(OffsetOperand(operand, 1), IR::U32(IR::Value(0u))); - return; - } -} - -void Translator::WriteMask64(const Decoder::Operand& operand, IR::U1 value) { - if (operand.kind != Decoder::OperandKind::Sgpr) { - WriteMask(operand, value); - return; - } - const auto reg = static_cast(operand.reg); - const auto mask = BallotMask(value); - ir.SetThreadBitScalarReg(reg, value); - ir.SetScalarMaskTag(reg, IR::U1(IR::Value(true))); - if (IR::RegIndex(reg) > 0u) { - ir.SetScalarMaskTag(static_cast(IR::RegIndex(reg) - 1u), - IR::U1(IR::Value(false))); - } - ir.SetScalarReg(reg, mask[0]); - if (IR::RegIndex(reg) + 1u < IR::NumScalarRegs) { - const auto high = static_cast(IR::RegIndex(reg) + 1u); - ir.SetScalarReg(high, mask[1]); - ir.SetThreadBitScalarReg(high, IR::U1(IR::Value(false))); - ir.SetScalarMaskTag(high, IR::U1(IR::Value(false))); + WriteRawU32(operand, mask[0]); + WriteRawU32(OffsetOperand(operand, 1), mask[1]); + return mask; } } @@ -910,6 +893,7 @@ void ValidateTranslateOptions(const TranslateOptions& options) { } switch (options.stage) { case ShaderType::Vertex: + case ShaderType::Mesh: if (options.vertex == nullptr) { EXIT("vertex shader translation has no vertex input metadata"); } @@ -1022,12 +1006,32 @@ IR::Program TranslateProgram(const Decoder::Program& decoded, const CFG::Graph& } const auto value = entry_ir.GetUserData(reg); entry_ir.SetScalarReg(reg, value); - entry_ir.SetThreadBitScalarReg(reg, entry_ir.INotEqual(value, IR::U32(IR::Value(0u)))); entry_ir.SetScalarMaskTag(reg, IR::U1(IR::Value(false))); } - entry_ir.SetExec(IR::U1(IR::Value(true))); - entry_ir.SetExecLo(IR::U32(IR::Value(1u))); - entry_ir.SetExecHi(IR::U32(IR::Value(0u))); + auto initial_exec = IR::U1(IR::Value(true)); + uint32_t total_threads = 0; + ShaderStageInputInfo stage_input {}; + if (options.stage == ShaderType::Compute) { + stage_input.compute = options.compute; + } else { + stage_input.vertex = options.vertex; + } + const auto* workgroup = ShaderWorkgroupInput(options.stage, stage_input); + if (workgroup != nullptr) { + total_threads = std::max(workgroup->threads_num[0], 1u) * + std::max(workgroup->threads_num[1], 1u) * + std::max(workgroup->threads_num[2], 1u); + if (options.wave_size == 64u && workgroup->host_subgroup_size == 32u && + total_threads % 64u != 0u) { + initial_exec = entry_ir.ULessThan(builtin(IR::StageInputKind::LocalInvocationIndex), + IR::U32(IR::Value(total_threads))); + } + } + entry_ir.SetExec(initial_exec); + const auto initial_mask = entry_ir.Emit(IR::ValueOpcode::Ballot, {initial_exec}); + entry_ir.SetExecLo(entry_ir.CompositeExtract(initial_mask, 0)); + entry_ir.SetExecHi(options.wave_size == 64u ? entry_ir.CompositeExtract(initial_mask, 1) + : IR::U32(IR::Value(0u))); if (options.stage == ShaderType::Compute) { const auto* cs = options.compute; const auto thread_ids = @@ -1046,9 +1050,6 @@ IR::Program TranslateProgram(const Decoder::Program& decoded, const CFG::Graph& } if (cs->tg_size_en) { const auto wave_size = cs->wave_size != 0u ? cs->wave_size : 64u; - const auto total_threads = std::max(cs->threads_num[0], 1u) * - std::max(cs->threads_num[1], 1u) * - std::max(cs->threads_num[2], 1u); const auto waves = std::min((total_threads + wave_size - 1u) / wave_size, 0x3fu); const auto local_index = builtin(IR::StageInputKind::LocalInvocationIndex); const auto wave_id = IR::U32( @@ -1062,6 +1063,53 @@ IR::Program TranslateProgram(const Decoder::Program& decoded, const CFG::Graph& entry_ir.BitwiseOr(entry_ir.BitwiseOr(wave_bits, IR::U32(IR::Value(waves))), first_bit)); } + } else if (options.stage == ShaderType::Mesh) { + const auto& mesh = options.vertex->mesh; + EXIT_NOT_IMPLEMENTED(options.wave_size != 64u || mesh.primitives_per_group == 0u || + mesh.vertices_per_group > 64u || total_threads > 15u * 64u); + const auto u32 = [](uint32_t value) { return IR::U32(IR::Value(value)); }; + const auto draw = [&](uint32_t index) { + return IR::U32( + entry_ir.Emit(IR::ValueOpcode::MeshDrawParameter, {IR::Value(index)})); + }; + const auto minimum = [&](IR::U32 lhs, IR::U32 rhs) { + return IR::U32(entry_ir.Emit(IR::ValueOpcode::UMin32, {lhs, rhs})); + }; + const auto subtract_saturate = [&](IR::U32 lhs, IR::U32 rhs) { + return entry_ir.ISub(lhs, minimum(lhs, rhs)); + }; + const auto local = builtin(IR::StageInputKind::LocalInvocationIndex); + const auto chunk = entry_ir.IMul(builtin(IR::StageInputKind::WorkgroupId, 0), + u32(mesh.primitives_per_group)); + const auto vertices = + minimum(subtract_saturate(draw(0), chunk), u32(mesh.vertices_per_group)); + const auto primitives = subtract_saturate(vertices, u32(2)); + const auto wave = entry_ir.ShiftRightLogical(local, u32(6)); + const auto wave_base = entry_ir.BitwiseAnd(local, u32(~63u)); + const auto vertex_count = minimum(subtract_saturate(vertices, wave_base), u32(64)); + const auto primitive_count = minimum(subtract_saturate(primitives, wave_base), u32(64)); + const auto wave_info = entry_ir.BitwiseOr(entry_ir.ShiftLeftLogical(wave, u32(24)), + u32(((total_threads + 63u) / 64u) << 28u)); + entry_ir.SetScalarReg( + static_cast(3), + entry_ir.BitwiseOr(wave_info, entry_ir.BitwiseOr(entry_ir.ShiftLeftLogical( + primitive_count, u32(8)), + vertex_count))); + // Triangle-strip adjacency addresses the ES records in LDS. Alternating the + // first two indices preserves winding across subgroup boundaries. + const auto parity = entry_ir.BitwiseAnd(entry_ir.IAdd(chunk, local), u32(1)); + const auto first = entry_ir.IAdd(local, parity); + const auto second = entry_ir.ISub(entry_ir.IAdd(local, u32(1)), parity); + entry_ir.SetVectorReg(static_cast(0), + entry_ir.BitwiseOr(entry_ir.ShiftLeftLogical(first, u32(2)), + entry_ir.ShiftLeftLogical(second, u32(18)))); + entry_ir.SetVectorReg(static_cast(1), + entry_ir.ShiftLeftLogical(entry_ir.IAdd(local, u32(2)), u32(2))); + entry_ir.SetVectorReg(static_cast(5), + entry_ir.IAdd(draw(1), entry_ir.IAdd(chunk, local))); + entry_ir.SetVectorReg( + static_cast(8), + entry_ir.IAdd(draw(2), builtin(IR::StageInputKind::WorkgroupId, 1))); } else if (options.stage == ShaderType::Pixel) { const auto* ps = options.pixel; if (ps->ps_perspective_center_vgpr != UINT32_MAX) { diff --git a/src/graphics/shader/recompiler/frontend/translate/Translator.h b/src/graphics/shader/recompiler/frontend/translate/Translator.h index 71bab6ae7..9687caca6 100644 --- a/src/graphics/shader/recompiler/frontend/translate/Translator.h +++ b/src/graphics/shader/recompiler/frontend/translate/Translator.h @@ -29,7 +29,7 @@ private: IR::U32 ReadScalarCode(uint32_t code); IR::U32 ApplyBitSourceModifiers(const Decoder::Operand& operand, IR::U32 value); IR::Value ReadOperand(const Decoder::Operand& operand, IR::Type type); - IR::U1 ThreadBit(IR::U32 low); + IR::U1 ThreadBit(const std::array& mask); void WriteRawU32(const Decoder::Operand& operand, IR::U32 value); IR::F32 ApplyF32ResultModifiers(const Decoder::Operand& operand, IR::F32 value); void WriteOperand(const Decoder::Operand& operand, IR::Value value); @@ -53,8 +53,8 @@ private: IR::U32 ConditionBit(const Decoder::Operand& operand); IR::U1 ReadMask(const Decoder::Operand& operand); IR::U1 ReadMaskValid(const Decoder::Operand& operand); - void WriteMask(const Decoder::Operand& operand, IR::U1 value); - void WriteMask64(const Decoder::Operand& operand, IR::U1 value); + std::array WriteMask(const Decoder::Operand& operand, IR::U1 value, + bool write_64 = false); void WriteCompareResult(const Decoder::Operand& operand, IR::U1 value); IR::MemoryFlags AddMemoryInfo(const IR::MemoryInfo& memory, uint32_t pc); @@ -228,7 +228,7 @@ private: void EmitControlNop(); void EmitWaitcnt(); void S_BARRIER(); - void S_SENDMSG(); + void S_SENDMSG(const Decoder::Instruction& inst); void S_TTRACEDATA(); void S_INST_PREFETCH(); void S_GETPC_B64(const Decoder::Instruction& inst); diff --git a/src/graphics/shader/recompiler/ir/ShaderIR.h b/src/graphics/shader/recompiler/ir/ShaderIR.h index 909f260a7..b98090cf2 100644 --- a/src/graphics/shader/recompiler/ir/ShaderIR.h +++ b/src/graphics/shader/recompiler/ir/ShaderIR.h @@ -271,6 +271,7 @@ static_assert(static_cast(DescriptorBindingKind::Count) == 43u); struct PushData { static constexpr uint32_t DwordCount = 32; + static constexpr uint32_t MeshDrawDwordCount = 3; static constexpr uint32_t NoStart = UINT32_MAX; std::array dwords {}; diff --git a/src/graphics/shader/recompiler/ir/opcodes/ValueOpcodes.cpp b/src/graphics/shader/recompiler/ir/opcodes/ValueOpcodes.cpp index a1f62c11b..4ffe10555 100644 --- a/src/graphics/shader/recompiler/ir/opcodes/ValueOpcodes.cpp +++ b/src/graphics/shader/recompiler/ir/opcodes/ValueOpcodes.cpp @@ -90,6 +90,7 @@ bool HasSideEffects(ValueOpcode opcode) { case ValueOpcode::Reference: case ValueOpcode::ReferenceU32: case ValueOpcode::SetAttribute: + case ValueOpcode::MeshAllocate: case ValueOpcode::Barrier: return true; default: return false; } diff --git a/src/graphics/shader/recompiler/ir/opcodes/ValueOpcodes.inc b/src/graphics/shader/recompiler/ir/opcodes/ValueOpcodes.inc index 995e3c58e..e15e30d9a 100644 --- a/src/graphics/shader/recompiler/ir/opcodes/ValueOpcodes.inc +++ b/src/graphics/shader/recompiler/ir/opcodes/ValueOpcodes.inc @@ -8,6 +8,8 @@ VALUE_OPCODE(ReferenceU32, Void, U32) // Translation and register-state pseudo operations. VALUE_OPCODE(GetUserData, U32, ScalarReg) VALUE_OPCODE(GetShaderBase, U64) +VALUE_OPCODE(MeshDrawParameter, U32, U32) +VALUE_OPCODE(MeshAllocate, Void, U32) VALUE_OPCODE(GetBuiltin, U32, U32, U32) VALUE_OPCODE(GetThreadBitScalarRegister, U1, ScalarReg) VALUE_OPCODE(SetThreadBitScalarRegister, Void, ScalarReg, U1) @@ -81,7 +83,6 @@ VALUE_OPCODE(BitFieldSExtract, U32, U32, U32, U32) VALUE_OPCODE(DppMoveU32, U32, U32, U1) VALUE_OPCODE(DppUpdateU32, U32, U32, U32, U1) VALUE_OPCODE(WqmU64, U64, U64) -VALUE_OPCODE(WqmMask, U1, U1) // Core value operations used to expand architectural side effects. VALUE_OPCODE(SelectU1, U1, U1, U1, U1) diff --git a/src/graphics/shader/recompiler/ir/passes/ShaderInfoCollection.cpp b/src/graphics/shader/recompiler/ir/passes/ShaderInfoCollection.cpp index 7d77b9cbf..37a4bb023 100644 --- a/src/graphics/shader/recompiler/ir/passes/ShaderInfoCollection.cpp +++ b/src/graphics/shader/recompiler/ir/passes/ShaderInfoCollection.cpp @@ -43,6 +43,7 @@ void AddOutput(ShaderInfo& info, StageOutputKind kind, uint32_t index, uint32_t void ValidateOptions(const Program& program, const ShaderInfoOptions& options) { switch (program.stage) { case ShaderType::Vertex: + case ShaderType::Mesh: if (options.vertex == nullptr) { return Fail("vertex shader has no input metadata"); } @@ -365,6 +366,7 @@ void CollectShaderInfo(Program& program, const ShaderInfoOptions& options) { }); switch (program.stage) { case ShaderType::Vertex: CollectVertexInputs(program, options.vertex, next); break; + case ShaderType::Mesh: break; case ShaderType::Pixel: CollectPixelInputs(program, options.pixel, next); break; case ShaderType::Compute: CollectComputeInputs(options.compute, next); break; default: return Fail("unsupported shader stage for info collection"); diff --git a/src/graphics/shader/recompiler/ir/passes/SharedMemoryBarrier.cpp b/src/graphics/shader/recompiler/ir/passes/SharedMemoryBarrier.cpp deleted file mode 100644 index 61d479536..000000000 --- a/src/graphics/shader/recompiler/ir/passes/SharedMemoryBarrier.cpp +++ /dev/null @@ -1,153 +0,0 @@ -#include "graphics/shader/recompiler/ir/passes/SharedMemoryBarrier.h" - -#include "graphics/shader/recompiler/ir/ShaderIR.h" - -#include -#include -#include -#include - -namespace Libs::Graphics::ShaderRecompiler::IR { -namespace { - -enum class LdsPhase { None, Read, Write }; - -LdsPhase GetLdsPhase(const Program& program, const Inst& inst) { - const auto access = SharedAccessOf(inst.GetOpcode()); - if (access != SharedAccess::Read && access != SharedAccess::Write) { - return LdsPhase::None; - } - const auto index = inst.Flags().index; - if (index >= program.memory_info.size() || - program.memory_info[index].kind != ResourceKind::Lds) { - return LdsPhase::None; - } - return access == SharedAccess::Read ? LdsPhase::Read : LdsPhase::Write; -} - -bool DependsOnInvocation(Value value) { - std::queue queue; - std::unordered_set visited; - queue.push(value); - while (!queue.empty()) { - value = queue.front().Resolve(); - queue.pop(); - auto* inst = value.TryInstruction(); - if (inst == nullptr || !visited.insert(inst).second) { - continue; - } - if (inst->GetOpcode() == ValueOpcode::LaneId) { - return true; - } - if (inst->GetOpcode() == ValueOpcode::GetBuiltin) { - const auto kind = static_cast(inst->Arg(0).U32()); - if (kind == StageInputKind::LocalInvocationId || - kind == StageInputKind::LocalInvocationIndex || - kind == StageInputKind::GlobalInvocationId) { - return true; - } - } - for (size_t index = 0; index < inst->NumArgs(); index++) { - queue.push(inst->Arg(index)); - } - } - return false; -} - -uint32_t InsertBarrier(Block& block, Block::iterator position) { - block.PrependNewInst(position, ValueOpcode::Barrier); - return 1; -} - -uint32_t InsertBlockBarriers(Program& program, Block& block) { - uint32_t inserted = 0; - LdsPhase phase = LdsPhase::None; - for (auto inst = block.begin(); inst != block.end(); ++inst) { - if (inst->GetOpcode() == ValueOpcode::Barrier) { - phase = LdsPhase::None; - continue; - } - const auto next = GetLdsPhase(program, *inst); - if (next == LdsPhase::None) { - continue; - } - if (phase != LdsPhase::None && phase != next) { - inserted += InsertBarrier(block, inst); - } - phase = next; - } - if (phase != LdsPhase::None) { - block.AppendNewInst(ValueOpcode::Barrier); - inserted++; - } - return inserted; -} - -uint32_t InsertMergeBarrier(Block& block) { - auto position = std::ranges::find_if( - block, [](const Inst& inst) { return inst.GetOpcode() != ValueOpcode::Phi; }); - if (position != block.end() && position->GetOpcode() == ValueOpcode::Barrier) { - return 0; - } - return InsertBarrier(block, position); -} - -} // namespace - -SharedMemoryBarrierStats InsertSharedMemoryBarriers(Program& program, uint32_t wave_size, - const ShaderComputeInputInfo& compute_info) { - SharedMemoryBarrierStats stats; - const auto threadgroup_size = compute_info.threads_num[0] * compute_info.threads_num[1] * - compute_info.threads_num[2]; - if (wave_size != 64u || !compute_info.needs_lds_barriers || - compute_info.lds_size_dwords == 0u || threadgroup_size != 64u) { - return stats; - } - if (program.blocks.size() != program.block_info.size()) { - return stats; - } - for (const auto& info: program.block_info) { - if (info.terminator.kind == CFG::TerminatorKind::ConditionalBranch && - info.terminator.merge_block == UINT32_MAX && DependsOnInvocation(info.condition)) { - return stats; - } - } - - std::unordered_map blocks; - for (size_t index = 0; index < program.blocks.size(); index++) { - blocks.emplace(program.block_info[index].id, program.blocks[index]); - } - - uint32_t divergence_depth = 0; - std::unordered_map divergence_ends; - std::unordered_set synchronized_merges; - for (size_t index = 0; index < program.blocks.size(); index++) { - const auto id = program.block_info[index].id; - if (const auto end = divergence_ends.find(id); end != divergence_ends.end()) { - EXIT_IF(end->second > divergence_depth); - divergence_depth -= end->second; - } - - if (divergence_depth == 0) { - stats.inserted_barriers += InsertBlockBarriers(program, *program.blocks[index]); - } - - const auto& info = program.block_info[index]; - const auto& term = info.terminator; - if (term.kind != CFG::TerminatorKind::ConditionalBranch || term.merge_block == UINT32_MAX || - !DependsOnInvocation(info.condition)) { - continue; - } - if (divergence_depth == 0 && synchronized_merges.insert(term.merge_block).second) { - const auto merge = blocks.find(term.merge_block); - if (merge != blocks.end()) { - stats.inserted_barriers += InsertMergeBarrier(*merge->second); - } - } - divergence_depth++; - divergence_ends[term.merge_block]++; - } - return stats; -} - -} // namespace Libs::Graphics::ShaderRecompiler::IR diff --git a/src/graphics/shader/recompiler/ir/passes/SharedMemoryBarrier.h b/src/graphics/shader/recompiler/ir/passes/SharedMemoryBarrier.h deleted file mode 100644 index 0e5e0b77c..000000000 --- a/src/graphics/shader/recompiler/ir/passes/SharedMemoryBarrier.h +++ /dev/null @@ -1,15 +0,0 @@ -#pragma once - -#include "graphics/shader/recompiler/ir/ShaderIR.h" - -namespace Libs::Graphics::ShaderRecompiler::IR { - -struct SharedMemoryBarrierStats { - uint32_t inserted_barriers = 0; -}; - -[[nodiscard]] SharedMemoryBarrierStats -InsertSharedMemoryBarriers(Program& program, uint32_t wave_size, - const ShaderComputeInputInfo& compute_info); - -} // namespace Libs::Graphics::ShaderRecompiler::IR diff --git a/src/graphics/shader/shader.cpp b/src/graphics/shader/shader.cpp index 0eed19cbd..c6f7171a1 100644 --- a/src/graphics/shader/shader.cpp +++ b/src/graphics/shader/shader.cpp @@ -723,23 +723,22 @@ static void ShaderGetStaticInputInfoPS( } static void ShaderGetStaticInputInfoCS(const HW::ComputeShaderInfo& regs, - const HW::ShaderRegisters& /*sh*/, - const ShaderMappedData& data, - ShaderComputeInputInfo& info) { + const HW::ShaderRegisters& /*sh*/, + const ShaderMappedData& data, ShaderComputeInputInfo& info) { const bool dispatch_thread_dimensions = info.dispatch_thread_dimensions; - const bool needs_lds_barriers = info.needs_lds_barriers; - info = {}; - info.dispatch_thread_dimensions = dispatch_thread_dimensions; - info.needs_lds_barriers = needs_lds_barriers; - info.threads_num[0] = regs.cs_regs.num_thread_x; - info.threads_num[1] = regs.cs_regs.num_thread_y; - info.threads_num[2] = regs.cs_regs.num_thread_z; - info.lds_size_dwords = static_cast(regs.cs_regs.lds_size) * 128u; - info.scratch_size_dwords = data.scratch_size_dwords; - info.group_id[0] = regs.cs_regs.tgid_x_en != 0; - info.group_id[1] = regs.cs_regs.tgid_y_en != 0; - info.group_id[2] = regs.cs_regs.tgid_z_en != 0; - info.wave_size = regs.cs_regs.wave_size; + const auto host_subgroup_size = info.host_subgroup_size; + info = {}; + info.dispatch_thread_dimensions = dispatch_thread_dimensions; + info.host_subgroup_size = host_subgroup_size; + info.threads_num[0] = regs.cs_regs.num_thread_x; + info.threads_num[1] = regs.cs_regs.num_thread_y; + info.threads_num[2] = regs.cs_regs.num_thread_z; + info.lds_size_dwords = static_cast(regs.cs_regs.lds_size) * 128u; + info.scratch_size_dwords = data.scratch_size_dwords; + info.group_id[0] = regs.cs_regs.tgid_x_en != 0; + info.group_id[1] = regs.cs_regs.tgid_y_en != 0; + info.group_id[2] = regs.cs_regs.tgid_z_en != 0; + info.wave_size = regs.cs_regs.wave_size; info.thread_ids_num = regs.cs_regs.tidig_comp_cnt + 1; info.tg_size_en = regs.cs_regs.tg_size_en != 0; @@ -768,6 +767,15 @@ void BuildStageStaticKey(const ShaderVertexInputInfo& info, std::vector(info.needs_lds_barriers)); key.push_back(static_cast(info.dispatch_thread_dimensions)); for (int i = 0; i < 3; i++) { key.push_back(info.threads_num[i]); @@ -834,15 +842,57 @@ void BuildStageStaticKey(const ShaderComputeInputInfo& info, std::vector(info.tg_size_en)); } -ShaderParams PrepareProgram(const HW::VertexShaderInfo& regs, const HW::ShaderRegisters& sh, - ShaderVertexInputInfo& info) { +ShaderParams PrepareProgram(const HW::VertexShaderInfo& regs, const HW::Context& context, + const HW::UserConfig& user_config, ShaderVertexInputInfo& info) { + const auto& sh = context.GetShaderRegisters(); const auto data = ShaderGetMappedData(regs.es_regs.data_addr, "ShaderGetInputInfoVS():"); - if (!ShaderGetStaticInputInfoVS(regs, sh, data, info)) { - EXIT("failed to prepare vertex shader program\n"); - } - return GetShaderParams( - regs.es_regs.data_addr, "ShaderRecompiler VS", GetDeclaredShaderHash(regs.es_regs.data_addr), + auto params = GetShaderParams( + regs.es_regs.data_addr, "ShaderRecompiler VS", + GetDeclaredShaderHash(regs.es_regs.data_addr), std::span(regs.gs_user_sgpr.value, regs.gs_regs.rsrc2.user_sgpr), data); + if ((context.GetShaderStages() & 0x20u) == 0) { + if (!ShaderGetStaticInputInfoVS(regs, sh, data, info)) { + EXIT("failed to prepare vertex shader program\n"); + } + return params; + } + EXIT_IF(regs.gs_regs.data_addr == 0); + const auto back = ShaderGetMappedData(regs.gs_regs.data_addr, "ShaderGetInputInfoGS():"); + const auto back_params = + GetShaderParams(regs.gs_regs.data_addr, "ShaderRecompiler GS", + GetDeclaredShaderHash(regs.gs_regs.data_addr), {}, back); + params.back_code = back_params.code; + const uint64_t hashes[] = {params.hash, back_params.hash}; + params.hash = XXH3_64bits(hashes, sizeof(hashes)); + info = {}; + info.pa_cl_vs_out_cntl = sh.m_paClVsOutCntl; + auto& mesh = info.mesh; + mesh.input_primitive = static_cast(user_config.GetPrimType()); + mesh.wave_size = (context.GetShaderStages() & 0x00400000u) != 0 ? 32u : 64u; + mesh.max_vertices = sh.m_geMaxOutputPerSubgroup; + mesh.provoking_vertex = context.GetModeControl().provoking_vtx_last ? 2u : 0u; + mesh.lds_size_dwords = static_cast(regs.gs_regs.rsrc2.lds_size) * 128u; + mesh.scratch_size_dwords = std::max(data.scratch_size_dwords, back.scratch_size_dwords); + EXIT_NOT_IMPLEMENTED(regs.gs_regs.rsrc1.gs_vgpr_component_count != 3u || + regs.gs_regs.rsrc2.es_vgpr_component_count != 3u); + const auto& group = user_config.GetGeControl(); + if (user_config.GetPrimType() != Prospero::PrimitiveType::kTriStrip || + sh.m_vgtGsOutPrimType != 2u || sh.m_vgtGsMaxVertOut < 3u || group.vertex_group_size < 3u || + mesh.max_vertices == 0u) { + EXIT("unsupported GS assembly: input=%u output=%u vertices=%u GE=%u/%u max_output=%u\n", + mesh.input_primitive, sh.m_vgtGsOutPrimType, sh.m_vgtGsMaxVertOut, + group.primitive_group_size, group.vertex_group_size, mesh.max_vertices); + } + mesh.max_primitives = group.primitive_group_size * (sh.m_vgtGsMaxVertOut - 2u); + mesh.primitives_per_group = std::min({static_cast(group.primitive_group_size), + static_cast(group.vertex_group_size) - 2u, + mesh.max_vertices / sh.m_vgtGsMaxVertOut}); + EXIT_IF(mesh.primitives_per_group == 0u); + mesh.vertices_per_group = mesh.primitives_per_group + 2u; + mesh.threads_num[0] = + ((mesh.max_vertices + mesh.wave_size - 1u) / mesh.wave_size) * mesh.wave_size; + mesh.threads_num[1] = mesh.threads_num[2] = 1u; + return params; } ShaderParams PrepareProgram( @@ -960,11 +1010,9 @@ void ShaderDbgDumpInputInfo(const ShaderComputeInputInfo& info) { "\t thread_ids_num = %d\n" "\t wave_size = %u\n" "\t lds_size_dwords = %u\n" - "\t needs_lds_barriers = %s\n" "\t threads_num = {%u, %u, %u}\n" "\t tg_size_en = %s\n", info.workgroup_register, info.thread_ids_num, info.wave_size, info.lds_size_dwords, - info.needs_lds_barriers ? "true" : "false", info.threads_num[0], info.threads_num[1], info.threads_num[2], info.tg_size_en ? "true" : "false"); LOGF("\t threadgroup_id = {%s, %s, %s}\n", info.group_id[0] ? "true" : "false", diff --git a/src/graphics/shader/shader.h b/src/graphics/shader/shader.h index 4e51d211f..daf3d5738 100644 --- a/src/graphics/shader/shader.h +++ b/src/graphics/shader/shader.h @@ -33,7 +33,7 @@ struct ComputeShaderInfo; struct ShaderRegisters; } // namespace HW -enum class ShaderType { Unknown, Vertex, Pixel, Fetch, Compute }; +enum class ShaderType { Unknown, Vertex, Pixel, Fetch, Compute, Mesh }; namespace ShaderRecompiler::IR { struct CompiledShaderInfo; @@ -63,6 +63,23 @@ struct ShaderClipSpaceTransform { bool enabled = false; }; +struct ShaderWorkgroupInputInfo { + uint32_t threads_num[3] = {0, 0, 0}; + uint32_t lds_size_dwords = 0; + uint32_t scratch_size_dwords = 0; + uint32_t host_subgroup_size = 64; + uint32_t wave_size = 64; +}; + +struct ShaderMeshInputInfo: ShaderWorkgroupInputInfo { + uint32_t input_primitive = 0; + uint32_t primitives_per_group = 0; + uint32_t vertices_per_group = 0; + uint32_t max_vertices = 0; + uint32_t max_primitives = 0; + uint32_t provoking_vertex = 0; +}; + struct ShaderVertexInputInfo { static constexpr int RES_MAX = 32; @@ -78,19 +95,15 @@ struct ShaderVertexInputInfo { uint32_t scratch_size_dwords = 0; uint32_t pa_cl_vs_out_cntl = 0; ShaderClipSpaceTransform clip_space; + ShaderMeshInputInfo mesh; bool fetch_external = false; bool fetch_embedded = false; }; -struct ShaderComputeInputInfo { - uint32_t threads_num[3] = {0, 0, 0}; +struct ShaderComputeInputInfo: ShaderWorkgroupInputInfo { uint32_t dispatch_threads_num[3] = {0, 0, 0}; - uint32_t lds_size_dwords = 0; - uint32_t scratch_size_dwords = 0; bool group_id[3] = {false, false, false}; bool dispatch_thread_dimensions = false; - bool needs_lds_barriers = false; - uint32_t wave_size = 64; int thread_ids_num = 0; int workgroup_register = 0; bool tg_size_en = false; @@ -130,6 +143,15 @@ union ShaderStageInputInfo { const ShaderComputeInputInfo* compute = nullptr; }; +inline const ShaderWorkgroupInputInfo* ShaderWorkgroupInput(ShaderType stage, + ShaderStageInputInfo input) { + switch (stage) { + case ShaderType::Compute: return input.compute; + case ShaderType::Mesh: return &input.vertex->mesh; + default: return nullptr; + } +} + uint32_t ShaderPixelParameterMappedLocation(const ShaderPixelInputInfo& info, uint32_t input); uint32_t ShaderPixelParameterLocation(const ShaderPixelInputInfo& info, std::span active_inputs, uint32_t input); diff --git a/src/graphics/shader/shaderCompiler.h b/src/graphics/shader/shaderCompiler.h index 4859a9fee..f201040ef 100644 --- a/src/graphics/shader/shaderCompiler.h +++ b/src/graphics/shader/shaderCompiler.h @@ -8,10 +8,16 @@ namespace Libs::Graphics { +namespace HW { +class Context; +class UserConfig; +} // namespace HW + struct ShaderParams { std::span code; std::span user_data; uint64_t hash = 0; + std::span back_code; [[nodiscard]] uint64_t Base() const { return reinterpret_cast(code.data()); @@ -22,8 +28,8 @@ void BuildStageStaticKey(const ShaderVertexInputInfo& input_info, std::vector& key); void BuildStageStaticKey(const ShaderComputeInputInfo& input_info, std::vector& key); -ShaderParams PrepareProgram(const HW::VertexShaderInfo& regs, const HW::ShaderRegisters& sh, - ShaderVertexInputInfo& input_info); +ShaderParams PrepareProgram(const HW::VertexShaderInfo& regs, const HW::Context& context, + const HW::UserConfig& user_config, ShaderVertexInputInfo& input_info); ShaderParams PrepareProgram( const HW::PixelShaderInfo& regs, const HW::ShaderRegisters& sh, std::span target_export_mapping, diff --git a/tests/ResourceTrackingTests.cpp b/tests/ResourceTrackingTests.cpp index f48762ee4..19bb6688b 100644 --- a/tests/ResourceTrackingTests.cpp +++ b/tests/ResourceTrackingTests.cpp @@ -640,7 +640,8 @@ void TestImagesSamplersAndAliases() { void TestSampleAdjustSamplerScratch() { Fixture fixture(ShaderType::Pixel); - const auto active = fixture.Emit(ValueOpcode::WqmMask, {Value(true)}); + const auto active = fixture.Emit( + ValueOpcode::IEqual32, {fixture.Emit(ValueOpcode::LaneId), Value(0u)}); const auto lane = fixture.Emit(ValueOpcode::SelectU32, {active, Value(1u), Value(0u)}); const auto low = @@ -686,7 +687,8 @@ void TestSampleAdjustSamplerScratch() { const auto CheckRejected = [](uint32_t flags, uint32_t shift, const char *message) { Fixture rejected(ShaderType::Pixel); - const auto condition = rejected.Emit(ValueOpcode::WqmMask, {Value(true)}); + const auto condition = rejected.Emit( + ValueOpcode::IEqual32, {rejected.Emit(ValueOpcode::LaneId), Value(0u)}); const auto bit = rejected.Emit(ValueOpcode::SelectU32, {condition, Value(1u), Value(0u)}); const auto dynamic = diff --git a/tests/ScalarProvenanceTests.cpp b/tests/ScalarProvenanceTests.cpp index cfebc9aab..f7fae3611 100644 --- a/tests/ScalarProvenanceTests.cpp +++ b/tests/ScalarProvenanceTests.cpp @@ -321,7 +321,8 @@ void TestRuntime64BitDescriptorOps() { void TestUniformFirstLaneSamplerLod() { Fixture fixture; - const auto active = fixture.Emit(ValueOpcode::WqmMask, {Value(true)}); + const auto active = fixture.Emit( + ValueOpcode::IEqual32, {fixture.Emit(ValueOpcode::LaneId), Value(0u)}); const auto stale = fixture.Emit( ValueOpcode::GetVectorRegister, {Value(static_cast(0))}); const auto write = [&](Value value, Value previous) { diff --git a/tests/ShaderRecompilerComputeTests.cpp b/tests/ShaderRecompilerComputeTests.cpp index f2cc3bf3c..ca3cb8b3a 100644 --- a/tests/ShaderRecompilerComputeTests.cpp +++ b/tests/ShaderRecompilerComputeTests.cpp @@ -1198,7 +1198,7 @@ void CheckSpirvText(const TestCase &test, const std::vector &spirv) { } } -CompiledShader CompileCase(const TestCase &test) { +CompiledShader CompileCase(const TestCase &test, u32 host_subgroup_size = 64) { auto user_data = MakeNativeUserData(test.has_user_data ? &test.user_data : nullptr); const auto uses_image = @@ -1218,7 +1218,9 @@ CompiledShader CompileCase(const TestCase &test) { ShaderRecompiler::CompileOptions options; options.stage = ShaderType::Compute; options.dump_ir = true; - options.input_info.compute = &test.compute_info; + auto compute_info = test.compute_info; + compute_info.host_subgroup_size = host_subgroup_size; + options.input_info.compute = &compute_info; options.user_data = user_data; options.scratch_dwords = test.compute_info.scratch_size_dwords; if (test.has_compute_info) { @@ -1553,6 +1555,13 @@ public: }; [[nodiscard]] vk::Device Device() const { return m_device; } + [[nodiscard]] u32 SubgroupSize() const { + vk::PhysicalDeviceSubgroupProperties subgroup{}; + vk::PhysicalDeviceProperties2 properties{}; + properties.pNext = &subgroup; + m_physical_device.getProperties2(&properties); + return subgroup.subgroupSize; + } [[nodiscard]] GraphicContext &RuntimeContext() { EnsureRuntimeContext(); return m_runtime_context; @@ -2278,7 +2287,11 @@ public: return packet; }; const auto finish_gpu = [&] { - gpu.SendCommandSync([&] { context.GetCommandScheduler().Finish(); }); + gpu.SendCommandSync([&] { + auto &scheduler = context.GetCommandScheduler(); + scheduler.Finish(); + scheduler.WaitPriorityOperations(scheduler.CurrentTick() - 1); + }); }; const auto wait_for_interrupt = [&](LibKernel::EventQueue::KernelEvent &event, int &count) { @@ -12064,7 +12077,7 @@ void CompareGraphicsWords(const GraphicsCase &test, } void RunCase(VulkanHarness *vulkan, const TestCase &test) { - auto compiled = CompileCase(test); + auto compiled = CompileCase(test, vulkan != nullptr ? vulkan->SubgroupSize() : 64u); if (test.image_descriptor_swizzle != DstSel(4, 5, 6, 7)) { Require(test.name, "resource specialization", !compiled.program.info.images.empty() && @@ -13207,7 +13220,7 @@ TestCase ScalarNotB64UpdatesScc() { return {"ScalarNotB64UpdatesScc", code, {}, - {0, 1, 0, 0, 1, 0, 0, 0xfffffffeu, 0xffffffffu}, + {0, 1, 0, 0, 1, 0, 1, 0xfffffffeu, 0xffffffffu}, {O::S_MOV_B32, O::S_CMP_EQ_U32, O::S_NOT_B64, O::S_CSELECT_B32, O::V_CMP_EQ_U32, O::S_MOV_B64, O::V_MOV_B32, O::BUFFER_STORE_DWORD, O::S_ENDPGM}}; @@ -13341,8 +13354,8 @@ TestCase ScalarSaveExecOps() { return {"ScalarSaveExecOps", code, {}, - // EXEC and saved EXEC values are invocation-local Booleans. - {1, 0, 1, 0, 1, 0, 1, 1, 0, 1, 0, 0, 0}, + {1, 0, 1, 0, 0xffffffffu, 0xffffffffu, 0xffffffffu, + 3, 0xffffffffu, 3, 2, 0xffffffffu, 1}, {O::S_MOV_B32, O::S_AND_SAVEEXEC_B64, O::S_ORN2_SAVEEXEC_B64, O::S_ANDN1_SAVEEXEC_B64, O::S_AND_SAVEEXEC_B32, O::S_ANDN1_SAVEEXEC_B32, O::S_MOV_B64, O::V_MOV_B32, @@ -13369,7 +13382,7 @@ TestCase ScalarOrn2SaveexecUsesSourceOrNotExec() { return {"ScalarOrn2SaveexecUsesSourceOrNotExec", code, {}, - {1, 0, 1, 0, 1}, + {0x0000000cu, 0x80000000u, 0xfffffff3u, 0x7fffffffu, 1}, {O::S_MOV_B32, O::S_ORN2_SAVEEXEC_B64, O::S_MOV_B64, O::V_MOV_B32, O::BUFFER_STORE_DWORD, O::S_ENDPGM}}; } @@ -13688,7 +13701,7 @@ TestCase ScalarSelectB64PreservesMaskProvenance() { return {"ScalarSelectB64PreservesMaskProvenance", code, {}, - {1, 0}, + {1, 1}, {O::S_MOV_B32, O::S_CMP_EQ_U32, O::S_CSELECT_B64, O::S_NOT_B64, O::S_CSELECT_B32, O::V_CMP_EQ_U32, O::S_MOV_B64, O::V_MOV_B32, O::BUFFER_STORE_DWORD, O::S_ENDPGM}}; @@ -13720,7 +13733,7 @@ TestCase ScalarWqmB64SelectsSccDomain() { return {"ScalarWqmB64SelectsSccDomain", code, {}, - {1, 0}, + {1, 1}, {O::S_MOV_B32, O::S_WQM_B64, O::S_CSELECT_B32, O::V_CMP_EQ_U32, O::S_MOV_B64, O::S_NOT_B64, O::V_MOV_B32, O::BUFFER_STORE_DWORD, O::S_ENDPGM}}; @@ -13754,7 +13767,7 @@ TestCase ScalarMaskProvenanceOverlapAndMixedBinary() { return {"ScalarMaskProvenanceOverlapAndMixedBinary", code, {}, - {0, 1}, + {1, 1}, {O::V_CMP_EQ_U32, O::S_MOV_B64, O::S_NOT_B64, O::S_CSELECT_B32, O::S_MOV_B32, O::S_NAND_B64, O::V_MOV_B32, O::BUFFER_STORE_DWORD, O::S_ENDPGM}}; @@ -14510,9 +14523,7 @@ TestCase VectorAddcWritesPerLaneCarryOut() { test.name = "VectorAddcWritesPerLaneCarryOut"; test.code = code; test.initial = {0x0000ffffu, 0u, 0x0000ffffu, 0u}; - // VCC is represented as the current invocation's carry bit, not a shared - // four-lane ballot word. - test.expected = {1u, 0u, 1u, 0u}; + test.expected = {5u, 5u, 5u, 5u}; test.opcodes = {O::V_MOV_B32, O::V_LSHLREV_B32, O::V_CMP_T_U32, O::BUFFER_LOAD_DWORD, O::V_ADDC_U32, O::BUFFER_STORE_DWORD, O::S_ENDPGM}; @@ -14680,8 +14691,7 @@ TestCase VectorVop3BCarryOutWritesSgprMask() { TestCase test; test.name = "VectorVop3BCarryOutWritesSgprMask"; test.code = code; - // Scalar mask destinations keep the current invocation's bit. - test.expected = std::vector(12, 1u); + test.expected = std::vector(12, 15u); test.opcodes = {O::V_MOV_B32, O::V_ADD_I32, O::V_SUB_I32, O::V_SUBREV_I32, O::BUFFER_STORE_DWORD, O::S_ENDPGM}; test.compute_info.threads_num[0] = 4; @@ -14713,7 +14723,7 @@ TestCase VectorVop3BCarryOutUsesEncodedSdst() { TestCase test; test.name = "VectorVop3BCarryOutUsesEncodedSdst"; test.code = code; - test.expected = std::vector(12, 1u); + test.expected = std::vector(12, 15u); test.opcodes = {O::V_MOV_B32, O::V_ADD_I32, O::V_SUB_I32, O::V_SUBREV_I32, O::BUFFER_STORE_DWORD, O::S_ENDPGM}; test.compute_info.threads_num[0] = 4; @@ -14752,7 +14762,7 @@ TestCase VectorVop3BSubCoU32UsesRdna2Opcode310() { TestCase test; test.name = "VectorVop3BSubCoU32UsesRdna2Opcode310"; test.code = code; - test.expected = {0xffffffffu, 0, 0xfffffffeu, 0x80000001u, 1, 0, 0, 1}; + test.expected = {0xffffffffu, 0, 0xfffffffeu, 0x80000001u, 9, 9, 9, 9}; test.opcodes = {O::V_MOV_B32, O::V_CMP_EQ_U32, O::V_CNDMASK_B32, O::V_SUB_I32, O::BUFFER_STORE_DWORD, O::S_ENDPGM}; test.compute_info.threads_num[0] = 4; @@ -15667,10 +15677,11 @@ TestCase Vop1MoveRelDestination() { AppendVMovLiteral(&code, 6, 0xbbbbbbbbu); AppendVMovLiteral(&code, 7, 0xccccccccu); code.push_back(EncodeSMovB32(124, InlineU32(2))); + code.push_back(EncodeSop1(0x04, 8, 126)); code.push_back(EncodeVopc(0xc2, InlineU32(0), 0)); code.push_back(EncodeSop1(0x04, 126, 106)); code.push_back(EncodeVop1(0x42, 5, Vgpr(1))); - code.push_back(EncodeSop1(0x04, 126, 193u)); + code.push_back(EncodeSop1(0x04, 126, 8)); code.push_back(EncodeVop2(0x1a, 4, InlineU32(2), 0)); AppendBufferStoreDword(&code, 7, 4); AppendEnd(&code); @@ -17035,7 +17046,7 @@ TestCase SimpleLoop() { O::S_BRANCH, O::V_MOV_B32, O::BUFFER_STORE_DWORD, O::S_ENDPGM}}; } -TestCase BranchVccnzUsesInvocationMask() { +TestCase BranchVccnzUsesWaveMask() { using O = ShaderOpcode; std::vector code = { @@ -17053,9 +17064,9 @@ TestCase BranchVccnzUsesInvocationMask() { AppendEnd(&code); TestCase test; - test.name = "BranchVccnzUsesInvocationMask"; + test.name = "BranchVccnzUsesWaveMask"; test.code = code; - test.expected = {42, 11, 11, 11, 42, 11, 11, 11}; + test.expected = std::vector(8, 42); test.opcodes = {O::V_MOV_B32, O::V_LSHLREV_B32, O::V_ADD_NC_U32, O::V_CMP_EQ_U32, O::S_CBRANCH_VCCNZ, O::S_BRANCH, O::BUFFER_STORE_DWORD, O::S_ENDPGM}; @@ -18747,7 +18758,7 @@ TestCase FlatSubdwordLoadsApplyByteOffset() { return test; } -TestCase BranchVccnzUsesCarryProducedInvocationMask() { +TestCase BranchVccnzUsesCarryProducedWaveMask() { using O = ShaderOpcode; std::vector code = { @@ -18765,9 +18776,9 @@ TestCase BranchVccnzUsesCarryProducedInvocationMask() { AppendEnd(&code); TestCase test; - test.name = "BranchVccnzUsesCarryProducedInvocationMask"; + test.name = "BranchVccnzUsesCarryProducedWaveMask"; test.code = code; - test.expected = {11, 42, 42, 42, 42, 42, 42, 42}; + test.expected = std::vector(8, 42); test.opcodes = {O::V_MOV_B32, O::V_LSHLREV_B32, O::V_ADD_NC_U32, O::V_CMP_F_U32, O::V_ADDC_U32, O::S_CBRANCH_VCCNZ, O::S_BRANCH, O::BUFFER_STORE_DWORD, O::S_ENDPGM}; @@ -18903,11 +18914,13 @@ TestCase ScratchIsPrivatePerInvocation() { TestCase test; test.name = "ScratchIsPrivatePerInvocation"; test.code = std::move(code); - test.expected = {0, 1, 2, 3}; + for (u32 lane = 0; lane < 64; lane++) { + test.expected.push_back(lane); + } test.opcodes = {O::S_MOV_B32, O::V_MOV_B32, O::FLAT_STORE_DWORD, O::FLAT_LOAD_DWORD, O::BUFFER_STORE_DWORD, O::S_ENDPGM}; - test.compute_info.threads_num[0] = 4; + test.compute_info.threads_num[0] = 64; test.compute_info.threads_num[1] = 1; test.compute_info.threads_num[2] = 1; test.compute_info.thread_ids_num = 1; @@ -19589,6 +19602,185 @@ TestCase DsBpermuteWave64UsesIndependentHalves() { return test; } +TestCase Wave64CrossHalfLaneAndLds() { + using O = ShaderOpcode; + std::vector code; + AppendVMovU32(&code, 1, 100); + code.push_back(EncodeVop2(0x25, 1, Vgpr(0), 1)); + AppendVop3(&code, 0x360, 8, Vgpr(1), InlineU32(47)); + AppendStoreSgprAtLaneDwordOffset(&code, 8, 0, 0); + code.push_back(EncodeSop1(0x04, 10, 126)); + code.push_back(EncodeVop2(0x1b, 2, InlineU32(63), 0)); + code.push_back(EncodeVopc(0xc2, InlineU32(40), 2)); + code.push_back(EncodeSop1(0x04, 126, 106)); + code.push_back(EncodeVop1(0x02, 9, Vgpr(1))); + code.push_back(EncodeSop1(0x04, 126, 10)); + AppendStoreSgprAtLaneDwordOffset(&code, 9, 0, 128); + AppendStoreSgprAtLaneDwordOffset(&code, 106, 0, 256); + AppendStoreSgprAtLaneDwordOffset(&code, 107, 0, 384); + code.push_back(EncodeVop2(0x1a, 2, InlineU32(2), 0)); + code.push_back(EncodeDs0(0x0d)); + code.push_back(EncodeDs1(0, 1, 2)); + code.push_back(EncodeSopp(0x0a, 0)); + code.push_back(EncodeVop2(0x1d, 3, InlineU32(64), 0)); + code.push_back(EncodeVop2(0x1a, 3, InlineU32(2), 3)); + code.push_back(EncodeDs0(0x36)); + code.push_back(EncodeDs1(4, 0, 3)); + AppendStoreVgprAtLaneDwordOffset(&code, 4, 0, 512); + AppendEnd(&code); + + TestCase test; + test.name = "Wave64CrossHalfLaneAndLds"; + test.code = std::move(code); + test.initial.resize(640); + test.expected.insert(test.expected.end(), 64, 147); + test.expected.insert(test.expected.end(), 64, 211); + test.expected.insert(test.expected.end(), 64, 140); + test.expected.insert(test.expected.end(), 64, 204); + test.expected.insert(test.expected.end(), 128, 0); + test.expected.insert(test.expected.end(), 128, 256); + for (u32 lane = 0; lane < 128; lane++) { + test.expected.push_back(100 + (lane ^ 64)); + } + test.opcodes = {O::V_MOV_B32, O::V_ADD_NC_U32, O::V_READLANE_B32, + O::V_READFIRSTLANE_B32, O::V_CMP_EQ_U32, O::S_MOV_B64, + O::V_LSHLREV_B32, O::V_AND_B32, O::V_XOR_B32, O::DS_WRITE_B32, + O::S_BARRIER, O::DS_READ_B32, O::BUFFER_STORE_DWORD, O::S_ENDPGM}; + test.required_spirv = {"OpGroupNonUniformBallot", "OpGroupNonUniformShuffle", + "OpControlBarrier"}; + test.ir_counts = {{"Barrier", 1}}; + test.compute_info.threads_num[0] = 128; + test.compute_info.threads_num[1] = 1; + test.compute_info.threads_num[2] = 1; + test.compute_info.thread_ids_num = 1; + test.has_compute_info = true; + return test; +} + +TestCase Wave64RawMasksAndScalarBranch() { + using O = ShaderOpcode; + std::vector code; + AppendVMovU32(&code, 1, 7); + AppendSMovLiteral(&code, 8, 33u << 16u); + code.push_back(EncodeSop2(0x29, 126, 193u, 8)); // captured ES BFE EXEC,-1,width + AppendStoreVgprAtLaneDwordOffset(&code, 1, 0, 0); + code.push_back(EncodeSop2(0x1f, 126, InlineU32(1), InlineU32(32))); + const auto exec_branch = code.size(); + code.push_back(0); + code.push_back(EncodeSopp(0x0a, 0)); + code.push_back(EncodeSop1(0x24, 10, 193u)); // s_and_saveexec_b64 s[10:11],-1 + const auto scc_branch = code.size(); + code.push_back(0); + AppendVMovU32(&code, 1, 9); + AppendStoreVgprAtLaneDwordOffset(&code, 1, 0, 64); + const auto restore = code.size(); + code[exec_branch] = EncodeSopp(0x08, restore - exec_branch - 1); + code[scc_branch] = EncodeSopp(0x04, restore - scc_branch - 1); + code.push_back(EncodeSop1(0x04, 126, 193u)); + code.push_back(EncodeVopc(0xc2, Vgpr(0), 0)); + code.push_back(EncodeSop1(0x04, 12, 106)); + code.push_back(EncodeSMovB32(13, InlineU32(1))); + AppendVop3(&code, 0x101, 2, InlineU32(0), InlineU32(11), 12); + AppendStoreVgprAtLaneDwordOffset(&code, 2, 0, 128); + code.push_back(EncodeSMovB32(127, InlineU32(1))); + AppendVMovU32(&code, 3, 13); + AppendStoreVgprAtLaneDwordOffset(&code, 3, 0, 192); + code.push_back(EncodeSop1(0x04, 126, InlineU32(1))); + AppendStoreVgprAtLaneDwordOffset(&code, 3, 0, 256); + code.push_back(EncodeSMovB32(107, InlineU32(2))); + code.push_back(EncodeSMovB32(127, 107)); + AppendStoreVgprAtLaneDwordOffset(&code, 1, 0, 320); + AppendEnd(&code); + + TestCase test; + test.name = "Wave64RawMasksAndScalarBranch"; + test.code = std::move(code); + test.initial.assign(384, 0xdeadbeef); + test.expected = test.initial; + for (u32 lane = 0; lane < 64; lane++) { + if (lane < 33) { + test.expected[lane] = 7; + test.expected[192 + lane] = 13; + } + test.expected[128 + lane] = lane < 33 ? 11 : 0; + } + test.expected[64 + 32] = 9; + test.expected[256] = 13; + test.expected[320] = 7; + test.expected[320 + 33] = 7; + test.opcodes = {O::V_MOV_B32, O::S_MOV_B32, O::S_BFE_U64, O::S_LSHL_B64, + O::S_CBRANCH_EXECZ, O::S_AND_SAVEEXEC_B64, O::S_CBRANCH_SCC0, + O::S_BARRIER, O::S_MOV_B64, O::V_CMP_EQ_U32, O::V_CNDMASK_B32, + O::V_LSHLREV_B32, O::BUFFER_STORE_DWORD, O::S_ENDPGM}; + test.compute_info.threads_num[0] = 64; + test.compute_info.threads_num[1] = 1; + test.compute_info.threads_num[2] = 1; + test.compute_info.thread_ids_num = 1; + test.has_compute_info = true; + return test; +} + +TestCase Wave64PartialMultidimensionalWorkgroup() { + using O = ShaderOpcode; + std::vector code; + code.push_back(EncodeVop2(0x25, 3, Vgpr(2), 2)); + code.push_back(EncodeVop2(0x25, 3, Vgpr(2), 3)); + code.push_back(EncodeVop2(0x25, 3, Vgpr(1), 3)); + code.push_back(EncodeVop2(0x1a, 3, InlineU32(3), 3)); + code.push_back(EncodeVop2(0x25, 3, Vgpr(0), 3)); + code.push_back(EncodeVop2(0x1a, 4, InlineU32(2), 3)); + AppendVMovU32(&code, 5, 1); + AppendBufferStoreOpcode(&code, 0x32, 5, 4, false); + AppendEnd(&code); + TestCase test; + test.name = "Wave64PartialMultidimensionalWorkgroup"; + test.code = std::move(code); + test.initial.resize(128); + test.expected.assign(96, 1); + test.expected.resize(128); + test.opcodes = {O::V_ADD_NC_U32, O::V_LSHLREV_B32, O::V_MOV_B32, + O::BUFFER_ATOMIC_ADD, O::S_ENDPGM}; + test.compute_info.threads_num[0] = 8; + test.compute_info.threads_num[1] = 3; + test.compute_info.threads_num[2] = 4; + test.compute_info.thread_ids_num = 3; + test.has_compute_info = true; + return test; +} + +TestCase Wave64AppendConsumeHighHalf() { + using O = ShaderOpcode; + std::vector code; + AppendSMovLiteral(&code, 124, 4); + code.push_back(EncodeSop2(0x1f, 126, 193u, InlineU32(32))); + code.push_back(EncodeDs0(0x3e, 0, true)); + code.push_back(EncodeDs1(1, 0, 0)); + AppendStoreVgprAtLaneDwordOffset(&code, 1, 0, 0); + code.push_back(EncodeDs0(0x3d, 0, true)); + code.push_back(EncodeDs1(2, 0, 0)); + AppendStoreVgprAtLaneDwordOffset(&code, 2, 0, 64); + AppendEnd(&code); + TestCase test; + test.name = "Wave64AppendConsumeHighHalf"; + test.code = std::move(code); + test.initial.assign(128, 0xdeadbeef); + test.expected = test.initial; + for (u32 lane = 32; lane < 64; lane++) { + test.expected[lane] = 10; + test.expected[64 + lane] = 42; + } + test.gds_initial = {10}; + test.expected_gds = {10}; + test.opcodes = {O::S_MOV_B32, O::S_LSHL_B64, O::DS_APPEND, O::DS_CONSUME, + O::V_LSHLREV_B32, O::BUFFER_STORE_DWORD, O::S_ENDPGM}; + test.compute_info.threads_num[0] = 64; + test.compute_info.threads_num[1] = 1; + test.compute_info.threads_num[2] = 1; + test.compute_info.thread_ids_num = 1; + test.has_compute_info = true; + return test; +} + TestCase BufferAtomicVariants() { using O = ShaderOpcode; @@ -21780,8 +21972,8 @@ std::vector MakeCases() { AddCase(VectorCompareInvertedMaskSelect); AddCase(BranchSelect); AddCase(SimpleLoop); - AddCase(BranchVccnzUsesInvocationMask); - AddCase(BranchVccnzUsesCarryProducedInvocationMask); + AddCase(BranchVccnzUsesWaveMask); + AddCase(BranchVccnzUsesCarryProducedWaveMask); AddCase(ScalarMemoryLoadVariants); AddCase(ScalarLoadSignedImmediateOffsetAddsSoffset); AddCase(ScalarLoadAlignsComponentsAndMasksAddress); @@ -21874,6 +22066,10 @@ std::vector MakeCases() { AddCase(DsSwizzleInvalidSourceLaneZero); AddCase(DsBpermuteCapturedExecOffsetAndWrap); AddCase(DsBpermuteWave64UsesIndependentHalves); + AddCase(Wave64CrossHalfLaneAndLds); + AddCase(Wave64RawMasksAndScalarBranch); + AddCase(Wave64PartialMultidimensionalWorkgroup); + AddCase(Wave64AppendConsumeHighHalf); AddCase(BufferAtomicVariants); AddCase(BufferAtomicCmpSwapExactRaw); AddCase(BufferAtomicGlc0DoesNotReturnOldValue); @@ -26093,6 +26289,30 @@ int main(int argc, char **argv) { return 0; } #endif + if (argc == 2 && std::strcmp(argv[1], "--wave64-only") == 0) { + VulkanHarness vulkan; + RunCase(&vulkan, DsBpermuteWave64UsesIndependentHalves()); + RunCase(&vulkan, Wave64CrossHalfLaneAndLds()); + RunCase(&vulkan, Wave64RawMasksAndScalarBranch()); + RunCase(&vulkan, Wave64PartialMultidimensionalWorkgroup()); + RunCase(&vulkan, Wave64AppendConsumeHighHalf()); + RunCase(&vulkan, VectorAddcWritesPerLaneCarryOut()); + RunCase(&vulkan, VectorVop3BCarryOutWritesSgprMask()); + RunCase(&vulkan, VectorVop3BCarryOutUsesEncodedSdst()); + RunCase(&vulkan, DispatcherIrreducibleControlFlow()); + RunCase(&vulkan, ScalarSaveExecOps()); + RunCase(&vulkan, ScalarOrn2SaveexecUsesSourceOrNotExec()); + RunCase(&vulkan, ScalarNotB64UpdatesScc()); + RunCase(&vulkan, ScalarSelectB64PreservesMaskProvenance()); + RunCase(&vulkan, ScalarWqmB64SelectsSccDomain()); + RunCase(&vulkan, ScalarMaskProvenanceOverlapAndMixedBinary()); + RunCase(&vulkan, ScratchIsPrivatePerInvocation()); + RunCase(&vulkan, Vop1MoveRelDestination()); + RunCase(&vulkan, BranchVccnzUsesWaveMask()); + RunCase(&vulkan, BranchVccnzUsesCarryProducedWaveMask()); + RunCase(&vulkan, ImageSampleAndGather()); + return 0; + } if (argc == 2 && std::strcmp(argv[1], "--clip-control-only") == 0) { CheckClipControlDepthClipState(); return 0; diff --git a/tests/shaderCfgTests.cpp b/tests/shaderCfgTests.cpp index 0154c2571..1bdf419fc 100644 --- a/tests/shaderCfgTests.cpp +++ b/tests/shaderCfgTests.cpp @@ -21,7 +21,6 @@ #include "graphics/shader/recompiler/ir/passes/ReadLaneElimination.h" #include "graphics/shader/recompiler/ir/passes/ResourceTracking.h" #include "graphics/shader/recompiler/ir/passes/ShaderInfoCollection.h" -#include "graphics/shader/recompiler/ir/passes/SharedMemoryBarrier.h" #include "graphics/shader/recompiler/ir/passes/SrtWalker.h" #include "graphics/shader/recompiler/ir/passes/SsaRewrite.h" #include "graphics/shader/shader.h" @@ -7882,9 +7881,9 @@ void TestNewShaderRecompilerCfgExecSccSharedArm() { "EXEC/SCC shared-arm SPIR-V has the wrong selection-merge count"); Check(SpirvInstructionOpcodeCount(result.spirv, 251) == 0u, "EXEC/SCC shared-arm SPIR-V unexpectedly used dispatcher OpSwitch"); - Check(!Common::ContainsStr(DisassembleSpirvBinary(result.spirv), + Check(Common::ContainsStr(DisassembleSpirvBinary(result.spirv), "OpGroupNonUniformBallot"), - "per-invocation EXEC/SCC branch reconstructed a native subgroup mask"); + "scalar mask SCC did not reduce the complete wave mask"); CheckSpirvBinaryValidates(result.spirv); } @@ -8727,6 +8726,43 @@ void TestNewShaderRecompilerSetpcBranch() { CheckSpirvBinaryValidates(result.spirv); } +void TestFusedShaderHandoffPreservesRegisters() { + using namespace ShaderRecompiler; + const uint32_t front[] = { + EncodeSMovB32(12, 255), 0x1003u, // three vertices and one primitive + EncodeSop1(0x20, 0, 6), // merged-stage handoff through s[6:7] + 0xffffffffu, // front shader metadata must not be decoded + }; + const uint32_t back[] = { + EncodeSMovB32(124, 12), // s_mov_b32 m0, s12 + EncodeSopp(0x10, 9), // s_sendmsg MSG_GS_ALLOC_REQ + EncodeSopp(0x01), + }; + ShaderVertexInputInfo input{}; + input.mesh.threads_num[0] = 192; + input.mesh.threads_num[1] = input.mesh.threads_num[2] = 1; + input.mesh.primitives_per_group = 62; + input.mesh.vertices_per_group = 64; + CompileOptions options{}; + options.stage = ShaderType::Mesh; + options.input_info.vertex = &input; + options.back_code = back; + auto translated = TranslateProgram(front, options); + uint32_t allocations = 0; + for (const auto* block: translated.program.blocks) { + for (const auto& inst: *block) { + if (inst.GetOpcode() != IR::ValueOpcode::MeshAllocate) { + continue; + } + const auto value = inst.Arg(0).Resolve(); + Check(value.IsImmediate() && value.U32() == 0x1003u, + "fused back shader lost the front shader's scalar register value"); + allocations++; + } + } + Check(allocations == 1u, "fused shader omitted the back shader allocation"); +} + void TestNewShaderRecompilerSetpcJumpTable() { const uint32_t shader[] = { EncodeSop2(0x07, 0, 0, 129), // s_min_u32 s0, s0, 1 @@ -9237,6 +9273,7 @@ void TestTypedEntryStateIsMinimal() { uint32_t set_exec_lo = 0; uint32_t set_exec_hi = 0; uint32_t ballots = 0; + const IR::Inst* ballot = nullptr; IR::Value exec; IR::Value exec_lo; IR::Value exec_hi; @@ -9245,6 +9282,7 @@ void TestTypedEntryStateIsMinimal() { switch (inst.GetOpcode()) { case IR::ValueOpcode::Ballot: ballots++; + ballot = &inst; break; case IR::ValueOpcode::SetExec: set_exec++; @@ -9274,11 +9312,19 @@ void TestTypedEntryStateIsMinimal() { } } } + const auto is_mask_word = [&](IR::Value value, uint32_t word) { + const auto* extract = value.ResolveInstruction(); + return extract != nullptr && + extract->GetOpcode() == IR::ValueOpcode::CompositeExtractU32x4 && + extract->Arg(0).ResolveInstruction() == ballot && + extract->Arg(1).IsImmediate() && extract->Arg(1).U32() == word; + }; Check(set_exec == 1u && set_exec_lo == 1u && set_exec_hi == 1u && - ballots == 0u && exec.IsImmediate() && exec.U1() && - exec_lo.IsImmediate() && exec_lo.U32() == 1u && - exec_hi.IsImmediate() && exec_hi.U32() == 0u, - "typed entry is not the local {1,0} invocation mask"); + ballots == 1u && exec.IsImmediate() && exec.U1() && + ballot->Arg(0) == exec && is_mask_word(exec_lo, 0u) && + (wave_size == 64u ? is_mask_word(exec_hi, 1u) + : exec_hi.IsImmediate() && exec_hi.U32() == 0u), + "typed entry did not materialize the active invocation mask once"); IR::RewriteToSsa(values.blocks); IR::RemoveIdentities(values.blocks); @@ -9609,21 +9655,9 @@ void TestValuePhiValidation() { #endif } -void TestWqmMaskSignatureAndU64ShiftConstantPropagation() { +void TestU64ShiftConstantPropagation() { using namespace ShaderRecompiler::IR; -#if KYTY_PLATFORM != KYTY_PLATFORM_WINDOWS - Program malformed; - malformed.block_storage.push_back(std::make_unique()); - malformed.blocks.push_back(malformed.block_storage.back().get()); - malformed.block_info.emplace_back(); - IREmitter malformed_ir(malformed.blocks.front()); - auto malformed_value = malformed_ir.Emit(ValueOpcode::WqmMask, {Value(true)}); - malformed_value.TryInstruction()->SetArg(0, Value(uint64_t{1})); - ExpectFatal([&] { ValidateProgram(malformed, false); }, - "invalid WqmMask operand type did not terminate IR validation"); -#endif - Program shifts; shifts.block_storage.push_back(std::make_unique()); shifts.blocks.push_back(shifts.block_storage.back().get()); @@ -10021,8 +10055,8 @@ void TestNewShaderRecompilerVertexExportUsesInvocationExecMask() { auto result = RecompileForTest(shader, options); CheckSpirvBinaryValidates(result.spirv); const auto source = DisassembleSpirvBinary(result.spirv); - Check(!Common::ContainsStr(source, "OpLoad %uint %gl_SubgroupInvocationID"), - "vertex EXEC guard depends on the native subgroup lane"); + Check(Common::ContainsStr(source, "OpLoad %uint %gl_SubgroupInvocationID"), + "raw EXEC=1 did not select guest lane zero"); Check(Common::ContainsStr(source, "OpBranchConditional"), "vertex export lost its per-invocation EXEC guard"); } @@ -10049,8 +10083,8 @@ void TestNewShaderRecompilerPerInvocationMasksWithoutMirrors() { Check( !Common::ContainsStr(source, "OpGroupNonUniformBallot"), "per-invocation VCC producer still materialized a shared subgroup mask"); - Check(!Common::ContainsStr(source, "OpLoad %uint %gl_SubgroupInvocationID"), - "per-invocation BFM EXEC prefix still selected native subgroup lanes"); + Check(Common::ContainsStr(source, "OpLoad %uint %gl_SubgroupInvocationID"), + "BFM EXEC prefix did not select the four requested guest lanes"); Check(!Common::ContainsStr(source, "%vcc_lo") && !Common::ContainsStr(source, "%vcc_hi"), "per-invocation comparison retained VCC register mirrors"); @@ -10067,8 +10101,8 @@ void TestNewShaderRecompilerPerInvocationMasksWithoutMirrors() { Check(Common::ContainsStr(wqm_source, "OpCapability GroupNonUniformBallot") && Common::ContainsStr(wqm_source, "OpGroupNonUniformBallot"), "per-invocation scalar WQM omitted its subgroup ballot capability"); - Check(SpirvInstructionOpcodeCount(result.spirv, 132u) == 2u, - "wave64 WQM did not compact exactly two ballot words"); + Check(SpirvInstructionOpcodeCount(result.spirv, 132u) == 1u, + "wave64 WQM did not expand its scalar word pair together"); auto wave32_options = options; wave32_options.wave_size = 32u; @@ -11352,9 +11386,9 @@ void TestComputeLdsAllocationIdentity() { "compute pipeline identity omitted the LDS allocation"); auto split_wave = lds_896; - split_wave.needs_lds_barriers = true; + split_wave.host_subgroup_size = 32; Check(MakeStageStaticKey(lds_896) != MakeStageStaticKey(split_wave), - "compute shader identity omitted split-wave LDS synchronization"); + "compute shader identity omitted the host subgroup size"); auto tg_size_disabled = lds_896; auto tg_size_enabled = lds_896; @@ -11418,135 +11452,6 @@ void TestComputeLdsAllocationIdentity() { CheckSpirvBinaryValidates(append_result.spirv); } -void TestWave64LdsSynchronization() { - const uint32_t shader[] = { - EncodeDs0(0x0d), // ds_write_b32 v0, v1 - EncodeDs1(0, 1, 0), - EncodeSopp(0x01), - }; - - const auto compile = [&](bool needs_lds_barriers, uint32_t wave_size, - uint32_t threads) { - ShaderComputeInputInfo input_info{}; - input_info.lds_size_dwords = 128; - input_info.needs_lds_barriers = needs_lds_barriers; - input_info.threads_num[0] = threads; - input_info.threads_num[1] = 1; - input_info.threads_num[2] = 1; - - auto options = MakeCompileOptions(ShaderType::Compute); - options.wave_size = wave_size; - options.input_info.compute = &input_info; - auto result = RecompileForTest(shader, options); - CheckSpirvBinaryValidates(result.spirv); - return result; - }; - - const auto native_wave64 = compile(false, 64, 64); - Check(!SpirvContainsOpcode(native_wave64.spirv, 224), - "native wave64 shader gained a synthetic workgroup barrier"); - - const auto split_wave64 = compile(true, 64, 64); - Check(SpirvContainsOpcode(split_wave64.spirv, 224), - "split wave64 LDS shader omitted workgroup synchronization"); - - const auto split_wave32 = compile(true, 32, 64); - Check(!SpirvContainsOpcode(split_wave32.spirv, 224), - "wave32 shader gained wave64 synchronization"); - - const auto larger_workgroup = compile(true, 64, 128); - Check(!SpirvContainsOpcode(larger_workgroup.spirv, 224), - "multi-wave workgroup gained unsupported synthetic synchronization"); -} - -void TestSharedMemoryBarrierSafety() { - using namespace ShaderRecompiler::IR; - - const auto make_program = [](size_t block_count) { - Program program; - program.memory_info.push_back({.kind = ResourceKind::Lds}); - for (size_t index = 0; index < block_count; index++) { - program.block_storage.push_back(std::make_unique()); - program.blocks.push_back(program.block_storage.back().get()); - program.block_info.emplace_back(); - program.block_info.back().id = static_cast(index); - } - return program; - }; - const auto append_write = [](Block& block) { - auto& inst = block.AppendNewInst( - ValueOpcode::WriteSharedU32, - {Value(0u), Value(1u), Value(true)}); - inst.SetFlags(MemoryFlags{.index = 0}); - }; - const auto append_read = [](Block& block) { - auto& inst = block.AppendNewInst(ValueOpcode::LoadSharedU32, - {Value(0u), Value(true)}); - inst.SetFlags(MemoryFlags{.index = 0}); - }; - const auto count_barriers = [](const Block& block) { - return std::ranges::count_if(block, [](const Inst& inst) { - return inst.GetOpcode() == ValueOpcode::Barrier; - }); - }; - const auto divergent_condition = [](Program& program, size_t block) { - IREmitter ir(program.blocks[block]); - const auto local_id = U32(ir.Emit( - ValueOpcode::GetBuiltin, - {Value(static_cast(StageInputKind::LocalInvocationId)), - Value(0u)})); - return ir.INotEqual(local_id, U32(Value(0u))); - }; - - ShaderComputeInputInfo compute_info{}; - compute_info.needs_lds_barriers = true; - compute_info.lds_size_dwords = 128; - compute_info.threads_num[0] = 64; - compute_info.threads_num[1] = 1; - compute_info.threads_num[2] = 1; - - auto phases = make_program(1); - append_write(*phases.blocks[0]); - append_read(*phases.blocks[0]); - const auto phase_stats = InsertSharedMemoryBarriers(phases, 64u, compute_info); - std::vector phase_opcodes; - for (const auto& inst : *phases.blocks[0]) { - phase_opcodes.push_back(inst.GetOpcode()); - } - Check(phase_stats.inserted_barriers == 2 && phase_opcodes.size() == 4 && - phase_opcodes[0] == ValueOpcode::WriteSharedU32 && - phase_opcodes[1] == ValueOpcode::Barrier && - phase_opcodes[2] == ValueOpcode::LoadSharedU32 && - phase_opcodes[3] == ValueOpcode::Barrier, - "LDS write/read phases were not separated by a barrier"); - - auto selection = make_program(3); - selection.block_info[0].terminator.kind = - ShaderRecompiler::CFG::TerminatorKind::ConditionalBranch; - selection.block_info[0].terminator.merge_block = 2; - selection.block_info[0].condition = divergent_condition(selection, 0); - append_write(*selection.blocks[1]); - append_read(*selection.blocks[2]); - const auto selection_stats = - InsertSharedMemoryBarriers(selection, 64u, compute_info); - Check(selection_stats.inserted_barriers == 2 && - count_barriers(*selection.blocks[1]) == 0 && - selection.blocks[2]->begin()->GetOpcode() == - ValueOpcode::Barrier, - "workgroup barrier was placed inside divergent selection flow"); - - auto loop = make_program(1); - loop.block_info[0].terminator.kind = - ShaderRecompiler::CFG::TerminatorKind::ConditionalBranch; - loop.block_info[0].condition = divergent_condition(loop, 0); - append_write(*loop.blocks[0]); - append_read(*loop.blocks[0]); - const auto loop_stats = InsertSharedMemoryBarriers(loop, 64u, compute_info); - Check(loop_stats.inserted_barriers == 0 && - count_barriers(*loop.blocks[0]) == 0, - "workgroup barrier was inserted into divergent loop control"); -} - void TestPixelProgramCacheBindingIdentity() { const uint32_t shader_01[] = {0xbf810000u}; const uint32_t shader_10[] = {0xbf810000u}; @@ -11914,8 +11819,8 @@ void TestNewShaderRecompilerSpirvSizeBaselines() { .conditional_branches = 1, .ballots = 1}, ShaderType::Vertex); - Check(Common::ContainsStr(wqm_result.ir_dump, "WqmMask"), - "WQM size fixture no longer reaches per-invocation WqmMask IR"); + Check(Common::ContainsStr(wqm_result.ir_dump, "WqmU64"), + "WQM size fixture no longer reaches scalar mask expansion"); const uint32_t dispatcher[] = { EncodeSopp(0x05, 2), // entry -> B, fallthrough A @@ -12028,6 +11933,7 @@ int main() { TestNewShaderRecompilerPixelImageSampleLodSelection(); TestNewShaderRecompilerBranchConditionForms(); TestNewShaderRecompilerSetpcBranch(); + TestFusedShaderHandoffPreservesRegisters(); TestNewShaderRecompilerSetpcJumpTable(); TestNewShaderRecompilerPrunesUnreachableSetpcMetadata(); TestNewShaderRecompilerSetpcDwordJumpTable(); @@ -12036,7 +11942,7 @@ int main() { TestFinalSsaRejectsRegisterStatePseudos(); #endif TestValuePhiValidation(); - TestWqmMaskSignatureAndU64ShiftConstantPropagation(); + TestU64ShiftConstantPropagation(); #if KYTY_PLATFORM != KYTY_PLATFORM_WINDOWS TestNativeWideValueValidation(); #endif @@ -12064,8 +11970,6 @@ int main() { TestGraphicsCreateInterpolantMapping(); TestNewShaderRecompilerPixelPipelineEntry(); TestComputeLdsAllocationIdentity(); - TestWave64LdsSynchronization(); - TestSharedMemoryBarrierSafety(); TestPixelProgramCacheBindingIdentity(); TestGraphicsPushConstantPlacement(); TestNewShaderRecompilerUnsupportedMemoryDecode();