diff --git a/src/graphics/host_gpu/renderer/renderDraw.cpp b/src/graphics/host_gpu/renderer/renderDraw.cpp index 873595104..4b39325a1 100644 --- a/src/graphics/host_gpu/renderer/renderDraw.cpp +++ b/src/graphics/host_gpu/renderer/renderDraw.cpp @@ -1032,6 +1032,7 @@ void RenderExecutor::ExecutePreparedDraw(uint64_t submit_id, CommandBuffer& buff uint32_t mesh_groups = 0; if (mesh_active) { const auto& mesh = state.vertex_info[0].mesh; + EXIT_NOT_IMPLEMENTED(mesh.fast_launch && (draw.IsIndexed() || primitive_restart_enable)); static std::atomic_bool restart_warned = false; if (primitive_restart_enable && !restart_warned.exchange(true, std::memory_order_relaxed)) { std::printf("Warning: primitive restart is not implemented for mesh shaders; " diff --git a/src/graphics/shader/recompiler/frontend/translate/Translate.cpp b/src/graphics/shader/recompiler/frontend/translate/Translate.cpp index 25478144f..2c918dc4f 100644 --- a/src/graphics/shader/recompiler/frontend/translate/Translate.cpp +++ b/src/graphics/shader/recompiler/frontend/translate/Translate.cpp @@ -1138,12 +1138,13 @@ IR::Program TranslateProgram(const Decoder::Program& decoded, const CFG::Graph& return entry_ir.ISub(lhs, minimum(lhs, rhs)); }; const auto local = builtin(IR::StageInputKind::LocalInvocationIndex); - const auto primitive_chunk = entry_ir.IMul(builtin(IR::StageInputKind::WorkgroupId, 0), - u32(mesh.primitives_per_group)); + const auto group = builtin(IR::StageInputKind::WorkgroupId, 0); + const auto primitive_chunk = mesh.fast_launch ? group : + entry_ir.IMul(group, u32(mesh.primitives_per_group)); const auto step = u32(mesh.InputPrimitiveStep()); const auto size = u32(mesh.InputPrimitiveSize()); - const auto chunk = entry_ir.IMul(primitive_chunk, step); - const auto vertices = + const auto chunk = mesh.fast_launch ? group : entry_ir.IMul(primitive_chunk, step); + const auto vertices = mesh.fast_launch ? u32(mesh.vertices_per_group) : minimum(subtract_saturate(draw(0), chunk), u32(mesh.vertices_per_group)); const auto primitives = entry_ir.Select( entry_ir.ULessThan(vertices, size), u32(0), @@ -1164,56 +1165,64 @@ IR::Program TranslateProgram(const Decoder::Program& decoded, const CFG::Graph& entry_ir.BitwiseOr(wave_info, entry_ir.BitwiseOr(entry_ir.ShiftLeftLogical( primitive_count, u32(8)), vertex_count))); - // GS adjacency addresses local ES records in LDS. Fans retain the draw's - // center in every subgroup; strip winding follows the global primitive. - const auto vertex = entry_ir.IMul(local, step); - auto first = vertex; - auto second = u32(0); - auto third = u32(0); - if (mesh.InputPrimitiveSize() >= 2u) { - second = entry_ir.IAdd(vertex, u32(1)); + if (mesh.fast_launch) { + // Fast launch broadcasts the subgroup's base vertex and instance to every lane. + entry_ir.SetVectorReg(static_cast(5), entry_ir.IAdd(draw(1), group)); + entry_ir.SetVectorReg( + static_cast(6), + entry_ir.IAdd(draw(2), builtin(IR::StageInputKind::WorkgroupId, 1))); + } else { + // GS adjacency addresses local ES records in LDS. Fans retain the draw's + // center in every subgroup; strip winding follows the global primitive. + const auto vertex = entry_ir.IMul(local, step); + auto first = vertex; + auto second = u32(0); + auto third = u32(0); + if (mesh.InputPrimitiveSize() >= 2u) { + second = entry_ir.IAdd(vertex, u32(1)); + } + if (mesh.InputPrimitiveSize() == 3u) { + third = entry_ir.IAdd(vertex, u32(2)); + } + auto input_vertex = entry_ir.IAdd(chunk, local); + if (mesh.input_primitive == static_cast(Prospero::PrimitiveType::kTriFan)) { + first = u32(0); + input_vertex = entry_ir.Select(entry_ir.IEqual(local, u32(0)), u32(0), input_vertex); + } else if (mesh.input_primitive == static_cast(Prospero::PrimitiveType::kTriStrip)) { + const auto parity = entry_ir.BitwiseAnd(entry_ir.IAdd(primitive_chunk, local), u32(1)); + first = entry_ir.IAdd(first, parity); + second = entry_ir.ISub(second, parity); + } + entry_ir.SetVectorReg(static_cast(0), + entry_ir.BitwiseOr(entry_ir.ShiftLeftLogical(first, u32(2)), + entry_ir.ShiftLeftLogical(second, u32(18)))); + entry_ir.SetVectorReg(static_cast(1), + entry_ir.ShiftLeftLogical(third, u32(2))); + const auto index_bytes = draw(3); + const auto indexed = entry_ir.INotEqual(index_bytes, u32(0)); + const auto index_low = draw(4); + const auto byte_offset = entry_ir.IAdd(entry_ir.BitwiseAnd(index_low, u32(3)), + entry_ir.IMul(input_vertex, index_bytes)); + const auto index_resource = entry_ir.Emit( + IR::ValueOpcode::GetAddressResource, + {entry_ir.BitwiseAnd(index_low, u32(~3u)), draw(5)}); + const auto memory_index = static_cast(result.memory_info.size()); + result.memory_info.push_back({.kind = IR::ResourceKind::Global}); + const auto packed_index = entry_ir.Emit( + IR::ValueOpcode::LoadAddressU32, + {index_resource, entry_ir.BitwiseAnd(byte_offset, u32(~3u)), u32(0), + entry_ir.LogicalAnd(indexed, entry_ir.ULessThan(local, vertices))}, + IR::MemoryFlags {.index = memory_index}); + const auto index = IR::U32(entry_ir.Emit( + IR::ValueOpcode::BitFieldUExtract, + {packed_index, entry_ir.IMul(entry_ir.BitwiseAnd(byte_offset, u32(3)), u32(8)), + entry_ir.IMul(index_bytes, u32(8))})); + entry_ir.SetVectorReg(static_cast(5), + entry_ir.IAdd(draw(1), entry_ir.Select(indexed, index, input_vertex))); + entry_ir.SetVectorReg( + static_cast(8), + entry_ir.IAdd(draw(2), builtin(IR::StageInputKind::WorkgroupId, 1))); } - if (mesh.InputPrimitiveSize() == 3u) { - third = entry_ir.IAdd(vertex, u32(2)); - } - auto input_vertex = entry_ir.IAdd(chunk, local); - if (mesh.input_primitive == static_cast(Prospero::PrimitiveType::kTriFan)) { - first = u32(0); - input_vertex = entry_ir.Select(entry_ir.IEqual(local, u32(0)), u32(0), input_vertex); - } else if (mesh.input_primitive == static_cast(Prospero::PrimitiveType::kTriStrip)) { - const auto parity = entry_ir.BitwiseAnd(entry_ir.IAdd(primitive_chunk, local), u32(1)); - first = entry_ir.IAdd(first, parity); - second = entry_ir.ISub(second, parity); - } - entry_ir.SetVectorReg(static_cast(0), - entry_ir.BitwiseOr(entry_ir.ShiftLeftLogical(first, u32(2)), - entry_ir.ShiftLeftLogical(second, u32(18)))); - entry_ir.SetVectorReg(static_cast(1), - entry_ir.ShiftLeftLogical(third, u32(2))); - const auto index_bytes = draw(3); - const auto indexed = entry_ir.INotEqual(index_bytes, u32(0)); - const auto index_low = draw(4); - const auto byte_offset = entry_ir.IAdd(entry_ir.BitwiseAnd(index_low, u32(3)), - entry_ir.IMul(input_vertex, index_bytes)); - const auto index_resource = entry_ir.Emit( - IR::ValueOpcode::GetAddressResource, - {entry_ir.BitwiseAnd(index_low, u32(~3u)), draw(5)}); - const auto memory_index = static_cast(result.memory_info.size()); - result.memory_info.push_back({.kind = IR::ResourceKind::Global}); - const auto packed_index = entry_ir.Emit( - IR::ValueOpcode::LoadAddressU32, - {index_resource, entry_ir.BitwiseAnd(byte_offset, u32(~3u)), u32(0), - entry_ir.LogicalAnd(indexed, entry_ir.ULessThan(local, vertices))}, - IR::MemoryFlags {.index = memory_index}); - const auto index = IR::U32(entry_ir.Emit( - IR::ValueOpcode::BitFieldUExtract, - {packed_index, entry_ir.IMul(entry_ir.BitwiseAnd(byte_offset, u32(3)), u32(8)), - entry_ir.IMul(index_bytes, u32(8))})); - entry_ir.SetVectorReg(static_cast(5), - entry_ir.IAdd(draw(1), entry_ir.Select(indexed, index, input_vertex))); - entry_ir.SetVectorReg( - static_cast(8), - entry_ir.IAdd(draw(2), builtin(IR::StageInputKind::WorkgroupId, 1))); } else if (options.stage == ShaderType::Local) { entry_ir.SetScalarReg(static_cast(3), IR::U32(IR::Value(64u))); entry_ir.SetVectorReg(static_cast(2), diff --git a/src/graphics/shader/shader.cpp b/src/graphics/shader/shader.cpp index 208cc95b7..7445d08e0 100644 --- a/src/graphics/shader/shader.cpp +++ b/src/graphics/shader/shader.cpp @@ -617,7 +617,8 @@ void BuildStageStaticKey(const ShaderVertexInputInfo& info, std::vector(mesh.fast_launch)}); } key.push_back(info.tess.input_control_points); if (info.tess.input_control_points != 0) { @@ -719,6 +720,9 @@ ShaderParams PrepareProgram(const HW::VertexShaderInfo& regs, const HW::Context& mesh.provoking_vertex = context.GetModeControl().provoking_vtx_last ? 2u : 0u; mesh.lds_size_dwords = static_cast(regs.gs_regs.rsrc2.lds_size) * 128u; mesh.scratch_size_dwords = data.scratch_size_dwords; + const auto fast_launch = (context.GetShaderStages() >> 19u) & 3u; + EXIT_NOT_IMPLEMENTED(fast_launch > 1u); + mesh.fast_launch = fast_launch != 0; if (data.type == Prospero::ShaderBinaryType::kGsFront) { EXIT_IF(regs.gs_regs.data_addr == 0); const auto [back, back_hash] = ShaderGetMappedData(regs.gs_regs.data_addr, "ShaderGetInputInfoGS():"); @@ -731,9 +735,20 @@ ShaderParams PrepareProgram(const HW::VertexShaderInfo& regs, const HW::Context& params.hash = XXH3_64bits(hashes, sizeof(hashes)); mesh.scratch_size_dwords = std::max(mesh.scratch_size_dwords, back.scratch_size_dwords); } - EXIT_NOT_IMPLEMENTED(regs.gs_regs.rsrc1.gs_vgpr_component_count != 3u || - regs.gs_regs.rsrc2.es_vgpr_component_count != 3u); const auto& group = user_config.GetGeControl(); + if (mesh.fast_launch) { + const auto& user_vgpr = user_config.GetGeUserVgprEn(); + EXIT_NOT_IMPLEMENTED(data.type != Prospero::ShaderBinaryType::kGs || + regs.gs_regs.rsrc1.gs_vgpr_component_count != 0u || + regs.gs_regs.rsrc2.es_vgpr_component_count != 1u || + user_config.GetPrimType() != Prospero::PrimitiveType::kPointList || + group.primitive_group_size != 1u || group.vertex_group_size != 1u || + user_vgpr.vgpr1 || user_vgpr.vgpr2 || user_vgpr.vgpr3 || + mesh.max_vertices != sh.m_vgtGsMaxVertOut); + } else { + EXIT_NOT_IMPLEMENTED(regs.gs_regs.rsrc1.gs_vgpr_component_count != 3u || + regs.gs_regs.rsrc2.es_vgpr_component_count != 3u); + } if ((user_config.GetPrimType() != Prospero::PrimitiveType::kPointList && user_config.GetPrimType() != Prospero::PrimitiveType::kLineList && user_config.GetPrimType() != Prospero::PrimitiveType::kTriFan && @@ -746,7 +761,8 @@ ShaderParams PrepareProgram(const HW::VertexShaderInfo& regs, const HW::Context& mesh.input_primitive, sh.m_vgtGsOutPrimType, sh.m_vgtGsMaxVertOut, group.primitive_group_size, group.vertex_group_size, mesh.max_vertices); } - mesh.max_primitives = group.primitive_group_size * (sh.m_vgtGsMaxVertOut - 2u); + mesh.max_primitives = mesh.fast_launch ? mesh.max_vertices : + group.primitive_group_size * (sh.m_vgtGsMaxVertOut - 2u); mesh.primitives_per_group = std::min({static_cast(group.primitive_group_size), mesh.InputPrimitiveCount(group.vertex_group_size), mesh.max_vertices / sh.m_vgtGsMaxVertOut}); diff --git a/src/graphics/shader/shader.h b/src/graphics/shader/shader.h index 22086ca1e..221b43cc5 100644 --- a/src/graphics/shader/shader.h +++ b/src/graphics/shader/shader.h @@ -88,6 +88,7 @@ struct ShaderMeshInputInfo: ShaderWorkgroupInputInfo { uint32_t max_vertices = 0; uint32_t max_primitives = 0; uint32_t provoking_vertex = 0; + bool fast_launch = false; [[nodiscard]] constexpr uint32_t InputPrimitiveSize() const { switch (static_cast(input_primitive)) { diff --git a/tests/shaderCfgTests.cpp b/tests/shaderCfgTests.cpp index 09cb2201f..9a35cb16c 100644 --- a/tests/shaderCfgTests.cpp +++ b/tests/shaderCfgTests.cpp @@ -10108,6 +10108,44 @@ void TestMergedShaderUserDataSnapshot() { Check(fan_input.mesh.primitives_per_group == 30 && fan_input.mesh.vertices_per_group == 32 && fan_input.mesh.max_vertices == 256 && fan_input.mesh.max_primitives == 192, "captured triangle-fan GS configuration lost its subgroup assembly limits"); + context.SetShaderStages(0x00482030u); + context.SetMaxOutputPerSubgroup(32); + context.SetGsMaxVertOut(32); + user_config.SetPrimitiveType(Prospero::PrimitiveType::kPointList); + user_config.SetGeControl({1, 1}); + regs.gs_regs.rsrc1.gs_vgpr_component_count = 0; + regs.gs_regs.rsrc2.es_vgpr_component_count = 1; + ShaderVertexInputInfo fast_input{}; + PrepareProgram(regs, context, user_config, fast_input); + Check(fast_input.mesh.fast_launch && fast_input.mesh.threads_num[0] == 32 && + fast_input.mesh.primitives_per_group == 1 && fast_input.mesh.vertices_per_group == 1 && + fast_input.mesh.max_vertices == 32 && fast_input.mesh.max_primitives == 32, + "captured fast-launch shader lost its per-group launch and output limits"); + auto ordinary_input = fast_input; + ordinary_input.mesh.fast_launch = false; + Check(MakeStageStaticKey(fast_input) != MakeStageStaticKey(ordinary_input), + "fast launch shared the ordinary input-assembly shader key"); + + const uint32_t fast_code[] = { + EncodeSMovB32(124, 255), 0x20020u, // allocate the full 32 vertices and primitives + EncodeSopp(0x10, 9), + EncodeVop1(0x01, 0, 256 + 5), + EncodeVop1(0x01, 1, 256 + 6), + EncodeExp0(0x0c, 0xf, false), EncodeExp1(0, 1, 0, 1), + EncodeExp0(0x14, 0x1), EncodeExp1(0, 0, 0, 0), EncodeSopp(0x01), + }; + options.input_info.vertex = &fast_input; + options.wave_size = fast_input.mesh.wave_size; + options.back_code = {}; + const auto fast_result = RecompileForTest(fast_code, options); + CheckSpirvBinaryValidates(fast_result.spirv); + Check(fast_result.program.memory_info.empty(), + "compiled fast-launch entry retained an unused index-buffer resource"); + const auto fast_source = DisassembleSpirvBinary(fast_result.spirv); + Check(SpirvSourceHasInstructionUsing(fast_source, "OpExecutionMode", "OutputVertices 32") && + SpirvSourceHasInstructionUsing(fast_source, "OpExecutionMode", "OutputPrimitivesEXT 32"), + "fast-launch SPIR-V truncated the mesh output capacity"); + } void TestEmbeddedFetchPreservesSharedScalarLoad() { @@ -10278,8 +10316,10 @@ void TestMeshInputAssembly() { uint32_t wave_info, first, second, third, byte_offset, vertex_id; bool fetch; uint32_t wave_size = 64; + bool fast_launch = false; + uint32_t threads = 256; }; - const Case cases[] = { + std::vector cases = { {Prospero::PrimitiveType::kTriList, 14, 177, 14, 2, 2, 0x1002, UINT32_MAX, 0x40000309, 6, 7, 8, 340, 0xabcc, true}, {Prospero::PrimitiveType::kTriList, 14, 177, 14, 9, 2, 0x1002, 0, @@ -10327,14 +10367,30 @@ void TestMeshInputAssembly() { {Prospero::PrimitiveType::kTriStrip, 40, 40, 0, 32, 0, 0, 11, 0x81000608, 32, 33, 34, 0, 43, false, 32}, }; + for (const uint32_t wave_size : {32u, 64u}) { + for (const uint32_t threads : {wave_size, 2u * wave_size}) { + for (const uint32_t group : {0u, 1u, 11u}) { + for (const uint32_t lane : {0u, wave_size - 1u, threads - 1u}) { + const uint32_t wave_info = ((threads / wave_size) << 28u) | + ((lane / wave_size) << 24u) | + (lane < wave_size ? 0x101u : 0u); + const uint32_t base_vertex = group == 11u ? UINT32_MAX : 11u; + cases.push_back({Prospero::PrimitiveType::kPointList, 1, 12, group, lane, 0, 0, + base_vertex, wave_info, 0, 0, 0, 0, group + base_vertex, false, + wave_size, true, threads}); + } + } + } + } for (const auto &test : cases) { ShaderVertexInputInfo input{}; auto &mesh = input.mesh; mesh.input_primitive = static_cast(test.topology); mesh.wave_size = test.wave_size; + mesh.fast_launch = test.fast_launch; mesh.primitives_per_group = mesh.InputPrimitiveCount(test.capacity); mesh.vertices_per_group = mesh.InputVertexCount(mesh.primitives_per_group); - mesh.threads_num[0] = 256; + mesh.threads_num[0] = test.threads; mesh.threads_num[1] = mesh.threads_num[2] = 1; Decoder::Program decoded; CFG::Graph graph; @@ -10367,16 +10423,21 @@ void TestMeshInputAssembly() { } } ConstantPropagationPass(program.blocks); - Check(load != nullptr && load->Arg(1).Resolve().U32() == test.byte_offset && - load->Arg(3).Resolve().U1() == test.fetch, - "mesh index fetch address or active-lane predicate is wrong"); - const auto *resource = load->Arg(0).ResolveInstruction(); - Check(resource != nullptr && resource->Arg(0).Resolve().U32() == (test.address_low & ~3u) && - resource->Arg(1).Resolve().U32() == 0x12 && - program.memory_info[load->Flags().index].kind == ResourceKind::Global, - "mesh index fetch lost its aligned guest address resource"); - load->ReplaceUsesWith(Value(test.fetch ? 0xabcd0123u : 0u)); - ConstantPropagationPass(program.blocks); + if (test.fast_launch) { + Check(load == nullptr && program.memory_info.empty(), + "fast launch emitted an ordinary input-assembly index resource"); + } else { + Check(load != nullptr && load->Arg(1).Resolve().U32() == test.byte_offset && + load->Arg(3).Resolve().U1() == test.fetch, + "mesh index fetch address or active-lane predicate is wrong"); + const auto *resource = load->Arg(0).ResolveInstruction(); + Check(resource != nullptr && resource->Arg(0).Resolve().U32() == (test.address_low & ~3u) && + resource->Arg(1).Resolve().U32() == 0x12 && + program.memory_info[load->Flags().index].kind == ResourceKind::Global, + "mesh index fetch lost its aligned guest address resource"); + load->ReplaceUsesWith(Value(test.fetch ? 0xabcd0123u : 0u)); + ConstantPropagationPass(program.blocks); + } std::array vgprs{}; uint32_t sgpr3 = 0; for (const auto &inst : *program.blocks.front()) { @@ -10387,7 +10448,8 @@ void TestMeshInputAssembly() { } } Check(sgpr3 == test.wave_info && vgprs[0] == ((test.first << 2) | (test.second << 18)) && - vgprs[1] == test.third * 4 && vgprs[5] == test.vertex_id && vgprs[8] == 9, + vgprs[1] == test.third * 4 && vgprs[5] == test.vertex_id && + vgprs[test.fast_launch ? 6 : 8] == 9u, "mesh prolog changed input assembly, wave counts, vertex ID, or instance ID"); } }