diff --git a/src/graphics/guest_gpu/command_processor/commandProcessor.h b/src/graphics/guest_gpu/command_processor/commandProcessor.h index 7729fd162..d47cbc3de 100644 --- a/src/graphics/guest_gpu/command_processor/commandProcessor.h +++ b/src/graphics/guest_gpu/command_processor/commandProcessor.h @@ -73,6 +73,9 @@ public: void SetIndexBufferSize(uint32_t index_buffer_size); void SetDrawIndirectArgsBaseAddress(uint64_t draw_indirect_args_base_addr); void SetDispatchIndirectArgsBaseAddress(uint64_t dispatch_indirect_args_base_addr); + [[nodiscard]] uint64_t GetDispatchIndirectArgsBaseAddress() const { + return m_dispatch_indirect_args_base_addr; + } void SetNumInstances(uint32_t num_instances); void DrawIndex(DrawIndexArgs args); void DrawIndexOffset(uint32_t index_offset, uint32_t index_count); @@ -99,7 +102,7 @@ public: void TriggerEopEventAtEndOfPipe(uint32_t interrupt_context_id); void DispatchDirect(uint32_t thread_group_x, uint32_t thread_group_y, uint32_t thread_group_z, uint32_t mode); - void DispatchIndirect(uint32_t data_offset, uint32_t mode); + void DispatchIndirect(uint64_t args_addr, uint32_t mode); void WaitFlipDone(uint32_t video_out_handle, uint32_t display_buffer_index); void TriggerEvent(uint32_t event_type, uint32_t event_index, uint64_t event_address = 0); diff --git a/src/graphics/guest_gpu/command_processor/pm4Handlers.cpp b/src/graphics/guest_gpu/command_processor/pm4Handlers.cpp index 168d7b647..c95983b69 100644 --- a/src/graphics/guest_gpu/command_processor/pm4Handlers.cpp +++ b/src/graphics/guest_gpu/command_processor/pm4Handlers.cpp @@ -1331,26 +1331,13 @@ KYTY_CP_OP_PARSER(CpOpDispatchIndirect) { EXIT_NOT_IMPLEMENTED(cmd_id != 0xc0011600 && cmd_id != 0xc0021600); if (cmd_id == 0xc0021600) { - struct DispatchIndirectArgs { - uint32_t thread_group_x; - uint32_t thread_group_y; - uint32_t thread_group_z; - }; - - auto* args = reinterpret_cast( - buffer[0] | (static_cast(buffer[1]) << 32u)); - uint32_t mode = buffer[2]; - - EXIT_NOT_IMPLEMENTED(args == nullptr); - cp.DispatchDirect(args->thread_group_x, args->thread_group_y, args->thread_group_z, mode); - + cp.DispatchIndirect(buffer[0] | (static_cast(buffer[1]) << 32u), buffer[2]); return 3; } - uint32_t data_offset = buffer[0]; - uint32_t mode = buffer[1]; - - cp.DispatchIndirect(data_offset, mode); + const auto base_addr = cp.GetDispatchIndirectArgsBaseAddress(); + EXIT_NOT_IMPLEMENTED(base_addr == 0); + cp.DispatchIndirect(base_addr + buffer[0], buffer[1]); return 2; } diff --git a/src/graphics/guest_gpu/graphicsRun.cpp b/src/graphics/guest_gpu/graphicsRun.cpp index 451fc1bd6..4debaef90 100644 --- a/src/graphics/guest_gpu/graphicsRun.cpp +++ b/src/graphics/guest_gpu/graphicsRun.cpp @@ -1090,19 +1090,16 @@ void CommandProcessor::DispatchDirect(uint32_t thread_group_x, uint32_t thread_g }*/ } -void CommandProcessor::DispatchIndirect(uint32_t data_offset, uint32_t mode) { - struct DispatchIndirectArgs { - uint32_t thread_group_x; - uint32_t thread_group_y; - uint32_t thread_group_z; - }; - - EXIT_NOT_IMPLEMENTED(m_dispatch_indirect_args_base_addr == 0); - - const auto args_addr = m_dispatch_indirect_args_base_addr + data_offset; - auto* args = reinterpret_cast(args_addr); - - DispatchDirect(args->thread_group_x, args->thread_group_y, args->thread_group_z, mode); +void CommandProcessor::DispatchIndirect(uint64_t args_addr, uint32_t mode) { + EXIT_NOT_IMPLEMENTED(args_addr == 0 || (args_addr & 3u) != 0); + if ((mode & Pm4::COMPUTE_DISPATCH_INITIATOR_USE_THREAD_DIMENSIONS) != 0) { + const auto* args = reinterpret_cast(args_addr); + DispatchDirect(args->x, args->y, args->z, mode); + return; + } + m_sh_ctx.SetCsWaveSize(Pm4::ComputeWaveSize(mode)); + CheckBuffer(); + m_renderer.GetRenderExecutor().DispatchIndirect(m_submit_id, CurrentBuffer(), args_addr, mode); } void CommandProcessor::DrawIndexAuto(DrawAutoArgs args) { diff --git a/src/graphics/guest_gpu/pm4.h b/src/graphics/guest_gpu/pm4.h index 7da943e3a..038bd02c2 100644 --- a/src/graphics/guest_gpu/pm4.h +++ b/src/graphics/guest_gpu/pm4.h @@ -910,6 +910,7 @@ constexpr uint32_t COMPUTE_PGM_RSRC1_WGP_MODE_MASK = 0x1; constexpr uint32_t COMPUTE_PGM_RSRC1_FWD_PROGRESS_SHIFT = 31; constexpr uint32_t COMPUTE_PGM_RSRC1_FWD_PROGRESS_MASK = 0x1; +constexpr uint32_t COMPUTE_DISPATCH_INITIATOR_USE_THREAD_DIMENSIONS = 1u << 5u; constexpr uint32_t COMPUTE_DISPATCH_INITIATOR_CS_W32_EN_SHIFT = 15; constexpr uint32_t COMPUTE_DISPATCH_INITIATOR_CS_W32_EN_MASK = 0x1; diff --git a/src/graphics/host_gpu/renderer/render.h b/src/graphics/host_gpu/renderer/render.h index 8c474e5b4..9d04104f5 100644 --- a/src/graphics/host_gpu/renderer/render.h +++ b/src/graphics/host_gpu/renderer/render.h @@ -45,6 +45,7 @@ enum class CommandBufferDebugOp : uint32_t { EopFlip, EopWriteBackFlip, EopOnlyFlip, + DispatchIndirect, Unknown, }; @@ -156,6 +157,8 @@ public: void DispatchDirect(uint64_t submit_id, CommandBuffer& buffer, uint32_t thread_group_x, uint32_t thread_group_y, uint32_t thread_group_z, uint32_t mode); + void DispatchIndirect(uint64_t submit_id, CommandBuffer& buffer, uint64_t args_addr, + uint32_t mode); [[nodiscard]] PreparedBindings PrepareBindings(const ShaderStageRuntime& runtime); void FindBuffers(PreparedBindings& bindings); diff --git a/src/graphics/host_gpu/renderer/renderCompute.cpp b/src/graphics/host_gpu/renderer/renderCompute.cpp index c057e3d14..f8bfbc2d1 100644 --- a/src/graphics/host_gpu/renderer/renderCompute.cpp +++ b/src/graphics/host_gpu/renderer/renderCompute.cpp @@ -9,6 +9,7 @@ #include "graphics/guest_gpu/gpu_defs.h" #include "graphics/guest_gpu/graphicsRun.h" #include "graphics/guest_gpu/hardwareContext.h" +#include "graphics/guest_gpu/pm4.h" #include "graphics/host_gpu/graphicContext.h" #include "graphics/host_gpu/renderer/image/imageInfo.h" #include "graphics/host_gpu/renderer/pipeline/descriptors.h" @@ -395,4 +396,60 @@ void RenderExecutor::DispatchDirect(uint64_t submit_id, CommandBuffer& buffer, ResetBindings(); } +void RenderExecutor::DispatchIndirect(uint64_t submit_id, CommandBuffer& buffer, + uint64_t args_addr, uint32_t mode) { + EXIT_IF(buffer.IsInvalid() || args_addr == 0 || (args_addr & 3u) != 0 || + (mode & Pm4::COMPUTE_DISPATCH_INITIATOR_USE_THREAD_DIMENSIONS) != 0); + m_context.GetCommandScheduler().PopPendingOperations(); + buffer.SetDebugInfo(static_cast(CommandBufferDebugOp::DispatchIndirect), submit_id, + static_cast(args_addr), static_cast(args_addr >> 32u), + 0, mode, buffer.GetShaders().GetCs().cs_regs.data_addr); + Common::LockGuard lock(m_context.GetMutex()); + const auto& cs_regs = buffer.GetShaders().GetCs(); + if (cs_regs.cs_regs.data_addr == 0) { + return; + } + ShaderComputeInputInfo input_info {}; + const auto compute_program = m_context.GetPipelineCache().GetComputeProgram( + cs_regs, buffer.GetRegisters().GetShaderRegisters(), input_info); + buffer.EndRendering(); + auto& pipeline = m_context.GetPipelineCache().GetComputePipeline(input_info, compute_program); + auto bindings = PrepareBindings(input_info.stage); + FindBuffers(bindings); + const auto& program = *input_info.stage.program; + if (program.info.uses_dma) { + m_context.PrepareBda(); + } + RebindImages(bindings); + // Acquiring arguments can merge cache buffers; finalize shader bindings afterward. + const auto [args_buffer, args_offset] = m_context.GetBufferCache().ObtainBuffer( + args_addr, sizeof(vk::DispatchIndirectCommand), false); + EXIT_IF(args_buffer == nullptr || (args_offset & 3u) != 0); + RebindBuffers(bindings); + PreparedBindings* descriptor_stage = &bindings; + CommitBindings(buffer, vk::PipelineBindPoint::eCompute, pipeline, + std::span {&descriptor_stage, 1u}); + const auto vk_buffer = buffer.Handle(); + const bool has_storage_writes = HasShaderBufferWrites(input_info.stage) || + std::any_of(program.info.images.begin(), program.info.images.end(), [](const auto& image) { + return image.written && image.resource_class == + ShaderRecompiler::IR::ImageResourceClass::Storage; + }); + if (has_storage_writes) { + ShaderWriteHazardBarrier(vk_buffer, vk::PipelineStageFlagBits::eComputeShader); + } + vk::MemoryBarrier barrier {}; + barrier.srcAccessMask = vk::AccessFlagBits::eShaderWrite | vk::AccessFlagBits::eTransferWrite; + barrier.dstAccessMask = vk::AccessFlagBits::eIndirectCommandRead; + vk_buffer.pipelineBarrier(vk::PipelineStageFlagBits::eAllGraphics | + vk::PipelineStageFlagBits::eComputeShader | + vk::PipelineStageFlagBits::eTransfer, + vk::PipelineStageFlagBits::eDrawIndirect, {}, + 1, &barrier, 0, nullptr, 0, nullptr); + vk_buffer.bindPipeline(vk::PipelineBindPoint::eCompute, pipeline.pipeline); + vk_buffer.dispatchIndirect(args_buffer->Handle(), args_offset); + ShaderAccessBarrier(vk_buffer, vk::PipelineStageFlagBits::eComputeShader); + ResetBindings(); +} + } // namespace Libs::Graphics diff --git a/tests/ShaderRecompilerComputeTests.cpp b/tests/ShaderRecompilerComputeTests.cpp index efe1a85c3..1e0aa82c9 100644 --- a/tests/ShaderRecompilerComputeTests.cpp +++ b/tests/ShaderRecompilerComputeTests.cpp @@ -8703,6 +8703,181 @@ public: return result; } + void CheckNativeIndirectDispatch() { + constexpr const char *name = "NativeIndirectDispatch"; + constexpr uintptr_t base = 0x0000000204600000ull; + constexpr uint64_t allocation_size = 0x100000; + constexpr uint64_t case_size = 0x10000; + constexpr uint32_t sentinel = 0xa5a5a5a5u; + struct DispatchCase { + std::array dimensions; + uint32_t mode; + uint32_t expected_threads; + bool transfer = false; + }; + constexpr std::array cases{ + DispatchCase{{2, 1, 1}, 0x41u, 8}, + DispatchCase{{0, 1, 1}, 0x41u, 0}, + DispatchCase{{1, 0, 1}, 0x41u, 0}, + DispatchCase{{1, 1, 0}, 0x41u, 0}, + DispatchCase{{3, 1, 1}, 0x41u, 12, true}, + DispatchCase{{8, 1, 1}, 0x61u, 8}, + }; + + // Separate DWORD descriptors preserve two owners until the indirect argument + // range spans them. The consumer's output aliases the second owner. + std::vector writer; + for (u32 i = 0; i < 3; i++) { + writer.push_back(EncodeVop1(0x01u, 1, 12u + i)); + writer.push_back(EncodeMubuf0(0x1cu, 0, false, false)); + writer.push_back(EncodeMubuf1(1, i, 0)); + } + AppendEnd(&writer); + std::vector consumer; + AppendVop3(&consumer, 0x346u, 1, 4, InlineU32(2), Vgpr(0)); + consumer.push_back(EncodeVop2(0x25u, 2, InlineU32(1), 1)); + consumer.push_back(EncodeVop2(0x1au, 1, InlineU32(2), 1)); + consumer.push_back(EncodeMubuf0(0x1cu)); + consumer.push_back(EncodeMubuf1(2, 0, 1)); + AppendEnd(&consumer); + for (const auto *code : {&writer, &consumer}) { + ShaderMapUserData(reinterpret_cast(code->data()), + {.type = Prospero::ShaderBinaryType::kCs, + .code_size_bytes = static_cast(code->size() * sizeof(u32))}); + } + + EnsureRuntimeContext(); + int64_t direct_offset = -1; + Require(name, "direct allocation", + Libs::LibKernel::Memory::KernelAllocateDirectMemory( + 0, Libs::LibKernel::Memory::KernelGetDirectMemorySize(), + allocation_size, case_size, 0, &direct_offset) == 0, + "indirect argument allocation failed"); + void *mapped = reinterpret_cast(base); + Require(name, "direct mapping", + Libs::LibKernel::Memory::KernelMapDirectMemory( + &mapped, allocation_size, 0x3, 0x10, direct_offset, case_size) == 0 && + mapped == reinterpret_cast(base), + "indirect argument mapping failed"); + std::memset(mapped, 0xa5, allocation_size); + const auto argument_address = [&](size_t index) { + return base + index * case_size + + (cases[index].transfer ? 0x200u : BufferCache::CACHING_PAGESIZE - 4u); + }; + for (size_t i = 0; i < cases.size(); i++) { + std::memset(reinterpret_cast(argument_address(i)), 0, 12); + } + + RenderContext context(m_runtime_context); + context.InitializeGpu(nullptr); + LibKernel::Memory::InstallGpuResources(&context); + context.GetGpu().SendCommandSync([&] { + CommandProcessor processor(context, 0); + processor.BufferInit(); + auto &scheduler = context.GetCommandScheduler(); + auto &cache = context.GetBufferCache(); + auto &shaders = processor.GetShCtx(); + context.MapMemory(base, allocation_size); + const auto set_buffer = [&](u32 sgpr, uint64_t address, u32 bytes) { + ShaderBufferResource descriptor{}; + descriptor.UpdateAddress48(address); + descriptor.fields[2] = bytes; + descriptor.fields[3] = DstSel(4, 5, 6, 7) | + (static_cast(Prospero::BufferFormat::k32UInt) << 12u); + for (u32 i = 0; i < 4; i++) { + shaders.SetCsUserSgpr(sgpr + i, descriptor.fields[i], + HW::UserSgprType::Unknown); + } + }; + for (size_t index = 0; index < cases.size(); index++) { + const auto &test = cases[index]; + const auto args = argument_address(index); + const auto output = base + index * case_size + BufferCache::CACHING_PAGESIZE + 0x100u; + if (test.transfer) { + for (u32 i = 0; i < 3; i++) { + auto [buffer, offset] = cache.ObtainBuffer(args + i * 4u, 4, true); + buffer->Fill(offset, 4, test.dimensions[i]); + } + } else { + shaders.SetCsShader({.data_addr = reinterpret_cast(writer.data()), + .num_thread_x = 1, .num_thread_y = 1, .num_thread_z = 1, + .wave_size = 64, .user_sgpr = 15}); + for (u32 i = 0; i < 3; i++) { + set_buffer(i * 4u, args + i * 4u, 4); + shaders.SetCsUserSgpr(12u + i, test.dimensions[i], HW::UserSgprType::Unknown); + } + processor.DispatchDirect(1, 1, 1, 0x41u); + } + std::array stale{}; + Require(name, "GPU-only argument write", + LibKernel::Memory::TryReadBacking(args, stale.data(), sizeof(stale)) && + stale == std::array{}, + "argument publication changed the CPU backing before indirect dispatch"); + const auto first_owner = BufferCacheTestAccess::PageOwner(cache, args); + const auto second_owner = BufferCacheTestAccess::PageOwner(cache, args + 8u); + if (!test.transfer) { + Require(name, "separate argument owners", + first_owner && second_owner && first_owner != second_owner, + "the indirect argument merge fixture already shared one owner"); + } + + shaders.SetCsShader({.data_addr = reinterpret_cast(consumer.data()), + .num_thread_x = 4, .num_thread_y = 1, .num_thread_z = 1, + .wave_size = 64, .user_sgpr = 4, .tgid_x_en = true}); + set_buffer(0, output, 16u * sizeof(u32)); + const auto tick = scheduler.CurrentTick(); + if (test.transfer) { + const std::array packet{static_cast(args), + static_cast(args >> 32u), test.mode}; + Require(name, "absolute indirect packet", + CpOpDispatchIndirect(processor, 0xc0021600u, packet.data(), 0, 0) == 3, + "the absolute indirect packet was not consumed"); + } else { + processor.SetDispatchIndirectArgsBaseAddress(base + index * case_size); + const std::array packet{ + static_cast(BufferCache::CACHING_PAGESIZE - 4u), test.mode}; + Require(name, "offset indirect packet", + CpOpDispatchIndirect(processor, 0xc0011600u, packet.data(), 0, 0) == 2, + "the offset indirect packet was not consumed"); + } + if (test.mode == 0x41u) { + Require(name, "asynchronous indirect dispatch", scheduler.CurrentTick() == tick, + "workgroup-count arguments caused a host submission or readback"); + if (!test.transfer) { + const auto merged = BufferCacheTestAccess::PageOwner(cache, args); + Require(name, "argument/output owner merge", + merged && merged != first_owner && merged != second_owner && + merged == BufferCacheTestAccess::PageOwner(cache, output), + "argument discovery did not merge the consumer's buffer owner"); + } + } + cache.ReadMemory(output, 16u * sizeof(u32)); + std::array actual{}; + Require(name, "output readback", + LibKernel::Memory::TryReadBacking(output, actual.data(), sizeof(actual)), + "consumer output could not be read back"); + for (u32 i = 0; i < actual.size(); i++) { + const auto expected = i < test.expected_threads ? i + 1u : sentinel; + Require(name, "indirect invocation coverage", actual[i] == expected, + "case " + std::to_string(index) + " word " + std::to_string(i) + + " expected " + Hex(expected) + ", got " + Hex(actual[i])); + } + } + context.UnmapMemory(base, allocation_size); + scheduler.Finish(); + }); + LibKernel::Memory::InstallGpuResources(nullptr); + context.ShutdownGpu(); + Require(name, "unmap direct backing", + Libs::LibKernel::Memory::KernelMunmap(base, allocation_size) == 0, + "indirect argument mapping release failed"); + Require(name, "release direct backing", + Libs::LibKernel::Memory::KernelReleaseDirectMemory( + direct_offset, allocation_size) == 0, + "indirect argument allocation release failed"); + std::printf("[gpu] %-32s ok\n", name); + } + void CheckRenderExecutorDccFixedClearFloat() { constexpr const char *name = "RenderExecutorDccFixedClearFloat"; constexpr uintptr_t base = 0x0000000204100000ull; @@ -32908,6 +33083,7 @@ int main(int argc, char **argv) { CheckDynamicRenderingState(); VulkanHarness vulkan; vulkan.CheckComputeMetaClearClassification(); + vulkan.CheckNativeIndirectDispatch(); vulkan.CheckRenderExecutorColorVolumeDiscovery(); vulkan.CheckRenderExecutorDccFixedClearFloat(); vulkan.CheckSampledDccClear(); @@ -33094,6 +33270,7 @@ int main(int argc, char **argv) { vulkan.CheckGpuMappedRangeLifecycle(); vulkan.CheckStreamBufferRing(); vulkan.CheckGpuTilerCpuParity(); + vulkan.CheckNativeIndirectDispatch(); #if KYTY_PLATFORM == KYTY_PLATFORM_WINDOWS vulkan.CheckRenderExecutorColorDiscovery(); vulkan.CheckRenderExecutorColorVolumeDiscovery();