diff --git a/src/graphics/guest_gpu/graphicsRun.cpp b/src/graphics/guest_gpu/graphicsRun.cpp index 8b367e232..2aac8dab6 100644 --- a/src/graphics/guest_gpu/graphicsRun.cpp +++ b/src/graphics/guest_gpu/graphicsRun.cpp @@ -32,11 +32,10 @@ namespace Libs::Graphics { -static thread_local CommandProcessor* g_current_processor = nullptr; -static thread_local Pm4Execution* g_current_execution = nullptr; -static thread_local uint32_t g_submission_pause_depth = 0; -static thread_local bool g_gpu_mutex_owned = false; -static thread_local bool g_gpu_thread = false; +static thread_local CommandProcessor* g_current_processor = nullptr; +static thread_local Pm4Execution* g_current_execution = nullptr; +static thread_local bool g_gpu_mutex_owned = false; +static thread_local bool g_gpu_thread = false; class GpuMutexLock final { public: @@ -98,8 +97,6 @@ public: bool trigger_agc_interrupt_on_done); void SubmitFlipPreparation(uint64_t request_id); void Done(); - void PauseSubmissions(); - void ResumeSubmissions(); void Shutdown(); [[nodiscard]] bool IsStopping(); void SendCommand(Common::UniqueFunction&& command); @@ -407,7 +404,7 @@ void CommandProcessor::WriteData(uint32_t* dst, const uint32_t* src, uint32_t dw uint32_t write_control) { const uint32_t dst_sel = ((write_control >> 30u) & 0x1u) | ((write_control >> 7u) & 0x1eu); const uint32_t cache_policy = (write_control >> 25u) & 0x3u; - const uint32_t increment = (write_control >> 16u) & 0x1u; + const uint32_t increment = (write_control >> 16u) & 0x1u; const uint32_t write_confirm = (write_control >> 20u) & 0x1u; switch (dst_sel) { @@ -696,26 +693,6 @@ bool GpuState::Process(Submission& submission) { return complete; } -void GpuState::PauseSubmissions() { - if (g_gpu_mutex_owned) { - EXIT("GPU submissions are already paused by this thread\n"); - } - g_gpu_mutex_owned = true; - m_submission_mutex.Lock(); - if (!IsGpuThread()) { - WaitLocked(); - } - m_renderer.GetCommandScheduler().DrainPriorityOperations(); -} - -void GpuState::ResumeSubmissions() { - if (!g_gpu_mutex_owned) { - EXIT("GPU submissions resumed without an active pause\n"); - } - m_submission_mutex.Unlock(); - g_gpu_mutex_owned = false; -} - Pm4ProcessResult CommandProcessor::Process(Pm4Execution& execution, uint32_t* buffer, uint32_t size_dw) { KYTY_PROFILER_BLOCK("CommandProcessor::Process"); @@ -1693,32 +1670,6 @@ int Gpu::GetFrameNum() const { return m_state->GetFrameNum(); } -void Gpu::PauseSubmissions() { - m_state->PauseSubmissions(); -} - -void Gpu::ResumeSubmissions() { - m_state->ResumeSubmissions(); -} - -Gpu::SubmissionLock::SubmissionLock(Gpu& gpu): m_gpu(gpu) { - if (g_current_processor != nullptr || g_submission_pause_depth == UINT32_MAX) { - EXIT("cannot acquire GPU submission lock in the current state\n"); - } - if (g_submission_pause_depth++ == 0) { - m_gpu.PauseSubmissions(); - } -} - -Gpu::SubmissionLock::~SubmissionLock() { - if (g_submission_pause_depth == 0) { - EXIT("GPU submission lock released without ownership\n"); - } - if (--g_submission_pause_depth == 0) { - m_gpu.ResumeSubmissions(); - } -} - bool Gpu::IsCommandProcessorThread() noexcept { return g_current_processor != nullptr; } @@ -1727,12 +1678,4 @@ CommandProcessor* Gpu::CurrentCommandProcessor() noexcept { return g_current_processor; } -bool Gpu::SubmissionLockHeld() noexcept { - return g_submission_pause_depth != 0; -} - -bool Gpu::MutexHeld() noexcept { - return g_gpu_mutex_owned; -} - } // namespace Libs::Graphics diff --git a/src/graphics/guest_gpu/graphicsRun.h b/src/graphics/guest_gpu/graphicsRun.h index aa6210daf..39ee55f3a 100644 --- a/src/graphics/guest_gpu/graphicsRun.h +++ b/src/graphics/guest_gpu/graphicsRun.h @@ -35,25 +35,8 @@ public: [[nodiscard]] static bool IsCommandProcessorThread() noexcept; [[nodiscard]] static CommandProcessor* CurrentCommandProcessor() noexcept; - [[nodiscard]] static bool SubmissionLockHeld() noexcept; - [[nodiscard]] static bool MutexHeld() noexcept; - - class SubmissionLock final { - public: - explicit SubmissionLock(Gpu& gpu); - ~SubmissionLock(); - KYTY_CLASS_NO_COPY(SubmissionLock); - - private: - Gpu& m_gpu; - }; private: - friend class SubmissionLock; - - void PauseSubmissions(); - void ResumeSubmissions(); - std::unique_ptr m_state; }; } // namespace Libs::Graphics diff --git a/src/graphics/host_gpu/renderer/cache/gpuResourceManager.cpp b/src/graphics/host_gpu/renderer/cache/gpuResourceManager.cpp index efbe25009..be000246f 100644 --- a/src/graphics/host_gpu/renderer/cache/gpuResourceManager.cpp +++ b/src/graphics/host_gpu/renderer/cache/gpuResourceManager.cpp @@ -7,7 +7,8 @@ namespace Libs::Graphics { GpuResourceManager::GpuResourceManager(GraphicContext& graphics, CommandScheduler& scheduler) - : m_buffer_cache(graphics, scheduler, m_page_manager, m_texture_cache, m_resource_mutex), + : m_scheduler(scheduler), + m_buffer_cache(graphics, scheduler, m_page_manager, m_texture_cache, m_resource_mutex), m_texture_cache(graphics, scheduler, m_page_manager, m_buffer_cache, m_resource_mutex) {} GpuResourceManager::~GpuResourceManager() = default; @@ -101,7 +102,22 @@ void GpuResourceManager::MapMemory(uint64_t vaddr, uint64_t size) { } void GpuResourceManager::UnmapMemory(uint64_t vaddr, uint64_t size) { + if (CommandScheduler::InDeferredOperation()) { + EXIT("unsupported memory unmap from an asynchronous GPU completion, " + "addr=0x%016" PRIx64 " size=0x%016" PRIx64 "\n", + vaddr, size); + } + if (m_resource_mutex.IsOwnedByCurrentThread()) { + EXIT("unsupported memory unmap from a pre-owned resource transaction, " + "addr=0x%016" PRIx64 " size=0x%016" PRIx64 "\n", + vaddr, size); + } const auto unmap = [this, vaddr, size] { + if (m_scheduler.Active()) { + const auto tick = m_scheduler.CurrentTick(); + m_scheduler.FinishCurrent(); + m_scheduler.WaitPriorityOperations(tick); + } m_buffer_cache.UnmapMemory(vaddr, size); m_texture_cache.UnmapMemory(vaddr, size); m_page_manager.OnGpuUnmap(vaddr, size); @@ -109,13 +125,9 @@ void GpuResourceManager::UnmapMemory(uint64_t vaddr, uint64_t size) { m_mapped_ranges.Subtract(vaddr, size); }; if (m_gpu == nullptr) { - if (m_resource_mutex.IsOwnedByCurrentThread()) { - EXIT("cannot synchronously unmap from a resource transaction\n"); - } unmap(); return; } - Gpu::SubmissionLock submissions(*m_gpu); m_gpu->SendCommandSync(unmap); } diff --git a/src/graphics/host_gpu/renderer/cache/gpuResourceManager.h b/src/graphics/host_gpu/renderer/cache/gpuResourceManager.h index 8a3098b87..d9aa17750 100644 --- a/src/graphics/host_gpu/renderer/cache/gpuResourceManager.h +++ b/src/graphics/host_gpu/renderer/cache/gpuResourceManager.h @@ -36,6 +36,7 @@ public: private: PageManager m_page_manager; ResourceMutex m_resource_mutex; + CommandScheduler& m_scheduler; BufferCache m_buffer_cache; TextureCache m_texture_cache; mutable std::shared_mutex m_mapped_ranges_mutex; diff --git a/src/libs/agc.cpp b/src/libs/agc.cpp index 8172e08e1..b3b50d738 100644 --- a/src/libs/agc.cpp +++ b/src/libs/agc.cpp @@ -221,57 +221,6 @@ static RegisterDefaults* get_internal_register_defaults(uint32_t ver) { return get_register_defaults(g_agc_internal_reg_defaults_by_version[index], &storage[index]); } -struct PendingGraphicsSegment { - uint32_t* start = nullptr; - uint32_t* end = nullptr; - uint32_t* range_end = nullptr; -}; - -static std::mutex g_pending_graphics_segment_mutex; -static PendingGraphicsSegment g_pending_graphics_segment; - -static void track_pending_graphics_segment_after_submit(uint32_t* dcb, uint32_t size_in_dwords) { - if (dcb == nullptr || size_in_dwords == 0) { - return; - } - - auto* segment_start = dcb + size_in_dwords; - auto* range_end = segment_start + 0xfffffu; - - std::lock_guard lock(g_pending_graphics_segment_mutex); - g_pending_graphics_segment.start = segment_start; - g_pending_graphics_segment.end = segment_start; - g_pending_graphics_segment.range_end = range_end; -} - -static void track_pending_graphics_allocation(uint32_t* cmd, uint32_t size_dw) { - if (cmd == nullptr || size_dw == 0) { - return; - } - - std::lock_guard lock(g_pending_graphics_segment_mutex); - auto* range_start = g_pending_graphics_segment.start; - auto* range_end = g_pending_graphics_segment.range_end; - if (range_start == nullptr || range_end == nullptr || cmd < range_start || cmd >= range_end) { - return; - } - - auto* cmd_end = cmd + size_dw; - if (cmd > g_pending_graphics_segment.end) { - static std::atomic log_count {0}; - if (log_count.fetch_add(1) < 64) { - LOGF("\t pending graphics segment: ignoring non-contiguous allocation cmd = " - "0x%016" PRIx64 ", tracked_end = 0x%016" PRIx64 "\n", - reinterpret_cast(cmd), - reinterpret_cast(g_pending_graphics_segment.end)); - } - return; - } - if (cmd_end > g_pending_graphics_segment.end && cmd_end <= range_end) { - g_pending_graphics_segment.end = cmd_end; - } -} - struct CommandBuffer { using Callback = KYTY_SYSV_ABI bool (*)(CommandBuffer*, uint32_t, void*); @@ -370,7 +319,6 @@ struct CommandBuffer { } auto* ret_ptr = cursor_up; cursor_up += size_dw; - track_pending_graphics_allocation(ret_ptr, size_dw); return ret_ptr; } }; @@ -1918,7 +1866,6 @@ uint32_t* KYTY_SYSV_ABI GraphicsCbReleaseMem(CommandBuffer* buf, uint8_t action, cmd[5] = static_cast(packet_data & 0xffffffffu); cmd[6] = static_cast((packet_data >> 32u) & 0xffffffffu); cmd[7] = interrupt_ctx_id & 0x07ffffffu; - return cmd; } @@ -3815,150 +3762,6 @@ static void submit_dcb(uint32_t* dcb, uint32_t size_in_dwords) { EXIT_IF(g_renderer == nullptr); g_renderer->GetGpu().Submit(dcb, size_in_dwords, nullptr, 0, !dcb_has_queued_interrupt(dcb, size_in_dwords)); - Gen5::track_pending_graphics_segment_after_submit(dcb, size_in_dwords); -} - -static std::vector collect_acb_wait_addresses(const uint32_t* acb, - uint32_t size_in_dwords) { - std::vector addresses; - - for (uint32_t offset = 0; offset < size_in_dwords;) { - auto cmd_id = acb[offset]; - auto len = KYTY_PM4_LEN(cmd_id); - if (len == 0 || len > size_in_dwords - offset) { - return addresses; - } - - auto op = (cmd_id >> 8u) & 0xffu; - if (op == Pm4::IT_NOP && KYTY_PM4_R(cmd_id) == Pm4::R_WAIT_MEM_32 && len >= 7) { - auto address = static_cast(acb[offset + 1]) | - (static_cast(acb[offset + 2]) << 32u); - if (address != 0) { - addresses.push_back(address); - } - } else if (op == Pm4::IT_NOP && KYTY_PM4_R(cmd_id) == Pm4::R_WAIT_MEM_64 && len >= 9) { - auto address = static_cast(acb[offset + 1]) | - (static_cast(acb[offset + 2]) << 32u); - if (address != 0) { - addresses.push_back(address); - } - } - - offset += len; - } - - return addresses; -} - -static bool acb_waits_for_address(const std::vector& wait_addresses, - uint64_t release_address) { - for (auto address: wait_addresses) { - if (address == release_address) { - return true; - } - } - - return false; -} - -static void flush_pending_graphics_segment_before_acb(const uint32_t* acb, - uint32_t acb_size_in_dwords) { - uint32_t* dcb = nullptr; - uint32_t size_in_dwords = 0; - auto wait_addresses = collect_acb_wait_addresses(acb, acb_size_in_dwords); - - { - std::lock_guard lock(Gen5::g_pending_graphics_segment_mutex); - if (!wait_addresses.empty() && Gen5::g_pending_graphics_segment.start != nullptr) { - auto* scan = Gen5::g_pending_graphics_segment.start; - auto* matched_end = Gen5::g_pending_graphics_segment.start; - while (scan < Gen5::g_pending_graphics_segment.end) { - auto cmd_id = *scan; - if (cmd_id == 0x80000000u) { - scan++; - continue; - } - if ((cmd_id & 0xC0000000u) != 0xC0000000u) { - break; - } - - auto len = KYTY_PM4_LEN(cmd_id); - if (len == 0 || - len > static_cast(Gen5::g_pending_graphics_segment.end - scan)) { - break; - } - - if (((cmd_id >> 8u) & 0xffu) == Pm4::IT_NOP && - KYTY_PM4_R(cmd_id) == Pm4::R_RELEASE_MEM && len >= 7) { - auto release_addr = - static_cast(scan[3]) | (static_cast(scan[4]) << 32u); - if (acb_waits_for_address(wait_addresses, release_addr)) { - matched_end = scan + len; - } - } - - scan += len; - } - - if (matched_end > Gen5::g_pending_graphics_segment.start) { - Gen5::g_pending_graphics_segment.end = matched_end; - } - } - - if (Gen5::g_pending_graphics_segment.start != nullptr && - Gen5::g_pending_graphics_segment.end > Gen5::g_pending_graphics_segment.start) { - auto* scan = Gen5::g_pending_graphics_segment.start; - auto* valid_end = Gen5::g_pending_graphics_segment.start; - while (scan < Gen5::g_pending_graphics_segment.end) { - auto cmd_id = *scan; - if (cmd_id == 0x80000000u) { - scan++; - valid_end = scan; - continue; - } - if ((cmd_id & 0xC0000000u) != 0xC0000000u) { - break; - } - - auto len = KYTY_PM4_LEN(cmd_id); - if (len == 0 || - len > static_cast(Gen5::g_pending_graphics_segment.end - scan)) { - break; - } - - scan += len; - valid_end = scan; - } - - if (valid_end < Gen5::g_pending_graphics_segment.end) { - static std::atomic log_count {0}; - if (log_count.fetch_add(1) < 64) { - LOGF("\t trimming pending graphics segment: addr = 0x%016" PRIx64 - ", old_dw = 0x%08" PRIx32 ", new_dw = 0x%08" PRIx32 "\n", - reinterpret_cast(Gen5::g_pending_graphics_segment.start), - static_cast(Gen5::g_pending_graphics_segment.end - - Gen5::g_pending_graphics_segment.start), - static_cast(valid_end - Gen5::g_pending_graphics_segment.start)); - } - Gen5::g_pending_graphics_segment.end = valid_end; - } - } - - if (Gen5::g_pending_graphics_segment.start == nullptr || - Gen5::g_pending_graphics_segment.end <= Gen5::g_pending_graphics_segment.start) { - return; - } - - dcb = Gen5::g_pending_graphics_segment.start; - size_in_dwords = static_cast(Gen5::g_pending_graphics_segment.end - - Gen5::g_pending_graphics_segment.start); - } - - LOGF("\t flushing pending graphics segment before ACB: addr = 0x%016" PRIx64 - ", dw_num = 0x%08" PRIx32 "\n", - reinterpret_cast(dcb), size_in_dwords); - - submit_dcb(dcb, size_in_dwords); } int KYTY_SYSV_ABI GraphicsDriverSubmitDcb(const Packet* packet) { @@ -4074,8 +3877,6 @@ static void submit_acb(uint32_t queue, uint32_t* acb, uint32_t size_in_dwords) { LOGF("\t acb[%u] = 0x%08" PRIx32 "\n", i, acb[i]); } - flush_pending_graphics_segment_before_acb(acb, size_in_dwords); - GraphicsDbgDumpDcb("a", size_in_dwords, acb); const bool trigger_interrupt_on_done = !dcb_has_queued_interrupt(acb, size_in_dwords); diff --git a/tests/ShaderRecompilerComputeTests.cpp b/tests/ShaderRecompilerComputeTests.cpp index b9490b6bd..8da259a46 100644 --- a/tests/ShaderRecompilerComputeTests.cpp +++ b/tests/ShaderRecompilerComputeTests.cpp @@ -58,6 +58,7 @@ #include #include #include +#include #include #include #include @@ -1450,22 +1451,91 @@ public: uint32_t ordered_suffix = 0; std::jthread ordered([&] { ordered_started.release(); - Gpu::SubmissionLock submissions(gpu); - gpu.SendCommandSync([&] { - ordered_suffix = suffix; - ordered_finished = true; - }); + gpu.Done(); + ordered_suffix = suffix; + ordered_finished = true; }); ordered_started.acquire(); gpu.SendCommandSync([&] { - Require("GpuCommandLane", "ordered barrier", !ordered_finished.load(), - "ordered host command overtook a queued submission"); + Require("GpuCommandLane", "submit done barrier", !ordered_finished.load(), + "submit done returned before a queued submission"); label = 1; }); ordered.join(); Require("GpuCommandLane", "ordered completion", prefix == 11 && suffix == 22 && ordered_suffix == 22 && ordered_finished.load(), - "submission barrier did not drain prior PM4 work"); + "submit done did not drain prior PM4 work"); + + auto& resources = context.GetGpuResources(); + constexpr uint64_t empty_unmap_base = 0x0000000200400000ull; + constexpr uint64_t empty_unmap_size = 0x4000; + resources.MapMemory(empty_unmap_base, empty_unmap_size); + label = 0; + prefix = 0; + suffix = 0; + gpu.Submit(commands.data(), static_cast(commands.size()), nullptr, 0); + + std::binary_semaphore unmap_complete {0}; + std::jthread unmap_thread([&] { + resources.UnmapMemory(empty_unmap_base, empty_unmap_size); + unmap_complete.release(); + }); + const bool unmap_returned = unmap_complete.try_acquire_for(std::chrono::seconds(2)); + gpu.SendCommandSync([&] { label = 1; }); + if (!unmap_returned) { + unmap_complete.acquire(); + } + unmap_thread.join(); + gpu.Done(); + Require("GpuCommandLane", "unmap queue progress", + unmap_returned && !resources.IsMapped(empty_unmap_base, empty_unmap_size) && + prefix == 11 && suffix == 22, + "an unrelated unmap waited for a blocked PM4 submission"); + + auto& scheduler = context.GetCommandScheduler(); + std::atomic normal_completed {false}; + gpu.SendCommandSync( + [&] { scheduler.DeferOperation([&] { normal_completed = true; }); }); + resources.MapMemory(empty_unmap_base, empty_unmap_size); + resources.UnmapMemory(empty_unmap_base, empty_unmap_size); + Require("GpuCommandLane", "unmap native completion", + normal_completed.load() && + !resources.IsMapped(empty_unmap_base, empty_unmap_size), + "unmap returned before an earlier native guest-memory callback"); + + std::binary_semaphore priority_entered {0}; + std::binary_semaphore release_priority {0}; + gpu.SendCommandSync([&] { + scheduler.DeferPriorityOperation([&] { + priority_entered.release(); + release_priority.acquire(); + }); + scheduler.Flush(); + }); + priority_entered.acquire(); + resources.MapMemory(empty_unmap_base, empty_unmap_size); + + std::binary_semaphore priority_unmap_entered {0}; + std::binary_semaphore priority_unmap_complete {0}; + std::jthread priority_unmap_thread([&] { + gpu.SendCommandSync([&] { + priority_unmap_entered.release(); + resources.UnmapMemory(empty_unmap_base, empty_unmap_size); + }); + priority_unmap_complete.release(); + }); + priority_unmap_entered.acquire(); + const bool unmap_overtook_priority = + priority_unmap_complete.try_acquire_for(std::chrono::seconds(1)); + release_priority.release(); + if (!unmap_overtook_priority) { + priority_unmap_complete.acquire(); + } + priority_unmap_thread.join(); + Require("GpuCommandLane", "unmap priority ordering", + !unmap_overtook_priority && + !resources.IsMapped(empty_unmap_base, empty_unmap_size), + "unmap returned before an earlier guest-memory callback"); constexpr uintptr_t fault_base = 0x0000000200500000ull; constexpr uint64_t fault_size = 0x10000; @@ -1483,7 +1553,6 @@ public: Require("GpuCommandLane", "processor fault allocation", fault_memory == reinterpret_cast(fault_base), "fixed processor-fault allocation failed"); - auto& resources = context.GetGpuResources(); resources.MapMemory(fault_base, fault_size); constexpr uint64_t immediate_dst = fault_base + 0x1000; @@ -1530,9 +1599,7 @@ public: Require("GpuCommandLane", "DMA_DATA packet assembly", dma_cursor == dma_commands.size(), "DMA_DATA GDS packet stream has the wrong size"); gpu.Submit(dma_commands.data(), static_cast(dma_commands.size()), nullptr, 0); - { - Gpu::SubmissionLock submissions(gpu); - } + gpu.Done(); constexpr uint32_t clean_fill_value = 0xdecafbad; gpu.SendCommandSyncWithProcessor([&](CommandProcessor&) { auto& buffer_cache = resources.GetBufferCache();