diff --git a/3rdparty/tracy b/3rdparty/tracy index 00c079cec..05cceee0d 160000 --- a/3rdparty/tracy +++ b/3rdparty/tracy @@ -1 +1 @@ -Subproject commit 00c079cec91e6a374519aa5719a073e7f4539c9d +Subproject commit 05cceee0df3b8d7c6fa87e9638af311dbabc63cb diff --git a/src/graphics/guest_gpu/command_processor/commandProcessor.h b/src/graphics/guest_gpu/command_processor/commandProcessor.h index 77e5e3283..899b34e55 100644 --- a/src/graphics/guest_gpu/command_processor/commandProcessor.h +++ b/src/graphics/guest_gpu/command_processor/commandProcessor.h @@ -50,23 +50,10 @@ public: void Reset(); - void BufferInit(); - void BufferFlush(); - void BufferFlushAndWait(); - void BufferWait(); - void BeginReadbackTransaction() { - if (m_readback_active) { - EXIT("nested command-processor readback transaction\n"); - } - m_readback_active = true; - } - void EndReadbackTransaction() { - if (!m_readback_active) { - EXIT("command-processor readback transaction is not active\n"); - } - m_readback_active = false; - } - + void BufferInit(); + void BufferFlush(); + void BufferFlushAndWait(); + void BufferWait(); HW::Context& GetCtx() { return m_ctx; } HW::UserConfig& GetUcfg() { return m_ucfg; } HW::Shader& GetShCtx() { return m_sh_ctx; } @@ -173,10 +160,9 @@ private: // Persistent draw state: indirect draws update it for subsequent draws. uint32_t m_num_instances = 1; - uint32_t m_de_count = 0; - uint32_t m_ce_count = 0; - bool m_ce_complete = false; - bool m_readback_active = false; + uint32_t m_de_count = 0; + uint32_t m_ce_count = 0; + bool m_ce_complete = false; uint32_t m_const_ram[0x3000] = {0}; diff --git a/src/graphics/guest_gpu/graphicsRun.cpp b/src/graphics/guest_gpu/graphicsRun.cpp index 44b49ffe2..ffc9941fc 100644 --- a/src/graphics/guest_gpu/graphicsRun.cpp +++ b/src/graphics/guest_gpu/graphicsRun.cpp @@ -36,6 +36,7 @@ static thread_local CommandProcessor* g_current_processor = nullptr; static thread_local Pm4Execution* g_current_execution = nullptr; static thread_local bool g_gpu_mutex_owned = false; static thread_local bool g_gpu_thread = false; +static thread_local GpuState* g_gpu_state = nullptr; class GpuMutexLock final { public: @@ -98,11 +99,10 @@ public: void SubmitFlipPreparation(uint64_t request_id); void Done(); void Shutdown(); - [[nodiscard]] bool IsStopping(); - void SendCommand(Common::UniqueFunction&& command); - void SendCommandSync(Common::UniqueFunction&& command); - void SendCommandSyncWithProcessor(Common::UniqueFunction&& command); - int GetFrameNum(); + [[nodiscard]] bool IsStopping(); + void SendCommand(Common::UniqueFunction&& command); + void SendCommandSync(Common::UniqueFunction&& command); + int GetFrameNum(); [[nodiscard]] static bool IsGpuThread() noexcept { return g_gpu_thread; } private: @@ -127,6 +127,7 @@ private: void WaitLocked(); void Enqueue(Submission submission); void WaitForIdle(); + void ProcessCommands(); bool Process(Submission& submission); static void ThreadRun(void* data); CommandProcessor& GetProcessor(uint32_t queue_id); @@ -139,6 +140,7 @@ private: Common::CondVar m_idle; std::array, QueueCount> m_queues; std::deque> m_commands; + std::atomic_uint32_t m_pending_commands {0}; uint32_t m_next_queue = 0; uint32_t m_submission_count = 0; bool m_processing = false; @@ -153,6 +155,8 @@ private: uint64_t m_submit_id = 0; std::atomic_int m_done_num = 0; std::jthread m_thread; + + friend class CommandProcessor; }; static bool GraphicsRunDebugDumpEnabled() { @@ -195,9 +199,25 @@ void GpuState::SendCommand(Common::UniqueFunction&& command) { Common::LockGuard lock(m_queue_mutex); EXIT_IF(!m_accepting); m_commands.push_back(std::move(command)); + m_pending_commands.fetch_add(1, std::memory_order_release); m_work_available.Signal(); } +void GpuState::ProcessCommands() { + EXIT_IF(!IsGpuThread()); + while (m_pending_commands.load(std::memory_order_acquire) != 0) { + Common::UniqueFunction command; + { + Common::LockGuard lock(m_queue_mutex); + EXIT_IF(m_commands.empty()); + command = std::move(m_commands.front()); + m_commands.pop_front(); + EXIT_IF(m_pending_commands.fetch_sub(1, std::memory_order_acq_rel) == 0); + } + command(); + } +} + void GpuState::SendCommandSync(Common::UniqueFunction&& command) { EXIT_IF(!command); if (IsGpuThread()) { @@ -212,17 +232,6 @@ void GpuState::SendCommandSync(Common::UniqueFunction&& command) { done.acquire(); } -void GpuState::SendCommandSyncWithProcessor( - Common::UniqueFunction&& command) { - EXIT_IF(!command); - SendCommandSync([this, operation = std::move(command)]() mutable { - EXIT_IF(g_current_processor != nullptr); - g_current_processor = m_gfx_cp.get(); - operation(*m_gfx_cp); - g_current_processor = nullptr; - }); -} - void GpuState::Submit(uint32_t* cmd_draw_buffer, uint32_t num_draw_dw, uint32_t* cmd_const_buffer, uint32_t num_const_dw, bool trigger_agc_interrupt_on_done) { GpuMutexLock lock(m_submission_mutex); @@ -509,6 +518,7 @@ void GpuState::ThreadRun(void* data) { EXIT_IF(gpu == nullptr); KYTY_PROFILER_THREAD("Thread_Gpu"); g_gpu_thread = true; + g_gpu_state = gpu; for (;;) { Submission submission; @@ -529,6 +539,7 @@ void GpuState::ThreadRun(void* data) { } else if (!gpu->m_commands.empty()) { command = std::move(gpu->m_commands.front()); gpu->m_commands.pop_front(); + EXIT_IF(gpu->m_pending_commands.fetch_sub(1, std::memory_order_acq_rel) == 0); gpu->m_processing = true; } else { int selected_queue = -1; @@ -560,6 +571,7 @@ void GpuState::ThreadRun(void* data) { } if (should_stop) { gpu->m_gfx_cp->BufferWait(); + g_gpu_state = nullptr; g_gpu_thread = false; return; } @@ -741,6 +753,9 @@ void CommandProcessor::SuspendPm4() { void CommandProcessor::ProcessPm4(Pm4Execution& execution, size_t stop_depth) { while (execution.m_buffer_stack.size() > stop_depth) { + if (g_gpu_state != nullptr) { + g_gpu_state->ProcessCommands(); + } const auto buffer_index = execution.m_buffer_stack.size() - 1; auto& cursor = execution.m_buffer_stack[buffer_index]; if (cursor.deferred_advance_dw != 0) { @@ -1650,10 +1665,6 @@ void Gpu::SendCommandSync(Common::UniqueFunction&& command) { m_state->SendCommandSync(std::move(command)); } -void Gpu::SendCommandSyncWithProcessor(Common::UniqueFunction&& command) { - m_state->SendCommandSyncWithProcessor(std::move(command)); -} - void Gpu::Submit(uint32_t* draw_commands, uint32_t draw_size_dw, uint32_t* constant_commands, uint32_t constant_size_dw, bool trigger_agc_interrupt_on_done) { EXIT_IF(draw_commands == nullptr || draw_size_dw == 0); @@ -1679,12 +1690,8 @@ int Gpu::GetFrameNum() const { return m_state->GetFrameNum(); } -bool Gpu::IsCommandProcessorThread() noexcept { - return g_current_processor != nullptr; -} - -CommandProcessor* Gpu::CurrentCommandProcessor() noexcept { - return g_current_processor; +bool Gpu::IsGpuThread() noexcept { + return GpuState::IsGpuThread(); } } // namespace Libs::Graphics diff --git a/src/graphics/guest_gpu/graphicsRun.h b/src/graphics/guest_gpu/graphicsRun.h index 39ee55f3a..dc9c2cf69 100644 --- a/src/graphics/guest_gpu/graphicsRun.h +++ b/src/graphics/guest_gpu/graphicsRun.h @@ -9,7 +9,6 @@ namespace Libs::Graphics { -class CommandProcessor; class GpuState; class RenderContext; @@ -23,7 +22,6 @@ public: [[nodiscard]] bool IsStopping(); void SendCommand(Common::UniqueFunction&& command); void SendCommandSync(Common::UniqueFunction&& command); - void SendCommandSyncWithProcessor(Common::UniqueFunction&& command); void Submit(uint32_t* draw_commands, uint32_t draw_size_dw, uint32_t* constant_commands, uint32_t constant_size_dw, bool trigger_agc_interrupt_on_done = false); @@ -33,8 +31,7 @@ public: void Done(); [[nodiscard]] int GetFrameNum() const; - [[nodiscard]] static bool IsCommandProcessorThread() noexcept; - [[nodiscard]] static CommandProcessor* CurrentCommandProcessor() noexcept; + [[nodiscard]] static bool IsGpuThread() noexcept; private: std::unique_ptr m_state; diff --git a/src/graphics/host_gpu/memoryTracker.h b/src/graphics/host_gpu/memoryTracker.h index 6a9b5390d..ff4cff8e1 100644 --- a/src/graphics/host_gpu/memoryTracker.h +++ b/src/graphics/host_gpu/memoryTracker.h @@ -29,45 +29,29 @@ public: void MarkRegionAsGpuModified(uint64_t vaddr, uint64_t size); void UnmarkRegionAsGpuModified(uint64_t vaddr, uint64_t size); void UntrackMemory(uint64_t vaddr, uint64_t size); + // Removes protection from a range and flushes GPU-owned data when required. template - void InvalidateRegion(uint64_t vaddr, uint64_t size, Flush&& on_flush) { + void InvalidateRegion(uint64_t vaddr, uint64_t size, Flush&& on_flush) noexcept { static_assert(std::is_invocable_v); CheckNotInUploadCallback(); ValidateRange(vaddr, size); - const auto update_cpu_state = [this, vaddr, size] { - std::lock_guard access(m_access_mutex); - std::vector managers; - Iterate(vaddr, size, [&](RegionManager* manager, uint64_t, uint64_t) { - managers.push_back(manager); - }); - std::vector> locks; - locks.reserve(managers.size()); - for (auto* manager: managers) { - locks.emplace_back(manager->lock); + Iterate(vaddr, size, [&](RegionManager* manager, uint64_t offset, uint64_t bytes) { + const bool should_flush = [&] { + // Perform both the GPU modification check and CPU state change with the lock in + // case the GPU thread is racing to mark the page modified. If a flush is needed, + // on_flush performs the CPU state change. + std::scoped_lock lock(manager->lock); + if (manager->IsModified(offset, bytes)) { + return true; + } + manager->ChangeState(manager->GetCpuAddr() + offset, bytes); + return false; + }(); + if (should_flush) { + on_flush(); } - const bool gpu_modified = Iterate( - vaddr, size, [](RegionManager* manager, uint64_t offset, uint64_t bytes) { - return manager->IsModified(offset, bytes); - }); - if (gpu_modified) { - return true; - } - Iterate(vaddr, size, - [](RegionManager* manager, uint64_t offset, uint64_t bytes) { - manager->ChangeState( - manager->GetCpuAddr() + offset, bytes); - }); - return false; - }; - - if (!update_cpu_state()) { - return; - } - std::forward(on_flush)(); - if (update_cpu_state()) { - EXIT("memory invalidation retained GPU-owned pages\n"); - } + }); } #if KYTY_BUILD == KYTY_BUILD_DEBUG void ValidateGpuDirtyPages(const RangeSet& dirty, uint64_t vaddr, uint64_t size, diff --git a/src/graphics/host_gpu/renderer/cache/bufferCache.cpp b/src/graphics/host_gpu/renderer/cache/bufferCache.cpp index 0d29be501..b84bc5ffd 100644 --- a/src/graphics/host_gpu/renderer/cache/bufferCache.cpp +++ b/src/graphics/host_gpu/renderer/cache/bufferCache.cpp @@ -3,15 +3,18 @@ #include "common/assert.h" #include "common/logging/log.h" #include "common/profiler.h" +#include "graphics/guest_gpu/graphicsRun.h" #include "graphics/host_gpu/graphicContext.h" #include "graphics/host_gpu/renderer/cache/resourceMutex.h" #include "graphics/host_gpu/renderer/cache/textureCache.h" #include "graphics/host_gpu/renderer/commandScheduler.h" #include "graphics/host_gpu/renderer/render.h" +#include "graphics/host_gpu/renderer/renderContext.h" #include "kernel/memory.h" #include #include +#include #include #include #include @@ -288,10 +291,29 @@ void BufferCache::InvalidateMemory(uint64_t vaddr, uint64_t size) { return; } m_memory_tracker.InvalidateRegion(vaddr, size, - [this, vaddr, size] { ReadMemory(vaddr, size); }); + [this, vaddr, size] { ReadMemory(vaddr, size, true); }); } -void BufferCache::ReadMemory(uint64_t vaddr, uint64_t size) { +void BufferCache::ReadMemory(uint64_t vaddr, uint64_t size, bool is_write) { + if (Gpu::IsGpuThread()) { + ReadMemoryOnGpu(vaddr, size, is_write); + return; + } + if (CommandScheduler::InDeferredOperation()) { + EXIT("unsupported buffer readback from an asynchronous GPU completion, " + "addr=0x%016" PRIx64 " size=0x%016" PRIx64 "\n", + vaddr, size); + } + if (m_resource_mutex.IsOwnedByCurrentThread()) { + EXIT("unsupported buffer readback from a pre-owned resource transaction, " + "addr=0x%016" PRIx64 " size=0x%016" PRIx64 "\n", + vaddr, size); + } + m_scheduler.Context().GetGpu().SendCommandSync( + [this, vaddr, size, is_write] { ReadMemoryOnGpu(vaddr, size, is_write); }); +} + +void BufferCache::ReadMemoryOnGpu(uint64_t vaddr, uint64_t size, bool is_write) { std::vector copies; { FaultSafeCacheLock lock(this, m_mutex); @@ -322,9 +344,16 @@ void BufferCache::ReadMemory(uint64_t vaddr, uint64_t size) { } } }); - } - if (copies.empty()) { - return; + if (copies.empty()) { + if (!is_write) { + return; + } + // A preceding read fault can consume the last GPU-owned copy after this write + // invalidation has already chosen to flush. Complete the CPU ownership handoff even + // though this callback no longer has bytes to download. + m_memory_tracker.MarkRegionAsCpuModified(vaddr, size); + return; + } } auto downloads = RecordDownloads(copies); m_scheduler.FinishCurrent(); @@ -336,6 +365,9 @@ void BufferCache::ReadMemory(uint64_t vaddr, uint64_t size) { } // The enumeration above covered whole dirty pages and every exact interval on them. m_memory_tracker.UnmarkRegionAsGpuModified(vaddr, size); + if (is_write) { + m_memory_tracker.MarkRegionAsCpuModified(vaddr, size); + } } } diff --git a/src/graphics/host_gpu/renderer/cache/bufferCache.h b/src/graphics/host_gpu/renderer/cache/bufferCache.h index 6e7406911..31644e0bd 100644 --- a/src/graphics/host_gpu/renderer/cache/bufferCache.h +++ b/src/graphics/host_gpu/renderer/cache/bufferCache.h @@ -46,7 +46,7 @@ public: KYTY_CLASS_NO_COPY(BufferCache); void InvalidateMemory(uint64_t vaddr, uint64_t size); - void ReadMemory(uint64_t vaddr, uint64_t size); + void ReadMemory(uint64_t vaddr, uint64_t size, bool is_write = false); void UnmapMemory(uint64_t vaddr, uint64_t size); [[nodiscard]] BufferBinding ObtainBuffer(CommandBuffer& command, uint64_t vaddr, uint64_t size, bool is_written = false, bool is_read = true, @@ -97,6 +97,7 @@ private: void PublishDownloads(std::span downloads); void QueueGarbageDownload(std::span copies, RetiredBuffer retire); void WriteHostMemory(uint64_t vaddr, std::span data); + void ReadMemoryOnGpu(uint64_t vaddr, uint64_t size, bool is_write); GraphicContext& m_graphics; CommandScheduler& m_scheduler; diff --git a/src/graphics/host_gpu/renderer/cache/gpuResourceManager.cpp b/src/graphics/host_gpu/renderer/cache/gpuResourceManager.cpp index be000246f..3432a3942 100644 --- a/src/graphics/host_gpu/renderer/cache/gpuResourceManager.cpp +++ b/src/graphics/host_gpu/renderer/cache/gpuResourceManager.cpp @@ -1,7 +1,6 @@ #include "graphics/host_gpu/renderer/cache/gpuResourceManager.h" #include "common/assert.h" -#include "graphics/guest_gpu/command_processor/commandProcessor.h" #include "graphics/guest_gpu/graphicsRun.h" #include "graphics/host_gpu/renderer/commandScheduler.h" namespace Libs::Graphics { @@ -18,69 +17,21 @@ bool GpuResourceManager::HandleFault(PageFaultAccess access, uint64_t fault_vadd if (!IsMapped(fault_vaddr, fault_size)) { return false; } - if (CommandScheduler::InDeferredOperation()) { - EXIT("unsupported guest-memory fault from an asynchronous GPU completion, " - "addr=0x%016" PRIx64 " access=%u\n", - fault_vaddr, static_cast(access)); + if (access == PageFaultAccess::Write) { + m_buffer_cache.InvalidateMemory(fault_vaddr, fault_size); + m_texture_cache.InvalidateMemory(fault_vaddr, fault_size); + } else { + m_buffer_cache.ReadMemory(fault_vaddr, fault_size); } - bool handled = false; - const auto resolve = [this, access, fault_vaddr, &handled](CommandProcessor& cp) { - cp.BeginReadbackTransaction(); - { - ResourceMutex::FaultScope fault(m_resource_mutex); - if (access == PageFaultAccess::Write) { - m_buffer_cache.InvalidateMemory(fault_vaddr, fault_size); - m_texture_cache.InvalidateMemory(fault_vaddr, fault_size); - } else { - m_buffer_cache.ReadMemory(fault_vaddr, fault_size); - } - handled = true; - } - cp.EndReadbackTransaction(); - }; - if (auto* cp = Gpu::CurrentCommandProcessor(); cp != nullptr) { - resolve(*cp); - return handled; - } - if (m_resource_mutex.IsOwnedByCurrentThread()) { - EXIT("unsupported page fault from a pre-owned resource transaction, addr=0x%016" PRIx64 - " access=%u\n", - fault_vaddr, static_cast(access)); - } - EXIT_IF(m_gpu == nullptr); - m_gpu->SendCommandSyncWithProcessor(resolve); - return handled; + return true; } bool GpuResourceManager::InvalidateMemory(uint64_t vaddr, uint64_t size) { if (!IsMapped(vaddr, size)) { return false; } - if (CommandScheduler::InDeferredOperation()) { - EXIT("unsupported memory invalidation from an asynchronous GPU completion, " - "addr=0x%016" PRIx64 " size=0x%016" PRIx64 "\n", - vaddr, size); - } - const auto resolve = [this, vaddr, size](CommandProcessor& cp) { - cp.BeginReadbackTransaction(); - { - ResourceMutex::FaultScope fault(m_resource_mutex); - m_buffer_cache.InvalidateMemory(vaddr, size); - m_texture_cache.InvalidateMemory(vaddr, size); - } - cp.EndReadbackTransaction(); - }; - if (auto* cp = Gpu::CurrentCommandProcessor(); cp != nullptr) { - resolve(*cp); - return true; - } - if (m_resource_mutex.IsOwnedByCurrentThread()) { - EXIT("unsupported memory invalidation from a pre-owned resource transaction, " - "addr=0x%016" PRIx64 " size=0x%016" PRIx64 "\n", - vaddr, size); - } - EXIT_IF(m_gpu == nullptr); - m_gpu->SendCommandSyncWithProcessor(resolve); + m_buffer_cache.InvalidateMemory(vaddr, size); + m_texture_cache.InvalidateMemory(vaddr, size); return true; } diff --git a/src/graphics/host_gpu/renderer/cache/resourceMutex.cpp b/src/graphics/host_gpu/renderer/cache/resourceMutex.cpp index 95dfe2a1c..6d5545c3e 100644 --- a/src/graphics/host_gpu/renderer/cache/resourceMutex.cpp +++ b/src/graphics/host_gpu/renderer/cache/resourceMutex.cpp @@ -4,18 +4,10 @@ namespace Libs::Graphics { -ResourceMutex::FaultScope::FaultScope(ResourceMutex& mutex): m_mutex(mutex) { - m_resource_preowned = m_mutex.BeginFault(); -} - -ResourceMutex::FaultScope::~FaultScope() { - m_mutex.EndFault(m_resource_preowned); -} - ResourceMutex::~ResourceMutex() { std::lock_guard state(m_state); - if (m_resource_owner != std::thread::id {} || m_fault_owner != std::thread::id {}) { - EXIT("ResourceMutex destroyed with an active owner or fault transaction\n"); + if (m_resource_owner != std::thread::id {}) { + EXIT("ResourceMutex destroyed with an active owner\n"); } } @@ -26,9 +18,6 @@ void ResourceMutex::lock() { if (m_resource_owner == current) { EXIT("recursive resource transaction\n"); } - if (m_fault_owner == current) { - EXIT("resource transaction re-entered from a page-fault callback\n"); - } } m_resource.lock(); std::lock_guard state(m_state); @@ -55,39 +44,4 @@ bool ResourceMutex::IsOwnedByCurrentThread() { return m_resource_owner == std::this_thread::get_id(); } -bool ResourceMutex::BeginFault() { - const auto current = std::this_thread::get_id(); - { - std::lock_guard state(m_state); - if (m_fault_owner == current) { - EXIT("nested resource page-fault transaction\n"); - } - if (m_resource_owner == current) { - m_fault_owner = current; - return true; - } - } - lock(); - std::lock_guard state(m_state); - if (m_resource_owner != current || m_fault_owner != std::thread::id {}) { - EXIT("resource fault transaction acquired inconsistent ownership\n"); - } - m_fault_owner = current; - return false; -} - -void ResourceMutex::EndFault(bool resource_preowned) { - const auto current = std::this_thread::get_id(); - { - std::lock_guard state(m_state); - if (m_fault_owner != current || m_resource_owner != current) { - EXIT("resource page-fault transaction ended with inconsistent ownership\n"); - } - m_fault_owner = {}; - } - if (!resource_preowned) { - unlock(); - } -} - } // namespace Libs::Graphics diff --git a/src/graphics/host_gpu/renderer/cache/resourceMutex.h b/src/graphics/host_gpu/renderer/cache/resourceMutex.h index d27b9680e..b9aaf18d2 100644 --- a/src/graphics/host_gpu/renderer/cache/resourceMutex.h +++ b/src/graphics/host_gpu/renderer/cache/resourceMutex.h @@ -8,21 +8,9 @@ namespace Libs::Graphics { -// Owner-tracked shared buffer/image transaction. External faults pause GPU submissions first; -// command-processor faults drain pending guest processors before entering this transaction. +// Owner-tracked shared buffer/image transaction. class ResourceMutex final { public: - class FaultScope final { - public: - explicit FaultScope(ResourceMutex& mutex); - ~FaultScope(); - KYTY_CLASS_NO_COPY(FaultScope); - - private: - ResourceMutex& m_mutex; - bool m_resource_preowned = false; - }; - ResourceMutex() = default; ~ResourceMutex(); KYTY_CLASS_NO_COPY(ResourceMutex); @@ -32,15 +20,9 @@ public: [[nodiscard]] bool IsOwnedByCurrentThread(); private: - friend class FaultScope; - - [[nodiscard]] bool BeginFault(); - void EndFault(bool resource_preowned); - std::mutex m_resource; std::mutex m_state; std::thread::id m_resource_owner; - std::thread::id m_fault_owner; }; } // namespace Libs::Graphics diff --git a/tests/MemoryTrackerTests.cpp b/tests/MemoryTrackerTests.cpp index 0ce664537..0b92c017f 100644 --- a/tests/MemoryTrackerTests.cpp +++ b/tests/MemoryTrackerTests.cpp @@ -6,7 +6,9 @@ #include #include #include +#include #include +#include #include #if KYTY_PLATFORM == KYTY_PLATFORM_WINDOWS @@ -265,6 +267,7 @@ void TestRangeInvalidation() { flushes++; tracker.ForEachDownloadRange(address + 16, size - 32, [](uint64_t, uint64_t) noexcept {}); + tracker.MarkRegionAsCpuModified(address + 16, size - 32); }); Check(flushes == 1 && !tracker.IsRegionGpuModified(address, size) && tracker.IsRegionCpuModified(address, size) && IsWritable(memory) && @@ -277,6 +280,53 @@ void TestRangeInvalidation() { Release(page_manager, memory, size); } +void TestGpuReacquisitionAfterInvalidation() { + TrackerHarness harness; + auto &tracker = harness.tracker; + auto &page_manager = harness.page_manager; + const auto page_size = page_manager.GetPageSize(); + auto *memory = Allocate(page_manager, 1); + const auto address = reinterpret_cast(memory); + + tracker.ForEachUploadRange( + address, page_size, true, [](uint64_t, uint64_t) noexcept {}, + []() noexcept {}); + Check(tracker.IsRegionGpuModified(address, page_size) && + !tracker.IsRegionCpuModified(address, page_size), + "reacquisition setup did not establish GPU ownership"); + + uint32_t flushes = 0; + uint32_t uploads = 0; + std::binary_semaphore reacquire{0}; + std::binary_semaphore reacquired{0}; + std::jthread publisher([&] { + reacquire.acquire(); + tracker.ForEachUploadRange( + address + 16, 32, true, + [&](uint64_t, uint64_t) noexcept { uploads++; }, []() noexcept {}); + reacquired.release(); + }); + tracker.InvalidateRegion(address + 16, 32, [&] { + flushes++; + tracker.ForEachDownloadRange(address + 16, 32, + [](uint64_t, uint64_t) noexcept {}); + tracker.MarkRegionAsCpuModified(address + 16, 32); + reacquire.release(); + reacquired.acquire(); + }); + publisher.join(); + Check(flushes == 1 && uploads == 1 && + tracker.IsRegionGpuModified(address, page_size) && + !tracker.IsRegionCpuModified(address, page_size) && + !IsWritable(memory), + "invalidation rejected a new generation of GPU ownership"); + + tracker.UnmarkRegionAsGpuModified(address, page_size); + tracker.MarkRegionAsCpuModified(address, page_size); + tracker.UntrackMemory(address, page_size); + Release(page_manager, memory, page_size); +} + void TestGpuDirtyBits() { TrackerHarness harness; auto &tracker = harness.tracker; @@ -635,6 +685,7 @@ int main(int argc, char **argv) { TestQueriesDoNotRequireMappedOwnership(); TestCpuDirtyUpload(); TestRangeInvalidation(); + TestGpuReacquisitionAfterInvalidation(); TestGpuDirtyBits(); TestExactDirtyIntervalsSharingTrackerPage(); TestGpuDownloadProtectionMirrors(); diff --git a/tests/ResourceMutexTests.cpp b/tests/ResourceMutexTests.cpp index a6937fb7d..fbde8b8a9 100644 --- a/tests/ResourceMutexTests.cpp +++ b/tests/ResourceMutexTests.cpp @@ -35,171 +35,50 @@ void YieldMany() { } } -void TestFaultBlocksPublisher() { - ResourceMutex mutex; - std::atomic_bool publisher_started {false}; - std::atomic_bool publisher_entered {false}; - std::thread publisher; +void TestOwnership() { + ResourceMutex mutex; + Check(!mutex.IsOwnedByCurrentThread(), "fresh mutex reported an owner"); { - ResourceMutex::FaultScope fault(mutex); - publisher = std::thread([&] { - publisher_started.store(true, std::memory_order_release); - std::lock_guard lock(mutex); - publisher_entered.store(true, std::memory_order_release); - }); - while (!publisher_started.load(std::memory_order_acquire)) { - std::this_thread::yield(); - } - YieldMany(); - Check(!publisher_entered.load(std::memory_order_acquire), - "publisher entered during active fault transaction"); + std::lock_guard lock(mutex); + Check(mutex.IsOwnedByCurrentThread(), "owner tracking was not installed"); } - publisher.join(); - Check(publisher_entered.load(std::memory_order_acquire), - "publisher did not resume after fault transaction"); + Check(!mutex.IsOwnedByCurrentThread(), "owner tracking survived unlock"); } -void TestFaultDrainsExistingOwner() { +void TestSerializesTransactions() { ResourceMutex mutex; std::unique_lock owner(mutex); - std::atomic_bool fault_started {false}; - std::atomic_bool fault_entered {false}; - std::atomic_bool release_fault {false}; - std::atomic_bool publisher_started {false}; - std::atomic_bool publisher_entered {false}; - std::thread fault([&] { - fault_started.store(true, std::memory_order_release); - ResourceMutex::FaultScope scope(mutex); - fault_entered.store(true, std::memory_order_release); - while (!release_fault.load(std::memory_order_acquire)) { - std::this_thread::yield(); - } - }); - while (!fault_started.load(std::memory_order_acquire)) { - std::this_thread::yield(); - } - YieldMany(); - Check(!fault_entered.load(std::memory_order_acquire), - "fault transaction did not wait for existing owner"); - owner.unlock(); - while (!fault_entered.load(std::memory_order_acquire)) { - std::this_thread::yield(); - } - std::thread publisher([&] { - publisher_started.store(true, std::memory_order_release); + std::atomic_bool contender_started {false}; + std::atomic_bool contender_entered {false}; + std::thread contender([&] { + contender_started.store(true, std::memory_order_release); std::lock_guard lock(mutex); - publisher_entered.store(true, std::memory_order_release); + contender_entered.store(true, std::memory_order_release); }); - while (!publisher_started.load(std::memory_order_acquire)) { + while (!contender_started.load(std::memory_order_acquire)) { std::this_thread::yield(); } YieldMany(); - Check(!publisher_entered.load(std::memory_order_acquire), - "publisher entered before drained fault completed"); - release_fault.store(true, std::memory_order_release); - fault.join(); - publisher.join(); - Check(publisher_entered.load(std::memory_order_acquire), - "publisher did not resume after drained fault"); -} - -void TestFaultScopesSerialize() { - ResourceMutex mutex; - std::atomic_bool first_entered {false}; - std::atomic_bool second_started {false}; - std::atomic_bool second_entered {false}; - std::atomic_bool release_first {false}; - std::atomic_bool release_second {false}; - std::atomic_bool publisher_entered {false}; - std::thread first([&] { - ResourceMutex::FaultScope scope(mutex); - first_entered.store(true, std::memory_order_release); - while (!release_first.load(std::memory_order_acquire)) { - std::this_thread::yield(); - } - }); - while (!first_entered.load(std::memory_order_acquire)) { - std::this_thread::yield(); - } - std::thread second([&] { - second_started.store(true, std::memory_order_release); - ResourceMutex::FaultScope scope(mutex); - second_entered.store(true, std::memory_order_release); - while (!release_second.load(std::memory_order_acquire)) { - std::this_thread::yield(); - } - }); - while (!second_started.load(std::memory_order_acquire)) { - std::this_thread::yield(); - } - YieldMany(); - Check(!second_entered.load(std::memory_order_acquire), - "second fault transaction entered before first completed"); - release_first.store(true, std::memory_order_release); - while (!second_entered.load(std::memory_order_acquire)) { - std::this_thread::yield(); - } - std::thread publisher([&] { - std::lock_guard lock(mutex); - publisher_entered.store(true, std::memory_order_release); - }); - YieldMany(); - Check(!publisher_entered.load(std::memory_order_acquire), - "publisher entered during second fault transaction"); - release_second.store(true, std::memory_order_release); - first.join(); - second.join(); - publisher.join(); - Check(publisher_entered.load(std::memory_order_acquire), - "publisher did not resume after serialized faults"); -} - -void TestPreownedFaultKeepsResourceTransaction() { - ResourceMutex mutex; - std::atomic_bool publisher_started {false}; - std::atomic_bool publisher_entered {false}; - std::thread publisher; - std::unique_lock owner(mutex); - { - ResourceMutex::FaultScope fault(mutex); - publisher = std::thread([&] { - publisher_started.store(true, std::memory_order_release); - std::lock_guard lock(mutex); - publisher_entered.store(true, std::memory_order_release); - }); - while (!publisher_started.load(std::memory_order_acquire)) { - std::this_thread::yield(); - } - YieldMany(); - Check(!publisher_entered.load(std::memory_order_acquire), - "publisher entered during preowned fault transaction"); - } - YieldMany(); - Check(!publisher_entered.load(std::memory_order_acquire), - "preowned fault released its outer resource transaction"); + Check(!contender_entered.load(std::memory_order_acquire), + "contender entered an active resource transaction"); owner.unlock(); - publisher.join(); - Check(publisher_entered.load(std::memory_order_acquire), - "publisher did not resume after outer transaction"); + contender.join(); + Check(contender_entered.load(std::memory_order_acquire), + "contender did not resume after resource transaction"); } #if KYTY_PLATFORM == KYTY_PLATFORM_WINDOWS -[[noreturn]] void RunDeathCase(const char* name) { +[[noreturn]] void RunDeathCase() { ResourceMutex mutex; - if (std::strcmp(name, "recursive-lock") == 0) { - std::lock_guard first(mutex); - std::lock_guard second(mutex); - } else if (std::strcmp(name, "nested-fault") == 0) { - ResourceMutex::FaultScope first(mutex); - ResourceMutex::FaultScope second(mutex); - } + std::lock_guard first(mutex); + std::lock_guard second(mutex); std::_Exit(0x7f); } -void CheckDeathCase(const char* name) { +void CheckDeathCase() { char path[MAX_PATH] {}; Check(GetModuleFileNameA(nullptr, path, MAX_PATH) != 0, "GetModuleFileName failed"); - std::string command = std::string("\"") + path + "\" --death " + name; + std::string command = std::string("\"") + path + "\" --death"; std::vector mutable_command(command.begin(), command.end()); mutable_command.push_back('\0'); STARTUPINFOA startup {sizeof(startup)}; @@ -224,20 +103,17 @@ void CheckDeathCase(const char* name) { int main(int argc, char** argv) { #if KYTY_PLATFORM == KYTY_PLATFORM_WINDOWS - if (argc == 3 && std::strcmp(argv[1], "--death") == 0) { - RunDeathCase(argv[2]); + if (argc == 2 && std::strcmp(argv[1], "--death") == 0) { + RunDeathCase(); } #else (void)argc; (void)argv; #endif - TestFaultBlocksPublisher(); - TestFaultDrainsExistingOwner(); - TestFaultScopesSerialize(); - TestPreownedFaultKeepsResourceTransaction(); + TestOwnership(); + TestSerializesTransactions(); #if KYTY_PLATFORM == KYTY_PLATFORM_WINDOWS - CheckDeathCase("recursive-lock"); - CheckDeathCase("nested-fault"); + CheckDeathCase(); #endif std::puts("ResourceMutexTests: all cases passed"); return 0; diff --git a/tests/ShaderRecompilerComputeTests.cpp b/tests/ShaderRecompilerComputeTests.cpp index dbc334434..bf34b51b8 100644 --- a/tests/ShaderRecompilerComputeTests.cpp +++ b/tests/ShaderRecompilerComputeTests.cpp @@ -58,6 +58,7 @@ #include #include +#include #include #include #include @@ -1372,12 +1373,14 @@ public: void CheckGpuMappedRangeLifecycle() { EnsureRuntimeContext(); - CommandScheduler scheduler(Renderer(), m_runtime_context); + auto& context = Renderer(); + CommandScheduler scheduler(context, m_runtime_context); HW::Context registers {}; HW::UserConfig user_config {}; HW::Shader shaders {}; scheduler.Begin(registers, user_config, shaders); - Gpu gpu(Renderer()); + context.InitializeGpu(nullptr); + auto& gpu = context.GetGpu(); GpuResourceManager resources(m_runtime_context, scheduler); resources.SetGpu(&gpu); @@ -1416,6 +1419,7 @@ public: resources.SetGpu(nullptr); scheduler.Finish(); + context.ShutdownGpu(); std::printf("[host] %-32s ok\n", "GpuMappedRangeLifecycle"); } @@ -1567,21 +1571,73 @@ public: gpu_thread = std::this_thread::get_id(); Require("GpuCommandLane", "FIFO", order == 1, "synchronous command overtook an older host command"); - Require("GpuCommandLane", "generic context", - !Gpu::IsCommandProcessorThread() && Gpu::CurrentCommandProcessor() == nullptr, - "generic host command manufactured a PM4 processor context"); + Require("GpuCommandLane", "GPU context", Gpu::IsGpuThread(), + "host command did not run on the GPU thread"); gpu.SendCommandSync([&nested_thread] { nested_thread = std::this_thread::get_id(); }); order = 2; }); - gpu.SendCommandSyncWithProcessor([&](CommandProcessor& processor) { - Require("GpuCommandLane", "processor context", - Gpu::IsCommandProcessorThread() && Gpu::CurrentCommandProcessor() == &processor, - "resource command did not receive its explicit processor context"); - }); Require("GpuCommandLane", "dispatch thread", order == 2 && gpu_thread != caller_thread && nested_thread == gpu_thread, "host command did not run on the GPU thread or nested sync deadlocked"); + alignas(uint32_t) uint32_t packet_marker_a = 0; + alignas(uint32_t) uint32_t packet_marker_b = 0; + const auto write_packet = [](uint32_t* packet, uint32_t* destination, uint32_t value) { + const auto destination_address = reinterpret_cast(destination); + packet[0] = KYTY_PM4(5, Pm4::IT_WRITE_DATA, 0); + packet[1] = 0; + packet[2] = static_cast(destination_address); + packet[3] = static_cast(destination_address >> 32u); + packet[4] = value; + }; + std::array polling_loop {}; + for (size_t i = 0; i < polling_loop.size() - 4; i += 2) { + polling_loop[i] = KYTY_PM4(2, Pm4::IT_NOP, Pm4::R_ZERO); + polling_loop[i + 1] = 0; + } + const auto loop_address = reinterpret_cast(polling_loop.data()); + polling_loop[1020] = KYTY_PM4(4, Pm4::IT_INDIRECT_BUFFER, 0); + polling_loop[1021] = static_cast(loop_address); + polling_loop[1022] = static_cast(loop_address >> 32u); + polling_loop[1023] = 0x0f200000u | static_cast(polling_loop.size()); + + std::array polling_commands {}; + write_packet(polling_commands.data(), &packet_marker_a, 11); + polling_commands[5] = KYTY_PM4(4, Pm4::IT_INDIRECT_BUFFER, 0); + polling_commands[6] = static_cast(loop_address); + polling_commands[7] = static_cast(loop_address >> 32u); + polling_commands[8] = 0x0f200000u | static_cast(polling_loop.size()); + write_packet(polling_commands.data() + 9, &packet_marker_b, 22); + std::binary_semaphore stream_gate_entered {0}; + std::binary_semaphore stream_gate_release {0}; + gpu.SendCommand([&] { + stream_gate_entered.release(); + stream_gate_release.acquire(); + }); + gpu.Submit(polling_commands.data(), polling_commands.size(), nullptr, 0); + stream_gate_entered.acquire(); + stream_gate_release.release(); + uint32_t packet_marker_a_at_callback = UINT32_MAX; + uint32_t packet_marker_b_at_callback = UINT32_MAX; + for (uint32_t attempt = 0; attempt < 8 && packet_marker_a_at_callback != 11; attempt++) { + gpu.SendCommandSync([&] { + if (packet_marker_a == 11 && packet_marker_b == 0) { + packet_marker_a_at_callback = packet_marker_a; + packet_marker_b_at_callback = packet_marker_b; + polling_loop[1020] = KYTY_PM4(2, Pm4::IT_NOP, Pm4::R_ZERO); + polling_loop[1021] = 0; + polling_loop[1022] = KYTY_PM4(2, Pm4::IT_NOP, Pm4::R_ZERO); + polling_loop[1023] = 0; + } + }); + std::this_thread::yield(); + } + gpu.Done(); + Require("GpuCommandLane", "packet-boundary command polling", + packet_marker_a_at_callback == 11 && packet_marker_b_at_callback == 0 && + packet_marker_a == 11 && packet_marker_b == 22, + "host command waited for an entire non-suspended PM4 stream"); + uint32_t label = 0; uint32_t prefix = 0; uint32_t suffix = 0; @@ -1760,7 +1816,7 @@ public: gpu.Submit(dma_commands.data(), static_cast(dma_commands.size()), nullptr, 0); gpu.Done(); constexpr uint32_t clean_fill_value = 0xdecafbad; - gpu.SendCommandSyncWithProcessor([&](CommandProcessor&) { + gpu.SendCommandSync([&] { auto& buffer_cache = resources.GetBufferCache(); Require("GpuCommandLane", "clean cached setup", buffer_cache.HasPageOverlap(clean_cached_fill, sizeof(source_words)) && @@ -1840,14 +1896,116 @@ public: clean_copy_words[1]}, "host-memory fill/copy did not update the clean cached mirror"); - gpu.SendCommandSyncWithProcessor([&](CommandProcessor& processor) { - Require("GpuCommandLane", "processor fault context", - Gpu::CurrentCommandProcessor() == &processor, - "processor resource test lost its command context"); - Require("GpuCommandLane", "processor memory invalidation", + gpu.SendCommandSync([&] { + Require("GpuCommandLane", "GPU memory invalidation", resources.InvalidateMemory(fault_base, sizeof(uint32_t)), - "processor memory invalidation did not find its mapped range"); + "GPU memory invalidation did not find its mapped range"); }); + + auto& buffer_cache = resources.GetBufferCache(); + Require("GpuCommandLane", "clean tracked fault setup", + buffer_cache.HasPageOverlap(immediate_dst, sizeof(uint32_t)) && + !buffer_cache.HasGpuDirtyBytes(immediate_dst, sizeof(uint32_t)), + "clean write-fault test address is not backed by a clean cached Buffer"); + constexpr uint64_t dirty_fault_address = fault_base + 0xd000; + constexpr uint32_t dirty_fault_value = 0x5a17c0deu; + constexpr uint32_t dirty_fault_stale = 0x0ddba11u; + std::memcpy(reinterpret_cast(dirty_fault_address), &dirty_fault_stale, + sizeof(dirty_fault_stale)); + gpu.SendCommandSync([&] { + auto dirty = buffer_cache.ObtainBuffer(scheduler.Current(), dirty_fault_address, + sizeof(dirty_fault_value), true, false); + Require("GpuCommandLane", "dirty tracked fault setup", dirty.owner != nullptr, + "dirty write-fault test could not create a cached Buffer"); + scheduler.Current().RetainResourceUntilFence(dirty.owner); + buffer_cache.FillBuffer(dirty_fault_address, sizeof(dirty_fault_value), + dirty_fault_value); + }); + Require("GpuCommandLane", "dirty tracked fault ownership", + buffer_cache.HasGpuDirtyBytes(dirty_fault_address, sizeof(dirty_fault_value)), + "dirty write-fault test address did not acquire GPU ownership"); + + std::binary_semaphore block_entered {0}; + std::binary_semaphore block_release {0}; + std::binary_semaphore clean_fault_complete {0}; + bool clean_fault_handled = false; + gpu.SendCommand([&] { + block_entered.release(); + block_release.acquire(); + }); + block_entered.acquire(); + std::jthread clean_fault_thread([&] { + clean_fault_handled = resources.HandleFault(PageFaultAccess::Write, immediate_dst); + clean_fault_complete.release(); + }); + const bool clean_fault_was_direct = + clean_fault_complete.try_acquire_for(std::chrono::seconds(1)); + if (!clean_fault_was_direct) { + block_release.release(); + clean_fault_complete.acquire(); + } + clean_fault_thread.join(); + Require("GpuCommandLane", "clean fault direct path", + clean_fault_was_direct && clean_fault_handled, + "a clean CPU write fault waited for the blocked GPU command lane"); + + std::binary_semaphore dirty_fault_complete {0}; + bool dirty_fault_handled = false; + std::jthread dirty_fault_thread([&] { + dirty_fault_handled = + resources.HandleFault(PageFaultAccess::Write, dirty_fault_address); + dirty_fault_complete.release(); + }); + const bool dirty_fault_bypassed_lane = + dirty_fault_complete.try_acquire_for(std::chrono::milliseconds(200)); + block_release.release(); + if (!dirty_fault_bypassed_lane) { + dirty_fault_complete.acquire(); + } + dirty_fault_thread.join(); + uint32_t dirty_fault_backing = 0; + std::memcpy(&dirty_fault_backing, reinterpret_cast(dirty_fault_address), + sizeof(dirty_fault_backing)); + Require("GpuCommandLane", "dirty fault synchronization", + !dirty_fault_bypassed_lane && dirty_fault_handled && + dirty_fault_backing == dirty_fault_value && + !buffer_cache.HasGpuDirtyBytes(dirty_fault_address, sizeof(dirty_fault_value)), + "a GPU-dirty write fault bypassed synchronization or lost the published bytes"); + + constexpr uint64_t empty_copy_fault_address = fault_base + 0xe000; + constexpr uint32_t empty_copy_fault_value = 0x6b28d1efu; + gpu.SendCommandSync([&] { + auto dirty = buffer_cache.ObtainBuffer(scheduler.Current(), empty_copy_fault_address, + sizeof(empty_copy_fault_value), true, false); + Require("GpuCommandLane", "empty-copy write setup", dirty.owner != nullptr, + "empty-copy write test could not create a cached Buffer"); + scheduler.Current().RetainResourceUntilFence(dirty.owner); + buffer_cache.FillBuffer(empty_copy_fault_address, sizeof(empty_copy_fault_value), + empty_copy_fault_value); + buffer_cache.ReadMemory(empty_copy_fault_address, sizeof(empty_copy_fault_value)); + const bool gpu_owned = buffer_cache.IsRegionGpuModified( + empty_copy_fault_address, sizeof(empty_copy_fault_value)); + const bool cpu_owned = buffer_cache.IsRegionCpuModified( + empty_copy_fault_address, sizeof(empty_copy_fault_value)); + Require("GpuCommandLane", "empty-copy read ownership", + !gpu_owned && !cpu_owned, + "readback did not leave clean ownership for the queued write invalidation"); + buffer_cache.ReadMemory(empty_copy_fault_address, sizeof(empty_copy_fault_value), true); + }); + uint32_t empty_copy_fault_backing = 0; + std::memcpy(&empty_copy_fault_backing, reinterpret_cast(empty_copy_fault_address), + sizeof(empty_copy_fault_backing)); + const bool empty_copy_cpu_owned = buffer_cache.IsRegionCpuModified( + empty_copy_fault_address, sizeof(empty_copy_fault_value)); + const bool empty_copy_gpu_owned = buffer_cache.IsRegionGpuModified( + empty_copy_fault_address, sizeof(empty_copy_fault_value)); + const bool empty_copy_dirty = + buffer_cache.HasGpuDirtyBytes(empty_copy_fault_address, sizeof(empty_copy_fault_value)); + Require("GpuCommandLane", "empty-copy write ownership", + empty_copy_fault_backing == empty_copy_fault_value && + empty_copy_cpu_owned && !empty_copy_gpu_owned && !empty_copy_dirty, + "write invalidation without a remaining copy did not establish CPU ownership"); + resources.UnmapMemory(fault_base, fault_size); Require("GpuCommandLane", "processor fault unmap", Libs::LibKernel::Memory::KernelMunmap(fault_base, fault_size) == 0, @@ -2073,12 +2231,14 @@ public: constexpr uint32_t ring_fault_second_value = 0x4e5f6071u; EnsureRuntimeContext(); - CommandScheduler scheduler(Renderer(), m_runtime_context); + auto& context = Renderer(); + CommandScheduler scheduler(context, m_runtime_context); HW::Context registers {}; HW::UserConfig user_config {}; HW::Shader shaders {}; scheduler.Begin(registers, user_config, shaders); - Gpu gpu(Renderer()); + context.InitializeGpu(nullptr); + auto& gpu = context.GetGpu(); int64_t direct_offset = -1; Require(name, "direct allocation", @@ -2471,7 +2631,7 @@ public: resources.UnmapMemory(base, allocation_size); scheduler.Finish(); } - gpu.Shutdown(); + context.ShutdownGpu(); Require(name, "unmap direct backing", Libs::LibKernel::Memory::KernelMunmap(base, allocation_size) == 0, "dirty-GC direct-memory mapping release failed"); @@ -2488,12 +2648,14 @@ public: constexpr uint64_t allocation_size = 0x2800000; constexpr uint64_t allocation_alignment = 0x200000; EnsureRuntimeContext(); - CommandScheduler scheduler(Renderer(), m_runtime_context); + auto& context = Renderer(); + CommandScheduler scheduler(context, m_runtime_context); HW::Context registers {}; HW::UserConfig user_config {}; HW::Shader shaders {}; scheduler.Begin(registers, user_config, shaders); - Gpu gpu(Renderer()); + context.InitializeGpu(nullptr); + auto& gpu = context.GetGpu(); int64_t direct_offset = -1; Require(name, "direct allocation", Libs::LibKernel::Memory::KernelAllocateDirectMemory( @@ -4949,7 +5111,7 @@ public: resources.UnmapMemory(base, allocation_size); scheduler.Finish(); } - gpu.Shutdown(); + context.ShutdownGpu(); Require(name, "unmap direct backing", Libs::LibKernel::Memory::KernelMunmap(base, allocation_size) == 0, "cache direct-memory mapping release failed");