#ifndef EMULATOR_SRC_GRAPHICS_HOST_GPU_MEMORYTRACKER_H_ #define EMULATOR_SRC_GRAPHICS_HOST_GPU_MEMORYTRACKER_H_ #include "common/assert.h" #include "graphics/host_gpu/pageManager.h" #include "graphics/host_gpu/rangeSet.h" #include "graphics/host_gpu/regionManager.h" #include #include #include #include #include #include #include namespace Libs::Graphics { class MemoryTracker final { public: explicit MemoryTracker(PageManager& page_manager); ~MemoryTracker(); KYTY_CLASS_NO_COPY(MemoryTracker); [[nodiscard]] bool IsRegionCpuModified(uint64_t vaddr, uint64_t size); [[nodiscard]] bool IsRegionGpuModified(uint64_t vaddr, uint64_t size); void MarkRegionAsCpuModified(uint64_t vaddr, uint64_t size); void MarkRegionAsGpuModified(uint64_t vaddr, uint64_t size); void UnmarkRegionAsGpuModified(uint64_t vaddr, uint64_t size); void UntrackMemory(uint64_t vaddr, uint64_t size); // Queue after changing ownership; a concurrent drain must not consume an event // before the dirty pages it describes become visible. void QueueBdaSync(uint64_t vaddr, uint64_t size); [[nodiscard]] RangeSet TakeBdaSync(); // Removes protection from a range and flushes GPU-owned data when required. template void InvalidateRegion(uint64_t vaddr, uint64_t size, Flush&& on_flush) noexcept { static_assert(std::is_invocable_v); CheckNotInUploadCallback(); Iterate(vaddr, size, [&](RegionManager* manager, uint64_t offset, uint64_t bytes) { const bool should_flush = [&] { // Perform both the GPU modification check and CPU state change with the lock in // case the GPU thread is racing to mark the page modified. If a flush is needed, // on_flush performs the CPU state change. std::scoped_lock lock(manager->lock); if (manager->IsModified(offset, bytes)) { return true; } manager->ChangeState(manager->GetCpuAddr() + offset, bytes); return false; }(); if (should_flush) { on_flush(); } }); QueueBdaSync(vaddr, size); } #if KYTY_BUILD == KYTY_BUILD_DEBUG void ValidateGpuDirtyPages(const RangeSet& dirty, uint64_t vaddr, uint64_t size, const char* operation) const noexcept; void ValidateGpuDirtyOwnership(const RangeSet& dirty, uint64_t vaddr, uint64_t size, const char* operation); #else void ValidateGpuDirtyPages(const RangeSet&, uint64_t, uint64_t, const char*) const noexcept {} void ValidateGpuDirtyOwnership(const RangeSet&, uint64_t, uint64_t, const char*) {} #endif template void ForEachDownloadRange(uint64_t vaddr, uint64_t size, Func&& func) { static_assert(std::is_nothrow_invocable_v); CheckNotInUploadCallback(); Iterate(vaddr, size, [&](RegionManager* manager, uint64_t offset, uint64_t bytes) { std::scoped_lock lock(manager->lock); const auto address = manager->GetCpuAddr() + offset; manager->template ForEachModifiedRange(address, bytes, func); if constexpr (clear) { manager->template ChangeState(address, bytes); } }); } template void ForEachUploadRange(uint64_t vaddr, uint64_t size, bool is_written, RangeFunc&& range_func, UploadFunc&& upload_func) { static_assert(std::is_nothrow_invocable_v); static_assert(std::is_nothrow_invocable_v); CheckNotInUploadCallback(); const auto* previous_upload_owner = std::exchange(s_upload_owner, this); Iterate(vaddr, size, [&](RegionManager* manager, uint64_t offset, uint64_t bytes) { manager->lock.lock(); manager->ForEachModifiedRange(manager->GetCpuAddr() + offset, bytes, range_func); if (!is_written) { manager->lock.unlock(); } }); upload_func(); if (is_written) { Iterate(vaddr, size, [](RegionManager* manager, uint64_t offset, uint64_t bytes) { manager->template ChangeState( manager->GetCpuAddr() + offset, bytes); manager->lock.unlock(); }); } s_upload_owner = previous_upload_owner; } private: static constexpr size_t REGION_COUNT = TRACKER_ADDRESS_SIZE / TRACKER_REGION_SIZE; inline static thread_local const MemoryTracker* s_upload_owner = nullptr; void CheckNotInUploadCallback() const noexcept { if (s_upload_owner == this) { EXIT("memory tracker re-entered from upload callback\n"); } } template bool Iterate(uint64_t vaddr, uint64_t size, Func&& func) { ValidateRange(vaddr, size); using Result = std::invoke_result_t; constexpr bool returns_bool = std::is_same_v; uint64_t remaining = size; uint64_t index = vaddr / TRACKER_REGION_SIZE; uint64_t offset = vaddr % TRACKER_REGION_SIZE; while (remaining != 0) { const auto bytes = std::min(TRACKER_REGION_SIZE - offset, remaining); auto* manager = m_regions[index].load(std::memory_order_acquire); if (manager == nullptr && create) { manager = GetOrCreateRegion(index); } if (manager != nullptr) { if constexpr (returns_bool) { if (func(manager, offset, bytes)) { return true; } } else { func(manager, offset, bytes); } } remaining -= bytes; offset = 0; index++; } return false; } static void ValidateRange(uint64_t vaddr, uint64_t size); RegionManager* GetOrCreateRegion(uint64_t index); std::unique_ptr[]> m_regions; std::vector> m_region_storage; std::mutex m_region_mutex; std::mutex m_bda_sync_mutex; RangeSet m_bda_sync; PageManager& m_page_manager; }; } // namespace Libs::Graphics #endif // EMULATOR_SRC_GRAPHICS_HOST_GPU_MEMORYTRACKER_H_