diff --git a/src/graphics/host_gpu/renderer/pipeline/imageHeap.cpp b/src/graphics/host_gpu/renderer/pipeline/imageHeap.cpp index dc8a65c9f..b9b1242eb 100644 --- a/src/graphics/host_gpu/renderer/pipeline/imageHeap.cpp +++ b/src/graphics/host_gpu/renderer/pipeline/imageHeap.cpp @@ -20,7 +20,7 @@ namespace { namespace IR = ShaderRecompiler::IR; -// Smaller tables are cheap to materialize on each draw and are often transient. +// Smaller tables are often written for one draw, so they enter on a second draw of a frame. constexpr uint64_t MinTableBytes = 64ull * 1024ull; constexpr uint32_t ElementMask = (1u << IR::ImageHeapKindShift) - 1u; @@ -133,12 +133,16 @@ ImageHeap::~ImageHeap() { bool ImageHeap::Acquire(const IR::ImageHeapRequest& request, IR::ImageHeapView& result) { KYTY_PROFILER_FUNCTION(); const auto size = request.size & ~uint64_t {3}; - if (size < MinTableBytes || request.count == 0u || request.step == 0u || + if (request.count == 0u || request.step == 0u || ((request.base | request.first | request.step) & 3u) != 0u) { return false; } auto memory = std::ranges::find_if( m_memories, [&](const auto& candidate) { return candidate->base == request.base; }); + if (memory == m_memories.end() && size < MinTableBytes && + m_sightings.try_emplace(request.base, m_draw).first->second == m_draw) { + return false; + } // A table follows its descriptor's size, as the first program of a draw acquires it. if (memory == m_memories.end() || ((*memory)->size != size && (*memory)->draw != m_draw)) { if (!m_context.IsMapped(request.base, size)) return false; @@ -593,26 +597,42 @@ void ImageHeap::UnmapMemory(uint64_t address, uint64_t size) { } else { m_unmapped.push_back({address, size}); } - auto& scheduler = m_context.GetCommandScheduler(); std::scoped_lock lock {m_mutex}; std::erase_if(m_memories, [&](const std::unique_ptr& memory) { if (address >= memory->base + memory->size || memory->base >= address + size) { return false; } - Unwatch(*memory); - for (auto& view: memory->views) { - std::erase(m_active, view.get()); - for (const auto& [value, index]: view->elements) { - Unreference(m_elements[index]); - view->released.push_back(index); - } - Retire(*view); - if (scheduler.Active()) { - scheduler.DeferOperation([buffer = std::move(view->buffer)] {}); - } - } + Drop(*memory); return true; }); } +void ImageHeap::RunGarbageCollector() { + // A smaller table rebuilt each frame moves to a new base. Larger ones keep their dead words. + std::scoped_lock lock {m_mutex}; + std::erase_if(m_memories, [&](const std::unique_ptr& memory) { + if (memory->size >= MinTableBytes || memory->draw > m_suspend_draw) return false; + Drop(*memory); + return true; + }); + m_suspend_draw = m_draw; + m_sightings.clear(); +} + +void ImageHeap::Drop(Memory& memory) { + auto& scheduler = m_context.GetCommandScheduler(); + Unwatch(memory); + for (auto& view: memory.views) { + std::erase(m_active, view.get()); + for (const auto& [value, index]: view->elements) { + Unreference(m_elements[index]); + view->released.push_back(index); + } + Retire(*view); + if (scheduler.Active()) { + scheduler.DeferOperation([buffer = std::move(view->buffer)] {}); + } + } +} + } // namespace Libs::Graphics diff --git a/src/graphics/host_gpu/renderer/pipeline/imageHeap.h b/src/graphics/host_gpu/renderer/pipeline/imageHeap.h index 44720178b..c3fb08bf1 100644 --- a/src/graphics/host_gpu/renderer/pipeline/imageHeap.h +++ b/src/graphics/host_gpu/renderer/pipeline/imageHeap.h @@ -26,7 +26,7 @@ struct ImageHeapTestAccess; // of a table owns device slot words, one per table dword, that select an array element. CPU // writes to a table fault in its snapshot, which each draw refreshes once; a view takes the // changes when a draw first acquires it. Changes to the images behind elements bump -// Image::heap_epoch. +// Image::heap_epoch. A smaller table leaves at a suspend point if the frame did not acquire it. class ImageHeap final: public ShaderRecompiler::IR::ImageHeapSource { public: ImageHeap(RenderContext& context, PageManager& page_manager); @@ -47,6 +47,8 @@ public: // CPU write fault, on any thread. void InvalidateMemory(uint64_t address, uint64_t size); void UnmapMemory(uint64_t address, uint64_t size); + // At a suspend point. + void RunGarbageCollector(); private: using DescriptorValue = ShaderRecompiler::IR::DescriptorValue; @@ -107,6 +109,7 @@ private: template void Watch(const Memory& memory, size_t begin, size_t end); void Unwatch(const Memory& memory); + void Drop(Memory& memory); void UpdateProbes(View& view, uint64_t begin, uint64_t end); void UpdateProbe(View& view, uint64_t probe); void Assign(View& view, size_t slot, uint32_t entry); @@ -125,9 +128,11 @@ private: std::mutex m_mutex; // Fault-visible Memory state and m_memories. std::vector> m_memories; std::vector m_active; // Acquired since the last commit. - uint64_t m_draw = 1; + uint64_t m_draw = 1; + uint64_t m_suspend_draw = 0; // m_draw at the last suspend point. std::vector m_elements; std::vector m_free; + std::unordered_map m_sightings; // Smaller tables: base, first draw. std::deque> m_retired; // Tick, element. std::vector> m_runs; std::vector m_scratch; diff --git a/src/graphics/host_gpu/renderer/renderContext.cpp b/src/graphics/host_gpu/renderer/renderContext.cpp index 83db56606..f08ce983d 100644 --- a/src/graphics/host_gpu/renderer/renderContext.cpp +++ b/src/graphics/host_gpu/renderer/renderContext.cpp @@ -147,6 +147,7 @@ void RenderContext::RunGarbageCollector() { m_fault_process_pending = false; m_buffer_cache.ProcessFaultBuffer(); } + m_image_heap.RunGarbageCollector(); m_texture_cache.ProcessDownloadImages(); m_texture_cache.RunGarbageCollector(); m_buffer_cache.RunGarbageCollector(); diff --git a/tests/ShaderRecompilerComputeTests.cpp b/tests/ShaderRecompilerComputeTests.cpp index a1ae255c8..615e841c0 100644 --- a/tests/ShaderRecompilerComputeTests.cpp +++ b/tests/ShaderRecompilerComputeTests.cpp @@ -5684,6 +5684,25 @@ public: ImageHeapTestAccess::MemoryCount(heap) == 0u && textures.GetImage(resident).heap_refs.load() == 0u, "unmapping the table kept its images pinned"); + // A smaller table enters once a second draw acquires it and leaves at the suspend point + // after a frame that did not acquire it. Larger tables stay. + const IR::ImageHeapRequest small{table, table_size / 2u, 32u, 0u, table_size / 64u, &sampled}; + Require(name, "first small draw", + acquire(sampled, idle_table) && !heap.Acquire(small, view) && + !heap.Acquire(small, view), + "a smaller table entered on its first draw"); + draw(); + Require(name, "second small draw", heap.Acquire(small, view), + "a smaller table was declined on its second draw"); + context.RunGarbageCollector(); + Require(name, "acquired frame", ImageHeapTestAccess::MemoryCount(heap) == 2u, + "a table left at the suspend point of a frame that acquired it"); + draw(); + context.RunGarbageCollector(); + Require(name, "idle frame", + ImageHeapTestAccess::MemoryCount(heap) == 1u && + textures.GetImage(resident).heap_refs.load() == 0u, + "a suspend point kept an idle smaller table pinned or dropped a larger one"); // The idle table stays watched; destroying the heap must unwatch it before the page // manager fails on live watches. Require(name, "idle table",