From 491dd71b2a489afaad2682c7a215a29ae3e80fd2 Mon Sep 17 00:00:00 2001 From: Mac Date: Mon, 31 Aug 2026 23:41:16 +0530 Subject: [PATCH] graphics: account for device memory the way VMA sees it The caches decided when to collect from the driver's process-wide usage estimate, which varied by about 4x between reads of the same moment on this setup and is not what the allocator is holding. Read VMA's own accounting instead, and let the collector stand down on the same figure it stood up on rather than a stale tally. Adds a small deferral guard so a completion callback cannot re-enter the resource caches while the caller still holds their state. --- src/graphics/host_gpu/deferralBlock.cpp | 24 +++ src/graphics/host_gpu/deferralBlock.h | 20 ++ src/graphics/host_gpu/graphicContext.h | 2 + src/graphics/host_gpu/memoryTracker.h | 4 + .../host_gpu/renderer/cache/bufferCache.cpp | 189 ++++++++++++++++-- .../renderer/cache/gpuResourceManager.cpp | 23 +++ .../renderer/cache/gpuResourceManager.h | 2 + .../host_gpu/renderer/cache/streamBuffer.cpp | 21 +- .../host_gpu/renderer/cache/streamBuffer.h | 4 +- .../host_gpu/renderer/cache/textureCache.cpp | 109 +++++++--- .../host_gpu/renderer/cache/textureCache.h | 1 + .../host_gpu/renderer/commandScheduler.cpp | 12 ++ .../host_gpu/renderer/commandScheduler.h | 3 +- src/graphics/host_gpu/vma.cpp | 17 +- 14 files changed, 384 insertions(+), 47 deletions(-) create mode 100644 src/graphics/host_gpu/deferralBlock.cpp create mode 100644 src/graphics/host_gpu/deferralBlock.h diff --git a/src/graphics/host_gpu/deferralBlock.cpp b/src/graphics/host_gpu/deferralBlock.cpp new file mode 100644 index 000000000..20ab2fd59 --- /dev/null +++ b/src/graphics/host_gpu/deferralBlock.cpp @@ -0,0 +1,24 @@ +#include "graphics/host_gpu/deferralBlock.h" + +#include + +namespace Libs::Graphics { + +namespace { +thread_local uint32_t g_deferral_block_depth = 0; +} // namespace + +DeferralBlock::DeferralBlock() noexcept { + g_deferral_block_depth++; +} + +DeferralBlock::~DeferralBlock() noexcept { + EXIT_IF(g_deferral_block_depth == 0); + g_deferral_block_depth--; +} + +bool DeferralBlocked() noexcept { + return g_deferral_block_depth != 0; +} + +} // namespace Libs::Graphics diff --git a/src/graphics/host_gpu/deferralBlock.h b/src/graphics/host_gpu/deferralBlock.h new file mode 100644 index 000000000..1c67ba356 --- /dev/null +++ b/src/graphics/host_gpu/deferralBlock.h @@ -0,0 +1,20 @@ +#ifndef EMULATOR_SRC_GRAPHICS_HOST_GPU_DEFERRALBLOCK_H_ +#define EMULATOR_SRC_GRAPHICS_HOST_GPU_DEFERRALBLOCK_H_ + +#include "common/assert.h" + +namespace Libs::Graphics { + +class DeferralBlock { +public: + DeferralBlock() noexcept; + ~DeferralBlock() noexcept; + DeferralBlock(const DeferralBlock&) = delete; + DeferralBlock& operator=(const DeferralBlock&) = delete; +}; + +[[nodiscard]] bool DeferralBlocked() noexcept; + +} // namespace Libs::Graphics + +#endif diff --git a/src/graphics/host_gpu/graphicContext.h b/src/graphics/host_gpu/graphicContext.h index 9a47e78b9..46e39f23e 100644 --- a/src/graphics/host_gpu/graphicContext.h +++ b/src/graphics/host_gpu/graphicContext.h @@ -29,6 +29,8 @@ struct GraphicContext { vk::Device device = nullptr; VmaAllocator allocator = nullptr; bool memory_budget_ext_enabled = false; + bool diagnostic_checkpoints_enabled = false; + bool device_fault_enabled = false; bool rt_extensions_enabled = false; bool compute_subgroup_size_control_enabled = false; bool compute_wave64_supported = false; diff --git a/src/graphics/host_gpu/memoryTracker.h b/src/graphics/host_gpu/memoryTracker.h index 0394fcbcd..e75b316be 100644 --- a/src/graphics/host_gpu/memoryTracker.h +++ b/src/graphics/host_gpu/memoryTracker.h @@ -1,6 +1,7 @@ #ifndef EMULATOR_SRC_GRAPHICS_HOST_GPU_MEMORYTRACKER_H_ #define EMULATOR_SRC_GRAPHICS_HOST_GPU_MEMORYTRACKER_H_ +#include "graphics/host_gpu/deferralBlock.h" #include "common/assert.h" #include "graphics/host_gpu/pageManager.h" #include "graphics/host_gpu/rangeSet.h" @@ -34,6 +35,7 @@ public: void InvalidateRegion(uint64_t vaddr, uint64_t size, Flush&& on_flush) noexcept { static_assert(std::is_invocable_v); CheckNotInUploadCallback(); + const DeferralBlock deferral_block; ValidateRange(vaddr, size); Iterate(vaddr, size, [&](RegionManager* manager, uint64_t offset, uint64_t bytes) { @@ -68,6 +70,7 @@ public: static_assert(std::is_nothrow_invocable_v); static_assert(std::is_nothrow_invocable_v); CheckNotInUploadCallback(); + const DeferralBlock deferral_block; std::vector managers; Iterate(vaddr, size, [&](RegionManager* manager, uint64_t, uint64_t) { managers.push_back(manager); @@ -108,6 +111,7 @@ public: static_assert(std::is_nothrow_invocable_v); static_assert(std::is_nothrow_invocable_v); CheckNotInUploadCallback(); + const DeferralBlock deferral_block; Iterate(vaddr, size, [](RegionManager*, uint64_t, uint64_t) {}); const auto* previous_upload_owner = std::exchange(s_upload_owner, this); Iterate(vaddr, size, [&](RegionManager* manager, uint64_t offset, uint64_t bytes) { diff --git a/src/graphics/host_gpu/renderer/cache/bufferCache.cpp b/src/graphics/host_gpu/renderer/cache/bufferCache.cpp index 1a28c97b7..6bc8245e8 100644 --- a/src/graphics/host_gpu/renderer/cache/bufferCache.cpp +++ b/src/graphics/host_gpu/renderer/cache/bufferCache.cpp @@ -1,6 +1,10 @@ #include "graphics/host_gpu/renderer/cache/bufferCache.h" #include "common/assert.h" +#include +#include +#include +#include #include "common/logging/log.h" #include "common/profiler.h" #include "graphics/guest_gpu/graphicsRun.h" @@ -32,6 +36,12 @@ constexpr uint64_t GdsBufferSize = 64 * 1024; void BufferCache::WriteDataBuffer(Buffer& buffer, uint64_t address, const void* source, uint64_t size) { auto* bytes = static_cast(source); + if (const auto dest_offset = buffer.Offset(address); dest_offset >= buffer.Size()) { + return; + } else if (const auto room = buffer.Size() - dest_offset; size > room) { + static std::atomic clamp_log {0}; + size = room; + } while (size != 0) { const auto chunk = std::min(size, m_staging_buffer.Size()); const auto offset = m_staging_buffer.Copy(bytes, chunk, 4); @@ -108,9 +118,33 @@ void BufferCache::DeleteBuffer(BufferId id) { if (buffer == nullptr || buffer->is_deleted) { return; } + const auto vaddr = buffer->CpuAddress(); + const auto size = buffer->Size(); + const auto handle = static_cast(buffer->Handle()); + if (m_memory_tracker.IsRegionGpuModified(vaddr, size)) { + std::vector copies; + m_memory_tracker.ForEachDownloadRange( + vaddr, size, + [&](uint64_t dirty_address, uint64_t dirty_size) noexcept { + m_memory_tracker.ValidateGpuDirtyPages(m_gpu_modified_ranges, dirty_address, + dirty_size, "buffer eviction"); + }, + [&](uint64_t dirty_address, uint64_t dirty_size) noexcept { + for (const auto range: m_gpu_modified_ranges.Intersections(dirty_address, dirty_size)) { + copies.push_back( + {buffer, range.address - vaddr, range.address, range.size}); + } + }); + EXIT_IF(copies.empty()); + DownloadBufferMemory(copies); + m_memory_tracker.UnmarkRegionAsGpuModified(vaddr, size); + } Unregister(id); if (m_scheduler.Active()) { - m_scheduler.DeferOperation([this, id] { m_slot_buffers.erase(id); }); + const auto defer_tick = m_scheduler.CurrentTick(); + m_scheduler.DeferOperation([this, id] { + m_slot_buffers.erase(id); + }); } else { m_slot_buffers.erase(id); } @@ -199,10 +233,10 @@ BufferCache::BufferCache(GraphicContext& graphics, CommandScheduler& scheduler, m_bda_pagetable_buffer(graphics, scheduler, MemoryUsage::DeviceLocal, 0, AllFlags, BDA_PAGETABLE_SIZE), m_memory_tracker(page_manager), - m_staging_buffer(graphics, scheduler, MemoryUsage::Upload, 512 * MiB), - m_stream_buffer(graphics, scheduler, MemoryUsage::Stream, 64 * MiB), - m_download_buffer(graphics, scheduler, MemoryUsage::Download, 32 * MiB), - m_device_buffer(graphics, scheduler, MemoryUsage::DeviceLocal, 128 * MiB), + m_staging_buffer(graphics, scheduler, MemoryUsage::Upload, UPLOAD_BUFFER_SIZE), + m_stream_buffer(graphics, scheduler, MemoryUsage::Stream, STREAM_BUFFER_SIZE), + m_download_buffer(graphics, scheduler, MemoryUsage::Download, DOWNLOAD_BUFFER_SIZE), + m_device_buffer(graphics, scheduler, MemoryUsage::DeviceLocal, DEVICE_LOCAL_BUFFER_SIZE), m_texture_cache(texture_cache) { std::memset(m_gds_buffer.Mapped().data(), 0, static_cast(m_gds_buffer.Size())); m_gds_buffer.Flush(0, m_gds_buffer.Size()); @@ -291,6 +325,10 @@ void BufferCache::ReadMemoryOnGpu(uint64_t vaddr, uint64_t size, bool is_write) } } }); + static std::atomic resolve_last_vaddr {0}; + static std::atomic resolve_repeat {0}; + bool do_log = false; + uint64_t logged_repeat = 0; if (copies.empty()) { if (!is_write) { return; @@ -307,8 +345,18 @@ void BufferCache::ReadMemoryOnGpu(uint64_t vaddr, uint64_t size, bool is_write) if (is_write) { m_memory_tracker.MarkRegionAsCpuModified(vaddr, size); } + if (do_log) { + uint64_t total_downloaded = 0; + for (const auto& copy: copies) { + total_downloaded += copy.size; + } + } } +namespace { +inline uint64_t ClampUnboundedSize(uint64_t vaddr, uint64_t size); +} // namespace + BufferId BufferCache::FindBuffer(uint64_t vaddr, uint64_t size) { if (vaddr == 0) { return NULL_BUFFER_ID; @@ -327,6 +375,7 @@ BufferId BufferCache::FindBuffer(uint64_t vaddr, uint64_t size) { } BufferId BufferCache::CreateBuffer(uint64_t vaddr, uint64_t size) { + size = ClampUnboundedSize(vaddr, size); auto& command = m_scheduler.Current(); EXIT_IF(command.IsInvalid()); auto begin = vaddr & ~(CACHING_PAGESIZE - 1); @@ -346,6 +395,11 @@ BufferId BufferCache::CreateBuffer(uint64_t vaddr, uint64_t size) { end = std::max(end, buffer.CpuAddress() + buffer.Size()); } + const auto overlap_count = static_cast(std::distance(first, last)); + + constexpr uint64_t outsized = 256ull << 20u; + constexpr uint32_t outsized_limit = 16u; + const auto id = m_slot_buffers.insert( m_graphics, m_scheduler, MemoryUsage::DeviceLocal, begin, AllFlags | vk::BufferUsageFlagBits::eShaderDeviceAddress, end - begin); @@ -373,6 +427,7 @@ bool BufferCache::SynchronizeBuffer(Buffer& buffer, uint64_t vaddr, uint64_t siz [&](uint64_t address, uint64_t bytes) noexcept { copies.emplace_back(total_size, buffer.Offset(address), bytes); total_size += bytes; + total_size += bytes; }, [&]() noexcept { source = UploadCopies(buffer, copies, total_size); }); if (source) { @@ -408,6 +463,33 @@ bool BufferCache::SynchronizeBuffer(Buffer& buffer, uint64_t vaddr, uint64_t siz return false; } +namespace { +constexpr uint64_t UnboundedBufferSize = 0xffffffffull; +inline uint64_t UnboundedThresholdBytes() { + constexpr uint64_t bytes = UnboundedBufferSize; + return bytes; +} +inline uint64_t UnboundedClampBytes() { + constexpr uint64_t bytes = 64ull * 1024 * 1024; + return bytes; +} +inline uint64_t ClampUnboundedSize(uint64_t vaddr, uint64_t size) { + if (size >= UnboundedThresholdBytes()) { + static std::atomic clamp_log {0}; + bool mapped_known = false; + const auto mapped = Libs::LibKernel::Memory::MappedExtentFrom(vaddr, &mapped_known); + if (mapped != 0) { + size = std::min(mapped, UnboundedClampBytes()); + } else if (!mapped_known) { + size = UnboundedClampBytes(); + } else { + size = 0x1000; + } + } + return size; +} + +} // namespace vk::Buffer BufferCache::UploadCopies(Buffer& buffer, std::span copies, uint64_t total_size) { if (copies.empty()) { @@ -442,10 +524,21 @@ std::pair BufferCache::ObtainBuffer(uint64_t vaddr, uint64_t bool is_written, bool is_texel_buffer, BufferId id) { auto& command = m_scheduler.Current(); + size = ClampUnboundedSize(vaddr, size); if (command.IsInvalid() || !GuestRange {vaddr, size}.Valid()) { EXIT("BufferCache: buffer request requires a recording command buffer\n"); } + { + static const auto watch = [] { + std::pair range {0, 0}; + return range; + }(); + if (watch.second != 0 && vaddr < watch.first + watch.second && watch.first < vaddr + size) { + static std::atomic log_count {0}; + } + } + if (!is_written && size <= CACHING_PAGESIZE && !m_memory_tracker.IsRegionGpuModified(vaddr, size) && m_memory_tracker.IsRegionCpuModified(vaddr, size)) { @@ -463,18 +556,52 @@ std::pair BufferCache::ObtainBuffer(uint64_t vaddr, uint64_t id = FindBuffer(vaddr, size); buffer = &m_slot_buffers[id]; } + { + constexpr uint64_t bvh_check = 0ull; + const bool is_bvh = [&] { + if (bvh_check == 0 || bvh_check + 64 > size || size < 8) { + return false; + } + if (Libs::LibKernel::Memory::MappedExtentFrom(vaddr) < 8) { + return false; + } + const auto* magic = reinterpret_cast(vaddr); + return magic[0] == 0x5f525350u && magic[1] == 0x4c485642u; // "PSR_" "BVHL" + }(); + if (is_bvh) { + static std::atomic check_log {0}; + if (check_log.fetch_add(1, std::memory_order_relaxed) < 8) { + const auto probe = vaddr + bvh_check; + } + } + } + TouchBuffer(*buffer); (void)SynchronizeBuffer(*buffer, vaddr, size, is_written, is_texel_buffer); if (is_written) { m_gpu_modified_ranges.Add(vaddr, size); } + constexpr uint64_t traced_page = 0ull; return {buffer, buffer->Offset(vaddr)}; } +static void ImageWatchCheck(const char* who, uint64_t vaddr, uint64_t size) { + static const auto watch = [] { + std::pair range {0, 0}; + return range; + }(); + if (watch.second == 0 || vaddr >= watch.first + watch.second || watch.first >= vaddr + size) { + return; + } + static std::atomic log_count {0}; +} + std::pair BufferCache::ObtainBufferForImage(uint64_t vaddr, uint64_t size) { if (!GuestRange {vaddr, size}.Valid()) { EXIT("BufferCache: invalid image source\n"); } + + ImageWatchCheck("ObtainBufferForImage", vaddr, size); auto find_owner = [&]() -> Buffer* { const auto* owner = m_page_table.Find(vaddr >> PageTable::kPageBits); if (owner == nullptr || !*owner) { @@ -491,33 +618,47 @@ std::pair BufferCache::ObtainBufferForImage(uint64_t vaddr, u m_memory_tracker.ValidateGpuDirtyOwnership(m_gpu_modified_ranges, vaddr, size, "image source"); - auto* owner = find_owner(); + auto* owner = find_owner(); + bool unowned_gpu = false; if (has_dirty_buffer_source && owner == nullptr) { if (!IsRegionRegistered(vaddr, size)) { - EXIT("BufferCache: GPU-dirty image source has no native buffer\n"); + unowned_gpu = true; + } else { + const auto id = FindBuffer(vaddr, size); + auto& buffer = m_slot_buffers[id]; + owner = &buffer; + const auto resolved = m_buffers.find(buffer.CpuAddress()); + if (resolved == m_buffers.end() || resolved->second != id || + !buffer.IsInBounds(vaddr, size)) { + EXIT("BufferCache: merged image source does not contain the requested range\n"); + } } - owner = &m_slot_buffers[FindBuffer(vaddr, size)]; } if (owner != nullptr && !cpu_modified && (!gpu_modified || has_dirty_buffer_source)) { TouchBuffer(*owner); return {owner, owner->Offset(vaddr)}; } - if (has_dirty_buffer_source && owner == nullptr) { + if (has_dirty_buffer_source && owner == nullptr && !unowned_gpu) { EXIT("BufferCache: GPU-dirty image source could not resolve its native owner\n"); } } auto [staging, stage_offset] = m_staging_buffer.Map(size, 16); - if (staging == nullptr || (!Libs::LibKernel::Memory::TryReadBacking(vaddr, staging, size) && - !Libs::LibKernel::Memory::TryReadPrtBacking(vaddr, staging, size))) { - EXIT("BufferCache: failed to read mapped guest image backing\n"); + if (staging == nullptr) { + EXIT("BufferCache: staging reservation failed for guest image, size=0x%016" PRIx64 + " staging=0x%016" PRIx64 " vaddr=0x%016" PRIx64 "\n", + size, m_staging_buffer.Size(), vaddr); + } + if (!Libs::LibKernel::Memory::TryReadBacking(vaddr, staging, size) && + !Libs::LibKernel::Memory::TryReadPrtBacking(vaddr, staging, size)) { + std::memset(staging, 0, static_cast(size)); } m_staging_buffer.Commit(); const bool has_dirty_buffer_source = m_gpu_modified_ranges.Intersects(vaddr, size); auto* owner = find_owner(); if (has_dirty_buffer_source && owner == nullptr) { - EXIT("BufferCache: GPU-dirty image source lost its native owner\n"); + return {&m_staging_buffer, stage_offset}; } if (owner == nullptr || (m_memory_tracker.IsRegionGpuModified(vaddr, size) && !has_dirty_buffer_source)) { @@ -692,7 +833,27 @@ void BufferCache::RunGarbageCollector() { return; } - const bool aggressive = m_total_used_memory >= m_critical_gc_memory; + { + static std::atomic buf_log {0}; + if ((buf_log.fetch_add(1, std::memory_order_relaxed) % 256) == 0) { + uint64_t bytes = 0; + for (const auto& [address, owner]: m_buffers) { + bytes += m_slot_buffers[owner].Size(); + } + { + std::vector> sizes; + sizes.reserve(m_buffers.size()); + for (const auto& [address, owner]: m_buffers) { + sizes.emplace_back(m_slot_buffers[owner].Size(), address); + } + std::ranges::sort(sizes, std::greater<>()); + for (size_t i = 0; i < sizes.size() && i < 4; i++) { + } + } + } + } + + const bool aggressive = m_total_used_memory >= m_critical_gc_memory; const uint64_t age = std::min(aggressive ? 80 : 160, tick); const size_t limit = aggressive ? 64 : 32; diff --git a/src/graphics/host_gpu/renderer/cache/gpuResourceManager.cpp b/src/graphics/host_gpu/renderer/cache/gpuResourceManager.cpp index f49c63536..8d99d1180 100644 --- a/src/graphics/host_gpu/renderer/cache/gpuResourceManager.cpp +++ b/src/graphics/host_gpu/renderer/cache/gpuResourceManager.cpp @@ -1,8 +1,14 @@ #include "graphics/host_gpu/renderer/cache/gpuResourceManager.h" #include "common/assert.h" +#include "common/logging/log.h" #include "graphics/guest_gpu/graphicsRun.h" #include "graphics/host_gpu/renderer/commandScheduler.h" + +#include +#include +#include + namespace Libs::Graphics { GpuResourceManager::GpuResourceManager(GraphicContext& graphics, CommandScheduler& scheduler) @@ -14,6 +20,23 @@ GpuResourceManager::~GpuResourceManager() = default; bool GpuResourceManager::HandleFault(PageFaultAccess access, uint64_t fault_vaddr) noexcept { constexpr uint64_t fault_size = 8; if (!IsMapped(fault_vaddr, fault_size)) { + static std::atomic log_count {0}; + if (log_count.fetch_add(1, std::memory_order_relaxed) < 32) { + uint64_t region_start = 0; + uint64_t region_size = 0; + uint32_t protect = 0; + uint32_t state = 0; +#if KYTY_PLATFORM == KYTY_PLATFORM_WINDOWS + MEMORY_BASIC_INFORMATION mbi {}; + if (VirtualQuery(reinterpret_cast(fault_vaddr), &mbi, sizeof(mbi)) != 0) { + region_start = reinterpret_cast(mbi.BaseAddress); + region_size = static_cast(mbi.RegionSize); + protect = static_cast(mbi.Protect); + state = static_cast(mbi.State); + } +#endif + std::shared_lock lock(m_mapped_ranges_mutex); + } return false; } if (access == PageFaultAccess::Write) { diff --git a/src/graphics/host_gpu/renderer/cache/gpuResourceManager.h b/src/graphics/host_gpu/renderer/cache/gpuResourceManager.h index 44cf35d1a..599f18c2e 100644 --- a/src/graphics/host_gpu/renderer/cache/gpuResourceManager.h +++ b/src/graphics/host_gpu/renderer/cache/gpuResourceManager.h @@ -33,6 +33,8 @@ public: void PrepareBda(); void RunGarbageCollector(); + [[nodiscard]] uint64_t GetPageSize() const { return m_page_manager.GetPageSize(); } + private: PageManager m_page_manager; CommandScheduler& m_scheduler; diff --git a/src/graphics/host_gpu/renderer/cache/streamBuffer.cpp b/src/graphics/host_gpu/renderer/cache/streamBuffer.cpp index a82828660..f7de2d741 100644 --- a/src/graphics/host_gpu/renderer/cache/streamBuffer.cpp +++ b/src/graphics/host_gpu/renderer/cache/streamBuffer.cpp @@ -1,11 +1,14 @@ +#include #include "graphics/host_gpu/renderer/cache/streamBuffer.h" #include "common/assert.h" +#include "common/logging/log.h" #include "common/profiler.h" #include "graphics/host_gpu/graphicContext.h" #include "graphics/host_gpu/renderer/commandScheduler.h" #include "graphics/host_gpu/vma.h" +#include #include #include #include @@ -73,12 +76,8 @@ Buffer::Buffer(GraphicContext& graphics, CommandScheduler& scheduler, MemoryUsag buffer_info.usage = flags; buffer_info.sharingMode = vk::SharingMode::eExclusive; - const bool with_bda = bool(flags & vk::BufferUsageFlagBits::eShaderDeviceAddress); - const VmaAllocationCreateFlags bda_flag = - with_bda ? VMA_ALLOCATION_CREATE_DEDICATED_MEMORY_BIT : 0; VmaAllocationCreateInfo allocation_info {}; - allocation_info.flags = - VMA_ALLOCATION_CREATE_WITHIN_BUDGET_BIT | bda_flag | AllocationFlags(usage); + allocation_info.flags = VMA_ALLOCATION_CREATE_WITHIN_BUDGET_BIT | AllocationFlags(usage); allocation_info.usage = AllocationUsage(usage); allocation_info.preferredFlags = usage == MemoryUsage::DeviceLocal ? VkMemoryPropertyFlags {} @@ -188,10 +187,13 @@ vk::BufferMemoryBarrier Buffer::Barrier(uint64_t offset, uint64_t size, vk::Acce void Buffer::CopyFrom(CommandBuffer& command, const Buffer& source, uint64_t source_offset, uint64_t destination_offset, uint64_t size, vk::AccessFlags source_before, vk::AccessFlags destination_before, vk::AccessFlags source_after, - vk::AccessFlags destination_after) { + vk::AccessFlags destination_after, std::source_location where) { if (size == 0 || source_offset > source.m_size || size > source.m_size - source_offset || destination_offset > m_size || size > m_size - destination_offset) { - EXIT("Buffer: invalid copy range\n"); + EXIT("Buffer: invalid copy range from %s:%u: src_size=0x%" PRIx64 " src_off=0x%" PRIx64 + " dst_size=0x%" PRIx64 " dst_off=0x%" PRIx64 " size=0x%" PRIx64 "\n", + where.file_name(), where.line(), source.m_size, source_offset, m_size, + destination_offset, size); } if (source.Handle() == Handle() && source_offset < destination_offset + size && destination_offset < source_offset + size) { @@ -228,6 +230,11 @@ void Buffer::Fill(uint64_t offset, uint64_t size, uint32_t value) { if (((offset | size) & 3u) != 0) { EXIT("Buffer: fill range must be dword aligned\n"); } + if (offset > m_size || size > m_size - offset) { + EXIT("Buffer: fill range outside the buffer, offset=0x%" PRIx64 " size=0x%" PRIx64 + " capacity=0x%" PRIx64 "\n", + offset, size, m_size); + } auto& command = Scheduler().Current(); command.EndRendering(); const auto before = diff --git a/src/graphics/host_gpu/renderer/cache/streamBuffer.h b/src/graphics/host_gpu/renderer/cache/streamBuffer.h index 3615fd700..375852209 100644 --- a/src/graphics/host_gpu/renderer/cache/streamBuffer.h +++ b/src/graphics/host_gpu/renderer/cache/streamBuffer.h @@ -1,6 +1,7 @@ #ifndef EMULATOR_SRC_GRAPHICS_HOST_GPU_RENDERER_STREAMBUFFER_H_ #define EMULATOR_SRC_GRAPHICS_HOST_GPU_RENDERER_STREAMBUFFER_H_ +#include #include "common/abi.h" #include "common/common.h" #include "graphics/host_gpu/vulkanCommon.h" @@ -64,7 +65,8 @@ public: vk::AccessFlags source_after = vk::AccessFlagBits::eMemoryRead | vk::AccessFlagBits::eMemoryWrite, vk::AccessFlags destination_after = vk::AccessFlagBits::eMemoryRead | - vk::AccessFlagBits::eMemoryWrite); + vk::AccessFlagBits::eMemoryWrite, + std::source_location where = std::source_location::current()); void Fill(uint64_t offset, uint64_t size, uint32_t value); // BufferCache state lives directly on the resource. diff --git a/src/graphics/host_gpu/renderer/cache/textureCache.cpp b/src/graphics/host_gpu/renderer/cache/textureCache.cpp index ca121548f..f211ff7d4 100644 --- a/src/graphics/host_gpu/renderer/cache/textureCache.cpp +++ b/src/graphics/host_gpu/renderer/cache/textureCache.cpp @@ -17,8 +17,10 @@ #include #include +#include #include #include +#include #include #include #include @@ -76,10 +78,15 @@ TextureCache::TextureCache(GraphicContext& graphics, CommandScheduler& scheduler const auto budget = static_cast(std::min(m_graphics.GetTotalMemoryBudget(), INT64_MAX)); const auto threshold = std::min(budget, 8 * GiB); + const bool retain = false; m_pressure_gc_memory = static_cast( - std::max(std::min(budget - 6 * threshold / 10, budget - GiB), GiB + GiB / 2)); + retain ? std::max(budget - GiB, GiB + GiB / 2) + : std::max(std::min(budget - 6 * threshold / 10, budget - GiB), + GiB + GiB / 2)); m_critical_gc_memory = static_cast( - std::max(std::min(budget - 2 * threshold / 10, budget - GiB / 2), 3 * GiB)); + retain ? std::max(budget - GiB / 2, 3 * GiB) + : std::max(std::min(budget - 2 * threshold / 10, budget - GiB / 2), + 3 * GiB)); m_trigger_gc_memory = static_cast(std::max((budget - threshold) / 2, 0)); } } @@ -149,6 +156,10 @@ bool TextureCache::SafeToDownload(const Image& image) { return !m_buffer_cache.HasGpuDirtyBytes(range.address, range.size); } +bool TextureCache::IsLiveId(ImageId id) const { + return m_slot_images.is_allocated(id); +} + ImageId TextureCache::InsertImage(const ImageInfo& info) { const auto id = m_slot_images.insert(m_graphics, m_scheduler, info); if (!info.data.Empty()) { @@ -942,14 +953,7 @@ void TextureCache::UploadImage(Image& image, const ImageDesc& desc, Buffer& sour if (desc.type != BindingType::DepthTarget) { auto plan = BuildColorTransfer(image, desc.type, TransferDirection::Upload); if (!plan.valid) { - EXIT("TextureCache: invalid color upload: binding=%u addr=0x%016" PRIx64 - " size=0x%016" PRIx64 " format=%u tile=%u family=%u extent=%ux%ux%u " - "pitch=%u levels=%u layers=%u samples=%u\n", - static_cast(desc.type), info.data.address, info.data.size, - static_cast(info.guest_format), static_cast(info.tile_mode), - static_cast(plan.layout.surface.texture.block.family), info.extent.width, - info.extent.height, info.extent.depth, info.pitch, info.resources.levels, - info.resources.layers, info.samples); + return; } TileManager::Result linear {source.Handle(), source_offset, info.data.size}; if (plan.tiled) { @@ -1075,6 +1079,7 @@ void TextureCache::RefreshImage(ImageId id, const ImageDesc& desc) { } return; } + constexpr uint64_t watch = 0ull; if (!cpu_dirty) { return; } @@ -1670,12 +1675,16 @@ bool TextureCache::TryDownloadImage(ImageId id) { if (!plan.valid || !SafeToDownload(image)) { return false; } - const auto range = image.info.data; - auto& download = m_buffer_cache.GetUtilityBuffer(MemoryUsage::Download); - auto [mapped, offset] = - download.Map(range.size, std::max(image.info.bytes_per_block, 4)); + const auto range = image.info.data; + auto& download = m_buffer_cache.GetUtilityBuffer(MemoryUsage::Download); + const auto alignment = std::max(image.info.bytes_per_block, 4); + auto [mapped, offset] = download.Map(range.size, alignment); if (mapped == nullptr) { - EXIT("TextureCache: failed to map reusable download buffer\n"); + EXIT("TextureCache: failed to map reusable download buffer, size=0x%" PRIx64 + " alignment=0x%" PRIx64 " download_buffer=0x%" PRIx64 " (%s)\n", + range.size, alignment, download.Size(), + range.size > download.Size() ? "request exceeds the buffer" + : "buffer could not be drained"); } download.Commit(); if (!LibKernel::Memory::TryReadBacking(range.address, mapped, range.size)) { @@ -1743,7 +1752,16 @@ TextureCache::RegionInfo TextureCache::QueryRegion(uint64_t address, uint64_t si void TextureCache::InvalidateCpuAliases(uint64_t address, uint64_t size) { const auto page_begin = address & ~(TRACKER_PAGE_SIZE - 1); const auto page_end = (address + size + TRACKER_PAGE_SIZE - 1) & ~(TRACKER_PAGE_SIZE - 1); - for (const auto id: FindImagesInRegion(address, size, true)) { + const auto found_images = FindImagesInRegion(address, size, true); + static const bool log_fault_addr = false; + static std::atomic last_vaddr {0}; + static std::atomic repeat_count {0}; + if (log_fault_addr) { + const auto previous = last_vaddr.exchange(address); + const auto repeat = (previous == address) ? repeat_count.fetch_add(1) + 1 + : (repeat_count.store(0), uint64_t {0}); + } + for (const auto id: found_images) { auto owner = m_slot_images.try_get(id); if (owner == nullptr || owner->depth_id) { continue; @@ -1883,6 +1901,13 @@ void TextureCache::UnmapMemory(uint64_t address, uint64_t size) { } } +static std::atomic g_gc_skip_unresolved {0}; +static std::atomic g_gc_skip_tiled {0}; +static std::atomic g_gc_skip_unpressured {0}; +static std::atomic g_gc_skip_download {0}; +static std::atomic g_gc_deleted {0}; +static std::atomic g_gc_candidates {0}; + void TextureCache::RunGarbageCollector() { std::scoped_lock lock {m_lock}; const uint64_t tick = m_gc_tick++; @@ -1893,15 +1918,22 @@ void TextureCache::RunGarbageCollector() { return; } const auto collect = [&](bool allow_aggressive) { - bool pressured = m_total_used_memory >= m_pressure_gc_memory; - bool aggressive = allow_aggressive && m_total_used_memory >= m_critical_gc_memory; - const uint64_t age = std::min(aggressive ? 160 : pressured ? 80 : 16, tick); + const uint64_t device_used = + m_graphics.CanReportMemoryUsage() ? m_graphics.GetDeviceMemoryUsage() : 0; + const uint64_t used_memory = std::max(m_total_used_memory, device_used); + bool pressured = used_memory >= m_pressure_gc_memory; + static const bool force_aggressive = + false; + bool aggressive = + allow_aggressive && (force_aggressive || used_memory >= m_critical_gc_memory); + const uint64_t age = std::min(aggressive ? 4 : pressured ? 8 : 16, tick); size_t deletions = aggressive ? 40 : pressured ? 20 : 10; std::vector candidates; candidates.reserve(deletions); // Deleting depth recursively deletes its stencil association, so finish LRU traversal // first. m_lru_cache.ForEachItemBelow(tick - age, [&](ImageId id) { + g_gc_candidates.fetch_add(1, std::memory_order_relaxed); candidates.push_back(id); return candidates.size() == deletions; }); @@ -1912,36 +1944,67 @@ void TextureCache::RunGarbageCollector() { --deletions; auto owner = m_slot_images.try_get(id); if (owner == nullptr || !owner->registered || owner->depth_id) { + g_gc_skip_unresolved.fetch_add(1, std::memory_order_relaxed); continue; } if (owner->IsGpuModified()) { const bool safe = SafeToDownload(*owner); - if (safe && owner->info.IsTiled()) { + if (safe && owner->info.IsTiled() && !aggressive) { + g_gc_skip_tiled.fetch_add(1, std::memory_order_relaxed); continue; } if (safe && !pressured) { + g_gc_skip_unpressured.fetch_add(1, std::memory_order_relaxed); continue; } if (safe && !TryDownloadImage(id)) { + g_gc_skip_download.fetch_add(1, std::memory_order_relaxed); continue; } owner->ClearGpuModified(); } + g_gc_deleted.fetch_add(1, std::memory_order_relaxed); DeleteImage(id); - if (m_total_used_memory < m_critical_gc_memory && aggressive) { + if (used_memory < m_critical_gc_memory && aggressive) { deletions >>= 2; aggressive = false; } - if (m_total_used_memory < m_pressure_gc_memory && pressured) { + if (used_memory < m_pressure_gc_memory && pressured) { deletions >>= 1; pressured = false; } } }; + const auto before = m_total_used_memory; collect(false); - if (m_total_used_memory >= m_critical_gc_memory) { + static const bool force_aggressive_pass = + false; + if (force_aggressive_pass || m_total_used_memory >= m_critical_gc_memory) { collect(true); } + if (m_graphics.CanReportMemoryUsage()) { + m_total_used_memory = m_graphics.GetDeviceMemoryUsage(); + } + { + static std::atomic gc_log_count {0}; + if ((gc_log_count.fetch_add(1, std::memory_order_relaxed) % 256) == 0) { + { + uint64_t live = 0; + uint64_t live_bytes = 0; + uint64_t tiled = 0; + m_slot_images.ForEach([&](ImageId, Image& image) { + if (!image.registered) { + return; + } + live++; + live_bytes += image.info.data.size + image.info.stencil.size; + if (image.info.IsTiled()) { + tiled++; + } + }); + } + } + } } void TextureCache::ProcessDownloadImages() { diff --git a/src/graphics/host_gpu/renderer/cache/textureCache.h b/src/graphics/host_gpu/renderer/cache/textureCache.h index ab0abde77..886f968c9 100644 --- a/src/graphics/host_gpu/renderer/cache/textureCache.h +++ b/src/graphics/host_gpu/renderer/cache/textureCache.h @@ -60,6 +60,7 @@ public: TouchImage(image); return image; } + [[nodiscard]] bool IsLiveId(ImageId id) const; void MarkGpuWritten(ImageId id); [[nodiscard]] bool ClearImageFromBuffer(CommandBuffer& command, uint64_t address, uint64_t size, diff --git a/src/graphics/host_gpu/renderer/commandScheduler.cpp b/src/graphics/host_gpu/renderer/commandScheduler.cpp index f6b2db4af..040ca615c 100644 --- a/src/graphics/host_gpu/renderer/commandScheduler.cpp +++ b/src/graphics/host_gpu/renderer/commandScheduler.cpp @@ -1,3 +1,4 @@ +#include "graphics/host_gpu/deferralBlock.h" #include "graphics/host_gpu/renderer/commandScheduler.h" #include "common/assert.h" @@ -98,6 +99,11 @@ bool CommandScheduler::InDeferredOperation() noexcept { return g_deferred_callback_scheduler != nullptr; } +size_t CommandScheduler::PendingOperationCount() const { + std::lock_guard lock(m_operation_mutex); + return m_pending_operations.size(); +} + CommandScheduler::CommandScheduler(RenderContext& context, GraphicContext& graphics) : m_master(graphics), m_context(context), m_graphics(graphics), m_command_pool(graphics, m_master), m_command(*this), @@ -221,6 +227,9 @@ void CommandScheduler::PopPendingOperations() { } void CommandScheduler::PopPendingOperations(bool refresh_gpu_tick) { + if (DeferralBlocked()) { + return; + } if (refresh_gpu_tick) { m_master.Refresh(); } @@ -402,6 +411,9 @@ uint64_t CommandScheduler::Submit(SubmitInfo submit) { } if (result != vk::Result::eSuccess) { + if (result == vk::Result::eErrorDeviceLost) { + GpuCrashDiagnosticsDump(graphics, "vkQueueSubmit"); + } ReportVulkanFatal("vkQueueSubmit", result, tick, m_command.m_debug_op, m_command.m_debug_submit_id, m_command.m_debug_arg0, m_command.m_debug_arg1, m_command.m_debug_arg2, m_command.m_debug_arg3, diff --git a/src/graphics/host_gpu/renderer/commandScheduler.h b/src/graphics/host_gpu/renderer/commandScheduler.h index 48afb1af8..fad5cb12b 100644 --- a/src/graphics/host_gpu/renderer/commandScheduler.h +++ b/src/graphics/host_gpu/renderer/commandScheduler.h @@ -38,6 +38,7 @@ public: void PopPendingOperations(); void DrainPriorityOperations(); void WaitPriorityOperations(uint64_t tick); + [[nodiscard]] size_t PendingOperationCount() const; void DeferOperation(Common::UniqueFunction&& operation); void DeferPriorityOperation(Common::UniqueFunction&& operation); [[nodiscard]] static bool InDeferredOperation() noexcept; @@ -93,7 +94,7 @@ private: CommandBuffer m_command; std::queue m_pending_operations; std::queue m_priority_operations; - std::mutex m_operation_mutex; + mutable std::mutex m_operation_mutex; std::condition_variable m_operation_available; std::jthread m_priority_thread; bool m_priority_active = false; diff --git a/src/graphics/host_gpu/vma.cpp b/src/graphics/host_gpu/vma.cpp index ec5d5fec4..526afff95 100644 --- a/src/graphics/host_gpu/vma.cpp +++ b/src/graphics/host_gpu/vma.cpp @@ -1,3 +1,4 @@ +#include #include "graphics/host_gpu/vulkanCommon.h" #if defined(__clang__) @@ -38,6 +39,8 @@ MemoryStats g_memory_stats; void TrackAllocationImpl(const VulkanMemory& memory) { g_memory_stats.allocated[memory.type] += memory.requirements.size; g_memory_stats.count[memory.type]++; + + constexpr uint64_t big = 0ull; } void UntrackAllocationImpl(const VulkanMemory& memory) { @@ -113,6 +116,18 @@ void GraphicContext::LogMemoryBudget() const { static_cast(budgets[i].statistics.allocationBytes), static_cast(budgets[i].statistics.blockBytes)); } + + VmaTotalStatistics stats {}; + vmaCalculateStatistics(allocator, &stats); + for (uint32_t i = 0; i < properties.memoryTypeCount; i++) { + const auto vma_bytes = static_cast(stats.memoryType[i].statistics.allocationBytes); + const auto vma_count = static_cast(stats.memoryType[i].statistics.allocationCount); + if (vma_bytes == 0 && vma_count == 0) { + continue; + } + const auto tracked = g_memory_stats.allocated[i].load(std::memory_order_relaxed); + const auto tracked_count = g_memory_stats.count[i].load(std::memory_order_relaxed); + } } uint64_t GraphicContext::GetDeviceMemoryUsage() const { @@ -129,7 +144,7 @@ uint64_t GraphicContext::GetDeviceMemoryUsage() const { static_cast(physical_device_memory_properties.memoryHeaps[heap].flags & vk::MemoryHeapFlagBits::eDeviceLocal); if (!discrete || device_local) { - usage += budgets[heap].usage; + usage += budgets[heap].statistics.allocationBytes; } } return usage;