From 861729fc6c6a2287d1bcc670f00ef8566553e3ad Mon Sep 17 00:00:00 2001 From: nmzik Date: Wed, 29 Jul 2026 03:37:09 +0200 Subject: [PATCH] Optimize texture cache tracking (#123) Optimize texture cache page tracking --- src/graphics/host_gpu/pageManager.cpp | 438 +++++++++++++----- .../host_gpu/renderer/cache/textureCache.cpp | 341 ++++++++------ .../host_gpu/renderer/cache/textureCache.h | 20 +- .../host_gpu/renderer/image/image.cpp | 305 ++++++------ src/graphics/host_gpu/renderer/image/image.h | 81 ++-- tests/PageManagerTests.cpp | 70 +++ tests/ShaderRecompilerComputeTests.cpp | 21 +- 7 files changed, 791 insertions(+), 485 deletions(-) diff --git a/src/graphics/host_gpu/pageManager.cpp b/src/graphics/host_gpu/pageManager.cpp index 2e490d11f..5ce0c3dc7 100644 --- a/src/graphics/host_gpu/pageManager.cpp +++ b/src/graphics/host_gpu/pageManager.cpp @@ -2,6 +2,7 @@ #include "graphics/host_gpu/regionDefinitions.h" +#include #include #include #include @@ -71,8 +72,8 @@ static int PageProtToPosix(uint32_t protection) { // Query the current protection of the page containing vaddr via the Mach VM map and // collapse it to the tracker's read/write tags (execute is irrelevant to write tracking). static uint32_t MachQueryPageProt(uint64_t vaddr) { - auto region_addr = static_cast(vaddr); - mach_vm_size_t region_size = 0; + auto region_addr = static_cast(vaddr); + mach_vm_size_t region_size = 0; vm_region_basic_info_data_64_t info {}; mach_msg_type_number_t count = VM_REGION_BASIC_INFO_COUNT_64; mach_port_t object_name = MACH_PORT_NULL; @@ -164,22 +165,27 @@ int ToHostProtection(uint32_t protection) { } } +struct HostMapping { + uint64_t end = 0; + uint32_t protection = UNKNOWN_PROTECTION; +}; + // Async-signal-safe lookup in the address-ordered /proc/self/maps. -uint32_t QueryHostProtection(uint64_t vaddr) noexcept { +HostMapping QueryHostMapping(uint64_t vaddr) noexcept { int fd = ::open("/proc/self/maps", O_RDONLY | O_CLOEXEC); // NOLINT if (fd < 0) { - return UNKNOWN_PROTECTION; + return {}; } enum class Field { Start, End, Perms, Rest }; - uint32_t result = UNKNOWN_PROTECTION; - auto field = Field::Start; - uint64_t start = 0; - uint64_t end = 0; - char perms[4] = {}; - uint32_t perms_len = 0; - bool line_valid = true; + HostMapping result {}; + auto field = Field::Start; + uint64_t start = 0; + uint64_t end = 0; + char perms[4] = {}; + uint32_t perms_len = 0; + bool line_valid = true; char buffer[8192]; @@ -248,10 +254,11 @@ uint32_t QueryHostProtection(uint64_t vaddr) noexcept { if (vaddr < start) { done = true; } else if (vaddr < end && perms_len >= 2) { - result = perms[1] == 'w' ? READ_WRITE_PROTECTION - : perms[0] == 'r' ? READ_ONLY_PROTECTION - : NO_ACCESS_PROTECTION; - done = true; + result.end = end; + result.protection = perms[1] == 'w' ? READ_WRITE_PROTECTION + : perms[0] == 'r' ? READ_ONLY_PROTECTION + : NO_ACCESS_PROTECTION; + done = true; } else { field = Field::Rest; } @@ -266,6 +273,10 @@ uint32_t QueryHostProtection(uint64_t vaddr) noexcept { ::close(fd); return result; } + +uint32_t QueryHostProtection(uint64_t vaddr) noexcept { + return QueryHostMapping(vaddr).protection; +} #endif class SpinGuard final { @@ -301,28 +312,48 @@ uint64_t PageEnd(uint64_t vaddr, uint64_t size) { struct PageManager::Impl { struct PageState { - std::atomic_flag lock = ATOMIC_FLAG_INIT; - uint32_t mappings = 0; - uint32_t gpu_read_mappings = 0; - uint32_t gpu_write_mappings = 0; - uint32_t write_watchers = 0; - uint32_t access_watchers = 0; - uint32_t original_protection = 0; - uint32_t backing_writer = 0; + std::atomic_flag lock = ATOMIC_FLAG_INIT; + uint32_t mappings = 0; + uint32_t gpu_read_mappings = 0; + uint32_t gpu_write_mappings = 0; + uint32_t write_watchers = 0; + uint32_t access_watchers = 0; + uint32_t original_protection = 0; + uint32_t backing_writer = 0; #if defined(__linux__) // Shadow the protection applied through Protect(). uint32_t current_protection = UNKNOWN_PROTECTION; #endif - bool resolving = false; - bool resolving_read_write = false; - bool late_read_pending = false; - bool late_write_pending = false; + bool resolving = false; + bool resolving_read_write = false; + bool late_read_pending = false; + bool late_write_pending = false; }; struct Region { std::array pages; }; + class PageRangeGuard final { + public: + explicit PageRangeGuard(std::span pages): m_pages(pages) { + for (auto* page: m_pages) { + while (page->lock.test_and_set(std::memory_order_acquire)) { + std::atomic_signal_fence(std::memory_order_seq_cst); + } + } + } + ~PageRangeGuard() { + for (auto it = m_pages.rbegin(); it != m_pages.rend(); ++it) { + (*it)->lock.clear(std::memory_order_release); + } + } + KYTY_CLASS_NO_COPY(PageRangeGuard); + + private: + std::span m_pages; + }; + Impl(PageFaultHandler handler, void* context): fault_handler(handler), fault_context(context) { if (fault_handler == nullptr) { Fatal("null fault handler"); @@ -337,8 +368,7 @@ struct PageManager::Impl { #elif defined(__APPLE__) // Under Rosetta the host page size is 4 KB, matching TRACKER_PAGE_SIZE. if (static_cast(getpagesize()) != PAGE_SIZE) { - Fatal("unsupported host page size 0x%08" PRIx32, - static_cast(getpagesize())); + Fatal("unsupported host page size 0x%08" PRIx32, static_cast(getpagesize())); } #else const auto host_page_size = ::sysconf(_SC_PAGESIZE); @@ -405,42 +435,57 @@ struct PageManager::Impl { if (old_protection == NO_ACCESS_PROTECTION && new_protection != NO_ACCESS_PROTECTION) { page.late_read_pending = true; } - if ((old_protection == NO_ACCESS_PROTECTION || - old_protection == READ_ONLY_PROTECTION) && + if ((old_protection == NO_ACCESS_PROTECTION || old_protection == READ_ONLY_PROTECTION) && new_protection == READ_WRITE_PROTECTION) { page.late_write_pending = true; } } - static uint32_t QueryProtection([[maybe_unused]] PageState& page, uint64_t vaddr) { + static void ValidateInitialProtection(std::span pages, uint64_t vaddr) { + const auto end = vaddr + pages.size() * PAGE_SIZE; #if KYTY_PLATFORM == KYTY_PLATFORM_WINDOWS - MEMORY_BASIC_INFORMATION info {}; - if (VirtualQuery(reinterpret_cast(static_cast(vaddr)), &info, - sizeof(info)) == 0 || - info.State != MEM_COMMIT || info.Protect != PAGE_READWRITE) { - Fatal("basic path requires PAGE_READWRITE at 0x%016" PRIx64 " (state=0x%08" PRIx32 - ", protection=0x%08" PRIx32 ")", - vaddr, static_cast(info.State), static_cast(info.Protect)); + for (auto address = vaddr; address < end;) { + MEMORY_BASIC_INFORMATION info {}; + if (VirtualQuery(reinterpret_cast(static_cast(address)), &info, + sizeof(info)) == 0 || + info.State != MEM_COMMIT || info.Protect != PAGE_READWRITE) { + Fatal("basic path requires PAGE_READWRITE at 0x%016" PRIx64 " (state=0x%08" PRIx32 + ", protection=0x%08" PRIx32 ")", + address, static_cast(info.State), + static_cast(info.Protect)); + } + const auto region_end = reinterpret_cast(info.BaseAddress) + info.RegionSize; + if (region_end <= address) { + Fatal("VirtualQuery returned an invalid region at 0x%016" PRIx64, address); + } + address = std::min(end, region_end); } - return info.Protect; #elif defined(__APPLE__) - const uint32_t protection = MachQueryPageProt(vaddr); - if (protection != PAGE_READWRITE) { - Fatal("basic path requires PAGE_READWRITE at 0x%016" PRIx64 " (protection=0x%08" PRIx32 - ")", - vaddr, protection); + for (auto address = vaddr; address < end; address += PAGE_SIZE) { + const uint32_t protection = MachQueryPageProt(address); + if (protection != PAGE_READWRITE) { + Fatal("basic path requires PAGE_READWRITE at 0x%016" PRIx64 + " (protection=0x%08" PRIx32 ")", + address, protection); + } } - return protection; #else - const auto host_protection = QueryHostProtection(vaddr); - if (host_protection != READ_WRITE_PROTECTION) { - Fatal("basic path requires a read/write mapping at 0x%016" PRIx64 - " (protection=0x%08" PRIx32 ")", - vaddr, host_protection); + for (auto address = vaddr; address < end;) { + const auto mapping = QueryHostMapping(address); + if (mapping.protection != READ_WRITE_PROTECTION || mapping.end <= address) { + Fatal("basic path requires a read/write mapping at 0x%016" PRIx64 + " (protection=0x%08" PRIx32 ")", + address, mapping.protection); + } + address = std::min(end, mapping.end); + } + for (auto* page: pages) { + page->current_protection = READ_WRITE_PROTECTION; } - page.current_protection = host_protection; - return host_protection; #endif + for (auto* page: pages) { + page->original_protection = READ_WRITE_PROTECTION; + } } static bool AllowsAccess([[maybe_unused]] const PageState& page, uint64_t vaddr, @@ -470,7 +515,8 @@ struct PageManager::Impl { const auto permitted = [](uint32_t protection, PageFaultAccess wanted) { switch (wanted) { case PageFaultAccess::Read: - return protection == READ_ONLY_PROTECTION || protection == READ_WRITE_PROTECTION; + return protection == READ_ONLY_PROTECTION || + protection == READ_WRITE_PROTECTION; case PageFaultAccess::Write: return protection == READ_WRITE_PROTECTION; default: return false; } @@ -483,26 +529,84 @@ struct PageManager::Impl { #endif } - static void Protect([[maybe_unused]] PageState& page, uint64_t vaddr, uint32_t protection, - uint32_t expected_old, bool fault_path) noexcept { + static void ProtectRange(std::span pages, uint64_t vaddr, uint32_t protection, + std::span expected_old, bool fault_path) noexcept { + const auto size = pages.size() * PAGE_SIZE; + if (pages.size() != expected_old.size()) { + FailFast("protection range state size mismatch"); + } #if KYTY_PLATFORM == KYTY_PLATFORM_WINDOWS - DWORD old_protection = 0; - if (VirtualProtect(reinterpret_cast(static_cast(vaddr)), PAGE_SIZE, - protection, &old_protection) == 0 || - old_protection != expected_old) { - if (fault_path) { - FailFast("VirtualProtect fault transition did not match expected protection"); + struct HostRange { + uint64_t begin = 0; + uint64_t end = 0; + }; + std::vector host_ranges; + const auto end = vaddr + size; + for (auto address = vaddr; address < end;) { + MEMORY_BASIC_INFORMATION info {}; + if (VirtualQuery(reinterpret_cast(static_cast(address)), &info, + sizeof(info)) == 0 || + info.State != MEM_COMMIT) { + if (fault_path) { + FailFast("VirtualProtect fault transition did not match expected protection"); + } + Fatal("invalid protection transition at 0x%016" PRIx64 ", state=0x%08" PRIx32 + ", new=0x%08" PRIx32, + address, static_cast(info.State), protection); + } + const auto region_end = reinterpret_cast(info.BaseAddress) + info.RegionSize; + const auto query_end = std::min(end, region_end); + if (query_end <= address) { + if (fault_path) { + FailFast("VirtualQuery returned an invalid fault transition region"); + } + Fatal("VirtualQuery returned an invalid region at 0x%016" PRIx64, address); + } + const auto first_page = static_cast((address - vaddr) / PAGE_SIZE); + const auto last_page = + static_cast((query_end - vaddr + PAGE_SIZE - 1) / PAGE_SIZE); + for (auto page = first_page; page < last_page; page++) { + if (info.Protect != expected_old[page]) { + if (fault_path) { + FailFast( + "VirtualProtect fault transition did not match expected protection"); + } + Fatal("invalid protection transition at 0x%016" PRIx64 ", actual=0x%08" PRIx32 + ", expected=0x%08" PRIx32 ", new=0x%08" PRIx32, + vaddr + page * PAGE_SIZE, static_cast(info.Protect), + expected_old[page], protection); + } + } + const auto allocation = reinterpret_cast(info.AllocationBase); + if (host_ranges.empty() || allocation != host_ranges.back().begin) { + host_ranges.push_back({allocation, query_end}); + } else { + host_ranges.back().end = query_end; + } + address = query_end; + } + for (auto range: host_ranges) { + range.begin = std::max(range.begin, vaddr); + DWORD old_protection = 0; + const auto first_page = static_cast((range.begin - vaddr) / PAGE_SIZE); + if (VirtualProtect(reinterpret_cast(static_cast(range.begin)), + range.end - range.begin, protection, &old_protection) == 0 || + old_protection != expected_old[first_page]) { + if (fault_path) { + FailFast("VirtualProtect fault transition did not match expected protection"); + } + Fatal("invalid protection transition at 0x%016" PRIx64 ", old=0x%08" PRIx32 + ", expected=0x%08" PRIx32 ", new=0x%08" PRIx32, + range.begin, static_cast(old_protection), expected_old[first_page], + protection); } - Fatal("invalid protection transition at 0x%016" PRIx64 ", old=0x%08" PRIx32 - ", expected=0x%08" PRIx32 ", new=0x%08" PRIx32, - vaddr, static_cast(old_protection), expected_old, protection); } #elif defined(__APPLE__) // mprotect cannot report the previous protection, so the expected_old comparison // is dropped; the tracker is the sole mutator of these pages and drives the // transition from its own shadow state. (void)expected_old; - if (mprotect(reinterpret_cast(static_cast(vaddr)), PAGE_SIZE, + if (mprotect(reinterpret_cast(static_cast(vaddr)), size, PageProtToPosix(protection)) != 0) { if (fault_path) { FailFast("mprotect fault transition failed"); @@ -510,16 +614,18 @@ struct PageManager::Impl { Fatal("mprotect failed at 0x%016" PRIx64 ", new=0x%08" PRIx32, vaddr, protection); } #else - if (page.current_protection != UNKNOWN_PROTECTION && - page.current_protection != expected_old) { - if (fault_path) { - FailFast("mprotect fault transition did not match expected protection"); + for (size_t i = 0; i < pages.size(); i++) { + const auto actual = pages[i]->current_protection; + if (actual != UNKNOWN_PROTECTION && actual != expected_old[i]) { + if (fault_path) { + FailFast("mprotect fault transition did not match expected protection"); + } + Fatal("invalid protection transition at 0x%016" PRIx64 ", old=0x%08" PRIx32 + ", expected=0x%08" PRIx32 ", new=0x%08" PRIx32, + vaddr + i * PAGE_SIZE, actual, expected_old[i], protection); } - Fatal("invalid protection transition at 0x%016" PRIx64 ", old=0x%08" PRIx32 - ", expected=0x%08" PRIx32 ", new=0x%08" PRIx32, - vaddr, page.current_protection, expected_old, protection); } - if (::mprotect(reinterpret_cast(static_cast(vaddr)), PAGE_SIZE, + if (::mprotect(reinterpret_cast(static_cast(vaddr)), size, ToHostProtection(protection)) != 0) { if (fault_path) { FailFast("mprotect failed on the fault path"); @@ -527,10 +633,19 @@ struct PageManager::Impl { Fatal("mprotect failed at 0x%016" PRIx64 ", new=0x%08" PRIx32 " (%s)", vaddr, protection, std::strerror(errno)); } - page.current_protection = protection; + for (auto* page: pages) { + page->current_protection = protection; + } #endif } + static void Protect(PageState& page, uint64_t vaddr, uint32_t protection, uint32_t expected_old, + bool fault_path) noexcept { + PageState* pages[] = {&page}; + uint32_t expected[] = {expected_old}; + ProtectRange(pages, vaddr, protection, expected, fault_path); + } + std::unique_ptr[]> regions; std::vector> region_storage; std::mutex region_mutex; @@ -634,65 +749,134 @@ void PageManager::UpdatePageWatchers(bool track, uint64_t vaddr, uint64_t size, if (mode != PageWatchMode::Write && mode != PageWatchMode::ReadWrite) { Fatal("invalid watcher mode"); } - const auto end = PageEnd(vaddr, size); - for (auto page_vaddr = PageStart(vaddr); page_vaddr < end; page_vaddr += PAGE_SIZE) { - auto* region = - track ? m_impl->GetOrCreateRegion(page_vaddr) : m_impl->FindRegion(page_vaddr); + const auto begin = PageStart(vaddr); + const auto end = PageEnd(vaddr, size); + for (auto chunk_begin = begin; chunk_begin < end;) { + const auto chunk_end = std::min(end, (chunk_begin / REGION_SIZE + 1) * REGION_SIZE); + auto* region = + track ? m_impl->GetOrCreateRegion(chunk_begin) : m_impl->FindRegion(chunk_begin); if (region == nullptr) { - Fatal("untracking unknown page 0x%016" PRIx64, page_vaddr); + Fatal("untracking unknown page 0x%016" PRIx64, chunk_begin); } - auto& page = m_impl->GetPage(*region, page_vaddr); - SpinGuard lock(page.lock); - if (page.resolving && track) { - FailFast("new page watcher raced active fault resolution"); + + const auto page_count = static_cast((chunk_end - chunk_begin) / PAGE_SIZE); + std::vector pages; + pages.reserve(page_count); + for (auto address = chunk_begin; address < chunk_end; address += PAGE_SIZE) { + pages.push_back(&m_impl->GetPage(*region, address)); } - if (page.mappings == 0) { - Fatal("watching unmapped page 0x%016" PRIx64, page_vaddr); + Impl::PageRangeGuard lock(pages); + + std::vector first_watchers(page_count); + for (size_t i = 0; i < page_count; i++) { + auto& page = *pages[i]; + const auto address = chunk_begin + i * PAGE_SIZE; + if (page.resolving && track) { + FailFast("new page watcher raced active fault resolution"); + } + if (page.mappings == 0) { + Fatal("watching unmapped page 0x%016" PRIx64, address); + } + auto& watchers = + (mode == PageWatchMode::ReadWrite ? page.access_watchers : page.write_watchers); + if (track) { + if (watchers == std::numeric_limits::max()) { + Fatal("watcher overflow at 0x%016" PRIx64, address); + } + first_watchers[i] = page.write_watchers == 0 && page.access_watchers == 0; + } else { + if (watchers == 0) { + Fatal("watcher underflow at 0x%016" PRIx64, address); + } + if (page.backing_writer != 0 && page.backing_writer != CurrentThread()) { + Fatal("backing write ownership changed at 0x%016" PRIx64, address); + } + } } - auto& watchers = - (mode == PageWatchMode::ReadWrite ? page.access_watchers : page.write_watchers); + if (track) { - if (watchers == std::numeric_limits::max()) { - Fatal("watcher overflow at 0x%016" PRIx64, page_vaddr); - } - const bool first_watcher = page.write_watchers == 0 && page.access_watchers == 0; - if (first_watcher) { - page.original_protection = Impl::QueryProtection(page, page_vaddr); - } - const auto old_protection = Impl::WatcherProtection(page); - watchers++; - const auto new_protection = Impl::WatcherProtection(page); - if (new_protection != old_protection) { - Impl::Protect(page, page_vaddr, new_protection, old_protection, false); - } - switch (new_protection) { - case NO_ACCESS_PROTECTION: - page.late_read_pending = false; - page.late_write_pending = false; - break; - case READ_ONLY_PROTECTION: page.late_write_pending = false; break; - default: break; - } - } else { - if (watchers == 0) { - Fatal("watcher underflow at 0x%016" PRIx64, page_vaddr); - } - if (page.backing_writer != 0 && page.backing_writer != CurrentThread()) { - Fatal("backing write ownership changed at 0x%016" PRIx64, page_vaddr); - } - const auto old_protection = Impl::WatcherProtection(page); - watchers--; - const auto new_protection = Impl::WatcherProtection(page); - if (page.backing_writer == 0 && new_protection != old_protection) { - Impl::Protect(page, page_vaddr, new_protection, old_protection, false); - } - if (page.backing_writer == 0) { - Impl::PublishDelayedFaults(page, old_protection, new_protection); - } - if (page.backing_writer == 0 && page.write_watchers == 0 && page.access_watchers == 0) { - page.original_protection = 0; + for (size_t first = 0; first < page_count;) { + while (first < page_count && first_watchers[first] == 0) { + first++; + } + auto last = first; + while (last < page_count && first_watchers[last] != 0) { + last++; + } + if (first != last) { + Impl::ValidateInitialProtection(std::span {pages}.subspan(first, last - first), + chunk_begin + first * PAGE_SIZE); + } + first = last; } } + + std::vector old_protections(page_count); + std::vector new_protections(page_count); + std::vector transitions(page_count); + for (size_t i = 0; i < page_count; i++) { + auto& page = *pages[i]; + auto& watchers = + (mode == PageWatchMode::ReadWrite ? page.access_watchers : page.write_watchers); + const auto old_protection = Impl::WatcherProtection(page); + if (track) { + watchers++; + } else { + watchers--; + } + const auto new_protection = Impl::WatcherProtection(page); + old_protections[i] = old_protection; + new_protections[i] = new_protection; + if (new_protection != old_protection && (track || page.backing_writer == 0)) { + transitions[i] = 1; + } + } + + for (size_t first = 0; first < page_count;) { + while (first < page_count && transitions[first] == 0) { + first++; + } + if (first == page_count) { + break; + } + const auto protection = new_protections[first]; + auto current = first + 1; + auto last = current; + for (; current < page_count && new_protections[current] == protection; current++) { + if (old_protections[current] != new_protections[current] && + transitions[current] == 0) { + break; + } + if (transitions[current] != 0) { + last = current + 1; + } + } + Impl::ProtectRange(std::span {pages}.subspan(first, last - first), + chunk_begin + first * PAGE_SIZE, protection, + std::span {old_protections}.subspan(first, last - first), false); + first = current; + } + + for (size_t i = 0; i < page_count; i++) { + auto& page = *pages[i]; + const auto protection = new_protections[i]; + if (track) { + switch (protection) { + case NO_ACCESS_PROTECTION: + page.late_read_pending = false; + page.late_write_pending = false; + break; + case READ_ONLY_PROTECTION: page.late_write_pending = false; break; + default: break; + } + } else if (page.backing_writer == 0) { + Impl::PublishDelayedFaults(page, old_protections[i], protection); + if (page.write_watchers == 0 && page.access_watchers == 0) { + page.original_protection = 0; + } + } + } + chunk_begin = chunk_end; } } diff --git a/src/graphics/host_gpu/renderer/cache/textureCache.cpp b/src/graphics/host_gpu/renderer/cache/textureCache.cpp index 1a59369ac..034f82be6 100644 --- a/src/graphics/host_gpu/renderer/cache/textureCache.cpp +++ b/src/graphics/host_gpu/renderer/cache/textureCache.cpp @@ -7,13 +7,13 @@ #include "graphics/guest_gpu/gpu_format.h" #include "graphics/guest_gpu/tile.h" #include "graphics/host_gpu/graphicContext.h" -#include "graphics/host_gpu/renderer/image/textureCommon.h" #include "graphics/host_gpu/renderer/cache/bufferCache.h" +#include "graphics/host_gpu/renderer/cache/resourceMutex.h" #include "graphics/host_gpu/renderer/commandScheduler.h" #include "graphics/host_gpu/renderer/image/imageView.h" -#include "graphics/host_gpu/renderer/render.h" -#include "graphics/host_gpu/renderer/cache/resourceMutex.h" +#include "graphics/host_gpu/renderer/image/textureCommon.h" #include "graphics/host_gpu/renderer/image/tiler.h" +#include "graphics/host_gpu/renderer/render.h" #include "kernel/memory.h" #include @@ -58,8 +58,8 @@ private: TextureCache::TextureCache(GraphicContext& graphics, CommandScheduler& scheduler, PageManager& page_manager, BufferCache& buffer_cache, ResourceMutex& resource_mutex) - : m_graphics(graphics), m_scheduler(scheduler), - m_memory_tracker(page_manager, PageWatchMode::Write), m_blit_helper(graphics, scheduler), + : m_graphics(graphics), m_scheduler(scheduler), m_page_manager(page_manager), + m_blit_helper(graphics, scheduler), m_tiler(std::make_unique(graphics, scheduler, buffer_cache.GetUtilityBuffer(MemoryUsage::Stream))), m_buffer_cache(buffer_cache), m_resource_mutex(resource_mutex), @@ -80,7 +80,7 @@ TextureCache::TextureCache(GraphicContext& graphics, CommandScheduler& scheduler TextureCache::~TextureCache() { for (uint32_t index = 0; index < m_slots.size(); index++) { if (m_slots[index].image != nullptr && m_slots[index].image->registered) { - UnregisterImage({index, m_slots[index].generation}, false); + UnregisterImage({index, m_slots[index].generation}); } m_slots[index].image.reset(); } @@ -117,8 +117,7 @@ bool TextureCache::SafeToDownload(const Image& image) { return false; } const auto range = image.info.data; - return !m_buffer_cache.HasGpuDirtyBytes(range.address, range.size) && - !m_memory_tracker.IsRegionCpuModified(range.address, range.size); + return !m_buffer_cache.HasGpuDirtyBytes(range.address, range.size); } Image& TextureCache::ResolveImage(ImageId id) { @@ -187,21 +186,17 @@ void TextureCache::RegisterImage(ImageId id) { m_total_used_memory += image.AccountedSize(); } -void TextureCache::UnregisterImage(ImageId id, bool release_tracking) { +void TextureCache::UnregisterImage(ImageId id) { auto& image = ResolveImage(id); if (!image.registered) { return; } + UntrackImage(id); std::vector releases; if (!m_image_owner_index.Unregister(id, releases)) { EXIT("TextureCache: image missing from owner index\n"); } m_lru_cache.Free(image.lru_id); - if (release_tracking) { - for (const auto& range: releases) { - m_memory_tracker.UntrackMemory(range.address, range.size); - } - } const auto accounted = image.AccountedSize(); if (accounted > m_total_used_memory) { EXIT("TextureCache: image accounting underflow\n"); @@ -210,7 +205,7 @@ void TextureCache::UnregisterImage(ImageId id, bool release_tracking) { image.registered = false; } -void TextureCache::DeleteImage(ImageId id, bool release_tracking) { +void TextureCache::DeleteImage(ImageId id) { auto owner = ResolveOwner(id); if (owner == nullptr || !owner->registered) { return; @@ -224,7 +219,7 @@ void TextureCache::DeleteImage(ImageId id, bool release_tracking) { } } for (const auto association: associations) { - ReleaseGpuTracking(association); + ClearGpuModified(association); DeleteImage(association); } } @@ -235,7 +230,7 @@ void TextureCache::DeleteImage(ImageId id, bool release_tracking) { if (owner->info.metadata.kind == ImageMetadataKind::Htile) { m_surface_metas.erase(owner->info.metadata.range.address); } - UnregisterImage(id, release_tracking); + UnregisterImage(id); const auto erase_slot = [this, id, retained = owner] { auto& slot = m_slots[id.index]; if (slot.generation != id.generation || slot.image != retained) { @@ -266,10 +261,10 @@ void TextureCache::DeleteImages(std::span ids, continue; } if (native_source == id) { - ReleaseGpuTracking(id); + ClearGpuModified(id); } else if (owner->IsGpuModified()) { DownloadImage(id); - ReleaseGpuTracking(id); + ClearGpuModified(id); } DeleteImage(id); } @@ -296,6 +291,121 @@ void TextureCache::TouchImage(Image& image) { } } +void TextureCache::TrackImage(ImageId id) { + auto& image = ResolveImage(id); + if (!image.registered) { + return; + } + const auto image_begin = image.info.data.address; + const auto image_end = image.info.data.End(); + if (image_begin == image.track_addr && image_end == image.track_addr_end) { + return; + } + if (!image.IsTracked()) { + image.track_addr = image_begin; + image.track_addr_end = image_end; + m_page_manager.UpdatePageWatchers(true, image_begin, image.info.data.size); + return; + } + if (image_begin < image.track_addr) { + TrackImageHead(id); + } + if (image.track_addr_end < image_end) { + TrackImageTail(id); + } +} + +void TextureCache::TrackImageHead(ImageId id) { + auto& image = ResolveImage(id); + if (!image.registered) { + return; + } + const auto image_begin = image.info.data.address; + if (image_begin == image.track_addr) { + return; + } + if (!image.IsTracked() || image_begin > image.track_addr) { + EXIT("TextureCache: invalid image head tracking range\n"); + } + const auto size = image.track_addr - image_begin; + image.track_addr = image_begin; + m_page_manager.UpdatePageWatchers(true, image_begin, size); +} + +void TextureCache::TrackImageTail(ImageId id) { + auto& image = ResolveImage(id); + if (!image.registered) { + return; + } + const auto image_end = image.info.data.End(); + if (image_end == image.track_addr_end) { + return; + } + if (!image.IsTracked() || image.track_addr_end > image_end) { + EXIT("TextureCache: invalid image tail tracking range\n"); + } + const auto address = image.track_addr_end; + const auto size = image_end - address; + image.track_addr_end = image_end; + m_page_manager.UpdatePageWatchers(true, address, size); +} + +void TextureCache::UntrackImage(ImageId id) { + auto& image = ResolveImage(id); + if (!image.IsTracked()) { + return; + } + const auto address = image.track_addr; + const auto size = image.track_addr_end - image.track_addr; + image.track_addr = 0; + image.track_addr_end = 0; + if (size != 0) { + m_page_manager.UpdatePageWatchers(false, address, size); + } +} + +void TextureCache::UntrackImageHead(ImageId id) { + auto& image = ResolveImage(id); + const auto begin = image.info.data.address; + if (!image.IsTracked() || begin < image.track_addr) { + return; + } + const auto address = (begin + TRACKER_PAGE_SIZE) & ~(TRACKER_PAGE_SIZE - 1); + const auto size = address - begin; + image.track_addr = address; + if (image.track_addr == image.track_addr_end) { + image.MarkMaybeCpuDirty(); + if (image.NeedsMaybeCpuHash()) { + image.SetMaybeCpuHash(image.HashGuestEdges()); + } + UntrackImage(id); + } + if (size != 0) { + m_page_manager.UpdatePageWatchers(false, begin, size); + } +} + +void TextureCache::UntrackImageTail(ImageId id) { + auto& image = ResolveImage(id); + const auto end = image.info.data.End(); + if (!image.IsTracked() || image.track_addr_end < end) { + return; + } + const auto address = end & ~(TRACKER_PAGE_SIZE - 1); + const auto size = end - address; + image.track_addr_end = address; + if (image.track_addr == image.track_addr_end) { + image.MarkMaybeCpuDirty(); + if (image.NeedsMaybeCpuHash()) { + image.SetMaybeCpuHash(image.HashGuestEdges()); + } + UntrackImage(id); + } + if (size != 0) { + m_page_manager.UpdatePageWatchers(false, address, size); + } +} + void TextureCache::TrackImageDownload(ImageId id) { std::lock_guard transaction(m_resource_mutex); CacheLock lock(*this, m_lock); @@ -376,9 +486,6 @@ void TextureCache::ValidateImageDesc(const ImageDesc& desc) const { } void TextureCache::PrepareImageCopy(Image& image) { - const auto range = image.info.data; - m_memory_tracker.ForEachUploadRange( - range.address, range.size, false, [](uint64_t, uint64_t) noexcept {}, []() noexcept {}); if (image.IsCpuDirty()) { image.RefreshComplete(); } @@ -471,6 +578,7 @@ void TextureCache::CopyImage(ImageId destination_id, ImageId source_id) { RefreshCopySource(source_id); auto& destination = ResolveImage(destination_id); auto& source = ResolveImage(source_id); + TrackImage(destination_id); if (source.backing.samples != destination.backing.samples) { EXIT("TextureCache: cannot issue an unequal-sample image copy\n"); } @@ -479,7 +587,6 @@ void TextureCache::CopyImage(ImageId destination_id, ImageId source_id) { if (source.info.data == destination.info.data) { destination.MarkBufferModified(); } - RestoreGpuTracking(destination); return; } const bool source_depth = source.info.IsDepth(); @@ -503,7 +610,6 @@ void TextureCache::CopyImage(ImageId destination_id, ImageId source_id) { destination.MarkGpuModified(); } destination.ClearBufferModified(); - RestoreGpuTracking(destination); } void TextureCache::CopyImageMip(ImageId destination_id, ImageId source_id, uint32_t mip, @@ -511,6 +617,7 @@ void TextureCache::CopyImageMip(ImageId destination_id, ImageId source_id, uint3 RefreshCopySource(source_id); auto& destination = ResolveImage(destination_id); auto& source = ResolveImage(source_id); + TrackImage(destination_id); if (source.IsBufferModified() || source.backing.samples != destination.backing.samples) { EXIT("TextureCache: invalid mip-copy ownership or sample count\n"); } @@ -520,7 +627,6 @@ void TextureCache::CopyImageMip(ImageId destination_id, ImageId source_id, uint3 if (source.IsGpuModified()) { destination.MarkGpuModified(); } - RestoreGpuTracking(destination); } ImageId TextureCache::ResolveDepthOverlap(const ImageInfo& requested, BindingType binding, @@ -590,7 +696,7 @@ ImageId TextureCache::ResolveDepthOverlap(const ImageInfo& requested, BindingTyp if (copied) { DeleteImages(std::array {cached_id}, cached_id); } else { - ReleaseGpuTracking(cached_id); + ClearGpuModified(cached_id); DeleteImage(cached_id); } return replacement_id; @@ -903,39 +1009,29 @@ void TextureCache::InitializeImage(ImageId id, const ImageDesc& desc) { if (image.info.data.Empty()) { return; } + TrackImage(id); if (image.info.metadata.compression != VideoOutCompression::Uncompressed) { - m_memory_tracker.ForEachUploadRange( - image.info.data.address, image.info.data.size, false, - [](uint64_t, uint64_t) noexcept {}, []() noexcept {}); if (image.IsCpuDirty()) { image.RefreshComplete(); } return; } if (image.info.samples > 1) { - RestoreGpuTracking(image); return; } - bool data_gpu_owned = false; - bool data_imported = false; - bool uploaded = false; - m_memory_tracker.ForEachUploadRange( - image.info.data.address, image.info.data.size, false, - [&](uint64_t, uint64_t) noexcept { uploaded = true; }, - [&]() noexcept { - uploaded |= image.IsBufferModified() || image.IsDefinitelyCpuDirty(); - if (!uploaded) { - return; - } - const auto source = - m_buffer_cache.ObtainBufferForImage(image.info.data.address, image.info.data.size); - if (source.buffer == nullptr) { - EXIT("TextureCache: failed to obtain image upload source\n"); - } - data_gpu_owned |= source.gpu_owned; - data_imported = true; - UploadImage(image, desc, *source.buffer, source.offset); - }); + bool data_gpu_owned = false; + bool data_imported = false; + const bool upload = image.IsBufferModified() || image.IsCpuDirty(); + if (upload) { + const auto source = + m_buffer_cache.ObtainBufferForImage(image.info.data.address, image.info.data.size); + if (source.buffer == nullptr) { + EXIT("TextureCache: failed to obtain image upload source\n"); + } + data_gpu_owned |= source.gpu_owned; + data_imported = true; + UploadImage(image, desc, *source.buffer, source.offset); + } if (data_imported) { image.ClearBufferModified(); } @@ -945,39 +1041,27 @@ void TextureCache::InitializeImage(ImageId id, const ImageDesc& desc) { if (image.IsCpuDirty()) { image.RefreshComplete(); } - RestoreGpuTracking(image); } void TextureCache::RefreshImage(ImageId id, const ImageDesc& desc) { - auto& image = ResolveImage(id); - bool unchanged_maybe = false; + TrackImage(id); + auto& image = ResolveImage(id); if (image.IsMaybeCpuDirty()) { const auto hash = image.HashGuestEdges(); if (image.NeedsMaybeCpuHash()) { image.SetMaybeCpuHash(hash); return; } - unchanged_maybe = !image.ResolveMaybeCpuHash(hash); - if (unchanged_maybe) { - m_memory_tracker.ForEachUploadRange( - image.info.data.address, image.info.data.size, false, - [](uint64_t, uint64_t) noexcept {}, []() noexcept {}); - } + (void)image.ResolveMaybeCpuHash(hash); } bool cpu_dirty = image.IsBufferModified() || image.IsDefinitelyCpuDirty(); - if (!unchanged_maybe) { - cpu_dirty |= - m_memory_tracker.IsRegionCpuModified(image.info.data.address, image.info.data.size); - } if (image.info.metadata.compression != VideoOutCompression::Uncompressed) { if (cpu_dirty) { EXIT("TextureCache: compressed guest image refresh is unsupported\n"); } - RestoreGpuTracking(image); return; } if (!cpu_dirty) { - RestoreGpuTracking(image); return; } InitializeImage(id, desc); @@ -1029,7 +1113,7 @@ ImageId TextureCache::FindImage(ImageDesc& desc, bool exact_format) { ImageId result {}; bool replacement_buffer = false; - bool replacing_image = false; + bool inserted_new = false; { std::lock_guard transaction(m_resource_mutex); CacheLock lock(*this, m_lock); @@ -1079,20 +1163,19 @@ ImageId TextureCache::FindImage(ImageDesc& desc, bool exact_format) { } replacement_buffer = resolved.IsBufferModified(); DeleteImage(result); - result = {}; - replacing_image = true; + result = {}; } } if (!result) { result = InsertImage(desc.info); + inserted_new = true; auto& inserted = ResolveImage(result); if (replacement_buffer || m_buffer_cache.HasGpuDirtyBytes(inserted.info.data.address, inserted.info.data.size)) { inserted.MarkBufferModified(); - } else if (replacing_image) { - m_memory_tracker.MarkRegionAsCpuModified(inserted.info.data.address, - inserted.info.data.size); } + } + if (inserted_new) { InitializeImage(result, desc); } else { RefreshImage(result, desc); @@ -1101,9 +1184,7 @@ ImageId TextureCache::FindImage(ImageDesc& desc, bool exact_format) { auto& image = ResolveImage(result); if (desc.type == BindingType::VideoOut && desc.info.metadata.compression != VideoOutCompression::Uncompressed) { - const bool guest_dirty = - image.IsBufferModified() || image.IsCpuDirty() || - m_memory_tracker.IsRegionCpuModified(image.info.data.address, image.info.data.size); + const bool guest_dirty = image.IsBufferModified() || image.IsCpuDirty(); const bool native_current = (image.usage.render_target || image.IsGpuModified()) && !guest_dirty; if (!native_current) { @@ -1252,6 +1333,7 @@ void TextureCache::MarkGpuWritten(ImageId id) { if (!image.registered || image.depth_id) { EXIT("TextureCache: cannot mark an unavailable image GPU-written\n"); } + TrackImage(id); CommitGpuWrite(image); } @@ -1265,13 +1347,10 @@ void TextureCache::CommitGpuWrite(Image& image) { } m_buffer_cache.InvalidateImageAliases(range.address, range.size); image.ClearBufferModified(); - m_memory_tracker.ForEachUploadRange( - range.address, range.size, true, [](uint64_t, uint64_t) noexcept {}, []() noexcept {}); if (image.IsCpuDirty()) { image.RefreshComplete(); } image.MarkGpuModified(); - RestoreGpuTracking(image); } bool TextureCache::ClearImageFromBuffer(CommandBuffer& command, uint64_t address, uint64_t size, @@ -1335,8 +1414,7 @@ bool TextureCache::ClearImageFromBuffer(CommandBuffer& command, uint64_t address if (m_buffer_cache.HasGpuDirtyBytes(address, size)) { m_buffer_cache.DiscardGpuDirtyBytes(address, size); } - if (image.IsBufferModified() || image.IsCpuDirty() || - m_memory_tracker.IsRegionCpuModified(image.info.data.address, image.info.data.size)) { + if (image.IsBufferModified() || image.IsCpuDirty()) { ImageDesc refresh {.info = image.info, .view_info = {}, .type = UploadBinding(image)}; InitializeImage(selected, refresh); if (image.info.samples == 1 && (image.IsBufferModified() || image.IsCpuDirty())) { @@ -1367,8 +1445,6 @@ void TextureCache::PrepareHostWrite(uint64_t address, uint64_t size) { } CacheLock lock(*this, m_lock); InvalidateCpuAliases(address, size); - m_memory_tracker.ForEachDownloadRange(address, size, [](uint64_t, uint64_t) noexcept {}); - m_memory_tracker.MarkRegionAsCpuModified(address, size); } void TextureCache::DownloadDepth(Image& image, Buffer& destination, uint64_t destination_offset) { @@ -1567,18 +1643,15 @@ bool TextureCache::SynchronizeImageToBuffer(ImageId id) { if (!plan.valid) { return false; } - const auto range = image.info.data; - const bool refresh = image.IsDefinitelyCpuDirty() || - m_memory_tracker.IsRegionCpuModified(range.address, range.size); - if (refresh) { + const auto range = image.info.data; + if (image.IsCpuDirty()) { RefreshImage(id, ImageDesc {.info = image.info, .view_info = {}, .type = UploadBinding(image)}); } if (!image.IsGpuModified()) { return true; } - if (image.IsDefinitelyCpuDirty() || image.IsBufferModified() || - m_memory_tracker.IsRegionCpuModified(range.address, range.size)) { + if (image.IsDefinitelyCpuDirty() || image.IsBufferModified()) { EXIT("TextureCache: image mirror source is not native-current\n"); } auto [destination, offset] = @@ -1591,7 +1664,7 @@ bool TextureCache::SynchronizeImageToBuffer(ImageId id) { m_buffer_cache.PublishImageBuffer(range.address, range.size); image.MarkBufferModified(); RetainImage(m_scheduler.Current(), id); - ReleaseGpuTracking(id); + ClearGpuModified(id); return true; } @@ -1633,7 +1706,7 @@ bool TextureCache::InvalidateMemoryFromGPU(uint64_t address, uint64_t size, if (!formatted_buffer_write) { EXIT("TextureCache: buffer write aliases GPU-modified image\n"); } - ReleaseGpuTracking(id); + ClearGpuModified(id); } owner->MarkBufferModified(); found = true; @@ -1660,50 +1733,40 @@ TextureCache::RegionInfo TextureCache::QueryRegion(uint64_t address, uint64_t si } void TextureCache::InvalidateCpuAliases(uint64_t address, uint64_t size) { + const auto page_begin = address & ~(TRACKER_PAGE_SIZE - 1); + const auto page_end = (address + size + TRACKER_PAGE_SIZE - 1) & ~(TRACKER_PAGE_SIZE - 1); for (const auto id: FindImagesInRegion(address, size, true)) { auto owner = ResolveOwner(id); if (owner == nullptr || owner->depth_id) { continue; } - owner->InvalidateCpuWrite(address, size); - if (owner->NeedsMaybeCpuHash()) { - owner->SetMaybeCpuHash(owner->HashGuestEdges()); + if (owner->Overlaps(address, size)) { + owner->InvalidateCpuWrite(address, size); + UntrackImage(id); + continue; + } + const auto image_begin = owner->info.data.address; + const auto image_end = owner->info.data.End(); + if (page_end < image_end) { + UntrackImageHead(id); + } else if (image_begin < page_begin) { + UntrackImageTail(id); + } else { + owner->MarkMaybeCpuDirty(); + if (owner->NeedsMaybeCpuHash()) { + owner->SetMaybeCpuHash(owner->HashGuestEdges()); + } + UntrackImage(id); } } } -void TextureCache::RestoreGpuTracking(const Image& image) { - if (!image.IsGpuModified()) { - return; - } - constexpr uint64_t page_mask = TRACKER_PAGE_SIZE - 1; - const auto range = image.info.data; - const auto begin = range.address & ~page_mask; - const auto end = (range.End() + page_mask) & ~page_mask; - for (auto page = begin; page < end; page += TRACKER_PAGE_SIZE) { - if (!m_memory_tracker.IsRegionGpuModified(page, TRACKER_PAGE_SIZE) && - !m_memory_tracker.IsRegionCpuModified(page, TRACKER_PAGE_SIZE)) { - m_memory_tracker.MarkRegionAsGpuModified(page, TRACKER_PAGE_SIZE); - } - } -} - -void TextureCache::ReleaseGpuTracking(ImageId id) { +void TextureCache::ClearGpuModified(ImageId id) { auto owner = ResolveOwner(id); if (owner == nullptr || !owner->IsGpuModified()) { return; } - const auto released = owner->info.data; owner->ClearGpuModified(); - m_memory_tracker.ForEachDownloadRange(released.address, released.size, - [](uint64_t, uint64_t) noexcept {}); - for (const auto candidate: FindImagesInRegion(released.address, released.size, true)) { - const auto survivor = ResolveOwner(candidate); - if (survivor != nullptr && survivor.get() != owner.get() && !survivor->depth_id) { - RestoreGpuTracking(*survivor); - } - } - RestoreGpuTracking(*owner); } bool TextureCache::IsMeta(uint64_t address) { @@ -1755,27 +1818,25 @@ bool TextureCache::InvalidateMemory(PageFaultAccess access, uint64_t address, ui return false; } if (phase == PageFaultPhase::Invalidate) { - const bool gpu_image = - m_memory_tracker.InvalidateVirtualGpuWrite(access, address, size, phase); - CpuFaultAction action = gpu_image ? CpuFaultAction::Download - : m_memory_tracker.BeginCpuFault(address, size, access); - { - CacheLock lock(*this, m_lock); + CacheLock lock(*this, m_lock); + const bool tracked = + std::ranges::any_of(FindImagesInRegion(address, size, true), [&](ImageId id) { + const auto owner = ResolveOwner(id); + return owner != nullptr && !owner->depth_id && owner->IsTracked(); + }); + if (tracked) { InvalidateCpuAliases(address, size); } - return action != CpuFaultAction::Untracked; + return tracked; } - - if (phase == PageFaultPhase::Complete) { - const bool gpu_image = m_memory_tracker.IsRegionGpuModified(address, size); - return gpu_image ? m_memory_tracker.InvalidateVirtualGpuWrite(access, address, size, phase) - : m_memory_tracker.CompleteCpuFault(address, size, access, false); - } - if (phase != PageFaultPhase::Release) { + if (phase != PageFaultPhase::Complete && phase != PageFaultPhase::Release) { return false; } - (void)m_memory_tracker.InvalidateVirtualGpuWrite(access, address, size, phase); - return true; + CacheLock lock(*this, m_lock); + return std::ranges::any_of(FindImagesInRegion(address, size, true), [&](ImageId id) { + const auto owner = ResolveOwner(id); + return owner != nullptr && !owner->depth_id; + }); } void TextureCache::UnmapMemory(uint64_t address, uint64_t size) { @@ -1794,16 +1855,10 @@ void TextureCache::UnmapMemory(uint64_t address, uint64_t size) { continue; } if (owner->IsGpuModified()) { - ReleaseGpuTracking(id); + ClearGpuModified(id); } DeleteImage(id); } - m_memory_tracker.UntrackMemory(address, size); - for (const auto id: FindImagesInRegion(address, size, true)) { - if (const auto survivor = ResolveOwner(id); survivor != nullptr) { - RestoreGpuTracking(*survivor); - } - } } void TextureCache::RunGarbageCollector() { @@ -1849,7 +1904,7 @@ void TextureCache::RunGarbageCollector() { if (safe && !TryDownloadImage(id)) { continue; } - ReleaseGpuTracking(id); + ClearGpuModified(id); } DeleteImage(id); if (m_total_used_memory < m_critical_gc_memory && aggressive) { diff --git a/src/graphics/host_gpu/renderer/cache/textureCache.h b/src/graphics/host_gpu/renderer/cache/textureCache.h index c1e40e454..493eb62d1 100644 --- a/src/graphics/host_gpu/renderer/cache/textureCache.h +++ b/src/graphics/host_gpu/renderer/cache/textureCache.h @@ -4,10 +4,11 @@ #include "common/abi.h" #include "common/common.h" #include "common/lruCache.h" -#include "graphics/host_gpu/memoryTracker.h" +#include "graphics/host_gpu/pageManager.h" +#include "graphics/host_gpu/regionManager.h" +#include "graphics/host_gpu/renderer/cache/multiLevelPageTable.h" #include "graphics/host_gpu/renderer/image/blitHelper.h" #include "graphics/host_gpu/renderer/image/image.h" -#include "graphics/host_gpu/renderer/cache/multiLevelPageTable.h" #include #include @@ -109,11 +110,17 @@ private: [[nodiscard]] ImageId InsertImage(const ImageInfo& info); [[nodiscard]] ImageId GetNullImage(const ImageDesc& desc); void RegisterImage(ImageId id); - void UnregisterImage(ImageId id, bool release_tracking); - void DeleteImage(ImageId id, bool release_tracking = true); + void UnregisterImage(ImageId id); + void DeleteImage(ImageId id); void DeleteImages(std::span ids, std::optional native_source = {}); void RetainImage(CommandBuffer& command, ImageId id); void TouchImage(Image& image); + void TrackImage(ImageId id); + void TrackImageHead(ImageId id); + void TrackImageTail(ImageId id); + void UntrackImage(ImageId id); + void UntrackImageHead(ImageId id); + void UntrackImageTail(ImageId id); void TrackImageDownload(ImageId id); void TrackImageDownloadLocked(ImageId id, Image& image); [[nodiscard]] static bool SameBacking(const ImageInfo& cached, const ImageInfo& requested, @@ -148,8 +155,7 @@ private: void ValidateImageDesc(const ImageDesc& desc) const; void InvalidateCpuAliases(uint64_t address, uint64_t size); - void RestoreGpuTracking(const Image& image); - void ReleaseGpuTracking(ImageId id); + void ClearGpuModified(ImageId id); [[nodiscard]] bool SynchronizeImageToBuffer(ImageId id); void DownloadImage(ImageId id); @@ -160,7 +166,7 @@ private: GraphicContext& m_graphics; CommandScheduler& m_scheduler; TrackingSpinLock m_lock; - MemoryTracker m_memory_tracker; + PageManager& m_page_manager; BlitHelper m_blit_helper; std::unique_ptr m_tiler; BufferCache& m_buffer_cache; diff --git a/src/graphics/host_gpu/renderer/image/image.cpp b/src/graphics/host_gpu/renderer/image/image.cpp index 464a0bb54..b025e3db8 100644 --- a/src/graphics/host_gpu/renderer/image/image.cpp +++ b/src/graphics/host_gpu/renderer/image/image.cpp @@ -2,10 +2,10 @@ #include "common/assert.h" #include "common/profiler.h" +#include "graphics/host_gpu/renderer/cache/streamBuffer.h" #include "graphics/host_gpu/renderer/commandScheduler.h" #include "graphics/host_gpu/renderer/image/imageView.h" #include "graphics/host_gpu/renderer/renderTarget.h" -#include "graphics/host_gpu/renderer/cache/streamBuffer.h" #include "kernel/memory.h" #include @@ -99,9 +99,9 @@ vk::ImageAspectFlags Image::FullAspectMask(vk::Format format) noexcept { } } -Image::Barriers Image::GetBarriers(vk::ImageLayout destination_layout, - vk::AccessFlags2 destination_access, - vk::PipelineStageFlags2 destination_stage, +Image::Barriers Image::GetBarriers(vk::ImageLayout destination_layout, + vk::AccessFlags2 destination_access, + vk::PipelineStageFlags2 destination_stage, std::optional range) { auto& state = backing.state; auto& subresource_states = backing.subresource_states; @@ -130,25 +130,25 @@ Image::Barriers Image::GetBarriers(vk::ImageLayout destination_layout, constexpr auto write_access = vk::AccessFlagBits2::eTransferWrite | vk::AccessFlagBits2::eShaderWrite | vk::AccessFlagBits2::eMemoryWrite; - const bool repeated_write = + const bool repeated_write = static_cast(subresource_state.access_mask & write_access); if (subresource_state.layout != destination_layout || subresource_state.access_mask != destination_access || repeated_write) { vk::ImageMemoryBarrier2 barrier {}; - barrier.srcStageMask = subresource_state.pl_stage; - barrier.srcAccessMask = subresource_state.access_mask; - barrier.dstStageMask = destination_stage; - barrier.dstAccessMask = destination_access; - barrier.oldLayout = subresource_state.layout; - barrier.newLayout = destination_layout; - barrier.srcQueueFamilyIndex = VK_QUEUE_FAMILY_IGNORED; - barrier.dstQueueFamilyIndex = VK_QUEUE_FAMILY_IGNORED; - barrier.image = backing.image; - barrier.subresourceRange.aspectMask = FullAspectMask(backing.format); - barrier.subresourceRange.baseMipLevel = level; - barrier.subresourceRange.levelCount = 1; - barrier.subresourceRange.baseArrayLayer = layer; - barrier.subresourceRange.layerCount = 1; + barrier.srcStageMask = subresource_state.pl_stage; + barrier.srcAccessMask = subresource_state.access_mask; + barrier.dstStageMask = destination_stage; + barrier.dstAccessMask = destination_access; + barrier.oldLayout = subresource_state.layout; + barrier.newLayout = destination_layout; + barrier.srcQueueFamilyIndex = VK_QUEUE_FAMILY_IGNORED; + barrier.dstQueueFamilyIndex = VK_QUEUE_FAMILY_IGNORED; + barrier.image = backing.image; + barrier.subresourceRange.aspectMask = FullAspectMask(backing.format); + barrier.subresourceRange.baseMipLevel = level; + barrier.subresourceRange.levelCount = 1; + barrier.subresourceRange.baseArrayLayer = layer; + barrier.subresourceRange.layerCount = 1; barriers.push_back(barrier); subresource_state = {destination_stage, destination_access, destination_layout}; } @@ -159,10 +159,10 @@ Image::Barriers Image::GetBarriers(vk::ImageLayout destination_layout, subresource_states.clear(); } } else { - constexpr auto write_access = vk::AccessFlagBits2::eTransferWrite | - vk::AccessFlagBits2::eShaderWrite | - vk::AccessFlagBits2::eMemoryWrite; - const bool repeated_write = static_cast(state.access_mask & write_access); + constexpr auto write_access = vk::AccessFlagBits2::eTransferWrite | + vk::AccessFlagBits2::eShaderWrite | + vk::AccessFlagBits2::eMemoryWrite; + const bool repeated_write = static_cast(state.access_mask & write_access); if (state.layout == destination_layout && state.access_mask == destination_access && !repeated_write) { return {}; @@ -191,8 +191,7 @@ Image::Barriers Image::GetBarriers(vk::ImageLayout destination_layout, } void Image::Transit(vk::ImageLayout destination_layout, vk::AccessFlags2 destination_access, - std::optional range, - vk::CommandBuffer command_buffer) { + std::optional range, vk::CommandBuffer command_buffer) { const auto transfer_access = vk::AccessFlagBits2::eTransferRead | vk::AccessFlagBits2::eTransferWrite; vk::PipelineStageFlags2 destination_stage {}; @@ -201,8 +200,8 @@ void Image::Transit(vk::ImageLayout destination_layout, vk::AccessFlags2 destina } if (!destination_access || static_cast(destination_access & ~vk::AccessFlags2 {transfer_access})) { - destination_stage |= vk::PipelineStageFlagBits2::eAllGraphics | - vk::PipelineStageFlagBits2::eComputeShader; + destination_stage |= + vk::PipelineStageFlagBits2::eAllGraphics | vk::PipelineStageFlagBits2::eComputeShader; } const auto barriers = GetBarriers(destination_layout, destination_access, destination_stage, range); @@ -218,10 +217,9 @@ void Image::Transit(vk::ImageLayout destination_layout, vk::AccessFlags2 destina command_buffer.pipelineBarrier2(dependency); } -void Image::Upload(std::span copies, vk::Buffer buffer, - uint64_t offset, uint64_t size) { - EXIT_IF(m_scheduler == nullptr || copies.empty() || buffer == nullptr || - size == 0); +void Image::Upload(std::span copies, vk::Buffer buffer, uint64_t offset, + uint64_t size) { + EXIT_IF(m_scheduler == nullptr || copies.empty() || buffer == nullptr || size == 0); m_scheduler->EndRendering(); vk::BufferMemoryBarrier2 buffer_barrier {}; buffer_barrier.srcStageMask = vk::PipelineStageFlagBits2::eAllCommands; @@ -234,16 +232,15 @@ void Image::Upload(std::span copies, vk::Buffer buffe buffer_barrier.offset = offset; buffer_barrier.size = size; const auto image_barriers = - GetBarriers(vk::ImageLayout::eTransferDstOptimal, - vk::AccessFlagBits2::eTransferWrite, + GetBarriers(vk::ImageLayout::eTransferDstOptimal, vk::AccessFlagBits2::eTransferWrite, vk::PipelineStageFlagBits2::eCopy, {}); vk::DependencyInfo dependency {}; - dependency.dependencyFlags = vk::DependencyFlagBits::eByRegion; + dependency.dependencyFlags = vk::DependencyFlagBits::eByRegion; dependency.bufferMemoryBarrierCount = 1; dependency.pBufferMemoryBarriers = &buffer_barrier; - dependency.imageMemoryBarrierCount = static_cast(image_barriers.size()); - dependency.pImageMemoryBarriers = image_barriers.data(); - auto command = m_scheduler->Current().Handle(); + dependency.imageMemoryBarrierCount = static_cast(image_barriers.size()); + dependency.pImageMemoryBarriers = image_barriers.data(); + auto command = m_scheduler->Current().Handle(); command.pipelineBarrier2(dependency); command.copyBufferToImage(buffer, backing.image, vk::ImageLayout::eTransferDstOptimal, static_cast(copies.size()), copies.data()); @@ -256,8 +253,7 @@ void Image::Upload(std::span copies, vk::Buffer buffe dependency.pImageMemoryBarriers = nullptr; command.pipelineBarrier2(dependency); Transit(vk::ImageLayout::eGeneral, - vk::AccessFlagBits2::eShaderRead | vk::AccessFlagBits2::eTransferRead, {}, - command); + vk::AccessFlagBits2::eShaderRead | vk::AccessFlagBits2::eTransferRead, {}, command); } void Image::Download(std::span copies, vk::Buffer buffer, @@ -265,7 +261,7 @@ void Image::Download(std::span copies, vk::Buffer buf EXIT_IF(m_scheduler == nullptr || copies.empty() || buffer == nullptr || size == 0); m_scheduler->EndRendering(); vk::BufferMemoryBarrier2 buffer_barrier {}; - buffer_barrier.srcStageMask = vk::PipelineStageFlagBits2::eAllCommands; + buffer_barrier.srcStageMask = vk::PipelineStageFlagBits2::eAllCommands; buffer_barrier.srcAccessMask = vk::AccessFlagBits2::eMemoryRead | vk::AccessFlagBits2::eMemoryWrite; buffer_barrier.dstStageMask = vk::PipelineStageFlagBits2::eCopy; @@ -276,16 +272,15 @@ void Image::Download(std::span copies, vk::Buffer buf buffer_barrier.offset = offset; buffer_barrier.size = size; const auto image_barriers = - GetBarriers(vk::ImageLayout::eTransferSrcOptimal, - vk::AccessFlagBits2::eTransferRead, + GetBarriers(vk::ImageLayout::eTransferSrcOptimal, vk::AccessFlagBits2::eTransferRead, vk::PipelineStageFlagBits2::eCopy, {}); vk::DependencyInfo dependency {}; - dependency.dependencyFlags = vk::DependencyFlagBits::eByRegion; + dependency.dependencyFlags = vk::DependencyFlagBits::eByRegion; dependency.bufferMemoryBarrierCount = 1; dependency.pBufferMemoryBarriers = &buffer_barrier; - dependency.imageMemoryBarrierCount = static_cast(image_barriers.size()); - dependency.pImageMemoryBarriers = image_barriers.data(); - auto command = m_scheduler->Current().Handle(); + dependency.imageMemoryBarrierCount = static_cast(image_barriers.size()); + dependency.pImageMemoryBarriers = image_barriers.data(); + auto command = m_scheduler->Current().Handle(); command.pipelineBarrier2(dependency); command.copyImageToBuffer(backing.image, vk::ImageLayout::eTransferSrcOptimal, buffer, static_cast(copies.size()), copies.data()); @@ -299,11 +294,11 @@ void Image::Download(std::span copies, vk::Buffer buf command.pipelineBarrier2(dependency); } -std::pair -Image::SanitizeCopyLayers(const Image& source, const Image& destination, uint32_t depth) { - const auto source_type = source.backing.image_type; - const auto destination_type = destination.backing.image_type; - uint32_t source_layers = source.backing.layers; +std::pair Image::SanitizeCopyLayers(const Image& source, + const Image& destination, uint32_t depth) { + const auto source_type = source.backing.image_type; + const auto destination_type = destination.backing.image_type; + uint32_t source_layers = source.backing.layers; uint32_t destination_layers = destination.backing.layers; if (source_type == vk::ImageType::e3D) { source_layers = 1; @@ -312,13 +307,10 @@ Image::SanitizeCopyLayers(const Image& source, const Image& destination, uint32_ destination_layers = 1; } if (source_type == destination_type) { - source_layers = destination_layers = - std::min(source_layers, destination_layers); - } else if (source_type == vk::ImageType::e2D && - destination_type == vk::ImageType::e3D) { + source_layers = destination_layers = std::min(source_layers, destination_layers); + } else if (source_type == vk::ImageType::e2D && destination_type == vk::ImageType::e3D) { source_layers = depth; - } else if (source_type == vk::ImageType::e3D && - destination_type == vk::ImageType::e2D) { + } else if (source_type == vk::ImageType::e3D && destination_type == vk::ImageType::e2D) { destination_layers = depth; } return {source_layers, destination_layers}; @@ -327,12 +319,11 @@ Image::SanitizeCopyLayers(const Image& source, const Image& destination, uint32_ void Image::CopyImage(Image& source) { EXIT_IF(m_scheduler == nullptr || source.backing.samples != backing.samples); m_scheduler->EndRendering(); - const uint32_t levels = - std::min(source.backing.mip_levels, backing.mip_levels); + const uint32_t levels = std::min(source.backing.mip_levels, backing.mip_levels); const uint32_t base_depth = backing.image_type == vk::ImageType::e3D ? backing.extent.depth : source.backing.extent.depth; - const auto source_aspect = + const auto source_aspect = FullAspectMask(source.backing.format) & ~vk::ImageAspectFlagBits::eStencil; const auto destination_aspect = FullAspectMask(backing.format) & ~vk::ImageAspectFlagBits::eStencil; @@ -342,8 +333,7 @@ void Image::CopyImage(Image& source) { const auto width = std::max(source.backing.extent.width >> level, 1u); const auto height = std::max(source.backing.extent.height >> level, 1u); const auto depth = std::max(base_depth >> level, 1u); - const auto [source_layers, destination_layers] = - SanitizeCopyLayers(source, *this, depth); + const auto [source_layers, destination_layers] = SanitizeCopyLayers(source, *this, depth); vk::ImageCopy copy {}; copy.srcSubresource = {source_aspect, level, 0, 1}; copy.dstSubresource = {destination_aspect, level, 0, 1}; @@ -351,8 +341,7 @@ void Image::CopyImage(Image& source) { if (source.backing.image_type == vk::ImageType::e3D) { copy.extent = {width, height, depth}; } else { - copy.srcSubresource.layerCount = - std::min(source_layers, destination_layers); + copy.srcSubresource.layerCount = std::min(source_layers, destination_layers); copy.dstSubresource.layerCount = copy.srcSubresource.layerCount; copy.extent = {width, height, 1}; } @@ -369,34 +358,30 @@ void Image::CopyImage(Image& source) { return; } auto command = m_scheduler->Current().Handle(); - source.Transit(vk::ImageLayout::eTransferSrcOptimal, - vk::AccessFlagBits2::eTransferRead, {}, command); - Transit(vk::ImageLayout::eTransferDstOptimal, - vk::AccessFlagBits2::eTransferWrite, {}, command); - command.copyImage(source.backing.image, vk::ImageLayout::eTransferSrcOptimal, - backing.image, vk::ImageLayout::eTransferDstOptimal, - static_cast(copies.size()), copies.data()); + source.Transit(vk::ImageLayout::eTransferSrcOptimal, vk::AccessFlagBits2::eTransferRead, {}, + command); + Transit(vk::ImageLayout::eTransferDstOptimal, vk::AccessFlagBits2::eTransferWrite, {}, command); + command.copyImage(source.backing.image, vk::ImageLayout::eTransferSrcOptimal, backing.image, + vk::ImageLayout::eTransferDstOptimal, static_cast(copies.size()), + copies.data()); Transit(vk::ImageLayout::eGeneral, - vk::AccessFlagBits2::eShaderRead | vk::AccessFlagBits2::eTransferRead, {}, - command); + vk::AccessFlagBits2::eShaderRead | vk::AccessFlagBits2::eTransferRead, {}, command); } void Image::Resolve(Image& source, const ImageSubresourceRange& source_range, const ImageSubresourceRange& destination_range) { EXIT_IF(m_scheduler == nullptr || backing.samples != 1 || source.backing.image_type != vk::ImageType::e2D || - backing.image_type != vk::ImageType::e2D || - source_range.level_count != 1 || destination_range.level_count != 1 || + backing.image_type != vk::ImageType::e2D || source_range.level_count != 1 || + destination_range.level_count != 1 || source_range.base_level >= source.backing.mip_levels || destination_range.base_level >= backing.mip_levels || source_range.base_layer >= source.backing.layers || destination_range.base_layer >= backing.layers); - const auto layers = std::min( - {source_range.layer_count, destination_range.layer_count, - source.backing.layers - source_range.base_layer, - backing.layers - destination_range.base_layer}); - const auto source_width = - std::max(source.backing.extent.width >> source_range.base_level, 1u); + const auto layers = std::min({source_range.layer_count, destination_range.layer_count, + source.backing.layers - source_range.base_layer, + backing.layers - destination_range.base_layer}); + const auto source_width = std::max(source.backing.extent.width >> source_range.base_level, 1u); const auto source_height = std::max(source.backing.extent.height >> source_range.base_level, 1u); const auto destination_width = @@ -404,43 +389,40 @@ void Image::Resolve(Image& source, const ImageSubresourceRange& source_range, const auto destination_height = std::max(backing.extent.height >> destination_range.base_level, 1u); const bool copy = source.backing.samples == 1; - EXIT_IF(layers == 0 || info.extent.width > source_width || - info.extent.height > source_height || info.extent.width > destination_width || - info.extent.height > destination_height || + EXIT_IF(layers == 0 || info.extent.width > source_width || info.extent.height > source_height || + info.extent.width > destination_width || info.extent.height > destination_height || (copy ? !ImageViewOps::FormatsCompatible(source.backing.format, backing.format) : source.backing.format != backing.format)); - auto resolved_source_range = source_range; - auto resolved_destination_range = destination_range; + auto resolved_source_range = source_range; + auto resolved_destination_range = destination_range; resolved_source_range.layer_count = layers; resolved_destination_range.layer_count = layers; const vk::Extent3D resolve_extent {info.extent.width, info.extent.height, 1}; m_scheduler->EndRendering(); auto command = m_scheduler->Current().Handle(); - source.Transit(vk::ImageLayout::eTransferSrcOptimal, - vk::AccessFlagBits2::eTransferRead, resolved_source_range, command); - Transit(vk::ImageLayout::eTransferDstOptimal, - vk::AccessFlagBits2::eTransferWrite, resolved_destination_range, command); + source.Transit(vk::ImageLayout::eTransferSrcOptimal, vk::AccessFlagBits2::eTransferRead, + resolved_source_range, command); + Transit(vk::ImageLayout::eTransferDstOptimal, vk::AccessFlagBits2::eTransferWrite, + resolved_destination_range, command); if (copy) { vk::ImageCopy region {}; - region.srcSubresource = {vk::ImageAspectFlagBits::eColor, - resolved_source_range.base_level, + region.srcSubresource = {vk::ImageAspectFlagBits::eColor, resolved_source_range.base_level, resolved_source_range.base_layer, layers}; region.dstSubresource = {vk::ImageAspectFlagBits::eColor, resolved_destination_range.base_level, resolved_destination_range.base_layer, layers}; - region.extent = resolve_extent; - command.copyImage(source.backing.image, vk::ImageLayout::eTransferSrcOptimal, - backing.image, vk::ImageLayout::eTransferDstOptimal, region); + region.extent = resolve_extent; + command.copyImage(source.backing.image, vk::ImageLayout::eTransferSrcOptimal, backing.image, + vk::ImageLayout::eTransferDstOptimal, region); } else { vk::ImageResolve region {}; - region.srcSubresource = {vk::ImageAspectFlagBits::eColor, - resolved_source_range.base_level, + region.srcSubresource = {vk::ImageAspectFlagBits::eColor, resolved_source_range.base_level, resolved_source_range.base_layer, layers}; region.dstSubresource = {vk::ImageAspectFlagBits::eColor, resolved_destination_range.base_level, resolved_destination_range.base_layer, layers}; - region.extent = resolve_extent; + region.extent = resolve_extent; command.resolveImage(source.backing.image, vk::ImageLayout::eTransferSrcOptimal, backing.image, vk::ImageLayout::eTransferDstOptimal, region); } @@ -454,22 +436,21 @@ uint32_t Image::CopyRows(uint64_t row_size, uint32_t rows, uint64_t capacity) no } void Image::CopyImageWithBuffer(Image& source, Buffer& buffer) { - EXIT_IF(m_scheduler == nullptr || buffer.Handle() == nullptr || - source.backing.samples != 1 || backing.samples != 1); + EXIT_IF(m_scheduler == nullptr || buffer.Handle() == nullptr || source.backing.samples != 1 || + backing.samples != 1); m_scheduler->EndRendering(); - const uint32_t levels = - std::min(source.backing.mip_levels, backing.mip_levels); - const auto source_aspect = + const uint32_t levels = std::min(source.backing.mip_levels, backing.mip_levels); + const auto source_aspect = FullAspectMask(source.backing.format) & ~vk::ImageAspectFlagBits::eStencil; const auto destination_aspect = FullAspectMask(backing.format) & ~vk::ImageAspectFlagBits::eStencil; - const auto source_bytes = DepthAspectTransferBytes(source.backing.format) != 0 - ? DepthAspectTransferBytes(source.backing.format) - : source.info.bytes_per_block; - const auto destination_bytes = DepthAspectTransferBytes(backing.format) != 0 - ? DepthAspectTransferBytes(backing.format) - : info.bytes_per_block; - const uint32_t source_block = source.info.IsBlock() ? 4u : 1u; + const auto source_bytes = DepthAspectTransferBytes(source.backing.format) != 0 + ? DepthAspectTransferBytes(source.backing.format) + : source.info.bytes_per_block; + const auto destination_bytes = DepthAspectTransferBytes(backing.format) != 0 + ? DepthAspectTransferBytes(backing.format) + : info.bytes_per_block; + const uint32_t source_block = source.info.IsBlock() ? 4u : 1u; const uint32_t destination_block = info.IsBlock() ? 4u : 1u; EXIT_IF(levels == 0 || source_bytes == 0 || source_bytes != destination_bytes || source_block != destination_block); @@ -484,74 +465,66 @@ void Image::CopyImageWithBuffer(Image& source, Buffer& buffer) { barrier.buffer = buffer.Handle(); barrier.offset = 0; vk::DependencyInfo dependency {}; - dependency.dependencyFlags = vk::DependencyFlagBits::eByRegion; + dependency.dependencyFlags = vk::DependencyFlagBits::eByRegion; dependency.bufferMemoryBarrierCount = 1; dependency.pBufferMemoryBarriers = &barrier; - auto command = m_scheduler->Current().Handle(); - source.Transit(vk::ImageLayout::eTransferSrcOptimal, - vk::AccessFlagBits2::eTransferRead, {}, command); - Transit(vk::ImageLayout::eTransferDstOptimal, - vk::AccessFlagBits2::eTransferWrite, {}, command); + auto command = m_scheduler->Current().Handle(); + source.Transit(vk::ImageLayout::eTransferSrcOptimal, vk::AccessFlagBits2::eTransferRead, {}, + command); + Transit(vk::ImageLayout::eTransferDstOptimal, vk::AccessFlagBits2::eTransferWrite, {}, command); for (uint32_t level = 0; level < levels; level++) { - const auto width = std::max(source.backing.extent.width >> level, 1u); - const auto height = std::max(source.backing.extent.height >> level, 1u); - const auto source_depth = source.backing.image_type == vk::ImageType::e3D - ? std::max(source.backing.extent.depth >> level, 1u) - : source.backing.layers; + const auto width = std::max(source.backing.extent.width >> level, 1u); + const auto height = std::max(source.backing.extent.height >> level, 1u); + const auto source_depth = source.backing.image_type == vk::ImageType::e3D + ? std::max(source.backing.extent.depth >> level, 1u) + : source.backing.layers; const auto destination_depth = backing.image_type == vk::ImageType::e3D ? std::max(backing.extent.depth >> level, 1u) : backing.layers; - const auto slices = std::min(source_depth, destination_depth); - const auto block_rows = (height + source_block - 1) / source_block; + const auto slices = std::min(source_depth, destination_depth); + const auto block_rows = (height + source_block - 1) / source_block; const auto row_size = static_cast((width + source_block - 1) / source_block) * source_bytes; const auto rows_per_copy = CopyRows(row_size, block_rows, buffer.Size()); EXIT_IF(slices == 0 || rows_per_copy == 0); for (uint32_t slice = 0; slice < slices; slice++) { - for (uint32_t block_row = 0; block_row < block_rows; - block_row += rows_per_copy) { - const auto copy_rows = std::min(rows_per_copy, block_rows - block_row); - const auto y = block_row * source_block; - const auto copy_height = - std::min(copy_rows * source_block, height - y); - const auto copy_size = row_size * copy_rows; + for (uint32_t block_row = 0; block_row < block_rows; block_row += rows_per_copy) { + const auto copy_rows = std::min(rows_per_copy, block_rows - block_row); + const auto y = block_row * source_block; + const auto copy_height = std::min(copy_rows * source_block, height - y); + const auto copy_size = row_size * copy_rows; vk::BufferImageCopy source_copy {}; source_copy.imageSubresource = { source_aspect, level, source.backing.image_type == vk::ImageType::e3D ? 0u : slice, 1}; - source_copy.imageOffset = { - 0, static_cast(y), - source.backing.image_type == vk::ImageType::e3D - ? static_cast(slice) - : 0}; - source_copy.imageExtent = {width, copy_height, 1}; - auto destination_copy = source_copy; + source_copy.imageOffset = {0, static_cast(y), + source.backing.image_type == vk::ImageType::e3D + ? static_cast(slice) + : 0}; + source_copy.imageExtent = {width, copy_height, 1}; + auto destination_copy = source_copy; destination_copy.imageSubresource = { destination_aspect, level, backing.image_type == vk::ImageType::e3D ? 0u : slice, 1}; destination_copy.imageOffset.z = - backing.image_type == vk::ImageType::e3D - ? static_cast(slice) - : 0; + backing.image_type == vk::ImageType::e3D ? static_cast(slice) : 0; barrier.size = copy_size; barrier.srcAccessMask = vk::AccessFlagBits2::eTransferRead; barrier.dstAccessMask = vk::AccessFlagBits2::eTransferWrite; command.pipelineBarrier2(dependency); command.copyImageToBuffer(source.backing.image, - vk::ImageLayout::eTransferSrcOptimal, - buffer.Handle(), source_copy); + vk::ImageLayout::eTransferSrcOptimal, buffer.Handle(), + source_copy); barrier.srcAccessMask = vk::AccessFlagBits2::eTransferWrite; barrier.dstAccessMask = vk::AccessFlagBits2::eTransferRead; command.pipelineBarrier2(dependency); command.copyBufferToImage(buffer.Handle(), backing.image, - vk::ImageLayout::eTransferDstOptimal, - destination_copy); + vk::ImageLayout::eTransferDstOptimal, destination_copy); } } } Transit(vk::ImageLayout::eGeneral, - vk::AccessFlagBits2::eShaderRead | vk::AccessFlagBits2::eTransferRead, {}, - command); + vk::AccessFlagBits2::eShaderRead | vk::AccessFlagBits2::eTransferRead, {}, command); } void Image::CopyMip(Image& source, uint32_t mip, uint32_t layer) { @@ -561,11 +534,9 @@ void Image::CopyMip(Image& source, uint32_t mip, uint32_t layer) { const auto width = std::max(backing.extent.width >> mip, 1u); const auto height = std::max(backing.extent.height >> mip, 1u); const auto depth = std::max(backing.extent.depth >> mip, 1u); - EXIT_IF(width != source.backing.extent.width || - height != source.backing.extent.height); - const auto [source_layers, destination_layers] = - SanitizeCopyLayers(source, *this, depth); - const auto aspects = FullAspectMask(source.backing.format); + EXIT_IF(width != source.backing.extent.width || height != source.backing.extent.height); + const auto [source_layers, destination_layers] = SanitizeCopyLayers(source, *this, depth); + const auto aspects = FullAspectMask(source.backing.format); EXIT_IF(aspects != FullAspectMask(backing.format)); std::array copies {}; uint32_t copy_count = 0; @@ -580,16 +551,13 @@ void Image::CopyMip(Image& source, uint32_t mip, uint32_t layer) { copy.extent = {width, height, depth}; } auto command = m_scheduler->Current().Handle(); - Transit(vk::ImageLayout::eTransferDstOptimal, - vk::AccessFlagBits2::eTransferWrite, {}, command); - source.Transit(vk::ImageLayout::eTransferSrcOptimal, - vk::AccessFlagBits2::eTransferRead, {}, command); - command.copyImage(source.backing.image, vk::ImageLayout::eTransferSrcOptimal, - backing.image, vk::ImageLayout::eTransferDstOptimal, copy_count, - copies.data()); + Transit(vk::ImageLayout::eTransferDstOptimal, vk::AccessFlagBits2::eTransferWrite, {}, command); + source.Transit(vk::ImageLayout::eTransferSrcOptimal, vk::AccessFlagBits2::eTransferRead, {}, + command); + command.copyImage(source.backing.image, vk::ImageLayout::eTransferSrcOptimal, backing.image, + vk::ImageLayout::eTransferDstOptimal, copy_count, copies.data()); Transit(vk::ImageLayout::eGeneral, - vk::AccessFlagBits2::eShaderRead | vk::AccessFlagBits2::eTransferRead, {}, - command); + vk::AccessFlagBits2::eShaderRead | vk::AccessFlagBits2::eTransferRead, {}, command); } namespace ImageOps { @@ -601,8 +569,7 @@ void Validate(const ImageInfo& info) { if (info.pixel_format == vk::Format::eUndefined) { const bool metadata_empty = info.metadata.range.address == 0 && info.metadata.range.size == 0 && - info.metadata.kind == ImageMetadataKind::None && - info.metadata.control == 0 && + info.metadata.kind == ImageMetadataKind::None && info.metadata.control == 0 && info.metadata.compression == VideoOutCompression::Uncompressed && !info.metadata.stencil_compressed; if (info.data.Empty() || info.HasStencil() || !metadata_empty || info.extent.width == 0 || @@ -615,9 +582,9 @@ void Validate(const ImageInfo& info) { } if (info.extent.width == 0 || info.extent.height == 0 || info.extent.depth == 0 || - info.resources.levels == 0 || - info.resources.levels > info.mip_layout.size() || info.resources.layers == 0 || - info.samples == 0 || vulkan_sample_count(info.samples) == vk::SampleCountFlagBits {} || + info.resources.levels == 0 || info.resources.levels > info.mip_layout.size() || + info.resources.layers == 0 || info.samples == 0 || + vulkan_sample_count(info.samples) == vk::SampleCountFlagBits {} || info.bytes_per_block == 0 || (info.data.address != 0 && info.pitch == 0)) { EXIT("invalid image geometry or format\n"); } @@ -688,11 +655,11 @@ uint32_t RenderTargetTransferFormat(uint32_t bytes_per_element) { } // namespace ImageOps -Image::Image(GraphicContext& graphics, CommandScheduler& scheduler, - const ImageInfo& image_info) +Image::Image(GraphicContext& graphics, CommandScheduler& scheduler, const ImageInfo& image_info) : info(image_info), m_graphics(&graphics), m_scheduler(&scheduler) { KYTY_PROFILER_FUNCTION(); ImageOps::Validate(info); + m_cpu_dirty = !info.data.Empty(); if (info.pixel_format == vk::Format::eUndefined) { return; } @@ -742,9 +709,9 @@ Image::Image(GraphicContext& graphics, CommandScheduler& scheduler, } uint64_t Image::HashGuestEdges() const { - constexpr uint64_t page_mask = TRACKER_PAGE_SIZE - 1; + constexpr uint64_t page_mask = TRACKER_PAGE_SIZE - 1; std::array bytes {}; - const auto range = info.data; + const auto range = info.data; const uint64_t head_end = std::min(range.End(), (range.address + page_mask) & ~page_mask); const uint64_t tail_begin = std::max(range.address, range.End() & ~page_mask); const uint64_t head_size = head_end - range.address; diff --git a/src/graphics/host_gpu/renderer/image/image.h b/src/graphics/host_gpu/renderer/image/image.h index 789119841..c09298301 100644 --- a/src/graphics/host_gpu/renderer/image/image.h +++ b/src/graphics/host_gpu/renderer/image/image.h @@ -66,16 +66,16 @@ public: [[nodiscard]] vk::ImageView FindView(const ImageViewInfo& view_info); void AssociateDepth(ImageId image_id) { depth_id = image_id; } using Barriers = std::vector; - [[nodiscard]] Barriers - GetBarriers(vk::ImageLayout destination_layout, vk::AccessFlags2 destination_access, - vk::PipelineStageFlags2 destination_stage, - std::optional range); + [[nodiscard]] Barriers GetBarriers(vk::ImageLayout destination_layout, + vk::AccessFlags2 destination_access, + vk::PipelineStageFlags2 destination_stage, + std::optional range); void Transit(vk::ImageLayout destination_layout, vk::AccessFlags2 destination_access, std::optional range, vk::CommandBuffer command_buffer); - void Upload(std::span copies, vk::Buffer buffer, - uint64_t offset, uint64_t size); - void Download(std::span copies, vk::Buffer buffer, - uint64_t offset, uint64_t size); + void Upload(std::span copies, vk::Buffer buffer, uint64_t offset, + uint64_t size); + void Download(std::span copies, vk::Buffer buffer, uint64_t offset, + uint64_t size); void CopyImage(Image& source); void Resolve(Image& source, const ImageSubresourceRange& source_range, const ImageSubresourceRange& destination_range); @@ -84,8 +84,8 @@ public: void InvalidateCpuWrite(uint64_t vaddr, uint64_t size) { if (ImageRangeOverlaps(info.data.address, info.data.size, vaddr, size)) { - m_cpu_dirty = true; - m_maybe_cpu_dirty = false; + m_cpu_dirty = true; + m_maybe_cpu_dirty = false; m_maybe_hash_valid = false; } else if (ImagePageRangesOverlap(info.data.address, info.data.size, vaddr, size)) { m_maybe_cpu_dirty = true; @@ -95,6 +95,11 @@ public: [[nodiscard]] bool IsCpuDirty() const { return m_cpu_dirty || m_maybe_cpu_dirty; } [[nodiscard]] bool IsDefinitelyCpuDirty() const { return m_cpu_dirty; } [[nodiscard]] bool IsMaybeCpuDirty() const { return m_maybe_cpu_dirty; } + void MarkMaybeCpuDirty() { + if (!m_cpu_dirty) { + m_maybe_cpu_dirty = true; + } + } [[nodiscard]] bool NeedsMaybeCpuHash() const { return m_maybe_cpu_dirty && !m_maybe_hash_valid; } @@ -102,14 +107,14 @@ public: if (!NeedsMaybeCpuHash()) { EXIT("image cannot initialize maybe-dirty hash\n"); } - m_maybe_cpu_hash = hash; + m_maybe_cpu_hash = hash; m_maybe_hash_valid = true; } [[nodiscard]] bool ResolveMaybeCpuHash(uint64_t hash) { if (!m_maybe_cpu_dirty || !m_maybe_hash_valid || m_cpu_dirty) { EXIT("image cannot resolve maybe-dirty hash\n"); } - m_maybe_cpu_dirty = false; + m_maybe_cpu_dirty = false; m_maybe_hash_valid = false; m_cpu_dirty |= hash != m_maybe_cpu_hash; return m_cpu_dirty; @@ -125,14 +130,15 @@ public: } [[nodiscard]] bool IsGpuModified() const noexcept { return m_gpu_modified; } - void MarkGpuModified() noexcept { m_gpu_modified = true; } - void ClearGpuModified() noexcept { m_gpu_modified = false; } + void MarkGpuModified() noexcept { m_gpu_modified = true; } + void ClearGpuModified() noexcept { m_gpu_modified = false; } [[nodiscard]] bool IsBufferModified() const noexcept { return m_buffer_modified; } - void MarkBufferModified() noexcept { m_buffer_modified = true; } - void ClearBufferModified() noexcept { m_buffer_modified = false; } + void MarkBufferModified() noexcept { m_buffer_modified = true; } + void ClearBufferModified() noexcept { m_buffer_modified = false; } - [[nodiscard]] bool Overlaps(uint64_t address, uint64_t size, bool pages = false) const noexcept { + [[nodiscard]] bool Overlaps(uint64_t address, uint64_t size, + bool pages = false) const noexcept { return pages ? ImagePageRangesOverlap(info.data.address, info.data.size, address, size) : ImageRangeOverlaps(info.data.address, info.data.size, address, size); } @@ -142,38 +148,41 @@ public: [[nodiscard]] bool SafeToDownload() const noexcept { return IsGpuModified() && !IsBufferModified() && !IsCpuDirty(); } + [[nodiscard]] bool IsTracked() const noexcept { return track_addr != 0 && track_addr_end != 0; } [[nodiscard]] uint64_t AccountedSize() const noexcept { return backing.image == nullptr ? 0 : (info.data.size + 1023) & ~uint64_t {1023}; } [[nodiscard]] uint64_t HashGuestEdges() const; - ImageInfo info; - VulkanImage backing; - ImageViewCache views; - ImageUsage usage; - ImageBinding binding; - bool registered = false; - ImageId depth_id {}; - uint64_t tick_accessed_last = 0; - size_t lru_id = 0; + ImageInfo info; + VulkanImage backing; + ImageViewCache views; + ImageUsage usage; + ImageBinding binding; + bool registered = false; + uint64_t track_addr = 0; + uint64_t track_addr_end = 0; + ImageId depth_id {}; + uint64_t tick_accessed_last = 0; + size_t lru_id = 0; private: friend struct ImageTestAccess; [[nodiscard]] static vk::ImageAspectFlags FullAspectMask(vk::Format format) noexcept; - [[nodiscard]] static uint32_t CopyRows(uint64_t row_size, uint32_t rows, - uint64_t capacity) noexcept; + [[nodiscard]] static uint32_t CopyRows(uint64_t row_size, uint32_t rows, + uint64_t capacity) noexcept; [[nodiscard]] static std::pair SanitizeCopyLayers(const Image& source, const Image& destination, uint32_t depth); - GraphicContext* m_graphics = nullptr; - CommandScheduler* m_scheduler = nullptr; - uint64_t m_maybe_cpu_hash = 0; - bool m_cpu_dirty = false; - bool m_maybe_cpu_dirty = false; - bool m_maybe_hash_valid = false; - bool m_gpu_modified = false; - bool m_buffer_modified = false; + GraphicContext* m_graphics = nullptr; + CommandScheduler* m_scheduler = nullptr; + uint64_t m_maybe_cpu_hash = 0; + bool m_cpu_dirty = false; + bool m_maybe_cpu_dirty = false; + bool m_maybe_hash_valid = false; + bool m_gpu_modified = false; + bool m_buffer_modified = false; }; namespace ImageOps { diff --git a/tests/PageManagerTests.cpp b/tests/PageManagerTests.cpp index 53b3124cc..054262030 100644 --- a/tests/PageManagerTests.cpp +++ b/tests/PageManagerTests.cpp @@ -577,6 +577,75 @@ void TestCrossRegionRange() { Check(VirtualFree(memory, 0, MEM_RELEASE) != 0, "VirtualFree failed"); } +void TestBatchedWatcherRanges() { + FaultContext context; + PageManager manager(InvalidateFault, &context); + context.manager = &manager; + const auto page_size = manager.GetPageSize(); + constexpr uint64_t region_size = 4ull * 1024ull * 1024ull; + constexpr uint64_t allocation_size = region_size * 3; + auto *memory = Allocate(allocation_size); + const auto address = reinterpret_cast(memory); + + manager.OnGpuMap(address, allocation_size); + + manager.UpdatePageWatchers(true, address + page_size, page_size); + manager.UpdatePageWatchers(true, address + page_size * 3, page_size); + manager.UpdatePageWatchers(true, address, page_size * 5); + for (uint64_t page = 0; page < 5; page++) { + Check(Protection(memory + page * page_size) == PAGE_READONLY, + "fragmented watch did not coalesce to read-only"); + } + manager.UpdatePageWatchers(false, address, page_size * 5); + Check(IsWritable(memory) && + Protection(memory + page_size) == PAGE_READONLY && + IsWritable(memory + page_size * 2) && + Protection(memory + page_size * 3) == PAGE_READONLY && + IsWritable(memory + page_size * 4), + "fragmented unwatch lost overlapping watcher counts"); + manager.UpdatePageWatchers(false, address + page_size, page_size); + manager.UpdatePageWatchers(false, address + page_size * 3, page_size); + + manager.UpdatePageWatchers(true, address, allocation_size); + Check(!IsWritable(memory) && + !IsWritable(memory + region_size) && + !IsWritable(memory + region_size * 2) && + !IsWritable(memory + allocation_size - page_size), + "large cross-region watch did not protect the full range"); + manager.UpdatePageWatchers(false, address, allocation_size); + Check(IsWritable(memory) && + IsWritable(memory + region_size) && + IsWritable(memory + region_size * 2) && + IsWritable(memory + allocation_size - page_size), + "large cross-region unwatch did not restore the full range"); + + manager.UpdatePageWatchers(true, address, page_size * 5); + manager.UpdatePageWatchers(true, address + page_size, page_size * 3, + Libs::Graphics::PageWatchMode::ReadWrite); + Check(Protection(memory) == PAGE_READONLY && + Protection(memory + page_size) == PAGE_NOACCESS && + Protection(memory + page_size * 2) == PAGE_NOACCESS && + Protection(memory + page_size * 3) == PAGE_NOACCESS && + Protection(memory + page_size * 4) == PAGE_READONLY, + "mixed watcher modes installed incorrect protections"); + manager.UpdatePageWatchers(false, address, page_size * 5); + Check(IsWritable(memory) && + Protection(memory + page_size) == PAGE_NOACCESS && + Protection(memory + page_size * 2) == PAGE_NOACCESS && + Protection(memory + page_size * 3) == PAGE_NOACCESS && + IsWritable(memory + page_size * 4), + "write unwatch incorrectly released read/write watchers"); + manager.UpdatePageWatchers(false, address + page_size, page_size * 3, + Libs::Graphics::PageWatchMode::ReadWrite); + Check(IsWritable(memory + page_size) && + IsWritable(memory + page_size * 2) && + IsWritable(memory + page_size * 3), + "read/write unwatch did not restore writable protection"); + + manager.OnGpuUnmap(address, allocation_size); + Check(VirtualFree(memory, 0, MEM_RELEASE) != 0, "VirtualFree failed"); +} + [[noreturn]] void RunDeathCase(const char *name) { FaultContext context; auto manager = std::make_unique(InvalidateFault, &context); @@ -795,6 +864,7 @@ int main(int argc, char **argv) { TestNativeAccessViolation(); TestInvalidLateWriteTokenIsConsumed(); TestCrossRegionRange(); + TestBatchedWatcherRanges(); TestConcurrentFault(); TestExternalDirtyTransferDuringResolution(); TestMappingDoesNotRequireCpuWriteAccess(); diff --git a/tests/ShaderRecompilerComputeTests.cpp b/tests/ShaderRecompilerComputeTests.cpp index 6233b655d..db7c99765 100644 --- a/tests/ShaderRecompilerComputeTests.cpp +++ b/tests/ShaderRecompilerComputeTests.cpp @@ -3195,6 +3195,10 @@ public: const auto fault_b_image = texture_cache.FindImage(fault_b_desc); texture_cache.MarkGpuWritten(fault_a_image); texture_cache.MarkGpuWritten(fault_b_image); + Require(name, "per-image watcher installation", + texture_cache.GetImage(fault_a_image).IsTracked() && + texture_cache.GetImage(fault_b_image).IsTracked(), + "same-page images did not install independent write watchers"); constexpr uint64_t padding_fault_offset = 0x8080; uint32_t write_only_read_a = 0; uint32_t write_only_read_b = 0; @@ -3212,13 +3216,24 @@ public: resources.HandleFault(PageFaultAccess::Write, base + padding_fault_offset) && texture_cache.GetImage(fault_a_image).IsGpuModified() && - texture_cache.GetImage(fault_b_image).IsGpuModified(), + texture_cache.GetImage(fault_b_image).IsGpuModified() && + !texture_cache.GetImage(fault_a_image).IsTracked() && + !texture_cache.GetImage(fault_b_image).IsTracked() && + texture_cache.GetImage(fault_a_image).IsMaybeCpuDirty() && + texture_cache.GetImage(fault_b_image).IsMaybeCpuDirty(), "a byte-disjoint CPU write discarded authoritative images"); + const auto retracked_a = texture_cache.FindImage(fault_a_desc); + const auto retracked_b = texture_cache.FindImage(fault_b_desc); Require(name, "same-page image re-track", - texture_cache.FindImage(fault_a_desc) == fault_a_image && - texture_cache.FindImage(fault_b_desc) == fault_b_image && + retracked_a == fault_a_image && retracked_b == fault_b_image && + texture_cache.GetImage(fault_a_image).IsTracked() && + texture_cache.GetImage(fault_b_image).IsTracked() && + !texture_cache.GetImage(fault_a_image).IsCpuDirty() && + !texture_cache.GetImage(fault_b_image).IsCpuDirty() && texture_cache.SynchronizeImageToBuffer(base + 0x8000, sizeof(fault_a)) && + texture_cache.GetImage(fault_a_image).IsTracked() && + texture_cache.GetImage(fault_b_image).IsTracked() && !texture_cache.GetImage(fault_a_image).IsGpuModified() && texture_cache.GetImage(fault_b_image).IsGpuModified(), "retiring one same-page image lost the surviving owner");