diff --git a/src/graphics/host_gpu/memoryTracker.h b/src/graphics/host_gpu/memoryTracker.h index 9798ba251..9053e4fde 100644 --- a/src/graphics/host_gpu/memoryTracker.h +++ b/src/graphics/host_gpu/memoryTracker.h @@ -38,10 +38,51 @@ public: bool downloaded) noexcept; [[nodiscard]] bool InvalidateRegion(uint64_t vaddr, uint64_t size, PageFaultPhase phase) noexcept; + template + void InvalidateRegion(uint64_t vaddr, uint64_t size, Flush&& on_flush) { + static_assert(std::is_invocable_v); + CheckNotInUploadCallback(); + ValidateRange(vaddr, size); + + const auto update_cpu_state = [this, vaddr, size] { + std::lock_guard access(m_access_mutex); + std::vector managers; + Iterate(vaddr, size, [&](RegionManager* manager, uint64_t, uint64_t) { + managers.push_back(manager); + }); + std::vector> locks; + locks.reserve(managers.size()); + for (auto* manager: managers) { + locks.emplace_back(manager->lock); + } + const bool gpu_modified = Iterate( + vaddr, size, [](RegionManager* manager, uint64_t offset, uint64_t bytes) { + return manager->IsModified(offset, bytes); + }); + if (gpu_modified) { + return true; + } + Iterate(vaddr, size, + [](RegionManager* manager, uint64_t offset, uint64_t bytes) { + const auto changed = manager->ChangeState( + manager->GetCpuAddr() + offset, bytes); + manager->ApplyProtection(changed, false); + }); + return false; + }; + + if (!update_cpu_state()) { + return; + } + std::forward(on_flush)(); + if (update_cpu_state()) { + EXIT("memory invalidation retained GPU-owned pages\n"); + } + } [[nodiscard]] bool InvalidateVirtualGpuWrite(PageFaultAccess access, uint64_t vaddr, uint64_t size, PageFaultPhase phase) noexcept; - void ValidateGpuDirtyPages(const RangeSet& dirty, uint64_t vaddr, uint64_t size, - const char* operation) const noexcept; + void ValidateGpuDirtyPages(const RangeSet& dirty, uint64_t vaddr, uint64_t size, + const char* operation) const noexcept; void ValidateGpuDirtyOwnership(const RangeSet& dirty, uint64_t vaddr, uint64_t size, const char* operation); diff --git a/src/graphics/host_gpu/pageManager.cpp b/src/graphics/host_gpu/pageManager.cpp index 5ce0c3dc7..7c3ab6d04 100644 --- a/src/graphics/host_gpu/pageManager.cpp +++ b/src/graphics/host_gpu/pageManager.cpp @@ -699,26 +699,6 @@ bool PageManager::IsMapped(uint64_t vaddr, uint64_t size) const noexcept { return true; } -bool PageManager::HasAnyMapping(uint64_t vaddr, uint64_t size) const noexcept { - if (g_in_fault_resolution || vaddr == 0 || size == 0 || vaddr >= ADDRESS_SIZE || - size > ADDRESS_SIZE - vaddr) { - return false; - } - const auto end = PageEnd(vaddr, size); - for (auto page_vaddr = PageStart(vaddr); page_vaddr < end; page_vaddr += PAGE_SIZE) { - auto* region = m_impl->FindRegion(page_vaddr); - if (region == nullptr) { - continue; - } - auto& page = m_impl->GetPage(*region, page_vaddr); - SpinGuard lock(page.lock); - if (page.mappings != 0) { - return true; - } - } - return false; -} - bool PageManager::HasGpuAccess(uint64_t vaddr, uint64_t size, GpuAccess access) const noexcept { if (access != GpuAccess::Read && access != GpuAccess::Write && access != GpuAccess::ReadWrite) { FailFast("HasGpuAccess received an invalid GPU access mode"); @@ -1150,22 +1130,4 @@ bool PageManager::HandleFault(PageFaultAccess access, uint64_t fault_vaddr) noex return true; } -bool PageManager::HandleWriteRange(uint64_t vaddr, uint64_t size) noexcept { - if (g_in_fault_resolution || vaddr == 0 || size == 0 || vaddr >= ADDRESS_SIZE || - size > ADDRESS_SIZE - vaddr) { - return false; - } - const auto end = PageEnd(vaddr, size); - for (auto page_vaddr = PageStart(vaddr); page_vaddr < end; page_vaddr += PAGE_SIZE) { - if (!IsMapped(page_vaddr, 1)) { - continue; - } - const auto fault_vaddr = std::max(page_vaddr, vaddr); - if (!HandleFault(PageFaultAccess::Write, fault_vaddr)) { - return false; - } - } - return true; -} - } // namespace Libs::Graphics diff --git a/src/graphics/host_gpu/pageManager.h b/src/graphics/host_gpu/pageManager.h index 50af34b87..5143b4229 100644 --- a/src/graphics/host_gpu/pageManager.h +++ b/src/graphics/host_gpu/pageManager.h @@ -41,7 +41,6 @@ public: [[nodiscard]] uint64_t GetPageSize() const; [[nodiscard]] bool IsTracked(uint64_t vaddr) const noexcept; [[nodiscard]] bool IsMapped(uint64_t vaddr, uint64_t size) const noexcept; - [[nodiscard]] bool HasAnyMapping(uint64_t vaddr, uint64_t size) const noexcept; [[nodiscard]] bool HasGpuAccess(uint64_t vaddr, uint64_t size, GpuAccess access) const noexcept; void UpdatePageWatchers(bool track, uint64_t vaddr, uint64_t size, @@ -50,9 +49,8 @@ public: void OnGpuUnmap(uint64_t vaddr, uint64_t size, GpuAccess access = GpuAccess::ReadWrite); [[nodiscard]] bool HandleFault(PageFaultAccess access, uint64_t fault_vaddr) noexcept; - [[nodiscard]] bool HandleWriteRange(uint64_t vaddr, uint64_t size) noexcept; [[nodiscard]] std::vector> - ReserveBackingWrites(std::span ranges); + ReserveBackingWrites(std::span ranges); private: void BeginBackingWrite(uint64_t vaddr, uint64_t size) noexcept; diff --git a/src/graphics/host_gpu/rangeSet.h b/src/graphics/host_gpu/rangeSet.h index 203775bc5..6a96cf55b 100644 --- a/src/graphics/host_gpu/rangeSet.h +++ b/src/graphics/host_gpu/rangeSet.h @@ -59,6 +59,16 @@ public: return result; } + [[nodiscard]] bool Contains(uint64_t address, uint64_t size) const { + const auto end = End(address, size); + auto it = m_ranges.upper_bound(address); + if (it == m_ranges.begin()) { + return false; + } + --it; + return it->first <= address && it->second >= end; + } + template void ForEachIntersection(uint64_t address, uint64_t size, Func&& func) const { const auto end = End(address, size); diff --git a/src/graphics/host_gpu/renderer/cache/bufferCache.cpp b/src/graphics/host_gpu/renderer/cache/bufferCache.cpp index 40c17e747..0f356a12e 100644 --- a/src/graphics/host_gpu/renderer/cache/bufferCache.cpp +++ b/src/graphics/host_gpu/renderer/cache/bufferCache.cpp @@ -4,10 +4,10 @@ #include "common/logging/log.h" #include "common/profiler.h" #include "graphics/host_gpu/graphicContext.h" -#include "graphics/host_gpu/renderer/commandScheduler.h" -#include "graphics/host_gpu/renderer/render.h" #include "graphics/host_gpu/renderer/cache/resourceMutex.h" #include "graphics/host_gpu/renderer/cache/textureCache.h" +#include "graphics/host_gpu/renderer/commandScheduler.h" +#include "graphics/host_gpu/renderer/render.h" #include "kernel/memory.h" #include @@ -183,9 +183,8 @@ BufferCache::RecordDownloads(std::span copies) { return {}; } - auto& download = m_download_buffer; - const auto [mapped, base_offset] = - download.Map(reservation_size, DOWNLOAD_ALIGNMENT); + auto& download = m_download_buffer; + const auto [mapped, base_offset] = download.Map(reservation_size, DOWNLOAD_ALIGNMENT); if (mapped == nullptr) { EXIT("BufferCache: download batch could not reserve the shared stream\n"); } @@ -215,40 +214,38 @@ void BufferCache::PublishDownloads(std::span downloads) { } } -void BufferCache::QueueGarbageDownload(std::span copies, - RetiredBuffer retire) { +void BufferCache::QueueGarbageDownload(std::span copies, RetiredBuffer retire) { if (copies.empty()) { return; } - auto downloads = RecordDownloads(copies); - const auto tick = m_scheduler.CurrentTick(); + auto downloads = RecordDownloads(copies); + const auto tick = m_scheduler.CurrentTick(); BeginBackingPublication(retire.address, retire.size, tick); - m_scheduler.DeferOperation( - [this, downloads = std::move(downloads), retire = std::move(retire), tick]() mutable { - PublishDownloads(downloads); - { - FaultSafeCacheLock lock(this, m_mutex); - if (m_memory_tracker.IsRegionGpuModified(retire.address, retire.size)) { - m_memory_tracker.ForEachDownloadRange( - retire.address, retire.size, - [&](uint64_t address, uint64_t size) noexcept { - m_memory_tracker.ValidateGpuDirtyPages( - m_gpu_modified_ranges, address, size, - "asynchronous garbage retirement"); - }, - [](uint64_t, uint64_t) noexcept {}); - } - for (const auto& range: downloads) { - m_gpu_modified_ranges.Subtract(range.address, range.size); - } - if (m_memory_tracker.IsRegionGpuModified(retire.address, retire.size) || - !m_gpu_modified_ranges.Intersections(retire.address, retire.size).empty()) { - EXIT("BufferCache: asynchronous garbage collection retained GPU ownership\n"); - } - m_memory_tracker.UntrackMemory(retire.address, retire.size); - } - CompleteBackingPublication(retire.address, retire.size, tick); - }); + m_scheduler.DeferOperation([this, downloads = std::move(downloads), retire = std::move(retire), + tick]() mutable { + PublishDownloads(downloads); + { + FaultSafeCacheLock lock(this, m_mutex); + if (m_memory_tracker.IsRegionGpuModified(retire.address, retire.size)) { + m_memory_tracker.ForEachDownloadRange( + retire.address, retire.size, + [&](uint64_t address, uint64_t size) noexcept { + m_memory_tracker.ValidateGpuDirtyPages(m_gpu_modified_ranges, address, size, + "asynchronous garbage retirement"); + }, + [](uint64_t, uint64_t) noexcept {}); + } + for (const auto& range: downloads) { + m_gpu_modified_ranges.Subtract(range.address, range.size); + } + if (m_memory_tracker.IsRegionGpuModified(retire.address, retire.size) || + !m_gpu_modified_ranges.Intersections(retire.address, retire.size).empty()) { + EXIT("BufferCache: asynchronous garbage collection retained GPU ownership\n"); + } + m_memory_tracker.UntrackMemory(retire.address, retire.size); + } + CompleteBackingPublication(retire.address, retire.size, tick); + }); } BufferCache::BufferCache(GraphicContext& graphics, CommandScheduler& scheduler, @@ -301,14 +298,13 @@ BufferCache::~BufferCache() { bool BufferCache::SynchronizeBacking(uint64_t vaddr, uint64_t size) { bool waited = false; for (;;) { - uint64_t tick = 0; + uint64_t tick = 0; const auto page_begin = vaddr & ~(TRACKER_PAGE_SIZE - 1); - const auto page_end = - (vaddr + size + TRACKER_PAGE_SIZE - 1) & ~(TRACKER_PAGE_SIZE - 1); + const auto page_end = (vaddr + size + TRACKER_PAGE_SIZE - 1) & ~(TRACKER_PAGE_SIZE - 1); CacheRange affected {.address = page_begin, .size = page_end - page_begin}; { FaultSafeCacheLock lock(this, m_mutex); - bool changed = true; + bool changed = true; while (changed) { changed = false; for (const auto& [address, cached]: m_buffers) { @@ -384,6 +380,72 @@ BufferBinding BufferCache::UploadTransient(const void* data, uint64_t size, uint return {owner, owner->Handle(), 0}; } +void BufferCache::InvalidateMemory(uint64_t vaddr, uint64_t size) { + if (vaddr == 0 || size == 0 || vaddr >= TRACKER_ADDRESS_SIZE || + size > TRACKER_ADDRESS_SIZE - vaddr) { + EXIT("BufferCache: invalid memory-invalidation range\n"); + } + (void)SynchronizeBacking(vaddr, size); + if (!HasPageOverlap(vaddr, size)) { + return; + } + m_memory_tracker.InvalidateRegion(vaddr, size, + [this, vaddr, size] { ReadMemory(vaddr, size); }); +} + +void BufferCache::ReadMemory(uint64_t vaddr, uint64_t size) { + std::vector copies; + { + FaultSafeCacheLock lock(this, m_mutex); + m_memory_tracker.ForEachDownloadRange( + vaddr, size, + [&](uint64_t address, uint64_t bytes) noexcept { + m_memory_tracker.ValidateGpuDirtyPages(m_gpu_modified_ranges, address, bytes, + "memory invalidation"); + }, + [&](uint64_t address, uint64_t bytes) noexcept { + for (const auto range: m_gpu_modified_ranges.Intersections(address, bytes)) { + for (uint64_t copied = 0; copied < range.size;) { + const auto copy_address = range.address + copied; + auto owner = m_buffers.upper_bound(copy_address); + if (owner == m_buffers.begin()) { + EXIT("BufferCache: invalidation readback has no buffer owner\n"); + } + auto& cached = *std::prev(owner)->second; + if (!cached.buffer->IsInBounds(copy_address, 1)) { + EXIT( + "BufferCache: invalidation readback is outside its buffer owner\n"); + } + const auto copy_size = std::min(range.size - copied, + cached.vaddr + cached.size - copy_address); + copies.push_back({cached.buffer, cached.buffer->Offset(copy_address), + copy_address, copy_size}); + copied += copy_size; + } + } + }); + } + if (copies.empty()) { + EXIT("BufferCache: GPU-owned invalidation has no dirty byte ranges\n"); + } + auto downloads = RecordDownloads(copies); + m_scheduler.FinishCurrent(); + PublishDownloads(downloads); + { + FaultSafeCacheLock lock(this, m_mutex); + m_memory_tracker.ForEachDownloadRange( + vaddr, size, + [&](uint64_t address, uint64_t bytes) noexcept { + m_memory_tracker.ValidateGpuDirtyPages(m_gpu_modified_ranges, address, bytes, + "memory invalidation completion"); + }, + [](uint64_t, uint64_t) noexcept {}); + for (const auto& range: downloads) { + m_gpu_modified_ranges.Subtract(range.address, range.size); + } + } +} + bool BufferCache::InvalidateMemory(PageFaultAccess access, uint64_t vaddr, uint64_t size, PageFaultPhase phase) noexcept { const auto page = vaddr & ~(TRACKER_PAGE_SIZE - 1); @@ -544,8 +606,8 @@ void BufferCache::UnmapMemory(uint64_t vaddr, uint64_t size) { m_memory_tracker.ForEachDownloadRange( begin, bytes, [&](uint64_t address, uint64_t download_size) noexcept { - m_memory_tracker.ValidateGpuDirtyPages( - m_gpu_modified_ranges, address, download_size, "unmap retirement"); + m_memory_tracker.ValidateGpuDirtyPages(m_gpu_modified_ranges, address, + download_size, "unmap retirement"); }, [](uint64_t, uint64_t) noexcept {}); } @@ -722,12 +784,11 @@ ImageBufferSource BufferCache::ObtainBufferForImage(uint64_t vaddr, uint64_t siz { FaultSafeCacheLock lock(this, m_mutex); - const bool cpu_modified = m_memory_tracker.IsRegionCpuModified(vaddr, size); - const bool gpu_modified = m_memory_tracker.IsRegionGpuModified(vaddr, size); - const auto dirty = m_gpu_modified_ranges.Intersections(vaddr, size); - const bool invalidated = - !m_image_invalidated_ranges.Intersections(vaddr, size).empty(); - const bool requested_gpu_owned = !dirty.empty(); + const bool cpu_modified = m_memory_tracker.IsRegionCpuModified(vaddr, size); + const bool gpu_modified = m_memory_tracker.IsRegionGpuModified(vaddr, size); + const auto dirty = m_gpu_modified_ranges.Intersections(vaddr, size); + const bool invalidated = !m_image_invalidated_ranges.Intersections(vaddr, size).empty(); + const bool requested_gpu_owned = !dirty.empty(); m_memory_tracker.ValidateGpuDirtyOwnership(m_gpu_modified_ranges, vaddr, size, "image source"); @@ -807,8 +868,8 @@ ImageBufferSource BufferCache::ObtainBufferForImage(uint64_t vaddr, uint64_t siz } FaultSafeCacheLock lock(this, m_mutex); - const auto dirty = m_gpu_modified_ranges.Intersections(vaddr, size); - const bool invalidated = !m_image_invalidated_ranges.Intersections(vaddr, size).empty(); + const auto dirty = m_gpu_modified_ranges.Intersections(vaddr, size); + const bool invalidated = !m_image_invalidated_ranges.Intersections(vaddr, size).empty(); const bool requested_gpu_owned = !dirty.empty(); auto owner = find_owner(); if (requested_gpu_owned && owner == m_buffers.end()) { @@ -831,9 +892,8 @@ ImageBufferSource BufferCache::ObtainBufferForImage(uint64_t vaddr, uint64_t siz [&]() noexcept { for (const auto& [address, upload_size]: uploads) { cached.buffer->CopyFrom( - m_scheduler.Current(), m_staging_buffer, - stage_offset + address - stage_address, cached.buffer->Offset(address), - upload_size, vk::AccessFlagBits::eHostWrite); + m_scheduler.Current(), m_staging_buffer, stage_offset + address - stage_address, + cached.buffer->Offset(address), upload_size, vk::AccessFlagBits::eHostWrite); } }); DiscardGpuDirtyBytesLocked(vaddr, size, "staged image source transfer"); @@ -917,9 +977,8 @@ std::pair, uint64_t> BufferCache::ObtainBufferForImageWr [&]() noexcept { for (const auto& [address, upload_size]: uploads) { cached.buffer->CopyFrom( - m_scheduler.Current(), m_staging_buffer, - stage_offset + address - stage_address, cached.buffer->Offset(address), - upload_size, vk::AccessFlagBits::eHostWrite); + m_scheduler.Current(), m_staging_buffer, stage_offset + address - stage_address, + cached.buffer->Offset(address), upload_size, vk::AccessFlagBits::eHostWrite); } }); return {cached.buffer, cached.buffer->Offset(vaddr)}; @@ -946,12 +1005,12 @@ void BufferCache::FillBuffer(uint64_t vaddr, uint64_t size, uint32_t value, bool const auto region = m_texture_cache.QueryRegion(vaddr, size); if (!HasGpuDirtyBytes(vaddr, size) && !region.gpu_image_bytes) { if (region.image_bytes) { - m_texture_cache.PrepareHostWrite(vaddr, size); + m_texture_cache.InvalidateMemory(vaddr, size); } std::array values; values.fill(value); - const std::span bytes { - reinterpret_cast(values.data()), sizeof(values)}; + const std::span bytes {reinterpret_cast(values.data()), + sizeof(values)}; for (uint64_t offset = 0; offset < size;) { const auto chunk = std::min(size - offset, bytes.size()); WriteHostMemory(vaddr + offset, bytes.first(chunk)); @@ -992,7 +1051,7 @@ void BufferCache::CopyBuffer(uint64_t dst_vaddr, uint64_t src_vaddr, uint64_t si if (src_memory) { (void)SynchronizeBacking(src_vaddr, size); } - const auto src_region = + const auto src_region = src_memory ? m_texture_cache.QueryRegion(src_vaddr, size) : TextureCache::RegionInfo {}; const auto dst_region = dst_memory ? m_texture_cache.QueryRegion(dst_vaddr, size) : TextureCache::RegionInfo {}; @@ -1004,7 +1063,7 @@ void BufferCache::CopyBuffer(uint64_t dst_vaddr, uint64_t src_vaddr, uint64_t si !HasGpuDirtyBytes(dst_vaddr, size) && !src_region.gpu_image_bytes && !dst_region.gpu_image_bytes) { if (dst_region.image_bytes) { - m_texture_cache.PrepareHostWrite(dst_vaddr, size); + m_texture_cache.InvalidateMemory(dst_vaddr, size); } std::array bytes; for (uint64_t offset = 0; offset < size;) { @@ -1037,10 +1096,10 @@ void BufferCache::CopyBuffer(uint64_t dst_vaddr, uint64_t src_vaddr, uint64_t si EXIT("BufferCache: resolved Vulkan copy ranges overlap\n"); } auto& source = src.owner != nullptr ? *std::static_pointer_cast(src.owner) - : src_gds ? m_gds_buffer - : m_stream_buffer; - auto& destination = dst.owner != nullptr ? *std::static_pointer_cast(dst.owner) - : m_gds_buffer; + : src_gds ? m_gds_buffer + : m_stream_buffer; + auto& destination = + dst.owner != nullptr ? *std::static_pointer_cast(dst.owner) : m_gds_buffer; if (source.Handle() != src.buffer || destination.Handle() != dst.buffer) { EXIT("BufferCache: resolved copy owner does not match its Vulkan handle\n"); } @@ -1107,7 +1166,7 @@ void BufferCache::BeginBackingPublication(uint64_t vaddr, uint64_t size, uint64_ void BufferCache::CompleteBackingPublication(uint64_t vaddr, uint64_t size, uint64_t tick) { std::lock_guard lock(m_publication_mutex); - const auto publication = + const auto publication = std::ranges::find_if(m_pending_backing_publications, [&](const auto& pending) { return pending.address == vaddr && pending.size == size && pending.tick == tick; }); @@ -1170,7 +1229,7 @@ void BufferCache::RunGarbageCollector() { const uint64_t age = std::min(aggressive ? 80 : 160, tick); const size_t limit = aggressive ? 64 : 32; - std::vector retires; + std::vector retires; std::vector>> dirty_retires; { FaultSafeCacheLock lock(this, m_mutex); @@ -1190,8 +1249,8 @@ void BufferCache::RunGarbageCollector() { } for (const auto address: candidates) { auto& cached = *m_buffers.at(address); - m_memory_tracker.ValidateGpuDirtyOwnership( - m_gpu_modified_ranges, cached.vaddr, cached.size, "garbage collection"); + m_memory_tracker.ValidateGpuDirtyOwnership(m_gpu_modified_ranges, cached.vaddr, + cached.size, "garbage collection"); retires.push_back({address, cached.size, cached.buffer}); // GC runs immediately before submission. Preserve every source referenced by commands // already recorded in the active batch. @@ -1205,13 +1264,13 @@ void BufferCache::RunGarbageCollector() { m_memory_tracker.ForEachDownloadRange( retire.address, retire.size, [&](uint64_t address, uint64_t size) noexcept { - m_memory_tracker.ValidateGpuDirtyPages( - m_gpu_modified_ranges, address, size, "garbage collection"); + m_memory_tracker.ValidateGpuDirtyPages(m_gpu_modified_ranges, address, size, + "garbage collection"); }, [&](uint64_t address, uint64_t size) noexcept { for (const auto range: m_gpu_modified_ranges.Intersections(address, size)) { - copies.push_back({retire.owner, range.address - retire.address, range.address, - range.size}); + copies.push_back({retire.owner, range.address - retire.address, + range.address, range.size}); } }); } diff --git a/src/graphics/host_gpu/renderer/cache/bufferCache.h b/src/graphics/host_gpu/renderer/cache/bufferCache.h index 3bfad0580..d390f5072 100644 --- a/src/graphics/host_gpu/renderer/cache/bufferCache.h +++ b/src/graphics/host_gpu/renderer/cache/bufferCache.h @@ -49,6 +49,7 @@ public: [[nodiscard]] bool InvalidateMemory(PageFaultAccess access, uint64_t vaddr, uint64_t size, PageFaultPhase phase) noexcept; + void InvalidateMemory(uint64_t vaddr, uint64_t size); void UnmapMemory(uint64_t vaddr, uint64_t size); [[nodiscard]] BufferBinding ObtainBuffer(CommandBuffer& command, uint64_t vaddr, uint64_t size, bool is_written = false, bool is_read = true, @@ -71,8 +72,8 @@ public: [[nodiscard]] bool IsRegionCpuModified(uint64_t vaddr, uint64_t size); [[nodiscard]] bool IsRegionGpuModified(uint64_t vaddr, uint64_t size); void InvalidateImageAliases(uint64_t vaddr, uint64_t size); - void BeginBackingPublication(uint64_t vaddr, uint64_t size, uint64_t tick); - void CompleteBackingPublication(uint64_t vaddr, uint64_t size, uint64_t tick); + void BeginBackingPublication(uint64_t vaddr, uint64_t size, uint64_t tick); + void CompleteBackingPublication(uint64_t vaddr, uint64_t size, uint64_t tick); [[nodiscard]] bool SynchronizeBacking(uint64_t vaddr, uint64_t size); void PublishImageBuffer(uint64_t vaddr, uint64_t size); void ValidateGpuAccess(uint64_t vaddr, uint64_t size, bool is_read, bool is_written) const; @@ -91,27 +92,26 @@ private: struct RetiredBuffer; struct FaultReadback; struct PendingBackingPublication; - static constexpr uint64_t DOWNLOAD_ALIGNMENT = 64; - [[nodiscard]] static uint64_t AlignDown(uint64_t value) noexcept; - [[nodiscard]] static uint64_t AlignUp(uint64_t value); + static constexpr uint64_t DOWNLOAD_ALIGNMENT = 64; + [[nodiscard]] static uint64_t AlignDown(uint64_t value) noexcept; + [[nodiscard]] static uint64_t AlignUp(uint64_t value); [[nodiscard]] static constexpr uint64_t AlignDownload(uint64_t size) noexcept { return (size + DOWNLOAD_ALIGNMENT - 1) & ~(DOWNLOAD_ALIGNMENT - 1); } [[nodiscard]] static bool PageOverlaps(uint64_t left, uint64_t left_size, uint64_t right, - uint64_t right_size) noexcept; - [[nodiscard]] static std::pair - DownloadEnvelope(const DownloadCopy& copy); - [[nodiscard]] static bool ResolveOverlap(CacheRange& merged, CacheRange candidate) noexcept; + uint64_t right_size) noexcept; + [[nodiscard]] static std::pair DownloadEnvelope(const DownloadCopy& copy); + [[nodiscard]] static bool ResolveOverlap(CacheRange& merged, CacheRange candidate) noexcept; void Upload(CommandBuffer& command, Buffer& destination, uint64_t destination_offset, const void* source, uint64_t size); [[nodiscard]] CachedBuffer& GetOrCreateBuffer(CommandBuffer& command, uint64_t vaddr, uint64_t size); - [[nodiscard]] std::vector - RecordDownloads(std::span copies); + [[nodiscard]] std::vector RecordDownloads(std::span copies); void PublishDownloads(std::span downloads); void QueueGarbageDownload(std::span copies, RetiredBuffer retire); void RefreshInvalidatedRanges(CommandBuffer& command, CachedBuffer& cached, uint64_t vaddr, uint64_t size, bool upload); + void ReadMemory(uint64_t vaddr, uint64_t size); void DiscardGpuDirtyBytesLocked(uint64_t vaddr, uint64_t size, const char* operation); void WriteHostMemory(uint64_t vaddr, std::span data); diff --git a/src/graphics/host_gpu/renderer/cache/gpuResourceManager.cpp b/src/graphics/host_gpu/renderer/cache/gpuResourceManager.cpp index 728af7e94..7ff188a19 100644 --- a/src/graphics/host_gpu/renderer/cache/gpuResourceManager.cpp +++ b/src/graphics/host_gpu/renderer/cache/gpuResourceManager.cpp @@ -68,47 +68,52 @@ bool GpuResourceManager::HandleFault(PageFaultAccess access, uint64_t fault_vadd return handled; } -void GpuResourceManager::PrepareHostWrite(uint64_t vaddr, uint64_t size) { - if (!m_page_manager.HasAnyMapping(vaddr, size)) { - return; +bool GpuResourceManager::InvalidateMemory(uint64_t vaddr, uint64_t size) { + if (!IsMapped(vaddr, size)) { + return false; } if (CommandScheduler::InDeferredOperation()) { - EXIT("unsupported host write from an asynchronous GPU completion, addr=0x%016" PRIx64 - " size=0x%016" PRIx64 "\n", + EXIT("unsupported memory invalidation from an asynchronous GPU completion, " + "addr=0x%016" PRIx64 " size=0x%016" PRIx64 "\n", vaddr, size); } - const auto handle_range = [this, vaddr, size] { - if (!m_page_manager.HandleWriteRange(vaddr, size)) { - EXIT("failed to prepare host write, addr=0x%016" PRIx64 " size=0x%016" PRIx64 "\n", - vaddr, size); - } - }; - const auto resolve = [this, &handle_range](CommandProcessor& cp) { + const auto resolve = [this, vaddr, size](CommandProcessor& cp) { cp.BeginReadbackTransaction(); { ResourceMutex::FaultScope fault(m_resource_mutex); - handle_range(); + m_buffer_cache.InvalidateMemory(vaddr, size); + m_texture_cache.InvalidateMemory(vaddr, size); } cp.EndReadbackTransaction(); }; if (auto* cp = Gpu::CurrentCommandProcessor(); cp != nullptr) { resolve(*cp); - return; + return true; } if (m_resource_mutex.IsOwnedByCurrentThread()) { - EXIT("unsupported host write from a pre-owned resource transaction, addr=0x%016" PRIx64 - " size=0x%016" PRIx64 "\n", + EXIT("unsupported memory invalidation from a pre-owned resource transaction, " + "addr=0x%016" PRIx64 " size=0x%016" PRIx64 "\n", vaddr, size); } EXIT_IF(m_gpu == nullptr); m_gpu->SendCommandSyncWithProcessor(resolve); + return true; } bool GpuResourceManager::IsMapped(uint64_t vaddr, uint64_t size) const noexcept { - return m_page_manager.IsMapped(vaddr, size); + if (vaddr == 0 || size == 0 || vaddr >= TRACKER_ADDRESS_SIZE || + size > TRACKER_ADDRESS_SIZE - vaddr) { + return false; + } + std::shared_lock lock(m_mapped_ranges_mutex); + return m_mapped_ranges.Contains(vaddr, size); } void GpuResourceManager::MapMemory(uint64_t vaddr, uint64_t size, GpuAccess access) { + { + std::lock_guard lock(m_mapped_ranges_mutex); + m_mapped_ranges.Add(vaddr, size); + } m_page_manager.OnGpuMap(vaddr, size, access); } @@ -120,6 +125,8 @@ void GpuResourceManager::UnmapMemory(uint64_t vaddr, uint64_t size, GpuAccess ac m_texture_cache.UnmapMemory(vaddr, size); m_buffer_cache.UnmapMemory(vaddr, size); m_page_manager.OnGpuUnmap(vaddr, size, access); + std::lock_guard lock(m_mapped_ranges_mutex); + m_mapped_ranges.Subtract(vaddr, size); }; if (m_gpu == nullptr) { if (m_resource_mutex.IsOwnedByCurrentThread()) { diff --git a/src/graphics/host_gpu/renderer/cache/gpuResourceManager.h b/src/graphics/host_gpu/renderer/cache/gpuResourceManager.h index c50897f9e..f2dc2f350 100644 --- a/src/graphics/host_gpu/renderer/cache/gpuResourceManager.h +++ b/src/graphics/host_gpu/renderer/cache/gpuResourceManager.h @@ -9,6 +9,7 @@ #include "graphics/host_gpu/renderer/cache/textureCache.h" #include +#include namespace Libs::Graphics { @@ -26,7 +27,7 @@ public: void SetGpu(Gpu* gpu) noexcept { m_gpu = gpu; } [[nodiscard]] bool HandleFault(PageFaultAccess access, uint64_t fault_vaddr) noexcept; - void PrepareHostWrite(uint64_t vaddr, uint64_t size); + [[nodiscard]] bool InvalidateMemory(uint64_t vaddr, uint64_t size); [[nodiscard]] bool IsMapped(uint64_t vaddr, uint64_t size) const noexcept; void MapMemory(uint64_t vaddr, uint64_t size, GpuAccess access); void UnmapMemory(uint64_t vaddr, uint64_t size, GpuAccess access); @@ -38,11 +39,13 @@ private: [[nodiscard]] bool InvalidateMemory(PageFaultAccess access, uint64_t vaddr, uint64_t size, PageFaultPhase phase) noexcept; - PageManager m_page_manager; - ResourceMutex m_resource_mutex; - BufferCache m_buffer_cache; - TextureCache m_texture_cache; - Gpu* m_gpu = nullptr; + PageManager m_page_manager; + ResourceMutex m_resource_mutex; + BufferCache m_buffer_cache; + TextureCache m_texture_cache; + mutable std::shared_mutex m_mapped_ranges_mutex; + RangeSet m_mapped_ranges; + Gpu* m_gpu = nullptr; }; } // namespace Libs::Graphics diff --git a/src/graphics/host_gpu/renderer/cache/textureCache.cpp b/src/graphics/host_gpu/renderer/cache/textureCache.cpp index 034f82be6..00e1e8fa7 100644 --- a/src/graphics/host_gpu/renderer/cache/textureCache.cpp +++ b/src/graphics/host_gpu/renderer/cache/textureCache.cpp @@ -1439,9 +1439,9 @@ bool TextureCache::ClearImageFromBuffer(CommandBuffer& command, uint64_t address return true; } -void TextureCache::PrepareHostWrite(uint64_t address, uint64_t size) { +void TextureCache::InvalidateMemory(uint64_t address, uint64_t size) { if (!GuestRange {address, size}.Valid()) { - EXIT("TextureCache: invalid host-write range\n"); + EXIT("TextureCache: invalid memory-invalidation range\n"); } CacheLock lock(*this, m_lock); InvalidateCpuAliases(address, size); diff --git a/src/graphics/host_gpu/renderer/cache/textureCache.h b/src/graphics/host_gpu/renderer/cache/textureCache.h index 493eb62d1..03af6c6d5 100644 --- a/src/graphics/host_gpu/renderer/cache/textureCache.h +++ b/src/graphics/host_gpu/renderer/cache/textureCache.h @@ -65,7 +65,7 @@ public: [[nodiscard]] bool ClearImageFromBuffer(CommandBuffer& command, uint64_t address, uint64_t size, uint32_t packed_clear); - void PrepareHostWrite(uint64_t address, uint64_t size); + void InvalidateMemory(uint64_t address, uint64_t size); [[nodiscard]] bool SynchronizeImageToBuffer(uint64_t address, uint64_t size); [[nodiscard]] bool InvalidateMemoryFromGPU(uint64_t address, uint64_t size, bool formatted_buffer_write = false); diff --git a/src/kernel/fileSystem.cpp b/src/kernel/fileSystem.cpp index 6f7ec7f4c..c4d5e65ad 100644 --- a/src/kernel/fileSystem.cpp +++ b/src/kernel/fileSystem.cpp @@ -238,8 +238,9 @@ static std::filesystem::path ResolvePathIgnoringCase(const std::filesystem::path } // Preserve unmatched components for the caller's ENOENT path. - std::filesystem::path resolved = path.has_root_path() ? path.root_path() : std::filesystem::path("."); - bool matched = true; + std::filesystem::path resolved = + path.has_root_path() ? path.root_path() : std::filesystem::path("."); + bool matched = true; for (const auto& component: path.relative_path()) { if (component.empty()) { @@ -568,11 +569,11 @@ int64_t KYTY_SYSV_ABI KernelRead(int d, void* buf, size_t nbytes) { file->mutex.Lock(); - bool is_invalid = file->f.IsInvalid(); - const auto pos = file->f.Tell(); - const auto file_size = file->f.Size(); - const auto remaining = pos < file_size ? file_size - pos : 0; - Memory::PrepareHostWrite(reinterpret_cast(buf), + bool is_invalid = file->f.IsInvalid(); + const auto pos = file->f.Tell(); + const auto file_size = file->f.Size(); + const auto remaining = pos < file_size ? file_size - pos : 0; + Memory::InvalidateMemory(reinterpret_cast(buf), std::min(nbytes, remaining)); uint32_t bytes_read = 0; file->f.Read(buf, static_cast(nbytes), &bytes_read); @@ -692,13 +693,12 @@ int64_t KYTY_SYSV_ABI KernelPread(int d, void* buf, size_t nbytes, int64_t offse file->mutex.Lock(); - bool is_invalid = file->f.IsInvalid(); - auto pos = file->f.Tell(); - const auto file_size = file->f.Size(); - const auto remaining = static_cast(offset) < file_size - ? file_size - static_cast(offset) - : 0; - Memory::PrepareHostWrite(reinterpret_cast(buf), + bool is_invalid = file->f.IsInvalid(); + auto pos = file->f.Tell(); + const auto file_size = file->f.Size(); + const auto remaining = + static_cast(offset) < file_size ? file_size - static_cast(offset) : 0; + Memory::InvalidateMemory(reinterpret_cast(buf), std::min(nbytes, remaining)); uint32_t bytes_read = 0; file->f.Seek(offset); diff --git a/src/kernel/memory.cpp b/src/kernel/memory.cpp index bc48c5d9c..1a912ae8a 100644 --- a/src/kernel/memory.cpp +++ b/src/kernel/memory.cpp @@ -66,8 +66,8 @@ constexpr int PAGE_TABLE_POOL_ENTRIES = static_cast(PAGE_TABLE_POOL_SIZE / PAGE_TABLE_GRANULARITY); constexpr uint64_t DEFAULT_FLEXIBLE_MEMORY_SIZE = 4ull * 1024ull * 1024ull * 1024ull; -static uint64_t g_flexible_memory_size = DEFAULT_FLEXIBLE_MEMORY_SIZE; -static Graphics::GpuResourceManager* g_gpu_resources = nullptr; +static uint64_t g_flexible_memory_size = DEFAULT_FLEXIBLE_MEMORY_SIZE; +static Graphics::GpuResourceManager* g_gpu_resources = nullptr; static Graphics::GpuResourceManager& GetGpuResources() { EXIT_IF(g_gpu_resources == nullptr); @@ -892,11 +892,11 @@ void WriteBacking(uint64_t vaddr, const void* data, uint64_t size) noexcept { } } -void PrepareHostWrite(uint64_t vaddr, uint64_t size) { +void InvalidateMemory(uint64_t vaddr, uint64_t size) { if (size == 0) { return; } - GetGpuResources().PrepareHostWrite(vaddr, size); + (void)GetGpuResources().InvalidateMemory(vaddr, size); } void InstallGpuResources(Graphics::GpuResourceManager* resources) noexcept { @@ -1915,24 +1915,24 @@ int32_t KYTY_SYSV_ABI KernelMapNamedFlexibleMemory(void** addr_in_out, size_t le EXIT_NOT_IMPLEMENTED(addr_in_out == nullptr); - constexpr size_t PAGE_SIZE = 0x4000; - constexpr size_t MAXIMUM_NAME_SIZE = 32; - constexpr uint64_t DEFAULT_PS5_BASE = 0x200000000; - constexpr int GUEST_MAP_FIXED = 0x10; - constexpr int GUEST_MAP_SHARED = 0x01; - constexpr int GUEST_MAP_PRIVATE = 0x02; - constexpr int GUEST_MAP_NO_OVERWRITE = 0x80; - constexpr int GUEST_MAP_VOID = 0x100; - constexpr int GUEST_MAP_STACK = 0x400; - constexpr int GUEST_MAP_NO_SYNC = 0x800; - constexpr int GUEST_MAP_ANON = 0x1000; - constexpr int GUEST_MAP_UNKNOWN_8000 = 0x8000; - constexpr int GUEST_MAP_NO_CORE = 0x20000; - constexpr int GUEST_MAP_NO_COALESCE = 0x400000; - constexpr int SUPPORTED_MAP_BITS = - GUEST_MAP_SHARED | GUEST_MAP_PRIVATE | GUEST_MAP_FIXED | GUEST_MAP_NO_OVERWRITE | - GUEST_MAP_VOID | GUEST_MAP_STACK | GUEST_MAP_NO_SYNC | GUEST_MAP_ANON | - GUEST_MAP_UNKNOWN_8000 | GUEST_MAP_NO_CORE | GUEST_MAP_NO_COALESCE; + constexpr size_t PAGE_SIZE = 0x4000; + constexpr size_t MAXIMUM_NAME_SIZE = 32; + constexpr uint64_t DEFAULT_PS5_BASE = 0x200000000; + constexpr int GUEST_MAP_FIXED = 0x10; + constexpr int GUEST_MAP_SHARED = 0x01; + constexpr int GUEST_MAP_PRIVATE = 0x02; + constexpr int GUEST_MAP_NO_OVERWRITE = 0x80; + constexpr int GUEST_MAP_VOID = 0x100; + constexpr int GUEST_MAP_STACK = 0x400; + constexpr int GUEST_MAP_NO_SYNC = 0x800; + constexpr int GUEST_MAP_ANON = 0x1000; + constexpr int GUEST_MAP_UNKNOWN_8000 = 0x8000; + constexpr int GUEST_MAP_NO_CORE = 0x20000; + constexpr int GUEST_MAP_NO_COALESCE = 0x400000; + constexpr int SUPPORTED_MAP_BITS = GUEST_MAP_SHARED | GUEST_MAP_PRIVATE | GUEST_MAP_FIXED | + GUEST_MAP_NO_OVERWRITE | GUEST_MAP_VOID | GUEST_MAP_STACK | + GUEST_MAP_NO_SYNC | GUEST_MAP_ANON | GUEST_MAP_UNKNOWN_8000 | + GUEST_MAP_NO_CORE | GUEST_MAP_NO_COALESCE; if (len == 0 || (len & (PAGE_SIZE - 1)) != 0) { return KERNEL_ERROR_EINVAL; @@ -3294,7 +3294,7 @@ int KYTY_SYSV_ABI KernelReserveVirtualRange(void** addr, size_t len, int flags, "\t alignment = 0x%016" PRIx64 "\n", in_addr, len, flags, alignment); - constexpr size_t PAGE_SIZE = 0x4000; + constexpr size_t PAGE_SIZE = 0x4000; constexpr int GUEST_MAP_FIXED = 0x10; constexpr int GUEST_MAP_NO_OVERWRITE = 0x80; diff --git a/src/kernel/memory.h b/src/kernel/memory.h index 8b31a7e3d..448d89323 100644 --- a/src/kernel/memory.h +++ b/src/kernel/memory.h @@ -99,13 +99,13 @@ struct KernelMemoryPoolBlockStats { static_assert(sizeof(KernelMemoryPoolBlockStats) == 16, "KernelMemoryPoolBlockStats struct size is incorrect"); -void RegisterCallbacks(callback_func_t alloc_func, callback_func_t free_func); -void SetFlexibleMemorySize(uint64_t size); -bool TryWriteBacking(uint64_t vaddr, const void* data, uint64_t size); -bool TryReadBacking(uint64_t vaddr, void* data, uint64_t size); -void WriteBacking(uint64_t vaddr, const void* data, uint64_t size) noexcept; -void PrepareHostWrite(uint64_t vaddr, uint64_t size); -void InstallGpuResources(Graphics::GpuResourceManager* resources) noexcept; +void RegisterCallbacks(callback_func_t alloc_func, callback_func_t free_func); +void SetFlexibleMemorySize(uint64_t size); +bool TryWriteBacking(uint64_t vaddr, const void* data, uint64_t size); +bool TryReadBacking(uint64_t vaddr, void* data, uint64_t size); +void WriteBacking(uint64_t vaddr, const void* data, uint64_t size) noexcept; +void InvalidateMemory(uint64_t vaddr, uint64_t size); +void InstallGpuResources(Graphics::GpuResourceManager* resources) noexcept; [[nodiscard]] bool HandleGpuFault(Graphics::PageFaultAccess access, uint64_t fault_vaddr) noexcept; int KYTY_SYSV_ABI KernelMapNamedFlexibleMemory(void** addr_in_out, size_t len, int prot, int flags, diff --git a/tests/MemoryTrackerTests.cpp b/tests/MemoryTrackerTests.cpp index d6861087a..78e55d818 100644 --- a/tests/MemoryTrackerTests.cpp +++ b/tests/MemoryTrackerTests.cpp @@ -669,6 +669,8 @@ void TestRangeSet() { ranges.Add(0x1000, 0x80); ranges.Add(0x1080, 0x80); ranges.Add(0x1200, 0x40); + Check(ranges.Contains(0x1010, 0xe0) && !ranges.Contains(0x1010, 0x200), + "range set containment did not require full coverage"); auto intersections = ranges.Intersections(0x1070, 0x1b0); Check(intersections.size() == 2 && intersections[0].address == 0x1070 && intersections[0].size == 0x90 && intersections[1].address == 0x1200 && @@ -682,6 +684,46 @@ void TestRangeSet() { "range set subtraction did not preserve both exact tails"); } +void TestRangeInvalidation() { + constexpr uintptr_t base = 0x0000000201000000ull; + TrackerHarness harness; + auto &tracker = harness.tracker; + auto &page_manager = harness.page_manager; + constexpr uint64_t size = Libs::Graphics::TRACKER_REGION_SIZE * 2; + auto *memory = static_cast( + VirtualAlloc(reinterpret_cast(base), size, MEM_RESERVE | MEM_COMMIT, + PAGE_READWRITE)); + Check(memory == reinterpret_cast(base), + "range invalidation allocation failed"); + const auto address = reinterpret_cast(memory); + page_manager.OnGpuMap(address, size); + + tracker.ForEachUploadRange( + address, size, true, [](uint64_t, uint64_t) noexcept {}, + []() noexcept {}); + Check(tracker.IsRegionGpuModified(address, size) && !IsWritable(memory), + "range invalidation setup did not establish GPU ownership"); + + uint32_t flushes = 0; + tracker.InvalidateRegion(address + 16, size - 32, [&] { + flushes++; + tracker.ForEachDownloadRange( + address + 16, size - 32, [](uint64_t, uint64_t) noexcept {}); + }); + Check(flushes == 1 && !tracker.IsRegionGpuModified(address, size) && + tracker.IsRegionCpuModified(address, size) && IsWritable(memory) && + IsWritable(memory + size - 1), + "range invalidation did not batch ownership transfer across regions"); + + tracker.InvalidateRegion(address + 16, size - 32, [&] { flushes++; }); + Check(flushes == 1, + "clean range invalidation unnecessarily requested a GPU flush"); + tracker.UntrackMemory(address, size); + page_manager.OnGpuUnmap(address, size); + Check(VirtualFree(memory, 0, MEM_RELEASE) != 0, + "range invalidation VirtualFree failed"); +} + void TestCpuDirtyUploadAndFault() { constexpr uintptr_t base = 0x0000000200010000ull; TrackerHarness harness; @@ -1207,6 +1249,7 @@ int main(int argc, char **argv) { TestSameSlabTrackerArbitration(); TestSharedMetadataAndImagePageFault(); TestRangeSet(); + TestRangeInvalidation(); TestGpuDirtyBits(); TestCrossRegionUpload(); TestFaultDuringUploadRemainsDirty(); diff --git a/tests/PageManagerTests.cpp b/tests/PageManagerTests.cpp index 054262030..a0c02ed78 100644 --- a/tests/PageManagerTests.cpp +++ b/tests/PageManagerTests.cpp @@ -301,36 +301,6 @@ void TestSharedWatcherFault() { Check(VirtualFree(memory, 0, MEM_RELEASE) != 0, "VirtualFree failed"); } -void TestMappedHostWriteRange() { - FaultContext context; - PageManager manager(InvalidateFault, &context); - context.manager = &manager; - const auto page_size = manager.GetPageSize(); - auto *memory = Allocate(page_size * 3); - const auto address = reinterpret_cast(memory); - - manager.OnGpuMap(address, page_size); - manager.OnGpuMap(address + page_size * 2, page_size); - manager.UpdatePageWatchers(true, address, page_size); - manager.UpdatePageWatchers(true, address + page_size * 2, page_size); - Check(manager.HasAnyMapping(address + 16, page_size * 3 - 32), - "host-write range did not find partial GPU mappings"); - Check(!manager.IsMapped(address, page_size * 3), - "partial GPU mappings were reported as a full mapping"); - Check(manager.HandleWriteRange(address + 16, page_size * 3 - 32), - "mapped host-write range was not handled"); - Check(context.calls.load(std::memory_order_relaxed) == 2, - "host-write range did not invalidate each mapped watched page"); - Check(IsWritable(memory) && IsWritable(memory + page_size * 2), - "host-write range did not restore writable protection"); - - manager.OnGpuUnmap(address, page_size); - manager.OnGpuUnmap(address + page_size * 2, page_size); - Check(!manager.HasAnyMapping(address, page_size * 3), - "host-write range retained stale GPU mappings"); - Check(VirtualFree(memory, 0, MEM_RELEASE) != 0, "VirtualFree failed"); -} - void TestReadWriteWatcherFault() { FaultContext context; PageManager manager(InvalidateFault, &context); @@ -855,7 +825,6 @@ int main(int argc, char **argv) { } TestWatchFaultAndUnwatch(); TestSharedWatcherFault(); - TestMappedHostWriteRange(); TestReadWriteWatcherFault(); TestPermittedMappedLateFaultsResume(); TestPartialMappingUnmapPreservesTokens(); diff --git a/tests/ShaderRecompilerComputeTests.cpp b/tests/ShaderRecompilerComputeTests.cpp index db7c99765..e6e4b052b 100644 --- a/tests/ShaderRecompilerComputeTests.cpp +++ b/tests/ShaderRecompilerComputeTests.cpp @@ -1679,7 +1679,9 @@ public: Require("GpuCommandLane", "processor fault context", Gpu::CurrentCommandProcessor() == &processor, "processor resource test lost its command context"); - resources.PrepareHostWrite(fault_base, sizeof(uint32_t)); + Require("GpuCommandLane", "processor memory invalidation", + resources.InvalidateMemory(fault_base, sizeof(uint32_t)), + "processor memory invalidation did not find its mapped range"); }); resources.UnmapMemory(fault_base, fault_size, GpuAccess::ReadWrite); Require("GpuCommandLane", "processor fault unmap",