diff --git a/src/CMakeLists.txt b/src/CMakeLists.txt index b77704ef9..79e19f6e6 100644 --- a/src/CMakeLists.txt +++ b/src/CMakeLists.txt @@ -289,6 +289,11 @@ add_executable(resource_mutex_tests EXCLUDE_FROM_ALL target_link_libraries(resource_mutex_tests common) target_include_directories(resource_mutex_tests PRIVATE ${inc_headers}) +add_executable(image_page_table_tests EXCLUDE_FROM_ALL + ../tests/ImagePageTableTests.cpp +) +target_include_directories(image_page_table_tests PRIVATE ${inc_headers}) + add_kyty_full_emulator_test(shader_recompiler_compute_tests ../tests/ShaderRecompilerComputeTests.cpp) add_kyty_full_emulator_test(virtual_memory_allocation_tests ../tests/VirtualMemoryAllocationTests.cpp) diff --git a/src/graphics/host_gpu/renderer/image.h b/src/graphics/host_gpu/renderer/image.h index 8099e204e..5a506c85e 100644 --- a/src/graphics/host_gpu/renderer/image.h +++ b/src/graphics/host_gpu/renderer/image.h @@ -4,6 +4,7 @@ #include "common/assert.h" #include "graphics/host_gpu/renderer/imageInfo.h" +#include #include #include @@ -11,30 +12,83 @@ namespace Libs::Graphics { struct Image final: ImageInfo { Image& operator=(const ImageInfo& value) { - if (m_cpu_dirty) { + if (IsCpuDirty()) { EXIT("dirty sampled image cannot be reassigned\n"); } static_cast(*this) = value; + m_track_begin = address; + m_track_end = address + size; + m_maybe_cpu_hash_valid = false; return *this; } void InvalidateCpuWrite(uint64_t vaddr, uint64_t size) { - if (ImagePageRangesOverlap(address, this->size, vaddr, size)) { + if (ImageRangeOverlaps(address, this->size, vaddr, size)) { m_cpu_dirty = true; + m_maybe_cpu_dirty = false; + m_maybe_cpu_hash_valid = false; + m_track_begin = m_track_end; + } else if (ImagePageRangesOverlap(address, this->size, vaddr, size)) { + constexpr uint64_t page_mask = 4096 - 1; + if (vaddr + size <= address) { + const auto next_page = (address + page_mask) & ~page_mask; + m_track_begin = std::min(m_track_end, std::max(m_track_begin, next_page)); + } else if (vaddr >= address + this->size) { + const auto page = (address + this->size) & ~page_mask; + m_track_end = std::max(m_track_begin, std::min(m_track_end, page)); + } + m_maybe_cpu_dirty = m_track_begin == m_track_end; } } - [[nodiscard]] bool IsCpuDirty() const { return m_cpu_dirty; } + [[nodiscard]] bool IsCpuDirty() const { return m_cpu_dirty || m_maybe_cpu_dirty; } + [[nodiscard]] bool IsDefinitelyCpuDirty() const { return m_cpu_dirty; } + [[nodiscard]] bool IsMaybeCpuDirty() const { return m_maybe_cpu_dirty; } + [[nodiscard]] bool NeedsMaybeCpuHash() const { + return m_maybe_cpu_dirty && !m_maybe_cpu_hash_valid; + } + [[nodiscard]] bool IsCpuTrackingComplete() const { + return m_track_begin == address && m_track_end == address + size; + } + void SetMaybeCpuHash(uint64_t hash) { + if (!NeedsMaybeCpuHash()) { + EXIT("sampled image cannot initialize maybe-dirty hash\n"); + } + m_maybe_cpu_hash = hash; + m_maybe_cpu_hash_valid = true; + } + [[nodiscard]] bool ResolveMaybeCpuHash(uint64_t hash) { + if (!m_maybe_cpu_dirty || !m_maybe_cpu_hash_valid || m_cpu_dirty) { + EXIT("sampled image cannot resolve maybe-dirty hash\n"); + } + m_maybe_cpu_dirty = false; + m_maybe_cpu_hash_valid = false; + m_cpu_dirty = hash != m_maybe_cpu_hash; + if (!m_cpu_dirty) { + m_track_begin = address; + m_track_end = address + size; + } + return m_cpu_dirty; + } void RefreshComplete() { - if (!m_cpu_dirty) { + if (!IsCpuDirty()) { EXIT("clean sampled image cannot complete a refresh\n"); } m_cpu_dirty = false; + m_maybe_cpu_dirty = false; + m_maybe_cpu_hash_valid = false; + m_track_begin = address; + m_track_end = address + size; } private: bool m_cpu_dirty = false; + bool m_maybe_cpu_dirty = false; + bool m_maybe_cpu_hash_valid = false; + uint64_t m_track_begin = 0; + uint64_t m_track_end = 0; + uint64_t m_maybe_cpu_hash = 0; }; struct ImageRetirementRange { @@ -60,8 +114,8 @@ FindImageRetirementConflict(std::span ranges) { if (ranges[retained].retire) { continue; } - if (ImagePageRangesOverlap(ranges[retired].address, ranges[retired].size, - ranges[retained].address, ranges[retained].size)) { + if (ImageRangeOverlaps(ranges[retired].address, ranges[retired].size, + ranges[retained].address, ranges[retained].size)) { return {retired, retained}; } } diff --git a/src/graphics/host_gpu/renderer/imageInfo.h b/src/graphics/host_gpu/renderer/imageInfo.h index 954076c44..5fcec1981 100644 --- a/src/graphics/host_gpu/renderer/imageInfo.h +++ b/src/graphics/host_gpu/renderer/imageInfo.h @@ -821,14 +821,11 @@ ClassifyRenderTargetOverlap(const ImageInfo& sampled, bool sampled_gpu_modified, if (!ImagePageRangesOverlap(sampled.address, sampled.size, target.address, target.size)) { return RenderTargetOverlap::None; } - const bool exact = sampled.address == target.address && sampled.size == target.size; - const bool page_isolated_overlap = - sampled.address % TRACKER_PAGE_SIZE == 0 && sampled.size % TRACKER_PAGE_SIZE == 0 && - target.address % TRACKER_PAGE_SIZE == 0 && target.size % TRACKER_PAGE_SIZE == 0 && - ImageRangeOverlaps(sampled.address, sampled.size, target.address, target.size); - return !sampled_gpu_modified && same_context && (exact || page_isolated_overlap) - ? RenderTargetOverlap::RetireSampled - : RenderTargetOverlap::Unsupported; + if (!ImageRangeOverlaps(sampled.address, sampled.size, target.address, target.size)) { + return RenderTargetOverlap::None; + } + return !sampled_gpu_modified && same_context ? RenderTargetOverlap::RetireSampled + : RenderTargetOverlap::Unsupported; } [[nodiscard]] inline RenderTargetOverlap @@ -886,6 +883,9 @@ ClassifySampledRenderTargetOverlap(const ImageInfo& sampled, const RenderTargetI if (!ImagePageRangesOverlap(sampled.address, sampled.size, target.address, target.size)) { return RenderTargetOverlap::None; } + if (!ImageRangeOverlaps(sampled.address, sampled.size, target.address, target.size)) { + return RenderTargetOverlap::None; + } return same_context && !target_buffer_modified ? RenderTargetOverlap::RetireTarget : RenderTargetOverlap::Unsupported; } diff --git a/src/graphics/host_gpu/renderer/multiLevelPageTable.h b/src/graphics/host_gpu/renderer/multiLevelPageTable.h new file mode 100644 index 000000000..3c74c4b75 --- /dev/null +++ b/src/graphics/host_gpu/renderer/multiLevelPageTable.h @@ -0,0 +1,342 @@ +#ifndef EMULATOR_SRC_GRAPHICS_HOST_GPU_RENDERER_MULTILEVELPAGETABLE_H_ +#define EMULATOR_SRC_GRAPHICS_HOST_GPU_RENDERER_MULTILEVELPAGETABLE_H_ + +#include +#include +#include +#include +#include +#include +#include +#include +#include + +namespace Libs::Graphics { + +// Sparse two-level lookup for texture-cache page ownership. +// Entries are selected by guest page number; queries never allocate L1 buckets. +template +class MultiLevelPageTable final { +public: + using Entry = EntryT; + + static_assert(AddressSpaceBits < 64); + static_assert(PageBits < AddressSpaceBits); + static_assert(FirstLevelBits > 0 && FirstLevelBits <= AddressSpaceBits - PageBits); + + static constexpr size_t kPageBits = PageBits; + static constexpr size_t kAddressSpaceBits = AddressSpaceBits; + static constexpr size_t kFirstLevelBits = FirstLevelBits; + static constexpr size_t kSecondLevelBits = AddressSpaceBits - FirstLevelBits - PageBits; + static constexpr size_t kFirstLevelEntries = size_t{1} << FirstLevelBits; + static constexpr size_t kBucketEntries = size_t{1} << kSecondLevelBits; + static constexpr size_t kPageCount = size_t{1} << (AddressSpaceBits - PageBits); + static constexpr uint64_t kAddressSpaceSize = uint64_t{1} << AddressSpaceBits; + + struct PageRange final { + size_t first = 0; + size_t last_exclusive = 0; + }; + + MultiLevelPageTable(): m_first_level(kFirstLevelEntries) {} + + [[nodiscard]] Entry* Find(size_t page) { + if (!IsValidPage(page)) { + return nullptr; + } + auto& bucket = m_first_level[FirstLevelIndex(page)]; + return bucket == nullptr ? nullptr : &(*bucket)[SecondLevelIndex(page)]; + } + + [[nodiscard]] const Entry* Find(size_t page) const { + if (!IsValidPage(page)) { + return nullptr; + } + const auto& bucket = m_first_level[FirstLevelIndex(page)]; + return bucket == nullptr ? nullptr : &(*bucket)[SecondLevelIndex(page)]; + } + + [[nodiscard]] Entry& GetOrCreate(size_t page) { + if (!IsValidPage(page)) { + throw std::out_of_range("MultiLevelPageTable page is outside the guest address space"); + } + auto& bucket = m_first_level[FirstLevelIndex(page)]; + if (bucket == nullptr) { + bucket = std::make_unique(); + ++m_allocated_buckets; + } + return (*bucket)[SecondLevelIndex(page)]; + } + + [[nodiscard]] Entry& operator[](size_t page) { return GetOrCreate(page); } + + [[nodiscard]] static constexpr bool IsValidPage(size_t page) { return page < kPageCount; } + + // Returns the half-open page interval touched by a non-empty byte range. + // An end exactly at 2^AddressSpaceBits is valid; wrapping or crossing it is not. + [[nodiscard]] static constexpr bool TryGetPageRange(uint64_t address, uint64_t size, PageRange& range) { + if (size == 0 || address >= kAddressSpaceSize || size > kAddressSpaceSize - address) { + return false; + } + const uint64_t end = address + size; + range.first = static_cast(address >> PageBits); + range.last_exclusive = static_cast(((end - 1) >> PageBits) + 1); + return true; + } + + [[nodiscard]] size_t AllocatedBucketCount() const { return m_allocated_buckets; } + +private: + using Bucket = std::array; + + [[nodiscard]] static constexpr size_t FirstLevelIndex(size_t page) { return page >> kSecondLevelBits; } + [[nodiscard]] static constexpr size_t SecondLevelIndex(size_t page) { return page & (kBucketEntries - 1); } + + std::vector> m_first_level; + size_t m_allocated_buckets = 0; +}; + +// Removes only the requested owner. Other owners in the same page entry remain intact. +template +[[nodiscard]] bool EraseExact(Container& owners, const Value& owner) { + const auto it = std::find(owners.begin(), owners.end(), owner); + if (it == owners.end()) { + return false; + } + owners.erase(it); + return true; +} + +// Multi-range ownership with 1 MiB candidate buckets and precise 4 KiB +// lifetime accounting. OwnerT only needs equality. +template +class MultiRangePageOwnerIndex final { +private: + struct Registration; + using MembershipList = std::vector; + +public: + struct ByteRange final { + uint64_t address = 0; + uint64_t size = 0; + }; + + using CoarseTable = MultiLevelPageTable; + using TrackingTable = MultiLevelPageTable; + + [[nodiscard]] bool Register(const OwnerT& owner, const std::vector& ranges) { + if (FindRegistration(owner) != m_registrations.end()) { + return false; + } + auto normalized = Normalize(ranges); + if (normalized.empty()) { + return false; + } + m_registrations.push_back({owner, std::move(normalized)}); + const Registration* registration = &m_registrations.back(); + for (const size_t page: CollectPages<20>(registration->ranges)) { + m_coarse_pages[page].push_back(registration); + } + for (const size_t page: CollectPages<12>(registration->ranges)) { + m_tracking_pages[page].push_back(registration); + } + return true; + } + + // No state changes if an expected membership is absent. Final releases are + // sorted and coalesced 4 KiB spans whose final owner disappeared. + [[nodiscard]] bool Unregister(const OwnerT& owner, std::vector& final_releases) { + final_releases.clear(); + auto registration = FindRegistration(owner); + if (registration == m_registrations.end()) { + return false; + } + const auto coarse_pages = CollectPages<20>(registration->ranges); + const auto tracking_pages = CollectPages<12>(registration->ranges); + const Registration* registration_ptr = &*registration; + if (!HasAllMemberships(m_coarse_pages, coarse_pages, registration_ptr) || + !HasAllMemberships(m_tracking_pages, tracking_pages, registration_ptr)) { + return false; + } + std::vector final_pages; + for (const size_t page: coarse_pages) { + (void)EraseExact(*m_coarse_pages.Find(page), registration_ptr); + } + for (const size_t page: tracking_pages) { + auto& owners = *m_tracking_pages.Find(page); + if (owners.size() == 1) { + final_pages.push_back(page); + } + (void)EraseExact(owners, registration_ptr); + } + m_registrations.erase(registration); + final_releases = CoalesceTrackingPages(final_pages); + return true; + } + + [[nodiscard]] std::vector Query(uint64_t address, uint64_t size) const { + return Query(address, size, [](const OwnerT&) { return true; }); + } + + template + [[nodiscard]] std::vector Query(uint64_t address, uint64_t size, Predicate&& predicate) const { + return QueryImpl(address, size, true, std::forward(predicate)); + } + + // Fault paths use page candidates: exact byte-disjoint owners sharing a + // touched 4 KiB page are intentionally retained. + [[nodiscard]] std::vector QueryCandidates(uint64_t address, uint64_t size) const { + return QueryCandidates(address, size, [](const OwnerT&) { return true; }); + } + + template + [[nodiscard]] std::vector QueryCandidates(uint64_t address, uint64_t size, Predicate&& predicate) const { + return QueryImpl(address, size, false, std::forward(predicate)); + } + + [[nodiscard]] size_t CoarseMembershipCount(size_t page) const { + const auto* owners = m_coarse_pages.Find(page); + return owners == nullptr ? 0 : owners->size(); + } + [[nodiscard]] size_t TrackingMembershipCount(size_t page) const { + const auto* owners = m_tracking_pages.Find(page); + return owners == nullptr ? 0 : owners->size(); + } + +private: + template + [[nodiscard]] std::vector QueryImpl(uint64_t address, uint64_t size, bool strict_bytes, + Predicate&& predicate) const { + typename CoarseTable::PageRange coarse_range{}; + typename TrackingTable::PageRange tracking_range{}; + if (!CoarseTable::TryGetPageRange(address, size, coarse_range) || + !TrackingTable::TryGetPageRange(address, size, tracking_range)) { + return {}; + } + MembershipList candidates; + for (size_t page = coarse_range.first; page < coarse_range.last_exclusive; ++page) { + if (const auto* owners = m_coarse_pages.Find(page); owners != nullptr) { + AppendUnique(candidates, *owners); + } + } + std::vector result; + for (const Registration* registration: candidates) { + if ((!strict_bytes || Overlaps(registration->ranges, address, size)) && + HasTrackingMembership(registration, tracking_range) && predicate(registration->owner)) { + result.push_back(registration->owner); + } + } + return result; + } + + struct Registration final { + OwnerT owner; + std::vector ranges; + }; + using RegistrationIterator = typename std::list::iterator; + using ConstRegistrationIterator = typename std::list::const_iterator; + + [[nodiscard]] RegistrationIterator FindRegistration(const OwnerT& owner) { + return std::find_if(m_registrations.begin(), m_registrations.end(), + [&](const Registration& item) { return item.owner == owner; }); + } + [[nodiscard]] ConstRegistrationIterator FindRegistration(const OwnerT& owner) const { + return std::find_if(m_registrations.begin(), m_registrations.end(), + [&](const Registration& item) { return item.owner == owner; }); + } + + [[nodiscard]] static std::vector Normalize(const std::vector& ranges) { + std::vector sorted; + for (const auto& range: ranges) { + typename CoarseTable::PageRange ignored{}; + if (!CoarseTable::TryGetPageRange(range.address, range.size, ignored)) { + return {}; + } + sorted.push_back(range); + } + std::sort(sorted.begin(), sorted.end(), + [](const ByteRange& lhs, const ByteRange& rhs) { return lhs.address < rhs.address; }); + std::vector merged; + for (const auto& range: sorted) { + if (merged.empty() || range.address > merged.back().address + merged.back().size) { + merged.push_back(range); + } else { + const uint64_t end = std::max(merged.back().address + merged.back().size, range.address + range.size); + merged.back().size = end - merged.back().address; + } + } + return merged; + } + + template + [[nodiscard]] static std::vector CollectPages(const std::vector& ranges) { + std::vector pages; + for (const auto& range: ranges) { + const size_t first = static_cast(range.address >> Bits); + const size_t last = static_cast((range.address + range.size - 1) >> Bits); + for (size_t page = first; page <= last; ++page) { + pages.push_back(page); + } + } + std::sort(pages.begin(), pages.end()); + pages.erase(std::unique(pages.begin(), pages.end()), pages.end()); + return pages; + } + + template + [[nodiscard]] static bool HasAllMemberships(const Table& table, const std::vector& pages, + const Registration* registration) { + for (const size_t page: pages) { + const auto* owners = table.Find(page); + if (owners == nullptr || std::find(owners->begin(), owners->end(), registration) == owners->end()) { + return false; + } + } + return true; + } + + [[nodiscard]] bool HasTrackingMembership(const Registration* registration, + const typename TrackingTable::PageRange& range) const { + for (size_t page = range.first; page < range.last_exclusive; ++page) { + const auto* owners = m_tracking_pages.Find(page); + if (owners != nullptr && std::find(owners->begin(), owners->end(), registration) != owners->end()) { + return true; + } + } + return false; + } + + [[nodiscard]] static bool Overlaps(const std::vector& ranges, uint64_t address, uint64_t size) { + const uint64_t end = address + size; + return std::any_of(ranges.begin(), ranges.end(), [&](const ByteRange& range) { + return range.address < end && address < range.address + range.size; + }); + } + static void AppendUnique(MembershipList& destination, const MembershipList& source) { + for (const Registration* registration: source) { + if (std::find(destination.begin(), destination.end(), registration) == destination.end()) { + destination.push_back(registration); + } + } + } + [[nodiscard]] static std::vector CoalesceTrackingPages(const std::vector& pages) { + std::vector result; + for (const size_t page: pages) { + const uint64_t address = static_cast(page) << 12; + if (!result.empty() && result.back().address + result.back().size == address) { + result.back().size += uint64_t{1} << 12; + } else { + result.push_back({address, uint64_t{1} << 12}); + } + } + return result; + } + + CoarseTable m_coarse_pages; + TrackingTable m_tracking_pages; + std::list m_registrations; +}; + +} // namespace Libs::Graphics + +#endif // EMULATOR_SRC_GRAPHICS_HOST_GPU_RENDERER_MULTILEVELPAGETABLE_H_ diff --git a/src/graphics/host_gpu/renderer/textureCache.cpp b/src/graphics/host_gpu/renderer/textureCache.cpp index 375b8b2e4..588615d0a 100644 --- a/src/graphics/host_gpu/renderer/textureCache.cpp +++ b/src/graphics/host_gpu/renderer/textureCache.cpp @@ -22,6 +22,8 @@ #include "graphics/shader/shader.h" #include "kernel/memory.h" +#include + #include #include #include @@ -37,6 +39,29 @@ namespace { thread_local const void* g_texture_cache_lock_owner = nullptr; thread_local const void* g_texture_fault_owner = nullptr; +[[nodiscard]] uint64_t HashSampledImageEdges(const Image& image) { + constexpr uint64_t page_mask = TRACKER_PAGE_SIZE - 1; + const uint64_t begin = image.address; + const uint64_t end = image.address + image.size; + const uint64_t head_end = std::min(end, (begin + page_mask) & ~page_mask); + const uint64_t tail_begin = std::max(begin, end & ~page_mask); + const uint64_t head_size = head_end - begin; + const uint64_t tail_size = tail_begin < head_end ? end - head_end : end - tail_begin; + std::array bytes {}; + if (head_size + tail_size > bytes.size()) { + EXIT("TextureCache: sampled-image edge hash range overflow\n"); + } + if ((head_size != 0 && !LibKernel::Memory::TryReadBacking(begin, bytes.data(), head_size)) || + (tail_size != 0 && + !LibKernel::Memory::TryReadBacking(tail_begin < head_end ? head_end : tail_begin, + bytes.data() + head_size, tail_size))) { + EXIT("TextureCache: failed to hash sampled-image edge backing, addr=0x%016" PRIx64 + " size=0x%016" PRIx64 "\n", + image.address, image.size); + } + return XXH3_64bits(bytes.data(), static_cast(head_size + tail_size)); +} + class FaultSafeTextureLock final { public: FaultSafeTextureLock(const void* owner, TrackingSpinLock& mutex): m_mutex(mutex) { @@ -87,13 +112,14 @@ struct TextureCache::CachedImage { bool gpu_modified = false; bool buffer_modified = false; bool stencil_initialized = false; + bool registered = false; ~CachedImage() { - if (ctx == nullptr || image == nullptr) { + if (ctx == nullptr || image == nullptr || registered) { EXIT("TextureCache: cached image destroyed with invalid resources, ctx=%p image=%p " - "kind=%u\n", + "kind=%u registered=%d\n", static_cast(ctx), static_cast(image), - static_cast(kind)); + static_cast(kind), registered); } switch (kind) { case Kind::Texture: @@ -191,6 +217,43 @@ struct TextureCache::CachedImage { } }; +void TextureCache::RegisterImageLocked(CachedImage& image) { + if (image.registered) { + EXIT("TextureCache: registering an already registered image\n"); + } + std::vector ranges; + ranges.reserve(image.RangeCount()); + for (uint32_t range = 0; range < image.RangeCount(); range++) { + ranges.push_back({image.Address(range), image.Size(range)}); + } + if (!m_image_owner_index.Register(&image, ranges)) { + EXIT("TextureCache: invalid or duplicate image registration\n"); + } + image.registered = true; +} + +void TextureCache::UnregisterImageLocked(CachedImage& image, bool release_tracking) { + if (!image.registered) { + EXIT("TextureCache: unregistering an unregistered image\n"); + } + std::vector final_releases; + if (!m_image_owner_index.Unregister(&image, final_releases)) { + EXIT("TextureCache: image missing from owner index\n"); + } + if (release_tracking) { + for (const auto& release: final_releases) { + m_memory_tracker.UntrackMemory(release.address, release.size); + } + } + image.registered = false; +} + +std::vector +TextureCache::FindImagesInRegionLocked(uint64_t vaddr, uint64_t size, bool page_overlap) { + return page_overlap ? m_image_owner_index.QueryCandidates(vaddr, size) + : m_image_owner_index.Query(vaddr, size); +} + struct TextureCache::ReadbackWorker { enum class State : uint32_t { Idle, @@ -653,12 +716,10 @@ void TextureCache::RetireImages(const std::vector& retire, "addr=0x%016" PRIx64 " size=0x%016" PRIx64 "\n", (*it)->Address(range), (*it)->Size(range)); } - // A native image transition transfers this registration to the replacement cache - // record. Keeping it GPU-owned also keeps guest page protection continuous. - if (!native_image) { - m_memory_tracker.UntrackMemory((*it)->Address(range), (*it)->Size(range)); - } } + // A native transition keeps tracker ownership continuous until the replacement is + // published. Normal retirement only releases 4 KiB pages with no remaining image owner. + UnregisterImageLocked(**it, !native_image); it = m_images.erase(it); removed++; } @@ -1500,9 +1561,9 @@ VkImageView TextureCache::GetStorageTextureStorageView(GraphicContext* } TextureCache::CachedImage* TextureCache::FindGpuReadbackPageCandidateLocked(uint64_t vaddr, - uint64_t size) const { + uint64_t size) { CachedImage* selected = nullptr; - for (const auto& cached: m_images) { + for (auto* cached: FindImagesInRegionLocked(vaddr, size, true)) { if (!cached->IsGpuReadbackPageCandidate(vaddr, size)) { continue; } @@ -1510,33 +1571,36 @@ TextureCache::CachedImage* TextureCache::FindGpuReadbackPageCandidateLocked(uint EXIT("TextureCache: CPU fault has multiple GPU-modified image page candidates, " "addr=0x%016" PRIx64 " size=0x%016" PRIx64 " first=%p second=%p\n", vaddr, size, static_cast(selected), - static_cast(cached.get())); + static_cast(cached)); } - selected = cached.get(); + selected = cached; } return selected; } void TextureCache::MarkSampledAliasesCpuDirtyLocked(uint64_t vaddr, uint64_t size) { - for (auto& cached: m_images) { + for (auto* cached: FindImagesInRegionLocked(vaddr, size, true)) { if (cached->kind != CachedImage::Kind::Texture) { continue; } if (cached->gpu_modified && - ImagePageRangesOverlap(cached->info.address, cached->info.size, vaddr, size)) { + ImageRangeOverlaps(cached->info.address, cached->info.size, vaddr, size)) { EXIT("TextureCache: CPU write overlaps a GPU-modified sampled texture, " "write=0x%016" PRIx64 "+0x%016" PRIx64 " image=0x%016" PRIx64 "+0x%016" PRIx64 "\n", vaddr, size, cached->info.address, cached->info.size); } cached->info.InvalidateCpuWrite(vaddr, size); + if (cached->info.NeedsMaybeCpuHash()) { + cached->info.SetMaybeCpuHash(HashSampledImageEdges(cached->info)); + } } } void TextureCache::RetireSampledTargetAliases(GraphicContext* ctx, const ImageInfo& requested) { std::vector retire; bool wait_idle = false; - for (const auto& cached: m_images) { + for (auto* cached: FindImagesInRegionLocked(requested.address, requested.size, true)) { if (cached->kind != CachedImage::Kind::RenderTarget) { continue; } @@ -1545,7 +1609,7 @@ void TextureCache::RetireSampledTargetAliases(GraphicContext* ctx, const ImageIn case RenderTargetOverlap::None: continue; case RenderTargetOverlap::RetireTarget: wait_idle |= cached->gpu_modified; - retire.push_back(cached.get()); + retire.push_back(cached); break; case RenderTargetOverlap::RetireSampled: case RenderTargetOverlap::PreserveStorage: @@ -1558,13 +1622,13 @@ void TextureCache::RetireSampledTargetAliases(GraphicContext* ctx, const ImageIn cached->ctx == ctx, cached->gpu_modified, cached->buffer_modified); } } - for (const auto& cached: m_images) { + for (auto* cached: FindImagesInRegionLocked(requested.address, requested.size, true)) { if (cached->kind != CachedImage::Kind::DepthTarget) { continue; } const auto overlap = ClassifyImageRangeOverlap(requested.address, requested.size, cached->depth.address, cached->depth.size); - if (overlap == ImageRangeOverlap::None) { + if (overlap == ImageRangeOverlap::None || overlap == ImageRangeOverlap::PageOnly) { continue; } const auto delta = cached->depth.address >= requested.address @@ -1578,8 +1642,7 @@ void TextureCache::RetireSampledTargetAliases(GraphicContext* ctx, const ImageIn const bool supported = cached->ctx == ctx && !cached->buffer_modified && cached->depth.stencil_address == 0 && cached->depth.stencil_size == 0 && cached->depth.layers == 1 && - (overlap == ImageRangeOverlap::PageOnly || - (contained && (exact_range || sampled_expansion))); + contained && (exact_range || sampled_expansion); if (!supported) { EXIT("TextureCache: unsupported sampled/depth-target alias, sampled=0x%016" PRIx64 "+0x%016" PRIx64 " depth=0x%016" PRIx64 "+0x%016" PRIx64 @@ -1599,7 +1662,7 @@ void TextureCache::RetireSampledTargetAliases(GraphicContext* ctx, const ImageIn cached->depth.tile_mode, cached->depth.htile_address, cached->depth.htile_size); } wait_idle |= cached->gpu_modified; - retire.push_back(cached.get()); + retire.push_back(cached); } if (retire.empty()) { return; @@ -1660,10 +1723,9 @@ void TextureCache::RetireSampledTargetAliases(GraphicContext* ctx, const ImageIn RetireImages(retire); } -void TextureCache::RetireStoragePageNeighbors(GraphicContext* ctx, const ImageInfo& requested) { +void TextureCache::ResolveStorageImageOverlaps(GraphicContext* ctx, const ImageInfo& requested) { std::vector retire; - bool wait_idle = false; - for (const auto& cached: m_images) { + for (auto* cached: FindImagesInRegionLocked(requested.address, requested.size, true)) { const bool tracker_gpu = m_memory_tracker.IsRegionGpuModified(cached->Address(), cached->Size()); switch (ClassifyStorageImageOverlap( @@ -1671,8 +1733,8 @@ void TextureCache::RetireStoragePageNeighbors(GraphicContext* ctx, const ImageIn cached->kind == CachedImage::Kind::Texture, cached->ctx == ctx, cached->gpu_modified, cached->buffer_modified, tracker_gpu)) { case StorageImageOverlap::None: continue; - case StorageImageOverlap::RetireSampled: retire.push_back(cached.get()); continue; - case StorageImageOverlap::PageNeighbor: break; + case StorageImageOverlap::RetireSampled: retire.push_back(cached); continue; + case StorageImageOverlap::PageNeighbor: continue; case StorageImageOverlap::Unsupported: EXIT("TextureCache: unsupported storage-image byte alias, requested=0x%016" PRIx64 "+0x%016" PRIx64 " existing=0x%016" PRIx64 "+0x%016" PRIx64 @@ -1681,25 +1743,8 @@ void TextureCache::RetireStoragePageNeighbors(GraphicContext* ctx, const ImageIn static_cast(cached->kind), cached->ctx == ctx, cached->gpu_modified, tracker_gpu, cached->buffer_modified); } - const bool tracker_cpu = - m_memory_tracker.IsRegionCpuModified(cached->Address(), cached->Size()); - if (cached->kind != CachedImage::Kind::StorageTexture || cached->ctx != ctx || - cached->buffer_modified || cached->info.IsCpuDirty() || - cached->gpu_modified != tracker_gpu || (tracker_gpu && tracker_cpu)) { - EXIT("TextureCache: unsupported storage-image page neighbor, requested=0x%016" PRIx64 - "+0x%016" PRIx64 " existing=0x%016" PRIx64 "+0x%016" PRIx64 - " kind=%u same_context=%d gpu=%d/%d cpu=%d buffer=%d image_cpu=%d\n", - requested.address, requested.size, cached->Address(), cached->Size(), - static_cast(cached->kind), cached->ctx == ctx, cached->gpu_modified, - tracker_gpu, tracker_cpu, cached->buffer_modified, cached->info.IsCpuDirty()); - } - wait_idle |= cached->gpu_modified; - retire.push_back(cached.get()); } RequireRetirementIsolation(retire, "storage neighbor", requested.address, requested.size); - if (wait_idle) { - VulkanDeviceWaitIdle(ctx); - } for (auto* cached: retire) { if (!cached->gpu_modified) { continue; @@ -1715,7 +1760,7 @@ void TextureCache::RetireStoragePageNeighbors(GraphicContext* ctx, const ImageIn void TextureCache::RetireStorageDepthAliasLocked(GraphicContext* ctx, const ImageInfo& requested) { CachedImage* selected = nullptr; - for (const auto& entry: m_images) { + for (auto* entry: FindImagesInRegionLocked(requested.address, requested.size, true)) { auto& cached = *entry; if (cached.kind != CachedImage::Kind::DepthTarget || !cached.OverlapsRange(requested.address, requested.size, true)) { @@ -1765,6 +1810,9 @@ TextureCache::~TextureCache() { if (ctx != nullptr) { VulkanDeviceWaitIdle(ctx); } + for (const auto& image: m_images) { + UnregisterImageLocked(*image, false); + } m_images.clear(); for (size_t i = 0; i < m_dummy_sampled_textures.size(); i++) { if (m_dummy_sampled_textures[i] != nullptr) { @@ -1881,6 +1929,15 @@ VulkanImage* TextureCache::FindTexture(CommandBuffer* command, GraphicContext* c match = cached; } if (match != nullptr) { + if (match->info.IsMaybeCpuDirty() && !match->info.IsDefinitelyCpuDirty()) { + const bool changed = + match->info.ResolveMaybeCpuHash(HashSampledImageEdges(match->info)); + if (!changed) { + m_memory_tracker.ForEachUploadRange( + info.address, info.size, false, [](uint64_t, uint64_t) noexcept {}, + []() noexcept {}); + } + } const bool buffer_dirty = match->buffer_modified; if (buffer_dirty) { if (match->gpu_modified || @@ -1952,6 +2009,7 @@ VulkanImage* TextureCache::FindTexture(CommandBuffer* command, GraphicContext* c auto* image = cached->image; command->RetainResourceUntilFence(cached); m_images.push_back(std::move(cached)); + RegisterImageLocked(*m_images.back()); return image; } @@ -2073,7 +2131,7 @@ StorageTextureVulkanImage* TextureCache::FindStorageTexture(CommandBuffer* com return static_cast(match->image); } - RetireStoragePageNeighbors(ctx, info); + ResolveStorageImageOverlaps(ctx, info); m_images.reserve(m_images.size() + 1); auto cached = std::make_shared(); cached->kind = CachedImage::Kind::StorageTexture; @@ -2094,6 +2152,7 @@ StorageTextureVulkanImage* TextureCache::FindStorageTexture(CommandBuffer* com auto* image = static_cast(cached->image); command->RetainResourceUntilFence(cached); m_images.push_back(std::move(cached)); + RegisterImageLocked(*m_images.back()); return image; } @@ -2233,6 +2292,9 @@ RenderTextureVulkanImage* TextureCache::FindRenderTarget(CommandBuffer* break; case CachedImage::Kind::VideoOut: break; } + if (overlap == RenderTargetOverlap::None) { + continue; + } bool supported = false; switch (overlap) { case RenderTargetOverlap::RetireSampled: @@ -2328,6 +2390,7 @@ RenderTextureVulkanImage* TextureCache::FindRenderTarget(CommandBuffer* auto* image = static_cast(cached->image); command->RetainResourceUntilFence(cached); m_images.push_back(std::move(cached)); + RegisterImageLocked(*m_images.back()); return image; } @@ -2649,6 +2712,7 @@ DepthStencilVulkanImage* TextureCache::FindDepthTarget(CommandBuffer* command, G auto* image = static_cast(cached->image); command->RetainResourceUntilFence(cached); m_images.push_back(std::move(cached)); + RegisterImageLocked(*m_images.back()); return image; } @@ -2715,6 +2779,7 @@ TextureCache::RegisterVideoOutSurfaces(GraphicContext* ctx, } result.push_back(static_cast(cached->image)); m_images.push_back(std::move(cached)); + RegisterImageLocked(*m_images.back()); } return result; } @@ -2813,7 +2878,7 @@ void TextureCache::UnregisterVideoOutSurfaces(const std::vectorAddress(), cached->Size()); cached->gpu_modified = false; } - m_memory_tracker.UntrackMemory(cached->Address(), cached->Size()); + UnregisterImageLocked(*cached, true); } for (auto* image: images) { auto it = std::find_if(m_images.begin(), m_images.end(), @@ -3485,8 +3550,8 @@ RenderTextureVulkanImage* TextureCache::FindRenderTargetByRange(CommandBuffer* c vaddr, size); } FaultSafeTextureLock lock(this, m_lock); - std::shared_ptr found; - for (auto& cached: m_images) { + CachedImage* found = nullptr; + for (auto* cached: FindImagesInRegionLocked(vaddr, size, false)) { if (cached->kind != CachedImage::Kind::RenderTarget || !ImageRangeOverlaps(vaddr, size, cached->Address(), cached->Size())) { continue; @@ -3503,14 +3568,19 @@ RenderTextureVulkanImage* TextureCache::FindRenderTargetByRange(CommandBuffer* c EXIT("TextureCache: incompatible or ambiguous render-target range, addr=0x%016" PRIx64 " size=0x%016" PRIx64 " cached=0x%016" PRIx64 "+0x%016" PRIx64 " previous=%p\n", vaddr, size, cached->Address(), cached->Size(), - static_cast(found.get())); + static_cast(found)); } found = cached; } if (found == nullptr) { return nullptr; } - command->RetainResourceUntilFence(found); + const auto owner = std::find_if(m_images.begin(), m_images.end(), + [found](const auto& image) { return image.get() == found; }); + if (owner == m_images.end()) { + EXIT("TextureCache: page-table render target has no cache owner\n"); + } + command->RetainResourceUntilFence(*owner); return static_cast(found->image); } @@ -3522,7 +3592,7 @@ bool TextureCache::HasGpuTargetPageOverlap(uint64_t vaddr, uint64_t size) { vaddr, size); } FaultSafeTextureLock lock(this, m_lock); - for (const auto& cached: m_images) { + for (const auto* cached: FindImagesInRegionLocked(vaddr, size, true)) { if (cached->kind != CachedImage::Kind::Texture && cached->OverlapsRange(vaddr, size, true)) { return true; @@ -3539,12 +3609,7 @@ bool TextureCache::HasPageOverlap(uint64_t vaddr, uint64_t size) { vaddr, size); } FaultSafeTextureLock lock(this, m_lock); - for (const auto& cached: m_images) { - if (cached->OverlapsRange(vaddr, size, true)) { - return true; - } - } - return false; + return !FindImagesInRegionLocked(vaddr, size, true).empty(); } bool TextureCache::HasRangeOverlap(uint64_t vaddr, uint64_t size) { @@ -3555,12 +3620,7 @@ bool TextureCache::HasRangeOverlap(uint64_t vaddr, uint64_t size) { vaddr, size); } FaultSafeTextureLock lock(this, m_lock); - for (const auto& cached: m_images) { - if (cached->OverlapsRange(vaddr, size, false)) { - return true; - } - } - return false; + return !FindImagesInRegionLocked(vaddr, size, false).empty(); } bool TextureCache::HasGpuModifiedRangeOverlap(uint64_t vaddr, uint64_t size) { @@ -3571,7 +3631,7 @@ bool TextureCache::HasGpuModifiedRangeOverlap(uint64_t vaddr, uint64_t size) { vaddr, size); } FaultSafeTextureLock lock(this, m_lock); - for (const auto& cached: m_images) { + for (const auto* cached: FindImagesInRegionLocked(vaddr, size, false)) { if (cached->gpu_modified && cached->OverlapsRange(vaddr, size, false)) { return true; } @@ -3997,14 +4057,6 @@ void TextureCache::UnmapMemory(uint64_t vaddr, uint64_t size) { } cached->gpu_modified = false; } - for (auto& cached: m_images) { - if (!cached->OverlapsRange(vaddr, size, false)) { - continue; - } - for (uint32_t i = 0; i < cached->RangeCount(); i++) { - m_memory_tracker.UntrackMemory(cached->Address(i), cached->Size(i)); - } - } for (auto it = m_surface_metas.begin(); it != m_surface_metas.end();) { const bool allocation_unmapped = ImageRangeOverlaps(vaddr, size, it->first, it->second.size); @@ -4034,6 +4086,7 @@ void TextureCache::UnmapMemory(uint64_t vaddr, uint64_t size) { ++it; continue; } + UnregisterImageLocked(**it, false); it = m_images.erase(it); } } diff --git a/src/graphics/host_gpu/renderer/textureCache.h b/src/graphics/host_gpu/renderer/textureCache.h index 85af4fcb3..f989d3950 100644 --- a/src/graphics/host_gpu/renderer/textureCache.h +++ b/src/graphics/host_gpu/renderer/textureCache.h @@ -6,6 +6,7 @@ #include "common/threads.h" #include "graphics/host_gpu/memoryTracker.h" #include "graphics/host_gpu/renderer/imageInfo.h" +#include "graphics/host_gpu/renderer/multiLevelPageTable.h" #include "graphics/host_gpu/renderer/tiler.h" #include @@ -124,6 +125,7 @@ public: private: struct CachedImage; + using ImageOwnerIndex = MultiRangePageOwnerIndex; struct ReadbackWorker; struct MetaDataInfo { uint64_t size = 0; @@ -135,13 +137,16 @@ private: [[nodiscard]] VkImageView GetImageView(GraphicContext* ctx, VulkanImage* image, const ImageViewInfo& info); [[nodiscard]] bool HasMetaOverlapLocked(uint64_t vaddr, uint64_t size) const; - [[nodiscard]] CachedImage* FindGpuReadbackPageCandidateLocked(uint64_t vaddr, - uint64_t size) const; + [[nodiscard]] CachedImage* FindGpuReadbackPageCandidateLocked(uint64_t vaddr, uint64_t size); void RequireNoMetaOverlapLocked(uint64_t vaddr, uint64_t size) const; void MarkSampledAliasesCpuDirtyLocked(uint64_t vaddr, uint64_t size); void RetireSampledTargetAliases(GraphicContext* ctx, const ImageInfo& requested); - void RetireStoragePageNeighbors(GraphicContext* ctx, const ImageInfo& requested); + void ResolveStorageImageOverlaps(GraphicContext* ctx, const ImageInfo& requested); void RetireStorageDepthAliasLocked(GraphicContext* ctx, const ImageInfo& requested); + void RegisterImageLocked(CachedImage& image); + void UnregisterImageLocked(CachedImage& image, bool release_tracking); + [[nodiscard]] std::vector + FindImagesInRegionLocked(uint64_t vaddr, uint64_t size, bool page_overlap); void RequireRetirementIsolation(const std::vector& retire, const char* operation, uint64_t address, uint64_t size) const; void RetireImages(const std::vector& retire, @@ -165,6 +170,7 @@ private: BufferCache& m_buffer_cache; ResourceMutex& m_resource_mutex; std::vector> m_images; + ImageOwnerIndex m_image_owner_index; std::map m_surface_metas; std::unique_ptr m_readback; std::vector m_buffer_transition_linear; diff --git a/tests/ImagePageTableTests.cpp b/tests/ImagePageTableTests.cpp new file mode 100644 index 000000000..ac83ea6f5 --- /dev/null +++ b/tests/ImagePageTableTests.cpp @@ -0,0 +1,138 @@ +#include "graphics/host_gpu/renderer/multiLevelPageTable.h" + +#include +#include +#include +#include +#include + +namespace { + +using Owners = std::vector; +using Table = Libs::Graphics::MultiLevelPageTable; +using OwnerIndex = Libs::Graphics::MultiRangePageOwnerIndex; + +void Check(bool value, const char* text) { + if (!value) { + std::fprintf(stderr, "ImagePageTableTests: failed: %s\n", text); + std::abort(); + } +} + +void TestMultiOwnerAndExactErase() { + Table table; + auto& owners = table.GetOrCreate(17); + owners.push_back(11); + owners.push_back(22); + + Check(table.Find(17) != nullptr && table.Find(17)->size() == 2, "both page owners are retained"); + Check(Libs::Graphics::EraseExact(owners, 11U), "registered owner is erased"); + Check(owners.size() == 1 && owners.front() == 22, "erasing one owner preserves its neighbor"); + Check(!Libs::Graphics::EraseExact(owners, 33U), "missing owner is reported without mutation"); +} + +void TestCrossBucketRange() { + Table::PageRange range{}; + constexpr uint64_t bucket_boundary = uint64_t{Table::kBucketEntries} << Table::kPageBits; + Check(Table::TryGetPageRange(bucket_boundary - 1, 2, range), "cross-bucket range is valid"); + Check(range.first == Table::kBucketEntries - 1 && range.last_exclusive == Table::kBucketEntries + 1, + "cross-bucket range covers both pages"); + + Table table; + table[range.first].push_back(1); + table[range.last_exclusive - 1].push_back(2); + Check(table.AllocatedBucketCount() == 2, "pages across the L1 boundary use distinct sparse buckets"); +} + +void TestQueriesDoNotAllocate() { + Table table; + Check(table.Find(123) == nullptr, "unallocated page query is empty"); + Check(table.AllocatedBucketCount() == 0, "mutable query does not allocate"); + const Table& const_table = table; + Check(const_table.Find(Table::kPageCount - 1) == nullptr, "const query is empty"); + Check(table.AllocatedBucketCount() == 0, "const query does not allocate"); + Check(table.Find(Table::kPageCount) == nullptr, "out-of-range query is empty"); + Check(table.AllocatedBucketCount() == 0, "out-of-range query does not allocate"); +} + +void TestAddressSpaceBoundaries() { + Table::PageRange range{}; + Check(Table::TryGetPageRange(Table::kAddressSpaceSize - 1, 1, range), "last guest byte is valid"); + Check(range.first == Table::kPageCount - 1 && range.last_exclusive == Table::kPageCount, + "last guest byte maps to the final page"); + Check(!Table::TryGetPageRange(0, 0, range), "empty ranges are rejected"); + Check(!Table::TryGetPageRange(Table::kAddressSpaceSize, 1, range), "first out-of-range byte is rejected"); + Check(!Table::TryGetPageRange(Table::kAddressSpaceSize - 1, 2, range), "crossing the address-space end is rejected"); + Check(!Table::TryGetPageRange(UINT64_MAX - 1, 4, range), "wrapping input is rejected"); + + Table table; + table.GetOrCreate(Table::kPageCount - 1).push_back(99); + Check(table.Find(Table::kPageCount - 1) != nullptr && table.Find(Table::kPageCount - 1)->front() == 99, + "final page supports allocating and nonallocating access"); + bool threw = false; + try { + (void)table.GetOrCreate(Table::kPageCount); + } catch (const std::out_of_range&) { + threw = true; + } + Check(threw, "allocating access hard-fails outside the guest address space"); +} + +void TestMultiRangeRegistrationDeduplicatesPages() { + OwnerIndex index; + // Depth and stencil-like planes overlap tracking pages and share one 1 MiB bucket. + Check(index.Register(7, {{0x101000, 0x2800}, {0x102000, 0x3000}}), "multi-range owner registers"); + Check(index.CoarseMembershipCount(1) == 1, "one owner is inserted once in a shared 1 MiB bucket"); + Check(index.TrackingMembershipCount(0x102) == 1, "overlapping planes insert one 4 KiB membership"); + Check(!index.Register(7, {{0x101000, 0x1000}}), "duplicate owner registration hard-fails"); + + const auto owners = index.Query(0x100000, 0x10000); + Check(owners.size() == 1 && owners.front() == 7, "multi-page query returns an owner once"); +} + +void TestSharedPageUnregisterLifecycle() { + OwnerIndex index; + const std::vector ranges{{0x202000, 0x2000}}; + Check(index.Register(11, ranges) && index.Register(22, ranges), "two owners register on identical pages"); + Check(index.CoarseMembershipCount(2) == 2 && index.TrackingMembershipCount(0x202) == 2, + "coarse and tracking pages retain both owners"); + + std::vector releases; + Check(index.Unregister(11, releases), "first owner unregisters"); + Check(releases.empty(), "shared tracking pages are not released with one owner remaining"); + Check(index.Query(0x202000, 1).size() == 1 && index.Query(0x202000, 1).front() == 22, + "unregistering one owner preserves the other"); + Check(!index.Unregister(11, releases), "missing membership hard-fails without mutation"); + + Check(index.Unregister(22, releases), "final owner unregisters"); + Check(releases.size() == 1 && releases.front().address == 0x202000 && releases.front().size == 0x2000, + "adjacent final-owner tracking pages return one contiguous release"); +} + +void TestStrictByteFilteringAndPredicate() { + OwnerIndex index; + Check(index.Register(31, {{0x300100, 0x100}}), "first byte-disjoint owner registers"); + Check(index.Register(32, {{0x300800, 0x100}}), "second byte-disjoint owner registers"); + Check(index.TrackingMembershipCount(0x300) == 2, "byte-disjoint owners share one tracking page"); + Check(index.Query(0x300400, 0x40).empty(), "page hit without byte overlap is filtered out"); + const auto page_candidates = index.QueryCandidates(0x300400, 0x40); + Check(page_candidates.size() == 2, "fault candidate query retains byte-disjoint owners on the touched page"); + const auto first = index.Query(0x300180, 0x10); + Check(first.size() == 1 && first.front() == 31, "strict byte overlap selects only the matching owner"); + const auto predicate_filtered = index.Query(0x300000, 0x1000, [](uint32_t owner) { return owner == 32; }); + Check(predicate_filtered.size() == 1 && predicate_filtered.front() == 32, "supplied predicate filters query owners"); +} + +} // namespace + +int main() { + TestMultiOwnerAndExactErase(); + TestCrossBucketRange(); + TestQueriesDoNotAllocate(); + TestAddressSpaceBoundaries(); + TestMultiRangeRegistrationDeduplicatesPages(); + TestSharedPageUnregisterLifecycle(); + TestStrictByteFilteringAndPredicate(); + std::printf("ImagePageTableTests: all cases passed\n"); + return 0; +} diff --git a/tests/ShaderRecompilerComputeTests.cpp b/tests/ShaderRecompilerComputeTests.cpp index 64af67109..0e4b9670a 100644 --- a/tests/ShaderRecompilerComputeTests.cpp +++ b/tests/ShaderRecompilerComputeTests.cpp @@ -11490,6 +11490,49 @@ void CheckImageOverlapResolution() { ClassifySampledOverlap(sampled_alias, sampled, false, false) == SampledOverlap::Unsupported, "cross-context sampled alias was admitted"); + ImageInfo dirty_info{}; + dirty_info.address = 0x500100; + dirty_info.size = 0x100; + Image dirty_image; + dirty_image = dirty_info; + dirty_image.InvalidateCpuWrite(0x500300, 0x10); + Require("ImageOverlapResolution", "sampled edge-page maybe dirty", + dirty_image.IsCpuDirty() && dirty_image.IsMaybeCpuDirty() && + !dirty_image.IsDefinitelyCpuDirty(), + "byte-disjoint write sharing a tracker page was not kept maybe-dirty"); + Require("ImageOverlapResolution", "sampled edge hash required", + dirty_image.NeedsMaybeCpuHash(), + "collapsed edge tracking did not request a baseline hash"); + dirty_image.SetMaybeCpuHash(0x1234); + Require("ImageOverlapResolution", "sampled unchanged edge hash", + !dirty_image.ResolveMaybeCpuHash(0x1234) && !dirty_image.IsCpuDirty() && + dirty_image.IsCpuTrackingComplete(), + "unchanged edge bytes forced an image upload"); + dirty_image.InvalidateCpuWrite(0x500300, 0x10); + dirty_image.SetMaybeCpuHash(0x1234); + Require("ImageOverlapResolution", "sampled changed edge hash", + dirty_image.ResolveMaybeCpuHash(0x5678) && + dirty_image.IsDefinitelyCpuDirty(), + "changed edge bytes were not promoted to definitely dirty"); + dirty_image.RefreshComplete(); + dirty_image.InvalidateCpuWrite(dirty_info.address, 1); + Require("ImageOverlapResolution", "sampled byte overlap dirty", + dirty_image.IsCpuDirty() && dirty_image.IsDefinitelyCpuDirty() && + !dirty_image.IsMaybeCpuDirty(), + "true byte overlap was not promoted to definitely dirty"); + ImageInfo two_page_info{}; + two_page_info.address = 0x600100; + two_page_info.size = 0x1800; + Image two_page_image; + two_page_image = two_page_info; + two_page_image.InvalidateCpuWrite(0x600000, 1); + Require("ImageOverlapResolution", "sampled head tracking shrink", + !two_page_image.IsCpuDirty(), + "a single disjoint edge fault prematurely dirtied a two-page image"); + two_page_image.InvalidateCpuWrite(0x601a00, 1); + Require("ImageOverlapResolution", "sampled head-tail collapse", + two_page_image.IsMaybeCpuDirty() && two_page_image.NeedsMaybeCpuHash(), + "both disjoint edge faults did not collapse logical tracking"); constexpr uint64_t storage_subrange = 0x649b0100; constexpr uint64_t storage_subrange_size = 0x800; constexpr uint64_t sampled_backing = 0x649a3000; @@ -11824,14 +11867,24 @@ void CheckImageOverlapResolution() { target.address++; Require("ImageOverlapResolution", "render target partial page", ClassifyRenderTargetOverlap(sampled, false, true, target) == - RenderTargetOverlap::Unsupported, - "partially shared tracker page was retired for a render target"); + RenderTargetOverlap::RetireSampled, + "true byte overlap within a partial tracker page was not retired"); + target.address = sampled.address + 0x6101; + target.size = 0x4000; + Require("ImageOverlapResolution", "render target unaligned chance overlap", + ClassifyRenderTargetOverlap(sampled, false, true, target) == + RenderTargetOverlap::RetireSampled, + "unaligned clean sampled chance overlap was not retired"); target.address = page_left.address + page_left.size; target.size = page_left.size; Require("ImageOverlapResolution", "render target shared page", ClassifyRenderTargetOverlap(page_left, false, true, target) == - RenderTargetOverlap::Unsupported, - "byte-disjoint allocation sharing a tracker page was silently admitted"); + RenderTargetOverlap::None, + "byte-disjoint allocation sharing a tracker page was treated as an alias"); + Require("ImageOverlapResolution", "sampled render-target shared page", + ClassifySampledRenderTargetOverlap(page_left, target, false, true) == + RenderTargetOverlap::None, + "reverse byte-disjoint tracker-page relationship was treated as an alias"); target.address = sampled.address + sampled.size; target.size = sampled.size; Require("ImageOverlapResolution", "render target adjacent",