From 26c41681600dd03702877aa2a04fa06682bac59f Mon Sep 17 00:00:00 2001 From: nmzik Date: Fri, 21 Aug 2026 20:11:38 +0200 Subject: [PATCH] refactor(renderer): simplify buffer and texture cache ownership --- src/common/slotVector.h | 132 ++++ src/graphics/guest_gpu/graphicsRun.cpp | 2 +- src/graphics/host_gpu/graphicContext.h | 1 - .../host_gpu/renderer/cache/bufferCache.cpp | 551 +++++++-------- .../host_gpu/renderer/cache/bufferCache.h | 112 ++- .../host_gpu/renderer/cache/streamBuffer.h | 4 + .../host_gpu/renderer/cache/textureCache.cpp | 543 +++++---------- .../host_gpu/renderer/cache/textureCache.h | 83 +-- .../host_gpu/renderer/commandScheduler.cpp | 21 +- src/graphics/host_gpu/renderer/context.cpp | 53 +- src/graphics/host_gpu/renderer/image/image.h | 11 +- .../renderer/pipeline/descriptorCache.h | 10 +- .../renderer/pipeline/descriptors.cpp | 207 +++--- src/graphics/host_gpu/renderer/render.h | 38 +- .../host_gpu/renderer/renderCompute.cpp | 8 +- src/graphics/host_gpu/renderer/renderDraw.cpp | 90 +-- src/graphics/host_gpu/renderer/sync.cpp | 2 +- src/graphics/host_gpu/renderer/sync.h | 2 +- src/graphics/host_gpu/vma.cpp | 14 - tests/ShaderRecompilerComputeTests.cpp | 643 ++++++++---------- 20 files changed, 1116 insertions(+), 1411 deletions(-) create mode 100644 src/common/slotVector.h diff --git a/src/common/slotVector.h b/src/common/slotVector.h new file mode 100644 index 000000000..b4c7d1049 --- /dev/null +++ b/src/common/slotVector.h @@ -0,0 +1,132 @@ +#ifndef EMULATOR_SRC_COMMON_SLOTVECTOR_H_ +#define EMULATOR_SRC_COMMON_SLOTVECTOR_H_ + +#include "common/assert.h" +#include "common/abi.h" + +#include +#include +#include +#include +#include +#include +#include +#include + +namespace Common { + +struct SlotId { + static constexpr uint32_t INVALID_INDEX = std::numeric_limits::max(); + + constexpr SlotId() noexcept = default; + constexpr SlotId(uint32_t value) noexcept: index(value), generation(1) {} + constexpr SlotId(uint32_t value, uint32_t slot_generation) noexcept + : index(value), generation(slot_generation) {} + + [[nodiscard]] constexpr explicit operator bool() const noexcept { return index != INVALID_INDEX; } + constexpr auto operator<=>(const SlotId&) const noexcept = default; + + uint32_t index = INVALID_INDEX; + uint32_t generation = 0; +}; + +// Stable-address slot storage for cache resources that are intentionally non-movable. +template +class SlotVector { +public: + SlotVector() = default; + KYTY_CLASS_NO_COPY(SlotVector); + + [[nodiscard]] T& operator[](SlotId id) noexcept { + EXIT_IF(!is_allocated(id)); + return *m_values[id.index].value; + } + + [[nodiscard]] const T& operator[](SlotId id) const noexcept { + EXIT_IF(!is_allocated(id)); + return *m_values[id.index].value; + } + + [[nodiscard]] T* try_get(SlotId id) noexcept { + return is_allocated(id) ? &*m_values[id.index].value : nullptr; + } + + [[nodiscard]] const T* try_get(SlotId id) const noexcept { + return is_allocated(id) ? &*m_values[id.index].value : nullptr; + } + + [[nodiscard]] bool is_allocated(SlotId id) const noexcept { + return id && id.index < m_values.size() && + m_values[id.index].generation == id.generation && m_values[id.index].value.has_value(); + } + + template + [[nodiscard]] SlotId insert(Args&&... args) { + uint32_t index = 0; + if (m_free_list.empty()) { + index = static_cast(m_values.size()); + m_values.emplace_back(); + m_values.back().value.emplace(std::forward(args)...); + } else { + index = m_free_list.back(); + m_free_list.pop_back(); + EXIT_IF(m_values[index].value.has_value()); + m_values[index].value.emplace(std::forward(args)...); + } + ++m_size; + return SlotId {index, m_values[index].generation}; + } + + void erase(SlotId id) noexcept { + EXIT_IF(!is_allocated(id)); + auto& slot = m_values[id.index]; + slot.value.reset(); + if (++slot.generation == 0) { + slot.generation = 1; + } + m_free_list.push_back(id.index); + --m_size; + } + + [[nodiscard]] size_t size() const noexcept { return m_size; } + [[nodiscard]] size_t capacity() const noexcept { return m_values.size(); } + + template + void ForEach(F&& fn) { + for (uint32_t index = 0; index < m_values.size(); ++index) { + if (m_values[index].value) { + fn(SlotId {index, m_values[index].generation}, *m_values[index].value); + } + } + } + + template + void ForEach(F&& fn) const { + for (uint32_t index = 0; index < m_values.size(); ++index) { + if (m_values[index].value) { + fn(SlotId {index, m_values[index].generation}, *m_values[index].value); + } + } + } + +private: + struct Slot { + std::optional value; + uint32_t generation = 1; + }; + + std::deque m_values; + std::vector m_free_list; + size_t m_size = 0; +}; + +} // namespace Common + +template <> +struct std::hash { + [[nodiscard]] size_t operator()(Common::SlotId id) const noexcept { + return std::hash {}((static_cast(id.generation) << 32u) | id.index); + } +}; + +#endif // EMULATOR_SRC_COMMON_SLOTVECTOR_H_ diff --git a/src/graphics/guest_gpu/graphicsRun.cpp b/src/graphics/guest_gpu/graphicsRun.cpp index 10444fe33..bcb0a4681 100644 --- a/src/graphics/guest_gpu/graphicsRun.cpp +++ b/src/graphics/guest_gpu/graphicsRun.cpp @@ -1275,7 +1275,7 @@ void CommandProcessor::WriteAtEndOfPipe(uint32_t cache_policy, uint32_t event_wr if (eop_event_type == 0x2f && cache_action == 0x00 && event_index == 0x06) { auto* dst = static_cast(dst_gpu_addr); SynchronizeGpu(); - Sync::ReadGds(m_renderer.GetBufferCache().GetGdsBuffer(), dst, value & 0xffffu, + Sync::ReadGds(*m_renderer.GetBufferCache().GetGdsBuffer(), dst, value & 0xffffu, value >> 16u); Sync::WriteAtEndOfPipeGds32(m_submit_id, CurrentBuffer(), dst, value & 0xffffu, value >> 16u); diff --git a/src/graphics/host_gpu/graphicContext.h b/src/graphics/host_gpu/graphicContext.h index e54f458a0..cdc633b4b 100644 --- a/src/graphics/host_gpu/graphicContext.h +++ b/src/graphics/host_gpu/graphicContext.h @@ -24,7 +24,6 @@ struct GraphicContext: public VulkanInstance { [[nodiscard]] uint64_t GetDeviceMemoryUsage() const; [[nodiscard]] uint64_t GetTotalMemoryBudget() const; void CreateBuffer(uint64_t size, VulkanBuffer& buffer); - void DeleteBuffer(VulkanBuffer& buffer); [[nodiscard]] bool CreateImage(const vk::ImageCreateInfo& info, VulkanImage& image); void DeleteImage(VulkanImage& image); void MapMemory(VulkanMemory& memory, void*& data); diff --git a/src/graphics/host_gpu/renderer/cache/bufferCache.cpp b/src/graphics/host_gpu/renderer/cache/bufferCache.cpp index 8123c42ce..11fa83784 100644 --- a/src/graphics/host_gpu/renderer/cache/bufferCache.cpp +++ b/src/graphics/host_gpu/renderer/cache/bufferCache.cpp @@ -28,60 +28,25 @@ constexpr uint64_t GdsBufferSize = 64 * 1024; } // namespace -uint64_t BufferCache::AlignDown(uint64_t value) noexcept { - return value & ~(CACHING_PAGE_SIZE - 1); -} - -uint64_t BufferCache::AlignUp(uint64_t value) { - if (value > UINT64_MAX - (CACHING_PAGE_SIZE - 1)) { - EXIT("BufferCache: address alignment overflow\n"); - } - return (value + CACHING_PAGE_SIZE - 1) & ~(CACHING_PAGE_SIZE - 1); -} - -void BufferCache::Upload(CommandBuffer& command, Buffer& destination, uint64_t destination_offset, - const void* source, uint64_t size) { +void BufferCache::WriteDataBuffer(Buffer& buffer, uint64_t address, const void* source, + uint64_t size) { auto* bytes = static_cast(source); while (size != 0) { - const auto chunk = std::min(size, m_staging_buffer.Size()); - const auto stage_offset = m_staging_buffer.Copy(bytes, chunk, 4); - destination.CopyFrom(command, m_staging_buffer, stage_offset, destination_offset, chunk, - vk::AccessFlagBits::eHostWrite); + const auto chunk = std::min(size, m_staging_buffer.Size()); + const auto offset = m_staging_buffer.Copy(bytes, chunk, 4); + buffer.CopyFrom(m_scheduler.Current(), m_staging_buffer, offset, buffer.Offset(address), + chunk, vk::AccessFlagBits::eHostWrite); bytes += chunk; - destination_offset += chunk; + address += chunk; size -= chunk; } } -bool BufferCache::ResolveOverlap(CacheRange& merged, CacheRange candidate) noexcept { - if (merged.address == 0 || merged.size == 0 || candidate.address == 0 || candidate.size == 0 || - merged.size > UINT64_MAX - merged.address || - candidate.size > UINT64_MAX - candidate.address) { - EXIT("BufferCache: invalid overlap-merge range\n"); - } - const auto merged_end = merged.address + merged.size; - const auto candidate_end = candidate.address + candidate.size; - if (merged.address >= candidate_end || candidate.address >= merged_end) { - return false; - } - const auto address = std::min(merged.address, candidate.address); - const auto end = std::max(merged_end, candidate_end); - merged = {.address = address, .size = end - address}; - return true; -} - -struct BufferCache::CachedBuffer { - uint64_t vaddr = 0; - uint64_t size = 0; - std::shared_ptr buffer; - uint64_t tick_accessed_last = 0; -}; - struct BufferCache::DownloadCopy { - std::shared_ptr owner; - uint64_t source_offset = 0; - uint64_t address = 0; - uint64_t size = 0; + Buffer* buffer = nullptr; + uint64_t source_offset = 0; + uint64_t address = 0; + uint64_t size = 0; }; struct BufferCache::DownloadRange { @@ -90,15 +55,37 @@ struct BufferCache::DownloadRange { uint64_t offset = 0; }; -struct BufferCache::RetiredBuffer { - uint64_t address = 0; - uint64_t size = 0; - std::shared_ptr owner; -}; +void BufferCache::Unregister(BufferId id) { + auto& buffer = m_slot_buffers[id]; + if (buffer.is_deleted) { + return; + } + const auto found = m_buffers.find(buffer.CpuAddress()); + EXIT_IF(found == m_buffers.end() || found->second != id); + m_buffers.erase(found); + if (buffer.Size() > m_total_used_memory) { + EXIT("BufferCache: allocation accounting underflow\n"); + } + m_total_used_memory -= buffer.Size(); + buffer.is_deleted = true; +} + +void BufferCache::DeleteBuffer(BufferId id) { + auto* buffer = m_slot_buffers.try_get(id); + if (buffer == nullptr || buffer->is_deleted) { + return; + } + Unregister(id); + if (m_scheduler.Active()) { + m_scheduler.DeferOperation([this, id] { m_slot_buffers.erase(id); }); + } else { + m_slot_buffers.erase(id); + } +} std::pair BufferCache::DownloadEnvelope(const DownloadCopy& copy) { - if (copy.owner == nullptr || copy.size == 0 || copy.source_offset > copy.owner->Size() || - copy.size > copy.owner->Size() - copy.source_offset) { + if (copy.buffer == nullptr || copy.size == 0 || copy.source_offset > copy.buffer->Size() || + copy.size > copy.buffer->Size() - copy.source_offset) { EXIT("BufferCache: invalid download copy\n"); } const auto begin = copy.source_offset & ~uint64_t {3}; @@ -107,7 +94,7 @@ std::pair BufferCache::DownloadEnvelope(const DownloadCopy& EXIT("BufferCache: download copy alignment overflow\n"); } const auto end = (copy.source_offset + copy.size + 3) & ~uint64_t {3}; - if (end > copy.owner->Size()) { + if (end > copy.buffer->Size()) { EXIT("BufferCache: aligned download copy exceeds its owner\n"); } return {begin, end - begin}; @@ -144,7 +131,7 @@ BufferCache::RecordDownloads(std::span copies) { for (const auto& copy: copies) { const auto [source_begin, envelope_size] = DownloadEnvelope(copy); const auto prefix = copy.source_offset - source_begin; - download.CopyFrom(m_scheduler.Current(), *copy.owner, source_begin, base_offset + cursor, + download.CopyFrom(m_scheduler.Current(), *copy.buffer, source_begin, base_offset + cursor, envelope_size, vk::AccessFlagBits::eMemoryWrite, vk::AccessFlags {}, vk::AccessFlagBits::eMemoryRead | vk::AccessFlagBits::eMemoryWrite, vk::AccessFlagBits::eHostRead); @@ -163,27 +150,29 @@ void BufferCache::PublishDownloads(std::span downloads) { } } -void BufferCache::QueueGarbageDownload(std::span copies, RetiredBuffer retire) { - if (copies.empty()) { - return; - } +void BufferCache::QueueGarbageDownload(std::span copies, BufferId id, + uint64_t address, uint64_t size) { + EXIT_IF(copies.empty()); auto downloads = RecordDownloads(copies); - m_scheduler.DeferOperation( - [this, downloads = std::move(downloads), retire = std::move(retire)]() mutable { - PublishDownloads(downloads); - for (const auto& range: downloads) { - m_gpu_modified_ranges.Subtract(range.address, range.size); - } - // ForEachDownloadRange reports full tracker pages, and every exact GPU-owned - // interval on those pages was downloaded and removed. Clearing the original - // query therefore cannot orphan a dirty sibling on an edge page. - m_memory_tracker.UnmarkRegionAsGpuModified(retire.address, retire.size); - if (m_memory_tracker.IsRegionGpuModified(retire.address, retire.size) || - !m_gpu_modified_ranges.Intersections(retire.address, retire.size).empty()) { - EXIT("BufferCache: asynchronous garbage collection retained GPU ownership\n"); - } - m_memory_tracker.UntrackMemory(retire.address, retire.size); - }); + auto retired = downloads; + m_scheduler.DeferPriorityOperation( + [this, downloads = std::move(downloads)]() mutable { PublishDownloads(downloads); }); + m_scheduler.DeferOperation([this, retired = std::move(retired), id, address, size]() mutable { + for (const auto& range: retired) { + m_gpu_modified_ranges.Subtract(range.address, range.size); + } + // ForEachDownloadRange reports full tracker pages, and every exact GPU-owned + // interval on those pages was downloaded and removed. Clearing the original + // query therefore cannot orphan a dirty sibling on an edge page. + m_memory_tracker.UnmarkRegionAsGpuModified(address, size); + if (m_memory_tracker.IsRegionGpuModified(address, size) || + !m_gpu_modified_ranges.Intersections(address, size).empty()) { + EXIT("BufferCache: asynchronous garbage collection retained GPU ownership\n"); + } + m_memory_tracker.UntrackMemory(address, size); + EXIT_IF(!m_slot_buffers[id].is_deleted); + m_slot_buffers.erase(id); + }); } BufferCache::BufferCache(GraphicContext& graphics, CommandScheduler& scheduler, @@ -198,6 +187,10 @@ BufferCache::BufferCache(GraphicContext& graphics, CommandScheduler& scheduler, m_texture_cache(texture_cache) { std::memset(m_gds_buffer.Mapped().data(), 0, static_cast(m_gds_buffer.Size())); m_gds_buffer.Flush(0, m_gds_buffer.Size()); + const auto null_id = + m_slot_buffers.insert(m_graphics, m_scheduler, MemoryUsage::DeviceLocal, 0, AllFlags, 16); + EXIT_IF(null_id != NULL_BUFFER_ID); + SetVulkanObjectNameF(m_graphics.device, GetBuffer(null_id).Handle(), "Kyty.NullBuffer"); if (!m_graphics.CanReportMemoryUsage()) { return; } @@ -216,38 +209,16 @@ BufferCache::~BufferCache() { if (!m_gpu_modified_ranges.Empty()) { EXIT("BufferCache: destroyed with pending GPU-modified ranges\n"); } - for (const auto& [vaddr, cached]: m_buffers) { + for (const auto& [vaddr, id]: m_buffers) { (void)vaddr; - if (m_memory_tracker.IsRegionGpuModified(cached->vaddr, cached->size)) { + const auto& buffer = m_slot_buffers[id]; + if (m_memory_tracker.IsRegionGpuModified(buffer.CpuAddress(), buffer.Size())) { EXIT("BufferCache: destroyed with GPU-modified buffer\n"); } } m_buffers.clear(); } -StreamBuffer& BufferCache::GetUtilityBuffer(MemoryUsage usage) noexcept { - switch (usage) { - case MemoryUsage::Upload: return m_staging_buffer; - case MemoryUsage::Stream: return m_stream_buffer; - case MemoryUsage::Download: return m_download_buffer; - case MemoryUsage::DeviceLocal: return m_device_buffer; - } - EXIT("BufferCache: invalid utility-buffer usage\n"); -} - -BufferBinding BufferCache::UploadTransient(const void* data, uint64_t size, uint64_t alignment) { - EXIT_IF(data == nullptr || size == 0); - if (auto [mapped, offset] = m_stream_buffer.Map(size, alignment, false); mapped != nullptr) { - std::memcpy(mapped, data, static_cast(size)); - m_stream_buffer.Commit(); - return {{}, m_stream_buffer.Handle(), offset}; - } - auto owner = - std::make_shared(m_graphics, m_scheduler, MemoryUsage::Upload, 0, AllFlags, size); - owner->Write(0, data, size); - return {owner, owner->Handle(), 0}; -} - void BufferCache::InvalidateMemory(uint64_t vaddr, uint64_t size) { if (vaddr == 0 || size == 0 || vaddr >= TRACKER_ADDRESS_SIZE || size > TRACKER_ADDRESS_SIZE - vaddr) { @@ -288,14 +259,14 @@ void BufferCache::ReadMemoryOnGpu(uint64_t vaddr, uint64_t size, bool is_write) if (owner == m_buffers.begin()) { EXIT("BufferCache: invalidation readback has no buffer owner\n"); } - auto& cached = *std::prev(owner)->second; - if (!cached.buffer->IsInBounds(copy_address, 1)) { + auto& buffer = m_slot_buffers[std::prev(owner)->second]; + if (!buffer.IsInBounds(copy_address, 1)) { EXIT("BufferCache: invalidation readback is outside its buffer owner\n"); } - const auto copy_size = - std::min(range.size - copied, cached.vaddr + cached.size - copy_address); - copies.push_back({cached.buffer, cached.buffer->Offset(copy_address), - copy_address, copy_size}); + const auto copy_size = std::min( + range.size - copied, buffer.CpuAddress() + buffer.Size() - copy_address); + copies.push_back( + {&buffer, buffer.Offset(copy_address), copy_address, copy_size}); copied += copy_size; } } @@ -330,38 +301,39 @@ void BufferCache::UnmapMemory(uint64_t vaddr, uint64_t size) { std::vector copies; std::vector> modified_buffers; std::vector> retired_buffers; - for (const auto& [begin, cached]: m_buffers) { - if (vaddr < begin + cached->size && begin < vaddr + size) { - retired_buffers.emplace_back(begin, cached->size); + for (const auto& [begin, id]: m_buffers) { + const auto& buffer = m_slot_buffers[id]; + if (vaddr < begin + buffer.Size() && begin < vaddr + size) { + retired_buffers.emplace_back(begin, buffer.Size()); } } - for (const auto& [begin, cached]: m_buffers) { - if (vaddr >= begin + cached->size || begin >= vaddr + size || - !m_memory_tracker.IsRegionGpuModified(begin, cached->size)) { + for (const auto& [begin, id]: m_buffers) { + const auto& buffer = m_slot_buffers[id]; + if (vaddr >= begin + buffer.Size() || begin >= vaddr + size || + !m_memory_tracker.IsRegionGpuModified(begin, buffer.Size())) { continue; } - const auto dirty = m_gpu_modified_ranges.Intersections(begin, cached->size); + const auto dirty = m_gpu_modified_ranges.Intersections(begin, buffer.Size()); if (dirty.empty()) { EXIT("BufferCache: GPU-modified buffer has no dirty ranges\n"); } - modified_buffers.emplace_back(begin, cached->size); + modified_buffers.emplace_back(begin, buffer.Size()); } for (const auto& [begin, bytes]: modified_buffers) { auto owner = m_buffers.find(begin); - if (owner == m_buffers.end() || owner->second->size != bytes) { + if (owner == m_buffers.end() || m_slot_buffers[owner->second].Size() != bytes) { EXIT("BufferCache: unmap owner changed during collection\n"); } - auto& cached = *owner->second; + auto& buffer = m_slot_buffers[owner->second]; m_memory_tracker.ForEachDownloadRange( - begin, cached.size, + begin, buffer.Size(), [&](uint64_t address, uint64_t bytes) noexcept { m_memory_tracker.ValidateGpuDirtyPages(m_gpu_modified_ranges, address, bytes, "unmap"); }, [&](uint64_t address, uint64_t bytes) noexcept { for (const auto& range: m_gpu_modified_ranges.Intersections(address, bytes)) { - copies.push_back( - {cached.buffer, range.address - begin, range.address, range.size}); + copies.push_back({&buffer, range.address - begin, range.address, range.size}); } }); } @@ -385,86 +357,92 @@ void BufferCache::UnmapMemory(uint64_t vaddr, uint64_t size) { EXIT("BufferCache: unmap retained dirty byte ranges\n"); } m_memory_tracker.UntrackMemory(vaddr, size); - for (auto it = m_buffers.begin(); it != m_buffers.end();) { - if (vaddr < it->first + it->second->size && it->first < vaddr + size) { - if (it->second->size > m_total_used_memory) { - EXIT("BufferCache: allocation accounting underflow\n"); - } - m_total_used_memory -= it->second->size; - it = m_buffers.erase(it); - } else { - ++it; + for (const auto& [begin, bytes]: retired_buffers) { + (void)bytes; + const auto found = m_buffers.find(begin); + if (found != m_buffers.end()) { + DeleteBuffer(found->second); } } } -BufferCache::CachedBuffer& BufferCache::GetOrCreateBuffer(CommandBuffer& command, uint64_t vaddr, - uint64_t size) { - const auto begin = AlignDown(vaddr); - const auto end = AlignUp(vaddr + size); +BufferId BufferCache::FindBuffer(uint64_t vaddr, uint64_t size) { + if (vaddr == 0) { + return NULL_BUFFER_ID; + } + if (size == 0 || vaddr >= TRACKER_ADDRESS_SIZE || size > TRACKER_ADDRESS_SIZE - vaddr) { + EXIT("BufferCache: invalid buffer discovery request\n"); + } + auto& command = m_scheduler.Current(); + EXIT_IF(command.IsInvalid() || command.IsExecute()); + auto begin = vaddr & ~(CACHING_PAGE_SIZE - 1); + auto end = (vaddr + size + CACHING_PAGE_SIZE - 1) & ~(CACHING_PAGE_SIZE - 1); const auto next = m_buffers.upper_bound(vaddr); if (next != m_buffers.begin()) { - auto owner = std::prev(next); - if (owner->second->buffer->IsInBounds(vaddr, size)) { - owner->second->tick_accessed_last = m_gc_tick; - return *owner->second; + const auto owner = std::prev(next); + auto& buffer = m_slot_buffers[owner->second]; + if (buffer.IsInBounds(vaddr, size)) { + buffer.tick_accessed_last = m_gc_tick; + return owner->second; } } - CacheRange merged {.address = begin, .size = end - begin}; - using Iterator = decltype(m_buffers.begin()); - std::vector overlaps; + auto merge = [&](const Buffer& buffer) { + const auto buffer_begin = buffer.CpuAddress(); + const auto buffer_end = buffer_begin + buffer.Size(); + if (begin >= buffer_end || buffer_begin >= end) { + return false; + } + begin = std::min(begin, buffer_begin); + end = std::max(end, buffer_end); + return true; + }; + std::vector overlaps; auto first = m_buffers.lower_bound(begin); if (first != m_buffers.begin()) { - auto previous = std::prev(first); - if (ResolveOverlap(merged, {previous->second->vaddr, previous->second->size})) { + const auto previous = std::prev(first); + if (merge(m_slot_buffers[previous->second])) { first = previous; } } for (auto candidate = first; candidate != m_buffers.end(); ++candidate) { - if (candidate->first >= merged.address + merged.size) { + if (candidate->first >= end) { break; } - if (ResolveOverlap(merged, {candidate->second->vaddr, candidate->second->size})) { - overlaps.push_back(candidate); + if (merge(m_slot_buffers[candidate->second])) { + overlaps.push_back(candidate->second); } } - auto cached = std::make_unique(); - cached->vaddr = merged.address; - cached->size = merged.size; - cached->tick_accessed_last = m_gc_tick; - cached->buffer = std::make_shared(m_graphics, m_scheduler, MemoryUsage::DeviceLocal, - merged.address, AllFlags, merged.size); - SetVulkanObjectNameF(m_graphics.device, cached->buffer->Handle(), - "Kyty.GameBuffer[guest=0x{:016x} size=0x{:x}]", cached->vaddr, - cached->size); + const auto id = m_slot_buffers.insert(m_graphics, m_scheduler, MemoryUsage::DeviceLocal, begin, + AllFlags, end - begin); + auto& buffer = m_slot_buffers[id]; + buffer.tick_accessed_last = m_gc_tick; + SetVulkanObjectNameF(m_graphics.device, buffer.Handle(), + "Kyty.GameBuffer[guest=0x{:016x} size=0x{:x}]", begin, end - begin); for (const auto overlap: overlaps) { - const auto& old = *overlap->second; - cached->buffer->CopyFrom(command, *old.buffer, 0, old.vaddr - cached->vaddr, old.size); - command.RetainResourceUntilFence(old.buffer); + const auto& old = m_slot_buffers[overlap]; + buffer.CopyFrom(command, old, 0, old.CpuAddress() - begin, old.Size()); } for (const auto overlap: overlaps) { - if (overlap->second->size > m_total_used_memory) { - EXIT("BufferCache: allocation accounting underflow\n"); - } - m_total_used_memory -= overlap->second->size; - m_buffers.erase(overlap); + DeleteBuffer(overlap); } - m_total_used_memory += cached->size; - return *m_buffers.emplace(cached->vaddr, std::move(cached)).first->second; + m_total_used_memory += buffer.Size(); + const auto [it, inserted] = m_buffers.emplace(begin, id); + (void)it; + EXIT_IF(!inserted); + return id; } -bool BufferCache::SynchronizeBuffer(CommandBuffer& command, Buffer& buffer, uint64_t vaddr, - uint64_t size, bool is_written, bool is_texel_buffer) { +bool BufferCache::SynchronizeBuffer(Buffer& buffer, uint64_t vaddr, uint64_t size, bool is_written, + bool is_texel_buffer) { std::vector> uploads; m_memory_tracker.ForEachUploadRange( vaddr, size, is_written, [&](uint64_t address, uint64_t bytes) noexcept { uploads.emplace_back(address, bytes); }, [&]() noexcept { for (const auto& [address, bytes]: uploads) { - Upload(command, buffer, buffer.Offset(address), - reinterpret_cast(address), bytes); + WriteDataBuffer(buffer, address, reinterpret_cast(address), bytes); } }); if (is_texel_buffer && !is_written) { @@ -473,71 +451,51 @@ bool BufferCache::SynchronizeBuffer(CommandBuffer& command, Buffer& buffer, uint return false; } -BufferBinding BufferCache::ObtainBuffer(CommandBuffer& command, uint64_t vaddr, uint64_t size, - bool is_written, bool is_read, bool is_formatted) { - if (command.IsInvalid() || command.IsExecute()) { +std::pair BufferCache::ObtainBuffer(uint64_t vaddr, uint64_t size, + bool is_written, bool is_texel_buffer, + BufferId id) { + auto& command = m_scheduler.Current(); + if (command.IsInvalid() || command.IsExecute() || vaddr == 0 || size == 0 || + vaddr >= TRACKER_ADDRESS_SIZE || size > TRACKER_ADDRESS_SIZE - vaddr) { EXIT("BufferCache: buffer request requires a recording command buffer\n"); } - if (is_read && !is_written && size <= CACHING_PAGE_SIZE && + if (!is_written && size <= CACHING_PAGE_SIZE && !m_memory_tracker.IsRegionGpuModified(vaddr, size) && m_memory_tracker.IsRegionCpuModified(vaddr, size)) { const auto alignment = std::max( m_graphics.physical_device_properties.limits.minUniformBufferOffsetAlignment, 1); - if (auto [mapped, offset] = m_stream_buffer.Map(size, alignment, false); - mapped != nullptr) { - if (Libs::LibKernel::Memory::TryReadBacking(vaddr, mapped, size)) { - m_stream_buffer.Commit(); - return {{}, m_stream_buffer.Handle(), offset}; - } - } else { - auto owner = std::make_shared(m_graphics, m_scheduler, MemoryUsage::Upload, 0, - AllFlags, size); - if (Libs::LibKernel::Memory::TryReadBacking(vaddr, owner->Mapped().data(), size)) { - owner->Flush(0, size); - return {owner, owner->Handle(), 0}; - } - } + auto [mapped, offset] = m_stream_buffer.Map(size, alignment); + EXIT_IF(mapped == nullptr || !Libs::LibKernel::Memory::TryReadBacking(vaddr, mapped, size)); + m_stream_buffer.Commit(); + return {&m_stream_buffer, offset}; } - if (is_formatted && is_written) { - (void)m_texture_cache.InvalidateMemoryFromGPU(vaddr, size, true); + auto* buffer = m_slot_buffers.try_get(id); + if (buffer == nullptr || buffer->is_deleted || !buffer->IsInBounds(vaddr, size)) { + id = FindBuffer(vaddr, size); + buffer = &m_slot_buffers[id]; } - - auto& cached = GetOrCreateBuffer(command, vaddr, size); - (void)SynchronizeBuffer(command, *cached.buffer, vaddr, size, is_written, - is_formatted && is_read); + (void)SynchronizeBuffer(*buffer, vaddr, size, is_written, is_texel_buffer); if (is_written) { m_gpu_modified_ranges.Add(vaddr, size); } - auto owner = cached.buffer; - uint64_t offset = cached.buffer->Offset(vaddr); - return {owner, owner->Handle(), offset}; + return {buffer, buffer->Offset(vaddr)}; } -std::shared_ptr BufferCache::ObtainNullBuffer() { - if (m_null_buffer != nullptr) { - return m_null_buffer; - } - m_null_buffer = std::make_shared(m_graphics, m_scheduler, MemoryUsage::DeviceLocal, 0, - AllFlags, 16); - const std::array zeros {}; - Upload(m_scheduler.Current(), *m_null_buffer, 0, zeros.data(), zeros.size()); - return m_null_buffer; -} - -ImageBufferSource BufferCache::ObtainBufferForImage(uint64_t vaddr, uint64_t size) { +std::pair BufferCache::ObtainBufferForImage(uint64_t vaddr, uint64_t size) { if (vaddr == 0 || size == 0 || vaddr >= TRACKER_ADDRESS_SIZE || size > TRACKER_ADDRESS_SIZE - vaddr) { EXIT("BufferCache: invalid image source\n"); } - auto find_owner = [&]() -> CachedBuffer* { + auto find_owner = [&]() -> Buffer* { auto owner = m_buffers.upper_bound(vaddr); if (owner == m_buffers.begin()) { return nullptr; } --owner; - return owner->second->buffer->IsInBounds(vaddr, size) ? owner->second.get() : nullptr; + auto* buffer = m_slot_buffers.try_get(owner->second); + return buffer != nullptr && buffer->IsInBounds(vaddr, size) ? buffer : nullptr; }; { @@ -553,17 +511,18 @@ ImageBufferSource BufferCache::ObtainBufferForImage(uint64_t vaddr, uint64_t siz if (!IsRegionRegistered(vaddr, size)) { EXIT("BufferCache: GPU-dirty image source has no native buffer\n"); } - auto& cached = GetOrCreateBuffer(m_scheduler.Current(), vaddr, size); - owner = &cached; - const auto resolved = m_buffers.find(cached.vaddr); - if (resolved == m_buffers.end() || resolved->second.get() != &cached || - !cached.buffer->IsInBounds(vaddr, size)) { + const auto id = FindBuffer(vaddr, size); + auto& buffer = m_slot_buffers[id]; + owner = &buffer; + const auto resolved = m_buffers.find(buffer.CpuAddress()); + if (resolved == m_buffers.end() || resolved->second != id || + !buffer.IsInBounds(vaddr, size)) { EXIT("BufferCache: merged image source does not contain the requested range\n"); } } if (owner != nullptr && !cpu_modified && (!gpu_modified || has_dirty_buffer_source)) { owner->tick_accessed_last = m_gc_tick; - return {owner->buffer.get(), owner->buffer->Offset(vaddr)}; + return {owner, owner->Offset(vaddr)}; } if (has_dirty_buffer_source && owner == nullptr) { EXIT("BufferCache: GPU-dirty image source could not resolve its native owner\n"); @@ -588,8 +547,7 @@ ImageBufferSource BufferCache::ObtainBufferForImage(uint64_t vaddr, uint64_t siz return {&m_staging_buffer, stage_offset}; } - auto& cached = *owner; - cached.tick_accessed_last = m_gc_tick; + owner->tick_accessed_last = m_gc_tick; std::vector> uploads; m_memory_tracker.ForEachUploadRange( vaddr, size, false, @@ -598,12 +556,12 @@ ImageBufferSource BufferCache::ObtainBufferForImage(uint64_t vaddr, uint64_t siz }, [&]() noexcept { for (const auto& [address, upload_size]: uploads) { - cached.buffer->CopyFrom( - m_scheduler.Current(), m_staging_buffer, stage_offset + address - vaddr, - cached.buffer->Offset(address), upload_size, vk::AccessFlagBits::eHostWrite); + owner->CopyFrom(m_scheduler.Current(), m_staging_buffer, + stage_offset + address - vaddr, owner->Offset(address), upload_size, + vk::AccessFlagBits::eHostWrite); } }); - return {cached.buffer.get(), cached.buffer->Offset(vaddr)}; + return {owner, owner->Offset(vaddr)}; } void BufferCache::WriteHostMemory(uint64_t vaddr, std::span data) { @@ -613,16 +571,16 @@ void BufferCache::WriteHostMemory(uint64_t vaddr, std::span data) Libs::LibKernel::Memory::WriteBacking(vaddr, data.data(), data.size()); const auto end = vaddr + data.size(); - for (auto& [address, cached]: m_buffers) { - const auto cached_end = address + cached->size; + for (const auto& [address, id]: m_buffers) { + auto& buffer = m_slot_buffers[id]; + const auto buffer_end = address + buffer.Size(); const auto begin = std::max(vaddr, address); - const auto range_end = std::min(end, cached_end); + const auto range_end = std::min(end, buffer_end); if (begin >= range_end) { continue; } - Upload(m_scheduler.Current(), *cached->buffer, cached->buffer->Offset(begin), - data.data() + begin - vaddr, range_end - begin); - cached->tick_accessed_last = m_gc_tick; + WriteDataBuffer(buffer, begin, data.data() + begin - vaddr, range_end - begin); + buffer.tick_accessed_last = m_gc_tick; } } @@ -660,12 +618,11 @@ void BufferCache::FillBuffer(uint64_t vaddr, uint64_t size, uint32_t value, bool } } - auto& command = m_scheduler.Current(); - auto dst = ObtainBuffer(command, vaddr, size, true, false, true); - EXIT_IF(dst.buffer == nullptr || dst.owner == nullptr); - command.RetainResourceUntilFence(dst.owner); - auto owner = std::static_pointer_cast(dst.owner); - owner->Fill(dst.offset, size, value); + m_texture_cache.InvalidateMemoryFromGPU(vaddr, size); + const auto id = FindBuffer(vaddr, size); + auto [dst, dst_offset] = ObtainBuffer(vaddr, size, true, true, id); + EXIT_IF(dst == nullptr); + dst->Fill(dst_offset, size, value); } void BufferCache::CopyBuffer(uint64_t dst_vaddr, uint64_t src_vaddr, uint64_t size, bool dst_gds, @@ -707,30 +664,20 @@ void BufferCache::CopyBuffer(uint64_t dst_vaddr, uint64_t src_vaddr, uint64_t si } auto& command = m_scheduler.Current(); - auto src = src_memory ? ObtainBuffer(command, src_vaddr, size, false, true, true) - : BufferBinding {.buffer = m_gds_buffer.Handle(), .offset = src_vaddr}; - auto dst = dst_memory ? ObtainBuffer(command, dst_vaddr, size, true, false, true) - : BufferBinding {.buffer = m_gds_buffer.Handle(), .offset = dst_vaddr}; - EXIT_IF(src.buffer == nullptr || dst.buffer == nullptr || (dst_memory && dst.owner == nullptr)); - if (src.owner != nullptr) { - command.RetainResourceUntilFence(src.owner); + if (dst_memory) { + m_texture_cache.InvalidateMemoryFromGPU(dst_vaddr, size); } - if (dst.owner != nullptr) { - command.RetainResourceUntilFence(dst.owner); - } - if (src.buffer == dst.buffer && src.offset < dst.offset + size && - dst.offset < src.offset + size) { + const auto src_id = src_memory ? FindBuffer(src_vaddr, size) : BufferId {}; + const auto dst_id = dst_memory ? FindBuffer(dst_vaddr, size) : BufferId {}; + auto [src, src_offset] = src_memory ? ObtainBuffer(src_vaddr, size, false, true, src_id) + : std::pair {&m_gds_buffer, src_vaddr}; + auto [dst, dst_offset] = dst_memory ? ObtainBuffer(dst_vaddr, size, true, true, dst_id) + : std::pair {&m_gds_buffer, dst_vaddr}; + EXIT_IF(src == nullptr || dst == nullptr); + if (src == dst && src_offset < dst_offset + size && dst_offset < src_offset + size) { EXIT("BufferCache: resolved Vulkan copy ranges overlap\n"); } - auto& source = src.owner != nullptr ? *std::static_pointer_cast(src.owner) - : src_gds ? m_gds_buffer - : m_stream_buffer; - auto& destination = - dst.owner != nullptr ? *std::static_pointer_cast(dst.owner) : m_gds_buffer; - if (source.Handle() != src.buffer || destination.Handle() != dst.buffer) { - EXIT("BufferCache: resolved copy owner does not match its Vulkan handle\n"); - } - destination.CopyFrom(command, source, src.offset, dst.offset, size); + dst->CopyFrom(command, *src, src_offset, dst_offset, size); } bool BufferCache::IsRegionRegistered(uint64_t vaddr, uint64_t size) { @@ -744,8 +691,8 @@ bool BufferCache::IsRegionRegistered(uint64_t vaddr, uint64_t size) { if (candidate == m_buffers.begin()) { return false; } - const auto& [address, cached] = *std::prev(candidate); - return address + cached->size > vaddr; + const auto& [address, id] = *std::prev(candidate); + return address + m_slot_buffers[id].Size() > vaddr; } bool BufferCache::IsRegionGpuModified(uint64_t vaddr, uint64_t size) { @@ -773,70 +720,56 @@ void BufferCache::RunGarbageCollector() { const uint64_t age = std::min(aggressive ? 80 : 160, tick); const size_t limit = aggressive ? 64 : 32; - std::vector retires; - std::vector>> dirty_retires; - { - std::vector candidates; - for (const auto& [address, owner]: m_buffers) { - const auto& cached = *owner; - if (tick - std::min(tick, cached.tick_accessed_last) < age) { - continue; - } + std::vector candidates; + for (const auto& [address, id]: m_buffers) { + const auto& buffer = m_slot_buffers[id]; + if (tick - std::min(tick, buffer.tick_accessed_last) >= age) { candidates.push_back(address); } - std::ranges::sort(candidates, [&](uint64_t left, uint64_t right) { - return m_buffers.at(left)->tick_accessed_last < m_buffers.at(right)->tick_accessed_last; - }); - if (candidates.size() > limit) { - candidates.resize(limit); - } - for (const auto address: candidates) { - auto& cached = *m_buffers.at(address); - m_memory_tracker.ValidateGpuDirtyOwnership(m_gpu_modified_ranges, cached.vaddr, - cached.size, "garbage collection"); - retires.push_back({address, cached.size, cached.buffer}); - // GC runs immediately before submission. Preserve every source referenced by commands - // already recorded in the active batch. - m_scheduler.Current().RetainResourceUntilFence(cached.buffer); - } - for (const auto& retire: retires) { - if (!m_memory_tracker.IsRegionGpuModified(retire.address, retire.size)) { - continue; - } - auto& copies = dirty_retires.emplace_back(retire, std::vector {}).second; + } + std::ranges::sort(candidates, [&](uint64_t left, uint64_t right) { + return m_slot_buffers[m_buffers.at(left)].tick_accessed_last < + m_slot_buffers[m_buffers.at(right)].tick_accessed_last; + }); + if (candidates.size() > limit) { + candidates.resize(limit); + } + + for (const auto address: candidates) { + auto found = m_buffers.find(address); + EXIT_IF(found == m_buffers.end()); + const auto id = found->second; + auto& buffer = m_slot_buffers[id]; + m_memory_tracker.ValidateGpuDirtyOwnership(m_gpu_modified_ranges, buffer.CpuAddress(), + buffer.Size(), "garbage collection"); + const bool dirty = m_memory_tracker.IsRegionGpuModified(buffer.CpuAddress(), buffer.Size()); + std::vector copies; + if (dirty) { m_memory_tracker.ForEachDownloadRange( - retire.address, retire.size, - [&](uint64_t address, uint64_t size) noexcept { - m_memory_tracker.ValidateGpuDirtyPages(m_gpu_modified_ranges, address, size, - "garbage collection"); + buffer.CpuAddress(), buffer.Size(), + [&](uint64_t dirty_address, uint64_t dirty_size) noexcept { + m_memory_tracker.ValidateGpuDirtyPages(m_gpu_modified_ranges, dirty_address, + dirty_size, "garbage collection"); }, - [&](uint64_t address, uint64_t size) noexcept { - for (const auto range: m_gpu_modified_ranges.Intersections(address, size)) { - copies.push_back({retire.owner, range.address - retire.address, + [&](uint64_t dirty_address, uint64_t dirty_size) noexcept { + for (const auto range: + m_gpu_modified_ranges.Intersections(dirty_address, dirty_size)) { + copies.push_back({&buffer, range.address - buffer.CpuAddress(), range.address, range.size}); } }); } - } - for (auto& [retire, copies]: dirty_retires) { - QueueGarbageDownload(copies, std::move(retire)); - } - - for (const auto& retire: retires) { - auto found = m_buffers.find(retire.address); - if (found == m_buffers.end() || found->second->size != retire.size || - found->second->buffer != retire.owner) { - EXIT("BufferCache: garbage-collection owner changed during download\n"); + const auto retire_address = buffer.CpuAddress(); + const auto retire_size = buffer.Size(); + if (dirty) { + EXIT_IF(copies.empty()); + Unregister(id); + QueueGarbageDownload(copies, id, retire_address, retire_size); + } else { + m_memory_tracker.UntrackMemory(retire_address, retire_size); + DeleteBuffer(id); } - if (!m_memory_tracker.IsRegionGpuModified(retire.address, retire.size)) { - m_memory_tracker.UntrackMemory(retire.address, retire.size); - } - if (retire.size > m_total_used_memory) { - EXIT("BufferCache: allocation accounting underflow\n"); - } - m_total_used_memory -= retire.size; - m_buffers.erase(found); } } diff --git a/src/graphics/host_gpu/renderer/cache/bufferCache.h b/src/graphics/host_gpu/renderer/cache/bufferCache.h index 8a5981988..2a8882915 100644 --- a/src/graphics/host_gpu/renderer/cache/bufferCache.h +++ b/src/graphics/host_gpu/renderer/cache/bufferCache.h @@ -3,12 +3,12 @@ #include "common/abi.h" #include "common/common.h" +#include "common/slotVector.h" #include "graphics/host_gpu/memoryTracker.h" #include "graphics/host_gpu/rangeSet.h" #include "graphics/host_gpu/renderer/cache/streamBuffer.h" #include -#include #include #include #include @@ -16,49 +16,44 @@ namespace Libs::Graphics { struct GraphicContext; -class CommandBuffer; class CommandScheduler; class TextureCache; -struct BufferBinding { - std::shared_ptr owner; - vk::Buffer buffer = nullptr; - uint64_t offset = 0; -}; - -struct ImageBufferSource { - Buffer* buffer = nullptr; - uint64_t offset = 0; -}; +using BufferId = Common::SlotId; +inline constexpr BufferId NULL_BUFFER_ID {0}; class BufferCache { public: static constexpr uint64_t CACHING_PAGE_SIZE = 16ull * 1024ull; - static constexpr uint64_t GetBufferOffset(uint64_t vaddr) { - return vaddr & (CACHING_PAGE_SIZE - 1); - } BufferCache(GraphicContext& graphics, CommandScheduler& scheduler, PageManager& page_manager, TextureCache& texture_cache); ~BufferCache(); KYTY_CLASS_NO_COPY(BufferCache); - void InvalidateMemory(uint64_t vaddr, uint64_t size); - void ReadMemory(uint64_t vaddr, uint64_t size, bool is_write = false); - void UnmapMemory(uint64_t vaddr, uint64_t size); - [[nodiscard]] BufferBinding ObtainBuffer(CommandBuffer& command, uint64_t vaddr, uint64_t size, - bool is_written = false, bool is_read = true, - bool is_formatted = false); - [[nodiscard]] StreamBuffer& GetUtilityBuffer(MemoryUsage usage) noexcept; - [[nodiscard]] Buffer& GetGdsBuffer() noexcept { return m_gds_buffer; } - [[nodiscard]] const Buffer& GetGdsBuffer() const noexcept { return m_gds_buffer; } - [[nodiscard]] BufferBinding UploadTransient(const void* data, uint64_t size, - uint64_t alignment); - [[nodiscard]] std::shared_ptr ObtainNullBuffer(); - [[nodiscard]] ImageBufferSource ObtainBufferForImage(uint64_t vaddr, uint64_t size); - void FillBuffer(uint64_t vaddr, uint64_t size, uint32_t value, bool is_gds = false); - void CopyBuffer(uint64_t dst_vaddr, uint64_t src_vaddr, uint64_t size, bool dst_gds = false, - bool src_gds = false); + void InvalidateMemory(uint64_t vaddr, uint64_t size); + void ReadMemory(uint64_t vaddr, uint64_t size, bool is_write = false); + void UnmapMemory(uint64_t vaddr, uint64_t size); + [[nodiscard]] Buffer& GetBuffer(BufferId id) { return m_slot_buffers[id]; } + [[nodiscard]] BufferId FindBuffer(uint64_t vaddr, uint64_t size); + [[nodiscard]] std::pair ObtainBuffer(uint64_t vaddr, uint64_t size, + bool is_written, + bool is_texel_buffer = false, + BufferId id = {}); + [[nodiscard]] StreamBuffer& GetUtilityBuffer(MemoryUsage usage) noexcept { + switch (usage) { + case MemoryUsage::Upload: return m_staging_buffer; + case MemoryUsage::Stream: return m_stream_buffer; + case MemoryUsage::Download: return m_download_buffer; + case MemoryUsage::DeviceLocal: return m_device_buffer; + } + EXIT("BufferCache: invalid utility-buffer usage\n"); + } + [[nodiscard]] const Buffer* GetGdsBuffer() const noexcept { return &m_gds_buffer; } + [[nodiscard]] std::pair ObtainBufferForImage(uint64_t vaddr, uint64_t size); + void FillBuffer(uint64_t vaddr, uint64_t size, uint32_t value, bool is_gds); + void CopyBuffer(uint64_t dst_vaddr, uint64_t src_vaddr, uint64_t size, bool dst_gds, + bool src_gds); // Cache-index and exact dirty-range queries require GPU-thread serialization. [[nodiscard]] bool IsRegionRegistered(uint64_t vaddr, uint64_t size); [[nodiscard]] bool HasGpuDirtyBytes(uint64_t vaddr, uint64_t size); @@ -69,51 +64,42 @@ public: private: friend struct BufferCacheTestAccess; - struct CacheRange { - uint64_t address = 0; - uint64_t size = 0; - }; - struct CachedBuffer; struct DownloadCopy; struct DownloadRange; - struct RetiredBuffer; static constexpr uint64_t DOWNLOAD_ALIGNMENT = 64; - [[nodiscard]] static uint64_t AlignDown(uint64_t value) noexcept; - [[nodiscard]] static uint64_t AlignUp(uint64_t value); [[nodiscard]] static constexpr uint64_t AlignDownload(uint64_t size) noexcept { return (size + DOWNLOAD_ALIGNMENT - 1) & ~(DOWNLOAD_ALIGNMENT - 1); } [[nodiscard]] static std::pair DownloadEnvelope(const DownloadCopy& copy); - [[nodiscard]] static bool ResolveOverlap(CacheRange& merged, CacheRange candidate) noexcept; - void Upload(CommandBuffer& command, Buffer& destination, uint64_t destination_offset, - const void* source, uint64_t size); - [[nodiscard]] CachedBuffer& GetOrCreateBuffer(CommandBuffer& command, uint64_t vaddr, - uint64_t size); - [[nodiscard]] bool SynchronizeBuffer(CommandBuffer& command, Buffer& buffer, uint64_t vaddr, - uint64_t size, bool is_written, bool is_texel_buffer); + void WriteDataBuffer(Buffer& buffer, uint64_t address, const void* source, uint64_t size); + void Unregister(BufferId id); + void DeleteBuffer(BufferId id); + [[nodiscard]] bool SynchronizeBuffer(Buffer& buffer, uint64_t vaddr, uint64_t size, + bool is_written, bool is_texel_buffer); [[nodiscard]] bool SynchronizeBufferFromImage(Buffer& buffer, uint64_t vaddr, uint64_t size); [[nodiscard]] std::vector RecordDownloads(std::span copies); void PublishDownloads(std::span downloads); - void QueueGarbageDownload(std::span copies, RetiredBuffer retire); + void QueueGarbageDownload(std::span copies, BufferId id, uint64_t address, + uint64_t size); void WriteHostMemory(uint64_t vaddr, std::span data); void ReadMemoryOnGpu(uint64_t vaddr, uint64_t size, bool is_write); - GraphicContext& m_graphics; - CommandScheduler& m_scheduler; - Buffer m_gds_buffer; - std::shared_ptr m_null_buffer; - std::map> m_buffers; - RangeSet m_gpu_modified_ranges; - MemoryTracker m_memory_tracker; - StreamBuffer m_staging_buffer; - StreamBuffer m_stream_buffer; - StreamBuffer m_download_buffer; - StreamBuffer m_device_buffer; - TextureCache& m_texture_cache; - uint64_t m_total_used_memory = 0; - uint64_t m_trigger_gc_memory = 1ull * 1024 * 1024 * 1024; - uint64_t m_critical_gc_memory = 2ull * 1024 * 1024 * 1024; - uint64_t m_gc_tick = 0; + GraphicContext& m_graphics; + CommandScheduler& m_scheduler; + Buffer m_gds_buffer; + Common::SlotVector m_slot_buffers; + std::map m_buffers; + RangeSet m_gpu_modified_ranges; + MemoryTracker m_memory_tracker; + StreamBuffer m_staging_buffer; + StreamBuffer m_stream_buffer; + StreamBuffer m_download_buffer; + StreamBuffer m_device_buffer; + TextureCache& m_texture_cache; + uint64_t m_total_used_memory = 0; + uint64_t m_trigger_gc_memory = 1ull * 1024 * 1024 * 1024; + uint64_t m_critical_gc_memory = 2ull * 1024 * 1024 * 1024; + uint64_t m_gc_tick = 0; }; } // namespace Libs::Graphics diff --git a/src/graphics/host_gpu/renderer/cache/streamBuffer.h b/src/graphics/host_gpu/renderer/cache/streamBuffer.h index d2f2c4b8c..9b1cf4159 100644 --- a/src/graphics/host_gpu/renderer/cache/streamBuffer.h +++ b/src/graphics/host_gpu/renderer/cache/streamBuffer.h @@ -65,6 +65,10 @@ public: vk::AccessFlagBits::eMemoryWrite); void Fill(uint64_t offset, uint64_t size, uint32_t value); + // BufferCache state lives directly on the resource. + bool is_deleted = false; + uint64_t tick_accessed_last = 0; + protected: [[nodiscard]] GraphicContext& Graphics() const noexcept { return *m_graphics; } [[nodiscard]] CommandScheduler& Scheduler() const noexcept { return *m_scheduler; } diff --git a/src/graphics/host_gpu/renderer/cache/textureCache.cpp b/src/graphics/host_gpu/renderer/cache/textureCache.cpp index 47726ce72..a4c001093 100644 --- a/src/graphics/host_gpu/renderer/cache/textureCache.cpp +++ b/src/graphics/host_gpu/renderer/cache/textureCache.cpp @@ -32,8 +32,6 @@ namespace { constexpr uint64_t NumFramesBeforeRemoval = 32; -thread_local const TextureCache* g_locked_cache = nullptr; - [[nodiscard]] const char* BindingTypeName(TextureCache::BindingType type) { switch (type) { case TextureCache::BindingType::Texture: return "Texture"; @@ -64,32 +62,13 @@ void NameImageBinding(GraphicContext& graphics, Image& image, vk::ImageView view view_info.level_count, view_info.base_layer, view_info.layer_count); } -class CacheLock final { -public: - CacheLock(const TextureCache& owner, TrackingSpinLock& lock): m_lock(lock) { - if (g_locked_cache != nullptr) { - EXIT("TextureCache: recursive cache lock\n"); - } - g_locked_cache = &owner; - m_lock.lock(); - } - ~CacheLock() { - m_lock.unlock(); - g_locked_cache = nullptr; - } - -private: - TrackingSpinLock& m_lock; -}; - } // namespace TextureCache::TextureCache(GraphicContext& graphics, CommandScheduler& scheduler, PageManager& page_manager, BufferCache& buffer_cache) : m_graphics(graphics), m_scheduler(scheduler), m_page_manager(page_manager), m_blit_helper(graphics, scheduler), - m_tiler(std::make_unique(graphics, scheduler, - buffer_cache.GetUtilityBuffer(MemoryUsage::Stream))), + m_tiler(graphics, scheduler, buffer_cache.GetUtilityBuffer(MemoryUsage::Stream)), m_buffer_cache(buffer_cache), m_readback_linear_images(Config::ReadbackLinearImagesEnabled()) { if (m_graphics.CanReportMemoryUsage()) { @@ -106,11 +85,14 @@ TextureCache::TextureCache(GraphicContext& graphics, CommandScheduler& scheduler } TextureCache::~TextureCache() { - for (uint32_t index = 0; index < m_slots.size(); index++) { - if (m_slots[index].image != nullptr && m_slots[index].image->registered) { - UnregisterImage({index, m_slots[index].generation}); + std::vector registered; + m_slot_images.ForEach([&](ImageId id, const Image& image) { + if (image.registered) { + registered.push_back(id); } - m_slots[index].image.reset(); + }); + for (const auto id: registered) { + UnregisterImage(id); } } @@ -167,51 +149,8 @@ bool TextureCache::SafeToDownload(const Image& image) { return !m_buffer_cache.HasGpuDirtyBytes(range.address, range.size); } -Image& TextureCache::ResolveImage(ImageId id) { - if (!id || id.index >= m_slots.size()) { - EXIT("TextureCache: invalid image id\n"); - } - auto& slot = m_slots[id.index]; - if (slot.generation != id.generation || slot.image == nullptr) { - EXIT("TextureCache: stale image id\n"); - } - return *slot.image; -} - -const Image& TextureCache::ResolveImage(ImageId id) const { - if (!id || id.index >= m_slots.size()) { - EXIT("TextureCache: invalid image id\n"); - } - const auto& slot = m_slots[id.index]; - if (slot.generation != id.generation || slot.image == nullptr) { - EXIT("TextureCache: stale image id\n"); - } - return *slot.image; -} - -std::shared_ptr TextureCache::ResolveOwner(ImageId id) const { - if (!id || id.index >= m_slots.size()) { - return {}; - } - const auto& slot = m_slots[id.index]; - return slot.generation == id.generation ? slot.image : nullptr; -} - ImageId TextureCache::InsertImage(const ImageInfo& info) { - uint32_t index = 0; - if (m_free_slots.empty()) { - index = static_cast(m_slots.size()); - m_slots.emplace_back(); - } else { - index = m_free_slots.back(); - m_free_slots.pop_back(); - } - auto& slot = m_slots[index]; - if (slot.image != nullptr) { - EXIT("TextureCache: occupied free image slot\n"); - } - slot.image = std::make_shared(m_graphics, m_scheduler, info); - const ImageId id {index, slot.generation}; + const auto id = m_slot_images.insert(m_graphics, m_scheduler, info); if (!info.data.Empty()) { RegisterImage(id); } @@ -219,7 +158,7 @@ ImageId TextureCache::InsertImage(const ImageInfo& info) { } void TextureCache::RegisterImage(ImageId id) { - auto& image = ResolveImage(id); + auto& image = m_slot_images[id]; if (image.registered || image.info.data.Empty()) { EXIT("TextureCache: invalid image registration\n"); } @@ -236,7 +175,7 @@ void TextureCache::RegisterImage(ImageId id) { } void TextureCache::UnregisterImage(ImageId id) { - auto& image = ResolveImage(id); + auto& image = m_slot_images[id]; if (!image.registered) { return; } @@ -261,69 +200,48 @@ void TextureCache::UnregisterImage(ImageId id) { } void TextureCache::DeleteImage(ImageId id) { - auto owner = ResolveOwner(id); - if (owner == nullptr || !owner->registered) { + auto* image = m_slot_images.try_get(id); + if (image == nullptr || !image->registered) { return; } - if (!owner->depth_id) { + if (!image->depth_id) { std::vector associations; - for (uint32_t index = 0; index < m_slots.size(); index++) { - const auto& slot = m_slots[index]; - if (slot.image != nullptr && slot.image->depth_id == id) { - associations.push_back({index, slot.generation}); + m_slot_images.ForEach([&](ImageId candidate, const Image& associated) { + if (associated.depth_id == id) { + associations.push_back(candidate); } - } + }); for (const auto association: associations) { - ClearGpuModified(association); + auto& associated = m_slot_images[association]; + if (associated.IsGpuModified()) { + associated.ClearGpuModified(); + } DeleteImage(association); } } - if (owner->IsGpuModified()) { + if (image->IsGpuModified()) { EXIT("TextureCache: deleting a GPU-modified image without resolving its contents\n"); } m_download_images.erase(id); - if (owner->info.HasMetadata()) { - m_surface_metas.erase(owner->info.metadata.range.address); + if (image->info.HasMetadata()) { + m_surface_metas.erase(image->info.metadata.range.address); } UnregisterImage(id); - const auto erase_slot = [this, id, retained = owner] { - auto& slot = m_slots[id.index]; - if (slot.generation != id.generation || slot.image != retained) { - EXIT("TextureCache: retired image slot changed before deferred erasure\n"); - } - slot.image.reset(); - if (++slot.generation == 0) { - slot.generation = 1; - } - m_free_slots.push_back(id.index); - }; if (m_scheduler.Active()) { - m_scheduler.DeferOperation(erase_slot); + m_scheduler.DeferOperation([this, id] { m_slot_images.erase(id); }); } else { - erase_slot(); + m_slot_images.erase(id); } } void TextureCache::FreeImage(ImageId id) { - ClearGpuModified(id); + auto& image = m_slot_images[id]; + if (image.IsGpuModified()) { + image.ClearGpuModified(); + } DeleteImage(id); } -void TextureCache::RetainImage(CommandBuffer& command, ImageId id) { - auto owner = ResolveOwner(id); - if (owner == nullptr) { - EXIT("TextureCache: retaining a stale image\n"); - } - if (owner->depth_id) { - auto depth = ResolveOwner(owner->depth_id); - if (depth == nullptr) { - EXIT("TextureCache: stencil association points to a stale depth image\n"); - } - command.RetainResourceUntilFence(std::move(depth)); - } - command.RetainResourceUntilFence(std::move(owner)); -} - void TextureCache::TouchImage(Image& image) { if (image.registered) { m_lru_cache.Touch(image.lru_id, m_gc_tick); @@ -339,7 +257,7 @@ void TextureCache::MarkAsMaybeDirty(ImageId id, Image& image) { } void TextureCache::TrackImage(ImageId id) { - auto& image = ResolveImage(id); + auto& image = m_slot_images[id]; if (!image.registered) { return; } @@ -363,7 +281,7 @@ void TextureCache::TrackImage(ImageId id) { } void TextureCache::TrackImageHead(ImageId id) { - auto& image = ResolveImage(id); + auto& image = m_slot_images[id]; if (!image.registered) { return; } @@ -380,7 +298,7 @@ void TextureCache::TrackImageHead(ImageId id) { } void TextureCache::TrackImageTail(ImageId id) { - auto& image = ResolveImage(id); + auto& image = m_slot_images[id]; if (!image.registered) { return; } @@ -398,7 +316,7 @@ void TextureCache::TrackImageTail(ImageId id) { } void TextureCache::UntrackImage(ImageId id) { - auto& image = ResolveImage(id); + auto& image = m_slot_images[id]; if (!image.IsTracked()) { return; } @@ -412,7 +330,7 @@ void TextureCache::UntrackImage(ImageId id) { } void TextureCache::UntrackImageHead(ImageId id) { - auto& image = ResolveImage(id); + auto& image = m_slot_images[id]; const auto begin = image.info.data.address; if (!image.IsTracked() || begin < image.track_addr) { return; @@ -429,7 +347,7 @@ void TextureCache::UntrackImageHead(ImageId id) { } void TextureCache::UntrackImageTail(ImageId id) { - auto& image = ResolveImage(id); + auto& image = m_slot_images[id]; const auto end = image.info.data.End(); if (!image.IsTracked() || image.track_addr_end < end) { return; @@ -445,13 +363,7 @@ void TextureCache::UntrackImageTail(ImageId id) { } } -void TextureCache::TrackImageDownload(ImageId id) { - CacheLock lock(*this, m_lock); - auto& image = ResolveImage(id); - TrackImageDownloadLocked(id, image); -} - -void TextureCache::TrackImageDownloadLocked(ImageId id, Image& image) { +void TextureCache::TrackImageDownload(ImageId id, Image& image) { if (m_readback_linear_images && !image.info.IsTiled() && !image.info.data.Empty()) { if (!image.IsGpuModified()) { EXIT("TextureCache: cannot enroll a non-GPU-owned image for download\n"); @@ -460,16 +372,6 @@ void TextureCache::TrackImageDownloadLocked(ImageId id, Image& image) { } } -Image& TextureCache::GetImage(ImageId id) { - auto& image = ResolveImage(id); - TouchImage(image); - return image; -} - -const Image& TextureCache::GetImage(ImageId id) const { - return ResolveImage(id); -} - TextureCache::ImageIds TextureCache::FindImagesInRegion(uint64_t address, uint64_t size, bool page_overlap) const { ImagePageTable::PageRange pages {}; @@ -479,11 +381,7 @@ TextureCache::ImageIds TextureCache::FindImagesInRegion(uint64_t address, uint64 uint32_t query_epoch = ++m_image_query_epoch; if (query_epoch == 0) { - for (const auto& slot: m_slots) { - if (slot.image != nullptr) { - slot.image->query_epoch = 0; - } - } + m_slot_images.ForEach([](ImageId, const Image& image) { image.query_epoch = 0; }); query_epoch = ++m_image_query_epoch; } @@ -494,19 +392,15 @@ TextureCache::ImageIds TextureCache::FindImagesInRegion(uint64_t address, uint64 continue; } owners->ForEach([&](ImageId id) { - if (!id || id.index >= m_slots.size()) { + auto* image = m_slot_images.try_get(id); + if (image == nullptr) { return; } - const auto& slot = m_slots[id.index]; - if (slot.generation != id.generation || slot.image == nullptr) { + if (image->query_epoch == query_epoch) { return; } - auto& image = *slot.image; - if (image.query_epoch == query_epoch) { - return; - } - image.query_epoch = query_epoch; - if (image.Overlaps(address, size, page_overlap)) { + image->query_epoch = query_epoch; + if (image->Overlaps(address, size, page_overlap)) { result.push_back(id); } }); @@ -515,10 +409,8 @@ TextureCache::ImageIds TextureCache::FindImagesInRegion(uint64_t address, uint64 } ImageId TextureCache::GetNullImage(const ImageDesc& desc) { - auto& command = m_scheduler.Current(); - const auto format = desc.info.pixel_format; + const auto format = desc.info.pixel_format; if (const auto found = m_null_images.find(format); found != m_null_images.end()) { - RetainImage(command, found->second); return found->second; } ImageInfo info {}; @@ -534,7 +426,6 @@ ImageId TextureCache::GetNullImage(const ImageDesc& desc) { info.mip_layout[0] = {0, info.bytes_per_block, 1, 1}; const auto id = InsertImage(info); m_null_images.emplace(format, id); - RetainImage(command, id); return id; } @@ -568,7 +459,7 @@ void TextureCache::PrepareImageCopy(Image& image) { } void TextureCache::RefreshCopySource(ImageId id) { - auto& image = ResolveImage(id); + auto& image = m_slot_images[id]; RefreshImage(id, ImageDesc {.info = image.info, .view_info = {}, .type = UploadBinding(image)}); if (image.IsDefinitelyCpuDirty()) { EXIT("TextureCache: image copy source remained CPU-dirty after refresh\n"); @@ -616,8 +507,8 @@ bool TextureCache::CopyD16(Image& destination, Image& source) { color_copies[layer].imageExtent = color.info.extent; } - auto depth_buffer = m_tiler->GetScratchBuffer(depth_size); - auto color_buffer = m_tiler->GetScratchBuffer(color_size); + auto depth_buffer = m_tiler.GetScratchBuffer(depth_size); + auto color_buffer = m_tiler.GetScratchBuffer(color_size); const TileManager::D16Layout promote_layout { .width = depth.info.extent.width, .height = depth.info.extent.height, @@ -630,20 +521,20 @@ bool TextureCache::CopyD16(Image& destination, Image& source) { const bool d32 = DepthAspectTransferFormat(depth.backing.format) == vk::Format::eD32Sfloat; if (source_depth) { source.Download(depth_copies, depth_buffer.buffer, depth_buffer.offset, depth_buffer.size); - m_tiler->ConvertD16(depth_buffer, color_buffer, TileManager::D16Direction::Demote, d32, - {.width = promote_layout.width, - .height = promote_layout.height, - .layers = promote_layout.layers, - .source_row_stride = promote_layout.target_row_stride, - .target_row_stride = promote_layout.source_row_stride, - .source_slice_stride = promote_layout.target_slice_stride, - .target_slice_stride = promote_layout.source_slice_stride}); + m_tiler.ConvertD16(depth_buffer, color_buffer, TileManager::D16Direction::Demote, d32, + {.width = promote_layout.width, + .height = promote_layout.height, + .layers = promote_layout.layers, + .source_row_stride = promote_layout.target_row_stride, + .target_row_stride = promote_layout.source_row_stride, + .source_slice_stride = promote_layout.target_slice_stride, + .target_slice_stride = promote_layout.source_slice_stride}); destination.Upload(color_copies, color_buffer.buffer, color_buffer.offset, color_buffer.size); } else { source.Download(color_copies, color_buffer.buffer, color_buffer.offset, color_buffer.size); - m_tiler->ConvertD16(color_buffer, depth_buffer, TileManager::D16Direction::Promote, d32, - promote_layout); + m_tiler.ConvertD16(color_buffer, depth_buffer, TileManager::D16Direction::Promote, d32, + promote_layout); destination.Upload(depth_copies, depth_buffer.buffer, depth_buffer.offset, depth_buffer.size); } @@ -652,8 +543,8 @@ bool TextureCache::CopyD16(Image& destination, Image& source) { void TextureCache::CopyImage(ImageId destination_id, ImageId source_id) { RefreshCopySource(source_id); - auto& destination = ResolveImage(destination_id); - auto& source = ResolveImage(source_id); + auto& destination = m_slot_images[destination_id]; + auto& source = m_slot_images[source_id]; TrackImage(destination_id); if (source.backing.samples != destination.backing.samples) { EXIT("TextureCache: cannot issue an unequal-sample image copy\n"); @@ -680,8 +571,6 @@ void TextureCache::CopyImage(ImageId destination_id, ImageId source_id) { auto& copy_buffer = m_buffer_cache.GetUtilityBuffer(MemoryUsage::DeviceLocal); destination.CopyImageWithBuffer(source, copy_buffer); } - RetainImage(m_scheduler.Current(), source_id); - RetainImage(m_scheduler.Current(), destination_id); if (source.IsGpuModified()) { destination.MarkGpuModified(); } @@ -691,15 +580,13 @@ void TextureCache::CopyImage(ImageId destination_id, ImageId source_id) { void TextureCache::CopyImageMip(ImageId destination_id, ImageId source_id, uint32_t mip, uint32_t layer) { RefreshCopySource(source_id); - auto& destination = ResolveImage(destination_id); - auto& source = ResolveImage(source_id); + auto& destination = m_slot_images[destination_id]; + auto& source = m_slot_images[source_id]; TrackImage(destination_id); if (source.IsBufferModified() || source.backing.samples != destination.backing.samples) { EXIT("TextureCache: invalid mip-copy ownership or sample count\n"); } destination.CopyMip(source, mip, layer); - RetainImage(m_scheduler.Current(), source_id); - RetainImage(m_scheduler.Current(), destination_id); if (source.IsGpuModified()) { destination.MarkGpuModified(); } @@ -707,7 +594,7 @@ void TextureCache::CopyImageMip(ImageId destination_id, ImageId source_id, uint3 ImageId TextureCache::ResolveDepthOverlap(const ImageInfo& requested, BindingType binding, ImageId cached_id) { - auto& cached = ResolveImage(cached_id); + auto& cached = m_slot_images[cached_id]; if (!cached.info.IsDepth() && !requested.IsDepth()) { return {}; } @@ -773,7 +660,7 @@ ImageId TextureCache::ResolveDepthOverlap(const ImageInfo& requested, BindingTyp } info.htile_clear_mask = 0; const auto replacement_id = InsertImage(info); - auto& replacement = ResolveImage(replacement_id); + auto& replacement = m_slot_images[replacement_id]; replacement.usage = cached.usage; if (cached.binding.is_bound || cached.binding.is_target) { cached.binding.needs_rebind = true; @@ -797,9 +684,6 @@ ImageId TextureCache::ResolveDepthOverlap(const ImageInfo& requested, BindingTyp } PrepareImageCopy(replacement); m_blit_helper.ReinterpretColorAsMsDepth(cached, replacement); - auto& command = m_scheduler.Current(); - RetainImage(command, cached_id); - RetainImage(command, replacement_id); CommitGpuWrite(replacement); } else { LOGF_COLOR(Log::Color::BrightYellow, @@ -813,7 +697,7 @@ ImageId TextureCache::ResolveDepthOverlap(const ImageInfo& requested, BindingTyp TextureCache::OverlapResult TextureCache::ResolveOverlap(const ImageInfo& requested, BindingType binding, ImageId cached_id, ImageId merged_id) { - auto owner = ResolveOwner(cached_id); + auto owner = m_slot_images.try_get(cached_id); if (owner == nullptr) { return {merged_id}; } @@ -861,7 +745,7 @@ TextureCache::OverlapResult TextureCache::ResolveOverlap(const ImageInfo& reques if (requested.pixel_format != cached.info.pixel_format || requested.data.size <= cached.info.data.size) { const auto result_id = merged_id ? merged_id : cached_id; - const auto result = ResolveOwner(result_id); + const auto result = m_slot_images.try_get(result_id); return {result != nullptr && ImageViewOps::FormatsCompatible(result->info.pixel_format, requested.pixel_format) ? result_id @@ -902,7 +786,7 @@ TextureCache::OverlapResult TextureCache::ResolveOverlap(const ImageInfo& reques if (cached.binding.is_target) { cached.binding.needs_rebind = true; if (merged_id) { - ResolveImage(merged_id).binding.is_target = true; + m_slot_images[merged_id].binding.is_target = true; } FreeImage(cached_id); return {merged_id}; @@ -920,8 +804,8 @@ TextureCache::OverlapResult TextureCache::ResolveOverlap(const ImageInfo& reques ImageId TextureCache::ExpandImage(const ImageInfo& info, ImageId source_id) { RefreshCopySource(source_id); const auto expanded_id = InsertImage(info); - auto& expanded = ResolveImage(expanded_id); - auto& source = ResolveImage(source_id); + auto& expanded = m_slot_images[expanded_id]; + auto& source = m_slot_images[source_id]; expanded.usage = source.usage; if (source.binding.is_bound || source.binding.is_target) { source.binding.needs_rebind = true; @@ -1069,11 +953,11 @@ void TextureCache::UploadImage(Image& image, const ImageDesc& desc, Buffer& sour } TileManager::Result linear {source.Handle(), source_offset, info.data.size}; if (plan.tiled) { - linear = m_tiler->Detile(source.Handle(), source_offset, info.data.size, - plan.linear_size, plan.tiles); + linear = m_tiler.Detile(source.Handle(), source_offset, info.data.size, + plan.linear_size, plan.tiles); } if (plan.swap_bgra16) { - linear = m_tiler->SwapBgra16(linear); + linear = m_tiler.SwapBgra16(linear); } upload(plan.regions, linear); return; @@ -1110,7 +994,7 @@ void TextureCache::UploadImage(Image& image, const ImageDesc& desc, Buffer& sour TileManager::Result linear {source.Handle(), source_offset, source.Size() - source_offset}; if (!tiles.empty()) { linear = - m_tiler->Detile(source.Handle(), source_offset, info.data.size, info.data.size, tiles); + m_tiler.Detile(source.Handle(), source_offset, info.data.size, info.data.size, tiles); } const auto transfer_bytes = DepthAspectTransferBytes(info.pixel_format); if (transfer_bytes != info.bytes_per_block) { @@ -1120,8 +1004,8 @@ void TextureCache::UploadImage(Image& image, const ImageDesc& desc, Buffer& sour texels_per_slice > UINT64_MAX / transfer_bytes); const uint64_t transfer_slice = texels_per_slice * transfer_bytes; EXIT_NOT_IMPLEMENTED(transfer_slice > UINT64_MAX / layers); - auto promoted = m_tiler->GetScratchBuffer(transfer_slice * layers); - m_tiler->ConvertD16( + auto promoted = m_tiler.GetScratchBuffer(transfer_slice * layers); + m_tiler.ConvertD16( linear, promoted, TileManager::D16Direction::Promote, info.pixel_format == vk::Format::eD32SfloatS8Uint, {.width = info.extent.width, @@ -1140,7 +1024,7 @@ void TextureCache::UploadImage(Image& image, const ImageDesc& desc, Buffer& sour } void TextureCache::InitializeImage(ImageId id, const ImageDesc& desc) { - auto& image = ResolveImage(id); + auto& image = m_slot_images[id]; if (image.info.data.Empty()) { return; } @@ -1157,13 +1041,13 @@ void TextureCache::InitializeImage(ImageId id, const ImageDesc& desc) { bool data_imported = false; const bool upload = image.IsBufferModified() || image.IsCpuDirty(); if (upload) { - const auto source = + const auto [source, source_offset] = m_buffer_cache.ObtainBufferForImage(image.info.data.address, image.info.data.size); - if (source.buffer == nullptr) { + if (source == nullptr) { EXIT("TextureCache: failed to obtain image upload source\n"); } data_imported = true; - UploadImage(image, desc, *source.buffer, source.offset); + UploadImage(image, desc, *source, source_offset); } if (data_imported) { image.ClearBufferModified(); @@ -1175,7 +1059,7 @@ void TextureCache::InitializeImage(ImageId id, const ImageDesc& desc) { void TextureCache::RefreshImage(ImageId id, const ImageDesc& desc) { TrackImage(id); - auto& image = ResolveImage(id); + auto& image = m_slot_images[id]; if (image.IsMaybeCpuDirty()) { const auto hash = image.HashGuestEdges(); if (image.NeedsMaybeCpuHash()) { @@ -1198,22 +1082,17 @@ void TextureCache::RefreshImage(ImageId id, const ImageDesc& desc) { } void TextureCache::AssociateStencil(ImageId depth_id, GuestRange stencil) { - CacheLock lock(*this, m_lock); - AssociateStencilLocked(depth_id, stencil); -} - -void TextureCache::AssociateStencilLocked(ImageId depth_id, GuestRange stencil) { if (!stencil.Valid()) { EXIT("TextureCache: invalid stencil association range\n"); } - auto& depth = ResolveImage(depth_id); + auto& depth = m_slot_images[depth_id]; if (!depth.info.IsDepth() || !depth.info.HasStencil()) { EXIT("TextureCache: stencil association requires a depth/stencil image\n"); } ImageId association {}; for (const auto id: FindImagesInRegion(stencil.address, stencil.size, false)) { - const auto owner = ResolveOwner(id); + const auto owner = m_slot_images.try_get(id); if (owner != nullptr && owner->info.data.address == stencil.address) { association = id; } @@ -1224,7 +1103,7 @@ void TextureCache::AssociateStencilLocked(ImageId depth_id, GuestRange stencil) info.extent = depth.info.extent; association = InsertImage(info); } - auto& record = ResolveImage(association); + auto& record = m_slot_images[association]; TouchImage(record); record.AssociateDepth(depth_id); } @@ -1236,18 +1115,18 @@ ImageId TextureCache::FindImage(ImageDesc& desc, bool exact_format) { } ValidateImageDesc(desc); if (desc.info.data.Empty()) { - CacheLock lock(*this, m_lock); + std::scoped_lock lock {m_lock}; return GetNullImage(desc); } ImageId result {}; { - CacheLock lock(*this, m_lock); - const auto candidates = + std::scoped_lock lock {m_lock}; + const auto candidates = FindImagesInRegion(desc.info.data.address, desc.info.data.size, false); for (const auto id: candidates) { - const auto& image = ResolveImage(id); + const auto& image = m_slot_images[id]; if (SameBacking(image.info, desc.info, exact_format)) { result = id; } @@ -1259,7 +1138,7 @@ ImageId TextureCache::FindImage(ImageDesc& desc, bool exact_format) { for (const auto candidate: candidates) { view_mip = -1; view_layer = -1; - const auto& merged_info = result ? ResolveImage(result).info : desc.info; + const auto& merged_info = result ? m_slot_images[result].info : desc.info; const auto overlap = ResolveOverlap(merged_info, desc.type, candidate, result); if (overlap.image) { result = overlap.image; @@ -1270,7 +1149,7 @@ ImageId TextureCache::FindImage(ImageDesc& desc, bool exact_format) { } if (result) { - auto& resolved = ResolveImage(result); + auto& resolved = m_slot_images[result]; if (exact_format && resolved.info.pixel_format != desc.info.pixel_format) { result = {}; } else if (resolved.info.resources < desc.info.resources) { @@ -1280,13 +1159,13 @@ ImageId TextureCache::FindImage(ImageDesc& desc, bool exact_format) { } if (!result) { result = InsertImage(desc.info); - auto& inserted = ResolveImage(result); + auto& inserted = m_slot_images[result]; if (m_buffer_cache.HasGpuDirtyBytes(inserted.info.data.address, inserted.info.data.size)) { inserted.MarkBufferModified(); } } - auto& image = ResolveImage(result); + auto& image = m_slot_images[result]; if (desc.type == BindingType::VideoOut && desc.info.metadata.compression != VideoOutCompression::Uncompressed) { const bool guest_dirty = image.IsBufferModified() || image.IsCpuDirty(); @@ -1305,14 +1184,13 @@ ImageId TextureCache::FindImage(ImageDesc& desc, bool exact_format) { } image.tick_accessed_last = m_scheduler.CurrentTick(); TouchImage(image); - RetainImage(command, result); } return result; } void TextureCache::UpdateImage(ImageId id) { - CacheLock lock(*this, m_lock); - auto& image = ResolveImage(id); + std::scoped_lock lock {m_lock}; + auto& image = m_slot_images[id]; TouchImage(image); RefreshImage(id, ImageDesc {.info = image.info, .type = UploadBinding(image)}); } @@ -1321,15 +1199,15 @@ ImageId TextureCache::FindImageFromRange(uint64_t address, uint64_t size, bool e if (!GuestRange {address, size}.Valid()) { return {}; } - CacheLock lock(*this, m_lock); + std::scoped_lock lock {m_lock}; std::vector matches; for (const auto id: FindImagesInRegion(address, size, false)) { - auto owner = ResolveOwner(id); + auto owner = m_slot_images.try_get(id); if (owner == nullptr || owner->info.data.address != address) { continue; } if (ensure_valid && owner->depth_id) { - owner = ResolveOwner(owner->depth_id); + owner = m_slot_images.try_get(owner->depth_id); } if (owner == nullptr || (ensure_valid && !owner->SafeToDownload())) { continue; @@ -1341,28 +1219,25 @@ ImageId TextureCache::FindImageFromRange(uint64_t address, uint64_t size, bool e selected = matches.front(); } else { for (const auto id: matches) { - const auto& image = ResolveImage(id); + const auto& image = m_slot_images[id]; if (image.info.data.size == size) { selected = id; break; } } } - if (selected) { - if (ensure_valid) { - const auto owner = ResolveOwner(selected); - if (owner != nullptr && owner->depth_id) { - selected = owner->depth_id; - } + if (selected && ensure_valid) { + const auto owner = m_slot_images.try_get(selected); + if (owner != nullptr && owner->depth_id) { + selected = owner->depth_id; } - RetainImage(m_scheduler.Current(), selected); } return selected; } vk::ImageView TextureCache::FindTexture(ImageId id, const ImageDesc& desc) { - CacheLock lock(*this, m_lock); - auto& image = ResolveImage(id); + std::scoped_lock lock {m_lock}; + auto& image = m_slot_images[id]; TouchImage(image); if (!image.info.data.Empty()) { if (!image.registered || image.depth_id || image.binding.needs_rebind) { @@ -1384,13 +1259,12 @@ vk::ImageView TextureCache::FindTexture(ImageId id, const ImageDesc& desc) { } CommitGpuWrite(image); } - TrackImageDownloadLocked(id, image); + TrackImageDownload(id, image); break; default: EXIT("TextureCache: invalid texture binding\n"); } const auto view = image.FindView(desc.view_info); NameImageBinding(m_graphics, image, view, desc.type, desc.view_info); - RetainImage(m_scheduler.Current(), id); return view; } @@ -1398,8 +1272,8 @@ vk::ImageView TextureCache::FindRenderTarget(ImageId id, const ImageDesc& desc) if (desc.type != BindingType::RenderTarget) { EXIT("TextureCache: invalid color-target binding\n"); } - CacheLock lock(*this, m_lock); - auto& image = ResolveImage(id); + std::scoped_lock lock {m_lock}; + auto& image = m_slot_images[id]; if (!image.registered || image.depth_id || image.binding.needs_rebind) { EXIT("TextureCache: color target requires rediscovery before final acquisition\n"); } @@ -1421,10 +1295,9 @@ vk::ImageView TextureCache::FindRenderTarget(ImageId id, const ImageDesc& desc) } } CommitGpuWrite(image); - TrackImageDownloadLocked(id, image); + TrackImageDownload(id, image); const auto view = image.FindView(desc.view_info); NameImageBinding(m_graphics, image, view, desc.type, desc.view_info); - RetainImage(m_scheduler.Current(), id); return view; } @@ -1432,8 +1305,8 @@ vk::ImageView TextureCache::FindDepthTarget(ImageId id, const ImageDesc& desc) { if (desc.type != BindingType::DepthTarget) { EXIT("TextureCache: invalid depth-target binding\n"); } - CacheLock lock(*this, m_lock); - auto& image = ResolveImage(id); + std::scoped_lock lock {m_lock}; + auto& image = m_slot_images[id]; if (!image.registered || image.depth_id || image.binding.needs_rebind) { EXIT("TextureCache: depth target requires rediscovery before final acquisition\n"); } @@ -1460,17 +1333,16 @@ vk::ImageView TextureCache::FindDepthTarget(ImageId id, const ImageDesc& desc) { } CommitGpuWrite(image); if (desc.info.HasStencil()) { - AssociateStencilLocked(id, desc.info.stencil); + AssociateStencil(id, desc.info.stencil); } const auto view = image.FindView(desc.view_info); NameImageBinding(m_graphics, image, view, desc.type, desc.view_info); - RetainImage(m_scheduler.Current(), id); return view; } void TextureCache::MarkGpuWritten(ImageId id) { - CacheLock lock(*this, m_lock); - auto& image = ResolveImage(id); + std::scoped_lock lock {m_lock}; + auto& image = m_slot_images[id]; if (!image.registered || image.depth_id) { EXIT("TextureCache: cannot mark an unavailable image GPU-written\n"); } @@ -1494,11 +1366,11 @@ bool TextureCache::ClearImageFromBuffer(CommandBuffer& command, uint64_t address if (command.IsInvalid() || !GuestRange {address, size}.Valid()) { EXIT("TextureCache: invalid image clear\n"); } - CacheLock lock(*this, m_lock); + std::scoped_lock lock {m_lock}; ImageId selected {}; vk::ImageAspectFlags aspect {}; for (const auto id: FindImagesInRegion(address, size, false)) { - auto owner = ResolveOwner(id); + auto owner = m_slot_images.try_get(id); if (owner == nullptr) { continue; } @@ -1508,7 +1380,7 @@ bool TextureCache::ClearImageFromBuffer(CommandBuffer& command, uint64_t address owner->info.data.size == size) { candidate = vk::ImageAspectFlagBits::eStencil; candidate_id = owner->depth_id; - owner = ResolveOwner(candidate_id); + owner = m_slot_images.try_get(candidate_id); if (owner == nullptr || owner->backing.image == nullptr || !owner->info.HasStencil()) { continue; } @@ -1529,7 +1401,7 @@ bool TextureCache::ClearImageFromBuffer(CommandBuffer& command, uint64_t address if (!selected) { return false; } - auto& image = ResolveImage(selected); + auto& image = m_slot_images[selected]; vk::ClearColorValue color_clear {}; float depth_clear = 0.0f; uint8_t stencil_clear = 0; @@ -1566,7 +1438,6 @@ bool TextureCache::ClearImageFromBuffer(CommandBuffer& command, uint64_t address image.backing.image, vk::ImageLayout::eTransferDstOptimal, &clear, 1, &range); } CommitGpuWrite(image); - RetainImage(command, selected); return true; } @@ -1574,7 +1445,7 @@ void TextureCache::InvalidateMemory(uint64_t address, uint64_t size) { if (!GuestRange {address, size}.Valid()) { EXIT("TextureCache: invalid memory-invalidation range\n"); } - CacheLock lock(*this, m_lock); + std::scoped_lock lock {m_lock}; InvalidateCpuAliases(address, size); } @@ -1621,8 +1492,8 @@ void TextureCache::DownloadDepth(Image& image, Buffer& destination, uint64_t des info.pitch}); tiles.back().surface_z = layer; } - m_tiler->TileImage(image, copies, destination.Handle(), destination_offset, info.data.size, - info.data.size, tiles); + m_tiler.TileImage(image, copies, destination.Handle(), destination_offset, info.data.size, + info.data.size, tiles); return; } EXIT_NOT_IMPLEMENTED(info.bytes_per_block != sizeof(uint16_t) || @@ -1630,21 +1501,21 @@ void TextureCache::DownloadDepth(Image& image, Buffer& destination, uint64_t des for (uint32_t layer = 0; layer < layers; layer++) { copies[layer].bufferOffset = transfer_slice * layer; } - auto host_linear = m_tiler->GetScratchBuffer(transfer_size); + auto host_linear = m_tiler.GetScratchBuffer(transfer_size); image.Download(copies, host_linear.buffer, 0, host_linear.size); const bool tiled = info.IsTiled(); - auto guest_linear = tiled ? m_tiler->GetScratchBuffer(info.data.size) + auto guest_linear = tiled ? m_tiler.GetScratchBuffer(info.data.size) : TileManager::Result {destination.Handle(), destination_offset, destination.Size() - destination_offset}; - m_tiler->ConvertD16(host_linear, guest_linear, TileManager::D16Direction::Demote, - DepthAspectTransferFormat(info.pixel_format) == vk::Format::eD32Sfloat, - {.width = info.extent.width, - .height = info.extent.height, - .layers = layers, - .source_row_stride = static_cast(info.pitch) * sizeof(uint32_t), - .target_row_stride = static_cast(info.pitch) * sizeof(uint16_t), - .source_slice_stride = transfer_slice, - .target_slice_stride = full_slice_size}); + m_tiler.ConvertD16(host_linear, guest_linear, TileManager::D16Direction::Demote, + DepthAspectTransferFormat(info.pixel_format) == vk::Format::eD32Sfloat, + {.width = info.extent.width, + .height = info.extent.height, + .layers = layers, + .source_row_stride = static_cast(info.pitch) * sizeof(uint32_t), + .target_row_stride = static_cast(info.pitch) * sizeof(uint16_t), + .source_slice_stride = transfer_slice, + .target_slice_stride = full_slice_size}); if (!tiled) { return; } @@ -1659,8 +1530,8 @@ void TextureCache::DownloadDepth(Image& image, Buffer& destination, uint64_t des full_slice_size, 0, info.extent.width, info.extent.height, 1, info.pitch}); tiles.back().surface_z = layer; } - m_tiler->Tile(guest_linear.buffer, guest_linear.offset, info.data.size, destination.Handle(), - destination_offset, info.data.size, tiles); + m_tiler.Tile(guest_linear.buffer, guest_linear.offset, info.data.size, destination.Handle(), + destination_offset, info.data.size, tiles); } void TextureCache::DownloadImageData(Image& image, Buffer& destination, uint64_t destination_offset, @@ -1681,10 +1552,10 @@ void TextureCache::DownloadImageData(Image& image, Buffer& destination, uint64_t : TileManager::ColorTransform::None; if (!color.tiled) { if (transform == TileManager::ColorTransform::SwapBgra16) { - auto linear = m_tiler->GetScratchBuffer(destination_size); + auto linear = m_tiler.GetScratchBuffer(destination_size); image.Download(color.regions, linear.buffer, 0, linear.size); - m_tiler->SwapBgra16(linear, - {destination.Handle(), destination_offset, destination_size}); + m_tiler.SwapBgra16(linear, + {destination.Handle(), destination_offset, destination_size}); return; } for (auto& copy: color.regions) { @@ -1694,20 +1565,20 @@ void TextureCache::DownloadImageData(Image& image, Buffer& destination, uint64_t return; } - m_tiler->TileImage(image, color.regions, destination.Handle(), destination_offset, - destination_size, color.linear_size, color.tiles, transform); + m_tiler.TileImage(image, color.regions, destination.Handle(), destination_offset, + destination_size, color.linear_size, color.tiles, transform); } bool BufferCache::SynchronizeBufferFromImage(Buffer& buffer, uint64_t vaddr, uint64_t size) { - CacheLock lock(m_texture_cache, m_texture_cache.m_lock); + std::scoped_lock lock {m_texture_cache.m_lock}; std::vector matches; for (const auto id: m_texture_cache.FindImagesInRegion(vaddr, size, false)) { - auto owner = m_texture_cache.ResolveOwner(id); + auto owner = m_texture_cache.m_slot_images.try_get(id); if (owner == nullptr || owner->info.data.address != vaddr) { continue; } if (owner->depth_id) { - owner = m_texture_cache.ResolveOwner(owner->depth_id); + owner = m_texture_cache.m_slot_images.try_get(owner->depth_id); } if (owner != nullptr && owner->SafeToDownload()) { matches.push_back(id); @@ -1719,7 +1590,7 @@ bool BufferCache::SynchronizeBufferFromImage(Buffer& buffer, uint64_t vaddr, uin selected = matches.front(); } else { for (const auto id: matches) { - const auto& image = m_texture_cache.ResolveImage(id); + const auto& image = m_texture_cache.m_slot_images[id]; if (image.info.data.size == size) { selected = id; break; @@ -1729,12 +1600,12 @@ bool BufferCache::SynchronizeBufferFromImage(Buffer& buffer, uint64_t vaddr, uin if (!selected) { return false; } - if (const auto owner = m_texture_cache.ResolveOwner(selected); + if (const auto owner = m_texture_cache.m_slot_images.try_get(selected); owner != nullptr && owner->depth_id) { selected = owner->depth_id; } - auto& image = m_texture_cache.ResolveImage(selected); + auto& image = m_texture_cache.m_slot_images[selected]; if (!buffer.IsInBounds(image.info.data.address, 1)) { return false; } @@ -1787,25 +1658,32 @@ bool BufferCache::SynchronizeBufferFromImage(Buffer& buffer, uint64_t vaddr, uin } } m_texture_cache.DownloadImageData(image, buffer, buf_offset, copy_size, std::move(plan)); - m_texture_cache.RetainImage(m_scheduler.Current(), selected); return true; } -std::pair TextureCache::MapDownload(uint64_t size, uint64_t alignment) { - if (size == 0) { - EXIT("TextureCache: cannot map an empty image download\n"); +bool TextureCache::TryDownloadImage(ImageId id) { + auto& image = m_slot_images[id]; + if (image.depth_id) { + return false; } - auto& download = m_buffer_cache.GetUtilityBuffer(MemoryUsage::Download); - auto mapping = download.Map(size, std::max(alignment, 4)); - if (mapping.first == nullptr) { + auto plan = BuildDownload(image); + if (!plan.valid || !SafeToDownload(image)) { + return false; + } + const auto range = image.info.data; + auto& download = m_buffer_cache.GetUtilityBuffer(MemoryUsage::Download); + auto [mapped, offset] = + download.Map(range.size, std::max(image.info.bytes_per_block, 4)); + if (mapped == nullptr) { EXIT("TextureCache: failed to map reusable download buffer\n"); } download.Commit(); - return mapping; -} + if (!LibKernel::Memory::TryReadBacking(range.address, mapped, range.size)) { + return false; + } + download.Flush(offset, range.size); -void TextureCache::QueueDownload(GuestRange range, StreamBuffer& download, uint8_t* mapped, - uint64_t offset) { + DownloadImageData(image, download, offset, range.size, std::move(plan)); vk::BufferMemoryBarrier barrier {}; barrier.sType = vk::StructureType::eBufferMemoryBarrier; barrier.srcAccessMask = vk::AccessFlagBits::eMemoryWrite | vk::AccessFlagBits::eTransferWrite | @@ -1824,61 +1702,24 @@ void TextureCache::QueueDownload(GuestRange range, StreamBuffer& download, uint8 download.Invalidate(offset, range.size); LibKernel::Memory::WriteBacking(range.address, mapped, range.size); }); -} - -bool TextureCache::TryDownloadImage(ImageId id) { - auto& image = ResolveImage(id); - if (image.depth_id) { - return false; - } - auto plan = BuildDownload(image); - if (!plan.valid || !SafeToDownload(image)) { - return false; - } - const auto range = image.info.data; - auto [mapped, offset] = MapDownload(range.size, image.info.bytes_per_block); - auto& download = m_buffer_cache.GetUtilityBuffer(MemoryUsage::Download); - if (!LibKernel::Memory::TryReadBacking(range.address, mapped, range.size)) { - return false; - } - download.Flush(offset, range.size); - - DownloadImageData(image, download, offset, range.size, std::move(plan)); - - QueueDownload(range, download, mapped, offset); return true; } -void TextureCache::DownloadImage(ImageId id) { - if (!TryDownloadImage(id)) { - EXIT("TextureCache: unsupported image readback\n"); - } - m_scheduler.FinishCurrent(); - m_scheduler.DrainPriorityOperations(); -} - -bool TextureCache::InvalidateMemoryFromGPU(uint64_t address, uint64_t size, - bool formatted_buffer_write) { +void TextureCache::InvalidateMemoryFromGPU(uint64_t address, uint64_t size) { if (!GuestRange {address, size}.Valid()) { - return false; + return; } - CacheLock lock(*this, m_lock); - bool found = false; + std::scoped_lock lock {m_lock}; for (const auto id: FindImagesInRegion(address, size, true)) { - auto& image = ResolveImage(id); + auto& image = m_slot_images[id]; if (image.depth_id || !image.Overlaps(address, size)) { continue; } if (image.IsGpuModified()) { - if (!formatted_buffer_write) { - EXIT("TextureCache: buffer write aliases GPU-modified image\n"); - } image.ClearGpuModified(); } image.MarkBufferModified(); - found = true; } - return found; } TextureCache::RegionInfo TextureCache::QueryRegion(uint64_t address, uint64_t size) { @@ -1886,9 +1727,9 @@ TextureCache::RegionInfo TextureCache::QueryRegion(uint64_t address, uint64_t si if (!GuestRange {address, size}.Valid()) { return result; } - CacheLock lock(*this, m_lock); + std::scoped_lock lock {m_lock}; for (const auto id: FindImagesInRegion(address, size, true)) { - const auto& image = ResolveImage(id); + const auto& image = m_slot_images[id]; if (image.depth_id) { continue; } @@ -1903,7 +1744,7 @@ void TextureCache::InvalidateCpuAliases(uint64_t address, uint64_t size) { const auto page_begin = address & ~(TRACKER_PAGE_SIZE - 1); const auto page_end = (address + size + TRACKER_PAGE_SIZE - 1) & ~(TRACKER_PAGE_SIZE - 1); for (const auto id: FindImagesInRegion(address, size, true)) { - auto owner = ResolveOwner(id); + auto owner = m_slot_images.try_get(id); if (owner == nullptr || owner->depth_id) { continue; } @@ -1924,23 +1765,15 @@ void TextureCache::InvalidateCpuAliases(uint64_t address, uint64_t size) { } } -void TextureCache::ClearGpuModified(ImageId id) { - auto owner = ResolveOwner(id); - if (owner == nullptr || !owner->IsGpuModified()) { - return; - } - owner->ClearGpuModified(); -} - bool TextureCache::IsMeta(uint64_t address) { - CacheLock lock(*this, m_lock); - const auto found = m_surface_metas.find(address); + std::scoped_lock lock {m_lock}; + const auto found = m_surface_metas.find(address); return found != m_surface_metas.end() && found->second.type != MetaDataInfo::Type::PendingDcc; } bool TextureCache::IsMetaCleared(uint64_t address, uint32_t slice, uint32_t* fill_value) { - CacheLock lock(*this, m_lock); - const auto found = m_surface_metas.find(address); + std::scoped_lock lock {m_lock}; + const auto found = m_surface_metas.find(address); if (found == m_surface_metas.end() || found->second.type == MetaDataInfo::Type::PendingDcc || slice >= 32) { return false; @@ -1952,8 +1785,8 @@ bool TextureCache::IsMetaCleared(uint64_t address, uint32_t slice, uint32_t* fil } bool TextureCache::ClearMeta(uint64_t address) { - CacheLock lock(*this, m_lock); - const auto found = m_surface_metas.find(address); + std::scoped_lock lock {m_lock}; + const auto found = m_surface_metas.find(address); if (found == m_surface_metas.end() || found->second.type == MetaDataInfo::Type::PendingDcc || found->second.type == MetaDataInfo::Type::Dcc) { // Preserve the broad metadata-clear operation for CMask/FMask/HTile. DCC requires a @@ -1984,8 +1817,8 @@ bool TextureCache::TryConsumeDccFill(uint64_t address, uint64_t size, uint32_t f default: return 0u; } }(); - CacheLock lock(*this, m_lock); - auto found = m_surface_metas.find(address); + std::scoped_lock lock {m_lock}; + auto found = m_surface_metas.find(address); if (found == m_surface_metas.end()) { // This dispatch may precede color-target discovery. Retain it in the shared // metadata map, but PendingDcc remains invisible to IsMeta until registration. Returning @@ -2013,8 +1846,8 @@ bool TextureCache::TryConsumeDccFill(uint64_t address, uint64_t size, uint32_t f } bool TextureCache::TouchMeta(uint64_t address, uint32_t slice, bool is_clear) { - CacheLock lock(*this, m_lock); - const auto found = m_surface_metas.find(address); + std::scoped_lock lock {m_lock}; + const auto found = m_surface_metas.find(address); if (found == m_surface_metas.end() || found->second.type == MetaDataInfo::Type::PendingDcc || slice >= 32) { return false; @@ -2031,8 +1864,8 @@ void TextureCache::UnmapMemory(uint64_t address, uint64_t size) { if (!GuestRange {address, size}.Valid()) { EXIT("TextureCache: invalid unmap range\n"); } - CacheLock lock(*this, m_lock); - const auto end = address + size; + std::scoped_lock lock {m_lock}; + const auto end = address + size; for (auto metadata = m_surface_metas.lower_bound(address); metadata != m_surface_metas.end() && metadata->first < end;) { metadata = m_surface_metas.erase(metadata); @@ -2042,20 +1875,20 @@ void TextureCache::UnmapMemory(uint64_t address, uint64_t size) { m_scheduler.Finish(); } for (const auto id: images) { - auto owner = ResolveOwner(id); + auto owner = m_slot_images.try_get(id); if (owner == nullptr) { continue; } if (owner->IsGpuModified()) { - ClearGpuModified(id); + owner->ClearGpuModified(); } DeleteImage(id); } } void TextureCache::RunGarbageCollector() { - CacheLock lock(*this, m_lock); - const uint64_t tick = m_gc_tick++; + std::scoped_lock lock {m_lock}; + const uint64_t tick = m_gc_tick++; if (m_graphics.CanReportMemoryUsage()) { m_total_used_memory = m_graphics.GetDeviceMemoryUsage(); } @@ -2080,7 +1913,7 @@ void TextureCache::RunGarbageCollector() { break; } --deletions; - auto owner = ResolveOwner(id); + auto owner = m_slot_images.try_get(id); if (owner == nullptr || !owner->registered || owner->depth_id) { continue; } @@ -2095,7 +1928,7 @@ void TextureCache::RunGarbageCollector() { if (safe && !TryDownloadImage(id)) { continue; } - ClearGpuModified(id); + owner->ClearGpuModified(); } DeleteImage(id); if (m_total_used_memory < m_critical_gc_memory && aggressive) { @@ -2115,9 +1948,9 @@ void TextureCache::RunGarbageCollector() { } void TextureCache::ProcessDownloadImages() { - CacheLock lock(*this, m_lock); + std::scoped_lock lock {m_lock}; for (const auto id: m_download_images) { - const auto owner = ResolveOwner(id); + const auto owner = m_slot_images.try_get(id); if (owner != nullptr && owner->registered && owner->IsGpuModified()) { (void)TryDownloadImage(id); } diff --git a/src/graphics/host_gpu/renderer/cache/textureCache.h b/src/graphics/host_gpu/renderer/cache/textureCache.h index 945226afc..ab0abde77 100644 --- a/src/graphics/host_gpu/renderer/cache/textureCache.h +++ b/src/graphics/host_gpu/renderer/cache/textureCache.h @@ -4,17 +4,17 @@ #include "common/abi.h" #include "common/common.h" #include "common/lruCache.h" +#include "common/slotVector.h" #include "graphics/host_gpu/pageManager.h" #include "graphics/host_gpu/regionManager.h" #include "graphics/host_gpu/renderer/cache/multiLevelPageTable.h" #include "graphics/host_gpu/renderer/image/blitHelper.h" #include "graphics/host_gpu/renderer/image/image.h" +#include "graphics/host_gpu/renderer/image/tiler.h" -#include #include -#include -#include -#include +#include +#include #include namespace Libs::Graphics { @@ -25,8 +25,6 @@ class BufferCache; class CommandBuffer; class CommandScheduler; class RenderExecutor; -class StreamBuffer; -class TileManager; struct TextureCacheTestAccess; class TextureCache { @@ -57,15 +55,17 @@ public: [[nodiscard]] vk::ImageView FindTexture(ImageId id, const ImageDesc& desc); [[nodiscard]] vk::ImageView FindRenderTarget(ImageId id, const ImageDesc& desc); [[nodiscard]] vk::ImageView FindDepthTarget(ImageId id, const ImageDesc& desc); - [[nodiscard]] Image& GetImage(ImageId id); - [[nodiscard]] const Image& GetImage(ImageId id) const; - void MarkGpuWritten(ImageId id); + [[nodiscard]] Image& GetImage(ImageId id) { + auto& image = m_slot_images[id]; + TouchImage(image); + return image; + } + void MarkGpuWritten(ImageId id); [[nodiscard]] bool ClearImageFromBuffer(CommandBuffer& command, uint64_t address, uint64_t size, uint32_t packed_clear); void InvalidateMemory(uint64_t address, uint64_t size); - [[nodiscard]] bool InvalidateMemoryFromGPU(uint64_t address, uint64_t size, - bool formatted_buffer_write = false); + void InvalidateMemoryFromGPU(uint64_t address, uint64_t size); [[nodiscard]] RegionInfo QueryRegion(uint64_t address, uint64_t size); [[nodiscard]] bool IsMeta(uint64_t address); @@ -86,16 +86,11 @@ private: struct ColorTransferPlan; struct DownloadPlan; - struct Slot { - std::shared_ptr image; - uint32_t generation = 1; - }; - struct MetaDataInfo { // A guest metadata-fill dispatch may initialize DCC before its render target is bound. // PendingDcc retains that exact fill until FindRenderTarget classifies the address, // without exposing an unconfirmed buffer address to the normal metadata heuristics. - // Keep all surface metadata in this one shadPS4-style entry so CMask/FMask can be + // Keep all surface metadata in one entry so CMask/FMask can be // registered beside HTile and DCC without introducing parallel tracking paths. enum class Type : uint8_t { PendingDcc, CMask, FMask, HTile, Dcc }; @@ -114,26 +109,21 @@ private: using ImageIds = InlinePageOwnerList; using ImagePageTable = MultiLevelPageTable; - [[nodiscard]] Image& ResolveImage(ImageId id); - [[nodiscard]] const Image& ResolveImage(ImageId id) const; - [[nodiscard]] std::shared_ptr ResolveOwner(ImageId id) const; - [[nodiscard]] ImageId InsertImage(const ImageInfo& info); - [[nodiscard]] ImageId GetNullImage(const ImageDesc& desc); - void RegisterImage(ImageId id); - void UnregisterImage(ImageId id); - void DeleteImage(ImageId id); - void FreeImage(ImageId id); - void RetainImage(CommandBuffer& command, ImageId id); - void TouchImage(Image& image); - void TrackImage(ImageId id); - void TrackImageHead(ImageId id); - void TrackImageTail(ImageId id); - void UntrackImage(ImageId id); - void UntrackImageHead(ImageId id); - void UntrackImageTail(ImageId id); - void MarkAsMaybeDirty(ImageId id, Image& image); - void TrackImageDownload(ImageId id); - void TrackImageDownloadLocked(ImageId id, Image& image); + [[nodiscard]] ImageId InsertImage(const ImageInfo& info); + [[nodiscard]] ImageId GetNullImage(const ImageDesc& desc); + void RegisterImage(ImageId id); + void UnregisterImage(ImageId id); + void DeleteImage(ImageId id); + void FreeImage(ImageId id); + void TouchImage(Image& image); + void TrackImage(ImageId id); + void TrackImageHead(ImageId id); + void TrackImageTail(ImageId id); + void UntrackImage(ImageId id); + void UntrackImageHead(ImageId id); + void UntrackImageTail(ImageId id); + void MarkAsMaybeDirty(ImageId id, Image& image); + void TrackImageDownload(ImageId id, Image& image); [[nodiscard]] static bool SameBacking(const ImageInfo& cached, const ImageInfo& requested, bool exact_format); [[nodiscard]] static BindingType UploadBinding(const Image& image); @@ -162,31 +152,24 @@ private: [[nodiscard]] bool CopyD16(Image& destination, Image& source); void CopyImage(ImageId destination, ImageId source); void AssociateStencil(ImageId depth, GuestRange stencil); - void AssociateStencilLocked(ImageId depth, GuestRange stencil); void CopyImageMip(ImageId destination, ImageId source, uint32_t mip, uint32_t layer); void ValidateImageDesc(const ImageDesc& desc) const; - void InvalidateCpuAliases(uint64_t address, uint64_t size); - void ClearGpuModified(ImageId id); - - void DownloadImage(ImageId id); - [[nodiscard]] bool TryDownloadImage(ImageId id); - [[nodiscard]] std::pair MapDownload(uint64_t size, uint64_t alignment); - void QueueDownload(GuestRange range, StreamBuffer& download, uint8_t* mapped, uint64_t offset); + void InvalidateCpuAliases(uint64_t address, uint64_t size); + [[nodiscard]] bool TryDownloadImage(ImageId id); GraphicContext& m_graphics; CommandScheduler& m_scheduler; TrackingSpinLock m_lock; PageManager& m_page_manager; BlitHelper m_blit_helper; - std::unique_ptr m_tiler; + TileManager m_tiler; BufferCache& m_buffer_cache; - std::vector m_slots; - std::vector m_free_slots; + Common::SlotVector m_slot_images; ImagePageTable m_image_page_table; - std::map m_null_images; + std::unordered_map m_null_images; Common::LeastRecentlyUsedCache m_lru_cache; - std::set m_download_images; + std::unordered_set m_download_images; std::map m_surface_metas; uint64_t m_total_used_memory = 0; uint64_t m_trigger_gc_memory = 0; diff --git a/src/graphics/host_gpu/renderer/commandScheduler.cpp b/src/graphics/host_gpu/renderer/commandScheduler.cpp index de5f32989..f56cd84db 100644 --- a/src/graphics/host_gpu/renderer/commandScheduler.cpp +++ b/src/graphics/host_gpu/renderer/commandScheduler.cpp @@ -226,12 +226,17 @@ void CommandScheduler::Wait(uint64_t tick) { EXIT_IF(tick > CurrentTick()); if (tick >= CurrentTick()) { // A stream-buffer wrap can wait while a draw is being prepared through a reference to - // Current(). Recycle the same command object so that reference remains valid. - FinishCurrent(); + // Current(). Recycle the same command object so that reference remains valid. Deferred + // resources are released only at the next GPU operation boundary. + SubmitInfo submit; + auto& submitted = SubmitCurrent(submit); + submitted.WaitForFenceAndReset(); + m_master.Refresh(); + submitted.Begin(); + m_recording = true; return; } m_master.Wait(tick); - PopPendingOperations(); } void CommandScheduler::PopPendingOperations() { @@ -243,17 +248,18 @@ void CommandScheduler::PopPendingOperations(bool refresh_gpu_tick) { m_master.Refresh(); } for (;;) { - Common::UniqueFunction callback; + PendingOperation operation; { std::lock_guard lock(m_operation_mutex); if (m_pending_operations.empty() || !m_master.IsFree(m_pending_operations.front().tick)) { return; } - callback = std::move(m_pending_operations.front().callback); + operation = std::move(m_pending_operations.front()); m_pending_operations.pop(); } - RunOperation(std::move(callback)); + WaitPriorityOperations(operation.tick); + RunOperation(std::move(operation.callback)); } } @@ -423,10 +429,8 @@ void CommandScheduler::BeginNext() { EXIT_IF(m_recording); auto candidate = FindReusableBuffer(m_master.KnownGpuTick()); - bool refreshed = false; if (candidate < 0) { m_master.Refresh(); - refreshed = true; candidate = FindReusableBuffer(m_master.KnownGpuTick()); } if (candidate < 0) { @@ -435,7 +439,6 @@ void CommandScheduler::BeginNext() { m_current = candidate; Current().WaitForFenceAndReset(); - PopPendingOperations(!refreshed); BindCurrent(); Current().Begin(); m_recording = true; diff --git a/src/graphics/host_gpu/renderer/context.cpp b/src/graphics/host_gpu/renderer/context.cpp index e72278cee..0edceccc7 100644 --- a/src/graphics/host_gpu/renderer/context.cpp +++ b/src/graphics/host_gpu/renderer/context.cpp @@ -19,7 +19,6 @@ #include #include #include -#include namespace Libs::Graphics { namespace { @@ -40,27 +39,6 @@ void ReportVulkanFatal(const char* what, vk::Result result, uint32_t slot, uint6 } // namespace -FenceResourceRetainer::~FenceResourceRetainer() { - if (!m_resources.empty()) { - EXIT("fence resource retainer destroyed before release\n"); - } -} - -void FenceResourceRetainer::Retain(std::shared_ptr resource) { - if (resource == nullptr) { - EXIT("cannot retain a null fence resource\n"); - } - if (std::ranges::none_of(m_resources, [&resource](const auto& retained) { - return retained.get() == resource.get(); - })) { - m_resources.push_back(std::move(resource)); - } -} - -void FenceResourceRetainer::ReleaseAfterFence() noexcept { - m_resources.clear(); -} - CommandBuffer::CommandBuffer(CommandScheduler& scheduler) : m_context(scheduler.Context()), m_scheduler(scheduler), m_graphics(scheduler.Graphics()), m_slot(scheduler.AllocateCommandBuffer()) {} @@ -90,26 +68,12 @@ void CommandBuffer::Release() { m_slot->busy = false; m_slot->Reset(); - ReleaseResourcesAfterFence(); + RecycleDescriptorsAfterFence(); m_slot = nullptr; EXIT_NOT_IMPLEMENTED(!IsInvalid()); } -void CommandBuffer::RetireBufferAfterFence(std::unique_ptr buffer) { - if (IsInvalid() || m_execute || buffer == nullptr || buffer->buffer == nullptr) { - EXIT("cannot retire a buffer on an invalid or submitted command buffer\n"); - } - m_retired_buffers.push_back(std::move(buffer)); -} - -void CommandBuffer::RetainResourceUntilFence(std::shared_ptr resource) { - if (IsInvalid() || m_execute) { - EXIT("cannot retain a resource on an invalid or submitted command buffer\n"); - } - m_fence_resources.Retain(std::move(resource)); -} - void CommandBuffer::RecycleDescriptorAfterFence(VulkanDescriptorSet& set) { m_descriptor_sets_after_fence.push_back(&set); } @@ -255,21 +219,8 @@ void CommandBuffer::FinalizeFence(bool reset_recording) { } } if (was_executed) { - ReleaseResourcesAfterFence(); + RecycleDescriptorsAfterFence(); } - DeleteBuffersAfterFence(); -} - -void CommandBuffer::ReleaseResourcesAfterFence() { - RecycleDescriptorsAfterFence(); - m_fence_resources.ReleaseAfterFence(); -} - -void CommandBuffer::DeleteBuffersAfterFence() { - for (const auto& buffer: m_retired_buffers) { - m_graphics.DeleteBuffer(*buffer); - } - m_retired_buffers.clear(); } void CommandBuffer::BeginRendering(const RenderState& state) const { diff --git a/src/graphics/host_gpu/renderer/image/image.h b/src/graphics/host_gpu/renderer/image/image.h index 2232036d8..a8bcec8ef 100644 --- a/src/graphics/host_gpu/renderer/image/image.h +++ b/src/graphics/host_gpu/renderer/image/image.h @@ -2,6 +2,7 @@ #define EMULATOR_SRC_GRAPHICS_HOST_GPU_RENDERER_IMAGE_H_ #include "common/assert.h" +#include "common/slotVector.h" #include "graphics/host_gpu/graphicContext.h" #include "graphics/host_gpu/renderer/image/imageInfo.h" @@ -19,15 +20,7 @@ class Buffer; class CommandScheduler; struct ImageTestAccess; -struct ImageId { - uint32_t index = std::numeric_limits::max(); - uint32_t generation = 0; - - [[nodiscard]] explicit operator bool() const noexcept { - return index != std::numeric_limits::max(); - } - auto operator<=>(const ImageId&) const = default; -}; +using ImageId = Common::SlotId; struct CachedImageView { ImageViewInfo info; diff --git a/src/graphics/host_gpu/renderer/pipeline/descriptorCache.h b/src/graphics/host_gpu/renderer/pipeline/descriptorCache.h index 3f3d704e6..ff35bd1ec 100644 --- a/src/graphics/host_gpu/renderer/pipeline/descriptorCache.h +++ b/src/graphics/host_gpu/renderer/pipeline/descriptorCache.h @@ -6,6 +6,7 @@ #include "common/common.h" #include "common/threads.h" #include "graphics/host_gpu/graphicContext.h" +#include "graphics/host_gpu/renderer/cache/bufferCache.h" #include "graphics/host_gpu/renderer/cache/textureCache.h" #include "graphics/host_gpu/vulkanCommon.h" #include "graphics/shader/shaderBindings.h" @@ -36,10 +37,9 @@ struct VulkanDescriptorSet { }; struct BufferView { - std::shared_ptr owner; - vk::Buffer buffer = nullptr; - vk::DeviceSize offset = 0; - vk::DeviceSize range = VK_WHOLE_SIZE; + vk::Buffer buffer = nullptr; + vk::DeviceSize offset = 0; + vk::DeviceSize range = VK_WHOLE_SIZE; }; class DescriptorCache { @@ -68,6 +68,8 @@ public: std::shared_ptr program; std::shared_ptr snapshot; NativeDescriptors resources; + std::vector buffer_ids; + std::vector address_ids; std::vector flattened_srt; std::vector user_data; vk::ShaderStageFlags shader_stage; diff --git a/src/graphics/host_gpu/renderer/pipeline/descriptors.cpp b/src/graphics/host_gpu/renderer/pipeline/descriptors.cpp index d2c8520ee..11ea9736d 100644 --- a/src/graphics/host_gpu/renderer/pipeline/descriptors.cpp +++ b/src/graphics/host_gpu/renderer/pipeline/descriptors.cpp @@ -56,9 +56,7 @@ static const char* ShaderStageResourceName(DescriptorCache::Stage stage) { } static void BindNullStorageBuffer(RenderContext& context, BufferView& dst) { - auto owner = context.GetBufferCache().ObtainNullBuffer(); - dst.buffer = owner->Handle(); - dst.owner = std::move(owner); + dst.buffer = context.GetBufferCache().GetBuffer(NULL_BUFFER_ID).Handle(); dst.offset = 0; dst.range = 16; } @@ -89,11 +87,11 @@ static bool IsMultisampledTexture(Prospero::ImageType type) { type == Prospero::ImageType::kColor2DMsaaArray; } -static BufferView NativeStorageBuffer(RenderContext& context, CommandBuffer& command_buffer, - const ShaderBufferResource& descriptor, - const ShaderRecompiler::IR::BufferResource& resource, - DescriptorCache::Stage stage, uint32_t slot, - uint32_t& buffer_offset) { +static BufferView +NativeStorageBuffer(RenderContext& context, const ShaderBufferResource& descriptor, + const ShaderRecompiler::IR::BufferResource& resource, + DescriptorCache::Stage stage, uint32_t slot, uint32_t& buffer_offset, + BufferId id) { BufferView result; buffer_offset = 0; @@ -115,19 +113,21 @@ static BufferView NativeStorageBuffer(RenderContext& context, CommandBuffer& com size > graphics.GetPhysicalDeviceProperties().limits.maxStorageBufferRange) { EXIT("storage buffer range or device alignment is unsupported\n"); } - auto binding = context.GetBufferCache().ObtainBuffer( - command_buffer, address, size, resource.written, resource.read, resource.formatted); - const auto aligned_offset = binding.offset - binding.offset % alignment; - const auto adjustment = binding.offset - aligned_offset; + auto [buffer, offset] = context.GetBufferCache().ObtainBuffer( + address, size, resource.written, resource.formatted, id); + const auto aligned_offset = offset - offset % alignment; + const auto adjustment = offset - aligned_offset; const auto max_range = graphics.GetPhysicalDeviceProperties().limits.maxStorageBufferRange; if (adjustment % sizeof(uint32_t) != 0 || adjustment >= 256 || size > max_range - adjustment) { EXIT("storage buffer offset adjustment is unsupported\n"); } buffer_offset = static_cast(adjustment); - result.owner = std::move(binding.owner); - result.buffer = binding.buffer; + result.buffer = buffer->Handle(); result.offset = aligned_offset; result.range = static_cast(size + adjustment); + if (resource.formatted && resource.written) { + context.GetTextureCache().InvalidateMemoryFromGPU(address, size); + } SetVulkanObjectNameF( graphics.device, result.buffer, "Kyty.{}.StorageBuffer[slot={} guest=0x{:016x} size=0x{:x} access={} formatted={}]", @@ -136,11 +136,10 @@ static BufferView NativeStorageBuffer(RenderContext& context, CommandBuffer& com return result; } -static BufferView -NativeAddressBuffer(RenderContext& context, CommandBuffer& command_buffer, - const ShaderRecompiler::IR::AddressResource& resource, - const ShaderRecompiler::IR::ResourceSnapshot::Address& address, - DescriptorCache::Stage stage, uint32_t slot, uint32_t& address_offset) { +static BufferView NativeAddressBuffer( + RenderContext& context, const ShaderRecompiler::IR::AddressResource& resource, + const ShaderRecompiler::IR::ResourceSnapshot::Address& address, + DescriptorCache::Stage stage, uint32_t slot, uint32_t& address_offset, BufferId id) { BufferView result; address_offset = 0; if (address.binding_base == 0) { @@ -163,15 +162,14 @@ NativeAddressBuffer(RenderContext& context, CommandBuffer& command_buffer, } const auto& graphics = context.GetGraphics(); const auto alignment = graphics.StorageMinAlignment(); - auto binding = - context.GetBufferCache().ObtainBuffer(command_buffer, address.binding_base, size); - const auto aligned_offset = binding.offset - binding.offset % alignment; - const auto adjustment = binding.offset - aligned_offset; + auto [buffer, offset] = + context.GetBufferCache().ObtainBuffer(address.binding_base, size, false, false, id); + const auto aligned_offset = offset - offset % alignment; + const auto adjustment = offset - aligned_offset; const auto max_range = graphics.GetPhysicalDeviceProperties().limits.maxStorageBufferRange; const auto visible_size = std::min(size, static_cast(max_range - adjustment)); address_offset = static_cast(adjustment); - result.owner = std::move(binding.owner); - result.buffer = binding.buffer; + result.buffer = buffer->Handle(); result.offset = aligned_offset; result.range = static_cast(visible_size + adjustment); SetVulkanObjectNameF(graphics.device, result.buffer, @@ -811,22 +809,19 @@ static vk::Sampler NativeSampler(RenderContext& context, return context.GetSamplerCache().GetSampler(descriptor); } -static BufferView NativeUpload(RenderContext& context, CommandBuffer& command_buffer, - std::span data) { +static BufferView NativeUpload(RenderContext& context, std::span data) { EXIT_IF(data.empty()); + auto& command_buffer = context.GetCommandScheduler().Current(); EXIT_IF(command_buffer.IsInvalid() || command_buffer.IsExecute()); - auto binding = context.GetBufferCache().UploadTransient(data.data(), data.size_bytes(), 256); - return {.owner = std::move(binding.owner), - .buffer = binding.buffer, - .offset = binding.offset, - .range = data.size_bytes()}; + auto& buffer = context.GetBufferCache().GetUtilityBuffer(MemoryUsage::Stream); + const auto offset = buffer.Copy(data.data(), data.size_bytes(), 256); + return {.buffer = buffer.Handle(), .offset = offset, .range = data.size_bytes()}; } void RenderExecutor::TrackImageBinding(ImageId id) { - auto image = m_context.GetTextureCache().ResolveOwner(id); - EXIT_IF(image == nullptr); - if (std::ranges::find(m_bound_images, image) == m_bound_images.end()) { - m_bound_images.push_back(std::move(image)); + EXIT_IF(m_context.GetTextureCache().m_slot_images.try_get(id) == nullptr); + if (std::ranges::find(m_bound_images, id) == m_bound_images.end()) { + m_bound_images.push_back(id); } } @@ -850,16 +845,17 @@ void RenderExecutor::BindRenderTarget(ImageId id) { } void RenderExecutor::ResetBindings() { - for (const auto& image: m_bound_images) { - image->binding = {}; + for (const auto id: m_bound_images) { + if (auto* image = m_context.GetTextureCache().m_slot_images.try_get(id); image != nullptr) { + image->binding = {}; + } } m_bound_images.clear(); } -DescriptorCache::PreparedBindings RenderExecutor::PrepareBindings(CommandBuffer& buffer, - const ShaderStageRuntime& runtime, - vk::ShaderStageFlags shader_stage, - DescriptorCache::Stage stage) { +DescriptorCache::PreparedBindings +RenderExecutor::PrepareBindings(const ShaderStageRuntime& runtime, + vk::ShaderStageFlags shader_stage, DescriptorCache::Stage stage) { KYTY_PROFILER_FUNCTION(); EXIT_IF(!runtime); const auto& program = *runtime.program; @@ -899,19 +895,68 @@ DescriptorCache::PreparedBindings RenderExecutor::PrepareBindings(CommandBuffer& prepared.user_data.resize(program.bindings.ShaderDataDwords()); if (ShaderRecompiler::IR::FindBinding( program.bindings, ShaderRecompiler::IR::DescriptorBindingKind::Gds) != nullptr) { - descriptors.gds.buffer = m_context.GetBufferCache().GetGdsBuffer().Handle(); + descriptors.gds.buffer = m_context.GetBufferCache().GetGdsBuffer()->Handle(); } return prepared; } -void RenderExecutor::RebindBuffers(CommandBuffer& buffer, - DescriptorCache::PreparedBindings& prepared) { +void RenderExecutor::FindBuffers(DescriptorCache::PreparedBindings& prepared) { + KYTY_PROFILER_FUNCTION(); + EXIT_IF(prepared.program == nullptr || prepared.snapshot == nullptr); + const auto& program = *prepared.program; + const auto& snapshot = *prepared.snapshot; + auto& cache = m_context.GetBufferCache(); + + prepared.buffer_ids.clear(); + prepared.buffer_ids.reserve(program.info.buffers.size()); + for (uint32_t i = 0; i < program.info.buffers.size(); i++) { + ShaderBufferResource descriptor; + CopyNativeDescriptor(snapshot.buffers[i], descriptor.fields); + const auto address = descriptor.Base48(); + const auto stride = descriptor.Stride(); + const auto records = descriptor.NumRecords(); + EXIT_IF(stride != 0 && records > UINT64_MAX / stride); + const auto requested_size = + stride != 0 ? static_cast(stride) * records : records; + if (address == 0 || requested_size == 0) { + prepared.buffer_ids.emplace_back(); + continue; + } + const auto size = Libs::LibKernel::Memory::ClampRangeSize(address, requested_size); + prepared.buffer_ids.push_back(cache.FindBuffer(address, size)); + } + + prepared.address_ids.clear(); + prepared.address_ids.reserve(program.info.addresses.size()); + for (uint32_t i = 0; i < program.info.addresses.size(); i++) { + const auto& resource = program.info.addresses[i]; + const auto& address = snapshot.addresses[i]; + if (address.binding_base == 0) { + prepared.address_ids.emplace_back(); + continue; + } + EXIT_IF(resource.written); + const auto limit = + resource.kind == ShaderRecompiler::IR::ResourceKind::Flat + ? ShaderRecompiler::IR::FlatAddressWindowSize + : static_cast(m_context.GetGraphics() + .GetPhysicalDeviceProperties() + .limits.maxStorageBufferRange); + uint64_t size = 0; + EXIT_IF(!HostMemoryQueryRange(address.binding_base, limit, HostMemoryAccess::Mapped, size)); + prepared.address_ids.push_back(cache.FindBuffer(address.binding_base, size)); + } +} + +void RenderExecutor::RebindBuffers(DescriptorCache::PreparedBindings& prepared) { KYTY_PROFILER_FUNCTION(); EXIT_IF(prepared.program == nullptr || prepared.snapshot == nullptr); const auto& program = *prepared.program; const auto& snapshot = *prepared.snapshot; auto& resources = prepared.resources; const auto& layout = program.bindings; + EXIT_IF(prepared.buffer_ids.size() != program.info.buffers.size() || + prepared.address_ids.size() != program.info.addresses.size()); resources.buffers.clear(); resources.buffers.reserve(program.info.buffers.size()); @@ -926,9 +971,9 @@ void RenderExecutor::RebindBuffers(CommandBuffer& buffer, ShaderBufferResource descriptor; CopyNativeDescriptor(snapshot.buffers[i], descriptor.fields); uint32_t buffer_offset = 0; - resources.buffers.push_back(NativeStorageBuffer(m_context, buffer, descriptor, + resources.buffers.push_back(NativeStorageBuffer(m_context, descriptor, program.info.buffers[i], prepared.stage, i, - buffer_offset)); + buffer_offset, prepared.buffer_ids[i])); pack_memory_offset(i, buffer_offset); } resources.addresses.clear(); @@ -936,14 +981,20 @@ void RenderExecutor::RebindBuffers(CommandBuffer& buffer, for (uint32_t i = 0; i < program.info.addresses.size(); i++) { uint32_t address_offset = 0; resources.addresses.push_back( - NativeAddressBuffer(m_context, buffer, program.info.addresses[i], snapshot.addresses[i], - prepared.stage, i, address_offset)); + NativeAddressBuffer(m_context, program.info.addresses[i], snapshot.addresses[i], + prepared.stage, i, address_offset, prepared.address_ids[i])); pack_memory_offset(static_cast(program.info.buffers.size()) + i, address_offset); } + if (!prepared.flattened_srt.empty()) { + resources.flattened_srt = NativeUpload(m_context, prepared.flattened_srt); + } + if (ShaderRecompiler::IR::FindBinding( + program.bindings, ShaderRecompiler::IR::DescriptorBindingKind::UserData) != nullptr) { + resources.user_data = NativeUpload(m_context, prepared.user_data); + } } -void RenderExecutor::RebindImages(CommandBuffer& buffer, - DescriptorCache::PreparedBindings& prepared) { +void RenderExecutor::RebindImages(DescriptorCache::PreparedBindings& prepared) { KYTY_PROFILER_FUNCTION(); EXIT_IF(prepared.program == nullptr || prepared.snapshot == nullptr); const auto& program = *prepared.program; @@ -952,7 +1003,7 @@ void RenderExecutor::RebindImages(CommandBuffer& buffer, EXIT_IF(images.size() != program.info.images.size()); auto& texture_cache = m_context.GetTextureCache(); for (uint32_t i = 0; i < program.info.images.size(); i++) { - const auto old_image = texture_cache.ResolveOwner(images[i].image_id); + const auto old_image = texture_cache.m_slot_images.try_get(images[i].image_id); if (old_image == nullptr || (!old_image->registered && !old_image->info.data.Empty()) || old_image->binding.needs_rebind) { if (old_image != nullptr) { @@ -962,6 +1013,8 @@ void RenderExecutor::RebindImages(CommandBuffer& buffer, BindImage(images[i].image_id, images[i].desc.type == TextureCache::BindingType::Storage); } + } + for (uint32_t i = 0; i < program.info.images.size(); i++) { auto& binding = images[i]; binding.mip_views.clear(); const auto& resource = program.info.images[i]; @@ -991,42 +1044,31 @@ void RenderExecutor::RebindImages(CommandBuffer& buffer, } RenderExecutor::GraphicsBindings -RenderExecutor::PrepareGraphicsBindings(CommandBuffer& buffer, const ShaderStageRuntime& vertex, +RenderExecutor::PrepareGraphicsBindings(const ShaderStageRuntime& vertex, const ShaderStageRuntime& pixel, bool pixel_active) { GraphicsBindings bindings { - .vertex = PrepareBindings(buffer, vertex, vk::ShaderStageFlagBits::eVertex, + .vertex = PrepareBindings(vertex, vk::ShaderStageFlagBits::eVertex, DescriptorCache::Stage::Vertex), }; - RebindBuffers(buffer, bindings.vertex); - RebindImages(buffer, bindings.vertex); if (pixel_active) { - bindings.pixel.emplace(PrepareBindings(buffer, pixel, vk::ShaderStageFlagBits::eFragment, + bindings.pixel.emplace(PrepareBindings(pixel, vk::ShaderStageFlagBits::eFragment, DescriptorCache::Stage::Pixel)); - RebindBuffers(buffer, *bindings.pixel); - RebindImages(buffer, *bindings.pixel); + } + FindBuffers(bindings.vertex); + if (bindings.pixel) { + FindBuffers(*bindings.pixel); + } + RebindBuffers(bindings.vertex); + if (bindings.pixel) { + RebindBuffers(*bindings.pixel); + } + RebindImages(bindings.vertex); + if (bindings.pixel) { + RebindImages(*bindings.pixel); } return bindings; } -static void RetainBindings(CommandBuffer& buffer, DescriptorCache::NativeDescriptors& resources) { - if (resources.flattened_srt.owner != nullptr) { - buffer.RetainResourceUntilFence(std::move(resources.flattened_srt.owner)); - } - if (resources.user_data.owner != nullptr) { - buffer.RetainResourceUntilFence(std::move(resources.user_data.owner)); - } - for (auto& view: resources.buffers) { - if (view.owner != nullptr) { - buffer.RetainResourceUntilFence(std::move(view.owner)); - } - } - for (auto& view: resources.addresses) { - if (view.owner != nullptr) { - buffer.RetainResourceUntilFence(std::move(view.owner)); - } - } -} - void RenderExecutor::CommitBindings(CommandBuffer& buffer, vk::PipelineBindPoint pipeline_bind_point, vk::PipelineLayout layout, @@ -1036,15 +1078,6 @@ void RenderExecutor::CommitBindings(CommandBuffer& buffer, prepared.committed); const auto& program = *prepared.program; auto& descriptors = prepared.resources; - if (!prepared.flattened_srt.empty()) { - descriptors.flattened_srt = NativeUpload(m_context, buffer, prepared.flattened_srt); - } - if (ShaderRecompiler::IR::FindBinding( - program.bindings, ShaderRecompiler::IR::DescriptorBindingKind::UserData) != nullptr) { - descriptors.user_data = NativeUpload(m_context, buffer, prepared.user_data); - } - RetainBindings(buffer, descriptors); - auto vk_buffer = buffer.Handle(); const auto shader_stages = ShaderPipelineStages(prepared.shader_stage); if (descriptors.gds.buffer != nullptr) { diff --git a/src/graphics/host_gpu/renderer/render.h b/src/graphics/host_gpu/renderer/render.h index d2ef08e1c..bf54c8e88 100644 --- a/src/graphics/host_gpu/renderer/render.h +++ b/src/graphics/host_gpu/renderer/render.h @@ -9,7 +9,6 @@ #include "graphics/host_gpu/vulkanCommon.h" #include -#include #include namespace Libs::Graphics { @@ -24,7 +23,6 @@ struct GraphicContext; struct ShaderBufferResource; struct ShaderComputeInputInfo; struct CommandSlot; -struct VulkanBuffer; struct VulkanDescriptorSet; struct RenderDepthInfo; struct RenderColorInfo; @@ -49,20 +47,6 @@ enum class CommandBufferDebugOp : uint32_t { Unknown, }; -class FenceResourceRetainer { -public: - FenceResourceRetainer() = default; - ~FenceResourceRetainer(); - KYTY_CLASS_NO_COPY(FenceResourceRetainer); - - void Retain(std::shared_ptr resource); - void ReleaseAfterFence() noexcept; - [[nodiscard]] bool Empty() const noexcept { return m_resources.empty(); } - -private: - std::vector> m_resources; -}; - struct SubmitInfo { static constexpr uint32_t MaxSemaphores = 3; @@ -108,8 +92,6 @@ public: void WaitForFenceOnly(); void WaitForFence(); void WaitForFenceAndReset(); - void RetireBufferAfterFence(std::unique_ptr buffer); - void RetainResourceUntilFence(std::shared_ptr resource); void RecycleDescriptorAfterFence(VulkanDescriptorSet& set); [[nodiscard]] vk::CommandBuffer Handle() const; @@ -120,8 +102,6 @@ public: private: void Release(); void FinalizeFence(bool reset_recording); - void ReleaseResourcesAfterFence(); - void DeleteBuffersAfterFence(); void RecycleDescriptorsAfterFence(); RenderContext& m_context; @@ -138,8 +118,6 @@ private: uint32_t m_debug_arg2 = 0; uint32_t m_debug_arg3 = 0; uint64_t m_debug_arg4 = 0; - std::vector> m_retired_buffers; - FenceResourceRetainer m_fence_resources; std::vector m_descriptor_sets_after_fence; mutable RenderState m_render_state; mutable bool m_rendering = false; @@ -182,10 +160,11 @@ public: uint32_t thread_group_y, uint32_t thread_group_z, uint32_t mode); [[nodiscard]] DescriptorCache::PreparedBindings - PrepareBindings(CommandBuffer& buffer, const ShaderStageRuntime& runtime, - vk::ShaderStageFlags shader_stage, DescriptorCache::Stage stage); - void RebindBuffers(CommandBuffer& buffer, DescriptorCache::PreparedBindings& bindings); - void RebindImages(CommandBuffer& buffer, DescriptorCache::PreparedBindings& bindings); + PrepareBindings(const ShaderStageRuntime& runtime, vk::ShaderStageFlags shader_stage, + DescriptorCache::Stage stage); + void FindBuffers(DescriptorCache::PreparedBindings& bindings); + void RebindBuffers(DescriptorCache::PreparedBindings& bindings); + void RebindImages(DescriptorCache::PreparedBindings& bindings); void CommitBindings(CommandBuffer& buffer, vk::PipelineBindPoint pipeline_bind_point, vk::PipelineLayout layout, DescriptorCache::PreparedBindings& bindings); @@ -198,8 +177,7 @@ private: [[nodiscard]] DescriptorCache::TextureBinding ResolveTexture(const ShaderRecompiler::IR::ImageResource& resource, const ShaderRecompiler::IR::DescriptorValue& value); - [[nodiscard]] GraphicsBindings PrepareGraphicsBindings(CommandBuffer& buffer, - const ShaderStageRuntime& vertex, + [[nodiscard]] GraphicsBindings PrepareGraphicsBindings(const ShaderStageRuntime& vertex, const ShaderStageRuntime& pixel, bool pixel_active); void ResolveRenderColorTarget(uint64_t submit_id, RenderCommandBuffer& buffer, @@ -229,8 +207,8 @@ private: [[nodiscard]] bool TryConsumeComputeMetaClear(const ShaderComputeInputInfo& input, const RenderCommandBuffer& buffer); - RenderContext& m_context; - std::vector> m_bound_images; + RenderContext& m_context; + std::vector m_bound_images; friend struct RenderExecutorTestAccess; }; diff --git a/src/graphics/host_gpu/renderer/renderCompute.cpp b/src/graphics/host_gpu/renderer/renderCompute.cpp index c863c45b7..4e9f59d05 100644 --- a/src/graphics/host_gpu/renderer/renderCompute.cpp +++ b/src/graphics/host_gpu/renderer/renderCompute.cpp @@ -168,6 +168,7 @@ void RenderExecutor::DispatchDirect(uint64_t submit_id, RenderCommandBuffer& buf uint32_t thread_group_x, uint32_t thread_group_y, uint32_t thread_group_z, uint32_t mode) { EXIT_IF(buffer.IsInvalid()); + m_context.GetCommandScheduler().PopPendingOperations(); auto& ctx = buffer.GetRegisters(); auto& sh_ctx = buffer.GetShaders(); @@ -333,10 +334,11 @@ void RenderExecutor::DispatchDirect(uint64_t submit_id, RenderCommandBuffer& buf buffer.EndRendering(); auto& pipeline = m_context.GetPipelineCache().CreateComputePipeline(input_info, sh_ctx.GetCs(), cs_shader); - auto bindings = PrepareBindings(buffer, input_info.stage, vk::ShaderStageFlagBits::eCompute, + auto bindings = PrepareBindings(input_info.stage, vk::ShaderStageFlagBits::eCompute, DescriptorCache::Stage::Compute); - RebindBuffers(buffer, bindings); - RebindImages(buffer, bindings); + FindBuffers(bindings); + RebindBuffers(bindings); + RebindImages(bindings); auto vk_buffer = buffer.Handle(); CommitBindings(buffer, vk::PipelineBindPoint::eCompute, pipeline.pipeline_layout, bindings); diff --git a/src/graphics/host_gpu/renderer/renderDraw.cpp b/src/graphics/host_gpu/renderer/renderDraw.cpp index 6ef035ae4..fbd4d4bdd 100644 --- a/src/graphics/host_gpu/renderer/renderDraw.cpp +++ b/src/graphics/host_gpu/renderer/renderDraw.cpp @@ -616,7 +616,7 @@ RenderState RenderExecutor::AcquireRenderTargets(CommandBuffer& buffer, RenderCo for (uint32_t i = 0; i < color_count; i++) { auto& target = colors[i]; EXIT_IF(!target.image_id); - const auto old_image = cache.ResolveOwner(target.image_id); + const auto old_image = cache.m_slot_images.try_get(target.image_id); if (old_image == nullptr || (!old_image->registered && !old_image->info.data.Empty()) || old_image->binding.needs_rebind) { if (old_image != nullptr) { @@ -668,7 +668,7 @@ RenderState RenderExecutor::AcquireRenderTargets(CommandBuffer& buffer, RenderCo attachment.is_clear = target.color_clear_enable || metadata_clear; } if (depth.image_id) { - const auto owner = cache.ResolveOwner(depth.image_id); + const auto owner = cache.m_slot_images.try_get(depth.image_id); if (owner == nullptr || !owner->registered || owner->binding.needs_rebind) { EXIT("depth target changed after render-state discovery\n"); } @@ -786,11 +786,10 @@ struct DrawIndexBufferSource { }; struct PreparedIndexBuffer { - std::shared_ptr owner; - vk::Buffer buffer = nullptr; - uint64_t size = 0; - vk::DeviceSize offset = 0; - vk::IndexType type = vk::IndexType::eUint16; + vk::Buffer buffer = nullptr; + uint64_t size = 0; + vk::DeviceSize offset = 0; + vk::IndexType type = vk::IndexType::eUint16; }; static uint64_t VertexBufferDescriptorSize(const ShaderVertexInputBuffer& buffer) { @@ -802,7 +801,7 @@ struct VertexBufferRange { uint64_t base_address = 0; uint64_t requested_end = 0; uint64_t acquired_end = 0; - BufferBinding binding; + std::pair binding; [[nodiscard]] uint64_t RequestedSize() const { return requested_end - base_address; } }; @@ -810,11 +809,9 @@ struct VertexBufferRange { struct PreparedVertexBuffers { static constexpr uint32_t MaxBuffers = ShaderVertexInputInfo::RES_MAX; - std::array buffers {}; - std::array offsets {}; - std::array, MaxBuffers> owners {}; - uint32_t count = 0; - uint32_t owner_count = 0; + std::array buffers {}; + std::array offsets {}; + uint32_t count = 0; }; static PreparedVertexBuffers AcquireVertexBuffers(RenderCommandBuffer& buffer, @@ -864,8 +861,9 @@ static PreparedVertexBuffers AcquireVertexBuffers(RenderCommandBuffer& b const auto size = Libs::LibKernel::Memory::ClampRangeSize(range.base_address, range.RequestedSize()); range.acquired_end = range.base_address + size; - range.binding = cache.ObtainBuffer(buffer, range.base_address, size); - SetVulkanObjectNameF(buffer.GetContext().GetGraphics().device, range.binding.buffer, + range.binding = cache.ObtainBuffer(range.base_address, size, false); + SetVulkanObjectNameF(buffer.GetContext().GetGraphics().device, + range.binding.first->Handle(), "Kyty.VertexBufferRange[guest=0x{:016x} size=0x{:x}]", range.base_address, size); } @@ -873,15 +871,13 @@ static PreparedVertexBuffers AcquireVertexBuffers(RenderCommandBuffer& b // Rebuild slot bindings, offsetting non-empty slots into their acquired merged range. PreparedVertexBuffers prepared; prepared.count = static_cast(vs_input_info.buffers_num); - std::shared_ptr null_owner; - vk::Buffer null_buffer = nullptr; + vk::Buffer null_buffer = nullptr; for (int i = 0; i < vs_input_info.buffers_num; i++) { const auto& vertex = vs_input_info.buffers[i]; const auto size = VertexBufferDescriptorSize(vertex); if (size == 0) { - if (null_owner == nullptr) { - null_owner = cache.ObtainNullBuffer(); - null_buffer = null_owner->Handle(); + if (null_buffer == nullptr) { + null_buffer = cache.GetBuffer(NULL_BUFFER_ID).Handle(); } prepared.buffers[i] = null_buffer; prepared.offsets[i] = 0; @@ -898,23 +894,14 @@ static PreparedVertexBuffers AcquireVertexBuffers(RenderCommandBuffer& b vertex.addr); } - prepared.buffers[i] = range->binding.buffer; - prepared.offsets[i] = range->binding.offset + vertex.addr - range->base_address; + prepared.buffers[i] = range->binding.first->Handle(); + prepared.offsets[i] = range->binding.second + vertex.addr - range->base_address; SetVulkanObjectNameF( buffer.GetContext().GetGraphics().device, prepared.buffers[i], "Kyty.VertexBuffer[slot={} guest=0x{:016x} size=0x{:x} stride={} records={}]", i, vertex.addr, size, vertex.stride, vertex.num_records); } - if (null_owner != nullptr) { - prepared.owners[prepared.owner_count++] = std::move(null_owner); - } - for (uint32_t i = 0; i < merged_count; i++) { - if (merged_ranges[i].binding.owner != nullptr) { - EXIT_IF(prepared.owner_count >= PreparedVertexBuffers::MaxBuffers); - prepared.owners[prepared.owner_count++] = std::move(merged_ranges[i].binding.owner); - } - } return prepared; } @@ -1100,17 +1087,14 @@ static PreparedIndexBuffer PrepareIndexBuffer(RenderCommandBuffer& buffe prepared.size = source.size; prepared.type = source.type; if (source.host_data != nullptr) { - auto binding = - buffer.GetContext().GetBufferCache().UploadTransient(source.host_data, source.size, 16); - prepared.owner = std::move(binding.owner); - prepared.buffer = binding.buffer; - prepared.offset = binding.offset; + auto& stream = buffer.GetContext().GetBufferCache().GetUtilityBuffer(MemoryUsage::Stream); + prepared.offset = stream.Copy(source.host_data, source.size, 16); + prepared.buffer = stream.Handle(); } else { - auto binding = - buffer.GetContext().GetBufferCache().ObtainBuffer(buffer, source.address, source.size); - prepared.owner = std::move(binding.owner); - prepared.buffer = binding.buffer; - prepared.offset = binding.offset; + auto [buffer_ptr, offset] = buffer.GetContext().GetBufferCache().ObtainBuffer( + source.address, source.size, false); + prepared.buffer = buffer_ptr->Handle(); + prepared.offset = offset; } if (source.host_data != nullptr) { SetVulkanObjectNameF(buffer.GetContext().GetGraphics().device, prepared.buffer, @@ -1124,13 +1108,7 @@ static PreparedIndexBuffer PrepareIndexBuffer(RenderCommandBuffer& buffe return prepared; } -static void CommitVertexBuffers(RenderCommandBuffer& buffer, vk::CommandBuffer vk_buffer, - PreparedVertexBuffers& prepared) { - for (uint32_t i = 0; i < prepared.owner_count; i++) { - if (prepared.owners[i] != nullptr) { - buffer.RetainResourceUntilFence(std::move(prepared.owners[i])); - } - } +static void CommitVertexBuffers(vk::CommandBuffer vk_buffer, const PreparedVertexBuffers& prepared) { for (uint32_t i = 0; i < prepared.count; i++) { EXIT_IF(prepared.buffers[i] == nullptr); } @@ -1140,14 +1118,10 @@ static void CommitVertexBuffers(RenderCommandBuffer& buffer, vk::CommandBuffer v } } -static void CommitIndexBuffer(RenderCommandBuffer& buffer, vk::CommandBuffer vk_buffer, - PreparedIndexBuffer& prepared) { +static void CommitIndexBuffer(vk::CommandBuffer vk_buffer, const PreparedIndexBuffer& prepared) { if (prepared.size == 0) { return; } - if (prepared.owner != nullptr) { - buffer.RetainResourceUntilFence(std::move(prepared.owner)); - } EXIT_IF(prepared.buffer == nullptr); vk_buffer.bindIndexBuffer(prepared.buffer, prepared.offset, prepared.type); } @@ -1230,8 +1204,8 @@ void RenderExecutor::ExecutePreparedDraw(uint64_t submit_id, RenderCommandBuffer auto& ucfg = buffer.GetUserConfig(); LogDrawPhase(draw.name, "PrepareBindings"); - auto bindings = PrepareGraphicsBindings(buffer, state.vs_input_info.stage, - state.ps_input_info.stage, state.ps_active); + auto bindings = PrepareGraphicsBindings(state.vs_input_info.stage, state.ps_input_info.stage, + state.ps_active); auto vertex_bindings = PrepareVertexBuffers(submit_id, buffer, draw, state.vs_input_info); auto index_binding = PrepareIndexBuffer(buffer, index_source); state.rendering = @@ -1255,7 +1229,7 @@ void RenderExecutor::ExecutePreparedDraw(uint64_t submit_id, RenderCommandBuffer if (set_auto_debug) { SetDrawDebugPhase(buffer, submit_id, draw, 0x200u); } - CommitVertexBuffers(buffer, vk_buffer, vertex_bindings); + CommitVertexBuffers(vk_buffer, vertex_bindings); CommitBindings(buffer, vk::PipelineBindPoint::eGraphics, pipeline.pipeline_layout, bindings.vertex); if (bindings.pixel.has_value()) { @@ -1265,7 +1239,7 @@ void RenderExecutor::ExecutePreparedDraw(uint64_t submit_id, RenderCommandBuffer CommitBindings(buffer, vk::PipelineBindPoint::eGraphics, pipeline.pipeline_layout, *bindings.pixel); } - CommitIndexBuffer(buffer, vk_buffer, index_binding); + CommitIndexBuffer(vk_buffer, index_binding); const auto dynamic_params = BuildGraphicsDynamicParams(buffer, state.color_info, state.color_count, state.depth_info); @@ -1310,6 +1284,7 @@ void RenderExecutor::DrawIndex(uint64_t submit_id, RenderCommandBuffer& buffer, KYTY_PROFILER_FUNCTION(); EXIT_IF(buffer.IsInvalid()); + m_context.GetCommandScheduler().PopPendingOperations(); auto& ucfg = buffer.GetUserConfig(); auto& sh_ctx = buffer.GetShaders(); @@ -1441,6 +1416,7 @@ void RenderExecutor::DrawAuto(uint64_t submit_id, RenderCommandBuffer& buffer, u KYTY_PROFILER_FUNCTION(); EXIT_IF(buffer.IsInvalid()); + m_context.GetCommandScheduler().PopPendingOperations(); auto& ucfg = buffer.GetUserConfig(); auto& sh_ctx = buffer.GetShaders(); diff --git a/src/graphics/host_gpu/renderer/sync.cpp b/src/graphics/host_gpu/renderer/sync.cpp index 94ebbadfb..e2afef6ed 100644 --- a/src/graphics/host_gpu/renderer/sync.cpp +++ b/src/graphics/host_gpu/renderer/sync.cpp @@ -390,7 +390,7 @@ int DeleteEqEvent(LibKernel::EventQueue::KernelEqueue eq, int id) { return result; } -void ReadGds(Buffer& gds, uint32_t* dst, uint32_t dw_offset, uint32_t dw_size) { +void ReadGds(const Buffer& gds, uint32_t* dst, uint32_t dw_offset, uint32_t dw_size) { const auto offset = uint64_t {dw_offset} * sizeof(uint32_t); const auto size = uint64_t {dw_size} * sizeof(uint32_t); EXIT_IF(dst == nullptr || offset > gds.Size() || size > gds.Size() - offset || diff --git a/src/graphics/host_gpu/renderer/sync.h b/src/graphics/host_gpu/renderer/sync.h index 1271a5ac6..7abf1d67f 100644 --- a/src/graphics/host_gpu/renderer/sync.h +++ b/src/graphics/host_gpu/renderer/sync.h @@ -62,7 +62,7 @@ void WriteAtEndOfPipeWithInterruptWriteBackFlip32(uint64_t submit_id, CommandBuf int AddEqEvent(RenderContext& renderer, LibKernel::EventQueue::KernelEqueue eq, int id, void* udata); int DeleteEqEvent(LibKernel::EventQueue::KernelEqueue eq, int id); -void ReadGds(Buffer& gds, uint32_t* dst, uint32_t dw_offset, uint32_t dw_size); +void ReadGds(const Buffer& gds, uint32_t* dst, uint32_t dw_offset, uint32_t dw_size); } // namespace Sync } // namespace Libs::Graphics diff --git a/src/graphics/host_gpu/vma.cpp b/src/graphics/host_gpu/vma.cpp index 82fb5fa26..6d2d29fee 100644 --- a/src/graphics/host_gpu/vma.cpp +++ b/src/graphics/host_gpu/vma.cpp @@ -199,20 +199,6 @@ void GraphicContext::CreateBuffer(uint64_t size, VulkanBuffer& buffer) { VulkanTrackAllocation(buffer.memory); } -void GraphicContext::DeleteBuffer(VulkanBuffer& buffer) { - KYTY_PROFILER_FUNCTION(); - EXIT_IF(allocator == nullptr || buffer.buffer == nullptr || - buffer.memory.allocation == nullptr); - - VulkanUntrackAllocation(buffer.memory); - vmaDestroyBuffer(allocator, buffer.buffer, buffer.memory.allocation); - buffer.buffer = nullptr; - buffer.memory.memory = nullptr; - buffer.memory.allocation = nullptr; - buffer.memory.allocation_info = {}; - buffer.memory.offset = 0; -} - bool GraphicContext::CreateImage(const vk::ImageCreateInfo& image_info, VulkanImage& image) { KYTY_PROFILER_FUNCTION(); EXIT_IF(allocator == nullptr || image.image != nullptr || image.memory.allocation != nullptr); diff --git a/tests/ShaderRecompilerComputeTests.cpp b/tests/ShaderRecompilerComputeTests.cpp index e46586796..d94fd3020 100644 --- a/tests/ShaderRecompilerComputeTests.cpp +++ b/tests/ShaderRecompilerComputeTests.cpp @@ -97,6 +97,10 @@ template concept HasGetDownloadBuffer = requires(Cache &cache) { cache.GetDownloadBuffer(uint64_t{1}); }; static_assert(!HasGetDownloadBuffer); +static_assert(std::same_as); +static_assert(std::same_as); template concept HasSynchronizeImageToBuffer = requires(Cache &cache) { @@ -114,13 +118,10 @@ template concept HasDiscardGpuDirtyBytes = requires(Cache &cache) { cache.DiscardGpuDirtyBytes(uint64_t{1}, uint64_t{1}); }; -template -concept HasGpuOwnedImageSource = requires(Source &source) { source.gpu_owned; }; static_assert(!HasSynchronizeImageToBuffer); static_assert(!HasObtainBufferForImageCopy); static_assert(!HasObtainBufferForImageWrite); static_assert(!HasDiscardGpuDirtyBytes); -static_assert(!HasGpuOwnedImageSource); template concept HasLegacyImageLayout = requires(Backing &backing) { backing.layout; }; @@ -136,6 +137,9 @@ static_assert(BlitHelper::ColorToMsDepthLayout == vk::ImageLayout::eDepthStencilAttachmentOptimal); struct BufferCacheTestAccess { + static_assert(std::same_as>); + static void SetGarbageCollectionThresholds(BufferCache &cache, uint64_t trigger, uint64_t critical) { @@ -178,6 +182,8 @@ struct TileManagerTestAccess { }; struct TextureCacheTestAccess { + static_assert(std::same_as>); static_assert(TextureCache::ImagePageTable::kPageBits == 20); static_assert(TextureCache::ImagePageTable::kAddressSpaceBits == 40); static_assert(TextureCache::ImagePageTable::kFirstLevelBits == 10); @@ -195,15 +201,14 @@ struct TextureCacheTestAccess { cache.m_gc_tick = tick; std::vector live; cache.m_lru_cache = {}; - for (auto &slot : cache.m_slots) { - if (slot.image != nullptr && slot.image->registered) { - slot.image->tick_accessed_last = cache.m_scheduler.CurrentTick(); - live.push_back({static_cast(&slot - cache.m_slots.data()), - slot.generation}); + cache.m_slot_images.ForEach([&](ImageId id, Image &image) { + if (image.registered) { + image.tick_accessed_last = cache.m_scheduler.CurrentTick(); + live.push_back(id); } - } + }); for (const auto id : oldest) { - const auto owner = cache.ResolveOwner(id); + const auto owner = cache.m_slot_images.try_get(id); if (owner != nullptr && owner->registered) { owner->tick_accessed_last = 0; owner->lru_id = cache.m_lru_cache.Insert(id, 0); @@ -211,13 +216,13 @@ struct TextureCacheTestAccess { } for (const auto id : live) { if (std::ranges::find(oldest, id) == oldest.end()) { - cache.ResolveImage(id).lru_id = cache.m_lru_cache.Insert(id, tick); + cache.m_slot_images[id].lru_id = cache.m_lru_cache.Insert(id, tick); } } } static bool Contains(const TextureCache &cache, ImageId id) { - const auto owner = cache.ResolveOwner(id); + const auto owner = cache.m_slot_images.try_get(id); return owner != nullptr && owner->registered; } @@ -291,8 +296,8 @@ struct TextureCacheTestAccess { cache.DeleteImage(id); } - static std::shared_ptr Owner(const TextureCache &cache, ImageId id) { - return cache.ResolveOwner(id); + static const Image *Owner(const TextureCache &cache, ImageId id) { + return cache.m_slot_images.try_get(id); } static bool PendingDownload(const TextureCache &cache, ImageId id) { @@ -304,11 +309,13 @@ struct TextureCacheTestAccess { } static void TrackDownload(TextureCache &cache, ImageId id) { - cache.TrackImageDownload(id); + std::lock_guard lock(cache.m_lock); + cache.TrackImageDownload(id, cache.m_slot_images[id]); } static void AssociateStencil(TextureCache &cache, ImageId depth, GuestRange stencil) { + std::lock_guard lock(cache.m_lock); cache.AssociateStencil(depth, stencil); } @@ -318,14 +325,20 @@ struct TextureCacheTestAccess { static std::pair MapDownload(TextureCache &cache, uint64_t size, uint64_t alignment) { - return cache.MapDownload(size, alignment); + auto &download = + cache.m_buffer_cache.GetUtilityBuffer(MemoryUsage::Download); + auto mapping = download.Map(size, std::max(alignment, 4)); + if (mapping.first != nullptr) { + download.Commit(); + } + return mapping; } static bool TryDownload(TextureCache &cache, ImageId id) { return cache.TryDownloadImage(id); } - static TileManager &Tiler(TextureCache &cache) { return *cache.m_tiler; } + static TileManager &Tiler(TextureCache &cache) { return cache.m_tiler; } }; struct RenderExecutorTestAccess { @@ -337,12 +350,10 @@ struct RenderExecutorTestAccess { } static auto PrepareGraphicsBindings(RenderExecutor &executor, - CommandBuffer &buffer, const ShaderStageRuntime &vertex, const ShaderStageRuntime &pixel, bool pixel_active) { - return executor.PrepareGraphicsBindings(buffer, vertex, pixel, - pixel_active); + return executor.PrepareGraphicsBindings(vertex, pixel, pixel_active); } static void CommitBindings(RenderExecutor &executor, CommandBuffer &buffer, @@ -382,8 +393,7 @@ struct RenderExecutorTestAccess { } static bool BoundImagesInOrder(const RenderExecutor &executor, - const std::shared_ptr &first, - const std::shared_ptr &second) { + ImageId first, ImageId second) { return executor.m_bound_images.size() == 2 && executor.m_bound_images[0] == first && executor.m_bound_images[1] == second; @@ -977,7 +987,7 @@ void CheckRectListShaders() { shaders.control.size() > 1 && shaders.control[1] == 0x00010500u && shaders.evaluation.size() > 1 && shaders.evaluation[1] == 0x00010500u, - "shadPS4-compatible vector selection requires SPIR-V 1.5"); + "vector selection requires SPIR-V 1.5"); ValidateSpirv(name, shaders.control); ValidateSpirv(name, shaders.evaluation); @@ -1452,10 +1462,13 @@ public: &completed] { completed = *owned; }); scheduler.Flush(); scheduler.Wait(first_tick); - Require("SchedulerTimeline", "first tick", + Require("SchedulerTimeline", "wait boundary", scheduler.CurrentTick() == first_tick + 1 && - scheduler.IsFree(first_tick) && completed == 1, - "timeline wait did not release its deferred operation"); + scheduler.IsFree(first_tick) && completed == 0, + "timeline wait released a resource in the middle of an operation"); + scheduler.PopPendingOperations(); + Require("SchedulerTimeline", "first tick", completed == 1, + "operation boundary did not release its deferred operation"); const auto second_tick = scheduler.CurrentTick(); std::atomic priority_in_callback{false}; @@ -1479,8 +1492,11 @@ public: scheduler.Wait(implicit_flush_tick); Require("SchedulerTimeline", "implicit flush", scheduler.CurrentTick() == implicit_flush_tick + 1 && - scheduler.IsFree(implicit_flush_tick) && completed == 3, - "waiting for the current tick did not flush it"); + scheduler.IsFree(implicit_flush_tick) && completed == 2, + "waiting for the current tick released a deferred resource"); + scheduler.PopPendingOperations(); + Require("SchedulerTimeline", "implicit boundary", completed == 3, + "operation boundary did not release the implicit tick callback"); vk::SemaphoreTypeCreateInfo timeline_type{}; timeline_type.sType = vk::StructureType::eSemaphoreTypeCreateInfo; @@ -1684,30 +1700,18 @@ public: "oversized download replaced or corrupted the fixed shared ring"); fixed_download->Commit(); - std::vector full_stream(64ull << 20, 0x5a); - const auto utility_tick = scheduler.CurrentTick(); - auto full_binding = - cache.UploadTransient(full_stream.data(), full_stream.size(), 16); const std::array overflow_data{ 0xa5, 0xa5, 0xa5, 0xa5, 0xa5, 0xa5, 0xa5, 0xa5, 0xa5, 0xa5, 0xa5, 0xa5, 0xa5, 0xa5, 0xa5, 0xa5}; - auto overflow_binding = - cache.UploadTransient(overflow_data.data(), overflow_data.size(), 16); - auto overflow_owner = std::static_pointer_cast( - overflow_binding.owner); + auto &upload_stream = cache.GetUtilityBuffer(MemoryUsage::Stream); + const auto stream_offset = + upload_stream.Copy(overflow_data.data(), overflow_data.size(), 16); Require( - "StreamBufferRing", "current tick overflow", - full_binding.owner == nullptr && - full_binding.buffer == - cache.GetUtilityBuffer(MemoryUsage::Stream).Handle() && - overflow_owner != nullptr && - overflow_binding.buffer == overflow_owner->Handle() && - overflow_binding.buffer != full_binding.buffer && - overflow_owner->Mapped().size() == overflow_data.size() && - std::memcmp(overflow_owner->Mapped().data(), overflow_data.data(), - overflow_data.size()) == 0 && - scheduler.CurrentTick() == utility_tick, - "current-tick stream overflow submitted partial state or lost bytes"); + "StreamBufferRing", "cache-owned stream upload", + upload_stream.Handle() != nullptr && stream_offset < upload_stream.Size() && + std::memcmp(upload_stream.Mapped().data() + stream_offset, overflow_data.data(), + overflow_data.size()) == 0, + "cache-owned stream upload lost bytes"); const auto atom = m_runtime_context.physical_device_properties.limits.nonCoherentAtomSize; @@ -2249,10 +2253,10 @@ public: "DMA prefetch modified ownership or did not establish a dirty " "cached page"); - buffer_cache.FillBuffer(clean_cached_fill, sizeof(source_words), - clean_fill_value); - buffer_cache.CopyBuffer(clean_cached_copy, clean_cached_source, - sizeof(clean_copy_words)); + buffer_cache.FillBuffer(clean_cached_fill, sizeof(source_words), + clean_fill_value, false); + buffer_cache.CopyBuffer(clean_cached_copy, clean_cached_source, + sizeof(clean_copy_words), false, false); Require("GpuCommandLane", "clean cached DMA ownership", !buffer_cache.HasGpuDirtyBytes(clean_cached_fill, sizeof(source_words)) && @@ -2355,15 +2359,13 @@ public: std::memcpy(reinterpret_cast(dirty_fault_address), &dirty_fault_stale, sizeof(dirty_fault_stale)); gpu.SendCommandSync([&] { - auto dirty = - buffer_cache.ObtainBuffer(scheduler.Current(), dirty_fault_address, - sizeof(dirty_fault_value), true, false); + auto dirty = buffer_cache.ObtainBuffer( + dirty_fault_address, sizeof(dirty_fault_value), true, false); Require("GpuCommandLane", "dirty tracked fault setup", - dirty.owner != nullptr, + dirty.first != nullptr, "dirty write-fault test could not create a cached Buffer"); - scheduler.Current().RetainResourceUntilFence(dirty.owner); - buffer_cache.FillBuffer(dirty_fault_address, sizeof(dirty_fault_value), - dirty_fault_value); + buffer_cache.FillBuffer(dirty_fault_address, sizeof(dirty_fault_value), + dirty_fault_value, false); }); Require("GpuCommandLane", "dirty tracked fault ownership", buffer_cache.HasGpuDirtyBytes(dirty_fault_address, @@ -2425,15 +2427,13 @@ public: constexpr uint32_t empty_copy_fault_value = 0x6b28d1efu; gpu.SendCommandSync([&] { auto dirty = buffer_cache.ObtainBuffer( - scheduler.Current(), empty_copy_fault_address, - sizeof(empty_copy_fault_value), true, false); + empty_copy_fault_address, sizeof(empty_copy_fault_value), true, false); Require("GpuCommandLane", "empty-copy write setup", - dirty.owner != nullptr, + dirty.first != nullptr, "empty-copy write test could not create a cached Buffer"); - scheduler.Current().RetainResourceUntilFence(dirty.owner); - buffer_cache.FillBuffer(empty_copy_fault_address, - sizeof(empty_copy_fault_value), - empty_copy_fault_value); + buffer_cache.FillBuffer(empty_copy_fault_address, + sizeof(empty_copy_fault_value), + empty_copy_fault_value, false); buffer_cache.ReadMemory(empty_copy_fault_address, sizeof(empty_copy_fault_value)); const bool gpu_owned = buffer_cache.IsRegionGpuModified( @@ -2760,11 +2760,9 @@ public: resources.MapMemory(base, allocation_size); const auto MarkGpuWrite = [&](uint64_t address, uint64_t size) { - auto allocation = - cache.ObtainBuffer(scheduler.Current(), address, size, true, false); - Require(name, "dirty allocation", allocation.owner != nullptr, + auto allocation = cache.ObtainBuffer(address, size, true, false); + Require(name, "dirty allocation", allocation.first != nullptr, "dirty-GC buffer allocation failed"); - scheduler.Current().RetainResourceUntilFence(allocation.owner); }; constexpr uint64_t index_offset = 0x180000; @@ -2774,17 +2772,13 @@ public: Require(name, "registered-range empty lookup", !cache.IsRegionRegistered(index_begin, index_span), "empty Buffer cache reported a registered range"); - auto index_left = - cache.ObtainBuffer(scheduler.Current(), index_begin + 0x100, - sizeof(uint32_t), false, false); - auto index_right = cache.ObtainBuffer( - scheduler.Current(), index_begin + 2 * index_page + 0x100, - sizeof(uint32_t), false, false); + const auto index_left = + cache.FindBuffer(index_begin + 0x100, sizeof(uint32_t)); + const auto index_right = cache.FindBuffer( + index_begin + 2 * index_page + 0x100, sizeof(uint32_t)); Require(name, "registered-range owners", - index_left.owner != nullptr && index_right.owner != nullptr, + index_left && index_right, "failed to create disjoint Buffer index owners"); - scheduler.Current().RetainResourceUntilFence(index_left.owner); - scheduler.Current().RetainResourceUntilFence(index_right.owner); Require( name, "registered-range boundaries", cache.IsRegionRegistered(index_begin, 1) && @@ -2797,15 +2791,23 @@ public: cache.IsRegionRegistered(index_begin - 1, index_span + 2), "indexed lookup mishandled a half-open boundary, gap, or broad " "overlap"); - auto index_bridge = - cache.ObtainBuffer(scheduler.Current(), index_begin + index_page - 1, - index_page + 2, false, false); + const auto index_bridge = + cache.FindBuffer(index_begin + index_page - 1, index_page + 2); Require( name, "registered-range merge", - index_bridge.owner != nullptr && + index_bridge && cache.IsRegionRegistered(index_begin + index_page, index_page), "bridging acquisition did not publish its merged Buffer range"); - scheduler.Current().RetainResourceUntilFence(index_bridge.owner); + const auto [resolved_left, resolved_left_offset] = cache.ObtainBuffer( + index_begin + 0x100, sizeof(uint32_t), true, false, index_left); + const auto [resolved_right, resolved_right_offset] = cache.ObtainBuffer( + index_begin + 2 * index_page + 0x100, sizeof(uint32_t), true, false, + index_right); + Require(name, "two-pass stale id resolution", + resolved_left == &cache.GetBuffer(index_bridge) && + resolved_right == resolved_left && + resolved_left_offset != resolved_right_offset, + "saved descriptor IDs did not resolve through the merged owner"); cache.UnmapMemory(index_begin, index_span); Require(name, "registered-range removal", !cache.IsRegionRegistered(index_begin, index_span), @@ -2813,9 +2815,9 @@ public: MarkGpuWrite(base + first_offset, sizeof(first_value)); MarkGpuWrite(base + second_offset, sizeof(second_value)); - cache.FillBuffer(base + first_offset, sizeof(first_value), first_value); - cache.FillBuffer(base + second_offset, sizeof(second_value), - second_value); + cache.FillBuffer(base + first_offset, sizeof(first_value), first_value, false); + cache.FillBuffer(base + second_offset, sizeof(second_value), + second_value, false); auto &download = BufferCacheTestAccess::DownloadBuffer(cache); auto *fixed_download = &download; const auto fixed_download_handle = download.Handle(); @@ -2829,11 +2831,11 @@ public: sizeof(ring_fault_first_value)); MarkGpuWrite(base + ring_fault_second_offset, sizeof(ring_fault_second_value)); - cache.FillBuffer(base + ring_fault_first_offset, - sizeof(ring_fault_first_value), ring_fault_first_value); - cache.FillBuffer(base + ring_fault_second_offset, - sizeof(ring_fault_second_value), - ring_fault_second_value); + cache.FillBuffer(base + ring_fault_first_offset, + sizeof(ring_fault_first_value), ring_fault_first_value, false); + cache.FillBuffer(base + ring_fault_second_offset, + sizeof(ring_fault_second_value), + ring_fault_second_value, false); Require(name, "fault-ring wrapped batch", resources.HandleFault(PageFaultAccess::Read, base + ring_fault_first_offset), @@ -2930,14 +2932,12 @@ public: "dirty GC did not publish exact ranges before " "clearing broad page ownership"); - auto unmap_allocation = - cache.ObtainBuffer(scheduler.Current(), base + unmap_offset, - sizeof(unmap_value), true, false); + auto unmap_allocation = cache.ObtainBuffer( + base + unmap_offset, sizeof(unmap_value), true, false); Require(name, "direct-unmap allocation", - unmap_allocation.owner != nullptr, + unmap_allocation.first != nullptr, "direct-unmap buffer allocation failed"); - scheduler.Current().RetainResourceUntilFence(unmap_allocation.owner); - cache.FillBuffer(base + unmap_offset, sizeof(unmap_value), unmap_value); + cache.FillBuffer(base + unmap_offset, sizeof(unmap_value), unmap_value, false); cache.UnmapMemory(base + 0x4000, 0x4000); uint32_t unmap_backing = 0; std::memcpy(&unmap_backing, memory + unmap_offset, sizeof(unmap_backing)); @@ -2948,33 +2948,28 @@ public: "publishing bytes"); auto partial_unmap_allocation = cache.ObtainBuffer( - scheduler.Current(), base + 0x8000, 0x8000, true, false); + base + 0x8000, 0x8000, true, false); Require(name, "partial-unmap allocation", - partial_unmap_allocation.owner != nullptr, + partial_unmap_allocation.first != nullptr, "cross-page cached buffer allocation failed"); - scheduler.Current().RetainResourceUntilFence( - partial_unmap_allocation.owner); - cache.FillBuffer(base + partial_unmap_offset, sizeof(partial_unmap_value), - partial_unmap_value); - cache.FillBuffer(base + partial_unmap_survivor_offset, - sizeof(partial_unmap_survivor_value), - partial_unmap_survivor_value); + cache.FillBuffer(base + partial_unmap_offset, sizeof(partial_unmap_value), + partial_unmap_value, false); + cache.FillBuffer(base + partial_unmap_survivor_offset, + sizeof(partial_unmap_survivor_value), + partial_unmap_survivor_value, false); cache.UnmapMemory(base + 0x8000, 0x4000); auto survivor = cache.ObtainBuffer( - scheduler.Current(), base + partial_unmap_survivor_offset, - sizeof(partial_unmap_survivor_value), false, true); - Require(name, "partial-unmap survivor", survivor.buffer != nullptr, + base + partial_unmap_survivor_offset, + sizeof(partial_unmap_survivor_value), false); + Require(name, "partial-unmap survivor", survivor.first != nullptr, "still-mapped cached-buffer remainder could not be recreated"); - if (survivor.owner != nullptr) { - scheduler.Current().RetainResourceUntilFence(survivor.owner); - } auto partial_unmap_readback = CreateHostBuffer(name, sizeof(partial_unmap_survivor_value), vk::BufferUsageFlagBits::eTransferDst, {0}); - const vk::BufferCopy survivor_copy{survivor.offset, 0, + const vk::BufferCopy survivor_copy{survivor.second, 0, sizeof(partial_unmap_survivor_value)}; scheduler.Current().Handle().copyBuffer( - survivor.buffer, partial_unmap_readback.buffer, 1, &survivor_copy); + survivor.first->Handle(), partial_unmap_readback.buffer, 1, &survivor_copy); vk::BufferMemoryBarrier survivor_barrier{}; survivor_barrier.sType = vk::StructureType::eBufferMemoryBarrier; survivor_barrier.srcAccessMask = vk::AccessFlagBits::eTransferWrite; @@ -3003,12 +2998,11 @@ public: std::memcpy(memory + large_offset + large_size - sizeof(large_stale), &large_stale, sizeof(large_stale)); auto large_allocation = cache.ObtainBuffer( - scheduler.Current(), base + large_offset, large_size, true, false); + base + large_offset, large_size, true, false); Require(name, "near-capacity dirty allocation", - large_allocation.owner != nullptr, + large_allocation.first != nullptr, "failed to allocate the near-capacity dirty native buffer"); - scheduler.Current().RetainResourceUntilFence(large_allocation.owner); - cache.FillBuffer(base + large_offset, large_size, large_value); + cache.FillBuffer(base + large_offset, large_size, large_value, false); for (uint32_t tick = 0; tick <= 160; tick++) { cache.RunGarbageCollector(); } @@ -3026,7 +3020,7 @@ public: const auto large_image_source = cache.ObtainBufferForImage(base + large_offset, sizeof(large_value)); Require(name, "fixed download after Buffer retirement", - large_image_source.buffer != nullptr && + large_image_source.first != nullptr && &BufferCacheTestAccess::DownloadBuffer(cache) == fixed_download && fixed_download->Handle() == fixed_download_handle && @@ -3055,23 +3049,18 @@ public: base + grouped_first_offset, &grouped_stale, sizeof(grouped_stale)); Libs::LibKernel::Memory::WriteBacking( base + grouped_second_offset, &grouped_stale, sizeof(grouped_stale)); - auto grouped_first = - cache.ObtainBuffer(scheduler.Current(), base + grouped_first_offset, - grouped_owner_size, true, false); - auto grouped_second = - cache.ObtainBuffer(scheduler.Current(), base + grouped_second_offset, - grouped_owner_size, true, false); + auto grouped_first = cache.ObtainBuffer( + base + grouped_first_offset, grouped_owner_size, true, false); + auto grouped_second = cache.ObtainBuffer( + base + grouped_second_offset, grouped_owner_size, true, false); Require(name, "per-owner GC allocation", - grouped_first.owner != nullptr && - grouped_second.owner != nullptr && - grouped_first.owner != grouped_second.owner, + grouped_first.first != nullptr && grouped_second.first != nullptr && + grouped_first.first != grouped_second.first, "disjoint GC candidates merged into one Buffer owner"); - scheduler.Current().RetainResourceUntilFence(grouped_first.owner); - scheduler.Current().RetainResourceUntilFence(grouped_second.owner); - cache.FillBuffer(base + grouped_first_offset, grouped_owner_size, - grouped_first_value); - cache.FillBuffer(base + grouped_second_offset, grouped_owner_size, - grouped_second_value); + cache.FillBuffer(base + grouped_first_offset, grouped_owner_size, + grouped_first_value, false); + cache.FillBuffer(base + grouped_second_offset, grouped_owner_size, + grouped_second_value, false); for (uint32_t tick = 0; tick <= 160; tick++) { cache.RunGarbageCollector(); } @@ -3107,20 +3096,14 @@ public: constexpr uint32_t disjoint_stale = 0x76543210u; std::memcpy(memory + disjoint_dirty_offset, &disjoint_stale, sizeof(disjoint_stale)); - auto wide_owner = - cache.ObtainBuffer(scheduler.Current(), base + disjoint_owner_offset, - disjoint_owner_size, false, true); - Require(name, "disjoint retirement owner", wide_owner.owner != nullptr, + auto wide_owner = cache.ObtainBuffer(base + disjoint_owner_offset, + disjoint_owner_size, false); + Require(name, "disjoint retirement owner", wide_owner.first != nullptr, "failed to create the clean cross-page retirement owner"); - scheduler.Current().RetainResourceUntilFence(wide_owner.owner); - auto dirty_alias = - cache.ObtainBuffer(scheduler.Current(), base + disjoint_dirty_offset, - sizeof(disjoint_value), true, false); - if (dirty_alias.owner != nullptr) { - scheduler.Current().RetainResourceUntilFence(dirty_alias.owner); - } - cache.FillBuffer(base + disjoint_dirty_offset, sizeof(disjoint_value), - disjoint_value); + (void)cache.ObtainBuffer(base + disjoint_dirty_offset, + sizeof(disjoint_value), true, false); + cache.FillBuffer(base + disjoint_dirty_offset, sizeof(disjoint_value), + disjoint_value, false); for (uint32_t tick = 0; tick <= 160; tick++) { cache.RunGarbageCollector(); } @@ -3148,12 +3131,11 @@ public: sizeof(disjoint_value)), "disjoint unmap lost the completed whole-owner publication"); auto disjoint_new_owner = cache.ObtainBuffer( - scheduler.Current(), base + disjoint_new_owner_offset, + base + disjoint_new_owner_offset, sizeof(disjoint_value), true, false); Require(name, "disjoint post-publication reacquire", - disjoint_new_owner.owner != nullptr, + disjoint_new_owner.first != nullptr, "disjoint acquisition failed after whole-owner publication"); - scheduler.Current().RetainResourceUntilFence(disjoint_new_owner.owner); uint32_t disjoint_backing = 0; Libs::LibKernel::Memory::TryReadBacking(base + disjoint_dirty_offset, &disjoint_backing, @@ -3175,21 +3157,15 @@ public: constexpr uint32_t reacquire_stale = 0x0badf00du; std::memcpy(memory + reacquire_dirty_offset, &reacquire_stale, sizeof(reacquire_stale)); - auto reacquire_owner = - cache.ObtainBuffer(scheduler.Current(), base + reacquire_owner_offset, - reacquire_owner_size, false, true); + auto reacquire_owner = cache.ObtainBuffer(base + reacquire_owner_offset, + reacquire_owner_size, false); Require(name, "reacquire retirement owner", - reacquire_owner.owner != nullptr, + reacquire_owner.first != nullptr, "failed to create a fresh cross-page retirement owner"); - scheduler.Current().RetainResourceUntilFence(reacquire_owner.owner); - auto reacquire_dirty = - cache.ObtainBuffer(scheduler.Current(), base + reacquire_dirty_offset, - sizeof(reacquire_value), true, false); - if (reacquire_dirty.owner != nullptr) { - scheduler.Current().RetainResourceUntilFence(reacquire_dirty.owner); - } - cache.FillBuffer(base + reacquire_dirty_offset, sizeof(reacquire_value), - reacquire_value); + (void)cache.ObtainBuffer(base + reacquire_dirty_offset, + sizeof(reacquire_value), true, false); + cache.FillBuffer(base + reacquire_dirty_offset, sizeof(reacquire_value), + reacquire_value, false); for (uint32_t tick = 0; tick <= 160; tick++) { cache.RunGarbageCollector(); } @@ -3206,12 +3182,11 @@ public: "fresh whole-owner publication completed before reacquisition"); scheduler.FinishCurrent(); auto reacquired = cache.ObtainBuffer( - scheduler.Current(), base + reacquire_disjoint_offset, + base + reacquire_disjoint_offset, sizeof(reacquire_value), true, false); Require(name, "independent disjoint post-publication reacquire", - reacquired.owner != nullptr, + reacquired.first != nullptr, "clean disjoint-half acquisition failed after publication"); - scheduler.Current().RetainResourceUntilFence(reacquired.owner); uint32_t reacquire_after = 0; Libs::LibKernel::Memory::TryReadBacking(base + reacquire_dirty_offset, &reacquire_after, @@ -3656,7 +3631,7 @@ public: }; // A formatted Buffer read must use the private - // shadPS4-shaped image-copy path. Use a request larger + // Exercise the cache-native image-copy path. Use a request larger // than the stream shortcut and poison guest backing // after upload so stale CPU staging cannot accidentally // satisfy the content check. @@ -3725,24 +3700,21 @@ public: "prefix, or transferred ownership"); auto &buffer_cache = resources.GetBufferCache(); - BufferBinding mip_formatted; + std::pair mip_formatted; bool formatted_owner_was_absent = false; gpu.SendCommandSync([&] { formatted_owner_was_absent = !buffer_cache.IsRegionRegistered( mip_desc.info.data.address, mip_desc.info.data.size); mip_formatted = buffer_cache.ObtainBuffer( - command, mip_desc.info.data.address, mip_desc.info.data.size, false, - true, true); + mip_desc.info.data.address, mip_desc.info.data.size, false, true); }); Require(name, "formatted cache fixture", formatted_owner_was_absent, "formatted image already had a cached Buffer owner"); Require(name, "formatted Buffer path", - mip_formatted.owner != nullptr && - mip_formatted.buffer != nullptr && + mip_formatted.first != nullptr && texture_cache.GetImage(mip_image).IsGpuModified(), "formatted read bypassed the cached image-copy " "path or transferred ownership"); - command.RetainResourceUntilFence(mip_formatted.owner); auto mip_prefix_readback = CreateHostBuffer( name, mip_prefix_size, vk::BufferUsageFlagBits::eTransferDst, std::vector(mip_prefix_size / sizeof(uint32_t), 0)); @@ -3753,10 +3725,10 @@ public: const vk::BufferCopy mip_prefix_copy{0, 0, mip_prefix_size}; command.Handle().copyBuffer( mip_prefix.Handle(), mip_prefix_readback.buffer, 1, &mip_prefix_copy); - TransferReadBarrier(mip_formatted.buffer, mip_guest_size); - const vk::BufferCopy mip_formatted_copy{mip_formatted.offset, 0, + TransferReadBarrier(mip_formatted.first->Handle(), mip_guest_size); + const vk::BufferCopy mip_formatted_copy{mip_formatted.second, 0, mip_guest_size}; - command.Handle().copyBuffer(mip_formatted.buffer, + command.Handle().copyBuffer(mip_formatted.first->Handle(), mip_formatted_readback.buffer, 1, &mip_formatted_copy); HostReadBarrier(mip_prefix_readback.buffer, mip_prefix_readback.size, @@ -4132,13 +4104,12 @@ public: constexpr uint64_t ms_stencil_offset = 0x80000; constexpr uint64_t ms_stencil_size = 0x10000; auto ms_stencil_owner = resources.GetBufferCache().ObtainBuffer( - command, base + ms_stencil_offset, ms_stencil_size, false, true); + base + ms_stencil_offset, ms_stencil_size, false); Require(name, "MS stencil buffer allocation", - ms_stencil_owner.owner != nullptr, + ms_stencil_owner.first != nullptr, "failed to create an unequal-sample stencil source"); - command.RetainResourceUntilFence(ms_stencil_owner.owner); - resources.GetBufferCache().FillBuffer(base + ms_stencil_offset, - ms_stencil_size, 0x41414141u); + resources.GetBufferCache().FillBuffer(base + ms_stencil_offset, + ms_stencil_size, 0x41414141u, false); auto ms_depth_desc = color_desc; ms_depth_desc.type = BindingType::DepthTarget; ms_depth_desc.info.stencil = {base + ms_stencil_offset, ms_stencil_size}; @@ -4360,20 +4331,17 @@ public: "GPU image did not accept a CPU write before Buffer mirroring"); std::memcpy(memory + 0x5000, &mirror_cpu_value, sizeof(mirror_cpu_value)); auto mirror_binding = resources.GetBufferCache().ObtainBuffer( - command, base + 0x5000, sizeof(mirror_value), false, true, true); + base + 0x5000, sizeof(mirror_value), false, true); Require(name, "CPU-dirty formatted mirror source", - mirror_binding.buffer != nullptr && + mirror_binding.first != nullptr && !texture_cache.GetImage(mirror_image).IsBufferModified(), "formatted mirror did not expose a readable Buffer source"); - if (mirror_binding.owner != nullptr) { - command.RetainResourceUntilFence(mirror_binding.owner); - } auto mirror_cpu_readback = CreateHostBuffer( name, sizeof(mirror_cpu_value), vk::BufferUsageFlagBits::eTransferDst, std::vector{0}); - const vk::BufferCopy mirror_cpu_copy{mirror_binding.offset, 0, + const vk::BufferCopy mirror_cpu_copy{mirror_binding.second, 0, sizeof(mirror_cpu_value)}; - command.Handle().copyBuffer(mirror_binding.buffer, + command.Handle().copyBuffer(mirror_binding.first->Handle(), mirror_cpu_readback.buffer, 1, &mirror_cpu_copy); HostReadBarrier(mirror_cpu_readback.buffer, mirror_cpu_readback.size, @@ -4402,17 +4370,14 @@ public: (void)texture_cache.FindTexture(exact_buffer_image, exact_buffer_desc); texture_cache.MarkGpuWritten(exact_buffer_image); auto exact_buffer_binding = resources.GetBufferCache().ObtainBuffer( - command, exact_buffer_desc.info.data.address, - exact_buffer_desc.info.data.size, false, true, true); + exact_buffer_desc.info.data.address, exact_buffer_desc.info.data.size, + false, true); Require( name, "exact replacement Buffer synchronization", - exact_buffer_binding.buffer != nullptr && + exact_buffer_binding.first != nullptr && !texture_cache.GetImage(exact_buffer_image).IsBufferModified() && texture_cache.GetImage(exact_buffer_image).IsGpuModified(), "exact replacement copy transferred cache ownership"); - if (exact_buffer_binding.owner != nullptr) { - command.RetainResourceUntilFence(exact_buffer_binding.owner); - } auto exact_float_desc = exact_buffer_desc; exact_float_desc.info.pixel_format = vk::Format::eR32Sfloat; exact_float_desc.info.guest_format = Prospero::BufferFormat::k32Float; @@ -4458,14 +4423,13 @@ public: {1, 1, 1}, 1, 4, 1); const auto clear_image = texture_cache.FindImage(clear_desc); auto buffer_write = resources.GetBufferCache().ObtainBuffer( - command, base + 0x6000, sizeof(clear_source), true, false, true); + base + 0x6000, sizeof(clear_source), true, true); + texture_cache.InvalidateMemoryFromGPU(base + 0x6000, + sizeof(clear_source)); Require(name, "buffer-write ownership", - buffer_write.buffer != nullptr && + buffer_write.first != nullptr && texture_cache.GetImage(clear_image).IsBufferModified(), "formatted buffer write did not supersede the image"); - if (buffer_write.owner != nullptr) { - command.RetainResourceUntilFence(buffer_write.owner); - } Require(name, "buffer-owned full image clear", texture_cache.ClearImageFromBuffer( command, base + 0x6000, sizeof(clear_source), 0xaabbccddu) && @@ -4488,18 +4452,16 @@ public: std::memcpy(memory + partial_clean_offset, &partial_clean_value, sizeof(partial_clean_value)); auto partial_write = resources.GetBufferCache().ObtainBuffer( - command, base + partial_image_offset, sizeof(partial_image_value), - true, false); + base + partial_image_offset, sizeof(partial_image_value), true); Require(name, "partial-page buffer allocation", - partial_write.owner != nullptr, + partial_write.first != nullptr, "partial-page GPU write did not create a native buffer"); - command.RetainResourceUntilFence(partial_write.owner); - resources.GetBufferCache().FillBuffer(base + partial_image_offset, - sizeof(partial_image_value), - partial_image_value); - resources.GetBufferCache().FillBuffer(base + partial_buffer_offset, - sizeof(partial_buffer_value), - partial_buffer_value); + resources.GetBufferCache().FillBuffer(base + partial_image_offset, + sizeof(partial_image_value), + partial_image_value, false); + resources.GetBufferCache().FillBuffer(base + partial_buffer_offset, + sizeof(partial_buffer_value), + partial_buffer_value, false); auto partial_desc = MakeLinearDesc( base + partial_image_offset, sizeof(partial_image_value), vk::Format::eR32Uint, Prospero::BufferFormat::k32UInt, @@ -4509,19 +4471,16 @@ public: !texture_cache.GetImage(partial_image).IsGpuModified(), "image upload incorrectly consumed buffer dirty ownership"); auto partial_image_mirror = resources.GetBufferCache().ObtainBuffer( - command, base + partial_image_offset, sizeof(partial_image_value), - false, true, true); + base + partial_image_offset, sizeof(partial_image_value), false, true); Require(name, "partial-page image mirror", - partial_image_mirror.buffer != nullptr && - partial_image_mirror.owner != nullptr && + partial_image_mirror.first != nullptr && !texture_cache.GetImage(partial_image).IsGpuModified(), "same-page image upload lost its buffer source"); - command.RetainResourceUntilFence(partial_image_mirror.owner); const auto partial_clean_source = resources.GetBufferCache().ObtainBufferForImage( base + partial_clean_offset, sizeof(partial_clean_value)); Require(name, "partial-page clean source", - partial_clean_source.buffer != nullptr, + partial_clean_source.first != nullptr, "clean same-page bytes inherited unrelated buffer ownership"); Require(name, "partial-page remaining fault", resources.HandleFault(PageFaultAccess::Read, @@ -4548,16 +4507,16 @@ public: resources.GetBufferCache().ObtainBufferForImage( base + partial_clean_offset, sizeof(partial_cpu_refresh_value)); Require(name, "partial-page CPU refresh source", - partial_cpu_refresh_source.buffer != nullptr, + partial_cpu_refresh_source.first != nullptr, "CPU-dirty same-page bytes did not resolve through the cached " "buffer"); auto partial_cpu_refresh_readback = CreateHostBuffer(name, sizeof(partial_cpu_refresh_value), vk::BufferUsageFlagBits::eTransferDst, {0}); const vk::BufferCopy partial_cpu_refresh_copy{ - partial_cpu_refresh_source.offset, 0, + partial_cpu_refresh_source.second, 0, sizeof(partial_cpu_refresh_value)}; - command.Handle().copyBuffer(partial_cpu_refresh_source.buffer->Handle(), + command.Handle().copyBuffer(partial_cpu_refresh_source.first->Handle(), partial_cpu_refresh_readback.buffer, 1, &partial_cpu_refresh_copy); vk::BufferMemoryBarrier partial_cpu_refresh_barrier{}; @@ -4623,17 +4582,14 @@ public: (void)texture_cache.FindTexture(retracked_a, fault_a_desc); (void)texture_cache.FindTexture(retracked_b, fault_b_desc); auto fault_a_mirror = resources.GetBufferCache().ObtainBuffer( - command, base + 0x8000, sizeof(fault_a), false, true, true); - if (fault_a_mirror.owner != nullptr) { - command.RetainResourceUntilFence(fault_a_mirror.owner); - } + base + 0x8000, sizeof(fault_a), false, true); Require(name, "same-page image re-track", retracked_a == fault_a_image && retracked_b == fault_b_image && texture_cache.GetImage(fault_a_image).IsTracked() && texture_cache.GetImage(fault_b_image).IsTracked() && !texture_cache.GetImage(fault_a_image).IsCpuDirty() && !texture_cache.GetImage(fault_b_image).IsCpuDirty() && - fault_a_mirror.buffer != nullptr && + fault_a_mirror.first != nullptr && texture_cache.GetImage(fault_a_image).IsTracked() && texture_cache.GetImage(fault_b_image).IsTracked() && texture_cache.GetImage(fault_a_image).IsGpuModified() && @@ -4668,9 +4624,9 @@ public: const auto publish_image = texture_cache.FindImage(publish_image_desc); (void)texture_cache.FindTexture(publish_image, publish_image_desc); texture_cache.MarkGpuWritten(publish_image); - resources.GetBufferCache().FillBuffer(base + publish_buffer_offset, - sizeof(publish_buffer_value), - publish_buffer_value); + resources.GetBufferCache().FillBuffer(base + publish_buffer_offset, + sizeof(publish_buffer_value), + publish_buffer_value, false); auto publish_replacement_desc = publish_image_desc; publish_replacement_desc.info.pixel_format = vk::Format::eR32Sfloat; publish_replacement_desc.info.guest_format = @@ -4797,8 +4753,8 @@ public: texture_cache.TouchMeta(base + metadata_b, 0, false) && !texture_cache.IsMetaCleared(base + metadata_b, 0), "per-slice metadata update incorrectly required prior GPU ownership"); - resources.GetBufferCache().FillBuffer(base + metadata_b, sizeof(uint32_t), - 0); + resources.GetBufferCache().FillBuffer(base + metadata_b, sizeof(uint32_t), + 0, false); Require( name, "metadata buffer fill state", texture_cache.IsMetaCleared(base + metadata_b, 0), @@ -4829,19 +4785,16 @@ public: unformatted_alias); texture_cache.MarkGpuWritten(unformatted_alias_image); auto unformatted_alias_buffer = resources.GetBufferCache().ObtainBuffer( - command, unformatted_alias.info.data.address, - unformatted_alias.info.data.size, false, true, false); + unformatted_alias.info.data.address, unformatted_alias.info.data.size, + false); Require( name, "unformatted buffer/image alias", - unformatted_alias_buffer.buffer != nullptr && + unformatted_alias_buffer.first != nullptr && texture_cache.GetImage(unformatted_alias_image).IsGpuModified() && !texture_cache.GetImage(unformatted_alias_image) .IsBufferModified(), "read-only unformatted buffer alias did not follow ordinary " "BufferCache acquisition"); - if (unformatted_alias_buffer.owner != nullptr) { - command.RetainResourceUntilFence(unformatted_alias_buffer.owner); - } constexpr uint64_t layered_offset = 0x14000; constexpr uint64_t layered_guest_size = 0x800; @@ -5012,20 +4965,19 @@ public: constexpr uint32_t mixed_cpu_value = 0x1234abcdu; constexpr uint32_t mixed_gpu_value = 0x9876fedcu; auto mixed_owner = resources.GetBufferCache().ObtainBuffer( - command, base + mixed_source_offset, mixed_source_size, true, true); + base + mixed_source_offset, mixed_source_size, true); Require(name, "mixed-page source allocation", - mixed_owner.owner != nullptr, + mixed_owner.first != nullptr, "mixed CPU/GPU image source did not create a containing buffer"); - command.RetainResourceUntilFence(mixed_owner.owner); Require(name, "mixed-page CPU write fault", resources.HandleFault(PageFaultAccess::Write, base + mixed_source_offset), "mixed image source could not dirty its first page"); std::memcpy(memory + mixed_source_offset, &mixed_cpu_value, sizeof(mixed_cpu_value)); - resources.GetBufferCache().FillBuffer(base + mixed_source_offset + 0x1000, - sizeof(mixed_gpu_value), - mixed_gpu_value); + resources.GetBufferCache().FillBuffer(base + mixed_source_offset + 0x1000, + sizeof(mixed_gpu_value), + mixed_gpu_value, false); auto mixed_source_desc = MakeLinearDesc( base + mixed_source_offset, mixed_source_size, vk::Format::eR32Uint, Prospero::BufferFormat::k32UInt, Prospero::ImageType::kColor2D, @@ -5073,13 +5025,12 @@ public: std::memcpy(memory + bridged_cpu_offset, bridged_cpu_values.data(), sizeof(bridged_cpu_values)); auto bridged_cpu_owner = resources.GetBufferCache().ObtainBuffer( - command, base + bridged_cpu_offset, bridged_texel_size, true, false); + base + bridged_cpu_offset, bridged_texel_size, true); Require(name, "bridged CPU owner allocation", - bridged_cpu_owner.owner != nullptr, + bridged_cpu_owner.first != nullptr, "bridged image source did not create its CPU-side cache owner"); - command.RetainResourceUntilFence(bridged_cpu_owner.owner); - resources.GetBufferCache().FillBuffer(base + bridged_cpu_offset, - bridged_texel_size, 0); + resources.GetBufferCache().FillBuffer(base + bridged_cpu_offset, + bridged_texel_size, 0, false); Require( name, "bridged CPU write fault", resources.HandleFault(PageFaultAccess::Write, @@ -5095,15 +5046,13 @@ public: sizeof(bridged_gpu_guest_values)); auto bridged_gpu_owner = resources.GetBufferCache().ObtainBuffer( - scheduler.Current(), base + bridged_gpu_offset, bridged_texel_size, - true, false); + base + bridged_gpu_offset, bridged_texel_size, true); Require(name, "bridged GPU owner allocation", - bridged_gpu_owner.owner != nullptr && - bridged_gpu_owner.owner != bridged_cpu_owner.owner, + bridged_gpu_owner.first != nullptr && + bridged_gpu_owner.first != bridged_cpu_owner.first, "bridged image source did not retain two disjoint cache owners"); - scheduler.Current().RetainResourceUntilFence(bridged_gpu_owner.owner); - resources.GetBufferCache().FillBuffer( - base + bridged_gpu_offset, bridged_texel_size, bridged_gpu_value); + resources.GetBufferCache().FillBuffer( + base + bridged_gpu_offset, bridged_texel_size, bridged_gpu_value, false); Require(name, "bridged ownership split", resources.GetBufferCache().HasGpuDirtyBytes( base + bridged_gpu_offset, bridged_texel_size) && @@ -5161,16 +5110,13 @@ public: (void)texture_cache.FindTexture(byte_mirror_image, byte_mirror_desc); texture_cache.MarkGpuWritten(byte_mirror_image); auto byte_mirror = resources.GetBufferCache().ObtainBuffer( - command, base + byte_mirror_offset, 1, false, true, true); + base + byte_mirror_offset, 1, false, true); Require( name, "byte image mirror", - byte_mirror.buffer != nullptr && + byte_mirror.first != nullptr && !texture_cache.GetImage(byte_mirror_image).IsBufferModified() && texture_cache.GetImage(byte_mirror_image).IsGpuModified(), "one-byte image copy transferred cache ownership"); - if (byte_mirror.owner != nullptr) { - command.RetainResourceUntilFence(byte_mirror.owner); - } const std::array bgra16_guest{0x3c00u, 0x4000u, 0x4200u, 0x4400u}; @@ -5508,15 +5454,13 @@ public: constexpr uint64_t dirty_sibling_offset = 0x334200; constexpr uint32_t dirty_sibling_value = 0xc001d00du; auto dirty_sibling = resources.GetBufferCache().ObtainBuffer( - scheduler.Current(), base + dirty_sibling_offset, - sizeof(dirty_sibling_value), true, false); + base + dirty_sibling_offset, sizeof(dirty_sibling_value), true); Require(name, "same-page dirty sibling allocation", - dirty_sibling.owner != nullptr, + dirty_sibling.first != nullptr, "failed to create the disjoint same-page Buffer owner"); - scheduler.Current().RetainResourceUntilFence(dirty_sibling.owner); - resources.GetBufferCache().FillBuffer(base + dirty_sibling_offset, - sizeof(dirty_sibling_value), - dirty_sibling_value); + resources.GetBufferCache().FillBuffer(base + dirty_sibling_offset, + sizeof(dirty_sibling_value), + dirty_sibling_value, false); auto exact_image_desc = MakeLinearDesc(base + exact_image_offset, sizeof(uint32_t), vk::Format::eR32Uint, Prospero::BufferFormat::k32UInt, @@ -5551,13 +5495,10 @@ public: auto gc_image_desc_b = gc_image_desc_a; gc_image_desc_b.info.data.address = base + gc_image_offsets[1]; auto clean_buffer_alias = resources.GetBufferCache().ObtainBuffer( - scheduler.Current(), gc_image_desc_a.info.data.address, - gc_image_desc_a.info.data.size, true, false); + gc_image_desc_a.info.data.address, gc_image_desc_a.info.data.size, true); Require(name, "clean pre-image buffer alias", - clean_buffer_alias.owner != nullptr && - clean_buffer_alias.buffer != nullptr, + clean_buffer_alias.first != nullptr, "failed to create the clean cached Buffer alias"); - scheduler.Current().RetainResourceUntilFence(clean_buffer_alias.owner); resources.GetBufferCache().ReadMemory(gc_image_desc_a.info.data.address, gc_image_desc_a.info.data.size); const std::array gc_images{texture_cache.FindImage(gc_image_desc_a), @@ -5625,15 +5566,11 @@ public: scheduler.FinishCurrent(); scheduler.DrainPriorityOperations(); auto refreshed_buffer_alias = resources.GetBufferCache().ObtainBuffer( - scheduler.Current(), gc_image_desc_a.info.data.address, - gc_image_desc_a.info.data.size, false, true); + gc_image_desc_a.info.data.address, gc_image_desc_a.info.data.size, + false); Require(name, "post-publication Buffer reacquire", - refreshed_buffer_alias.buffer != nullptr, + refreshed_buffer_alias.first != nullptr, "Buffer lookup failed after the retired image publication"); - if (refreshed_buffer_alias.owner != nullptr) { - scheduler.Current().RetainResourceUntilFence( - refreshed_buffer_alias.owner); - } std::array gc_after_completion{}; for (size_t index = 0; index < gc_image_offsets.size(); index++) { Libs::LibKernel::Memory::TryReadBacking(base + gc_image_offsets[index], @@ -5645,19 +5582,15 @@ public: gc_after_completion == gc_image_values, "one submission did not publish both deferred image readbacks"); Require(name, "refreshed post-image buffer alias", - refreshed_buffer_alias.buffer != nullptr, + refreshed_buffer_alias.first != nullptr, "failed to reacquire the cached Buffer alias after image " "publication"); - if (refreshed_buffer_alias.owner != nullptr) { - scheduler.Current().RetainResourceUntilFence( - refreshed_buffer_alias.owner); - } auto alias_readback = CreateHostBuffer( name, sizeof(uint32_t), vk::BufferUsageFlagBits::eTransferDst, {0}); - const vk::BufferCopy alias_copy{refreshed_buffer_alias.offset, 0, + const vk::BufferCopy alias_copy{refreshed_buffer_alias.second, 0, sizeof(uint32_t)}; scheduler.Current().Handle().copyBuffer( - refreshed_buffer_alias.buffer, alias_readback.buffer, 1, &alias_copy); + refreshed_buffer_alias.first->Handle(), alias_readback.buffer, 1, &alias_copy); HostReadBarrier(alias_readback.buffer, alias_readback.size, vk::PipelineStageFlagBits::eTransfer, vk::AccessFlagBits::eTransferWrite); @@ -6437,15 +6370,14 @@ public: TextureCacheTestAccess::TryDownload(cache, id), "tiled BGRA16 guest readback was rejected"); auto mirror = resources.GetBufferCache().ObtainBuffer( - scheduler.Current(), base, total.size, false, true, true); + base, total.size, false, true); Require(name, "mirror owner", - mirror.buffer != nullptr && mirror.owner != nullptr, + mirror.first != nullptr, "tiled BGRA16 mirror has no BufferCache owner"); - scheduler.Current().RetainResourceUntilFence(mirror.owner); auto mirror_readback = CreateHostBuffer( name, 8, vk::BufferUsageFlagBits::eTransferDst, {0, 0}); - const vk::BufferCopy copy{mirror.offset, 0, 8}; - scheduler.Current().Handle().copyBuffer(mirror.buffer, + const vk::BufferCopy copy{mirror.second, 0, 8}; + scheduler.Current().Handle().copyBuffer(mirror.first->Handle(), mirror_readback.buffer, 1, ©); vk::BufferMemoryBarrier barrier{}; barrier.sType = vk::StructureType::eBufferMemoryBarrier; @@ -6638,15 +6570,14 @@ public: TextureCacheTestAccess::TryDownload(texture_cache, storage_id), "the 3D render target could not be queued for guest-layout readback"); auto mirror = resources.GetBufferCache().ObtainBuffer( - scheduler.Current(), base, allocation_size, false, true, true); + base, allocation_size, false, true); Require(name, "volume mirror", - mirror.buffer != nullptr && mirror.owner != nullptr, + mirror.first != nullptr, "the 3D render-target readback has no BufferCache owner"); - scheduler.Current().RetainResourceUntilFence(mirror.owner); auto slice_probe = CreateHostBuffer(name, 4, vk::BufferUsageFlagBits::eTransferDst, {0}); - const vk::BufferCopy copy{mirror.offset + 31 * slice_size, 0, 4}; - scheduler.Current().Handle().copyBuffer(mirror.buffer, slice_probe.buffer, + const vk::BufferCopy copy{mirror.second + 31 * slice_size, 0, 4}; + scheduler.Current().Handle().copyBuffer(mirror.first->Handle(), slice_probe.buffer, 1, ©); vk::BufferMemoryBarrier barrier{}; barrier.sType = vk::StructureType::eBufferMemoryBarrier; @@ -6997,9 +6928,9 @@ public: std::make_shared( std::move(null_snapshot))}; auto null_bindings = executor.PrepareBindings( - scheduler.Current(), null_runtime, vk::ShaderStageFlagBits::eCompute, + null_runtime, vk::ShaderStageFlagBits::eCompute, DescriptorCache::Stage::Compute); - executor.RebindImages(scheduler.Current(), null_bindings); + executor.RebindImages(null_bindings); Require(name, "null descriptor count", null_bindings.resources.images.size() == 3, "null descriptor preparation lost an image binding"); @@ -7161,7 +7092,7 @@ public: std::move(overwide_mipped_storage_binding)); mipped_prepared.resources.images.push_back( std::move(sampled_overwide_resolved)); - executor.RebindImages(scheduler.Current(), mipped_prepared); + executor.RebindImages(mipped_prepared); auto &plain_mipped_binding = mipped_prepared.resources.images[0]; auto &mipped_binding = mipped_prepared.resources.images[1]; auto &overwide_mipped_binding = mipped_prepared.resources.images[2]; @@ -7345,8 +7276,8 @@ public: std::make_shared( std::move(storage_snapshot))}; auto storage_discovery = executor.PrepareBindings( - scheduler.Current(), storage_runtime, - vk::ShaderStageFlagBits::eVertex, DescriptorCache::Stage::Vertex); + storage_runtime, vk::ShaderStageFlagBits::eVertex, + DescriptorCache::Stage::Vertex); const auto storage_id = storage_discovery.resources.images[0].image_id; Require(name, "storage prefetch purity", storage_discovery.resources.images[0].image_view == nullptr && @@ -7399,8 +7330,7 @@ public: std::make_shared( std::move(ordered_snapshot))}; auto ordered_bindings = RenderExecutorTestAccess::PrepareGraphicsBindings( - executor, scheduler.Current(), storage_runtime, - ordered_sampled_runtime, true); + executor, storage_runtime, ordered_sampled_runtime, true); const auto ordered_sampled_id = ordered_bindings.pixel->resources.images[0].image_id; Require(name, "VS-before-PS retained-owner order", @@ -7408,18 +7338,14 @@ public: storage_id && ordered_sampled_id != storage_id && RenderExecutorTestAccess::BoundImagesInOrder( - executor, - TextureCacheTestAccess::Owner(texture_cache, storage_id), - TextureCacheTestAccess::Owner(texture_cache, - ordered_sampled_id)), + executor, storage_id, ordered_sampled_id), "production graphics binding did not retain vertex resources " "before pixel resources"); RenderExecutorTestAccess::ResetBindings(executor); auto graphics_bindings = RenderExecutorTestAccess::PrepareGraphicsBindings( - executor, scheduler.Current(), storage_runtime, sampled_runtime, - true); + executor, storage_runtime, sampled_runtime, true); const auto &storage_binding = graphics_bindings.vertex.resources.images[0]; const auto &sampled_binding = @@ -7469,8 +7395,7 @@ public: storage_runtime.resources}; auto writable_alias_bindings = RenderExecutorTestAccess::PrepareGraphicsBindings( - executor, scheduler.Current(), vertex_sampled_runtime, - pixel_storage_runtime, true); + executor, vertex_sampled_runtime, pixel_storage_runtime, true); Require(name, "late writable alias promotion", texture_cache.GetImage(storage_id).binding.force_general, "an already sampled image was not promoted when a later " @@ -7941,9 +7866,9 @@ public: std::move(array_snapshot))}; auto array_binding = executor.PrepareBindings( - scheduler.Current(), array_runtime, vk::ShaderStageFlagBits::eCompute, + array_runtime, vk::ShaderStageFlagBits::eCompute, DescriptorCache::Stage::Compute); - executor.RebindImages(scheduler.Current(), array_binding); + executor.RebindImages(array_binding); const auto expanded_array_id = array_binding.resources.images[0].image_id; const auto &expanded_array = texture_cache.GetImage(expanded_array_id); Require( @@ -8016,9 +7941,9 @@ public: std::move(colliding_msaa_program), std::move(colliding_msaa_snapshot)}; auto colliding_msaa_binding = executor.PrepareBindings( - scheduler.Current(), colliding_msaa_runtime, - vk::ShaderStageFlagBits::eCompute, DescriptorCache::Stage::Compute); - executor.RebindImages(scheduler.Current(), colliding_msaa_binding); + colliding_msaa_runtime, vk::ShaderStageFlagBits::eCompute, + DescriptorCache::Stage::Compute); + executor.RebindImages(colliding_msaa_binding); const auto &resolved_colliding_msaa = colliding_msaa_binding.resources.images[0]; Require( @@ -8076,9 +8001,9 @@ public: ShaderStageRuntime msaa_runtime{std::move(msaa_program), std::move(msaa_snapshot)}; auto msaa_binding = executor.PrepareBindings( - scheduler.Current(), msaa_runtime, vk::ShaderStageFlagBits::eCompute, + msaa_runtime, vk::ShaderStageFlagBits::eCompute, DescriptorCache::Stage::Compute); - executor.RebindImages(scheduler.Current(), msaa_binding); + executor.RebindImages(msaa_binding); const auto &resolved_msaa = msaa_binding.resources.images[0]; Require( name, "MSAA descriptor backing reuse", @@ -8117,9 +8042,9 @@ public: ShaderStageRuntime msaa_array_runtime{std::move(msaa_array_program), std::move(msaa_array_snapshot)}; auto msaa_array_binding = executor.PrepareBindings( - scheduler.Current(), msaa_array_runtime, - vk::ShaderStageFlagBits::eCompute, DescriptorCache::Stage::Compute); - executor.RebindImages(scheduler.Current(), msaa_array_binding); + msaa_array_runtime, vk::ShaderStageFlagBits::eCompute, + DescriptorCache::Stage::Compute); + executor.RebindImages(msaa_array_binding); const auto &resolved_msaa_array = msaa_array_binding.resources.images[0]; Require(name, "MSAA array backing expansion", resolved_msaa_array.image_id != msaa_target_id && @@ -8190,10 +8115,8 @@ public: make_target_desc(base + target_mip_size, target_mip_size, {1, 1, 1}); const auto target_subresource_id = texture_cache.FindImage(target_subresource); - auto target_subresource_owner = + const auto *target_subresource_owner = TextureCacheTestAccess::Owner(texture_cache, target_subresource_id); - std::weak_ptr retired_target = - target_subresource_owner; const auto &discovered_target = texture_cache.GetImage(target_subresource_id); Require(name, "cache-only target discovery", @@ -8225,8 +8148,6 @@ public: target_parent_id), "target overlap did not transfer target state to the merged " "owner"); - target_subresource_owner.reset(); - RenderColorInfo rebound_color{}; rebound_color.type = RenderColorType::RenderTexture; rebound_color.desc = target_subresource; @@ -8242,14 +8163,11 @@ public: rebound_depth.samples = 1; rebound_depth.image_id = depth_id; scheduler.FinishCurrent(); - auto retired_owner = retired_target.lock(); Require( name, "deferred target slot erasure", TextureCacheTestAccess::Owner(texture_cache, target_subresource_id) == - nullptr && - retired_owner != nullptr && retired_owner->binding.needs_rebind, - "the displaced target did not survive only through RenderExecutor " - "ownership"); + nullptr, + "the displaced target slot was not erased after GPU completion"); auto rebound_rendering = RenderExecutorTestAccess::AcquireRenderTargets( executor, scheduler.Current(), &rebound_color, 1, rebound_depth); const auto stencil_proxy_id = texture_cache.FindImageFromRange( @@ -8280,15 +8198,11 @@ public: scheduler.EndRendering(); RenderExecutorTestAccess::ResetBindings(executor); Require(name, "draw-scoped target reset", - !retired_owner->binding.is_target && - !retired_owner->binding.needs_rebind && + TextureCacheTestAccess::Owner(texture_cache, + target_subresource_id) == nullptr && !texture_cache.GetImage(target_parent_id).binding.is_target && !texture_cache.GetImage(depth_id).binding.is_target, "RenderExecutor retained target binding state after reset"); - retired_owner.reset(); - Require(name, "RenderExecutor-owned retired lifetime", - retired_target.expired(), - "a retired target owner survived after RenderExecutor reset"); constexpr uint64_t ordered_color_address = base + 0x80000; auto ordered_color_desc = @@ -8368,7 +8282,7 @@ public: std::move(snapshot))}; auto prepared = context.GetRenderExecutor().PrepareBindings( - scheduler.Current(), runtime, vk::ShaderStageFlagBits::eCompute, + runtime, vk::ShaderStageFlagBits::eCompute, DescriptorCache::Stage::Compute); const auto sampled_stencil_id = prepared.resources.images[0].image_id; Require(name, "first stencil discovery", @@ -8381,7 +8295,7 @@ public: "the first sampled-stencil lookup did not remain an ordinary " "discovery before final depth acquisition"); - context.GetRenderExecutor().RebindImages(scheduler.Current(), prepared); + context.GetRenderExecutor().RebindImages(prepared); Require(name, "first stencil acquisition", prepared.resources.images[0].image_id == sampled_stencil_id && prepared.resources.images[0].image_view != nullptr && @@ -8403,7 +8317,7 @@ public: RenderExecutorTestAccess::ResetBindings(executor); auto redirected = context.GetRenderExecutor().PrepareBindings( - scheduler.Current(), runtime, vk::ShaderStageFlagBits::eCompute, + runtime, vk::ShaderStageFlagBits::eCompute, DescriptorCache::Stage::Compute); Require(name, "redirected owner discovery", redirected.resources.images.size() == 1 && @@ -8413,7 +8327,7 @@ public: !texture_cache.GetImage(sampled_stencil_id).binding.is_bound, "the established stencil association did not redirect the next " "discovery to the depth owner"); - context.GetRenderExecutor().RebindImages(scheduler.Current(), redirected); + context.GetRenderExecutor().RebindImages(redirected); Require(name, "second-pass stencil acquisition", redirected.resources.images[0].image_id == depth_id && redirected.resources.images[0].image_view != nullptr && @@ -8441,9 +8355,9 @@ public: std::make_shared( std::move(stencil_storage_snapshot))}; auto storage_redirected = executor.PrepareBindings( - scheduler.Current(), stencil_storage_runtime, - vk::ShaderStageFlagBits::eCompute, DescriptorCache::Stage::Compute); - executor.RebindImages(scheduler.Current(), storage_redirected); + stencil_storage_runtime, vk::ShaderStageFlagBits::eCompute, + DescriptorCache::Stage::Compute); + executor.RebindImages(storage_redirected); Require( name, "storage stencil acquisition", storage_redirected.resources.images[0].image_id == depth_id && @@ -22492,38 +22406,35 @@ void CheckDepthTargetFootprints() { std::printf("[host] %-32s ok\n", "DepthTargetFootprints"); } -struct FenceLifetimeProbe { - explicit FenceLifetimeProbe(bool *destroyed) : destroyed(destroyed) { +struct SlotLifetimeProbe { + explicit SlotLifetimeProbe(bool *destroyed) : destroyed(destroyed) { if (destroyed == nullptr || *destroyed) { EXIT("fence-lifetime probe has invalid construction state\n"); } } - ~FenceLifetimeProbe() { *destroyed = true; } + ~SlotLifetimeProbe() { *destroyed = true; } bool *destroyed = nullptr; }; -void CheckSharedFenceResourceLifetime() { +void CheckSlotVectorLifetime() { bool destroyed = false; - auto image = std::make_shared(&destroyed); - FenceResourceRetainer first; - FenceResourceRetainer second; - first.Retain(image); - second.Retain(image); - first.Retain(image); - image.reset(); - Require("SharedFenceResourceLifetime", "retained", - !destroyed && !first.Empty() && !second.Empty(), - "cache removal destroyed an image retained by command buffers"); - first.ReleaseAfterFence(); - Require("SharedFenceResourceLifetime", "first fence", - !destroyed && first.Empty() && !second.Empty(), - "first command-buffer fence destroyed another buffer's image"); - second.ReleaseAfterFence(); - Require("SharedFenceResourceLifetime", "last fence", - destroyed && second.Empty(), - "last referencing command-buffer fence did not destroy the image"); - std::printf("[host] %-32s ok\n", "SharedFenceResourceLifetime"); + Common::SlotVector slots; + const auto first = slots.insert(&destroyed); + Require("SlotVectorLifetime", "inserted", + slots.is_allocated(first) && slots.try_get(first) != nullptr && + !destroyed, + "slot insertion did not expose its cache-owned object"); + slots.erase(first); + const bool first_destroyed = destroyed; + destroyed = false; + const auto second = slots.insert(&destroyed); + Require("SlotVectorLifetime", "generation", + first_destroyed && !destroyed && first.index == second.index && + first != second && + !slots.is_allocated(first) && slots.is_allocated(second), + "reused slot accepted a stale cache resource id"); + std::printf("[host] %-32s ok\n", "SlotVectorLifetime"); } void CheckEmbeddedFetchLaneSpill() { @@ -23563,7 +23474,7 @@ int main(int argc, char **argv) { CheckDepthAttachmentWrites(); CheckDynamicRenderingState(); CheckDepthTargetFootprints(); - CheckSharedFenceResourceLifetime(); + CheckSlotVectorLifetime(); #else if (argc != 1) { std::fprintf(stderr, "unknown test selector: %s\n", argv[1]);