diff --git a/src/graphics/host_gpu/graphicContext.h b/src/graphics/host_gpu/graphicContext.h index 7d19dbecc..8e7a41877 100644 --- a/src/graphics/host_gpu/graphicContext.h +++ b/src/graphics/host_gpu/graphicContext.h @@ -49,14 +49,15 @@ struct GraphicContext: public VulkanInstance { }; struct VulkanMemory { - vk::MemoryRequirements requirements = {}; - vk::MemoryPropertyFlags property = {}; - vk::DeviceMemory memory = nullptr; - VmaAllocation allocation = nullptr; - VmaAllocationInfo allocation_info = {}; - vk::DeviceSize offset = 0; - uint32_t type = 0; - uint64_t unique_id = 0; + vk::MemoryRequirements requirements = {}; + vk::MemoryPropertyFlags property = {}; + vk::MemoryPropertyFlags preferred_property = {}; + vk::DeviceMemory memory = nullptr; + VmaAllocation allocation = nullptr; + VmaAllocationInfo allocation_info = {}; + vk::DeviceSize offset = 0; + uint32_t type = 0; + uint64_t unique_id = 0; }; enum class VulkanImageType { diff --git a/src/graphics/host_gpu/renderer/textureCache.cpp b/src/graphics/host_gpu/renderer/textureCache.cpp index ee24f10d2..cdc216127 100644 --- a/src/graphics/host_gpu/renderer/textureCache.cpp +++ b/src/graphics/host_gpu/renderer/textureCache.cpp @@ -3015,11 +3015,9 @@ void TextureCache::SynchronizeColorImageToBufferLocked(CachedImage& cached, uint } // This is the CPU Tiler backend for the image-to-buffer synchronization seam. - // The vectors and Vulkan staging allocation retain capacity; a future PS5 GPU tiler can replace - // this block without changing alias classification or ownership transitions. + // The guest vector and Vulkan staging allocation retain capacity; a future PS5 GPU tiler can + // replace this block without changing alias classification or ownership transitions. Transfer::WaitForGraphicsIdle(cached.ctx); - m_buffer_transition_linear.resize(target.size); - std::fill(m_buffer_transition_linear.begin(), m_buffer_transition_linear.end(), 0); std::vector regions; if (storage) { auto layout = TextureCalcUploadLayout( @@ -3040,24 +3038,23 @@ void TextureCache::SynchronizeColorImageToBufferLocked(CachedImage& cached, uint regions = Transfer::MakeLayeredImageBufferCopies(target.layers, slice_size, target.pitch, target.width, target.height); } - Transfer::DownloadImage(cached.ctx, m_buffer_transition_linear.data(), target.size, regions, - cached.image, cached.image->layout); - if (storage) { - m_buffer_transition_guest.resize(target.size); - m_tiler.TileImage(m_buffer_transition_guest.data(), m_buffer_transition_linear.data(), - cached.info); - Libs::LibKernel::Memory::WriteBacking(target.address, m_buffer_transition_guest.data(), - target.size); - } else if (tiled) { - m_buffer_transition_guest.resize(target.size); - m_tiler.TileImage(m_buffer_transition_guest.data(), m_buffer_transition_linear.data(), - target); - Libs::LibKernel::Memory::WriteBacking(target.address, m_buffer_transition_guest.data(), - target.size); - } else { - Libs::LibKernel::Memory::WriteBacking(target.address, m_buffer_transition_linear.data(), - target.size); - } + Transfer::ProcessDownloadedImage( + cached.ctx, target.size, regions, cached.image, cached.image->layout, + [&](std::span linear) { + if (storage) { + m_buffer_transition_guest.resize(target.size); + m_tiler.TileImage(m_buffer_transition_guest.data(), linear.data(), cached.info); + Libs::LibKernel::Memory::WriteBacking( + target.address, m_buffer_transition_guest.data(), target.size); + } else if (tiled) { + m_buffer_transition_guest.resize(target.size); + m_tiler.TileImage(m_buffer_transition_guest.data(), linear.data(), target); + Libs::LibKernel::Memory::WriteBacking( + target.address, m_buffer_transition_guest.data(), target.size); + } else { + Libs::LibKernel::Memory::WriteBacking(target.address, linear.data(), target.size); + } + }); m_memory_tracker.ForEachDownloadRange(target.address, target.size, [](uint64_t, uint64_t) noexcept {}); // Only the impending buffer-write range needs publication into BufferCache ownership. The @@ -3104,16 +3101,16 @@ void TextureCache::SynchronizeDepthImageToBufferLocked(CachedImage& cached, uint has_stencil, has_htile, cached.gpu_modified, cached.buffer_modified); } Transfer::WaitForGraphicsIdle(cached.ctx); - m_buffer_transition_linear.resize(info.size); - std::fill(m_buffer_transition_linear.begin(), m_buffer_transition_linear.end(), 0); const auto regions = Transfer::MakeLayeredImageBufferCopies( 1, info.size, info.pitch, info.width, info.height, vk::ImageAspectFlagBits::eDepth); - Transfer::DownloadImage(cached.ctx, m_buffer_transition_linear.data(), info.size, regions, - cached.image, cached.image->layout); - m_buffer_transition_guest.resize(info.size); - m_tiler.TileImage(m_buffer_transition_guest.data(), m_buffer_transition_linear.data(), info); - Libs::LibKernel::Memory::WriteBacking(info.address, m_buffer_transition_guest.data(), - info.size); + Transfer::ProcessDownloadedImage( + cached.ctx, info.size, regions, cached.image, cached.image->layout, + [&](std::span linear) { + m_buffer_transition_guest.resize(info.size); + m_tiler.TileImage(m_buffer_transition_guest.data(), linear.data(), info); + Libs::LibKernel::Memory::WriteBacking(info.address, m_buffer_transition_guest.data(), + info.size); + }); m_memory_tracker.ForEachDownloadRange(info.address, info.size, [](uint64_t, uint64_t) noexcept {}); m_buffer_cache.PublishImageBacking(write_address, write_size); diff --git a/src/graphics/host_gpu/renderer/textureCache.h b/src/graphics/host_gpu/renderer/textureCache.h index bd1e1c848..d9d7c2d47 100644 --- a/src/graphics/host_gpu/renderer/textureCache.h +++ b/src/graphics/host_gpu/renderer/textureCache.h @@ -176,7 +176,6 @@ private: ImageOwnerIndex m_image_owner_index; std::map m_surface_metas; std::unique_ptr m_readback; - std::vector m_buffer_transition_linear; std::vector m_buffer_transition_guest; }; diff --git a/src/graphics/host_gpu/transfer.cpp b/src/graphics/host_gpu/transfer.cpp index 0e39b050f..948f3e395 100644 --- a/src/graphics/host_gpu/transfer.cpp +++ b/src/graphics/host_gpu/transfer.cpp @@ -351,12 +351,28 @@ public: void DownloadFromImage(GraphicContext* ctx, void* dst_data, uint64_t size, std::span regions, VulkanImage* src_image, vk::ImageLayout src_layout) { - Common::LockGuard lock(m_mutex); - EnsureBuffer(ctx, size, vk::BufferUsageFlagBits::eTransferDst); - ExecuteImmediateCommands([&](CommandBuffer* command, vk::CommandBuffer) { - RecordImageToBuffer(*command, *src_image, m_buffer, regions, src_layout); - }); - std::memcpy(dst_data, m_mapped_data, size); + WithDownloadedImage(ctx, size, regions, src_image, src_layout, + [&](std::span data) { + std::memcpy(dst_data, data.data(), data.size()); + }); + } + + void ProcessDownloadedImage(GraphicContext* ctx, uint64_t size, + std::span regions, + VulkanImage* src_image, vk::ImageLayout src_layout, + const DownloadedImageConsumer& consumer) { + WithDownloadedImage(ctx, size, regions, src_image, src_layout, + [&](std::span data) { + if (m_host_cached) { + consumer(data); + return; + } + // Scattered CPU tiler reads can be much slower from uncached mapped + // memory. Preserve the original sequential-copy path as a fallback. + m_cached_readback.resize(data.size()); + std::memcpy(m_cached_readback.data(), data.data(), data.size()); + consumer(m_cached_readback); + }); } void Release(GraphicContext* ctx) { @@ -366,11 +382,26 @@ public: VulkanUnmapMemory(ctx, &m_buffer.memory); VulkanDeleteBuffer(ctx, &m_buffer); } - m_capacity = 0; - m_mapped_data = nullptr; + m_capacity = 0; + m_mapped_data = nullptr; + m_host_cached = false; } private: + template + void WithDownloadedImage(GraphicContext* ctx, uint64_t size, + std::span regions, + VulkanImage* src_image, vk::ImageLayout src_layout, + const Consumer& consumer) { + Common::LockGuard lock(m_mutex); + EnsureBuffer(ctx, size, vk::BufferUsageFlagBits::eTransferDst); + ExecuteImmediateCommands([&](CommandBuffer* command, vk::CommandBuffer) { + RecordImageToBuffer(*command, *src_image, m_buffer, regions, src_layout); + }); + consumer(std::span(static_cast(m_mapped_data), + static_cast(size))); + } + template void RecordUpload(GraphicContext* ctx, const void* src_data, uint64_t size, const Recorder& recorder) { @@ -400,17 +431,30 @@ private: m_buffer.usage = usage; m_buffer.memory.property = vk::MemoryPropertyFlagBits::eHostVisible | vk::MemoryPropertyFlagBits::eHostCoherent; + // CPU readback benefits from cached host memory. Keep it optional so Vulkan devices + // without a coherent cached type can fall back to the original sequential-copy path. + m_buffer.memory.preferred_property = + usage & vk::BufferUsageFlagBits::eTransferDst + ? vk::MemoryPropertyFlags(vk::MemoryPropertyFlagBits::eHostCached) + : vk::MemoryPropertyFlags {}; VulkanCreateBuffer(ctx, size, &m_buffer); m_capacity = size; + const auto& properties = ctx->GetPhysicalDeviceMemoryProperties(); + EXIT_IF(m_buffer.memory.type >= properties.memoryTypeCount); + m_host_cached = + static_cast(properties.memoryTypes[m_buffer.memory.type].propertyFlags & + vk::MemoryPropertyFlagBits::eHostCached); VulkanMapMemory(ctx, &m_buffer.memory, &m_mapped_data); } } - Common::Mutex m_mutex; - VulkanBuffer m_buffer; - uint64_t m_capacity = 0; - void* m_mapped_data = nullptr; + Common::Mutex m_mutex; + VulkanBuffer m_buffer; + uint64_t m_capacity = 0; + void* m_mapped_data = nullptr; + bool m_host_cached = false; + std::vector m_cached_readback; }; // Do not replace with one @@ -927,6 +971,16 @@ void DownloadImage(GraphicContext* ctx, void* dst_data, uint64_t size, ->DownloadFromImage(ctx, dst_data, size, regions, src_image, src_layout); } +void ProcessDownloadedImage(GraphicContext* ctx, uint64_t size, + std::span regions, VulkanImage* src_image, + vk::ImageLayout src_layout, + const DownloadedImageConsumer& consumer) { + KYTY_PROFILER_FUNCTION(); + EXIT_IF(size == 0 || regions.empty() || !consumer); + GetStagingBuffer(StagingBufferType::ReadBack) + ->ProcessDownloadedImage(ctx, size, regions, src_image, src_layout, consumer); +} + void UploadImage(GraphicContext* ctx, VulkanImage* image, const void* src_data, uint64_t size, std::span regions, vk::ImageLayout dst_layout) { EXIT_IF(size == 0 || regions.empty()); diff --git a/src/graphics/host_gpu/transfer.h b/src/graphics/host_gpu/transfer.h index b65380066..4fb4dc480 100644 --- a/src/graphics/host_gpu/transfer.h +++ b/src/graphics/host_gpu/transfer.h @@ -5,6 +5,7 @@ #include "common/common.h" #include "graphics/host_gpu/vulkanCommon.h" +#include #include #include #include @@ -73,6 +74,8 @@ MakeLayeredImageBufferCopies(uint32_t layers, uint64_t slice_size, uint32_t pitc enum class StagingBufferType { Texture, Vertex, ReadBack }; +using DownloadedImageConsumer = std::function)>; + class ScratchBuffer { public: explicit ScratchBuffer(uint64_t size); @@ -107,6 +110,13 @@ void DownloadImage(GraphicContext* ctx, void* dst_data, uint64_t size, uint32_t void DownloadImage(GraphicContext* ctx, void* dst_data, uint64_t size, std::span regions, VulkanImage* src_image, vk::ImageLayout src_layout); +// Invokes consumer synchronously after the image copy fence while the readback staging buffer is +// reserved. The supplied span is valid only for the call; the consumer must not retain it or +// re-enter Transfer. +void ProcessDownloadedImage(GraphicContext* ctx, uint64_t size, + std::span regions, VulkanImage* src_image, + vk::ImageLayout src_layout, + const DownloadedImageConsumer& consumer); void UploadBuffer(GraphicContext* ctx, StagingBufferType type, VulkanBuffer* dst_buffer, uint64_t dst_offset, const void* src_data, uint64_t size); void CopyBuffer(VulkanBuffer* src_buffer, VulkanBuffer* dst_buffer, uint64_t size); diff --git a/src/graphics/host_gpu/vma.cpp b/src/graphics/host_gpu/vma.cpp index 63e812c6a..adc16cc1b 100644 --- a/src/graphics/host_gpu/vma.cpp +++ b/src/graphics/host_gpu/vma.cpp @@ -127,6 +127,8 @@ void VulkanCreateBuffer(GraphicContext* ctx, uint64_t size, VulkanBuffer* buffer VmaAllocationCreateInfo alloc_info {}; alloc_info.requiredFlags = static_cast(buffer->memory.property); + alloc_info.preferredFlags = + static_cast(buffer->memory.preferred_property); vk::Buffer::CType native_buffer = VK_NULL_HANDLE; const auto result = static_cast(vmaCreateBuffer( @@ -170,6 +172,8 @@ bool VulkanCreateImage(GraphicContext* ctx, const vk::ImageCreateInfo& image_inf auto& memory = image->memory; VmaAllocationCreateInfo alloc_info {}; alloc_info.requiredFlags = static_cast(memory.property); + alloc_info.preferredFlags = + static_cast(memory.preferred_property); vk::Image::CType native_image = VK_NULL_HANDLE; const auto result = static_cast(vmaCreateImage(