diff --git a/src/graphics/host_gpu/renderer/cache/textureCache.cpp b/src/graphics/host_gpu/renderer/cache/textureCache.cpp index b9d4f6341..3b654a0eb 100644 --- a/src/graphics/host_gpu/renderer/cache/textureCache.cpp +++ b/src/graphics/host_gpu/renderer/cache/textureCache.cpp @@ -583,7 +583,7 @@ bool TextureCache::CopyD16(Image& destination, Image& source) { } auto depth_buffer = m_tiler.GetScratchBuffer(depth_size); - auto color_buffer = m_tiler.GetScratchBuffer(color_size); + auto color_buffer = m_tiler.GetScratchBuffer(color_size, depth_buffer.buffer); const TileManager::D16Layout promote_layout { .width = depth.info.extent.width, .height = depth.info.extent.height, @@ -1071,7 +1071,7 @@ void TextureCache::UploadImage(Image& image, Buffer& source, uint64_t source_off texels_per_slice > UINT64_MAX / transfer_bytes); const uint64_t transfer_slice = texels_per_slice * transfer_bytes; EXIT_NOT_IMPLEMENTED(transfer_slice > UINT64_MAX / layers); - auto promoted = m_tiler.GetScratchBuffer(transfer_slice * layers); + auto promoted = m_tiler.GetScratchBuffer(transfer_slice * layers, linear.buffer); m_tiler.ConvertD16( linear, promoted, TileManager::D16Direction::Promote, info.pixel_format == vk::Format::eD32SfloatS8Uint, @@ -1679,7 +1679,7 @@ void TextureCache::DownloadDepth(Image& image, Buffer& destination, uint64_t des auto host_linear = m_tiler.GetScratchBuffer(transfer_size); image.Download(copies, host_linear.buffer, 0, host_linear.size); const bool tiled = info.IsTiled(); - auto guest_linear = tiled ? m_tiler.GetScratchBuffer(info.data.size) + auto guest_linear = tiled ? m_tiler.GetScratchBuffer(info.data.size, host_linear.buffer) : TileManager::Result {destination.Handle(), destination_offset, destination.Size() - destination_offset}; m_tiler.ConvertD16(host_linear, guest_linear, TileManager::D16Direction::Demote, diff --git a/src/graphics/host_gpu/renderer/image/tiler.cpp b/src/graphics/host_gpu/renderer/image/tiler.cpp index 06f20cf80..f0de271ec 100644 --- a/src/graphics/host_gpu/renderer/image/tiler.cpp +++ b/src/graphics/host_gpu/renderer/image/tiler.cpp @@ -88,30 +88,6 @@ TileManager::~TileManager() { } } -TileManager::Scratch TileManager::AllocateScratch(uint64_t size) { - EXIT_IF(size == 0); - vk::BufferCreateInfo create {}; - create.size = size; - create.usage = vk::BufferUsageFlagBits::eStorageBuffer | vk::BufferUsageFlagBits::eTransferSrc | - vk::BufferUsageFlagBits::eTransferDst; - - VmaAllocationCreateInfo allocate {}; - allocate.usage = VMA_MEMORY_USAGE_AUTO_PREFER_DEVICE; - VkBuffer buffer = VK_NULL_HANDLE; - VmaAllocation memory = nullptr; - const auto raw = static_cast(create); - RequireVulkanSuccess(static_cast(vmaCreateBuffer( - m_graphics.allocator, &raw, &allocate, &buffer, &memory, nullptr)), - "allocate TileManager scratch buffer"); - return {buffer, memory, size}; -} - -void TileManager::DeferDestroy(Scratch scratch) { - auto allocator = m_graphics.allocator; - m_scheduler.DeferOperation( - [allocator, scratch] { vmaDestroyBuffer(allocator, scratch.buffer, scratch.allocation); }); -} - void TileManager::Prepare(bool tile, uint64_t tiled_capacity, uint64_t linear_capacity, std::span infos, uint64_t source_base, uint64_t target_base, std::vector& dispatches) { @@ -369,8 +345,7 @@ TileManager::Result TileManager::Detile(vk::Buffer tiled, uint64_t tiled_offset, const uint64_t source_base = tiled_offset & (descriptor_alignment - 1); std::vector dispatches; Prepare(false, tiled_capacity, linear_capacity, infos, source_base, 0, dispatches); - auto scratch = AllocateScratch(Common::AlignUp(linear_capacity, 4)); - DeferDestroy(scratch); + auto scratch = GetScratchBuffer(linear_capacity, tiled); Record(tiled, tiled_offset, tiled_capacity, scratch.buffer, 0, scratch.size, dispatches, true); return {scratch.buffer, 0, linear_capacity}; @@ -400,11 +375,9 @@ void TileManager::TileImage(Image& image, std::span r std::max(limits.minStorageBufferOffsetAlignment, 4); const uint64_t target_base = tiled_offset & (descriptor_alignment - 1); std::vector dispatches; - // Reserve all stream parameters before creating a scheduler-lived scratch dependency: - // StreamBuffer::Map is allowed to submit the current tick when it wraps. + // Reserve stream parameters before recording the image download and conversion. Prepare(true, tiled_capacity, linear_capacity, infos, 0, target_base, dispatches); - auto linear = AllocateScratch(Common::AlignUp(linear_capacity, 4)); - DeferDestroy(linear); + auto linear = GetScratchBuffer(linear_capacity, tiled); image.Download(regions, linear.buffer, 0, linear.size); Result source {linear.buffer, 0, linear.size}; if (transform == ColorTransform::SwapBgra16) { @@ -414,10 +387,19 @@ void TileManager::TileImage(Image& image, std::span r dispatches, false); } -TileManager::Result TileManager::GetScratchBuffer(uint64_t size) { - auto scratch = AllocateScratch(Common::AlignUp(size, 4)); - DeferDestroy(scratch); - return {scratch.buffer, 0, scratch.size}; +TileManager::Result TileManager::GetScratchBuffer(uint64_t size, vk::Buffer input) { + EXIT_IF(size == 0); + size = Common::AlignUp(size, 4); + auto& buffer = m_scratch[m_scratch[0] && m_scratch[0]->Handle() == input ? 1 : 0]; + if (!buffer || buffer->Size() < size) { + if (buffer) { + m_scheduler.DeferOperation([old = std::move(buffer)]() mutable { old.reset(); }); + } + buffer = std::make_unique(m_graphics, m_scheduler, MemoryUsage::DeviceLocal, 0, + vk::BufferUsageFlagBits::eStorageBuffer | vk::BufferUsageFlagBits::eTransferSrc | + vk::BufferUsageFlagBits::eTransferDst, size); + } + return {buffer->Handle(), 0, size}; } TileManager::StorageBinding TileManager::BindStorage(Result buffer, uint64_t size) const { @@ -668,9 +650,7 @@ void TileManager::SwapBgra16(Result input, Result output, uint32_t pixels) { TileManager::Result TileManager::SwapBgra16(Result input) { EXIT_NOT_IMPLEMENTED(input.size == 0 || input.size % 8u != 0 || input.size / 8u > UINT32_MAX); - auto output = AllocateScratch(input.size); - DeferDestroy(output); - Result result {output.buffer, 0, output.size}; + auto result = GetScratchBuffer(input.size, input.buffer); SwapBgra16(input, result, static_cast(input.size / 8u)); return result; } diff --git a/src/graphics/host_gpu/renderer/image/tiler.h b/src/graphics/host_gpu/renderer/image/tiler.h index e280952c7..9ba3499f1 100644 --- a/src/graphics/host_gpu/renderer/image/tiler.h +++ b/src/graphics/host_gpu/renderer/image/tiler.h @@ -6,13 +6,14 @@ #include "graphics/host_gpu/vulkanCommon.h" #include +#include #include #include -#include namespace Libs::Graphics { class CommandScheduler; +class Buffer; class Image; class StreamBuffer; struct GraphicContext; @@ -62,7 +63,7 @@ public: ~TileManager(); KYTY_CLASS_NO_COPY(TileManager); - // The returned device-local buffer remains alive through the current scheduler tick. + // Consume scratch results before the next acquisition, or pass their buffer as input. [[nodiscard]] Result Detile(vk::Buffer tiled, uint64_t tiled_offset, uint64_t tiled_capacity, uint64_t linear_capacity, std::span infos); void Tile(vk::Buffer linear, uint64_t linear_offset, uint64_t linear_capacity, vk::Buffer tiled, @@ -71,7 +72,7 @@ public: uint64_t tiled_offset, uint64_t tiled_capacity, uint64_t linear_capacity, std::span infos, ColorTransform transform = ColorTransform::None); - [[nodiscard]] Result GetScratchBuffer(uint64_t size); + [[nodiscard]] Result GetScratchBuffer(uint64_t size, vk::Buffer input = nullptr); void ConvertD16(Result source, Result target, D16Direction direction, bool d32, const D16Layout& layout); [[nodiscard]] Result SwapBgra16(Result input); @@ -105,23 +106,16 @@ private: uint32_t pipeline_slot = 0; uint64_t params_offset = 0; }; - struct Scratch { - vk::Buffer buffer = nullptr; - VmaAllocation allocation = nullptr; - uint64_t size = 0; - }; struct StorageBinding { vk::DescriptorBufferInfo info; uint32_t base = 0; }; - [[nodiscard]] Scratch AllocateScratch(uint64_t size); [[nodiscard]] StorageBinding BindStorage(Result buffer, uint64_t size) const; [[nodiscard]] static uint32_t ConversionRows(uint64_t offset, uint64_t row_stride, uint64_t active, uint32_t remaining, uint64_t alignment, uint64_t max_range, uint32_t max_groups) noexcept; - void DeferDestroy(Scratch scratch); void Prepare(bool tile, uint64_t tiled_capacity, uint64_t linear_capacity, std::span infos, uint64_t source_base, uint64_t target_base, std::vector& dispatches); @@ -134,6 +128,7 @@ private: GraphicContext& m_graphics; CommandScheduler& m_scheduler; StreamBuffer& m_stream_buffer; + std::array, 2> m_scratch; vk::DescriptorSetLayout m_descriptor_layout = nullptr; vk::PipelineLayout m_pipeline_layout = nullptr; std::array m_pipelines {}; diff --git a/tests/ShaderRecompilerComputeTests.cpp b/tests/ShaderRecompilerComputeTests.cpp index ef9f03fa5..18c95754e 100644 --- a/tests/ShaderRecompilerComputeTests.cpp +++ b/tests/ShaderRecompilerComputeTests.cpp @@ -15298,6 +15298,40 @@ public: source_stage, vk::PipelineStageFlagBits::eHost, {}, 0, nullptr, 1, &barrier, 0, nullptr); }; + auto scratch_input = CreateHostBuffer(name, 32, AllFlags, + {0x00020001u, 0x00040003u, 0x00060005u, 0x00080007u, + 0x000a0009u, 0x000c000bu, 0x000e000du, 0x0010000fu}); + auto scratch_output = CreateHostBuffer(name, 56, AllFlags, {}); + const auto copy_scratch = [&](TileManager::Result result, uint64_t offset) { + const vk::BufferCopy copy{result.offset, offset, result.size}; + scheduler.Current().Handle().copyBuffer(result.buffer, scratch_output.buffer, 1, ©); + }; + // Keep all four conversions in flight to check growth, overwrite ordering, + // the requested span on reuse, and two simultaneously live workspaces. + const auto first = tile_manager.SwapBgra16({scratch_input.buffer, 0, 8}); + copy_scratch(first, 0); + const auto grown = tile_manager.SwapBgra16({scratch_input.buffer, 0, 32}); + copy_scratch(grown, 8); + const auto reused = tile_manager.SwapBgra16({scratch_input.buffer, 8, 8}); + copy_scratch(reused, 40); + const auto paired = tile_manager.SwapBgra16(reused); + copy_scratch(paired, 48); + Require(name, "scratch workspace reuse", + first.buffer != grown.buffer && grown.buffer == reused.buffer && + reused.buffer != paired.buffer && reused.size == 8 && paired.size == 8, + "scratch growth, input exclusion, or requested span was lost"); + host_barrier(scratch_output.buffer, scratch_output.size, + vk::PipelineStageFlagBits::eTransfer, vk::AccessFlagBits::eTransferWrite); + scheduler.Finish(); + Require(name, "scratch workspace GPU contents", + ReadBuffer(name, scratch_output, 14) == std::vector{ + 0x00020003u, 0x00040001u, + 0x00020003u, 0x00040001u, 0x00060007u, 0x00080005u, + 0x000a000bu, 0x000c0009u, 0x000e000fu, 0x0010000du, + 0x00060007u, 0x00080005u, 0x00060005u, 0x00080007u}, + "scratch reuse overwrote an earlier result or its live conversion input"); + DestroyBuffer(&scratch_output); + DestroyBuffer(&scratch_input); const auto gpu_detile = [&](const std::vector &tiled, std::vector *linear, uint64_t tiled_capacity, uint64_t linear_capacity,