diff --git a/src/graphics/guest_gpu/command_processor/commandProcessor.h b/src/graphics/guest_gpu/command_processor/commandProcessor.h index a202f1484..eb490d648 100644 --- a/src/graphics/guest_gpu/command_processor/commandProcessor.h +++ b/src/graphics/guest_gpu/command_processor/commandProcessor.h @@ -196,6 +196,7 @@ public: template void WaitRegMem(uint32_t func, const T* addr, T ref, T mask, uint32_t poll, uint32_t wait_op); void WriteData(uint32_t* dst, const uint32_t* src, uint32_t dw_num, uint32_t write_control); + void WriteReferenceClock(uint64_t dst_address, uint32_t num_bytes); void DmaData(uint8_t engine, uint8_t dst_sel, uint8_t dst_cache_policy, uint64_t dst_address_or_offset, uint8_t src_sel, uint8_t src_cache_policy, uint64_t src_address_or_offset_or_immediate, uint32_t num_bytes, diff --git a/src/graphics/guest_gpu/command_processor/pm4Handlers.cpp b/src/graphics/guest_gpu/command_processor/pm4Handlers.cpp index af0e2ca19..a964c989c 100644 --- a/src/graphics/guest_gpu/command_processor/pm4Handlers.cpp +++ b/src/graphics/guest_gpu/command_processor/pm4Handlers.cpp @@ -2224,6 +2224,15 @@ KYTY_CP_OP_PARSER(CpOpCopyData) { const uint32_t num_bytes = ((control >> 16u) & 0x1u) != 0 ? 8u : 4u; const uint64_t src = buffer[1] | (static_cast(buffer[2]) << 32u); const uint64_t dst = buffer[3] | (static_cast(buffer[4]) << 32u); + if (src_sel == (9u << 1u)) { + if (dst_sel != (2u << 1u) || dst == 0 || (dst & (num_bytes - 1u)) != 0) { + EXIT("unsupported reference-clock copyData, src_sel=0x%02" PRIx32 + " dst_sel=0x%02" PRIx32 " dst=0x%016" PRIx64 " size=%u\n", + src_sel, dst_sel, dst, num_bytes); + } + cp->WriteReferenceClock(dst, num_bytes); + return 5; + } const auto dma_src = CopyDataSrcToDma(src_sel); if (dma_src == 2 && num_bytes == 8) { EXIT("unsupported 64-bit immediate copyData\n"); diff --git a/src/graphics/guest_gpu/graphicsRun.cpp b/src/graphics/guest_gpu/graphicsRun.cpp index 581439792..930988dab 100644 --- a/src/graphics/guest_gpu/graphicsRun.cpp +++ b/src/graphics/guest_gpu/graphicsRun.cpp @@ -661,6 +661,22 @@ void CommandProcessor::WriteData(uint32_t* dst, const uint32_t* src, uint32_t dw memcpy(dst, src, static_cast(dw_num) * 4); } +void CommandProcessor::WriteReferenceClock(uint64_t dst_address, uint32_t num_bytes) { + Common::LockGuard lock(m_mutex); + if (dst_address == 0 || (num_bytes != sizeof(uint32_t) && num_bytes != sizeof(uint64_t)) || + (dst_address & (num_bytes - 1u)) != 0) { + EXIT("invalid reference-clock copy, dst=0x%016" PRIx64 " size=%u\n", dst_address, + num_bytes); + } + const auto value = GraphicsRenderReadReferenceClock(); + CommandProcessorGuestAccessScope guest_access(CommandProcessorGuestAccess::DirectFence, + dst_address, num_bytes); + std::memcpy(reinterpret_cast(dst_address), &value, num_bytes); + LOGF("\t copy_data reference clock: dst=0x%016" PRIx64 " value=0x%016" PRIx64 + " size=%u\n", + dst_address, value, num_bytes); +} + void CommandProcessor::DmaData(uint8_t engine, uint8_t dst_sel, uint8_t dst_cache_policy, uint64_t dst_address_or_offset, uint8_t src_sel, uint8_t src_cache_policy, diff --git a/src/graphics/guest_gpu/tile.cpp b/src/graphics/guest_gpu/tile.cpp index c14ac0907..34df092c3 100644 --- a/src/graphics/guest_gpu/tile.cpp +++ b/src/graphics/guest_gpu/tile.cpp @@ -1919,7 +1919,32 @@ void TileConvertTiledToLinearStandard256B(void* dst, const void* src, uint32_t f } } -// Prospero depth-tile address table for 2D, 1xAA surfaces by element size. +// Prospero depth/stencil-tile address table for 2D, 1xAA surfaces by element size. +static constexpr uint32_t Depth64KB8XOffsetBytes(uint32_t x) { + uint32_t offset = 0; + offset ^= x & 0x0001u; + offset ^= (x << 1u) & 0x0004u; + offset ^= (x << 2u) & 0x0010u; + offset ^= (x << 3u) & 0x0040u; + offset ^= (x << 5u) & 0x0300u; + offset ^= (x << 4u) & 0x0400u; + offset ^= (x << 6u) & 0x0800u; + offset ^= (x << 7u) & 0x2000u; + offset ^= (x << 8u) & 0x8000u; + return offset; +} + +static constexpr uint32_t Depth64KB8YOffsetBytes(uint32_t y) { + uint32_t offset = 0; + offset ^= (y << 1u) & 0x0002u; + offset ^= (y << 2u) & 0x0008u; + offset ^= (y << 3u) & 0x00a0u; + offset ^= (y << 5u) & 0x0f00u; + offset ^= (y << 6u) & 0x1000u; + offset ^= (y << 7u) & 0x4000u; + return offset; +} + static constexpr uint32_t Depth64KB16XOffsetBytes(uint32_t x) { uint32_t offset = 0; offset ^= (x << 1u) & 0x0002u; @@ -1964,6 +1989,9 @@ static constexpr uint32_t Depth64KB32YOffsetBytes(uint32_t y) { return offset; } +static_assert(Depth64KB8XOffsetBytes(2) == 0x0004u); +static_assert(Depth64KB8YOffsetBytes(1) == 0x0002u); +static_assert((Depth64KB8XOffsetBytes(3) ^ Depth64KB8YOffsetBytes(5)) == 0x0027u); static_assert(Depth64KB16XOffsetBytes(2) == 0x0008u); static_assert(Depth64KB16YOffsetBytes(1) == 0x0004u); static_assert((Depth64KB16XOffsetBytes(3) ^ Depth64KB16YOffsetBytes(5)) == 0x004eu); @@ -1971,6 +1999,20 @@ static_assert(Depth64KB32XOffsetBytes(2) == 0x0010u); static_assert(Depth64KB32YOffsetBytes(1) == 0x0008u); static_assert((Depth64KB32XOffsetBytes(3) ^ Depth64KB32YOffsetBytes(5)) == 0x009cu); +struct Depth64KB8Tables { + std::array x_words {}; + std::array y_words {}; + + constexpr Depth64KB8Tables() { + for (uint32_t x = 0; x < x_words.size(); x++) { + x_words[x] = Depth64KB8XOffsetBytes(x); + } + for (uint32_t y = 0; y < y_words.size(); y++) { + y_words[y] = Depth64KB8YOffsetBytes(y); + } + } +}; + struct Depth64KB16Tables { std::array x_words {}; std::array y_words {}; @@ -2089,15 +2131,19 @@ static void TileConvertDepth(void* dst, const void* src, uint32_t format, uint32 uint32_t height, uint32_t pitch, uint64_t size) { KYTY_PROFILER_FUNCTION(); const uint32_t bytes_per_element = Prospero::NumBytesPerElement(format); - const uint32_t block_width = bytes_per_element == 2 ? 256u : 128u; + const bool supported_bpe = + bytes_per_element == 1 || bytes_per_element == 2 || bytes_per_element == 4; + const uint32_t block_width = supported_bpe && bytes_per_element <= 2 ? 256u : 128u; + const uint32_t block_height = supported_bpe ? 65536u / (block_width * bytes_per_element) : 0; if (dst == nullptr || src == nullptr || width == 0 || height == 0 || pitch < width || - (bytes_per_element != 2 && bytes_per_element != 4) || pitch % block_width != 0 || + !supported_bpe || pitch % block_width != 0 || size == 0 || size % 65536u != 0) { EXIT("unsupported depth conversion, dst=%p src=%p format=%u " "extent=%ux%u pitch=%u size=0x%016" PRIx64 "\n", dst, src, format, width, height, pitch, size); } - const uint64_t block_rows = (static_cast(height) + 127u) / 128u; + const uint64_t block_rows = + (static_cast(height) + block_height - 1u) / block_height; const uint64_t blocks_per_row = pitch / block_width; if (blocks_per_row > UINT64_MAX / block_rows || blocks_per_row * block_rows > UINT64_MAX / 65536u || @@ -2106,7 +2152,11 @@ static void TileConvertDepth(void* dst, const void* src, uint32_t format, uint32 "extent=%ux%u pitch=%u bpe=%u size=0x%016" PRIx64 "\n", width, height, pitch, bytes_per_element, size); } - if (bytes_per_element == 2) { + if (bytes_per_element == 1) { + static constexpr Depth64KB8Tables tables; + TileConvertDepthTyped(dst, src, width, height, pitch, size, + tables.x_words, tables.y_words); + } else if (bytes_per_element == 2) { static constexpr Depth64KB16Tables tables; TileConvertDepthTyped(dst, src, width, height, pitch, size, tables.x_words, tables.y_words); diff --git a/src/graphics/host_gpu/renderer/bufferCache.cpp b/src/graphics/host_gpu/renderer/bufferCache.cpp index a36563e0b..e7bb85a18 100644 --- a/src/graphics/host_gpu/renderer/bufferCache.cpp +++ b/src/graphics/host_gpu/renderer/bufferCache.cpp @@ -1207,6 +1207,7 @@ BufferImageCopySource BufferCache::ObtainBufferForImage(uint64_t vaddr, uint64_t (vaddr & (TRACKER_PAGE_SIZE - 1)) == 0 && (size & (TRACKER_PAGE_SIZE - 1)) == 0); } FaultSafeCacheLock lock(this, m_mutex); + const bool cpu_dirty = m_memory_tracker.IsRegionCpuModified(vaddr, size); auto use_coherent_guest_backing = [&](const CachedBuffer& overlap) { const bool gpu_modified = m_memory_tracker.IsRegionGpuModified(vaddr, size); const auto dirty_ranges = m_gpu_modified_ranges.Intersections(vaddr, size); @@ -1219,7 +1220,7 @@ BufferImageCopySource BufferCache::ObtainBufferForImage(uint64_t vaddr, uint64_t // A clean/CPU-current image range may coexist with a partial cached-buffer view because // guest backing is authoritative. The PS5 detiler consumes that backing at the existing // Tiler seam. - return BufferImageCopySource {nullptr, 0, vaddr, size, true}; + return BufferImageCopySource {nullptr, 0, vaddr, size, true, cpu_dirty}; }; auto it = m_buffers.upper_bound(vaddr); if (it == m_buffers.begin()) { @@ -1227,7 +1228,7 @@ BufferImageCopySource BufferCache::ObtainBufferForImage(uint64_t vaddr, uint64_t PageOverlaps(vaddr, size, it->second->vaddr, it->second->size)) { return use_coherent_guest_backing(*it->second); } - return {nullptr, 0, vaddr, size, true}; + return {nullptr, 0, vaddr, size, true, cpu_dirty}; } --it; auto& cached = *it->second; @@ -1241,7 +1242,7 @@ BufferImageCopySource BufferCache::ObtainBufferForImage(uint64_t vaddr, uint64_t PageOverlaps(vaddr, size, next->second->vaddr, next->second->size)) { return use_coherent_guest_backing(*next->second); } - return {nullptr, 0, vaddr, size, true}; + return {nullptr, 0, vaddr, size, true, cpu_dirty}; } if (cached.ctx == nullptr || cached.buffer == nullptr || cached.buffer->buffer == nullptr) { EXIT("BufferCache: inconsistent image source, addr=0x%016" PRIx64 " size=0x%016" PRIx64 @@ -1250,7 +1251,7 @@ BufferImageCopySource BufferCache::ObtainBufferForImage(uint64_t vaddr, uint64_t static_cast(cached.buffer.get())); } const bool gpu_modified = m_memory_tracker.IsRegionGpuModified(vaddr, size); - const bool cpu_modified = m_memory_tracker.IsRegionCpuModified(vaddr, size); + const bool cpu_modified = cpu_dirty; if (gpu_modified && cpu_modified) { EXIT("BufferCache: image source is both CPU- and GPU-modified, addr=0x%016" PRIx64 " size=0x%016" PRIx64 "\n", @@ -1333,7 +1334,7 @@ BufferImageCopySource BufferCache::ObtainBufferForImage(uint64_t vaddr, uint64_t " size=0x%016" PRIx64 "\n", vaddr, size); } - return {cached.buffer.get(), offset, vaddr, size, true}; + return {cached.buffer.get(), offset, vaddr, size, true, cpu_dirty}; } namespace { diff --git a/src/graphics/host_gpu/renderer/bufferCache.h b/src/graphics/host_gpu/renderer/bufferCache.h index 4a9fafede..116d330c7 100644 --- a/src/graphics/host_gpu/renderer/bufferCache.h +++ b/src/graphics/host_gpu/renderer/bufferCache.h @@ -26,6 +26,9 @@ struct BufferImageCopySource { uint64_t address = 0; uint64_t size = 0; bool cpu_current = false; + // True when the guest range was CPU-dirty before coherence resolution. ObtainBufferForImage + // may consume that tracker state while publishing the same bytes to a cached buffer. + bool cpu_dirty = false; }; struct BufferCacheRange { diff --git a/src/graphics/host_gpu/renderer/depthRenderTarget.cpp b/src/graphics/host_gpu/renderer/depthRenderTarget.cpp index 469ed89cd..58ee6cc92 100644 --- a/src/graphics/host_gpu/renderer/depthRenderTarget.cpp +++ b/src/graphics/host_gpu/renderer/depthRenderTarget.cpp @@ -340,6 +340,8 @@ void ResolveRenderDepthTarget(uint64_t submit_id, CommandBuffer* buffer, const H (r->stencil_test_enable && (stencil_face_accesses_attachment(r->stencil_static_front, r->stencil_dynamic_front) || stencil_face_accesses_attachment(r->stencil_static_back, r->stencil_dynamic_back))); + info.stencil_htile_compressed = + has_stencil && has_htile && !z.stencil_info.tile_stencil_disable; r->vulkan_buffer = cache->FindDepthTarget(buffer, g_render_ctx->GetGraphicCtx(), info); if (meta_clear && !cache->TouchMeta(z.htile_data_base_addr, z.depth_view.slice_start, false)) { DepthFatal("failed to consume HTile clear state"); diff --git a/src/graphics/host_gpu/renderer/imageInfo.h b/src/graphics/host_gpu/renderer/imageInfo.h index 695c8cf36..660afb8dc 100644 --- a/src/graphics/host_gpu/renderer/imageInfo.h +++ b/src/graphics/host_gpu/renderer/imageInfo.h @@ -86,6 +86,7 @@ struct DepthTargetInfo { bool depth_load_clear = false; bool stencil_load_clear = false; bool stencil_access = false; + bool stencil_htile_compressed = false; }; enum class VideoOutCompression : uint8_t { Uncompressed, Dcc256_64_64, Unsupported }; @@ -171,6 +172,7 @@ struct VideoOutPixelFormatInfo { } enum class DepthOverlap : uint8_t { None, RetireSampled, Unsupported }; +enum class DepthTransitionSource : uint8_t { None, Guest, Native }; enum class RenderTargetOverlap : uint8_t { None, RetireSampled, RetireTarget, Unsupported }; enum class SampledOverlap : uint8_t { None, ReadOnlyAlias, Unsupported }; enum class StorageSampledOverlap : uint8_t { None, ExactImage, Unsupported }; @@ -178,6 +180,19 @@ enum class HostWriteOverlap : uint8_t { None, InvalidateImage, Unsupported }; enum class BufferImageAlias : uint8_t { None, VideoOutWrite, RenderTargetWrite, Unsupported }; enum class MetaImageOverlap : uint8_t { RetainSampled, RetireTarget, Unsupported }; +[[nodiscard]] inline constexpr DepthTransitionSource +SelectDepthTransitionSource(bool depth_load_clear, bool sampled_native_available, + bool sampled_cpu_dirty, bool sampled_buffer_modified, + bool buffer_overlap, bool buffer_cpu_dirty) noexcept { + if (depth_load_clear) { + return DepthTransitionSource::None; + } + return sampled_native_available && !sampled_cpu_dirty && !sampled_buffer_modified && + !(buffer_overlap && buffer_cpu_dirty) + ? DepthTransitionSource::Native + : DepthTransitionSource::Guest; +} + [[nodiscard]] inline MetaImageOverlap ClassifyMetaImageOverlap(bool sampled, bool render_target, bool gpu_modified, bool buffer_modified) { @@ -281,12 +296,17 @@ enum class MetaImageOverlap : uint8_t { RetainSampled, RetireTarget, Unsupported } [[nodiscard]] inline bool CanLoadStencilAttachment(const DepthTargetInfo& target, - bool stencil_initialized) { + bool stencil_initialized) { const bool has_stencil = target.stencil_address != 0 || target.stencil_size != 0; return !has_stencil || !target.stencil_access || target.stencil_load_clear || stencil_initialized; } +[[nodiscard]] inline bool CanLoadRawStencilPlane(const DepthTargetInfo& target) { + const bool has_stencil = target.stencil_address != 0 || target.stencil_size != 0; + return has_stencil && !target.stencil_htile_compressed; +} + [[nodiscard]] inline bool IsDepthTargetRangeCompatible(const DepthTargetInfo& target, uint64_t address, uint64_t size) { if (address == 0 || size == 0 || address > UINT64_MAX - size) { @@ -414,6 +434,22 @@ ClassifyBufferImageAlias(uint64_t buffer_address, uint64_t buffer_size, uint64_t return DepthOverlap::None; } const bool has_stencil = depth.stencil_address != 0 || depth.stencil_size != 0; + const bool exact_depth_format = + sampled.format == depth.guest_format && + ((depth.guest_format == Prospero::GpuEnumValue(Prospero::BufferFormat::k16UNorm) && + depth.format == VK_FORMAT_D16_UNORM && depth.bytes_per_element == 2) || + (depth.guest_format == Prospero::GpuEnumValue(Prospero::BufferFormat::k32Float) && + depth.format == VK_FORMAT_D32_SFLOAT && depth.bytes_per_element == 4)); + const bool exact_depth_load = + !has_stencil && !depth.depth_load_clear && sampled.address == depth.address && + sampled.size == depth.size && sampled.width == depth.width && sampled.height == depth.height && + sampled.pitch == depth.pitch && sampled.base_level == 0 && sampled.levels == 1 && + sampled.view_levels == 1 && sampled.tile == depth.tile_mode && sampled.depth == 1 && + sampled.type == Prospero::GpuEnumValue(Prospero::ImageType::kColor2D) && + sampled.base_array == 0 && exact_depth_format; + if (!sampled_gpu_modified && exact_depth_load) { + return DepthOverlap::RetireSampled; + } if (sampled.address == depth.address && !sampled_gpu_modified && depth.depth_load_clear && (!has_stencil || (depth.stencil_address != 0 && depth.stencil_size != 0 && depth.stencil_load_clear))) { diff --git a/src/graphics/host_gpu/renderer/imageView.h b/src/graphics/host_gpu/renderer/imageView.h index 79c015d34..14753f1b7 100644 --- a/src/graphics/host_gpu/renderer/imageView.h +++ b/src/graphics/host_gpu/renderer/imageView.h @@ -88,6 +88,7 @@ namespace Libs::Graphics { switch (swizzle) { case DstSel(4, 4, 4, 4): return VulkanImage::VIEW_DEPTH_TEXTURE; case DstSel(4, 0, 0, 0): return VulkanImage::VIEW_R000; + case DstSel(4, 0, 0, 1): return VulkanImage::VIEW_R001; default: break; } } diff --git a/src/graphics/host_gpu/renderer/render.h b/src/graphics/host_gpu/renderer/render.h index ed07f232e..853a057d0 100644 --- a/src/graphics/host_gpu/renderer/render.h +++ b/src/graphics/host_gpu/renderer/render.h @@ -127,6 +127,10 @@ void RenderDispatchDirect(uint64_t submit_id, CommandBuffer* buffer, HW::Context void GraphicsRenderInit(); void GraphicsRenderCreateContext(); +[[nodiscard]] bool GraphicsScaleReferenceClock(uint64_t host_ticks, uint64_t host_frequency, + uint64_t* value); +[[nodiscard]] uint64_t GraphicsRenderReadReferenceClock(); + [[nodiscard]] bool ResolveComputeImageClear(const ShaderComputeInputInfo& input, uint32_t group_x, uint32_t group_y, uint32_t group_z, uint32_t mode, ShaderBufferResource* descriptor, diff --git a/src/graphics/host_gpu/renderer/sync.cpp b/src/graphics/host_gpu/renderer/sync.cpp index 3637037b6..ddc3a93ff 100644 --- a/src/graphics/host_gpu/renderer/sync.cpp +++ b/src/graphics/host_gpu/renderer/sync.cpp @@ -14,11 +14,47 @@ #include "libs/errno.h" #include +#include namespace Libs::Graphics { constexpr int GRAPHICS_EVENT_QUEUED_GRAPHICS_INTERRUPT = 0x00; constexpr int GRAPHICS_EVENT_EOP = 0x40; +constexpr uint64_t GRAPHICS_REFERENCE_CLOCK_FREQUENCY = 100000000; + +bool GraphicsScaleReferenceClock(uint64_t host_ticks, uint64_t host_frequency, uint64_t* value) { + if (host_frequency == 0 || value == nullptr) { + return false; + } + + const auto whole_seconds = host_ticks / host_frequency; + const auto remainder = host_ticks % host_frequency; + constexpr auto MAX_VALUE = std::numeric_limits::max(); + if (whole_seconds > MAX_VALUE / GRAPHICS_REFERENCE_CLOCK_FREQUENCY || + remainder > MAX_VALUE / GRAPHICS_REFERENCE_CLOCK_FREQUENCY) { + return false; + } + + const auto whole_value = whole_seconds * GRAPHICS_REFERENCE_CLOCK_FREQUENCY; + const auto fractional_value = + (remainder * GRAPHICS_REFERENCE_CLOCK_FREQUENCY) / host_frequency; + if (whole_value > MAX_VALUE - fractional_value) { + return false; + } + *value = whole_value + fractional_value; + return true; +} + +uint64_t GraphicsRenderReadReferenceClock() { + const auto host_frequency = LibKernel::KernelGetTscFrequency(); + const auto host_ticks = LibKernel::KernelReadTsc(); + uint64_t value = 0; + if (!GraphicsScaleReferenceClock(host_ticks, host_frequency, &value)) { + EXIT("cannot scale host clock, ticks=0x%016" PRIx64 " frequency=%" PRIu64 "\n", + host_ticks, host_frequency); + } + return value; +} static void SubmitLabel32(CommandBuffer* buffer, uint32_t* dst, uint32_t value, LabelCallback callback_1 = nullptr, LabelCallback callback_2 = nullptr, @@ -158,7 +194,7 @@ void GraphicsRenderWriteAtEndOfPipeClockCounter(uint64_t submit_id, CommandBuffe buffer->SetDebugInfo(static_cast(CommandBufferDebugOp::EopWrite), submit_id, 8, 0, 0, 0, reinterpret_cast(dst_gpu_addr)); - const auto value = LibKernel::KernelReadTsc(); + const auto value = GraphicsRenderReadReferenceClock(); PublishImmediateFence(dst_gpu_addr, value); LOGF_COLOR(Log::Color::BrightGreen, "EndOfPipe Signal!!! [0x%016" PRIx64 "] <- Clock: 0x%016" PRIx64 "\n", @@ -176,7 +212,7 @@ void GraphicsRenderWriteAtEndOfPipeClockCounterWithWriteBack(uint64_t subm buffer->SetDebugInfo(static_cast(CommandBufferDebugOp::EopWriteBack), submit_id, 8, 0, 0, 0, reinterpret_cast(dst_gpu_addr)); - const auto value = LibKernel::KernelReadTsc(); + const auto value = GraphicsRenderReadReferenceClock(); PublishImmediateFence(dst_gpu_addr, value); LOGF_COLOR(Log::Color::BrightGreen, "EndOfPipe Signal!!! [0x%016" PRIx64 "] <- Clock: 0x%016" PRIx64 "\n", diff --git a/src/graphics/host_gpu/renderer/textureCache.cpp b/src/graphics/host_gpu/renderer/textureCache.cpp index acb1769e1..ee4c47637 100644 --- a/src/graphics/host_gpu/renderer/textureCache.cpp +++ b/src/graphics/host_gpu/renderer/textureCache.cpp @@ -591,7 +591,8 @@ bool Equal(const DepthTargetInfo& left, const DepthTargetInfo& right) { left.htile_size == right.htile_size && left.format == right.format && left.guest_format == right.guest_format && left.width == right.width && left.height == right.height && left.pitch == right.pitch && - left.bytes_per_element == right.bytes_per_element && left.tile_mode == right.tile_mode; + left.bytes_per_element == right.bytes_per_element && left.tile_mode == right.tile_mode && + left.stencil_htile_compressed == right.stencil_htile_compressed; } [[nodiscard]] bool IsCoherentGuestImageSource(const BufferImageCopySource& source, @@ -863,6 +864,11 @@ void CreateDepthViews(GraphicContext* ctx, DepthStencilVulkanImage* image) { {VK_COMPONENT_SWIZZLE_R, VK_COMPONENT_SWIZZLE_ZERO, VK_COMPONENT_SWIZZLE_ZERO, VK_COMPONENT_SWIZZLE_ZERO}, 0, 0, 1, 1); + UtilCreateImageView(ctx, image, VulkanImage::VIEW_R001, VK_IMAGE_VIEW_TYPE_2D, + VK_IMAGE_ASPECT_DEPTH_BIT, + {VK_COMPONENT_SWIZZLE_R, VK_COMPONENT_SWIZZLE_ZERO, + VK_COMPONENT_SWIZZLE_ZERO, VK_COMPONENT_SWIZZLE_ONE}, + 0, 0, 1, 1); } DepthStencilVulkanImage* CreateDepthTarget(GraphicContext* ctx, const DepthTargetInfo& info, @@ -1840,6 +1846,7 @@ DepthStencilVulkanImage* TextureCache::FindDepthTarget(CommandBuffer* command, G PageOverlaps(info.address, info.size, info.htile_address, info.htile_size) || (has_stencil && PageOverlaps(info.stencil_address, info.stencil_size, info.htile_address, info.htile_size)))) || + (info.stencil_htile_compressed && (!has_stencil || !has_htile)) || !(info.guest_format == Prospero::GpuEnumValue(Prospero::BufferFormat::k16UNorm) ? !has_stencil && info.format == VK_FORMAT_D16_UNORM && info.bytes_per_element == 2 : info.guest_format == Prospero::GpuEnumValue(Prospero::BufferFormat::k32Float) && @@ -1871,14 +1878,23 @@ DepthStencilVulkanImage* TextureCache::FindDepthTarget(CommandBuffer* command, G " elements=0x%016" PRIx64 "\n", info.stencil_size, elements); } + if (has_stencil && !info.stencil_load_clear && !CanLoadRawStencilPlane(info)) { + EXIT("TextureCache: HTile-compressed stencil load is unsupported, addr=0x%016" PRIx64 + " size=0x%016" PRIx64 " htile=0x%016" PRIx64 "+0x%016" PRIx64 "\n", + info.stencil_address, info.stencil_size, info.htile_address, info.htile_size); + } if (has_htile) { RegisterMeta(info.htile_address, info.htile_size); } std::lock_guard transaction(m_resource_mutex); - const auto depth_source = m_buffer_cache.ObtainBufferForImage(info.address, info.size); - if (has_stencil) { - (void)m_buffer_cache.ObtainBufferForImage(info.stencil_address, info.stencil_size); - } + // BufferCache treats an untracked guest range as CPU-current. That is useful when + // uploading from guest memory, but it is not evidence that a clean native image is + // stale. Only a genuinely overlapping buffer participates in transition selection. + const bool depth_buffer_overlap = m_buffer_cache.HasPageOverlap(info.address, info.size); + const auto depth_source = m_buffer_cache.ObtainBufferForImage(info.address, info.size); + const auto stencil_source = + has_stencil ? m_buffer_cache.ObtainBufferForImage(info.stencil_address, info.stencil_size) + : BufferImageCopySource {}; FaultSafeTextureLock lock(this, m_lock); RequireNoMetaOverlapLocked(info.address, info.size); if (has_stencil) { @@ -1900,37 +1916,32 @@ DepthStencilVulkanImage* TextureCache::FindDepthTarget(CommandBuffer* command, G if (has_stencil && info.stencil_load_clear) { match->stencil_initialized = true; } - if (!match->gpu_modified && m_memory_tracker.IsRegionCpuModified(info.address, info.size)) { - if (has_stencil && match->stencil_initialized && !info.stencil_load_clear) { - EXIT("TextureCache: depth refresh would discard initialized stencil, " - "addr=0x%016" PRIx64 "\n", - info.address); - } + if (!match->gpu_modified && depth_source.cpu_dirty) { m_memory_tracker.ForEachUploadRange( info.address, info.size, false, [](uint64_t, uint64_t) noexcept {}, [&]() noexcept { m_tiler.DetileImage(ctx, - static_cast(match->image), - match->depth, depth_source, true); + static_cast(match->image), + match->depth, depth_source, true); }); } - if (!match->gpu_modified && has_stencil && - m_memory_tracker.IsRegionCpuModified(info.stencil_address, info.stencil_size)) { - if (!info.stencil_load_clear) { - if (info.stencil_access) { - EXIT("TextureCache: stencil refresh from guest memory is unsupported, " - "addr=0x%016" PRIx64 " size=0x%016" PRIx64 "\n", - info.stencil_address, info.stencil_size); - } - } else { - m_memory_tracker.ForEachUploadRange( - info.stencil_address, info.stencil_size, false, - [](uint64_t, uint64_t) noexcept {}, []() noexcept {}); - } + if (!match->gpu_modified && has_stencil && stencil_source.cpu_dirty) { + m_memory_tracker.ForEachUploadRange( + info.stencil_address, info.stencil_size, false, + [](uint64_t, uint64_t) noexcept {}, + [&]() noexcept { + if (!info.stencil_load_clear) { + m_tiler.DetileStencil( + ctx, static_cast(match->image), match->depth, + stencil_source, true); + match->stencil_initialized = true; + } + }); } return static_cast(match->image); } std::vector retire; + std::shared_ptr sampled_depth_source; for (const auto& entry: m_images) { auto& cached = *entry; const bool overlaps = @@ -1947,6 +1958,9 @@ DepthStencilVulkanImage* TextureCache::FindDepthTarget(CommandBuffer* command, G info.address, info.size, static_cast(cached.kind), cached.Address(), cached.Size()); } + if (!info.depth_load_clear && sampled_depth_source == nullptr) { + sampled_depth_source = entry; + } retire.push_back(&cached); } std::vector ranges; @@ -1979,19 +1993,64 @@ DepthStencilVulkanImage* TextureCache::FindDepthTarget(CommandBuffer* command, G cached->ctx = ctx; cached->stencil_initialized = !has_stencil || info.stencil_load_clear; cached->image = CreateDepthTarget(ctx, info, &cached->memory); + if (sampled_depth_source != nullptr && + (sampled_depth_source->image->type != VulkanImageType::Texture || + sampled_depth_source->image->format != Prospero::SurfaceFormat(info.guest_format) || + sampled_depth_source->image->extent.width != info.width || + sampled_depth_source->image->extent.height != info.height)) { + EXIT("TextureCache: sampled-depth native source is inconsistent, format=%d/%d " + "extent=%ux%u/%ux%u type=%u\n", + static_cast(sampled_depth_source->image->format), + static_cast(Prospero::SurfaceFormat(info.guest_format)), + sampled_depth_source->image->extent.width, + sampled_depth_source->image->extent.height, info.width, info.height, + static_cast(sampled_depth_source->image->type)); + } + const auto transition_source = SelectDepthTransitionSource( + info.depth_load_clear, sampled_depth_source != nullptr, + sampled_depth_source != nullptr && sampled_depth_source->info.IsCpuDirty(), + sampled_depth_source != nullptr && sampled_depth_source->buffer_modified, + depth_buffer_overlap, depth_source.cpu_dirty); + LOGF("TextureCache: depth transition source=%u sampled=%d sampled_cpu_dirty=%d " + "sampled_buffer_dirty=%d buffer_overlap=%d buffer_cpu_dirty=%d addr=0x%016" PRIx64 + " size=0x%016" PRIx64 "\n", + static_cast(transition_source), sampled_depth_source != nullptr, + sampled_depth_source != nullptr && sampled_depth_source->info.IsCpuDirty(), + sampled_depth_source != nullptr && sampled_depth_source->buffer_modified, + depth_buffer_overlap, depth_source.cpu_dirty, info.address, info.size); m_memory_tracker.ForEachUploadRange( info.address, info.size, false, [](uint64_t, uint64_t) noexcept {}, [&]() noexcept { - if (!info.depth_load_clear) { - m_tiler.DetileImage(ctx, - static_cast(cached->image), - info, depth_source, false); + switch (transition_source) { + case DepthTransitionSource::None: break; + case DepthTransitionSource::Guest: + m_tiler.DetileImage(ctx, static_cast(cached->image), + info, depth_source, false); + break; + case DepthTransitionSource::Native: + command->RetainResourceUntilFence(sampled_depth_source); + UtilCopyImageWithBuffer( + command, ctx, sampled_depth_source->image, VK_IMAGE_ASPECT_COLOR_BIT, + cached->image, VK_IMAGE_ASPECT_DEPTH_BIT, info.bytes_per_element, + static_cast(VK_IMAGE_LAYOUT_DEPTH_STENCIL_ATTACHMENT_OPTIMAL)); + LOGF("TextureCache: preserved sampled depth through native buffer copy, " + "addr=0x%016" PRIx64 " size=0x%016" PRIx64 "\n", + info.address, info.size); + break; } }); if (has_stencil) { m_memory_tracker.ForEachUploadRange( - info.stencil_address, info.stencil_size, false, [](uint64_t, uint64_t) noexcept {}, - []() noexcept {}); + info.stencil_address, info.stencil_size, false, + [](uint64_t, uint64_t) noexcept {}, + [&]() noexcept { + if (!info.stencil_load_clear) { + m_tiler.DetileStencil( + ctx, static_cast(cached->image), info, + stencil_source, false); + } + }); + cached->stencil_initialized = true; } auto* image = static_cast(cached->image); m_images.push_back(std::move(cached)); diff --git a/src/graphics/host_gpu/renderer/tiler.cpp b/src/graphics/host_gpu/renderer/tiler.cpp index b1173ce9a..c77a3137e 100644 --- a/src/graphics/host_gpu/renderer/tiler.cpp +++ b/src/graphics/host_gpu/renderer/tiler.cpp @@ -50,8 +50,8 @@ void Tiler::DetileImage(GraphicContext* ctx, GpuTextureVulkanImage* image, } void Tiler::DetileImage(GraphicContext* ctx, DepthStencilVulkanImage* image, - const DepthTargetInfo& info, const BufferImageCopySource& source, - bool refresh) const { + const DepthTargetInfo& info, const BufferImageCopySource& source, + bool refresh) const { const bool d16 = info.guest_format == Prospero::GpuEnumValue(Prospero::BufferFormat::k16UNorm) && info.format == VK_FORMAT_D16_UNORM && info.bytes_per_element == 2; const bool d32 = @@ -80,7 +80,54 @@ void Tiler::DetileImage(GraphicContext* ctx, DepthStencilVulkanImage* image, TileConvertTiledToLinearDepth(linear.Data(), reinterpret_cast(source.address), info.guest_format, info.width, info.height, info.pitch, info.size); - UtilFillDepthImage(ctx, image, linear.Data(), info.size, info.pitch); + UtilFillImage(ctx, image, linear.Data(), info.size, info.pitch, VK_IMAGE_ASPECT_DEPTH_BIT); +} + +void Tiler::DetileStencil(GraphicContext* ctx, DepthStencilVulkanImage* image, + const DepthTargetInfo& info, const BufferImageCopySource& source, + bool refresh) const { + TileSizeAlign stencil_size {}; + TileSizeAlign htile_size {}; + TileSizeAlign depth_size {}; + const auto stencil_format = Prospero::GpuEnumValue(Prospero::BufferFormat::k8UInt); + const auto stencil_pitch = TileGetTexturePitch( + stencil_format, info.width, 1, Prospero::GpuEnumValue(Prospero::TileMode::kDepth)); + const bool supported_layout = + TileGetDepthSize(info.width, info.height, 0, + Prospero::GpuEnumValue(Prospero::DepthFormat::kZ32F), + Prospero::GpuEnumValue(Prospero::StencilFormat::k8UInt), + info.htile_address != 0, &stencil_size, &htile_size, &depth_size) && + stencil_size.size == info.stencil_size && stencil_size.align == 65536 && + stencil_pitch != 0; + if (ctx == nullptr || image == nullptr || info.address == 0 || info.size == 0 || + info.stencil_address == 0 || info.stencil_size == 0 || info.width == 0 || + info.height == 0 || info.format != VK_FORMAT_D32_SFLOAT_S8_UINT || + info.guest_format != Prospero::GpuEnumValue(Prospero::BufferFormat::k32Float) || + info.bytes_per_element != 4 || + info.tile_mode != Prospero::GpuEnumValue(Prospero::TileMode::kDepth) || + info.stencil_htile_compressed || !supported_layout || !source.cpu_current || + source.address != info.stencil_address || + source.size != info.stencil_size || (source.buffer == nullptr && source.offset != 0)) { + EXIT("Tiler: unsupported stencil detile, ctx=%p image=%p source_buffer=%p " + "source=0x%016" PRIx64 "+0x%016" PRIx64 " offset=0x%016" PRIx64 + " current=%d stencil=0x%016" PRIx64 "+0x%016" PRIx64 + " extent=%ux%u depth_pitch=%u stencil_pitch=%u tile=%u format=%d guest_format=%u " + "bpe=%u\n", + static_cast(ctx), static_cast(image), + static_cast(source.buffer), source.address, source.size, source.offset, + source.cpu_current, info.stencil_address, info.stencil_size, info.width, info.height, + info.pitch, stencil_pitch, info.tile_mode, static_cast(info.format), + info.guest_format, info.bytes_per_element); + } + if (refresh) { + VulkanDeviceWaitIdle(ctx); + } + UtilScratchBuffer linear(info.stencil_size); + TileConvertTiledToLinearDepth(linear.Data(), reinterpret_cast(source.address), + stencil_format, info.width, info.height, stencil_pitch, + info.stencil_size); + UtilFillImage(ctx, image, linear.Data(), info.stencil_size, stencil_pitch, + VK_IMAGE_ASPECT_STENCIL_BIT); } void Tiler::TileImage(void* dst, const void* src, const RenderTargetInfo& info) const { diff --git a/src/graphics/host_gpu/renderer/tiler.h b/src/graphics/host_gpu/renderer/tiler.h index c137d407c..5226c0816 100644 --- a/src/graphics/host_gpu/renderer/tiler.h +++ b/src/graphics/host_gpu/renderer/tiler.h @@ -21,6 +21,9 @@ public: void DetileImage(GraphicContext* ctx, DepthStencilVulkanImage* image, const DepthTargetInfo& info, const BufferImageCopySource& source, bool refresh) const; + void DetileStencil(GraphicContext* ctx, DepthStencilVulkanImage* image, + const DepthTargetInfo& info, const BufferImageCopySource& source, + bool refresh) const; void TileImage(void* dst, const void* src, const RenderTargetInfo& info) const; void TileImage(void* dst, const void* src, const DepthTargetInfo& info) const; }; diff --git a/src/graphics/host_gpu/utils.cpp b/src/graphics/host_gpu/utils.cpp index 4239b20ac..8813401e9 100644 --- a/src/graphics/host_gpu/utils.cpp +++ b/src/graphics/host_gpu/utils.cpp @@ -12,6 +12,7 @@ #include #include +#include #include #include @@ -59,9 +60,6 @@ bool UtilBufferIsTiled(uint64_t vaddr, uint64_t size) { } void VulkanDeviceWaitIdle(GraphicContext* ctx) { - EXIT_IF(ctx == nullptr); - EXIT_IF(ctx->device == nullptr); - Common::Mutex* locked[GraphicContext::QUEUES_NUM] {}; int locked_num = 0; @@ -99,8 +97,6 @@ void VulkanDeviceWaitIdle(GraphicContext* ctx) { } void UtilResetImageViews(VulkanImage* image) { - EXIT_IF(image == nullptr); - for (auto& view: image->image_view) { view = nullptr; } @@ -111,8 +107,6 @@ void UtilCreateImageView(GraphicContext* ctx, VulkanImage* image, int view_index VkComponentMapping components, uint32_t base_array_layer, uint32_t base_mip_level, uint32_t layer_count, uint32_t level_count, VkFormat view_format, VkImageUsageFlags view_usage) { - EXIT_IF(ctx == nullptr); - EXIT_IF(image == nullptr); if (view_index < 0 || view_index >= VulkanImage::VIEW_MAX || image->image == nullptr || image->image_view[view_index] != nullptr) { EXIT("invalid image-view creation target: image=%p index=%d current_view=%d\n", @@ -153,6 +147,142 @@ static void SetImageLayout(VkCommandBuffer buffer, VulkanImage* dst_image, uint3 uint32_t levels, VkImageAspectFlags aspect_mask, VkImageLayout old_image_layout, VkImageLayout new_image_layout); +template +static void ExecuteImmediateUtilCommands(const Recorder& recorder) { + // Keep synchronous utility submission behind one boundary so adopting a central scheduler does + // not touch every caller. + CommandBuffer command(GraphicContext::QUEUE_UTIL); + command.Begin(); + recorder(&command, command.GetPool()->buffers[command.GetIndex()]); + command.End(); + command.Execute(); + command.WaitForFence(); +} + +[[nodiscard]] static bool IsSingleImageAspect(VkImageAspectFlags aspect) { + return aspect == VK_IMAGE_ASPECT_COLOR_BIT || aspect == VK_IMAGE_ASPECT_DEPTH_BIT || + aspect == VK_IMAGE_ASPECT_STENCIL_BIT; +} + +[[nodiscard]] static VkBufferImageCopy +MakeBufferImageCopy(VkDeviceSize buffer_offset, uint32_t buffer_pitch, VkImageAspectFlags aspect, + uint32_t mip_level, uint32_t array_layer, VkOffset3D image_offset, + VkExtent3D image_extent) { + VkBufferImageCopy copy {}; + copy.bufferOffset = buffer_offset; + copy.bufferRowLength = buffer_pitch != image_extent.width ? buffer_pitch : 0; + copy.bufferImageHeight = 0; + copy.imageSubresource.aspectMask = aspect; + copy.imageSubresource.mipLevel = mip_level; + copy.imageSubresource.baseArrayLayer = array_layer; + copy.imageSubresource.layerCount = 1; + copy.imageOffset = image_offset; + copy.imageExtent = image_extent; + return copy; +} + +static void SetBufferMemoryBarrier(VkCommandBuffer command, VkBuffer buffer, VkDeviceSize offset, + VkDeviceSize size, VkAccessFlags src_access, + VkAccessFlags dst_access, VkPipelineStageFlags src_stage, + VkPipelineStageFlags dst_stage) { + VkBufferMemoryBarrier barrier {}; + barrier.sType = VK_STRUCTURE_TYPE_BUFFER_MEMORY_BARRIER; + barrier.srcAccessMask = src_access; + barrier.dstAccessMask = dst_access; + barrier.srcQueueFamilyIndex = VK_QUEUE_FAMILY_IGNORED; + barrier.dstQueueFamilyIndex = VK_QUEUE_FAMILY_IGNORED; + barrier.buffer = buffer; + barrier.offset = offset; + barrier.size = size; + vkCmdPipelineBarrier(command, src_stage, dst_stage, 0, 0, nullptr, 1, &barrier, 0, nullptr); +} + +[[nodiscard]] static VkImageAspectFlags GetDepthStencilAspects(VkFormat format) { + switch (format) { + case VK_FORMAT_D16_UNORM: + case VK_FORMAT_D32_SFLOAT: return VK_IMAGE_ASPECT_DEPTH_BIT; + case VK_FORMAT_D16_UNORM_S8_UINT: + case VK_FORMAT_D24_UNORM_S8_UINT: + case VK_FORMAT_D32_SFLOAT_S8_UINT: + return VK_IMAGE_ASPECT_DEPTH_BIT | VK_IMAGE_ASPECT_STENCIL_BIT; + default: EXIT("unsupported depth/stencil image format: %d\n", static_cast(format)); + } +} + +[[nodiscard]] static VkImageAspectFlags GetTransferAspects(const VulkanImage& image, + VkImageAspectFlags copy_aspect) { + switch (copy_aspect) { + case VK_IMAGE_ASPECT_COLOR_BIT: return VK_IMAGE_ASPECT_COLOR_BIT; + case VK_IMAGE_ASPECT_DEPTH_BIT: + case VK_IMAGE_ASPECT_STENCIL_BIT: { + const auto aspects = GetDepthStencilAspects(image.format); + if ((copy_aspect & aspects) == 0) { + EXIT("image transfer aspect is unavailable: format=%d copy=0x%x available=0x%x\n", + static_cast(image.format), copy_aspect, aspects); + } + return aspects; + } + default: EXIT("unsupported image transfer aspect: 0x%x\n", copy_aspect); + } +} + +static void RecordBufferToImageCopy(CommandBuffer& command, VulkanBuffer& src_buffer, + VulkanImage& dst_image, + std::span regions, + VkImageAspectFlags transition_aspects, + VkImageLayout initial_layout, VkImageLayout final_layout) { + auto* vk_command = command.GetPool()->buffers[command.GetIndex()]; + SetBufferMemoryBarrier(vk_command, src_buffer.buffer, 0, VK_WHOLE_SIZE, + VK_ACCESS_MEMORY_WRITE_BIT, VK_ACCESS_TRANSFER_READ_BIT, + VK_PIPELINE_STAGE_ALL_COMMANDS_BIT, VK_PIPELINE_STAGE_TRANSFER_BIT); + SetImageLayout(vk_command, &dst_image, 0, VK_REMAINING_MIP_LEVELS, transition_aspects, + initial_layout, VK_IMAGE_LAYOUT_TRANSFER_DST_OPTIMAL); + vkCmdCopyBufferToImage(vk_command, src_buffer.buffer, dst_image.image, + VK_IMAGE_LAYOUT_TRANSFER_DST_OPTIMAL, + static_cast(regions.size()), regions.data()); + SetImageLayout(vk_command, &dst_image, 0, VK_REMAINING_MIP_LEVELS, transition_aspects, + VK_IMAGE_LAYOUT_TRANSFER_DST_OPTIMAL, final_layout); +} + +[[nodiscard]] static std::span +ConvertBufferImageCopies(std::span regions) { + static thread_local std::vector vk_regions; + vk_regions.resize(regions.size()); + + for (size_t i = 0; i < regions.size(); i++) { + const auto& r = regions[i]; + vk_regions[i] = + MakeBufferImageCopy(r.offset, r.pitch, VK_IMAGE_ASPECT_COLOR_BIT, r.dst_level, + r.dst_layer, {r.dst_x, r.dst_y, r.dst_z}, + {r.width, (r.copy_height != 0 ? r.copy_height : r.height), 1}); + } + return vk_regions; +} + +static void RecordImageToBuffer(CommandBuffer& command, VulkanImage& src_image, + VulkanBuffer& dst_buffer, uint32_t dst_pitch, + VkImageLayout final_layout, VkImageAspectFlags aspect) { + auto* vk_command = command.GetPool()->buffers[command.GetIndex()]; + const auto transition_aspects = GetTransferAspects(src_image, aspect); + + SetImageLayout(vk_command, &src_image, 0, VK_REMAINING_MIP_LEVELS, transition_aspects, + src_image.layout, VK_IMAGE_LAYOUT_TRANSFER_SRC_OPTIMAL); + + const auto region = MakeBufferImageCopy(0, dst_pitch, aspect, 0, 0, {0, 0, 0}, + {src_image.extent.width, src_image.extent.height, 1}); + SetBufferMemoryBarrier(vk_command, dst_buffer.buffer, 0, VK_WHOLE_SIZE, + VK_ACCESS_MEMORY_READ_BIT, VK_ACCESS_TRANSFER_WRITE_BIT, + VK_PIPELINE_STAGE_ALL_COMMANDS_BIT, VK_PIPELINE_STAGE_TRANSFER_BIT); + vkCmdCopyImageToBuffer(vk_command, src_image.image, VK_IMAGE_LAYOUT_TRANSFER_SRC_OPTIMAL, + dst_buffer.buffer, 1, ®ion); + SetBufferMemoryBarrier(vk_command, dst_buffer.buffer, 0, VK_WHOLE_SIZE, + VK_ACCESS_TRANSFER_WRITE_BIT, VK_ACCESS_HOST_READ_BIT, + VK_PIPELINE_STAGE_TRANSFER_BIT, VK_PIPELINE_STAGE_HOST_BIT); + + SetImageLayout(vk_command, &src_image, 0, VK_REMAINING_MIP_LEVELS, transition_aspects, + VK_IMAGE_LAYOUT_TRANSFER_SRC_OPTIMAL, final_layout); +} + class ReusableStagingBuffer { public: ReusableStagingBuffer() = default; @@ -160,118 +290,49 @@ public: KYTY_CLASS_NO_COPY(ReusableStagingBuffer); void UploadToBuffer(GraphicContext* ctx, VulkanBuffer* dst_buffer, const void* src_data, - uint64_t size, uint64_t dst_offset = 0) { - Common::LockGuard lock(m_mutex); + uint64_t size, uint64_t dst_offset) { + RecordUpload(ctx, src_data, size, [&](CommandBuffer*, VkCommandBuffer vk_command) { + SetBufferMemoryBarrier(vk_command, m_buffer.buffer, 0, size, VK_ACCESS_MEMORY_WRITE_BIT, + VK_ACCESS_TRANSFER_READ_BIT, VK_PIPELINE_STAGE_ALL_COMMANDS_BIT, + VK_PIPELINE_STAGE_TRANSFER_BIT); + SetBufferMemoryBarrier(vk_command, dst_buffer->buffer, dst_offset, size, + VK_ACCESS_MEMORY_READ_BIT | VK_ACCESS_MEMORY_WRITE_BIT, + VK_ACCESS_TRANSFER_WRITE_BIT, VK_PIPELINE_STAGE_ALL_COMMANDS_BIT, + VK_PIPELINE_STAGE_TRANSFER_BIT); + const VkBufferCopy copy_region {.srcOffset = 0, .dstOffset = dst_offset, .size = size}; + vkCmdCopyBuffer(vk_command, m_buffer.buffer, dst_buffer->buffer, 1, ©_region); + SetBufferMemoryBarrier( + vk_command, dst_buffer->buffer, dst_offset, size, VK_ACCESS_TRANSFER_WRITE_BIT, + VK_ACCESS_VERTEX_ATTRIBUTE_READ_BIT | VK_ACCESS_INDEX_READ_BIT | + VK_ACCESS_SHADER_READ_BIT | VK_ACCESS_SHADER_WRITE_BIT, + VK_PIPELINE_STAGE_TRANSFER_BIT, + VK_PIPELINE_STAGE_VERTEX_INPUT_BIT | VK_PIPELINE_STAGE_VERTEX_SHADER_BIT | + VK_PIPELINE_STAGE_FRAGMENT_SHADER_BIT | VK_PIPELINE_STAGE_COMPUTE_SHADER_BIT); + }); + } - CopyFromHost(ctx, src_data, size, VK_BUFFER_USAGE_TRANSFER_SRC_BIT); - VulkanDeviceWaitIdle(ctx); - - CommandBuffer buffer(GraphicContext::QUEUE_UTIL); - EXIT_NOT_IMPLEMENTED(buffer.IsInvalid()); - auto* vk_buffer = buffer.GetPool()->buffers[buffer.GetIndex()]; - buffer.Begin(); - VkBufferMemoryBarrier pre_barrier {}; - pre_barrier.sType = VK_STRUCTURE_TYPE_BUFFER_MEMORY_BARRIER; - pre_barrier.srcAccessMask = VK_ACCESS_MEMORY_READ_BIT | VK_ACCESS_MEMORY_WRITE_BIT; - pre_barrier.dstAccessMask = VK_ACCESS_TRANSFER_WRITE_BIT; - pre_barrier.srcQueueFamilyIndex = VK_QUEUE_FAMILY_IGNORED; - pre_barrier.dstQueueFamilyIndex = VK_QUEUE_FAMILY_IGNORED; - pre_barrier.buffer = dst_buffer->buffer; - pre_barrier.offset = dst_offset; - pre_barrier.size = size; - vkCmdPipelineBarrier(vk_buffer, VK_PIPELINE_STAGE_ALL_COMMANDS_BIT, - VK_PIPELINE_STAGE_TRANSFER_BIT, 0, 0, nullptr, 1, &pre_barrier, 0, - nullptr); - VkBufferCopy copy_region {}; - copy_region.dstOffset = dst_offset; - copy_region.size = size; - vkCmdCopyBuffer(vk_buffer, m_buffer.buffer, dst_buffer->buffer, 1, ©_region); - VkBufferMemoryBarrier post_barrier {}; - post_barrier.sType = VK_STRUCTURE_TYPE_BUFFER_MEMORY_BARRIER; - post_barrier.srcAccessMask = VK_ACCESS_TRANSFER_WRITE_BIT; - post_barrier.dstAccessMask = VK_ACCESS_VERTEX_ATTRIBUTE_READ_BIT | - VK_ACCESS_INDEX_READ_BIT | VK_ACCESS_SHADER_READ_BIT | - VK_ACCESS_SHADER_WRITE_BIT; - post_barrier.srcQueueFamilyIndex = VK_QUEUE_FAMILY_IGNORED; - post_barrier.dstQueueFamilyIndex = VK_QUEUE_FAMILY_IGNORED; - post_barrier.buffer = dst_buffer->buffer; - post_barrier.offset = dst_offset; - post_barrier.size = size; - vkCmdPipelineBarrier( - vk_buffer, VK_PIPELINE_STAGE_TRANSFER_BIT, - VK_PIPELINE_STAGE_VERTEX_INPUT_BIT | VK_PIPELINE_STAGE_VERTEX_SHADER_BIT | - VK_PIPELINE_STAGE_FRAGMENT_SHADER_BIT | VK_PIPELINE_STAGE_COMPUTE_SHADER_BIT, - 0, 0, nullptr, 1, &post_barrier, 0, nullptr); - buffer.End(); - buffer.Execute(); - buffer.WaitForFence(); + void UploadToImage(GraphicContext* ctx, VulkanImage* image, const void* src_data, uint64_t size, + uint32_t src_pitch, VkImageAspectFlags copy_aspect, + VkImageLayout initial_layout, VkImageLayout final_layout) { + const auto transition_aspects = GetTransferAspects(*image, copy_aspect); + RecordUpload(ctx, src_data, size, [&](CommandBuffer* command, VkCommandBuffer) { + const auto region = MakeBufferImageCopy(0, src_pitch, copy_aspect, 0, 0, {0, 0, 0}, + {image->extent.width, image->extent.height, 1}); + RecordBufferToImageCopy(*command, m_buffer, *image, + std::span(®ion, 1), + transition_aspects, initial_layout, final_layout); + }); } void UploadToImage(GraphicContext* ctx, VulkanImage* dst_image, const void* src_data, - uint64_t size, uint32_t src_pitch, uint64_t dst_layout) { - Common::LockGuard lock(m_mutex); - - CopyFromHost(ctx, src_data, size, VK_BUFFER_USAGE_TRANSFER_SRC_BIT); - - CommandBuffer buffer(GraphicContext::QUEUE_UTIL); - // buffer.SetQueue(GraphicContext::QUEUE_UTIL); - - EXIT_NOT_IMPLEMENTED(buffer.IsInvalid()); - - buffer.Begin(); - UtilBufferToImage(&buffer, &m_buffer, src_pitch, dst_image, dst_layout); - buffer.End(); - buffer.Execute(); - buffer.WaitForFence(); - } - - void UploadToDepthImage(GraphicContext* ctx, DepthStencilVulkanImage* image, - const void* src_data, uint64_t size, uint32_t src_pitch) { - Common::LockGuard lock(m_mutex); - CopyFromHost(ctx, src_data, size, VK_BUFFER_USAGE_TRANSFER_SRC_BIT); - CommandBuffer buffer(GraphicContext::QUEUE_UTIL); - EXIT_NOT_IMPLEMENTED(buffer.IsInvalid()); - buffer.Begin(); - auto* vk_buffer = buffer.GetPool()->buffers[buffer.GetIndex()]; - VkImageAspectFlags aspects = VK_IMAGE_ASPECT_DEPTH_BIT; - if (image->format == VK_FORMAT_D16_UNORM_S8_UINT || - image->format == VK_FORMAT_D24_UNORM_S8_UINT || - image->format == VK_FORMAT_D32_SFLOAT_S8_UINT) { - aspects |= VK_IMAGE_ASPECT_STENCIL_BIT; - } - SetImageLayout(vk_buffer, image, 0, 1, aspects, VK_IMAGE_LAYOUT_UNDEFINED, - VK_IMAGE_LAYOUT_TRANSFER_DST_OPTIMAL); - VkBufferImageCopy region {}; - region.bufferRowLength = src_pitch == image->extent.width ? 0 : src_pitch; - region.imageSubresource.aspectMask = VK_IMAGE_ASPECT_DEPTH_BIT; - region.imageSubresource.layerCount = 1; - region.imageExtent = {image->extent.width, image->extent.height, 1}; - vkCmdCopyBufferToImage(vk_buffer, m_buffer.buffer, image->image, - VK_IMAGE_LAYOUT_TRANSFER_DST_OPTIMAL, 1, ®ion); - SetImageLayout(vk_buffer, image, 0, 1, aspects, VK_IMAGE_LAYOUT_TRANSFER_DST_OPTIMAL, - VK_IMAGE_LAYOUT_DEPTH_STENCIL_ATTACHMENT_OPTIMAL); - buffer.End(); - buffer.Execute(); - buffer.WaitForFence(); - } - - void UploadToImage(GraphicContext* ctx, VulkanImage* dst_image, const void* src_data, - uint64_t size, const std::vector& regions, + uint64_t size, std::span regions, uint64_t dst_layout) { - Common::LockGuard lock(m_mutex); - - CopyFromHost(ctx, src_data, size, VK_BUFFER_USAGE_TRANSFER_SRC_BIT); - - CommandBuffer buffer(GraphicContext::QUEUE_UTIL); - // buffer.SetQueue(GraphicContext::QUEUE_UTIL); - - EXIT_NOT_IMPLEMENTED(buffer.IsInvalid()); - - buffer.Begin(); - UtilBufferToImage(&buffer, &m_buffer, dst_image, regions, dst_layout); - buffer.End(); - buffer.Execute(); - buffer.WaitForFence(); + RecordUpload(ctx, src_data, size, [&](CommandBuffer* command, VkCommandBuffer) { + RecordBufferToImageCopy(*command, m_buffer, *dst_image, + ConvertBufferImageCopies(regions), VK_IMAGE_ASPECT_COLOR_BIT, + VK_IMAGE_LAYOUT_UNDEFINED, + static_cast(dst_layout)); + }); } void DownloadFromImage(GraphicContext* ctx, void* dst_data, uint64_t size, uint32_t dst_pitch, @@ -280,36 +341,34 @@ public: EnsureBuffer(ctx, size, VK_BUFFER_USAGE_TRANSFER_DST_BIT); - CommandBuffer buffer(GraphicContext::QUEUE_UTIL); - // buffer.SetQueue(GraphicContext::QUEUE_UTIL); - - EXIT_NOT_IMPLEMENTED(buffer.IsInvalid()); - - buffer.Begin(); - UtilImageToBuffer(&buffer, src_image, &m_buffer, dst_pitch, src_layout, aspect); - buffer.End(); - buffer.Execute(); - buffer.WaitForFence(); + ExecuteImmediateUtilCommands([&](CommandBuffer* command, VkCommandBuffer) { + RecordImageToBuffer(*command, *src_image, m_buffer, dst_pitch, + static_cast(src_layout), aspect); + }); std::memcpy(dst_data, m_mapped_data, size); } private: + template + void RecordUpload(GraphicContext* ctx, const void* src_data, uint64_t size, + const Recorder& recorder) { + Common::LockGuard lock(m_mutex); + CopyFromHost(ctx, src_data, size, VK_BUFFER_USAGE_TRANSFER_SRC_BIT); + if constexpr (WaitIdle) { + VulkanDeviceWaitIdle(ctx); + } + ExecuteImmediateUtilCommands(recorder); + } + void CopyFromHost(GraphicContext* ctx, const void* src_data, uint64_t size, VkBufferUsageFlags usage) { - EXIT_IF(ctx == nullptr); - EXIT_IF(src_data == nullptr); - EXIT_IF(size == 0); - EnsureBuffer(ctx, size, usage); std::memcpy(m_mapped_data, src_data, size); } void EnsureBuffer(GraphicContext* ctx, uint64_t size, VkBufferUsageFlags usage) { - EXIT_IF(ctx == nullptr); - EXIT_IF(size == 0); - if (m_capacity < size || m_buffer.usage != usage) { if (m_buffer.buffer != nullptr) { VulkanUnmapMemory(ctx, &m_buffer.memory); @@ -324,7 +383,6 @@ private: m_capacity = size; VulkanMapMemory(ctx, &m_buffer.memory, &m_mapped_data); - EXIT_IF(m_mapped_data == nullptr); } } @@ -337,7 +395,6 @@ private: // Do not replace with one static ReusableStagingBuffer g_texture_staging_buffer; static ReusableStagingBuffer g_vertex_staging_buffer; -static ReusableStagingBuffer g_index_staging_buffer; static ReusableStagingBuffer g_readback_staging_buffer; static Common::Mutex g_compressed_image_copy_mutex; @@ -387,7 +444,6 @@ static uint32_t GetFormatTexelSize(VkFormat format) { } static VulkanBuffer* GetCompressedImageCopyBuffer(GraphicContext* ctx, uint64_t size) { - EXIT_IF(ctx == nullptr); EXIT_IF(size == 0); constexpr uint64_t buffer_size = 64ull * 1024ull * 1024ull; @@ -409,7 +465,6 @@ static ReusableStagingBuffer* GetStagingBuffer(StagingBufferType type) { switch (type) { case StagingBufferType::Texture: return &g_texture_staging_buffer; case StagingBufferType::Vertex: return &g_vertex_staging_buffer; - case StagingBufferType::Index: return &g_index_staging_buffer; case StagingBufferType::ReadBack: return &g_readback_staging_buffer; default: EXIT("unknown staging buffer type\n"); } @@ -420,9 +475,6 @@ static ReusableStagingBuffer* GetStagingBuffer(StagingBufferType type) { static void SetImageLayout(VkCommandBuffer buffer, VulkanImage* dst_image, uint32_t base_level, uint32_t levels, VkImageAspectFlags aspect_mask, VkImageLayout old_image_layout, VkImageLayout new_image_layout) { - EXIT_IF(buffer == nullptr); - EXIT_IF(dst_image == nullptr); - if ((old_image_layout == VK_IMAGE_LAYOUT_TRANSFER_DST_OPTIMAL && new_image_layout == VK_IMAGE_LAYOUT_TRANSFER_SRC_OPTIMAL) || (old_image_layout == VK_IMAGE_LAYOUT_TRANSFER_DST_OPTIMAL && @@ -477,6 +529,9 @@ static void SetImageLayout(VkCommandBuffer buffer, VulkanImage* dst_image, uint3 case VK_IMAGE_LAYOUT_TRANSFER_DST_OPTIMAL: image_memory_barrier.srcAccessMask = VK_ACCESS_TRANSFER_WRITE_BIT; break; + case VK_IMAGE_LAYOUT_TRANSFER_SRC_OPTIMAL: + image_memory_barrier.srcAccessMask = VK_ACCESS_TRANSFER_READ_BIT; + break; case VK_IMAGE_LAYOUT_PREINITIALIZED: image_memory_barrier.srcAccessMask = VK_ACCESS_HOST_WRITE_BIT; break; @@ -528,7 +583,7 @@ static void SetImageLayout(VkCommandBuffer buffer, VulkanImage* dst_image, uint3 static bool CopyBlockImageToCompressedImage(VkCommandBuffer vk_buffer, const ImageImageCopy& r, VulkanImage* dst_image) { - if (r.src_image == nullptr || dst_image == nullptr || !UtilIsBcFormat(dst_image->format)) { + if (!UtilIsBcFormat(dst_image->format)) { return false; } @@ -548,46 +603,26 @@ static bool CopyBlockImageToCompressedImage(VkCommandBuffer vk_buffer, const Ima auto* copy_buffer = GetCompressedImageCopyBuffer(ctx, static_cast(block_width) * block_height * dst_block_size); - VkBufferImageCopy src_region {}; - src_region.bufferOffset = 0; - src_region.bufferRowLength = 0; - src_region.bufferImageHeight = 0; - src_region.imageSubresource.aspectMask = VK_IMAGE_ASPECT_COLOR_BIT; - src_region.imageSubresource.mipLevel = r.src_level; - src_region.imageSubresource.baseArrayLayer = r.src_layer; - src_region.imageSubresource.layerCount = 1; - src_region.imageOffset = {r.src_x, r.src_y, 0}; - src_region.imageExtent = {block_width, block_height, 1}; + const auto src_region = + MakeBufferImageCopy(0, 0, VK_IMAGE_ASPECT_COLOR_BIT, r.src_level, r.src_layer, + {r.src_x, r.src_y, 0}, {block_width, block_height, 1}); auto src_layout = r.src_image->layout; SetImageLayout(vk_buffer, r.src_image, r.src_level, 1, VK_IMAGE_ASPECT_COLOR_BIT, src_layout, VK_IMAGE_LAYOUT_TRANSFER_SRC_OPTIMAL); + SetBufferMemoryBarrier(vk_buffer, copy_buffer->buffer, 0, VK_WHOLE_SIZE, + VK_ACCESS_TRANSFER_READ_BIT, VK_ACCESS_TRANSFER_WRITE_BIT, + VK_PIPELINE_STAGE_TRANSFER_BIT, VK_PIPELINE_STAGE_TRANSFER_BIT); vkCmdCopyImageToBuffer(vk_buffer, r.src_image->image, VK_IMAGE_LAYOUT_TRANSFER_SRC_OPTIMAL, copy_buffer->buffer, 1, &src_region); + SetBufferMemoryBarrier(vk_buffer, copy_buffer->buffer, 0, VK_WHOLE_SIZE, + VK_ACCESS_TRANSFER_WRITE_BIT, VK_ACCESS_TRANSFER_READ_BIT, + VK_PIPELINE_STAGE_TRANSFER_BIT, VK_PIPELINE_STAGE_TRANSFER_BIT); - VkBufferMemoryBarrier barrier {}; - barrier.sType = VK_STRUCTURE_TYPE_BUFFER_MEMORY_BARRIER; - barrier.srcAccessMask = VK_ACCESS_TRANSFER_WRITE_BIT; - barrier.dstAccessMask = VK_ACCESS_TRANSFER_READ_BIT; - barrier.srcQueueFamilyIndex = VK_QUEUE_FAMILY_IGNORED; - barrier.dstQueueFamilyIndex = VK_QUEUE_FAMILY_IGNORED; - barrier.buffer = copy_buffer->buffer; - barrier.offset = 0; - barrier.size = VK_WHOLE_SIZE; - vkCmdPipelineBarrier(vk_buffer, VK_PIPELINE_STAGE_TRANSFER_BIT, VK_PIPELINE_STAGE_TRANSFER_BIT, - 0, 0, nullptr, 1, &barrier, 0, nullptr); - - VkBufferImageCopy dst_region {}; - dst_region.bufferOffset = 0; - dst_region.bufferRowLength = 0; - dst_region.bufferImageHeight = 0; - dst_region.imageSubresource.aspectMask = VK_IMAGE_ASPECT_COLOR_BIT; - dst_region.imageSubresource.mipLevel = r.dst_level; - dst_region.imageSubresource.baseArrayLayer = r.dst_layer; - dst_region.imageSubresource.layerCount = 1; - dst_region.imageOffset = {r.dst_x, r.dst_y, 0}; - dst_region.imageExtent = {r.width, r.height, 1}; + const auto dst_region = + MakeBufferImageCopy(0, 0, VK_IMAGE_ASPECT_COLOR_BIT, r.dst_level, r.dst_layer, + {r.dst_x, r.dst_y, 0}, {r.width, r.height, 1}); vkCmdCopyBufferToImage(vk_buffer, copy_buffer->buffer, dst_image->image, VK_IMAGE_LAYOUT_TRANSFER_DST_OPTIMAL, 1, &dst_region); @@ -598,117 +633,8 @@ static bool CopyBlockImageToCompressedImage(VkCommandBuffer vk_buffer, const Ima return true; } -void UtilBufferToImage(CommandBuffer* buffer, VulkanBuffer* src_buffer, uint32_t src_pitch, - VulkanImage* dst_image, uint64_t dst_layout) { - EXIT_IF(src_buffer == nullptr); - EXIT_IF(src_buffer->buffer == nullptr); - EXIT_IF(dst_image == nullptr); - EXIT_IF(dst_image->image == nullptr); - - auto* vk_buffer = buffer->GetPool()->buffers[buffer->GetIndex()]; - - SetImageLayout(vk_buffer, dst_image, 0, 1, VK_IMAGE_ASPECT_COLOR_BIT, VK_IMAGE_LAYOUT_UNDEFINED, - VK_IMAGE_LAYOUT_TRANSFER_DST_OPTIMAL); - - VkBufferImageCopy region {}; - region.bufferOffset = 0; - region.bufferRowLength = (src_pitch != dst_image->extent.width ? src_pitch : 0); - region.bufferImageHeight = 0; - - region.imageSubresource.aspectMask = VK_IMAGE_ASPECT_COLOR_BIT; - region.imageSubresource.mipLevel = 0; - region.imageSubresource.baseArrayLayer = 0; - region.imageSubresource.layerCount = 1; - - region.imageOffset = {0, 0, 0}; - region.imageExtent = {dst_image->extent.width, dst_image->extent.height, 1}; - - vkCmdCopyBufferToImage(vk_buffer, src_buffer->buffer, dst_image->image, - VK_IMAGE_LAYOUT_TRANSFER_DST_OPTIMAL, 1, ®ion); - - SetImageLayout(vk_buffer, dst_image, 0, 1, VK_IMAGE_ASPECT_COLOR_BIT, - VK_IMAGE_LAYOUT_TRANSFER_DST_OPTIMAL, static_cast(dst_layout)); -} - -void UtilImageToBuffer(CommandBuffer* buffer, VulkanImage* src_image, VulkanBuffer* dst_buffer, - uint32_t dst_pitch, uint64_t src_layout, VkImageAspectFlags aspect) { - EXIT_IF(dst_buffer == nullptr); - EXIT_IF(dst_buffer->buffer == nullptr); - EXIT_IF(src_image == nullptr); - EXIT_IF(src_image->image == nullptr); - if (aspect != VK_IMAGE_ASPECT_COLOR_BIT && aspect != VK_IMAGE_ASPECT_DEPTH_BIT && - aspect != VK_IMAGE_ASPECT_STENCIL_BIT) { - EXIT("unsupported image readback aspect mask: 0x%x\n", aspect); - } - - auto* vk_buffer = buffer->GetPool()->buffers[buffer->GetIndex()]; - - SetImageLayout(vk_buffer, src_image, 0, 1, aspect, src_image->layout, - VK_IMAGE_LAYOUT_TRANSFER_SRC_OPTIMAL); - - VkBufferImageCopy region {}; - region.bufferOffset = 0; - region.bufferRowLength = (dst_pitch != src_image->extent.width ? dst_pitch : 0); - region.bufferImageHeight = 0; - - region.imageSubresource.aspectMask = aspect; - region.imageSubresource.mipLevel = 0; - region.imageSubresource.baseArrayLayer = 0; - region.imageSubresource.layerCount = 1; - - region.imageOffset = {0, 0, 0}; - region.imageExtent = {src_image->extent.width, src_image->extent.height, 1}; - - vkCmdCopyImageToBuffer(vk_buffer, src_image->image, VK_IMAGE_LAYOUT_TRANSFER_SRC_OPTIMAL, - dst_buffer->buffer, 1, ®ion); - - SetImageLayout(vk_buffer, src_image, 0, 1, aspect, VK_IMAGE_LAYOUT_TRANSFER_SRC_OPTIMAL, - static_cast(src_layout)); -} - -void UtilBufferToImage(CommandBuffer* buffer, VulkanBuffer* src_buffer, VulkanImage* dst_image, - const std::vector& regions, uint64_t dst_layout) { - EXIT_IF(src_buffer == nullptr); - EXIT_IF(src_buffer->buffer == nullptr); - EXIT_IF(dst_image == nullptr); - EXIT_IF(dst_image->image == nullptr); - - auto* vk_buffer = buffer->GetPool()->buffers[buffer->GetIndex()]; - - static thread_local std::vector vk_regions; - vk_regions.resize(regions.size()); - - uint32_t index = 0; - for (const auto& r: regions) { - auto& region = vk_regions[index]; - region.bufferOffset = r.offset; - region.bufferRowLength = (r.width != r.pitch ? r.pitch : 0); - region.bufferImageHeight = 0; - region.imageSubresource.aspectMask = VK_IMAGE_ASPECT_COLOR_BIT; - region.imageSubresource.mipLevel = r.dst_level; - region.imageSubresource.baseArrayLayer = r.dst_layer; - region.imageSubresource.layerCount = 1; - region.imageOffset = {r.dst_x, r.dst_y, r.dst_z}; - region.imageExtent = {r.width, (r.copy_height != 0 ? r.copy_height : r.height), 1}; - index++; - } - - SetImageLayout(vk_buffer, dst_image, 0, VK_REMAINING_MIP_LEVELS, VK_IMAGE_ASPECT_COLOR_BIT, - VK_IMAGE_LAYOUT_UNDEFINED, VK_IMAGE_LAYOUT_TRANSFER_DST_OPTIMAL); - - vkCmdCopyBufferToImage(vk_buffer, src_buffer->buffer, dst_image->image, - VK_IMAGE_LAYOUT_TRANSFER_DST_OPTIMAL, - static_cast(vk_regions.size()), vk_regions.data()); - - SetImageLayout(vk_buffer, dst_image, 0, VK_REMAINING_MIP_LEVELS, VK_IMAGE_ASPECT_COLOR_BIT, - VK_IMAGE_LAYOUT_TRANSFER_DST_OPTIMAL, static_cast(dst_layout)); -} - -void UtilImageToImage(CommandBuffer* buffer, const std::vector& regions, +void UtilImageToImage(CommandBuffer* buffer, std::span regions, VulkanImage* dst_image, uint64_t dst_layout) { - EXIT_IF(dst_image == nullptr); - EXIT_IF(dst_image->image == nullptr); - auto* vk_buffer = buffer->GetPool()->buffers[buffer->GetIndex()]; bool same_image_copy = false; @@ -799,11 +725,80 @@ void UtilImageToImage(CommandBuffer* buffer, const std::vector& VK_IMAGE_LAYOUT_TRANSFER_DST_OPTIMAL, static_cast(dst_layout)); } -void UtilBlitImage(CommandBuffer* buffer, VulkanImage* src_image, VulkanSwapchain* dst_swapchain) { - EXIT_IF(src_image == nullptr); - EXIT_IF(src_image->image == nullptr); - EXIT_IF(dst_swapchain == nullptr); +void UtilCopyImageWithBuffer(CommandBuffer* buffer, GraphicContext* ctx, VulkanImage* src_image, + VkImageAspectFlags src_aspect, VulkanImage* dst_image, + VkImageAspectFlags dst_aspect, uint32_t bytes_per_element, + uint64_t dst_layout) { + if (!IsSingleImageAspect(src_aspect) || !IsSingleImageAspect(dst_aspect) || + (bytes_per_element != 2 && bytes_per_element != 4) || src_image->layers != 1 || + dst_image->layers != 1 || src_image->mip_levels != 1 || dst_image->mip_levels != 1 || + src_image->extent.width == 0 || src_image->extent.height == 0 || + src_image->extent.width != dst_image->extent.width || + src_image->extent.height != dst_image->extent.height) { + EXIT("unsupported image-buffer-image copy, src_aspect=0x%x dst_aspect=0x%x bpe=%u " + "src=%ux%u/%u/%u dst=%ux%u/%u/%u\n", + src_aspect, dst_aspect, bytes_per_element, src_image->extent.width, + src_image->extent.height, src_image->layers, src_image->mip_levels, + dst_image->extent.width, dst_image->extent.height, dst_image->layers, + dst_image->mip_levels); + } + const uint64_t row_bytes = static_cast(src_image->extent.width) * bytes_per_element; + constexpr uint64_t MAX_COPY_BUFFER_SIZE = 16ull * 1024ull * 1024ull; + if (row_bytes == 0 || row_bytes > MAX_COPY_BUFFER_SIZE) { + EXIT("unsupported image-buffer-image row size: 0x%016" PRIx64 "\n", row_bytes); + } + const auto rows_per_chunk = static_cast( + std::min(src_image->extent.height, MAX_COPY_BUFFER_SIZE / row_bytes)); + const auto copy_buffer_size = row_bytes * rows_per_chunk; + + // Kyty does not yet have a cross-queue scheduler. Drain submitted work once before + // recording the recreate copy; commands already recorded on this buffer remain ordered by the + // barriers below. The preservation policy stays isolated here so a future scheduler can replace + // this synchronization without changing texture-cache ownership logic. + VulkanDeviceWaitIdle(ctx); + auto* copy_buffer = new VulkanBuffer; + copy_buffer->usage = VK_BUFFER_USAGE_TRANSFER_SRC_BIT | VK_BUFFER_USAGE_TRANSFER_DST_BIT; + copy_buffer->memory.property = VK_MEMORY_PROPERTY_DEVICE_LOCAL_BIT; + VulkanCreateBuffer(ctx, copy_buffer_size, copy_buffer); + buffer->DeleteAfterFence(copy_buffer); + + auto* vk_buffer = buffer->GetPool()->buffers[buffer->GetIndex()]; + const auto src_layout = src_image->layout; + const auto final_layout = static_cast(dst_layout); + SetImageLayout(vk_buffer, src_image, 0, 1, src_aspect, src_layout, + VK_IMAGE_LAYOUT_TRANSFER_SRC_OPTIMAL); + SetImageLayout(vk_buffer, dst_image, 0, 1, dst_aspect, dst_image->layout, + VK_IMAGE_LAYOUT_TRANSFER_DST_OPTIMAL); + + for (uint32_t row = 0; row < src_image->extent.height; row += rows_per_chunk) { + const auto rows = std::min(rows_per_chunk, src_image->extent.height - row); + auto copy = MakeBufferImageCopy(0, 0, src_aspect, 0, 0, {0, static_cast(row), 0}, + {src_image->extent.width, rows, 1}); + vkCmdCopyImageToBuffer(vk_buffer, src_image->image, VK_IMAGE_LAYOUT_TRANSFER_SRC_OPTIMAL, + copy_buffer->buffer, 1, ©); + + SetBufferMemoryBarrier(vk_buffer, copy_buffer->buffer, 0, copy_buffer_size, + VK_ACCESS_TRANSFER_WRITE_BIT, VK_ACCESS_TRANSFER_READ_BIT, + VK_PIPELINE_STAGE_TRANSFER_BIT, VK_PIPELINE_STAGE_TRANSFER_BIT); + + copy.imageSubresource.aspectMask = dst_aspect; + vkCmdCopyBufferToImage(vk_buffer, copy_buffer->buffer, dst_image->image, + VK_IMAGE_LAYOUT_TRANSFER_DST_OPTIMAL, 1, ©); + if (row + rows < src_image->extent.height) { + SetBufferMemoryBarrier(vk_buffer, copy_buffer->buffer, 0, copy_buffer_size, + VK_ACCESS_TRANSFER_READ_BIT, VK_ACCESS_TRANSFER_WRITE_BIT, + VK_PIPELINE_STAGE_TRANSFER_BIT, VK_PIPELINE_STAGE_TRANSFER_BIT); + } + } + + SetImageLayout(vk_buffer, src_image, 0, 1, src_aspect, VK_IMAGE_LAYOUT_TRANSFER_SRC_OPTIMAL, + src_layout); + SetImageLayout(vk_buffer, dst_image, 0, 1, dst_aspect, VK_IMAGE_LAYOUT_TRANSFER_DST_OPTIMAL, + final_layout); +} + +void UtilBlitImage(CommandBuffer* buffer, VulkanImage* src_image, VulkanSwapchain* dst_swapchain) { auto* vk_buffer = buffer->GetPool()->buffers[buffer->GetIndex()]; VulkanImage swapchain_image(VulkanImageType::Unknown); @@ -850,10 +845,7 @@ void UtilBlitImage(CommandBuffer* buffer, VulkanImage* src_image, VulkanSwapchai void VulkanCreateBuffer(GraphicContext* gctx, uint64_t size, VulkanBuffer* buffer) { KYTY_PROFILER_FUNCTION(); - EXIT_IF(gctx == nullptr); - EXIT_IF(buffer == nullptr); EXIT_IF(buffer->buffer != nullptr); - EXIT_IF(gctx->allocator == nullptr); VkBufferCreateInfo buffer_info {}; buffer_info.sType = VK_STRUCTURE_TYPE_BUFFER_CREATE_INFO; @@ -886,9 +878,6 @@ void VulkanCreateBuffer(GraphicContext* gctx, uint64_t size, VulkanBuffer* buffe void VulkanDeleteBuffer(GraphicContext* gctx, VulkanBuffer* buffer) { KYTY_PROFILER_FUNCTION(); - EXIT_IF(buffer == nullptr); - EXIT_IF(gctx == nullptr); - EXIT_IF(gctx->allocator == nullptr); EXIT_IF(buffer->memory.allocation == nullptr); VulkanUntrackAllocation(&buffer->memory); @@ -904,12 +893,7 @@ bool VulkanCreateImage(GraphicContext* gctx, const VkImageCreateInfo* image_info VulkanImage* image, VulkanMemory* memory) { KYTY_PROFILER_FUNCTION(); - EXIT_IF(gctx == nullptr); - EXIT_IF(gctx->allocator == nullptr); - EXIT_IF(image_info == nullptr); - EXIT_IF(image == nullptr); EXIT_IF(image->image != nullptr); - EXIT_IF(memory == nullptr); EXIT_IF(memory->allocation != nullptr); VmaAllocationCreateInfo alloc_info {}; @@ -938,10 +922,6 @@ bool VulkanCreateImage(GraphicContext* gctx, const VkImageCreateInfo* image_info void VulkanDeleteImage(GraphicContext* gctx, VulkanImage* image, VulkanMemory* memory) { KYTY_PROFILER_FUNCTION(); - EXIT_IF(gctx == nullptr); - EXIT_IF(gctx->allocator == nullptr); - EXIT_IF(image == nullptr); - EXIT_IF(memory == nullptr); EXIT_IF(memory->allocation == nullptr); VulkanUntrackAllocation(memory); @@ -957,153 +937,83 @@ void UtilFillImage(GraphicContext* ctx, VulkanImage* dst_image, const void* src_ uint32_t src_pitch, uint64_t dst_layout) { KYTY_PROFILER_FUNCTION(); - EXIT_IF(ctx == nullptr); - EXIT_IF(dst_image == nullptr); - EXIT_IF(src_data == nullptr); + EXIT_IF(size == 0); GetStagingBuffer(StagingBufferType::Texture) - ->UploadToImage(ctx, dst_image, src_data, size, src_pitch, dst_layout); + ->UploadToImage(ctx, dst_image, src_data, size, src_pitch, VK_IMAGE_ASPECT_COLOR_BIT, + VK_IMAGE_LAYOUT_UNDEFINED, static_cast(dst_layout)); } void UtilFillBuffer(GraphicContext* ctx, void* dst_data, uint64_t size, uint32_t dst_pitch, VulkanImage* src_image, uint64_t src_layout, VkImageAspectFlags aspect) { KYTY_PROFILER_FUNCTION(); - EXIT_IF(ctx == nullptr); - EXIT_IF(src_image == nullptr); - EXIT_IF(dst_data == nullptr); + EXIT_IF(size == 0); GetStagingBuffer(StagingBufferType::ReadBack) ->DownloadFromImage(ctx, dst_data, size, dst_pitch, src_image, src_layout, aspect); } -void UtilSetDepthLayoutOptimal(DepthStencilVulkanImage* image) { - CommandBuffer buffer(GraphicContext::QUEUE_UTIL); - // buffer.SetQueue(GraphicContext::QUEUE_UTIL); - - EXIT_NOT_IMPLEMENTED(buffer.IsInvalid()); - - buffer.Begin(); - - auto* vk_buffer = buffer.GetPool()->buffers[buffer.GetIndex()]; - - VkImageAspectFlags aspect_mask = VK_IMAGE_ASPECT_DEPTH_BIT; - - if (image->format == VK_FORMAT_D24_UNORM_S8_UINT || - image->format == VK_FORMAT_D32_SFLOAT_S8_UINT) { - aspect_mask |= VK_IMAGE_ASPECT_STENCIL_BIT; - } - - SetImageLayout(vk_buffer, image, 0, 1, aspect_mask, VK_IMAGE_LAYOUT_UNDEFINED, - VK_IMAGE_LAYOUT_DEPTH_STENCIL_ATTACHMENT_OPTIMAL); - - buffer.End(); - buffer.Execute(); - buffer.WaitForFence(); -} - void UtilSetImageLayoutOptimal(VulkanImage* image) { - CommandBuffer buffer(GraphicContext::QUEUE_UTIL); - // buffer.SetQueue(GraphicContext::QUEUE_UTIL); - - EXIT_NOT_IMPLEMENTED(buffer.IsInvalid()); - - buffer.Begin(); - - auto* vk_buffer = buffer.GetPool()->buffers[buffer.GetIndex()]; - - VkImageAspectFlags aspect_mask = VK_IMAGE_ASPECT_COLOR_BIT; - - SetImageLayout(vk_buffer, image, 0, 1, aspect_mask, image->layout, VK_IMAGE_LAYOUT_GENERAL); - - buffer.End(); - buffer.Execute(); - buffer.WaitForFence(); + ExecuteImmediateUtilCommands([&](CommandBuffer*, VkCommandBuffer vk_command) { + SetImageLayout(vk_command, image, 0, VK_REMAINING_MIP_LEVELS, VK_IMAGE_ASPECT_COLOR_BIT, + image->layout, VK_IMAGE_LAYOUT_GENERAL); + }); } void UtilFillImage(GraphicContext* ctx, VulkanImage* image, const void* src_data, uint64_t size, - const std::vector& regions, uint64_t dst_layout) { - EXIT_IF(ctx == nullptr); - EXIT_IF(image == nullptr); - EXIT_IF(src_data == nullptr); + std::span regions, uint64_t dst_layout) { + EXIT_IF(size == 0 || regions.empty()); GetStagingBuffer(StagingBufferType::Texture) ->UploadToImage(ctx, image, src_data, size, regions, dst_layout); } -void UtilFillImage(GraphicContext* ctx, const std::vector& regions, +void UtilFillImage(GraphicContext* ctx, std::span regions, VulkanImage* dst_image, uint64_t dst_layout) { - EXIT_IF(ctx == nullptr); - EXIT_IF(dst_image == nullptr); - - CommandBuffer buffer(GraphicContext::QUEUE_UTIL); - // buffer.SetQueue(GraphicContext::QUEUE_UTIL); - - EXIT_NOT_IMPLEMENTED(buffer.IsInvalid()); - - buffer.Begin(); - UtilImageToImage(&buffer, regions, dst_image, dst_layout); - buffer.End(); - buffer.Execute(); - buffer.WaitForFence(); + ExecuteImmediateUtilCommands([&](CommandBuffer* command, VkCommandBuffer) { + UtilImageToImage(command, regions, dst_image, dst_layout); + }); } -void UtilUploadBuffer(GraphicContext* ctx, StagingBufferType type, VulkanBuffer* dst_buffer, - const void* src_data, uint64_t size) { - EXIT_IF(ctx == nullptr); - EXIT_IF(dst_buffer == nullptr); - EXIT_IF(dst_buffer->buffer == nullptr); - EXIT_IF(src_data == nullptr); - - GetStagingBuffer(type)->UploadToBuffer(ctx, dst_buffer, src_data, size); -} - -void UtilFillDepthImage(GraphicContext* ctx, DepthStencilVulkanImage* dst_image, - const void* src_data, uint64_t size, uint32_t src_pitch) { - EXIT_IF(ctx == nullptr || dst_image == nullptr || src_data == nullptr); +void UtilFillImage(GraphicContext* ctx, DepthStencilVulkanImage* image, const void* src_data, + uint64_t size, uint32_t src_pitch, VkImageAspectFlags aspect) { + EXIT_IF(size == 0); GetStagingBuffer(StagingBufferType::Texture) - ->UploadToDepthImage(ctx, dst_image, src_data, size, src_pitch); + ->UploadToImage(ctx, image, src_data, size, src_pitch, aspect, image->layout, + VK_IMAGE_LAYOUT_DEPTH_STENCIL_ATTACHMENT_OPTIMAL); } void UtilUploadBuffer(GraphicContext* ctx, StagingBufferType type, VulkanBuffer* dst_buffer, uint64_t dst_offset, const void* src_data, uint64_t size) { - EXIT_IF(ctx == nullptr); - EXIT_IF(dst_buffer == nullptr || dst_buffer->buffer == nullptr || src_data == nullptr); + EXIT_IF(size == 0); EXIT_IF(dst_offset > dst_buffer->buffer_size || size > dst_buffer->buffer_size - dst_offset); GetStagingBuffer(type)->UploadToBuffer(ctx, dst_buffer, src_data, size, dst_offset); } void UtilCopyBuffer(VulkanBuffer* src_buffer, VulkanBuffer* dst_buffer, uint64_t size) { - EXIT_IF(src_buffer == nullptr); - EXIT_IF(src_buffer->buffer == nullptr); - EXIT_IF(dst_buffer == nullptr); - EXIT_IF(dst_buffer->buffer == nullptr); + EXIT_IF(size == 0 || size > src_buffer->buffer_size || size > dst_buffer->buffer_size); - CommandBuffer buffer(GraphicContext::QUEUE_UTIL); - // buffer.SetQueue(GraphicContext::QUEUE_UTIL); - - EXIT_NOT_IMPLEMENTED(buffer.IsInvalid()); - - auto* vk_buffer = buffer.GetPool()->buffers[buffer.GetIndex()]; - - buffer.Begin(); - - VkBufferCopy copy_region {}; - copy_region.srcOffset = 0; - copy_region.dstOffset = 0; - copy_region.size = size; - - vkCmdCopyBuffer(vk_buffer, src_buffer->buffer, dst_buffer->buffer, 1, ©_region); - - buffer.End(); - buffer.Execute(); - buffer.WaitForFence(); + ExecuteImmediateUtilCommands([&](CommandBuffer*, VkCommandBuffer vk_command) { + SetBufferMemoryBarrier(vk_command, src_buffer->buffer, 0, size, VK_ACCESS_MEMORY_WRITE_BIT, + VK_ACCESS_TRANSFER_READ_BIT, VK_PIPELINE_STAGE_ALL_COMMANDS_BIT, + VK_PIPELINE_STAGE_TRANSFER_BIT); + SetBufferMemoryBarrier(vk_command, dst_buffer->buffer, 0, size, + VK_ACCESS_MEMORY_READ_BIT | VK_ACCESS_MEMORY_WRITE_BIT, + VK_ACCESS_TRANSFER_WRITE_BIT, VK_PIPELINE_STAGE_ALL_COMMANDS_BIT, + VK_PIPELINE_STAGE_TRANSFER_BIT); + const VkBufferCopy copy_region {.srcOffset = 0, .dstOffset = 0, .size = size}; + vkCmdCopyBuffer(vk_command, src_buffer->buffer, dst_buffer->buffer, 1, ©_region); + SetBufferMemoryBarrier(vk_command, dst_buffer->buffer, 0, size, + VK_ACCESS_TRANSFER_WRITE_BIT, + VK_ACCESS_MEMORY_READ_BIT | VK_ACCESS_MEMORY_WRITE_BIT, + VK_PIPELINE_STAGE_TRANSFER_BIT, VK_PIPELINE_STAGE_ALL_COMMANDS_BIT); + }); } void UtilDownloadBuffer(GraphicContext* ctx, VulkanBuffer* src_buffer, uint64_t src_offset, void* dst_data, uint64_t size) { - EXIT_IF(ctx == nullptr || src_buffer == nullptr || src_buffer->buffer == nullptr || - dst_data == nullptr || size == 0); + EXIT_IF(size == 0); EXIT_IF(src_offset > src_buffer->buffer_size || size > src_buffer->buffer_size - src_offset); const auto family = ctx->queues[GraphicContext::QUEUE_UTIL].family; EXIT_IF(family == static_cast(-1)); @@ -1116,7 +1026,6 @@ void UtilDownloadBuffer(GraphicContext* ctx, VulkanBuffer* src_buffer, uint64_t if ((src_buffer->memory.property & VK_MEMORY_PROPERTY_HOST_VISIBLE_BIT) != 0) { void* mapped = nullptr; VulkanMapMemory(ctx, &src_buffer->memory, &mapped); - EXIT_IF(mapped == nullptr); std::memcpy(dst_data, static_cast(mapped) + src_offset, size); VulkanUnmapMemory(ctx, &src_buffer->memory); return; @@ -1129,30 +1038,19 @@ void UtilDownloadBuffer(GraphicContext* ctx, VulkanBuffer* src_buffer, uint64_t VK_MEMORY_PROPERTY_HOST_CACHED_BIT; VulkanCreateBuffer(ctx, size, &readback); - CommandBuffer command(GraphicContext::QUEUE_UTIL); - EXIT_NOT_IMPLEMENTED(command.IsInvalid()); - auto* vk_buffer = command.GetPool()->buffers[command.GetIndex()]; - command.Begin(); - VkBufferMemoryBarrier barrier {}; - barrier.sType = VK_STRUCTURE_TYPE_BUFFER_MEMORY_BARRIER; - barrier.srcAccessMask = VK_ACCESS_MEMORY_WRITE_BIT; - barrier.dstAccessMask = VK_ACCESS_TRANSFER_READ_BIT; - barrier.srcQueueFamilyIndex = VK_QUEUE_FAMILY_IGNORED; - barrier.dstQueueFamilyIndex = VK_QUEUE_FAMILY_IGNORED; - barrier.buffer = src_buffer->buffer; - barrier.offset = src_offset; - barrier.size = size; - vkCmdPipelineBarrier(vk_buffer, VK_PIPELINE_STAGE_ALL_COMMANDS_BIT, - VK_PIPELINE_STAGE_TRANSFER_BIT, 0, 0, nullptr, 1, &barrier, 0, nullptr); - const VkBufferCopy copy {.srcOffset = src_offset, .dstOffset = 0, .size = size}; - vkCmdCopyBuffer(vk_buffer, src_buffer->buffer, readback.buffer, 1, ©); - command.End(); - command.Execute(); - command.WaitForFence(); + ExecuteImmediateUtilCommands([&](CommandBuffer*, VkCommandBuffer vk_command) { + SetBufferMemoryBarrier(vk_command, src_buffer->buffer, src_offset, size, + VK_ACCESS_MEMORY_WRITE_BIT, VK_ACCESS_TRANSFER_READ_BIT, + VK_PIPELINE_STAGE_ALL_COMMANDS_BIT, VK_PIPELINE_STAGE_TRANSFER_BIT); + const VkBufferCopy copy {.srcOffset = src_offset, .dstOffset = 0, .size = size}; + vkCmdCopyBuffer(vk_command, src_buffer->buffer, readback.buffer, 1, ©); + SetBufferMemoryBarrier(vk_command, readback.buffer, 0, size, VK_ACCESS_TRANSFER_WRITE_BIT, + VK_ACCESS_HOST_READ_BIT, VK_PIPELINE_STAGE_TRANSFER_BIT, + VK_PIPELINE_STAGE_HOST_BIT); + }); void* mapped = nullptr; VulkanMapMemory(ctx, &readback.memory, &mapped); - EXIT_IF(mapped == nullptr); std::memcpy(dst_data, mapped, size); VulkanUnmapMemory(ctx, &readback.memory); VulkanDeleteBuffer(ctx, &readback); diff --git a/src/graphics/host_gpu/utils.h b/src/graphics/host_gpu/utils.h index 968200cb4..9ada13ad8 100644 --- a/src/graphics/host_gpu/utils.h +++ b/src/graphics/host_gpu/utils.h @@ -4,8 +4,8 @@ #include "common/abi.h" #include "common/common.h" +#include #include -#include #include namespace Libs::Graphics { @@ -47,7 +47,7 @@ struct ImageImageCopy { int dst_z = 0; }; -enum class StagingBufferType { Texture, Vertex, Index, ReadBack }; +enum class StagingBufferType { Texture, Vertex, ReadBack }; class UtilScratchBuffer { public: @@ -61,36 +61,30 @@ private: void* m_data = nullptr; }; -void UtilBufferToImage(CommandBuffer* buffer, VulkanBuffer* src_buffer, uint32_t src_pitch, - VulkanImage* dst_image, uint64_t dst_layout); -void UtilBufferToImage(CommandBuffer* buffer, VulkanBuffer* src_buffer, VulkanImage* dst_image, - const std::vector& regions, uint64_t dst_layout); -void UtilImageToBuffer(CommandBuffer* buffer, VulkanImage* src_image, VulkanBuffer* dst_buffer, - uint32_t dst_pitch, uint64_t src_layout, - VkImageAspectFlags aspect = VK_IMAGE_ASPECT_COLOR_BIT); -void UtilImageToImage(CommandBuffer* buffer, const std::vector& regions, +void UtilImageToImage(CommandBuffer* buffer, std::span regions, VulkanImage* dst_image, uint64_t dst_layout); +void UtilCopyImageWithBuffer(CommandBuffer* buffer, GraphicContext* ctx, VulkanImage* src_image, + VkImageAspectFlags src_aspect, VulkanImage* dst_image, + VkImageAspectFlags dst_aspect, uint32_t bytes_per_element, + uint64_t dst_layout); void UtilBlitImage(CommandBuffer* buffer, VulkanImage* src_image, VulkanSwapchain* dst_swapchain); void UtilFillImage(GraphicContext* ctx, VulkanImage* dst_image, const void* src_data, uint64_t size, uint32_t src_pitch, uint64_t dst_layout); -void UtilFillDepthImage(GraphicContext* ctx, DepthStencilVulkanImage* dst_image, - const void* src_data, uint64_t size, uint32_t src_pitch); +void UtilFillImage(GraphicContext* ctx, DepthStencilVulkanImage* dst_image, const void* src_data, + uint64_t size, uint32_t src_pitch, VkImageAspectFlags aspect); void UtilFillImage(GraphicContext* ctx, VulkanImage* dst_image, const void* src_data, uint64_t size, - const std::vector& regions, uint64_t dst_layout); -void UtilFillImage(GraphicContext* ctx, const std::vector& regions, + std::span regions, uint64_t dst_layout); +void UtilFillImage(GraphicContext* ctx, std::span regions, VulkanImage* dst_image, uint64_t dst_layout); void UtilFillBuffer(GraphicContext* ctx, void* dst_data, uint64_t size, uint32_t dst_pitch, VulkanImage* src_image, uint64_t src_layout, VkImageAspectFlags aspect = VK_IMAGE_ASPECT_COLOR_BIT); -void UtilUploadBuffer(GraphicContext* ctx, StagingBufferType type, VulkanBuffer* dst_buffer, - const void* src_data, uint64_t size); void UtilUploadBuffer(GraphicContext* ctx, StagingBufferType type, VulkanBuffer* dst_buffer, uint64_t dst_offset, const void* src_data, uint64_t size); void UtilCopyBuffer(VulkanBuffer* src_buffer, VulkanBuffer* dst_buffer, uint64_t size); void UtilDownloadBuffer(GraphicContext* ctx, VulkanBuffer* src_buffer, uint64_t src_offset, void* dst_data, uint64_t size); bool UtilBufferIsTiled(uint64_t vaddr, uint64_t size); -void UtilSetDepthLayoutOptimal(DepthStencilVulkanImage* image); void UtilSetImageLayoutOptimal(VulkanImage* image); void UtilResetImageViews(VulkanImage* image); void UtilCreateImageView(GraphicContext* ctx, VulkanImage* image, int view_index, diff --git a/tests/ShaderRecompilerComputeTests.cpp b/tests/ShaderRecompilerComputeTests.cpp index 16582552f..4fdaf0561 100644 --- a/tests/ShaderRecompilerComputeTests.cpp +++ b/tests/ShaderRecompilerComputeTests.cpp @@ -7777,9 +7777,9 @@ TestCase ImageStoreRgbOneUsesInverseSwizzle() { AppendVMovU32(&code, 21, 2); AppendVMovU32(&code, 22, 0); AppendVMovLiteral(&code, 0, 0x3f800000u); - AppendVMovLiteral(&code, 1, 0x3f000000u); - AppendVMovLiteral(&code, 2, 0x3e800000u); - AppendVMovLiteral(&code, 3, 0x3f400000u); + AppendVMovLiteral(&code, 1, 0x3f008081u); + AppendVMovLiteral(&code, 2, 0x3e808081u); + AppendVMovLiteral(&code, 3, 0x3f40c0c1u); code.push_back(EncodeMimg0(0x08, 0xf)); code.push_back(EncodeMimg1(0, 20)); AppendEnd(&code); @@ -7809,9 +7809,9 @@ TestCase ImageStoreBgraUsesInverseSwizzle() { AppendVMovU32(&code, 21, 2); AppendVMovU32(&code, 22, 0); AppendVMovLiteral(&code, 0, 0x3f800000u); - AppendVMovLiteral(&code, 1, 0x3f000000u); - AppendVMovLiteral(&code, 2, 0x3e800000u); - AppendVMovLiteral(&code, 3, 0x3f400000u); + AppendVMovLiteral(&code, 1, 0x3f008081u); // 128/255 + AppendVMovLiteral(&code, 2, 0x3e808081u); // 64/255 + AppendVMovLiteral(&code, 3, 0x3f3fbfc0u); // 191/255 code.push_back(EncodeMimg0(0x08, 0xf)); code.push_back(EncodeMimg1(0, 20)); AppendEnd(&code); @@ -8788,6 +8788,12 @@ void CheckSampledColorViews() { DstSel(4, 0, 0, 0)) == VulkanImage::VIEW_R000, "D16 depth target did not select its R000 depth-aspect view"); + Require("SampledColorViews", "D32S8 R001 depth target", + SelectSampledDepthView(VK_FORMAT_D32_SFLOAT_S8_UINT, + VK_FORMAT_R32_SFLOAT, + DstSel(4, 0, 0, 1)) == + VulkanImage::VIEW_R001, + "D32S8 depth target did not select its R001 depth-aspect view"); Require("SampledColorViews", "storage identity", SelectStorageColorView(VK_FORMAT_R8G8B8A8_UNORM, VK_FORMAT_R8G8B8A8_UNORM, @@ -9601,6 +9607,67 @@ void CheckDepthTargetTileRoundTrip() { std::printf("[host] %-32s ok\n", "DepthTargetTileRoundTrip"); } +void CheckStencilTargetTileRoundTrip() { + struct Case { + uint32_t width; + uint32_t height; + }; + constexpr std::array cases{Case{8, 8}, Case{257, 259}, Case{1920, 1080}}; + constexpr auto format = Prospero::GpuEnumValue(Prospero::BufferFormat::k8UInt); + constexpr auto tile = Prospero::GpuEnumValue(Prospero::TileMode::kDepth); + for (const auto test : cases) { + const auto pitch = TileGetTexturePitch(format, test.width, 1, tile); + TileSizeAlign stencil{}; + TileSizeAlign htile{}; + TileSizeAlign depth{}; + Require("StencilTargetTileRoundTrip", "layout", + pitch >= test.width && pitch % 256 == 0 && + TileGetDepthSize(test.width, test.height, 0, + Prospero::GpuEnumValue(Prospero::DepthFormat::kZ32F), + Prospero::GpuEnumValue(Prospero::StencilFormat::k8UInt), false, + &stencil, &htile, &depth) && + stencil.size != 0 && stencil.align == 0x10000, + "Prospero stencil layout was rejected"); + std::vector linear(stencil.size, 0); + for (uint32_t y = 0; y < test.height; y++) { + for (uint32_t x = 0; x < test.width; x++) { + linear[static_cast(y) * pitch + x] = + static_cast((x * 43u + y * 71u + 19u) & 0xffu); + } + } + std::vector tiled(stencil.size, 0xcd); + std::vector restored(stencil.size, 0xa5); + TileConvertLinearToTiledDepth(tiled.data(), linear.data(), format, test.width, + test.height, pitch, stencil.size); + TileConvertTiledToLinearDepth(restored.data(), tiled.data(), format, test.width, + test.height, pitch, stencil.size); + for (uint32_t y = 0; y < test.height; y++) { + const auto row = static_cast(y) * pitch; + Require("StencilTargetTileRoundTrip", "contents", + std::memcmp(linear.data() + row, restored.data() + row, test.width) == 0, + "linear-to-tiled PS5 stencil conversion did not round-trip"); + } + if (test.width == 8 && test.height == 8) { + Require("StencilTargetTileRoundTrip", "Prospero S8 SW_64KB_Z_X anchors", + tiled[0] == linear[0] && tiled[1] == linear[1] && + tiled[4] == linear[2] && tiled[2] == linear[pitch] && + tiled[0x27] == linear[5 * pitch + 3], + "Prospero S8 stencil address anchors changed"); + } + if (test.width == 257 && test.height == 259) { + Require("StencilTargetTileRoundTrip", "Prospero S8 block order", + tiled[0x10000] == linear[256] && tiled[0x20000] == linear[256 * pitch], + "Prospero S8 stencil block order changed"); + } + if (test.width == 1920 && test.height == 1080) { + Require("StencilTargetTileRoundTrip", "PPSA01880 footprint", + pitch == 2048 && stencil.size == 0x280000, + "captured PPSA01880 stencil footprint changed"); + } + } + std::printf("[host] %-32s ok\n", "StencilTargetTileRoundTrip"); +} + void CheckStorageTextureGpuOwnedRebindState() { constexpr uintptr_t base = 0x0000000200200000ull; constexpr uint64_t size = 0x10000; @@ -10625,10 +10692,83 @@ void CheckImageOverlapResolution() { DepthOverlap::Unsupported, "GPU-owned sampled image was admitted for retirement"); depth.depth_load_clear = false; - Require("ImageOverlapResolution", "depth load", + Require("ImageOverlapResolution", "incompatible depth load", ClassifyDepthOverlap(sampled, false, depth) == DepthOverlap::Unsupported, - "load-preserving depth use was admitted for retirement"); + "layout-unknown depth load was admitted for retirement"); + + ImageInfo ppsa01880_sampled_depth{}; + ppsa01880_sampled_depth.address = 0x1095200000ull; + ppsa01880_sampled_depth.size = 0x10000000ull; + ppsa01880_sampled_depth.format = + Prospero::GpuEnumValue(Prospero::BufferFormat::k32Float); + ppsa01880_sampled_depth.width = 8192; + ppsa01880_sampled_depth.height = 8192; + ppsa01880_sampled_depth.pitch = 8192; + ppsa01880_sampled_depth.tile = + Prospero::GpuEnumValue(Prospero::TileMode::kDepth); + ppsa01880_sampled_depth.type = + Prospero::GpuEnumValue(Prospero::ImageType::kColor2D); + DepthTargetInfo ppsa01880_depth{}; + ppsa01880_depth.address = ppsa01880_sampled_depth.address; + ppsa01880_depth.size = ppsa01880_sampled_depth.size; + ppsa01880_depth.format = VK_FORMAT_D32_SFLOAT; + ppsa01880_depth.guest_format = ppsa01880_sampled_depth.format; + ppsa01880_depth.width = ppsa01880_sampled_depth.width; + ppsa01880_depth.height = ppsa01880_sampled_depth.height; + ppsa01880_depth.pitch = ppsa01880_sampled_depth.pitch; + ppsa01880_depth.bytes_per_element = 4; + ppsa01880_depth.tile_mode = ppsa01880_sampled_depth.tile; + Require("ImageOverlapResolution", "PPSA01880 exact depth load", + ClassifyDepthOverlap(ppsa01880_sampled_depth, false, + ppsa01880_depth) == + DepthOverlap::RetireSampled, + "captured CPU-owned R32F/D32F depth layout was not retired"); + Require("ImageOverlapResolution", "PPSA01880 GPU depth load", + ClassifyDepthOverlap(ppsa01880_sampled_depth, true, + ppsa01880_depth) == + DepthOverlap::Unsupported, + "captured GPU-owned sampled depth was admitted for retirement"); + auto mismatched_sampled_depth = ppsa01880_sampled_depth; + mismatched_sampled_depth.pitch++; + auto mismatched_depth = ppsa01880_depth; + mismatched_depth.stencil_address = ppsa01880_depth.address + ppsa01880_depth.size; + mismatched_depth.stencil_size = 0x4000000; + Require("ImageOverlapResolution", "PPSA01880 depth layout guards", + ClassifyDepthOverlap(mismatched_sampled_depth, false, + ppsa01880_depth) == + DepthOverlap::Unsupported && + ClassifyDepthOverlap(ppsa01880_sampled_depth, false, + mismatched_depth) == + DepthOverlap::Unsupported, + "pitch-mismatched or stencil-bearing depth load was admitted"); + mismatched_sampled_depth = ppsa01880_sampled_depth; + mismatched_sampled_depth.levels = 2; + mismatched_sampled_depth.view_levels = 2; + mismatched_depth = ppsa01880_depth; + mismatched_depth.format = VK_FORMAT_D16_UNORM; + Require("ImageOverlapResolution", "PPSA01880 depth topology guards", + ClassifyDepthOverlap(mismatched_sampled_depth, false, + ppsa01880_depth) == + DepthOverlap::Unsupported && + ClassifyDepthOverlap(ppsa01880_sampled_depth, false, + mismatched_depth) == + DepthOverlap::Unsupported, + "mipmapped or format-mismatched depth load was admitted"); + Require("ImageOverlapResolution", "depth transition source", + SelectDepthTransitionSource(true, true, false, false, false, true) == + DepthTransitionSource::None && + SelectDepthTransitionSource(false, true, false, false, false, true) == + DepthTransitionSource::Native && + SelectDepthTransitionSource(false, false, false, false, false, false) == + DepthTransitionSource::Guest && + SelectDepthTransitionSource(false, true, true, false, false, false) == + DepthTransitionSource::Guest && + SelectDepthTransitionSource(false, true, false, true, false, false) == + DepthTransitionSource::Guest && + SelectDepthTransitionSource(false, true, false, false, true, true) == + DepthTransitionSource::Guest, + "clear, clean-native, or overlapping dirty-buffer depth preservation policy regressed"); depth.depth_load_clear = true; depth.stencil_address = sampled.address + 0x6000; depth.stencil_size = 0x2000; @@ -10657,6 +10797,15 @@ void CheckImageOverlapResolution() { Require("ImageOverlapResolution", "stencil clear initializes", CanLoadStencilAttachment(depth, false), "stencil clear did not initialize the attachment"); + depth.stencil_htile_compressed = false; + Require("ImageOverlapResolution", "raw stencil load", + CanLoadRawStencilPlane(depth), + "uncompressed guest stencil plane was rejected"); + depth.stencil_htile_compressed = true; + Require("ImageOverlapResolution", "compressed stencil load", + !CanLoadRawStencilPlane(depth), + "HTile-compressed guest stencil plane was silently admitted"); + depth.stencil_htile_compressed = false; depth.address = sampled.address; depth.size = 0xff0000; depth.stencil_address = sampled.address + 0x1000000; @@ -11041,6 +11190,31 @@ void CheckEmbeddedFetchLaneSpill() { std::printf("[host] %-32s ok\n", "EmbeddedFetchLaneSpill"); } +void CheckReferenceClockScale() { + uint64_t value = 0; + Require("ReferenceClockScale", "zero", + GraphicsScaleReferenceClock(0, 3000000000ull, &value) && value == 0, + "zero host tick did not produce a zero GPU clock"); + Require("ReferenceClockScale", "fractional second", + GraphicsScaleReferenceClock(1500000000ull, 3000000000ull, &value) && + value == 50000000ull, + "host half-second did not scale to 50,000,000 ticks"); + Require("ReferenceClockScale", "whole and fractional", + GraphicsScaleReferenceClock(3750000000ull, 3000000000ull, &value) && + value == 125000000ull, + "host 1.25 seconds did not scale to 125,000,000 ticks"); + Require("ReferenceClockScale", "monotonic floor", + GraphicsScaleReferenceClock(3750000001ull, 3000000000ull, &value) && + value == 125000000ull, + "sub-reference-tick increment did not use a monotonic floor"); + Require("ReferenceClockScale", "guards", + !GraphicsScaleReferenceClock(1, 0, &value) && + !GraphicsScaleReferenceClock(1, 1, nullptr) && + !GraphicsScaleReferenceClock(UINT64_MAX, 1, &value), + "invalid frequency, destination, or overflow was accepted"); + std::printf("[host] %-32s ok\n", "ReferenceClockScale"); +} + } // namespace } // namespace Libs::Graphics @@ -11049,6 +11223,10 @@ int main(int argc, char **argv) { EnsureConfigInitialized(); TileInit(); + if (argc == 2 && std::strcmp(argv[1], "--reference-clock-only") == 0) { + CheckReferenceClockScale(); + return 0; + } #if KYTY_PLATFORM == KYTY_PLATFORM_WINDOWS if (argc == 2 && std::strcmp(argv[1], "--image-overlap-only") == 0) { CheckImageOverlapResolution(); @@ -11096,6 +11274,7 @@ int main(int argc, char **argv) { } if (argc == 2 && std::strcmp(argv[1], "--depth-readback-only") == 0) { CheckDepthTargetTileRoundTrip(); + CheckStencilTargetTileRoundTrip(); CheckDepthTargetFootprints(); return 0; } @@ -11126,6 +11305,7 @@ int main(int argc, char **argv) { CheckStorageImageSwizzleSpecializationId(); CheckRenderTargetTileRoundTrip(); CheckDepthTargetTileRoundTrip(); + CheckStencilTargetTileRoundTrip(); CheckStorageTextureVolumeUploadLayout(); CheckStorageTextureGpuOwnedRebindState(); CheckStorageTextureSampledReuse(); @@ -11145,6 +11325,7 @@ int main(int argc, char **argv) { (void)argc; (void)argv; #endif + CheckReferenceClockScale(); CheckEmbeddedFetchVertexOffset(); CheckEmbeddedFetchLaneSpill(); CheckPs5GameExampleImageClearRuntimeShape();