diff --git a/src/graphics/host_gpu/graphicContext.h b/src/graphics/host_gpu/graphicContext.h index 6c455fb78..683f18f65 100644 --- a/src/graphics/host_gpu/graphicContext.h +++ b/src/graphics/host_gpu/graphicContext.h @@ -110,8 +110,15 @@ struct VideoOutVulkanImage: public VulkanImage { }; struct DepthStencilVulkanImage: public VulkanImage { + struct AttachmentView { + uint32_t base_layer = 0; + uint32_t layer_count = 1; + VkImageView view = nullptr; + }; DepthStencilVulkanImage(): VulkanImage(VulkanImageType::DepthStencil) {} - bool compressed = false; + bool compressed = false; + std::mutex attachment_view_mutex; + std::vector attachment_views; }; struct GpuTextureVulkanImage: public VulkanImage { @@ -138,22 +145,30 @@ struct StorageTextureVulkanImage: public GpuTextureVulkanImage { struct RenderTextureVulkanImage: public VulkanImage { struct AttachmentView { - VkFormat format = VK_FORMAT_UNDEFINED; - uint32_t level = 0; - VkImageView view = nullptr; + VkFormat format = VK_FORMAT_UNDEFINED; + uint32_t level = 0; + uint32_t base_layer = 0; + uint32_t layer_count = 1; + VkImageView view = nullptr; }; struct SampledView { - VkFormat format = VK_FORMAT_UNDEFINED; - uint32_t base_level = 0; - uint32_t level_count = 0; - int variant = VIEW_DEFAULT; - VkImageView view = nullptr; + VkFormat format = VK_FORMAT_UNDEFINED; + VkImageViewType type = VK_IMAGE_VIEW_TYPE_2D; + uint32_t base_level = 0; + uint32_t level_count = 0; + uint32_t base_layer = 0; + uint32_t layer_count = 1; + int variant = VIEW_DEFAULT; + VkImageView view = nullptr; }; struct StorageView { - VkFormat format = VK_FORMAT_UNDEFINED; - uint32_t base_level = 0; - uint32_t level_count = 0; - VkImageView view = nullptr; + VkFormat format = VK_FORMAT_UNDEFINED; + VkImageViewType type = VK_IMAGE_VIEW_TYPE_2D; + uint32_t base_level = 0; + uint32_t level_count = 0; + uint32_t base_layer = 0; + uint32_t layer_count = 1; + VkImageView view = nullptr; }; RenderTextureVulkanImage(): VulkanImage(VulkanImageType::RenderTexture) {} VkImageView render_view[16] = {}; diff --git a/src/graphics/host_gpu/renderer/bufferCache.cpp b/src/graphics/host_gpu/renderer/bufferCache.cpp index e7bb85a18..113a4f4a7 100644 --- a/src/graphics/host_gpu/renderer/bufferCache.cpp +++ b/src/graphics/host_gpu/renderer/bufferCache.cpp @@ -28,7 +28,7 @@ namespace Libs::Graphics { namespace { thread_local const void* g_cache_lock_owner = nullptr; -constexpr uint32_t READBACK_MAX_RANGES = 256; +constexpr uint32_t READBACK_MAX_RANGES = 256; constexpr uint64_t READBACK_COPY_ALIGNMENT = 64; constexpr uint64_t READBACK_CAPACITY = TRACKER_PAGE_SIZE + READBACK_MAX_RANGES * (READBACK_COPY_ALIGNMENT - 1); @@ -42,24 +42,26 @@ constexpr bool ShouldLogCommandProcessorReadback(uint64_t serial) noexcept { } struct CommandProcessorReadbackState { - BufferCache* cache = nullptr; - PageFaultAccess access = PageFaultAccess::Unknown; - uint64_t vaddr = 0; - uint64_t size = 0; - uint64_t page = 0; - uint64_t serial = 0; - uint64_t dirty_bytes = 0; + BufferCache* cache = nullptr; + PageFaultAccess access = PageFaultAccess::Unknown; + uint64_t vaddr = 0; + uint64_t size = 0; + uint64_t page = 0; + uint64_t serial = 0; + uint64_t dirty_bytes = 0; uint64_t packed_bytes = 0; - uint32_t dirty_count = 0; + uint32_t dirty_count = 0; std::chrono::steady_clock::time_point started {}; bool installed = false; }; thread_local CommandProcessorReadbackState g_command_processor_readback; -std::atomic g_command_processor_readback_serial {0}; +std::atomic g_command_processor_readback_serial {0}; struct SharedVulkanBufferOwner { - explicit SharedVulkanBufferOwner(GraphicContext* context): ctx(context) { EXIT_IF(ctx == nullptr); } + explicit SharedVulkanBufferOwner(GraphicContext* context): ctx(context) { + EXIT_IF(ctx == nullptr); + } ~SharedVulkanBufferOwner() { if (buffer.buffer != nullptr) { @@ -68,7 +70,7 @@ struct SharedVulkanBufferOwner { } GraphicContext* ctx = nullptr; - VulkanBuffer buffer; + VulkanBuffer buffer; }; std::shared_ptr MakeSharedVulkanBuffer(GraphicContext* ctx) { @@ -203,8 +205,8 @@ struct BufferCache::CachedBuffer { }; struct BufferCache::CommandProcessorReadbackResources { - GraphicContext* ctx = nullptr; - int queue = -1; + GraphicContext* ctx = nullptr; + int queue = -1; VulkanBuffer buffer {}; void* mapped = nullptr; std::unique_ptr command; @@ -257,8 +259,8 @@ struct BufferCache::ReadbackWorker { } auto current = state.load(std::memory_order_acquire); while (current == State::InitRequested || - (current == State::Claimed && (command == nullptr || mapped == nullptr || - readback.buffer == nullptr))) { + (current == State::Claimed && + (command == nullptr || mapped == nullptr || readback.buffer == nullptr))) { state.wait(current, std::memory_order_acquire); current = state.load(std::memory_order_acquire); } @@ -306,9 +308,8 @@ struct BufferCache::ReadbackWorker { const bool submissions_prepaused_now = GraphicsRunSubmissionLockHeld(); const bool unsafe_gpu_lock = GraphicsRunGpuLockHeld() && !submissions_prepaused_now; if (GraphicsRunIsCommandProcessorThread() || unsafe_gpu_lock || LabelInCallback() || - g_cache_lock_owner != nullptr || - ctx == nullptr || command == nullptr || mapped == nullptr || - readback.buffer == nullptr) { + g_cache_lock_owner != nullptr || ctx == nullptr || command == nullptr || + mapped == nullptr || readback.buffer == nullptr) { EXIT("BufferCache: unsafe readback request context, command_thread=%d " "submission_lock=%d " "gpu_lock=%d label_callback=%d cache_lock=%p ctx=%p command=%p mapped=%p " @@ -328,10 +329,10 @@ struct BufferCache::ReadbackWorker { state.wait(expected, std::memory_order_acquire); expected = State::Idle; } - access = fault_access; - vaddr = fault_vaddr; - size = fault_size; - range_count = 0; + access = fault_access; + vaddr = fault_vaddr; + size = fault_size; + range_count = 0; submissions_prepaused = submissions_prepaused_now; state.store(State::Requested, std::memory_order_release); state.notify_all(); @@ -368,7 +369,7 @@ struct BufferCache::ReadbackWorker { for (uint32_t i = 0; i < range_count; i++) { const auto& range = ranges[i]; Libs::LibKernel::Memory::WriteBacking(range.address, data.data() + range.offset, - range.size); + range.size); } if (!cache.m_memory_tracker.CompleteCpuFault(vaddr, size, access, true)) { EXIT("BufferCache: failed to complete downloaded CPU fault, addr=0x%016" PRIx64 @@ -476,7 +477,7 @@ struct BufferCache::ReadbackWorker { } const auto page = vaddr & ~(TRACKER_PAGE_SIZE - 1); FaultSafeCacheLock lock(&cache, cache.m_mutex); - auto it = cache.m_buffers.upper_bound(vaddr); + auto it = cache.m_buffers.upper_bound(vaddr); if (it == cache.m_buffers.begin()) { EXIT("BufferCache: readback address has no cached buffer, addr=0x%016" PRIx64 "\n", vaddr); @@ -564,10 +565,10 @@ struct BufferCache::ReadbackWorker { VulkanBuffer readback {}; void* mapped = nullptr; std::unique_ptr command; - PageFaultAccess access = PageFaultAccess::Unknown; - uint64_t vaddr = 0; - uint64_t size = 0; - uint32_t range_count = 0; + PageFaultAccess access = PageFaultAccess::Unknown; + uint64_t vaddr = 0; + uint64_t size = 0; + uint32_t range_count = 0; bool submissions_prepaused = false; std::array ranges {}; std::array data {}; @@ -662,8 +663,8 @@ void BufferCache::RequestCommandProcessorReadback(PageFaultAccess access, uint64 } const bool log_detail = ShouldLogCommandProcessorReadback(serial); if (log_detail) { - LOGF("BufferCache: command-processor readback request serial=%" PRIu64 - " addr=0x%016" PRIx64 " size=0x%016" PRIx64 " access=%u poll=%d\n", + LOGF("BufferCache: command-processor readback request serial=%" PRIu64 " addr=0x%016" PRIx64 + " size=0x%016" PRIx64 " access=%u poll=%d\n", serial, vaddr, size, static_cast(access), GraphicsRunIsGuestMemoryPoll()); Log::Flush(); } @@ -681,8 +682,8 @@ void BufferCache::RequestCommandProcessorReadback(PageFaultAccess access, uint64 g_cache_lock_owner = this; m_mutex.Lock(); if (log_detail) { - LOGF("BufferCache: command-processor cache locked serial=%" PRIu64 - " page=0x%016" PRIx64 "\n", + LOGF("BufferCache: command-processor cache locked serial=%" PRIu64 " page=0x%016" PRIx64 + "\n", serial, page); Log::Flush(); } @@ -715,12 +716,12 @@ void BufferCache::RequestCommandProcessorReadback(PageFaultAccess access, uint64 } ValidateDirtyPages(m_gpu_modified_ranges, page, TRACKER_PAGE_SIZE, "command-processor"); if (m_cp_readback_resources == nullptr) { - auto resources = std::make_unique(); - resources->ctx = cached.ctx; - resources->buffer.usage = VK_BUFFER_USAGE_TRANSFER_DST_BIT; + auto resources = std::make_unique(); + resources->ctx = cached.ctx; + resources->buffer.usage = VK_BUFFER_USAGE_TRANSFER_DST_BIT; resources->buffer.memory.property = VK_MEMORY_PROPERTY_HOST_VISIBLE_BIT | - VK_MEMORY_PROPERTY_HOST_COHERENT_BIT | - VK_MEMORY_PROPERTY_HOST_CACHED_BIT; + VK_MEMORY_PROPERTY_HOST_COHERENT_BIT | + VK_MEMORY_PROPERTY_HOST_CACHED_BIT; VulkanCreateBuffer(cached.ctx, READBACK_CAPACITY, &resources->buffer); VulkanMapMemory(cached.ctx, &resources->buffer.memory, &resources->mapped); m_cp_readback_resources = std::move(resources); @@ -739,7 +740,8 @@ void BufferCache::RequestCommandProcessorReadback(PageFaultAccess access, uint64 resources.command = std::make_unique(static_cast(queue)); resources.queue = static_cast(queue); } - if (resources.command->IsInvalid() || resources.command->GetQueue() != static_cast(queue)) { + if (resources.command->IsInvalid() || + resources.command->GetQueue() != static_cast(queue)) { EXIT("BufferCache: failed to initialize owning-queue readback command buffer, queue=%u\n", queue); } @@ -750,18 +752,18 @@ void BufferCache::RequestCommandProcessorReadback(PageFaultAccess access, uint64 } std::array copies {}; std::array barriers {}; - uint64_t dirty_bytes = 0; - uint64_t packed_bytes = 0; - uint32_t dirty_count = 0; + uint64_t dirty_bytes = 0; + uint64_t packed_bytes = 0; + uint32_t dirty_count = 0; for (const auto& range: dirty) { if ((range.address - cached.vaddr) % 4 != 0 || range.size % 4 != 0) { EXIT("BufferCache: invalid command-processor dirty range, addr=0x%016" PRIx64 " size=0x%016" PRIx64 " page=0x%016" PRIx64 " packed=0x%016" PRIx64 "\n", range.address, range.size, page, dirty_bytes); } - copies[dirty_count] = {.srcOffset = range.address - cached.vaddr, - .dstOffset = packed_bytes, - .size = range.size}; + copies[dirty_count] = {.srcOffset = range.address - cached.vaddr, + .dstOffset = packed_bytes, + .size = range.size}; auto& barrier = barriers[dirty_count]; barrier.sType = VK_STRUCTURE_TYPE_BUFFER_MEMORY_BARRIER; barrier.srcAccessMask = VK_ACCESS_MEMORY_WRITE_BIT; @@ -771,7 +773,7 @@ void BufferCache::RequestCommandProcessorReadback(PageFaultAccess access, uint64 barrier.buffer = cached.buffer->buffer; barrier.offset = copies[dirty_count].srcOffset; barrier.size = range.size; - const auto slot_size = AlignReadbackCopySize(range.size); + const auto slot_size = AlignReadbackCopySize(range.size); if (packed_bytes > READBACK_CAPACITY || slot_size > READBACK_CAPACITY - packed_bytes) { EXIT("BufferCache: aligned command-processor readback exceeds persistent capacity, " "packed=%" PRIu64 " size=%" PRIu64 " capacity=%" PRIu64 "\n", @@ -787,8 +789,7 @@ void BufferCache::RequestCommandProcessorReadback(PageFaultAccess access, uint64 const auto started = std::chrono::steady_clock::now(); if (log_detail) { LOGF("BufferCache: command-processor readback begin serial=%" PRIu64 - " queue=%u page=0x%016" PRIx64 " ranges=%u bytes=%" PRIu64 - " packed=%" PRIu64 "\n", + " queue=%u page=0x%016" PRIx64 " ranges=%u bytes=%" PRIu64 " packed=%" PRIu64 "\n", serial, queue, page, dirty_count, dirty_bytes, packed_bytes); Log::Flush(); } @@ -808,20 +809,19 @@ void BufferCache::RequestCommandProcessorReadback(PageFaultAccess access, uint64 } for (uint32_t i = 0; i < dirty_count; i++) { Libs::LibKernel::Memory::WriteBacking( - dirty[i].address, - static_cast(resources.mapped) + copies[i].dstOffset, dirty[i].size); + dirty[i].address, static_cast(resources.mapped) + copies[i].dstOffset, + dirty[i].size); } - g_command_processor_readback = { - .cache = this, - .access = access, - .vaddr = vaddr, - .size = size, - .page = page, - .serial = serial, - .dirty_bytes = dirty_bytes, - .packed_bytes = packed_bytes, - .dirty_count = dirty_count, - .started = started}; + g_command_processor_readback = {.cache = this, + .access = access, + .vaddr = vaddr, + .size = size, + .page = page, + .serial = serial, + .dirty_bytes = dirty_bytes, + .packed_bytes = packed_bytes, + .dirty_count = dirty_count, + .started = started}; } bool BufferCache::CompleteCommandProcessorReadback(PageFaultAccess access, uint64_t vaddr, @@ -866,13 +866,13 @@ bool BufferCache::ReleaseCommandProcessorReadback(PageFaultAccess access, uint64 const auto elapsed_micro = std::chrono::duration_cast( std::chrono::steady_clock::now() - pending.started) .count(); - pending = {}; + pending = {}; m_mutex.Unlock(); g_cache_lock_owner = nullptr; GraphicsRunEndCommandProcessorReadback(); if (ShouldLogCommandProcessorReadback(serial)) { - LOGF("BufferCache: command-processor readback release serial=%" PRIu64 - " page=0x%016" PRIx64 " ranges=%u bytes=%" PRIu64 " elapsed_us=%" PRIi64 "\n", + LOGF("BufferCache: command-processor readback release serial=%" PRIu64 " page=0x%016" PRIx64 + " ranges=%u bytes=%" PRIu64 " elapsed_us=%" PRIi64 "\n", serial, page, dirty_count, dirty_bytes, static_cast(elapsed_micro)); } return true; @@ -971,10 +971,10 @@ std::pair BufferCache::ObtainBuffer(CommandBuffer* com !m_texture_cache->IsMetaGpuModified(vaddr, size)) { std::array guest_data; if (Libs::LibKernel::Memory::TryReadBacking(vaddr, guest_data.data(), size)) { - VulkanBuffer* stream_buffer = nullptr; - VkDeviceSize stream_offset = 0; - VkDeviceSize stream_range = 0; - const auto stream_alignment = std::max(ctx->StorageMinAlignment(), 16); + VulkanBuffer* stream_buffer = nullptr; + VkDeviceSize stream_offset = 0; + VkDeviceSize stream_range = 0; + const auto stream_alignment = std::max(ctx->StorageMinAlignment(), 16); if (UploadHostData(command, ctx, guest_data.data(), size, stream_alignment, &stream_buffer, &stream_offset, &stream_range)) { return {stream_buffer, stream_offset}; @@ -992,9 +992,9 @@ std::pair BufferCache::ObtainBuffer(CommandBuffer* com // performs the image ownership transition below. if (m_texture_cache->HasPageOverlap(begin, end - begin) && m_texture_cache->HasRangeOverlap(vaddr, size)) { - const bool coherent_read = - is_read && !is_written && !m_memory_tracker.IsRegionGpuModified(vaddr, size) && - !m_texture_cache->HasGpuModifiedRangeOverlap(vaddr, size); + const bool coherent_read = is_read && !is_written && + !m_memory_tracker.IsRegionGpuModified(vaddr, size) && + !m_texture_cache->HasGpuModifiedRangeOverlap(vaddr, size); if (!coherent_read) { if (!is_written || !is_formatted) { EXIT("BufferCache: unsupported buffer/image alias, addr=0x%016" PRIx64 @@ -1041,7 +1041,7 @@ std::pair BufferCache::ObtainBuffer(CommandBuffer* com } if (!overlaps.empty()) { - for (const auto overlap: overlaps) { + for (const auto& overlap: overlaps) { auto& old = *overlap->second; if (old.ctx != ctx || old.buffer == nullptr || old.buffer->buffer == nullptr) { EXIT("BufferCache: invalid overlapping buffer owner, addr=0x%016" PRIx64 @@ -1071,10 +1071,10 @@ std::pair BufferCache::ObtainBuffer(CommandBuffer* com }); } } - auto cached = std::make_unique(); - cached->vaddr = merged.address; - cached->size = merged.size; - cached->ctx = ctx; + auto cached = std::make_unique(); + cached->vaddr = merged.address; + cached->size = merged.size; + cached->ctx = ctx; cached->buffer = MakeSharedVulkanBuffer(ctx); cached->buffer->usage = VK_BUFFER_USAGE_TRANSFER_SRC_BIT | VK_BUFFER_USAGE_TRANSFER_DST_BIT | @@ -1086,7 +1086,7 @@ std::pair BufferCache::ObtainBuffer(CommandBuffer* com auto* vk_buffer = command->GetPool()->buffers[command->GetIndex()]; std::vector before; before.reserve(overlaps.size() + 1); - for (const auto overlap: overlaps) { + for (const auto& overlap: overlaps) { VkBufferMemoryBarrier barrier {}; barrier.sType = VK_STRUCTURE_TYPE_BUFFER_MEMORY_BARRIER; barrier.srcAccessMask = VK_ACCESS_MEMORY_WRITE_BIT; @@ -1111,7 +1111,7 @@ std::pair BufferCache::ObtainBuffer(CommandBuffer* com VK_PIPELINE_STAGE_TRANSFER_BIT, VK_DEPENDENCY_BY_REGION_BIT, 0, nullptr, static_cast(before.size()), before.data(), 0, nullptr); - for (const auto overlap: overlaps) { + for (const auto& overlap: overlaps) { const auto& old = *overlap->second; const VkBufferCopy copy { .srcOffset = 0, .dstOffset = old.vaddr - cached->vaddr, .size = old.size}; @@ -1129,7 +1129,7 @@ std::pair BufferCache::ObtainBuffer(CommandBuffer* com vkCmdPipelineBarrier(vk_buffer, VK_PIPELINE_STAGE_TRANSFER_BIT, VK_PIPELINE_STAGE_ALL_COMMANDS_BIT, VK_DEPENDENCY_BY_REGION_BIT, 0, nullptr, 1, &after, 0, nullptr); - for (const auto overlap: overlaps) { + for (const auto& overlap: overlaps) { command->RetainResourceUntilFence(overlap->second->buffer); m_buffers.erase(overlap); } @@ -1166,7 +1166,8 @@ bool BufferCache::UploadHostData(CommandBuffer* command, GraphicContext* ctx, co if (command == nullptr || command->IsInvalid() || command->IsExecute() || ctx == nullptr) { EXIT("BufferCache: host stream upload requires a recording command buffer\n"); } - return command->m_host_stream.Copy(ctx, src, size, alignment, out_buffer, out_offset, out_range); + return command->m_host_stream.Copy(ctx, src, size, alignment, out_buffer, out_offset, + out_range); } VulkanBuffer* BufferCache::ObtainNullBuffer(CommandBuffer* command, GraphicContext* ctx) { @@ -1177,7 +1178,7 @@ VulkanBuffer* BufferCache::ObtainNullBuffer(CommandBuffer* command, GraphicConte if (m_null_buffer == nullptr) { // robustBufferAccess makes every fetch safe; TODO: Use a // persistent 16-byte fallback when Vulkan null vertex descriptors are unavailable. - auto buffer = MakeSharedVulkanBuffer(ctx); + auto buffer = MakeSharedVulkanBuffer(ctx); buffer->usage = VK_BUFFER_USAGE_STORAGE_BUFFER_BIT | VK_BUFFER_USAGE_VERTEX_BUFFER_BIT | VK_BUFFER_USAGE_INDEX_BUFFER_BIT; buffer->memory.property = static_cast(VK_MEMORY_PROPERTY_HOST_VISIBLE_BIT) | @@ -1207,57 +1208,28 @@ BufferImageCopySource BufferCache::ObtainBufferForImage(uint64_t vaddr, uint64_t (vaddr & (TRACKER_PAGE_SIZE - 1)) == 0 && (size & (TRACKER_PAGE_SIZE - 1)) == 0); } FaultSafeCacheLock lock(this, m_mutex); - const bool cpu_dirty = m_memory_tracker.IsRegionCpuModified(vaddr, size); - auto use_coherent_guest_backing = [&](const CachedBuffer& overlap) { - const bool gpu_modified = m_memory_tracker.IsRegionGpuModified(vaddr, size); - const auto dirty_ranges = m_gpu_modified_ranges.Intersections(vaddr, size); - if (gpu_modified || !dirty_ranges.empty()) { - EXIT("BufferCache: GPU-modified image source requires a containing cached buffer, " - "addr=0x%016" PRIx64 " size=0x%016" PRIx64 - " buffer=0x%016" PRIx64 "+0x%016" PRIx64 " tracker_dirty=%d byte_ranges=%zu\n", - vaddr, size, overlap.vaddr, overlap.size, gpu_modified, dirty_ranges.size()); + const bool cpu_modified = m_memory_tracker.IsRegionCpuModified(vaddr, size); + const bool gpu_modified = m_memory_tracker.IsRegionGpuModified(vaddr, size); + const auto dirty_ranges = m_gpu_modified_ranges.Intersections(vaddr, size); + if (gpu_modified != !dirty_ranges.empty()) { + EXIT("BufferCache: image-source tracker and byte ownership disagree, addr=0x%016" PRIx64 + " size=0x%016" PRIx64 " tracker_dirty=%d byte_ranges=%zu\n", + vaddr, size, gpu_modified, dirty_ranges.size()); + } + auto find_owner = [&](uint64_t address, uint64_t bytes) noexcept -> CachedBuffer& { + auto owner = m_buffers.upper_bound(address); + if (owner == m_buffers.begin()) { + EXIT("BufferCache: GPU image-source bytes have no cached owner\n"); } - // A clean/CPU-current image range may coexist with a partial cached-buffer view because - // guest backing is authoritative. The PS5 detiler consumes that backing at the existing - // Tiler seam. - return BufferImageCopySource {nullptr, 0, vaddr, size, true, cpu_dirty}; + --owner; + auto& cached = *owner->second; + const auto offset = address - cached.vaddr; + if (offset > cached.size || bytes > cached.size - offset || cached.ctx == nullptr || + cached.buffer == nullptr || cached.buffer->buffer == nullptr) { + EXIT("BufferCache: GPU image-source bytes have no containing native buffer\n"); + } + return cached; }; - auto it = m_buffers.upper_bound(vaddr); - if (it == m_buffers.begin()) { - if (it != m_buffers.end() && - PageOverlaps(vaddr, size, it->second->vaddr, it->second->size)) { - return use_coherent_guest_backing(*it->second); - } - return {nullptr, 0, vaddr, size, true, cpu_dirty}; - } - --it; - auto& cached = *it->second; - const auto offset = vaddr - cached.vaddr; - if (offset > cached.size || size > cached.size - offset) { - const auto next = std::next(it); - if (PageOverlaps(vaddr, size, cached.vaddr, cached.size)) { - return use_coherent_guest_backing(cached); - } - if (next != m_buffers.end() && - PageOverlaps(vaddr, size, next->second->vaddr, next->second->size)) { - return use_coherent_guest_backing(*next->second); - } - return {nullptr, 0, vaddr, size, true, cpu_dirty}; - } - if (cached.ctx == nullptr || cached.buffer == nullptr || cached.buffer->buffer == nullptr) { - EXIT("BufferCache: inconsistent image source, addr=0x%016" PRIx64 " size=0x%016" PRIx64 - " buffer=0x%016" PRIx64 "+0x%016" PRIx64 " ctx=%p native=%p\n", - vaddr, size, cached.vaddr, cached.size, static_cast(cached.ctx), - static_cast(cached.buffer.get())); - } - const bool gpu_modified = m_memory_tracker.IsRegionGpuModified(vaddr, size); - const bool cpu_modified = cpu_dirty; - if (gpu_modified && cpu_modified) { - EXIT("BufferCache: image source is both CPU- and GPU-modified, addr=0x%016" PRIx64 - " size=0x%016" PRIx64 "\n", - vaddr, size); - } - const auto ranges = m_gpu_modified_ranges.Intersections(vaddr, size); if (gpu_modified) { if (!GraphicsRunIsCommandProcessorThread() && !GraphicsRunSubmissionLockHeld() && !LabelInCallback()) { @@ -1267,13 +1239,15 @@ BufferImageCopySource BufferCache::ObtainBufferForImage(uint64_t vaddr, uint64_t vaddr, size, GraphicsRunIsCommandProcessorThread(), GraphicsRunSubmissionLockHeld(), LabelInCallback()); } - if (ranges.empty()) { - EXIT("BufferCache: GPU-dirty image read has no dirty byte ranges, addr=0x%016" PRIx64 - " size=0x%016" PRIx64 "\n", - vaddr, size); + std::vector waited; + for (const auto& range: dirty_ranges) { + auto& owner = find_owner(range.address, range.size); + if (std::find(waited.begin(), waited.end(), owner.ctx) == waited.end()) { + VulkanDeviceWaitIdle(owner.ctx); + waited.push_back(owner.ctx); + } } - VulkanDeviceWaitIdle(cached.ctx); - auto backing_writes = ReserveBackingWrites(m_page_manager, ranges); + auto backing_writes = ReserveBackingWrites(m_page_manager, dirty_ranges); uint64_t downloaded = 0; m_memory_tracker.ForEachDownloadRange( vaddr, size, @@ -1288,9 +1262,10 @@ BufferImageCopySource BufferCache::ObtainBufferForImage(uint64_t vaddr, uint64_t address, bytes); } for (const auto& range: dirty) { - const auto buffer_offset = range.address - cached.vaddr; + auto& owner = find_owner(range.address, range.size); + const auto buffer_offset = range.address - owner.vaddr; std::vector data(range.size); - UtilDownloadBuffer(cached.ctx, cached.buffer.get(), buffer_offset, data.data(), + UtilDownloadBuffer(owner.ctx, owner.buffer.get(), buffer_offset, data.data(), range.size); Libs::LibKernel::Memory::WriteBacking(range.address, data.data(), data.size()); downloaded += range.size; @@ -1302,10 +1277,19 @@ BufferImageCopySource BufferCache::ObtainBufferForImage(uint64_t vaddr, uint64_t vaddr, size); } m_gpu_modified_ranges.Subtract(vaddr, size); - } else if (!ranges.empty()) { - EXIT("BufferCache: CPU-current image source retains GPU-dirty bytes, addr=0x%016" PRIx64 - " size=0x%016" PRIx64 "\n", - vaddr, size); + } + auto owner = m_buffers.upper_bound(vaddr); + if (owner == m_buffers.begin()) { + return {nullptr, 0, vaddr, size, true, cpu_modified}; + } + --owner; + auto& cached = *owner->second; + const auto offset = vaddr - cached.vaddr; + if (offset > cached.size || size > cached.size - offset) { + return {nullptr, 0, vaddr, size, true, cpu_modified}; + } + if (cached.ctx == nullptr || cached.buffer == nullptr || cached.buffer->buffer == nullptr) { + EXIT("BufferCache: containing image source is inconsistent\n"); } if (cpu_modified) { std::vector> uploads; @@ -1334,7 +1318,7 @@ BufferImageCopySource BufferCache::ObtainBufferForImage(uint64_t vaddr, uint64_t " size=0x%016" PRIx64 "\n", vaddr, size); } - return {cached.buffer.get(), offset, vaddr, size, true, cpu_dirty}; + return {cached.buffer.get(), offset, vaddr, size, true, cpu_modified}; } namespace { @@ -1384,7 +1368,7 @@ void BufferCache::FillBuffer(CommandBuffer* command, GraphicContext* ctx, uint64 ValidateGpuAccess(vaddr, size, false, true); { std::lock_guard transaction(m_resource_mutex); - const bool image_overlap = m_texture_cache->HasPageOverlap(vaddr, size); + const bool image_overlap = m_texture_cache->HasRangeOverlap(vaddr, size); const bool buffer_overlap = HasPageOverlap(vaddr, size); const bool buffer_gpu_modified = IsRegionGpuModified(vaddr, size); if (!buffer_overlap && !buffer_gpu_modified) { @@ -1400,7 +1384,7 @@ void BufferCache::FillBuffer(CommandBuffer* command, GraphicContext* ctx, uint64 " size=0x%016" PRIx64 "\n", vaddr, size); } - if (m_texture_cache->HasMetaOverlap(vaddr, size)) { + if (m_texture_cache->HasMetaRangeOverlap(vaddr, size)) { LOGF("BufferCache: GPU fill overlaps virtual metadata, addr=0x%016" PRIx64 " size=0x%016" PRIx64 "\n", vaddr, size); @@ -1439,15 +1423,15 @@ void BufferCache::CopyBuffer(CommandBuffer* command, GraphicContext* ctx, uint64 ValidateGpuAccess(dst_vaddr, size, false, true); { std::lock_guard transaction(m_resource_mutex); - const bool src_image = m_texture_cache->HasPageOverlap(src_vaddr, size); - const bool dst_image = m_texture_cache->HasPageOverlap(dst_vaddr, size); + const bool src_image_gpu = m_texture_cache->HasGpuModifiedRangeOverlap(src_vaddr, size); + const bool src_meta = m_texture_cache->HasMetaRangeOverlap(src_vaddr, size); if (m_texture_cache->HasGpuTargetPageOverlap(src_vaddr, size)) { EXIT("BufferCache: GPU copy aliases target pages, src=0x%016" PRIx64 " dst=0x%016" PRIx64 " size=0x%016" PRIx64 "\n", src_vaddr, dst_vaddr, size); } if (!HasPageOverlap(dst_vaddr, size) && !IsRegionGpuModified(src_vaddr, size) && - !IsRegionGpuModified(dst_vaddr, size)) { + !IsRegionGpuModified(dst_vaddr, size) && !src_image_gpu) { if (m_texture_cache->IsMetaGpuModified(src_vaddr, size)) { LOGF("BufferCache: host copy reads virtual metadata, src=0x%016" PRIx64 " size=0x%016" PRIx64 "\n", @@ -1456,13 +1440,19 @@ void BufferCache::CopyBuffer(CommandBuffer* command, GraphicContext* ctx, uint64 "src=0x%016" PRIx64 " size=0x%016" PRIx64 "\n", src_vaddr, size); } - if (dst_image) { + if (m_texture_cache->HasRangeOverlap(dst_vaddr, size)) { m_texture_cache->PrepareHostWrite(dst_vaddr, size); } std::memcpy(reinterpret_cast(dst_vaddr), reinterpret_cast(src_vaddr), size); return; } + if (src_image_gpu) { + EXIT("BufferCache: GPU copy source aliases GPU-modified image, src=0x%016" PRIx64 + " dst=0x%016" PRIx64 " size=0x%016" PRIx64 "\n", + src_vaddr, dst_vaddr, size); + } + m_texture_cache->InvalidateMemoryFromGPU(dst_vaddr, size); // A clean target destination is handled above like a protected host write. Target // aliases that require an actual GPU buffer copy remain unsupported. if (m_texture_cache->HasGpuTargetPageOverlap(dst_vaddr, size)) { @@ -1470,13 +1460,7 @@ void BufferCache::CopyBuffer(CommandBuffer* command, GraphicContext* ctx, uint64 " dst=0x%016" PRIx64 " size=0x%016" PRIx64 "\n", src_vaddr, dst_vaddr, size); } - if (src_image || dst_image) { - EXIT("BufferCache: GPU copy requires image-buffer synchronization, src=0x%016" PRIx64 - " dst=0x%016" PRIx64 " size=0x%016" PRIx64 "\n", - src_vaddr, dst_vaddr, size); - } - if (m_texture_cache->HasMetaOverlap(src_vaddr, size) || - m_texture_cache->HasMetaOverlap(dst_vaddr, size)) { + if (src_meta || m_texture_cache->HasMetaRangeOverlap(dst_vaddr, size)) { LOGF("BufferCache: GPU copy overlaps virtual metadata, src=0x%016" PRIx64 " dst=0x%016" PRIx64 " size=0x%016" PRIx64 "\n", src_vaddr, dst_vaddr, size); @@ -1606,7 +1590,7 @@ void BufferCache::SetTextureCache(TextureCache& texture_cache) { m_texture_cache = &texture_cache; } -//TODO: add LRU cache +// TODO: add LRU cache void BufferCache::RegisterForDelete(VulkanBuffer* buffer) { FaultSafeCacheLock lock(this, m_mutex); diff --git a/src/graphics/host_gpu/renderer/colorRenderTarget.cpp b/src/graphics/host_gpu/renderer/colorRenderTarget.cpp index 54f93b0cd..c995f04c2 100644 --- a/src/graphics/host_gpu/renderer/colorRenderTarget.cpp +++ b/src/graphics/host_gpu/renderer/colorRenderTarget.cpp @@ -315,9 +315,20 @@ void ResolveRenderColorTarget(uint64_t submit_id, CommandBuffer* buffer, const H } const bool msaa_compat = color_msaa_single_sample_compatible(rt.attrib.num_samples, rt.attrib.num_fragments); - if ((!msaa_compat && (rt.attrib.num_samples != 0 || rt.attrib.num_fragments != 0)) || - rt.view.base_array_slice_index != 0 || render_target_slice_offset != 0) { - EXIT("multisampled or sliced render targets are unsupported\n"); + if (!msaa_compat && (rt.attrib.num_samples != 0 || rt.attrib.num_fragments != 0)) { + EXIT("multisampled render targets are unsupported\n"); + } + const auto view = ResolveTargetViewInfo( + rt.view.base_array_slice_index, rt.view.last_array_slice_index, render_target_slice_offset); + switch (view.type) { + case TargetViewType::Image2D: break; + case TargetViewType::Image2DArray: + EXIT("layered render-target views are unsupported: base=%u count=%u\n", view.base_layer, + view.layer_count); + case TargetViewType::Unsupported: + EXIT("invalid render-target view: base=%u last=%u draw_offset=%u\n", + rt.view.base_array_slice_index, rt.view.last_array_slice_index, + render_target_slice_offset); } const uint32_t levels = rt.attrib2.num_mip_levels + 1u; if (levels == 0 || levels > 16 || rt.view.current_mip_level >= levels) { @@ -409,15 +420,16 @@ void ResolveRenderColorTarget(uint64_t submit_id, CommandBuffer* buffer, const H } else { size = static_cast(pitch) * height * bytes_per_element; } - if (size == 0 || size > TRACKER_ADDRESS_SIZE - rt.base.addr) { + if (size == 0 || size > UINT64_MAX / view.image_layers) { EXIT("render-target memory footprint is invalid\n"); } + const auto backing_size = size * view.image_layers; + if (backing_size > TRACKER_ADDRESS_SIZE - rt.base.addr) { + EXIT("render-target backing range is invalid\n"); + } - const auto effective_slice = rt.view.base_array_slice_index + render_target_slice_offset; - const auto effective_base = rt.base.addr + static_cast(size) * effective_slice; - - auto video_image = Presentation::DisplayBufferFind(effective_base, true); - bool render_to_texture = (video_image.image == nullptr); + auto video_image = Presentation::DisplayBufferFind(rt.base.addr, true); + bool render_to_texture = view.base_layer != 0 || video_image.image == nullptr; if (!render_to_texture && (levels != 1 || rt.view.current_mip_level != 0)) { EXIT("mipmapped display render targets are unsupported\n"); } @@ -430,7 +442,7 @@ void ResolveRenderColorTarget(uint64_t submit_id, CommandBuffer* buffer, const H " extent=%ux%u view_mip=%u view_extent=%ux%u levels=%u pitch=%u" " fmt=0x%08" PRIx32 " nfmt=0x%08" PRIx32 " order=0x%08" PRIx32 " tile=%s target=%s video_size=0x%016" PRIx64 " video_pitch=%" PRIu64 "\n", - rt_slot, effective_base, size, width, height, rt.view.current_mip_level, + rt_slot, rt.base.addr, backing_size, width, height, rt.view.current_mip_level, view_extent.width, view_extent.height, levels, pitch, rt.info.format, rt.info.channel_type, rt.info.channel_order, tile ? "tiled" : "linear", render_to_texture ? "RenderTexture" : "DisplayBuffer", video_image.size, @@ -443,8 +455,8 @@ void ResolveRenderColorTarget(uint64_t submit_id, CommandBuffer* buffer, const H (void)reuse_existing_render_texture; RenderTargetInfo target {}; - target.address = effective_base; - target.size = size; + target.address = rt.base.addr; + target.size = backing_size; target.format = rt_format.format; target.width = width; target.height = height; @@ -452,18 +464,20 @@ void ResolveRenderColorTarget(uint64_t submit_id, CommandBuffer* buffer, const H target.bytes_per_element = rt_format.bytes_per_element; target.tile_mode = rt.attrib3.tile_mode; target.levels = levels; + target.layers = view.image_layers; auto* texture_cache = g_render_ctx->GetTextureCache(); auto* buffer_vulkan = texture_cache->FindRenderTarget(buffer, g_render_ctx->GetGraphicCtx(), target); r->type = RenderColorType::RenderTexture; - r->base_addr = effective_base; + r->base_addr = rt.base.addr; r->vulkan_buffer = buffer_vulkan; r->vulkan_view = texture_cache->GetRenderTargetAttachmentView( - g_render_ctx->GetGraphicCtx(), buffer_vulkan, target.format, rt.view.current_mip_level); + g_render_ctx->GetGraphicCtx(), buffer_vulkan, target.format, rt.view.current_mip_level, + view.base_layer, view.layer_count); r->format = target.format; r->extent = view_extent; r->base_mip_level = rt.view.current_mip_level; - r->buffer_size = size; + r->buffer_size = backing_size; } else { const auto layout = static_cast(rt.info.format); const auto type = static_cast(rt.info.channel_type); @@ -489,7 +503,7 @@ void ResolveRenderColorTarget(uint64_t submit_id, CommandBuffer* buffer, const H EXIT_NOT_IMPLEMENTED(video_image.size < size); EXIT_NOT_IMPLEMENTED(video_image.pitch != pitch); r->type = RenderColorType::DisplayBuffer; - r->base_addr = effective_base; + r->base_addr = rt.base.addr; r->vulkan_buffer = video_image.image; r->vulkan_view = video_image.image->image_view[VulkanImage::VIEW_DEFAULT]; r->format = video_image.image->format; diff --git a/src/graphics/host_gpu/renderer/context.cpp b/src/graphics/host_gpu/renderer/context.cpp index 743f68b1a..db093ad38 100644 --- a/src/graphics/host_gpu/renderer/context.cpp +++ b/src/graphics/host_gpu/renderer/context.cpp @@ -608,7 +608,7 @@ void CommandBuffer::BeginRenderPass(VulkanFramebuffer* framebuffer, RenderColorI image_memory_barrier.subresourceRange.baseMipLevel = 0; image_memory_barrier.subresourceRange.levelCount = VK_REMAINING_MIP_LEVELS; image_memory_barrier.subresourceRange.baseArrayLayer = 0; - image_memory_barrier.subresourceRange.layerCount = 1; + image_memory_barrier.subresourceRange.layerCount = colors[i].vulkan_buffer->layers; vkCmdPipelineBarrier(buffer, VK_PIPELINE_STAGE_TOP_OF_PIPE_BIT, VK_PIPELINE_STAGE_COLOR_ATTACHMENT_OUTPUT_BIT, 0, 0, nullptr, 0, @@ -646,7 +646,7 @@ void CommandBuffer::BeginRenderPass(VulkanFramebuffer* framebuffer, RenderColorI image_memory_barrier.subresourceRange.baseMipLevel = 0; image_memory_barrier.subresourceRange.levelCount = 1; image_memory_barrier.subresourceRange.baseArrayLayer = 0; - image_memory_barrier.subresourceRange.layerCount = 1; + image_memory_barrier.subresourceRange.layerCount = depth->vulkan_buffer->layers; vkCmdPipelineBarrier( buffer, VK_PIPELINE_STAGE_ALL_GRAPHICS_BIT | VK_PIPELINE_STAGE_COMPUTE_SHADER_BIT, diff --git a/src/graphics/host_gpu/renderer/depthRenderTarget.cpp b/src/graphics/host_gpu/renderer/depthRenderTarget.cpp index 58ee6cc92..2f8702eae 100644 --- a/src/graphics/host_gpu/renderer/depthRenderTarget.cpp +++ b/src/graphics/host_gpu/renderer/depthRenderTarget.cpp @@ -115,6 +115,16 @@ void ResolveRenderDepthTarget(uint64_t submit_id, CommandBuffer* buffer, const H const bool msaa_compat = depth_msaa_single_sample_compatible(z.z_info.num_samples); const bool htile_stencil_compat = depth_htile_stencil_acceleration_compatible( has_stencil, has_htile, z.stencil_info.tile_stencil_disable); + const auto view = ResolveTargetViewInfo(z.depth_view.slice_start, z.depth_view.slice_max); + switch (view.type) { + case TargetViewType::Image2D: break; + case TargetViewType::Image2DArray: + DepthFatal("layered depth views are unsupported: base=%u count=%u", view.base_layer, + view.layer_count); + case TargetViewType::Unsupported: + DepthFatal("invalid depth view: base=%u last=%u", z.depth_view.slice_start, + z.depth_view.slice_max); + } // Prospero defines the compression-disable bits as tile writeback policy. Vulkan attachments // expose the same logical depth/stencil values regardless of the driver's backing compression. if ((stencil_active && !has_stencil) || rc.resummarize_enable || rc.copy_centroid || @@ -123,8 +133,7 @@ void ResolveRenderDepthTarget(uint64_t submit_id, CommandBuffer* buffer, const H z.stencil_info.partially_resident || z.z_info.plane_compression != 0 || (z.z_info.num_samples != 0 && !msaa_compat) || z.z_info.num_mip_levels != 0 || z.z_info.tile_mode_index != 0 || z.z_info.zrange_precision > 1 || - z.depth_view.current_mip_level != 0 || z.depth_view.slice_start != 0 || - z.depth_view.slice_max != 0 || z.depth_info.addr5_swizzle_mask != 0 || + z.depth_view.current_mip_level != 0 || z.depth_info.addr5_swizzle_mask != 0 || z.depth_info.array_mode != 0 || z.depth_info.pipe_config != 0 || z.depth_info.bank_width != 0 || z.depth_info.bank_height != 0 || z.depth_info.macro_tile_aspect != 0 || z.depth_info.num_banks != 0 || @@ -172,6 +181,9 @@ void ResolveRenderDepthTarget(uint64_t submit_id, CommandBuffer* buffer, const H if (z.htile_data_base_addr == 0 || (z.htile_data_base_addr & 0x7fffu) != 0) { DepthFatal("invalid HTile metadata address"); } + if (z.depth_view.slice_max >= 32) { + DepthFatal("HTile clear tracking supports at most 32 slices"); + } } else if (z.htile_data_base_addr != 0) { DepthFatal("HTile address without an enabled tile surface"); } @@ -234,9 +246,6 @@ void ResolveRenderDepthTarget(uint64_t submit_id, CommandBuffer* buffer, const H (has_htile != (htile_size.align == 32768 && htile_size.size != 0))) { DepthFatal("unsupported depth/stencil/HTile footprint"); } - if (has_htile) { - g_render_ctx->GetTextureCache()->RegisterMeta(z.htile_data_base_addr, htile_size.size); - } } else { TileGetTextureTotalSize(guest_format, width, height, 1, pitch, 1, Prospero::GpuEnumValue(Prospero::TileMode::kDepth), false, @@ -253,16 +262,32 @@ void ResolveRenderDepthTarget(uint64_t submit_id, CommandBuffer* buffer, const H z.pitch_height_valid ? 1u : 0u, (static_cast(z.slice_div64_minus1) + 1u) * 64u); } + if (depth_size.size > UINT64_MAX / view.image_layers || + stencil_size.size > UINT64_MAX / view.image_layers || + htile_size.size > UINT64_MAX / view.image_layers) { + DepthFatal("layered depth footprint overflow"); + } + const auto depth_backing_size = depth_size.size * view.image_layers; + const auto stencil_backing_size = stencil_size.size * view.image_layers; + const auto htile_backing_size = htile_size.size * view.image_layers; + if (depth_backing_size > TRACKER_ADDRESS_SIZE - z.z_read_base_addr || + (has_stencil && stencil_backing_size > TRACKER_ADDRESS_SIZE - z.stencil_read_base_addr) || + (has_htile && htile_backing_size > TRACKER_ADDRESS_SIZE - z.htile_data_base_addr)) { + DepthFatal("layered depth backing range is invalid"); + } r->htile = has_htile; r->width = width; r->height = height; - r->depth_buffer_size = depth_size.size; + r->depth_buffer_size = depth_backing_size; r->depth_buffer_vaddr = z.z_read_base_addr; - r->stencil_buffer_size = has_stencil ? stencil_size.size : 0; + r->stencil_buffer_size = has_stencil ? stencil_backing_size : 0; r->stencil_buffer_vaddr = has_stencil ? z.stencil_read_base_addr : 0; - r->htile_buffer_size = has_htile ? htile_size.size : 0; + r->htile_buffer_size = has_htile ? htile_backing_size : 0; r->htile_buffer_vaddr = has_htile ? z.htile_data_base_addr : 0; auto* cache = g_render_ctx->GetTextureCache(); + if (has_htile) { + cache->RegisterMeta(r->htile_buffer_vaddr, r->htile_buffer_size, view.image_layers); + } if (has_htile && rc.depth_clear_enable && !cache->ClearMeta(z.htile_data_base_addr)) { DepthFatal("failed to acquire HTile metadata for a depth clear"); } @@ -333,6 +358,7 @@ void ResolveRenderDepthTarget(uint64_t submit_id, CommandBuffer* buffer, const H info.pitch = pitch; info.bytes_per_element = bytes; info.tile_mode = Prospero::GpuEnumValue(Prospero::TileMode::kDepth); + info.layers = view.image_layers; info.depth_load_clear = r->depth_load_clear_enable; info.stencil_load_clear = rc.stencil_clear_enable; info.stencil_access = @@ -343,6 +369,8 @@ void ResolveRenderDepthTarget(uint64_t submit_id, CommandBuffer* buffer, const H info.stencil_htile_compressed = has_stencil && has_htile && !z.stencil_info.tile_stencil_disable; r->vulkan_buffer = cache->FindDepthTarget(buffer, g_render_ctx->GetGraphicCtx(), info); + r->vulkan_view = cache->GetDepthTargetAttachmentView( + g_render_ctx->GetGraphicCtx(), r->vulkan_buffer, view.base_layer, view.layer_count); if (meta_clear && !cache->TouchMeta(z.htile_data_base_addr, z.depth_view.slice_start, false)) { DepthFatal("failed to consume HTile clear state"); } diff --git a/src/graphics/host_gpu/renderer/descriptors.cpp b/src/graphics/host_gpu/renderer/descriptors.cpp index fb4807356..1d5e68493 100644 --- a/src/graphics/host_gpu/renderer/descriptors.cpp +++ b/src/graphics/host_gpu/renderer/descriptors.cpp @@ -238,12 +238,44 @@ static TextureVariant NativeTextureVariant(const ShaderRecompiler::IR::ImageReso } static bool IsSupportedSampledColorResource(const ShaderRecompiler::IR::ImageResource& resource) { - return resource.kind == ShaderRecompiler::IR::ResourceKind::Image && - resource.dimension == ShaderRecompiler::Decoder::ImageDimension::Dim2D && + bool supported_dimension = false; + switch (resource.dimension) { + case ShaderRecompiler::Decoder::ImageDimension::Dim2D: + case ShaderRecompiler::Decoder::ImageDimension::Dim2DArray: + supported_dimension = true; + break; + default: break; + } + return resource.kind == ShaderRecompiler::IR::ResourceKind::Image && supported_dimension && resource.mip_mode == ShaderRecompiler::IR::ImageMipMode::None && resource.read && !resource.written && !resource.atomic && !resource.depth_compare; } +struct TargetTextureViewInfo { + VkImageViewType type = VK_IMAGE_VIEW_TYPE_MAX_ENUM; + uint32_t base_layer = 0; + uint32_t layer_count = 0; +}; + +static TargetTextureViewInfo +ResolveTargetTextureView(const ShaderRecompiler::IR::ImageResource& resource, + Prospero::ImageType type, uint32_t base_layer, uint32_t image_layers) { + switch (type) { + case Prospero::ImageType::kColor2D: + return resource.dimension == ShaderRecompiler::Decoder::ImageDimension::Dim2D && + base_layer == 0 && image_layers == 1 + ? TargetTextureViewInfo {VK_IMAGE_VIEW_TYPE_2D, 0, 1} + : TargetTextureViewInfo {}; + case Prospero::ImageType::kColor2DArray: + return resource.dimension == ShaderRecompiler::Decoder::ImageDimension::Dim2DArray && + base_layer < image_layers + ? TargetTextureViewInfo {VK_IMAGE_VIEW_TYPE_2D_ARRAY, base_layer, + image_layers - base_layer} + : TargetTextureViewInfo {}; + default: return {}; + } +} + static bool IsSupportedSampledDepthFormat(VkFormat image_format, uint32_t guest_format, VkFormat view_format) { const bool d16 = image_format == VK_FORMAT_D16_UNORM && @@ -418,15 +450,17 @@ NativeTexture(uint64_t submit_id, CommandBuffer* command_buffer, EXIT("unsupported texture mip view: base=%u last=%u levels=%u\n", base_level, last_level, levels); } - const auto view_levels = last_level - base_level + 1u; - const auto depth = static_cast(descriptor.Depth()) + 1u; - const auto tile = descriptor.TileMode(); - const auto format = descriptor.Format(); - const auto view_format = TextureGetFormat(format, storage ? TextureFormatUsage::Storage - : TextureFormatUsage::Sampled); - const auto type = static_cast(descriptor.Type()); - const auto pitch = TileGetTexturePitch(format, width, levels, tile); - const auto swizzle = descriptor.DstSelXYZW(); + const auto view_levels = last_level - base_level + 1u; + const auto depth = static_cast(descriptor.Depth()) + 1u; + const auto tile = descriptor.TileMode(); + const auto format = descriptor.Format(); + const auto view_format = TextureGetFormat(format, storage ? TextureFormatUsage::Storage + : TextureFormatUsage::Sampled); + const auto type = static_cast(descriptor.Type()); + const auto target_view = + ResolveTargetTextureView(resource, type, descriptor.BaseArray5(), depth); + const auto pitch = TileGetTexturePitch(format, width, levels, tile); + const auto swizzle = descriptor.DstSelXYZW(); TileSizeAlign size; TileGetTextureTotalSize(format, width, height, depth, pitch, levels, tile, type == Prospero::ImageType::kColor3D, &size); @@ -465,11 +499,13 @@ NativeTexture(uint64_t submit_id, CommandBuffer* command_buffer, } else { if (!(storage ? IsSupportedStorageImageResource(resource) : IsSupportedSampledColorResource(resource)) || - image->type != VulkanImageType::RenderTexture || image->layers != 1 || - width != image->extent.width || height != image->extent.height || depth != 1 || + image->type != VulkanImageType::RenderTexture || width != image->extent.width || + height != image->extent.height || (storage ? levels != 1 || base_level != 0 : levels != image->mip_levels || base_level >= levels) || - descriptor.BaseArray5() != 0 || type != Prospero::ImageType::kColor2D) { + target_view.type == VK_IMAGE_VIEW_TYPE_MAX_ENUM || + target_view.base_layer >= image->layers || + target_view.layer_count > image->layers - target_view.base_layer) { EXIT("unsupported cached render-target image view: storage=%d resource=%u" " image_type=%u layers=%u extent=%ux%u/%ux%u depth=%u" " levels=%u/%u base_level=%u base_array=%u descriptor_type=%u\n", @@ -485,12 +521,14 @@ NativeTexture(uint64_t submit_id, CommandBuffer* command_buffer, image_view = g_render_ctx->GetTextureCache()->GetRenderTargetStorageView( g_render_ctx->GetGraphicCtx(), static_cast(image), view_format, base_level, - view_levels); + view_levels, target_view.type, target_view.base_layer, + target_view.layer_count); } else { image_view = g_render_ctx->GetTextureCache()->GetRenderTargetSampledView( g_render_ctx->GetGraphicCtx(), static_cast(image), view_format, view, - base_level, view_levels); + base_level, view_levels, target_view.type, target_view.base_layer, + target_view.layer_count); } } if (image_view == nullptr && image->image_view[view] == nullptr) { diff --git a/src/graphics/host_gpu/renderer/framebufferCache.cpp b/src/graphics/host_gpu/renderer/framebufferCache.cpp index 57a59f1bd..f6ecabaae 100644 --- a/src/graphics/host_gpu/renderer/framebufferCache.cpp +++ b/src/graphics/host_gpu/renderer/framebufferCache.cpp @@ -70,6 +70,7 @@ VulkanFramebuffer* FramebufferCache::CreateFramebuffer(RenderColorInfo* colors, } depth->format = VK_FORMAT_UNDEFINED; depth->vulkan_buffer = nullptr; + depth->vulkan_view = nullptr; depth->depth_test_enable = false; depth->depth_write_enable = false; depth->depth_bounds_test_enable = false; @@ -97,6 +98,7 @@ VulkanFramebuffer* FramebufferCache::CreateFramebuffer(RenderColorInfo* colors, } if (color_match && f.depth_id == (with_depth ? depth->vulkan_buffer->memory.unique_id : 0) && + f.depth_view == (with_depth ? depth->vulkan_view : nullptr) && f.depth_clear_enable == depth->depth_load_clear_enable && f.stencil_clear_enable == depth->stencil_clear_enable && f.depth_read_only == depth_read_only) { @@ -237,7 +239,10 @@ VulkanFramebuffer* FramebufferCache::CreateFramebuffer(RenderColorInfo* colors, views[i] = colors[i].vulkan_view; } if (with_depth) { - views[color_count] = depth->vulkan_buffer->image_view[VulkanImage::VIEW_DEFAULT]; + if (depth->vulkan_view == nullptr) { + EXIT("Framebuffer: depth attachment view is missing\n"); + } + views[color_count] = depth->vulkan_view; } VkFramebufferCreateInfo framebuffer_info {}; @@ -268,6 +273,7 @@ VulkanFramebuffer* FramebufferCache::CreateFramebuffer(RenderColorInfo* colors, fnew.color_layout[i] = color_layout[i]; } fnew.depth_id = (with_depth ? depth->vulkan_buffer->memory.unique_id : 0); + fnew.depth_view = (with_depth ? depth->vulkan_view : nullptr); fnew.depth_clear_enable = depth->depth_load_clear_enable; fnew.stencil_clear_enable = depth->stencil_clear_enable; fnew.depth_read_only = depth_read_only; diff --git a/src/graphics/host_gpu/renderer/framebufferCache.h b/src/graphics/host_gpu/renderer/framebufferCache.h index 64e36abac..cfd141903 100644 --- a/src/graphics/host_gpu/renderer/framebufferCache.h +++ b/src/graphics/host_gpu/renderer/framebufferCache.h @@ -63,6 +63,7 @@ private: uint64_t image_id[RENDER_COLOR_ATTACHMENTS_MAX] = {}; VkImageView color_view[RENDER_COLOR_ATTACHMENTS_MAX] = {}; uint64_t depth_id = 0; + VkImageView depth_view = nullptr; bool color_clear_enable[RENDER_COLOR_ATTACHMENTS_MAX] = {}; VkImageLayout color_layout[RENDER_COLOR_ATTACHMENTS_MAX] = {}; bool depth_clear_enable = false; diff --git a/src/graphics/host_gpu/renderer/imageInfo.h b/src/graphics/host_gpu/renderer/imageInfo.h index 5cf5db4e7..bccede86b 100644 --- a/src/graphics/host_gpu/renderer/imageInfo.h +++ b/src/graphics/host_gpu/renderer/imageInfo.h @@ -39,6 +39,7 @@ struct RenderTargetInfo { uint32_t bytes_per_element = 0; uint32_t tile_mode = 0; uint32_t levels = 1; + uint32_t layers = 1; }; // Common image-to-buffer copy description. Storage images and render targets keep distinct @@ -70,22 +71,23 @@ MakeColorImageTransferInfo(const RenderTargetInfo& info) noexcept { } struct DepthTargetInfo { - uint64_t address = 0; - uint64_t size = 0; - uint64_t stencil_address = 0; - uint64_t stencil_size = 0; - uint64_t htile_address = 0; - uint64_t htile_size = 0; - VkFormat format = VK_FORMAT_UNDEFINED; - uint32_t guest_format = 0; - uint32_t width = 0; - uint32_t height = 0; - uint32_t pitch = 0; - uint32_t bytes_per_element = 0; - uint32_t tile_mode = 0; - bool depth_load_clear = false; - bool stencil_load_clear = false; - bool stencil_access = false; + uint64_t address = 0; + uint64_t size = 0; + uint64_t stencil_address = 0; + uint64_t stencil_size = 0; + uint64_t htile_address = 0; + uint64_t htile_size = 0; + VkFormat format = VK_FORMAT_UNDEFINED; + uint32_t guest_format = 0; + uint32_t width = 0; + uint32_t height = 0; + uint32_t pitch = 0; + uint32_t bytes_per_element = 0; + uint32_t tile_mode = 0; + uint32_t layers = 1; + bool depth_load_clear = false; + bool stencil_load_clear = false; + bool stencil_access = false; bool stencil_htile_compressed = false; }; @@ -163,25 +165,29 @@ struct VideoOutPixelFormatInfo { } [[nodiscard]] inline bool IsSupportedVideoOutFormat(uint32_t guest_format, VkFormat format) { - const bool rgba8 = guest_format == Prospero::GpuEnumValue(Prospero::BufferFormat::k8_8_8_8Srgb) && - (format == VK_FORMAT_R8G8B8A8_SRGB || format == VK_FORMAT_B8G8R8A8_SRGB); - const bool rgb10 = guest_format == Prospero::GpuEnumValue(Prospero::BufferFormat::k10_10_10_2UNorm) && - (format == VK_FORMAT_A2B10G10R10_UNORM_PACK32 || - format == VK_FORMAT_A2R10G10B10_UNORM_PACK32); + const bool rgba8 = + guest_format == Prospero::GpuEnumValue(Prospero::BufferFormat::k8_8_8_8Srgb) && + (format == VK_FORMAT_R8G8B8A8_SRGB || format == VK_FORMAT_B8G8R8A8_SRGB); + const bool rgb10 = + guest_format == Prospero::GpuEnumValue(Prospero::BufferFormat::k10_10_10_2UNorm) && + (format == VK_FORMAT_A2B10G10R10_UNORM_PACK32 || + format == VK_FORMAT_A2R10G10B10_UNORM_PACK32); return rgba8 || rgb10; } -enum class DepthOverlap : uint8_t { None, RetireSampled, Unsupported }; +enum class DepthOverlap : uint8_t { None, RetireSampled, ExpandTarget, Unsupported }; enum class DepthTransitionSource : uint8_t { None, Guest, Native }; enum class RenderTargetOverlap : uint8_t { None, RetireSampled, PreserveStorage, + ExpandTarget, RetireTarget, Unsupported }; enum class SampledOverlap : uint8_t { None, ReadOnlyAlias, Unsupported }; enum class StorageSampledOverlap : uint8_t { None, ExactImage, Unsupported }; +enum class StorageImageOverlap : uint8_t { None, RetireSampled, PageNeighbor, Unsupported }; enum class HostWriteOverlap : uint8_t { None, InvalidateImage, Unsupported }; enum class BufferImageAlias : uint8_t { None, VideoOutWrite, RenderTargetWrite, Unsupported }; enum class MetaImageOverlap : uint8_t { RetainSampled, RetireTarget, Unsupported }; @@ -295,14 +301,15 @@ SelectDepthTransitionSource(bool depth_load_clear, bool sampled_native_available uint64_t address, uint64_t size) { const bool d32 = target.format == VK_FORMAT_D32_SFLOAT || target.format == VK_FORMAT_D32_SFLOAT_S8_UINT; - return address == target.address && size == target.size && d32 && + return address == target.address && size == target.size && target.layers == 1 && d32 && target.guest_format == Prospero::GpuEnumValue(Prospero::BufferFormat::k32Float) && - target.bytes_per_element == 4 && target.tile_mode == Prospero::GpuEnumValue(Prospero::TileMode::kDepth) && + target.bytes_per_element == 4 && + target.tile_mode == Prospero::GpuEnumValue(Prospero::TileMode::kDepth) && target.htile_address == 0 && target.htile_size == 0; } [[nodiscard]] inline bool CanLoadStencilAttachment(const DepthTargetInfo& target, - bool stencil_initialized) { + bool stencil_initialized) { const bool has_stencil = target.stencil_address != 0 || target.stencil_size != 0; return !has_stencil || !target.stencil_access || target.stencil_load_clear || stencil_initialized; @@ -448,11 +455,12 @@ ClassifyBufferImageAlias(uint64_t buffer_address, uint64_t buffer_size, uint64_t depth.format == VK_FORMAT_D32_SFLOAT && depth.bytes_per_element == 4)); const bool exact_depth_load = !has_stencil && !depth.depth_load_clear && sampled.address == depth.address && - sampled.size == depth.size && sampled.width == depth.width && sampled.height == depth.height && - sampled.pitch == depth.pitch && sampled.base_level == 0 && sampled.levels == 1 && - sampled.view_levels == 1 && sampled.tile == depth.tile_mode && sampled.depth == 1 && + sampled.size == depth.size && sampled.width == depth.width && + sampled.height == depth.height && sampled.pitch == depth.pitch && sampled.base_level == 0 && + sampled.levels == 1 && sampled.view_levels == 1 && sampled.tile == depth.tile_mode && + sampled.depth == 1 && sampled.type == Prospero::GpuEnumValue(Prospero::ImageType::kColor2D) && - sampled.base_array == 0 && exact_depth_format; + depth.layers == 1 && sampled.base_array == 0 && exact_depth_format; if (!sampled_gpu_modified && exact_depth_load) { return DepthOverlap::RetireSampled; } @@ -480,10 +488,11 @@ ClassifyRenderTargetOverlap(const ImageInfo& sampled, bool sampled_gpu_modified, : RenderTargetOverlap::Unsupported; } -[[nodiscard]] inline RenderTargetOverlap ClassifyStorageRenderTargetOverlap( - const ImageInfo& storage, VkFormat storage_format, bool storage_gpu_modified, - bool storage_buffer_modified, bool storage_cpu_dirty, bool same_context, - const RenderTargetInfo& target) { +[[nodiscard]] inline RenderTargetOverlap +ClassifyStorageRenderTargetOverlap(const ImageInfo& storage, VkFormat storage_format, + bool storage_gpu_modified, bool storage_buffer_modified, + bool storage_cpu_dirty, bool same_context, + const RenderTargetInfo& target) { if (!ImagePageRangesOverlap(storage.address, storage.size, target.address, target.size)) { return RenderTargetOverlap::None; } @@ -494,7 +503,7 @@ ClassifyRenderTargetOverlap(const ImageInfo& sampled, bool sampled_gpu_modified, storage.base_level == 0 && storage.levels == 1 && storage.view_levels == 1 && storage.tile == target.tile_mode && storage.depth == 1 && storage.type == Prospero::GpuEnumValue(Prospero::ImageType::kColor2D) && - storage.base_array == 0 && target.levels == 1; + storage.base_array == 0 && target.levels == 1 && target.layers == 1; return exact_native_image && storage_gpu_modified && !storage_buffer_modified && !storage_cpu_dirty && same_context ? RenderTargetOverlap::PreserveStorage @@ -521,9 +530,85 @@ ClassifyRenderTargetOverlap(const ImageInfo& sampled, bool sampled_gpu_modified, cached.pitch == requested.pitch && cached.bytes_per_element == requested.bytes_per_element && cached.tile_mode == requested.tile_mode && cached.levels == requested.levels && + cached.layers == requested.layers && IsRgba8SrgbReinterpretation(cached.format, requested.format); } +[[nodiscard]] inline RenderTargetOverlap +ClassifySampledRenderTargetOverlap(const ImageInfo& sampled, const RenderTargetInfo& target, + bool target_buffer_modified, bool same_context) { + if (!ImagePageRangesOverlap(sampled.address, sampled.size, target.address, target.size)) { + return RenderTargetOverlap::None; + } + return same_context && !target_buffer_modified ? RenderTargetOverlap::RetireTarget + : RenderTargetOverlap::Unsupported; +} + +[[nodiscard]] inline StorageImageOverlap +ClassifyStorageImageOverlap(uint64_t requested_address, uint64_t requested_size, + uint64_t cached_address, uint64_t cached_size, bool sampled, + bool same_context, bool gpu_modified, bool buffer_modified, + bool tracker_gpu_modified) { + if (!ImagePageRangesOverlap(requested_address, requested_size, cached_address, cached_size)) { + return StorageImageOverlap::None; + } + if (!ImageRangeOverlaps(requested_address, requested_size, cached_address, cached_size)) { + return StorageImageOverlap::PageNeighbor; + } + return sampled && same_context && !gpu_modified && !buffer_modified && !tracker_gpu_modified + ? StorageImageOverlap::RetireSampled + : StorageImageOverlap::Unsupported; +} + +[[nodiscard]] inline constexpr bool LayeredBackingContains(uint64_t container_size, + uint32_t container_layers, + uint64_t view_size, + uint32_t view_layers) { + return container_layers >= view_layers && container_layers != 0 && view_layers != 0 && + container_size % container_layers == 0 && view_size % view_layers == 0 && + container_size / container_layers == view_size / view_layers; +} + +[[nodiscard]] inline constexpr bool +LayeredPlaneContains(uint64_t container_address, uint64_t container_size, uint32_t container_layers, + uint64_t view_address, uint64_t view_size, uint32_t view_layers) { + return container_address == 0 ? view_address == 0 && view_size == 0 + : view_address == container_address && + LayeredBackingContains(container_size, container_layers, + view_size, view_layers); +} + +[[nodiscard]] inline bool +IsCompatibleRenderTargetBacking(const RenderTargetInfo& cached, + const RenderTargetInfo& requested) noexcept { + return cached.address == requested.address && + LayeredBackingContains(cached.size, cached.layers, requested.size, requested.layers) && + cached.width == requested.width && cached.height == requested.height && + cached.pitch == requested.pitch && + cached.bytes_per_element == requested.bytes_per_element && + cached.tile_mode == requested.tile_mode && cached.levels == requested.levels && + (cached.format == requested.format || + IsRgba8SrgbReinterpretation(cached.format, requested.format)); +} + +[[nodiscard]] inline bool +IsCompatibleDepthTargetBacking(const DepthTargetInfo& cached, + const DepthTargetInfo& requested) noexcept { + return cached.address == requested.address && + LayeredBackingContains(cached.size, cached.layers, requested.size, requested.layers) && + LayeredPlaneContains(cached.stencil_address, cached.stencil_size, cached.layers, + requested.stencil_address, requested.stencil_size, + requested.layers) && + LayeredPlaneContains(cached.htile_address, cached.htile_size, cached.layers, + requested.htile_address, requested.htile_size, requested.layers) && + cached.format == requested.format && cached.guest_format == requested.guest_format && + cached.width == requested.width && cached.height == requested.height && + cached.pitch == requested.pitch && + cached.bytes_per_element == requested.bytes_per_element && + cached.tile_mode == requested.tile_mode && + cached.stencil_htile_compressed == requested.stencil_htile_compressed; +} + [[nodiscard]] inline RenderTargetOverlap ClassifyRenderTargetOverlap(const RenderTargetInfo& cached, bool cached_gpu_modified, bool cached_buffer_modified, bool same_context, @@ -531,8 +616,15 @@ ClassifyRenderTargetOverlap(const RenderTargetInfo& cached, bool cached_gpu_modi if (!ImagePageRangesOverlap(cached.address, cached.size, requested.address, requested.size)) { return RenderTargetOverlap::None; } + const bool expand = requested.layers > cached.layers && + IsCompatibleRenderTargetBacking(requested, cached) && + cached.format == requested.format; + if (expand && cached_gpu_modified && !cached_buffer_modified && same_context) { + return RenderTargetOverlap::ExpandTarget; + } // For an equal-address allocation-pool entry, a changed block raster or block size is a new - // image allocation, not a view of the old image. In Kyty we can only retire an already-published target with page-isolated storage :) + // image allocation, not a view of the old image. In Kyty we can only retire an + // already-published target with page-isolated storage :) const bool page_isolated = cached.address % TRACKER_PAGE_SIZE == 0 && cached.size % TRACKER_PAGE_SIZE == 0 && requested.address % TRACKER_PAGE_SIZE == 0 && requested.size % TRACKER_PAGE_SIZE == 0; @@ -545,6 +637,21 @@ ClassifyRenderTargetOverlap(const RenderTargetInfo& cached, bool cached_gpu_modi : RenderTargetOverlap::Unsupported; } +[[nodiscard]] inline DepthOverlap ClassifyDepthTargetOverlap(const DepthTargetInfo& cached, + bool cached_gpu_modified, + bool cached_buffer_modified, + bool same_context, + const DepthTargetInfo& requested) { + if (!ImagePageRangesOverlap(cached.address, cached.size, requested.address, requested.size)) { + return DepthOverlap::None; + } + const bool expand = + requested.layers > cached.layers && IsCompatibleDepthTargetBacking(requested, cached); + return expand && cached_gpu_modified && !cached_buffer_modified && same_context + ? DepthOverlap::ExpandTarget + : DepthOverlap::Unsupported; +} + } // namespace Libs::Graphics #endif // EMULATOR_SRC_GRAPHICS_HOST_GPU_RENDERER_IMAGEINFO_H_ diff --git a/src/graphics/host_gpu/renderer/renderState.h b/src/graphics/host_gpu/renderer/renderState.h index 19e807480..5b9d45268 100644 --- a/src/graphics/host_gpu/renderer/renderState.h +++ b/src/graphics/host_gpu/renderer/renderState.h @@ -40,6 +40,24 @@ inline constexpr bool color_msaa_single_sample_compatible(uint32_t encoded_sampl depth_msaa_single_sample_compatible(encoded_fragments); } +enum class TargetViewType : uint8_t { Image2D, Image2DArray, Unsupported }; + +struct TargetViewInfo { + TargetViewType type = TargetViewType::Unsupported; + uint32_t base_layer = 0; + uint32_t layer_count = 0; + uint32_t image_layers = 0; +}; + +inline constexpr TargetViewInfo ResolveTargetViewInfo(uint32_t base_layer, uint32_t last_layer, + uint32_t draw_layer_offset = 0) { + if (base_layer > last_layer || draw_layer_offset != 0) { + return {}; + } + return {base_layer == last_layer ? TargetViewType::Image2D : TargetViewType::Image2DArray, + base_layer, last_layer - base_layer + 1u, last_layer + 1u}; +} + inline constexpr bool depth_htile_stencil_acceleration_compatible(bool has_stencil, bool has_htile, bool acceleration_disabled) { return acceleration_disabled || (has_stencil && has_htile); @@ -147,8 +165,8 @@ static_assert(std::is_trivially_copyable_v); static_assert(std::is_standard_layout_v); static_assert(alignof(PipelineDynamicParameters) == 1); static_assert(sizeof(PipelineDynamicParameters) == - sizeof(bool) + sizeof(float[3]) + sizeof(float[3]) + sizeof(int[4]) + - sizeof(float) + sizeof(uint32_t) + sizeof(bool[RENDER_COLOR_ATTACHMENTS_MAX]) + + sizeof(bool) + sizeof(float[3]) + sizeof(float[3]) + sizeof(int[4]) + sizeof(float) + + sizeof(uint32_t) + sizeof(bool[RENDER_COLOR_ATTACHMENTS_MAX]) + sizeof(PipelineStencilDynamicState) * 2); struct RenderDepthInfo { @@ -183,6 +201,7 @@ struct RenderDepthInfo { PipelineStencilDynamicState stencil_dynamic_front; PipelineStencilDynamicState stencil_dynamic_back; DepthStencilVulkanImage* vulkan_buffer = nullptr; + VkImageView vulkan_view = nullptr; uint64_t vaddr[3] = {}; uint64_t size[3] = {}; int vaddr_num = 0; diff --git a/src/graphics/host_gpu/renderer/textureCache.cpp b/src/graphics/host_gpu/renderer/textureCache.cpp index 2f6fb2cbb..b6ef60a52 100644 --- a/src/graphics/host_gpu/renderer/textureCache.cpp +++ b/src/graphics/host_gpu/renderer/textureCache.cpp @@ -3,9 +3,9 @@ #include "common/assert.h" #include "common/logging/log.h" #include "common/profiler.h" -#include "graphics/guest_gpu/graphicsRun.h" #include "graphics/guest_gpu/gpu_defs.h" #include "graphics/guest_gpu/gpu_format.h" +#include "graphics/guest_gpu/graphicsRun.h" #include "graphics/guest_gpu/tile.h" #include "graphics/host_gpu/graphicContext.h" #include "graphics/host_gpu/hostMemory.h" @@ -64,6 +64,17 @@ bool PageOverlaps(uint64_t left, uint64_t left_size, uint64_t right, uint64_t ri return ImagePageRangesOverlap(left, left_size, right, right_size); } +enum class ImageRangeOverlap : uint8_t { None, PageOnly, Bytes }; + +ImageRangeOverlap ClassifyImageRangeOverlap(uint64_t left, uint64_t left_size, uint64_t right, + uint64_t right_size) { + if (Overlaps(left, left_size, right, right_size)) { + return ImageRangeOverlap::Bytes; + } + return PageOverlaps(left, left_size, right, right_size) ? ImageRangeOverlap::PageOnly + : ImageRangeOverlap::None; +} + } // namespace struct TextureCache::CachedImage { @@ -231,9 +242,9 @@ struct TextureCache::ReadbackWorker { state.wait(expected, std::memory_order_acquire); expected = State::Idle; } - access = fault_access; - vaddr = fault_vaddr; - size = fault_size; + access = fault_access; + vaddr = fault_vaddr; + size = fault_size; submissions_prepaused = submissions_prepaused_now; state.store(State::Requested, std::memory_order_release); state.notify_all(); @@ -302,44 +313,54 @@ struct TextureCache::ReadbackWorker { } [[nodiscard]] ReadbackRange DownloadDepthTarget(CachedImage& cached) { - const auto& info = cached.depth; - const bool has_stencil = info.stencil_address != 0 || info.stencil_size != 0; - const bool has_htile = info.htile_address != 0 || info.htile_size != 0; - const bool fault_in_depth = vaddr >= info.address && vaddr < info.address + info.size && - size <= info.address + info.size - vaddr; - const bool d16 = info.guest_format == Prospero::GpuEnumValue(Prospero::BufferFormat::k16UNorm) && - info.format == VK_FORMAT_D16_UNORM && info.bytes_per_element == 2; - const bool d32 = info.guest_format == Prospero::GpuEnumValue(Prospero::BufferFormat::k32Float) && - info.format == VK_FORMAT_D32_SFLOAT && info.bytes_per_element == 4; + const auto& info = cached.depth; + const bool has_stencil = info.stencil_address != 0 || info.stencil_size != 0; + const bool has_htile = info.htile_address != 0 || info.htile_size != 0; + const bool fault_in_depth = vaddr >= info.address && vaddr < info.address + info.size && + size <= info.address + info.size - vaddr; + const bool d16 = + info.guest_format == Prospero::GpuEnumValue(Prospero::BufferFormat::k16UNorm) && + info.format == VK_FORMAT_D16_UNORM && info.bytes_per_element == 2; + const bool d32 = + info.guest_format == Prospero::GpuEnumValue(Prospero::BufferFormat::k32Float) && + info.format == VK_FORMAT_D32_SFLOAT && info.bytes_per_element == 4; TileSizeAlign expected_stencil {}; TileSizeAlign expected_htile {}; TileSizeAlign expected_depth {}; const bool prospero_layout = (d16 || d32) && TileGetDepthSize(info.width, info.height, 0, - Prospero::GpuEnumValue(d16 ? Prospero::DepthFormat::kZ16 : Prospero::DepthFormat::kZ32F), - Prospero::GpuEnumValue(Prospero::StencilFormat::kInvalid), has_htile, &expected_stencil, - &expected_htile, &expected_depth); - const auto expected_pitch = prospero_layout - ? TileGetTexturePitch(info.guest_format, info.width, 1, - Prospero::GpuEnumValue(Prospero::TileMode::kDepth)) - : 0u; + Prospero::GpuEnumValue(d16 ? Prospero::DepthFormat::kZ16 + : Prospero::DepthFormat::kZ32F), + Prospero::GpuEnumValue(Prospero::StencilFormat::kInvalid), has_htile, + &expected_stencil, &expected_htile, &expected_depth); + const auto expected_pitch = + prospero_layout + ? TileGetTexturePitch(info.guest_format, info.width, 1, + Prospero::GpuEnumValue(Prospero::TileMode::kDepth)) + : 0u; + const bool layered_sizes = + info.layers != 0 && info.size <= UINT32_MAX && + expected_depth.size <= UINT64_MAX / info.layers && + info.size == expected_depth.size * info.layers && + (!has_htile || (expected_htile.size <= UINT64_MAX / info.layers && + info.htile_size == expected_htile.size * info.layers)); if (!fault_in_depth || has_stencil || info.address == 0 || info.size == 0 || info.width == 0 || info.height == 0 || info.pitch < info.width || - info.tile_mode != Prospero::GpuEnumValue(Prospero::TileMode::kDepth) || !prospero_layout || - info.pitch != expected_pitch || info.size != expected_depth.size || - expected_depth.align != 65536u || - (has_htile && - (info.htile_size != expected_htile.size || expected_htile.align != 32768u))) { + info.tile_mode != Prospero::GpuEnumValue(Prospero::TileMode::kDepth) || + !prospero_layout || info.pitch != expected_pitch || !layered_sizes || + expected_depth.align != 65536u || cached.image->layers != info.layers || + (has_htile && expected_htile.align != 32768u)) { EXIT("TextureCache: unsupported depth-image readback layout, fault=0x%016" PRIx64 "+0x%016" PRIx64 " depth=0x%016" PRIx64 "+0x%016" PRIx64 " stencil=0x%016" PRIx64 "+0x%016" PRIx64 " htile=0x%016" PRIx64 "+0x%016" PRIx64 - " extent=%ux%u pitch=%u/%u tile=%u format=%d guest_format=%u bpe=%u " + " extent=%ux%u pitch=%u/%u layers=%u/%u tile=%u format=%d guest_format=%u bpe=%u " "expected_depth=0x%016" PRIx64 " expected_htile=0x%016" PRIx64 "\n", vaddr, size, info.address, info.size, info.stencil_address, info.stencil_size, info.htile_address, info.htile_size, info.width, info.height, info.pitch, - expected_pitch, info.tile_mode, static_cast(info.format), info.guest_format, - info.bytes_per_element, expected_depth.size, expected_htile.size); + expected_pitch, info.layers, cached.image->layers, info.tile_mode, + static_cast(info.format), info.guest_format, info.bytes_per_element, + expected_depth.size, expected_htile.size); } const auto rows = static_cast(info.height - 1); if (rows > (UINT64_MAX - info.width) / info.pitch) { @@ -350,9 +371,10 @@ struct TextureCache::ReadbackWorker { EXIT("TextureCache: depth-image readback size overflow\n"); } const auto linear_size = linear_elements * info.bytes_per_element; + const auto slice_size = info.size / info.layers; const bool meta_overlap = cache.HasMetaOverlapLocked(info.address, info.size); const bool buffer_overlap = cache.m_buffer_cache.HasPageOverlap(info.address, info.size); - if (linear_size > info.size || cached.image->format != info.format || + if (linear_size > slice_size || cached.image->format != info.format || cached.image->extent.width != info.width || cached.image->extent.height != info.height || meta_overlap || buffer_overlap) { EXIT("TextureCache: depth-image readback storage is unsupported, " @@ -365,8 +387,20 @@ struct TextureCache::ReadbackWorker { } download.resize(info.size); std::fill(download.begin(), download.end(), 0); - UtilFillBuffer(cached.ctx, download.data(), linear_size, info.pitch, cached.image, - cached.image->layout, VK_IMAGE_ASPECT_DEPTH_BIT); + std::vector regions; + regions.reserve(info.layers); + for (uint32_t layer = 0; layer < info.layers; layer++) { + ImageBufferCopy region {}; + region.offset = static_cast(slice_size * layer); + region.pitch = info.pitch; + region.width = info.width; + region.height = info.height; + region.src_layer = layer; + region.aspect = VK_IMAGE_ASPECT_DEPTH_BIT; + regions.push_back(region); + } + UtilFillBuffer(cached.ctx, download.data(), info.size, regions, cached.image, + cached.image->layout); guest.resize(info.size); cache.m_tiler.TileImage(guest.data(), download.data(), info); Libs::LibKernel::Memory::WriteBacking(info.address, guest.data(), info.size); @@ -376,20 +410,33 @@ struct TextureCache::ReadbackWorker { [[nodiscard]] ReadbackRange DownloadColorImage(CachedImage& cached) { const bool storage = cached.kind == CachedImage::Kind::StorageTexture; const bool target = cached.kind == CachedImage::Kind::RenderTarget; - const auto info = - storage ? MakeColorImageTransferInfo(cached.info, Prospero::SurfaceFormat(cached.info.format), - Prospero::NumBytesPerElement(cached.info.format)) - : MakeColorImageTransferInfo(cached.target); - const bool linear = info.tile_mode == Prospero::GpuEnumValue(Prospero::TileMode::kLinear); - const bool tiled = target && info.tile_mode == Prospero::GpuEnumValue(Prospero::TileMode::kRenderTarget); + const auto info = storage ? MakeColorImageTransferInfo( + cached.info, Prospero::SurfaceFormat(cached.info.format), + Prospero::NumBytesPerElement(cached.info.format)) + : MakeColorImageTransferInfo(cached.target); + const bool linear = info.tile_mode == Prospero::GpuEnumValue(Prospero::TileMode::kLinear); + const bool tiled = + target && info.tile_mode == Prospero::GpuEnumValue(Prospero::TileMode::kRenderTarget); + bool single_layer_storage = false; + if (storage) { + switch (static_cast(cached.info.type)) { + case Prospero::ImageType::kColor2D: + case Prospero::ImageType::kColor2DArray: + single_layer_storage = cached.info.depth == 1; + break; + default: break; + } + } const bool basic_storage = - !storage || (cached.info.type == Prospero::GpuEnumValue(Prospero::ImageType::kColor2D) && - cached.info.depth == 1 && cached.info.base_level == 0 && + !storage || (single_layer_storage && cached.info.base_level == 0 && cached.info.levels == 1 && cached.info.base_array == 0 && linear); - const auto rows = static_cast(info.height - 1); + const auto layers = target ? cached.target.layers : 1u; + const auto rows = static_cast(info.height - 1); if ((!linear && !tiled) || !basic_storage || info.address == 0 || info.size == 0 || info.width == 0 || info.height == 0 || info.levels != 1 || info.pitch < info.width || - info.bytes_per_element == 0 || rows > (UINT64_MAX - info.width) / info.pitch) { + info.bytes_per_element == 0 || layers == 0 || info.size > UINT32_MAX || + info.size % layers != 0 || cached.image->layers != layers || + rows > (UINT64_MAX - info.width) / info.pitch) { EXIT("TextureCache: unsupported color-image readback layout, addr=0x%016" PRIx64 " size=0x%016" PRIx64 " extent=%ux%u pitch=%u bpe=%u levels=%u tile=%u kind=%u\n", info.address, info.size, info.width, info.height, info.pitch, @@ -401,9 +448,10 @@ struct TextureCache::ReadbackWorker { EXIT("TextureCache: color-image readback size overflow\n"); } const auto linear_size = linear_elements * info.bytes_per_element; + const auto slice_size = info.size / layers; const bool meta_overlap = cache.HasMetaOverlapLocked(info.address, info.size); const bool buffer_overlap = cache.m_buffer_cache.HasPageOverlap(info.address, info.size); - if (linear_size > info.size || cached.image->format != info.format || + if (linear_size > slice_size || cached.image->format != info.format || cached.image->extent.width != info.width || cached.image->extent.height != info.height || (tiled && info.bytes_per_element != 1 && info.bytes_per_element != 2 && @@ -419,7 +467,18 @@ struct TextureCache::ReadbackWorker { } download.resize(info.size); std::fill(download.begin(), download.end(), 0); - UtilFillBuffer(cached.ctx, download.data(), linear_size, info.pitch, cached.image, + std::vector regions; + regions.reserve(layers); + for (uint32_t layer = 0; layer < layers; layer++) { + ImageBufferCopy region {}; + region.offset = static_cast(slice_size * layer); + region.pitch = info.pitch; + region.width = info.width; + region.height = info.height; + region.src_layer = layer; + regions.push_back(region); + } + UtilFillBuffer(cached.ctx, download.data(), info.size, regions, cached.image, cached.image->layout); if (tiled) { guest.resize(info.size); @@ -453,8 +512,8 @@ struct TextureCache::ReadbackWorker { submissions.emplace(); } FaultSafeTextureLock lock(&cache, cache.m_lock); - CachedImage* selected = cache.FindGpuReadbackPageCandidateLocked(vaddr, size); - const bool render_target = + CachedImage* selected = cache.FindGpuReadbackPageCandidateLocked(vaddr, size); + const bool render_target = selected != nullptr && selected->kind == CachedImage::Kind::RenderTarget; const bool storage_texture = selected != nullptr && selected->kind == CachedImage::Kind::StorageTexture; @@ -510,7 +569,7 @@ struct TextureCache::ReadbackWorker { transfer.address, transfer.size); } selected->gpu_modified = false; - submissions_prepaused = false; + submissions_prepaused = false; state.store(State::Idle, std::memory_order_release); state.notify_all(); } @@ -518,61 +577,101 @@ struct TextureCache::ReadbackWorker { TextureCache& cache; std::atomic state {State::Idle}; - PageFaultAccess access = PageFaultAccess::Unknown; - uint64_t vaddr = 0; - uint64_t size = 0; + PageFaultAccess access = PageFaultAccess::Unknown; + uint64_t vaddr = 0; + uint64_t size = 0; bool submissions_prepaused = false; std::vector download; std::vector guest; std::thread thread; }; +void TextureCache::RequireRetirementIsolation(const std::vector& retire, + const char* operation, uint64_t address, + uint64_t size) const { + if (retire.empty()) { + return; + } + std::vector ranges; + ranges.reserve(m_images.size() * 2); + for (const auto& cached: m_images) { + const bool retiring = std::find(retire.begin(), retire.end(), cached.get()) != retire.end(); + for (uint32_t range = 0; range < cached->RangeCount(); range++) { + ranges.push_back({cached->Address(range), cached->Size(range), retiring}); + } + } + const auto conflict = FindImageRetirementConflict(ranges); + if (conflict.Exists()) { + const auto& retired = ranges[conflict.retired]; + const auto& retained = ranges[conflict.retained]; + EXIT("TextureCache: %s retirement leaves a tracked page alias, request=0x%016" PRIx64 + "+0x%016" PRIx64 " retired=0x%016" PRIx64 "+0x%016" PRIx64 " retained=0x%016" PRIx64 + "+0x%016" PRIx64 "\n", + operation, address, size, retired.address, retired.size, retained.address, + retained.size); + } +} + void TextureCache::RetireImages(const std::vector& retire, - const CachedImage* native_storage_source) { + const CachedImage* native_image_source) { if (retire.empty()) { return; } // FindTexture and FindRenderTarget retain cache records on their recording commands. Erasing // the lookup owner therefore defers Vulkan destruction until the last referencing fence. - size_t removed = 0; - bool native_storage_retired = false; + size_t removed = 0; + bool native_image_retired = false; for (auto it = m_images.begin(); it != m_images.end();) { if (std::find(retire.begin(), retire.end(), it->get()) == retire.end()) { ++it; continue; } - const bool sampled = (*it)->kind == CachedImage::Kind::Texture; - const bool target = (*it)->kind == CachedImage::Kind::RenderTarget; - const bool native_storage = it->get() == native_storage_source; - if (native_storage) { - if ((*it)->kind != CachedImage::Kind::StorageTexture || !(*it)->gpu_modified || - (*it)->buffer_modified || (*it)->info.IsCpuDirty() || - !m_memory_tracker.IsRegionGpuModified((*it)->Address(), (*it)->Size())) { - EXIT("TextureCache: invalid native storage-image retirement, kind=%u " - "gpu_modified=%d buffer_modified=%d cpu_dirty=%d tracker_gpu_modified=%d\n", + const bool sampled = (*it)->kind == CachedImage::Kind::Texture; + const bool storage = (*it)->kind == CachedImage::Kind::StorageTexture; + const bool target = (*it)->kind == CachedImage::Kind::RenderTarget; + const bool native_image = it->get() == native_image_source; + if (native_image) { + bool source_valid = (*it)->gpu_modified && !(*it)->buffer_modified; + switch ((*it)->kind) { + case CachedImage::Kind::StorageTexture: + source_valid = source_valid && !(*it)->info.IsCpuDirty(); + break; + case CachedImage::Kind::RenderTarget: + case CachedImage::Kind::DepthTarget: break; + case CachedImage::Kind::Texture: + case CachedImage::Kind::VideoOut: source_valid = false; break; + } + for (uint32_t range = 0; source_valid && range < (*it)->RangeCount(); range++) { + source_valid = + m_memory_tracker.IsRegionGpuModified((*it)->Address(range), (*it)->Size(range)); + } + if (!source_valid) { + EXIT("TextureCache: invalid native image retirement, kind=%u gpu_modified=%d " + "buffer_modified=%d cpu_dirty=%d\n", static_cast((*it)->kind), (*it)->gpu_modified, - (*it)->buffer_modified, (*it)->info.IsCpuDirty(), - m_memory_tracker.IsRegionGpuModified((*it)->Address(), (*it)->Size())); + (*it)->buffer_modified, + (*it)->kind == CachedImage::Kind::StorageTexture && (*it)->info.IsCpuDirty()); } (*it)->gpu_modified = false; - native_storage_retired = true; + native_image_retired = true; } - if ((!sampled && !target && !native_storage) || (*it)->gpu_modified || + if ((!sampled && !storage && !target && !native_image) || (*it)->gpu_modified || + (storage && ((*it)->buffer_modified || (*it)->info.IsCpuDirty())) || (target && (*it)->buffer_modified)) { EXIT("TextureCache: invalid image retirement, kind=%u gpu_modified=%d " "buffer_modified=%d\n", static_cast((*it)->kind), (*it)->gpu_modified, (*it)->buffer_modified); } for (uint32_t range = 0; range < (*it)->RangeCount(); range++) { - if (target && + if (target && !native_image && m_memory_tracker.IsRegionGpuModified((*it)->Address(range), (*it)->Size(range))) { EXIT("TextureCache: clean target retirement retained tracker GPU ownership, " "addr=0x%016" PRIx64 " size=0x%016" PRIx64 "\n", (*it)->Address(range), (*it)->Size(range)); } - // An exact storage-to-target transition transfers this registration to the new cache + // A native image transition transfers this registration to the replacement cache // record. Keeping it GPU-owned also keeps guest page protection continuous. - if (!native_storage) { + if (!native_image) { m_memory_tracker.UntrackMemory((*it)->Address(range), (*it)->Size(range)); } } @@ -583,8 +682,8 @@ void TextureCache::RetireImages(const std::vector& retire, EXIT("TextureCache: image retirement set mismatch, requested=%zu removed=%zu\n", retire.size(), removed); } - if (native_storage_source != nullptr && !native_storage_retired) { - EXIT("TextureCache: native storage-image retirement source was not retired\n"); + if (native_image_source != nullptr && !native_image_retired) { + EXIT("TextureCache: native image retirement source was not retired\n"); } } @@ -605,7 +704,7 @@ bool Equal(const RenderTargetInfo& left, const RenderTargetInfo& right) { left.format == right.format && left.width == right.width && left.height == right.height && left.pitch == right.pitch && left.bytes_per_element == right.bytes_per_element && left.tile_mode == right.tile_mode && - left.levels == right.levels; + left.levels == right.levels && left.layers == right.layers; } bool Equal(const DepthTargetInfo& left, const DepthTargetInfo& right) { @@ -616,17 +715,67 @@ bool Equal(const DepthTargetInfo& left, const DepthTargetInfo& right) { left.guest_format == right.guest_format && left.width == right.width && left.height == right.height && left.pitch == right.pitch && left.bytes_per_element == right.bytes_per_element && left.tile_mode == right.tile_mode && + left.layers == right.layers && left.stencil_htile_compressed == right.stencil_htile_compressed; } -[[nodiscard]] bool IsCoherentGuestImageSource(const BufferImageCopySource& source, - uint64_t address, uint64_t size) { +[[nodiscard]] bool IsCoherentGuestImageSource(const BufferImageCopySource& source, uint64_t address, + uint64_t size) { // The current PS5 Tiler consumes coherent guest backing directly. A native buffer is an // optional future GPU-detiler source or staging fallback. return source.cpu_current && source.address == address && source.size == size && (source.buffer != nullptr || source.offset == 0); } +[[nodiscard]] BufferImageCopySource SelectSourceRange(const BufferImageCopySource& source, + uint64_t address, uint64_t size) { + if (address < source.address || size > source.size || + address - source.address > source.size - size || + (source.buffer != nullptr && source.offset > UINT64_MAX - (address - source.address))) { + EXIT("TextureCache: image source subrange is invalid\n"); + } + const auto offset = source.buffer != nullptr ? source.offset + address - source.address : 0; + return {source.buffer, offset, address, size, source.cpu_current}; +} + +[[nodiscard]] DepthTargetInfo SelectDepthLayers(const DepthTargetInfo& info, uint32_t base_layer, + uint32_t layer_count) { + if (info.layers == 0 || base_layer >= info.layers || layer_count == 0 || + layer_count > info.layers - base_layer || info.size % info.layers != 0 || + info.stencil_size % info.layers != 0 || info.htile_size % info.layers != 0) { + EXIT("TextureCache: depth layer range is invalid\n"); + } + auto result = info; + result.layers = layer_count; + for (auto [address, size]: {std::pair {&result.address, &result.size}, + std::pair {&result.stencil_address, &result.stencil_size}, + std::pair {&result.htile_address, &result.htile_size}}) { + const auto slice_size = *size / info.layers; + *address += slice_size * base_layer; + *size = slice_size * layer_count; + } + return result; +} + +void AppendLayerCopies(std::vector& regions, VulkanImage* source, + VkImageAspectFlags aspect) { + for (uint32_t layer = 0; layer < source->layers; layer++) { + for (uint32_t level = 0; level < source->mip_levels; level++) { + ImageImageCopy region {}; + region.src_image = source; + region.src_aspect = aspect; + region.dst_aspect = aspect; + region.src_level = level; + region.dst_level = level; + region.src_layer = layer; + region.dst_layer = layer; + region.width = std::max(source->extent.width >> level, 1u); + region.height = std::max(source->extent.height >> level, 1u); + regions.push_back(region); + } + } +} + VkImageAspectFlags DepthAspectMask(VkFormat format) { return VK_IMAGE_ASPECT_DEPTH_BIT | (format == VK_FORMAT_D16_UNORM_S8_UINT || format == VK_FORMAT_D24_UNORM_S8_UINT || @@ -679,8 +828,7 @@ bool RenderTargetSupportsStorage(GraphicContext* ctx, VkFormat format, VkImageCr VkImageCreateFlags RenderTargetCreateFlags(VkFormat format) { const bool compatible_format_view = - IsRgba8SrgbViewFormat(format) || - BgraToRgbaSampledViewFormat(format) != VK_FORMAT_UNDEFINED; + IsRgba8SrgbViewFormat(format) || BgraToRgbaSampledViewFormat(format) != VK_FORMAT_UNDEFINED; return compatible_format_view ? VK_IMAGE_CREATE_MUTABLE_FORMAT_BIT | VK_IMAGE_CREATE_EXTENDED_USAGE_BIT : VkImageCreateFlags {0}; @@ -711,9 +859,10 @@ void CreateRenderTargetView(GraphicContext* ctx, VulkanImage* image, int index, VkComponentSwizzle a, VkImageViewType type = VK_IMAGE_VIEW_TYPE_2D, VkFormat view_format = VK_FORMAT_UNDEFINED, VkImageUsageFlags view_usage = 0, uint32_t level_count = 0) { - UtilCreateImageView(ctx, image, index, type, VK_IMAGE_ASPECT_COLOR_BIT, {r, g, b, a}, 0, 0, 1, - level_count == 0 ? image->mip_levels : level_count, view_format, - view_usage); + const auto layer_count = type == VK_IMAGE_VIEW_TYPE_2D_ARRAY ? image->layers : 1u; + UtilCreateImageView(ctx, image, index, type, VK_IMAGE_ASPECT_COLOR_BIT, {r, g, b, a}, 0, 0, + layer_count, level_count == 0 ? image->mip_levels : level_count, + view_format, view_usage); } void CreateRenderAttachmentViews(GraphicContext* ctx, RenderTextureVulkanImage* image) { @@ -761,11 +910,26 @@ void CreateRenderTargetViews(GraphicContext* ctx, RenderTextureVulkanImage* imag CreateRenderTargetView(ctx, image, VulkanImage::VIEW_000R, VK_COMPONENT_SWIZZLE_ZERO, VK_COMPONENT_SWIZZLE_ZERO, VK_COMPONENT_SWIZZLE_ZERO, VK_COMPONENT_SWIZZLE_R); + if (image->layers > 1) { + CreateRenderTargetView(ctx, image, VulkanImage::VIEW_DEFAULT_ARRAY, + VK_COMPONENT_SWIZZLE_IDENTITY, VK_COMPONENT_SWIZZLE_IDENTITY, + VK_COMPONENT_SWIZZLE_IDENTITY, VK_COMPONENT_SWIZZLE_IDENTITY, + VK_IMAGE_VIEW_TYPE_2D_ARRAY); + CreateRenderTargetView(ctx, image, VulkanImage::VIEW_BGRA_ARRAY, VK_COMPONENT_SWIZZLE_B, + VK_COMPONENT_SWIZZLE_G, VK_COMPONENT_SWIZZLE_R, + VK_COMPONENT_SWIZZLE_IDENTITY, VK_IMAGE_VIEW_TYPE_2D_ARRAY); + } if (FormatSupportsStorage(ctx, image->format)) { CreateRenderTargetView(ctx, image, VulkanImage::VIEW_STORAGE, VK_COMPONENT_SWIZZLE_IDENTITY, VK_COMPONENT_SWIZZLE_IDENTITY, VK_COMPONENT_SWIZZLE_IDENTITY, VK_COMPONENT_SWIZZLE_IDENTITY, VK_IMAGE_VIEW_TYPE_2D, VK_FORMAT_UNDEFINED, 0, 1); + if (image->layers > 1) { + CreateRenderTargetView(ctx, image, VulkanImage::VIEW_STORAGE_ARRAY, + VK_COMPONENT_SWIZZLE_IDENTITY, VK_COMPONENT_SWIZZLE_IDENTITY, + VK_COMPONENT_SWIZZLE_IDENTITY, VK_COMPONENT_SWIZZLE_IDENTITY, + VK_IMAGE_VIEW_TYPE_2D_ARRAY, VK_FORMAT_UNDEFINED, 0, 1); + } } const auto rgba_view_format = BgraToRgbaSampledViewFormat(image->format); if (rgba_view_format != VK_FORMAT_UNDEFINED) { @@ -793,55 +957,77 @@ void CreateVideoOutViews(GraphicContext* ctx, VideoOutVulkanImage* image) { VK_COMPONENT_SWIZZLE_IDENTITY, VK_IMAGE_VIEW_TYPE_2D_ARRAY); } -void UploadRenderTarget(GraphicContext* ctx, RenderTextureVulkanImage* image, - const RenderTargetInfo& info, bool refresh) { +[[nodiscard]] uint32_t RenderTargetTransferFormat(uint32_t bytes_per_element) { + switch (bytes_per_element) { + case 1: return Prospero::GpuEnumValue(Prospero::BufferFormat::k8UNorm); + case 2: return Prospero::GpuEnumValue(Prospero::BufferFormat::k16UNorm); + case 4: return Prospero::GpuEnumValue(Prospero::BufferFormat::k32Float); + case 8: return Prospero::GpuEnumValue(Prospero::BufferFormat::k16_16_16_16Float); + case 16: return Prospero::GpuEnumValue(Prospero::BufferFormat::k32_32_32_32Float); + default: + EXIT("TextureCache: unsupported render-target element size: %u\n", bytes_per_element); + } +} + +void UploadRenderTargetLayers(GraphicContext* ctx, RenderTextureVulkanImage* image, + const RenderTargetInfo& info, uint32_t base_layer, + uint32_t layer_count, bool refresh) { + if (info.layers == 0 || info.size % info.layers != 0 || layer_count == 0 || + base_layer >= info.layers || layer_count > info.layers - base_layer || image == nullptr || + base_layer >= image->layers || layer_count > image->layers - base_layer) { + EXIT("TextureCache: invalid render-target layer upload, base=%u count=%u " + "info_layers=%u image_layers=%u size=0x%016" PRIx64 "\n", + base_layer, layer_count, info.layers, image != nullptr ? image->layers : 0, info.size); + } if (refresh) { VulkanDeviceWaitIdle(ctx); } - image->layout = VK_IMAGE_LAYOUT_UNDEFINED; - if (info.levels > 1) { - uint32_t format = 0; - switch (info.bytes_per_element) { - case 1: format = Prospero::GpuEnumValue(Prospero::BufferFormat::k8UNorm); break; - case 2: format = Prospero::GpuEnumValue(Prospero::BufferFormat::k16UNorm); break; - case 4: format = Prospero::GpuEnumValue(Prospero::BufferFormat::k32Float); break; - case 8: format = Prospero::GpuEnumValue(Prospero::BufferFormat::k16_16_16_16Float); break; - case 16: format = Prospero::GpuEnumValue(Prospero::BufferFormat::k32_32_32_32Float); break; - default: - EXIT("TextureCache: unsupported render-target mip element size: %u\n", - info.bytes_per_element); - } - auto layout = TextureCalcUploadLayout(format, info.width, info.height, info.levels, 1, - info.pitch, info.tile_mode, info.size, false, false, - false, "TextureCache render target"); - if (!layout.fmt_tiled_render_target || layout.pitch != info.pitch) { + const auto slice_size = info.size / info.layers; + const auto upload_size = slice_size * layer_count; + if (info.levels > 1 || info.layers > 1) { + const auto format = RenderTargetTransferFormat(info.bytes_per_element); + auto layout = TextureCalcUploadLayout(format, info.width, info.height, info.levels, + layer_count, info.pitch, info.tile_mode, upload_size, + false, false, false, "TextureCache render target"); + const bool tiled = + info.tile_mode == Prospero::GpuEnumValue(Prospero::TileMode::kRenderTarget); + if ((tiled && !layout.fmt_tiled_render_target) || layout.pitch != info.pitch) { EXIT("TextureCache: unsupported render-target mip upload layout, pitch=%u/%u tile=%u\n", info.pitch, layout.pitch, info.tile_mode); } auto regions = TextureBuildUploadRegions( - layout, info.format, info.width, info.height, 1, info.levels, false, false, + layout, info.format, info.width, info.height, layer_count, info.levels, true, false, TextureUploadDestination::MipLevels, TextureUploadSliceLayout::MipChainPerSlice); - TextureUploadGuestImage(ctx, image, reinterpret_cast(info.address), info.size, - regions, layout, format, info.width, info.height, 1, info.levels, - TextureUploadSliceLayout::MipChainPerSlice, - "TextureCache render target", - static_cast(VK_IMAGE_LAYOUT_GENERAL)); + for (auto& region: regions) { + region.dst_layer += base_layer; + } + const auto source_address = info.address + slice_size * base_layer; + TextureUploadGuestImage( + ctx, image, reinterpret_cast(source_address), upload_size, regions, layout, + format, info.width, info.height, layer_count, info.levels, + TextureUploadSliceLayout::MipChainPerSlice, "TextureCache render target", + static_cast(VK_IMAGE_LAYOUT_GENERAL)); return; } if (info.tile_mode == Prospero::GpuEnumValue(Prospero::TileMode::kRenderTarget) && - UtilBufferIsTiled(info.address, info.size)) { - UtilScratchBuffer scratch(info.size); + UtilBufferIsTiled(info.address, slice_size)) { + UtilScratchBuffer scratch(slice_size); TileConvertTiledToLinearRenderTarget( scratch.Data(), reinterpret_cast(info.address), info.width, info.height, - info.pitch, info.bytes_per_element, info.size); - UtilFillImage(ctx, image, scratch.Data(), info.size, info.pitch, + info.pitch, info.bytes_per_element, slice_size); + UtilFillImage(ctx, image, scratch.Data(), slice_size, info.pitch, static_cast(VK_IMAGE_LAYOUT_GENERAL)); } else { - UtilFillImage(ctx, image, reinterpret_cast(info.address), info.size, + UtilFillImage(ctx, image, reinterpret_cast(info.address), slice_size, info.pitch, static_cast(VK_IMAGE_LAYOUT_GENERAL)); } } +void UploadRenderTarget(GraphicContext* ctx, RenderTextureVulkanImage* image, + const RenderTargetInfo& info, bool refresh) { + UploadRenderTargetLayers(ctx, image, info, 0, info.layers, refresh); +} + RenderTextureVulkanImage* CreateRenderTarget(GraphicContext* ctx, const RenderTargetInfo& info, VulkanMemory* memory) { auto* image = new RenderTextureVulkanImage; @@ -849,6 +1035,7 @@ RenderTextureVulkanImage* CreateRenderTarget(GraphicContext* ctx, const RenderTa image->extent.height = info.height; image->format = info.format; image->mip_levels = info.levels; + image->layers = info.layers; image->layout = VK_IMAGE_LAYOUT_UNDEFINED; UtilResetImageViews(image); VkImageCreateInfo create {}; @@ -857,7 +1044,7 @@ RenderTextureVulkanImage* CreateRenderTarget(GraphicContext* ctx, const RenderTa create.imageType = VK_IMAGE_TYPE_2D; create.extent = {info.width, info.height, 1}; create.mipLevels = info.levels; - create.arrayLayers = 1; + create.arrayLayers = info.layers; create.format = info.format; create.tiling = VK_IMAGE_TILING_OPTIMAL; create.initialLayout = VK_IMAGE_LAYOUT_UNDEFINED; @@ -896,6 +1083,28 @@ void CreateDepthViews(GraphicContext* ctx, DepthStencilVulkanImage* image) { {VK_COMPONENT_SWIZZLE_R, VK_COMPONENT_SWIZZLE_ZERO, VK_COMPONENT_SWIZZLE_ZERO, VK_COMPONENT_SWIZZLE_ONE}, 0, 0, 1, 1); + if (image->layers > 1) { + UtilCreateImageView(ctx, image, VulkanImage::VIEW_DEFAULT_ARRAY, + VK_IMAGE_VIEW_TYPE_2D_ARRAY, DepthAspectMask(image->format), + {VK_COMPONENT_SWIZZLE_IDENTITY, VK_COMPONENT_SWIZZLE_IDENTITY, + VK_COMPONENT_SWIZZLE_IDENTITY, VK_COMPONENT_SWIZZLE_IDENTITY}, + 0, 0, image->layers, 1); + UtilCreateImageView(ctx, image, VulkanImage::VIEW_DEPTH_TEXTURE_ARRAY, + VK_IMAGE_VIEW_TYPE_2D_ARRAY, VK_IMAGE_ASPECT_DEPTH_BIT, + {VK_COMPONENT_SWIZZLE_R, VK_COMPONENT_SWIZZLE_R, VK_COMPONENT_SWIZZLE_R, + VK_COMPONENT_SWIZZLE_R}, + 0, 0, image->layers, 1); + UtilCreateImageView(ctx, image, VulkanImage::VIEW_R000_ARRAY, VK_IMAGE_VIEW_TYPE_2D_ARRAY, + VK_IMAGE_ASPECT_DEPTH_BIT, + {VK_COMPONENT_SWIZZLE_R, VK_COMPONENT_SWIZZLE_ZERO, + VK_COMPONENT_SWIZZLE_ZERO, VK_COMPONENT_SWIZZLE_ZERO}, + 0, 0, image->layers, 1); + UtilCreateImageView(ctx, image, VulkanImage::VIEW_R001_ARRAY, VK_IMAGE_VIEW_TYPE_2D_ARRAY, + VK_IMAGE_ASPECT_DEPTH_BIT, + {VK_COMPONENT_SWIZZLE_R, VK_COMPONENT_SWIZZLE_ZERO, + VK_COMPONENT_SWIZZLE_ZERO, VK_COMPONENT_SWIZZLE_ONE}, + 0, 0, image->layers, 1); + } } DepthStencilVulkanImage* CreateDepthTarget(GraphicContext* ctx, const DepthTargetInfo& info, @@ -905,7 +1114,7 @@ DepthStencilVulkanImage* CreateDepthTarget(GraphicContext* ctx, const DepthTarge create.imageType = VK_IMAGE_TYPE_2D; create.extent = {info.width, info.height, 1}; create.mipLevels = 1; - create.arrayLayers = 1; + create.arrayLayers = info.layers; create.format = info.format; create.tiling = VK_IMAGE_TILING_OPTIMAL; create.initialLayout = VK_IMAGE_LAYOUT_UNDEFINED; @@ -924,6 +1133,7 @@ DepthStencilVulkanImage* CreateDepthTarget(GraphicContext* ctx, const DepthTarge image->extent.width = info.width; image->extent.height = info.height; image->guest_pitch = info.pitch; + image->layers = info.layers; image->format = info.format; image->layout = VK_IMAGE_LAYOUT_UNDEFINED; image->compressed = false; @@ -951,9 +1161,10 @@ void ValidateVideoOutInfo(GraphicContext* ctx, const VideoOutInfo& info) { info.address >= TRACKER_ADDRESS_SIZE || info.size > TRACKER_ADDRESS_SIZE - info.address || (info.address & 0xffffu) != 0 || info.width == 0 || info.height == 0 || info.width > 16384 || info.height > 16384 || info.pitch < info.width || - info.tile_mode != Prospero::GpuEnumValue(Prospero::TileMode::kRenderTarget) || info.bytes_per_element != 4 || - compression == VideoOutCompression::Unsupported || compression != info.compression || - metadata_invalid || !IsSupportedVideoOutFormat(info.guest_format, info.format)) { + info.tile_mode != Prospero::GpuEnumValue(Prospero::TileMode::kRenderTarget) || + info.bytes_per_element != 4 || compression == VideoOutCompression::Unsupported || + compression != info.compression || metadata_invalid || + !IsSupportedVideoOutFormat(info.guest_format, info.format)) { EXIT("TextureCache: unsupported video-out surface, ctx=%p addr=0x%016" PRIx64 " size=0x%016" PRIx64 " metadata=0x%016" PRIx64 " dcc=0x%08" PRIx32 " extent=%ux%u pitch=%u tile=%u guest_format=%u bpe=%u vk_format=%d\n", @@ -1027,7 +1238,7 @@ void UploadVideoOut(GraphicContext* ctx, VideoOutVulkanImage* image, const Video } // namespace TextureCache::TextureCache(PageManager& page_manager, BufferCache& buffer_cache, - ResourceMutex& resource_mutex) + ResourceMutex& resource_mutex) : m_memory_tracker(page_manager), m_metadata_tracker(page_manager, PageWatchMode::Write), m_buffer_cache(buffer_cache), m_resource_mutex(resource_mutex) { if (!Common::Thread::IsMainThread()) { @@ -1038,21 +1249,25 @@ TextureCache::TextureCache(PageManager& page_manager, BufferCache& buffer_cache, VkImageView TextureCache::GetRenderTargetAttachmentView(GraphicContext* ctx, RenderTextureVulkanImage* image, - VkFormat format, uint32_t level) { + VkFormat format, uint32_t level, + uint32_t base_layer, uint32_t layer_count) { if (ctx == nullptr || image == nullptr || image->image == nullptr || - format == VK_FORMAT_UNDEFINED || level >= image->mip_levels || level >= 16) { + format == VK_FORMAT_UNDEFINED || level >= image->mip_levels || level >= 16 || + layer_count == 0 || base_layer >= image->layers || + layer_count > image->layers - base_layer) { EXIT("TextureCache: invalid render-target attachment view, image=%p format=%d" - " level=%u image_levels=%u\n", + " level=%u image_levels=%u base_layer=%u layer_count=%u image_layers=%u\n", static_cast(image), static_cast(format), level, - image != nullptr ? image->mip_levels : 0); + image != nullptr ? image->mip_levels : 0, base_layer, layer_count, + image != nullptr ? image->layers : 0); } - if (format == image->format) { + if (format == image->format && base_layer == 0 && layer_count == 1) { if (image->render_view[level] == nullptr) { EXIT("TextureCache: base render-target attachment view is missing, level=%u\n", level); } return image->render_view[level]; } - if (!IsRgba8SrgbReinterpretation(image->format, format)) { + if (format != image->format && !IsRgba8SrgbReinterpretation(image->format, format)) { EXIT("TextureCache: incompatible render-target attachment view, image_format=%d" " view_format=%d level=%u\n", static_cast(image->format), static_cast(format), level); @@ -1060,7 +1275,8 @@ VkImageView TextureCache::GetRenderTargetAttachmentView(GraphicContext* std::lock_guard lock(image->attachment_view_mutex); for (const auto& cached: image->attachment_views) { - if (cached.format == format && cached.level == level) { + if (cached.format == format && cached.level == level && cached.base_layer == base_layer && + cached.layer_count == layer_count) { return cached.view; } } @@ -1072,42 +1288,91 @@ VkImageView TextureCache::GetRenderTargetAttachmentView(GraphicContext* create.sType = VK_STRUCTURE_TYPE_IMAGE_VIEW_CREATE_INFO; create.pNext = &usage; create.image = image->image; - create.viewType = VK_IMAGE_VIEW_TYPE_2D; + create.viewType = layer_count == 1 ? VK_IMAGE_VIEW_TYPE_2D : VK_IMAGE_VIEW_TYPE_2D_ARRAY; create.format = format; create.components = {VK_COMPONENT_SWIZZLE_IDENTITY, VK_COMPONENT_SWIZZLE_IDENTITY, VK_COMPONENT_SWIZZLE_IDENTITY, VK_COMPONENT_SWIZZLE_IDENTITY}; create.subresourceRange.aspectMask = VK_IMAGE_ASPECT_COLOR_BIT; create.subresourceRange.baseMipLevel = level; create.subresourceRange.levelCount = 1; - create.subresourceRange.baseArrayLayer = 0; - create.subresourceRange.layerCount = 1; + create.subresourceRange.baseArrayLayer = base_layer; + create.subresourceRange.layerCount = layer_count; VkImageView view = nullptr; const auto result = vkCreateImageView(ctx->device, &create, nullptr, &view); if (result != VK_SUCCESS || view == nullptr) { - EXIT("TextureCache: failed to create compatible render-target attachment view," - " result=%d image_format=%d view_format=%d level=%u\n", + EXIT("TextureCache: failed to create render-target attachment view," + " result=%d image_format=%d view_format=%d level=%u base_layer=%u" + " layer_count=%u\n", static_cast(result), static_cast(image->format), static_cast(format), - level); + level, base_layer, layer_count); } - image->attachment_views.push_back({format, level, view}); - LOGF("TextureCache: created compatible render-target attachment view:" - " image_format=%d view_format=%d level=%u extent=%ux%u\n", - static_cast(image->format), static_cast(format), level, image->extent.width, - image->extent.height); + image->attachment_views.push_back({format, level, base_layer, layer_count, view}); + LOGF("TextureCache: created render-target attachment view:" + " image_format=%d view_format=%d level=%u layer=%u+%u extent=%ux%u\n", + static_cast(image->format), static_cast(format), level, base_layer, layer_count, + image->extent.width, image->extent.height); + return view; +} + +VkImageView TextureCache::GetDepthTargetAttachmentView(GraphicContext* ctx, + DepthStencilVulkanImage* image, + uint32_t base_layer, uint32_t layer_count) { + if (ctx == nullptr || image == nullptr || image->image == nullptr || layer_count == 0 || + base_layer >= image->layers || layer_count > image->layers - base_layer) { + EXIT("TextureCache: invalid depth-target attachment view, image=%p base_layer=%u " + "layer_count=%u image_layers=%u\n", + static_cast(image), base_layer, layer_count, + image != nullptr ? image->layers : 0); + } + if (base_layer == 0 && layer_count == 1) { + return image->image_view[VulkanImage::VIEW_DEFAULT]; + } + std::lock_guard lock(image->attachment_view_mutex); + for (const auto& cached: image->attachment_views) { + if (cached.base_layer == base_layer && cached.layer_count == layer_count) { + return cached.view; + } + } + VkImageViewCreateInfo create {}; + create.sType = VK_STRUCTURE_TYPE_IMAGE_VIEW_CREATE_INFO; + create.image = image->image; + create.viewType = layer_count == 1 ? VK_IMAGE_VIEW_TYPE_2D : VK_IMAGE_VIEW_TYPE_2D_ARRAY; + create.format = image->format; + create.components = {VK_COMPONENT_SWIZZLE_IDENTITY, VK_COMPONENT_SWIZZLE_IDENTITY, + VK_COMPONENT_SWIZZLE_IDENTITY, VK_COMPONENT_SWIZZLE_IDENTITY}; + create.subresourceRange.aspectMask = DepthAspectMask(image->format); + create.subresourceRange.baseMipLevel = 0; + create.subresourceRange.levelCount = 1; + create.subresourceRange.baseArrayLayer = base_layer; + create.subresourceRange.layerCount = layer_count; + VkImageView view = nullptr; + const auto result = vkCreateImageView(ctx->device, &create, nullptr, &view); + if (result != VK_SUCCESS || view == nullptr) { + EXIT("TextureCache: failed to create depth-target attachment view, result=%d " + "format=%d base_layer=%u layer_count=%u\n", + static_cast(result), static_cast(image->format), base_layer, layer_count); + } + image->attachment_views.push_back({base_layer, layer_count, view}); return view; } VkImageView TextureCache::GetRenderTargetSampledView(GraphicContext* ctx, RenderTextureVulkanImage* image, VkFormat view_format, int variant, - uint32_t base_level, uint32_t level_count) { + uint32_t base_level, uint32_t level_count, + VkImageViewType type, uint32_t base_layer, + uint32_t layer_count) { if (ctx == nullptr || image == nullptr || image->image == nullptr || base_level >= 16 || view_format == VK_FORMAT_UNDEFINED || level_count == 0 || - base_level + level_count > image->mip_levels) { + base_level + level_count > image->mip_levels || layer_count == 0 || + base_layer >= image->layers || layer_count > image->layers - base_layer || + (type != VK_IMAGE_VIEW_TYPE_2D && type != VK_IMAGE_VIEW_TYPE_2D_ARRAY) || + (type == VK_IMAGE_VIEW_TYPE_2D && layer_count != 1)) { EXIT("TextureCache: invalid render-target sampled view, image=%p variant=%d" - " view_format=%d base=%u count=%u image_levels=%u\n", + " view_format=%d mip=%u+%u layer=%u+%u type=%d image_levels=%u image_layers=%u\n", static_cast(image), variant, static_cast(view_format), base_level, - level_count, image != nullptr ? image->mip_levels : 0); + level_count, base_layer, layer_count, static_cast(type), + image != nullptr ? image->mip_levels : 0, image != nullptr ? image->layers : 0); } const bool compatible_variant_format = view_format == image->format || (variant == VulkanImage::VIEW_BGRA_TO_RGBA && @@ -1120,18 +1385,29 @@ VkImageView TextureCache::GetRenderTargetSampledView(GraphicContext* c const auto default_view_format = variant == VulkanImage::VIEW_BGRA_TO_RGBA ? BgraToRgbaSampledViewFormat(image->format) : image->format; - if (view_format == default_view_format && base_level == 0 && level_count == image->mip_levels) { - if (variant < 0 || variant >= VulkanImage::VIEW_MAX || - image->image_view[variant] == nullptr) { - EXIT("TextureCache: full render-target sampled view is missing, variant=%d\n", variant); + int default_variant = variant; + if (type == VK_IMAGE_VIEW_TYPE_2D_ARRAY) { + switch (variant) { + case VulkanImage::VIEW_DEFAULT: + default_variant = VulkanImage::VIEW_DEFAULT_ARRAY; + break; + case VulkanImage::VIEW_BGRA: default_variant = VulkanImage::VIEW_BGRA_ARRAY; break; + default: default_variant = VulkanImage::VIEW_MAX; break; } - return image->image_view[variant]; + } + const bool full_view = + base_level == 0 && level_count == image->mip_levels && base_layer == 0 && + layer_count == (type == VK_IMAGE_VIEW_TYPE_2D_ARRAY ? image->layers : 1u); + if (view_format == default_view_format && full_view && default_variant >= 0 && + default_variant < VulkanImage::VIEW_MAX && image->image_view[default_variant] != nullptr) { + return image->image_view[default_variant]; } std::lock_guard lock(image->sampled_view_mutex); for (const auto& cached: image->sampled_views) { - if (cached.format == view_format && cached.variant == variant && - cached.base_level == base_level && cached.level_count == level_count) { + if (cached.format == view_format && cached.type == type && cached.variant == variant && + cached.base_level == base_level && cached.level_count == level_count && + cached.base_layer == base_layer && cached.layer_count == layer_count) { return cached.view; } } @@ -1182,37 +1458,43 @@ VkImageView TextureCache::GetRenderTargetSampledView(GraphicContext* c create.sType = VK_STRUCTURE_TYPE_IMAGE_VIEW_CREATE_INFO; create.pNext = &usage; create.image = image->image; - create.viewType = VK_IMAGE_VIEW_TYPE_2D; + create.viewType = type; create.format = view_format; create.components = components; create.subresourceRange.aspectMask = VK_IMAGE_ASPECT_COLOR_BIT; create.subresourceRange.baseMipLevel = base_level; create.subresourceRange.levelCount = level_count; - create.subresourceRange.baseArrayLayer = 0; - create.subresourceRange.layerCount = 1; + create.subresourceRange.baseArrayLayer = base_layer; + create.subresourceRange.layerCount = layer_count; VkImageView view = nullptr; const auto result = vkCreateImageView(ctx->device, &create, nullptr, &view); if (result != VK_SUCCESS || view == nullptr) { EXIT("TextureCache: failed to create render-target sampled subresource view," - " result=%d view_format=%d variant=%d base=%u count=%u\n", + " result=%d view_format=%d variant=%d mip=%u+%u layer=%u+%u type=%d\n", static_cast(result), static_cast(view_format), variant, base_level, - level_count); + level_count, base_layer, layer_count, static_cast(type)); } - image->sampled_views.push_back({view_format, base_level, level_count, variant, view}); + image->sampled_views.push_back( + {view_format, type, base_level, level_count, base_layer, layer_count, variant, view}); return view; } VkImageView TextureCache::GetRenderTargetStorageView(GraphicContext* ctx, RenderTextureVulkanImage* image, VkFormat view_format, uint32_t base_level, - uint32_t level_count) { + uint32_t level_count, VkImageViewType type, + uint32_t base_layer, uint32_t layer_count) { if (ctx == nullptr || image == nullptr || image->image == nullptr || view_format == VK_FORMAT_UNDEFINED || level_count == 0 || - base_level + level_count > image->mip_levels) { + base_level + level_count > image->mip_levels || layer_count == 0 || + base_layer >= image->layers || layer_count > image->layers - base_layer || + (type != VK_IMAGE_VIEW_TYPE_2D && type != VK_IMAGE_VIEW_TYPE_2D_ARRAY) || + (type == VK_IMAGE_VIEW_TYPE_2D && layer_count != 1)) { EXIT("TextureCache: invalid render-target storage view, image=%p view_format=%d" - " base=%u count=%u image_levels=%u\n", + " mip=%u+%u layer=%u+%u type=%d image_levels=%u image_layers=%u\n", static_cast(image), static_cast(view_format), base_level, - level_count, image != nullptr ? image->mip_levels : 0); + level_count, base_layer, layer_count, static_cast(type), + image != nullptr ? image->mip_levels : 0, image != nullptr ? image->layers : 0); } const bool exact = view_format == image->format; const bool compatible = view_format == BgraSrgbStorageViewFormat(image->format); @@ -1223,18 +1505,21 @@ VkImageView TextureCache::GetRenderTargetStorageView(GraphicContext* c level_count); } if (exact) { - if (image->image_view[VulkanImage::VIEW_STORAGE] == nullptr) { - EXIT("TextureCache: base render-target storage view is missing\n"); - } - if (base_level == 0 && level_count == 1) { - return image->image_view[VulkanImage::VIEW_STORAGE]; + const auto index = type == VK_IMAGE_VIEW_TYPE_2D_ARRAY ? VulkanImage::VIEW_STORAGE_ARRAY + : VulkanImage::VIEW_STORAGE; + const bool full_view = + base_level == 0 && level_count == 1 && base_layer == 0 && + layer_count == (type == VK_IMAGE_VIEW_TYPE_2D_ARRAY ? image->layers : 1u); + if (full_view && image->image_view[index] != nullptr) { + return image->image_view[index]; } } std::lock_guard lock(image->storage_view_mutex); for (const auto& cached: image->storage_views) { - if (cached.format == view_format && cached.base_level == base_level && - cached.level_count == level_count) { + if (cached.format == view_format && cached.type == type && + cached.base_level == base_level && cached.level_count == level_count && + cached.base_layer == base_layer && cached.layer_count == layer_count) { return cached.view; } } @@ -1252,15 +1537,15 @@ VkImageView TextureCache::GetRenderTargetStorageView(GraphicContext* c create.sType = VK_STRUCTURE_TYPE_IMAGE_VIEW_CREATE_INFO; create.pNext = &usage; create.image = image->image; - create.viewType = VK_IMAGE_VIEW_TYPE_2D; + create.viewType = type; create.format = view_format; create.components = {VK_COMPONENT_SWIZZLE_IDENTITY, VK_COMPONENT_SWIZZLE_IDENTITY, VK_COMPONENT_SWIZZLE_IDENTITY, VK_COMPONENT_SWIZZLE_IDENTITY}; create.subresourceRange.aspectMask = VK_IMAGE_ASPECT_COLOR_BIT; create.subresourceRange.baseMipLevel = base_level; create.subresourceRange.levelCount = level_count; - create.subresourceRange.baseArrayLayer = 0; - create.subresourceRange.layerCount = 1; + create.subresourceRange.baseArrayLayer = base_layer; + create.subresourceRange.layerCount = layer_count; VkImageView view = nullptr; const auto result = vkCreateImageView(ctx->device, &create, nullptr, &view); if (result != VK_SUCCESS || view == nullptr) { @@ -1269,7 +1554,8 @@ VkImageView TextureCache::GetRenderTargetStorageView(GraphicContext* c static_cast(result), static_cast(image->format), static_cast(view_format), base_level, level_count); } - image->storage_views.push_back({view_format, base_level, level_count, view}); + image->storage_views.push_back( + {view_format, type, base_level, level_count, base_layer, layer_count, view}); LOGF("TextureCache: created compatible render-target storage view: image_format=%d" " view_format=%d base=%u count=%u extent=%ux%u\n", static_cast(image->format), static_cast(view_format), base_level, level_count, @@ -1280,8 +1566,10 @@ VkImageView TextureCache::GetRenderTargetStorageView(GraphicContext* c VkImageView TextureCache::GetStorageTextureSampledView(GraphicContext* ctx, StorageTextureVulkanImage* image, const ImageInfo& info) { - const bool image_2d = info.type == Prospero::GpuEnumValue(Prospero::ImageType::kColor2D) && info.depth == 1; - const bool image_3d = info.type == Prospero::GpuEnumValue(Prospero::ImageType::kColor3D) && info.depth != 0; + const bool image_2d = + info.type == Prospero::GpuEnumValue(Prospero::ImageType::kColor2D) && info.depth == 1; + const bool image_3d = + info.type == Prospero::GpuEnumValue(Prospero::ImageType::kColor3D) && info.depth != 0; if (ctx == nullptr || image == nullptr || image->image == nullptr || (!image_2d && !image_3d) || info.base_array != 0 || info.levels != image->mip_levels || info.base_level >= info.levels || info.view_levels == 0 || @@ -1374,6 +1662,102 @@ void TextureCache::MarkSampledAliasesCpuDirtyLocked(uint64_t vaddr, uint64_t siz } } +void TextureCache::RetireSampledTargetAliases(GraphicContext* ctx, const ImageInfo& requested) { + std::vector retire; + bool wait_idle = false; + for (const auto& cached: m_images) { + if (cached->kind != CachedImage::Kind::RenderTarget) { + continue; + } + switch (ClassifySampledRenderTargetOverlap(requested, cached->target, + cached->buffer_modified, cached->ctx == ctx)) { + case RenderTargetOverlap::None: continue; + case RenderTargetOverlap::RetireTarget: + wait_idle |= cached->gpu_modified; + retire.push_back(cached.get()); + break; + case RenderTargetOverlap::RetireSampled: + case RenderTargetOverlap::PreserveStorage: + case RenderTargetOverlap::ExpandTarget: + case RenderTargetOverlap::Unsupported: + EXIT("TextureCache: unsupported sampled/render-target alias, sampled=0x%016" PRIx64 + "+0x%016" PRIx64 " target=0x%016" PRIx64 "+0x%016" PRIx64 + " same_context=%d gpu=%d buffer=%d\n", + requested.address, requested.size, cached->target.address, cached->target.size, + cached->ctx == ctx, cached->gpu_modified, cached->buffer_modified); + } + } + if (retire.empty()) { + return; + } + RequireRetirementIsolation(retire, "sampled target", requested.address, requested.size); + if (wait_idle) { + VulkanDeviceWaitIdle(ctx); + } + for (auto* cached: retire) { + if (!cached->gpu_modified) { + continue; + } + const auto transfer = m_readback->DownloadColorImage(*cached); + m_memory_tracker.ForEachDownloadRange(transfer.address, transfer.size, + [](uint64_t, uint64_t) noexcept {}); + cached->gpu_modified = false; + } + RetireImages(retire); +} + +void TextureCache::RetireStoragePageNeighbors(GraphicContext* ctx, const ImageInfo& requested) { + std::vector retire; + bool wait_idle = false; + for (const auto& cached: m_images) { + const bool tracker_gpu = + m_memory_tracker.IsRegionGpuModified(cached->Address(), cached->Size()); + switch (ClassifyStorageImageOverlap( + requested.address, requested.size, cached->Address(), cached->Size(), + cached->kind == CachedImage::Kind::Texture, cached->ctx == ctx, cached->gpu_modified, + cached->buffer_modified, tracker_gpu)) { + case StorageImageOverlap::None: continue; + case StorageImageOverlap::RetireSampled: retire.push_back(cached.get()); continue; + case StorageImageOverlap::PageNeighbor: break; + case StorageImageOverlap::Unsupported: + EXIT("TextureCache: unsupported storage-image byte alias, requested=0x%016" PRIx64 + "+0x%016" PRIx64 " existing=0x%016" PRIx64 "+0x%016" PRIx64 + " kind=%u same_context=%d gpu=%d/%d buffer=%d\n", + requested.address, requested.size, cached->Address(), cached->Size(), + static_cast(cached->kind), cached->ctx == ctx, cached->gpu_modified, + tracker_gpu, cached->buffer_modified); + } + const bool tracker_cpu = + m_memory_tracker.IsRegionCpuModified(cached->Address(), cached->Size()); + if (cached->kind != CachedImage::Kind::StorageTexture || cached->ctx != ctx || + cached->buffer_modified || cached->info.IsCpuDirty() || + cached->gpu_modified != tracker_gpu || (tracker_gpu && tracker_cpu)) { + EXIT("TextureCache: unsupported storage-image page neighbor, requested=0x%016" PRIx64 + "+0x%016" PRIx64 " existing=0x%016" PRIx64 "+0x%016" PRIx64 + " kind=%u same_context=%d gpu=%d/%d cpu=%d buffer=%d image_cpu=%d\n", + requested.address, requested.size, cached->Address(), cached->Size(), + static_cast(cached->kind), cached->ctx == ctx, cached->gpu_modified, + tracker_gpu, tracker_cpu, cached->buffer_modified, cached->info.IsCpuDirty()); + } + wait_idle |= cached->gpu_modified; + retire.push_back(cached.get()); + } + RequireRetirementIsolation(retire, "storage neighbor", requested.address, requested.size); + if (wait_idle) { + VulkanDeviceWaitIdle(ctx); + } + for (auto* cached: retire) { + if (!cached->gpu_modified) { + continue; + } + const auto transfer = m_readback->DownloadColorImage(*cached); + m_memory_tracker.ForEachDownloadRange(transfer.address, transfer.size, + [](uint64_t, uint64_t) noexcept {}); + cached->gpu_modified = false; + } + RetireImages(retire); +} + TextureCache::~TextureCache() { m_readback.reset(); GraphicContext* ctx = nullptr; @@ -1411,7 +1795,11 @@ VulkanImage* TextureCache::FindTexture(CommandBuffer* command, GraphicContext* c static_cast(command), static_cast(ctx), info.address, info.size, info.width, info.height, info.depth, info.levels); } - std::lock_guard transaction(m_resource_mutex); + std::lock_guard transaction(m_resource_mutex); + { + FaultSafeTextureLock lock(this, m_lock); + RetireSampledTargetAliases(ctx, info); + } BufferImageCopySource source {nullptr, 0, info.address, info.size, true}; if (m_buffer_cache.HasPageOverlap(info.address, info.size)) { // ObtainBufferForImage publishes dirty native-buffer bytes when necessary and otherwise @@ -1513,8 +1901,8 @@ VulkanImage* TextureCache::FindTexture(CommandBuffer* command, GraphicContext* c info.address, info.size, false, [](uint64_t, uint64_t) noexcept {}, [&]() noexcept { m_tiler.DetileImage(match->ctx, - static_cast(match->image), - match->info, source, true, false); + static_cast(match->image), + match->info, source, true, false); }); if (match->info.IsCpuDirty()) { match->info.RefreshComplete(); @@ -1563,9 +1951,8 @@ VulkanImage* TextureCache::FindTexture(CommandBuffer* command, GraphicContext* c m_memory_tracker.ForEachUploadRange( info.address, info.size, false, [](uint64_t, uint64_t) noexcept {}, [&]() noexcept { - m_tiler.DetileImage(cached->ctx, - static_cast(cached->image), - cached->info, source, false, false); + m_tiler.DetileImage(cached->ctx, static_cast(cached->image), + cached->info, source, false, false); }); TextureCreateImageViews(ctx, cached->image, components, info.type, info.base_array, info.base_level, info.view_levels, info.depth, true, @@ -1579,9 +1966,10 @@ VulkanImage* TextureCache::FindTexture(CommandBuffer* command, GraphicContext* c StorageTextureVulkanImage* TextureCache::FindStorageTexture(CommandBuffer* command, GraphicContext* ctx, const ImageInfo& info) { - const bool supported_type = info.type == Prospero::GpuEnumValue(Prospero::ImageType::kColor2D) || - info.type == Prospero::GpuEnumValue(Prospero::ImageType::kColor2DArray) || - info.type == Prospero::GpuEnumValue(Prospero::ImageType::kColor3D); + const bool supported_type = + info.type == Prospero::GpuEnumValue(Prospero::ImageType::kColor2D) || + info.type == Prospero::GpuEnumValue(Prospero::ImageType::kColor2DArray) || + info.type == Prospero::GpuEnumValue(Prospero::ImageType::kColor3D); const bool supported_depth_tile = info.tile == Prospero::GpuEnumValue(Prospero::TileMode::kDepth) && IsSupportedStorageDepthTile(info.format, info.type, info.width, info.height, info.depth); @@ -1603,16 +1991,24 @@ StorageTextureVulkanImage* TextureCache::FindStorageTexture(CommandBuffer* com } if ((info.type == Prospero::GpuEnumValue(Prospero::ImageType::kColor2D) && info.depth != 1) || (info.type == Prospero::GpuEnumValue(Prospero::ImageType::kColor3D) && info.depth == 0) || - (info.type == Prospero::GpuEnumValue(Prospero::ImageType::kColor2DArray) && info.base_array >= info.depth)) { + (info.type == Prospero::GpuEnumValue(Prospero::ImageType::kColor2DArray) && + info.base_array >= info.depth)) { EXIT("TextureCache: storage-image type and depth disagree, type=%u depth=%u\n", info.type, info.depth); } - std::lock_guard transaction(m_resource_mutex); - if (m_buffer_cache.HasPageOverlap(info.address, info.size)) { - EXIT("TextureCache: storage image aliases buffer pages, addr=0x%016" PRIx64 - " size=0x%016" PRIx64 "\n", - info.address, info.size); + std::lock_guard transaction(m_resource_mutex); + const bool buffer_overlap = m_buffer_cache.HasPageOverlap(info.address, info.size); + BufferImageCopySource source {nullptr, 0, info.address, info.size, true}; + if (buffer_overlap) { + source = m_buffer_cache.ObtainBufferForImage(info.address, info.size); + if (!IsCoherentGuestImageSource(source, info.address, info.size)) { + EXIT("TextureCache: storage-image buffer source is inconsistent, addr=0x%016" PRIx64 + " size=0x%016" PRIx64 " source=%p source_addr=0x%016" PRIx64 + " source_size=0x%016" PRIx64 " current=%d\n", + info.address, info.size, static_cast(source.buffer), source.address, + source.size, source.cpu_current); + } } FaultSafeTextureLock lock(this, m_lock); RequireNoMetaOverlapLocked(info.address, info.size); @@ -1631,6 +2027,11 @@ StorageTextureVulkanImage* TextureCache::FindStorageTexture(CommandBuffer* com match = cached; } if (match != nullptr) { + if (buffer_overlap) { + EXIT("TextureCache: exact storage image aliases buffer pages, addr=0x%016" PRIx64 + " size=0x%016" PRIx64 "\n", + info.address, info.size); + } const bool cpu_modified = m_memory_tracker.IsRegionCpuModified(info.address, info.size); const bool gpu_modified = m_memory_tracker.IsRegionGpuModified(info.address, info.size); if (match->gpu_modified != gpu_modified || (gpu_modified && cpu_modified)) { @@ -1660,17 +2061,8 @@ StorageTextureVulkanImage* TextureCache::FindStorageTexture(CommandBuffer* com return static_cast(match->image); } - for (const auto& cached: m_images) { - if (cached->OverlapsRange(info.address, info.size, true)) { - EXIT("TextureCache: unsupported storage-image alias, requested=0x%016" PRIx64 - "+0x%016" PRIx64 " existing=0x%016" PRIx64 "+0x%016" PRIx64 - " kind=%u gpu_modified=%d\n", - info.address, info.size, cached->Address(), cached->Size(), - static_cast(cached->kind), cached->gpu_modified); - } - } + RetireStoragePageNeighbors(ctx, info); uint64_t readable = 0; - //For debugging purposes if (!HostMemoryQueryReadable(info.address, info.size, &readable) || readable < info.size) { EXIT("TextureCache: new storage image has unreadable guest backing, addr=0x%016" PRIx64 " size=0x%016" PRIx64 " readable=0x%016" PRIx64 "\n", @@ -1688,9 +2080,8 @@ StorageTextureVulkanImage* TextureCache::FindStorageTexture(CommandBuffer* com m_memory_tracker.ForEachUploadRange( info.address, info.size, true, [](uint64_t, uint64_t) noexcept {}, [&]() noexcept { - m_tiler.DetileImage( - cached->ctx, static_cast(cached->image), cached->info, - {nullptr, 0, cached->info.address, cached->info.size, true}, false, true); + m_tiler.DetileImage(cached->ctx, static_cast(cached->image), + cached->info, source, false, true); }); cached->gpu_modified = true; TextureCreateImageViews(ctx, cached->image, components, info.type, 0, 0, 1, info.depth, false, @@ -1708,7 +2099,7 @@ RenderTextureVulkanImage* TextureCache::FindRenderTarget(CommandBuffer* info.address >= TRACKER_ADDRESS_SIZE || info.size > TRACKER_ADDRESS_SIZE - info.address || info.format == VK_FORMAT_UNDEFINED || info.width == 0 || info.height == 0 || info.pitch < info.width || info.bytes_per_element == 0 || info.levels == 0 || - info.levels > 16 || + info.levels > 16 || info.layers == 0 || info.size % info.layers != 0 || (info.tile_mode != Prospero::GpuEnumValue(Prospero::TileMode::kLinear) && info.tile_mode != Prospero::GpuEnumValue(Prospero::TileMode::kRenderTarget))) { EXIT("TextureCache: invalid render-target request, ctx=%p addr=0x%016" PRIx64 @@ -1722,7 +2113,8 @@ RenderTextureVulkanImage* TextureCache::FindRenderTarget(CommandBuffer* !TileGetRenderTargetMipLayout(info.width, info.height, info.pitch, info.bytes_per_element, info.levels, &layout, nullptr, nullptr) || - layout.size != info.size) { + layout.size > UINT64_MAX / info.layers || + static_cast(layout.size) * info.layers != info.size) { EXIT("TextureCache: unsupported render-target mip layout, size=0x%016" PRIx64 " expected=0x%08x extent=%ux%u pitch=%u bpe=%u levels=%u tile=%u\n", info.size, layout.size, info.width, info.height, info.pitch, @@ -1736,7 +2128,8 @@ RenderTextureVulkanImage* TextureCache::FindRenderTarget(CommandBuffer* } const auto elements = rows * info.pitch + info.width; if (elements > UINT64_MAX / info.bytes_per_element || - info.size < elements * info.bytes_per_element) { + elements * info.bytes_per_element > UINT64_MAX / info.layers || + info.size < elements * info.bytes_per_element * info.layers) { EXIT("TextureCache: invalid render-target storage, size=0x%016" PRIx64 " elements=0x%016" PRIx64 " bpe=%u\n", info.size, elements, info.bytes_per_element); @@ -1754,7 +2147,8 @@ RenderTextureVulkanImage* TextureCache::FindRenderTarget(CommandBuffer* std::shared_ptr match; for (auto& cached: m_images) { if (cached->kind != CachedImage::Kind::RenderTarget || cached->ctx != ctx || - (!Equal(info, cached->target) && !IsCompatibleRenderTargetView(cached->target, info))) { + (!Equal(info, cached->target) && !IsCompatibleRenderTargetView(cached->target, info) && + !IsCompatibleRenderTargetBacking(cached->target, info))) { continue; } match = cached; @@ -1776,15 +2170,15 @@ RenderTextureVulkanImage* TextureCache::FindRenderTarget(CommandBuffer* info.address, info.size, false, [](uint64_t, uint64_t) noexcept {}, [&]() noexcept { UploadRenderTarget(ctx, static_cast(match->image), - match->target, true); + info, true); }); match->buffer_modified = false; } command->RetainResourceUntilFence(match); return static_cast(match->image); } - std::vector retire; - std::shared_ptr native_storage_source; + std::vector retire; + std::shared_ptr native_image_source; for (const auto& entry: m_images) { auto& cached = *entry; if (!cached.OverlapsRange(info.address, info.size, true)) { @@ -1798,15 +2192,15 @@ RenderTextureVulkanImage* TextureCache::FindRenderTarget(CommandBuffer* break; case CachedImage::Kind::StorageTexture: overlap = ClassifyStorageRenderTargetOverlap( - cached.info, cached.image->format, cached.gpu_modified, - cached.buffer_modified, + cached.info, cached.image->format, cached.gpu_modified, cached.buffer_modified, cached.info.IsCpuDirty() || m_memory_tracker.IsRegionCpuModified(cached.info.address, cached.info.size), cached.ctx == ctx, info); break; case CachedImage::Kind::RenderTarget: - overlap = ClassifyRenderTargetOverlap(cached.target, cached.gpu_modified, - cached.buffer_modified, cached.ctx == ctx, info); + overlap = + ClassifyRenderTargetOverlap(cached.target, cached.gpu_modified, + cached.buffer_modified, cached.ctx == ctx, info); break; case CachedImage::Kind::DepthTarget: case CachedImage::Kind::VideoOut: break; @@ -1818,9 +2212,16 @@ RenderTextureVulkanImage* TextureCache::FindRenderTarget(CommandBuffer* break; case RenderTargetOverlap::PreserveStorage: supported = cached.kind == CachedImage::Kind::StorageTexture && - native_storage_source == nullptr; + native_image_source == nullptr; if (supported) { - native_storage_source = entry; + native_image_source = entry; + } + break; + case RenderTargetOverlap::ExpandTarget: + supported = cached.kind == CachedImage::Kind::RenderTarget && + native_image_source == nullptr; + if (supported) { + native_image_source = entry; } break; case RenderTargetOverlap::RetireTarget: @@ -1843,40 +2244,39 @@ RenderTextureVulkanImage* TextureCache::FindRenderTarget(CommandBuffer* } retire.push_back(&cached); } - std::vector ranges; - ranges.reserve(m_images.size() * 2); - for (const auto& entry: m_images) { - const bool retiring = std::find(retire.begin(), retire.end(), entry.get()) != retire.end(); - for (uint32_t range = 0; range < entry->RangeCount(); range++) { - ranges.push_back({entry->Address(range), entry->Size(range), retiring}); - } - } - const auto conflict = FindImageRetirementConflict(ranges); - if (conflict.Exists()) { - const auto& retired = ranges[conflict.retired]; - const auto& retained = ranges[conflict.retained]; - EXIT("TextureCache: sampled retirement leaves a tracked page alias, target=0x%016" PRIx64 - "+0x%016" PRIx64 " retired=0x%016" PRIx64 "+0x%016" PRIx64 " retained=0x%016" PRIx64 - "+0x%016" PRIx64 "\n", - info.address, info.size, retired.address, retired.size, retained.address, - retained.size); - } - RetireImages(retire, native_storage_source.get()); - auto cached = std::make_shared(); - cached->kind = CachedImage::Kind::RenderTarget; - cached->target = info; - cached->ctx = ctx; - cached->image = CreateRenderTarget(ctx, info, &cached->memory); - const bool preserve_native = native_storage_source != nullptr; + RequireRetirementIsolation(retire, "render target", info.address, info.size); + RetireImages(retire, native_image_source.get()); + auto cached = std::make_shared(); + cached->kind = CachedImage::Kind::RenderTarget; + cached->target = info; + cached->ctx = ctx; + cached->image = CreateRenderTarget(ctx, info, &cached->memory); + const bool preserve_native = native_image_source != nullptr; if (preserve_native) { - command->RetainResourceUntilFence(native_storage_source); - ImageImageCopy region {}; - region.src_image = native_storage_source->image; - region.src_level = 0; - region.dst_level = 0; - region.width = info.width; - region.height = info.height; - UtilImageToImage(command, std::span(®ion, 1), cached->image, + command->RetainResourceUntilFence(native_image_source); + if (native_image_source->kind == CachedImage::Kind::RenderTarget) { + const auto& old = native_image_source->target; + const auto tail_address = info.address + old.size; + const auto tail_size = info.size - old.size; + if (old.layers >= info.layers || tail_size == 0 || + !IsCoherentGuestImageSource(target_source, info.address, info.size)) { + EXIT("TextureCache: invalid render-target expansion source, old_layers=%u " + "new_layers=%u old_size=0x%016" PRIx64 " new_size=0x%016" PRIx64 "\n", + old.layers, info.layers, old.size, info.size); + } + m_memory_tracker.ForEachUploadRange( + tail_address, tail_size, true, [](uint64_t, uint64_t) noexcept {}, + [&]() noexcept { + UploadRenderTargetLayers(ctx, + static_cast(cached->image), + info, old.layers, info.layers - old.layers, false); + }); + } + std::vector regions; + regions.reserve(static_cast(native_image_source->image->layers) * + native_image_source->image->mip_levels); + AppendLayerCopies(regions, native_image_source->image, VK_IMAGE_ASPECT_COLOR_BIT); + UtilImageToImage(command, regions, cached->image, static_cast(RENDER_COLOR_IMAGE_LAYOUT)); } else { m_memory_tracker.ForEachUploadRange( @@ -1888,9 +2288,10 @@ RenderTextureVulkanImage* TextureCache::FindRenderTarget(CommandBuffer* } cached->gpu_modified = preserve_native; if (preserve_native) { - LOGF("TextureCache: preserved storage image through native render-target copy, " - "addr=0x%016" PRIx64 " size=0x%016" PRIx64 " format=%d extent=%ux%u\n", - info.address, info.size, static_cast(info.format), info.width, info.height); + LOGF("TextureCache: preserved native image through render-target recreation, " + "addr=0x%016" PRIx64 " size=0x%016" PRIx64 " format=%d extent=%ux%u layers=%u\n", + info.address, info.size, static_cast(info.format), info.width, info.height, + info.layers); } auto* image = static_cast(cached->image); command->RetainResourceUntilFence(cached); @@ -1900,26 +2301,27 @@ RenderTextureVulkanImage* TextureCache::FindRenderTarget(CommandBuffer* DepthStencilVulkanImage* TextureCache::FindDepthTarget(CommandBuffer* command, GraphicContext* ctx, const DepthTargetInfo& info) { -//same as in findRenderTarget, but with additional checks for stencil and htile const bool has_stencil = info.stencil_address != 0 || info.stencil_size != 0; const bool has_htile = info.htile_address != 0 || info.htile_size != 0; if (command == nullptr || ctx == nullptr || info.address == 0 || info.size == 0 || info.address >= TRACKER_ADDRESS_SIZE || info.size > TRACKER_ADDRESS_SIZE - info.address || (info.address & 0xffffu) != 0 || info.width == 0 || info.height == 0 || - info.pitch < info.width || info.tile_mode != Prospero::GpuEnumValue(Prospero::TileMode::kDepth) || + info.pitch < info.width || info.layers == 0 || info.size % info.layers != 0 || + info.tile_mode != Prospero::GpuEnumValue(Prospero::TileMode::kDepth) || (has_stencil && (info.stencil_address == 0 || info.stencil_size == 0 || - info.stencil_address >= TRACKER_ADDRESS_SIZE || + info.stencil_size % info.layers != 0 || info.stencil_address >= TRACKER_ADDRESS_SIZE || info.stencil_size > TRACKER_ADDRESS_SIZE - info.stencil_address || (info.stencil_address & 0xffffu) != 0 || PageOverlaps(info.address, info.size, info.stencil_address, info.stencil_size))) || ((info.htile_address == 0) != (info.htile_size == 0)) || - (has_htile && (info.htile_address >= TRACKER_ADDRESS_SIZE || - info.htile_size > TRACKER_ADDRESS_SIZE - info.htile_address || - (info.htile_address & 0x7fffu) != 0 || - PageOverlaps(info.address, info.size, info.htile_address, info.htile_size) || - (has_stencil && PageOverlaps(info.stencil_address, info.stencil_size, - info.htile_address, info.htile_size)))) || + (has_htile && + (info.htile_size % info.layers != 0 || info.htile_address >= TRACKER_ADDRESS_SIZE || + info.htile_size > TRACKER_ADDRESS_SIZE - info.htile_address || + (info.htile_address & 0x7fffu) != 0 || + PageOverlaps(info.address, info.size, info.htile_address, info.htile_size) || + (has_stencil && PageOverlaps(info.stencil_address, info.stencil_size, info.htile_address, + info.htile_size)))) || (info.stencil_htile_compressed && (!has_stencil || !has_htile)) || !(info.guest_format == Prospero::GpuEnumValue(Prospero::BufferFormat::k16UNorm) ? !has_stencil && info.format == VK_FORMAT_D16_UNORM && info.bytes_per_element == 2 @@ -1942,12 +2344,14 @@ DepthStencilVulkanImage* TextureCache::FindDepthTarget(CommandBuffer* command, G } const auto elements = rows * info.pitch + info.width; if (elements > UINT64_MAX / info.bytes_per_element || - info.size < elements * info.bytes_per_element) { + elements * info.bytes_per_element > UINT64_MAX / info.layers || + info.size < elements * info.bytes_per_element * info.layers) { EXIT("TextureCache: depth storage is too small, size=0x%016" PRIx64 " elements=0x%016" PRIx64 " bpe=%u\n", info.size, elements, info.bytes_per_element); } - if (has_stencil && info.stencil_size < elements) { + if (has_stencil && + (elements > UINT64_MAX / info.layers || info.stencil_size < elements * info.layers)) { EXIT("TextureCache: stencil storage is too small, size=0x%016" PRIx64 " elements=0x%016" PRIx64 "\n", info.stencil_size, elements); @@ -1958,14 +2362,14 @@ DepthStencilVulkanImage* TextureCache::FindDepthTarget(CommandBuffer* command, G info.stencil_address, info.stencil_size, info.htile_address, info.htile_size); } if (has_htile) { - RegisterMeta(info.htile_address, info.htile_size); + RegisterMeta(info.htile_address, info.htile_size, info.layers); } std::lock_guard transaction(m_resource_mutex); // BufferCache treats an untracked guest range as CPU-current. That is useful when // uploading from guest memory, but it is not evidence that a clean native image is // stale. Only a genuinely overlapping buffer participates in transition selection. const bool depth_buffer_overlap = m_buffer_cache.HasPageOverlap(info.address, info.size); - const auto depth_source = m_buffer_cache.ObtainBufferForImage(info.address, info.size); + const auto depth_source = m_buffer_cache.ObtainBufferForImage(info.address, info.size); const auto stencil_source = has_stencil ? m_buffer_cache.ObtainBufferForImage(info.stencil_address, info.stencil_size) : BufferImageCopySource {}; @@ -1974,12 +2378,13 @@ DepthStencilVulkanImage* TextureCache::FindDepthTarget(CommandBuffer* command, G if (has_stencil) { RequireNoMetaOverlapLocked(info.stencil_address, info.stencil_size); } - CachedImage* match = nullptr; + std::shared_ptr match; for (auto& cached: m_images) { - if (cached->kind != CachedImage::Kind::DepthTarget || !Equal(info, cached->depth)) { + if (cached->kind != CachedImage::Kind::DepthTarget || cached->ctx != ctx || + (!Equal(info, cached->depth) && !IsCompatibleDepthTargetBacking(cached->depth, info))) { continue; } - match = cached.get(); + match = cached; } if (match != nullptr) { if (!CanLoadStencilAttachment(info, match->stencil_initialized)) { @@ -1994,28 +2399,28 @@ DepthStencilVulkanImage* TextureCache::FindDepthTarget(CommandBuffer* command, G m_memory_tracker.ForEachUploadRange( info.address, info.size, false, [](uint64_t, uint64_t) noexcept {}, [&]() noexcept { - m_tiler.DetileImage(ctx, - static_cast(match->image), - match->depth, depth_source, true); + m_tiler.DetileImage(ctx, static_cast(match->image), + info, depth_source, true); }); } if (!match->gpu_modified && has_stencil && stencil_source.cpu_dirty) { m_memory_tracker.ForEachUploadRange( - info.stencil_address, info.stencil_size, false, - [](uint64_t, uint64_t) noexcept {}, + info.stencil_address, info.stencil_size, false, [](uint64_t, uint64_t) noexcept {}, [&]() noexcept { if (!info.stencil_load_clear) { - m_tiler.DetileStencil( - ctx, static_cast(match->image), match->depth, - stencil_source, true); + m_tiler.DetileStencil(ctx, + static_cast(match->image), + info, stencil_source, true); match->stencil_initialized = true; } }); } + command->RetainResourceUntilFence(match); return static_cast(match->image); } - std::vector retire; + std::vector retire; std::shared_ptr sampled_depth_source; + std::shared_ptr native_depth_source; for (const auto& entry: m_images) { auto& cached = *entry; const bool overlaps = @@ -2024,39 +2429,52 @@ DepthStencilVulkanImage* TextureCache::FindDepthTarget(CommandBuffer* command, G if (!overlaps) { continue; } - if (cached.kind != CachedImage::Kind::Texture || cached.ctx != ctx || - ClassifyDepthOverlap(cached.info, cached.gpu_modified, info) != - DepthOverlap::RetireSampled) { + DepthOverlap overlap = DepthOverlap::Unsupported; + switch (cached.kind) { + case CachedImage::Kind::Texture: + overlap = cached.ctx == ctx + ? ClassifyDepthOverlap(cached.info, cached.gpu_modified, info) + : DepthOverlap::Unsupported; + break; + case CachedImage::Kind::DepthTarget: + overlap = + ClassifyDepthTargetOverlap(cached.depth, cached.gpu_modified, + cached.buffer_modified, cached.ctx == ctx, info); + break; + case CachedImage::Kind::StorageTexture: + case CachedImage::Kind::RenderTarget: + case CachedImage::Kind::VideoOut: break; + } + bool supported = false; + switch (overlap) { + case DepthOverlap::RetireSampled: + supported = cached.kind == CachedImage::Kind::Texture; + if (supported && !info.depth_load_clear && sampled_depth_source == nullptr) { + sampled_depth_source = entry; + } + break; + case DepthOverlap::ExpandTarget: + supported = + cached.kind == CachedImage::Kind::DepthTarget && native_depth_source == nullptr; + if (supported) { + native_depth_source = entry; + } + break; + case DepthOverlap::None: + case DepthOverlap::Unsupported: break; + } + if (!supported) { EXIT("TextureCache: unsupported depth-target alias, depth=0x%016" PRIx64 "+0x%016" PRIx64 " existing_kind=%u existing=0x%016" PRIx64 "+0x%016" PRIx64 "\n", info.address, info.size, static_cast(cached.kind), cached.Address(), cached.Size()); } - if (!info.depth_load_clear && sampled_depth_source == nullptr) { - sampled_depth_source = entry; - } retire.push_back(&cached); } - std::vector ranges; - ranges.reserve(m_images.size() * 2); - for (const auto& entry: m_images) { - const bool retiring = std::find(retire.begin(), retire.end(), entry.get()) != retire.end(); - for (uint32_t range = 0; range < entry->RangeCount(); range++) { - ranges.push_back({entry->Address(range), entry->Size(range), retiring}); - } - } - const auto conflict = FindImageRetirementConflict(ranges); - if (conflict.Exists()) { - const auto& retired = ranges[conflict.retired]; - const auto& retained = ranges[conflict.retained]; - EXIT("TextureCache: sampled retirement leaves a tracked page alias, depth=0x%016" PRIx64 - "+0x%016" PRIx64 " retired=0x%016" PRIx64 "+0x%016" PRIx64 " retained=0x%016" PRIx64 - "+0x%016" PRIx64 "\n", - info.address, info.size, retired.address, retired.size, retained.address, - retained.size); - } - RetireImages(retire); - if (!CanLoadStencilAttachment(info, false)) { + RequireRetirementIsolation(retire, "depth target", info.address, info.size); + RetireImages(retire, native_depth_source.get()); + if (!CanLoadStencilAttachment(info, native_depth_source != nullptr && + native_depth_source->stencil_initialized)) { EXIT("TextureCache: new stencil target requires a clear before stencil access, " "addr=0x%016" PRIx64 " size=0x%016" PRIx64 "\n", info.stencil_address, info.stencil_size); @@ -2065,68 +2483,108 @@ DepthStencilVulkanImage* TextureCache::FindDepthTarget(CommandBuffer* command, G cached->kind = CachedImage::Kind::DepthTarget; cached->depth = info; cached->ctx = ctx; - cached->stencil_initialized = !has_stencil || info.stencil_load_clear; + cached->stencil_initialized = !has_stencil; cached->image = CreateDepthTarget(ctx, info, &cached->memory); - if (sampled_depth_source != nullptr && - (sampled_depth_source->image->type != VulkanImageType::Texture || - sampled_depth_source->image->format != Prospero::SurfaceFormat(info.guest_format) || - sampled_depth_source->image->extent.width != info.width || - sampled_depth_source->image->extent.height != info.height)) { - EXIT("TextureCache: sampled-depth native source is inconsistent, format=%d/%d " - "extent=%ux%u/%ux%u type=%u\n", - static_cast(sampled_depth_source->image->format), - static_cast(Prospero::SurfaceFormat(info.guest_format)), - sampled_depth_source->image->extent.width, - sampled_depth_source->image->extent.height, info.width, info.height, - static_cast(sampled_depth_source->image->type)); - } - const auto transition_source = SelectDepthTransitionSource( - info.depth_load_clear, sampled_depth_source != nullptr, - sampled_depth_source != nullptr && sampled_depth_source->info.IsCpuDirty(), - sampled_depth_source != nullptr && sampled_depth_source->buffer_modified, - depth_buffer_overlap, depth_source.cpu_dirty); - LOGF("TextureCache: depth transition source=%u sampled=%d sampled_cpu_dirty=%d " - "sampled_buffer_dirty=%d buffer_overlap=%d buffer_cpu_dirty=%d addr=0x%016" PRIx64 - " size=0x%016" PRIx64 "\n", - static_cast(transition_source), sampled_depth_source != nullptr, - sampled_depth_source != nullptr && sampled_depth_source->info.IsCpuDirty(), - sampled_depth_source != nullptr && sampled_depth_source->buffer_modified, - depth_buffer_overlap, depth_source.cpu_dirty, info.address, info.size); - m_memory_tracker.ForEachUploadRange( - info.address, info.size, false, [](uint64_t, uint64_t) noexcept {}, - [&]() noexcept { - switch (transition_source) { - case DepthTransitionSource::None: break; - case DepthTransitionSource::Guest: - m_tiler.DetileImage(ctx, static_cast(cached->image), - info, depth_source, false); - break; - case DepthTransitionSource::Native: - command->RetainResourceUntilFence(sampled_depth_source); - UtilCopyImageWithBuffer( - command, ctx, sampled_depth_source->image, VK_IMAGE_ASPECT_COLOR_BIT, - cached->image, VK_IMAGE_ASPECT_DEPTH_BIT, info.bytes_per_element, - static_cast(VK_IMAGE_LAYOUT_DEPTH_STENCIL_ATTACHMENT_OPTIMAL)); - LOGF("TextureCache: preserved sampled depth through native buffer copy, " - "addr=0x%016" PRIx64 " size=0x%016" PRIx64 "\n", - info.address, info.size); - break; - } - }); - if (has_stencil) { + if (native_depth_source != nullptr) { + const auto& old = native_depth_source->depth; + if (old.layers >= info.layers || native_depth_source->image->layers != old.layers) { + EXIT("TextureCache: invalid depth expansion source, old_layers=%u new_layers=%u\n", + old.layers, info.layers); + } + command->RetainResourceUntilFence(native_depth_source); + const auto tail = SelectDepthLayers(info, old.layers, info.layers - old.layers); + const auto depth_tail = SelectSourceRange(depth_source, tail.address, tail.size); + if (!info.depth_load_clear && + !IsCoherentGuestImageSource(depth_tail, tail.address, tail.size)) { + EXIT("TextureCache: depth expansion tail is not CPU-current\n"); + } m_memory_tracker.ForEachUploadRange( - info.stencil_address, info.stencil_size, false, - [](uint64_t, uint64_t) noexcept {}, + tail.address, tail.size, true, [](uint64_t, uint64_t) noexcept {}, [&]() noexcept { - if (!info.stencil_load_clear) { - m_tiler.DetileStencil( - ctx, static_cast(cached->image), info, - stencil_source, false); + if (!info.depth_load_clear) { + m_tiler.DetileImage(ctx, static_cast(cached->image), + tail, depth_tail, false, old.layers); } }); - cached->stencil_initialized = true; + if (has_stencil) { + const auto stencil_tail = + SelectSourceRange(stencil_source, tail.stencil_address, tail.stencil_size); + if (!info.stencil_load_clear && + !IsCoherentGuestImageSource(stencil_tail, tail.stencil_address, + tail.stencil_size)) { + EXIT("TextureCache: stencil expansion tail is not CPU-current\n"); + } + m_memory_tracker.ForEachUploadRange( + tail.stencil_address, tail.stencil_size, true, [](uint64_t, uint64_t) noexcept {}, + [&]() noexcept { + if (!info.stencil_load_clear) { + m_tiler.DetileStencil(ctx, + static_cast(cached->image), + tail, stencil_tail, false, old.layers); + } + }); + cached->stencil_initialized = true; + } + std::vector regions; + regions.reserve(native_depth_source->image->layers * (has_stencil ? 2u : 1u)); + AppendLayerCopies(regions, native_depth_source->image, VK_IMAGE_ASPECT_DEPTH_BIT); + if (has_stencil) { + AppendLayerCopies(regions, native_depth_source->image, VK_IMAGE_ASPECT_STENCIL_BIT); + } + UtilImageToImage(command, regions, cached->image, + static_cast(VK_IMAGE_LAYOUT_DEPTH_STENCIL_ATTACHMENT_OPTIMAL)); + cached->gpu_modified = true; + LOGF("TextureCache: expanded native depth target, addr=0x%016" PRIx64 " size=0x%016" PRIx64 + " layers=%u->%u\n", + info.address, info.size, old.layers, info.layers); + } else { + if (sampled_depth_source != nullptr && + (sampled_depth_source->image->type != VulkanImageType::Texture || + sampled_depth_source->image->format != Prospero::SurfaceFormat(info.guest_format) || + sampled_depth_source->image->extent.width != info.width || + sampled_depth_source->image->extent.height != info.height)) { + EXIT("TextureCache: sampled-depth native source is inconsistent\n"); + } + const auto transition_source = SelectDepthTransitionSource( + info.depth_load_clear, sampled_depth_source != nullptr, + sampled_depth_source != nullptr && sampled_depth_source->info.IsCpuDirty(), + sampled_depth_source != nullptr && sampled_depth_source->buffer_modified, + depth_buffer_overlap, depth_source.cpu_dirty); + m_memory_tracker.ForEachUploadRange( + info.address, info.size, false, [](uint64_t, uint64_t) noexcept {}, + [&]() noexcept { + switch (transition_source) { + case DepthTransitionSource::None: break; + case DepthTransitionSource::Guest: + m_tiler.DetileImage(ctx, + static_cast(cached->image), + info, depth_source, false); + break; + case DepthTransitionSource::Native: + command->RetainResourceUntilFence(sampled_depth_source); + UtilCopyImageWithBuffer( + command, ctx, sampled_depth_source->image, VK_IMAGE_ASPECT_COLOR_BIT, + cached->image, VK_IMAGE_ASPECT_DEPTH_BIT, info.bytes_per_element, + static_cast( + VK_IMAGE_LAYOUT_DEPTH_STENCIL_ATTACHMENT_OPTIMAL)); + break; + } + }); + if (has_stencil) { + m_memory_tracker.ForEachUploadRange( + info.stencil_address, info.stencil_size, false, [](uint64_t, uint64_t) noexcept {}, + [&]() noexcept { + if (!info.stencil_load_clear) { + m_tiler.DetileStencil(ctx, + static_cast(cached->image), + info, stencil_source, false); + } + }); + cached->stencil_initialized = true; + } } auto* image = static_cast(cached->image); + command->RetainResourceUntilFence(cached); m_images.push_back(std::move(cached)); return image; } @@ -2584,20 +3042,11 @@ void TextureCache::MarkGpuWritten(VulkanImage* image) { } m_buffer_cache.ValidateGpuAccess(cached->depth.htile_address, cached->depth.htile_size, false, true); - const bool cpu_modified = m_metadata_tracker.IsRegionCpuModified( - cached->depth.htile_address, cached->depth.htile_size); - if (meta->second.gpu_modified && cpu_modified) { - EXIT("TextureCache: depth write races CPU-modified HTile metadata, " - "addr=0x%016" PRIx64 " size=0x%016" PRIx64 "\n", - cached->depth.htile_address, cached->depth.htile_size); - } - if (!meta->second.gpu_modified) { - m_metadata_tracker.ForEachUploadRange( - cached->depth.htile_address, cached->depth.htile_size, true, - [](uint64_t, uint64_t) noexcept {}, []() noexcept {}); - meta->second.gpu_modified = true; - } - meta->second.clear_mask = 0; + m_metadata_tracker.ForEachUploadRange( + cached->depth.htile_address, cached->depth.htile_size, true, + [](uint64_t, uint64_t) noexcept {}, []() noexcept {}); + meta->second.gpu_modified = true; + meta->second.clear_mask = 0; } } if (!cached->gpu_modified) { @@ -2655,10 +3104,11 @@ void TextureCache::PrepareHostWrite(uint64_t vaddr, uint64_t size) { } void TextureCache::SynchronizeRenderTargetToBufferLocked(CachedImage& cached) { - const auto& target = cached.target; - const bool linear = target.tile_mode == Prospero::GpuEnumValue(Prospero::TileMode::kLinear); - const bool tiled = target.tile_mode == Prospero::GpuEnumValue(Prospero::TileMode::kRenderTarget); - const auto rows = static_cast(target.height - 1); + const auto& target = cached.target; + const bool linear = target.tile_mode == Prospero::GpuEnumValue(Prospero::TileMode::kLinear); + const bool tiled = + target.tile_mode == Prospero::GpuEnumValue(Prospero::TileMode::kRenderTarget); + const auto rows = static_cast(target.height - 1); TileSizeAlign exact {}; const bool exact_tiled = tiled && TileGetRenderTargetSize(target.width, target.height, target.pitch, @@ -2703,13 +3153,13 @@ void TextureCache::SynchronizeRenderTargetToBufferLocked(CachedImage& cached) { cached.image, cached.image->layout); if (tiled) { m_buffer_transition_guest.resize(target.size); - m_tiler.TileImage(m_buffer_transition_guest.data(), - m_buffer_transition_linear.data(), target); + m_tiler.TileImage(m_buffer_transition_guest.data(), m_buffer_transition_linear.data(), + target); Libs::LibKernel::Memory::WriteBacking(target.address, m_buffer_transition_guest.data(), - target.size); + target.size); } else { Libs::LibKernel::Memory::WriteBacking(target.address, m_buffer_transition_linear.data(), - target.size); + target.size); } m_memory_tracker.ForEachDownloadRange(target.address, target.size, [](uint64_t, uint64_t) noexcept {}); @@ -2730,12 +3180,28 @@ void TextureCache::InvalidateMemoryFromGPU(uint64_t vaddr, uint64_t size, vaddr, size); } FaultSafeTextureLock lock(this, m_lock); - auto match = m_images.end(); - BufferImageAlias action = BufferImageAlias::None; - bool sampled_write = false; + for (auto it = m_surface_metas.begin(); it != m_surface_metas.end();) { + switch (ClassifyImageRangeOverlap(vaddr, size, it->first, it->second.size)) { + case ImageRangeOverlap::None: + case ImageRangeOverlap::PageOnly: ++it; continue; + case ImageRangeOverlap::Bytes: break; + } + LOGF("TextureCache: discarding overwritten virtual metadata, write=0x%016" PRIx64 + "+0x%016" PRIx64 " metadata=0x%016" PRIx64 "+0x%016" PRIx64 "\n", + vaddr, size, it->first, it->second.size); + if (it->second.gpu_modified) { + m_metadata_tracker.ForEachDownloadRange(it->first, it->second.size, + [](uint64_t, uint64_t) noexcept {}); + } + m_metadata_tracker.UntrackMemory(it->first, it->second.size); + it = m_surface_metas.erase(it); + } + auto match = m_images.end(); + BufferImageAlias action = BufferImageAlias::None; + bool sampled_write = false; for (auto it = m_images.begin(); it != m_images.end(); ++it) { auto& cached = **it; - if (!cached.OverlapsRange(vaddr, size, true)) { + if (!cached.OverlapsRange(vaddr, size, false)) { continue; } const auto alias = @@ -2762,11 +3228,6 @@ void TextureCache::InvalidateMemoryFromGPU(uint64_t vaddr, uint64_t size, if (match == m_images.end()) { return; } - if (HasMetaOverlapLocked(vaddr, size)) { - EXIT("TextureCache: GPU invalidation overlaps metadata, addr=0x%016" PRIx64 - " size=0x%016" PRIx64 "\n", - vaddr, size); - } if (sampled_write) { (**match).buffer_modified = true; return; @@ -2831,8 +3292,16 @@ RenderTextureVulkanImage* TextureCache::FindRenderTargetByRange(CommandBuffer* c !Overlaps(vaddr, size, cached->Address(), cached->Size())) { continue; } - if (vaddr != cached->Address() || size != cached->Size() || found != nullptr) { - EXIT("TextureCache: ambiguous or partial render-target range, addr=0x%016" PRIx64 + const auto slice_size = + cached->target.layers != 0 ? cached->target.size / cached->target.layers : 0; + const bool contained = vaddr == cached->target.address && slice_size != 0 && + cached->target.size % cached->target.layers == 0 && + size <= cached->target.size && size % slice_size == 0; + if (!contained) { + continue; + } + if (found != nullptr) { + EXIT("TextureCache: incompatible or ambiguous render-target range, addr=0x%016" PRIx64 " size=0x%016" PRIx64 " cached=0x%016" PRIx64 "+0x%016" PRIx64 " previous=%p\n", vaddr, size, cached->Address(), cached->Size(), static_cast(found.get())); @@ -2911,12 +3380,13 @@ bool TextureCache::HasGpuModifiedRangeOverlap(uint64_t vaddr, uint64_t size) { return false; } -void TextureCache::RegisterMeta(uint64_t vaddr, uint64_t size) { +void TextureCache::RegisterMeta(uint64_t vaddr, uint64_t size, uint32_t layers) { if (vaddr == 0 || size == 0 || vaddr >= TRACKER_ADDRESS_SIZE || - size > TRACKER_ADDRESS_SIZE - vaddr || (vaddr & 0x7fffu) != 0) { + size > TRACKER_ADDRESS_SIZE - vaddr || (vaddr & 0x7fffu) != 0 || layers == 0 || + layers > 32 || size % layers != 0) { EXIT("TextureCache: invalid metadata registration, addr=0x%016" PRIx64 " size=0x%016" PRIx64 - "\n", - vaddr, size); + " layers=%u\n", + vaddr, size, layers); } std::lock_guard transaction(m_resource_mutex); if (m_buffer_cache.HasPageOverlap(vaddr, size)) { @@ -2932,25 +3402,32 @@ void TextureCache::RegisterMeta(uint64_t vaddr, uint64_t size) { } } FaultSafeTextureLock lock(this, m_lock); - const auto existing = m_surface_metas.find(vaddr); + auto existing = m_surface_metas.find(vaddr); + uint64_t range_vaddr = vaddr; + uint64_t range_size = size; if (existing != m_surface_metas.end()) { - if (existing->second.size != size) { - EXIT("TextureCache: metadata address reused with a different size, addr=0x%016" PRIx64 - " requested=0x%016" PRIx64 " existing=0x%016" PRIx64 "\n", - vaddr, size, existing->second.size); + const auto slice_size = size / layers; + const auto old_slice_size = existing->second.size / existing->second.layers; + if (slice_size != old_slice_size || + (size > existing->second.size) != (layers > existing->second.layers)) { + EXIT("TextureCache: incompatible metadata backing growth\n"); } - return; + if (layers <= existing->second.layers) { + return; + } + range_vaddr += existing->second.size; + range_size -= existing->second.size; } for (const auto& [address, meta]: m_surface_metas) { - if (PageOverlaps(vaddr, size, address, meta.size)) { + if (address != vaddr && PageOverlaps(range_vaddr, range_size, address, meta.size)) { LOGF("TextureCache: registering overlapping metadata views, requested=0x%016" PRIx64 "+0x%016" PRIx64 " existing=0x%016" PRIx64 "+0x%016" PRIx64 "\n", - vaddr, size, address, meta.size); + range_vaddr, range_size, address, meta.size); } } std::vector retire; for (const auto& cached: m_images) { - if (!cached->OverlapsRange(vaddr, size, true)) { + if (!cached->OverlapsRange(range_vaddr, range_size, true)) { continue; } const auto overlap = @@ -2961,7 +3438,7 @@ void TextureCache::RegisterMeta(uint64_t vaddr, uint64_t size) { EXIT("TextureCache: metadata aliases image pages, addr=0x%016" PRIx64 " size=0x%016" PRIx64 " image_kind=%u image=0x%016" PRIx64 "+0x%016" PRIx64 " gpu_modified=%d buffer_modified=%d\n", - vaddr, size, static_cast(cached->kind), cached->Address(), + range_vaddr, range_size, static_cast(cached->kind), cached->Address(), cached->Size(), cached->gpu_modified, cached->buffer_modified); } if (overlap == MetaImageOverlap::RetireTarget) { @@ -2970,36 +3447,24 @@ void TextureCache::RegisterMeta(uint64_t vaddr, uint64_t size) { } LOGF("TextureCache: metadata aliases a CPU-current sampled image, metadata=0x%016" PRIx64 "+0x%016" PRIx64 " image=0x%016" PRIx64 "+0x%016" PRIx64 "\n", - vaddr, size, cached->Address(), cached->Size()); + range_vaddr, range_size, cached->Address(), cached->Size()); } if (!retire.empty()) { - std::vector ranges; - ranges.reserve(m_images.size() * 2); - for (const auto& cached: m_images) { - const bool retiring = - std::find(retire.begin(), retire.end(), cached.get()) != retire.end(); - for (uint32_t range = 0; range < cached->RangeCount(); range++) { - ranges.push_back({cached->Address(range), cached->Size(range), retiring}); - } - } - const auto conflict = FindImageRetirementConflict(ranges); - if (conflict.Exists()) { - const auto& retired = ranges[conflict.retired]; - const auto& retained = ranges[conflict.retained]; - EXIT("TextureCache: metadata retirement leaves a tracked page alias, " - "metadata=0x%016" PRIx64 "+0x%016" PRIx64 " retired=0x%016" PRIx64 "+0x%016" PRIx64 - " retained=0x%016" PRIx64 "+0x%016" PRIx64 "\n", - vaddr, size, retired.address, retired.size, retained.address, retained.size); - } + RequireRetirementIsolation(retire, "metadata", range_vaddr, range_size); for (const auto* cached: retire) { LOGF("TextureCache: retiring a CPU-current render target for metadata reuse, " "metadata=0x%016" PRIx64 "+0x%016" PRIx64 " target=0x%016" PRIx64 "+0x%016" PRIx64 "\n", - vaddr, size, cached->Address(), cached->Size()); + range_vaddr, range_size, cached->Address(), cached->Size()); } RetireImages(retire); } - m_surface_metas.emplace(vaddr, MetaDataInfo {size}); + if (existing == m_surface_metas.end()) { + m_surface_metas.emplace(vaddr, MetaDataInfo {.size = size, .layers = layers}); + } else { + existing->second.size = size; + existing->second.layers = layers; + } } bool TextureCache::IsMeta(uint64_t vaddr) { @@ -3017,6 +3482,19 @@ bool TextureCache::IsMetaRange(uint64_t vaddr, uint64_t size) { return it != m_surface_metas.end() && it->second.size == size; } +bool TextureCache::HasMetaRangeOverlap(uint64_t vaddr, uint64_t size) { + if (vaddr == 0 || size == 0 || vaddr >= TRACKER_ADDRESS_SIZE || + size > TRACKER_ADDRESS_SIZE - vaddr) { + EXIT("TextureCache: invalid metadata-range query, addr=0x%016" PRIx64 " size=0x%016" PRIx64 + "\n", + vaddr, size); + } + FaultSafeTextureLock lock(this, m_lock); + return std::any_of(m_surface_metas.begin(), m_surface_metas.end(), [&](const auto& entry) { + return Overlaps(vaddr, size, entry.first, entry.second.size); + }); +} + bool TextureCache::HasMetaOverlap(uint64_t vaddr, uint64_t size) { if (vaddr == 0 || size == 0 || vaddr >= TRACKER_ADDRESS_SIZE || size > TRACKER_ADDRESS_SIZE - vaddr) { @@ -3042,7 +3520,7 @@ bool TextureCache::IsMetaGpuModified(uint64_t vaddr, uint64_t size) { } FaultSafeTextureLock lock(this, m_lock); for (const auto& [address, meta]: m_surface_metas) { - if (PageOverlaps(vaddr, size, address, meta.size) && + if (Overlaps(vaddr, size, address, meta.size) && m_metadata_tracker.IsRegionGpuModified(vaddr, size)) { return true; } @@ -3068,19 +3546,27 @@ void TextureCache::RequireNoMetaOverlapLocked(uint64_t vaddr, uint64_t size) con } bool TextureCache::IsMetaCleared(uint64_t vaddr, uint32_t slice) { - if (slice >= 32) { - EXIT("TextureCache: metadata clear slice out of range, addr=0x%016" PRIx64 " slice=%u\n", - vaddr, slice); - } FaultSafeTextureLock lock(this, m_lock); const auto it = m_surface_metas.find(vaddr); - if (it != m_surface_metas.end() && it->second.gpu_modified && - m_metadata_tracker.IsRegionCpuModified(vaddr, it->second.size)) { - EXIT("TextureCache: GPU-owned metadata became CPU-modified, addr=0x%016" PRIx64 - " size=0x%016" PRIx64 "\n", - vaddr, it->second.size); + if (it == m_surface_metas.end()) { + return false; } - return it != m_surface_metas.end() && (it->second.clear_mask & (1u << slice)) != 0; + if (slice >= it->second.layers) { + EXIT("TextureCache: metadata clear slice out of range, addr=0x%016" PRIx64 + " slice=%u layers=%u\n", + vaddr, slice, it->second.layers); + } + if ((it->second.clear_mask & (1u << slice)) == 0) { + return false; + } + const auto slice_size = it->second.size / it->second.layers; + const auto slice_addr = vaddr + slice_size * slice; + if (!it->second.gpu_modified || + !m_metadata_tracker.IsRegionGpuModified(slice_addr, slice_size) || + m_metadata_tracker.IsRegionCpuModified(slice_addr, slice_size)) { + EXIT("TextureCache: cleared metadata slice is not GPU-owned\n"); + } + return true; } bool TextureCache::ClearMeta(uint64_t vaddr) { @@ -3093,28 +3579,30 @@ bool TextureCache::ClearMeta(uint64_t vaddr) { m_metadata_tracker.ForEachUploadRange( vaddr, it->second.size, true, [](uint64_t, uint64_t) noexcept {}, []() noexcept {}); it->second.gpu_modified = true; - it->second.clear_mask = UINT32_MAX; + it->second.clear_mask = it->second.layers == 32 ? UINT32_MAX : (1u << it->second.layers) - 1u; return true; } bool TextureCache::TouchMeta(uint64_t vaddr, uint32_t slice, bool is_clear) { - if (slice >= 32) { - EXIT("TextureCache: metadata update slice out of range, addr=0x%016" PRIx64 " slice=%u\n", - vaddr, slice); - } FaultSafeTextureLock lock(this, m_lock); const auto it = m_surface_metas.find(vaddr); if (it == m_surface_metas.end()) { return false; } - if (!it->second.gpu_modified) { + if (slice >= it->second.layers) { + EXIT("TextureCache: metadata update slice out of range\n"); + } + const auto slice_size = it->second.size / it->second.layers; + const auto slice_addr = vaddr + slice_size * slice; + if (!it->second.gpu_modified || + !m_metadata_tracker.IsRegionGpuModified(slice_addr, slice_size)) { EXIT("TextureCache: metadata update requires GPU ownership, addr=0x%016" PRIx64 "\n", vaddr); } - if (m_metadata_tracker.IsRegionCpuModified(vaddr, it->second.size)) { + if (m_metadata_tracker.IsRegionCpuModified(slice_addr, slice_size)) { EXIT("TextureCache: metadata update races CPU modification, addr=0x%016" PRIx64 " size=0x%016" PRIx64 "\n", - vaddr, it->second.size); + slice_addr, slice_size); } if (is_clear) { it->second.clear_mask |= 1u << slice; @@ -3351,7 +3839,8 @@ static constexpr int DummyTextureIndex(bool uint_format, bool image_3d) { } static constexpr uint32_t DummyTextureSwizzle() { - return Prospero::GpuEnumValue(Prospero::CompSwizzle::kRed) | (Prospero::GpuEnumValue(Prospero::CompSwizzle::kGreen) << 3u) | + return Prospero::GpuEnumValue(Prospero::CompSwizzle::kRed) | + (Prospero::GpuEnumValue(Prospero::CompSwizzle::kGreen) << 3u) | (Prospero::GpuEnumValue(Prospero::CompSwizzle::kBlue) << 6u) | (Prospero::GpuEnumValue(Prospero::CompSwizzle::kAlpha) << 9u); } @@ -3361,19 +3850,21 @@ static TextureImageCreateParams MakeDummyTextureParams(bool uint_format, bool im const char* owner) { TextureImageCreateParams params {}; params.fmt = static_cast( - Prospero::GpuEnumValue(uint_format ? Prospero::BufferFormat::k8_8_8_8UInt : Prospero::BufferFormat::k8_8_8_8UNorm)); - params.width = 1; - params.height = 1; - params.base_level = 0; - params.levels = 1; - params.depth = 1; - params.type = Prospero::GpuEnumValue(image_3d ? Prospero::ImageType::kColor3D : Prospero::ImageType::kColor2D); - params.swizzle = DummyTextureSwizzle(); - params.format_usage = usage; - params.required_format_usage = usage; - params.view_usage = usage; - params.image_layout = TextureUploadDestination::MipLevels; - params.allow_cube_view = true; + Prospero::GpuEnumValue(uint_format ? Prospero::BufferFormat::k8_8_8_8UInt + : Prospero::BufferFormat::k8_8_8_8UNorm)); + params.width = 1; + params.height = 1; + params.base_level = 0; + params.levels = 1; + params.depth = 1; + params.type = Prospero::GpuEnumValue(image_3d ? Prospero::ImageType::kColor3D + : Prospero::ImageType::kColor2D); + params.swizzle = DummyTextureSwizzle(); + params.format_usage = usage; + params.required_format_usage = usage; + params.view_usage = usage; + params.image_layout = TextureUploadDestination::MipLevels; + params.allow_cube_view = true; params.storage_swizzle_fallback = TextureHasFormatUsage(usage, TextureFormatUsage::Storage); params.owner = owner; return params; @@ -3492,6 +3983,13 @@ void TextureCache::DeleteDepthStencil(GraphicContext* ctx, DepthStencilVulkanIma EXIT_IF(image == nullptr); g_render_ctx->GetFramebufferCache()->FreeFramebufferByDepth(image); + for (auto& cached: image->attachment_views) { + if (cached.view != nullptr) { + vkDestroyImageView(ctx->device, cached.view, nullptr); + cached.view = nullptr; + } + } + image->attachment_views.clear(); DeleteImageViews(ctx, image); VulkanDeleteImage(ctx, image, mem); delete image; diff --git a/src/graphics/host_gpu/renderer/textureCache.h b/src/graphics/host_gpu/renderer/textureCache.h index dc3b8540f..39a18aaa3 100644 --- a/src/graphics/host_gpu/renderer/textureCache.h +++ b/src/graphics/host_gpu/renderer/textureCache.h @@ -31,7 +31,8 @@ class ResourceMutex; class TextureCache { public: - TextureCache(PageManager& page_manager, BufferCache& buffer_cache, ResourceMutex& resource_mutex); + TextureCache(PageManager& page_manager, BufferCache& buffer_cache, + ResourceMutex& resource_mutex); ~TextureCache(); KYTY_CLASS_NO_COPY(TextureCache); @@ -61,31 +62,41 @@ public: uint64_t vaddr, uint64_t size); [[nodiscard]] VkImageView GetRenderTargetAttachmentView(GraphicContext* ctx, RenderTextureVulkanImage* image, - VkFormat format, uint32_t level); + VkFormat format, uint32_t level, + uint32_t base_layer, + uint32_t layer_count); + [[nodiscard]] VkImageView GetDepthTargetAttachmentView(GraphicContext* ctx, + DepthStencilVulkanImage* image, + uint32_t base_layer, + uint32_t layer_count); [[nodiscard]] VkImageView GetRenderTargetSampledView(GraphicContext* ctx, RenderTextureVulkanImage* image, VkFormat view_format, int variant, - uint32_t base_level, uint32_t level_count); + uint32_t base_level, uint32_t level_count, + VkImageViewType type, uint32_t base_layer, + uint32_t layer_count); [[nodiscard]] VkImageView GetRenderTargetStorageView(GraphicContext* ctx, RenderTextureVulkanImage* image, VkFormat view_format, uint32_t base_level, - uint32_t level_count); + uint32_t level_count, VkImageViewType type, + uint32_t base_layer, uint32_t layer_count); [[nodiscard]] VkImageView GetStorageTextureSampledView(GraphicContext* ctx, StorageTextureVulkanImage* image, const ImageInfo& info); [[nodiscard]] DepthStencilVulkanImage* FindDepthTargetByRange(uint64_t vaddr, uint64_t size); [[nodiscard]] bool HasPageOverlap(uint64_t vaddr, uint64_t size); [[nodiscard]] bool HasRangeOverlap(uint64_t vaddr, uint64_t size); - [[nodiscard]] bool HasGpuModifiedRangeOverlap(uint64_t vaddr, uint64_t size); - [[nodiscard]] bool HasGpuTargetPageOverlap(uint64_t vaddr, uint64_t size); - void RegisterMeta(uint64_t vaddr, uint64_t size); - [[nodiscard]] bool IsMeta(uint64_t vaddr); - [[nodiscard]] bool IsMetaRange(uint64_t vaddr, uint64_t size); - [[nodiscard]] bool HasMetaOverlap(uint64_t vaddr, uint64_t size); - [[nodiscard]] bool IsMetaGpuModified(uint64_t vaddr, uint64_t size); - [[nodiscard]] bool IsMetaCleared(uint64_t vaddr, uint32_t slice); - [[nodiscard]] bool ClearMeta(uint64_t vaddr); - [[nodiscard]] bool TouchMeta(uint64_t vaddr, uint32_t slice, bool is_clear); + [[nodiscard]] bool HasGpuModifiedRangeOverlap(uint64_t vaddr, uint64_t size); + [[nodiscard]] bool HasGpuTargetPageOverlap(uint64_t vaddr, uint64_t size); + void RegisterMeta(uint64_t vaddr, uint64_t size, uint32_t layers = 1); + [[nodiscard]] bool IsMeta(uint64_t vaddr); + [[nodiscard]] bool IsMetaRange(uint64_t vaddr, uint64_t size); + [[nodiscard]] bool HasMetaRangeOverlap(uint64_t vaddr, uint64_t size); + [[nodiscard]] bool HasMetaOverlap(uint64_t vaddr, uint64_t size); + [[nodiscard]] bool IsMetaGpuModified(uint64_t vaddr, uint64_t size); + [[nodiscard]] bool IsMetaCleared(uint64_t vaddr, uint32_t slice); + [[nodiscard]] bool ClearMeta(uint64_t vaddr); + [[nodiscard]] bool TouchMeta(uint64_t vaddr, uint32_t slice, bool is_clear); [[nodiscard]] bool InvalidateMemory(PageFaultAccess access, uint64_t vaddr, uint64_t size, PageFaultPhase phase) noexcept; void UnmapMemory(uint64_t vaddr, uint64_t size); @@ -106,6 +117,7 @@ private: struct ReadbackWorker; struct MetaDataInfo { uint64_t size = 0; + uint32_t layers = 1; uint32_t clear_mask = 0; bool gpu_modified = false; }; @@ -115,9 +127,13 @@ private: uint64_t size) const; void RequireNoMetaOverlapLocked(uint64_t vaddr, uint64_t size) const; void MarkSampledAliasesCpuDirtyLocked(uint64_t vaddr, uint64_t size); + void RetireSampledTargetAliases(GraphicContext* ctx, const ImageInfo& requested); + void RetireStoragePageNeighbors(GraphicContext* ctx, const ImageInfo& requested); + void RequireRetirementIsolation(const std::vector& retire, const char* operation, + uint64_t address, uint64_t size) const; void RetireImages(const std::vector& retire, - const CachedImage* native_storage_source = nullptr); - void SynchronizeRenderTargetToBufferLocked(CachedImage& cached); + const CachedImage* native_image_source = nullptr); + void SynchronizeRenderTargetToBufferLocked(CachedImage& cached); Common::Mutex m_dummy_mutex; std::array m_dummy_sampled_textures {}; diff --git a/src/graphics/host_gpu/renderer/tiler.cpp b/src/graphics/host_gpu/renderer/tiler.cpp index c77a3137e..5284efc4c 100644 --- a/src/graphics/host_gpu/renderer/tiler.cpp +++ b/src/graphics/host_gpu/renderer/tiler.cpp @@ -8,11 +8,12 @@ #include "graphics/host_gpu/renderer/image.h" #include "graphics/host_gpu/utils.h" +#include + namespace Libs::Graphics { -void Tiler::DetileImage(GraphicContext* ctx, GpuTextureVulkanImage* image, - const ImageInfo& info, const BufferImageCopySource& source, - bool refresh, bool storage) const { +void Tiler::DetileImage(GraphicContext* ctx, GpuTextureVulkanImage* image, const ImageInfo& info, + const BufferImageCopySource& source, bool refresh, bool storage) const { if (ctx == nullptr || image == nullptr || info.address == 0 || info.size == 0 || info.width == 0 || info.height == 0 || info.depth == 0 || info.levels == 0 || info.levels >= 16 || !source.cpu_current || source.address != info.address || @@ -50,16 +51,19 @@ void Tiler::DetileImage(GraphicContext* ctx, GpuTextureVulkanImage* image, } void Tiler::DetileImage(GraphicContext* ctx, DepthStencilVulkanImage* image, - const DepthTargetInfo& info, const BufferImageCopySource& source, - bool refresh) const { - const bool d16 = info.guest_format == Prospero::GpuEnumValue(Prospero::BufferFormat::k16UNorm) && - info.format == VK_FORMAT_D16_UNORM && info.bytes_per_element == 2; + const DepthTargetInfo& info, const BufferImageCopySource& source, + bool refresh, uint32_t base_layer) const { + const bool d16 = + info.guest_format == Prospero::GpuEnumValue(Prospero::BufferFormat::k16UNorm) && + info.format == VK_FORMAT_D16_UNORM && info.bytes_per_element == 2; const bool d32 = info.guest_format == Prospero::GpuEnumValue(Prospero::BufferFormat::k32Float) && (info.format == VK_FORMAT_D32_SFLOAT || info.format == VK_FORMAT_D32_SFLOAT_S8_UINT) && info.bytes_per_element == 4; if (ctx == nullptr || image == nullptr || info.address == 0 || info.size == 0 || - info.width == 0 || info.height == 0 || info.pitch < info.width || + info.width == 0 || info.height == 0 || info.pitch < info.width || info.layers == 0 || + info.size % info.layers != 0 || base_layer > image->layers || + info.layers > image->layers - base_layer || info.size > UINT32_MAX || info.tile_mode != Prospero::GpuEnumValue(Prospero::TileMode::kDepth) || (!d16 && !d32) || !source.cpu_current || source.address != info.address || source.size != info.size || (source.buffer == nullptr && source.offset != 0)) { @@ -76,38 +80,55 @@ void Tiler::DetileImage(GraphicContext* ctx, DepthStencilVulkanImage* image, if (refresh) { VulkanDeviceWaitIdle(ctx); } - UtilScratchBuffer linear(info.size); - TileConvertTiledToLinearDepth(linear.Data(), reinterpret_cast(source.address), - info.guest_format, info.width, info.height, info.pitch, - info.size); - UtilFillImage(ctx, image, linear.Data(), info.size, info.pitch, VK_IMAGE_ASPECT_DEPTH_BIT); + const auto slice_size = info.size / info.layers; + UtilScratchBuffer linear(info.size); + std::vector regions; + regions.reserve(info.layers); + for (uint32_t layer = 0; layer < info.layers; layer++) { + auto* linear_slice = static_cast(linear.Data()) + slice_size * layer; + auto* guest_slice = reinterpret_cast(source.address) + slice_size * layer; + TileConvertTiledToLinearDepth(linear_slice, guest_slice, info.guest_format, info.width, + info.height, info.pitch, slice_size); + BufferImageCopy region {}; + region.offset = static_cast(slice_size * layer); + region.pitch = info.pitch; + region.width = info.width; + region.height = info.height; + region.dst_layer = base_layer + layer; + region.aspect = VK_IMAGE_ASPECT_DEPTH_BIT; + regions.push_back(region); + } + UtilFillImage(ctx, image, linear.Data(), info.size, regions, + VK_IMAGE_LAYOUT_DEPTH_STENCIL_ATTACHMENT_OPTIMAL); } void Tiler::DetileStencil(GraphicContext* ctx, DepthStencilVulkanImage* image, const DepthTargetInfo& info, const BufferImageCopySource& source, - bool refresh) const { + bool refresh, uint32_t base_layer) const { TileSizeAlign stencil_size {}; TileSizeAlign htile_size {}; TileSizeAlign depth_size {}; - const auto stencil_format = Prospero::GpuEnumValue(Prospero::BufferFormat::k8UInt); - const auto stencil_pitch = TileGetTexturePitch( - stencil_format, info.width, 1, Prospero::GpuEnumValue(Prospero::TileMode::kDepth)); - const bool supported_layout = + const auto stencil_format = Prospero::GpuEnumValue(Prospero::BufferFormat::k8UInt); + const auto stencil_pitch = TileGetTexturePitch( + stencil_format, info.width, 1, Prospero::GpuEnumValue(Prospero::TileMode::kDepth)); + const bool supported_layout = TileGetDepthSize(info.width, info.height, 0, Prospero::GpuEnumValue(Prospero::DepthFormat::kZ32F), Prospero::GpuEnumValue(Prospero::StencilFormat::k8UInt), info.htile_address != 0, &stencil_size, &htile_size, &depth_size) && - stencil_size.size == info.stencil_size && stencil_size.align == 65536 && + info.layers != 0 && info.stencil_size % info.layers == 0 && + stencil_size.size == info.stencil_size / info.layers && stencil_size.align == 65536 && stencil_pitch != 0; if (ctx == nullptr || image == nullptr || info.address == 0 || info.size == 0 || info.stencil_address == 0 || info.stencil_size == 0 || info.width == 0 || info.height == 0 || info.format != VK_FORMAT_D32_SFLOAT_S8_UINT || info.guest_format != Prospero::GpuEnumValue(Prospero::BufferFormat::k32Float) || - info.bytes_per_element != 4 || + info.bytes_per_element != 4 || base_layer > image->layers || + info.layers > image->layers - base_layer || info.stencil_size > UINT32_MAX || info.tile_mode != Prospero::GpuEnumValue(Prospero::TileMode::kDepth) || info.stencil_htile_compressed || !supported_layout || !source.cpu_current || - source.address != info.stencil_address || - source.size != info.stencil_size || (source.buffer == nullptr && source.offset != 0)) { + source.address != info.stencil_address || source.size != info.stencil_size || + (source.buffer == nullptr && source.offset != 0)) { EXIT("Tiler: unsupported stencil detile, ctx=%p image=%p source_buffer=%p " "source=0x%016" PRIx64 "+0x%016" PRIx64 " offset=0x%016" PRIx64 " current=%d stencil=0x%016" PRIx64 "+0x%016" PRIx64 @@ -122,12 +143,26 @@ void Tiler::DetileStencil(GraphicContext* ctx, DepthStencilVulkanImage* image, if (refresh) { VulkanDeviceWaitIdle(ctx); } - UtilScratchBuffer linear(info.stencil_size); - TileConvertTiledToLinearDepth(linear.Data(), reinterpret_cast(source.address), - stencil_format, info.width, info.height, stencil_pitch, - info.stencil_size); - UtilFillImage(ctx, image, linear.Data(), info.stencil_size, stencil_pitch, - VK_IMAGE_ASPECT_STENCIL_BIT); + const auto slice_size = info.stencil_size / info.layers; + UtilScratchBuffer linear(info.stencil_size); + std::vector regions; + regions.reserve(info.layers); + for (uint32_t layer = 0; layer < info.layers; layer++) { + auto* linear_slice = static_cast(linear.Data()) + slice_size * layer; + auto* guest_slice = reinterpret_cast(source.address) + slice_size * layer; + TileConvertTiledToLinearDepth(linear_slice, guest_slice, stencil_format, info.width, + info.height, stencil_pitch, slice_size); + BufferImageCopy region {}; + region.offset = static_cast(slice_size * layer); + region.pitch = stencil_pitch; + region.width = info.width; + region.height = info.height; + region.dst_layer = base_layer + layer; + region.aspect = VK_IMAGE_ASPECT_STENCIL_BIT; + regions.push_back(region); + } + UtilFillImage(ctx, image, linear.Data(), info.stencil_size, regions, + VK_IMAGE_LAYOUT_DEPTH_STENCIL_ATTACHMENT_OPTIMAL); } void Tiler::TileImage(void* dst, const void* src, const RenderTargetInfo& info) const { @@ -135,7 +170,8 @@ void Tiler::TileImage(void* dst, const void* src, const RenderTargetInfo& info) info.bytes_per_element == 4 || info.bytes_per_element == 8; if (dst == nullptr || src == nullptr || info.address == 0 || info.size == 0 || info.width == 0 || info.height == 0 || info.pitch < info.width || - info.tile_mode != Prospero::GpuEnumValue(Prospero::TileMode::kRenderTarget) || info.levels != 1 || + info.tile_mode != Prospero::GpuEnumValue(Prospero::TileMode::kRenderTarget) || + info.levels != 1 || info.layers == 0 || info.size % info.layers != 0 || !supported_element) { EXIT("Tiler: unsupported render-target tile, dst=%p src=%p " "addr=0x%016" PRIx64 "+0x%016" PRIx64 @@ -143,8 +179,13 @@ void Tiler::TileImage(void* dst, const void* src, const RenderTargetInfo& info) dst, src, info.address, info.size, info.width, info.height, info.pitch, info.levels, info.tile_mode, info.bytes_per_element); } - TileConvertLinearToTiledRenderTarget(dst, src, info.width, info.height, info.pitch, - info.bytes_per_element, info.size); + const auto slice_size = info.size / info.layers; + for (uint32_t layer = 0; layer < info.layers; layer++) { + auto* guest_slice = static_cast(dst) + slice_size * layer; + auto* linear_slice = static_cast(src) + slice_size * layer; + TileConvertLinearToTiledRenderTarget(guest_slice, linear_slice, info.width, info.height, + info.pitch, info.bytes_per_element, slice_size); + } } void Tiler::TileImage(void* dst, const void* src, const DepthTargetInfo& info) const { @@ -156,7 +197,8 @@ void Tiler::TileImage(void* dst, const void* src, const DepthTargetInfo& info) c if (dst == nullptr || src == nullptr || info.address == 0 || info.size == 0 || info.stencil_address != 0 || info.stencil_size != 0 || info.width == 0 || info.height == 0 || info.pitch < info.width || - info.tile_mode != Prospero::GpuEnumValue(Prospero::TileMode::kDepth) || !supported_format) { + info.tile_mode != Prospero::GpuEnumValue(Prospero::TileMode::kDepth) || info.layers == 0 || + info.size % info.layers != 0 || !supported_format) { EXIT("Tiler: unsupported depth-target tile, dst=%p src=%p " "depth=0x%016" PRIx64 "+0x%016" PRIx64 " stencil=0x%016" PRIx64 "+0x%016" PRIx64 " extent=%ux%u pitch=%u tile=%u format=%d guest_format=%u bpe=%u\n", @@ -164,8 +206,13 @@ void Tiler::TileImage(void* dst, const void* src, const DepthTargetInfo& info) c info.height, info.pitch, info.tile_mode, static_cast(info.format), info.guest_format, info.bytes_per_element); } - TileConvertLinearToTiledDepth(dst, src, info.guest_format, info.width, info.height, info.pitch, - info.size); + const auto slice_size = info.size / info.layers; + for (uint32_t layer = 0; layer < info.layers; layer++) { + auto* guest_slice = static_cast(dst) + slice_size * layer; + auto* linear_slice = static_cast(src) + slice_size * layer; + TileConvertLinearToTiledDepth(guest_slice, linear_slice, info.guest_format, info.width, + info.height, info.pitch, slice_size); + } } } // namespace Libs::Graphics diff --git a/src/graphics/host_gpu/renderer/tiler.h b/src/graphics/host_gpu/renderer/tiler.h index 5226c0816..5cc19ab08 100644 --- a/src/graphics/host_gpu/renderer/tiler.h +++ b/src/graphics/host_gpu/renderer/tiler.h @@ -19,11 +19,11 @@ public: void DetileImage(GraphicContext* ctx, GpuTextureVulkanImage* image, const ImageInfo& info, const BufferImageCopySource& source, bool refresh, bool storage) const; void DetileImage(GraphicContext* ctx, DepthStencilVulkanImage* image, - const DepthTargetInfo& info, const BufferImageCopySource& source, - bool refresh) const; + const DepthTargetInfo& info, const BufferImageCopySource& source, bool refresh, + uint32_t base_layer = 0) const; void DetileStencil(GraphicContext* ctx, DepthStencilVulkanImage* image, const DepthTargetInfo& info, const BufferImageCopySource& source, - bool refresh) const; + bool refresh, uint32_t base_layer = 0) const; void TileImage(void* dst, const void* src, const RenderTargetInfo& info) const; void TileImage(void* dst, const void* src, const DepthTargetInfo& info) const; }; diff --git a/src/graphics/host_gpu/utils.cpp b/src/graphics/host_gpu/utils.cpp index 8813401e9..21caaf679 100644 --- a/src/graphics/host_gpu/utils.cpp +++ b/src/graphics/host_gpu/utils.cpp @@ -251,38 +251,58 @@ ConvertBufferImageCopies(std::span regions) { for (size_t i = 0; i < regions.size(); i++) { const auto& r = regions[i]; - vk_regions[i] = - MakeBufferImageCopy(r.offset, r.pitch, VK_IMAGE_ASPECT_COLOR_BIT, r.dst_level, - r.dst_layer, {r.dst_x, r.dst_y, r.dst_z}, - {r.width, (r.copy_height != 0 ? r.copy_height : r.height), 1}); + vk_regions[i] = MakeBufferImageCopy( + r.offset, r.pitch, r.aspect, r.dst_level, r.dst_layer, {r.dst_x, r.dst_y, r.dst_z}, + {r.width, (r.copy_height != 0 ? r.copy_height : r.height), 1}); + } + return vk_regions; +} + +[[nodiscard]] static std::span +ConvertImageBufferCopies(std::span regions) { + static thread_local std::vector vk_regions; + vk_regions.resize(regions.size()); + for (size_t i = 0; i < regions.size(); i++) { + const auto& r = regions[i]; + vk_regions[i] = MakeBufferImageCopy( + r.offset, r.pitch, r.aspect, r.src_level, r.src_layer, {r.src_x, r.src_y, r.src_z}, + {r.width, r.copy_height != 0 ? r.copy_height : r.height, 1}); } return vk_regions; } static void RecordImageToBuffer(CommandBuffer& command, VulkanImage& src_image, - VulkanBuffer& dst_buffer, uint32_t dst_pitch, - VkImageLayout final_layout, VkImageAspectFlags aspect) { - auto* vk_command = command.GetPool()->buffers[command.GetIndex()]; - const auto transition_aspects = GetTransferAspects(src_image, aspect); - - SetImageLayout(vk_command, &src_image, 0, VK_REMAINING_MIP_LEVELS, transition_aspects, - src_image.layout, VK_IMAGE_LAYOUT_TRANSFER_SRC_OPTIMAL); - - const auto region = MakeBufferImageCopy(0, dst_pitch, aspect, 0, 0, {0, 0, 0}, - {src_image.extent.width, src_image.extent.height, 1}); + VulkanBuffer& dst_buffer, std::span regions, + VkImageLayout final_layout) { + auto* vk_command = command.GetPool()->buffers[command.GetIndex()]; + VkImageAspectFlags aspects = 0; + for (const auto& region: regions) { + aspects |= GetTransferAspects(src_image, region.aspect); + } + SetImageLayout(vk_command, &src_image, 0, VK_REMAINING_MIP_LEVELS, aspects, src_image.layout, + VK_IMAGE_LAYOUT_TRANSFER_SRC_OPTIMAL); SetBufferMemoryBarrier(vk_command, dst_buffer.buffer, 0, VK_WHOLE_SIZE, VK_ACCESS_MEMORY_READ_BIT, VK_ACCESS_TRANSFER_WRITE_BIT, VK_PIPELINE_STAGE_ALL_COMMANDS_BIT, VK_PIPELINE_STAGE_TRANSFER_BIT); + const auto copies = ConvertImageBufferCopies(regions); vkCmdCopyImageToBuffer(vk_command, src_image.image, VK_IMAGE_LAYOUT_TRANSFER_SRC_OPTIMAL, - dst_buffer.buffer, 1, ®ion); + dst_buffer.buffer, static_cast(copies.size()), copies.data()); SetBufferMemoryBarrier(vk_command, dst_buffer.buffer, 0, VK_WHOLE_SIZE, VK_ACCESS_TRANSFER_WRITE_BIT, VK_ACCESS_HOST_READ_BIT, VK_PIPELINE_STAGE_TRANSFER_BIT, VK_PIPELINE_STAGE_HOST_BIT); - - SetImageLayout(vk_command, &src_image, 0, VK_REMAINING_MIP_LEVELS, transition_aspects, + SetImageLayout(vk_command, &src_image, 0, VK_REMAINING_MIP_LEVELS, aspects, VK_IMAGE_LAYOUT_TRANSFER_SRC_OPTIMAL, final_layout); } +static void RecordImageToBuffer(CommandBuffer& command, VulkanImage& src_image, + VulkanBuffer& dst_buffer, uint32_t dst_pitch, + VkImageLayout final_layout, VkImageAspectFlags aspect) { + const ImageBufferCopy region { + 0, dst_pitch, 0, src_image.extent.width, src_image.extent.height, 0, 0, 0, 0, 0, aspect}; + RecordImageToBuffer(command, src_image, dst_buffer, + std::span(®ion, 1), final_layout); +} + class ReusableStagingBuffer { public: ReusableStagingBuffer() = default; @@ -328,10 +348,13 @@ public: uint64_t size, std::span regions, uint64_t dst_layout) { RecordUpload(ctx, src_data, size, [&](CommandBuffer* command, VkCommandBuffer) { + VkImageAspectFlags transition_aspects = 0; + for (const auto& region: regions) { + transition_aspects |= GetTransferAspects(*dst_image, region.aspect); + } RecordBufferToImageCopy(*command, m_buffer, *dst_image, - ConvertBufferImageCopies(regions), VK_IMAGE_ASPECT_COLOR_BIT, - VK_IMAGE_LAYOUT_UNDEFINED, - static_cast(dst_layout)); + ConvertBufferImageCopies(regions), transition_aspects, + dst_image->layout, static_cast(dst_layout)); }); } @@ -349,6 +372,18 @@ public: std::memcpy(dst_data, m_mapped_data, size); } + void DownloadFromImage(GraphicContext* ctx, void* dst_data, uint64_t size, + std::span regions, VulkanImage* src_image, + uint64_t src_layout) { + Common::LockGuard lock(m_mutex); + EnsureBuffer(ctx, size, VK_BUFFER_USAGE_TRANSFER_DST_BIT); + ExecuteImmediateUtilCommands([&](CommandBuffer* command, VkCommandBuffer) { + RecordImageToBuffer(*command, *src_image, m_buffer, regions, + static_cast(src_layout)); + }); + std::memcpy(dst_data, m_mapped_data, size); + } + private: template void RecordUpload(GraphicContext* ctx, const void* src_data, uint64_t size, @@ -583,7 +618,8 @@ static void SetImageLayout(VkCommandBuffer buffer, VulkanImage* dst_image, uint3 static bool CopyBlockImageToCompressedImage(VkCommandBuffer vk_buffer, const ImageImageCopy& r, VulkanImage* dst_image) { - if (!UtilIsBcFormat(dst_image->format)) { + if (r.src_aspect != VK_IMAGE_ASPECT_COLOR_BIT || r.dst_aspect != VK_IMAGE_ASPECT_COLOR_BIT || + !UtilIsBcFormat(dst_image->format)) { return false; } @@ -637,8 +673,10 @@ void UtilImageToImage(CommandBuffer* buffer, std::span reg VulkanImage* dst_image, uint64_t dst_layout) { auto* vk_buffer = buffer->GetPool()->buffers[buffer->GetIndex()]; - bool same_image_copy = false; + bool same_image_copy = false; + VkImageAspectFlags dst_transition_aspects = 0; for (const auto& r: regions) { + dst_transition_aspects |= GetTransferAspects(*dst_image, r.dst_aspect); if (r.src_image == dst_image) { same_image_copy = true; break; @@ -646,18 +684,18 @@ void UtilImageToImage(CommandBuffer* buffer, std::span reg } if (same_image_copy) { - SetImageLayout(vk_buffer, dst_image, 0, VK_REMAINING_MIP_LEVELS, VK_IMAGE_ASPECT_COLOR_BIT, + SetImageLayout(vk_buffer, dst_image, 0, VK_REMAINING_MIP_LEVELS, dst_transition_aspects, dst_image->layout, VK_IMAGE_LAYOUT_GENERAL); for (const auto& r: regions) { VkImageCopy region; - region.srcSubresource.aspectMask = VK_IMAGE_ASPECT_COLOR_BIT; + region.srcSubresource.aspectMask = r.src_aspect; region.srcSubresource.mipLevel = r.src_level; region.srcSubresource.baseArrayLayer = r.src_layer; region.srcSubresource.layerCount = 1; region.srcOffset = {r.src_x, r.src_y, r.src_z}; - region.dstSubresource.aspectMask = VK_IMAGE_ASPECT_COLOR_BIT; + region.dstSubresource.aspectMask = r.dst_aspect; region.dstSubresource.mipLevel = r.dst_level; region.dstSubresource.baseArrayLayer = r.dst_layer; region.dstSubresource.layerCount = 1; @@ -668,8 +706,9 @@ void UtilImageToImage(CommandBuffer* buffer, std::span reg auto restore_layout = VK_IMAGE_LAYOUT_GENERAL; if (r.src_image != dst_image) { restore_layout = r.src_image->layout; - SetImageLayout(vk_buffer, r.src_image, r.src_level, 1, VK_IMAGE_ASPECT_COLOR_BIT, - restore_layout, VK_IMAGE_LAYOUT_TRANSFER_SRC_OPTIMAL); + SetImageLayout(vk_buffer, r.src_image, r.src_level, 1, + GetTransferAspects(*r.src_image, r.src_aspect), restore_layout, + VK_IMAGE_LAYOUT_TRANSFER_SRC_OPTIMAL); src_layout = VK_IMAGE_LAYOUT_TRANSFER_SRC_OPTIMAL; } @@ -677,51 +716,56 @@ void UtilImageToImage(CommandBuffer* buffer, std::span reg VK_IMAGE_LAYOUT_GENERAL, 1, ®ion); if (r.src_image != dst_image) { - SetImageLayout(vk_buffer, r.src_image, r.src_level, 1, VK_IMAGE_ASPECT_COLOR_BIT, + SetImageLayout(vk_buffer, r.src_image, r.src_level, 1, + GetTransferAspects(*r.src_image, r.src_aspect), VK_IMAGE_LAYOUT_TRANSFER_SRC_OPTIMAL, restore_layout); } } - SetImageLayout(vk_buffer, dst_image, 0, VK_REMAINING_MIP_LEVELS, VK_IMAGE_ASPECT_COLOR_BIT, + SetImageLayout(vk_buffer, dst_image, 0, VK_REMAINING_MIP_LEVELS, dst_transition_aspects, VK_IMAGE_LAYOUT_GENERAL, static_cast(dst_layout)); return; } - SetImageLayout(vk_buffer, dst_image, 0, VK_REMAINING_MIP_LEVELS, VK_IMAGE_ASPECT_COLOR_BIT, - VK_IMAGE_LAYOUT_UNDEFINED, VK_IMAGE_LAYOUT_TRANSFER_DST_OPTIMAL); + SetImageLayout(vk_buffer, dst_image, 0, VK_REMAINING_MIP_LEVELS, dst_transition_aspects, + dst_image->layout, VK_IMAGE_LAYOUT_TRANSFER_DST_OPTIMAL); for (const auto& r: regions) { VkImageCopy region; auto src_layout = r.src_image->layout; - if (CopyBlockImageToCompressedImage(vk_buffer, r, dst_image)) { + if (r.src_aspect == VK_IMAGE_ASPECT_COLOR_BIT && + r.dst_aspect == VK_IMAGE_ASPECT_COLOR_BIT && + CopyBlockImageToCompressedImage(vk_buffer, r, dst_image)) { continue; } - region.srcSubresource.aspectMask = VK_IMAGE_ASPECT_COLOR_BIT; + region.srcSubresource.aspectMask = r.src_aspect; region.srcSubresource.mipLevel = r.src_level; region.srcSubresource.baseArrayLayer = r.src_layer; region.srcSubresource.layerCount = 1; region.srcOffset = {r.src_x, r.src_y, r.src_z}; - region.dstSubresource.aspectMask = VK_IMAGE_ASPECT_COLOR_BIT; + region.dstSubresource.aspectMask = r.dst_aspect; region.dstSubresource.mipLevel = r.dst_level; region.dstSubresource.baseArrayLayer = r.dst_layer; region.dstSubresource.layerCount = 1; region.dstOffset = {r.dst_x, r.dst_y, r.dst_z}; region.extent = {r.width, r.height, 1}; - SetImageLayout(vk_buffer, r.src_image, r.src_level, 1, VK_IMAGE_ASPECT_COLOR_BIT, - src_layout, VK_IMAGE_LAYOUT_TRANSFER_SRC_OPTIMAL); + SetImageLayout(vk_buffer, r.src_image, r.src_level, 1, + GetTransferAspects(*r.src_image, r.src_aspect), src_layout, + VK_IMAGE_LAYOUT_TRANSFER_SRC_OPTIMAL); vkCmdCopyImage(vk_buffer, r.src_image->image, VK_IMAGE_LAYOUT_TRANSFER_SRC_OPTIMAL, dst_image->image, VK_IMAGE_LAYOUT_TRANSFER_DST_OPTIMAL, 1, ®ion); - SetImageLayout(vk_buffer, r.src_image, r.src_level, 1, VK_IMAGE_ASPECT_COLOR_BIT, + SetImageLayout(vk_buffer, r.src_image, r.src_level, 1, + GetTransferAspects(*r.src_image, r.src_aspect), VK_IMAGE_LAYOUT_TRANSFER_SRC_OPTIMAL, src_layout); } - SetImageLayout(vk_buffer, dst_image, 0, VK_REMAINING_MIP_LEVELS, VK_IMAGE_ASPECT_COLOR_BIT, + SetImageLayout(vk_buffer, dst_image, 0, VK_REMAINING_MIP_LEVELS, dst_transition_aspects, VK_IMAGE_LAYOUT_TRANSFER_DST_OPTIMAL, static_cast(dst_layout)); } @@ -954,6 +998,15 @@ void UtilFillBuffer(GraphicContext* ctx, void* dst_data, uint64_t size, uint32_t ->DownloadFromImage(ctx, dst_data, size, dst_pitch, src_image, src_layout, aspect); } +void UtilFillBuffer(GraphicContext* ctx, void* dst_data, uint64_t size, + std::span regions, VulkanImage* src_image, + uint64_t src_layout) { + KYTY_PROFILER_FUNCTION(); + EXIT_IF(size == 0 || regions.empty()); + GetStagingBuffer(StagingBufferType::ReadBack) + ->DownloadFromImage(ctx, dst_data, size, regions, src_image, src_layout); +} + void UtilSetImageLayoutOptimal(VulkanImage* image) { ExecuteImmediateUtilCommands([&](CommandBuffer*, VkCommandBuffer vk_command) { SetImageLayout(vk_command, image, 0, VK_REMAINING_MIP_LEVELS, VK_IMAGE_ASPECT_COLOR_BIT, diff --git a/src/graphics/host_gpu/utils.h b/src/graphics/host_gpu/utils.h index 9ada13ad8..a7f5e0500 100644 --- a/src/graphics/host_gpu/utils.h +++ b/src/graphics/host_gpu/utils.h @@ -19,32 +19,49 @@ struct DepthStencilVulkanImage; struct VulkanSwapchain; struct BufferImageCopy { - uint32_t offset; - uint32_t pitch; - uint32_t dst_level; - uint32_t width; - uint32_t height; - uint32_t copy_height = 0; - uint32_t dst_layer = 0; - int dst_x; - int dst_y; - int dst_z = 0; + uint32_t offset; + uint32_t pitch; + uint32_t dst_level; + uint32_t width; + uint32_t height; + uint32_t copy_height = 0; + uint32_t dst_layer = 0; + int dst_x; + int dst_y; + int dst_z = 0; + VkImageAspectFlags aspect = VK_IMAGE_ASPECT_COLOR_BIT; +}; + +struct ImageBufferCopy { + uint32_t offset; + uint32_t pitch; + uint32_t src_level; + uint32_t width; + uint32_t height; + uint32_t copy_height = 0; + uint32_t src_layer = 0; + int src_x; + int src_y; + int src_z = 0; + VkImageAspectFlags aspect = VK_IMAGE_ASPECT_COLOR_BIT; }; struct ImageImageCopy { - VulkanImage* src_image; - uint32_t src_level; - uint32_t dst_level; - uint32_t width; - uint32_t height; - uint32_t src_layer = 0; - uint32_t dst_layer = 0; - int src_x; - int src_y; - int src_z = 0; - int dst_x; - int dst_y; - int dst_z = 0; + VulkanImage* src_image; + uint32_t src_level; + uint32_t dst_level; + uint32_t width; + uint32_t height; + uint32_t src_layer = 0; + uint32_t dst_layer = 0; + VkImageAspectFlags src_aspect = VK_IMAGE_ASPECT_COLOR_BIT; + VkImageAspectFlags dst_aspect = VK_IMAGE_ASPECT_COLOR_BIT; + int src_x; + int src_y; + int src_z = 0; + int dst_x; + int dst_y; + int dst_z = 0; }; enum class StagingBufferType { Texture, Vertex, ReadBack }; @@ -79,6 +96,9 @@ void UtilFillImage(GraphicContext* ctx, std::span regions, void UtilFillBuffer(GraphicContext* ctx, void* dst_data, uint64_t size, uint32_t dst_pitch, VulkanImage* src_image, uint64_t src_layout, VkImageAspectFlags aspect = VK_IMAGE_ASPECT_COLOR_BIT); +void UtilFillBuffer(GraphicContext* ctx, void* dst_data, uint64_t size, + std::span regions, VulkanImage* src_image, + uint64_t src_layout); void UtilUploadBuffer(GraphicContext* ctx, StagingBufferType type, VulkanBuffer* dst_buffer, uint64_t dst_offset, const void* src_data, uint64_t size); void UtilCopyBuffer(VulkanBuffer* src_buffer, VulkanBuffer* dst_buffer, uint64_t size); diff --git a/tests/ShaderRecompilerComputeTests.cpp b/tests/ShaderRecompilerComputeTests.cpp index dd7c26677..0a3577477 100644 --- a/tests/ShaderRecompilerComputeTests.cpp +++ b/tests/ShaderRecompilerComputeTests.cpp @@ -10080,6 +10080,50 @@ void CheckOverlappingMetadataViews() { std::printf("[host] %-32s ok\n", "OverlappingMetadataViews"); } +void CheckGpuMetadataReuse() { + constexpr uintptr_t base = 0x0000000200010000ull; + constexpr uint64_t allocation_size = 0x20000; + constexpr uint64_t metadata_size = 0x8000; + auto *memory = static_cast(VirtualAlloc( + reinterpret_cast(base), allocation_size, + MEM_RESERVE | MEM_COMMIT, PAGE_READWRITE)); + Require("GpuMetadataReuse", "allocation", + memory == reinterpret_cast(base), + "fixed VirtualAlloc failed"); + + ResourceMutex resource_mutex; + CacheFaultContext fault_context; + PageManager page_manager(CacheFault, &fault_context); + BufferCache buffer_cache(page_manager, resource_mutex); + TextureCache texture_cache(page_manager, buffer_cache, resource_mutex); + fault_context.texture = &texture_cache; + buffer_cache.SetTextureCache(texture_cache); + page_manager.OnGpuMap(base, allocation_size); + + texture_cache.RegisterMeta(base, metadata_size); + Require("GpuMetadataReuse", "clear", texture_cache.ClearMeta(base), + "metadata clear setup failed"); + texture_cache.InvalidateMemoryFromGPU(base, allocation_size); + Require("GpuMetadataReuse", "discard", + !texture_cache.IsMeta(base) && + !texture_cache.HasMetaRangeOverlap(base, allocation_size), + "fully overwritten metadata identity was retained"); + texture_cache.RegisterMeta(base, metadata_size); + Require("GpuMetadataReuse", "re-register", + texture_cache.IsMetaRange(base, metadata_size) && + texture_cache.ClearMeta(base), + "metadata identity could not be reused after a GPU overwrite"); + texture_cache.InvalidateMemoryFromGPU(base + 0x1000, 0x1000); + Require("GpuMetadataReuse", "partial discard", !texture_cache.IsMeta(base), + "partially overwritten metadata identity was retained"); + + texture_cache.UnmapMemory(base, allocation_size); + page_manager.OnGpuUnmap(base, allocation_size); + Require("GpuMetadataReuse", "free", VirtualFree(memory, 0, MEM_RELEASE) != 0, + "VirtualFree failed"); + std::printf("[host] %-32s ok\n", "GpuMetadataReuse"); +} + void CheckMetadataReuseDescriptors() { ValidateMetadataReuseTexture(BasicMetadataReuseResource(), BasicMetadataReuseDescriptor(), 0x10000); @@ -10219,6 +10263,22 @@ void CheckImageOverlapResolution() { ClassifySampledOverlap(sampled_alias, sampled, false, false) == SampledOverlap::Unsupported, "cross-context sampled alias was admitted"); + constexpr uint64_t storage_subrange = 0x649b0100; + constexpr uint64_t storage_subrange_size = 0x800; + constexpr uint64_t sampled_backing = 0x649a3000; + constexpr uint64_t sampled_backing_size = 0x24000; + Require("ImageOverlapResolution", "storage retires clean sampled backing", + ClassifyStorageImageOverlap(storage_subrange, storage_subrange_size, + sampled_backing, sampled_backing_size, + true, true, false, false, false) == + StorageImageOverlap::RetireSampled, + "clean sampled subrange was not retired before storage creation"); + Require("ImageOverlapResolution", "storage preserves dirty sampled backing", + ClassifyStorageImageOverlap(storage_subrange, storage_subrange_size, + sampled_backing, sampled_backing_size, + true, true, true, false, true) == + StorageImageOverlap::Unsupported, + "GPU-owned sampled subrange was admitted as storage backing"); ImageInfo page_left = sampled; page_left.size = TRACKER_PAGE_SIZE / 2; ImageInfo same_page = page_left; @@ -10539,6 +10599,25 @@ void CheckImageOverlapResolution() { ClassifyRenderTargetOverlap(sampled, true, true, target) == RenderTargetOverlap::Unsupported, "GPU-owned sampled allocation was retired for a render target"); + ImageInfo partial_target_sample{}; + partial_target_sample.address = 0x79b01000; + partial_target_sample.size = 0x20000; + RenderTargetInfo layered_target{}; + layered_target.address = 0x79b10000; + layered_target.size = 0x60000; + Require("ImageOverlapResolution", "sampled partial target transition", + ClassifySampledRenderTargetOverlap(partial_target_sample, + layered_target, false, true) == + RenderTargetOverlap::RetireTarget, + "sampled partial target alias was not routed through readback"); + Require("ImageOverlapResolution", "sampled partial target rejection", + ClassifySampledRenderTargetOverlap(partial_target_sample, + layered_target, true, true) == + RenderTargetOverlap::Unsupported && + ClassifySampledRenderTargetOverlap( + partial_target_sample, layered_target, false, false) == + RenderTargetOverlap::Unsupported, + "buffer-owned or cross-context target alias was admitted"); ImageInfo storage{}; storage.address = 0x112cd0000ull; @@ -11375,6 +11454,7 @@ int main(int argc, char **argv) { CheckStorageTextureAccessPermissions(); CheckMetaOverlapDeaths(); CheckOverlappingMetadataViews(); + CheckGpuMetadataReuse(); CheckMetadataReuseDescriptors(); CheckImageOverlapResolution(); CheckMsaaCompatibility();