diff --git a/src/CMakeLists.txt b/src/CMakeLists.txt index 79e19f6e6..2114bf05a 100644 --- a/src/CMakeLists.txt +++ b/src/CMakeLists.txt @@ -137,7 +137,7 @@ config_compiler_and_linker() add_subdirectory("${KYTY_THIRD_PARTY_DIR}" "${CMAKE_CURRENT_BINARY_DIR}/3rdparty") add_subdirectory(common) -file(GLOB kyty_emulator_src +file(GLOB kyty_emulator_src CONFIGURE_DEPENDS libs/*.cpp libs/*.h graphics/*.cpp diff --git a/src/graphics/host_gpu/graphicContext.h b/src/graphics/host_gpu/graphicContext.h index f134b0332..ebae4e95e 100644 --- a/src/graphics/host_gpu/graphicContext.h +++ b/src/graphics/host_gpu/graphicContext.h @@ -75,7 +75,7 @@ struct ImageViewInfo { uint32_t base_layer = 0; uint32_t layer_count = 1; uint32_t swizzle = 0; - bool is_storage = false; + VkImageUsageFlags usage = VK_IMAGE_USAGE_SAMPLED_BIT; bool operator==(const ImageViewInfo&) const = default; }; @@ -121,15 +121,8 @@ struct VideoOutVulkanImage: public VulkanImage { }; struct DepthStencilVulkanImage: public VulkanImage { - struct AttachmentView { - uint32_t base_layer = 0; - uint32_t layer_count = 1; - VkImageView view = nullptr; - }; DepthStencilVulkanImage(): VulkanImage(VulkanImageType::DepthStencil) {} - bool compressed = false; - std::mutex attachment_view_mutex; - std::vector attachment_views; + bool compressed = false; }; struct GpuTextureVulkanImage: public VulkanImage { @@ -145,17 +138,7 @@ struct StorageTextureVulkanImage: public GpuTextureVulkanImage { }; struct RenderTextureVulkanImage: public VulkanImage { - struct AttachmentView { - VkFormat format = VK_FORMAT_UNDEFINED; - uint32_t level = 0; - uint32_t base_layer = 0; - uint32_t layer_count = 1; - VkImageView view = nullptr; - }; RenderTextureVulkanImage(): VulkanImage(VulkanImageType::RenderTexture) {} - VkImageView render_view[16] = {}; - std::mutex attachment_view_mutex; - std::vector attachment_views; }; struct VulkanBuffer { diff --git a/src/graphics/host_gpu/renderer/bufferCache.cpp b/src/graphics/host_gpu/renderer/bufferCache.cpp index b21d51fdc..450b4abdf 100644 --- a/src/graphics/host_gpu/renderer/bufferCache.cpp +++ b/src/graphics/host_gpu/renderer/bufferCache.cpp @@ -676,6 +676,7 @@ std::pair BufferCache::ObtainBuffer(CommandBuffer* com const auto begin = AlignDown(vaddr); const auto end = AlignUp(vaddr + size); std::lock_guard transaction(m_resource_mutex); + const auto texture_region = m_texture_cache->QueryRegion(vaddr, size); // Use the stream-buffer fast path before image/buffer alias handling. Clean image and metadata // views may coexist with a small CPU-current read; Kyty's separate image trackers require // GPU-dirty ownership guards here. Read physical backing so @@ -683,8 +684,7 @@ std::pair BufferCache::ObtainBuffer(CommandBuffer* com if (is_read && !is_written && size <= CACHING_PAGE_SIZE && !m_memory_tracker.IsRegionGpuModified(vaddr, size) && m_memory_tracker.IsRegionCpuModified(vaddr, size) && - !m_texture_cache->HasGpuModifiedRangeOverlap(vaddr, size) && - !m_texture_cache->IsMetaGpuModified(vaddr, size)) { + !texture_region.gpu_image_bytes && !texture_region.gpu_metadata_bytes) { std::array guest_data; if (Libs::LibKernel::Memory::TryReadBacking(vaddr, guest_data.data(), size)) { VulkanBuffer* stream_buffer = nullptr; @@ -697,7 +697,10 @@ std::pair BufferCache::ObtainBuffer(CommandBuffer* com } } } - if (m_texture_cache->HasMetaOverlap(begin, end - begin)) { + const auto texture_pages = begin == vaddr && end - begin == size + ? texture_region + : m_texture_cache->QueryRegion(begin, end - begin); + if (texture_pages.metadata_pages) { EXIT("BufferCache: buffer aliases metadata pages, addr=0x%016" PRIx64 " size=0x%016" PRIx64 "\n", begin, end - begin); @@ -706,11 +709,10 @@ std::pair BufferCache::ObtainBuffer(CommandBuffer* com // an edge page. Clean read-only buffer and image views may coexist; Kyty retains a hard failure // when either cache owns newer GPU bytes. Writable buffers delegate the ownership transition // to TextureCache, which distinguishes raw texture-data writes from formatted target paths. - if (m_texture_cache->HasPageOverlap(begin, end - begin) && - m_texture_cache->HasRangeOverlap(vaddr, size)) { + if (texture_pages.image_pages && texture_region.image_bytes) { const bool coherent_read = is_read && !is_written && !m_memory_tracker.IsRegionGpuModified(vaddr, size) && - !m_texture_cache->HasGpuModifiedRangeOverlap(vaddr, size); + !texture_region.gpu_image_bytes; if (!coherent_read) { if (!is_written) { EXIT("BufferCache: unsupported buffer/image alias, addr=0x%016" PRIx64 @@ -1084,7 +1086,8 @@ void BufferCache::FillBuffer(CommandBuffer* command, GraphicContext* ctx, uint64 ValidateGpuAccess(vaddr, size, false, true); { std::lock_guard transaction(m_resource_mutex); - const bool image_overlap = m_texture_cache->HasRangeOverlap(vaddr, size); + const auto texture_region = m_texture_cache->QueryRegion(vaddr, size); + const bool image_overlap = texture_region.image_bytes; const bool buffer_overlap = HasPageOverlap(vaddr, size); const bool buffer_gpu_modified = IsRegionGpuModified(vaddr, size); if (!buffer_overlap && !buffer_gpu_modified) { @@ -1100,7 +1103,7 @@ void BufferCache::FillBuffer(CommandBuffer* command, GraphicContext* ctx, uint64 " size=0x%016" PRIx64 "\n", vaddr, size); } - if (m_texture_cache->HasMetaRangeOverlap(vaddr, size)) { + if (texture_region.metadata_bytes) { LOGF("BufferCache: GPU fill overlaps virtual metadata, addr=0x%016" PRIx64 " size=0x%016" PRIx64 "\n", vaddr, size); @@ -1140,16 +1143,18 @@ void BufferCache::CopyBuffer(CommandBuffer* command, GraphicContext* ctx, uint64 bool dst_image_transition = false; { std::lock_guard transaction(m_resource_mutex); - const bool src_image_gpu = m_texture_cache->HasGpuModifiedRangeOverlap(src_vaddr, size); - const bool src_meta = m_texture_cache->HasMetaRangeOverlap(src_vaddr, size); - if (m_texture_cache->HasGpuTargetPageOverlap(src_vaddr, size)) { + const auto src_region = m_texture_cache->QueryRegion(src_vaddr, size); + const auto dst_region = m_texture_cache->QueryRegion(dst_vaddr, size); + const bool src_image_gpu = src_region.gpu_image_bytes; + const bool src_meta = src_region.metadata_bytes; + if (src_region.non_sampled_pages) { EXIT("BufferCache: GPU copy aliases target pages, src=0x%016" PRIx64 " dst=0x%016" PRIx64 " size=0x%016" PRIx64 "\n", src_vaddr, dst_vaddr, size); } if (!HasPageOverlap(dst_vaddr, size) && !IsRegionGpuModified(src_vaddr, size) && !IsRegionGpuModified(dst_vaddr, size) && !src_image_gpu) { - if (m_texture_cache->IsMetaGpuModified(src_vaddr, size)) { + if (src_region.gpu_metadata_bytes) { LOGF("BufferCache: host copy reads virtual metadata, src=0x%016" PRIx64 " size=0x%016" PRIx64 "\n", src_vaddr, size); @@ -1157,7 +1162,7 @@ void BufferCache::CopyBuffer(CommandBuffer* command, GraphicContext* ctx, uint64 "src=0x%016" PRIx64 " size=0x%016" PRIx64 "\n", src_vaddr, size); } - if (m_texture_cache->HasRangeOverlap(dst_vaddr, size)) { + if (dst_region.image_bytes) { m_texture_cache->PrepareHostWrite(dst_vaddr, size); } std::memcpy(reinterpret_cast(dst_vaddr), @@ -1170,14 +1175,15 @@ void BufferCache::CopyBuffer(CommandBuffer* command, GraphicContext* ctx, uint64 src_vaddr, dst_vaddr, size); } dst_image_transition = m_texture_cache->InvalidateMemoryFromGPU(dst_vaddr, size); + const auto transitioned_dst = m_texture_cache->QueryRegion(dst_vaddr, size); // A clean target destination is handled above like a protected host write. Target // aliases that require an actual GPU buffer copy remain unsupported. - if (m_texture_cache->HasGpuTargetPageOverlap(dst_vaddr, size)) { + if (transitioned_dst.non_sampled_pages) { EXIT("BufferCache: GPU copy aliases target pages, src=0x%016" PRIx64 " dst=0x%016" PRIx64 " size=0x%016" PRIx64 "\n", src_vaddr, dst_vaddr, size); } - if (src_meta || m_texture_cache->HasMetaRangeOverlap(dst_vaddr, size)) { + if (src_meta || transitioned_dst.metadata_bytes) { LOGF("BufferCache: GPU copy overlaps virtual metadata, src=0x%016" PRIx64 " dst=0x%016" PRIx64 " size=0x%016" PRIx64 "\n", src_vaddr, dst_vaddr, size); diff --git a/src/graphics/host_gpu/renderer/context.cpp b/src/graphics/host_gpu/renderer/context.cpp index 9bafdd875..cf7031318 100644 --- a/src/graphics/host_gpu/renderer/context.cpp +++ b/src/graphics/host_gpu/renderer/context.cpp @@ -40,10 +40,10 @@ public: } return m_pool[id]; } + void DeleteAll(); private: void Create(int id); - void DeleteAll(); VulkanCommandPool* m_pool[GraphicContext::QUEUES_NUM] = {}; }; @@ -78,6 +78,12 @@ void GraphicsRenderInit() { g_render_ctx = new RenderContext; } +void GraphicsRenderReleaseThreadCommandPools() { + if (g_render_ctx != nullptr) { + g_command_pool.DeleteAll(); + } +} + void GraphicsRenderCreateContext() { EXIT_IF(g_render_ctx == nullptr); diff --git a/src/graphics/host_gpu/renderer/descriptors.cpp b/src/graphics/host_gpu/renderer/descriptors.cpp index f3d98f7b0..20758f1be 100644 --- a/src/graphics/host_gpu/renderer/descriptors.cpp +++ b/src/graphics/host_gpu/renderer/descriptors.cpp @@ -567,8 +567,8 @@ NativeTexture(uint64_t submit_id, CommandBuffer* command_buffer, descriptor.MsaaDepth(); if (image == nullptr) { if (check_depth) { - image = - g_render_ctx->GetTextureCache()->FindDepthTargetByRange(address, size.size, true); + image = g_render_ctx->GetTextureCache()->FindDepthTargetByRange( + command_buffer, address, size.size, true); } else { image = g_render_ctx->GetTextureCache()->FindRenderTargetByRange(command_buffer, address, size.size); @@ -699,7 +699,7 @@ NativeTexture(uint64_t submit_id, CommandBuffer* command_buffer, } if (image == nullptr) { auto* texture_cache = g_render_ctx->GetTextureCache(); - const bool metadata_read = texture_cache->HasMetaOverlap(address, size.size); + const bool metadata_read = texture_cache->QueryRegion(address, size.size).metadata_pages; if (storage && metadata_read) { EXIT("storage texture overlaps surface metadata\n"); } diff --git a/src/graphics/host_gpu/renderer/dummyTextureCache.cpp b/src/graphics/host_gpu/renderer/dummyTextureCache.cpp new file mode 100644 index 000000000..91114cdee --- /dev/null +++ b/src/graphics/host_gpu/renderer/dummyTextureCache.cpp @@ -0,0 +1,58 @@ +#include "graphics/host_gpu/renderer/dummyTextureCache.h" + +#include "common/assert.h" +#include "graphics/host_gpu/renderer/image.h" +#include "graphics/host_gpu/utils.h" + +namespace Libs::Graphics { + +namespace { + +[[nodiscard]] constexpr size_t DummyTextureIndex(bool uint_format, bool image_3d) noexcept { + return (image_3d ? 2u : 0u) + (uint_format ? 1u : 0u); +} + +} // namespace + +DummyTextureCache::~DummyTextureCache() { + Common::LockGuard lock(m_mutex); + if (m_ctx == nullptr) { + return; + } + + VulkanDeviceWaitIdle(m_ctx); + const auto destroy = [this](auto& slots) { + for (auto& slot: slots) { + if (slot.image != nullptr) { + ImageOps::Destroy(m_ctx, slot.image, &slot.memory); + slot.image = nullptr; + } + } + }; + destroy(m_sampled); + destroy(m_storage); +} + +VulkanImage* DummyTextureCache::Get(GraphicContext* ctx, Usage usage, bool uint_format, + bool image_3d) { + Common::LockGuard lock(m_mutex); + if (ctx == nullptr) { + EXIT("TextureCache: dummy texture requires a graphics context\n"); + } + if (m_ctx != nullptr && m_ctx != ctx) { + EXIT("TextureCache: dummy texture context changed, previous=%p current=%p usage=%u\n", + static_cast(m_ctx), static_cast(ctx), + static_cast(usage)); + } + m_ctx = ctx; + + auto& slots = usage == Usage::Storage ? m_storage : m_sampled; + auto& slot = slots[DummyTextureIndex(uint_format, image_3d)]; + if (slot.image == nullptr) { + slot.image = ImageOps::CreateDummyTexture(ctx, uint_format, image_3d, + usage == Usage::Storage, &slot.memory); + } + return slot.image; +} + +} // namespace Libs::Graphics diff --git a/src/graphics/host_gpu/renderer/dummyTextureCache.h b/src/graphics/host_gpu/renderer/dummyTextureCache.h new file mode 100644 index 000000000..2dc060923 --- /dev/null +++ b/src/graphics/host_gpu/renderer/dummyTextureCache.h @@ -0,0 +1,38 @@ +#ifndef EMULATOR_SRC_GRAPHICS_HOST_GPU_RENDERER_DUMMYTEXTURECACHE_H_ +#define EMULATOR_SRC_GRAPHICS_HOST_GPU_RENDERER_DUMMYTEXTURECACHE_H_ + +#include "common/abi.h" +#include "common/threads.h" +#include "graphics/host_gpu/graphicContext.h" + +#include +#include + +namespace Libs::Graphics { + +class DummyTextureCache final { +public: + enum class Usage: uint8_t { Sampled, Storage }; + + DummyTextureCache() = default; + ~DummyTextureCache(); + KYTY_CLASS_NO_COPY(DummyTextureCache); + + [[nodiscard]] VulkanImage* Get(GraphicContext* ctx, Usage usage, bool uint_format, + bool image_3d); + +private: + struct Slot { + GpuTextureVulkanImage* image = nullptr; + VulkanMemory memory {}; + }; + + Common::Mutex m_mutex; + std::array m_sampled {}; + std::array m_storage {}; + GraphicContext* m_ctx = nullptr; +}; + +} // namespace Libs::Graphics + +#endif // EMULATOR_SRC_GRAPHICS_HOST_GPU_RENDERER_DUMMYTEXTURECACHE_H_ diff --git a/src/graphics/host_gpu/renderer/image.cpp b/src/graphics/host_gpu/renderer/image.cpp new file mode 100644 index 000000000..d3fdd395d --- /dev/null +++ b/src/graphics/host_gpu/renderer/image.cpp @@ -0,0 +1,470 @@ +#include "graphics/host_gpu/renderer/image.h" + +#include "common/assert.h" +#include "common/profiler.h" +#include "graphics/guest_gpu/gpu_defs.h" +#include "graphics/guest_gpu/tile.h" +#include "graphics/host_gpu/graphicContext.h" +#include "graphics/host_gpu/objects/textureCommon.h" +#include "graphics/host_gpu/regionDefinitions.h" +#include "graphics/host_gpu/renderer/framebufferCache.h" +#include "graphics/host_gpu/renderer/imageView.h" +#include "graphics/host_gpu/renderer/renderContext.h" +#include "graphics/host_gpu/utils.h" +#include "graphics/shader/shader.h" + +#include + +namespace Libs::Graphics { + +namespace { + +TextureImageCreateParams MakeImageParams(const ImageInfo& info, bool storage) { + TextureImageCreateParams params {}; + params.fmt = info.format; + params.width = info.width; + params.height = info.height; + params.base_level = SelectImageBackingBaseLevel(storage, info.base_level); + params.levels = info.levels; + params.depth = info.depth; + params.type = info.type; + // Storage image views use identity component mapping. The guest storage write mapping is + // validated before this point and intentionally does not become a Vulkan view swizzle. + params.swizzle = storage ? DstSel(4, 5, 6, 7) : info.swizzle; + params.format_usage = TextureFormatUsage::Sampled | TextureFormatUsage::Storage; + params.required_format_usage = storage + ? TextureFormatUsage::Sampled | TextureFormatUsage::Storage + : TextureFormatUsage::Sampled; + params.view_usage = storage ? TextureFormatUsage::Sampled | TextureFormatUsage::Storage + : TextureFormatUsage::Sampled; + params.image_layout = TextureUploadDestination::MipLevels; + params.allow_cube_view = !storage; + params.compatible_format_views = + storage && + (IsRgba8SrgbViewFormat(TextureGetFormat(info.format, params.format_usage)) || + info.format == Prospero::GpuEnumValue(Prospero::BufferFormat::k32UInt) || + info.format == Prospero::GpuEnumValue(Prospero::BufferFormat::k32Float)); + params.owner = storage ? "StorageTextureCache" : "TextureCache"; + return params; +} + +bool RenderTargetSupportsStorage(GraphicContext* ctx, VkFormat format, VkImageCreateFlags flags) { + const auto compatible = SrgbStorageViewFormat(format); + const auto required_flags = + VK_IMAGE_CREATE_MUTABLE_FORMAT_BIT | VK_IMAGE_CREATE_EXTENDED_USAGE_BIT; + const bool compatible_views = (flags & required_flags) == required_flags; + return ImageViewOps::FormatSupportsStorage(ctx, format) || + (compatible_views && compatible != VK_FORMAT_UNDEFINED && + ImageViewOps::FormatSupportsStorage(ctx, compatible)); +} + +VkImageCreateFlags RenderTargetCreateFlags(VkFormat format) { + const bool compatible_format_view = + IsRgba8SrgbViewFormat(format) || BgraToRgbaSampledViewFormat(format) != VK_FORMAT_UNDEFINED || + format == VK_FORMAT_R8G8B8A8_UINT || format == VK_FORMAT_R16G16B16A16_SFLOAT || + format == VK_FORMAT_R16G16B16A16_UINT; + return compatible_format_view + ? VK_IMAGE_CREATE_MUTABLE_FORMAT_BIT | VK_IMAGE_CREATE_EXTENDED_USAGE_BIT + : VkImageCreateFlags {0}; +} + +VkImageUsageFlags RenderTargetUsage(GraphicContext* ctx, VkFormat format, + VkImageCreateFlags flags) { + auto usage = static_cast(VK_IMAGE_USAGE_COLOR_ATTACHMENT_BIT) | + static_cast(VK_IMAGE_USAGE_TRANSFER_SRC_BIT) | + static_cast(VK_IMAGE_USAGE_TRANSFER_DST_BIT) | + static_cast(VK_IMAGE_USAGE_SAMPLED_BIT); + if (RenderTargetSupportsStorage(ctx, format, flags)) { + usage |= VK_IMAGE_USAGE_STORAGE_BIT; + } + VkImageFormatProperties properties {}; + if (ctx->GetImageFormatProperties(format, VK_IMAGE_TYPE_2D, VK_IMAGE_TILING_OPTIMAL, usage, + flags, &properties) != VK_SUCCESS) { + EXIT("TextureCache: render-target format does not support required usage, format=%d " + "usage=0x%x\n", + static_cast(format), usage); + } + return usage; +} + +[[nodiscard]] uint32_t RenderTargetTransferFormatImpl(uint32_t bytes_per_element) { + switch (bytes_per_element) { + case 1: return Prospero::GpuEnumValue(Prospero::BufferFormat::k8UNorm); + case 2: return Prospero::GpuEnumValue(Prospero::BufferFormat::k16UNorm); + case 4: return Prospero::GpuEnumValue(Prospero::BufferFormat::k32Float); + case 8: return Prospero::GpuEnumValue(Prospero::BufferFormat::k16_16_16_16Float); + case 16: return Prospero::GpuEnumValue(Prospero::BufferFormat::k32_32_32_32Float); + default: + EXIT("TextureCache: unsupported render-target element size: %u\n", bytes_per_element); + } +} + +static constexpr uint32_t DummyTextureSwizzle() { + return Prospero::GpuEnumValue(Prospero::CompSwizzle::kRed) | + (Prospero::GpuEnumValue(Prospero::CompSwizzle::kGreen) << 3u) | + (Prospero::GpuEnumValue(Prospero::CompSwizzle::kBlue) << 6u) | + (Prospero::GpuEnumValue(Prospero::CompSwizzle::kAlpha) << 9u); +} + +TextureImageCreateParams MakeDummyTextureParams(bool uint_format, bool image_3d, + TextureFormatUsage usage, + const char* owner) { + TextureImageCreateParams params {}; + params.fmt = static_cast( + Prospero::GpuEnumValue(uint_format ? Prospero::BufferFormat::k8_8_8_8UInt + : Prospero::BufferFormat::k8_8_8_8UNorm)); + params.width = 1; + params.height = 1; + params.base_level = 0; + params.levels = 1; + params.depth = 1; + params.type = Prospero::GpuEnumValue(image_3d ? Prospero::ImageType::kColor3D + : Prospero::ImageType::kColor2D); + params.swizzle = DummyTextureSwizzle(); + params.format_usage = usage; + params.required_format_usage = usage; + params.view_usage = usage; + params.image_layout = TextureUploadDestination::MipLevels; + params.allow_cube_view = true; + params.storage_swizzle_fallback = TextureHasFormatUsage(usage, TextureFormatUsage::Storage); + params.owner = owner; + return params; +} + +} // namespace + +namespace ImageOps { + +uint32_t RenderTargetTransferFormat(uint32_t bytes_per_element) { + return RenderTargetTransferFormatImpl(bytes_per_element); +} + +GpuTextureVulkanImage* CreateTexture(GraphicContext* ctx, const ImageInfo& info, bool storage, + VulkanMemory* memory, VkComponentMapping* components) { + if (components == nullptr) { + EXIT("TextureCache: invalid texture component output\n"); + } + auto* image = storage ? static_cast(new StorageTextureVulkanImage) + : new TextureVulkanImage; + *components = TextureCreateImage(ctx, image, memory, MakeImageParams(info, storage)); + return image; +} + +void CreateTextureViews(GraphicContext* ctx, GpuTextureVulkanImage* image, const ImageInfo& info, + bool storage, VkComponentMapping components) { + if (storage) { + TextureCreateImageViews(ctx, image, components, info.type, 0, 0, 1, info.depth, false, + TextureFormatUsage::Sampled | TextureFormatUsage::Storage); + } else { + TextureCreateImageViews(ctx, image, components, info.type, info.base_array, info.base_level, + info.view_levels, info.depth, true, TextureFormatUsage::Sampled); + } +} + +void UploadRenderTargetLayers(GraphicContext* ctx, RenderTextureVulkanImage* image, + const RenderTargetInfo& info, uint32_t base_layer, + uint32_t layer_count, bool refresh) { + if (info.layers == 0 || info.size % info.layers != 0 || layer_count == 0 || + base_layer >= info.layers || layer_count > info.layers - base_layer || image == nullptr || + base_layer >= image->layers || layer_count > image->layers - base_layer) { + EXIT("TextureCache: invalid render-target layer upload, base=%u count=%u " + "info_layers=%u image_layers=%u size=0x%016" PRIx64 "\n", + base_layer, layer_count, info.layers, image != nullptr ? image->layers : 0, info.size); + } + if (refresh) { + VulkanDeviceWaitIdle(ctx); + } + const auto slice_size = info.size / info.layers; + const auto upload_size = slice_size * layer_count; + const bool standard64 = IsSupportedStandard64RenderTarget(info); + if (standard64 || info.levels > 1 || info.layers > 1) { + const auto format = RenderTargetTransferFormat(info.bytes_per_element); + auto layout = TextureCalcUploadLayout(format, info.width, info.height, info.levels, + layer_count, info.pitch, info.tile_mode, upload_size, + false, false, false, "TextureCache render target"); + const bool render_target_tiled = + info.tile_mode == Prospero::GpuEnumValue(Prospero::TileMode::kRenderTarget); + if (!standard64 && ((render_target_tiled && !layout.fmt_tiled_render_target) || + layout.pitch != info.pitch)) { + EXIT("TextureCache: unsupported render-target mip upload layout, pitch=%u/%u tile=%u\n", + info.pitch, layout.pitch, info.tile_mode); + } + auto regions = TextureBuildUploadRegions( + layout, info.format, info.width, info.height, layer_count, info.levels, true, false, + TextureUploadDestination::MipLevels, TextureUploadSliceLayout::MipChainPerSlice); + for (auto& region: regions) { + region.dst_layer += base_layer; + } + const auto source_address = info.address + slice_size * base_layer; + TextureUploadGuestImage( + ctx, image, reinterpret_cast(source_address), upload_size, regions, layout, + format, info.width, info.height, layer_count, info.levels, + TextureUploadSliceLayout::MipChainPerSlice, "TextureCache render target", + static_cast(VK_IMAGE_LAYOUT_GENERAL)); + return; + } + if (info.tile_mode == Prospero::GpuEnumValue(Prospero::TileMode::kRenderTarget) && + UtilBufferIsTiled(info.address, slice_size)) { + UtilScratchBuffer scratch(slice_size); + TileConvertTiledToLinearRenderTarget( + scratch.Data(), reinterpret_cast(info.address), info.width, info.height, + info.pitch, info.bytes_per_element, slice_size); + UtilFillImage(ctx, image, scratch.Data(), slice_size, info.pitch, + static_cast(VK_IMAGE_LAYOUT_GENERAL)); + } else { + UtilFillImage(ctx, image, reinterpret_cast(info.address), slice_size, + info.pitch, static_cast(VK_IMAGE_LAYOUT_GENERAL)); + } +} + +void UploadRenderTarget(GraphicContext* ctx, RenderTextureVulkanImage* image, + const RenderTargetInfo& info, bool refresh) { + UploadRenderTargetLayers(ctx, image, info, 0, info.layers, refresh); +} + +RenderTextureVulkanImage* CreateRenderTarget(GraphicContext* ctx, const RenderTargetInfo& info, + VulkanMemory* memory) { + auto* image = new RenderTextureVulkanImage; + image->extent.width = info.width; + image->extent.height = info.height; + image->format = info.format; + image->mip_levels = info.levels; + image->layers = info.layers; + image->layout = VK_IMAGE_LAYOUT_UNDEFINED; + UtilResetImageViews(image); + VkImageCreateInfo create {}; + create.sType = VK_STRUCTURE_TYPE_IMAGE_CREATE_INFO; + create.flags = RenderTargetCreateFlags(info.format); + create.imageType = VK_IMAGE_TYPE_2D; + create.extent = {info.width, info.height, 1}; + create.mipLevels = info.levels; + create.arrayLayers = info.layers; + create.format = info.format; + create.tiling = VK_IMAGE_TILING_OPTIMAL; + create.initialLayout = VK_IMAGE_LAYOUT_UNDEFINED; + create.usage = RenderTargetUsage(ctx, info.format, create.flags); + create.sharingMode = VK_SHARING_MODE_EXCLUSIVE; + create.samples = VK_SAMPLE_COUNT_1_BIT; + memory->property = VK_MEMORY_PROPERTY_DEVICE_LOCAL_BIT; + if (!VulkanCreateImage(ctx, &create, image, memory)) { + EXIT("TextureCache: failed to create render target, addr=0x%016" PRIx64 + " extent=%ux%u format=%d\n", + info.address, info.width, info.height, static_cast(info.format)); + } + image->memory = *memory; + ImageViewOps::CreateRenderTargetViews(ctx, image); + return image; +} + +DepthStencilVulkanImage* CreateDepthTarget(GraphicContext* ctx, const DepthTargetInfo& info, + VulkanMemory* memory) { + VkImageCreateInfo create {}; + create.sType = VK_STRUCTURE_TYPE_IMAGE_CREATE_INFO; + create.imageType = VK_IMAGE_TYPE_2D; + create.extent = {info.width, info.height, 1}; + create.mipLevels = 1; + create.arrayLayers = info.layers; + create.format = info.format; + create.tiling = VK_IMAGE_TILING_OPTIMAL; + create.initialLayout = VK_IMAGE_LAYOUT_UNDEFINED; + create.usage = DepthTargetImageUsage(); + create.sharingMode = VK_SHARING_MODE_EXCLUSIVE; + create.samples = VK_SAMPLE_COUNT_1_BIT; + VkImageFormatProperties properties {}; + if (ctx->GetImageFormatProperties(info.format, VK_IMAGE_TYPE_2D, VK_IMAGE_TILING_OPTIMAL, + create.usage, 0, &properties) != VK_SUCCESS) { + EXIT("TextureCache: depth format does not support required usage, format=%d usage=0x%x\n", + static_cast(info.format), create.usage); + } + auto* image = new DepthStencilVulkanImage; + image->extent.width = info.width; + image->extent.height = info.height; + image->guest_pitch = info.pitch; + image->layers = info.layers; + image->format = info.format; + image->layout = VK_IMAGE_LAYOUT_UNDEFINED; + image->compressed = false; + UtilResetImageViews(image); + memory->property = VK_MEMORY_PROPERTY_DEVICE_LOCAL_BIT; + if (!VulkanCreateImage(ctx, &create, image, memory)) { + EXIT("TextureCache: failed to create depth target, addr=0x%016" PRIx64 + " extent=%ux%u format=%d\n", + info.address, info.width, info.height, static_cast(info.format)); + } + image->memory = *memory; + ImageViewOps::CreateDepthViews(ctx, image); + return image; +} + +void ValidateVideoOut(GraphicContext* ctx, const VideoOutInfo& info) { + const auto compression = + ClassifyVideoOutCompression(info.compression != VideoOutCompression::Uncompressed, + info.metadata_address, info.dcc_control, 0); + const bool metadata_invalid = compression == VideoOutCompression::Dcc256_64_64 && + (info.metadata_address >= TRACKER_ADDRESS_SIZE || + (info.metadata_address >= info.address && + info.metadata_address < info.address + info.size)); + if (ctx == nullptr || info.address == 0 || info.size == 0 || + info.address >= TRACKER_ADDRESS_SIZE || info.size > TRACKER_ADDRESS_SIZE - info.address || + (info.address & 0xffffu) != 0 || info.width == 0 || info.height == 0 || + info.width > 16384 || info.height > 16384 || info.pitch < info.width || + info.tile_mode != Prospero::GpuEnumValue(Prospero::TileMode::kRenderTarget) || + compression == VideoOutCompression::Unsupported || compression != info.compression || + metadata_invalid || !IsSupportedVideoOutFormat(info)) { + EXIT("TextureCache: unsupported video-out surface, ctx=%p addr=0x%016" PRIx64 + " size=0x%016" PRIx64 " metadata=0x%016" PRIx64 " dcc=0x%08" PRIx32 + " extent=%ux%u pitch=%u tile=%u guest_format=%u bpe=%u vk_format=%d\n", + static_cast(ctx), info.address, info.size, info.metadata_address, + info.dcc_control, info.width, info.height, info.pitch, info.tile_mode, + info.guest_format, info.bytes_per_element, static_cast(info.format)); + } + TileSizeAlign exact {}; + TileGetTextureTotalSize(info.guest_format, info.width, info.height, 1, info.pitch, 1, + info.tile_mode, false, &exact); + if (exact.align != 65536 || exact.size != info.size || + TileGetTexturePitch(info.guest_format, info.width, 1, info.tile_mode) != info.pitch) { + EXIT("TextureCache: video-out tile layout mismatch, addr=0x%016" PRIx64 + " size=0x%016" PRIx64 " expected_size=0x%016" PRIx64 " align=0x%016" PRIx64 + " pitch=%u\n", + info.address, info.size, exact.size, exact.align, info.pitch); + } + (void)RenderTargetUsage(ctx, info.format, 0); +} + +VideoOutVulkanImage* CreateVideoOut(GraphicContext* ctx, const VideoOutInfo& info, + VulkanMemory* memory) { + auto* image = new VideoOutVulkanImage; + image->extent.width = info.width; + image->extent.height = info.height; + image->format = info.format; + image->layout = VK_IMAGE_LAYOUT_UNDEFINED; + UtilResetImageViews(image); + VkImageCreateInfo create {}; + create.sType = VK_STRUCTURE_TYPE_IMAGE_CREATE_INFO; + create.imageType = VK_IMAGE_TYPE_2D; + create.extent = {info.width, info.height, 1}; + create.mipLevels = 1; + create.arrayLayers = 1; + create.format = info.format; + create.tiling = VK_IMAGE_TILING_OPTIMAL; + create.initialLayout = VK_IMAGE_LAYOUT_UNDEFINED; + create.flags = RenderTargetCreateFlags(info.format); + create.usage = RenderTargetUsage(ctx, info.format, create.flags); + create.sharingMode = VK_SHARING_MODE_EXCLUSIVE; + create.samples = VK_SAMPLE_COUNT_1_BIT; + memory->property = VK_MEMORY_PROPERTY_DEVICE_LOCAL_BIT; + if (!VulkanCreateImage(ctx, &create, image, memory)) { + EXIT("TextureCache: failed to create video-out image, addr=0x%016" PRIx64 + " extent=%ux%u format=%d\n", + info.address, info.width, info.height, static_cast(info.format)); + } + image->memory = *memory; + ImageViewOps::CreateVideoOutViews(ctx, image); + return image; +} + +void UploadVideoOut(GraphicContext* ctx, VideoOutVulkanImage* image, const VideoOutInfo& info, + bool refresh) { + if (info.compression != VideoOutCompression::Uncompressed) { + EXIT("TextureCache: compressed video-out guest upload is unsupported, " + "addr=0x%016" PRIx64 " metadata=0x%016" PRIx64 " dcc=0x%08" PRIx32 "\n", + info.address, info.metadata_address, info.dcc_control); + } + if (refresh) { + VulkanDeviceWaitIdle(ctx); + } + image->layout = VK_IMAGE_LAYOUT_UNDEFINED; + UtilScratchBuffer scratch(info.size); + TileConvertTiledToLinearRenderTarget( + scratch.Data(), reinterpret_cast(info.address), info.width, info.height, + info.pitch, info.bytes_per_element, info.size); + if (info.bgra16) { + auto* pixels = static_cast(scratch.Data()); + for (uint64_t i = 0; i < info.size / sizeof(uint16_t); i += 4) { + std::swap(pixels[i], pixels[i + 2]); + } + } + UtilFillImage(ctx, image, scratch.Data(), info.size, info.pitch, + static_cast(VK_IMAGE_LAYOUT_GENERAL)); +} + +GpuTextureVulkanImage* CreateDummyTexture(GraphicContext* ctx, bool uint_format, bool image_3d, + bool storage, VulkanMemory* memory) { + if (memory == nullptr || memory->allocation != nullptr) { + EXIT("TextureCache: invalid dummy texture memory slot, slot=%p allocation=%p storage=%d\n", + static_cast(memory), + memory == nullptr ? nullptr : static_cast(memory->allocation), storage); + } + auto* image = storage ? static_cast(new StorageTextureVulkanImage) + : new TextureVulkanImage; + auto usage = storage ? TextureFormatUsage::Storage : TextureFormatUsage::Sampled; + auto layout = storage ? VK_IMAGE_LAYOUT_GENERAL : VK_IMAGE_LAYOUT_SHADER_READ_ONLY_OPTIMAL; + auto owner = storage ? "DummyStorageTexture" : "DummySampledTexture"; + + auto params = MakeDummyTextureParams(uint_format, image_3d, usage, owner); + auto components = TextureCreateImage(ctx, image, memory, params); + + static constexpr uint32_t zero = 0; + UtilFillImage(ctx, image, &zero, sizeof(zero), 1, static_cast(layout)); + TextureCreateImageViews(ctx, image, components, params.type, 0, params.base_level, + params.levels, params.depth, params.allow_cube_view, params.view_usage); + return image; +} + +void Destroy(GraphicContext* ctx, GpuTextureVulkanImage* image, VulkanMemory* memory) { + KYTY_PROFILER_BLOCK("TextureCache::DeleteGpuTexture"); + + EXIT_IF(ctx == nullptr); + EXIT_IF(image == nullptr); + + ImageViewOps::DestroyViews(ctx, image); + VulkanDeleteImage(ctx, image, memory); + + switch (image->type) { + case VulkanImageType::Texture: delete static_cast(image); break; + case VulkanImageType::StorageTexture: + delete static_cast(image); + break; + default: EXIT("unsupported gpu texture image type: %d\n", static_cast(image->type)); + } +} + +void Destroy(GraphicContext* ctx, RenderTextureVulkanImage* image, VulkanMemory* memory) { + KYTY_PROFILER_BLOCK("TextureCache::DeleteRenderTexture"); + + EXIT_IF(ctx == nullptr); + EXIT_IF(image == nullptr); + + g_render_ctx->GetFramebufferCache()->FreeFramebufferByColor(image); + ImageViewOps::DestroyViews(ctx, image); + VulkanDeleteImage(ctx, image, memory); + delete image; +} + +void Destroy(GraphicContext* ctx, DepthStencilVulkanImage* image, VulkanMemory* memory) { + KYTY_PROFILER_BLOCK("TextureCache::DeleteDepthStencil"); + + EXIT_IF(ctx == nullptr); + EXIT_IF(image == nullptr); + + g_render_ctx->GetFramebufferCache()->FreeFramebufferByDepth(image); + ImageViewOps::DestroyViews(ctx, image); + VulkanDeleteImage(ctx, image, memory); + delete image; +} + +void Destroy(GraphicContext* ctx, VideoOutVulkanImage* image, VulkanMemory* memory) { + KYTY_PROFILER_BLOCK("TextureCache::DeleteVideoOut"); + + EXIT_IF(ctx == nullptr); + EXIT_IF(image == nullptr); + + g_render_ctx->GetFramebufferCache()->FreeFramebufferByColor(image); + ImageViewOps::DestroyViews(ctx, image); + VulkanDeleteImage(ctx, image, memory); + delete image; +} + +} // namespace ImageOps + +} // namespace Libs::Graphics diff --git a/src/graphics/host_gpu/renderer/image.h b/src/graphics/host_gpu/renderer/image.h index 5a506c85e..ea49c1d46 100644 --- a/src/graphics/host_gpu/renderer/image.h +++ b/src/graphics/host_gpu/renderer/image.h @@ -10,6 +10,13 @@ namespace Libs::Graphics { +struct DepthStencilVulkanImage; +struct GpuTextureVulkanImage; +struct GraphicContext; +struct RenderTextureVulkanImage; +struct VideoOutVulkanImage; +struct VulkanMemory; + struct Image final: ImageInfo { Image& operator=(const ImageInfo& value) { if (IsCpuDirty()) { @@ -91,6 +98,43 @@ private: uint64_t m_maybe_cpu_hash = 0; }; +namespace ImageOps { + +[[nodiscard]] GpuTextureVulkanImage* +CreateTexture(GraphicContext* ctx, const ImageInfo& info, bool storage, VulkanMemory* memory, + VkComponentMapping* components); +void CreateTextureViews(GraphicContext* ctx, GpuTextureVulkanImage* image, const ImageInfo& info, + bool storage, VkComponentMapping components); + +[[nodiscard]] RenderTextureVulkanImage* +CreateRenderTarget(GraphicContext* ctx, const RenderTargetInfo& info, VulkanMemory* memory); +[[nodiscard]] uint32_t RenderTargetTransferFormat(uint32_t bytes_per_element); +void UploadRenderTargetLayers(GraphicContext* ctx, RenderTextureVulkanImage* image, + const RenderTargetInfo& info, uint32_t base_layer, + uint32_t layer_count, bool refresh); +void UploadRenderTarget(GraphicContext* ctx, RenderTextureVulkanImage* image, + const RenderTargetInfo& info, bool refresh); + +[[nodiscard]] DepthStencilVulkanImage* +CreateDepthTarget(GraphicContext* ctx, const DepthTargetInfo& info, VulkanMemory* memory); + +void ValidateVideoOut(GraphicContext* ctx, const VideoOutInfo& info); +[[nodiscard]] VideoOutVulkanImage* +CreateVideoOut(GraphicContext* ctx, const VideoOutInfo& info, VulkanMemory* memory); +void UploadVideoOut(GraphicContext* ctx, VideoOutVulkanImage* image, const VideoOutInfo& info, + bool refresh); + +[[nodiscard]] GpuTextureVulkanImage* +CreateDummyTexture(GraphicContext* ctx, bool uint_format, bool image_3d, bool storage, + VulkanMemory* memory); + +void Destroy(GraphicContext* ctx, GpuTextureVulkanImage* image, VulkanMemory* memory); +void Destroy(GraphicContext* ctx, RenderTextureVulkanImage* image, VulkanMemory* memory); +void Destroy(GraphicContext* ctx, DepthStencilVulkanImage* image, VulkanMemory* memory); +void Destroy(GraphicContext* ctx, VideoOutVulkanImage* image, VulkanMemory* memory); + +} // namespace ImageOps + struct ImageRetirementRange { uint64_t address = 0; uint64_t size = 0; diff --git a/src/graphics/host_gpu/renderer/imageView.cpp b/src/graphics/host_gpu/renderer/imageView.cpp new file mode 100644 index 000000000..f48fc0447 --- /dev/null +++ b/src/graphics/host_gpu/renderer/imageView.cpp @@ -0,0 +1,369 @@ +#include "graphics/host_gpu/renderer/imageView.h" + +#include "common/assert.h" +#include "graphics/host_gpu/objects/textureCommon.h" +#include "graphics/host_gpu/renderer/textureCache.h" +#include "graphics/host_gpu/utils.h" + +#include + +namespace Libs::Graphics { + +namespace { + +void CreateRenderTargetView(GraphicContext* ctx, VulkanImage* image, int index, + VkComponentSwizzle r, VkComponentSwizzle g, VkComponentSwizzle b, + VkComponentSwizzle a, VkImageViewType type = VK_IMAGE_VIEW_TYPE_2D, + VkFormat view_format = VK_FORMAT_UNDEFINED, + VkImageUsageFlags view_usage = 0, uint32_t level_count = 0) { + const auto layer_count = type == VK_IMAGE_VIEW_TYPE_2D_ARRAY ? image->layers : 1u; + UtilCreateImageView(ctx, image, index, type, VK_IMAGE_ASPECT_COLOR_BIT, {r, g, b, a}, 0, 0, + layer_count, level_count == 0 ? image->mip_levels : level_count, + view_format, view_usage); +} + +} // namespace + +namespace ImageViewOps { + +VkImageAspectFlags DepthAspectMask(VkFormat format) noexcept { + return VK_IMAGE_ASPECT_DEPTH_BIT | + (format == VK_FORMAT_D16_UNORM_S8_UINT || format == VK_FORMAT_D24_UNORM_S8_UINT || + format == VK_FORMAT_D32_SFLOAT_S8_UINT + ? VK_IMAGE_ASPECT_STENCIL_BIT + : 0u); +} + +bool FormatSupportsStorage(GraphicContext* ctx, VkFormat format) { + const auto properties = ctx->GetFormatProperties(format); + return (properties.optimalTilingFeatures & VK_FORMAT_FEATURE_STORAGE_IMAGE_BIT) != 0; +} + +void CreateRenderTargetViews(GraphicContext* ctx, RenderTextureVulkanImage* image) { + CreateRenderTargetView(ctx, image, VulkanImage::VIEW_DEFAULT, VK_COMPONENT_SWIZZLE_IDENTITY, + VK_COMPONENT_SWIZZLE_IDENTITY, VK_COMPONENT_SWIZZLE_IDENTITY, + VK_COMPONENT_SWIZZLE_IDENTITY); + if (image->layers > 1) { + CreateRenderTargetView(ctx, image, VulkanImage::VIEW_DEFAULT_ARRAY, + VK_COMPONENT_SWIZZLE_IDENTITY, VK_COMPONENT_SWIZZLE_IDENTITY, + VK_COMPONENT_SWIZZLE_IDENTITY, VK_COMPONENT_SWIZZLE_IDENTITY, + VK_IMAGE_VIEW_TYPE_2D_ARRAY); + } + if (FormatSupportsStorage(ctx, image->format)) { + CreateRenderTargetView(ctx, image, VulkanImage::VIEW_STORAGE, VK_COMPONENT_SWIZZLE_IDENTITY, + VK_COMPONENT_SWIZZLE_IDENTITY, VK_COMPONENT_SWIZZLE_IDENTITY, + VK_COMPONENT_SWIZZLE_IDENTITY, VK_IMAGE_VIEW_TYPE_2D, + VK_FORMAT_UNDEFINED, 0, 1); + if (image->layers > 1) { + CreateRenderTargetView(ctx, image, VulkanImage::VIEW_STORAGE_ARRAY, + VK_COMPONENT_SWIZZLE_IDENTITY, VK_COMPONENT_SWIZZLE_IDENTITY, + VK_COMPONENT_SWIZZLE_IDENTITY, VK_COMPONENT_SWIZZLE_IDENTITY, + VK_IMAGE_VIEW_TYPE_2D_ARRAY, VK_FORMAT_UNDEFINED, 0, 1); + } + } +} + +void CreateDepthViews(GraphicContext* ctx, DepthStencilVulkanImage* image) { + UtilCreateImageView(ctx, image, VulkanImage::VIEW_DEFAULT, VK_IMAGE_VIEW_TYPE_2D, + DepthAspectMask(image->format), + {VK_COMPONENT_SWIZZLE_IDENTITY, VK_COMPONENT_SWIZZLE_IDENTITY, + VK_COMPONENT_SWIZZLE_IDENTITY, VK_COMPONENT_SWIZZLE_IDENTITY}, + 0, 0, 1, 1); +} + +void CreateVideoOutViews(GraphicContext* ctx, VideoOutVulkanImage* image) { + CreateRenderTargetView(ctx, image, VulkanImage::VIEW_DEFAULT, VK_COMPONENT_SWIZZLE_IDENTITY, + VK_COMPONENT_SWIZZLE_IDENTITY, VK_COMPONENT_SWIZZLE_IDENTITY, + VK_COMPONENT_SWIZZLE_IDENTITY); + if ((image->format == VK_FORMAT_R8G8B8A8_SRGB || image->format == VK_FORMAT_B8G8R8A8_SRGB) && + FormatSupportsStorage(ctx, VK_FORMAT_R8G8B8A8_UINT)) { + CreateRenderTargetView(ctx, image, VulkanImage::VIEW_STORAGE, VK_COMPONENT_SWIZZLE_IDENTITY, + VK_COMPONENT_SWIZZLE_IDENTITY, VK_COMPONENT_SWIZZLE_IDENTITY, + VK_COMPONENT_SWIZZLE_IDENTITY, VK_IMAGE_VIEW_TYPE_2D, + VK_FORMAT_R8G8B8A8_UINT, VK_IMAGE_USAGE_STORAGE_BIT, 1); + } +} + +void DestroyViews(GraphicContext* ctx, VulkanImage* image) { + EXIT_IF(ctx == nullptr); + EXIT_IF(image == nullptr); + + for (auto& cached: image->view_cache.views) { + if (cached.view != nullptr) { + vkDestroyImageView(ctx->device, cached.view, nullptr); + cached.view = nullptr; + } + } + image->view_cache.views.clear(); + for (auto& view: image->image_view) { + if (view != nullptr) { + vkDestroyImageView(ctx->device, view, nullptr); + view = nullptr; + } + } +} + +} // namespace ImageViewOps + +VkImageView TextureCache::GetRenderTargetAttachmentView(GraphicContext* ctx, + RenderTextureVulkanImage* image, + VkFormat format, uint32_t level, + uint32_t base_layer, uint32_t layer_count) { + if (ctx == nullptr || image == nullptr || image->image == nullptr || + format == VK_FORMAT_UNDEFINED || level >= image->mip_levels || level >= 16 || + layer_count == 0 || base_layer >= image->layers || + layer_count > image->layers - base_layer) { + EXIT("TextureCache: invalid render-target attachment view, image=%p format=%d" + " level=%u image_levels=%u base_layer=%u layer_count=%u image_layers=%u\n", + static_cast(image), static_cast(format), level, + image != nullptr ? image->mip_levels : 0, base_layer, layer_count, + image != nullptr ? image->layers : 0); + } + if (format != image->format && !IsRgba8SrgbReinterpretation(image->format, format)) { + EXIT("TextureCache: incompatible render-target attachment view, image_format=%d" + " view_format=%d level=%u\n", + static_cast(image->format), static_cast(format), level); + } + + return GetImageView( + ctx, image, + {format, layer_count == 1 ? VK_IMAGE_VIEW_TYPE_2D : VK_IMAGE_VIEW_TYPE_2D_ARRAY, + VK_IMAGE_ASPECT_COLOR_BIT, level, 1, base_layer, layer_count, DstSel(4, 5, 6, 7), + VK_IMAGE_USAGE_COLOR_ATTACHMENT_BIT}); +} + +VkImageView TextureCache::GetDepthTargetAttachmentView(GraphicContext* ctx, + DepthStencilVulkanImage* image, + uint32_t base_layer, uint32_t layer_count) { + if (ctx == nullptr || image == nullptr || image->image == nullptr || layer_count == 0 || + base_layer >= image->layers || layer_count > image->layers - base_layer) { + EXIT("TextureCache: invalid depth-target attachment view, image=%p base_layer=%u " + "layer_count=%u image_layers=%u\n", + static_cast(image), base_layer, layer_count, + image != nullptr ? image->layers : 0); + } + return GetImageView( + ctx, image, + {image->format, layer_count == 1 ? VK_IMAGE_VIEW_TYPE_2D : VK_IMAGE_VIEW_TYPE_2D_ARRAY, + ImageViewOps::DepthAspectMask(image->format), 0, 1, base_layer, layer_count, + DstSel(4, 5, 6, 7), VK_IMAGE_USAGE_DEPTH_STENCIL_ATTACHMENT_BIT}); +} + +VkImageView TextureCache::GetImageView(GraphicContext* ctx, VulkanImage* image, + const ImageViewInfo& info) { + const bool supported_type = info.type == VK_IMAGE_VIEW_TYPE_2D || + info.type == VK_IMAGE_VIEW_TYPE_2D_ARRAY || + info.type == VK_IMAGE_VIEW_TYPE_3D; + const bool supported_usage = info.usage == VK_IMAGE_USAGE_SAMPLED_BIT || + info.usage == VK_IMAGE_USAGE_STORAGE_BIT || + info.usage == VK_IMAGE_USAGE_COLOR_ATTACHMENT_BIT || + info.usage == VK_IMAGE_USAGE_DEPTH_STENCIL_ATTACHMENT_BIT; + const bool valid_shape = + (info.type == VK_IMAGE_VIEW_TYPE_2D && info.layer_count == 1) || + info.type == VK_IMAGE_VIEW_TYPE_2D_ARRAY || + (info.type == VK_IMAGE_VIEW_TYPE_3D && info.base_layer == 0 && info.layer_count == 1); + if (ctx == nullptr || image == nullptr || image->image == nullptr || + info.format == VK_FORMAT_UNDEFINED || info.aspect == 0 || info.level_count == 0 || + info.base_level >= (image != nullptr ? image->mip_levels : 0) || + info.level_count > image->mip_levels - info.base_level || info.layer_count == 0 || + info.base_layer >= image->layers || info.layer_count > image->layers - info.base_layer || + !supported_type || !valid_shape || !supported_usage) { + EXIT("TextureCache: invalid dynamic image view, image=%p format=%d aspect=0x%x" + " swizzle=0x%03x mip=%u+%u layer=%u+%u type=%d usage=0x%x" + " image_levels=%u image_layers=%u\n", + static_cast(image), static_cast(info.format), info.aspect, + info.swizzle, info.base_level, info.level_count, info.base_layer, info.layer_count, + static_cast(info.type), info.usage, image != nullptr ? image->mip_levels : 0, + image != nullptr ? image->layers : 0); + } + + auto& cache = image->view_cache; + std::lock_guard lock(cache.mutex); + for (const auto& cached: cache.views) { + if (cached.info == info) { + return cached.view; + } + } + + VkImageViewUsageCreateInfo usage {}; + usage.sType = VK_STRUCTURE_TYPE_IMAGE_VIEW_USAGE_CREATE_INFO; + usage.usage = info.usage; + VkImageViewCreateInfo create {}; + create.sType = VK_STRUCTURE_TYPE_IMAGE_VIEW_CREATE_INFO; + create.pNext = &usage; + create.image = image->image; + create.viewType = info.type; + create.format = info.format; + create.components = info.usage == VK_IMAGE_USAGE_SAMPLED_BIT + ? TextureGetComponentMapping(info.swizzle) + : VkComponentMapping {}; + create.subresourceRange.aspectMask = info.aspect; + create.subresourceRange.baseMipLevel = info.base_level; + create.subresourceRange.levelCount = info.level_count; + create.subresourceRange.baseArrayLayer = info.base_layer; + create.subresourceRange.layerCount = info.layer_count; + VkImageView view = nullptr; + const auto result = vkCreateImageView(ctx->device, &create, nullptr, &view); + if (result != VK_SUCCESS || view == nullptr) { + EXIT("TextureCache: failed to create dynamic image view, result=%d format=%d" + " aspect=0x%x swizzle=0x%03x mip=%u+%u layer=%u+%u type=%d usage=0x%x\n", + static_cast(result), static_cast(info.format), info.aspect, info.swizzle, + info.base_level, info.level_count, info.base_layer, info.layer_count, + static_cast(info.type), info.usage); + } + cache.views.push_back({info, view}); + return view; +} + +VkImageView TextureCache::GetDepthTargetSampledView(GraphicContext* ctx, + DepthStencilVulkanImage* image, + VkFormat view_format, uint32_t swizzle, + uint32_t base_level, uint32_t level_count, + VkImageViewType type, uint32_t base_layer, + uint32_t layer_count) { + if (ctx == nullptr || image == nullptr || image->image == nullptr || + view_format == VK_FORMAT_UNDEFINED || + !IsSupportedSampledDepthView(image->format, view_format, swizzle)) { + EXIT("TextureCache: invalid sampled depth-target view, image=%p image_format=%d" + " view_format=%d swizzle=0x%03x mip=%u+%u layer=%u+%u type=%d" + " image_levels=%u image_layers=%u\n", + static_cast(image), + image != nullptr ? static_cast(image->format) : VK_FORMAT_UNDEFINED, + static_cast(view_format), swizzle, base_level, level_count, base_layer, + layer_count, static_cast(type), image != nullptr ? image->mip_levels : 0, + image != nullptr ? image->layers : 0); + } + return GetImageView(ctx, image, + {image->format, type, VK_IMAGE_ASPECT_DEPTH_BIT, base_level, level_count, + base_layer, layer_count, swizzle}); +} + +VkImageView TextureCache::GetSampledColorView(GraphicContext* ctx, VulkanImage* image, + VkFormat view_format, uint32_t swizzle, + uint32_t base_level, uint32_t level_count, + VkImageViewType type, uint32_t base_layer, + uint32_t layer_count) { + if (ctx == nullptr || image == nullptr || image->image == nullptr || + view_format == VK_FORMAT_UNDEFINED || base_level >= 16 || + (type != VK_IMAGE_VIEW_TYPE_2D && type != VK_IMAGE_VIEW_TYPE_2D_ARRAY) || + !IsSupportedSampledColorView(image->format, view_format, swizzle)) { + EXIT("TextureCache: invalid sampled color view, image=%p swizzle=0x%03x" + " view_format=%d mip=%u+%u layer=%u+%u type=%d image_levels=%u image_layers=%u\n", + static_cast(image), swizzle, static_cast(view_format), base_level, + level_count, base_layer, layer_count, static_cast(type), + image != nullptr ? image->mip_levels : 0, image != nullptr ? image->layers : 0); + } + const auto precreated_view = type == VK_IMAGE_VIEW_TYPE_2D_ARRAY + ? VulkanImage::VIEW_DEFAULT_ARRAY + : VulkanImage::VIEW_DEFAULT; + const bool full_view = + base_level == 0 && level_count == image->mip_levels && base_layer == 0 && + layer_count == (type == VK_IMAGE_VIEW_TYPE_2D_ARRAY ? image->layers : 1u); + if (view_format == image->format && swizzle == DstSel(4, 5, 6, 7) && full_view && + image->image_view[precreated_view] != nullptr) { + return image->image_view[precreated_view]; + } + return GetImageView(ctx, image, + {view_format, type, VK_IMAGE_ASPECT_COLOR_BIT, base_level, level_count, + base_layer, layer_count, swizzle}); +} + +VkImageView TextureCache::GetRenderTargetStorageView(GraphicContext* ctx, + RenderTextureVulkanImage* image, + VkFormat view_format, uint32_t base_level, + uint32_t level_count, VkImageViewType type, + uint32_t base_layer, uint32_t layer_count) { + if (ctx == nullptr || image == nullptr || image->image == nullptr || + view_format == VK_FORMAT_UNDEFINED || + (type != VK_IMAGE_VIEW_TYPE_2D && type != VK_IMAGE_VIEW_TYPE_2D_ARRAY)) { + EXIT("TextureCache: invalid render-target storage view, image=%p view_format=%d" + " mip=%u+%u layer=%u+%u type=%d image_levels=%u image_layers=%u\n", + static_cast(image), static_cast(view_format), base_level, + level_count, base_layer, layer_count, static_cast(type), + image != nullptr ? image->mip_levels : 0, image != nullptr ? image->layers : 0); + } + const bool exact = view_format == image->format; + const bool compatible = view_format == BgraSrgbStorageViewFormat(image->format); + if (!exact && !compatible) { + EXIT("TextureCache: incompatible render-target storage view, image_format=%d" + " view_format=%d base=%u count=%u\n", + static_cast(image->format), static_cast(view_format), base_level, + level_count); + } + if (exact) { + const auto index = type == VK_IMAGE_VIEW_TYPE_2D_ARRAY ? VulkanImage::VIEW_STORAGE_ARRAY + : VulkanImage::VIEW_STORAGE; + const bool full_view = + base_level == 0 && level_count == 1 && base_layer == 0 && + layer_count == (type == VK_IMAGE_VIEW_TYPE_2D_ARRAY ? image->layers : 1u); + if (full_view && image->image_view[index] != nullptr) { + return image->image_view[index]; + } + } + + if (compatible && !ImageViewOps::FormatSupportsStorage(ctx, view_format)) { + EXIT("TextureCache: compatible render-target storage format lacks storage support," + " image_format=%d view_format=%d base=%u count=%u\n", + static_cast(image->format), static_cast(view_format), base_level, + level_count); + } + return GetImageView(ctx, image, + {view_format, type, VK_IMAGE_ASPECT_COLOR_BIT, base_level, level_count, + base_layer, layer_count, DstSel(4, 5, 6, 7), + VK_IMAGE_USAGE_STORAGE_BIT}); +} + +VkImageView TextureCache::GetStorageTextureSampledView(GraphicContext* ctx, + StorageTextureVulkanImage* image, + const ImageInfo& info) { + const auto shape = + SelectStorageSampledViewShape(info.type, info.depth, image != nullptr ? image->layers : 0); + if (ctx == nullptr || image == nullptr || image->image == nullptr || + shape == StorageSampledViewShape::Unsupported || info.base_array != 0 || + info.levels != image->mip_levels || info.base_level >= info.levels || + info.view_levels == 0 || info.base_level + info.view_levels > info.levels) { + EXIT("TextureCache: invalid sampled view of storage texture, image=%p type=%u depth=%u" + " base=%u levels=%u view_levels=%u image_levels=%u base_array=%u\n", + static_cast(image), info.type, info.depth, info.base_level, info.levels, + info.view_levels, image != nullptr ? image->mip_levels : 0, info.base_array); + } + const auto view_format = TextureGetFormat(info.format, TextureFormatUsage::Sampled); + if (view_format != image->format && !IsRgba8SrgbReinterpretation(image->format, view_format) && + !IsR32UintFloatReinterpretation(image->format, view_format)) { + EXIT("TextureCache: incompatible sampled view of storage texture, image_format=%d" + " view_format=%d swizzle=0x%03x\n", + static_cast(image->format), static_cast(view_format), info.swizzle); + } + + VkImageViewType type = VK_IMAGE_VIEW_TYPE_MAX_ENUM; + switch (shape) { + case StorageSampledViewShape::Image2D: type = VK_IMAGE_VIEW_TYPE_2D; break; + case StorageSampledViewShape::Image2DArray: type = VK_IMAGE_VIEW_TYPE_2D_ARRAY; break; + case StorageSampledViewShape::Image3D: type = VK_IMAGE_VIEW_TYPE_3D; break; + case StorageSampledViewShape::Unsupported: + EXIT("TextureCache: unsupported sampled storage-image view shape\n"); + } + const auto layer_count = shape == StorageSampledViewShape::Image2DArray ? info.depth : 1u; + return GetImageView(ctx, image, + {view_format, type, VK_IMAGE_ASPECT_COLOR_BIT, info.base_level, + info.view_levels, 0, layer_count, info.swizzle}); +} + +VkImageView TextureCache::GetStorageTextureStorageView(GraphicContext* ctx, + StorageTextureVulkanImage* image, + uint32_t base_level) { + if (ctx == nullptr || image == nullptr || image->image == nullptr || + base_level >= (image != nullptr ? image->mip_levels : 0)) { + EXIT("TextureCache: invalid storage-texture mip view, image=%p level=%u levels=%u\n", + static_cast(image), base_level, image != nullptr ? image->mip_levels : 0); + } + if (base_level == 0) { + return image->image_view[VulkanImage::VIEW_DEFAULT]; + } + return GetImageView(ctx, image, + {image->format, VK_IMAGE_VIEW_TYPE_2D, VK_IMAGE_ASPECT_COLOR_BIT, + base_level, 1, 0, 1, DstSel(4, 5, 6, 7), + VK_IMAGE_USAGE_STORAGE_BIT}); +} + +} // namespace Libs::Graphics diff --git a/src/graphics/host_gpu/renderer/imageView.h b/src/graphics/host_gpu/renderer/imageView.h index 99041fe4a..b563cd83b 100644 --- a/src/graphics/host_gpu/renderer/imageView.h +++ b/src/graphics/host_gpu/renderer/imageView.h @@ -214,6 +214,18 @@ ValidateStorageImageResource(const ShaderRecompiler::IR::ImageResource& resource } } +namespace ImageViewOps { + +[[nodiscard]] VkImageAspectFlags DepthAspectMask(VkFormat format) noexcept; +[[nodiscard]] bool FormatSupportsStorage(GraphicContext* ctx, VkFormat format); + +void CreateRenderTargetViews(GraphicContext* ctx, RenderTextureVulkanImage* image); +void CreateDepthViews(GraphicContext* ctx, DepthStencilVulkanImage* image); +void CreateVideoOutViews(GraphicContext* ctx, VideoOutVulkanImage* image); +void DestroyViews(GraphicContext* ctx, VulkanImage* image); + +} // namespace ImageViewOps + } // namespace Libs::Graphics #endif // EMULATOR_SRC_GRAPHICS_HOST_GPU_RENDERER_IMAGEVIEW_H_ diff --git a/src/graphics/host_gpu/renderer/render.h b/src/graphics/host_gpu/renderer/render.h index f67fb22df..fa889bdef 100644 --- a/src/graphics/host_gpu/renderer/render.h +++ b/src/graphics/host_gpu/renderer/render.h @@ -142,6 +142,7 @@ void RenderDispatchDirect(uint64_t submit_id, CommandBuffer* buffer, HW::Context void GraphicsRenderInit(); void GraphicsRenderCreateContext(); +void GraphicsRenderReleaseThreadCommandPools(); [[nodiscard]] bool ResolveComputeImageClear(const ShaderComputeInputInfo& input, uint32_t group_x, uint32_t group_y, uint32_t group_z, uint32_t mode, diff --git a/src/graphics/host_gpu/renderer/renderCompute.cpp b/src/graphics/host_gpu/renderer/renderCompute.cpp index be2355a90..8687b7878 100644 --- a/src/graphics/host_gpu/renderer/renderCompute.cpp +++ b/src/graphics/host_gpu/renderer/renderCompute.cpp @@ -267,7 +267,8 @@ static bool TryConsumeComputeMetaClear(const ShaderComputeInputInfo& input, cons continue; } if (resource.written || !resource.read || resource.atomic || descriptor.Base48() == 0 || - descriptor_size == 0 || cache->HasMetaOverlap(descriptor.Base48(), descriptor_size)) { + descriptor_size == 0 || + cache->QueryRegion(descriptor.Base48(), descriptor_size).metadata_pages) { EXIT("unsupported HTile clear side-buffer access\n"); } g_render_ctx->GetBufferCache()->ValidateGpuAccess(descriptor.Base48(), descriptor_size, @@ -349,9 +350,7 @@ static bool TryConsumeComputeImageClear(const ShaderComputeInputInfo& input, Com return false; } auto* cache = g_render_ctx->GetTextureCache(); - if (!cache->ClearColorImageFromBuffer(command, descriptor.Base48(), size, packed_clear) && - !cache->ClearDepthImageFromBuffer(command, descriptor.Base48(), size, packed_clear) && - !cache->ClearStencilImageFromBuffer(command, descriptor.Base48(), size, packed_clear)) { + if (!cache->ClearImageFromBuffer(command, descriptor.Base48(), size, packed_clear)) { return false; } static std::atomic logged_clears {0}; diff --git a/src/graphics/host_gpu/renderer/renderTargetBarriers.cpp b/src/graphics/host_gpu/renderer/renderTargetBarriers.cpp index 09d63cd16..dfa55ef37 100644 --- a/src/graphics/host_gpu/renderer/renderTargetBarriers.cpp +++ b/src/graphics/host_gpu/renderer/renderTargetBarriers.cpp @@ -217,7 +217,8 @@ void GraphicsRenderDepthStencilBarrier(CommandBuffer* buffer, uint64_t vaddr, ui Common::LockGuard lock(g_render_ctx->GetMutex()); auto* vk_buffer = buffer->GetPool()->buffers[buffer->GetIndex()]; - auto* native = g_render_ctx->GetTextureCache()->FindDepthTargetByRange(vaddr, size); + auto* native = + g_render_ctx->GetTextureCache()->FindDepthTargetByRange(buffer, vaddr, size); if (native == nullptr) { EXIT("depth-target barrier range has no cached image\n"); } diff --git a/src/graphics/host_gpu/renderer/textureCache.cpp b/src/graphics/host_gpu/renderer/textureCache.cpp index bad06fd7a..c25a02fee 100644 --- a/src/graphics/host_gpu/renderer/textureCache.cpp +++ b/src/graphics/host_gpu/renderer/textureCache.cpp @@ -11,7 +11,7 @@ #include "graphics/host_gpu/objects/label.h" #include "graphics/host_gpu/objects/textureCommon.h" #include "graphics/host_gpu/renderer/bufferCache.h" -#include "graphics/host_gpu/renderer/framebufferCache.h" +#include "graphics/host_gpu/renderer/dummyTextureCache.h" #include "graphics/host_gpu/renderer/image.h" #include "graphics/host_gpu/renderer/imageView.h" #include "graphics/host_gpu/renderer/render.h" @@ -19,7 +19,6 @@ #include "graphics/host_gpu/renderer/renderTargetBarriers.h" #include "graphics/host_gpu/renderer/resourceMutex.h" #include "graphics/host_gpu/utils.h" -#include "graphics/shader/shader.h" #include "kernel/memory.h" #include @@ -124,20 +123,16 @@ struct TextureCache::CachedImage { switch (kind) { case Kind::Texture: case Kind::StorageTexture: - TextureCache::DeleteGpuTexture(ctx, static_cast(image), - &memory); + ImageOps::Destroy(ctx, static_cast(image), &memory); break; case Kind::RenderTarget: - TextureCache::DeleteRenderTexture( - ctx, static_cast(image), &memory); + ImageOps::Destroy(ctx, static_cast(image), &memory); break; case Kind::DepthTarget: - TextureCache::DeleteDepthStencil(ctx, static_cast(image), - &memory); + ImageOps::Destroy(ctx, static_cast(image), &memory); break; case Kind::VideoOut: - TextureCache::DeleteVideoOut(ctx, static_cast(image), - &memory); + ImageOps::Destroy(ctx, static_cast(image), &memory); break; } image = nullptr; @@ -248,6 +243,21 @@ void TextureCache::UnregisterImageLocked(CachedImage& image, bool release_tracki image.registered = false; } +VulkanImage* TextureCache::PublishImage(CommandBuffer* command, + std::shared_ptr image) { + if (command == nullptr || image == nullptr || image->image == nullptr || image->registered) { + EXIT("TextureCache: invalid image publication, command=%p record=%p image=%p registered=%d\n", + static_cast(command), static_cast(image.get()), + image != nullptr ? static_cast(image->image) : nullptr, + image != nullptr && image->registered); + } + auto* result = image->image; + command->RetainResourceUntilFence(image); + m_images.push_back(std::move(image)); + RegisterImageLocked(*m_images.back()); + return result; +} + std::vector TextureCache::FindImagesInRegionLocked(uint64_t vaddr, uint64_t size, bool page_overlap) { return page_overlap ? m_image_owner_index.QueryCandidates(vaddr, size) @@ -829,406 +839,6 @@ void AppendLayerCopies(std::vector& regions, VulkanImage* source } } -VkImageAspectFlags DepthAspectMask(VkFormat format) { - return VK_IMAGE_ASPECT_DEPTH_BIT | - (format == VK_FORMAT_D16_UNORM_S8_UINT || format == VK_FORMAT_D24_UNORM_S8_UINT || - format == VK_FORMAT_D32_SFLOAT_S8_UINT - ? VK_IMAGE_ASPECT_STENCIL_BIT - : 0u); -} - -TextureImageCreateParams MakeImageParams(const ImageInfo& info, bool storage) { - TextureImageCreateParams params {}; - params.fmt = info.format; - params.width = info.width; - params.height = info.height; - params.base_level = SelectImageBackingBaseLevel(storage, info.base_level); - params.levels = info.levels; - params.depth = info.depth; - params.type = info.type; - // Storage image views use identity component mapping. The guest storage write mapping is - // validated before this point and intentionally does not become a Vulkan view swizzle. - params.swizzle = storage ? DstSel(4, 5, 6, 7) : info.swizzle; - params.format_usage = TextureFormatUsage::Sampled | TextureFormatUsage::Storage; - params.required_format_usage = storage - ? TextureFormatUsage::Sampled | TextureFormatUsage::Storage - : TextureFormatUsage::Sampled; - params.view_usage = storage ? TextureFormatUsage::Sampled | TextureFormatUsage::Storage - : TextureFormatUsage::Sampled; - params.image_layout = TextureUploadDestination::MipLevels; - params.allow_cube_view = !storage; - params.compatible_format_views = - storage && - (IsRgba8SrgbViewFormat(TextureGetFormat(info.format, params.format_usage)) || - info.format == Prospero::GpuEnumValue(Prospero::BufferFormat::k32UInt) || - info.format == Prospero::GpuEnumValue(Prospero::BufferFormat::k32Float)); - params.owner = storage ? "StorageTextureCache" : "TextureCache"; - return params; -} - -bool FormatSupportsStorage(GraphicContext* ctx, VkFormat format) { - const auto properties = ctx->GetFormatProperties(format); - return (properties.optimalTilingFeatures & VK_FORMAT_FEATURE_STORAGE_IMAGE_BIT) != 0; -} - -bool RenderTargetSupportsStorage(GraphicContext* ctx, VkFormat format, VkImageCreateFlags flags) { - const auto compatible = SrgbStorageViewFormat(format); - const auto required_flags = - VK_IMAGE_CREATE_MUTABLE_FORMAT_BIT | VK_IMAGE_CREATE_EXTENDED_USAGE_BIT; - const bool compatible_views = (flags & required_flags) == required_flags; - return FormatSupportsStorage(ctx, format) || - (compatible_views && compatible != VK_FORMAT_UNDEFINED && - FormatSupportsStorage(ctx, compatible)); -} - -VkImageCreateFlags RenderTargetCreateFlags(VkFormat format) { - const bool compatible_format_view = - IsRgba8SrgbViewFormat(format) || BgraToRgbaSampledViewFormat(format) != VK_FORMAT_UNDEFINED || - format == VK_FORMAT_R8G8B8A8_UINT || format == VK_FORMAT_R16G16B16A16_SFLOAT || - format == VK_FORMAT_R16G16B16A16_UINT; - return compatible_format_view - ? VK_IMAGE_CREATE_MUTABLE_FORMAT_BIT | VK_IMAGE_CREATE_EXTENDED_USAGE_BIT - : VkImageCreateFlags {0}; -} - -VkImageUsageFlags RenderTargetUsage(GraphicContext* ctx, VkFormat format, - VkImageCreateFlags flags) { - auto usage = static_cast(VK_IMAGE_USAGE_COLOR_ATTACHMENT_BIT) | - static_cast(VK_IMAGE_USAGE_TRANSFER_SRC_BIT) | - static_cast(VK_IMAGE_USAGE_TRANSFER_DST_BIT) | - static_cast(VK_IMAGE_USAGE_SAMPLED_BIT); - if (RenderTargetSupportsStorage(ctx, format, flags)) { - usage |= VK_IMAGE_USAGE_STORAGE_BIT; - } - VkImageFormatProperties properties {}; - if (ctx->GetImageFormatProperties(format, VK_IMAGE_TYPE_2D, VK_IMAGE_TILING_OPTIMAL, usage, - flags, &properties) != VK_SUCCESS) { - EXIT("TextureCache: render-target format does not support required usage, format=%d " - "usage=0x%x\n", - static_cast(format), usage); - } - return usage; -} - -void CreateRenderTargetView(GraphicContext* ctx, VulkanImage* image, int index, - VkComponentSwizzle r, VkComponentSwizzle g, VkComponentSwizzle b, - VkComponentSwizzle a, VkImageViewType type = VK_IMAGE_VIEW_TYPE_2D, - VkFormat view_format = VK_FORMAT_UNDEFINED, - VkImageUsageFlags view_usage = 0, uint32_t level_count = 0) { - const auto layer_count = type == VK_IMAGE_VIEW_TYPE_2D_ARRAY ? image->layers : 1u; - UtilCreateImageView(ctx, image, index, type, VK_IMAGE_ASPECT_COLOR_BIT, {r, g, b, a}, 0, 0, - layer_count, level_count == 0 ? image->mip_levels : level_count, - view_format, view_usage); -} - -void CreateRenderAttachmentViews(GraphicContext* ctx, RenderTextureVulkanImage* image) { - for (uint32_t level = 0; level < image->mip_levels; level++) { - VkImageViewCreateInfo create {}; - create.sType = VK_STRUCTURE_TYPE_IMAGE_VIEW_CREATE_INFO; - create.image = image->image; - create.viewType = VK_IMAGE_VIEW_TYPE_2D; - create.format = image->format; - create.components = {VK_COMPONENT_SWIZZLE_IDENTITY, VK_COMPONENT_SWIZZLE_IDENTITY, - VK_COMPONENT_SWIZZLE_IDENTITY, VK_COMPONENT_SWIZZLE_IDENTITY}; - create.subresourceRange.aspectMask = VK_IMAGE_ASPECT_COLOR_BIT; - create.subresourceRange.baseMipLevel = level; - create.subresourceRange.levelCount = 1; - create.subresourceRange.baseArrayLayer = 0; - create.subresourceRange.layerCount = 1; - const auto result = - vkCreateImageView(ctx->device, &create, nullptr, &image->render_view[level]); - if (result != VK_SUCCESS || image->render_view[level] == nullptr) { - EXIT("TextureCache: failed to create render-target mip view, result=%d level=%u\n", - static_cast(result), level); - } - } -} - -void CreateRenderTargetViews(GraphicContext* ctx, RenderTextureVulkanImage* image) { - CreateRenderTargetView(ctx, image, VulkanImage::VIEW_DEFAULT, VK_COMPONENT_SWIZZLE_IDENTITY, - VK_COMPONENT_SWIZZLE_IDENTITY, VK_COMPONENT_SWIZZLE_IDENTITY, - VK_COMPONENT_SWIZZLE_IDENTITY); - if (image->layers > 1) { - CreateRenderTargetView(ctx, image, VulkanImage::VIEW_DEFAULT_ARRAY, - VK_COMPONENT_SWIZZLE_IDENTITY, VK_COMPONENT_SWIZZLE_IDENTITY, - VK_COMPONENT_SWIZZLE_IDENTITY, VK_COMPONENT_SWIZZLE_IDENTITY, - VK_IMAGE_VIEW_TYPE_2D_ARRAY); - } - if (FormatSupportsStorage(ctx, image->format)) { - CreateRenderTargetView(ctx, image, VulkanImage::VIEW_STORAGE, VK_COMPONENT_SWIZZLE_IDENTITY, - VK_COMPONENT_SWIZZLE_IDENTITY, VK_COMPONENT_SWIZZLE_IDENTITY, - VK_COMPONENT_SWIZZLE_IDENTITY, VK_IMAGE_VIEW_TYPE_2D, - VK_FORMAT_UNDEFINED, 0, 1); - if (image->layers > 1) { - CreateRenderTargetView(ctx, image, VulkanImage::VIEW_STORAGE_ARRAY, - VK_COMPONENT_SWIZZLE_IDENTITY, VK_COMPONENT_SWIZZLE_IDENTITY, - VK_COMPONENT_SWIZZLE_IDENTITY, VK_COMPONENT_SWIZZLE_IDENTITY, - VK_IMAGE_VIEW_TYPE_2D_ARRAY, VK_FORMAT_UNDEFINED, 0, 1); - } - } - CreateRenderAttachmentViews(ctx, image); -} - -void CreateVideoOutViews(GraphicContext* ctx, VideoOutVulkanImage* image) { - CreateRenderTargetView(ctx, image, VulkanImage::VIEW_DEFAULT, VK_COMPONENT_SWIZZLE_IDENTITY, - VK_COMPONENT_SWIZZLE_IDENTITY, VK_COMPONENT_SWIZZLE_IDENTITY, - VK_COMPONENT_SWIZZLE_IDENTITY); - if ((image->format == VK_FORMAT_R8G8B8A8_SRGB || image->format == VK_FORMAT_B8G8R8A8_SRGB) && - FormatSupportsStorage(ctx, VK_FORMAT_R8G8B8A8_UINT)) { - CreateRenderTargetView(ctx, image, VulkanImage::VIEW_STORAGE, VK_COMPONENT_SWIZZLE_IDENTITY, - VK_COMPONENT_SWIZZLE_IDENTITY, VK_COMPONENT_SWIZZLE_IDENTITY, - VK_COMPONENT_SWIZZLE_IDENTITY, VK_IMAGE_VIEW_TYPE_2D, - VK_FORMAT_R8G8B8A8_UINT, VK_IMAGE_USAGE_STORAGE_BIT, 1); - } -} - -[[nodiscard]] uint32_t RenderTargetTransferFormat(uint32_t bytes_per_element) { - switch (bytes_per_element) { - case 1: return Prospero::GpuEnumValue(Prospero::BufferFormat::k8UNorm); - case 2: return Prospero::GpuEnumValue(Prospero::BufferFormat::k16UNorm); - case 4: return Prospero::GpuEnumValue(Prospero::BufferFormat::k32Float); - case 8: return Prospero::GpuEnumValue(Prospero::BufferFormat::k16_16_16_16Float); - case 16: return Prospero::GpuEnumValue(Prospero::BufferFormat::k32_32_32_32Float); - default: - EXIT("TextureCache: unsupported render-target element size: %u\n", bytes_per_element); - } -} - -void UploadRenderTargetLayers(GraphicContext* ctx, RenderTextureVulkanImage* image, - const RenderTargetInfo& info, uint32_t base_layer, - uint32_t layer_count, bool refresh) { - if (info.layers == 0 || info.size % info.layers != 0 || layer_count == 0 || - base_layer >= info.layers || layer_count > info.layers - base_layer || image == nullptr || - base_layer >= image->layers || layer_count > image->layers - base_layer) { - EXIT("TextureCache: invalid render-target layer upload, base=%u count=%u " - "info_layers=%u image_layers=%u size=0x%016" PRIx64 "\n", - base_layer, layer_count, info.layers, image != nullptr ? image->layers : 0, info.size); - } - if (refresh) { - VulkanDeviceWaitIdle(ctx); - } - const auto slice_size = info.size / info.layers; - const auto upload_size = slice_size * layer_count; - const bool standard64 = IsSupportedStandard64RenderTarget(info); - if (standard64 || info.levels > 1 || info.layers > 1) { - const auto format = RenderTargetTransferFormat(info.bytes_per_element); - auto layout = TextureCalcUploadLayout(format, info.width, info.height, info.levels, - layer_count, info.pitch, info.tile_mode, upload_size, - false, false, false, "TextureCache render target"); - const bool render_target_tiled = - info.tile_mode == Prospero::GpuEnumValue(Prospero::TileMode::kRenderTarget); - if (!standard64 && ((render_target_tiled && !layout.fmt_tiled_render_target) || - layout.pitch != info.pitch)) { - EXIT("TextureCache: unsupported render-target mip upload layout, pitch=%u/%u tile=%u\n", - info.pitch, layout.pitch, info.tile_mode); - } - auto regions = TextureBuildUploadRegions( - layout, info.format, info.width, info.height, layer_count, info.levels, true, false, - TextureUploadDestination::MipLevels, TextureUploadSliceLayout::MipChainPerSlice); - for (auto& region: regions) { - region.dst_layer += base_layer; - } - const auto source_address = info.address + slice_size * base_layer; - TextureUploadGuestImage( - ctx, image, reinterpret_cast(source_address), upload_size, regions, layout, - format, info.width, info.height, layer_count, info.levels, - TextureUploadSliceLayout::MipChainPerSlice, "TextureCache render target", - static_cast(VK_IMAGE_LAYOUT_GENERAL)); - return; - } - if (info.tile_mode == Prospero::GpuEnumValue(Prospero::TileMode::kRenderTarget) && - UtilBufferIsTiled(info.address, slice_size)) { - UtilScratchBuffer scratch(slice_size); - TileConvertTiledToLinearRenderTarget( - scratch.Data(), reinterpret_cast(info.address), info.width, info.height, - info.pitch, info.bytes_per_element, slice_size); - UtilFillImage(ctx, image, scratch.Data(), slice_size, info.pitch, - static_cast(VK_IMAGE_LAYOUT_GENERAL)); - } else { - UtilFillImage(ctx, image, reinterpret_cast(info.address), slice_size, - info.pitch, static_cast(VK_IMAGE_LAYOUT_GENERAL)); - } -} - -void UploadRenderTarget(GraphicContext* ctx, RenderTextureVulkanImage* image, - const RenderTargetInfo& info, bool refresh) { - UploadRenderTargetLayers(ctx, image, info, 0, info.layers, refresh); -} - -RenderTextureVulkanImage* CreateRenderTarget(GraphicContext* ctx, const RenderTargetInfo& info, - VulkanMemory* memory) { - auto* image = new RenderTextureVulkanImage; - image->extent.width = info.width; - image->extent.height = info.height; - image->format = info.format; - image->mip_levels = info.levels; - image->layers = info.layers; - image->layout = VK_IMAGE_LAYOUT_UNDEFINED; - UtilResetImageViews(image); - VkImageCreateInfo create {}; - create.sType = VK_STRUCTURE_TYPE_IMAGE_CREATE_INFO; - create.flags = RenderTargetCreateFlags(info.format); - create.imageType = VK_IMAGE_TYPE_2D; - create.extent = {info.width, info.height, 1}; - create.mipLevels = info.levels; - create.arrayLayers = info.layers; - create.format = info.format; - create.tiling = VK_IMAGE_TILING_OPTIMAL; - create.initialLayout = VK_IMAGE_LAYOUT_UNDEFINED; - create.usage = RenderTargetUsage(ctx, info.format, create.flags); - create.sharingMode = VK_SHARING_MODE_EXCLUSIVE; - create.samples = VK_SAMPLE_COUNT_1_BIT; - memory->property = VK_MEMORY_PROPERTY_DEVICE_LOCAL_BIT; - if (!VulkanCreateImage(ctx, &create, image, memory)) { - EXIT("TextureCache: failed to create render target, addr=0x%016" PRIx64 - " extent=%ux%u format=%d\n", - info.address, info.width, info.height, static_cast(info.format)); - } - image->memory = *memory; - CreateRenderTargetViews(ctx, image); - return image; -} - -void CreateDepthViews(GraphicContext* ctx, DepthStencilVulkanImage* image) { - UtilCreateImageView(ctx, image, VulkanImage::VIEW_DEFAULT, VK_IMAGE_VIEW_TYPE_2D, - DepthAspectMask(image->format), - {VK_COMPONENT_SWIZZLE_IDENTITY, VK_COMPONENT_SWIZZLE_IDENTITY, - VK_COMPONENT_SWIZZLE_IDENTITY, VK_COMPONENT_SWIZZLE_IDENTITY}, - 0, 0, 1, 1); -} - -DepthStencilVulkanImage* CreateDepthTarget(GraphicContext* ctx, const DepthTargetInfo& info, - VulkanMemory* memory) { - VkImageCreateInfo create {}; - create.sType = VK_STRUCTURE_TYPE_IMAGE_CREATE_INFO; - create.imageType = VK_IMAGE_TYPE_2D; - create.extent = {info.width, info.height, 1}; - create.mipLevels = 1; - create.arrayLayers = info.layers; - create.format = info.format; - create.tiling = VK_IMAGE_TILING_OPTIMAL; - create.initialLayout = VK_IMAGE_LAYOUT_UNDEFINED; - create.usage = DepthTargetImageUsage(); - create.sharingMode = VK_SHARING_MODE_EXCLUSIVE; - create.samples = VK_SAMPLE_COUNT_1_BIT; - VkImageFormatProperties properties {}; - if (ctx->GetImageFormatProperties(info.format, VK_IMAGE_TYPE_2D, VK_IMAGE_TILING_OPTIMAL, - create.usage, 0, &properties) != VK_SUCCESS) { - EXIT("TextureCache: depth format does not support required usage, format=%d usage=0x%x\n", - static_cast(info.format), create.usage); - } - auto* image = new DepthStencilVulkanImage; - image->extent.width = info.width; - image->extent.height = info.height; - image->guest_pitch = info.pitch; - image->layers = info.layers; - image->format = info.format; - image->layout = VK_IMAGE_LAYOUT_UNDEFINED; - image->compressed = false; - UtilResetImageViews(image); - memory->property = VK_MEMORY_PROPERTY_DEVICE_LOCAL_BIT; - if (!VulkanCreateImage(ctx, &create, image, memory)) { - EXIT("TextureCache: failed to create depth target, addr=0x%016" PRIx64 - " extent=%ux%u format=%d\n", - info.address, info.width, info.height, static_cast(info.format)); - } - image->memory = *memory; - CreateDepthViews(ctx, image); - return image; -} - -void ValidateVideoOutInfo(GraphicContext* ctx, const VideoOutInfo& info) { - const auto compression = - ClassifyVideoOutCompression(info.compression != VideoOutCompression::Uncompressed, - info.metadata_address, info.dcc_control, 0); - const bool metadata_invalid = compression == VideoOutCompression::Dcc256_64_64 && - (info.metadata_address >= TRACKER_ADDRESS_SIZE || - (info.metadata_address >= info.address && - info.metadata_address < info.address + info.size)); - if (ctx == nullptr || info.address == 0 || info.size == 0 || - info.address >= TRACKER_ADDRESS_SIZE || info.size > TRACKER_ADDRESS_SIZE - info.address || - (info.address & 0xffffu) != 0 || info.width == 0 || info.height == 0 || - info.width > 16384 || info.height > 16384 || info.pitch < info.width || - info.tile_mode != Prospero::GpuEnumValue(Prospero::TileMode::kRenderTarget) || - compression == VideoOutCompression::Unsupported || compression != info.compression || - metadata_invalid || !IsSupportedVideoOutFormat(info)) { - EXIT("TextureCache: unsupported video-out surface, ctx=%p addr=0x%016" PRIx64 - " size=0x%016" PRIx64 " metadata=0x%016" PRIx64 " dcc=0x%08" PRIx32 - " extent=%ux%u pitch=%u tile=%u guest_format=%u bpe=%u vk_format=%d\n", - static_cast(ctx), info.address, info.size, info.metadata_address, - info.dcc_control, info.width, info.height, info.pitch, info.tile_mode, - info.guest_format, info.bytes_per_element, static_cast(info.format)); - } - TileSizeAlign exact {}; - TileGetTextureTotalSize(info.guest_format, info.width, info.height, 1, info.pitch, 1, - info.tile_mode, false, &exact); - if (exact.align != 65536 || exact.size != info.size || - TileGetTexturePitch(info.guest_format, info.width, 1, info.tile_mode) != info.pitch) { - EXIT("TextureCache: video-out tile layout mismatch, addr=0x%016" PRIx64 - " size=0x%016" PRIx64 " expected_size=0x%016" PRIx64 " align=0x%016" PRIx64 - " pitch=%u\n", - info.address, info.size, exact.size, exact.align, info.pitch); - } - (void)RenderTargetUsage(ctx, info.format, 0); -} - -VideoOutVulkanImage* CreateVideoOut(GraphicContext* ctx, const VideoOutInfo& info, - VulkanMemory* memory) { - auto* image = new VideoOutVulkanImage; - image->extent.width = info.width; - image->extent.height = info.height; - image->format = info.format; - image->layout = VK_IMAGE_LAYOUT_UNDEFINED; - UtilResetImageViews(image); - VkImageCreateInfo create {}; - create.sType = VK_STRUCTURE_TYPE_IMAGE_CREATE_INFO; - create.imageType = VK_IMAGE_TYPE_2D; - create.extent = {info.width, info.height, 1}; - create.mipLevels = 1; - create.arrayLayers = 1; - create.format = info.format; - create.tiling = VK_IMAGE_TILING_OPTIMAL; - create.initialLayout = VK_IMAGE_LAYOUT_UNDEFINED; - create.flags = RenderTargetCreateFlags(info.format); - create.usage = RenderTargetUsage(ctx, info.format, create.flags); - create.sharingMode = VK_SHARING_MODE_EXCLUSIVE; - create.samples = VK_SAMPLE_COUNT_1_BIT; - memory->property = VK_MEMORY_PROPERTY_DEVICE_LOCAL_BIT; - if (!VulkanCreateImage(ctx, &create, image, memory)) { - EXIT("TextureCache: failed to create video-out image, addr=0x%016" PRIx64 - " extent=%ux%u format=%d\n", - info.address, info.width, info.height, static_cast(info.format)); - } - image->memory = *memory; - CreateVideoOutViews(ctx, image); - return image; -} - -void UploadVideoOut(GraphicContext* ctx, VideoOutVulkanImage* image, const VideoOutInfo& info, - bool refresh) { - if (info.compression != VideoOutCompression::Uncompressed) { - EXIT("TextureCache: compressed video-out guest upload is unsupported, " - "addr=0x%016" PRIx64 " metadata=0x%016" PRIx64 " dcc=0x%08" PRIx32 "\n", - info.address, info.metadata_address, info.dcc_control); - } - if (refresh) { - VulkanDeviceWaitIdle(ctx); - } - image->layout = VK_IMAGE_LAYOUT_UNDEFINED; - UtilScratchBuffer scratch(info.size); - TileConvertTiledToLinearRenderTarget( - scratch.Data(), reinterpret_cast(info.address), info.width, info.height, - info.pitch, info.bytes_per_element, info.size); - if (info.bgra16) { - auto* pixels = static_cast(scratch.Data()); - for (uint64_t i = 0; i < info.size / sizeof(uint16_t); i += 4) { - std::swap(pixels[i], pixels[i + 2]); - } - } - UtilFillImage(ctx, image, scratch.Data(), info.size, info.pitch, - static_cast(VK_IMAGE_LAYOUT_GENERAL)); -} - } // namespace TextureCache::TextureCache(PageManager& page_manager, BufferCache& buffer_cache, @@ -1238,328 +848,10 @@ TextureCache::TextureCache(PageManager& page_manager, BufferCache& buffer_cache, if (!Common::Thread::IsMainThread()) { EXIT("TextureCache: construction is restricted to the main thread\n"); } + m_dummy_textures = std::make_unique(); m_readback = std::make_unique(*this); } -VkImageView TextureCache::GetRenderTargetAttachmentView(GraphicContext* ctx, - RenderTextureVulkanImage* image, - VkFormat format, uint32_t level, - uint32_t base_layer, uint32_t layer_count) { - if (ctx == nullptr || image == nullptr || image->image == nullptr || - format == VK_FORMAT_UNDEFINED || level >= image->mip_levels || level >= 16 || - layer_count == 0 || base_layer >= image->layers || - layer_count > image->layers - base_layer) { - EXIT("TextureCache: invalid render-target attachment view, image=%p format=%d" - " level=%u image_levels=%u base_layer=%u layer_count=%u image_layers=%u\n", - static_cast(image), static_cast(format), level, - image != nullptr ? image->mip_levels : 0, base_layer, layer_count, - image != nullptr ? image->layers : 0); - } - if (format == image->format && base_layer == 0 && layer_count == 1) { - if (image->render_view[level] == nullptr) { - EXIT("TextureCache: base render-target attachment view is missing, level=%u\n", level); - } - return image->render_view[level]; - } - if (format != image->format && !IsRgba8SrgbReinterpretation(image->format, format)) { - EXIT("TextureCache: incompatible render-target attachment view, image_format=%d" - " view_format=%d level=%u\n", - static_cast(image->format), static_cast(format), level); - } - - std::lock_guard lock(image->attachment_view_mutex); - for (const auto& cached: image->attachment_views) { - if (cached.format == format && cached.level == level && cached.base_layer == base_layer && - cached.layer_count == layer_count) { - return cached.view; - } - } - - VkImageViewUsageCreateInfo usage {}; - usage.sType = VK_STRUCTURE_TYPE_IMAGE_VIEW_USAGE_CREATE_INFO; - usage.usage = VK_IMAGE_USAGE_COLOR_ATTACHMENT_BIT; - VkImageViewCreateInfo create {}; - create.sType = VK_STRUCTURE_TYPE_IMAGE_VIEW_CREATE_INFO; - create.pNext = &usage; - create.image = image->image; - create.viewType = layer_count == 1 ? VK_IMAGE_VIEW_TYPE_2D : VK_IMAGE_VIEW_TYPE_2D_ARRAY; - create.format = format; - create.components = {VK_COMPONENT_SWIZZLE_IDENTITY, VK_COMPONENT_SWIZZLE_IDENTITY, - VK_COMPONENT_SWIZZLE_IDENTITY, VK_COMPONENT_SWIZZLE_IDENTITY}; - create.subresourceRange.aspectMask = VK_IMAGE_ASPECT_COLOR_BIT; - create.subresourceRange.baseMipLevel = level; - create.subresourceRange.levelCount = 1; - create.subresourceRange.baseArrayLayer = base_layer; - create.subresourceRange.layerCount = layer_count; - VkImageView view = nullptr; - const auto result = vkCreateImageView(ctx->device, &create, nullptr, &view); - if (result != VK_SUCCESS || view == nullptr) { - EXIT("TextureCache: failed to create render-target attachment view," - " result=%d image_format=%d view_format=%d level=%u base_layer=%u" - " layer_count=%u\n", - static_cast(result), static_cast(image->format), static_cast(format), - level, base_layer, layer_count); - } - image->attachment_views.push_back({format, level, base_layer, layer_count, view}); - LOGF("TextureCache: created render-target attachment view:" - " image_format=%d view_format=%d level=%u layer=%u+%u extent=%ux%u\n", - static_cast(image->format), static_cast(format), level, base_layer, layer_count, - image->extent.width, image->extent.height); - return view; -} - -VkImageView TextureCache::GetDepthTargetAttachmentView(GraphicContext* ctx, - DepthStencilVulkanImage* image, - uint32_t base_layer, uint32_t layer_count) { - if (ctx == nullptr || image == nullptr || image->image == nullptr || layer_count == 0 || - base_layer >= image->layers || layer_count > image->layers - base_layer) { - EXIT("TextureCache: invalid depth-target attachment view, image=%p base_layer=%u " - "layer_count=%u image_layers=%u\n", - static_cast(image), base_layer, layer_count, - image != nullptr ? image->layers : 0); - } - if (base_layer == 0 && layer_count == 1) { - return image->image_view[VulkanImage::VIEW_DEFAULT]; - } - std::lock_guard lock(image->attachment_view_mutex); - for (const auto& cached: image->attachment_views) { - if (cached.base_layer == base_layer && cached.layer_count == layer_count) { - return cached.view; - } - } - VkImageViewCreateInfo create {}; - create.sType = VK_STRUCTURE_TYPE_IMAGE_VIEW_CREATE_INFO; - create.image = image->image; - create.viewType = layer_count == 1 ? VK_IMAGE_VIEW_TYPE_2D : VK_IMAGE_VIEW_TYPE_2D_ARRAY; - create.format = image->format; - create.components = {VK_COMPONENT_SWIZZLE_IDENTITY, VK_COMPONENT_SWIZZLE_IDENTITY, - VK_COMPONENT_SWIZZLE_IDENTITY, VK_COMPONENT_SWIZZLE_IDENTITY}; - create.subresourceRange.aspectMask = DepthAspectMask(image->format); - create.subresourceRange.baseMipLevel = 0; - create.subresourceRange.levelCount = 1; - create.subresourceRange.baseArrayLayer = base_layer; - create.subresourceRange.layerCount = layer_count; - VkImageView view = nullptr; - const auto result = vkCreateImageView(ctx->device, &create, nullptr, &view); - if (result != VK_SUCCESS || view == nullptr) { - EXIT("TextureCache: failed to create depth-target attachment view, result=%d " - "format=%d base_layer=%u layer_count=%u\n", - static_cast(result), static_cast(image->format), base_layer, layer_count); - } - image->attachment_views.push_back({base_layer, layer_count, view}); - return view; -} - -VkImageView TextureCache::GetImageView(GraphicContext* ctx, VulkanImage* image, - const ImageViewInfo& info) { - const bool supported_type = info.type == VK_IMAGE_VIEW_TYPE_2D || - info.type == VK_IMAGE_VIEW_TYPE_2D_ARRAY || - info.type == VK_IMAGE_VIEW_TYPE_3D; - const bool valid_shape = - (info.type == VK_IMAGE_VIEW_TYPE_2D && info.layer_count == 1) || - info.type == VK_IMAGE_VIEW_TYPE_2D_ARRAY || - (info.type == VK_IMAGE_VIEW_TYPE_3D && info.base_layer == 0 && info.layer_count == 1); - if (ctx == nullptr || image == nullptr || image->image == nullptr || - info.format == VK_FORMAT_UNDEFINED || info.aspect == 0 || info.level_count == 0 || - info.base_level >= (image != nullptr ? image->mip_levels : 0) || - info.level_count > image->mip_levels - info.base_level || info.layer_count == 0 || - info.base_layer >= image->layers || info.layer_count > image->layers - info.base_layer || - !supported_type || !valid_shape) { - EXIT("TextureCache: invalid dynamic image view, image=%p format=%d aspect=0x%x" - " swizzle=0x%03x mip=%u+%u layer=%u+%u type=%d storage=%d" - " image_levels=%u image_layers=%u\n", - static_cast(image), static_cast(info.format), info.aspect, - info.swizzle, info.base_level, info.level_count, info.base_layer, info.layer_count, - static_cast(info.type), info.is_storage, image != nullptr ? image->mip_levels : 0, - image != nullptr ? image->layers : 0); - } - - auto& cache = image->view_cache; - std::lock_guard lock(cache.mutex); - for (const auto& cached: cache.views) { - if (cached.info == info) { - return cached.view; - } - } - - VkImageViewUsageCreateInfo usage {}; - usage.sType = VK_STRUCTURE_TYPE_IMAGE_VIEW_USAGE_CREATE_INFO; - usage.usage = info.is_storage ? VK_IMAGE_USAGE_STORAGE_BIT : VK_IMAGE_USAGE_SAMPLED_BIT; - VkImageViewCreateInfo create {}; - create.sType = VK_STRUCTURE_TYPE_IMAGE_VIEW_CREATE_INFO; - create.pNext = &usage; - create.image = image->image; - create.viewType = info.type; - create.format = info.format; - create.components = - info.is_storage ? VkComponentMapping {} : TextureGetComponentMapping(info.swizzle); - create.subresourceRange.aspectMask = info.aspect; - create.subresourceRange.baseMipLevel = info.base_level; - create.subresourceRange.levelCount = info.level_count; - create.subresourceRange.baseArrayLayer = info.base_layer; - create.subresourceRange.layerCount = info.layer_count; - VkImageView view = nullptr; - const auto result = vkCreateImageView(ctx->device, &create, nullptr, &view); - if (result != VK_SUCCESS || view == nullptr) { - EXIT("TextureCache: failed to create dynamic image view, result=%d format=%d" - " aspect=0x%x swizzle=0x%03x mip=%u+%u layer=%u+%u type=%d storage=%d\n", - static_cast(result), static_cast(info.format), info.aspect, info.swizzle, - info.base_level, info.level_count, info.base_layer, info.layer_count, - static_cast(info.type), info.is_storage); - } - cache.views.push_back({info, view}); - return view; -} - -VkImageView TextureCache::GetDepthTargetSampledView(GraphicContext* ctx, - DepthStencilVulkanImage* image, - VkFormat view_format, uint32_t swizzle, - uint32_t base_level, uint32_t level_count, - VkImageViewType type, uint32_t base_layer, - uint32_t layer_count) { - if (ctx == nullptr || image == nullptr || image->image == nullptr || - view_format == VK_FORMAT_UNDEFINED || - !IsSupportedSampledDepthView(image->format, view_format, swizzle)) { - EXIT("TextureCache: invalid sampled depth-target view, image=%p image_format=%d" - " view_format=%d swizzle=0x%03x mip=%u+%u layer=%u+%u type=%d" - " image_levels=%u image_layers=%u\n", - static_cast(image), - image != nullptr ? static_cast(image->format) : VK_FORMAT_UNDEFINED, - static_cast(view_format), swizzle, base_level, level_count, base_layer, - layer_count, static_cast(type), image != nullptr ? image->mip_levels : 0, - image != nullptr ? image->layers : 0); - } - return GetImageView(ctx, image, - {image->format, type, VK_IMAGE_ASPECT_DEPTH_BIT, base_level, level_count, - base_layer, layer_count, swizzle}); -} - -VkImageView TextureCache::GetSampledColorView(GraphicContext* ctx, VulkanImage* image, - VkFormat view_format, uint32_t swizzle, - uint32_t base_level, uint32_t level_count, - VkImageViewType type, uint32_t base_layer, - uint32_t layer_count) { - if (ctx == nullptr || image == nullptr || image->image == nullptr || - view_format == VK_FORMAT_UNDEFINED || base_level >= 16 || - (type != VK_IMAGE_VIEW_TYPE_2D && type != VK_IMAGE_VIEW_TYPE_2D_ARRAY) || - !IsSupportedSampledColorView(image->format, view_format, swizzle)) { - EXIT("TextureCache: invalid sampled color view, image=%p swizzle=0x%03x" - " view_format=%d mip=%u+%u layer=%u+%u type=%d image_levels=%u image_layers=%u\n", - static_cast(image), swizzle, static_cast(view_format), base_level, - level_count, base_layer, layer_count, static_cast(type), - image != nullptr ? image->mip_levels : 0, image != nullptr ? image->layers : 0); - } - const auto precreated_view = type == VK_IMAGE_VIEW_TYPE_2D_ARRAY - ? VulkanImage::VIEW_DEFAULT_ARRAY - : VulkanImage::VIEW_DEFAULT; - const bool full_view = - base_level == 0 && level_count == image->mip_levels && base_layer == 0 && - layer_count == (type == VK_IMAGE_VIEW_TYPE_2D_ARRAY ? image->layers : 1u); - if (view_format == image->format && swizzle == DstSel(4, 5, 6, 7) && full_view && - image->image_view[precreated_view] != nullptr) { - return image->image_view[precreated_view]; - } - return GetImageView(ctx, image, - {view_format, type, VK_IMAGE_ASPECT_COLOR_BIT, base_level, level_count, - base_layer, layer_count, swizzle}); -} - -VkImageView TextureCache::GetRenderTargetStorageView(GraphicContext* ctx, - RenderTextureVulkanImage* image, - VkFormat view_format, uint32_t base_level, - uint32_t level_count, VkImageViewType type, - uint32_t base_layer, uint32_t layer_count) { - if (ctx == nullptr || image == nullptr || image->image == nullptr || - view_format == VK_FORMAT_UNDEFINED || - (type != VK_IMAGE_VIEW_TYPE_2D && type != VK_IMAGE_VIEW_TYPE_2D_ARRAY)) { - EXIT("TextureCache: invalid render-target storage view, image=%p view_format=%d" - " mip=%u+%u layer=%u+%u type=%d image_levels=%u image_layers=%u\n", - static_cast(image), static_cast(view_format), base_level, - level_count, base_layer, layer_count, static_cast(type), - image != nullptr ? image->mip_levels : 0, image != nullptr ? image->layers : 0); - } - const bool exact = view_format == image->format; - const bool compatible = view_format == BgraSrgbStorageViewFormat(image->format); - if (!exact && !compatible) { - EXIT("TextureCache: incompatible render-target storage view, image_format=%d" - " view_format=%d base=%u count=%u\n", - static_cast(image->format), static_cast(view_format), base_level, - level_count); - } - if (exact) { - const auto index = type == VK_IMAGE_VIEW_TYPE_2D_ARRAY ? VulkanImage::VIEW_STORAGE_ARRAY - : VulkanImage::VIEW_STORAGE; - const bool full_view = - base_level == 0 && level_count == 1 && base_layer == 0 && - layer_count == (type == VK_IMAGE_VIEW_TYPE_2D_ARRAY ? image->layers : 1u); - if (full_view && image->image_view[index] != nullptr) { - return image->image_view[index]; - } - } - - if (compatible && !FormatSupportsStorage(ctx, view_format)) { - EXIT("TextureCache: compatible render-target storage format lacks storage support," - " image_format=%d view_format=%d base=%u count=%u\n", - static_cast(image->format), static_cast(view_format), base_level, - level_count); - } - return GetImageView(ctx, image, - {view_format, type, VK_IMAGE_ASPECT_COLOR_BIT, base_level, level_count, - base_layer, layer_count, DstSel(4, 5, 6, 7), true}); -} - -VkImageView TextureCache::GetStorageTextureSampledView(GraphicContext* ctx, - StorageTextureVulkanImage* image, - const ImageInfo& info) { - const auto shape = - SelectStorageSampledViewShape(info.type, info.depth, image != nullptr ? image->layers : 0); - if (ctx == nullptr || image == nullptr || image->image == nullptr || - shape == StorageSampledViewShape::Unsupported || info.base_array != 0 || - info.levels != image->mip_levels || info.base_level >= info.levels || - info.view_levels == 0 || info.base_level + info.view_levels > info.levels) { - EXIT("TextureCache: invalid sampled view of storage texture, image=%p type=%u depth=%u" - " base=%u levels=%u view_levels=%u image_levels=%u base_array=%u\n", - static_cast(image), info.type, info.depth, info.base_level, info.levels, - info.view_levels, image != nullptr ? image->mip_levels : 0, info.base_array); - } - const auto view_format = TextureGetFormat(info.format, TextureFormatUsage::Sampled); - if (view_format != image->format && !IsRgba8SrgbReinterpretation(image->format, view_format) && - !IsR32UintFloatReinterpretation(image->format, view_format)) { - EXIT("TextureCache: incompatible sampled view of storage texture, image_format=%d" - " view_format=%d swizzle=0x%03x\n", - static_cast(image->format), static_cast(view_format), info.swizzle); - } - - VkImageViewType type = VK_IMAGE_VIEW_TYPE_MAX_ENUM; - switch (shape) { - case StorageSampledViewShape::Image2D: type = VK_IMAGE_VIEW_TYPE_2D; break; - case StorageSampledViewShape::Image2DArray: type = VK_IMAGE_VIEW_TYPE_2D_ARRAY; break; - case StorageSampledViewShape::Image3D: type = VK_IMAGE_VIEW_TYPE_3D; break; - case StorageSampledViewShape::Unsupported: - EXIT("TextureCache: unsupported sampled storage-image view shape\n"); - } - const auto layer_count = shape == StorageSampledViewShape::Image2DArray ? info.depth : 1u; - return GetImageView(ctx, image, - {view_format, type, VK_IMAGE_ASPECT_COLOR_BIT, info.base_level, - info.view_levels, 0, layer_count, info.swizzle}); -} - -VkImageView TextureCache::GetStorageTextureStorageView(GraphicContext* ctx, - StorageTextureVulkanImage* image, - uint32_t base_level) { - if (ctx == nullptr || image == nullptr || image->image == nullptr || - base_level >= (image != nullptr ? image->mip_levels : 0)) { - EXIT("TextureCache: invalid storage-texture mip view, image=%p level=%u levels=%u\n", - static_cast(image), base_level, image != nullptr ? image->mip_levels : 0); - } - if (base_level == 0) { - return image->image_view[VulkanImage::VIEW_DEFAULT]; - } - return GetImageView(ctx, image, - {image->format, VK_IMAGE_VIEW_TYPE_2D, VK_IMAGE_ASPECT_COLOR_BIT, - base_level, 1, 0, 1, DstSel(4, 5, 6, 7), true}); -} - TextureCache::CachedImage* TextureCache::FindGpuReadbackPageCandidateLocked(uint64_t vaddr, uint64_t size) { CachedImage* selected = nullptr; @@ -1801,31 +1093,14 @@ void TextureCache::RetireStorageDepthAliasLocked(GraphicContext* ctx, TextureCache::~TextureCache() { m_readback.reset(); - GraphicContext* ctx = nullptr; if (!m_images.empty()) { - ctx = m_images.front()->ctx; - } else { - ctx = m_dummy_ctx; - } - if (ctx != nullptr) { - VulkanDeviceWaitIdle(ctx); + VulkanDeviceWaitIdle(m_images.front()->ctx); } for (const auto& image: m_images) { UnregisterImageLocked(*image, false); } m_images.clear(); - for (size_t i = 0; i < m_dummy_sampled_textures.size(); i++) { - if (m_dummy_sampled_textures[i] != nullptr) { - DeleteGpuTexture(ctx, static_cast(m_dummy_sampled_textures[i]), - m_dummy_sampled_memory[i]); - delete m_dummy_sampled_memory[i]; - } - if (m_dummy_storage_textures[i] != nullptr) { - DeleteGpuTexture(ctx, static_cast(m_dummy_storage_textures[i]), - m_dummy_storage_memory[i]); - delete m_dummy_storage_memory[i]; - } - } + m_dummy_textures.reset(); } VulkanImage* TextureCache::FindTexture(CommandBuffer* command, GraphicContext* ctx, @@ -1990,27 +1265,21 @@ VulkanImage* TextureCache::FindTexture(CommandBuffer* command, GraphicContext* c } } m_images.reserve(m_images.size() + 1); - auto cached = std::make_shared(); - cached->kind = CachedImage::Kind::Texture; - cached->info = info; - cached->ctx = ctx; - cached->image = new TextureVulkanImage; - const auto components = - TextureCreateImage(ctx, cached->image, &cached->memory, MakeImageParams(info, false)); + auto cached = std::make_shared(); + cached->kind = CachedImage::Kind::Texture; + cached->info = info; + cached->ctx = ctx; + VkComponentMapping components {}; + cached->image = ImageOps::CreateTexture(ctx, info, false, &cached->memory, &components); m_memory_tracker.ForEachUploadRange( info.address, info.size, false, [](uint64_t, uint64_t) noexcept {}, [&]() noexcept { m_tiler.DetileImage(cached->ctx, static_cast(cached->image), cached->info, source, false, false); }); - TextureCreateImageViews(ctx, cached->image, components, info.type, info.base_array, - info.base_level, info.view_levels, info.depth, true, - TextureFormatUsage::Sampled); - auto* image = cached->image; - command->RetainResourceUntilFence(cached); - m_images.push_back(std::move(cached)); - RegisterImageLocked(*m_images.back()); - return image; + ImageOps::CreateTextureViews(ctx, static_cast(cached->image), info, + false, components); + return PublishImage(command, std::move(cached)); } StorageTextureVulkanImage* TextureCache::FindStorageTexture(CommandBuffer* command, @@ -2133,13 +1402,12 @@ StorageTextureVulkanImage* TextureCache::FindStorageTexture(CommandBuffer* com ResolveStorageImageOverlaps(ctx, info); m_images.reserve(m_images.size() + 1); - auto cached = std::make_shared(); - cached->kind = CachedImage::Kind::StorageTexture; - cached->info = info; - cached->ctx = ctx; - cached->image = new StorageTextureVulkanImage; - const auto components = - TextureCreateImage(ctx, cached->image, &cached->memory, MakeImageParams(info, true)); + auto cached = std::make_shared(); + cached->kind = CachedImage::Kind::StorageTexture; + cached->info = info; + cached->ctx = ctx; + VkComponentMapping components {}; + cached->image = ImageOps::CreateTexture(ctx, info, true, &cached->memory, &components); m_memory_tracker.ForEachUploadRange( info.address, info.size, true, [](uint64_t, uint64_t) noexcept {}, [&]() noexcept { @@ -2147,13 +1415,9 @@ StorageTextureVulkanImage* TextureCache::FindStorageTexture(CommandBuffer* com cached->info, source, false, true); }); cached->gpu_modified = true; - TextureCreateImageViews(ctx, cached->image, components, info.type, 0, 0, 1, info.depth, false, - TextureFormatUsage::Sampled | TextureFormatUsage::Storage); - auto* image = static_cast(cached->image); - command->RetainResourceUntilFence(cached); - m_images.push_back(std::move(cached)); - RegisterImageLocked(*m_images.back()); - return image; + ImageOps::CreateTextureViews(ctx, static_cast(cached->image), info, true, + components); + return static_cast(PublishImage(command, std::move(cached))); } RenderTextureVulkanImage* TextureCache::FindRenderTarget(CommandBuffer* command, @@ -2188,7 +1452,7 @@ RenderTextureVulkanImage* TextureCache::FindRenderTarget(CommandBuffer* } } if (standard64) { - const auto format = RenderTargetTransferFormat(info.bytes_per_element); + const auto format = ImageOps::RenderTargetTransferFormat(info.bytes_per_element); const auto expected_pitch = TileGetTexturePitch(format, info.width, info.levels, info.tile_mode); TileSizeAlign layout {}; @@ -2249,8 +1513,8 @@ RenderTextureVulkanImage* TextureCache::FindRenderTarget(CommandBuffer* m_memory_tracker.ForEachUploadRange( info.address, info.size, false, [](uint64_t, uint64_t) noexcept {}, [&]() noexcept { - UploadRenderTarget(ctx, static_cast(match->image), - info, true); + ImageOps::UploadRenderTarget( + ctx, static_cast(match->image), info, true); }); match->buffer_modified = false; } @@ -2350,7 +1614,7 @@ RenderTextureVulkanImage* TextureCache::FindRenderTarget(CommandBuffer* cached->kind = CachedImage::Kind::RenderTarget; cached->target = info; cached->ctx = ctx; - cached->image = CreateRenderTarget(ctx, info, &cached->memory); + cached->image = ImageOps::CreateRenderTarget(ctx, info, &cached->memory); const bool preserve_native = native_image_source != nullptr; if (preserve_native) { command->RetainResourceUntilFence(native_image_source); @@ -2367,9 +1631,9 @@ RenderTextureVulkanImage* TextureCache::FindRenderTarget(CommandBuffer* m_memory_tracker.ForEachUploadRange( tail_address, tail_size, true, [](uint64_t, uint64_t) noexcept {}, [&]() noexcept { - UploadRenderTargetLayers(ctx, - static_cast(cached->image), - info, old.layers, info.layers - old.layers, false); + ImageOps::UploadRenderTargetLayers( + ctx, static_cast(cached->image), info, old.layers, + info.layers - old.layers, false); }); } std::vector regions; @@ -2382,16 +1646,12 @@ RenderTextureVulkanImage* TextureCache::FindRenderTarget(CommandBuffer* m_memory_tracker.ForEachUploadRange( info.address, info.size, false, [](uint64_t, uint64_t) noexcept {}, [&]() noexcept { - UploadRenderTarget(ctx, static_cast(cached->image), info, - false); + ImageOps::UploadRenderTarget( + ctx, static_cast(cached->image), info, false); }); } cached->gpu_modified = preserve_native; - auto* image = static_cast(cached->image); - command->RetainResourceUntilFence(cached); - m_images.push_back(std::move(cached)); - RegisterImageLocked(*m_images.back()); - return image; + return static_cast(PublishImage(command, std::move(cached))); } DepthStencilVulkanImage* TextureCache::FindDepthTarget(CommandBuffer* command, GraphicContext* ctx, @@ -2635,7 +1895,7 @@ DepthStencilVulkanImage* TextureCache::FindDepthTarget(CommandBuffer* command, G cached->depth = info; cached->ctx = ctx; cached->stencil_initialized = !has_stencil; - cached->image = CreateDepthTarget(ctx, info, &cached->memory); + cached->image = ImageOps::CreateDepthTarget(ctx, info, &cached->memory); if (discarded_depth_source != nullptr) { command->RetainResourceUntilFence(discarded_depth_source); } @@ -2737,11 +1997,7 @@ DepthStencilVulkanImage* TextureCache::FindDepthTarget(CommandBuffer* command, G cached->stencil_initialized = true; } } - auto* image = static_cast(cached->image); - command->RetainResourceUntilFence(cached); - m_images.push_back(std::move(cached)); - RegisterImageLocked(*m_images.back()); - return image; + return static_cast(PublishImage(command, std::move(cached))); } std::vector @@ -2751,7 +2007,7 @@ TextureCache::RegisterVideoOutSurfaces(GraphicContext* ctx, EXIT("TextureCache: video-out registration requires surfaces\n"); } for (const auto& info: infos) { - ValidateVideoOutInfo(ctx, info); + ImageOps::ValidateVideoOut(ctx, info); } for (size_t i = 0; i < infos.size(); i++) { for (size_t j = i + 1; j < infos.size(); j++) { @@ -2789,13 +2045,13 @@ TextureCache::RegisterVideoOutSurfaces(GraphicContext* ctx, cached->kind = CachedImage::Kind::VideoOut; cached->video_out = info; cached->ctx = ctx; - cached->image = CreateVideoOut(ctx, info, &cached->memory); + cached->image = ImageOps::CreateVideoOut(ctx, info, &cached->memory); if (info.compression == VideoOutCompression::Uncompressed) { m_memory_tracker.ForEachUploadRange( info.address, info.size, false, [](uint64_t, uint64_t) noexcept {}, [&]() noexcept { - UploadVideoOut(ctx, static_cast(cached->image), info, - false); + ImageOps::UploadVideoOut( + ctx, static_cast(cached->image), info, false); }); } else { // A compressed guest surface cannot be decoded without its DCC metadata. Establish the @@ -2862,7 +2118,7 @@ void TextureCache::RefreshVideoOut(VideoOutVulkanImage* image, bool render_targe } m_memory_tracker.ForEachUploadRange( info.address, info.size, false, [](uint64_t, uint64_t) noexcept {}, - [&]() noexcept { UploadVideoOut(cached.ctx, image, info, true); }); + [&]() noexcept { ImageOps::UploadVideoOut(cached.ctx, image, info, true); }); } void TextureCache::UnregisterVideoOutSurfaces(const std::vector& images) { @@ -2915,8 +2171,8 @@ void TextureCache::UnregisterVideoOutSurfaces(const std::vectorIsInvalid() || vaddr == 0 || size == 0 || vaddr >= TRACKER_ADDRESS_SIZE || size > TRACKER_ADDRESS_SIZE - vaddr) { EXIT("TextureCache: invalid compute image clear, command=%p addr=0x%016" PRIx64 @@ -2926,28 +2182,67 @@ bool TextureCache::ClearColorImageFromBuffer(CommandBuffer* command, uint64_t va m_buffer_cache.ValidateGpuAccess(vaddr, size, false, true); std::lock_guard transaction(m_resource_mutex); FaultSafeTextureLock lock(this, m_lock); + enum class ClearAspect : uint8_t { None, Color, Depth, Stencil }; + auto classify = [vaddr, size](const CachedImage& cached) { + const bool color = cached.kind == CachedImage::Kind::VideoOut || + cached.kind == CachedImage::Kind::RenderTarget; + if (color && cached.Address() == vaddr && cached.Size() == size) { + return ClearAspect::Color; + } + if (cached.kind != CachedImage::Kind::DepthTarget) { + return ClearAspect::None; + } + if (CanNativeClearDepthFromBuffer(cached.depth, vaddr, size)) { + return ClearAspect::Depth; + } + if (cached.RangeCount() == 2 && cached.Address(1) == vaddr && cached.Size(1) == size) { + return ClearAspect::Stencil; + } + return ClearAspect::None; + }; + std::shared_ptr match; + ClearAspect aspect = ClearAspect::None; + bool incompatible = false; + uint32_t matches = 0; for (const auto& cached: m_images) { if (!cached->OverlapsRange(vaddr, size, true)) { continue; } - const bool color = cached->kind == CachedImage::Kind::VideoOut || - cached->kind == CachedImage::Kind::RenderTarget; - const bool exact = cached->Address() == vaddr && cached->Size() == size; - if (!color || !exact) { - return false; + const auto candidate = classify(*cached); + if (candidate == ClearAspect::None) { + incompatible = true; + continue; } - if (match != nullptr) { - EXIT("TextureCache: compute image clear has ambiguous exact aliases, " - "addr=0x%016" PRIx64 " size=0x%016" PRIx64 " first=%p second=%p\n", - vaddr, size, static_cast(match->image), - static_cast(cached->image)); + if (aspect == ClearAspect::None) { + aspect = candidate; + match = cached; + } else if (candidate != aspect) { + incompatible = true; + continue; } - match = cached; + matches++; } - if (match == nullptr) { + if (matches > 1) { + EXIT("TextureCache: compute image clear has ambiguous exact aliases, " + "addr=0x%016" PRIx64 " size=0x%016" PRIx64 " aspect=%u matches=%u\n", + vaddr, size, static_cast(aspect), matches); + } + if (incompatible || match == nullptr) { return false; } + + float depth_clear = 0.0f; + if (aspect == ClearAspect::Depth && + !DecodePackedDepthClear(match->image->format, packed_clear, &depth_clear)) { + return false; + } + uint8_t stencil_clear = 0; + if (aspect == ClearAspect::Stencil && + !DecodePackedStencilClear(packed_clear, &stencil_clear)) { + return false; + } + RequireNoMetaOverlapLocked(vaddr, size); const bool buffer_overlap = m_buffer_cache.HasPageOverlap(vaddr, size); const bool buffer_cpu_modified = @@ -2956,16 +2251,15 @@ bool TextureCache::ClearColorImageFromBuffer(CommandBuffer* command, uint64_t va buffer_overlap && m_buffer_cache.IsRegionGpuModified(vaddr, size); BufferImageCopySource source {nullptr, 0, vaddr, size, true}; if (buffer_overlap && !buffer_cpu_modified && !buffer_gpu_modified) { - // A prior buffer clear can leave one clean cached buffer after this range becomes a native - // color target. Allow coexisting cache views only after the shared ownership - // helper proves the stale source is coherent native-buffer or guest backing. source = m_buffer_cache.ObtainBufferForImage(vaddr, size); } const bool buffer_source_valid = !buffer_overlap || IsCoherentGuestImageSource(source, vaddr, size); const bool image_cpu_modified = m_memory_tracker.IsRegionCpuModified(vaddr, size); - if (!buffer_source_valid || buffer_cpu_modified || buffer_gpu_modified || - match->buffer_modified || image_cpu_modified) { + const bool invalid_common = !buffer_source_valid || buffer_cpu_modified || + buffer_gpu_modified || match->buffer_modified || + image_cpu_modified; + if (aspect == ClearAspect::Color && invalid_common) { EXIT("TextureCache: compute image clear requires exclusive GPU image ownership, " "addr=0x%016" PRIx64 " size=0x%016" PRIx64 " buffer_overlap=%d source_valid=%d buffer_cpu_modified=%d" @@ -2973,176 +2267,61 @@ bool TextureCache::ClearColorImageFromBuffer(CommandBuffer* command, uint64_t va vaddr, size, buffer_overlap, buffer_source_valid, buffer_cpu_modified, buffer_gpu_modified, match->buffer_modified, image_cpu_modified); } - VkClearColorValue clear {}; - if (!DecodePackedColorClear(match->image->format, packed_clear, &clear)) { - return false; + if (aspect != ClearAspect::Color && (!match->gpu_modified || invalid_common)) { + const bool depth = aspect == ClearAspect::Depth; + const char* plane = depth ? "depth" : "stencil"; + const char* source_name = depth ? "buffer" : "stencil"; + EXIT("TextureCache: compute %s clear requires a GPU-owned depth image and a clean " + "%s source, addr=0x%016" PRIx64 " size=0x%016" PRIx64 + " gpu_modified=%d buffer_overlap=%d source_valid=%d buffer_cpu_modified=%d" + " buffer_gpu_modified=%d buffer_modified=%d image_cpu_modified=%d\n", + plane, source_name, vaddr, size, match->gpu_modified, buffer_overlap, + buffer_source_valid, buffer_cpu_modified, buffer_gpu_modified, + match->buffer_modified, image_cpu_modified); } + auto* vk_buffer = command->GetPool()->buffers[command->GetIndex()]; - GraphicsRenderColorImageBarrier(vk_buffer, match->image, VK_IMAGE_LAYOUT_TRANSFER_DST_OPTIMAL); - const VkImageSubresourceRange range {VK_IMAGE_ASPECT_COLOR_BIT, 0, VK_REMAINING_MIP_LEVELS, 0, - match->image->layers}; - vkCmdClearColorImage(vk_buffer, match->image->image, match->image->layout, &clear, 1, &range); - GraphicsRenderColorImageBarrier(vk_buffer, match->image, RENDER_COLOR_IMAGE_LAYOUT); - if (!match->gpu_modified) { - m_memory_tracker.MarkRegionAsGpuModified(vaddr, size); - match->gpu_modified = true; - } - command->RetainResourceUntilFence(match); - return true; -} - -bool TextureCache::ClearDepthImageFromBuffer(CommandBuffer* command, uint64_t vaddr, uint64_t size, - uint32_t packed_clear) { - if (command == nullptr || command->IsInvalid() || vaddr == 0 || size == 0 || - vaddr >= TRACKER_ADDRESS_SIZE || size > TRACKER_ADDRESS_SIZE - vaddr) { - EXIT("TextureCache: invalid compute depth clear, command=%p addr=0x%016" PRIx64 - " size=0x%016" PRIx64 "\n", - static_cast(command), vaddr, size); - } - m_buffer_cache.ValidateGpuAccess(vaddr, size, false, true); - std::lock_guard transaction(m_resource_mutex); - FaultSafeTextureLock lock(this, m_lock); - std::shared_ptr match; - for (const auto& cached: m_images) { - if (!cached->OverlapsRange(vaddr, size, true)) { - continue; - } - const bool exact_depth = cached->kind == CachedImage::Kind::DepthTarget && - CanNativeClearDepthFromBuffer(cached->depth, vaddr, size); - if (!exact_depth) { + if (aspect == ClearAspect::Color) { + VkClearColorValue clear {}; + if (!DecodePackedColorClear(match->image->format, packed_clear, &clear)) { return false; } - if (match != nullptr) { - EXIT("TextureCache: compute depth clear has ambiguous exact aliases, " - "addr=0x%016" PRIx64 " size=0x%016" PRIx64 " first=%p second=%p\n", - vaddr, size, static_cast(match->image), - static_cast(cached->image)); + GraphicsRenderColorImageBarrier(vk_buffer, match->image, + VK_IMAGE_LAYOUT_TRANSFER_DST_OPTIMAL); + const VkImageSubresourceRange range {VK_IMAGE_ASPECT_COLOR_BIT, 0, + VK_REMAINING_MIP_LEVELS, 0, + match->image->layers}; + vkCmdClearColorImage(vk_buffer, match->image->image, match->image->layout, &clear, 1, + &range); + GraphicsRenderColorImageBarrier(vk_buffer, match->image, RENDER_COLOR_IMAGE_LAYOUT); + if (!match->gpu_modified) { + m_memory_tracker.MarkRegionAsGpuModified(vaddr, size); + match->gpu_modified = true; } - match = cached; - } - if (match == nullptr) { - return false; - } - float depth_clear = 0.0f; - if (!DecodePackedDepthClear(match->image->format, packed_clear, &depth_clear)) { - return false; - } - RequireNoMetaOverlapLocked(vaddr, size); - const bool buffer_overlap = m_buffer_cache.HasPageOverlap(vaddr, size); - const bool buffer_cpu_modified = - buffer_overlap && m_buffer_cache.IsRegionCpuModified(vaddr, size); - const bool buffer_gpu_modified = - buffer_overlap && m_buffer_cache.IsRegionGpuModified(vaddr, size); - BufferImageCopySource source {nullptr, 0, vaddr, size, true}; - if (buffer_overlap && !buffer_cpu_modified && !buffer_gpu_modified) { - source = m_buffer_cache.ObtainBufferForImage(vaddr, size); - } - const bool buffer_source_valid = - !buffer_overlap || IsCoherentGuestImageSource(source, vaddr, size); - const bool image_cpu_modified = m_memory_tracker.IsRegionCpuModified(vaddr, size); - if (!match->gpu_modified || !buffer_source_valid || buffer_cpu_modified || - buffer_gpu_modified || match->buffer_modified || image_cpu_modified) { - EXIT("TextureCache: compute depth clear requires a GPU-owned depth image and a clean " - "buffer source, addr=0x%016" PRIx64 " size=0x%016" PRIx64 - " gpu_modified=%d buffer_overlap=%d source_valid=%d buffer_cpu_modified=%d" - " buffer_gpu_modified=%d buffer_modified=%d image_cpu_modified=%d\n", - vaddr, size, match->gpu_modified, buffer_overlap, buffer_source_valid, - buffer_cpu_modified, buffer_gpu_modified, match->buffer_modified, image_cpu_modified); + command->RetainResourceUntilFence(match); + return true; } + if (match->image->layout == VK_IMAGE_LAYOUT_UNDEFINED || match->image->layout == VK_IMAGE_LAYOUT_PREINITIALIZED) { - EXIT("TextureCache: compute depth clear has invalid source layout %u\n", + EXIT("TextureCache: compute %s clear has invalid source layout %u\n", + aspect == ClearAspect::Depth ? "depth" : "stencil", static_cast(match->image->layout)); } - auto* vk_buffer = command->GetPool()->buffers[command->GetIndex()]; const auto old_layout = match->image->layout; GraphicsRenderDepthStencilImageBarrier(vk_buffer, match->image, VK_IMAGE_LAYOUT_TRANSFER_DST_OPTIMAL); - const VkClearDepthStencilValue clear {depth_clear, 0}; - const VkImageSubresourceRange range {VK_IMAGE_ASPECT_DEPTH_BIT, 0, VK_REMAINING_MIP_LEVELS, 0, - match->image->layers}; - vkCmdClearDepthStencilImage(vk_buffer, match->image->image, match->image->layout, &clear, 1, - &range); - GraphicsRenderDepthStencilImageBarrier(vk_buffer, match->image, old_layout); - command->RetainResourceUntilFence(match); - return true; -} - -bool TextureCache::ClearStencilImageFromBuffer(CommandBuffer* command, uint64_t vaddr, - uint64_t size, uint32_t packed_clear) { - if (command == nullptr || command->IsInvalid() || vaddr == 0 || size == 0 || - vaddr >= TRACKER_ADDRESS_SIZE || size > TRACKER_ADDRESS_SIZE - vaddr) { - EXIT("TextureCache: invalid compute stencil clear, command=%p addr=0x%016" PRIx64 - " size=0x%016" PRIx64 "\n", - static_cast(command), vaddr, size); - } - uint8_t stencil_clear = 0; - if (!DecodePackedStencilClear(packed_clear, &stencil_clear)) { - return false; - } - m_buffer_cache.ValidateGpuAccess(vaddr, size, false, true); - std::lock_guard transaction(m_resource_mutex); - FaultSafeTextureLock lock(this, m_lock); - std::shared_ptr match; - for (const auto& cached: m_images) { - if (!cached->OverlapsRange(vaddr, size, true)) { - continue; - } - const bool exact_stencil = cached->kind == CachedImage::Kind::DepthTarget && - cached->RangeCount() == 2 && cached->Address(1) == vaddr && - cached->Size(1) == size; - if (!exact_stencil) { - return false; - } - if (match != nullptr) { - EXIT("TextureCache: compute stencil clear has ambiguous exact aliases, " - "addr=0x%016" PRIx64 " size=0x%016" PRIx64 " first=%p second=%p\n", - vaddr, size, static_cast(match->image), - static_cast(cached->image)); - } - match = cached; - } - if (match == nullptr) { - return false; - } - RequireNoMetaOverlapLocked(vaddr, size); - const bool buffer_overlap = m_buffer_cache.HasPageOverlap(vaddr, size); - const bool buffer_cpu_modified = - buffer_overlap && m_buffer_cache.IsRegionCpuModified(vaddr, size); - const bool buffer_gpu_modified = - buffer_overlap && m_buffer_cache.IsRegionGpuModified(vaddr, size); - BufferImageCopySource source {nullptr, 0, vaddr, size, true}; - if (buffer_overlap && !buffer_cpu_modified && !buffer_gpu_modified) { - source = m_buffer_cache.ObtainBufferForImage(vaddr, size); - } - const bool buffer_source_valid = - !buffer_overlap || IsCoherentGuestImageSource(source, vaddr, size); - const bool image_cpu_modified = m_memory_tracker.IsRegionCpuModified(vaddr, size); - if (!match->gpu_modified || !buffer_source_valid || buffer_cpu_modified || - buffer_gpu_modified || match->buffer_modified || image_cpu_modified) { - EXIT("TextureCache: compute stencil clear requires a GPU-owned depth image and a clean " - "stencil source, addr=0x%016" PRIx64 " size=0x%016" PRIx64 - " gpu_modified=%d buffer_overlap=%d source_valid=%d buffer_cpu_modified=%d" - " buffer_gpu_modified=%d buffer_modified=%d image_cpu_modified=%d\n", - vaddr, size, match->gpu_modified, buffer_overlap, buffer_source_valid, - buffer_cpu_modified, buffer_gpu_modified, match->buffer_modified, image_cpu_modified); - } - if (match->image->layout == VK_IMAGE_LAYOUT_UNDEFINED || - match->image->layout == VK_IMAGE_LAYOUT_PREINITIALIZED) { - EXIT("TextureCache: compute stencil clear has invalid source layout %u\n", - static_cast(match->image->layout)); - } - auto* vk_buffer = command->GetPool()->buffers[command->GetIndex()]; - const auto old_layout = match->image->layout; - GraphicsRenderDepthStencilImageBarrier(vk_buffer, match->image, - VK_IMAGE_LAYOUT_TRANSFER_DST_OPTIMAL); - const VkClearDepthStencilValue clear {0.0f, stencil_clear}; - const VkImageSubresourceRange range {VK_IMAGE_ASPECT_STENCIL_BIT, 0, VK_REMAINING_MIP_LEVELS, 0, + const VkClearDepthStencilValue clear {depth_clear, stencil_clear}; + const auto clear_aspect = static_cast( + aspect == ClearAspect::Depth ? VK_IMAGE_ASPECT_DEPTH_BIT : VK_IMAGE_ASPECT_STENCIL_BIT); + const VkImageSubresourceRange range {clear_aspect, 0, VK_REMAINING_MIP_LEVELS, 0, match->image->layers}; vkCmdClearDepthStencilImage(vk_buffer, match->image->image, match->image->layout, &clear, 1, &range); GraphicsRenderDepthStencilImageBarrier(vk_buffer, match->image, old_layout); - match->stencil_initialized = true; + if (aspect == ClearAspect::Stencil) { + match->stencil_initialized = true; + } command->RetainResourceUntilFence(match); return true; } @@ -3529,17 +2708,18 @@ bool TextureCache::InvalidateMemoryFromGPU(uint64_t vaddr, uint64_t size, return false; } -DepthStencilVulkanImage* TextureCache::FindDepthTargetByRange(uint64_t vaddr, uint64_t size, +DepthStencilVulkanImage* TextureCache::FindDepthTargetByRange(CommandBuffer* command, + uint64_t vaddr, uint64_t size, bool allow_containing_sampled) { - if (vaddr == 0 || size == 0 || vaddr >= TRACKER_ADDRESS_SIZE || + if (command == nullptr || vaddr == 0 || size == 0 || vaddr >= TRACKER_ADDRESS_SIZE || size > TRACKER_ADDRESS_SIZE - vaddr) { EXIT("TextureCache: invalid depth-target range query, addr=0x%016" PRIx64 " size=0x%016" PRIx64 "\n", vaddr, size); } - FaultSafeTextureLock lock(this, m_lock); - DepthStencilVulkanImage* found = nullptr; - for (auto& cached: m_images) { + FaultSafeTextureLock lock(this, m_lock); + CachedImage* found = nullptr; + for (auto* cached: FindImagesInRegionLocked(vaddr, size, false)) { if (cached->kind != CachedImage::Kind::DepthTarget || !cached->OverlapsRange(vaddr, size, false)) { continue; @@ -3564,9 +2744,18 @@ DepthStencilVulkanImage* TextureCache::FindDepthTargetByRange(uint64_t vaddr, ui " size=0x%016" PRIx64 "\n", vaddr, size); } - found = static_cast(cached->image); + found = cached; } - return found; + if (found == nullptr) { + return nullptr; + } + const auto owner = std::find_if(m_images.begin(), m_images.end(), + [found](const auto& image) { return image.get() == found; }); + if (owner == m_images.end()) { + EXIT("TextureCache: page-table depth target has no cache owner\n"); + } + command->RetainResourceUntilFence(*owner); + return static_cast(found->image); } RenderTextureVulkanImage* TextureCache::FindRenderTargetByRange(CommandBuffer* command, @@ -3612,59 +2801,30 @@ RenderTextureVulkanImage* TextureCache::FindRenderTargetByRange(CommandBuffer* c return static_cast(found->image); } -bool TextureCache::HasGpuTargetPageOverlap(uint64_t vaddr, uint64_t size) { +TextureCache::RegionInfo TextureCache::QueryRegion(uint64_t vaddr, uint64_t size) { if (vaddr == 0 || size == 0 || vaddr >= TRACKER_ADDRESS_SIZE || size > TRACKER_ADDRESS_SIZE - vaddr) { - EXIT("TextureCache: invalid GPU-target overlap query, addr=0x%016" PRIx64 + EXIT("TextureCache: invalid region query, addr=0x%016" PRIx64 " size=0x%016" PRIx64 "\n", vaddr, size); } FaultSafeTextureLock lock(this, m_lock); - for (const auto* cached: FindImagesInRegionLocked(vaddr, size, true)) { - if (cached->kind != CachedImage::Kind::Texture && - cached->OverlapsRange(vaddr, size, true)) { - return true; - } + RegionInfo result; + const auto candidates = FindImagesInRegionLocked(vaddr, size, true); + result.image_pages = !candidates.empty(); + for (const auto* cached: candidates) { + const bool bytes = cached->OverlapsRange(vaddr, size, false); + result.image_bytes |= bytes; + result.gpu_image_bytes |= bytes && cached->gpu_modified; + result.non_sampled_pages |= cached->kind != CachedImage::Kind::Texture; } - return false; -} - -bool TextureCache::HasPageOverlap(uint64_t vaddr, uint64_t size) { - if (vaddr == 0 || size == 0 || vaddr >= TRACKER_ADDRESS_SIZE || - size > TRACKER_ADDRESS_SIZE - vaddr) { - EXIT("TextureCache: invalid image page-overlap query, addr=0x%016" PRIx64 - " size=0x%016" PRIx64 "\n", - vaddr, size); + for (const auto& [address, metadata]: m_surface_metas) { + result.metadata_pages |= ImagePageRangesOverlap(vaddr, size, address, metadata.size); + result.metadata_bytes |= ImageRangeOverlaps(vaddr, size, address, metadata.size); } - FaultSafeTextureLock lock(this, m_lock); - return !FindImagesInRegionLocked(vaddr, size, true).empty(); -} - -bool TextureCache::HasRangeOverlap(uint64_t vaddr, uint64_t size) { - if (vaddr == 0 || size == 0 || vaddr >= TRACKER_ADDRESS_SIZE || - size > TRACKER_ADDRESS_SIZE - vaddr) { - EXIT("TextureCache: invalid image-range-overlap query, addr=0x%016" PRIx64 - " size=0x%016" PRIx64 "\n", - vaddr, size); - } - FaultSafeTextureLock lock(this, m_lock); - return !FindImagesInRegionLocked(vaddr, size, false).empty(); -} - -bool TextureCache::HasGpuModifiedRangeOverlap(uint64_t vaddr, uint64_t size) { - if (vaddr == 0 || size == 0 || vaddr >= TRACKER_ADDRESS_SIZE || - size > TRACKER_ADDRESS_SIZE - vaddr) { - EXIT("TextureCache: invalid GPU-modified image-overlap query, addr=0x%016" PRIx64 - " size=0x%016" PRIx64 "\n", - vaddr, size); - } - FaultSafeTextureLock lock(this, m_lock); - for (const auto* cached: FindImagesInRegionLocked(vaddr, size, false)) { - if (cached->gpu_modified && cached->OverlapsRange(vaddr, size, false)) { - return true; - } - } - return false; + result.gpu_metadata_bytes = + result.metadata_bytes && m_metadata_tracker.IsRegionGpuModified(vaddr, size); + return result; } void TextureCache::RegisterMeta(uint64_t vaddr, uint64_t size, uint32_t layers) { @@ -3770,52 +2930,6 @@ bool TextureCache::IsMetaRange(uint64_t vaddr, uint64_t size) { return it != m_surface_metas.end() && it->second.size == size; } -bool TextureCache::HasMetaRangeOverlap(uint64_t vaddr, uint64_t size) { - if (vaddr == 0 || size == 0 || vaddr >= TRACKER_ADDRESS_SIZE || - size > TRACKER_ADDRESS_SIZE - vaddr) { - EXIT("TextureCache: invalid metadata-range query, addr=0x%016" PRIx64 " size=0x%016" PRIx64 - "\n", - vaddr, size); - } - FaultSafeTextureLock lock(this, m_lock); - return std::any_of(m_surface_metas.begin(), m_surface_metas.end(), [&](const auto& entry) { - return ImageRangeOverlaps(vaddr, size, entry.first, entry.second.size); - }); -} - -bool TextureCache::HasMetaOverlap(uint64_t vaddr, uint64_t size) { - if (vaddr == 0 || size == 0 || vaddr >= TRACKER_ADDRESS_SIZE || - size > TRACKER_ADDRESS_SIZE - vaddr) { - EXIT("TextureCache: invalid metadata-overlap query, addr=0x%016" PRIx64 - " size=0x%016" PRIx64 "\n", - vaddr, size); - } - FaultSafeTextureLock lock(this, m_lock); - for (const auto& [address, meta]: m_surface_metas) { - if (ImagePageRangesOverlap(vaddr, size, address, meta.size)) { - return true; - } - } - return false; -} - -bool TextureCache::IsMetaGpuModified(uint64_t vaddr, uint64_t size) { - if (vaddr == 0 || size == 0 || vaddr >= TRACKER_ADDRESS_SIZE || - size > TRACKER_ADDRESS_SIZE - vaddr) { - EXIT("TextureCache: invalid metadata ownership query, addr=0x%016" PRIx64 - " size=0x%016" PRIx64 "\n", - vaddr, size); - } - FaultSafeTextureLock lock(this, m_lock); - for (const auto& [address, meta]: m_surface_metas) { - if (ImageRangeOverlaps(vaddr, size, address, meta.size) && - m_metadata_tracker.IsRegionGpuModified(vaddr, size)) { - return true; - } - } - return false; -} - bool TextureCache::HasMetaOverlapLocked(uint64_t vaddr, uint64_t size) const { for (const auto& [address, meta]: m_surface_metas) { if (ImagePageRangesOverlap(vaddr, size, address, meta.size)) { @@ -4119,199 +3233,16 @@ void TextureCache::UnmapMemory(uint64_t vaddr, uint64_t size) { } } -static constexpr int DummyTextureIndex(bool uint_format, bool image_3d) { - return (image_3d ? 2 : 0) + (uint_format ? 1 : 0); -} - -static constexpr uint32_t DummyTextureSwizzle() { - return Prospero::GpuEnumValue(Prospero::CompSwizzle::kRed) | - (Prospero::GpuEnumValue(Prospero::CompSwizzle::kGreen) << 3u) | - (Prospero::GpuEnumValue(Prospero::CompSwizzle::kBlue) << 6u) | - (Prospero::GpuEnumValue(Prospero::CompSwizzle::kAlpha) << 9u); -} - -static TextureImageCreateParams MakeDummyTextureParams(bool uint_format, bool image_3d, - TextureFormatUsage usage, - const char* owner) { - TextureImageCreateParams params {}; - params.fmt = static_cast( - Prospero::GpuEnumValue(uint_format ? Prospero::BufferFormat::k8_8_8_8UInt - : Prospero::BufferFormat::k8_8_8_8UNorm)); - params.width = 1; - params.height = 1; - params.base_level = 0; - params.levels = 1; - params.depth = 1; - params.type = Prospero::GpuEnumValue(image_3d ? Prospero::ImageType::kColor3D - : Prospero::ImageType::kColor2D); - params.swizzle = DummyTextureSwizzle(); - params.format_usage = usage; - params.required_format_usage = usage; - params.view_usage = usage; - params.image_layout = TextureUploadDestination::MipLevels; - params.allow_cube_view = true; - params.storage_swizzle_fallback = TextureHasFormatUsage(usage, TextureFormatUsage::Storage); - params.owner = owner; - return params; -} - -static VulkanImage* CreateDummyTexture(GraphicContext* ctx, bool uint_format, bool image_3d, - bool storage, VulkanMemory** memory) { - if (memory == nullptr || *memory != nullptr) { - EXIT("TextureCache: invalid dummy texture memory slot, slot=%p current=%p storage=%d\n", - static_cast(memory), - memory == nullptr ? nullptr : static_cast(*memory), storage); - } - auto* image = (storage ? static_cast(new StorageTextureVulkanImage) - : new TextureVulkanImage); - auto usage = (storage ? TextureFormatUsage::Storage : TextureFormatUsage::Sampled); - auto layout = (storage ? VK_IMAGE_LAYOUT_GENERAL : VK_IMAGE_LAYOUT_SHADER_READ_ONLY_OPTIMAL); - auto owner = (storage ? "DummyStorageTexture" : "DummySampledTexture"); - - *memory = new VulkanMemory {}; - auto params = MakeDummyTextureParams(uint_format, image_3d, usage, owner); - auto components = TextureCreateImage(ctx, image, *memory, params); - - static constexpr uint32_t zero = 0; - UtilFillImage(ctx, image, &zero, sizeof(zero), 1, static_cast(layout)); - TextureCreateImageViews(ctx, image, components, params.type, 0, params.base_level, - params.levels, params.depth, params.allow_cube_view, params.view_usage); - return image; -} - -void TextureCache::DeleteImageViews(GraphicContext* ctx, VulkanImage* image) { - EXIT_IF(ctx == nullptr); - EXIT_IF(image == nullptr); - - for (auto& cached: image->view_cache.views) { - if (cached.view != nullptr) { - vkDestroyImageView(ctx->device, cached.view, nullptr); - cached.view = nullptr; - } - } - image->view_cache.views.clear(); - for (auto& view: image->image_view) { - if (view != nullptr) { - vkDestroyImageView(ctx->device, view, nullptr); - view = nullptr; - } - } -} - -void TextureCache::DeleteGpuTexture(GraphicContext* ctx, GpuTextureVulkanImage* image, - VulkanMemory* mem) { - KYTY_PROFILER_BLOCK("TextureCache::DeleteGpuTexture"); - - EXIT_IF(ctx == nullptr); - EXIT_IF(image == nullptr); - - DeleteImageViews(ctx, image); - VulkanDeleteImage(ctx, image, mem); - - switch (image->type) { - case VulkanImageType::Texture: delete static_cast(image); break; - case VulkanImageType::StorageTexture: - delete static_cast(image); - break; - default: EXIT("unsupported gpu texture image type: %d\n", static_cast(image->type)); - } -} - -void TextureCache::DeleteRenderTexture(GraphicContext* ctx, RenderTextureVulkanImage* image, - VulkanMemory* mem) { - KYTY_PROFILER_BLOCK("TextureCache::DeleteRenderTexture"); - - EXIT_IF(ctx == nullptr); - EXIT_IF(image == nullptr); - - g_render_ctx->GetFramebufferCache()->FreeFramebufferByColor(image); - for (auto& view: image->render_view) { - if (view != nullptr) { - vkDestroyImageView(ctx->device, view, nullptr); - view = nullptr; - } - } - for (auto& cached: image->attachment_views) { - if (cached.view != nullptr) { - vkDestroyImageView(ctx->device, cached.view, nullptr); - cached.view = nullptr; - } - } - image->attachment_views.clear(); - DeleteImageViews(ctx, image); - VulkanDeleteImage(ctx, image, mem); - delete image; -} - -void TextureCache::DeleteDepthStencil(GraphicContext* ctx, DepthStencilVulkanImage* image, - VulkanMemory* mem) { - KYTY_PROFILER_BLOCK("TextureCache::DeleteDepthStencil"); - - EXIT_IF(ctx == nullptr); - EXIT_IF(image == nullptr); - - g_render_ctx->GetFramebufferCache()->FreeFramebufferByDepth(image); - for (auto& cached: image->attachment_views) { - if (cached.view != nullptr) { - vkDestroyImageView(ctx->device, cached.view, nullptr); - cached.view = nullptr; - } - } - image->attachment_views.clear(); - DeleteImageViews(ctx, image); - VulkanDeleteImage(ctx, image, mem); - delete image; -} - -void TextureCache::DeleteVideoOut(GraphicContext* ctx, VideoOutVulkanImage* image, - VulkanMemory* mem) { - KYTY_PROFILER_BLOCK("TextureCache::DeleteVideoOut"); - - EXIT_IF(ctx == nullptr); - EXIT_IF(image == nullptr); - - g_render_ctx->GetFramebufferCache()->FreeFramebufferByColor(image); - DeleteImageViews(ctx, image); - VulkanDeleteImage(ctx, image, mem); - delete image; -} - VulkanImage* TextureCache::GetDummySampledTexture(bool uint_format, bool image_3d) { KYTY_PROFILER_BLOCK("TextureCache::GetDummySampledTexture"); - - Common::LockGuard lock(m_dummy_mutex); - auto* ctx = g_render_ctx->GetGraphicCtx(); - if (m_dummy_ctx != nullptr && m_dummy_ctx != ctx) { - EXIT("TextureCache: sampled dummy texture context changed, previous=%p current=%p\n", - static_cast(m_dummy_ctx), static_cast(ctx)); - } - m_dummy_ctx = ctx; - const auto index = DummyTextureIndex(uint_format, image_3d); - auto*& image = m_dummy_sampled_textures[index]; - if (image == nullptr) { - image = - CreateDummyTexture(ctx, uint_format, image_3d, false, &m_dummy_sampled_memory[index]); - } - return image; + return m_dummy_textures->Get(g_render_ctx->GetGraphicCtx(), DummyTextureCache::Usage::Sampled, + uint_format, image_3d); } VulkanImage* TextureCache::GetDummyStorageTexture(bool uint_format, bool image_3d) { KYTY_PROFILER_BLOCK("TextureCache::GetDummyStorageTexture"); - - Common::LockGuard lock(m_dummy_mutex); - auto* ctx = g_render_ctx->GetGraphicCtx(); - if (m_dummy_ctx != nullptr && m_dummy_ctx != ctx) { - EXIT("TextureCache: storage dummy texture context changed, previous=%p current=%p\n", - static_cast(m_dummy_ctx), static_cast(ctx)); - } - m_dummy_ctx = ctx; - const auto index = DummyTextureIndex(uint_format, image_3d); - auto*& image = m_dummy_storage_textures[index]; - if (image == nullptr) { - image = - CreateDummyTexture(ctx, uint_format, image_3d, true, &m_dummy_storage_memory[index]); - } - return image; + return m_dummy_textures->Get(g_render_ctx->GetGraphicCtx(), DummyTextureCache::Usage::Storage, + uint_format, image_3d); } } // namespace Libs::Graphics diff --git a/src/graphics/host_gpu/renderer/textureCache.h b/src/graphics/host_gpu/renderer/textureCache.h index f989d3950..f79aa1e86 100644 --- a/src/graphics/host_gpu/renderer/textureCache.h +++ b/src/graphics/host_gpu/renderer/textureCache.h @@ -9,7 +9,6 @@ #include "graphics/host_gpu/renderer/multiLevelPageTable.h" #include "graphics/host_gpu/renderer/tiler.h" -#include #include #include #include @@ -29,10 +28,21 @@ struct VulkanImage; struct VulkanMemory; class BufferCache; class CommandBuffer; +class DummyTextureCache; class ResourceMutex; class TextureCache { public: + struct RegionInfo { + bool image_pages = false; + bool image_bytes = false; + bool gpu_image_bytes = false; + bool non_sampled_pages = false; + bool metadata_pages = false; + bool metadata_bytes = false; + bool gpu_metadata_bytes = false; + }; + TextureCache(PageManager& page_manager, BufferCache& buffer_cache, ResourceMutex& resource_mutex); ~TextureCache(); @@ -50,12 +60,8 @@ public: RegisterVideoOutSurfaces(GraphicContext* ctx, const std::vector& infos); void RefreshVideoOut(VideoOutVulkanImage* image, bool render_target = false); void UnregisterVideoOutSurfaces(const std::vector& images); - [[nodiscard]] bool ClearColorImageFromBuffer(CommandBuffer* command, uint64_t vaddr, - uint64_t size, uint32_t packed_clear); - [[nodiscard]] bool ClearDepthImageFromBuffer(CommandBuffer* command, uint64_t vaddr, - uint64_t size, uint32_t packed_clear); - [[nodiscard]] bool ClearStencilImageFromBuffer(CommandBuffer* command, uint64_t vaddr, - uint64_t size, uint32_t packed_clear); + [[nodiscard]] bool ClearImageFromBuffer(CommandBuffer* command, uint64_t vaddr, uint64_t size, + uint32_t packed_clear); void MarkGpuWritten(VulkanImage* image); void PrepareHostWrite(uint64_t vaddr, uint64_t size); [[nodiscard]] bool InvalidateMemoryFromGPU(uint64_t vaddr, uint64_t size, @@ -94,17 +100,12 @@ public: StorageTextureVulkanImage* image, uint32_t base_level); [[nodiscard]] DepthStencilVulkanImage* - FindDepthTargetByRange(uint64_t vaddr, uint64_t size, bool allow_containing_sampled = false); - [[nodiscard]] bool HasPageOverlap(uint64_t vaddr, uint64_t size); - [[nodiscard]] bool HasRangeOverlap(uint64_t vaddr, uint64_t size); - [[nodiscard]] bool HasGpuModifiedRangeOverlap(uint64_t vaddr, uint64_t size); - [[nodiscard]] bool HasGpuTargetPageOverlap(uint64_t vaddr, uint64_t size); + FindDepthTargetByRange(CommandBuffer* command, uint64_t vaddr, uint64_t size, + bool allow_containing_sampled = false); + [[nodiscard]] RegionInfo QueryRegion(uint64_t vaddr, uint64_t size); void RegisterMeta(uint64_t vaddr, uint64_t size, uint32_t layers = 1); [[nodiscard]] bool IsMeta(uint64_t vaddr); [[nodiscard]] bool IsMetaRange(uint64_t vaddr, uint64_t size); - [[nodiscard]] bool HasMetaRangeOverlap(uint64_t vaddr, uint64_t size); - [[nodiscard]] bool HasMetaOverlap(uint64_t vaddr, uint64_t size); - [[nodiscard]] bool IsMetaGpuModified(uint64_t vaddr, uint64_t size); [[nodiscard]] bool IsMetaCleared(uint64_t vaddr, uint32_t slice); [[nodiscard]] bool ClearMeta(uint64_t vaddr); [[nodiscard]] bool TouchMeta(uint64_t vaddr, uint32_t slice, bool is_clear); @@ -112,14 +113,6 @@ public: PageFaultPhase phase) noexcept; void UnmapMemory(uint64_t vaddr, uint64_t size); - static void DeleteGpuTexture(GraphicContext* ctx, GpuTextureVulkanImage* image, - VulkanMemory* mem); - static void DeleteRenderTexture(GraphicContext* ctx, RenderTextureVulkanImage* image, - VulkanMemory* mem); - static void DeleteDepthStencil(GraphicContext* ctx, DepthStencilVulkanImage* image, - VulkanMemory* mem); - static void DeleteVideoOut(GraphicContext* ctx, VideoOutVulkanImage* image, VulkanMemory* mem); - VulkanImage* GetDummySampledTexture(bool uint_format, bool image_3d); VulkanImage* GetDummyStorageTexture(bool uint_format, bool image_3d); @@ -133,7 +126,6 @@ private: uint32_t clear_mask = 0; bool gpu_modified = false; }; - static void DeleteImageViews(GraphicContext* ctx, VulkanImage* image); [[nodiscard]] VkImageView GetImageView(GraphicContext* ctx, VulkanImage* image, const ImageViewInfo& info); [[nodiscard]] bool HasMetaOverlapLocked(uint64_t vaddr, uint64_t size) const; @@ -145,6 +137,8 @@ private: void RetireStorageDepthAliasLocked(GraphicContext* ctx, const ImageInfo& requested); void RegisterImageLocked(CachedImage& image); void UnregisterImageLocked(CachedImage& image, bool release_tracking); + [[nodiscard]] VulkanImage* PublishImage(CommandBuffer* command, + std::shared_ptr image); [[nodiscard]] std::vector FindImagesInRegionLocked(uint64_t vaddr, uint64_t size, bool page_overlap); void RequireRetirementIsolation(const std::vector& retire, const char* operation, @@ -156,12 +150,7 @@ private: void SynchronizeDepthImageToBufferLocked(CachedImage& cached, uint64_t write_address, uint64_t write_size); - Common::Mutex m_dummy_mutex; - std::array m_dummy_sampled_textures {}; - std::array m_dummy_storage_textures {}; - std::array m_dummy_sampled_memory {}; - std::array m_dummy_storage_memory {}; - GraphicContext* m_dummy_ctx = nullptr; + std::unique_ptr m_dummy_textures; TrackingSpinLock m_lock; std::mutex m_fault_mutex; MemoryTracker m_memory_tracker; diff --git a/src/graphics/host_gpu/utils.cpp b/src/graphics/host_gpu/utils.cpp index 014756b04..c249f277c 100644 --- a/src/graphics/host_gpu/utils.cpp +++ b/src/graphics/host_gpu/utils.cpp @@ -405,6 +405,17 @@ public: std::memcpy(dst_data, m_mapped_data, size); } + void Release(GraphicContext* ctx) { + Common::LockGuard lock(m_mutex); + if (m_buffer.buffer != nullptr) { + EXIT_IF(ctx == nullptr || m_mapped_data == nullptr); + VulkanUnmapMemory(ctx, &m_buffer.memory); + VulkanDeleteBuffer(ctx, &m_buffer); + } + m_capacity = 0; + m_mapped_data = nullptr; + } + private: template void RecordUpload(GraphicContext* ctx, const void* src_data, uint64_t size, @@ -1078,6 +1089,17 @@ void UtilUploadBuffer(GraphicContext* ctx, StagingBufferType type, VulkanBuffer* GetStagingBuffer(type)->UploadToBuffer(ctx, dst_buffer, src_data, size, dst_offset); } +void UtilReleaseCachedResources(GraphicContext* ctx) { + EXIT_IF(ctx == nullptr); + g_texture_staging_buffer.Release(ctx); + g_vertex_staging_buffer.Release(ctx); + g_readback_staging_buffer.Release(ctx); + Common::LockGuard lock(g_compressed_image_copy_mutex); + if (g_compressed_image_copy_buffer.buffer != nullptr) { + VulkanDeleteBuffer(ctx, &g_compressed_image_copy_buffer); + } +} + void UtilCopyBuffer(VulkanBuffer* src_buffer, VulkanBuffer* dst_buffer, uint64_t size) { EXIT_IF(size == 0 || size > src_buffer->buffer_size || size > dst_buffer->buffer_size); diff --git a/src/graphics/host_gpu/utils.h b/src/graphics/host_gpu/utils.h index 2635a5e19..4a5edb354 100644 --- a/src/graphics/host_gpu/utils.h +++ b/src/graphics/host_gpu/utils.h @@ -110,6 +110,7 @@ void UtilFillBuffer(GraphicContext* ctx, void* dst_data, uint64_t size, uint64_t src_layout); void UtilUploadBuffer(GraphicContext* ctx, StagingBufferType type, VulkanBuffer* dst_buffer, uint64_t dst_offset, const void* src_data, uint64_t size); +void UtilReleaseCachedResources(GraphicContext* ctx); void UtilCopyBuffer(VulkanBuffer* src_buffer, VulkanBuffer* dst_buffer, uint64_t size); void UtilDownloadBuffer(GraphicContext* ctx, VulkanBuffer* src_buffer, uint64_t src_offset, void* dst_data, uint64_t size); diff --git a/tests/ShaderRecompilerComputeTests.cpp b/tests/ShaderRecompilerComputeTests.cpp index 5131cd347..d57135dd0 100644 --- a/tests/ShaderRecompilerComputeTests.cpp +++ b/tests/ShaderRecompilerComputeTests.cpp @@ -15,6 +15,7 @@ #include "graphics/host_gpu/renderer/image.h" #include "graphics/host_gpu/renderer/imageView.h" #include "graphics/host_gpu/renderer/render.h" +#include "graphics/host_gpu/renderer/renderContext.h" #include "graphics/host_gpu/renderer/sync.h" #include "graphics/host_gpu/renderer/renderDraw.h" #include "graphics/host_gpu/renderer/renderTarget.h" @@ -1138,6 +1139,14 @@ public: &context, &image, image.format, 0, 1, VK_IMAGE_VIEW_TYPE_2D, 0, 1); const auto storage_again = texture_cache.GetRenderTargetStorageView( &context, &image, image.format, 0, 1, VK_IMAGE_VIEW_TYPE_2D, 0, 1); + const auto attachment = texture_cache.GetRenderTargetAttachmentView( + &context, &image, image.format, 0, 0, 1); + const auto attachment_again = texture_cache.GetRenderTargetAttachmentView( + &context, &image, image.format, 0, 0, 1); + const auto attachment_mip = texture_cache.GetRenderTargetAttachmentView( + &context, &image, image.format, 1, 0, 1); + const auto attachment_array = texture_cache.GetRenderTargetAttachmentView( + &context, &image, image.format, 0, 0, 2); Require(name, "cache identity", first != VK_NULL_HANDLE && first_again == first && second != VK_NULL_HANDLE && second != first && @@ -1148,14 +1157,17 @@ public: reinterpreted_format != VK_NULL_HANDLE && reinterpreted_format != native_format && storage != VK_NULL_HANDLE && storage != native_format && - storage_again == storage && image.view_cache.views.size() == 8, - "view cache omitted usage, swizzle, format, type, mip, or layer " - "identity"); + storage_again == storage && attachment != VK_NULL_HANDLE && + attachment_again == attachment && attachment != native_format && + attachment != storage && attachment_mip != VK_NULL_HANDLE && + attachment_mip != attachment && attachment_array != VK_NULL_HANDLE && + attachment_array != attachment && image.view_cache.views.size() == 11, + "view cache omitted attachment usage, swizzle, format, type, mip, " + "or layer identity"); - for (const auto &view : image.view_cache.views) { - vkDestroyImageView(m_device, view.view, nullptr); - } - image.view_cache.views.clear(); + ImageViewOps::DestroyViews(&context, &image); + Require(name, "view teardown", image.view_cache.views.empty(), + "dynamic render-target views survived DestroyViews"); vkDestroyImage(m_device, image_handle, nullptr); vkFreeMemory(m_device, memory, nullptr); std::printf("[host] %-32s ok\n", name); @@ -1232,19 +1244,30 @@ public: const auto array_view = texture_cache.GetDepthTargetSampledView( &context, &image, VK_FORMAT_R32_SFLOAT, replicated, 0, 1, VK_IMAGE_VIEW_TYPE_2D_ARRAY, 0, 2); + const auto attachment = texture_cache.GetDepthTargetAttachmentView( + &context, &image, 0, 1); + const auto attachment_again = texture_cache.GetDepthTargetAttachmentView( + &context, &image, 0, 1); + const auto attachment_layer = texture_cache.GetDepthTargetAttachmentView( + &context, &image, 1, 1); + const auto attachment_array = texture_cache.GetDepthTargetAttachmentView( + &context, &image, 0, 2); Require(name, "cache identity", first != VK_NULL_HANDLE && first_again == first && different_swizzle != VK_NULL_HANDLE && different_swizzle != first && different_layer != VK_NULL_HANDLE && different_layer != first && array_view != VK_NULL_HANDLE && array_view != first && - image.view_cache.views.size() == 4, - "sampled depth cache omitted swizzle, type, or layer identity"); + attachment != VK_NULL_HANDLE && attachment_again == attachment && + attachment != first && attachment_layer != VK_NULL_HANDLE && + attachment_layer != attachment && attachment_array != VK_NULL_HANDLE && + attachment_array != attachment && image.view_cache.views.size() == 7, + "depth view cache omitted attachment usage, swizzle, type, or layer " + "identity"); - for (const auto &view : image.view_cache.views) { - vkDestroyImageView(m_device, view.view, nullptr); - } - image.view_cache.views.clear(); + ImageViewOps::DestroyViews(&context, &image); + Require(name, "view teardown", image.view_cache.views.empty(), + "dynamic depth-target views survived DestroyViews"); vkDestroyImage(m_device, image_handle, nullptr); vkFreeMemory(m_device, memory, nullptr); std::printf("[host] %-32s ok\n", name); @@ -1255,7 +1278,7 @@ public: auto backing = CreateImage2D(name, 8, 8, VK_FORMAT_R8G8B8A8_UNORM, VK_IMAGE_USAGE_COLOR_ATTACHMENT_BIT | VK_IMAGE_USAGE_SAMPLED_BIT, - {}, 1, VK_IMAGE_LAYOUT_UNDEFINED); + {}, 1, VK_IMAGE_LAYOUT_SHADER_READ_ONLY_OPTIMAL); GraphicContext context{}; context.physical_device = m_physical_device; @@ -1296,14 +1319,123 @@ public: "video-out sampled views did not use identity fast path and lazy " "mappings"); - for (const auto &view : image.view_cache.views) { - vkDestroyImageView(m_device, view.view, nullptr); - } - image.view_cache.views.clear(); + ImageViewOps::DestroyViews(&context, &image); + Require(name, "view teardown", + image.view_cache.views.empty() && + std::all_of(std::begin(image.image_view), + std::end(image.image_view), + [](VkImageView view) { return view == VK_NULL_HANDLE; }), + "dynamic or fixed video-out views survived DestroyViews"); + backing.view = VK_NULL_HANDLE; DestroyImage(&backing); std::printf("[host] %-32s ok\n", name); } +#if KYTY_PLATFORM == KYTY_PLATFORM_WINDOWS + void CheckQueryRegionImageClassification() { + constexpr const char *name = "QueryRegionImageClassification"; + constexpr uintptr_t base = 0x0000000200100000ull; + constexpr uint64_t allocation_size = 0x4000; + auto *memory = static_cast(VirtualAlloc( + reinterpret_cast(base), allocation_size, + MEM_RESERVE | MEM_COMMIT, PAGE_READWRITE)); + Require(name, "allocation", memory == reinterpret_cast(base), + "fixed VirtualAlloc failed"); + std::memset(memory, 0, allocation_size); + + EnsureRuntimeContext(); + ResourceMutex resource_mutex; + PageManager page_manager(RejectUnexpectedPageFault, nullptr); + BufferCache buffer_cache(page_manager, resource_mutex); + TextureCache texture_cache(page_manager, buffer_cache, resource_mutex); + buffer_cache.SetTextureCache(texture_cache); + page_manager.OnGpuMap(base, allocation_size); + + constexpr auto format = + Prospero::GpuEnumValue(Prospero::BufferFormat::k8_8_8_8UNorm); + constexpr auto linear = + Prospero::GpuEnumValue(Prospero::TileMode::kLinear); + ImageInfo sampled_info{}; + sampled_info.address = base + 0x100; + sampled_info.format = format; + sampled_info.width = 4; + sampled_info.height = 4; + sampled_info.pitch = TileGetTexturePitch(format, sampled_info.width, 1, linear); + sampled_info.tile = linear; + sampled_info.type = Prospero::GpuEnumValue(Prospero::ImageType::kColor2D); + TileSizeAlign sampled_size{}; + TileGetTextureTotalSize(format, sampled_info.width, sampled_info.height, + sampled_info.depth, sampled_info.pitch, + sampled_info.levels, linear, false, &sampled_size); + sampled_info.size = sampled_size.size; + Require(name, "sampled fixture", + sampled_info.size != 0 && sampled_size.align != 0 && + sampled_info.size <= 0x600 && + sampled_info.address % sampled_size.align == 0, + "sampled image does not fit the intended sub-page fixture"); + + RenderTargetInfo target_info{}; + target_info.address = base + 0x1100; + target_info.size = 0x100; + target_info.format = VK_FORMAT_R8G8B8A8_UNORM; + target_info.width = 4; + target_info.height = 4; + target_info.pitch = 4; + target_info.bytes_per_element = 4; + target_info.tile_mode = linear; + + { + CommandBuffer command(GraphicContext::QUEUE_GFX); + auto *sampled = + texture_cache.FindTexture(&command, &m_runtime_context, sampled_info, false); + auto *target = + texture_cache.FindRenderTarget(&command, &m_runtime_context, target_info); + Require(name, "registration", sampled != nullptr && target != nullptr, + "sampled or non-sampled image registration failed"); + + const auto sampled_bytes = + texture_cache.QueryRegion(sampled_info.address + 0x20, 0x20); + const auto sampled_page = texture_cache.QueryRegion(base + 0x800, 0x20); + Require(name, "sampled classification", + sampled_bytes.image_pages && sampled_bytes.image_bytes && + !sampled_bytes.gpu_image_bytes && + !sampled_bytes.non_sampled_pages && sampled_page.image_pages && + !sampled_page.image_bytes && !sampled_page.gpu_image_bytes && + !sampled_page.non_sampled_pages, + "sampled image page and byte ownership were not separated"); + + const auto target_bytes = + texture_cache.QueryRegion(target_info.address + 0x20, 0x20); + const auto target_page = + texture_cache.QueryRegion(target_info.address + 0x400, 0x20); + Require(name, "non-sampled classification", + target_bytes.image_pages && target_bytes.image_bytes && + !target_bytes.gpu_image_bytes && target_bytes.non_sampled_pages && + target_page.image_pages && !target_page.image_bytes && + !target_page.gpu_image_bytes && target_page.non_sampled_pages, + "non-sampled image page and byte ownership were not separated"); + + texture_cache.MarkGpuWritten(target); + const auto gpu_bytes = + texture_cache.QueryRegion(target_info.address + 0x20, 0x20); + const auto gpu_page = + texture_cache.QueryRegion(target_info.address + 0x400, 0x20); + Require(name, "GPU ownership", + gpu_bytes.image_pages && gpu_bytes.image_bytes && + gpu_bytes.gpu_image_bytes && gpu_bytes.non_sampled_pages && + gpu_page.image_pages && !gpu_page.image_bytes && + !gpu_page.gpu_image_bytes && gpu_page.non_sampled_pages, + "GPU ownership escaped the target's exact byte range"); + + texture_cache.UnmapMemory(base, allocation_size); + page_manager.OnGpuUnmap(base, allocation_size); + } + Require(name, "free", VirtualFree(memory, 0, MEM_RELEASE) != 0, + "VirtualFree failed"); + std::printf("[host] %-32s ok\n", name); + } +#endif + Buffer CreateStorageBuffer(const char *shader_name, const std::vector &initial, size_t dword_count) { @@ -2238,6 +2370,37 @@ public: } private: + void EnsureRuntimeContext() { + if (m_runtime_context.allocator != VK_NULL_HANDLE) { + return; + } + m_runtime_context.instance = m_instance; + m_runtime_context.physical_device = m_physical_device; + m_runtime_context.device = m_device; + vkGetPhysicalDeviceProperties(m_physical_device, + &m_runtime_context.physical_device_properties); + m_runtime_context.physical_device_memory_properties = m_memory_properties; + for (auto &queue : m_runtime_context.queues) { + queue.mutex = &m_runtime_queue_mutex; + queue.family = m_queue_family; + queue.index = 0; + queue.vk_queue = m_queue; + } + + VmaAllocatorCreateInfo allocator_info{}; + allocator_info.instance = m_instance; + allocator_info.physicalDevice = m_physical_device; + allocator_info.device = m_device; + allocator_info.vulkanApiVersion = VK_API_VERSION_1_2; + RequireVk("VulkanHarness", "runtime context", + vmaCreateAllocator(&allocator_info, &m_runtime_context.allocator), + "vmaCreateAllocator"); + if (g_render_ctx == nullptr) { + GraphicsRenderInit(); + } + g_render_ctx->SetGraphicCtx(&m_runtime_context); + } + void Init() { VkApplicationInfo app{}; app.sType = VK_STRUCTURE_TYPE_APPLICATION_INFO; @@ -2330,6 +2493,12 @@ private: void Destroy() { if (m_device != VK_NULL_HANDLE) { vkDeviceWaitIdle(m_device); + if (m_runtime_context.allocator != VK_NULL_HANDLE) { + GraphicsRenderReleaseThreadCommandPools(); + UtilReleaseCachedResources(&m_runtime_context); + vmaDestroyAllocator(m_runtime_context.allocator); + m_runtime_context.allocator = VK_NULL_HANDLE; + } if (m_command_pool != VK_NULL_HANDLE) { vkDestroyCommandPool(m_device, m_command_pool, nullptr); } @@ -2565,6 +2734,8 @@ private: VkCommandPool m_command_pool = VK_NULL_HANDLE; u32 m_queue_family = 0; VkPhysicalDeviceMemoryProperties m_memory_properties{}; + Common::Mutex m_runtime_queue_mutex; + GraphicContext m_runtime_context{}; }; void CompareWords(const TestCase &test, const char *stage, @@ -11135,7 +11306,7 @@ void CheckOverlappingMetadataViews() { page_manager.HandleFault(PageFaultAccess::Write, second), "shared metadata page did not transfer to CPU ownership"); Require("OverlappingMetadataViews", "ownership", - !texture_cache.IsMetaGpuModified(second, 0x1000), + !texture_cache.QueryRegion(second, 0x1000).gpu_metadata_bytes, "shared metadata page retained GPU ownership after a CPU fault"); texture_cache.UnmapMemory(base, allocation_size); @@ -11145,6 +11316,62 @@ void CheckOverlappingMetadataViews() { std::printf("[host] %-32s ok\n", "OverlappingMetadataViews"); } +void CheckQueryRegionAggregation() { + constexpr uintptr_t base = 0x0000000200010000ull; + constexpr uint64_t allocation_size = 0x10000; + constexpr uint64_t metadata_size = 0x180; + auto *memory = static_cast(VirtualAlloc( + reinterpret_cast(base), allocation_size, + MEM_RESERVE | MEM_COMMIT, PAGE_READWRITE)); + Require("QueryRegionAggregation", "allocation", + memory == reinterpret_cast(base), "fixed VirtualAlloc failed"); + + ResourceMutex resource_mutex; + CacheFaultContext fault_context; + PageManager page_manager(CacheFault, &fault_context); + BufferCache buffer_cache(page_manager, resource_mutex); + TextureCache texture_cache(page_manager, buffer_cache, resource_mutex); + fault_context.texture = &texture_cache; + buffer_cache.SetTextureCache(texture_cache); + page_manager.OnGpuMap(base, allocation_size); + + const auto empty = texture_cache.QueryRegion(base + 0x20, 0x40); + Require("QueryRegionAggregation", "empty", + !empty.image_pages && !empty.image_bytes && !empty.gpu_image_bytes && + !empty.non_sampled_pages && !empty.metadata_pages && + !empty.metadata_bytes && !empty.gpu_metadata_bytes, + "empty region reported cached ownership"); + + texture_cache.RegisterMeta(base, metadata_size); + const auto bytes = texture_cache.QueryRegion(base + 0x20, 0x40); + const auto page_only = texture_cache.QueryRegion(base + 0x400, 0x40); + const auto disjoint = texture_cache.QueryRegion(base + 0x1000, 0x40); + Require("QueryRegionAggregation", "page and byte overlap", + bytes.metadata_pages && bytes.metadata_bytes && + !bytes.gpu_metadata_bytes && page_only.metadata_pages && + !page_only.metadata_bytes && !page_only.gpu_metadata_bytes && + !disjoint.metadata_pages && !disjoint.metadata_bytes && + !bytes.image_pages && !bytes.image_bytes && + !bytes.gpu_image_bytes && !bytes.non_sampled_pages, + "metadata page candidates were not separated from exact byte overlap"); + + Require("QueryRegionAggregation", "clear", texture_cache.ClearMeta(base), + "metadata clear setup failed"); + const auto gpu_bytes = texture_cache.QueryRegion(base + 0x20, 0x40); + const auto gpu_page_only = texture_cache.QueryRegion(base + 0x400, 0x40); + Require("QueryRegionAggregation", "GPU ownership", + gpu_bytes.metadata_pages && gpu_bytes.metadata_bytes && + gpu_bytes.gpu_metadata_bytes && gpu_page_only.metadata_pages && + !gpu_page_only.metadata_bytes && !gpu_page_only.gpu_metadata_bytes, + "GPU metadata ownership escaped its exact registered byte range"); + + texture_cache.UnmapMemory(base, allocation_size); + page_manager.OnGpuUnmap(base, allocation_size); + Require("QueryRegionAggregation", "free", VirtualFree(memory, 0, MEM_RELEASE) != 0, + "VirtualFree failed"); + std::printf("[host] %-32s ok\n", "QueryRegionAggregation"); +} + void CheckGpuMetadataReuse() { constexpr uintptr_t base = 0x0000000200010000ull; constexpr uint64_t allocation_size = 0x20000; @@ -11173,7 +11400,7 @@ void CheckGpuMetadataReuse() { texture_cache.InvalidateMemoryFromGPU(base, allocation_size); Require("GpuMetadataReuse", "discard", !full_image_transition && !texture_cache.IsMeta(base) && - !texture_cache.HasMetaRangeOverlap(base, allocation_size), + !texture_cache.QueryRegion(base, allocation_size).metadata_bytes, "metadata-only overwrite retained identity or claimed an image transition"); texture_cache.RegisterMeta(base, metadata_size, layers); Require("GpuMetadataReuse", "re-register", @@ -12721,8 +12948,8 @@ void CheckHostDmaMetadataReuse() { reinterpret_cast(memory + 0x1ffc)[0] == 0x11223344, "post-clear CPU DMA fill did not publish backing"); Require("HostDmaMetadataReuse", "identity", - texture_cache.HasMetaOverlap(base, metadata_size) && - !texture_cache.IsMetaGpuModified(base, 0x2000) && + texture_cache.QueryRegion(base, metadata_size).metadata_pages && + !texture_cache.QueryRegion(base, 0x2000).gpu_metadata_bytes && VirtualQuery(memory, &protection, sizeof(protection)) != 0 && protection.Protect == PAGE_READWRITE, "CPU writes erased metadata identity or retained virtual ownership"); @@ -12827,6 +13054,9 @@ int main(int argc, char **argv) { } if (argc == 2 && std::strcmp(argv[1], "--image-overlap-only") == 0) { CheckImageOverlapResolution(); + CheckQueryRegionAggregation(); + VulkanHarness vulkan; + vulkan.CheckQueryRegionImageClassification(); return 0; } if (argc == 2 && std::strcmp(argv[1], "--htile-clear-only") == 0) { @@ -12935,6 +13165,7 @@ int main(int argc, char **argv) { CheckGpuMetadataReuse(); CheckMetadataReuseDescriptors(); CheckImageOverlapResolution(); + CheckQueryRegionAggregation(); CheckMsaaCompatibility(); CheckDepthHtileStencilCompatibility(); CheckStencilAttachmentAccess(); @@ -12951,6 +13182,7 @@ int main(int argc, char **argv) { CheckEmbeddedFetchLaneSpill(); CheckPs5GameExampleImageClearRuntimeShape(); VulkanHarness vulkan; + vulkan.CheckQueryRegionImageClassification(); vulkan.CheckMutableStorageSrgbView(); vulkan.CheckMutableRenderTargetBgraStorageView(); vulkan.CheckRenderTargetViewCache();