From 999f87576e335f523b8a515d686d97fa0b685f2a Mon Sep 17 00:00:00 2001 From: nmzik Date: Sun, 19 Jul 2026 06:26:01 +0200 Subject: [PATCH] graphics: add native PS5 MSAA and safe image allocation reuse --- src/graphics/guest_gpu/tile.cpp | 108 ++- src/graphics/guest_gpu/tile.h | 12 +- src/graphics/host_gpu/graphicContext.h | 6 +- .../host_gpu/renderer/colorRenderTarget.cpp | 70 +- .../host_gpu/renderer/colorRenderTarget.h | 1 + src/graphics/host_gpu/renderer/context.cpp | 8 + src/graphics/host_gpu/renderer/debug.cpp | 13 +- .../host_gpu/renderer/depthRenderTarget.cpp | 142 ++- .../host_gpu/renderer/depthRenderTarget.h | 1 + .../host_gpu/renderer/descriptors.cpp | 6 + .../host_gpu/renderer/framebufferCache.cpp | 42 +- .../host_gpu/renderer/framebufferCache.h | 5 +- src/graphics/host_gpu/renderer/image.cpp | 46 +- src/graphics/host_gpu/renderer/imageInfo.h | 272 +++++- src/graphics/host_gpu/renderer/imageView.cpp | 2 +- .../host_gpu/renderer/pipelineCache.cpp | 8 + .../host_gpu/renderer/pipelineCache.h | 6 +- .../host_gpu/renderer/renderCompute.cpp | 72 +- src/graphics/host_gpu/renderer/renderTarget.h | 16 +- src/graphics/host_gpu/renderer/shaders.cpp | 4 +- .../host_gpu/renderer/textureCache.cpp | 813 +++++++++++++----- src/graphics/host_gpu/renderer/textureCache.h | 25 +- src/graphics/host_gpu/renderer/tiler.cpp | 39 +- src/graphics/host_gpu/renderer/tiler.h | 1 + src/graphics/host_gpu/vulkanInstance.h | 1 + .../presentation/window/vulkanWindow.cpp | 4 +- src/graphics/shader/shader.cpp | 7 +- src/graphics/shader/shader.h | 1 + tests/ShaderRecompilerComputeTests.cpp | 557 +++++++++--- 29 files changed, 1685 insertions(+), 603 deletions(-) diff --git a/src/graphics/guest_gpu/tile.cpp b/src/graphics/guest_gpu/tile.cpp index 6499fba65..744a21c1b 100644 --- a/src/graphics/guest_gpu/tile.cpp +++ b/src/graphics/guest_gpu/tile.cpp @@ -11,6 +11,7 @@ #include #include +#include #include #include #include @@ -329,6 +330,30 @@ static bool Gen5Thin64KBBlockSizeFromElementBytes(uint32_t bytes_per_element, ui return false; } +static bool Gen5Msaa64KBBlockSizeFromElementBytes(uint32_t bytes_per_element, + uint32_t num_fragments_log2, + uint32_t* block_width, uint32_t* block_height) { + if (num_fragments_log2 == 0) { + return Gen5Thin64KBBlockSizeFromElementBytes(bytes_per_element, block_width, block_height); + } + if (num_fragments_log2 > 3 || !std::has_single_bit(bytes_per_element) || + bytes_per_element > 16) { + return false; + } + // AGC kLog2BlockSizeMsaa. Row zero is the ordinary thin 64 KiB layout; rows 1..3 + // describe 2x, 4x and 8x depth/render-target blocks respectively. + static constexpr uint8_t LOG2_BLOCK[4][5][2] = { + {{8, 8}, {8, 7}, {7, 7}, {7, 6}, {6, 6}}, + {{7, 8}, {7, 7}, {6, 7}, {6, 6}, {5, 6}}, + {{7, 7}, {7, 6}, {6, 6}, {6, 5}, {5, 5}}, + {{6, 7}, {6, 6}, {5, 6}, {5, 5}, {4, 5}}, + }; + const auto bytes_log2 = std::countr_zero(bytes_per_element); + *block_width = 1u << LOG2_BLOCK[num_fragments_log2][bytes_log2][0]; + *block_height = 1u << LOG2_BLOCK[num_fragments_log2][bytes_log2][1]; + return true; +} + struct Gen5MipTailLocation { uint32_t x; uint32_t y; @@ -2262,35 +2287,59 @@ void TileConvertLinearToTiledRenderTarget(void* dst, const void* src, uint32_t w } } +bool TileGetHtileSize(uint32_t width, uint32_t height, TileSizeAlign* htile_size) { + *htile_size = {}; + if (width == 0 || width > 16384 || height == 0 || height > 16384) { + return false; + } + // Prospero HTile stores one DWORD per depth tile. Its 32 KiB allocation blocks cover + // 1024x512 pixels, independently of the attachment's fragment count. + const uint64_t size = static_cast(AlignUp(width, 1024u) / 1024u) * + (AlignUp(height, 512u) / 512u) * 32768u; + if (size == 0 || size > UINT32_MAX) { + return false; + } + *htile_size = {static_cast(size), 32768}; + return true; +} + bool TileGetDepthSize(uint32_t width, uint32_t height, uint32_t pitch, uint32_t z_format, uint32_t stencil_format, bool htile, TileSizeAlign* stencil_size, - TileSizeAlign* htile_size, TileSizeAlign* depth_size) { + TileSizeAlign* htile_size, TileSizeAlign* depth_size, + uint32_t num_fragments_log2) { EXIT_IF(pitch != 0); - // Prospero derives uncompressed depth/stencil as independent 64 KiB block surfaces and HTile - // as 32 KiB metadata blocks covering 1024x512 pixels for a single-mip, single-slice target. + // Prospero derives uncompressed depth/stencil as independent 64 KiB block surfaces. if (width > 0 && width <= 16384 && height > 0 && height <= 16384 && - (z_format == 1 || z_format == 3) && stencil_format <= 1) { - const uint32_t depth_bytes = z_format == 1 ? 2u : 4u; - const uint32_t depth_block_width = z_format == 1 ? 256u : 128u; + (z_format == 1 || z_format == 3) && stencil_format <= 1 && num_fragments_log2 <= 3) { + const uint32_t depth_bytes = z_format == 1 ? 2u : 4u; + uint32_t depth_block_width = 0; + uint32_t depth_block_height = 0; + uint32_t stencil_block_width = 0; + uint32_t stencil_block_height = 0; + const bool valid_blocks = + Gen5Msaa64KBBlockSizeFromElementBytes(depth_bytes, num_fragments_log2, + &depth_block_width, &depth_block_height) && + (stencil_format == 0 || + Gen5Msaa64KBBlockSizeFromElementBytes(1, num_fragments_log2, &stencil_block_width, + &stencil_block_height)); + const uint32_t fragments = 1u << num_fragments_log2; const uint64_t depth_bytes_total = - static_cast(AlignUp(width, depth_block_width)) * AlignUp(height, 128u) * - depth_bytes; + valid_blocks ? static_cast(AlignUp(width, depth_block_width)) * + AlignUp(height, depth_block_height) * depth_bytes * fragments + : 0; const uint64_t stencil_bytes_total = - stencil_format == 1 - ? static_cast(AlignUp(width, 256u)) * AlignUp(height, 256u) + stencil_format == 1 && valid_blocks + ? static_cast(AlignUp(width, stencil_block_width)) * + AlignUp(height, stencil_block_height) * fragments : 0; - const uint64_t htile_bytes_total = - htile ? static_cast(AlignUp(width, 1024u) / 1024u) * - (AlignUp(height, 512u) / 512u) * 32768u - : 0; - if (depth_bytes_total <= UINT32_MAX && stencil_bytes_total <= UINT32_MAX && - htile_bytes_total <= UINT32_MAX) { + TileSizeAlign calculated_htile {}; + const bool htile_valid = !htile || TileGetHtileSize(width, height, &calculated_htile); + if (depth_bytes_total <= UINT32_MAX && stencil_bytes_total <= UINT32_MAX && htile_valid) { *depth_size = {static_cast(depth_bytes_total), 65536}; *stencil_size = stencil_format == 1 ? TileSizeAlign {static_cast(stencil_bytes_total), 65536} : TileSizeAlign {}; - *htile_size = htile ? TileSizeAlign {static_cast(htile_bytes_total), 32768} - : TileSizeAlign {}; + *htile_size = calculated_htile; return true; } } @@ -2300,11 +2349,12 @@ bool TileGetDepthSize(uint32_t width, uint32_t height, uint32_t pitch, uint32_t return false; } -uint32_t TileGetRenderTargetPitch(uint32_t width, uint32_t bytes_per_element) { +uint32_t TileGetRenderTargetPitch(uint32_t width, uint32_t bytes_per_element, + uint32_t num_fragments_log2) { uint32_t block_width = 0; uint32_t block_height = 0; - if (width == 0 || - !Gen5Thin64KBBlockSizeFromElementBytes(bytes_per_element, &block_width, &block_height)) { + if (width == 0 || !Gen5Msaa64KBBlockSizeFromElementBytes(bytes_per_element, num_fragments_log2, + &block_width, &block_height)) { return 0; } const uint64_t pitch = (static_cast(width) + block_width - 1u) & @@ -2312,19 +2362,27 @@ uint32_t TileGetRenderTargetPitch(uint32_t width, uint32_t bytes_per_element) { return pitch <= UINT32_MAX ? static_cast(pitch) : 0; } +uint32_t TileGetDepthPitch(uint32_t width, uint32_t bytes_per_element, + uint32_t num_fragments_log2) { + return TileGetRenderTargetPitch(width, bytes_per_element, num_fragments_log2); +} + bool TileGetRenderTargetSize(uint32_t width, uint32_t height, uint32_t pitch, - uint32_t bytes_per_element, TileSizeAlign* total_size) { + uint32_t bytes_per_element, TileSizeAlign* total_size, + uint32_t num_fragments_log2) { *total_size = {}; uint32_t block_width = 0; uint32_t block_height = 0; if (height == 0 || pitch == 0 || - !Gen5Thin64KBBlockSizeFromElementBytes(bytes_per_element, &block_width, &block_height) || - pitch != TileGetRenderTargetPitch(width, bytes_per_element)) { + !Gen5Msaa64KBBlockSizeFromElementBytes(bytes_per_element, num_fragments_log2, &block_width, + &block_height) || + pitch != TileGetRenderTargetPitch(width, bytes_per_element, num_fragments_log2)) { return false; } const uint64_t padded_height = (static_cast(height) + block_height - 1u) & ~static_cast(block_height - 1u); - const uint64_t size = static_cast(pitch) * padded_height * bytes_per_element; + const uint64_t size = static_cast(pitch) * padded_height * bytes_per_element * + (1u << num_fragments_log2); if (size == 0 || size > UINT32_MAX) { return false; } diff --git a/src/graphics/guest_gpu/tile.h b/src/graphics/guest_gpu/tile.h index 49a3d8cb0..85016c204 100644 --- a/src/graphics/guest_gpu/tile.h +++ b/src/graphics/guest_gpu/tile.h @@ -80,12 +80,18 @@ bool TileGetStandard4KBVolumeLayout(uint32_t format, uint32_t* bytes_per_element uint32_t* texels_per_element_tall, uint32_t* block_width_log2, uint32_t* block_height_log2, uint32_t* block_depth_log2); +bool TileGetHtileSize(uint32_t width, uint32_t height, TileSizeAlign* htile_size); bool TileGetDepthSize(uint32_t width, uint32_t height, uint32_t pitch, uint32_t z_format, uint32_t stencil_format, bool htile, TileSizeAlign* stencil_size, - TileSizeAlign* htile_size, TileSizeAlign* depth_size); -uint32_t TileGetRenderTargetPitch(uint32_t width, uint32_t bytes_per_element); + TileSizeAlign* htile_size, TileSizeAlign* depth_size, + uint32_t num_fragments_log2 = 0); +uint32_t TileGetRenderTargetPitch(uint32_t width, uint32_t bytes_per_element, + uint32_t num_fragments_log2 = 0); +uint32_t TileGetDepthPitch(uint32_t width, uint32_t bytes_per_element, + uint32_t num_fragments_log2 = 0); bool TileGetRenderTargetSize(uint32_t width, uint32_t height, uint32_t pitch, - uint32_t bytes_per_element, TileSizeAlign* total_size); + uint32_t bytes_per_element, TileSizeAlign* total_size, + uint32_t num_fragments_log2 = 0); bool TileGetRenderTargetMipLayout(uint32_t width, uint32_t height, uint32_t pitch, uint32_t bytes_per_element, uint32_t levels, TileSizeAlign* total_size, TileSizeOffset* level_sizes, diff --git a/src/graphics/host_gpu/graphicContext.h b/src/graphics/host_gpu/graphicContext.h index bde77d295..7d19dbecc 100644 --- a/src/graphics/host_gpu/graphicContext.h +++ b/src/graphics/host_gpu/graphicContext.h @@ -111,6 +111,7 @@ struct VulkanImage { uint32_t guest_pitch = 0; uint32_t layers = 1; uint32_t mip_levels = 1; + uint32_t samples = 1; vk::Image image = nullptr; vk::ImageView image_view[VIEW_MAX] = {}; vk::ImageLayout layout = vk::ImageLayout::eUndefined; @@ -124,7 +125,9 @@ struct VideoOutVulkanImage: public VulkanImage { struct DepthStencilVulkanImage: public VulkanImage { DepthStencilVulkanImage(): VulkanImage(VulkanImageType::DepthStencil) {} - bool compressed = false; + bool compressed = false; + bool initial_depth_clear_pending = false; + bool initial_stencil_clear_pending = false; }; struct GpuTextureVulkanImage: public VulkanImage { @@ -141,6 +144,7 @@ struct StorageTextureVulkanImage: public GpuTextureVulkanImage { struct RenderTextureVulkanImage: public VulkanImage { RenderTextureVulkanImage(): VulkanImage(VulkanImageType::RenderTexture) {} + bool initial_clear_pending = false; }; struct VulkanBuffer { diff --git a/src/graphics/host_gpu/renderer/colorRenderTarget.cpp b/src/graphics/host_gpu/renderer/colorRenderTarget.cpp index 0960fc81b..39652b3c3 100644 --- a/src/graphics/host_gpu/renderer/colorRenderTarget.cpp +++ b/src/graphics/host_gpu/renderer/colorRenderTarget.cpp @@ -66,14 +66,15 @@ void ResolveRenderColorTarget(uint64_t submit_id, CommandBuffer* buffer, const H r->extent = {}; r->base_mip_level = 0; r->buffer_size = 0; + r->samples = 1; r->color_clear_enable = false; r->color_clear_value = {}; return; } - const bool msaa_compat = - color_msaa_single_sample_compatible(rt.attrib.num_samples, rt.attrib.num_fragments); - if (!msaa_compat && (rt.attrib.num_samples != 0 || rt.attrib.num_fragments != 0)) { - EXIT("multisampled render targets are unsupported\n"); + const auto samples = render_sample_count(rt.attrib.num_fragments); + if (samples == 0 || rt.attrib.num_samples != rt.attrib.num_fragments) { + EXIT("unsupported render-target sample configuration: samples=%u fragments=%u\n", + rt.attrib.num_samples, rt.attrib.num_fragments); } const auto view = ResolveTargetViewInfo( rt.view.base_array_slice_index, rt.view.last_array_slice_index, render_target_slice_offset); @@ -93,16 +94,6 @@ void ResolveRenderColorTarget(uint64_t submit_id, CommandBuffer* buffer, const H EXIT("unsupported render-target mip range: current=%u levels=%u\n", rt.view.current_mip_level, levels); } - if (msaa_compat) { - static std::atomic logged_fragments = 0; - const uint32_t bit = 1u << rt.attrib.num_fragments; - if ((logged_fragments.fetch_or(bit, std::memory_order_relaxed) & bit) == 0) { - LOGF("RenderColorTarget: compatibility: rendering PS5 %ux samples/fragments as " - "single-sample\n", - bit); - } - } - if (graphics_debug_dump_enabled()) { static std::atomic_uint log_count = 0; const auto log_id = log_count.fetch_add(1, std::memory_order_relaxed); @@ -144,6 +135,9 @@ void ResolveRenderColorTarget(uint64_t submit_id, CommandBuffer* buffer, const H if (!tile && levels > 1) { EXIT("linear mipmapped render targets are unsupported\n"); } + if (samples > 1 && (!tile || levels != 1)) { + EXIT("multisampled render targets require a single-mip tiled surface\n"); + } width = rt.attrib2.width + 1; height = rt.attrib2.height + 1; @@ -156,13 +150,13 @@ void ResolveRenderColorTarget(uint64_t submit_id, CommandBuffer* buffer, const H if (standard64 && (rt.attrib3.dimension != 1 || rt.attrib3.depth != 0 || levels != 1 || rt.view.current_mip_level != 0 || view.base_layer != 0 || view.image_layers != 1 || - rt.attrib.num_samples != 0 || rt.attrib.num_fragments != 0 || bytes_per_element != 4 || - rt.pitch.pitch_div8_minus1 != 0 || (rt.base.addr & 0xffffu) != 0 || - rt.info.fmask_compression_enable || rt.info.fmask_data_compression_disable || - rt.info.fmask_one_frag_mode || rt.info.cmask_fast_clear_enable || - rt.info.dcc_compression_enable || rt.info.cmask_is_linear != 0 || - rt.info.cmask_addr_type != 0 || rt.info.alt_tile_mode || rt.cmask.addr != 0 || - rt.fmask.addr != 0 || rt.dcc_addr.addr != 0 || rt.dcc.data_write_on_dcc_clear_to_reg)) { + samples != 1 || bytes_per_element != 4 || rt.pitch.pitch_div8_minus1 != 0 || + (rt.base.addr & 0xffffu) != 0 || rt.info.fmask_compression_enable || + rt.info.fmask_data_compression_disable || rt.info.fmask_one_frag_mode || + rt.info.cmask_fast_clear_enable || rt.info.dcc_compression_enable || + rt.info.cmask_is_linear != 0 || rt.info.cmask_addr_type != 0 || rt.info.alt_tile_mode || + rt.cmask.addr != 0 || rt.fmask.addr != 0 || rt.dcc_addr.addr != 0 || + rt.dcc.data_write_on_dcc_clear_to_reg)) { EXIT("unsupported Standard64KB render target: addr=0x%016" PRIx64 " dimension=%u depth=%u levels=%u layer=%u/%u samples=%u fragments=%u bpe=%u" " cmask=0x%016" PRIx64 " fmask=0x%016" PRIx64 " dcc=0x%016" PRIx64 "\n", @@ -176,7 +170,7 @@ void ResolveRenderColorTarget(uint64_t submit_id, CommandBuffer* buffer, const H pitch = standard64 ? TileGetTexturePitch(Prospero::GpuEnumValue(Prospero::BufferFormat::k32Float), width, levels, rt.attrib3.tile_mode) - : TileGetRenderTargetPitch(width, bytes_per_element); + : TileGetRenderTargetPitch(width, bytes_per_element, rt.attrib.num_fragments); if (pitch == 0) { EXIT("unsupported render-target pitch: width=%u bytes=%u\n", width, bytes_per_element); } @@ -194,10 +188,10 @@ void ResolveRenderColorTarget(uint64_t submit_id, CommandBuffer* buffer, const H valid_layout = layout.size != 0 && layout.align == 65536; } else { valid_layout = - levels == 1 - ? TileGetRenderTargetSize(width, height, pitch, bytes_per_element, &layout) - : TileGetRenderTargetMipLayout(width, height, pitch, bytes_per_element, levels, - &layout, nullptr, nullptr); + levels == 1 ? TileGetRenderTargetSize(width, height, pitch, bytes_per_element, + &layout, rt.attrib.num_fragments) + : TileGetRenderTargetMipLayout(width, height, pitch, bytes_per_element, + levels, &layout, nullptr, nullptr); } if (!valid_layout) { EXIT("unsupported render-target layout: %ux%u pitch=%u bytes=%u levels=%u\n", width, @@ -211,7 +205,7 @@ void ResolveRenderColorTarget(uint64_t submit_id, CommandBuffer* buffer, const H (static_cast(rt.slice.slice_div64_minus1) + 1u) * 64u, size); } } else { - size = static_cast(pitch) * height * bytes_per_element; + size = static_cast(pitch) * height * bytes_per_element * samples; } if (size == 0 || size > UINT64_MAX / view.image_layers) { EXIT("render-target memory footprint is invalid\n"); @@ -241,10 +235,10 @@ void ResolveRenderColorTarget(uint64_t submit_id, CommandBuffer* buffer, const H LOGF("RenderColorTarget: slot=%" PRIu32 " addr=0x%010" PRIx64 " size=0x%016" PRIx64 " extent=%ux%u view_mip=%u view_extent=%ux%u levels=%u pitch=%u" " fmt=0x%08" PRIx32 " nfmt=0x%08" PRIx32 " order=0x%08" PRIx32 - " tile=%s target=%s video_size=0x%016" PRIx64 " video_pitch=%" PRIu64 "\n", + " samples=%u tile=%s target=%s video_size=0x%016" PRIx64 " video_pitch=%" PRIu64 "\n", rt_slot, rt.base.addr, backing_size, width, height, rt.view.current_mip_level, view_extent.width, view_extent.height, levels, pitch, rt.info.format, - rt.info.channel_type, rt.info.channel_order, tile ? "tiled" : "linear", + rt.info.channel_type, rt.info.channel_order, samples, tile ? "tiled" : "linear", render_to_texture ? "RenderTexture" : "DisplayBuffer", video_image.size, video_image.pitch); } @@ -262,6 +256,7 @@ void ResolveRenderColorTarget(uint64_t submit_id, CommandBuffer* buffer, const H target.tile_mode = rt.attrib3.tile_mode; target.levels = levels; target.layers = view.image_layers; + target.samples = samples; auto* texture_cache = g_render_ctx->GetTextureCache(); auto* buffer_vulkan = texture_cache->FindRenderTarget(buffer, g_render_ctx->GetGraphicCtx(), target); @@ -271,12 +266,18 @@ void ResolveRenderColorTarget(uint64_t submit_id, CommandBuffer* buffer, const H r->vulkan_view = texture_cache->GetRenderTargetAttachmentView( g_render_ctx->GetGraphicCtx(), buffer_vulkan, target.format, rt.view.current_mip_level, view.base_layer, view.layer_count); - r->format = target.format; - r->extent = view_extent; - r->base_mip_level = rt.view.current_mip_level; - r->buffer_size = backing_size; - r->export_mapping = target_format.export_mapping; + r->format = target.format; + r->extent = view_extent; + r->base_mip_level = rt.view.current_mip_level; + r->buffer_size = backing_size; + r->samples = samples; + r->export_mapping = target_format.export_mapping; + r->color_clear_enable = buffer_vulkan->initial_clear_pending; + r->color_clear_value = {}; } else { + if (samples != 1) { + EXIT("multisampled display render targets are unsupported\n"); + } const auto layout = static_cast(rt.info.format); const auto type = static_cast(rt.info.channel_type); const auto order = static_cast(rt.info.channel_order); @@ -312,6 +313,7 @@ void ResolveRenderColorTarget(uint64_t submit_id, CommandBuffer* buffer, const H r->extent = video_image.image->extent; r->base_mip_level = 0; r->buffer_size = video_image.size; + r->samples = 1; r->export_mapping = target_format.export_mapping; } } diff --git a/src/graphics/host_gpu/renderer/colorRenderTarget.h b/src/graphics/host_gpu/renderer/colorRenderTarget.h index 9e9b66c87..b99346b37 100644 --- a/src/graphics/host_gpu/renderer/colorRenderTarget.h +++ b/src/graphics/host_gpu/renderer/colorRenderTarget.h @@ -33,6 +33,7 @@ struct RenderColorInfo { uint64_t base_addr = 0; uint64_t buffer_size = 0; uint32_t target_slot = 0; + uint32_t samples = 1; Prospero::ColorComponentMapping export_mapping; bool color_clear_enable = false; vk::ClearColorValue color_clear_value {}; diff --git a/src/graphics/host_gpu/renderer/context.cpp b/src/graphics/host_gpu/renderer/context.cpp index d60ad7623..6a669be5b 100644 --- a/src/graphics/host_gpu/renderer/context.cpp +++ b/src/graphics/host_gpu/renderer/context.cpp @@ -580,6 +580,14 @@ void CommandBuffer::BeginRenderPass(VulkanFramebuffer* framebuffer, RenderColorI for (uint32_t i = 0; i < color_count; i++) { colors[i].vulkan_buffer->layout = RENDER_COLOR_IMAGE_LAYOUT; + if (colors[i].vulkan_buffer->type == VulkanImageType::RenderTexture) { + static_cast(colors[i].vulkan_buffer)->initial_clear_pending = + false; + } + } + if (with_depth) { + depth->vulkan_buffer->initial_depth_clear_pending = false; + depth->vulkan_buffer->initial_stencil_clear_pending = false; } } diff --git a/src/graphics/host_gpu/renderer/debug.cpp b/src/graphics/host_gpu/renderer/debug.cpp index 613fe220e..077d323aa 100644 --- a/src/graphics/host_gpu/renderer/debug.cpp +++ b/src/graphics/host_gpu/renderer/debug.cpp @@ -334,7 +334,7 @@ static void RtCheck(const HW::RenderTarget& rt) { if (rt.attrib.num_samples != 0x00000000 || rt.attrib.num_fragments != 0x00000000) { static bool logged = false; if (!logged) { - LOGF("RenderTarget: temporary: rendering PS5 MSAA color target as single-sample, " + LOGF("RenderTarget: using native PS5 MSAA color target, " "samples=0x%08" PRIx32 " fragments=0x%08" PRIx32 "\n", rt.attrib.num_samples, rt.attrib.num_fragments); logged = true; @@ -534,7 +534,7 @@ static void ZCheck(const HW::DepthRenderTarget& z) { if (z.z_info.num_samples != 0x00000000) { static bool logged = false; if (!logged) { - LOGF("DepthTarget: temporary: ignoring num_samples=0x%08" PRIx32 "\n", + LOGF("DepthTarget: using native num_samples=0x%08" PRIx32 "\n", z.z_info.num_samples); logged = true; } @@ -932,7 +932,7 @@ static void EqaaCheck(const HW::EqaaControl& c) { c.incoherent_eqaa_reads || c.interpolate_comp_z || c.static_anchor_associations) { static std::atomic log_count {0}; if (log_count.fetch_add(1) < 16) { - LOGF("\t warning: unsupported PS5 EQAA state, rendering with single-sample fallback\n"); + LOGF("\t warning: unsupported PS5 EQAA controls use native Vulkan MSAA defaults\n"); } } } @@ -962,8 +962,8 @@ static void AaCheck(const HW::AaSampleControl& c, const HW::AaConfig& cf) { cf.max_sample_dist != 0 || cf.msaa_exposed_samples != 0) { static std::atomic log_count {0}; if (log_count.fetch_add(1) < 16) { - LOGF("\t warning: unsupported PS5 AA/MSAA state, rendering with single-sample " - "fallback: samples=%" PRIu8 ", exposed=%" PRIu8 ", max_dist=%" PRIu8 "\n", + LOGF("\t warning: unsupported PS5 sample locations use native Vulkan locations: " + "samples=%" PRIu8 ", exposed=%" PRIu8 ", max_dist=%" PRIu8 "\n", cf.msaa_num_samples, cf.msaa_exposed_samples, cf.max_sample_dist); } } @@ -1058,8 +1058,7 @@ static void VpCheck(const HW::ScreenViewport& vp, const HW::ScanModeControl& smc static std::atomic log_count {0}; if (log_count.fetch_add(1) < 16) { - LOGF("\t warning: unsupported PS5 MSAA raster state, rendering with single-sample " - "fallback\n"); + LOGF("\t warning: unsupported PS5 MSAA raster controls use native Vulkan defaults\n"); } } // EXIT_NOT_IMPLEMENTED(smc.vport_scissor_enable); diff --git a/src/graphics/host_gpu/renderer/depthRenderTarget.cpp b/src/graphics/host_gpu/renderer/depthRenderTarget.cpp index f0c91419d..4076c80d2 100644 --- a/src/graphics/host_gpu/renderer/depthRenderTarget.cpp +++ b/src/graphics/host_gpu/renderer/depthRenderTarget.cpp @@ -63,29 +63,30 @@ static bool UsesStencilOpValue(uint8_t fail, uint8_t pass, uint8_t depth_fail) { [[nodiscard]] static vk::Format ResolveHostDepthAttachmentFormat(GraphicContext* ctx, const DepthFormatPolicy& policy, - bool has_stencil) { + bool has_stencil, + uint32_t samples) { + if (ctx == nullptr) { + return vk::Format::eUndefined; + } + const auto required_samples = vulkan_sample_count(samples); + const auto supports = [&](vk::Format format) { + vk::ImageFormatProperties properties {}; + return format != vk::Format::eUndefined && + ctx->GetImageFormatProperties(format, vk::ImageType::e2D, vk::ImageTiling::eOptimal, + DepthTargetImageUsage(), vk::ImageCreateFlags {}, + &properties) == vk::Result::eSuccess && + static_cast(properties.sampleCounts & required_samples); + }; if (!has_stencil) { - return policy.depth_attachment_format; + return supports(policy.depth_attachment_format) ? policy.depth_attachment_format + : vk::Format::eUndefined; } - switch (policy.depth_format) { - case Prospero::DepthFormat::kZ32F: return policy.stencil_attachment_formats.front(); - case Prospero::DepthFormat::kZ16: { - if (ctx == nullptr) { - return vk::Format::eUndefined; - } - for (const auto format: policy.stencil_attachment_formats) { - vk::ImageFormatProperties properties {}; - if (ctx->GetImageFormatProperties(format, vk::ImageType::e2D, - vk::ImageTiling::eOptimal, - DepthTargetImageUsage(), vk::ImageCreateFlags {}, - &properties) == vk::Result::eSuccess) { - return format; - } - } - return vk::Format::eUndefined; + for (const auto format: policy.stencil_attachment_formats) { + if (supports(format)) { + return format; } - default: return vk::Format::eUndefined; } + return vk::Format::eUndefined; } // NOLINTNEXTLINE(readability-function-cognitive-complexity) @@ -142,8 +143,11 @@ void ResolveRenderDepthTarget(uint64_t submit_id, CommandBuffer* buffer, const H } const bool has_stencil = z.stencil_info.format != Prospero::GpuEnumValue(Prospero::StencilFormat::kInvalid); - const bool has_htile = z.z_info.tile_surface_enable; - const bool msaa_compat = depth_msaa_single_sample_compatible(z.z_info.num_samples); + const bool has_htile = z.z_info.tile_surface_enable; + const auto samples = render_sample_count(z.z_info.num_samples); + if (samples == 0) { + DepthFatal("unsupported depth fragment count: %u", z.z_info.num_samples); + } const bool htile_stencil_compat = depth_htile_stencil_acceleration_compatible( has_stencil, has_htile, z.stencil_info.tile_stencil_disable); const auto view = ResolveTargetViewInfo(z.depth_view.slice_start, z.depth_view.slice_max); @@ -162,28 +166,20 @@ void ResolveRenderDepthTarget(uint64_t submit_id, CommandBuffer* buffer, const H rc.copy_sample != 0 || z.z_info.expclear_enabled || z.stencil_info.expclear_enabled || z.z_info.embedded_sample_locations || z.z_info.partially_resident || z.stencil_info.partially_resident || z.z_info.plane_compression != 0 || - (z.z_info.num_samples != 0 && !msaa_compat) || z.z_info.num_mip_levels != 0 || - z.z_info.tile_mode_index != 0 || z.z_info.zrange_precision > 1 || - z.depth_view.current_mip_level != 0 || z.depth_info.addr5_swizzle_mask != 0 || - z.depth_info.array_mode != 0 || z.depth_info.pipe_config != 0 || - z.depth_info.bank_width != 0 || z.depth_info.bank_height != 0 || - z.depth_info.macro_tile_aspect != 0 || z.depth_info.num_banks != 0 || - z.htile_surface.linear != 0 || z.htile_surface.full_cache != 0 || - z.htile_surface.htile_uses_preload_win != 0 || z.htile_surface.preload != 0 || - z.htile_surface.prefetch_width != 0 || z.htile_surface.prefetch_height != 0 || - z.htile_surface.dst_outside_zero_to_one != 0 || z.z_read_base_addr == 0 || - z.z_write_base_addr != z.z_read_base_addr || (z.z_read_base_addr & 0xffffu) != 0 || + z.z_info.num_mip_levels != 0 || z.z_info.tile_mode_index != 0 || + z.z_info.zrange_precision > 1 || z.depth_view.current_mip_level != 0 || + z.depth_info.addr5_swizzle_mask != 0 || z.depth_info.array_mode != 0 || + z.depth_info.pipe_config != 0 || z.depth_info.bank_width != 0 || + z.depth_info.bank_height != 0 || z.depth_info.macro_tile_aspect != 0 || + z.depth_info.num_banks != 0 || z.htile_surface.linear != 0 || + z.htile_surface.full_cache != 0 || z.htile_surface.htile_uses_preload_win != 0 || + z.htile_surface.preload != 0 || z.htile_surface.prefetch_width != 0 || + z.htile_surface.prefetch_height != 0 || z.htile_surface.dst_outside_zero_to_one != 0 || + z.z_read_base_addr == 0 || z.z_write_base_addr != z.z_read_base_addr || + (z.z_read_base_addr & 0xffffu) != 0 || dc.zfunc > static_cast(vk::CompareOp::eAlways)) { DepthFatal("unsupported depth register state"); } - if (msaa_compat) { - static std::atomic logged_fragments = 0; - const uint32_t bit = 1u << z.z_info.num_samples; - if ((logged_fragments.fetch_or(bit, std::memory_order_relaxed) & bit) == 0) { - LOGF("DepthTarget: compatibility: rendering PS5 %ux depth fragments as single-sample\n", - bit); - } - } if (has_stencil) { // Prospero defines Hi-Stencil as HTile-backed acceleration of the logical stencil plane. // Keep the plane native in Vulkan while tracking HTile separately. @@ -236,55 +232,35 @@ void ResolveRenderDepthTarget(uint64_t submit_id, CommandBuffer* buffer, const H DepthFatal("unsupported depth/stencil format pair"); } const auto ideal_format = DepthAttachmentFormat(*policy, has_stencil); - r->format = - ResolveHostDepthAttachmentFormat(g_render_ctx->GetGraphicCtx(), *policy, has_stencil); + r->format = ResolveHostDepthAttachmentFormat(g_render_ctx->GetGraphicCtx(), *policy, + has_stencil, samples); if (r->format == vk::Format::eUndefined) { DepthFatal("no host depth/stencil format supports required usage for %s", VulkanToString(ideal_format).c_str()); } const uint32_t guest_format = Prospero::GpuEnumValue(policy->guest_format); const uint32_t bytes = policy->bytes_per_element; - const auto pitch = TileGetTexturePitch(guest_format, width, 1, - Prospero::GpuEnumValue(Prospero::TileMode::kDepth)); + const auto pitch = TileGetDepthPitch(width, bytes, z.z_info.num_samples); if (z.pitch_height_valid && ((static_cast(z.pitch_div8_minus1) + 1u) * 8u != pitch || (static_cast(z.height_div8_minus1) + 1u) * 8u != ((static_cast(height) + 7u) & ~7ull))) { DepthFatal("encoded depth pitch or height mismatch"); } - const uint32_t block_width = bytes == 2 ? 256u : 128u; - const uint64_t padded_width = - (static_cast(pitch) + block_width - 1u) & ~(block_width - 1u); - const uint64_t padded_height = - (static_cast(height) + 127u) & ~static_cast(127u); - if (padded_width > UINT64_MAX / padded_height || - padded_width * padded_height > UINT64_MAX / bytes) { - DepthFatal("depth footprint overflow"); + TileSizeAlign depth_size {}; + TileSizeAlign stencil_size {}; + TileSizeAlign htile_size {}; + if (!TileGetDepthSize(width, height, 0, z.z_info.format, z.stencil_info.format, has_htile, + &stencil_size, &htile_size, &depth_size, z.z_info.num_samples) || + depth_size.align != 65536 || depth_size.size == 0 || + (has_stencil != (stencil_size.align == 65536 && stencil_size.size != 0)) || + (has_htile != (htile_size.align == 32768 && htile_size.size != 0))) { + DepthFatal("unsupported depth/stencil/HTile footprint"); } - const uint64_t expected_size = padded_width * padded_height * bytes; - TileSizeAlign depth_size {}; - TileSizeAlign stencil_size {}; - TileSizeAlign htile_size {}; - if (has_stencil || has_htile) { - if (!TileGetDepthSize(width, height, 0, z.z_info.format, z.stencil_info.format, has_htile, - &stencil_size, &htile_size, &depth_size) || - depth_size.align != 65536 || depth_size.size != expected_size || - (has_stencil != (stencil_size.align == 65536 && stencil_size.size != 0)) || - (has_htile != (htile_size.align == 32768 && htile_size.size != 0))) { - DepthFatal("unsupported depth/stencil/HTile footprint"); - } - } else { - TileGetTextureTotalSize(guest_format, width, height, 1, pitch, 1, - Prospero::GpuEnumValue(Prospero::TileMode::kDepth), false, - &depth_size); - } - if (expected_size == 0 || expected_size > UINT32_MAX || depth_size.align != 65536 || - depth_size.size != expected_size || - (z.pitch_height_valid && - (static_cast(z.slice_div64_minus1) + 1u) * 64u != expected_size)) { + if (z.pitch_height_valid && + (static_cast(z.slice_div64_minus1) + 1u) * 64u != depth_size.size) { DepthFatal("depth footprint mismatch: extent=%ux%u pitch=%u expected=0x%016" PRIx64 - " calculated=0x%016" PRIx64 "/0x%016" PRIx64 - " encoded_valid=%u encoded=0x%016" PRIx64, - width, height, pitch, expected_size, depth_size.size, depth_size.align, + " align=0x%016" PRIx64 " encoded_valid=%u encoded=0x%016" PRIx64, + width, height, pitch, depth_size.size, depth_size.align, z.pitch_height_valid ? 1u : 0u, (static_cast(z.slice_div64_minus1) + 1u) * 64u); } @@ -304,6 +280,7 @@ void ResolveRenderDepthTarget(uint64_t submit_id, CommandBuffer* buffer, const H r->htile = has_htile; r->width = width; r->height = height; + r->samples = samples; r->depth_buffer_size = depth_backing_size; r->depth_buffer_vaddr = z.z_read_base_addr; r->stencil_buffer_size = has_stencil ? stencil_backing_size : 0; @@ -312,7 +289,8 @@ void ResolveRenderDepthTarget(uint64_t submit_id, CommandBuffer* buffer, const H r->htile_buffer_vaddr = has_htile ? z.htile_data_base_addr : 0; auto* cache = g_render_ctx->GetTextureCache(); if (has_htile) { - cache->RegisterMeta(r->htile_buffer_vaddr, r->htile_buffer_size, view.image_layers); + cache->RegisterMeta(g_render_ctx->GetGraphicCtx(), r->htile_buffer_vaddr, + r->htile_buffer_size, view.image_layers); } if (has_htile && rc.depth_clear_enable && !cache->ClearMeta(z.htile_data_base_addr)) { DepthFatal("failed to acquire HTile metadata for a depth clear"); @@ -386,7 +364,9 @@ void ResolveRenderDepthTarget(uint64_t submit_id, CommandBuffer* buffer, const H info.bytes_per_element = bytes; info.tile_mode = Prospero::GpuEnumValue(Prospero::TileMode::kDepth); info.layers = view.image_layers; + info.samples = samples; info.depth_load_clear = r->depth_load_clear_enable; + info.depth_access = depth_active; info.stencil_load_clear = rc.stencil_clear_enable; info.stencil_access = r->stencil_clear_enable || @@ -398,6 +378,14 @@ void ResolveRenderDepthTarget(uint64_t submit_id, CommandBuffer* buffer, const H r->vulkan_buffer = cache->FindDepthTarget(buffer, g_render_ctx->GetGraphicCtx(), info); r->vulkan_view = cache->GetDepthTargetAttachmentView( g_render_ctx->GetGraphicCtx(), r->vulkan_buffer, view.base_layer, view.layer_count); + if (r->vulkan_buffer->initial_depth_clear_pending) { + r->depth_load_clear_enable = true; + r->depth_clear_value = 0.0f; + } + if (r->vulkan_buffer->initial_stencil_clear_pending) { + r->stencil_clear_enable = true; + r->stencil_clear_value = 0; + } if (meta_clear && !cache->TouchMeta(z.htile_data_base_addr, z.depth_view.slice_start, false)) { DepthFatal("failed to consume HTile clear state"); } diff --git a/src/graphics/host_gpu/renderer/depthRenderTarget.h b/src/graphics/host_gpu/renderer/depthRenderTarget.h index 799cd1295..4211a6d2e 100644 --- a/src/graphics/host_gpu/renderer/depthRenderTarget.h +++ b/src/graphics/host_gpu/renderer/depthRenderTarget.h @@ -25,6 +25,7 @@ struct RenderDepthInfo { vk::Format format = vk::Format::eUndefined; uint32_t width = 0; uint32_t height = 0; + uint32_t samples = 1; bool htile = false; uint64_t depth_buffer_size = 0; uint64_t depth_buffer_vaddr = 0; diff --git a/src/graphics/host_gpu/renderer/descriptors.cpp b/src/graphics/host_gpu/renderer/descriptors.cpp index fe6a8a8c7..447ec8d70 100644 --- a/src/graphics/host_gpu/renderer/descriptors.cpp +++ b/src/graphics/host_gpu/renderer/descriptors.cpp @@ -285,6 +285,12 @@ TargetTextureViewInfo ResolveTargetTextureView(const ShaderRecompiler::IR::Image base_layer == 0 && image_layers == 1 ? TargetTextureViewInfo {vk::ImageViewType::e2D, 0, 1} : TargetTextureViewInfo {}; + case Prospero::ImageType::kCube: + if (resource.dimension != ShaderRecompiler::Decoder::ImageDimension::Dim2DArray || + base_layer >= image_layers || (image_layers - base_layer) % 6u != 0) { + return {}; + } + return {vk::ImageViewType::e2DArray, base_layer, image_layers - base_layer}; case Prospero::ImageType::kColor2DArray: if (resource.dimension == ShaderRecompiler::Decoder::ImageDimension::Dim2D && base_layer == 0 && image_layers == 1) { diff --git a/src/graphics/host_gpu/renderer/framebufferCache.cpp b/src/graphics/host_gpu/renderer/framebufferCache.cpp index 58971dbe9..6e081e130 100644 --- a/src/graphics/host_gpu/renderer/framebufferCache.cpp +++ b/src/graphics/host_gpu/renderer/framebufferCache.cpp @@ -30,6 +30,7 @@ VulkanFramebuffer* FramebufferCache::CreateFramebuffer(RenderColorInfo* colors, uint32_t color_count = 0; VulkanImage* first_color = nullptr; vk::Extent2D first_color_extent = {}; + uint32_t attachment_samples = 0; for (uint32_t i = 0; i < requested_color_count; i++) { with_color[i] = (colors[i].vulkan_buffer != nullptr); if (!with_color[i]) { @@ -38,6 +39,7 @@ VulkanFramebuffer* FramebufferCache::CreateFramebuffer(RenderColorInfo* colors, if (first_color == nullptr) { first_color = colors[i].vulkan_buffer; first_color_extent = colors[i].extent; + attachment_samples = colors[i].samples; } else if (colors[i].extent.width != first_color_extent.width || colors[i].extent.height != first_color_extent.height) { LOGF("Framebuffer: temporary: dropping mismatched MRT%u attachment color0=%ux%u " @@ -47,8 +49,34 @@ VulkanFramebuffer* FramebufferCache::CreateFramebuffer(RenderColorInfo* colors, with_color[i] = false; break; } + if (colors[i].samples != attachment_samples || + colors[i].vulkan_buffer->samples != colors[i].samples) { + EXIT("Framebuffer: mismatched color attachment samples at slot %u, expected=%u " + "requested=%u image=%u\n", + i, attachment_samples, colors[i].samples, colors[i].vulkan_buffer->samples); + } color_count++; } + if (with_depth) { + if (depth->samples != depth->vulkan_buffer->samples) { + EXIT("Framebuffer: depth attachment sample identity mismatch, requested=%u image=%u\n", + depth->samples, depth->vulkan_buffer->samples); + } + if (attachment_samples == 0) { + attachment_samples = depth->samples; + } else if (attachment_samples != depth->samples) { + EXIT( + "Framebuffer: mixed color/depth sample counts are unsupported, color=%u depth=%u\n", + attachment_samples, depth->samples); + } + } + if (!with_depth && color_count == 0) { + LOGF("Framebuffer: warning: no color or depth attachment\n"); + return nullptr; + } + if (vulkan_sample_count(attachment_samples) == vk::SampleCountFlagBits {}) { + EXIT("Framebuffer: invalid attachment sample count %u\n", attachment_samples); + } vk::ImageLayout color_layout[RENDER_COLOR_ATTACHMENTS_MAX] = {}; for (auto& layout: color_layout) { layout = RENDER_COLOR_IMAGE_LAYOUT; @@ -107,11 +135,6 @@ VulkanFramebuffer* FramebufferCache::CreateFramebuffer(RenderColorInfo* colors, } } - if (!with_depth && color_count == 0) { - LOGF("Framebuffer: warning: no color or depth attachment\n"); - return nullptr; - } - EXIT_NOT_IMPLEMENTED(with_depth && first_color != nullptr && (first_color_extent.width != depth->vulkan_buffer->extent.width || first_color_extent.height != depth->vulkan_buffer->extent.height)); @@ -123,6 +146,7 @@ VulkanFramebuffer* FramebufferCache::CreateFramebuffer(RenderColorInfo* colors, auto* framebuffer = new VulkanFramebuffer; framebuffer->render_pass = nullptr; framebuffer->framebuffer = nullptr; + framebuffer->samples = attachment_samples; for (uint32_t i = 0; i < RENDER_COLOR_ATTACHMENTS_MAX; i++) { framebuffer->color_layout[i] = color_layout[i]; } @@ -136,7 +160,7 @@ VulkanFramebuffer* FramebufferCache::CreateFramebuffer(RenderColorInfo* colors, for (uint32_t i = 0; i < color_count; i++) { attachments[i].flags = {}; attachments[i].format = colors[i].format; - attachments[i].samples = vk::SampleCountFlagBits::e1; + attachments[i].samples = vulkan_sample_count(attachment_samples); attachments[i].loadOp = (colors[i].color_clear_enable ? vk::AttachmentLoadOp::eClear : vk::AttachmentLoadOp::eLoad); attachments[i].storeOp = vk::AttachmentStoreOp::eStore; @@ -149,7 +173,7 @@ VulkanFramebuffer* FramebufferCache::CreateFramebuffer(RenderColorInfo* colors, const uint32_t depth_attachment = color_count; attachments[depth_attachment].flags = {}; attachments[depth_attachment].format = depth->format; - attachments[depth_attachment].samples = vk::SampleCountFlagBits::e1; + attachments[depth_attachment].samples = vulkan_sample_count(attachment_samples); attachments[depth_attachment].loadOp = (depth->depth_load_clear_enable ? vk::AttachmentLoadOp::eClear : vk::AttachmentLoadOp::eLoad); @@ -229,8 +253,8 @@ VulkanFramebuffer* FramebufferCache::CreateFramebuffer(RenderColorInfo* colors, for (uint32_t i = 0; i < color_count; i++) { color_formats[i] = colors[i].format; } - framebuffer->render_pass_id = - render_pass_compat_id(color_count, color_formats, with_depth, depth->format, depth_layout); + framebuffer->render_pass_id = render_pass_compat_id( + color_count, color_formats, with_depth, depth->format, depth_layout, attachment_samples); EXIT_NOT_IMPLEMENTED(framebuffer->render_pass == nullptr); diff --git a/src/graphics/host_gpu/renderer/framebufferCache.h b/src/graphics/host_gpu/renderer/framebufferCache.h index 223d70a0f..e467ce836 100644 --- a/src/graphics/host_gpu/renderer/framebufferCache.h +++ b/src/graphics/host_gpu/renderer/framebufferCache.h @@ -22,13 +22,14 @@ struct VulkanFramebuffer { vk::RenderPass render_pass = nullptr; uint64_t render_pass_id = 0; vk::Framebuffer framebuffer = nullptr; + uint32_t samples = 1; vk::ImageLayout color_layout[RENDER_COLOR_ATTACHMENTS_MAX] = {}; vk::ImageLayout depth_layout = vk::ImageLayout::eDepthStencilAttachmentOptimal; }; inline uint64_t render_pass_compat_id(uint32_t color_count, const vk::Format* color_formats, bool with_depth, vk::Format depth_format, - vk::ImageLayout depth_layout) { + vk::ImageLayout depth_layout, uint32_t samples) { uint64_t id = 0xcbf29ce484222325ull; auto mix = [&id](uint64_t v) { id ^= v; @@ -42,7 +43,7 @@ inline uint64_t render_pass_compat_id(uint32_t color_count, const vk::Format* co mix(with_depth ? 1u : 0u); mix(static_cast(depth_format)); mix(static_cast(depth_layout)); - mix(static_cast(vk::SampleCountFlagBits::e1)); + mix(samples); return id; } diff --git a/src/graphics/host_gpu/renderer/image.cpp b/src/graphics/host_gpu/renderer/image.cpp index a12a66fc2..fa7d843bf 100644 --- a/src/graphics/host_gpu/renderer/image.cpp +++ b/src/graphics/host_gpu/renderer/image.cpp @@ -10,6 +10,7 @@ #include "graphics/host_gpu/renderer/framebufferCache.h" #include "graphics/host_gpu/renderer/imageView.h" #include "graphics/host_gpu/renderer/renderContext.h" +#include "graphics/host_gpu/renderer/renderTarget.h" #include "graphics/host_gpu/transfer.h" #include "graphics/host_gpu/vma.h" #include "graphics/shader/shader.h" @@ -71,20 +72,24 @@ vk::ImageCreateFlags RenderTargetCreateFlags(vk::Format format) { } vk::ImageUsageFlags RenderTargetUsage(GraphicContext* ctx, vk::Format format, - vk::ImageCreateFlags flags) { + vk::ImageCreateFlags flags, uint32_t samples) { auto usage = static_cast(vk::ImageUsageFlagBits::eColorAttachment) | static_cast(vk::ImageUsageFlagBits::eTransferSrc) | - static_cast(vk::ImageUsageFlagBits::eTransferDst) | - static_cast(vk::ImageUsageFlagBits::eSampled); - if (RenderTargetSupportsStorage(ctx, format, flags)) { - usage |= vk::ImageUsageFlagBits::eStorage; + static_cast(vk::ImageUsageFlagBits::eTransferDst); + if (samples == 1) { + usage |= vk::ImageUsageFlagBits::eSampled; + if (RenderTargetSupportsStorage(ctx, format, flags)) { + usage |= vk::ImageUsageFlagBits::eStorage; + } } vk::ImageFormatProperties properties {}; if (ctx->GetImageFormatProperties(format, vk::ImageType::e2D, vk::ImageTiling::eOptimal, usage, - flags, &properties) != vk::Result::eSuccess) { + flags, &properties) != vk::Result::eSuccess || + !static_cast(properties.sampleCounts & vulkan_sample_count(samples))) { EXIT("TextureCache: render-target format does not support required usage, format=%d " - "usage=0x%x\n", - static_cast(format), static_cast(usage)); + "usage=0x%x samples=%u supported=0x%x\n", + static_cast(format), static_cast(usage), samples, + static_cast(properties.sampleCounts)); } return usage; } @@ -172,6 +177,10 @@ void UploadRenderTargetLayers(GraphicContext* ctx, RenderTextureVulkanImage* ima "info_layers=%u image_layers=%u size=0x%016" PRIx64 "\n", base_layer, layer_count, info.layers, image != nullptr ? image->layers : 0, info.size); } + if (info.samples != 1 || image->samples != 1) { + EXIT("TextureCache: multisampled render-target upload is unsupported, samples=%u/%u\n", + info.samples, image->samples); + } if (refresh) { Transfer::WaitForGraphicsIdle(ctx); } @@ -230,6 +239,7 @@ RenderTextureVulkanImage* CreateRenderTarget(GraphicContext* ctx, const RenderTa image->format = info.format; image->mip_levels = info.levels; image->layers = info.layers; + image->samples = info.samples; image->layout = vk::ImageLayout::eUndefined; vk::ImageCreateInfo create {}; create.sType = vk::StructureType::eImageCreateInfo; @@ -241,9 +251,9 @@ RenderTextureVulkanImage* CreateRenderTarget(GraphicContext* ctx, const RenderTa create.format = info.format; create.tiling = vk::ImageTiling::eOptimal; create.initialLayout = vk::ImageLayout::eUndefined; - create.usage = RenderTargetUsage(ctx, info.format, create.flags); + create.usage = RenderTargetUsage(ctx, info.format, create.flags, info.samples); create.sharingMode = vk::SharingMode::eExclusive; - create.samples = vk::SampleCountFlagBits::e1; + create.samples = vulkan_sample_count(info.samples); image->memory.property = vk::MemoryPropertyFlagBits::eDeviceLocal; if (!VulkanCreateImage(ctx, create, image)) { EXIT("TextureCache: failed to create render target, addr=0x%016" PRIx64 @@ -266,20 +276,24 @@ DepthStencilVulkanImage* CreateDepthTarget(GraphicContext* ctx, const DepthTarge create.initialLayout = vk::ImageLayout::eUndefined; create.usage = DepthTargetImageUsage(); create.sharingMode = vk::SharingMode::eExclusive; - create.samples = vk::SampleCountFlagBits::e1; + create.samples = vulkan_sample_count(info.samples); vk::ImageFormatProperties properties {}; if (ctx->GetImageFormatProperties(info.format, vk::ImageType::e2D, vk::ImageTiling::eOptimal, create.usage, vk::ImageCreateFlags {}, - &properties) != vk::Result::eSuccess) { - EXIT("TextureCache: depth format does not support required usage, format=%d usage=0x%x\n", + &properties) != vk::Result::eSuccess || + !static_cast(properties.sampleCounts & create.samples)) { + EXIT("TextureCache: depth format does not support required usage, format=%d usage=0x%x " + "samples=%u supported=0x%x\n", static_cast(info.format), - static_cast(create.usage)); + static_cast(create.usage), info.samples, + static_cast(properties.sampleCounts)); } auto* image = new DepthStencilVulkanImage; image->extent.width = info.width; image->extent.height = info.height; image->guest_pitch = info.pitch; image->layers = info.layers; + image->samples = info.samples; image->format = info.format; image->layout = vk::ImageLayout::eUndefined; image->compressed = false; @@ -326,7 +340,7 @@ void ValidateVideoOut(GraphicContext* ctx, const VideoOutInfo& info) { " pitch=%u\n", info.address, info.size, exact.size, exact.align, info.pitch); } - (void)RenderTargetUsage(ctx, info.format, vk::ImageCreateFlags {}); + (void)RenderTargetUsage(ctx, info.format, vk::ImageCreateFlags {}, 1); } VideoOutVulkanImage* CreateVideoOut(GraphicContext* ctx, const VideoOutInfo& info) { @@ -345,7 +359,7 @@ VideoOutVulkanImage* CreateVideoOut(GraphicContext* ctx, const VideoOutInfo& inf create.tiling = vk::ImageTiling::eOptimal; create.initialLayout = vk::ImageLayout::eUndefined; create.flags = RenderTargetCreateFlags(info.format); - create.usage = RenderTargetUsage(ctx, info.format, create.flags); + create.usage = RenderTargetUsage(ctx, info.format, create.flags, 1); create.sharingMode = vk::SharingMode::eExclusive; create.samples = vk::SampleCountFlagBits::e1; image->memory.property = vk::MemoryPropertyFlagBits::eDeviceLocal; diff --git a/src/graphics/host_gpu/renderer/imageInfo.h b/src/graphics/host_gpu/renderer/imageInfo.h index 239573cc1..c21572760 100644 --- a/src/graphics/host_gpu/renderer/imageInfo.h +++ b/src/graphics/host_gpu/renderer/imageInfo.h @@ -41,6 +41,7 @@ struct RenderTargetInfo { uint32_t tile_mode = 0; uint32_t levels = 1; uint32_t layers = 1; + uint32_t samples = 1; }; // Common image-to-buffer copy description. Storage images and render targets keep distinct @@ -55,20 +56,21 @@ struct ColorImageTransferInfo { uint32_t bytes_per_element = 0; uint32_t tile_mode = 0; uint32_t levels = 1; + uint32_t samples = 1; }; [[nodiscard]] inline ColorImageTransferInfo MakeColorImageTransferInfo(const ImageInfo& info, vk::Format format, uint32_t bytes_per_element) noexcept { - return {info.address, info.size, format, info.width, info.height, - info.pitch, bytes_per_element, info.tile, info.levels}; + return {info.address, info.size, format, info.width, info.height, + info.pitch, bytes_per_element, info.tile, info.levels, 1}; } [[nodiscard]] inline ColorImageTransferInfo MakeColorImageTransferInfo(const RenderTargetInfo& info) noexcept { return { - info.address, info.size, info.format, info.width, info.height, info.pitch, - info.bytes_per_element, info.tile_mode, info.levels}; + info.address, info.size, info.format, info.width, info.height, info.pitch, + info.bytes_per_element, info.tile_mode, info.levels, info.samples}; } struct DepthTargetInfo { @@ -86,7 +88,9 @@ struct DepthTargetInfo { uint32_t bytes_per_element = 0; uint32_t tile_mode = 0; uint32_t layers = 1; + uint32_t samples = 1; bool depth_load_clear = false; + bool depth_access = false; bool stencil_load_clear = false; bool stencil_access = false; bool stencil_htile_compressed = false; @@ -225,6 +229,13 @@ FindGuestDepthFormatPolicy(uint32_t guest_format) noexcept { : info.format == policy->depth_attachment_format); } +[[nodiscard]] inline constexpr bool IsSupportedDepthReadbackFormat(const DepthTargetInfo& info) { + const bool has_stencil = info.stencil_address != 0 || info.stencil_size != 0; + return IsSupportedDepthTargetFormat(info) && + DepthAspectTransferBytes(info.format) == info.bytes_per_element && + (!has_stencil || !info.stencil_htile_compressed); +} + enum class VideoOutCompression : uint8_t { Uncompressed, Dcc256_256_0, Dcc256_64_64, Unsupported }; struct VideoOutInfo { @@ -335,12 +346,14 @@ enum class DepthOverlap : uint8_t { RetireStorage, ExpandTarget, DiscardTarget, + RecreateTarget, Unsupported }; enum class DepthTransitionSource : uint8_t { None, Guest, Native }; enum class RenderTargetOverlap : uint8_t { None, RetireSampled, + RetireStorage, PreserveStorage, ExpandTarget, RetireTarget, @@ -373,8 +386,24 @@ enum class BufferImageWrite : uint8_t { Unsupported }; +[[nodiscard]] inline constexpr bool +HasGuestCurrentImageOwnership(bool image_gpu_modified, bool buffer_modified, bool cpu_dirty, + bool tracker_gpu_modified, bool same_context) noexcept { + return !image_gpu_modified && !buffer_modified && !cpu_dirty && !tracker_gpu_modified && + same_context; +} + enum class StorageBufferRebind : uint8_t { Reuse, RefreshFromBacking, Unsupported }; -enum class MetaImageOverlap : uint8_t { RetainSampled, RetireTarget, Unsupported }; +enum class MetaImageOverlap : uint8_t { RetainSampled, RetireImage, Unsupported }; + +[[nodiscard]] inline constexpr bool CanRetireGuestCurrentDepthForMetadataReuse( + bool depth_gpu_modified, bool depth_buffer_modified, bool depth_tracker_gpu_modified, + bool metadata_gpu_modified, bool metadata_tracker_gpu_modified, uint32_t metadata_clear_mask, + bool same_context) noexcept { + return !metadata_gpu_modified && !metadata_tracker_gpu_modified && metadata_clear_mask == 0 && + HasGuestCurrentImageOwnership(depth_gpu_modified, depth_buffer_modified, false, + depth_tracker_gpu_modified, same_context); +} [[nodiscard]] inline constexpr uint32_t SelectImageBackingBaseLevel(bool storage, uint32_t view_base_level) { @@ -434,7 +463,7 @@ IsSupportedDisplayRenderTargetTileMode(uint32_t tile_mode) noexcept { IsSupportedStandard64RenderTarget(const RenderTargetInfo& info) noexcept { if (info.tile_mode != Prospero::GpuEnumValue(Prospero::TileMode::kStandard64KB) || info.address == 0 || (info.address & 0xffffu) != 0 || info.width == 0 || info.height == 0 || - info.bytes_per_element != 4 || info.levels != 1 || info.layers != 1) { + info.bytes_per_element != 4 || info.levels != 1 || info.layers != 1 || info.samples != 1) { return false; } const auto expected_pitch = (static_cast(info.width) + 127u) & ~uint64_t {127u}; @@ -462,14 +491,15 @@ SelectDepthTransitionSource(bool depth_load_clear, bool sampled_native_available : DepthTransitionSource::Guest; } -[[nodiscard]] inline MetaImageOverlap ClassifyMetaImageOverlap(bool sampled, bool render_target, +[[nodiscard]] inline MetaImageOverlap ClassifyMetaImageOverlap(bool sampled, bool writable_image, bool gpu_modified, - bool buffer_modified) { + bool buffer_modified, bool cpu_dirty, + bool same_context) { if (sampled && !gpu_modified) { return MetaImageOverlap::RetainSampled; } - if (render_target && !gpu_modified && !buffer_modified) { - return MetaImageOverlap::RetireTarget; + if (writable_image && !gpu_modified && !buffer_modified && !cpu_dirty && same_context) { + return MetaImageOverlap::RetireImage; } return MetaImageOverlap::Unsupported; } @@ -559,6 +589,7 @@ SelectDepthTransitionSource(bool depth_load_clear, bool sampled_native_available const bool d32 = target.format == vk::Format::eD32Sfloat || target.format == vk::Format::eD32SfloatS8Uint; return address == target.address && size == target.size && target.layers == 1 && d32 && + target.samples == 1 && target.guest_format == Prospero::GpuEnumValue(Prospero::BufferFormat::k32Float) && target.bytes_per_element == 4 && target.tile_mode == Prospero::GpuEnumValue(Prospero::TileMode::kDepth) && @@ -826,8 +857,11 @@ ClassifyStorageBufferRebind(bool buffer_overlap, bool cached_gpu_modified, // Mirrors storage -> depth-target binding transition. An inaccessible stencil aspect // does not require the otherwise necessary image content copy. -[[nodiscard]] inline DepthOverlap ClassifyStorageDepthOverlap(const ImageInfo& storage, - const DepthTargetInfo& depth) { +[[nodiscard]] inline DepthOverlap +ClassifyStorageDepthOverlap(const ImageInfo& storage, bool storage_gpu_modified, + bool storage_buffer_modified, bool storage_cpu_dirty, + bool tracker_gpu_modified, bool same_context, + const DepthTargetInfo& depth) { const bool overlaps_depth = ImageRangeOverlaps(storage.address, storage.size, depth.address, depth.size); const bool overlaps_stencil = @@ -836,11 +870,47 @@ ClassifyStorageBufferRebind(bool buffer_overlap, bool cached_gpu_modified, if (!overlaps_depth && !overlaps_stencil) { return DepthOverlap::None; } - return !overlaps_depth && overlaps_stencil && !depth.stencil_access + const bool guest_current = + HasGuestCurrentImageOwnership(storage_gpu_modified, storage_buffer_modified, + storage_cpu_dirty, tracker_gpu_modified, same_context); + const bool aspects_discarded = + (!overlaps_depth || !depth.depth_access) && (!overlaps_stencil || !depth.stencil_access); + return guest_current || (!storage_buffer_modified && !storage_cpu_dirty && same_context && + aspects_discarded) ? DepthOverlap::RetireStorage : DepthOverlap::Unsupported; } +[[nodiscard]] inline bool CanRetireGuestCurrentDepthForSampled( + const ImageInfo& sampled, const DepthTargetInfo& depth, bool depth_gpu_modified, + bool depth_buffer_modified, bool depth_tracker_gpu_modified, bool stencil_tracker_gpu_modified, + bool same_context) noexcept { + const bool overlaps_depth = + ImageRangeOverlaps(sampled.address, sampled.size, depth.address, depth.size); + const bool overlaps_stencil = + depth.stencil_address != 0 && ImageRangeOverlaps(sampled.address, sampled.size, + depth.stencil_address, depth.stencil_size); + return (overlaps_depth || overlaps_stencil) && + HasGuestCurrentImageOwnership(depth_gpu_modified, depth_buffer_modified, false, + depth_tracker_gpu_modified || stencil_tracker_gpu_modified, + same_context); +} + +[[nodiscard]] inline bool +CanRetireGuestCurrentSampledForDepth(const ImageInfo& sampled, const DepthTargetInfo& depth, + bool sampled_gpu_modified, bool sampled_buffer_modified, + bool sampled_cpu_dirty, bool tracker_gpu_modified, + bool same_context, bool guest_source_current) noexcept { + const bool overlaps_depth = + ImageRangeOverlaps(sampled.address, sampled.size, depth.address, depth.size); + const bool overlaps_stencil = + depth.stencil_address != 0 && ImageRangeOverlaps(sampled.address, sampled.size, + depth.stencil_address, depth.stencil_size); + return (overlaps_depth || overlaps_stencil) && guest_source_current && + HasGuestCurrentImageOwnership(sampled_gpu_modified, sampled_buffer_modified, + sampled_cpu_dirty, tracker_gpu_modified, same_context); +} + [[nodiscard]] inline RenderTargetOverlap ClassifyRenderTargetOverlap(const ImageInfo& sampled, bool sampled_gpu_modified, bool same_context, const RenderTargetInfo& target) { @@ -857,11 +927,14 @@ ClassifyRenderTargetOverlap(const ImageInfo& sampled, bool sampled_gpu_modified, [[nodiscard]] inline RenderTargetOverlap ClassifyStorageRenderTargetOverlap(const ImageInfo& storage, vk::Format storage_format, bool storage_gpu_modified, bool storage_buffer_modified, - bool storage_cpu_dirty, bool same_context, - const RenderTargetInfo& target) { + bool storage_cpu_dirty, bool tracker_gpu_modified, + bool same_context, const RenderTargetInfo& target) { if (!ImagePageRangesOverlap(storage.address, storage.size, target.address, target.size)) { return RenderTargetOverlap::None; } + if (!ImageRangeOverlaps(storage.address, storage.size, target.address, target.size)) { + return RenderTargetOverlap::None; + } const bool exact_native_image = storage.address == target.address && storage.size == target.size && storage_format == target.format && storage.width == target.width && @@ -869,10 +942,14 @@ ClassifyStorageRenderTargetOverlap(const ImageInfo& storage, vk::Format storage_ storage.base_level == 0 && storage.levels == 1 && storage.view_levels == 1 && storage.tile == target.tile_mode && storage.depth == 1 && storage.type == Prospero::GpuEnumValue(Prospero::ImageType::kColor2D) && - storage.base_array == 0 && target.levels == 1 && target.layers == 1; - return exact_native_image && storage_gpu_modified && !storage_buffer_modified && - !storage_cpu_dirty && same_context - ? RenderTargetOverlap::PreserveStorage + storage.base_array == 0 && target.levels == 1 && target.layers == 1 && target.samples == 1; + if (exact_native_image && storage_gpu_modified && tracker_gpu_modified && + !storage_buffer_modified && !storage_cpu_dirty && same_context) { + return RenderTargetOverlap::PreserveStorage; + } + return HasGuestCurrentImageOwnership(storage_gpu_modified, storage_buffer_modified, + storage_cpu_dirty, tracker_gpu_modified, same_context) + ? RenderTargetOverlap::RetireStorage : RenderTargetOverlap::Unsupported; } @@ -899,7 +976,7 @@ ClassifyStorageRenderTargetOverlap(const ImageInfo& storage, vk::Format storage_ cached.pitch == requested.pitch && cached.bytes_per_element == requested.bytes_per_element && cached.tile_mode == requested.tile_mode && cached.levels == requested.levels && - cached.layers == requested.layers && + cached.layers == requested.layers && cached.samples == requested.samples && IsRgba8SrgbReinterpretation(cached.format, requested.format); } @@ -959,6 +1036,7 @@ IsCompatibleRenderTargetBacking(const RenderTargetInfo& cached, cached.pitch == requested.pitch && cached.bytes_per_element == requested.bytes_per_element && cached.tile_mode == requested.tile_mode && cached.levels == requested.levels && + cached.samples == requested.samples && (cached.format == requested.format || IsRgba8SrgbReinterpretation(cached.format, requested.format)); } @@ -977,34 +1055,46 @@ IsCompatibleDepthTargetBacking(const DepthTargetInfo& cached, cached.width == requested.width && cached.height == requested.height && cached.pitch == requested.pitch && cached.bytes_per_element == requested.bytes_per_element && - cached.tile_mode == requested.tile_mode && + cached.tile_mode == requested.tile_mode && cached.samples == requested.samples && cached.stencil_htile_compressed == requested.stencil_htile_compressed; } [[nodiscard]] inline RenderTargetOverlap ClassifyRenderTargetOverlap(const RenderTargetInfo& cached, bool cached_gpu_modified, - bool cached_buffer_modified, bool same_context, + bool cached_buffer_modified, bool tracker_gpu_modified, + bool same_context, bool guest_source_current, const RenderTargetInfo& requested) { if (!ImagePageRangesOverlap(cached.address, cached.size, requested.address, requested.size)) { return RenderTargetOverlap::None; } + if (!ImageRangeOverlaps(cached.address, cached.size, requested.address, requested.size)) { + return RenderTargetOverlap::None; + } const bool expand = requested.layers > cached.layers && IsCompatibleRenderTargetBacking(requested, cached) && cached.format == requested.format; - if (expand && cached_gpu_modified && !cached_buffer_modified && same_context) { + if (expand && cached_gpu_modified && tracker_gpu_modified && !cached_buffer_modified && + same_context) { return RenderTargetOverlap::ExpandTarget; } - // For an equal-address allocation-pool entry, a changed block raster or block size is a new - // image allocation, not a view of the old image. In Kyty we can only retire an - // already-published target with page-isolated storage. + // A clean, page-isolated overlap is allocation-pool reuse, including a contained subrange. + // Compatible views are handled before this classifier and are never retired here. const bool page_isolated = cached.address % TRACKER_PAGE_SIZE == 0 && cached.size % TRACKER_PAGE_SIZE == 0 && requested.address % TRACKER_PAGE_SIZE == 0 && requested.size % TRACKER_PAGE_SIZE == 0; - const bool pool_storage_shape_changed = cached.pitch != requested.pitch || - cached.height != requested.height || - cached.bytes_per_element != requested.bytes_per_element; - return cached.address == requested.address && page_isolated && pool_storage_shape_changed && - !cached_gpu_modified && !cached_buffer_modified && same_context + const bool incompatible_format = cached.format != requested.format && + !IsRgba8SrgbReinterpretation(cached.format, requested.format); + const bool pool_storage_shape_changed = + incompatible_format || cached.width != requested.width || + cached.height != requested.height || cached.pitch != requested.pitch || + cached.bytes_per_element != requested.bytes_per_element || + cached.tile_mode != requested.tile_mode || cached.levels != requested.levels || + cached.layers != requested.layers || cached.samples != requested.samples; + const bool new_allocation = cached.address != requested.address || + cached.size != requested.size || pool_storage_shape_changed; + return page_isolated && new_allocation && guest_source_current && + HasGuestCurrentImageOwnership(cached_gpu_modified, cached_buffer_modified, false, + tracker_gpu_modified, same_context) ? RenderTargetOverlap::RetireTarget : RenderTargetOverlap::Unsupported; } @@ -1014,7 +1104,24 @@ ClassifyRenderTargetOverlap(const RenderTargetInfo& cached, bool cached_gpu_modi bool cached_buffer_modified, bool same_context, const DepthTargetInfo& requested) { - if (!ImagePageRangesOverlap(cached.address, cached.size, requested.address, requested.size)) { + const auto overlaps = [](uint64_t left, uint64_t left_size, uint64_t right, uint64_t right_size, + bool pages) { + if (left == 0 || left_size == 0 || right == 0 || right_size == 0) { + return false; + } + return pages ? ImagePageRangesOverlap(left, left_size, right, right_size) + : ImageRangeOverlaps(left, left_size, right, right_size); + }; + const auto planes_overlap = [&](bool pages) { + return overlaps(cached.address, cached.size, requested.address, requested.size, pages) || + overlaps(cached.address, cached.size, requested.stencil_address, + requested.stencil_size, pages) || + overlaps(cached.stencil_address, cached.stencil_size, requested.address, + requested.size, pages) || + overlaps(cached.stencil_address, cached.stencil_size, requested.stencil_address, + requested.stencil_size, pages); + }; + if (!planes_overlap(true) || !planes_overlap(false)) { return DepthOverlap::None; } const bool expand = @@ -1025,19 +1132,100 @@ ClassifyRenderTargetOverlap(const RenderTargetInfo& cached, bool cached_gpu_modi const bool exact_discard = requested.depth_load_clear && cached_gpu_modified && !cached_buffer_modified && same_context && cached.address == requested.address && cached.size == requested.size && - cached.stencil_address == 0 && cached.stencil_size == 0 && cached.htile_address == 0 && - cached.htile_size == 0 && requested.stencil_address == 0 && requested.stencil_size == 0 && + cached.samples == requested.samples && cached.stencil_address == 0 && + cached.stencil_size == 0 && cached.htile_address == 0 && cached.htile_size == 0 && + requested.stencil_address == 0 && requested.stencil_size == 0 && requested.htile_address == 0 && requested.htile_size == 0; - return exact_discard ? DepthOverlap::DiscardTarget : DepthOverlap::Unsupported; + if (exact_discard) { + return DepthOverlap::DiscardTarget; + } + const bool exact_plane_rebind = + (cached.address == requested.address && cached.size == requested.size) || + (cached.address == requested.stencil_address && cached.size == requested.stencil_size) || + (cached.stencil_address == requested.address && cached.stencil_size == requested.size) || + (cached.stencil_address == requested.stencil_address && + cached.stencil_size == requested.stencil_size); + const bool same_shape = cached.format == requested.format && + cached.guest_format == requested.guest_format && + cached.width == requested.width && cached.height == requested.height && + cached.pitch == requested.pitch && + cached.bytes_per_element == requested.bytes_per_element && + cached.tile_mode == requested.tile_mode && cached.layers == 1 && + requested.layers == 1 && cached.samples == 1 && requested.samples == 1; + return exact_plane_rebind && same_shape && !cached_buffer_modified && same_context + ? DepthOverlap::RecreateTarget + : DepthOverlap::Unsupported; } -[[nodiscard]] inline bool CanRetireBufferOwnedDepthForRenderTarget( - const DepthTargetInfo& depth, bool gpu_modified, bool buffer_modified, bool same_context, - bool containing_buffer_source, const RenderTargetInfo& target) noexcept { - return !gpu_modified && buffer_modified && same_context && containing_buffer_source && - depth.address == target.address && depth.size == target.size && - depth.stencil_address == 0 && depth.stencil_size == 0 && depth.htile_address == 0 && - depth.htile_size == 0; +[[nodiscard]] inline bool CanRecreateDepthForRenderTarget(const DepthTargetInfo& depth, + bool gpu_modified, bool buffer_modified, + bool tracker_gpu_modified, + bool same_context, + bool guest_source_current, + const RenderTargetInfo& target) noexcept { + const bool exact_depth = depth.address == target.address && depth.size == target.size; + const bool exact_stencil = + depth.stencil_address == target.address && depth.stencil_size == target.size; + const auto contains = [&](uint64_t address, uint64_t size) { + if (address == 0 || size == 0 || address < target.address) { + return false; + } + const auto offset = address - target.address; + return offset <= target.size && size <= target.size - offset; + }; + const bool page_isolated_rebind = + target.address % TRACKER_PAGE_SIZE == 0 && target.size % TRACKER_PAGE_SIZE == 0 && + ((contains(depth.address, depth.size) && depth.address % TRACKER_PAGE_SIZE == 0 && + depth.size % TRACKER_PAGE_SIZE == 0) || + (contains(depth.stencil_address, depth.stencil_size) && + depth.stencil_address % TRACKER_PAGE_SIZE == 0 && + depth.stencil_size % TRACKER_PAGE_SIZE == 0)); + const bool ownership_consistent = gpu_modified == tracker_gpu_modified && !buffer_modified; + const bool source_available = + ((exact_depth || exact_stencil) && gpu_modified) || guest_source_current; + return same_context && ownership_consistent && source_available && + (exact_depth || exact_stencil || page_isolated_rebind) && depth.layers == 1 && + target.levels == 1 && target.layers == 1 && depth.samples == 1 && target.samples == 1; +} + +[[nodiscard]] inline bool +CanRecreateRenderTargetForDepth(const RenderTargetInfo& target, bool gpu_modified, + bool buffer_modified, bool tracker_gpu_modified, bool same_context, + bool guest_source_current, const DepthTargetInfo& depth) noexcept { + const bool exact_depth = target.address == depth.address && target.size == depth.size; + const bool exact_stencil = + target.address == depth.stencil_address && target.size == depth.stencil_size; + const auto overlaps_plane = [&](uint64_t address, uint64_t size) { + return address != 0 && size != 0 && + ImageRangeOverlaps(target.address, target.size, address, size); + }; + const bool page_isolated_rebind = + target.address % TRACKER_PAGE_SIZE == 0 && target.size % TRACKER_PAGE_SIZE == 0 && + ((overlaps_plane(depth.address, depth.size) && depth.address % TRACKER_PAGE_SIZE == 0 && + depth.size % TRACKER_PAGE_SIZE == 0) || + (overlaps_plane(depth.stencil_address, depth.stencil_size) && + depth.stencil_address % TRACKER_PAGE_SIZE == 0 && + depth.stencil_size % TRACKER_PAGE_SIZE == 0)); + const bool source_available = + (gpu_modified && (exact_depth || exact_stencil)) || guest_source_current; + return gpu_modified == tracker_gpu_modified && !buffer_modified && same_context && + source_available && (exact_depth || exact_stencil || page_isolated_rebind) && + target.levels == 1 && target.layers == 1 && depth.layers == 1 && target.samples == 1 && + depth.samples == 1; +} + +[[nodiscard]] inline bool RequiresMultisampleDepthRefresh(const DepthTargetInfo& info, + bool buffer_modified, + bool depth_cpu_modified, + bool stencil_cpu_modified) noexcept { + if (info.samples == 1) { + return false; + } + const bool depth_refresh = + info.depth_access && !info.depth_load_clear && (buffer_modified || depth_cpu_modified); + const bool stencil_refresh = info.stencil_access && !info.stencil_load_clear && + (buffer_modified || stencil_cpu_modified); + return depth_refresh || stencil_refresh; } } // namespace Libs::Graphics diff --git a/src/graphics/host_gpu/renderer/imageView.cpp b/src/graphics/host_gpu/renderer/imageView.cpp index db9fdc43d..bd5dcf613 100644 --- a/src/graphics/host_gpu/renderer/imageView.cpp +++ b/src/graphics/host_gpu/renderer/imageView.cpp @@ -94,7 +94,7 @@ void CreateRenderTargetViews(GraphicContext* ctx, RenderTextureVulkanImage* imag vk::ComponentSwizzle::eIdentity, vk::ComponentSwizzle::eIdentity, vk::ImageViewType::e2DArray); } - if (FormatSupportsStorage(ctx, image->format)) { + if (image->samples == 1 && FormatSupportsStorage(ctx, image->format)) { CreateRenderTargetView(ctx, image, VulkanImage::VIEW_STORAGE, vk::ComponentSwizzle::eIdentity, vk::ComponentSwizzle::eIdentity, vk::ComponentSwizzle::eIdentity, vk::ComponentSwizzle::eIdentity, diff --git a/src/graphics/host_gpu/renderer/pipelineCache.cpp b/src/graphics/host_gpu/renderer/pipelineCache.cpp index f5c4e5f6a..26cd72822 100644 --- a/src/graphics/host_gpu/renderer/pipelineCache.cpp +++ b/src/graphics/host_gpu/renderer/pipelineCache.cpp @@ -8,6 +8,7 @@ #include "graphics/host_gpu/renderer/debug.h" #include "graphics/host_gpu/renderer/depthRenderTarget.h" #include "graphics/host_gpu/renderer/framebufferCache.h" +#include "graphics/host_gpu/renderer/renderContext.h" #include #include @@ -89,6 +90,13 @@ PipelineCache::GraphicsPipeline* PipelineCache::CreateGraphicsPipeline( static_params.negative_one_to_one = !ctx->GetClipControl().dx_clip_space; static_params.topology = topology; + static_params.samples = framebuffer->samples; + static_params.sample_shading_enable = + ps_active && framebuffer->samples > 1 && ps_input_info->ps_sample_shading; + if (static_params.sample_shading_enable && + !g_render_ctx->GetGraphicCtx()->sample_rate_shading_enabled) { + EXIT("Pipeline: sample-rate shading is required but unsupported by the host\n"); + } static_params.with_depth = (depth->format != vk::Format::eUndefined && depth->vulkan_buffer != nullptr); static_params.depth_test_enable = depth->depth_test_enable; diff --git a/src/graphics/host_gpu/renderer/pipelineCache.h b/src/graphics/host_gpu/renderer/pipelineCache.h index 5757aa457..f17ed86f9 100644 --- a/src/graphics/host_gpu/renderer/pipelineCache.h +++ b/src/graphics/host_gpu/renderer/pipelineCache.h @@ -35,6 +35,8 @@ struct PipelineStaticParameters { bool negative_one_to_one = false; int scissor_ltrb[4] = {0}; vk::PrimitiveTopology topology = vk::PrimitiveTopology::ePointList; + uint32_t samples = 1; + bool sample_shading_enable = false; bool with_depth = false; bool depth_test_enable = false; bool depth_write_enable = false; @@ -74,8 +76,8 @@ static_assert(std::is_standard_layout_v); static_assert(alignof(PipelineStaticParameters) == 1); static_assert(sizeof(PipelineStaticParameters) == sizeof(float[3]) + sizeof(float[3]) + sizeof(bool) + sizeof(int[4]) + - sizeof(vk::PrimitiveTopology) + sizeof(bool) * 3 + sizeof(vk::CompareOp) + - sizeof(bool) + sizeof(float) * 2 + sizeof(bool) + + sizeof(vk::PrimitiveTopology) + sizeof(uint32_t) + sizeof(bool) * 4 + + sizeof(vk::CompareOp) + sizeof(bool) + sizeof(float) * 2 + sizeof(bool) + sizeof(PipelineStencilStaticState) * 2 + sizeof(uint32_t) + sizeof(uint32_t[RENDER_COLOR_ATTACHMENTS_MAX]) + sizeof(bool) * 3 + sizeof(uint8_t[RENDER_COLOR_ATTACHMENTS_MAX]) * 6 + diff --git a/src/graphics/host_gpu/renderer/renderCompute.cpp b/src/graphics/host_gpu/renderer/renderCompute.cpp index 8d2bcc7ba..e2e33838d 100644 --- a/src/graphics/host_gpu/renderer/renderCompute.cpp +++ b/src/graphics/host_gpu/renderer/renderCompute.cpp @@ -59,18 +59,16 @@ bool ResolveHtileClearTarget(const HW::DepthRenderTarget& z, uint64_t descriptor const bool has_stencil = z.stencil_info.format != Prospero::GpuEnumValue(Prospero::StencilFormat::kInvalid); const auto* depth_policy = FindDepthFormatPolicy(z.z_info.format); - const bool msaa_compat = depth_msaa_single_sample_compatible(z.z_info.num_samples); const bool supported_depth_state = z.z_info.tile_surface_enable && depth_policy != nullptr && z.z_info.tile_mode_index == 0 && - (z.z_info.num_samples == 0 || msaa_compat) && z.z_info.zrange_precision <= 1 && - !z.z_info.expclear_enabled && !z.z_info.embedded_sample_locations && - !z.z_info.partially_resident && z.z_info.num_mip_levels == 0 && - z.z_info.plane_compression == 0 && z.depth_view.current_mip_level == 0 && - z.depth_view.slice_start == 0 && z.depth_view.slice_max == 0 && - z.depth_info.addr5_swizzle_mask == 0 && z.depth_info.array_mode == 0 && - z.depth_info.pipe_config == 0 && z.depth_info.bank_width == 0 && - z.depth_info.bank_height == 0 && z.depth_info.macro_tile_aspect == 0 && - z.depth_info.num_banks == 0; + z.z_info.num_samples <= 3 && z.z_info.zrange_precision <= 1 && !z.z_info.expclear_enabled && + !z.z_info.embedded_sample_locations && !z.z_info.partially_resident && + z.z_info.num_mip_levels == 0 && z.z_info.plane_compression == 0 && + z.depth_view.current_mip_level == 0 && z.depth_view.slice_start == 0 && + z.depth_view.slice_max == 0 && z.depth_info.addr5_swizzle_mask == 0 && + z.depth_info.array_mode == 0 && z.depth_info.pipe_config == 0 && + z.depth_info.bank_width == 0 && z.depth_info.bank_height == 0 && + z.depth_info.macro_tile_aspect == 0 && z.depth_info.num_banks == 0; const bool supported_stencil_state = z.stencil_info.tile_mode_index == 0 && z.stencil_info.tile_split == 0 && !z.stencil_info.expclear_enabled && @@ -100,14 +98,6 @@ bool ResolveHtileClearTarget(const HW::DepthRenderTarget& z, uint64_t descriptor !supported_addresses) { return false; } - if (msaa_compat) { - static std::atomic logged_fragments = 0; - const uint32_t bit = 1u << z.z_info.num_samples; - if ((logged_fragments.fetch_or(bit, std::memory_order_relaxed) & bit) == 0) { - LOGF("HTileClear: compatibility: treating PS5 %ux depth fragments as single-sample\n", - bit); - } - } const bool size_xy_valid = z.size.valid; const bool wh_valid = z.width_height_valid && z.width != 0 && z.height != 0; @@ -133,36 +123,8 @@ bool ResolveHtileClearTarget(const HW::DepthRenderTarget& z, uint64_t descriptor (z.pitch_div8_minus1 != 0 || z.height_div8_minus1 != 0 || z.slice_div64_minus1 != 0))) { return false; } - - const uint32_t guest_format = Prospero::GpuEnumValue(depth_policy->guest_format); - const uint32_t bytes = depth_policy->bytes_per_element; - const uint32_t pitch = TileGetTexturePitch(guest_format, width, 1, - Prospero::GpuEnumValue(Prospero::TileMode::kDepth)); - if (z.pitch_height_valid && ((static_cast(z.pitch_div8_minus1) + 1u) * 8u != pitch || - (static_cast(z.height_div8_minus1) + 1u) * 8u != - ((static_cast(height) + 7u) & ~7ull))) { - return false; - } - const uint32_t block_width = bytes == 2 ? 256u : 128u; - const uint64_t padded_width = - (static_cast(pitch) + block_width - 1u) & ~(block_width - 1u); - const uint64_t padded_height = (static_cast(height) + 127u) & ~127ull; - if (padded_width > UINT64_MAX / padded_height || - padded_width * padded_height > UINT64_MAX / bytes) { - return false; - } - const uint64_t expected_depth_size = padded_width * padded_height * bytes; - TileSizeAlign depth_size {}; - TileSizeAlign stencil_size {}; - TileSizeAlign htile_size {}; - if (!TileGetDepthSize(width, height, 0, z.z_info.format, z.stencil_info.format, true, - &stencil_size, &htile_size, &depth_size) || - expected_depth_size == 0 || expected_depth_size > UINT32_MAX || depth_size.align != 65536 || - depth_size.size != expected_depth_size || - (has_stencil != (stencil_size.align == 65536 && stencil_size.size != 0)) || - htile_size.align != 32768 || htile_size.size == 0 || htile_size.size != descriptor_size || - (z.pitch_height_valid && - (static_cast(z.slice_div64_minus1) + 1u) * 64u != expected_depth_size)) { + TileSizeAlign htile_size {}; + if (!TileGetHtileSize(width, height, &htile_size) || htile_size.size != descriptor_size) { return false; } *resolved = {.address = z.htile_data_base_addr, .size = htile_size.size}; @@ -232,9 +194,19 @@ static bool TryConsumeComputeMetaClear(const ShaderComputeInputInfo& input, cons HtileClearTarget target {}; if (current_references != 0) { if (!ResolveHtileClearTarget(z, described_meta_size, &target)) { - EXIT("unsupported HTile compute-clear target state\n"); + EXIT("unsupported HTile compute-clear target state: current=%u registered=%u " + "meta=0x%016" PRIx64 "+0x%016" PRIx64 " depth=0x%016" PRIx64 "/0x%016" PRIx64 + " stencil=0x%016" PRIx64 "/0x%016" PRIx64 + " extent=%d:%ux%u wh=%d:%ux%u pitch=%d:%u/%u/%u zfmt=%u sfmt=%u samples=%u\n", + current_references, registered_writes, meta_addr, described_meta_size, + z.z_read_base_addr, z.z_write_base_addr, z.stencil_read_base_addr, + z.stencil_write_base_addr, z.size.valid, static_cast(z.size.x_max) + 1u, + static_cast(z.size.y_max) + 1u, z.width_height_valid, z.width, z.height, + z.pitch_height_valid, z.pitch_div8_minus1, z.height_div8_minus1, + z.slice_div64_minus1, z.z_info.format, z.stencil_info.format, + z.z_info.num_samples); } - cache->RegisterMeta(target.address, target.size); + cache->RegisterMeta(g_render_ctx->GetGraphicCtx(), target.address, target.size); } else { target = registered_target; } diff --git a/src/graphics/host_gpu/renderer/renderTarget.h b/src/graphics/host_gpu/renderer/renderTarget.h index c6b3584dd..23a79f3cc 100644 --- a/src/graphics/host_gpu/renderer/renderTarget.h +++ b/src/graphics/host_gpu/renderer/renderTarget.h @@ -10,14 +10,18 @@ namespace Libs::Graphics { static constexpr uint32_t RENDER_COLOR_ATTACHMENTS_MAX = 8; -inline constexpr bool depth_msaa_single_sample_compatible(uint32_t encoded_fragments) { - return encoded_fragments == 1 || encoded_fragments == 2; +[[nodiscard]] inline constexpr uint32_t render_sample_count(uint32_t encoded_samples) { + return encoded_samples <= 3 ? 1u << encoded_samples : 0; } -inline constexpr bool color_msaa_single_sample_compatible(uint32_t encoded_samples, - uint32_t encoded_fragments) { - return encoded_samples == encoded_fragments && - depth_msaa_single_sample_compatible(encoded_fragments); +[[nodiscard]] inline constexpr vk::SampleCountFlagBits vulkan_sample_count(uint32_t samples) { + switch (samples) { + case 1: return vk::SampleCountFlagBits::e1; + case 2: return vk::SampleCountFlagBits::e2; + case 4: return vk::SampleCountFlagBits::e4; + case 8: return vk::SampleCountFlagBits::e8; + default: return {}; + } } enum class TargetViewType : uint8_t { Image2D, Image2DArray, Unsupported }; diff --git a/src/graphics/host_gpu/renderer/shaders.cpp b/src/graphics/host_gpu/renderer/shaders.cpp index 0c712d94b..bac4a10b9 100644 --- a/src/graphics/host_gpu/renderer/shaders.cpp +++ b/src/graphics/host_gpu/renderer/shaders.cpp @@ -755,8 +755,8 @@ void CreatePipelineInternal(PipelineCache::GraphicsPipeline* pipeline, vk::Rende multisampling.sType = vk::StructureType::ePipelineMultisampleStateCreateInfo; multisampling.pNext = nullptr; multisampling.flags = {}; - multisampling.sampleShadingEnable = VK_FALSE; - multisampling.rasterizationSamples = vk::SampleCountFlagBits::e1; + multisampling.sampleShadingEnable = static_params.sample_shading_enable ? VK_TRUE : VK_FALSE; + multisampling.rasterizationSamples = vulkan_sample_count(static_params.samples); multisampling.minSampleShading = 1.0f; multisampling.pSampleMask = nullptr; multisampling.alphaToCoverageEnable = VK_FALSE; diff --git a/src/graphics/host_gpu/renderer/textureCache.cpp b/src/graphics/host_gpu/renderer/textureCache.cpp index 4248b5607..403d48fce 100644 --- a/src/graphics/host_gpu/renderer/textureCache.cpp +++ b/src/graphics/host_gpu/renderer/textureCache.cpp @@ -24,6 +24,7 @@ #include #include +#include #include #include #include @@ -38,6 +39,25 @@ namespace { thread_local const void* g_texture_cache_lock_owner = nullptr; thread_local const void* g_texture_fault_owner = nullptr; +[[nodiscard]] bool GuestRangeIsZero(uint64_t address, uint64_t size) noexcept { + const auto* bytes = reinterpret_cast(address); + while (size >= sizeof(uint64_t)) { + uint64_t word = 0; + std::memcpy(&word, bytes, sizeof(word)); + if (word != 0) { + return false; + } + bytes += sizeof(word); + size -= sizeof(word); + } + while (size-- != 0) { + if (*bytes++ != 0) { + return false; + } + } + return true; +} + [[nodiscard]] uint64_t HashSampledImageEdges(const Image& image) { constexpr uint64_t page_mask = TRACKER_PAGE_SIZE - 1; const uint64_t begin = image.address; @@ -265,6 +285,20 @@ struct TextureCache::ReadbackWorker { uint64_t address; uint64_t size; }; + struct ReadbackTransfer { + std::array ranges {}; + uint32_t count = 0; + + void Add(uint64_t address, uint64_t size) { + if (address == 0 || size == 0 || count == ranges.size()) { + EXIT("TextureCache: invalid image readback transfer range\n"); + } + ranges[count++] = {address, size}; + } + [[nodiscard]] std::span Ranges() const { + return {ranges.data(), count}; + } + }; static_assert(std::atomic::is_always_lock_free); explicit ReadbackWorker(TextureCache& owner): cache(owner), thread([this] { Run(); }) {} @@ -374,94 +408,142 @@ struct TextureCache::ReadbackWorker { } } - [[nodiscard]] ReadbackRange DownloadDepthTarget(CachedImage& cached, - bool require_fault_range = true) { - const auto& info = cached.depth; - const bool has_stencil = info.stencil_address != 0 || info.stencil_size != 0; - const bool has_htile = info.htile_address != 0 || info.htile_size != 0; - const bool fault_in_depth = vaddr >= info.address && vaddr < info.address + info.size && - size <= info.address + info.size - vaddr; - const bool d16 = + [[nodiscard]] ReadbackTransfer DownloadDepthTarget(CachedImage& cached, + bool require_fault_range = true) { + const auto& info = cached.depth; + if (info.samples != 1 || cached.image->samples != 1) { + EXIT("TextureCache: multisampled depth-image readback is unsupported, samples=%u/%u\n", + info.samples, cached.image->samples); + } + const bool has_stencil = info.stencil_address != 0 || info.stencil_size != 0; + const bool has_htile = info.htile_address != 0 || info.htile_size != 0; + const bool fault_in_depth = vaddr >= info.address && vaddr < info.address + info.size && + size <= info.address + info.size - vaddr; + const bool fault_in_stencil = has_stencil && vaddr >= info.stencil_address && + vaddr < info.stencil_address + info.stencil_size && + size <= info.stencil_address + info.stencil_size - vaddr; + const bool d16 = info.guest_format == Prospero::GpuEnumValue(Prospero::BufferFormat::k16UNorm) && - info.format == vk::Format::eD16Unorm && info.bytes_per_element == 2; + info.bytes_per_element == 2; const bool d32 = info.guest_format == Prospero::GpuEnumValue(Prospero::BufferFormat::k32Float) && - info.format == vk::Format::eD32Sfloat && info.bytes_per_element == 4; + info.bytes_per_element == 4; TileSizeAlign expected_stencil {}; TileSizeAlign expected_htile {}; TileSizeAlign expected_depth {}; const bool prospero_layout = - (d16 || d32) && + (d16 || d32) && IsSupportedDepthReadbackFormat(info) && TileGetDepthSize(info.width, info.height, 0, Prospero::GpuEnumValue(d16 ? Prospero::DepthFormat::kZ16 : Prospero::DepthFormat::kZ32F), - Prospero::GpuEnumValue(Prospero::StencilFormat::kInvalid), has_htile, - &expected_stencil, &expected_htile, &expected_depth); + Prospero::GpuEnumValue(has_stencil + ? Prospero::StencilFormat::k8UInt + : Prospero::StencilFormat::kInvalid), + has_htile, &expected_stencil, &expected_htile, &expected_depth); const auto expected_pitch = prospero_layout ? TileGetTexturePitch(info.guest_format, info.width, 1, Prospero::GpuEnumValue(Prospero::TileMode::kDepth)) : 0u; + const auto expected_stencil_pitch = + prospero_layout && has_stencil + ? TileGetTexturePitch(Prospero::GpuEnumValue(Prospero::BufferFormat::k8UInt), + info.width, 1, + Prospero::GpuEnumValue(Prospero::TileMode::kDepth)) + : 0u; const bool layered_sizes = info.layers != 0 && info.size <= UINT32_MAX && expected_depth.size <= UINT64_MAX / info.layers && info.size == expected_depth.size * info.layers && + (!has_stencil || (info.stencil_size <= UINT32_MAX && + expected_stencil.size <= UINT64_MAX / info.layers && + info.stencil_size == expected_stencil.size * info.layers)) && (!has_htile || (expected_htile.size <= UINT64_MAX / info.layers && info.htile_size == expected_htile.size * info.layers)); - if ((require_fault_range && !fault_in_depth) || has_stencil || info.address == 0 || + if ((require_fault_range && !fault_in_depth && !fault_in_stencil) || info.address == 0 || info.size == 0 || info.width == 0 || info.height == 0 || info.pitch < info.width || info.tile_mode != Prospero::GpuEnumValue(Prospero::TileMode::kDepth) || !prospero_layout || info.pitch != expected_pitch || !layered_sizes || expected_depth.align != 65536u || cached.image->layers != info.layers || + (has_stencil && (info.stencil_address == 0 || expected_stencil_pitch < info.width || + expected_stencil.align != 65536u)) || (has_htile && expected_htile.align != 32768u)) { EXIT("TextureCache: unsupported depth-image readback layout, fault=0x%016" PRIx64 "+0x%016" PRIx64 " depth=0x%016" PRIx64 "+0x%016" PRIx64 " stencil=0x%016" PRIx64 "+0x%016" PRIx64 " htile=0x%016" PRIx64 "+0x%016" PRIx64 " extent=%ux%u pitch=%u/%u layers=%u/%u tile=%u format=%d guest_format=%u bpe=%u " - "expected_depth=0x%016" PRIx64 " expected_htile=0x%016" PRIx64 "\n", + "expected_depth=0x%016" PRIx64 " expected_stencil=0x%016" PRIx64 + " expected_htile=0x%016" PRIx64 "\n", vaddr, size, info.address, info.size, info.stencil_address, info.stencil_size, info.htile_address, info.htile_size, info.width, info.height, info.pitch, expected_pitch, info.layers, cached.image->layers, info.tile_mode, static_cast(info.format), info.guest_format, info.bytes_per_element, - expected_depth.size, expected_htile.size); + expected_depth.size, expected_stencil.size, expected_htile.size); } const auto rows = static_cast(info.height - 1); - if (rows > (UINT64_MAX - info.width) / info.pitch) { + if (rows > (UINT64_MAX - info.width) / info.pitch || + (has_stencil && rows > (UINT64_MAX - info.width) / expected_stencil_pitch)) { EXIT("TextureCache: depth-image readback size overflow\n"); } - const auto linear_elements = rows * info.pitch + info.width; - if (linear_elements > UINT64_MAX / info.bytes_per_element) { + const auto depth_linear_elements = rows * info.pitch + info.width; + const auto stencil_linear_elements = rows * expected_stencil_pitch + info.width; + if (depth_linear_elements > UINT64_MAX / info.bytes_per_element) { EXIT("TextureCache: depth-image readback size overflow\n"); } - const auto linear_size = linear_elements * info.bytes_per_element; - const auto slice_size = info.size / info.layers; - const bool meta_overlap = cache.HasMetaOverlapLocked(info.address, info.size); - const bool buffer_overlap = cache.m_buffer_cache.HasPageOverlap(info.address, info.size); - if (linear_size > slice_size || cached.image->format != info.format || + const auto depth_linear_size = depth_linear_elements * info.bytes_per_element; + const auto depth_slice_size = info.size / info.layers; + const auto stencil_slice_size = has_stencil ? info.stencil_size / info.layers : 0; + const bool meta_overlap = + cache.HasMetaOverlapLocked(info.address, info.size) || + (has_stencil && cache.HasMetaOverlapLocked(info.stencil_address, info.stencil_size)); + const bool buffer_overlap = cache.m_buffer_cache.HasPageOverlap(info.address, info.size) || + (has_stencil && cache.m_buffer_cache.HasPageOverlap( + info.stencil_address, info.stencil_size)); + const auto transfer_size = info.size + info.stencil_size; + if (depth_linear_size > depth_slice_size || + (has_stencil && stencil_linear_elements > stencil_slice_size) || + transfer_size > UINT32_MAX || cached.image->format != info.format || cached.image->extent.width != info.width || cached.image->extent.height != info.height || meta_overlap || buffer_overlap) { EXIT("TextureCache: depth-image readback storage is unsupported, " "depth=0x%016" PRIx64 "+0x%016" PRIx64 " linear=0x%016" PRIx64 " format=%d/%d extent=%ux%u/%ux%u meta=%d buffer=%d\n", - info.address, info.size, linear_size, static_cast(cached.image->format), + info.address, info.size, depth_linear_size, static_cast(cached.image->format), static_cast(info.format), cached.image->extent.width, cached.image->extent.height, info.width, info.height, meta_overlap, buffer_overlap); } - download.resize(info.size); - std::fill(download.begin(), download.end(), 0); - const auto regions = - Transfer::MakeLayeredImageBufferCopies(info.layers, slice_size, info.pitch, info.width, - info.height, vk::ImageAspectFlagBits::eDepth); - Transfer::DownloadImage(cached.ctx, download.data(), info.size, regions, cached.image, + download.resize(transfer_size); + auto regions = Transfer::MakeLayeredImageBufferCopies(info.layers, depth_slice_size, + info.pitch, info.width, info.height, + vk::ImageAspectFlagBits::eDepth); + if (has_stencil) { + auto stencil_regions = Transfer::MakeLayeredImageBufferCopies( + info.layers, stencil_slice_size, expected_stencil_pitch, info.width, info.height, + vk::ImageAspectFlagBits::eStencil); + for (auto& region: stencil_regions) { + region.offset += static_cast(info.size); + } + regions.insert(regions.end(), stencil_regions.begin(), stencil_regions.end()); + } + Transfer::DownloadImage(cached.ctx, download.data(), transfer_size, regions, cached.image, cached.image->layout); guest.resize(info.size); cache.m_tiler.TileImage(guest.data(), download.data(), info); Libs::LibKernel::Memory::WriteBacking(info.address, guest.data(), info.size); - return {info.address, info.size}; + ReadbackTransfer transfer; + transfer.Add(info.address, info.size); + if (has_stencil) { + guest.resize(info.stencil_size); + cache.m_tiler.TileStencil(guest.data(), download.data() + info.size, info); + Libs::LibKernel::Memory::WriteBacking(info.stencil_address, guest.data(), + info.stencil_size); + transfer.Add(info.stencil_address, info.stencil_size); + } + return transfer; } - [[nodiscard]] ReadbackRange DownloadColorImage(CachedImage& cached) { + [[nodiscard]] ReadbackTransfer DownloadColorImage(CachedImage& cached) { const bool storage = cached.kind == CachedImage::Kind::StorageTexture; const bool target = cached.kind == CachedImage::Kind::RenderTarget; const auto info = @@ -487,12 +569,13 @@ struct TextureCache::ReadbackWorker { (single_layer_storage && cached.info.base_level == 0 && cached.info.levels == 1 && cached.info.base_array == 0 && (linear || tiled_storage)); const auto layers = target ? cached.target.layers : 1u; - if ((!linear && !tiled_target && !tiled_storage) || !basic_storage || info.levels != 1 || - info.size > UINT32_MAX) { + if (info.samples != 1 || (!linear && !tiled_target && !tiled_storage) || !basic_storage || + info.levels != 1 || info.size > UINT32_MAX) { EXIT("TextureCache: unsupported color-image readback layout, addr=0x%016" PRIx64 - " size=0x%016" PRIx64 " extent=%ux%u pitch=%u bpe=%u levels=%u tile=%u kind=%u\n", + " size=0x%016" PRIx64 + " extent=%ux%u pitch=%u bpe=%u levels=%u samples=%u tile=%u kind=%u\n", info.address, info.size, info.width, info.height, info.pitch, - info.bytes_per_element, info.levels, info.tile_mode, + info.bytes_per_element, info.levels, info.samples, info.tile_mode, static_cast(cached.kind)); } const auto slice_size = info.size / layers; @@ -528,7 +611,9 @@ struct TextureCache::ReadbackWorker { } else { Libs::LibKernel::Memory::WriteBacking(info.address, download.data(), info.size); } - return {info.address, info.size}; + ReadbackTransfer transfer; + transfer.Add(info.address, info.size); + return transfer; } void Run() noexcept { @@ -583,16 +668,23 @@ struct TextureCache::ReadbackWorker { vaddr, size); } const auto fault_page = vaddr & ~(TRACKER_PAGE_SIZE - 1); - const auto target_end = transfer.address + transfer.size; const auto page_end = fault_page + TRACKER_PAGE_SIZE; - if (fault_page > transfer.address) { - cache.m_memory_tracker.ForEachDownloadRange( - transfer.address, fault_page - transfer.address, - [](uint64_t, uint64_t) noexcept {}); - } - if (page_end < target_end) { - cache.m_memory_tracker.ForEachDownloadRange( - page_end, target_end - page_end, [](uint64_t, uint64_t) noexcept {}); + for (const auto& range: transfer.Ranges()) { + const auto range_end = range.address + range.size; + if (fault_page < range_end && page_end > range.address) { + if (fault_page > range.address) { + cache.m_memory_tracker.ForEachDownloadRange( + range.address, fault_page - range.address, + [](uint64_t, uint64_t) noexcept {}); + } + if (page_end < range_end) { + cache.m_memory_tracker.ForEachDownloadRange( + page_end, range_end - page_end, [](uint64_t, uint64_t) noexcept {}); + } + } else { + cache.m_memory_tracker.ForEachDownloadRange( + range.address, range.size, [](uint64_t, uint64_t) noexcept {}); + } } state.store(State::Ready, std::memory_order_release); @@ -604,10 +696,12 @@ struct TextureCache::ReadbackWorker { } state.wait(current, std::memory_order_acquire); } - if (cache.m_memory_tracker.IsRegionGpuModified(transfer.address, transfer.size)) { - EXIT("TextureCache: completed image readback retained GPU ownership, " - "addr=0x%016" PRIx64 " size=0x%016" PRIx64 "\n", - transfer.address, transfer.size); + for (const auto& range: transfer.Ranges()) { + if (cache.m_memory_tracker.IsRegionGpuModified(range.address, range.size)) { + EXIT("TextureCache: completed image readback retained GPU ownership, " + "addr=0x%016" PRIx64 " size=0x%016" PRIx64 "\n", + range.address, range.size); + } } selected->gpu_modified = false; submissions_prepaused = false; @@ -727,6 +821,80 @@ void TextureCache::RetireImages(const std::vector& retire, } } +void TextureCache::RetireDepthMetadataLocked(const std::vector& retire, + uint64_t preserve_address) { + std::vector addresses; + for (auto* cached: retire) { + if (cached->kind != CachedImage::Kind::DepthTarget || cached->depth.htile_address == 0) { + continue; + } + const bool retained_owner = + std::any_of(m_images.begin(), m_images.end(), [&](const auto& other) { + return std::find(retire.begin(), retire.end(), other.get()) == retire.end() && + other->kind == CachedImage::Kind::DepthTarget && + other->depth.htile_address == cached->depth.htile_address && + other->depth.htile_size == cached->depth.htile_size; + }); + if (!retained_owner && cached->depth.htile_address != preserve_address && + std::find(addresses.begin(), addresses.end(), cached->depth.htile_address) == + addresses.end()) { + addresses.push_back(cached->depth.htile_address); + } + } + for (const auto address: addresses) { + auto metadata = m_surface_metas.find(address); + auto owner = std::find_if(retire.begin(), retire.end(), [&](const auto* cached) { + return cached->kind == CachedImage::Kind::DepthTarget && + cached->depth.htile_address == address; + }); + if (metadata == m_surface_metas.end() || owner == retire.end() || + metadata->second.size != (*owner)->depth.htile_size) { + EXIT("TextureCache: retiring depth target has invalid HTile metadata, " + "addr=0x%016" PRIx64 " size=0x%016" PRIx64 "\n", + address, owner != retire.end() ? (*owner)->depth.htile_size : 0); + } + if (metadata->second.gpu_modified) { + m_metadata_tracker.ForEachDownloadRange(metadata->first, metadata->second.size, + [](uint64_t, uint64_t) noexcept {}); + } + m_metadata_tracker.UntrackMemory(metadata->first, metadata->second.size); + m_surface_metas.erase(metadata); + } +} + +void TextureCache::MaterializeImagesToGuestLocked( + GraphicContext* ctx, const std::vector>& images) { + if (std::any_of(images.begin(), images.end(), + [](const auto& cached) { return cached->gpu_modified; })) { + Transfer::WaitForGraphicsIdle(ctx); + } + for (const auto& cached: images) { + if (!cached->gpu_modified) { + continue; + } + if (cached->ctx != ctx || cached->buffer_modified) { + EXIT("TextureCache: image recreation has invalid ownership, kind=%u ctx=%p/%p " + "buffer_modified=%d\n", + static_cast(cached->kind), static_cast(cached->ctx), + static_cast(ctx), cached->buffer_modified); + } + if (cached->kind != CachedImage::Kind::DepthTarget && + cached->kind != CachedImage::Kind::RenderTarget && + cached->kind != CachedImage::Kind::StorageTexture) { + EXIT("TextureCache: unsupported image recreation source kind %u\n", + static_cast(cached->kind)); + } + const auto transfer = cached->kind == CachedImage::Kind::DepthTarget + ? m_readback->DownloadDepthTarget(*cached, false) + : m_readback->DownloadColorImage(*cached); + for (const auto& range: transfer.Ranges()) { + m_memory_tracker.ForEachDownloadRange(range.address, range.size, + [](uint64_t, uint64_t) noexcept {}); + } + cached->gpu_modified = false; + } +} + namespace { bool Equal(const ImageInfo& left, const ImageInfo& right) { @@ -752,7 +920,8 @@ bool Equal(const RenderTargetInfo& left, const RenderTargetInfo& right) { left.format == right.format && left.width == right.width && left.height == right.height && left.pitch == right.pitch && left.bytes_per_element == right.bytes_per_element && left.tile_mode == right.tile_mode && - left.levels == right.levels && left.layers == right.layers; + left.levels == right.levels && left.layers == right.layers && + left.samples == right.samples; } bool Equal(const DepthTargetInfo& left, const DepthTargetInfo& right) { @@ -763,7 +932,7 @@ bool Equal(const DepthTargetInfo& left, const DepthTargetInfo& right) { left.guest_format == right.guest_format && left.width == right.width && left.height == right.height && left.pitch == right.pitch && left.bytes_per_element == right.bytes_per_element && left.tile_mode == right.tile_mode && - left.layers == right.layers && + left.layers == right.layers && left.samples == right.samples && left.stencil_htile_compressed == right.stencil_htile_compressed; } @@ -888,6 +1057,7 @@ void TextureCache::RetireSampledTargetAliases(GraphicContext* ctx, const ImageIn retire.push_back(cached); break; case RenderTargetOverlap::RetireSampled: + case RenderTargetOverlap::RetireStorage: case RenderTargetOverlap::PreserveStorage: case RenderTargetOverlap::ExpandTarget: case RenderTargetOverlap::Unsupported: @@ -902,9 +1072,13 @@ void TextureCache::RetireSampledTargetAliases(GraphicContext* ctx, const ImageIn if (cached->kind != CachedImage::Kind::DepthTarget) { continue; } - const auto overlap = ClassifyImageRangeOverlap(requested.address, requested.size, + const bool overlaps_depth = ImageRangeOverlaps(requested.address, requested.size, cached->depth.address, cached->depth.size); - if (overlap == ImageRangeOverlap::None || overlap == ImageRangeOverlap::PageOnly) { + const bool overlaps_stencil = + cached->depth.stencil_address != 0 && + ImageRangeOverlaps(requested.address, requested.size, cached->depth.stencil_address, + cached->depth.stencil_size); + if (!overlaps_depth && !overlaps_stencil) { continue; } const auto delta = cached->depth.address >= requested.address @@ -915,27 +1089,38 @@ void TextureCache::RetireSampledTargetAliases(GraphicContext* ctx, const ImageIn const bool sampled_expansion = IsSampledDepthExpansion(requested, cached->depth); const bool exact_range = requested.address == cached->depth.address && requested.size == cached->depth.size; - const bool supported = cached->ctx == ctx && !cached->buffer_modified && - cached->depth.stencil_address == 0 && - cached->depth.stencil_size == 0 && cached->depth.layers == 1 && - contained && (exact_range || sampled_expansion); + const bool depth_tracker_gpu = + m_memory_tracker.IsRegionGpuModified(cached->depth.address, cached->depth.size); + const bool stencil_tracker_gpu = + cached->depth.stencil_address != 0 && + m_memory_tracker.IsRegionGpuModified(cached->depth.stencil_address, + cached->depth.stencil_size); + const bool clean_pool_alias = CanRetireGuestCurrentDepthForSampled( + requested, cached->depth, cached->gpu_modified, cached->buffer_modified, + depth_tracker_gpu, stencil_tracker_gpu, cached->ctx == ctx); + const bool native_transition = + cached->ctx == ctx && !cached->buffer_modified && cached->depth.stencil_address == 0 && + cached->depth.stencil_size == 0 && cached->depth.layers == 1 && contained && + (exact_range || sampled_expansion); + const bool supported = clean_pool_alias || native_transition; if (!supported) { EXIT("TextureCache: unsupported sampled/depth-target alias, sampled=0x%016" PRIx64 "+0x%016" PRIx64 " depth=0x%016" PRIx64 "+0x%016" PRIx64 - " overlap=%u contained=%d same_context=%d gpu=%d buffer=%d stencil=0x%016" PRIx64 - "+0x%016" PRIx64 " layers=%u expansion=%d" + " overlaps=%d/%d contained=%d same_context=%d gpu=%d/%d/%d buffer=%d" + " stencil=0x%016" PRIx64 "+0x%016" PRIx64 " layers=%u expansion=%d" " sampled_info={fmt=%u extent=%ux%u pitch=%u levels=%u/%u tile=%u type=%u}" " depth_info={fmt=%u host=%d extent=%ux%u pitch=%u tile=%u htile=0x%016" PRIx64 "+0x%016" PRIx64 "}\n", requested.address, requested.size, cached->depth.address, cached->depth.size, - static_cast(overlap), contained, cached->ctx == ctx, - cached->gpu_modified, cached->buffer_modified, cached->depth.stencil_address, - cached->depth.stencil_size, cached->depth.layers, sampled_expansion, - requested.format, requested.width, requested.height, requested.pitch, - requested.levels, requested.view_levels, requested.tile, requested.type, - cached->depth.guest_format, static_cast(cached->depth.format), - cached->depth.width, cached->depth.height, cached->depth.pitch, - cached->depth.tile_mode, cached->depth.htile_address, cached->depth.htile_size); + overlaps_depth, overlaps_stencil, contained, cached->ctx == ctx, + cached->gpu_modified, depth_tracker_gpu, stencil_tracker_gpu, + cached->buffer_modified, cached->depth.stencil_address, cached->depth.stencil_size, + cached->depth.layers, sampled_expansion, requested.format, requested.width, + requested.height, requested.pitch, requested.levels, requested.view_levels, + requested.tile, requested.type, cached->depth.guest_format, + static_cast(cached->depth.format), cached->depth.width, cached->depth.height, + cached->depth.pitch, cached->depth.tile_mode, cached->depth.htile_address, + cached->depth.htile_size); } wait_idle |= cached->gpu_modified; retire.push_back(cached); @@ -954,48 +1139,13 @@ void TextureCache::RetireSampledTargetAliases(GraphicContext* ctx, const ImageIn const auto transfer = cached->kind == CachedImage::Kind::DepthTarget ? m_readback->DownloadDepthTarget(*cached, false) : m_readback->DownloadColorImage(*cached); - m_memory_tracker.ForEachDownloadRange(transfer.address, transfer.size, - [](uint64_t, uint64_t) noexcept {}); + for (const auto& range: transfer.Ranges()) { + m_memory_tracker.ForEachDownloadRange(range.address, range.size, + [](uint64_t, uint64_t) noexcept {}); + } cached->gpu_modified = false; } - std::vector retire_depth_metadata; - for (auto* cached: retire) { - if (cached->kind != CachedImage::Kind::DepthTarget || cached->depth.htile_address == 0) { - continue; - } - const bool retained_owner = - std::any_of(m_images.begin(), m_images.end(), [&](const auto& other) { - return std::find(retire.begin(), retire.end(), other.get()) == retire.end() && - other->kind == CachedImage::Kind::DepthTarget && - other->depth.htile_address == cached->depth.htile_address && - other->depth.htile_size == cached->depth.htile_size; - }); - if (!retained_owner && - std::find(retire_depth_metadata.begin(), retire_depth_metadata.end(), - cached->depth.htile_address) == retire_depth_metadata.end()) { - retire_depth_metadata.push_back(cached->depth.htile_address); - } - } - for (const auto address: retire_depth_metadata) { - auto metadata = m_surface_metas.find(address); - auto owner = std::find_if(retire.begin(), retire.end(), [&](const auto* cached) { - return cached->kind == CachedImage::Kind::DepthTarget && - cached->depth.htile_address == address; - }); - if (metadata == m_surface_metas.end() || owner == retire.end() || - metadata->second.size != (*owner)->depth.htile_size) { - EXIT( - "TextureCache: retiring depth target has invalid HTile metadata, addr=0x%016" PRIx64 - " size=0x%016" PRIx64 "\n", - address, owner != retire.end() ? (*owner)->depth.htile_size : 0); - } - if (metadata->second.gpu_modified) { - m_metadata_tracker.ForEachDownloadRange(metadata->first, metadata->second.size, - [](uint64_t, uint64_t) noexcept {}); - } - m_metadata_tracker.UntrackMemory(metadata->first, metadata->second.size); - m_surface_metas.erase(metadata); - } + RetireDepthMetadataLocked(retire); RetireImages(retire); } @@ -1026,8 +1176,10 @@ void TextureCache::ResolveStorageImageOverlaps(GraphicContext* ctx, const ImageI continue; } const auto transfer = m_readback->DownloadColorImage(*cached); - m_memory_tracker.ForEachDownloadRange(transfer.address, transfer.size, - [](uint64_t, uint64_t) noexcept {}); + for (const auto& range: transfer.Ranges()) { + m_memory_tracker.ForEachDownloadRange(range.address, range.size, + [](uint64_t, uint64_t) noexcept {}); + } cached->gpu_modified = false; } RetireImages(retire); @@ -1068,8 +1220,10 @@ void TextureCache::RetireStorageDepthAliasLocked(GraphicContext* ctx, const Imag } Transfer::WaitForGraphicsIdle(ctx); const auto transfer = m_readback->DownloadDepthTarget(*selected, false); - m_memory_tracker.ForEachDownloadRange(transfer.address, transfer.size, - [](uint64_t, uint64_t) noexcept {}); + for (const auto& range: transfer.Ranges()) { + m_memory_tracker.ForEachDownloadRange(range.address, range.size, + [](uint64_t, uint64_t) noexcept {}); + } selected->gpu_modified = false; RetireImages({selected}); } @@ -1312,7 +1466,7 @@ StorageTextureVulkanImage* TextureCache::FindStorageTexture(CommandBuffer* com } } FaultSafeTextureLock lock(this, m_lock); - RequireNoMetaOverlapLocked(info.address, info.size); + ResolveImageMetadataOverlapsLocked(ctx, info.address, info.size); if (supported_depth_tile && info.format == Prospero::GpuEnumValue(Prospero::BufferFormat::k32UInt)) { RetireStorageDepthAliasLocked(ctx, info); @@ -1405,18 +1559,38 @@ RenderTextureVulkanImage* TextureCache::FindRenderTarget(CommandBuffer* GraphicContext* ctx, const RenderTargetInfo& info) { const bool standard64 = IsSupportedStandard64RenderTarget(info); + const bool valid_samples = + info.samples == 1 || info.samples == 2 || info.samples == 4 || info.samples == 8; if (info.address == 0 || info.size == 0 || info.address >= TRACKER_ADDRESS_SIZE || info.size > TRACKER_ADDRESS_SIZE - info.address || info.format == vk::Format::eUndefined || info.width == 0 || info.height == 0 || info.pitch < info.width || info.bytes_per_element == 0 || info.levels == 0 || info.levels > 16 || info.layers == 0 || - info.size % info.layers != 0 || + info.size % info.layers != 0 || !valid_samples || + (info.samples > 1 && + (info.levels != 1 || info.layers != 1 || + info.tile_mode != Prospero::GpuEnumValue(Prospero::TileMode::kRenderTarget))) || (info.tile_mode != Prospero::GpuEnumValue(Prospero::TileMode::kLinear) && info.tile_mode != Prospero::GpuEnumValue(Prospero::TileMode::kRenderTarget) && !standard64)) { EXIT("TextureCache: invalid render-target request, ctx=%p addr=0x%016" PRIx64 - " size=0x%016" PRIx64 " extent=%ux%u pitch=%u bpe=%u tile=%u format=%d\n", + " size=0x%016" PRIx64 " extent=%ux%u pitch=%u bpe=%u samples=%u tile=%u format=%d\n", static_cast(ctx), info.address, info.size, info.width, info.height, - info.pitch, info.bytes_per_element, info.tile_mode, static_cast(info.format)); + info.pitch, info.bytes_per_element, info.samples, info.tile_mode, + static_cast(info.format)); + } + if (info.levels == 1 && + info.tile_mode == Prospero::GpuEnumValue(Prospero::TileMode::kRenderTarget)) { + TileSizeAlign layout {}; + const auto encoded_samples = static_cast(std::countr_zero(info.samples)); + if (!TileGetRenderTargetSize(info.width, info.height, info.pitch, info.bytes_per_element, + &layout, encoded_samples) || + layout.size > UINT64_MAX / info.layers || + static_cast(layout.size) * info.layers != info.size) { + EXIT("TextureCache: invalid render-target block layout, size=0x%016" PRIx64 + " expected=0x%08x extent=%ux%u pitch=%u bpe=%u samples=%u\n", + info.size, layout.size, info.width, info.height, info.pitch, + info.bytes_per_element, info.samples); + } } if (info.levels > 1) { TileSizeAlign layout {}; @@ -1467,8 +1641,7 @@ RenderTextureVulkanImage* TextureCache::FindRenderTarget(CommandBuffer* // partial ownership and inconsistent state as hard failures. target_source = m_buffer_cache.ObtainBufferForImage(info.address, info.size); } - FaultSafeTextureLock lock(this, m_lock); - RequireNoMetaOverlapLocked(info.address, info.size); + FaultSafeTextureLock lock(this, m_lock); std::shared_ptr match; for (auto& cached: m_images) { if (cached->kind != CachedImage::Kind::RenderTarget || cached->ctx != ctx || @@ -1479,6 +1652,13 @@ RenderTextureVulkanImage* TextureCache::FindRenderTarget(CommandBuffer* match = cached; } if (match != nullptr) { + ResolveImageMetadataOverlapsLocked(ctx, info.address, info.size); + if (info.samples > 1 && (match->buffer_modified || + m_memory_tracker.IsRegionCpuModified(info.address, info.size))) { + EXIT("TextureCache: multisampled render-target refresh is unsupported, " + "addr=0x%016" PRIx64 " size=0x%016" PRIx64 " samples=%u\n", + info.address, info.size, info.samples); + } if (match->buffer_modified) { if (match->gpu_modified || !IsCoherentGuestImageSource(target_source, info.address, info.size)) { @@ -1502,9 +1682,9 @@ RenderTextureVulkanImage* TextureCache::FindRenderTarget(CommandBuffer* command->RetainResourceUntilFence(match); return static_cast(match->image); } - std::vector retire; - std::vector buffer_owned_depth_retire; - std::shared_ptr native_image_source; + std::vector retire; + std::vector> recreated_depth_sources; + std::shared_ptr native_image_source; for (const auto& entry: m_images) { auto& cached = *entry; if (!cached.OverlapsRange(info.address, info.size, true)) { @@ -1521,20 +1701,29 @@ RenderTextureVulkanImage* TextureCache::FindRenderTarget(CommandBuffer* cached.info, cached.image->format, cached.gpu_modified, cached.buffer_modified, cached.info.IsCpuDirty() || m_memory_tracker.IsRegionCpuModified(cached.info.address, cached.info.size), + m_memory_tracker.IsRegionGpuModified(cached.info.address, cached.info.size), cached.ctx == ctx, info); break; case CachedImage::Kind::RenderTarget: - overlap = - ClassifyRenderTargetOverlap(cached.target, cached.gpu_modified, - cached.buffer_modified, cached.ctx == ctx, info); + overlap = ClassifyRenderTargetOverlap( + cached.target, cached.gpu_modified, cached.buffer_modified, + m_memory_tracker.IsRegionGpuModified(cached.target.address, cached.target.size), + cached.ctx == ctx, + IsCoherentGuestImageSource(target_source, info.address, info.size), info); break; - case CachedImage::Kind::DepthTarget: - if (CanRetireBufferOwnedDepthForRenderTarget( + case CachedImage::Kind::DepthTarget: { + bool tracker_gpu_modified = false; + for (uint32_t range = 0; range < cached.RangeCount(); range++) { + tracker_gpu_modified |= m_memory_tracker.IsRegionGpuModified( + cached.Address(range), cached.Size(range)); + } + if (CanRecreateDepthForRenderTarget( cached.depth, cached.gpu_modified, cached.buffer_modified, - cached.ctx == ctx, target_source.buffer != nullptr, info)) { + tracker_gpu_modified, cached.ctx == ctx, + IsCoherentGuestImageSource(target_source, info.address, info.size), info)) { overlap = RenderTargetOverlap::RetireTarget; } - break; + } break; case CachedImage::Kind::VideoOut: break; } if (overlap == RenderTargetOverlap::None) { @@ -1545,6 +1734,9 @@ RenderTextureVulkanImage* TextureCache::FindRenderTarget(CommandBuffer* case RenderTargetOverlap::RetireSampled: supported = cached.kind == CachedImage::Kind::Texture; break; + case RenderTargetOverlap::RetireStorage: + supported = cached.kind == CachedImage::Kind::StorageTexture; + break; case RenderTargetOverlap::PreserveStorage: supported = cached.kind == CachedImage::Kind::StorageTexture && native_image_source == nullptr; @@ -1560,12 +1752,10 @@ RenderTextureVulkanImage* TextureCache::FindRenderTarget(CommandBuffer* } break; case RenderTargetOverlap::RetireTarget: - supported = - cached.kind == CachedImage::Kind::RenderTarget || - (cached.kind == CachedImage::Kind::DepthTarget && target_buffer_overlap && - IsCoherentGuestImageSource(target_source, info.address, info.size)); + supported = cached.kind == CachedImage::Kind::RenderTarget || + cached.kind == CachedImage::Kind::DepthTarget; if (supported && cached.kind == CachedImage::Kind::DepthTarget) { - buffer_owned_depth_retire.push_back(&cached); + recreated_depth_sources.push_back(entry); } break; case RenderTargetOverlap::None: @@ -1586,12 +1776,15 @@ RenderTextureVulkanImage* TextureCache::FindRenderTarget(CommandBuffer* retire.push_back(&cached); } RequireRetirementIsolation(retire, "render target", info.address, info.size); - for (auto* cached: buffer_owned_depth_retire) { - // The exact formatted buffer owns the current bytes. Clear the stale-image marker only - // after coherence validation so normal retirement can remove the obsolete Vulkan shape. + MaterializeImagesToGuestLocked(ctx, recreated_depth_sources); + for (const auto& cached: recreated_depth_sources) { + // The exact guest range owns the current bytes. Clear a stale buffer marker only after + // source-coherence validation so retirement can replace the obsolete Vulkan shape. cached->buffer_modified = false; } + RetireDepthMetadataLocked(retire); RetireImages(retire, native_image_source.get()); + ResolveImageMetadataOverlapsLocked(ctx, info.address, info.size); auto cached = std::make_shared(); cached->kind = CachedImage::Kind::RenderTarget; cached->target = info; @@ -1624,12 +1817,25 @@ RenderTextureVulkanImage* TextureCache::FindRenderTarget(CommandBuffer* AppendLayerCopies(regions, native_image_source->image, vk::ImageAspectFlagBits::eColor); Transfer::CopyImage(command, regions, cached->image, RENDER_COLOR_IMAGE_LAYOUT); } else { - m_memory_tracker.ForEachUploadRange( - info.address, info.size, false, [](uint64_t, uint64_t) noexcept {}, - [&]() noexcept { - ImageOps::UploadRenderTarget( - ctx, static_cast(cached->image), info, false); - }); + if (info.samples > 1) { + if (!IsCoherentGuestImageSource(target_source, info.address, info.size) || + !GuestRangeIsZero(info.address, info.size)) { + EXIT("TextureCache: nonzero multisampled render-target upload is unsupported, " + "addr=0x%016" PRIx64 " size=0x%016" PRIx64 " samples=%u\n", + info.address, info.size, info.samples); + } + m_memory_tracker.ForEachUploadRange( + info.address, info.size, false, [](uint64_t, uint64_t) noexcept {}, + []() noexcept {}); + static_cast(cached->image)->initial_clear_pending = true; + } else { + m_memory_tracker.ForEachUploadRange( + info.address, info.size, false, [](uint64_t, uint64_t) noexcept {}, + [&]() noexcept { + ImageOps::UploadRenderTarget( + ctx, static_cast(cached->image), info, false); + }); + } } cached->gpu_modified = preserve_native; return static_cast(PublishImage(command, std::move(cached))); @@ -1639,10 +1845,13 @@ DepthStencilVulkanImage* TextureCache::FindDepthTarget(CommandBuffer* command, G const DepthTargetInfo& info) { const bool has_stencil = info.stencil_address != 0 || info.stencil_size != 0; const bool has_htile = info.htile_address != 0 || info.htile_size != 0; + const bool valid_samples = + info.samples == 1 || info.samples == 2 || info.samples == 4 || info.samples == 8; if (info.address == 0 || info.size == 0 || info.address >= TRACKER_ADDRESS_SIZE || info.size > TRACKER_ADDRESS_SIZE - info.address || (info.address & 0xffffu) != 0 || info.width == 0 || info.height == 0 || info.pitch < info.width || info.layers == 0 || - info.size % info.layers != 0 || info.size > UINT32_MAX || info.stencil_size > UINT32_MAX || + !valid_samples || (info.samples > 1 && info.layers != 1) || info.size % info.layers != 0 || + info.size > UINT32_MAX || info.stencil_size > UINT32_MAX || info.tile_mode != Prospero::GpuEnumValue(Prospero::TileMode::kDepth) || (has_stencil && (info.stencil_address == 0 || info.stencil_size == 0 || @@ -1663,27 +1872,54 @@ DepthStencilVulkanImage* TextureCache::FindDepthTarget(CommandBuffer* command, G !IsSupportedDepthTargetFormat(info)) { EXIT("TextureCache: unsupported depth target, command=%p ctx=%p depth=0x%016" PRIx64 "+0x%016" PRIx64 " stencil=0x%016" PRIx64 "+0x%016" PRIx64 " htile=0x%016" PRIx64 - "+0x%016" PRIx64 " extent=%ux%u pitch=%u tile=%u format=%d guest_format=%u bpe=%u\n", + "+0x%016" PRIx64 + " extent=%ux%u pitch=%u samples=%u tile=%u format=%d guest_format=%u bpe=%u\n", static_cast(command), static_cast(ctx), info.address, info.size, info.stencil_address, info.stencil_size, info.htile_address, - info.htile_size, info.width, info.height, info.pitch, info.tile_mode, + info.htile_size, info.width, info.height, info.pitch, info.samples, info.tile_mode, static_cast(info.format), info.guest_format, info.bytes_per_element); } + const auto* depth_policy = FindGuestDepthFormatPolicy(info.guest_format); + TileSizeAlign expected_depth {}; + TileSizeAlign expected_stencil {}; + TileSizeAlign expected_htile {}; + const auto encoded_samples = static_cast(std::countr_zero(info.samples)); + if (depth_policy == nullptr || depth_policy->bytes_per_element != info.bytes_per_element || + info.pitch != TileGetDepthPitch(info.width, info.bytes_per_element, encoded_samples) || + !TileGetDepthSize( + info.width, info.height, 0, Prospero::GpuEnumValue(depth_policy->depth_format), + Prospero::GpuEnumValue(has_stencil ? Prospero::StencilFormat::k8UInt + : Prospero::StencilFormat::kInvalid), + has_htile, &expected_stencil, &expected_htile, &expected_depth, encoded_samples) || + expected_depth.size > UINT64_MAX / info.layers || + static_cast(expected_depth.size) * info.layers != info.size || + (has_stencil && + (expected_stencil.size > UINT64_MAX / info.layers || + static_cast(expected_stencil.size) * info.layers != info.stencil_size)) || + (has_htile && + (expected_htile.size > UINT64_MAX / info.layers || + static_cast(expected_htile.size) * info.layers != info.htile_size))) { + EXIT("TextureCache: invalid depth-target block layout, depth=0x%016" PRIx64 "+0x%016" PRIx64 + " stencil=0x%016" PRIx64 "+0x%016" PRIx64 " htile=0x%016" PRIx64 "+0x%016" PRIx64 + " extent=%ux%u pitch=%u samples=%u\n", + info.address, info.size, info.stencil_address, info.stencil_size, info.htile_address, + info.htile_size, info.width, info.height, info.pitch, info.samples); + } const auto rows = static_cast(info.height - 1); if (rows > (UINT64_MAX - info.width) / info.pitch) { EXIT("TextureCache: depth-target element count overflow, extent=%ux%u pitch=%u\n", info.width, info.height, info.pitch); } const auto elements = rows * info.pitch + info.width; - if (elements > UINT64_MAX / info.bytes_per_element || - elements * info.bytes_per_element > UINT64_MAX / info.layers || - info.size < elements * info.bytes_per_element * info.layers) { + if (elements > UINT64_MAX / info.bytes_per_element / info.samples || + elements * info.bytes_per_element * info.samples > UINT64_MAX / info.layers || + info.size < elements * info.bytes_per_element * info.samples * info.layers) { EXIT("TextureCache: depth storage is too small, size=0x%016" PRIx64 " elements=0x%016" PRIx64 " bpe=%u\n", info.size, elements, info.bytes_per_element); } - if (has_stencil && - (elements > UINT64_MAX / info.layers || info.stencil_size < elements * info.layers)) { + if (has_stencil && (elements > UINT64_MAX / info.samples / info.layers || + info.stencil_size < elements * info.samples * info.layers)) { EXIT("TextureCache: stencil storage is too small, size=0x%016" PRIx64 " elements=0x%016" PRIx64 "\n", info.stencil_size, elements); @@ -1694,7 +1930,7 @@ DepthStencilVulkanImage* TextureCache::FindDepthTarget(CommandBuffer* command, G info.stencil_address, info.stencil_size, info.htile_address, info.htile_size); } if (has_htile) { - RegisterMeta(info.htile_address, info.htile_size, info.layers); + RegisterMeta(ctx, info.htile_address, info.htile_size, info.layers); } std::lock_guard transaction(m_resource_mutex); // BufferCache treats an untracked guest range as CPU-current. That is useful when @@ -1706,9 +1942,9 @@ DepthStencilVulkanImage* TextureCache::FindDepthTarget(CommandBuffer* command, G has_stencil ? m_buffer_cache.ObtainBufferForImage(info.stencil_address, info.stencil_size) : BufferImageCopySource {}; FaultSafeTextureLock lock(this, m_lock); - RequireNoMetaOverlapLocked(info.address, info.size); + ResolveImageMetadataOverlapsLocked(ctx, info.address, info.size); if (has_stencil) { - RequireNoMetaOverlapLocked(info.stencil_address, info.stencil_size); + ResolveImageMetadataOverlapsLocked(ctx, info.stencil_address, info.stencil_size); } std::shared_ptr match; for (auto& cached: m_images) { @@ -1719,6 +1955,20 @@ DepthStencilVulkanImage* TextureCache::FindDepthTarget(CommandBuffer* command, G match = cached; } if (match != nullptr) { + const bool depth_cpu_modified = + m_memory_tracker.IsRegionCpuModified(info.address, info.size); + const bool stencil_cpu_modified = + has_stencil && + m_memory_tracker.IsRegionCpuModified(info.stencil_address, info.stencil_size); + if (RequiresMultisampleDepthRefresh(info, match->buffer_modified, depth_cpu_modified, + stencil_cpu_modified)) { + EXIT("TextureCache: multisampled depth-target refresh is unsupported, " + "addr=0x%016" PRIx64 " size=0x%016" PRIx64 + " samples=%u access=%d/%d clear=%d/%d buffer=%d cpu=%d/%d\n", + info.address, info.size, info.samples, info.depth_access, info.stencil_access, + info.depth_load_clear, info.stencil_load_clear, match->buffer_modified, + depth_cpu_modified, stencil_cpu_modified); + } if (!CanLoadStencilAttachment(info, match->stencil_initialized)) { EXIT("TextureCache: stencil load requires initialized contents, addr=0x%016" PRIx64 " size=0x%016" PRIx64 "\n", @@ -1768,11 +2018,12 @@ DepthStencilVulkanImage* TextureCache::FindDepthTarget(CommandBuffer* command, G command->RetainResourceUntilFence(match); return static_cast(match->image); } - std::vector retire; - std::shared_ptr sampled_depth_source; - std::shared_ptr native_depth_source; - std::shared_ptr discarded_depth_source; - std::shared_ptr retired_storage_source; + std::vector retire; + std::shared_ptr sampled_depth_source; + std::shared_ptr native_depth_source; + std::shared_ptr discarded_depth_source; + std::shared_ptr retired_storage_source; + std::vector> recreated_target_sources; for (const auto& entry: m_images) { auto& cached = *entry; const bool overlaps = @@ -1783,43 +2034,88 @@ DepthStencilVulkanImage* TextureCache::FindDepthTarget(CommandBuffer* command, G } DepthOverlap overlap = DepthOverlap::Unsupported; switch (cached.kind) { - case CachedImage::Kind::Texture: - overlap = cached.ctx == ctx - ? ClassifyDepthOverlap(cached.info, cached.gpu_modified, info) - : DepthOverlap::Unsupported; - break; + case CachedImage::Kind::Texture: { + const auto native_overlap = + cached.ctx == ctx ? ClassifyDepthOverlap(cached.info, cached.gpu_modified, info) + : DepthOverlap::Unsupported; + const bool overlaps_depth = ImageRangeOverlaps( + cached.info.address, cached.info.size, info.address, info.size); + const bool overlaps_stencil = + has_stencil && ImageRangeOverlaps(cached.info.address, cached.info.size, + info.stencil_address, info.stencil_size); + const bool guest_source_current = + (!overlaps_depth || info.depth_load_clear || + IsCoherentGuestImageSource(depth_source, info.address, info.size)) && + (!overlaps_stencil || info.stencil_load_clear || + IsCoherentGuestImageSource(stencil_source, info.stencil_address, + info.stencil_size)); + overlap = native_overlap; + if (overlap == DepthOverlap::Unsupported && + CanRetireGuestCurrentSampledForDepth( + cached.info, info, cached.gpu_modified, cached.buffer_modified, + cached.info.IsCpuDirty(), + m_memory_tracker.IsRegionGpuModified(cached.info.address, cached.info.size), + cached.ctx == ctx, guest_source_current)) { + overlap = DepthOverlap::RetireSampled; + } + if (overlap == DepthOverlap::RetireSampled && !info.depth_load_clear && + native_overlap == DepthOverlap::RetireSampled && + sampled_depth_source == nullptr) { + sampled_depth_source = entry; + } + } break; case CachedImage::Kind::DepthTarget: overlap = ClassifyDepthTargetOverlap(cached.depth, cached.gpu_modified, cached.buffer_modified, cached.ctx == ctx, info); break; case CachedImage::Kind::StorageTexture: - overlap = cached.ctx == ctx ? ClassifyStorageDepthOverlap(cached.info, info) - : DepthOverlap::Unsupported; + overlap = ClassifyStorageDepthOverlap( + cached.info, cached.gpu_modified, cached.buffer_modified, + cached.info.IsCpuDirty() || + m_memory_tracker.IsRegionCpuModified(cached.info.address, cached.info.size), + m_memory_tracker.IsRegionGpuModified(cached.info.address, cached.info.size), + cached.ctx == ctx, info); break; - case CachedImage::Kind::RenderTarget: + case CachedImage::Kind::RenderTarget: { + const bool overlaps_depth = ImageRangeOverlaps( + cached.target.address, cached.target.size, info.address, info.size); + const bool overlaps_stencil = + has_stencil && ImageRangeOverlaps(cached.target.address, cached.target.size, + info.stencil_address, info.stencil_size); + const bool guest_source_current = + (!overlaps_depth || info.depth_load_clear || + IsCoherentGuestImageSource(depth_source, info.address, info.size)) && + (!overlaps_stencil || info.stencil_load_clear || + IsCoherentGuestImageSource(stencil_source, info.stencil_address, + info.stencil_size)); + overlap = CanRecreateRenderTargetForDepth( + cached.target, cached.gpu_modified, cached.buffer_modified, + m_memory_tracker.IsRegionGpuModified(cached.target.address, + cached.target.size), + cached.ctx == ctx, guest_source_current, info) + ? DepthOverlap::RecreateTarget + : DepthOverlap::Unsupported; + } break; case CachedImage::Kind::VideoOut: break; } bool supported = false; switch (overlap) { case DepthOverlap::RetireSampled: supported = cached.kind == CachedImage::Kind::Texture; - if (supported && !info.depth_load_clear && sampled_depth_source == nullptr) { - sampled_depth_source = entry; - } break; case DepthOverlap::RetireStorage: supported = cached.kind == CachedImage::Kind::StorageTexture && - cached.gpu_modified && !cached.buffer_modified && - !cached.info.IsCpuDirty() && retired_storage_source == nullptr && - native_depth_source == nullptr && discarded_depth_source == nullptr; + retired_storage_source == nullptr && native_depth_source == nullptr && + discarded_depth_source == nullptr && recreated_target_sources.empty(); if (supported) { retired_storage_source = entry; } break; case DepthOverlap::ExpandTarget: supported = cached.kind == CachedImage::Kind::DepthTarget && - native_depth_source == nullptr && retired_storage_source == nullptr; + native_depth_source == nullptr && retired_storage_source == nullptr && + recreated_target_sources.empty(); if (supported) { native_depth_source = entry; } @@ -1827,11 +2123,21 @@ DepthStencilVulkanImage* TextureCache::FindDepthTarget(CommandBuffer* command, G case DepthOverlap::DiscardTarget: supported = cached.kind == CachedImage::Kind::DepthTarget && discarded_depth_source == nullptr && native_depth_source == nullptr; - supported = supported && retired_storage_source == nullptr; + supported = supported && retired_storage_source == nullptr && + recreated_target_sources.empty(); if (supported) { discarded_depth_source = entry; } break; + case DepthOverlap::RecreateTarget: + supported = (cached.kind == CachedImage::Kind::DepthTarget || + cached.kind == CachedImage::Kind::RenderTarget) && + native_depth_source == nullptr && discarded_depth_source == nullptr && + retired_storage_source == nullptr; + if (supported) { + recreated_target_sources.push_back(entry); + } + break; case DepthOverlap::None: case DepthOverlap::Unsupported: break; } @@ -1844,15 +2150,19 @@ DepthStencilVulkanImage* TextureCache::FindDepthTarget(CommandBuffer* command, G retire.push_back(&cached); } RequireRetirementIsolation(retire, "depth target", info.address, info.size); + MaterializeImagesToGuestLocked(ctx, recreated_target_sources); if (retired_storage_source != nullptr) { - m_memory_tracker.UnmarkRegionAsGpuModified(retired_storage_source->info.address, - retired_storage_source->info.size); - retired_storage_source->gpu_modified = false; + if (retired_storage_source->gpu_modified) { + m_memory_tracker.UnmarkRegionAsGpuModified(retired_storage_source->info.address, + retired_storage_source->info.size); + retired_storage_source->gpu_modified = false; + } } const auto* transition_source = native_depth_source != nullptr ? native_depth_source.get() : (discarded_depth_source != nullptr ? discarded_depth_source.get() : nullptr); + RetireDepthMetadataLocked(retire, info.htile_address); RetireImages(retire, transition_source); const bool coherent_guest_stencil = has_stencil && @@ -1866,12 +2176,36 @@ DepthStencilVulkanImage* TextureCache::FindDepthTarget(CommandBuffer* command, G "addr=0x%016" PRIx64 " size=0x%016" PRIx64 "\n", info.stencil_address, info.stencil_size); } + if (info.samples > 1 && (native_depth_source != nullptr || sampled_depth_source != nullptr || + retired_storage_source != nullptr)) { + EXIT("TextureCache: multisampled depth native transition is unsupported, " + "addr=0x%016" PRIx64 " size=0x%016" PRIx64 " samples=%u\n", + info.address, info.size, info.samples); + } + const bool initial_depth_clear = info.samples > 1 && !info.depth_load_clear; + const bool initial_stencil_clear = info.samples > 1 && has_stencil && !info.stencil_load_clear; + if (initial_depth_clear && + (!IsCoherentGuestImageSource(depth_source, info.address, info.size) || + !GuestRangeIsZero(info.address, info.size))) { + EXIT("TextureCache: nonzero multisampled depth upload is unsupported, " + "addr=0x%016" PRIx64 " size=0x%016" PRIx64 " samples=%u\n", + info.address, info.size, info.samples); + } + if (initial_stencil_clear && + (!coherent_guest_stencil || !GuestRangeIsZero(info.stencil_address, info.stencil_size))) { + EXIT("TextureCache: nonzero multisampled stencil upload is unsupported, " + "addr=0x%016" PRIx64 " size=0x%016" PRIx64 " samples=%u\n", + info.stencil_address, info.stencil_size, info.samples); + } auto cached = std::make_shared(); cached->kind = CachedImage::Kind::DepthTarget; cached->depth = info; cached->ctx = ctx; - cached->stencil_initialized = !has_stencil; + cached->stencil_initialized = !has_stencil || info.stencil_load_clear || initial_stencil_clear; cached->image = ImageOps::CreateDepthTarget(ctx, info); + auto* depth_image = static_cast(cached->image); + depth_image->initial_depth_clear_pending = initial_depth_clear; + depth_image->initial_stencil_clear_pending = initial_stencil_clear; if (discarded_depth_source != nullptr) { command->RetainResourceUntilFence(discarded_depth_source); } @@ -1929,6 +2263,19 @@ DepthStencilVulkanImage* TextureCache::FindDepthTarget(CommandBuffer* command, G vk::ImageLayout::eDepthStencilAttachmentOptimal); cached->gpu_modified = true; } else { + if (info.samples > 1) { + // Multisample guest planes cannot be copied through Vulkan buffer-image commands. + // Their verified-zero initial contents are materialized by the first render pass. + m_memory_tracker.ForEachUploadRange( + info.address, info.size, false, [](uint64_t, uint64_t) noexcept {}, + []() noexcept {}); + if (has_stencil) { + m_memory_tracker.ForEachUploadRange( + info.stencil_address, info.stencil_size, false, + [](uint64_t, uint64_t) noexcept {}, []() noexcept {}); + } + return static_cast(PublishImage(command, std::move(cached))); + } if (sampled_depth_source != nullptr && (sampled_depth_source->image->type != VulkanImageType::Texture || sampled_depth_source->image->format != VulkanFormat(info.guest_format) || @@ -2004,7 +2351,7 @@ TextureCache::RegisterVideoOutSurfaces(GraphicContext* ctx, } FaultSafeTextureLock lock(this, m_lock); for (const auto& info: infos) { - RequireNoMetaOverlapLocked(info.address, info.size); + ResolveImageMetadataOverlapsLocked(ctx, info.address, info.size); for (const auto& cached: m_images) { if (cached->OverlapsRange(info.address, info.size, true)) { EXIT( @@ -2794,7 +3141,8 @@ TextureCache::RegionInfo TextureCache::QueryRegion(uint64_t vaddr, uint64_t size return result; } -void TextureCache::RegisterMeta(uint64_t vaddr, uint64_t size, uint32_t layers) { +void TextureCache::RegisterMeta(GraphicContext* ctx, uint64_t vaddr, uint64_t size, + uint32_t layers) { if (vaddr == 0 || size == 0 || vaddr >= TRACKER_ADDRESS_SIZE || size > TRACKER_ADDRESS_SIZE - vaddr || (vaddr & 0x7fffu) != 0 || layers == 0 || layers > 32 || size % layers != 0) { @@ -2845,18 +3193,27 @@ void TextureCache::RegisterMeta(uint64_t vaddr, uint64_t size, uint32_t layers) if (!cached->OverlapsRange(range_vaddr, range_size, true)) { continue; } + const bool sampled = cached->kind == CachedImage::Kind::Texture; + const bool writable_image = cached->kind == CachedImage::Kind::StorageTexture || + cached->kind == CachedImage::Kind::RenderTarget || + cached->kind == CachedImage::Kind::DepthTarget; + const bool gpu_modified = cached->gpu_modified || m_memory_tracker.IsRegionGpuModified( + cached->Address(), cached->Size()); + const bool cpu_dirty = + cached->kind == CachedImage::Kind::StorageTexture && + (cached->info.IsCpuDirty() || + m_memory_tracker.IsRegionCpuModified(cached->info.address, cached->info.size)); const auto overlap = - ClassifyMetaImageOverlap(cached->kind == CachedImage::Kind::Texture, - cached->kind == CachedImage::Kind::RenderTarget, - cached->gpu_modified, cached->buffer_modified); + ClassifyMetaImageOverlap(sampled, writable_image, gpu_modified, cached->buffer_modified, + cpu_dirty, cached->ctx == ctx); if (overlap == MetaImageOverlap::Unsupported) { EXIT("TextureCache: metadata aliases image pages, addr=0x%016" PRIx64 " size=0x%016" PRIx64 " image_kind=%u image=0x%016" PRIx64 "+0x%016" PRIx64 - " gpu_modified=%d buffer_modified=%d\n", + " gpu_modified=%d buffer_modified=%d same_context=%d\n", range_vaddr, range_size, static_cast(cached->kind), cached->Address(), - cached->Size(), cached->gpu_modified, cached->buffer_modified); + cached->Size(), cached->gpu_modified, cached->buffer_modified, cached->ctx == ctx); } - if (overlap == MetaImageOverlap::RetireTarget) { + if (overlap == MetaImageOverlap::RetireImage) { retire.push_back(cached.get()); continue; } @@ -2867,11 +3224,13 @@ void TextureCache::RegisterMeta(uint64_t vaddr, uint64_t size, uint32_t layers) if (!retire.empty()) { RequireRetirementIsolation(retire, "metadata", range_vaddr, range_size); for (const auto* cached: retire) { - LOGF("TextureCache: retiring a CPU-current render target for metadata reuse, " - "metadata=0x%016" PRIx64 "+0x%016" PRIx64 " target=0x%016" PRIx64 "+0x%016" PRIx64 - "\n", - range_vaddr, range_size, cached->Address(), cached->Size()); + LOGF("TextureCache: retiring a guest-current image for metadata reuse, " + "metadata=0x%016" PRIx64 "+0x%016" PRIx64 " image=0x%016" PRIx64 "+0x%016" PRIx64 + " kind=%u\n", + range_vaddr, range_size, cached->Address(), cached->Size(), + static_cast(cached->kind)); } + RetireDepthMetadataLocked(retire, vaddr); RetireImages(retire); } if (existing == m_surface_metas.end()) { @@ -2907,13 +3266,77 @@ bool TextureCache::HasMetaOverlapLocked(uint64_t vaddr, uint64_t size) const { } void TextureCache::RequireNoMetaOverlapLocked(uint64_t vaddr, uint64_t size) const { - if (HasMetaOverlapLocked(vaddr, size)) { - EXIT("TextureCache: image range overlaps virtual metadata, addr=0x%016" PRIx64 - " size=0x%016" PRIx64 "\n", - vaddr, size); + for (const auto& [address, meta]: m_surface_metas) { + if (!ImagePageRangesOverlap(vaddr, size, address, meta.size)) { + continue; + } + const auto owner = std::find_if(m_images.begin(), m_images.end(), [&](const auto& cached) { + return cached->kind == CachedImage::Kind::DepthTarget && + cached->depth.htile_address == address && cached->depth.htile_size == meta.size; + }); + EXIT("TextureCache: image range overlaps virtual metadata, image=0x%016" PRIx64 + "+0x%016" PRIx64 " metadata=0x%016" PRIx64 "+0x%016" PRIx64 + " layers=%u gpu=%d clear=0x%08x owner=%p owner_depth=0x%016" PRIx64 "+0x%016" PRIx64 + " owner_gpu=%d owner_buffer=%d\n", + vaddr, size, address, meta.size, meta.layers, meta.gpu_modified, meta.clear_mask, + owner != m_images.end() ? static_cast(owner->get()) : nullptr, + owner != m_images.end() ? (*owner)->depth.address : 0, + owner != m_images.end() ? (*owner)->depth.size : 0, + owner != m_images.end() && (*owner)->gpu_modified, + owner != m_images.end() && (*owner)->buffer_modified); } } +void TextureCache::ResolveImageMetadataOverlapsLocked(GraphicContext* ctx, uint64_t vaddr, + uint64_t size) { + std::vector retire; + for (const auto& [address, meta]: m_surface_metas) { + if (!ImagePageRangesOverlap(vaddr, size, address, meta.size)) { + continue; + } + bool found_owner = false; + for (const auto& cached: m_images) { + if (cached->kind != CachedImage::Kind::DepthTarget || + cached->depth.htile_address != address) { + continue; + } + found_owner = true; + bool tracker_gpu_modified = false; + for (uint32_t range = 0; range < cached->RangeCount(); range++) { + tracker_gpu_modified |= m_memory_tracker.IsRegionGpuModified(cached->Address(range), + cached->Size(range)); + } + const bool metadata_tracker_gpu_modified = + m_metadata_tracker.IsRegionGpuModified(address, meta.size); + if (cached->depth.htile_size != meta.size || + !CanRetireGuestCurrentDepthForMetadataReuse( + cached->gpu_modified, cached->buffer_modified, tracker_gpu_modified, + meta.gpu_modified, metadata_tracker_gpu_modified, meta.clear_mask, + cached->ctx == ctx)) { + EXIT("TextureCache: image range overlaps owned metadata, image=0x%016" PRIx64 + "+0x%016" PRIx64 " metadata=0x%016" PRIx64 "+0x%016" PRIx64 + " layers=%u clear=0x%08x same_context=%d gpu=%d/%d/%d buffer=%d\n", + vaddr, size, address, meta.size, meta.layers, meta.clear_mask, + cached->ctx == ctx, cached->gpu_modified, tracker_gpu_modified, + metadata_tracker_gpu_modified, cached->buffer_modified); + } + if (std::find(retire.begin(), retire.end(), cached.get()) == retire.end()) { + retire.push_back(cached.get()); + } + } + if (!found_owner) { + EXIT("TextureCache: image range overlaps unowned metadata, image=0x%016" PRIx64 + "+0x%016" PRIx64 " metadata=0x%016" PRIx64 "+0x%016" PRIx64 + " layers=%u gpu=%d clear=0x%08x\n", + vaddr, size, address, meta.size, meta.layers, meta.gpu_modified, meta.clear_mask); + } + } + RequireRetirementIsolation(retire, "image metadata", vaddr, size); + RetireDepthMetadataLocked(retire); + RetireImages(retire); + RequireNoMetaOverlapLocked(vaddr, size); +} + bool TextureCache::IsMetaCleared(uint64_t vaddr, uint32_t slice) { FaultSafeTextureLock lock(this, m_lock); const auto it = m_surface_metas.find(vaddr); diff --git a/src/graphics/host_gpu/renderer/textureCache.h b/src/graphics/host_gpu/renderer/textureCache.h index fc9e33c11..d0e8f5338 100644 --- a/src/graphics/host_gpu/renderer/textureCache.h +++ b/src/graphics/host_gpu/renderer/textureCache.h @@ -101,15 +101,15 @@ public: FindDepthTargetByRange(CommandBuffer* command, uint64_t vaddr, uint64_t size, bool allow_containing_sampled = false); [[nodiscard]] RegionInfo QueryRegion(uint64_t vaddr, uint64_t size); - void RegisterMeta(uint64_t vaddr, uint64_t size, uint32_t layers = 1); - [[nodiscard]] bool IsMeta(uint64_t vaddr); - [[nodiscard]] bool IsMetaRange(uint64_t vaddr, uint64_t size); - [[nodiscard]] bool IsMetaCleared(uint64_t vaddr, uint32_t slice); - [[nodiscard]] bool ClearMeta(uint64_t vaddr); - [[nodiscard]] bool TouchMeta(uint64_t vaddr, uint32_t slice, bool is_clear); - [[nodiscard]] bool InvalidateMemory(PageFaultAccess access, uint64_t vaddr, uint64_t size, - PageFaultPhase phase) noexcept; - void UnmapMemory(uint64_t vaddr, uint64_t size); + void RegisterMeta(GraphicContext* ctx, uint64_t vaddr, uint64_t size, uint32_t layers = 1); + [[nodiscard]] bool IsMeta(uint64_t vaddr); + [[nodiscard]] bool IsMetaRange(uint64_t vaddr, uint64_t size); + [[nodiscard]] bool IsMetaCleared(uint64_t vaddr, uint32_t slice); + [[nodiscard]] bool ClearMeta(uint64_t vaddr); + [[nodiscard]] bool TouchMeta(uint64_t vaddr, uint32_t slice, bool is_clear); + [[nodiscard]] bool InvalidateMemory(PageFaultAccess access, uint64_t vaddr, uint64_t size, + PageFaultPhase phase) noexcept; + void UnmapMemory(uint64_t vaddr, uint64_t size); VulkanImage* GetDummySampledTexture(bool uint_format, bool image_3d); VulkanImage* GetDummyStorageTexture(bool uint_format, bool image_3d); @@ -129,7 +129,8 @@ private: [[nodiscard]] bool HasMetaOverlapLocked(uint64_t vaddr, uint64_t size) const; [[nodiscard]] CachedImage* FindGpuReadbackPageCandidateLocked(uint64_t vaddr, uint64_t size); void RequireNoMetaOverlapLocked(uint64_t vaddr, uint64_t size) const; - void MarkSampledAliasesCpuDirtyLocked(uint64_t vaddr, uint64_t size); + void ResolveImageMetadataOverlapsLocked(GraphicContext* ctx, uint64_t vaddr, uint64_t size); + void MarkSampledAliasesCpuDirtyLocked(uint64_t vaddr, uint64_t size); void RetireSampledTargetAliases(GraphicContext* ctx, const ImageInfo& requested); void ResolveStorageImageOverlaps(GraphicContext* ctx, const ImageInfo& requested); void RetireStorageDepthAliasLocked(GraphicContext* ctx, const ImageInfo& requested); @@ -143,6 +144,10 @@ private: uint64_t address, uint64_t size) const; void RetireImages(const std::vector& retire, const CachedImage* native_image_source = nullptr); + void RetireDepthMetadataLocked(const std::vector& retire, + uint64_t preserve_address = 0); + void MaterializeImagesToGuestLocked(GraphicContext* ctx, + const std::vector>& images); void SynchronizeColorImageToBufferLocked(CachedImage& cached, uint64_t write_address, uint64_t write_size); void SynchronizeDepthImageToBufferLocked(CachedImage& cached, uint64_t write_address, diff --git a/src/graphics/host_gpu/renderer/tiler.cpp b/src/graphics/host_gpu/renderer/tiler.cpp index 6cc23ef1e..a00048f98 100644 --- a/src/graphics/host_gpu/renderer/tiler.cpp +++ b/src/graphics/host_gpu/renderer/tiler.cpp @@ -77,6 +77,10 @@ void Tiler::DetileImage(GraphicContext* ctx, GpuTextureVulkanImage* image, const void Tiler::DetileImage(GraphicContext* ctx, DepthStencilVulkanImage* image, const DepthTargetInfo& info, const BufferImageCopySource& source, bool refresh, uint32_t base_layer) const { + if (info.samples != 1 || image == nullptr || image->samples != 1) { + EXIT("Tiler: multisampled depth upload is unsupported, samples=%u/%u\n", info.samples, + image != nullptr ? image->samples : 0); + } if (refresh) { Transfer::WaitForGraphicsIdle(ctx); } @@ -118,6 +122,10 @@ void Tiler::DetileImage(GraphicContext* ctx, DepthStencilVulkanImage* image, void Tiler::DetileStencil(GraphicContext* ctx, DepthStencilVulkanImage* image, const DepthTargetInfo& info, const BufferImageCopySource& source, bool refresh, uint32_t base_layer) const { + if (info.samples != 1 || image == nullptr || image->samples != 1) { + EXIT("Tiler: multisampled stencil upload is unsupported, samples=%u/%u\n", info.samples, + image != nullptr ? image->samples : 0); + } const auto stencil_format = Prospero::GpuEnumValue(Prospero::BufferFormat::k8UInt); const auto stencil_pitch = TileGetTexturePitch( stencil_format, info.width, 1, Prospero::GpuEnumValue(Prospero::TileMode::kDepth)); @@ -150,7 +158,7 @@ void Tiler::TileImage(void* dst, const void* src, const RenderTargetInfo& info) const bool standard64 = IsSupportedStandard64RenderTarget(info); if ((info.tile_mode != Prospero::GpuEnumValue(Prospero::TileMode::kRenderTarget) && !standard64) || - info.levels != 1) { + info.levels != 1 || info.samples != 1) { EXIT("Tiler: unsupported render-target tile, dst=%p src=%p " "addr=0x%016" PRIx64 "+0x%016" PRIx64 " extent=%ux%u pitch=%u levels=%u tile=%u bpe=%u\n", @@ -203,9 +211,8 @@ void Tiler::TileImage(void* dst, const void* src, const ImageInfo& info) const { } void Tiler::TileImage(void* dst, const void* src, const DepthTargetInfo& info) const { - if (info.stencil_address != 0 || info.stencil_size != 0 || - info.tile_mode != Prospero::GpuEnumValue(Prospero::TileMode::kDepth) || - !IsSupportedDepthTargetFormat(info)) { + if (info.samples != 1 || info.tile_mode != Prospero::GpuEnumValue(Prospero::TileMode::kDepth) || + !IsSupportedDepthReadbackFormat(info)) { EXIT("Tiler: unsupported depth-target tile, dst=%p src=%p " "depth=0x%016" PRIx64 "+0x%016" PRIx64 " stencil=0x%016" PRIx64 "+0x%016" PRIx64 " extent=%ux%u pitch=%u tile=%u format=%d guest_format=%u bpe=%u\n", @@ -214,6 +221,7 @@ void Tiler::TileImage(void* dst, const void* src, const DepthTargetInfo& info) c info.guest_format, info.bytes_per_element); } const auto slice_size = info.size / info.layers; + std::memset(dst, 0, info.size); for (uint32_t layer = 0; layer < info.layers; layer++) { auto* guest_slice = static_cast(dst) + slice_size * layer; auto* linear_slice = static_cast(src) + slice_size * layer; @@ -222,4 +230,27 @@ void Tiler::TileImage(void* dst, const void* src, const DepthTargetInfo& info) c } } +void Tiler::TileStencil(void* dst, const void* src, const DepthTargetInfo& info) const { + const auto format = Prospero::GpuEnumValue(Prospero::BufferFormat::k8UInt); + const auto pitch = TileGetTexturePitch(format, info.width, 1, + Prospero::GpuEnumValue(Prospero::TileMode::kDepth)); + if (info.samples != 1 || info.stencil_address == 0 || info.stencil_size == 0 || + info.layers == 0 || info.stencil_size % info.layers != 0 || + !IsSupportedDepthReadbackFormat(info)) { + EXIT("Tiler: unsupported stencil-target tile, dst=%p src=%p " + "stencil=0x%016" PRIx64 "+0x%016" PRIx64 + " extent=%ux%u pitch=%u layers=%u format=%d compressed=%d\n", + dst, src, info.stencil_address, info.stencil_size, info.width, info.height, pitch, + info.layers, static_cast(info.format), info.stencil_htile_compressed); + } + const auto slice_size = info.stencil_size / info.layers; + std::memset(dst, 0, info.stencil_size); + for (uint32_t layer = 0; layer < info.layers; layer++) { + auto* guest_slice = static_cast(dst) + slice_size * layer; + auto* linear_slice = static_cast(src) + slice_size * layer; + TileConvertLinearToTiledDepth(guest_slice, linear_slice, format, info.width, info.height, + pitch, slice_size); + } +} + } // namespace Libs::Graphics diff --git a/src/graphics/host_gpu/renderer/tiler.h b/src/graphics/host_gpu/renderer/tiler.h index cede1a43f..29bfde95b 100644 --- a/src/graphics/host_gpu/renderer/tiler.h +++ b/src/graphics/host_gpu/renderer/tiler.h @@ -27,6 +27,7 @@ public: void TileImage(void* dst, const void* src, const RenderTargetInfo& info) const; void TileImage(void* dst, const void* src, const ImageInfo& info) const; void TileImage(void* dst, const void* src, const DepthTargetInfo& info) const; + void TileStencil(void* dst, const void* src, const DepthTargetInfo& info) const; }; } // namespace Libs::Graphics diff --git a/src/graphics/host_gpu/vulkanInstance.h b/src/graphics/host_gpu/vulkanInstance.h index beb115d6e..0bc8b5f5f 100644 --- a/src/graphics/host_gpu/vulkanInstance.h +++ b/src/graphics/host_gpu/vulkanInstance.h @@ -43,6 +43,7 @@ struct VulkanInstance { bool memory_budget_ext_enabled = false; bool rt_extensions_enabled = false; bool subgroup_size_control_enabled = false; + bool sample_rate_shading_enabled = false; uint32_t subgroup_size = 0; uint32_t min_subgroup_size = 0; uint32_t max_subgroup_size = 0; diff --git a/src/graphics/presentation/window/vulkanWindow.cpp b/src/graphics/presentation/window/vulkanWindow.cpp index 090793380..a10124ae7 100644 --- a/src/graphics/presentation/window/vulkanWindow.cpp +++ b/src/graphics/presentation/window/vulkanWindow.cpp @@ -556,7 +556,7 @@ static void VulkanInitSubgroupSizeControl(vk::PhysicalDevice physical_device, Gr static vk::Device VulkanCreateDevice(vk::PhysicalDevice physical_device, vk::SurfaceKHR surface, const VulkanExtensions* r, const VulkanQueues& queues, const std::vector& device_extensions, - const GraphicContext* ctx) { + GraphicContext* ctx) { EXIT_IF(physical_device == nullptr); EXIT_IF(r == nullptr); EXIT_IF(surface == nullptr); @@ -633,6 +633,8 @@ static vk::Device VulkanCreateDevice(vk::PhysicalDevice physical_device, vk::Sur device_features.shaderImageGatherExtended = VK_TRUE; device_features.independentBlend = VK_TRUE; device_features.tessellationShader = VK_TRUE; + device_features.sampleRateShading = supported_features2.features.sampleRateShading; + ctx->sample_rate_shading_enabled = device_features.sampleRateShading == VK_TRUE; device_features.vertexPipelineStoresAndAtomics = supported_features2.features.vertexPipelineStoresAndAtomics; diff --git a/src/graphics/shader/shader.cpp b/src/graphics/shader/shader.cpp index 7ce1a142f..9dec733a2 100644 --- a/src/graphics/shader/shader.cpp +++ b/src/graphics/shader/shader.cpp @@ -819,6 +819,7 @@ static void ShaderGetStaticInputInfoPS( ps_info->ps_pos_z = (active_inputs & 0x00000400u) != 0; ps_info->ps_pos_w = (active_inputs & 0x00000800u) != 0; ps_info->ps_front_face = (active_inputs & 0x00001000u) != 0; + ps_info->ps_sample_shading = (active_inputs & 0x00000011u) != 0; ps_info->ps_no_perspective = (sh->ps_input_ena & sh->ps_input_addr & 0x00000020u) != 0; ps_info->ps_pixel_kill_enable = sh->db_shader_control.shader_kill_enable; ps_info->ps_depth_export_enable = sh->db_shader_control.shader_z_export_enable; @@ -1260,6 +1261,7 @@ void ShaderDbgDumpInputInfo(const ShaderPixelInputInfo* info) { "\t ps_pos_z = %s\n" "\t ps_pos_w = %s\n" "\t ps_front_face = %s\n" + "\t ps_sample_shading = %s\n" "\t ps_no_perspective = %s\n" "\t ps_pixel_kill_enable = %s\n" "\t ps_early_z = %s\n" @@ -1267,8 +1269,9 @@ void ShaderDbgDumpInputInfo(const ShaderPixelInputInfo* info) { info->input_num, info->ps_system_input_base, info->ps_pos_x ? "true" : "false", info->ps_pos_y ? "true" : "false", info->ps_pos_z ? "true" : "false", info->ps_pos_w ? "true" : "false", info->ps_front_face ? "true" : "false", - info->ps_no_perspective ? "true" : "false", info->ps_pixel_kill_enable ? "true" : "false", - info->ps_early_z ? "true" : "false", info->ps_execute_on_noop ? "true" : "false"); + info->ps_sample_shading ? "true" : "false", info->ps_no_perspective ? "true" : "false", + info->ps_pixel_kill_enable ? "true" : "false", info->ps_early_z ? "true" : "false", + info->ps_execute_on_noop ? "true" : "false"); for (uint32_t i = 0; i < info->input_num; i++) { LOGF("\t interpolator_settings[%u] = %u\n", i, info->interpolator_settings[i]); diff --git a/src/graphics/shader/shader.h b/src/graphics/shader/shader.h index 7d182437d..0f8327f10 100644 --- a/src/graphics/shader/shader.h +++ b/src/graphics/shader/shader.h @@ -113,6 +113,7 @@ struct ShaderPixelInputInfo { bool ps_pixel_kill_enable = false; bool ps_depth_export_enable = false; bool ps_sample_mask_export_enable = false; + bool ps_sample_shading = false; bool ps_early_z = false; bool ps_execute_on_noop = false; ShaderStageRuntime stage; diff --git a/tests/ShaderRecompilerComputeTests.cpp b/tests/ShaderRecompilerComputeTests.cpp index 1ad0d96d2..9b4f43148 100644 --- a/tests/ShaderRecompilerComputeTests.cpp +++ b/tests/ShaderRecompilerComputeTests.cpp @@ -9244,6 +9244,45 @@ void CheckSampledColorViews() { vk::Format::eR8G8B8A8Unorm, DstSel(4, 5, 6, 7)) == DstSel(4, 5, 6, 7), "RGBA did not select the identity view"); + ShaderRecompiler::IR::ImageResource cube_resource{}; + cube_resource.kind = ShaderRecompiler::IR::ResourceKind::Image; + cube_resource.dimension = + ShaderRecompiler::Decoder::ImageDimension::Dim2DArray; + cube_resource.read = true; + const auto cube_view = + ResolveTargetTextureView(cube_resource, Prospero::ImageType::kCube, 0, 6); + Require("SampledColorViews", "PPSA17337 cubemap render target", + cube_view.type == vk::ImageViewType::e2DArray && + cube_view.base_layer == 0 && cube_view.layer_count == 6, + "captured six-face cubemap did not resolve to a 2D-array view"); + const auto cube_array_view = ResolveTargetTextureView( + cube_resource, Prospero::ImageType::kCube, 6, 18); + Require("SampledColorViews", "cubemap array subview", + cube_array_view.type == vk::ImageViewType::e2DArray && + cube_array_view.base_layer == 6 && + cube_array_view.layer_count == 12, + "nonzero-base multi-cube view did not preserve whole face groups"); + auto non_array_cube_resource = cube_resource; + non_array_cube_resource.dimension = + ShaderRecompiler::Decoder::ImageDimension::Dim2D; + Require("SampledColorViews", "cubemap hard guards", + ResolveTargetTextureView(non_array_cube_resource, + Prospero::ImageType::kCube, 0, 6) + .type == + static_cast(VK_IMAGE_VIEW_TYPE_MAX_ENUM) && + ResolveTargetTextureView(cube_resource, + Prospero::ImageType::kCube, 0, 7) + .type == + static_cast(VK_IMAGE_VIEW_TYPE_MAX_ENUM) && + ResolveTargetTextureView(cube_resource, + Prospero::ImageType::kCube, 1, 6) + .type == + static_cast(VK_IMAGE_VIEW_TYPE_MAX_ENUM) && + ResolveTargetTextureView(cube_resource, + Prospero::ImageType::kCube, 6, 6) + .type == + static_cast(VK_IMAGE_VIEW_TYPE_MAX_ENUM), + "non-array or partial cubemap views were accepted"); uint32_t valid_swizzles = 0; for (uint32_t swizzle = 0; swizzle <= 0xfffu; swizzle++) { bool expected = true; @@ -11133,23 +11172,41 @@ void CheckStorageTextureDepthAlias() { depth.stencil_address = storage.address; depth.stencil_size = storage.size; Require("StorageTextureDepthAlias", "inactive stencil", - ClassifyStorageDepthOverlap(storage, depth) == - DepthOverlap::RetireStorage, + ClassifyStorageDepthOverlap(storage, true, false, false, true, true, + depth) == DepthOverlap::RetireStorage, "inactive stencil storage ownership was not retired"); depth.stencil_access = true; Require("StorageTextureDepthAlias", "active stencil", - ClassifyStorageDepthOverlap(storage, depth) == - DepthOverlap::Unsupported, + ClassifyStorageDepthOverlap(storage, true, false, false, true, true, + depth) == DepthOverlap::Unsupported, "active stencil storage contents were discarded"); depth.stencil_access = false; storage.address = depth.address; + depth.depth_access = true; Require("StorageTextureDepthAlias", "depth aspect", - ClassifyStorageDepthOverlap(storage, depth) == - DepthOverlap::Unsupported, + ClassifyStorageDepthOverlap(storage, true, false, false, true, true, + depth) == DepthOverlap::Unsupported, "depth-aspect storage contents were discarded"); + Require("StorageTextureDepthAlias", "clean depth transition", + ClassifyStorageDepthOverlap(storage, false, false, false, false, true, + depth) == DepthOverlap::RetireStorage, + "guest-current storage image was not retired for a depth target"); + Require( + "StorageTextureDepthAlias", "depth ownership guards", + ClassifyStorageDepthOverlap(storage, false, true, false, false, true, + depth) == DepthOverlap::Unsupported && + ClassifyStorageDepthOverlap(storage, false, false, true, false, true, + depth) == DepthOverlap::Unsupported && + ClassifyStorageDepthOverlap(storage, false, false, false, true, true, + depth) == DepthOverlap::Unsupported && + ClassifyStorageDepthOverlap(storage, false, false, false, false, + false, + depth) == DepthOverlap::Unsupported, + "incoherent storage-to-depth transition was admitted"); storage.address = 0x2000000000ull; Require("StorageTextureDepthAlias", "disjoint", - ClassifyStorageDepthOverlap(storage, depth) == DepthOverlap::None, + ClassifyStorageDepthOverlap(storage, true, true, true, true, false, + depth) == DepthOverlap::None, "disjoint storage image was classified as an alias"); std::printf("[host] %-32s ok\n", "StorageTextureDepthAlias"); } @@ -11238,11 +11295,11 @@ void CheckStorageTextureAccessPermissions() { TextureCache texture_cache(page_manager, buffer_cache, resource_mutex); buffer_cache.SetTextureCache(texture_cache); page_manager.OnGpuMap(address, allocation_size); - texture_cache.RegisterMeta(address, 0x8000); auto *ctx = reinterpret_cast(1); + texture_cache.RegisterMeta(ctx, address, 0x8000); if (std::strcmp(kind, "metadata-size") == 0) { - texture_cache.RegisterMeta(address, 0x10000); + texture_cache.RegisterMeta(ctx, address, 0x10000); } else if (std::strcmp(kind, "texture") == 0) { ImageInfo info{}; info.address = address; @@ -11377,8 +11434,8 @@ void CheckOverlappingMetadataViews() { buffer_cache.SetTextureCache(texture_cache); page_manager.OnGpuMap(base, allocation_size); - texture_cache.RegisterMeta(base, metadata_size); - texture_cache.RegisterMeta(second, metadata_size); + texture_cache.RegisterMeta(nullptr, base, metadata_size); + texture_cache.RegisterMeta(nullptr, second, metadata_size); Require("OverlappingMetadataViews", "registration", texture_cache.IsMeta(base) && texture_cache.IsMeta(second) && texture_cache.IsMetaRange(base, metadata_size) && @@ -11432,7 +11489,7 @@ void CheckQueryRegionAggregation() { !empty.metadata_bytes && !empty.gpu_metadata_bytes, "empty region reported cached ownership"); - texture_cache.RegisterMeta(base, metadata_size); + texture_cache.RegisterMeta(nullptr, base, metadata_size); const auto bytes = texture_cache.QueryRegion(base + 0x20, 0x40); const auto page_only = texture_cache.QueryRegion(base + 0x400, 0x40); const auto disjoint = texture_cache.QueryRegion(base + 0x1000, 0x40); @@ -11485,7 +11542,7 @@ void CheckGpuMetadataReuse() { buffer_cache.SetTextureCache(texture_cache); page_manager.OnGpuMap(base, allocation_size); - texture_cache.RegisterMeta(base, metadata_size, layers); + texture_cache.RegisterMeta(nullptr, base, metadata_size, layers); Require("GpuMetadataReuse", "clear", texture_cache.ClearMeta(base), "metadata clear setup failed"); const bool full_image_transition = @@ -11495,7 +11552,7 @@ void CheckGpuMetadataReuse() { !texture_cache.QueryRegion(base, allocation_size).metadata_bytes, "metadata-only overwrite retained identity or claimed an image " "transition"); - texture_cache.RegisterMeta(base, metadata_size, layers); + texture_cache.RegisterMeta(nullptr, base, metadata_size, layers); Require("GpuMetadataReuse", "re-register", texture_cache.IsMetaRange(base, metadata_size) && texture_cache.ClearMeta(base), @@ -11730,6 +11787,8 @@ void CheckBufferImageWrites() { old_depth.address = 0x4a5c90000ull; old_depth.size = 0x10000; old_depth.width = old_depth.height = 32; + old_depth.pitch = 32; + old_depth.bytes_per_element = 4; old_depth.layers = 1; DepthTargetInfo cleared_depth = old_depth; cleared_depth.width = cleared_depth.height = 64; @@ -11745,21 +11804,137 @@ void CheckBufferImageWrites() { ClassifyDepthTargetOverlap(old_depth, true, false, true, cleared_depth) == DepthOverlap::Unsupported, "loaded depth reinterpretation discarded live native contents"); + RenderTargetInfo old_color{}; + old_color.address = 0x4a5cb0000ull; + old_color.size = 0x10000; + old_color.width = old_color.height = old_color.pitch = 128; + old_color.bytes_per_element = 4; + old_color.format = vk::Format::eR8G8B8A8Unorm; + auto reinterpreted_color = old_color; + reinterpreted_color.format = vk::Format::eR32Uint; + Require("BufferImageWrite", "clean color format recreation", + ClassifyRenderTargetOverlap(old_color, false, false, false, true, + true, reinterpreted_color) == + RenderTargetOverlap::RetireTarget && + ClassifyRenderTargetOverlap(old_color, true, false, true, true, + true, reinterpreted_color) == + RenderTargetOverlap::Unsupported, + "clean equal-base color allocation did not preserve GPU-owned format " + "failures"); + auto pool_depth = old_depth; + pool_depth.stencil_address = old_depth.address + 0x20000; + pool_depth.stencil_size = old_depth.size; + auto shifted_depth = pool_depth; + shifted_depth.address = old_depth.address - 0x20000; + shifted_depth.stencil_address = old_depth.address; + Require("BufferImageWrite", "shifted depth-plane recreation", + ClassifyDepthTargetOverlap(pool_depth, true, false, true, + shifted_depth) == + DepthOverlap::RecreateTarget && + ClassifyDepthTargetOverlap(pool_depth, true, true, true, + shifted_depth) == + DepthOverlap::Unsupported, + "exact stencil/depth pool reuse did not require clean single-sample " + "recreation"); + shifted_depth.samples = 8; + Require( + "BufferImageWrite", "multisampled depth-plane recreation", + ClassifyDepthTargetOverlap(pool_depth, true, false, true, + shifted_depth) == DepthOverlap::Unsupported, + "multisampled depth-plane reuse bypassed the unsupported readback guard"); + RenderTargetInfo pool_color{}; + pool_color.address = shifted_depth.stencil_address; + pool_color.size = shifted_depth.stencil_size; + shifted_depth.samples = 1; + Require( + "BufferImageWrite", "color/depth pool recreation", + CanRecreateRenderTargetForDepth(pool_color, false, false, false, true, + true, shifted_depth) && + !CanRecreateRenderTargetForDepth(pool_color, false, true, false, true, + true, shifted_depth), + "exact color-plane reuse did not require clean single-sample recreation"); + pool_color.size *= 2; + Require( + "BufferImageWrite", "larger color/depth pool recreation", + CanRecreateRenderTargetForDepth(pool_color, false, false, false, true, + true, shifted_depth), + "equal-base page-isolated color allocation was not recreated for depth"); + auto contained_depth = shifted_depth; + contained_depth.address = pool_color.address + TRACKER_PAGE_SIZE; + contained_depth.size = TRACKER_PAGE_SIZE; + contained_depth.stencil_address = 0; + contained_depth.stencil_size = 0; + Require( + "BufferImageWrite", "contained color/depth pool recreation", + CanRecreateRenderTargetForDepth(pool_color, false, false, false, true, + true, contained_depth) && + !CanRecreateRenderTargetForDepth(pool_color, false, false, false, + true, false, contained_depth) && + !CanRecreateRenderTargetForDepth(pool_color, true, false, false, true, + true, contained_depth), + "contained depth allocation bypassed source or tracker ownership guards"); + pool_color.samples = 8; + Require( + "BufferImageWrite", "multisampled color/depth recreation", + !CanRecreateRenderTargetForDepth(pool_color, false, false, false, true, + true, shifted_depth), + "multisampled color-plane reuse bypassed the unsupported readback guard"); + auto multisample_depth = old_depth; + multisample_depth.samples = 8; + multisample_depth.depth_access = true; + multisample_depth.stencil_access = true; + Require("BufferImageWrite", "multisampled depth refresh state", + !RequiresMultisampleDepthRefresh(multisample_depth, false, false, + false) && + RequiresMultisampleDepthRefresh(multisample_depth, false, true, + false) && + RequiresMultisampleDepthRefresh(multisample_depth, false, false, + true) && + RequiresMultisampleDepthRefresh(multisample_depth, true, false, + false), + "multisampled refresh used consumed source history instead of " + "current plane ownership"); RenderTargetInfo color_alias{}; color_alias.address = old_depth.address; color_alias.size = old_depth.size; + color_alias.width = color_alias.height = 32; + color_alias.pitch = 32; + color_alias.bytes_per_element = 4; + Require("BufferImageWrite", "guest-backed depth to color", + CanRecreateDepthForRenderTarget(old_depth, false, false, false, true, + true, color_alias) && + CanRecreateDepthForRenderTarget(old_depth, true, false, true, + true, false, color_alias) && + !CanRecreateDepthForRenderTarget(old_depth, true, true, true, + true, true, color_alias), + "depth-to-color recreation did not require coherent guest ownership"); + Require("BufferImageWrite", "guest-current depth to color", + !CanRecreateDepthForRenderTarget(old_depth, false, false, false, true, + false, color_alias), + "stale guest backing authorized depth-to-color recreation"); + auto mismatched_color = color_alias; + mismatched_color.address += 0x1000; + Require("BufferImageWrite", "mismatched depth to color", + !CanRecreateDepthForRenderTarget(old_depth, false, false, false, true, + true, mismatched_color), + "mismatched depth shape was discarded as a color allocation"); + color_alias.address = pool_depth.stencil_address; Require( - "BufferImageWrite", "buffer-owned depth to color", - CanRetireBufferOwnedDepthForRenderTarget(old_depth, false, true, true, - true, color_alias) && - !CanRetireBufferOwnedDepthForRenderTarget(old_depth, true, false, - true, true, color_alias), - "depth-to-color recreation did not require coherent buffer ownership"); - Require( - "BufferImageWrite", "page-neighbor depth to color", - !CanRetireBufferOwnedDepthForRenderTarget(old_depth, false, true, true, - false, color_alias), - "same-page byte-disjoint buffer authorized depth-to-color recreation"); + "BufferImageWrite", "stencil-plane depth to color", + CanRecreateDepthForRenderTarget(pool_depth, false, false, false, true, + true, color_alias), + "exact stencil-plane reuse was not treated as whole-image recreation"); + auto containing_color = color_alias; + containing_color.address = pool_depth.stencil_address - 0x40000; + containing_color.size = 0x80000; + Require("BufferImageWrite", "contained depth allocation to color", + CanRecreateDepthForRenderTarget(pool_depth, false, false, false, true, + true, containing_color) && + !CanRecreateDepthForRenderTarget(pool_depth, false, false, false, + true, false, containing_color) && + !CanRecreateDepthForRenderTarget(pool_depth, false, false, true, + true, true, containing_color), + "contained guest-current depth allocation bypassed ownership guards"); TileSizeAlign storage{}; Require("BufferImageWrite", "target layout", @@ -12269,33 +12444,52 @@ void CheckImageOverlapResolution() { storage_target.tile_mode = storage.tile; Require("ImageOverlapResolution", "storage render-target native transition", ClassifyStorageRenderTargetOverlap( - storage, storage_target.format, true, false, false, true, + storage, storage_target.format, true, false, false, true, true, storage_target) == RenderTargetOverlap::PreserveStorage, "exact GPU-owned RGBA16F storage image was not preserved for a " "render target"); + auto partial_storage_target = storage_target; + partial_storage_target.address += 0x10000; + partial_storage_target.size = 0x10000; + Require("ImageOverlapResolution", "clean storage render-target transition", + ClassifyStorageRenderTargetOverlap( + storage, storage_target.format, false, false, false, false, true, + partial_storage_target) == RenderTargetOverlap::RetireStorage, + "clean storage allocation was not retired for an overlapping render " + "target"); auto mismatched_storage_target = storage_target; mismatched_storage_target.width--; - Require("ImageOverlapResolution", "storage render-target guards", + Require( + "ImageOverlapResolution", "storage render-target guards", + ClassifyStorageRenderTargetOverlap( + storage, storage_target.format, true, true, false, true, true, + storage_target) == RenderTargetOverlap::Unsupported && ClassifyStorageRenderTargetOverlap( - storage, storage_target.format, false, false, false, true, + storage, storage_target.format, true, false, true, true, true, storage_target) == RenderTargetOverlap::Unsupported && - ClassifyStorageRenderTargetOverlap( - storage, storage_target.format, true, true, false, true, - storage_target) == RenderTargetOverlap::Unsupported && - ClassifyStorageRenderTargetOverlap( - storage, storage_target.format, true, false, true, true, - storage_target) == RenderTargetOverlap::Unsupported && - ClassifyStorageRenderTargetOverlap( - storage, storage_target.format, true, false, false, false, - storage_target) == RenderTargetOverlap::Unsupported && - ClassifyStorageRenderTargetOverlap( - storage, vk::Format::eR32G32B32A32Sfloat, true, false, false, - true, storage_target) == RenderTargetOverlap::Unsupported && - ClassifyStorageRenderTargetOverlap(storage, storage_target.format, - true, false, false, true, - mismatched_storage_target) == - RenderTargetOverlap::Unsupported, - "unsupported storage-to-render-target transition was admitted"); + ClassifyStorageRenderTargetOverlap( + storage, storage_target.format, true, false, false, true, false, + storage_target) == RenderTargetOverlap::Unsupported && + ClassifyStorageRenderTargetOverlap( + storage, vk::Format::eR32G32B32A32Sfloat, true, false, false, + true, true, storage_target) == RenderTargetOverlap::Unsupported && + ClassifyStorageRenderTargetOverlap( + storage, storage_target.format, true, false, false, true, true, + mismatched_storage_target) == RenderTargetOverlap::Unsupported && + ClassifyStorageRenderTargetOverlap( + storage, storage_target.format, false, false, false, true, true, + storage_target) == RenderTargetOverlap::Unsupported && + ClassifyStorageRenderTargetOverlap( + storage, storage_target.format, true, false, false, false, true, + storage_target) == RenderTargetOverlap::Unsupported, + "unsupported storage-to-render-target transition was admitted"); + partial_storage_target.address = storage.address + storage.size; + Require("ImageOverlapResolution", "storage render-target adjacency", + ClassifyStorageRenderTargetOverlap( + storage, storage_target.format, false, false, false, false, true, + partial_storage_target) == RenderTargetOverlap::None, + "byte-disjoint storage and render-target allocations were treated as " + "aliases"); Require("ImageOverlapResolution", "render target context", ClassifyRenderTargetOverlap(sampled, false, false, target) == RenderTargetOverlap::Unsupported, @@ -12362,40 +12556,51 @@ void CheckImageOverlapResolution() { replacement_target.height = 128; replacement_target.bytes_per_element = 8; Require("ImageOverlapResolution", "render target allocation-pool replacement", - ClassifyRenderTargetOverlap(old_target, false, false, true, - replacement_target) == + ClassifyRenderTargetOverlap(old_target, false, false, false, true, + true, replacement_target) == RenderTargetOverlap::RetireTarget, "clean same-base render-target pool allocation was not retired"); Require("ImageOverlapResolution", "render target allocation-pool GPU owner", - ClassifyRenderTargetOverlap(old_target, true, false, true, + ClassifyRenderTargetOverlap(old_target, true, false, true, true, true, replacement_target) == RenderTargetOverlap::Unsupported, "GPU-owned render-target pool allocation was retired"); Require("ImageOverlapResolution", "render target allocation-pool buffer owner", - ClassifyRenderTargetOverlap(old_target, false, true, true, - replacement_target) == + ClassifyRenderTargetOverlap(old_target, false, true, false, true, + true, replacement_target) == RenderTargetOverlap::Unsupported, "buffer-dirty render-target pool allocation was retired"); Require("ImageOverlapResolution", "render target allocation-pool context", - ClassifyRenderTargetOverlap(old_target, false, false, false, - replacement_target) == + ClassifyRenderTargetOverlap(old_target, false, false, false, false, + true, replacement_target) == RenderTargetOverlap::Unsupported, "cross-context render-target pool allocation was retired"); auto offset_target = replacement_target; offset_target.address += TRACKER_PAGE_SIZE; Require("ImageOverlapResolution", "render target allocation-pool offset", - ClassifyRenderTargetOverlap(old_target, false, false, true, - offset_target) == - RenderTargetOverlap::Unsupported, - "offset render-target overlap was treated as allocation-pool " - "replacement"); + ClassifyRenderTargetOverlap(old_target, false, false, false, true, + true, offset_target) == + RenderTargetOverlap::RetireTarget, + "clean offset render-target allocation was not retired"); + auto contained_target = old_target; + contained_target.address += TRACKER_PAGE_SIZE; + contained_target.size = TRACKER_PAGE_SIZE; + Require("ImageOverlapResolution", + "render target contained allocation-pool reuse", + ClassifyRenderTargetOverlap(old_target, false, false, false, true, + true, contained_target) == + RenderTargetOverlap::RetireTarget && + ClassifyRenderTargetOverlap(old_target, false, false, false, true, + false, contained_target) == + RenderTargetOverlap::Unsupported, + "contained render-target allocation bypassed guest ownership guards"); auto partial_page_target = replacement_target; partial_page_target.address++; Require("ImageOverlapResolution", "render target allocation-pool partial page", - ClassifyRenderTargetOverlap(old_target, false, false, true, - partial_page_target) == + ClassifyRenderTargetOverlap(old_target, false, false, false, true, + true, partial_page_target) == RenderTargetOverlap::Unsupported, "partially shared tracker page was treated as allocation-pool " "replacement"); @@ -12406,8 +12611,8 @@ void CheckImageOverlapResolution() { "exact RGBA8 UNORM-to-sRGB target was not recognized as a compatible " "view"); Require("ImageOverlapResolution", "render target sRGB no retirement", - ClassifyRenderTargetOverlap(old_target, false, false, true, - same_shape_target) == + ClassifyRenderTargetOverlap(old_target, false, false, false, true, + true, same_shape_target) == RenderTargetOverlap::Unsupported, "compatible render-target view fell through to target retirement"); auto incompatible_same_shape_target = same_shape_target; @@ -12415,10 +12620,11 @@ void CheckImageOverlapResolution() { Require("ImageOverlapResolution", "render target incompatible same shape", !IsCompatibleRenderTargetView(old_target, incompatible_same_shape_target) && - ClassifyRenderTargetOverlap(old_target, false, false, true, + ClassifyRenderTargetOverlap(old_target, false, false, false, true, + true, incompatible_same_shape_target) == - RenderTargetOverlap::Unsupported, - "incompatible same-storage render-target format was admitted"); + RenderTargetOverlap::RetireTarget, + "clean incompatible render-target format was not recreated"); RenderTargetInfo ppsa02604_unorm_target{}; ppsa02604_unorm_target.address = 0x79c50000ull; @@ -12441,29 +12647,107 @@ void CheckImageOverlapResolution() { adjacent_target.address = old_target.address + old_target.size; Require( "ImageOverlapResolution", "render target allocation-pool adjacent", - ClassifyRenderTargetOverlap(old_target, false, false, true, + ClassifyRenderTargetOverlap(old_target, false, false, false, true, true, adjacent_target) == RenderTargetOverlap::None, "adjacent render targets were treated as allocation-pool replacement"); Require("ImageOverlapResolution", "metadata retains sampled image", - ClassifyMetaImageOverlap(true, false, false, false) == + ClassifyMetaImageOverlap(true, false, false, false, false, true) == MetaImageOverlap::RetainSampled, "CPU-current sampled metadata alias was rejected"); - Require("ImageOverlapResolution", "metadata retires render target", - ClassifyMetaImageOverlap(false, true, false, false) == - MetaImageOverlap::RetireTarget, - "CPU-current render target was not retired for metadata reuse"); + Require("ImageOverlapResolution", "metadata retires writable image", + ClassifyMetaImageOverlap(false, true, false, false, false, true) == + MetaImageOverlap::RetireImage, + "guest-current writable image was not retired for metadata reuse"); Require("ImageOverlapResolution", "metadata rejects GPU target", - ClassifyMetaImageOverlap(false, true, true, false) == + ClassifyMetaImageOverlap(false, true, true, false, false, true) == MetaImageOverlap::Unsupported && - ClassifyMetaImageOverlap(false, true, false, true) == + ClassifyMetaImageOverlap(false, true, false, true, false, true) == + MetaImageOverlap::Unsupported && + ClassifyMetaImageOverlap(false, true, false, false, true, true) == MetaImageOverlap::Unsupported, - "unordered or buffer-dirty render target was admitted for metadata " + "unordered or dirty writable image was admitted for metadata " "reuse"); Require("ImageOverlapResolution", "metadata rejects unsupported image", - ClassifyMetaImageOverlap(false, false, false, false) == + ClassifyMetaImageOverlap(false, false, false, false, false, true) == MetaImageOverlap::Unsupported, "unsupported cached image kind was admitted for metadata reuse"); + Require("ImageOverlapResolution", "metadata rejects cross-context target", + ClassifyMetaImageOverlap(false, true, false, false, false, false) == + MetaImageOverlap::Unsupported, + "cross-context writable image was retired for metadata reuse"); + Require( + "ImageOverlapResolution", "guest-current depth metadata pool reuse", + CanRetireGuestCurrentDepthForMetadataReuse(false, false, false, false, + false, 0, true), + "guest-current depth owner was not retired when its HTile allocation was " + "reused"); + Require( + "ImageOverlapResolution", "depth metadata pool ownership guards", + !CanRetireGuestCurrentDepthForMetadataReuse(true, false, true, false, + false, 0, true) && + !CanRetireGuestCurrentDepthForMetadataReuse(false, true, false, false, + false, 0, true) && + !CanRetireGuestCurrentDepthForMetadataReuse(false, false, false, true, + true, 1, true) && + !CanRetireGuestCurrentDepthForMetadataReuse(false, false, false, + false, false, 0, false), + "GPU-owned, buffer-owned, cleared, or cross-context HTile metadata was " + "admitted for pool reuse"); + + ImageInfo pooled_sampled{}; + pooled_sampled.address = 0x71141000; + pooled_sampled.size = 0x40000; + DepthTargetInfo pooled_depth{}; + pooled_depth.address = 0x71150000; + pooled_depth.size = 0x10000; + pooled_depth.stencil_address = 0x71170000; + pooled_depth.stencil_size = 0x10000; + Require("ImageOverlapResolution", "guest-current sampled/depth pool alias", + CanRetireGuestCurrentDepthForSampled( + pooled_sampled, pooled_depth, false, false, false, false, true), + "guest-current contained depth target was not retired for a sampled " + "allocation"); + Require( + "ImageOverlapResolution", "sampled/depth pool ownership guards", + !CanRetireGuestCurrentDepthForSampled(pooled_sampled, pooled_depth, true, + false, true, true, true) && + !CanRetireGuestCurrentDepthForSampled( + pooled_sampled, pooled_depth, false, true, false, false, true) && + !CanRetireGuestCurrentDepthForSampled( + pooled_sampled, pooled_depth, false, false, false, false, false), + "GPU-owned, buffer-owned, or cross-context sampled/depth alias was " + "admitted"); + ImageInfo offset_sampled{}; + offset_sampled.address = 0x6c6cd000; + offset_sampled.size = 0x10000; + DepthTargetInfo offset_depth{}; + offset_depth.address = 0x6c6d0000; + offset_depth.size = 0x10000; + Require("ImageOverlapResolution", "guest-current sampled to depth pool alias", + CanRetireGuestCurrentSampledForDepth(offset_sampled, offset_depth, + false, false, false, false, true, + true), + "offset guest-current sampled allocation was not retired for depth " + "reuse"); + Require( + "ImageOverlapResolution", "sampled to depth pool ownership guards", + !CanRetireGuestCurrentSampledForDepth(offset_sampled, offset_depth, true, + false, false, true, true, true) && + !CanRetireGuestCurrentSampledForDepth(offset_sampled, offset_depth, + false, true, false, false, true, + true) && + !CanRetireGuestCurrentSampledForDepth(offset_sampled, offset_depth, + false, false, true, false, true, + true) && + !CanRetireGuestCurrentSampledForDepth(offset_sampled, offset_depth, + false, false, false, false, + false, true) && + !CanRetireGuestCurrentSampledForDepth(offset_sampled, offset_depth, + false, false, false, false, + true, false), + "dirty, cross-context, or stale sampled allocation was admitted for " + "depth reuse"); const std::array isolated_retirement{ ImageRetirementRange{sampled.address, TRACKER_PAGE_SIZE, true}, @@ -12717,32 +13001,44 @@ void CheckImageOverlapResolution() { std::printf("[host] %-32s ok\n", "ImageOverlapResolution"); } -void CheckMsaaCompatibility() { - Require("MsaaCompatibility", "single sample", - !depth_msaa_single_sample_compatible(0), - "native single-sample state was classified as compatibility"); - Require("MsaaCompatibility", "two fragments", - depth_msaa_single_sample_compatible(1), - "two-fragment compatibility state was rejected"); - Require("MsaaCompatibility", "four fragments", - depth_msaa_single_sample_compatible(2), - "existing four-fragment compatibility state regressed"); - Require("MsaaCompatibility", "eight fragments", - !depth_msaa_single_sample_compatible(3), - "unsupported eight-fragment state was silently admitted"); - Require("MsaaCompatibility", "color two fragments", - color_msaa_single_sample_compatible(1, 1), - "matching two-sample color compatibility state was rejected"); - Require("MsaaCompatibility", "color four fragments", - color_msaa_single_sample_compatible(2, 2), - "existing matching four-sample color compatibility state regressed"); - Require("MsaaCompatibility", "color mismatch", - !color_msaa_single_sample_compatible(2, 1), - "mismatched color sample/fragment state was silently admitted"); - Require("MsaaCompatibility", "color eight fragments", - !color_msaa_single_sample_compatible(3, 3), - "unsupported eight-sample color state was silently admitted"); - std::printf("[host] %-32s ok\n", "MsaaCompatibility"); +void CheckNativeMsaaState() { + Require("NativeMsaaState", "sample encoding", + render_sample_count(0) == 1 && render_sample_count(1) == 2 && + render_sample_count(2) == 4 && render_sample_count(3) == 8 && + render_sample_count(4) == 0, + "PS5 sample encodings were not mapped exactly"); + Require("NativeMsaaState", "Vulkan sample mapping", + vulkan_sample_count(1) == vk::SampleCountFlagBits::e1 && + vulkan_sample_count(2) == vk::SampleCountFlagBits::e2 && + vulkan_sample_count(4) == vk::SampleCountFlagBits::e4 && + vulkan_sample_count(8) == vk::SampleCountFlagBits::e8 && + vulkan_sample_count(3) == vk::SampleCountFlagBits{}, + "native sample counts were not mapped exactly to Vulkan"); + + TileSizeAlign color{}; + const auto color_pitch = TileGetRenderTargetPitch(1920, 8, 3); + Require("NativeMsaaState", "8x color footprint", + color_pitch == 1920 && + TileGetRenderTargetSize(1920, 1080, color_pitch, 8, &color, 3) && + color.align == 0x10000 && color.size == 0x07f80000, + "AGC 8x R16G16B16A16 color footprint was not preserved"); + + TileSizeAlign depth{}; + TileSizeAlign stencil{}; + TileSizeAlign htile{}; + Require( + "NativeMsaaState", "8x depth/stencil footprint", + TileGetDepthPitch(1920, 4, 3) == 1920 && + TileGetDepthSize( + 1920, 1080, 0, + Prospero::GpuEnumValue(Prospero::DepthFormat::kZ32F), + Prospero::GpuEnumValue(Prospero::StencilFormat::k8UInt), true, + &stencil, &htile, &depth, 3) && + depth.align == 0x10000 && depth.size == 0x03fc0000 && + stencil.align == 0x10000 && stencil.size == 0x010e0000 && + htile.align == 0x8000 && htile.size == 0x00030000, + "AGC 8x depth/stencil or fragment-independent HTile footprint regressed"); + std::printf("[host] %-32s ok\n", "NativeMsaaState"); } void CheckDepthHtileStencilCompatibility() { @@ -12837,34 +13133,39 @@ void CheckDepthTargetFootprints() { uint32_t bytes_per_element; bool stencil; bool supported; + bool readback; }; constexpr TargetFormatCase target_cases[] = { {"D16", vk::Format::eD16Unorm, - Prospero::GpuEnumValue(Prospero::BufferFormat::k16UNorm), 2, false, + Prospero::GpuEnumValue(Prospero::BufferFormat::k16UNorm), 2, false, true, true}, {"D16S8", vk::Format::eD16UnormS8Uint, - Prospero::GpuEnumValue(Prospero::BufferFormat::k16UNorm), 2, true, true}, + Prospero::GpuEnumValue(Prospero::BufferFormat::k16UNorm), 2, true, true, + true}, {"D16 via D24S8", vk::Format::eD24UnormS8Uint, - Prospero::GpuEnumValue(Prospero::BufferFormat::k16UNorm), 2, true, true}, + Prospero::GpuEnumValue(Prospero::BufferFormat::k16UNorm), 2, true, true, + false}, {"D16 via D32S8", vk::Format::eD32SfloatS8Uint, - Prospero::GpuEnumValue(Prospero::BufferFormat::k16UNorm), 2, true, true}, + Prospero::GpuEnumValue(Prospero::BufferFormat::k16UNorm), 2, true, true, + false}, {"D32", vk::Format::eD32Sfloat, - Prospero::GpuEnumValue(Prospero::BufferFormat::k32Float), 4, false, + Prospero::GpuEnumValue(Prospero::BufferFormat::k32Float), 4, false, true, true}, {"D32S8", vk::Format::eD32SfloatS8Uint, - Prospero::GpuEnumValue(Prospero::BufferFormat::k32Float), 4, true, true}, + Prospero::GpuEnumValue(Prospero::BufferFormat::k32Float), 4, true, true, + true}, {"D16 plus stencil mismatch", vk::Format::eD16Unorm, - Prospero::GpuEnumValue(Prospero::BufferFormat::k16UNorm), 2, true, + Prospero::GpuEnumValue(Prospero::BufferFormat::k16UNorm), 2, true, false, false}, {"fallback without stencil", vk::Format::eD24UnormS8Uint, Prospero::GpuEnumValue(Prospero::BufferFormat::k16UNorm), 2, false, - false}, + false, false}, {"D32 guest via D24", vk::Format::eD24UnormS8Uint, - Prospero::GpuEnumValue(Prospero::BufferFormat::k32Float), 4, true, + Prospero::GpuEnumValue(Prospero::BufferFormat::k32Float), 4, true, false, false}, {"D16 byte mismatch", vk::Format::eD16Unorm, Prospero::GpuEnumValue(Prospero::BufferFormat::k16UNorm), 4, false, - false}, + false, false}, }; for (const auto &test : target_cases) { DepthTargetInfo target{}; @@ -12874,9 +13175,23 @@ void CheckDepthTargetFootprints() { target.stencil_address = test.stencil ? 0x10000 : 0; target.stencil_size = test.stencil ? 0x10000 : 0; Require("DepthTargetFootprints", test.name, - IsSupportedDepthTargetFormat(target) == test.supported, - "host/guest depth format policy changed"); + IsSupportedDepthTargetFormat(target) == test.supported && + IsSupportedDepthReadbackFormat(target) == test.readback, + "host/guest depth format or exact readback policy changed"); } + DepthTargetInfo compressed_stencil{}; + compressed_stencil.format = vk::Format::eD32SfloatS8Uint; + compressed_stencil.guest_format = + Prospero::GpuEnumValue(Prospero::BufferFormat::k32Float); + compressed_stencil.bytes_per_element = 4; + compressed_stencil.stencil_address = 0x10000; + compressed_stencil.stencil_size = 0x10000; + compressed_stencil.stencil_htile_compressed = true; + Require( + "DepthTargetFootprints", "compressed stencil readback hard guard", + IsSupportedDepthTargetFormat(compressed_stencil) && + !IsSupportedDepthReadbackFormat(compressed_stencil), + "compressed stencil unexpectedly entered the raw-plane readback path"); struct TransferPlaneCase { const char *name; vk::Format attachment; @@ -13098,6 +13413,20 @@ void CheckHtileClearTargetResolution() { resolved.address == derived_z16s8.htile_data_base_addr && resolved.size == htile_size.size, "extent-derived Z16S8 HTile target was rejected"); + + auto derived_8x = derived; + derived_8x.z_info.num_samples = 3; + auto malformed_fragments = derived; + malformed_fragments.z_info.num_samples = 4; + Require("HtileClearTargetResolution", "fragment-independent metadata plane", + ResolveHtileClearTarget(derived_8x, htile_size.size, &resolved) && + resolved.address == derived_8x.htile_data_base_addr && + resolved.size == htile_size.size && + render_sample_count(derived_8x.z_info.num_samples) == 8 && + !ResolveHtileClearTarget(malformed_fragments, htile_size.size, + &resolved), + "HTile clear was coupled to attachment MSAA support or admitted an " + "invalid fragment field"); std::printf("[host] %-32s ok\n", "HtileClearTargetResolution"); } @@ -13154,8 +13483,8 @@ void CheckHostDmaMetadataReuse() { fault_context.texture = &texture_cache; buffer_cache.SetTextureCache(texture_cache); page_manager.OnGpuMap(base, allocation_size); - texture_cache.RegisterMeta(base, metadata_size); auto *ctx = reinterpret_cast(1); + texture_cache.RegisterMeta(ctx, base, metadata_size); Require("HostDmaMetadataReuse", "clear", texture_cache.ClearMeta(base), "metadata clear setup failed"); @@ -13400,7 +13729,7 @@ int main(int argc, char **argv) { CheckMetadataReuseDescriptors(); CheckImageOverlapResolution(); CheckQueryRegionAggregation(); - CheckMsaaCompatibility(); + CheckNativeMsaaState(); CheckDepthHtileStencilCompatibility(); CheckStencilAttachmentAccess(); CheckDepthTargetFootprints();