render: clean up texture transfers and compute fills

src/graphics/host_gpu/renderer/cache/textureCache.cpp

@@ -130,10 +130,11 @@ bool TextureCache::SameBacking(const ImageInfo& cached, const ImageInfo& request
 
 TextureCache::BindingType TextureCache::UploadBinding(const Image& image) {
 	if (image.info.IsDepth()) {
-		return image.info.tile_mode == Prospero::TileMode::kDepth ||
-		               image.info.tile_mode == Prospero::TileMode::kLinear
-		           ? BindingType::DepthTarget
-		           : BindingType::Texture;
+		if (image.info.tile_mode == Prospero::TileMode::kDepth ||
+		    image.info.tile_mode == Prospero::TileMode::kLinear) {
+			return BindingType::DepthTarget;
+		}
+		return BindingType::Texture;
 	}
 	if (image.usage.render_target) {
 		return BindingType::RenderTarget;
@@ -823,7 +824,6 @@ struct TextureCache::TextureTransferPlan {
 	std::vector<vk::BufferImageCopy> regions;
 	std::vector<GpuTileInfo>         tiles;
 	uint64_t                         linear_size = 0;
-	bool                             tiled       = false;
 	bool                             swap_bgra16 = false;
 	bool                             valid       = false;
 };
@@ -846,52 +846,44 @@ TextureCache::TextureTransferPlan
 TextureCache::BuildTextureTransfer(const Image& image, BindingType binding,
                                     TransferDirection direction) const {
 	const auto& info             = image.info;
+	const bool  upload           = direction == TransferDirection::Upload;
+	const bool  render_target    = binding == BindingType::RenderTarget;
+	const bool  video_out        = binding == BindingType::VideoOut;
 	auto        format           = info.guest_format;
 	uint32_t    layers           = info.TransferLayers();
 	bool        volume           = info.IsVolume();
-	bool        allow_depth_tile = direction == TransferDirection::Upload;
-	const char* owner =
-	    direction == TransferDirection::Upload ? "TextureCache" : "TextureCache readback";
+	bool        allow_depth_tile = upload;
+	const char* owner            = "TextureCache readback";
 
 	TextureTransferPlan plan;
-	if (direction == TransferDirection::Upload) {
-		switch (binding) {
-			case BindingType::Texture: break;
-			case BindingType::Storage: owner = "StorageTextureCache"; break;
-			case BindingType::RenderTarget:
-				if (info.resources.layers == 0 || info.data.size % info.resources.layers != 0 ||
-				    info.samples != 1 || image.backing.samples != 1) {
-					EXIT("TextureCache: invalid color-attachment upload\n");
-				}
-				format           = ImageOps::RenderTargetTransferFormat(info.bytes_per_block);
-				allow_depth_tile = true;
-				plan.swap_bgra16 = info.bgra16;
-				owner            = "RenderTarget";
-				break;
-			case BindingType::VideoOut:
-				if (info.resources.layers == 0 || info.data.size % info.resources.layers != 0 ||
-				    info.samples != 1 || image.backing.samples != 1 ||
-				    info.metadata.compression != VideoOutCompression::Uncompressed) {
-					EXIT("TextureCache: invalid color-attachment upload\n");
-				}
-				format           = info.guest_format;
-				layers           = info.resources.layers;
-				volume           = false;
-				allow_depth_tile = false;
-				plan.swap_bgra16 = info.bgra16;
-				owner            = "VideoOut";
-				break;
-			case BindingType::DepthTarget: return plan;
+	plan.swap_bgra16 = info.bgra16 && (!upload || render_target || video_out);
+	if (render_target) {
+		format = ImageOps::RenderTargetTransferFormat(info.bytes_per_block);
+	}
+	if (video_out) {
+		allow_depth_tile = false;
+	} else if (render_target || binding == BindingType::Storage) {
+		allow_depth_tile = true;
+	}
+	if (upload) {
+		if ((render_target || video_out) &&
+		    (info.resources.layers == 0 || info.data.size % info.resources.layers != 0 ||
+		     info.samples != 1 || image.backing.samples != 1)) {
+			EXIT("TextureCache: invalid color-attachment upload\n");
 		}
-	} else {
-		if (binding == BindingType::DepthTarget) {
-			return plan;
+		owner = "TextureCache";
+		if (render_target) {
+			owner = "RenderTarget";
+		} else if (binding == BindingType::Storage) {
+			owner = "StorageTextureCache";
+		} else if (video_out) {
+			if (info.metadata.compression != VideoOutCompression::Uncompressed) {
+				EXIT("TextureCache: invalid color-attachment upload\n");
+			}
+			layers = info.resources.layers;
+			volume = false;
+			owner  = "VideoOut";
 		}
-		format           = binding == BindingType::RenderTarget
-		                       ? ImageOps::RenderTargetTransferFormat(info.bytes_per_block)
-		                       : info.guest_format;
-		allow_depth_tile = binding == BindingType::Storage || binding == BindingType::RenderTarget;
-		plan.swap_bgra16 = info.bgra16;
 	}
 
 	plan.layout  = TextureCalcUploadLayout(format, info.extent.width, info.extent.height,
@@ -903,8 +895,7 @@ TextureCache::BuildTextureTransfer(const Image& image, BindingType binding,
 			region.imageSubresource.aspectMask = vk::ImageAspectFlagBits::eDepth;
 		}
 	}
-	plan.tiled   = plan.layout.surface.description.tile_mode != Prospero::TileMode::kLinear;
-	if (plan.tiled) {
+	if (plan.layout.surface.description.tile_mode != Prospero::TileMode::kLinear) {
 		if (!TextureBuildGpuTileInfos(info.data.size, plan.regions, plan.layout,
 		                              info.resources.levels, plan.tiles)) {
 			return plan;
@@ -959,7 +950,7 @@ void TextureCache::UploadImage(Image& image, Buffer& source, uint64_t source_off
 			     info.resources.layers, info.samples);
 		}
 		TileManager::Result linear {source.Handle(), source_offset, info.data.size};
-		if (plan.tiled) {
+		if (!plan.tiles.empty()) {
 			linear = m_tiler.Detile(source.Handle(), source_offset, info.data.size,
 			                        plan.linear_size, plan.tiles);
 		}
@@ -1568,7 +1559,7 @@ void TextureCache::DownloadImageData(Image& image, Buffer& destination, uint64_t
 	auto&      texture   = plan.texture;
 	const auto transform = texture.swap_bgra16 ? TileManager::ColorTransform::SwapBgra16
 	                                           : TileManager::ColorTransform::None;
-	if (!texture.tiled) {
+	if (texture.tiles.empty()) {
 		if (transform == TileManager::ColorTransform::SwapBgra16) {
 			auto linear = m_tiler.GetScratchBuffer(destination_size);
 			image.Download(texture.regions, linear.buffer, 0, linear.size);
@@ -1666,7 +1657,7 @@ bool BufferCache::SynchronizeBufferFromImage(Buffer& buffer, uint64_t vaddr, uin
 		if (texture.regions.empty()) {
 			return false;
 		}
-		if (texture.tiled) {
+		if (!texture.tiles.empty()) {
 			texture.tiles.clear();
 			if (!TextureBuildGpuTileInfos(copy_size, texture.regions, texture.layout, levels,
 			                              texture.tiles)) {

src/graphics/host_gpu/renderer/image/imageView.h

@@ -92,15 +92,6 @@ SelectSampledColorView(vk::Format image_format, vk::Format view_format, uint32_t
 	}
 }
 
-[[nodiscard]] inline uint32_t
-SelectSampledDepthView(vk::Format image_format, vk::Format view_format, uint32_t swizzle) noexcept {
-	if (IsSupportedSampledDepthView(image_format, view_format, swizzle)) {
-		return swizzle;
-	}
-	EXIT("unsupported sampled depth image view: image_format=%d view_format=%d swizzle=0x%03x\n",
-	     static_cast<int>(image_format), static_cast<int>(view_format), swizzle);
-}
-
 [[nodiscard]] inline bool
 IsSupportedSampledDepthResource(const ShaderRecompiler::IR::ImageResource& resource) noexcept {
 	if (resource.resource_class != ShaderRecompiler::IR::ImageResourceClass::Sampled) {

src/graphics/host_gpu/renderer/pipeline/descriptors.cpp

@@ -335,40 +335,36 @@ bool IsSupportedDepthTextureEncoding(const ShaderTextureResource& descriptor, co
 }
 
 static void ValidateSampledDepthBinding(const ShaderRecompiler::IR::ImageResource& resource,
-                                       const ShaderTextureResource& descriptor, const Image* image,
-                                       vk::Format view_format, uint64_t size) {
-	const bool resource_ok = IsSupportedSampledDepthResource(resource);
-	const bool descriptor_ok =
-	    image != nullptr && IsSupportedSampledDepthDescriptor(descriptor, *image, resource.r128);
-	const bool encoding_ok =
-	    image != nullptr && IsSupportedDepthTextureEncoding(descriptor, *image, resource.r128);
-	const bool format_ok =
-	    image != nullptr && IsSupportedSampledDepthFormat(image->info.pixel_format, view_format);
-	if (resource_ok && descriptor_ok && encoding_ok && format_ok && size != 0) {
+                                        const ShaderTextureResource& descriptor, const Image& image,
+                                        vk::Format view_format, uint64_t size) {
+	const bool resource_ok   = IsSupportedSampledDepthResource(resource);
+	const bool descriptor_ok = IsSupportedSampledDepthDescriptor(descriptor, image, resource.r128);
+	const bool encoding_ok   = IsSupportedDepthTextureEncoding(descriptor, image, resource.r128);
+	const bool view_ok =
+	    IsSupportedSampledDepthView(image.info.pixel_format, view_format, descriptor.DstSelXYZW());
+	if (resource_ok && descriptor_ok && encoding_ok && view_ok) {
 		return;
 	}
 	const auto descriptor_pitch =
 	    TileGetTexturePitch(descriptor.Format(), static_cast<uint32_t>(descriptor.Width5()) + 1u,
 	                        descriptor.TileMode());
-	EXIT("unsupported sampled depth image: resource=%d descriptor=%d encoding=%d format=%d "
+	EXIT("unsupported sampled depth image: resource=%d descriptor=%d encoding=%d view=%d "
 	     "class=%u numeric=%u dimension=%u mip_mode=%u read=%d written=%d atomic=%d compare=%d "
 	     "guest_format=%u swizzle=0x%03x image_format=%d view_format=%d image_layers=%u "
 	     "descriptor_type=%u base_array=%u depth=%u descriptor_pitch=%u target_pitch=%u "
 	     "addr=0x%016" PRIx64 " size=0x%016" PRIx64
 	     " dwords=%08x,%08x,%08x,%08x,%08x,%08x,%08x,%08x\n",
-	     resource_ok, descriptor_ok, encoding_ok, format_ok,
+	     resource_ok, descriptor_ok, encoding_ok, view_ok,
 	     static_cast<uint32_t>(resource.resource_class),
 	     static_cast<uint32_t>(resource.numeric_class), static_cast<uint32_t>(resource.dimension),
 	     static_cast<uint32_t>(resource.mip_mode), resource.read, resource.written, resource.atomic,
 	     resource.depth_compare, static_cast<uint32_t>(descriptor.Format()),
-	     descriptor.DstSelXYZW(),
-	     image == nullptr ? static_cast<int>(vk::Format::eUndefined)
-	                      : static_cast<int>(image->info.pixel_format),
-	     static_cast<int>(view_format), image == nullptr ? 0u : image->info.resources.layers,
+	     descriptor.DstSelXYZW(), static_cast<int>(image.info.pixel_format),
+	     static_cast<int>(view_format), image.info.resources.layers,
 	     static_cast<uint32_t>(descriptor.Type()), descriptor.BaseArray5(), descriptor.Depth(),
-	     descriptor_pitch, image == nullptr ? 0u : image->info.pitch, descriptor.Base40(), size,
-	     descriptor.fields[0], descriptor.fields[1], descriptor.fields[2], descriptor.fields[3],
-	     descriptor.fields[4], descriptor.fields[5], descriptor.fields[6], descriptor.fields[7]);
+	     descriptor_pitch, image.info.pitch, descriptor.Base40(), size, descriptor.fields[0],
+	     descriptor.fields[1], descriptor.fields[2], descriptor.fields[3], descriptor.fields[4],
+	     descriptor.fields[5], descriptor.fields[6], descriptor.fields[7]);
 }
 
 static bool IsSupportedStorageTextureDescriptor(const ShaderRecompiler::IR::ImageResource& resource,
@@ -747,10 +743,12 @@ TextureBinding RenderExecutor::ResolveTexture(const ShaderRecompiler::IR::ImageR
 		ValidateStorageTexture(resource, descriptor, size.size);
 	}
 
-	const auto* depth_format = resource.depth_compare ? FindGuestDepthFormatPolicy(format) : nullptr;
-	EXIT_NOT_IMPLEMENTED(resource.depth_compare && depth_format == nullptr);
-	const auto pixel_format =
-	    depth_format != nullptr ? depth_format->depth_attachment_format : surface_format.vk_format;
+	auto pixel_format = surface_format.vk_format;
+	if (resource.depth_compare) {
+		const auto* depth_format = FindGuestDepthFormatPolicy(format);
+		EXIT_NOT_IMPLEMENTED(depth_format == nullptr);
+		pixel_format = depth_format->depth_attachment_format;
+	}
 	const auto storage_view_format = storage && format == Prospero::BufferFormat::k32SInt
 	                                     ? vk::Format::eR32Uint
 	                                     : SrgbStorageViewFormat(pixel_format);
@@ -789,9 +787,7 @@ TextureBinding RenderExecutor::ResolveTexture(const ShaderRecompiler::IR::ImageR
 		if (storage) {
 			EXIT("depth target cannot be bound as a storage image\n");
 		}
-		ValidateSampledDepthBinding(resource, descriptor, image, pixel_format, size.size);
-		(void)SelectSampledDepthView(image->info.pixel_format, pixel_format,
-		                             descriptor.DstSelXYZW());
+		ValidateSampledDepthBinding(resource, descriptor, *image, pixel_format, size.size);
 	} else if (storage) {
 		ValidateStorageColorView(image->info.pixel_format, view_format, descriptor.DstSelXYZW());
 	} else {

src/graphics/host_gpu/renderer/render.h

@@ -219,7 +219,7 @@ private:
 	friend struct RenderExecutorTestAccess;
 };
 
-[[nodiscard]] bool ResolveComputeImageClear(const ShaderComputeInputInfo& input, uint32_t group_x,
+[[nodiscard]] bool ResolveComputeBufferFill(const ShaderComputeInputInfo& input, uint32_t group_x,
                                             uint32_t group_y, uint32_t group_z, uint32_t mode,
                                             ShaderBufferResource& descriptor,
                                             uint32_t& packed_clear, uint64_t& size);

src/graphics/host_gpu/renderer/renderCompute.cpp

@@ -39,9 +39,6 @@ namespace Libs::Graphics {
 static uint64_t BufferDescriptorSize(const ShaderBufferResource& descriptor) {
 	const uint64_t records = descriptor.NumRecords();
 	const uint64_t stride  = descriptor.Stride();
-	if (stride != 0 && records > UINT64_MAX / stride) {
-		EXIT("compute buffer descriptor footprint overflow\n");
-	}
 	return stride == 0 ? records : records * stride;
 }
 
@@ -51,9 +48,8 @@ static bool FillSourcesDisjoint(std::span<const ShaderRecompiler::IR::Descriptor
 		if (i == output_buffer) continue;
 		const auto source = DecodeNativeDescriptor<ShaderBufferResource>(sources[i]);
 		const auto bytes  = BufferDescriptorSize(source);
-		if (source.Base48() > UINT64_MAX - bytes ||
-		    (source.Base48() < destination.End() &&
-		     destination.address < source.Base48() + bytes)) return false;
+		if (source.Base48() < destination.End() && destination.address < source.Base48() + bytes)
+			return false;
 	}
 	return true;
 }
@@ -91,30 +87,23 @@ bool RenderExecutor::TryConsumeComputeMetaClear(const ShaderComputeInputInfo& in
 	return false;
 }
 
-bool ResolveComputeImageClear(const ShaderComputeInputInfo& input, uint32_t group_x,
+bool ResolveComputeBufferFill(const ShaderComputeInputInfo& input, uint32_t group_x,
                               uint32_t group_y, uint32_t group_z, uint32_t mode,
                               ShaderBufferResource& resolved_descriptor, uint32_t& resolved_clear,
                               uint64_t& resolved_size) {
-	const auto& program   = *input.stage.program;
 	const auto& resources = input.stage.resources;
 	const auto& fill      = resources.uniform_fill;
-	if (fill.kind != ShaderRecompiler::IR::UniformFillKind::Buffer || fill.words == 0 ||
-	    fill.words > 4 || fill.resource >= resources.buffers.size() ||
-	    resources.buffers.size() != program.info.buffers.size()) {
+	if (fill.kind != ShaderRecompiler::IR::UniformFillKind::Buffer) {
 		return false;
 	}
-	const auto           element_size = fill.words * sizeof(uint32_t);
-	const auto&          resource   = program.info.buffers[fill.resource];
-	const auto&          raw        = resources.buffers[fill.resource];
-	const auto           descriptor = DecodeNativeDescriptor<ShaderBufferResource>(raw);
+	const auto element_size = fill.words * sizeof(uint32_t);
+	const auto descriptor =
+	    DecodeNativeDescriptor<ShaderBufferResource>(resources.buffers[fill.resource]);
 	constexpr std::array formats {
 	    Prospero::BufferFormat::k32UInt, Prospero::BufferFormat::k32_32UInt,
 	    Prospero::BufferFormat::k32_32_32UInt, Prospero::BufferFormat::k32_32_32_32UInt};
-	if (!resource.formatted || !resource.written || resource.read || resource.atomic ||
-	    resource.scalar || descriptor.Stride() != element_size ||
-	    descriptor.Format() != formats[fill.words - 1] || descriptor.SwizzleEnabled() ||
-	    descriptor.IndexStride() != 0 || descriptor.AddTid() ||
-	    resource.packed_stride != descriptor.PackedStride() || raw.dword_count != 4 ||
+	if (descriptor.Stride() != element_size || descriptor.Format() != formats[fill.words - 1] ||
+	    descriptor.SwizzleEnabled() || descriptor.IndexStride() != 0 || descriptor.AddTid() ||
 	    descriptor.Base48() == 0) {
 		return false;
 	}
@@ -128,7 +117,6 @@ bool ResolveComputeImageClear(const ShaderComputeInputInfo& input, uint32_t grou
 	                                 : static_cast<uint64_t>(group_x) * input.threads_num[0];
 	const auto     size        = BufferDescriptorSize(descriptor);
 	if (invocations != descriptor.NumRecords() || size == 0 || size > UINT32_MAX ||
-	    descriptor.Base48() > UINT64_MAX - size ||
 	    (input.dispatch_thread_dimensions &&
 	     (group_x % input.threads_num[0] != 0 || input.dispatch_threads_num[0] != group_x ||
 	      input.dispatch_threads_num[1] != 1 || input.dispatch_threads_num[2] != 1))) {
@@ -150,23 +138,17 @@ bool RenderExecutor::TryConsumeComputeImageClear(const ShaderComputeInputInfo& i
 	const auto& fill      = resources.uniform_fill;
 	auto&       cache     = command.GetContext().GetTextureCache();
 	if (fill.kind == ShaderRecompiler::IR::UniformFillKind::Image) {
-		if (mode != 0x41u || input.dispatch_thread_dimensions || fill.words != 1 || fill.value > 255 ||
-		    resources.images.size() != 1 || program.info.images.size() != 1 || fill.resource != 0 ||
-		    resources.images[0].dword_count != 8 || input.threads_num[2] != 1 ||
-		    fill.group_stride[2] != 1) return false;
+		if (mode != 0x41u || input.dispatch_thread_dimensions || fill.value > 255 ||
+		    input.threads_num[2] != 1)
+			return false;
 		const auto  descriptor = DecodeNativeDescriptor<ShaderTextureResource>(resources.images[0]);
 		const auto& resource   = program.info.images[0];
 		if (descriptor.IsNull() || descriptor.Format() != Prospero::BufferFormat::k8UInt ||
-		    descriptor.Type() != Prospero::ImageType::kColor2DArray ||
-		    descriptor.MetaCompress() || descriptor.WriteCompress() ||
-		    descriptor.BaseLevel() > descriptor.LastLevel() ||
+		    descriptor.Type() != Prospero::ImageType::kColor2DArray || descriptor.MetaCompress() ||
+		    descriptor.WriteCompress() || descriptor.BaseLevel() > descriptor.LastLevel() ||
 		    descriptor.BaseLevel() > descriptor.MaxMip() ||
-		    descriptor.BaseArray5() > descriptor.Depth() ||
-		    descriptor.DstSelX() != 4 || (resource.shader_swizzle & 7u) != 4 ||
-		    resource.numeric_class != Prospero::TextureNumericClass::Uint ||
-		    resource.dimension != ShaderRecompiler::Decoder::ImageDimension::Dim2DArray ||
-		    resource.read || resource.atomic || !resource.written ||
-		    resource.mip_mode != ShaderRecompiler::IR::ImageMipMode::None) return false;
+		    descriptor.BaseArray5() > descriptor.Depth() || descriptor.DstSelX() != 4)
+			return false;
 		const std::array extents {
 		    std::max(1u, (descriptor.Width5() + 1u) >> descriptor.BaseLevel()),
 		    std::max(1u, (descriptor.Height5() + 1u) >> descriptor.BaseLevel()),
@@ -202,7 +184,7 @@ bool RenderExecutor::TryConsumeComputeImageClear(const ShaderComputeInputInfo& i
 	ShaderBufferResource descriptor;
 	uint32_t             packed_clear = 0;
 	uint64_t             size         = 0;
-	if (!ResolveComputeImageClear(input, group_x, group_y, group_z, mode, descriptor, packed_clear,
+	if (!ResolveComputeBufferFill(input, group_x, group_y, group_z, mode, descriptor, packed_clear,
 	                              size)) {
 		return false;
 	}
@@ -219,10 +201,7 @@ bool RenderExecutor::TryConsumeComputeImageClear(const ShaderComputeInputInfo& i
 			     registered_metadata ? "tracked" : "deferred", input.stage.program->shader_hash,
 			     descriptor.Base48(), size, packed_clear);
 		}
-		if (registered_metadata) {
-			return true;
-		}
-		return false;
+		return registered_metadata;
 	}
 	static std::atomic<uint32_t> logged_clears {0};
 	if (logged_clears.fetch_add(1, std::memory_order_relaxed) < 32) {