shader: fix crash from packed ancillary used as DPP old operand (#1769)

shader: emit plain EXEC selects for DPP writes that reach every active lane (Street Fighter 6)

.gitignore

@@ -6,3 +6,4 @@ _Build/vscode-clang/
 _Build/
 cmake-build-debug/
 _PipelineCache/
+CMakeUserPresets.json

CMakeLists.txt

@@ -265,6 +265,7 @@ set(gpu_tiler_shader_names
 	promote_d16
 	demote_d16
 	color_transform
+	color_clear
 )
 file(GLOB gpu_tiler_shader_includes CONFIGURE_DEPENDS "${gpu_tiler_shader_dir}/gpu_tiler_*.inc")
 foreach(shader_name IN LISTS gpu_tiler_shader_names)
@@ -657,6 +658,8 @@ if(BUILD_TESTING)
 	add_test(NAME shader_recompiler_compute COMMAND $<TARGET_FILE:shader_recompiler_compute_tests>)
 	add_test(NAME shader_recompiler_tessellation
 		COMMAND $<TARGET_FILE:shader_recompiler_compute_tests> --tessellation-only)
+	add_test(NAME shader_recompiler_float_min_max
+		COMMAND $<TARGET_FILE:shader_recompiler_compute_tests> --float-min-max-only)
 	add_test(NAME shader_recompiler_alignbyte
 		COMMAND $<TARGET_FILE:shader_recompiler_compute_tests> --alignbyte-only)
 	add_test(NAME graphics_pipeline_rasterization
@@ -673,6 +676,12 @@ if(BUILD_TESTING)
 		COMMAND $<TARGET_FILE:shader_recompiler_compute_tests> --stream-buffer-only)
 	add_test(NAME gpu_command_lane
 		COMMAND $<TARGET_FILE:shader_recompiler_compute_tests> --gpu-command-lane-only)
+	add_test(NAME wait_until_safe_for_rendering
+		COMMAND $<TARGET_FILE:shader_recompiler_compute_tests> --wait-until-safe-for-rendering-only)
+	add_test(NAME depth_bits
+		COMMAND $<TARGET_FILE:shader_recompiler_compute_tests> --depth-bits-only)
+	add_test(NAME msaa_storage
+		COMMAND $<TARGET_FILE:shader_recompiler_compute_tests> --msaa-storage-only)
 	add_test(NAME pm4_context_state
 		COMMAND $<TARGET_FILE:shader_recompiler_compute_tests> --context-state-only)
 	add_test(NAME gpu_tiler

README.md

@@ -46,6 +46,16 @@ or graphical glitches, so please include the version you tested when reporting a
 ## Screenshots
 
 <table align="center">
+  <tr>
+    <td align="center">
+      <strong>Cyberpunk 2077</strong><br>
+      <img src="docs/screenshots/ps5-07.png" width="300" alt="Cyberpunk 2077 running in KytyPS5">
+    </td>
+    <td align="center">
+      <strong>MLB The Show 23</strong><br>
+      <img src="docs/screenshots/ps5-08.png" width="300" alt="MLB The Show 23 running in KytyPS5">
+    </td>
+  </tr>
   <tr>
     <td align="center">
       <strong>Astro Bot</strong><br>

docs/logo.svg

@@ -0,0 +1,15 @@
+<svg xmlns="http://www.w3.org/2000/svg" viewBox="0 0 512 512"><defs>
+<linearGradient id="body" x1="0" y1="0" x2="0" y2="1"><stop offset="0" stop-color="#ffffff"/><stop offset="1" stop-color="#e3e9f2"/></linearGradient>
+</defs>
+<g transform="translate(256 268) scale(1.16) translate(-256 -268)" stroke-linejoin="round" stroke-linecap="round">
+<path d="M100 196 L120 104 Q124 92 134 99 L214 160 Z" fill="url(#body)"/><path d="M412 196 L392 104 Q388 92 378 99 L298 160 Z" fill="url(#body)"/>
+<path d="M128 164 L137 128 L174 156 Z" fill="#0b0d12"/><path d="M384 164 L375 128 L338 156 Z" fill="#0b0d12"/>
+<g fill="none" stroke="#0b0d12" stroke-width="16"><path d="M100 196 L120 104 Q124 92 134 99 L214 160 Z"/><path d="M412 196 L392 104 Q388 92 378 99 L298 160 Z"/></g>
+<path d="M256 154 C330 154 380 160 406 178 C432 196 447 244 459 302 C471 358 466 398 430 408 C400 416 380 398 360 368 L326 318 C316 304 306 300 290 300 L222 300 C206 300 196 304 186 318 L152 368 C132 398 112 416 82 408 C46 398 41 358 53 302 C65 244 80 196 106 178 C132 160 182 154 256 154 Z" fill="url(#body)"/>
+<path d="M256 154 C330 154 380 160 406 178 C432 196 447 244 459 302 C471 358 466 398 430 408 C400 416 380 398 360 368 L326 318 C316 304 306 300 290 300 L222 300 C206 300 196 304 186 318 L152 368 C132 398 112 416 82 408 C46 398 41 358 53 302 C65 244 80 196 106 178 C132 160 182 154 256 154 Z" fill="none" stroke="#0b0d12" stroke-width="16"/>
+<g fill="none" stroke="#0b0d12" stroke-width="9"><path d="M82 260 C100 274 118 288 128 304 C134 316 121 350 108 388"/><path d="M430 260 C412 274 394 288 384 304 C378 316 391 350 404 388"/></g>
+<path d="M64 322 L82 340 M62 340 L76 354 M448 322 L430 340 M450 340 L436 354" fill="none" stroke="#0b0d12" stroke-width="7"/>
+<path d="M168 230 Q194 202 220 230" fill="none" stroke="#0b0d12" stroke-width="15"/>
+<circle cx="322" cy="220" r="22" fill="#0b0d12"/>
+<path d="M238 252 L274 252 L256 276 Z" fill="#0b0d12" stroke="#0b0d12" stroke-width="8"/>
+</g></svg>

src/common/archive.h

@@ -5,6 +5,7 @@
 
 #include <cstdint>
 #include <filesystem>
+#include <functional>
 #include <memory>
 #include <vector>
 
@@ -45,6 +46,12 @@ private:
 // Accepts either a host archive filename or a virtual path inside the archive.
 [[nodiscard]] std::shared_ptr<ArchiveReader> OpenArchive(const std::filesystem::path& path);
 
+// Packs a host directory into a new ZArchive outside it. progress gets 0 < done <= total and
+// returns false to cancel. A failed or cancelled pack removes the partial archive.
+[[nodiscard]] bool PackArchive(const std::filesystem::path& directory,
+                               const std::filesystem::path& archive,
+                               const std::function<bool(uint64_t done, uint64_t total)>& progress);
+
 } // namespace Common
 
 #endif /* KYTY_COMMON_ARCHIVE_H_ */

src/common/archivePack.cpp

@@ -0,0 +1,84 @@
+// Kept out of archive.cpp so the statically linked emulator does not pull in the writer and the
+// zstd compressor.
+#include "common/archive.h"
+
+#include "common/stringUtils.h"
+
+#include <algorithm>
+#include <zarchive/zarchivewriter.h>
+
+namespace Common {
+
+bool PackArchive(const std::filesystem::path& directory, const std::filesystem::path& archive,
+                 const std::function<bool(uint64_t done, uint64_t total)>& progress) {
+	namespace fs = std::filesystem;
+	struct Output {
+		File file;
+		bool ok = true;
+	} output;
+	// Create the archive before walking, so a re-run started while a cancelled pack is still
+	// walking sees it exists instead of losing its archive to that pack's cleanup.
+	if (!output.file.Create(archive)) {
+		return false;
+	}
+	struct Entry {
+		fs::path path;
+		uint64_t size;
+		bool     is_file;
+	};
+	std::vector<Entry> entries;
+	uint64_t           total = 0;
+	std::error_code    error;
+	// Follow directory links, as the emulator does when it runs the folder.
+	fs::recursive_directory_iterator it(directory, fs::directory_options::follow_directory_symlink,
+	                                    error);
+	for (; !error && it != fs::end(it); it.increment(error)) {
+		const bool is_file = !it->is_directory(error);
+		const auto size    = is_file && !error ? it->file_size(error) : 0;
+		if (error) {
+			break;
+		}
+		entries.push_back({it->path(), size, is_file});
+		total += size;
+	}
+	output.ok = !error;
+	ZArchiveWriter writer(
+	    [](int32_t, void*) {},
+	    [](const void* data, size_t size, void* opaque) {
+		    auto&    out     = *static_cast<Output*>(opaque);
+		    uint32_t written = 0;
+		    out.file.Write(data, static_cast<uint32_t>(size), &written);
+		    out.ok = out.ok && written == size;
+	    },
+	    &output);
+	std::vector<uint8_t> buffer(1u << 20u);
+	uint64_t             done = 0;
+	for (size_t index = 0; output.ok && index < entries.size(); index++) {
+		const auto& entry = entries[index];
+		const auto  name  = PathToGenericString(entry.path.lexically_relative(directory));
+		File        input;
+		output.ok = entry.is_file ? writer.StartNewFile(name.c_str()) &&
+		                                input.Open(entry.path, File::Mode::Read)
+		                          : writer.MakeDir(name.c_str());
+		for (uint64_t left = entry.size; output.ok && left != 0;) {
+			uint32_t read = 0;
+			input.Read(buffer.data(), static_cast<uint32_t>(std::min<uint64_t>(left, buffer.size())),
+			           &read);
+			writer.AppendData(buffer.data(), read);
+			done += read;
+			left -= read;
+			output.ok = output.ok && read != 0 && progress(done, total);
+		}
+	}
+	if (output.ok) {
+		writer.Finalize();
+	}
+	output.ok = output.ok && output.file.Flush();
+	output.file.Close();
+	if (!output.ok) {
+		File::DeleteFile(archive);
+	}
+	return output.ok;
+}
+
+} // namespace Common

src/common/debugCounters.h

@@ -29,6 +29,9 @@ enum class Counter {
 	TextureUploadBytes,
 	TextureDownloadBytes,
 	TextureEvictions,
+	ColorMetadataReadbacks, // CPU blocked reading back GPU-written DCC/CMASK metadata.
+	ColorMetadataReadbackNs,
+	ColorMetadataChecks, // GPU clear-key checks of DCC/CMASK metadata.
 	BufferUploadBytes,
 	BufferDownloadBytes,
 	BufferEvictions,

src/common/emulatorConfig.cpp

@@ -185,6 +185,10 @@ bool ReadbackLinearImagesEnabled() {
 	return g_config->readback_linear_images;
 }
 
+bool ReadbackTiledImagesEnabled() {
+	return g_config->readback_tiled_images;
+}
+
 bool SyncRawImageBuffersEnabled() {
 	return g_config->sync_raw_image_buffers;
 }

src/common/emulatorConfig.h

@@ -85,6 +85,7 @@ struct ConfigOptions {
 	bool                   gpu_assisted_validation_enabled = false;
 	bool                   renderdoc_enabled           = false;
 	bool                   readback_linear_images      = false;
+	bool                   readback_tiled_images       = false;
 	bool                   sync_raw_image_buffers      = false;
 	bool                   tessellation_enabled        = false;
 	bool                   trophy_enabled              = true;
@@ -150,6 +151,7 @@ bool GpuAssistedValidationEnabled();
 
 bool RenderDocEnabled();
 bool ReadbackLinearImagesEnabled();
+bool ReadbackTiledImagesEnabled();
 bool SyncRawImageBuffersEnabled();
 bool TessellationEnabled();
 bool TrophyEnabled();

src/graphics/guest_gpu/command_processor/commandProcessor.h

@@ -154,6 +154,8 @@ private:
 	                      uint32_t interrupt_context_id);
 	void ProcessPm4(Pm4Execution& execution);
 	void SuspendPm4();
+	uint32_t            NumInstances();
+	const void*         IndexAddress(uint32_t first_index) const;
 	CommandScheduler&   GetScheduler() const { return m_renderer.GetCommandScheduler(); }
 	CommandBuffer&      CurrentBuffer() { return GetScheduler().Current(); }
 
@@ -169,8 +171,10 @@ private:
 	uint64_t         m_index_base_addr                  = 0;
 	uint64_t         m_draw_indirect_args_base_addr     = 0;
 	uint64_t         m_dispatch_indirect_args_base_addr = 0;
-	// Persistent draw state: indirect draws update it for subsequent draws.
-	uint32_t m_num_instances = 1;
+	// Persistent draw state. Indirect draws leave the address of their last instance count,
+	// read only when a later draw inherits it.
+	uint32_t m_num_instances      = 1;
+	uint64_t m_num_instances_addr = 0;
 
 	uint32_t m_de_count    = 0;
 	uint32_t m_ce_count    = 0;

src/graphics/guest_gpu/graphicsRun.cpp

@@ -30,6 +30,7 @@
 #include <mutex>
 #include <semaphore>
 #include <thread>
+#include <utility>
 #include <vector>
 
 namespace Libs::Graphics {
@@ -39,21 +40,6 @@ static thread_local Pm4Execution*     g_current_execution = nullptr;
 static thread_local bool              g_gpu_thread        = false;
 static thread_local GuestGpu*         g_gpu_state         = nullptr;
 
-struct DrawIndirectArgs {
-	uint32_t vertex_count_per_instance;
-	uint32_t instance_count;
-	uint32_t start_vertex_location;
-	uint32_t start_instance_location;
-};
-
-struct DrawIndexedIndirectArgs {
-	uint32_t index_count_per_instance;
-	uint32_t instance_count;
-	uint32_t start_index_location;
-	uint32_t base_vertex_location;
-	uint32_t start_instance_location;
-};
-
 static bool GraphicsRunDebugDumpEnabled() {
 	return Config::GraphicsDebugDumpEnabled() &&
 	       Config::GetPrintfDirection() != Config::LogDirection::Silent;
@@ -106,6 +92,12 @@ void GuestGpu::SendCommand(Common::UniqueFunction<void>&& command) {
 	m_work_available.Signal();
 }
 
+void GuestGpu::Wake() {
+	Common::LockGuard lock(m_queue_mutex);
+	m_wake_pending = true;
+	m_work_available.Signal();
+}
+
 void GuestGpu::ProcessCommands() {
 	EXIT_IF(!IsGpuThread());
 	while (m_pending_commands.load(std::memory_order_acquire) != 0) {
@@ -206,6 +198,7 @@ void CommandProcessor::Reset() {
 	m_index_buffer_size                = 0;
 	m_index_base_addr                  = 0;
 	m_num_instances                    = 1;
+	m_num_instances_addr               = 0;
 	m_predicate_skip                   = false;
 	m_user_data_marker                 = HW::UserSgprType::Unknown;
 	m_draw_indirect_args_base_addr     = 0;
@@ -478,7 +471,10 @@ void GuestGpu::ThreadRun(void* data) {
 				}
 				if (selected_queue < 0) {
 					gpu->m_processing = false;
-					gpu->m_work_available.WaitFor(&gpu->m_queue_mutex, 100);
+					// A wake that came before this wait must not be lost.
+					if (!std::exchange(gpu->m_wake_pending, false)) {
+						gpu->m_work_available.WaitFor(&gpu->m_queue_mutex, 100);
+					}
 					for (auto& queue: gpu->m_queues) {
 						if (!queue.empty()) {
 							queue.front().blocked = false;
@@ -784,7 +780,16 @@ void CommandProcessor::SetNumInstances(uint32_t num_instances) {
 		num_instances = 1;
 	}
 
-	m_num_instances = num_instances;
+	m_num_instances      = num_instances;
+	m_num_instances_addr = 0;
+}
+
+uint32_t CommandProcessor::NumInstances() {
+	if (m_num_instances_addr != 0) {
+		m_num_instances      = *reinterpret_cast<const uint32_t*>(m_num_instances_addr);
+		m_num_instances_addr = 0;
+	}
+	return m_num_instances;
 }
 
 void CommandProcessor::SetPredication(uint32_t condition, uint32_t op, uint32_t wait_op,
@@ -841,16 +846,12 @@ void CommandProcessor::SetPredication(uint32_t condition, uint32_t op, uint32_t
 void CommandProcessor::DrawIndex(DrawIndexArgs args) {
 	args.index_type_and_size = m_index_type_and_size;
 	if (args.instance_count == 0) {
-		args.instance_count = m_num_instances;
-	}
-	if (GraphicsRunDebugDumpEnabled() && (args.base_vertex != 0 || args.first_instance != 0)) {
-		LOGF("\t draw indexed offsets: base_vertex = %" PRId32 ", first_instance = %" PRIu32 "\n",
-		     args.base_vertex, args.first_instance);
+		args.instance_count = NumInstances();
 	}
 	m_renderer.GetRenderExecutor().DrawIndex(m_submit_id, CurrentBuffer(), args);
 }
 
-void CommandProcessor::DrawIndexOffset(uint32_t index_offset, uint32_t index_count) {
+const void* CommandProcessor::IndexAddress(uint32_t first_index) const {
 	uint64_t index_size = 0;
 	switch (m_index_type_and_size) {
 		case 0: index_size = 2; break;
@@ -858,11 +859,11 @@ void CommandProcessor::DrawIndexOffset(uint32_t index_offset, uint32_t index_cou
 		case 2: index_size = 1; break;
 		default: EXIT("unknown index_type_and_size: %u\n", m_index_type_and_size);
 	}
+	return reinterpret_cast<const void*>(m_index_base_addr + first_index * index_size);
+}
 
-	auto* index_addr = reinterpret_cast<const void*>(
-	    m_index_base_addr + static_cast<uint64_t>(index_offset) * index_size);
-
-	DrawIndex({.index_count = index_count, .index_addr = index_addr});
+void CommandProcessor::DrawIndexOffset(uint32_t index_offset, uint32_t index_count) {
+	DrawIndex({.index_count = index_count, .index_addr = IndexAddress(index_offset)});
 }
 
 static void PatchIndirectDrawOffsets(CommandProcessor& cp, IndirectDrawRegisters registers,
@@ -883,7 +884,8 @@ static void PatchIndirectDrawOffsets(CommandProcessor& cp, IndirectDrawRegisters
 
 void CommandProcessor::DrawIndirect(uint32_t data_offset, IndirectDrawRegisters registers,
                                     uint32_t draw_initiator, bool indexed) {
-	const auto args_size = indexed ? sizeof(DrawIndexedIndirectArgs) : sizeof(DrawIndirectArgs);
+	const auto args_size =
+	    indexed ? sizeof(vk::DrawIndexedIndirectCommand) : sizeof(vk::DrawIndirectCommand);
 	DrawIndirectMulti(data_offset, 1, nullptr, args_size, registers, draw_initiator, indexed);
 }
 
@@ -893,77 +895,65 @@ void CommandProcessor::DrawIndirectMulti(uint32_t data_offset, uint32_t max_coun
                                          uint32_t draw_initiator, bool indexed) {
 	EXIT_NOT_IMPLEMENTED((draw_initiator & ~0x20u) != (indexed ? 0u : 2u));
 	EXIT_NOT_IMPLEMENTED(m_draw_indirect_args_base_addr == 0);
+	// The guest records match Vulkan's indirect commands byte for byte.
+	EXIT_NOT_IMPLEMENTED((stride_in_bytes & 3u) != 0 ||
+	                     stride_in_bytes < (indexed ? sizeof(vk::DrawIndexedIndirectCommand)
+	                                                : sizeof(vk::DrawIndirectCommand)));
 
-	uint32_t draw_count = max_count_or_count;
-	if (count_addr != nullptr) {
-		draw_count = *count_addr;
-		if (draw_count > max_count_or_count) {
-			draw_count = max_count_or_count;
-		}
-	}
-
+	const auto draw_count = count_addr == nullptr
+	                            ? max_count_or_count
+	                            : std::min(uint32_t {*count_addr}, max_count_or_count);
 	if (draw_count == 0) {
 		return;
 	}
 	Common::DebugCounters::Add(Common::DebugCounters::Counter::IndirectDraws, draw_count);
 
-	const auto args_size = indexed ? sizeof(DrawIndexedIndirectArgs) : sizeof(DrawIndirectArgs);
-	EXIT_NOT_IMPLEMENTED(stride_in_bytes < args_size);
-
-	uint64_t index_size = 0;
-	if (indexed) {
-		switch (m_index_type_and_size) {
-			case 0: index_size = 2; break;
-			case 1: index_size = 4; break;
-			case 2: index_size = 1; break;
-			default: EXIT("unknown index_type_and_size: %u\n", m_index_type_and_size);
-		}
+	auto&      executor  = m_renderer.GetRenderExecutor();
+	const auto args_addr = m_draw_indirect_args_base_addr + data_offset;
+	// NUM_INSTANCES keeps the last draw's instance count.
+	m_num_instances_addr = args_addr + static_cast<uint64_t>(draw_count - 1) * stride_in_bytes +
+	                       offsetof(vk::DrawIndirectCommand, instanceCount);
+	// With no SGPRs to patch, the GPU reads the records if the shaders allow. Shaders apply
+	// the records' offsets only through those SGPRs; Vulkan applies them, which matches
+	// while they are zero.
+	const DrawIndirectArgs records {args_addr, draw_count, stride_in_bytes};
+	if (registers.vertex_offset == Pm4::SH_NOP && registers.instance_offset == Pm4::SH_NOP &&
+	    registers.index_offset == Pm4::SH_NOP && (!indexed || m_index_buffer_size != 0) &&
+	    (indexed ? executor.DrawIndex(m_submit_id, CurrentBuffer(),
+	                                  {.index_count         = m_index_buffer_size,
+	                                   .index_addr          = IndexAddress(0),
+	                                   .index_type_and_size = m_index_type_and_size,
+	                                   .indirect            = records})
+	             : executor.DrawAuto(m_submit_id, CurrentBuffer(), {.indirect = records}))) {
+		return;
 	}
 
+	const uint32_t max_indices = m_index_buffer_size != 0 ? m_index_buffer_size : UINT32_MAX;
 	for (uint32_t i = 0; i < draw_count; i++) {
-		const auto args_addr = m_draw_indirect_args_base_addr + data_offset +
-		                       static_cast<uint64_t>(i) * stride_in_bytes;
-
+		const auto addr = args_addr + static_cast<uint64_t>(i) * stride_in_bytes;
 		if (!indexed) {
-			auto* args = reinterpret_cast<const DrawIndirectArgs*>(args_addr);
-			PatchIndirectDrawOffsets(*this, registers, args->start_vertex_location,
-			                         args->start_instance_location, 0);
-			m_num_instances = args->instance_count;
-			DrawIndexAuto({.vertex_count   = args->vertex_count_per_instance,
-			               .instance_count = args->instance_count,
-			               .first_vertex   = args->start_vertex_location,
-			               .first_instance = args->start_instance_location,
-			               .offset_source  = DrawOffsetSource::IndirectArgs});
+			const auto& args = *reinterpret_cast<const vk::DrawIndirectCommand*>(addr);
+			PatchIndirectDrawOffsets(*this, registers, args.firstVertex, args.firstInstance, 0);
+			executor.DrawAuto(m_submit_id, CurrentBuffer(),
+			                  {.vertex_count   = args.vertexCount,
+			                   .instance_count = args.instanceCount,
+			                   .first_vertex   = args.firstVertex,
+			                   .first_instance = args.firstInstance,
+			                   .offset_source  = DrawOffsetSource::IndirectArgs});
 			continue;
 		}
 
-		auto* args = reinterpret_cast<const DrawIndexedIndirectArgs*>(args_addr);
-		PatchIndirectDrawOffsets(*this, registers, args->base_vertex_location,
-		                         args->start_instance_location, args->start_index_location);
-
-		auto* index_addr = reinterpret_cast<const void*>(
-		    m_index_base_addr + static_cast<uint64_t>(args->start_index_location) * index_size);
-
-		const uint32_t index_count =
-		    (m_index_buffer_size != 0
-		         ? std::min(args->index_count_per_instance, m_index_buffer_size)
-		         : args->index_count_per_instance);
-		if (GraphicsRunDebugDumpEnabled() && index_count != args->index_count_per_instance) {
-			static std::atomic<uint32_t> log_count {0};
-			if (log_count.fetch_add(1, std::memory_order_relaxed) < 64) {
-				LOGF("\t DrawIndexIndirectMulti: clamped index_count from %" PRIu32 " to %" PRIu32
-				     " using INDEX_BUFFER_SIZE\n",
-				     args->index_count_per_instance, index_count);
-			}
-		}
-
-		m_num_instances = args->instance_count;
-		DrawIndex({.index_count    = index_count,
-		           .index_addr     = index_addr,
-		           .instance_count = args->instance_count,
-		           .base_vertex    = static_cast<int32_t>(args->base_vertex_location),
-		           .first_instance = args->start_instance_location,
-		           .offset_source  = DrawOffsetSource::IndirectArgs});
+		const auto& args = *reinterpret_cast<const vk::DrawIndexedIndirectCommand*>(addr);
+		PatchIndirectDrawOffsets(*this, registers, static_cast<uint32_t>(args.vertexOffset),
+		                         args.firstInstance, args.firstIndex);
+		executor.DrawIndex(m_submit_id, CurrentBuffer(),
+		                   {.index_count         = std::min(args.indexCount, max_indices),
+		                    .index_addr          = IndexAddress(args.firstIndex),
+		                    .instance_count      = args.instanceCount,
+		                    .index_type_and_size = m_index_type_and_size,
+		                    .base_vertex         = args.vertexOffset,
+		                    .first_instance      = args.firstInstance,
+		                    .offset_source       = DrawOffsetSource::IndirectArgs});
 	}
 }
 
@@ -1012,16 +1002,17 @@ void CommandProcessor::DispatchIndirect(uint64_t args_addr, uint32_t mode) {
 
 void CommandProcessor::DrawIndexAuto(DrawAutoArgs args) {
 	if (args.instance_count == 0) {
-		args.instance_count = m_num_instances;
+		args.instance_count = NumInstances();
 	}
 	m_renderer.GetRenderExecutor().DrawAuto(m_submit_id, CurrentBuffer(), args);
 }
 
 void CommandProcessor::WaitFlipDone(uint32_t video_out_handle, uint32_t display_buffer_index) {
-	BufferFlush();
-
-	m_renderer.GetVideoOut().WaitFlipDone(static_cast<int>(video_out_handle),
-	                                      static_cast<int>(display_buffer_index));
+	// Stall only this command processor, not the host command lane or other queues.
+	if (!m_renderer.GetVideoOut().IsSafeForRendering(static_cast<int>(video_out_handle),
+	                                                 static_cast<int>(display_buffer_index))) {
+		SuspendPm4();
+	}
 }
 
 template <typename T>

src/graphics/guest_gpu/graphicsRun.h

@@ -31,6 +31,8 @@ public:
 	[[nodiscard]] bool IsStopping();
 	void               SendCommand(Common::UniqueFunction<void>&& command);
 	void               SendCommandSync(Common::UniqueFunction<void>&& command);
+	// Retries blocked submissions now instead of after the 100 ms stall timeout.
+	void               Wake();
 
 	// Submitted command memory is borrowed and must remain valid until GPU execution completes.
 	void Submit(std::span<const uint32_t> draw_commands,
@@ -87,6 +89,7 @@ private:
 	bool                                           m_processing        = false;
 	bool                                           m_accepting         = true;
 	bool                                           m_stopping          = false;
+	bool                                           m_wake_pending      = false;
 	bool                                           m_shutdown_complete = false;
 	// Completion callbacks can outlive GuestGpu during renderer shutdown.
 	std::shared_ptr<std::binary_semaphore> m_suspend_point_ready =

src/graphics/host_gpu/graphicContext.h

@@ -35,6 +35,7 @@ struct GraphicContext {
 	bool                               attachment_feedback_loop_enabled      = false;
 	bool                               provoking_vertex_last_enabled         = false;
 	bool                               maintenance8_enabled                  = false;
+	bool                               conditional_rendering_enabled         = false;
 	bool                               supports_block_texel_view              = false;
 	bool                                      mesh_shader_enabled                   = false;
 	vk::PhysicalDeviceMeshShaderPropertiesEXT mesh_shader_properties                = {};
@@ -42,6 +43,7 @@ struct GraphicContext {
 	uint32_t                           min_subgroup_size                     = 0;
 	uint32_t                           max_subgroup_size                     = 0;
 	uint32_t                           max_push_descriptors                  = 0;
+	vk::DescriptorSetLayout            image_heap_layout                     = nullptr;
 	vk::ShaderStageFlags               required_subgroup_size_stages         = {};
 	Common::Mutex                      queue_mutex;
 	uint32_t                           queue_family = static_cast<uint32_t>(-1);

src/graphics/host_gpu/renderer/cache/bufferCache.cpp

@@ -174,6 +174,7 @@ bool BufferCache::DownloadBufferMemory(Buffer& buffer, uint64_t vaddr, uint64_t
 			    total_size += Common::AlignUp(end - start, 64);
 		    });
 		    m_gpu_modified_ranges.Subtract(address, bytes);
+		    ForgetColorMetadataChecks(address, bytes);
 	    });
 	if (copies.empty()) {
 		return false;
@@ -519,6 +520,7 @@ std::pair<Buffer*, uint64_t> BufferCache::ObtainBuffer(uint64_t vaddr, uint64_t
 	(void)SynchronizeBuffer(buffer, vaddr, size, is_written, is_texel_buffer);
 	if (is_written) {
 		m_gpu_modified_ranges.Add(vaddr, size);
+		ForgetColorMetadataChecks(vaddr, size);
 	}
 	return {&buffer, buffer.Offset(vaddr)};
 }
@@ -623,6 +625,29 @@ bool BufferCache::IsRegionRegistered(uint64_t vaddr, uint64_t size) {
 	return address + m_slot_buffers[id].Size() > vaddr;
 }
 
+void BufferCache::MarkColorMetadataChecked(uint64_t vaddr, uint64_t size, uint32_t keys) {
+	for (uint32_t slot = 0; slot < m_checked_color_metadata.size(); slot++) {
+		if ((keys & (1u << slot)) != 0) {
+			m_checked_color_metadata[slot].Add(vaddr, size);
+		}
+	}
+}
+
+bool BufferCache::IsColorMetadataChecked(uint64_t vaddr, uint64_t size, uint32_t keys) const {
+	for (uint32_t slot = 0; slot < m_checked_color_metadata.size(); slot++) {
+		if ((keys & (1u << slot)) != 0 && !m_checked_color_metadata[slot].Contains(vaddr, size)) {
+			return false;
+		}
+	}
+	return true;
+}
+
+void BufferCache::ForgetColorMetadataChecks(uint64_t vaddr, uint64_t size) {
+	for (auto& checked: m_checked_color_metadata) {
+		checked.Subtract(vaddr, size);
+	}
+}
+
 bool BufferCache::IsRegionGpuModified(uint64_t vaddr, uint64_t size) {
 	return m_memory_tracker.IsRegionGpuModified(vaddr, size);
 }

src/graphics/host_gpu/renderer/cache/bufferCache.h

@@ -11,6 +11,7 @@
 #include "graphics/host_gpu/renderer/cache/multiLevelPageTable.h"
 #include "graphics/host_gpu/renderer/cache/streamBuffer.h"
 
+#include <array>
 #include <map>
 #include <span>
 #include <utility>
@@ -79,6 +80,10 @@ public:
 	[[nodiscard]] bool HasGpuDirtyBytes(uint64_t vaddr, uint64_t size);
 	[[nodiscard]] bool IsRegionCpuModified(uint64_t vaddr, uint64_t size);
 	[[nodiscard]] bool IsRegionGpuModified(uint64_t vaddr, uint64_t size);
+	// Color metadata the GPU found free of the clear keys in `keys` (bits of code >> 5) stays so
+	// until a GPU write or download.
+	void               MarkColorMetadataChecked(uint64_t vaddr, uint64_t size, uint32_t keys);
+	[[nodiscard]] bool IsColorMetadataChecked(uint64_t vaddr, uint64_t size, uint32_t keys) const;
 	void               ProcessFaultBuffer();
 	void               SynchronizeBda(const RangeSet& mapped);
 	void               QueueBdaSync(uint64_t vaddr, uint64_t size) {
@@ -124,6 +129,7 @@ private:
 	// Synchronous downloads publish before returning; asynchronous callers wait before reuse.
 	template <bool async>
 	[[nodiscard]] bool DownloadBufferMemory(Buffer& buffer, uint64_t vaddr, uint64_t size);
+	void               ForgetColorMetadataChecks(uint64_t vaddr, uint64_t size);
 
 	GraphicContext&                                   m_graphics;
 	CommandScheduler&                                 m_scheduler;
@@ -135,6 +141,8 @@ private:
 	BufferMap                                         m_buffers;
 	PageTable                                         m_page_table;
 	RangeSet                                          m_gpu_modified_ranges;
+	// One per TileManager::ColorClearSlots key slot.
+	std::array<RangeSet, 8>                           m_checked_color_metadata;
 	MemoryTracker                                     m_memory_tracker;
 	StreamBuffer                                      m_staging_buffer;
 	StreamBuffer                                      m_stream_buffer;

src/graphics/host_gpu/renderer/cache/textureCache.cpp

@@ -34,6 +34,8 @@ namespace Libs::Graphics {
 namespace {
 
 constexpr uint64_t NumFramesBeforeRemoval = 32;
+// Largest GPU-written tiled image (in texels) that is written back to guest memory on submit.
+constexpr uint64_t SmallTiledReadbackTexels = 64 * 64;
 
 [[nodiscard]] bool DecodeColorClear(const TextureCache::ImageDesc& desc, uint8_t code,
                                   vk::ClearColorValue& clear) {
@@ -122,7 +124,8 @@ TextureCache::TextureCache(GraphicContext& graphics, CommandScheduler& scheduler
       m_blit_helper(graphics, scheduler),
       m_tiler(graphics, scheduler, buffer_cache.GetUtilityBuffer(MemoryUsage::Stream)),
       m_buffer_cache(buffer_cache),
-      m_readback_linear_images(Config::ReadbackLinearImagesEnabled()) {
+      m_readback_linear_images(Config::ReadbackLinearImagesEnabled()),
+      m_readback_tiled_images(Config::ReadbackTiledImagesEnabled()) {
 	if (m_graphics.CanReportMemoryUsage()) {
 		constexpr int64_t GiB = 1024ll * 1024 * 1024;
 		const auto        budget =
@@ -226,6 +229,7 @@ void TextureCache::UnregisterImage(ImageId id) {
 	if (!image.registered) {
 		return;
 	}
+	image.HeapChanged();
 	UntrackImage(id);
 	ImagePageTable::PageRange pages {};
 	if (!ImagePageTable::TryGetPageRange(image.info.data.address, image.info.data.size, pages)) {
@@ -375,6 +379,7 @@ void TextureCache::UntrackImageHead(ImageId id) {
 	const auto address = Common::AlignDown(begin + TRACKER_PAGE_SIZE, TRACKER_PAGE_SIZE);
 	const auto size    = address - begin;
 	image.track_addr   = address;
+	image.HeapChanged();
 	if (image.track_addr == image.track_addr_end) {
 		MarkAsMaybeDirty(id, image);
 	}
@@ -392,6 +397,7 @@ void TextureCache::UntrackImageTail(ImageId id) {
 	const auto address   = Common::AlignDown(end, TRACKER_PAGE_SIZE);
 	const auto size      = end - address;
 	image.track_addr_end = address;
+	image.HeapChanged();
 	if (image.track_addr == image.track_addr_end) {
 		MarkAsMaybeDirty(id, image);
 	}
@@ -401,7 +407,14 @@ void TextureCache::UntrackImageTail(ImageId id) {
 }
 
 void TextureCache::TrackImageDownload(ImageId id, Image& image) {
-	if (m_readback_linear_images && !image.info.IsTiled()) {
+	// Small tiled targets are written back too: games read GPU results such as reductions and
+	// simulation state from them on the CPU (PPSA04263 keeps the origin of its ocean wave grid
+	// in a 1x1 R32G32 target, and its water physics read it to sample the wave heights).
+	const auto& info   = image.info;
+	const auto  texels = static_cast<uint64_t>(info.extent.width) * info.extent.height *
+	                     info.extent.depth * info.resources.layers;
+	if (info.IsTiled() ? m_readback_tiled_images && texels <= SmallTiledReadbackTexels
+	                   : m_readback_linear_images) {
 		if (!image.IsGpuModified()) {
 			EXIT("TextureCache: cannot enroll a non-GPU-owned image for download\n");
 		}
@@ -785,7 +798,8 @@ TextureCache::OverlapResult TextureCache::ResolveOverlap(const ImageInfo& reques
 	auto&      cached       = *owner;
 	const auto current_tick = m_scheduler.CurrentTick();
 	const bool safe_to_delete =
-	    current_tick - std::min(current_tick, cached.tick_accessed_last) > NumFramesBeforeRemoval;
+	    current_tick - std::min(current_tick, cached.tick_accessed_last) > NumFramesBeforeRemoval &&
+	    cached.heap_refs.load(std::memory_order_relaxed) == 0u;
 
 	const uint32_t requested_block = requested.bytes_per_block * requested.samples;
 	const uint32_t cached_block    = cached.info.bytes_per_block * cached.info.samples;
@@ -1057,12 +1071,78 @@ void TextureCache::MaterializeColorClear(ImageId id, const ImageDesc& desc,
 	if (first >= layers || count > layers - first) {
 		EXIT("TextureCache: color view exceeds its native metadata slices\n");
 	}
-	// Finish native metadata writes before reading backing bytes. This can submit the scheduler,
-	// so discovery runs before final draw uploads and never holds the texture lock across it.
-	if (m_buffer_cache.IsRegionGpuModified(range.address, range.size)) {
-		m_buffer_cache.ReadMemory(range.address, range.size, false);
+	const auto       slice_size = range.size / layers;
+	const GuestRange viewed {range.address + slice_size * first, slice_size * count};
+	if (m_buffer_cache.IsRegionGpuModified(viewed.address, viewed.size)) {
+		std::array<vk::ClearAttachment, TileManager::ColorClearSlots> clears {};
+		uint32_t keys = 0;
+		for (uint32_t key = 0; key < clears.size(); key++) {
+			clears[key].aspectMask = vk::ImageAspectFlagBits::eColor;
+			if (DecodeColorClear(desc, static_cast<uint8_t>(key << 5u),
+			                     clears[key].clearValue.color)) {
+				keys |= 1u << key;
+			}
+		}
+		// No selected key to clear, or none left since the last check.
+		if (m_buffer_cache.IsColorMetadataChecked(viewed.address, viewed.size, keys)) {
+			return;
+		}
+		{
+			std::scoped_lock lock {m_lock};
+			auto&            image = m_slot_images[id];
+			// Compressed images cannot upload, so they must already hold native contents.
+			const bool native_current = (image.usage.render_target || image.IsGpuModified()) &&
+			                            !image.IsBufferModified() && !image.IsCpuDirty();
+			// 2D-array attachment views select 2D layers or, of a volume, depth slices.
+			if (m_graphics.conditional_rendering_enabled &&
+			    image.backing.image_type != vk::ImageType::e1D &&
+			    count <= m_graphics.GetPhysicalDeviceProperties().limits.maxFramebufferLayers &&
+			    (image.backing.usage & vk::ImageUsageFlagBits::eColorAttachment) &&
+			    (m_graphics.GetFormatProperties(view.format).optimalTilingFeatures &
+			     vk::FormatFeatureFlagBits::eColorAttachment) &&
+			    (desc.info.metadata.compression == VideoOutCompression::Uncompressed ||
+			     native_current)) {
+				// The GPU selects each slice's clear, so the CPU never waits for the metadata.
+				Common::DebugCounters::Add(Common::DebugCounters::Counter::ColorMetadataChecks);
+				InitializeImage(id);
+				const bool expand = desc.type != BindingType::VideoOut;
+				const auto [buffer, offset] =
+				    m_buffer_cache.ObtainBuffer(viewed.address, viewed.size, expand);
+				const auto predicates = m_tiler.CheckColorClear(
+				    {buffer->Handle(), offset, viewed.size}, count, keys, expand);
+				if (expand) {
+					m_buffer_cache.MarkColorMetadataChecked(viewed.address, viewed.size, keys);
+				}
+				const auto command = m_scheduler.Current().Handle();
+				const auto area    = BeginColorRendering(image, view.format, view.base_level,
+				                                         image_first, count, nullptr, command);
+				vk::ClearRect rect {.rect = area, .layerCount = 1};
+				for (uint32_t slice = 0; slice < count; slice++) {
+					rect.baseArrayLayer = slice;
+					for (uint32_t key = 0; key < clears.size(); key++) {
+						if ((keys & (1u << key)) != 0) {
+							const vk::ConditionalRenderingBeginInfoEXT predicate {
+							    .buffer = predicates,
+							    .offset = (slice * clears.size() + key) * sizeof(uint32_t)};
+							command.beginConditionalRenderingEXT(&predicate);
+							command.clearAttachments(1, &clears[key], 1, &rect);
+							command.endConditionalRenderingEXT();
+						}
+					}
+				}
+				command.endRendering();
+				CommitGpuWrite(image);
+				return;
+			}
+		}
+		// Finish native metadata writes before reading backing bytes. This can submit the
+		// scheduler, so discovery runs before final draw uploads and never holds the texture
+		// lock across it.
+		Common::DebugCounters::ScopedTimer timer(
+		    Common::DebugCounters::Counter::ColorMetadataReadbacks,
+		    Common::DebugCounters::Counter::ColorMetadataReadbackNs);
+		m_buffer_cache.ReadMemory(viewed.address, viewed.size, false);
 	}
-	const auto slice_size = range.size / layers;
 	for (uint32_t slice = 0; slice < count; slice++) {
 		const auto address = range.address + slice_size * (first + slice);
 		uint8_t code = 0;
@@ -1564,29 +1644,8 @@ void TextureCache::ClearImage(CommandBuffer& command, ImageId id, vk::Format for
 	if (format != image.backing.format || (image.info.IsVolume() && !full_image)) {
 		EXIT_NOT_IMPLEMENTED(range.aspectMask != vk::ImageAspectFlagBits::eColor ||
 		                     range.levelCount != 1);
-		ImageViewInfo view {};
-		view.format = format;
-		view.type   = range.layerCount == 1 ? vk::ImageViewType::e2D : vk::ImageViewType::e2DArray;
-		view.base_level  = range.baseMipLevel;
-		view.base_layer  = range.baseArrayLayer;
-		view.layer_count = range.layerCount;
-		view.usage       = vk::ImageUsageFlagBits::eColorAttachment;
-		image.Transit(vk::ImageLayout::eColorAttachmentOptimal,
-		              vk::AccessFlagBits2::eColorAttachmentWrite, {}, command.Handle());
-		vk::RenderingAttachmentInfo attachment {};
-		attachment.imageView   = image.FindView(view);
-		attachment.imageLayout = vk::ImageLayout::eColorAttachmentOptimal;
-		attachment.loadOp      = vk::AttachmentLoadOp::eClear;
-		attachment.storeOp     = vk::AttachmentStoreOp::eStore;
-		attachment.clearValue  = clear;
-		vk::RenderingInfo rendering {};
-		rendering.renderArea.extent = {
-		    std::max(image.info.extent.width >> range.baseMipLevel, 1u),
-		    std::max(image.info.extent.height >> range.baseMipLevel, 1u)};
-		rendering.layerCount           = range.layerCount;
-		rendering.colorAttachmentCount = 1;
-		rendering.pColorAttachments    = &attachment;
-		command.Handle().beginRendering(&rendering);
+		BeginColorRendering(image, format, range.baseMipLevel, range.baseArrayLayer,
+		                    range.layerCount, &clear, command.Handle());
 		command.Handle().endRendering();
 		CommitGpuWrite(image);
 		return;
@@ -1609,6 +1668,40 @@ void TextureCache::ClearImage(CommandBuffer& command, ImageId id, vk::Format for
 	CommitGpuWrite(image);
 }
 
+vk::Rect2D TextureCache::BeginColorRendering(Image& image, vk::Format format, uint32_t level,
+                                             uint32_t layer, uint32_t layers,
+                                             const vk::ClearValue* clear,
+                                             vk::CommandBuffer     command) {
+	ImageViewInfo view {};
+	view.format      = format;
+	view.type        = layers == 1 ? vk::ImageViewType::e2D : vk::ImageViewType::e2DArray;
+	view.base_level  = level;
+	view.base_layer  = layer;
+	view.layer_count = layers;
+	view.usage       = vk::ImageUsageFlagBits::eColorAttachment;
+	image.Transit(vk::ImageLayout::eColorAttachmentOptimal,
+	              vk::AccessFlagBits2::eColorAttachmentRead |
+	                  vk::AccessFlagBits2::eColorAttachmentWrite,
+	              {}, command);
+	vk::RenderingAttachmentInfo attachment {};
+	attachment.imageView   = image.FindView(view);
+	attachment.imageLayout = vk::ImageLayout::eColorAttachmentOptimal;
+	attachment.loadOp      = vk::AttachmentLoadOp::eLoad;
+	attachment.storeOp     = vk::AttachmentStoreOp::eStore;
+	if (clear != nullptr) {
+		attachment.loadOp     = vk::AttachmentLoadOp::eClear;
+		attachment.clearValue = *clear;
+	}
+	vk::RenderingInfo rendering {};
+	rendering.renderArea.extent    = {std::max(image.info.extent.width >> level, 1u),
+	                                  std::max(image.info.extent.height >> level, 1u)};
+	rendering.layerCount           = layers;
+	rendering.colorAttachmentCount = 1;
+	rendering.pColorAttachments    = &attachment;
+	command.beginRendering(&rendering);
+	return rendering.renderArea;
+}
+
 void TextureCache::InvalidateMemory(uint64_t address, uint64_t size) {
 	if (!GuestRange {address, size}.Valid()) {
 		EXIT("TextureCache: invalid memory-invalidation range\n");
@@ -1742,6 +1835,7 @@ bool BufferCache::SynchronizeBufferFromImage(Buffer& buffer, uint64_t vaddr, uin
 	     copy_size != image.info.data.size)) {
 		return false;
 	}
+	ForgetColorMetadataChecks(image.info.data.address, copy_size);
 	m_texture_cache.DownloadImage(image, buffer, buf_offset, copy_size, levels);
 	return true;
 }
@@ -1925,6 +2019,11 @@ void TextureCache::RunGarbageCollector() {
 			if (owner == nullptr || !owner->registered || owner->depth_id) {
 				continue;
 			}
+			if (owner->heap_refs.load(std::memory_order_relaxed) != 0u) {
+				// Resident heaps sample it without touching it.
+				TouchImage(*owner);
+				continue;
+			}
 			if (owner->IsGpuModified()) {
 				const bool safe = owner->SafeToDownload();
 				if (safe && owner->info.IsTiled()) {
@@ -1956,7 +2055,7 @@ void TextureCache::RunGarbageCollector() {
 }
 
 void TextureCache::ProcessDownloadImages() {
-	if (!m_readback_linear_images) {
+	if (!m_readback_linear_images && !m_readback_tiled_images) {
 		return;
 	}
 	std::scoped_lock lock {m_lock};

src/graphics/host_gpu/renderer/cache/textureCache.h

@@ -150,6 +150,12 @@ private:
 	// Caller holds m_lock. Volume layer ranges select depth slices.
 	void ClearImage(CommandBuffer& command, ImageId id, vk::Format format,
 	                const vk::ImageSubresourceRange& range, const vk::ClearValue& clear);
+	// Caller holds m_lock and ends rendering. Begins rendering `layers` layers of one color mip
+	// through a `format` view, clearing them or, when `clear` is null, loading them; returns the
+	// render area.
+	vk::Rect2D BeginColorRendering(Image& image, vk::Format format, uint32_t level,
+	                               uint32_t layer, uint32_t layers, const vk::ClearValue* clear,
+	                               vk::CommandBuffer command);
 	void PrepareImageCopy(Image& image);
 	void RefreshCopySource(ImageId id);
 	[[nodiscard]] bool CopyD16(Image& destination, Image& source);
@@ -180,10 +186,12 @@ private:
 	uint64_t         m_gc_tick                = 0;
 	mutable uint32_t m_image_query_epoch      = 0;
 	bool             m_readback_linear_images = false;
+	bool             m_readback_tiled_images  = false;
 
 	friend struct TextureCacheTestAccess;
 	friend class BufferCache;
 	friend class RenderExecutor;
+	friend class ImageHeap;
 	friend bool BcAlias::Matches(const ImageInfo&, const ImageInfo&);
 	friend bool BcAlias::Publish(TextureCache&, Image&);
 	friend void BcAlias::Resolve(TextureCache&, ImageId, ImageId);

src/graphics/host_gpu/renderer/image/image.cpp

@@ -30,16 +30,18 @@ namespace {
 	}
 }
 
-[[nodiscard]] vk::ImageCreateFlags ImageCreateFlags(const GraphicContext& graphics,
-                                                   const ImageInfo& info) {
+[[nodiscard]] vk::ImageCreateFlags
+ImageCreateFlags(const GraphicContext& graphics, const ImageInfo& info, vk::ImageUsageFlags usage) {
 	vk::ImageCreateFlags flags {};
 	if (DepthAspectTransferFormat(info.pixel_format) == vk::Format::eUndefined) {
 		flags |= vk::ImageCreateFlagBits::eMutableFormat;
-		flags |= vk::ImageCreateFlagBits::eExtendedUsage;
 		if (info.IsBlock() && graphics.supports_block_texel_view) {
 			flags |= vk::ImageCreateFlagBits::eBlockTexelViewCompatible;
 		}
 	}
+	if (usage & vk::ImageUsageFlagBits::eStorage) {
+		flags |= vk::ImageCreateFlagBits::eExtendedUsage;
+	}
 	if (info.IsVolume()) {
 		flags |= vk::ImageCreateFlagBits::e2DArrayCompatible;
 	}
@@ -59,7 +61,7 @@ namespace {
 			const auto storage = usage | vk::ImageUsageFlagBits::eStorage;
 			if (graphics.GetImageFormatProperties(info.pixel_format, HostImageType(info.type),
 			                                      vk::ImageTiling::eOptimal, storage,
-			                                      ImageCreateFlags(graphics, info),
+			                                      ImageCreateFlags(graphics, info, storage),
 			                                      nullptr) == vk::Result::eSuccess) {
 				usage = storage;
 			} else {
@@ -88,7 +90,8 @@ namespace {
 	if (HasFormatFeature(properties, vk::FormatFeatureFlagBits::eColorAttachment)) {
 		usage |= vk::ImageUsageFlagBits::eColorAttachment;
 	}
-	if (info.samples == 1) {
+	if (graphics.GetPhysicalDeviceProperties().limits.storageImageSampleCounts &
+	    static_cast<vk::SampleCountFlagBits>(info.samples)) {
 		usage |= vk::ImageUsageFlagBits::eStorage;
 	}
 	return usage;
@@ -209,6 +212,13 @@ Image::Barriers Image::GetBarriers(vk::ImageLayout                      destinat
 		barriers.push_back(barrier);
 	}
 
+	// Resident heaps sample their images in the read state; only leaving it invalidates them.
+	if (std::ranges::any_of(barriers, [&](const vk::ImageMemoryBarrier2& barrier) {
+		    return barrier.oldLayout == ReadLayout() &&
+		           barrier.srcAccessMask == vk::AccessFlagBits2::eShaderRead;
+	    })) {
+		HeapChanged();
+	}
 	state = {destination_stage, destination_access, destination_layout};
 	return barriers;
 }
@@ -702,7 +712,8 @@ Image::Image(GraphicContext& graphics, CommandScheduler& scheduler, const ImageI
 	}
 
 	vk::ImageCreateInfo create {};
-	create.flags         = ImageCreateFlags(graphics, info);
+	create.usage         = ImageUsageFlags(graphics, info);
+	create.flags         = ImageCreateFlags(graphics, info, create.usage);
 	create.imageType     = HostImageType(info.type);
 	create.extent        = info.extent;
 	create.mipLevels     = info.resources.levels;
@@ -710,7 +721,6 @@ Image::Image(GraphicContext& graphics, CommandScheduler& scheduler, const ImageI
 	create.format        = info.pixel_format;
 	create.tiling        = vk::ImageTiling::eOptimal;
 	create.initialLayout = vk::ImageLayout::eUndefined;
-	create.usage         = ImageUsageFlags(graphics, info);
 	create.samples       = vulkan_sample_count(info.samples);
 
 	vk::ImageFormatProperties properties {};

src/graphics/host_gpu/renderer/image/image.h

@@ -7,6 +7,7 @@
 #include "graphics/host_gpu/graphicContext.h"
 #include "graphics/host_gpu/renderer/image/imageInfo.h"
 
+#include <atomic>
 #include <compare>
 #include <limits>
 #include <optional>
@@ -71,6 +72,7 @@ public:
 	void MarkCpuDirty() {
 		m_cpu_dirty       = true;
 		m_maybe_cpu_dirty = false;
+		HeapChanged();
 	}
 
 	[[nodiscard]] bool IsCpuDirty() const { return m_cpu_dirty || m_maybe_cpu_dirty; }
@@ -80,6 +82,7 @@ public:
 		if (!IsCpuDirty()) {
 			m_maybe_cpu_hash  = HashGuestEdges();
 			m_maybe_cpu_dirty = true;
+			HeapChanged();
 		}
 	}
 	[[nodiscard]] bool ResolveMaybeCpuHash(uint64_t hash) {
@@ -104,7 +107,10 @@ public:
 	void               ClearGpuModified() noexcept { m_gpu_modified = false; }
 
 	[[nodiscard]] bool IsBufferModified() const noexcept { return m_buffer_modified; }
-	void               MarkBufferModified() noexcept { m_buffer_modified = true; }
+	void               MarkBufferModified() noexcept {
+		m_buffer_modified = true;
+		HeapChanged();
+	}
 	void               ClearBufferModified() noexcept { m_buffer_modified = false; }
 
 	[[nodiscard]] bool Overlaps(uint64_t address, uint64_t size,
@@ -120,6 +126,19 @@ public:
 		return backing.image == nullptr ? 0 : Common::AlignUp(info.data.size, 1024);
 	}
 	[[nodiscard]] uint64_t HashGuestEdges() const;
+	[[nodiscard]] vk::ImageLayout ReadLayout() const noexcept {
+		return info.IsDepth() ? vk::ImageLayout::eDepthStencilReadOnlyOptimal
+		                      : vk::ImageLayout::eShaderReadOnlyOptimal;
+	}
+	// Resident image heaps sample an image without rediscovering it until one changes its
+	// contents, tracking, identity or layout.
+	void HeapChanged() const noexcept {
+		if (heap_refs.load(std::memory_order_relaxed) != 0u) {
+			heap_epoch.fetch_add(1u, std::memory_order_relaxed);
+		}
+	}
+
+	static inline std::atomic<uint64_t> heap_epoch {0};
 
 	ImageInfo        info;
 	VulkanImage      backing;
@@ -137,6 +156,8 @@ public:
 	ImageSubresourceRange stencil_subresources;
 	uint64_t         tick_accessed_last = 0;
 	size_t           lru_id             = 0;
+	// Resident image heap elements that select this image.
+	std::atomic<uint32_t> heap_refs {0};
 
 private:
 	friend struct ImageTestAccess;

src/graphics/host_gpu/renderer/image/imageView.h

@@ -101,6 +101,7 @@ IsSupportedStorageImageResource(const ShaderRecompiler::IR::ImageResource& resou
 	       (resource.dimension == ShaderRecompiler::Decoder::ImageDimension::Dim1D ||
 	        resource.dimension == ShaderRecompiler::Decoder::ImageDimension::Dim1DArray ||
 	        resource.dimension == ShaderRecompiler::Decoder::ImageDimension::Dim2D ||
+	        resource.dimension == ShaderRecompiler::Decoder::ImageDimension::Dim2DMsaa ||
 	        resource.dimension == ShaderRecompiler::Decoder::ImageDimension::Dim3D ||
 	        resource.dimension == ShaderRecompiler::Decoder::ImageDimension::Dim2DArray) &&
 	       ((resource.mip_mode == ShaderRecompiler::IR::ImageMipMode::None &&

src/graphics/host_gpu/renderer/image/tiler.cpp

@@ -14,6 +14,7 @@
 #include "gpu_tiler_shaders/gpu_tiler_standard64_3d_spv.h"
 #include "gpu_tiler_shaders/gpu_tiler_standard64_spv.h"
 #include "gpu_tiler_shaders/gpu_tiler_color_transform_spv.h"
+#include "gpu_tiler_shaders/gpu_tiler_color_clear_spv.h"
 #include "graphics/host_gpu/graphicContext.h"
 #include "graphics/host_gpu/renderer/cache/streamBuffer.h"
 #include "graphics/host_gpu/renderer/commandScheduler.h"
@@ -71,20 +72,11 @@ TileManager::~TileManager() {
 			m_graphics.device.destroyPipeline(pipeline, nullptr);
 		}
 	}
-	if (m_d16_to_d24 != nullptr) {
-		m_graphics.device.destroyPipeline(m_d16_to_d24, nullptr);
-	}
-	if (m_d16_to_d32 != nullptr) {
-		m_graphics.device.destroyPipeline(m_d16_to_d32, nullptr);
-	}
-	if (m_d24_to_d16 != nullptr) {
-		m_graphics.device.destroyPipeline(m_d24_to_d16, nullptr);
-	}
-	if (m_d32_to_d16 != nullptr) {
-		m_graphics.device.destroyPipeline(m_d32_to_d16, nullptr);
-	}
-	if (m_color_transform != nullptr) {
-		m_graphics.device.destroyPipeline(m_color_transform, nullptr);
+	for (auto pipeline: {m_d16_to_d24, m_d16_to_d32, m_d24_to_d16, m_d32_to_d16, m_color_transform,
+	                     m_color_clear}) {
+		if (pipeline != nullptr) {
+			m_graphics.device.destroyPipeline(pipeline, nullptr);
+		}
 	}
 	if (m_pipeline_layout != nullptr) {
 		m_graphics.device.destroyPipelineLayout(m_pipeline_layout, nullptr);
@@ -220,42 +212,37 @@ vk::Pipeline TileManager::GetPipeline(uint32_t slot) {
 	if (m_pipelines[slot] != nullptr) {
 		return m_pipelines[slot];
 	}
-	struct Shader {
-		const uint32_t* code;
-		size_t          words;
-	};
-	static constexpr std::array<Shader, FamilyCount> shaders {{
-	    {GPU_TILER_STANDARD256_SPV, std::size(GPU_TILER_STANDARD256_SPV)},
-	    {GPU_TILER_STANDARD4_SPV, std::size(GPU_TILER_STANDARD4_SPV)},
-	    {GPU_TILER_STANDARD4_3D_SPV, std::size(GPU_TILER_STANDARD4_3D_SPV)},
-	    {GPU_TILER_STANDARD64_SPV, std::size(GPU_TILER_STANDARD64_SPV)},
-	    {GPU_TILER_STANDARD64_3D_SPV, std::size(GPU_TILER_STANDARD64_3D_SPV)},
-	    {GPU_TILER_PRT_SPV, std::size(GPU_TILER_PRT_SPV)},
-	    {GPU_TILER_PRT_3D_SPV, std::size(GPU_TILER_PRT_3D_SPV)},
-	    {GPU_TILER_RENDER_TARGET_SPV, std::size(GPU_TILER_RENDER_TARGET_SPV)},
-	    {GPU_TILER_DEPTH_SPV, std::size(GPU_TILER_DEPTH_SPV)},
-	}};
-	const uint32_t                                   element_index = slot % BytesPerElementCount;
+	static constexpr std::array<std::span<const uint32_t>, FamilyCount> shaders {
+	    GPU_TILER_STANDARD256_SPV, GPU_TILER_STANDARD4_SPV,     GPU_TILER_STANDARD4_3D_SPV,
+	    GPU_TILER_STANDARD64_SPV,  GPU_TILER_STANDARD64_3D_SPV, GPU_TILER_PRT_SPV,
+	    GPU_TILER_PRT_3D_SPV,      GPU_TILER_RENDER_TARGET_SPV, GPU_TILER_DEPTH_SPV};
+	const uint32_t                   element_index   = slot % BytesPerElementCount;
 	const uint32_t                   direction_index = slot / (FamilyCount * BytesPerElementCount);
 	const uint32_t                   family_index    = (slot / BytesPerElementCount) % FamilyCount;
 	const uint32_t                   values[] {1u << element_index, direction_index};
 	const vk::SpecializationMapEntry entries[] {{0, 0, 4}, {1, 4, 4}};
 	const vk::SpecializationInfo     specialization {2, entries, sizeof(values), values};
-	const auto module =
-	    CompileSPV({shaders[family_index].code, shaders[family_index].words}, m_graphics.device);
+	m_pipelines[slot] = CreatePipeline(shaders[family_index], &specialization);
+	return m_pipelines[slot];
+}
+
+vk::Pipeline TileManager::CreatePipeline(std::span<const uint32_t>     code,
+                                         const vk::SpecializationInfo* specialization) const {
+	const auto                        module = CompileSPV(code, m_graphics.device);
 	vk::PipelineShaderStageCreateInfo stage {};
 	stage.stage               = vk::ShaderStageFlagBits::eCompute;
 	stage.module              = module;
 	stage.pName               = "main";
-	stage.pSpecializationInfo = &specialization;
+	stage.pSpecializationInfo = specialization;
 	vk::ComputePipelineCreateInfo create {};
-	create.stage  = stage;
-	create.layout = m_pipeline_layout;
-	const auto result =
-	    m_graphics.device.createComputePipelines(nullptr, 1, &create, nullptr, &m_pipelines[slot]);
+	create.stage          = stage;
+	create.layout         = m_pipeline_layout;
+	vk::Pipeline pipeline = nullptr;
+	const auto   result =
+	    m_graphics.device.createComputePipelines(nullptr, 1, &create, nullptr, &pipeline);
 	m_graphics.device.destroyShaderModule(module, nullptr);
 	RequireVulkanSuccess(result, "create TileManager pipeline");
-	return m_pipelines[slot];
+	return pipeline;
 }
 
 void TileManager::Record(vk::Buffer source, uint64_t source_offset, uint64_t source_capacity,
@@ -434,39 +421,21 @@ uint32_t TileManager::ConversionRows(uint64_t offset, uint64_t row_stride, uint6
 
 void TileManager::ConvertD16(Result source, Result target, D16Direction direction, bool d32,
                              const D16Layout& layout) {
-	vk::Pipeline* pipeline_pointer = nullptr;
+	vk::Pipeline*             pipeline_pointer = nullptr;
+	std::span<const uint32_t> code;
 	if (direction == D16Direction::Promote) {
 		pipeline_pointer = d32 ? &m_d16_to_d32 : &m_d16_to_d24;
+		code             = GPU_TILER_PROMOTE_D16_SPV;
 	} else {
 		pipeline_pointer = d32 ? &m_d32_to_d16 : &m_d24_to_d16;
+		code             = GPU_TILER_DEMOTE_D16_SPV;
 	}
 	auto& pipeline = *pipeline_pointer;
 	if (pipeline == nullptr) {
 		const uint32_t                   value = d32 ? 1u : 0u;
 		const vk::SpecializationMapEntry entry {0, 0, sizeof(value)};
 		const vk::SpecializationInfo     specialization {1, &entry, sizeof(value), &value};
-		const uint32_t*                  code  = nullptr;
-		size_t                           words = 0;
-		if (direction == D16Direction::Promote) {
-			code  = GPU_TILER_PROMOTE_D16_SPV;
-			words = std::size(GPU_TILER_PROMOTE_D16_SPV);
-		} else {
-			code  = GPU_TILER_DEMOTE_D16_SPV;
-			words = std::size(GPU_TILER_DEMOTE_D16_SPV);
-		}
-		const auto module = CompileSPV({code, words}, m_graphics.device);
-		vk::PipelineShaderStageCreateInfo stage {};
-		stage.stage               = vk::ShaderStageFlagBits::eCompute;
-		stage.module              = module;
-		stage.pName               = "main";
-		stage.pSpecializationInfo = &specialization;
-		vk::ComputePipelineCreateInfo create {};
-		create.stage  = stage;
-		create.layout = m_pipeline_layout;
-		const auto result =
-		    m_graphics.device.createComputePipelines(nullptr, 1, &create, nullptr, &pipeline);
-		m_graphics.device.destroyShaderModule(module, nullptr);
-		RequireVulkanSuccess(result, "create D16 conversion pipeline");
+		pipeline = CreatePipeline(code, &specialization);
 	}
 
 	const uint64_t source_element =
@@ -599,18 +568,7 @@ void TileManager::TransformColor(Result input, Result output, ColorTransform tra
 	        output.size < input.size);
 	const auto pixels = static_cast<uint32_t>(input.size / element_bytes);
 	if (m_color_transform == nullptr) {
-		const auto module = CompileSPV(GPU_TILER_COLOR_TRANSFORM_SPV, m_graphics.device);
-		vk::PipelineShaderStageCreateInfo stage {};
-		stage.stage  = vk::ShaderStageFlagBits::eCompute;
-		stage.module = module;
-		stage.pName  = "main";
-		vk::ComputePipelineCreateInfo create {};
-		create.stage  = stage;
-		create.layout = m_pipeline_layout;
-		const auto result =
-		    m_graphics.device.createComputePipelines(nullptr, 1, &create, nullptr, &m_color_transform);
-		m_graphics.device.destroyShaderModule(module, nullptr);
-		RequireVulkanSuccess(result, "create color transform pipeline");
+		m_color_transform = CreatePipeline(GPU_TILER_COLOR_TRANSFORM_SPV, nullptr);
 	}
 	const auto input_binding  = BindStorage(input, input.size);
 	const auto output_binding = BindStorage(output, input.size);
@@ -670,4 +628,60 @@ TileManager::Result TileManager::TransformColor(Result input, ColorTransform tra
 	return result;
 }
 
+vk::Buffer TileManager::CheckColorClear(Result metadata, uint32_t slices, uint32_t keys,
+                                        bool expand) {
+	struct Check {
+		uint32_t metadata_base;
+		uint32_t words;
+		uint32_t keys;
+		uint32_t expand;
+	};
+	static_assert(sizeof(Check) <= sizeof(Conversion));
+	const uint32_t max_slices =
+	    m_graphics.GetPhysicalDeviceProperties().limits.maxFramebufferLayers;
+	EXIT_IF(slices == 0 || slices > max_slices || metadata.size % (4ull * slices) != 0 ||
+	        metadata.size / 4 > UINT32_MAX);
+	if (m_color_clear == nullptr) {
+		m_color_clear      = CreatePipeline(GPU_TILER_COLOR_CLEAR_SPV, nullptr);
+		m_clear_predicates = std::make_unique<Buffer>(
+		    m_graphics, m_scheduler, MemoryUsage::DeviceLocal, 0,
+		    vk::BufferUsageFlagBits::eStorageBuffer |
+		        vk::BufferUsageFlagBits::eConditionalRenderingEXT,
+		    uint64_t {max_slices} * ColorClearSlots * sizeof(uint32_t));
+	}
+	const auto source = BindStorage(metadata, metadata.size);
+	const vk::DescriptorBufferInfo infos[] {source.info,
+	                                        {m_clear_predicates->Handle(), 0, VK_WHOLE_SIZE}};
+	std::array<vk::WriteDescriptorSet, 2> writes {};
+	for (uint32_t index = 0; index < writes.size(); index++) {
+		writes[index].dstBinding      = index;
+		writes[index].descriptorCount = 1;
+		writes[index].descriptorType  = vk::DescriptorType::eStorageBuffer;
+		writes[index].pBufferInfo     = &infos[index];
+	}
+	// Orders the metadata writers and the previous check's predicate reads before this check.
+	vk::MemoryBarrier barrier {
+	    .srcAccessMask = vk::AccessFlagBits::eMemoryWrite,
+	    .dstAccessMask = vk::AccessFlagBits::eShaderRead | vk::AccessFlagBits::eShaderWrite};
+	m_scheduler.EndRendering();
+	auto command = m_scheduler.Current().Handle();
+	command.pipelineBarrier(vk::PipelineStageFlagBits::eAllCommands,
+	                        vk::PipelineStageFlagBits::eComputeShader, {}, 1, &barrier, 0, nullptr,
+	                        0, nullptr);
+	command.bindPipeline(vk::PipelineBindPoint::eCompute, m_color_clear);
+	command.pushDescriptorSetKHR(vk::PipelineBindPoint::eCompute, m_pipeline_layout, 0,
+	                             static_cast<uint32_t>(writes.size()), writes.data());
+	const Check push {source.base, static_cast<uint32_t>(metadata.size / 4 / slices), keys,
+	                  expand};
+	command.pushConstants(m_pipeline_layout, vk::ShaderStageFlagBits::eCompute, 0, sizeof(push),
+	                      &push);
+	command.dispatch(slices, 1, 1);
+	barrier.srcAccessMask = vk::AccessFlagBits::eShaderWrite;
+	barrier.dstAccessMask = vk::AccessFlagBits::eMemoryRead | vk::AccessFlagBits::eMemoryWrite;
+	command.pipelineBarrier(vk::PipelineStageFlagBits::eComputeShader,
+	                        vk::PipelineStageFlagBits::eAllCommands, {}, 1, &barrier, 0, nullptr,
+	                        0, nullptr);
+	return m_clear_predicates->Handle();
+}
+
 } // namespace Libs::Graphics

src/graphics/host_gpu/renderer/image/tiler.h

@@ -54,6 +54,13 @@ public:
 	                                const D16Layout& layout);
 	[[nodiscard]] Result TransformColor(Result input, ColorTransform transform, bool to_host);
 	void TransformColor(Result input, Result output, ColorTransform transform, bool to_host);
+	// Color-clear keys are multiples of 0x20, so key >> 5 selects one of these slots.
+	static constexpr uint32_t ColorClearSlots = 8;
+	// Returns the buffer whose dword slice * ColorClearSlots + (key >> 5) is set when every byte
+	// of that metadata slice equals a key whose bit `keys` selects; a hit expands to 0xff when
+	// `expand` is set.
+	[[nodiscard]] vk::Buffer CheckColorClear(Result metadata, uint32_t slices, uint32_t keys,
+	                                         bool expand);
 
 private:
 	friend struct TileManagerTestAccess;
@@ -117,6 +124,8 @@ private:
 	                            vk::Buffer target, uint64_t target_offset, uint64_t target_capacity,
 	                            const Tiling& tiling, bool clear_target);
 	[[nodiscard]] vk::Pipeline GetPipeline(uint32_t slot);
+	[[nodiscard]] vk::Pipeline CreatePipeline(std::span<const uint32_t>     code,
+	                                          const vk::SpecializationInfo* specialization) const;
 
 	GraphicContext&                         m_graphics;
 	CommandScheduler&                       m_scheduler;
@@ -130,6 +139,8 @@ private:
 	vk::Pipeline                            m_d24_to_d16      = nullptr;
 	vk::Pipeline                            m_d32_to_d16      = nullptr;
 	vk::Pipeline                            m_color_transform = nullptr;
+	vk::Pipeline                            m_color_clear     = nullptr;
+	std::unique_ptr<Buffer>                 m_clear_predicates;
 };
 
 } // namespace Libs::Graphics

src/graphics/host_gpu/renderer/pipeline/descriptors.cpp

@@ -254,6 +254,10 @@ static bool IsSupportedStorageTextureDescriptor(const ShaderRecompiler::IR::Imag
 	const bool is_3d = resource.dimension == ShaderRecompiler::Decoder::ImageDimension::Dim3D &&
 	                   descriptor.Type() == Prospero::ImageType::kColor3D &&
 	                   descriptor.BaseArray5() == 0;
+	const bool is_2d_msaa =
+	    resource.dimension == ShaderRecompiler::Decoder::ImageDimension::Dim2DMsaa &&
+	    descriptor.Type() == Prospero::ImageType::kColor2DMsaa && descriptor.Depth() == 0 &&
+	    descriptor.BaseArray5() == 0;
 	TileTextureBlockLayout tile_layout {};
 	bool                   supported_tile = false;
 	switch (tile) {
@@ -284,8 +288,8 @@ static bool IsSupportedStorageTextureDescriptor(const ShaderRecompiler::IR::Imag
 	const bool supported_swizzle =
 	    IsValidImageSwizzle(swizzle) &&
 	    (swizzle == DstSel(4, 5, 6, 7) || !resource.read || resource.atomic);
-	return (is_1d || is_1d_array || is_2d || is_2d_array || is_3d) && supported_tile &&
-	       descriptor.BaseLevel() <= descriptor.LastLevel() &&
+	return (is_1d || is_1d_array || is_2d || is_2d_array || is_3d || is_2d_msaa) &&
+	       supported_tile && descriptor.BaseLevel() <= descriptor.LastLevel() &&
 	       descriptor.MinLod() == 0 && supported_swizzle && descriptor.BCSwizzle() == 0 &&
 	       !descriptor.MsaaDepth();
 }
@@ -356,7 +360,7 @@ static ImageViewInfo TextureViewInfo(const ShaderRecompiler::IR::ImageResource&
 	}
 	view.usage = storage ? vk::ImageUsageFlagBits::eStorage : vk::ImageUsageFlagBits::eSampled;
 	view.mapping =
-	    storage || surface_format.conversion_format != Prospero::BufferFormat::kInvalid
+	    storage || resource.conversion_format != Prospero::BufferFormat::kInvalid
 	        ? vk::ComponentMapping {}
 	        : TextureGetComponentMapping(descriptor.DstSelXYZW(), surface_format.host_to_storage);
 	switch (resource.dimension) {
@@ -515,7 +519,9 @@ TextureBinding RenderExecutor::ResolveTexture(const ShaderRecompiler::IR::ImageR
 	}
 	const auto samples = multisampled ? 1u << last_level : 1u;
 	const auto depth          = static_cast<uint32_t>(descriptor.Depth()) + 1u;
-	const auto format         = descriptor.Format();
+	const auto format         = ShaderRecompiler::IR::ReadsDepthBits(resource)
+	                                ? Prospero::BufferFormat::k16UNorm
+	                                : descriptor.Format();
 	const auto surface_format = TextureGetSurfaceFormatInfo(format);
 	const bool shader_conversion =
 	    surface_format.conversion_format != Prospero::BufferFormat::kInvalid;
@@ -579,6 +585,10 @@ TextureBinding RenderExecutor::ResolveTexture(const ShaderRecompiler::IR::ImageR
 	const auto alignment = desc.info.DataAlignment();
 	EXIT_NOT_IMPLEMENTED(desc.info.data.size == 0 || alignment == 0 ||
 	                     (address & (static_cast<uint64_t>(alignment) - 1u)) != 0);
+	// Image heaps keep the descriptors of unmapped textures, which PS5 never samples.
+	if (resource.heap && !m_context.IsMapped(address, desc.info.data.size)) {
+		return {.layout = vk::ImageLayout::eGeneral};
+	}
 	if (storage) {
 		ValidateStorageTexture(resource, descriptor, desc.info.data.size);
 	}
@@ -684,6 +694,10 @@ void RenderExecutor::PrepareBindings(const ShaderStageRuntime& runtime,
 	prepared.samplers.clear();
 	prepared.shader_data.clear();
 	for (uint32_t i = 0; i < program.info.images.size(); i++) {
+		if (program.info.images[i].heap) {
+			prepared.images[i] = {};
+			continue;
+		}
 		auto binding = ResolveTexture(program.info.images[i], snapshot.images[i]);
 		BindImage(binding.image_id, binding.desc.type == TextureCache::BindingType::Storage);
 		binding.mip_views.swap(prepared.images[i].mip_views);
@@ -846,14 +860,19 @@ void RenderExecutor::RebindImages(PreparedBindings& prepared) {
 
 void RenderExecutor::PrepareGraphicsBindings(std::span<PreparedBindings* const> stages,
                                              std::span<RenderColorInfo> colors) {
-	bool uses_dma = false;
+	bool uses_dma        = false;
+	bool uses_image_heap = false;
 	FindBuffers(stages);
 	for (auto* stage: stages) {
 		uses_dma |= stage->runtime->program->info.uses_dma;
+		uses_image_heap |= stage->runtime->program->info.uses_image_heap;
 	}
 	if (uses_dma) {
 		m_context.PrepareBda();
 	}
+	if (uses_image_heap) {
+		m_context.GetImageHeap().Rebind(*this);
+	}
 	for (auto* stage: stages) {
 		RebindImages(*stage);
 	}
@@ -953,6 +972,9 @@ void RenderExecutor::CommitBindings(CommandBuffer&                     buffer,
 			}
 		}
 	}
+	if (pipeline.uses_image_heap) {
+		m_context.GetImageHeap().Commit(buffer, pipeline_bind_point, pipeline.pipeline_layout);
+	}
 	m_descriptor_images.resize(image_descriptor_count);
 	m_descriptor_writes.clear();
 	m_descriptor_writes.reserve(write_count);
@@ -1010,9 +1032,7 @@ void RenderExecutor::CommitBindings(CommandBuffer&                     buffer,
 				              vk::AccessFlagBits2::eShaderRead | vk::AccessFlagBits2::eShaderWrite,
 				              range, vk_buffer);
 			} else {
-				image.Transit(image.info.IsDepth() ? vk::ImageLayout::eDepthStencilReadOnlyOptimal
-				                                   : vk::ImageLayout::eShaderReadOnlyOptimal,
-				              vk::AccessFlagBits2::eShaderRead, range, vk_buffer);
+				image.Transit(image.ReadLayout(), vk::AccessFlagBits2::eShaderRead, range, vk_buffer);
 			}
 			binding.layout = image.backing.state.layout;
 		}
@@ -1083,6 +1103,7 @@ void RenderExecutor::CommitBindings(CommandBuffer&                     buffer,
 		for (uint32_t i = 0; i < program.info.images.size(); ++i) {
 			const auto& resource = program.info.images[i];
 			const auto& texture = descriptors.images[i];
+			if (resource.heap) continue;
 			EXIT_IF(texture.image_id &&
 			        resource.mip_count != (texture.mip_views.empty() ? 1u : texture.mip_views.size()));
 			const auto kind = *ShaderRecompiler::IR::DescriptorBindingForImage(resource);

src/graphics/host_gpu/renderer/pipeline/imageHeap.cpp

@@ -0,0 +1,544 @@
+#include "graphics/host_gpu/renderer/pipeline/imageHeap.h"
+
+#include "common/alignment.h"
+#include "common/assert.h"
+#include "common/profiler.h"
+#include "graphics/host_gpu/pageManager.h"
+#include "graphics/host_gpu/regionDefinitions.h"
+#include "graphics/host_gpu/renderer/cache/streamBuffer.h"
+#include "graphics/host_gpu/renderer/render.h"
+#include "graphics/host_gpu/renderer/renderContext.h"
+#include "graphics/shader/recompiler/ir/passes/ResourceMaterialization.h"
+#include "kernel/memory.h"
+
+#include <algorithm>
+#include <xxhash.h>
+
+namespace Libs::Graphics {
+namespace {
+
+namespace IR = ShaderRecompiler::IR;
+
+// Smaller tables are cheap to materialize on each draw and are often transient.
+constexpr uint64_t MinTableBytes = 64ull * 1024ull;
+constexpr uint32_t ElementMask   = (1u << IR::ImageHeapKindShift) - 1u;
+
+template <typename Pred, typename Fn>
+void ForEachRun(size_t begin, size_t end, Pred&& pred, Fn&& fn) {
+	while (begin < end) {
+		if (!pred(begin)) {
+			++begin;
+			continue;
+		}
+		auto run = begin + 1u;
+		while (run < end && pred(run)) ++run;
+		fn(begin, run);
+		begin = run;
+	}
+}
+
+// Grows [begin, end) to cover [first, last).
+void Widen(size_t& begin, size_t& end, size_t first, size_t last) {
+	const bool empty = begin == end;
+	begin            = empty ? first : std::min(begin, first);
+	end              = empty ? last : std::max(end, last);
+}
+
+// Guest bytes of a table that its tracker pages [begin, end) cover.
+std::pair<uint64_t, uint64_t> PageRange(uint64_t base, uint64_t size, size_t begin, size_t end) {
+	const auto origin = Common::AlignDown(base, TRACKER_PAGE_SIZE);
+	return {std::max(base, origin + begin * TRACKER_PAGE_SIZE),
+	        std::min(base + size, origin + end * TRACKER_PAGE_SIZE)};
+}
+
+} // namespace
+
+size_t ImageHeap::DescriptorHash::operator()(const DescriptorValue& value) const noexcept {
+	return XXH3_64bits(value.dwords.data(), sizeof(value.dwords));
+}
+
+ImageHeap::ImageHeap(RenderContext& context, PageManager& page_manager)
+    : m_context(context), m_page_manager(page_manager) {
+	auto& graphics = context.GetGraphics();
+	vk::PhysicalDeviceVulkan12Properties limits {};
+	vk::PhysicalDeviceProperties2        properties {};
+	properties.pNext = &limits;
+	graphics.physical_device.getProperties2(&properties);
+	EXIT_NOT_IMPLEMENTED(std::min({limits.maxDescriptorSetUpdateAfterBindSampledImages,
+	                               limits.maxPerStageDescriptorUpdateAfterBindSampledImages,
+	                               limits.maxPerStageUpdateAfterBindResources}) <
+	                     IR::ImageHeapCapacity);
+
+	const vk::DescriptorBindingFlags binding_flags =
+	    vk::DescriptorBindingFlagBits::ePartiallyBound |
+	    vk::DescriptorBindingFlagBits::eUpdateAfterBind |
+	    vk::DescriptorBindingFlagBits::eUpdateUnusedWhilePending;
+	vk::DescriptorSetLayoutBindingFlagsCreateInfo flags_info {};
+	flags_info.bindingCount  = 1;
+	flags_info.pBindingFlags = &binding_flags;
+	const vk::DescriptorSetLayoutBinding binding {0, vk::DescriptorType::eSampledImage,
+	                                              IR::ImageHeapCapacity,
+	                                              vk::ShaderStageFlagBits::eAll, nullptr};
+	vk::DescriptorSetLayoutCreateInfo layout_info {};
+	layout_info.pNext        = &flags_info;
+	layout_info.flags        = vk::DescriptorSetLayoutCreateFlagBits::eUpdateAfterBindPool;
+	layout_info.bindingCount = 1;
+	layout_info.pBindings    = &binding;
+	EXIT_IF(graphics.device.createDescriptorSetLayout(&layout_info, nullptr, &m_layout) !=
+	        vk::Result::eSuccess);
+	const vk::DescriptorPoolSize pool_size {vk::DescriptorType::eSampledImage,
+	                                        IR::ImageHeapCapacity};
+	vk::DescriptorPoolCreateInfo pool_info {};
+	pool_info.flags         = vk::DescriptorPoolCreateFlagBits::eUpdateAfterBind;
+	pool_info.maxSets       = 1;
+	pool_info.poolSizeCount = 1;
+	pool_info.pPoolSizes    = &pool_size;
+	EXIT_IF(graphics.device.createDescriptorPool(&pool_info, nullptr, &m_pool) !=
+	        vk::Result::eSuccess);
+	vk::DescriptorSetAllocateInfo allocate {};
+	allocate.descriptorPool     = m_pool;
+	allocate.descriptorSetCount = 1;
+	allocate.pSetLayouts        = &m_layout;
+	EXIT_IF(graphics.device.allocateDescriptorSets(&allocate, &m_set) != vk::Result::eSuccess);
+	graphics.image_heap_layout = m_layout;
+	// Slot word 0 selects element 0, which stays a null image.
+	m_elements.push_back({.binding = {.layout = vk::ImageLayout::eShaderReadOnlyOptimal}});
+	m_writes.push_back(0u);
+	WriteDescriptors();
+}
+
+ImageHeap::~ImageHeap() {
+	for (const auto& memory: m_memories) Unwatch(*memory);
+	auto& graphics             = m_context.GetGraphics();
+	graphics.image_heap_layout = nullptr;
+	graphics.device.destroyDescriptorPool(m_pool, nullptr);
+	graphics.device.destroyDescriptorSetLayout(m_layout, nullptr);
+}
+
+bool ImageHeap::Acquire(const IR::ImageHeapRequest& request, IR::ImageHeapView& result) {
+	KYTY_PROFILER_FUNCTION();
+	const auto size = request.size & ~uint64_t {3};
+	if (size < MinTableBytes || request.count == 0u || request.step == 0u ||
+	    ((request.base | request.first | request.step) & 3u) != 0u) {
+		return false;
+	}
+	auto memory = std::ranges::find_if(m_memories, [&](const auto& candidate) {
+		return candidate->base == request.base && candidate->size == size;
+	});
+	if (memory == m_memories.end()) {
+		if (!m_context.IsMapped(request.base, size)) return false;
+		auto created   = std::make_unique<Memory>();
+		created->base  = request.base;
+		created->size  = size;
+		created->words.resize(size / 4u);
+		created->dead.resize(size / 4u);
+		created->dirty_pages.assign((Common::AlignUp(request.base + size, TRACKER_PAGE_SIZE) -
+		                             Common::AlignDown(request.base, TRACKER_PAGE_SIZE)) /
+		                                TRACKER_PAGE_SIZE,
+		                            uint8_t {1});
+		std::scoped_lock lock {m_mutex};
+		memory = m_memories.insert(m_memories.end(), std::move(created));
+	}
+	// A draw refreshes a table and a view takes its changes once, so all programs of the draw see
+	// one snapshot and one set of kinds.
+	if ((*memory)->draw != m_draw) {
+		if (!Refresh(**memory)) return false;
+		(*memory)->draw = m_draw;
+	}
+	auto&      views   = (*memory)->views;
+	const auto profile = IR::ImageHeapResource(*request.root, 0u);
+	auto       view    = std::ranges::find_if(views, [&](const auto& candidate) {
+		return candidate->step == request.step && candidate->first == request.first &&
+		       candidate->count == request.count && candidate->profile == profile;
+	});
+	if (view == views.end()) {
+		const auto epoch   = Image::heap_epoch.load(std::memory_order_relaxed);
+		auto       created = std::make_unique<View>(View {.memory        = **memory,
+		                                                  .step          = request.step,
+		                                                  .first         = request.first,
+		                                                  .count         = request.count,
+		                                                  .profile       = profile,
+		                                                  .resolve_epoch = epoch,
+		                                                  .layout_epoch  = epoch});
+		created->slots.resize(size / 4u + 1u);
+		created->buffer = std::make_unique<Buffer>(
+		    m_context.GetGraphics(), m_context.GetCommandScheduler(), MemoryUsage::DeviceLocal, 0,
+		    vk::BufferUsageFlagBits::eTransferDst | vk::BufferUsageFlagBits::eStorageBuffer |
+		        vk::BufferUsageFlagBits::eShaderDeviceAddress,
+		    size + 4u);
+		UpdateProbes(*created, 0u, size);
+		// The first upload also clears the slots no probe selects.
+		created->dirty_begin = 0;
+		created->dirty_end   = created->slots.size();
+		view = views.insert(views.end(), std::move(created));
+	} else {
+		// Empty after the first acquire in a draw, since the table refreshes once per draw.
+		UpdateProbes(**view, (*view)->changed_begin, (*view)->changed_end);
+		(*view)->changed_begin = (*view)->changed_end = 0;
+	}
+	if (std::ranges::find(m_active, view->get()) == m_active.end()) m_active.push_back(view->get());
+	const auto& selected = **view;
+	result = {.slots      = selected.buffer->BufferDeviceAddress(),
+	          .slot_count = static_cast<uint32_t>(selected.slots.size() - 1u),
+	          .kinds      = selected.kinds};
+	return true;
+}
+
+bool ImageHeap::Refresh(Memory& memory) {
+	// GPU writes leave no fault; reading the table downloads them.
+	const bool gpu_dirty = m_context.GetBufferCache().HasGpuDirtyBytes(memory.base, memory.size);
+	const auto pages     = memory.dirty_pages.size();
+	m_runs.clear();
+	{
+		std::scoped_lock lock {m_mutex};
+		// Watch before reading, so a write during the read faults the page again.
+		ForEachRun(
+		    0, pages, [&](size_t page) { return memory.dirty_pages[page] != 0u; },
+		    [&](size_t begin, size_t end) {
+			    std::fill(memory.dirty_pages.begin() + begin, memory.dirty_pages.begin() + end,
+			              uint8_t {0});
+			    Watch<true>(memory, begin, end);
+			    m_runs.emplace_back(begin, end);
+		    });
+	}
+	if (gpu_dirty) m_runs.assign(1u, {0u, pages});
+	for (size_t run = 0; run < m_runs.size(); ++run) {
+		const auto [address, end] = PageRange(memory.base, memory.size, m_runs[run].first,
+		                                      m_runs[run].second);
+		m_scratch.resize((end - address) / 4u);
+		if (!LibKernel::Memory::TryReadBufferBacking(address, m_scratch.data(), end - address)) {
+			std::scoped_lock lock {m_mutex};
+			for (; run < m_runs.size(); ++run) {
+				MarkDirty(memory, m_runs[run].first, m_runs[run].second);
+			}
+			return false;
+		}
+		const auto words = std::span(memory.words).subspan((address - memory.base) / 4u,
+		                                                   m_scratch.size());
+		size_t     first = 0;
+		size_t     last  = words.size();
+		while (first < last && words[first] == m_scratch[first]) ++first;
+		while (first < last && words[last - 1u] == m_scratch[last - 1u]) --last;
+		if (first == last) continue;
+		std::copy(m_scratch.begin() + first, m_scratch.begin() + last, words.begin() + first);
+		const auto offset = address - memory.base;
+		for (auto& view: memory.views) {
+			Widen(view->changed_begin, view->changed_end, offset + first * 4u, offset + last * 4u);
+		}
+	}
+	return true;
+}
+
+void ImageHeap::MarkDirty(Memory& memory, size_t begin, size_t end) {
+	ForEachRun(
+	    begin, end, [&](size_t page) { return memory.dirty_pages[page] == 0u; },
+	    [&](size_t first, size_t last) {
+		    std::fill(memory.dirty_pages.begin() + first, memory.dirty_pages.begin() + last,
+		              uint8_t {1});
+		    Watch<false>(memory, first, last);
+	    });
+}
+
+template <bool track>
+void ImageHeap::Watch(const Memory& memory, size_t begin, size_t end) {
+	const auto [address, limit] = PageRange(memory.base, memory.size, begin, end);
+	m_page_manager.UpdatePageWatchers<track>(address, limit - address);
+}
+
+void ImageHeap::Unwatch(const Memory& memory) {
+	ForEachRun(
+	    0, memory.dirty_pages.size(), [&](size_t page) { return memory.dirty_pages[page] == 0u; },
+	    [&](size_t begin, size_t end) { Watch<false>(memory, begin, end); });
+}
+
+void ImageHeap::InvalidateMemory(uint64_t address, uint64_t size) {
+	std::scoped_lock lock {m_mutex};
+	for (auto& memory: m_memories) {
+		const auto begin = std::max(address, memory->base);
+		const auto end   = std::min(address + size, memory->base + memory->size);
+		if (begin >= end) continue;
+		const auto origin = Common::AlignDown(memory->base, TRACKER_PAGE_SIZE);
+		MarkDirty(*memory, (begin - origin) / TRACKER_PAGE_SIZE,
+		          (end - 1u - origin) / TRACKER_PAGE_SIZE + 1u);
+	}
+}
+
+void ImageHeap::UpdateProbes(View& view, uint64_t begin, uint64_t end) {
+	// A probe reads its descriptor from first + probe * step.
+	const uint64_t bytes = view.profile.r128 ? 16u : 32u;
+	auto probe = begin < view.first + bytes ? 0u : (begin - view.first - bytes) / view.step + 1u;
+	for (; probe < view.count && view.first + probe * view.step < end; ++probe) {
+		UpdateProbe(view, probe);
+	}
+}
+
+void ImageHeap::UpdateProbe(View& view, uint64_t probe) {
+	const auto&     words  = view.memory.words;
+	const auto      offset = (view.first + probe * view.step) / 4u;
+	DescriptorValue value {.dword_count = 8u};
+	std::copy_n(words.begin() + offset,
+	            std::min<size_t>(view.profile.r128 ? 4u : 8u, words.size() - offset),
+	            value.dwords.begin());
+	// A buried word stays null in every view until its descriptor changes.
+	auto& dead = view.memory.dead[offset];
+	if (dead != 0u && dead != DescriptorHash {}(value)) dead = 0u;
+	const auto kind = dead != 0u ? 0u : IR::ImageHeapKind(view.profile, value);
+	Assign(view, offset, kind == 0u ? 0u : Intern(view, kind, value));
+}
+
+void ImageHeap::Assign(View& view, size_t slot, uint32_t entry) {
+	const auto previous = std::exchange(view.slots[slot], entry);
+	Release(view, previous);
+	if (previous != entry) Widen(view.dirty_begin, view.dirty_end, slot, slot + 1u);
+}
+
+void ImageHeap::Bury() {
+	if (m_unmapped.empty()) return;
+	for (const auto& memory: m_memories) {
+		for (const auto& view: memory->views) {
+			bool buried = false;
+			for (const auto& [value, index]: view->elements) {
+				const auto& binding = m_elements[index].binding;
+				if (binding.image_id && std::ranges::any_of(m_unmapped, [&](GuestRange range) {
+					    return ImageRangeOverlaps(binding.desc.info.data, range);
+				    })) {
+					m_elements[index].stale = buried = true;
+				}
+			}
+			if (!buried) continue;
+			for (uint64_t probe = 0; probe < view->count; ++probe) {
+				const auto  slot    = (view->first + probe * view->step) / 4u;
+				const auto& element = m_elements[view->slots[slot] & ElementMask];
+				if (!element.stale) continue;
+				memory->dead[slot] = DescriptorHash {}(element.value);
+				Assign(*view, slot, 0u);
+			}
+		}
+	}
+	m_unmapped.clear();
+}
+
+uint32_t ImageHeap::Intern(View& view, uint32_t kind, const DescriptorValue& value) {
+	const auto arm = static_cast<uint32_t>(std::ranges::find(view.kind_keys, kind) -
+	                                       view.kind_keys.begin());
+	if (arm == view.kind_keys.size()) {
+		view.kind_keys.push_back(kind);
+		view.kinds.push_back(value);
+	}
+	auto [found, inserted] = view.elements.try_emplace(value, 0u);
+	if (inserted) {
+		found->second    = Allocate();
+		auto& element    = m_elements[found->second];
+		element.value    = value;
+		element.kind     = kind;
+		view.pending.push_back(found->second);
+	}
+	++m_elements[found->second].refs;
+	return arm << IR::ImageHeapKindShift | found->second;
+}
+
+void ImageHeap::Release(View& view, uint32_t entry) {
+	const auto index = entry & ElementMask;
+	if (index == 0u || --m_elements[index].refs != 0u) return;
+	auto& element = m_elements[index];
+	if (const auto found = view.elements.find(element.value);
+	    found != view.elements.end() && found->second == index) {
+		view.elements.erase(found);
+	}
+	Unreference(element);
+	view.released.push_back(index);
+}
+
+void ImageHeap::Unreference(Element& element) {
+	if (auto* image = m_context.GetTextureCache().m_slot_images.try_get(element.binding.image_id)) {
+		image->heap_refs.fetch_sub(1u, std::memory_order_relaxed);
+		image->tick_accessed_last = m_context.GetCommandScheduler().CurrentTick();
+	}
+}
+
+uint32_t ImageHeap::Allocate() {
+	auto& scheduler = m_context.GetCommandScheduler();
+	while (!m_retired.empty() && scheduler.IsFree(m_retired.front().first)) {
+		m_free.push_back(m_retired.front().second);
+		m_retired.pop_front();
+	}
+	if (m_free.empty()) {
+		if (m_elements.size() == IR::ImageHeapCapacity) {
+			EXIT("image heaps exceed %u resident images\n", IR::ImageHeapCapacity);
+		}
+		m_elements.emplace_back();
+		return static_cast<uint32_t>(m_elements.size() - 1u);
+	}
+	const auto index = m_free.back();
+	m_free.pop_back();
+	m_elements[index] = {};
+	return index;
+}
+
+void ImageHeap::Retire(View& view) {
+	const auto tick = m_context.GetCommandScheduler().CurrentTick();
+	for (const auto index: view.released) m_retired.emplace_back(tick, index);
+	view.released.clear();
+}
+
+void ImageHeap::Rebind(RenderExecutor& executor) {
+	KYTY_PROFILER_FUNCTION();
+	auto&      textures = m_context.GetTextureCache();
+	const auto epoch    = Image::heap_epoch.load(std::memory_order_relaxed);
+	for (auto* view: m_active) {
+		bool       stale = false;
+		const auto mark  = [&](uint32_t index) { m_elements[index].stale = stale = true; };
+		if (view->resolve_epoch != epoch) {
+			view->resolve_epoch = epoch;
+			for (const auto& [value, index]: view->elements) {
+				const auto& binding = m_elements[index].binding;
+				if (!binding.image_id) continue;
+				const auto* image = textures.m_slot_images.try_get(binding.image_id);
+				if (image == nullptr || !image->registered || image->depth_id ||
+				    image->binding.needs_rebind) {
+					mark(index);
+				} else {
+					(void)textures.FindTexture(binding.image_id, binding.desc);
+				}
+			}
+		}
+		// Discovery applies the color metadata clears written since the last draw.
+		std::erase_if(view->dcc, [&](uint32_t index) {
+			const auto& element = m_elements[index];
+			if (element.refs == 0u || element.stale) return true;
+			if (executor.ResolveTexture(IR::ImageHeapResource(view->profile, element.kind),
+			                            element.value).image_id == element.binding.image_id) {
+				return false;
+			}
+			mark(index);
+			return true;
+		});
+		if (stale) {
+			// A changed image needs a new element; pending draws still read the old one. The new
+			// element keeps the old value, since a view takes table changes only when acquired.
+			std::erase_if(view->elements,
+			              [&](const auto& item) { return m_elements[item.second].stale; });
+			for (uint64_t probe = 0; probe < view->count; ++probe) {
+				const auto  slot    = (view->first + probe * view->step) / 4u;
+				const auto& element = m_elements[view->slots[slot] & ElementMask];
+				if (!element.stale) continue;
+				const auto value = element.value; // Intern may grow m_elements.
+				Assign(*view, slot, Intern(*view, element.kind, value));
+			}
+		}
+		for (const auto index: view->pending) {
+			if (m_elements[index].refs == 0u) continue;
+			auto binding = executor.ResolveTexture(
+			    IR::ImageHeapResource(view->profile, m_elements[index].kind),
+			    m_elements[index].value);
+			if (binding.image_id) {
+				auto& image = textures.GetImage(binding.image_id);
+				image.heap_refs.fetch_add(1u, std::memory_order_relaxed);
+				binding.layout     = image.ReadLayout();
+				binding.image_view = textures.FindTexture(binding.image_id, binding.desc);
+				if (binding.desc.info.metadata.kind == ImageMetadataKind::Dcc) {
+					view->dcc.push_back(index);
+				}
+				view->away.push_back(index);
+			}
+			m_elements[index].binding = std::move(binding);
+			m_writes.push_back(index);
+		}
+		view->pending.clear();
+	}
+	WriteDescriptors();
+}
+
+void ImageHeap::WriteDescriptors() {
+	if (m_writes.empty()) return;
+	std::vector<vk::DescriptorImageInfo> infos;
+	std::vector<vk::WriteDescriptorSet>  writes(m_writes.size());
+	infos.reserve(m_writes.size());
+	for (size_t write = 0; write < m_writes.size(); ++write) {
+		const auto& binding = m_elements[m_writes[write]].binding;
+		infos.push_back({nullptr, binding.image_view, binding.layout});
+		writes[write].dstSet          = m_set;
+		writes[write].dstArrayElement = m_writes[write];
+		writes[write].descriptorCount = 1;
+		writes[write].descriptorType  = vk::DescriptorType::eSampledImage;
+		writes[write].pImageInfo      = &infos.back();
+	}
+	m_context.GetGraphics().device.updateDescriptorSets(static_cast<uint32_t>(writes.size()),
+	                                                    writes.data(), 0, nullptr);
+	m_writes.clear();
+}
+
+void ImageHeap::Commit(CommandBuffer& buffer, vk::PipelineBindPoint bind_point,
+                       vk::PipelineLayout layout) {
+	KYTY_PROFILER_FUNCTION();
+	auto&      textures = m_context.GetTextureCache();
+	const auto epoch    = Image::heap_epoch.load(std::memory_order_relaxed);
+	for (auto* view: m_active) {
+		if (view->dirty_begin != view->dirty_end) {
+			const auto slots = std::span(view->slots).subspan(view->dirty_begin,
+			                                                  view->dirty_end - view->dirty_begin);
+			auto& stream = m_context.GetBufferCache().GetUtilityBuffer(MemoryUsage::Stream);
+			const auto offset = stream.Copy(slots.data(), slots.size_bytes(), sizeof(uint32_t));
+			view->buffer->CopyFrom(buffer, stream, offset, view->dirty_begin * sizeof(uint32_t),
+			                       slots.size_bytes());
+			view->dirty_begin = view->dirty_end = 0;
+		}
+		Retire(*view);
+		if (view->layout_epoch != epoch) {
+			view->layout_epoch = epoch;
+			view->away.clear();
+			for (const auto& [value, index]: view->elements) view->away.push_back(index);
+		}
+		// This draw's own bindings own the layouts of its attachments and storage images. An image
+		// this draw's discovery replaced lives until the tick ends, and its slots still select it.
+		std::erase_if(view->away, [&](uint32_t index) {
+			const auto& element = m_elements[index];
+			auto*       image   = textures.m_slot_images.try_get(element.binding.image_id);
+			if (element.refs == 0u || image == nullptr) return true;
+			if (image->binding.is_target || image->binding.shader_write ||
+			    image->binding.force_general) {
+				return false;
+			}
+			const auto& view_info = element.binding.desc.view_info;
+			image->Transit(image->ReadLayout(), vk::AccessFlagBits2::eShaderRead,
+			               ImageSubresourceRange {view_info.base_level, view_info.level_count,
+			                                      view_info.base_layer, view_info.layer_count},
+			               buffer.Handle());
+			return true;
+		});
+	}
+	m_active.clear();
+	buffer.Handle().bindDescriptorSets(bind_point, layout, IR::ImageHeapSet, 1, &m_set, 0,
+	                                   nullptr);
+}
+
+void ImageHeap::UnmapMemory(uint64_t address, uint64_t size) {
+	// Descriptors of textures in the range stay null even once it is mapped again.
+	if (!m_unmapped.empty() && m_unmapped.back().End() == address) {
+		m_unmapped.back().size += size;
+	} else {
+		m_unmapped.push_back({address, size});
+	}
+	auto&           scheduler = m_context.GetCommandScheduler();
+	std::scoped_lock lock {m_mutex};
+	std::erase_if(m_memories, [&](const std::unique_ptr<Memory>& memory) {
+		if (address >= memory->base + memory->size || memory->base >= address + size) {
+			return false;
+		}
+		Unwatch(*memory);
+		for (auto& view: memory->views) {
+			std::erase(m_active, view.get());
+			for (const auto& [value, index]: view->elements) {
+				Unreference(m_elements[index]);
+				view->released.push_back(index);
+			}
+			Retire(*view);
+			if (scheduler.Active()) {
+				scheduler.DeferOperation([buffer = std::move(view->buffer)] {});
+			}
+		}
+		return true;
+	});
+}
+
+} // namespace Libs::Graphics

src/graphics/host_gpu/renderer/pipeline/imageHeap.h

@@ -0,0 +1,138 @@
+#ifndef EMULATOR_SRC_GRAPHICS_HOST_GPU_RENDERER_IMAGEHEAP_H_
+#define EMULATOR_SRC_GRAPHICS_HOST_GPU_RENDERER_IMAGEHEAP_H_
+
+#include "common/common.h"
+#include "graphics/host_gpu/renderer/pipeline/descriptors.h"
+#include "graphics/shader/recompiler/ir/passes/SrtWalker.h"
+
+#include <deque>
+#include <memory>
+#include <mutex>
+#include <unordered_map>
+#include <utility>
+#include <vector>
+
+namespace Libs::Graphics {
+
+class Buffer;
+class CommandBuffer;
+class PageManager;
+class RenderContext;
+class RenderExecutor;
+struct ImageHeapTestAccess;
+
+// Guest image tables that shaders index by GPU-computed offsets, kept resident in one bindless
+// sampled-image array. Each view of a table owns device slot words, one per table dword, that
+// select an array element. CPU writes to a table fault in its snapshot, which each draw refreshes
+// once; a view takes the changes when a draw first acquires it. Changes to the images behind
+// elements bump Image::heap_epoch.
+class ImageHeap final: public ShaderRecompiler::IR::ImageHeapSource {
+public:
+	ImageHeap(RenderContext& context, PageManager& page_manager);
+	~ImageHeap();
+	KYTY_CLASS_NO_COPY(ImageHeap);
+
+	// Starts a draw or dispatch, before its programs acquire views.
+	void BeginDraw() {
+		++m_draw;
+		Bury();
+	}
+	bool Acquire(const ShaderRecompiler::IR::ImageHeapRequest& request,
+	             ShaderRecompiler::IR::ImageHeapView&          result) override;
+	// Image discovery: resolves new elements and rediscovers changed images.
+	void Rebind(RenderExecutor& executor);
+	// Recording: uploads slots, moves resident images to their read layout and binds the array.
+	void Commit(CommandBuffer& buffer, vk::PipelineBindPoint bind_point, vk::PipelineLayout layout);
+	// CPU write fault, on any thread.
+	void InvalidateMemory(uint64_t address, uint64_t size);
+	void UnmapMemory(uint64_t address, uint64_t size);
+
+private:
+	using DescriptorValue = ShaderRecompiler::IR::DescriptorValue;
+
+	struct DescriptorHash {
+		size_t operator()(const DescriptorValue& value) const noexcept;
+	};
+
+	struct Element {
+		DescriptorValue value;
+		TextureBinding  binding;
+		uint32_t        kind  = 0;
+		uint32_t        refs  = 0; // Slots of the owning view that select it.
+		bool            stale = false;
+	};
+
+	struct Memory;
+
+	struct View {
+		Memory&                             memory;
+		uint64_t                            step  = 0;
+		uint64_t                            first = 0;
+		uint64_t                            count = 0;
+		ShaderRecompiler::IR::ImageResource profile;
+		std::vector<uint32_t>               slots; // Plus a trailing null slot.
+		std::vector<uint32_t>               kind_keys;
+		std::vector<DescriptorValue>        kinds; // One representative per kind_keys entry.
+		std::unordered_map<DescriptorValue, uint32_t, DescriptorHash> elements;
+		std::vector<uint32_t>   pending;  // Not yet resolved.
+		std::vector<uint32_t>   released; // No slot selects them after the next upload.
+		std::vector<uint32_t>   away;     // Not yet moved to the read layout.
+		std::vector<uint32_t>   dcc;      // Color metadata clears apply on each discovery.
+		std::unique_ptr<Buffer> buffer;
+		size_t                  dirty_begin   = 0; // Slots to upload.
+		size_t                  dirty_end     = 0;
+		size_t                  changed_begin = 0; // Table bytes changed since it was probed.
+		size_t                  changed_end   = 0;
+		uint64_t                resolve_epoch = 0;
+		uint64_t                layout_epoch  = 0;
+	};
+
+	struct Memory {
+		uint64_t                           base = 0;
+		uint64_t                           size = 0;
+		std::vector<uint32_t>              words;
+		std::vector<uint8_t>               dirty_pages; // Unwatched, so unread since a write.
+		std::vector<uint64_t>              dead;        // Per word, a buried descriptor's hash.
+		uint64_t                           draw = 0;    // Last draw that refreshed it.
+		std::vector<std::unique_ptr<View>> views;
+	};
+
+	[[nodiscard]] bool Refresh(Memory& memory);
+	void               MarkDirty(Memory& memory, size_t begin, size_t end);
+	template <bool track>
+	void     Watch(const Memory& memory, size_t begin, size_t end);
+	void     Unwatch(const Memory& memory);
+	void     UpdateProbes(View& view, uint64_t begin, uint64_t end);
+	void     UpdateProbe(View& view, uint64_t probe);
+	void     Assign(View& view, size_t slot, uint32_t entry);
+	void     Bury();
+	uint32_t Intern(View& view, uint32_t kind, const DescriptorValue& value);
+	void     Release(View& view, uint32_t entry);
+	void     Unreference(Element& element);
+	uint32_t Allocate();
+	void     Retire(View& view);
+	void     WriteDescriptors();
+
+	RenderContext&                       m_context;
+	PageManager&                         m_page_manager;
+	std::mutex                           m_mutex; // Fault-visible Memory state and m_memories.
+	std::vector<std::unique_ptr<Memory>> m_memories;
+	std::vector<View*>                   m_active; // Acquired since the last commit.
+	uint64_t                             m_draw = 1;
+	std::vector<Element>                 m_elements;
+	std::vector<uint32_t>                m_free;
+	std::deque<std::pair<uint64_t, uint32_t>> m_retired; // Tick, element.
+	std::vector<std::pair<size_t, size_t>>    m_runs;
+	std::vector<uint32_t>                     m_scratch;
+	std::vector<uint32_t>                     m_writes; // Elements whose descriptors changed.
+	std::vector<GuestRange>                   m_unmapped; // Since the last Bury.
+	vk::DescriptorSetLayout                   m_layout = nullptr;
+	vk::DescriptorPool                        m_pool   = nullptr;
+	vk::DescriptorSet                         m_set    = nullptr;
+
+	friend struct ImageHeapTestAccess;
+};
+
+} // namespace Libs::Graphics
+
+#endif // EMULATOR_SRC_GRAPHICS_HOST_GPU_RENDERER_IMAGEHEAP_H_

src/graphics/host_gpu/renderer/pipeline/pipelineCache.cpp

@@ -308,7 +308,7 @@ struct PipelineCache::ProgramCache {
 		auto result = ShaderRecompiler::CompileProgram(std::move(translated), options,
 		                                               specialization, push_data_start_dword);
 		if (!ValidateShaderSpirv(options.dump_label, options.shader_hash, result.spirv,
-		                         options.maintenance8_enabled)) {
+		                         options.profile->maintenance8)) {
 			DumpShaderSpirv(stage_name, options.shader_hash, result.spirv);
 			EXIT("%s failed hash=0x%016" PRIx64 ": SPIR-V validation failed\n", options.dump_label,
 			     options.shader_hash);
@@ -353,6 +353,7 @@ struct PipelineCache::ProgramCache {
 		    .user_data                  = user_data,
 		    .shader_base                = params.Base(),
 		    .read_specialization_memory = ReadShaderGuestMemory,
+		    .image_heap                 = image_heap,
 		};
 		if constexpr (std::is_same_v<InputInfo, ShaderComputeInputInfo>) {
 			runtime.workgroup_counts = input_info.workgroup_counts;
@@ -398,7 +399,7 @@ struct PipelineCache::ProgramCache {
 		}
 		ShaderRecompiler::CompileOptions options;
 		options.stage       = stage;
-		options.maintenance8_enabled = graphics.maintenance8_enabled;
+		options.profile     = &profile;
 		options.shader_hash = params.hash;
 		options.user_data   = user_data;
 		options.back_code      = params.back_code;
@@ -480,7 +481,13 @@ struct PipelineCache::ProgramCache {
 		return permutation.handle;
 	}
 
-	explicit ProgramCache(GraphicContext& graphics): graphics(graphics) {
+	ProgramCache(GraphicContext& graphics, ShaderRecompiler::IR::ImageHeapSource* image_heap)
+	    : graphics(graphics), image_heap(image_heap),
+	      // AMD GPUs (vendor 0x1002) run NMin/NMax as v_min/v_max, which give the guest's NaN and
+	      // signed-zero results; GLSL.std.450 alone does not order -0 below +0.
+	      profile {.maintenance8 = graphics.maintenance8_enabled,
+		           .native_float_min_max =
+		               graphics.physical_device_properties.vendorID == 0x1002u} {
 		lookup_key.static_state.reserve(MaxStaticKeyWords);
 	}
 	~ProgramCache() {
@@ -495,11 +502,14 @@ struct PipelineCache::ProgramCache {
 	std::unordered_map<ProgramKey, SourceEntry, ProgramKeyHash> programs;
 	ProgramKey                                                  lookup_key;
 	GraphicContext&                                             graphics;
+	ShaderRecompiler::IR::ImageHeapSource*                      image_heap;
+	const ShaderRecompiler::Spirv::Profile                      profile;
 	uint64_t                                                    next_shader_id = 0;
 };
 
-PipelineCache::PipelineCache(GraphicContext& graphics)
-    : m_graphics(graphics), m_program_cache(std::make_unique<ProgramCache>(graphics)),
+PipelineCache::PipelineCache(GraphicContext&                         graphics,
+                             ShaderRecompiler::IR::ImageHeapSource* image_heap)
+    : m_graphics(graphics), m_program_cache(std::make_unique<ProgramCache>(graphics, image_heap)),
       m_next_driver_cache_save(std::chrono::steady_clock::now() + DriverCacheSaveInterval) {
 	EXIT_NOT_IMPLEMENTED(!Common::Thread::IsMainThread());
 	InitializeDriverCache();

src/graphics/host_gpu/renderer/pipeline/pipelineCache.h

@@ -30,6 +30,10 @@ class UserConfig;
 struct ComputeShaderInfo;
 } // namespace HW
 
+namespace ShaderRecompiler::IR {
+class ImageHeapSource;
+} // namespace ShaderRecompiler::IR
+
 #pragma pack(push, 1)
 
 struct PipelineStaticParameters {
@@ -104,7 +108,7 @@ struct ShaderProgram {
 // The owning renderer serializes access, including saves while the GPU is running.
 class PipelineCache {
 public:
-	explicit PipelineCache(GraphicContext& graphics);
+	PipelineCache(GraphicContext& graphics, ShaderRecompiler::IR::ImageHeapSource* image_heap);
 	~PipelineCache();
 	KYTY_CLASS_NO_COPY(PipelineCache);
 	void Save();
@@ -114,6 +118,7 @@ public:
 		vk::Pipeline            pipeline              = nullptr;
 		vk::DescriptorSetLayout descriptor_set_layout = nullptr;
 		bool                    uses_push_descriptors = false;
+		bool                    uses_image_heap       = false;
 	};
 
 	struct GraphicsPrograms {

src/graphics/host_gpu/renderer/pipeline/shaderResourceBarrier.cpp

@@ -52,6 +52,7 @@ vk::MemoryBarrier MakeShaderWriteDependency() {
 	barrier.dstAccessMask = vk::AccessFlagBits::eShaderRead | vk::AccessFlagBits::eShaderWrite |
 	                        vk::AccessFlagBits::eVertexAttributeRead |
 	                        vk::AccessFlagBits::eIndexRead | vk::AccessFlagBits::eUniformRead |
+	                        vk::AccessFlagBits::eIndirectCommandRead |
 	                        vk::AccessFlagBits::eTransferRead | vk::AccessFlagBits::eTransferWrite |
 	                        vk::AccessFlagBits::eColorAttachmentRead |
 	                        vk::AccessFlagBits::eColorAttachmentWrite;

src/graphics/host_gpu/renderer/pipeline/shaders.cpp

@@ -473,20 +473,23 @@ void CreatePipelineInternal(GraphicContext& graphics, PipelineCache::Pipeline& p
 	for (const auto& stage: vertex_info) {
 		const auto native_stage = NativeShaderStage(stage.logical_stage);
 		AddLayoutBindings(descriptor_bindings, *stage.stage.program, native_stage);
+		pipeline.uses_image_heap |= stage.stage.program->info.uses_image_heap;
 		graphics_stages |= native_stage;
 	}
 	if (ps_active) {
 		EXIT_IF(!ps_input_info->stage);
 		AddLayoutBindings(descriptor_bindings, *ps_input_info->stage.program,
 		                  vk::ShaderStageFlagBits::eFragment);
+		pipeline.uses_image_heap |= ps_input_info->stage.program->info.uses_image_heap;
 	}
 	CreateDescriptorLayout(graphics, pipeline, descriptor_bindings);
 	const vk::PushConstantRange push_constants {graphics_stages, 0,
 	                                            ShaderRecompiler::IR::NativePushConstantSize};
+	const std::array set_layouts {pipeline.descriptor_set_layout, graphics.image_heap_layout};
 
 	vk::PipelineLayoutCreateInfo pipeline_layout_info {};
-	pipeline_layout_info.setLayoutCount         = 1;
-	pipeline_layout_info.pSetLayouts            = &pipeline.descriptor_set_layout;
+	pipeline_layout_info.setLayoutCount         = pipeline.uses_image_heap ? 2u : 1u;
+	pipeline_layout_info.pSetLayouts            = set_layouts.data();
 	pipeline_layout_info.pushConstantRangeCount = 1;
 	pipeline_layout_info.pPushConstantRanges    = &push_constants;
 
@@ -615,13 +618,15 @@ void CreatePipelineInternal(GraphicContext& graphics, PipelineCache::Pipeline& p
 	std::vector<vk::DescriptorSetLayoutBinding> descriptor_bindings;
 	AddLayoutBindings(descriptor_bindings, *input_info.stage.program,
 	                  vk::ShaderStageFlagBits::eCompute);
+	pipeline.uses_image_heap = input_info.stage.program->info.uses_image_heap;
 	CreateDescriptorLayout(graphics, pipeline, descriptor_bindings);
 	const vk::PushConstantRange push_constants {vk::ShaderStageFlagBits::eCompute, 0,
 	                                            ShaderRecompiler::IR::NativePushConstantSize};
+	const std::array set_layouts {pipeline.descriptor_set_layout, graphics.image_heap_layout};
 
 	vk::PipelineLayoutCreateInfo pipeline_layout_info {};
-	pipeline_layout_info.setLayoutCount         = 1;
-	pipeline_layout_info.pSetLayouts            = &pipeline.descriptor_set_layout;
+	pipeline_layout_info.setLayoutCount         = pipeline.uses_image_heap ? 2u : 1u;
+	pipeline_layout_info.pSetLayouts            = set_layouts.data();
 	pipeline_layout_info.pushConstantRangeCount = 1;
 	pipeline_layout_info.pPushConstantRanges    = &push_constants;
 

src/graphics/host_gpu/renderer/render.h

@@ -48,6 +48,13 @@ enum class DrawOffsetSource : uint8_t {
 	IndirectArgs,
 };
 
+// Nonzero address: the GPU reads count Vulkan indirect draw records, stride bytes apart.
+struct DrawIndirectArgs {
+	uint64_t address = 0;
+	uint32_t count   = 0;
+	uint32_t stride  = 0;
+};
+
 struct DrawIndexArgs {
 	uint32_t         index_count                = 0;
 	const void*      index_addr                 = nullptr;
@@ -57,6 +64,8 @@ struct DrawIndexArgs {
 	uint32_t         first_instance             = 0;
 	DrawOffsetSource offset_source              = DrawOffsetSource::DrawState;
 	uint32_t         render_target_slice_offset = 0;
+	// With indirect records, index_count is INDEX_BUFFER_SIZE.
+	DrawIndirectArgs indirect;
 };
 
 struct DrawAutoArgs {
@@ -66,6 +75,7 @@ struct DrawAutoArgs {
 	uint32_t         first_instance             = 0;
 	DrawOffsetSource offset_source              = DrawOffsetSource::DrawState;
 	uint32_t         render_target_slice_offset = 0;
+	DrawIndirectArgs indirect;
 };
 
 struct SubmitInfo {
@@ -161,18 +171,19 @@ public:
 	void CommitBindings(CommandBuffer& buffer, vk::PipelineBindPoint pipeline_bind_point,
 	                    const PipelineCache::Pipeline&     pipeline,
 	                    std::span<PreparedBindings* const> bindings);
+	[[nodiscard]] TextureBinding ResolveTexture(const ShaderRecompiler::IR::ImageResource&   resource,
+	                                            const ShaderRecompiler::IR::DescriptorValue& value);
 
 private:
-	void DrawIndex(uint64_t submit_id, CommandBuffer& buffer, const DrawIndexArgs& args);
-	void DrawAuto(uint64_t submit_id, CommandBuffer& buffer, const DrawAutoArgs& args);
+	// False, with nothing drawn, when the draw's indirect records need CPU counts.
+	bool DrawIndex(uint64_t submit_id, CommandBuffer& buffer, const DrawIndexArgs& args);
+	bool DrawAuto(uint64_t submit_id, CommandBuffer& buffer, const DrawAutoArgs& args);
 
 	struct GraphicsBindings {
 		std::array<PreparedBindings, 3> vertex;
 		std::optional<PreparedBindings> pixel;
 	};
 
-	[[nodiscard]] TextureBinding ResolveTexture(const ShaderRecompiler::IR::ImageResource& resource,
-	                                            const ShaderRecompiler::IR::DescriptorValue& value);
 	void PrepareGraphicsBindings(std::span<PreparedBindings* const> stages,
 	                             std::span<RenderColorInfo> colors);
 	void ResolveRenderColorTarget(CommandBuffer& buffer, RenderColorInfo& target,

src/graphics/host_gpu/renderer/renderCompute.cpp

@@ -311,6 +311,7 @@ void RenderExecutor::DispatchDirect(uint64_t submit_id, CommandBuffer& buffer,
 			input_info.workgroup_counts[axis] = threads / size + (threads % size != 0u);
 		}
 	}
+	m_context.GetImageHeap().BeginDraw();
 	const auto compute_program =
 	    m_context.GetPipelineCache().GetComputeProgram(cs_regs, sh_regs, input_info);
 	if (!compute_program) return;
@@ -363,6 +364,9 @@ void RenderExecutor::DispatchDirect(uint64_t submit_id, CommandBuffer& buffer,
 	if (program.info.uses_dma) {
 		m_context.PrepareBda();
 	}
+	if (program.info.uses_image_heap) {
+		m_context.GetImageHeap().Rebind(*this);
+	}
 	RebindImages(bindings);
 	BindSharedMemory(m_context, input_info, bindings);
 	RebindBuffers(bindings);
@@ -497,6 +501,7 @@ void RenderExecutor::DispatchIndirect(uint64_t submit_id, CommandBuffer& buffer,
 	ShaderComputeInputInfo input_info {};
 	input_info.async_compute = async_compute;
 	input_info.indirect_args_address = args_addr;
+	m_context.GetImageHeap().BeginDraw();
 	const auto compute_program = m_context.GetPipelineCache().GetComputeProgram(
 	    cs_regs, buffer.GetRegisters().GetShaderRegisters(), input_info);
 	if (!compute_program) return;
@@ -511,6 +516,9 @@ void RenderExecutor::DispatchIndirect(uint64_t submit_id, CommandBuffer& buffer,
 		m_context.PrepareBda();
 	}
 	BindSharedMemory(m_context, input_info, bindings, args_addr);
+	if (program.info.uses_image_heap) {
+		m_context.GetImageHeap().Rebind(*this);
+	}
 	RebindImages(bindings);
 	// Acquiring arguments can merge cache buffers; finalize shader bindings afterward.
 	const auto [args_buffer, args_offset] = m_context.GetBufferCache().ObtainBuffer(

src/graphics/host_gpu/renderer/renderContext.cpp

@@ -13,9 +13,10 @@ namespace Libs::Graphics {
 RenderContext::RenderContext(GraphicContext& graphics)
     : m_graphics(graphics), m_render_executor(*this), m_command_scheduler(*this, graphics),
       m_descriptor_heap(graphics, m_command_scheduler.GetMasterSemaphore()),
-      m_pipeline_cache(graphics), m_sampler_cache(graphics),
+      m_pipeline_cache(graphics, &m_image_heap), m_sampler_cache(graphics),
       m_buffer_cache(graphics, m_command_scheduler, m_page_manager, m_texture_cache),
-      m_texture_cache(graphics, m_command_scheduler, m_page_manager, m_buffer_cache) {
+      m_texture_cache(graphics, m_command_scheduler, m_page_manager, m_buffer_cache),
+      m_image_heap(*this, m_page_manager) {
 	EXIT_NOT_IMPLEMENTED(!Common::Thread::IsMainThread());
 }
 
@@ -64,6 +65,7 @@ bool RenderContext::HandleFault(PageFaultAccess access, uint64_t fault_vaddr) no
 	if (access == PageFaultAccess::Write) {
 		m_buffer_cache.InvalidateMemory(fault_vaddr, fault_size);
 		m_texture_cache.InvalidateMemory(fault_vaddr, fault_size);
+		m_image_heap.InvalidateMemory(fault_vaddr, fault_size);
 	} else {
 		m_buffer_cache.ReadMemory(fault_vaddr, fault_size);
 	}
@@ -76,6 +78,7 @@ bool RenderContext::InvalidateMemory(uint64_t vaddr, uint64_t size) {
 	}
 	m_buffer_cache.InvalidateMemory(vaddr, size);
 	m_texture_cache.InvalidateMemory(vaddr, size);
+	m_image_heap.InvalidateMemory(vaddr, size);
 	return true;
 }
 
@@ -112,6 +115,7 @@ void RenderContext::UnmapMemory(uint64_t vaddr, uint64_t size) {
 		}
 		m_buffer_cache.InvalidateMemory(vaddr, size);
 		m_texture_cache.UnmapMemory(vaddr, size);
+		m_image_heap.UnmapMemory(vaddr, size);
 		std::lock_guard lock(m_mapped_ranges_mutex);
 		m_mapped_ranges.Subtract(vaddr, size);
 	};

src/graphics/host_gpu/renderer/renderContext.h

@@ -12,6 +12,7 @@
 #include "graphics/host_gpu/renderer/cache/textureCache.h"
 #include "graphics/host_gpu/renderer/commandScheduler.h"
 #include "graphics/host_gpu/renderer/pipeline/descriptorHeap.h"
+#include "graphics/host_gpu/renderer/pipeline/imageHeap.h"
 #include "graphics/host_gpu/renderer/pipeline/pipelineCache.h"
 #include "kernel/eventQueue.h"
 
@@ -43,6 +44,7 @@ public:
 	CommandScheduler&   GetCommandScheduler() { return m_command_scheduler; }
 	PipelineCache&      GetPipelineCache() { return m_pipeline_cache; }
 	DescriptorHeap&     GetDescriptorHeap() { return m_descriptor_heap; }
+	ImageHeap&          GetImageHeap() { return m_image_heap; }
 	SamplerCache&       GetSamplerCache() { return m_sampler_cache; }
 	BufferCache&        GetBufferCache() { return m_buffer_cache; }
 	TextureCache&       GetTextureCache() { return m_texture_cache; }
@@ -76,6 +78,7 @@ private:
 	PageManager               m_page_manager;
 	BufferCache               m_buffer_cache;
 	TextureCache              m_texture_cache;
+	ImageHeap                 m_image_heap;
 	mutable std::shared_mutex m_mapped_ranges_mutex;
 	RangeSet                  m_mapped_ranges;
 	std::unique_ptr<GuestGpu> m_gpu;

src/graphics/host_gpu/renderer/renderDraw.cpp

@@ -631,10 +631,11 @@ bool RenderExecutor::ConsumeMetadataColorOperation(CommandBuffer& buffer,
 	       mode == static_cast<uint8_t>(CbColorMode::DccDecompress);
 }
 
+// vertex_offset is the first vertex of non-indexed draws.
 struct DrawEmitInfo {
-	int32_t  vertex_offset = 0;
-	uint32_t first_vertex  = 0;
-	uint32_t first_instance = 0;
+	int32_t          vertex_offset  = 0;
+	uint32_t         first_instance = 0;
+	DrawIndirectArgs indirect;
 };
 
 struct DrawIndexBufferSource {
@@ -892,6 +893,7 @@ static void RefreshShaders(CommandBuffer& buffer, const DrawCallInfo& draw,
 		}
 	}
 	auto& pipeline_cache = buffer.GetContext().GetPipelineCache();
+	buffer.GetContext().GetImageHeap().BeginDraw();
 	if (draw.IsIndexed()) {
 		LogDrawPhase(draw.Name(), "GetGraphicsPrograms");
 	}
@@ -1043,8 +1045,8 @@ static void EmitDrawPrimitives(const HW::UserConfig& ucfg, vk::CommandBuffer vk_
 				vk_buffer.drawIndexed(draw.index_count, draw.instance_count, 0, emit.vertex_offset,
 				                      emit.first_instance);
 			} else {
-				vk_buffer.draw(draw.index_count, draw.instance_count, emit.first_vertex,
-				               emit.first_instance);
+				vk_buffer.draw(draw.index_count, draw.instance_count,
+				               static_cast<uint32_t>(emit.vertex_offset), emit.first_instance);
 			}
 			break;
 		case Prospero::PrimitiveType::kQuadListLegacy:
@@ -1054,7 +1056,8 @@ static void EmitDrawPrimitives(const HW::UserConfig& ucfg, vk::CommandBuffer vk_
 					vk_buffer.drawIndexed(4, draw.instance_count, i, emit.vertex_offset,
 					                      emit.first_instance);
 				} else {
-					vk_buffer.draw(4, draw.instance_count, i + emit.first_vertex,
+					vk_buffer.draw(4, draw.instance_count,
+					               i + static_cast<uint32_t>(emit.vertex_offset),
 					               emit.first_instance);
 				}
 			}
@@ -1075,7 +1078,7 @@ bool RenderExecutor::CopyDepthToColor(CommandBuffer& buffer, const DrawCallInfo&
 	EXIT_NOT_IMPLEMENTED(!m_context.GetGraphics().maintenance8_enabled ||
 	                     !Prospero::IsRectList(ucfg.GetPrimType()) ||
 	                     draw.IsIndexed() || draw.index_count != 3 || draw.instance_count != 1 ||
-	                     emit.first_vertex != 0 || emit.first_instance != 0 ||
+	                     emit.vertex_offset != 0 || emit.first_instance != 0 ||
 	                     state.color_count != 1 || color.target_slot != 0 || !depth.image_id ||
 	                     color.desc.info.samples != 1 ||
 	                     depth.desc.info.extent.width != color.Extent().width ||
@@ -1189,6 +1192,14 @@ void RenderExecutor::ExecutePreparedDraw(uint64_t submit_id, CommandBuffer& buff
 		vertex_bindings = AcquireVertexBuffers(buffer, state.vertex_info[0]);
 		index_binding   = PrepareIndexBuffer(buffer, index_source);
 	}
+	std::pair<Buffer*, uint64_t> args_buffer {};
+	if (emit.indirect.address != 0) {
+		const auto size = static_cast<uint64_t>(emit.indirect.count - 1) * emit.indirect.stride +
+		                  (draw.IsIndexed() ? sizeof(vk::DrawIndexedIndirectCommand)
+		                                    : sizeof(vk::DrawIndirectCommand));
+		args_buffer = m_context.GetBufferCache().ObtainBuffer(emit.indirect.address, size, false);
+		EXIT_IF((args_buffer.second & 3u) != 0);
+	}
 	if (draw.IsIndexed()) {
 		LogDrawPhase(draw.Name(), "CreatePipeline");
 	}
@@ -1225,12 +1236,12 @@ void RenderExecutor::ExecutePreparedDraw(uint64_t submit_id, CommandBuffer& buff
 	if (draw_dwords != 0) {
 		std::array<uint32_t, PushData::MeshDrawDwordCount> draw_data {};
 		if (mesh_active) {
-			draw_data = {
-			    draw.index_count,
-			    draw.IsIndexed() ? static_cast<uint32_t>(emit.vertex_offset) : emit.first_vertex,
-			    emit.first_instance, index_source.guest_element_size,
-			    static_cast<uint32_t>(index_source.address),
-			    static_cast<uint32_t>(index_source.address >> 32u)};
+			draw_data = {draw.index_count,
+			             static_cast<uint32_t>(emit.vertex_offset),
+			             emit.first_instance,
+			             index_source.guest_element_size,
+			             static_cast<uint32_t>(index_source.address),
+			             static_cast<uint32_t>(index_source.address >> 32u)};
 		} else {
 			draw_data[0] = buffer.GetUserConfig().GetObjectId();
 		}
@@ -1275,8 +1286,14 @@ void RenderExecutor::ExecutePreparedDraw(uint64_t submit_id, CommandBuffer& buff
 	}
 	if (mesh_active) {
 		vk_buffer.drawMeshTasksEXT(mesh_groups, draw.instance_count, 1);
-	} else {
+	} else if (args_buffer.first == nullptr) {
 		EmitDrawPrimitives(ucfg, vk_buffer, draw, emit);
+	} else if (draw.IsIndexed()) {
+		vk_buffer.drawIndexedIndirect(args_buffer.first->Handle(), args_buffer.second,
+		                              emit.indirect.count, emit.indirect.stride);
+	} else {
+		vk_buffer.drawIndirect(args_buffer.first->Handle(), args_buffer.second,
+		                       emit.indirect.count, emit.indirect.stride);
 	}
 	Common::DebugCounters::Add(Common::DebugCounters::Counter::Draws);
 	Common::DebugCounters::Add(Common::DebugCounters::Counter::DrawInstances, draw.instance_count);
@@ -1294,7 +1311,30 @@ void RenderExecutor::ExecutePreparedDraw(uint64_t submit_id, CommandBuffer& buff
 	}
 }
 
-void RenderExecutor::DrawIndex(uint64_t submit_id, CommandBuffer& buffer,
+// Vulkan offsets for a prepared draw, or nullopt when its indirect records need CPU counts, as
+// mesh and quad-list draws do. Indirect records carry their own offsets.
+static std::optional<DrawEmitInfo> ResolveDrawEmit(const HW::UserConfig&        ucfg,
+                                                   const ShaderVertexInputInfo& vs,
+                                                   DrawOffsetSource             source,
+                                                   int32_t                      first_vertex,
+                                                   uint32_t                     first_instance,
+                                                   const DrawIndirectArgs&      indirect) {
+	if (indirect.address != 0) {
+		if (vs.stage.program->stage == ShaderType::Mesh ||
+		    ucfg.GetPrimType() == Prospero::PrimitiveType::kQuadListLegacy) {
+			return std::nullopt;
+		}
+		return DrawEmitInfo {.indirect = indirect};
+	}
+	if (source == DrawOffsetSource::IndirectArgs) {
+		// Native fetches consume the patched SGPRs; rewritten fetches need Vulkan offsets.
+		return vs.fetch_embedded ? DrawEmitInfo {first_vertex, first_instance} : DrawEmitInfo {};
+	}
+	const auto [vertex_offset, instance_offset] = ResolveDrawOffsets(ucfg.GetIndexOffset(), vs);
+	return DrawEmitInfo {vertex_offset + first_vertex, instance_offset};
+}
+
+bool RenderExecutor::DrawIndex(uint64_t submit_id, CommandBuffer& buffer,
                                const DrawIndexArgs& args) {
 	KYTY_PROFILER_FUNCTION();
 
@@ -1309,19 +1349,19 @@ void RenderExecutor::DrawIndex(uint64_t submit_id, CommandBuffer& buffer,
 	                    reinterpret_cast<uint64_t>(args.index_addr));
 
 	Common::LockGuard lock(m_context.GetMutex());
-	if (args.index_count == 0 || args.instance_count == 0) {
-		return;
+	if (args.index_count == 0 || (args.instance_count == 0 && args.indirect.address == 0)) {
+		return true;
 	}
 
 	if (ConsumeMetadataColorOperation(buffer, args.render_target_slice_offset) ||
 	    DepthStencilCopy(buffer) ||
 	    ResolveColorTargets(buffer, args.render_target_slice_offset)) {
 		ResetBindings();
-		return;
+		return true;
 	}
 
 	if (!DrawHasValidVertexShader(sh_ctx)) {
-		return;
+		return true;
 	}
 
 	if (graphics_debug_dump_enabled()) {
@@ -1347,7 +1387,7 @@ void RenderExecutor::DrawIndex(uint64_t submit_id, CommandBuffer& buffer,
 
 	vk::PrimitiveTopology topology = vk::PrimitiveTopology::ePointList;
 	if (!GetDrawTopology(ucfg, topology)) {
-		return;
+		return true;
 	}
 
 	DrawIndexBufferSource index_source {};
@@ -1386,30 +1426,23 @@ void RenderExecutor::DrawIndex(uint64_t submit_id, CommandBuffer& buffer,
 	DrawRenderState state {};
 	if (!PrepareDrawRenderState(buffer, draw, args.render_target_slice_offset, state)) {
 		ResetBindings();
-		return;
+		return true;
 	}
 
 	LogDrawStateIfNeeded(buffer, draw, state, args.index_type_and_size,
 	                     args.index_addr);
-
-	const bool indirect = args.offset_source == DrawOffsetSource::IndirectArgs;
-	const auto [vertex_offset, instance_offset] =
-	    indirect ? std::pair<int32_t, uint32_t> {0, args.first_instance}
-	             : ResolveDrawOffsets(ucfg.GetIndexOffset(), state.vertex_info[0]);
-
-	DrawEmitInfo emit {};
-	// Native fetches consume the patched SGPRs; rewritten fetches need Vulkan offsets.
-	const bool native_indirect = indirect && !state.vertex_info[0].fetch_embedded;
-	emit.vertex_offset  = native_indirect ? 0 : vertex_offset + args.base_vertex;
-	emit.first_instance = native_indirect ? 0 : instance_offset;
-
-	ExecutePreparedDraw(submit_id, buffer, draw, state, topology, emit, index_source,
-	                    primitive_restart);
+	const auto emit = ResolveDrawEmit(ucfg, state.vertex_info[0], args.offset_source,
+	                                  args.base_vertex, args.first_instance, args.indirect);
+	if (emit) {
+		ExecutePreparedDraw(submit_id, buffer, draw, state, topology, *emit, index_source,
+		                    primitive_restart);
+	}
 	ResetBindings();
+	return emit.has_value();
 }
 
 // NOLINTNEXTLINE(readability-function-cognitive-complexity)
-void RenderExecutor::DrawAuto(uint64_t submit_id, CommandBuffer& buffer, const DrawAutoArgs& args) {
+bool RenderExecutor::DrawAuto(uint64_t submit_id, CommandBuffer& buffer, const DrawAutoArgs& args) {
 	KYTY_PROFILER_FUNCTION();
 
 	EXIT_IF(buffer.IsInvalid());
@@ -1423,19 +1456,19 @@ void RenderExecutor::DrawAuto(uint64_t submit_id, CommandBuffer& buffer, const D
 	                    args.first_instance);
 
 	Common::LockGuard lock(m_context.GetMutex());
-	if (args.vertex_count == 0 || args.instance_count == 0) {
-		return;
+	if (args.indirect.address == 0 && (args.vertex_count == 0 || args.instance_count == 0)) {
+		return true;
 	}
 
 	if (ConsumeMetadataColorOperation(buffer, args.render_target_slice_offset) ||
 	    DepthStencilCopy(buffer) ||
 	    ResolveColorTargets(buffer, args.render_target_slice_offset)) {
 		ResetBindings();
-		return;
+		return true;
 	}
 
 	if (!DrawHasValidVertexShader(sh_ctx)) {
-		return;
+		return true;
 	}
 
 	if (graphics_debug_dump_enabled()) {
@@ -1461,12 +1494,12 @@ void RenderExecutor::DrawAuto(uint64_t submit_id, CommandBuffer& buffer, const D
 	vk::PrimitiveTopology topology = vk::PrimitiveTopology::ePointList;
 	if (!GetDrawTopology(ucfg, topology)) {
 		ResetBindings();
-		return;
+		return true;
 	}
 	DrawRenderState state {};
 	if (!PrepareDrawRenderState(buffer, draw, args.render_target_slice_offset, state)) {
 		ResetBindings();
-		return;
+		return true;
 	}
 
 	const bool rect_list = Prospero::IsRectList(ucfg.GetPrimType());
@@ -1480,24 +1513,18 @@ void RenderExecutor::DrawAuto(uint64_t submit_id, CommandBuffer& buffer, const D
 			     sh_ctx.GetVs().es_regs.data_addr, sh_ctx.GetVs().gs_regs.data_addr);
 		}
 		ResetBindings();
-		return;
+		return true;
 	}
 
 	LogDrawStateIfNeeded(buffer, draw, state, 0, nullptr);
-
-	const bool indirect = args.offset_source == DrawOffsetSource::IndirectArgs;
-	const auto [vertex_offset, instance_offset] =
-	    indirect ? std::pair<int32_t, uint32_t> {0, args.first_instance}
-	             : ResolveDrawOffsets(ucfg.GetIndexOffset(), state.vertex_info[0]);
-	DrawEmitInfo emit {};
-	const bool native_indirect = indirect && !state.vertex_info[0].fetch_embedded;
-	emit.first_vertex = native_indirect ? 0 :
-	    static_cast<uint32_t>(vertex_offset + static_cast<int32_t>(args.first_vertex));
-	emit.first_instance = native_indirect ? 0 : instance_offset;
-
-	DrawIndexBufferSource index_source {};
-	ExecutePreparedDraw(submit_id, buffer, draw, state, topology, emit, index_source, false);
+	const auto emit = ResolveDrawEmit(ucfg, state.vertex_info[0], args.offset_source,
+	                                  static_cast<int32_t>(args.first_vertex), args.first_instance,
+	                                  args.indirect);
+	if (emit) {
+		ExecutePreparedDraw(submit_id, buffer, draw, state, topology, *emit, {}, false);
+	}
 	ResetBindings();
+	return emit.has_value();
 }
 
 bool RenderExecutor::ResolveColorTargets(CommandBuffer& buffer, uint32_t render_target_slice_offset) {

src/graphics/host_gpu/shaders/gpu_tiler_color_clear.comp

@@ -0,0 +1,50 @@
+#version 450
+
+// One workgroup per metadata slice; see TileManager::CheckColorClear.
+layout(local_size_x = 256) in;
+
+layout(set = 0, binding = 0, std430) buffer Metadata { uint data[]; } metadata;
+layout(set = 0, binding = 1, std430) writeonly buffer Predicates { uint data[]; } predicates;
+
+layout(push_constant, std430) uniform Check {
+	uint metadata_base;
+	uint words;
+	uint keys;
+	uint expand;
+} params;
+
+const uint Slots = 8u; // TileManager::ColorClearSlots
+
+shared uint key;
+shared uint mismatch;
+
+void main() {
+	const uint slice = gl_WorkGroupID.x;
+	const uint base  = (params.metadata_base >> 2u) + slice * params.words;
+	// Each invocation reads, and may rewrite, only its own words.
+	if (gl_LocalInvocationIndex == 0u) {
+		key      = metadata.data[base] & 0xffu;
+		mismatch = 0u;
+	}
+	barrier();
+	const uint pattern = key * 0x01010101u;
+	bool       same    = true;
+	for (uint i = gl_LocalInvocationIndex; same && i < params.words; i += gl_WorkGroupSize.x) {
+		same = metadata.data[base + i] == pattern;
+	}
+	if (!same) {
+		atomicOr(mismatch, 1u);
+	}
+	barrier();
+	const uint slot = key >> 5u;
+	const bool hit  = mismatch == 0u && (key & 0x1fu) == 0u && (params.keys & (1u << slot)) != 0u;
+	if (gl_LocalInvocationIndex < Slots) {
+		predicates.data[slice * Slots + gl_LocalInvocationIndex] =
+		    uint(hit && slot == gl_LocalInvocationIndex);
+	}
+	if (hit && params.expand != 0u) {
+		for (uint i = gl_LocalInvocationIndex; i < params.words; i += gl_WorkGroupSize.x) {
+			metadata.data[base + i] = 0xffffffffu;
+		}
+	}
+}

src/graphics/presentation/performanceOverlay.cpp

@@ -310,6 +310,10 @@ void GpuActivityRows(const Snapshot& shown) {
 	Row("CPU waiting on GPU",
 	    Fmt("%.0f ms/s  (%.0f waits/s)", Rate(shown, Counter::GpuWaitNs) / 1e6,
 	        Rate(shown, Counter::GpuWaits)));
+	Row("DCC/CMASK readbacks",
+	    Fmt("%.0f ms/s  (%.0f/s)", Rate(shown, Counter::ColorMetadataReadbackNs) / 1e6,
+	        Rate(shown, Counter::ColorMetadataReadbacks)));
+	Row("DCC/CMASK GPU checks", Fmt("%.0f/s", Rate(shown, Counter::ColorMetadataChecks)));
 }
 
 void CompilationRows(const Snapshot& shown) {

src/graphics/presentation/videoOut.cpp

@@ -221,7 +221,7 @@ public:
 	void WaitForSubmitSlot(VideoOutConfig& cfg);
 	bool Flip(uint32_t micros);
 	void GetFlipStatus(VideoOutConfig& cfg, VideoOutFlipStatus& out);
-	void Wait(VideoOutConfig& cfg, int index);
+	bool IsSafeForRendering(VideoOutConfig& cfg, int index);
 
 private:
 	enum class RequestState { Reserved, Recording, Ready, Presenting };
@@ -240,6 +240,7 @@ private:
 		Graphics::Presenter::Frame* frame;
 		bool                        premultiplied_alpha;
 	};
+	void SignalDoneLocked();
 
 	Graphics::Presenter& m_presenter;
 	Common::Mutex        m_mutex;
@@ -250,6 +251,7 @@ private:
 	std::list<Request>   m_cpu_requests;
 	std::list<Request>   m_cancelled_requests;
 	bool                 m_processing      = false;
+	bool                 m_gpu_waiting     = false;
 	uint64_t             m_next_request_id = 1;
 };
 
@@ -953,7 +955,7 @@ bool FlipQueue::Reserve(VideoOutConfig& cfg, int index, int64_t flip_arg, FlipRe
 				}
 			}
 		}
-		m_done_cond_var.SignalAll();
+		SignalDoneLocked();
 	}
 	if (cfg.flip_status.flipPendingNum >= VIDEO_OUT_FLIP_QUEUE_CAPACITY) {
 		Common::DebugCounters::Add(Common::DebugCounters::Counter::FlipsRejected);
@@ -1033,7 +1035,7 @@ void FlipQueue::Cancel(VideoOutConfig& cfg) {
 			it = queue->erase(it);
 		}
 	}
-	m_done_cond_var.SignalAll();
+	SignalDoneLocked();
 	m_submit_slot_cond_var.SignalAll();
 	m_submit_cond_var.SignalAll();
 	m_mutex.Unlock();
@@ -1205,19 +1207,26 @@ void FlipQueue::WaitForSubmitSlot(VideoOutConfig& cfg) {
 	}
 }
 
-void FlipQueue::Wait(VideoOutConfig& cfg, int index) {
-	Common::LockGuard lock(m_mutex);
-
-	auto has_request = [this, &cfg, index] {
-		auto matches = [&cfg, index](const auto& r) { return r.cfg == &cfg && r.index == index; };
-		return std::any_of(m_requests.begin(), m_requests.end(), matches) ||
-		       std::any_of(m_cpu_requests.begin(), m_cpu_requests.end(), matches);
-	};
-	while (has_request()) {
-		m_done_cond_var.Wait(&m_mutex);
+void FlipQueue::SignalDoneLocked() {
+	m_done_cond_var.SignalAll();
+	// A stalled GPU cannot drain and shut down without re-checking under m_mutex, so it is alive here.
+	if (m_gpu_waiting) {
+		m_gpu_waiting = false;
+		m_presenter.Renderer().GetGpu().Wake();
 	}
 }
 
+// A display buffer is safe for rendering once it is neither displayed nor waiting to be.
+bool FlipQueue::IsSafeForRendering(VideoOutConfig& cfg, int index) {
+	Common::LockGuard lock(m_mutex);
+	auto matches = [&cfg, index](const auto& r) { return r.cfg == &cfg && r.index == index; };
+	const bool safe = cfg.flip_status.currentBuffer != index &&
+	                  std::none_of(m_requests.begin(), m_requests.end(), matches) &&
+	                  std::none_of(m_cpu_requests.begin(), m_cpu_requests.end(), matches);
+	m_gpu_waiting = !safe;
+	return safe;
+}
+
 bool FlipQueue::Flip(uint32_t micros) {
 	KYTY_PROFILER_BLOCK("FlipQueue::Flip");
 
@@ -1311,7 +1320,7 @@ bool FlipQueue::Flip(uint32_t micros) {
 		it = m_requests.erase(it);
 	}
 	m_processing = false;
-	m_done_cond_var.SignalAll();
+	SignalDoneLocked();
 	m_submit_slot_cond_var.SignalAll();
 	m_mutex.Unlock();
 	for (size_t i = count; i != 0; i--) {
@@ -1694,12 +1703,12 @@ void VideoOutDriver::WaitForSubmitSlot(int handle) {
 	m_impl->GetFlipQueue().WaitForSubmitSlot(*cfg);
 }
 
-void VideoOutDriver::WaitFlipDone(int handle, int index) {
+bool VideoOutDriver::IsSafeForRendering(int handle, int index) {
 	auto* ctx = m_impl->Get(handle);
 	EXIT_IF(ctx == nullptr);
 
 	EXIT_NOT_IMPLEMENTED(!IsValidBufferIndex(index));
-	m_impl->GetFlipQueue().Wait(*ctx, index);
+	return m_impl->GetFlipQueue().IsSafeForRendering(*ctx, index);
 }
 
 KYTY_SYSV_ABI int VideoOutGetFlipStatus(int handle, VideoOutFlipStatus* status) {

src/graphics/presentation/videoOut.h

@@ -38,7 +38,7 @@ public:
 	void CompleteFlip(uint64_t request_id);
 	void SubmitFlipPreparation(uint64_t request_id);
 	void WaitForSubmitSlot(int handle);
-	void WaitFlipDone(int handle, int index);
+	[[nodiscard]] bool IsSafeForRendering(int handle, int index);
 
 	[[nodiscard]] Impl& State() noexcept;
 

src/graphics/presentation/window/vulkanWindow.cpp

@@ -65,6 +65,9 @@ vk::PhysicalDeviceVulkan12Features WindowContext::RequiredVulkan12Features() noe
 	features.shaderSampledImageArrayNonUniformIndexing = VK_TRUE;
 	features.shaderStorageImageArrayNonUniformIndexing = VK_TRUE;
 	features.shaderStorageBufferArrayNonUniformIndexing = VK_TRUE;
+	features.descriptorBindingSampledImageUpdateAfterBind = VK_TRUE;
+	features.descriptorBindingUpdateUnusedWhilePending    = VK_TRUE;
+	features.descriptorBindingPartiallyBound              = VK_TRUE;
 	return features;
 }
 
@@ -295,6 +298,14 @@ static void VulkanFindPhysicalDevice(vk::Instance instance, vk::SurfaceKHR surfa
 		check_feature(features12.shaderStorageBufferArrayNonUniformIndexing,
 		              "shaderStorageBufferArrayNonUniformIndexing",
 		              required_features12.shaderStorageBufferArrayNonUniformIndexing);
+		check_feature(features12.descriptorBindingSampledImageUpdateAfterBind,
+		              "descriptorBindingSampledImageUpdateAfterBind",
+		              required_features12.descriptorBindingSampledImageUpdateAfterBind);
+		check_feature(features12.descriptorBindingUpdateUnusedWhilePending,
+		              "descriptorBindingUpdateUnusedWhilePending",
+		              required_features12.descriptorBindingUpdateUnusedWhilePending);
+		check_feature(features12.descriptorBindingPartiallyBound, "descriptorBindingPartiallyBound",
+		              required_features12.descriptorBindingPartiallyBound);
 		check_feature(features13.robustImageAccess, "robustImageAccess");
 		check_feature(robustness2.nullDescriptor, "nullDescriptor");
 		check_feature(features13.dynamicRendering, "dynamicRendering",
@@ -328,6 +339,9 @@ static void VulkanFindPhysicalDevice(vk::Instance instance, vk::SurfaceKHR surfa
 
 		check_feature(device_features2.features.independentBlend, "independentBlend");
 		check_feature(device_features2.features.tessellationShader, "tessellationShader");
+		check_feature(device_features2.features.drawIndirectFirstInstance,
+		              "drawIndirectFirstInstance");
+		check_feature(device_features2.features.multiDrawIndirect, "multiDrawIndirect");
 
 		if (!skip_device) {
 			auto available_extensions = EnumerateVulkan<vk::ExtensionProperties>(
@@ -499,6 +513,13 @@ static vk::Device VulkanCreateDevice(GraphicContext& graphics,
 		image_atomic_int64.pNext = supported_features2.pNext;
 		supported_features2.pNext = &image_atomic_int64;
 	}
+	const bool conditional_rendering_extension =
+	    HasExtension(device_extensions, VK_EXT_CONDITIONAL_RENDERING_EXTENSION_NAME);
+	vk::PhysicalDeviceConditionalRenderingFeaturesEXT conditional_rendering {};
+	if (conditional_rendering_extension) {
+		conditional_rendering.pNext = supported_features2.pNext;
+		supported_features2.pNext   = &conditional_rendering;
+	}
 	const bool barycentric_extension =
 	    HasExtension(device_extensions, VK_KHR_FRAGMENT_SHADER_BARYCENTRIC_EXTENSION_NAME);
 	vk::PhysicalDeviceFragmentShaderBarycentricFeaturesKHR supported_barycentric {};
@@ -572,6 +593,8 @@ static vk::Device VulkanCreateDevice(GraphicContext& graphics,
 	    feedback_extensions && feedback_layout.attachmentFeedbackLoopLayout &&
 	    feedback_dynamic.attachmentFeedbackLoopDynamicState;
 	graphics.maintenance8_enabled = maintenance8_extension && maintenance8.maintenance8;
+	graphics.conditional_rendering_enabled =
+	    conditional_rendering_extension && conditional_rendering.conditionalRendering;
 	LOGF("Vulkan maintenance8 support: %s\n",
 	     graphics.maintenance8_enabled ? "true" : "false");
 	LOGF("Vulkan depth feedback support: %s\n",
@@ -597,10 +620,14 @@ static vk::Device VulkanCreateDevice(GraphicContext& graphics,
 	device_features.depthClamp  = VK_TRUE;
 #endif
 	device_features.shaderStorageImageWriteWithoutFormat = VK_TRUE;
+	device_features.shaderStorageImageMultisample        =
+	    supported_features2.features.shaderStorageImageMultisample;
 	device_features.shaderImageGatherExtended            = VK_TRUE;
 	device_features.independentBlend                     = VK_TRUE;
 	device_features.dualSrcBlend                         = VK_TRUE;
 	device_features.tessellationShader                   = VK_TRUE;
+	device_features.drawIndirectFirstInstance            = VK_TRUE;
+	device_features.multiDrawIndirect                    = VK_TRUE;
 	device_features.sampleRateShading                    = VK_TRUE;
 	device_features.depthBiasClamp                       = VK_TRUE;
 	device_features.shaderClipDistance                   = VK_TRUE;
@@ -609,6 +636,11 @@ static vk::Device VulkanCreateDevice(GraphicContext& graphics,
 	device_features.multiViewport                        = VK_TRUE;
 	device_features.fillModeNonSolid                      = VK_TRUE;
 	device_features.vertexPipelineStoresAndAtomics       = VK_TRUE;
+	// Storage image sample counts beyond 1x require the feature.
+	if (!device_features.shaderStorageImageMultisample) {
+		graphics.physical_device_properties.limits.storageImageSampleCounts =
+		    vk::SampleCountFlagBits::e1;
+	}
 	graphics.sample_rate_shading_enabled                 = true;
 	device_features.shaderInt64 = VK_TRUE;
 	device_features.shaderFloat64 =
@@ -670,6 +702,10 @@ static vk::Device VulkanCreateDevice(GraphicContext& graphics,
 		maintenance8.pNext = const_cast<void*>(create_info.pNext);
 		create_info.pNext = &maintenance8;
 	}
+	if (graphics.conditional_rendering_enabled) {
+		conditional_rendering.pNext = const_cast<void*>(create_info.pNext);
+		create_info.pNext = &conditional_rendering;
+	}
 	create_info.pQueueCreateInfos       = &queue_create_info;
 	create_info.queueCreateInfoCount    = 1;
 	create_info.enabledExtensionCount   = static_cast<uint32_t>(device_extensions.size());
@@ -1046,7 +1082,8 @@ void WindowContext::CreateVulkan() {
 		                             VK_EXT_MESH_SHADER_EXTENSION_NAME,
 		                             VK_KHR_WORKGROUP_MEMORY_EXPLICIT_LAYOUT_EXTENSION_NAME,
 		                             VK_EXT_DEPTH_RANGE_UNRESTRICTED_EXTENSION_NAME,
-		                             VK_KHR_FRAGMENT_SHADER_BARYCENTRIC_EXTENSION_NAME}) {
+		                             VK_KHR_FRAGMENT_SHADER_BARYCENTRIC_EXTENSION_NAME,
+		                             VK_EXT_CONDITIONAL_RENDERING_EXTENSION_NAME}) {
 			if (HasExtension(available_extensions, extension)) {
 				device_extensions.push_back(extension);
 			}

src/graphics/shader/recompiler/ShaderRecompiler.cpp

@@ -930,8 +930,8 @@ CompileResult CompileProgram(TranslateResult translated, const CompileOptions& o
 
 	LOGF("%s phase begin: stage=%s hash=0x%016" PRIx64 " SPIR-V EmitProgram\n",
 	     GetDumpLabel(options), StageName(ir.stage), ir.shader_hash);
-	auto spirv = Spirv::EmitProgram(ir, options.input_info, push_data_start_dword,
-	                                options.maintenance8_enabled, options.emit_debug_info);
+	auto spirv = Spirv::EmitProgram(ir, options.input_info, push_data_start_dword, *options.profile,
+	                                options.emit_debug_info);
 	LOGF("%s phase end: stage=%s hash=0x%016" PRIx64 " SPIR-V EmitProgram words=%" PRIu64
 	     " elapsed_ms=%" PRIu64 "\n",
 	     GetDumpLabel(options), StageName(ir.stage), ir.shader_hash,

src/graphics/shader/recompiler/ShaderRecompiler.h

@@ -3,6 +3,7 @@
 
 #include "common/common.h"
 #include "common/stringUtils.h"
+#include "graphics/shader/recompiler/backend/spirv/SpirvEmitter.h"
 #include "graphics/shader/recompiler/ir/passes/ResourceMaterialization.h"
 #include "graphics/shader/recompiler/frontend/decode/ShaderDecoder.h"
 #include "graphics/shader/shader.h"
@@ -23,7 +24,7 @@ struct CompileOptions {
 	bool                        dump_ir                    = true;
 	bool                        early_dump                 = false;
 	bool                        emit_debug_info            = false;
-	bool                        maintenance8_enabled       = false;
+	const Spirv::Profile*       profile                    = &Spirv::DefaultProfile;
 	const char*                 dump_label                 = nullptr;
 	std::span<const uint32_t>   user_data;
 	std::span<const uint32_t>   back_code;

src/graphics/shader/recompiler/backend/spirv/SpirvBuilder.cpp

@@ -169,6 +169,9 @@ uint32_t Builder::AddString(const char* text) {
 }
 
 void Builder::AddName(uint32_t target, const char* name) {
+	if (!m_emit_names) {
+		return;
+	}
 	std::vector<uint32_t> operands = {target};
 	AppendString(operands, name);
 	AppendInstruction(m_debug, spv::OpName, operands);

src/graphics/shader/recompiler/backend/spirv/SpirvBuilder.h

@@ -67,6 +67,7 @@ public:
 	}
 
 	uint32_t AddString(const char* text);
+	void EnableNames() { m_emit_names = true; }
 	void AddName(uint32_t target, const char* name);
 	template <typename... Args>
 	void AddAnnotation(spv::Op opcode, const Args&... operands) {
@@ -159,6 +160,7 @@ private:
 	std::map<std::string, uint32_t>           m_import_ids;
 	std::map<std::vector<uint32_t>, uint32_t> m_declaration_ids;
 	size_t                                    m_unpatched_phi_incomings = 0;
+	bool                                      m_emit_names              = false;
 };
 
 } // namespace Libs::Graphics::ShaderRecompiler::Spirv

src/graphics/shader/recompiler/backend/spirv/SpirvEmitter.cpp

@@ -161,6 +161,7 @@ void ValidateNativeProgram(const IR::Program& program, bool lds_storage) {
 		if (!kind.has_value()) {
 			Fail(program, "native shader plan has an invalid image class");
 		}
+		if (image.heap) continue;
 		if (image.mip_count == 0u ||
 		    (image.mip_mode != IR::ImageMipMode::Dynamic && image.mip_count != 1u)) {
 			Fail(program, "native shader plan has an invalid image mip descriptor count");
@@ -204,7 +205,7 @@ void ValidateNativeProgram(const IR::Program& program, bool lds_storage) {
 } // namespace
 
 std::vector<uint32_t> EmitProgram(IR::Program& program, ShaderStageInputInfo input_info,
-                                  uint32_t push_data_start_dword, bool maintenance8_enabled,
+                                  uint32_t push_data_start_dword, const Profile& profile,
                                   bool emit_debug_info) {
 	using namespace Emitter;
 
@@ -220,12 +221,13 @@ std::vector<uint32_t> EmitProgram(IR::Program& program, ShaderStageInputInfo inp
 	}
 	IR::ValidateProgram(program, true);
 	program.bindings = {.push_data_start_dword = push_data_start_dword};
-	EmitterState state(program, input_info, maintenance8_enabled);
+	EmitterState state(program, input_info, profile);
 	if (emit_debug_info) {
 		char name[64];
 		std::snprintf(name, sizeof(name), "rdna2-%016" PRIx64 "-block-byte-pc-plus-one",
 		              program.shader_hash);
 		state.debug_source = state.builder.AddString(name);
+		state.builder.EnableNames();
 	}
 	const auto* workgroup = ShaderWorkgroupInput(program.stage, input_info);
 	state.lane_count =

src/graphics/shader/recompiler/backend/spirv/SpirvEmitter.h

@@ -9,11 +9,19 @@
 
 namespace Libs::Graphics::ShaderRecompiler::Spirv {
 
+// Host device capabilities that change the emitted SPIR-V. Built once per device.
+struct Profile {
+	bool maintenance8         = false; // Dynamic texel offsets on image samples.
+	bool native_float_min_max = false; // NMin/NMax give the guest's NaN and signed-zero results.
+};
+
+inline constexpr Profile DefaultProfile {};
+
 // Resets and writes backend IDs and descriptor indices; do not emit the same program concurrently.
 std::vector<uint32_t> EmitProgram(IR::Program& program, ShaderStageInputInfo input_info,
-                                  uint32_t push_data_start_dword = 0,
-                                  bool maintenance8_enabled = false,
-                                  bool emit_debug_info = false);
+                                  uint32_t       push_data_start_dword = 0,
+                                  const Profile& profile               = DefaultProfile,
+                                  bool           emit_debug_info       = false);
 
 } // namespace Libs::Graphics::ShaderRecompiler::Spirv
 

src/graphics/shader/recompiler/backend/spirv/spirvEmitterAlu.cpp

@@ -5,18 +5,6 @@
 namespace Libs::Graphics::ShaderRecompiler::Spirv::Emitter {
 namespace {
 
-struct Pair {
-	uint32_t low  = 0;
-	uint32_t high = 0;
-};
-
-Pair ExtractPair(EmitterState& state, uint32_t value) {
-	Pair result {state.builder.AllocateId(), state.builder.AllocateId()};
-	state.builder.AddFunction(spv::OpCompositeExtract, TypeU32(state), result.low, value, 0);
-	state.builder.AddFunction(spv::OpCompositeExtract, TypeU32(state), result.high, value, 1);
-	return result;
-}
-
 uint32_t EmitMinMaxF64(EmitterState& state, uint32_t lhs, uint32_t rhs, bool max_value) {
 	const auto bits_type = TypeU32Vector(state, 2);
 	const auto lhs_bits = Unary(state, spv::OpBitcast, bits_type, lhs);
@@ -32,11 +20,10 @@ uint32_t EmitMinMaxF64(EmitterState& state, uint32_t lhs, uint32_t rhs, bool max
 	result = Select(state, TypeF64(state), equal,
 	                Unary(state, spv::OpBitcast, TypeF64(state), equal_bits), result);
 
-	// Non-IEEE mode selects the other operand for NaN, including rhs when both are NaN.
-	result = Select(state, TypeF64(state), Unary(state, spv::OpIsNan, TypeBool(state), rhs),
-	                lhs, result);
-	return Select(state, TypeF64(state), Unary(state, spv::OpIsNan, TypeBool(state), lhs),
-	              rhs, result);
+	// Non-IEEE mode selects the other operand for NaN, and lhs when both are NaN. The ordered
+	// compares already pick rhs for a NaN lhs.
+	return Select(state, TypeF64(state), Unary(state, spv::OpIsNan, TypeBool(state), rhs),
+	              lhs, result);
 }
 
 uint32_t EmitMulHigh(EmitterState& state, uint32_t lhs, uint32_t rhs, bool signed_value) {
@@ -106,35 +93,23 @@ uint32_t EmitF32ToF16RtzBits(EmitterState& state, uint32_t value) {
 }
 
 uint32_t EmitF32ToU32(EmitterState& state, uint32_t src, bool signed_value) {
-	const auto trunc         = EmitTruncF32Value(state, src);
-	const auto converted_raw = state.builder.AllocateId();
-	if (signed_value) {
-		const auto converted_i = state.builder.AllocateId();
-		state.builder.AddFunction(spv::OpConvertFToS, TypeI32(state), converted_i, trunc);
-		state.builder.AddFunction(spv::OpBitcast, TypeU32(state), converted_raw, converted_i);
-	} else {
-		state.builder.AddFunction(spv::OpConvertFToU, TypeU32(state), converted_raw, trunc);
-	}
-	const auto nan = Unary(state, spv::OpIsNan, TypeBool(state), src);
-	if (signed_value) {
-		const auto below = Binary(state, spv::OpFOrdLessThanEqual, TypeBool(state), src,
-		                          ConstantF32(state, 0xcf000000u));
-		const auto above = Binary(state, spv::OpFOrdGreaterThanEqual, TypeBool(state), src,
-		                          ConstantF32(state, 0x4f000000u));
-		const auto high =
-		    Select(state, TypeU32(state), above, ConstantU32(state, 0x7fffffffu), converted_raw);
-		const auto low =
-		    Select(state, TypeU32(state), below, ConstantU32(state, 0x80000000u), high);
-		return Select(state, TypeU32(state), nan, ConstantU32(state, 0), low);
-	}
-	const auto below =
-	    Binary(state, spv::OpFOrdLessThanEqual, TypeBool(state), src, ConstantF32(state, 0));
-	const auto above = Binary(state, spv::OpFOrdGreaterThanEqual, TypeBool(state), src,
-	                          ConstantF32(state, 0x4f800000u));
-	const auto zero  = Binary(state, spv::OpLogicalOr, TypeBool(state), nan, below);
+	// Clamping first keeps the truncating conversion defined; NaN and values past the largest
+	// float below the integer limit are selected afterwards.
+	const auto clamped =
+	    EmitExt(state, TypeF32(state), GLSLstd450NClamp,
+	            {src, ConstantF32(state, signed_value ? 0xcf000000u : 0u),
+	             ConstantF32(state, signed_value ? 0x4effffffu : 0x4f7fffffu)});
+	const auto converted = signed_value
+	                           ? Unary(state, spv::OpBitcast, TypeU32(state),
+	                                   Unary(state, spv::OpConvertFToS, TypeI32(state), clamped))
+	                           : Unary(state, spv::OpConvertFToU, TypeU32(state), clamped);
+	const auto above     = Binary(state, spv::OpFOrdGreaterThanEqual, TypeBool(state), src,
+	                              ConstantF32(state, signed_value ? 0x4f000000u : 0x4f800000u));
 	const auto high =
-	    Select(state, TypeU32(state), above, ConstantU32(state, 0xffffffffu), converted_raw);
-	return Select(state, TypeU32(state), zero, ConstantU32(state, 0), high);
+	    Select(state, TypeU32(state), above,
+	           ConstantU32(state, signed_value ? 0x7fffffffu : 0xffffffffu), converted);
+	return Select(state, TypeU32(state), Unary(state, spv::OpIsNan, TypeBool(state), src),
+	              ConstantU32(state, 0), high);
 }
 
 } // namespace
@@ -160,18 +135,6 @@ uint32_t EmitFPMad32(EmitterState& state, uint32_t a, uint32_t b, uint32_t c) {
 	return sum;
 }
 
-uint32_t EmitFindUMsb64(EmitterState& state, uint32_t value) {
-	// GLSL FindUMsb is restricted to 32-bit components.
-	const auto unpacked = Unary(state, spv::OpBitcast, TypeU32Vector(state, 2), value);
-	const auto pair =
-	    ExtractPair(state, EmitExt(state, TypeU32Vector(state, 2), GLSLstd450FindUMsb, {unpacked}));
-	const auto high_found =
-	    Binary(state, spv::OpINotEqual, TypeBool(state), pair.high, ConstantU32(state, UINT32_MAX));
-	return Select(state, TypeU32(state), high_found,
-	              Binary(state, spv::OpIAdd, TypeU32(state), pair.high, ConstantU32(state, 32)),
-	              pair.low);
-}
-
 uint32_t EmitConvertU16U32(EmitterState& state, uint32_t arg0) {
 	return EmitNative<spv::OpBitwiseAnd, IR::Type::U16>(state, arg0, ConstantU32(state, 0xffffu));
 }
@@ -235,10 +198,9 @@ uint32_t EmitCompositeExtractU32x2(EmitterState& state, uint32_t arg0, IR::Value
 }
 
 uint32_t EmitPackFloat2x16Rtz(EmitterState& state, uint32_t arg0, uint32_t arg1) {
-	const auto low  = EmitF32ToF16RtzBits(state, arg0);
-	const auto high = Binary(state, spv::OpShiftLeftLogical, TypeU32(state),
-	                         EmitF32ToF16RtzBits(state, arg1), ConstantU32(state, 16));
-	return EmitOrU32(state, low, high);
+	return EmitNative<spv::OpBitFieldInsert, IR::Type::U32>(
+	    state, EmitF32ToF16RtzBits(state, arg0), EmitF32ToF16RtzBits(state, arg1),
+	    ConstantU32(state, 16), ConstantU32(state, 16));
 }
 
 uint32_t EmitFPSaturate32(EmitterState& state, uint32_t arg0) {
@@ -255,14 +217,6 @@ uint32_t EmitUMulHi(EmitterState& state, uint32_t arg0, uint32_t arg1) {
 	return EmitMulHigh(state, arg0, arg1, false);
 }
 
-uint32_t EmitBitCount64(EmitterState& state, uint32_t arg0) {
-	// Vulkan bit-count operands remain 32-bit, including when counting a U64.
-	const auto unpacked = Unary(state, spv::OpBitcast, TypeU32Vector(state, 2), arg0);
-	const auto pair =
-	    ExtractPair(state, Unary(state, spv::OpBitCount, TypeU32Vector(state, 2), unpacked));
-	return Binary(state, spv::OpIAdd, TypeU32(state), pair.low, pair.high);
-}
-
 uint32_t EmitSMinTri32(EmitterState& state, uint32_t arg0, uint32_t arg1, uint32_t arg2) {
 	return EmitSMin32(state, arg0, EmitSMin32(state, arg1, arg2));
 }
@@ -339,8 +293,8 @@ uint32_t EmitFPSqrt(EmitterState& state, uint32_t arg0) {
 }
 
 uint32_t EmitFPExp2(EmitterState& state, uint32_t arg0) {
-	return EmitExt(state, TypeF32(state), GLSLstd450Exp2,
-	               {EmitFlushF32DenormToSignedZero(state, arg0)});
+	// A denormal input already rounds to 1.0, the result for its flushed zero.
+	return EmitExt(state, TypeF32(state), GLSLstd450Exp2, {arg0});
 }
 
 uint32_t EmitFPLog2(EmitterState& state, uint32_t arg0) {

src/graphics/shader/recompiler/backend/spirv/spirvEmitterAluHelpers.cpp

@@ -95,8 +95,8 @@ uint32_t EmitClassMaskF16(EmitterState& state, uint32_t bits, uint32_t mask) {
 
 static uint32_t EmitMinMaxF32(EmitterState& state, uint32_t lhs, uint32_t rhs, uint32_t lhs_bits,
                               uint32_t rhs_bits, bool max_value) {
-	// RDNA2 V_MIN/V_MAX_F32 return the other source for a NaN source (S1 when both are NaN)
-	// and order -0 below +0. An unordered compare keeps S0 for a NaN S1 and, unlike NMin/NMax,
+	// PS5 V_MIN/V_MAX_F32 return the other source for a NaN source (S0 when both are NaN)
+	// and order -0 below +0. An ordered compare takes S1 for a NaN S0 and, unlike NMin/NMax,
 	// always returns the exact bits of one source.
 	const auto either    = EmitOrU32(state, lhs_bits, rhs_bits);
 	const auto both_zero = EmitCompareU32Constant(state, spv::OpIEqual,
@@ -104,15 +104,21 @@ static uint32_t EmitMinMaxF32(EmitterState& state, uint32_t lhs, uint32_t rhs, u
 	const auto zero =
 	    EmitBitcastU32ToF32(state, max_value ? EmitAndU32(state, lhs_bits, rhs_bits) : either);
 	const auto keep_lhs =
-	    Binary(state, max_value ? spv::OpFUnordGreaterThanEqual : spv::OpFUnordLessThan,
+	    Binary(state, max_value ? spv::OpFOrdGreaterThanEqual : spv::OpFOrdLessThan,
 		       TypeBool(state), lhs, rhs);
 	const auto ordered = Select(state, TypeF32(state), keep_lhs, lhs, rhs);
 	const auto numeric = Select(state, TypeF32(state), both_zero, zero, ordered);
-	return Select(state, TypeF32(state), Unary(state, spv::OpIsNan, TypeBool(state), lhs), rhs,
+	return Select(state, TypeF32(state), Unary(state, spv::OpIsNan, TypeBool(state), rhs), lhs,
 	              numeric);
 }
 
 uint32_t EmitMinMaxF32Value(EmitterState& state, uint32_t lhs, uint32_t rhs, bool max_value) {
+	if (state.profile.native_float_min_max) {
+		const auto result = state.builder.AllocateId();
+		state.builder.AddFunction(spv::OpExtInst, TypeF32(state), result, GlslStd450(state),
+		                          max_value ? GLSLstd450NMax : GLSLstd450NMin, lhs, rhs);
+		return result;
+	}
 	return EmitMinMaxF32(state, lhs, rhs, EmitBitcastF32ToU32(state, lhs),
 	                     EmitBitcastF32ToU32(state, rhs), max_value);
 }
@@ -121,9 +127,10 @@ uint32_t EmitFPMedTri32(EmitterState& state, uint32_t a, uint32_t b, uint32_t c)
 	const auto a_bits = EmitBitcastF32ToU32(state, a);
 	const auto b_bits = EmitBitcastF32ToU32(state, b);
 	const auto c_bits = EmitBitcastF32ToU32(state, c);
-	const auto min_ab = EmitMinMaxF32(state, a, b, a_bits, b_bits, false);
+	// The NaN path is RDNA2 MIN3, which keeps the later source when two are NaN.
+	const auto min_ba = EmitMinMaxF32(state, b, a, b_bits, a_bits, false);
 	const auto min3 =
-	    EmitMinMaxF32(state, min_ab, c, EmitBitcastF32ToU32(state, min_ab), c_bits, false);
+	    EmitMinMaxF32(state, c, min_ba, c_bits, EmitBitcastF32ToU32(state, min_ba), false);
 	const auto max_ab = EmitMinMaxF32(state, a, b, a_bits, b_bits, true);
 	const auto max3 =
 	    EmitMinMaxF32(state, max_ab, c, EmitBitcastF32ToU32(state, max_ab), c_bits, true);
@@ -144,53 +151,30 @@ uint32_t EmitFPMedTri32(EmitterState& state, uint32_t a, uint32_t b, uint32_t c)
 }
 
 uint32_t EmitFlushF32DenormToSignedZero(EmitterState& state, uint32_t value) {
-	const auto bits      = state.builder.AllocateId();
-	const auto abs_bits  = state.builder.AllocateId();
-	const auto sign_bits = state.builder.AllocateId();
-	const auto subnormal = state.builder.AllocateId();
-	const auto selected  = state.builder.AllocateId();
-	const auto ret       = state.builder.AllocateId();
-	state.builder.AddFunction(spv::OpBitcast, TypeU32(state), bits, value);
-	state.builder.AddFunction(spv::OpBitwiseAnd, TypeU32(state), abs_bits, bits,
-	                          ConstantU32(state, 0x7fffffffu));
-	state.builder.AddFunction(spv::OpBitwiseAnd, TypeU32(state), sign_bits, bits,
-	                          ConstantU32(state, 0x80000000u));
-	state.builder.AddFunction(spv::OpULessThan, TypeBool(state), subnormal, abs_bits,
-	                          ConstantU32(state, 0x00800000u));
-	state.builder.AddFunction(spv::OpSelect, TypeU32(state), selected, subnormal, sign_bits, bits);
-	state.builder.AddFunction(spv::OpBitcast, TypeF32(state), ret, selected);
-	return ret;
+	// x * 0 keeps the sign of a denormal; NaN and infinity fail the ordered compare.
+	const auto magnitude = state.builder.AllocateId();
+	state.builder.AddFunction(spv::OpExtInst, TypeF32(state), magnitude, GlslStd450(state),
+	                          GLSLstd450FAbs, value);
+	const auto denormal = Binary(state, spv::OpFOrdLessThan, TypeBool(state), magnitude,
+	                             ConstantF32(state, 0x00800000u));
+	return Select(state, TypeF32(state), denormal,
+	              Binary(state, spv::OpFMul, TypeF32(state), value, ConstantF32(state, 0)), value);
 }
 
 uint32_t EmitTrigCycleF32(EmitterState& state, uint32_t src, bool preserve_signed_zero) {
-	const auto fract        = state.builder.AllocateId();
-	const auto bits         = state.builder.AllocateId();
-	const auto abs_bits     = state.builder.AllocateId();
-	const auto large        = state.builder.AllocateId();
-	const auto finite       = state.builder.AllocateId();
-	const auto large_finite = state.builder.AllocateId();
-	const auto reduced      = state.builder.AllocateId();
+	// Finite values of magnitude 2^23 and above are integers, so fract already gives them +0.
+	const auto fract = state.builder.AllocateId();
 	state.builder.AddFunction(spv::OpExtInst, TypeF32(state), fract, GlslStd450(state),
 	                          GLSLstd450Fract, src);
-	state.builder.AddFunction(spv::OpBitcast, TypeU32(state), bits, src);
-	state.builder.AddFunction(spv::OpBitwiseAnd, TypeU32(state), abs_bits, bits,
-	                          ConstantU32(state, 0x7fffffffu));
-	state.builder.AddFunction(spv::OpUGreaterThanEqual, TypeBool(state), large, abs_bits,
-	                          ConstantU32(state, 0x4b000000u));
-	state.builder.AddFunction(spv::OpULessThan, TypeBool(state), finite, abs_bits,
-	                          ConstantU32(state, 0x7f800000u));
-	state.builder.AddFunction(spv::OpLogicalAnd, TypeBool(state), large_finite, large, finite);
-	state.builder.AddFunction(spv::OpSelect, TypeF32(state), reduced, large_finite,
-	                          ConstantF32(state, 0), fract);
 	if (!preserve_signed_zero) {
-		return reduced;
+		return fract;
 	}
-	const auto zero = state.builder.AllocateId();
-	const auto ret  = state.builder.AllocateId();
-	state.builder.AddFunction(spv::OpIEqual, TypeBool(state), zero, abs_bits,
-	                          ConstantU32(state, 0));
-	state.builder.AddFunction(spv::OpSelect, TypeF32(state), ret, zero, src, reduced);
-	return ret;
+	const auto abs_bits = Binary(state, spv::OpBitwiseAnd, TypeU32(state),
+	                             Unary(state, spv::OpBitcast, TypeU32(state), src),
+	                             ConstantU32(state, 0x7fffffffu));
+	return Select(state, TypeF32(state),
+	              Binary(state, spv::OpIEqual, TypeBool(state), abs_bits, ConstantU32(state, 0)),
+	              src, fract);
 }
 
 uint32_t EmitF16BitsToF32(EmitterState& state, uint32_t bits) {

src/graphics/shader/recompiler/backend/spirv/spirvEmitterAnalysis.cpp

@@ -120,7 +120,14 @@ uint32_t ImageType(EmitterState& state, const IR::ImageResource& image) {
 	} else {
 		EXIT("invalid image resource class");
 	}
+	if (image.dimension == ImageDimension::Dim1D || image.dimension == ImageDimension::Dim1DArray) {
+		state.builder.RequireCapability(sampled == 1 ? spv::CapabilitySampled1D
+		                                             : spv::CapabilityImage1D);
+	}
 	const auto& info = ImageDimensionInfoFor(image.dimension);
+	if (sampled == 2 && info.multisampled != 0u) {
+		state.builder.RequireCapability(spv::CapabilityStorageImageMultisample);
+	}
 	const auto  scalar_type =
 	    image.atomic64 ? TypeU64(state) : ImageScalarType(state, image.numeric_class);
 	return state.builder.Type(spv::OpTypeImage, scalar_type,
@@ -144,14 +151,20 @@ void DecorateNonUniformDescriptor(EmitterState& state, uint32_t id, spv::Capabil
 	state.builder.RequireCapability(capability);
 	state.builder.AddAnnotation(spv::OpDecorate, id, spv::DecorationNonUniform);
 }
+
+ImageDefinition& ImageDescriptorDefinition(EmitterState& state, const IR::ImageResource& image,
+                                           IR::DescriptorBindingKind kind) {
+	return (image.heap ? state.heap_images : state.images)[IR::ImageBindingIndex(kind)];
+}
 } // namespace
 
 uint32_t LoadImageDescriptor(EmitterState& state, uint32_t resource, uint32_t mip,
                              uint32_t array_index) {
 	const auto pointer = ImageDescriptorPointer(state, resource, mip, array_index);
 	const auto image = state.builder.AllocateId();
-	const auto kind = *IR::DescriptorBindingForImage(state.program.info.images[resource]);
-	state.builder.AddFunction(spv::OpLoad, state.images[IR::ImageBindingIndex(kind)].type,
+	const auto& resource_info = state.program.info.images[resource];
+	const auto kind = *IR::DescriptorBindingForImage(resource_info);
+	state.builder.AddFunction(spv::OpLoad, ImageDescriptorDefinition(state, resource_info, kind).type,
 	                          image, pointer);
 	if (array_index != 0u) {
 		DecorateNonUniformDescriptor(state, image,
@@ -184,13 +197,13 @@ uint32_t MakeSampledImage(EmitterState& state, uint32_t resource, uint32_t sampl
 	EXIT_IF(image_resource.resource_class != IR::ImageResourceClass::Sampled);
 	const auto  image          = LoadImageDescriptor(state, resource, mip, array_index);
 	const auto  sampled_image = state.builder.AllocateId();
-	const auto kind = *IR::DescriptorBindingForImage(image_resource);
-	auto& sampled_type = state.images[IR::ImageBindingIndex(kind)].sampled_type;
-	if (sampled_type == 0) {
-		sampled_type = state.builder.Type(
-		    spv::OpTypeSampledImage, state.images[IR::ImageBindingIndex(kind)].type);
+	auto& definition = ImageDescriptorDefinition(state, image_resource,
+	                                             *IR::DescriptorBindingForImage(image_resource));
+	if (definition.sampled_type == 0) {
+		definition.sampled_type = state.builder.Type(spv::OpTypeSampledImage, definition.type);
 	}
-	state.builder.AddFunction(spv::OpSampledImage, sampled_type, sampled_image, image, sampler_id);
+	state.builder.AddFunction(spv::OpSampledImage, definition.sampled_type, sampled_image, image,
+	                          sampler_id);
 	if (array_index != 0u || sampler_dynamic) {
 		DecorateNonUniformDescriptor(state, sampled_image, spv::CapabilitySampledImageArrayNonUniformIndexing);
 	}
@@ -202,26 +215,31 @@ uint32_t ImageDescriptorPointer(EmitterState& state, uint32_t resource, uint32_t
 	const auto& image = state.program.info.images.at(resource);
 	EXIT_IF(mip >= image.mip_count);
 	const auto kind = IR::DescriptorBindingForImage(image);
-	EXIT_IF(!kind.has_value());
+	EXIT_IF(!kind.has_value() || (image.heap && array_index == 0u));
 	if (array_index == 0u) {
 		array_index = ConstantU32(state, image.descriptor_index + mip);
 	}
-	const auto& definition = state.images[IR::ImageBindingIndex(*kind)];
+	const auto& definition = ImageDescriptorDefinition(state, image, *kind);
 	return DescriptorElementPointer(state, definition.pointer_type, definition.variable,
 	                                array_index, *kind, resource,
 	                                "image descriptor array was not emitted");
 }
 
 void EmitStorageImageWrite(EmitterState& state, uint32_t resource, uint32_t mip_lod, uint32_t coord,
-                           uint32_t texel, uint32_t array_index) {
+                           uint32_t texel, uint32_t array_index, uint32_t sample) {
 	const auto& image = state.program.info.images.at(resource);
 	EXIT_IF(image.resource_class != IR::ImageResourceClass::Storage);
 	if (!image.atomic) {
 		state.builder.RequireCapability(spv::CapabilityStorageImageWriteWithoutFormat);
 	}
 	const auto EmitWrite = [&](uint32_t mip) {
-		state.builder.AddFunction(spv::OpImageWrite, LoadImageDescriptor(state, resource, mip, array_index),
-		                          coord, texel);
+		const auto descriptor = LoadImageDescriptor(state, resource, mip, array_index);
+		if (sample == 0u) {
+			state.builder.AddFunction(spv::OpImageWrite, descriptor, coord, texel);
+		} else {
+			state.builder.AddFunction(spv::OpImageWrite, descriptor, coord, texel,
+			                          spv::ImageOperandsSampleMask, sample);
+		}
 	};
 	if (image.mip_mode != IR::ImageMipMode::Dynamic) {
 		EmitWrite(0);

src/graphics/shader/recompiler/backend/spirv/spirvEmitterFlow.cpp

@@ -4,6 +4,7 @@
 
 #include <algorithm>
 #include <atomic>
+#include <optional>
 
 namespace Libs::Graphics::ShaderRecompiler::Spirv::Emitter {
 namespace {
@@ -114,51 +115,98 @@ uint32_t EmitBuiltinU32(EmitterState& state, IR::StageInputKind kind, uint32_t c
 
 uint32_t EmitDppWriteCondition(ValueEmitContext& ctx, const IR::DppMoveFlags& flags,
                                uint32_t exec) {
-	auto&      state      = ctx.state;
-	const auto lane       = EmitSubgroupLocalInvocationId(state);
-	const auto bank_shift = state.builder.AllocateId();
-	const auto row_shift  = state.builder.AllocateId();
-	const auto bank       = state.builder.AllocateId();
-	const auto row        = state.builder.AllocateId();
-	const auto bank_bit   = state.builder.AllocateId();
-	const auto row_bit    = state.builder.AllocateId();
-	const auto bank_hit   = state.builder.AllocateId();
-	const auto row_hit    = state.builder.AllocateId();
-	const auto bank_ok    = state.builder.AllocateId();
-	const auto row_ok     = state.builder.AllocateId();
-	const auto masks_ok   = state.builder.AllocateId();
-	state.builder.AddFunction(spv::OpShiftRightLogical, TypeU32(state), bank_shift, lane,
-	                          ConstantU32(state, 2));
-	state.builder.AddFunction(spv::OpShiftRightLogical, TypeU32(state), row_shift, lane,
-	                          ConstantU32(state, 4));
-	state.builder.AddFunction(spv::OpBitwiseAnd, TypeU32(state), bank, bank_shift,
-	                          ConstantU32(state, 3));
-	state.builder.AddFunction(spv::OpBitwiseAnd, TypeU32(state), row, row_shift,
-	                          ConstantU32(state, 3));
-	state.builder.AddFunction(spv::OpShiftLeftLogical, TypeU32(state), bank_bit,
-	                          ConstantU32(state, 1), bank);
-	state.builder.AddFunction(spv::OpShiftLeftLogical, TypeU32(state), row_bit,
-	                          ConstantU32(state, 1), row);
-	state.builder.AddFunction(spv::OpBitwiseAnd, TypeU32(state), bank_hit,
-	                          ConstantU32(state, flags.bank_mask), bank_bit);
-	state.builder.AddFunction(spv::OpBitwiseAnd, TypeU32(state), row_hit,
-	                          ConstantU32(state, flags.row_mask), row_bit);
-	state.builder.AddFunction(spv::OpINotEqual, TypeBool(state), bank_ok, bank_hit,
-	                          ConstantU32(state, 0));
-	state.builder.AddFunction(spv::OpINotEqual, TypeBool(state), row_ok, row_hit,
-	                          ConstantU32(state, 0));
-	state.builder.AddFunction(spv::OpLogicalAnd, TypeBool(state), masks_ok, bank_ok, row_ok);
-	uint32_t writable = masks_ok;
-	if (!flags.bound_control) {
-		const auto target  = EmitDppTargetLane(state, flags);
-		const auto bounded = state.builder.AllocateId();
-		state.builder.AddFunction(spv::OpLogicalAnd, TypeBool(state), bounded, writable,
-		                          target.valid);
-		writable = bounded;
+	auto&    state    = ctx.state;
+	uint32_t writable = exec;
+	uint32_t lane     = 0;
+	// Each mask bit enables writes to one bank (4 lanes) or row (16 lanes); a full mask enables all.
+	const auto enable = [&](uint32_t mask, uint32_t group_shift) {
+		if (mask == 0xfu) return;
+		lane = lane != 0 ? lane : EmitSubgroupLocalInvocationId(state);
+		const auto group = EmitNative<spv::OpBitFieldUExtract, IR::Type::U32>(
+		    state, lane, ConstantU32(state, group_shift), ConstantU32(state, 2));
+		const auto bit = EmitNative<spv::OpBitFieldUExtract, IR::Type::U32>(
+		    state, ConstantU32(state, mask), group, ConstantU32(state, 1));
+		writable = Binary(state, spv::OpLogicalAnd, TypeBool(state), writable,
+		                  Binary(state, spv::OpINotEqual, TypeBool(state), bit, ConstantU32(state, 0)));
+	};
+	enable(flags.bank_mask, 2u);
+	enable(flags.row_mask, 4u);
+	// Only row shifts (0x101-0x10f, 0x111-0x11f) can vacate a lane; other controls always read one.
+	const bool row_shift = flags.control > 0x100u && flags.control < 0x120u &&
+	                       flags.control != 0x110u;
+	if (!flags.bound_control && row_shift) {
+		writable = Binary(state, spv::OpLogicalAnd, TypeBool(state), writable,
+		                  EmitDppTargetLane(state, flags).valid);
 	}
-	const auto result = state.builder.AllocateId();
-	state.builder.AddFunction(spv::OpLogicalAnd, TypeBool(state), result, exec, writable);
-	return result;
+	return writable;
+}
+
+uint32_t LoadVertexParameter(EmitterState& state, const InputBinding& input, uint32_t vertex,
+                             uint32_t chan) {
+	const auto pointer = state.builder.AllocateId();
+	const auto value   = state.builder.AllocateId();
+	state.builder.AddFunction(spv::OpAccessChain,
+	                          TypePointer(state, spv::StorageClassInput, TypeF32(state)), pointer,
+	                          input.variable_id, ConstantU32(state, vertex),
+	                          ConstantU32(state, chan & 3u));
+	state.builder.AddFunction(spv::OpLoad, TypeF32(state), value, pointer);
+	return value;
+}
+
+// Interpolates a pixel input channel to f32. A half (0 low, 1 high) selects one f16 of a packed
+// FP16_INTERP_MODE dword before blending; smooth inputs read by halves are always per-vertex.
+uint32_t EmitPixelAttribute(EmitterState& state, const InputBinding& input, uint32_t attr,
+                            uint32_t chan, std::optional<uint32_t> half) {
+	const auto select = [&](uint32_t value) {
+		if (!half) {
+			return value;
+		}
+		const auto bits     = state.builder.AllocateId();
+		const auto unpacked = state.builder.AllocateId();
+		const auto word     = state.builder.AllocateId();
+		state.builder.AddFunction(spv::OpBitcast, TypeU32(state), bits, value);
+		state.builder.AddFunction(spv::OpExtInst, TypeF32Vector(state, 2), unpacked,
+		                          GlslStd450(state), GLSLstd450UnpackHalf2x16, bits);
+		state.builder.AddFunction(spv::OpCompositeExtract, TypeF32(state), word, unpacked, *half);
+		return word;
+	};
+	if (!input.per_vertex) {
+		const auto vector    = state.builder.AllocateId();
+		const auto component = state.builder.AllocateId();
+		state.builder.AddFunction(spv::OpLoad, TypeF32Vector(state, 4), vector, input.variable_id);
+		state.builder.AddFunction(spv::OpCompositeExtract, TypeF32(state), component, vector,
+		                          chan & 3u);
+		return select(component);
+	}
+	if (PixelParameterIsFlat(state, attr)) {
+		const auto vertex =
+		    state.input_info.pixel->parameter_mode == ShaderPixelParameterMode::LastVertex ? 2u : 0u;
+		return select(LoadVertexParameter(state, input, vertex, chan));
+	}
+	const auto barycentric_kind = state.input_info.pixel->ps_no_perspective
+	                                  ? IR::StageInputKind::BaryCoordNoPerspective
+	                                  : IR::StageInputKind::BaryCoordSmooth;
+	const auto barycentric      = InputVariableForKind(state, barycentric_kind);
+	uint32_t   sum              = 0;
+	for (uint32_t vertex = 0; vertex < 3u; vertex++) {
+		const auto pointer = state.builder.AllocateId();
+		const auto weight  = state.builder.AllocateId();
+		const auto product = state.builder.AllocateId();
+		state.builder.AddFunction(spv::OpAccessChain,
+		                          TypePointer(state, spv::StorageClassInput, TypeF32(state)),
+		                          pointer, barycentric, ConstantU32(state, vertex));
+		state.builder.AddFunction(spv::OpLoad, TypeF32(state), weight, pointer);
+		state.builder.AddFunction(spv::OpFMul, TypeF32(state), product,
+		                          select(LoadVertexParameter(state, input, vertex, chan)), weight);
+		if (vertex == 0u) {
+			sum = product;
+		} else {
+			const auto next = state.builder.AllocateId();
+			state.builder.AddFunction(spv::OpFAdd, TypeF32(state), next, sum, product);
+			sum = next;
+		}
+	}
+	return sum;
 }
 
 uint32_t EmitAttribute(EmitterState& state, uint32_t attr, uint32_t chan) {
@@ -169,57 +217,9 @@ uint32_t EmitAttribute(EmitterState& state, uint32_t attr, uint32_t chan) {
 	if (state.program.stage == ShaderType::Vertex || state.program.stage == ShaderType::Local) {
 		return EmitVertexParameterComponentU32(state, *input, chan & 3u);
 	}
-	const auto load_per_vertex = [&](uint32_t vertex) {
-		const auto pointer = state.builder.AllocateId();
-		const auto value   = state.builder.AllocateId();
-		state.builder.AddFunction(
-		    spv::OpAccessChain, TypePointer(state, spv::StorageClassInput, TypeF32(state)), pointer,
-		    input->variable_id, ConstantU32(state, vertex), ConstantU32(state, chan & 3u));
-		state.builder.AddFunction(spv::OpLoad, TypeF32(state), value, pointer);
-		return value;
-	};
-	if (input->per_vertex) {
-		if (PixelParameterIsFlat(state, attr)) {
-			const auto value = load_per_vertex(
-			    state.input_info.pixel->parameter_mode == ShaderPixelParameterMode::LastVertex ? 2u : 0u);
-			const auto bits  = state.builder.AllocateId();
-			state.builder.AddFunction(spv::OpBitcast, TypeU32(state), bits, value);
-			return bits;
-		}
-		const auto barycentric_kind = state.input_info.pixel->ps_no_perspective
-		                                  ? IR::StageInputKind::BaryCoordNoPerspective
-		                                  : IR::StageInputKind::BaryCoordSmooth;
-		const auto barycentric      = InputVariableForKind(state, barycentric_kind);
-		uint32_t   sum              = 0;
-		for (uint32_t vertex = 0; vertex < 3u; vertex++) {
-			const auto pointer = state.builder.AllocateId();
-			const auto weight  = state.builder.AllocateId();
-			const auto product = state.builder.AllocateId();
-			state.builder.AddFunction(spv::OpAccessChain,
-			                          TypePointer(state, spv::StorageClassInput, TypeF32(state)),
-			                          pointer, barycentric, ConstantU32(state, vertex));
-			state.builder.AddFunction(spv::OpLoad, TypeF32(state), weight, pointer);
-			state.builder.AddFunction(spv::OpFMul, TypeF32(state), product, load_per_vertex(vertex),
-			                          weight);
-			if (vertex == 0u) {
-				sum = product;
-			} else {
-				const auto next = state.builder.AllocateId();
-				state.builder.AddFunction(spv::OpFAdd, TypeF32(state), next, sum, product);
-				sum = next;
-			}
-		}
-		const auto bits = state.builder.AllocateId();
-		state.builder.AddFunction(spv::OpBitcast, TypeU32(state), bits, sum);
-		return bits;
-	}
-	const auto vector    = state.builder.AllocateId();
-	const auto component = state.builder.AllocateId();
-	const auto bits      = state.builder.AllocateId();
-	state.builder.AddFunction(spv::OpLoad, TypeF32Vector(state, 4), vector, input->variable_id);
-	state.builder.AddFunction(spv::OpCompositeExtract, TypeF32(state), component, vector,
-	                          chan & 3u);
-	state.builder.AddFunction(spv::OpBitcast, TypeU32(state), bits, component);
+	const auto value = EmitPixelAttribute(state, *input, attr, chan, std::nullopt);
+	const auto bits  = state.builder.AllocateId();
+	state.builder.AddFunction(spv::OpBitcast, TypeU32(state), bits, value);
 	return bits;
 }
 
@@ -230,25 +230,16 @@ uint32_t EmitInterpolationParameter(ValueEmitContext& ctx, uint32_t attr, uint32
 	if (!input->per_vertex) {
 		return EmitAttribute(ctx.state, attr, chan);
 	}
-	const auto load_vertex = [&](uint32_t vertex) {
-		const auto pointer = state.builder.AllocateId();
-		const auto value   = state.builder.AllocateId();
-		state.builder.AddFunction(
-		    spv::OpAccessChain, TypePointer(state, spv::StorageClassInput, TypeF32(state)), pointer,
-		    input->variable_id, ConstantU32(state, vertex), ConstantU32(state, chan & 3u));
-		state.builder.AddFunction(spv::OpLoad, TypeF32(state), value, pointer);
-		return value;
-	};
-
 	uint32_t selected_vertex = (mode + 1u) % 3u;
 	if (mode == 2u && PixelParameterIsFlat(state, attr)) {
 		selected_vertex =
 		    state.input_info.pixel->parameter_mode == ShaderPixelParameterMode::LastVertex ? 2u : 0u;
 	}
-	uint32_t value = load_vertex(selected_vertex);
+	uint32_t value = LoadVertexParameter(state, *input, selected_vertex, chan);
 	if (!PixelParameterIsCustom(state, attr) && mode < 2u) {
 		const auto delta = state.builder.AllocateId();
-		state.builder.AddFunction(spv::OpFSub, TypeF32(state), delta, value, load_vertex(0));
+		state.builder.AddFunction(spv::OpFSub, TypeF32(state), delta, value,
+		                          LoadVertexParameter(state, *input, 0, chan));
 		value = delta;
 	}
 	const auto bits = state.builder.AllocateId();
@@ -874,6 +865,12 @@ uint32_t EmitGetAttribute(ValueEmitContext& ctx, const IR::Inst& inst) {
 	return EmitAttribute(ctx.state, inst.Arg(0).U32(), inst.Arg(1).U32());
 }
 
+uint32_t EmitGetAttributeF16(ValueEmitContext& ctx, const IR::Inst& inst) {
+	const auto attr = inst.Arg(0).U32();
+	return EmitPixelAttribute(ctx.state, *InputBindingForParameter(ctx.state, attr), attr,
+	                          inst.Arg(1).U32(), inst.Arg(2).U32());
+}
+
 uint32_t EmitGetInterpolationParameter(ValueEmitContext& ctx, const IR::Inst& inst) {
 	return EmitInterpolationParameter(ctx, inst.Arg(0).U32(), inst.Arg(1).U32(), inst.Arg(2).U32());
 }

src/graphics/shader/recompiler/backend/spirv/spirvEmitterHelpers.cpp

@@ -4,25 +4,16 @@ namespace Libs::Graphics::ShaderRecompiler::Spirv::Emitter {
 
 DppTargetLane EmitDppPermTargetLane(EmitterState& state, uint32_t subid, uint32_t control,
                                     uint32_t lane_bits) {
-	const auto lane_mask  = (1u << lane_bits) - 1u;
-	const auto group_base = state.builder.AllocateId();
-	const auto lane      = state.builder.AllocateId();
-	const auto shift     = state.builder.AllocateId();
-	const auto selected0 = state.builder.AllocateId();
-	const auto selected  = state.builder.AllocateId();
-	const auto target    = state.builder.AllocateId();
-	state.builder.AddFunction(spv::OpBitwiseAnd, TypeU32(state), group_base, subid,
-	                          ConstantU32(state, ~lane_mask));
-	state.builder.AddFunction(spv::OpBitwiseAnd, TypeU32(state), lane, subid,
-	                          ConstantU32(state, lane_mask));
-	state.builder.AddFunction(spv::OpIMul, TypeU32(state), shift, lane,
-	                          ConstantU32(state, lane_bits));
-	state.builder.AddFunction(spv::OpShiftRightLogical, TypeU32(state), selected0,
-	                          ConstantU32(state, control), shift);
-	state.builder.AddFunction(spv::OpBitwiseAnd, TypeU32(state), selected, selected0,
-	                          ConstantU32(state, lane_mask));
-	state.builder.AddFunction(spv::OpBitwiseOr, TypeU32(state), target, group_base, selected);
-	return {target, ConstantBool(state, true)};
+	// Each lane of a group takes its lane_bits-wide source index from control.
+	const auto lane = Binary(state, spv::OpBitwiseAnd, TypeU32(state), subid,
+	                         ConstantU32(state, (1u << lane_bits) - 1u));
+	const auto selected = EmitNative<spv::OpBitFieldUExtract, IR::Type::U32>(
+	    state, ConstantU32(state, control),
+	    Binary(state, spv::OpIMul, TypeU32(state), lane, ConstantU32(state, lane_bits)),
+	    ConstantU32(state, lane_bits));
+	return {EmitNative<spv::OpBitFieldInsert, IR::Type::U32>(
+	            state, subid, selected, ConstantU32(state, 0), ConstantU32(state, lane_bits)),
+	        ConstantBool(state, true)};
 }
 
 DppTargetLane EmitDppRowShiftTargetLane(EmitterState& state, uint32_t subid, uint32_t amount,
@@ -53,42 +44,19 @@ DppTargetLane EmitDppRowShiftTargetLane(EmitterState& state, uint32_t subid, uin
 
 DppTargetLane EmitDppRowRotateRightTargetLane(EmitterState& state, uint32_t subid,
                                               uint32_t amount) {
-	const auto row      = state.builder.AllocateId();
-	const auto lane     = state.builder.AllocateId();
-	const auto in_high  = state.builder.AllocateId();
-	const auto minus    = state.builder.AllocateId();
-	const auto plus     = state.builder.AllocateId();
-	const auto selected = state.builder.AllocateId();
-	const auto target   = state.builder.AllocateId();
-	state.builder.AddFunction(spv::OpBitwiseAnd, TypeU32(state), row, subid,
-	                          ConstantU32(state, 0xfffffff0u));
-	state.builder.AddFunction(spv::OpBitwiseAnd, TypeU32(state), lane, subid,
-	                          ConstantU32(state, 15));
-	state.builder.AddFunction(spv::OpUGreaterThanEqual, TypeBool(state), in_high, lane,
-	                          ConstantU32(state, amount));
-	state.builder.AddFunction(spv::OpISub, TypeU32(state), minus, lane, ConstantU32(state, amount));
-	state.builder.AddFunction(spv::OpIAdd, TypeU32(state), plus, lane,
-	                          ConstantU32(state, 16u - amount));
-	state.builder.AddFunction(spv::OpSelect, TypeU32(state), selected, in_high, minus, plus);
-	state.builder.AddFunction(spv::OpBitwiseOr, TypeU32(state), target, row, selected);
-	return {target, ConstantBool(state, true)};
+	// Rotation wraps within the row, so only the low four lane bits change.
+	const auto rotated = Binary(state, spv::OpISub, TypeU32(state), subid, ConstantU32(state, amount));
+	return {EmitNative<spv::OpBitFieldInsert, IR::Type::U32>(state, subid, rotated,
+	                                                        ConstantU32(state, 0),
+	                                                        ConstantU32(state, 4)),
+	        ConstantBool(state, true)};
 }
 
 DppTargetLane EmitDppMirrorTargetLane(EmitterState& state, uint32_t subid, bool half_row) {
-	const auto base_mask = half_row ? 0xfffffff8u : 0xfffffff0u;
-	const auto lane_mask = half_row ? 7u : 15u;
-	const auto base      = state.builder.AllocateId();
-	const auto lane      = state.builder.AllocateId();
-	const auto mirrored  = state.builder.AllocateId();
-	const auto target    = state.builder.AllocateId();
-	state.builder.AddFunction(spv::OpBitwiseAnd, TypeU32(state), base, subid,
-	                          ConstantU32(state, base_mask));
-	state.builder.AddFunction(spv::OpBitwiseAnd, TypeU32(state), lane, subid,
-	                          ConstantU32(state, lane_mask));
-	state.builder.AddFunction(spv::OpISub, TypeU32(state), mirrored, ConstantU32(state, lane_mask),
-	                          lane);
-	state.builder.AddFunction(spv::OpBitwiseOr, TypeU32(state), target, base, mirrored);
-	return {target, ConstantBool(state, true)};
+	// Mirroring lane i of a row or half row reads lane (size - 1 - i), which flips its low bits.
+	return {Binary(state, spv::OpBitwiseXor, TypeU32(state), subid,
+	               ConstantU32(state, half_row ? 7u : 15u)),
+	        ConstantBool(state, true)};
 }
 
 DppTargetLane EmitDppTargetLane(EmitterState& state, const IR::DppMoveFlags& flags) {
@@ -117,13 +85,10 @@ DppTargetLane EmitDppTargetLane(EmitterState& state, const IR::DppMoveFlags& fla
 	}
 	// row_share:N reads lane N of the caller's row of 16.
 	if (control >= 0x150u && control <= 0x15fu) {
-		const auto row    = state.builder.AllocateId();
-		const auto target = state.builder.AllocateId();
-		state.builder.AddFunction(spv::OpBitwiseAnd, TypeU32(state), row, subid,
-		                          ConstantU32(state, 0xfffffff0u));
-		state.builder.AddFunction(spv::OpBitwiseOr, TypeU32(state), target, row,
-		                          ConstantU32(state, control & 0xfu));
-		return {target, ConstantBool(state, true)};
+		return {EmitNative<spv::OpBitFieldInsert, IR::Type::U32>(
+		            state, subid, ConstantU32(state, control & 0xfu), ConstantU32(state, 0),
+		            ConstantU32(state, 4)),
+		        ConstantBool(state, true)};
 	}
 	if (control >= 0x160u && control <= 0x16fu) {
 		const auto target = state.builder.AllocateId();

src/graphics/shader/recompiler/backend/spirv/spirvEmitterImage.cpp

@@ -389,6 +389,7 @@ Format::BufferFormatInfo ImageConversionFormat(const EmitterState&   state,
 	const auto format = image.conversion_format;
 	if (format == Prospero::BufferFormat::kInvalid) return {};
 	const auto info = Format::GetFormatInfo(format);
+	if (IR::ReadsDepthBits(image)) return info;
 	const bool raw_signed = IR::RawSignedStorageFormat(image, format) != Prospero::BufferFormat::kInvalid;
 	EXIT_IF((Prospero::RemapTextureFormat(format) == format && !raw_signed) || info.component_count == 0u ||
 	        info.component_count > 4u);
@@ -424,6 +425,7 @@ uint32_t UnpackImageTexel(ValueEmitContext& ctx, const IR::MemoryInfo& mem, uint
 
 	const auto numeric_class = ctx.state.program.info.images[mem.resource].numeric_class;
 	const auto scalar_type   = ImageScalarType(ctx.state, numeric_class);
+	const bool integer       = info.packed_bitfield || info.type == Format::ComponentType::Uint;
 	uint32_t   packed        = 0;
 	if (info.packed_bitfield) {
 		packed = ctx.state.builder.AllocateId();
@@ -441,8 +443,16 @@ uint32_t UnpackImageTexel(ValueEmitContext& ctx, const IR::MemoryInfo& mem, uint
 			ctx.state.builder.AddFunction(spv::OpCompositeExtract, scalar_type,
 			                              components[component], texel, component);
 			// UNorm backing preserves the guest's filtering; scaling precedes swizzle constants.
+			const auto unorm_max = static_cast<float>((1u << info.component_bits[component]) - 1u);
 			components[component] = Binary(ctx.state, spv::OpFMul, scalar_type, components[component],
-			                               ConstantF32Value(ctx.state, 255.0f));
+			                               ConstantF32Value(ctx.state, unorm_max));
+			if (integer) {
+				// Depth bits: round back to the stored UNorm code.
+				components[component] =
+				    Unary(ctx.state, spv::OpConvertFToU, TypeU32(ctx.state),
+				          Binary(ctx.state, spv::OpFAdd, scalar_type, components[component],
+				                 ConstantF32Value(ctx.state, 0.5f)));
+			}
 		}
 	}
 	for (uint32_t component = info.component_count; component < 4u; component++) {
@@ -454,20 +464,19 @@ uint32_t UnpackImageTexel(ValueEmitContext& ctx, const IR::MemoryInfo& mem, uint
 	for (uint32_t component = 0; component < 4u; component++) {
 		const auto selector = (swizzle >> (component * 3u)) & 7u;
 		if (selector == 1u) {
-			selected[component] = info.packed_bitfield
+			selected[component] = integer
 			    ? ConstantU32(ctx.state, Format::FormattedConstantBits(info, Format::FormattedSourceKind::One))
 			    : ConstantF32Value(ctx.state, 1.0f);
 		} else if (selector >= 4u) {
 			selected[component] = components[selector - 4u];
 		} else {
-			selected[component] = info.packed_bitfield ? ConstantU32(ctx.state, 0u)
-			                                          : ZeroF32(ctx.state);
+			selected[component] = integer ? ConstantU32(ctx.state, 0u) : ZeroF32(ctx.state);
 		}
 	}
 	const auto result = ctx.state.builder.AllocateId();
 	ctx.state.builder.AddFunction(spv::OpCompositeConstruct,
-	                              ImageVectorType(ctx.state, numeric_class, 4), result, selected[0],
-	                              selected[1], selected[2], selected[3]);
+	                              integer ? TypeU32Vector(ctx.state, 4) : TypeF32Vector(ctx.state, 4),
+	                              result, selected[0], selected[1], selected[2], selected[3]);
 	return info.type == Format::ComponentType::Unorm
 	           ? Unary(ctx.state, spv::OpBitcast, TypeF32Vector(ctx.state, 4), result) : result;
 }
@@ -691,6 +700,21 @@ auto EmitImageAccess(ValueEmitContext& ctx, const IR::Inst& inst, uint32_t resul
 	    image.indirect_resources.size() < 2u) {
 		ctx.Fail(inst, "has no indirect image runtime mapping");
 	}
+	if (image.heap) {
+		const auto entry = EmitImageHeapEntry(state, key, image.indirect_mapping_offset);
+		const auto kinds = static_cast<uint32_t>(image.indirect_resources.size()) - 1u;
+		// One kind leaves every slot word a bare element.
+		if (kinds == 1u) return emit(image.indirect_resources[1], entry);
+		const auto element = Binary(state, spv::OpBitwiseAnd, TypeU32(state), entry,
+		                            ConstantU32(state, (1u << IR::ImageHeapKindShift) - 1u));
+		const auto EmitKind = [&](uint32_t kind) {
+			return emit(image.indirect_resources[kind + 1u], element);
+		};
+		return EmitIndexSwitch(state,
+		                       Binary(state, spv::OpShiftRightLogical, TypeU32(state), entry,
+		                              ConstantU32(state, IR::ImageHeapKindShift)),
+		                       kinds, result_type, EmitKind);
+	}
 	const auto selected = EmitIndirectResourceIndex(
 	    state, key, image.indirect_mapping_offset, image.indirect_search_iterations, 0u);
 	const auto SameType = [&](uint32_t lhs, uint32_t rhs) {
@@ -870,14 +894,19 @@ void EmitImage(ValueEmitContext& ctx, const IR::Inst& inst) {
 	}
 	if (op == IR::ValueOpcode::ImageQueryLod) {
 		state.builder.RequireCapability(spv::CapabilityImageQuery);
-		const auto dimension = image.dimension;
-		const auto sampled = MakeSampledImage(state, mem.resource,
-		                                     LoadSamplerDescriptor(state, mem.sampler));
-		const auto lod       = state.builder.AllocateId();
-		state.builder.AddFunction(
-		    spv::OpImageQueryLod, TypeF32Vector(state, 2), lod, sampled,
-		    CoordF32(ctx, mem, *address, 0, ImageDimensionInfoFor(dimension).spatial_components,
-		             image.cube));
+		const auto sampler = LoadSamplerDescriptor(state, mem.sampler);
+		const auto lod = EmitImageAccess(ctx, inst, TypeF32Vector(state, 2),
+		    [&](uint32_t resource, uint32_t array_index) {
+			    const auto& candidate = state.program.info.images[resource];
+			    const auto query = state.builder.AllocateId();
+			    state.builder.AddFunction(
+			        spv::OpImageQueryLod, TypeF32Vector(state, 2), query,
+			        MakeSampledImage(state, resource, sampler, 0u, array_index),
+			        CoordF32(ctx, mem, *address, 0,
+			                 ImageDimensionInfoFor(candidate.dimension).spatial_components,
+			                 candidate.cube));
+			    return query;
+		    });
 		uint32_t values[4] = {ConstantU32(state, 0), ConstantU32(state, 0), ConstantU32(state, 0),
 		                      ConstantU32(state, 0)};
 		for (uint32_t index = 0; index < 2u; index++) {
@@ -932,9 +961,12 @@ void EmitImage(ValueEmitContext& ctx, const IR::Inst& inst) {
 				const auto mip_lod = candidate.mip_mode == IR::ImageMipMode::Dynamic
 				    ? LodU32(ctx, mem, *address, mem.image_dimension) : 0u;
 				const auto coord = CoordU32(ctx, mem, *address, candidate.dimension);
+				const auto& dimension_info = ImageDimensionInfoFor(candidate.dimension);
+				const auto sample = dimension_info.multisampled != 0u
+				    ? AddressU32(ctx, mem, *address, dimension_info.coordinate_components) : 0u;
 				const auto texel = StoreTexel(ctx, selected, ctx.Arg(inst, 2),
 				    candidate.numeric_class == Prospero::TextureNumericClass::Uint);
-				EmitStorageImageWrite(state, resource, mip_lod, coord, texel, array_index);
+				EmitStorageImageWrite(state, resource, mip_lod, coord, texel, array_index, sample);
 			});
 		});
 		return;
@@ -1057,8 +1089,8 @@ void EmitImage(ValueEmitContext& ctx, const IR::Inst& inst) {
 			operands[1] = AddressF32(ctx, mem, *address, layout.bias);
 			operand_count = 2;
 		}
-		if (layout.offset != NoImageComponent &&
-		    !offset_value.IsImmediate() && !state.maintenance8_enabled) {
+		if (layout.offset != NoImageComponent && !offset_value.IsImmediate() &&
+		    !state.profile.maintenance8) {
 			ctx.Fail(inst, "requires maintenance8 for a dynamic sample texel offset");
 		}
 		const auto samplers = LoadImageSamplers(ctx, inst);

src/graphics/shader/recompiler/backend/spirv/spirvEmitterInstructions.h

@@ -62,8 +62,6 @@ EMIT_NATIVE(IMul32, OpIMul, U32, uint32_t, uint32_t)
 EMIT_NATIVE(UDiv32, OpUDiv, U32, uint32_t, uint32_t)
 EMIT_NATIVE(IAdd64, OpIAdd, U64, uint32_t, uint32_t)
 EMIT_NATIVE(ISub64, OpISub, U64, uint32_t, uint32_t)
-EMIT_NATIVE(IMul64, OpIMul, U64, uint32_t, uint32_t)
-EMIT_NATIVE(IAddCarry32, OpIAddCarry, U32x2, uint32_t, uint32_t)
 uint32_t EmitSMulHi(EmitterState& state, uint32_t arg0, uint32_t arg1);
 uint32_t EmitUMulHi(EmitterState& state, uint32_t arg0, uint32_t arg1);
 inline constexpr auto EmitIAbs32 = EmitGlsl<GLSLstd450SAbs, IR::Type::U32, uint32_t>;
@@ -92,10 +90,8 @@ EMIT_NATIVE(BitwiseNot32, OpNot, U32, uint32_t)
 EMIT_NATIVE(BitwiseAnd64, OpBitwiseAnd, U64, uint32_t, uint32_t)
 EMIT_NATIVE(BitReverse32, OpBitReverse, U32, uint32_t)
 EMIT_NATIVE(BitCount32, OpBitCount, U32, uint32_t)
-uint32_t EmitBitCount64(EmitterState& state, uint32_t arg0);
 inline constexpr auto EmitFindILsb32 = EmitGlsl<GLSLstd450FindILsb, IR::Type::U32, uint32_t>;
 inline constexpr auto EmitFindUMsb32 = EmitGlsl<GLSLstd450FindUMsb, IR::Type::U32, uint32_t>;
-uint32_t EmitFindUMsb64(EmitterState& state, uint32_t arg0);
 inline constexpr auto EmitSMin32 = EmitGlsl<GLSLstd450SMin, IR::Type::U32, uint32_t, uint32_t>;
 inline constexpr auto EmitSMax32 = EmitGlsl<GLSLstd450SMax, IR::Type::U32, uint32_t, uint32_t>;
 inline constexpr auto EmitUMin32 = EmitGlsl<GLSLstd450UMin, IR::Type::U32, uint32_t, uint32_t>;
@@ -227,10 +223,7 @@ uint32_t              EmitGetUserData(EmitterState& state, IR::ScalarReg reg);
 uint32_t              EmitGetDispatchThreadExtent(ValueEmitContext& ctx, const IR::Inst& inst);
 uint32_t              EmitGetBuiltin(ValueEmitContext& ctx, IR::Value kind, IR::Value index);
 uint32_t              EmitUndefU1(EmitterState& state, const IR::Inst& inst);
-inline constexpr auto EmitUndefU8  = EmitUndefU1;
-inline constexpr auto EmitUndefU16 = EmitUndefU1;
 inline constexpr auto EmitUndefU32 = EmitUndefU1;
-inline constexpr auto EmitUndefU64 = EmitUndefU1;
 uint32_t              EmitDppMoveU32(ValueEmitContext& ctx, const IR::Inst& inst);
 uint32_t              EmitDppUpdateU32(ValueEmitContext& ctx, const IR::Inst& inst);
 uint32_t              EmitWqmU64(EmitterState& state, uint32_t value);
@@ -243,6 +236,7 @@ uint32_t              EmitSubgroupUMinU32(ValueEmitContext& ctx, const IR::Inst&
 uint32_t              EmitWriteLane(ValueEmitContext& ctx, const IR::Inst& inst);
 uint32_t              EmitPermlane16U32(ValueEmitContext& ctx, const IR::Inst& inst);
 uint32_t              EmitGetAttribute(ValueEmitContext& ctx, const IR::Inst& inst);
+uint32_t              EmitGetAttributeF16(ValueEmitContext& ctx, const IR::Inst& inst);
 uint32_t              EmitGetInterpolationParameter(ValueEmitContext& ctx, const IR::Inst& inst);
 void                  EmitSetAttribute(ValueEmitContext& ctx, const IR::Inst& inst);
 uint32_t              EmitGetShaderBase(ValueEmitContext& ctx);

src/graphics/shader/recompiler/backend/spirv/spirvEmitterInternal.h

@@ -5,6 +5,7 @@
 #include "common/stringUtils.h"
 #include "graphics/shader/recompiler/BufferFormat.h"
 #include "graphics/shader/recompiler/backend/spirv/SpirvBuilder.h"
+#include "graphics/shader/recompiler/backend/spirv/SpirvEmitter.h"
 #include "graphics/shader/recompiler/ir/ShaderIR.h"
 #include "graphics/shader/recompiler/ir/passes/ResourceMaterialization.h"
 
@@ -73,9 +74,9 @@ struct ImageDefinition {
 };
 
 struct EmitterState {
-	EmitterState(IR::Program& program_, ShaderStageInputInfo input_info_, bool maintenance8_enabled_)
+	EmitterState(IR::Program& program_, ShaderStageInputInfo input_info_, const Profile& profile_)
 	    : builder(program_.stage == ShaderType::Mesh ? 0x00010400u : 0x00010300u),
-	      program(program_), input_info(input_info_), maintenance8_enabled(maintenance8_enabled_) {
+	      program(program_), input_info(input_info_), profile(profile_) {
 		if (ShaderWorkgroupInput(program.stage, input_info) != nullptr) {
 			lds_storage_class = spv::StorageClassWorkgroup;
 		}
@@ -88,7 +89,7 @@ struct EmitterState {
 	Builder                                          builder;
 	IR::Program&                                     program;
 	ShaderStageInputInfo                             input_info;
-	bool                                            maintenance8_enabled;
+	const Profile&                                      profile;
 	uint32_t                                        debug_source = 0;
 	uint32_t                                        void_type = 0;
 	uint32_t                                        bool_type = 0;
@@ -129,6 +130,7 @@ struct EmitterState {
 	uint32_t                                         lds_u64_variable        = 0;
 	std::array<uint32_t, 2>                          scratch_variable {};
 	std::array<ImageDefinition, IR::ImageBindingCount> images {};
+	std::array<ImageDefinition, IR::ImageBindingCount> heap_images {};
 	uint32_t                   sampler_variable                      = 0;
 	uint32_t                   sampler_type                          = 0;
 	uint32_t                   sampler_pointer_type                  = 0;
@@ -338,8 +340,9 @@ uint32_t MakeSampledImage(EmitterState& state, uint32_t resource, uint32_t sampl
                           uint32_t mip = 0, uint32_t array_index = 0,
                           bool sampler_dynamic = false);
 
+// sample is 0 (no id) for single-sampled images.
 void EmitStorageImageWrite(EmitterState& state, uint32_t resource, uint32_t mip_lod, uint32_t coord,
-                           uint32_t texel, uint32_t array_index);
+                           uint32_t texel, uint32_t array_index, uint32_t sample);
 
 spv::ExecutionModel ExecutionModelForStage(ShaderType stage);
 
@@ -429,6 +432,8 @@ struct MemoryResourceAccess {
 
 uint32_t EmitIndirectResourceIndex(EmitterState& state, uint32_t key, uint32_t mapping_offset,
                                    uint32_t search_iterations, uint32_t default_resource);
+// Slot word of an image heap byte offset; offsets past the heap select the null element.
+uint32_t EmitImageHeapEntry(EmitterState& state, uint32_t key, uint32_t mapping_offset);
 MemoryResourceAccess PrepareMemoryResourceAccess(EmitterState& state, const IR::MemoryInfo& mem,
                                                  uint32_t array_index = 0u);
 
@@ -517,8 +522,6 @@ uint32_t EmitClassMaskF16(EmitterState& state, uint32_t bits, uint32_t mask);
 
 uint32_t EmitMinMaxF32Value(EmitterState& state, uint32_t lhs, uint32_t rhs, bool max_value);
 
-inline constexpr auto EmitTruncF32Value = EmitGlsl<GLSLstd450Trunc, IR::Type::F32, uint32_t>;
-
 uint32_t EmitFlushF32DenormToSignedZero(EmitterState& state, uint32_t value);
 
 uint32_t EmitTrigCycleF32(EmitterState& state, uint32_t src, bool preserve_signed_zero);
@@ -579,6 +582,10 @@ auto EmitIndexSwitch(EmitterState& state, uint32_t index, uint32_t count,
 
 template <typename Fn>
 void EmitIfCondition(EmitterState& state, uint32_t condition, Fn&& fn) {
+	if (condition == ConstantBool(state, true)) {
+		fn();
+		return;
+	}
 	const auto then_label  = state.builder.AllocateId();
 	const auto merge_label = state.builder.AllocateId();
 	state.builder.AddFunction(spv::OpSelectionMerge, merge_label, spv::SelectionControlMaskNone);
@@ -592,6 +599,7 @@ void EmitIfCondition(EmitterState& state, uint32_t condition, Fn&& fn) {
 template <typename Fn>
 uint32_t EmitValueOrDefaultIfCondition(EmitterState& state, uint32_t condition, uint32_t type,
                                        uint32_t default_value, Fn&& fn) {
+	if (condition == ConstantBool(state, true)) return fn();
 	const auto then_label  = state.builder.AllocateId();
 	const auto header     = state.current_label;
 	const auto merge_label = state.builder.AllocateId();

src/graphics/shader/recompiler/backend/spirv/spirvEmitterMemory.cpp

@@ -1605,9 +1605,12 @@ void EmitReadConstBuffer(ValueEmitContext& ctx, const IR::Inst& inst) {
 	}
 	auto& state        = ctx.state;
 	mem.kind           = IR::ResourceKind::ScalarBuffer;
-	auto index = Binary(state, spv::OpShiftRightLogical, TypeU32(state), ctx.Arg(inst, 1),
-	                    ConstantU32(state, 2));
-	if (mem.offset >= 4u) {
+	const auto offset = inst.Arg(1).Resolve();
+	auto       index  = offset.IsImmediate()
+	                        ? ConstantU32(state, (offset.U32() >> 2u) + (mem.offset >> 2u))
+	                        : Binary(state, spv::OpShiftRightLogical, TypeU32(state),
+	                                 ctx.Arg(inst, 1), ConstantU32(state, 2));
+	if (!offset.IsImmediate() && mem.offset >= 4u) {
 		index = Binary(state, spv::OpIAdd, TypeU32(state), index, ConstantU32(state, mem.offset >> 2u));
 	}
 	const auto access    = PrepareMemoryResourceAccess(state, mem);

src/graphics/shader/recompiler/backend/spirv/spirvEmitterMemoryHelpers.cpp

@@ -38,19 +38,39 @@ uint32_t EmitBinaryU32(EmitterState& state, spv::Op opcode, uint32_t lhs, uint32
 	return ret;
 }
 
+static uint32_t LoadFlattenedSrt(EmitterState& state, uint32_t index) {
+	const auto pointer = state.builder.AllocateId();
+	state.builder.AddFunction(spv::OpAccessChain, TypeStorageBufferElementPointer(state), pointer,
+	                          state.flattened_srt_variable, ConstantU32(state, 0), index);
+	const auto value = state.builder.AllocateId();
+	state.builder.AddFunction(spv::OpLoad, TypeU32(state), value, pointer);
+	return value;
+}
+
+uint32_t EmitImageHeapEntry(EmitterState& state, uint32_t key, uint32_t mapping_offset) {
+	const auto Mapping = [&](uint32_t word) {
+		return LoadFlattenedSrt(state, ConstantU32(state, mapping_offset + word));
+	};
+	// Keys past the table select its trailing null slot.
+	const auto index = EmitUMin32(state, Binary(state, spv::OpShiftRightLogical, TypeU32(state), key,
+	                                            ConstantU32(state, 2u)),
+	                              Mapping(0u));
+	const auto offset = Unary(state, spv::OpUConvert, TypeU64(state),
+	                          Binary(state, spv::OpShiftLeftLogical, TypeU32(state), index,
+	                                 ConstantU32(state, 2u)));
+	const auto pointer = Unary(state, spv::OpConvertUToPtr, TypePhysicalU32Pointer(state),
+	                           Binary(state, spv::OpIAdd, TypeU64(state),
+	                                  PackU64(state, Mapping(1u), Mapping(2u)), offset));
+	const auto entry = state.builder.AllocateId();
+	state.builder.AddFunction(spv::OpLoad, TypeU32(state), entry, pointer,
+	                          spv::MemoryAccessAlignedMask, 4u);
+	return entry;
+}
+
 uint32_t EmitIndirectResourceIndex(EmitterState& state, uint32_t key, uint32_t mapping_offset,
                                    uint32_t search_iterations, uint32_t default_resource) {
-	const auto LoadMapping = [&](uint32_t index) {
-		const auto pointer = state.builder.AllocateId();
-		state.builder.AddFunction(spv::OpAccessChain, TypeStorageBufferElementPointer(state),
-		                          pointer, state.flattened_srt_variable, ConstantU32(state, 0),
-		                          index);
-		const auto value = state.builder.AllocateId();
-		state.builder.AddFunction(spv::OpLoad, TypeU32(state), value, pointer);
-		return value;
-	};
 	const auto mapping = ConstantU32(state, mapping_offset);
-	const auto count = LoadMapping(mapping);
+	const auto count = LoadFlattenedSrt(state, mapping);
 	if (search_iterations == 0u) {
 		const auto index = Binary(state, spv::OpShiftRightLogical, TypeU32(state), key,
 		                          ConstantU32(state, 2u));
@@ -58,7 +78,7 @@ uint32_t EmitIndirectResourceIndex(EmitterState& state, uint32_t key, uint32_t m
 		const auto safe = Select(state, TypeU32(state), valid, index, ConstantU32(state, 0u));
 		const auto entry = Binary(state, spv::OpIAdd, TypeU32(state), safe,
 		                          ConstantU32(state, mapping_offset + 1u));
-		return Select(state, TypeU32(state), valid, LoadMapping(entry),
+		return Select(state, TypeU32(state), valid, LoadFlattenedSrt(state, entry),
 		              ConstantU32(state, default_resource));
 	}
 	const auto entry_at = [&](uint32_t index) {
@@ -76,7 +96,7 @@ uint32_t EmitIndirectResourceIndex(EmitterState& state, uint32_t key, uint32_t m
 		                        ConstantU32(state, 1u));
 		const auto probe = Select(state, TypeU32(state), active, mid, ConstantU32(state, 0u));
 		const auto less = Binary(state, spv::OpULessThan, TypeBool(state),
-		                         LoadMapping(entry_at(probe)), key);
+		                         LoadFlattenedSrt(state, entry_at(probe)), key);
 		low = Select(state, TypeU32(state),
 		             Binary(state, spv::OpLogicalAnd, TypeBool(state), active, less),
 		             Binary(state, spv::OpIAdd, TypeU32(state), mid, ConstantU32(state, 1u)), low);
@@ -87,9 +107,10 @@ uint32_t EmitIndirectResourceIndex(EmitterState& state, uint32_t key, uint32_t m
 	const auto in_range = Binary(state, spv::OpULessThan, TypeBool(state), low, count);
 	const auto entry = entry_at(Select(state, TypeU32(state), in_range, low, ConstantU32(state, 0u)));
 	const auto match = Binary(state, spv::OpLogicalAnd, TypeBool(state), in_range,
-	                          Binary(state, spv::OpIEqual, TypeBool(state), LoadMapping(entry), key));
-	const auto resource = LoadMapping(Binary(state, spv::OpIAdd, TypeU32(state), entry,
-	                                         ConstantU32(state, 1u)));
+	                          Binary(state, spv::OpIEqual, TypeBool(state),
+	                                 LoadFlattenedSrt(state, entry), key));
+	const auto resource = LoadFlattenedSrt(
+	    state, Binary(state, spv::OpIAdd, TypeU32(state), entry, ConstantU32(state, 1u)));
 	return Select(state, TypeU32(state), match, resource, ConstantU32(state, default_resource));
 }
 

src/graphics/shader/recompiler/backend/spirv/spirvEmitterModule.cpp

@@ -206,6 +206,17 @@ uint32_t F32ArrayType(EmitterState& state, uint32_t count) {
 	return state.builder.Type(spv::OpTypeArray, TypeF32(state), ConstantU32(state, count));
 }
 
+void DefineImageArray(EmitterState& state, ImageDefinition& definition, const IR::ImageResource& image,
+                      uint32_t count, uint32_t set, uint32_t binding) {
+	definition.type     = ImageType(state, image);
+	const auto array    = state.builder.Type(spv::OpTypeArray, definition.type, ConstantU32(state, count));
+	definition.variable = state.builder.DefineGlobalVariable(
+	    TypePointer(state, spv::StorageClassUniformConstant, array), spv::StorageClassUniformConstant);
+	state.builder.AddAnnotation(spv::OpDecorate, definition.variable, spv::DecorationDescriptorSet, set);
+	state.builder.AddAnnotation(spv::OpDecorate, definition.variable, spv::DecorationBinding, binding);
+	definition.pointer_type = TypePointer(state, spv::StorageClassUniformConstant, definition.type);
+}
+
 void DefineDescriptors(EmitterState& state) {
 	using Kind = IR::DescriptorBindingKind;
 	auto& info = state.program.info;
@@ -222,6 +233,14 @@ void DefineDescriptors(EmitterState& state) {
 		    (image.mip_mode != IR::ImageMipMode::Dynamic && image.mip_count != 1u)) {
 			EXIT("shader image has an invalid descriptor type or mip count");
 		}
+		if (image.heap) {
+			// Heap kinds alias one array; the root itself is never accessed.
+			auto& definition = state.heap_images[IR::ImageBindingIndex(*kind)];
+			if (image.indirect_resources.empty() && definition.variable == 0) {
+				DefineImageArray(state, definition, image, IR::ImageHeapCapacity, IR::ImageHeapSet, 0u);
+			}
+			continue;
+		}
 		auto& count = counts[static_cast<size_t>(*kind)];
 		image.descriptor_index = count;
 		if (count == 0) image_types[IR::ImageBindingIndex(*kind)] = &image;
@@ -331,21 +350,11 @@ void DefineDescriptors(EmitterState& state) {
 			default: {
 				EXIT_IF(IR::ImageBindingResourceClass(kind) ==
 				        IR::ImageResourceClass::None);
-				const auto& image = *image_types[IR::ImageBindingIndex(kind)];
-				const auto  name  = "image_" + std::to_string(static_cast<uint32_t>(kind));
 				auto& definition = state.images[IR::ImageBindingIndex(kind)];
-				definition.type = ImageType(state, image);
-				definition.variable = Define(ArrayType(definition.type), name.c_str(),
-				                             spv::StorageClassUniformConstant);
-				definition.pointer_type = TypePointer(
-				    state, spv::StorageClassUniformConstant, definition.type);
-				if (image.dimension == ImageDimension::Dim1D ||
-				    image.dimension == ImageDimension::Dim1DArray) {
-					state.builder.RequireCapability(image.resource_class ==
-					                                        IR::ImageResourceClass::Sampled
-					                                    ? spv::CapabilitySampled1D
-					                                    : spv::CapabilityImage1D);
-				}
+				DefineImageArray(state, definition, *image_types[IR::ImageBindingIndex(kind)],
+				                 counts[index], 0u, IR::NativeBinding(state.program.stage, kind));
+				state.builder.AddName(definition.variable,
+				                      ("image_" + std::to_string(static_cast<uint32_t>(kind))).c_str());
 				break;
 			}
 		}
@@ -752,7 +761,7 @@ void DefineModule(EmitterState& state) {
 	if (state.program.info.buffer_u16) {
 		state.builder.RequireCapability(spv::CapabilityStorageBuffer16BitAccess);
 	}
-	if (state.program.info.uses_dma) {
+	if (state.program.info.uses_dma || state.program.info.uses_image_heap) {
 		state.builder.RequireCapability(spv::CapabilityInt64);
 		state.builder.RequireCapability(spv::CapabilityPhysicalStorageBufferAddresses);
 		state.builder.RequireExtension("SPV_KHR_physical_storage_buffer");
@@ -820,7 +829,7 @@ void DefineModule(EmitterState& state) {
 		state.builder.RequireExtension("SPV_KHR_fragment_shader_barycentric");
 	}
 	state.builder.RequireExtension("SPV_KHR_float_controls");
-	state.builder.AddMemoryModel(state.program.info.uses_dma
+	state.builder.AddMemoryModel(state.program.info.uses_dma || state.program.info.uses_image_heap
 	                                 ? spv::AddressingModelPhysicalStorageBuffer64
 	                                 : spv::AddressingModelLogical,
 	                             spv::MemoryModelGLSL450);

src/graphics/shader/recompiler/backend/spirv/spirvEmitterProgram.cpp

@@ -16,10 +16,8 @@ namespace {
 void EmitKillIfBoolFalse(EmitterState& state, uint32_t active) {
 	const auto kill_label  = state.builder.AllocateId();
 	const auto merge_label = state.builder.AllocateId();
-	const auto inactive    = state.builder.AllocateId();
-	state.builder.AddFunction(spv::OpLogicalNot, TypeBool(state), inactive, active);
 	state.builder.AddFunction(spv::OpSelectionMerge, merge_label, spv::SelectionControlMaskNone);
-	state.builder.AddFunction(spv::OpBranchConditional, inactive, kill_label, merge_label);
+	state.builder.AddFunction(spv::OpBranchConditional, active, merge_label, kill_label);
 	EmitLabel(state, kill_label);
 	state.builder.AddFunction(spv::OpKill);
 	EmitLabel(state, merge_label);
@@ -495,29 +493,24 @@ uint32_t ValueEmitContext::Ballot(IR::Value predicate) {
 }
 
 uint32_t ValueEmitContext::FirstLane(uint32_t ballot) {
-	if (other_half == nullptr) {
-		const auto result = state.builder.AllocateId();
-		state.builder.AddFunction(spv::OpGroupNonUniformBallotFindLSB, TypeU32(state), result,
-		                          ConstantU32(state, spv::ScopeSubgroup), ballot);
-		return result;
+	// An empty ballot (EXEC zero) gives lane 0, which V_READFIRSTLANE reads on hardware. FindILsb
+	// returns -1 for zero, unlike BallotFindLSB, whose result is then undefined.
+	const auto first = [&](uint32_t word) {
+		const auto bits = state.builder.AllocateId();
+		state.builder.AddFunction(spv::OpCompositeExtract, TypeU32(state), bits, ballot, word);
+		return std::pair {bits, EmitGlsl<GLSLstd450FindILsb, IR::Type::U32>(state, bits)};
+	};
+	const auto [low, low_first] = first(0);
+	if (state.program.wave_size == 32u) {
+		return EmitGlsl<GLSLstd450SMax, IR::Type::U32>(state, low_first, ConstantU32(state, 0));
 	}
-	const auto low        = state.builder.AllocateId();
-	const auto high       = state.builder.AllocateId();
-	const auto low_first  = state.builder.AllocateId();
-	const auto high_first = state.builder.AllocateId();
-	const auto low_active = state.builder.AllocateId();
-	const auto result     = state.builder.AllocateId();
-	state.builder.AddFunction(spv::OpCompositeExtract, TypeU32(state), low, ballot, 0);
-	state.builder.AddFunction(spv::OpCompositeExtract, TypeU32(state), high, ballot, 1);
-	state.builder.AddFunction(spv::OpExtInst, TypeU32(state), low_first, GlslStd450(state),
-	                          GLSLstd450FindILsb, low);
-	state.builder.AddFunction(spv::OpExtInst, TypeU32(state), high_first, GlslStd450(state),
-	                          GLSLstd450FindILsb, high);
-	state.builder.AddFunction(spv::OpINotEqual, TypeBool(state), low_active, low,
-	                          ConstantU32(state, 0));
-	state.builder.AddFunction(spv::OpSelect, TypeU32(state), result, low_active, low_first,
-	                          EmitAddU32(state, high_first, ConstantU32(state, 32)));
-	return result;
+	const auto high_first = first(1).second;
+	const auto high_lane  = EmitGlsl<GLSLstd450SMax, IR::Type::U32>(
+	    state, EmitBinaryU32(state, spv::OpBitwiseOr, high_first, ConstantU32(state, 32)),
+	    ConstantU32(state, 0));
+	return Select(state, TypeU32(state),
+	              Binary(state, spv::OpINotEqual, TypeBool(state), low, ConstantU32(state, 0)),
+	              low_first, high_lane);
 }
 
 uint32_t ValueEmitContext::Shuffle(const IR::Inst& inst, size_t index, uint32_t lane) {

src/graphics/shader/recompiler/frontend/decode/MemoryOps.cpp

@@ -6,13 +6,14 @@ namespace Libs::Graphics::ShaderRecompiler::Decoder {
 namespace {
 
 struct MemoryOpcodeInfo {
-	uint32_t encoding    = 0;
-	Opcode   decoded     = Opcode::UNSUPPORTED;
-	uint32_t data_dwords = 1;
-	uint32_t data_bits   = 32;
-	bool     data_signed = false;
-	bool     typed       = false;
-	bool     formatted   = false;
+	uint32_t encoding        = 0;
+	Opcode   decoded         = Opcode::UNSUPPORTED;
+	uint32_t data_dwords     = 1;
+	uint32_t data_bits       = 32;
+	bool     data_signed     = false;
+	bool     typed           = false;
+	bool     formatted       = false;
+	uint32_t data_components = 0; // D16 formats pack two components per dword
 };
 
 constexpr MemoryOpcodeInfo SMEM_OPCODE_LIST[] = {
@@ -73,9 +74,13 @@ constexpr MemoryOpcodeInfo MUBUF_OPCODE_LIST[] = {
     {0x71u, Opcode::BUFFER_GL0_INV, 0, 0},
     {0x72u, Opcode::BUFFER_GL1_INV, 0, 0},
     {0x80u, Opcode::BUFFER_LOAD_FORMAT_D16_X, 1, 16, false, false, true},
-    {0x83u, Opcode::BUFFER_LOAD_FORMAT_D16_XYZW, 2, 16, false, false, true},
+    {0x81u, Opcode::BUFFER_LOAD_FORMAT_D16_XY, 1, 16, false, false, true, 2},
+    {0x82u, Opcode::BUFFER_LOAD_FORMAT_D16_XYZ, 2, 16, false, false, true, 3},
+    {0x83u, Opcode::BUFFER_LOAD_FORMAT_D16_XYZW, 2, 16, false, false, true, 4},
     {0x84u, Opcode::BUFFER_STORE_FORMAT_D16_X, 1, 16, false, false, true},
-    {0x87u, Opcode::BUFFER_STORE_FORMAT_D16_XYZW, 2, 16, false, false, true},
+    {0x85u, Opcode::BUFFER_STORE_FORMAT_D16_XY, 1, 16, false, false, true, 2},
+    {0x86u, Opcode::BUFFER_STORE_FORMAT_D16_XYZ, 2, 16, false, false, true, 3},
+    {0x87u, Opcode::BUFFER_STORE_FORMAT_D16_XYZW, 2, 16, false, false, true, 4},
 };
 
 constexpr MemoryOpcodeInfo MTBUF_OPCODE_LIST[] = {
@@ -158,12 +163,13 @@ void ApplyMemoryInfo(Instruction& inst, const MemoryOpcodeInfo* info) {
 		inst.opcode = Opcode::UNSUPPORTED;
 		return;
 	}
-	inst.opcode      = info->decoded;
-	inst.data_dwords = info->data_dwords;
-	inst.data_bits   = info->data_bits;
-	inst.data_signed = info->data_signed;
-	inst.typed       = info->typed;
-	inst.formatted   = info->formatted;
+	inst.opcode          = info->decoded;
+	inst.data_dwords     = info->data_dwords;
+	inst.data_bits       = info->data_bits;
+	inst.data_signed     = info->data_signed;
+	inst.typed           = info->typed;
+	inst.formatted       = info->formatted;
+	inst.data_components = info->data_components;
 }
 
 } // namespace
@@ -323,12 +329,7 @@ void DecodeMubuf(uint32_t pc, std::span<const uint32_t> code, uint32_t word_inde
 	}
 
 	DecodeVectorGpr(vdata, inst.dst);
-	if (inst.opcode == Opcode::BUFFER_LOAD_FORMAT_D16_XYZW ||
-	    inst.opcode == Opcode::BUFFER_STORE_FORMAT_D16_XYZW) {
-		inst.data_components = 4u;
-	}
-	if (inst.opcode == Opcode::BUFFER_LOAD_FORMAT_D16_X ||
-	    inst.opcode == Opcode::BUFFER_LOAD_UBYTE_D16 ||
+	if (inst.opcode == Opcode::BUFFER_LOAD_UBYTE_D16 ||
 	    inst.opcode == Opcode::BUFFER_LOAD_SHORT_D16 ||
 	    inst.opcode == Opcode::BUFFER_LOAD_SHORT_D16_HI) {
 		inst.dst.sdwa_sel = inst.opcode == Opcode::BUFFER_LOAD_SHORT_D16_HI ? 5u : 4u;

src/graphics/shader/recompiler/frontend/decode/ShaderDecoder.cpp

@@ -636,6 +636,8 @@ std::string InstructionToString(const Instruction& inst) {
 		case Opcode::BUFFER_LOAD_FORMAT_XYZ:
 		case Opcode::BUFFER_LOAD_FORMAT_XYZW:
 		case Opcode::BUFFER_LOAD_FORMAT_D16_X:
+		case Opcode::BUFFER_LOAD_FORMAT_D16_XY:
+		case Opcode::BUFFER_LOAD_FORMAT_D16_XYZ:
 		case Opcode::BUFFER_LOAD_FORMAT_D16_XYZW:
 		case Opcode::BUFFER_STORE_FORMAT_X:
 		case Opcode::BUFFER_STORE_FORMAT_XY:
@@ -643,6 +645,8 @@ std::string InstructionToString(const Instruction& inst) {
 		case Opcode::BUFFER_STORE_FORMAT_XYZW:
 		case Opcode::BUFFER_STORE_FORMAT_D16_X:
 		case Opcode::BUFFER_STORE_FORMAT_D16_HI_X:
+		case Opcode::BUFFER_STORE_FORMAT_D16_XY:
+		case Opcode::BUFFER_STORE_FORMAT_D16_XYZ:
 		case Opcode::BUFFER_STORE_FORMAT_D16_XYZW:
 		case Opcode::BUFFER_LOAD_UBYTE:
 		case Opcode::BUFFER_LOAD_UBYTE_D16:

src/graphics/shader/recompiler/frontend/decode/ShaderDecoder.h

@@ -547,6 +547,8 @@ enum class Opcode {
 	BUFFER_LOAD_FORMAT_XYZ,
 	BUFFER_LOAD_FORMAT_XYZW,
 	BUFFER_LOAD_FORMAT_D16_X,
+	BUFFER_LOAD_FORMAT_D16_XY,
+	BUFFER_LOAD_FORMAT_D16_XYZ,
 	BUFFER_LOAD_FORMAT_D16_XYZW,
 	BUFFER_STORE_FORMAT_X,
 	BUFFER_STORE_FORMAT_XY,
@@ -554,6 +556,8 @@ enum class Opcode {
 	BUFFER_STORE_FORMAT_XYZW,
 	BUFFER_STORE_FORMAT_D16_X,
 	BUFFER_STORE_FORMAT_D16_HI_X,
+	BUFFER_STORE_FORMAT_D16_XY,
+	BUFFER_STORE_FORMAT_D16_XYZ,
 	BUFFER_STORE_FORMAT_D16_XYZW,
 	BUFFER_LOAD_UBYTE,
 	BUFFER_LOAD_UBYTE_D16,
@@ -713,6 +717,8 @@ enum class Opcode {
 	V_INTERP_P1_F32,
 	V_INTERP_P2_F32,
 	V_INTERP_MOV_F32,
+	V_INTERP_P1LL_F16,
+	V_INTERP_P2_F16,
 
 	S_NOP,
 	S_WAITCNT,

src/graphics/shader/recompiler/frontend/decode/VectorAluOps.cpp

@@ -391,6 +391,8 @@ constexpr OpcodeMap VOP3_OPCODE_LIST[] = {
     {0x377u, Opcode::V_PERMLANE16_B32},
     {0x378u, Opcode::V_PERMLANEX16_B32},
     {0x34bu, Opcode::V_FMA_F16},
+    {0x342u, Opcode::V_INTERP_P1LL_F16},
+    {0x35au, Opcode::V_INTERP_P2_F16},
     {0x36du, Opcode::V_ADD3_U32},
     {0x14fu, Opcode::V_ALIGNBYTE_B32},
 };
@@ -1553,6 +1555,9 @@ bool HasUnsupportedNativeVop3Modifiers(Opcode opcode, bool permlane, bool mad_mi
 	}
 	switch (opcode) {
 		case Opcode::V_LDEXP_F32: return (abs & ~1u) != 0u || op_sel != 0u || (neg & ~1u) != 0u;
+		case Opcode::V_INTERP_P1LL_F16:
+		case Opcode::V_INTERP_P2_F16:
+			return abs != 0u || (op_sel & 0x7u) != 0u || clamp != 0u || omod != 0u || neg != 0u;
 		case Opcode::V_CNDMASK_B32:
 			return (abs & ~0x3u) != 0u || op_sel != 0u || clamp != 0u || omod != 0u ||
 			       (neg & ~0x3u) != 0u;
@@ -1664,6 +1669,17 @@ bool IsVopcCompareExec(Opcode opcode) {
 	}
 }
 
+// VINTRP and VOP3 interpolation share one layout: src0 = I/J VGPR, src1 = attr, src2 = chan.
+void SetInterpolationAttribute(Instruction& inst, uint32_t attr, uint32_t chan) {
+	inst.src1.kind       = OperandKind::IntegerInlineConstant;
+	inst.src1.value      = attr;
+	inst.src1.signed_val = static_cast<int32_t>(attr);
+	inst.src2.kind       = OperandKind::IntegerInlineConstant;
+	inst.src2.value      = chan;
+	inst.src2.signed_val = static_cast<int32_t>(chan);
+	inst.src_count       = 3;
+}
+
 } // namespace
 
 void DecodeVop2(uint32_t pc, std::span<const uint32_t> code, uint32_t word_index,
@@ -1830,6 +1846,15 @@ void DecodeVop3(uint32_t pc, std::span<const uint32_t> code, uint32_t word_index
 		inst.src_count = 0;
 		return;
 	}
+	if (inst.opcode == Opcode::V_INTERP_P1LL_F16 || inst.opcode == Opcode::V_INTERP_P2_F16) {
+		// SRC0 packs {attr_word, attr_chan, attr}; attr_word selects the high f16 of the dword.
+		DecodeVectorGpr(vdst, inst.dst);
+		DecodeScalarSource(src1, pc, inst.src0);
+		SetInterpolationAttribute(inst, src0 & 0x3fu, (src0 >> 6u) & 0x3u);
+		inst.src1.op_sel  = (src0 & 0x100u) != 0u;
+		inst.dst.sdwa_sel = (op_sel & 0x8u) != 0u ? 5u : 4u;
+		return;
+	}
 	if (compare_exec) {
 		inst.dst.kind = OperandKind::ExecLo;
 	} else if (scalar_dst) {
@@ -2004,13 +2029,7 @@ void DecodeVintrp(uint32_t pc, std::span<const uint32_t> code, uint32_t word_ind
 	} else {
 		DecodeVectorGpr(vsrc, inst.src0);
 	}
-	inst.src1.kind       = OperandKind::IntegerInlineConstant;
-	inst.src1.value      = attr;
-	inst.src1.signed_val = static_cast<int32_t>(attr);
-	inst.src2.kind       = OperandKind::IntegerInlineConstant;
-	inst.src2.value      = chan;
-	inst.src2.signed_val = static_cast<int32_t>(chan);
-	inst.src_count       = 3;
+	SetInterpolationAttribute(inst, attr, chan);
 }
 
 } // namespace Libs::Graphics::ShaderRecompiler::Decoder

src/graphics/shader/recompiler/frontend/translate/Attribute.cpp

@@ -73,14 +73,19 @@ void Translator::TranslateEmbeddedFetch(const Decoder::Instruction& inst, uint32
 	}
 }
 
-void Translator::V_INTERP_P1_F32() {}
-
 void Translator::V_INTERP_P2_F32(const Decoder::Instruction& inst) {
 	const auto value = ir.Emit(IR::ValueOpcode::GetAttribute,
 	                           {IR::Value(inst.src1.value), IR::Value(inst.src2.value)});
 	WriteOperand(inst.dst, value);
 }
 
+void Translator::V_INTERP_P2_F16(const Decoder::Instruction& inst) {
+	const auto value = ir.Emit(IR::ValueOpcode::GetAttributeF16,
+	                           {IR::Value(inst.src1.value), IR::Value(inst.src2.value),
+	                            IR::Value(uint32_t {inst.src1.op_sel})});
+	WriteF16(inst.dst, IR::F32(value));
+}
+
 void Translator::V_INTERP_MOV_F32(const Decoder::Instruction& inst) {
 	if (inst.src0.value >= 3u) {
 		EXIT("v_interp_mov_f32 mode %u is reserved at pc 0x%08x", inst.src0.value, inst.pc);
@@ -108,8 +113,11 @@ void Translator::EXP(const Decoder::Instruction& inst) {
 
 void Translator::EmitInterpolation(const Decoder::Instruction& inst) {
 	switch (inst.opcode) {
-		case Decoder::Opcode::V_INTERP_P1_F32: V_INTERP_P1_F32(); return;
+		// P2 interpolates the whole attribute, so the P1 halves have nothing to compute.
+		case Decoder::Opcode::V_INTERP_P1_F32:
+		case Decoder::Opcode::V_INTERP_P1LL_F16: return;
 		case Decoder::Opcode::V_INTERP_P2_F32: V_INTERP_P2_F32(inst); return;
+		case Decoder::Opcode::V_INTERP_P2_F16: V_INTERP_P2_F16(inst); return;
 		case Decoder::Opcode::V_INTERP_MOV_F32: V_INTERP_MOV_F32(inst); return;
 		default: return FailMissingTranslation(inst);
 	}

src/graphics/shader/recompiler/frontend/translate/Control.cpp

@@ -82,19 +82,17 @@ void Translator::S_SAVEEXEC(const Decoder::Instruction& inst, IR::ValueOpcode op
 }
 
 void Translator::ADD_U32(const Decoder::Instruction& inst, bool vector, bool use_carry_in) {
-	const auto zero     = IR::U32(IR::Value(0u));
-	const auto lhs      = ReadU32(inst.src0);
-	const auto rhs      = ReadU32(inst.src1);
-	const auto carry_in = !use_carry_in ? zero
-	                      : vector      ? ConditionBit(inst.src2)
-	                                    : ConditionBit(ConditionOperand(Decoder::OperandKind::Scc));
-	const auto add0     = ir.Emit(IR::ValueOpcode::IAddCarry32, {lhs, rhs});
-	const auto partial  = ir.CompositeExtract(add0, 0);
-	const auto carry0   = ir.CompositeExtract(add0, 1);
-	const auto add1     = ir.Emit(IR::ValueOpcode::IAddCarry32, {partial, carry_in});
-	const auto result   = ir.CompositeExtract(add1, 0);
-	const auto carry1   = ir.CompositeExtract(add1, 1);
-	const auto carry    = ir.INotEqual(ir.BitwiseOr(carry0, carry1), zero);
+	const auto lhs     = ReadU32(inst.src0);
+	const auto rhs     = ReadU32(inst.src1);
+	const auto partial = ir.IAdd(lhs, rhs);
+	auto       result  = partial;
+	auto       carry   = ir.ULessThan(partial, lhs);
+	if (use_carry_in) {
+		// The carry-in is 0 or 1, so the second add carries out only when it wraps to zero.
+		result = ir.IAdd(partial, vector ? ConditionBit(inst.src2)
+		                                 : ConditionBit(ConditionOperand(Decoder::OperandKind::Scc)));
+		carry  = ir.LogicalOr(carry, ir.ULessThan(result, partial));
+	}
 	WriteOperand(DestinationOperand(inst), result);
 	if (vector) {
 		WriteMask(inst.dst2, ir.LogicalAnd(ir.GetExec(), carry));
@@ -142,29 +140,25 @@ void Translator::S_ABSDIFF_I32(const Decoder::Instruction& inst) {
 }
 
 void Translator::S_ADD_SUB_I32(const Decoder::Instruction& inst, bool subtract) {
-	const auto lhs      = ReadU32(inst.src0);
-	const auto rhs      = ReadU32(inst.src1);
-	const auto result   = subtract ? ir.ISub(lhs, rhs) : ir.IAdd(lhs, rhs);
-	const auto shift    = IR::U32(IR::Value(31u));
-	const auto lhs_sign = ir.ShiftRightLogical(lhs, shift);
-	const auto rhs_sign = ir.ShiftRightLogical(rhs, shift);
-	const auto out_sign = ir.ShiftRightLogical(result, shift);
-	const auto inputs = subtract ? ir.INotEqual(lhs_sign, rhs_sign) : ir.IEqual(lhs_sign, rhs_sign);
-	const auto changed = ir.INotEqual(lhs_sign, out_sign);
+	const auto lhs    = ReadU32(inst.src0);
+	const auto rhs    = ReadU32(inst.src1);
+	const auto result = subtract ? ir.ISub(lhs, rhs) : ir.IAdd(lhs, rhs);
+	// Signed overflow moves the result past lhs the wrong way for the sign of rhs.
+	const auto ge = [&](IR::Value a, IR::Value b) {
+		return IR::U1(ir.Emit(IR::ValueOpcode::SGreaterThanEqual32, {a, b}));
+	};
+	const auto moved = subtract ? ge(lhs, result) : ge(result, lhs);
 	WriteOperand(DestinationOperand(inst), result);
-	ir.SetScc(ir.LogicalAnd(inputs, changed));
+	ir.SetScc(IR::U1(ir.Emit(IR::ValueOpcode::LogicalXor, {moved, ge(rhs, IR::Value(0u))})));
 }
 
 void Translator::S_LSHL_ADD_U32(const Decoder::Instruction& inst, uint32_t shift_amount) {
-	const auto lhs           = ReadU32(inst.src0);
-	const auto shift         = IR::U32(IR::Value(shift_amount));
-	const auto rhs           = ReadU32(inst.src1);
-	const auto shifted       = ir.ShiftLeftLogical(lhs, shift);
-	const auto result        = ir.IAdd(shifted, rhs);
-	const auto add_carry     = ir.ULessThan(result, shifted);
-	const auto inverse_shift = ir.ISub(IR::U32(IR::Value(32u)), shift);
-	const auto shifted_out   = ir.ShiftRightLogical(lhs, inverse_shift);
-	const auto shift_carry   = ir.INotEqual(shifted_out, IR::U32(IR::Value(0u)));
+	const auto lhs         = ReadU32(inst.src0);
+	const auto rhs         = ReadU32(inst.src1);
+	const auto shifted     = ir.ShiftLeftLogical(lhs, IR::U32(IR::Value(shift_amount)));
+	const auto result      = ir.IAdd(shifted, rhs);
+	const auto add_carry   = ir.ULessThan(result, shifted);
+	const auto shift_carry = ir.UGreaterThan(lhs, IR::U32(IR::Value(~0u >> shift_amount)));
 	WriteOperand(DestinationOperand(inst), result);
 	ir.SetScc(ir.LogicalOr(add_carry, shift_carry));
 }

src/graphics/shader/recompiler/frontend/translate/Convert.cpp

@@ -6,43 +6,10 @@ IR::F32 Translator::SelectF32(IR::U1 condition, IR::F32 true_value, IR::F32 fals
 	return IR::F32(ir.Emit(IR::ValueOpcode::SelectF32, {condition, true_value, false_value}));
 }
 
-IR::U32 Translator::ConvertF32ToU32Saturated(IR::F32 value, float upper_bound, float safe_upper,
-                                             uint32_t upper_result) {
-	const auto zero = IR::F32(IR::Value::F32(0.0f));
-	const auto nan  = IR::U1(ir.Emit(IR::ValueOpcode::FPIsNan32, {value}));
-	const auto low  = IR::U1(ir.Emit(IR::ValueOpcode::FPOrdLessThanEqual32, {value, zero}));
-	const auto high = IR::U1(
-	    ir.Emit(IR::ValueOpcode::FPOrdGreaterThanEqual32, {value, IR::Value::F32(upper_bound)}));
-	const auto truncated = IR::F32(ir.Emit(IR::ValueOpcode::FPTrunc32, {value}));
-	const auto safe_low  = SelectF32(ir.LogicalOr(nan, low), zero, truncated);
-	const auto safe      = SelectF32(high, IR::F32(IR::Value::F32(safe_upper)), safe_low);
-	const auto converted = IR::U32(ir.Emit(IR::ValueOpcode::ConvertU32F32, {safe}));
-	return ir.Select(high, IR::U32(IR::Value(upper_result)), converted);
-}
-
-IR::U32 Translator::ConvertF32ToI32Saturated(IR::F32 value, float lower_bound, float upper_bound,
-                                             float safe_upper, uint32_t lower_result,
-                                             uint32_t upper_result) {
-	const auto nan = IR::U1(ir.Emit(IR::ValueOpcode::FPIsNan32, {value}));
-	const auto low = IR::U1(
-	    ir.Emit(IR::ValueOpcode::FPOrdLessThanEqual32, {value, IR::Value::F32(lower_bound)}));
-	const auto high = IR::U1(
-	    ir.Emit(IR::ValueOpcode::FPOrdGreaterThanEqual32, {value, IR::Value::F32(upper_bound)}));
-	const auto truncated    = IR::F32(ir.Emit(IR::ValueOpcode::FPTrunc32, {value}));
-	const auto safe_low     = SelectF32(low, IR::F32(IR::Value::F32(lower_bound)), truncated);
-	const auto safe_high    = SelectF32(high, IR::F32(IR::Value::F32(safe_upper)), safe_low);
-	const auto safe         = SelectF32(nan, IR::F32(IR::Value::F32(0.0f)), safe_high);
-	const auto converted    = IR::U32(ir.Emit(IR::ValueOpcode::ConvertS32F32, {safe}));
-	const auto clamped_high = ir.Select(high, IR::U32(IR::Value(upper_result)), converted);
-	const auto clamped      = ir.Select(low, IR::U32(IR::Value(lower_result)), clamped_high);
-	return ir.Select(nan, IR::U32(IR::Value(0u)), clamped);
-}
-
 void Translator::V_CVT_F32_UBYTE(const Decoder::Instruction& inst, uint32_t byte_index) {
 	const auto source = ReadU32(SourceAt(inst, 0));
-	const auto byte =
-	    ir.BitwiseAnd(ir.ShiftRightLogical(source, IR::U32(IR::Value(byte_index * 8u))),
-	                  IR::U32(IR::Value(0xffu)));
+	const auto byte   = ir.Emit(IR::ValueOpcode::BitFieldUExtract,
+	                            {source, IR::Value(byte_index * 8u), IR::Value(8u)});
 	WriteOperand(DestinationOperand(inst), ir.Emit(IR::ValueOpcode::ConvertF32U32, {byte}));
 }
 
@@ -58,15 +25,12 @@ void Translator::V_CVT_F32_I32(const Decoder::Instruction& inst) {
 
 void Translator::V_CVT_U32_F32(const Decoder::Instruction& inst) {
 	const auto value = IR::F32(ReadOperand(SourceAt(inst, 0), IR::Type::F32));
-	WriteOperand(DestinationOperand(inst),
-	             ConvertF32ToU32Saturated(value, 4294967296.0f, 4294967040.0f, 0xffffffffu));
+	WriteOperand(DestinationOperand(inst), ir.Emit(IR::ValueOpcode::ConvertU32F32, {value}));
 }
 
 void Translator::V_CVT_I32_F32(const Decoder::Instruction& inst) {
 	const auto value = IR::F32(ReadOperand(SourceAt(inst, 0), IR::Type::F32));
-	WriteOperand(DestinationOperand(inst),
-	             ConvertF32ToI32Saturated(value, -2147483648.0f, 2147483648.0f, 2147483520.0f,
-	                                      0x80000000u, 0x7fffffffu));
+	WriteOperand(DestinationOperand(inst), ir.Emit(IR::ValueOpcode::ConvertS32F32, {value}));
 }
 
 void Translator::V_CVT_F16_F32(const Decoder::Instruction& inst) {
@@ -86,31 +50,30 @@ void Translator::V_CVT_F16_16(const Decoder::Instruction& inst, bool signed_valu
 
 void Translator::V_CVT_16_F16(const Decoder::Instruction& inst, bool signed_value) {
 	const auto value = ReadF16AsF32(inst.src0);
+	// The 32-bit conversion already saturates and maps NaN to zero.
 	if (signed_value) {
-		const auto converted =
-		    ConvertF32ToI32Saturated(value, -32768.0f, 32768.0f, 32767.0f, 0xffff8000u, 0x7fffu);
-		Write16Bits(DestinationOperand(inst), ir.BitwiseAnd(converted, IR::U32(IR::Value(0xffffu))));
+		const auto converted = ir.Emit(IR::ValueOpcode::ConvertS32F32, {value});
+		const auto low = ir.Emit(IR::ValueOpcode::SMax32, {converted, IR::Value(0xffff8000u)});
+		Write16Bits(DestinationOperand(inst),
+		            IR::U32(ir.Emit(IR::ValueOpcode::SMin32, {low, IR::Value(0x7fffu)})));
 		return;
 	}
+	const auto converted = ir.Emit(IR::ValueOpcode::ConvertU32F32, {value});
 	Write16Bits(DestinationOperand(inst),
-	         ConvertF32ToU32Saturated(value, 65536.0f, 65535.0f, 0xffffu));
+	            IR::U32(ir.Emit(IR::ValueOpcode::UMin32, {converted, IR::Value(0xffffu)})));
 }
 
 void Translator::V_CVT_RPI_I32_F32(const Decoder::Instruction& inst) {
 	const auto source  = IR::F32(ReadOperand(SourceAt(inst, 0), IR::Type::F32));
 	const auto biased  = IR::F32(ir.Emit(IR::ValueOpcode::FPAdd32, {source, IR::Value::F32(0.5f)}));
 	const auto rounded = IR::F32(ir.Emit(IR::ValueOpcode::FPFloor32, {biased}));
-	WriteOperand(DestinationOperand(inst),
-	             ConvertF32ToI32Saturated(rounded, -2147483648.0f, 2147483648.0f, 2147483520.0f,
-	                                      0x80000000u, 0x7fffffffu));
+	WriteOperand(DestinationOperand(inst), ir.Emit(IR::ValueOpcode::ConvertS32F32, {rounded}));
 }
 
 void Translator::V_CVT_FLR_I32_F32(const Decoder::Instruction& inst) {
 	const auto source  = IR::F32(ReadOperand(SourceAt(inst, 0), IR::Type::F32));
 	const auto rounded = IR::F32(ir.Emit(IR::ValueOpcode::FPFloor32, {source}));
-	WriteOperand(DestinationOperand(inst),
-	             ConvertF32ToI32Saturated(rounded, -2147483648.0f, 2147483648.0f, 2147483520.0f,
-	                                      0x80000000u, 0x7fffffffu));
+	WriteOperand(DestinationOperand(inst), ir.Emit(IR::ValueOpcode::ConvertS32F32, {rounded}));
 }
 
 void Translator::V_FREXP_EXP_I32_F32(const Decoder::Instruction& inst) {
@@ -157,25 +120,24 @@ void Translator::V_CVT_PKNORM_F32(const Decoder::Instruction& inst, IR::ValueOpc
 
 void Translator::V_CVT_PK_U8_F32(const Decoder::Instruction& inst) {
 	const auto source = IR::F32(ReadOperand(SourceAt(inst, 0), IR::Type::F32));
-	const auto byte   = ConvertF32ToU32Saturated(source, 255.0f, 255.0f, 255u);
+	const auto byte =
+	    IR::U32(ir.Emit(IR::ValueOpcode::UMin32,
+		                {ir.Emit(IR::ValueOpcode::ConvertU32F32, {source}), IR::Value(255u)}));
 	const auto index  = ir.BitwiseAnd(ReadU32(SourceAt(inst, 1)), IR::U32(IR::Value(3u)));
 	const auto shift  = ir.ShiftLeftLogical(index, IR::U32(IR::Value(3u)));
-	const auto mask   = ir.ShiftLeftLogical(IR::U32(IR::Value(0xffu)), shift);
-	const auto base   = ir.BitwiseAnd(ReadU32(SourceAt(inst, 2)), ir.BitwiseNot(mask));
-	WriteOperand(DestinationOperand(inst), ir.BitwiseOr(base, ir.ShiftLeftLogical(byte, shift)));
+	WriteOperand(DestinationOperand(inst),
+	             ir.Emit(IR::ValueOpcode::BitFieldInsert,
+	                     {ReadU32(SourceAt(inst, 2)), byte, shift, IR::Value(8u)}));
 }
 
 void Translator::V_PACK_B32_F16(const Decoder::Instruction& inst) {
-	const auto low = Read16LaneBits(inst.src0, false);
-	const auto high =
-	    ir.ShiftLeftLogical(Read16LaneBits(inst.src1, false), IR::U32(IR::Value(16u)));
-	WriteOperand(DestinationOperand(inst), ir.BitwiseOr(low, high));
+	WriteOperand(DestinationOperand(inst),
+	             PackU16Lanes(Read16LaneBits(inst.src0, false), Read16LaneBits(inst.src1, false)));
 }
 
 IR::U32 Translator::PackU16Lanes(IR::U32 low, IR::U32 high) {
-	const auto mask = IR::U32(IR::Value(0xffffu));
-	return ir.BitwiseOr(ir.BitwiseAnd(low, mask),
-	                    ir.ShiftLeftLogical(ir.BitwiseAnd(high, mask), IR::U32(IR::Value(16u))));
+	return IR::U32(
+	    ir.Emit(IR::ValueOpcode::BitFieldInsert, {low, high, IR::Value(16u), IR::Value(16u)}));
 }
 
 } // namespace Libs::Graphics::ShaderRecompiler::Frontend

src/graphics/shader/recompiler/frontend/translate/Float.cpp

@@ -222,13 +222,10 @@ void Translator::FloatCube(const Decoder::Instruction& inst, uint32_t result_kin
 			break;
 		case 1u: result = select_face(SelectF32(x_neg, z, nz), x, SelectF32(z_neg, nx, x)); break;
 		case 2u: result = select_face(ny, SelectF32(y_neg, nz, z), ny); break;
-		case 3u: {
-			const auto two = IR::F32(IR::Value::F32(2.0f));
-			result         = select_face(IR::F32(ir.Emit(IR::ValueOpcode::FPMul32, {x, two})),
-			                             IR::F32(ir.Emit(IR::ValueOpcode::FPMul32, {y, two})),
-			                             IR::F32(ir.Emit(IR::ValueOpcode::FPMul32, {z, two})));
+		case 3u:
+			result = IR::F32(
+			    ir.Emit(IR::ValueOpcode::FPMul32, {select_face(x, y, z), IR::Value::F32(2.0f)}));
 			break;
-		}
 		default: EXIT("invalid cube result kind");
 	}
 	WriteOperand(DestinationOperand(inst), result);

src/graphics/shader/recompiler/frontend/translate/Integer.cpp

@@ -9,7 +9,7 @@ void Translator::Integer16Shift(const Decoder::Instruction& inst, IR::ValueOpcod
 	const auto value  = ReadU16AsU32(inst.src1, arithmetic);
 	const auto count  = ir.BitwiseAnd(ReadU16AsU32(inst.src0, false), IR::U32(IR::Value(15u)));
 	const auto result = IR::U32(ir.Emit(opcode, {value, count}));
-	Write16Bits(DestinationOperand(inst), ir.BitwiseAnd(result, IR::U32(IR::Value(0xffffu))));
+	Write16Bits(DestinationOperand(inst), result);
 }
 
 void Translator::Integer16Binary(const Decoder::Instruction& inst, IR::ValueOpcode opcode,
@@ -17,7 +17,7 @@ void Translator::Integer16Binary(const Decoder::Instruction& inst, IR::ValueOpco
 	const auto lhs    = ReadU16AsU32(inst.src0, sign);
 	const auto rhs    = ReadU16AsU32(inst.src1, sign);
 	const auto result = IR::U32(ir.Emit(opcode, {lhs, rhs}));
-	Write16Bits(DestinationOperand(inst), ir.BitwiseAnd(result, IR::U32(IR::Value(0xffffu))));
+	Write16Bits(DestinationOperand(inst), result);
 }
 
 void Translator::V_MAD_I16(const Decoder::Instruction& inst) {
@@ -28,7 +28,7 @@ void Translator::V_MAD_I16(const Decoder::Instruction& inst) {
 		result = IR::U32(ir.Emit(IR::ValueOpcode::SMax32, {result, IR::Value(0xffff8000u)}));
 		result = IR::U32(ir.Emit(IR::ValueOpcode::SMin32, {result, IR::Value(0x7fffu)}));
 	}
-	Write16Bits(DestinationOperand(inst), ir.BitwiseAnd(result, IR::U32(IR::Value(0xffffu))));
+	Write16Bits(DestinationOperand(inst), result);
 }
 
 void Translator::Integer16Ternary(const Decoder::Instruction& inst, IR::ValueOpcode opcode,
@@ -221,19 +221,21 @@ void Translator::V_XOR3_B32(const Decoder::Instruction& inst) {
 	WriteOperand(DestinationOperand(inst), result);
 }
 
-void Translator::S_FF_I32_B64(const Decoder::Instruction& inst, bool find_zero) {
+void Translator::S_FF_I32_B64(const Decoder::Instruction& inst, bool find_zero, bool from_msb) {
 	auto source = ReadU32Pair(inst.src0);
-	if (find_zero) {
-		source[0] = ir.BitwiseNot(source[0]);
-		source[1] = ir.BitwiseNot(source[1]);
+	for (auto& word: source) {
+		word = find_zero ? ir.BitwiseNot(word) : word;
 	}
-	const auto low_lsb       = IR::U32(ir.Emit(IR::ValueOpcode::FindILsb32, {source[0]}));
-	const auto high_lsb      = IR::U32(ir.Emit(IR::ValueOpcode::FindILsb32, {source[1]}));
-	const auto high_position = ir.IAdd(high_lsb, IR::U32(IR::Value(32u)));
-	const auto result        = ir.Select(ir.INotEqual(source[0], IR::U32(IR::Value(0u))), low_lsb,
-	                                     ir.Select(ir.INotEqual(source[1], IR::U32(IR::Value(0u))),
-	                                               high_position, IR::U32(IR::Value(0xffffffffu))));
-	WriteOperand(DestinationOperand(inst), result);
+	if (from_msb) {
+		// The leading bit index is the trailing bit index of the bit-reversed 64-bit value.
+		source = {IR::U32(ir.Emit(IR::ValueOpcode::BitReverse32, {source[1]})),
+		          IR::U32(ir.Emit(IR::ValueOpcode::BitReverse32, {source[0]}))};
+	}
+	// FindILsb returns ~0 for zero, so OR-ing 32 keeps a missing high bit at ~0 (-1).
+	const auto low  = ir.Emit(IR::ValueOpcode::FindILsb32, {source[0]});
+	const auto high = ir.BitwiseOr(IR::U32(ir.Emit(IR::ValueOpcode::FindILsb32, {source[1]})),
+	                               IR::U32(IR::Value(32u)));
+	WriteOperand(DestinationOperand(inst), ir.Emit(IR::ValueOpcode::UMin32, {low, high}));
 }
 
 void Translator::V_FFBH_32(const Decoder::Instruction& inst, bool sign) {
@@ -248,17 +250,6 @@ void Translator::V_FFBH_32(const Decoder::Instruction& inst, bool sign) {
 	WriteOperand(DestinationOperand(inst), result);
 }
 
-void Translator::S_FLBIT_I32_B64(const Decoder::Instruction& inst) {
-	const auto source   = ReadU64(inst.src0);
-	const auto msb      = IR::U32(ir.Emit(IR::ValueOpcode::FindUMsb64, {source}));
-	const auto position = ir.ISub(IR::U32(IR::Value(63u)), msb);
-	const auto nonzero =
-	    IR::U1(ir.Emit(IR::ValueOpcode::INotEqual64,
-	                   {source, ir.ConstructU64(IR::U32(IR::Value(0u)), IR::U32(IR::Value(0u)))}));
-	const auto result = ir.Select(nonzero, position, IR::U32(IR::Value(0xffffffffu)));
-	WriteOperand(DestinationOperand(inst), result);
-}
-
 void Translator::Integer24(const Decoder::Instruction& inst, bool sign, bool addend) {
 	const auto extract24 = [&](IR::U32 value) {
 		return IR::U32(
@@ -277,7 +268,7 @@ void Translator::Integer24(const Decoder::Instruction& inst, bool sign, bool add
 void Translator::V_MAD_U64_U32(const Decoder::Instruction& inst) {
 	const auto lhs = ReadU32(inst.src0);
 	const auto rhs = ReadU32(inst.src1);
-	// Keep the 32x32 product in words: native widening IMul64 fails NVIDIA compilation
+	// Keep the 32x32 product in words: a native widening 64-bit multiply fails NVIDIA compilation
 	// in large vertex shaders (Astro's Playroom). The 64-bit addition stays native.
 	const auto product = ir.ConstructU64(
 	    ir.IMul(lhs, rhs), IR::U32(ir.Emit(IR::ValueOpcode::UMulHi, {lhs, rhs})));
@@ -336,8 +327,7 @@ void Translator::V_BCNT_U32_B32(const Decoder::Instruction& inst) {
 void Translator::V_MBCNT_U32_B32(const Decoder::Instruction& inst, bool low) {
 	const auto lane  = IR::U32(ir.Emit(IR::ValueOpcode::LaneId));
 	const auto local = ir.BitwiseAnd(lane, IR::U32(IR::Value(31u)));
-	const auto below =
-	    ir.ISub(ir.ShiftLeftLogical(IR::U32(IR::Value(1u)), local), IR::U32(IR::Value(1u)));
+	const auto below = RightMask32(local);
 	const auto high_lane =
 	    IR::U1(ir.Emit(IR::ValueOpcode::UGreaterThanEqual32, {lane, IR::Value(32u)}));
 	const auto thread_mask = low ? ir.Select(high_lane, IR::U32(IR::Value(0xffffffffu)), below)
@@ -389,9 +379,8 @@ void Translator::S_QUADMASK_B64(const Decoder::Instruction& inst) {
 void Translator::BFM_B32(const Decoder::Instruction& inst) {
 	const auto count  = ir.BitwiseAnd(ReadU32(inst.src0), IR::U32(IR::Value(31u)));
 	const auto offset = ir.BitwiseAnd(ReadU32(inst.src1), IR::U32(IR::Value(31u)));
-	const auto result = ir.Emit(IR::ValueOpcode::BitFieldInsert,
-	                            {IR::Value(0u), IR::Value(0xffffffffu), offset, count});
-	WriteOperand(DestinationOperand(inst), result);
+	// Mask bits shifted past bit 31 are dropped.
+	WriteOperand(DestinationOperand(inst), ir.ShiftLeftLogical(RightMask32(count), offset));
 }
 
 IR::U32 Translator::RightMask32(IR::U32 count) {
@@ -399,15 +388,6 @@ IR::U32 Translator::RightMask32(IR::U32 count) {
 	                       {IR::Value(0u), IR::Value(0xffffffffu), IR::Value(0u), count}));
 }
 
-IR::U64 Translator::RightMask64(IR::U32 count) {
-	const auto below32 = IR::U1(ir.Emit(IR::ValueOpcode::ULessThan32, {count, IR::Value(32u)}));
-	const auto above32 = IR::U1(ir.Emit(IR::ValueOpcode::UGreaterThan32, {count, IR::Value(32u)}));
-	const auto low_count = ir.Select(below32, count, IR::U32(IR::Value(32u)));
-	const auto high_count =
-	    ir.Select(above32, ir.ISub(count, IR::U32(IR::Value(32u))), IR::U32(IR::Value(0u)));
-	return ir.ConstructU64(RightMask32(low_count), RightMask32(high_count));
-}
-
 void Translator::S_BFM_B64(const Decoder::Instruction& inst) {
 	const auto count  = ReadU32(inst.src0);
 	const auto offset = ReadU32(inst.src1);
@@ -439,12 +419,15 @@ void Translator::S_BFE_U64(const Decoder::Instruction& inst) {
 	const auto field  = ReadU32(inst.src1);
 	const auto offset =
 	    IR::U32(ir.Emit(IR::ValueOpcode::BitFieldUExtract, {field, IR::Value(0u), IR::Value(6u)}));
-	const auto raw_count =
+	const auto count =
 	    IR::U32(ir.Emit(IR::ValueOpcode::BitFieldUExtract, {field, IR::Value(16u), IR::Value(7u)}));
-	const auto available = ir.ISub(IR::U32(IR::Value(64u)), offset);
-	const auto count     = IR::U32(ir.Emit(IR::ValueOpcode::UMin32, {raw_count, available}));
-	const auto shifted   = IR::U64(ir.Emit(IR::ValueOpcode::ShiftRightLogical64, {source, offset}));
-	const auto result    = ir.Emit(IR::ValueOpcode::BitwiseAnd64, {shifted, RightMask64(count)});
+	// The shift already clears bits past the source; widths of 64 and more keep every bit.
+	const auto low_count = IR::U32(ir.Emit(IR::ValueOpcode::UMin32, {count, IR::Value(32u)}));
+	const auto high_count =
+	    IR::U32(ir.Emit(IR::ValueOpcode::UMin32, {ir.ISub(count, low_count), IR::Value(32u)}));
+	const auto mask    = ir.ConstructU64(RightMask32(low_count), RightMask32(high_count));
+	const auto shifted = IR::U64(ir.Emit(IR::ValueOpcode::ShiftRightLogical64, {source, offset}));
+	const auto result  = ir.Emit(IR::ValueOpcode::BitwiseAnd64, {shifted, mask});
 	WriteOperand(DestinationOperand(inst), result);
 	ir.SetScc(IR::U1(ir.Emit(IR::ValueOpcode::INotEqual64, {result, IR::Value(uint64_t {0})})));
 }

src/graphics/shader/recompiler/frontend/translate/Memory.cpp

@@ -455,12 +455,17 @@ void Translator::BUFFER_LOAD(const Decoder::Instruction& inst) {
 	            AddMemoryInfo(memory, inst.pc));
 	if (bits != 32u) {
 		WriteOperand(inst.dst, WidenSubdword(loaded, bits, sign));
-	} else if (memory.data_dwords == 1u) {
-		WriteOperand(inst.dst, loaded);
-	} else {
-		for (uint32_t component = 0; component < memory.data_dwords; component++) {
-			WriteOperand(OffsetOperand(inst.dst, component),
-			             ir.CompositeExtract(loaded, component));
+		return;
+	}
+	for (uint32_t component = 0; component < memory.data_dwords; component++) {
+		const auto dst = OffsetOperand(inst.dst, component);
+		const auto value =
+		    memory.data_dwords == 1u ? IR::U32(loaded) : ir.CompositeExtract(loaded, component);
+		// D16 formats with an odd component count keep the high half of their final VGPR.
+		if (memory.data_bits == 16u && component * 2u + 1u == memory.component_count) {
+			Write16Bits(dst, value);
+		} else {
+			WriteOperand(dst, value);
 		}
 	}
 }
@@ -992,6 +997,8 @@ void Translator::EmitMemory(const Decoder::Instruction& inst) {
 		case Decoder::Opcode::BUFFER_LOAD_FORMAT_XYZ:
 		case Decoder::Opcode::BUFFER_LOAD_FORMAT_XYZW:
 		case Decoder::Opcode::BUFFER_LOAD_FORMAT_D16_X:
+		case Decoder::Opcode::BUFFER_LOAD_FORMAT_D16_XY:
+		case Decoder::Opcode::BUFFER_LOAD_FORMAT_D16_XYZ:
 		case Decoder::Opcode::BUFFER_LOAD_FORMAT_D16_XYZW:
 		case Decoder::Opcode::TBUFFER_LOAD_FORMAT_X:
 		case Decoder::Opcode::TBUFFER_LOAD_FORMAT_XY:
@@ -1012,6 +1019,8 @@ void Translator::EmitMemory(const Decoder::Instruction& inst) {
 		case Decoder::Opcode::BUFFER_STORE_FORMAT_XYZW:
 		case Decoder::Opcode::BUFFER_STORE_FORMAT_D16_X:
 		case Decoder::Opcode::BUFFER_STORE_FORMAT_D16_HI_X:
+		case Decoder::Opcode::BUFFER_STORE_FORMAT_D16_XY:
+		case Decoder::Opcode::BUFFER_STORE_FORMAT_D16_XYZ:
 		case Decoder::Opcode::BUFFER_STORE_FORMAT_D16_XYZW:
 		case Decoder::Opcode::TBUFFER_STORE_FORMAT_X:
 		case Decoder::Opcode::TBUFFER_STORE_FORMAT_XY:

src/graphics/shader/recompiler/frontend/translate/Scalar.cpp

@@ -238,10 +238,10 @@ void Translator::EmitScalar(const Decoder::Instruction& inst) {
 			return ComposedIntegerBinary(inst, IR::ValueOpcode::BitwiseOr32, false, true, true);
 		case O::S_XNOR_B32:
 			return ComposedIntegerBinary(inst, IR::ValueOpcode::BitwiseXor32, false, true, true);
-		case O::S_FF0_I32_B64: return S_FF_I32_B64(inst, true);
-		case O::S_FF1_I32_B64: return S_FF_I32_B64(inst, false);
+		case O::S_FF0_I32_B64: return S_FF_I32_B64(inst, true, false);
+		case O::S_FF1_I32_B64: return S_FF_I32_B64(inst, false, false);
 		case O::S_FLBIT_I32_B32: return V_FFBH_32(inst, false);
-		case O::S_FLBIT_I32_B64: return S_FLBIT_I32_B64(inst);
+		case O::S_FLBIT_I32_B64: return S_FF_I32_B64(inst, false, true);
 
 		case O::S_BITSET0_B32: return S_BITSET_B32(inst, false);
 		case O::S_BITSET1_B32: return S_BITSET_B32(inst, true);

src/graphics/shader/recompiler/frontend/translate/Translate.cpp

@@ -296,15 +296,11 @@ void Translator::WriteRawU32(const Decoder::Operand& operand, IR::U32 value) {
 				value               = ir.ShiftLeftLogical(extended, IR::U32(IR::Value(offset)));
 				break;
 			}
-			case 2: {
-				const uint32_t field_mask = width == 32u ? 0xffffffffu : (1u << width) - 1u;
-				const auto     inserted =
-				    ir.ShiftLeftLogical(ir.BitwiseAnd(value, IR::U32(IR::Value(field_mask))),
-				                        IR::U32(IR::Value(offset)));
-				const auto cleared = ir.BitwiseAnd(ReadRawU32(PlainOperand(operand)),
-				                                   IR::U32(IR::Value(~(field_mask << offset))));
-				value              = ir.BitwiseOr(cleared, inserted);
-			} break;
+			case 2:
+				value = IR::U32(ir.Emit(IR::ValueOpcode::BitFieldInsert,
+				                        {ReadRawU32(PlainOperand(operand)), value, IR::Value(offset),
+				                         IR::Value(width)}));
+				break;
 			default: EXIT("reserved SDWA DST_U mode");
 		}
 	}
@@ -323,7 +319,10 @@ void Translator::WriteRawU32(const Decoder::Operand& operand, IR::U32 value) {
 		case Decoder::OperandKind::Vgpr: {
 			const auto reg = static_cast<IR::VectorReg>(operand.reg);
 			const auto old = ir.GetVectorReg(reg);
-			if (operand.dpp) {
+			// DPP skips EXEC lanes only through row/bank masks, or for invalid sources without
+			// BOUND_CTRL (dpp8 reads zero); otherwise it is a plain masked write.
+			if (operand.dpp && (operand.dpp_row_mask != 0xfu || operand.dpp_bank_mask != 0xfu ||
+			                    (!operand.dpp_bound_ctrl && !operand.dpp8))) {
 				value = IR::U32(ir.Emit(IR::ValueOpcode::DppUpdateU32, {value, old, ir.GetExec()},
 				                        DppFlags(operand)));
 			} else {
@@ -453,7 +452,10 @@ void Translator::Write16Bits(const Decoder::Operand& operand, IR::U32 value) {
 	destination.omod   = 0u;
 	destination.op_sel = false;
 	destination.clamp  = false;
-	WriteRawU32(destination, ir.BitwiseAnd(value, IR::U32(IR::Value(0xffffu))));
+	// A field selector keeps only the field's bits; a full-dword SDWA selector zero-extends.
+	WriteRawU32(destination, destination.sdwa_sel == 6u
+	                             ? ir.BitwiseAnd(value, IR::U32(IR::Value(0xffffu)))
+	                             : value);
 }
 
 void Translator::WriteF16(const Decoder::Operand& operand, IR::F32 value) {
@@ -740,14 +742,16 @@ void Translator::WriteCompareResult(const Decoder::Operand& operand, IR::U1 valu
 void Translator::AddBranchCondition(const CFG::Graph& graph, const CFG::BasicBlock& source,
                                     IR::Block& info) {
 	const auto native_condition = [&](CFG::BranchCondition kind) -> IR::U1 {
+		// A zero kind negates the vote of the nonzero kind that follows it in the enum.
+		const bool zero = kind == CFG::BranchCondition::SccZero ||
+		                  kind == CFG::BranchCondition::VccZero ||
+		                  kind == CFG::BranchCondition::ExecZero;
+		if (zero) kind = static_cast<CFG::BranchCondition>(static_cast<int>(kind) + 1);
 		IR::U1 condition;
 		switch (kind) {
 			case CFG::BranchCondition::Always: condition = IR::U1(IR::Value(true)); break;
-			case CFG::BranchCondition::SccZero: condition = ir.LogicalNot(ir.GetScc()); break;
 			case CFG::BranchCondition::SccNonZero: condition = ir.GetScc(); break;
-			case CFG::BranchCondition::VccZero: condition = ir.LogicalNot(ir.GetVcc()); break;
 			case CFG::BranchCondition::VccNonZero: condition = ir.GetVcc(); break;
-			case CFG::BranchCondition::ExecZero: condition = ir.LogicalNot(ir.GetExec()); break;
 			case CFG::BranchCondition::ExecNonZero: condition = ir.GetExec(); break;
 			case CFG::BranchCondition::ScalarInstruction:
 				EXIT_IF(instruction_branch_condition.IsEmpty());
@@ -755,7 +759,8 @@ void Translator::AddBranchCondition(const CFG::Graph& graph, const CFG::BasicBlo
 				break;
 			default: EXIT("block %u has an invalid native branch condition", source.id);
 		}
-		return IR::U1(ir.Emit(IR::ValueOpcode::ConditionRef, {condition}, kind));
+		const auto vote = IR::U1(ir.Emit(IR::ValueOpcode::ConditionRef, {condition}, kind));
+		return zero ? ir.LogicalNot(vote) : vote;
 	};
 	const auto expression = [&](auto&& self, uint32_t index) -> IR::U1 {
 		const auto& value = graph.expressions.at(index);

src/graphics/shader/recompiler/frontend/translate/Translator.h

@@ -113,11 +113,6 @@ private:
 	void DS_ORDERED_COUNT(const Decoder::Instruction& inst);
 
 	IR::F32 SelectF32(IR::U1 condition, IR::F32 true_value, IR::F32 false_value);
-	IR::U32 ConvertF32ToU32Saturated(IR::F32 value, float upper_bound, float safe_upper,
-	                                 uint32_t high_result);
-	IR::U32 ConvertF32ToI32Saturated(IR::F32 value, float lower_bound, float upper_bound,
-	                                 float safe_upper, uint32_t lower_result,
-	                                 uint32_t upper_result);
 	IR::U32 PackU16Lanes(IR::U32 low, IR::U32 high);
 	void EmitCompareResult(const Decoder::Instruction& inst, IR::U1 value, bool scalar, bool cmpx);
 	void EmitCompareConstant(const Decoder::Instruction& inst, bool value, bool scalar, bool cmpx);
@@ -187,9 +182,8 @@ private:
 	void V_AND_OR_B32(const Decoder::Instruction& inst);
 	void V_OR3_B32(const Decoder::Instruction& inst);
 	void V_XOR3_B32(const Decoder::Instruction& inst);
-	void S_FF_I32_B64(const Decoder::Instruction& inst, bool find_zero);
+	void S_FF_I32_B64(const Decoder::Instruction& inst, bool find_zero, bool from_msb);
 	void V_FFBH_32(const Decoder::Instruction& inst, bool sign);
-	void S_FLBIT_I32_B64(const Decoder::Instruction& inst);
 	void Integer24(const Decoder::Instruction& inst, bool sign, bool addend);
 	void V_MAD_U64_U32(const Decoder::Instruction& inst);
 	void V_SAD_U32(const Decoder::Instruction& inst);
@@ -202,7 +196,6 @@ private:
 	void S_QUADMASK_B64(const Decoder::Instruction& inst);
 	void BFM_B32(const Decoder::Instruction& inst);
 	IR::U32 RightMask32(IR::U32 count);
-	IR::U64 RightMask64(IR::U32 count);
 	void    S_BFM_B64(const Decoder::Instruction& inst);
 	void    S_BFE_U32(const Decoder::Instruction& inst, bool sign);
 	void    S_BFE_U64(const Decoder::Instruction& inst);
@@ -247,8 +240,8 @@ private:
 	void V_READLANE_B32(const Decoder::Instruction& inst);
 	void V_WRITELANE_B32(const Decoder::Instruction& inst);
 	void V_PERMLANE16_B32(const Decoder::Instruction& inst, bool x16);
-	void V_INTERP_P1_F32();
 	void V_INTERP_P2_F32(const Decoder::Instruction& inst);
+	void V_INTERP_P2_F16(const Decoder::Instruction& inst);
 	void V_INTERP_MOV_F32(const Decoder::Instruction& inst);
 	void EXP(const Decoder::Instruction& inst);
 

src/graphics/shader/recompiler/frontend/translate/Vector.cpp

@@ -6,6 +6,8 @@ void Translator::EmitVector(const Decoder::Instruction& inst) {
 	using O = Decoder::Opcode;
 	switch (inst.opcode) {
 		case O::V_NOP: return;
+		case O::V_INTERP_P1LL_F16:
+		case O::V_INTERP_P2_F16: return EmitInterpolation(inst);
 		case O::V_ADD_I32: ADD_U32(inst, true, false); return;
 		case O::V_ADDC_U32: ADD_U32(inst, true, true); return;
 		case O::V_SUB_I32: SUB_U32(inst, true, false); return;

src/graphics/shader/recompiler/ir/IREmitter.cpp

@@ -6,11 +6,6 @@ IREmitter::IREmitter(Block* value_block): block(value_block) {
 	EXIT_IF(block == nullptr);
 }
 
-void IREmitter::SetBlock(Block* value_block) {
-	EXIT_IF(value_block == nullptr);
-	block = value_block;
-}
-
 Value IREmitter::Emit(ValueOpcode opcode, std::initializer_list<Value> args, uint64_t flags) {
 	auto& inst = block->AppendNewInst(opcode, args, flags);
 	return TypeOf(opcode) == Type::Void ? Value {} : Value(&inst);
@@ -132,16 +127,6 @@ U32 IREmitter::BitCastU32(F32 value) {
 	return U32(Emit(ValueOpcode::BitCastU32F32, {value}));
 }
 
-F16 IREmitter::BitCastF16(U32 value) {
-	const auto half = U16(Emit(ValueOpcode::ConvertU16U32, {value}));
-	return F16(Emit(ValueOpcode::BitCastF16U16, {half}));
-}
-
-U32 IREmitter::BitCastU32(F16 value) {
-	const auto half = U16(Emit(ValueOpcode::BitCastU16F16, {value}));
-	return U32(Emit(ValueOpcode::ConvertU32U16, {half}));
-}
-
 U64 IREmitter::ConstructU64(U32 low, U32 high) {
 	return U64(Emit(ValueOpcode::CompositeConstructU64, {low, high}));
 }

src/graphics/shader/recompiler/ir/IREmitter.h

@@ -12,8 +12,6 @@ class IREmitter {
 public:
 	explicit IREmitter(Block* block);
 
-	void SetBlock(Block* block);
-
 	Value Emit(ValueOpcode opcode, std::initializer_list<Value> args = {}, uint64_t flags = 0);
 
 	template <typename T>
@@ -54,8 +52,6 @@ public:
 
 	F32 BitCastF32(U32 value);
 	U32 BitCastU32(F32 value);
-	F16 BitCastF16(U32 value);
-	U32 BitCastU32(F16 value);
 	U64 ConstructU64(U32 low, U32 high);
 	U32 CompositeExtract(Value composite, uint32_t index);
 

src/graphics/shader/recompiler/ir/Program.cpp

@@ -487,7 +487,7 @@ void ValidateProgram(const Program& program, bool require_ssa) {
 				const auto access = BufferAccessOf(inst.GetOpcode());
 				const bool packed_formatted =
 				    (access == BufferAccess::Read || access == BufferAccess::Write) && memory.formatted &&
-				    memory.data_bits == 16u && memory.component_count == 4u && buffer_components == 2u;
+				    memory.data_bits == 16u && (memory.component_count + 1u) / 2u == buffer_components;
 				if (buffer_components > 1u &&
 				    (!vector_buffer || (memory.data_bits != 32u && !packed_formatted) ||
 				     memory.data_dwords != buffer_components || memory.component_index != 0u)) {

src/graphics/shader/recompiler/ir/ShaderIR.h

@@ -169,6 +169,7 @@ struct ImageResource {
 	bool                          depth_compare     = false;
 	bool                          cube              = false;
 	bool                          r128              = false;
+	bool                          heap              = false; // Resident image heap root or kind.
 	uint32_t                      indirect_root     = NoIndirectImage;
 	uint32_t                      indirect_mapping_offset   = 0;
 	uint32_t                      indirect_search_iterations = 0;
@@ -188,6 +189,12 @@ RawSignedStorageFormat(const ImageResource& image, Prospero::BufferFormat format
 	return Prospero::BufferFormat::kInvalid;
 }
 
+// A raw load of a kDepth k16UInt T# reads a D16 surface's codes. Vulkan cannot view D16 as
+// R16Uint, so the host binds it as UNorm and the shader converts back.
+[[nodiscard]] constexpr bool ReadsDepthBits(const ImageResource& image) {
+	return image.conversion_format == Prospero::BufferFormat::k16UInt;
+}
+
 struct SamplerResource {
 	static constexpr uint32_t NoIndirectSampler = UINT32_MAX;
 
@@ -337,7 +344,12 @@ struct StageOutput {
 inline constexpr uint32_t FirstImageBinding           = 1u;
 inline constexpr uint32_t FirstComparisonImageBinding = 22u;
 inline constexpr uint32_t FirstStorageImageBinding    = 29u;
-inline constexpr uint32_t ImageBindingCount           = 48u;
+inline constexpr uint32_t ImageBindingCount           = 50u;
+// Resident image heaps share one sampled-image array at binding 0 of this set. A heap slot
+// word is kind << ImageHeapKindShift | element; element 0 is the null descriptor.
+inline constexpr uint32_t ImageHeapSet                = 1u;
+inline constexpr uint32_t ImageHeapCapacity           = 65536u;
+inline constexpr uint32_t ImageHeapKindShift          = 20u;
 
 enum class DescriptorBindingKind : uint32_t {
 	Buffers  = 0u,
@@ -351,8 +363,8 @@ enum class DescriptorBindingKind : uint32_t {
 	Count,
 };
 
-static_assert(static_cast<uint32_t>(DescriptorBindingKind::Samplers) == 49u);
-static_assert(static_cast<uint32_t>(DescriptorBindingKind::Count) == 56u);
+static_assert(static_cast<uint32_t>(DescriptorBindingKind::Samplers) == 51u);
+static_assert(static_cast<uint32_t>(DescriptorBindingKind::Count) == 58u);
 
 struct PushData {
 	static constexpr uint32_t DwordCount = 32;
@@ -413,8 +425,8 @@ DescriptorBindingForImage(const ImageResource& image) {
 	constexpr uint32_t SampledUintBinding  = 8u;
 	constexpr uint32_t SampledSintBinding  = 15u;
 	constexpr uint32_t StorageFloatBinding = FirstStorageImageBinding;
-	constexpr uint32_t StorageUintBinding  = StorageFloatBinding + 5u;
-	constexpr uint32_t AtomicUintBinding   = StorageUintBinding + 5u;
+	constexpr uint32_t StorageUintBinding  = StorageFloatBinding + 6u;
+	constexpr uint32_t AtomicUintBinding   = StorageUintBinding + 6u;
 
 	uint32_t base    = 0;
 	bool     sampled = false;
@@ -461,10 +473,10 @@ DescriptorBindingForImage(const ImageResource& image) {
 		case Decoder::ImageDimension::Dim2D: dimension = 2u; break;
 		case Decoder::ImageDimension::Dim2DArray: dimension = 3u; break;
 		case Decoder::ImageDimension::Dim2DMsaa:
-			if (!sampled) {
+			if (image.atomic) {
 				return std::nullopt;
 			}
-			dimension = 4u;
+			dimension = sampled ? 4u : 5u;
 			break;
 		case Decoder::ImageDimension::Dim2DMsaaArray:
 			if (!sampled) {
@@ -523,6 +535,7 @@ struct ShaderInfo {
 	bool                            uses_lds                      = false;
 	bool                            uses_gds                      = false;
 	bool                            uses_flattened_srt            = false;
+	bool                            uses_image_heap               = false;
 	bool                            uses_dispatch_threads         = false;
 	bool                            bvh                           = false;
 	bool                            subgroup_ballot               = false;

src/graphics/shader/recompiler/ir/opcodes/ValueOpcodes.inc

@@ -44,10 +44,7 @@ VALUE_OPCODE(GetM0, U32)
 VALUE_OPCODE(SetM0, Void, U32)
 
 VALUE_OPCODE(UndefU1, U1)
-VALUE_OPCODE(UndefU8, U8)
-VALUE_OPCODE(UndefU16, U16)
 VALUE_OPCODE(UndefU32, U32)
-VALUE_OPCODE(UndefU64, U64)
 
 // Value construction and bit-preserving conversion.
 VALUE_OPCODE(BitCastU16F16, U16, F16)
@@ -100,11 +97,9 @@ VALUE_OPCODE(SelectU1, U1, U1, U1, U1)
 VALUE_OPCODE(SelectF32, F32, U1, F32, F32)
 VALUE_OPCODE(IAdd32, U32, U32, U32)
 VALUE_OPCODE(IAdd64, U64, U64, U64)
-VALUE_OPCODE(IAddCarry32, U32x2, U32, U32)
 VALUE_OPCODE(ISub32, U32, U32, U32)
 VALUE_OPCODE(ISub64, U64, U64, U64)
 VALUE_OPCODE(IMul32, U32, U32, U32)
-VALUE_OPCODE(IMul64, U64, U64, U64)
 VALUE_OPCODE(UDiv32, U32, U32, U32)
 VALUE_OPCODE(SMulHi, U32, U32, U32)
 VALUE_OPCODE(UMulHi, U32, U32, U32)
@@ -122,9 +117,7 @@ VALUE_OPCODE(BitwiseXor32, U32, U32, U32)
 VALUE_OPCODE(BitwiseNot32, U32, U32)
 VALUE_OPCODE(BitReverse32, U32, U32)
 VALUE_OPCODE(BitCount32, U32, U32)
-VALUE_OPCODE(BitCount64, U32, U64)
 VALUE_OPCODE(FindUMsb32, U32, U32)
-VALUE_OPCODE(FindUMsb64, U32, U64)
 VALUE_OPCODE(FindILsb32, U32, U32)
 VALUE_OPCODE(SMin32, U32, U32, U32)
 VALUE_OPCODE(UMin32, U32, U32, U32)
@@ -345,6 +338,7 @@ VALUE_OPCODE(ImageAtomicXor64, U64, ImageResource, ImageAddress, U64, U1)
 VALUE_OPCODE(ImageAtomicFMin32, U32, ImageResource, ImageAddress, U32, U1)
 VALUE_OPCODE(ImageAtomicFMax32, U32, ImageResource, ImageAddress, U32, U1)
 VALUE_OPCODE(GetAttribute, U32, U32, U32)
+VALUE_OPCODE(GetAttributeF16, F32, U32, U32, U32)
 VALUE_OPCODE(GetInterpolationParameter, U32, U32, U32, U32)
 VALUE_OPCODE(SetAttribute, Void, U32x4, U1)
 

src/graphics/shader/recompiler/ir/passes/ConstantPropagation.cpp

@@ -244,17 +244,6 @@ bool FoldCompositeExtract(Inst& inst, ValueOpcode construct, size_t components)
 		Replace(inst, producer->Arg(component));
 		return true;
 	}
-	if (component < 2u && producer != nullptr &&
-	    producer->GetOpcode() == ValueOpcode::IAddCarry32) {
-		const auto lhs = producer->Arg(0).Resolve();
-		const auto rhs = producer->Arg(1).Resolve();
-		if (IsImmediate(lhs, Type::U32) && IsImmediate(rhs, Type::U32)) {
-			const auto sum = static_cast<uint64_t>(lhs.U32()) + rhs.U32();
-			Replace(inst, Value(component == 0u ? static_cast<uint32_t>(sum)
-			                                    : static_cast<uint32_t>(sum >> 32u)));
-			return true;
-		}
-	}
 	return false;
 }
 
@@ -511,11 +500,6 @@ void FoldInstruction(Block& block, Block::iterator instruction,
 				ReplaceBinaryIdentity(inst, Type::U32, 1u);
 			}
 			return;
-		case ValueOpcode::IMul64:
-			if (!FoldU64(inst, [](uint64_t a, uint64_t b) { return a * b; })) {
-				ReplaceBinaryIdentity(inst, Type::U64, 1u);
-			}
-			return;
 		case ValueOpcode::WqmU64: {
 			const auto value = Arg(inst, 0);
 			if (IsImmediate(value, Type::U64)) {
@@ -667,17 +651,15 @@ void FoldInstruction(Block& block, Block::iterator instruction,
 			}
 			return;
 		}
-		case ValueOpcode::BitCount32: {
+		case ValueOpcode::BitCount32:
+		case ValueOpcode::FindILsb32: {
 			const auto value = Arg(inst, 0);
 			if (IsImmediate(value, Type::U32)) {
-				Replace(inst, Value(static_cast<uint32_t>(std::popcount(value.U32()))));
-			}
-			return;
-		}
-		case ValueOpcode::BitCount64: {
-			const auto value = Arg(inst, 0);
-			if (IsImmediate(value, Type::U64)) {
-				Replace(inst, Value(static_cast<uint32_t>(std::popcount(value.U64()))));
+				const auto bits = value.U32();
+				Replace(inst, Value(static_cast<uint32_t>(
+				    inst.GetOpcode() == ValueOpcode::BitCount32 ? std::popcount(bits)
+				    : bits == 0u                                ? -1
+				                                                : std::countr_zero(bits))));
 			}
 			return;
 		}
@@ -690,7 +672,9 @@ void FoldInstruction(Block& block, Block::iterator instruction,
 			});
 			return;
 		case ValueOpcode::UMin32:
-			FoldU32(inst, [](uint32_t a, uint32_t b) { return std::min(a, b); });
+			if (!FoldU32(inst, [](uint32_t a, uint32_t b) { return std::min(a, b); })) {
+				ReplaceBinaryIdentity(inst, Type::U32, ~0u);
+			}
 			return;
 		case ValueOpcode::UMax32:
 			FoldU32(inst, [](uint32_t a, uint32_t b) { return std::max(a, b); });

src/graphics/shader/recompiler/ir/passes/ResourceMaterialization.cpp

@@ -681,6 +681,32 @@ bool MaterializeIndirectDescriptor(const ResourcePlan&                         p
 			const auto count = table_size >= first + 4u
 			    ? std::min((table_size - first - 4u) / step + 1u, domain)
 			    : 0u;
+			descriptors[resource_index] = {.dword_count = dword_count};
+			if constexpr (std::is_same_v<Specialization, ResourceSpecialization::Image>) {
+				const auto& image = program.info.images[resource_index];
+				// A resident heap maps slot words instead of materializing its entries.
+				ImageHeapView view;
+				if (image_heap && runtime.image_heap != nullptr &&
+				    image.resource_class == ImageResourceClass::Sampled &&
+				    image.mip_mode != ImageMipMode::Dynamic &&
+				    runtime.image_heap->Acquire({table_base, table_size, step, first, count, &image},
+				                                view)) {
+					snapshot.specialization_reads.emplace_back(table_base, table_size);
+					if (view.kinds.empty()) return true;
+					auto root = root_resource;
+					root.indirect_root = resource_index;
+					root.indirect_mapping_offset = static_cast<uint32_t>(snapshot.flattened_srt.size());
+					root.heap = true;
+					specializations[resource_index] = root;
+					snapshot.flattened_srt.insert(snapshot.flattened_srt.end(),
+					    {view.slot_count, static_cast<uint32_t>(view.slots), static_cast<uint32_t>(view.slots >> 32u)});
+					for (const auto& kind: view.kinds) {
+						descriptors.push_back(kind);
+						specializations.push_back(root);
+					}
+					return true;
+				}
+			}
 			if (child_count == 0u || (!image_heap && count * child_count > MaxIndirectDescriptorProbes))
 				return failed("candidate count", fmt::format("rows={} limit={}", count, MaxIndirectDescriptorProbes));
 			// Affine SMEM offsets wrap in U32 before adding the aligned immediate.
@@ -692,7 +718,6 @@ bool MaterializeIndirectDescriptor(const ResourcePlan&                         p
 					return read_failed("image heap read", 0u, 0u, table_base, 0u, heap_words.size() * sizeof(uint32_t));
 			}
 			const auto mapping_offset = snapshot.flattened_srt.size();
-			descriptors[resource_index] = {.dword_count = dword_count};
 			snapshot.flattened_srt.resize(mapping_offset + 2u, 0u);
 			for (uint32_t word = 0; word < count; ++word) {
 				const auto table_offset = affine ? first + word * step : uint64_t {word} * 4u;
@@ -932,7 +957,12 @@ static bool BuildResourceSpecialization(const ResourcePlan& program, ResourceSna
 				return SpecializationFail("FMASK requires a direct image load");
 			}
 		}
-		image.conversion_format = ImageConversionFormat(format);
+		// See ReadsDepthBits; raw loads only, since filtering would blend the codes.
+		const bool depth_bits =
+		    !storage && base.unfiltered_access_only && format == Prospero::BufferFormat::k16UInt &&
+		    static_cast<Prospero::TileMode>((descriptor.dwords[3] >> 20u) & 0x1fu) ==
+		        Prospero::TileMode::kDepth;
+		image.conversion_format = depth_bits ? format : ImageConversionFormat(format);
 		if (storage || image.conversion_format != Prospero::BufferFormat::kInvalid) {
 			image.shader_swizzle = DescriptorImageSwizzle(descriptor);
 		}
@@ -940,7 +970,7 @@ static bool BuildResourceSpecialization(const ResourcePlan& program, ResourceSna
 		const bool raw_sint_storage = raw_sint_format != Prospero::BufferFormat::kInvalid;
 		if (raw_sint_format == Prospero::BufferFormat::k16UInt) image.conversion_format = format;
 		image.numeric_class = Prospero::SampledTextureNumericClass(
-		    Prospero::RemapTextureFormat(format));
+		    depth_bits ? Prospero::BufferFormat::k16UNorm : Prospero::RemapTextureFormat(format));
 		if (!storage && format == Prospero::BufferFormat::k11_11_10UNorm) {
 			if (image.indirect_root != ImageResource::NoIndirectImage) {
 				return SpecializationFail("packed UNorm sampling requires a direct image");
@@ -1005,10 +1035,12 @@ static bool BuildResourceSpecialization(const ResourcePlan& program, ResourceSna
 		const auto key_count = root.indirect_mapping_offset < snapshot.flattened_srt.size()
 		                           ? snapshot.flattened_srt[root.indirect_mapping_offset]
 		                           : 0u;
+		// A heap maps its slot count and slot address; tables map each key.
+		size_t mapping_words = 3u;
+		if (!root.heap)
+			mapping_words = 1u + size_t {key_count} * (root.indirect_search_iterations == 0u ? 1u : 2u);
 		if (key_count == 0u ||
-		    static_cast<size_t>(root.indirect_mapping_offset) + 1u +
-		            static_cast<size_t>(key_count) * (root.indirect_search_iterations == 0u ? 1u : 2u) >
-		        snapshot.flattened_srt.size()) {
+		    root.indirect_mapping_offset + mapping_words > snapshot.flattened_srt.size()) {
 			return SpecializationFail("indirect image specialization has an invalid key mapping");
 		}
 		uint32_t exemplar       = ImageResource::NoIndirectImage;
@@ -1744,6 +1776,43 @@ bool MaterializeResources(const ResourcePlan& program, const SrtRuntime& runtime
 	return BuildResourceSpecialization(program, snapshot, specialization);
 }
 
+uint32_t ImageHeapKind(const ImageResource& root, DescriptorValue& value) {
+	// Mirrors BuildResourceSpecialization for sampled roots: entries it would reject, and
+	// entries that need shader-side conversion, read as null instead of failing the draw.
+	const auto format = static_cast<Prospero::BufferFormat>((value.dwords[1] >> 20u) & 0x1ffu);
+	const auto tile = static_cast<Prospero::TileMode>((value.dwords[3] >> 20u) & 0x1fu);
+	const auto numeric_class =
+	    Prospero::SampledTextureNumericClass(Prospero::RemapTextureFormat(format));
+	if (NullImageDescriptor(value) || !ValidImageDescriptor(value, root.r128) ||
+	    DescriptorDimension(value, root.dimension) != root.dimension ||
+	    Prospero::IsFmaskTextureFormat(format) || format == Prospero::BufferFormat::k11_11_10UNorm ||
+	    ImageConversionFormat(format) != Prospero::BufferFormat::kInvalid ||
+	    (root.unfiltered_access_only && format == Prospero::BufferFormat::k16UInt &&
+	     tile == Prospero::TileMode::kDepth) ||
+	    numeric_class == Prospero::TextureNumericClass::Unsupported ||
+	    (root.depth_compare && numeric_class != Prospero::TextureNumericClass::Float)) {
+		value.dwords.fill(0u);
+		return 0u;
+	}
+	return 1u + static_cast<uint32_t>(numeric_class) * 2u + (DescriptorIsCube(value) ? 1u : 0u);
+}
+
+ImageResource ImageHeapResource(const ImageResource& root, uint32_t kind) {
+	ImageResource resource;
+	resource.resource_class         = ImageResourceClass::Sampled;
+	resource.read                   = true;
+	resource.heap                   = true;
+	resource.dimension              = root.dimension;
+	resource.unfiltered_access_only = root.unfiltered_access_only;
+	resource.depth_compare          = root.depth_compare;
+	resource.r128                   = root.r128;
+	if (kind != 0u) {
+		resource.numeric_class = static_cast<Prospero::TextureNumericClass>((kind - 1u) / 2u);
+		resource.cube          = ((kind - 1u) & 1u) != 0u;
+	}
+	return resource;
+}
+
 void ApplyResourceSpecialization(Program& program, const ResourceSpecialization& specialization) {
 	EXIT_IF(!program.resource_tracking_complete || program.shader_info_complete);
 	EXIT_IF(program.info.buffers.size() > specialization.buffers.size() ||
@@ -1794,6 +1863,7 @@ void ApplyResourceSpecialization(Program& program, const ResourceSpecialization&
 		image.indirect_mapping_offset    = source.indirect_mapping_offset;
 		image.indirect_search_iterations = source.indirect_search_iterations;
 		image.cube                       = source.cube;
+		image.heap                       = source.heap;
 		image.indirect_resources.clear();
 	}
 	for (uint32_t index = 0; index < images.size(); index++) {

src/graphics/shader/recompiler/ir/passes/ResourceMaterialization.h

@@ -31,6 +31,7 @@ struct ResourceSpecialization {
 		uint32_t                      indirect_search_iterations = 0;
 		bool                          cube                       = false;
 		bool                          fmask                      = false;
+		bool                          heap                       = false;
 		bool                          operator==(const Image&) const = default;
 	};
 
@@ -59,6 +60,13 @@ bool MaterializeResources(const ResourcePlan& program, const SrtRuntime& runtime
 // Applies an already-derived specialization to native IR before layout and emission.
 void ApplyResourceSpecialization(Program& program, const ResourceSpecialization& specialization);
 
+// Kind of an image heap entry as root sees it, or 0 (and a zeroed value) when root reads it as
+// null. Entries of one kind specialize identically.
+uint32_t ImageHeapKind(const ImageResource& root, DescriptorValue& value);
+// The sampled resource that root resolves entries of kind as; kind 0 gives the identity of
+// root's view of a heap.
+ImageResource ImageHeapResource(const ImageResource& root, uint32_t kind);
+
 } // namespace Libs::Graphics::ShaderRecompiler::IR
 
 #endif /* EMULATOR_INCLUDE_EMULATOR_GRAPHICS_SHADER_RECOMPILER_RESOURCEMATERIALIZATION_H_ */

src/graphics/shader/recompiler/ir/passes/ResourceTracking.cpp

@@ -56,6 +56,17 @@ uint32_t PossibleU32Bits(Value value, const auto& normalize) {
 				}
 				break;
 			}
+			case ValueOpcode::BitFieldInsert: {
+				const auto offset = inst->Arg(2).Resolve();
+				const auto count  = inst->Arg(3).Resolve();
+				if (offset.IsImmediate() && count.IsImmediate() && offset.U32() < 32u &&
+				    count.U32() <= 32u - offset.U32()) {
+					const auto field = (count.U32() == 32u ? UINT32_MAX : (1u << count.U32()) - 1u)
+					                   << offset.U32();
+					bits = (arg(0) & ~field) | ((arg(1) << offset.U32()) & field);
+				}
+				break;
+			}
 			default: break;
 		}
 		return bits;

src/graphics/shader/recompiler/ir/passes/ShaderInfoCollection.cpp

@@ -373,7 +373,8 @@ void Visit(Program& program, ShaderStageInputInfo input_info, InputUsage& inputs
 		}
 	}
 	switch (inst.GetOpcode()) {
-		case ValueOpcode::GetAttribute: {
+		case ValueOpcode::GetAttribute:
+		case ValueOpcode::GetAttributeF16: {
 			if (!inst.Arg(0).IsImmediate() || inst.Arg(0).GetType() != Type::U32 ||
 			    !inst.Arg(1).IsImmediate() || inst.Arg(1).GetType() != Type::U32) {
 				return Fail("typed attribute reference is not constant");
@@ -395,6 +396,15 @@ void Visit(Program& program, ShaderStageInputInfo input_info, InputUsage& inputs
 				auto& components = inputs.components[inst.Arg(0).U32()];
 				components = std::max(components, static_cast<uint8_t>(inst.Arg(1).U32() + 1u));
 			}
+			if (inst.GetOpcode() == ValueOpcode::GetAttributeF16) {
+				if (program.stage != ShaderType::Pixel || !inst.Arg(2).IsImmediate()) {
+					return Fail("f16 attribute reference is invalid");
+				}
+				// FP16_INTERP_MODE packs two halves per dword, so smooth inputs blend unpacked
+				// vertices; flat inputs unpack the provoking vertex.
+				const auto input = inst.Arg(0).U32();
+				inputs.per_vertex[input] |= !ShaderPixelParameterIsFlat(*input_info.pixel, input);
+			}
 			break;
 		}
 		case ValueOpcode::GetInterpolationParameter: {
@@ -584,6 +594,7 @@ void CollectShaderInfo(Program& program, ShaderStageInputInfo input_info) {
 	};
 	info.uses_flattened_srt |= std::ranges::any_of(info.buffers, uses_mapping) ||
 	                           std::ranges::any_of(info.images, uses_mapping);
+	info.uses_image_heap = std::ranges::any_of(info.images, &ImageResource::heap);
 	switch (program.stage) {
 		case ShaderType::Vertex:
 		case ShaderType::Local: CollectVertexInputs(input_info.vertex, info, inputs.components); break;

src/graphics/shader/recompiler/ir/passes/SrtWalker.cpp

@@ -134,11 +134,9 @@ bool IsRuntimeUniformOp(ValueOpcode op) {
 		case ValueOpcode::BitFieldSExtract:
 		case ValueOpcode::IAdd32:
 		case ValueOpcode::IAdd64:
-		case ValueOpcode::IAddCarry32:
 		case ValueOpcode::ISub32:
 		case ValueOpcode::ISub64:
 		case ValueOpcode::IMul32:
-		case ValueOpcode::IMul64:
 		case ValueOpcode::UMulHi:
 		case ValueOpcode::UMin32:
 		case ValueOpcode::UMax32:
@@ -259,9 +257,7 @@ private:
 			}
 			return finish(Validate(inst->Arg(1)));
 		}
-		if (op == ValueOpcode::UndefU1 || op == ValueOpcode::UndefU8 ||
-		    op == ValueOpcode::UndefU16 || op == ValueOpcode::UndefU32 ||
-		    op == ValueOpcode::UndefU64 || op == ValueOpcode::Void) {
+		if (op == ValueOpcode::UndefU1 || op == ValueOpcode::UndefU32 || op == ValueOpcode::Void) {
 			return finish(false);
 		}
 		if (op == ValueOpcode::GetUserData) {
@@ -341,9 +337,7 @@ private:
 			const auto* source = inst->NumArgs() == 2 ? inst->Arg(0).ResolveInstruction() : nullptr;
 			const auto  index  = inst->NumArgs() == 2 ? inst->Arg(1).Resolve() : Value {};
 			if (source == nullptr || !index.IsImmediate() || index.GetType() != Type::U32 ||
-			    index.U32() >= 2u ||
-			    (source->GetOpcode() != ValueOpcode::CompositeConstructU32x2 &&
-			     source->GetOpcode() != ValueOpcode::IAddCarry32)) {
+			    index.U32() >= 2u || source->GetOpcode() != ValueOpcode::CompositeConstructU32x2) {
 				return finish(false);
 			}
 		}
@@ -656,18 +650,6 @@ bool SrtWalker::EvaluateExtract(const Inst& inst, uint64_t& result) {
 	if (source->GetOpcode() == ValueOpcode::CompositeConstructU32x2) {
 		return EvaluateWide(source->Arg(component), result);
 	}
-	if (source->GetOpcode() == ValueOpcode::IAddCarry32) {
-		uint64_t lhs = 0;
-		uint64_t rhs = 0;
-		if (!Arg(*source, 0, lhs) || !Arg(*source, 1, rhs)) {
-			return false;
-		}
-		const auto sum =
-		    static_cast<uint64_t>(static_cast<uint32_t>(lhs)) + static_cast<uint32_t>(rhs);
-		result =
-		    component == 0u ? static_cast<uint32_t>(sum) : static_cast<uint32_t>(sum >> 32u);
-		return true;
-	}
 	return false;
 }
 
@@ -832,12 +814,6 @@ bool SrtWalker::EvaluateInst(const Inst& inst, uint64_t& result) {
 				return true;
 			}
 			return false;
-		case ValueOpcode::IMul64:
-			if (binary()) {
-				result = a * b;
-				return true;
-			}
-			return false;
 		case ValueOpcode::UMulHi:
 			if (binary()) {
 				result = (static_cast<uint64_t>(static_cast<uint32_t>(a)) * static_cast<uint32_t>(b)) >> 32u;
@@ -861,12 +837,11 @@ bool SrtWalker::EvaluateInst(const Inst& inst, uint64_t& result) {
 			return false;
 		case ValueOpcode::ConvertU32F32:
 			if (Arg(inst, 0, a)) {
+				// The conversion saturates and maps NaN to zero.
 				const auto value = Float32(a);
-				if (!std::isfinite(value) || value < 0.0f ||
-				    static_cast<double>(value) > UINT32_MAX) {
-					return false;
-				}
-				result = static_cast<uint32_t>(value);
+				result           = std::isnan(value) || value <= 0.0f ? 0u
+				                   : value >= 4294967296.0f ? UINT32_MAX
+				                                            : static_cast<uint32_t>(value);
 				return true;
 			}
 			return false;
@@ -1143,10 +1118,7 @@ bool SrtWalker::EvaluateInst(const Inst& inst, uint64_t& result) {
 			}
 			return false;
 		case ValueOpcode::UndefU1:
-		case ValueOpcode::UndefU8:
-		case ValueOpcode::UndefU16:
-		case ValueOpcode::UndefU32:
-		case ValueOpcode::UndefU64: return false;
+		case ValueOpcode::UndefU32: return false;
 		default: break;
 	}
 	return false;

src/graphics/shader/recompiler/ir/passes/SrtWalker.h

@@ -12,6 +12,33 @@ class Value;
 
 using SrtMemoryReader = bool (*)(void* userdata, uint64_t address, std::span<uint32_t> values);
 
+// Image descriptors at base + first + k * step, k < count, within size bytes that a sampled
+// root selects by GPU-computed byte offset. The host keeps them resident instead of
+// materializing them.
+struct ImageHeapRequest {
+	uint64_t             base  = 0;
+	uint64_t             size  = 0;
+	uint64_t             step  = 0;
+	uint64_t             first = 0;
+	uint64_t             count = 0;
+	const ImageResource* root  = nullptr;
+};
+
+struct ImageHeapView {
+	uint64_t                         slots      = 0; // Device address of one slot per heap dword.
+	uint32_t                         slot_count = 0; // Keys past it select a trailing null slot.
+	std::span<const DescriptorValue> kinds; // One representative descriptor per kind.
+};
+
+class ImageHeapSource {
+public:
+	// False when the host does not keep the table resident.
+	virtual bool Acquire(const ImageHeapRequest& request, ImageHeapView& view) = 0;
+
+protected:
+	~ImageHeapSource() = default;
+};
+
 struct SrtRuntime {
 	std::span<const uint32_t> user_data;
 	uint64_t                  shader_base                = 0;
@@ -20,6 +47,7 @@ struct SrtRuntime {
 	SrtMemoryReader           read_specialization_memory = nullptr;
 	std::span<const uint32_t> workgroup_counts;
 	uint64_t                  indirect_args_address = 0;
+	ImageHeapSource*          image_heap            = nullptr;
 };
 
 class SrtReadCapture {

src/kernel/fileSystem.cpp

@@ -410,6 +410,11 @@ std::filesystem::path GetRealFilename(const std::string& mounted_file_name) {
 	return g_mount_points->ResolvePath(mounted_file_name);
 }
 
+bool IsOpenDescriptor(int d) {
+	const auto* file = g_files->GetFile(d);
+	return file != nullptr && file->opened;
+}
+
 // NOLINTNEXTLINE(readability-function-cognitive-complexity)
 int KYTY_SYSV_ABI KernelOpen(const char* path, int flags, uint16_t mode) {
 	PRINT_NAME();

src/kernel/fileSystem.h

@@ -56,6 +56,8 @@ void                  Mount(const std::filesystem::path& folder, const std::stri
 void                  Umount(const std::string& folder_or_point);
 // Returns an empty path when the guest path has no mounted filesystem.
 std::filesystem::path GetRealFilename(const std::string& mounted_file_name);
+// Returns whether d is an open file, directory or device descriptor.
+bool                  IsOpenDescriptor(int d);
 
 int KYTY_SYSV_ABI     KernelOpen(const char* path, int flags, uint16_t mode);
 int KYTY_SYSV_ABI     KernelClose(int d);

src/launcher/forms/configuration_edit_dialog.ui

@@ -168,12 +168,18 @@
              </widget>
             </item>
             <item row="6" column="1">
+             <widget class="QCheckBox" name="checkBox_readback_tiled">
+              <property name="toolTip"><string>Read back small writable tiled images</string></property>
+              <property name="text"><string>Enable tiled readback</string></property>
+             </widget>
+            </item>
+            <item row="7" column="1">
              <widget class="QCheckBox" name="checkBox_tessellation">
               <property name="toolTip"><string>Draw tessellated geometry. Disabled by default; unsupported tessellation may cause crashes.</string></property>
               <property name="text"><string>Enable tessellation support (experimental)</string></property>
              </widget>
             </item>
-            <item row="7" column="1">
+            <item row="8" column="1">
              <widget class="QCheckBox" name="checkBox_hdr">
               <property name="toolTip"><string>Enable HDR10 output for supported games. Requires HDR enabled on the host display.</string></property>
               <property name="text"><string>Enable HDR display support</string></property>

src/launcher/icons/kyty-small.svg

@@ -0,0 +1,15 @@
+<svg xmlns="http://www.w3.org/2000/svg" viewBox="0 0 512 512"><defs>
+<linearGradient id="body" x1="0" y1="0" x2="0" y2="1"><stop offset="0" stop-color="#ffffff"/><stop offset="1" stop-color="#e3e9f2"/></linearGradient>
+</defs>
+<g transform="translate(256 268) scale(1.16) translate(-256 -268)" stroke-linejoin="round" stroke-linecap="round">
+<path d="M100 196 L120 104 Q124 92 134 99 L214 160 Z" fill="url(#body)"/><path d="M412 196 L392 104 Q388 92 378 99 L298 160 Z" fill="url(#body)"/>
+<path d="M116 181 L137 121 L190 157 Z" fill="#0b0d12"/><path d="M396 181 L375 121 L322 157 Z" fill="#0b0d12"/>
+<g fill="none" stroke="#0b0d12" stroke-width="20"><path d="M100 196 L120 104 Q124 92 134 99 L214 160 Z"/><path d="M412 196 L392 104 Q388 92 378 99 L298 160 Z"/></g>
+<path d="M256 154 C330 154 380 160 406 178 C432 196 447 244 459 302 C471 358 466 398 430 408 C400 416 380 398 360 368 L326 318 C316 304 306 300 290 300 L222 300 C206 300 196 304 186 318 L152 368 C132 398 112 416 82 408 C46 398 41 358 53 302 C65 244 80 196 106 178 C132 160 182 154 256 154 Z" fill="url(#body)"/>
+<path d="M256 154 C330 154 380 160 406 178 C432 196 447 244 459 302 C471 358 466 398 430 408 C400 416 380 398 360 368 L326 318 C316 304 306 300 290 300 L222 300 C206 300 196 304 186 318 L152 368 C132 398 112 416 82 408 C46 398 41 358 53 302 C65 244 80 196 106 178 C132 160 182 154 256 154 Z" fill="none" stroke="#0b0d12" stroke-width="20"/>
+<g fill="none" stroke="#0b0d12" stroke-width="14"><path d="M82 260 C100 274 118 288 128 304 C134 316 121 350 108 388"/><path d="M430 260 C412 274 394 288 384 304 C378 316 391 350 404 388"/></g>
+<path d="M72 316 L92 336 M70 336 L86 352 M440 316 L420 336 M442 336 L426 352" fill="none" stroke="#0b0d12" stroke-width="10"/>
+<path d="M179 230 Q194 210 209 230" fill="none" stroke="#0b0d12" stroke-width="16"/>
+<circle cx="318" cy="222" r="21" fill="#0b0d12"/>
+<path d="M240 256 L272 256 L256 278 Z" fill="#0b0d12" stroke="#0b0d12" stroke-width="8"/>
+</g></svg>

src/launcher/icons/kyty.svg

@@ -0,0 +1,15 @@
+<svg xmlns="http://www.w3.org/2000/svg" viewBox="0 0 512 512"><defs>
+<linearGradient id="body" x1="0" y1="0" x2="0" y2="1"><stop offset="0" stop-color="#ffffff"/><stop offset="1" stop-color="#e3e9f2"/></linearGradient>
+</defs>
+<g transform="translate(256 268) scale(1.16) translate(-256 -268)" stroke-linejoin="round" stroke-linecap="round">
+<path d="M100 196 L120 104 Q124 92 134 99 L214 160 Z" fill="url(#body)"/><path d="M412 196 L392 104 Q388 92 378 99 L298 160 Z" fill="url(#body)"/>
+<path d="M128 164 L137 128 L174 156 Z" fill="#0b0d12"/><path d="M384 164 L375 128 L338 156 Z" fill="#0b0d12"/>
+<g fill="none" stroke="#0b0d12" stroke-width="16"><path d="M100 196 L120 104 Q124 92 134 99 L214 160 Z"/><path d="M412 196 L392 104 Q388 92 378 99 L298 160 Z"/></g>
+<path d="M256 154 C330 154 380 160 406 178 C432 196 447 244 459 302 C471 358 466 398 430 408 C400 416 380 398 360 368 L326 318 C316 304 306 300 290 300 L222 300 C206 300 196 304 186 318 L152 368 C132 398 112 416 82 408 C46 398 41 358 53 302 C65 244 80 196 106 178 C132 160 182 154 256 154 Z" fill="url(#body)"/>
+<path d="M256 154 C330 154 380 160 406 178 C432 196 447 244 459 302 C471 358 466 398 430 408 C400 416 380 398 360 368 L326 318 C316 304 306 300 290 300 L222 300 C206 300 196 304 186 318 L152 368 C132 398 112 416 82 408 C46 398 41 358 53 302 C65 244 80 196 106 178 C132 160 182 154 256 154 Z" fill="none" stroke="#0b0d12" stroke-width="16"/>
+<g fill="none" stroke="#0b0d12" stroke-width="9"><path d="M82 260 C100 274 118 288 128 304 C134 316 121 350 108 388"/><path d="M430 260 C412 274 394 288 384 304 C378 316 391 350 404 388"/></g>
+<path d="M64 322 L82 340 M62 340 L76 354 M448 322 L430 340 M450 340 L436 354" fill="none" stroke="#0b0d12" stroke-width="7"/>
+<path d="M168 230 Q194 202 220 230" fill="none" stroke="#0b0d12" stroke-width="15"/>
+<circle cx="322" cy="220" r="22" fill="#0b0d12"/>
+<path d="M238 252 L274 252 L256 276 Z" fill="#0b0d12" stroke="#0b0d12" stroke-width="8"/>
+</g></svg>

src/launcher/include/configuration.h

@@ -126,6 +126,7 @@ public:
 	bool                   hdr_enabled                 = false;
 	bool                   hide_cursor_enabled         = false;
 	bool                   readback_linear_images      = false;
+	bool                   readback_tiled_images       = false;
 	bool                   sync_raw_image_buffers      = false;
 	bool                   tessellation_enabled        = false;
 	bool                   trophy_enabled              = true;
@@ -165,6 +166,7 @@ public:
 		hdr_enabled                 = other.hdr_enabled;
 		hide_cursor_enabled         = other.hide_cursor_enabled;
 		readback_linear_images      = other.readback_linear_images;
+		readback_tiled_images       = other.readback_tiled_images;
 		sync_raw_image_buffers      = other.sync_raw_image_buffers;
 		tessellation_enabled        = other.tessellation_enabled;
 		trophy_enabled              = other.trophy_enabled;

src/launcher/include/configurationListWidget.h

@@ -68,6 +68,7 @@ protected slots:
 	void list_itemDoubleClicked(QTreeWidgetItem* witem, int column);
 	void open_game_folder();
 	void remove_save_data();
+	void compress_game();
 	void filter_configurations(const QString& text);
 
 private:

src/launcher/resources.qrc

@@ -7,6 +7,8 @@
   <file alias="grid-view.svg">icons/grid-view.svg</file>
   <file alias="remove-configuration.svg">icons/remove-configuration.svg</file>
   <file alias="trophy.svg">icons/trophy.svg</file>
+  <file alias="kyty.svg">icons/kyty.svg</file>
+  <file alias="kyty-small.svg">icons/kyty-small.svg</file>
   <file alias="hidden-trophy.png">../../assets/images/hidden-trophy.png</file>
  </qresource>
  <qresource prefix="/styles">

src/launcher/src/configuration.cpp

@@ -13,6 +13,7 @@ QVariantMap Configuration::GameSettings() const {
 	KYTY_CFG_SET(hdr_enabled);
 	KYTY_CFG_SET(hide_cursor_enabled);
 	KYTY_CFG_SET(readback_linear_images);
+	KYTY_CFG_SET(readback_tiled_images);
 	KYTY_CFG_SET(sync_raw_image_buffers);
 	KYTY_CFG_SET(tessellation_enabled);
 	KYTY_CFG_SET(trophy_enabled);
@@ -62,6 +63,7 @@ void Configuration::ReadGameSettingsValues(const Settings& s) {
 	KYTY_CFG_GET(hdr_enabled);
 	KYTY_CFG_GET(hide_cursor_enabled);
 	KYTY_CFG_GET(readback_linear_images);
+	KYTY_CFG_GET(readback_tiled_images);
 	KYTY_CFG_GET(sync_raw_image_buffers);
 	KYTY_CFG_GET(tessellation_enabled);
 	trophy_enabled   = s.value("trophy_enabled", trophy_enabled).toBool();

src/launcher/src/configurationEditDialog.cpp

@@ -286,6 +286,7 @@ void ConfigurationEditDialog::Init(const Configuration& info) {
 	m_ui->checkBox_hdr->setChecked(info.hdr_enabled);
 	m_ui->checkBox_hide_cursor->setChecked(info.hide_cursor_enabled);
 	m_ui->checkBox_readback->setChecked(info.readback_linear_images);
+	m_ui->checkBox_readback_tiled->setChecked(info.readback_tiled_images);
 	m_ui->checkBox_sync_raw_image_buffers->setChecked(info.sync_raw_image_buffers);
 	m_ui->checkBox_tessellation->setChecked(info.tessellation_enabled);
 	m_ui->checkBox_skip_notice_screen->setChecked(info.skip_notice_screen);
@@ -479,6 +480,7 @@ static void UpdateInfo(Configuration& info, Ui::ConfigurationEditDialog& ui, boo
 	info.hdr_enabled               = ui.checkBox_hdr->isChecked();
 	info.hide_cursor_enabled       = ui.checkBox_hide_cursor->isChecked();
 	info.readback_linear_images    = ui.checkBox_readback->isChecked();
+	info.readback_tiled_images     = ui.checkBox_readback_tiled->isChecked();
 	info.sync_raw_image_buffers    = ui.checkBox_sync_raw_image_buffers->isChecked();
 	info.tessellation_enabled      = ui.checkBox_tessellation->isChecked();
 	info.skip_notice_screen        = ui.checkBox_skip_notice_screen->isChecked();

src/launcher/src/configurationListWidget.cpp

@@ -27,6 +27,7 @@
 #include <QFile>
 #include <QFileDialog>
 #include <QFileInfo>
+#include <QFutureWatcher>
 #include <QHeaderView>
 #include <QIcon>
 #include <QInputDialog>
@@ -42,6 +43,8 @@
 #include <QPalette>
 #include <QPointer>
 #include <QProcess>
+#include <QProgressDialog>
+#include <QPromise>
 #include <QPushButton>
 #include <QRegularExpression>
 #include <QSaveFile>
@@ -54,6 +57,7 @@
 #include <QToolButton>
 #include <QTreeWidget>
 #include <QUrl>
+#include <QtConcurrentRun>
 #include <QtCore>
 
 #ifdef __linux__
@@ -62,6 +66,7 @@
 #include <QDBusPendingCallWatcher>
 #endif
 
+#include <atomic>
 #include <memory>
 #include <vector>
 
@@ -1171,6 +1176,71 @@ void ConfigurationListWidget::remove_save_data() {
 	}
 }
 
+void ConfigurationListWidget::compress_game() {
+	auto* item = static_cast<ConfigurationItem*>(m_ui->cfgs_list->currentItem());
+	if (item == nullptr) {
+		return;
+	}
+
+	const auto base    = item->GetInfo().basedir;
+	const auto archive = base + QStringLiteral(".zar");
+	const auto title   = tr("Compress to ZAR");
+	if (QFileInfo::exists(archive)) {
+		QMessageBox::warning(this, title,
+		                     tr("%1 already exists.").arg(QDir::toNativeSeparators(archive)));
+		return;
+	}
+
+	// Cancel through a flag, not the future, so a pack that already succeeded keeps its result.
+	auto  canceled = std::make_shared<std::atomic_bool>(false);
+	auto* progress = new QProgressDialog(
+	    tr("Compressing %1...").arg(QDir::toNativeSeparators(base)), tr("Cancel"), 0, 1000, this);
+	auto* watcher  = new QFutureWatcher<bool>(this);
+	// Stay modal through finalization and fsync, which follow the last progress step.
+	progress->setAutoReset(false);
+	connect(progress, &QProgressDialog::canceled, this, [canceled] { *canceled = true; });
+	connect(watcher, &QFutureWatcher<bool>::progressValueChanged, progress,
+	        &QProgressDialog::setValue);
+	connect(watcher, &QFutureWatcher<bool>::finished, this,
+	        [this, progress, watcher, canceled, base, archive, title] {
+		        // finished can arrive inside the modal dialog's setValue event processing.
+		        progress->deleteLater();
+		        watcher->deleteLater();
+		        if (!watcher->result()) {
+			        if (!*canceled) {
+				        QMessageBox::warning(
+				            this, title,
+				            tr("Could not compress %1.").arg(QDir::toNativeSeparators(base)));
+			        }
+			        return;
+		        }
+		        if (QMessageBox::Yes ==
+		                QMessageBox::question(this, title,
+		                                      tr("Created %1.\n\nDelete the original folder %2?")
+		                                          .arg(QDir::toNativeSeparators(archive),
+		                                               QDir::toNativeSeparators(base)),
+		                                      QMessageBox::Yes | QMessageBox::No, QMessageBox::No) &&
+		            !QDir(base).removeRecursively()) {
+			        QMessageBox::warning(
+			            this, title, tr("Could not remove %1.").arg(QDir::toNativeSeparators(base)));
+		        }
+		        ScanGameDirectory();
+	        });
+	progress->open();
+	watcher->setFuture(QtConcurrent::run(
+	    [directory = GameContent::ToPath(base), path = GameContent::ToPath(archive),
+	     canceled](QPromise<bool>& promise) {
+		    promise.setProgressRange(0, 1000);
+		    // A pack cancelled while queued must not touch a path a later pack may already own.
+		    promise.addResult(
+		        !*canceled &&
+		        Common::PackArchive(directory, path, [&](uint64_t done, uint64_t total) {
+			        promise.setProgressValue(static_cast<int>(done * 1000 / total));
+			        return !*canceled;
+		        }));
+	    }));
+}
+
 void ConfigurationListWidget::filter_configurations(const QString& text) {
 	const auto query              = text.trimmed();
 	const bool has_query          = !query.isEmpty();
@@ -1268,6 +1338,8 @@ void ConfigurationListWidget::ShowContextMenu(QTreeWidgetItem* witem) {
 		        }
 	        });
 	action_addcont->setEnabled(item != nullptr);
+	QAction* action_compress =
+	    menu.addAction(tr("Compress to ZAR..."), this, SLOT(compress_game()));
 	QAction* action_remove_save_data =
 	    menu.addAction(style()->standardIcon(QStyle::SP_DialogDiscardButton),
 	                   tr("Remove save data..."), this, SLOT(remove_save_data()));
@@ -1291,6 +1363,7 @@ void ConfigurationListWidget::ShowContextMenu(QTreeWidgetItem* witem) {
 		const auto& base = item->GetInfo().basedir;
 		action_open_folder->setDisabled(!QDir(base).exists() && !GameContent::IsArchive(base));
 		action_view_trophies->setDisabled(!has_trophy_data);
+		action_compress->setDisabled(item->IsRunning() || !QDir(base).exists());
 		action_remove_save_data->setDisabled(item->IsRunning() || save_data_dirs.isEmpty());
 		action_edit->setDisabled(item->IsRunning());
 		action_delete->setDisabled(item->IsRunning() || !item->GetInfo().custom_settings);
@@ -1298,6 +1371,7 @@ void ConfigurationListWidget::ShowContextMenu(QTreeWidgetItem* witem) {
 		action_run->setDisabled(true);
 		action_open_folder->setDisabled(true);
 		action_view_trophies->setDisabled(true);
+		action_compress->setDisabled(true);
 		action_remove_save_data->setDisabled(true);
 		action_edit->setDisabled(true);
 		action_delete->setDisabled(true);

src/launcher/src/main.cpp

@@ -2,9 +2,21 @@
 #include "launcherTheme.h"
 
 #include <QApplication>
+#include <QIcon>
 
 int main(int argc, char* argv[]) {
 	QApplication a(argc, argv);
+	// Title bars and taskbars get the bolder small-size artwork.
+	const QIcon small_art(QStringLiteral(":/icons/kyty-small.svg"));
+	const QIcon large_art(QStringLiteral(":/icons/kyty.svg"));
+	QIcon       icon;
+	for (int size: {16, 24, 32, 48, 64}) {
+		icon.addPixmap(small_art.pixmap(QSize(size, size), 1.0));
+	}
+	for (int size: {128, 256}) {
+		icon.addPixmap(large_art.pixmap(QSize(size, size), 1.0));
+	}
+	QApplication::setWindowIcon(icon);
 	LauncherTheme::Initialize(a);
 
 	MainDialog w;

src/launcher/src/mainDialog.cpp

@@ -279,6 +279,7 @@ static QStringList CreateEmulatorArgs(const Configuration& info) {
 	AddFlag(args, info.hdr_enabled, "--hdr");
 	AddFlag(args, info.hide_cursor_enabled, "--hide-cursor");
 	AddFlag(args, info.readback_linear_images, "--readback-linear-images");
+	AddFlag(args, info.readback_tiled_images, "--readback-tiled-images");
 	AddFlag(args, info.sync_raw_image_buffers, "--sync-raw-image-buffers");
 	AddFlag(args, !info.trophy_enabled, "--no-trophy-notifications");
 	args << "--trophy-duration" << QString::number(info.trophy_duration_seconds);

src/launcher/src/updateChecker.cpp

@@ -4,6 +4,7 @@
 
 #include <QAbstractButton>
 #include <QApplication>
+#include <QDateTime>
 #include <QDebug>
 #include <QDesktopServices>
 #include <QDir>
@@ -96,7 +97,27 @@ bool UpdateChecker::IsSupported() {
 
 void UpdateChecker::ShowPreviousResult(QWidget* parent) {
 	const auto directory = InstallDirectory();
-	QFile      result(QDir(directory).filePath(QStringLiteral(".kyty-update-result")));
+	if (!directory.isEmpty()) {
+		// The helper cannot delete its own executable on Windows, so its workspace is
+		// removed by the next launcher start.
+		const QDir install(directory);
+		for (const auto& name:
+		     install.entryList({QStringLiteral(".kyty-update-??????")},
+		                       QDir::Dirs | QDir::Hidden | QDir::NoSymLinks | QDir::NoDotAndDotDot)) {
+			QDir       workspace(install.filePath(name));
+			QDateTime  newest = QFileInfo(workspace.path()).lastModified();
+			for (const auto& entry: workspace.entryInfoList(QDir::Files | QDir::Dirs | QDir::Hidden |
+			                                                QDir::NoDotAndDotDot)) {
+				newest = qMax(newest, entry.lastModified());
+			}
+			// A workspace touched recently may belong to a running download or helper. Windows
+			// lists a file still being downloaded with a stale time, so allow a generous margin.
+			if (newest.secsTo(QDateTime::currentDateTime()) > 60 * 60) {
+				workspace.removeRecursively();
+			}
+		}
+	}
+	QFile result(QDir(directory).filePath(QStringLiteral(".kyty-update-result")));
 	if (directory.isEmpty() || !result.open(QIODevice::ReadOnly)) {
 		return;
 	}

src/libs/libAudio2.cpp

@@ -178,7 +178,6 @@ struct AudioOut2ContextState {
 	uint32_t               num_grains          = 512;
 	uint64_t               last_update         = 0;
 	int                    object_audio_handle = 0;
-	std::vector<float>     object_mix;
 };
 
 struct AudioOut2PortStateEntry {
@@ -212,8 +211,18 @@ struct AudioOut2LatencyState {
 	uint32_t latency_us = 0;
 };
 
+// Per-context copy of what a push hands the backend. The backend can sleep while pacing, so a push
+// holds only its context's mutex across it; closing the context's handles waits on that mutex.
+struct AudioOut2Submission {
+	Common::Mutex                                                   mutex;
+	std::array<std::vector<uint8_t>, AudioInternal::OUT_PORTS_MAX>  pcm;
+	std::array<std::array<float, 16>, AudioInternal::OUT_PORTS_MAX> gains;
+	std::vector<float>                                              object_mix;
+};
+
 static Common::Mutex                              g_audioout2_state_mutex;
 static std::array<AudioOut2ContextState, 16>      g_audioout2_contexts;
+static std::array<AudioOut2Submission, 16>        g_audioout2_submissions;
 static std::array<AudioOut2PortStateEntry, 256>   g_audioout2_ports;
 static std::vector<AudioOut2UserHandle>          g_audioout2_users;
 static Common::Mutex                              g_audioout2_speaker_array_mutex;
@@ -249,6 +258,10 @@ static AudioOut2ContextState* audioout2_find_context_locked(AudioOut2ContextHand
 	return nullptr;
 }
 
+static AudioOut2Submission& audioout2_submission(const AudioOut2ContextState& context) {
+	return g_audioout2_submissions[&context - g_audioout2_contexts.data()];
+}
+
 static uint32_t audioout2_grain_micros(uint32_t grains) {
 	const auto sample_count = (grains == 0 ? 512u : grains);
 	return std::max<uint32_t>((sample_count * 1000000u) / 48000u, 1000u);
@@ -372,11 +385,10 @@ static bool audioout2_context_has_queueable_device(const AudioOut2ContextState&
 	return false;
 }
 
-static void audioout2_mix_object(AudioOut2ContextState&         context,
+static void audioout2_mix_object(std::vector<float>& mix, uint32_t num_grains,
                                  const AudioOut2PortStateEntry& object) {
-	auto& mix = context.object_mix;
 	if (mix.empty()) {
-		mix.resize(static_cast<size_t>(context.num_grains) * 2, 0.0f);
+		mix.resize(static_cast<size_t>(num_grains) * 2, 0.0f);
 	}
 
 	// Center ordinary objects. First-order stereo ambisonics uses left/right-facing
@@ -393,7 +405,7 @@ static void audioout2_mix_object(AudioOut2ContextState&         context,
 	}
 	left *= object.gains[0];
 	right *= object.gains[0];
-	for (uint32_t frame = 0; frame < context.num_grains; frame++) {
+	for (uint32_t frame = 0; frame < num_grains; frame++) {
 		const float sample =
 		    object.audio_format == AudioInternal::Format::FloatMono
 		        ? reinterpret_cast<const float*>(object.pcm_data.data())[frame]
@@ -403,15 +415,16 @@ static void audioout2_mix_object(AudioOut2ContextState&         context,
 	}
 }
 
-static void audioout2_queue_context_audio(AudioOut2ContextHandle ctx, bool blocking) {
-	// The backend consumes the buffers synchronously, but can block while pacing SDL's queue.
-	// Keep both PCM storage and port handles alive until it returns.
-	Common::LockGuard lock(g_audioout2_state_mutex);
-	auto*             context = audioout2_find_context_locked(ctx);
+static void audioout2_queue_context_audio(AudioOut2ContextHandle ctx, AudioOut2Submission& out,
+                                          bool blocking) {
+	Common::LockGuard submit_lock(out.mutex);
+	g_audioout2_state_mutex.Lock();
+	auto* context = audioout2_find_context_locked(ctx);
 	if (context == nullptr) {
+		g_audioout2_state_mutex.Unlock();
 		return;
 	}
-	context->object_mix.clear();
+	out.object_mix.clear();
 	std::vector<AudioInternal::OutputParam> params;
 	params.reserve(AudioInternal::OUT_PORTS_MAX);
 	for (const auto& state: g_audioout2_ports) {
@@ -419,14 +432,18 @@ static void audioout2_queue_context_audio(AudioOut2ContextHandle ctx, bool block
 			continue;
 		}
 		if (audioout2_port_type_is_object(state.port_type)) {
-			audioout2_mix_object(*context, state);
+			audioout2_mix_object(out.object_mix, context->num_grains, state);
 		} else if (state.audio_handle > 0 && params.size() < AudioInternal::OUT_PORTS_MAX) {
-			params.push_back({state.audio_handle, state.pcm_data.data(), state.gains.data()});
+			const auto i = params.size();
+			out.pcm[i]   = state.pcm_data;
+			out.gains[i] = state.gains;
+			params.push_back({state.audio_handle, out.pcm[i].data(), out.gains[i].data()});
 		}
 	}
-	if (!context->object_mix.empty() && params.size() < AudioInternal::OUT_PORTS_MAX) {
-		params.push_back({context->object_audio_handle, context->object_mix.data()});
+	if (!out.object_mix.empty() && params.size() < AudioInternal::OUT_PORTS_MAX) {
+		params.push_back({context->object_audio_handle, out.object_mix.data()});
 	}
+	g_audioout2_state_mutex.Unlock();
 	if (!params.empty()) {
 		(void)AudioInternal::AudioOutOutputs(params.data(), static_cast<uint32_t>(params.size()),
 		                                     blocking);
@@ -528,7 +545,8 @@ int KYTY_SYSV_ABI AudioOut2ContextDestroy(AudioOut2ContextHandle ctx) {
 	size_t audio_handles_num = 0;
 
 	g_audioout2_state_mutex.Lock();
-	if (auto* state = audioout2_find_context_locked(ctx); state != nullptr) {
+	auto* state = audioout2_find_context_locked(ctx);
+	if (state != nullptr) {
 		if (state->object_audio_handle > 0) {
 			audio_handles[audio_handles_num++] = state->object_audio_handle;
 		}
@@ -544,6 +562,10 @@ int KYTY_SYSV_ABI AudioOut2ContextDestroy(AudioOut2ContextHandle ctx) {
 	}
 	g_audioout2_state_mutex.Unlock();
 
+	if (state != nullptr) {
+		// Wait for an in-flight push that may still use the handles.
+		Common::LockGuard wait(audioout2_submission(*state).mutex);
+	}
 	for (size_t i = 0; i < audio_handles_num; i++) {
 		audioout2_close_audio_handle(audio_handles[i]);
 	}
@@ -595,8 +617,9 @@ int KYTY_SYSV_ABI AudioOut2ContextPush(AudioOut2ContextHandle ctx, uint32_t bloc
 				if (state->queued < state->queue_depth) {
 					state->queued++;
 				}
+				auto& submission = audioout2_submission(*state);
 				g_audioout2_state_mutex.Unlock();
-				audioout2_queue_context_audio(ctx, blocking != 0);
+				audioout2_queue_context_audio(ctx, submission, blocking != 0);
 				return OK;
 			}
 		}
@@ -729,14 +752,20 @@ int KYTY_SYSV_ABI AudioOut2PortDestroy(AudioOut2PortHandle port) {
 	PRINT_NAME();
 	LOGF("\t port = 0x%016" PRIx64 "\n", port);
 
-	int audio_handle = 0;
+	int                          audio_handle = 0;
+	const AudioOut2ContextState* context      = nullptr;
 	g_audioout2_state_mutex.Lock();
 	if (auto* state = audioout2_find_port_locked(port); state != nullptr) {
 		audio_handle = state->audio_handle;
+		context      = audioout2_find_context_locked(state->context);
 		*state       = AudioOut2PortStateEntry {};
 	}
 	g_audioout2_state_mutex.Unlock();
 
+	if (audio_handle > 0 && context != nullptr) {
+		// Wait for an in-flight push that may still use the handle.
+		Common::LockGuard wait(audioout2_submission(*context).mutex);
+	}
 	audioout2_close_audio_handle(audio_handle);
 
 	return OK;