Associate stencil when depth is acquired

Co-authored-by: Daniil Shushakov <4shushakov@gmail.com>

src/common/file.cpp

@@ -453,13 +453,8 @@ std::vector<File::DirEntry> File::GetDirEntries(const std::filesystem::path& pat
 	std::vector<File::DirEntry> ret;
 	ret.reserve(files.size());
 
-	for (const auto& f: files) {
-		File::DirEntry r {};
-
-		r.name    = f.name;
-		r.is_file = f.is_file;
-
-		ret.push_back(std::move(r));
+	for (auto& f: files) {
+		ret.emplace_back(std::move(f.name), f.is_file);
 	}
 
 	return ret;

src/graphics/guest_gpu/command_processor/commandProcessor.h

@@ -73,6 +73,9 @@ public:
 	void SetIndexBufferSize(uint32_t index_buffer_size);
 	void SetDrawIndirectArgsBaseAddress(uint64_t draw_indirect_args_base_addr);
 	void SetDispatchIndirectArgsBaseAddress(uint64_t dispatch_indirect_args_base_addr);
+	[[nodiscard]] uint64_t GetDispatchIndirectArgsBaseAddress() const {
+		return m_dispatch_indirect_args_base_addr;
+	}
 	void SetNumInstances(uint32_t num_instances);
 	void DrawIndex(DrawIndexArgs args);
 	void DrawIndexOffset(uint32_t index_offset, uint32_t index_count);
@@ -99,7 +102,7 @@ public:
 	void TriggerEopEventAtEndOfPipe(uint32_t interrupt_context_id);
 	void DispatchDirect(uint32_t thread_group_x, uint32_t thread_group_y, uint32_t thread_group_z,
 	                    uint32_t mode);
-	void DispatchIndirect(uint32_t data_offset, uint32_t mode);
+	void DispatchIndirect(uint64_t args_addr, uint32_t mode);
 	void WaitFlipDone(uint32_t video_out_handle, uint32_t display_buffer_index);
 	void TriggerEvent(uint32_t event_type, uint32_t event_index, uint64_t event_address = 0);
 
@@ -148,7 +151,6 @@ private:
 	void SuspendPm4();
 	CommandScheduler&   GetScheduler() const { return m_renderer.GetCommandScheduler(); }
 	CommandBuffer&      CurrentBuffer() { return GetScheduler().Current(); }
-	void                CheckBuffer() const { GetScheduler().CheckActive(); }
 
 	RenderContext&   m_renderer;
 	HW::Context      m_ctx;

src/graphics/guest_gpu/command_processor/pm4Handlers.cpp

@@ -1331,26 +1331,13 @@ KYTY_CP_OP_PARSER(CpOpDispatchIndirect) {
 	EXIT_NOT_IMPLEMENTED(cmd_id != 0xc0011600 && cmd_id != 0xc0021600);
 
 	if (cmd_id == 0xc0021600) {
-		struct DispatchIndirectArgs {
-			uint32_t thread_group_x;
-			uint32_t thread_group_y;
-			uint32_t thread_group_z;
-		};
-
-		auto* args = reinterpret_cast<const DispatchIndirectArgs*>(
-		    buffer[0] | (static_cast<uint64_t>(buffer[1]) << 32u));
-		uint32_t mode = buffer[2];
-
-		EXIT_NOT_IMPLEMENTED(args == nullptr);
-		cp.DispatchDirect(args->thread_group_x, args->thread_group_y, args->thread_group_z, mode);
-
+		cp.DispatchIndirect(buffer[0] | (static_cast<uint64_t>(buffer[1]) << 32u), buffer[2]);
 		return 3;
 	}
 
-	uint32_t data_offset = buffer[0];
-	uint32_t mode        = buffer[1];
-
-	cp.DispatchIndirect(data_offset, mode);
+	const auto base_addr = cp.GetDispatchIndirectArgsBaseAddress();
+	EXIT_NOT_IMPLEMENTED(base_addr == 0);
+	cp.DispatchIndirect(base_addr + buffer[0], buffer[1]);
 
 	return 2;
 }

src/graphics/guest_gpu/graphicsRun.cpp

@@ -874,13 +874,11 @@ void CommandProcessor::SetPredication(uint32_t condition, uint32_t op, uint32_t
 }
 
 void CommandProcessor::DrawIndex(DrawIndexArgs args) {
-	CheckBuffer();
-
 	args.index_type_and_size = m_index_type_and_size;
 	if (args.instance_count == 0) {
 		args.instance_count = m_num_instances;
 	}
-	if (args.base_vertex != 0 || args.first_instance != 0) {
+	if (GraphicsRunDebugDumpEnabled() && (args.base_vertex != 0 || args.first_instance != 0)) {
 		LOGF("\t draw indexed offsets: base_vertex = %" PRId32 ", first_instance = %" PRIu32 "\n",
 		     args.base_vertex, args.first_instance);
 	}
@@ -912,17 +910,6 @@ void CommandProcessor::DrawIndirect(uint32_t data_offset, uint32_t draw_initiato
 	if (!indexed) {
 		DrawIndirectArgs args {};
 		std::memcpy(&args, args_addr, sizeof(args));
-		if (args.instance_count != 1u || args.start_vertex_location != 0u ||
-		    args.start_instance_location != 0u) {
-			static std::atomic<uint32_t> log_count {0};
-			if (log_count.fetch_add(1) < 64) {
-				LOGF("\t warning: partial DrawIndirect args: vertex_count=%" PRIu32
-				     ", instance_count=%" PRIu32 ", start_vertex=%" PRIu32
-				     ", start_instance=%" PRIu32 "\n",
-				     args.vertex_count_per_instance, args.instance_count,
-				     args.start_vertex_location, args.start_instance_location);
-			}
-		}
 		m_num_instances = args.instance_count;
 		DrawIndexAuto({.vertex_count   = args.vertex_count_per_instance,
 		               .instance_count = args.instance_count,
@@ -934,16 +921,6 @@ void CommandProcessor::DrawIndirect(uint32_t data_offset, uint32_t draw_initiato
 
 	DrawIndexedIndirectArgs args {};
 	std::memcpy(&args, args_addr, sizeof(args));
-	if (args.base_vertex_location != 0u || args.start_instance_location != 0u) {
-		static std::atomic<uint32_t> log_count {0};
-		if (log_count.fetch_add(1) < 64) {
-			LOGF("\t warning: partial DrawIndexIndirect args: index_count=%" PRIu32
-			     ", instance_count=%" PRIu32 ", start_index=%" PRIu32 ", base_vertex=%" PRIu32
-			     ", start_instance=%" PRIu32 "\n",
-			     args.index_count_per_instance, args.instance_count, args.start_index_location,
-			     args.base_vertex_location, args.start_instance_location);
-		}
-	}
 
 	uint64_t index_size = 0;
 	switch (m_index_type_and_size) {
@@ -1005,17 +982,6 @@ void CommandProcessor::DrawIndirectMulti(uint32_t data_offset, uint32_t max_coun
 
 		if (!indexed) {
 			auto* args = reinterpret_cast<const DrawIndirectArgs*>(args_addr);
-			if (args->instance_count != 1u || args->start_vertex_location != 0u ||
-			    args->start_instance_location != 0u) {
-				static std::atomic<uint32_t> log_count {0};
-				if (log_count.fetch_add(1) < 64) {
-					LOGF("\t warning: partial DrawIndirectMulti args[%u]: vertex_count=%" PRIu32
-					     ", instance_count=%" PRIu32 ", start_vertex=%" PRIu32
-					     ", start_instance=%" PRIu32 "\n",
-					     i, args->vertex_count_per_instance, args->instance_count,
-					     args->start_vertex_location, args->start_instance_location);
-				}
-			}
 			m_num_instances = args->instance_count;
 			DrawIndexAuto({.vertex_count   = args->vertex_count_per_instance,
 			               .instance_count = args->instance_count,
@@ -1026,17 +992,6 @@ void CommandProcessor::DrawIndirectMulti(uint32_t data_offset, uint32_t max_coun
 		}
 
 		auto* args = reinterpret_cast<const DrawIndexedIndirectArgs*>(args_addr);
-		if (args->base_vertex_location != 0u || args->start_instance_location != 0u) {
-			static std::atomic<uint32_t> log_count {0};
-			if (log_count.fetch_add(1) < 64) {
-				LOGF("\t warning: partial DrawIndexIndirectMulti args[%u]: index_count=%" PRIu32
-				     ", instance_count=%" PRIu32 ", start_index=%" PRIu32 ", base_vertex=%" PRIu32
-				     ", start_instance=%" PRIu32 "\n",
-				     i, args->index_count_per_instance, args->instance_count,
-				     args->start_index_location, args->base_vertex_location,
-				     args->start_instance_location);
-			}
-		}
 
 		uint64_t index_size = 0;
 		switch (m_index_type_and_size) {
@@ -1082,7 +1037,6 @@ void CommandProcessor::DispatchDirect(uint32_t thread_group_x, uint32_t thread_g
 	// uint32_t local_z   = 1;
 
 	{
-		CheckBuffer();
 		frame_num = m_renderer.GetGpu().GetFrameNum();
 		if (GraphicsRunDebugDumpEnabled()) {
 			static std::atomic<uint32_t> log_count {0};
@@ -1133,24 +1087,18 @@ void CommandProcessor::DispatchDirect(uint32_t thread_group_x, uint32_t thread_g
 	}*/
 }
 
-void CommandProcessor::DispatchIndirect(uint32_t data_offset, uint32_t mode) {
-	struct DispatchIndirectArgs {
-		uint32_t thread_group_x;
-		uint32_t thread_group_y;
-		uint32_t thread_group_z;
-	};
-
-	EXIT_NOT_IMPLEMENTED(m_dispatch_indirect_args_base_addr == 0);
-
-	const auto args_addr = m_dispatch_indirect_args_base_addr + data_offset;
-	auto*      args      = reinterpret_cast<const DispatchIndirectArgs*>(args_addr);
-
-	DispatchDirect(args->thread_group_x, args->thread_group_y, args->thread_group_z, mode);
+void CommandProcessor::DispatchIndirect(uint64_t args_addr, uint32_t mode) {
+	EXIT_NOT_IMPLEMENTED(args_addr == 0 || (args_addr & 3u) != 0);
+	if ((mode & Pm4::COMPUTE_DISPATCH_INITIATOR_USE_THREAD_DIMENSIONS) != 0) {
+		const auto* args = reinterpret_cast<const vk::DispatchIndirectCommand*>(args_addr);
+		DispatchDirect(args->x, args->y, args->z, mode);
+		return;
+	}
+	m_sh_ctx.SetCsWaveSize(Pm4::ComputeWaveSize(mode));
+	m_renderer.GetRenderExecutor().DispatchIndirect(m_submit_id, CurrentBuffer(), args_addr, mode);
 }
 
 void CommandProcessor::DrawIndexAuto(DrawAutoArgs args) {
-	CheckBuffer();
-
 	if (args.instance_count == 0) {
 		args.instance_count = m_num_instances;
 	}
@@ -1172,7 +1120,7 @@ void CommandProcessor::WriteAtEndOfPipe(uint32_t cache_policy, uint32_t event_wr
                                         uint32_t interrupt_context_id) {
 	static_assert(sizeof(T) == sizeof(uint32_t) || sizeof(T) == sizeof(uint64_t));
 
-	CheckBuffer();
+	auto& command = CurrentBuffer();
 
 	if (GraphicsRunDebugDumpEnabled()) {
 		const auto bits      = static_cast<unsigned>(sizeof(T) * 8u);
@@ -1203,7 +1151,7 @@ void CommandProcessor::WriteAtEndOfPipe(uint32_t cache_policy, uint32_t event_wr
 		case 0x03: with_interrupt = false; break;
 		case 0x01:
 			if (!IsAsyncComputeQueue()) {
-				Sync::TriggerEopEventAtEndOfPipe(CurrentBuffer(), m_interrupt_event_id,
+				Sync::TriggerEopEventAtEndOfPipe(command, m_interrupt_event_id,
 				                                 interrupt_context_id);
 				return;
 			}
@@ -1220,17 +1168,17 @@ void CommandProcessor::WriteAtEndOfPipe(uint32_t cache_policy, uint32_t event_wr
 
 		if (with_interrupt) {
 			if (with_writeback) {
-				Sync::WriteAtEndOfPipeWithInterruptWriteBack32(m_submit_id, CurrentBuffer(), dst,
+				Sync::WriteAtEndOfPipeWithInterruptWriteBack32(m_submit_id, command, dst,
 				                                               data, m_interrupt_event_id,
 				                                               interrupt_context_id);
 			} else {
-				Sync::WriteAtEndOfPipeWithInterrupt32(m_submit_id, CurrentBuffer(), dst, data,
+				Sync::WriteAtEndOfPipeWithInterrupt32(m_submit_id, command, dst, data,
 				                                      m_interrupt_event_id, interrupt_context_id);
 			}
 		} else if (with_writeback) {
-			Sync::WriteAtEndOfPipeWithWriteBack32(m_submit_id, CurrentBuffer(), dst, data);
+			Sync::WriteAtEndOfPipeWithWriteBack32(m_submit_id, command, dst, data);
 		} else {
-			Sync::WriteAtEndOfPipe32(m_submit_id, CurrentBuffer(), dst, data);
+			Sync::WriteAtEndOfPipe32(m_submit_id, command, dst, data);
 		}
 	};
 
@@ -1242,7 +1190,7 @@ void CommandProcessor::WriteAtEndOfPipe(uint32_t cache_policy, uint32_t event_wr
 					SynchronizeGpu();
 					Sync::ReadGds(*m_renderer.GetBufferCache().GetGdsBuffer(), dst, value & 0xffffu,
 					              value >> 16u);
-					Sync::WriteAtEndOfPipeGds32(m_submit_id, CurrentBuffer(), dst, value & 0xffffu,
+					Sync::WriteAtEndOfPipeGds32(m_submit_id, command, dst, value & 0xffffu,
 					                            value >> 16u);
 					if (with_interrupt) {
 						m_renderer.TriggerInterrupt(m_interrupt_event_id, interrupt_context_id);
@@ -1271,18 +1219,18 @@ void CommandProcessor::WriteAtEndOfPipe(uint32_t cache_policy, uint32_t event_wr
 					if (with_interrupt) {
 						if (with_writeback) {
 							Sync::WriteAtEndOfPipeWithInterruptWriteBack64(
-							    m_submit_id, CurrentBuffer(), dst, value, m_interrupt_event_id,
+							    m_submit_id, command, dst, value, m_interrupt_event_id,
 							    interrupt_context_id);
 						} else {
-							Sync::WriteAtEndOfPipeWithInterrupt64(m_submit_id, CurrentBuffer(), dst,
+							Sync::WriteAtEndOfPipeWithInterrupt64(m_submit_id, command, dst,
 							                                      value, m_interrupt_event_id,
 							                                      interrupt_context_id);
 						}
 					} else if (with_writeback) {
-						Sync::WriteAtEndOfPipeWithWriteBack64(m_submit_id, CurrentBuffer(), dst,
+						Sync::WriteAtEndOfPipeWithWriteBack64(m_submit_id, command, dst,
 						                                      value);
 					} else {
-						Sync::WriteAtEndOfPipe64(m_submit_id, CurrentBuffer(), dst, value);
+						Sync::WriteAtEndOfPipe64(m_submit_id, command, dst, value);
 					}
 				};
 
@@ -1363,10 +1311,10 @@ void CommandProcessor::WriteAtEndOfPipe(uint32_t cache_policy, uint32_t event_wr
 						    (eop_event_type == 0x28 && event_index == 0x00)) {
 							if (with_interrupt) {
 								Sync::WriteAtEndOfPipeWithInterrupt64(
-								    m_submit_id, CurrentBuffer(), dst, clock, m_interrupt_event_id,
+								    m_submit_id, command, dst, clock, m_interrupt_event_id,
 								    interrupt_context_id);
 							} else {
-								Sync::WriteAtEndOfPipeClockCounter(m_submit_id, CurrentBuffer(),
+								Sync::WriteAtEndOfPipeClockCounter(m_submit_id, command,
 								                                   dst, clock);
 							}
 							return;
@@ -1378,11 +1326,11 @@ void CommandProcessor::WriteAtEndOfPipe(uint32_t cache_policy, uint32_t event_wr
 						    (eop_event_type == 0x28 && event_index == 0x00)) {
 							if (with_interrupt) {
 								Sync::WriteAtEndOfPipeWithInterruptWriteBack64(
-								    m_submit_id, CurrentBuffer(), dst, clock, m_interrupt_event_id,
+								    m_submit_id, command, dst, clock, m_interrupt_event_id,
 								    interrupt_context_id);
 							} else {
 								Sync::WriteAtEndOfPipeClockCounterWithWriteBack(
-								    m_submit_id, CurrentBuffer(), dst, clock);
+								    m_submit_id, command, dst, clock);
 							}
 							return;
 						}
@@ -1420,8 +1368,6 @@ void CommandProcessor::WriteAtEndOfPipe64(uint32_t cache_policy, uint32_t event_
 }
 
 void CommandProcessor::EmitGlobalBarrier() {
-	CheckBuffer();
-
 	Common::LockGuard lock(m_renderer.GetMutex());
 
 	vk::MemoryBarrier2 barrier {};
@@ -1438,8 +1384,6 @@ void CommandProcessor::EmitGlobalBarrier() {
 }
 
 void CommandProcessor::TriggerEopEventAtEndOfPipe(uint32_t interrupt_context_id) {
-	CheckBuffer();
-
 	Sync::TriggerEopEventAtEndOfPipe(CurrentBuffer(), m_interrupt_event_id, interrupt_context_id);
 }
 
@@ -1523,8 +1467,6 @@ void CommandProcessor::TriggerEvent(uint32_t event_type, uint32_t event_index,
 }
 
 void CommandProcessor::Flip() {
-	CheckBuffer();
-
 	if (GraphicsRunDebugDumpEnabled()) {
 		LOGF("CommandProcessor::Flip()\n");
 	}
@@ -1538,7 +1480,7 @@ void CommandProcessor::Flip() {
 }
 
 void CommandProcessor::Flip(void* dst_gpu_addr, uint32_t value) {
-	CheckBuffer();
+	auto& command = CurrentBuffer();
 
 	if (GraphicsRunDebugDumpEnabled()) {
 		LOGF("CommandProcessor::Flip()\n"
@@ -1548,7 +1490,6 @@ void CommandProcessor::Flip(void* dst_gpu_addr, uint32_t value) {
 	}
 
 	std::memcpy(dst_gpu_addr, &value, sizeof(value));
-	auto& command = CurrentBuffer();
 	auto request = Sync::PrepareVideoOutFlip(command, m_flip.handle, m_flip.index, m_flip.flip_mode,
 	                                         m_flip.flip_arg);
 	Sync::WriteAtEndOfPipeWithFlip32(m_submit_id, command, static_cast<uint32_t*>(dst_gpu_addr),
@@ -1559,7 +1500,7 @@ void CommandProcessor::Flip(void* dst_gpu_addr, uint32_t value) {
 
 void CommandProcessor::FlipWithInterrupt(uint32_t eop_event_type, uint32_t cache_action,
                                          void* dst_gpu_addr, uint32_t value) {
-	CheckBuffer();
+	auto& command = CurrentBuffer();
 
 	if (GraphicsRunDebugDumpEnabled()) {
 		LOGF("CommandProcessor::FlipWithInterrupt()\n"
@@ -1574,7 +1515,6 @@ void CommandProcessor::FlipWithInterrupt(uint32_t eop_event_type, uint32_t cache
 		EXIT("unknown event type\n");
 	}
 	std::memcpy(dst_gpu_addr, &value, sizeof(value));
-	auto& command = CurrentBuffer();
 	auto request = Sync::PrepareVideoOutFlip(command, m_flip.handle, m_flip.index, m_flip.flip_mode,
 	                                         m_flip.flip_arg);
 	Sync::WriteAtEndOfPipeWithInterruptWriteBackFlip32(
@@ -1584,7 +1524,7 @@ void CommandProcessor::FlipWithInterrupt(uint32_t eop_event_type, uint32_t cache
 }
 
 void CommandProcessor::PrepareCpuFlip(uint64_t request_id) {
-	CheckBuffer();
+	auto& command = CurrentBuffer();
 	if (g_current_processor != nullptr) {
 		EXIT("invalid graphics-thread CPU flip preparation\n");
 	}
@@ -1594,7 +1534,7 @@ void CommandProcessor::PrepareCpuFlip(uint64_t request_id) {
 	};
 	ProcessorScope processor_scope(*this);
 
-	m_renderer.GetVideoOut().PrepareFlip(request_id, CurrentBuffer());
+	m_renderer.GetVideoOut().PrepareFlip(request_id, command);
 	GetScheduler().Flush();
 	m_renderer.GetVideoOut().CompleteFlip(request_id);
 }

src/graphics/guest_gpu/pm4.h

@@ -910,6 +910,7 @@ constexpr uint32_t COMPUTE_PGM_RSRC1_WGP_MODE_MASK    = 0x1;
 constexpr uint32_t COMPUTE_PGM_RSRC1_FWD_PROGRESS_SHIFT = 31;
 constexpr uint32_t COMPUTE_PGM_RSRC1_FWD_PROGRESS_MASK  = 0x1;
 
+constexpr uint32_t COMPUTE_DISPATCH_INITIATOR_USE_THREAD_DIMENSIONS = 1u << 5u;
 constexpr uint32_t COMPUTE_DISPATCH_INITIATOR_CS_W32_EN_SHIFT = 15;
 constexpr uint32_t COMPUTE_DISPATCH_INITIATOR_CS_W32_EN_MASK  = 0x1;
 

src/graphics/host_gpu/renderer/cache/bufferCache.cpp

@@ -129,7 +129,7 @@ bool BufferCache::DownloadBufferMemory(Buffer& buffer, uint64_t vaddr, uint64_t
 
 	const auto [mapped, offset] = m_download_buffer.Map(total_size, 64);
 	if (mapped == nullptr) {
-		EXIT("BufferCache: download exceeds 32 MiB staging buffer capacity\n");
+		EXIT("BufferCache: download exceeds 64 MiB staging buffer capacity\n");
 	}
 	m_download_buffer.Commit();
 	for (auto& copy: copies) {
@@ -183,7 +183,7 @@ BufferCache::BufferCache(GraphicContext& graphics, CommandScheduler& scheduler,
       m_memory_tracker(page_manager),
       m_staging_buffer(graphics, scheduler, MemoryUsage::Upload, 512 * MiB),
       m_stream_buffer(graphics, scheduler, MemoryUsage::Stream, 64 * MiB),
-      m_download_buffer(graphics, scheduler, MemoryUsage::Download, 32 * MiB),
+      m_download_buffer(graphics, scheduler, MemoryUsage::Download, 64 * MiB),
       m_device_buffer(graphics, scheduler, MemoryUsage::DeviceLocal, 128 * MiB),
       m_texture_cache(texture_cache) {
 	std::memset(m_gds_buffer.Mapped().data(), 0, static_cast<size_t>(m_gds_buffer.Size()));

src/graphics/host_gpu/renderer/cache/textureCache.cpp

@@ -138,14 +138,15 @@ void NameImageBinding(GraphicContext& graphics, Image& image, vk::ImageView view
 }
 
 [[nodiscard]] std::vector<vk::BufferImageCopy> BuildDepthCopies(const ImageInfo& info,
-                                                                uint64_t         slice_stride) {
+                                                              uint64_t slice_stride,
+                                                              vk::ImageAspectFlags aspect) {
 	std::vector<vk::BufferImageCopy> copies(info.resources.layers);
 	for (uint32_t layer = 0; layer < info.resources.layers; ++layer) {
 		auto& copy             = copies[layer];
 		copy.bufferOffset      = slice_stride * layer;
 		copy.bufferRowLength   = info.pitch;
 		copy.bufferImageHeight = info.extent.height;
-		copy.imageSubresource  = {vk::ImageAspectFlagBits::eDepth, 0, layer, 1};
+		copy.imageSubresource  = {aspect, 0, layer, 1};
 		copy.imageExtent       = {info.extent.width, info.extent.height, 1};
 	}
 	return copies;
@@ -1031,16 +1032,17 @@ TextureCache::ImageDownload TextureCache::BuildDownload(const Image& image) cons
 }
 
 void TextureCache::UploadImage(Image& image, Buffer& source, uint64_t source_offset) {
-	const auto& info    = image.info;
-	const auto  binding = UploadBinding(image);
+	auto& destination = image.depth_id ? m_slot_images[image.depth_id] : image;
+	const auto binding = image.depth_id ? BindingType::DepthTarget : UploadBinding(image);
 	const auto  upload  = [&](std::vector<vk::BufferImageCopy>& copies, TileManager::Result linear) {
 		for (auto& copy: copies) {
 			copy.bufferOffset += linear.offset;
 		}
-		image.Upload(copies, linear.buffer, linear.offset, linear.size);
+		destination.Upload(copies, linear.buffer, linear.offset, linear.size);
 	};
 
 	if (binding != BindingType::DepthTarget) {
+		const auto& info = image.info;
 		auto transfer = BuildTextureTransfer(image, binding, TransferDirection::Upload);
 		if (!transfer.valid) {
 			EXIT("TextureCache: invalid texture upload: binding=%u addr=0x%016" PRIx64
@@ -1064,21 +1066,32 @@ void TextureCache::UploadImage(Image& image, Buffer& source, uint64_t source_off
 		return;
 	}
 
-	if (info.samples != 1 || image.backing.samples != 1 ||
+	// The stencil plane has its own row pitch and shares the native image
+	// with the depth plane.
+	auto info = destination.info;
+	if (image.depth_id) {
+		info.data            = image.info.data;
+		info.guest_format    = Prospero::BufferFormat::k8UInt;
+		info.bytes_per_block = 1;
+		if (info.IsTiled()) info.pitch = TileGetDepthPitch(info.extent.width, 1, 0);
+	}
+	if (info.samples != 1 || destination.backing.samples != 1 ||
 	    info.resources.layers == 0 || info.data.size % info.resources.layers != 0 ||
 	    Prospero::NumBytesPerElement(info.guest_format) != info.bytes_per_block) {
 		EXIT("TextureCache: invalid depth upload\n");
 	}
 	const auto          layers          = info.resources.layers;
 	const auto          full_slice_size = info.data.size / layers;
-	auto                copies          = BuildDepthCopies(info, full_slice_size);
+	auto copies = BuildDepthCopies(info, full_slice_size, image.depth_id
+	                                                        ? vk::ImageAspectFlagBits::eStencil
+	                                                        : vk::ImageAspectFlagBits::eDepth);
 	TileManager::Result linear {source.Handle(), source_offset, source.Size() - source_offset};
 	if (info.IsTiled()) {
 		const auto tiles = BuildDepthTiles(info);
 		linear =
 		    m_tiler.Detile(source.Handle(), source_offset, info.data.size, info.data.size, tiles);
 	}
-	const auto transfer_bytes = DepthAspectTransferBytes(info.pixel_format);
+	const auto transfer_bytes = image.depth_id ? 1u : DepthAspectTransferBytes(info.pixel_format);
 	if (transfer_bytes != info.bytes_per_block) {
 		const uint64_t texels_per_slice = static_cast<uint64_t>(info.pitch) * info.extent.height;
 		EXIT_NOT_IMPLEMENTED(info.bytes_per_block != sizeof(uint16_t) ||
@@ -1202,8 +1215,13 @@ void TextureCache::MaterializeDccClear(ImageId id, const ImageDesc& desc,
 }
 
 void TextureCache::RefreshImage(ImageId id) {
-	TrackImage(id);
 	auto& image = m_slot_images[id];
+	if (image.depth_id &&
+	    (m_slot_images[image.depth_id].info.metadata.stencil_compressed ||
+	     m_slot_images[image.depth_id].info.samples != 1)) {
+		return;
+	}
+	TrackImage(id);
 	if (image.IsMaybeCpuDirty()) {
 		const auto hash = image.HashGuestEdges();
 		if (image.NeedsMaybeCpuHash()) {
@@ -1225,7 +1243,7 @@ void TextureCache::RefreshImage(ImageId id) {
 	InitializeImage(id);
 }
 
-void TextureCache::AssociateStencil(ImageId depth_id, GuestRange stencil) {
+ImageId TextureCache::AssociateStencil(ImageId depth_id, GuestRange stencil) {
 	if (!stencil.Valid()) {
 		EXIT("TextureCache: invalid stencil association range\n");
 	}
@@ -1251,6 +1269,7 @@ void TextureCache::AssociateStencil(ImageId depth_id, GuestRange stencil) {
 	auto& record = m_slot_images[association];
 	TouchImage(record);
 	record.depth_id = depth_id;
+	return association;
 }
 
 ImageId TextureCache::FindImage(ImageDesc& desc, bool exact_format) {
@@ -1396,6 +1415,19 @@ vk::ImageView TextureCache::FindTexture(ImageId id, const ImageDesc& desc) {
 	}
 	if (!image.info.data.Empty()) {
 		RefreshImage(id);
+		if (image.info.HasStencil() &&
+		    desc.info.data.address >= image.info.stencil.address &&
+		    desc.info.data.End() <= image.info.stencil.End()) {
+			for (const auto stencil_id:
+			     FindImagesInRegion(image.info.stencil.address, image.info.stencil.size, false)) {
+				const auto* stencil = m_slot_images.try_get(stencil_id);
+				if (stencil != nullptr && stencil->depth_id == id &&
+				    stencil->info.data == image.info.stencil) {
+					RefreshImage(stencil_id);
+					break;
+				}
+			}
+		}
 	}
 	switch (desc.type) {
 		case BindingType::Texture: break;
@@ -1444,16 +1476,17 @@ vk::ImageView TextureCache::FindDepthTarget(ImageId id, const ImageDesc& desc) {
 	TouchImage(image);
 	image.MarkGpuModified();
 	image.usage.depth_target = true;
-	RefreshImage(id);
+	image.info.stencil = desc.info.stencil;
+	image.info.metadata = desc.info.metadata;
 	if (desc.info.HasMetadata()) {
-		image.info.metadata = desc.info.metadata;
 		m_surface_metas.emplace(desc.info.metadata.range.address,
 		                        MetaDataInfo {.type       = MetaDataInfo::Type::HTile,
 		                                      .clear_mask = image.info.htile_clear_mask});
 	}
+	RefreshImage(id);
 	CommitGpuWrite(image);
 	if (desc.info.HasStencil()) {
-		AssociateStencil(id, desc.info.stencil);
+		RefreshImage(AssociateStencil(id, desc.info.stencil));
 	}
 	const auto view = image.FindView(desc.view_info);
 	NameImageBinding(m_graphics, image, view, desc.type, desc.view_info);
@@ -1468,11 +1501,16 @@ void TextureCache::MarkGpuWritten(ImageId id) {
 	}
 	TrackImage(id);
 	CommitGpuWrite(image);
+	if (image.info.HasStencil()) {
+		const auto stencil_id = AssociateStencil(id, image.info.stencil);
+		TrackImage(stencil_id);
+		CommitGpuWrite(m_slot_images[stencil_id]);
+	}
 }
 
 void TextureCache::CommitGpuWrite(Image& image) {
-	if (image.depth_id || image.backing.image == nullptr) {
-		EXIT("TextureCache: stencil association cannot own image contents\n");
+	if (!image.depth_id && image.backing.image == nullptr) {
+		EXIT("TextureCache: GPU writes require a native image or stencil association\n");
 	}
 	image.ClearBufferModified();
 	if (image.IsCpuDirty()) {
@@ -1556,9 +1594,10 @@ void TextureCache::ClearImage(CommandBuffer& command, ImageId id, vk::Format for
 	        range.layerCount == 0 || range.baseArrayLayer >= layers ||
 	        range.layerCount > layers - range.baseArrayLayer ||
 	        (range.aspectMask & aspects) != range.aspectMask);
-	const bool full_image = range.aspectMask == aspects && range.baseMipLevel == 0 &&
-	                        range.levelCount == image.info.resources.levels &&
-	                        range.baseArrayLayer == 0 && range.layerCount == layers;
+	const bool full_subresources = range.baseMipLevel == 0 &&
+	                               range.levelCount == image.info.resources.levels &&
+	                               range.baseArrayLayer == 0 && range.layerCount == layers;
+	const bool full_image = range.aspectMask == aspects && full_subresources;
 	TrackImage(id);
 	if (!full_image && (image.IsBufferModified() || image.IsCpuDirty())) {
 		InitializeImage(id);
@@ -1566,6 +1605,15 @@ void TextureCache::ClearImage(CommandBuffer& command, ImageId id, vk::Format for
 			EXIT("TextureCache: image clear retained guest ownership\n");
 		}
 	}
+	if (image.info.HasStencil() && (range.aspectMask & vk::ImageAspectFlagBits::eStencil)) {
+		const auto stencil_id = AssociateStencil(id, image.info.stencil);
+		if (!full_subresources) {
+			RefreshImage(stencil_id);
+		} else {
+			TrackImage(stencil_id);
+			CommitGpuWrite(m_slot_images[stencil_id]);
+		}
+	}
 	command.EndRendering();
 	// Transfer clears use the backing format; aliased clears must encode through their view.
 	if (format != image.backing.format || (image.info.IsVolume() && !full_image)) {
@@ -1638,7 +1686,7 @@ void TextureCache::DownloadDepth(Image& image, Buffer& destination, uint64_t des
 	EXIT_NOT_IMPLEMENTED(transfer_slice > UINT64_MAX / layers);
 	const uint64_t transfer_size = transfer_slice * layers;
 	EXIT_NOT_IMPLEMENTED(guest_slice > full_slice_size);
-	auto copies = BuildDepthCopies(info, full_slice_size);
+	auto copies = BuildDepthCopies(info, full_slice_size, vk::ImageAspectFlagBits::eDepth);
 	if (transfer_bytes == info.bytes_per_block) {
 		if (!info.IsTiled()) {
 			for (auto& copy: copies) {
@@ -1831,7 +1879,7 @@ void TextureCache::InvalidateMemoryFromGPU(uint64_t address, uint64_t size) {
 	std::scoped_lock lock {m_lock};
 	for (const auto id: FindImagesInRegion(address, size, true)) {
 		auto& image = m_slot_images[id];
-		if (image.depth_id || !image.Overlaps(address, size)) {
+		if (!image.Overlaps(address, size)) {
 			continue;
 		}
 		if (image.IsGpuModified()) {

src/graphics/host_gpu/renderer/cache/textureCache.h

@@ -157,7 +157,7 @@ private:
 	void RefreshCopySource(ImageId id);
 	[[nodiscard]] bool CopyD16(Image& destination, Image& source);
 	void               CopyImage(ImageId destination, ImageId source);
-	void               AssociateStencil(ImageId depth, GuestRange stencil);
+	[[nodiscard]] ImageId AssociateStencil(ImageId depth, GuestRange stencil);
 	void CopyImageMip(ImageId destination, ImageId source, uint32_t mip, uint32_t layer);
 	void ValidateImageDesc(const ImageDesc& desc) const;
 

src/graphics/host_gpu/renderer/debug.cpp

@@ -209,17 +209,16 @@ static void RtCheck(const HW::RenderTarget& rt) {
 		}
 		if (rt.info.fmask_compression_enable) {
 			EXIT_NOT_IMPLEMENTED(rt.attrib.num_samples == 0 && rt.attrib.num_fragments == 0);
+			// Native MSAA stores expanded samples, independent of FMASK metadata compression.
 			static bool logged = false;
 			if (!logged) {
-				LOGF("RenderTarget: using native Vulkan MSAA without guest FMASK metadata, "
+				LOGF("RenderTarget: using expanded native Vulkan MSAA samples, "
 				     "fmask=0x%016" PRIx64 "\n",
 				     rt.fmask.addr);
 				logged = true;
 			}
 		}
 
-		EXIT_NOT_IMPLEMENTED(rt.info.fmask_data_compression_disable != false);
-
 		if (rt.info.cmask_fast_clear_enable || rt.info.dcc_compression_enable) {
 			static bool logged = false;
 			if (!logged) {

src/graphics/host_gpu/renderer/pipeline/descriptors.cpp

@@ -184,17 +184,18 @@ bool IsSupportedDepthTextureEncoding(const ShaderTextureResource& descriptor, bo
 	}
 	const bool full = common && descriptor.fields[4] == field4_expected &&
 	                  descriptor.fields[5] == field5_expected;
-	if (!full || (descriptor.fields[6] == 0 && descriptor.fields[7] != 0) ||
+	if (!full ||
 	    (descriptor.MsaaDepth() && !IsMultisampledTexture(descriptor.Type()))) {
 		return false;
 	}
-	if (descriptor.fields[6] == 0) {
+	const auto metadata_control = descriptor.fields[6] & 0x00ffffffu;
+	if (metadata_control == 0) {
 		return true;
 	}
 	constexpr uint32_t htile_control = 0x00280000u;
 	const uint32_t expected_control  = htile_control | (descriptor.MsaaDepth() ? (1u << 10u) : 0u);
 	const auto     metadata_addr     = descriptor.MetaAddr() << 8u;
-	return (descriptor.fields[6] & 0x00ffffffu) == expected_control && metadata_addr != 0 &&
+	return metadata_control == expected_control && metadata_addr != 0 &&
 	       metadata_addr < TRACKER_ADDRESS_SIZE && (metadata_addr & 0x7fffu) == 0 &&
 	       descriptor.TileMode() == Prospero::TileMode::kDepth;
 }
@@ -550,8 +551,14 @@ TextureBinding RenderExecutor::ResolveTexture(const ShaderRecompiler::IR::ImageR
 	const bool multisampled    = IsMultisampledTexture(type);
 	const auto max_mip         = resource.r128 ? last_level : descriptor.MaxMip();
 	const auto physical_levels = multisampled ? 1u : static_cast<uint32_t>(max_mip) + 1u;
+	// IMAGE_STORE addresses BASE_LEVEL; only IMAGE_STORE_MIP selects other view mips.
+	const bool single_storage_mip =
+	    storage && resource.mip_mode != ShaderRecompiler::IR::ImageMipMode::DynamicStorage;
+	const auto view_levels = multisampled || single_storage_mip
+	                             ? 1u
+	                             : static_cast<uint32_t>(last_level - base_level) + 1u;
 	const auto levels =
-	    multisampled ? 1u : std::max(physical_levels, static_cast<uint32_t>(last_level) + 1u);
+	    multisampled ? 1u : std::max(physical_levels, base_level + view_levels);
 	const auto tile       = descriptor.TileMode();
 	const bool depth_tile = tile == Prospero::TileMode::kDepth;
 	const bool msaa_tile  = depth_tile || tile == Prospero::TileMode::kRenderTarget;
@@ -574,8 +581,6 @@ TextureBinding RenderExecutor::ResolveTexture(const ShaderRecompiler::IR::ImageR
 		     descriptor.fields[6], descriptor.fields[7]);
 	}
 	const auto samples = multisampled ? 1u << last_level : 1u;
-	const auto view_levels =
-	    multisampled ? 1u : static_cast<uint32_t>(last_level - base_level) + 1u;
 	const auto depth          = static_cast<uint32_t>(descriptor.Depth()) + 1u;
 	const auto format         = descriptor.Format();
 	const auto surface_format = TextureGetSurfaceFormatInfo(format);
@@ -748,7 +753,7 @@ PreparedBindings RenderExecutor::PrepareBindings(const ShaderStageRuntime& runti
 	KYTY_PROFILER_FUNCTION();
 	EXIT_IF(!runtime);
 	const auto& program  = *runtime.program;
-	const auto& snapshot = runtime.resources;
+	const auto& snapshot = *runtime.resources;
 	PreparedBindings prepared;
 	prepared.runtime = &runtime;
 	prepared.images.reserve(program.info.images.size());
@@ -777,7 +782,7 @@ void RenderExecutor::FindBuffers(PreparedBindings& prepared) {
 	KYTY_PROFILER_FUNCTION();
 	EXIT_IF(prepared.runtime == nullptr || !*prepared.runtime);
 	const auto& program  = *prepared.runtime->program;
-	const auto& snapshot = prepared.runtime->resources;
+	const auto& snapshot = *prepared.runtime->resources;
 	auto&       cache    = m_context.GetBufferCache();
 
 	prepared.buffer_sources.clear();
@@ -799,7 +804,7 @@ void RenderExecutor::RebindBuffers(PreparedBindings& prepared) {
 	KYTY_PROFILER_FUNCTION();
 	EXIT_IF(prepared.runtime == nullptr || !*prepared.runtime);
 	const auto& program   = *prepared.runtime->program;
-	const auto& snapshot  = prepared.runtime->resources;
+	const auto& snapshot  = *prepared.runtime->resources;
 	const auto& layout    = program.bindings;
 	EXIT_IF(prepared.buffer_sources.size() != program.info.buffers.size());
 
@@ -834,7 +839,7 @@ void RenderExecutor::RebindImages(PreparedBindings& prepared) {
 	KYTY_PROFILER_FUNCTION();
 	EXIT_IF(prepared.runtime == nullptr || !*prepared.runtime);
 	const auto& program  = *prepared.runtime->program;
-	const auto& snapshot = prepared.runtime->resources;
+	const auto& snapshot = *prepared.runtime->resources;
 	auto&       images   = prepared.images;
 	EXIT_IF(images.size() != program.info.images.size());
 	auto& texture_cache = m_context.GetTextureCache();
@@ -866,11 +871,7 @@ void RenderExecutor::RebindImages(PreparedBindings& prepared) {
 			}
 			binding.image_view = binding.mip_views.front();
 		} else {
-			auto desc = binding.desc;
-			if (desc.type == TextureCache::BindingType::Storage) {
-				desc.view_info.level_count = 1;
-			}
-			binding.image_view = texture_cache.FindTexture(binding.image_id, desc);
+			binding.image_view = texture_cache.FindTexture(binding.image_id, binding.desc);
 		}
 		auto&      image   = texture_cache.GetImage(binding.image_id);
 		const bool storage = binding.desc.type == TextureCache::BindingType::Storage;

src/graphics/host_gpu/renderer/pipeline/pipelineCache.cpp

@@ -92,9 +92,9 @@ void PipelineCacheLog(fmt::format_string<Args...> format, Args&&... args) {
 	Log::WriteToConsoleAndLog(message);
 }
 
-bool ReadShaderGuestMemory(void*, uint64_t address, uint32_t* value) {
-	return value != nullptr &&
-	       Libs::LibKernel::Memory::TryReadGpuCleanBacking(address, value, sizeof(*value));
+bool ReadShaderGuestMemory(void*, uint64_t address, std::span<uint32_t> values) {
+	return !values.empty() &&
+	       Libs::LibKernel::Memory::TryReadGpuCleanBacking(address, values.data(), values.size_bytes());
 }
 
 void DumpShaderSpirv(const char* stage_name, uint64_t shader_hash,
@@ -200,8 +200,10 @@ struct PipelineCache::ProgramCache {
 			permutations.reserve(8);
 		}
 
-		ShaderRecompiler::IR::ResourcePlan resource_plan;
-		std::vector<Permutation>           permutations;
+		ShaderRecompiler::IR::ResourcePlan           resource_plan;
+		ShaderRecompiler::IR::ResourceSnapshot       resources;
+		ShaderRecompiler::IR::ResourceSpecialization specialization;
+		std::vector<Permutation>                    permutations;
 	};
 
 	struct ProgramKeyHash {
@@ -274,33 +276,33 @@ struct PipelineCache::ProgramCache {
 			stage = ShaderType::Compute;
 		}
 
+		const auto user_data = std::span(params.user_data).first(params.user_data_count);
 		lookup_key.stage           = stage;
 		lookup_key.hash            = params.hash;
-		lookup_key.user_data_count = static_cast<uint32_t>(params.user_data.size());
+		lookup_key.user_data_count = params.user_data_count;
 		lookup_key.code_size       = static_cast<uint32_t>(params.code.size());
 		BuildStageStaticKey(input_info, lookup_key.static_state);
 		auto                                         entry = programs.find(lookup_key);
-		ShaderRecompiler::IR::ResourceSnapshot       resources;
-		ShaderRecompiler::IR::ResourceSpecialization specialization;
 		const ShaderRecompiler::IR::SrtRuntime       runtime {
-		    .user_data                  = params.user_data,
+		    .user_data                  = user_data,
 		    .shader_base                = params.Base(),
 		    .read_specialization_memory = ReadShaderGuestMemory,
 		};
 		if (entry != programs.end()) {
 			EXIT_IF(!ShaderRecompiler::IR::MaterializeResources(
-			    entry->second.resource_plan, runtime, resources, specialization));
+			    entry->second.resource_plan, runtime, entry->second.resources,
+			    entry->second.specialization));
 			if (const auto permutation = std::ranges::find_if(
 			        entry->second.permutations, [&](const Permutation& candidate) {
 				        const auto& layout = candidate.program.bindings;
 				        return layout.push_data_start_dword ==
 				                   ShaderRecompiler::IR::PushData::StartFor(
 				                       push_data_cursor, layout.ShaderDataDwords()) &&
-				               candidate.specialization == specialization;
+				               candidate.specialization == entry->second.specialization;
 			        });
 			    permutation != entry->second.permutations.end()) {
 				input_info.stage = {.program   = &permutation->program,
-				                    .resources = std::move(resources)};
+				                    .resources = &entry->second.resources};
 				permutation->program.bindings.AdvancePushData(push_data_cursor);
 				return permutation->handle;
 			}
@@ -328,7 +330,7 @@ struct PipelineCache::ProgramCache {
 		ShaderRecompiler::CompileOptions options;
 		options.stage       = stage;
 		options.shader_hash = params.hash;
-		options.user_data   = params.user_data;
+		options.user_data   = user_data;
 		options.back_code      = params.back_code;
 		options.dump_ir     = Config::GetShaderLogDirection() != Config::LogDirection::Silent;
 		options.early_dump  = options.dump_ir;
@@ -337,6 +339,7 @@ struct PipelineCache::ProgramCache {
 
 		if constexpr (std::is_same_v<InputInfo, ShaderVertexInputInfo>) {
 			options.user_data_base = 8;
+			options.wave_size = input_info.wave_size;
 			if (stage == ShaderType::Mesh || stage == ShaderType::TessellationControl) {
 				options.user_data_base = 0;
 				options.wave_size = stage == ShaderType::Mesh ? input_info.mesh.wave_size : 64u;
@@ -346,15 +349,16 @@ struct PipelineCache::ProgramCache {
 		}
 		auto translated = ShaderRecompiler::TranslateProgram(params.code, options);
 		if (entry == programs.end()) {
-			auto resource_plan = ShaderRecompiler::IR::ExtractResourcePlan(translated.program);
-			EXIT_IF(!ShaderRecompiler::IR::MaterializeResources(resource_plan, runtime, resources,
-			                                                    specialization));
-			entry = programs.try_emplace(lookup_key, std::move(resource_plan)).first;
+			entry = programs.try_emplace(lookup_key,
+			    ShaderRecompiler::IR::ExtractResourcePlan(translated.program)).first;
+			EXIT_IF(!ShaderRecompiler::IR::MaterializeResources(
+			    entry->second.resource_plan, runtime, entry->second.resources,
+			    entry->second.specialization));
 		}
 		entry->second.permutations.push_back(CompilePermutation(
-		    params, options, std::move(translated), std::move(specialization), push_data_cursor));
+		    params, options, std::move(translated), entry->second.specialization, push_data_cursor));
 		const auto& permutation = entry->second.permutations.back();
-		input_info.stage = {.program = &permutation.program, .resources = std::move(resources)};
+		input_info.stage = {.program = &permutation.program, .resources = &entry->second.resources};
 		permutation.program.bindings.AdvancePushData(push_data_cursor);
 
 		std::array<size_t, static_cast<size_t>(ShaderType::TessellationEvaluation) + 1> counts {};

src/graphics/host_gpu/renderer/pipeline/shaderResourceBarrier.cpp

@@ -90,7 +90,7 @@ vk::BufferMemoryBarrier MakeGdsDependency(vk::Buffer buffer) {
 bool HasShaderBufferWrites(const ShaderStageRuntime& runtime) {
 	EXIT_IF(!runtime);
 	const auto& program   = *runtime.program;
-	const auto& resources = runtime.resources;
+	const auto& resources = *runtime.resources;
 	EXIT_IF(resources.buffers.size() != program.info.buffers.size());
 	bool has_writes = false;
 	for (uint32_t i = 0; i < program.info.buffers.size(); i++) {

src/graphics/host_gpu/renderer/render.h

@@ -45,6 +45,7 @@ enum class CommandBufferDebugOp : uint32_t {
 	EopFlip,
 	EopWriteBackFlip,
 	EopOnlyFlip,
+	DispatchIndirect,
 	Unknown,
 };
 
@@ -156,6 +157,8 @@ public:
 
 	void DispatchDirect(uint64_t submit_id, CommandBuffer& buffer, uint32_t thread_group_x,
 	                    uint32_t thread_group_y, uint32_t thread_group_z, uint32_t mode);
+	void DispatchIndirect(uint64_t submit_id, CommandBuffer& buffer, uint64_t args_addr,
+	                      uint32_t mode);
 
 	[[nodiscard]] PreparedBindings PrepareBindings(const ShaderStageRuntime& runtime);
 	void                           FindBuffers(PreparedBindings& bindings);

src/graphics/host_gpu/renderer/renderCompute.cpp

@@ -9,6 +9,7 @@
 #include "graphics/guest_gpu/gpu_defs.h"
 #include "graphics/guest_gpu/graphicsRun.h"
 #include "graphics/guest_gpu/hardwareContext.h"
+#include "graphics/guest_gpu/pm4.h"
 #include "graphics/host_gpu/graphicContext.h"
 #include "graphics/host_gpu/renderer/image/imageInfo.h"
 #include "graphics/host_gpu/renderer/pipeline/descriptors.h"
@@ -51,7 +52,7 @@ static bool FillSourcesDisjoint(std::span<const ShaderRecompiler::IR::Descriptor
 bool RenderExecutor::TryConsumeComputeMetaClear(const ShaderComputeInputInfo& input,
                                                 const CommandBuffer&          buffer) {
 	const auto& program   = *input.stage.program;
-	const auto& resources = input.stage.resources;
+	const auto& resources = *input.stage.resources;
 	if (resources.buffers.size() != program.info.buffers.size()) {
 		EXIT("compute runtime buffer count does not match shader metadata\n");
 	}
@@ -85,7 +86,7 @@ bool ResolveComputeBufferFill(const ShaderComputeInputInfo& input, uint32_t grou
                               uint32_t group_y, uint32_t group_z, uint32_t mode,
                               ShaderBufferResource& resolved_descriptor, uint32_t& resolved_clear,
                               uint64_t& resolved_size) {
-	const auto& resources = input.stage.resources;
+	const auto& resources = *input.stage.resources;
 	const auto& fill      = resources.uniform_fill;
 	if (fill.kind != ShaderRecompiler::IR::UniformFillKind::Buffer) {
 		return false;
@@ -128,7 +129,7 @@ bool RenderExecutor::TryConsumeComputeImageClear(const ShaderComputeInputInfo& i
                                                 CommandBuffer& command, uint32_t group_x,
                                                 uint32_t group_y, uint32_t group_z, uint32_t mode) {
 	const auto& program   = *input.stage.program;
-	const auto& resources = input.stage.resources;
+	const auto& resources = *input.stage.resources;
 	const auto& fill      = resources.uniform_fill;
 	auto&       cache     = command.GetContext().GetTextureCache();
 	if (fill.kind == ShaderRecompiler::IR::UniformFillKind::Image) {
@@ -262,7 +263,7 @@ void RenderExecutor::DispatchDirect(uint64_t submit_id, CommandBuffer& buffer,
 	}
 
 	const auto& program   = *input_info.stage.program;
-	const auto& resources = input_info.stage.resources;
+	const auto& resources = *input_info.stage.resources;
 	if (TryConsumeComputeMetaClear(input_info, buffer)) {
 		ResetBindings();
 		return;
@@ -395,4 +396,60 @@ void RenderExecutor::DispatchDirect(uint64_t submit_id, CommandBuffer& buffer,
 	ResetBindings();
 }
 
+void RenderExecutor::DispatchIndirect(uint64_t submit_id, CommandBuffer& buffer,
+                                      uint64_t args_addr, uint32_t mode) {
+	EXIT_IF(buffer.IsInvalid() || args_addr == 0 || (args_addr & 3u) != 0 ||
+	        (mode & Pm4::COMPUTE_DISPATCH_INITIATOR_USE_THREAD_DIMENSIONS) != 0);
+	m_context.GetCommandScheduler().PopPendingOperations();
+	buffer.SetDebugInfo(static_cast<uint32_t>(CommandBufferDebugOp::DispatchIndirect), submit_id,
+	                    static_cast<uint32_t>(args_addr), static_cast<uint32_t>(args_addr >> 32u),
+	                    0, mode, buffer.GetShaders().GetCs().cs_regs.data_addr);
+	Common::LockGuard lock(m_context.GetMutex());
+	const auto& cs_regs = buffer.GetShaders().GetCs();
+	if (cs_regs.cs_regs.data_addr == 0) {
+		return;
+	}
+	ShaderComputeInputInfo input_info {};
+	const auto compute_program = m_context.GetPipelineCache().GetComputeProgram(
+	    cs_regs, buffer.GetRegisters().GetShaderRegisters(), input_info);
+	buffer.EndRendering();
+	auto& pipeline = m_context.GetPipelineCache().GetComputePipeline(input_info, compute_program);
+	auto bindings = PrepareBindings(input_info.stage);
+	FindBuffers(bindings);
+	const auto& program = *input_info.stage.program;
+	if (program.info.uses_dma) {
+		m_context.PrepareBda();
+	}
+	RebindImages(bindings);
+	// Acquiring arguments can merge cache buffers; finalize shader bindings afterward.
+	const auto [args_buffer, args_offset] = m_context.GetBufferCache().ObtainBuffer(
+	    args_addr, sizeof(vk::DispatchIndirectCommand), false);
+	EXIT_IF(args_buffer == nullptr || (args_offset & 3u) != 0);
+	RebindBuffers(bindings);
+	PreparedBindings* descriptor_stage = &bindings;
+	CommitBindings(buffer, vk::PipelineBindPoint::eCompute, pipeline,
+	               std::span {&descriptor_stage, 1u});
+	const auto vk_buffer = buffer.Handle();
+	const bool has_storage_writes = HasShaderBufferWrites(input_info.stage) ||
+	    std::any_of(program.info.images.begin(), program.info.images.end(), [](const auto& image) {
+		    return image.written && image.resource_class ==
+		                                ShaderRecompiler::IR::ImageResourceClass::Storage;
+	    });
+	if (has_storage_writes) {
+		ShaderWriteHazardBarrier(vk_buffer, vk::PipelineStageFlagBits::eComputeShader);
+	}
+	vk::MemoryBarrier barrier {};
+	barrier.srcAccessMask = vk::AccessFlagBits::eShaderWrite | vk::AccessFlagBits::eTransferWrite;
+	barrier.dstAccessMask = vk::AccessFlagBits::eIndirectCommandRead;
+	vk_buffer.pipelineBarrier(vk::PipelineStageFlagBits::eAllGraphics |
+	                              vk::PipelineStageFlagBits::eComputeShader |
+	                              vk::PipelineStageFlagBits::eTransfer,
+	                          vk::PipelineStageFlagBits::eDrawIndirect, {},
+	                          1, &barrier, 0, nullptr, 0, nullptr);
+	vk_buffer.bindPipeline(vk::PipelineBindPoint::eCompute, pipeline.pipeline);
+	vk_buffer.dispatchIndirect(args_buffer->Handle(), args_offset);
+	ShaderAccessBarrier(vk_buffer, vk::PipelineStageFlagBits::eComputeShader);
+	ResetBindings();
+}
+
 } // namespace Libs::Graphics

src/graphics/host_gpu/renderer/renderDraw.cpp

@@ -58,7 +58,7 @@ std::pair<int32_t, uint32_t> ResolveDrawOffsets(uint32_t index_offset,
 
 	EXIT_IF(!vs_input_info.stage);
 	const auto& program   = *vs_input_info.stage.program;
-	const auto& resources = vs_input_info.stage.resources;
+	const auto& resources = *vs_input_info.stage.resources;
 	if (index_offset == 0 &&
 	    program.info.vertex_offset_sgpr >= static_cast<int32_t>(program.user_data_base)) {
 		const auto index =
@@ -461,7 +461,6 @@ RenderState RenderExecutor::AcquireRenderTargets(CommandBuffer& buffer, RenderCo
 	state.height                = std::numeric_limits<uint32_t>::max();
 	state.num_layers            = std::numeric_limits<uint32_t>::max();
 	state.num_color_attachments = 0;
-	uint32_t attachment_samples = 0;
 	for (uint32_t i = 0; i < color_count; i++) {
 		auto& target = colors[i];
 		EXIT_IF(!target.image_id);
@@ -482,12 +481,6 @@ RenderState RenderExecutor::AcquireRenderTargets(CommandBuffer& buffer, RenderCo
 		                     target.desc.view_info.base_level, target.desc.view_info.base_layer,
 		                     target.desc.view_info.layer_count);
 		EXIT_IF(image.backing.samples != target.desc.info.samples || image_view == nullptr);
-		if (attachment_samples == 0) {
-			attachment_samples = target.desc.info.samples;
-		} else if (attachment_samples != target.desc.info.samples) {
-			EXIT("mixed color attachment sample counts are unsupported: %u and %u\n",
-			     attachment_samples, target.desc.info.samples);
-		}
 		const auto& view   = target.desc.view_info;
 		const auto  layout = image.binding.is_bound ? vk::ImageLayout::eGeneral
 		                                            : vk::ImageLayout::eColorAttachmentOptimal;
@@ -536,12 +529,6 @@ RenderState RenderExecutor::AcquireRenderTargets(CommandBuffer& buffer, RenderCo
 		                     image.info.data.address, depth.desc.view_info.base_layer,
 		                     depth.desc.view_info.layer_count);
 		EXIT_IF(image_view == nullptr || image.backing.samples != depth.desc.info.samples);
-		if (attachment_samples == 0) {
-			attachment_samples = depth.desc.info.samples;
-		} else if (attachment_samples != depth.desc.info.samples) {
-			EXIT("mixed color/depth sample counts are unsupported: %u and %u\n", attachment_samples,
-			     depth.desc.info.samples);
-		}
 		const bool feedback = depth.depth_write_enable && pixel &&
 		    std::ranges::any_of(pixel->images, [&](const TextureBinding& binding) {
 			    if (binding.image_id != depth.image_id ||
@@ -592,9 +579,6 @@ RenderState RenderExecutor::AcquireRenderTargets(CommandBuffer& buffer, RenderCo
 		const auto& limits = buffer.GetGraphics().GetPhysicalDeviceProperties().limits;
 		state.width        = limits.maxFramebufferWidth;
 		state.height       = limits.maxFramebufferHeight;
-	} else if (attachment_samples == 0 ||
-	           vulkan_sample_count(attachment_samples) == vk::SampleCountFlagBits {}) {
-		EXIT("render state has no valid attachments\n");
 	}
 	if (state.num_layers == std::numeric_limits<uint32_t>::max()) {
 		state.num_layers = 1;
@@ -1109,10 +1093,6 @@ void RenderExecutor::ExecutePreparedDraw(uint64_t submit_id, CommandBuffer& buff
 		vertex_bindings = AcquireVertexBuffers(buffer, state.vertex_info[0]);
 		index_binding   = PrepareIndexBuffer(buffer, index_source);
 	}
-	const auto rendering =
-	    AcquireRenderTargets(buffer, state.color_info, state.color_count, state.depth_info,
-	                         bindings.pixel);
-
 	if (draw.IsIndexed()) {
 		LogDrawPhase(draw.Name(), "CreatePipeline");
 	}
@@ -1120,6 +1100,9 @@ void RenderExecutor::ExecutePreparedDraw(uint64_t submit_id, CommandBuffer& buff
 	    std::span {state.color_info, state.color_count}, state.depth_info, vertex_stages, buffer,
 	    state.ps_active ? &state.ps_input_info : nullptr, topology, primitive_restart_enable,
 	    state.programs);
+	const auto rendering =
+	    AcquireRenderTargets(buffer, state.color_info, state.color_count, state.depth_info,
+	                         bindings.pixel);
 
 	// Resource preparation above may synchronously finish and restart the scheduler. From this
 	// point onward, every operation targets the current command buffer and cannot touch guest

src/graphics/presentation/videoOut.cpp

@@ -20,6 +20,7 @@
 #include "kernel/pthread.h"
 #include "libs/errno.h"
 #include "libs/libs.h"
+#include "loader/systemContent.h"
 
 #include <algorithm>
 #include <array>
@@ -1654,8 +1655,12 @@ KYTY_SYSV_ABI int VideoOutGetOutputStatus(int handle, VideoOutOutputStatus* stat
 		return VIDEO_OUT_ERROR_INVALID_HANDLE;
 	}
 
+	int32_t attribute3 = 0;
+	Loader::SystemContentParamSfoGetInt("ATTRIBUTE3", &attribute3);
 	ctx->mutex.Lock();
-	status->resolution   = (ctx->width >= 3840 || ctx->height >= 2160 ? 2u : 1u);
+	// Primary output reports 4K unless param.json Video-out Info enables resolution detection.
+	status->resolution =
+	    ((attribute3 & 4) != 0 && ctx->width < 3840 && ctx->height < 2160 ? 1u : 2u);
 	status->dynamicRange = 1;
 	status->refreshRate =
 	    (ctx->output_mode == VIDEO_OUT_OUTPUT_MODE_119_88HZ || Config::GetVblankFrequency() >= 119

src/graphics/shader/recompiler/backend/spirv/SpirvEmitter.cpp

@@ -132,12 +132,23 @@ void ValidateNativeProgram(const IR::Program& program) {
 			              program.memory_info[index].planning_only;
 		       });
 	};
+	const auto indirect_buffer_handle = [&](const IR::Inst& handle) {
+		return program.info.uses_dma && handle.NumArgs() == 4u && !handle.Uses().empty() &&
+		       std::ranges::all_of(handle.Uses(), [&](const IR::Use& use) {
+			       if (IR::BufferAccessOf(use.user->GetOpcode()) != IR::BufferAccess::Read) {
+				       return false;
+			       }
+			       const auto index = use.user->Flags<IR::MemoryFlags>().index;
+			       return index < program.memory_info.size() &&
+			              program.memory_info[index].kind == IR::ResourceKind::IndirectBuffer;
+		       });
+	};
 	for (const auto* block: program.blocks) {
 		for (const auto& inst: *block) {
 			const auto dense = inst.Flags<uint32_t>();
 			switch (inst.GetOpcode()) {
 				case IR::ValueOpcode::GetBufferResource:
-					if (planning_only_handle(inst)) {
+					if (planning_only_handle(inst) || indirect_buffer_handle(inst)) {
 						break;
 					}
 					if (dense >= program.info.buffers.size()) {
@@ -212,6 +223,9 @@ Emitter::SpirvRequirements Emitter::AnalyzeProgramRequirements(const IR::Program
 					Fail(program, "buffer operation has invalid memory metadata");
 				}
 				const auto& memory = program.memory_info[memory_index];
+				if (memory.kind == IR::ResourceKind::IndirectBuffer) {
+					requirements.subgroup_local_invocation_id = true;
+				}
 				if (memory.kind == IR::ResourceKind::Buffer) {
 					if (memory.resource >= program.info.buffers.size()) {
 						Fail(program, "buffer operation has invalid resource metadata");

src/graphics/shader/recompiler/backend/spirv/spirvEmitterFlow.cpp

@@ -559,6 +559,10 @@ void EmitVoid(ValueEmitContext&) {}
 
 void EmitBarrier(EmitterState& state) {
 	const auto tessellation = state.program.stage == ShaderType::TessellationControl;
+	if (!tessellation && ShaderWorkgroupInput(state.program.stage, state.input_info) == nullptr) {
+		// Independent graphics invocations have no native workgroup left to synchronize.
+		return;
+	}
 	const auto memory_scope = tessellation ? spv::ScopeInvocation : spv::ScopeWorkgroup;
 	const auto semantics    = tessellation ? spv::MemorySemanticsMaskNone
 	                                       : spv::MemorySemanticsAcquireReleaseMask |

src/graphics/shader/recompiler/backend/spirv/spirvEmitterImage.cpp

@@ -63,10 +63,10 @@ uint32_t AddressF32(ValueEmitContext& ctx, const IR::MemoryInfo& mem, const IR::
 	           : Unary(ctx.state, spv::OpBitcast, TypeF32(ctx.state), value);
 }
 
-ImageSampleLayout Layout(const IR::MemoryInfo& mem, ImageDimension dimension) {
+ImageSampleLayout Layout(const IR::MemoryInfo& mem) {
 	ImageSampleLayout layout;
 	uint32_t          cursor = 0;
-	const auto&       info   = ImageDimensionInfoFor(dimension);
+	const auto&       info   = ImageDimensionInfoFor(mem.image_dimension);
 	if (HasFlag(mem, Decoder::ImageSampleFlagOffset)) layout.offset = cursor++;
 	if (HasFlag(mem, Decoder::ImageSampleFlagBias)) layout.bias = cursor++;
 	if (HasFlag(mem, Decoder::ImageSampleFlagCompare)) layout.dref = cursor++;
@@ -104,8 +104,7 @@ uint32_t CubeLayer(EmitterState& state, uint32_t value) {
 }
 
 uint32_t CoordF32(ValueEmitContext& ctx, const IR::MemoryInfo& mem, const IR::Inst& address,
-                  uint32_t first, uint32_t components) {
-	const bool cube = ctx.state.program.info.images.at(mem.resource).cube;
+                  uint32_t first, uint32_t components, bool cube = false) {
 	auto x = AddressF32(ctx, mem, address, first);
 	if (components == 1u) return x;
 	auto y = mem.image_address_components > first + 1u ? AddressF32(ctx, mem, address, first + 1u)
@@ -576,7 +575,8 @@ void EmitImage(ValueEmitContext& ctx, const IR::Inst& inst) {
 		const auto lod       = state.builder.AllocateId();
 		state.builder.AddFunction(
 		    spv::OpImageQueryLod, TypeF32Vector(state, 2), lod, sampled,
-		    CoordF32(ctx, mem, *address, 0, ImageDimensionInfoFor(dimension).spatial_components));
+		    CoordF32(ctx, mem, *address, 0, ImageDimensionInfoFor(dimension).spatial_components,
+		             image.cube));
 		uint32_t values[4] = {ConstantU32(state, 0), ConstantU32(state, 0), ConstantU32(state, 0),
 		                      ConstantU32(state, 0)};
 		for (uint32_t index = 0; index < 2u; index++) {
@@ -637,7 +637,7 @@ void EmitImage(ValueEmitContext& ctx, const IR::Inst& inst) {
 	if (op == IR::ValueOpcode::ImageSampleRaw || op == IR::ValueOpcode::ImageGatherRaw) {
 		const auto  dimension      = image.dimension;
 		const auto& dimension_info = ImageDimensionInfoFor(dimension);
-		const auto  layout         = Layout(mem, dimension);
+		const auto  layout         = Layout(mem);
 		const auto  numeric_class  = image.numeric_class;
 		const bool  dref           = HasFlag(mem, Decoder::ImageSampleFlagCompare);
 		if (dref && state.program.info.images[mem.resource].conversion_format !=
@@ -645,9 +645,9 @@ void EmitImage(ValueEmitContext& ctx, const IR::Inst& inst) {
 			ctx.Fail(inst, "uses depth comparison with a packed integer image");
 			return;
 		}
-		const auto coord =
-		    CoordF32(ctx, mem, *address, layout.coord, dimension_info.coordinate_components);
 		if (op == IR::ValueOpcode::ImageGatherRaw) {
+			const auto coord = CoordF32(ctx, mem, *address, layout.coord,
+			                            dimension_info.coordinate_components, image.cube);
 			if (HasFlag(mem, Decoder::ImageSampleFlagLod)) {
 				static std::atomic_flag warned = ATOMIC_FLAG_INIT;
 				if (!warned.test_and_set(std::memory_order_relaxed)) {
@@ -752,6 +752,11 @@ void EmitImage(ValueEmitContext& ctx, const IR::Inst& inst) {
 			operands.push_back(AddressF32(ctx, mem, *address, layout.bias));
 		}
 		const auto EmitSample = [&](uint32_t resource) {
+			const auto& candidate = state.program.info.images[resource];
+			const auto coord =
+			    CoordF32(ctx, mem, *address, layout.coord,
+			             ImageDimensionInfoFor(candidate.dimension).coordinate_components,
+			             candidate.cube);
 			const auto            sampled = MakeSampledImage(state, resource, mem.sampler);
 			const auto            sample  = state.builder.AllocateId();
 			std::vector<uint32_t> sample_operands {result_type, sample, sampled, coord};

src/graphics/shader/recompiler/backend/spirv/spirvEmitterInternal.h

@@ -494,22 +494,18 @@ template <typename Fn>
 uint32_t EmitValueOrDefaultIfCondition(EmitterState& state, uint32_t condition, uint32_t type,
                                        uint32_t default_value, Fn&& fn) {
 	const auto then_label  = state.builder.AllocateId();
-	const auto then_exit   = state.builder.AllocateId();
-	const auto else_label  = state.builder.AllocateId();
+	const auto header     = state.current_label;
 	const auto merge_label = state.builder.AllocateId();
 	state.builder.AddFunction(spv::OpSelectionMerge, merge_label, spv::SelectionControlMaskNone);
-	state.builder.AddFunction(spv::OpBranchConditional, condition, then_label, else_label);
+	state.builder.AddFunction(spv::OpBranchConditional, condition, then_label, merge_label);
 	EmitLabel(state, then_label);
 	const auto then_value = fn();
-	state.builder.AddFunction(spv::OpBranch, then_exit);
-	EmitLabel(state, then_exit);
-	state.builder.AddFunction(spv::OpBranch, merge_label);
-	EmitLabel(state, else_label);
+	const auto then_exit  = state.current_label;
 	state.builder.AddFunction(spv::OpBranch, merge_label);
 	EmitLabel(state, merge_label);
 	const auto value = state.builder.AllocateId();
 	state.builder.AddFunction(spv::OpPhi, type, value, then_value, then_exit, default_value,
-	                          else_label);
+	                          header);
 	return value;
 }
 

src/graphics/shader/recompiler/backend/spirv/spirvEmitterMemory.cpp

@@ -27,61 +27,71 @@ uint32_t EmitDsMaskedLaneRead(EmitterState& state, uint32_t source, uint32_t tar
 	return Select(state, TypeU32(state), source_active, shuffled, ConstantU32(state, 0));
 }
 
-uint32_t BufferByteAddress(ValueEmitContext& ctx, const IR::Inst& inst, const IR::MemoryInfo& mem,
-                           uint32_t index, uint32_t offset, uint32_t soffset) {
-	auto&          state   = ctx.state;
-	const uint32_t packed  = StorageBufferPackedStride(state, mem);
-	const uint32_t stride  = packed & 0x3fffu;
-	const bool     swizzle = stride != 0u && ((packed >> 14u) & 1u) != 0u;
-	if (((packed >> 20u) & 1u) != 0u) {
-		const auto lane = Binary(state, spv::OpBitwiseAnd, TypeU32(state),
-		                         EmitSubgroupLocalInvocationId(state), ConstantU32(state, 63));
-		index           = Binary(state, spv::OpIAdd, TypeU32(state), index, lane);
-	}
-	if (mem.offset != 0u) {
-		offset = Binary(state, spv::OpIAdd, TypeU32(state), offset, ConstantU32(state, mem.offset));
-	}
+struct BufferAddress {
+	uint32_t offset;
+	uint32_t byte;
+};
 
-	uint32_t address = 0;
-	if (!swizzle) {
-		if (stride == 0u) {
-			address = offset;
-		} else {
-			const auto indexed = stride == 1u ? index
-			                                  : Binary(state, spv::OpIMul, TypeU32(state), index,
-			                                           ConstantU32(state, stride));
-			address            = Binary(state, spv::OpIAdd, TypeU32(state), indexed, offset);
-		}
-	} else {
-		const uint32_t stride_enum  = (packed >> 16u) & 3u;
-		const uint32_t index_stride = 8u << stride_enum;
-		const auto     index_msb    = Binary(state, spv::OpShiftRightLogical, TypeU32(state), index,
-		                                     ConstantU32(state, stride_enum + 3u));
-		const auto     index_lsb    = Binary(state, spv::OpBitwiseAnd, TypeU32(state), index,
-		                                     ConstantU32(state, index_stride - 1u));
-		const auto     offset_msb =
+BufferAddress CalculateBufferAddress(EmitterState& state, uint32_t index, uint32_t offset,
+                                     uint32_t soffset, uint32_t immediate, uint32_t stride,
+                                     uint32_t swizzle, uint32_t index_stride) {
+	const auto zero = ConstantU32(state, 0);
+	const auto one  = ConstantU32(state, 1);
+	const auto add = [&](uint32_t lhs, uint32_t rhs) {
+		return lhs == zero ? rhs : rhs == zero ? lhs
+		                                      : Binary(state, spv::OpIAdd, TypeU32(state), lhs, rhs);
+	};
+	const auto mul = [&](uint32_t lhs, uint32_t rhs) {
+		return lhs == zero || rhs == zero ? zero
+		       : lhs == one              ? rhs
+		       : rhs == one              ? lhs
+		                                 : Binary(state, spv::OpIMul, TypeU32(state), lhs, rhs);
+	};
+	if (immediate != 0u) {
+		offset = add(offset, ConstantU32(state, immediate));
+	}
+	auto address = add(mul(index, stride), offset);
+	if (swizzle != 0u) {
+		const auto index_shift =
+		    Binary(state, spv::OpIAdd, TypeU32(state), index_stride, ConstantU32(state, 3));
+		const auto indices = Binary(state, spv::OpShiftLeftLogical, TypeU32(state),
+		                            ConstantU32(state, 1), index_shift);
+		const auto index_msb =
+		    Binary(state, spv::OpShiftRightLogical, TypeU32(state), index, index_shift);
+		const auto index_lsb =
+		    Binary(state, spv::OpBitwiseAnd, TypeU32(state), index,
+		           Binary(state, spv::OpISub, TypeU32(state), indices, ConstantU32(state, 1)));
+		const auto offset_msb =
 		    Binary(state, spv::OpBitwiseAnd, TypeU32(state), offset, ConstantU32(state, ~3u));
 		const auto offset_lsb =
 		    Binary(state, spv::OpBitwiseAnd, TypeU32(state), offset, ConstantU32(state, 3u));
-		const auto indexed_msb = stride == 1u ? index_msb
-		                                      : Binary(state, spv::OpIMul, TypeU32(state),
-		                                               index_msb, ConstantU32(state, stride));
-		const auto msb = Binary(state, spv::OpIMul, TypeU32(state),
-		                        Binary(state, spv::OpIAdd, TypeU32(state), indexed_msb, offset_msb),
-		                        ConstantU32(state, index_stride));
-		const auto lsb = Binary(state, spv::OpIAdd, TypeU32(state),
-		                        Binary(state, spv::OpShiftLeftLogical, TypeU32(state), index_lsb,
-		                               ConstantU32(state, 2u)),
-		                        offset_lsb);
-		address        = Binary(state, spv::OpIAdd, TypeU32(state), msb, lsb);
+		const auto msb = mul(add(mul(index_msb, stride), offset_msb), indices);
+		const auto lsb = add(Binary(state, spv::OpShiftLeftLogical, TypeU32(state), index_lsb,
+		                            ConstantU32(state, 2u)), offset_lsb);
+		address = Select(state, TypeU32(state), swizzle, add(msb, lsb), address);
 	}
+	return {offset, add(address, soffset)};
+}
 
-	const auto soffset_value = inst.Arg(3).Resolve();
-	if (soffset_value.IsImmediate() && soffset_value.GetType() == IR::Type::U32 &&
-	    soffset_value.U32() == 0u) {
-		return address;
+uint32_t BufferLane(EmitterState& state) {
+	return Binary(state, spv::OpBitwiseAnd, TypeU32(state), EmitSubgroupLocalInvocationId(state),
+	              ConstantU32(state, 63));
+}
+
+uint32_t BufferByteAddress(ValueEmitContext& ctx, const IR::Inst& inst, const IR::MemoryInfo& mem) {
+	auto&      state  = ctx.state;
+	const auto packed = StorageBufferPackedStride(state, mem);
+	const auto stride = packed & 0x3fffu;
+	auto       index  = ctx.Arg(inst, 1);
+	if ((packed & (1u << 20u)) != 0u) {
+		index = Binary(state, spv::OpIAdd, TypeU32(state), index, BufferLane(state));
 	}
-	return Binary(state, spv::OpIAdd, TypeU32(state), address, soffset);
+	const bool swizzle = stride != 0u && (packed & (1u << 14u)) != 0u;
+	return CalculateBufferAddress(state, index, ctx.Arg(inst, 2), ctx.Arg(inst, 3), mem.offset,
+	                              ConstantU32(state, stride),
+	                              swizzle ? ConstantBool(state, true) : 0u,
+	                              ConstantU32(state, (packed >> 16u) & 3u))
+	    .byte;
 }
 
 uint32_t AddU64Low(EmitterState& state, uint32_t low, uint32_t high, uint32_t add_low,
@@ -196,11 +206,8 @@ uint32_t LoadBdaDword(ValueEmitContext& ctx, uint32_t address) {
 	});
 }
 
-uint32_t LoadBda(ValueEmitContext& ctx, const IR::Inst& inst, const IR::MemoryInfo& mem,
-	             uint32_t bits) {
-	auto&      state   = ctx.state;
-	const auto address = GuestAddress(ctx, inst, mem);
-	const auto active  = ctx.Arg(inst, inst.NumArgs() - 1);
+uint32_t LoadBda(ValueEmitContext& ctx, uint32_t address, uint32_t active, uint32_t bits) {
+	auto& state = ctx.state;
 	return EmitValueOrZeroIfCondition(state, active, [&]() {
 		const auto aligned = Binary(state, spv::OpBitwiseAnd, TypeScalarU64(state), address,
 		                            ConstantDeviceAddress(state, ~uint64_t {3}));
@@ -236,8 +243,7 @@ uint32_t LoadBda(ValueEmitContext& ctx, const IR::Inst& inst, const IR::MemoryIn
 
 uint32_t ByteAddress(ValueEmitContext& ctx, const IR::Inst& inst, const IR::MemoryInfo& mem) {
 	if (mem.kind == IR::ResourceKind::Buffer) {
-		return BufferByteAddress(ctx, inst, mem, ctx.Arg(inst, 1), ctx.Arg(inst, 2),
-		                         ctx.Arg(inst, 3));
+		return BufferByteAddress(ctx, inst, mem);
 	}
 	if (mem.kind == IR::ResourceKind::Lds || mem.kind == IR::ResourceKind::Gds) {
 		if (mem.offset == 0u) {
@@ -734,12 +740,86 @@ void FormattedStore(ValueEmitContext& ctx, const IR::Inst& inst, const IR::Memor
 	});
 }
 
+uint32_t LoadIndirectBuffer(ValueEmitContext& ctx, const IR::Inst& inst, uint32_t components) {
+	auto&       state   = ctx.state;
+	const auto& handle  = *inst.Arg(0).ResolveInstruction();
+	const auto  word1   = ctx.Arg(handle, 1);
+	const auto  records = ctx.Arg(handle, 2);
+	const auto  word3   = ctx.Arg(handle, 3);
+	const auto  field   = [&](uint32_t word, uint32_t first, uint32_t count) {
+		return EmitBitFieldUExtract(state, word, ConstantU32(state, first),
+		                            ConstantU32(state, count));
+	};
+	const auto nonzero = [&](uint32_t value) {
+		return Binary(state, spv::OpINotEqual, TypeBool(state), value, ConstantU32(state, 0));
+	};
+	const auto has_dword = [&](uint32_t offset, uint32_t size) {
+		return AndCondition(
+		    state,
+		    Binary(state, spv::OpUGreaterThanEqual, TypeBool(state), size, ConstantU32(state, 4)),
+		    Binary(state, spv::OpULessThanEqual, TypeBool(state), offset,
+		           Binary(state, spv::OpISub, TypeU32(state), size, ConstantU32(state, 4))));
+	};
+	const auto stride       = field(word1, 16, 14);
+	const auto swizzle      = AndCondition(state, nonzero(stride), nonzero(field(word1, 31, 1)));
+	const auto index_stride = field(word3, 21, 2);
+	const auto add_tid      = nonzero(field(word3, 23, 1));
+	const auto index =
+	    Binary(state, spv::OpIAdd, TypeU32(state), ctx.Arg(inst, 1),
+	           Select(state, TypeU32(state), add_tid, BufferLane(state), ConstantU32(state, 0)));
+	const auto soffset = ctx.Arg(inst, 3);
+	const auto base    = DeviceAddressFromWords(state, ctx.Arg(handle, 0), field(word1, 0, 16));
+	const auto valid_format    = nonzero(field(word3, 12, 7));
+	const auto mode            = field(word3, 28, 2);
+	const auto index_in_bounds = Binary(state, spv::OpULessThan, TypeBool(state), index, records);
+	const auto scalar_in_bounds =
+	    Binary(state, spv::OpULessThanEqual, TypeBool(state), soffset, records);
+	const auto raw_records = Binary(state, spv::OpISub, TypeU32(state), records, soffset);
+	const auto raw_index_in_bounds =
+	    Binary(state, spv::OpULessThan, TypeBool(state), index, raw_records);
+	std::array<uint32_t, 4> values {};
+	for (uint32_t component = 0; component < components; component++) {
+		const auto address = CalculateBufferAddress(state, index, ctx.Arg(inst, 2), soffset,
+		                                            ctx.Memory(inst).offset + component * 4u,
+		                                            stride, swizzle, index_stride);
+		// RDNA2 raw DWORD vectors check each component, unlike formatted accesses.
+		const auto structured_bounds =
+		    AndCondition(state, index_in_bounds,
+		                 Binary(state, spv::OpULessThan, TypeBool(state), address.offset, stride));
+		// OOB_SELECT=3 reduces NUM_RECORDS by SOFFSET before the offset/index check.
+		const auto raw_bounds = AndCondition(
+		    state, scalar_in_bounds,
+		    Select(state, TypeBool(state), swizzle,
+		           AndCondition(state, raw_index_in_bounds, has_dword(address.offset, stride)),
+		           has_dword(address.offset, raw_records)));
+		auto in_bounds =
+		    Select(state, TypeBool(state),
+		           Binary(state, spv::OpIEqual, TypeBool(state), mode, ConstantU32(state, 0)),
+		           structured_bounds, index_in_bounds);
+		in_bounds = Select(
+		    state, TypeBool(state),
+		    Binary(state, spv::OpULessThan, TypeBool(state), mode, ConstantU32(state, 2)),
+		    in_bounds,
+		    Select(state, TypeBool(state),
+		           Binary(state, spv::OpIEqual, TypeBool(state), mode, ConstantU32(state, 2)),
+		           nonzero(records), raw_bounds));
+		const auto guest =
+		    Binary(state, spv::OpIAdd, TypeScalarU64(state), base,
+		           Unary(state, spv::OpUConvert, TypeScalarU64(state), address.byte));
+		values[component] = LoadBda(ctx, guest, AndCondition(state, valid_format, in_bounds), 32u);
+	}
+	return ConstructU32Composite(state, components, values);
+}
+
 uint32_t LoadWideBuffer(ValueEmitContext& ctx, const IR::Inst& inst, uint32_t components) {
 	auto& state = ctx.state;
 	return EmitValueOrDefaultIfCondition(
 	    state, ctx.Arg(inst, inst.NumArgs() - 1), TypeU32Composite(state, components),
 	    ConstantU32CompositeZero(state, components), [&]() {
 		    const auto mem      = ctx.Memory(inst);
+		    if (mem.kind == IR::ResourceKind::IndirectBuffer) {
+			    return LoadIndirectBuffer(ctx, inst, components);
+		    }
 		    const auto resource = PrepareMemoryResourceAccess(state, mem);
 		    const auto info = Format::GetFormatInfo(
 		        mem.formatted ? BufferFormat(ctx, mem) : Prospero::BufferFormat::kInvalid);
@@ -1088,7 +1168,8 @@ void EmitLoadMemory(ValueEmitContext& ctx, const IR::Inst& inst) {
 		value = LoadWideShared(ctx, inst, shared_components);
 	else if (address_info.access == IR::AddressAccess::Read &&
 	         mem.kind != IR::ResourceKind::Scratch)
-		value = LoadBda(ctx, inst, mem, address_info.data_bits);
+		value = LoadBda(ctx, GuestAddress(ctx, inst, mem), ctx.Arg(inst, inst.NumArgs() - 1),
+		                address_info.data_bits);
 	else if (op == IR::ValueOpcode::LoadBufferU32 && mem.formatted)
 		value = FormattedLoad(ctx, inst, mem);
 	else if (inst.GetType() == IR::Type::U8)

src/graphics/shader/recompiler/backend/spirv/spirvEmitterMesh.cpp

@@ -1,4 +1,4 @@
-#include "graphics/shader/recompiler/backend/spirv/spirvEmitterInternal.h"
+#include "graphics/shader/recompiler/backend/spirv/spirvEmitterInstructions.h"
 
 namespace Libs::Graphics::ShaderRecompiler::Spirv::Emitter {
 namespace {
@@ -127,10 +127,7 @@ void EmitMeshEntryPoint(EmitterState& state) {
 	state.builder.AddFunction(spv::OpFunctionCall, TypeVoid(state), state.builder.AllocateId(),
 	                          state.mesh_guest_func);
 	// All guest waves finish before the uniform Vulkan allocation and output stores.
-	state.builder.AddFunction(spv::OpControlBarrier, ConstantU32(state, spv::ScopeWorkgroup),
-	                          ConstantU32(state, spv::ScopeWorkgroup),
-	                          ConstantU32(state, spv::MemorySemanticsAcquireReleaseMask |
-	                                                 spv::MemorySemanticsWorkgroupMemoryMask));
+	EmitBarrier(state);
 	const auto vertices   = MeshLoad(state, state.mesh_allocation, spv::StorageClassWorkgroup,
 	                                 TypeU32(state), ConstantU32(state, 0));
 	const auto primitives = MeshLoad(state, state.mesh_allocation, spv::StorageClassWorkgroup,

src/graphics/shader/recompiler/frontend/decode/VectorAluOps.cpp

@@ -527,8 +527,8 @@ constexpr Vop1SdwaRule VOP1_SDWA_RULES[] = {
     {Opcode::V_CVT_F32_F16, SdwaSelWords() | SdwaSelFull(), 0, 0, true},
     {Opcode::V_CVT_F16_F32, SdwaSelBytes() | SdwaSelWords() | SdwaSelFull(), SdwaSelWords(),
      SdwaSelBytes() | SdwaSelWords() | SdwaSelFull(), true},
-    {Opcode::V_CVT_F16_U16, SdwaSelWords() | SdwaSelFull(), SdwaSelWords(),
-     SdwaSelWords() | SdwaSelFull(), false},
+    {Opcode::V_CVT_F16_U16, SdwaSelBytes() | SdwaSelWords() | SdwaSelFull(), SdwaSelWords(),
+     SdwaSelBytes() | SdwaSelWords() | SdwaSelFull(), false},
     {Opcode::V_CVT_U16_F16, SdwaSelWords() | SdwaSelFull(), SdwaSelWords(),
      SdwaSelWords() | SdwaSelFull(), true},
     {Opcode::V_CVT_F16_I16, SdwaSelWords() | SdwaSelFull(), SdwaSelWords(),
@@ -618,13 +618,14 @@ bool SupportsVop1Clamp(Opcode opcode) {
 }
 
 bool ValidateVop1Sdwa(Instruction& inst, uint32_t opcode, uint32_t modifier) {
-	const auto dst_sel  = (modifier >> 8u) & 0x7u;
-	const auto dst_u    = (modifier >> 11u) & 0x3u;
-	const auto clamp    = (modifier >> 13u) & 0x1u;
-	const auto omod     = (modifier >> 14u) & 0x3u;
-	const auto src0_sel = (modifier >> 16u) & 0x7u;
-	const auto src0_neg = (modifier >> 20u) & 0x1u;
-	const auto src0_abs = (modifier >> 21u) & 0x1u;
+	const auto dst_sel   = (modifier >> 8u) & 0x7u;
+	const auto dst_u     = (modifier >> 11u) & 0x3u;
+	const auto clamp     = (modifier >> 13u) & 0x1u;
+	const auto omod      = (modifier >> 14u) & 0x3u;
+	const auto src0_sel  = (modifier >> 16u) & 0x7u;
+	const auto src0_sext = (modifier >> 19u) & 0x1u;
+	const auto src0_neg  = (modifier >> 20u) & 0x1u;
+	const auto src0_abs  = (modifier >> 21u) & 0x1u;
 
 	if (src0_sel > 6u || dst_sel > 6u) {
 		SetUnsupported(inst, Family::VOP1, opcode, "VOP1 SDWA selector is invalid");
@@ -640,7 +641,8 @@ bool ValidateVop1Sdwa(Instruction& inst, uint32_t opcode, uint32_t modifier) {
 		               "VOP1 SDWA destination selector is not supported");
 		return false;
 	}
-	if (!IsVop1SdwaSourceSupported(inst.opcode, src0_sel, src0_neg != 0u, src0_abs != 0u)) {
+	if (!IsVop1SdwaSourceSupported(inst.opcode, src0_sel, src0_neg != 0u, src0_abs != 0u) ||
+	    (inst.opcode == Opcode::V_CVT_F16_U16 && src0_sel <= 3u && src0_sext != 0u)) {
 		SetUnsupported(inst, Family::VOP1, opcode, "VOP1 SDWA source selector is not supported");
 		return false;
 	}

src/graphics/shader/recompiler/frontend/translate/Translate.cpp

@@ -1225,6 +1225,12 @@ IR::Program TranslateProgram(const Decoder::Program& decoded, const CFG::Graph&
 				                      builtin(IR::StageInputKind::PackedAncillary));
 			}
 		} else if (options.stage == ShaderType::Vertex) {
+			// Vulkan owns primitive assembly; each vertex subgroup is one NGG wave.
+			// Keep its full lane extent: mbcnt(-1) uses lane ordinals, not active counts.
+			entry_ir.SetScalarReg(static_cast<IR::ScalarReg>(2),
+			                      IR::U32(IR::Value(options.wave_size << 12u)));
+			entry_ir.SetScalarReg(static_cast<IR::ScalarReg>(3),
+			                      IR::U32(IR::Value((1u << 28u) | options.wave_size)));
 			entry_ir.SetVectorReg(static_cast<IR::VectorReg>(5),
 			                      builtin(IR::StageInputKind::VertexIndex));
 			entry_ir.SetVectorReg(static_cast<IR::VectorReg>(8),

src/graphics/shader/recompiler/ir/Program.cpp

@@ -447,13 +447,20 @@ void ValidateProgram(const Program& program, bool require_ssa) {
 					                        ValueOpcodeName(inst.GetOpcode())));
 				}
 				const auto& memory = program.memory_info[memory_index];
-				if (memory.kind != ResourceKind::Buffer &&
-				    memory.kind != ResourceKind::ScalarBuffer) {
+				const bool  vector_buffer = memory.kind == ResourceKind::Buffer ||
+				                            memory.kind == ResourceKind::IndirectBuffer;
+				if (!vector_buffer && memory.kind != ResourceKind::ScalarBuffer) {
 					return Fail(fmt::format("{} has a non-buffer resource kind",
 					                        ValueOpcodeName(inst.GetOpcode())));
 				}
+				if (memory.kind == ResourceKind::IndirectBuffer &&
+				    (memory.formatted || memory.typed ||
+				     (inst.GetOpcode() != ValueOpcode::LoadBufferU32x2 &&
+				      inst.GetOpcode() != ValueOpcode::LoadBufferU32x4))) {
+					return Fail("indirect buffer requires a raw DWORD x2/x4 load");
+				}
 				if (buffer_components > 1u &&
-				    (memory.kind != ResourceKind::Buffer || memory.data_bits != 32u ||
+				    (!vector_buffer || memory.data_bits != 32u ||
 				     memory.data_dwords != buffer_components || memory.component_index != 0u)) {
 					return Fail(fmt::format("{} has inconsistent native-wide metadata",
 					                        ValueOpcodeName(inst.GetOpcode())));

src/graphics/shader/recompiler/ir/ShaderIR.h

@@ -14,6 +14,7 @@
 
 #include <array>
 #include <bit>
+#include <deque>
 #include <list>
 #include <memory>
 #include <optional>
@@ -27,6 +28,7 @@ enum class ResourceKind {
 	ScalarBuffer,
 	ScalarAddress,
 	Buffer,
+	IndirectBuffer,
 	Flat,
 	Global,
 	Scratch,
@@ -457,11 +459,13 @@ struct BlockInfo {
 
 struct DescriptorSource {
 	struct IndirectImage {
-		uint32_t material_source = 0;
-		uint32_t heap_source     = 0;
+		uint32_t material_source = UINT32_MAX;
+		uint32_t table_source    = 0;
 		uint32_t selector_stride = 0;
 		uint32_t selector_offset = 0;
 		uint32_t key_arg         = 0;
+		uint32_t table_offset    = 0;
+		uint32_t key_count       = 0;
 
 		bool operator==(const IndirectImage& other) const = default;
 	};
@@ -505,9 +509,19 @@ struct UniformFillPlan {
 	std::array<Value, 4> values;
 };
 
-// Immutable runtime resource analysis retained by the shader cache. It owns descriptor/SRT,
-// uniform condition and fill values without retaining translated blocks.
+// Resource analysis retained by the shader cache. It owns immutable descriptor/SRT,
+// condition and fill values without translated blocks, plus reusable evaluation scratch.
 struct ResourcePlan {
+	struct EvaluationContext {
+		struct Entry {
+			uint64_t value      = 0;
+			uint64_t generation = 0;
+		};
+
+		std::vector<Entry> values;
+		uint64_t           generation = 0;
+	};
+
 	ResourcePlan() = default;
 	~ResourcePlan();
 
@@ -524,7 +538,6 @@ struct ResourcePlan {
 	std::vector<MemoryInfo>             memory_info;
 	std::vector<DescriptorSource>       descriptor_sources;
 	std::vector<ResourceBlock>          control_flow;
-	std::vector<uint32_t>               materialization_sources;
 	std::vector<SrtRead>                srt_reads;
 	std::vector<uint8_t>                clean_flat_slots;
 	bool                                requires_specialization_memory = false;
@@ -532,6 +545,14 @@ struct ResourcePlan {
 	bool                                resource_tracking_complete = false;
 	ShaderInfo                          info;
 	UniformFillPlan                     uniform_fill;
+	// GPU-thread scratch for nested clean/EXEC memos, activity and material keys.
+	mutable std::deque<EvaluationContext> evaluation_contexts;
+	mutable uint32_t                       evaluation_value_count = 0;
+	mutable uint32_t                       evaluation_depth       = 0;
+	mutable std::vector<uint8_t>            active_sources;
+	mutable std::vector<uint8_t>            visited_blocks;
+	mutable std::vector<uint32_t>           pending_blocks;
+	mutable std::vector<uint32_t>           material_keys;
 };
 
 struct Program: ResourcePlan {

src/graphics/shader/recompiler/ir/Value.h

@@ -121,6 +121,13 @@ public:
 	[[nodiscard]] Block*                  PhiBlock(size_t index) const;
 	[[nodiscard]] Block*                  Parent() const;
 	[[nodiscard]] const std::vector<Use>& Uses() const;
+	// Runtime indices belong to the resource plan that owns this instruction.
+	[[nodiscard]] uint32_t EvaluationIndex(uint32_t& count) const {
+		if (evaluation_index == UINT32_MAX) {
+			evaluation_index = count++;
+		}
+		return evaluation_index;
+	}
 
 	void SetParent(Block* block);
 	void SetArg(size_t index, Value value);
@@ -155,6 +162,7 @@ private:
 	std::vector<Value>  args;
 	std::vector<Block*> phi_blocks;
 	std::vector<Use>    uses;
+	mutable uint32_t    evaluation_index = UINT32_MAX;
 };
 
 } // namespace Libs::Graphics::ShaderRecompiler::IR

src/graphics/shader/recompiler/ir/passes/ResourceMaterialization.cpp

@@ -21,18 +21,6 @@ namespace {
 constexpr uint64_t AddressMask            = 0x0000ffffffffffffull;
 constexpr uint64_t MaxIndirectImageProbes = 65536u;
 
-struct IndirectImage {
-	uint32_t                     resource = 0;
-	std::vector<uint32_t>        keys;
-	std::vector<uint32_t>        candidates;
-	std::vector<DescriptorValue> descriptors;
-};
-
-struct MaterializedSnapshot {
-	ResourceSnapshot           resources;
-	std::vector<IndirectImage> indirect_images;
-};
-
 bool SpecializationFail(std::string_view message) {
 	std::fprintf(stderr, "shader resource specialization failed: %.*s\n",
 	             static_cast<int>(message.size()), message.data());
@@ -74,6 +62,13 @@ bool NullImageDescriptor(const DescriptorValue& descriptor) {
 }
 
 bool ValidImageDescriptor(const DescriptorValue& descriptor, bool r128 = false) {
+	const auto& words = descriptor.dwords;
+	// Reject texture descriptors with nonzero reserved bits.
+	if ((words[1] & 0x20000000u) != 0u || (words[2] & 0x70003000u) != 0u ||
+	    (!r128 && ((words[4] & 0xe000e000u) != 0u || (words[5] & 0xf9000000u) != 0u ||
+	               (words[6] & 0x00007b00u) != 0u))) {
+		return false;
+	}
 	const auto type   = static_cast<Prospero::ImageType>((descriptor.dwords[3] >> 28u) & 0xfu);
 	const auto format = static_cast<Prospero::BufferFormat>((descriptor.dwords[1] >> 20u) & 0x1ffu);
 	if (type < Prospero::ImageType::kColor1D || format == Prospero::BufferFormat::kInvalid ||
@@ -84,6 +79,13 @@ bool ValidImageDescriptor(const DescriptorValue& descriptor, bool r128 = false)
 	    type != Prospero::ImageType::kColor2DMsaa) {
 		return false;
 	}
+	const bool array = type == Prospero::ImageType::kColor1DArray ||
+	                   type == Prospero::ImageType::kColor2DArray ||
+	                   type == Prospero::ImageType::kColor2DMsaaArray ||
+	                   type == Prospero::ImageType::kCube;
+	if (array && ((words[4] >> 16u) & 0x1fffu) > (words[4] & 0x1fffu)) {
+		return false;
+	}
 	if (type == Prospero::ImageType::kColor2DMsaa ||
 	    type == Prospero::ImageType::kColor2DMsaaArray) {
 		const auto base_level = (descriptor.dwords[3] >> 12u) & 0xfu;
@@ -173,200 +175,155 @@ void MarkCleanFlatSlots(const ResourcePlan& program, const DescriptorSource* sou
 	}
 }
 
-bool ReadSpecializationWord(const SrtRuntime& runtime, uint64_t address, uint32_t& word) {
-	return runtime.read_specialization_memory != nullptr &&
-	       runtime.read_specialization_memory(runtime.userdata, address, &word);
-}
-
-bool ReadScalarBufferWord(const ShaderBufferResource& descriptor, uint32_t dynamic_offset,
-                          uint32_t immediate_offset, const SrtRuntime& runtime, uint32_t& word) {
-	const auto byte_offset = static_cast<uint64_t>(dynamic_offset) + immediate_offset;
-	const auto aligned     = byte_offset & ~uint64_t {3};
-	const auto size        = descriptor.GetSize();
-	if (aligned > size || size - aligned < sizeof(uint32_t)) {
-		word = 0;
+bool ReadScalarTable(uint64_t base, uint64_t size, uint32_t dynamic_offset,
+                     const SrtRuntime& runtime, std::span<uint32_t> words) {
+	const auto offset = static_cast<uint64_t>(dynamic_offset) & ~uint64_t {3};
+	const auto count = std::min<uint64_t>(words.size(), offset < size ? (size - offset) / 4u : 0u);
+	std::ranges::fill(words.subspan(count), 0u);
+	if (count == 0u) {
 		return true;
 	}
-	const auto base = descriptor.Base48() & ~uint64_t {3};
-	if (aligned > AddressMask - base) {
+	base &= AddressMask & ~uint64_t {3};
+	if (offset > AddressMask - base) {
 		return false;
 	}
-	const auto address = base + aligned;
-	if (!ReadSpecializationWord(runtime, address, word)) {
-		return false;
-	}
-	return true;
+	const auto address = base + offset;
+	const auto prefix = words.first(count);
+	return prefix.size_bytes() - 1u <= AddressMask - address &&
+	       runtime.read_specialization_memory != nullptr &&
+	       runtime.read_specialization_memory(runtime.userdata, address, prefix);
 }
 
-bool MaterializeIndirectImage(const DescriptorSource::IndirectImage& indirect,
-                              const DescriptorValue&                 material_value,
-                              const DescriptorValue& heap_value, bool r128,
-                              const SrtRuntime& runtime, IndirectImage& result) {
-	ShaderBufferResource material;
-	ShaderBufferResource heap;
-	if (!DecodeBufferDescriptor(material_value, material) ||
-	    !DecodeBufferDescriptor(heap_value, heap)) {
+bool MaterializeIndirectImage(const ResourcePlan& program,
+                              const DescriptorSource::IndirectImage& indirect,
+                              const DescriptorValue& material_value,
+                              const DescriptorValue& table_value, uint32_t image_index,
+                              const SrtRuntime& runtime, ResourceSnapshot& snapshot,
+                              ResourceSpecialization& specialization) {
+	uint64_t table_base = 0;
+	uint64_t table_size = UINT64_MAX; // Scalar addresses have no buffer descriptor bounds.
+	ShaderBufferResource table;
+	if (table_value.dword_count == 2u) {
+		table_base = (static_cast<uint64_t>(table_value.dwords[1]) << 32u) | table_value.dwords[0];
+	} else if (DecodeBufferDescriptor(table_value, table)) {
+		table_base = table.Base48();
+		table_size = table.GetSize();
+	} else {
 		return false;
 	}
-	if (material.Stride() != indirect.selector_stride) {
-		return false;
-	}
-
-	// S_BUFFER_LOAD ignores vector-buffer swizzle/add-thread fields. The shader computes the
-	// record stride explicitly; enumerate every wrapped 32-bit offset that can pass bounds.
-	const auto period      = uint64_t {1} << 32u;
-	const auto step        = std::gcd<uint64_t>(indirect.selector_stride, period);
-	const auto residue     = static_cast<uint64_t>(indirect.selector_offset) % step;
-	const auto size        = material.GetSize();
-	const auto limit       = std::min<uint64_t>(UINT32_MAX, size + 3u);
-	const auto probe_count = residue <= limit ? (limit - residue) / step + 1u : 0u;
-	if (probe_count > MaxIndirectImageProbes) {
-		return false;
-	}
-
-	std::vector<uint32_t>        keys {0u};
-	std::unordered_set<uint32_t> seen {0u};
-	keys.reserve(static_cast<size_t>(probe_count) + 1u);
-	seen.reserve(static_cast<size_t>(probe_count) + 1u);
-	for (uint64_t offset = residue; offset <= limit && probe_count != 0u; offset += step) {
-		uint32_t key = 0;
-		if (!ReadScalarBufferWord(material, static_cast<uint32_t>(offset), 0u, runtime, key)) {
+	auto& keys = program.material_keys;
+	keys.clear();
+	if (indirect.material_source == UINT32_MAX) {
+		if (table_value.dword_count != 2u || indirect.key_count == 0u ||
+		    indirect.key_count > MaxIndirectImageProbes) {
 			return false;
 		}
-		if (seen.insert(key).second) {
+		keys.resize(indirect.key_count);
+		std::iota(keys.begin(), keys.end(), 0u);
+	} else {
+		ShaderBufferResource material;
+		if (!DecodeBufferDescriptor(material_value, material) || table_value.dword_count != 4u ||
+		    material.Stride() != indirect.selector_stride) {
+			return false;
+		}
+		// Enumerate every wrapped scalar-buffer offset that can pass the descriptor bounds.
+		const auto step = std::gcd<uint64_t>(indirect.selector_stride, uint64_t {1} << 32u);
+		const auto residue = static_cast<uint64_t>(indirect.selector_offset) % step;
+		const auto limit = std::min<uint64_t>(UINT32_MAX, material.GetSize() + 3u);
+		const auto probe_count = residue <= limit ? (limit - residue) / step + 1u : 0u;
+		if (probe_count > MaxIndirectImageProbes) {
+			return false;
+		}
+		keys.reserve(static_cast<size_t>(probe_count) + 1u);
+		keys.push_back(0u);
+		for (uint64_t offset = residue; offset <= limit && probe_count != 0u; offset += step) {
+			uint32_t key = 0;
+			if (!ReadScalarTable(material.Base48(), material.GetSize(),
+			                     static_cast<uint32_t>(offset), runtime, {&key, 1})) {
+				return false;
+			}
 			keys.push_back(key);
+			if (limit - offset < step) {
+				break;
+			}
 		}
-		if (limit - offset < step) {
-			break;
-		}
+		std::ranges::sort(keys);
+		keys.erase(std::unique(keys.begin(), keys.end()), keys.end());
 	}
 
-	IndirectImage next;
-	next.keys = std::move(keys);
-	next.candidates.reserve(next.keys.size());
-	next.descriptors.reserve(
-	    std::min(next.keys.size(), static_cast<size_t>(ShaderInfo::MaxImages)));
-	for (const auto key: next.keys) {
+	const auto children_begin = snapshot.images.size();
+	const auto mapping_offset = snapshot.flattened_srt.size();
+	const auto root_image = specialization.images[image_index];
+	snapshot.flattened_srt.resize(mapping_offset + 1u + keys.size() * 2u);
+	snapshot.flattened_srt[mapping_offset] = static_cast<uint32_t>(keys.size());
+	for (uint32_t entry = 0; entry < keys.size(); ++entry) {
+		const auto key = keys[entry];
 		DescriptorValue candidate;
-		candidate.dword_count  = 8u;
-		const auto heap_offset = key << 5u;
-		for (uint32_t dword = 0; dword < candidate.dword_count; dword++) {
-			if (!ReadScalarBufferWord(heap, heap_offset, dword * sizeof(uint32_t), runtime,
-			                          candidate.dwords[dword])) {
-				return false;
-			}
+		candidate.dword_count = 8u;
+		const auto table_offset = (key << 5u) + indirect.table_offset;
+		if (!ReadScalarTable(table_base, table_size, table_offset, runtime, candidate.dwords)) {
+			return false;
 		}
-		if (NullImageDescriptor(candidate) || !ValidImageDescriptor(candidate, r128)) {
+		if (NullImageDescriptor(candidate) ||
+		    !ValidImageDescriptor(candidate, program.info.images[image_index].r128)) {
 			candidate.dwords.fill(0);
 		}
-		const auto found = std::ranges::find(next.descriptors, candidate);
-		if (found == next.descriptors.end()) {
-			if (next.descriptors.size() >= ShaderInfo::MaxImages) {
-				return false;
+		uint32_t ordinal = 0;
+		if (entry == 0) {
+			snapshot.images[image_index] = candidate;
+		} else if (snapshot.images[image_index] != candidate) {
+			const auto found = std::find(snapshot.images.begin() + children_begin,
+			                             snapshot.images.end(), candidate);
+			ordinal = static_cast<uint32_t>(found - snapshot.images.begin() - children_begin + 1u);
+			if (found == snapshot.images.end()) {
+				if (snapshot.images.size() >= ShaderInfo::MaxImages) {
+					return false;
+				}
+				snapshot.images.push_back(candidate);
+				auto child = root_image;
+				child.indirect_root = image_index;
+				specialization.images.push_back(child);
 			}
-			next.descriptors.push_back(candidate);
-			next.candidates.push_back(static_cast<uint32_t>(next.descriptors.size() - 1u));
-		} else {
-			next.candidates.push_back(static_cast<uint32_t>(found - next.descriptors.begin()));
 		}
+		snapshot.flattened_srt[mapping_offset + 1u + entry * 2u] = key;
+		snapshot.flattened_srt[mapping_offset + 2u + entry * 2u] = ordinal;
+	}
+	if (snapshot.images.size() == children_begin) {
+		snapshot.flattened_srt.resize(mapping_offset);
+	} else {
+		auto& root = specialization.images[image_index];
+		root.indirect_root = image_index;
+		root.indirect_mapping_offset = static_cast<uint32_t>(mapping_offset);
+		root.indirect_search_iterations = std::bit_width(keys.size());
 	}
-	result = std::move(next);
 	return true;
 }
 
 } // namespace
 
-static bool MaterializeSnapshot(const ResourcePlan& program, const SrtRuntime& runtime,
-                                MaterializedSnapshot& snapshot) {
-	if (!program.resource_tracking_complete) {
-		return false;
-	}
-
-	if (program.requires_specialization_memory && runtime.read_specialization_memory == nullptr) {
-		return false;
-	}
-	std::vector<DescriptorValue> values;
-	std::vector<uint32_t>        flattened_srt;
-	std::vector<uint8_t>         active_sources;
-	if (!EvaluateRuntimeSources(program, program.materialization_sources, runtime, values,
-	                            flattened_srt, program.clean_flat_slots, active_sources)) {
-		return false;
-	}
-
-	auto&                   next  = snapshot.resources;
-	const auto&             fill  = program.uniform_fill;
-	const auto              words = fill.fill.words;
-	std::array<uint32_t, 4> stored {};
-	if (words != 0 &&
-	    EvaluateUniformValues(program, std::span(fill.values).first(words), runtime,
-	                          std::span(stored).first(words)) &&
-	    std::all_of(stored.begin(), stored.begin() + words,
-	                [&](uint32_t value) { return value == stored[0]; })) {
-		next.uniform_fill       = fill.fill;
-		next.uniform_fill.value = stored[0];
-	}
-	auto cursor = values.begin();
-	next.buffers.assign(cursor, cursor + program.info.buffers.size());
-	cursor += program.info.buffers.size();
-	next.flattened_srt = std::move(flattened_srt);
-	next.images.resize(program.info.images.size());
-	for (uint32_t image_index = 0; image_index < program.info.images.size(); image_index++) {
-		const auto& image  = program.info.images[image_index];
-		const auto* source = Source(program, image.source);
-		if (source != nullptr && source->indirect_image.has_value()) {
-			if (!active_sources[image.source]) {
-				next.images[image_index].dword_count = 8u;
-				continue;
-			}
-			const std::array requests {source->indirect_image->material_source,
-			                           source->indirect_image->heap_source};
-			SrtRuntime       clean_runtime = runtime;
-			clean_runtime.read_memory      = runtime.read_specialization_memory;
-			std::vector<DescriptorValue> tables;
-			if (!EvaluateDescriptorSources(program, requests, clean_runtime, tables)) {
-				return false;
-			}
-			const auto&   material = tables[0];
-			const auto&   heap     = tables[1];
-			IndirectImage table;
-			if (!MaterializeIndirectImage(*source->indirect_image, material, heap, image.r128,
-			                              runtime, table)) {
-				return false;
-			}
-			next.images[image_index] = table.descriptors[table.candidates[0]];
-			if (table.descriptors.size() > 1u) {
-				table.resource = image_index;
-				snapshot.indirect_images.push_back(std::move(table));
-			}
-		} else {
-			auto descriptor = *cursor++;
-			if (!ValidImageDescriptor(descriptor, image.r128)) {
-				descriptor.dwords.fill(0);
-			}
-			next.images[image_index] = descriptor;
-		}
-	}
-	next.samplers.assign(cursor, cursor + program.info.samplers.size());
-	next.user_data.assign(runtime.user_data.begin(), runtime.user_data.end());
-	return true;
-}
-
 struct SamplerPlan {
 	std::array<uint32_t, ShaderInfo::MaxSamplers> point_sampler {};
 	uint32_t                                      sampler_count = 0;
 };
 
 struct ImageRemap {
-	explicit ImageRemap(const ResourceSpecialization& specialization) {
-		for (const auto& image: specialization.images) {
-			indices.push_back(image.fmask ? UINT32_MAX : count++);
+	explicit ImageRemap(const ResourceSpecialization& specialization)
+	    : source_count(static_cast<uint32_t>(specialization.images.size())) {
+		EXIT_IF(specialization.images.size() > indices.size());
+		for (uint32_t index = 0; index < source_count; index++) {
+			indices[index] = specialization.images[index].fmask ? UINT32_MAX : count++;
 		}
 	}
 
+	uint32_t operator[](uint32_t index) const {
+		EXIT_IF(index >= source_count);
+		return indices[index];
+	}
+
 	template <typename T>
 	void Apply(std::vector<T>& images) const {
-		EXIT_IF(images.size() != indices.size());
-		for (uint32_t index = 0; index < indices.size(); index++) {
+		EXIT_IF(images.size() != source_count);
+		for (uint32_t index = 0; index < source_count; index++) {
 			if (indices[index] != UINT32_MAX && indices[index] != index) {
 				images[indices[index]] = std::move(images[index]);
 			}
@@ -374,75 +331,21 @@ struct ImageRemap {
 		images.resize(count);
 	}
 
-	std::vector<uint32_t> indices;
-	uint32_t              count = 0;
+private:
+	std::array<uint32_t, ShaderInfo::MaxImages> indices;
+	uint32_t                                    source_count;
+	uint32_t                                    count = 0;
 };
 
 template <typename Images>
 bool BuildSamplerPlan(const ShaderInfo& base, const Images& images, SamplerPlan& plan);
 
-static bool BuildResourceSpecialization(const ResourcePlan& program, MaterializedSnapshot snapshot,
-                                        ResourceSnapshot&       specialized_snapshot,
+static bool BuildResourceSpecialization(const ResourcePlan& program, ResourceSnapshot& snapshot,
                                         ResourceSpecialization& specialization) {
-	auto                   next_snapshot = std::move(snapshot.resources);
-	ResourceSpecialization next_specialization;
-	next_specialization.buffers.reserve(program.info.buffers.size());
-	size_t image_count   = program.info.images.size();
-	size_t mapping_words = 0;
-	for (const auto& table: snapshot.indirect_images) {
-		if (table.resource >= program.info.images.size() || table.descriptors.size() < 2u ||
-		    image_count + table.descriptors.size() - 1u > ShaderInfo::MaxImages) {
-			return SpecializationFail(
-			    "indirect image candidates exceed the dense image resource limit");
-		}
-		image_count += table.descriptors.size() - 1u;
-		mapping_words += 1u + table.keys.size() * 2u;
-	}
-	next_snapshot.images.reserve(image_count);
-	next_snapshot.flattened_srt.reserve(next_snapshot.flattened_srt.size() + mapping_words);
-	next_specialization.images.reserve(image_count);
-	for (const auto& image: program.info.images) {
-		next_specialization.images.push_back({
-		    .numeric_class              = image.numeric_class,
-		    .dimension                  = image.dimension,
-		    .mip_count                  = image.mip_count,
-		    .conversion_format          = image.conversion_format,
-		    .shader_swizzle             = image.shader_swizzle,
-		    .indirect_root              = image.indirect_root,
-		    .indirect_mapping_offset    = image.indirect_mapping_offset,
-		    .indirect_search_iterations = image.indirect_search_iterations,
-		    .cube                       = image.cube,
-		});
-	}
-	for (const auto& table: snapshot.indirect_images) {
-		const auto root_image = next_specialization.images[table.resource];
-		for (uint32_t candidate = 1; candidate < table.descriptors.size(); candidate++) {
-			auto image          = root_image;
-			image.indirect_root = table.resource;
-			next_specialization.images.push_back(image);
-			next_snapshot.images.push_back(table.descriptors[candidate]);
-		}
-		auto& root                      = next_specialization.images[table.resource];
-		root.indirect_root              = table.resource;
-		root.indirect_mapping_offset    = static_cast<uint32_t>(next_snapshot.flattened_srt.size());
-		root.indirect_search_iterations = std::bit_width(table.keys.size());
-		next_snapshot.flattened_srt.resize(next_snapshot.flattened_srt.size() + 1u +
-		                                   table.keys.size() * 2u);
-		std::vector<uint32_t> order(table.keys.size());
-		std::iota(order.begin(), order.end(), 0u);
-		std::ranges::sort(order, {}, [&](uint32_t index) { return table.keys[index]; });
-		next_snapshot.flattened_srt[root.indirect_mapping_offset] =
-		    static_cast<uint32_t>(table.keys.size());
-		for (uint32_t entry = 0; entry < order.size(); entry++) {
-			const auto source                   = order[entry];
-			const auto offset                   = root.indirect_mapping_offset + 1u + entry * 2u;
-			next_snapshot.flattened_srt[offset] = table.keys[source];
-			next_snapshot.flattened_srt[offset + 1] = table.candidates[source];
-		}
-		next_snapshot.images[table.resource] = table.descriptors[0];
-	}
+	specialization.buffers.clear();
+	specialization.buffers.reserve(program.info.buffers.size());
 	for (uint32_t i = 0; i < program.info.buffers.size(); i++) {
-		auto&                descriptor_value = next_snapshot.buffers[i];
+		auto&                descriptor_value = snapshot.buffers[i];
 		ShaderBufferResource descriptor;
 		if (!DecodeBufferDescriptor(descriptor_value, descriptor)) {
 			return SpecializationFail(fmt::format("buffer descriptor {} has invalid width", i));
@@ -459,7 +362,7 @@ static bool BuildResourceSpecialization(const ResourcePlan& program, Materialize
 		} else if (!swizzle) {
 			packed_stride &= ~(3u << 16u);
 		}
-		next_specialization.buffers.push_back({
+		specialization.buffers.push_back({
 		    .packed_stride     = packed_stride,
 		    .descriptor_format = program.info.buffers[i].formatted
 		                             ? descriptor.Format()
@@ -468,9 +371,9 @@ static bool BuildResourceSpecialization(const ResourcePlan& program, Materialize
 		        program.info.buffers[i].formatted ? descriptor.DstSelXYZW() : DstSel(4, 5, 6, 7),
 		});
 	}
-	for (uint32_t i = 0; i < next_specialization.images.size(); i++) {
-		const auto& descriptor = next_snapshot.images[i];
-		auto&       image      = next_specialization.images[i];
+	for (uint32_t i = 0; i < specialization.images.size(); i++) {
+		const auto& descriptor = snapshot.images[i];
+		auto&       image      = specialization.images[i];
 		const auto  base_index = i < program.info.images.size() ? i : image.indirect_root;
 		if (base_index >= program.info.images.size()) {
 			return SpecializationFail(fmt::format("image resource {} has an invalid root", i));
@@ -545,42 +448,46 @@ static bool BuildResourceSpecialization(const ResourcePlan& program, Materialize
 			                static_cast<uint32_t>(format)));
 		}
 	}
-	for (uint32_t root_index = 0; root_index < next_specialization.images.size(); root_index++) {
-		auto& root = next_specialization.images[root_index];
+	for (uint32_t root_index = 0; root_index < specialization.images.size(); root_index++) {
+		auto& root = specialization.images[root_index];
 		if (root.indirect_root != root_index) {
 			continue;
 		}
-		const auto key_count = root.indirect_mapping_offset < next_snapshot.flattened_srt.size()
-		                           ? next_snapshot.flattened_srt[root.indirect_mapping_offset]
+		const auto key_count = root.indirect_mapping_offset < snapshot.flattened_srt.size()
+		                           ? snapshot.flattened_srt[root.indirect_mapping_offset]
 		                           : 0u;
 		if (root.indirect_search_iterations == 0u || key_count < 2u ||
 		    static_cast<size_t>(root.indirect_mapping_offset) + 1u +
 		            static_cast<size_t>(key_count) * 2u >
-		        next_snapshot.flattened_srt.size()) {
+		        snapshot.flattened_srt.size()) {
 			return SpecializationFail("indirect image specialization has an invalid key mapping");
 		}
 		uint32_t exemplar       = ImageResource::NoIndirectImage;
 		uint32_t resource_count = 0;
-		for (uint32_t resource = 0; resource < next_specialization.images.size(); resource++) {
-			if (next_specialization.images[resource].indirect_root != root_index) {
+		for (uint32_t resource = 0; resource < specialization.images.size(); resource++) {
+			if (specialization.images[resource].indirect_root != root_index) {
 				continue;
 			}
 			resource_count++;
 			if (exemplar == ImageResource::NoIndirectImage &&
-			    !NullImageDescriptor(next_snapshot.images[resource])) {
+			    !NullImageDescriptor(snapshot.images[resource])) {
 				exemplar = resource;
 			}
 		}
 		if (resource_count < 2u || exemplar == ImageResource::NoIndirectImage) {
 			return SpecializationFail("indirect image specialization has no typed candidate");
 		}
-		const auto& image_class = next_specialization.images[exemplar];
-		for (uint32_t candidate = 0; candidate < next_specialization.images.size(); candidate++) {
-			auto& image = next_specialization.images[candidate];
+		const auto& image_class = specialization.images[exemplar];
+		const auto is_2d = [](Decoder::ImageDimension dimension) {
+			return dimension == Decoder::ImageDimension::Dim2D ||
+			       dimension == Decoder::ImageDimension::Dim2DArray;
+		};
+		for (uint32_t candidate = 0; candidate < specialization.images.size(); candidate++) {
+			auto& image = specialization.images[candidate];
 			if (image.indirect_root != root_index) {
 				continue;
 			}
-			if (NullImageDescriptor(next_snapshot.images[candidate])) {
+			if (NullImageDescriptor(snapshot.images[candidate])) {
 				image.numeric_class     = image_class.numeric_class;
 				image.dimension         = image_class.dimension;
 				image.mip_count         = image_class.mip_count;
@@ -588,12 +495,13 @@ static bool BuildResourceSpecialization(const ResourcePlan& program, Materialize
 				image.shader_swizzle    = image_class.shader_swizzle;
 				image.cube              = image_class.cube;
 			}
+			const bool same_coordinates = image.dimension == image_class.dimension &&
+			                              image.cube == image_class.cube;
 			if (image.numeric_class != image_class.numeric_class ||
-			    image.dimension != image_class.dimension ||
+			    (!same_coordinates && !(is_2d(image.dimension) && is_2d(image_class.dimension))) ||
 			    image.mip_count != image_class.mip_count ||
 			    image.conversion_format != image_class.conversion_format ||
-			    image.shader_swizzle != image_class.shader_swizzle ||
-			    image.cube != image_class.cube) {
+			    image.shader_swizzle != image_class.shader_swizzle) {
 				return SpecializationFail(
 				    fmt::format("indirect image table at pc 0x{:08x} has incompatible candidates",
 				                program.info.images[root_index].first_use_pc));
@@ -601,18 +509,16 @@ static bool BuildResourceSpecialization(const ResourcePlan& program, Materialize
 		}
 	}
 	SamplerPlan sampler_plan;
-	if (!BuildSamplerPlan(program.info, next_specialization.images, sampler_plan)) {
+	if (!BuildSamplerPlan(program.info, specialization.images, sampler_plan)) {
 		return SpecializationFail("specialized sampler layout exceeds its resource limit");
 	}
 	for (uint32_t index = 0; index < program.info.samplers.size(); index++) {
 		const auto target = sampler_plan.point_sampler[index];
 		if (target != UINT32_MAX && target >= program.info.samplers.size()) {
-			next_snapshot.samplers.push_back(next_snapshot.samplers[index]);
+			snapshot.samplers.push_back(snapshot.samplers[index]);
 		}
 	}
-	ImageRemap(next_specialization).Apply(next_snapshot.images);
-	specialization       = std::move(next_specialization);
-	specialized_snapshot = std::move(next_snapshot);
+	ImageRemap(specialization).Apply(snapshot.images);
 	return true;
 }
 
@@ -926,31 +832,16 @@ ResourcePlan ExtractResourcePlan(const Program& program) {
 	for (uint32_t i = 0; i < plan.uniform_fill.fill.words; ++i) {
 		plan.uniform_fill.values[i] = Clone(plan.uniform_fill.values[i]);
 	}
-	plan.materialization_sources.reserve(plan.info.buffers.size() + plan.info.images.size() +
-	                                     plan.info.samplers.size());
-	for (const auto& buffer: plan.info.buffers) {
-		plan.materialization_sources.push_back(buffer.source);
-	}
-	for (const auto& image: plan.info.images) {
-		const auto* source = Source(plan, image.source);
-		if (source != nullptr && source->indirect_image.has_value()) {
-			plan.requires_specialization_memory = true;
-		} else {
-			plan.materialization_sources.push_back(image.source);
-		}
-	}
-	for (const auto& sampler: plan.info.samplers) {
-		plan.materialization_sources.push_back(sampler.source);
-	}
 	plan.clean_flat_slots.resize(plan.srt_reads.size());
 	for (const auto& image: plan.info.images) {
 		const auto* source = Source(plan, image.source);
 		if (source == nullptr || !source->indirect_image.has_value()) {
 			continue;
 		}
+		plan.requires_specialization_memory = true;
 		MarkCleanFlatSlots(plan, Source(plan, source->indirect_image->material_source),
 		                   plan.clean_flat_slots);
-		MarkCleanFlatSlots(plan, Source(plan, source->indirect_image->heap_source),
+		MarkCleanFlatSlots(plan, Source(plan, source->indirect_image->table_source),
 		                   plan.clean_flat_slots);
 	}
 	return plan;
@@ -958,11 +849,99 @@ ResourcePlan ExtractResourcePlan(const Program& program) {
 
 bool MaterializeResources(const ResourcePlan& program, const SrtRuntime& runtime,
                           ResourceSnapshot& snapshot, ResourceSpecialization& specialization) {
-	MaterializedSnapshot materialized;
-	if (!MaterializeSnapshot(program, runtime, materialized)) {
+	if (!program.resource_tracking_complete ||
+	    (program.requires_specialization_memory && runtime.read_specialization_memory == nullptr)) {
 		return false;
 	}
-	return BuildResourceSpecialization(program, std::move(materialized), snapshot, specialization);
+	SrtWalker clean(program, CleanRuntime(runtime));
+	SrtWalker walker(program, runtime, program.clean_flat_slots, &clean);
+	const auto active = clean.FindActiveSources();
+	if (!walker.RefreshFlatBuffer(snapshot.flattened_srt)) {
+		return false;
+	}
+	snapshot.uniform_fill = {};
+	const auto& fill = program.uniform_fill;
+	const auto words = fill.fill.words;
+	std::array<uint32_t, 4> stored {};
+	bool uniform_fill = words != 0;
+	for (uint32_t i = 0; i < words && uniform_fill; ++i) {
+		uniform_fill = clean.Evaluate(fill.values[i], stored[i]) && stored[i] == stored[0];
+	}
+	if (uniform_fill) {
+		snapshot.uniform_fill = fill.fill;
+		snapshot.uniform_fill.value = stored[0];
+	}
+	const auto evaluate = [&](uint32_t source, DescriptorValue& value) {
+		if (source >= program.descriptor_sources.size()) {
+			return false;
+		}
+		if (active.empty() || active[source]) {
+			return walker.EvaluateDescriptor(source, value);
+		}
+		value = {};
+		value.dword_count = program.descriptor_sources[source].dword_count;
+		return true;
+	};
+	snapshot.buffers.resize(program.info.buffers.size());
+	for (uint32_t i = 0; i < program.info.buffers.size(); ++i) {
+		if (!evaluate(program.info.buffers[i].source, snapshot.buffers[i])) {
+			return false;
+		}
+	}
+	snapshot.images.resize(program.info.images.size());
+	specialization.images.clear();
+	specialization.images.reserve(program.info.images.size());
+	for (const auto& image: program.info.images) {
+		specialization.images.push_back({
+		    .numeric_class = image.numeric_class,
+		    .dimension = image.dimension,
+		    .mip_count = image.mip_count,
+		    .conversion_format = image.conversion_format,
+		    .shader_swizzle = image.shader_swizzle,
+		    .indirect_root = image.indirect_root,
+		    .indirect_mapping_offset = image.indirect_mapping_offset,
+		    .indirect_search_iterations = image.indirect_search_iterations,
+		    .cube = image.cube,
+		});
+	}
+	for (uint32_t i = 0; i < program.info.images.size(); ++i) {
+		const auto& image = program.info.images[i];
+		const auto* source = Source(program, image.source);
+		if (source == nullptr) {
+			return false;
+		}
+		if (source->indirect_image.has_value()) {
+			snapshot.images[i] = {.dword_count = 8u};
+			if (!active.empty() && !active[image.source]) {
+				continue;
+			}
+			const auto& indirect = *source->indirect_image;
+			DescriptorValue material;
+			DescriptorValue table;
+			if ((indirect.material_source != UINT32_MAX &&
+			     !clean.EvaluateDescriptor(indirect.material_source, material)) ||
+			    !clean.EvaluateDescriptor(indirect.table_source, table) ||
+			    !MaterializeIndirectImage(program, indirect, material, table, i, runtime, snapshot,
+			                              specialization)) {
+				return false;
+			}
+		} else {
+			if (!evaluate(image.source, snapshot.images[i])) {
+				return false;
+			}
+			if (!ValidImageDescriptor(snapshot.images[i], image.r128)) {
+				snapshot.images[i].dwords.fill(0);
+			}
+		}
+	}
+	snapshot.samplers.resize(program.info.samplers.size());
+	for (uint32_t i = 0; i < program.info.samplers.size(); ++i) {
+		if (!evaluate(program.info.samplers[i].source, snapshot.samplers[i])) {
+			return false;
+		}
+	}
+	snapshot.user_data.assign(runtime.user_data.begin(), runtime.user_data.end());
+	return BuildResourceSpecialization(program, snapshot, specialization);
 }
 
 void ApplyResourceSpecialization(Program& program, const ResourceSpecialization& specialization) {
@@ -1075,29 +1054,29 @@ void ApplyResourceSpecialization(Program& program, const ResourceSpecialization&
 	for (auto* block: program.blocks) {
 		for (auto& inst: *block) {
 			if (inst.GetOpcode() == ValueOpcode::GetImageResource) {
-				inst.SetFlags(image_remap.indices.at(inst.Flags<uint32_t>()));
+				inst.SetFlags(image_remap[inst.Flags<uint32_t>()]);
 			}
 		}
 	}
 	for (auto& memory: memory_info) {
 		if (memory.kind == ResourceKind::Image && !memory.planning_only) {
-			memory.resource = image_remap.indices.at(memory.resource);
+			memory.resource = image_remap[memory.resource];
 		}
 	}
 	for (auto& buffer: buffers) {
 		if (buffer.image_alias != BufferResource::NoImageAlias) {
-			buffer.image_alias = image_remap.indices.at(buffer.image_alias);
+			buffer.image_alias = image_remap[buffer.image_alias];
 		}
 	}
 	for (auto& pair: sampled_pairs) {
-		pair.image = image_remap.indices.at(pair.image);
+		pair.image = image_remap[pair.image];
 	}
 	for (auto& image: images) {
 		if (image.indirect_root != ImageResource::NoIndirectImage) {
-			image.indirect_root = image_remap.indices.at(image.indirect_root);
+			image.indirect_root = image_remap[image.indirect_root];
 		}
 		for (auto& resource: image.indirect_resources) {
-			resource = image_remap.indices.at(resource);
+			resource = image_remap[resource];
 		}
 	}
 	image_remap.Apply(images);

src/graphics/shader/recompiler/ir/passes/ResourceMaterialization.h

@@ -39,8 +39,7 @@ struct ResourceSpecialization {
 // its values and is independent of the translated shader CFG.
 ResourcePlan ExtractResourcePlan(const Program& program);
 
-// Resolves and specializes the immutable resource plan in one transaction. On failure both
-// destinations are unchanged.
+// Refreshes cached resources and specialization in place. A failed refresh must not be used.
 bool MaterializeResources(const ResourcePlan& program, const SrtRuntime& runtime,
                           ResourceSnapshot& snapshot, ResourceSpecialization& specialization);
 

src/graphics/shader/recompiler/ir/passes/ResourceTracking.cpp

@@ -169,7 +169,7 @@ private:
 		std::abort();
 	}
 
-	Value LowerDescriptorPhi(Value value) {
+	Value LowerDescriptorPhi(Value value, const Block* use) {
 		value           = value.Resolve();
 		const auto* phi = value.TryInstruction();
 		if (m_shader_writes || phi == nullptr || phi->GetOpcode() != ValueOpcode::Phi ||
@@ -177,17 +177,53 @@ private:
 		    m_program.blocks.size() != m_program.block_info.size()) {
 			return value;
 		}
-		for (const auto& [original, selected]: m_descriptor_selections) {
-			if (original == phi) {
-				return selected;
-			}
-		}
-		const auto* merge  = phi->Parent();
+		const auto* merge = phi->Parent();
 		const auto* branch = phi->PhiBlock(0);
 		if (merge == nullptr || branch == nullptr || phi->PhiBlock(1) == nullptr ||
 		    branch == phi->PhiBlock(1)) {
 			return value;
 		}
+		// Structurization can merge the descriptor and its use predicate in parallel Phis.
+		// Match their incoming blocks to exclude only edges that cannot reach this use.
+		if (use != nullptr && use->ImmPredecessors().size() == 1u &&
+		    use->ImmPredecessors()[0] == merge) {
+			const auto merge_it = std::ranges::find(m_program.blocks, merge);
+			const auto use_it   = std::ranges::find(m_program.blocks, use);
+			if (merge_it != m_program.blocks.end() && use_it != m_program.blocks.end()) {
+				const auto& info = m_program.block_info[merge_it - m_program.blocks.begin()];
+				const auto& term = info.terminator;
+				const auto id    = m_program.block_info[use_it - m_program.blocks.begin()].id;
+				const auto* condition = info.condition.Resolve().TryInstruction();
+				if (term.kind == CFG::TerminatorKind::ConditionalBranch &&
+				    term.true_block != term.false_block &&
+				    (id == term.true_block || id == term.false_block) && condition != nullptr &&
+				    condition->GetOpcode() == ValueOpcode::Phi && condition->GetType() == Type::U1 &&
+				    condition->Parent() == merge && condition->NumArgs() == 2u &&
+				    condition->NumPhiBlocks() == 2u) {
+					const bool taken = id == term.true_block;
+					for (uint32_t skipped = 0; skipped < 2u; skipped++) {
+						const auto excluded = condition->Arg(skipped).Resolve();
+						const auto included = condition->Arg(skipped ^ 1u).Resolve();
+						if (!excluded.IsImmediate() || excluded.GetType() != Type::U1 ||
+						    excluded.U1() == taken ||
+						    (included.IsImmediate() && included.U1() != taken)) {
+							continue;
+						}
+						for (uint32_t selected = 0; selected < 2u; selected++) {
+							if (phi->PhiBlock(selected) == condition->PhiBlock(skipped ^ 1u) &&
+							    phi->PhiBlock(selected ^ 1u) == condition->PhiBlock(skipped)) {
+								return phi->Arg(selected);
+							}
+						}
+					}
+				}
+			}
+		}
+		for (const auto& [original, selected]: m_descriptor_selections) {
+			if (original == phi) {
+				return selected;
+			}
+		}
 		if (branch->ImmSuccessors().size() != 2u) {
 			if (branch->ImmPredecessors().size() != 1u) {
 				return value;
@@ -247,7 +283,7 @@ private:
 		}
 		descriptor.dword_count = width;
 		for (uint32_t i = 0; i < width; i++) {
-			descriptor.dwords[i] = LowerDescriptorPhi(handle.Arg(i));
+			descriptor.dwords[i] = LowerDescriptorPhi(handle.Arg(i), handle.Parent());
 		}
 		if (sample_adjust) {
 			descriptor.dwords[3] = CanonicalizeSampleAdjustDword3(descriptor.dwords[3]);
@@ -308,16 +344,26 @@ private:
 	}
 
 	const MemoryInfo* ScalarReadMemory(const Inst& read, uint32_t& index) const {
-		if (read.GetOpcode() != ValueOpcode::ReadConstBuffer || read.NumArgs() != 2u) {
+		const bool address = read.GetOpcode() == ValueOpcode::LoadAddressU32;
+		if (!(address ? read.NumArgs() == 4u
+		              : read.GetOpcode() == ValueOpcode::ReadConstBuffer && read.NumArgs() == 2u)) {
 			return nullptr;
 		}
+		if (address) {
+			const auto high = read.Arg(2).Resolve();
+			const auto enabled = read.Arg(3).Resolve();
+			if (!high.IsImmediate() || high.GetType() != Type::U32 || high.U32() != 0u ||
+			    !enabled.IsImmediate() || enabled.GetType() != Type::U1 || !enabled.U1()) {
+				return nullptr;
+			}
+		}
 		index = read.Flags<MemoryFlags>().index;
 		if (index >= m_program.memory_info.size()) {
 			return nullptr;
 		}
 		const auto& memory = m_program.memory_info[index];
-		return memory.kind == ResourceKind::ScalarBuffer && memory.data_bits == 32u &&
-		               memory.data_dwords == 1u
+		return memory.kind == (address ? ResourceKind::ScalarAddress : ResourceKind::ScalarBuffer) &&
+		               memory.data_bits == 32u && memory.data_dwords == 1u
 		           ? &memory
 		           : nullptr;
 	}
@@ -340,18 +386,16 @@ private:
 		return true;
 	}
 
-	bool MakeRuntimeBufferSource(const Inst& handle, uint32_t pc, uint32_t& source,
-	                             DescriptorSource& descriptor) {
-		if (handle.GetOpcode() != ValueOpcode::GetBufferResource) {
+	bool MakeRuntimeTableSource(const Inst& handle, uint32_t pc,
+	                            DescriptorSource& descriptor) {
+		const auto width = handle.GetOpcode() == ValueOpcode::GetBufferResource ? 4u
+		                 : handle.GetOpcode() == ValueOpcode::GetAddressResource ? 2u : 0u;
+		if (width == 0u) {
 			return false;
 		}
-		MakeSource(handle, 4u, false, false, descriptor, pc);
+		MakeSource(handle, width, false, false, descriptor, pc);
 		uint32_t bad_dword = 0;
-		if (!ValidateSource(descriptor, bad_dword)) {
-			return false;
-		}
-		source = InternSource(descriptor);
-		return true;
+		return ValidateSource(descriptor, bad_dword);
 	}
 
 	bool MatchMaterialOffset(Value value, Value& selector, uint32_t& stride,
@@ -388,106 +432,183 @@ private:
 		       selector_inst->GetOpcode() == ValueOpcode::ReadFirstLane;
 	}
 
+	bool NonzeroOnEntry(Value value, const Block* block) const {
+		if (m_program.blocks.size() != m_program.block_info.size()) {
+			return false;
+		}
+		// Each unique predecessor must execute before this use. Stop at joins: an
+		// unrelated comparison is not a bound on FindILsb's zero-input sentinel.
+		for (size_t depth = 0; block != nullptr && depth < m_program.blocks.size(); ++depth) {
+			if (block->ImmPredecessors().size() != 1u) {
+				return false;
+			}
+			const auto* previous = block->ImmPredecessors()[0];
+			const auto current_it = std::ranges::find(m_program.blocks, block);
+			const auto previous_it = std::ranges::find(m_program.blocks, previous);
+			if (current_it == m_program.blocks.end() || previous_it == m_program.blocks.end()) {
+				return false;
+			}
+			const auto& info = m_program.block_info[previous_it - m_program.blocks.begin()];
+			const auto id = m_program.block_info[current_it - m_program.blocks.begin()].id;
+			const auto& term = info.terminator;
+			if (term.kind == CFG::TerminatorKind::ConditionalBranch &&
+			    (term.true_block == id) != (term.false_block == id)) {
+				bool positive = term.true_block == id;
+				const auto* test = info.condition.Resolve().TryInstruction();
+				while (test != nullptr && test->GetOpcode() == ValueOpcode::LogicalNot) {
+					positive = !positive;
+					test = test->Arg(0).Resolve().TryInstruction();
+				}
+				if (test != nullptr && test->NumArgs() == 2u &&
+				    ((test->GetOpcode() == ValueOpcode::INotEqual32 && positive) ||
+				     (test->GetOpcode() == ValueOpcode::IEqual32 && !positive))) {
+					for (uint32_t arg = 0; arg < 2u; ++arg) {
+						uint32_t immediate;
+						if (ImmediateU32(test->Arg(arg), immediate) && immediate == 0u &&
+						    EquivalentValue(m_program, test->Arg(arg ^ 1u), value)) {
+							return true;
+						}
+					}
+				}
+			}
+			block = previous;
+		}
+		return false;
+	}
+
+	bool MatchTableOffset(Value value, Value& key, uint32_t& offset) const {
+		offset = 0;
+		for (;;) {
+			const auto* inst = value.Resolve().TryInstruction();
+			if (inst == nullptr || inst->NumArgs() != 2u) {
+				return false;
+			}
+			uint32_t immediate;
+			if (inst->GetOpcode() == ValueOpcode::ShiftLeftLogical32 &&
+			    ImmediateU32(inst->Arg(1), immediate) && immediate == 5u) {
+				key = inst->Arg(0).Resolve();
+				return key.GetType() == Type::U32;
+			}
+			if (inst->GetOpcode() != ValueOpcode::IAdd32) {
+				return false;
+			}
+			if (ImmediateU32(inst->Arg(0), immediate)) {
+				value = inst->Arg(1);
+			} else if (ImmediateU32(inst->Arg(1), immediate)) {
+				value = inst->Arg(0);
+			} else {
+				return false;
+			}
+			// These additions are shader U32 arithmetic, before the scalar memory offset.
+			offset += immediate;
+		}
+	}
+
 	bool TryMakeIndirectImage(Inst& handle, uint32_t pc, IndirectImagePlan& plan) {
 		if (handle.GetOpcode() != ValueOpcode::GetImageResource || handle.NumArgs() != 8u) {
 			return false;
 		}
-
-		std::array<Inst*, 8> heap_reads {};
-		Inst*                heap_handle = nullptr;
-		Value                heap_offset;
-		for (uint32_t dword = 0; dword < heap_reads.size(); dword++) {
-			heap_reads[dword] = handle.Arg(dword).Resolve().TryInstruction();
-			if (heap_reads[dword] == nullptr) {
+		Inst* table_handle = nullptr;
+		Value key;
+		uint32_t table_offset = 0;
+		for (uint32_t dword = 0; dword < plan.reads.size(); ++dword) {
+			auto* read = handle.Arg(dword).Resolve().TryInstruction();
+			if (read == nullptr) {
 				return false;
 			}
-			uint32_t    memory_index = 0;
-			const auto* memory       = ScalarReadMemory(*heap_reads[dword], memory_index);
-			if (memory == nullptr || memory->offset != dword * sizeof(uint32_t) ||
-			    !MemoryIndexBelongsTo(memory_index, *heap_reads[dword])) {
+			uint32_t memory_index = 0;
+			const auto* memory = ScalarReadMemory(*read, memory_index);
+			if (memory == nullptr || memory->offset > INT32_MAX || (memory->offset & 3u) != 0u ||
+			    !MemoryIndexBelongsTo(memory_index, *read)) {
 				return false;
 			}
-			auto* current_handle = heap_reads[dword]->Arg(0).Resolve().TryInstruction();
+			auto* current_handle = read->Arg(0).Resolve().TryInstruction();
+			Value current_key;
+			uint32_t offset = 0;
 			if (current_handle == nullptr ||
-			    (heap_handle != nullptr && current_handle != heap_handle)) {
+			    current_handle->GetOpcode() != (memory->kind == ResourceKind::ScalarAddress
+			                                      ? ValueOpcode::GetAddressResource
+			                                      : ValueOpcode::GetBufferResource) ||
+			    (memory->kind == ResourceKind::ScalarAddress && read->Parent() != handle.Parent()) ||
+			    (table_handle != nullptr &&
+			     !EquivalentValue(m_program, Value(table_handle), Value(current_handle))) ||
+			    !MatchTableOffset(read->Arg(1), current_key, offset) ||
+			    memory->offset > UINT32_MAX - offset) {
 				return false;
 			}
-			heap_handle = current_handle;
+			offset += memory->offset;
 			if (dword == 0u) {
-				heap_offset = heap_reads[dword]->Arg(1).Resolve();
-			} else if (!EquivalentValue(m_program, heap_offset, heap_reads[dword]->Arg(1))) {
+				key = current_key;
+				table_offset = offset;
+			} else if (!EquivalentValue(m_program, key, current_key) ||
+			           static_cast<uint64_t>(table_offset) + dword * sizeof(uint32_t) != offset) {
 				return false;
 			}
-			plan.memory[dword] = memory_index;
-			plan.reads[dword]  = heap_reads[dword];
-		}
-
-		const auto* shift        = heap_offset.TryInstruction();
-		uint32_t    shift_amount = 0;
-		if (shift == nullptr || shift->GetOpcode() != ValueOpcode::ShiftLeftLogical32 ||
-		    shift->NumArgs() != 2u || !ImmediateU32(shift->Arg(1), shift_amount) ||
-		    shift_amount != 5u) {
-			return false;
-		}
-		auto* material_read = shift->Arg(0).Resolve().TryInstruction();
-		if (material_read == nullptr) {
-			return false;
-		}
-		uint32_t    material_memory_index = 0;
-		const auto* material_memory       = ScalarReadMemory(*material_read, material_memory_index);
-		if (material_memory == nullptr || material_memory->offset != 0u ||
-		    !MemoryIndexBelongsTo(material_memory_index, *material_read)) {
-			return false;
-		}
-		auto* material_handle = material_read->Arg(0).Resolve().TryInstruction();
-		if (material_handle == nullptr) {
-			return false;
-		}
-
-		Value    selector;
-		uint32_t selector_stride = 0;
-		uint32_t selector_offset = 0;
-		if (!MatchMaterialOffset(material_read->Arg(1), selector, selector_stride,
-		                         selector_offset)) {
-			return false;
-		}
-
-		const std::array<const Inst*, 1> material_users {shift};
-		std::array<const Inst*, 8>       heap_users {};
-		std::copy(heap_reads.begin(), heap_reads.end(), heap_users.begin());
-		const std::array<const Inst*, 1> image_users {&handle};
-		if (!UsesOnly(*material_read, material_users) || !UsesOnly(*shift, heap_users)) {
-			return false;
-		}
-		for (const auto* read: heap_reads) {
+			table_handle = current_handle;
+			const std::array<const Inst*, 1> image_users {&handle};
 			if (!UsesOnly(*read, image_users)) {
 				return false;
 			}
+			plan.memory[dword] = memory_index;
+			plan.reads[dword] = read;
 		}
 
-		DescriptorSource material_source;
-		DescriptorSource heap_source;
-		uint32_t         material_source_index = 0;
-		uint32_t         heap_source_index     = 0;
-		if (!MakeRuntimeBufferSource(*material_handle, pc, material_source_index,
-		                             material_source) ||
-		    !MakeRuntimeBufferSource(*heap_handle, pc, heap_source_index, heap_source)) {
+		DescriptorSource table_source;
+		if (!MakeRuntimeTableSource(*table_handle, pc, table_source)) {
 			return false;
 		}
-
+		DescriptorSource material_source;
+		DescriptorSource::IndirectImage indirect;
+		indirect.table_offset = table_offset;
+		if (table_source.dword_count == 2u) {
+			const auto* selector = key.Resolve().TryInstruction();
+			if (m_shader_writes || selector == nullptr ||
+			    selector->GetOpcode() != ValueOpcode::FindILsb32 || selector->NumArgs() != 1u ||
+			    !NonzeroOnEntry(selector->Arg(0), handle.Parent()) ||
+			    (table_offset & 3u) != 0u || table_offset > UINT32_MAX - (32u * 32u - 1u)) {
+				return false;
+			}
+			indirect.key_count = 32u;
+		} else {
+			auto* material_read = key.Resolve().TryInstruction();
+			uint32_t material_memory_index = 0;
+			const auto* memory = material_read != nullptr
+			                         ? ScalarReadMemory(*material_read, material_memory_index) : nullptr;
+			if (table_offset != 0u || memory == nullptr || memory->kind != ResourceKind::ScalarBuffer ||
+			    memory->offset != 0u || !MemoryIndexBelongsTo(material_memory_index, *material_read)) {
+				return false;
+			}
+			Value selector;
+			if (!MatchMaterialOffset(material_read->Arg(1), selector, indirect.selector_stride,
+			                         indirect.selector_offset)) {
+				return false;
+			}
+			const auto* shift = plan.reads[0]->Arg(1).Resolve().TryInstruction();
+			const std::array<const Inst*, 1> material_users {shift};
+			if (!UsesOnly(*material_read, material_users) || !UsesOnly(*shift, plan.reads)) {
+				return false;
+			}
+			const auto* material_handle = material_read->Arg(0).Resolve().TryInstruction();
+			if (material_handle == nullptr ||
+			    material_handle->GetOpcode() != ValueOpcode::GetBufferResource ||
+			    !MakeRuntimeTableSource(*material_handle, pc, material_source)) {
+				return false;
+			}
+			indirect.material_source = InternSource(material_source);
+		}
+		indirect.table_source = InternSource(table_source);
 		DescriptorSource image_source;
 		image_source.dword_count = 8u;
-		std::copy(material_source.dwords.begin(), material_source.dwords.begin() + 4u,
-		          image_source.dwords.begin());
-		std::copy(heap_source.dwords.begin(), heap_source.dwords.begin() + 4u,
-		          image_source.dwords.begin() + 4u);
-		image_source.indirect_image = DescriptorSource::IndirectImage {
-		    material_source_index, heap_source_index, selector_stride, selector_offset, 0u};
-
+		image_source.dwords.fill(Value(0u));
+		std::copy_n(material_source.dwords.begin(), material_source.dword_count,
+		            image_source.dwords.begin());
+		std::copy_n(table_source.dwords.begin(), table_source.dword_count,
+		            image_source.dwords.begin() + 4u);
+		image_source.indirect_image = indirect;
 		plan.handle = &handle;
 		plan.source = InternSource(image_source);
-		plan.key    = Value(material_read);
-		plan.roots  = image_source.dwords;
+		plan.key = key;
+		plan.roots = image_source.dwords;
 		return true;
 	}
 
@@ -526,7 +647,7 @@ private:
 		}
 	}
 
-	void GetHandle(Value value, ValueOpcode expected, uint32_t width, uint32_t pc, Inst*& handle,
+	bool GetHandle(Value value, ValueOpcode expected, uint32_t width, uint32_t pc, Inst*& handle,
 	               uint32_t& source, bool sampler = false, bool sample_adjust = false) {
 		handle = value.Resolve().TryInstruction();
 		if (handle == nullptr || handle->GetOpcode() != expected) {
@@ -535,21 +656,17 @@ private:
 		DescriptorSource descriptor;
 		MakeSource(*handle, width, sampler, sample_adjust, descriptor, pc);
 		uint32_t bad_dword = 0;
-		if (expected == ValueOpcode::GetImageResource) {
-			for (; bad_dword < descriptor.dword_count; bad_dword++) {
-				const auto* value = descriptor.dwords[bad_dword].Resolve().TryInstruction();
-				if (value != nullptr && value->GetOpcode() == ValueOpcode::ReadConstBuffer) {
-					Fail(pc, fmt::format("{} dword {} is not a valid runtime value",
-					                     ValueOpcodeName(expected), bad_dword));
-				}
-			}
-			bad_dword = 0;
-		}
 		if (!ValidateSource(descriptor, bad_dword)) {
+			if (expected == ValueOpcode::GetBufferResource &&
+			    std::all_of(descriptor.dwords.begin(), descriptor.dwords.begin() + width,
+			                [](Value word) { return word.Resolve().GetType() == Type::U32; })) {
+				return false;
+			}
 			Fail(pc, fmt::format("{} dword {} is not a valid runtime value",
 			                     ValueOpcodeName(expected), bad_dword));
 		}
 		source = InternSource(descriptor);
+		return true;
 	}
 
 	void ValidateAddressHandle(Value value, uint32_t pc) const {
@@ -720,7 +837,18 @@ private:
 		uint32_t resource = 0;
 
 		if (buffer != BufferAccess::None) {
-			GetHandle(inst.Arg(0), ValueOpcode::GetBufferResource, 4, flags.pc, handle, source);
+			if (!GetHandle(inst.Arg(0), ValueOpcode::GetBufferResource, 4, flags.pc, handle,
+			               source)) {
+				if (memory.kind != ResourceKind::Buffer || memory.formatted || memory.typed ||
+				    (op != ValueOpcode::LoadBufferU32x2 && op != ValueOpcode::LoadBufferU32x4)) {
+					Fail(flags.pc,
+					     "buffer descriptor is not a valid runtime value; GPU-selected access "
+					     "requires a raw DWORD x2/x4 load");
+				}
+				m_program.memory_info[flags.index].kind = ResourceKind::IndirectBuffer;
+				m_info.uses_dma                         = true;
+				return;
+			}
 			resource = AddBuffer(source, memory, op, flags.pc);
 			if (resource == UINT32_MAX) {
 				Fail(flags.pc, "buffer resource limit exceeded");

src/graphics/shader/recompiler/ir/passes/SrtWalker.h

@@ -9,7 +9,7 @@ namespace Libs::Graphics::ShaderRecompiler::IR {
 
 class Value;
 
-using SrtMemoryReader = bool (*)(void* userdata, uint64_t address, uint32_t* value);
+using SrtMemoryReader = bool (*)(void* userdata, uint64_t address, std::span<uint32_t> values);
 
 struct SrtRuntime {
 	std::span<const uint32_t> user_data;
@@ -26,26 +26,42 @@ enum class RuntimeValueType { Any, Integer };
 void BuildSrtPlan(Program& program);
 bool ValidateRuntimeValue(const ResourcePlan& program, Value value,
                           RuntimeValueType type = RuntimeValueType::Any);
-bool EvaluateUniformValues(const ResourcePlan& program, std::span<const Value> values,
-                            const SrtRuntime& runtime, std::span<uint32_t> results);
+// Uses the strict reader for values that affect shader specialization.
+SrtRuntime CleanRuntime(SrtRuntime runtime);
 
-bool EvaluateDescriptorSource(const ResourcePlan& program, uint32_t source,
-                              const SrtRuntime& runtime, DescriptorValue& result);
+// One memoized evaluation session shared by the entire shader resource refresh.
+class SrtWalker {
+public:
+	SrtWalker(const ResourcePlan& program, const SrtRuntime& runtime,
+	          std::span<const uint8_t> clean_flat_slots = {}, SrtWalker* clean_evaluator = nullptr,
+	          Value active_mask = {});
+	~SrtWalker();
+	SrtWalker(const SrtWalker&)            = delete;
+	SrtWalker& operator=(const SrtWalker&) = delete;
 
-// Evaluates one runtime snapshot transactionally. Scalar values and ReadConst results shared by
-// several descriptors are memoized once across the batch.
-bool EvaluateDescriptorSources(const ResourcePlan& program, std::span<const uint32_t> sources,
-                               const SrtRuntime& runtime, std::vector<DescriptorValue>& results);
+	bool Evaluate(Value value, uint32_t& result);
+	bool EvaluateDescriptor(uint32_t source, DescriptorValue& result);
+	// An empty span means that all sources are active.
+	std::span<const uint8_t> FindActiveSources();
+	bool RefreshFlatBuffer(std::vector<uint32_t>& flat);
 
-// Evaluates potentially reachable descriptor sources and the flattened immediate SRT with one
-// memoized scalar walk. Inactive descriptors are zero; on failure no destination is changed.
-bool EvaluateRuntimeSources(const ResourcePlan& program, std::span<const uint32_t> sources,
-                            const SrtRuntime& runtime, std::vector<DescriptorValue>& results,
-                            std::vector<uint32_t>& flat, std::span<const uint8_t> clean_flat_slots,
-                            std::vector<uint8_t>& active_sources);
+private:
+	static ResourcePlan::EvaluationContext& AcquireContext(const ResourcePlan& program);
+	static float Float32(uint64_t bits);
+	bool EvaluateWide(Value value, uint64_t& result);
+	bool Arg(const Inst& inst, size_t index, uint64_t& result);
+	bool EvaluatePhi(const Inst& inst, uint64_t& result);
+	bool EvaluateExtract(const Inst& inst, uint64_t& result);
+	bool EvaluateRawRead(const Inst& inst, uint64_t& result);
+	bool EvaluateInst(const Inst& inst, uint64_t& result);
 
-bool WalkSrt(const ResourcePlan& program, const SrtRuntime& runtime,
-             std::vector<uint32_t>& flat);
+	const ResourcePlan&              m_program;
+	SrtRuntime                      m_runtime;
+	std::span<const uint8_t>         m_clean_flat_slots;
+	SrtWalker*                      m_clean_evaluator = nullptr;
+	Value                           m_active_mask;
+	ResourcePlan::EvaluationContext& m_context;
+};
 
 } // namespace Libs::Graphics::ShaderRecompiler::IR
 

src/graphics/shader/shader.cpp

@@ -108,7 +108,7 @@ static uint64_t GetDeclaredShaderHash(uint64_t shader_addr) {
 
 static ShaderParams GetShaderParams(uint64_t shader_addr, const char* label, uint64_t declared_hash,
 	                                std::span<const uint32_t> user_data,
-	                                const ShaderMappedData& data) {
+	                                const ShaderMappedData& data, uint32_t user_data_base = 0) {
 	if (data.code_size_bytes == 0 || data.code_size_bytes % sizeof(uint32_t) != 0) {
 		EXIT("%s hash=0x%016" PRIx64 " shader=0x%016" PRIx64
 		     " has invalid AGC shader_size=0x%08" PRIx32 "\n",
@@ -116,12 +116,16 @@ static ShaderParams GetShaderParams(uint64_t shader_addr, const char* label, uin
 	}
 	const auto code_words = data.code_size_bytes / sizeof(uint32_t);
 	const auto code = std::span {reinterpret_cast<const uint32_t*>(shader_addr), code_words};
-	return {
-	    .code      = code,
-	    .user_data = std::vector<uint32_t>(user_data.begin(), user_data.end()),
-	    .hash      = declared_hash != 0 ? declared_hash
-	                                    : XXH3_64bits(code.data(), code.size_bytes()),
+	ShaderParams params {
+	    .code            = code,
+	    .user_data_count = static_cast<uint32_t>(user_data.size()) + user_data_base,
+	    .hash            = declared_hash != 0 ? declared_hash
+	                                          : XXH3_64bits(code.data(), code.size_bytes()),
 	};
+	EXIT_IF(user_data.size() > HW::UserSgprInfo::SGPRS_MAX ||
+	        params.user_data_count > params.user_data.size());
+	std::copy(user_data.begin(), user_data.end(), params.user_data.begin() + user_data_base);
+	return params;
 }
 
 #if 0
@@ -651,6 +655,7 @@ void BuildStageStaticKey(const ShaderVertexInputInfo& info, std::vector<uint32_t
 	key.push_back(static_cast<uint32_t>(info.fetch_attrib_reg));
 	key.push_back(static_cast<uint32_t>(info.fetch_buffer_reg));
 	key.push_back(info.resources_num);
+	key.push_back(info.wave_size);
 	key.push_back(info.scratch_size_dwords);
 	key.push_back(info.pa_cl_vs_out_cntl);
 	key.push_back(static_cast<uint32_t>(info.clip_space.enabled));
@@ -755,20 +760,22 @@ ShaderParams PrepareProgram(const HW::VertexShaderInfo& regs, const HW::Context&
                             const HW::UserConfig& user_config, ShaderVertexInputInfo& info) {
 	const auto& sh     = context.GetShaderRegisters();
 	const auto data = ShaderGetMappedData(regs.es_regs.data_addr, "ShaderGetInputInfoVS():");
+	const bool merged = (context.GetShaderStages() & 0x20u) != 0;
 	auto        params = GetShaderParams(
 	    regs.es_regs.data_addr, "ShaderRecompiler VS",
 	    GetDeclaredShaderHash(regs.es_regs.data_addr),
-	    std::span<const uint32_t>(regs.gs_user_sgpr.value, regs.gs_regs.rsrc2.user_sgpr), data);
-	if ((context.GetShaderStages() & 0x20u) == 0) {
+	    std::span<const uint32_t>(regs.gs_user_sgpr.value, regs.gs_regs.rsrc2.user_sgpr), data,
+	    merged ? 8u : 0u);
+	if (!merged) {
 		if (!ShaderGetStaticVertexInputInfo(regs.es_regs.data_addr, regs.gs_user_sgpr,
 		                                    regs.gs_regs.rsrc2.user_sgpr, sh, data, info)) {
 			EXIT("failed to prepare vertex shader program\n");
 		}
+		info.wave_size = (context.GetShaderStages() & 0x00400000u) != 0 ? 32u : 64u;
 		return params;
 	}
 	// NGG user SGPRs start at s8; a separately compiled GS back half also receives
 	// its user-data pointer in s0:s1.
-	params.user_data.insert(params.user_data.begin(), 8u, 0u);
 	info                     = {};
 	info.logical_stage       = ShaderType::Mesh;
 	info.pa_cl_vs_out_cntl   = sh.m_paClVsOutCntl;
@@ -839,14 +846,13 @@ PrepareTessellationPrograms(const HW::VertexShaderInfo& regs, const HW::Context&
 	    GetShaderParams(regs.ls_regs.data_addr, "ShaderRecompiler LS",
 	                    GetDeclaredShaderHash(regs.ls_regs.data_addr), local_users, local),
 	    GetShaderParams(regs.hs_regs.data_addr, "ShaderRecompiler HS",
-	                    GetDeclaredShaderHash(regs.hs_regs.data_addr), local_users, control),
+	                    GetDeclaredShaderHash(regs.hs_regs.data_addr), local_users, control, 8u),
 	    GetShaderParams(regs.es_regs.data_addr, "ShaderRecompiler TES",
 	                    GetDeclaredShaderHash(regs.es_regs.data_addr), evaluation_users,
 	                    evaluation),
 	};
 	// The fused HS back half receives its separate user-data address in s0:s1.
 	// RDNA2 reserves s0:s7 before the native HS user SGPRs.
-	params[1].user_data.insert(params[1].user_data.begin(), 8u, 0u);
 	params[1].user_data[0] = static_cast<uint32_t>(regs.hs_regs.user_data_addr);
 	params[1].user_data[1] = static_cast<uint32_t>(regs.hs_regs.user_data_addr >> 32u);
 

src/graphics/shader/shader.h

@@ -51,10 +51,10 @@ struct CompiledShaderInfo;
 
 struct ShaderStageRuntime {
 	const ShaderRecompiler::IR::CompiledShaderInfo* program = nullptr;
-	ShaderRecompiler::IR::ResourceSnapshot          resources;
+	const ShaderRecompiler::IR::ResourceSnapshot*   resources = nullptr;
 
 	[[nodiscard]] explicit operator bool() const {
-		return program != nullptr;
+		return program != nullptr && resources != nullptr;
 	}
 };
 
@@ -134,6 +134,7 @@ struct ShaderVertexInputInfo {
 	int                     fetch_attrib_reg    = 0;
 	int                     fetch_buffer_reg    = 0;
 	int                     buffers_num         = 0;
+	uint32_t                wave_size           = 64;
 	uint32_t                scratch_size_dwords = 0;
 	uint32_t                pa_cl_vs_out_cntl    = 0;
 	ShaderClipSpaceTransform clip_space;

src/graphics/shader/shaderCompiler.h

@@ -3,6 +3,7 @@
 
 #include "graphics/shader/shader.h"
 
+#include <array>
 #include <span>
 #include <vector>
 
@@ -15,7 +16,8 @@ class UserConfig;
 
 struct ShaderParams {
 	std::span<const uint32_t> code;
-	std::vector<uint32_t>     user_data;
+	std::array<uint32_t, 40>  user_data {}; // 32 user SGPRs plus the merged-stage s0:s7 prefix.
+	uint32_t                  user_data_count = 0;
 	uint64_t                  hash = 0;
 	std::span<const uint32_t> back_code;
 

src/libs/ajm.cpp

@@ -1196,12 +1196,6 @@ int KYTY_SYSV_ABI AjmBatchJobRunSplit(AjmBatchInfo* info, uint32_t instance, uin
 	(void)AjmGetInstanceFormat(instance, nullptr, &decoder, &gapless);
 	AjmWriteSideband(flags, sideband_output, sideband_output_size, decoder, gapless, decode_result);
 
-	LOGF("\t instance = 0x%08" PRIx32 ", flags = 0x%016" PRIx64 ", input_buffers = 0x%016" PRIx64
-	     ", input_num = 0x%016" PRIx64 ", output_buffers = 0x%016" PRIx64
-	     ", output_num = 0x%016" PRIx64 "\n",
-	     instance, flags, reinterpret_cast<uint64_t>(input_buffers),
-	     static_cast<uint64_t>(input_buffers_num), reinterpret_cast<uint64_t>(output_buffers),
-	     static_cast<uint64_t>(output_buffers_num));
 	return AjmAppendJob(info, 16 * (input_buffers_num + output_buffers_num) + 32, "run-split");
 }
 

src/libs/ajm/atrac9_decoder.h

@@ -464,6 +464,14 @@ private:
 			const auto* chunk      = input + offset;
 			const auto  chunk_size = static_cast<size_t>(AjmReadLe32(chunk + 4));
 			const auto  payload    = offset + 8;
+			// The data chunk describes the whole stream; this job may contain only its prefix.
+			if (AjmFourCcEquals(chunk, 'd', 'a', 't', 'a')) {
+				*data_offset                  = payload;
+				result->input_consumed        = payload;
+				result->format                = GetFormat();
+				result->total_decoded_samples = m_total_decoded_samples;
+				return true;
+			}
 			if (payload > input_size || chunk_size > input_size - payload) {
 				result->result = AJM_RESULT_PARTIAL_INPUT;
 				return false;
@@ -496,12 +504,6 @@ private:
 					LOGF("AJM ATRAC9 gapless: total=%" PRIu32 ", skip=%" PRIu16 "\n",
 					     gapless_decode.total_samples, gapless_decode.skip_samples);
 				}
-			} else if (AjmFourCcEquals(chunk, 'd', 'a', 't', 'a')) {
-				*data_offset                  = payload;
-				result->input_consumed        = payload;
-				result->format                = GetFormat();
-				result->total_decoded_samples = m_total_decoded_samples;
-				return true;
 			}
 
 			const auto padded_size = chunk_size + (chunk_size & 1u);

src/libs/avPlayer.cpp

@@ -852,11 +852,15 @@ public:
 		emit_event(stop_event, AVPLAYER_EVENT_STATE_STOP);
 		return 0;
 	}
-	void Pause() {
+	int Pause() {
 		std::scoped_lock lifecycle_lock(lifecycle_mutex);
 		std::lock_guard  lock(mutex);
+		if (state != State::Playing) {
+			return AVPLAYER_ERROR_OPERATION_FAILED;
+		}
 		paused     = true;
 		pause_time = std::chrono::steady_clock::now();
+		return 0;
 	}
 	void Resume() {
 		std::scoped_lock lifecycle_lock(lifecycle_mutex);
@@ -890,6 +894,9 @@ public:
 		bool             was_paused = false;
 		{
 			std::lock_guard lock(mutex);
+			if (state != State::Playing) {
+				return AVPLAYER_ERROR_OPERATION_FAILED;
+			}
 			was_paused = paused;
 		}
 		return StartImpl(ms, was_paused, true);
@@ -1682,6 +1689,7 @@ struct AvPlayerInternal {
 	AvPlayerPostInitData     post_init {};
 	bool                     auto_start        = false;
 	int32_t                  video_buffers     = 2;
+	uint32_t                 sync_mode         = 0;
 	uint32_t                 start_bandwidth   = 0;
 	uint32_t                 minimum_bandwidth = 0;
 	uint32_t                 maximum_bandwidth = 0;
@@ -1730,6 +1738,7 @@ static int add_source(AvPlayerInternal* h, const std::string& filename, AvPlayer
 	if (auto rc = s->Init(filename, type); rc < 0) {
 		return rc;
 	}
+	s->SetSync(h->sync_mode);
 	h->source = std::move(s);
 	emit_event(h->event, AVPLAYER_EVENT_STATE_READY);
 	if (h->auto_start) {
@@ -1915,9 +1924,11 @@ int KYTY_SYSV_ABI AvPlayerPause(AvPlayerInternal* h) {
 	if (h == nullptr || h->source == nullptr) {
 		return AVPLAYER_ERROR_INVALID_PARAMS;
 	}
-	h->source->Pause();
-	emit_event(h->event, AVPLAYER_EVENT_STATE_PAUSE);
-	return 0;
+	auto rc = h->source->Pause();
+	if (rc == 0) {
+		emit_event(h->event, AVPLAYER_EVENT_STATE_PAUSE);
+	}
+	return rc;
 }
 int KYTY_SYSV_ABI AvPlayerResume(AvPlayerInternal* h) {
 	PRINT_NAME();
@@ -1951,13 +1962,16 @@ int KYTY_SYSV_ABI AvPlayerSetTrickSpeed(AvPlayerInternal* h, int32_t trick_speed
 }
 int KYTY_SYSV_ABI AvPlayerSetAvSyncMode(AvPlayerInternal* h, uint32_t sync_mode) {
 	PRINT_NAME();
-	if (h == nullptr || h->source == nullptr) {
+	if (h == nullptr) {
 		return AVPLAYER_ERROR_INVALID_PARAMS;
 	}
 	if (sync_mode > 1) {
 		return AVPLAYER_ERROR_NOT_SUPPORTED;
 	}
-	h->source->SetSync(sync_mode);
+	h->sync_mode = sync_mode;
+	if (h->source != nullptr) {
+		h->source->SetSync(sync_mode);
+	}
 	return 0;
 }
 int KYTY_SYSV_ABI AvPlayerSetAvailableBandwidth(AvPlayerInternal* h, uint32_t start_bandwidth,

src/libs/libSaveData.cpp

@@ -27,6 +27,7 @@ namespace SaveData {
 
 // TODO(): specify dir at launcher
 static constexpr char     SAVE_DATA_DIR[]      = "_SaveData";
+static constexpr uint64_t SAVE_DATA_BLOCKS_MIN = 48;
 static constexpr uint64_t SAVE_DATA_BLOCKS_MAX = 16384;
 
 struct SceSaveDataDirName {
@@ -305,9 +306,9 @@ static bool valid_memory_range(const SaveDataMemoryData& data, size_t size) {
 	       data.buf_size <= size - static_cast<size_t>(data.offset);
 }
 
-// Keep memory.dat as raw guest bytes. Replacing a flushed temporary file protects the
+// Replacing a flushed temporary file protects the
 // previous save if writing fails or the emulator exits partway through an update.
-static int write_memory_file(const std::filesystem::path& path, const void* data, uint32_t size) {
+static int write_save_file(const std::filesystem::path& path, const void* data, uint32_t size) {
 	const auto   temporary = std::filesystem::path(path.string() + ".tmp");
 	Common::File file;
 	if (!file.Create(temporary)) {
@@ -329,6 +330,21 @@ static int write_memory_file(const std::filesystem::path& path, const void* data
 	return SAVE_DATA_ERROR_INTERNAL;
 }
 
+static int read_save_blocks(const std::filesystem::path& directory, uint64_t* blocks) {
+	Common::File file(directory / "sce_sys" / "blocks.bin", Common::File::Mode::Read);
+	if (file.IsInvalid() || file.Size() != sizeof(*blocks)) {
+		return SAVE_DATA_ERROR_BROKEN;
+	}
+	uint64_t value = 0;
+	uint32_t read = 0;
+	file.Read(&value, sizeof(value), &read);
+	if (read != sizeof(value) || value < SAVE_DATA_BLOCKS_MIN || value > SAVE_DATA_BLOCKS_MAX) {
+		return SAVE_DATA_ERROR_BROKEN;
+	}
+	*blocks = value;
+	return OK;
+}
+
 static int save_memory(const std::filesystem::path& directory, const SaveDataMemory& memory) {
 	std::error_code error;
 	std::filesystem::create_directories(directory, error);
@@ -336,10 +352,10 @@ static int save_memory(const std::filesystem::path& directory, const SaveDataMem
 		return SAVE_DATA_ERROR_INTERNAL;
 	}
 	// Display metadata is a separate host sidecar, leaving memory.dat as raw guest bytes.
-	int result = write_memory_file(directory / "param.bin", &memory.param, sizeof(memory.param));
+	int result = write_save_file(directory / "param.bin", &memory.param, sizeof(memory.param));
 	if (result == OK) {
 		result =
-		    write_memory_file(directory / "memory.dat", memory.data.data(), memory.data.size());
+		    write_save_file(directory / "memory.dat", memory.data.data(), memory.data.size());
 	}
 	return result;
 }
@@ -561,9 +577,14 @@ int KYTY_SYSV_ABI SaveDataDirNameSearch(const SaveDataDirNameSearchCond* cond,
 			result->params[i] = {};
 		}
 		if (result->infos != nullptr) {
-			result->infos[i]             = {};
-			result->infos[i].blocks      = SAVE_DATA_BLOCKS_MAX;
-			result->infos[i].free_blocks = SAVE_DATA_BLOCKS_MAX;
+			auto& info = result->infos[i];
+			info = {};
+			const int status = read_save_blocks(root / dir_list[i], &info.blocks);
+			if (status != OK) {
+				return status;
+			}
+			// Host-directory saves report their allocation as free space.
+			info.free_blocks = info.blocks;
 		}
 	}
 
@@ -621,10 +642,22 @@ int KYTY_SYSV_ABI SaveDataMount3(const SaveDataMount3* mount, SaveDataMountResul
 
 	bool created = false;
 	if ((create || create2) && !Common::File::IsDirectoryExisting(mount_dir)) {
-		Common::File::CreateDirectories(mount_dir);
+		if (mount->blocks < SAVE_DATA_BLOCKS_MIN || mount->blocks > SAVE_DATA_BLOCKS_MAX) {
+			return SAVE_DATA_ERROR_PARAMETER;
+		}
+		const auto metadata = std::filesystem::path(mount_dir) / "sce_sys";
+		std::error_code error;
+		std::filesystem::create_directories(metadata, error);
+		if (error) {
+			return SAVE_DATA_ERROR_INTERNAL;
+		}
+		const int status = write_save_file(metadata / "blocks.bin", &mount->blocks,
+		                                   sizeof(mount->blocks));
+		if (status != OK) {
+			std::filesystem::remove_all(mount_dir, error);
+			return status;
+		}
 		created = true;
-
-		EXIT_NOT_IMPLEMENTED((!Common::File::IsDirectoryExisting(mount_dir)));
 	}
 
 	return mount_save_data(slot, dir_name, mount_dir, created ? 1u : 0u, mount_result);
@@ -1101,13 +1134,23 @@ int KYTY_SYSV_ABI SaveDataGetMountInfo(const SaveDataMountPoint* mount_point,
                                        SaveDataMountInfo*        info) {
 	PRINT_NAME();
 
-	EXIT_NOT_IMPLEMENTED(mount_point == nullptr);
-	EXIT_NOT_IMPLEMENTED(info == nullptr);
-
+	if (mount_point == nullptr || info == nullptr ||
+	    std::memchr(mount_point->data, 0, sizeof(mount_point->data)) == nullptr) {
+		return SAVE_DATA_ERROR_PARAMETER;
+	}
+	Common::LockGuard lock(g_mount_mutex);
+	if (g_mount_slots.Find(mount_point->data) == SaveDataMountSlots::FULL) {
+		return SAVE_DATA_ERROR_NOT_MOUNTED;
+	}
+	uint64_t blocks = 0;
+	const int status = read_save_blocks(
+	    LibKernel::FileSystem::GetRealFilename(mount_point->data), &blocks);
+	if (status != OK) {
+		return status;
+	}
 	*info = {};
-
-	info->blocks      = SAVE_DATA_BLOCKS_MAX;
-	info->free_blocks = SAVE_DATA_BLOCKS_MAX;
+	info->blocks = blocks;
+	info->free_blocks = blocks;
 
 	return OK;
 }

src/loader/systemContent.cpp

@@ -164,6 +164,7 @@ void Psf::OpenJson(const std::filesystem::path& file_name) {
 	add_string_param("contentId", "CONTENT_ID");
 	add_string_param("contentVersion", "APP_VER");
 	add_string_param("appVersion", "APP_VER");
+	add_int_param("attribute3", "ATTRIBUTE3");
 	add_int_param("userDefinedParam1", "USER_DEFINED_PARAM_1");
 	add_int_param("userDefinedParam2", "USER_DEFINED_PARAM_2");
 	add_int_param("userDefinedParam3", "USER_DEFINED_PARAM_3");

tests/ResourceMaterializationTests.cpp

@@ -14,7 +14,7 @@ void Check(bool value, const char *text) {
   }
 }
 
-bool RejectSpecializationRead(void *userdata, uint64_t, uint32_t *) {
+bool RejectSpecializationRead(void *userdata, uint64_t, std::span<uint32_t>) {
   ++*static_cast<uint32_t *>(userdata);
   return false;
 }
@@ -219,19 +219,13 @@ void TestUnbasedFlatCacheHitMaterializes() {
         "unbased FLAT plan produced unexpected descriptors");
 }
 
-void TestFailedMaterializationPreservesPriorStage() {
+void TestFailedMaterializationRejectsStage() {
   using namespace Libs::Graphics::ShaderRecompiler::IR;
   auto plan = UserDataBufferPlan();
   ResourceSnapshot snapshot;
-  snapshot.user_data.push_back(0xfeedbeefu);
   ResourceSpecialization specialization;
-  specialization.buffers.push_back({.packed_stride = 7});
   Check(!MaterializeResources(plan, {}, snapshot, specialization),
         "missing runtime user data did not reject the cached stage");
-  Check(snapshot.user_data == std::vector<uint32_t>{0xfeedbeefu} &&
-            specialization.buffers.size() == 1 &&
-            specialization.buffers[0].packed_stride == 7,
-        "failed cache materialization changed its destinations");
 }
 
 void TestMixedSamplerDuplicatesTheCorrectSnapshot() {
@@ -268,7 +262,7 @@ int main() {
   TestMappedSrtUsesDirectReaderByDefault();
   TestIntegerRuntimeValueFollowsSrtReads();
   TestUnbasedFlatCacheHitMaterializes();
-  TestFailedMaterializationPreservesPriorStage();
+  TestFailedMaterializationRejectsStage();
   TestMixedSamplerDuplicatesTheCorrectSnapshot();
   std::puts("ResourceMaterializationTests: all cases passed");
   return 0;

tests/ResourceTrackingTests.cpp

@@ -30,14 +30,6 @@ void Check(bool condition, const char *message) {
   }
 }
 
-bool SameResourceSnapshot(const ResourceSnapshot &lhs,
-                          const ResourceSnapshot &rhs) {
-  return lhs.buffers == rhs.buffers && lhs.images == rhs.images &&
-         lhs.samplers == rhs.samplers &&
-         lhs.flattened_srt == rhs.flattened_srt &&
-         lhs.user_data == rhs.user_data && lhs.uniform_fill == rhs.uniform_fill;
-}
-
 template <typename F>
 void CheckFatal(F &&function, std::string_view expected, const char *message) {
   try {
@@ -146,14 +138,16 @@ struct TestMemory {
   uint32_t fail_after = UINT32_MAX;
 };
 
-bool ReadTestMemory(void *userdata, uint64_t address, uint32_t *value) {
+bool ReadTestMemory(void *userdata, uint64_t address, std::span<uint32_t> values) {
   auto *memory = static_cast<TestMemory *>(userdata);
-  if (memory == nullptr || value == nullptr || address < memory->base ||
-      address - memory->base >= memory->words.size() * sizeof(uint32_t) ||
+  if (memory == nullptr || address < memory->base ||
+      values.size_bytes() > memory->words.size() * sizeof(uint32_t) ||
+      address - memory->base > memory->words.size() * sizeof(uint32_t) - values.size_bytes() ||
       memory->reads >= memory->fail_after) {
     return false;
   }
-  *value = memory->words[(address - memory->base) / sizeof(uint32_t)];
+  std::copy_n(memory->words.begin() + (address - memory->base) / sizeof(uint32_t),
+               values.size(), values.begin());
   memory->reads++;
   return true;
 }
@@ -162,16 +156,31 @@ struct LinearTestMemory {
   uint64_t base = 0x1000;
   std::vector<uint32_t> words = std::vector<uint32_t>(0x2200 / 4);
   uint64_t fail_address = UINT64_MAX;
+  uint64_t watched_address = UINT64_MAX;
+  uint32_t watched_reads = 0;
+  size_t watched_dwords = 0;
+  uint32_t reads = 0;
+  uint32_t descriptor_reads = 0;
 };
 
-bool ReadLinearTestMemory(void *userdata, uint64_t address, uint32_t *value) {
+bool ReadLinearTestMemory(void *userdata, uint64_t address, std::span<uint32_t> values) {
   auto *memory = static_cast<LinearTestMemory *>(userdata);
-  if (memory == nullptr || value == nullptr || address < memory->base ||
-      address - memory->base >= memory->words.size() * sizeof(uint32_t) ||
-      (address & 3u) != 0u || address == memory->fail_address) {
+  if (memory == nullptr || address < memory->base ||
+      values.size_bytes() > memory->words.size() * sizeof(uint32_t) ||
+      address - memory->base > memory->words.size() * sizeof(uint32_t) - values.size_bytes() ||
+      (address & 3u) != 0u ||
+      (memory->fail_address >= address && memory->fail_address - address < values.size_bytes())) {
     return false;
   }
-  *value = memory->words[(address - memory->base) / sizeof(uint32_t)];
+  std::copy_n(memory->words.begin() + (address - memory->base) / sizeof(uint32_t),
+               values.size(), values.begin());
+  ++memory->reads;
+  if (values.size() == 8u) ++memory->descriptor_reads;
+  if (memory->watched_address >= address &&
+      memory->watched_address - address < values.size_bytes()) {
+    ++memory->watched_reads;
+    memory->watched_dwords = values.size();
+  }
   return true;
 }
 
@@ -213,8 +222,11 @@ MakeIndirectImageFixture(bool malformed, uint32_t material_immediate = 0,
                       fixture->AddMemory(shared_buffer, 0x10d8));
     fixture->Emit(ValueOpcode::ReferenceU32, {load});
   }
+  const auto invocation = fixture->Emit(
+      ValueOpcode::GetBuiltin,
+      {Value(static_cast<uint32_t>(StageInputKind::GlobalInvocationId)), Value(0u)});
   const auto selector = fixture->Emit(ValueOpcode::ReadFirstLane,
-                                      {fixture->UserData(8), Value(true)});
+                                      {invocation, Value(true)});
   const auto record =
       fixture->Emit(ValueOpcode::IMul32, {selector, Value(224u)});
   const auto member = fixture->Emit(ValueOpcode::IAdd32, {record, Value(4u)});
@@ -315,14 +327,26 @@ void TestInvariantIndirectImageMaterialization() {
                        snapshot.images[0].dwords.begin()),
         "invariant indirect image table did not materialize");
 
-  const auto prior_snapshot = snapshot;
-  const auto prior_specialization = specialization;
+  user_data[5] = 0u;
+  user_data[6] = 19u;
+  memory.fail_address = 0x2010u;
+  memory.watched_address = 0x2000u;
+  Check(MaterializeResources(resource_plan, runtime, snapshot, specialization) &&
+            memory.watched_dwords == 4u &&
+            std::equal(image_descriptor.begin(), image_descriptor.end(),
+                       snapshot.images[0].dwords.begin()),
+        "partial scalar-buffer descriptor read crossed bounds instead of zeroing its tail");
+  memory.fail_address = 0x2008u;
+  Check(!MaterializeResources(resource_plan, runtime, snapshot, specialization),
+        "unreadable memory inside the descriptor prefix was accepted");
+  user_data[5] = 16u << 16u;
+  user_data[6] = 4u;
+  memory.watched_address = UINT64_MAX;
+
   memory.fail_address = 0x1004u;
   Check(!MaterializeResources(resource_plan, runtime, snapshot,
-                              specialization) &&
-            SameResourceSnapshot(snapshot, prior_snapshot) &&
-            specialization == prior_specialization,
-        "rejected planning memory read mutated the snapshot");
+                              specialization),
+        "unreadable material-table selector was accepted");
   memory.fail_address = UINT64_MAX;
 
   memory.words[(0x1000u - memory.base + 36u) / 4u] = 1u;
@@ -358,6 +382,40 @@ void TestInvariantIndirectImageMaterialization() {
             dynamic_snapshot.images.size() == 2 &&
             dynamic_specialization.images.size() == 2,
         "dynamic indirect image table did not materialize");
+  const auto second_image = (0x2020u - memory.base) / 4u;
+  memory.words[second_image + 1u] |= 3u << 30u;
+  memory.words[second_image + 2u] = 3u << 14u;
+  memory.words[second_image + 3u] =
+      Libs::Graphics::DstSel(4, 5, 6, 7) |
+      (static_cast<uint32_t>(Libs::Graphics::Prospero::ImageType::kCube) << 28u);
+  memory.words[second_image + 4u] = 11u;
+  ResourceSnapshot mixed_snapshot;
+  ResourceSpecialization mixed_specialization;
+  Check(MaterializeResources(resource_plan, runtime, mixed_snapshot,
+                             mixed_specialization) &&
+            mixed_snapshot.images.size() == 2 &&
+            mixed_specialization.images.size() == 2 &&
+            mixed_specialization.images[0].dimension ==
+                Decoder::ImageDimension::Dim2D &&
+            !mixed_specialization.images[0].cube &&
+            mixed_specialization.images[1].dimension ==
+                Decoder::ImageDimension::Dim2DArray &&
+            mixed_specialization.images[1].cube &&
+            std::equal(mixed_snapshot.images[1].dwords.begin(),
+                       mixed_snapshot.images[1].dwords.end(),
+                       memory.words.begin() + second_image),
+        "mixed 2D and cube candidates were rejected or discarded");
+  memory.words[second_image + 1u] =
+      static_cast<uint32_t>(
+          Libs::Graphics::Prospero::BufferFormat::k32_32_32_32UInt)
+          << 20u |
+      (3u << 30u);
+  Check(!MaterializeResources(resource_plan, runtime, mixed_snapshot,
+                              mixed_specialization),
+        "indirect images with different numeric classes were accepted");
+  for (const auto dword : {1u, 2u, 3u, 4u}) {
+    memory.words[second_image + dword] = image_descriptor[dword];
+  }
   ApplyResourceSpecialization(fixture->program, dynamic_specialization);
   Check(fixture->program.info.images.size() == 2 &&
             fixture->program.info.images[0].indirect_root == 0 &&
@@ -430,15 +488,36 @@ void TestInvariantIndirectImageMaterialization() {
                                    .userdata = &memory,
                                    .read_specialization_memory =
                                        ReadLinearTestMemory};
-  const auto memory_backed_prior_snapshot = snapshot;
-  const auto memory_backed_prior_specialization = specialization;
   Check(!MaterializeResources(memory_backed_plan, memory_backed_runtime,
-                              snapshot, specialization) &&
-            SameResourceSnapshot(snapshot, memory_backed_prior_snapshot) &&
-            specialization == memory_backed_prior_specialization,
-        "rejected indirect table descriptor read mutated the snapshot");
+                              snapshot, specialization),
+        "unreadable indirect table descriptor was accepted");
   memory.fail_address = UINT64_MAX;
 
+  memory.watched_address = 0x3100u;
+  memory.watched_reads = 0;
+  Check(MaterializeResources(memory_backed_plan, memory_backed_runtime,
+                             snapshot, specialization) && memory.watched_reads == 1u,
+        "descriptor, flattened SRT and indirect-table roots repeated a clean pointer read");
+  const auto* buffer_storage = snapshot.buffers.data();
+  const auto* image_storage = snapshot.images.data();
+  const auto* flat_storage = snapshot.flattened_srt.data();
+  const auto* user_data_storage = snapshot.user_data.data();
+  const auto* buffer_specialization_storage = specialization.buffers.data();
+  const auto* image_specialization_storage = specialization.images.data();
+  const auto old_address = snapshot.images[0].dwords[0];
+  memory.words[(0x2000u - memory.base) / 4u] += 0x100u;
+  memory.watched_reads = 0;
+  Check(MaterializeResources(memory_backed_plan, memory_backed_runtime,
+                             snapshot, specialization) && memory.watched_reads == 1u &&
+            snapshot.images[0].dwords[0] == old_address + 0x100u,
+        "cache refresh reused stale table contents or repeated its clean pointer read");
+  Check(snapshot.buffers.data() == buffer_storage && snapshot.images.data() == image_storage &&
+            snapshot.flattened_srt.data() == flat_storage &&
+            snapshot.user_data.data() == user_data_storage &&
+            specialization.buffers.data() == buffer_specialization_storage &&
+            specialization.images.data() == image_specialization_storage,
+        "a same-shape refresh discarded the runtime output storage");
+
   auto malformed = MakeIndirectImageFixture(true);
   BuildSrtPlan(malformed->program);
   CheckFatal([&] { TrackResources(malformed->program); }, "not a valid runtime value",
@@ -457,6 +536,322 @@ void TestInvariantIndirectImageMaterialization() {
         "wrapped scalar immediate entered the invariant image proof");
 }
 
+void TestGuardedDirectImageTable() {
+  namespace CFG = Libs::Graphics::ShaderRecompiler::CFG;
+  enum class Guard { Nonzero, Zero, Unrelated, Bypass };
+  const auto make_plan = [](Guard guard) {
+    Fixture fixture(ShaderType::Pixel);
+    auto *entry = fixture.block;
+    auto *middle = fixture.AddBlock();
+    auto *before_sample = fixture.AddBlock();
+    auto *sample = fixture.AddBlock();
+    auto *exit = fixture.AddBlock();
+    entry->AddBranch(middle);
+    entry->AddBranch(exit);
+    middle->AddBranch(before_sample);
+    before_sample->AddBranch(sample);
+    sample->AddBranch(exit);
+    if (guard == Guard::Bypass) exit->AddBranch(sample);
+    const auto mask = fixture.Emit(ValueOpcode::ReadFirstLane,
+        {fixture.Emit(ValueOpcode::GetAttribute, {Value(0u), Value(0u)}), Value(true)});
+    const auto nonzero = fixture.Emit(
+        ValueOpcode::INotEqual32,
+        {Value(0u), guard == Guard::Unrelated ? fixture.UserData(2) : mask});
+    fixture.program.block_info[0].condition =
+        fixture.Emit(ValueOpcode::LogicalNot, {nonzero});
+    fixture.program.block_info[0].terminator = {
+        .kind = CFG::TerminatorKind::ConditionalBranch,
+        .true_block = guard == Guard::Zero ? 1u : 4u,
+        .false_block = guard == Guard::Zero ? 4u : 1u};
+    for (uint32_t block = 1; block < 4; ++block) {
+      fixture.program.block_info[block].terminator = {
+          .kind = CFG::TerminatorKind::Branch, .true_block = block + 1u};
+    }
+    fixture.program.block_info[4].terminator = {
+        .kind = guard == Guard::Bypass ? CFG::TerminatorKind::Branch
+                                      : CFG::TerminatorKind::Return,
+        .true_block = 3u};
+    const auto srt = fixture.Address(fixture.UserData(0), fixture.UserData(1));
+    std::array<Value, 2> pointer;
+    for (uint32_t word = 0; word < pointer.size(); ++word) {
+      MemoryInfo memory;
+      memory.kind = ResourceKind::ScalarAddress;
+      memory.offset = word * 4u;
+      pointer[word] = fixture.Emit(
+          ValueOpcode::LoadAddressU32, {srt, Value(0u), Value(0u), Value(true)},
+          fixture.AddMemory(memory, 0x20));
+    }
+    fixture.block = sample;
+    const auto key = fixture.Emit(ValueOpcode::FindILsb32, {mask});
+    const auto offset = fixture.Emit(ValueOpcode::IAdd32,
+        {fixture.Emit(ValueOpcode::ShiftLeftLogical32, {key, Value(5u)}),
+         Value(344u)});
+    std::array<Value, 8> words;
+    for (uint32_t group = 0; group < 2u; ++group) {
+      // Separate equivalent pointer handles mirror the two scalar x4 loads.
+      const auto table = fixture.Address(pointer[0], pointer[1]);
+      const auto group_offset = group == 0u ? offset : fixture.Emit(
+          ValueOpcode::IAdd32, {offset, Value(16u)});
+      for (uint32_t word = 0; word < 4u; ++word) {
+        MemoryInfo memory;
+        memory.kind = ResourceKind::ScalarAddress;
+        memory.offset = word * 4u;
+        words[group * 4u + word] = fixture.Emit(
+            ValueOpcode::LoadAddressU32,
+            {table, group_offset, Value(0u), Value(true)},
+            fixture.AddMemory(memory, 0x100 + group * 8u));
+      }
+    }
+    const auto image = fixture.Image(words, 0x128);
+    const auto sampler = fixture.Sampler({Value(0u), Value(0u), Value(0u), Value(0u)});
+    MemoryInfo memory;
+    memory.kind = ResourceKind::Image;
+    memory.image_dimension = Decoder::ImageDimension::Dim2D;
+    fixture.Emit(ValueOpcode::ImageSampleRaw, {image, sampler, fixture.ImageAddress()},
+                 fixture.AddMemory(memory, 0x128));
+    fixture.PlanAndTrack();
+    const auto source = fixture.program.info.images[0].source;
+    const auto &indirect = fixture.program.descriptor_sources[source].indirect_image;
+    Check(indirect && indirect->material_source == UINT32_MAX &&
+              indirect->selector_stride == 0u && indirect->table_offset == 344u &&
+              indirect->key_count == 32u &&
+              fixture.program.descriptor_sources[indirect->table_source].dword_count == 2u,
+          "guarded direct image table lost its pointer or proven selector range");
+    return ExtractResourcePlan(fixture.program);
+  };
+
+  auto plan = make_plan(Guard::Nonzero);
+  for (const auto guard : {Guard::Zero, Guard::Unrelated, Guard::Bypass}) {
+    CheckFatal([&] { make_plan(guard); }, "not a valid runtime value",
+               "direct table accepted a selector without a dominating nonzero guard");
+  }
+  LinearTestMemory memory;
+  constexpr uint64_t table = 0x1800u + 344u;
+  memory.words[0] = 0x1800u;
+  const auto fill_table = [](LinearTestMemory &memory, uint64_t base) {
+    for (uint32_t key = 0; key < 32u; ++key) {
+      const auto word = (base - memory.base) / 4u + key * 8u;
+      memory.words[word] = 0x100u + key;
+      memory.words[word + 1u] = static_cast<uint32_t>(
+          Libs::Graphics::Prospero::BufferFormat::k32_32_32_32Float) << 20u;
+      memory.words[word + 3u] = Libs::Graphics::DstSel(4, 5, 6, 7) |
+          (static_cast<uint32_t>(Libs::Graphics::Prospero::ImageType::kColor2D) << 28u);
+    }
+  };
+  fill_table(memory, table);
+  std::array<uint32_t, 2> user_data{0x1000u, 0u};
+  SrtRuntime runtime{.user_data = user_data, .read_memory = ReadLinearTestMemory,
+                     .userdata = &memory,
+                     .read_specialization_memory = ReadLinearTestMemory};
+  ResourceSnapshot snapshot;
+  ResourceSpecialization specialization;
+  Check(MaterializeResources(plan, runtime, snapshot, specialization) &&
+            snapshot.images.size() == 32u && specialization.images.size() == 32u &&
+            snapshot.flattened_srt[specialization.images[0].indirect_mapping_offset] == 32u &&
+            memory.reads == 34u && memory.descriptor_reads == 32u,
+        "direct table did not retain all 32 reachable descriptors");
+  const auto captured_word = (table - memory.base) / 4u + 16u * 8u;
+  const auto original_descriptor = snapshot.images[16].dwords;
+  const std::array<uint32_t, 8> captured_invalid{
+      0x101f0000u, 0xcb500000u, 0x001fc01fu, 0xd0970facu,
+      0x86000000u, 0x00500003u, 0x00000400u, 0x00005204u};
+  std::copy(captured_invalid.begin(), captured_invalid.end(),
+             memory.words.begin() + captured_word);
+  Check(MaterializeResources(plan, runtime, snapshot, specialization) &&
+            snapshot.images.size() == 32u &&
+            snapshot.flattened_srt[specialization.images[0].indirect_mapping_offset] == 32u &&
+            std::ranges::all_of(snapshot.images[16].dwords,
+                                [](uint32_t word) { return word == 0u; }),
+        "captured non-descriptor record became a host image or lost its key mapping");
+  std::copy(original_descriptor.begin(), original_descriptor.end(),
+             memory.words.begin() + captured_word);
+  memory.words[(table - memory.base) / 4u + 31u * 8u] = 0x987u;
+  Check(MaterializeResources(plan, runtime, snapshot, specialization) &&
+            snapshot.images[31].dwords[0] == 0x987u,
+        "direct table refresh reused stale descriptor contents");
+  memory.fail_address = table + 31u * 32u + 28u;
+  Check(!MaterializeResources(plan, runtime, snapshot, specialization),
+        "direct table accepted an unreadable final descriptor word");
+
+  LinearTestMemory wrapping;
+  wrapping.base = 0u;
+  wrapping.words[0x1000u / 4u] = 0xffffff00u;
+  wrapping.words[0x1000u / 4u + 1u] = 0xffffu;
+  wrapping.watched_address = 88u;
+  fill_table(wrapping, 88u);
+  runtime.userdata = &wrapping;
+  Check(!MaterializeResources(plan, runtime, snapshot, specialization) &&
+            wrapping.watched_reads == 0u,
+        "direct table wrapped its descriptor address at the 48-bit boundary");
+
+  LinearTestMemory endpoint;
+  endpoint.base = (uint64_t{1} << 48u) - 0x1000u;
+  const auto crossing = (uint64_t{1} << 48u) - 16u;
+  endpoint.words[0] = static_cast<uint32_t>(crossing - 344u);
+  endpoint.words[1] = static_cast<uint32_t>((crossing - 344u) >> 32u);
+  fill_table(endpoint, crossing);
+  endpoint.watched_address = crossing;
+  user_data = {static_cast<uint32_t>(endpoint.base),
+               static_cast<uint32_t>(endpoint.base >> 32u)};
+  runtime.userdata = &endpoint;
+  Check(!MaterializeResources(plan, runtime, snapshot, specialization) &&
+            endpoint.watched_reads == 0u,
+        "batched descriptor read crossed the 48-bit endpoint");
+}
+
+void TestImageDescriptorFields() {
+  constexpr std::array<std::pair<uint32_t, uint32_t>, 5> reserved{
+      {{1u, 0x20000000u}, {2u, 0x70003000u}, {4u, 0xe000e000u},
+       {5u, 0xf9000000u}, {6u, 0x00007b00u}}};
+  for (const bool r128 : {false, true}) {
+    Fixture fixture;
+    std::array<Value, 8> words;
+    for (uint32_t word = 0; word < words.size(); ++word) {
+      words[word] = fixture.UserData(word);
+    }
+    const auto image = fixture.Image(words);
+    const auto sampler = fixture.Sampler({Value(0u), Value(0u), Value(0u), Value(0u)});
+    MemoryInfo memory;
+    memory.kind = ResourceKind::Image;
+    memory.image_dimension = Decoder::ImageDimension::Dim2D;
+    memory.image_r128 = r128;
+    fixture.Emit(ValueOpcode::ImageSampleRaw, {image, sampler, fixture.ImageAddress()},
+                 fixture.AddMemory(memory, 0x80));
+    fixture.PlanAndTrack();
+    auto plan = ExtractResourcePlan(fixture.program);
+    std::array<uint32_t, 8> user_data{};
+    user_data[0] = 0x100u;
+    user_data[1] = static_cast<uint32_t>(
+        Libs::Graphics::Prospero::BufferFormat::k32_32_32_32Float) << 20u;
+    user_data[3] = Libs::Graphics::DstSel(4, 5, 6, 7) |
+        (static_cast<uint32_t>(Libs::Graphics::Prospero::ImageType::kColor2D) << 28u);
+    const SrtRuntime runtime{.user_data = user_data};
+    ResourceSnapshot snapshot;
+    ResourceSpecialization specialization;
+    const auto is_null = [&] {
+      return std::ranges::all_of(snapshot.images[0].dwords,
+                                 [](uint32_t word) { return word == 0u; });
+    };
+    Check(MaterializeResources(plan, runtime, snapshot, specialization) &&
+              snapshot.images[0].dwords == user_data,
+          "valid texture descriptor was rejected");
+    const auto valid_descriptor = user_data;
+    // Keep RESOURCE_LEVEL set in this valid texture descriptor.
+    user_data = {0x0208a200u, 0xca900000u, 0x800fc00fu, 0x90960facu,
+                 0u, 0x60u, 0u, 0u};
+    Check(MaterializeResources(plan, runtime, snapshot, specialization) &&
+              snapshot.images[0].dwords == user_data,
+          "instruction-ready texture descriptor lost RESOURCE_LEVEL or became null");
+    user_data = valid_descriptor;
+    for (const auto [word, mask] : reserved) {
+      for (uint32_t bits = mask; bits != 0u; bits &= bits - 1u) {
+        const auto bit = bits & (0u - bits);
+        user_data[word] |= bit;
+        Check(MaterializeResources(plan, runtime, snapshot, specialization) &&
+                  (r128 && word >= 4u ? snapshot.images[0].dwords == user_data : is_null()),
+              "reserved descriptor bits or ignored R128 upper words were misclassified");
+        user_data[word] &= ~bit;
+      }
+    }
+    user_data[5] = 0x06800000u;
+    user_data[6] = 0x010880ffu;
+    user_data[7] = 0x1234u;
+    Check(MaterializeResources(plan, runtime, snapshot, specialization) &&
+              snapshot.images[0].dwords == user_data,
+          "defined mip-statistics, PRT, or metadata fields were treated as reserved");
+    user_data[3] |= 3u << 16u;
+    Check(MaterializeResources(plan, runtime, snapshot, specialization) &&
+              snapshot.images[0].dwords == user_data,
+          "view LAST_LEVEL above physical MAX_MIP was rejected");
+    if (!r128) {
+      user_data[3] = (user_data[3] & 0x0fffffffu) |
+          (static_cast<uint32_t>(Libs::Graphics::Prospero::ImageType::kColor2DArray) << 28u);
+      user_data[4] = 3u | (4u << 16u);
+      Check(MaterializeResources(plan, runtime, snapshot, specialization) && is_null(),
+            "array view starting after its last slice was accepted");
+      for (const auto base : {1u, 3u}) {
+        user_data[4] = 3u | (base << 16u);
+        Check(MaterializeResources(plan, runtime, snapshot, specialization) &&
+                  snapshot.images[0].dwords == user_data,
+              "valid array view with a nonzero base slice was rejected");
+      }
+    }
+  }
+}
+
+void TestUniformScalarBufferImage() {
+  Fixture fixture(ShaderType::Pixel);
+  std::array<Value, 4> material_words;
+  std::array<Value, 4> heap_words;
+  for (uint32_t dword = 0; dword < 4; dword++) {
+    material_words[dword] = fixture.UserData(dword);
+    heap_words[dword] = fixture.UserData(dword + 4u);
+  }
+  const auto material = fixture.Buffer(material_words);
+  const auto heap = fixture.Buffer(heap_words);
+  const auto selector = fixture.Emit(ValueOpcode::ShiftLeftLogical32,
+                                     {fixture.UserData(8), Value(2u)});
+  MemoryInfo scalar;
+  scalar.kind = ResourceKind::ScalarBuffer;
+  scalar.offset = 0x40u;
+  const auto key = fixture.Emit(ValueOpcode::ReadConstBuffer,
+                                {material, selector}, fixture.AddMemory(scalar, 0x100));
+  const auto record = fixture.Emit(ValueOpcode::IMul32, {key, Value(48u)});
+  std::array<Value, 8> image_words;
+  for (uint32_t dword = 0; dword < image_words.size(); dword++) {
+    scalar.offset = 0x10u + dword * sizeof(uint32_t);
+    image_words[dword] = fixture.Emit(ValueOpcode::ReadConstBuffer,
+                                      {heap, record}, fixture.AddMemory(scalar, 0x200));
+  }
+  const auto image = fixture.Image(image_words);
+  const auto sampler = fixture.Sampler({Value(0u), Value(0u), Value(0u), Value(0u)});
+  MemoryInfo sample;
+  sample.kind = ResourceKind::Image;
+  sample.image_dimension = Decoder::ImageDimension::Dim2D;
+  fixture.Emit(ValueOpcode::ImageSampleRaw,
+                {image, sampler, fixture.ImageAddress()}, fixture.AddMemory(sample, 0x228));
+  fixture.PlanAndTrack();
+  const auto plan = ExtractResourcePlan(fixture.program);
+
+  std::array<uint32_t, 9> user_data{0x1000u, 0u, 0x100u, 0u,
+                                    0x2000u, 0u, 0x100u, 0u, 0u};
+  LinearTestMemory memory;
+  memory.words[0x44u / 4u] = 1u;
+  std::array<uint32_t, 8> descriptor{};
+  descriptor[0] = 0x20u;
+  descriptor[1] = static_cast<uint32_t>(
+                      Libs::Graphics::Prospero::BufferFormat::k32_32_32_32Float)
+                  << 20u;
+  descriptor[2] = 3u | (3u << 14u);
+  descriptor[3] = Libs::Graphics::DstSel(4, 5, 6, 7) |
+                  (static_cast<uint32_t>(Libs::Graphics::Prospero::ImageType::kColor2D)
+                   << 28u);
+  for (uint32_t index = 0; index < 2; index++) {
+    std::copy(descriptor.begin(), descriptor.end(),
+              memory.words.begin() + (0x1010u + index * 48u) / 4u);
+    memory.words[(0x1010u + index * 48u) / 4u] += index;
+  }
+  SrtRuntime runtime{.user_data = user_data,
+                     .read_memory = ReadLinearTestMemory,
+                     .userdata = &memory,
+                     .read_specialization_memory = ReadLinearTestMemory};
+  ResourceSnapshot snapshot;
+  ResourceSpecialization specialization;
+  const auto materializes = [&](uint32_t address) {
+    return MaterializeResources(plan, runtime, snapshot, specialization) &&
+           snapshot.images.size() == 1 && snapshot.images[0].dwords[0] == address;
+  };
+  Check(materializes(0x20u), "nested draw-uniform image descriptor did not materialize");
+  user_data[8] = 1u;
+  Check(materializes(0x21u), "changed draw selector reused the previous image descriptor");
+  memory.words[0x44u / 4u] = 0u;
+  memory.words[0x1010u / 4u] = 0x30u;
+  Check(materializes(0x30u), "changed scalar table memory reused the previous image descriptor");
+  memory.fail_address = 0x1044u;
+  Check(!MaterializeResources(plan, runtime, snapshot, specialization),
+        "unavailable nested image table was accepted");
+}
+
 void TestComputeBufferFill() {
   struct Options {
     bool scalar = false;
@@ -524,12 +919,12 @@ void TestComputeBufferFill() {
     std::array<uint32_t, 4> userdata{0x200000u, 4u << 16, 0x4000u, 0x14204u};
     ResourceSnapshot snapshot;
     ResourceSpecialization specialization;
-    const auto Read = +[](void *data, uint64_t address, uint32_t *word) {
+    const auto Read = +[](void *data, uint64_t address, std::span<uint32_t> words) {
       auto &memory = *static_cast<TestMemory *>(data);
-      if (address != memory.base)
+      if (address != memory.base || words.size() != 1u)
         return false;
       ++memory.reads;
-      *word = memory.words[0];
+      words[0] = memory.words[0];
       return true;
     };
     Check(MaterializeResources(
@@ -551,6 +946,12 @@ void TestComputeBufferFill() {
                 snapshot.uniform_fill.value ==
                     (options.scalar ? 0x40404040u : 0u),
             "fill proof lost address coverage or the actual stored scalar");
+      if (options.scalar && options.clean) {
+        Check(MaterializeResources(plan,
+                  {.user_data = userdata, .read_memory = Read, .userdata = &memory},
+                  snapshot, specialization) && snapshot.uniform_fill.words == 0,
+              "an unavailable clean value retained a previous uniform fill");
+      }
     }
   };
   Run({});
@@ -669,12 +1070,12 @@ void TestRuntimeUnsignedMinDescriptor() {
   SrtRuntime runtime{.user_data = user_data};
   DescriptorValue value;
   const auto source = fixture.program.info.buffers[0].source;
-  Check(EvaluateDescriptorSource(fixture.program, source, runtime, value) &&
+  Check(SrtWalker(fixture.program, runtime).EvaluateDescriptor(source, value) &&
             value.dwords[3] == 0x100u,
         "runtime descriptor unsigned minimum did not clamp its first operand");
   user_data[0] = 0x80u;
   Check(
-      EvaluateDescriptorSource(fixture.program, source, runtime, value) &&
+      SrtWalker(fixture.program, runtime).EvaluateDescriptor(source, value) &&
           value.dwords[3] == 0x80u,
       "runtime descriptor unsigned minimum did not preserve its first operand");
 }
@@ -785,7 +1186,7 @@ void TestSampleAdjustSamplerScratch() {
   std::array<uint32_t, 4> user_data{4u, 1u, 2u, 0x80000abcu};
   SrtRuntime runtime{.user_data = user_data};
   DescriptorValue descriptor;
-  Check(EvaluateDescriptorSource(fixture.program, source, runtime, descriptor) &&
+  Check(SrtWalker(fixture.program, runtime).EvaluateDescriptor(source, descriptor) &&
             descriptor.dwords[3] == 0x80000abcu,
         "SampleAdjust canonicalization lost sampler border fields");
 
@@ -892,11 +1293,11 @@ void TestFmaskLoadSpecialization() {
         "FMASK load did not lower to a value vector");
   result = vector->Arg(0);
   uint32_t value = 0;
-  Check(EvaluateUniformValues(plan, {&result, 1}, {.user_data = user_data}, {&value, 1}) &&
+  Check(SrtWalker(fixture.program, {.user_data = user_data}).Evaluate(result, value) &&
             value == 0x76543210u,
         "FMASK load did not return the native sample-to-fragment mapping");
   user_data[8] = 1;
-  Check(EvaluateUniformValues(plan, {&result, 1}, {.user_data = user_data}, {&value, 1}) &&
+  Check(SrtWalker(fixture.program, {.user_data = user_data}).Evaluate(result, value) &&
             value == 0u,
         "inactive FMASK load did not preserve the execution mask");
   ShaderComputeInputInfo compute{};
@@ -1046,13 +1447,9 @@ void TestDynamicStorageMipTracking() {
   changed_user_data[3] =
       (changed_user_data[3] & ~((0xfu << 12u) | (0xfu << 16u))) |
       (4u << 12u) | (3u << 16u);
-  const auto valid_snapshot = changed_snapshot;
-  const auto valid_specialization = changed_specialization;
   Check(!MaterializeResources(resource_plan, {.user_data = changed_user_data},
-                              changed_snapshot, changed_specialization) &&
-            SameResourceSnapshot(changed_snapshot, valid_snapshot) &&
-            changed_specialization == valid_specialization,
-        "an inverted dynamic storage mip range was accepted or mutated output");
+                              changed_snapshot, changed_specialization),
+        "an inverted dynamic storage mip range was accepted");
 }
 
 void TestSrtFlatteningAndRuntimeMemoization() {
@@ -1093,28 +1490,38 @@ void TestSrtFlatteningAndRuntimeMemoization() {
   SrtRuntime runtime{.user_data = user_data,
                      .read_memory = ReadTestMemory,
                      .userdata = &memory};
-  std::vector<DescriptorValue> descriptors;
+  DescriptorValue descriptor;
   std::vector<uint32_t> flat;
-  std::vector<uint8_t> active_sources;
   const uint32_t request = fixture.program.info.buffers[0].source;
-  Check(EvaluateRuntimeSources(fixture.program, std::span{&request, 1}, runtime,
-                               descriptors, flat, {}, active_sources),
-        "typed runtime source evaluation failed");
-  Check(descriptors.size() == 1 && descriptors[0].dwords[0] == 0xdeadbeefu &&
+  const auto refresh = [&](const ResourcePlan& plan) {
+    SrtWalker walker(plan, runtime);
+    return walker.EvaluateDescriptor(request, descriptor) && walker.RefreshFlatBuffer(flat);
+  };
+  Check(refresh(fixture.program), "typed runtime source evaluation failed");
+  Check(descriptor.dwords[0] == 0xdeadbeefu &&
             flat == std::vector<uint32_t>{0xdeadbeefu} && memory.reads == 1,
         "descriptor and flat SRT evaluation did not share one memoized read");
 
+  memory.reads = 0;
+  memory.words[1] = 0x12345678u;
+  Check(refresh(fixture.program) && descriptor.dwords[0] == 0x12345678u &&
+            flat == std::vector<uint32_t>{0x12345678u} && memory.reads == 1,
+        "repeated runtime evaluation reused stale scalar memory");
+
   memory.reads = 0;
   memory.fail_after = 0;
-  descriptors = {{{1u}, 1u}};
-  flat = {2u};
-  active_sources = {3u};
-  Check(!EvaluateRuntimeSources(fixture.program, std::span{&request, 1},
-                                runtime, descriptors, flat, {}, active_sources) &&
-            descriptors == std::vector<DescriptorValue>{{{1u}, 1u}} &&
-            flat == std::vector<uint32_t>{2u} &&
-            active_sources == std::vector<uint8_t>{3u},
-        "runtime evaluation failure was not transactional");
+  Check(!refresh(fixture.program), "unavailable scalar memory was accepted");
+  memory.fail_after = UINT32_MAX;
+  Check(refresh(fixture.program) && descriptor.dwords[0] == 0x12345678u && memory.reads == 1,
+        "failed runtime evaluation left a value marked as visiting");
+
+  auto detached = ExtractResourcePlan(fixture.program);
+  Check(refresh(detached), "detached resource plan did not evaluate");
+  auto moved = std::move(detached);
+  memory.reads = 0;
+  memory.words[1] = 0x87654321u;
+  Check(refresh(moved) && descriptor.dwords[0] == 0x87654321u && memory.reads == 1,
+        "moving a cached resource plan lost its evaluation state");
 
   ShaderComputeInputInfo compute{};
   CollectShaderInfo(fixture.program, {.compute = &compute});
@@ -1154,8 +1561,7 @@ void TestDynamicSrtReadRemainsExplicit() {
                      .read_memory = ReadTestMemory,
                      .userdata = &memory};
   DescriptorValue value;
-  Check(EvaluateDescriptorSource(fixture.program,
-                                 fixture.program.info.buffers[0].source, runtime, value) &&
+  Check(SrtWalker(fixture.program, runtime).EvaluateDescriptor(fixture.program.info.buffers[0].source, value) &&
             value.dwords[0] == 0xabcdef01u && memory.reads == 1,
         "dynamic typed scalar descriptor source was not evaluated");
 
@@ -1330,7 +1736,8 @@ void TestConditionalSamplerPhi() {
           const uint32_t first = (flag < 0) != reverse ? 480 / 4 : 448 / 4;
           memory.fail_address = 0x1000 + (first == 448 / 4 ? 480u : 448u);
           DescriptorValue selected;
-          Check(EvaluateDescriptorSource(plan, source, runtime, selected),
+          SrtWalker clean(plan, CleanRuntime(runtime));
+          Check(SrtWalker(plan, runtime, {}, &clean).EvaluateDescriptor(source, selected),
                 "conditional sampler did not survive detached plan lifetime");
           for (uint32_t word = 0; word < 4; ++word) {
             Check(selected.dwords[word] == memory.words[first + word],
@@ -1340,13 +1747,17 @@ void TestConditionalSamplerPhi() {
         DescriptorValue selected;
         auto no_clean_reader = runtime;
         no_clean_reader.read_specialization_memory = nullptr;
-        Check(
-            !EvaluateDescriptorSource(plan, source, no_clean_reader, selected),
-            "conditional sampler used unchecked memory for its predicate");
+        {
+          SrtWalker clean(plan, CleanRuntime(no_clean_reader));
+          Check(!SrtWalker(plan, no_clean_reader, {}, &clean).EvaluateDescriptor(source, selected),
+                "conditional sampler used unchecked memory for its predicate");
+        }
         memory.fail_address = 0x2000 + 196;
-        Check(!EvaluateDescriptorSource(plan, source, runtime, selected),
-              "conditional sampler ignored unavailable coherent predicate "
-              "memory");
+        {
+          SrtWalker clean(plan, CleanRuntime(runtime));
+          Check(!SrtWalker(plan, runtime, {}, &clean).EvaluateDescriptor(source, selected),
+                "conditional sampler ignored unavailable coherent predicate memory");
+        }
       }
     }
     CheckFatal([&] { ConditionalSamplerPlan(diamond, false, false, true); },
@@ -1359,6 +1770,90 @@ void TestConditionalSamplerPhi() {
   }
 }
 
+void TestGuardedSamplerPhi() {
+  namespace CFG = Libs::Graphics::ShaderRecompiler::CFG;
+  const auto make_plan = [](bool take_true, bool reverse_phi, bool bypass,
+                            bool mismatched, bool ambiguous) {
+    Fixture fixture(ShaderType::Pixel);
+    auto *entry = fixture.block;
+    auto *loaded = fixture.AddBlock();
+    auto *merge = fixture.AddBlock();
+    auto *sample = fixture.AddBlock();
+    auto *exit = fixture.AddBlock();
+    entry->AddBranch(loaded);
+    entry->AddBranch(merge);
+    loaded->AddBranch(merge);
+    merge->AddBranch(sample);
+    merge->AddBranch(exit);
+    sample->AddBranch(exit);
+    if (bypass) exit->AddBranch(sample);
+    const auto lane = fixture.Emit(ValueOpcode::LaneId);
+    const auto predicate = fixture.Emit(ValueOpcode::IEqual32, {lane, Value(0u)});
+    fixture.program.block_info[0].condition = predicate;
+    fixture.program.block_info[0].terminator = {
+        .kind = CFG::TerminatorKind::ConditionalBranch,
+        .true_block = 1u, .false_block = 2u};
+    fixture.program.block_info[1].terminator = {
+        .kind = CFG::TerminatorKind::Branch, .true_block = 2u};
+    auto &guard = merge->AppendNewInst(ValueOpcode::Phi, {},
+                                       static_cast<uint64_t>(Type::U1));
+    guard.AddPhiOperand(entry, ambiguous ? predicate : Value(!take_true));
+    guard.AddPhiOperand(mismatched ? sample : loaded, predicate);
+    fixture.program.block_info[2].condition = Value(&guard);
+    fixture.program.block_info[2].terminator = {
+        .kind = CFG::TerminatorKind::ConditionalBranch,
+        .true_block = take_true ? 3u : 4u,
+        .false_block = take_true ? 4u : 3u};
+    fixture.program.block_info[3].terminator = {
+        .kind = CFG::TerminatorKind::Branch, .true_block = 4u};
+    fixture.program.block_info[4].terminator = {
+        .kind = bypass ? CFG::TerminatorKind::Branch : CFG::TerminatorKind::Return,
+        .true_block = 3u};
+    std::array<Value, 4> words;
+    for (uint32_t word = 0; word < words.size(); ++word) {
+      // An arbitrary excluded value proves this is control-flow reasoning, not null filtering.
+      const auto first = Value(0xbad000u + word);
+      const auto second = fixture.UserData(word);
+      auto &phi = merge->AppendNewInst(ValueOpcode::Phi, {},
+                                       static_cast<uint64_t>(Type::U32));
+      phi.AddPhiOperand(reverse_phi ? loaded : entry, reverse_phi ? second : first);
+      phi.AddPhiOperand(reverse_phi ? entry : loaded, reverse_phi ? first : second);
+      words[word] = Value(&phi);
+    }
+    fixture.block = sample;
+    const auto image = fixture.Image({Value(0u), Value(0u), Value(0u), Value(0u),
+                                      Value(0u), Value(0u), Value(0u), Value(0u)});
+    const auto sampler = fixture.Sampler(words);
+    MemoryInfo memory;
+    memory.kind = ResourceKind::Image;
+    memory.image_dimension = Decoder::ImageDimension::Dim2D;
+    fixture.Emit(ValueOpcode::ImageSampleRaw,
+                  {image, sampler, fixture.ImageAddress()},
+                  fixture.AddMemory(memory, 0x2a0));
+    fixture.PlanAndTrack();
+    Check(words[0].ResolveInstruction()->GetOpcode() == ValueOpcode::Phi,
+          "guarded host descriptor selection rewrote the GPU Phi");
+    return ExtractResourcePlan(fixture.program);
+  };
+  for (const bool take_true : {false, true}) {
+    for (const bool reverse_phi : {false, true}) {
+      auto plan = make_plan(take_true, reverse_phi, false, false, false);
+      const std::array<uint32_t, 4> user_data{0x444u, 0x555u, 0x666u, 0x777u};
+      DescriptorValue selected;
+      Check(SrtWalker(plan, {.user_data = user_data}).EvaluateDescriptor(
+                plan.info.samplers[0].source, selected) &&
+                std::equal(user_data.begin(), user_data.end(), selected.dwords.begin()),
+            "guarded sampler did not match descriptor and predicate predecessors");
+    }
+  }
+  CheckFatal([&] { make_plan(true, false, true, false, false); },
+              "not a valid runtime value", "sampler guard accepted an unguarded path");
+  CheckFatal([&] { make_plan(true, false, false, true, false); },
+              "not a valid runtime value", "sampler guard ignored predecessor identity");
+  CheckFatal([&] { make_plan(true, false, false, false, true); },
+              "not a valid runtime value", "sampler guard discarded a reachable alternative");
+}
+
 void TestLoopCycleEnteredThroughRuntimeValue() {
   Fixture fixture;
   auto *entry = fixture.block;
@@ -1403,8 +1898,7 @@ void TestInvariantLoopPhi() {
   std::array<uint32_t, 1> user_data{0x12345678u};
   SrtRuntime runtime{.user_data = user_data};
   DescriptorValue descriptor;
-  Check(EvaluateDescriptorSource(fixture.program,
-                                 fixture.program.info.buffers[0].source, runtime, descriptor) &&
+  Check(SrtWalker(fixture.program, runtime).EvaluateDescriptor(fixture.program.info.buffers[0].source, descriptor) &&
             descriptor.dwords[0] == user_data[0],
         "loop-invariant descriptor phi was not evaluated through typed SSA");
 }
@@ -1630,11 +2124,9 @@ void TestConditionalBufferMaterialization() {
   runtime.user_data = std::span(user_data).first(4);
   Check(MaterializeResources(plan, runtime, snapshot, specialization),
         "untaken branch evaluated its unavailable descriptor");
-  const auto prior = snapshot;
   memory.words[0] = 1;
-  Check(!MaterializeResources(plan, runtime, snapshot, specialization) &&
-            SameResourceSnapshot(snapshot, prior),
-        "taken branch accepted an unavailable descriptor or changed the snapshot");
+  Check(!MaterializeResources(plan, runtime, snapshot, specialization),
+        "taken branch accepted an unavailable descriptor");
 
   runtime.user_data = user_data;
   const auto CheckActive = [&] {
@@ -1699,7 +2191,7 @@ void TestConditionalIndirectImageMaterialization() {
   uint32_t reads = 0;
   const SrtRuntime runtime{
       .user_data = user_data, .userdata = &reads,
-      .read_specialization_memory = [](void *data, uint64_t, uint32_t *) {
+      .read_specialization_memory = [](void *data, uint64_t, std::span<uint32_t>) {
         ++*static_cast<uint32_t *>(data);
         return false;
       }};
@@ -2013,10 +2505,14 @@ int main() {
     Run("FMASK load specialization", TestFmaskLoadSpecialization);
     Run("dynamic storage mips", TestDynamicStorageMipTracking);
     Run("invariant indirect images", TestInvariantIndirectImageMaterialization);
+    Run("guarded direct image table", TestGuardedDirectImageTable);
+    Run("image descriptor fields", TestImageDescriptorFields);
+    Run("draw-uniform scalar image", TestUniformScalarBufferImage);
     Run("SRT runtime", TestSrtFlatteningAndRuntimeMemoization);
     Run("dynamic SRT", TestDynamicSrtReadRemainsExplicit);
     Run("phi validation", TestPhiValidation);
     Run("conditional sampler phi", TestConditionalSamplerPhi);
+    Run("guarded sampler phi", TestGuardedSamplerPhi);
     Run("runtime-rooted loop", TestLoopCycleEnteredThroughRuntimeValue);
     Run("invariant loop phi", TestInvariantLoopPhi);
     Run("DMA address materialization", TestDmaAddressMaterialization);

tests/SaveDataMemoryTests.cpp

@@ -17,6 +17,7 @@
 namespace {
 std::string           g_test_title = "PPSA21564";
 std::filesystem::path g_mounted_directory;
+std::map<std::string, std::filesystem::path> g_test_mounts;
 } // namespace
 
 namespace Loader {
@@ -33,10 +34,14 @@ Common::Time GetTime() {
 } // namespace Loader
 
 namespace Libs::LibKernel::FileSystem {
-void Mount(const std::filesystem::path& directory, const std::string&) {
+void Mount(const std::filesystem::path& directory, const std::string& point) {
 	g_mounted_directory = directory;
+	g_test_mounts[point] = directory;
+}
+void Umount(const std::string& point) { g_test_mounts.erase(point); }
+std::filesystem::path GetRealFilename(const std::string& point) {
+	return g_test_mounts.at(point);
 }
-void Umount(const std::string&) {}
 } // namespace Libs::LibKernel::FileSystem
 
 namespace {
@@ -337,6 +342,7 @@ void TestClassicSavePaths() {
 		mount.user_id    = 1;
 		mount.dir_name   = &name;
 		mount.mount_mode = text == "save.1" ? 1 : 4;
+		mount.blocks     = 48;
 		SaveDataMountResult result {};
 		CHECK(SaveDataMount3(&mount, &result) == OK);
 		CHECK(g_mounted_directory == fs::path("_SaveData") / "CLASSIC" / text);
@@ -385,6 +391,70 @@ void TestClassicSavePaths() {
 	CHECK(Search(1) == names);
 }
 
+void TestSaveAllocations() {
+	Reset("CAPACITY");
+	std::array<SceSaveDataDirName, 2> names {DirName("Options"), DirName("Player")};
+	const std::array<uint64_t, 2> allocations {48, 96};
+	for (const auto blocks : {0u, 47u, 16385u}) {
+		struct SaveDataMount3 invalid {};
+		invalid.user_id = 1;
+		invalid.dir_name = &names[0];
+		invalid.mount_mode = 4;
+		invalid.blocks = blocks;
+		SaveDataMountResult result {};
+		CHECK(SaveDataMount3(&invalid, &result) == SAVE_DATA_ERROR_PARAMETER);
+		CHECK(!fs::exists("_SaveData/CAPACITY/Options"));
+	}
+	for (size_t i = 0; i < names.size(); i++) {
+		struct SaveDataMount3 mount {};
+		mount.user_id = 1;
+		mount.dir_name = &names[i];
+		mount.mount_mode = 4;
+		mount.blocks = allocations[i];
+		SaveDataMountResult result {};
+		CHECK(SaveDataMount3(&mount, &result) == OK);
+		CHECK(result.mount_status == 1);
+		std::ofstream(g_mounted_directory / "USR-DATA") << "saved payload";
+		CHECK(SaveDataUmount2(0, &result.mount_point) == OK);
+		SaveDataMountInfo info {};
+		CHECK(SaveDataGetMountInfo(&result.mount_point, &info) == SAVE_DATA_ERROR_NOT_MOUNTED);
+	}
+	Reset("CAPACITY");
+	std::array<SceSaveDataDirName, 2> found {};
+	std::array<SaveDataSearchInfo, 2> infos {};
+	SaveDataDirNameSearchCond cond {};
+	cond.user_id = 1;
+	SaveDataDirNameSearchResult search {};
+	search.dir_names = found.data();
+	search.dir_names_num = found.size();
+	search.infos = infos.data();
+	CHECK(SaveDataDirNameSearch(&cond, &search) == OK && search.set_num == 2);
+	for (size_t i = 0; i < names.size(); i++) {
+		CHECK(std::string(found[i].data) == names[i].data);
+		CHECK(infos[i].blocks == allocations[i] && infos[i].free_blocks == allocations[i]);
+		struct SaveDataMount3 mount {};
+		mount.user_id = 1;
+		mount.dir_name = &names[i];
+		mount.mount_mode = 34;
+		mount.blocks = 16384;
+		SaveDataMountResult result {};
+		CHECK(SaveDataMount3(&mount, &result) == OK && result.mount_status == 0);
+		std::ofstream(g_mounted_directory / "USR-DATA", std::ios::app) << "more data";
+		SaveDataMountInfo info {};
+		CHECK(SaveDataGetMountInfo(&result.mount_point, &info) == OK);
+		CHECK(info.blocks == allocations[i] && info.free_blocks == allocations[i]);
+		CHECK(SaveDataUmount2(0, &result.mount_point) == OK);
+	}
+	CHECK(infos[0].blocks + infos[1].blocks == 144);
+	const fs::path metadata = "_SaveData/CAPACITY/Options/sce_sys/blocks.bin";
+	fs::resize_file(metadata, 7);
+	CHECK(SaveDataDirNameSearch(&cond, &search) == SAVE_DATA_ERROR_BROKEN);
+	CHECK(fs::file_size(metadata) == 7);
+	CHECK(fs::remove(metadata));
+	CHECK(SaveDataDirNameSearch(&cond, &search) == SAVE_DATA_ERROR_BROKEN);
+	CHECK(!fs::exists(metadata));
+}
+
 void RunChild(const fs::path& executable, const char* mode) {
 #ifdef _WIN32
 	CHECK(_spawnl(_P_WAIT, executable.string().c_str(), executable.string().c_str(), mode,
@@ -425,6 +495,7 @@ int main(int argc, char** argv) {
 	TestIsolationAndSync();
 	TestFailedWritePreservesSave();
 	TestClassicSavePaths();
+	TestSaveAllocations();
 	CHECK(SaveDataTerminate() == OK);
 	fs::current_path(previous);
 	fs::remove_all(temp);

tests/ScalarProvenanceTests.cpp

@@ -79,13 +79,14 @@ struct TestMemory {
   uint32_t reads = 0;
 };
 
-bool ReadMemory(void *userdata, uint64_t address, uint32_t *value) {
+bool ReadMemory(void *userdata, uint64_t address, std::span<uint32_t> values) {
   auto &memory = *static_cast<TestMemory *>(userdata);
-  const auto it = memory.words.find(address);
-  if (it == memory.words.end()) {
-    return false;
+  for (auto &value : values) {
+    const auto it = memory.words.find(address);
+    if (it == memory.words.end()) return false;
+    value = it->second;
+    address += sizeof(uint32_t);
   }
-  *value = it->second;
   memory.reads++;
   return true;
 }
@@ -122,7 +123,7 @@ void TestImmediateFlatteningAndGvn() {
   TestMemory memory_image{{{0x1020u, 0xfeedbeefu}}};
   SrtRuntime runtime{.read_memory = ReadMemory, .userdata = &memory_image};
   std::vector<uint32_t> flat;
-  Check(WalkSrt(fixture.program, runtime, flat), "flattened SRT walk failed");
+  Check(SrtWalker(fixture.program, runtime).RefreshFlatBuffer(flat), "flattened SRT walk failed");
   Check(flat == std::vector<uint32_t>{0xfeedbeefu} && memory_image.reads == 1,
         "flattened SRT did not evaluate its canonical read once");
 }
@@ -139,7 +140,7 @@ void TestRawScalarComponentAlignment() {
   TestMemory memory_image{{{0x1000u, 0x12345678u}}};
   SrtRuntime runtime{.read_memory = ReadMemory, .userdata = &memory_image};
   std::vector<uint32_t> flat;
-  Check(WalkSrt(fixture.program, runtime, flat), "raw scalar SRT walk failed");
+  Check(SrtWalker(fixture.program, runtime).RefreshFlatBuffer(flat), "raw scalar SRT walk failed");
   Check(
       flat == std::vector<uint32_t>{0x12345678u} && memory_image.reads == 1,
       "raw scalar base, immediate, and offset were not aligned independently");
@@ -195,7 +196,7 @@ void TestNestedSrtWalk() {
   TestMemory memory_image{{{0x1000u, 0x2000u}, {0x2000u, 0xabcdef01u}}};
   SrtRuntime runtime{.read_memory = ReadMemory, .userdata = &memory_image};
   std::vector<uint32_t> flat;
-  Check(WalkSrt(fixture.program, runtime, flat), "nested SRT walk failed");
+  Check(SrtWalker(fixture.program, runtime).RefreshFlatBuffer(flat), "nested SRT walk failed");
   Check(flat == std::vector<uint32_t>({0x2000u, 0xabcdef01u}),
         "nested typed SRT reads were not evaluated in dependency order");
 }
@@ -218,7 +219,7 @@ void TestShaderBaseAndUserData() {
   SrtRuntime runtime{.user_data = user_data,
                      .shader_base = 0x12345678abcdef00ull};
   DescriptorValue result;
-  Check(EvaluateDescriptorSource(fixture.program, 0, runtime, result),
+  Check(SrtWalker(fixture.program, runtime).EvaluateDescriptor(0, result),
         "shader-relative descriptor evaluation failed");
   Check(result.dword_count == 3 && result.dwords[0] == 0xabcdef00u &&
             result.dwords[1] == 0x12345678u && result.dwords[2] == 0x24u,
@@ -243,7 +244,7 @@ void TestCarryAndBitFields() {
       {.dwords = {low, high, inserted, sign}, .dword_count = 4});
 
   DescriptorValue result;
-  Check(EvaluateDescriptorSource(fixture.program, 0, {}, result),
+  Check(SrtWalker(fixture.program, {}).EvaluateDescriptor(0, result),
         "carry and bit-field descriptor evaluation failed");
   Check(result.dwords[0] == 1u && result.dwords[1] == 1u &&
             result.dwords[2] == 0x89abcdefu && result.dwords[3] == 0xffffffffu,
@@ -267,14 +268,11 @@ void TestInvariantAndDivergentPhi() {
       {.dwords = {Value(&divergent)}, .dword_count = 1});
 
   DescriptorValue result;
-  Check(EvaluateDescriptorSource(fixture.program, 0, {}, result) &&
+  Check(SrtWalker(fixture.program, {}).EvaluateDescriptor(0, result) &&
             result.dwords[0] == 7u,
         "loop-invariant typed phi was rejected");
-  result.dword_count = 4;
-  result.dwords[0] = 0xdeadbeefu;
-  Check(!EvaluateDescriptorSource(fixture.program, 1, {}, result) &&
-            result.dword_count == 4 && result.dwords[0] == 0xdeadbeefu,
-        "divergent phi did not fail transactionally");
+  Check(!SrtWalker(fixture.program, {}).EvaluateDescriptor(1, result),
+        "divergent phi was accepted");
 }
 
 void TestControlDependentStandaloneLoadStaysTyped() {
@@ -314,7 +312,7 @@ void TestRuntime64BitDescriptorOps() {
       {.dwords = {low, high}, .dword_count = 2});
 
   DescriptorValue result;
-  Check(EvaluateDescriptorSource(fixture.program, 0, {}, result) &&
+  Check(SrtWalker(fixture.program, {}).EvaluateDescriptor(0, result) &&
             result.dwords[0] == 0xabcdu && result.dwords[1] == 0x1235u,
         "64-bit typed descriptor arithmetic evaluation is incorrect");
 }
@@ -389,11 +387,11 @@ void TestUniformFirstLaneSamplerLod() {
   user_data[6] = 3u;
   SrtRuntime runtime{.user_data = user_data};
   DescriptorValue result;
-  Check(EvaluateDescriptorSource(fixture.program, 0, runtime, result) &&
+  Check(SrtWalker(fixture.program, runtime).EvaluateDescriptor(0, result) &&
             result.dwords[0] == 0x00300300u,
         "uniform sampler LOD evaluated incorrectly");
   user_data[6] = 20u;
-  Check(EvaluateDescriptorSource(fixture.program, 0, runtime, result) &&
+  Check(SrtWalker(fixture.program, runtime).EvaluateDescriptor(0, result) &&
             result.dwords[0] == 0x00ffffffu,
         "uniform sampler LOD clamp evaluated incorrectly");
 }
@@ -455,7 +453,7 @@ void TestConstantBufferBounds() {
   TestMemory memory_image{{{0x300cu, 0xa5a5a5a5u}}};
   SrtRuntime runtime{.read_memory = ReadMemory, .userdata = &memory_image};
   std::vector<uint32_t> flat;
-  Check(WalkSrt(fixture.program, runtime, flat) &&
+  Check(SrtWalker(fixture.program, runtime).RefreshFlatBuffer(flat) &&
             flat == std::vector<uint32_t>{0xa5a5a5a5u},
         "constant-buffer SRT walk failed");
 
@@ -470,10 +468,8 @@ void TestConstantBufferBounds() {
   overflow.Emit(ValueOpcode::GetBufferResource,
                 {overflow_read, Value(0u), Value(16u), Value(0u)});
   overflow.Plan();
-  flat = {0x55u};
-  Check(!WalkSrt(overflow.program, runtime, flat) &&
-            flat == std::vector<uint32_t>{0x55u},
-        "out-of-bounds constant-buffer walk was not transactional");
+  Check(!SrtWalker(overflow.program, runtime).RefreshFlatBuffer(flat),
+        "out-of-bounds constant-buffer walk was accepted");
 }
 
 void TestReadLaneElimination() {
@@ -560,10 +556,8 @@ void TestUndefinedRuntimeValueFails() {
   fixture.program.descriptor_sources.push_back(
       {.dwords = {undef}, .dword_count = 1});
   DescriptorValue result;
-  result.dword_count = 3;
-  Check(!EvaluateDescriptorSource(fixture.program, 0, {}, result) &&
-            result.dword_count == 3,
-        "undefined typed descriptor source did not fail transactionally");
+  Check(!SrtWalker(fixture.program, {}).EvaluateDescriptor(0, result),
+        "undefined typed descriptor source was accepted");
 }
 
 } // namespace

tests/shaderCfgTests.cpp

@@ -107,11 +107,11 @@ ShaderRecompiler::CompileOptions MakeCompileOptions(ShaderType stage) {
   return options;
 }
 
-bool ReadHostTestMemory(void *, uint64_t address, uint32_t *value) {
-  if (address == 0 || value == nullptr) {
+bool ReadHostTestMemory(void *, uint64_t address, std::span<uint32_t> values) {
+  if (address == 0 || values.empty()) {
     return false;
   }
-  std::memcpy(value, reinterpret_cast<const void *>(address), sizeof(*value));
+  std::memcpy(values.data(), reinterpret_cast<const void *>(address), values.size_bytes());
   return true;
 }
 
@@ -153,14 +153,16 @@ void CompilePixelRuntime(const ShaderParams &params,
                          ShaderPixelInputInfo &input_info) {
   auto options = MakeCompileOptions(ShaderType::Pixel);
   options.shader_hash = params.hash;
-  options.user_data = params.user_data;
+  options.user_data = std::span(params.user_data).first(params.user_data_count);
 
   options.input_info.pixel = &input_info;
   auto result = RecompileForTest(params.code, options);
   static std::deque<ShaderRecompiler::IR::CompiledShaderInfo> programs;
+  static std::deque<ShaderRecompiler::IR::ResourceSnapshot> resources;
   programs.push_back(std::move(result.program).TakeCompiledInfo());
   input_info.stage.program = &programs.back();
-  input_info.stage.resources = std::move(result.resources);
+  resources.push_back(std::move(result.resources));
+  input_info.stage.resources = &resources.back();
 }
 
 template <typename InputInfo>
@@ -1154,20 +1156,17 @@ void TestNativeShaderResourceDependencies() {
       .info = program.info,
       .bindings = program.bindings,
   };
-  ShaderStageRuntime runtime{.program = &compiled, .resources = resources};
+  ShaderStageRuntime runtime{.program = &compiled, .resources = &resources};
   Check(HasShaderBufferWrites(runtime),
         "graphics/compute write predicate lost nonempty written buffers");
   set_buffer(0, 0, 16, 3);
   set_buffer(2, 0x2000, 0, 0);
-  runtime.resources = resources;
   Check(!HasShaderBufferWrites(runtime),
         "graphics/compute write predicate included null, empty, or read-only buffers");
   set_buffer(2, 0x2000, 0, 64);
-  runtime.resources = resources;
   Check(HasShaderBufferWrites(runtime),
         "graphics/compute write predicate lost a byte-addressed buffer");
   set_buffer(2, 0x2000, 0x3FFF, UINT32_MAX);
-  runtime.resources = resources;
   Check(HasShaderBufferWrites(runtime),
         "graphics/compute write predicate lost a maximum-size strided buffer");
 
@@ -1426,11 +1425,8 @@ void SetImageTestFormat(std::array<uint32_t, 64> *data, uint32_t srsrc,
   (*data)[format_dword] = static_cast<uint32_t>(format) << 20u;
 }
 
-bool ReadZeroTestMemory(void *, uint64_t, uint32_t *value) {
-  if (value == nullptr) {
-    return false;
-  }
-  *value = 0;
+bool ReadZeroTestMemory(void *, uint64_t, std::span<uint32_t> values) {
+  std::ranges::fill(values, 0u);
   return true;
 }
 
@@ -1679,45 +1675,14 @@ void TestNewShaderRecompilerSMovB32() {
   CheckSpirvBinaryValidates(result.spirv);
 }
 
-void TestNewShaderRecompilerSoppMarkers() {
+void TestShaderStageBarriers() {
   const uint32_t shader[] = {
-      EncodeSopp(0x00, 3),    // s_nop 3
-      EncodeSopp(0x0c, 0),    // s_waitcnt 0
-      EncodeSopp(0x10, 0x0f), // s_sendmsg 15
-      EncodeSopp(0x16, 0x2a), // s_ttracedata 42
-      EncodeSopp(0x20, 1),    // s_inst_prefetch 1
       EncodeSopp(0x0a, 0),    // s_barrier
       EncodeSopp(0x01, 0),    // s_endpgm
   };
 
-  auto options = MakeCompileOptions(ShaderType::Compute);
-  options.dump_ir = true;
-
-  auto result = RecompileForTest(shader, options);
-  Check((result.decoded_dump.find("s_nop 0x00000003") != std::string::npos),
-        "new decoder did not decode SOPP s_nop");
-  Check((result.decoded_dump.find("s_waitcnt 0x00000000") != std::string::npos),
-        "new decoder did not decode SOPP s_waitcnt");
-  Check((result.decoded_dump.find("s_sendmsg 0x0000000f") != std::string::npos),
-        "new decoder did not decode SOPP s_sendmsg");
-  Check((result.decoded_dump.find("s_ttracedata 0x0000002a") != std::string::npos),
-        "new decoder did not decode SOPP s_ttracedata");
-  Check((result.decoded_dump.find("s_inst_prefetch 0x00000001") != std::string::npos),
-        "new decoder did not decode SOPP s_inst_prefetch");
-  Check((result.decoded_dump.find("s_barrier") != std::string::npos),
-        "new decoder did not decode SOPP s_barrier");
-  Check((result.ir_dump.find("ControlNop null, 0x00000003") != std::string::npos),
-        "SOPP s_nop did not lower to an IR marker");
-  Check((result.ir_dump.find("Waitcnt null, 0x00000000") != std::string::npos),
-        "SOPP s_waitcnt did not lower to an IR marker");
-  Check((result.ir_dump.find("Sendmsg null, 0x0000000f") != std::string::npos),
-        "SOPP s_sendmsg did not lower to an IR marker");
-  Check((result.ir_dump.find("TtraceData null, 0x0000002a") != std::string::npos),
-        "SOPP s_ttracedata did not lower to an IR marker");
-  Check((result.ir_dump.find("InstPrefetch null, 0x00000001") != std::string::npos),
-        "SOPP s_inst_prefetch did not lower to an IR marker");
-  Check((result.ir_dump.find("Barrier null") != std::string::npos),
-        "SOPP s_barrier did not lower to an IR marker");
+  const auto result =
+      RecompileForTest(shader, MakeCompileOptions(ShaderType::Compute));
   Check(SpirvContainsOpcode(result.spirv, 224),
         "SPIR-V binary does not contain OpControlBarrier");
   Check(
@@ -1725,6 +1690,70 @@ void TestNewShaderRecompilerSoppMarkers() {
           result.spirv.end(),
       "SPIR-V barrier does not use workgroup acquire-release memory semantics");
   CheckSpirvBinaryValidates(result.spirv);
+
+  const auto vertex_result =
+      RecompileForTest(shader, MakeCompileOptions(ShaderType::Vertex));
+  CheckSpirvBinaryValidates(vertex_result.spirv);
+  Check(!SpirvContainsOpcode(vertex_result.spirv, 224),
+        "independent vertex invocations retained a workgroup barrier");
+}
+
+void TestNggVertexEntryState() {
+  using namespace ShaderRecompiler;
+  // PPSA03309 shader 5a39eb2021a5d2c1: retain its NGG prologue and replace
+  // the resource-dependent vertex body with a position export.
+  const uint32_t shader[] = {
+      0xbfa00003u, 0x93ebff03u, 0x00040018u, 0xbefe03c1u,
+      0x9380ff02u, 0x00090016u, 0x9381ff02u, 0x0009000cu,
+      0xbf8a0000u, 0xbf076b80u, 0xbf850003u, 0x8f6a8c00u,
+      0x887c6a01u, 0xbf900009u, 0xd7650001u, 0x000100c1u,
+      0xd7460001u, 0x04050a6bu, 0x7da80200u, 0xbf880002u,
+      0xf8000941u, 0x00000000u, 0xbf8cff0fu, 0xbefe03c1u,
+      0x7da80201u, EncodeSopp(0x08, 2),
+      EncodeExp0(0x0c, 0xf), EncodeExp1(5, 5, 5, 5), EncodeSopp(0x01),
+  };
+  HW::VertexShaderInfo regs{};
+  regs.es_regs.data_addr = reinterpret_cast<uint64_t>(shader);
+  ShaderUserData user_data{};
+  ShaderMappedData mapped{};
+  mapped.user_data = &user_data;
+  mapped.code_size_bytes = sizeof(shader);
+  ShaderMapUserData(regs.es_regs.data_addr, mapped);
+  std::vector<uint32_t> previous_key;
+  for (const uint32_t wave_size : {32u, 64u}) {
+    HW::Context context;
+    context.SetShaderStages(wave_size == 32u ? 0x00400000u : 0u);
+    ShaderVertexInputInfo input{};
+    const auto params = PrepareProgram(regs, context, HW::UserConfig{}, input);
+    Check(input.wave_size == wave_size,
+          "vertex preparation lost the native NGG wave size");
+    const auto key = MakeStageStaticKey(input);
+    Check(previous_key != key, "NGG wave sizes shared a shader cache key");
+    previous_key = key;
+    auto options = MakeCompileOptions(ShaderType::Vertex);
+    options.user_data_base = 8;
+    options.user_data = std::span(params.user_data).first(params.user_data_count);
+    options.input_info.vertex = &input;
+    options.wave_size = input.wave_size;
+    const auto result = RecompileForTest(params.code, options);
+    bool live_vertex_range = false;
+    for (const auto *block : result.program.blocks) {
+      for (const auto &inst : *block) {
+        if (inst.GetOpcode() == IR::ValueOpcode::UGreaterThan32) {
+          const auto count = inst.Arg(0).Resolve();
+          live_vertex_range |= count.IsImmediate() && count.U32() == wave_size;
+        }
+      }
+    }
+    Check(live_vertex_range && std::ranges::any_of(result.program.info.outputs,
+              [](const auto &output) {
+                return output.kind == IR::StageOutputKind::Position;
+              }),
+          "NGG launch counts suppressed the vertex export");
+    Check(result.program.wave_size == wave_size,
+          "NGG wave size was lost during translation");
+    CheckSpirvBinaryValidates(result.spirv);
+  }
 }
 
 void TestNewShaderRecompilerSopkWaitcntMarkers() {
@@ -3433,6 +3462,13 @@ void TestNewShaderRecompilerCapturedVop1SdwaByteConvert() {
   Check(SpirvContainsOpcode(result.spirv, 112),
         "captured SDWA byte conversion did not emit OpConvertUToF");
   CheckSpirvBinaryValidates(result.spirv);
+
+  const uint32_t byte_sext[] = {0x7e20a0f9u, 0x000b1412u};
+  ShaderRecompiler::Decoder::Instruction decoded;
+  ShaderRecompiler::Decoder::DecodeInstruction(byte_sext, 0, decoded);
+  Check(decoded.word_count == 2u &&
+            decoded.opcode == ShaderRecompiler::Decoder::Opcode::UNSUPPORTED,
+        "V_CVT_F16_U16 accepted unimplemented SDWA byte sign extension");
 }
 
 void TestNewShaderRecompilerVop1SdwaNotDestination() {
@@ -9606,7 +9642,7 @@ void TestMergedShaderUserDataSnapshot() {
   regs.gs_user_sgpr.value[0]++;
   ShaderVertexInputInfo second_input{};
   const auto second = PrepareProgram(regs, context, user_config, second_input);
-  Check(first.user_data.size() == 12 && second.user_data.size() == 12 &&
+  Check(first.user_data_count == 12 && second.user_data_count == 12 &&
             std::equal(front_data.begin(), front_data.end(), first.user_data.begin() + 8) &&
             second.user_data[8] == front_data[0] + 1,
         "merged shader parameters did not snapshot ordinary user SGPRs at s8");
@@ -9617,7 +9653,7 @@ void TestMergedShaderUserDataSnapshot() {
   CompileOptions options{};
   options.stage = ShaderType::Mesh;
   options.user_data_base = 0;
-  options.user_data = first.user_data;
+  options.user_data = std::span(first.user_data).first(first.user_data_count);
   options.input_info.vertex = &first_input;
   options.back_code = first.back_code;
   const auto translated = TranslateProgram(first.code, options);
@@ -9625,15 +9661,14 @@ void TestMergedShaderUserDataSnapshot() {
   Check(program.info.buffers.size() == 2 && program.srt_reads.size() == 4,
         "merged shader lost front user SGPRs or the back-stage SRT load");
   for (const auto *params : {&first, &second}) {
-    const IR::SrtRuntime runtime{.user_data = params->user_data,
+    const auto user_data = std::span(params->user_data).first(params->user_data_count);
+    const IR::SrtRuntime runtime{.user_data = user_data,
                                  .read_memory = ReadHostTestMemory};
     IR::DescriptorValue front_descriptor, back_descriptor;
     const auto &table = params == &first ? first_table : second_table;
-    Check(IR::EvaluateDescriptorSource(program, program.info.buffers[0].source,
-                                       runtime, front_descriptor) &&
-              IR::EvaluateDescriptorSource(program, program.info.buffers[1].source,
-                                             runtime, back_descriptor) &&
-              std::equal(params->user_data.begin() + 8, params->user_data.end(),
+    Check(IR::SrtWalker(program, runtime).EvaluateDescriptor(program.info.buffers[0].source, front_descriptor) &&
+              IR::SrtWalker(program, runtime).EvaluateDescriptor(program.info.buffers[1].source, back_descriptor) &&
+              std::equal(user_data.begin() + 8, user_data.end(),
                          front_descriptor.dwords.begin()) &&
               std::equal(table.begin(), table.end(), back_descriptor.dwords.begin()),
           "merged shader resource plan did not follow the current s0:s1 pointer and s8 data");
@@ -9651,12 +9686,27 @@ void TestMergedShaderUserDataSnapshot() {
     ShaderVertexInputInfo input{};
     const auto params = PrepareProgram(regs, context, user_config, input);
     Check(params.back_code.empty() && params.hash == XXH3_64bits(monolithic, sizeof(monolithic)) &&
-              input.mesh.scratch_size_dwords == 3 && params.user_data.size() == 12 &&
+              input.mesh.scratch_size_dwords == 3 && params.user_data_count == 12 &&
               params.user_data[0] == 0 && params.user_data[1] == 0 &&
-              std::equal(second.user_data.begin() + 8, second.user_data.end(),
+              std::equal(second.user_data.begin() + 8, second.user_data.begin() + second.user_data_count,
                          params.user_data.begin() + 8),
           "monolithic NGG shader used stale GS-back state or lost its s8 user data");
   }
+  regs.gs_regs.rsrc2.user_sgpr = HW::UserSgprInfo::SGPRS_MAX;
+  for (uint32_t i = 4; i < HW::UserSgprInfo::SGPRS_MAX; ++i) {
+    regs.gs_user_sgpr.value[i] = 0x10001000u + i;
+  }
+  ShaderVertexInputInfo full_input{};
+  const auto full = PrepareProgram(regs, context, user_config, full_input);
+  Check(full.user_data_count == 8u + HW::UserSgprInfo::SGPRS_MAX &&
+            std::ranges::all_of(std::span(full.user_data).first(8),
+                               [](uint32_t word) { return word == 0; }) &&
+            std::equal(std::begin(regs.gs_user_sgpr.value), std::end(regs.gs_user_sgpr.value),
+                       full.user_data.begin() + 8),
+        "merged shader did not preserve the full user-SGPR range after its reserved prefix");
+  regs.gs_user_sgpr.value[HW::UserSgprInfo::SGPRS_MAX - 1]++;
+  Check(full.user_data.back() == 0x10001000u + HW::UserSgprInfo::SGPRS_MAX - 1,
+        "merged shader parameters borrowed mutable register storage");
   context.SetMaxOutputPerSubgroup(256);
   context.SetGsMaxVertOut(8);
   user_config.SetPrimitiveType(Prospero::PrimitiveType::kTriFan);
@@ -12048,11 +12098,11 @@ void CheckFlattenedReadSlots(const ShaderRecompiler::IR::Program &program,
   }
 }
 
-bool ReadSrtHostDword(void *, uint64_t address, uint32_t *value) {
-  if (address == 0 || value == nullptr) {
+bool ReadSrtHostDword(void *, uint64_t address, std::span<uint32_t> values) {
+  if (address == 0 || values.empty()) {
     return false;
   }
-  std::memcpy(value, reinterpret_cast<const void *>(address), sizeof(*value));
+  std::memcpy(values.data(), reinterpret_cast<const void *>(address), values.size_bytes());
   return true;
 }
 
@@ -12061,18 +12111,17 @@ struct SrtHostRange {
   size_t count;
 };
 
-bool ReadSrtHostRangeDword(void *userdata, uint64_t address, uint32_t *value) {
+bool ReadSrtHostRangeDword(void *userdata, uint64_t address, std::span<uint32_t> values) {
   const auto *range = static_cast<const SrtHostRange *>(userdata);
-  if (range == nullptr || range->data == nullptr || range->count == 0 ||
-      value == nullptr) {
+  if (range == nullptr || range->data == nullptr || range->count == 0 || values.empty()) {
     return false;
   }
   const auto base = reinterpret_cast<uint64_t>(range->data);
   const auto size = range->count * sizeof(uint32_t);
-  if (address < base || address - base > size - sizeof(uint32_t)) {
+  if (address < base || values.size_bytes() > size || address - base > size - values.size_bytes()) {
     return false;
   }
-  std::memcpy(value, reinterpret_cast<const void *>(address), sizeof(*value));
+  std::memcpy(values.data(), reinterpret_cast<const void *>(address), values.size_bytes());
   return true;
 }
 
@@ -12241,8 +12290,7 @@ void TestTypedDescriptorRealCarryAndScalarLoads() {
   ShaderRecompiler::IR::SrtRuntime carry_runtime{carry_user_data, shader_base,
                                                  nullptr, nullptr};
   ShaderRecompiler::IR::DescriptorValue carry_value;
-  Check(ShaderRecompiler::IR::EvaluateDescriptorSource(
-            carry_ir, carry_source_index, carry_runtime, carry_value) &&
+  Check(ShaderRecompiler::IR::SrtWalker(carry_ir, carry_runtime).EvaluateDescriptor(carry_source_index, carry_value) &&
             carry_value.dwords[0] == static_cast<uint32_t>(expected_pc) &&
             carry_value.dwords[1] == static_cast<uint32_t>(expected_pc >> 32u),
         "S_GETPC shader-base or add/addc carry evaluation was incorrect");
@@ -12315,9 +12363,7 @@ void TestTypedDescriptorRealCarryAndScalarLoads() {
             TypedDescriptorSource(inline_sampler_ir,
                                   inline_sampler_ir.info.samplers[0].source) !=
                 nullptr &&
-            ShaderRecompiler::IR::EvaluateDescriptorSource(
-                inline_sampler_ir, inline_sampler_ir.info.samplers[0].source,
-                runtime, sampler) &&
+            ShaderRecompiler::IR::SrtWalker(inline_sampler_ir, runtime).EvaluateDescriptor(inline_sampler_ir.info.samplers[0].source, sampler) &&
             sampler.dwords[0] == 0 && sampler.dwords[1] == 0x00fff000u &&
             sampler.dwords[2] == 0x09500000u && sampler.dwords[3] == 0,
         "real inline sampler construction was unresolved or evaluated "
@@ -12353,7 +12399,7 @@ void TestSrtWalkerRealSmemTranslation() {
   std::vector<uint32_t> flat;
   const ShaderRecompiler::IR::SrtRuntime runtime{user_data, 0, ReadSrtHostDword,
                                                  nullptr};
-  const auto walked = ShaderRecompiler::IR::WalkSrt(ir, runtime, flat);
+  const auto walked = ShaderRecompiler::IR::SrtWalker(ir, runtime).RefreshFlatBuffer(flat);
   Check(walked, "SRT walk failed");
   Check(flat.size() == table.size() &&
             std::equal(flat.begin(), flat.end(), table.begin()),
@@ -12382,7 +12428,7 @@ void TestSrtWalkerVccBaseTranslation() {
   const ShaderRecompiler::IR::SrtRuntime runtime{user_data, 0, ReadSrtHostDword,
                                                  nullptr};
   std::vector<uint32_t> flat;
-  Check(ShaderRecompiler::IR::WalkSrt(ir, runtime, flat), "SRT walk failed");
+  Check(ShaderRecompiler::IR::SrtWalker(ir, runtime).RefreshFlatBuffer(flat), "SRT walk failed");
   Check(flat.size() == table.size() &&
             std::equal(flat.begin(), flat.end(), table.begin()),
         "typed SSA lost an SMEM base copied through VCC");
@@ -12410,18 +12456,15 @@ void TestSrtWalkerRealSBufferTranslation() {
   std::vector<uint32_t> flat;
   const ShaderRecompiler::IR::SrtRuntime runtime{user_data, 0, ReadSrtHostDword,
                                                  nullptr};
-  const auto walked = ShaderRecompiler::IR::WalkSrt(ir, runtime, flat);
+  const auto walked = ShaderRecompiler::IR::SrtWalker(ir, runtime).RefreshFlatBuffer(flat);
   Check(walked, "SRT walk failed");
   Check(flat.size() == 4 &&
             std::equal(flat.begin(), flat.end(), table.begin() + 1),
         "real S_BUFFER_LOAD walk used the wrong final alignment");
 
   user_data[10] = 4 * sizeof(uint32_t);
-  const auto flat_before_failure = flat;
-  const auto bounds_walked = ShaderRecompiler::IR::WalkSrt(ir, runtime, flat);
+  const auto bounds_walked = ShaderRecompiler::IR::SrtWalker(ir, runtime).RefreshFlatBuffer(flat);
   Check(!bounds_walked, "real S_BUFFER_LOAD walk ignored descriptor bounds");
-  Check(flat == flat_before_failure,
-        "failed real S_BUFFER_LOAD walk changed the prior flat snapshot");
   CheckFlattenedReadSlots(
       ir, 4, "real S_BUFFER_LOAD patch used the wrong flat offsets");
 
@@ -12435,7 +12478,7 @@ void TestSrtWalkerRealSBufferTranslation() {
                  static_cast<uint32_t>(std::size(negative_shader)),
                  negative_ir);
   user_data[10] = sizeof(table);
-  Check(!ShaderRecompiler::IR::WalkSrt(negative_ir, runtime, flat),
+  Check(!ShaderRecompiler::IR::SrtWalker(negative_ir, runtime).RefreshFlatBuffer(flat),
         "real S_BUFFER_LOAD walk accepted a negative immediate");
 }
 
@@ -12474,7 +12517,7 @@ void TestScalarMemorySourcesCapturedBeforeWrites() {
     const ShaderRecompiler::IR::SrtRuntime runtime{
         user_data, 0, ReadSrtHostRangeDword, &range};
     std::vector<uint32_t> flat;
-    Check(ShaderRecompiler::IR::WalkSrt(ir, runtime, flat), "SRT walk failed");
+    Check(ShaderRecompiler::IR::SrtWalker(ir, runtime).RefreshFlatBuffer(flat), "SRT walk failed");
     Check(flat.size() == table.size() &&
               std::equal(flat.begin(), flat.end(), table.begin()),
           "overlapping scalar-memory load did not capture its sources before "
@@ -12808,7 +12851,7 @@ void TestComputeLdsAllocationIdentity() {
           "COMPUTE_PGM_RSRC2 LDS allocation units were not decoded");
     auto options = MakeCompileOptions(ShaderType::Compute);
     options.shader_hash = params.hash;
-    options.user_data = params.user_data;
+    options.user_data = std::span(params.user_data).first(params.user_data_count);
     options.input_info.compute = &input_info;
     options.wave_size = input_info.wave_size;
 
@@ -12874,7 +12917,7 @@ void TestComputeLdsAllocationIdentity() {
         "AGC per-thread scratch size was not propagated");
   auto scratch_options = MakeCompileOptions(ShaderType::Compute);
   scratch_options.shader_hash = scratch_params.hash;
-  scratch_options.user_data = scratch_params.user_data;
+  scratch_options.user_data = std::span(scratch_params.user_data).first(scratch_params.user_data_count);
   scratch_options.input_info.compute = &scratch_info;
   scratch_options.wave_size = scratch_info.wave_size;
 
@@ -13443,6 +13486,8 @@ int main() {
   TestNewShaderRecompilerSpirvSizeBaselines();
   TestDemandDrivenSpirvDeclarations();
   TestNewShaderRecompilerSMovB32();
+  TestShaderStageBarriers();
+  TestNggVertexEntryState();
   TestNewShaderRecompilerClipDisabledPosition();
   TestNewShaderRecompilerAuxPositionExports();
   TestNewShaderRecompilerNativeWideScalarMemoryIr();