tests: fix macOS extended-memory alias regression check (#889)

tests: use a macOS-owned address for extended memory alias checks

README.md

@@ -72,8 +72,8 @@ or graphical glitches, so please include the version you tested when reporting a
       <img src="docs/screenshots/ps5-02.png" width="300" alt="Demon's Souls running in KytyPS5">
     </td>
     <td align="center">
-      <strong>Hellboy</strong><br>
-      <img src="docs/screenshots/ps5-06.png" width="300" alt="Hellboy running in KytyPS5">
+      <strong>UFC 5</strong><br>
+      <img src="docs/screenshots/ps5-06.png" width="300" alt="UFC 5 running in KytyPS5">
     </td>
   </tr>
 </table>

src/graphics/host_gpu/regionDefinitions.h

@@ -3,6 +3,7 @@
 
 #include "common/bitArray.h"
 #include "common/common.h"
+#include "kernel/memory.h"
 
 #include <compare>
 
@@ -10,7 +11,8 @@ namespace Libs::Graphics {
 
 constexpr uint64_t TRACKER_PAGE_SIZE    = 4ull * 1024ull;
 constexpr uint64_t TRACKER_REGION_SIZE  = 4ull * 1024ull * 1024ull;
-constexpr uint64_t TRACKER_ADDRESS_SIZE = 1ull << 40u;
+constexpr uint64_t TRACKER_ADDRESS_SIZE = 1ull << 44u;
+constexpr uint64_t LOWER_ADDRESS_SIZE   = 1ull << 40u;
 constexpr size_t   TRACKER_REGION_PAGES = TRACKER_REGION_SIZE / TRACKER_PAGE_SIZE;
 
 struct GuestRange {
@@ -19,8 +21,11 @@ struct GuestRange {
 
 	[[nodiscard]] constexpr bool Empty() const noexcept { return address == 0 && size == 0; }
 	[[nodiscard]] constexpr bool Valid() const noexcept {
-		return address != 0 && size != 0 && address < TRACKER_ADDRESS_SIZE &&
-		       size <= TRACKER_ADDRESS_SIZE - address;
+		if (address == 0 || size == 0) return false;
+		if (address < LOWER_ADDRESS_SIZE) return size <= LOWER_ADDRESS_SIZE - address;
+		constexpr auto base = LibKernel::Memory::kExtendedMemoryBase;
+		constexpr auto length = LibKernel::Memory::kExtendedMemorySize;
+		return address >= base && address - base < length && size <= length - (address - base);
 	}
 	[[nodiscard]] constexpr bool     ValidOrEmpty() const noexcept { return Empty() || Valid(); }
 	[[nodiscard]] constexpr uint64_t End() const noexcept { return address + size; }

src/graphics/host_gpu/renderer/cache/bufferCache.cpp

@@ -55,7 +55,8 @@ template <bool insert>
 void BufferCache::ChangeRegister(BufferId id) {
 	auto& buffer = m_slot_buffers[id];
 	PageTable::PageRange pages {};
-	EXIT_IF(!PageTable::TryGetPageRange(buffer.CpuAddress(), buffer.Size(), pages));
+	EXIT_IF(!(GuestRange {buffer.CpuAddress(), buffer.Size()}.Valid()) ||
+	        !PageTable::TryGetPageRange(buffer.CpuAddress(), buffer.Size(), pages));
 	for (size_t page = pages.first; page < pages.last_exclusive; ++page) {
 		if constexpr (insert) {
 			m_page_table[page] = id;
@@ -64,6 +65,7 @@ void BufferCache::ChangeRegister(BufferId id) {
 		}
 	}
 	const auto size_pages = pages.last_exclusive - pages.first;
+	const auto table_offset = PageIndex(buffer.CpuAddress()) * sizeof(vk::DeviceAddress);
 	if constexpr (insert) {
 		const auto [it, inserted] = m_buffers.emplace(buffer.CpuAddress(), id);
 		(void)it;
@@ -75,7 +77,7 @@ void BufferCache::ChangeRegister(BufferId id) {
 		for (uint64_t i = 0; i < size_pages; ++i) {
 			addresses.push_back(buffer.BufferDeviceAddress() + (i << CACHING_PAGEBITS));
 		}
-		WriteDataBuffer(m_bda_pagetable_buffer, pages.first * sizeof(vk::DeviceAddress),
+		WriteDataBuffer(m_bda_pagetable_buffer, table_offset,
 		                addresses.data(), addresses.size() * sizeof(vk::DeviceAddress));
 	} else {
 		const auto found = m_buffers.find(buffer.CpuAddress());
@@ -84,7 +86,7 @@ void BufferCache::ChangeRegister(BufferId id) {
 		EXIT_IF(buffer.Size() > m_total_used_memory);
 		m_total_used_memory -= buffer.Size();
 		m_lru_cache.Free(buffer.lru_id);
-		m_bda_pagetable_buffer.Fill(pages.first * sizeof(vk::DeviceAddress),
+		m_bda_pagetable_buffer.Fill(table_offset,
 		                            size_pages * sizeof(vk::DeviceAddress), 0);
 		buffer.is_deleted = true;
 	}
@@ -312,10 +314,14 @@ BufferCache::OverlapResult BufferCache::ResolveOverlaps(uint64_t vaddr, uint64_t
 			// Reserve space in the incoming stream's direction of growth.
 			// The old buffer extending left of the request predicts growth to the right, and vice versa.
 			if (expands_left) {
-				end += std::min(StreamLeapSize, PageTable::kAddressSpaceSize - end);
+				end += std::min(StreamLeapSize, (vaddr < LOWER_ADDRESS_SIZE ? LOWER_ADDRESS_SIZE
+				                                       : LibKernel::Memory::kExtendedMemoryBase +
+				                                             LibKernel::Memory::kExtendedMemorySize) - end);
 			}
 			if (expands_right) {
-				const auto minimum = CACHING_PAGESIZE * 2;
+				const auto minimum = vaddr < LOWER_ADDRESS_SIZE
+				                         ? CACHING_PAGESIZE * 2
+				                         : LibKernel::Memory::kExtendedMemoryBase;
 				if (begin > minimum) {
 					begin -= std::min(StreamLeapSize, begin - minimum);
 				}

src/graphics/host_gpu/renderer/cache/bufferCache.h

@@ -29,10 +29,22 @@ class BufferCache {
 public:
 	static constexpr uint32_t CACHING_PAGEBITS  = 14;
 	static constexpr uint64_t CACHING_PAGESIZE  = uint64_t {1} << CACHING_PAGEBITS;
-	static constexpr uint64_t CACHING_NUMPAGES  = uint64_t {1} << (40 - CACHING_PAGEBITS);
+	static constexpr uint64_t CACHING_NUMPAGES  = (LOWER_ADDRESS_SIZE + LibKernel::Memory::kExtendedMemorySize) >> CACHING_PAGEBITS;
 	static constexpr uint64_t BDA_PAGETABLE_SIZE =
 	    CACHING_NUMPAGES * sizeof(vk::DeviceAddress);
 
+	static constexpr uint64_t PageIndex(uint64_t address) {
+		return (address < LOWER_ADDRESS_SIZE
+		            ? address
+		            : address - LibKernel::Memory::kExtendedMemoryBase + LOWER_ADDRESS_SIZE) >>
+		       CACHING_PAGEBITS;
+	}
+	static constexpr uint64_t GuestAddress(uint64_t offset) {
+		return offset < LOWER_ADDRESS_SIZE
+		           ? offset
+		           : offset - LOWER_ADDRESS_SIZE + LibKernel::Memory::kExtendedMemoryBase;
+	}
+
 	BufferCache(GraphicContext& graphics, CommandScheduler& scheduler, PageManager& page_manager,
 	            TextureCache& texture_cache);
 	~BufferCache();
@@ -88,7 +100,7 @@ private:
 		bool                has_stream_leap;
 	};
 
-	using PageTable = MultiLevelPageTable<BufferId, CACHING_PAGEBITS, 40, 16>;
+	using PageTable = MultiLevelPageTable<BufferId, CACHING_PAGEBITS, 44, 20>;
 	static_assert(CACHING_PAGESIZE == (uint64_t {1} << PageTable::kPageBits));
 	void WriteDataBuffer(Buffer& buffer, uint64_t address, const void* source, uint64_t size);
 	void TouchBuffer(const Buffer& buffer);

src/graphics/host_gpu/renderer/cache/faultManager.cpp

@@ -134,8 +134,9 @@ void FaultManager::ProcessFaultBuffer() {
 		const auto* faults = std::bit_cast<const uint64_t*>(mapped);
 		const auto  count  = static_cast<uint32_t>(faults[0]);
 		for (uint32_t index = 1; index <= count; ++index) {
-			fault_ranges.Add(faults[index], BufferCache::CACHING_PAGESIZE);
-			LOGF("Accessed non-GPU cached memory at 0x%016" PRIx64 "\n", faults[index]);
+			const auto address = BufferCache::GuestAddress(faults[index]);
+			fault_ranges.Add(address, BufferCache::CACHING_PAGESIZE);
+			LOGF("Accessed non-GPU cached memory at 0x%016" PRIx64 "\n", address);
 		}
 		fault_ranges.ForEach([this](uint64_t start, uint64_t end) {
 			EXIT_IF(end - start > std::numeric_limits<uint32_t>::max());

src/graphics/host_gpu/renderer/cache/samplerCache.cpp

@@ -14,10 +14,10 @@ SamplerCache::~SamplerCache() {
 	}
 }
 
-vk::Sampler SamplerCache::GetSampler(const ShaderSamplerResource& r) {
+vk::Sampler SamplerCache::GetSampler(const ShaderSamplerResource& r, bool integer_border) {
 	Common::LockGuard lock(m_mutex);
 
-	const SamplerKey key {r.fields[0], r.fields[1], r.fields[2], r.fields[3]};
+	const SamplerKey key {r.fields[0], r.fields[1], r.fields[2], r.fields[3], integer_border};
 	if (auto iter = m_samplers.find(key); iter != m_samplers.end()) {
 		return iter->second;
 	}
@@ -93,23 +93,23 @@ vk::Sampler SamplerCache::GetSampler(const ShaderSamplerResource& r) {
 		return vk::SamplerAddressMode::eClampToBorder;
 	};
 
-	vk::BorderColor border = vk::BorderColor::eIntTransparentBlack;
+	vk::BorderColor border = integer_border ? vk::BorderColor::eIntTransparentBlack
+	                                       : vk::BorderColor::eFloatTransparentBlack;
 	switch (static_cast<Prospero::SamplerBorderColor>(r.BorderColorType())) {
-		case Prospero::SamplerBorderColor::kTransBlack:
-			border = vk::BorderColor::eIntTransparentBlack;
-			break;
+		case Prospero::SamplerBorderColor::kTransBlack: break;
 		case Prospero::SamplerBorderColor::kOpaqueBlack:
-			border = vk::BorderColor::eIntOpaqueBlack;
+			border = integer_border ? vk::BorderColor::eIntOpaqueBlack
+			                        : vk::BorderColor::eFloatOpaqueBlack;
 			break;
 		case Prospero::SamplerBorderColor::kOpaqueWhite:
-			border = vk::BorderColor::eIntOpaqueWhite;
+			border = integer_border ? vk::BorderColor::eIntOpaqueWhite
+			                        : vk::BorderColor::eFloatOpaqueWhite;
 			break;
 		case Prospero::SamplerBorderColor::kFromTable:
 			LOGF(
 			    "temporary: approximating table border color as transparent black, index = %" PRIu16
 			    "\n",
 			    r.BorderColorPtr());
-			border = vk::BorderColor::eIntTransparentBlack;
 			break;
 		default: EXIT("unknown border color: %d", static_cast<int>(r.BorderColorType()));
 	}

src/graphics/host_gpu/renderer/cache/samplerCache.h

@@ -24,10 +24,10 @@ public:
 	~SamplerCache();
 	KYTY_CLASS_NO_COPY(SamplerCache);
 
-	vk::Sampler GetSampler(const ShaderSamplerResource& r);
+	vk::Sampler GetSampler(const ShaderSamplerResource& r, bool integer_border);
 
 private:
-	using SamplerKey = std::array<uint32_t, 4>;
+	using SamplerKey = std::array<uint32_t, 5>;
 
 	struct SamplerKeyHash {
 		std::size_t operator()(const SamplerKey& key) const {

src/graphics/host_gpu/renderer/cache/textureCache.h

@@ -91,9 +91,9 @@ private:
 	};
 
 	using ImageIds       = InlinePageOwnerList<ImageId, 16>;
-	using ImagePageTable = MultiLevelPageTable<ImageIds, 20, 40, 10>;
+	using ImagePageTable = MultiLevelPageTable<ImageIds, 20, 44, 14>;
 
-	// Callers have validated the nonempty 40-bit range with TryGetPageRange.
+	// Callers have validated the nonempty 44-bit range with TryGetPageRange.
 	template <typename Func>
 	static void ForEachPage(uint64_t address, size_t size, Func&& func) {
 		using FuncReturn = typename std::invoke_result<Func, uint64_t>::type;

src/graphics/host_gpu/renderer/image/image.cpp

@@ -628,10 +628,8 @@ void Validate(const ImageInfo& info) {
 			break;
 		case ImageMetadataKind::Dcc:
 		case ImageMetadataKind::Cmask:
-			if (info.metadata.range.address == 0 ||
-			    info.metadata.range.address >= TRACKER_ADDRESS_SIZE ||
-			    (info.metadata.range.size != 0 &&
-			     info.metadata.range.size > TRACKER_ADDRESS_SIZE - info.metadata.range.address) ||
+			if (!GuestRange {info.metadata.range.address,
+			                 std::max<uint64_t>(info.metadata.range.size, 1)}.Valid() ||
 			    info.metadata.compression == VideoOutCompression::Unsupported) {
 				EXIT("invalid color metadata\n");
 			}

src/graphics/host_gpu/renderer/pipeline/descriptors.cpp

@@ -196,8 +196,8 @@ bool IsSupportedDepthTextureEncoding(const ShaderTextureResource& descriptor, bo
 	constexpr uint32_t htile_control = 0x00280000u;
 	const uint32_t expected_control  = htile_control | (descriptor.MsaaDepth() ? (1u << 10u) : 0u);
 	const auto     metadata_addr     = descriptor.MetaAddr() << 8u;
-	return metadata_control == expected_control && metadata_addr != 0 &&
-	       metadata_addr < TRACKER_ADDRESS_SIZE && (metadata_addr & 0x7fffu) == 0 &&
+	return metadata_control == expected_control && GuestRange {metadata_addr, 1}.Valid() &&
+	       (metadata_addr & 0x7fffu) == 0 &&
 	       descriptor.TileMode() == Prospero::TileMode::kDepth;
 }
 
@@ -738,14 +738,15 @@ static vk::Sampler NativeSampler(RenderContext&                       context,
                                  const ShaderRecompiler::IR::CompiledShaderInfo& program,
                                  uint32_t index,
                                  const ShaderRecompiler::IR::DescriptorValue& value) {
-	auto descriptor = DecodeNativeDescriptor<ShaderSamplerResource>(value);
-	if (!program.info.samplers[index].depth_compare) {
+	auto        descriptor = DecodeNativeDescriptor<ShaderSamplerResource>(value);
+	const auto& sampler = program.info.samplers[index];
+	if (!sampler.depth_compare) {
 		descriptor.fields[0] &= ~(0x7u << 12u);
 	}
-	if (program.info.samplers[index].force_point_filtering) {
+	if (sampler.force_point_filtering) {
 		descriptor.SetPointFiltering();
 	}
-	return context.GetSamplerCache().GetSampler(descriptor);
+	return context.GetSamplerCache().GetSampler(descriptor, sampler.integer_border);
 }
 
 static vk::DescriptorBufferInfo NativeUpload(RenderContext&            context,

src/graphics/host_gpu/renderer/pipeline/pipelineCache.cpp

@@ -29,7 +29,6 @@
 #include <span>
 #include <spirv-tools/libspirv.hpp>
 #include <string_view>
-#include <tuple>
 #include <utility>
 #include <vector>
 #include <xxhash.h>
@@ -116,33 +115,22 @@ void DumpShaderSpirv(const char* stage_name, uint64_t shader_hash,
 }
 
 void DumpShaderOriginal(const char* stage_name, uint64_t shader_hash,
-                        std::span<const uint32_t> code, const std::string& decoded_dump) {
+                        std::span<const uint32_t> code) {
 	if (!Config::GraphicsDebugDumpEnabled()) {
 		return;
 	}
 	EXIT_IF(code.empty());
 	static std::atomic_int id = 0;
-	const auto base = Config::GetShaderLogFolder() / "original" /
-	                  fmt::format("{:04d}_new_shader_{}_{:016x}", id++, stage_name, shader_hash);
-	Common::File::CreateDirectories(base.parent_path());
-	for (const auto& [suffix, data, size]: {
-	         std::tuple {".bin", static_cast<const void*>(code.data()), code.size_bytes()},
-	         std::tuple {".rdna2", static_cast<const void*>(decoded_dump.data()),
-	                     decoded_dump.size()},
-	     }) {
-		if (size == 0) {
-			continue;
-		}
-		auto path = base;
-		path += suffix;
-		Common::File file(path);
-		if (file.IsInvalid()) {
-			const auto path_text = Common::PathToString(path);
-			LOGF_COLOR(Log::Color::BrightRed, "Can't create file: %s\n", path_text.c_str());
-		} else {
-			file.Write(data, size);
-		}
+	const auto path = Config::GetShaderLogFolder() / "original" /
+	                  fmt::format("{:04d}_new_shader_{}_{:016x}.bin", id++, stage_name, shader_hash);
+	Common::File::CreateDirectories(path.parent_path());
+	Common::File file(path);
+	if (file.IsInvalid()) {
+		const auto path_text = Common::PathToString(path);
+		LOGF_COLOR(Log::Color::BrightRed, "Can't create file: %s\n", path_text.c_str());
+		return;
 	}
+	file.Write(code.data(), code.size_bytes());
 }
 
 bool ValidateShaderSpirv(const char* label, uint64_t shader_hash,
@@ -225,25 +213,13 @@ struct PipelineCache::ProgramCache {
 
 	static constexpr std::size_t MaxStaticKeyWords = 13 + ShaderVertexInputInfo::RES_MAX * 13;
 
-	Permutation CompilePermutation(const ShaderParams&                          params,
+	Permutation CompilePermutation(const char*                                  stage_name,
 	                               const ShaderRecompiler::CompileOptions&      options,
 	                               ShaderRecompiler::TranslateResult            translated,
 	                               ShaderRecompiler::IR::ResourceSpecialization specialization,
 	                               uint32_t push_data_start_dword) {
-		const char* stage_name = nullptr;
-		switch (options.stage) {
-			case ShaderType::Vertex: stage_name = "vs"; break;
-			case ShaderType::Mesh: stage_name = "ms"; break;
-			case ShaderType::Local: stage_name = "ls"; break;
-			case ShaderType::TessellationControl: stage_name = "hs"; break;
-			case ShaderType::TessellationEvaluation: stage_name = "ds"; break;
-			case ShaderType::Pixel: stage_name = "ps"; break;
-			case ShaderType::Compute: stage_name = "cs"; break;
-			default: EXIT("invalid pipeline shader stage\n");
-		}
 		auto result = ShaderRecompiler::CompileProgram(std::move(translated), options,
 		                                               specialization, push_data_start_dword);
-		DumpShaderOriginal(stage_name, options.shader_hash, params.code, result.decoded_dump);
 		if (!ValidateShaderSpirv(options.dump_label, options.shader_hash, result.spirv)) {
 			DumpShaderSpirv(stage_name, options.shader_hash, result.spirv);
 			EXIT("%s failed hash=0x%016" PRIx64 ": SPIR-V validation failed\n", options.dump_label,
@@ -321,14 +297,15 @@ struct PipelineCache::ProgramCache {
 			stage_input.compute = &input_info;
 		}
 		const char* label = nullptr;
+		const char* stage_name = nullptr;
 		switch (stage) {
-			case ShaderType::Vertex: label = "ShaderRecompiler VS"; break;
-			case ShaderType::Mesh: label = "ShaderRecompiler MS"; break;
-			case ShaderType::Local: label = "ShaderRecompiler LS"; break;
-			case ShaderType::TessellationControl: label = "ShaderRecompiler HS"; break;
-			case ShaderType::TessellationEvaluation: label = "ShaderRecompiler DS"; break;
-			case ShaderType::Pixel: label = "ShaderRecompiler PS"; break;
-			case ShaderType::Compute: label = "ShaderRecompiler CS"; break;
+			case ShaderType::Vertex: label = "ShaderRecompiler VS"; stage_name = "vs"; break;
+			case ShaderType::Mesh: label = "ShaderRecompiler MS"; stage_name = "ms"; break;
+			case ShaderType::Local: label = "ShaderRecompiler LS"; stage_name = "ls"; break;
+			case ShaderType::TessellationControl: label = "ShaderRecompiler HS"; stage_name = "hs"; break;
+			case ShaderType::TessellationEvaluation: label = "ShaderRecompiler DS"; stage_name = "ds"; break;
+			case ShaderType::Pixel: label = "ShaderRecompiler PS"; stage_name = "ps"; break;
+			case ShaderType::Compute: label = "ShaderRecompiler CS"; stage_name = "cs"; break;
 			default: EXIT("invalid pipeline shader stage\n");
 		}
 		ShaderRecompiler::CompileOptions options;
@@ -351,6 +328,7 @@ struct PipelineCache::ProgramCache {
 		} else {
 			options.wave_size = input_info.wave_size;
 		}
+		DumpShaderOriginal(stage_name, options.shader_hash, params.code);
 		auto translated = ShaderRecompiler::TranslateProgram(params.code, options);
 		if (translated.skip_dispatch) {
 			entry = programs.try_emplace(lookup_key, ShaderRecompiler::IR::ResourcePlan {}).first;
@@ -365,7 +343,7 @@ struct PipelineCache::ProgramCache {
 			    entry->second.specialization));
 		}
 		entry->second.permutations.push_back(CompilePermutation(
-		    params, options, std::move(translated), entry->second.specialization, push_data_cursor));
+		    stage_name, options, std::move(translated), entry->second.specialization, push_data_cursor));
 		const auto& permutation = entry->second.permutations.back();
 		input_info.stage = {.program = &permutation.program, .resources = &entry->second.resources};
 		permutation.program.bindings.AdvancePushData(push_data_cursor);

src/graphics/host_gpu/renderer/renderDraw.cpp

@@ -575,12 +575,17 @@ RenderState RenderExecutor::AcquireRenderTargets(CommandBuffer& buffer, RenderCo
 	return state;
 }
 
-static bool DrawHasActivePixelShader(const CommandBuffer& buffer) {
-	const auto& ctx              = buffer.GetRegisters();
-	const auto& sh_regs          = ctx.GetShaderRegisters();
-	const bool  has_color_output = (ctx.GetRenderTargetMask() & sh_regs.m_cbShaderMask) != 0;
-	return buffer.GetShaders().GetPs().ps_regs.data_addr != 0 &&
-	       (has_color_output || PixelShaderHasDepthOrCoverageSideEffects(sh_regs));
+static uint32_t DrawColorOutputMask(const HW::Context& ctx) {
+	const auto& sh_regs     = ctx.GetShaderRegisters();
+	const auto  write_mask  = ctx.GetRenderTargetMask() & sh_regs.m_cbShaderMask;
+	uint32_t    output_mask = 0;
+	for (uint32_t slot = 0; slot < RENDER_COLOR_ATTACHMENTS_MAX; slot++) {
+		if (sh_regs.target_output_mode[slot] != 0 &&
+		    render_target_mask_slot(write_mask, slot) != 0) {
+			output_mask |= 1u << slot;
+		}
+	}
+	return output_mask;
 }
 
 enum class CbColorMode : uint8_t {
@@ -847,7 +852,7 @@ static bool ResolvePrimitiveRestart(const CommandBuffer& buffer,
 }
 
 static void RefreshShaders(CommandBuffer& buffer, const DrawCallInfo& draw,
-                           DrawRenderState& state) {
+                           uint32_t color_output_mask, DrawRenderState& state) {
 	auto& ctx    = buffer.GetRegisters();
 	auto& sh_ctx = buffer.GetShaders();
 
@@ -861,7 +866,7 @@ static void RefreshShaders(CommandBuffer& buffer, const DrawCallInfo& draw,
 	    target_export_mapping {};
 	for (uint32_t slot = 0; slot < RENDER_COLOR_ATTACHMENTS_MAX; slot++) {
 		const auto& rt = ctx.GetRenderTarget(slot);
-		if (rt.base.addr != 0 && render_target_mask_slot(ctx.GetRenderTargetMask(), slot) != 0) {
+		if ((color_output_mask & (1u << slot)) != 0 && rt.base.addr != 0) {
 			target_export_mapping[slot] =
 			    TextureGetRenderTargetFormat(rt.info.format, rt.info.channel_type,
 			                                 rt.info.channel_order)
@@ -880,8 +885,12 @@ static void RefreshShaders(CommandBuffer& buffer, const DrawCallInfo& draw,
 bool RenderExecutor::PrepareDrawRenderState(CommandBuffer& buffer, const DrawCallInfo& draw,
                                             uint32_t            render_target_slice_offset,
 	                                        DrawRenderState& state) {
-	state.ps_active = DrawHasActivePixelShader(buffer);
-	RefreshShaders(buffer, draw, state);
+	const auto& shader_regs       = buffer.GetRegisters().GetShaderRegisters();
+	const auto  color_output_mask = DrawColorOutputMask(buffer.GetRegisters());
+	state.ps_active = buffer.GetShaders().GetPs().ps_regs.data_addr != 0 &&
+	                  (color_output_mask != 0 ||
+	                   PixelShaderHasDepthOrCoverageSideEffects(shader_regs));
+	RefreshShaders(buffer, draw, color_output_mask, state);
 	uint32_t mrt_mask = 0;
 	if (state.ps_active) {
 		for (const auto& output: state.ps_input_info.stage.program->info.outputs) {
@@ -890,6 +899,7 @@ bool RenderExecutor::PrepareDrawRenderState(CommandBuffer& buffer, const DrawCal
 			}
 		}
 	}
+	mrt_mask &= color_output_mask;
 	if (draw.IsIndexed()) {
 		LogDrawPhase(draw.Name(), "ResolveRenderColorTarget");
 	}

src/graphics/shader/recompiler/ShaderRecompiler.cpp

@@ -15,7 +15,6 @@
 #include "graphics/shader/recompiler/ir/passes/ResourceMaterialization.h"
 #include "graphics/shader/recompiler/ir/passes/ResourceTracking.h"
 #include "graphics/shader/recompiler/ir/passes/ShaderInfoCollection.h"
-#include "graphics/shader/recompiler/ir/passes/SrtWalker.h"
 #include "graphics/shader/recompiler/ir/passes/SsaRewrite.h"
 
 #include <algorithm>
@@ -548,30 +547,40 @@ TranslateResult TranslateProgram(std::span<const uint32_t> code, const CompileOp
 
 	LOGF("%s phase begin: stage=%s hash=0x%016" PRIx64 " CFG BuildGraph\n", GetDumpLabel(options),
 	     StageName(options.stage), options.shader_hash);
-	auto cfg = CFG::BuildGraph(decoded);
+	auto native_cfg = CFG::BuildGraph(decoded);
+	CFG::Graph structured_cfg;
+	auto* selected_cfg = &native_cfg;
 	LOGF("%s phase end: stage=%s hash=0x%016" PRIx64 " CFG BuildGraph blocks=%" PRIu64
 	     " loops=%" PRIu64 " back_edges=%" PRIu64 " elapsed_ms=%" PRIu64 "\n",
 	     GetDumpLabel(options), StageName(options.stage), options.shader_hash,
-	     static_cast<uint64_t>(cfg.blocks.size()), static_cast<uint64_t>(cfg.natural_loops.size()),
-	     static_cast<uint64_t>(cfg.back_edges.size()), phase_ms());
-	if (cfg.irreducible) {
-		LogDispatcherFallback(options, cfg, "build");
+	     static_cast<uint64_t>(native_cfg.blocks.size()),
+	     static_cast<uint64_t>(native_cfg.natural_loops.size()),
+	     static_cast<uint64_t>(native_cfg.back_edges.size()), phase_ms());
+	if (native_cfg.irreducible) {
+		LogDispatcherFallback(options, native_cfg, "build");
 	} else {
 		LOGF("%s phase begin: stage=%s hash=0x%016" PRIx64 " CFG Structurize\n",
 		     GetDumpLabel(options), StageName(options.stage), options.shader_hash);
-		if (!CFG::Structurize(cfg)) {
-			LogDispatcherFallback(options, cfg, "structurize");
+		structured_cfg = CFG::Structurize(native_cfg);
+		if (structured_cfg.unsupported) {
+			native_cfg.unsupported = true;
+			native_cfg.failure_kind = structured_cfg.failure_kind;
+			native_cfg.failure_block = structured_cfg.failure_block;
+			native_cfg.unsupported_reason = structured_cfg.unsupported_reason;
+			LogDispatcherFallback(options, native_cfg, "structurize");
 		} else {
+			selected_cfg = &structured_cfg;
 			LOGF("%s structured CFG success: blocks=%" PRIu64 "\n", GetDumpLabel(options),
-			     static_cast<uint64_t>(cfg.blocks.size()));
+			     static_cast<uint64_t>(selected_cfg->blocks.size()));
 		}
 		LOGF("%s phase end: stage=%s hash=0x%016" PRIx64 " CFG Structurize blocks=%" PRIu64
 		     " loops=%" PRIu64 " elapsed_ms=%" PRIu64 "\n",
 		     GetDumpLabel(options), StageName(options.stage), options.shader_hash,
-		     static_cast<uint64_t>(cfg.blocks.size()),
-		     static_cast<uint64_t>(cfg.natural_loops.size()), phase_ms());
+		     static_cast<uint64_t>(selected_cfg->blocks.size()),
+		     static_cast<uint64_t>(selected_cfg->natural_loops.size()), phase_ms());
 	}
 
+	const auto& cfg = *selected_cfg;
 	Frontend::EmbeddedFetchPlan embedded_fetch;
 	if ((options.stage == ShaderType::Vertex || options.stage == ShaderType::Local) &&
 	    options.input_info.vertex != nullptr && options.input_info.vertex->fetch_embedded) {
@@ -614,15 +623,21 @@ TranslateResult TranslateProgram(std::span<const uint32_t> code, const CompileOp
 		IR::EliminateDeadCode(ir.blocks);
 	}
 	LowerTessellationMemory(ir, options);
-	IR::BuildSrtPlan(ir);
-	IR::EliminateDeadCode(ir.blocks);
-	IR::TrackResources(ir);
+	std::string cfg_dump;
+	if (options.dump_ir) {
+		cfg_dump = CFG::GraphToString(cfg);
+		if (options.early_dump) {
+			LOGF("%s native IR before resource tracking:\n%s", GetDumpLabel(options),
+			     MakeIrDump(cfg_dump, ir).c_str());
+		}
+	}
+	IR::TrackResources(ir, decoded, native_cfg);
 	IR::EliminateDeadCode(ir.blocks);
 	TranslateResult result;
 	result.program = std::move(ir);
 	if (options.dump_ir) {
 		result.decoded_dump = std::move(decoded_dump);
-		result.cfg_dump     = CFG::GraphToString(cfg);
+		result.cfg_dump     = std::move(cfg_dump);
 	}
 	return result;
 }
@@ -634,6 +649,35 @@ CompileResult CompileProgram(TranslateResult translated, const CompileOptions& o
 	const auto emit_begin = std::chrono::steady_clock::now();
 	auto& ir = translated.program;
 	IR::ApplyResourceSpecialization(ir, specialization);
+	// The resource plan owns host descriptor evaluation now. Keep only dependencies consumed
+	// by GPU memory operations; bound descriptor dwords must not retain shader instructions.
+	for (auto& inst: ir.value_storage) {
+		inst.Invalidate();
+	}
+	for (auto* block: ir.blocks) {
+		for (auto& inst: *block) {
+			const auto op = inst.GetOpcode();
+			uint32_t first = 0;
+			if (op == IR::ValueOpcode::GetBufferResource) {
+				if (std::ranges::any_of(inst.Uses(), [&](const IR::Use& use) {
+					return ir.memory_info[use.user->Flags<IR::MemoryFlags>().index].kind ==
+					       IR::ResourceKind::IndirectBuffer;
+				})) {
+					continue;
+				}
+			} else if (op == IR::ValueOpcode::GetImageResource) {
+				const auto resource = inst.Flags<uint32_t>();
+				first = resource < ir.info.images.size() &&
+				                ir.info.images[resource].indirect_root == resource ? 1u : 0u;
+			} else if (op != IR::ValueOpcode::GetSamplerResource) {
+				continue;
+			}
+			for (size_t index = first; index < inst.NumArgs(); index++) {
+				inst.SetArg(index, IR::Value(0u));
+			}
+		}
+	}
+	ir.value_storage.clear();
 	IR::RemoveIdentities(ir.blocks);
 	IR::EliminateDeadCode(ir.blocks);
 

src/graphics/shader/recompiler/backend/spirv/SpirvEmitter.cpp

@@ -62,12 +62,7 @@ void ValidateNativeProgram(const IR::Program& program) {
 		Expect(Kind::BdaPagetable);
 		Expect(Kind::FaultBuffer);
 	}
-	const bool uses_flattened_runtime =
-	    !program.srt_reads.empty() ||
-	     std::ranges::any_of(program.info.images, [](const IR::ImageResource& image) {
-		     return image.indirect_search_iterations != 0u;
-	     });
-	if (uses_flattened_runtime) {
+	if (IR::UsesFlattenedSrt(program)) {
 		Expect(Kind::FlattenedSrt);
 	}
 	if (program.bindings.ShaderDataDwords() != 0 && !program.bindings.UsesPushData()) {

src/graphics/shader/recompiler/backend/spirv/spirvEmitterAlu.cpp

@@ -307,6 +307,11 @@ uint32_t EmitConvertF32F64(EmitterState& state, uint32_t arg0) {
 	                  Unary(state, spv::OpBitcast, TypeF32(state), clamped), converted));
 }
 
+uint32_t EmitConvertF64F32(EmitterState& state, uint32_t arg0) {
+	return EmitNative<spv::OpFConvert, IR::Type::F64>(state,
+	                                                  EmitFlushF32DenormToSignedZero(state, arg0));
+}
+
 uint32_t EmitCompositeExtractU64(EmitterState& state, uint32_t arg0, IR::Value arg1) {
 	return EmitNative<spv::OpCompositeExtract, IR::Type::U32>(state, arg0, arg1.U32());
 }

src/graphics/shader/recompiler/backend/spirv/spirvEmitterAluHelpers.cpp

@@ -203,9 +203,7 @@ uint32_t EmitFlushF32DenormToSignedZero(EmitterState& state, uint32_t value) {
 	const auto bits      = state.builder.AllocateId();
 	const auto abs_bits  = state.builder.AllocateId();
 	const auto sign_bits = state.builder.AllocateId();
-	const auto non_zero  = state.builder.AllocateId();
 	const auto subnormal = state.builder.AllocateId();
-	const auto flush     = state.builder.AllocateId();
 	const auto selected  = state.builder.AllocateId();
 	const auto ret       = state.builder.AllocateId();
 	state.builder.AddFunction(spv::OpBitcast, TypeU32(state), bits, value);
@@ -213,12 +211,9 @@ uint32_t EmitFlushF32DenormToSignedZero(EmitterState& state, uint32_t value) {
 	                          ConstantU32(state, 0x7fffffffu));
 	state.builder.AddFunction(spv::OpBitwiseAnd, TypeU32(state), sign_bits, bits,
 	                          ConstantU32(state, 0x80000000u));
-	state.builder.AddFunction(spv::OpINotEqual, TypeBool(state), non_zero, abs_bits,
-	                          ConstantU32(state, 0));
 	state.builder.AddFunction(spv::OpULessThan, TypeBool(state), subnormal, abs_bits,
 	                          ConstantU32(state, 0x00800000u));
-	state.builder.AddFunction(spv::OpLogicalAnd, TypeBool(state), flush, non_zero, subnormal);
-	state.builder.AddFunction(spv::OpSelect, TypeU32(state), selected, flush, sign_bits, bits);
+	state.builder.AddFunction(spv::OpSelect, TypeU32(state), selected, subnormal, sign_bits, bits);
 	state.builder.AddFunction(spv::OpBitcast, TypeF32(state), ret, selected);
 	return ret;
 }

src/graphics/shader/recompiler/backend/spirv/spirvEmitterFlow.cpp

@@ -636,6 +636,29 @@ uint32_t EmitDppUpdateU32(ValueEmitContext& ctx, const IR::Inst& inst) {
 	                                                ctx.Arg(inst, 1));
 }
 
+uint32_t EmitConditionRef(ValueEmitContext& ctx, const IR::Inst& inst) {
+	if (ctx.other_half == nullptr) return ctx.Arg(inst, 0);
+	// A native scalar branch makes one decision for both emulated wave halves.
+	if (ctx.half != 0) return ctx.other_half->Def(IR::Value(&inst));
+	const auto kind = inst.Flags<CFG::BranchCondition>();
+	if (kind == CFG::BranchCondition::ScalarInstruction) return ctx.Arg(inst, 0);
+	const auto ballot = ctx.Ballot(inst.Arg(0));
+	const auto low = ctx.state.builder.AllocateId();
+	const auto high = ctx.state.builder.AllocateId();
+	const auto combined = ctx.state.builder.AllocateId();
+	const auto result = ctx.state.builder.AllocateId();
+	ctx.state.builder.AddFunction(spv::OpCompositeExtract, TypeU32(ctx.state), low, ballot, 0);
+	ctx.state.builder.AddFunction(spv::OpCompositeExtract, TypeU32(ctx.state), high, ballot, 1);
+	const bool zero = kind == CFG::BranchCondition::ExecZero ||
+	                  kind == CFG::BranchCondition::VccZero || kind == CFG::BranchCondition::SccZero;
+	ctx.state.builder.AddFunction(zero ? spv::OpBitwiseAnd : spv::OpBitwiseOr,
+	                              TypeU32(ctx.state), combined, low, high);
+	ctx.state.builder.AddFunction(zero ? spv::OpIEqual : spv::OpINotEqual,
+	                              TypeBool(ctx.state), result, combined,
+	                              ConstantU32(ctx.state, zero ? ~0u : 0u));
+	return result;
+}
+
 uint32_t EmitBallot(ValueEmitContext& ctx, IR::Value predicate) {
 	return ctx.Ballot(predicate);
 }

src/graphics/shader/recompiler/backend/spirv/spirvEmitterInstructions.h

@@ -2,6 +2,8 @@
 
 #include "graphics/shader/recompiler/backend/spirv/spirvEmitterInternal.h"
 
+#include <bit>
+
 namespace Libs::Graphics::ShaderRecompiler::Spirv::Emitter {
 // Operand signatures distinguish SPIR-V IDs (uint32_t), raw literals/values
 // (IR::Value), and instructions whose metadata or operand loads must stay lazy.
@@ -29,7 +31,9 @@ uint32_t EmitConvertSigned32ToFloat(EmitterState& state, uint32_t arg0) {
 inline constexpr auto EmitConvertF32S32 = EmitConvertSigned32ToFloat<IR::Type::F32>;
 inline constexpr auto EmitConvertF64S32 = EmitConvertSigned32ToFloat<IR::Type::F64>;
 uint32_t              EmitConvertF32F64(EmitterState& state, uint32_t arg0);
+uint32_t              EmitConvertF64F32(EmitterState& state, uint32_t arg0);
 EMIT_NATIVE(ConvertF32U32, OpConvertUToF, F32, uint32_t)
+EMIT_NATIVE(ConvertF64U32, OpConvertUToF, F64, uint32_t)
 EMIT_NATIVE(CompositeConstructU64, OpCompositeConstruct, U64, uint32_t, uint32_t)
 EMIT_NATIVE(CompositeConstructU32x2, OpCompositeConstruct, U32x2, uint32_t, uint32_t)
 EMIT_NATIVE(CompositeConstructU32x3, OpCompositeConstruct, U32x3, uint32_t, uint32_t, uint32_t)
@@ -112,18 +116,34 @@ EMIT_NATIVE(LogicalOr, OpLogicalOr, U1, uint32_t, uint32_t)
 EMIT_NATIVE(LogicalAnd, OpLogicalAnd, U1, uint32_t, uint32_t)
 EMIT_NATIVE(LogicalXor, OpLogicalNotEqual, U1, uint32_t, uint32_t)
 EMIT_NATIVE(LogicalNot, OpLogicalNot, U1, uint32_t)
-EMIT_NATIVE(FPOrdEqual32, OpFOrdEqual, U1, uint32_t, uint32_t)
-EMIT_NATIVE(FPUnordEqual32, OpFUnordEqual, U1, uint32_t, uint32_t)
-EMIT_NATIVE(FPOrdNotEqual32, OpFOrdNotEqual, U1, uint32_t, uint32_t)
-EMIT_NATIVE(FPUnordNotEqual32, OpFUnordNotEqual, U1, uint32_t, uint32_t)
-EMIT_NATIVE(FPOrdLessThan32, OpFOrdLessThan, U1, uint32_t, uint32_t)
-EMIT_NATIVE(FPUnordLessThan32, OpFUnordLessThan, U1, uint32_t, uint32_t)
-EMIT_NATIVE(FPOrdGreaterThan32, OpFOrdGreaterThan, U1, uint32_t, uint32_t)
-EMIT_NATIVE(FPUnordGreaterThan32, OpFUnordGreaterThan, U1, uint32_t, uint32_t)
-EMIT_NATIVE(FPOrdLessThanEqual32, OpFOrdLessThanEqual, U1, uint32_t, uint32_t)
-EMIT_NATIVE(FPUnordLessThanEqual32, OpFUnordLessThanEqual, U1, uint32_t, uint32_t)
-EMIT_NATIVE(FPOrdGreaterThanEqual32, OpFOrdGreaterThanEqual, U1, uint32_t, uint32_t)
-EMIT_NATIVE(FPUnordGreaterThanEqual32, OpFUnordGreaterThanEqual, U1, uint32_t, uint32_t)
+template <spv::Op opcode>
+uint32_t EmitFloatCompare32(ValueEmitContext& ctx, const IR::Inst& inst) {
+	// SPIR-V's DenormFlushToZero mode does not require comparison operands to flush.
+	const bool flush = inst.Flags<IR::FPCompareFlags>().flush_input_denorms;
+	const auto operand = [&](size_t index) {
+		const auto value = inst.Arg(index);
+		if (flush && value.IsImmediate()) {
+			const auto bits = std::bit_cast<uint32_t>(value.F32Value());
+			return ConstantF32(ctx.state, (bits & 0x7fffffffu) < 0x00800000u
+			                                  ? bits & 0x80000000u : bits);
+		}
+		const auto id = ctx.Arg(inst, index);
+		return flush ? EmitFlushF32DenormToSignedZero(ctx.state, id) : id;
+	};
+	return EmitNative<opcode, IR::Type::U1>(ctx.state, operand(0), operand(1));
+}
+inline constexpr auto EmitFPOrdEqual32 = EmitFloatCompare32<spv::OpFOrdEqual>;
+inline constexpr auto EmitFPUnordEqual32 = EmitFloatCompare32<spv::OpFUnordEqual>;
+inline constexpr auto EmitFPOrdNotEqual32 = EmitFloatCompare32<spv::OpFOrdNotEqual>;
+inline constexpr auto EmitFPUnordNotEqual32 = EmitFloatCompare32<spv::OpFUnordNotEqual>;
+inline constexpr auto EmitFPOrdLessThan32 = EmitFloatCompare32<spv::OpFOrdLessThan>;
+inline constexpr auto EmitFPUnordLessThan32 = EmitFloatCompare32<spv::OpFUnordLessThan>;
+inline constexpr auto EmitFPOrdGreaterThan32 = EmitFloatCompare32<spv::OpFOrdGreaterThan>;
+inline constexpr auto EmitFPUnordGreaterThan32 = EmitFloatCompare32<spv::OpFUnordGreaterThan>;
+inline constexpr auto EmitFPOrdLessThanEqual32 = EmitFloatCompare32<spv::OpFOrdLessThanEqual>;
+inline constexpr auto EmitFPUnordLessThanEqual32 = EmitFloatCompare32<spv::OpFUnordLessThanEqual>;
+inline constexpr auto EmitFPOrdGreaterThanEqual32 = EmitFloatCompare32<spv::OpFOrdGreaterThanEqual>;
+inline constexpr auto EmitFPUnordGreaterThanEqual32 = EmitFloatCompare32<spv::OpFUnordGreaterThanEqual>;
 uint32_t              EmitFPIsNan32(EmitterState& state, uint32_t arg0);
 inline constexpr auto EmitFPCmpClass32 = EmitClassMaskF32;
 EMIT_NATIVE(FPAdd32, OpFAdd, F32, uint32_t, uint32_t)
@@ -182,6 +202,7 @@ uint32_t              EmitDppUpdateU32(ValueEmitContext& ctx, const IR::Inst& in
 uint32_t              EmitWqmU64(EmitterState& state, uint32_t value);
 uint32_t              EmitLaneId(EmitterState& state);
 uint32_t              EmitBallot(ValueEmitContext& ctx, IR::Value predicate);
+uint32_t              EmitConditionRef(ValueEmitContext& ctx, const IR::Inst& inst);
 uint32_t              EmitReadFirstLane(ValueEmitContext& ctx, const IR::Inst& inst);
 uint32_t              EmitReadLane(ValueEmitContext& ctx, const IR::Inst& inst);
 uint32_t              EmitWriteLane(ValueEmitContext& ctx, const IR::Inst& inst);

src/graphics/shader/recompiler/backend/spirv/spirvEmitterMemory.cpp

@@ -146,7 +146,12 @@ uint32_t GuestAddress(ValueEmitContext& ctx, const IR::Inst& inst, const IR::Mem
 			ctx.Fail(inst, "has no address base pair");
 			return ConstantDeviceAddress(state, 0);
 		}
-		const auto base = DeviceAddressFromWords(state, ctx.Arg(*handle, 0), ctx.Arg(*handle, 1));
+		auto base_low = ctx.Arg(*handle, 0);
+		if (mem.kind == IR::ResourceKind::ScalarAddress) {
+			base_low = Binary(state, spv::OpBitwiseAnd, TypeU32(state), base_low,
+			                  ConstantU32(state, ~3u));
+		}
+		const auto base = DeviceAddressFromWords(state, base_low, ctx.Arg(*handle, 1));
 		address         = Binary(state, spv::OpIAdd, TypeScalarU64(state), base,
 		                         Unary(state, spv::OpUConvert, TypeScalarU64(state), low));
 	}
@@ -970,7 +975,14 @@ void DefineGetBdaPointer(EmitterState& state) {
 	state.builder.AddFunction(spv::OpFunctionParameter, type, address);
 	EmitLabel(state, entry_label);
 
-	const auto page64        = Binary(state, spv::OpShiftRightLogical, type, address,
+	const auto extended = Binary(state, spv::OpUGreaterThanEqual, TypeBool(state), address,
+	                             ConstantDeviceAddress(state, LibKernel::Memory::kExtendedMemoryBase));
+	const auto packed = Select(state, type, extended,
+	                           Binary(state, spv::OpISub, type, address,
+	                                  ConstantDeviceAddress(state,
+	                                      LibKernel::Memory::kExtendedMemoryBase - LOWER_ADDRESS_SIZE)),
+	                           address);
+	const auto page64        = Binary(state, spv::OpShiftRightLogical, type, packed,
 	                                  ConstantDeviceAddress(state, BufferCache::CACHING_PAGEBITS));
 	const auto page          = Unary(state, spv::OpUConvert, TypeU32(state), page64);
 	const auto entry_pointer = state.builder.AllocateId();
@@ -1099,16 +1111,27 @@ uint32_t EmitAppendConsume(ValueEmitContext& ctx, const IR::Inst& inst) {
 	if (ctx.half == 1) {
 		return ctx.other_half->Def(IR::Value(const_cast<IR::Inst*>(&inst)));
 	}
-	const auto m0 = ctx.Arg(inst, 0);
-	const auto base =
-	    Binary(state, spv::OpShiftRightLogical, TypeU32(state), m0, ConstantU32(state, 16));
-	const auto size =
-	    Binary(state, spv::OpBitwiseAnd, TypeU32(state), m0, ConstantU32(state, 0xffffu));
-	const auto address = Binary(state, spv::OpIAdd, TypeU32(state), base,
-	                            ConstantU32(state, ctx.Memory(inst).offset));
+	const auto mem           = ctx.Memory(inst);
+	const auto offset        = mem.offset & 0xfffcu;
+	auto       address       = ConstantU32(state, offset);
+	uint32_t   region_bounds = 0;
+	if (mem.kind == IR::ResourceKind::Gds) {
+		const auto m0 = ctx.Arg(inst, 0);
+		const auto base =
+		    Binary(state, spv::OpShiftRightLogical, TypeU32(state), m0, ConstantU32(state, 16));
+		const auto size =
+		    Binary(state, spv::OpBitwiseAnd, TypeU32(state), m0, ConstantU32(state, 0xffffu));
+		address = Binary(state, spv::OpIAdd, TypeU32(state), base, address);
+		// The entire M0 region must fit the PS5's 48 KiB GDS partition.
+		region_bounds = AndCondition(
+		    state, Binary(state, spv::OpULessThan, TypeBool(state),
+		                  ConstantU32(state, offset + 3u), size),
+		    Binary(state, spv::OpULessThanEqual, TypeBool(state),
+		           Binary(state, spv::OpIAdd, TypeU32(state), base, size),
+		           ConstantU32(state, 0xc000u)));
+	}
 	const auto raw_index =
 	    Binary(state, spv::OpShiftRightLogical, TypeU32(state), address, ConstantU32(state, 2));
-	const auto mem    = ctx.Memory(inst);
 	const auto access = PrepareMemoryResourceAccess(state, mem);
 	const auto index  = EmitMemoryElementIndex(state, access, raw_index);
 	const auto exec   = ctx.Arg(inst, 1);
@@ -1127,19 +1150,17 @@ uint32_t EmitAppendConsume(ValueEmitContext& ctx, const IR::Inst& inst) {
 	        : first;
 	const auto is_first       = Binary(state, spv::OpIEqual, TypeBool(state),
 	                                   EmitSubgroupLocalInvocationId(state), source_lane);
-	const auto storage_bounds = EmitMemoryElementInBounds(state, access, index);
-	const auto m0_bounds =
-	    mem.kind == IR::ResourceKind::Gds
-	        ? Binary(state, spv::OpINotEqual, TypeBool(state), size, ConstantU32(state, 0))
-	        : Binary(state, spv::OpULessThan, TypeBool(state),
-	                 ConstantU32(state, ctx.Memory(inst).offset + 3u), size);
+	auto bounds = EmitMemoryElementInBounds(state, access, index);
+	if (mem.kind == IR::ResourceKind::Gds) {
+		bounds = AndCondition(state, bounds, region_bounds);
+	}
 	const auto condition = AndCondition(
 	    state, is_first,
 	    AndCondition(state,
 	                 state.lane_count == 2 ? Binary(state, spv::OpINotEqual, TypeBool(state), count,
 	                                                ConstantU32(state, 0))
 	                                       : exec,
-	                 AndCondition(state, storage_bounds, m0_bounds)));
+	                 bounds));
 	const auto atomic = EmitValueOrZeroIfCondition(state, condition, [&]() {
 		const auto value = state.builder.AllocateId();
 		state.builder.AddFunction(append ? spv::OpAtomicIAdd : spv::OpAtomicISub, TypeU32(state),
@@ -1173,10 +1194,11 @@ void EmitReadConstBuffer(ValueEmitContext& ctx, const IR::Inst& inst) {
 	if (mem.planning_only) return;
 	auto& state        = ctx.state;
 	mem.kind           = IR::ResourceKind::ScalarBuffer;
-	const auto address = Binary(state, spv::OpIAdd, TypeU32(state), ctx.Arg(inst, 1),
-	                            ConstantU32(state, mem.offset));
-	const auto index =
-	    Binary(state, spv::OpShiftRightLogical, TypeU32(state), address, ConstantU32(state, 2));
+	auto index = Binary(state, spv::OpShiftRightLogical, TypeU32(state), ctx.Arg(inst, 1),
+	                    ConstantU32(state, 2));
+	if (mem.offset >= 4u) {
+		index = Binary(state, spv::OpIAdd, TypeU32(state), index, ConstantU32(state, mem.offset >> 2u));
+	}
 	const auto access    = PrepareMemoryResourceAccess(state, mem);
 	const auto element   = EmitMemoryElementIndex(state, access, index);
 	const auto condition = EmitMemoryElementInBounds(state, access, element);
@@ -1198,6 +1220,8 @@ void EmitLoadMemory(ValueEmitContext& ctx, const IR::Inst& inst) {
 		value = LoadWideBuffer(ctx, inst, buffer_components);
 	else if (shared_components > 1u)
 		value = LoadWideShared(ctx, inst, shared_components);
+	else if (mem.kind == IR::ResourceKind::ScalarAddress)
+		value = LoadBdaDword(ctx, GuestAddress(ctx, inst, mem));
 	else if (address_info.access == IR::AddressAccess::Read &&
 	         mem.kind != IR::ResourceKind::Scratch)
 		value = LoadBda(ctx, GuestAddress(ctx, inst, mem), ctx.Arg(inst, inst.NumArgs() - 1),

src/graphics/shader/recompiler/backend/spirv/spirvEmitterModule.cpp

@@ -684,8 +684,8 @@ void DefineModule(EmitterState& state) {
 	state.builder.AddExecutionMode(state.main_func, spv::ExecutionModeSignedZeroInfNanPreserve,
 	                               32u);
 	if (state.requirements.float64) {
-		EXIT_NOT_IMPLEMENTED(state.program.stage != ShaderType::Compute);
-		EXIT_NOT_IMPLEMENTED(state.input_info.compute->float_mode != 0xc0);
+		EXIT_NOT_IMPLEMENTED(state.program.stage == ShaderType::Compute &&
+		                     state.input_info.compute->float_mode != 0xc0);
 		// MODE=0xc0 uses round-to-nearest-even and preserves FP64 input/output denormals.
 		state.builder.RequireCapability(spv::CapabilityFloat64);
 		state.builder.RequireCapability(spv::CapabilityRoundingModeRTE);

src/graphics/shader/recompiler/backend/spirv/spirvEmitterProgram.cpp

@@ -97,30 +97,6 @@ void EmitReturn(ValueEmitContext& ctx) {
 	ctx.state.builder.AddFunction(spv::OpReturn);
 }
 
-uint32_t BranchCondition(ValueEmitContext& ctx, const IR::BlockInfo& info) {
-	// Scalar-instruction conditions already test the full wave's raw register values.
-	if (ctx.other_half == nullptr ||
-	    info.terminator.condition == CFG::BranchCondition::ScalarInstruction ||
-	    info.terminator.condition == CFG::BranchCondition::GotoVariable) {
-		return ctx.Def(info.condition);
-	}
-	const auto ballot = ctx.Ballot(info.condition);
-	const auto low    = ctx.state.builder.AllocateId();
-	const auto high   = ctx.state.builder.AllocateId();
-	const auto result = ctx.state.builder.AllocateId();
-	ctx.state.builder.AddFunction(spv::OpCompositeExtract, TypeU32(ctx.state), low, ballot, 0);
-	ctx.state.builder.AddFunction(spv::OpCompositeExtract, TypeU32(ctx.state), high, ballot, 1);
-	const auto kind     = info.terminator.condition;
-	const bool zero     = kind == CFG::BranchCondition::ExecZero ||
-	                      kind == CFG::BranchCondition::VccZero ||
-	                      kind == CFG::BranchCondition::SccZero;
-	const auto combined =
-	    EmitBinaryU32(ctx.state, zero ? spv::OpBitwiseAnd : spv::OpBitwiseOr, low, high);
-	ctx.state.builder.AddFunction(zero ? spv::OpIEqual : spv::OpINotEqual, TypeBool(ctx.state),
-	                              result, combined, ConstantU32(ctx.state, zero ? ~0u : 0u));
-	return result;
-}
-
 void EmitStructuredTerminator(ValueEmitContext& ctx, const IR::Block* block,
                               const IR::BlockInfo& info) {
 	const auto& program = ctx.state.program;
@@ -160,7 +136,7 @@ void EmitStructuredTerminator(ValueEmitContext& ctx, const IR::Block* block,
 				EmitReturn(ctx);
 				return;
 			}
-			const auto condition = BranchCondition(ctx, info);
+			const auto condition = ctx.Def(info.condition);
 			emit_merge();
 			ctx.state.builder.AddFunction(spv::OpBranchConditional, condition,
 			                              ctx.Label(true_block), ctx.Label(false_block));
@@ -193,7 +169,7 @@ uint32_t EmitDispatcherNextPc(ValueEmitContext& ctx, const DispatcherFunctionSta
 			EmitDispatcherTarget(ctx, dispatcher, block, term.false_block);
 			const auto selected = ctx.state.builder.AllocateId();
 			ctx.state.builder.AddFunction(
-			    spv::OpSelect, TypeU32(ctx.state), selected, BranchCondition(ctx, info),
+			    spv::OpSelect, TypeU32(ctx.state), selected, ctx.Def(info.condition),
 			    ConstantU32(ctx.state, term.true_block), ConstantU32(ctx.state, term.false_block));
 			return selected;
 		}

src/graphics/shader/recompiler/frontend/cfg/ShaderCFG.h

@@ -18,10 +18,17 @@ enum class BranchCondition {
 	ExecZero,
 	ExecNonZero,
 	ScalarInstruction,
-	GotoVariable,
+	Expression,
 	Unknown
 };
 
+struct ConditionExpression {
+	enum class Op { Constant, Variable, Native, Not, Or };
+	Op       op;
+	uint32_t lhs;
+	uint32_t rhs = UINT32_MAX;
+};
+
 enum class TerminatorKind { Branch, ConditionalBranch, IndirectBranch, Return, Unsupported };
 
 enum class FailureKind {
@@ -48,8 +55,7 @@ struct Terminator {
 	std::vector<uint32_t> indirect_targets;
 	std::vector<uint32_t> indirect_selector_values;
 	std::vector<uint32_t> indirect_selector_targets;
-	uint32_t              goto_variable = UINT32_MAX;
-	int32_t               goto_value    = -1;
+	uint32_t              expression    = UINT32_MAX;
 	bool                  loop_header   = false;
 };
 
@@ -59,10 +65,11 @@ struct BasicBlock {
 	uint32_t              end_pc     = 0;
 	uint32_t              inst_begin = 0;
 	uint32_t              inst_end   = 0;
+	struct Assignment { uint32_t variable; uint32_t expression; };
+	std::vector<Assignment> assignments;
 	std::vector<uint32_t> predecessors;
 	std::vector<uint32_t> successors;
 	std::vector<uint32_t> dominators;
-	std::vector<uint32_t> post_dominators;
 	Terminator            terminator;
 };
 
@@ -73,12 +80,9 @@ struct BackEdge {
 };
 
 struct NaturalLoop {
-	uint32_t              header         = UINT32_MAX;
-	uint32_t              latch          = UINT32_MAX;
-	uint32_t              merge          = UINT32_MAX;
-	uint32_t              continue_block = UINT32_MAX;
+	uint32_t              header = UINT32_MAX;
+	uint32_t              latch  = UINT32_MAX;
 	std::vector<uint32_t> body_blocks;
-	std::vector<uint32_t> exit_blocks;
 };
 
 struct StronglyConnectedComponent {
@@ -89,6 +93,7 @@ struct StronglyConnectedComponent {
 
 struct Graph {
 	std::vector<BasicBlock>                 blocks;
+	std::vector<ConditionExpression>        expressions;
 	std::vector<BackEdge>                   back_edges;
 	std::vector<NaturalLoop>                natural_loops;
 	std::vector<StronglyConnectedComponent> components;
@@ -105,14 +110,12 @@ struct Graph {
 	const BasicBlock* FindBlockByPc(uint32_t pc) const;
 	BasicBlock*       FindBlockByPc(uint32_t pc);
 	bool              Dominates(uint32_t dominator, uint32_t block) const;
-	bool              PostDominates(uint32_t post_dominator, uint32_t block) const;
-	uint32_t          FindNearestCommonPostDominator(uint32_t block_a, uint32_t block_b) const;
 };
 
 Graph       BuildGraph(const Decoder::Program& program);
-// Commits structured control flow on success; preserves the original graph with
-// failure diagnostics on failure. failure_block is an original block ID or UINT32_MAX.
-bool        Structurize(Graph& graph);
+// Returns structured control flow or failure diagnostics without changing the native graph.
+// On failure, failure_block is an original block ID or UINT32_MAX.
+Graph       Structurize(const Graph& graph);
 std::string BranchConditionToString(BranchCondition condition);
 std::string FailureKindToString(FailureKind kind);
 std::string GraphToString(const Graph& graph);

src/graphics/shader/recompiler/frontend/decode/ShaderDecoder.h

@@ -156,6 +156,8 @@ enum class Opcode {
 	V_DOT2C_F32_F16,
 	V_CVT_F64_I32,
 	V_CVT_F32_F64,
+	V_CVT_F64_F32,
+	V_CVT_F64_U32,
 	V_CVT_F32_I32,
 	V_CVT_F32_U32,
 	V_CVT_U32_F32,

src/graphics/shader/recompiler/frontend/decode/VectorAluOps.cpp

@@ -89,6 +89,8 @@ constexpr OpcodeMap VOP1_OPCODE_LIST[] = {
     {0x02u, Opcode::V_READFIRSTLANE_B32},
     {0x04u, Opcode::V_CVT_F64_I32},
     {0x0fu, Opcode::V_CVT_F32_F64},
+    {0x10u, Opcode::V_CVT_F64_F32},
+    {0x16u, Opcode::V_CVT_F64_U32},
     {0x2fu, Opcode::V_RCP_F64},
     {0x05u, Opcode::V_CVT_F32_I32},
     {0x06u, Opcode::V_CVT_F32_U32},
@@ -151,6 +153,8 @@ constexpr OpcodeMap VOP3_ENCODED_VOP1_OPCODE_LIST[] = {
     {0x02u, Opcode::V_READFIRSTLANE_B32},
     {0x04u, Opcode::V_CVT_F64_I32},
     {0x0fu, Opcode::V_CVT_F32_F64},
+    {0x10u, Opcode::V_CVT_F64_F32},
+    {0x16u, Opcode::V_CVT_F64_U32},
     {0x2fu, Opcode::V_RCP_F64},
     {0x05u, Opcode::V_CVT_F32_I32},
     {0x06u, Opcode::V_CVT_F32_U32},
@@ -461,6 +465,7 @@ bool IsVopcCompareExec(Opcode opcode);
 bool IsVop1FloatSourceOpcode(Opcode opcode) {
 	switch (opcode) {
 		case Opcode::V_CVT_F32_F64:
+		case Opcode::V_CVT_F64_F32:
 		case Opcode::V_RCP_F64:
 		case Opcode::V_MOV_B32:
 		case Opcode::V_CVT_F32_F16:
@@ -721,6 +726,7 @@ void ApplyDppModifier(Operand& operand, uint32_t modifier, uint32_t encoding) {
 void DecodeVop1Dpp(uint32_t pc, std::span<const uint32_t> code, uint32_t word_index,
                    uint32_t opcode, uint32_t vdst, Instruction& inst) {
 	if (inst.opcode == Opcode::V_CVT_F64_I32 || inst.opcode == Opcode::V_CVT_F32_F64 ||
+	    inst.opcode == Opcode::V_CVT_F64_F32 || inst.opcode == Opcode::V_CVT_F64_U32 ||
 	    inst.opcode == Opcode::V_RCP_F64) {
 		SetUnsupported(inst, Family::VOP1, opcode, "FP64 instructions do not support DPP");
 		return;

src/graphics/shader/recompiler/frontend/translate/Compare.cpp

@@ -44,7 +44,8 @@ void Translator::EmitFloatCompare(const Decoder::Instruction& inst, IR::ValueOpc
 	    half ? IR::Value(ReadF16AsF32(inst.src0)) : ReadOperand(inst.src0, IR::Type::F32);
 	const auto rhs =
 	    half ? IR::Value(ReadF16AsF32(inst.src1)) : ReadOperand(inst.src1, IR::Type::F32);
-	EmitCompareResult(inst, IR::U1(ir.Emit(opcode, {lhs, rhs})), false, cmpx);
+	const IR::FPCompareFlags flags{.flush_input_denorms = !half && flush_f32_inputs};
+	EmitCompareResult(inst, IR::U1(ir.Emit(opcode, {lhs, rhs}, flags)), false, cmpx);
 }
 
 void Translator::EmitFloatOrderedCompare(const Decoder::Instruction& inst, bool ordered, bool cmpx) {

src/graphics/shader/recompiler/frontend/translate/Memory.cpp

@@ -235,6 +235,7 @@ Decoder::Operand MemorySourceAt(const Decoder::Instruction& decoded, uint32_t in
 				return index == 0u ? decoded.src0 : MakeImmediate(decoded.offset & 0xffffu);
 			case Decoder::Opcode::DS_CONSUME:
 			case Decoder::Opcode::DS_APPEND:
+				return decoded.gds ? MakeM0Operand() : MakeImmediate(0);
 			case Decoder::Opcode::DS_READ_ADDTID_B32: return MakeM0Operand();
 			case Decoder::Opcode::DS_WRITE_ADDTID_B32:
 				return index == 0u ? decoded.src1 : MakeM0Operand();
@@ -845,8 +846,7 @@ void Translator::DS_MINMAX_F32(const Decoder::Instruction& inst, IR::ValueOpcode
 void Translator::DS_APPEND_CONSUME(const Decoder::Instruction& inst, IR::ValueOpcode opcode) {
 	const auto memory = MemoryInfoFromDecoded(inst);
 	WriteOperand(inst.dst, ir.Emit(opcode,
-	                               {ReadU32(MemorySourceAt(inst, 0)), ir.GetExec(), ir.GetExecLo(),
-	                                ir.GetExecHi()},
+	                               {ReadU32(MemorySourceAt(inst, 0)), ir.GetExec()},
 	                               AddMemoryInfo(memory, inst.pc)));
 }
 

src/graphics/shader/recompiler/frontend/translate/Translate.cpp

@@ -317,6 +317,7 @@ void Translator::WriteRawU32(const Decoder::Operand& operand, IR::U32 value) {
 		case Decoder::OperandKind::Sgpr: {
 			const auto reg = static_cast<IR::ScalarReg>(operand.reg);
 			ir.SetScalarReg(reg, value);
+			program.scalar_writes.push_back({current_pc, reg});
 			ir.SetScalarMaskTag(reg, IR::U1(IR::Value(false)));
 			if (IR::RegIndex(reg) > 0u) {
 				ir.SetScalarMaskTag(static_cast<IR::ScalarReg>(IR::RegIndex(reg) - 1u),
@@ -699,11 +700,13 @@ std::array<IR::U32, 2> Translator::WriteMask(const Decoder::Operand& operand, IR
 				                    IR::U1(IR::Value(false)));
 			}
 			ir.SetScalarReg(reg, mask[0]);
+			program.scalar_writes.push_back({current_pc, reg});
 			// A wave32 VALU mask destination must not overwrite the neighboring SGPR.
 			if ((write_64 || program.wave_size == 64u) &&
 			    IR::RegIndex(reg) + 1u < IR::NumScalarRegs) {
 				const auto high = static_cast<IR::ScalarReg>(IR::RegIndex(reg) + 1u);
 				ir.SetScalarReg(high, mask[1]);
+				program.scalar_writes.push_back({current_pc, high});
 				ir.SetThreadBitScalarReg(high, IR::U1(IR::Value(false)));
 				ir.SetScalarMaskTag(high, IR::U1(IR::Value(false)));
 			}
@@ -746,55 +749,61 @@ void Translator::WriteCompareResult(const Decoder::Operand& operand, IR::U1 valu
 	WriteMask(operand, ir.LogicalAnd(ir.GetExec(), value));
 }
 
-void Translator::AddBranchCondition(const CFG::BasicBlock& source, IR::BlockInfo& info) {
-	if (source.terminator.goto_value >= 0) {
-		if (source.terminator.goto_variable == UINT32_MAX) {
-			EXIT("block %u sets an invalid goto variable", source.id);
+void Translator::AddBranchCondition(const CFG::Graph& graph, const CFG::BasicBlock& source,
+                                    IR::BlockInfo& info) {
+	const auto native_condition = [&](CFG::BranchCondition kind) -> IR::U1 {
+		IR::U1 condition;
+		switch (kind) {
+			case CFG::BranchCondition::Always: condition = IR::U1(IR::Value(true)); break;
+			case CFG::BranchCondition::SccZero: condition = ir.LogicalNot(ir.GetScc()); break;
+			case CFG::BranchCondition::SccNonZero: condition = ir.GetScc(); break;
+			case CFG::BranchCondition::VccZero: condition = ir.LogicalNot(ir.GetVcc()); break;
+			case CFG::BranchCondition::VccNonZero: condition = ir.GetVcc(); break;
+			case CFG::BranchCondition::ExecZero: condition = ir.LogicalNot(ir.GetExec()); break;
+			case CFG::BranchCondition::ExecNonZero: condition = ir.GetExec(); break;
+			case CFG::BranchCondition::ScalarInstruction:
+				EXIT_IF(instruction_branch_condition.IsEmpty());
+				condition = instruction_branch_condition;
+				break;
+			default: EXIT("block %u has an invalid native branch condition", source.id);
 		}
-		ir.SetGotoVariable(source.terminator.goto_variable,
-		                   IR::U1(IR::Value(source.terminator.goto_value != 0)));
+		return IR::U1(ir.Emit(IR::ValueOpcode::ConditionRef, {condition}, kind));
+	};
+	const auto expression = [&](auto&& self, uint32_t index) -> IR::U1 {
+		const auto& value = graph.expressions.at(index);
+		switch (value.op) {
+			case CFG::ConditionExpression::Op::Constant: return IR::U1(IR::Value(value.lhs != 0));
+			case CFG::ConditionExpression::Op::Variable: return ir.GetGotoVariable(value.lhs);
+			case CFG::ConditionExpression::Op::Native:
+				return native_condition(static_cast<CFG::BranchCondition>(value.lhs));
+			case CFG::ConditionExpression::Op::Not: return ir.LogicalNot(self(self, value.lhs));
+			case CFG::ConditionExpression::Op::Or:
+				return ir.LogicalOr(self(self, value.lhs), self(self, value.rhs));
+		}
+		EXIT("invalid CFG condition expression");
+	};
+	for (const auto& assignment: source.assignments) {
+		ir.SetGotoVariable(assignment.variable, expression(expression, assignment.expression));
 	}
-	if (source.terminator.kind == CFG::TerminatorKind::IndirectBranch) {
-		if (source.terminator.indirect_selector_code != UINT32_MAX) {
-			info.indirect_target = ReadScalarCode(source.terminator.indirect_selector_code);
-		} else if (source.terminator.indirect_pc_sgpr != UINT32_MAX) {
-			info.indirect_target =
-			    ir.GetScalarReg(static_cast<IR::ScalarReg>(source.terminator.indirect_pc_sgpr));
+	const auto& term = source.terminator;
+	if (term.kind == CFG::TerminatorKind::IndirectBranch) {
+		if (term.indirect_selector_code != UINT32_MAX) {
+			info.indirect_target = ReadScalarCode(term.indirect_selector_code);
+		} else if (term.indirect_pc_sgpr != UINT32_MAX) {
+			info.indirect_target = ir.GetScalarReg(static_cast<IR::ScalarReg>(term.indirect_pc_sgpr));
 		} else {
 			EXIT("block %u has no indirect branch selector", source.id);
 		}
 		ir.Emit(IR::ValueOpcode::ReferenceU32, {info.indirect_target});
 		return;
 	}
-	if (source.terminator.kind != CFG::TerminatorKind::ConditionalBranch) {
-		return;
+	if (term.kind == CFG::TerminatorKind::ConditionalBranch) {
+		const auto condition = term.expression != UINT32_MAX
+		                           ? expression(expression, term.expression)
+		                           : native_condition(term.condition);
+		info.condition = condition;
+		ir.Emit(IR::ValueOpcode::Reference, {condition});
 	}
-	// EXEC and VCC are invocation-local Boolean masks. Branching on that Boolean lets inactive
-	// invocations leave the region without reconstructing a host-subgroup mask.
-	IR::U1 condition;
-	switch (source.terminator.condition) {
-		case CFG::BranchCondition::Always: condition = IR::U1(IR::Value(true)); break;
-		case CFG::BranchCondition::SccZero: condition = ir.LogicalNot(ir.GetScc()); break;
-		case CFG::BranchCondition::SccNonZero: condition = ir.GetScc(); break;
-		case CFG::BranchCondition::VccZero: condition = ir.LogicalNot(ir.GetVcc()); break;
-		case CFG::BranchCondition::VccNonZero: condition = ir.GetVcc(); break;
-		case CFG::BranchCondition::ExecZero: condition = ir.LogicalNot(ir.GetExec()); break;
-		case CFG::BranchCondition::ExecNonZero: condition = ir.GetExec(); break;
-		case CFG::BranchCondition::ScalarInstruction:
-			EXIT_IF(instruction_branch_condition.IsEmpty());
-			condition = instruction_branch_condition;
-			break;
-		case CFG::BranchCondition::GotoVariable:
-			if (source.terminator.goto_variable == UINT32_MAX) {
-				EXIT("block %u reads an invalid goto variable", source.id);
-			}
-			condition = ir.GetGotoVariable(source.terminator.goto_variable);
-			break;
-		case CFG::BranchCondition::Unknown:
-			EXIT("block %u has an unknown branch condition", source.id);
-	}
-	info.condition = condition;
-	ir.Emit(IR::ValueOpcode::Reference, {condition});
 }
 
 namespace {
@@ -867,7 +876,9 @@ void IncludeInstructionVectorRegisters(const Decoder::Instruction& inst, uint32_
 	include_vector(inst.src2);
 	include_vector(inst.src3);
 	switch (inst.opcode) {
-		case Decoder::Opcode::V_CVT_F64_I32: include_vector(inst.dst, 2u); break;
+		case Decoder::Opcode::V_CVT_F64_I32:
+		case Decoder::Opcode::V_CVT_F64_F32:
+		case Decoder::Opcode::V_CVT_F64_U32: include_vector(inst.dst, 2u); break;
 		case Decoder::Opcode::V_FMA_F64: include_vector(inst.src2, 2u); [[fallthrough]];
 		case Decoder::Opcode::V_MUL_F64: include_vector(inst.src1, 2u); [[fallthrough]];
 		case Decoder::Opcode::V_RCP_F64: include_vector(inst.dst, 2u); [[fallthrough]];
@@ -1272,9 +1283,11 @@ IR::Program TranslateProgram(const Decoder::Program& decoded, const CFG::Graph&
 			                      builtin(IR::StageInputKind::InstanceIndex));
 		}
 	}
+	const bool flush_f32_inputs = options.stage == ShaderType::Compute &&
+	                             (options.input_info.compute->float_mode & 0x10u) == 0;
 	for (const auto& cfg_block: cfg.blocks) {
 		const auto typed_index = block_indices.at(cfg_block.id);
-		Translator translator(result, result.blocks[typed_index], vector_limit);
+		Translator translator(result, result.blocks[typed_index], vector_limit, flush_f32_inputs);
 		for (uint32_t index = cfg_block.inst_begin; index < cfg_block.inst_end; index++) {
 			const auto& instruction = decoded.instructions[index];
 			if (IsCodeTableLoad(cfg, instruction.pc)) {
@@ -1297,7 +1310,7 @@ IR::Program TranslateProgram(const Decoder::Program& decoded, const CFG::Graph&
 			}
 			translator.TranslateInstruction(instruction);
 		}
-		translator.AddBranchCondition(cfg_block, result.block_info[typed_index]);
+		translator.AddBranchCondition(cfg, cfg_block, result.block_info[typed_index]);
 	}
 	IR::ValidateProgram(result, false);
 	return result;

src/graphics/shader/recompiler/frontend/translate/Translator.h

@@ -9,13 +9,15 @@ namespace Libs::Graphics::ShaderRecompiler::Frontend {
 
 class Translator {
 public:
-	Translator(IR::Program& program, IR::Block* block, uint32_t vector_limit)
-	    : program(program), ir(block), current_vector_limit(vector_limit) {}
+	Translator(IR::Program& program, IR::Block* block, uint32_t vector_limit,
+	           bool flush_f32_inputs)
+	    : program(program), ir(block), current_vector_limit(vector_limit),
+	      flush_f32_inputs(flush_f32_inputs) {}
 
 	void TranslateInstruction(const Decoder::Instruction& inst);
 	void TranslateEmbeddedFetch(const Decoder::Instruction& inst, uint32_t attribute,
 	                            uint32_t component_count, const ShaderBufferResource& resource);
-	void AddBranchCondition(const CFG::BasicBlock& source, IR::BlockInfo& info);
+	void AddBranchCondition(const CFG::Graph& graph, const CFG::BasicBlock& source, IR::BlockInfo& info);
 
 private:
 	const Decoder::Operand& SourceAt(const Decoder::Instruction& inst, uint32_t index);
@@ -261,6 +263,7 @@ private:
 	Decoder::Opcode current_opcode       = Decoder::Opcode::UNKNOWN;
 	uint32_t        current_pc           = 0;
 	uint32_t        current_vector_limit = 1;
+	bool            flush_f32_inputs;
 };
 
 } // namespace Libs::Graphics::ShaderRecompiler::Frontend

src/graphics/shader/recompiler/frontend/translate/Vector.cpp

@@ -299,6 +299,8 @@ void Translator::EmitVector(const Decoder::Instruction& inst) {
 		case O::V_CVT_F32_I32: V_CVT_F32_I32(inst); return;
 		case O::V_CVT_F64_I32: return FloatUnary(inst, IR::ValueOpcode::ConvertF64S32);
 		case O::V_CVT_F32_F64: return FloatUnary(inst, IR::ValueOpcode::ConvertF32F64);
+		case O::V_CVT_F64_F32: return FloatUnary(inst, IR::ValueOpcode::ConvertF64F32);
+		case O::V_CVT_F64_U32: return FloatUnary(inst, IR::ValueOpcode::ConvertF64U32);
 		case O::V_CVT_U32_F32: V_CVT_U32_F32(inst); return;
 		case O::V_CVT_I32_F32: V_CVT_I32_F32(inst); return;
 		case O::V_CVT_F16_F32: V_CVT_F16_F32(inst); return;

src/graphics/shader/recompiler/ir/Program.cpp

@@ -81,7 +81,8 @@ bool EquivalentValue(const ResourcePlan& program, Value left, Value right,
 		    program.memory_info[li] != program.memory_info[ri]) {
 			return false;
 		}
-	} else if (lhs->Flags<uint64_t>() != rhs->Flags<uint64_t>()) {
+	} else if (lhs->GetOpcode() != ValueOpcode::ReadConst &&
+	           lhs->Flags<uint64_t>() != rhs->Flags<uint64_t>()) {
 		return false;
 	}
 	for (size_t index = 0; index < lhs->NumArgs(); index++) {

src/graphics/shader/recompiler/ir/ShaderIR.h

@@ -147,6 +147,7 @@ struct SamplerResource {
 	uint32_t first_use_pc          = 0;
 	bool     force_point_filtering = false;
 	bool     depth_compare         = false;
+	bool     integer_border        = false;
 
 	bool operator==(const SamplerResource& other) const = default;
 };
@@ -548,7 +549,7 @@ struct ResourcePlan {
 	std::vector<SrtRead>                srt_reads;
 	std::vector<uint8_t>                clean_flat_slots;
 	bool                                requires_specialization_memory = false;
-	bool                                has_address_writes = false;
+	bool                                capture_specialization_reads = false;
 	bool                                srt_plan_complete          = false;
 	bool                                resource_tracking_complete = false;
 	ShaderInfo                          info;
@@ -582,10 +583,12 @@ struct Program: ResourcePlan {
 	CFG::FailureKind              cfg_failure_kind    = CFG::FailureKind::None;
 	std::string                   fallback_reason;
 	std::vector<BlockInfo>        block_info;
+	struct ScalarWrite { uint32_t pc; ScalarReg reg; };
+	std::vector<ScalarWrite>      scalar_writes;
 	// Typed memory and export instructions reference shader-local metadata by dense index.
 	// Decoder-only details (such as NSA register numbers) have already become IR operands.
 	std::vector<ExportInfo>       export_info;
-	std::vector<Value>            dynamic_reads;
+	bool                          has_address_writes = false;
 	bool                          shader_info_complete = false;
 	BindingLayout                 bindings;
 	bool                          binding_layout_complete = false;

src/graphics/shader/recompiler/ir/opcodes/ValueOpcodes.h

@@ -52,6 +52,10 @@ struct PermlaneFlags {
 static_assert(sizeof(PermlaneFlags) <= sizeof(uint64_t));
 static_assert(std::is_trivially_copyable_v<PermlaneFlags>);
 
+struct FPCompareFlags {
+	bool flush_input_denorms = false;
+};
+
 struct MemoryFlags {
 	uint32_t index = 0;
 	uint32_t pc    = 0;

src/graphics/shader/recompiler/ir/opcodes/ValueOpcodes.inc

@@ -4,6 +4,7 @@ VALUE_OPCODE(Identity, Opaque, Opaque)
 VALUE_OPCODE(Void, Void)
 VALUE_OPCODE(Reference, Void, U1)
 VALUE_OPCODE(ReferenceU32, Void, U32)
+VALUE_OPCODE(ConditionRef, U1, U1)
 
 // Translation and register-state pseudo operations.
 VALUE_OPCODE(GetUserData, U32, ScalarReg)
@@ -65,7 +66,9 @@ VALUE_OPCODE(ConvertU32F32, U32, F32)
 VALUE_OPCODE(ConvertF32S32, F32, U32)
 VALUE_OPCODE(ConvertF64S32, F64, U32)
 VALUE_OPCODE(ConvertF32F64, F32, F64)
+VALUE_OPCODE(ConvertF64F32, F64, F32)
 VALUE_OPCODE(ConvertF32U32, F32, U32)
+VALUE_OPCODE(ConvertF64U32, F64, U32)
 VALUE_OPCODE(CompositeConstructU64, U64, U32, U32)
 VALUE_OPCODE(CompositeConstructU32x2, U32x2, U32, U32)
 VALUE_OPCODE(CompositeConstructU32x3, U32x3, U32, U32, U32)
@@ -274,8 +277,8 @@ VALUE_OPCODE(SharedAtomicUMax32, U32, U32, U32, U1)
 VALUE_OPCODE(SharedAtomicAnd32, U32, U32, U32, U1)
 VALUE_OPCODE(SharedAtomicOr32, U32, U32, U32, U1)
 VALUE_OPCODE(SharedAtomicXor32, U32, U32, U32, U1)
-VALUE_OPCODE(DataAppend, U32, U32, U1, U32, U32)
-VALUE_OPCODE(DataConsume, U32, U32, U1, U32, U32)
+VALUE_OPCODE(DataAppend, U32, U32, U1)
+VALUE_OPCODE(DataConsume, U32, U32, U1)
 VALUE_OPCODE(SwizzleU32, U32, U32, U32, U1)
 
 VALUE_OPCODE(ImageQueryDimensions, U32x4, ImageResource, ImageAddress)

src/graphics/shader/recompiler/ir/passes/BindingLayout.cpp

@@ -85,6 +85,16 @@ bool CollectMemoryResources(const Program& program, std::vector<uint32_t>& buffe
 	return uses_gds;
 }
 
+bool UsesFlattenedSrt(const Program& program) {
+	return std::ranges::any_of(program.blocks, [](const Block* block) {
+		return std::ranges::any_of(*block, [](const Inst& inst) {
+			return inst.GetOpcode() == ValueOpcode::ReadConst;
+		});
+	}) || std::ranges::any_of(program.info.images, [](const ImageResource& image) {
+		return image.indirect_search_iterations != 0u;
+	});
+}
+
 void AllocateBindings(Program& program, uint32_t push_data_start_dword) {
 	if (!program.shader_info_complete || program.binding_layout_complete) {
 		EXIT("shader binding layout failed: %s", !program.shader_info_complete
@@ -145,12 +155,7 @@ void AllocateBindings(Program& program, uint32_t push_data_start_dword) {
 		AddBinding(next, DescriptorBindingKind::BdaPagetable);
 		AddBinding(next, DescriptorBindingKind::FaultBuffer);
 	}
-	const bool uses_flattened_runtime =
-	    !program.srt_reads.empty() ||
-	    std::ranges::any_of(program.info.images, [](const ImageResource& image) {
-		    return image.indirect_search_iterations != 0u;
-	    });
-	if (uses_flattened_runtime) {
+	if (UsesFlattenedSrt(program)) {
 		AddBinding(next, DescriptorBindingKind::FlattenedSrt);
 	}
 

src/graphics/shader/recompiler/ir/passes/BindingLayout.h

@@ -9,6 +9,7 @@ void AllocateBindings(Program& program, uint32_t push_data_start_dword = 0);
 
 // Collect live direct buffer indices and report whether the shader accesses GDS.
 bool CollectMemoryResources(const Program& program, std::vector<uint32_t>& buffers);
+bool UsesFlattenedSrt(const Program& program);
 
 const DescriptorBinding* FindBinding(const BindingLayout& layout, DescriptorBindingKind kind);
 

src/graphics/shader/recompiler/ir/passes/ConstantPropagation.cpp

@@ -630,6 +630,11 @@ void FoldInstruction(Block& block, Block::iterator instruction,
 				}
 			}
 			return;
+		case ValueOpcode::ConditionRef: {
+			const auto value = Arg(inst, 0);
+			if (IsImmediate(value, Type::U1)) Replace(inst, value);
+			return;
+		}
 		case ValueOpcode::LogicalNot: {
 			const auto value = Arg(inst, 0);
 			if (IsImmediate(value, Type::U1)) {

src/graphics/shader/recompiler/ir/passes/ResourceMaterialization.cpp

@@ -107,14 +107,16 @@ Prospero::BufferFormat ImageConversionFormat(Prospero::BufferFormat format) {
 	                                                      : Prospero::BufferFormat::kInvalid;
 }
 
-bool RequiresPointSampler(const ImageResource& image) {
-	return image.numeric_class == Prospero::TextureNumericClass::Sint ||
-	       image.conversion_format != Prospero::BufferFormat::kInvalid;
-}
+enum class SamplerClass : uint8_t { Float, Integer, PointInteger };
 
-bool RequiresPointSampler(const ResourceSpecialization::Image& image) {
-	return image.numeric_class == Prospero::TextureNumericClass::Sint ||
-	       image.conversion_format != Prospero::BufferFormat::kInvalid;
+template <typename Image>
+SamplerClass ClassifySampler(const Image& image) {
+	if (image.numeric_class == Prospero::TextureNumericClass::Sint ||
+	    image.conversion_format != Prospero::BufferFormat::kInvalid) {
+		return SamplerClass::PointInteger;
+	}
+	return image.numeric_class == Prospero::TextureNumericClass::Uint ? SamplerClass::Integer
+	                                                               : SamplerClass::Float;
 }
 
 bool DescriptorIsCube(const DescriptorValue& descriptor) {
@@ -223,9 +225,9 @@ void MarkCleanFlatSlots(const ResourcePlan& program, const DescriptorSource* sou
 	}
 }
 
-bool ReadScalarTable(uint64_t base, uint64_t size, uint32_t dynamic_offset,
+bool ReadScalarTable(uint64_t base, uint64_t size, uint64_t dynamic_offset,
                      const SrtRuntime& runtime, std::span<uint32_t> words) {
-	const auto offset = static_cast<uint64_t>(dynamic_offset) & ~uint64_t {3};
+	const auto offset = dynamic_offset & ~uint64_t {3};
 	const auto count = std::min<uint64_t>(words.size(), offset < size ? (size - offset) / 4u : 0u);
 	std::ranges::fill(words.subspan(count), 0u);
 	if (count == 0u) {
@@ -304,26 +306,24 @@ bool MaterializeIndirectImage(const ResourcePlan& program,
 		    material.Stride() != indirect.selector_stride) {
 			return false;
 		}
-		// Enumerate every wrapped scalar-buffer offset that can pass the descriptor bounds.
-		const auto step = std::gcd<uint64_t>(indirect.selector_stride, uint64_t {1} << 32u);
-		const auto residue = static_cast<uint64_t>(indirect.selector_offset) % step;
-		const auto limit = std::min<uint64_t>(UINT32_MAX, material.GetSize() + 3u);
-		const auto probe_count = residue <= limit ? (limit - residue) / step + 1u : 0u;
+		// The first aligned offset includes the immediate added after shader U32 arithmetic.
+		const auto step = std::max<uint64_t>(4u,
+		    std::gcd<uint64_t>(indirect.selector_stride, uint64_t {1} << 32u));
+		const uint64_t first = indirect.selector_offset;
+		const auto size = material.GetSize();
+		const auto limit = std::min(first + (uint64_t {1} << 32u) - step, size >= 4u ? size - 4u : 0u);
+		const auto probe_count = size >= 4u && first <= limit ? (limit - first) / step + 1u : 0u;
 		if (probe_count > MaxIndirectImageProbes) {
 			return false;
 		}
 		keys.reserve(static_cast<size_t>(probe_count) + 1u);
 		keys.push_back(0u);
-		for (uint64_t offset = residue; offset <= limit && probe_count != 0u; offset += step) {
+		for (uint64_t probe = 0, offset = first; probe < probe_count; ++probe, offset += step) {
 			uint32_t key = 0;
-			if (!ReadScalarTable(material.Base48(), material.GetSize(),
-			                     static_cast<uint32_t>(offset), runtime, {&key, 1})) {
+			if (!ReadScalarTable(material.Base48(), size, offset, runtime, {&key, 1})) {
 				return false;
 			}
 			keys.push_back(key);
-			if (limit - offset < step) {
-				break;
-			}
 		}
 		std::ranges::sort(keys);
 		keys.erase(std::unique(keys.begin(), keys.end()), keys.end());
@@ -380,8 +380,13 @@ bool MaterializeIndirectImage(const ResourcePlan& program,
 } // namespace
 
 struct SamplerPlan {
-	std::array<uint32_t, ShaderInfo::MaxSamplers> point_sampler {};
-	uint32_t                                      sampler_count = 0;
+	struct Binding {
+		uint32_t     source;
+		SamplerClass type;
+	};
+	std::array<std::array<uint32_t, 3>, ShaderInfo::MaxSamplers> mapping;
+	std::array<Binding, ShaderInfo::MaxSamplers>                bindings;
+	uint32_t                                                  sampler_count = 0;
 };
 
 struct ImageRemap {
@@ -595,11 +600,9 @@ static bool BuildResourceSpecialization(const ResourcePlan& program, ResourceSna
 	if (!BuildSamplerPlan(program.info, specialization.images, sampler_plan)) {
 		return SpecializationFail("specialized sampler layout exceeds its resource limit");
 	}
-	for (uint32_t index = 0; index < program.info.samplers.size(); index++) {
-		const auto target = sampler_plan.point_sampler[index];
-		if (target != UINT32_MAX && target >= program.info.samplers.size()) {
-			snapshot.samplers.push_back(snapshot.samplers[index]);
-		}
+	for (uint32_t index = static_cast<uint32_t>(program.info.samplers.size());
+	     index < sampler_plan.sampler_count; index++) {
+		snapshot.samplers.push_back(snapshot.samplers[sampler_plan.bindings[index].source]);
 	}
 	ImageRemap(specialization).Apply(snapshot.images);
 	return true;
@@ -607,29 +610,29 @@ static bool BuildResourceSpecialization(const ResourcePlan& program, ResourceSna
 
 template <typename Images>
 bool BuildSamplerPlan(const ShaderInfo& base, const Images& images, SamplerPlan& plan) {
-	if (base.samplers.size() > plan.point_sampler.size()) {
+	if (base.samplers.size() > plan.mapping.size()) {
 		return false;
 	}
 	std::array<uint8_t, ShaderInfo::MaxSamplers> usage {};
-	plan.point_sampler.fill(UINT32_MAX);
 	plan.sampler_count = static_cast<uint32_t>(base.samplers.size());
 	for (const auto& pair: base.sampled_pairs) {
 		if (pair.image >= images.size() || pair.sampler >= base.samplers.size()) {
 			return false;
 		}
-		usage[pair.sampler] |= RequiresPointSampler(images[pair.image]) ? 2u : 1u;
+		usage[pair.sampler] |= 1u << static_cast<uint32_t>(ClassifySampler(images[pair.image]));
 	}
 	for (uint32_t index = 0; index < base.samplers.size(); index++) {
-		if ((usage[index] & 2u) == 0u) {
-			continue;
-		}
-		if ((usage[index] & 1u) == 0u) {
-			plan.point_sampler[index] = index;
-		} else {
-			if (plan.sampler_count >= ShaderInfo::MaxSamplers) {
-				return false;
-			}
-			plan.point_sampler[index] = plan.sampler_count++;
+		auto& mapping = plan.mapping[index];
+		mapping.fill(UINT32_MAX);
+		const auto classes = usage[index] == 0u ? 1u : usage[index];
+		bool       first   = true;
+		for (uint32_t type = 0; type < mapping.size(); type++) {
+			if ((classes & (1u << type)) == 0u) continue;
+			const auto target = first ? index : plan.sampler_count++;
+			if (target >= ShaderInfo::MaxSamplers) return false;
+			mapping[type]         = target;
+			plan.bindings[target] = {index, static_cast<SamplerClass>(type)};
+			first                = false;
 		}
 	}
 	return true;
@@ -862,7 +865,6 @@ ResourcePlan ExtractResourcePlan(const Program& program) {
 	plan.memory_info                = program.memory_info;
 	plan.srt_plan_complete          = program.srt_plan_complete;
 	plan.resource_tracking_complete = program.resource_tracking_complete;
-	plan.has_address_writes         = program.has_address_writes;
 
 	std::unordered_map<const Inst*, Inst*> cloned;
 	std::function<Value(Value)>            Clone = [&](Value value) -> Value {
@@ -921,17 +923,25 @@ ResourcePlan ExtractResourcePlan(const Program& program) {
 		plan.uniform_fill.values[i] = Clone(plan.uniform_fill.values[i]);
 	}
 	plan.clean_flat_slots.resize(plan.srt_reads.size());
+	bool masked_image = false;
 	for (const auto& image: plan.info.images) {
 		const auto* source = Source(plan, image.source);
 		if (source == nullptr || !source->indirect_image.has_value()) {
 			continue;
 		}
 		plan.requires_specialization_memory = true;
+		masked_image |= !source->indirect_image->selector_mask.IsEmpty();
 		MarkCleanFlatSlots(plan, Source(plan, source->indirect_image->material_source),
 		                   plan.clean_flat_slots, source->indirect_image->selector_mask);
 		MarkCleanFlatSlots(plan, Source(plan, source->indirect_image->table_source),
 		                   plan.clean_flat_slots);
 	}
+	if (masked_image) {
+		plan.resource_tracking_complete &= !program.has_address_writes &&
+		    !std::ranges::any_of(plan.info.images, &ImageResource::written);
+		plan.capture_specialization_reads =
+		    std::ranges::any_of(plan.info.buffers, &BufferResource::written);
+	}
 	return plan;
 }
 
@@ -941,18 +951,7 @@ bool MaterializeResources(const ResourcePlan& program, const SrtRuntime& runtime
 	    (program.requires_specialization_memory && runtime.read_specialization_memory == nullptr)) {
 		return false;
 	}
-	const bool masked_image = std::ranges::any_of(program.info.images, [&](const auto& image) {
-		const auto* source = Source(program, image.source);
-		return source != nullptr && source->indirect_image.has_value() &&
-		       !source->indirect_image->selector_mask.IsEmpty();
-	});
-	if (masked_image &&
-	    (program.has_address_writes ||
-	     std::ranges::any_of(program.info.images, &ImageResource::written))) {
-		return false;
-	}
-	const bool capture_reads = masked_image &&
-	    std::ranges::any_of(program.info.buffers, &BufferResource::written);
+	const bool capture_reads = program.capture_specialization_reads;
 	auto& reads = program.specialization_reads;
 	ReadCapture capture {runtime, reads};
 	SrtRuntime observed = runtime;
@@ -1105,25 +1104,18 @@ void ApplyResourceSpecialization(Program& program, const ResourceSpecialization&
 	auto samplers      = program.info.samplers;
 	auto sampled_pairs = program.info.sampled_pairs;
 	samplers.reserve(sampler_plan.sampler_count);
-	for (uint32_t index = 0; index < program.info.samplers.size(); index++) {
-		const auto target = sampler_plan.point_sampler[index];
-		if (target == UINT32_MAX) {
-			continue;
-		}
-		if (target == index) {
-			samplers[index].force_point_filtering = true;
-		} else {
-			EXIT_IF(target != samplers.size());
-			auto sampler                  = samplers[index];
-			sampler.force_point_filtering = true;
-			samplers.push_back(std::move(sampler));
+	for (uint32_t index = 0; index < sampler_plan.sampler_count; index++) {
+		const auto& binding = sampler_plan.bindings[index];
+		if (index >= program.info.samplers.size()) {
+			samplers.push_back(program.info.samplers[binding.source]);
 		}
+		samplers[index].force_point_filtering = binding.type == SamplerClass::PointInteger;
+		samplers[index].integer_border        = binding.type != SamplerClass::Float;
 	}
 	for (auto& pair: sampled_pairs) {
-		if (RequiresPointSampler(images[pair.image])) {
-			EXIT_IF(sampler_plan.point_sampler[pair.sampler] == UINT32_MAX);
-			pair.sampler = sampler_plan.point_sampler[pair.sampler];
-		}
+		const auto type = static_cast<uint32_t>(ClassifySampler(images[pair.image]));
+		pair.sampler = sampler_plan.mapping[pair.sampler][type];
+		EXIT_IF(pair.sampler == UINT32_MAX);
 		samplers[pair.sampler].depth_compare |= images[pair.image].depth_compare;
 	}
 
@@ -1201,10 +1193,11 @@ void ApplyResourceSpecialization(Program& program, const ResourceSpecialization&
 				inst.ReplaceUsesWith(Value(&*result));
 				continue;
 			}
-			if (image_opcode.needs_sampler && RequiresPointSampler(image) &&
+			if (image_opcode.needs_sampler &&
 			    memory.sampler < program.info.samplers.size()) {
-				EXIT_IF(sampler_plan.point_sampler[memory.sampler] == UINT32_MAX);
-				memory.sampler = sampler_plan.point_sampler[memory.sampler];
+				const auto type = static_cast<uint32_t>(ClassifySampler(image));
+				memory.sampler = sampler_plan.mapping[memory.sampler][type];
+				EXIT_IF(memory.sampler == UINT32_MAX);
 			}
 			EXIT_IF(image.indirect_root == memory.resource &&
 			        inst.GetOpcode() != ValueOpcode::ImageSampleRaw);

src/graphics/shader/recompiler/ir/passes/ResourceTracking.h

@@ -5,9 +5,8 @@
 
 namespace Libs::Graphics::ShaderRecompiler::IR {
 
-// Collects immutable resource topology from typed SSA handles, interns their resolved dwords in
-// descriptor_sources, then writes dense indices to handle flags and MemoryInfo.
-void TrackResources(Program& program);
+// Resolves native descriptor sources, plans their scalar reads, and assigns dense resource bindings.
+void TrackResources(Program& program, const Decoder::Program& decoded, const CFG::Graph& native_cfg);
 
 } // namespace Libs::Graphics::ShaderRecompiler::IR
 

src/graphics/shader/recompiler/ir/passes/SrtWalker.cpp

@@ -7,7 +7,6 @@
 #include <bit>
 #include <cmath>
 #include <cstring>
-#include <fmt/format.h>
 #include <unordered_set>
 
 namespace Libs::Graphics::ShaderRecompiler::IR {
@@ -23,21 +22,6 @@ namespace {
 
 constexpr uint64_t AddressMask = 0x0000ffffffffffffull;
 
-const char* StageName(ShaderType stage) {
-	switch (stage) {
-		case ShaderType::Vertex: return "vertex";
-		case ShaderType::Pixel: return "pixel";
-		case ShaderType::Fetch: return "fetch";
-		case ShaderType::Compute: return "compute";
-		default: return "unknown";
-	}
-}
-
-std::string Diagnostic(const ResourcePlan& program, uint32_t pc, const std::string& message) {
-	return fmt::format("shader SRT: hash=0x{:016x} stage={} pc=0x{:08x} {}", program.shader_hash,
-	                   StageName(program.stage), pc, message);
-}
-
 bool AddSignedAddress(uint64_t base, int64_t offset, uint64_t& result) {
 	if (base > AddressMask) {
 		return false;
@@ -89,6 +73,7 @@ bool IsRuntimeSelect(ValueOpcode op) {
 
 bool IsRuntimeUniformOp(ValueOpcode op) {
 	switch (op) {
+		case ValueOpcode::ConditionRef:
 		case ValueOpcode::BitCastU32F32:
 		case ValueOpcode::BitCastF32U32:
 		case ValueOpcode::ConvertU32F32:
@@ -312,151 +297,6 @@ private:
 	std::unordered_set<const Inst*> m_validated_dependencies;
 };
 
-class PlanBuilder {
-public:
-	explicit PlanBuilder(Program& program): m_program(program) {}
-
-	void Run() {
-		m_program.srt_reads.clear();
-		m_program.dynamic_reads.clear();
-		for (auto* block: m_program.blocks) {
-			for (auto& inst: *block) {
-				const auto op = inst.GetOpcode();
-				if (op == ValueOpcode::LoadAddressU32 || op == ValueOpcode::ReadConstBuffer) {
-					const auto flags = inst.Flags<MemoryFlags>();
-					if (flags.index < m_program.memory_info.size()) {
-						const auto kind       = m_program.memory_info[flags.index].kind;
-						const bool crosswired = (op == ValueOpcode::LoadAddressU32 &&
-						                         kind == ResourceKind::ScalarBuffer) ||
-						                        (op == ValueOpcode::ReadConstBuffer &&
-						                         kind == ResourceKind::ScalarAddress);
-						if (crosswired) {
-							Fail(flags.pc,
-							     fmt::format("{} has incompatible scalar memory metadata",
-							                 ValueOpcodeName(op)));
-						}
-					}
-				}
-				if (IsDescriptorHandle(inst.GetOpcode())) {
-					for (size_t index = 0; index < inst.NumArgs(); index++) {
-						Collect(inst.Arg(index), 0);
-					}
-				}
-			}
-		}
-		for (auto* block: m_program.blocks) {
-			for (auto& inst: *block) {
-				if (inst.GetOpcode() == ValueOpcode::LoadAddressU32 && IsRawRead(m_program, inst) &&
-				    inst.Arg(1).Resolve().IsImmediate() &&
-				    ValidateRuntimeValue(m_program, Value(&inst))) {
-					Collect(Value(&inst), inst.Flags<MemoryFlags>().pc);
-				}
-			}
-		}
-		PatchReads();
-	}
-
-private:
-	struct Patch {
-		Inst*    inst = nullptr;
-		uint32_t slot = 0;
-		bool     keep = false;
-	};
-
-	[[noreturn]] void Fail(uint32_t pc, const std::string& message) const {
-		const auto diagnostic = Diagnostic(m_program, pc, message);
-		EXIT("shader SRT planning failed: %s", diagnostic.c_str());
-		std::abort();
-	}
-
-	void Collect(Value value, uint32_t use_pc) {
-		value = value.Resolve();
-		if (value.IsImmediate()) {
-			return;
-		}
-		auto* inst = value.TryInstruction();
-		if (inst == nullptr) {
-			Fail(use_pc, "invalid typed planning value");
-		}
-		const auto cycle = std::ranges::find(m_visiting, inst);
-		if (cycle != m_visiting.end()) {
-			const auto contains_phi = std::any_of(cycle, m_visiting.end(), [](const Inst* value) {
-				return value->GetOpcode() == ValueOpcode::Phi;
-			});
-			if (contains_phi) {
-				return;
-			}
-			Fail(use_pc, fmt::format("cyclic typed planning value {} without a phi",
-			                         ValueOpcodeName(inst->GetOpcode())));
-		}
-		if (std::ranges::find(m_visited, inst) != m_visited.end()) {
-			return;
-		}
-		m_visiting.push_back(inst);
-		for (size_t index = 0; index < inst->NumArgs(); index++) {
-			Collect(inst->Arg(index), use_pc);
-		}
-		m_visiting.pop_back();
-		m_visited.push_back(inst);
-		if (!IsRawRead(m_program, *inst)) {
-			return;
-		}
-		const auto offset = inst->Arg(1).Resolve();
-		if (!offset.IsImmediate() || offset.GetType() != Type::U32) {
-			if (std::ranges::find(m_program.dynamic_reads, value) ==
-			    m_program.dynamic_reads.end()) {
-				m_program.dynamic_reads.push_back(value);
-			}
-			return;
-		}
-		for (uint32_t slot = 0; slot < m_program.srt_reads.size(); slot++) {
-			if (EquivalentValue(m_program, value, m_program.srt_reads[slot].value)) {
-				m_patches.push_back({inst, slot, false});
-				return;
-			}
-		}
-		const auto slot = static_cast<uint32_t>(m_program.srt_reads.size());
-		m_program.srt_reads.push_back({value, slot});
-		m_patches.push_back({inst, slot, true});
-	}
-
-	void PatchReads() {
-		for (const auto& patch: m_patches) {
-			auto* block = patch.inst->Parent();
-			auto& list  = block->Instructions();
-			auto  where =
-			    std::ranges::find_if(list, [&](const Inst& inst) { return &inst == patch.inst; });
-			const auto resource =
-			    Value(&*block->PrependNewInst(where, ValueOpcode::GetSrtResource));
-			const auto flat = Value(&*block->PrependNewInst(where, ValueOpcode::ReadConst,
-			                                                {resource, Value(patch.slot)}));
-			const auto uses = patch.inst->Uses();
-			for (const auto& use: uses) {
-				use.user->SetArg(use.operand, flat);
-			}
-			for (auto& info: m_program.block_info) {
-				if (info.condition.Resolve() == Value(patch.inst)) {
-					info.condition = flat;
-				}
-				if (info.indirect_target.Resolve() == Value(patch.inst)) {
-					info.indirect_target = flat;
-				}
-			}
-			if (patch.keep) {
-				const auto memory = patch.inst->Flags<MemoryFlags>().index;
-				if (memory < m_program.memory_info.size()) {
-					m_program.memory_info[memory].planning_only = true;
-				}
-				block->AppendNewInst(ValueOpcode::ReferenceU32, {Value(patch.inst)});
-			}
-		}
-	}
-
-	Program&           m_program;
-	std::vector<Inst*> m_visiting;
-	std::vector<Inst*> m_visited;
-	std::vector<Patch> m_patches;
-};
 
 } // namespace
 
@@ -616,16 +456,15 @@ bool SrtWalker::EvaluateRawRead(const Inst& inst, uint64_t& result) {
 			return false;
 		}
 		const auto byte_offset =
-		    static_cast<uint64_t>(immediate) + static_cast<uint32_t>(offset);
-		const auto aligned = byte_offset & ~uint64_t {3};
+		    (static_cast<uint64_t>(immediate) & ~uint64_t {3}) + (static_cast<uint32_t>(offset) & ~3u);
 		const auto stride  = (static_cast<uint32_t>(high) >> 16u) & 0x3fffu;
 		const auto size = stride == 0u
 		                      ? static_cast<uint64_t>(static_cast<uint32_t>(records))
 		                      : static_cast<uint64_t>(stride) * static_cast<uint32_t>(records);
-		if (aligned > size || size - aligned < sizeof(uint32_t)) {
+		if (byte_offset > size || size - byte_offset < sizeof(uint32_t)) {
 			return false;
 		}
-		address = ((base & ~uint64_t {3}) + byte_offset) & ~uint64_t {3};
+		address = (base & ~uint64_t {3}) + byte_offset;
 	} else {
 		const auto relative = (immediate & ~int64_t {3}) +
 		                      static_cast<int64_t>(static_cast<uint32_t>(offset) & ~3u);
@@ -780,14 +619,18 @@ bool SrtWalker::EvaluateInst(const Inst& inst, uint64_t& result) {
 			}
 			return false;
 		case ValueOpcode::FPOrdLessThanEqual32:
-			if (binary()) {
-				result = Float32(a) <= Float32(b);
-				return true;
-			}
-			return false;
 		case ValueOpcode::FPOrdGreaterThanEqual32:
 			if (binary()) {
-				result = Float32(a) >= Float32(b);
+				const auto operand = [&](uint64_t bits) {
+					if (inst.Flags<FPCompareFlags>().flush_input_denorms &&
+					    (bits & 0x7fffffffu) < 0x00800000u) {
+						bits &= 0x80000000u;
+					}
+					return Float32(bits);
+				};
+				result = inst.GetOpcode() == ValueOpcode::FPOrdLessThanEqual32
+				             ? operand(a) <= operand(b)
+				             : operand(a) >= operand(b);
 				return true;
 			}
 			return false;
@@ -970,6 +813,7 @@ bool SrtWalker::EvaluateInst(const Inst& inst, uint64_t& result) {
 				return true;
 			}
 			return false;
+		case ValueOpcode::ConditionRef: return Arg(inst, 0, result);
 		case ValueOpcode::LogicalNot:
 			if (Arg(inst, 0, a)) {
 				result = a == 0u;
@@ -1061,13 +905,5 @@ bool ValidateRuntimeValue(const ResourcePlan& program, Value value, RuntimeValue
 	return RuntimeValidator(program, type).Run(value);
 }
 
-void BuildSrtPlan(Program& program) {
-	if (program.resource_tracking_complete) {
-		EXIT("shader SRT planning failed: cannot rebuild SRT after resource tracking");
-	}
-	program.srt_plan_complete = false;
-	PlanBuilder(program).Run();
-	program.srt_plan_complete = true;
-}
 
 } // namespace Libs::Graphics::ShaderRecompiler::IR

src/graphics/shader/recompiler/ir/passes/SrtWalker.h

@@ -21,9 +21,6 @@ struct SrtRuntime {
 
 enum class RuntimeValueType { Any, Integer };
 
-// Collects reachable ReadConst values. Immediate offsets receive compact flat-buffer slots;
-// dynamic offsets remain explicit and are never assigned a fake slot.
-void BuildSrtPlan(Program& program);
 bool ValidateRuntimeValue(const ResourcePlan& program, Value value,
                           RuntimeValueType type = RuntimeValueType::Any);
 // Uses the strict reader for values that affect shader specialization.

src/kernel/memory.cpp

@@ -78,8 +78,7 @@ static Graphics::RenderContext& GetGpuResources() {
 }
 
 static bool IsGpuAddressRange(uint64_t vaddr, uint64_t size) {
-	constexpr uint64_t GPU_ADDRESS_LIMIT = 1ull << 40u;
-	return vaddr != 0 && size != 0 && vaddr < GPU_ADDRESS_LIMIT && size < GPU_ADDRESS_LIMIT - vaddr;
+	return Graphics::GuestRange {vaddr, size}.Valid();
 }
 
 static void MapGpuRange(uint64_t vaddr, uint64_t size) {
@@ -660,6 +659,9 @@ static bool     g_test_fail_next_fixed_reserve_range_add     = false;
 
 class PhysicalMemory {
 public:
+	enum class AllocationKind { Direct, Pooled, Automatic };
+	using PhysicalRanges = std::vector<std::pair<uint64_t, uint64_t>>;
+
 	struct AllocatedBlock {
 		uint64_t            start_addr;
 		uint64_t            size;
@@ -671,7 +673,7 @@ public:
 		VirtualMemory::Mode mode;
 		GpuAccessMode       gpu_mode;
 		int                 memory_type;
-		bool                pool_expansion;
+		AllocationKind      kind;
 		char                name[KERNEL_MAXIMUM_NAME_LENGTH];
 	};
 
@@ -690,7 +692,10 @@ public:
 	}
 
 	bool Alloc(uint64_t search_start, uint64_t search_end, size_t len, size_t alignment,
-	           uint64_t* phys_addr_out, int memory_type, bool pool_expansion = false);
+	           uint64_t* phys_addr_out, int memory_type,
+	           AllocationKind kind = AllocationKind::Direct);
+	bool ReserveAutomatic(uint64_t size, PhysicalRanges* ranges);
+	void RestoreAutomatic(const PhysicalRanges& ranges);
 	bool Available(uint64_t search_start, uint64_t search_end, size_t alignment,
 	               uint64_t* phys_addr_out, uint64_t* size_out);
 	bool Release(uint64_t start, size_t len, uint64_t* vaddr, uint64_t* size,
@@ -718,12 +723,14 @@ public:
 	[[nodiscard]] const std::vector<AllocatedBlock>& GetMappings() const { return m_mappings; }
 
 private:
-	void ConsumeFreeRange(std::map<uint64_t, uint64_t>::iterator range, uint64_t start,
-	                      uint64_t size);
-	void AddFreeRange(uint64_t start, uint64_t size);
+	static void RemoveFreeRange(std::map<uint64_t, uint64_t>& ranges, uint64_t start,
+	                            uint64_t size);
+	static void AddFreeRange(std::map<uint64_t, uint64_t>& ranges, uint64_t start, uint64_t size);
+	void        ReclaimAutomatic(uint64_t start, uint64_t size);
 
 	std::map<uint64_t, AllocatedBlock> m_physical;
 	std::map<uint64_t, uint64_t>       m_free;
+	std::map<uint64_t, uint64_t>       m_automatic_free;
 	std::vector<AllocatedBlock>        m_mappings;
 	Common::Mutex                      m_mutex;
 };
@@ -1093,7 +1100,7 @@ void SetFlexibleMemorySize(uint64_t size) {
 }
 
 bool PhysicalMemory::Alloc(uint64_t search_start, uint64_t search_end, size_t len, size_t alignment,
-                           uint64_t* phys_addr_out, int memory_type, bool pool_expansion) {
+                           uint64_t* phys_addr_out, int memory_type, AllocationKind kind) {
 	if (phys_addr_out == nullptr) {
 		return false;
 	}
@@ -1128,10 +1135,13 @@ bool PhysicalMemory::Alloc(uint64_t search_start, uint64_t search_end, size_t le
 		b.prot           = 0;
 		b.mode           = VirtualMemory::Mode::NoAccess;
 		b.memory_type    = memory_type;
-		b.pool_expansion = pool_expansion;
+		b.kind           = kind;
 
-		ConsumeFreeRange(range, free_pos, len);
+		RemoveFreeRange(m_free, free_pos, len);
 		EXIT_IF(!m_physical.emplace(b.start_addr, b).second);
+		if (kind == AllocationKind::Automatic) {
+			AddFreeRange(m_automatic_free, free_pos, len);
+		}
 
 		*phys_addr_out = free_pos;
 		return true;
@@ -1180,35 +1190,99 @@ bool PhysicalMemory::Available(uint64_t search_start, uint64_t search_end, size_
 	return true;
 }
 
-void PhysicalMemory::ConsumeFreeRange(std::map<uint64_t, uint64_t>::iterator range, uint64_t start,
-                                      uint64_t size) {
-	const auto range_start = range->first;
-	const auto range_end   = range->first + range->second;
-	m_free.erase(range);
-	if (range_start < start) {
-		m_free.emplace(range_start, start - range_start);
+void PhysicalMemory::RemoveFreeRange(std::map<uint64_t, uint64_t>& ranges, uint64_t start,
+                                     uint64_t size) {
+	const auto end   = start + size;
+	auto       range = ranges.upper_bound(start);
+	if (range != ranges.begin()) {
+		--range;
 	}
-	if (start + size < range_end) {
-		m_free.emplace(start + size, range_end - start - size);
+	while (range != ranges.end() && range->first < end) {
+		const auto range_start = range->first;
+		const auto range_end   = range_start + range->second;
+		if (range_end <= start) {
+			++range;
+			continue;
+		}
+		range = ranges.erase(range);
+		if (range_start < start) {
+			ranges.emplace(range_start, start - range_start);
+		}
+		if (end < range_end) {
+			ranges.emplace(end, range_end - end);
+		}
 	}
 }
 
-void PhysicalMemory::AddFreeRange(uint64_t start, uint64_t size) {
+void PhysicalMemory::AddFreeRange(std::map<uint64_t, uint64_t>& ranges, uint64_t start,
+                                  uint64_t size) {
 	auto end  = start + size;
-	auto next = m_free.lower_bound(start);
-	if (next != m_free.begin()) {
+	auto next = ranges.lower_bound(start);
+	if (next != ranges.begin()) {
 		auto previous = std::prev(next);
 		if (previous->first + previous->second >= start) {
 			start = previous->first;
 			end   = std::max(end, previous->first + previous->second);
-			next  = m_free.erase(previous);
+			next  = ranges.erase(previous);
 		}
 	}
-	while (next != m_free.end() && next->first <= end) {
+	while (next != ranges.end() && next->first <= end) {
 		end  = std::max(end, next->first + next->second);
-		next = m_free.erase(next);
+		next = ranges.erase(next);
+	}
+	ranges.emplace(start, end - start);
+}
+
+bool PhysicalMemory::ReserveAutomatic(uint64_t size, PhysicalRanges* ranges) {
+	Common::LockGuard lock(m_mutex);
+	for (const auto& [start, available]: m_automatic_free) {
+		const auto length = std::min(size, available);
+		ranges->emplace_back(start, length);
+		size -= length;
+		if (size == 0) {
+			for (const auto& [offset, length]: *ranges) {
+				RemoveFreeRange(m_automatic_free, offset, length);
+			}
+			return true;
+		}
+	}
+	ranges->clear();
+	return false;
+}
+
+void PhysicalMemory::RestoreAutomatic(const PhysicalRanges& ranges) {
+	Common::LockGuard lock(m_mutex);
+	for (const auto& [start, size]: ranges) {
+		AddFreeRange(m_automatic_free, start, size);
+	}
+}
+
+void PhysicalMemory::ReclaimAutomatic(uint64_t start, uint64_t size) {
+	const auto end   = start + size;
+	auto       owner = m_physical.upper_bound(start);
+	if (owner != m_physical.begin()) {
+		--owner;
+	}
+	bool automatic = false;
+	for (; owner != m_physical.end() && owner->first < end; ++owner) {
+		const auto& block = owner->second;
+		if (block.kind != AllocationKind::Automatic || block.start_addr + block.size <= start) {
+			continue;
+		}
+		const auto first = std::max(start, block.start_addr);
+		const auto last  = std::min(end, block.start_addr + block.size);
+		AddFreeRange(m_automatic_free, first, last - first);
+		automatic = true;
+	}
+	if (automatic) {
+		for (const auto& mapping: m_mappings) {
+			const auto first = std::max(start, mapping.start_addr);
+			const auto last  = std::min(end, mapping.start_addr + mapping.size);
+			if (first < last) {
+				RemoveFreeRange(m_automatic_free, first, last - first);
+			}
+		}
 	}
-	m_free.emplace(start, end - start);
 }
 
 bool PhysicalMemory::Release(uint64_t start, size_t len, uint64_t* vaddr, uint64_t* size,
@@ -1225,18 +1299,22 @@ bool PhysicalMemory::Release(uint64_t start, size_t len, uint64_t* vaddr, uint64
 	}
 	auto  it = std::prev(next);
 	auto& b  = it->second;
-	if (b.pool_expansion || start < b.start_addr || start >= b.start_addr + b.size ||
-	    len > b.start_addr + b.size - start) {
+	if (b.kind == AllocationKind::Pooled || start < b.start_addr ||
+	    start >= b.start_addr + b.size || len > b.start_addr + b.size - start) {
 		return false;
 	}
 
+	if (b.kind == AllocationKind::Automatic) {
+		RemoveFreeRange(m_automatic_free, start, len);
+	}
+
 	if (start == b.start_addr && len == b.size) {
 		*vaddr    = b.map_vaddr;
 		*size     = b.map_size;
 		*gpu_mode = b.gpu_mode;
 
 		m_physical.erase(it);
-		AddFreeRange(start, len);
+		AddFreeRange(m_free, start, len);
 		return true;
 	}
 	if (start > b.start_addr && start + len < b.start_addr + b.size) {
@@ -1261,7 +1339,7 @@ bool PhysicalMemory::Release(uint64_t start, size_t len, uint64_t* vaddr, uint64
 		}
 
 		m_physical.emplace(right.start_addr, right);
-		AddFreeRange(start, len);
+		AddFreeRange(m_free, start, len);
 		return true;
 	}
 	if (start == b.start_addr && len < b.size) {
@@ -1278,7 +1356,7 @@ bool PhysicalMemory::Release(uint64_t start, size_t len, uint64_t* vaddr, uint64
 			remaining.map_size -= len;
 		}
 		m_physical.emplace(remaining.start_addr, remaining);
-		AddFreeRange(start, len);
+		AddFreeRange(m_free, start, len);
 		return true;
 	}
 	if (start > b.start_addr && start + len == b.start_addr + b.size) {
@@ -1290,7 +1368,7 @@ bool PhysicalMemory::Release(uint64_t start, size_t len, uint64_t* vaddr, uint64
 		if (b.map_vaddr != 0) {
 			b.map_size = b.size;
 		}
-		AddFreeRange(start, len);
+		AddFreeRange(m_free, start, len);
 		return true;
 	}
 
@@ -1311,6 +1389,7 @@ bool PhysicalMemory::Map(uint64_t vaddr, uint64_t phys_addr, size_t len, int pro
 		return false;
 	}
 	auto first = std::prev(next);
+	bool has_automatic = false;
 	while (current < phys_addr + len) {
 		auto block = m_physical.upper_bound(current);
 		if (block == m_physical.begin()) {
@@ -1318,10 +1397,11 @@ bool PhysicalMemory::Map(uint64_t vaddr, uint64_t phys_addr, size_t len, int pro
 		}
 		--block;
 		const auto block_end = block->second.start_addr + block->second.size;
-		if (block->second.pool_expansion || current < block->second.start_addr ||
+		if (block->second.kind == AllocationKind::Pooled || current < block->second.start_addr ||
 		    current >= block_end) {
 			return false;
 		}
+		has_automatic |= block->second.kind == AllocationKind::Automatic;
 		current = std::min<uint64_t>(phys_addr + len, block_end);
 		if (current < phys_addr + len) {
 			const auto following = std::next(block);
@@ -1342,6 +1422,9 @@ bool PhysicalMemory::Map(uint64_t vaddr, uint64_t phys_addr, size_t len, int pro
 	mapping.mode           = mode;
 	mapping.gpu_mode       = gpu_mode;
 	m_mappings.push_back(mapping);
+	if (has_automatic) {
+		RemoveFreeRange(m_automatic_free, phys_addr, len);
+	}
 
 	return true;
 }
@@ -1362,7 +1445,7 @@ bool PhysicalMemory::CanMapDirect(uint64_t phys_addr, size_t len) {
 		}
 		--block;
 		const auto block_end = block->second.start_addr + block->second.size;
-		if (block->second.pool_expansion || current < block->second.start_addr ||
+		if (block->second.kind == AllocationKind::Pooled || current < block->second.start_addr ||
 		    current >= block_end) {
 			return false;
 		}
@@ -1380,11 +1463,12 @@ bool PhysicalMemory::CanMapDirect(uint64_t phys_addr, size_t len) {
 bool PhysicalMemory::ReleasePoolExpansion(uint64_t phys_addr, size_t len) {
 	Common::LockGuard lock(m_mutex);
 	const auto        it = m_physical.find(phys_addr);
-	if (it == m_physical.end() || !it->second.pool_expansion || it->second.size != len) {
+	if (it == m_physical.end() || it->second.kind != AllocationKind::Pooled ||
+	    it->second.size != len) {
 		return false;
 	}
 	m_physical.erase(it);
-	AddFreeRange(phys_addr, len);
+	AddFreeRange(m_free, phys_addr, len);
 	return true;
 }
 
@@ -1407,7 +1491,7 @@ bool PhysicalMemory::GetAllocatedSpan(uint64_t phys_addr, size_t len,
 		}
 		--block;
 		const auto block_end = block->second.start_addr + block->second.size;
-		if (block->second.pool_expansion || current < block->second.start_addr ||
+		if (block->second.kind == AllocationKind::Pooled || current < block->second.start_addr ||
 		    current >= block_end) {
 			blocks->clear();
 			return false;
@@ -1463,49 +1547,39 @@ bool PhysicalMemory::Unmap(uint64_t vaddr, uint64_t size, GpuAccessMode* gpu_mod
 		}
 	};
 
-	size_t index = 0;
-	for (auto& b: m_mappings) {
-		if (b.map_vaddr == vaddr && b.map_size == size) {
-			*gpu_mode             = b.gpu_mode;
-			const auto host_vaddr = b.host_vaddr;
-			const auto host_size  = b.host_size;
-
+	for (size_t index = 0; index < m_mappings.size(); ++index) {
+		auto& b = m_mappings[index];
+		if (vaddr < b.map_vaddr || vaddr >= b.map_vaddr + b.map_size ||
+		    size > b.map_vaddr + b.map_size - vaddr) {
+			continue;
+		}
+		*gpu_mode             = b.gpu_mode;
+		const auto phys_addr  = b.start_addr + vaddr - b.map_vaddr;
+		const auto host_vaddr = b.host_vaddr;
+		const auto host_size  = b.host_size;
+		if (vaddr == b.map_vaddr && size == b.map_size) {
 			m_mappings.erase(m_mappings.begin() + static_cast<std::ptrdiff_t>(index));
 			set_host_release_if_last(host_vaddr, host_size);
-
-			return true;
-		}
-		if (vaddr > b.map_vaddr && vaddr + size < b.map_vaddr + b.map_size) {
-			*gpu_mode = b.gpu_mode;
-
+		} else if (vaddr > b.map_vaddr && vaddr + size < b.map_vaddr + b.map_size) {
 			AllocatedBlock right = b;
 			right.start_addr += (vaddr + size) - b.map_vaddr;
 			right.size      = b.map_vaddr + b.map_size - (vaddr + size);
 			right.map_size  = right.size;
 			right.map_vaddr = vaddr + size;
-
 			b.size     = vaddr - b.map_vaddr;
 			b.map_size = b.size;
 			m_mappings.push_back(right);
-			return true;
-		}
-		if (vaddr == b.map_vaddr && size < b.map_size) {
-			*gpu_mode = b.gpu_mode;
-
+		} else if (vaddr == b.map_vaddr) {
 			b.start_addr += size;
 			b.size -= size;
 			b.map_vaddr += size;
 			b.map_size -= size;
-			return true;
-		}
-		if (vaddr > b.map_vaddr && vaddr + size == b.map_vaddr + b.map_size) {
-			*gpu_mode = b.gpu_mode;
-
+		} else {
 			b.size     = vaddr - b.map_vaddr;
 			b.map_size = b.size;
-			return true;
 		}
-		index++;
+		ReclaimAutomatic(phys_addr, size);
+		return true;
 	}
 
 	return false;
@@ -2726,10 +2800,8 @@ int KYTY_SYSV_ABI KernelDirectMemoryQuery(int64_t offset, int flags, void* info,
 	return OK;
 }
 
-int KYTY_SYSV_ABI KernelAllocateDirectMemory(int64_t search_start, int64_t search_end, size_t len,
-                                             size_t alignment, int memory_type,
-                                             int64_t* phys_addr_out) {
-	PRINT_NAME();
+int AllocateDirectMemory(int64_t search_start, int64_t search_end, size_t len, size_t alignment,
+                         int memory_type, int64_t* phys_addr_out, bool automatic) {
 
 	std::lock_guard<std::recursive_mutex> memory_operation_lock(g_memory_operation_mutex);
 
@@ -2748,7 +2820,10 @@ int KYTY_SYSV_ABI KernelAllocateDirectMemory(int64_t search_start, int64_t searc
 	}
 
 	uint64_t addr = 0;
-	if (!g_physical_memory->Alloc(search_start, search_end, len, alignment, &addr, memory_type)) {
+	const auto kind = automatic ? PhysicalMemory::AllocationKind::Automatic
+	                            : PhysicalMemory::AllocationKind::Direct;
+	if (!g_physical_memory->Alloc(search_start, search_end, len, alignment, &addr, memory_type,
+	                              kind)) {
 		LOGF_COLOR(Log::Color::Red, "\t[Fail]\n");
 		return KERNEL_ERROR_EAGAIN;
 	}
@@ -2760,6 +2835,14 @@ int KYTY_SYSV_ABI KernelAllocateDirectMemory(int64_t search_start, int64_t searc
 	return OK;
 }
 
+int KYTY_SYSV_ABI KernelAllocateDirectMemory(int64_t search_start, int64_t search_end, size_t len,
+                                             size_t alignment, int memory_type,
+                                             int64_t* phys_addr_out) {
+	PRINT_NAME();
+	return AllocateDirectMemory(search_start, search_end, len, alignment, memory_type,
+	                            phys_addr_out);
+}
+
 int KYTY_SYSV_ABI KernelAllocateMainDirectMemory(size_t len, size_t alignment, int memory_type,
                                                  int64_t* phys_addr_out) {
 	PRINT_NAME();
@@ -3082,6 +3165,36 @@ int KYTY_SYSV_ABI KernelMapDirectMemory2(void** addr, size_t len, int type, int
 	return ret;
 }
 
+int MapAutomaticMemory(uint64_t vaddr, size_t size, int type, int prot) {
+	std::lock_guard<std::recursive_mutex> memory_operation_lock(g_memory_operation_mutex);
+	constexpr uint64_t                    page_size = 0x4000;
+	if (size == 0 || (vaddr & (page_size - 1)) != 0 || (size & (page_size - 1)) != 0 ||
+	    vaddr < kExtendedMemoryBase || vaddr >= kExtendedMemoryBase + kExtendedMemorySize ||
+	    size > kExtendedMemoryBase + kExtendedMemorySize - vaddr) {
+		return KERNEL_ERROR_EINVAL;
+	}
+
+	PhysicalMemory::PhysicalRanges ranges;
+	if (!g_physical_memory->ReserveAutomatic(size, &ranges)) {
+		return KERNEL_ERROR_EAGAIN;
+	}
+	uint64_t mapped = 0;
+	for (const auto& [offset, length]: ranges) {
+		void*     address = reinterpret_cast<void*>(vaddr + mapped);
+		const int result  = KernelMapDirectMemory2(&address, length, type, prot, 0x10,
+		                                           static_cast<int64_t>(offset), page_size);
+		if (result != OK) {
+			if (mapped != 0) {
+				EXIT_IF(KernelMunmap(vaddr, mapped) != OK);
+			}
+			g_physical_memory->RestoreAutomatic(ranges);
+			return result;
+		}
+		mapped += length;
+	}
+	return OK;
+}
+
 int KYTY_SYSV_ABI KernelMapNamedDirectMemory(void** addr, size_t len, int prot, int flags,
                                              int64_t direct_memory_start, size_t alignment,
                                              const char* name) {
@@ -3892,7 +4005,7 @@ int KYTY_SYSV_ABI KernelMemoryPoolExpand(int64_t search_start, int64_t search_en
 	uint64_t   phys_addr           = 0;
 	if (!g_physical_memory->Alloc(static_cast<uint64_t>(search_start),
 	                              static_cast<uint64_t>(search_end), len, effective_alignment,
-	                              &phys_addr, 0, true)) {
+	                              &phys_addr, 0, PhysicalMemory::AllocationKind::Pooled)) {
 		return KERNEL_ERROR_ENOMEM;
 	}
 

src/kernel/memory.h

@@ -24,6 +24,8 @@ struct Lifecycle {
 using callback_func_t = void (*)(uintptr_t addr, size_t size);
 
 constexpr uint32_t KERNEL_MAXIMUM_NAME_LENGTH = 32;
+constexpr uint64_t kExtendedMemoryBase       = 0x080000000000ull;
+constexpr uint64_t kExtendedMemorySize       = 512ull * 1024 * 1024 * 1024;
 
 struct VirtualQueryInfo {
 	uintptr_t start;
@@ -107,6 +109,9 @@ static_assert(sizeof(KernelMemoryPoolBlockStats) == 16,
 
 void                   RegisterCallbacks(callback_func_t alloc_func, callback_func_t free_func);
 void                   SetFlexibleMemorySize(uint64_t size);
+int AllocateDirectMemory(int64_t search_start, int64_t search_end, size_t size, size_t alignment,
+                         int memory_type, int64_t* phys_addr_out, bool automatic = false);
+int MapAutomaticMemory(uint64_t vaddr, size_t size, int type, int prot);
 bool                   TryWriteBacking(uint64_t vaddr, const void* data, uint64_t size);
 bool                   TryReadBacking(uint64_t vaddr, void* data, uint64_t size);
 bool                   TryReadGpuCleanBacking(uint64_t vaddr, void* data, uint64_t size);

src/kernel/memoryAddressSpace.inc

@@ -3,7 +3,6 @@
 
 static constexpr uint64_t HOST_SYSTEM_MANAGED_MIN  = 0x0000040000ull;
 static constexpr uint64_t HOST_SYSTEM_MANAGED_MAX  = 0x07ffffbfffull;
-static constexpr uint64_t HOST_SYSTEM_RESERVED_MIN = HOST_SYSTEM_MANAGED_MAX + 1u;
 #if defined(__APPLE__) && (defined(__x86_64__) || defined(_M_X64))
 static constexpr uint64_t HOST_SYSTEM_RESERVED_MAX = 0x0fbfffffffull;
 static constexpr uint64_t HOST_USER_MIN            = 0x7000000000ull;
@@ -1077,51 +1076,51 @@ private:
 #endif
 
 	void ReserveGuestRegions() {
+		const std::array<std::pair<uint64_t, uint64_t>, 3> regions {{
+		    {HOST_SYSTEM_MANAGED_MIN, HOST_SYSTEM_RESERVED_MAX + 1u},
+		    {HOST_USER_MIN, HOST_USER_MAX + 1u},
+		    {kExtendedMemoryBase, kExtendedMemoryBase + kExtendedMemorySize},
+		}};
+
 #if KYTY_PLATFORM == KYTY_PLATFORM_WINDOWS
 		constexpr uint64_t MinimumRegionSize = 0x1000000;
 		const auto         process           = GetCurrentProcess();
-		const auto         guest_end         = HOST_USER_MAX + 1u;
-		auto               next_addr         = HOST_SYSTEM_MANAGED_MIN;
-
-		while (next_addr < guest_end) {
-			MEMORY_BASIC_INFORMATION info {};
-			if (VirtualQuery(reinterpret_cast<const void*>(next_addr), &info, sizeof(info)) == 0) {
-				EXIT("failed to inspect guest address space at 0x%016" PRIx64 "\n", next_addr);
-			}
-
-			const auto region_base = reinterpret_cast<uint64_t>(info.BaseAddress);
-			const auto region_end =
-			    std::min<uint64_t>(guest_end, region_base + static_cast<uint64_t>(info.RegionSize));
-			const auto reserve_start =
-			    AlignUp(std::max<uint64_t>(region_base, HOST_SYSTEM_MANAGED_MIN), m_granularity);
-			const auto reserve_end = AlignDown(region_end, m_granularity);
-
-			if (info.State == MEM_FREE && reserve_end > reserve_start &&
-			    reserve_end - reserve_start > MinimumRegionSize) {
-				auto* ptr = VirtualAlloc2(
-				    process, reinterpret_cast<void*>(reserve_start), reserve_end - reserve_start,
-				    MEM_RESERVE | MEM_RESERVE_PLACEHOLDER, PAGE_NOACCESS, nullptr, 0);
-				if (reinterpret_cast<uint64_t>(ptr) != reserve_start) {
-					EXIT("failed to reserve guest address space at 0x%016" PRIx64
-					     ", size 0x%016" PRIx64 "\n",
-					     reserve_start, reserve_end - reserve_start);
+		for (const auto& [guest_start, guest_end]: regions) {
+			auto next_addr = guest_start;
+			while (next_addr < guest_end) {
+				MEMORY_BASIC_INFORMATION info {};
+				if (VirtualQuery(reinterpret_cast<const void*>(next_addr), &info, sizeof(info)) == 0) {
+					EXIT("failed to inspect guest address space at 0x%016" PRIx64 "\n", next_addr);
 				}
-				AddFreeUnlocked(reserve_start, reserve_end - reserve_start);
-				m_owned.emplace_back(reserve_start, reserve_end - reserve_start);
-			}
 
-			if (region_end <= next_addr) {
-				EXIT("guest address-space scan did not advance at 0x%016" PRIx64 "\n", next_addr);
+				const auto region_base = reinterpret_cast<uint64_t>(info.BaseAddress);
+				const auto region_end = std::min<uint64_t>(
+				    guest_end, region_base + static_cast<uint64_t>(info.RegionSize));
+				const auto reserve_start =
+				    AlignUp(std::max(region_base, guest_start), m_granularity);
+				const auto reserve_end = AlignDown(region_end, m_granularity);
+
+				if (info.State == MEM_FREE && reserve_end > reserve_start &&
+				    reserve_end - reserve_start > MinimumRegionSize) {
+					auto* ptr = VirtualAlloc2(
+					    process, reinterpret_cast<void*>(reserve_start), reserve_end - reserve_start,
+					    MEM_RESERVE | MEM_RESERVE_PLACEHOLDER, PAGE_NOACCESS, nullptr, 0);
+					if (reinterpret_cast<uint64_t>(ptr) != reserve_start) {
+						EXIT("failed to reserve guest address space at 0x%016" PRIx64
+						     ", size 0x%016" PRIx64 "\n",
+						     reserve_start, reserve_end - reserve_start);
+					}
+					AddFreeUnlocked(reserve_start, reserve_end - reserve_start);
+					m_owned.emplace_back(reserve_start, reserve_end - reserve_start);
+				}
+
+				if (region_end <= next_addr) {
+					EXIT("guest address-space scan did not advance at 0x%016" PRIx64 "\n", next_addr);
+				}
+				next_addr = region_end;
 			}
-			next_addr = region_end;
 		}
 #else
-		const std::array<std::pair<uint64_t, uint64_t>, 3> regions {{
-		    {HOST_SYSTEM_MANAGED_MIN, HOST_SYSTEM_MANAGED_MAX + 1u},
-		    {HOST_SYSTEM_RESERVED_MIN, HOST_SYSTEM_RESERVED_MAX + 1u},
-		    {HOST_USER_MIN, HOST_USER_MAX + 1u},
-		}};
-
 #if KYTY_PLATFORM == KYTY_PLATFORM_LINUX && !defined(__APPLE__)
 		std::vector<std::pair<uint64_t, uint64_t>> occupied;
 		FILE*                                      maps = fopen("/proc/self/maps", "r");

src/libs/libAmpr.cpp

@@ -685,8 +685,6 @@ constexpr int      PROT_AMPR_READ                = 0x40;
 constexpr int      PROT_AMPR_WRITE               = 0x80;
 constexpr int      PROT_ACP_READ                 = 0x100;
 constexpr int      PROT_ACP_WRITE                = 0x200;
-constexpr uint64_t AMM_VA_START                  = 0x0000001000000000ull;
-constexpr uint64_t AMM_VA_SIZE                   = 0x0000001000000000ull;
 constexpr uint64_t APR_MAX_READ_LENGTH           = 0x0000000100000000ull;
 constexpr uint64_t APR_MAX_FILE_OFFSET           = 0x0000010000000000ull;
 constexpr uint64_t APR_MAX_APP_ADDRESS           = 0x0000f00000000000ull;
@@ -744,19 +742,6 @@ struct CommandBufferState {
 	uint64_t                         gather_scatter_file_offset = 0;
 };
 
-struct AmmAutoPoolRange {
-	uint64_t start = 0;
-	uint64_t size  = 0;
-	uint64_t used  = 0;
-};
-
-struct AmmVirtualAddressRanges {
-	uint64_t va_start          = 0;
-	uint64_t va_end            = 0;
-	uint64_t multimap_va_start = 0;
-	uint64_t multimap_va_end   = 0;
-};
-
 struct AmmUsageStatsData {
 	uint64_t size_in_bytes                                    = 0;
 	uint16_t num_page_table_pool_entries                      = 0;
@@ -770,8 +755,6 @@ static_assert(sizeof(AmmUsageStatsData) == 0x18);
 static std::mutex                                       g_command_buffer_mutex;
 static std::unordered_map<uint64_t, CommandBufferState> g_command_buffers;
 static std::unordered_map<uint64_t, uint64_t>           g_command_buffer_aliases;
-static std::mutex                                       g_amm_auto_pool_mutex;
-static std::vector<AmmAutoPoolRange>                    g_amm_auto_pool;
 using CommandBufferIterator = std::unordered_map<uint64_t, CommandBufferState>::iterator;
 
 static bool HasQueuedCommands(const CommandBufferState& state) {
@@ -1227,48 +1210,20 @@ static int NormalizeAmmProtection(int prot) {
 	return normalized;
 }
 
-static bool AllocateAmmAutoDirectMemory(uint64_t size, int memory_type, uint64_t* dmem_offset) {
-	if (dmem_offset == nullptr || size == 0) {
-		return false;
-	}
-
-	std::scoped_lock lock(g_amm_auto_pool_mutex);
-
-	for (auto& range: g_amm_auto_pool) {
-		const auto aligned_used = (range.used + (AMM_PAGE_SIZE - 1u)) & ~(AMM_PAGE_SIZE - 1u);
-		if (aligned_used <= range.size && size <= range.size - aligned_used) {
-			*dmem_offset = range.start + aligned_used;
-			range.used   = aligned_used + size;
-			return true;
-		}
-	}
-
-	int64_t allocated = 0;
-	if (LibKernel::Memory::KernelAllocateDirectMemory(
-	        0, LibKernel::Memory::KernelGetDirectMemorySize(), size, AMM_PAGE_SIZE, memory_type,
-	        &allocated) != OK) {
-		return false;
-	}
-
-	*dmem_offset = static_cast<uint64_t>(allocated);
-	return true;
-}
-
 static int ExecuteAmmMapCommand(const CommandBufferState::AmmMapCommand& command) {
 	if (!ValidateAmmMapArgs(command.va, command.size)) {
 		return LibKernel::KERNEL_ERROR_EINVAL;
 	}
 
-	uint64_t dmem_offset = command.dmem_offset;
-	if (command.kind == AmmCommandKind::MapAuto &&
-	    !AllocateAmmAutoDirectMemory(command.size, command.type, &dmem_offset)) {
-		return LibKernel::KERNEL_ERROR_EAGAIN;
+	if (command.kind == AmmCommandKind::MapAuto) {
+		return LibKernel::Memory::MapAutomaticMemory(command.va, command.size, command.type,
+		                                             NormalizeAmmProtection(command.prot));
 	}
 
 	void* addr = reinterpret_cast<void*>(command.va);
 	return LibKernel::Memory::KernelMapDirectMemory2(
 	    &addr, command.size, command.type, NormalizeAmmProtection(command.prot), AMM_MAP_FIXED,
-	    static_cast<int64_t>(dmem_offset), AMM_PAGE_SIZE);
+	    static_cast<int64_t>(command.dmem_offset), AMM_PAGE_SIZE);
 }
 
 static bool AppendAmmMapRecord(uint64_t                                 command_buffer,
@@ -2360,25 +2315,15 @@ static int KYTY_SYSV_ABI AmmGiveDirectMemory(int64_t search_start, int64_t searc
                                              int64_t* dmem_offset) {
 	PRINT_NAME();
 
-	if (dmem_offset == nullptr || size == 0 ||
-	    (usage != AMM_USAGE_DIRECT && usage != AMM_USAGE_AUTO)) {
+	constexpr uint64_t block_size = 0x200000;
+	if (dmem_offset == nullptr || size == 0 || (size & (block_size - 1)) != 0 ||
+	    (align & (block_size - 1)) != 0 || (usage != AMM_USAGE_DIRECT && usage != AMM_USAGE_AUTO)) {
 		return LibKernel::KERNEL_ERROR_EINVAL;
 	}
 
-	int64_t allocated = 0;
-	int     result = LibKernel::Memory::KernelAllocateDirectMemory(search_start, search_end, size,
-	                                                               align, 0, &allocated);
-	if (result != OK) {
-		return result;
-	}
-
-	*dmem_offset = allocated;
-	if (usage == AMM_USAGE_AUTO) {
-		std::scoped_lock lock(g_amm_auto_pool_mutex);
-		g_amm_auto_pool.push_back(AmmAutoPoolRange {static_cast<uint64_t>(allocated), size, 0});
-	}
-
-	return OK;
+	return LibKernel::Memory::AllocateDirectMemory(search_start, search_end, size,
+	                                               align != 0 ? align : block_size, 0, dmem_offset,
+	                                               usage == AMM_USAGE_AUTO);
 }
 
 static void KYTY_SYSV_ABI AmmGetVirtualAddressRanges(uint64_t* va_start, uint64_t* va_end,
@@ -2386,19 +2331,18 @@ static void KYTY_SYSV_ABI AmmGetVirtualAddressRanges(uint64_t* va_start, uint64_
                                                      uint64_t* multimap_va_end) {
 	PRINT_NAME();
 
+	constexpr auto end = LibKernel::Memory::kExtendedMemoryBase + LibKernel::Memory::kExtendedMemorySize;
 	if (va_start != nullptr) {
-		AprShared::WriteGuest(reinterpret_cast<uint64_t>(va_start), AMM_VA_START);
+		AprShared::WriteGuest(reinterpret_cast<uint64_t>(va_start), LibKernel::Memory::kExtendedMemoryBase);
 	}
 	if (va_end != nullptr) {
-		AprShared::WriteGuest(reinterpret_cast<uint64_t>(va_end), AMM_VA_START + AMM_VA_SIZE);
+		AprShared::WriteGuest(reinterpret_cast<uint64_t>(va_end), end);
 	}
 	if (multimap_va_start != nullptr) {
-		AprShared::WriteGuest(reinterpret_cast<uint64_t>(multimap_va_start),
-		                      AMM_VA_START + AMM_VA_SIZE / 2u);
+		AprShared::WriteGuest(reinterpret_cast<uint64_t>(multimap_va_start), end);
 	}
 	if (multimap_va_end != nullptr) {
-		AprShared::WriteGuest(reinterpret_cast<uint64_t>(multimap_va_end),
-		                      AMM_VA_START + AMM_VA_SIZE);
+		AprShared::WriteGuest(reinterpret_cast<uint64_t>(multimap_va_end), end);
 	}
 }
 

src/libs/ngs2.cpp

@@ -1435,6 +1435,11 @@ static void Ngs2FinishBlock(Ngs2VoiceInternal& voice) {
 		}
 		++block.repeated_count;
 		block.cursor = 0;
+		if (voice.decoder != nullptr) {
+			block.data_cursor = 0;
+			voice.compressed_input.clear();
+			voice.decoder->Reset();
+		}
 	}
 	const Ngs2VoiceCallbackInfo info {voice.callback_data,
 	                                  reinterpret_cast<uintptr_t>(&voice),
@@ -2396,8 +2401,7 @@ int KYTY_SYSV_ABI Ngs2VoiceControl(uintptr_t voice_handle, const Ngs2VoiceParamH
 								block.num_samples      = 0;
 							}
 							EXIT_NOT_IMPLEMENTED(block.num_repeats != 0 &&
-							                     (voice->rack->type != Ngs2RackType::Sampler ||
-							                      voice->decoder != nullptr));
+							                     voice->rack->type != Ngs2RackType::Sampler);
 							EXIT_NOT_IMPLEMENTED(
 							    voice->decoder == nullptr &&
 							    (uint64_t(block.num_skip_samples) + block.num_samples) *

tests/ImagePageTableTests.cpp

@@ -9,9 +9,9 @@
 namespace {
 
 using Owners     = std::vector<uint32_t>;
-using Table      = Libs::Graphics::MultiLevelPageTable<Owners>;
+using Table      = Libs::Graphics::MultiLevelPageTable<Owners, 20, 44, 14>;
 using PageOwners = Libs::Graphics::InlinePageOwnerList<uint32_t, 16>;
-using OwnerTable = Libs::Graphics::MultiLevelPageTable<PageOwners, 20, 40, 10>;
+using OwnerTable = Libs::Graphics::MultiLevelPageTable<PageOwners, 20, 44, 14>;
 
 void Check(bool value, const char* text) {
 	if (!value) {

tests/MemoryTrackerTests.cpp

@@ -219,17 +219,24 @@ void TestRangeSet() {
 void TestGuestRange() {
   constexpr GuestRange empty{};
   constexpr GuestRange first_byte{1, 1};
-  constexpr GuestRange last_byte{TRACKER_ADDRESS_SIZE - 1, 1};
+  constexpr uint64_t extended_end = Libs::LibKernel::Memory::kExtendedMemoryBase +
+                                    Libs::LibKernel::Memory::kExtendedMemorySize;
+  constexpr GuestRange last_byte{extended_end - 1, 1};
 
   static_assert(empty.Empty() && !empty.Valid() && empty.ValidOrEmpty());
   static_assert(!first_byte.Empty() && first_byte.Valid() &&
                 first_byte.ValidOrEmpty() && first_byte.End() == 2);
-  static_assert(last_byte.Valid() && last_byte.End() == TRACKER_ADDRESS_SIZE);
+  static_assert(last_byte.Valid() && last_byte.End() == extended_end);
 
   Check(!GuestRange{0, 1}.Empty() && !GuestRange{0, 1}.ValidOrEmpty(),
         "zero-address nonempty guest range is rejected");
   Check(!GuestRange{1, 0}.Empty() && !GuestRange{1, 0}.ValidOrEmpty(),
         "nonzero-address empty guest range is rejected");
+  Check(GuestRange{Libs::LibKernel::Memory::kExtendedMemoryBase, 1}.Valid() &&
+            !GuestRange{Libs::Graphics::LOWER_ADDRESS_SIZE, 1}.Valid() &&
+            !GuestRange{Libs::LibKernel::Memory::kExtendedMemoryBase - 1, 2}.Valid() &&
+            !GuestRange{extended_end - 1, 2}.Valid(),
+        "extended range and gap boundaries are enforced");
   Check(!GuestRange{TRACKER_ADDRESS_SIZE, 1}.Valid(),
         "first address beyond the guest range is rejected");
   Check(!GuestRange{TRACKER_ADDRESS_SIZE - 1, 2}.Valid(),

tests/Ngs2SamplerTests.cpp

@@ -712,6 +712,74 @@ void TestLoopCallbacksAndExit() {
 	Ngs2VoiceControl(reinterpret_cast<uintptr_t>(&pending.voice), &exit_loop);
 }
 
+void TestAtrac9Looping() {
+	// Four encoded mono frames with one nonzero spectral coefficient, then superframe padding.
+	// The block skips one frame plus one sample and ends inside the third frame.
+	constexpr std::array<uint8_t, 36> frame {0, 0, 0x04, 0x20, 0x04, 0xc0, 0, 0, 0x1f, 0xc0};
+	std::array<uint8_t, 192> encoded {};
+	for (size_t i = 0; i < 4; ++i) {
+		std::copy(std::begin(frame), std::end(frame), encoded.begin() + i * sizeof(frame));
+		if (i != 0) encoded[i * sizeof(frame)] |= 0x80;
+	}
+	const auto make_voice = [&](uint32_t repeats, LoopCallbacks& log) {
+		auto f = std::make_unique<Fixture>(24000);
+		f->voice.SetupSampler({NGS2_WAVEFORM_TYPE_ATRAC9, 1, 24000, 0xfe4005f0});
+		f->voice.state          = Ngs2VoicePlayState::Playing;
+		f->voice.callback       = reinterpret_cast<uintptr_t>(&RecordLoopCallback);
+		f->voice.callback_data  = reinterpret_cast<uintptr_t>(&log);
+		f->voice.callback_flags = 3;
+		const Ngs2WaveformBlock block {0, encoded.size(), repeats, 129, 137, 0, 123};
+		struct Blocks {
+			Ngs2VoiceParamHeader     header {32, 0, 0x10000001};
+			const uint8_t*           data;
+			uint32_t                 flags, count;
+			const Ngs2WaveformBlock* blocks;
+		} param {{32, 0, 0x10000001}, encoded.data(), 0, 1, &block};
+		Check(Ngs2VoiceControl(reinterpret_cast<uintptr_t>(&f->voice), &param.header) == OK,
+		      "compressed loop queue failed");
+		return f;
+	};
+	LoopCallbacks baseline_log, finite_log, split_log, infinite_log;
+	auto baseline = make_voice(0, baseline_log);
+	baseline->Render(274);
+	const auto reference = baseline->voice.samples;
+	Check(std::ranges::any_of(reference, [](float sample) { return std::abs(sample) > 0.00001f; }),
+	      "synthetic compressed fixture produced no decoded signal");
+	auto finite = make_voice(2, finite_log);
+	finite->Render(822);
+	Check(finite->voice.state == Ngs2VoicePlayState::Empty &&
+	          finite->voice.decoded_samples == 411 && finite->voice.decoded_bytes == 576 &&
+	          finite_log.events == std::vector<std::pair<uint32_t, uint32_t>> {{2, 1}, {2, 2}, {1, 2}},
+	      "compressed finite loop lost samples, input progress, or completion callbacks");
+	for (size_t i = 0; i < 3; ++i) {
+		Check(std::equal(reference.begin(), reference.end() - 1,
+		                 finite->voice.samples.begin() + i * 274),
+		      "compressed loop did not restore decoder history and skip samples");
+		const float tail = i == 2 ? reference.back() : std::lerp(reference[272], reference[0], 0.5f);
+		Check(finite->voice.samples[i * 274 + 273] == tail,
+		      "compressed loop dropped the final frame or broke boundary interpolation");
+	}
+	auto split = make_voice(2, split_log);
+	split->Render(273);
+	auto joined = split->voice.samples;
+	split->Render(549);
+	joined.insert(joined.end(), split->voice.samples.begin(), split->voice.samples.end());
+	Check(joined == finite->voice.samples && split_log.events == finite_log.events,
+	      "compressed loop changed across render grains");
+	auto infinite = make_voice(UINT32_MAX, infinite_log);
+	infinite->Render(300);
+	Check(infinite->voice.state == Ngs2VoicePlayState::Playing &&
+	          infinite->voice.blocks.front().info.num_repeats == UINT32_MAX,
+	      "compressed infinite loop did not rewind");
+	Ngs2VoiceParamHeader exit_loop {sizeof(Ngs2VoiceParamHeader), 0, 0x10000004};
+	Ngs2VoiceControl(reinterpret_cast<uintptr_t>(&infinite->voice), &exit_loop);
+	infinite->Render(248);
+	Check(infinite->voice.state == Ngs2VoicePlayState::Empty &&
+	          infinite->voice.decoded_samples == 274 &&
+	          infinite_log.events == std::vector<std::pair<uint32_t, uint32_t>> {{2, 1}, {1, 1}},
+	      "compressed loop exit failed to finish the active iteration");
+}
+
 void TestWaveformReadAddress() {
 	Fixture       f(48000, 2);
 	const int16_t pcm[] = {-100, -100, 8192, 4096, 16384, 8192};
@@ -921,6 +989,7 @@ int main() {
 	TestSamplerReuseResetsRouting();
 	TestDeferredSamplerSetup();
 	TestLoopCallbacksAndExit();
+	TestAtrac9Looping();
 	TestWaveformReadAddress();
 	TestFilterTailDuringStarvation();
 	TestFiniteFilterTail();

tests/PageManagerTests.cpp

@@ -136,8 +136,8 @@ bool ProtectAddressSpace(uint64_t vaddr, uint64_t size,
                         &old_protection) != 0;
 }
 
-uint8_t *Allocate(uint64_t size, uint32_t protection = PAGE_READWRITE) {
-  constexpr uintptr_t test_address = 0x0000000200010000ull;
+uint8_t *Allocate(uint64_t size, uint32_t protection = PAGE_READWRITE,
+                  uintptr_t test_address = 0x0000000200010000ull) {
 #if KYTY_PLATFORM == KYTY_PLATFORM_WINDOWS
   auto *memory = static_cast<uint8_t *>(
       VirtualAlloc(reinterpret_cast<void *>(test_address), size,
@@ -155,11 +155,11 @@ uint8_t *Allocate(uint64_t size, uint32_t protection = PAGE_READWRITE) {
   return memory;
 }
 
-void TestWatchAndUnwatch() {
+void TestWatchAndUnwatch(uint64_t base = 0x0000000200010000ull) {
   g_protection_calls = 0;
   PageManager manager;
   const auto page_size = manager.GetPageSize();
-  auto *memory = Allocate(page_size * 2);
+  auto *memory = Allocate(page_size * 2, PAGE_READWRITE, base);
   const auto address = reinterpret_cast<uint64_t>(memory);
 
   manager.UpdatePageWatchers<true>(address, page_size);
@@ -580,6 +580,7 @@ int main(int argc, char **argv) {
     RunDeathCase(argv[2]);
   }
   TestWatchAndUnwatch();
+  TestWatchAndUnwatch(Libs::LibKernel::Memory::kExtendedMemoryBase);
   TestSharedWatcherCounts();
   TestCrossRegionRange();
   TestBatchedWatcherRanges();

tests/ResourceTrackingTests.cpp

@@ -1,6 +1,7 @@
 #include "graphics/guest_gpu/gpu_defs.h"
 #include "graphics/shader/recompiler/ir/ShaderIR.h"
 #include "graphics/shader/recompiler/ir/passes/BindingLayout.h"
+#include "graphics/shader/recompiler/ir/passes/ConstantPropagation.h"
 #include "graphics/shader/recompiler/ir/passes/DeadCodeElimination.h"
 #include "graphics/shader/recompiler/ir/passes/ResourceMaterialization.h"
 #include "graphics/shader/recompiler/ir/passes/ResourceTracking.h"
@@ -126,8 +127,28 @@ struct Fixture {
   }
 
   void PlanAndTrack() {
-    BuildSrtPlan(program);
-    TrackResources(program);
+    for (size_t index = 0; index < program.block_info.size(); ++index) {
+      const auto condition = program.block_info[index].condition;
+      if (!condition.IsEmpty())
+        Emit(ValueOpcode::Reference, {condition}, 0, program.blocks[index]);
+    }
+    for (auto *target : program.blocks) {
+      for (auto &inst : *target) {
+        if (inst.HasUses() || inst.MayHaveSideEffects() ||
+            (BufferAccessOf(inst.GetOpcode()) == BufferAccess::None &&
+             AddressOpcodeInfoOf(inst.GetOpcode()).access == AddressAccess::None &&
+             ImageOpcodeInfoOf(inst.GetOpcode()).access == ImageAccess::None))
+          continue;
+        auto value = Value(&inst);
+        if (value.GetType() == Type::U32x4)
+          value = Emit(ValueOpcode::CompositeExtractU32x4, {value, Value(0u)}, 0, target);
+        if (value.GetType() == Type::U8)
+          value = Emit(ValueOpcode::ConvertU32U8, {value}, 0, target);
+        Check(value.GetType() == Type::U32, "unhandled memory result type in fixture");
+        Emit(ValueOpcode::ReferenceU32, {value}, 0, target);
+      }
+    }
+    TrackResources(program, {}, {});
   }
 };
 
@@ -185,8 +206,9 @@ bool ReadLinearTestMemory(void *userdata, uint64_t address, std::span<uint32_t>
 }
 
 std::unique_ptr<Fixture>
-MakeIndirectImageFixture(bool malformed, uint32_t material_immediate = 0,
-                         bool memory_backed_material = false) {
+MakeIndirectImageFixture(bool malformed, uint32_t material_immediate = 4,
+                         bool memory_backed_material = false, uint32_t member_offset = 0,
+                         uint32_t material_stride = 224) {
   auto fixture = std::make_unique<Fixture>();
   std::array<Value, 4> material_words;
   std::array<Value, 4> heap_words;
@@ -228,9 +250,9 @@ MakeIndirectImageFixture(bool malformed, uint32_t material_immediate = 0,
   const auto selector = fixture->Emit(ValueOpcode::ReadFirstLane,
                                       {invocation, Value(true)});
   const auto record =
-      fixture->Emit(ValueOpcode::IMul32, {selector, Value(224u)});
-  const auto member = fixture->Emit(ValueOpcode::IAdd32, {record, Value(4u)});
-  fixture->Emit(ValueOpcode::ReferenceU32, {record});
+      fixture->Emit(ValueOpcode::IMul32, {selector, Value(material_stride)});
+  const auto member = member_offset == 0 ? record :
+      fixture->Emit(ValueOpcode::IAdd32, {record, Value(member_offset)});
   fixture->Emit(ValueOpcode::ReferenceU32, {member});
   MemoryInfo material_scalar;
   material_scalar.kind = ResourceKind::ScalarBuffer;
@@ -277,7 +299,9 @@ void TestInvariantIndirectImageMaterialization() {
 
   Check(fixture->program.info.buffers.size() == 1 &&
             fixture->program.info.images.size() == 1 &&
-            fixture->program.dynamic_reads.size() == 1,
+            std::ranges::any_of(*fixture->block, [](const Inst &inst) {
+              return inst.GetOpcode() == ValueOpcode::ReadConstBuffer;
+            }),
         "indirect image key was not retained as a scalar-buffer read");
   const auto source = fixture->program.info.images[0].source;
   Check(source < fixture->program.descriptor_sources.size() &&
@@ -474,7 +498,7 @@ void TestInvariantIndirectImageMaterialization() {
             rebound_specialization != collapsed_specialization,
         "larger indirect candidate topology reused the old specialization");
 
-  auto memory_backed = MakeIndirectImageFixture(false, 0u, true);
+  auto memory_backed = MakeIndirectImageFixture(false, 4u, true);
   memory_backed->PlanAndTrack();
   auto memory_backed_plan = ExtractResourcePlan(memory_backed->program);
   EliminateDeadCode(memory_backed->program.blocks);
@@ -519,28 +543,50 @@ void TestInvariantIndirectImageMaterialization() {
         "a same-shape refresh discarded the runtime output storage");
 
   auto malformed = MakeIndirectImageFixture(true);
-  BuildSrtPlan(malformed->program);
-  CheckFatal([&] { TrackResources(malformed->program); }, "not a valid runtime value",
+
+  CheckFatal([&] { malformed->PlanAndTrack(); }, "not a valid runtime value",
              "malformed indirect image pattern was accepted");
   Check(!malformed->program.resource_tracking_complete &&
             malformed->program.info.images.empty() &&
             malformed->program.descriptor_sources.empty(),
         "malformed indirect image pattern was partially accepted");
 
-  auto wrapped_immediate = MakeIndirectImageFixture(false, 4u);
-  BuildSrtPlan(wrapped_immediate->program);
-  CheckFatal([&] { TrackResources(wrapped_immediate->program); },
+  auto negative_immediate = MakeIndirectImageFixture(false, 0xfffffffcu);
+  CheckFatal([&] { negative_immediate->PlanAndTrack(); },
              "not a valid runtime value",
-             "wrapped scalar immediate entered the invariant image proof");
-  Check(!wrapped_immediate->program.resource_tracking_complete,
-        "wrapped scalar immediate entered the invariant image proof");
+             "negative scalar immediate entered the indirect image proof");
+
+  for (const auto [immediate, member, first, stride] :
+       {std::array{0u, 4u, 4u, 224u}, std::array{4u, 4u, 8u, 224u},
+        std::array{36u, 0u, 36u, 224u}, std::array{1u, 3u, 0u, 224u},
+        std::array{1u, 3u, 0u, 1u}, std::array{1u, 3u, 0u, 2u}}) {
+    auto split_offset = MakeIndirectImageFixture(false, immediate, false, member, stride);
+    split_offset->PlanAndTrack();
+    const auto split_plan = ExtractResourcePlan(split_offset->program);
+    LinearTestMemory split_memory;
+    std::copy(image_descriptor.begin(), image_descriptor.end(),
+              split_memory.words.begin() + 0x1000u / 4u);
+    split_memory.fail_address = first == 36u ? 0x1004u : UINT64_MAX;
+    split_memory.watched_address = split_memory.base + first;
+    user_data[1] = stride << 16u;
+    user_data[2] = stride < 4u ? 8u / stride : 1u;
+    SrtRuntime split_runtime{.user_data = user_data,
+                             .userdata = &split_memory,
+                             .read_specialization_memory = ReadLinearTestMemory};
+    Check(MaterializeResources(split_plan, split_runtime, snapshot, specialization) &&
+              snapshot.images.size() == 1 &&
+              snapshot.images[0].dwords == image_descriptor &&
+              split_memory.watched_reads == 1u,
+          "indirect scalar offsets did not align and bound their components independently");
+  }
 }
 
 void TestGuardedDirectImageTable() {
   namespace CFG = Libs::Graphics::ShaderRecompiler::CFG;
-  enum class Guard { Nonzero, Zero, Unrelated, Bypass };
+  enum class Guard { Nonzero, SccNonZero, Plain, Zero, Unrelated, Bypass, ExecZero, VccZero };
   const auto make_plan = [](Guard guard) {
     Fixture fixture(ShaderType::Pixel);
+    fixture.program.wave_size = 64u;
     auto *entry = fixture.block;
     auto *middle = fixture.AddBlock();
     auto *before_sample = fixture.AddBlock();
@@ -557,8 +603,19 @@ void TestGuardedDirectImageTable() {
     const auto nonzero = fixture.Emit(
         ValueOpcode::INotEqual32,
         {Value(0u), guard == Guard::Unrelated ? fixture.UserData(2) : mask});
-    fixture.program.block_info[0].condition =
-        fixture.Emit(ValueOpcode::LogicalNot, {nonzero});
+    const auto kind = guard == Guard::ExecZero ? CFG::BranchCondition::ExecZero
+                    : guard == Guard::VccZero ? CFG::BranchCondition::VccZero
+                                             : CFG::BranchCondition::SccZero;
+    auto condition = nonzero;
+    if (guard == Guard::SccNonZero) {
+      condition = fixture.Emit(ValueOpcode::ConditionRef, {condition},
+                               CFG::BranchCondition::SccNonZero);
+    }
+    condition = fixture.Emit(ValueOpcode::LogicalNot, {condition});
+    if (guard != Guard::Plain && guard != Guard::SccNonZero) {
+      condition = fixture.Emit(ValueOpcode::ConditionRef, {condition}, kind);
+    }
+    fixture.program.block_info[0].condition = condition;
     fixture.program.block_info[0].terminator = {
         .kind = CFG::TerminatorKind::ConditionalBranch,
         .true_block = guard == Guard::Zero ? 1u : 4u,
@@ -622,7 +679,10 @@ void TestGuardedDirectImageTable() {
   };
 
   auto plan = make_plan(Guard::Nonzero);
-  for (const auto guard : {Guard::Zero, Guard::Unrelated, Guard::Bypass}) {
+  make_plan(Guard::SccNonZero);
+  make_plan(Guard::Plain);
+  for (const auto guard : {Guard::Zero, Guard::Unrelated, Guard::Bypass,
+                           Guard::ExecZero, Guard::VccZero}) {
     CheckFatal([&] { make_plan(guard); }, "not a valid runtime value",
                "direct table accepted a selector without a dominating nonzero guard");
   }
@@ -703,55 +763,183 @@ void TestGuardedDirectImageTable() {
 void TestBoundedComputeImageLoop() {
   namespace CFG = Libs::Graphics::ShaderRecompiler::CFG;
   enum class Variant {
-    Bounded, WrongGuard, EntryBypass, ExitBypass, GuardBlock, WrongStep
+    Bounded, Plain, Nonzero, TrueEdge, WrongGuard, EntryBypass, ExitBypass, GuardBlock,
+    WrongStep, DivergentBound, DivergentKey, Disjunction, WrongPolarity,
+    IncrementBypass, PreviousBound, Masked, MaskedWrongGuard, MaskResurrection, StatusOverwrite,
+    GuardedDiamond, GuardedDiamondBypass
   };
   const auto make_plan = [](Variant variant) {
-    Fixture fixture;
+    const bool diamond = variant >= Variant::GuardedDiamond;
+    Fixture fixture(diamond ? ShaderType::Vertex : ShaderType::Compute);
+    fixture.program.wave_size = 64u;
+    const bool masked = variant >= Variant::Masked;
     auto *entry = fixture.block;
     auto *header = fixture.AddBlock();
     auto *body = fixture.AddBlock();
     auto *latch = fixture.AddBlock();
     auto *exit = fixture.AddBlock();
+    auto *compare = masked ? fixture.AddBlock() : header;
+    auto *guard = masked ? fixture.AddBlock() : header;
+    auto *increment = masked ? fixture.AddBlock() : latch;
+    auto *final_exit = variant == Variant::PreviousBound ? fixture.AddBlock() : exit;
     entry->AddBranch(header);
-    header->AddBranch(exit);
-    header->AddBranch(body);
-    body->AddBranch(latch);
-    latch->AddBranch(header);
-    if (variant == Variant::EntryBypass) entry->AddBranch(body);
-    if (variant == Variant::ExitBypass) exit->AddBranch(body);
-    fixture.program.block_info[0].terminator = {
-        .kind = variant == Variant::EntryBypass ? CFG::TerminatorKind::ConditionalBranch
-                                               : CFG::TerminatorKind::Branch,
-        .true_block = 1u, .false_block = 2u};
-    fixture.program.block_info[1].terminator = {
-        .kind = CFG::TerminatorKind::ConditionalBranch,
-        .true_block = 4u, .false_block = 2u};
-    fixture.program.block_info[2].terminator = {
-        .kind = CFG::TerminatorKind::Branch, .true_block = 3u};
-    fixture.program.block_info[3].terminator = {
-        .kind = CFG::TerminatorKind::Branch, .true_block = 1u};
-    fixture.program.block_info[4].terminator = {
-        .kind = variant == Variant::ExitBypass ? CFG::TerminatorKind::Branch
-                                              : CFG::TerminatorKind::Return,
-        .true_block = 2u};
+    if (!masked) {
+      header->AddBranch(exit);
+      header->AddBranch(body);
+      body->AddBranch(latch);
+      latch->AddBranch(header);
+      if (variant == Variant::PreviousBound) latch->AddBranch(final_exit);
+      if (variant == Variant::EntryBypass) entry->AddBranch(body);
+      if (variant == Variant::ExitBypass) exit->AddBranch(body);
+      if (variant == Variant::IncrementBypass || variant == Variant::PreviousBound)
+        exit->AddBranch(latch);
+      fixture.program.block_info[0].terminator = {
+          .kind = variant == Variant::EntryBypass ? CFG::TerminatorKind::ConditionalBranch
+                                                 : CFG::TerminatorKind::Branch,
+          .true_block = 1u, .false_block = 2u};
+      fixture.program.block_info[1].terminator = {
+          .kind = CFG::TerminatorKind::ConditionalBranch,
+          .true_block = variant == Variant::TrueEdge ? 2u : 4u,
+          .false_block = variant == Variant::TrueEdge ? 4u : 2u};
+      fixture.program.block_info[2].terminator = {
+          .kind = CFG::TerminatorKind::Branch, .true_block = 3u};
+      fixture.program.block_info[3].terminator = {
+          .kind = variant == Variant::PreviousBound ? CFG::TerminatorKind::ConditionalBranch
+                                                   : CFG::TerminatorKind::Branch,
+          .true_block = 1u, .false_block = 5u};
+      fixture.program.block_info[4].terminator = {
+          .kind = final_exit != exit || variant == Variant::ExitBypass ||
+                          variant == Variant::IncrementBypass
+                      ? CFG::TerminatorKind::Branch : CFG::TerminatorKind::Return,
+          .true_block = final_exit != exit || variant == Variant::IncrementBypass ? 3u : 2u};
+      if (final_exit != exit)
+        fixture.program.block_info[5].terminator.kind = CFG::TerminatorKind::Return;
+    }
 
     auto &phi = header->AppendNewInst(ValueOpcode::Phi, {},
                                       static_cast<uint64_t>(Type::U32));
     const auto key = Value(&phi);
-    const auto count = fixture.UserData(2);
+    auto *previous = variant == Variant::PreviousBound ?
+        &header->AppendNewInst(ValueOpcode::Phi, {}, static_cast<uint64_t>(Type::U1)) : nullptr;
+    const auto local = fixture.Emit(
+        ValueOpcode::GetBuiltin,
+        {Value(static_cast<uint32_t>(diamond ? StageInputKind::VertexIndex
+                                            : StageInputKind::LocalInvocationId)), Value(0u)});
+    const auto count = variant == Variant::DivergentBound ? local : fixture.UserData(2);
     const auto in_range = fixture.Emit(ValueOpcode::SLessThan32,
                                        {variant == Variant::WrongGuard
                                             ? Value(0u) : key,
-                                        count}, 0, header);
-    const auto allowed = fixture.Emit(ValueOpcode::LogicalAnd,
-                                      {in_range, Value(true)}, 0, header);
-    fixture.program.block_info[1].condition =
-        fixture.Emit(ValueOpcode::LogicalNot, {allowed}, 0, header);
+                                        count}, 0, compare);
+    const auto initial_active = fixture.Emit(ValueOpcode::INotEqual32,
+                                             {local, Value(0u)}, 0, entry);
+    if (masked) {
+      auto *active_phi = diamond ? nullptr : &header->AppendNewInst(
+          ValueOpcode::Phi, {}, static_cast<uint64_t>(Type::U1));
+      auto &saved_phi = header->AppendNewInst(ValueOpcode::Phi, {},
+                                              static_cast<uint64_t>(Type::U1));
+      auto &status_phi = header->AppendNewInst(ValueOpcode::Phi, {},
+                                               static_cast<uint64_t>(Type::U32));
+      const auto saved = Value(&saved_phi);
+      const auto status = Value(&status_phi);
+      const auto active = diamond ? fixture.Emit(ValueOpcode::LogicalAnd,
+          {saved, fixture.Emit(ValueOpcode::UGreaterThanEqual32,
+                               {Value(0u), status}, 0, header)}, 0, header) : Value(active_phi);
+      const auto mask = fixture.Emit(ValueOpcode::LogicalOr,
+          {fixture.Emit(ValueOpcode::LogicalAnd, {in_range, active}, 0, compare),
+           fixture.Emit(ValueOpcode::LogicalNot, {active}, 0, compare)}, 0, compare);
+      auto next_saved = fixture.Emit(ValueOpcode::LogicalAnd,
+          {variant == Variant::MaskResurrection ? Value(true) : saved, mask}, 0, compare);
+      const auto execute = fixture.Emit(ValueOpcode::LogicalAnd, {active, mask}, 0, compare);
+      const auto body_status = fixture.Emit(ValueOpcode::SelectU32,
+          {execute, local, status}, 0, guard);
+      const auto next_active = fixture.Emit(ValueOpcode::LogicalAnd,
+          {next_saved, fixture.Emit(ValueOpcode::UGreaterThanEqual32,
+                                   {Value(3u), body_status}, 0, latch)}, 0, latch);
+      auto next_status = fixture.Emit(ValueOpcode::SelectU32,
+          {variant == Variant::StatusOverwrite ? Value(true) : next_active,
+           Value(0u), body_status}, 0,
+          variant == Variant::GuardedDiamondBypass ? latch : increment);
+      if (variant == Variant::GuardedDiamondBypass) {
+        auto &merged_saved = increment->AppendNewInst(ValueOpcode::Phi, {},
+                                                      static_cast<uint64_t>(Type::U1));
+        merged_saved.AddPhiOperand(compare, saved);
+        merged_saved.AddPhiOperand(latch, next_saved);
+        next_saved = Value(&merged_saved);
+        auto &merged_status = increment->AppendNewInst(ValueOpcode::Phi, {},
+                                                       static_cast<uint64_t>(Type::U32));
+        merged_status.AddPhiOperand(compare, status);
+        merged_status.AddPhiOperand(latch, next_status);
+        next_status = Value(&merged_status);
+      }
+      if (active_phi != nullptr) {
+        active_phi->AddPhiOperand(entry, initial_active);
+        active_phi->AddPhiOperand(increment, next_active);
+      }
+      saved_phi.AddPhiOperand(entry, initial_active);
+      saved_phi.AddPhiOperand(increment, next_saved);
+      status_phi.AddPhiOperand(entry, diamond ? Value(0u) : local);
+      status_phi.AddPhiOperand(increment, next_status);
+      const auto branch = [&](uint32_t index, uint32_t yes, uint32_t no,
+                              Value predicate, CFG::BranchCondition kind) {
+        auto *block = fixture.program.blocks[index];
+        block->AddBranch(fixture.program.blocks[yes]);
+        block->AddBranch(fixture.program.blocks[no]);
+        const auto inverse = fixture.Emit(ValueOpcode::LogicalNot, {predicate}, 0, block);
+        fixture.program.block_info[index].condition =
+            fixture.Emit(ValueOpcode::ConditionRef, {inverse}, kind, block);
+        fixture.program.block_info[index].terminator = {
+            .kind = CFG::TerminatorKind::ConditionalBranch,
+            .true_block = yes, .false_block = no};
+      };
+      fixture.program.block_info[0].terminator = {
+          .kind = CFG::TerminatorKind::Branch, .true_block = 1u};
+      branch(1u, 4u, 5u, active, CFG::BranchCondition::ExecZero);
+      branch(5u, variant == Variant::GuardedDiamondBypass ? 7u : 4u, 6u,
+             diamond ? execute : mask,
+             diamond ? CFG::BranchCondition::ExecZero : CFG::BranchCondition::VccZero);
+      const auto image_guard = variant == Variant::MaskedWrongGuard ? active :
+          fixture.Emit(ValueOpcode::LogicalAnd,
+              {fixture.Emit(ValueOpcode::LogicalAnd, {execute, in_range}, 0, guard),
+               fixture.Emit(ValueOpcode::IEqual32, {local, Value(1u)}, 0, guard)}, 0, guard);
+      branch(6u, 3u, 2u, image_guard, CFG::BranchCondition::ExecZero);
+      body->AddBranch(latch);
+      fixture.program.block_info[2].terminator = {
+          .kind = CFG::TerminatorKind::Branch, .true_block = 3u};
+      branch(3u, 4u, 7u, next_active, CFG::BranchCondition::ExecZero);
+      increment->AddBranch(header);
+      fixture.program.block_info[7].terminator = {
+          .kind = CFG::TerminatorKind::Branch, .true_block = 1u};
+      fixture.program.block_info[4].terminator.kind = CFG::TerminatorKind::Return;
+    } else {
+      const auto active = initial_active;
+      const auto allowed = fixture.Emit(variant == Variant::Disjunction
+                                           ? ValueOpcode::LogicalOr : ValueOpcode::LogicalAnd,
+                                        {in_range, active}, 0, header);
+      const bool nonzero = variant == Variant::Nonzero || variant == Variant::TrueEdge;
+      auto condition = allowed;
+      if (!nonzero)
+        condition = fixture.Emit(ValueOpcode::LogicalNot, {condition}, 0, header);
+      if (variant != Variant::Plain)
+        condition = fixture.Emit(ValueOpcode::ConditionRef, {condition},
+            nonzero ? CFG::BranchCondition::ExecNonZero
+                    : CFG::BranchCondition::ExecZero, header);
+      if (variant == Variant::Nonzero || variant == Variant::WrongPolarity)
+        condition = fixture.Emit(ValueOpcode::LogicalNot, {condition}, 0, header);
+      fixture.program.block_info[1].condition = condition;
+      if (variant == Variant::PreviousBound) {
+        previous->AddPhiOperand(entry, Value(false));
+        previous->AddPhiOperand(latch, in_range);
+        const auto continuing = fixture.Emit(ValueOpcode::LogicalOr,
+            {in_range, Value(previous)}, 0, latch);
+        fixture.program.block_info[3].condition = fixture.Emit(ValueOpcode::ConditionRef,
+            {continuing}, CFG::BranchCondition::SccNonZero, latch);
+      }
+    }
     const auto step = fixture.Emit(ValueOpcode::IAdd32,
                                    {key, Value(variant == Variant::WrongStep ? 2u : 1u)},
-                                   0, latch);
-    phi.AddPhiOperand(entry, Value(0u));
-    phi.AddPhiOperand(latch, step);
+                                   0, increment);
+    phi.AddPhiOperand(entry, variant == Variant::DivergentKey ? local : Value(0u));
+    phi.AddPhiOperand(increment, step);
 
     fixture.block = variant == Variant::GuardBlock ? header : body;
     const auto table = fixture.Address(fixture.UserData(0), fixture.UserData(1));
@@ -787,6 +975,18 @@ void TestBoundedComputeImageLoop() {
   };
 
   auto plan = make_plan(Variant::Bounded);
+  make_plan(Variant::Plain);
+  make_plan(Variant::Nonzero);
+  make_plan(Variant::TrueEdge);
+  make_plan(Variant::Masked);
+  make_plan(Variant::GuardedDiamond);
+  for (const auto variant : {Variant::IncrementBypass, Variant::PreviousBound,
+                             Variant::MaskedWrongGuard,
+                             Variant::MaskResurrection, Variant::StatusOverwrite,
+                             Variant::GuardedDiamondBypass}) {
+    CheckFatal([&] { make_plan(variant); }, "not a valid runtime value",
+               "compute image loop allowed an unbounded induction or mask resurrection");
+  }
   CheckFatal([&] { make_plan(Variant::WrongGuard); },
              "not a valid runtime value",
              "compute image loop accepted an unrelated guard");
@@ -802,6 +1002,11 @@ void TestBoundedComputeImageLoop() {
   CheckFatal([&] { make_plan(Variant::WrongStep); },
              "not a valid runtime value",
              "compute image loop accepted a two-step induction");
+  for (const auto variant : {Variant::DivergentBound, Variant::DivergentKey,
+                             Variant::Disjunction, Variant::WrongPolarity}) {
+    CheckFatal([&] { make_plan(variant); }, "not a valid runtime value",
+               "compute image loop accepted a guard without a uniform bound");
+  }
 
   LinearTestMemory memory;
   const auto table = 0x1800u + 0x6b0u;
@@ -849,8 +1054,21 @@ void TestBoundedComputeImageLoop() {
 
 void TestUniformizedMaterialImageKeys() {
   namespace CFG = Libs::Graphics::ShaderRecompiler::CFG;
-  const auto make_plan = [](bool wrong_update, bool wrong_equality) {
+  enum class Variant {
+    Valid, Plain, WrongUpdate, WrongEquality, WrongExit, WrongCarry, WrongBackedge,
+    AndNot, Subset, FirstLane, FirstLaneDirect, FirstLaneAndNot, EmptyFirstEntry, WrongFirstBackedge,
+    WideningFirstEntry, WideningFirstBackedge, WrongFirstLoadMask
+  };
+  const auto make_plan = [](Variant variant) {
     Fixture fixture;
+    fixture.program.wave_size = 64u;
+    const bool first_lane = variant >= Variant::FirstLane;
+    const bool first_lane_loop = first_lane && variant != Variant::FirstLaneDirect;
+    const uint32_t sample_entry_id = first_lane_loop ? 10u : 9u;
+    const auto branch = [&](Value predicate, CFG::BranchCondition kind, Block *block) {
+      return variant == Variant::Plain ? predicate : fixture.Emit(
+          ValueOpcode::ConditionRef, {predicate}, kind, block);
+    };
     auto *entry = fixture.block;
     auto *header = fixture.AddBlock();
     auto *inactive = fixture.AddBlock();
@@ -860,6 +1078,8 @@ void TestUniformizedMaterialImageKeys() {
     auto *choose = fixture.AddBlock();
     auto *sample = fixture.AddBlock();
     auto *done = fixture.AddBlock();
+    auto *sample_header = first_lane_loop ? fixture.AddBlock() : nullptr;
+    auto *sample_entry = first_lane ? fixture.AddBlock() : nullptr;
     entry->AddBranch(header);
     header->AddBranch(inactive);
     inactive->AddBranch(merge);
@@ -869,23 +1089,44 @@ void TestUniformizedMaterialImageKeys() {
     bit->AddBranch(header);
     bit->AddBranch(merge);
     merge->AddBranch(choose);
-    choose->AddBranch(sample);
+    choose->AddBranch(first_lane ? sample_entry : sample);
     choose->AddBranch(done);
+    if (first_lane) {
+      sample_entry->AddBranch(first_lane_loop ? sample_header : sample);
+      fixture.program.block_info[sample_entry_id].terminator = {
+          .kind = CFG::TerminatorKind::Branch, .true_block = first_lane_loop ? 9u : 7u};
+    }
+    if (first_lane_loop) {
+      sample_header->AddBranch(sample);
+      sample->AddBranch(sample_header);
+      fixture.program.block_info[9].terminator = {
+          .kind = CFG::TerminatorKind::Branch, .true_block = 7u,
+          .merge_block = 8u, .continue_block = 7u, .loop_header = true};
+    }
     sample->AddBranch(done);
     fixture.program.block_info[0].terminator = {
         .kind = CFG::TerminatorKind::Branch, .true_block = 1u};
     fixture.program.block_info[1].terminator = {
         .kind = CFG::TerminatorKind::Branch, .true_block = 2u};
-    const auto active_on_entry = fixture.Emit(
+    const auto enabled = fixture.Emit(
         ValueOpcode::INotEqual32, {fixture.UserData(5u), Value(0u)}, 0, entry);
+    const auto local = fixture.Emit(ValueOpcode::GetBuiltin,
+        {Value(static_cast<uint32_t>(StageInputKind::LocalInvocationId)), Value(0u)}, 0, entry);
+    const auto active_on_entry = fixture.Emit(ValueOpcode::LogicalAnd,
+        {enabled, fixture.Emit(ValueOpcode::INotEqual32, {local, Value(0u)}, 0, entry)},
+        0, entry);
     auto &active_phi = header->AppendNewInst(
         ValueOpcode::Phi, {}, static_cast<uint64_t>(Type::U1));
     auto &mask_phi = header->AppendNewInst(
         ValueOpcode::Phi, {}, static_cast<uint64_t>(Type::U32));
+    auto &carry_phi = header->AppendNewInst(
+        ValueOpcode::Phi, {}, static_cast<uint64_t>(Type::U32));
     const auto mask = Value(&mask_phi);
     const auto active = Value(&active_phi);
-    fixture.program.block_info[2].condition = fixture.Emit(
-        ValueOpcode::LogicalNot, {active}, 0, inactive);
+    fixture.program.block_info[2].condition = branch(
+        fixture.Emit(ValueOpcode::LogicalNot, {active}, 0, inactive),
+        variant == Variant::WrongExit ? CFG::BranchCondition::ExecNonZero
+                                      : CFG::BranchCondition::ExecZero, inactive);
     fixture.program.block_info[2].terminator = {
         .kind = CFG::TerminatorKind::ConditionalBranch,
         .true_block = 5u, .false_block = 3u};
@@ -893,8 +1134,9 @@ void TestUniformizedMaterialImageKeys() {
         ValueOpcode::INotEqual32, {Value(0u), mask}, 0, sentinel);
     const auto bit_guard = fixture.Emit(
         ValueOpcode::LogicalAnd, {active, nonzero}, 0, sentinel);
-    fixture.program.block_info[3].condition = fixture.Emit(
-        ValueOpcode::LogicalNot, {bit_guard}, 0, sentinel);
+    fixture.program.block_info[3].condition = branch(
+        fixture.Emit(ValueOpcode::LogicalNot, {bit_guard}, 0, sentinel),
+        CFG::BranchCondition::ExecZero, sentinel);
     fixture.program.block_info[3].terminator = {
         .kind = CFG::TerminatorKind::ConditionalBranch,
         .true_block = 5u, .false_block = 4u};
@@ -903,12 +1145,19 @@ void TestUniformizedMaterialImageKeys() {
         ValueOpcode::BitwiseAnd32, {first, Value(31u)}, 0, bit);
     const auto one_bit = fixture.Emit(
         ValueOpcode::ShiftLeftLogical32, {Value(1u), position}, 0, bit);
+    const bool and_not = variant == Variant::AndNot || variant == Variant::FirstLaneAndNot;
+    const auto removed = and_not
+        ? fixture.Emit(ValueOpcode::BitwiseNot32, {one_bit}, 0, bit)
+        : variant == Variant::Subset ? fixture.UserData(7u) : one_bit;
     const auto cleared = fixture.Emit(
-        wrong_update ? ValueOpcode::BitwiseOr32 : ValueOpcode::BitwiseXor32,
-        {mask, one_bit}, 0, bit);
+        variant == Variant::WrongUpdate ? ValueOpcode::BitwiseOr32
+        : and_not || variant == Variant::Subset ? ValueOpcode::BitwiseAnd32
+                                              : ValueOpcode::BitwiseXor32,
+        {mask, removed}, 0, bit);
     const auto continuation = fixture.Emit(
         ValueOpcode::LogicalAnd, {bit_guard, active_on_entry}, 0, bit);
-    fixture.program.block_info[4].condition = continuation;
+    fixture.program.block_info[4].condition = branch(
+        continuation, CFG::BranchCondition::ExecNonZero, bit);
     fixture.program.block_info[4].terminator = {
         .kind = CFG::TerminatorKind::ConditionalBranch,
         .true_block = 1u, .false_block = 5u};
@@ -917,13 +1166,18 @@ void TestUniformizedMaterialImageKeys() {
     mask_phi.AddPhiOperand(entry, fixture.UserData(4u));
     mask_phi.AddPhiOperand(bit, cleared);
     const auto arbitrary = fixture.UserData(6u);
+    const auto carry = Value(&carry_phi);
     const auto sentinel_index = fixture.Emit(
-        ValueOpcode::SelectU32, {active, Value(32u), arbitrary}, 0, sentinel);
+        ValueOpcode::SelectU32,
+        {active, Value(32u), variant == Variant::WrongCarry ? arbitrary : carry},
+        0, sentinel);
     const auto bit_index = fixture.Emit(
         ValueOpcode::SelectU32, {bit_guard, first, sentinel_index}, 0, bit);
+    carry_phi.AddPhiOperand(entry, arbitrary);
+    carry_phi.AddPhiOperand(bit, variant == Variant::WrongBackedge ? arbitrary : bit_index);
     auto &index_phi = merge->AppendNewInst(
         ValueOpcode::Phi, {}, static_cast<uint64_t>(Type::U32));
-    index_phi.AddPhiOperand(inactive, arbitrary);
+    index_phi.AddPhiOperand(inactive, carry);
     index_phi.AddPhiOperand(sentinel, sentinel_index);
     index_phi.AddPhiOperand(bit, bit_index);
     const auto index = Value(&index_phi);
@@ -950,24 +1204,54 @@ void TestUniformizedMaterialImageKeys() {
     const auto base = fixture.Address(fixture.UserData(0u), fixture.UserData(1u));
     MemoryInfo material_memory;
     material_memory.kind = ResourceKind::Global;
+    const auto load_mask = variant == Variant::WrongFirstLoadMask
+        ? fixture.Emit(ValueOpcode::LogicalNot, {material_guard}, 0, choose)
+        : material_guard;
     const auto loaded = fixture.Emit(
         ValueOpcode::LoadAddressU32,
-        {base, material_offset, Value(0u), material_guard},
+        {base, material_offset, Value(0u), load_mask},
         fixture.AddMemory(material_memory, 0x1a88u), choose);
-    const auto local = fixture.Emit(
+    const auto local_key = fixture.Emit(
         ValueOpcode::SelectU32, {material_guard, loaded, arbitrary}, 0, choose);
-    const auto key = fixture.Emit(
-        ValueOpcode::ReadLane, {local, Value(0u)}, 0, choose);
+    Inst *sample_active_phi = nullptr;
+    auto sample_active = material_guard;
+    if (first_lane) {
+      const auto initial = variant == Variant::EmptyFirstEntry ? Value(false)
+          : variant == Variant::WideningFirstEntry ? enabled
+          : fixture.Emit(ValueOpcode::LogicalAnd,
+              {material_guard, fixture.Emit(ValueOpcode::SLessThanEqual32,
+                  {Value(0u), local_key}, 0, choose)}, 0, choose);
+      sample_active = initial;
+      if (first_lane_loop) {
+        sample_active_phi = &sample_header->AppendNewInst(
+            ValueOpcode::Phi, {}, static_cast<uint64_t>(Type::U1));
+        sample_active_phi->AddPhiOperand(sample_entry, initial);
+        sample_active = Value(sample_active_phi);
+      }
+      fixture.program.block_info[6].condition = branch(
+          variant == Variant::EmptyFirstEntry ? enabled : initial,
+          CFG::BranchCondition::ExecNonZero, choose);
+      fixture.program.block_info[6].terminator = {
+          .kind = CFG::TerminatorKind::ConditionalBranch,
+          .true_block = sample_entry_id, .false_block = 8u};
+    }
+    auto *key_block = first_lane ? sample : choose;
+    const auto key = first_lane
+        ? fixture.Emit(ValueOpcode::ReadFirstLane, {local_key, sample_active}, 0, key_block)
+        : fixture.Emit(ValueOpcode::ReadLane, {local_key, Value(0u)}, 0, key_block);
     const auto compared = fixture.Emit(
         ValueOpcode::IEqual32,
-        {key, wrong_equality ? arbitrary : local}, 0, choose);
+        {key, variant == Variant::WrongEquality ? arbitrary : local_key}, 0, key_block);
     const auto sample_guard = fixture.Emit(
-        ValueOpcode::LogicalAnd, {material_guard, compared}, 0, choose);
-    fixture.program.block_info[6].condition = fixture.Emit(
-        ValueOpcode::LogicalNot, {sample_guard}, 0, choose);
-    fixture.program.block_info[6].terminator = {
-        .kind = CFG::TerminatorKind::ConditionalBranch,
-        .true_block = 8u, .false_block = 7u};
+        ValueOpcode::LogicalAnd, {sample_active, compared}, 0, key_block);
+    if (!first_lane) {
+      fixture.program.block_info[6].condition = branch(
+          fixture.Emit(ValueOpcode::LogicalNot, {sample_guard}, 0, choose),
+          CFG::BranchCondition::ExecZero, choose);
+      fixture.program.block_info[6].terminator = {
+          .kind = CFG::TerminatorKind::ConditionalBranch,
+          .true_block = 8u, .false_block = 7u};
+    }
     const auto table_offset = fixture.Emit(
         ValueOpcode::IAdd32,
         {fixture.Emit(ValueOpcode::ShiftLeftLogical32,
@@ -997,10 +1281,33 @@ void TestUniformizedMaterialImageKeys() {
     MemoryInfo image_memory;
     image_memory.kind = ResourceKind::Image;
     image_memory.image_dimension = Decoder::ImageDimension::Dim2D;
-    fixture.Emit(ValueOpcode::ImageSampleRaw, {image, sampler, address},
-                 fixture.AddMemory(image_memory, 0x1aecu), sample);
-    fixture.program.block_info[7].terminator = {
-        .kind = CFG::TerminatorKind::Branch, .true_block = 8u};
+    const auto sampled = fixture.Emit(ValueOpcode::ImageSampleRaw, {image, sampler, address},
+        fixture.AddMemory(image_memory, 0x1aecu), sample);
+    if (first_lane) {
+      const auto component = fixture.Emit(
+          ValueOpcode::CompositeExtractU32x4, {sampled, Value(0u)}, 0, sample);
+      fixture.Emit(ValueOpcode::ReferenceU32,
+          {fixture.Emit(ValueOpcode::SelectU32,
+              {variant == Variant::EmptyFirstEntry ? material_guard : sample_guard,
+               component, Value(0u)}, 0, sample)}, 0, sample);
+    }
+    if (first_lane_loop) {
+      auto remaining = fixture.Emit(ValueOpcode::LogicalAnd,
+          {sample_active, fixture.Emit(ValueOpcode::LogicalNot,
+              {sample_guard}, 0, sample)}, 0, sample);
+      if (variant == Variant::WideningFirstBackedge)
+        remaining = fixture.Emit(ValueOpcode::LogicalOr, {remaining, enabled}, 0, sample);
+      sample_active_phi->AddPhiOperand(sample, remaining);
+      fixture.program.block_info[7].condition = branch(
+          variant == Variant::WrongFirstBackedge ? sample_guard : remaining,
+          CFG::BranchCondition::ExecNonZero, sample);
+      fixture.program.block_info[7].terminator = {
+          .kind = CFG::TerminatorKind::ConditionalBranch,
+          .true_block = 9u, .false_block = 8u};
+    } else {
+      fixture.program.block_info[7].terminator = {
+          .kind = CFG::TerminatorKind::Branch, .true_block = 8u};
+    }
     fixture.program.block_info[8].terminator.kind = CFG::TerminatorKind::Return;
     const auto output = fixture.Emit(
         ValueOpcode::GetBufferResource,
@@ -1024,7 +1331,13 @@ void TestUniformizedMaterialImageKeys() {
           "uniformized image key lost its finite material range");
     return ExtractResourcePlan(fixture.program);
   };
-  auto plan = make_plan(false, false);
+  auto plan = make_plan(Variant::Valid);
+  make_plan(Variant::Plain);
+  make_plan(Variant::AndNot);
+  make_plan(Variant::Subset);
+  make_plan(Variant::FirstLane);
+  make_plan(Variant::FirstLaneDirect);
+  plan = make_plan(Variant::FirstLaneAndNot);
   Check(plan.requires_specialization_memory &&
             plan.descriptor_sources[plan.info.images[0].source]
                 .indirect_image->selector_mask.Resolve().TryInstruction() != nullptr,
@@ -1078,10 +1391,22 @@ void TestUniformizedMaterialImageKeys() {
   user_data[8] = first_table;
   Check(!MaterializeResources(plan, runtime, snapshot, specialization),
         "written buffer alias with an image record was accepted");
-  CheckFatal([&] { make_plan(true, false); }, "not a valid runtime value",
+  CheckFatal([&] { make_plan(Variant::WrongUpdate); }, "not a valid runtime value",
              "non-clearing material mask was accepted");
-  CheckFatal([&] { make_plan(false, true); }, "not a valid runtime value",
+  CheckFatal([&] { make_plan(Variant::WrongEquality); }, "not a valid runtime value",
              "unrelated ReadLane key was accepted");
+  CheckFatal([&] { make_plan(Variant::WrongExit); }, "not a valid runtime value",
+             "one inactive lane bypassed material index initialization for active lanes");
+  for (const auto variant : {Variant::WrongCarry, Variant::WrongBackedge}) {
+    CheckFatal([&] { make_plan(variant); }, "not a valid runtime value",
+               "inactive material lane did not preserve its selected index across iterations");
+  }
+  for (const auto variant : {Variant::EmptyFirstEntry, Variant::WrongFirstBackedge,
+                            Variant::WideningFirstEntry, Variant::WideningFirstBackedge,
+                            Variant::WrongFirstLoadMask}) {
+    CheckFatal([&] { make_plan(variant); }, "not a valid runtime value",
+               "first-lane material key escaped its active load mask");
+  }
 }
 
 void TestImageDescriptorFields() {
@@ -1405,7 +1730,7 @@ void TestDenseBufferTracking() {
             fixture.program.memory_info[other_flags.index].resource == 1,
         "typed memory metadata was not patched to dense indices");
 
-  CheckFatal([&] { TrackResources(fixture.program); }, "already tracked",
+  CheckFatal([&] { fixture.PlanAndTrack(); }, "already tracked",
              "resource tracking allowed a second mutation pass");
 }
 
@@ -1601,8 +1926,8 @@ void TestSampleAdjustSamplerScratch() {
     rejected.Emit(ValueOpcode::ImageSampleRaw,
                   {rejected_image, rejected_sampler, rejected.ImageAddress()},
                   rejected.AddMemory(rejected_memory, 0x200));
-    BuildSrtPlan(rejected.program);
-    CheckFatal([&] { TrackResources(rejected.program); },
+
+    CheckFatal([&] { rejected.PlanAndTrack(); },
                "not a valid runtime value", message);
   };
   CheckRejected(0u, 12u,
@@ -1936,7 +2261,8 @@ void TestDynamicSrtReadRemainsExplicit() {
   fixture.PlanAndTrack();
 
   Check(fixture.program.srt_reads.empty() &&
-            fixture.program.dynamic_reads.size() == 1 &&
+            read.ResolveInstruction()->GetOpcode() == ValueOpcode::LoadAddressU32 &&
+            !fixture.program.memory_info[0].planning_only &&
             fixture.program.info.uses_dma,
         "dynamic scalar read was incorrectly flattened or lost");
   std::array<uint32_t, 3> user_data{0x1000u, 0u, 4u};
@@ -1990,8 +2316,7 @@ void TestPhiValidation() {
                {handle, Value(0u), Value(0u), Value(0u), Value(true)},
                fixture.AddMemory(memory, 20), merge);
 
-  BuildSrtPlan(fixture.program);
-  CheckFatal([&] { TrackResources(fixture.program); }, "not a valid runtime value",
+  CheckFatal([&] { fixture.PlanAndTrack(); }, "not a valid runtime value",
              "control-dependent descriptor phi was accepted");
   Check(!fixture.program.resource_tracking_complete &&
             fixture.program.info.buffers.empty() &&
@@ -2092,7 +2417,9 @@ ResourcePlan ConditionalSamplerPlan(bool diamond, bool reverse, bool reverse_phi
                                {result, Value(0u)})});
   }
   fixture.PlanAndTrack();
-  Check(fixture.program.value_storage.size() == 4,
+  Check(std::ranges::count_if(fixture.program.value_storage, [](const Inst &inst) {
+          return inst.GetOpcode() == ValueOpcode::SelectU32;
+        }) == 4,
         "repeated sampler uses retained duplicate planning selections");
   Check(sampler_words[0].ResolveInstruction()->GetOpcode() == ValueOpcode::Phi,
         "host descriptor selection changed the GPU Phi");
@@ -2155,90 +2482,6 @@ void TestConditionalSamplerPhi() {
   }
 }
 
-void TestGuardedSamplerPhi() {
-  namespace CFG = Libs::Graphics::ShaderRecompiler::CFG;
-  const auto make_plan = [](bool take_true, bool reverse_phi, bool bypass,
-                            bool mismatched, bool ambiguous) {
-    Fixture fixture(ShaderType::Pixel);
-    auto *entry = fixture.block;
-    auto *loaded = fixture.AddBlock();
-    auto *merge = fixture.AddBlock();
-    auto *sample = fixture.AddBlock();
-    auto *exit = fixture.AddBlock();
-    entry->AddBranch(loaded);
-    entry->AddBranch(merge);
-    loaded->AddBranch(merge);
-    merge->AddBranch(sample);
-    merge->AddBranch(exit);
-    sample->AddBranch(exit);
-    if (bypass) exit->AddBranch(sample);
-    const auto lane = fixture.Emit(ValueOpcode::LaneId);
-    const auto predicate = fixture.Emit(ValueOpcode::IEqual32, {lane, Value(0u)});
-    fixture.program.block_info[0].condition = predicate;
-    fixture.program.block_info[0].terminator = {
-        .kind = CFG::TerminatorKind::ConditionalBranch,
-        .true_block = 1u, .false_block = 2u};
-    fixture.program.block_info[1].terminator = {
-        .kind = CFG::TerminatorKind::Branch, .true_block = 2u};
-    auto &guard = merge->AppendNewInst(ValueOpcode::Phi, {},
-                                       static_cast<uint64_t>(Type::U1));
-    guard.AddPhiOperand(entry, ambiguous ? predicate : Value(!take_true));
-    guard.AddPhiOperand(mismatched ? sample : loaded, predicate);
-    fixture.program.block_info[2].condition = Value(&guard);
-    fixture.program.block_info[2].terminator = {
-        .kind = CFG::TerminatorKind::ConditionalBranch,
-        .true_block = take_true ? 3u : 4u,
-        .false_block = take_true ? 4u : 3u};
-    fixture.program.block_info[3].terminator = {
-        .kind = CFG::TerminatorKind::Branch, .true_block = 4u};
-    fixture.program.block_info[4].terminator = {
-        .kind = bypass ? CFG::TerminatorKind::Branch : CFG::TerminatorKind::Return,
-        .true_block = 3u};
-    std::array<Value, 4> words;
-    for (uint32_t word = 0; word < words.size(); ++word) {
-      // An arbitrary excluded value proves this is control-flow reasoning, not null filtering.
-      const auto first = Value(0xbad000u + word);
-      const auto second = fixture.UserData(word);
-      auto &phi = merge->AppendNewInst(ValueOpcode::Phi, {},
-                                       static_cast<uint64_t>(Type::U32));
-      phi.AddPhiOperand(reverse_phi ? loaded : entry, reverse_phi ? second : first);
-      phi.AddPhiOperand(reverse_phi ? entry : loaded, reverse_phi ? first : second);
-      words[word] = Value(&phi);
-    }
-    fixture.block = sample;
-    const auto image = fixture.Image({Value(0u), Value(0u), Value(0u), Value(0u),
-                                      Value(0u), Value(0u), Value(0u), Value(0u)});
-    const auto sampler = fixture.Sampler(words);
-    MemoryInfo memory;
-    memory.kind = ResourceKind::Image;
-    memory.image_dimension = Decoder::ImageDimension::Dim2D;
-    fixture.Emit(ValueOpcode::ImageSampleRaw,
-                  {image, sampler, fixture.ImageAddress()},
-                  fixture.AddMemory(memory, 0x2a0));
-    fixture.PlanAndTrack();
-    Check(words[0].ResolveInstruction()->GetOpcode() == ValueOpcode::Phi,
-          "guarded host descriptor selection rewrote the GPU Phi");
-    return ExtractResourcePlan(fixture.program);
-  };
-  for (const bool take_true : {false, true}) {
-    for (const bool reverse_phi : {false, true}) {
-      auto plan = make_plan(take_true, reverse_phi, false, false, false);
-      const std::array<uint32_t, 4> user_data{0x444u, 0x555u, 0x666u, 0x777u};
-      DescriptorValue selected;
-      Check(SrtWalker(plan, {.user_data = user_data}).EvaluateDescriptor(
-                plan.info.samplers[0].source, selected) &&
-                std::equal(user_data.begin(), user_data.end(), selected.dwords.begin()),
-            "guarded sampler did not match descriptor and predicate predecessors");
-    }
-  }
-  CheckFatal([&] { make_plan(true, false, true, false, false); },
-              "not a valid runtime value", "sampler guard accepted an unguarded path");
-  CheckFatal([&] { make_plan(true, false, false, true, false); },
-              "not a valid runtime value", "sampler guard ignored predecessor identity");
-  CheckFatal([&] { make_plan(true, false, false, false, true); },
-              "not a valid runtime value", "sampler guard discarded a reachable alternative");
-}
-
 void TestLoopCycleEnteredThroughRuntimeValue() {
   Fixture fixture;
   auto *entry = fixture.block;
@@ -2252,11 +2495,24 @@ void TestLoopCycleEnteredThroughRuntimeValue() {
                                     {Value(&phi), Value(0xffffffffu)}, 0, loop);
   phi.AddPhiOperand(entry, initial);
   phi.AddPhiOperand(loop, carried);
-  fixture.Emit(ValueOpcode::GetBufferResource,
+  const auto handle = fixture.Emit(ValueOpcode::GetBufferResource,
                {carried, Value(0u), Value(0u), Value(0u)}, MemoryFlags{0, 12},
                loop);
-
-  BuildSrtPlan(fixture.program);
+  MemoryInfo memory;
+  memory.kind = ResourceKind::Buffer;
+  fixture.Emit(ValueOpcode::LoadBufferU32,
+               {handle, Value(0u), Value(0u), Value(0u), Value(true)},
+               fixture.AddMemory(memory, 16), loop);
+  ConstantPropagationPass(fixture.program.blocks);
+  RemoveIdentities(fixture.program.blocks);
+  fixture.PlanAndTrack();
+  const std::array<uint32_t, 1> user_data{0x4000u};
+  DescriptorValue descriptor;
+  Check(fixture.program.info.buffers.size() == 1u &&
+            SrtWalker(fixture.program, {.user_data = user_data}).EvaluateDescriptor(
+                fixture.program.info.buffers[0].source, descriptor) &&
+            descriptor.dwords[0] == user_data[0],
+        "runtime-rooted invariant loop lost its buffer source");
 }
 
 void TestInvariantLoopPhi() {
@@ -2600,14 +2856,15 @@ void TestShaderInfoAndBindingLayout() {
   fixture.Emit(ValueOpcode::LoadBufferU32,
                {handle, Value(0u), Value(0u), Value(0u), Value(true)},
                fixture.AddMemory(buffer, 4));
-  fixture.Emit(
+  const auto invocation = fixture.Emit(
       ValueOpcode::GetBuiltin,
       {Value(static_cast<uint32_t>(StageInputKind::GlobalInvocationId)),
        Value(2u)});
-  fixture.Emit(ValueOpcode::BitwiseXor32, {Value(1u), Value(2u)});
+  const auto value = fixture.Emit(ValueOpcode::BitwiseXor32,
+                                   {invocation, Value(2u)});
   MemoryInfo gds;
   gds.kind = ResourceKind::Gds;
-  fixture.Emit(ValueOpcode::WriteSharedU32, {Value(0u), Value(1u), Value(true)},
+  fixture.Emit(ValueOpcode::WriteSharedU32, {Value(0u), value, Value(true)},
                fixture.AddMemory(gds, 8));
   fixture.PlanAndTrack();
 
@@ -2851,8 +3108,8 @@ void TestResourceLimitIsTransactional() {
                  {handle, Value(0u), Value(0u), Value(0u), Value(true)},
                  fixture.AddMemory(memory, index * 4u));
   }
-  BuildSrtPlan(fixture.program);
-  CheckFatal([&] { TrackResources(fixture.program); },
+
+  CheckFatal([&] { fixture.PlanAndTrack(); },
              "buffer resource limit exceeded",
              "resource-limit failure was not reported");
   Check(!fixture.program.resource_tracking_complete &&
@@ -2870,9 +3127,9 @@ void TestMalformedMemoryKindsRejected() {
     fixture.Emit(ValueOpcode::StoreAddressU32,
                  {address, Value(0u), Value(0u), Value(1u), Value(true)},
                  fixture.AddMemory(memory, 4));
-    BuildSrtPlan(fixture.program);
+
     CheckFatal(
-        [&] { TrackResources(fixture.program); },
+        [&] { fixture.PlanAndTrack(); },
         "address operation has invalid resource kind",
         "resource tracking accepted an address opcode with buffer metadata");
   }
@@ -2887,9 +3144,9 @@ void TestMalformedMemoryKindsRejected() {
     fixture.Emit(ValueOpcode::ImageRead,
                  {image, fixture.ImageAddress(), Value(true)},
                  fixture.AddMemory(memory, 8));
-    BuildSrtPlan(fixture.program);
+
     CheckFatal(
-        [&] { TrackResources(fixture.program); },
+        [&] { fixture.PlanAndTrack(); },
         "image operation has invalid resource kind",
         "resource tracking accepted an image opcode with address metadata");
   }
@@ -2924,7 +3181,6 @@ int main() {
     Run("dynamic SRT", TestDynamicSrtReadRemainsExplicit);
     Run("phi validation", TestPhiValidation);
     Run("conditional sampler phi", TestConditionalSamplerPhi);
-    Run("guarded sampler phi", TestGuardedSamplerPhi);
     Run("runtime-rooted loop", TestLoopCycleEnteredThroughRuntimeValue);
     Run("invariant loop phi", TestInvariantLoopPhi);
     Run("DMA address materialization", TestDmaAddressMaterialization);
@@ -2981,3 +3237,4 @@ void DbgExit(int) { throw std::runtime_error("typed IR assertion failed"); }
 #include "graphics/shader/recompiler/ir/Value.cpp"
 #include "graphics/shader/recompiler/ir/opcodes/ValueOpcodes.cpp"
 #include "graphics/shader/recompiler/ir/passes/DeadCodeElimination.cpp"
+#include "graphics/shader/recompiler/ir/passes/ConstantPropagation.cpp"

tests/ScalarProvenanceTests.cpp

@@ -2,6 +2,7 @@
 #include "graphics/shader/recompiler/ir/passes/ConstantPropagation.h"
 #include "graphics/shader/recompiler/ir/passes/DeadCodeElimination.h"
 #include "graphics/shader/recompiler/ir/passes/ReadLaneElimination.h"
+#include "graphics/shader/recompiler/ir/passes/ResourceTracking.h"
 #include "graphics/shader/recompiler/ir/passes/SrtWalker.h"
 
 #include <cstdlib>
@@ -71,7 +72,7 @@ struct Fixture {
     return static_cast<uint32_t>(program.memory_info.size() - 1u);
   }
 
-  void Plan() { BuildSrtPlan(program); }
+  void Plan() { TrackResources(program, {}, {}); }
 };
 
 struct TestMemory {
@@ -102,6 +103,16 @@ Value RawRead(Fixture &fixture, Value address, Value offset, uint32_t memory,
                             0x80, block);
 }
 
+void LoadBuffer(Fixture &fixture, std::array<Value, 4> words) {
+  const auto handle = fixture.Emit(ValueOpcode::GetBufferResource,
+                                   {words[0], words[1], words[2], words[3]});
+  const auto loaded = fixture.EmitMemory(
+      ValueOpcode::LoadBufferU32,
+      {handle, Value(0u), Value(0u), Value(0u), Value(true)},
+      fixture.AddMemory(ResourceKind::Buffer));
+  fixture.Emit(ValueOpcode::ReferenceU32, {loaded});
+}
+
 void TestImmediateFlatteningAndGvn() {
   Fixture fixture;
   const auto memory = fixture.AddMemory(ResourceKind::ScalarAddress, 0x20);
@@ -109,14 +120,11 @@ void TestImmediateFlatteningAndGvn() {
       fixture, Address(fixture, Value(0x1000u), Value(0u)), Value(0u), memory);
   const auto second = RawRead(
       fixture, Address(fixture, Value(0x1000u), Value(0u)), Value(0u), memory);
-  fixture.Emit(ValueOpcode::GetBufferResource,
-               {first, second, Value(16u), Value(0u)});
+  LoadBuffer(fixture, {first, second, Value(16u), Value(0u)});
 
   fixture.Plan();
   Check(fixture.program.srt_reads.size() == 1,
         "equivalent typed scalar reads were not coalesced");
-  Check(fixture.program.dynamic_reads.empty(),
-        "immediate scalar read was classified as dynamic");
   Check(fixture.program.memory_info[memory].planning_only,
         "flattened raw read was not kept as a planning-only root");
 
@@ -133,8 +141,7 @@ void TestRawScalarComponentAlignment() {
   const auto memory = fixture.AddMemory(ResourceKind::ScalarAddress, 2);
   const auto read = RawRead(
       fixture, Address(fixture, Value(0x1003u), Value(0u)), Value(2u), memory);
-  fixture.Emit(ValueOpcode::GetBufferResource,
-               {read, Value(0u), Value(16u), Value(0u)});
+  LoadBuffer(fixture, {read, Value(0u), Value(16u), Value(0u)});
   fixture.Plan();
 
   TestMemory memory_image{{{0x1000u, 0x12345678u}}};
@@ -150,7 +157,7 @@ void TestScalarMemoryDomainMismatchFails() {
   Fixture raw;
   const auto raw_memory = raw.AddMemory(ResourceKind::ScalarBuffer);
   RawRead(raw, Address(raw, Value(0x1000u), Value(0u)), Value(0u), raw_memory);
-  CheckFatal([&] { BuildSrtPlan(raw.program); },
+  CheckFatal([&] { TrackResources(raw.program, {}, {}); },
              "incompatible scalar memory metadata",
              "raw scalar load accepted descriptor-buffer metadata");
 
@@ -161,7 +168,7 @@ void TestScalarMemoryDomainMismatchFails() {
                   {Value(0x1000u), Value(0u), Value(16u), Value(0u)});
   buffer.EmitMemory(ValueOpcode::ReadConstBuffer, {resource, Value(0u)},
                     buffer_memory);
-  CheckFatal([&] { BuildSrtPlan(buffer.program); },
+  CheckFatal([&] { TrackResources(buffer.program, {}, {}); },
              "incompatible scalar memory metadata",
              "descriptor scalar load accepted raw-address metadata");
 }
@@ -173,12 +180,12 @@ void TestDynamicReadRemainsTyped() {
                                    {Value(static_cast<ScalarReg>(2))});
   const auto read = RawRead(
       fixture, Address(fixture, Value(0x1000u), Value(0u)), offset, memory);
-  fixture.Emit(ValueOpcode::GetBufferResource,
-               {read, Value(0u), Value(16u), Value(0u)});
+  LoadBuffer(fixture, {read, Value(0u), Value(16u), Value(0u)});
 
   fixture.Plan();
   Check(fixture.program.srt_reads.empty() &&
-            fixture.program.dynamic_reads == std::vector<Value>{read},
+            read.ResolveInstruction()->GetOpcode() == ValueOpcode::LoadAddressU32 &&
+            !fixture.program.memory_info[memory].planning_only,
         "dynamic scalar read received a fake flattened slot");
 }
 
@@ -189,8 +196,7 @@ void TestNestedSrtWalk() {
       fixture, Address(fixture, Value(0x1000u), Value(0u)), Value(0u), memory);
   const auto value =
       RawRead(fixture, Address(fixture, pointer, Value(0u)), Value(0u), memory);
-  fixture.Emit(ValueOpcode::GetBufferResource,
-               {value, Value(0u), Value(16u), Value(0u)});
+  LoadBuffer(fixture, {value, Value(0u), Value(16u), Value(0u)});
   fixture.Plan();
 
   TestMemory memory_image{{{0x1000u, 0x2000u}, {0x2000u, 0xabcdef01u}}};
@@ -211,7 +217,7 @@ void TestShaderBaseAndUserData() {
   const auto user = fixture.Emit(ValueOpcode::GetUserData,
                                  {Value(static_cast<ScalarReg>(2))});
   const auto sum = fixture.Emit(ValueOpcode::IAdd32, {user, Value(4u)});
-  fixture.Plan();
+
   fixture.program.descriptor_sources.push_back(
       {.dwords = {low, high, sum}, .dword_count = 3});
 
@@ -239,7 +245,7 @@ void TestCarryAndBitFields() {
                    {Value(0u), Value(0x89abcdefu), Value(0u), Value(32u)});
   const auto sign = fixture.Emit(ValueOpcode::BitFieldSExtract,
                                  {Value(0x000000f0u), Value(4u), Value(4u)});
-  fixture.Plan();
+
   fixture.program.descriptor_sources.push_back(
       {.dwords = {low, high, inserted, sign}, .dword_count = 4});
 
@@ -261,7 +267,7 @@ void TestInvariantAndDivergentPhi() {
   divergent.SetFlags(Type::U32);
   divergent.AddPhiOperand(&fixture.BlockAt(0), Value(7u));
   divergent.AddPhiOperand(&fixture.BlockAt(1), Value(9u));
-  fixture.Plan();
+
   fixture.program.descriptor_sources.push_back(
       {.dwords = {Value(&invariant)}, .dword_count = 1});
   fixture.program.descriptor_sources.push_back(
@@ -285,6 +291,7 @@ void TestControlDependentStandaloneLoadStaysTyped() {
   const auto read =
       RawRead(fixture, Address(fixture, Value(&base), Value(0u), 2), Value(0u),
               memory, 2);
+  fixture.Emit(ValueOpcode::ReferenceU32, {read}, 0, 2);
   fixture.Plan();
   Check(fixture.program.srt_reads.empty() &&
             read.ResolveInstruction()->GetOpcode() ==
@@ -307,7 +314,7 @@ void TestRuntime64BitDescriptorOps() {
       fixture.Emit(ValueOpcode::CompositeExtractU64, {combined, Value(0u)});
   const auto high =
       fixture.Emit(ValueOpcode::CompositeExtractU64, {combined, Value(1u)});
-  fixture.Plan();
+
   fixture.program.descriptor_sources.push_back(
       {.dwords = {low, high}, .dword_count = 2});
 
@@ -374,7 +381,6 @@ void TestUniformFirstLaneSamplerLod() {
                                   {packed, active});
   const auto divergent = fixture.Emit(
       ValueOpcode::ReadFirstLane, {stale, active});
-  fixture.Plan();
 
   Check(ValidateRuntimeValue(fixture.program, first),
         "uniform sampler LOD construction was rejected");
@@ -396,6 +402,49 @@ void TestUniformFirstLaneSamplerLod() {
         "uniform sampler LOD clamp evaluated incorrectly");
 }
 
+void TestFloatComparisonDescriptorInputs() {
+  struct Input {
+    uint32_t bits;
+    uint32_t less_equal;
+    uint32_t greater_equal;
+  };
+  constexpr std::array inputs = {
+      Input{0x00000001u, 0, 1}, Input{0x80000001u, 1, 0},
+      Input{0x007fffffu, 0, 1}, Input{0x807fffffu, 1, 0},
+      Input{0x00800000u, 0, 1}, Input{0x80800000u, 1, 0},
+      Input{0x00000000u, 1, 1}, Input{0x80000000u, 1, 1},
+      Input{0x7fc00000u, 0, 0}};
+  for (const bool flush : {false, true}) {
+    Fixture fixture;
+    const auto user = fixture.Emit(
+        ValueOpcode::GetUserData, {Value(static_cast<ScalarReg>(2))});
+    const auto value = fixture.Emit(ValueOpcode::BitCastF32U32, {user});
+    const auto less_equal = fixture.Emit(ValueOpcode::FPOrdLessThanEqual32,
+                                         {value, Value::F32(0.0f)});
+    const auto greater_equal = fixture.Emit(ValueOpcode::FPOrdGreaterThanEqual32,
+                                            {value, Value::F32(0.0f)});
+    less_equal.Instruction()->SetFlags(FPCompareFlags{flush});
+    greater_equal.Instruction()->SetFlags(FPCompareFlags{flush});
+    const auto low = fixture.Emit(ValueOpcode::SelectU32,
+                                  {less_equal, Value(1u), Value(0u)});
+    const auto high = fixture.Emit(ValueOpcode::SelectU32,
+                                   {greater_equal, Value(1u), Value(0u)});
+    fixture.program.descriptor_sources.push_back(
+        {.dwords = {low, high}, .dword_count = 2});
+
+    for (size_t index = 0; index < inputs.size(); index++) {
+      const auto &input = inputs[index];
+      const std::array user_data{input.bits};
+      DescriptorValue result;
+      Check(SrtWalker(fixture.program, {.user_data = user_data})
+                    .EvaluateDescriptor(0, result) &&
+                result.dwords[0] == (flush && index < 4 ? 1 : input.less_equal) &&
+                result.dwords[1] == (flush && index < 4 ? 1 : input.greater_equal),
+            "descriptor comparison disagrees with native FP32 input mode");
+    }
+  }
+}
+
 void TestSharedIntegerRuntimeDependencies() {
   Fixture fixture;
   const auto lane = fixture.Emit(ValueOpcode::LaneId);
@@ -439,37 +488,34 @@ void TestSharedIntegerRuntimeDependencies() {
 }
 
 void TestConstantBufferBounds() {
-  Fixture fixture;
-  const auto memory = fixture.AddMemory(ResourceKind::ScalarBuffer);
-  const auto buffer =
-      fixture.Emit(ValueOpcode::GetBufferResource,
-                   {Value(0x3000u), Value(0u), Value(16u), Value(0u)});
-  const auto read = fixture.EmitMemory(ValueOpcode::ReadConstBuffer,
-                                       {buffer, Value(12u)}, memory);
-  fixture.Emit(ValueOpcode::GetBufferResource,
-               {read, Value(0u), Value(16u), Value(0u)});
-  fixture.Plan();
+  struct Case {
+    uint32_t offset;
+    uint32_t immediate;
+    bool valid;
+    uint32_t expected;
+  };
+  for (const auto &test : {Case{12u, 0u, true, 0xa5a5a5a5u},
+                           Case{3u, 1u, true, 0x12345678u},
+                           Case{0xfffffffcu, 4u, false, 0u},
+                           Case{16u, 0u, false, 0u}}) {
+    Fixture fixture;
+    const auto memory = fixture.AddMemory(ResourceKind::ScalarBuffer, test.immediate);
+    const auto buffer =
+        fixture.Emit(ValueOpcode::GetBufferResource,
+                     {Value(0x3000u), Value(0u), Value(16u), Value(0u)});
+    const auto read = fixture.EmitMemory(ValueOpcode::ReadConstBuffer,
+                                         {buffer, Value(test.offset)}, memory);
+    LoadBuffer(fixture, {read, Value(0u), Value(16u), Value(0u)});
+    fixture.Plan();
 
-  TestMemory memory_image{{{0x300cu, 0xa5a5a5a5u}}};
-  SrtRuntime runtime{.read_memory = ReadMemory, .userdata = &memory_image};
-  std::vector<uint32_t> flat;
-  Check(SrtWalker(fixture.program, runtime).RefreshFlatBuffer(flat) &&
-            flat == std::vector<uint32_t>{0xa5a5a5a5u},
-        "constant-buffer SRT walk failed");
-
-  Fixture overflow;
-  const auto overflow_memory = overflow.AddMemory(ResourceKind::ScalarBuffer);
-  const auto overflow_buffer =
-      overflow.Emit(ValueOpcode::GetBufferResource,
-                    {Value(0x3000u), Value(0u), Value(16u), Value(0u)});
-  const auto overflow_read =
-      overflow.EmitMemory(ValueOpcode::ReadConstBuffer,
-                          {overflow_buffer, Value(16u)}, overflow_memory);
-  overflow.Emit(ValueOpcode::GetBufferResource,
-                {overflow_read, Value(0u), Value(16u), Value(0u)});
-  overflow.Plan();
-  Check(!SrtWalker(overflow.program, runtime).RefreshFlatBuffer(flat),
-        "out-of-bounds constant-buffer walk was accepted");
+    TestMemory memory_image{{{0x3000u, 0x12345678u}, {0x300cu, 0xa5a5a5a5u}}};
+    SrtRuntime runtime{.read_memory = ReadMemory, .userdata = &memory_image};
+    std::vector<uint32_t> flat;
+    Check(SrtWalker(fixture.program, runtime).RefreshFlatBuffer(flat) == test.valid,
+          "constant-buffer walk misaligned or wrapped its offset components");
+    Check(test.valid ? flat == std::vector<uint32_t>{test.expected} : memory_image.reads == 0,
+          "constant-buffer walk read the wrong word or accessed an out-of-bounds address");
+  }
 }
 
 void TestReadLaneElimination() {
@@ -552,7 +598,7 @@ void TestControlFlowValueSurvivesReadLaneFolding() {
 void TestUndefinedRuntimeValueFails() {
   Fixture fixture;
   const auto undef = fixture.Emit(ValueOpcode::UndefU32);
-  fixture.Plan();
+
   fixture.program.descriptor_sources.push_back(
       {.dwords = {undef}, .dword_count = 1});
   DescriptorValue result;
@@ -591,6 +637,7 @@ int main() {
     TestControlDependentStandaloneLoadStaysTyped();
     TestRuntime64BitDescriptorOps();
     TestUniformFirstLaneSamplerLod();
+    TestFloatComparisonDescriptorInputs();
     TestSharedIntegerRuntimeDependencies();
     TestConstantBufferBounds();
     TestReadLaneElimination();
@@ -611,6 +658,7 @@ int main() {
 #include "../src/graphics/shader/recompiler/ir/Program.cpp"
 #include "../src/graphics/shader/recompiler/ir/Type.cpp"
 #include "../src/graphics/shader/recompiler/ir/Value.cpp"
+#include "../src/graphics/shader/recompiler/ir/passes/ResourceTracking.cpp"
 #include "../src/graphics/shader/recompiler/ir/opcodes/ValueOpcodes.cpp"
 #include "../src/graphics/shader/recompiler/ir/passes/ConstantPropagation.cpp"
 #include "../src/graphics/shader/recompiler/ir/passes/DeadCodeElimination.cpp"

tests/VirtualMemoryAllocationTests.cpp

@@ -2163,6 +2163,51 @@ void TestFixedReserveRangeAddRollbackKeepsPlaceholder() {
 	std::printf("[host]    %-48s ok\n", test);
 }
 
+void TestExtendedAndUserMappingsDoNotAlias() {
+	const char* test = "ExtendedAndUserMappingsDoNotAlias";
+	// macOS starts its ordinary guest range at 448 GiB.
+	constexpr uint64_t user_address = 0x7000000000ull;
+	const uint64_t addresses[] {
+	    user_address,
+	    Libs::LibKernel::Memory::kExtendedMemoryBase + user_address,
+	    Libs::LibKernel::Memory::kExtendedMemoryBase + Libs::LibKernel::Memory::kExtendedMemorySize -
+	        SceKernelPageSize,
+	};
+	int64_t physical = 0;
+	CheckOk(test,
+	        Libs::LibKernel::Memory::KernelAllocateDirectMemory(
+	            0, Libs::LibKernel::Memory::KernelGetDirectMemorySize(),
+	            SceKernelPageSize * std::size(addresses), SceKernelPageSize, SceKernelMtypeC,
+	            &physical),
+	        "KernelAllocateDirectMemory");
+	for (size_t i = 0; i < std::size(addresses); ++i) {
+		void* mapped = reinterpret_cast<void*>(addresses[i]);
+		CheckOk(test,
+		        Libs::LibKernel::Memory::KernelMapDirectMemory(
+		            &mapped, SceKernelPageSize, SceKernelProtCpuRw,
+		            SceKernelMapFixed | SceKernelMapNoOverwrite,
+		            physical + i * SceKernelPageSize, SceKernelPageSize),
+		        "KernelMapDirectMemory");
+		Check(test, mapped == reinterpret_cast<void*>(addresses[i]), "fixed mapping moved");
+		*static_cast<uint64_t*>(mapped) = 0x0123456700000000ull + i;
+	}
+	for (size_t i = 0; i < std::size(addresses); ++i) {
+		uint64_t value = 0;
+		Check(test,
+		      Libs::LibKernel::Memory::TryReadBacking(addresses[i], &value, sizeof(value)) &&
+		          value == 0x0123456700000000ull + i &&
+		          *reinterpret_cast<uint64_t*>(addresses[i]) == value,
+		      "Extended and user mappings overlap or lost their backing");
+		CheckOk(test, Libs::LibKernel::Memory::KernelMunmap(addresses[i], SceKernelPageSize),
+		        "KernelMunmap");
+	}
+	CheckOk(test,
+	        Libs::LibKernel::Memory::KernelReleaseDirectMemory(
+	            physical, SceKernelPageSize * std::size(addresses)),
+	        "KernelReleaseDirectMemory");
+	std::printf("[host]    %-48s ok\n", test);
+}
+
 void TestLargeHintedReserveHostsSmallDirectMap() {
 	const char* test = "LargeHintedReserveHostsSmallDirectMap";
 
@@ -2220,6 +2265,128 @@ void TestLargeHintedReserveHostsSmallDirectMap() {
 	std::printf("[host]    %-48s ok\n", test);
 }
 
+void TestAutomaticMemoryReuseAndAliases() {
+	using namespace Libs::LibKernel::Memory;
+	const char*        test     = "AutomaticMemoryReuseAndAliases";
+	constexpr uint64_t block    = 0x200000;
+	constexpr uint64_t page     = SceKernelPageSize;
+	constexpr uint64_t base     = kExtendedMemoryBase + 0x20000000;
+	const auto         end      = static_cast<int64_t>(KernelGetDirectMemorySize());
+	int64_t            physical = -1;
+	CheckOk(test, AllocateDirectMemory(0, end, block, block, 0, &physical, true),
+	        "allocate automatic backing");
+	CheckOk(test, MapAutomaticMemory(base, block, SceKernelMtypeC, SceKernelProtCpuRw),
+	        "MapAutomaticMemory");
+	*reinterpret_cast<uint64_t*>(base + page * 3) = 0x0123456789abcdefull;
+	auto* first_alias                             = reinterpret_cast<void*>(base + block * 2);
+	auto* second_alias                            = reinterpret_cast<void*>(base + block * 3);
+	CheckOk(test,
+	        KernelMapDirectMemory(&first_alias, page * 3, SceKernelProtCpuRw, SceKernelMapFixed,
+	                              physical + page, page),
+	        "KernelMapDirectMemory(first alias)");
+	CheckOk(test,
+	        KernelMapDirectMemory(&second_alias, page * 2, SceKernelProtCpuRw, SceKernelMapFixed,
+	                              physical + page * 2, page),
+	        "KernelMapDirectMemory(overlapping alias)");
+	TestFailPhysicalMemoryUnmapAfter(1);
+	CheckFailed(test, KernelCheckedReleaseDirectMemory(physical, block),
+	            "KernelCheckedReleaseDirectMemory(rollback)");
+	Check(test,
+	      MapAutomaticMemory(base + block * 4, page, SceKernelMtypeC, SceKernelProtCpuRw) ==
+	          Libs::LibKernel::KERNEL_ERROR_EAGAIN,
+	      "restored mappings remained available to the automatic allocator");
+	CheckOk(test, KernelMunmap(base, block), "KernelMunmap(original)");
+	Check(test,
+	      MapAutomaticMemory(base, block, SceKernelMtypeC, SceKernelProtCpuRw) ==
+	          Libs::LibKernel::KERNEL_ERROR_EAGAIN,
+	      "automatic allocation reused aliased pages or ordinary physical memory");
+	CheckOk(test, MapAutomaticMemory(base, block - page * 3, SceKernelMtypeC, SceKernelProtCpuRw),
+	        "MapAutomaticMemory(fragmented reuse)");
+	Check(test,
+	      Query(test, base).offset == static_cast<uint64_t>(physical) &&
+	          Query(test, base + page).offset == static_cast<uint64_t>(physical + page * 4),
+	      "automatic allocation did not skip the union of overlapping aliases");
+	CheckOk(test, KernelMunmap(reinterpret_cast<uint64_t>(first_alias), page * 3),
+	        "KernelMunmap(first alias)");
+	CheckOk(test, MapAutomaticMemory(base + block * 4, page, SceKernelMtypeC, SceKernelProtCpuRw),
+	        "MapAutomaticMemory(partial alias release)");
+	Check(test, Query(test, base + block * 4).offset == static_cast<uint64_t>(physical + page),
+	      "partial alias release did not reclaim its unreferenced page");
+	CheckOk(test, KernelMunmap(reinterpret_cast<uint64_t>(second_alias), page),
+	        "KernelMunmap(partial second alias)");
+	CheckOk(test, MapAutomaticMemory(base + block * 5, page, SceKernelMtypeC, SceKernelProtCpuRw),
+	        "MapAutomaticMemory(partial unmap)");
+	Check(test, Query(test, base + block * 5).offset == static_cast<uint64_t>(physical + page * 2),
+	      "partial unmap did not reclaim the matching physical page");
+	Check(test, *reinterpret_cast<uint64_t*>(base + block * 3 + page) == 0x0123456789abcdefull,
+	      "automatic reuse changed a surviving alias");
+	CheckOk(test, KernelCheckedReleaseDirectMemory(physical, block),
+	        "KernelCheckedReleaseDirectMemory(donation)");
+	ExpectUnmapped(test, base);
+	ExpectUnmapped(test, base + block * 3 + page);
+	Check(test,
+	      MapAutomaticMemory(base, page, SceKernelMtypeC, SceKernelProtCpuRw) ==
+	          Libs::LibKernel::KERNEL_ERROR_EAGAIN,
+	      "released donation remained in the automatic free list");
+	int64_t reused = -1;
+	CheckOk(test,
+	        KernelAllocateDirectMemory(physical, physical + block, block, block, SceKernelMtypeC,
+	                                   &reused),
+	        "KernelAllocateDirectMemory(released donation)");
+	Check(test, reused == physical, "donation did not return to the ordinary allocator");
+	CheckOk(test, KernelCheckedReleaseDirectMemory(reused, block), "release ordinary reuse");
+	std::printf("[host]    %-48s ok\n", test);
+}
+
+void TestAutomaticMemoryFragmentedMapRollback() {
+	using namespace Libs::LibKernel::Memory;
+	const char*        test   = "AutomaticMemoryFragmentedMapRollback";
+	constexpr uint64_t block  = 0x200000;
+	constexpr uint64_t base   = kExtendedMemoryBase + 0x22000000;
+	const auto         end    = static_cast<int64_t>(KernelGetDirectMemorySize());
+	int64_t            first  = -1;
+	int64_t            gap    = -1;
+	int64_t            second = -1;
+	CheckOk(test, AllocateDirectMemory(0, end, block, block, 0, &first, true), "first donation");
+	CheckOk(test, AllocateDirectMemory(first + block, end, block, block, 0, &gap),
+	        "ordinary allocation between donations");
+	CheckOk(test, AllocateDirectMemory(gap + block, end, block, block, 0, &second, true),
+	        "second donation");
+	Check(test, gap == first + block && second == gap + block,
+	      "test setup did not produce adjacent ownership ranges");
+	auto* address = reinterpret_cast<void*>(base);
+	CheckOk(test, KernelReserveVirtualRange(&address, block, SceKernelMapFixed, block),
+	        "reserve first mapping piece");
+	TestFailNextFixedReserveRangeRegistration();
+	CheckFailed(test, MapAutomaticMemory(base, block * 2, SceKernelMtypeC, SceKernelProtCpuRw),
+	            "MapAutomaticMemory(second-piece failure)");
+	ExpectUnmapped(test, base);
+	CheckOk(test, MapAutomaticMemory(base, block * 2, SceKernelMtypeC, SceKernelProtCpuRw),
+	        "MapAutomaticMemory(after rollback)");
+	Check(test,
+	      Query(test, base).offset == static_cast<uint64_t>(first) &&
+	          Query(test, base + block).offset == static_cast<uint64_t>(second),
+	      "fragmented map did not preserve both donated physical ranges");
+	auto* alias = reinterpret_cast<void*>(base + block * 3);
+	CheckOk(
+	    test,
+	    KernelMapDirectMemory(&alias, block * 2, SceKernelProtCpuRw, SceKernelMapFixed, gap, block),
+	    "alias across ordinary and donated ownership");
+	CheckOk(test, KernelMunmap(base, block * 2), "unmap fragmented allocation");
+	CheckFailed(test, MapAutomaticMemory(base, block * 2, SceKernelMtypeC, SceKernelProtCpuRw),
+	            "automatic allocation while second donation remains aliased");
+	CheckOk(test, KernelMunmap(reinterpret_cast<uint64_t>(alias), block * 2),
+	        "unmap mixed ownership alias");
+	CheckFailed(test, MapAutomaticMemory(base, block * 3, SceKernelMtypeC, SceKernelProtCpuRw),
+	            "automatic allocation beyond donated capacity");
+	CheckOk(test, MapAutomaticMemory(base, block * 2, SceKernelMtypeC, SceKernelProtCpuRw),
+	        "reuse both donations after final alias removal");
+	CheckOk(test, KernelCheckedReleaseDirectMemory(first, block), "release first donation");
+	CheckOk(test, KernelCheckedReleaseDirectMemory(gap, block), "release ordinary allocation");
+	CheckOk(test, KernelCheckedReleaseDirectMemory(second, block), "release second donation");
+	std::printf("[host]    %-48s ok\n", test);
+}
+
 void TestMemoryPoolAlignmentContracts() {
 	const char* test = "MemoryPoolAlignmentContracts";
 	void*       addr = nullptr;
@@ -3155,6 +3322,9 @@ int main(int argc, char** argv) {
 	RunTest(TestFixedReserveRollbackSkipsUntouchedChunks);
 	RunTest(TestFixedReserveRangeAddRollbackKeepsPlaceholder);
 	RunTest(TestLargeHintedReserveHostsSmallDirectMap);
+	RunTest(TestExtendedAndUserMappingsDoNotAlias);
+	RunTest(TestAutomaticMemoryReuseAndAliases);
+	RunTest(TestAutomaticMemoryFragmentedMapRollback);
 	RunTest(TestMemoryPoolAlignmentContracts);
 	RunTest(TestProsperoSampleMemoryPoolExpandCommit);
 	RunTest(TestFragmentedMemoryPoolBacking);