filesystem: keep unmapped guest paths out of the host namespace

* libs: fix CP932 iconv conversion on Windows and macOS

SDL3's SDL_SYSTEM_ICONV build option defaults off on Windows and macOS,
so SDL falls back to its own bundled iconv there, which only implements
ASCII/Latin-1/UTF/UCS. SDL_iconv_open("UTF-8", "CP932") always failed as
a result, crashing games that convert Shift-JIS strings via
sceCesMbcsStrToUtf8Str (e.g. save-data dialogs).

Add a small CesIconv shim selected at compile time: Windows converts one
character at a time via the Win32 codepage-932 APIs, replicating
SDL_iconv's chunked semantics and error codes; macOS calls real POSIX
iconv directly, since libiconv is always present there and does support
CP932, unlike SDL's fallback. Linux is untouched, since SDL already
links real system iconv by default there.

Co-Authored-By: Claude Sonnet 5 <noreply@anthropic.com>

* libs: fix SaveDataDialog infinite loop on save

SaveDataDialogOpen() jumped straight from SAVE_STATUS_INITIALIZED (1) to
SAVE_STATUS_FINISHED (3), skipping SAVE_STATUS_RUNNING (2) entirely. At
least one title's save-write dialog stage only accepts a FINISHED status
after observing a RUNNING poll first; without it, the title assumed the
Open() request never took effect and re-issued it forever, hanging the
save flow (this only became visible once the CP932 crash above stopped
masking it).

Open() now starts the dialog as RUNNING. UpdateStatus() advances the
state machine, reporting RUNNING once before flipping to FINISHED for
subsequent polls; GetStatus() is untouched, staying a pure peek with no
side effects, matching the real SDK's pump-vs-peek split between the two
calls.

Co-Authored-By: Claude Sonnet 5 <noreply@anthropic.com>

* Add iconv linking to macOS

* Fix missing macOS iconv link for tests

* Fix CP932 edge cases and save dialog status consistency

---------

Co-authored-by: Claude Sonnet 5 <noreply@anthropic.com>
Co-authored-by: nmzik <Nmzik@mail.ru>

3rdparty/CMakeLists.txt

@@ -29,6 +29,10 @@ FetchContent_Declare(
 FetchContent_MakeAvailable(xbyak zydis)
 
 set(SDL_SHARED OFF CACHE BOOL "" FORCE)
+if(APPLE)
+	# SDL's built-in iconv fallback does not support CP932.
+	set(SDL_SYSTEM_ICONV ON CACHE BOOL "Use system iconv for CP932 conversion" FORCE)
+endif()
 add_subdirectory(SDL3)
 
 add_library(Vulkan-Headers INTERFACE)

CMakeLists.txt

@@ -490,6 +490,17 @@ add_executable(save_data_memory_tests EXCLUDE_FROM_ALL
 target_link_libraries(save_data_memory_tests common fmt::fmt)
 target_include_directories(save_data_memory_tests PRIVATE ${inc_headers})
 
+add_executable(ces_tests EXCLUDE_FROM_ALL "${KYTY_TESTS_DIR}/CesTests.cpp")
+target_link_libraries(ces_tests common fmt::fmt SDL3::SDL3-static)
+target_include_directories(ces_tests PRIVATE ${inc_headers})
+
+add_executable(save_data_dialog_tests EXCLUDE_FROM_ALL
+	"${KYTY_TESTS_DIR}/SaveDataDialogTests.cpp"
+	"${KYTY_SOURCE_DIR}/libs/dialog.cpp"
+)
+target_link_libraries(save_data_dialog_tests common fmt::fmt)
+target_include_directories(save_data_dialog_tests PRIVATE ${inc_headers})
+
 add_kyty_full_emulator_test(http_uri_parse_tests
 	"${KYTY_TESTS_DIR}/HttpUriParseTests.cpp")
 
@@ -581,6 +592,8 @@ if(BUILD_TESTING)
 	add_test(NAME ngs2_sampler COMMAND $<TARGET_FILE:ngs2_sampler_tests>)
 	add_test(NAME pad_haptics COMMAND $<TARGET_FILE:pad_haptics_tests>)
 	add_test(NAME save_data_memory COMMAND $<TARGET_FILE:save_data_memory_tests>)
+	add_test(NAME ces COMMAND $<TARGET_FILE:ces_tests>)
+	add_test(NAME save_data_dialog COMMAND $<TARGET_FILE:save_data_dialog_tests>)
 	add_test(NAME http_uri_parse COMMAND $<TARGET_FILE:http_uri_parse_tests>)
 	add_test(NAME shader_recompiler_compute COMMAND $<TARGET_FILE:shader_recompiler_compute_tests>)
 	add_test(NAME shader_recompiler_alignbyte
@@ -650,6 +663,8 @@ if(BUILD_TESTING)
 		ngs2_sampler_tests
 		pad_haptics_tests
 		save_data_memory_tests
+		ces_tests
+		save_data_dialog_tests
 		http_uri_parse_tests
 	)
 endif()

README.md

@@ -7,6 +7,10 @@
 [![Status](https://img.shields.io/badge/status-active%20development-orange.svg)](#current-status)
 [![License](https://img.shields.io/badge/license-GPL--2.0-blue.svg)](LICENSE)
 
+**[Weekly updates](https://github.com/KytyPS5/KytyPS5/discussions/862)** — game progress, recent fixes and ongoing development.
+
+**[Development on Discord](https://discord.gg/UNrkMqGaBg)** — KytyPS5 development.
+
 KytyPS5 is a free and open-source PlayStation 5 emulator written in C++ for Windows and Linux,
 with experimental macOS support. It is based on a heavily modified version of
 [Kyty](https://github.com/InoriRus/Kyty). The project is in active development, and behavior
@@ -68,8 +72,8 @@ or graphical glitches, so please include the version you tested when reporting a
       <img src="docs/screenshots/ps5-02.png" width="300" alt="Demon's Souls running in KytyPS5">
     </td>
     <td align="center">
-      <strong>Hellboy</strong><br>
-      <img src="docs/screenshots/ps5-06.png" width="300" alt="Hellboy running in KytyPS5">
+      <strong>UFC 5</strong><br>
+      <img src="docs/screenshots/ps5-06.png" width="300" alt="UFC 5 running in KytyPS5">
     </td>
   </tr>
 </table>

src/common/platform/sysLinuxFileIO.cpp

@@ -54,7 +54,7 @@ struct sys_file_t {
 #endif
 
 static std::filesystem::path get_internal_name(const std::filesystem::path& name) {
-	return name.is_absolute() ? name : (std::filesystem::path(".") / name);
+	return name.empty() || name.is_absolute() ? name : (std::filesystem::path(".") / name);
 }
 
 // Pass access-pattern hints to the host.

src/graphics/host_gpu/regionDefinitions.h

@@ -3,6 +3,7 @@
 
 #include "common/bitArray.h"
 #include "common/common.h"
+#include "kernel/memory.h"
 
 #include <compare>
 
@@ -10,7 +11,8 @@ namespace Libs::Graphics {
 
 constexpr uint64_t TRACKER_PAGE_SIZE    = 4ull * 1024ull;
 constexpr uint64_t TRACKER_REGION_SIZE  = 4ull * 1024ull * 1024ull;
-constexpr uint64_t TRACKER_ADDRESS_SIZE = 1ull << 40u;
+constexpr uint64_t TRACKER_ADDRESS_SIZE = 1ull << 44u;
+constexpr uint64_t LOWER_ADDRESS_SIZE   = 1ull << 40u;
 constexpr size_t   TRACKER_REGION_PAGES = TRACKER_REGION_SIZE / TRACKER_PAGE_SIZE;
 
 struct GuestRange {
@@ -19,8 +21,11 @@ struct GuestRange {
 
 	[[nodiscard]] constexpr bool Empty() const noexcept { return address == 0 && size == 0; }
 	[[nodiscard]] constexpr bool Valid() const noexcept {
-		return address != 0 && size != 0 && address < TRACKER_ADDRESS_SIZE &&
-		       size <= TRACKER_ADDRESS_SIZE - address;
+		if (address == 0 || size == 0) return false;
+		if (address < LOWER_ADDRESS_SIZE) return size <= LOWER_ADDRESS_SIZE - address;
+		constexpr auto base = LibKernel::Memory::kExtendedMemoryBase;
+		constexpr auto length = LibKernel::Memory::kExtendedMemorySize;
+		return address >= base && address - base < length && size <= length - (address - base);
 	}
 	[[nodiscard]] constexpr bool     ValidOrEmpty() const noexcept { return Empty() || Valid(); }
 	[[nodiscard]] constexpr uint64_t End() const noexcept { return address + size; }

src/graphics/host_gpu/renderer/cache/bufferCache.cpp

@@ -55,7 +55,8 @@ template <bool insert>
 void BufferCache::ChangeRegister(BufferId id) {
 	auto& buffer = m_slot_buffers[id];
 	PageTable::PageRange pages {};
-	EXIT_IF(!PageTable::TryGetPageRange(buffer.CpuAddress(), buffer.Size(), pages));
+	EXIT_IF(!(GuestRange {buffer.CpuAddress(), buffer.Size()}.Valid()) ||
+	        !PageTable::TryGetPageRange(buffer.CpuAddress(), buffer.Size(), pages));
 	for (size_t page = pages.first; page < pages.last_exclusive; ++page) {
 		if constexpr (insert) {
 			m_page_table[page] = id;
@@ -64,6 +65,7 @@ void BufferCache::ChangeRegister(BufferId id) {
 		}
 	}
 	const auto size_pages = pages.last_exclusive - pages.first;
+	const auto table_offset = PageIndex(buffer.CpuAddress()) * sizeof(vk::DeviceAddress);
 	if constexpr (insert) {
 		const auto [it, inserted] = m_buffers.emplace(buffer.CpuAddress(), id);
 		(void)it;
@@ -75,7 +77,7 @@ void BufferCache::ChangeRegister(BufferId id) {
 		for (uint64_t i = 0; i < size_pages; ++i) {
 			addresses.push_back(buffer.BufferDeviceAddress() + (i << CACHING_PAGEBITS));
 		}
-		WriteDataBuffer(m_bda_pagetable_buffer, pages.first * sizeof(vk::DeviceAddress),
+		WriteDataBuffer(m_bda_pagetable_buffer, table_offset,
 		                addresses.data(), addresses.size() * sizeof(vk::DeviceAddress));
 	} else {
 		const auto found = m_buffers.find(buffer.CpuAddress());
@@ -84,7 +86,7 @@ void BufferCache::ChangeRegister(BufferId id) {
 		EXIT_IF(buffer.Size() > m_total_used_memory);
 		m_total_used_memory -= buffer.Size();
 		m_lru_cache.Free(buffer.lru_id);
-		m_bda_pagetable_buffer.Fill(pages.first * sizeof(vk::DeviceAddress),
+		m_bda_pagetable_buffer.Fill(table_offset,
 		                            size_pages * sizeof(vk::DeviceAddress), 0);
 		buffer.is_deleted = true;
 	}
@@ -312,10 +314,14 @@ BufferCache::OverlapResult BufferCache::ResolveOverlaps(uint64_t vaddr, uint64_t
 			// Reserve space in the incoming stream's direction of growth.
 			// The old buffer extending left of the request predicts growth to the right, and vice versa.
 			if (expands_left) {
-				end += std::min(StreamLeapSize, PageTable::kAddressSpaceSize - end);
+				end += std::min(StreamLeapSize, (vaddr < LOWER_ADDRESS_SIZE ? LOWER_ADDRESS_SIZE
+				                                       : LibKernel::Memory::kExtendedMemoryBase +
+				                                             LibKernel::Memory::kExtendedMemorySize) - end);
 			}
 			if (expands_right) {
-				const auto minimum = CACHING_PAGESIZE * 2;
+				const auto minimum = vaddr < LOWER_ADDRESS_SIZE
+				                         ? CACHING_PAGESIZE * 2
+				                         : LibKernel::Memory::kExtendedMemoryBase;
 				if (begin > minimum) {
 					begin -= std::min(StreamLeapSize, begin - minimum);
 				}

src/graphics/host_gpu/renderer/cache/bufferCache.h

@@ -29,10 +29,22 @@ class BufferCache {
 public:
 	static constexpr uint32_t CACHING_PAGEBITS  = 14;
 	static constexpr uint64_t CACHING_PAGESIZE  = uint64_t {1} << CACHING_PAGEBITS;
-	static constexpr uint64_t CACHING_NUMPAGES  = uint64_t {1} << (40 - CACHING_PAGEBITS);
+	static constexpr uint64_t CACHING_NUMPAGES  = (LOWER_ADDRESS_SIZE + LibKernel::Memory::kExtendedMemorySize) >> CACHING_PAGEBITS;
 	static constexpr uint64_t BDA_PAGETABLE_SIZE =
 	    CACHING_NUMPAGES * sizeof(vk::DeviceAddress);
 
+	static constexpr uint64_t PageIndex(uint64_t address) {
+		return (address < LOWER_ADDRESS_SIZE
+		            ? address
+		            : address - LibKernel::Memory::kExtendedMemoryBase + LOWER_ADDRESS_SIZE) >>
+		       CACHING_PAGEBITS;
+	}
+	static constexpr uint64_t GuestAddress(uint64_t offset) {
+		return offset < LOWER_ADDRESS_SIZE
+		           ? offset
+		           : offset - LOWER_ADDRESS_SIZE + LibKernel::Memory::kExtendedMemoryBase;
+	}
+
 	BufferCache(GraphicContext& graphics, CommandScheduler& scheduler, PageManager& page_manager,
 	            TextureCache& texture_cache);
 	~BufferCache();
@@ -88,7 +100,7 @@ private:
 		bool                has_stream_leap;
 	};
 
-	using PageTable = MultiLevelPageTable<BufferId, CACHING_PAGEBITS, 40, 16>;
+	using PageTable = MultiLevelPageTable<BufferId, CACHING_PAGEBITS, 44, 20>;
 	static_assert(CACHING_PAGESIZE == (uint64_t {1} << PageTable::kPageBits));
 	void WriteDataBuffer(Buffer& buffer, uint64_t address, const void* source, uint64_t size);
 	void TouchBuffer(const Buffer& buffer);

src/graphics/host_gpu/renderer/cache/faultManager.cpp

@@ -134,8 +134,9 @@ void FaultManager::ProcessFaultBuffer() {
 		const auto* faults = std::bit_cast<const uint64_t*>(mapped);
 		const auto  count  = static_cast<uint32_t>(faults[0]);
 		for (uint32_t index = 1; index <= count; ++index) {
-			fault_ranges.Add(faults[index], BufferCache::CACHING_PAGESIZE);
-			LOGF("Accessed non-GPU cached memory at 0x%016" PRIx64 "\n", faults[index]);
+			const auto address = BufferCache::GuestAddress(faults[index]);
+			fault_ranges.Add(address, BufferCache::CACHING_PAGESIZE);
+			LOGF("Accessed non-GPU cached memory at 0x%016" PRIx64 "\n", address);
 		}
 		fault_ranges.ForEach([this](uint64_t start, uint64_t end) {
 			EXIT_IF(end - start > std::numeric_limits<uint32_t>::max());

src/graphics/host_gpu/renderer/cache/samplerCache.cpp

@@ -14,10 +14,10 @@ SamplerCache::~SamplerCache() {
 	}
 }
 
-vk::Sampler SamplerCache::GetSampler(const ShaderSamplerResource& r) {
+vk::Sampler SamplerCache::GetSampler(const ShaderSamplerResource& r, bool integer_border) {
 	Common::LockGuard lock(m_mutex);
 
-	const SamplerKey key {r.fields[0], r.fields[1], r.fields[2], r.fields[3]};
+	const SamplerKey key {r.fields[0], r.fields[1], r.fields[2], r.fields[3], integer_border};
 	if (auto iter = m_samplers.find(key); iter != m_samplers.end()) {
 		return iter->second;
 	}
@@ -93,23 +93,23 @@ vk::Sampler SamplerCache::GetSampler(const ShaderSamplerResource& r) {
 		return vk::SamplerAddressMode::eClampToBorder;
 	};
 
-	vk::BorderColor border = vk::BorderColor::eIntTransparentBlack;
+	vk::BorderColor border = integer_border ? vk::BorderColor::eIntTransparentBlack
+	                                       : vk::BorderColor::eFloatTransparentBlack;
 	switch (static_cast<Prospero::SamplerBorderColor>(r.BorderColorType())) {
-		case Prospero::SamplerBorderColor::kTransBlack:
-			border = vk::BorderColor::eIntTransparentBlack;
-			break;
+		case Prospero::SamplerBorderColor::kTransBlack: break;
 		case Prospero::SamplerBorderColor::kOpaqueBlack:
-			border = vk::BorderColor::eIntOpaqueBlack;
+			border = integer_border ? vk::BorderColor::eIntOpaqueBlack
+			                        : vk::BorderColor::eFloatOpaqueBlack;
 			break;
 		case Prospero::SamplerBorderColor::kOpaqueWhite:
-			border = vk::BorderColor::eIntOpaqueWhite;
+			border = integer_border ? vk::BorderColor::eIntOpaqueWhite
+			                        : vk::BorderColor::eFloatOpaqueWhite;
 			break;
 		case Prospero::SamplerBorderColor::kFromTable:
 			LOGF(
 			    "temporary: approximating table border color as transparent black, index = %" PRIu16
 			    "\n",
 			    r.BorderColorPtr());
-			border = vk::BorderColor::eIntTransparentBlack;
 			break;
 		default: EXIT("unknown border color: %d", static_cast<int>(r.BorderColorType()));
 	}

src/graphics/host_gpu/renderer/cache/samplerCache.h

@@ -24,10 +24,10 @@ public:
 	~SamplerCache();
 	KYTY_CLASS_NO_COPY(SamplerCache);
 
-	vk::Sampler GetSampler(const ShaderSamplerResource& r);
+	vk::Sampler GetSampler(const ShaderSamplerResource& r, bool integer_border);
 
 private:
-	using SamplerKey = std::array<uint32_t, 4>;
+	using SamplerKey = std::array<uint32_t, 5>;
 
 	struct SamplerKeyHash {
 		std::size_t operator()(const SamplerKey& key) const {

src/graphics/host_gpu/renderer/cache/textureCache.h

@@ -91,9 +91,9 @@ private:
 	};
 
 	using ImageIds       = InlinePageOwnerList<ImageId, 16>;
-	using ImagePageTable = MultiLevelPageTable<ImageIds, 20, 40, 10>;
+	using ImagePageTable = MultiLevelPageTable<ImageIds, 20, 44, 14>;
 
-	// Callers have validated the nonempty 40-bit range with TryGetPageRange.
+	// Callers have validated the nonempty 44-bit range with TryGetPageRange.
 	template <typename Func>
 	static void ForEachPage(uint64_t address, size_t size, Func&& func) {
 		using FuncReturn = typename std::invoke_result<Func, uint64_t>::type;

src/graphics/host_gpu/renderer/image/image.cpp

@@ -628,10 +628,8 @@ void Validate(const ImageInfo& info) {
 			break;
 		case ImageMetadataKind::Dcc:
 		case ImageMetadataKind::Cmask:
-			if (info.metadata.range.address == 0 ||
-			    info.metadata.range.address >= TRACKER_ADDRESS_SIZE ||
-			    (info.metadata.range.size != 0 &&
-			     info.metadata.range.size > TRACKER_ADDRESS_SIZE - info.metadata.range.address) ||
+			if (!GuestRange {info.metadata.range.address,
+			                 std::max<uint64_t>(info.metadata.range.size, 1)}.Valid() ||
 			    info.metadata.compression == VideoOutCompression::Unsupported) {
 				EXIT("invalid color metadata\n");
 			}

src/graphics/host_gpu/renderer/pipeline/descriptors.cpp

@@ -196,8 +196,8 @@ bool IsSupportedDepthTextureEncoding(const ShaderTextureResource& descriptor, bo
 	constexpr uint32_t htile_control = 0x00280000u;
 	const uint32_t expected_control  = htile_control | (descriptor.MsaaDepth() ? (1u << 10u) : 0u);
 	const auto     metadata_addr     = descriptor.MetaAddr() << 8u;
-	return metadata_control == expected_control && metadata_addr != 0 &&
-	       metadata_addr < TRACKER_ADDRESS_SIZE && (metadata_addr & 0x7fffu) == 0 &&
+	return metadata_control == expected_control && GuestRange {metadata_addr, 1}.Valid() &&
+	       (metadata_addr & 0x7fffu) == 0 &&
 	       descriptor.TileMode() == Prospero::TileMode::kDepth;
 }
 
@@ -738,14 +738,15 @@ static vk::Sampler NativeSampler(RenderContext&                       context,
                                  const ShaderRecompiler::IR::CompiledShaderInfo& program,
                                  uint32_t index,
                                  const ShaderRecompiler::IR::DescriptorValue& value) {
-	auto descriptor = DecodeNativeDescriptor<ShaderSamplerResource>(value);
-	if (!program.info.samplers[index].depth_compare) {
+	auto        descriptor = DecodeNativeDescriptor<ShaderSamplerResource>(value);
+	const auto& sampler = program.info.samplers[index];
+	if (!sampler.depth_compare) {
 		descriptor.fields[0] &= ~(0x7u << 12u);
 	}
-	if (program.info.samplers[index].force_point_filtering) {
+	if (sampler.force_point_filtering) {
 		descriptor.SetPointFiltering();
 	}
-	return context.GetSamplerCache().GetSampler(descriptor);
+	return context.GetSamplerCache().GetSampler(descriptor, sampler.integer_border);
 }
 
 static vk::DescriptorBufferInfo NativeUpload(RenderContext&            context,

src/graphics/host_gpu/renderer/pipeline/pipelineCache.cpp

@@ -29,7 +29,6 @@
 #include <span>
 #include <spirv-tools/libspirv.hpp>
 #include <string_view>
-#include <tuple>
 #include <utility>
 #include <vector>
 #include <xxhash.h>
@@ -116,33 +115,22 @@ void DumpShaderSpirv(const char* stage_name, uint64_t shader_hash,
 }
 
 void DumpShaderOriginal(const char* stage_name, uint64_t shader_hash,
-                        std::span<const uint32_t> code, const std::string& decoded_dump) {
+                        std::span<const uint32_t> code) {
 	if (!Config::GraphicsDebugDumpEnabled()) {
 		return;
 	}
 	EXIT_IF(code.empty());
 	static std::atomic_int id = 0;
-	const auto base = Config::GetShaderLogFolder() / "original" /
-	                  fmt::format("{:04d}_new_shader_{}_{:016x}", id++, stage_name, shader_hash);
-	Common::File::CreateDirectories(base.parent_path());
-	for (const auto& [suffix, data, size]: {
-	         std::tuple {".bin", static_cast<const void*>(code.data()), code.size_bytes()},
-	         std::tuple {".rdna2", static_cast<const void*>(decoded_dump.data()),
-	                     decoded_dump.size()},
-	     }) {
-		if (size == 0) {
-			continue;
-		}
-		auto path = base;
-		path += suffix;
-		Common::File file(path);
-		if (file.IsInvalid()) {
-			const auto path_text = Common::PathToString(path);
-			LOGF_COLOR(Log::Color::BrightRed, "Can't create file: %s\n", path_text.c_str());
-		} else {
-			file.Write(data, size);
-		}
+	const auto path = Config::GetShaderLogFolder() / "original" /
+	                  fmt::format("{:04d}_new_shader_{}_{:016x}.bin", id++, stage_name, shader_hash);
+	Common::File::CreateDirectories(path.parent_path());
+	Common::File file(path);
+	if (file.IsInvalid()) {
+		const auto path_text = Common::PathToString(path);
+		LOGF_COLOR(Log::Color::BrightRed, "Can't create file: %s\n", path_text.c_str());
+		return;
 	}
+	file.Write(code.data(), code.size_bytes());
 }
 
 bool ValidateShaderSpirv(const char* label, uint64_t shader_hash,
@@ -225,25 +213,13 @@ struct PipelineCache::ProgramCache {
 
 	static constexpr std::size_t MaxStaticKeyWords = 13 + ShaderVertexInputInfo::RES_MAX * 13;
 
-	Permutation CompilePermutation(const ShaderParams&                          params,
+	Permutation CompilePermutation(const char*                                  stage_name,
 	                               const ShaderRecompiler::CompileOptions&      options,
 	                               ShaderRecompiler::TranslateResult            translated,
 	                               ShaderRecompiler::IR::ResourceSpecialization specialization,
 	                               uint32_t push_data_start_dword) {
-		const char* stage_name = nullptr;
-		switch (options.stage) {
-			case ShaderType::Vertex: stage_name = "vs"; break;
-			case ShaderType::Mesh: stage_name = "ms"; break;
-			case ShaderType::Local: stage_name = "ls"; break;
-			case ShaderType::TessellationControl: stage_name = "hs"; break;
-			case ShaderType::TessellationEvaluation: stage_name = "ds"; break;
-			case ShaderType::Pixel: stage_name = "ps"; break;
-			case ShaderType::Compute: stage_name = "cs"; break;
-			default: EXIT("invalid pipeline shader stage\n");
-		}
 		auto result = ShaderRecompiler::CompileProgram(std::move(translated), options,
 		                                               specialization, push_data_start_dword);
-		DumpShaderOriginal(stage_name, options.shader_hash, params.code, result.decoded_dump);
 		if (!ValidateShaderSpirv(options.dump_label, options.shader_hash, result.spirv)) {
 			DumpShaderSpirv(stage_name, options.shader_hash, result.spirv);
 			EXIT("%s failed hash=0x%016" PRIx64 ": SPIR-V validation failed\n", options.dump_label,
@@ -321,14 +297,15 @@ struct PipelineCache::ProgramCache {
 			stage_input.compute = &input_info;
 		}
 		const char* label = nullptr;
+		const char* stage_name = nullptr;
 		switch (stage) {
-			case ShaderType::Vertex: label = "ShaderRecompiler VS"; break;
-			case ShaderType::Mesh: label = "ShaderRecompiler MS"; break;
-			case ShaderType::Local: label = "ShaderRecompiler LS"; break;
-			case ShaderType::TessellationControl: label = "ShaderRecompiler HS"; break;
-			case ShaderType::TessellationEvaluation: label = "ShaderRecompiler DS"; break;
-			case ShaderType::Pixel: label = "ShaderRecompiler PS"; break;
-			case ShaderType::Compute: label = "ShaderRecompiler CS"; break;
+			case ShaderType::Vertex: label = "ShaderRecompiler VS"; stage_name = "vs"; break;
+			case ShaderType::Mesh: label = "ShaderRecompiler MS"; stage_name = "ms"; break;
+			case ShaderType::Local: label = "ShaderRecompiler LS"; stage_name = "ls"; break;
+			case ShaderType::TessellationControl: label = "ShaderRecompiler HS"; stage_name = "hs"; break;
+			case ShaderType::TessellationEvaluation: label = "ShaderRecompiler DS"; stage_name = "ds"; break;
+			case ShaderType::Pixel: label = "ShaderRecompiler PS"; stage_name = "ps"; break;
+			case ShaderType::Compute: label = "ShaderRecompiler CS"; stage_name = "cs"; break;
 			default: EXIT("invalid pipeline shader stage\n");
 		}
 		ShaderRecompiler::CompileOptions options;
@@ -351,6 +328,7 @@ struct PipelineCache::ProgramCache {
 		} else {
 			options.wave_size = input_info.wave_size;
 		}
+		DumpShaderOriginal(stage_name, options.shader_hash, params.code);
 		auto translated = ShaderRecompiler::TranslateProgram(params.code, options);
 		if (translated.skip_dispatch) {
 			entry = programs.try_emplace(lookup_key, ShaderRecompiler::IR::ResourcePlan {}).first;
@@ -365,7 +343,7 @@ struct PipelineCache::ProgramCache {
 			    entry->second.specialization));
 		}
 		entry->second.permutations.push_back(CompilePermutation(
-		    params, options, std::move(translated), entry->second.specialization, push_data_cursor));
+		    stage_name, options, std::move(translated), entry->second.specialization, push_data_cursor));
 		const auto& permutation = entry->second.permutations.back();
 		input_info.stage = {.program = &permutation.program, .resources = &entry->second.resources};
 		permutation.program.bindings.AdvancePushData(push_data_cursor);

src/graphics/host_gpu/renderer/renderDraw.cpp

@@ -575,12 +575,17 @@ RenderState RenderExecutor::AcquireRenderTargets(CommandBuffer& buffer, RenderCo
 	return state;
 }
 
-static bool DrawHasActivePixelShader(const CommandBuffer& buffer) {
-	const auto& ctx              = buffer.GetRegisters();
-	const auto& sh_regs          = ctx.GetShaderRegisters();
-	const bool  has_color_output = (ctx.GetRenderTargetMask() & sh_regs.m_cbShaderMask) != 0;
-	return buffer.GetShaders().GetPs().ps_regs.data_addr != 0 &&
-	       (has_color_output || PixelShaderHasDepthOrCoverageSideEffects(sh_regs));
+static uint32_t DrawColorOutputMask(const HW::Context& ctx) {
+	const auto& sh_regs     = ctx.GetShaderRegisters();
+	const auto  write_mask  = ctx.GetRenderTargetMask() & sh_regs.m_cbShaderMask;
+	uint32_t    output_mask = 0;
+	for (uint32_t slot = 0; slot < RENDER_COLOR_ATTACHMENTS_MAX; slot++) {
+		if (sh_regs.target_output_mode[slot] != 0 &&
+		    render_target_mask_slot(write_mask, slot) != 0) {
+			output_mask |= 1u << slot;
+		}
+	}
+	return output_mask;
 }
 
 enum class CbColorMode : uint8_t {
@@ -847,7 +852,7 @@ static bool ResolvePrimitiveRestart(const CommandBuffer& buffer,
 }
 
 static void RefreshShaders(CommandBuffer& buffer, const DrawCallInfo& draw,
-                           DrawRenderState& state) {
+                           uint32_t color_output_mask, DrawRenderState& state) {
 	auto& ctx    = buffer.GetRegisters();
 	auto& sh_ctx = buffer.GetShaders();
 
@@ -861,7 +866,7 @@ static void RefreshShaders(CommandBuffer& buffer, const DrawCallInfo& draw,
 	    target_export_mapping {};
 	for (uint32_t slot = 0; slot < RENDER_COLOR_ATTACHMENTS_MAX; slot++) {
 		const auto& rt = ctx.GetRenderTarget(slot);
-		if (rt.base.addr != 0 && render_target_mask_slot(ctx.GetRenderTargetMask(), slot) != 0) {
+		if ((color_output_mask & (1u << slot)) != 0 && rt.base.addr != 0) {
 			target_export_mapping[slot] =
 			    TextureGetRenderTargetFormat(rt.info.format, rt.info.channel_type,
 			                                 rt.info.channel_order)
@@ -880,8 +885,12 @@ static void RefreshShaders(CommandBuffer& buffer, const DrawCallInfo& draw,
 bool RenderExecutor::PrepareDrawRenderState(CommandBuffer& buffer, const DrawCallInfo& draw,
                                             uint32_t            render_target_slice_offset,
 	                                        DrawRenderState& state) {
-	state.ps_active = DrawHasActivePixelShader(buffer);
-	RefreshShaders(buffer, draw, state);
+	const auto& shader_regs       = buffer.GetRegisters().GetShaderRegisters();
+	const auto  color_output_mask = DrawColorOutputMask(buffer.GetRegisters());
+	state.ps_active = buffer.GetShaders().GetPs().ps_regs.data_addr != 0 &&
+	                  (color_output_mask != 0 ||
+	                   PixelShaderHasDepthOrCoverageSideEffects(shader_regs));
+	RefreshShaders(buffer, draw, color_output_mask, state);
 	uint32_t mrt_mask = 0;
 	if (state.ps_active) {
 		for (const auto& output: state.ps_input_info.stage.program->info.outputs) {
@@ -890,6 +899,7 @@ bool RenderExecutor::PrepareDrawRenderState(CommandBuffer& buffer, const DrawCal
 			}
 		}
 	}
+	mrt_mask &= color_output_mask;
 	if (draw.IsIndexed()) {
 		LogDrawPhase(draw.Name(), "ResolveRenderColorTarget");
 	}

src/graphics/shader/recompiler/ShaderRecompiler.cpp

@@ -15,7 +15,6 @@
 #include "graphics/shader/recompiler/ir/passes/ResourceMaterialization.h"
 #include "graphics/shader/recompiler/ir/passes/ResourceTracking.h"
 #include "graphics/shader/recompiler/ir/passes/ShaderInfoCollection.h"
-#include "graphics/shader/recompiler/ir/passes/SrtWalker.h"
 #include "graphics/shader/recompiler/ir/passes/SsaRewrite.h"
 
 #include <algorithm>
@@ -548,30 +547,40 @@ TranslateResult TranslateProgram(std::span<const uint32_t> code, const CompileOp
 
 	LOGF("%s phase begin: stage=%s hash=0x%016" PRIx64 " CFG BuildGraph\n", GetDumpLabel(options),
 	     StageName(options.stage), options.shader_hash);
-	auto cfg = CFG::BuildGraph(decoded);
+	auto native_cfg = CFG::BuildGraph(decoded);
+	CFG::Graph structured_cfg;
+	auto* selected_cfg = &native_cfg;
 	LOGF("%s phase end: stage=%s hash=0x%016" PRIx64 " CFG BuildGraph blocks=%" PRIu64
 	     " loops=%" PRIu64 " back_edges=%" PRIu64 " elapsed_ms=%" PRIu64 "\n",
 	     GetDumpLabel(options), StageName(options.stage), options.shader_hash,
-	     static_cast<uint64_t>(cfg.blocks.size()), static_cast<uint64_t>(cfg.natural_loops.size()),
-	     static_cast<uint64_t>(cfg.back_edges.size()), phase_ms());
-	if (cfg.irreducible) {
-		LogDispatcherFallback(options, cfg, "build");
+	     static_cast<uint64_t>(native_cfg.blocks.size()),
+	     static_cast<uint64_t>(native_cfg.natural_loops.size()),
+	     static_cast<uint64_t>(native_cfg.back_edges.size()), phase_ms());
+	if (native_cfg.irreducible) {
+		LogDispatcherFallback(options, native_cfg, "build");
 	} else {
 		LOGF("%s phase begin: stage=%s hash=0x%016" PRIx64 " CFG Structurize\n",
 		     GetDumpLabel(options), StageName(options.stage), options.shader_hash);
-		if (!CFG::Structurize(cfg)) {
-			LogDispatcherFallback(options, cfg, "structurize");
+		structured_cfg = CFG::Structurize(native_cfg);
+		if (structured_cfg.unsupported) {
+			native_cfg.unsupported = true;
+			native_cfg.failure_kind = structured_cfg.failure_kind;
+			native_cfg.failure_block = structured_cfg.failure_block;
+			native_cfg.unsupported_reason = structured_cfg.unsupported_reason;
+			LogDispatcherFallback(options, native_cfg, "structurize");
 		} else {
+			selected_cfg = &structured_cfg;
 			LOGF("%s structured CFG success: blocks=%" PRIu64 "\n", GetDumpLabel(options),
-			     static_cast<uint64_t>(cfg.blocks.size()));
+			     static_cast<uint64_t>(selected_cfg->blocks.size()));
 		}
 		LOGF("%s phase end: stage=%s hash=0x%016" PRIx64 " CFG Structurize blocks=%" PRIu64
 		     " loops=%" PRIu64 " elapsed_ms=%" PRIu64 "\n",
 		     GetDumpLabel(options), StageName(options.stage), options.shader_hash,
-		     static_cast<uint64_t>(cfg.blocks.size()),
-		     static_cast<uint64_t>(cfg.natural_loops.size()), phase_ms());
+		     static_cast<uint64_t>(selected_cfg->blocks.size()),
+		     static_cast<uint64_t>(selected_cfg->natural_loops.size()), phase_ms());
 	}
 
+	const auto& cfg = *selected_cfg;
 	Frontend::EmbeddedFetchPlan embedded_fetch;
 	if ((options.stage == ShaderType::Vertex || options.stage == ShaderType::Local) &&
 	    options.input_info.vertex != nullptr && options.input_info.vertex->fetch_embedded) {
@@ -614,15 +623,21 @@ TranslateResult TranslateProgram(std::span<const uint32_t> code, const CompileOp
 		IR::EliminateDeadCode(ir.blocks);
 	}
 	LowerTessellationMemory(ir, options);
-	IR::BuildSrtPlan(ir);
-	IR::EliminateDeadCode(ir.blocks);
-	IR::TrackResources(ir);
+	std::string cfg_dump;
+	if (options.dump_ir) {
+		cfg_dump = CFG::GraphToString(cfg);
+		if (options.early_dump) {
+			LOGF("%s native IR before resource tracking:\n%s", GetDumpLabel(options),
+			     MakeIrDump(cfg_dump, ir).c_str());
+		}
+	}
+	IR::TrackResources(ir, decoded, native_cfg);
 	IR::EliminateDeadCode(ir.blocks);
 	TranslateResult result;
 	result.program = std::move(ir);
 	if (options.dump_ir) {
 		result.decoded_dump = std::move(decoded_dump);
-		result.cfg_dump     = CFG::GraphToString(cfg);
+		result.cfg_dump     = std::move(cfg_dump);
 	}
 	return result;
 }
@@ -634,6 +649,35 @@ CompileResult CompileProgram(TranslateResult translated, const CompileOptions& o
 	const auto emit_begin = std::chrono::steady_clock::now();
 	auto& ir = translated.program;
 	IR::ApplyResourceSpecialization(ir, specialization);
+	// The resource plan owns host descriptor evaluation now. Keep only dependencies consumed
+	// by GPU memory operations; bound descriptor dwords must not retain shader instructions.
+	for (auto& inst: ir.value_storage) {
+		inst.Invalidate();
+	}
+	for (auto* block: ir.blocks) {
+		for (auto& inst: *block) {
+			const auto op = inst.GetOpcode();
+			uint32_t first = 0;
+			if (op == IR::ValueOpcode::GetBufferResource) {
+				if (std::ranges::any_of(inst.Uses(), [&](const IR::Use& use) {
+					return ir.memory_info[use.user->Flags<IR::MemoryFlags>().index].kind ==
+					       IR::ResourceKind::IndirectBuffer;
+				})) {
+					continue;
+				}
+			} else if (op == IR::ValueOpcode::GetImageResource) {
+				const auto resource = inst.Flags<uint32_t>();
+				first = resource < ir.info.images.size() &&
+				                ir.info.images[resource].indirect_root == resource ? 1u : 0u;
+			} else if (op != IR::ValueOpcode::GetSamplerResource) {
+				continue;
+			}
+			for (size_t index = first; index < inst.NumArgs(); index++) {
+				inst.SetArg(index, IR::Value(0u));
+			}
+		}
+	}
+	ir.value_storage.clear();
 	IR::RemoveIdentities(ir.blocks);
 	IR::EliminateDeadCode(ir.blocks);
 

src/graphics/shader/recompiler/backend/spirv/SpirvEmitter.cpp

@@ -62,12 +62,7 @@ void ValidateNativeProgram(const IR::Program& program) {
 		Expect(Kind::BdaPagetable);
 		Expect(Kind::FaultBuffer);
 	}
-	const bool uses_flattened_runtime =
-	    !program.srt_reads.empty() ||
-	     std::ranges::any_of(program.info.images, [](const IR::ImageResource& image) {
-		     return image.indirect_search_iterations != 0u;
-	     });
-	if (uses_flattened_runtime) {
+	if (IR::UsesFlattenedSrt(program)) {
 		Expect(Kind::FlattenedSrt);
 	}
 	if (program.bindings.ShaderDataDwords() != 0 && !program.bindings.UsesPushData()) {

src/graphics/shader/recompiler/backend/spirv/spirvEmitterAlu.cpp

@@ -307,6 +307,11 @@ uint32_t EmitConvertF32F64(EmitterState& state, uint32_t arg0) {
 	                  Unary(state, spv::OpBitcast, TypeF32(state), clamped), converted));
 }
 
+uint32_t EmitConvertF64F32(EmitterState& state, uint32_t arg0) {
+	return EmitNative<spv::OpFConvert, IR::Type::F64>(state,
+	                                                  EmitFlushF32DenormToSignedZero(state, arg0));
+}
+
 uint32_t EmitCompositeExtractU64(EmitterState& state, uint32_t arg0, IR::Value arg1) {
 	return EmitNative<spv::OpCompositeExtract, IR::Type::U32>(state, arg0, arg1.U32());
 }

src/graphics/shader/recompiler/backend/spirv/spirvEmitterAluHelpers.cpp

@@ -203,9 +203,7 @@ uint32_t EmitFlushF32DenormToSignedZero(EmitterState& state, uint32_t value) {
 	const auto bits      = state.builder.AllocateId();
 	const auto abs_bits  = state.builder.AllocateId();
 	const auto sign_bits = state.builder.AllocateId();
-	const auto non_zero  = state.builder.AllocateId();
 	const auto subnormal = state.builder.AllocateId();
-	const auto flush     = state.builder.AllocateId();
 	const auto selected  = state.builder.AllocateId();
 	const auto ret       = state.builder.AllocateId();
 	state.builder.AddFunction(spv::OpBitcast, TypeU32(state), bits, value);
@@ -213,12 +211,9 @@ uint32_t EmitFlushF32DenormToSignedZero(EmitterState& state, uint32_t value) {
 	                          ConstantU32(state, 0x7fffffffu));
 	state.builder.AddFunction(spv::OpBitwiseAnd, TypeU32(state), sign_bits, bits,
 	                          ConstantU32(state, 0x80000000u));
-	state.builder.AddFunction(spv::OpINotEqual, TypeBool(state), non_zero, abs_bits,
-	                          ConstantU32(state, 0));
 	state.builder.AddFunction(spv::OpULessThan, TypeBool(state), subnormal, abs_bits,
 	                          ConstantU32(state, 0x00800000u));
-	state.builder.AddFunction(spv::OpLogicalAnd, TypeBool(state), flush, non_zero, subnormal);
-	state.builder.AddFunction(spv::OpSelect, TypeU32(state), selected, flush, sign_bits, bits);
+	state.builder.AddFunction(spv::OpSelect, TypeU32(state), selected, subnormal, sign_bits, bits);
 	state.builder.AddFunction(spv::OpBitcast, TypeF32(state), ret, selected);
 	return ret;
 }

src/graphics/shader/recompiler/backend/spirv/spirvEmitterFlow.cpp

@@ -636,6 +636,29 @@ uint32_t EmitDppUpdateU32(ValueEmitContext& ctx, const IR::Inst& inst) {
 	                                                ctx.Arg(inst, 1));
 }
 
+uint32_t EmitConditionRef(ValueEmitContext& ctx, const IR::Inst& inst) {
+	if (ctx.other_half == nullptr) return ctx.Arg(inst, 0);
+	// A native scalar branch makes one decision for both emulated wave halves.
+	if (ctx.half != 0) return ctx.other_half->Def(IR::Value(&inst));
+	const auto kind = inst.Flags<CFG::BranchCondition>();
+	if (kind == CFG::BranchCondition::ScalarInstruction) return ctx.Arg(inst, 0);
+	const auto ballot = ctx.Ballot(inst.Arg(0));
+	const auto low = ctx.state.builder.AllocateId();
+	const auto high = ctx.state.builder.AllocateId();
+	const auto combined = ctx.state.builder.AllocateId();
+	const auto result = ctx.state.builder.AllocateId();
+	ctx.state.builder.AddFunction(spv::OpCompositeExtract, TypeU32(ctx.state), low, ballot, 0);
+	ctx.state.builder.AddFunction(spv::OpCompositeExtract, TypeU32(ctx.state), high, ballot, 1);
+	const bool zero = kind == CFG::BranchCondition::ExecZero ||
+	                  kind == CFG::BranchCondition::VccZero || kind == CFG::BranchCondition::SccZero;
+	ctx.state.builder.AddFunction(zero ? spv::OpBitwiseAnd : spv::OpBitwiseOr,
+	                              TypeU32(ctx.state), combined, low, high);
+	ctx.state.builder.AddFunction(zero ? spv::OpIEqual : spv::OpINotEqual,
+	                              TypeBool(ctx.state), result, combined,
+	                              ConstantU32(ctx.state, zero ? ~0u : 0u));
+	return result;
+}
+
 uint32_t EmitBallot(ValueEmitContext& ctx, IR::Value predicate) {
 	return ctx.Ballot(predicate);
 }

src/graphics/shader/recompiler/backend/spirv/spirvEmitterInstructions.h

@@ -2,6 +2,8 @@
 
 #include "graphics/shader/recompiler/backend/spirv/spirvEmitterInternal.h"
 
+#include <bit>
+
 namespace Libs::Graphics::ShaderRecompiler::Spirv::Emitter {
 // Operand signatures distinguish SPIR-V IDs (uint32_t), raw literals/values
 // (IR::Value), and instructions whose metadata or operand loads must stay lazy.
@@ -29,7 +31,9 @@ uint32_t EmitConvertSigned32ToFloat(EmitterState& state, uint32_t arg0) {
 inline constexpr auto EmitConvertF32S32 = EmitConvertSigned32ToFloat<IR::Type::F32>;
 inline constexpr auto EmitConvertF64S32 = EmitConvertSigned32ToFloat<IR::Type::F64>;
 uint32_t              EmitConvertF32F64(EmitterState& state, uint32_t arg0);
+uint32_t              EmitConvertF64F32(EmitterState& state, uint32_t arg0);
 EMIT_NATIVE(ConvertF32U32, OpConvertUToF, F32, uint32_t)
+EMIT_NATIVE(ConvertF64U32, OpConvertUToF, F64, uint32_t)
 EMIT_NATIVE(CompositeConstructU64, OpCompositeConstruct, U64, uint32_t, uint32_t)
 EMIT_NATIVE(CompositeConstructU32x2, OpCompositeConstruct, U32x2, uint32_t, uint32_t)
 EMIT_NATIVE(CompositeConstructU32x3, OpCompositeConstruct, U32x3, uint32_t, uint32_t, uint32_t)
@@ -112,18 +116,34 @@ EMIT_NATIVE(LogicalOr, OpLogicalOr, U1, uint32_t, uint32_t)
 EMIT_NATIVE(LogicalAnd, OpLogicalAnd, U1, uint32_t, uint32_t)
 EMIT_NATIVE(LogicalXor, OpLogicalNotEqual, U1, uint32_t, uint32_t)
 EMIT_NATIVE(LogicalNot, OpLogicalNot, U1, uint32_t)
-EMIT_NATIVE(FPOrdEqual32, OpFOrdEqual, U1, uint32_t, uint32_t)
-EMIT_NATIVE(FPUnordEqual32, OpFUnordEqual, U1, uint32_t, uint32_t)
-EMIT_NATIVE(FPOrdNotEqual32, OpFOrdNotEqual, U1, uint32_t, uint32_t)
-EMIT_NATIVE(FPUnordNotEqual32, OpFUnordNotEqual, U1, uint32_t, uint32_t)
-EMIT_NATIVE(FPOrdLessThan32, OpFOrdLessThan, U1, uint32_t, uint32_t)
-EMIT_NATIVE(FPUnordLessThan32, OpFUnordLessThan, U1, uint32_t, uint32_t)
-EMIT_NATIVE(FPOrdGreaterThan32, OpFOrdGreaterThan, U1, uint32_t, uint32_t)
-EMIT_NATIVE(FPUnordGreaterThan32, OpFUnordGreaterThan, U1, uint32_t, uint32_t)
-EMIT_NATIVE(FPOrdLessThanEqual32, OpFOrdLessThanEqual, U1, uint32_t, uint32_t)
-EMIT_NATIVE(FPUnordLessThanEqual32, OpFUnordLessThanEqual, U1, uint32_t, uint32_t)
-EMIT_NATIVE(FPOrdGreaterThanEqual32, OpFOrdGreaterThanEqual, U1, uint32_t, uint32_t)
-EMIT_NATIVE(FPUnordGreaterThanEqual32, OpFUnordGreaterThanEqual, U1, uint32_t, uint32_t)
+template <spv::Op opcode>
+uint32_t EmitFloatCompare32(ValueEmitContext& ctx, const IR::Inst& inst) {
+	// SPIR-V's DenormFlushToZero mode does not require comparison operands to flush.
+	const bool flush = inst.Flags<IR::FPCompareFlags>().flush_input_denorms;
+	const auto operand = [&](size_t index) {
+		const auto value = inst.Arg(index);
+		if (flush && value.IsImmediate()) {
+			const auto bits = std::bit_cast<uint32_t>(value.F32Value());
+			return ConstantF32(ctx.state, (bits & 0x7fffffffu) < 0x00800000u
+			                                  ? bits & 0x80000000u : bits);
+		}
+		const auto id = ctx.Arg(inst, index);
+		return flush ? EmitFlushF32DenormToSignedZero(ctx.state, id) : id;
+	};
+	return EmitNative<opcode, IR::Type::U1>(ctx.state, operand(0), operand(1));
+}
+inline constexpr auto EmitFPOrdEqual32 = EmitFloatCompare32<spv::OpFOrdEqual>;
+inline constexpr auto EmitFPUnordEqual32 = EmitFloatCompare32<spv::OpFUnordEqual>;
+inline constexpr auto EmitFPOrdNotEqual32 = EmitFloatCompare32<spv::OpFOrdNotEqual>;
+inline constexpr auto EmitFPUnordNotEqual32 = EmitFloatCompare32<spv::OpFUnordNotEqual>;
+inline constexpr auto EmitFPOrdLessThan32 = EmitFloatCompare32<spv::OpFOrdLessThan>;
+inline constexpr auto EmitFPUnordLessThan32 = EmitFloatCompare32<spv::OpFUnordLessThan>;
+inline constexpr auto EmitFPOrdGreaterThan32 = EmitFloatCompare32<spv::OpFOrdGreaterThan>;
+inline constexpr auto EmitFPUnordGreaterThan32 = EmitFloatCompare32<spv::OpFUnordGreaterThan>;
+inline constexpr auto EmitFPOrdLessThanEqual32 = EmitFloatCompare32<spv::OpFOrdLessThanEqual>;
+inline constexpr auto EmitFPUnordLessThanEqual32 = EmitFloatCompare32<spv::OpFUnordLessThanEqual>;
+inline constexpr auto EmitFPOrdGreaterThanEqual32 = EmitFloatCompare32<spv::OpFOrdGreaterThanEqual>;
+inline constexpr auto EmitFPUnordGreaterThanEqual32 = EmitFloatCompare32<spv::OpFUnordGreaterThanEqual>;
 uint32_t              EmitFPIsNan32(EmitterState& state, uint32_t arg0);
 inline constexpr auto EmitFPCmpClass32 = EmitClassMaskF32;
 EMIT_NATIVE(FPAdd32, OpFAdd, F32, uint32_t, uint32_t)
@@ -182,6 +202,7 @@ uint32_t              EmitDppUpdateU32(ValueEmitContext& ctx, const IR::Inst& in
 uint32_t              EmitWqmU64(EmitterState& state, uint32_t value);
 uint32_t              EmitLaneId(EmitterState& state);
 uint32_t              EmitBallot(ValueEmitContext& ctx, IR::Value predicate);
+uint32_t              EmitConditionRef(ValueEmitContext& ctx, const IR::Inst& inst);
 uint32_t              EmitReadFirstLane(ValueEmitContext& ctx, const IR::Inst& inst);
 uint32_t              EmitReadLane(ValueEmitContext& ctx, const IR::Inst& inst);
 uint32_t              EmitWriteLane(ValueEmitContext& ctx, const IR::Inst& inst);

src/graphics/shader/recompiler/backend/spirv/spirvEmitterMemory.cpp

@@ -146,7 +146,12 @@ uint32_t GuestAddress(ValueEmitContext& ctx, const IR::Inst& inst, const IR::Mem
 			ctx.Fail(inst, "has no address base pair");
 			return ConstantDeviceAddress(state, 0);
 		}
-		const auto base = DeviceAddressFromWords(state, ctx.Arg(*handle, 0), ctx.Arg(*handle, 1));
+		auto base_low = ctx.Arg(*handle, 0);
+		if (mem.kind == IR::ResourceKind::ScalarAddress) {
+			base_low = Binary(state, spv::OpBitwiseAnd, TypeU32(state), base_low,
+			                  ConstantU32(state, ~3u));
+		}
+		const auto base = DeviceAddressFromWords(state, base_low, ctx.Arg(*handle, 1));
 		address         = Binary(state, spv::OpIAdd, TypeScalarU64(state), base,
 		                         Unary(state, spv::OpUConvert, TypeScalarU64(state), low));
 	}
@@ -970,7 +975,14 @@ void DefineGetBdaPointer(EmitterState& state) {
 	state.builder.AddFunction(spv::OpFunctionParameter, type, address);
 	EmitLabel(state, entry_label);
 
-	const auto page64        = Binary(state, spv::OpShiftRightLogical, type, address,
+	const auto extended = Binary(state, spv::OpUGreaterThanEqual, TypeBool(state), address,
+	                             ConstantDeviceAddress(state, LibKernel::Memory::kExtendedMemoryBase));
+	const auto packed = Select(state, type, extended,
+	                           Binary(state, spv::OpISub, type, address,
+	                                  ConstantDeviceAddress(state,
+	                                      LibKernel::Memory::kExtendedMemoryBase - LOWER_ADDRESS_SIZE)),
+	                           address);
+	const auto page64        = Binary(state, spv::OpShiftRightLogical, type, packed,
 	                                  ConstantDeviceAddress(state, BufferCache::CACHING_PAGEBITS));
 	const auto page          = Unary(state, spv::OpUConvert, TypeU32(state), page64);
 	const auto entry_pointer = state.builder.AllocateId();
@@ -1099,16 +1111,27 @@ uint32_t EmitAppendConsume(ValueEmitContext& ctx, const IR::Inst& inst) {
 	if (ctx.half == 1) {
 		return ctx.other_half->Def(IR::Value(const_cast<IR::Inst*>(&inst)));
 	}
-	const auto m0 = ctx.Arg(inst, 0);
-	const auto base =
-	    Binary(state, spv::OpShiftRightLogical, TypeU32(state), m0, ConstantU32(state, 16));
-	const auto size =
-	    Binary(state, spv::OpBitwiseAnd, TypeU32(state), m0, ConstantU32(state, 0xffffu));
-	const auto address = Binary(state, spv::OpIAdd, TypeU32(state), base,
-	                            ConstantU32(state, ctx.Memory(inst).offset));
+	const auto mem           = ctx.Memory(inst);
+	const auto offset        = mem.offset & 0xfffcu;
+	auto       address       = ConstantU32(state, offset);
+	uint32_t   region_bounds = 0;
+	if (mem.kind == IR::ResourceKind::Gds) {
+		const auto m0 = ctx.Arg(inst, 0);
+		const auto base =
+		    Binary(state, spv::OpShiftRightLogical, TypeU32(state), m0, ConstantU32(state, 16));
+		const auto size =
+		    Binary(state, spv::OpBitwiseAnd, TypeU32(state), m0, ConstantU32(state, 0xffffu));
+		address = Binary(state, spv::OpIAdd, TypeU32(state), base, address);
+		// The entire M0 region must fit the PS5's 48 KiB GDS partition.
+		region_bounds = AndCondition(
+		    state, Binary(state, spv::OpULessThan, TypeBool(state),
+		                  ConstantU32(state, offset + 3u), size),
+		    Binary(state, spv::OpULessThanEqual, TypeBool(state),
+		           Binary(state, spv::OpIAdd, TypeU32(state), base, size),
+		           ConstantU32(state, 0xc000u)));
+	}
 	const auto raw_index =
 	    Binary(state, spv::OpShiftRightLogical, TypeU32(state), address, ConstantU32(state, 2));
-	const auto mem    = ctx.Memory(inst);
 	const auto access = PrepareMemoryResourceAccess(state, mem);
 	const auto index  = EmitMemoryElementIndex(state, access, raw_index);
 	const auto exec   = ctx.Arg(inst, 1);
@@ -1127,19 +1150,17 @@ uint32_t EmitAppendConsume(ValueEmitContext& ctx, const IR::Inst& inst) {
 	        : first;
 	const auto is_first       = Binary(state, spv::OpIEqual, TypeBool(state),
 	                                   EmitSubgroupLocalInvocationId(state), source_lane);
-	const auto storage_bounds = EmitMemoryElementInBounds(state, access, index);
-	const auto m0_bounds =
-	    mem.kind == IR::ResourceKind::Gds
-	        ? Binary(state, spv::OpINotEqual, TypeBool(state), size, ConstantU32(state, 0))
-	        : Binary(state, spv::OpULessThan, TypeBool(state),
-	                 ConstantU32(state, ctx.Memory(inst).offset + 3u), size);
+	auto bounds = EmitMemoryElementInBounds(state, access, index);
+	if (mem.kind == IR::ResourceKind::Gds) {
+		bounds = AndCondition(state, bounds, region_bounds);
+	}
 	const auto condition = AndCondition(
 	    state, is_first,
 	    AndCondition(state,
 	                 state.lane_count == 2 ? Binary(state, spv::OpINotEqual, TypeBool(state), count,
 	                                                ConstantU32(state, 0))
 	                                       : exec,
-	                 AndCondition(state, storage_bounds, m0_bounds)));
+	                 bounds));
 	const auto atomic = EmitValueOrZeroIfCondition(state, condition, [&]() {
 		const auto value = state.builder.AllocateId();
 		state.builder.AddFunction(append ? spv::OpAtomicIAdd : spv::OpAtomicISub, TypeU32(state),
@@ -1173,10 +1194,11 @@ void EmitReadConstBuffer(ValueEmitContext& ctx, const IR::Inst& inst) {
 	if (mem.planning_only) return;
 	auto& state        = ctx.state;
 	mem.kind           = IR::ResourceKind::ScalarBuffer;
-	const auto address = Binary(state, spv::OpIAdd, TypeU32(state), ctx.Arg(inst, 1),
-	                            ConstantU32(state, mem.offset));
-	const auto index =
-	    Binary(state, spv::OpShiftRightLogical, TypeU32(state), address, ConstantU32(state, 2));
+	auto index = Binary(state, spv::OpShiftRightLogical, TypeU32(state), ctx.Arg(inst, 1),
+	                    ConstantU32(state, 2));
+	if (mem.offset >= 4u) {
+		index = Binary(state, spv::OpIAdd, TypeU32(state), index, ConstantU32(state, mem.offset >> 2u));
+	}
 	const auto access    = PrepareMemoryResourceAccess(state, mem);
 	const auto element   = EmitMemoryElementIndex(state, access, index);
 	const auto condition = EmitMemoryElementInBounds(state, access, element);
@@ -1198,6 +1220,8 @@ void EmitLoadMemory(ValueEmitContext& ctx, const IR::Inst& inst) {
 		value = LoadWideBuffer(ctx, inst, buffer_components);
 	else if (shared_components > 1u)
 		value = LoadWideShared(ctx, inst, shared_components);
+	else if (mem.kind == IR::ResourceKind::ScalarAddress)
+		value = LoadBdaDword(ctx, GuestAddress(ctx, inst, mem));
 	else if (address_info.access == IR::AddressAccess::Read &&
 	         mem.kind != IR::ResourceKind::Scratch)
 		value = LoadBda(ctx, GuestAddress(ctx, inst, mem), ctx.Arg(inst, inst.NumArgs() - 1),

src/graphics/shader/recompiler/backend/spirv/spirvEmitterModule.cpp

@@ -684,8 +684,8 @@ void DefineModule(EmitterState& state) {
 	state.builder.AddExecutionMode(state.main_func, spv::ExecutionModeSignedZeroInfNanPreserve,
 	                               32u);
 	if (state.requirements.float64) {
-		EXIT_NOT_IMPLEMENTED(state.program.stage != ShaderType::Compute);
-		EXIT_NOT_IMPLEMENTED(state.input_info.compute->float_mode != 0xc0);
+		EXIT_NOT_IMPLEMENTED(state.program.stage == ShaderType::Compute &&
+		                     state.input_info.compute->float_mode != 0xc0);
 		// MODE=0xc0 uses round-to-nearest-even and preserves FP64 input/output denormals.
 		state.builder.RequireCapability(spv::CapabilityFloat64);
 		state.builder.RequireCapability(spv::CapabilityRoundingModeRTE);

src/graphics/shader/recompiler/backend/spirv/spirvEmitterProgram.cpp

@@ -97,30 +97,6 @@ void EmitReturn(ValueEmitContext& ctx) {
 	ctx.state.builder.AddFunction(spv::OpReturn);
 }
 
-uint32_t BranchCondition(ValueEmitContext& ctx, const IR::BlockInfo& info) {
-	// Scalar-instruction conditions already test the full wave's raw register values.
-	if (ctx.other_half == nullptr ||
-	    info.terminator.condition == CFG::BranchCondition::ScalarInstruction ||
-	    info.terminator.condition == CFG::BranchCondition::GotoVariable) {
-		return ctx.Def(info.condition);
-	}
-	const auto ballot = ctx.Ballot(info.condition);
-	const auto low    = ctx.state.builder.AllocateId();
-	const auto high   = ctx.state.builder.AllocateId();
-	const auto result = ctx.state.builder.AllocateId();
-	ctx.state.builder.AddFunction(spv::OpCompositeExtract, TypeU32(ctx.state), low, ballot, 0);
-	ctx.state.builder.AddFunction(spv::OpCompositeExtract, TypeU32(ctx.state), high, ballot, 1);
-	const auto kind     = info.terminator.condition;
-	const bool zero     = kind == CFG::BranchCondition::ExecZero ||
-	                      kind == CFG::BranchCondition::VccZero ||
-	                      kind == CFG::BranchCondition::SccZero;
-	const auto combined =
-	    EmitBinaryU32(ctx.state, zero ? spv::OpBitwiseAnd : spv::OpBitwiseOr, low, high);
-	ctx.state.builder.AddFunction(zero ? spv::OpIEqual : spv::OpINotEqual, TypeBool(ctx.state),
-	                              result, combined, ConstantU32(ctx.state, zero ? ~0u : 0u));
-	return result;
-}
-
 void EmitStructuredTerminator(ValueEmitContext& ctx, const IR::Block* block,
                               const IR::BlockInfo& info) {
 	const auto& program = ctx.state.program;
@@ -160,7 +136,7 @@ void EmitStructuredTerminator(ValueEmitContext& ctx, const IR::Block* block,
 				EmitReturn(ctx);
 				return;
 			}
-			const auto condition = BranchCondition(ctx, info);
+			const auto condition = ctx.Def(info.condition);
 			emit_merge();
 			ctx.state.builder.AddFunction(spv::OpBranchConditional, condition,
 			                              ctx.Label(true_block), ctx.Label(false_block));
@@ -193,7 +169,7 @@ uint32_t EmitDispatcherNextPc(ValueEmitContext& ctx, const DispatcherFunctionSta
 			EmitDispatcherTarget(ctx, dispatcher, block, term.false_block);
 			const auto selected = ctx.state.builder.AllocateId();
 			ctx.state.builder.AddFunction(
-			    spv::OpSelect, TypeU32(ctx.state), selected, BranchCondition(ctx, info),
+			    spv::OpSelect, TypeU32(ctx.state), selected, ctx.Def(info.condition),
 			    ConstantU32(ctx.state, term.true_block), ConstantU32(ctx.state, term.false_block));
 			return selected;
 		}

src/graphics/shader/recompiler/frontend/cfg/ShaderCFG.h

@@ -18,10 +18,17 @@ enum class BranchCondition {
 	ExecZero,
 	ExecNonZero,
 	ScalarInstruction,
-	GotoVariable,
+	Expression,
 	Unknown
 };
 
+struct ConditionExpression {
+	enum class Op { Constant, Variable, Native, Not, Or };
+	Op       op;
+	uint32_t lhs;
+	uint32_t rhs = UINT32_MAX;
+};
+
 enum class TerminatorKind { Branch, ConditionalBranch, IndirectBranch, Return, Unsupported };
 
 enum class FailureKind {
@@ -48,8 +55,7 @@ struct Terminator {
 	std::vector<uint32_t> indirect_targets;
 	std::vector<uint32_t> indirect_selector_values;
 	std::vector<uint32_t> indirect_selector_targets;
-	uint32_t              goto_variable = UINT32_MAX;
-	int32_t               goto_value    = -1;
+	uint32_t              expression    = UINT32_MAX;
 	bool                  loop_header   = false;
 };
 
@@ -59,10 +65,11 @@ struct BasicBlock {
 	uint32_t              end_pc     = 0;
 	uint32_t              inst_begin = 0;
 	uint32_t              inst_end   = 0;
+	struct Assignment { uint32_t variable; uint32_t expression; };
+	std::vector<Assignment> assignments;
 	std::vector<uint32_t> predecessors;
 	std::vector<uint32_t> successors;
 	std::vector<uint32_t> dominators;
-	std::vector<uint32_t> post_dominators;
 	Terminator            terminator;
 };
 
@@ -73,12 +80,9 @@ struct BackEdge {
 };
 
 struct NaturalLoop {
-	uint32_t              header         = UINT32_MAX;
-	uint32_t              latch          = UINT32_MAX;
-	uint32_t              merge          = UINT32_MAX;
-	uint32_t              continue_block = UINT32_MAX;
+	uint32_t              header = UINT32_MAX;
+	uint32_t              latch  = UINT32_MAX;
 	std::vector<uint32_t> body_blocks;
-	std::vector<uint32_t> exit_blocks;
 };
 
 struct StronglyConnectedComponent {
@@ -89,6 +93,7 @@ struct StronglyConnectedComponent {
 
 struct Graph {
 	std::vector<BasicBlock>                 blocks;
+	std::vector<ConditionExpression>        expressions;
 	std::vector<BackEdge>                   back_edges;
 	std::vector<NaturalLoop>                natural_loops;
 	std::vector<StronglyConnectedComponent> components;
@@ -105,14 +110,12 @@ struct Graph {
 	const BasicBlock* FindBlockByPc(uint32_t pc) const;
 	BasicBlock*       FindBlockByPc(uint32_t pc);
 	bool              Dominates(uint32_t dominator, uint32_t block) const;
-	bool              PostDominates(uint32_t post_dominator, uint32_t block) const;
-	uint32_t          FindNearestCommonPostDominator(uint32_t block_a, uint32_t block_b) const;
 };
 
 Graph       BuildGraph(const Decoder::Program& program);
-// Commits structured control flow on success; preserves the original graph with
-// failure diagnostics on failure. failure_block is an original block ID or UINT32_MAX.
-bool        Structurize(Graph& graph);
+// Returns structured control flow or failure diagnostics without changing the native graph.
+// On failure, failure_block is an original block ID or UINT32_MAX.
+Graph       Structurize(const Graph& graph);
 std::string BranchConditionToString(BranchCondition condition);
 std::string FailureKindToString(FailureKind kind);
 std::string GraphToString(const Graph& graph);

src/graphics/shader/recompiler/frontend/decode/ShaderDecoder.h

@@ -156,6 +156,8 @@ enum class Opcode {
 	V_DOT2C_F32_F16,
 	V_CVT_F64_I32,
 	V_CVT_F32_F64,
+	V_CVT_F64_F32,
+	V_CVT_F64_U32,
 	V_CVT_F32_I32,
 	V_CVT_F32_U32,
 	V_CVT_U32_F32,

src/graphics/shader/recompiler/frontend/decode/VectorAluOps.cpp

@@ -89,6 +89,8 @@ constexpr OpcodeMap VOP1_OPCODE_LIST[] = {
     {0x02u, Opcode::V_READFIRSTLANE_B32},
     {0x04u, Opcode::V_CVT_F64_I32},
     {0x0fu, Opcode::V_CVT_F32_F64},
+    {0x10u, Opcode::V_CVT_F64_F32},
+    {0x16u, Opcode::V_CVT_F64_U32},
     {0x2fu, Opcode::V_RCP_F64},
     {0x05u, Opcode::V_CVT_F32_I32},
     {0x06u, Opcode::V_CVT_F32_U32},
@@ -151,6 +153,8 @@ constexpr OpcodeMap VOP3_ENCODED_VOP1_OPCODE_LIST[] = {
     {0x02u, Opcode::V_READFIRSTLANE_B32},
     {0x04u, Opcode::V_CVT_F64_I32},
     {0x0fu, Opcode::V_CVT_F32_F64},
+    {0x10u, Opcode::V_CVT_F64_F32},
+    {0x16u, Opcode::V_CVT_F64_U32},
     {0x2fu, Opcode::V_RCP_F64},
     {0x05u, Opcode::V_CVT_F32_I32},
     {0x06u, Opcode::V_CVT_F32_U32},
@@ -461,6 +465,7 @@ bool IsVopcCompareExec(Opcode opcode);
 bool IsVop1FloatSourceOpcode(Opcode opcode) {
 	switch (opcode) {
 		case Opcode::V_CVT_F32_F64:
+		case Opcode::V_CVT_F64_F32:
 		case Opcode::V_RCP_F64:
 		case Opcode::V_MOV_B32:
 		case Opcode::V_CVT_F32_F16:
@@ -721,6 +726,7 @@ void ApplyDppModifier(Operand& operand, uint32_t modifier, uint32_t encoding) {
 void DecodeVop1Dpp(uint32_t pc, std::span<const uint32_t> code, uint32_t word_index,
                    uint32_t opcode, uint32_t vdst, Instruction& inst) {
 	if (inst.opcode == Opcode::V_CVT_F64_I32 || inst.opcode == Opcode::V_CVT_F32_F64 ||
+	    inst.opcode == Opcode::V_CVT_F64_F32 || inst.opcode == Opcode::V_CVT_F64_U32 ||
 	    inst.opcode == Opcode::V_RCP_F64) {
 		SetUnsupported(inst, Family::VOP1, opcode, "FP64 instructions do not support DPP");
 		return;

src/graphics/shader/recompiler/frontend/translate/Compare.cpp

@@ -44,7 +44,8 @@ void Translator::EmitFloatCompare(const Decoder::Instruction& inst, IR::ValueOpc
 	    half ? IR::Value(ReadF16AsF32(inst.src0)) : ReadOperand(inst.src0, IR::Type::F32);
 	const auto rhs =
 	    half ? IR::Value(ReadF16AsF32(inst.src1)) : ReadOperand(inst.src1, IR::Type::F32);
-	EmitCompareResult(inst, IR::U1(ir.Emit(opcode, {lhs, rhs})), false, cmpx);
+	const IR::FPCompareFlags flags{.flush_input_denorms = !half && flush_f32_inputs};
+	EmitCompareResult(inst, IR::U1(ir.Emit(opcode, {lhs, rhs}, flags)), false, cmpx);
 }
 
 void Translator::EmitFloatOrderedCompare(const Decoder::Instruction& inst, bool ordered, bool cmpx) {

src/graphics/shader/recompiler/frontend/translate/Memory.cpp

@@ -235,6 +235,7 @@ Decoder::Operand MemorySourceAt(const Decoder::Instruction& decoded, uint32_t in
 				return index == 0u ? decoded.src0 : MakeImmediate(decoded.offset & 0xffffu);
 			case Decoder::Opcode::DS_CONSUME:
 			case Decoder::Opcode::DS_APPEND:
+				return decoded.gds ? MakeM0Operand() : MakeImmediate(0);
 			case Decoder::Opcode::DS_READ_ADDTID_B32: return MakeM0Operand();
 			case Decoder::Opcode::DS_WRITE_ADDTID_B32:
 				return index == 0u ? decoded.src1 : MakeM0Operand();
@@ -845,8 +846,7 @@ void Translator::DS_MINMAX_F32(const Decoder::Instruction& inst, IR::ValueOpcode
 void Translator::DS_APPEND_CONSUME(const Decoder::Instruction& inst, IR::ValueOpcode opcode) {
 	const auto memory = MemoryInfoFromDecoded(inst);
 	WriteOperand(inst.dst, ir.Emit(opcode,
-	                               {ReadU32(MemorySourceAt(inst, 0)), ir.GetExec(), ir.GetExecLo(),
-	                                ir.GetExecHi()},
+	                               {ReadU32(MemorySourceAt(inst, 0)), ir.GetExec()},
 	                               AddMemoryInfo(memory, inst.pc)));
 }
 

src/graphics/shader/recompiler/frontend/translate/Translate.cpp

@@ -317,6 +317,7 @@ void Translator::WriteRawU32(const Decoder::Operand& operand, IR::U32 value) {
 		case Decoder::OperandKind::Sgpr: {
 			const auto reg = static_cast<IR::ScalarReg>(operand.reg);
 			ir.SetScalarReg(reg, value);
+			program.scalar_writes.push_back({current_pc, reg});
 			ir.SetScalarMaskTag(reg, IR::U1(IR::Value(false)));
 			if (IR::RegIndex(reg) > 0u) {
 				ir.SetScalarMaskTag(static_cast<IR::ScalarReg>(IR::RegIndex(reg) - 1u),
@@ -699,11 +700,13 @@ std::array<IR::U32, 2> Translator::WriteMask(const Decoder::Operand& operand, IR
 				                    IR::U1(IR::Value(false)));
 			}
 			ir.SetScalarReg(reg, mask[0]);
+			program.scalar_writes.push_back({current_pc, reg});
 			// A wave32 VALU mask destination must not overwrite the neighboring SGPR.
 			if ((write_64 || program.wave_size == 64u) &&
 			    IR::RegIndex(reg) + 1u < IR::NumScalarRegs) {
 				const auto high = static_cast<IR::ScalarReg>(IR::RegIndex(reg) + 1u);
 				ir.SetScalarReg(high, mask[1]);
+				program.scalar_writes.push_back({current_pc, high});
 				ir.SetThreadBitScalarReg(high, IR::U1(IR::Value(false)));
 				ir.SetScalarMaskTag(high, IR::U1(IR::Value(false)));
 			}
@@ -746,55 +749,61 @@ void Translator::WriteCompareResult(const Decoder::Operand& operand, IR::U1 valu
 	WriteMask(operand, ir.LogicalAnd(ir.GetExec(), value));
 }
 
-void Translator::AddBranchCondition(const CFG::BasicBlock& source, IR::BlockInfo& info) {
-	if (source.terminator.goto_value >= 0) {
-		if (source.terminator.goto_variable == UINT32_MAX) {
-			EXIT("block %u sets an invalid goto variable", source.id);
+void Translator::AddBranchCondition(const CFG::Graph& graph, const CFG::BasicBlock& source,
+                                    IR::BlockInfo& info) {
+	const auto native_condition = [&](CFG::BranchCondition kind) -> IR::U1 {
+		IR::U1 condition;
+		switch (kind) {
+			case CFG::BranchCondition::Always: condition = IR::U1(IR::Value(true)); break;
+			case CFG::BranchCondition::SccZero: condition = ir.LogicalNot(ir.GetScc()); break;
+			case CFG::BranchCondition::SccNonZero: condition = ir.GetScc(); break;
+			case CFG::BranchCondition::VccZero: condition = ir.LogicalNot(ir.GetVcc()); break;
+			case CFG::BranchCondition::VccNonZero: condition = ir.GetVcc(); break;
+			case CFG::BranchCondition::ExecZero: condition = ir.LogicalNot(ir.GetExec()); break;
+			case CFG::BranchCondition::ExecNonZero: condition = ir.GetExec(); break;
+			case CFG::BranchCondition::ScalarInstruction:
+				EXIT_IF(instruction_branch_condition.IsEmpty());
+				condition = instruction_branch_condition;
+				break;
+			default: EXIT("block %u has an invalid native branch condition", source.id);
 		}
-		ir.SetGotoVariable(source.terminator.goto_variable,
-		                   IR::U1(IR::Value(source.terminator.goto_value != 0)));
+		return IR::U1(ir.Emit(IR::ValueOpcode::ConditionRef, {condition}, kind));
+	};
+	const auto expression = [&](auto&& self, uint32_t index) -> IR::U1 {
+		const auto& value = graph.expressions.at(index);
+		switch (value.op) {
+			case CFG::ConditionExpression::Op::Constant: return IR::U1(IR::Value(value.lhs != 0));
+			case CFG::ConditionExpression::Op::Variable: return ir.GetGotoVariable(value.lhs);
+			case CFG::ConditionExpression::Op::Native:
+				return native_condition(static_cast<CFG::BranchCondition>(value.lhs));
+			case CFG::ConditionExpression::Op::Not: return ir.LogicalNot(self(self, value.lhs));
+			case CFG::ConditionExpression::Op::Or:
+				return ir.LogicalOr(self(self, value.lhs), self(self, value.rhs));
+		}
+		EXIT("invalid CFG condition expression");
+	};
+	for (const auto& assignment: source.assignments) {
+		ir.SetGotoVariable(assignment.variable, expression(expression, assignment.expression));
 	}
-	if (source.terminator.kind == CFG::TerminatorKind::IndirectBranch) {
-		if (source.terminator.indirect_selector_code != UINT32_MAX) {
-			info.indirect_target = ReadScalarCode(source.terminator.indirect_selector_code);
-		} else if (source.terminator.indirect_pc_sgpr != UINT32_MAX) {
-			info.indirect_target =
-			    ir.GetScalarReg(static_cast<IR::ScalarReg>(source.terminator.indirect_pc_sgpr));
+	const auto& term = source.terminator;
+	if (term.kind == CFG::TerminatorKind::IndirectBranch) {
+		if (term.indirect_selector_code != UINT32_MAX) {
+			info.indirect_target = ReadScalarCode(term.indirect_selector_code);
+		} else if (term.indirect_pc_sgpr != UINT32_MAX) {
+			info.indirect_target = ir.GetScalarReg(static_cast<IR::ScalarReg>(term.indirect_pc_sgpr));
 		} else {
 			EXIT("block %u has no indirect branch selector", source.id);
 		}
 		ir.Emit(IR::ValueOpcode::ReferenceU32, {info.indirect_target});
 		return;
 	}
-	if (source.terminator.kind != CFG::TerminatorKind::ConditionalBranch) {
-		return;
+	if (term.kind == CFG::TerminatorKind::ConditionalBranch) {
+		const auto condition = term.expression != UINT32_MAX
+		                           ? expression(expression, term.expression)
+		                           : native_condition(term.condition);
+		info.condition = condition;
+		ir.Emit(IR::ValueOpcode::Reference, {condition});
 	}
-	// EXEC and VCC are invocation-local Boolean masks. Branching on that Boolean lets inactive
-	// invocations leave the region without reconstructing a host-subgroup mask.
-	IR::U1 condition;
-	switch (source.terminator.condition) {
-		case CFG::BranchCondition::Always: condition = IR::U1(IR::Value(true)); break;
-		case CFG::BranchCondition::SccZero: condition = ir.LogicalNot(ir.GetScc()); break;
-		case CFG::BranchCondition::SccNonZero: condition = ir.GetScc(); break;
-		case CFG::BranchCondition::VccZero: condition = ir.LogicalNot(ir.GetVcc()); break;
-		case CFG::BranchCondition::VccNonZero: condition = ir.GetVcc(); break;
-		case CFG::BranchCondition::ExecZero: condition = ir.LogicalNot(ir.GetExec()); break;
-		case CFG::BranchCondition::ExecNonZero: condition = ir.GetExec(); break;
-		case CFG::BranchCondition::ScalarInstruction:
-			EXIT_IF(instruction_branch_condition.IsEmpty());
-			condition = instruction_branch_condition;
-			break;
-		case CFG::BranchCondition::GotoVariable:
-			if (source.terminator.goto_variable == UINT32_MAX) {
-				EXIT("block %u reads an invalid goto variable", source.id);
-			}
-			condition = ir.GetGotoVariable(source.terminator.goto_variable);
-			break;
-		case CFG::BranchCondition::Unknown:
-			EXIT("block %u has an unknown branch condition", source.id);
-	}
-	info.condition = condition;
-	ir.Emit(IR::ValueOpcode::Reference, {condition});
 }
 
 namespace {
@@ -867,7 +876,9 @@ void IncludeInstructionVectorRegisters(const Decoder::Instruction& inst, uint32_
 	include_vector(inst.src2);
 	include_vector(inst.src3);
 	switch (inst.opcode) {
-		case Decoder::Opcode::V_CVT_F64_I32: include_vector(inst.dst, 2u); break;
+		case Decoder::Opcode::V_CVT_F64_I32:
+		case Decoder::Opcode::V_CVT_F64_F32:
+		case Decoder::Opcode::V_CVT_F64_U32: include_vector(inst.dst, 2u); break;
 		case Decoder::Opcode::V_FMA_F64: include_vector(inst.src2, 2u); [[fallthrough]];
 		case Decoder::Opcode::V_MUL_F64: include_vector(inst.src1, 2u); [[fallthrough]];
 		case Decoder::Opcode::V_RCP_F64: include_vector(inst.dst, 2u); [[fallthrough]];
@@ -1272,9 +1283,11 @@ IR::Program TranslateProgram(const Decoder::Program& decoded, const CFG::Graph&
 			                      builtin(IR::StageInputKind::InstanceIndex));
 		}
 	}
+	const bool flush_f32_inputs = options.stage == ShaderType::Compute &&
+	                             (options.input_info.compute->float_mode & 0x10u) == 0;
 	for (const auto& cfg_block: cfg.blocks) {
 		const auto typed_index = block_indices.at(cfg_block.id);
-		Translator translator(result, result.blocks[typed_index], vector_limit);
+		Translator translator(result, result.blocks[typed_index], vector_limit, flush_f32_inputs);
 		for (uint32_t index = cfg_block.inst_begin; index < cfg_block.inst_end; index++) {
 			const auto& instruction = decoded.instructions[index];
 			if (IsCodeTableLoad(cfg, instruction.pc)) {
@@ -1297,7 +1310,7 @@ IR::Program TranslateProgram(const Decoder::Program& decoded, const CFG::Graph&
 			}
 			translator.TranslateInstruction(instruction);
 		}
-		translator.AddBranchCondition(cfg_block, result.block_info[typed_index]);
+		translator.AddBranchCondition(cfg, cfg_block, result.block_info[typed_index]);
 	}
 	IR::ValidateProgram(result, false);
 	return result;

src/graphics/shader/recompiler/frontend/translate/Translator.h

@@ -9,13 +9,15 @@ namespace Libs::Graphics::ShaderRecompiler::Frontend {
 
 class Translator {
 public:
-	Translator(IR::Program& program, IR::Block* block, uint32_t vector_limit)
-	    : program(program), ir(block), current_vector_limit(vector_limit) {}
+	Translator(IR::Program& program, IR::Block* block, uint32_t vector_limit,
+	           bool flush_f32_inputs)
+	    : program(program), ir(block), current_vector_limit(vector_limit),
+	      flush_f32_inputs(flush_f32_inputs) {}
 
 	void TranslateInstruction(const Decoder::Instruction& inst);
 	void TranslateEmbeddedFetch(const Decoder::Instruction& inst, uint32_t attribute,
 	                            uint32_t component_count, const ShaderBufferResource& resource);
-	void AddBranchCondition(const CFG::BasicBlock& source, IR::BlockInfo& info);
+	void AddBranchCondition(const CFG::Graph& graph, const CFG::BasicBlock& source, IR::BlockInfo& info);
 
 private:
 	const Decoder::Operand& SourceAt(const Decoder::Instruction& inst, uint32_t index);
@@ -261,6 +263,7 @@ private:
 	Decoder::Opcode current_opcode       = Decoder::Opcode::UNKNOWN;
 	uint32_t        current_pc           = 0;
 	uint32_t        current_vector_limit = 1;
+	bool            flush_f32_inputs;
 };
 
 } // namespace Libs::Graphics::ShaderRecompiler::Frontend

src/graphics/shader/recompiler/frontend/translate/Vector.cpp

@@ -299,6 +299,8 @@ void Translator::EmitVector(const Decoder::Instruction& inst) {
 		case O::V_CVT_F32_I32: V_CVT_F32_I32(inst); return;
 		case O::V_CVT_F64_I32: return FloatUnary(inst, IR::ValueOpcode::ConvertF64S32);
 		case O::V_CVT_F32_F64: return FloatUnary(inst, IR::ValueOpcode::ConvertF32F64);
+		case O::V_CVT_F64_F32: return FloatUnary(inst, IR::ValueOpcode::ConvertF64F32);
+		case O::V_CVT_F64_U32: return FloatUnary(inst, IR::ValueOpcode::ConvertF64U32);
 		case O::V_CVT_U32_F32: V_CVT_U32_F32(inst); return;
 		case O::V_CVT_I32_F32: V_CVT_I32_F32(inst); return;
 		case O::V_CVT_F16_F32: V_CVT_F16_F32(inst); return;

src/graphics/shader/recompiler/ir/Program.cpp

@@ -81,7 +81,8 @@ bool EquivalentValue(const ResourcePlan& program, Value left, Value right,
 		    program.memory_info[li] != program.memory_info[ri]) {
 			return false;
 		}
-	} else if (lhs->Flags<uint64_t>() != rhs->Flags<uint64_t>()) {
+	} else if (lhs->GetOpcode() != ValueOpcode::ReadConst &&
+	           lhs->Flags<uint64_t>() != rhs->Flags<uint64_t>()) {
 		return false;
 	}
 	for (size_t index = 0; index < lhs->NumArgs(); index++) {

src/graphics/shader/recompiler/ir/ShaderIR.h

@@ -147,6 +147,7 @@ struct SamplerResource {
 	uint32_t first_use_pc          = 0;
 	bool     force_point_filtering = false;
 	bool     depth_compare         = false;
+	bool     integer_border        = false;
 
 	bool operator==(const SamplerResource& other) const = default;
 };
@@ -548,7 +549,7 @@ struct ResourcePlan {
 	std::vector<SrtRead>                srt_reads;
 	std::vector<uint8_t>                clean_flat_slots;
 	bool                                requires_specialization_memory = false;
-	bool                                has_address_writes = false;
+	bool                                capture_specialization_reads = false;
 	bool                                srt_plan_complete          = false;
 	bool                                resource_tracking_complete = false;
 	ShaderInfo                          info;
@@ -582,10 +583,12 @@ struct Program: ResourcePlan {
 	CFG::FailureKind              cfg_failure_kind    = CFG::FailureKind::None;
 	std::string                   fallback_reason;
 	std::vector<BlockInfo>        block_info;
+	struct ScalarWrite { uint32_t pc; ScalarReg reg; };
+	std::vector<ScalarWrite>      scalar_writes;
 	// Typed memory and export instructions reference shader-local metadata by dense index.
 	// Decoder-only details (such as NSA register numbers) have already become IR operands.
 	std::vector<ExportInfo>       export_info;
-	std::vector<Value>            dynamic_reads;
+	bool                          has_address_writes = false;
 	bool                          shader_info_complete = false;
 	BindingLayout                 bindings;
 	bool                          binding_layout_complete = false;

src/graphics/shader/recompiler/ir/opcodes/ValueOpcodes.h

@@ -52,6 +52,10 @@ struct PermlaneFlags {
 static_assert(sizeof(PermlaneFlags) <= sizeof(uint64_t));
 static_assert(std::is_trivially_copyable_v<PermlaneFlags>);
 
+struct FPCompareFlags {
+	bool flush_input_denorms = false;
+};
+
 struct MemoryFlags {
 	uint32_t index = 0;
 	uint32_t pc    = 0;

src/graphics/shader/recompiler/ir/opcodes/ValueOpcodes.inc

@@ -4,6 +4,7 @@ VALUE_OPCODE(Identity, Opaque, Opaque)
 VALUE_OPCODE(Void, Void)
 VALUE_OPCODE(Reference, Void, U1)
 VALUE_OPCODE(ReferenceU32, Void, U32)
+VALUE_OPCODE(ConditionRef, U1, U1)
 
 // Translation and register-state pseudo operations.
 VALUE_OPCODE(GetUserData, U32, ScalarReg)
@@ -65,7 +66,9 @@ VALUE_OPCODE(ConvertU32F32, U32, F32)
 VALUE_OPCODE(ConvertF32S32, F32, U32)
 VALUE_OPCODE(ConvertF64S32, F64, U32)
 VALUE_OPCODE(ConvertF32F64, F32, F64)
+VALUE_OPCODE(ConvertF64F32, F64, F32)
 VALUE_OPCODE(ConvertF32U32, F32, U32)
+VALUE_OPCODE(ConvertF64U32, F64, U32)
 VALUE_OPCODE(CompositeConstructU64, U64, U32, U32)
 VALUE_OPCODE(CompositeConstructU32x2, U32x2, U32, U32)
 VALUE_OPCODE(CompositeConstructU32x3, U32x3, U32, U32, U32)
@@ -274,8 +277,8 @@ VALUE_OPCODE(SharedAtomicUMax32, U32, U32, U32, U1)
 VALUE_OPCODE(SharedAtomicAnd32, U32, U32, U32, U1)
 VALUE_OPCODE(SharedAtomicOr32, U32, U32, U32, U1)
 VALUE_OPCODE(SharedAtomicXor32, U32, U32, U32, U1)
-VALUE_OPCODE(DataAppend, U32, U32, U1, U32, U32)
-VALUE_OPCODE(DataConsume, U32, U32, U1, U32, U32)
+VALUE_OPCODE(DataAppend, U32, U32, U1)
+VALUE_OPCODE(DataConsume, U32, U32, U1)
 VALUE_OPCODE(SwizzleU32, U32, U32, U32, U1)
 
 VALUE_OPCODE(ImageQueryDimensions, U32x4, ImageResource, ImageAddress)

src/graphics/shader/recompiler/ir/passes/BindingLayout.cpp

@@ -85,6 +85,16 @@ bool CollectMemoryResources(const Program& program, std::vector<uint32_t>& buffe
 	return uses_gds;
 }
 
+bool UsesFlattenedSrt(const Program& program) {
+	return std::ranges::any_of(program.blocks, [](const Block* block) {
+		return std::ranges::any_of(*block, [](const Inst& inst) {
+			return inst.GetOpcode() == ValueOpcode::ReadConst;
+		});
+	}) || std::ranges::any_of(program.info.images, [](const ImageResource& image) {
+		return image.indirect_search_iterations != 0u;
+	});
+}
+
 void AllocateBindings(Program& program, uint32_t push_data_start_dword) {
 	if (!program.shader_info_complete || program.binding_layout_complete) {
 		EXIT("shader binding layout failed: %s", !program.shader_info_complete
@@ -145,12 +155,7 @@ void AllocateBindings(Program& program, uint32_t push_data_start_dword) {
 		AddBinding(next, DescriptorBindingKind::BdaPagetable);
 		AddBinding(next, DescriptorBindingKind::FaultBuffer);
 	}
-	const bool uses_flattened_runtime =
-	    !program.srt_reads.empty() ||
-	    std::ranges::any_of(program.info.images, [](const ImageResource& image) {
-		    return image.indirect_search_iterations != 0u;
-	    });
-	if (uses_flattened_runtime) {
+	if (UsesFlattenedSrt(program)) {
 		AddBinding(next, DescriptorBindingKind::FlattenedSrt);
 	}
 

src/graphics/shader/recompiler/ir/passes/BindingLayout.h

@@ -9,6 +9,7 @@ void AllocateBindings(Program& program, uint32_t push_data_start_dword = 0);
 
 // Collect live direct buffer indices and report whether the shader accesses GDS.
 bool CollectMemoryResources(const Program& program, std::vector<uint32_t>& buffers);
+bool UsesFlattenedSrt(const Program& program);
 
 const DescriptorBinding* FindBinding(const BindingLayout& layout, DescriptorBindingKind kind);
 

src/graphics/shader/recompiler/ir/passes/ConstantPropagation.cpp

@@ -630,6 +630,11 @@ void FoldInstruction(Block& block, Block::iterator instruction,
 				}
 			}
 			return;
+		case ValueOpcode::ConditionRef: {
+			const auto value = Arg(inst, 0);
+			if (IsImmediate(value, Type::U1)) Replace(inst, value);
+			return;
+		}
 		case ValueOpcode::LogicalNot: {
 			const auto value = Arg(inst, 0);
 			if (IsImmediate(value, Type::U1)) {

src/graphics/shader/recompiler/ir/passes/ResourceMaterialization.cpp

@@ -107,14 +107,16 @@ Prospero::BufferFormat ImageConversionFormat(Prospero::BufferFormat format) {
 	                                                      : Prospero::BufferFormat::kInvalid;
 }
 
-bool RequiresPointSampler(const ImageResource& image) {
-	return image.numeric_class == Prospero::TextureNumericClass::Sint ||
-	       image.conversion_format != Prospero::BufferFormat::kInvalid;
-}
+enum class SamplerClass : uint8_t { Float, Integer, PointInteger };
 
-bool RequiresPointSampler(const ResourceSpecialization::Image& image) {
-	return image.numeric_class == Prospero::TextureNumericClass::Sint ||
-	       image.conversion_format != Prospero::BufferFormat::kInvalid;
+template <typename Image>
+SamplerClass ClassifySampler(const Image& image) {
+	if (image.numeric_class == Prospero::TextureNumericClass::Sint ||
+	    image.conversion_format != Prospero::BufferFormat::kInvalid) {
+		return SamplerClass::PointInteger;
+	}
+	return image.numeric_class == Prospero::TextureNumericClass::Uint ? SamplerClass::Integer
+	                                                               : SamplerClass::Float;
 }
 
 bool DescriptorIsCube(const DescriptorValue& descriptor) {
@@ -223,9 +225,9 @@ void MarkCleanFlatSlots(const ResourcePlan& program, const DescriptorSource* sou
 	}
 }
 
-bool ReadScalarTable(uint64_t base, uint64_t size, uint32_t dynamic_offset,
+bool ReadScalarTable(uint64_t base, uint64_t size, uint64_t dynamic_offset,
                      const SrtRuntime& runtime, std::span<uint32_t> words) {
-	const auto offset = static_cast<uint64_t>(dynamic_offset) & ~uint64_t {3};
+	const auto offset = dynamic_offset & ~uint64_t {3};
 	const auto count = std::min<uint64_t>(words.size(), offset < size ? (size - offset) / 4u : 0u);
 	std::ranges::fill(words.subspan(count), 0u);
 	if (count == 0u) {
@@ -304,26 +306,24 @@ bool MaterializeIndirectImage(const ResourcePlan& program,
 		    material.Stride() != indirect.selector_stride) {
 			return false;
 		}
-		// Enumerate every wrapped scalar-buffer offset that can pass the descriptor bounds.
-		const auto step = std::gcd<uint64_t>(indirect.selector_stride, uint64_t {1} << 32u);
-		const auto residue = static_cast<uint64_t>(indirect.selector_offset) % step;
-		const auto limit = std::min<uint64_t>(UINT32_MAX, material.GetSize() + 3u);
-		const auto probe_count = residue <= limit ? (limit - residue) / step + 1u : 0u;
+		// The first aligned offset includes the immediate added after shader U32 arithmetic.
+		const auto step = std::max<uint64_t>(4u,
+		    std::gcd<uint64_t>(indirect.selector_stride, uint64_t {1} << 32u));
+		const uint64_t first = indirect.selector_offset;
+		const auto size = material.GetSize();
+		const auto limit = std::min(first + (uint64_t {1} << 32u) - step, size >= 4u ? size - 4u : 0u);
+		const auto probe_count = size >= 4u && first <= limit ? (limit - first) / step + 1u : 0u;
 		if (probe_count > MaxIndirectImageProbes) {
 			return false;
 		}
 		keys.reserve(static_cast<size_t>(probe_count) + 1u);
 		keys.push_back(0u);
-		for (uint64_t offset = residue; offset <= limit && probe_count != 0u; offset += step) {
+		for (uint64_t probe = 0, offset = first; probe < probe_count; ++probe, offset += step) {
 			uint32_t key = 0;
-			if (!ReadScalarTable(material.Base48(), material.GetSize(),
-			                     static_cast<uint32_t>(offset), runtime, {&key, 1})) {
+			if (!ReadScalarTable(material.Base48(), size, offset, runtime, {&key, 1})) {
 				return false;
 			}
 			keys.push_back(key);
-			if (limit - offset < step) {
-				break;
-			}
 		}
 		std::ranges::sort(keys);
 		keys.erase(std::unique(keys.begin(), keys.end()), keys.end());
@@ -380,8 +380,13 @@ bool MaterializeIndirectImage(const ResourcePlan& program,
 } // namespace
 
 struct SamplerPlan {
-	std::array<uint32_t, ShaderInfo::MaxSamplers> point_sampler {};
-	uint32_t                                      sampler_count = 0;
+	struct Binding {
+		uint32_t     source;
+		SamplerClass type;
+	};
+	std::array<std::array<uint32_t, 3>, ShaderInfo::MaxSamplers> mapping;
+	std::array<Binding, ShaderInfo::MaxSamplers>                bindings;
+	uint32_t                                                  sampler_count = 0;
 };
 
 struct ImageRemap {
@@ -595,11 +600,9 @@ static bool BuildResourceSpecialization(const ResourcePlan& program, ResourceSna
 	if (!BuildSamplerPlan(program.info, specialization.images, sampler_plan)) {
 		return SpecializationFail("specialized sampler layout exceeds its resource limit");
 	}
-	for (uint32_t index = 0; index < program.info.samplers.size(); index++) {
-		const auto target = sampler_plan.point_sampler[index];
-		if (target != UINT32_MAX && target >= program.info.samplers.size()) {
-			snapshot.samplers.push_back(snapshot.samplers[index]);
-		}
+	for (uint32_t index = static_cast<uint32_t>(program.info.samplers.size());
+	     index < sampler_plan.sampler_count; index++) {
+		snapshot.samplers.push_back(snapshot.samplers[sampler_plan.bindings[index].source]);
 	}
 	ImageRemap(specialization).Apply(snapshot.images);
 	return true;
@@ -607,29 +610,29 @@ static bool BuildResourceSpecialization(const ResourcePlan& program, ResourceSna
 
 template <typename Images>
 bool BuildSamplerPlan(const ShaderInfo& base, const Images& images, SamplerPlan& plan) {
-	if (base.samplers.size() > plan.point_sampler.size()) {
+	if (base.samplers.size() > plan.mapping.size()) {
 		return false;
 	}
 	std::array<uint8_t, ShaderInfo::MaxSamplers> usage {};
-	plan.point_sampler.fill(UINT32_MAX);
 	plan.sampler_count = static_cast<uint32_t>(base.samplers.size());
 	for (const auto& pair: base.sampled_pairs) {
 		if (pair.image >= images.size() || pair.sampler >= base.samplers.size()) {
 			return false;
 		}
-		usage[pair.sampler] |= RequiresPointSampler(images[pair.image]) ? 2u : 1u;
+		usage[pair.sampler] |= 1u << static_cast<uint32_t>(ClassifySampler(images[pair.image]));
 	}
 	for (uint32_t index = 0; index < base.samplers.size(); index++) {
-		if ((usage[index] & 2u) == 0u) {
-			continue;
-		}
-		if ((usage[index] & 1u) == 0u) {
-			plan.point_sampler[index] = index;
-		} else {
-			if (plan.sampler_count >= ShaderInfo::MaxSamplers) {
-				return false;
-			}
-			plan.point_sampler[index] = plan.sampler_count++;
+		auto& mapping = plan.mapping[index];
+		mapping.fill(UINT32_MAX);
+		const auto classes = usage[index] == 0u ? 1u : usage[index];
+		bool       first   = true;
+		for (uint32_t type = 0; type < mapping.size(); type++) {
+			if ((classes & (1u << type)) == 0u) continue;
+			const auto target = first ? index : plan.sampler_count++;
+			if (target >= ShaderInfo::MaxSamplers) return false;
+			mapping[type]         = target;
+			plan.bindings[target] = {index, static_cast<SamplerClass>(type)};
+			first                = false;
 		}
 	}
 	return true;
@@ -862,7 +865,6 @@ ResourcePlan ExtractResourcePlan(const Program& program) {
 	plan.memory_info                = program.memory_info;
 	plan.srt_plan_complete          = program.srt_plan_complete;
 	plan.resource_tracking_complete = program.resource_tracking_complete;
-	plan.has_address_writes         = program.has_address_writes;
 
 	std::unordered_map<const Inst*, Inst*> cloned;
 	std::function<Value(Value)>            Clone = [&](Value value) -> Value {
@@ -921,17 +923,25 @@ ResourcePlan ExtractResourcePlan(const Program& program) {
 		plan.uniform_fill.values[i] = Clone(plan.uniform_fill.values[i]);
 	}
 	plan.clean_flat_slots.resize(plan.srt_reads.size());
+	bool masked_image = false;
 	for (const auto& image: plan.info.images) {
 		const auto* source = Source(plan, image.source);
 		if (source == nullptr || !source->indirect_image.has_value()) {
 			continue;
 		}
 		plan.requires_specialization_memory = true;
+		masked_image |= !source->indirect_image->selector_mask.IsEmpty();
 		MarkCleanFlatSlots(plan, Source(plan, source->indirect_image->material_source),
 		                   plan.clean_flat_slots, source->indirect_image->selector_mask);
 		MarkCleanFlatSlots(plan, Source(plan, source->indirect_image->table_source),
 		                   plan.clean_flat_slots);
 	}
+	if (masked_image) {
+		plan.resource_tracking_complete &= !program.has_address_writes &&
+		    !std::ranges::any_of(plan.info.images, &ImageResource::written);
+		plan.capture_specialization_reads =
+		    std::ranges::any_of(plan.info.buffers, &BufferResource::written);
+	}
 	return plan;
 }
 
@@ -941,18 +951,7 @@ bool MaterializeResources(const ResourcePlan& program, const SrtRuntime& runtime
 	    (program.requires_specialization_memory && runtime.read_specialization_memory == nullptr)) {
 		return false;
 	}
-	const bool masked_image = std::ranges::any_of(program.info.images, [&](const auto& image) {
-		const auto* source = Source(program, image.source);
-		return source != nullptr && source->indirect_image.has_value() &&
-		       !source->indirect_image->selector_mask.IsEmpty();
-	});
-	if (masked_image &&
-	    (program.has_address_writes ||
-	     std::ranges::any_of(program.info.images, &ImageResource::written))) {
-		return false;
-	}
-	const bool capture_reads = masked_image &&
-	    std::ranges::any_of(program.info.buffers, &BufferResource::written);
+	const bool capture_reads = program.capture_specialization_reads;
 	auto& reads = program.specialization_reads;
 	ReadCapture capture {runtime, reads};
 	SrtRuntime observed = runtime;
@@ -1105,25 +1104,18 @@ void ApplyResourceSpecialization(Program& program, const ResourceSpecialization&
 	auto samplers      = program.info.samplers;
 	auto sampled_pairs = program.info.sampled_pairs;
 	samplers.reserve(sampler_plan.sampler_count);
-	for (uint32_t index = 0; index < program.info.samplers.size(); index++) {
-		const auto target = sampler_plan.point_sampler[index];
-		if (target == UINT32_MAX) {
-			continue;
-		}
-		if (target == index) {
-			samplers[index].force_point_filtering = true;
-		} else {
-			EXIT_IF(target != samplers.size());
-			auto sampler                  = samplers[index];
-			sampler.force_point_filtering = true;
-			samplers.push_back(std::move(sampler));
+	for (uint32_t index = 0; index < sampler_plan.sampler_count; index++) {
+		const auto& binding = sampler_plan.bindings[index];
+		if (index >= program.info.samplers.size()) {
+			samplers.push_back(program.info.samplers[binding.source]);
 		}
+		samplers[index].force_point_filtering = binding.type == SamplerClass::PointInteger;
+		samplers[index].integer_border        = binding.type != SamplerClass::Float;
 	}
 	for (auto& pair: sampled_pairs) {
-		if (RequiresPointSampler(images[pair.image])) {
-			EXIT_IF(sampler_plan.point_sampler[pair.sampler] == UINT32_MAX);
-			pair.sampler = sampler_plan.point_sampler[pair.sampler];
-		}
+		const auto type = static_cast<uint32_t>(ClassifySampler(images[pair.image]));
+		pair.sampler = sampler_plan.mapping[pair.sampler][type];
+		EXIT_IF(pair.sampler == UINT32_MAX);
 		samplers[pair.sampler].depth_compare |= images[pair.image].depth_compare;
 	}
 
@@ -1201,10 +1193,11 @@ void ApplyResourceSpecialization(Program& program, const ResourceSpecialization&
 				inst.ReplaceUsesWith(Value(&*result));
 				continue;
 			}
-			if (image_opcode.needs_sampler && RequiresPointSampler(image) &&
+			if (image_opcode.needs_sampler &&
 			    memory.sampler < program.info.samplers.size()) {
-				EXIT_IF(sampler_plan.point_sampler[memory.sampler] == UINT32_MAX);
-				memory.sampler = sampler_plan.point_sampler[memory.sampler];
+				const auto type = static_cast<uint32_t>(ClassifySampler(image));
+				memory.sampler = sampler_plan.mapping[memory.sampler][type];
+				EXIT_IF(memory.sampler == UINT32_MAX);
 			}
 			EXIT_IF(image.indirect_root == memory.resource &&
 			        inst.GetOpcode() != ValueOpcode::ImageSampleRaw);

src/graphics/shader/recompiler/ir/passes/ResourceTracking.h

@@ -5,9 +5,8 @@
 
 namespace Libs::Graphics::ShaderRecompiler::IR {
 
-// Collects immutable resource topology from typed SSA handles, interns their resolved dwords in
-// descriptor_sources, then writes dense indices to handle flags and MemoryInfo.
-void TrackResources(Program& program);
+// Resolves native descriptor sources, plans their scalar reads, and assigns dense resource bindings.
+void TrackResources(Program& program, const Decoder::Program& decoded, const CFG::Graph& native_cfg);
 
 } // namespace Libs::Graphics::ShaderRecompiler::IR
 

src/graphics/shader/recompiler/ir/passes/SrtWalker.cpp

@@ -7,7 +7,6 @@
 #include <bit>
 #include <cmath>
 #include <cstring>
-#include <fmt/format.h>
 #include <unordered_set>
 
 namespace Libs::Graphics::ShaderRecompiler::IR {
@@ -23,21 +22,6 @@ namespace {
 
 constexpr uint64_t AddressMask = 0x0000ffffffffffffull;
 
-const char* StageName(ShaderType stage) {
-	switch (stage) {
-		case ShaderType::Vertex: return "vertex";
-		case ShaderType::Pixel: return "pixel";
-		case ShaderType::Fetch: return "fetch";
-		case ShaderType::Compute: return "compute";
-		default: return "unknown";
-	}
-}
-
-std::string Diagnostic(const ResourcePlan& program, uint32_t pc, const std::string& message) {
-	return fmt::format("shader SRT: hash=0x{:016x} stage={} pc=0x{:08x} {}", program.shader_hash,
-	                   StageName(program.stage), pc, message);
-}
-
 bool AddSignedAddress(uint64_t base, int64_t offset, uint64_t& result) {
 	if (base > AddressMask) {
 		return false;
@@ -89,6 +73,7 @@ bool IsRuntimeSelect(ValueOpcode op) {
 
 bool IsRuntimeUniformOp(ValueOpcode op) {
 	switch (op) {
+		case ValueOpcode::ConditionRef:
 		case ValueOpcode::BitCastU32F32:
 		case ValueOpcode::BitCastF32U32:
 		case ValueOpcode::ConvertU32F32:
@@ -312,151 +297,6 @@ private:
 	std::unordered_set<const Inst*> m_validated_dependencies;
 };
 
-class PlanBuilder {
-public:
-	explicit PlanBuilder(Program& program): m_program(program) {}
-
-	void Run() {
-		m_program.srt_reads.clear();
-		m_program.dynamic_reads.clear();
-		for (auto* block: m_program.blocks) {
-			for (auto& inst: *block) {
-				const auto op = inst.GetOpcode();
-				if (op == ValueOpcode::LoadAddressU32 || op == ValueOpcode::ReadConstBuffer) {
-					const auto flags = inst.Flags<MemoryFlags>();
-					if (flags.index < m_program.memory_info.size()) {
-						const auto kind       = m_program.memory_info[flags.index].kind;
-						const bool crosswired = (op == ValueOpcode::LoadAddressU32 &&
-						                         kind == ResourceKind::ScalarBuffer) ||
-						                        (op == ValueOpcode::ReadConstBuffer &&
-						                         kind == ResourceKind::ScalarAddress);
-						if (crosswired) {
-							Fail(flags.pc,
-							     fmt::format("{} has incompatible scalar memory metadata",
-							                 ValueOpcodeName(op)));
-						}
-					}
-				}
-				if (IsDescriptorHandle(inst.GetOpcode())) {
-					for (size_t index = 0; index < inst.NumArgs(); index++) {
-						Collect(inst.Arg(index), 0);
-					}
-				}
-			}
-		}
-		for (auto* block: m_program.blocks) {
-			for (auto& inst: *block) {
-				if (inst.GetOpcode() == ValueOpcode::LoadAddressU32 && IsRawRead(m_program, inst) &&
-				    inst.Arg(1).Resolve().IsImmediate() &&
-				    ValidateRuntimeValue(m_program, Value(&inst))) {
-					Collect(Value(&inst), inst.Flags<MemoryFlags>().pc);
-				}
-			}
-		}
-		PatchReads();
-	}
-
-private:
-	struct Patch {
-		Inst*    inst = nullptr;
-		uint32_t slot = 0;
-		bool     keep = false;
-	};
-
-	[[noreturn]] void Fail(uint32_t pc, const std::string& message) const {
-		const auto diagnostic = Diagnostic(m_program, pc, message);
-		EXIT("shader SRT planning failed: %s", diagnostic.c_str());
-		std::abort();
-	}
-
-	void Collect(Value value, uint32_t use_pc) {
-		value = value.Resolve();
-		if (value.IsImmediate()) {
-			return;
-		}
-		auto* inst = value.TryInstruction();
-		if (inst == nullptr) {
-			Fail(use_pc, "invalid typed planning value");
-		}
-		const auto cycle = std::ranges::find(m_visiting, inst);
-		if (cycle != m_visiting.end()) {
-			const auto contains_phi = std::any_of(cycle, m_visiting.end(), [](const Inst* value) {
-				return value->GetOpcode() == ValueOpcode::Phi;
-			});
-			if (contains_phi) {
-				return;
-			}
-			Fail(use_pc, fmt::format("cyclic typed planning value {} without a phi",
-			                         ValueOpcodeName(inst->GetOpcode())));
-		}
-		if (std::ranges::find(m_visited, inst) != m_visited.end()) {
-			return;
-		}
-		m_visiting.push_back(inst);
-		for (size_t index = 0; index < inst->NumArgs(); index++) {
-			Collect(inst->Arg(index), use_pc);
-		}
-		m_visiting.pop_back();
-		m_visited.push_back(inst);
-		if (!IsRawRead(m_program, *inst)) {
-			return;
-		}
-		const auto offset = inst->Arg(1).Resolve();
-		if (!offset.IsImmediate() || offset.GetType() != Type::U32) {
-			if (std::ranges::find(m_program.dynamic_reads, value) ==
-			    m_program.dynamic_reads.end()) {
-				m_program.dynamic_reads.push_back(value);
-			}
-			return;
-		}
-		for (uint32_t slot = 0; slot < m_program.srt_reads.size(); slot++) {
-			if (EquivalentValue(m_program, value, m_program.srt_reads[slot].value)) {
-				m_patches.push_back({inst, slot, false});
-				return;
-			}
-		}
-		const auto slot = static_cast<uint32_t>(m_program.srt_reads.size());
-		m_program.srt_reads.push_back({value, slot});
-		m_patches.push_back({inst, slot, true});
-	}
-
-	void PatchReads() {
-		for (const auto& patch: m_patches) {
-			auto* block = patch.inst->Parent();
-			auto& list  = block->Instructions();
-			auto  where =
-			    std::ranges::find_if(list, [&](const Inst& inst) { return &inst == patch.inst; });
-			const auto resource =
-			    Value(&*block->PrependNewInst(where, ValueOpcode::GetSrtResource));
-			const auto flat = Value(&*block->PrependNewInst(where, ValueOpcode::ReadConst,
-			                                                {resource, Value(patch.slot)}));
-			const auto uses = patch.inst->Uses();
-			for (const auto& use: uses) {
-				use.user->SetArg(use.operand, flat);
-			}
-			for (auto& info: m_program.block_info) {
-				if (info.condition.Resolve() == Value(patch.inst)) {
-					info.condition = flat;
-				}
-				if (info.indirect_target.Resolve() == Value(patch.inst)) {
-					info.indirect_target = flat;
-				}
-			}
-			if (patch.keep) {
-				const auto memory = patch.inst->Flags<MemoryFlags>().index;
-				if (memory < m_program.memory_info.size()) {
-					m_program.memory_info[memory].planning_only = true;
-				}
-				block->AppendNewInst(ValueOpcode::ReferenceU32, {Value(patch.inst)});
-			}
-		}
-	}
-
-	Program&           m_program;
-	std::vector<Inst*> m_visiting;
-	std::vector<Inst*> m_visited;
-	std::vector<Patch> m_patches;
-};
 
 } // namespace
 
@@ -616,16 +456,15 @@ bool SrtWalker::EvaluateRawRead(const Inst& inst, uint64_t& result) {
 			return false;
 		}
 		const auto byte_offset =
-		    static_cast<uint64_t>(immediate) + static_cast<uint32_t>(offset);
-		const auto aligned = byte_offset & ~uint64_t {3};
+		    (static_cast<uint64_t>(immediate) & ~uint64_t {3}) + (static_cast<uint32_t>(offset) & ~3u);
 		const auto stride  = (static_cast<uint32_t>(high) >> 16u) & 0x3fffu;
 		const auto size = stride == 0u
 		                      ? static_cast<uint64_t>(static_cast<uint32_t>(records))
 		                      : static_cast<uint64_t>(stride) * static_cast<uint32_t>(records);
-		if (aligned > size || size - aligned < sizeof(uint32_t)) {
+		if (byte_offset > size || size - byte_offset < sizeof(uint32_t)) {
 			return false;
 		}
-		address = ((base & ~uint64_t {3}) + byte_offset) & ~uint64_t {3};
+		address = (base & ~uint64_t {3}) + byte_offset;
 	} else {
 		const auto relative = (immediate & ~int64_t {3}) +
 		                      static_cast<int64_t>(static_cast<uint32_t>(offset) & ~3u);
@@ -780,14 +619,18 @@ bool SrtWalker::EvaluateInst(const Inst& inst, uint64_t& result) {
 			}
 			return false;
 		case ValueOpcode::FPOrdLessThanEqual32:
-			if (binary()) {
-				result = Float32(a) <= Float32(b);
-				return true;
-			}
-			return false;
 		case ValueOpcode::FPOrdGreaterThanEqual32:
 			if (binary()) {
-				result = Float32(a) >= Float32(b);
+				const auto operand = [&](uint64_t bits) {
+					if (inst.Flags<FPCompareFlags>().flush_input_denorms &&
+					    (bits & 0x7fffffffu) < 0x00800000u) {
+						bits &= 0x80000000u;
+					}
+					return Float32(bits);
+				};
+				result = inst.GetOpcode() == ValueOpcode::FPOrdLessThanEqual32
+				             ? operand(a) <= operand(b)
+				             : operand(a) >= operand(b);
 				return true;
 			}
 			return false;
@@ -970,6 +813,7 @@ bool SrtWalker::EvaluateInst(const Inst& inst, uint64_t& result) {
 				return true;
 			}
 			return false;
+		case ValueOpcode::ConditionRef: return Arg(inst, 0, result);
 		case ValueOpcode::LogicalNot:
 			if (Arg(inst, 0, a)) {
 				result = a == 0u;
@@ -1061,13 +905,5 @@ bool ValidateRuntimeValue(const ResourcePlan& program, Value value, RuntimeValue
 	return RuntimeValidator(program, type).Run(value);
 }
 
-void BuildSrtPlan(Program& program) {
-	if (program.resource_tracking_complete) {
-		EXIT("shader SRT planning failed: cannot rebuild SRT after resource tracking");
-	}
-	program.srt_plan_complete = false;
-	PlanBuilder(program).Run();
-	program.srt_plan_complete = true;
-}
 
 } // namespace Libs::Graphics::ShaderRecompiler::IR

src/graphics/shader/recompiler/ir/passes/SrtWalker.h

@@ -21,9 +21,6 @@ struct SrtRuntime {
 
 enum class RuntimeValueType { Any, Integer };
 
-// Collects reachable ReadConst values. Immediate offsets receive compact flat-buffer slots;
-// dynamic offsets remain explicit and are never assigned a fake slot.
-void BuildSrtPlan(Program& program);
 bool ValidateRuntimeValue(const ResourcePlan& program, Value value,
                           RuntimeValueType type = RuntimeValueType::Any);
 // Uses the strict reader for values that affect shader specialization.

src/kernel/fileSystem.cpp

@@ -358,7 +358,8 @@ std::filesystem::path MountPoints::ResolvePath(const std::string& mounted_name)
 #endif
 	}
 
-	return mounted_name;
+	// Unmounted guest paths must not fall through to the host filesystem.
+	return {};
 }
 
 void Initialize() {
@@ -461,6 +462,10 @@ int KYTY_SYSV_ABI KernelOpen(const char* path, int flags, uint16_t mode) {
 	}
 
 	file->real_name = g_mount_points->ResolvePath(file->name);
+	if (file->real_name.empty()) {
+		g_files->DeleteDescriptor(descriptor);
+		return KERNEL_ERROR_ENOENT;
+	}
 
 	if (trunc && rw_mode == Common::File::Mode::Read) {
 		g_files->DeleteDescriptor(descriptor);
@@ -1255,7 +1260,7 @@ int KYTY_SYSV_ABI KernelRename(const char* from, const char* to) {
 	auto real_from = g_mount_points->ResolvePath(from_path);
 	auto real_to   = g_mount_points->ResolvePath(to_path);
 
-	if (!Common::File::IsFileExisting(real_from)) {
+	if (real_to.empty() || !Common::File::IsFileExisting(real_from)) {
 		return KERNEL_ERROR_ENOENT;
 	}
 
@@ -1335,6 +1340,9 @@ int KYTY_SYSV_ABI KernelMkdir(const char* path, uint16_t mode) {
 	     path, mode);
 
 	auto real_name = g_mount_points->ResolvePath(std::string(path));
+	if (real_name.empty()) {
+		return KERNEL_ERROR_ENOENT;
+	}
 
 	if (Common::File::IsDirectoryExisting(real_name)) {
 		return KERNEL_ERROR_EEXIST;

src/kernel/fileSystem.h

@@ -54,6 +54,7 @@ struct Lifecycle {
 
 void                  Mount(const std::filesystem::path& folder, const std::string& point);
 void                  Umount(const std::string& folder_or_point);
+// Returns an empty path when the guest path has no mounted filesystem.
 std::filesystem::path GetRealFilename(const std::string& mounted_file_name);
 
 int KYTY_SYSV_ABI     KernelOpen(const char* path, int flags, uint16_t mode);

src/kernel/memory.cpp

@@ -78,8 +78,7 @@ static Graphics::RenderContext& GetGpuResources() {
 }
 
 static bool IsGpuAddressRange(uint64_t vaddr, uint64_t size) {
-	constexpr uint64_t GPU_ADDRESS_LIMIT = 1ull << 40u;
-	return vaddr != 0 && size != 0 && vaddr < GPU_ADDRESS_LIMIT && size < GPU_ADDRESS_LIMIT - vaddr;
+	return Graphics::GuestRange {vaddr, size}.Valid();
 }
 
 static void MapGpuRange(uint64_t vaddr, uint64_t size) {
@@ -660,6 +659,9 @@ static bool     g_test_fail_next_fixed_reserve_range_add     = false;
 
 class PhysicalMemory {
 public:
+	enum class AllocationKind { Direct, Pooled, Automatic };
+	using PhysicalRanges = std::vector<std::pair<uint64_t, uint64_t>>;
+
 	struct AllocatedBlock {
 		uint64_t            start_addr;
 		uint64_t            size;
@@ -671,7 +673,7 @@ public:
 		VirtualMemory::Mode mode;
 		GpuAccessMode       gpu_mode;
 		int                 memory_type;
-		bool                pool_expansion;
+		AllocationKind      kind;
 		char                name[KERNEL_MAXIMUM_NAME_LENGTH];
 	};
 
@@ -690,7 +692,10 @@ public:
 	}
 
 	bool Alloc(uint64_t search_start, uint64_t search_end, size_t len, size_t alignment,
-	           uint64_t* phys_addr_out, int memory_type, bool pool_expansion = false);
+	           uint64_t* phys_addr_out, int memory_type,
+	           AllocationKind kind = AllocationKind::Direct);
+	bool ReserveAutomatic(uint64_t size, PhysicalRanges* ranges);
+	void RestoreAutomatic(const PhysicalRanges& ranges);
 	bool Available(uint64_t search_start, uint64_t search_end, size_t alignment,
 	               uint64_t* phys_addr_out, uint64_t* size_out);
 	bool Release(uint64_t start, size_t len, uint64_t* vaddr, uint64_t* size,
@@ -718,12 +723,14 @@ public:
 	[[nodiscard]] const std::vector<AllocatedBlock>& GetMappings() const { return m_mappings; }
 
 private:
-	void ConsumeFreeRange(std::map<uint64_t, uint64_t>::iterator range, uint64_t start,
-	                      uint64_t size);
-	void AddFreeRange(uint64_t start, uint64_t size);
+	static void RemoveFreeRange(std::map<uint64_t, uint64_t>& ranges, uint64_t start,
+	                            uint64_t size);
+	static void AddFreeRange(std::map<uint64_t, uint64_t>& ranges, uint64_t start, uint64_t size);
+	void        ReclaimAutomatic(uint64_t start, uint64_t size);
 
 	std::map<uint64_t, AllocatedBlock> m_physical;
 	std::map<uint64_t, uint64_t>       m_free;
+	std::map<uint64_t, uint64_t>       m_automatic_free;
 	std::vector<AllocatedBlock>        m_mappings;
 	Common::Mutex                      m_mutex;
 };
@@ -1093,7 +1100,7 @@ void SetFlexibleMemorySize(uint64_t size) {
 }
 
 bool PhysicalMemory::Alloc(uint64_t search_start, uint64_t search_end, size_t len, size_t alignment,
-                           uint64_t* phys_addr_out, int memory_type, bool pool_expansion) {
+                           uint64_t* phys_addr_out, int memory_type, AllocationKind kind) {
 	if (phys_addr_out == nullptr) {
 		return false;
 	}
@@ -1128,10 +1135,13 @@ bool PhysicalMemory::Alloc(uint64_t search_start, uint64_t search_end, size_t le
 		b.prot           = 0;
 		b.mode           = VirtualMemory::Mode::NoAccess;
 		b.memory_type    = memory_type;
-		b.pool_expansion = pool_expansion;
+		b.kind           = kind;
 
-		ConsumeFreeRange(range, free_pos, len);
+		RemoveFreeRange(m_free, free_pos, len);
 		EXIT_IF(!m_physical.emplace(b.start_addr, b).second);
+		if (kind == AllocationKind::Automatic) {
+			AddFreeRange(m_automatic_free, free_pos, len);
+		}
 
 		*phys_addr_out = free_pos;
 		return true;
@@ -1180,35 +1190,99 @@ bool PhysicalMemory::Available(uint64_t search_start, uint64_t search_end, size_
 	return true;
 }
 
-void PhysicalMemory::ConsumeFreeRange(std::map<uint64_t, uint64_t>::iterator range, uint64_t start,
-                                      uint64_t size) {
-	const auto range_start = range->first;
-	const auto range_end   = range->first + range->second;
-	m_free.erase(range);
-	if (range_start < start) {
-		m_free.emplace(range_start, start - range_start);
+void PhysicalMemory::RemoveFreeRange(std::map<uint64_t, uint64_t>& ranges, uint64_t start,
+                                     uint64_t size) {
+	const auto end   = start + size;
+	auto       range = ranges.upper_bound(start);
+	if (range != ranges.begin()) {
+		--range;
 	}
-	if (start + size < range_end) {
-		m_free.emplace(start + size, range_end - start - size);
+	while (range != ranges.end() && range->first < end) {
+		const auto range_start = range->first;
+		const auto range_end   = range_start + range->second;
+		if (range_end <= start) {
+			++range;
+			continue;
+		}
+		range = ranges.erase(range);
+		if (range_start < start) {
+			ranges.emplace(range_start, start - range_start);
+		}
+		if (end < range_end) {
+			ranges.emplace(end, range_end - end);
+		}
 	}
 }
 
-void PhysicalMemory::AddFreeRange(uint64_t start, uint64_t size) {
+void PhysicalMemory::AddFreeRange(std::map<uint64_t, uint64_t>& ranges, uint64_t start,
+                                  uint64_t size) {
 	auto end  = start + size;
-	auto next = m_free.lower_bound(start);
-	if (next != m_free.begin()) {
+	auto next = ranges.lower_bound(start);
+	if (next != ranges.begin()) {
 		auto previous = std::prev(next);
 		if (previous->first + previous->second >= start) {
 			start = previous->first;
 			end   = std::max(end, previous->first + previous->second);
-			next  = m_free.erase(previous);
+			next  = ranges.erase(previous);
 		}
 	}
-	while (next != m_free.end() && next->first <= end) {
+	while (next != ranges.end() && next->first <= end) {
 		end  = std::max(end, next->first + next->second);
-		next = m_free.erase(next);
+		next = ranges.erase(next);
+	}
+	ranges.emplace(start, end - start);
+}
+
+bool PhysicalMemory::ReserveAutomatic(uint64_t size, PhysicalRanges* ranges) {
+	Common::LockGuard lock(m_mutex);
+	for (const auto& [start, available]: m_automatic_free) {
+		const auto length = std::min(size, available);
+		ranges->emplace_back(start, length);
+		size -= length;
+		if (size == 0) {
+			for (const auto& [offset, length]: *ranges) {
+				RemoveFreeRange(m_automatic_free, offset, length);
+			}
+			return true;
+		}
+	}
+	ranges->clear();
+	return false;
+}
+
+void PhysicalMemory::RestoreAutomatic(const PhysicalRanges& ranges) {
+	Common::LockGuard lock(m_mutex);
+	for (const auto& [start, size]: ranges) {
+		AddFreeRange(m_automatic_free, start, size);
+	}
+}
+
+void PhysicalMemory::ReclaimAutomatic(uint64_t start, uint64_t size) {
+	const auto end   = start + size;
+	auto       owner = m_physical.upper_bound(start);
+	if (owner != m_physical.begin()) {
+		--owner;
+	}
+	bool automatic = false;
+	for (; owner != m_physical.end() && owner->first < end; ++owner) {
+		const auto& block = owner->second;
+		if (block.kind != AllocationKind::Automatic || block.start_addr + block.size <= start) {
+			continue;
+		}
+		const auto first = std::max(start, block.start_addr);
+		const auto last  = std::min(end, block.start_addr + block.size);
+		AddFreeRange(m_automatic_free, first, last - first);
+		automatic = true;
+	}
+	if (automatic) {
+		for (const auto& mapping: m_mappings) {
+			const auto first = std::max(start, mapping.start_addr);
+			const auto last  = std::min(end, mapping.start_addr + mapping.size);
+			if (first < last) {
+				RemoveFreeRange(m_automatic_free, first, last - first);
+			}
+		}
 	}
-	m_free.emplace(start, end - start);
 }
 
 bool PhysicalMemory::Release(uint64_t start, size_t len, uint64_t* vaddr, uint64_t* size,
@@ -1225,18 +1299,22 @@ bool PhysicalMemory::Release(uint64_t start, size_t len, uint64_t* vaddr, uint64
 	}
 	auto  it = std::prev(next);
 	auto& b  = it->second;
-	if (b.pool_expansion || start < b.start_addr || start >= b.start_addr + b.size ||
-	    len > b.start_addr + b.size - start) {
+	if (b.kind == AllocationKind::Pooled || start < b.start_addr ||
+	    start >= b.start_addr + b.size || len > b.start_addr + b.size - start) {
 		return false;
 	}
 
+	if (b.kind == AllocationKind::Automatic) {
+		RemoveFreeRange(m_automatic_free, start, len);
+	}
+
 	if (start == b.start_addr && len == b.size) {
 		*vaddr    = b.map_vaddr;
 		*size     = b.map_size;
 		*gpu_mode = b.gpu_mode;
 
 		m_physical.erase(it);
-		AddFreeRange(start, len);
+		AddFreeRange(m_free, start, len);
 		return true;
 	}
 	if (start > b.start_addr && start + len < b.start_addr + b.size) {
@@ -1261,7 +1339,7 @@ bool PhysicalMemory::Release(uint64_t start, size_t len, uint64_t* vaddr, uint64
 		}
 
 		m_physical.emplace(right.start_addr, right);
-		AddFreeRange(start, len);
+		AddFreeRange(m_free, start, len);
 		return true;
 	}
 	if (start == b.start_addr && len < b.size) {
@@ -1278,7 +1356,7 @@ bool PhysicalMemory::Release(uint64_t start, size_t len, uint64_t* vaddr, uint64
 			remaining.map_size -= len;
 		}
 		m_physical.emplace(remaining.start_addr, remaining);
-		AddFreeRange(start, len);
+		AddFreeRange(m_free, start, len);
 		return true;
 	}
 	if (start > b.start_addr && start + len == b.start_addr + b.size) {
@@ -1290,7 +1368,7 @@ bool PhysicalMemory::Release(uint64_t start, size_t len, uint64_t* vaddr, uint64
 		if (b.map_vaddr != 0) {
 			b.map_size = b.size;
 		}
-		AddFreeRange(start, len);
+		AddFreeRange(m_free, start, len);
 		return true;
 	}
 
@@ -1311,6 +1389,7 @@ bool PhysicalMemory::Map(uint64_t vaddr, uint64_t phys_addr, size_t len, int pro
 		return false;
 	}
 	auto first = std::prev(next);
+	bool has_automatic = false;
 	while (current < phys_addr + len) {
 		auto block = m_physical.upper_bound(current);
 		if (block == m_physical.begin()) {
@@ -1318,10 +1397,11 @@ bool PhysicalMemory::Map(uint64_t vaddr, uint64_t phys_addr, size_t len, int pro
 		}
 		--block;
 		const auto block_end = block->second.start_addr + block->second.size;
-		if (block->second.pool_expansion || current < block->second.start_addr ||
+		if (block->second.kind == AllocationKind::Pooled || current < block->second.start_addr ||
 		    current >= block_end) {
 			return false;
 		}
+		has_automatic |= block->second.kind == AllocationKind::Automatic;
 		current = std::min<uint64_t>(phys_addr + len, block_end);
 		if (current < phys_addr + len) {
 			const auto following = std::next(block);
@@ -1342,6 +1422,9 @@ bool PhysicalMemory::Map(uint64_t vaddr, uint64_t phys_addr, size_t len, int pro
 	mapping.mode           = mode;
 	mapping.gpu_mode       = gpu_mode;
 	m_mappings.push_back(mapping);
+	if (has_automatic) {
+		RemoveFreeRange(m_automatic_free, phys_addr, len);
+	}
 
 	return true;
 }
@@ -1362,7 +1445,7 @@ bool PhysicalMemory::CanMapDirect(uint64_t phys_addr, size_t len) {
 		}
 		--block;
 		const auto block_end = block->second.start_addr + block->second.size;
-		if (block->second.pool_expansion || current < block->second.start_addr ||
+		if (block->second.kind == AllocationKind::Pooled || current < block->second.start_addr ||
 		    current >= block_end) {
 			return false;
 		}
@@ -1380,11 +1463,12 @@ bool PhysicalMemory::CanMapDirect(uint64_t phys_addr, size_t len) {
 bool PhysicalMemory::ReleasePoolExpansion(uint64_t phys_addr, size_t len) {
 	Common::LockGuard lock(m_mutex);
 	const auto        it = m_physical.find(phys_addr);
-	if (it == m_physical.end() || !it->second.pool_expansion || it->second.size != len) {
+	if (it == m_physical.end() || it->second.kind != AllocationKind::Pooled ||
+	    it->second.size != len) {
 		return false;
 	}
 	m_physical.erase(it);
-	AddFreeRange(phys_addr, len);
+	AddFreeRange(m_free, phys_addr, len);
 	return true;
 }
 
@@ -1407,7 +1491,7 @@ bool PhysicalMemory::GetAllocatedSpan(uint64_t phys_addr, size_t len,
 		}
 		--block;
 		const auto block_end = block->second.start_addr + block->second.size;
-		if (block->second.pool_expansion || current < block->second.start_addr ||
+		if (block->second.kind == AllocationKind::Pooled || current < block->second.start_addr ||
 		    current >= block_end) {
 			blocks->clear();
 			return false;
@@ -1463,49 +1547,39 @@ bool PhysicalMemory::Unmap(uint64_t vaddr, uint64_t size, GpuAccessMode* gpu_mod
 		}
 	};
 
-	size_t index = 0;
-	for (auto& b: m_mappings) {
-		if (b.map_vaddr == vaddr && b.map_size == size) {
-			*gpu_mode             = b.gpu_mode;
-			const auto host_vaddr = b.host_vaddr;
-			const auto host_size  = b.host_size;
-
+	for (size_t index = 0; index < m_mappings.size(); ++index) {
+		auto& b = m_mappings[index];
+		if (vaddr < b.map_vaddr || vaddr >= b.map_vaddr + b.map_size ||
+		    size > b.map_vaddr + b.map_size - vaddr) {
+			continue;
+		}
+		*gpu_mode             = b.gpu_mode;
+		const auto phys_addr  = b.start_addr + vaddr - b.map_vaddr;
+		const auto host_vaddr = b.host_vaddr;
+		const auto host_size  = b.host_size;
+		if (vaddr == b.map_vaddr && size == b.map_size) {
 			m_mappings.erase(m_mappings.begin() + static_cast<std::ptrdiff_t>(index));
 			set_host_release_if_last(host_vaddr, host_size);
-
-			return true;
-		}
-		if (vaddr > b.map_vaddr && vaddr + size < b.map_vaddr + b.map_size) {
-			*gpu_mode = b.gpu_mode;
-
+		} else if (vaddr > b.map_vaddr && vaddr + size < b.map_vaddr + b.map_size) {
 			AllocatedBlock right = b;
 			right.start_addr += (vaddr + size) - b.map_vaddr;
 			right.size      = b.map_vaddr + b.map_size - (vaddr + size);
 			right.map_size  = right.size;
 			right.map_vaddr = vaddr + size;
-
 			b.size     = vaddr - b.map_vaddr;
 			b.map_size = b.size;
 			m_mappings.push_back(right);
-			return true;
-		}
-		if (vaddr == b.map_vaddr && size < b.map_size) {
-			*gpu_mode = b.gpu_mode;
-
+		} else if (vaddr == b.map_vaddr) {
 			b.start_addr += size;
 			b.size -= size;
 			b.map_vaddr += size;
 			b.map_size -= size;
-			return true;
-		}
-		if (vaddr > b.map_vaddr && vaddr + size == b.map_vaddr + b.map_size) {
-			*gpu_mode = b.gpu_mode;
-
+		} else {
 			b.size     = vaddr - b.map_vaddr;
 			b.map_size = b.size;
-			return true;
 		}
-		index++;
+		ReclaimAutomatic(phys_addr, size);
+		return true;
 	}
 
 	return false;
@@ -2726,10 +2800,8 @@ int KYTY_SYSV_ABI KernelDirectMemoryQuery(int64_t offset, int flags, void* info,
 	return OK;
 }
 
-int KYTY_SYSV_ABI KernelAllocateDirectMemory(int64_t search_start, int64_t search_end, size_t len,
-                                             size_t alignment, int memory_type,
-                                             int64_t* phys_addr_out) {
-	PRINT_NAME();
+int AllocateDirectMemory(int64_t search_start, int64_t search_end, size_t len, size_t alignment,
+                         int memory_type, int64_t* phys_addr_out, bool automatic) {
 
 	std::lock_guard<std::recursive_mutex> memory_operation_lock(g_memory_operation_mutex);
 
@@ -2748,7 +2820,10 @@ int KYTY_SYSV_ABI KernelAllocateDirectMemory(int64_t search_start, int64_t searc
 	}
 
 	uint64_t addr = 0;
-	if (!g_physical_memory->Alloc(search_start, search_end, len, alignment, &addr, memory_type)) {
+	const auto kind = automatic ? PhysicalMemory::AllocationKind::Automatic
+	                            : PhysicalMemory::AllocationKind::Direct;
+	if (!g_physical_memory->Alloc(search_start, search_end, len, alignment, &addr, memory_type,
+	                              kind)) {
 		LOGF_COLOR(Log::Color::Red, "\t[Fail]\n");
 		return KERNEL_ERROR_EAGAIN;
 	}
@@ -2760,6 +2835,14 @@ int KYTY_SYSV_ABI KernelAllocateDirectMemory(int64_t search_start, int64_t searc
 	return OK;
 }
 
+int KYTY_SYSV_ABI KernelAllocateDirectMemory(int64_t search_start, int64_t search_end, size_t len,
+                                             size_t alignment, int memory_type,
+                                             int64_t* phys_addr_out) {
+	PRINT_NAME();
+	return AllocateDirectMemory(search_start, search_end, len, alignment, memory_type,
+	                            phys_addr_out);
+}
+
 int KYTY_SYSV_ABI KernelAllocateMainDirectMemory(size_t len, size_t alignment, int memory_type,
                                                  int64_t* phys_addr_out) {
 	PRINT_NAME();
@@ -3082,6 +3165,36 @@ int KYTY_SYSV_ABI KernelMapDirectMemory2(void** addr, size_t len, int type, int
 	return ret;
 }
 
+int MapAutomaticMemory(uint64_t vaddr, size_t size, int type, int prot) {
+	std::lock_guard<std::recursive_mutex> memory_operation_lock(g_memory_operation_mutex);
+	constexpr uint64_t                    page_size = 0x4000;
+	if (size == 0 || (vaddr & (page_size - 1)) != 0 || (size & (page_size - 1)) != 0 ||
+	    vaddr < kExtendedMemoryBase || vaddr >= kExtendedMemoryBase + kExtendedMemorySize ||
+	    size > kExtendedMemoryBase + kExtendedMemorySize - vaddr) {
+		return KERNEL_ERROR_EINVAL;
+	}
+
+	PhysicalMemory::PhysicalRanges ranges;
+	if (!g_physical_memory->ReserveAutomatic(size, &ranges)) {
+		return KERNEL_ERROR_EAGAIN;
+	}
+	uint64_t mapped = 0;
+	for (const auto& [offset, length]: ranges) {
+		void*     address = reinterpret_cast<void*>(vaddr + mapped);
+		const int result  = KernelMapDirectMemory2(&address, length, type, prot, 0x10,
+		                                           static_cast<int64_t>(offset), page_size);
+		if (result != OK) {
+			if (mapped != 0) {
+				EXIT_IF(KernelMunmap(vaddr, mapped) != OK);
+			}
+			g_physical_memory->RestoreAutomatic(ranges);
+			return result;
+		}
+		mapped += length;
+	}
+	return OK;
+}
+
 int KYTY_SYSV_ABI KernelMapNamedDirectMemory(void** addr, size_t len, int prot, int flags,
                                              int64_t direct_memory_start, size_t alignment,
                                              const char* name) {
@@ -3892,7 +4005,7 @@ int KYTY_SYSV_ABI KernelMemoryPoolExpand(int64_t search_start, int64_t search_en
 	uint64_t   phys_addr           = 0;
 	if (!g_physical_memory->Alloc(static_cast<uint64_t>(search_start),
 	                              static_cast<uint64_t>(search_end), len, effective_alignment,
-	                              &phys_addr, 0, true)) {
+	                              &phys_addr, 0, PhysicalMemory::AllocationKind::Pooled)) {
 		return KERNEL_ERROR_ENOMEM;
 	}
 

src/kernel/memory.h

@@ -24,6 +24,8 @@ struct Lifecycle {
 using callback_func_t = void (*)(uintptr_t addr, size_t size);
 
 constexpr uint32_t KERNEL_MAXIMUM_NAME_LENGTH = 32;
+constexpr uint64_t kExtendedMemoryBase       = 0x080000000000ull;
+constexpr uint64_t kExtendedMemorySize       = 512ull * 1024 * 1024 * 1024;
 
 struct VirtualQueryInfo {
 	uintptr_t start;
@@ -107,6 +109,9 @@ static_assert(sizeof(KernelMemoryPoolBlockStats) == 16,
 
 void                   RegisterCallbacks(callback_func_t alloc_func, callback_func_t free_func);
 void                   SetFlexibleMemorySize(uint64_t size);
+int AllocateDirectMemory(int64_t search_start, int64_t search_end, size_t size, size_t alignment,
+                         int memory_type, int64_t* phys_addr_out, bool automatic = false);
+int MapAutomaticMemory(uint64_t vaddr, size_t size, int type, int prot);
 bool                   TryWriteBacking(uint64_t vaddr, const void* data, uint64_t size);
 bool                   TryReadBacking(uint64_t vaddr, void* data, uint64_t size);
 bool                   TryReadGpuCleanBacking(uint64_t vaddr, void* data, uint64_t size);

src/kernel/memoryAddressSpace.inc

@@ -3,7 +3,6 @@
 
 static constexpr uint64_t HOST_SYSTEM_MANAGED_MIN  = 0x0000040000ull;
 static constexpr uint64_t HOST_SYSTEM_MANAGED_MAX  = 0x07ffffbfffull;
-static constexpr uint64_t HOST_SYSTEM_RESERVED_MIN = HOST_SYSTEM_MANAGED_MAX + 1u;
 #if defined(__APPLE__) && (defined(__x86_64__) || defined(_M_X64))
 static constexpr uint64_t HOST_SYSTEM_RESERVED_MAX = 0x0fbfffffffull;
 static constexpr uint64_t HOST_USER_MIN            = 0x7000000000ull;
@@ -1077,51 +1076,51 @@ private:
 #endif
 
 	void ReserveGuestRegions() {
+		const std::array<std::pair<uint64_t, uint64_t>, 3> regions {{
+		    {HOST_SYSTEM_MANAGED_MIN, HOST_SYSTEM_RESERVED_MAX + 1u},
+		    {HOST_USER_MIN, HOST_USER_MAX + 1u},
+		    {kExtendedMemoryBase, kExtendedMemoryBase + kExtendedMemorySize},
+		}};
+
 #if KYTY_PLATFORM == KYTY_PLATFORM_WINDOWS
 		constexpr uint64_t MinimumRegionSize = 0x1000000;
 		const auto         process           = GetCurrentProcess();
-		const auto         guest_end         = HOST_USER_MAX + 1u;
-		auto               next_addr         = HOST_SYSTEM_MANAGED_MIN;
-
-		while (next_addr < guest_end) {
-			MEMORY_BASIC_INFORMATION info {};
-			if (VirtualQuery(reinterpret_cast<const void*>(next_addr), &info, sizeof(info)) == 0) {
-				EXIT("failed to inspect guest address space at 0x%016" PRIx64 "\n", next_addr);
-			}
-
-			const auto region_base = reinterpret_cast<uint64_t>(info.BaseAddress);
-			const auto region_end =
-			    std::min<uint64_t>(guest_end, region_base + static_cast<uint64_t>(info.RegionSize));
-			const auto reserve_start =
-			    AlignUp(std::max<uint64_t>(region_base, HOST_SYSTEM_MANAGED_MIN), m_granularity);
-			const auto reserve_end = AlignDown(region_end, m_granularity);
-
-			if (info.State == MEM_FREE && reserve_end > reserve_start &&
-			    reserve_end - reserve_start > MinimumRegionSize) {
-				auto* ptr = VirtualAlloc2(
-				    process, reinterpret_cast<void*>(reserve_start), reserve_end - reserve_start,
-				    MEM_RESERVE | MEM_RESERVE_PLACEHOLDER, PAGE_NOACCESS, nullptr, 0);
-				if (reinterpret_cast<uint64_t>(ptr) != reserve_start) {
-					EXIT("failed to reserve guest address space at 0x%016" PRIx64
-					     ", size 0x%016" PRIx64 "\n",
-					     reserve_start, reserve_end - reserve_start);
+		for (const auto& [guest_start, guest_end]: regions) {
+			auto next_addr = guest_start;
+			while (next_addr < guest_end) {
+				MEMORY_BASIC_INFORMATION info {};
+				if (VirtualQuery(reinterpret_cast<const void*>(next_addr), &info, sizeof(info)) == 0) {
+					EXIT("failed to inspect guest address space at 0x%016" PRIx64 "\n", next_addr);
 				}
-				AddFreeUnlocked(reserve_start, reserve_end - reserve_start);
-				m_owned.emplace_back(reserve_start, reserve_end - reserve_start);
-			}
 
-			if (region_end <= next_addr) {
-				EXIT("guest address-space scan did not advance at 0x%016" PRIx64 "\n", next_addr);
+				const auto region_base = reinterpret_cast<uint64_t>(info.BaseAddress);
+				const auto region_end = std::min<uint64_t>(
+				    guest_end, region_base + static_cast<uint64_t>(info.RegionSize));
+				const auto reserve_start =
+				    AlignUp(std::max(region_base, guest_start), m_granularity);
+				const auto reserve_end = AlignDown(region_end, m_granularity);
+
+				if (info.State == MEM_FREE && reserve_end > reserve_start &&
+				    reserve_end - reserve_start > MinimumRegionSize) {
+					auto* ptr = VirtualAlloc2(
+					    process, reinterpret_cast<void*>(reserve_start), reserve_end - reserve_start,
+					    MEM_RESERVE | MEM_RESERVE_PLACEHOLDER, PAGE_NOACCESS, nullptr, 0);
+					if (reinterpret_cast<uint64_t>(ptr) != reserve_start) {
+						EXIT("failed to reserve guest address space at 0x%016" PRIx64
+						     ", size 0x%016" PRIx64 "\n",
+						     reserve_start, reserve_end - reserve_start);
+					}
+					AddFreeUnlocked(reserve_start, reserve_end - reserve_start);
+					m_owned.emplace_back(reserve_start, reserve_end - reserve_start);
+				}
+
+				if (region_end <= next_addr) {
+					EXIT("guest address-space scan did not advance at 0x%016" PRIx64 "\n", next_addr);
+				}
+				next_addr = region_end;
 			}
-			next_addr = region_end;
 		}
 #else
-		const std::array<std::pair<uint64_t, uint64_t>, 3> regions {{
-		    {HOST_SYSTEM_MANAGED_MIN, HOST_SYSTEM_MANAGED_MAX + 1u},
-		    {HOST_SYSTEM_RESERVED_MIN, HOST_SYSTEM_RESERVED_MAX + 1u},
-		    {HOST_USER_MIN, HOST_USER_MAX + 1u},
-		}};
-
 #if KYTY_PLATFORM == KYTY_PLATFORM_LINUX && !defined(__APPLE__)
 		std::vector<std::pair<uint64_t, uint64_t>> occupied;
 		FILE*                                      maps = fopen("/proc/self/maps", "r");

src/libs/dialog.cpp

@@ -6,6 +6,7 @@
 #include "libs/errno.h"
 #include "libs/libs.h"
 
+#include <atomic>
 #include <cstdio>
 #include <cstring>
 #include <mutex>
@@ -306,6 +307,7 @@ LIB_NAME("SaveDataDialog", "SaveDataDialog");
 
 constexpr int SAVE_STATUS_NONE        = 0;
 constexpr int SAVE_STATUS_INITIALIZED = 1;
+constexpr int SAVE_STATUS_RUNNING     = 2;
 constexpr int SAVE_STATUS_FINISHED    = 3;
 constexpr int SAVE_RESULT_OK          = 0;
 constexpr int SAVE_BUTTON_ID_OK       = 1;
@@ -351,18 +353,20 @@ struct SaveDataDialogResult {
 	uint8_t  reserved[32];
 };
 
-static int   g_save_status    = SAVE_STATUS_NONE;
-static int   g_save_mode      = 0;
-static void* g_save_user_data = nullptr;
-static char  g_save_dir_name[sizeof(SaveDataDirName::data)] {};
+static std::atomic<int> g_save_status         = SAVE_STATUS_NONE;
+static bool             g_save_running_polled = false;
+static int              g_save_mode           = 0;
+static void*            g_save_user_data      = nullptr;
+static char             g_save_dir_name[sizeof(SaveDataDirName::data)] {};
 
 int KYTY_SYSV_ABI SaveDataDialogInitialize() {
 	PRINT_NAME();
 
-	g_save_status      = SAVE_STATUS_INITIALIZED;
-	g_save_mode        = 0;
-	g_save_user_data   = nullptr;
-	g_save_dir_name[0] = '\0';
+	g_save_status         = SAVE_STATUS_INITIALIZED;
+	g_save_running_polled = false;
+	g_save_mode           = 0;
+	g_save_user_data      = nullptr;
+	g_save_dir_name[0]    = '\0';
 
 	return OK;
 }
@@ -376,6 +380,14 @@ int KYTY_SYSV_ABI SaveDataDialogGetStatus() {
 int KYTY_SYSV_ABI SaveDataDialogUpdateStatus() {
 	PRINT_NAME();
 
+	// Some games require a RUNNING update before FINISHED.
+	if (g_save_status == SAVE_STATUS_RUNNING) {
+		if (g_save_running_polled) {
+			g_save_status = SAVE_STATUS_FINISHED;
+		}
+		g_save_running_polled = true;
+	}
+
 	return g_save_status;
 }
 
@@ -431,7 +443,8 @@ int KYTY_SYSV_ABI SaveDataDialogOpen(const void* param) {
 		}
 	}
 
-	g_save_status = SAVE_STATUS_FINISHED;
+	g_save_status         = SAVE_STATUS_RUNNING;
+	g_save_running_polled = false;
 
 	return OK;
 }
@@ -441,7 +454,8 @@ int KYTY_SYSV_ABI SaveDataDialogClose(const void* close_param) {
 
 	LOGF("\t close_param = 0x%016" PRIx64 "\n", reinterpret_cast<uint64_t>(close_param));
 
-	g_save_status = SAVE_STATUS_FINISHED;
+	g_save_status         = SAVE_STATUS_FINISHED;
+	g_save_running_polled = false;
 
 	return OK;
 }
@@ -455,10 +469,11 @@ int KYTY_SYSV_ABI SaveDataDialogIsReadyToDisplay() {
 int KYTY_SYSV_ABI SaveDataDialogTerminate() {
 	PRINT_NAME();
 
-	g_save_status      = SAVE_STATUS_NONE;
-	g_save_mode        = 0;
-	g_save_user_data   = nullptr;
-	g_save_dir_name[0] = '\0';
+	g_save_status         = SAVE_STATUS_NONE;
+	g_save_running_polled = false;
+	g_save_mode           = 0;
+	g_save_user_data      = nullptr;
+	g_save_dir_name[0]    = '\0';
 
 	return 0;
 }

src/libs/libAmpr.cpp

@@ -685,8 +685,6 @@ constexpr int      PROT_AMPR_READ                = 0x40;
 constexpr int      PROT_AMPR_WRITE               = 0x80;
 constexpr int      PROT_ACP_READ                 = 0x100;
 constexpr int      PROT_ACP_WRITE                = 0x200;
-constexpr uint64_t AMM_VA_START                  = 0x0000001000000000ull;
-constexpr uint64_t AMM_VA_SIZE                   = 0x0000001000000000ull;
 constexpr uint64_t APR_MAX_READ_LENGTH           = 0x0000000100000000ull;
 constexpr uint64_t APR_MAX_FILE_OFFSET           = 0x0000010000000000ull;
 constexpr uint64_t APR_MAX_APP_ADDRESS           = 0x0000f00000000000ull;
@@ -744,19 +742,6 @@ struct CommandBufferState {
 	uint64_t                         gather_scatter_file_offset = 0;
 };
 
-struct AmmAutoPoolRange {
-	uint64_t start = 0;
-	uint64_t size  = 0;
-	uint64_t used  = 0;
-};
-
-struct AmmVirtualAddressRanges {
-	uint64_t va_start          = 0;
-	uint64_t va_end            = 0;
-	uint64_t multimap_va_start = 0;
-	uint64_t multimap_va_end   = 0;
-};
-
 struct AmmUsageStatsData {
 	uint64_t size_in_bytes                                    = 0;
 	uint16_t num_page_table_pool_entries                      = 0;
@@ -770,8 +755,6 @@ static_assert(sizeof(AmmUsageStatsData) == 0x18);
 static std::mutex                                       g_command_buffer_mutex;
 static std::unordered_map<uint64_t, CommandBufferState> g_command_buffers;
 static std::unordered_map<uint64_t, uint64_t>           g_command_buffer_aliases;
-static std::mutex                                       g_amm_auto_pool_mutex;
-static std::vector<AmmAutoPoolRange>                    g_amm_auto_pool;
 using CommandBufferIterator = std::unordered_map<uint64_t, CommandBufferState>::iterator;
 
 static bool HasQueuedCommands(const CommandBufferState& state) {
@@ -1227,48 +1210,20 @@ static int NormalizeAmmProtection(int prot) {
 	return normalized;
 }
 
-static bool AllocateAmmAutoDirectMemory(uint64_t size, int memory_type, uint64_t* dmem_offset) {
-	if (dmem_offset == nullptr || size == 0) {
-		return false;
-	}
-
-	std::scoped_lock lock(g_amm_auto_pool_mutex);
-
-	for (auto& range: g_amm_auto_pool) {
-		const auto aligned_used = (range.used + (AMM_PAGE_SIZE - 1u)) & ~(AMM_PAGE_SIZE - 1u);
-		if (aligned_used <= range.size && size <= range.size - aligned_used) {
-			*dmem_offset = range.start + aligned_used;
-			range.used   = aligned_used + size;
-			return true;
-		}
-	}
-
-	int64_t allocated = 0;
-	if (LibKernel::Memory::KernelAllocateDirectMemory(
-	        0, LibKernel::Memory::KernelGetDirectMemorySize(), size, AMM_PAGE_SIZE, memory_type,
-	        &allocated) != OK) {
-		return false;
-	}
-
-	*dmem_offset = static_cast<uint64_t>(allocated);
-	return true;
-}
-
 static int ExecuteAmmMapCommand(const CommandBufferState::AmmMapCommand& command) {
 	if (!ValidateAmmMapArgs(command.va, command.size)) {
 		return LibKernel::KERNEL_ERROR_EINVAL;
 	}
 
-	uint64_t dmem_offset = command.dmem_offset;
-	if (command.kind == AmmCommandKind::MapAuto &&
-	    !AllocateAmmAutoDirectMemory(command.size, command.type, &dmem_offset)) {
-		return LibKernel::KERNEL_ERROR_EAGAIN;
+	if (command.kind == AmmCommandKind::MapAuto) {
+		return LibKernel::Memory::MapAutomaticMemory(command.va, command.size, command.type,
+		                                             NormalizeAmmProtection(command.prot));
 	}
 
 	void* addr = reinterpret_cast<void*>(command.va);
 	return LibKernel::Memory::KernelMapDirectMemory2(
 	    &addr, command.size, command.type, NormalizeAmmProtection(command.prot), AMM_MAP_FIXED,
-	    static_cast<int64_t>(dmem_offset), AMM_PAGE_SIZE);
+	    static_cast<int64_t>(command.dmem_offset), AMM_PAGE_SIZE);
 }
 
 static bool AppendAmmMapRecord(uint64_t                                 command_buffer,
@@ -2360,25 +2315,15 @@ static int KYTY_SYSV_ABI AmmGiveDirectMemory(int64_t search_start, int64_t searc
                                              int64_t* dmem_offset) {
 	PRINT_NAME();
 
-	if (dmem_offset == nullptr || size == 0 ||
-	    (usage != AMM_USAGE_DIRECT && usage != AMM_USAGE_AUTO)) {
+	constexpr uint64_t block_size = 0x200000;
+	if (dmem_offset == nullptr || size == 0 || (size & (block_size - 1)) != 0 ||
+	    (align & (block_size - 1)) != 0 || (usage != AMM_USAGE_DIRECT && usage != AMM_USAGE_AUTO)) {
 		return LibKernel::KERNEL_ERROR_EINVAL;
 	}
 
-	int64_t allocated = 0;
-	int     result = LibKernel::Memory::KernelAllocateDirectMemory(search_start, search_end, size,
-	                                                               align, 0, &allocated);
-	if (result != OK) {
-		return result;
-	}
-
-	*dmem_offset = allocated;
-	if (usage == AMM_USAGE_AUTO) {
-		std::scoped_lock lock(g_amm_auto_pool_mutex);
-		g_amm_auto_pool.push_back(AmmAutoPoolRange {static_cast<uint64_t>(allocated), size, 0});
-	}
-
-	return OK;
+	return LibKernel::Memory::AllocateDirectMemory(search_start, search_end, size,
+	                                               align != 0 ? align : block_size, 0, dmem_offset,
+	                                               usage == AMM_USAGE_AUTO);
 }
 
 static void KYTY_SYSV_ABI AmmGetVirtualAddressRanges(uint64_t* va_start, uint64_t* va_end,
@@ -2386,19 +2331,18 @@ static void KYTY_SYSV_ABI AmmGetVirtualAddressRanges(uint64_t* va_start, uint64_
                                                      uint64_t* multimap_va_end) {
 	PRINT_NAME();
 
+	constexpr auto end = LibKernel::Memory::kExtendedMemoryBase + LibKernel::Memory::kExtendedMemorySize;
 	if (va_start != nullptr) {
-		AprShared::WriteGuest(reinterpret_cast<uint64_t>(va_start), AMM_VA_START);
+		AprShared::WriteGuest(reinterpret_cast<uint64_t>(va_start), LibKernel::Memory::kExtendedMemoryBase);
 	}
 	if (va_end != nullptr) {
-		AprShared::WriteGuest(reinterpret_cast<uint64_t>(va_end), AMM_VA_START + AMM_VA_SIZE);
+		AprShared::WriteGuest(reinterpret_cast<uint64_t>(va_end), end);
 	}
 	if (multimap_va_start != nullptr) {
-		AprShared::WriteGuest(reinterpret_cast<uint64_t>(multimap_va_start),
-		                      AMM_VA_START + AMM_VA_SIZE / 2u);
+		AprShared::WriteGuest(reinterpret_cast<uint64_t>(multimap_va_start), end);
 	}
 	if (multimap_va_end != nullptr) {
-		AprShared::WriteGuest(reinterpret_cast<uint64_t>(multimap_va_end),
-		                      AMM_VA_START + AMM_VA_SIZE);
+		AprShared::WriteGuest(reinterpret_cast<uint64_t>(multimap_va_end), end);
 	}
 }
 

src/libs/libCes.cpp

@@ -7,6 +7,16 @@
 #include <array>
 #include <cstring>
 
+#if defined(_WIN32)
+#ifndef WIN32_LEAN_AND_MEAN
+#define WIN32_LEAN_AND_MEAN
+#endif
+#ifndef NOMINMAX
+#define NOMINMAX
+#endif
+#include <windows.h>
+#endif
+
 namespace Libs {
 
 namespace LibCes {
@@ -213,6 +223,54 @@ static int KYTY_SYSV_ABI CesMbcsUcsContextInit(CesContext* context, const CesPro
 	return 0;
 }
 
+#if defined(_WIN32)
+
+static size_t ConvertCp932ToUtf8(const char** inbuf, size_t* inbytesleft, char** outbuf,
+                                 size_t* outbytesleft) {
+	const auto* in = reinterpret_cast<const uint8_t*>(*inbuf);
+	while (*inbytesleft != 0) {
+		const uint8_t lead = in[0];
+		// Win32 accepts 0x80 as U+0080; the SJis1997 profile rejects this byte.
+		if (lead == 0x80) {
+			*inbuf = reinterpret_cast<const char*>(in);
+			return SDL_ICONV_EILSEQ;
+		}
+		const bool is_lead_byte = (lead >= 0x81 && lead <= 0x9f) || (lead >= 0xe0 && lead <= 0xfc);
+		const size_t char_len   = is_lead_byte ? 2 : 1;
+		if (*inbytesleft < char_len) {
+			*inbuf = reinterpret_cast<const char*>(in);
+			return SDL_ICONV_EINVAL;
+		}
+
+		wchar_t   wide[2];
+		const int wide_len =
+		    MultiByteToWideChar(932, MB_ERR_INVALID_CHARS, reinterpret_cast<const char*>(in),
+		                        static_cast<int>(char_len), wide, 2);
+		if (wide_len <= 0) {
+			*inbuf = reinterpret_cast<const char*>(in);
+			return SDL_ICONV_EILSEQ;
+		}
+
+		char      utf8[8];
+		const int utf8_len =
+		    WideCharToMultiByte(CP_UTF8, 0, wide, wide_len, utf8, sizeof(utf8), nullptr, nullptr);
+		if (utf8_len <= 0 || static_cast<size_t>(utf8_len) > *outbytesleft) {
+			*inbuf = reinterpret_cast<const char*>(in);
+			return SDL_ICONV_E2BIG;
+		}
+
+		std::memcpy(*outbuf, utf8, static_cast<size_t>(utf8_len));
+		*outbuf += utf8_len;
+		*outbytesleft -= static_cast<size_t>(utf8_len);
+		in += char_len;
+		*inbytesleft -= char_len;
+	}
+	*inbuf = reinterpret_cast<const char*>(in);
+	return 0;
+}
+
+#endif
+
 static int ConvertMbcsToUtf8(CesContext* context, const uint8_t* source, uint32_t source_max,
                              uint32_t* source_len, uint8_t* destination, uint32_t destination_max,
                              uint32_t* destination_len, bool measure) {
@@ -246,8 +304,10 @@ static int ConvertMbcsToUtf8(CesContext* context, const uint8_t* source, uint32_
 		++source_size;
 	}
 	const bool bounded_end = source_max != 0 && source_size == source_max;
-	const auto converter   = SDL_iconv_open("UTF-8", context->profile->encoding);
+#if !defined(_WIN32)
+	const auto converter = SDL_iconv_open("UTF-8", context->profile->encoding);
 	EXIT_IF(converter == reinterpret_cast<SDL_iconv_t>(-1));
+#endif
 
 	const char*           input      = reinterpret_cast<const char*>(source);
 	size_t                input_left = source_size;
@@ -258,7 +318,11 @@ static int ConvertMbcsToUtf8(CesContext* context, const uint8_t* source, uint32_
 		char* output = measure ? scratch.data() : reinterpret_cast<char*>(destination) + produced;
 		const size_t capacity    = measure ? scratch.size() : destination_max - 1 - produced;
 		size_t       output_left = capacity;
-		const auto   status      = SDL_iconv(converter, &input, &input_left, &output, &output_left);
+#if defined(_WIN32)
+		const auto status = ConvertCp932ToUtf8(&input, &input_left, &output, &output_left);
+#else
+		const auto status = SDL_iconv(converter, &input, &input_left, &output, &output_left);
+#endif
 		produced += capacity - output_left;
 		if (status == SDL_ICONV_E2BIG) {
 			if (measure) {
@@ -277,7 +341,9 @@ static int ConvertMbcsToUtf8(CesContext* context, const uint8_t* source, uint32_
 		}
 		break;
 	} while (input_left != 0);
+#if !defined(_WIN32)
 	SDL_iconv_close(converter);
+#endif
 
 	if (!measure) {
 		destination[produced] = 0;

src/libs/ngs2.cpp

@@ -1435,6 +1435,11 @@ static void Ngs2FinishBlock(Ngs2VoiceInternal& voice) {
 		}
 		++block.repeated_count;
 		block.cursor = 0;
+		if (voice.decoder != nullptr) {
+			block.data_cursor = 0;
+			voice.compressed_input.clear();
+			voice.decoder->Reset();
+		}
 	}
 	const Ngs2VoiceCallbackInfo info {voice.callback_data,
 	                                  reinterpret_cast<uintptr_t>(&voice),
@@ -2396,8 +2401,7 @@ int KYTY_SYSV_ABI Ngs2VoiceControl(uintptr_t voice_handle, const Ngs2VoiceParamH
 								block.num_samples      = 0;
 							}
 							EXIT_NOT_IMPLEMENTED(block.num_repeats != 0 &&
-							                     (voice->rack->type != Ngs2RackType::Sampler ||
-							                      voice->decoder != nullptr));
+							                     voice->rack->type != Ngs2RackType::Sampler);
 							EXIT_NOT_IMPLEMENTED(
 							    voice->decoder == nullptr &&
 							    (uint64_t(block.num_skip_samples) + block.num_samples) *

tests/CesTests.cpp

@@ -0,0 +1,127 @@
+#include "libs/libCes.cpp"
+
+#include <string>
+#include <vector>
+
+namespace Loader {
+void SymbolDatabase::Add(const SymbolResolve&, uint64_t, const std::string&) {}
+namespace Timer {
+double GetTimeMs() {
+	return 0.0;
+}
+} // namespace Timer
+} // namespace Loader
+
+namespace {
+using namespace Libs::LibCes;
+
+int failures = 0;
+
+#define CHECK(condition)                                                                           \
+	do {                                                                                           \
+		if (!(condition)) {                                                                        \
+			std::fprintf(stderr, "CesTests:%d: %s\n", __LINE__, #condition);                       \
+			++failures;                                                                            \
+		}                                                                                          \
+	} while (false)
+
+void CheckConversion(CesContext* context, const std::vector<uint8_t>& source,
+                     const std::string& expected) {
+	std::vector<uint8_t> output(expected.size() + 2, 0xcc);
+	uint32_t             consumed = 0;
+	uint32_t             produced = 0;
+	CHECK(CesMbcsStrToUtf8Str(context, source.data(), 0, &consumed, output.data(),
+	                          static_cast<uint32_t>(expected.size() + 1), &produced) == 0);
+	CHECK(consumed == source.size() - 1);
+	CHECK(produced == expected.size());
+	CHECK(std::memcmp(output.data(), expected.data(), expected.size()) == 0);
+	CHECK(output[expected.size()] == 0);
+	CHECK(output.back() == 0xcc);
+	CHECK(CesMbcsStrGetUtf8Len(context, source.data(), 0, &consumed, &produced) == 0);
+	CHECK(consumed == source.size() - 1);
+	CHECK(produced == expected.size());
+}
+
+void TestMappingsAndMeasurement(CesContext* context) {
+	CheckConversion(context, {0}, "");
+	CheckConversion(context, {'S', 'a', 'v', 'e', ' ', 0x93, 0xfa, 0x96, 0x7b, 0xb6, 0},
+	                "Save \xe6\x97\xa5\xe6\x9c\xac\xef\xbd\xb6");
+	// CP932 maps this character to U+FF5E, unlike strict Shift-JIS (U+301C).
+	CheckConversion(context, {0x81, 0x60, 0}, "\xef\xbd\x9e");
+	std::vector<uint8_t> source;
+	std::string          expected;
+	for (int i = 0; i < 200; ++i) {
+		source.insert(source.end(), {0x82, 0xa0});
+		expected += "\xe3\x81\x82";
+	}
+	source.push_back(0);
+	CheckConversion(context, source, expected);
+}
+
+void TestPartialOutput(CesContext* context) {
+	const uint8_t source[] = {'A', 0x82, 0xa0, 'B', 0};
+	for (uint32_t capacity = 1; capacity <= 7; ++capacity) {
+		std::array<uint8_t, 8> output;
+		output.fill(0xcc);
+		uint32_t  consumed = 99;
+		uint32_t  produced = 99;
+		const int status =
+		    CesMbcsStrToUtf8Str(context, source, 0, &consumed, output.data(), capacity, &produced);
+		const uint32_t expected_consumed = capacity == 1   ? 0
+		                                   : capacity < 5  ? 1
+		                                   : capacity == 5 ? 3
+		                                                   : 4;
+		const uint32_t expected_produced = capacity == 1   ? 0
+		                                   : capacity < 5  ? 1
+		                                   : capacity == 5 ? 4
+		                                                   : 5;
+		CHECK(status == (capacity < 6 ? CES_ERROR_DST_BUFFER_END : 0));
+		CHECK(consumed == expected_consumed);
+		CHECK(produced == expected_produced);
+		CHECK(std::memcmp(output.data(),
+		                  "A\xe3\x81\x82"
+		                  "B",
+		                  expected_produced) == 0);
+		CHECK(output[expected_produced] == 0);
+		CHECK(output[capacity] == 0xcc);
+	}
+}
+
+void TestErrors(CesContext* context) {
+	struct Case {
+		std::vector<uint8_t> source;
+		uint32_t             limit;
+		int                  error;
+	};
+	const Case cases[] = {
+	    {{'A', 0x80, 0}, 0, CES_ERROR_INVALID_ENCODE},
+	    {{'A', 0x82, 0}, 0, CES_ERROR_INVALID_ENCODE},
+	    {{'A', 0x82, 0xa0, 0}, 2, CES_ERROR_SRC_BUFFER_END},
+	    {{'A', 0x82, 0x20, 0}, 0, CES_ERROR_INVALID_ENCODE},
+	    {{'A', 0x81, 0xad, 0}, 0, CES_ERROR_UNASSIGNED_CODE},
+	};
+	for (const auto& test: cases) {
+		std::array<uint8_t, 16> output {};
+		uint32_t                consumed = 99;
+		uint32_t                produced = 99;
+		CHECK(CesMbcsStrToUtf8Str(context, test.source.data(), test.limit, &consumed, output.data(),
+		                          output.size(), &produced) == test.error);
+		CHECK(consumed == 1 && produced == 1);
+		CHECK(output[0] == 'A' && output[1] == 0);
+		CHECK(CesMbcsStrGetUtf8Len(context, test.source.data(), test.limit, &consumed, &produced) ==
+		      test.error);
+		CHECK(consumed == 1 && produced == 1);
+	}
+}
+} // namespace
+
+int main() {
+	CesProfile profile {};
+	CesContext context {};
+	CHECK(CesUcsProfileInitSJis1997Cp932(&profile) == &profile);
+	CHECK(CesMbcsUcsContextInit(&context, &profile) == 0);
+	TestMappingsAndMeasurement(&context);
+	TestPartialOutput(&context);
+	TestErrors(&context);
+	return failures == 0 ? 0 : 1;
+}

tests/ImagePageTableTests.cpp

@@ -9,9 +9,9 @@
 namespace {
 
 using Owners     = std::vector<uint32_t>;
-using Table      = Libs::Graphics::MultiLevelPageTable<Owners>;
+using Table      = Libs::Graphics::MultiLevelPageTable<Owners, 20, 44, 14>;
 using PageOwners = Libs::Graphics::InlinePageOwnerList<uint32_t, 16>;
-using OwnerTable = Libs::Graphics::MultiLevelPageTable<PageOwners, 20, 40, 10>;
+using OwnerTable = Libs::Graphics::MultiLevelPageTable<PageOwners, 20, 44, 14>;
 
 void Check(bool value, const char* text) {
 	if (!value) {

tests/KernelFileSystemTests.cpp

@@ -132,7 +132,7 @@ void CheckMountRoot(const std::filesystem::path &root) {
   FileSystem::Mount(root, "/app0");
   Check(FileSystem::GetRealFilename("/app0/rpf.cache") == root / "rpf.cache",
         "resolve mount descendant");
-  Check(FileSystem::GetRealFilename("/app01/rpf.cache") == "/app01/rpf.cache",
+  Check(FileSystem::GetRealFilename("/app01/rpf.cache").empty(),
         "mount prefix must end at a path component");
 
   for (const char *path : {"/app0", "/app0/"}) {
@@ -162,18 +162,58 @@ void CheckMountRoot(const std::filesystem::path &root) {
     }
   }
   FileSystem::Umount("/app0");
-  Check(FileSystem::GetRealFilename("/app0/rpf.cache") == "/app0/rpf.cache",
+  Check(FileSystem::GetRealFilename("/app0/rpf.cache").empty(),
         "unmount by guest path");
   for (const auto &folder : {root, root / ""}) {
     for (const auto &host : {root, root / ""}) {
       FileSystem::Mount(folder, "/app0");
       FileSystem::Umount(Common::PathToGenericString(host));
-      Check(FileSystem::GetRealFilename("/app0/rpf.cache") == "/app0/rpf.cache",
+      Check(FileSystem::GetRealFilename("/app0/rpf.cache").empty(),
             "unmount by host path with or without trailing separator");
     }
   }
 }
 
+void CheckUnmappedPaths(const std::filesystem::path &root) {
+  const auto host_file = root / "host-only.dat";
+  const auto host_path = Common::PathToGenericString(host_file);
+  Common::File fixture;
+  Check(fixture.Create(host_file), "create unmapped host file");
+  fixture.Close();
+
+  FileSystem::FileStat stat {};
+  Check(FileSystem::GetRealFilename(host_path).empty() &&
+            FileSystem::KernelOpen(host_path.c_str(), 0, 0) ==
+                Libs::LibKernel::KERNEL_ERROR_ENOENT &&
+            FileSystem::KernelStat(host_path.c_str(), &stat) ==
+                Libs::LibKernel::KERNEL_ERROR_ENOENT &&
+            FileSystem::KernelCheckReachability(host_path.c_str()) ==
+                Libs::LibKernel::KERNEL_ERROR_ENOENT,
+        "existing host files are absent from the guest namespace");
+  Check(FileSystem::KernelUnlink(host_path.c_str()) ==
+            Libs::LibKernel::KERNEL_ERROR_ENOENT &&
+            FileSystem::KernelRmdir(Common::PathToGenericString(root).c_str()) ==
+                Libs::LibKernel::KERNEL_ERROR_ENOENT &&
+            std::filesystem::exists(host_file),
+        "unmapped host files and directories cannot be removed");
+
+  const auto missing = Common::PathToGenericString(root / "unmapped-create");
+  Check(FileSystem::KernelOpen(missing.c_str(), 0x601, 0777) ==
+            Libs::LibKernel::KERNEL_ERROR_ENOENT &&
+            FileSystem::KernelMkdir(missing.c_str(), 0777) ==
+                Libs::LibKernel::KERNEL_ERROR_ENOENT &&
+            !std::filesystem::exists(root / "unmapped-create"),
+        "creation requires a mounted guest destination");
+
+  FileSystem::Mount(root, "/app0");
+  Check(FileSystem::KernelRename("/app0/host-only.dat", missing.c_str()) ==
+            Libs::LibKernel::KERNEL_ERROR_ENOENT &&
+            std::filesystem::exists(host_file) &&
+            !std::filesystem::exists(root / "unmapped-create"),
+        "rename to an unmapped destination preserves the source");
+  FileSystem::Umount("/app0");
+}
+
 void CheckUnicodePaths(const std::filesystem::path &root) {
   constexpr std::string_view HostDirectory =
       "Test\xc3\xa9-\xe6\x97\xa5\xe6\x9c\xac\xe8\xaa\x9e";
@@ -548,6 +588,7 @@ int main() {
   TempDirectory temporary;
   FileSystem::Initialize();
   CheckMountRoot(temporary.Path());
+  CheckUnmappedPaths(temporary.Path());
   CheckUnicodePaths(temporary.Path());
   CheckUnicodeLogPath(temporary.Path());
   CheckDirectoryStream(temporary.Path());

tests/MemoryTrackerTests.cpp

@@ -219,17 +219,24 @@ void TestRangeSet() {
 void TestGuestRange() {
   constexpr GuestRange empty{};
   constexpr GuestRange first_byte{1, 1};
-  constexpr GuestRange last_byte{TRACKER_ADDRESS_SIZE - 1, 1};
+  constexpr uint64_t extended_end = Libs::LibKernel::Memory::kExtendedMemoryBase +
+                                    Libs::LibKernel::Memory::kExtendedMemorySize;
+  constexpr GuestRange last_byte{extended_end - 1, 1};
 
   static_assert(empty.Empty() && !empty.Valid() && empty.ValidOrEmpty());
   static_assert(!first_byte.Empty() && first_byte.Valid() &&
                 first_byte.ValidOrEmpty() && first_byte.End() == 2);
-  static_assert(last_byte.Valid() && last_byte.End() == TRACKER_ADDRESS_SIZE);
+  static_assert(last_byte.Valid() && last_byte.End() == extended_end);
 
   Check(!GuestRange{0, 1}.Empty() && !GuestRange{0, 1}.ValidOrEmpty(),
         "zero-address nonempty guest range is rejected");
   Check(!GuestRange{1, 0}.Empty() && !GuestRange{1, 0}.ValidOrEmpty(),
         "nonzero-address empty guest range is rejected");
+  Check(GuestRange{Libs::LibKernel::Memory::kExtendedMemoryBase, 1}.Valid() &&
+            !GuestRange{Libs::Graphics::LOWER_ADDRESS_SIZE, 1}.Valid() &&
+            !GuestRange{Libs::LibKernel::Memory::kExtendedMemoryBase - 1, 2}.Valid() &&
+            !GuestRange{extended_end - 1, 2}.Valid(),
+        "extended range and gap boundaries are enforced");
   Check(!GuestRange{TRACKER_ADDRESS_SIZE, 1}.Valid(),
         "first address beyond the guest range is rejected");
   Check(!GuestRange{TRACKER_ADDRESS_SIZE - 1, 2}.Valid(),

tests/Ngs2SamplerTests.cpp

@@ -712,6 +712,74 @@ void TestLoopCallbacksAndExit() {
 	Ngs2VoiceControl(reinterpret_cast<uintptr_t>(&pending.voice), &exit_loop);
 }
 
+void TestAtrac9Looping() {
+	// Four encoded mono frames with one nonzero spectral coefficient, then superframe padding.
+	// The block skips one frame plus one sample and ends inside the third frame.
+	constexpr std::array<uint8_t, 36> frame {0, 0, 0x04, 0x20, 0x04, 0xc0, 0, 0, 0x1f, 0xc0};
+	std::array<uint8_t, 192> encoded {};
+	for (size_t i = 0; i < 4; ++i) {
+		std::copy(std::begin(frame), std::end(frame), encoded.begin() + i * sizeof(frame));
+		if (i != 0) encoded[i * sizeof(frame)] |= 0x80;
+	}
+	const auto make_voice = [&](uint32_t repeats, LoopCallbacks& log) {
+		auto f = std::make_unique<Fixture>(24000);
+		f->voice.SetupSampler({NGS2_WAVEFORM_TYPE_ATRAC9, 1, 24000, 0xfe4005f0});
+		f->voice.state          = Ngs2VoicePlayState::Playing;
+		f->voice.callback       = reinterpret_cast<uintptr_t>(&RecordLoopCallback);
+		f->voice.callback_data  = reinterpret_cast<uintptr_t>(&log);
+		f->voice.callback_flags = 3;
+		const Ngs2WaveformBlock block {0, encoded.size(), repeats, 129, 137, 0, 123};
+		struct Blocks {
+			Ngs2VoiceParamHeader     header {32, 0, 0x10000001};
+			const uint8_t*           data;
+			uint32_t                 flags, count;
+			const Ngs2WaveformBlock* blocks;
+		} param {{32, 0, 0x10000001}, encoded.data(), 0, 1, &block};
+		Check(Ngs2VoiceControl(reinterpret_cast<uintptr_t>(&f->voice), &param.header) == OK,
+		      "compressed loop queue failed");
+		return f;
+	};
+	LoopCallbacks baseline_log, finite_log, split_log, infinite_log;
+	auto baseline = make_voice(0, baseline_log);
+	baseline->Render(274);
+	const auto reference = baseline->voice.samples;
+	Check(std::ranges::any_of(reference, [](float sample) { return std::abs(sample) > 0.00001f; }),
+	      "synthetic compressed fixture produced no decoded signal");
+	auto finite = make_voice(2, finite_log);
+	finite->Render(822);
+	Check(finite->voice.state == Ngs2VoicePlayState::Empty &&
+	          finite->voice.decoded_samples == 411 && finite->voice.decoded_bytes == 576 &&
+	          finite_log.events == std::vector<std::pair<uint32_t, uint32_t>> {{2, 1}, {2, 2}, {1, 2}},
+	      "compressed finite loop lost samples, input progress, or completion callbacks");
+	for (size_t i = 0; i < 3; ++i) {
+		Check(std::equal(reference.begin(), reference.end() - 1,
+		                 finite->voice.samples.begin() + i * 274),
+		      "compressed loop did not restore decoder history and skip samples");
+		const float tail = i == 2 ? reference.back() : std::lerp(reference[272], reference[0], 0.5f);
+		Check(finite->voice.samples[i * 274 + 273] == tail,
+		      "compressed loop dropped the final frame or broke boundary interpolation");
+	}
+	auto split = make_voice(2, split_log);
+	split->Render(273);
+	auto joined = split->voice.samples;
+	split->Render(549);
+	joined.insert(joined.end(), split->voice.samples.begin(), split->voice.samples.end());
+	Check(joined == finite->voice.samples && split_log.events == finite_log.events,
+	      "compressed loop changed across render grains");
+	auto infinite = make_voice(UINT32_MAX, infinite_log);
+	infinite->Render(300);
+	Check(infinite->voice.state == Ngs2VoicePlayState::Playing &&
+	          infinite->voice.blocks.front().info.num_repeats == UINT32_MAX,
+	      "compressed infinite loop did not rewind");
+	Ngs2VoiceParamHeader exit_loop {sizeof(Ngs2VoiceParamHeader), 0, 0x10000004};
+	Ngs2VoiceControl(reinterpret_cast<uintptr_t>(&infinite->voice), &exit_loop);
+	infinite->Render(248);
+	Check(infinite->voice.state == Ngs2VoicePlayState::Empty &&
+	          infinite->voice.decoded_samples == 274 &&
+	          infinite_log.events == std::vector<std::pair<uint32_t, uint32_t>> {{2, 1}, {1, 1}},
+	      "compressed loop exit failed to finish the active iteration");
+}
+
 void TestWaveformReadAddress() {
 	Fixture       f(48000, 2);
 	const int16_t pcm[] = {-100, -100, 8192, 4096, 16384, 8192};
@@ -921,6 +989,7 @@ int main() {
 	TestSamplerReuseResetsRouting();
 	TestDeferredSamplerSetup();
 	TestLoopCallbacksAndExit();
+	TestAtrac9Looping();
 	TestWaveformReadAddress();
 	TestFilterTailDuringStarvation();
 	TestFiniteFilterTail();

tests/PageManagerTests.cpp

@@ -136,8 +136,8 @@ bool ProtectAddressSpace(uint64_t vaddr, uint64_t size,
                         &old_protection) != 0;
 }
 
-uint8_t *Allocate(uint64_t size, uint32_t protection = PAGE_READWRITE) {
-  constexpr uintptr_t test_address = 0x0000000200010000ull;
+uint8_t *Allocate(uint64_t size, uint32_t protection = PAGE_READWRITE,
+                  uintptr_t test_address = 0x0000000200010000ull) {
 #if KYTY_PLATFORM == KYTY_PLATFORM_WINDOWS
   auto *memory = static_cast<uint8_t *>(
       VirtualAlloc(reinterpret_cast<void *>(test_address), size,
@@ -155,11 +155,11 @@ uint8_t *Allocate(uint64_t size, uint32_t protection = PAGE_READWRITE) {
   return memory;
 }
 
-void TestWatchAndUnwatch() {
+void TestWatchAndUnwatch(uint64_t base = 0x0000000200010000ull) {
   g_protection_calls = 0;
   PageManager manager;
   const auto page_size = manager.GetPageSize();
-  auto *memory = Allocate(page_size * 2);
+  auto *memory = Allocate(page_size * 2, PAGE_READWRITE, base);
   const auto address = reinterpret_cast<uint64_t>(memory);
 
   manager.UpdatePageWatchers<true>(address, page_size);
@@ -580,6 +580,7 @@ int main(int argc, char **argv) {
     RunDeathCase(argv[2]);
   }
   TestWatchAndUnwatch();
+  TestWatchAndUnwatch(Libs::LibKernel::Memory::kExtendedMemoryBase);
   TestSharedWatcherCounts();
   TestCrossRegionRange();
   TestBatchedWatcherRanges();

tests/ResourceTrackingTests.cpp

@@ -1,6 +1,7 @@
 #include "graphics/guest_gpu/gpu_defs.h"
 #include "graphics/shader/recompiler/ir/ShaderIR.h"
 #include "graphics/shader/recompiler/ir/passes/BindingLayout.h"
+#include "graphics/shader/recompiler/ir/passes/ConstantPropagation.h"
 #include "graphics/shader/recompiler/ir/passes/DeadCodeElimination.h"
 #include "graphics/shader/recompiler/ir/passes/ResourceMaterialization.h"
 #include "graphics/shader/recompiler/ir/passes/ResourceTracking.h"
@@ -126,8 +127,28 @@ struct Fixture {
   }
 
   void PlanAndTrack() {
-    BuildSrtPlan(program);
-    TrackResources(program);
+    for (size_t index = 0; index < program.block_info.size(); ++index) {
+      const auto condition = program.block_info[index].condition;
+      if (!condition.IsEmpty())
+        Emit(ValueOpcode::Reference, {condition}, 0, program.blocks[index]);
+    }
+    for (auto *target : program.blocks) {
+      for (auto &inst : *target) {
+        if (inst.HasUses() || inst.MayHaveSideEffects() ||
+            (BufferAccessOf(inst.GetOpcode()) == BufferAccess::None &&
+             AddressOpcodeInfoOf(inst.GetOpcode()).access == AddressAccess::None &&
+             ImageOpcodeInfoOf(inst.GetOpcode()).access == ImageAccess::None))
+          continue;
+        auto value = Value(&inst);
+        if (value.GetType() == Type::U32x4)
+          value = Emit(ValueOpcode::CompositeExtractU32x4, {value, Value(0u)}, 0, target);
+        if (value.GetType() == Type::U8)
+          value = Emit(ValueOpcode::ConvertU32U8, {value}, 0, target);
+        Check(value.GetType() == Type::U32, "unhandled memory result type in fixture");
+        Emit(ValueOpcode::ReferenceU32, {value}, 0, target);
+      }
+    }
+    TrackResources(program, {}, {});
   }
 };
 
@@ -185,8 +206,9 @@ bool ReadLinearTestMemory(void *userdata, uint64_t address, std::span<uint32_t>
 }
 
 std::unique_ptr<Fixture>
-MakeIndirectImageFixture(bool malformed, uint32_t material_immediate = 0,
-                         bool memory_backed_material = false) {
+MakeIndirectImageFixture(bool malformed, uint32_t material_immediate = 4,
+                         bool memory_backed_material = false, uint32_t member_offset = 0,
+                         uint32_t material_stride = 224) {
   auto fixture = std::make_unique<Fixture>();
   std::array<Value, 4> material_words;
   std::array<Value, 4> heap_words;
@@ -228,9 +250,9 @@ MakeIndirectImageFixture(bool malformed, uint32_t material_immediate = 0,
   const auto selector = fixture->Emit(ValueOpcode::ReadFirstLane,
                                       {invocation, Value(true)});
   const auto record =
-      fixture->Emit(ValueOpcode::IMul32, {selector, Value(224u)});
-  const auto member = fixture->Emit(ValueOpcode::IAdd32, {record, Value(4u)});
-  fixture->Emit(ValueOpcode::ReferenceU32, {record});
+      fixture->Emit(ValueOpcode::IMul32, {selector, Value(material_stride)});
+  const auto member = member_offset == 0 ? record :
+      fixture->Emit(ValueOpcode::IAdd32, {record, Value(member_offset)});
   fixture->Emit(ValueOpcode::ReferenceU32, {member});
   MemoryInfo material_scalar;
   material_scalar.kind = ResourceKind::ScalarBuffer;
@@ -277,7 +299,9 @@ void TestInvariantIndirectImageMaterialization() {
 
   Check(fixture->program.info.buffers.size() == 1 &&
             fixture->program.info.images.size() == 1 &&
-            fixture->program.dynamic_reads.size() == 1,
+            std::ranges::any_of(*fixture->block, [](const Inst &inst) {
+              return inst.GetOpcode() == ValueOpcode::ReadConstBuffer;
+            }),
         "indirect image key was not retained as a scalar-buffer read");
   const auto source = fixture->program.info.images[0].source;
   Check(source < fixture->program.descriptor_sources.size() &&
@@ -474,7 +498,7 @@ void TestInvariantIndirectImageMaterialization() {
             rebound_specialization != collapsed_specialization,
         "larger indirect candidate topology reused the old specialization");
 
-  auto memory_backed = MakeIndirectImageFixture(false, 0u, true);
+  auto memory_backed = MakeIndirectImageFixture(false, 4u, true);
   memory_backed->PlanAndTrack();
   auto memory_backed_plan = ExtractResourcePlan(memory_backed->program);
   EliminateDeadCode(memory_backed->program.blocks);
@@ -519,28 +543,50 @@ void TestInvariantIndirectImageMaterialization() {
         "a same-shape refresh discarded the runtime output storage");
 
   auto malformed = MakeIndirectImageFixture(true);
-  BuildSrtPlan(malformed->program);
-  CheckFatal([&] { TrackResources(malformed->program); }, "not a valid runtime value",
+
+  CheckFatal([&] { malformed->PlanAndTrack(); }, "not a valid runtime value",
              "malformed indirect image pattern was accepted");
   Check(!malformed->program.resource_tracking_complete &&
             malformed->program.info.images.empty() &&
             malformed->program.descriptor_sources.empty(),
         "malformed indirect image pattern was partially accepted");
 
-  auto wrapped_immediate = MakeIndirectImageFixture(false, 4u);
-  BuildSrtPlan(wrapped_immediate->program);
-  CheckFatal([&] { TrackResources(wrapped_immediate->program); },
+  auto negative_immediate = MakeIndirectImageFixture(false, 0xfffffffcu);
+  CheckFatal([&] { negative_immediate->PlanAndTrack(); },
              "not a valid runtime value",
-             "wrapped scalar immediate entered the invariant image proof");
-  Check(!wrapped_immediate->program.resource_tracking_complete,
-        "wrapped scalar immediate entered the invariant image proof");
+             "negative scalar immediate entered the indirect image proof");
+
+  for (const auto [immediate, member, first, stride] :
+       {std::array{0u, 4u, 4u, 224u}, std::array{4u, 4u, 8u, 224u},
+        std::array{36u, 0u, 36u, 224u}, std::array{1u, 3u, 0u, 224u},
+        std::array{1u, 3u, 0u, 1u}, std::array{1u, 3u, 0u, 2u}}) {
+    auto split_offset = MakeIndirectImageFixture(false, immediate, false, member, stride);
+    split_offset->PlanAndTrack();
+    const auto split_plan = ExtractResourcePlan(split_offset->program);
+    LinearTestMemory split_memory;
+    std::copy(image_descriptor.begin(), image_descriptor.end(),
+              split_memory.words.begin() + 0x1000u / 4u);
+    split_memory.fail_address = first == 36u ? 0x1004u : UINT64_MAX;
+    split_memory.watched_address = split_memory.base + first;
+    user_data[1] = stride << 16u;
+    user_data[2] = stride < 4u ? 8u / stride : 1u;
+    SrtRuntime split_runtime{.user_data = user_data,
+                             .userdata = &split_memory,
+                             .read_specialization_memory = ReadLinearTestMemory};
+    Check(MaterializeResources(split_plan, split_runtime, snapshot, specialization) &&
+              snapshot.images.size() == 1 &&
+              snapshot.images[0].dwords == image_descriptor &&
+              split_memory.watched_reads == 1u,
+          "indirect scalar offsets did not align and bound their components independently");
+  }
 }
 
 void TestGuardedDirectImageTable() {
   namespace CFG = Libs::Graphics::ShaderRecompiler::CFG;
-  enum class Guard { Nonzero, Zero, Unrelated, Bypass };
+  enum class Guard { Nonzero, SccNonZero, Plain, Zero, Unrelated, Bypass, ExecZero, VccZero };
   const auto make_plan = [](Guard guard) {
     Fixture fixture(ShaderType::Pixel);
+    fixture.program.wave_size = 64u;
     auto *entry = fixture.block;
     auto *middle = fixture.AddBlock();
     auto *before_sample = fixture.AddBlock();
@@ -557,8 +603,19 @@ void TestGuardedDirectImageTable() {
     const auto nonzero = fixture.Emit(
         ValueOpcode::INotEqual32,
         {Value(0u), guard == Guard::Unrelated ? fixture.UserData(2) : mask});
-    fixture.program.block_info[0].condition =
-        fixture.Emit(ValueOpcode::LogicalNot, {nonzero});
+    const auto kind = guard == Guard::ExecZero ? CFG::BranchCondition::ExecZero
+                    : guard == Guard::VccZero ? CFG::BranchCondition::VccZero
+                                             : CFG::BranchCondition::SccZero;
+    auto condition = nonzero;
+    if (guard == Guard::SccNonZero) {
+      condition = fixture.Emit(ValueOpcode::ConditionRef, {condition},
+                               CFG::BranchCondition::SccNonZero);
+    }
+    condition = fixture.Emit(ValueOpcode::LogicalNot, {condition});
+    if (guard != Guard::Plain && guard != Guard::SccNonZero) {
+      condition = fixture.Emit(ValueOpcode::ConditionRef, {condition}, kind);
+    }
+    fixture.program.block_info[0].condition = condition;
     fixture.program.block_info[0].terminator = {
         .kind = CFG::TerminatorKind::ConditionalBranch,
         .true_block = guard == Guard::Zero ? 1u : 4u,
@@ -622,7 +679,10 @@ void TestGuardedDirectImageTable() {
   };
 
   auto plan = make_plan(Guard::Nonzero);
-  for (const auto guard : {Guard::Zero, Guard::Unrelated, Guard::Bypass}) {
+  make_plan(Guard::SccNonZero);
+  make_plan(Guard::Plain);
+  for (const auto guard : {Guard::Zero, Guard::Unrelated, Guard::Bypass,
+                           Guard::ExecZero, Guard::VccZero}) {
     CheckFatal([&] { make_plan(guard); }, "not a valid runtime value",
                "direct table accepted a selector without a dominating nonzero guard");
   }
@@ -703,55 +763,183 @@ void TestGuardedDirectImageTable() {
 void TestBoundedComputeImageLoop() {
   namespace CFG = Libs::Graphics::ShaderRecompiler::CFG;
   enum class Variant {
-    Bounded, WrongGuard, EntryBypass, ExitBypass, GuardBlock, WrongStep
+    Bounded, Plain, Nonzero, TrueEdge, WrongGuard, EntryBypass, ExitBypass, GuardBlock,
+    WrongStep, DivergentBound, DivergentKey, Disjunction, WrongPolarity,
+    IncrementBypass, PreviousBound, Masked, MaskedWrongGuard, MaskResurrection, StatusOverwrite,
+    GuardedDiamond, GuardedDiamondBypass
   };
   const auto make_plan = [](Variant variant) {
-    Fixture fixture;
+    const bool diamond = variant >= Variant::GuardedDiamond;
+    Fixture fixture(diamond ? ShaderType::Vertex : ShaderType::Compute);
+    fixture.program.wave_size = 64u;
+    const bool masked = variant >= Variant::Masked;
     auto *entry = fixture.block;
     auto *header = fixture.AddBlock();
     auto *body = fixture.AddBlock();
     auto *latch = fixture.AddBlock();
     auto *exit = fixture.AddBlock();
+    auto *compare = masked ? fixture.AddBlock() : header;
+    auto *guard = masked ? fixture.AddBlock() : header;
+    auto *increment = masked ? fixture.AddBlock() : latch;
+    auto *final_exit = variant == Variant::PreviousBound ? fixture.AddBlock() : exit;
     entry->AddBranch(header);
-    header->AddBranch(exit);
-    header->AddBranch(body);
-    body->AddBranch(latch);
-    latch->AddBranch(header);
-    if (variant == Variant::EntryBypass) entry->AddBranch(body);
-    if (variant == Variant::ExitBypass) exit->AddBranch(body);
-    fixture.program.block_info[0].terminator = {
-        .kind = variant == Variant::EntryBypass ? CFG::TerminatorKind::ConditionalBranch
-                                               : CFG::TerminatorKind::Branch,
-        .true_block = 1u, .false_block = 2u};
-    fixture.program.block_info[1].terminator = {
-        .kind = CFG::TerminatorKind::ConditionalBranch,
-        .true_block = 4u, .false_block = 2u};
-    fixture.program.block_info[2].terminator = {
-        .kind = CFG::TerminatorKind::Branch, .true_block = 3u};
-    fixture.program.block_info[3].terminator = {
-        .kind = CFG::TerminatorKind::Branch, .true_block = 1u};
-    fixture.program.block_info[4].terminator = {
-        .kind = variant == Variant::ExitBypass ? CFG::TerminatorKind::Branch
-                                              : CFG::TerminatorKind::Return,
-        .true_block = 2u};
+    if (!masked) {
+      header->AddBranch(exit);
+      header->AddBranch(body);
+      body->AddBranch(latch);
+      latch->AddBranch(header);
+      if (variant == Variant::PreviousBound) latch->AddBranch(final_exit);
+      if (variant == Variant::EntryBypass) entry->AddBranch(body);
+      if (variant == Variant::ExitBypass) exit->AddBranch(body);
+      if (variant == Variant::IncrementBypass || variant == Variant::PreviousBound)
+        exit->AddBranch(latch);
+      fixture.program.block_info[0].terminator = {
+          .kind = variant == Variant::EntryBypass ? CFG::TerminatorKind::ConditionalBranch
+                                                 : CFG::TerminatorKind::Branch,
+          .true_block = 1u, .false_block = 2u};
+      fixture.program.block_info[1].terminator = {
+          .kind = CFG::TerminatorKind::ConditionalBranch,
+          .true_block = variant == Variant::TrueEdge ? 2u : 4u,
+          .false_block = variant == Variant::TrueEdge ? 4u : 2u};
+      fixture.program.block_info[2].terminator = {
+          .kind = CFG::TerminatorKind::Branch, .true_block = 3u};
+      fixture.program.block_info[3].terminator = {
+          .kind = variant == Variant::PreviousBound ? CFG::TerminatorKind::ConditionalBranch
+                                                   : CFG::TerminatorKind::Branch,
+          .true_block = 1u, .false_block = 5u};
+      fixture.program.block_info[4].terminator = {
+          .kind = final_exit != exit || variant == Variant::ExitBypass ||
+                          variant == Variant::IncrementBypass
+                      ? CFG::TerminatorKind::Branch : CFG::TerminatorKind::Return,
+          .true_block = final_exit != exit || variant == Variant::IncrementBypass ? 3u : 2u};
+      if (final_exit != exit)
+        fixture.program.block_info[5].terminator.kind = CFG::TerminatorKind::Return;
+    }
 
     auto &phi = header->AppendNewInst(ValueOpcode::Phi, {},
                                       static_cast<uint64_t>(Type::U32));
     const auto key = Value(&phi);
-    const auto count = fixture.UserData(2);
+    auto *previous = variant == Variant::PreviousBound ?
+        &header->AppendNewInst(ValueOpcode::Phi, {}, static_cast<uint64_t>(Type::U1)) : nullptr;
+    const auto local = fixture.Emit(
+        ValueOpcode::GetBuiltin,
+        {Value(static_cast<uint32_t>(diamond ? StageInputKind::VertexIndex
+                                            : StageInputKind::LocalInvocationId)), Value(0u)});
+    const auto count = variant == Variant::DivergentBound ? local : fixture.UserData(2);
     const auto in_range = fixture.Emit(ValueOpcode::SLessThan32,
                                        {variant == Variant::WrongGuard
                                             ? Value(0u) : key,
-                                        count}, 0, header);
-    const auto allowed = fixture.Emit(ValueOpcode::LogicalAnd,
-                                      {in_range, Value(true)}, 0, header);
-    fixture.program.block_info[1].condition =
-        fixture.Emit(ValueOpcode::LogicalNot, {allowed}, 0, header);
+                                        count}, 0, compare);
+    const auto initial_active = fixture.Emit(ValueOpcode::INotEqual32,
+                                             {local, Value(0u)}, 0, entry);
+    if (masked) {
+      auto *active_phi = diamond ? nullptr : &header->AppendNewInst(
+          ValueOpcode::Phi, {}, static_cast<uint64_t>(Type::U1));
+      auto &saved_phi = header->AppendNewInst(ValueOpcode::Phi, {},
+                                              static_cast<uint64_t>(Type::U1));
+      auto &status_phi = header->AppendNewInst(ValueOpcode::Phi, {},
+                                               static_cast<uint64_t>(Type::U32));
+      const auto saved = Value(&saved_phi);
+      const auto status = Value(&status_phi);
+      const auto active = diamond ? fixture.Emit(ValueOpcode::LogicalAnd,
+          {saved, fixture.Emit(ValueOpcode::UGreaterThanEqual32,
+                               {Value(0u), status}, 0, header)}, 0, header) : Value(active_phi);
+      const auto mask = fixture.Emit(ValueOpcode::LogicalOr,
+          {fixture.Emit(ValueOpcode::LogicalAnd, {in_range, active}, 0, compare),
+           fixture.Emit(ValueOpcode::LogicalNot, {active}, 0, compare)}, 0, compare);
+      auto next_saved = fixture.Emit(ValueOpcode::LogicalAnd,
+          {variant == Variant::MaskResurrection ? Value(true) : saved, mask}, 0, compare);
+      const auto execute = fixture.Emit(ValueOpcode::LogicalAnd, {active, mask}, 0, compare);
+      const auto body_status = fixture.Emit(ValueOpcode::SelectU32,
+          {execute, local, status}, 0, guard);
+      const auto next_active = fixture.Emit(ValueOpcode::LogicalAnd,
+          {next_saved, fixture.Emit(ValueOpcode::UGreaterThanEqual32,
+                                   {Value(3u), body_status}, 0, latch)}, 0, latch);
+      auto next_status = fixture.Emit(ValueOpcode::SelectU32,
+          {variant == Variant::StatusOverwrite ? Value(true) : next_active,
+           Value(0u), body_status}, 0,
+          variant == Variant::GuardedDiamondBypass ? latch : increment);
+      if (variant == Variant::GuardedDiamondBypass) {
+        auto &merged_saved = increment->AppendNewInst(ValueOpcode::Phi, {},
+                                                      static_cast<uint64_t>(Type::U1));
+        merged_saved.AddPhiOperand(compare, saved);
+        merged_saved.AddPhiOperand(latch, next_saved);
+        next_saved = Value(&merged_saved);
+        auto &merged_status = increment->AppendNewInst(ValueOpcode::Phi, {},
+                                                       static_cast<uint64_t>(Type::U32));
+        merged_status.AddPhiOperand(compare, status);
+        merged_status.AddPhiOperand(latch, next_status);
+        next_status = Value(&merged_status);
+      }
+      if (active_phi != nullptr) {
+        active_phi->AddPhiOperand(entry, initial_active);
+        active_phi->AddPhiOperand(increment, next_active);
+      }
+      saved_phi.AddPhiOperand(entry, initial_active);
+      saved_phi.AddPhiOperand(increment, next_saved);
+      status_phi.AddPhiOperand(entry, diamond ? Value(0u) : local);
+      status_phi.AddPhiOperand(increment, next_status);
+      const auto branch = [&](uint32_t index, uint32_t yes, uint32_t no,
+                              Value predicate, CFG::BranchCondition kind) {
+        auto *block = fixture.program.blocks[index];
+        block->AddBranch(fixture.program.blocks[yes]);
+        block->AddBranch(fixture.program.blocks[no]);
+        const auto inverse = fixture.Emit(ValueOpcode::LogicalNot, {predicate}, 0, block);
+        fixture.program.block_info[index].condition =
+            fixture.Emit(ValueOpcode::ConditionRef, {inverse}, kind, block);
+        fixture.program.block_info[index].terminator = {
+            .kind = CFG::TerminatorKind::ConditionalBranch,
+            .true_block = yes, .false_block = no};
+      };
+      fixture.program.block_info[0].terminator = {
+          .kind = CFG::TerminatorKind::Branch, .true_block = 1u};
+      branch(1u, 4u, 5u, active, CFG::BranchCondition::ExecZero);
+      branch(5u, variant == Variant::GuardedDiamondBypass ? 7u : 4u, 6u,
+             diamond ? execute : mask,
+             diamond ? CFG::BranchCondition::ExecZero : CFG::BranchCondition::VccZero);
+      const auto image_guard = variant == Variant::MaskedWrongGuard ? active :
+          fixture.Emit(ValueOpcode::LogicalAnd,
+              {fixture.Emit(ValueOpcode::LogicalAnd, {execute, in_range}, 0, guard),
+               fixture.Emit(ValueOpcode::IEqual32, {local, Value(1u)}, 0, guard)}, 0, guard);
+      branch(6u, 3u, 2u, image_guard, CFG::BranchCondition::ExecZero);
+      body->AddBranch(latch);
+      fixture.program.block_info[2].terminator = {
+          .kind = CFG::TerminatorKind::Branch, .true_block = 3u};
+      branch(3u, 4u, 7u, next_active, CFG::BranchCondition::ExecZero);
+      increment->AddBranch(header);
+      fixture.program.block_info[7].terminator = {
+          .kind = CFG::TerminatorKind::Branch, .true_block = 1u};
+      fixture.program.block_info[4].terminator.kind = CFG::TerminatorKind::Return;
+    } else {
+      const auto active = initial_active;
+      const auto allowed = fixture.Emit(variant == Variant::Disjunction
+                                           ? ValueOpcode::LogicalOr : ValueOpcode::LogicalAnd,
+                                        {in_range, active}, 0, header);
+      const bool nonzero = variant == Variant::Nonzero || variant == Variant::TrueEdge;
+      auto condition = allowed;
+      if (!nonzero)
+        condition = fixture.Emit(ValueOpcode::LogicalNot, {condition}, 0, header);
+      if (variant != Variant::Plain)
+        condition = fixture.Emit(ValueOpcode::ConditionRef, {condition},
+            nonzero ? CFG::BranchCondition::ExecNonZero
+                    : CFG::BranchCondition::ExecZero, header);
+      if (variant == Variant::Nonzero || variant == Variant::WrongPolarity)
+        condition = fixture.Emit(ValueOpcode::LogicalNot, {condition}, 0, header);
+      fixture.program.block_info[1].condition = condition;
+      if (variant == Variant::PreviousBound) {
+        previous->AddPhiOperand(entry, Value(false));
+        previous->AddPhiOperand(latch, in_range);
+        const auto continuing = fixture.Emit(ValueOpcode::LogicalOr,
+            {in_range, Value(previous)}, 0, latch);
+        fixture.program.block_info[3].condition = fixture.Emit(ValueOpcode::ConditionRef,
+            {continuing}, CFG::BranchCondition::SccNonZero, latch);
+      }
+    }
     const auto step = fixture.Emit(ValueOpcode::IAdd32,
                                    {key, Value(variant == Variant::WrongStep ? 2u : 1u)},
-                                   0, latch);
-    phi.AddPhiOperand(entry, Value(0u));
-    phi.AddPhiOperand(latch, step);
+                                   0, increment);
+    phi.AddPhiOperand(entry, variant == Variant::DivergentKey ? local : Value(0u));
+    phi.AddPhiOperand(increment, step);
 
     fixture.block = variant == Variant::GuardBlock ? header : body;
     const auto table = fixture.Address(fixture.UserData(0), fixture.UserData(1));
@@ -787,6 +975,18 @@ void TestBoundedComputeImageLoop() {
   };
 
   auto plan = make_plan(Variant::Bounded);
+  make_plan(Variant::Plain);
+  make_plan(Variant::Nonzero);
+  make_plan(Variant::TrueEdge);
+  make_plan(Variant::Masked);
+  make_plan(Variant::GuardedDiamond);
+  for (const auto variant : {Variant::IncrementBypass, Variant::PreviousBound,
+                             Variant::MaskedWrongGuard,
+                             Variant::MaskResurrection, Variant::StatusOverwrite,
+                             Variant::GuardedDiamondBypass}) {
+    CheckFatal([&] { make_plan(variant); }, "not a valid runtime value",
+               "compute image loop allowed an unbounded induction or mask resurrection");
+  }
   CheckFatal([&] { make_plan(Variant::WrongGuard); },
              "not a valid runtime value",
              "compute image loop accepted an unrelated guard");
@@ -802,6 +1002,11 @@ void TestBoundedComputeImageLoop() {
   CheckFatal([&] { make_plan(Variant::WrongStep); },
              "not a valid runtime value",
              "compute image loop accepted a two-step induction");
+  for (const auto variant : {Variant::DivergentBound, Variant::DivergentKey,
+                             Variant::Disjunction, Variant::WrongPolarity}) {
+    CheckFatal([&] { make_plan(variant); }, "not a valid runtime value",
+               "compute image loop accepted a guard without a uniform bound");
+  }
 
   LinearTestMemory memory;
   const auto table = 0x1800u + 0x6b0u;
@@ -849,8 +1054,21 @@ void TestBoundedComputeImageLoop() {
 
 void TestUniformizedMaterialImageKeys() {
   namespace CFG = Libs::Graphics::ShaderRecompiler::CFG;
-  const auto make_plan = [](bool wrong_update, bool wrong_equality) {
+  enum class Variant {
+    Valid, Plain, WrongUpdate, WrongEquality, WrongExit, WrongCarry, WrongBackedge,
+    AndNot, Subset, FirstLane, FirstLaneDirect, FirstLaneAndNot, EmptyFirstEntry, WrongFirstBackedge,
+    WideningFirstEntry, WideningFirstBackedge, WrongFirstLoadMask
+  };
+  const auto make_plan = [](Variant variant) {
     Fixture fixture;
+    fixture.program.wave_size = 64u;
+    const bool first_lane = variant >= Variant::FirstLane;
+    const bool first_lane_loop = first_lane && variant != Variant::FirstLaneDirect;
+    const uint32_t sample_entry_id = first_lane_loop ? 10u : 9u;
+    const auto branch = [&](Value predicate, CFG::BranchCondition kind, Block *block) {
+      return variant == Variant::Plain ? predicate : fixture.Emit(
+          ValueOpcode::ConditionRef, {predicate}, kind, block);
+    };
     auto *entry = fixture.block;
     auto *header = fixture.AddBlock();
     auto *inactive = fixture.AddBlock();
@@ -860,6 +1078,8 @@ void TestUniformizedMaterialImageKeys() {
     auto *choose = fixture.AddBlock();
     auto *sample = fixture.AddBlock();
     auto *done = fixture.AddBlock();
+    auto *sample_header = first_lane_loop ? fixture.AddBlock() : nullptr;
+    auto *sample_entry = first_lane ? fixture.AddBlock() : nullptr;
     entry->AddBranch(header);
     header->AddBranch(inactive);
     inactive->AddBranch(merge);
@@ -869,23 +1089,44 @@ void TestUniformizedMaterialImageKeys() {
     bit->AddBranch(header);
     bit->AddBranch(merge);
     merge->AddBranch(choose);
-    choose->AddBranch(sample);
+    choose->AddBranch(first_lane ? sample_entry : sample);
     choose->AddBranch(done);
+    if (first_lane) {
+      sample_entry->AddBranch(first_lane_loop ? sample_header : sample);
+      fixture.program.block_info[sample_entry_id].terminator = {
+          .kind = CFG::TerminatorKind::Branch, .true_block = first_lane_loop ? 9u : 7u};
+    }
+    if (first_lane_loop) {
+      sample_header->AddBranch(sample);
+      sample->AddBranch(sample_header);
+      fixture.program.block_info[9].terminator = {
+          .kind = CFG::TerminatorKind::Branch, .true_block = 7u,
+          .merge_block = 8u, .continue_block = 7u, .loop_header = true};
+    }
     sample->AddBranch(done);
     fixture.program.block_info[0].terminator = {
         .kind = CFG::TerminatorKind::Branch, .true_block = 1u};
     fixture.program.block_info[1].terminator = {
         .kind = CFG::TerminatorKind::Branch, .true_block = 2u};
-    const auto active_on_entry = fixture.Emit(
+    const auto enabled = fixture.Emit(
         ValueOpcode::INotEqual32, {fixture.UserData(5u), Value(0u)}, 0, entry);
+    const auto local = fixture.Emit(ValueOpcode::GetBuiltin,
+        {Value(static_cast<uint32_t>(StageInputKind::LocalInvocationId)), Value(0u)}, 0, entry);
+    const auto active_on_entry = fixture.Emit(ValueOpcode::LogicalAnd,
+        {enabled, fixture.Emit(ValueOpcode::INotEqual32, {local, Value(0u)}, 0, entry)},
+        0, entry);
     auto &active_phi = header->AppendNewInst(
         ValueOpcode::Phi, {}, static_cast<uint64_t>(Type::U1));
     auto &mask_phi = header->AppendNewInst(
         ValueOpcode::Phi, {}, static_cast<uint64_t>(Type::U32));
+    auto &carry_phi = header->AppendNewInst(
+        ValueOpcode::Phi, {}, static_cast<uint64_t>(Type::U32));
     const auto mask = Value(&mask_phi);
     const auto active = Value(&active_phi);
-    fixture.program.block_info[2].condition = fixture.Emit(
-        ValueOpcode::LogicalNot, {active}, 0, inactive);
+    fixture.program.block_info[2].condition = branch(
+        fixture.Emit(ValueOpcode::LogicalNot, {active}, 0, inactive),
+        variant == Variant::WrongExit ? CFG::BranchCondition::ExecNonZero
+                                      : CFG::BranchCondition::ExecZero, inactive);
     fixture.program.block_info[2].terminator = {
         .kind = CFG::TerminatorKind::ConditionalBranch,
         .true_block = 5u, .false_block = 3u};
@@ -893,8 +1134,9 @@ void TestUniformizedMaterialImageKeys() {
         ValueOpcode::INotEqual32, {Value(0u), mask}, 0, sentinel);
     const auto bit_guard = fixture.Emit(
         ValueOpcode::LogicalAnd, {active, nonzero}, 0, sentinel);
-    fixture.program.block_info[3].condition = fixture.Emit(
-        ValueOpcode::LogicalNot, {bit_guard}, 0, sentinel);
+    fixture.program.block_info[3].condition = branch(
+        fixture.Emit(ValueOpcode::LogicalNot, {bit_guard}, 0, sentinel),
+        CFG::BranchCondition::ExecZero, sentinel);
     fixture.program.block_info[3].terminator = {
         .kind = CFG::TerminatorKind::ConditionalBranch,
         .true_block = 5u, .false_block = 4u};
@@ -903,12 +1145,19 @@ void TestUniformizedMaterialImageKeys() {
         ValueOpcode::BitwiseAnd32, {first, Value(31u)}, 0, bit);
     const auto one_bit = fixture.Emit(
         ValueOpcode::ShiftLeftLogical32, {Value(1u), position}, 0, bit);
+    const bool and_not = variant == Variant::AndNot || variant == Variant::FirstLaneAndNot;
+    const auto removed = and_not
+        ? fixture.Emit(ValueOpcode::BitwiseNot32, {one_bit}, 0, bit)
+        : variant == Variant::Subset ? fixture.UserData(7u) : one_bit;
     const auto cleared = fixture.Emit(
-        wrong_update ? ValueOpcode::BitwiseOr32 : ValueOpcode::BitwiseXor32,
-        {mask, one_bit}, 0, bit);
+        variant == Variant::WrongUpdate ? ValueOpcode::BitwiseOr32
+        : and_not || variant == Variant::Subset ? ValueOpcode::BitwiseAnd32
+                                              : ValueOpcode::BitwiseXor32,
+        {mask, removed}, 0, bit);
     const auto continuation = fixture.Emit(
         ValueOpcode::LogicalAnd, {bit_guard, active_on_entry}, 0, bit);
-    fixture.program.block_info[4].condition = continuation;
+    fixture.program.block_info[4].condition = branch(
+        continuation, CFG::BranchCondition::ExecNonZero, bit);
     fixture.program.block_info[4].terminator = {
         .kind = CFG::TerminatorKind::ConditionalBranch,
         .true_block = 1u, .false_block = 5u};
@@ -917,13 +1166,18 @@ void TestUniformizedMaterialImageKeys() {
     mask_phi.AddPhiOperand(entry, fixture.UserData(4u));
     mask_phi.AddPhiOperand(bit, cleared);
     const auto arbitrary = fixture.UserData(6u);
+    const auto carry = Value(&carry_phi);
     const auto sentinel_index = fixture.Emit(
-        ValueOpcode::SelectU32, {active, Value(32u), arbitrary}, 0, sentinel);
+        ValueOpcode::SelectU32,
+        {active, Value(32u), variant == Variant::WrongCarry ? arbitrary : carry},
+        0, sentinel);
     const auto bit_index = fixture.Emit(
         ValueOpcode::SelectU32, {bit_guard, first, sentinel_index}, 0, bit);
+    carry_phi.AddPhiOperand(entry, arbitrary);
+    carry_phi.AddPhiOperand(bit, variant == Variant::WrongBackedge ? arbitrary : bit_index);
     auto &index_phi = merge->AppendNewInst(
         ValueOpcode::Phi, {}, static_cast<uint64_t>(Type::U32));
-    index_phi.AddPhiOperand(inactive, arbitrary);
+    index_phi.AddPhiOperand(inactive, carry);
     index_phi.AddPhiOperand(sentinel, sentinel_index);
     index_phi.AddPhiOperand(bit, bit_index);
     const auto index = Value(&index_phi);
@@ -950,24 +1204,54 @@ void TestUniformizedMaterialImageKeys() {
     const auto base = fixture.Address(fixture.UserData(0u), fixture.UserData(1u));
     MemoryInfo material_memory;
     material_memory.kind = ResourceKind::Global;
+    const auto load_mask = variant == Variant::WrongFirstLoadMask
+        ? fixture.Emit(ValueOpcode::LogicalNot, {material_guard}, 0, choose)
+        : material_guard;
     const auto loaded = fixture.Emit(
         ValueOpcode::LoadAddressU32,
-        {base, material_offset, Value(0u), material_guard},
+        {base, material_offset, Value(0u), load_mask},
         fixture.AddMemory(material_memory, 0x1a88u), choose);
-    const auto local = fixture.Emit(
+    const auto local_key = fixture.Emit(
         ValueOpcode::SelectU32, {material_guard, loaded, arbitrary}, 0, choose);
-    const auto key = fixture.Emit(
-        ValueOpcode::ReadLane, {local, Value(0u)}, 0, choose);
+    Inst *sample_active_phi = nullptr;
+    auto sample_active = material_guard;
+    if (first_lane) {
+      const auto initial = variant == Variant::EmptyFirstEntry ? Value(false)
+          : variant == Variant::WideningFirstEntry ? enabled
+          : fixture.Emit(ValueOpcode::LogicalAnd,
+              {material_guard, fixture.Emit(ValueOpcode::SLessThanEqual32,
+                  {Value(0u), local_key}, 0, choose)}, 0, choose);
+      sample_active = initial;
+      if (first_lane_loop) {
+        sample_active_phi = &sample_header->AppendNewInst(
+            ValueOpcode::Phi, {}, static_cast<uint64_t>(Type::U1));
+        sample_active_phi->AddPhiOperand(sample_entry, initial);
+        sample_active = Value(sample_active_phi);
+      }
+      fixture.program.block_info[6].condition = branch(
+          variant == Variant::EmptyFirstEntry ? enabled : initial,
+          CFG::BranchCondition::ExecNonZero, choose);
+      fixture.program.block_info[6].terminator = {
+          .kind = CFG::TerminatorKind::ConditionalBranch,
+          .true_block = sample_entry_id, .false_block = 8u};
+    }
+    auto *key_block = first_lane ? sample : choose;
+    const auto key = first_lane
+        ? fixture.Emit(ValueOpcode::ReadFirstLane, {local_key, sample_active}, 0, key_block)
+        : fixture.Emit(ValueOpcode::ReadLane, {local_key, Value(0u)}, 0, key_block);
     const auto compared = fixture.Emit(
         ValueOpcode::IEqual32,
-        {key, wrong_equality ? arbitrary : local}, 0, choose);
+        {key, variant == Variant::WrongEquality ? arbitrary : local_key}, 0, key_block);
     const auto sample_guard = fixture.Emit(
-        ValueOpcode::LogicalAnd, {material_guard, compared}, 0, choose);
-    fixture.program.block_info[6].condition = fixture.Emit(
-        ValueOpcode::LogicalNot, {sample_guard}, 0, choose);
-    fixture.program.block_info[6].terminator = {
-        .kind = CFG::TerminatorKind::ConditionalBranch,
-        .true_block = 8u, .false_block = 7u};
+        ValueOpcode::LogicalAnd, {sample_active, compared}, 0, key_block);
+    if (!first_lane) {
+      fixture.program.block_info[6].condition = branch(
+          fixture.Emit(ValueOpcode::LogicalNot, {sample_guard}, 0, choose),
+          CFG::BranchCondition::ExecZero, choose);
+      fixture.program.block_info[6].terminator = {
+          .kind = CFG::TerminatorKind::ConditionalBranch,
+          .true_block = 8u, .false_block = 7u};
+    }
     const auto table_offset = fixture.Emit(
         ValueOpcode::IAdd32,
         {fixture.Emit(ValueOpcode::ShiftLeftLogical32,
@@ -997,10 +1281,33 @@ void TestUniformizedMaterialImageKeys() {
     MemoryInfo image_memory;
     image_memory.kind = ResourceKind::Image;
     image_memory.image_dimension = Decoder::ImageDimension::Dim2D;
-    fixture.Emit(ValueOpcode::ImageSampleRaw, {image, sampler, address},
-                 fixture.AddMemory(image_memory, 0x1aecu), sample);
-    fixture.program.block_info[7].terminator = {
-        .kind = CFG::TerminatorKind::Branch, .true_block = 8u};
+    const auto sampled = fixture.Emit(ValueOpcode::ImageSampleRaw, {image, sampler, address},
+        fixture.AddMemory(image_memory, 0x1aecu), sample);
+    if (first_lane) {
+      const auto component = fixture.Emit(
+          ValueOpcode::CompositeExtractU32x4, {sampled, Value(0u)}, 0, sample);
+      fixture.Emit(ValueOpcode::ReferenceU32,
+          {fixture.Emit(ValueOpcode::SelectU32,
+              {variant == Variant::EmptyFirstEntry ? material_guard : sample_guard,
+               component, Value(0u)}, 0, sample)}, 0, sample);
+    }
+    if (first_lane_loop) {
+      auto remaining = fixture.Emit(ValueOpcode::LogicalAnd,
+          {sample_active, fixture.Emit(ValueOpcode::LogicalNot,
+              {sample_guard}, 0, sample)}, 0, sample);
+      if (variant == Variant::WideningFirstBackedge)
+        remaining = fixture.Emit(ValueOpcode::LogicalOr, {remaining, enabled}, 0, sample);
+      sample_active_phi->AddPhiOperand(sample, remaining);
+      fixture.program.block_info[7].condition = branch(
+          variant == Variant::WrongFirstBackedge ? sample_guard : remaining,
+          CFG::BranchCondition::ExecNonZero, sample);
+      fixture.program.block_info[7].terminator = {
+          .kind = CFG::TerminatorKind::ConditionalBranch,
+          .true_block = 9u, .false_block = 8u};
+    } else {
+      fixture.program.block_info[7].terminator = {
+          .kind = CFG::TerminatorKind::Branch, .true_block = 8u};
+    }
     fixture.program.block_info[8].terminator.kind = CFG::TerminatorKind::Return;
     const auto output = fixture.Emit(
         ValueOpcode::GetBufferResource,
@@ -1024,7 +1331,13 @@ void TestUniformizedMaterialImageKeys() {
           "uniformized image key lost its finite material range");
     return ExtractResourcePlan(fixture.program);
   };
-  auto plan = make_plan(false, false);
+  auto plan = make_plan(Variant::Valid);
+  make_plan(Variant::Plain);
+  make_plan(Variant::AndNot);
+  make_plan(Variant::Subset);
+  make_plan(Variant::FirstLane);
+  make_plan(Variant::FirstLaneDirect);
+  plan = make_plan(Variant::FirstLaneAndNot);
   Check(plan.requires_specialization_memory &&
             plan.descriptor_sources[plan.info.images[0].source]
                 .indirect_image->selector_mask.Resolve().TryInstruction() != nullptr,
@@ -1078,10 +1391,22 @@ void TestUniformizedMaterialImageKeys() {
   user_data[8] = first_table;
   Check(!MaterializeResources(plan, runtime, snapshot, specialization),
         "written buffer alias with an image record was accepted");
-  CheckFatal([&] { make_plan(true, false); }, "not a valid runtime value",
+  CheckFatal([&] { make_plan(Variant::WrongUpdate); }, "not a valid runtime value",
              "non-clearing material mask was accepted");
-  CheckFatal([&] { make_plan(false, true); }, "not a valid runtime value",
+  CheckFatal([&] { make_plan(Variant::WrongEquality); }, "not a valid runtime value",
              "unrelated ReadLane key was accepted");
+  CheckFatal([&] { make_plan(Variant::WrongExit); }, "not a valid runtime value",
+             "one inactive lane bypassed material index initialization for active lanes");
+  for (const auto variant : {Variant::WrongCarry, Variant::WrongBackedge}) {
+    CheckFatal([&] { make_plan(variant); }, "not a valid runtime value",
+               "inactive material lane did not preserve its selected index across iterations");
+  }
+  for (const auto variant : {Variant::EmptyFirstEntry, Variant::WrongFirstBackedge,
+                            Variant::WideningFirstEntry, Variant::WideningFirstBackedge,
+                            Variant::WrongFirstLoadMask}) {
+    CheckFatal([&] { make_plan(variant); }, "not a valid runtime value",
+               "first-lane material key escaped its active load mask");
+  }
 }
 
 void TestImageDescriptorFields() {
@@ -1405,7 +1730,7 @@ void TestDenseBufferTracking() {
             fixture.program.memory_info[other_flags.index].resource == 1,
         "typed memory metadata was not patched to dense indices");
 
-  CheckFatal([&] { TrackResources(fixture.program); }, "already tracked",
+  CheckFatal([&] { fixture.PlanAndTrack(); }, "already tracked",
              "resource tracking allowed a second mutation pass");
 }
 
@@ -1601,8 +1926,8 @@ void TestSampleAdjustSamplerScratch() {
     rejected.Emit(ValueOpcode::ImageSampleRaw,
                   {rejected_image, rejected_sampler, rejected.ImageAddress()},
                   rejected.AddMemory(rejected_memory, 0x200));
-    BuildSrtPlan(rejected.program);
-    CheckFatal([&] { TrackResources(rejected.program); },
+
+    CheckFatal([&] { rejected.PlanAndTrack(); },
                "not a valid runtime value", message);
   };
   CheckRejected(0u, 12u,
@@ -1936,7 +2261,8 @@ void TestDynamicSrtReadRemainsExplicit() {
   fixture.PlanAndTrack();
 
   Check(fixture.program.srt_reads.empty() &&
-            fixture.program.dynamic_reads.size() == 1 &&
+            read.ResolveInstruction()->GetOpcode() == ValueOpcode::LoadAddressU32 &&
+            !fixture.program.memory_info[0].planning_only &&
             fixture.program.info.uses_dma,
         "dynamic scalar read was incorrectly flattened or lost");
   std::array<uint32_t, 3> user_data{0x1000u, 0u, 4u};
@@ -1990,8 +2316,7 @@ void TestPhiValidation() {
                {handle, Value(0u), Value(0u), Value(0u), Value(true)},
                fixture.AddMemory(memory, 20), merge);
 
-  BuildSrtPlan(fixture.program);
-  CheckFatal([&] { TrackResources(fixture.program); }, "not a valid runtime value",
+  CheckFatal([&] { fixture.PlanAndTrack(); }, "not a valid runtime value",
              "control-dependent descriptor phi was accepted");
   Check(!fixture.program.resource_tracking_complete &&
             fixture.program.info.buffers.empty() &&
@@ -2092,7 +2417,9 @@ ResourcePlan ConditionalSamplerPlan(bool diamond, bool reverse, bool reverse_phi
                                {result, Value(0u)})});
   }
   fixture.PlanAndTrack();
-  Check(fixture.program.value_storage.size() == 4,
+  Check(std::ranges::count_if(fixture.program.value_storage, [](const Inst &inst) {
+          return inst.GetOpcode() == ValueOpcode::SelectU32;
+        }) == 4,
         "repeated sampler uses retained duplicate planning selections");
   Check(sampler_words[0].ResolveInstruction()->GetOpcode() == ValueOpcode::Phi,
         "host descriptor selection changed the GPU Phi");
@@ -2155,90 +2482,6 @@ void TestConditionalSamplerPhi() {
   }
 }
 
-void TestGuardedSamplerPhi() {
-  namespace CFG = Libs::Graphics::ShaderRecompiler::CFG;
-  const auto make_plan = [](bool take_true, bool reverse_phi, bool bypass,
-                            bool mismatched, bool ambiguous) {
-    Fixture fixture(ShaderType::Pixel);
-    auto *entry = fixture.block;
-    auto *loaded = fixture.AddBlock();
-    auto *merge = fixture.AddBlock();
-    auto *sample = fixture.AddBlock();
-    auto *exit = fixture.AddBlock();
-    entry->AddBranch(loaded);
-    entry->AddBranch(merge);
-    loaded->AddBranch(merge);
-    merge->AddBranch(sample);
-    merge->AddBranch(exit);
-    sample->AddBranch(exit);
-    if (bypass) exit->AddBranch(sample);
-    const auto lane = fixture.Emit(ValueOpcode::LaneId);
-    const auto predicate = fixture.Emit(ValueOpcode::IEqual32, {lane, Value(0u)});
-    fixture.program.block_info[0].condition = predicate;
-    fixture.program.block_info[0].terminator = {
-        .kind = CFG::TerminatorKind::ConditionalBranch,
-        .true_block = 1u, .false_block = 2u};
-    fixture.program.block_info[1].terminator = {
-        .kind = CFG::TerminatorKind::Branch, .true_block = 2u};
-    auto &guard = merge->AppendNewInst(ValueOpcode::Phi, {},
-                                       static_cast<uint64_t>(Type::U1));
-    guard.AddPhiOperand(entry, ambiguous ? predicate : Value(!take_true));
-    guard.AddPhiOperand(mismatched ? sample : loaded, predicate);
-    fixture.program.block_info[2].condition = Value(&guard);
-    fixture.program.block_info[2].terminator = {
-        .kind = CFG::TerminatorKind::ConditionalBranch,
-        .true_block = take_true ? 3u : 4u,
-        .false_block = take_true ? 4u : 3u};
-    fixture.program.block_info[3].terminator = {
-        .kind = CFG::TerminatorKind::Branch, .true_block = 4u};
-    fixture.program.block_info[4].terminator = {
-        .kind = bypass ? CFG::TerminatorKind::Branch : CFG::TerminatorKind::Return,
-        .true_block = 3u};
-    std::array<Value, 4> words;
-    for (uint32_t word = 0; word < words.size(); ++word) {
-      // An arbitrary excluded value proves this is control-flow reasoning, not null filtering.
-      const auto first = Value(0xbad000u + word);
-      const auto second = fixture.UserData(word);
-      auto &phi = merge->AppendNewInst(ValueOpcode::Phi, {},
-                                       static_cast<uint64_t>(Type::U32));
-      phi.AddPhiOperand(reverse_phi ? loaded : entry, reverse_phi ? second : first);
-      phi.AddPhiOperand(reverse_phi ? entry : loaded, reverse_phi ? first : second);
-      words[word] = Value(&phi);
-    }
-    fixture.block = sample;
-    const auto image = fixture.Image({Value(0u), Value(0u), Value(0u), Value(0u),
-                                      Value(0u), Value(0u), Value(0u), Value(0u)});
-    const auto sampler = fixture.Sampler(words);
-    MemoryInfo memory;
-    memory.kind = ResourceKind::Image;
-    memory.image_dimension = Decoder::ImageDimension::Dim2D;
-    fixture.Emit(ValueOpcode::ImageSampleRaw,
-                  {image, sampler, fixture.ImageAddress()},
-                  fixture.AddMemory(memory, 0x2a0));
-    fixture.PlanAndTrack();
-    Check(words[0].ResolveInstruction()->GetOpcode() == ValueOpcode::Phi,
-          "guarded host descriptor selection rewrote the GPU Phi");
-    return ExtractResourcePlan(fixture.program);
-  };
-  for (const bool take_true : {false, true}) {
-    for (const bool reverse_phi : {false, true}) {
-      auto plan = make_plan(take_true, reverse_phi, false, false, false);
-      const std::array<uint32_t, 4> user_data{0x444u, 0x555u, 0x666u, 0x777u};
-      DescriptorValue selected;
-      Check(SrtWalker(plan, {.user_data = user_data}).EvaluateDescriptor(
-                plan.info.samplers[0].source, selected) &&
-                std::equal(user_data.begin(), user_data.end(), selected.dwords.begin()),
-            "guarded sampler did not match descriptor and predicate predecessors");
-    }
-  }
-  CheckFatal([&] { make_plan(true, false, true, false, false); },
-              "not a valid runtime value", "sampler guard accepted an unguarded path");
-  CheckFatal([&] { make_plan(true, false, false, true, false); },
-              "not a valid runtime value", "sampler guard ignored predecessor identity");
-  CheckFatal([&] { make_plan(true, false, false, false, true); },
-              "not a valid runtime value", "sampler guard discarded a reachable alternative");
-}
-
 void TestLoopCycleEnteredThroughRuntimeValue() {
   Fixture fixture;
   auto *entry = fixture.block;
@@ -2252,11 +2495,24 @@ void TestLoopCycleEnteredThroughRuntimeValue() {
                                     {Value(&phi), Value(0xffffffffu)}, 0, loop);
   phi.AddPhiOperand(entry, initial);
   phi.AddPhiOperand(loop, carried);
-  fixture.Emit(ValueOpcode::GetBufferResource,
+  const auto handle = fixture.Emit(ValueOpcode::GetBufferResource,
                {carried, Value(0u), Value(0u), Value(0u)}, MemoryFlags{0, 12},
                loop);
-
-  BuildSrtPlan(fixture.program);
+  MemoryInfo memory;
+  memory.kind = ResourceKind::Buffer;
+  fixture.Emit(ValueOpcode::LoadBufferU32,
+               {handle, Value(0u), Value(0u), Value(0u), Value(true)},
+               fixture.AddMemory(memory, 16), loop);
+  ConstantPropagationPass(fixture.program.blocks);
+  RemoveIdentities(fixture.program.blocks);
+  fixture.PlanAndTrack();
+  const std::array<uint32_t, 1> user_data{0x4000u};
+  DescriptorValue descriptor;
+  Check(fixture.program.info.buffers.size() == 1u &&
+            SrtWalker(fixture.program, {.user_data = user_data}).EvaluateDescriptor(
+                fixture.program.info.buffers[0].source, descriptor) &&
+            descriptor.dwords[0] == user_data[0],
+        "runtime-rooted invariant loop lost its buffer source");
 }
 
 void TestInvariantLoopPhi() {
@@ -2600,14 +2856,15 @@ void TestShaderInfoAndBindingLayout() {
   fixture.Emit(ValueOpcode::LoadBufferU32,
                {handle, Value(0u), Value(0u), Value(0u), Value(true)},
                fixture.AddMemory(buffer, 4));
-  fixture.Emit(
+  const auto invocation = fixture.Emit(
       ValueOpcode::GetBuiltin,
       {Value(static_cast<uint32_t>(StageInputKind::GlobalInvocationId)),
        Value(2u)});
-  fixture.Emit(ValueOpcode::BitwiseXor32, {Value(1u), Value(2u)});
+  const auto value = fixture.Emit(ValueOpcode::BitwiseXor32,
+                                   {invocation, Value(2u)});
   MemoryInfo gds;
   gds.kind = ResourceKind::Gds;
-  fixture.Emit(ValueOpcode::WriteSharedU32, {Value(0u), Value(1u), Value(true)},
+  fixture.Emit(ValueOpcode::WriteSharedU32, {Value(0u), value, Value(true)},
                fixture.AddMemory(gds, 8));
   fixture.PlanAndTrack();
 
@@ -2851,8 +3108,8 @@ void TestResourceLimitIsTransactional() {
                  {handle, Value(0u), Value(0u), Value(0u), Value(true)},
                  fixture.AddMemory(memory, index * 4u));
   }
-  BuildSrtPlan(fixture.program);
-  CheckFatal([&] { TrackResources(fixture.program); },
+
+  CheckFatal([&] { fixture.PlanAndTrack(); },
              "buffer resource limit exceeded",
              "resource-limit failure was not reported");
   Check(!fixture.program.resource_tracking_complete &&
@@ -2870,9 +3127,9 @@ void TestMalformedMemoryKindsRejected() {
     fixture.Emit(ValueOpcode::StoreAddressU32,
                  {address, Value(0u), Value(0u), Value(1u), Value(true)},
                  fixture.AddMemory(memory, 4));
-    BuildSrtPlan(fixture.program);
+
     CheckFatal(
-        [&] { TrackResources(fixture.program); },
+        [&] { fixture.PlanAndTrack(); },
         "address operation has invalid resource kind",
         "resource tracking accepted an address opcode with buffer metadata");
   }
@@ -2887,9 +3144,9 @@ void TestMalformedMemoryKindsRejected() {
     fixture.Emit(ValueOpcode::ImageRead,
                  {image, fixture.ImageAddress(), Value(true)},
                  fixture.AddMemory(memory, 8));
-    BuildSrtPlan(fixture.program);
+
     CheckFatal(
-        [&] { TrackResources(fixture.program); },
+        [&] { fixture.PlanAndTrack(); },
         "image operation has invalid resource kind",
         "resource tracking accepted an image opcode with address metadata");
   }
@@ -2924,7 +3181,6 @@ int main() {
     Run("dynamic SRT", TestDynamicSrtReadRemainsExplicit);
     Run("phi validation", TestPhiValidation);
     Run("conditional sampler phi", TestConditionalSamplerPhi);
-    Run("guarded sampler phi", TestGuardedSamplerPhi);
     Run("runtime-rooted loop", TestLoopCycleEnteredThroughRuntimeValue);
     Run("invariant loop phi", TestInvariantLoopPhi);
     Run("DMA address materialization", TestDmaAddressMaterialization);
@@ -2981,3 +3237,4 @@ void DbgExit(int) { throw std::runtime_error("typed IR assertion failed"); }
 #include "graphics/shader/recompiler/ir/Value.cpp"
 #include "graphics/shader/recompiler/ir/opcodes/ValueOpcodes.cpp"
 #include "graphics/shader/recompiler/ir/passes/DeadCodeElimination.cpp"
+#include "graphics/shader/recompiler/ir/passes/ConstantPropagation.cpp"

tests/SaveDataDialogTests.cpp

@@ -0,0 +1,166 @@
+#include "libs/dialog.h"
+
+#include <cstddef>
+#include <cstdint>
+#include <cstdio>
+#include <cstring>
+
+namespace Loader::Timer {
+double GetTimeMs() {
+	return 0.0;
+}
+} // namespace Loader::Timer
+
+namespace {
+namespace Save = Libs::Dialog::SaveDataDialog;
+
+struct BaseParam {
+	uint64_t size;
+	uint8_t  reserved[36];
+	uint32_t magic;
+};
+
+struct Items {
+	int32_t     user_id;
+	uint32_t    pad0;
+	const void* title_id;
+	const void* dir_names;
+	uint32_t    dir_names_num;
+	uint32_t    pad1;
+	const void* new_item;
+	int32_t     focus_pos;
+	uint32_t    pad2;
+	const void* focus_dir_name;
+	int32_t     item_style;
+	uint8_t     reserved[36];
+};
+
+struct UserMessage {
+	int32_t     button_type;
+	int32_t     message_type;
+	const char* message;
+	uint8_t     reserved[32];
+};
+
+struct Param {
+	BaseParam    base;
+	int32_t      size;
+	int32_t      mode;
+	int32_t      display_type;
+	uint32_t     pad0;
+	void*        animation;
+	Items*       items;
+	UserMessage* user_message;
+	void*        system_message;
+	void*        error_code;
+	void*        progress_bar;
+	void*        user_data;
+	void*        option;
+	void*        wizard;
+	uint8_t      reserved[16];
+};
+
+static_assert(sizeof(BaseParam) == 48);
+static_assert(sizeof(Items) == 96);
+static_assert(sizeof(Param) == 152);
+static_assert(offsetof(Param, items) == 72);
+static_assert(offsetof(Param, user_data) == 112);
+
+bool CheckLatestStatus(const char* stage) {
+	const int updated = Save::SaveDataDialogUpdateStatus();
+	const int current = Save::SaveDataDialogGetStatus();
+	if (updated != current) {
+		std::fprintf(stderr,
+		             "SaveDataDialogTests: %s: UpdateStatus returned %d but the current "
+		             "status is %d\n",
+		             stage, updated, current);
+		return false;
+	}
+	return true;
+}
+
+bool CheckStatus(int actual, int expected, const char* stage) {
+	if (actual != expected) {
+		std::fprintf(stderr, "SaveDataDialogTests: %s: expected %d, got %d\n", stage, expected,
+		             actual);
+		return false;
+	}
+	return true;
+}
+
+bool TestRunningThenFinished(Param& param) {
+	bool passed = CheckStatus(Save::SaveDataDialogOpen(&param), 0, "Open");
+	for (int poll = 0; poll < 3; ++poll) {
+		passed =
+		    CheckStatus(Save::SaveDataDialogGetStatus(), 2, "peek before first update") && passed;
+	}
+	passed = CheckStatus(Save::SaveDataDialogUpdateStatus(), 2, "first update") && passed;
+	for (int poll = 0; poll < 3; ++poll) {
+		passed =
+		    CheckStatus(Save::SaveDataDialogGetStatus(), 2, "peek after first update") && passed;
+	}
+	passed = CheckStatus(Save::SaveDataDialogUpdateStatus(), 3, "second update") && passed;
+	passed = CheckStatus(Save::SaveDataDialogGetStatus(), 3, "finished peek") && passed;
+	passed = CheckStatus(Save::SaveDataDialogUpdateStatus(), 3, "finished update") && passed;
+	return passed;
+}
+
+bool TestCloseAndReset(Param& param) {
+	bool passed = CheckStatus(Save::SaveDataDialogOpen(&param), 0, "Open before Close");
+	passed =
+	    CheckStatus(Save::SaveDataDialogClose(nullptr), 0, "Close before first poll") && passed;
+	passed = CheckStatus(Save::SaveDataDialogUpdateStatus(), 3, "poll after Close") && passed;
+	passed = CheckStatus(Save::SaveDataDialogGetStatus(), 3, "peek after Close") && passed;
+	passed = TestRunningThenFinished(param) && passed;
+
+	passed = CheckStatus(Save::SaveDataDialogOpen(&param), 0, "Open before Terminate") && passed;
+	passed = CheckStatus(Save::SaveDataDialogUpdateStatus(), 2, "poll before Terminate") && passed;
+	passed = CheckStatus(Save::SaveDataDialogTerminate(), 0, "Terminate while running") && passed;
+	passed = CheckStatus(Save::SaveDataDialogGetStatus(), 0, "peek after Terminate") && passed;
+	passed = CheckStatus(Save::SaveDataDialogUpdateStatus(), 0, "poll after Terminate") && passed;
+	passed = CheckStatus(Save::SaveDataDialogInitialize(), 0, "reinitialize") && passed;
+	passed = CheckStatus(Save::SaveDataDialogUpdateStatus(), 1, "initialized update") && passed;
+	passed = TestRunningThenFinished(param) && passed;
+	return passed;
+}
+} // namespace
+
+int main(int argc, char** argv) {
+	const bool consistency_only = argc == 2 && std::strcmp(argv[1], "--consistency-only") == 0;
+	if (argc != 1 && !consistency_only) {
+		std::fprintf(stderr, "usage: SaveDataDialogTests [--consistency-only]\n");
+		return 2;
+	}
+	Items items {};
+	items.user_id       = 1000;
+	items.dir_names_num = 1; // No save-data information is displayed with a null dir_names.
+	UserMessage message {};
+	message.message = "Status polling";
+	Param param {};
+	param.base.size = sizeof(BaseParam);
+	param.base.magic =
+	    static_cast<uint32_t>(0xc0d1a109u + reinterpret_cast<uintptr_t>(&param.base));
+	param.size         = sizeof(Param);
+	param.mode         = 2; // USER_MSG
+	param.display_type = 1; // SAVE
+	param.items        = &items;
+	param.user_message = &message;
+
+	if (Libs::Dialog::CommonDialog::CommonDialogInitialize() != 0 ||
+	    Save::SaveDataDialogInitialize() != 0) {
+		return 2;
+	}
+	bool passed = CheckLatestStatus("initialized");
+	if (Save::SaveDataDialogOpen(&param) != 0) {
+		return 2;
+	}
+	passed = CheckLatestStatus("first poll after Open") && passed;
+	passed = CheckLatestStatus("second poll after Open") && passed;
+	if (!consistency_only) {
+		passed = TestRunningThenFinished(param) && passed;
+		passed = TestCloseAndReset(param) && passed;
+	}
+	Save::SaveDataDialogTerminate();
+	passed = CheckLatestStatus("terminated") && passed;
+	return passed ? 0 : 1;
+}

tests/ScalarProvenanceTests.cpp

@@ -2,6 +2,7 @@
 #include "graphics/shader/recompiler/ir/passes/ConstantPropagation.h"
 #include "graphics/shader/recompiler/ir/passes/DeadCodeElimination.h"
 #include "graphics/shader/recompiler/ir/passes/ReadLaneElimination.h"
+#include "graphics/shader/recompiler/ir/passes/ResourceTracking.h"
 #include "graphics/shader/recompiler/ir/passes/SrtWalker.h"
 
 #include <cstdlib>
@@ -71,7 +72,7 @@ struct Fixture {
     return static_cast<uint32_t>(program.memory_info.size() - 1u);
   }
 
-  void Plan() { BuildSrtPlan(program); }
+  void Plan() { TrackResources(program, {}, {}); }
 };
 
 struct TestMemory {
@@ -102,6 +103,16 @@ Value RawRead(Fixture &fixture, Value address, Value offset, uint32_t memory,
                             0x80, block);
 }
 
+void LoadBuffer(Fixture &fixture, std::array<Value, 4> words) {
+  const auto handle = fixture.Emit(ValueOpcode::GetBufferResource,
+                                   {words[0], words[1], words[2], words[3]});
+  const auto loaded = fixture.EmitMemory(
+      ValueOpcode::LoadBufferU32,
+      {handle, Value(0u), Value(0u), Value(0u), Value(true)},
+      fixture.AddMemory(ResourceKind::Buffer));
+  fixture.Emit(ValueOpcode::ReferenceU32, {loaded});
+}
+
 void TestImmediateFlatteningAndGvn() {
   Fixture fixture;
   const auto memory = fixture.AddMemory(ResourceKind::ScalarAddress, 0x20);
@@ -109,14 +120,11 @@ void TestImmediateFlatteningAndGvn() {
       fixture, Address(fixture, Value(0x1000u), Value(0u)), Value(0u), memory);
   const auto second = RawRead(
       fixture, Address(fixture, Value(0x1000u), Value(0u)), Value(0u), memory);
-  fixture.Emit(ValueOpcode::GetBufferResource,
-               {first, second, Value(16u), Value(0u)});
+  LoadBuffer(fixture, {first, second, Value(16u), Value(0u)});
 
   fixture.Plan();
   Check(fixture.program.srt_reads.size() == 1,
         "equivalent typed scalar reads were not coalesced");
-  Check(fixture.program.dynamic_reads.empty(),
-        "immediate scalar read was classified as dynamic");
   Check(fixture.program.memory_info[memory].planning_only,
         "flattened raw read was not kept as a planning-only root");
 
@@ -133,8 +141,7 @@ void TestRawScalarComponentAlignment() {
   const auto memory = fixture.AddMemory(ResourceKind::ScalarAddress, 2);
   const auto read = RawRead(
       fixture, Address(fixture, Value(0x1003u), Value(0u)), Value(2u), memory);
-  fixture.Emit(ValueOpcode::GetBufferResource,
-               {read, Value(0u), Value(16u), Value(0u)});
+  LoadBuffer(fixture, {read, Value(0u), Value(16u), Value(0u)});
   fixture.Plan();
 
   TestMemory memory_image{{{0x1000u, 0x12345678u}}};
@@ -150,7 +157,7 @@ void TestScalarMemoryDomainMismatchFails() {
   Fixture raw;
   const auto raw_memory = raw.AddMemory(ResourceKind::ScalarBuffer);
   RawRead(raw, Address(raw, Value(0x1000u), Value(0u)), Value(0u), raw_memory);
-  CheckFatal([&] { BuildSrtPlan(raw.program); },
+  CheckFatal([&] { TrackResources(raw.program, {}, {}); },
              "incompatible scalar memory metadata",
              "raw scalar load accepted descriptor-buffer metadata");
 
@@ -161,7 +168,7 @@ void TestScalarMemoryDomainMismatchFails() {
                   {Value(0x1000u), Value(0u), Value(16u), Value(0u)});
   buffer.EmitMemory(ValueOpcode::ReadConstBuffer, {resource, Value(0u)},
                     buffer_memory);
-  CheckFatal([&] { BuildSrtPlan(buffer.program); },
+  CheckFatal([&] { TrackResources(buffer.program, {}, {}); },
              "incompatible scalar memory metadata",
              "descriptor scalar load accepted raw-address metadata");
 }
@@ -173,12 +180,12 @@ void TestDynamicReadRemainsTyped() {
                                    {Value(static_cast<ScalarReg>(2))});
   const auto read = RawRead(
       fixture, Address(fixture, Value(0x1000u), Value(0u)), offset, memory);
-  fixture.Emit(ValueOpcode::GetBufferResource,
-               {read, Value(0u), Value(16u), Value(0u)});
+  LoadBuffer(fixture, {read, Value(0u), Value(16u), Value(0u)});
 
   fixture.Plan();
   Check(fixture.program.srt_reads.empty() &&
-            fixture.program.dynamic_reads == std::vector<Value>{read},
+            read.ResolveInstruction()->GetOpcode() == ValueOpcode::LoadAddressU32 &&
+            !fixture.program.memory_info[memory].planning_only,
         "dynamic scalar read received a fake flattened slot");
 }
 
@@ -189,8 +196,7 @@ void TestNestedSrtWalk() {
       fixture, Address(fixture, Value(0x1000u), Value(0u)), Value(0u), memory);
   const auto value =
       RawRead(fixture, Address(fixture, pointer, Value(0u)), Value(0u), memory);
-  fixture.Emit(ValueOpcode::GetBufferResource,
-               {value, Value(0u), Value(16u), Value(0u)});
+  LoadBuffer(fixture, {value, Value(0u), Value(16u), Value(0u)});
   fixture.Plan();
 
   TestMemory memory_image{{{0x1000u, 0x2000u}, {0x2000u, 0xabcdef01u}}};
@@ -211,7 +217,7 @@ void TestShaderBaseAndUserData() {
   const auto user = fixture.Emit(ValueOpcode::GetUserData,
                                  {Value(static_cast<ScalarReg>(2))});
   const auto sum = fixture.Emit(ValueOpcode::IAdd32, {user, Value(4u)});
-  fixture.Plan();
+
   fixture.program.descriptor_sources.push_back(
       {.dwords = {low, high, sum}, .dword_count = 3});
 
@@ -239,7 +245,7 @@ void TestCarryAndBitFields() {
                    {Value(0u), Value(0x89abcdefu), Value(0u), Value(32u)});
   const auto sign = fixture.Emit(ValueOpcode::BitFieldSExtract,
                                  {Value(0x000000f0u), Value(4u), Value(4u)});
-  fixture.Plan();
+
   fixture.program.descriptor_sources.push_back(
       {.dwords = {low, high, inserted, sign}, .dword_count = 4});
 
@@ -261,7 +267,7 @@ void TestInvariantAndDivergentPhi() {
   divergent.SetFlags(Type::U32);
   divergent.AddPhiOperand(&fixture.BlockAt(0), Value(7u));
   divergent.AddPhiOperand(&fixture.BlockAt(1), Value(9u));
-  fixture.Plan();
+
   fixture.program.descriptor_sources.push_back(
       {.dwords = {Value(&invariant)}, .dword_count = 1});
   fixture.program.descriptor_sources.push_back(
@@ -285,6 +291,7 @@ void TestControlDependentStandaloneLoadStaysTyped() {
   const auto read =
       RawRead(fixture, Address(fixture, Value(&base), Value(0u), 2), Value(0u),
               memory, 2);
+  fixture.Emit(ValueOpcode::ReferenceU32, {read}, 0, 2);
   fixture.Plan();
   Check(fixture.program.srt_reads.empty() &&
             read.ResolveInstruction()->GetOpcode() ==
@@ -307,7 +314,7 @@ void TestRuntime64BitDescriptorOps() {
       fixture.Emit(ValueOpcode::CompositeExtractU64, {combined, Value(0u)});
   const auto high =
       fixture.Emit(ValueOpcode::CompositeExtractU64, {combined, Value(1u)});
-  fixture.Plan();
+
   fixture.program.descriptor_sources.push_back(
       {.dwords = {low, high}, .dword_count = 2});
 
@@ -374,7 +381,6 @@ void TestUniformFirstLaneSamplerLod() {
                                   {packed, active});
   const auto divergent = fixture.Emit(
       ValueOpcode::ReadFirstLane, {stale, active});
-  fixture.Plan();
 
   Check(ValidateRuntimeValue(fixture.program, first),
         "uniform sampler LOD construction was rejected");
@@ -396,6 +402,49 @@ void TestUniformFirstLaneSamplerLod() {
         "uniform sampler LOD clamp evaluated incorrectly");
 }
 
+void TestFloatComparisonDescriptorInputs() {
+  struct Input {
+    uint32_t bits;
+    uint32_t less_equal;
+    uint32_t greater_equal;
+  };
+  constexpr std::array inputs = {
+      Input{0x00000001u, 0, 1}, Input{0x80000001u, 1, 0},
+      Input{0x007fffffu, 0, 1}, Input{0x807fffffu, 1, 0},
+      Input{0x00800000u, 0, 1}, Input{0x80800000u, 1, 0},
+      Input{0x00000000u, 1, 1}, Input{0x80000000u, 1, 1},
+      Input{0x7fc00000u, 0, 0}};
+  for (const bool flush : {false, true}) {
+    Fixture fixture;
+    const auto user = fixture.Emit(
+        ValueOpcode::GetUserData, {Value(static_cast<ScalarReg>(2))});
+    const auto value = fixture.Emit(ValueOpcode::BitCastF32U32, {user});
+    const auto less_equal = fixture.Emit(ValueOpcode::FPOrdLessThanEqual32,
+                                         {value, Value::F32(0.0f)});
+    const auto greater_equal = fixture.Emit(ValueOpcode::FPOrdGreaterThanEqual32,
+                                            {value, Value::F32(0.0f)});
+    less_equal.Instruction()->SetFlags(FPCompareFlags{flush});
+    greater_equal.Instruction()->SetFlags(FPCompareFlags{flush});
+    const auto low = fixture.Emit(ValueOpcode::SelectU32,
+                                  {less_equal, Value(1u), Value(0u)});
+    const auto high = fixture.Emit(ValueOpcode::SelectU32,
+                                   {greater_equal, Value(1u), Value(0u)});
+    fixture.program.descriptor_sources.push_back(
+        {.dwords = {low, high}, .dword_count = 2});
+
+    for (size_t index = 0; index < inputs.size(); index++) {
+      const auto &input = inputs[index];
+      const std::array user_data{input.bits};
+      DescriptorValue result;
+      Check(SrtWalker(fixture.program, {.user_data = user_data})
+                    .EvaluateDescriptor(0, result) &&
+                result.dwords[0] == (flush && index < 4 ? 1 : input.less_equal) &&
+                result.dwords[1] == (flush && index < 4 ? 1 : input.greater_equal),
+            "descriptor comparison disagrees with native FP32 input mode");
+    }
+  }
+}
+
 void TestSharedIntegerRuntimeDependencies() {
   Fixture fixture;
   const auto lane = fixture.Emit(ValueOpcode::LaneId);
@@ -439,37 +488,34 @@ void TestSharedIntegerRuntimeDependencies() {
 }
 
 void TestConstantBufferBounds() {
-  Fixture fixture;
-  const auto memory = fixture.AddMemory(ResourceKind::ScalarBuffer);
-  const auto buffer =
-      fixture.Emit(ValueOpcode::GetBufferResource,
-                   {Value(0x3000u), Value(0u), Value(16u), Value(0u)});
-  const auto read = fixture.EmitMemory(ValueOpcode::ReadConstBuffer,
-                                       {buffer, Value(12u)}, memory);
-  fixture.Emit(ValueOpcode::GetBufferResource,
-               {read, Value(0u), Value(16u), Value(0u)});
-  fixture.Plan();
+  struct Case {
+    uint32_t offset;
+    uint32_t immediate;
+    bool valid;
+    uint32_t expected;
+  };
+  for (const auto &test : {Case{12u, 0u, true, 0xa5a5a5a5u},
+                           Case{3u, 1u, true, 0x12345678u},
+                           Case{0xfffffffcu, 4u, false, 0u},
+                           Case{16u, 0u, false, 0u}}) {
+    Fixture fixture;
+    const auto memory = fixture.AddMemory(ResourceKind::ScalarBuffer, test.immediate);
+    const auto buffer =
+        fixture.Emit(ValueOpcode::GetBufferResource,
+                     {Value(0x3000u), Value(0u), Value(16u), Value(0u)});
+    const auto read = fixture.EmitMemory(ValueOpcode::ReadConstBuffer,
+                                         {buffer, Value(test.offset)}, memory);
+    LoadBuffer(fixture, {read, Value(0u), Value(16u), Value(0u)});
+    fixture.Plan();
 
-  TestMemory memory_image{{{0x300cu, 0xa5a5a5a5u}}};
-  SrtRuntime runtime{.read_memory = ReadMemory, .userdata = &memory_image};
-  std::vector<uint32_t> flat;
-  Check(SrtWalker(fixture.program, runtime).RefreshFlatBuffer(flat) &&
-            flat == std::vector<uint32_t>{0xa5a5a5a5u},
-        "constant-buffer SRT walk failed");
-
-  Fixture overflow;
-  const auto overflow_memory = overflow.AddMemory(ResourceKind::ScalarBuffer);
-  const auto overflow_buffer =
-      overflow.Emit(ValueOpcode::GetBufferResource,
-                    {Value(0x3000u), Value(0u), Value(16u), Value(0u)});
-  const auto overflow_read =
-      overflow.EmitMemory(ValueOpcode::ReadConstBuffer,
-                          {overflow_buffer, Value(16u)}, overflow_memory);
-  overflow.Emit(ValueOpcode::GetBufferResource,
-                {overflow_read, Value(0u), Value(16u), Value(0u)});
-  overflow.Plan();
-  Check(!SrtWalker(overflow.program, runtime).RefreshFlatBuffer(flat),
-        "out-of-bounds constant-buffer walk was accepted");
+    TestMemory memory_image{{{0x3000u, 0x12345678u}, {0x300cu, 0xa5a5a5a5u}}};
+    SrtRuntime runtime{.read_memory = ReadMemory, .userdata = &memory_image};
+    std::vector<uint32_t> flat;
+    Check(SrtWalker(fixture.program, runtime).RefreshFlatBuffer(flat) == test.valid,
+          "constant-buffer walk misaligned or wrapped its offset components");
+    Check(test.valid ? flat == std::vector<uint32_t>{test.expected} : memory_image.reads == 0,
+          "constant-buffer walk read the wrong word or accessed an out-of-bounds address");
+  }
 }
 
 void TestReadLaneElimination() {
@@ -552,7 +598,7 @@ void TestControlFlowValueSurvivesReadLaneFolding() {
 void TestUndefinedRuntimeValueFails() {
   Fixture fixture;
   const auto undef = fixture.Emit(ValueOpcode::UndefU32);
-  fixture.Plan();
+
   fixture.program.descriptor_sources.push_back(
       {.dwords = {undef}, .dword_count = 1});
   DescriptorValue result;
@@ -591,6 +637,7 @@ int main() {
     TestControlDependentStandaloneLoadStaysTyped();
     TestRuntime64BitDescriptorOps();
     TestUniformFirstLaneSamplerLod();
+    TestFloatComparisonDescriptorInputs();
     TestSharedIntegerRuntimeDependencies();
     TestConstantBufferBounds();
     TestReadLaneElimination();
@@ -611,6 +658,7 @@ int main() {
 #include "../src/graphics/shader/recompiler/ir/Program.cpp"
 #include "../src/graphics/shader/recompiler/ir/Type.cpp"
 #include "../src/graphics/shader/recompiler/ir/Value.cpp"
+#include "../src/graphics/shader/recompiler/ir/passes/ResourceTracking.cpp"
 #include "../src/graphics/shader/recompiler/ir/opcodes/ValueOpcodes.cpp"
 #include "../src/graphics/shader/recompiler/ir/passes/ConstantPropagation.cpp"
 #include "../src/graphics/shader/recompiler/ir/passes/DeadCodeElimination.cpp"

tests/VirtualMemoryAllocationTests.cpp

@@ -2163,6 +2163,51 @@ void TestFixedReserveRangeAddRollbackKeepsPlaceholder() {
 	std::printf("[host]    %-48s ok\n", test);
 }
 
+void TestExtendedAndUserMappingsDoNotAlias() {
+	const char* test = "ExtendedAndUserMappingsDoNotAlias";
+	// macOS starts its ordinary guest range at 448 GiB.
+	constexpr uint64_t user_address = 0x7000000000ull;
+	const uint64_t addresses[] {
+	    user_address,
+	    Libs::LibKernel::Memory::kExtendedMemoryBase + user_address,
+	    Libs::LibKernel::Memory::kExtendedMemoryBase + Libs::LibKernel::Memory::kExtendedMemorySize -
+	        SceKernelPageSize,
+	};
+	int64_t physical = 0;
+	CheckOk(test,
+	        Libs::LibKernel::Memory::KernelAllocateDirectMemory(
+	            0, Libs::LibKernel::Memory::KernelGetDirectMemorySize(),
+	            SceKernelPageSize * std::size(addresses), SceKernelPageSize, SceKernelMtypeC,
+	            &physical),
+	        "KernelAllocateDirectMemory");
+	for (size_t i = 0; i < std::size(addresses); ++i) {
+		void* mapped = reinterpret_cast<void*>(addresses[i]);
+		CheckOk(test,
+		        Libs::LibKernel::Memory::KernelMapDirectMemory(
+		            &mapped, SceKernelPageSize, SceKernelProtCpuRw,
+		            SceKernelMapFixed | SceKernelMapNoOverwrite,
+		            physical + i * SceKernelPageSize, SceKernelPageSize),
+		        "KernelMapDirectMemory");
+		Check(test, mapped == reinterpret_cast<void*>(addresses[i]), "fixed mapping moved");
+		*static_cast<uint64_t*>(mapped) = 0x0123456700000000ull + i;
+	}
+	for (size_t i = 0; i < std::size(addresses); ++i) {
+		uint64_t value = 0;
+		Check(test,
+		      Libs::LibKernel::Memory::TryReadBacking(addresses[i], &value, sizeof(value)) &&
+		          value == 0x0123456700000000ull + i &&
+		          *reinterpret_cast<uint64_t*>(addresses[i]) == value,
+		      "Extended and user mappings overlap or lost their backing");
+		CheckOk(test, Libs::LibKernel::Memory::KernelMunmap(addresses[i], SceKernelPageSize),
+		        "KernelMunmap");
+	}
+	CheckOk(test,
+	        Libs::LibKernel::Memory::KernelReleaseDirectMemory(
+	            physical, SceKernelPageSize * std::size(addresses)),
+	        "KernelReleaseDirectMemory");
+	std::printf("[host]    %-48s ok\n", test);
+}
+
 void TestLargeHintedReserveHostsSmallDirectMap() {
 	const char* test = "LargeHintedReserveHostsSmallDirectMap";
 
@@ -2220,6 +2265,128 @@ void TestLargeHintedReserveHostsSmallDirectMap() {
 	std::printf("[host]    %-48s ok\n", test);
 }
 
+void TestAutomaticMemoryReuseAndAliases() {
+	using namespace Libs::LibKernel::Memory;
+	const char*        test     = "AutomaticMemoryReuseAndAliases";
+	constexpr uint64_t block    = 0x200000;
+	constexpr uint64_t page     = SceKernelPageSize;
+	constexpr uint64_t base     = kExtendedMemoryBase + 0x20000000;
+	const auto         end      = static_cast<int64_t>(KernelGetDirectMemorySize());
+	int64_t            physical = -1;
+	CheckOk(test, AllocateDirectMemory(0, end, block, block, 0, &physical, true),
+	        "allocate automatic backing");
+	CheckOk(test, MapAutomaticMemory(base, block, SceKernelMtypeC, SceKernelProtCpuRw),
+	        "MapAutomaticMemory");
+	*reinterpret_cast<uint64_t*>(base + page * 3) = 0x0123456789abcdefull;
+	auto* first_alias                             = reinterpret_cast<void*>(base + block * 2);
+	auto* second_alias                            = reinterpret_cast<void*>(base + block * 3);
+	CheckOk(test,
+	        KernelMapDirectMemory(&first_alias, page * 3, SceKernelProtCpuRw, SceKernelMapFixed,
+	                              physical + page, page),
+	        "KernelMapDirectMemory(first alias)");
+	CheckOk(test,
+	        KernelMapDirectMemory(&second_alias, page * 2, SceKernelProtCpuRw, SceKernelMapFixed,
+	                              physical + page * 2, page),
+	        "KernelMapDirectMemory(overlapping alias)");
+	TestFailPhysicalMemoryUnmapAfter(1);
+	CheckFailed(test, KernelCheckedReleaseDirectMemory(physical, block),
+	            "KernelCheckedReleaseDirectMemory(rollback)");
+	Check(test,
+	      MapAutomaticMemory(base + block * 4, page, SceKernelMtypeC, SceKernelProtCpuRw) ==
+	          Libs::LibKernel::KERNEL_ERROR_EAGAIN,
+	      "restored mappings remained available to the automatic allocator");
+	CheckOk(test, KernelMunmap(base, block), "KernelMunmap(original)");
+	Check(test,
+	      MapAutomaticMemory(base, block, SceKernelMtypeC, SceKernelProtCpuRw) ==
+	          Libs::LibKernel::KERNEL_ERROR_EAGAIN,
+	      "automatic allocation reused aliased pages or ordinary physical memory");
+	CheckOk(test, MapAutomaticMemory(base, block - page * 3, SceKernelMtypeC, SceKernelProtCpuRw),
+	        "MapAutomaticMemory(fragmented reuse)");
+	Check(test,
+	      Query(test, base).offset == static_cast<uint64_t>(physical) &&
+	          Query(test, base + page).offset == static_cast<uint64_t>(physical + page * 4),
+	      "automatic allocation did not skip the union of overlapping aliases");
+	CheckOk(test, KernelMunmap(reinterpret_cast<uint64_t>(first_alias), page * 3),
+	        "KernelMunmap(first alias)");
+	CheckOk(test, MapAutomaticMemory(base + block * 4, page, SceKernelMtypeC, SceKernelProtCpuRw),
+	        "MapAutomaticMemory(partial alias release)");
+	Check(test, Query(test, base + block * 4).offset == static_cast<uint64_t>(physical + page),
+	      "partial alias release did not reclaim its unreferenced page");
+	CheckOk(test, KernelMunmap(reinterpret_cast<uint64_t>(second_alias), page),
+	        "KernelMunmap(partial second alias)");
+	CheckOk(test, MapAutomaticMemory(base + block * 5, page, SceKernelMtypeC, SceKernelProtCpuRw),
+	        "MapAutomaticMemory(partial unmap)");
+	Check(test, Query(test, base + block * 5).offset == static_cast<uint64_t>(physical + page * 2),
+	      "partial unmap did not reclaim the matching physical page");
+	Check(test, *reinterpret_cast<uint64_t*>(base + block * 3 + page) == 0x0123456789abcdefull,
+	      "automatic reuse changed a surviving alias");
+	CheckOk(test, KernelCheckedReleaseDirectMemory(physical, block),
+	        "KernelCheckedReleaseDirectMemory(donation)");
+	ExpectUnmapped(test, base);
+	ExpectUnmapped(test, base + block * 3 + page);
+	Check(test,
+	      MapAutomaticMemory(base, page, SceKernelMtypeC, SceKernelProtCpuRw) ==
+	          Libs::LibKernel::KERNEL_ERROR_EAGAIN,
+	      "released donation remained in the automatic free list");
+	int64_t reused = -1;
+	CheckOk(test,
+	        KernelAllocateDirectMemory(physical, physical + block, block, block, SceKernelMtypeC,
+	                                   &reused),
+	        "KernelAllocateDirectMemory(released donation)");
+	Check(test, reused == physical, "donation did not return to the ordinary allocator");
+	CheckOk(test, KernelCheckedReleaseDirectMemory(reused, block), "release ordinary reuse");
+	std::printf("[host]    %-48s ok\n", test);
+}
+
+void TestAutomaticMemoryFragmentedMapRollback() {
+	using namespace Libs::LibKernel::Memory;
+	const char*        test   = "AutomaticMemoryFragmentedMapRollback";
+	constexpr uint64_t block  = 0x200000;
+	constexpr uint64_t base   = kExtendedMemoryBase + 0x22000000;
+	const auto         end    = static_cast<int64_t>(KernelGetDirectMemorySize());
+	int64_t            first  = -1;
+	int64_t            gap    = -1;
+	int64_t            second = -1;
+	CheckOk(test, AllocateDirectMemory(0, end, block, block, 0, &first, true), "first donation");
+	CheckOk(test, AllocateDirectMemory(first + block, end, block, block, 0, &gap),
+	        "ordinary allocation between donations");
+	CheckOk(test, AllocateDirectMemory(gap + block, end, block, block, 0, &second, true),
+	        "second donation");
+	Check(test, gap == first + block && second == gap + block,
+	      "test setup did not produce adjacent ownership ranges");
+	auto* address = reinterpret_cast<void*>(base);
+	CheckOk(test, KernelReserveVirtualRange(&address, block, SceKernelMapFixed, block),
+	        "reserve first mapping piece");
+	TestFailNextFixedReserveRangeRegistration();
+	CheckFailed(test, MapAutomaticMemory(base, block * 2, SceKernelMtypeC, SceKernelProtCpuRw),
+	            "MapAutomaticMemory(second-piece failure)");
+	ExpectUnmapped(test, base);
+	CheckOk(test, MapAutomaticMemory(base, block * 2, SceKernelMtypeC, SceKernelProtCpuRw),
+	        "MapAutomaticMemory(after rollback)");
+	Check(test,
+	      Query(test, base).offset == static_cast<uint64_t>(first) &&
+	          Query(test, base + block).offset == static_cast<uint64_t>(second),
+	      "fragmented map did not preserve both donated physical ranges");
+	auto* alias = reinterpret_cast<void*>(base + block * 3);
+	CheckOk(
+	    test,
+	    KernelMapDirectMemory(&alias, block * 2, SceKernelProtCpuRw, SceKernelMapFixed, gap, block),
+	    "alias across ordinary and donated ownership");
+	CheckOk(test, KernelMunmap(base, block * 2), "unmap fragmented allocation");
+	CheckFailed(test, MapAutomaticMemory(base, block * 2, SceKernelMtypeC, SceKernelProtCpuRw),
+	            "automatic allocation while second donation remains aliased");
+	CheckOk(test, KernelMunmap(reinterpret_cast<uint64_t>(alias), block * 2),
+	        "unmap mixed ownership alias");
+	CheckFailed(test, MapAutomaticMemory(base, block * 3, SceKernelMtypeC, SceKernelProtCpuRw),
+	            "automatic allocation beyond donated capacity");
+	CheckOk(test, MapAutomaticMemory(base, block * 2, SceKernelMtypeC, SceKernelProtCpuRw),
+	        "reuse both donations after final alias removal");
+	CheckOk(test, KernelCheckedReleaseDirectMemory(first, block), "release first donation");
+	CheckOk(test, KernelCheckedReleaseDirectMemory(gap, block), "release ordinary allocation");
+	CheckOk(test, KernelCheckedReleaseDirectMemory(second, block), "release second donation");
+	std::printf("[host]    %-48s ok\n", test);
+}
+
 void TestMemoryPoolAlignmentContracts() {
 	const char* test = "MemoryPoolAlignmentContracts";
 	void*       addr = nullptr;
@@ -3155,6 +3322,9 @@ int main(int argc, char** argv) {
 	RunTest(TestFixedReserveRollbackSkipsUntouchedChunks);
 	RunTest(TestFixedReserveRangeAddRollbackKeepsPlaceholder);
 	RunTest(TestLargeHintedReserveHostsSmallDirectMap);
+	RunTest(TestExtendedAndUserMappingsDoNotAlias);
+	RunTest(TestAutomaticMemoryReuseAndAliases);
+	RunTest(TestAutomaticMemoryFragmentedMapRollback);
 	RunTest(TestMemoryPoolAlignmentContracts);
 	RunTest(TestProsperoSampleMemoryPoolExpandCommit);
 	RunTest(TestFragmentedMemoryPoolBacking);