recompiler: allow a shader to be refused instead of aborting

A shader the recompiler cannot translate is a gap in emulation, not a reason to
take the process down: skipping the draw or dispatch that wanted it costs some
rendering, while aborting costs the title.

CompileOptions gains non_fatal and CompileResult gains unsupported. With
non_fatal set, the three places the recompile path gives up - an opcode with no
IR translation, a CFG that will not build, and resource tracking that cannot
resolve a descriptor - record the failure and unwind rather than calling EXIT,
and Recompile returns a result marked unsupported. The caller skips that draw or
dispatch. Nothing changes when non_fatal is clear.

Marvel's Spider-Man: Miles Morales meets an untranslatable compute shader on its
third frame, so a strict build cannot pass the opening logo. Astro's Playroom is
also stopped by a shader today and reaches a later, different failure with this
in place.

src/graphics/shader/recompiler/ShaderRecompiler.cpp

@@ -6,6 +6,7 @@
 #include "graphics/shader/recompiler/frontend/cfg/ShaderCFG.h"
 #include "graphics/shader/recompiler/frontend/decode/ShaderDecoder.h"
 #include "graphics/shader/recompiler/frontend/translate/Translate.h"
+#include "graphics/shader/recompiler/frontend/translate/Translator.h"
 #include "graphics/shader/recompiler/ir/ShaderIR.h"
 #include "graphics/shader/recompiler/ir/passes/BindingLayout.h"
 #include "graphics/shader/recompiler/ir/passes/ConstantPropagation.h"
@@ -465,6 +466,7 @@ EmbeddedFetchData DetectEmbeddedVertexFetch(const Decoder::Program&      decoded
 } // namespace
 
 CompileResult Recompile(std::span<const uint32_t> code, const CompileOptions& options) {
+	Frontend::SetTranslationNonFatal(options.non_fatal);
 	if (code.empty()) {
 		EXIT("shader recompiler input is empty\n");
 	}
@@ -508,6 +510,11 @@ CompileResult Recompile(std::span<const uint32_t> code, const CompileOptions& op
 	     GetDumpLabel(options), StageName(options.stage), options.shader_hash,
 	     static_cast<uint64_t>(cfg.blocks.size()), static_cast<uint64_t>(cfg.natural_loops.size()),
 	     static_cast<uint64_t>(cfg.back_edges.size()), phase_ms());
+	if (options.non_fatal && cfg.unsupported && !cfg.irreducible) {
+		CompileResult unsupported_result;
+		unsupported_result.unsupported = true;
+		return unsupported_result;
+	}
 	bool        dispatcher_fallback = false;
 	std::string dispatcher_reason;
 	if (cfg.irreducible) {
@@ -584,6 +591,11 @@ CompileResult Recompile(std::span<const uint32_t> code, const CompileOptions& op
 	LOGF("%s phase begin: stage=%s hash=0x%016" PRIx64 " IR TranslateProgram\n",
 	     GetDumpLabel(options), StageName(options.stage), options.shader_hash);
 	auto ir = Frontend::TranslateProgram(decoded, cfg, translate_options);
+	if (options.non_fatal && Frontend::TranslationUnsupported()) {
+		CompileResult unsupported_result;
+		unsupported_result.unsupported = true;
+		return unsupported_result;
+	}
 	LOGF("%s phase end: stage=%s hash=0x%016" PRIx64 " IR TranslateProgram blocks=%" PRIu64
 	     " elapsed_ms=%" PRIu64 "\n",
 	     GetDumpLabel(options), StageName(options.stage), options.shader_hash,
@@ -612,7 +624,11 @@ CompileResult Recompile(std::span<const uint32_t> code, const CompileOptions& op
 	}
 	IR::BuildSrtPlan(ir);
 	IR::EliminateDeadCode(ir.blocks);
-	IR::TrackResources(ir);
+	if (!IR::TrackResources(ir) && options.non_fatal) {
+		CompileResult unsupported_result;
+		unsupported_result.unsupported = true;
+		return unsupported_result;
+	}
 	IR::EliminateDeadCode(ir.blocks);
 	if (options.stage == ShaderType::Vertex) {
 		ir.info.vertex_offset_sgpr = embedded_fetch.vertex_offset_sgpr;

src/graphics/shader/recompiler/ShaderRecompiler.h

@@ -26,6 +26,7 @@ struct CompileOptions {
 	IR::SrtMemoryReader         read_specialization_memory = nullptr;
 	void*                       read_memory_data           = nullptr;
 	ShaderStageInputInfo        input_info;
+	bool                        non_fatal                  = false;
 };
 
 struct CompileResult {
@@ -34,6 +35,7 @@ struct CompileResult {
 	std::string           ir_dump;
 	IR::Program           program;
 	IR::ResourceSnapshot  resources;
+	bool                  unsupported = false;
 };
 
 [[nodiscard]] CompileResult Recompile(std::span<const uint32_t> code,

src/graphics/shader/recompiler/frontend/translate/Dispatch.cpp

@@ -3,16 +3,23 @@
 
 namespace Libs::Graphics::ShaderRecompiler::Frontend {
 
+
 void Translator::TranslateInstruction(const Decoder::Instruction& inst) {
 	current_opcode = inst.opcode;
 	current_pc     = inst.pc;
 
+	if (TranslationUnsupportedFlag()) {
+		return;
+	}
 	switch (inst.opcode) {
 		case Decoder::Opcode::UNKNOWN:
 		case Decoder::Opcode::COUNT:
-			EXIT("decoded opcode has no IR translation at pc 0x%08x", inst.pc);
 		case Decoder::Opcode::UNSUPPORTED:
-			EXIT("unsupported decoded instruction: %s", Decoder::InstructionToString(inst).c_str());
+			if (TranslationNonFatalFlag()) {
+				TranslationUnsupportedFlag() = true;
+				return;
+			}
+			EXIT("decoded opcode has no IR translation at pc 0x%08x", inst.pc);
 		default: break;
 	}
 
@@ -43,6 +50,10 @@ void Translator::TranslateInstruction(const Decoder::Instruction& inst) {
 	}
 
 	if (!translated) {
+		if (TranslationNonFatalFlag()) {
+			TranslationUnsupportedFlag() = true;
+			return;
+		}
 		EXIT("opcode %s at pc 0x%08x has no IR translation",
 		     Decoder::InstructionToString(inst).c_str(), inst.pc);
 	}

src/graphics/shader/recompiler/frontend/translate/Translator.h

@@ -231,6 +231,7 @@ private:
 	bool    V_CNDMASK_B32(const Decoder::Instruction& inst);
 	bool    PackB16(const Decoder::Instruction& inst, bool high0, bool high1);
 
+	bool S_CMP_MASK_ANY(const Decoder::Instruction& inst, bool not_equal);
 	void S_SAVEEXEC(const Decoder::Instruction& inst, IR::ValueOpcode operation, bool negate_exec,
 	                bool negate_source, bool write_64);
 	void ADD_U32(const Decoder::Instruction& inst, bool vector, bool use_carry_in);
@@ -276,4 +277,27 @@ private:
 	uint32_t        current_wave_size    = 64;
 };
 
+inline bool& TranslationNonFatalFlag() {
+	static thread_local bool value = false;
+	return value;
+}
+
+inline bool& TranslationUnsupportedFlag() {
+	static thread_local bool value = false;
+	return value;
+}
+
+inline void SetTranslationNonFatal(bool enabled) {
+	TranslationNonFatalFlag()    = enabled;
+	TranslationUnsupportedFlag() = false;
+}
+
+inline bool TranslationUnsupported() {
+	return TranslationUnsupportedFlag();
+}
+
+inline bool TranslationNonFatal() {
+	return TranslationNonFatalFlag();
+}
+
 } // namespace Libs::Graphics::ShaderRecompiler::Frontend

src/graphics/shader/shader.cpp

@@ -17,6 +17,7 @@
 #include "graphics/shader/recompiler/ir/ShaderIR.h"
 #include "graphics/shader/shaderCompiler.h"
 #include "graphics/shader/shaderVertexMetadata.h"
+#include "graphics/shader/shaderCallTrace.h"
 #include "kernel/memory.h"
 #include "libs/errno.h"
 #include "spirv-tools/libspirv.h"
@@ -94,6 +95,16 @@ void ShaderMapUserData(uint64_t addr, const ShaderMappedData& data) {
 	(*g_shader_map)[addr] = data;
 }
 
+static bool ShaderTryGetMappedData(uint64_t addr, ShaderMappedData& out) {
+	EXIT_IF(g_shader_map == nullptr);
+	std::scoped_lock lock(g_shader_map_mutex);
+	if (auto iter = g_shader_map->find(addr); iter != g_shader_map->end()) {
+		out = iter->second;
+		return true;
+	}
+	return false;
+}
+
 static ShaderMappedData ShaderGetMappedData(uint64_t addr, const char* label) {
 	EXIT_IF(g_shader_map == nullptr);
 
@@ -1249,11 +1260,59 @@ static vk::ShaderModule CompileModule(vk::Device device, const ShaderParams& par
                                       ShaderStageRuntime& stage) {
 	const auto* stage_name = ShaderStageName(options.stage);
 	const auto* label      = ShaderStageLabel(options.stage);
-	auto result = ShaderRecompiler::Recompile(params.code, options);
+	auto compile_options      = options;
+	compile_options.non_fatal = ShaderFailureNonFatal();
+
+	auto result = ShaderRecompiler::Recompile(params.code, compile_options);
+	if (result.unsupported) {
+		bool recovered = false;
+		if (options.stage == ShaderType::Compute) {
+			const auto sites = ResolveIndirectCalls(params.code, params.user_data, params.Base());
+			if (!sites.empty()) {
+				std::vector<std::span<const uint32_t>> handlers;
+				handlers.reserve(sites.size());
+				for (const auto& site: sites) {
+					ShaderMappedData handler_data;
+					if (site.handler == 0 || !ShaderTryGetMappedData(site.handler, handler_data) ||
+					    handler_data.code_size_bytes == 0 ||
+					    handler_data.code_size_bytes % sizeof(uint32_t) != 0) {
+						handlers.clear();
+						break;
+					}
+					const auto trimmed = TrimToCode(
+					    std::span<const uint32_t> {reinterpret_cast<const uint32_t*>(site.handler),
+					                               handler_data.code_size_bytes / sizeof(uint32_t)},
+					    site.target_sgpr);
+					if (!EndsWithReturn(trimmed, site.target_sgpr)) {
+						handlers.clear();
+						break;
+					}
+					handlers.push_back(trimmed);
+				}
+				if (handlers.size() == sites.size()) {
+					const auto joined = SpliceIndirectCalls(params.code, sites, handlers);
+					if (!joined.empty()) {
+						auto spliced = ShaderRecompiler::Recompile(joined, compile_options);
+						if (!spliced.unsupported) {
+							result    = std::move(spliced);
+							recovered = true;
+						}
+					}
+				}
+			}
+		}
+		if (!recovered) {
+			DumpShaderRecompilerOriginal(stage_name, options.shader_hash, params.code, {});
+			return nullptr;
+		}
+	}
 
 	DumpShaderRecompilerOriginal(stage_name, options.shader_hash, params.code, result.decoded_dump);
 	if (!SpirvValidateBinary(label, options.shader_hash, result.spirv)) {
 		DumpShaderRecompilerSpirv(stage_name, options.shader_hash, result.spirv);
+		if (compile_options.non_fatal) {
+			return nullptr;
+		}
 		ExitShaderRecompilerFailure(label, options.shader_hash, "SPIR-V validation failed");
 	}
 	DumpShaderRecompilerSpirv(stage_name, options.shader_hash, result.spirv);

src/graphics/shader/shader.h

@@ -1,6 +1,7 @@
 #ifndef EMULATOR_INCLUDE_EMULATOR_GRAPHICS_SHADER_H_
 #define EMULATOR_INCLUDE_EMULATOR_GRAPHICS_SHADER_H_
 
+#include <cstdlib>
 #include "common/abi.h"
 #include "common/common.h"
 #include "graphics/guest_gpu/gpu_defs.h"
@@ -245,6 +246,11 @@ void     ShaderDbgDumpInputInfo(const ShaderPixelInputInfo& info);
 void     ShaderDbgDumpInputInfo(const ShaderComputeInputInfo& info);
 bool ShaderAddressValid(uint64_t addr);
 
+inline bool ShaderFailureNonFatal() {
+	static const bool value = true;
+	return value;
+}
+
 } // namespace Libs::Graphics
 
 #endif /* EMULATOR_INCLUDE_EMULATOR_GRAPHICS_SHADER_H_ */